dtgitea
/
dataops
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

add workflow partner1site,dev

This commit is contained in:
root 2025-10-16 17:38:37 +08:00
parent 3610eac55c
commit 2cda50e344
10 changed files with 679 additions and 245 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
dev/workflow/TK_Cust/partner1site/Partner客户拜访记录/t01_partner_visit.sql
View File

@ -1,7 +1,8 @@
DROP TABLE IF EXISTS p20_pdm.t01_partner_visit; DROP TABLE IF EXISTS p20_pdm.t01_partner_visit;
CREATE TABLE IF NOT EXISTS p20_pdm.t01_partner_visit ( CREATE TABLE IF NOT EXISTS p20_pdm.t01_partner_visit (
insert_date date id varchar(100)
, insert_date date
, visitor varchar(100) , visitor varchar(100)
, dealer_name varchar(100) , dealer_name varchar(100)
, customer_name varchar(100) , customer_name varchar(100)
@ -35,9 +36,10 @@ CREATE TABLE IF NOT EXISTS p20_pdm.t01_partner_visit (
, Etl_Tx_Dt timestamp(0) , Etl_Tx_Dt timestamp(0)
, Src_Sysname varchar(50) , Src_Sysname varchar(50)
, Src_Table varchar(50) , Src_Table varchar(50)
,primary key( id )
); );
COMMENT ON COLUMN p20_pdm.t01_partner_visit.id IS '主键ID';
COMMENT ON COLUMN p20_pdm.t01_partner_visit.insert_date IS '录入时间'; COMMENT ON COLUMN p20_pdm.t01_partner_visit.insert_date IS '录入时间';
COMMENT ON COLUMN p20_pdm.t01_partner_visit.visitor IS '拜访人'; COMMENT ON COLUMN p20_pdm.t01_partner_visit.visitor IS '拜访人';
COMMENT ON COLUMN p20_pdm.t01_partner_visit.dealer_name IS '代理商名称'; COMMENT ON COLUMN p20_pdm.t01_partner_visit.dealer_name IS '代理商名称';

349
dev/workflow/TK_Cust/partner1site/Partner客户拜访记录/t01_partner_visit_agi.sql
View File

@ -4,11 +4,11 @@
/*Brilliance stems from wisdoms. */ /*Brilliance stems from wisdoms. */
/*************Head Section**************************************************************************/ /*************Head Section**************************************************************************/
/*Script Use: Periodically load data to :t01_partner_visit(Partner客户拜访记录) */ /*Script Use: Periodically load data to :t01_partner_visit(Partner客户拜访记录) */
/*Create Date:2025-09-30 21:24:49 */ /*Create Date:2025-10-16 17:37:32 */
/*SDM Developed By: dev */ /*SDM Developed By: dev */
/*SDM Developed Date: 2025-09-29 */ /*SDM Developed Date: 2025-09-29 */
/*SDM Checked By: dev */ /*SDM Checked By: dev */
/*SDM Checked Date: 2025-09-30 */ /*SDM Checked Date: 2025-10-16 */
/*Script Developed By: dev */ /*Script Developed By: dev */
/*Script Checked By: dev */ /*Script Checked By: dev */
/*Source table 1: p10_sa.s98_s_partner_summary_visit */ /*Source table 1: p10_sa.s98_s_partner_summary_visit */
@ -16,7 +16,7 @@
/*Target Table:t01_partner_visit */ /*Target Table:t01_partner_visit */
/*ETL Job Name:t01_partner_visit */ /*ETL Job Name:t01_partner_visit */
/*ETL Frequency:None */ /*ETL Frequency:None */
/*ETL Policy:F1 */ /*ETL Policy:F2 */
/********************************************************************************************/ /********************************************************************************************/
/*******Main Section**************************************************************************/ /*******Main Section**************************************************************************/
\set ON_ERROR_STOP on \set ON_ERROR_STOP on
@ -31,11 +31,19 @@ CREATE TEMPORARY TABLE t01_partner_visit_agi_CUR_I
ON COMMIT PRESERVE ROWS; ON COMMIT PRESERVE ROWS;
/*创建临时表加载不同数据 */
CREATE TEMPORARY TABLE t01_partner_visit_agi_INS
( LIKE :PDMDB.t01_partner_visit)
ON COMMIT PRESERVE ROWS;
/*****************************************************************************************************/ /*****************************************************************************************************/
/* GROUP 1:Source Table:s98_s_partner_summary_visit***************************************************/ /* GROUP 1:Source Table:s98_s_partner_summary_visit***************************************************/
/*****************************************************************************************************/ /*****************************************************************************************************/
INSERT INTO t01_partner_visit_agi_CUR_I ( INSERT INTO t01_partner_visit_agi_CUR_I (
insert_date /*录入时间*/ id /*主键ID*/
,insert_date /*录入时间*/
,visitor /*拜访人*/ ,visitor /*拜访人*/
,dealer_name /*代理商名称*/ ,dealer_name /*代理商名称*/
,customer_name /*客户名称*/ ,customer_name /*客户名称*/
@ -71,7 +79,8 @@ INSERT INTO t01_partner_visit_agi_CUR_I (
,Src_Table /*来源表*/ ,Src_Table /*来源表*/
) )
SELECT SELECT
p0.insert_date::date /*insert_date*/ COALESCE(TRIM(CAST(p0.id AS varchar(100))),'') /*id*/
,p0.insert_date::date /*insert_date*/
,COALESCE(TRIM(CAST(p0.visitor AS varchar(100))),'') /*visitor*/ ,COALESCE(TRIM(CAST(p0.visitor AS varchar(100))),'') /*visitor*/
,COALESCE(TRIM(CAST(p0.dealer_name AS varchar(100))),'') /*dealer_name*/ ,COALESCE(TRIM(CAST(p0.dealer_name AS varchar(100))),'') /*dealer_name*/
,COALESCE(TRIM(CAST(p0.customer_name AS varchar(100))),'') /*customer_name*/ ,COALESCE(TRIM(CAST(p0.customer_name AS varchar(100))),'') /*customer_name*/
@ -111,87 +120,259 @@ FROM p10_sa.s98_s_partner_summary_visit p0
/*从目标表中删除所有数据 t01_partner_visit(Partner客户拜访记录) */ /*将不同数据插入到临时表 */
DELETE FROM :PDMDB.t01_partner_visit ;INSERT INTO t01_partner_visit_agi_INS (
WHERE ETL_JOB=:ETLJOB; insert_date /*录入时间*/
,visitor /*拜访人*/
,dealer_name /*代理商名称*/
,customer_name /*客户名称*/
,contact_name /*联系人*/
,contact_department /*联系人部门*/
,customer_category_type /*产品分类*/
,intention_product /*意向产品*/
,into_report /*是否进入报备*/
,contact_ways /*联络方式*/
,attachment_number /*附件数量*/
,visit_remark /*拜访备注*/
,comment_number /*评论数*/
,like_number /*点赞数*/
,area /*大区*/
,contact_province /*省份*/
,contact_city /*城市*/
,contact_district /*区域*/
,contact_phone /*联系电话*/
,contact_email /*联系人邮件*/
,contact_address /*地址*/
,like_name /*点赞人*/
,alias /*别名*/
,industry /*行业*/
,sub_industry /*子行业*/
,tsm_names_by_alias /*tsm名*/
,customer_category /*客户类别*/
,id /*主键ID*/
,Etl_Batch_No /*作业批次号*/
,Etl_First_Dt /*最初入库时间*/
,Etl_Job /*作业名称*/
,Etl_Proc_Dt /*本次入库时间*/
,Etl_Tx_Dt /*作业运行时间*/
,Src_Sysname /*来源系统*/
,Src_Table /*来源表*/
)
SELECT
P1.insert_date /*录入时间*/
,P1.visitor /*拜访人*/
,P1.dealer_name /*代理商名称*/
,P1.customer_name /*客户名称*/
,P1.contact_name /*联系人*/
,P1.contact_department /*联系人部门*/
,P1.customer_category_type /*产品分类*/
,P1.intention_product /*意向产品*/
,P1.into_report /*是否进入报备*/
,P1.contact_ways /*联络方式*/
,P1.attachment_number /*附件数量*/
,P1.visit_remark /*拜访备注*/
,P1.comment_number /*评论数*/
,P1.like_number /*点赞数*/
,P1.area /*大区*/
,P1.contact_province /*省份*/
,P1.contact_city /*城市*/
,P1.contact_district /*区域*/
,P1.contact_phone /*联系电话*/
,P1.contact_email /*联系人邮件*/
,P1.contact_address /*地址*/
,P1.like_name /*点赞人*/
,P1.alias /*别名*/
,P1.industry /*行业*/
,P1.sub_industry /*子行业*/
,P1.tsm_names_by_alias /*tsm名*/
,P1.customer_category /*客户类别*/
,P1.id /*主键ID*/
,P1.Etl_Batch_No /*作业批次号*/
,P1.Etl_First_Dt /*最初入库时间*/
,P1.Etl_Job /*作业名称*/
,P1.Etl_Proc_Dt /*本次入库时间*/
,P1.Etl_Tx_Dt /*作业运行时间*/
,P1.Src_Sysname /*来源系统*/
,P1.Src_Table /*来源表*/
FROM t01_partner_visit_agi_CUR_I P1
LEFT JOIN :PDMDB.t01_partner_visit P2
ON P1.insert_date = P2.insert_date
AND P1.visitor = P2.visitor
AND P1.dealer_name = P2.dealer_name
AND P1.customer_name = P2.customer_name
AND P1.contact_name = P2.contact_name
AND P1.contact_department = P2.contact_department
AND P1.customer_category_type = P2.customer_category_type
AND P1.intention_product = P2.intention_product
AND P1.into_report = P2.into_report
AND P1.contact_ways = P2.contact_ways
AND P1.attachment_number = P2.attachment_number
AND P1.visit_remark = P2.visit_remark
AND P1.comment_number = P2.comment_number
AND P1.like_number = P2.like_number
AND P1.area = P2.area
AND P1.contact_province = P2.contact_province
AND P1.contact_city = P2.contact_city
AND P1.contact_district = P2.contact_district
AND P1.contact_phone = P2.contact_phone
AND P1.contact_email = P2.contact_email
AND P1.contact_address = P2.contact_address
AND P1.like_name = P2.like_name
AND P1.alias = P2.alias
AND P1.industry = P2.industry
AND P1.sub_industry = P2.sub_industry
AND P1.tsm_names_by_alias = P2.tsm_names_by_alias
AND P1.customer_category = P2.customer_category
AND P1.id = P2.id
WHERE P2.insert_date IS NULL
OR P2.visitor IS NULL
OR P2.dealer_name IS NULL
OR P2.customer_name IS NULL
OR P2.contact_name IS NULL
OR P2.contact_department IS NULL
OR P2.customer_category_type IS NULL
OR P2.intention_product IS NULL
OR P2.into_report IS NULL
OR P2.contact_ways IS NULL
OR P2.attachment_number IS NULL
OR P2.visit_remark IS NULL
OR P2.comment_number IS NULL
OR P2.like_number IS NULL
OR P2.area IS NULL
OR P2.contact_province IS NULL
OR P2.contact_city IS NULL
OR P2.contact_district IS NULL
OR P2.contact_phone IS NULL
OR P2.contact_email IS NULL
OR P2.contact_address IS NULL
OR P2.like_name IS NULL
OR P2.alias IS NULL
OR P2.industry IS NULL
OR P2.sub_industry IS NULL
OR P2.tsm_names_by_alias IS NULL
OR P2.customer_category IS NULL
OR P2.id IS NULL
;
/*将新增数据插入到目标表 */ /*将新增数据插入到目标表 */
;INSERT INTO :PDMDB.t01_partner_visit ( ;INSERT INTO :PDMDB.t01_partner_visit (
insert_date /*录入时间*/ insert_date /*录入时间*/
,visitor /*拜访人*/ ,visitor /*拜访人*/
,dealer_name /*代理商名称*/ ,dealer_name /*代理商名称*/
,customer_name /*客户名称*/ ,customer_name /*客户名称*/
,contact_name /*联系人*/ ,contact_name /*联系人*/
,contact_department /*联系人部门*/ ,contact_department /*联系人部门*/
,customer_category_type /*产品分类*/ ,customer_category_type /*产品分类*/
,intention_product /*意向产品*/ ,intention_product /*意向产品*/
,into_report /*是否进入报备*/ ,into_report /*是否进入报备*/
,contact_ways /*联络方式*/ ,contact_ways /*联络方式*/
,attachment_number /*附件数量*/ ,attachment_number /*附件数量*/
,visit_remark /*拜访备注*/ ,visit_remark /*拜访备注*/
,comment_number /*评论数*/ ,comment_number /*评论数*/
,like_number /*点赞数*/ ,like_number /*点赞数*/
,area /*大区*/ ,area /*大区*/
,contact_province /*省份*/ ,contact_province /*省份*/
,contact_city /*城市*/ ,contact_city /*城市*/
,contact_district /*区域*/ ,contact_district /*区域*/
,contact_phone /*联系电话*/ ,contact_phone /*联系电话*/
,contact_email /*联系人邮件*/ ,contact_email /*联系人邮件*/
,contact_address /*地址*/ ,contact_address /*地址*/
,like_name /*点赞人*/ ,like_name /*点赞人*/
,alias /*别名*/ ,alias /*别名*/
,industry /*行业*/ ,industry /*行业*/
,sub_industry /*子行业*/ ,sub_industry /*子行业*/
,tsm_names_by_alias /*tsm名*/ ,tsm_names_by_alias /*tsm名*/
,customer_category /*客户类别*/ ,customer_category /*客户类别*/
,Etl_Batch_No /*作业批次号*/ ,id /*主键ID*/
,Etl_First_Dt /*最初入库时间*/ ,Etl_Batch_No /*作业批次号*/
,Etl_Job /*作业名称*/ ,Etl_First_Dt /*最初入库时间*/
,Etl_Proc_Dt /*本次入库时间*/ ,Etl_Job /*作业名称*/
,Etl_Tx_Dt /*作业运行时间*/ ,Etl_Proc_Dt /*本次入库时间*/
,Src_Sysname /*来源系统*/ ,Etl_Tx_Dt /*作业运行时间*/
,Src_Table /*来源表*/ ,Src_Sysname /*来源系统*/
,Src_Table /*来源表*/
)
SELECT )
P1.insert_date /*录入时间*/ SELECT
,P1.visitor /*拜访人*/ P1.insert_date /*录入时间*/
,P1.dealer_name /*代理商名称*/ ,P1.visitor /*拜访人*/
,P1.customer_name /*客户名称*/ ,P1.dealer_name /*代理商名称*/
,P1.contact_name /*联系人*/ ,P1.customer_name /*客户名称*/
,P1.contact_department /*联系人部门*/ ,P1.contact_name /*联系人*/
,P1.customer_category_type /*产品分类*/ ,P1.contact_department /*联系人部门*/
,P1.intention_product /*意向产品*/ ,P1.customer_category_type /*产品分类*/
,P1.into_report /*是否进入报备*/ ,P1.intention_product /*意向产品*/
,P1.contact_ways /*联络方式*/ ,P1.into_report /*是否进入报备*/
,P1.attachment_number /*附件数量*/ ,P1.contact_ways /*联络方式*/
,P1.visit_remark /*拜访备注*/ ,P1.attachment_number /*附件数量*/
,P1.comment_number /*评论数*/ ,P1.visit_remark /*拜访备注*/
,P1.like_number /*点赞数*/ ,P1.comment_number /*评论数*/
,P1.area /*大区*/ ,P1.like_number /*点赞数*/
,P1.contact_province /*省份*/ ,P1.area /*大区*/
,P1.contact_city /*城市*/ ,P1.contact_province /*省份*/
,P1.contact_district /*区域*/ ,P1.contact_city /*城市*/
,P1.contact_phone /*联系电话*/ ,P1.contact_district /*区域*/
,P1.contact_email /*联系人邮件*/ ,P1.contact_phone /*联系电话*/
,P1.contact_address /*地址*/ ,P1.contact_email /*联系人邮件*/
,P1.like_name /*点赞人*/ ,P1.contact_address /*地址*/
,P1.alias /*别名*/ ,P1.like_name /*点赞人*/
,P1.industry /*行业*/ ,P1.alias /*别名*/
,P1.sub_industry /*子行业*/ ,P1.industry /*行业*/
,P1.tsm_names_by_alias /*tsm名*/ ,P1.sub_industry /*子行业*/
,P1.customer_category /*客户类别*/ ,P1.tsm_names_by_alias /*tsm名*/
,P1.Etl_Batch_No /*作业批次号*/ ,P1.customer_category /*客户类别*/
,P1.Etl_First_Dt /*最初入库时间*/ ,P1.id /*主键ID*/
,P1.Etl_Job /*作业名称*/ ,P1.Etl_Batch_No /*作业批次号*/
,P1.Etl_Proc_Dt /*本次入库时间*/ ,P1.Etl_First_Dt /*最初入库时间*/
,P1.Etl_Tx_Dt /*作业运行时间*/ ,P1.Etl_Job /*作业名称*/
,P1.Src_Sysname /*来源系统*/ ,P1.Etl_Proc_Dt /*本次入库时间*/
,P1.Src_Table /*来源表*/ ,P1.Etl_Tx_Dt /*作业运行时间*/
,P1.Src_Sysname /*来源系统*/
FROM t01_partner_visit_agi_CUR_I P1 ,P1.Src_Table /*来源表*/
FROM t01_partner_visit_agi_INS P1
ON CONFLICT ( id)
DO UPDATE SET
id=excluded.id
,insert_date=excluded.insert_date
,visitor=excluded.visitor
,dealer_name=excluded.dealer_name
,customer_name=excluded.customer_name
,contact_name=excluded.contact_name
,contact_department=excluded.contact_department
,customer_category_type=excluded.customer_category_type
,intention_product=excluded.intention_product
,into_report=excluded.into_report
,contact_ways=excluded.contact_ways
,attachment_number=excluded.attachment_number
,visit_remark=excluded.visit_remark
,comment_number=excluded.comment_number
,like_number=excluded.like_number
,area=excluded.area
,contact_province=excluded.contact_province
,contact_city=excluded.contact_city
,contact_district=excluded.contact_district
,contact_phone=excluded.contact_phone
,contact_email=excluded.contact_email
,contact_address=excluded.contact_address
,like_name=excluded.like_name
,alias=excluded.alias
,industry=excluded.industry
,sub_industry=excluded.sub_industry
,tsm_names_by_alias=excluded.tsm_names_by_alias
,customer_category=excluded.customer_category
,Etl_Batch_No=excluded.Etl_Batch_No
,Etl_First_Dt=excluded.Etl_First_Dt
,Etl_Job=excluded.Etl_Job
,Etl_Proc_Dt=excluded.Etl_Proc_Dt
,Etl_Tx_Dt=excluded.Etl_Tx_Dt
,Src_Sysname=excluded.Src_Sysname
,Src_Table=excluded.Src_Table
; ;
/*****程序结束退出 */ /*****程序结束退出 */
\q \q

38
dev/workflow/TK_Cust/partner1site/partner one site联系人信息/d_partner_contact.sql Normal file
View File

@ -0,0 +1,38 @@
DROP TABLE IF EXISTS p30_common.d_partner_contact;
CREATE TABLE IF NOT EXISTS p30_common.d_partner_contact (
contact_id bigint
, contact_name VARCHAR(200)
, province VARCHAR(50)
, city VARCHAR(50)
, address text
, department VARCHAR(50)
, email VARCHAR(50)
, phone VARCHAR(50)
, Etl_Batch_No varchar(50)
, Etl_First_Dt timestamp(0)
, Etl_Job varchar(200)
, Etl_Proc_Dt timestamp(0)
, Etl_Tx_Dt timestamp(0)
, Src_Sysname varchar(50)
, Src_Table varchar(50)
,primary key( contact_id )
);
COMMENT ON COLUMN p30_common.d_partner_contact.contact_id IS '联系人ID';
COMMENT ON COLUMN p30_common.d_partner_contact.contact_name IS '联系人名称';
COMMENT ON COLUMN p30_common.d_partner_contact.province IS '省份';
COMMENT ON COLUMN p30_common.d_partner_contact.city IS '城市';
COMMENT ON COLUMN p30_common.d_partner_contact.address IS '地址';
COMMENT ON COLUMN p30_common.d_partner_contact.department IS '部门';
COMMENT ON COLUMN p30_common.d_partner_contact.email IS '邮箱';
COMMENT ON COLUMN p30_common.d_partner_contact.phone IS '电话';
COMMENT ON COLUMN p30_common.d_partner_contact.Etl_Batch_No IS '作业批次号';
COMMENT ON COLUMN p30_common.d_partner_contact.Etl_First_Dt IS '最初入库时间';
COMMENT ON COLUMN p30_common.d_partner_contact.Etl_Job IS '作业名称';
COMMENT ON COLUMN p30_common.d_partner_contact.Etl_Proc_Dt IS '本次入库时间';
COMMENT ON COLUMN p30_common.d_partner_contact.Etl_Tx_Dt IS '作业运行时间';
COMMENT ON COLUMN p30_common.d_partner_contact.Src_Sysname IS '来源系统';
COMMENT ON COLUMN p30_common.d_partner_contact.Src_Table IS '来源表';
COMMENT ON TABLE p30_common.d_partner_contact IS 'partner one site联系人信息';

199
dev/workflow/TK_Cust/partner1site/partner one site联系人信息/d_partner_contact_agi.sql Normal file
View File

@ -0,0 +1,199 @@
/***************************************************************************************************/
/*script in Sql, generate by SdmCreateScript 2024(by IdgValue, qihang.feng@idgvalue.COM) */
/*VERSION 01.10 revised on 2020-08-25 */
/*Brilliance stems from wisdoms. */
/*************Head Section**************************************************************************/
/*Script Use: Periodically load data to :d_partner_contact(partner one site联系人信息) */
/*Create Date:2025-10-14 18:24:13 */
/*SDM Developed By: dev */
/*SDM Developed Date: 2025-10-14 */
/*SDM Checked By: dev */
/*SDM Checked Date: 2025-10-14 */
/*Script Developed By: dev */
/*Script Checked By: dev */
/*Source table 1: :PDMDB.t01_partner_customer_and_contact */
/*Job Type: Inbound transform (Tier 1 to Tier 2) */
/*Target Table:d_partner_contact */
/*ETL Job Name:d_partner_contact */
/*ETL Frequency:None */
/*ETL Policy:F2 */
/********************************************************************************************/
/*******Main Section**************************************************************************/
\set ON_ERROR_STOP on
\set AUTOCOMMIT on
\timing on
/*创建临时表加载当前数据 */
CREATE TEMPORARY TABLE d_partner_contact_agi_CUR_I
( LIKE :COMMDB.d_partner_contact)
ON COMMIT PRESERVE ROWS;
/*创建临时表加载不同数据 */
CREATE TEMPORARY TABLE d_partner_contact_agi_INS
( LIKE :COMMDB.d_partner_contact)
ON COMMIT PRESERVE ROWS;
/*****************************************************************************************************/
/* GROUP 1:Source Table:t01_partner_customer_and_contact**********************************************/
/*****************************************************************************************************/
INSERT INTO d_partner_contact_agi_CUR_I (
contact_id /*联系人ID*/
,contact_name /*联系人名称*/
,province /*省份*/
,city /*城市*/
,address /*地址*/
,department /*部门*/
,email /*邮箱*/
,phone /*电话*/
,Etl_Batch_No /*作业批次号*/
,Etl_First_Dt /*最初入库时间*/
,Etl_Job /*作业名称*/
,Etl_Proc_Dt /*本次入库时间*/
,Etl_Tx_Dt /*作业运行时间*/
,Src_Sysname /*来源系统*/
,Src_Table /*来源表*/
)
SELECT
COALESCE(p0.customer_contact_id,0) /*contact_id*/
,COALESCE(TRIM(p0.contact_name),'') /*contact_name*/
,COALESCE(TRIM(p0.province),'') /*province*/
,COALESCE(TRIM(p0.city),'') /*city*/
,COALESCE(TRIM(p0.address),'') /*address*/
,COALESCE(TRIM(p0.department),'') /*department*/
,COALESCE(TRIM(p0.email),'') /*email*/
,replace(replace(p0.phone_number,'+86',''),' ','') /*phone*/
,0 /*Etl_Batch_No*/
,TO_DATE(:TXDATE,'YYYYMMDD') /*Etl_First_Dt*/
,:ETLJOB /*Etl_Job*/
,current_timestamp(0) /*Etl_Proc_Dt*/
,TO_DATE(:TXDATE,'YYYYMMDD') /*Etl_Tx_Dt*/
,Substr('t01_partner_customer_and_contact',1,3) /*Src_Sysname*/
,'t01_partner_customer_and_contact' /*Src_Table*/
FROM :PDMDB.t01_partner_customer_and_contact p0
;
/*将不同数据插入到临时表 */
;INSERT INTO d_partner_contact_agi_INS (
contact_name /*联系人名称*/
,province /*省份*/
,city /*城市*/
,address /*地址*/
,department /*部门*/
,email /*邮箱*/
,phone /*电话*/
,contact_id /*联系人ID*/
,Etl_Batch_No /*作业批次号*/
,Etl_First_Dt /*最初入库时间*/
,Etl_Job /*作业名称*/
,Etl_Proc_Dt /*本次入库时间*/
,Etl_Tx_Dt /*作业运行时间*/
,Src_Sysname /*来源系统*/
,Src_Table /*来源表*/
)
SELECT
P1.contact_name /*联系人名称*/
,P1.province /*省份*/
,P1.city /*城市*/
,P1.address /*地址*/
,P1.department /*部门*/
,P1.email /*邮箱*/
,P1.phone /*电话*/
,P1.contact_id /*联系人ID*/
,P1.Etl_Batch_No /*作业批次号*/
,P1.Etl_First_Dt /*最初入库时间*/
,P1.Etl_Job /*作业名称*/
,P1.Etl_Proc_Dt /*本次入库时间*/
,P1.Etl_Tx_Dt /*作业运行时间*/
,P1.Src_Sysname /*来源系统*/
,P1.Src_Table /*来源表*/
FROM d_partner_contact_agi_CUR_I P1
LEFT JOIN :COMMDB.d_partner_contact P2
ON P1.contact_name = P2.contact_name
AND P1.province = P2.province
AND P1.city = P2.city
AND P1.address = P2.address
AND P1.department = P2.department
AND P1.email = P2.email
AND P1.phone = P2.phone
AND P1.contact_id = P2.contact_id
WHERE P2.contact_name IS NULL
OR P2.province IS NULL
OR P2.city IS NULL
OR P2.address IS NULL
OR P2.department IS NULL
OR P2.email IS NULL
OR P2.phone IS NULL
OR P2.contact_id IS NULL
;
/*将新增数据插入到目标表 */
;INSERT INTO :COMMDB.d_partner_contact (
contact_name /*联系人名称*/
,province /*省份*/
,city /*城市*/
,address /*地址*/
,department /*部门*/
,email /*邮箱*/
,phone /*电话*/
,contact_id /*联系人ID*/
,Etl_Batch_No /*作业批次号*/
,Etl_First_Dt /*最初入库时间*/
,Etl_Job /*作业名称*/
,Etl_Proc_Dt /*本次入库时间*/
,Etl_Tx_Dt /*作业运行时间*/
,Src_Sysname /*来源系统*/
,Src_Table /*来源表*/
)
SELECT
P1.contact_name /*联系人名称*/
,P1.province /*省份*/
,P1.city /*城市*/
,P1.address /*地址*/
,P1.department /*部门*/
,P1.email /*邮箱*/
,P1.phone /*电话*/
,P1.contact_id /*联系人ID*/
,P1.Etl_Batch_No /*作业批次号*/
,P1.Etl_First_Dt /*最初入库时间*/
,P1.Etl_Job /*作业名称*/
,P1.Etl_Proc_Dt /*本次入库时间*/
,P1.Etl_Tx_Dt /*作业运行时间*/
,P1.Src_Sysname /*来源系统*/
,P1.Src_Table /*来源表*/
FROM d_partner_contact_agi_INS P1
ON CONFLICT ( contact_id)
DO UPDATE SET
contact_id=excluded.contact_id
,contact_name=excluded.contact_name
,province=excluded.province
,city=excluded.city
,address=excluded.address
,department=excluded.department
,email=excluded.email
,phone=excluded.phone
,Etl_Batch_No=excluded.Etl_Batch_No
,Etl_First_Dt=excluded.Etl_First_Dt
,Etl_Job=excluded.Etl_Job
,Etl_Proc_Dt=excluded.Etl_Proc_Dt
,Etl_Tx_Dt=excluded.Etl_Tx_Dt
,Src_Sysname=excluded.Src_Sysname
,Src_Table=excluded.Src_Table
;
/*****程序结束退出 */
\q

11
dev/workflow/TK_Cust/partner1site/partner1site/wf_dag_partner1site.py
View File

@ -182,6 +182,14 @@ params={'my_param':"t01_partner_visit_agi"},
depends_on_past=False, depends_on_past=False,
retries=3, retries=3,
dag=dag) dag=dag)
d_partner_contact = SSHOperator(
ssh_hook=sshHook,
task_id='d_partner_contact',
command='/data/airflow/etl/COM/run_psql.sh {{ ds_nodash }} {{params.my_param}} >>/data/airflow/logs/run_tpt_{{ds_nodash}}.log 2>&1 ',
params={'my_param':"d_partner_contact_agi"},
depends_on_past=False,
retries=3,
dag=dag)
part_summary_visit_load >> partner_summary_visit_9060 part_summary_visit_load >> partner_summary_visit_9060
part_summary_report_load >> partner_summary_report_6257 part_summary_report_load >> partner_summary_report_6257
part_summary_pos_load >> partner_summary_pos_4937 part_summary_pos_load >> partner_summary_pos_4937
@ -190,4 +198,5 @@ partner_summary_pos_4937 >> t01_partner_pos
partner_summary_report_6257 >> t01_partner_report partner_summary_report_6257 >> t01_partner_report
partner_summary_visit_9060 >> t01_partner_visit partner_summary_visit_9060 >> t01_partner_visit
partner_summary_customer_5702 >> t01_partner_customer_and_contact partner_summary_customer_5702 >> t01_partner_customer_and_contact
t01_partner_customer_and_contact >> task_failed t01_partner_customer_and_contact >> d_partner_contact
d_partner_contact >> task_failed

8
dev/workflow/TK_Cust/partner1site/partner_summary_visit/S98_S_partner_summary_visit.sql
View File

@ -22,6 +22,7 @@ insert into p10_sa.S98_S_partner_summary_visit
, customer_category_type , customer_category_type
, customer_name , customer_name
, dealer_name , dealer_name
, id
, industry , industry
, insert_date , insert_date
, intention_product , intention_product
@ -32,7 +33,6 @@ insert into p10_sa.S98_S_partner_summary_visit
, tsm_names_by_alias , tsm_names_by_alias
, visit_remark , visit_remark
, visitor , visitor
, id
, etl_tx_dt ) , etl_tx_dt )
select select
alias alias
@ -52,6 +52,7 @@ insert into p10_sa.S98_S_partner_summary_visit
, customer_category_type , customer_category_type
, customer_name , customer_name
, dealer_name , dealer_name
, id
, industry , industry
, insert_date , insert_date
, intention_product , intention_product
@ -62,7 +63,6 @@ insert into p10_sa.S98_S_partner_summary_visit
, tsm_names_by_alias , tsm_names_by_alias
, visit_remark , visit_remark
, visitor , visitor
, id
, etl_tx_dt , etl_tx_dt
from p00_tal.S98_S_partner_summary_visit from p00_tal.S98_S_partner_summary_visit
; ;
@ -87,6 +87,7 @@ insert into p12_sfull.S98_S_partner_summary_visit
, customer_category_type , customer_category_type
, customer_name , customer_name
, dealer_name , dealer_name
, id
, industry , industry
, insert_date , insert_date
, intention_product , intention_product
@ -97,7 +98,6 @@ insert into p12_sfull.S98_S_partner_summary_visit
, tsm_names_by_alias , tsm_names_by_alias
, visit_remark , visit_remark
, visitor , visitor
, id
, etl_tx_dt ) , etl_tx_dt )
select select
alias alias
@ -117,6 +117,7 @@ insert into p12_sfull.S98_S_partner_summary_visit
, customer_category_type , customer_category_type
, customer_name , customer_name
, dealer_name , dealer_name
, id
, industry , industry
, insert_date , insert_date
, intention_product , intention_product
@ -127,7 +128,6 @@ insert into p12_sfull.S98_S_partner_summary_visit
, tsm_names_by_alias , tsm_names_by_alias
, visit_remark , visit_remark
, visitor , visitor
, id
, etl_tx_dt , etl_tx_dt
from p10_sa.S98_S_partner_summary_visit from p10_sa.S98_S_partner_summary_visit
; ;

2
dev/workflow/TK_Cust/partner1site/partner_summary_visit/sa_foreign_tables.sql
View File

@ -17,6 +17,7 @@ CREATE FOREIGN TABLE if not exists p00_tal.S98_S_partner_summary_visit (
, customer_category_type TEXT , customer_category_type TEXT
, customer_name TEXT , customer_name TEXT
, dealer_name TEXT , dealer_name TEXT
, id TEXT
, industry TEXT , industry TEXT
, insert_date TEXT , insert_date TEXT
, intention_product TEXT , intention_product TEXT
@ -27,7 +28,6 @@ CREATE FOREIGN TABLE if not exists p00_tal.S98_S_partner_summary_visit (
, tsm_names_by_alias TEXT , tsm_names_by_alias TEXT
, visit_remark TEXT , visit_remark TEXT
, visitor TEXT , visitor TEXT
, id TEXT
, etl_tx_dt TIMESTAMP , etl_tx_dt TIMESTAMP
) )

8
dev/workflow/TK_Cust/partner1site/partner_summary_visit/sa_tables.sql
View File

@ -17,6 +17,7 @@ create table if not exists p10_sa.S98_S_partner_summary_visit (
, customer_category_type TEXT , customer_category_type TEXT
, customer_name TEXT , customer_name TEXT
, dealer_name TEXT , dealer_name TEXT
, id TEXT
, industry TEXT , industry TEXT
, insert_date TEXT , insert_date TEXT
, intention_product TEXT , intention_product TEXT
@ -27,7 +28,6 @@ create table if not exists p10_sa.S98_S_partner_summary_visit (
, tsm_names_by_alias TEXT , tsm_names_by_alias TEXT
, visit_remark TEXT , visit_remark TEXT
, visitor TEXT , visitor TEXT
, id TEXT
, etl_tx_dt TIMESTAMP , etl_tx_dt TIMESTAMP
) ; ) ;
@ -48,6 +48,7 @@ create table if not exists p10_sa.S98_S_partner_summary_visit (
COMMENT ON COLUMN p10_sa.S98_S_partner_summary_visit.customer_category_type IS ''; COMMENT ON COLUMN p10_sa.S98_S_partner_summary_visit.customer_category_type IS '';
COMMENT ON COLUMN p10_sa.S98_S_partner_summary_visit.customer_name IS ''; COMMENT ON COLUMN p10_sa.S98_S_partner_summary_visit.customer_name IS '';
COMMENT ON COLUMN p10_sa.S98_S_partner_summary_visit.dealer_name IS ''; COMMENT ON COLUMN p10_sa.S98_S_partner_summary_visit.dealer_name IS '';
COMMENT ON COLUMN p10_sa.S98_S_partner_summary_visit.id IS '';
COMMENT ON COLUMN p10_sa.S98_S_partner_summary_visit.industry IS ''; COMMENT ON COLUMN p10_sa.S98_S_partner_summary_visit.industry IS '';
COMMENT ON COLUMN p10_sa.S98_S_partner_summary_visit.insert_date IS ''; COMMENT ON COLUMN p10_sa.S98_S_partner_summary_visit.insert_date IS '';
COMMENT ON COLUMN p10_sa.S98_S_partner_summary_visit.intention_product IS ''; COMMENT ON COLUMN p10_sa.S98_S_partner_summary_visit.intention_product IS '';
@ -58,7 +59,6 @@ create table if not exists p10_sa.S98_S_partner_summary_visit (
COMMENT ON COLUMN p10_sa.S98_S_partner_summary_visit.tsm_names_by_alias IS ''; COMMENT ON COLUMN p10_sa.S98_S_partner_summary_visit.tsm_names_by_alias IS '';
COMMENT ON COLUMN p10_sa.S98_S_partner_summary_visit.visit_remark IS ''; COMMENT ON COLUMN p10_sa.S98_S_partner_summary_visit.visit_remark IS '';
COMMENT ON COLUMN p10_sa.S98_S_partner_summary_visit.visitor IS ''; COMMENT ON COLUMN p10_sa.S98_S_partner_summary_visit.visitor IS '';
COMMENT ON COLUMN p10_sa.S98_S_partner_summary_visit.id IS '';
COMMENT ON COLUMN p10_sa.S98_S_partner_summary_visit.etl_tx_dt IS ''; COMMENT ON COLUMN p10_sa.S98_S_partner_summary_visit.etl_tx_dt IS '';
COMMENT ON TABLE p10_sa.S98_S_partner_summary_visit IS ''; COMMENT ON TABLE p10_sa.S98_S_partner_summary_visit IS '';
@ -83,6 +83,7 @@ create table if not exists p12_sfull.S98_S_partner_summary_visit (
, customer_category_type TEXT , customer_category_type TEXT
, customer_name TEXT , customer_name TEXT
, dealer_name TEXT , dealer_name TEXT
, id TEXT
, industry TEXT , industry TEXT
, insert_date TEXT , insert_date TEXT
, intention_product TEXT , intention_product TEXT
@ -93,7 +94,6 @@ create table if not exists p12_sfull.S98_S_partner_summary_visit (
, tsm_names_by_alias TEXT , tsm_names_by_alias TEXT
, visit_remark TEXT , visit_remark TEXT
, visitor TEXT , visitor TEXT
, id TEXT
, etl_tx_dt TIMESTAMP , etl_tx_dt TIMESTAMP
) ; ) ;
@ -114,6 +114,7 @@ create table if not exists p12_sfull.S98_S_partner_summary_visit (
COMMENT ON COLUMN p12_sfull.S98_S_partner_summary_visit.customer_category_type IS ''; COMMENT ON COLUMN p12_sfull.S98_S_partner_summary_visit.customer_category_type IS '';
COMMENT ON COLUMN p12_sfull.S98_S_partner_summary_visit.customer_name IS ''; COMMENT ON COLUMN p12_sfull.S98_S_partner_summary_visit.customer_name IS '';
COMMENT ON COLUMN p12_sfull.S98_S_partner_summary_visit.dealer_name IS ''; COMMENT ON COLUMN p12_sfull.S98_S_partner_summary_visit.dealer_name IS '';
COMMENT ON COLUMN p12_sfull.S98_S_partner_summary_visit.id IS '';
COMMENT ON COLUMN p12_sfull.S98_S_partner_summary_visit.industry IS ''; COMMENT ON COLUMN p12_sfull.S98_S_partner_summary_visit.industry IS '';
COMMENT ON COLUMN p12_sfull.S98_S_partner_summary_visit.insert_date IS ''; COMMENT ON COLUMN p12_sfull.S98_S_partner_summary_visit.insert_date IS '';
COMMENT ON COLUMN p12_sfull.S98_S_partner_summary_visit.intention_product IS ''; COMMENT ON COLUMN p12_sfull.S98_S_partner_summary_visit.intention_product IS '';
@ -124,7 +125,6 @@ create table if not exists p12_sfull.S98_S_partner_summary_visit (
COMMENT ON COLUMN p12_sfull.S98_S_partner_summary_visit.tsm_names_by_alias IS ''; COMMENT ON COLUMN p12_sfull.S98_S_partner_summary_visit.tsm_names_by_alias IS '';
COMMENT ON COLUMN p12_sfull.S98_S_partner_summary_visit.visit_remark IS ''; COMMENT ON COLUMN p12_sfull.S98_S_partner_summary_visit.visit_remark IS '';
COMMENT ON COLUMN p12_sfull.S98_S_partner_summary_visit.visitor IS ''; COMMENT ON COLUMN p12_sfull.S98_S_partner_summary_visit.visitor IS '';
COMMENT ON COLUMN p12_sfull.S98_S_partner_summary_visit.id IS '';
COMMENT ON COLUMN p12_sfull.S98_S_partner_summary_visit.etl_tx_dt IS ''; COMMENT ON COLUMN p12_sfull.S98_S_partner_summary_visit.etl_tx_dt IS '';
COMMENT ON TABLE p12_sfull.S98_S_partner_summary_visit IS ''; COMMENT ON TABLE p12_sfull.S98_S_partner_summary_visit IS '';

285
dev/workflow/TK_Cust/partner1site/获取客户拜访数据/part_summary_visit_feign.py
View File

@ -1,143 +1,142 @@
#!/usr/bin/env python3 #!/usr/bin/env python3
# -*- coding: utf-8 -*- # -*- coding: utf-8 -*-
""" """
Partner1site 全接口抓取脚本 Partner1site 全接口抓取脚本
分页结束条件：hasNext == False 分页结束条件：hasNext == False
""" """
import random import random
import hmac import hmac
import hashlib import hashlib
import base64 import base64
import requests import requests
import json import json
import uuid import uuid
from datetime import datetime, timezone, timedelta from datetime import datetime, timezone, timedelta
from typing import Dict, Any from typing import Dict, Any
import psycopg2 import psycopg2
# ======= 配置区 ======= # ======= 配置区 =======
ACCESS_KEY = "75c4ab4d-6a67-4aed-8b1d-5bb64fd36afc" ACCESS_KEY = "75c4ab4d-6a67-4aed-8b1d-5bb64fd36afc"
SECRET_KEY = "117347a7dd066a50a4d2973c5f3d5ba9101094c5" SECRET_KEY = "117347a7dd066a50a4d2973c5f3d5ba9101094c5"
PG_DSN = dict( PG_DSN = dict(
database="dataops_db", database="dataops_db",
user="dbuser_dba", user="dbuser_dba",
password="EmBRxnmmjnE3", password="EmBRxnmmjnE3",
host="124.221.232.219", host="124.221.232.219",
port="5432" port="5432"
) )
API_ID = "2460976d-00c1-47d9-84b2-33e66d68" # 外部传入 api_id，占位符 API_ID = "2460976d-00c1-47d9-84b2-33e66d68" # 外部传入 api_id，占位符
# ====================== # ======================
class Partner1SiteClient: class Partner1SiteClient:
"""Partner1site API 客户端""" """Partner1site API 客户端"""
def __init__(self, access_key: str, secret_key: str): def __init__(self, access_key: str, secret_key: str):
self.ak = access_key self.ak = access_key
self.sk = secret_key self.sk = secret_key
@staticmethod @staticmethod
def urlsafe_b64encode(data: bytes) -> str: def urlsafe_b64encode(data: bytes) -> str:
return base64.urlsafe_b64encode(data).decode() return base64.urlsafe_b64encode(data).decode()
def gen_token(self, expire_sec: int = 600) -> str: def gen_token(self, expire_sec: int = 600) -> str:
"""生成 API Token""" """生成 API Token"""
random_num = str(random.randint(100000, 999999)) random_num = str(random.randint(100000, 999999))
deadline = int(datetime.now(timezone.utc).timestamp()) + expire_sec deadline = int(datetime.now(timezone.utc).timestamp()) + expire_sec
parm_str = f"{random_num}:{deadline}" parm_str = f"{random_num}:{deadline}"
enc_parm = self.urlsafe_b64encode(parm_str.encode()) enc_parm = self.urlsafe_b64encode(parm_str.encode())
sign = hmac.new(self.sk.encode(), enc_parm.encode(), hashlib.sha1).digest() sign = hmac.new(self.sk.encode(), enc_parm.encode(), hashlib.sha1).digest()
enc_sign = self.urlsafe_b64encode(sign.hex().encode()) enc_sign = self.urlsafe_b64encode(sign.hex().encode())
return f"{self.ak}:{enc_sign}:{enc_parm}" return f"{self.ak}:{enc_sign}:{enc_parm}"
def fetch_all_pages(self, api_name: str, params: Dict[str, Any]): def fetch_all_pages(self, api_name: str, params: Dict[str, Any]):
"""通用分页请求（结束条件：hasNext == False）""" """通用分页请求（结束条件：hasNext == False）"""
# if api_name not in BASE_URLS: # if api_name not in BASE_URLS:
# raise ValueError(f"未知 API 数据来源: {api_name}") # raise ValueError(f"未知 API 数据来源: {api_name}")
base_url = 'http://onesite.tek.cn/api/summary/visits' base_url = 'http://onesite.tek.cn/api/summary/visits'
all_data = [] all_data = []
page_num = 0 page_num = 0
page_size = 1000 # 固定每页大小 page_size = 1000 # 固定每页大小
while True: while True:
token = self.gen_token() token = self.gen_token()
params_with_paging = dict(params) params_with_paging = dict(params)
params_with_paging.update({ params_with_paging.update({
"token": token, "token": token,
"size": page_size, "size": page_size,
"page": page_num "page": page_num
}) })
resp = requests.get(base_url, params=params_with_paging, timeout=120) resp = requests.get(base_url, params=params_with_paging, timeout=30)
resp.raise_for_status() resp.raise_for_status()
data_json = resp.json() data_json = resp.json()
if data_json.get("code") != 100 or not data_json.get("success", False): if data_json.get("code") != 100 or not data_json.get("success", False):
raise RuntimeError(f"获取客户拜访数据 API 错误: {data_json.get('message')}") raise RuntimeError(f"获取客户拜访数据 API 错误: {data_json.get('message')}")
content = data_json.get("data", {}).get("content", []) content = data_json.get("data", {}).get("content", [])
all_data.extend(content) all_data.extend(content)
total_elements = data_json.get("data", {}).get("totalElements") total_elements = data_json.get("data", {}).get("totalElements")
has_next = data_json.get("data", {}).get("hasNext", False) has_next = data_json.get("data", {}).get("hasNext", False)
print(f"[获取客户拜访数据] 页码 {page_num} -> 本页 {len(content)} 条，累计 {len(all_data)} 条 / 总数 {total_elements}") print(f"[获取客户拜访数据] 页码 {page_num} -> 本页 {len(content)} 条，累计 {len(all_data)} 条 / 总数 {total_elements}")
if not has_next: if not has_next:
break break
page_num += 1 page_num += 1
return all_data return all_data
def save_json_to_pg(data: list, api_id: str) -> None: def save_json_to_pg(data: list, api_id: str) -> None:
"""写入 PostgreSQL：软删历史 + 插入新数据""" """写入 PostgreSQL：软删历史 + 插入新数据"""
print(f"[save_to_pg] API={api_id} 写入 PG，记录数={len(data)}") print(f"[save_to_pg] API={api_id} 写入 PG，记录数={len(data)}")
sql = """ sql = """
UPDATE data_api.api_data UPDATE data_api.api_data
SET is_loaded = '1' SET is_loaded = '1'
WHERE api_id = %s; WHERE api_id = %s;
INSERT INTO data_api.api_data INSERT INTO data_api.api_data
(id, api_id, data, total_num, is_loaded, status, (id, api_id, data, total_num, is_loaded, status,
request_tm, execute_tm, remark) request_tm, execute_tm, remark)
VALUES (%s, %s, %s, %s, '0', '0', VALUES (%s, %s, %s, %s, '0', '0',
current_timestamp(0), current_timestamp(0), ''); current_timestamp(0), current_timestamp(0), '');
""" """
try: try:
with psycopg2.connect(**PG_DSN) as conn: with psycopg2.connect(**PG_DSN) as conn:
with conn.cursor() as cur: with conn.cursor() as cur:
cur.execute(sql, cur.execute(sql,
(api_id, (api_id,
str(uuid.uuid4()), str(uuid.uuid4()),
api_id, api_id,
json.dumps(data, ensure_ascii=False), json.dumps(data, ensure_ascii=False),
len(data))) len(data)))
conn.commit() conn.commit()
print(f"[save_to_pg] API={api_id} 写入完成") print(f"[save_to_pg] API={api_id} 写入完成")
except Exception as e: except Exception as e:
raise RuntimeError(f"PG写入错误: {e}") raise RuntimeError(f"PG写入错误: {e}")
def get_previous_date(days: int = 0) -> str: def get_previous_date(days: int = 0) -> str:
return (datetime.now() - timedelta(days=days)).strftime("%Y-%m-%d") return (datetime.now() - timedelta(days=days)).strftime("%Y-%m-%d")
def main(): def main():
client = Partner1SiteClient(ACCESS_KEY, SECRET_KEY) client = Partner1SiteClient(ACCESS_KEY, SECRET_KEY)
data = client.fetch_all_pages( data = client.fetch_all_pages(
api_name="visits", api_name="visits",
params={'startInsertDate':get_previous_date(7)} params={'startInsertDate':'2025-8-20',} # 拉全量，若有默认时间限制可改成 {"startInsertDate":"2000-01-01","endInsertDate":get_previous_date(0)}
# params={'startInsertDate':'2000-1-1'} )
) save_json_to_pg(data, API_ID)
save_json_to_pg(data, API_ID)
if __name__ == "__main__":
if __name__ == "__main__": main()
main()

18
dev/workflow/TK_Cust/partner1site/获取客户拜访数据/part_summary_visit_load.sql
View File

@ -21,8 +21,11 @@ insert into data_api.partner_summary_visit (
, contact_ways , contact_ways
, customer_category , customer_category
, customer_category_type , customer_category_type
, customer_contact_id
, customer_id
, customer_name , customer_name
, dealer_name , dealer_name
, id
, industry , industry
, insert_date , insert_date
, intention_product , intention_product
@ -32,8 +35,7 @@ insert into data_api.partner_summary_visit (
, sub_industry , sub_industry
, tsm_names_by_alias , tsm_names_by_alias
, visit_remark , visit_remark
, visitor , visitor
, id
,etl_tx_dt ,etl_tx_dt
) )
select select
@ -52,8 +54,11 @@ select
, case when trim(both from contact_ways)='' then null else contact_ways::text end contact_ways , case when trim(both from contact_ways)='' then null else contact_ways::text end contact_ways
, case when trim(both from customer_category)='' then null else customer_category::text end customer_category , case when trim(both from customer_category)='' then null else customer_category::text end customer_category
, case when trim(both from customer_category_type)='' then null else customer_category_type::text end customer_category_type , case when trim(both from customer_category_type)='' then null else customer_category_type::text end customer_category_type
, case when trim(both from customer_contact_id)='' then null else customer_contact_id::text end customer_contact_id
, case when trim(both from customer_id)='' then null else customer_id::text end customer_id
, case when trim(both from customer_name)='' then null else customer_name::text end customer_name , case when trim(both from customer_name)='' then null else customer_name::text end customer_name
, case when trim(both from dealer_name)='' then null else dealer_name::text end dealer_name , case when trim(both from dealer_name)='' then null else dealer_name::text end dealer_name
, case when trim(both from id)='' then null else id::text end id
, case when trim(both from industry)='' then null else industry::text end industry , case when trim(both from industry)='' then null else industry::text end industry
, case when trim(both from insert_date)='' then null else insert_date::text end insert_date , case when trim(both from insert_date)='' then null else insert_date::text end insert_date
, case when trim(both from intention_product)='' then null else intention_product::text end intention_product , case when trim(both from intention_product)='' then null else intention_product::text end intention_product
@ -63,8 +68,7 @@ select
, case when trim(both from sub_industry)='' then null else sub_industry::text end sub_industry , case when trim(both from sub_industry)='' then null else sub_industry::text end sub_industry
, case when trim(both from tsm_names_by_alias)='' then null else tsm_names_by_alias::text end tsm_names_by_alias , case when trim(both from tsm_names_by_alias)='' then null else tsm_names_by_alias::text end tsm_names_by_alias
, case when trim(both from visit_remark)='' then null else visit_remark::text end visit_remark , case when trim(both from visit_remark)='' then null else visit_remark::text end visit_remark
, case when trim(both from visitor)='' then null else visitor::text end visitor , case when trim(both from visitor)='' then null else visitor::text end visitor
, case when trim(both from id)='' then null else id::text end id
,etl_tx_dt ,etl_tx_dt
from ( from (
select select
@ -83,8 +87,11 @@ select
, (json_array_elements(data::json)::json->>'contactWays') contact_ways , (json_array_elements(data::json)::json->>'contactWays') contact_ways
, (json_array_elements(data::json)::json->>'customerCategory') customer_category , (json_array_elements(data::json)::json->>'customerCategory') customer_category
, (json_array_elements(data::json)::json->>'customerCategoryType') customer_category_type , (json_array_elements(data::json)::json->>'customerCategoryType') customer_category_type
, (json_array_elements(data::json)::json->>'customerContactId') customer_contact_id
, (json_array_elements(data::json)::json->>'customerId') customer_id
, (json_array_elements(data::json)::json->>'customerName') customer_name , (json_array_elements(data::json)::json->>'customerName') customer_name
, (json_array_elements(data::json)::json->>'dealerName') dealer_name , (json_array_elements(data::json)::json->>'dealerName') dealer_name
, (json_array_elements(data::json)::json->>'id') id
, (json_array_elements(data::json)::json->>'industry') industry , (json_array_elements(data::json)::json->>'industry') industry
, (json_array_elements(data::json)::json->>'insertDate') insert_date , (json_array_elements(data::json)::json->>'insertDate') insert_date
, (json_array_elements(data::json)::json->>'intentionProduct') intention_product , (json_array_elements(data::json)::json->>'intentionProduct') intention_product
@ -94,8 +101,7 @@ select
, (json_array_elements(data::json)::json->>'subIndustry') sub_industry , (json_array_elements(data::json)::json->>'subIndustry') sub_industry
, (json_array_elements(data::json)::json->>'tsmNamesByAlias') tsm_names_by_alias , (json_array_elements(data::json)::json->>'tsmNamesByAlias') tsm_names_by_alias
, (json_array_elements(data::json)::json->>'visitRemark') visit_remark , (json_array_elements(data::json)::json->>'visitRemark') visit_remark
, (json_array_elements(data::json)::json->>'visitor') visitor , (json_array_elements(data::json)::json->>'visitor') visitor
, (json_array_elements(data::json)::json->>'id') id
,CURRENT_TIMESTAMP(0) etl_tx_dt ,CURRENT_TIMESTAMP(0) etl_tx_dt
from (select * from data_api.api_data from (select * from data_api.api_data
WHERE api_id='2460976d-00c1-47d9-84b2-33e66d68' and is_loaded = '0' order by request_tm desc limit 1) p )p; WHERE api_id='2460976d-00c1-47d9-84b2-33e66d68' and is_loaded = '0' order by request_tm desc limit 1) p )p;