diff --git a/dev/workflow/TK_Cust/partner1site/Partner客户拜访记录/t01_partner_visit.sql b/dev/workflow/TK_Cust/partner1site/Partner客户拜访记录/t01_partner_visit.sql index dee71a7..a9e5414 100644 --- a/dev/workflow/TK_Cust/partner1site/Partner客户拜访记录/t01_partner_visit.sql +++ b/dev/workflow/TK_Cust/partner1site/Partner客户拜访记录/t01_partner_visit.sql @@ -1,7 +1,8 @@ DROP TABLE IF EXISTS p20_pdm.t01_partner_visit; CREATE TABLE IF NOT EXISTS p20_pdm.t01_partner_visit ( - insert_date date + id varchar(100) + , insert_date date , visitor varchar(100) , dealer_name varchar(100) , customer_name varchar(100) @@ -35,9 +36,10 @@ CREATE TABLE IF NOT EXISTS p20_pdm.t01_partner_visit ( , Etl_Tx_Dt timestamp(0) , Src_Sysname varchar(50) , Src_Table varchar(50) - + ,primary key( id ) ); + COMMENT ON COLUMN p20_pdm.t01_partner_visit.id IS '主键ID'; COMMENT ON COLUMN p20_pdm.t01_partner_visit.insert_date IS '录入时间'; COMMENT ON COLUMN p20_pdm.t01_partner_visit.visitor IS '拜访人'; COMMENT ON COLUMN p20_pdm.t01_partner_visit.dealer_name IS '代理商名称'; diff --git a/dev/workflow/TK_Cust/partner1site/Partner客户拜访记录/t01_partner_visit_agi.sql b/dev/workflow/TK_Cust/partner1site/Partner客户拜访记录/t01_partner_visit_agi.sql index 9ea0e9a..b492f13 100644 --- a/dev/workflow/TK_Cust/partner1site/Partner客户拜访记录/t01_partner_visit_agi.sql +++ b/dev/workflow/TK_Cust/partner1site/Partner客户拜访记录/t01_partner_visit_agi.sql @@ -4,11 +4,11 @@ /*Brilliance stems from wisdoms. */ /*************Head Section**************************************************************************/ /*Script Use: Periodically load data to :t01_partner_visit(Partner客户拜访记录) */ -/*Create Date:2025-09-30 21:24:49 */ +/*Create Date:2025-10-16 17:37:32 */ /*SDM Developed By: dev */ /*SDM Developed Date: 2025-09-29 */ /*SDM Checked By: dev */ -/*SDM Checked Date: 2025-09-30 */ +/*SDM Checked Date: 2025-10-16 */ /*Script Developed By: dev */ /*Script Checked By: dev */ /*Source table 1: p10_sa.s98_s_partner_summary_visit */ @@ -16,7 +16,7 @@ /*Target Table:t01_partner_visit */ /*ETL Job Name:t01_partner_visit */ /*ETL Frequency:None */ -/*ETL Policy:F1 */ +/*ETL Policy:F2 */ /********************************************************************************************/ /*******Main Section**************************************************************************/ \set ON_ERROR_STOP on @@ -31,11 +31,19 @@ CREATE TEMPORARY TABLE t01_partner_visit_agi_CUR_I ON COMMIT PRESERVE ROWS; + +/*创建临时表加载不同数据 */ +CREATE TEMPORARY TABLE t01_partner_visit_agi_INS +( LIKE :PDMDB.t01_partner_visit) +ON COMMIT PRESERVE ROWS; + + /*****************************************************************************************************/ /* GROUP 1:Source Table:s98_s_partner_summary_visit***************************************************/ /*****************************************************************************************************/ INSERT INTO t01_partner_visit_agi_CUR_I ( - insert_date /*录入时间*/ + id /*主键ID*/ + ,insert_date /*录入时间*/ ,visitor /*拜访人*/ ,dealer_name /*代理商名称*/ ,customer_name /*客户名称*/ @@ -71,7 +79,8 @@ INSERT INTO t01_partner_visit_agi_CUR_I ( ,Src_Table /*来源表*/ ) SELECT - p0.insert_date::date /*insert_date*/ + COALESCE(TRIM(CAST(p0.id AS varchar(100))),'') /*id*/ + ,p0.insert_date::date /*insert_date*/ ,COALESCE(TRIM(CAST(p0.visitor AS varchar(100))),'') /*visitor*/ ,COALESCE(TRIM(CAST(p0.dealer_name AS varchar(100))),'') /*dealer_name*/ ,COALESCE(TRIM(CAST(p0.customer_name AS varchar(100))),'') /*customer_name*/ @@ -111,87 +120,259 @@ FROM p10_sa.s98_s_partner_summary_visit p0 -/*从目标表中删除所有数据 t01_partner_visit(Partner客户拜访记录) */ -DELETE FROM :PDMDB.t01_partner_visit -WHERE ETL_JOB=:ETLJOB; - - +/*将不同数据插入到临时表 */ +;INSERT INTO t01_partner_visit_agi_INS ( + insert_date /*录入时间*/ + ,visitor /*拜访人*/ + ,dealer_name /*代理商名称*/ + ,customer_name /*客户名称*/ + ,contact_name /*联系人*/ + ,contact_department /*联系人部门*/ + ,customer_category_type /*产品分类*/ + ,intention_product /*意向产品*/ + ,into_report /*是否进入报备*/ + ,contact_ways /*联络方式*/ + ,attachment_number /*附件数量*/ + ,visit_remark /*拜访备注*/ + ,comment_number /*评论数*/ + ,like_number /*点赞数*/ + ,area /*大区*/ + ,contact_province /*省份*/ + ,contact_city /*城市*/ + ,contact_district /*区域*/ + ,contact_phone /*联系电话*/ + ,contact_email /*联系人邮件*/ + ,contact_address /*地址*/ + ,like_name /*点赞人*/ + ,alias /*别名*/ + ,industry /*行业*/ + ,sub_industry /*子行业*/ + ,tsm_names_by_alias /*tsm名*/ + ,customer_category /*客户类别*/ + ,id /*主键ID*/ + ,Etl_Batch_No /*作业批次号*/ + ,Etl_First_Dt /*最初入库时间*/ + ,Etl_Job /*作业名称*/ + ,Etl_Proc_Dt /*本次入库时间*/ + ,Etl_Tx_Dt /*作业运行时间*/ + ,Src_Sysname /*来源系统*/ + ,Src_Table /*来源表*/ + +) + SELECT + P1.insert_date /*录入时间*/ + ,P1.visitor /*拜访人*/ + ,P1.dealer_name /*代理商名称*/ + ,P1.customer_name /*客户名称*/ + ,P1.contact_name /*联系人*/ + ,P1.contact_department /*联系人部门*/ + ,P1.customer_category_type /*产品分类*/ + ,P1.intention_product /*意向产品*/ + ,P1.into_report /*是否进入报备*/ + ,P1.contact_ways /*联络方式*/ + ,P1.attachment_number /*附件数量*/ + ,P1.visit_remark /*拜访备注*/ + ,P1.comment_number /*评论数*/ + ,P1.like_number /*点赞数*/ + ,P1.area /*大区*/ + ,P1.contact_province /*省份*/ + ,P1.contact_city /*城市*/ + ,P1.contact_district /*区域*/ + ,P1.contact_phone /*联系电话*/ + ,P1.contact_email /*联系人邮件*/ + ,P1.contact_address /*地址*/ + ,P1.like_name /*点赞人*/ + ,P1.alias /*别名*/ + ,P1.industry /*行业*/ + ,P1.sub_industry /*子行业*/ + ,P1.tsm_names_by_alias /*tsm名*/ + ,P1.customer_category /*客户类别*/ + ,P1.id /*主键ID*/ + ,P1.Etl_Batch_No /*作业批次号*/ + ,P1.Etl_First_Dt /*最初入库时间*/ + ,P1.Etl_Job /*作业名称*/ + ,P1.Etl_Proc_Dt /*本次入库时间*/ + ,P1.Etl_Tx_Dt /*作业运行时间*/ + ,P1.Src_Sysname /*来源系统*/ + ,P1.Src_Table /*来源表*/ + +FROM t01_partner_visit_agi_CUR_I P1 +LEFT JOIN :PDMDB.t01_partner_visit P2 +ON P1.insert_date = P2.insert_date + AND P1.visitor = P2.visitor + AND P1.dealer_name = P2.dealer_name + AND P1.customer_name = P2.customer_name + AND P1.contact_name = P2.contact_name + AND P1.contact_department = P2.contact_department + AND P1.customer_category_type = P2.customer_category_type + AND P1.intention_product = P2.intention_product + AND P1.into_report = P2.into_report + AND P1.contact_ways = P2.contact_ways + AND P1.attachment_number = P2.attachment_number + AND P1.visit_remark = P2.visit_remark + AND P1.comment_number = P2.comment_number + AND P1.like_number = P2.like_number + AND P1.area = P2.area + AND P1.contact_province = P2.contact_province + AND P1.contact_city = P2.contact_city + AND P1.contact_district = P2.contact_district + AND P1.contact_phone = P2.contact_phone + AND P1.contact_email = P2.contact_email + AND P1.contact_address = P2.contact_address + AND P1.like_name = P2.like_name + AND P1.alias = P2.alias + AND P1.industry = P2.industry + AND P1.sub_industry = P2.sub_industry + AND P1.tsm_names_by_alias = P2.tsm_names_by_alias + AND P1.customer_category = P2.customer_category + AND P1.id = P2.id + +WHERE P2.insert_date IS NULL + OR P2.visitor IS NULL + OR P2.dealer_name IS NULL + OR P2.customer_name IS NULL + OR P2.contact_name IS NULL + OR P2.contact_department IS NULL + OR P2.customer_category_type IS NULL + OR P2.intention_product IS NULL + OR P2.into_report IS NULL + OR P2.contact_ways IS NULL + OR P2.attachment_number IS NULL + OR P2.visit_remark IS NULL + OR P2.comment_number IS NULL + OR P2.like_number IS NULL + OR P2.area IS NULL + OR P2.contact_province IS NULL + OR P2.contact_city IS NULL + OR P2.contact_district IS NULL + OR P2.contact_phone IS NULL + OR P2.contact_email IS NULL + OR P2.contact_address IS NULL + OR P2.like_name IS NULL + OR P2.alias IS NULL + OR P2.industry IS NULL + OR P2.sub_industry IS NULL + OR P2.tsm_names_by_alias IS NULL + OR P2.customer_category IS NULL + OR P2.id IS NULL + +; /*将新增数据插入到目标表 */ ;INSERT INTO :PDMDB.t01_partner_visit ( - insert_date /*录入时间*/ - ,visitor /*拜访人*/ - ,dealer_name /*代理商名称*/ - ,customer_name /*客户名称*/ - ,contact_name /*联系人*/ - ,contact_department /*联系人部门*/ - ,customer_category_type /*产品分类*/ - ,intention_product /*意向产品*/ - ,into_report /*是否进入报备*/ - ,contact_ways /*联络方式*/ - ,attachment_number /*附件数量*/ - ,visit_remark /*拜访备注*/ - ,comment_number /*评论数*/ - ,like_number /*点赞数*/ - ,area /*大区*/ - ,contact_province /*省份*/ - ,contact_city /*城市*/ - ,contact_district /*区域*/ - ,contact_phone /*联系电话*/ - ,contact_email /*联系人邮件*/ - ,contact_address /*地址*/ - ,like_name /*点赞人*/ - ,alias /*别名*/ - ,industry /*行业*/ - ,sub_industry /*子行业*/ - ,tsm_names_by_alias /*tsm名*/ - ,customer_category /*客户类别*/ - ,Etl_Batch_No /*作业批次号*/ - ,Etl_First_Dt /*最初入库时间*/ - ,Etl_Job /*作业名称*/ - ,Etl_Proc_Dt /*本次入库时间*/ - ,Etl_Tx_Dt /*作业运行时间*/ - ,Src_Sysname /*来源系统*/ - ,Src_Table /*来源表*/ - - ) - SELECT - P1.insert_date /*录入时间*/ - ,P1.visitor /*拜访人*/ - ,P1.dealer_name /*代理商名称*/ - ,P1.customer_name /*客户名称*/ - ,P1.contact_name /*联系人*/ - ,P1.contact_department /*联系人部门*/ - ,P1.customer_category_type /*产品分类*/ - ,P1.intention_product /*意向产品*/ - ,P1.into_report /*是否进入报备*/ - ,P1.contact_ways /*联络方式*/ - ,P1.attachment_number /*附件数量*/ - ,P1.visit_remark /*拜访备注*/ - ,P1.comment_number /*评论数*/ - ,P1.like_number /*点赞数*/ - ,P1.area /*大区*/ - ,P1.contact_province /*省份*/ - ,P1.contact_city /*城市*/ - ,P1.contact_district /*区域*/ - ,P1.contact_phone /*联系电话*/ - ,P1.contact_email /*联系人邮件*/ - ,P1.contact_address /*地址*/ - ,P1.like_name /*点赞人*/ - ,P1.alias /*别名*/ - ,P1.industry /*行业*/ - ,P1.sub_industry /*子行业*/ - ,P1.tsm_names_by_alias /*tsm名*/ - ,P1.customer_category /*客户类别*/ - ,P1.Etl_Batch_No /*作业批次号*/ - ,P1.Etl_First_Dt /*最初入库时间*/ - ,P1.Etl_Job /*作业名称*/ - ,P1.Etl_Proc_Dt /*本次入库时间*/ - ,P1.Etl_Tx_Dt /*作业运行时间*/ - ,P1.Src_Sysname /*来源系统*/ - ,P1.Src_Table /*来源表*/ - - FROM t01_partner_visit_agi_CUR_I P1 - + insert_date /*录入时间*/ + ,visitor /*拜访人*/ + ,dealer_name /*代理商名称*/ + ,customer_name /*客户名称*/ + ,contact_name /*联系人*/ + ,contact_department /*联系人部门*/ + ,customer_category_type /*产品分类*/ + ,intention_product /*意向产品*/ + ,into_report /*是否进入报备*/ + ,contact_ways /*联络方式*/ + ,attachment_number /*附件数量*/ + ,visit_remark /*拜访备注*/ + ,comment_number /*评论数*/ + ,like_number /*点赞数*/ + ,area /*大区*/ + ,contact_province /*省份*/ + ,contact_city /*城市*/ + ,contact_district /*区域*/ + ,contact_phone /*联系电话*/ + ,contact_email /*联系人邮件*/ + ,contact_address /*地址*/ + ,like_name /*点赞人*/ + ,alias /*别名*/ + ,industry /*行业*/ + ,sub_industry /*子行业*/ + ,tsm_names_by_alias /*tsm名*/ + ,customer_category /*客户类别*/ + ,id /*主键ID*/ + ,Etl_Batch_No /*作业批次号*/ + ,Etl_First_Dt /*最初入库时间*/ + ,Etl_Job /*作业名称*/ + ,Etl_Proc_Dt /*本次入库时间*/ + ,Etl_Tx_Dt /*作业运行时间*/ + ,Src_Sysname /*来源系统*/ + ,Src_Table /*来源表*/ + +) +SELECT + P1.insert_date /*录入时间*/ + ,P1.visitor /*拜访人*/ + ,P1.dealer_name /*代理商名称*/ + ,P1.customer_name /*客户名称*/ + ,P1.contact_name /*联系人*/ + ,P1.contact_department /*联系人部门*/ + ,P1.customer_category_type /*产品分类*/ + ,P1.intention_product /*意向产品*/ + ,P1.into_report /*是否进入报备*/ + ,P1.contact_ways /*联络方式*/ + ,P1.attachment_number /*附件数量*/ + ,P1.visit_remark /*拜访备注*/ + ,P1.comment_number /*评论数*/ + ,P1.like_number /*点赞数*/ + ,P1.area /*大区*/ + ,P1.contact_province /*省份*/ + ,P1.contact_city /*城市*/ + ,P1.contact_district /*区域*/ + ,P1.contact_phone /*联系电话*/ + ,P1.contact_email /*联系人邮件*/ + ,P1.contact_address /*地址*/ + ,P1.like_name /*点赞人*/ + ,P1.alias /*别名*/ + ,P1.industry /*行业*/ + ,P1.sub_industry /*子行业*/ + ,P1.tsm_names_by_alias /*tsm名*/ + ,P1.customer_category /*客户类别*/ + ,P1.id /*主键ID*/ + ,P1.Etl_Batch_No /*作业批次号*/ + ,P1.Etl_First_Dt /*最初入库时间*/ + ,P1.Etl_Job /*作业名称*/ + ,P1.Etl_Proc_Dt /*本次入库时间*/ + ,P1.Etl_Tx_Dt /*作业运行时间*/ + ,P1.Src_Sysname /*来源系统*/ + ,P1.Src_Table /*来源表*/ + +FROM t01_partner_visit_agi_INS P1 +ON CONFLICT ( id) +DO UPDATE SET + id=excluded.id + ,insert_date=excluded.insert_date + ,visitor=excluded.visitor + ,dealer_name=excluded.dealer_name + ,customer_name=excluded.customer_name + ,contact_name=excluded.contact_name + ,contact_department=excluded.contact_department + ,customer_category_type=excluded.customer_category_type + ,intention_product=excluded.intention_product + ,into_report=excluded.into_report + ,contact_ways=excluded.contact_ways + ,attachment_number=excluded.attachment_number + ,visit_remark=excluded.visit_remark + ,comment_number=excluded.comment_number + ,like_number=excluded.like_number + ,area=excluded.area + ,contact_province=excluded.contact_province + ,contact_city=excluded.contact_city + ,contact_district=excluded.contact_district + ,contact_phone=excluded.contact_phone + ,contact_email=excluded.contact_email + ,contact_address=excluded.contact_address + ,like_name=excluded.like_name + ,alias=excluded.alias + ,industry=excluded.industry + ,sub_industry=excluded.sub_industry + ,tsm_names_by_alias=excluded.tsm_names_by_alias + ,customer_category=excluded.customer_category + ,Etl_Batch_No=excluded.Etl_Batch_No + ,Etl_First_Dt=excluded.Etl_First_Dt + ,Etl_Job=excluded.Etl_Job + ,Etl_Proc_Dt=excluded.Etl_Proc_Dt + ,Etl_Tx_Dt=excluded.Etl_Tx_Dt + ,Src_Sysname=excluded.Src_Sysname + ,Src_Table=excluded.Src_Table + + ; /*****程序结束退出 */ \q diff --git a/dev/workflow/TK_Cust/partner1site/partner one site联系人信息/d_partner_contact.sql b/dev/workflow/TK_Cust/partner1site/partner one site联系人信息/d_partner_contact.sql new file mode 100644 index 0000000..f2cc989 --- /dev/null +++ b/dev/workflow/TK_Cust/partner1site/partner one site联系人信息/d_partner_contact.sql @@ -0,0 +1,38 @@ + +DROP TABLE IF EXISTS p30_common.d_partner_contact; +CREATE TABLE IF NOT EXISTS p30_common.d_partner_contact ( + contact_id bigint + , contact_name VARCHAR(200) + , province VARCHAR(50) + , city VARCHAR(50) + , address text + , department VARCHAR(50) + , email VARCHAR(50) + , phone VARCHAR(50) + , Etl_Batch_No varchar(50) + , Etl_First_Dt timestamp(0) + , Etl_Job varchar(200) + , Etl_Proc_Dt timestamp(0) + , Etl_Tx_Dt timestamp(0) + , Src_Sysname varchar(50) + , Src_Table varchar(50) + ,primary key( contact_id ) +); + + COMMENT ON COLUMN p30_common.d_partner_contact.contact_id IS '联系人ID'; + COMMENT ON COLUMN p30_common.d_partner_contact.contact_name IS '联系人名称'; + COMMENT ON COLUMN p30_common.d_partner_contact.province IS '省份'; + COMMENT ON COLUMN p30_common.d_partner_contact.city IS '城市'; + COMMENT ON COLUMN p30_common.d_partner_contact.address IS '地址'; + COMMENT ON COLUMN p30_common.d_partner_contact.department IS '部门'; + COMMENT ON COLUMN p30_common.d_partner_contact.email IS '邮箱'; + COMMENT ON COLUMN p30_common.d_partner_contact.phone IS '电话'; + COMMENT ON COLUMN p30_common.d_partner_contact.Etl_Batch_No IS '作业批次号'; + COMMENT ON COLUMN p30_common.d_partner_contact.Etl_First_Dt IS '最初入库时间'; + COMMENT ON COLUMN p30_common.d_partner_contact.Etl_Job IS '作业名称'; + COMMENT ON COLUMN p30_common.d_partner_contact.Etl_Proc_Dt IS '本次入库时间'; + COMMENT ON COLUMN p30_common.d_partner_contact.Etl_Tx_Dt IS '作业运行时间'; + COMMENT ON COLUMN p30_common.d_partner_contact.Src_Sysname IS '来源系统'; + COMMENT ON COLUMN p30_common.d_partner_contact.Src_Table IS '来源表'; + +COMMENT ON TABLE p30_common.d_partner_contact IS 'partner one site联系人信息'; \ No newline at end of file diff --git a/dev/workflow/TK_Cust/partner1site/partner one site联系人信息/d_partner_contact_agi.sql b/dev/workflow/TK_Cust/partner1site/partner one site联系人信息/d_partner_contact_agi.sql new file mode 100644 index 0000000..2217f19 --- /dev/null +++ b/dev/workflow/TK_Cust/partner1site/partner one site联系人信息/d_partner_contact_agi.sql @@ -0,0 +1,199 @@ +/***************************************************************************************************/ +/*script in Sql, generate by SdmCreateScript 2024(by IdgValue, qihang.feng@idgvalue.COM) */ +/*VERSION 01.10 revised on 2020-08-25 */ +/*Brilliance stems from wisdoms. */ +/*************Head Section**************************************************************************/ +/*Script Use: Periodically load data to :d_partner_contact(partner one site联系人信息) */ +/*Create Date:2025-10-14 18:24:13 */ +/*SDM Developed By: dev */ +/*SDM Developed Date: 2025-10-14 */ +/*SDM Checked By: dev */ +/*SDM Checked Date: 2025-10-14 */ +/*Script Developed By: dev */ +/*Script Checked By: dev */ +/*Source table 1: :PDMDB.t01_partner_customer_and_contact */ +/*Job Type: Inbound transform (Tier 1 to Tier 2) */ +/*Target Table:d_partner_contact */ +/*ETL Job Name:d_partner_contact */ +/*ETL Frequency:None */ +/*ETL Policy:F2 */ +/********************************************************************************************/ +/*******Main Section**************************************************************************/ +\set ON_ERROR_STOP on +\set AUTOCOMMIT on +\timing on + + + +/*创建临时表加载当前数据 */ +CREATE TEMPORARY TABLE d_partner_contact_agi_CUR_I + ( LIKE :COMMDB.d_partner_contact) +ON COMMIT PRESERVE ROWS; + + + +/*创建临时表加载不同数据 */ +CREATE TEMPORARY TABLE d_partner_contact_agi_INS +( LIKE :COMMDB.d_partner_contact) +ON COMMIT PRESERVE ROWS; + + +/*****************************************************************************************************/ +/* GROUP 1:Source Table:t01_partner_customer_and_contact**********************************************/ +/*****************************************************************************************************/ +INSERT INTO d_partner_contact_agi_CUR_I ( + contact_id /*联系人ID*/ + ,contact_name /*联系人名称*/ + ,province /*省份*/ + ,city /*城市*/ + ,address /*地址*/ + ,department /*部门*/ + ,email /*邮箱*/ + ,phone /*电话*/ + ,Etl_Batch_No /*作业批次号*/ + ,Etl_First_Dt /*最初入库时间*/ + ,Etl_Job /*作业名称*/ + ,Etl_Proc_Dt /*本次入库时间*/ + ,Etl_Tx_Dt /*作业运行时间*/ + ,Src_Sysname /*来源系统*/ + ,Src_Table /*来源表*/ + ) +SELECT + COALESCE(p0.customer_contact_id,0) /*contact_id*/ + ,COALESCE(TRIM(p0.contact_name),'') /*contact_name*/ + ,COALESCE(TRIM(p0.province),'') /*province*/ + ,COALESCE(TRIM(p0.city),'') /*city*/ + ,COALESCE(TRIM(p0.address),'') /*address*/ + ,COALESCE(TRIM(p0.department),'') /*department*/ + ,COALESCE(TRIM(p0.email),'') /*email*/ + ,replace(replace(p0.phone_number,'+86',''),' ','') /*phone*/ + ,0 /*Etl_Batch_No*/ + ,TO_DATE(:TXDATE,'YYYYMMDD') /*Etl_First_Dt*/ + ,:ETLJOB /*Etl_Job*/ + ,current_timestamp(0) /*Etl_Proc_Dt*/ + ,TO_DATE(:TXDATE,'YYYYMMDD') /*Etl_Tx_Dt*/ + ,Substr('t01_partner_customer_and_contact',1,3) /*Src_Sysname*/ + ,'t01_partner_customer_and_contact' /*Src_Table*/ + +FROM :PDMDB.t01_partner_customer_and_contact p0 +; + + + +/*将不同数据插入到临时表 */ +;INSERT INTO d_partner_contact_agi_INS ( + contact_name /*联系人名称*/ + ,province /*省份*/ + ,city /*城市*/ + ,address /*地址*/ + ,department /*部门*/ + ,email /*邮箱*/ + ,phone /*电话*/ + ,contact_id /*联系人ID*/ + ,Etl_Batch_No /*作业批次号*/ + ,Etl_First_Dt /*最初入库时间*/ + ,Etl_Job /*作业名称*/ + ,Etl_Proc_Dt /*本次入库时间*/ + ,Etl_Tx_Dt /*作业运行时间*/ + ,Src_Sysname /*来源系统*/ + ,Src_Table /*来源表*/ + +) + SELECT + P1.contact_name /*联系人名称*/ + ,P1.province /*省份*/ + ,P1.city /*城市*/ + ,P1.address /*地址*/ + ,P1.department /*部门*/ + ,P1.email /*邮箱*/ + ,P1.phone /*电话*/ + ,P1.contact_id /*联系人ID*/ + ,P1.Etl_Batch_No /*作业批次号*/ + ,P1.Etl_First_Dt /*最初入库时间*/ + ,P1.Etl_Job /*作业名称*/ + ,P1.Etl_Proc_Dt /*本次入库时间*/ + ,P1.Etl_Tx_Dt /*作业运行时间*/ + ,P1.Src_Sysname /*来源系统*/ + ,P1.Src_Table /*来源表*/ + +FROM d_partner_contact_agi_CUR_I P1 +LEFT JOIN :COMMDB.d_partner_contact P2 +ON P1.contact_name = P2.contact_name + AND P1.province = P2.province + AND P1.city = P2.city + AND P1.address = P2.address + AND P1.department = P2.department + AND P1.email = P2.email + AND P1.phone = P2.phone + AND P1.contact_id = P2.contact_id + +WHERE P2.contact_name IS NULL + OR P2.province IS NULL + OR P2.city IS NULL + OR P2.address IS NULL + OR P2.department IS NULL + OR P2.email IS NULL + OR P2.phone IS NULL + OR P2.contact_id IS NULL + +; +/*将新增数据插入到目标表 */ +;INSERT INTO :COMMDB.d_partner_contact ( + contact_name /*联系人名称*/ + ,province /*省份*/ + ,city /*城市*/ + ,address /*地址*/ + ,department /*部门*/ + ,email /*邮箱*/ + ,phone /*电话*/ + ,contact_id /*联系人ID*/ + ,Etl_Batch_No /*作业批次号*/ + ,Etl_First_Dt /*最初入库时间*/ + ,Etl_Job /*作业名称*/ + ,Etl_Proc_Dt /*本次入库时间*/ + ,Etl_Tx_Dt /*作业运行时间*/ + ,Src_Sysname /*来源系统*/ + ,Src_Table /*来源表*/ + +) +SELECT + P1.contact_name /*联系人名称*/ + ,P1.province /*省份*/ + ,P1.city /*城市*/ + ,P1.address /*地址*/ + ,P1.department /*部门*/ + ,P1.email /*邮箱*/ + ,P1.phone /*电话*/ + ,P1.contact_id /*联系人ID*/ + ,P1.Etl_Batch_No /*作业批次号*/ + ,P1.Etl_First_Dt /*最初入库时间*/ + ,P1.Etl_Job /*作业名称*/ + ,P1.Etl_Proc_Dt /*本次入库时间*/ + ,P1.Etl_Tx_Dt /*作业运行时间*/ + ,P1.Src_Sysname /*来源系统*/ + ,P1.Src_Table /*来源表*/ + +FROM d_partner_contact_agi_INS P1 +ON CONFLICT ( contact_id) +DO UPDATE SET + contact_id=excluded.contact_id + ,contact_name=excluded.contact_name + ,province=excluded.province + ,city=excluded.city + ,address=excluded.address + ,department=excluded.department + ,email=excluded.email + ,phone=excluded.phone + ,Etl_Batch_No=excluded.Etl_Batch_No + ,Etl_First_Dt=excluded.Etl_First_Dt + ,Etl_Job=excluded.Etl_Job + ,Etl_Proc_Dt=excluded.Etl_Proc_Dt + ,Etl_Tx_Dt=excluded.Etl_Tx_Dt + ,Src_Sysname=excluded.Src_Sysname + ,Src_Table=excluded.Src_Table + + +; +/*****程序结束退出 */ +\q + diff --git a/dev/workflow/TK_Cust/partner1site/partner1site/wf_dag_partner1site.py b/dev/workflow/TK_Cust/partner1site/partner1site/wf_dag_partner1site.py index 46a7849..522e30e 100644 --- a/dev/workflow/TK_Cust/partner1site/partner1site/wf_dag_partner1site.py +++ b/dev/workflow/TK_Cust/partner1site/partner1site/wf_dag_partner1site.py @@ -182,6 +182,14 @@ params={'my_param':"t01_partner_visit_agi"}, depends_on_past=False, retries=3, dag=dag) +d_partner_contact = SSHOperator( +ssh_hook=sshHook, +task_id='d_partner_contact', +command='/data/airflow/etl/COM/run_psql.sh {{ ds_nodash }} {{params.my_param}} >>/data/airflow/logs/run_tpt_{{ds_nodash}}.log 2>&1 ', +params={'my_param':"d_partner_contact_agi"}, +depends_on_past=False, +retries=3, +dag=dag) part_summary_visit_load >> partner_summary_visit_9060 part_summary_report_load >> partner_summary_report_6257 part_summary_pos_load >> partner_summary_pos_4937 @@ -190,4 +198,5 @@ partner_summary_pos_4937 >> t01_partner_pos partner_summary_report_6257 >> t01_partner_report partner_summary_visit_9060 >> t01_partner_visit partner_summary_customer_5702 >> t01_partner_customer_and_contact -t01_partner_customer_and_contact >> task_failed +t01_partner_customer_and_contact >> d_partner_contact +d_partner_contact >> task_failed diff --git a/dev/workflow/TK_Cust/partner1site/partner_summary_visit/S98_S_partner_summary_visit.sql b/dev/workflow/TK_Cust/partner1site/partner_summary_visit/S98_S_partner_summary_visit.sql index d7216fa..6d4afe7 100644 --- a/dev/workflow/TK_Cust/partner1site/partner_summary_visit/S98_S_partner_summary_visit.sql +++ b/dev/workflow/TK_Cust/partner1site/partner_summary_visit/S98_S_partner_summary_visit.sql @@ -22,6 +22,7 @@ insert into p10_sa.S98_S_partner_summary_visit , customer_category_type , customer_name , dealer_name + , id , industry , insert_date , intention_product @@ -32,7 +33,6 @@ insert into p10_sa.S98_S_partner_summary_visit , tsm_names_by_alias , visit_remark , visitor - , id , etl_tx_dt ) select alias @@ -52,6 +52,7 @@ insert into p10_sa.S98_S_partner_summary_visit , customer_category_type , customer_name , dealer_name + , id , industry , insert_date , intention_product @@ -62,7 +63,6 @@ insert into p10_sa.S98_S_partner_summary_visit , tsm_names_by_alias , visit_remark , visitor - , id , etl_tx_dt from p00_tal.S98_S_partner_summary_visit ; @@ -87,6 +87,7 @@ insert into p12_sfull.S98_S_partner_summary_visit , customer_category_type , customer_name , dealer_name + , id , industry , insert_date , intention_product @@ -97,7 +98,6 @@ insert into p12_sfull.S98_S_partner_summary_visit , tsm_names_by_alias , visit_remark , visitor - , id , etl_tx_dt ) select alias @@ -117,6 +117,7 @@ insert into p12_sfull.S98_S_partner_summary_visit , customer_category_type , customer_name , dealer_name + , id , industry , insert_date , intention_product @@ -127,7 +128,6 @@ insert into p12_sfull.S98_S_partner_summary_visit , tsm_names_by_alias , visit_remark , visitor - , id , etl_tx_dt from p10_sa.S98_S_partner_summary_visit ; diff --git a/dev/workflow/TK_Cust/partner1site/partner_summary_visit/sa_foreign_tables.sql b/dev/workflow/TK_Cust/partner1site/partner_summary_visit/sa_foreign_tables.sql index 54d381f..40a4d7d 100644 --- a/dev/workflow/TK_Cust/partner1site/partner_summary_visit/sa_foreign_tables.sql +++ b/dev/workflow/TK_Cust/partner1site/partner_summary_visit/sa_foreign_tables.sql @@ -17,6 +17,7 @@ CREATE FOREIGN TABLE if not exists p00_tal.S98_S_partner_summary_visit ( , customer_category_type TEXT , customer_name TEXT , dealer_name TEXT + , id TEXT , industry TEXT , insert_date TEXT , intention_product TEXT @@ -27,7 +28,6 @@ CREATE FOREIGN TABLE if not exists p00_tal.S98_S_partner_summary_visit ( , tsm_names_by_alias TEXT , visit_remark TEXT , visitor TEXT - , id TEXT , etl_tx_dt TIMESTAMP ) diff --git a/dev/workflow/TK_Cust/partner1site/partner_summary_visit/sa_tables.sql b/dev/workflow/TK_Cust/partner1site/partner_summary_visit/sa_tables.sql index 5c486aa..b0fac01 100644 --- a/dev/workflow/TK_Cust/partner1site/partner_summary_visit/sa_tables.sql +++ b/dev/workflow/TK_Cust/partner1site/partner_summary_visit/sa_tables.sql @@ -17,6 +17,7 @@ create table if not exists p10_sa.S98_S_partner_summary_visit ( , customer_category_type TEXT , customer_name TEXT , dealer_name TEXT + , id TEXT , industry TEXT , insert_date TEXT , intention_product TEXT @@ -27,7 +28,6 @@ create table if not exists p10_sa.S98_S_partner_summary_visit ( , tsm_names_by_alias TEXT , visit_remark TEXT , visitor TEXT - , id TEXT , etl_tx_dt TIMESTAMP ) ; @@ -48,6 +48,7 @@ create table if not exists p10_sa.S98_S_partner_summary_visit ( COMMENT ON COLUMN p10_sa.S98_S_partner_summary_visit.customer_category_type IS ''; COMMENT ON COLUMN p10_sa.S98_S_partner_summary_visit.customer_name IS ''; COMMENT ON COLUMN p10_sa.S98_S_partner_summary_visit.dealer_name IS ''; + COMMENT ON COLUMN p10_sa.S98_S_partner_summary_visit.id IS ''; COMMENT ON COLUMN p10_sa.S98_S_partner_summary_visit.industry IS ''; COMMENT ON COLUMN p10_sa.S98_S_partner_summary_visit.insert_date IS ''; COMMENT ON COLUMN p10_sa.S98_S_partner_summary_visit.intention_product IS ''; @@ -58,7 +59,6 @@ create table if not exists p10_sa.S98_S_partner_summary_visit ( COMMENT ON COLUMN p10_sa.S98_S_partner_summary_visit.tsm_names_by_alias IS ''; COMMENT ON COLUMN p10_sa.S98_S_partner_summary_visit.visit_remark IS ''; COMMENT ON COLUMN p10_sa.S98_S_partner_summary_visit.visitor IS ''; - COMMENT ON COLUMN p10_sa.S98_S_partner_summary_visit.id IS ''; COMMENT ON COLUMN p10_sa.S98_S_partner_summary_visit.etl_tx_dt IS ''; COMMENT ON TABLE p10_sa.S98_S_partner_summary_visit IS ''; @@ -83,6 +83,7 @@ create table if not exists p12_sfull.S98_S_partner_summary_visit ( , customer_category_type TEXT , customer_name TEXT , dealer_name TEXT + , id TEXT , industry TEXT , insert_date TEXT , intention_product TEXT @@ -93,7 +94,6 @@ create table if not exists p12_sfull.S98_S_partner_summary_visit ( , tsm_names_by_alias TEXT , visit_remark TEXT , visitor TEXT - , id TEXT , etl_tx_dt TIMESTAMP ) ; @@ -114,6 +114,7 @@ create table if not exists p12_sfull.S98_S_partner_summary_visit ( COMMENT ON COLUMN p12_sfull.S98_S_partner_summary_visit.customer_category_type IS ''; COMMENT ON COLUMN p12_sfull.S98_S_partner_summary_visit.customer_name IS ''; COMMENT ON COLUMN p12_sfull.S98_S_partner_summary_visit.dealer_name IS ''; + COMMENT ON COLUMN p12_sfull.S98_S_partner_summary_visit.id IS ''; COMMENT ON COLUMN p12_sfull.S98_S_partner_summary_visit.industry IS ''; COMMENT ON COLUMN p12_sfull.S98_S_partner_summary_visit.insert_date IS ''; COMMENT ON COLUMN p12_sfull.S98_S_partner_summary_visit.intention_product IS ''; @@ -124,7 +125,6 @@ create table if not exists p12_sfull.S98_S_partner_summary_visit ( COMMENT ON COLUMN p12_sfull.S98_S_partner_summary_visit.tsm_names_by_alias IS ''; COMMENT ON COLUMN p12_sfull.S98_S_partner_summary_visit.visit_remark IS ''; COMMENT ON COLUMN p12_sfull.S98_S_partner_summary_visit.visitor IS ''; - COMMENT ON COLUMN p12_sfull.S98_S_partner_summary_visit.id IS ''; COMMENT ON COLUMN p12_sfull.S98_S_partner_summary_visit.etl_tx_dt IS ''; COMMENT ON TABLE p12_sfull.S98_S_partner_summary_visit IS ''; diff --git a/dev/workflow/TK_Cust/partner1site/获取客户拜访数据/part_summary_visit_feign.py b/dev/workflow/TK_Cust/partner1site/获取客户拜访数据/part_summary_visit_feign.py index 6ffc8f6..6ae1764 100644 --- a/dev/workflow/TK_Cust/partner1site/获取客户拜访数据/part_summary_visit_feign.py +++ b/dev/workflow/TK_Cust/partner1site/获取客户拜访数据/part_summary_visit_feign.py @@ -1,143 +1,142 @@ -#!/usr/bin/env python3 -# -*- coding: utf-8 -*- -""" -Partner1site 全接口抓取脚本 -分页结束条件：hasNext == False -""" - -import random -import hmac -import hashlib -import base64 -import requests -import json -import uuid -from datetime import datetime, timezone, timedelta -from typing import Dict, Any -import psycopg2 - -# ======= 配置区 ======= -ACCESS_KEY = "75c4ab4d-6a67-4aed-8b1d-5bb64fd36afc" -SECRET_KEY = "117347a7dd066a50a4d2973c5f3d5ba9101094c5" - -PG_DSN = dict( - database="dataops_db", - user="dbuser_dba", - password="EmBRxnmmjnE3", - host="124.221.232.219", - port="5432" -) - -API_ID = "2460976d-00c1-47d9-84b2-33e66d68" # 外部传入 api_id，占位符 -# ====================== - - -class Partner1SiteClient: - """Partner1site API 客户端""" - - def __init__(self, access_key: str, secret_key: str): - self.ak = access_key - self.sk = secret_key - - @staticmethod - def urlsafe_b64encode(data: bytes) -> str: - return base64.urlsafe_b64encode(data).decode() - - def gen_token(self, expire_sec: int = 600) -> str: - """生成 API Token""" - random_num = str(random.randint(100000, 999999)) - deadline = int(datetime.now(timezone.utc).timestamp()) + expire_sec - parm_str = f"{random_num}:{deadline}" - enc_parm = self.urlsafe_b64encode(parm_str.encode()) - sign = hmac.new(self.sk.encode(), enc_parm.encode(), hashlib.sha1).digest() - enc_sign = self.urlsafe_b64encode(sign.hex().encode()) - return f"{self.ak}:{enc_sign}:{enc_parm}" - - def fetch_all_pages(self, api_name: str, params: Dict[str, Any]): - """通用分页请求（结束条件：hasNext == False）""" - # if api_name not in BASE_URLS: - # raise ValueError(f"未知 API 数据来源: {api_name}") - - base_url = 'http://onesite.tek.cn/api/summary/visits' - all_data = [] - page_num = 0 - page_size = 1000 # 固定每页大小 - - while True: - token = self.gen_token() - params_with_paging = dict(params) - params_with_paging.update({ - "token": token, - "size": page_size, - "page": page_num - }) - - resp = requests.get(base_url, params=params_with_paging, timeout=120) - resp.raise_for_status() - data_json = resp.json() - - if data_json.get("code") != 100 or not data_json.get("success", False): - raise RuntimeError(f"获取客户拜访数据 API 错误: {data_json.get('message')}") - - content = data_json.get("data", {}).get("content", []) - all_data.extend(content) - total_elements = data_json.get("data", {}).get("totalElements") - has_next = data_json.get("data", {}).get("hasNext", False) - - print(f"[获取客户拜访数据] 页码 {page_num} -> 本页 {len(content)} 条，累计 {len(all_data)} 条 / 总数 {total_elements}") - - if not has_next: - break - - page_num += 1 - - return all_data - - -def save_json_to_pg(data: list, api_id: str) -> None: - """写入 PostgreSQL：软删历史 + 插入新数据""" - print(f"[save_to_pg] API={api_id} 写入 PG，记录数={len(data)}") - sql = """ - UPDATE data_api.api_data - SET is_loaded = '1' - WHERE api_id = %s; - - INSERT INTO data_api.api_data - (id, api_id, data, total_num, is_loaded, status, - request_tm, execute_tm, remark) - VALUES (%s, %s, %s, %s, '0', '0', - current_timestamp(0), current_timestamp(0), ''); - """ - try: - with psycopg2.connect(**PG_DSN) as conn: - with conn.cursor() as cur: - cur.execute(sql, - (api_id, - str(uuid.uuid4()), - api_id, - json.dumps(data, ensure_ascii=False), - len(data))) - conn.commit() - print(f"[save_to_pg] API={api_id} 写入完成") - except Exception as e: - raise RuntimeError(f"PG写入错误: {e}") - - -def get_previous_date(days: int = 0) -> str: - return (datetime.now() - timedelta(days=days)).strftime("%Y-%m-%d") - - -def main(): - client = Partner1SiteClient(ACCESS_KEY, SECRET_KEY) - - data = client.fetch_all_pages( - api_name="visits", - params={'startInsertDate':get_previous_date(7)} - # params={'startInsertDate':'2000-1-1'} - ) - save_json_to_pg(data, API_ID) - - - -if __name__ == "__main__": - main() +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +""" +Partner1site 全接口抓取脚本 +分页结束条件：hasNext == False +""" + +import random +import hmac +import hashlib +import base64 +import requests +import json +import uuid +from datetime import datetime, timezone, timedelta +from typing import Dict, Any +import psycopg2 + +# ======= 配置区 ======= +ACCESS_KEY = "75c4ab4d-6a67-4aed-8b1d-5bb64fd36afc" +SECRET_KEY = "117347a7dd066a50a4d2973c5f3d5ba9101094c5" + +PG_DSN = dict( + database="dataops_db", + user="dbuser_dba", + password="EmBRxnmmjnE3", + host="124.221.232.219", + port="5432" +) + +API_ID = "2460976d-00c1-47d9-84b2-33e66d68" # 外部传入 api_id，占位符 +# ====================== + + +class Partner1SiteClient: + """Partner1site API 客户端""" + + def __init__(self, access_key: str, secret_key: str): + self.ak = access_key + self.sk = secret_key + + @staticmethod + def urlsafe_b64encode(data: bytes) -> str: + return base64.urlsafe_b64encode(data).decode() + + def gen_token(self, expire_sec: int = 600) -> str: + """生成 API Token""" + random_num = str(random.randint(100000, 999999)) + deadline = int(datetime.now(timezone.utc).timestamp()) + expire_sec + parm_str = f"{random_num}:{deadline}" + enc_parm = self.urlsafe_b64encode(parm_str.encode()) + sign = hmac.new(self.sk.encode(), enc_parm.encode(), hashlib.sha1).digest() + enc_sign = self.urlsafe_b64encode(sign.hex().encode()) + return f"{self.ak}:{enc_sign}:{enc_parm}" + + def fetch_all_pages(self, api_name: str, params: Dict[str, Any]): + """通用分页请求（结束条件：hasNext == False）""" + # if api_name not in BASE_URLS: + # raise ValueError(f"未知 API 数据来源: {api_name}") + + base_url = 'http://onesite.tek.cn/api/summary/visits' + all_data = [] + page_num = 0 + page_size = 1000 # 固定每页大小 + + while True: + token = self.gen_token() + params_with_paging = dict(params) + params_with_paging.update({ + "token": token, + "size": page_size, + "page": page_num + }) + + resp = requests.get(base_url, params=params_with_paging, timeout=30) + resp.raise_for_status() + data_json = resp.json() + + if data_json.get("code") != 100 or not data_json.get("success", False): + raise RuntimeError(f"获取客户拜访数据 API 错误: {data_json.get('message')}") + + content = data_json.get("data", {}).get("content", []) + all_data.extend(content) + total_elements = data_json.get("data", {}).get("totalElements") + has_next = data_json.get("data", {}).get("hasNext", False) + + print(f"[获取客户拜访数据] 页码 {page_num} -> 本页 {len(content)} 条，累计 {len(all_data)} 条 / 总数 {total_elements}") + + if not has_next: + break + + page_num += 1 + + return all_data + + +def save_json_to_pg(data: list, api_id: str) -> None: + """写入 PostgreSQL：软删历史 + 插入新数据""" + print(f"[save_to_pg] API={api_id} 写入 PG，记录数={len(data)}") + sql = """ + UPDATE data_api.api_data + SET is_loaded = '1' + WHERE api_id = %s; + + INSERT INTO data_api.api_data + (id, api_id, data, total_num, is_loaded, status, + request_tm, execute_tm, remark) + VALUES (%s, %s, %s, %s, '0', '0', + current_timestamp(0), current_timestamp(0), ''); + """ + try: + with psycopg2.connect(**PG_DSN) as conn: + with conn.cursor() as cur: + cur.execute(sql, + (api_id, + str(uuid.uuid4()), + api_id, + json.dumps(data, ensure_ascii=False), + len(data))) + conn.commit() + print(f"[save_to_pg] API={api_id} 写入完成") + except Exception as e: + raise RuntimeError(f"PG写入错误: {e}") + + +def get_previous_date(days: int = 0) -> str: + return (datetime.now() - timedelta(days=days)).strftime("%Y-%m-%d") + + +def main(): + client = Partner1SiteClient(ACCESS_KEY, SECRET_KEY) + + data = client.fetch_all_pages( + api_name="visits", + params={'startInsertDate':'2025-8-20',} # 拉全量，若有默认时间限制可改成 {"startInsertDate":"2000-01-01","endInsertDate":get_previous_date(0)} + ) + save_json_to_pg(data, API_ID) + + + +if __name__ == "__main__": + main() diff --git a/dev/workflow/TK_Cust/partner1site/获取客户拜访数据/part_summary_visit_load.sql b/dev/workflow/TK_Cust/partner1site/获取客户拜访数据/part_summary_visit_load.sql index c7f0f35..e71e239 100644 --- a/dev/workflow/TK_Cust/partner1site/获取客户拜访数据/part_summary_visit_load.sql +++ b/dev/workflow/TK_Cust/partner1site/获取客户拜访数据/part_summary_visit_load.sql @@ -21,8 +21,11 @@ insert into data_api.partner_summary_visit ( , contact_ways , customer_category , customer_category_type + , customer_contact_id + , customer_id , customer_name , dealer_name + , id , industry , insert_date , intention_product @@ -32,8 +35,7 @@ insert into data_api.partner_summary_visit ( , sub_industry , tsm_names_by_alias , visit_remark - , visitor - , id + , visitor ,etl_tx_dt ) select @@ -52,8 +54,11 @@ select , case when trim(both from contact_ways)='' then null else contact_ways::text end contact_ways , case when trim(both from customer_category)='' then null else customer_category::text end customer_category , case when trim(both from customer_category_type)='' then null else customer_category_type::text end customer_category_type + , case when trim(both from customer_contact_id)='' then null else customer_contact_id::text end customer_contact_id + , case when trim(both from customer_id)='' then null else customer_id::text end customer_id , case when trim(both from customer_name)='' then null else customer_name::text end customer_name , case when trim(both from dealer_name)='' then null else dealer_name::text end dealer_name + , case when trim(both from id)='' then null else id::text end id , case when trim(both from industry)='' then null else industry::text end industry , case when trim(both from insert_date)='' then null else insert_date::text end insert_date , case when trim(both from intention_product)='' then null else intention_product::text end intention_product @@ -63,8 +68,7 @@ select , case when trim(both from sub_industry)='' then null else sub_industry::text end sub_industry , case when trim(both from tsm_names_by_alias)='' then null else tsm_names_by_alias::text end tsm_names_by_alias , case when trim(both from visit_remark)='' then null else visit_remark::text end visit_remark - , case when trim(both from visitor)='' then null else visitor::text end visitor - , case when trim(both from id)='' then null else id::text end id + , case when trim(both from visitor)='' then null else visitor::text end visitor ,etl_tx_dt from ( select @@ -83,8 +87,11 @@ select , (json_array_elements(data::json)::json->>'contactWays') contact_ways , (json_array_elements(data::json)::json->>'customerCategory') customer_category , (json_array_elements(data::json)::json->>'customerCategoryType') customer_category_type + , (json_array_elements(data::json)::json->>'customerContactId') customer_contact_id + , (json_array_elements(data::json)::json->>'customerId') customer_id , (json_array_elements(data::json)::json->>'customerName') customer_name , (json_array_elements(data::json)::json->>'dealerName') dealer_name + , (json_array_elements(data::json)::json->>'id') id , (json_array_elements(data::json)::json->>'industry') industry , (json_array_elements(data::json)::json->>'insertDate') insert_date , (json_array_elements(data::json)::json->>'intentionProduct') intention_product @@ -94,8 +101,7 @@ select , (json_array_elements(data::json)::json->>'subIndustry') sub_industry , (json_array_elements(data::json)::json->>'tsmNamesByAlias') tsm_names_by_alias , (json_array_elements(data::json)::json->>'visitRemark') visit_remark - , (json_array_elements(data::json)::json->>'visitor') visitor - , (json_array_elements(data::json)::json->>'id') id + , (json_array_elements(data::json)::json->>'visitor') visitor ,CURRENT_TIMESTAMP(0) etl_tx_dt from (select * from data_api.api_data WHERE api_id='2460976d-00c1-47d9-84b2-33e66d68' and is_loaded = '0' order by request_tm desc limit 1) p )p;