dtgitea
/
dataops
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

add workflow 荟聚API_1,dev

This commit is contained in:
root 2024-04-11 11:05:30 +08:00
parent 92351441a4
commit ce205d50ff
5 changed files with 100 additions and 40 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

44
dev/workflow/TK_Cust/tk_api_1/scrm_contact_merge/S98_S_scrm_contact_merge.sql
View File

@ -5,8 +5,7 @@
delete from p10_sa.S98_S_scrm_contact_merge delete from p10_sa.S98_S_scrm_contact_merge
; ;
insert into p10_sa.S98_S_scrm_contact_merge insert into p10_sa.S98_S_scrm_contact_merge
( mergedcustomerid ( merged_customer_id
, id
, customer_id , customer_id
, date , date
, event , event
@ -14,15 +13,20 @@ insert into p10_sa.S98_S_scrm_contact_merge
, last_updated , last_updated
, c_name , c_name
, c_type , c_type
, contentname , content_name
, source , source
, tag , tag
, c_keyword , c_keyword
, attr2 , attr2
, id
, customer_id_str
, external_id
, id_str
, merged_customer_name
, target_id
, etl_tx_dt ) , etl_tx_dt )
select select
mergedcustomerid merged_customer_id
, id
, customer_id , customer_id
, date , date
, event , event
@ -30,11 +34,17 @@ insert into p10_sa.S98_S_scrm_contact_merge
, last_updated , last_updated
, c_name , c_name
, c_type , c_type
, contentname , content_name
, source , source
, tag , tag
, c_keyword , c_keyword
, attr2 , attr2
, id
, customer_id_str
, external_id
, id_str
, merged_customer_name
, target_id
, etl_tx_dt , etl_tx_dt
from p00_tal.S98_S_scrm_contact_merge from p00_tal.S98_S_scrm_contact_merge
; ;
@ -42,8 +52,7 @@ insert into p10_sa.S98_S_scrm_contact_merge
; ;
; ;
insert into p12_sfull.S98_S_scrm_contact_merge insert into p12_sfull.S98_S_scrm_contact_merge
( mergedcustomerid ( merged_customer_id
, id
, customer_id , customer_id
, date , date
, event , event
@ -51,15 +60,20 @@ insert into p12_sfull.S98_S_scrm_contact_merge
, last_updated , last_updated
, c_name , c_name
, c_type , c_type
, contentname , content_name
, source , source
, tag , tag
, c_keyword , c_keyword
, attr2 , attr2
, id
, customer_id_str
, external_id
, id_str
, merged_customer_name
, target_id
, etl_tx_dt ) , etl_tx_dt )
select select
mergedcustomerid merged_customer_id
, id
, customer_id , customer_id
, date , date
, event , event
@ -67,11 +81,17 @@ insert into p12_sfull.S98_S_scrm_contact_merge
, last_updated , last_updated
, c_name , c_name
, c_type , c_type
, contentname , content_name
, source , source
, tag , tag
, c_keyword , c_keyword
, attr2 , attr2
, id
, customer_id_str
, external_id
, id_str
, merged_customer_name
, target_id
, etl_tx_dt , etl_tx_dt
from p10_sa.S98_S_scrm_contact_merge from p10_sa.S98_S_scrm_contact_merge
; ;

11
dev/workflow/TK_Cust/tk_api_1/scrm_contact_merge/sa_foreign_tables.sql
View File

@ -1,7 +1,6 @@
CREATE FOREIGN TABLE if not exists p00_tal.S98_S_scrm_contact_merge ( CREATE FOREIGN TABLE if not exists p00_tal.S98_S_scrm_contact_merge (
mergedcustomerid TEXT merged_customer_id TEXT
, id TEXT
, customer_id TEXT , customer_id TEXT
, date TEXT , date TEXT
, event TEXT , event TEXT
@ -9,11 +8,17 @@ CREATE FOREIGN TABLE if not exists p00_tal.S98_S_scrm_contact_merge (
, last_updated TEXT , last_updated TEXT
, c_name TEXT , c_name TEXT
, c_type TEXT , c_type TEXT
, contentname TEXT , content_name TEXT
, source TEXT , source TEXT
, tag TEXT , tag TEXT
, c_keyword TEXT , c_keyword TEXT
, attr2 TEXT , attr2 TEXT
, id TEXT
, customer_id_str TEXT
, external_id TEXT
, id_str TEXT
, merged_customer_name TEXT
, target_id TEXT
, etl_tx_dt TIMESTAMP , etl_tx_dt TIMESTAMP
) )

44
dev/workflow/TK_Cust/tk_api_1/scrm_contact_merge/sa_tables.sql
View File

@ -1,7 +1,6 @@
create table if not exists p10_sa.S98_S_scrm_contact_merge ( create table if not exists p10_sa.S98_S_scrm_contact_merge (
mergedcustomerid TEXT merged_customer_id TEXT
, id TEXT
, customer_id TEXT , customer_id TEXT
, date TEXT , date TEXT
, event TEXT , event TEXT
@ -9,16 +8,21 @@ create table if not exists p10_sa.S98_S_scrm_contact_merge (
, last_updated TEXT , last_updated TEXT
, c_name TEXT , c_name TEXT
, c_type TEXT , c_type TEXT
, contentname TEXT , content_name TEXT
, source TEXT , source TEXT
, tag TEXT , tag TEXT
, c_keyword TEXT , c_keyword TEXT
, attr2 TEXT , attr2 TEXT
, id TEXT
, customer_id_str TEXT
, external_id TEXT
, id_str TEXT
, merged_customer_name TEXT
, target_id TEXT
, etl_tx_dt TIMESTAMP , etl_tx_dt TIMESTAMP
) ; ) ;
COMMENT ON COLUMN p10_sa.S98_S_scrm_contact_merge.mergedcustomerid IS ''; COMMENT ON COLUMN p10_sa.S98_S_scrm_contact_merge.merged_customer_id IS '';
COMMENT ON COLUMN p10_sa.S98_S_scrm_contact_merge.id IS '';
COMMENT ON COLUMN p10_sa.S98_S_scrm_contact_merge.customer_id IS ''; COMMENT ON COLUMN p10_sa.S98_S_scrm_contact_merge.customer_id IS '';
COMMENT ON COLUMN p10_sa.S98_S_scrm_contact_merge.date IS ''; COMMENT ON COLUMN p10_sa.S98_S_scrm_contact_merge.date IS '';
COMMENT ON COLUMN p10_sa.S98_S_scrm_contact_merge.event IS ''; COMMENT ON COLUMN p10_sa.S98_S_scrm_contact_merge.event IS '';
@ -26,11 +30,17 @@ create table if not exists p10_sa.S98_S_scrm_contact_merge (
COMMENT ON COLUMN p10_sa.S98_S_scrm_contact_merge.last_updated IS ''; COMMENT ON COLUMN p10_sa.S98_S_scrm_contact_merge.last_updated IS '';
COMMENT ON COLUMN p10_sa.S98_S_scrm_contact_merge.c_name IS ''; COMMENT ON COLUMN p10_sa.S98_S_scrm_contact_merge.c_name IS '';
COMMENT ON COLUMN p10_sa.S98_S_scrm_contact_merge.c_type IS ''; COMMENT ON COLUMN p10_sa.S98_S_scrm_contact_merge.c_type IS '';
COMMENT ON COLUMN p10_sa.S98_S_scrm_contact_merge.contentname IS ''; COMMENT ON COLUMN p10_sa.S98_S_scrm_contact_merge.content_name IS '';
COMMENT ON COLUMN p10_sa.S98_S_scrm_contact_merge.source IS ''; COMMENT ON COLUMN p10_sa.S98_S_scrm_contact_merge.source IS '';
COMMENT ON COLUMN p10_sa.S98_S_scrm_contact_merge.tag IS ''; COMMENT ON COLUMN p10_sa.S98_S_scrm_contact_merge.tag IS '';
COMMENT ON COLUMN p10_sa.S98_S_scrm_contact_merge.c_keyword IS ''; COMMENT ON COLUMN p10_sa.S98_S_scrm_contact_merge.c_keyword IS '';
COMMENT ON COLUMN p10_sa.S98_S_scrm_contact_merge.attr2 IS ''; COMMENT ON COLUMN p10_sa.S98_S_scrm_contact_merge.attr2 IS '';
COMMENT ON COLUMN p10_sa.S98_S_scrm_contact_merge.id IS '';
COMMENT ON COLUMN p10_sa.S98_S_scrm_contact_merge.customer_id_str IS '';
COMMENT ON COLUMN p10_sa.S98_S_scrm_contact_merge.external_id IS '';
COMMENT ON COLUMN p10_sa.S98_S_scrm_contact_merge.id_str IS '';
COMMENT ON COLUMN p10_sa.S98_S_scrm_contact_merge.merged_customer_name IS '';
COMMENT ON COLUMN p10_sa.S98_S_scrm_contact_merge.target_id IS '';
COMMENT ON COLUMN p10_sa.S98_S_scrm_contact_merge.etl_tx_dt IS ''; COMMENT ON COLUMN p10_sa.S98_S_scrm_contact_merge.etl_tx_dt IS '';
COMMENT ON TABLE p10_sa.S98_S_scrm_contact_merge IS ''; COMMENT ON TABLE p10_sa.S98_S_scrm_contact_merge IS '';
@ -38,8 +48,7 @@ COMMENT ON TABLE p10_sa.S98_S_scrm_contact_merge IS '';
create table if not exists p12_sfull.S98_S_scrm_contact_merge ( create table if not exists p12_sfull.S98_S_scrm_contact_merge (
mergedcustomerid TEXT merged_customer_id TEXT
, id TEXT
, customer_id TEXT , customer_id TEXT
, date TEXT , date TEXT
, event TEXT , event TEXT
@ -47,16 +56,21 @@ create table if not exists p12_sfull.S98_S_scrm_contact_merge (
, last_updated TEXT , last_updated TEXT
, c_name TEXT , c_name TEXT
, c_type TEXT , c_type TEXT
, contentname TEXT , content_name TEXT
, source TEXT , source TEXT
, tag TEXT , tag TEXT
, c_keyword TEXT , c_keyword TEXT
, attr2 TEXT , attr2 TEXT
, id TEXT
, customer_id_str TEXT
, external_id TEXT
, id_str TEXT
, merged_customer_name TEXT
, target_id TEXT
, etl_tx_dt TIMESTAMP , etl_tx_dt TIMESTAMP
) ; ) ;
COMMENT ON COLUMN p12_sfull.S98_S_scrm_contact_merge.mergedcustomerid IS ''; COMMENT ON COLUMN p12_sfull.S98_S_scrm_contact_merge.merged_customer_id IS '';
COMMENT ON COLUMN p12_sfull.S98_S_scrm_contact_merge.id IS '';
COMMENT ON COLUMN p12_sfull.S98_S_scrm_contact_merge.customer_id IS ''; COMMENT ON COLUMN p12_sfull.S98_S_scrm_contact_merge.customer_id IS '';
COMMENT ON COLUMN p12_sfull.S98_S_scrm_contact_merge.date IS ''; COMMENT ON COLUMN p12_sfull.S98_S_scrm_contact_merge.date IS '';
COMMENT ON COLUMN p12_sfull.S98_S_scrm_contact_merge.event IS ''; COMMENT ON COLUMN p12_sfull.S98_S_scrm_contact_merge.event IS '';
@ -64,11 +78,17 @@ create table if not exists p12_sfull.S98_S_scrm_contact_merge (
COMMENT ON COLUMN p12_sfull.S98_S_scrm_contact_merge.last_updated IS ''; COMMENT ON COLUMN p12_sfull.S98_S_scrm_contact_merge.last_updated IS '';
COMMENT ON COLUMN p12_sfull.S98_S_scrm_contact_merge.c_name IS ''; COMMENT ON COLUMN p12_sfull.S98_S_scrm_contact_merge.c_name IS '';
COMMENT ON COLUMN p12_sfull.S98_S_scrm_contact_merge.c_type IS ''; COMMENT ON COLUMN p12_sfull.S98_S_scrm_contact_merge.c_type IS '';
COMMENT ON COLUMN p12_sfull.S98_S_scrm_contact_merge.contentname IS ''; COMMENT ON COLUMN p12_sfull.S98_S_scrm_contact_merge.content_name IS '';
COMMENT ON COLUMN p12_sfull.S98_S_scrm_contact_merge.source IS ''; COMMENT ON COLUMN p12_sfull.S98_S_scrm_contact_merge.source IS '';
COMMENT ON COLUMN p12_sfull.S98_S_scrm_contact_merge.tag IS ''; COMMENT ON COLUMN p12_sfull.S98_S_scrm_contact_merge.tag IS '';
COMMENT ON COLUMN p12_sfull.S98_S_scrm_contact_merge.c_keyword IS ''; COMMENT ON COLUMN p12_sfull.S98_S_scrm_contact_merge.c_keyword IS '';
COMMENT ON COLUMN p12_sfull.S98_S_scrm_contact_merge.attr2 IS ''; COMMENT ON COLUMN p12_sfull.S98_S_scrm_contact_merge.attr2 IS '';
COMMENT ON COLUMN p12_sfull.S98_S_scrm_contact_merge.id IS '';
COMMENT ON COLUMN p12_sfull.S98_S_scrm_contact_merge.customer_id_str IS '';
COMMENT ON COLUMN p12_sfull.S98_S_scrm_contact_merge.external_id IS '';
COMMENT ON COLUMN p12_sfull.S98_S_scrm_contact_merge.id_str IS '';
COMMENT ON COLUMN p12_sfull.S98_S_scrm_contact_merge.merged_customer_name IS '';
COMMENT ON COLUMN p12_sfull.S98_S_scrm_contact_merge.target_id IS '';
COMMENT ON COLUMN p12_sfull.S98_S_scrm_contact_merge.etl_tx_dt IS ''; COMMENT ON COLUMN p12_sfull.S98_S_scrm_contact_merge.etl_tx_dt IS '';
COMMENT ON TABLE p12_sfull.S98_S_scrm_contact_merge IS ''; COMMENT ON TABLE p12_sfull.S98_S_scrm_contact_merge IS '';

2
dev/workflow/TK_Cust/tk_api_1/合并SCRM CONTACT/custom_events_update_feign.py
View File

@ -51,7 +51,7 @@ while 'error' in auth and i < 60:
print('开始请求数据总数。') print('开始请求数据总数。')
url='https://api.huiju.cool/v2/customerEvents' url='https://api.huiju.cool/v2/customerEvents'
header={} header={}
body={'access_token':auth['access_token'],'limit':'','sort':'lastUpdated','lastUpdated[le]':formatted_current_date,'lastUpdated[ge]':formatted_previous_date,} body={'access_token':auth['access_token'],'limit':'','event':'merge_customer','sort':'lastUpdated','lastUpdated[le]':formatted_current_date,'lastUpdated[ge]':formatted_previous_date,}
dataReqL=requests.get(url,headers=header,params=body) dataReqL=requests.get(url,headers=header,params=body)
resL=json.loads(dataReqL.text) resL=json.loads(dataReqL.text)
# print(resL) # print(resL)

39
dev/workflow/TK_Cust/tk_api_1/合并SCRM CONTACT/custom_events_update_load.sql
View File

@ -6,8 +6,7 @@
DELETE FROM data_api.scrm_contact_merge; DELETE FROM data_api.scrm_contact_merge;
insert into data_api.scrm_contact_merge ( insert into data_api.scrm_contact_merge (
mergedCustomerId merged_customer_id
, id
, customer_id , customer_id
, date , date
, event , event
@ -15,16 +14,21 @@ insert into data_api.scrm_contact_merge (
, last_updated , last_updated
, c_name , c_name
, c_type , c_type
, contentName , content_name
, source , source
, tag , tag
, c_keyword , c_keyword
, attr2 , attr2
, id
, customer_id_str
, external_id
, id_str
, merged_customer_name
, target_id
,etl_tx_dt ,etl_tx_dt
) )
select select
case when trim(both from mergedCustomerId)='' then null else mergedCustomerId::text end mergedCustomerId case when trim(both from merged_customer_id)='' then null else merged_customer_id::text end merged_customer_id
, case when trim(both from id)='' then null else id::text end id
, case when trim(both from customer_id)='' then null else customer_id::text end customer_id , case when trim(both from customer_id)='' then null else customer_id::text end customer_id
, case when trim(both from date)='' then null else date::text end date , case when trim(both from date)='' then null else date::text end date
, case when trim(both from event)='' then null else event::text end event , case when trim(both from event)='' then null else event::text end event
@ -32,16 +36,21 @@ select
, case when trim(both from last_updated)='' then null else last_updated::text end last_updated , case when trim(both from last_updated)='' then null else last_updated::text end last_updated
, case when trim(both from c_name)='' then null else c_name::text end c_name , case when trim(both from c_name)='' then null else c_name::text end c_name
, case when trim(both from c_type)='' then null else c_type::text end c_type , case when trim(both from c_type)='' then null else c_type::text end c_type
, case when trim(both from contentName)='' then null else contentName::text end contentName , case when trim(both from content_name)='' then null else content_name::text end content_name
, case when trim(both from source)='' then null else source::text end source , case when trim(both from source)='' then null else source::text end source
, case when trim(both from tag)='' then null else tag::text end tag , case when trim(both from tag)='' then null else tag::text end tag
, case when trim(both from c_keyword)='' then null else c_keyword::text end c_keyword , case when trim(both from c_keyword)='' then null else c_keyword::text end c_keyword
, case when trim(both from attr2)='' then null else attr2::text end attr2 , case when trim(both from attr2)='' then null else attr2::text end attr2
, case when trim(both from id)='' then null else id::text end id
, case when trim(both from customer_id_str)='' then null else customer_id_str::text end customer_id_str
, case when trim(both from external_id)='' then null else external_id::text end external_id
, case when trim(both from id_str)='' then null else id_str::text end id_str
, case when trim(both from merged_customer_name)='' then null else merged_customer_name::text end merged_customer_name
, case when trim(both from target_id)='' then null else target_id::text end target_id
,etl_tx_dt ,etl_tx_dt
from ( from (
select select
(json_array_elements(data::json)::json->>'mergedCustomerId') mergedCustomerId (json_array_elements(data::json)::json->>'mergedCustomerId') merged_customer_id
, (json_array_elements(data::json)::json->>'id') id
, (json_array_elements(data::json)::json->>'customerId') customer_id , (json_array_elements(data::json)::json->>'customerId') customer_id
, (json_array_elements(data::json)::json->>'date') date , (json_array_elements(data::json)::json->>'date') date
, (json_array_elements(data::json)::json->>'event') event , (json_array_elements(data::json)::json->>'event') event
@ -49,11 +58,17 @@ select
, (json_array_elements(data::json)::json->>'lastUpdated') last_updated , (json_array_elements(data::json)::json->>'lastUpdated') last_updated
, (json_array_elements(data::json)::json->>'c_name') c_name , (json_array_elements(data::json)::json->>'c_name') c_name
, (json_array_elements(data::json)::json->>'c_type') c_type , (json_array_elements(data::json)::json->>'c_type') c_type
, (json_array_elements(data::json)::json->>'contentName') contentName , (json_array_elements(data::json)::json->>'contentName') content_name
, (json_array_elements(data::json)::json->>'source') source , (json_array_elements(data::json)::json->>'source') source
, (json_array_elements(data::json)::json->>'tag') tag , (json_array_elements(data::json)::json->>'tag') tag
, (json_array_elements(data::json)::json->>'c_keyword') c_keyword , (json_array_elements(data::json)::json->>'c_keyword') c_keyword
, (json_array_elements(data::json)::json->>'attr2') attr2 , (json_array_elements(data::json)::json->>'attr2') attr2
, (json_array_elements(data::json)::json->>'id') id
, (json_array_elements(data::json)::json->>'customerIdStr') customer_id_str
, (json_array_elements(data::json)::json->>'externalId') external_id
, (json_array_elements(data::json)::json->>'idStr') id_str
, (json_array_elements(data::json)::json->>'mergedCustomerName') merged_customer_name
, (json_array_elements(data::json)::json->>'targetId') target_id
,CURRENT_TIMESTAMP(0) etl_tx_dt ,CURRENT_TIMESTAMP(0) etl_tx_dt
from (select * from data_api.api_data from (select * from data_api.api_data
WHERE api_id='ff5c7bf6-0d18-4201-9501-8fdd6152' and is_loaded = '0' order by request_tm desc limit 1) p )p; WHERE api_id='ff5c7bf6-0d18-4201-9501-8fdd6152' and is_loaded = '0' order by request_tm desc limit 1) p )p;