dtgitea
/
dataops
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

add workflow 荟聚API_1,dev

This commit is contained in:
root 2024-04-11 11:05:30 +08:00
parent 92351441a4
commit ce205d50ff
5 changed files with 100 additions and 40 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

44
dev/workflow/TK_Cust/tk_api_1/scrm_contact_merge/S98_S_scrm_contact_merge.sql
View File

@ -5,8 +5,7 @@
delete from p10_sa.S98_S_scrm_contact_merge
;
insert into p10_sa.S98_S_scrm_contact_merge
( mergedcustomerid
, id
( merged_customer_id
, customer_id
, date
, event
@ -14,15 +13,20 @@ insert into p10_sa.S98_S_scrm_contact_merge
, last_updated
, c_name
, c_type
, contentname
, content_name
, source
, tag
, c_keyword
, attr2
, id
, customer_id_str
, external_id
, id_str
, merged_customer_name
, target_id
, etl_tx_dt )
select
mergedcustomerid
, id
merged_customer_id
, customer_id
, date
, event
@ -30,11 +34,17 @@ insert into p10_sa.S98_S_scrm_contact_merge
, last_updated
, c_name
, c_type
, contentname
, content_name
, source
, tag
, c_keyword
, attr2
, id
, customer_id_str
, external_id
, id_str
, merged_customer_name
, target_id
, etl_tx_dt
from p00_tal.S98_S_scrm_contact_merge
;
@ -42,8 +52,7 @@ insert into p10_sa.S98_S_scrm_contact_merge
;
;
insert into p12_sfull.S98_S_scrm_contact_merge
( mergedcustomerid
, id
( merged_customer_id
, customer_id
, date
, event
@ -51,15 +60,20 @@ insert into p12_sfull.S98_S_scrm_contact_merge
, last_updated
, c_name
, c_type
, contentname
, content_name
, source
, tag
, c_keyword
, attr2
, id
, customer_id_str
, external_id
, id_str
, merged_customer_name
, target_id
, etl_tx_dt )
select
mergedcustomerid
, id
merged_customer_id
, customer_id
, date
, event
@ -67,11 +81,17 @@ insert into p12_sfull.S98_S_scrm_contact_merge
, last_updated
, c_name
, c_type
, contentname
, content_name
, source
, tag
, c_keyword
, attr2
, id
, customer_id_str
, external_id
, id_str
, merged_customer_name
, target_id
, etl_tx_dt
from p10_sa.S98_S_scrm_contact_merge
;

11
dev/workflow/TK_Cust/tk_api_1/scrm_contact_merge/sa_foreign_tables.sql
View File

@ -1,7 +1,6 @@
CREATE FOREIGN TABLE if not exists p00_tal.S98_S_scrm_contact_merge (
mergedcustomerid TEXT
, id TEXT
merged_customer_id TEXT
, customer_id TEXT
, date TEXT
, event TEXT
@ -9,11 +8,17 @@ CREATE FOREIGN TABLE if not exists p00_tal.S98_S_scrm_contact_merge (
, last_updated TEXT
, c_name TEXT
, c_type TEXT
, contentname TEXT
, content_name TEXT
, source TEXT
, tag TEXT
, c_keyword TEXT
, attr2 TEXT
, id TEXT
, customer_id_str TEXT
, external_id TEXT
, id_str TEXT
, merged_customer_name TEXT
, target_id TEXT
, etl_tx_dt TIMESTAMP
)

44
dev/workflow/TK_Cust/tk_api_1/scrm_contact_merge/sa_tables.sql
View File

@ -1,7 +1,6 @@
create table if not exists p10_sa.S98_S_scrm_contact_merge (
mergedcustomerid TEXT
, id TEXT
merged_customer_id TEXT
, customer_id TEXT
, date TEXT
, event TEXT
@ -9,16 +8,21 @@ create table if not exists p10_sa.S98_S_scrm_contact_merge (
, last_updated TEXT
, c_name TEXT
, c_type TEXT
, contentname TEXT
, content_name TEXT
, source TEXT
, tag TEXT
, c_keyword TEXT
, attr2 TEXT
, id TEXT
, customer_id_str TEXT
, external_id TEXT
, id_str TEXT
, merged_customer_name TEXT
, target_id TEXT
, etl_tx_dt TIMESTAMP
) ;
COMMENT ON COLUMN p10_sa.S98_S_scrm_contact_merge.mergedcustomerid IS '';
COMMENT ON COLUMN p10_sa.S98_S_scrm_contact_merge.id IS '';
COMMENT ON COLUMN p10_sa.S98_S_scrm_contact_merge.merged_customer_id IS '';
COMMENT ON COLUMN p10_sa.S98_S_scrm_contact_merge.customer_id IS '';
COMMENT ON COLUMN p10_sa.S98_S_scrm_contact_merge.date IS '';
COMMENT ON COLUMN p10_sa.S98_S_scrm_contact_merge.event IS '';
@ -26,11 +30,17 @@ create table if not exists p10_sa.S98_S_scrm_contact_merge (
COMMENT ON COLUMN p10_sa.S98_S_scrm_contact_merge.last_updated IS '';
COMMENT ON COLUMN p10_sa.S98_S_scrm_contact_merge.c_name IS '';
COMMENT ON COLUMN p10_sa.S98_S_scrm_contact_merge.c_type IS '';
COMMENT ON COLUMN p10_sa.S98_S_scrm_contact_merge.contentname IS '';
COMMENT ON COLUMN p10_sa.S98_S_scrm_contact_merge.content_name IS '';
COMMENT ON COLUMN p10_sa.S98_S_scrm_contact_merge.source IS '';
COMMENT ON COLUMN p10_sa.S98_S_scrm_contact_merge.tag IS '';
COMMENT ON COLUMN p10_sa.S98_S_scrm_contact_merge.c_keyword IS '';
COMMENT ON COLUMN p10_sa.S98_S_scrm_contact_merge.attr2 IS '';
COMMENT ON COLUMN p10_sa.S98_S_scrm_contact_merge.id IS '';
COMMENT ON COLUMN p10_sa.S98_S_scrm_contact_merge.customer_id_str IS '';
COMMENT ON COLUMN p10_sa.S98_S_scrm_contact_merge.external_id IS '';
COMMENT ON COLUMN p10_sa.S98_S_scrm_contact_merge.id_str IS '';
COMMENT ON COLUMN p10_sa.S98_S_scrm_contact_merge.merged_customer_name IS '';
COMMENT ON COLUMN p10_sa.S98_S_scrm_contact_merge.target_id IS '';
COMMENT ON COLUMN p10_sa.S98_S_scrm_contact_merge.etl_tx_dt IS '';
COMMENT ON TABLE p10_sa.S98_S_scrm_contact_merge IS '';
@ -38,8 +48,7 @@ COMMENT ON TABLE p10_sa.S98_S_scrm_contact_merge IS '';
create table if not exists p12_sfull.S98_S_scrm_contact_merge (
mergedcustomerid TEXT
, id TEXT
merged_customer_id TEXT
, customer_id TEXT
, date TEXT
, event TEXT
@ -47,16 +56,21 @@ create table if not exists p12_sfull.S98_S_scrm_contact_merge (
, last_updated TEXT
, c_name TEXT
, c_type TEXT
, contentname TEXT
, content_name TEXT
, source TEXT
, tag TEXT
, c_keyword TEXT
, attr2 TEXT
, id TEXT
, customer_id_str TEXT
, external_id TEXT
, id_str TEXT
, merged_customer_name TEXT
, target_id TEXT
, etl_tx_dt TIMESTAMP
) ;
COMMENT ON COLUMN p12_sfull.S98_S_scrm_contact_merge.mergedcustomerid IS '';
COMMENT ON COLUMN p12_sfull.S98_S_scrm_contact_merge.id IS '';
COMMENT ON COLUMN p12_sfull.S98_S_scrm_contact_merge.merged_customer_id IS '';
COMMENT ON COLUMN p12_sfull.S98_S_scrm_contact_merge.customer_id IS '';
COMMENT ON COLUMN p12_sfull.S98_S_scrm_contact_merge.date IS '';
COMMENT ON COLUMN p12_sfull.S98_S_scrm_contact_merge.event IS '';
@ -64,11 +78,17 @@ create table if not exists p12_sfull.S98_S_scrm_contact_merge (
COMMENT ON COLUMN p12_sfull.S98_S_scrm_contact_merge.last_updated IS '';
COMMENT ON COLUMN p12_sfull.S98_S_scrm_contact_merge.c_name IS '';
COMMENT ON COLUMN p12_sfull.S98_S_scrm_contact_merge.c_type IS '';
COMMENT ON COLUMN p12_sfull.S98_S_scrm_contact_merge.contentname IS '';
COMMENT ON COLUMN p12_sfull.S98_S_scrm_contact_merge.content_name IS '';
COMMENT ON COLUMN p12_sfull.S98_S_scrm_contact_merge.source IS '';
COMMENT ON COLUMN p12_sfull.S98_S_scrm_contact_merge.tag IS '';
COMMENT ON COLUMN p12_sfull.S98_S_scrm_contact_merge.c_keyword IS '';
COMMENT ON COLUMN p12_sfull.S98_S_scrm_contact_merge.attr2 IS '';
COMMENT ON COLUMN p12_sfull.S98_S_scrm_contact_merge.id IS '';
COMMENT ON COLUMN p12_sfull.S98_S_scrm_contact_merge.customer_id_str IS '';
COMMENT ON COLUMN p12_sfull.S98_S_scrm_contact_merge.external_id IS '';
COMMENT ON COLUMN p12_sfull.S98_S_scrm_contact_merge.id_str IS '';
COMMENT ON COLUMN p12_sfull.S98_S_scrm_contact_merge.merged_customer_name IS '';
COMMENT ON COLUMN p12_sfull.S98_S_scrm_contact_merge.target_id IS '';
COMMENT ON COLUMN p12_sfull.S98_S_scrm_contact_merge.etl_tx_dt IS '';
COMMENT ON TABLE p12_sfull.S98_S_scrm_contact_merge IS '';

2
dev/workflow/TK_Cust/tk_api_1/合并SCRM CONTACT/custom_events_update_feign.py
View File

@ -51,7 +51,7 @@ while 'error' in auth and i < 60:
print('开始请求数据总数。')
url='https://api.huiju.cool/v2/customerEvents'
header={}
body={'access_token':auth['access_token'],'limit':'','sort':'lastUpdated','lastUpdated[le]':formatted_current_date,'lastUpdated[ge]':formatted_previous_date,}
body={'access_token':auth['access_token'],'limit':'','event':'merge_customer','sort':'lastUpdated','lastUpdated[le]':formatted_current_date,'lastUpdated[ge]':formatted_previous_date,}
dataReqL=requests.get(url,headers=header,params=body)
resL=json.loads(dataReqL.text)
# print(resL)

33
dev/workflow/TK_Cust/tk_api_1/合并SCRM CONTACT/custom_events_update_load.sql
View File

@ -6,8 +6,7 @@
DELETE FROM data_api.scrm_contact_merge;
insert into data_api.scrm_contact_merge (
mergedCustomerId
, id
merged_customer_id
, customer_id
, date
, event
@ -15,16 +14,21 @@ insert into data_api.scrm_contact_merge (
, last_updated
, c_name
, c_type
, contentName
, content_name
, source
, tag
, c_keyword
, attr2
, id
, customer_id_str
, external_id
, id_str
, merged_customer_name
, target_id
,etl_tx_dt
)
select
case when trim(both from mergedCustomerId)='' then null else mergedCustomerId::text end mergedCustomerId
, case when trim(both from id)='' then null else id::text end id
case when trim(both from merged_customer_id)='' then null else merged_customer_id::text end merged_customer_id
, case when trim(both from customer_id)='' then null else customer_id::text end customer_id
, case when trim(both from date)='' then null else date::text end date
, case when trim(both from event)='' then null else event::text end event
@ -32,16 +36,21 @@ select
, case when trim(both from last_updated)='' then null else last_updated::text end last_updated
, case when trim(both from c_name)='' then null else c_name::text end c_name
, case when trim(both from c_type)='' then null else c_type::text end c_type
, case when trim(both from contentName)='' then null else contentName::text end contentName
, case when trim(both from content_name)='' then null else content_name::text end content_name
, case when trim(both from source)='' then null else source::text end source
, case when trim(both from tag)='' then null else tag::text end tag
, case when trim(both from c_keyword)='' then null else c_keyword::text end c_keyword
, case when trim(both from attr2)='' then null else attr2::text end attr2
, case when trim(both from id)='' then null else id::text end id
, case when trim(both from customer_id_str)='' then null else customer_id_str::text end customer_id_str
, case when trim(both from external_id)='' then null else external_id::text end external_id
, case when trim(both from id_str)='' then null else id_str::text end id_str
, case when trim(both from merged_customer_name)='' then null else merged_customer_name::text end merged_customer_name
, case when trim(both from target_id)='' then null else target_id::text end target_id
,etl_tx_dt
from (
select
(json_array_elements(data::json)::json->>'mergedCustomerId') mergedCustomerId
, (json_array_elements(data::json)::json->>'id') id
(json_array_elements(data::json)::json->>'mergedCustomerId') merged_customer_id
, (json_array_elements(data::json)::json->>'customerId') customer_id
, (json_array_elements(data::json)::json->>'date') date
, (json_array_elements(data::json)::json->>'event') event
@ -49,11 +58,17 @@ select
, (json_array_elements(data::json)::json->>'lastUpdated') last_updated
, (json_array_elements(data::json)::json->>'c_name') c_name
, (json_array_elements(data::json)::json->>'c_type') c_type
, (json_array_elements(data::json)::json->>'contentName') contentName
, (json_array_elements(data::json)::json->>'contentName') content_name
, (json_array_elements(data::json)::json->>'source') source
, (json_array_elements(data::json)::json->>'tag') tag
, (json_array_elements(data::json)::json->>'c_keyword') c_keyword
, (json_array_elements(data::json)::json->>'attr2') attr2
, (json_array_elements(data::json)::json->>'id') id
, (json_array_elements(data::json)::json->>'customerIdStr') customer_id_str
, (json_array_elements(data::json)::json->>'externalId') external_id
, (json_array_elements(data::json)::json->>'idStr') id_str
, (json_array_elements(data::json)::json->>'mergedCustomerName') merged_customer_name
, (json_array_elements(data::json)::json->>'targetId') target_id
,CURRENT_TIMESTAMP(0) etl_tx_dt
from (select * from data_api.api_data
WHERE api_id='ff5c7bf6-0d18-4201-9501-8fdd6152' and is_loaded = '0' order by request_tm desc limit 1) p )p;