dtgitea
/
dataops
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

add workflow 荟聚API,dev

This commit is contained in:
root 2024-01-31 16:02:04 +08:00
parent 8fb569274d
commit 826f4263fc
26 changed files with 1940 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

34
TK_Cust/dev/tk_api/customer_event_meta/S98_S_customer_event_meta.sql Normal file
View File

@ -0,0 +1,34 @@
/*******Main Section**************************************************************************/
\set ON_ERROR_STOP on
\set AUTOCOMMIT on
\timing on
delete from p10_sa.S98_S_customer_event_meta
;
insert into p10_sa.S98_S_customer_event_meta
( group_name
, group_label
, events
, etl_tx_dt )
select
group_name
, group_label
, events
, etl_tx_dt
from p00_tal.S98_S_customer_event_meta
;
delete from p12_sfull.S98_S_customer_event_meta
;
;
insert into p12_sfull.S98_S_customer_event_meta
( group_name
, group_label
, events
, etl_tx_dt )
select
group_name
, group_label
, events
, etl_tx_dt
from p10_sa.S98_S_customer_event_meta
;
\q

15
TK_Cust/dev/tk_api/customer_event_meta/sa_foreign_tables.sql Normal file
View File

@ -0,0 +1,15 @@
CREATE FOREIGN TABLE if not exists p00_tal.S98_S_customer_event_meta (
group_name TEXT
, group_label TEXT
, events TEXT
, etl_tx_dt TIMESTAMP
)
SERVER pgsql_server_S98_S OPTIONS(schema_name 'data_api', table_name 'customer_event_meta' );

31
TK_Cust/dev/tk_api/customer_event_meta/sa_tables.sql Normal file
View File

@ -0,0 +1,31 @@
create table if not exists p10_sa.S98_S_customer_event_meta (
group_name TEXT
, group_label TEXT
, events TEXT
, etl_tx_dt TIMESTAMP
) ;
COMMENT ON COLUMN p10_sa.S98_S_customer_event_meta.group_name IS '';
COMMENT ON COLUMN p10_sa.S98_S_customer_event_meta.group_label IS '';
COMMENT ON COLUMN p10_sa.S98_S_customer_event_meta.events IS '';
COMMENT ON COLUMN p10_sa.S98_S_customer_event_meta.etl_tx_dt IS '';
COMMENT ON TABLE p10_sa.S98_S_customer_event_meta IS '';
create table if not exists p12_sfull.S98_S_customer_event_meta (
group_name TEXT
, group_label TEXT
, events TEXT
, etl_tx_dt TIMESTAMP
) ;
COMMENT ON COLUMN p12_sfull.S98_S_customer_event_meta.group_name IS '';
COMMENT ON COLUMN p12_sfull.S98_S_customer_event_meta.group_label IS '';
COMMENT ON COLUMN p12_sfull.S98_S_customer_event_meta.events IS '';
COMMENT ON COLUMN p12_sfull.S98_S_customer_event_meta.etl_tx_dt IS '';
COMMENT ON TABLE p12_sfull.S98_S_customer_event_meta IS '';

66
TK_Cust/dev/tk_api/customer_events/S98_S_customer_events.sql Normal file
View File

@ -0,0 +1,66 @@
/*******Main Section**************************************************************************/
\set ON_ERROR_STOP on
\set AUTOCOMMIT on
\timing on
delete from p10_sa.S98_S_customer_events
;
insert into p10_sa.S98_S_customer_events
( id_str
, date
, last_updated
, customer_id_str
, create_method
, customer_id
, external_id
, id
, source
, event
, content_name
, etl_tx_dt )
select
id_str
, date
, last_updated
, customer_id_str
, create_method
, customer_id
, external_id
, id
, source
, event
, content_name
, etl_tx_dt
from p00_tal.S98_S_customer_events
;
delete from p12_sfull.S98_S_customer_events
;
;
insert into p12_sfull.S98_S_customer_events
( id_str
, date
, last_updated
, customer_id_str
, create_method
, customer_id
, external_id
, id
, source
, event
, content_name
, etl_tx_dt )
select
id_str
, date
, last_updated
, customer_id_str
, create_method
, customer_id
, external_id
, id
, source
, event
, content_name
, etl_tx_dt
from p10_sa.S98_S_customer_events
;
\q

23
TK_Cust/dev/tk_api/customer_events/sa_foreign_tables.sql Normal file
View File

@ -0,0 +1,23 @@
CREATE FOREIGN TABLE if not exists p00_tal.S98_S_customer_events (
id_str TEXT
, date TEXT
, last_updated TEXT
, customer_id_str TEXT
, create_method TEXT
, customer_id TEXT
, external_id TEXT
, id TEXT
, source TEXT
, event TEXT
, content_name TEXT
, etl_tx_dt TIMESTAMP
)
SERVER pgsql_server_S98_S OPTIONS(schema_name 'data_api', table_name 'customer_events' );

63
TK_Cust/dev/tk_api/customer_events/sa_tables.sql Normal file
View File

@ -0,0 +1,63 @@
create table if not exists p10_sa.S98_S_customer_events (
id_str TEXT
, date TEXT
, last_updated TEXT
, customer_id_str TEXT
, create_method TEXT
, customer_id TEXT
, external_id TEXT
, id TEXT
, source TEXT
, event TEXT
, content_name TEXT
, etl_tx_dt TIMESTAMP
) ;
COMMENT ON COLUMN p10_sa.S98_S_customer_events.id_str IS '';
COMMENT ON COLUMN p10_sa.S98_S_customer_events.date IS '';
COMMENT ON COLUMN p10_sa.S98_S_customer_events.last_updated IS '';
COMMENT ON COLUMN p10_sa.S98_S_customer_events.customer_id_str IS '';
COMMENT ON COLUMN p10_sa.S98_S_customer_events.create_method IS '';
COMMENT ON COLUMN p10_sa.S98_S_customer_events.customer_id IS '';
COMMENT ON COLUMN p10_sa.S98_S_customer_events.external_id IS '';
COMMENT ON COLUMN p10_sa.S98_S_customer_events.id IS '';
COMMENT ON COLUMN p10_sa.S98_S_customer_events.source IS '';
COMMENT ON COLUMN p10_sa.S98_S_customer_events.event IS '';
COMMENT ON COLUMN p10_sa.S98_S_customer_events.content_name IS '';
COMMENT ON COLUMN p10_sa.S98_S_customer_events.etl_tx_dt IS '';
COMMENT ON TABLE p10_sa.S98_S_customer_events IS '';
create table if not exists p12_sfull.S98_S_customer_events (
id_str TEXT
, date TEXT
, last_updated TEXT
, customer_id_str TEXT
, create_method TEXT
, customer_id TEXT
, external_id TEXT
, id TEXT
, source TEXT
, event TEXT
, content_name TEXT
, etl_tx_dt TIMESTAMP
) ;
COMMENT ON COLUMN p12_sfull.S98_S_customer_events.id_str IS '';
COMMENT ON COLUMN p12_sfull.S98_S_customer_events.date IS '';
COMMENT ON COLUMN p12_sfull.S98_S_customer_events.last_updated IS '';
COMMENT ON COLUMN p12_sfull.S98_S_customer_events.customer_id_str IS '';
COMMENT ON COLUMN p12_sfull.S98_S_customer_events.create_method IS '';
COMMENT ON COLUMN p12_sfull.S98_S_customer_events.customer_id IS '';
COMMENT ON COLUMN p12_sfull.S98_S_customer_events.external_id IS '';
COMMENT ON COLUMN p12_sfull.S98_S_customer_events.id IS '';
COMMENT ON COLUMN p12_sfull.S98_S_customer_events.source IS '';
COMMENT ON COLUMN p12_sfull.S98_S_customer_events.event IS '';
COMMENT ON COLUMN p12_sfull.S98_S_customer_events.content_name IS '';
COMMENT ON COLUMN p12_sfull.S98_S_customer_events.etl_tx_dt IS '';
COMMENT ON TABLE p12_sfull.S98_S_customer_events IS '';

130
TK_Cust/dev/tk_api/customer_events_lastupdated/S98_S_customer_events_lastupdated.sql Normal file
View File

@ -0,0 +1,130 @@
/*******Main Section**************************************************************************/
\set ON_ERROR_STOP on
\set AUTOCOMMIT on
\timing on
delete from p10_sa.S98_S_customer_events_lastupdated
;
insert into p10_sa.S98_S_customer_events_lastupdated
( date
, screen_width
, target_name
, ip_city
, ip_country
, platform
, last_updated
, page_type
, os_version
, browser
, browser_version
, customer_id
, id
, tag
, event
, id_str
, os
, target_id
, customer_id_str
, ip
, screen_height
, external_id
, ip_county
, page_id
, url
, ip_province
, device
, etl_tx_dt )
select
date
, screen_width
, target_name
, ip_city
, ip_country
, platform
, last_updated
, page_type
, os_version
, browser
, browser_version
, customer_id
, id
, tag
, event
, id_str
, os
, target_id
, customer_id_str
, ip
, screen_height
, external_id
, ip_county
, page_id
, url
, ip_province
, device
, etl_tx_dt
from p00_tal.S98_S_customer_events_lastupdated
;
delete from p12_sfull.S98_S_customer_events_lastupdated
;
;
insert into p12_sfull.S98_S_customer_events_lastupdated
( date
, screen_width
, target_name
, ip_city
, ip_country
, platform
, last_updated
, page_type
, os_version
, browser
, browser_version
, customer_id
, id
, tag
, event
, id_str
, os
, target_id
, customer_id_str
, ip
, screen_height
, external_id
, ip_county
, page_id
, url
, ip_province
, device
, etl_tx_dt )
select
date
, screen_width
, target_name
, ip_city
, ip_country
, platform
, last_updated
, page_type
, os_version
, browser
, browser_version
, customer_id
, id
, tag
, event
, id_str
, os
, target_id
, customer_id_str
, ip
, screen_height
, external_id
, ip_county
, page_id
, url
, ip_province
, device
, etl_tx_dt
from p10_sa.S98_S_customer_events_lastupdated
;
\q

39
TK_Cust/dev/tk_api/customer_events_lastupdated/sa_foreign_tables.sql Normal file
View File

@ -0,0 +1,39 @@
CREATE FOREIGN TABLE if not exists p00_tal.S98_S_customer_events_lastupdated (
date TEXT
, screen_width TEXT
, target_name TEXT
, ip_city TEXT
, ip_country TEXT
, platform TEXT
, last_updated TEXT
, page_type TEXT
, os_version TEXT
, browser TEXT
, browser_version TEXT
, customer_id TEXT
, id TEXT
, tag TEXT
, event TEXT
, id_str TEXT
, os TEXT
, target_id TEXT
, customer_id_str TEXT
, ip TEXT
, screen_height TEXT
, external_id TEXT
, ip_county TEXT
, page_id TEXT
, url TEXT
, ip_province TEXT
, device TEXT
, etl_tx_dt TIMESTAMP
)
SERVER pgsql_server_S98_S OPTIONS(schema_name 'data_api', table_name 'customer_events_lastupdated' );

127
TK_Cust/dev/tk_api/customer_events_lastupdated/sa_tables.sql Normal file
View File

@ -0,0 +1,127 @@
create table if not exists p10_sa.S98_S_customer_events_lastupdated (
date TEXT
, screen_width TEXT
, target_name TEXT
, ip_city TEXT
, ip_country TEXT
, platform TEXT
, last_updated TEXT
, page_type TEXT
, os_version TEXT
, browser TEXT
, browser_version TEXT
, customer_id TEXT
, id TEXT
, tag TEXT
, event TEXT
, id_str TEXT
, os TEXT
, target_id TEXT
, customer_id_str TEXT
, ip TEXT
, screen_height TEXT
, external_id TEXT
, ip_county TEXT
, page_id TEXT
, url TEXT
, ip_province TEXT
, device TEXT
, etl_tx_dt TIMESTAMP
) ;
COMMENT ON COLUMN p10_sa.S98_S_customer_events_lastupdated.date IS '';
COMMENT ON COLUMN p10_sa.S98_S_customer_events_lastupdated.screen_width IS '';
COMMENT ON COLUMN p10_sa.S98_S_customer_events_lastupdated.target_name IS '';
COMMENT ON COLUMN p10_sa.S98_S_customer_events_lastupdated.ip_city IS '';
COMMENT ON COLUMN p10_sa.S98_S_customer_events_lastupdated.ip_country IS '';
COMMENT ON COLUMN p10_sa.S98_S_customer_events_lastupdated.platform IS '';
COMMENT ON COLUMN p10_sa.S98_S_customer_events_lastupdated.last_updated IS '';
COMMENT ON COLUMN p10_sa.S98_S_customer_events_lastupdated.page_type IS '';
COMMENT ON COLUMN p10_sa.S98_S_customer_events_lastupdated.os_version IS '';
COMMENT ON COLUMN p10_sa.S98_S_customer_events_lastupdated.browser IS '';
COMMENT ON COLUMN p10_sa.S98_S_customer_events_lastupdated.browser_version IS '';
COMMENT ON COLUMN p10_sa.S98_S_customer_events_lastupdated.customer_id IS '';
COMMENT ON COLUMN p10_sa.S98_S_customer_events_lastupdated.id IS '';
COMMENT ON COLUMN p10_sa.S98_S_customer_events_lastupdated.tag IS '';
COMMENT ON COLUMN p10_sa.S98_S_customer_events_lastupdated.event IS '';
COMMENT ON COLUMN p10_sa.S98_S_customer_events_lastupdated.id_str IS '';
COMMENT ON COLUMN p10_sa.S98_S_customer_events_lastupdated.os IS '';
COMMENT ON COLUMN p10_sa.S98_S_customer_events_lastupdated.target_id IS '';
COMMENT ON COLUMN p10_sa.S98_S_customer_events_lastupdated.customer_id_str IS '';
COMMENT ON COLUMN p10_sa.S98_S_customer_events_lastupdated.ip IS '';
COMMENT ON COLUMN p10_sa.S98_S_customer_events_lastupdated.screen_height IS '';
COMMENT ON COLUMN p10_sa.S98_S_customer_events_lastupdated.external_id IS '';
COMMENT ON COLUMN p10_sa.S98_S_customer_events_lastupdated.ip_county IS '';
COMMENT ON COLUMN p10_sa.S98_S_customer_events_lastupdated.page_id IS '';
COMMENT ON COLUMN p10_sa.S98_S_customer_events_lastupdated.url IS '';
COMMENT ON COLUMN p10_sa.S98_S_customer_events_lastupdated.ip_province IS '';
COMMENT ON COLUMN p10_sa.S98_S_customer_events_lastupdated.device IS '';
COMMENT ON COLUMN p10_sa.S98_S_customer_events_lastupdated.etl_tx_dt IS '';
COMMENT ON TABLE p10_sa.S98_S_customer_events_lastupdated IS '';
create table if not exists p12_sfull.S98_S_customer_events_lastupdated (
date TEXT
, screen_width TEXT
, target_name TEXT
, ip_city TEXT
, ip_country TEXT
, platform TEXT
, last_updated TEXT
, page_type TEXT
, os_version TEXT
, browser TEXT
, browser_version TEXT
, customer_id TEXT
, id TEXT
, tag TEXT
, event TEXT
, id_str TEXT
, os TEXT
, target_id TEXT
, customer_id_str TEXT
, ip TEXT
, screen_height TEXT
, external_id TEXT
, ip_county TEXT
, page_id TEXT
, url TEXT
, ip_province TEXT
, device TEXT
, etl_tx_dt TIMESTAMP
) ;
COMMENT ON COLUMN p12_sfull.S98_S_customer_events_lastupdated.date IS '';
COMMENT ON COLUMN p12_sfull.S98_S_customer_events_lastupdated.screen_width IS '';
COMMENT ON COLUMN p12_sfull.S98_S_customer_events_lastupdated.target_name IS '';
COMMENT ON COLUMN p12_sfull.S98_S_customer_events_lastupdated.ip_city IS '';
COMMENT ON COLUMN p12_sfull.S98_S_customer_events_lastupdated.ip_country IS '';
COMMENT ON COLUMN p12_sfull.S98_S_customer_events_lastupdated.platform IS '';
COMMENT ON COLUMN p12_sfull.S98_S_customer_events_lastupdated.last_updated IS '';
COMMENT ON COLUMN p12_sfull.S98_S_customer_events_lastupdated.page_type IS '';
COMMENT ON COLUMN p12_sfull.S98_S_customer_events_lastupdated.os_version IS '';
COMMENT ON COLUMN p12_sfull.S98_S_customer_events_lastupdated.browser IS '';
COMMENT ON COLUMN p12_sfull.S98_S_customer_events_lastupdated.browser_version IS '';
COMMENT ON COLUMN p12_sfull.S98_S_customer_events_lastupdated.customer_id IS '';
COMMENT ON COLUMN p12_sfull.S98_S_customer_events_lastupdated.id IS '';
COMMENT ON COLUMN p12_sfull.S98_S_customer_events_lastupdated.tag IS '';
COMMENT ON COLUMN p12_sfull.S98_S_customer_events_lastupdated.event IS '';
COMMENT ON COLUMN p12_sfull.S98_S_customer_events_lastupdated.id_str IS '';
COMMENT ON COLUMN p12_sfull.S98_S_customer_events_lastupdated.os IS '';
COMMENT ON COLUMN p12_sfull.S98_S_customer_events_lastupdated.target_id IS '';
COMMENT ON COLUMN p12_sfull.S98_S_customer_events_lastupdated.customer_id_str IS '';
COMMENT ON COLUMN p12_sfull.S98_S_customer_events_lastupdated.ip IS '';
COMMENT ON COLUMN p12_sfull.S98_S_customer_events_lastupdated.screen_height IS '';
COMMENT ON COLUMN p12_sfull.S98_S_customer_events_lastupdated.external_id IS '';
COMMENT ON COLUMN p12_sfull.S98_S_customer_events_lastupdated.ip_county IS '';
COMMENT ON COLUMN p12_sfull.S98_S_customer_events_lastupdated.page_id IS '';
COMMENT ON COLUMN p12_sfull.S98_S_customer_events_lastupdated.url IS '';
COMMENT ON COLUMN p12_sfull.S98_S_customer_events_lastupdated.ip_province IS '';
COMMENT ON COLUMN p12_sfull.S98_S_customer_events_lastupdated.device IS '';
COMMENT ON COLUMN p12_sfull.S98_S_customer_events_lastupdated.etl_tx_dt IS '';
COMMENT ON TABLE p12_sfull.S98_S_customer_events_lastupdated IS '';

130
TK_Cust/dev/tk_api/scrm_contact/S98_S_scrm_contact.sql Normal file
View File

@ -0,0 +1,130 @@
/*******Main Section**************************************************************************/
\set ON_ERROR_STOP on
\set AUTOCOMMIT on
\timing on
delete from p10_sa.S98_S_scrm_contact
;
insert into p10_sa.S98_S_scrm_contact
( img
, not_acceptsms
, gender
, create_method
, mobile_verified
, language
, is_member
, source
, is_referrer
, create_from
, last_updated
, date_created
, acceptsms
, id
, subscribe_scene
, id_str
, is_employee
, version
, not_accept_email
, date_join
, content_name
, update_method
, email_verified
, is_anonymous
, accept_email
, create_from_name
, status
, etl_tx_dt )
select
img
, not_acceptsms
, gender
, create_method
, mobile_verified
, language
, is_member
, source
, is_referrer
, create_from
, last_updated
, date_created
, acceptsms
, id
, subscribe_scene
, id_str
, is_employee
, version
, not_accept_email
, date_join
, content_name
, update_method
, email_verified
, is_anonymous
, accept_email
, create_from_name
, status
, etl_tx_dt
from p00_tal.S98_S_scrm_contact
where etl_tx_dt >= CURRENT_DATE;
delete from p12_sfull.S98_S_scrm_contact
where (id) in (select id from p10_sa.S98_S_scrm_contact) ;
;
insert into p12_sfull.S98_S_scrm_contact
( img
, not_acceptsms
, gender
, create_method
, mobile_verified
, language
, is_member
, source
, is_referrer
, create_from
, last_updated
, date_created
, acceptsms
, id
, subscribe_scene
, id_str
, is_employee
, version
, not_accept_email
, date_join
, content_name
, update_method
, email_verified
, is_anonymous
, accept_email
, create_from_name
, status
, etl_tx_dt )
select
img
, not_acceptsms
, gender
, create_method
, mobile_verified
, language
, is_member
, source
, is_referrer
, create_from
, last_updated
, date_created
, acceptsms
, id
, subscribe_scene
, id_str
, is_employee
, version
, not_accept_email
, date_join
, content_name
, update_method
, email_verified
, is_anonymous
, accept_email
, create_from_name
, status
, etl_tx_dt
from p10_sa.S98_S_scrm_contact
;
\q

39
TK_Cust/dev/tk_api/scrm_contact/sa_foreign_tables.sql Normal file
View File

@ -0,0 +1,39 @@
CREATE FOREIGN TABLE if not exists p00_tal.S98_S_scrm_contact (
img TEXT
, not_acceptsms TEXT
, gender TEXT
, create_method TEXT
, mobile_verified TEXT
, language TEXT
, is_member TEXT
, source TEXT
, is_referrer TEXT
, create_from TEXT
, last_updated TEXT
, date_created TEXT
, acceptsms TEXT
, id TEXT
, subscribe_scene TEXT
, id_str TEXT
, is_employee TEXT
, version TEXT
, not_accept_email TEXT
, date_join TEXT
, content_name TEXT
, update_method TEXT
, email_verified TEXT
, is_anonymous TEXT
, accept_email TEXT
, create_from_name TEXT
, status TEXT
, etl_tx_dt TIMESTAMP
)
SERVER pgsql_server_S98_S OPTIONS(schema_name 'data_api', table_name 'scrm_contact' );

127
TK_Cust/dev/tk_api/scrm_contact/sa_tables.sql Normal file
View File

@ -0,0 +1,127 @@
create table if not exists p10_sa.S98_S_scrm_contact (
img TEXT
, not_acceptsms TEXT
, gender TEXT
, create_method TEXT
, mobile_verified TEXT
, language TEXT
, is_member TEXT
, source TEXT
, is_referrer TEXT
, create_from TEXT
, last_updated TEXT
, date_created TEXT
, acceptsms TEXT
, id TEXT
, subscribe_scene TEXT
, id_str TEXT
, is_employee TEXT
, version TEXT
, not_accept_email TEXT
, date_join TEXT
, content_name TEXT
, update_method TEXT
, email_verified TEXT
, is_anonymous TEXT
, accept_email TEXT
, create_from_name TEXT
, status TEXT
, etl_tx_dt TIMESTAMP
) ;
COMMENT ON COLUMN p10_sa.S98_S_scrm_contact.img IS '';
COMMENT ON COLUMN p10_sa.S98_S_scrm_contact.not_acceptsms IS '';
COMMENT ON COLUMN p10_sa.S98_S_scrm_contact.gender IS '';
COMMENT ON COLUMN p10_sa.S98_S_scrm_contact.create_method IS '';
COMMENT ON COLUMN p10_sa.S98_S_scrm_contact.mobile_verified IS '';
COMMENT ON COLUMN p10_sa.S98_S_scrm_contact.language IS '';
COMMENT ON COLUMN p10_sa.S98_S_scrm_contact.is_member IS '';
COMMENT ON COLUMN p10_sa.S98_S_scrm_contact.source IS '';
COMMENT ON COLUMN p10_sa.S98_S_scrm_contact.is_referrer IS '';
COMMENT ON COLUMN p10_sa.S98_S_scrm_contact.create_from IS '';
COMMENT ON COLUMN p10_sa.S98_S_scrm_contact.last_updated IS '';
COMMENT ON COLUMN p10_sa.S98_S_scrm_contact.date_created IS '';
COMMENT ON COLUMN p10_sa.S98_S_scrm_contact.acceptsms IS '';
COMMENT ON COLUMN p10_sa.S98_S_scrm_contact.id IS '';
COMMENT ON COLUMN p10_sa.S98_S_scrm_contact.subscribe_scene IS '';
COMMENT ON COLUMN p10_sa.S98_S_scrm_contact.id_str IS '';
COMMENT ON COLUMN p10_sa.S98_S_scrm_contact.is_employee IS '';
COMMENT ON COLUMN p10_sa.S98_S_scrm_contact.version IS '';
COMMENT ON COLUMN p10_sa.S98_S_scrm_contact.not_accept_email IS '';
COMMENT ON COLUMN p10_sa.S98_S_scrm_contact.date_join IS '';
COMMENT ON COLUMN p10_sa.S98_S_scrm_contact.content_name IS '';
COMMENT ON COLUMN p10_sa.S98_S_scrm_contact.update_method IS '';
COMMENT ON COLUMN p10_sa.S98_S_scrm_contact.email_verified IS '';
COMMENT ON COLUMN p10_sa.S98_S_scrm_contact.is_anonymous IS '';
COMMENT ON COLUMN p10_sa.S98_S_scrm_contact.accept_email IS '';
COMMENT ON COLUMN p10_sa.S98_S_scrm_contact.create_from_name IS '';
COMMENT ON COLUMN p10_sa.S98_S_scrm_contact.status IS '';
COMMENT ON COLUMN p10_sa.S98_S_scrm_contact.etl_tx_dt IS '';
COMMENT ON TABLE p10_sa.S98_S_scrm_contact IS '';
create table if not exists p12_sfull.S98_S_scrm_contact (
img TEXT
, not_acceptsms TEXT
, gender TEXT
, create_method TEXT
, mobile_verified TEXT
, language TEXT
, is_member TEXT
, source TEXT
, is_referrer TEXT
, create_from TEXT
, last_updated TEXT
, date_created TEXT
, acceptsms TEXT
, id TEXT
, subscribe_scene TEXT
, id_str TEXT
, is_employee TEXT
, version TEXT
, not_accept_email TEXT
, date_join TEXT
, content_name TEXT
, update_method TEXT
, email_verified TEXT
, is_anonymous TEXT
, accept_email TEXT
, create_from_name TEXT
, status TEXT
, etl_tx_dt TIMESTAMP
) ;
COMMENT ON COLUMN p12_sfull.S98_S_scrm_contact.img IS '';
COMMENT ON COLUMN p12_sfull.S98_S_scrm_contact.not_acceptsms IS '';
COMMENT ON COLUMN p12_sfull.S98_S_scrm_contact.gender IS '';
COMMENT ON COLUMN p12_sfull.S98_S_scrm_contact.create_method IS '';
COMMENT ON COLUMN p12_sfull.S98_S_scrm_contact.mobile_verified IS '';
COMMENT ON COLUMN p12_sfull.S98_S_scrm_contact.language IS '';
COMMENT ON COLUMN p12_sfull.S98_S_scrm_contact.is_member IS '';
COMMENT ON COLUMN p12_sfull.S98_S_scrm_contact.source IS '';
COMMENT ON COLUMN p12_sfull.S98_S_scrm_contact.is_referrer IS '';
COMMENT ON COLUMN p12_sfull.S98_S_scrm_contact.create_from IS '';
COMMENT ON COLUMN p12_sfull.S98_S_scrm_contact.last_updated IS '';
COMMENT ON COLUMN p12_sfull.S98_S_scrm_contact.date_created IS '';
COMMENT ON COLUMN p12_sfull.S98_S_scrm_contact.acceptsms IS '';
COMMENT ON COLUMN p12_sfull.S98_S_scrm_contact.id IS '';
COMMENT ON COLUMN p12_sfull.S98_S_scrm_contact.subscribe_scene IS '';
COMMENT ON COLUMN p12_sfull.S98_S_scrm_contact.id_str IS '';
COMMENT ON COLUMN p12_sfull.S98_S_scrm_contact.is_employee IS '';
COMMENT ON COLUMN p12_sfull.S98_S_scrm_contact.version IS '';
COMMENT ON COLUMN p12_sfull.S98_S_scrm_contact.not_accept_email IS '';
COMMENT ON COLUMN p12_sfull.S98_S_scrm_contact.date_join IS '';
COMMENT ON COLUMN p12_sfull.S98_S_scrm_contact.content_name IS '';
COMMENT ON COLUMN p12_sfull.S98_S_scrm_contact.update_method IS '';
COMMENT ON COLUMN p12_sfull.S98_S_scrm_contact.email_verified IS '';
COMMENT ON COLUMN p12_sfull.S98_S_scrm_contact.is_anonymous IS '';
COMMENT ON COLUMN p12_sfull.S98_S_scrm_contact.accept_email IS '';
COMMENT ON COLUMN p12_sfull.S98_S_scrm_contact.create_from_name IS '';
COMMENT ON COLUMN p12_sfull.S98_S_scrm_contact.status IS '';
COMMENT ON COLUMN p12_sfull.S98_S_scrm_contact.etl_tx_dt IS '';
COMMENT ON TABLE p12_sfull.S98_S_scrm_contact IS '';

138
TK_Cust/dev/tk_api/scrm_contact_lastupdated/S98_S_scrm_contact_lastupdated.sql Normal file
View File

@ -0,0 +1,138 @@
/*******Main Section**************************************************************************/
\set ON_ERROR_STOP on
\set AUTOCOMMIT on
\timing on
delete from p10_sa.S98_S_scrm_contact_lastupdated
;
insert into p10_sa.S98_S_scrm_contact_lastupdated
( img
, not_acceptsms
, gender
, display_name
, create_method
, mobile_verified
, language
, is_member
, source
, is_referrer
, create_from
, last_updated
, date_created
, acceptsms
, id
, subscribe_scene
, id_str
, is_employee
, nick_name
, version
, not_accept_email
, date_join
, content_name
, update_method
, email_verified
, is_anonymous
, accept_email
, create_from_name
, status
, etl_tx_dt )
select
img
, not_acceptsms
, gender
, display_name
, create_method
, mobile_verified
, language
, is_member
, source
, is_referrer
, create_from
, last_updated
, date_created
, acceptsms
, id
, subscribe_scene
, id_str
, is_employee
, nick_name
, version
, not_accept_email
, date_join
, content_name
, update_method
, email_verified
, is_anonymous
, accept_email
, create_from_name
, status
, etl_tx_dt
from p00_tal.S98_S_scrm_contact_lastupdated
;
delete from p12_sfull.S98_S_scrm_contact_lastupdated
;
;
insert into p12_sfull.S98_S_scrm_contact_lastupdated
( img
, not_acceptsms
, gender
, display_name
, create_method
, mobile_verified
, language
, is_member
, source
, is_referrer
, create_from
, last_updated
, date_created
, acceptsms
, id
, subscribe_scene
, id_str
, is_employee
, nick_name
, version
, not_accept_email
, date_join
, content_name
, update_method
, email_verified
, is_anonymous
, accept_email
, create_from_name
, status
, etl_tx_dt )
select
img
, not_acceptsms
, gender
, display_name
, create_method
, mobile_verified
, language
, is_member
, source
, is_referrer
, create_from
, last_updated
, date_created
, acceptsms
, id
, subscribe_scene
, id_str
, is_employee
, nick_name
, version
, not_accept_email
, date_join
, content_name
, update_method
, email_verified
, is_anonymous
, accept_email
, create_from_name
, status
, etl_tx_dt
from p10_sa.S98_S_scrm_contact_lastupdated
;
\q

41
TK_Cust/dev/tk_api/scrm_contact_lastupdated/sa_foreign_tables.sql Normal file
View File

@ -0,0 +1,41 @@
CREATE FOREIGN TABLE if not exists p00_tal.S98_S_scrm_contact_lastupdated (
img TEXT
, not_acceptsms TEXT
, gender TEXT
, display_name TEXT
, create_method TEXT
, mobile_verified TEXT
, language TEXT
, is_member TEXT
, source TEXT
, is_referrer TEXT
, create_from TEXT
, last_updated TEXT
, date_created TEXT
, acceptsms TEXT
, id TEXT
, subscribe_scene TEXT
, id_str TEXT
, is_employee TEXT
, nick_name TEXT
, version TEXT
, not_accept_email TEXT
, date_join TEXT
, content_name TEXT
, update_method TEXT
, email_verified TEXT
, is_anonymous TEXT
, accept_email TEXT
, create_from_name TEXT
, status TEXT
, etl_tx_dt TIMESTAMP
)
SERVER pgsql_server_S98_S OPTIONS(schema_name 'data_api', table_name 'scrm_contact_lastupdated' );

135
TK_Cust/dev/tk_api/scrm_contact_lastupdated/sa_tables.sql Normal file
View File

@ -0,0 +1,135 @@
create table if not exists p10_sa.S98_S_scrm_contact_lastupdated (
img TEXT
, not_acceptsms TEXT
, gender TEXT
, display_name TEXT
, create_method TEXT
, mobile_verified TEXT
, language TEXT
, is_member TEXT
, source TEXT
, is_referrer TEXT
, create_from TEXT
, last_updated TEXT
, date_created TEXT
, acceptsms TEXT
, id TEXT
, subscribe_scene TEXT
, id_str TEXT
, is_employee TEXT
, nick_name TEXT
, version TEXT
, not_accept_email TEXT
, date_join TEXT
, content_name TEXT
, update_method TEXT
, email_verified TEXT
, is_anonymous TEXT
, accept_email TEXT
, create_from_name TEXT
, status TEXT
, etl_tx_dt TIMESTAMP
) ;
COMMENT ON COLUMN p10_sa.S98_S_scrm_contact_lastupdated.img IS '';
COMMENT ON COLUMN p10_sa.S98_S_scrm_contact_lastupdated.not_acceptsms IS '';
COMMENT ON COLUMN p10_sa.S98_S_scrm_contact_lastupdated.gender IS '';
COMMENT ON COLUMN p10_sa.S98_S_scrm_contact_lastupdated.display_name IS '';
COMMENT ON COLUMN p10_sa.S98_S_scrm_contact_lastupdated.create_method IS '';
COMMENT ON COLUMN p10_sa.S98_S_scrm_contact_lastupdated.mobile_verified IS '';
COMMENT ON COLUMN p10_sa.S98_S_scrm_contact_lastupdated.language IS '';
COMMENT ON COLUMN p10_sa.S98_S_scrm_contact_lastupdated.is_member IS '';
COMMENT ON COLUMN p10_sa.S98_S_scrm_contact_lastupdated.source IS '';
COMMENT ON COLUMN p10_sa.S98_S_scrm_contact_lastupdated.is_referrer IS '';
COMMENT ON COLUMN p10_sa.S98_S_scrm_contact_lastupdated.create_from IS '';
COMMENT ON COLUMN p10_sa.S98_S_scrm_contact_lastupdated.last_updated IS '';
COMMENT ON COLUMN p10_sa.S98_S_scrm_contact_lastupdated.date_created IS '';
COMMENT ON COLUMN p10_sa.S98_S_scrm_contact_lastupdated.acceptsms IS '';
COMMENT ON COLUMN p10_sa.S98_S_scrm_contact_lastupdated.id IS '';
COMMENT ON COLUMN p10_sa.S98_S_scrm_contact_lastupdated.subscribe_scene IS '';
COMMENT ON COLUMN p10_sa.S98_S_scrm_contact_lastupdated.id_str IS '';
COMMENT ON COLUMN p10_sa.S98_S_scrm_contact_lastupdated.is_employee IS '';
COMMENT ON COLUMN p10_sa.S98_S_scrm_contact_lastupdated.nick_name IS '';
COMMENT ON COLUMN p10_sa.S98_S_scrm_contact_lastupdated.version IS '';
COMMENT ON COLUMN p10_sa.S98_S_scrm_contact_lastupdated.not_accept_email IS '';
COMMENT ON COLUMN p10_sa.S98_S_scrm_contact_lastupdated.date_join IS '';
COMMENT ON COLUMN p10_sa.S98_S_scrm_contact_lastupdated.content_name IS '';
COMMENT ON COLUMN p10_sa.S98_S_scrm_contact_lastupdated.update_method IS '';
COMMENT ON COLUMN p10_sa.S98_S_scrm_contact_lastupdated.email_verified IS '';
COMMENT ON COLUMN p10_sa.S98_S_scrm_contact_lastupdated.is_anonymous IS '';
COMMENT ON COLUMN p10_sa.S98_S_scrm_contact_lastupdated.accept_email IS '';
COMMENT ON COLUMN p10_sa.S98_S_scrm_contact_lastupdated.create_from_name IS '';
COMMENT ON COLUMN p10_sa.S98_S_scrm_contact_lastupdated.status IS '';
COMMENT ON COLUMN p10_sa.S98_S_scrm_contact_lastupdated.etl_tx_dt IS '';
COMMENT ON TABLE p10_sa.S98_S_scrm_contact_lastupdated IS '';
create table if not exists p12_sfull.S98_S_scrm_contact_lastupdated (
img TEXT
, not_acceptsms TEXT
, gender TEXT
, display_name TEXT
, create_method TEXT
, mobile_verified TEXT
, language TEXT
, is_member TEXT
, source TEXT
, is_referrer TEXT
, create_from TEXT
, last_updated TEXT
, date_created TEXT
, acceptsms TEXT
, id TEXT
, subscribe_scene TEXT
, id_str TEXT
, is_employee TEXT
, nick_name TEXT
, version TEXT
, not_accept_email TEXT
, date_join TEXT
, content_name TEXT
, update_method TEXT
, email_verified TEXT
, is_anonymous TEXT
, accept_email TEXT
, create_from_name TEXT
, status TEXT
, etl_tx_dt TIMESTAMP
) ;
COMMENT ON COLUMN p12_sfull.S98_S_scrm_contact_lastupdated.img IS '';
COMMENT ON COLUMN p12_sfull.S98_S_scrm_contact_lastupdated.not_acceptsms IS '';
COMMENT ON COLUMN p12_sfull.S98_S_scrm_contact_lastupdated.gender IS '';
COMMENT ON COLUMN p12_sfull.S98_S_scrm_contact_lastupdated.display_name IS '';
COMMENT ON COLUMN p12_sfull.S98_S_scrm_contact_lastupdated.create_method IS '';
COMMENT ON COLUMN p12_sfull.S98_S_scrm_contact_lastupdated.mobile_verified IS '';
COMMENT ON COLUMN p12_sfull.S98_S_scrm_contact_lastupdated.language IS '';
COMMENT ON COLUMN p12_sfull.S98_S_scrm_contact_lastupdated.is_member IS '';
COMMENT ON COLUMN p12_sfull.S98_S_scrm_contact_lastupdated.source IS '';
COMMENT ON COLUMN p12_sfull.S98_S_scrm_contact_lastupdated.is_referrer IS '';
COMMENT ON COLUMN p12_sfull.S98_S_scrm_contact_lastupdated.create_from IS '';
COMMENT ON COLUMN p12_sfull.S98_S_scrm_contact_lastupdated.last_updated IS '';
COMMENT ON COLUMN p12_sfull.S98_S_scrm_contact_lastupdated.date_created IS '';
COMMENT ON COLUMN p12_sfull.S98_S_scrm_contact_lastupdated.acceptsms IS '';
COMMENT ON COLUMN p12_sfull.S98_S_scrm_contact_lastupdated.id IS '';
COMMENT ON COLUMN p12_sfull.S98_S_scrm_contact_lastupdated.subscribe_scene IS '';
COMMENT ON COLUMN p12_sfull.S98_S_scrm_contact_lastupdated.id_str IS '';
COMMENT ON COLUMN p12_sfull.S98_S_scrm_contact_lastupdated.is_employee IS '';
COMMENT ON COLUMN p12_sfull.S98_S_scrm_contact_lastupdated.nick_name IS '';
COMMENT ON COLUMN p12_sfull.S98_S_scrm_contact_lastupdated.version IS '';
COMMENT ON COLUMN p12_sfull.S98_S_scrm_contact_lastupdated.not_accept_email IS '';
COMMENT ON COLUMN p12_sfull.S98_S_scrm_contact_lastupdated.date_join IS '';
COMMENT ON COLUMN p12_sfull.S98_S_scrm_contact_lastupdated.content_name IS '';
COMMENT ON COLUMN p12_sfull.S98_S_scrm_contact_lastupdated.update_method IS '';
COMMENT ON COLUMN p12_sfull.S98_S_scrm_contact_lastupdated.email_verified IS '';
COMMENT ON COLUMN p12_sfull.S98_S_scrm_contact_lastupdated.is_anonymous IS '';
COMMENT ON COLUMN p12_sfull.S98_S_scrm_contact_lastupdated.accept_email IS '';
COMMENT ON COLUMN p12_sfull.S98_S_scrm_contact_lastupdated.create_from_name IS '';
COMMENT ON COLUMN p12_sfull.S98_S_scrm_contact_lastupdated.status IS '';
COMMENT ON COLUMN p12_sfull.S98_S_scrm_contact_lastupdated.etl_tx_dt IS '';
COMMENT ON TABLE p12_sfull.S98_S_scrm_contact_lastupdated IS '';

41
TK_Cust/dev/tk_api/合并SCRM CONTACT/custom_events_update_feign.py Normal file
View File

@ -0,0 +1,41 @@
# coding: utf-8
import requests
import json
import psycopg2
import uuid
import datetime
# 获取当前日期
current_date = datetime.date.today()
previous_date = current_date - datetime.timedelta(days=2)
formatted_current_date = current_date.strftime("%Y-%m-%dT%H:%M:%SZ")
formatted_previous_date = previous_date.strftime("%Y-%m-%dT%H:%M:%SZ")
print('开始加载数据：custom_events_update:合并SCRM CONTACT')
authUrl='https://api.huiju.cool/v2/oauth2/token?app_id=cl037b184ebccd97c&secret=deaa83fdd8d385eec0d1d04d34282c4ec8c33b60&grant_type=client_credentials'
print('开始请求令牌。')
authRequest=requests.get(authUrl)
auth=json.loads(authRequest.text)
print('开始请求数据总数。')
url='https://api.huiju.cool/v2/customerEvents'
header={}
body={'access_token':auth['access_token'],'sort':'lastUpdated','lastUpdated[ge]':formatted_previous_date,'lastUpdated[le]':formatted_current_date,'limit':'',}
dataReqL=requests.get(url,headers=header,params=body)
resL=json.loads(dataReqL.text)
# print(resL)
dataList=resL['data']
total=len(dataList)
conn = psycopg2.connect(database="dataops_db", user="dbuser_dops", password="MIgTi3jA",
host="172.17.0.8", port="5432")
print('数据库连接成功')
dataId=str(uuid.uuid4())
print('临时id：'+dataId)
json_object = json.dumps(dataList)
cur=conn.cursor()
sql="update data_api.api_data set is_loaded = '1' where api_id = 'ff5c7bf6-0d18-4201-9501-8fdd6152';INSERT INTO data_api.api_data (id,api_id,data,total_num,is_loaded,status,request_tm,execute_tm,remark) values (%s,%s,%s,%s,'0','0',current_timestamp(0),current_timestamp(0),'')"
cur.execute(sql,[dataId,'ff5c7bf6-0d18-4201-9501-8fdd6152',json_object,total])
conn.commit()
cur.close()
conn.close()
print('加载数据结束：custom_events_update:合并SCRM CONTACT')

105
TK_Cust/dev/tk_api/合并SCRM CONTACT/custom_events_update_load.sql Normal file
View File

@ -0,0 +1,105 @@
/*******Main Section**************************************************************************/
\set ON_ERROR_STOP on
\set AUTOCOMMIT on
\timing on
DELETE FROM data_api.customer_events_lastupdated;
insert into data_api.customer_events_lastupdated (
date
, screen_width
, target_name
, ip_city
, ip_country
, platform
, last_updated
, page_type
, os_version
, browser
, browser_version
, customer_id
, id
, tag
, event
, id_str
, os
, target_id
, customer_id_str
, ip
, screen_height
, external_id
, ip_county
, page_id
, url
, ip_province
, device
,etl_tx_dt
)
select
case when trim(both from date)='' then null else date::text end date
, case when trim(both from screen_width)='' then null else screen_width::text end screen_width
, case when trim(both from target_name)='' then null else target_name::text end target_name
, case when trim(both from ip_city)='' then null else ip_city::text end ip_city
, case when trim(both from ip_country)='' then null else ip_country::text end ip_country
, case when trim(both from platform)='' then null else platform::text end platform
, case when trim(both from last_updated)='' then null else last_updated::text end last_updated
, case when trim(both from page_type)='' then null else page_type::text end page_type
, case when trim(both from os_version)='' then null else os_version::text end os_version
, case when trim(both from browser)='' then null else browser::text end browser
, case when trim(both from browser_version)='' then null else browser_version::text end browser_version
, case when trim(both from customer_id)='' then null else customer_id::text end customer_id
, case when trim(both from id)='' then null else id::text end id
, case when trim(both from tag)='' then null else tag::text end tag
, case when trim(both from event)='' then null else event::text end event
, case when trim(both from id_str)='' then null else id_str::text end id_str
, case when trim(both from os)='' then null else os::text end os
, case when trim(both from target_id)='' then null else target_id::text end target_id
, case when trim(both from customer_id_str)='' then null else customer_id_str::text end customer_id_str
, case when trim(both from ip)='' then null else ip::text end ip
, case when trim(both from screen_height)='' then null else screen_height::text end screen_height
, case when trim(both from external_id)='' then null else external_id::text end external_id
, case when trim(both from ip_county)='' then null else ip_county::text end ip_county
, case when trim(both from page_id)='' then null else page_id::text end page_id
, case when trim(both from url)='' then null else url::text end url
, case when trim(both from ip_province)='' then null else ip_province::text end ip_province
, case when trim(both from device)='' then null else device::text end device
,etl_tx_dt
from (
select
(json_array_elements(data::json)::json->>'date') date
, (json_array_elements(data::json)::json->>'screenWidth') screen_width
, (json_array_elements(data::json)::json->>'targetName') target_name
, (json_array_elements(data::json)::json->>'ipCity') ip_city
, (json_array_elements(data::json)::json->>'ipCountry') ip_country
, (json_array_elements(data::json)::json->>'platform') platform
, (json_array_elements(data::json)::json->>'lastUpdated') last_updated
, (json_array_elements(data::json)::json->>'pageType') page_type
, (json_array_elements(data::json)::json->>'osVersion') os_version
, (json_array_elements(data::json)::json->>'browser') browser
, (json_array_elements(data::json)::json->>'browserVersion') browser_version
, (json_array_elements(data::json)::json->>'customerId') customer_id
, (json_array_elements(data::json)::json->>'id') id
, (json_array_elements(data::json)::json->>'tag') tag
, (json_array_elements(data::json)::json->>'event') event
, (json_array_elements(data::json)::json->>'idStr') id_str
, (json_array_elements(data::json)::json->>'os') os
, (json_array_elements(data::json)::json->>'targetId') target_id
, (json_array_elements(data::json)::json->>'customerIdStr') customer_id_str
, (json_array_elements(data::json)::json->>'ip') ip
, (json_array_elements(data::json)::json->>'screenHeight') screen_height
, (json_array_elements(data::json)::json->>'externalId') external_id
, (json_array_elements(data::json)::json->>'ipCounty') ip_county
, (json_array_elements(data::json)::json->>'pageId') page_id
, (json_array_elements(data::json)::json->>'url') url
, (json_array_elements(data::json)::json->>'ipProvince') ip_province
, (json_array_elements(data::json)::json->>'device') device
,CURRENT_TIMESTAMP(0) etl_tx_dt
from (select * from data_api.api_data
WHERE api_id='ff5c7bf6-0d18-4201-9501-8fdd6152' and is_loaded = '0' order by request_tm desc limit 1) p )p;
update data_api.api_data
set is_loaded = '1' ,
status = '1',
request_tm = current_timestamp(0)
where api_id='ff5c7bf6-0d18-4201-9501-8fdd6152';
\q

41
TK_Cust/dev/tk_api/更新scrm CONTACT/update_scrm_contact_feign.py Normal file
View File

@ -0,0 +1,41 @@
# coding: utf-8
import requests
import json
import psycopg2
import uuid
import datetime
# 获取当前日期
current_date = datetime.date.today()
previous_date = current_date - datetime.timedelta(days=2)
formatted_current_date = current_date.strftime("%Y-%m-%dT%H:%M:%SZ")
formatted_previous_date = previous_date.strftime("%Y-%m-%dT%H:%M:%SZ")
print('开始加载数据：update_scrm_contact:更新scrm CONTACT')
authUrl='https://api.huiju.cool/v2/oauth2/token?app_id=cl037b184ebccd97c&secret=deaa83fdd8d385eec0d1d04d34282c4ec8c33b60&grant_type=client_credentials'
print('开始请求令牌。')
authRequest=requests.get(authUrl)
auth=json.loads(authRequest.text)
print('开始请求数据总数。')
url='https://api.huiju.cool/v2/customers'
header={}
body={'access_token':auth['access_token'],'sort':'lastUpdated','lastUpdated[ge]':formatted_previous_date,'lastUpdated[le]':formatted_current_date,'limit':'',}
dataReqL=requests.get(url,headers=header,params=body)
resL=json.loads(dataReqL.text)
# print(resL)
dataList=resL['data']
total=len(dataList)
conn = psycopg2.connect(database="dataops_db", user="dbuser_dops", password="MIgTi3jA",
host="172.17.0.8", port="5432")
print('数据库连接成功')
dataId=str(uuid.uuid4())
print('临时id：'+dataId)
json_object = json.dumps(dataList)
cur=conn.cursor()
sql="update data_api.api_data set is_loaded = '1' where api_id = '03d918ea-00d1-448a-8b9b-6a0b1a33';INSERT INTO data_api.api_data (id,api_id,data,total_num,is_loaded,status,request_tm,execute_tm,remark) values (%s,%s,%s,%s,'0','0',current_timestamp(0),current_timestamp(0),'')"
cur.execute(sql,[dataId,'03d918ea-00d1-448a-8b9b-6a0b1a33',json_object,total])
conn.commit()
cur.close()
conn.close()
print('加载数据结束：update_scrm_contact:更新scrm CONTACT')

111
TK_Cust/dev/tk_api/更新scrm CONTACT/update_scrm_contact_load.sql Normal file
View File

@ -0,0 +1,111 @@
/*******Main Section**************************************************************************/
\set ON_ERROR_STOP on
\set AUTOCOMMIT on
\timing on
DELETE FROM data_api.scrm_contact_lastUpdated;
insert into data_api.scrm_contact_lastUpdated (
img
, not_acceptsms
, gender
, display_name
, create_method
, mobile_verified
, language
, is_member
, source
, is_referrer
, create_from
, last_updated
, date_created
, acceptsms
, id
, subscribe_scene
, id_str
, is_employee
, nick_name
, version
, not_accept_email
, date_join
, content_name
, update_method
, email_verified
, is_anonymous
, accept_email
, create_from_name
, status
,etl_tx_dt
)
select
case when trim(both from img)='' then null else img::text end img
, case when trim(both from not_acceptsms)='' then null else not_acceptsms::text end not_acceptsms
, case when trim(both from gender)='' then null else gender::text end gender
, case when trim(both from display_name)='' then null else display_name::text end display_name
, case when trim(both from create_method)='' then null else create_method::text end create_method
, case when trim(both from mobile_verified)='' then null else mobile_verified::text end mobile_verified
, case when trim(both from language)='' then null else language::text end language
, case when trim(both from is_member)='' then null else is_member::text end is_member
, case when trim(both from source)='' then null else source::text end source
, case when trim(both from is_referrer)='' then null else is_referrer::text end is_referrer
, case when trim(both from create_from)='' then null else create_from::text end create_from
, case when trim(both from last_updated)='' then null else last_updated::text end last_updated
, case when trim(both from date_created)='' then null else date_created::text end date_created
, case when trim(both from acceptsms)='' then null else acceptsms::text end acceptsms
, case when trim(both from id)='' then null else id::text end id
, case when trim(both from subscribe_scene)='' then null else subscribe_scene::text end subscribe_scene
, case when trim(both from id_str)='' then null else id_str::text end id_str
, case when trim(both from is_employee)='' then null else is_employee::text end is_employee
, case when trim(both from nick_name)='' then null else nick_name::text end nick_name
, case when trim(both from version)='' then null else version::text end version
, case when trim(both from not_accept_email)='' then null else not_accept_email::text end not_accept_email
, case when trim(both from date_join)='' then null else date_join::text end date_join
, case when trim(both from content_name)='' then null else content_name::text end content_name
, case when trim(both from update_method)='' then null else update_method::text end update_method
, case when trim(both from email_verified)='' then null else email_verified::text end email_verified
, case when trim(both from is_anonymous)='' then null else is_anonymous::text end is_anonymous
, case when trim(both from accept_email)='' then null else accept_email::text end accept_email
, case when trim(both from create_from_name)='' then null else create_from_name::text end create_from_name
, case when trim(both from status)='' then null else status::text end status
,etl_tx_dt
from (
select
(json_array_elements(data::json)::json->>'img') img
, (json_array_elements(data::json)::json->>'notAcceptsms') not_acceptsms
, (json_array_elements(data::json)::json->>'gender') gender
, (json_array_elements(data::json)::json->>'displayName') display_name
, (json_array_elements(data::json)::json->>'createMethod') create_method
, (json_array_elements(data::json)::json->>'mobileVerified') mobile_verified
, (json_array_elements(data::json)::json->>'language') language
, (json_array_elements(data::json)::json->>'isMember') is_member
, (json_array_elements(data::json)::json->>'source') source
, (json_array_elements(data::json)::json->>'isReferrer') is_referrer
, (json_array_elements(data::json)::json->>'createFrom') create_from
, (json_array_elements(data::json)::json->>'lastUpdated') last_updated
, (json_array_elements(data::json)::json->>'dateCreated') date_created
, (json_array_elements(data::json)::json->>'acceptsms') acceptsms
, (json_array_elements(data::json)::json->>'id') id
, (json_array_elements(data::json)::json->>'subscribeScene') subscribe_scene
, (json_array_elements(data::json)::json->>'idStr') id_str
, (json_array_elements(data::json)::json->>'isEmployee') is_employee
, (json_array_elements(data::json)::json->>'nickName') nick_name
, (json_array_elements(data::json)::json->>'version') version
, (json_array_elements(data::json)::json->>'notAcceptEmail') not_accept_email
, (json_array_elements(data::json)::json->>'dateJoin') date_join
, (json_array_elements(data::json)::json->>'contentName') content_name
, (json_array_elements(data::json)::json->>'updateMethod') update_method
, (json_array_elements(data::json)::json->>'emailVerified') email_verified
, (json_array_elements(data::json)::json->>'isAnonymous') is_anonymous
, (json_array_elements(data::json)::json->>'acceptEmail') accept_email
, (json_array_elements(data::json)::json->>'createFromName') create_from_name
, (json_array_elements(data::json)::json->>'status') status
,CURRENT_TIMESTAMP(0) etl_tx_dt
from (select * from data_api.api_data
WHERE api_id='03d918ea-00d1-448a-8b9b-6a0b1a33' and is_loaded = '0' order by request_tm desc limit 1) p )p;
update data_api.api_data
set is_loaded = '1' ,
status = '1',
request_tm = current_timestamp(0)
where api_id='03d918ea-00d1-448a-8b9b-6a0b1a33';
\q

186
TK_Cust/dev/tk_api/荟聚API/wf_dag_tk_api.py Normal file
View File

@ -0,0 +1,186 @@
#!/usr/bin/python
# -*- encoding=utf-8 -*-
from airflow import DAG
from datetime import datetime, timedelta
from airflow.contrib.hooks.ssh_hook import SSHHook
from airflow.contrib.operators.ssh_operator import SSHOperator
from airflow.sensors.external_task_sensor import ExternalTaskSensor
import json
from airflow.operators.email_operator import EmailOperator
from airflow.utils.trigger_rule import TriggerRule
sshHook = SSHHook(ssh_conn_id ='ssh_air')
default_args = {
'owner': 'info@idgvalue.com',
'email': [''],
'email_on_failure': True,
'email_on_retry':True,
'start_date': datetime(2022, 9, 12),
'depends_on_past': False,
'retries': 6,
'retry_delay': timedelta(minutes=10),
}
dag = DAG('wf_dag_tk_api', default_args=default_args,
schedule_interval="0 0 * * *",
catchup=False,
dagrun_timeout=timedelta(minutes=160),
max_active_runs=3)
task_failed = EmailOperator (
dag=dag,
trigger_rule=TriggerRule.ONE_FAILED,
task_id="task_failed",
to=["info@idgvalue.com"],
cc=[""],
subject="tk_api_failed",
html_content='<h3>您好，tk_api作业失败，请及时处理" </h3>')
scrm_contact_feign = SSHOperator(
ssh_hook=sshHook,
task_id='scrm_contact_feign',
command='python3 /data/airflow/etl/API/scrm_contact_feign.py',
depends_on_past=False,
retries=3,
dag=dag)
scrm_contact_load = SSHOperator(
ssh_hook=sshHook,
task_id='scrm_contact_load',
command='/data/airflow/etl/API/run_psql.sh {{ ds_nodash }} {{params.my_param }}',
params={'my_param':"scrm_contact_load"},
depends_on_past=False,
retries=3,
dag=dag)
scrm_contact_feign >> scrm_contact_load
scrm_contact_1129 = SSHOperator(
ssh_hook=sshHook,
task_id='scrm_contact_1129',
command='/data/airflow/etl/SA/run_sa.sh {{ ds_nodash }} {{ params.my_param }} >>/data/airflow/logs/run_psql_{{ds_nodash}}.log 2>&1 ',
params={'my_param':"S98_S_scrm_contact"},
depends_on_past=False,
retries=3,
dag=dag)
custom_events_update_feign = SSHOperator(
ssh_hook=sshHook,
task_id='custom_events_update_feign',
command='python3 /data/airflow/etl/API/custom_events_update_feign.py',
depends_on_past=False,
retries=3,
dag=dag)
custom_events_update_load = SSHOperator(
ssh_hook=sshHook,
task_id='custom_events_update_load',
command='/data/airflow/etl/API/run_psql.sh {{ ds_nodash }} {{params.my_param }}',
params={'my_param':"custom_events_update_load"},
depends_on_past=False,
retries=3,
dag=dag)
custom_events_update_feign >> custom_events_update_load
update_scrm_contact_feign = SSHOperator(
ssh_hook=sshHook,
task_id='update_scrm_contact_feign',
command='python3 /data/airflow/etl/API/update_scrm_contact_feign.py',
depends_on_past=False,
retries=3,
dag=dag)
update_scrm_contact_load = SSHOperator(
ssh_hook=sshHook,
task_id='update_scrm_contact_load',
command='/data/airflow/etl/API/run_psql.sh {{ ds_nodash }} {{params.my_param }}',
params={'my_param':"update_scrm_contact_load"},
depends_on_past=False,
retries=3,
dag=dag)
update_scrm_contact_feign >> update_scrm_contact_load
customer_events_feign = SSHOperator(
ssh_hook=sshHook,
task_id='customer_events_feign',
command='python3 /data/airflow/etl/API/customer_events_feign.py',
depends_on_past=False,
retries=3,
dag=dag)
customer_events_load = SSHOperator(
ssh_hook=sshHook,
task_id='customer_events_load',
command='/data/airflow/etl/API/run_psql.sh {{ ds_nodash }} {{params.my_param }}',
params={'my_param':"customer_events_load"},
depends_on_past=False,
retries=3,
dag=dag)
customer_events_feign >> customer_events_load
customer_event_meta_feign = SSHOperator(
ssh_hook=sshHook,
task_id='customer_event_meta_feign',
command='python3 /data/airflow/etl/API/customer_event_meta_feign.py',
depends_on_past=False,
retries=3,
dag=dag)
customer_event_meta_load = SSHOperator(
ssh_hook=sshHook,
task_id='customer_event_meta_load',
command='/data/airflow/etl/API/run_psql.sh {{ ds_nodash }} {{params.my_param }}',
params={'my_param':"customer_event_meta_load"},
depends_on_past=False,
retries=3,
dag=dag)
customer_event_meta_feign >> customer_event_meta_load
scrm_contact_lastupdated_4112 = SSHOperator(
ssh_hook=sshHook,
task_id='scrm_contact_lastupdated_4112',
command='/data/airflow/etl/SA/run_sa.sh {{ ds_nodash }} {{ params.my_param }} >>/data/airflow/logs/run_psql_{{ds_nodash}}.log 2>&1 ',
params={'my_param':"S98_S_scrm_contact_lastupdated"},
depends_on_past=False,
retries=3,
dag=dag)
customer_event_meta_2268 = SSHOperator(
ssh_hook=sshHook,
task_id='customer_event_meta_2268',
command='/data/airflow/etl/SA/run_sa.sh {{ ds_nodash }} {{ params.my_param }} >>/data/airflow/logs/run_psql_{{ds_nodash}}.log 2>&1 ',
params={'my_param':"S98_S_customer_event_meta"},
depends_on_past=False,
retries=3,
dag=dag)
customer_events_lastupdated_8895 = SSHOperator(
ssh_hook=sshHook,
task_id='customer_events_lastupdated_8895',
command='/data/airflow/etl/SA/run_sa.sh {{ ds_nodash }} {{ params.my_param }} >>/data/airflow/logs/run_psql_{{ds_nodash}}.log 2>&1 ',
params={'my_param':"S98_S_customer_events_lastupdated"},
depends_on_past=False,
retries=3,
dag=dag)
customer_events_3292 = SSHOperator(
ssh_hook=sshHook,
task_id='customer_events_3292',
command='/data/airflow/etl/SA/run_sa.sh {{ ds_nodash }} {{ params.my_param }} >>/data/airflow/logs/run_psql_{{ds_nodash}}.log 2>&1 ',
params={'my_param':"S98_S_customer_events"},
depends_on_past=False,
retries=3,
dag=dag)
scrm_contact_load >> scrm_contact_1129
update_scrm_contact_load >> scrm_contact_lastupdated_4112
customer_event_meta_load >> customer_event_meta_2268
custom_events_update_load >> customer_events_lastupdated_8895
customer_events_load >> customer_events_3292
customer_events_3292 >> task_failed

41
TK_Cust/dev/tk_api/获取SCRM contact/scrm_contact_feign.py Normal file
View File

@ -0,0 +1,41 @@
# coding: utf-8
import requests
import json
import psycopg2
import uuid
import datetime
# 获取当前日期
current_date = datetime.date.today()
previous_date = current_date - datetime.timedelta(days=2)
formatted_current_date = current_date.strftime("%Y-%m-%dT%H:%M:%SZ")
formatted_previous_date = previous_date.strftime("%Y-%m-%dT%H:%M:%SZ")
print('开始加载数据：scrm_contact:获取SCRM contact')
authUrl='https://api.huiju.cool/v2/oauth2/token?app_id=cl037b184ebccd97c&secret=deaa83fdd8d385eec0d1d04d34282c4ec8c33b60&grant_type=client_credentials'
print('开始请求令牌。')
authRequest=requests.get(authUrl)
auth=json.loads(authRequest.text)
print('开始请求数据总数。')
url='https://api.huiju.cool/v2/customers'
header={}
body={'access_token':auth['access_token'],'sort':'dateJoin','dateJoin[ge]':formatted_previous_date,'dateJoin[le]':formatted_current_date,'limit':'',}
dataReqL=requests.get(url,headers=header,params=body)
resL=json.loads(dataReqL.text)
# print(resL)
dataList=resL['data']
total=len(dataList)
conn = psycopg2.connect(database="dataops_db", user="dbuser_dops", password="MIgTi3jA",
host="172.17.0.8", port="5432")
print('数据库连接成功')
dataId=str(uuid.uuid4())
print('临时id：'+dataId)
json_object = json.dumps(dataList)
cur=conn.cursor()
sql="update data_api.api_data set is_loaded = '1' where api_id = 'cab1a761-b8e5-4db4-a769-88db3ec1';INSERT INTO data_api.api_data (id,api_id,data,total_num,is_loaded,status,request_tm,execute_tm,remark) values (%s,%s,%s,%s,'0','0',current_timestamp(0),current_timestamp(0),'')"
cur.execute(sql,[dataId,'cab1a761-b8e5-4db4-a769-88db3ec1',json_object,total])
conn.commit()
cur.close()
conn.close()
print('加载数据结束：scrm_contact:获取SCRM contact')

105
TK_Cust/dev/tk_api/获取SCRM contact/scrm_contact_load.sql Normal file
View File

@ -0,0 +1,105 @@
/*******Main Section**************************************************************************/
\set ON_ERROR_STOP on
\set AUTOCOMMIT on
\timing on
DELETE FROM data_api.scrm_contact;
insert into data_api.scrm_contact (
img
, not_acceptsms
, gender
, create_method
, mobile_verified
, language
, is_member
, source
, is_referrer
, create_from
, last_updated
, date_created
, acceptsms
, id
, subscribe_scene
, id_str
, is_employee
, version
, not_accept_email
, date_join
, content_name
, update_method
, email_verified
, is_anonymous
, accept_email
, create_from_name
, status
,etl_tx_dt
)
select
case when trim(both from img)='' then null else img::text end img
, case when trim(both from not_acceptsms)='' then null else not_acceptsms::text end not_acceptsms
, case when trim(both from gender)='' then null else gender::text end gender
, case when trim(both from create_method)='' then null else create_method::text end create_method
, case when trim(both from mobile_verified)='' then null else mobile_verified::text end mobile_verified
, case when trim(both from language)='' then null else language::text end language
, case when trim(both from is_member)='' then null else is_member::text end is_member
, case when trim(both from source)='' then null else source::text end source
, case when trim(both from is_referrer)='' then null else is_referrer::text end is_referrer
, case when trim(both from create_from)='' then null else create_from::text end create_from
, case when trim(both from last_updated)='' then null else last_updated::text end last_updated
, case when trim(both from date_created)='' then null else date_created::text end date_created
, case when trim(both from acceptsms)='' then null else acceptsms::text end acceptsms
, case when trim(both from id)='' then null else id::text end id
, case when trim(both from subscribe_scene)='' then null else subscribe_scene::text end subscribe_scene
, case when trim(both from id_str)='' then null else id_str::text end id_str
, case when trim(both from is_employee)='' then null else is_employee::text end is_employee
, case when trim(both from version)='' then null else version::text end version
, case when trim(both from not_accept_email)='' then null else not_accept_email::text end not_accept_email
, case when trim(both from date_join)='' then null else date_join::text end date_join
, case when trim(both from content_name)='' then null else content_name::text end content_name
, case when trim(both from update_method)='' then null else update_method::text end update_method
, case when trim(both from email_verified)='' then null else email_verified::text end email_verified
, case when trim(both from is_anonymous)='' then null else is_anonymous::text end is_anonymous
, case when trim(both from accept_email)='' then null else accept_email::text end accept_email
, case when trim(both from create_from_name)='' then null else create_from_name::text end create_from_name
, case when trim(both from status)='' then null else status::text end status
,etl_tx_dt
from (
select
(json_array_elements(data::json)::json->>'img') img
, (json_array_elements(data::json)::json->>'notAcceptsms') not_acceptsms
, (json_array_elements(data::json)::json->>'gender') gender
, (json_array_elements(data::json)::json->>'createMethod') create_method
, (json_array_elements(data::json)::json->>'mobileVerified') mobile_verified
, (json_array_elements(data::json)::json->>'language') language
, (json_array_elements(data::json)::json->>'isMember') is_member
, (json_array_elements(data::json)::json->>'source') source
, (json_array_elements(data::json)::json->>'isReferrer') is_referrer
, (json_array_elements(data::json)::json->>'createFrom') create_from
, (json_array_elements(data::json)::json->>'lastUpdated') last_updated
, (json_array_elements(data::json)::json->>'dateCreated') date_created
, (json_array_elements(data::json)::json->>'acceptsms') acceptsms
, (json_array_elements(data::json)::json->>'id') id
, (json_array_elements(data::json)::json->>'subscribeScene') subscribe_scene
, (json_array_elements(data::json)::json->>'idStr') id_str
, (json_array_elements(data::json)::json->>'isEmployee') is_employee
, (json_array_elements(data::json)::json->>'version') version
, (json_array_elements(data::json)::json->>'notAcceptEmail') not_accept_email
, (json_array_elements(data::json)::json->>'dateJoin') date_join
, (json_array_elements(data::json)::json->>'contentName') content_name
, (json_array_elements(data::json)::json->>'updateMethod') update_method
, (json_array_elements(data::json)::json->>'emailVerified') email_verified
, (json_array_elements(data::json)::json->>'isAnonymous') is_anonymous
, (json_array_elements(data::json)::json->>'acceptEmail') accept_email
, (json_array_elements(data::json)::json->>'createFromName') create_from_name
, (json_array_elements(data::json)::json->>'status') status
,CURRENT_TIMESTAMP(0) etl_tx_dt
from (select * from data_api.api_data
WHERE api_id='cab1a761-b8e5-4db4-a769-88db3ec1' and is_loaded = '0' order by request_tm desc limit 1) p )p;
update data_api.api_data
set is_loaded = '1' ,
status = '1',
request_tm = current_timestamp(0)
where api_id='cab1a761-b8e5-4db4-a769-88db3ec1';
\q

41
TK_Cust/dev/tk_api/获取线索事件/customer_events_feign.py Normal file
View File

@ -0,0 +1,41 @@
# coding: utf-8
import requests
import json
import psycopg2
import uuid
import datetime
# 获取当前日期
current_date = datetime.date.today()
previous_date = current_date - datetime.timedelta(days=2)
formatted_current_date = current_date.strftime("%Y-%m-%dT%H:%M:%SZ")
formatted_previous_date = previous_date.strftime("%Y-%m-%dT%H:%M:%SZ")
print('开始加载数据：customer_events:获取线索事件')
authUrl='https://api.huiju.cool/v2/oauth2/token?app_id=cl037b184ebccd97c&secret=deaa83fdd8d385eec0d1d04d34282c4ec8c33b60&grant_type=client_credentials'
print('开始请求令牌。')
authRequest=requests.get(authUrl)
auth=json.loads(authRequest.text)
print('开始请求数据总数。')
url='https://api.huiju.cool/v2/customerEvents'
header={}
body={'access_token':auth['access_token'],'limit':'','sort':'date','event':'create_customer','date[ge]':formatted_previous_date,'date[le]':formatted_current_date,}
dataReqL=requests.get(url,headers=header,params=body)
resL=json.loads(dataReqL.text)
# print(resL)
dataList=resL['data']
total=len(dataList)
conn = psycopg2.connect(database="dataops_db", user="dbuser_dops", password="MIgTi3jA",
host="172.17.0.8", port="5432")
print('数据库连接成功')
dataId=str(uuid.uuid4())
print('临时id：'+dataId)
json_object = json.dumps(dataList)
cur=conn.cursor()
sql="update data_api.api_data set is_loaded = '1' where api_id = '747d33fa-a0e0-421d-aa9b-4ca4517b';INSERT INTO data_api.api_data (id,api_id,data,total_num,is_loaded,status,request_tm,execute_tm,remark) values (%s,%s,%s,%s,'0','0',current_timestamp(0),current_timestamp(0),'')"
cur.execute(sql,[dataId,'747d33fa-a0e0-421d-aa9b-4ca4517b',json_object,total])
conn.commit()
cur.close()
conn.close()
print('加载数据结束：customer_events:获取线索事件')

57
TK_Cust/dev/tk_api/获取线索事件/customer_events_load.sql Normal file
View File

@ -0,0 +1,57 @@
/*******Main Section**************************************************************************/
\set ON_ERROR_STOP on
\set AUTOCOMMIT on
\timing on
DELETE FROM data_api.customer_events;
insert into data_api.customer_events (
id_str
, date
, last_updated
, customer_id_str
, create_method
, customer_id
, external_id
, id
, source
, event
, content_name
,etl_tx_dt
)
select
case when trim(both from id_str)='' then null else id_str::text end id_str
, case when trim(both from date)='' then null else date::text end date
, case when trim(both from last_updated)='' then null else last_updated::text end last_updated
, case when trim(both from customer_id_str)='' then null else customer_id_str::text end customer_id_str
, case when trim(both from create_method)='' then null else create_method::text end create_method
, case when trim(both from customer_id)='' then null else customer_id::text end customer_id
, case when trim(both from external_id)='' then null else external_id::text end external_id
, case when trim(both from id)='' then null else id::text end id
, case when trim(both from source)='' then null else source::text end source
, case when trim(both from event)='' then null else event::text end event
, case when trim(both from content_name)='' then null else content_name::text end content_name
,etl_tx_dt
from (
select
(json_array_elements(data::json)::json->>'idStr') id_str
, (json_array_elements(data::json)::json->>'date') date
, (json_array_elements(data::json)::json->>'lastUpdated') last_updated
, (json_array_elements(data::json)::json->>'customerIdStr') customer_id_str
, (json_array_elements(data::json)::json->>'createMethod') create_method
, (json_array_elements(data::json)::json->>'customerId') customer_id
, (json_array_elements(data::json)::json->>'externalId') external_id
, (json_array_elements(data::json)::json->>'id') id
, (json_array_elements(data::json)::json->>'source') source
, (json_array_elements(data::json)::json->>'event') event
, (json_array_elements(data::json)::json->>'contentName') content_name
,CURRENT_TIMESTAMP(0) etl_tx_dt
from (select * from data_api.api_data
WHERE api_id='747d33fa-a0e0-421d-aa9b-4ca4517b' and is_loaded = '0' order by request_tm desc limit 1) p )p;
update data_api.api_data
set is_loaded = '1' ,
status = '1',
request_tm = current_timestamp(0)
where api_id='747d33fa-a0e0-421d-aa9b-4ca4517b';
\q

41
TK_Cust/dev/tk_api/获取线索事件元数据/customer_event_meta_feign.py Normal file
View File

@ -0,0 +1,41 @@
# coding: utf-8
import requests
import json
import psycopg2
import uuid
import datetime
# 获取当前日期
current_date = datetime.date.today()
previous_date = current_date - datetime.timedelta(days=2)
formatted_current_date = current_date.strftime("%Y-%m-%dT%H:%M:%SZ")
formatted_previous_date = previous_date.strftime("%Y-%m-%dT%H:%M:%SZ")
print('开始加载数据：customer_event_meta:获取线索事件元数据')
authUrl='https://api.huiju.cool/v2/oauth2/token?app_id=cl037b184ebccd97c&secret=deaa83fdd8d385eec0d1d04d34282c4ec8c33b60&grant_type=client_credentials'
print('开始请求令牌。')
authRequest=requests.get(authUrl)
auth=json.loads(authRequest.text)
print('开始请求数据总数。')
url='https://api.huiju.cool/v2/metaService/getCustomerEventMeta'
header={}
body={'access_token':auth['access_token'],}
dataReqL=requests.get(url,headers=header,params=body)
resL=json.loads(dataReqL.text)
# print(resL)
dataList=resL['data']
total=len(dataList)
conn = psycopg2.connect(database="dataops_db", user="dbuser_dops", password="MIgTi3jA",
host="172.17.0.8", port="5432")
print('数据库连接成功')
dataId=str(uuid.uuid4())
print('临时id：'+dataId)
json_object = json.dumps(dataList)
cur=conn.cursor()
sql="update data_api.api_data set is_loaded = '1' where api_id = '057169de-4aa7-4f73-8894-221b4fd0';INSERT INTO data_api.api_data (id,api_id,data,total_num,is_loaded,status,request_tm,execute_tm,remark) values (%s,%s,%s,%s,'0','0',current_timestamp(0),current_timestamp(0),'')"
cur.execute(sql,[dataId,'057169de-4aa7-4f73-8894-221b4fd0',json_object,total])
conn.commit()
cur.close()
conn.close()
print('加载数据结束：customer_event_meta:获取线索事件元数据')

33
TK_Cust/dev/tk_api/获取线索事件元数据/customer_event_meta_load.sql Normal file
View File

@ -0,0 +1,33 @@
/*******Main Section**************************************************************************/
\set ON_ERROR_STOP on
\set AUTOCOMMIT on
\timing on
DELETE FROM data_api.customer_event_meta;
insert into data_api.customer_event_meta (
group_name
, group_label
, events
,etl_tx_dt
)
select
case when trim(both from group_name)='' then null else group_name::text end group_name
, case when trim(both from group_label)='' then null else group_label::text end group_label
, case when trim(both from events)='' then null else events::text end events
,etl_tx_dt
from (
select
(json_array_elements(data::json)::json->>'groupName') group_name
, (json_array_elements(data::json)::json->>'groupLabel') group_label
, (json_array_elements(data::json)::json->>'events') events
,CURRENT_TIMESTAMP(0) etl_tx_dt
from (select * from data_api.api_data
WHERE api_id='057169de-4aa7-4f73-8894-221b4fd0' and is_loaded = '0' order by request_tm desc limit 1) p )p;
update data_api.api_data
set is_loaded = '1' ,
status = '1',
request_tm = current_timestamp(0)
where api_id='057169de-4aa7-4f73-8894-221b4fd0';
\q