diff --git a/dev/workflow/TK_Cust/tk_api_2/customer_event_meta/S98_S_customer_event_meta.sql b/dev/workflow/TK_Cust/tk_api_2/customer_event_meta/S98_S_customer_event_meta.sql new file mode 100644 index 0000000..16c1069 --- /dev/null +++ b/dev/workflow/TK_Cust/tk_api_2/customer_event_meta/S98_S_customer_event_meta.sql @@ -0,0 +1,34 @@ +/*******Main Section**************************************************************************/ +\set ON_ERROR_STOP on +\set AUTOCOMMIT on +\timing on +delete from p10_sa.S98_S_customer_event_meta +; +insert into p10_sa.S98_S_customer_event_meta +( events + , group_label + , group_name + , etl_tx_dt ) + select + events + , group_label + , group_name + , etl_tx_dt + from p00_tal.S98_S_customer_event_meta + ; + delete from p12_sfull.S98_S_customer_event_meta +; +; +insert into p12_sfull.S98_S_customer_event_meta +( events + , group_label + , group_name + , etl_tx_dt ) + select + events + , group_label + , group_name + , etl_tx_dt + from p10_sa.S98_S_customer_event_meta +; +\q \ No newline at end of file diff --git a/dev/workflow/TK_Cust/tk_api_2/customer_event_meta/sa_foreign_tables.sql b/dev/workflow/TK_Cust/tk_api_2/customer_event_meta/sa_foreign_tables.sql new file mode 100644 index 0000000..7c2e653 --- /dev/null +++ b/dev/workflow/TK_Cust/tk_api_2/customer_event_meta/sa_foreign_tables.sql @@ -0,0 +1,15 @@ + +CREATE FOREIGN TABLE if not exists p00_tal.S98_S_customer_event_meta ( + events TEXT + , group_label TEXT + , group_name TEXT + , etl_tx_dt TIMESTAMP +) + + +SERVER pgsql_server_S98_S OPTIONS(schema_name 'data_api', table_name 'customer_event_meta' ); + + + + + diff --git a/dev/workflow/TK_Cust/tk_api_2/customer_event_meta/sa_tables.sql b/dev/workflow/TK_Cust/tk_api_2/customer_event_meta/sa_tables.sql new file mode 100644 index 0000000..ef3dfa5 --- /dev/null +++ b/dev/workflow/TK_Cust/tk_api_2/customer_event_meta/sa_tables.sql @@ -0,0 +1,31 @@ + +create table if not exists p10_sa.S98_S_customer_event_meta ( + events TEXT + , group_label TEXT + , group_name TEXT + , etl_tx_dt TIMESTAMP +) ; + + COMMENT ON COLUMN p10_sa.S98_S_customer_event_meta.events IS ''; + COMMENT ON COLUMN p10_sa.S98_S_customer_event_meta.group_label IS ''; + COMMENT ON COLUMN p10_sa.S98_S_customer_event_meta.group_name IS ''; + COMMENT ON COLUMN p10_sa.S98_S_customer_event_meta.etl_tx_dt IS ''; + +COMMENT ON TABLE p10_sa.S98_S_customer_event_meta IS ''; + + + +create table if not exists p12_sfull.S98_S_customer_event_meta ( + events TEXT + , group_label TEXT + , group_name TEXT + , etl_tx_dt TIMESTAMP +) ; + + COMMENT ON COLUMN p12_sfull.S98_S_customer_event_meta.events IS ''; + COMMENT ON COLUMN p12_sfull.S98_S_customer_event_meta.group_label IS ''; + COMMENT ON COLUMN p12_sfull.S98_S_customer_event_meta.group_name IS ''; + COMMENT ON COLUMN p12_sfull.S98_S_customer_event_meta.etl_tx_dt IS ''; + +COMMENT ON TABLE p12_sfull.S98_S_customer_event_meta IS ''; + diff --git a/dev/workflow/TK_Cust/tk_api_2/customer_events/S98_S_customer_events.sql b/dev/workflow/TK_Cust/tk_api_2/customer_events/S98_S_customer_events.sql new file mode 100644 index 0000000..621ed13 --- /dev/null +++ b/dev/workflow/TK_Cust/tk_api_2/customer_events/S98_S_customer_events.sql @@ -0,0 +1,66 @@ +/*******Main Section**************************************************************************/ +\set ON_ERROR_STOP on +\set AUTOCOMMIT on +\timing on +delete from p10_sa.S98_S_customer_events +; +insert into p10_sa.S98_S_customer_events +( id + , customer_id + , event + , last_updated + , source + , content_name + , date + , create_method + , customer_id_str + , external_id + , id_str + , etl_tx_dt ) + select + id + , customer_id + , event + , last_updated + , source + , content_name + , date + , create_method + , customer_id_str + , external_id + , id_str + , etl_tx_dt + from p00_tal.S98_S_customer_events + ; + delete from p12_sfull.S98_S_customer_events +; +; +insert into p12_sfull.S98_S_customer_events +( id + , customer_id + , event + , last_updated + , source + , content_name + , date + , create_method + , customer_id_str + , external_id + , id_str + , etl_tx_dt ) + select + id + , customer_id + , event + , last_updated + , source + , content_name + , date + , create_method + , customer_id_str + , external_id + , id_str + , etl_tx_dt + from p10_sa.S98_S_customer_events +; +\q \ No newline at end of file diff --git a/dev/workflow/TK_Cust/tk_api_2/customer_events/sa_foreign_tables.sql b/dev/workflow/TK_Cust/tk_api_2/customer_events/sa_foreign_tables.sql new file mode 100644 index 0000000..d58b150 --- /dev/null +++ b/dev/workflow/TK_Cust/tk_api_2/customer_events/sa_foreign_tables.sql @@ -0,0 +1,23 @@ + +CREATE FOREIGN TABLE if not exists p00_tal.S98_S_customer_events ( + id TEXT + , customer_id TEXT + , event TEXT + , last_updated TEXT + , source TEXT + , content_name TEXT + , date TEXT + , create_method TEXT + , customer_id_str TEXT + , external_id TEXT + , id_str TEXT + , etl_tx_dt TIMESTAMP +) + + +SERVER pgsql_server_S98_S OPTIONS(schema_name 'data_api', table_name 'customer_events' ); + + + + + diff --git a/dev/workflow/TK_Cust/tk_api_2/customer_events/sa_tables.sql b/dev/workflow/TK_Cust/tk_api_2/customer_events/sa_tables.sql new file mode 100644 index 0000000..f596132 --- /dev/null +++ b/dev/workflow/TK_Cust/tk_api_2/customer_events/sa_tables.sql @@ -0,0 +1,63 @@ + +create table if not exists p10_sa.S98_S_customer_events ( + id TEXT + , customer_id TEXT + , event TEXT + , last_updated TEXT + , source TEXT + , content_name TEXT + , date TEXT + , create_method TEXT + , customer_id_str TEXT + , external_id TEXT + , id_str TEXT + , etl_tx_dt TIMESTAMP +) ; + + COMMENT ON COLUMN p10_sa.S98_S_customer_events.id IS ''; + COMMENT ON COLUMN p10_sa.S98_S_customer_events.customer_id IS ''; + COMMENT ON COLUMN p10_sa.S98_S_customer_events.event IS ''; + COMMENT ON COLUMN p10_sa.S98_S_customer_events.last_updated IS ''; + COMMENT ON COLUMN p10_sa.S98_S_customer_events.source IS ''; + COMMENT ON COLUMN p10_sa.S98_S_customer_events.content_name IS ''; + COMMENT ON COLUMN p10_sa.S98_S_customer_events.date IS ''; + COMMENT ON COLUMN p10_sa.S98_S_customer_events.create_method IS ''; + COMMENT ON COLUMN p10_sa.S98_S_customer_events.customer_id_str IS ''; + COMMENT ON COLUMN p10_sa.S98_S_customer_events.external_id IS ''; + COMMENT ON COLUMN p10_sa.S98_S_customer_events.id_str IS ''; + COMMENT ON COLUMN p10_sa.S98_S_customer_events.etl_tx_dt IS ''; + +COMMENT ON TABLE p10_sa.S98_S_customer_events IS ''; + + + +create table if not exists p12_sfull.S98_S_customer_events ( + id TEXT + , customer_id TEXT + , event TEXT + , last_updated TEXT + , source TEXT + , content_name TEXT + , date TEXT + , create_method TEXT + , customer_id_str TEXT + , external_id TEXT + , id_str TEXT + , etl_tx_dt TIMESTAMP +) ; + + COMMENT ON COLUMN p12_sfull.S98_S_customer_events.id IS ''; + COMMENT ON COLUMN p12_sfull.S98_S_customer_events.customer_id IS ''; + COMMENT ON COLUMN p12_sfull.S98_S_customer_events.event IS ''; + COMMENT ON COLUMN p12_sfull.S98_S_customer_events.last_updated IS ''; + COMMENT ON COLUMN p12_sfull.S98_S_customer_events.source IS ''; + COMMENT ON COLUMN p12_sfull.S98_S_customer_events.content_name IS ''; + COMMENT ON COLUMN p12_sfull.S98_S_customer_events.date IS ''; + COMMENT ON COLUMN p12_sfull.S98_S_customer_events.create_method IS ''; + COMMENT ON COLUMN p12_sfull.S98_S_customer_events.customer_id_str IS ''; + COMMENT ON COLUMN p12_sfull.S98_S_customer_events.external_id IS ''; + COMMENT ON COLUMN p12_sfull.S98_S_customer_events.id_str IS ''; + COMMENT ON COLUMN p12_sfull.S98_S_customer_events.etl_tx_dt IS ''; + +COMMENT ON TABLE p12_sfull.S98_S_customer_events IS ''; + diff --git a/dev/workflow/TK_Cust/tk_api_2/customer_events_open_content_page/S98_S_customer_events_open_content_page.sql b/dev/workflow/TK_Cust/tk_api_2/customer_events_open_content_page/S98_S_customer_events_open_content_page.sql new file mode 100644 index 0000000..71ac8ee --- /dev/null +++ b/dev/workflow/TK_Cust/tk_api_2/customer_events_open_content_page/S98_S_customer_events_open_content_page.sql @@ -0,0 +1,126 @@ +/*******Main Section**************************************************************************/ +\set ON_ERROR_STOP on +\set AUTOCOMMIT on +\timing on +delete from p10_sa.S98_S_customer_events_open_content_page +; +insert into p10_sa.S98_S_customer_events_open_content_page +( id + , customer_id + , event + , last_updated + , date + , browser + , browser_version + , customer_id_str + , device + , external_id + , id_str + , ip + , ip_city + , ip_country + , ip_province + , os + , os_version + , page_id + , page_type + , platform + , screen_height + , screen_width + , short_id + , target_id + , target_name + , url + , etl_tx_dt ) + select + id + , customer_id + , event + , last_updated + , date + , browser + , browser_version + , customer_id_str + , device + , external_id + , id_str + , ip + , ip_city + , ip_country + , ip_province + , os + , os_version + , page_id + , page_type + , platform + , screen_height + , screen_width + , short_id + , target_id + , target_name + , url + , etl_tx_dt + from p00_tal.S98_S_customer_events_open_content_page + ; + delete from p12_sfull.S98_S_customer_events_open_content_page +; +; +insert into p12_sfull.S98_S_customer_events_open_content_page +( id + , customer_id + , event + , last_updated + , date + , browser + , browser_version + , customer_id_str + , device + , external_id + , id_str + , ip + , ip_city + , ip_country + , ip_province + , os + , os_version + , page_id + , page_type + , platform + , screen_height + , screen_width + , short_id + , target_id + , target_name + , url + , etl_tx_dt ) + select + id + , customer_id + , event + , last_updated + , date + , browser + , browser_version + , customer_id_str + , device + , external_id + , id_str + , ip + , ip_city + , ip_country + , ip_province + , os + , os_version + , page_id + , page_type + , platform + , screen_height + , screen_width + , short_id + , target_id + , target_name + , url + , etl_tx_dt + from p10_sa.S98_S_customer_events_open_content_page +; +\q \ No newline at end of file diff --git a/dev/workflow/TK_Cust/tk_api_2/customer_events_open_content_page/sa_foreign_tables.sql b/dev/workflow/TK_Cust/tk_api_2/customer_events_open_content_page/sa_foreign_tables.sql new file mode 100644 index 0000000..efe7004 --- /dev/null +++ b/dev/workflow/TK_Cust/tk_api_2/customer_events_open_content_page/sa_foreign_tables.sql @@ -0,0 +1,38 @@ + +CREATE FOREIGN TABLE if not exists p00_tal.S98_S_customer_events_open_content_page ( + id TEXT + , customer_id TEXT + , event TEXT + , last_updated TEXT + , date TEXT + , browser TEXT + , browser_version TEXT + , customer_id_str TEXT + , device TEXT + , external_id TEXT + , id_str TEXT + , ip TEXT + , ip_city TEXT + , ip_country TEXT + , ip_province TEXT + , os TEXT + , os_version TEXT + , page_id TEXT + , page_type TEXT + , platform TEXT + , screen_height TEXT + , screen_width TEXT + , short_id TEXT + , target_id TEXT + , target_name TEXT + , url TEXT + , etl_tx_dt TIMESTAMP +) + + +SERVER pgsql_server_S98_S OPTIONS(schema_name 'data_api', table_name 'customer_events_open_content_page' ); + + + + + diff --git a/dev/workflow/TK_Cust/tk_api_2/customer_events_open_content_page/sa_tables.sql b/dev/workflow/TK_Cust/tk_api_2/customer_events_open_content_page/sa_tables.sql new file mode 100644 index 0000000..ceab08a --- /dev/null +++ b/dev/workflow/TK_Cust/tk_api_2/customer_events_open_content_page/sa_tables.sql @@ -0,0 +1,123 @@ + +create table if not exists p10_sa.S98_S_customer_events_open_content_page ( + id TEXT + , customer_id TEXT + , event TEXT + , last_updated TEXT + , date TEXT + , browser TEXT + , browser_version TEXT + , customer_id_str TEXT + , device TEXT + , external_id TEXT + , id_str TEXT + , ip TEXT + , ip_city TEXT + , ip_country TEXT + , ip_province TEXT + , os TEXT + , os_version TEXT + , page_id TEXT + , page_type TEXT + , platform TEXT + , screen_height TEXT + , screen_width TEXT + , short_id TEXT + , target_id TEXT + , target_name TEXT + , url TEXT + , etl_tx_dt TIMESTAMP +) ; + + COMMENT ON COLUMN p10_sa.S98_S_customer_events_open_content_page.id IS ''; + COMMENT ON COLUMN p10_sa.S98_S_customer_events_open_content_page.customer_id IS ''; + COMMENT ON COLUMN p10_sa.S98_S_customer_events_open_content_page.event IS ''; + COMMENT ON COLUMN p10_sa.S98_S_customer_events_open_content_page.last_updated IS ''; + COMMENT ON COLUMN p10_sa.S98_S_customer_events_open_content_page.date IS ''; + COMMENT ON COLUMN p10_sa.S98_S_customer_events_open_content_page.browser IS ''; + COMMENT ON COLUMN p10_sa.S98_S_customer_events_open_content_page.browser_version IS ''; + COMMENT ON COLUMN p10_sa.S98_S_customer_events_open_content_page.customer_id_str IS ''; + COMMENT ON COLUMN p10_sa.S98_S_customer_events_open_content_page.device IS ''; + COMMENT ON COLUMN p10_sa.S98_S_customer_events_open_content_page.external_id IS ''; + COMMENT ON COLUMN p10_sa.S98_S_customer_events_open_content_page.id_str IS ''; + COMMENT ON COLUMN p10_sa.S98_S_customer_events_open_content_page.ip IS ''; + COMMENT ON COLUMN p10_sa.S98_S_customer_events_open_content_page.ip_city IS ''; + COMMENT ON COLUMN p10_sa.S98_S_customer_events_open_content_page.ip_country IS ''; + COMMENT ON COLUMN p10_sa.S98_S_customer_events_open_content_page.ip_province IS ''; + COMMENT ON COLUMN p10_sa.S98_S_customer_events_open_content_page.os IS ''; + COMMENT ON COLUMN p10_sa.S98_S_customer_events_open_content_page.os_version IS ''; + COMMENT ON COLUMN p10_sa.S98_S_customer_events_open_content_page.page_id IS ''; + COMMENT ON COLUMN p10_sa.S98_S_customer_events_open_content_page.page_type IS ''; + COMMENT ON COLUMN p10_sa.S98_S_customer_events_open_content_page.platform IS ''; + COMMENT ON COLUMN p10_sa.S98_S_customer_events_open_content_page.screen_height IS ''; + COMMENT ON COLUMN p10_sa.S98_S_customer_events_open_content_page.screen_width IS ''; + COMMENT ON COLUMN p10_sa.S98_S_customer_events_open_content_page.short_id IS ''; + COMMENT ON COLUMN p10_sa.S98_S_customer_events_open_content_page.target_id IS ''; + COMMENT ON COLUMN p10_sa.S98_S_customer_events_open_content_page.target_name IS ''; + COMMENT ON COLUMN p10_sa.S98_S_customer_events_open_content_page.url IS ''; + COMMENT ON COLUMN p10_sa.S98_S_customer_events_open_content_page.etl_tx_dt IS ''; + +COMMENT ON TABLE p10_sa.S98_S_customer_events_open_content_page IS ''; + + + +create table if not exists p12_sfull.S98_S_customer_events_open_content_page ( + id TEXT + , customer_id TEXT + , event TEXT + , last_updated TEXT + , date TEXT + , browser TEXT + , browser_version TEXT + , customer_id_str TEXT + , device TEXT + , external_id TEXT + , id_str TEXT + , ip TEXT + , ip_city TEXT + , ip_country TEXT + , ip_province TEXT + , os TEXT + , os_version TEXT + , page_id TEXT + , page_type TEXT + , platform TEXT + , screen_height TEXT + , screen_width TEXT + , short_id TEXT + , target_id TEXT + , target_name TEXT + , url TEXT + , etl_tx_dt TIMESTAMP +) ; + + COMMENT ON COLUMN p12_sfull.S98_S_customer_events_open_content_page.id IS ''; + COMMENT ON COLUMN p12_sfull.S98_S_customer_events_open_content_page.customer_id IS ''; + COMMENT ON COLUMN p12_sfull.S98_S_customer_events_open_content_page.event IS ''; + COMMENT ON COLUMN p12_sfull.S98_S_customer_events_open_content_page.last_updated IS ''; + COMMENT ON COLUMN p12_sfull.S98_S_customer_events_open_content_page.date IS ''; + COMMENT ON COLUMN p12_sfull.S98_S_customer_events_open_content_page.browser IS ''; + COMMENT ON COLUMN p12_sfull.S98_S_customer_events_open_content_page.browser_version IS ''; + COMMENT ON COLUMN p12_sfull.S98_S_customer_events_open_content_page.customer_id_str IS ''; + COMMENT ON COLUMN p12_sfull.S98_S_customer_events_open_content_page.device IS ''; + COMMENT ON COLUMN p12_sfull.S98_S_customer_events_open_content_page.external_id IS ''; + COMMENT ON COLUMN p12_sfull.S98_S_customer_events_open_content_page.id_str IS ''; + COMMENT ON COLUMN p12_sfull.S98_S_customer_events_open_content_page.ip IS ''; + COMMENT ON COLUMN p12_sfull.S98_S_customer_events_open_content_page.ip_city IS ''; + COMMENT ON COLUMN p12_sfull.S98_S_customer_events_open_content_page.ip_country IS ''; + COMMENT ON COLUMN p12_sfull.S98_S_customer_events_open_content_page.ip_province IS ''; + COMMENT ON COLUMN p12_sfull.S98_S_customer_events_open_content_page.os IS ''; + COMMENT ON COLUMN p12_sfull.S98_S_customer_events_open_content_page.os_version IS ''; + COMMENT ON COLUMN p12_sfull.S98_S_customer_events_open_content_page.page_id IS ''; + COMMENT ON COLUMN p12_sfull.S98_S_customer_events_open_content_page.page_type IS ''; + COMMENT ON COLUMN p12_sfull.S98_S_customer_events_open_content_page.platform IS ''; + COMMENT ON COLUMN p12_sfull.S98_S_customer_events_open_content_page.screen_height IS ''; + COMMENT ON COLUMN p12_sfull.S98_S_customer_events_open_content_page.screen_width IS ''; + COMMENT ON COLUMN p12_sfull.S98_S_customer_events_open_content_page.short_id IS ''; + COMMENT ON COLUMN p12_sfull.S98_S_customer_events_open_content_page.target_id IS ''; + COMMENT ON COLUMN p12_sfull.S98_S_customer_events_open_content_page.target_name IS ''; + COMMENT ON COLUMN p12_sfull.S98_S_customer_events_open_content_page.url IS ''; + COMMENT ON COLUMN p12_sfull.S98_S_customer_events_open_content_page.etl_tx_dt IS ''; + +COMMENT ON TABLE p12_sfull.S98_S_customer_events_open_content_page IS ''; + diff --git a/dev/workflow/TK_Cust/tk_api_2/customer_events_open_page/S98_S_customer_events_open_page.sql b/dev/workflow/TK_Cust/tk_api_2/customer_events_open_page/S98_S_customer_events_open_page.sql new file mode 100644 index 0000000..1a8a35c --- /dev/null +++ b/dev/workflow/TK_Cust/tk_api_2/customer_events_open_page/S98_S_customer_events_open_page.sql @@ -0,0 +1,142 @@ +/*******Main Section**************************************************************************/ +\set ON_ERROR_STOP on +\set AUTOCOMMIT on +\timing on +delete from p10_sa.S98_S_customer_events_open_page +; +insert into p10_sa.S98_S_customer_events_open_page +( id + , customer_id + , event + , last_updated + , date + , browser + , browser_version + , customer_id_str + , device + , domain + , external_id + , id_str + , ip + , ip_city + , ip_country + , ip_province + , target_name + , os + , os_version + , page_id + , page_type + , path + , platform + , root_domain + , score + , screen_height + , screen_width + , short_id + , target_id + , url + , etl_tx_dt ) + select + id + , customer_id + , event + , last_updated + , date + , browser + , browser_version + , customer_id_str + , device + , domain + , external_id + , id_str + , ip + , ip_city + , ip_country + , ip_province + , target_name + , os + , os_version + , page_id + , page_type + , path + , platform + , root_domain + , score + , screen_height + , screen_width + , short_id + , target_id + , url + , etl_tx_dt + from p00_tal.S98_S_customer_events_open_page + ; + delete from p12_sfull.S98_S_customer_events_open_page +; +; +insert into p12_sfull.S98_S_customer_events_open_page +( id + , customer_id + , event + , last_updated + , date + , browser + , browser_version + , customer_id_str + , device + , domain + , external_id + , id_str + , ip + , ip_city + , ip_country + , ip_province + , target_name + , os + , os_version + , page_id + , page_type + , path + , platform + , root_domain + , score + , screen_height + , screen_width + , short_id + , target_id + , url + , etl_tx_dt ) + select + id + , customer_id + , event + , last_updated + , date + , browser + , browser_version + , customer_id_str + , device + , domain + , external_id + , id_str + , ip + , ip_city + , ip_country + , ip_province + , target_name + , os + , os_version + , page_id + , page_type + , path + , platform + , root_domain + , score + , screen_height + , screen_width + , short_id + , target_id + , url + , etl_tx_dt + from p10_sa.S98_S_customer_events_open_page +; +\q \ No newline at end of file diff --git a/dev/workflow/TK_Cust/tk_api_2/customer_events_open_page/sa_foreign_tables.sql b/dev/workflow/TK_Cust/tk_api_2/customer_events_open_page/sa_foreign_tables.sql new file mode 100644 index 0000000..18e9910 --- /dev/null +++ b/dev/workflow/TK_Cust/tk_api_2/customer_events_open_page/sa_foreign_tables.sql @@ -0,0 +1,42 @@ + +CREATE FOREIGN TABLE if not exists p00_tal.S98_S_customer_events_open_page ( + id TEXT + , customer_id TEXT + , event TEXT + , last_updated TEXT + , date TEXT + , browser TEXT + , browser_version TEXT + , customer_id_str TEXT + , device TEXT + , domain TEXT + , external_id TEXT + , id_str TEXT + , ip TEXT + , ip_city TEXT + , ip_country TEXT + , ip_province TEXT + , target_name TEXT + , os TEXT + , os_version TEXT + , page_id TEXT + , page_type TEXT + , path TEXT + , platform TEXT + , root_domain TEXT + , score TEXT + , screen_height TEXT + , screen_width TEXT + , short_id TEXT + , target_id TEXT + , url TEXT + , etl_tx_dt TIMESTAMP +) + + +SERVER pgsql_server_S98_S OPTIONS(schema_name 'data_api', table_name 'customer_events_open_page' ); + + + + + diff --git a/dev/workflow/TK_Cust/tk_api_2/customer_events_open_page/sa_tables.sql b/dev/workflow/TK_Cust/tk_api_2/customer_events_open_page/sa_tables.sql new file mode 100644 index 0000000..e47717e --- /dev/null +++ b/dev/workflow/TK_Cust/tk_api_2/customer_events_open_page/sa_tables.sql @@ -0,0 +1,139 @@ + +create table if not exists p10_sa.S98_S_customer_events_open_page ( + id TEXT + , customer_id TEXT + , event TEXT + , last_updated TEXT + , date TEXT + , browser TEXT + , browser_version TEXT + , customer_id_str TEXT + , device TEXT + , domain TEXT + , external_id TEXT + , id_str TEXT + , ip TEXT + , ip_city TEXT + , ip_country TEXT + , ip_province TEXT + , target_name TEXT + , os TEXT + , os_version TEXT + , page_id TEXT + , page_type TEXT + , path TEXT + , platform TEXT + , root_domain TEXT + , score TEXT + , screen_height TEXT + , screen_width TEXT + , short_id TEXT + , target_id TEXT + , url TEXT + , etl_tx_dt TIMESTAMP +) ; + + COMMENT ON COLUMN p10_sa.S98_S_customer_events_open_page.id IS ''; + COMMENT ON COLUMN p10_sa.S98_S_customer_events_open_page.customer_id IS ''; + COMMENT ON COLUMN p10_sa.S98_S_customer_events_open_page.event IS ''; + COMMENT ON COLUMN p10_sa.S98_S_customer_events_open_page.last_updated IS ''; + COMMENT ON COLUMN p10_sa.S98_S_customer_events_open_page.date IS ''; + COMMENT ON COLUMN p10_sa.S98_S_customer_events_open_page.browser IS ''; + COMMENT ON COLUMN p10_sa.S98_S_customer_events_open_page.browser_version IS ''; + COMMENT ON COLUMN p10_sa.S98_S_customer_events_open_page.customer_id_str IS ''; + COMMENT ON COLUMN p10_sa.S98_S_customer_events_open_page.device IS ''; + COMMENT ON COLUMN p10_sa.S98_S_customer_events_open_page.domain IS ''; + COMMENT ON COLUMN p10_sa.S98_S_customer_events_open_page.external_id IS ''; + COMMENT ON COLUMN p10_sa.S98_S_customer_events_open_page.id_str IS ''; + COMMENT ON COLUMN p10_sa.S98_S_customer_events_open_page.ip IS ''; + COMMENT ON COLUMN p10_sa.S98_S_customer_events_open_page.ip_city IS ''; + COMMENT ON COLUMN p10_sa.S98_S_customer_events_open_page.ip_country IS ''; + COMMENT ON COLUMN p10_sa.S98_S_customer_events_open_page.ip_province IS ''; + COMMENT ON COLUMN p10_sa.S98_S_customer_events_open_page.target_name IS ''; + COMMENT ON COLUMN p10_sa.S98_S_customer_events_open_page.os IS ''; + COMMENT ON COLUMN p10_sa.S98_S_customer_events_open_page.os_version IS ''; + COMMENT ON COLUMN p10_sa.S98_S_customer_events_open_page.page_id IS ''; + COMMENT ON COLUMN p10_sa.S98_S_customer_events_open_page.page_type IS ''; + COMMENT ON COLUMN p10_sa.S98_S_customer_events_open_page.path IS ''; + COMMENT ON COLUMN p10_sa.S98_S_customer_events_open_page.platform IS ''; + COMMENT ON COLUMN p10_sa.S98_S_customer_events_open_page.root_domain IS ''; + COMMENT ON COLUMN p10_sa.S98_S_customer_events_open_page.score IS ''; + COMMENT ON COLUMN p10_sa.S98_S_customer_events_open_page.screen_height IS ''; + COMMENT ON COLUMN p10_sa.S98_S_customer_events_open_page.screen_width IS ''; + COMMENT ON COLUMN p10_sa.S98_S_customer_events_open_page.short_id IS ''; + COMMENT ON COLUMN p10_sa.S98_S_customer_events_open_page.target_id IS ''; + COMMENT ON COLUMN p10_sa.S98_S_customer_events_open_page.url IS ''; + COMMENT ON COLUMN p10_sa.S98_S_customer_events_open_page.etl_tx_dt IS ''; + +COMMENT ON TABLE p10_sa.S98_S_customer_events_open_page IS ''; + + + +create table if not exists p12_sfull.S98_S_customer_events_open_page ( + id TEXT + , customer_id TEXT + , event TEXT + , last_updated TEXT + , date TEXT + , browser TEXT + , browser_version TEXT + , customer_id_str TEXT + , device TEXT + , domain TEXT + , external_id TEXT + , id_str TEXT + , ip TEXT + , ip_city TEXT + , ip_country TEXT + , ip_province TEXT + , target_name TEXT + , os TEXT + , os_version TEXT + , page_id TEXT + , page_type TEXT + , path TEXT + , platform TEXT + , root_domain TEXT + , score TEXT + , screen_height TEXT + , screen_width TEXT + , short_id TEXT + , target_id TEXT + , url TEXT + , etl_tx_dt TIMESTAMP +) ; + + COMMENT ON COLUMN p12_sfull.S98_S_customer_events_open_page.id IS ''; + COMMENT ON COLUMN p12_sfull.S98_S_customer_events_open_page.customer_id IS ''; + COMMENT ON COLUMN p12_sfull.S98_S_customer_events_open_page.event IS ''; + COMMENT ON COLUMN p12_sfull.S98_S_customer_events_open_page.last_updated IS ''; + COMMENT ON COLUMN p12_sfull.S98_S_customer_events_open_page.date IS ''; + COMMENT ON COLUMN p12_sfull.S98_S_customer_events_open_page.browser IS ''; + COMMENT ON COLUMN p12_sfull.S98_S_customer_events_open_page.browser_version IS ''; + COMMENT ON COLUMN p12_sfull.S98_S_customer_events_open_page.customer_id_str IS ''; + COMMENT ON COLUMN p12_sfull.S98_S_customer_events_open_page.device IS ''; + COMMENT ON COLUMN p12_sfull.S98_S_customer_events_open_page.domain IS ''; + COMMENT ON COLUMN p12_sfull.S98_S_customer_events_open_page.external_id IS ''; + COMMENT ON COLUMN p12_sfull.S98_S_customer_events_open_page.id_str IS ''; + COMMENT ON COLUMN p12_sfull.S98_S_customer_events_open_page.ip IS ''; + COMMENT ON COLUMN p12_sfull.S98_S_customer_events_open_page.ip_city IS ''; + COMMENT ON COLUMN p12_sfull.S98_S_customer_events_open_page.ip_country IS ''; + COMMENT ON COLUMN p12_sfull.S98_S_customer_events_open_page.ip_province IS ''; + COMMENT ON COLUMN p12_sfull.S98_S_customer_events_open_page.target_name IS ''; + COMMENT ON COLUMN p12_sfull.S98_S_customer_events_open_page.os IS ''; + COMMENT ON COLUMN p12_sfull.S98_S_customer_events_open_page.os_version IS ''; + COMMENT ON COLUMN p12_sfull.S98_S_customer_events_open_page.page_id IS ''; + COMMENT ON COLUMN p12_sfull.S98_S_customer_events_open_page.page_type IS ''; + COMMENT ON COLUMN p12_sfull.S98_S_customer_events_open_page.path IS ''; + COMMENT ON COLUMN p12_sfull.S98_S_customer_events_open_page.platform IS ''; + COMMENT ON COLUMN p12_sfull.S98_S_customer_events_open_page.root_domain IS ''; + COMMENT ON COLUMN p12_sfull.S98_S_customer_events_open_page.score IS ''; + COMMENT ON COLUMN p12_sfull.S98_S_customer_events_open_page.screen_height IS ''; + COMMENT ON COLUMN p12_sfull.S98_S_customer_events_open_page.screen_width IS ''; + COMMENT ON COLUMN p12_sfull.S98_S_customer_events_open_page.short_id IS ''; + COMMENT ON COLUMN p12_sfull.S98_S_customer_events_open_page.target_id IS ''; + COMMENT ON COLUMN p12_sfull.S98_S_customer_events_open_page.url IS ''; + COMMENT ON COLUMN p12_sfull.S98_S_customer_events_open_page.etl_tx_dt IS ''; + +COMMENT ON TABLE p12_sfull.S98_S_customer_events_open_page IS ''; + diff --git a/dev/workflow/TK_Cust/tk_api_2/荟聚API_2/wf_dag_tk_api_2.py b/dev/workflow/TK_Cust/tk_api_2/荟聚API_2/wf_dag_tk_api_2.py new file mode 100644 index 0000000..4953457 --- /dev/null +++ b/dev/workflow/TK_Cust/tk_api_2/荟聚API_2/wf_dag_tk_api_2.py @@ -0,0 +1,156 @@ +#!/usr/bin/python +# -*- encoding=utf-8 -*- +from airflow import DAG +from datetime import datetime, timedelta +from airflow.contrib.hooks.ssh_hook import SSHHook +from airflow.contrib.operators.ssh_operator import SSHOperator +from airflow.sensors.external_task_sensor import ExternalTaskSensor +import json + +from airflow.operators.email_operator import EmailOperator +from airflow.utils.trigger_rule import TriggerRule + + +sshHook = SSHHook(ssh_conn_id ='ssh_air') +default_args = { +'owner': 'info@idgvalue.com', +'email_on_failure': True, +'email_on_retry':True, +'start_date': datetime(2024, 1, 1), +'depends_on_past': False, +'retries': 6, +'retry_delay': timedelta(minutes=10), +} + +dag = DAG('wf_dag_tk_api_2', default_args=default_args, +schedule_interval="10 0 * * *", +catchup=False, +dagrun_timeout=timedelta(minutes=160), +max_active_runs=3) + +task_failed = EmailOperator ( + dag=dag, + trigger_rule=TriggerRule.ONE_FAILED, + task_id="task_failed", + to=["info@idgvalue.com"], + cc=[""], + subject="tk_api_2_failed", + html_content='<h3>您好,tk_api_2作业失败,请及时处理" </h3>') + +customer_events_feign = SSHOperator( +ssh_hook=sshHook, +task_id='customer_events_feign', +command='python3 /data/airflow/etl/API/customer_events_feign.py', +depends_on_past=False, +retries=3, +dag=dag) + +customer_events_load = SSHOperator( +ssh_hook=sshHook, +task_id='customer_events_load', +command='/data/airflow/etl/API/run_psql.sh {{ ds_nodash }} {{params.my_param }}', +params={'my_param':"customer_events_load"}, +depends_on_past=False, +retries=3, +dag=dag) + +customer_events_feign >> customer_events_load + +custom_events_open_page_feign = SSHOperator( +ssh_hook=sshHook, +task_id='custom_events_open_page_feign', +command='python3 /data/airflow/etl/API/custom_events_open_page_feign.py', +depends_on_past=False, +retries=3, +dag=dag) + +custom_events_open_page_load = SSHOperator( +ssh_hook=sshHook, +task_id='custom_events_open_page_load', +command='/data/airflow/etl/API/run_psql.sh {{ ds_nodash }} {{params.my_param }}', +params={'my_param':"custom_events_open_page_load"}, +depends_on_past=False, +retries=3, +dag=dag) + +custom_events_open_page_feign >> custom_events_open_page_load + +custom_events_open_content_page_feign = SSHOperator( +ssh_hook=sshHook, +task_id='custom_events_open_content_page_feign', +command='python3 /data/airflow/etl/API/custom_events_open_content_page_feign.py', +depends_on_past=False, +retries=3, +dag=dag) + +custom_events_open_content_page_load = SSHOperator( +ssh_hook=sshHook, +task_id='custom_events_open_content_page_load', +command='/data/airflow/etl/API/run_psql.sh {{ ds_nodash }} {{params.my_param }}', +params={'my_param':"custom_events_open_content_page_load"}, +depends_on_past=False, +retries=3, +dag=dag) + +custom_events_open_content_page_feign >> custom_events_open_content_page_load + +customer_event_meta_feign = SSHOperator( +ssh_hook=sshHook, +task_id='customer_event_meta_feign', +command='python3 /data/airflow/etl/API/customer_event_meta_feign.py', +depends_on_past=False, +retries=3, +dag=dag) + +customer_event_meta_load = SSHOperator( +ssh_hook=sshHook, +task_id='customer_event_meta_load', +command='/data/airflow/etl/API/run_psql.sh {{ ds_nodash }} {{params.my_param }}', +params={'my_param':"customer_event_meta_load"}, +depends_on_past=False, +retries=3, +dag=dag) + +customer_event_meta_feign >> customer_event_meta_load + +customer_events_open_content_page_9684 = SSHOperator( +ssh_hook=sshHook, +task_id='customer_events_open_content_page_9684', +command='/data/airflow/etl/SA/run_sa.sh {{ ds_nodash }} {{ params.my_param }} >>/data/airflow/logs/run_psql_{{ds_nodash}}.log 2>&1 ', +params={'my_param':"S98_S_customer_events_open_content_page"}, +depends_on_past=False, +retries=3, +dag=dag) + +customer_event_meta_2268 = SSHOperator( +ssh_hook=sshHook, +task_id='customer_event_meta_2268', +command='/data/airflow/etl/SA/run_sa.sh {{ ds_nodash }} {{ params.my_param }} >>/data/airflow/logs/run_psql_{{ds_nodash}}.log 2>&1 ', +params={'my_param':"S98_S_customer_event_meta"}, +depends_on_past=False, +retries=3, +dag=dag) + +customer_events_open_page_7519 = SSHOperator( +ssh_hook=sshHook, +task_id='customer_events_open_page_7519', +command='/data/airflow/etl/SA/run_sa.sh {{ ds_nodash }} {{ params.my_param }} >>/data/airflow/logs/run_psql_{{ds_nodash}}.log 2>&1 ', +params={'my_param':"S98_S_customer_events_open_page"}, +depends_on_past=False, +retries=3, +dag=dag) + +customer_events_3292 = SSHOperator( +ssh_hook=sshHook, +task_id='customer_events_3292', +command='/data/airflow/etl/SA/run_sa.sh {{ ds_nodash }} {{ params.my_param }} >>/data/airflow/logs/run_psql_{{ds_nodash}}.log 2>&1 ', +params={'my_param':"S98_S_customer_events"}, +depends_on_past=False, +retries=3, +dag=dag) + +customer_events_3292 >> customer_events_feign +customer_events_open_content_page_9684 >> custom_events_open_content_page_feign +customer_event_meta_2268 >> customer_event_meta_feign +customer_events_open_page_7519 >> custom_events_open_page_feign +custom_events_open_page_load >> task_failed diff --git a/dev/workflow/TK_Cust/tk_api_2/获取线索事件/customer_events_feign.py b/dev/workflow/TK_Cust/tk_api_2/获取线索事件/customer_events_feign.py new file mode 100644 index 0000000..34050b2 --- /dev/null +++ b/dev/workflow/TK_Cust/tk_api_2/获取线索事件/customer_events_feign.py @@ -0,0 +1,72 @@ +# coding: utf-8 +import requests +import json +import psycopg2 +import uuid +import datetime +import time +import hashlib +import time + +#全局变量,便于参数使用的预设值 +current_date = datetime.date.today() # 获取当前日期 +previous_date = current_date - datetime.timedelta(days=1) # 获取前一天日期 +formatted_current_date = current_date.strftime("%Y-%m-%dT%H:%M:%SZ") # 获取当前日期 - 标准化 +formatted_previous_date = previous_date.strftime("%Y-%m-%dT%H:%M:%SZ") # 获取前一天日期 - 标准化 +timestamp = time.time() # 为Unix time,即从"1970-01-01 00:00:00"至今的秒数; +sign_version = 'v2' # 签名版本号,固定值v2 +nonce = str(uuid.uuid4()) + +#获取签名令牌 +def sign_data(email, open_api_token, timestamp, nonce, sign_version): + # 按照指定的格式拼接字符串 + data_to_sign = f"{email}&{open_api_token}&{timestamp}&{nonce}&{sign_version}" + # 使用SHA256算法计算哈希值 + sha256_hash = hashlib.sha256(data_to_sign.encode()).hexdigest() + return sha256_hash + +#获取鉴权token +def get_token(url): + #请求鉴权接口 + authRequest=requests.get(url) + #解析结果 + if not authRequest: #若为空时,返回空 + return + auth=json.loads(authRequest.text) + return auth + +print('开始加载数据:customer_events:获取线索事件') +authUrl='https://api.huiju.cool/v2/oauth2/token?app_id=cl037b184ebccd97c&secret=deaa83fdd8d385eec0d1d04d34282c4ec8c33b60&grant_type=client_credentials' + +print('开始请求令牌。') +#authRequest=requests.get(authUrl) +#auth=json.loads(authRequest.text) +auth = get_token(authUrl) +#循环判断auth是否为空,若为空,等待30s后重新请求 +i = 0 +while not auth['access_token'] and i < 60: + time.sleep(60) + auth = get_token(authUrl) + i = i + 1 +print('开始请求数据总数。') +url='https://api.huiju.cool/v2/customerEvents' +header={} +body={'access_token':auth['access_token'],'limit':'2000','sort':'date','event':'create_customer','date[ge]':formatted_previous_date,'date[le]':formatted_current_date,} +dataReqL=requests.get(url,headers=header,params=body) +resL=json.loads(dataReqL.text) +# print(resL) +dataList=resL['data'] +total=len(dataList) +conn = psycopg2.connect(database="dataops_db", user="dbuser_dops", password="MIgTi3jA", + host="172.17.0.8", port="5432") +print('数据库连接成功') +dataId=str(uuid.uuid4()) +print('临时id:'+dataId) +json_object = json.dumps(dataList) +cur=conn.cursor() +sql="update data_api.api_data set is_loaded = '1' where api_id = '747d33fa-a0e0-421d-aa9b-4ca4517b';INSERT INTO data_api.api_data (id,api_id,data,total_num,is_loaded,status,request_tm,execute_tm,remark) values (%s,%s,%s,%s,'0','0',current_timestamp(0),current_timestamp(0),'')" +cur.execute(sql,[dataId,'747d33fa-a0e0-421d-aa9b-4ca4517b',json_object,total]) +conn.commit() +cur.close() +conn.close() +print('加载数据结束:customer_events:获取线索事件') \ No newline at end of file diff --git a/dev/workflow/TK_Cust/tk_api_2/获取线索事件/customer_events_load.sql b/dev/workflow/TK_Cust/tk_api_2/获取线索事件/customer_events_load.sql new file mode 100644 index 0000000..6ef17c4 --- /dev/null +++ b/dev/workflow/TK_Cust/tk_api_2/获取线索事件/customer_events_load.sql @@ -0,0 +1,57 @@ +/*******Main Section**************************************************************************/ +\set ON_ERROR_STOP on +\set AUTOCOMMIT on +\timing on + +DELETE FROM data_api.customer_events; + +insert into data_api.customer_events ( + id + , customer_id + , event + , last_updated + , source + , content_name + , date + , create_method + , customer_id_str + , external_id + , id_str + ,etl_tx_dt +) +select + case when trim(both from id)='' then null else id::text end id + , case when trim(both from customer_id)='' then null else customer_id::text end customer_id + , case when trim(both from event)='' then null else event::text end event + , case when trim(both from last_updated)='' then null else last_updated::text end last_updated + , case when trim(both from source)='' then null else source::text end source + , case when trim(both from content_name)='' then null else content_name::text end content_name + , case when trim(both from date)='' then null else date::text end date + , case when trim(both from create_method)='' then null else create_method::text end create_method + , case when trim(both from customer_id_str)='' then null else customer_id_str::text end customer_id_str + , case when trim(both from external_id)='' then null else external_id::text end external_id + , case when trim(both from id_str)='' then null else id_str::text end id_str +,etl_tx_dt +from ( +select + (json_array_elements(data::json)::json->>'id') id + , (json_array_elements(data::json)::json->>'customerId') customer_id + , (json_array_elements(data::json)::json->>'event') event + , (json_array_elements(data::json)::json->>'lastUpdated') last_updated + , (json_array_elements(data::json)::json->>'source') source + , (json_array_elements(data::json)::json->>'contentName') content_name + , (json_array_elements(data::json)::json->>'date') date + , (json_array_elements(data::json)::json->>'createMethod') create_method + , (json_array_elements(data::json)::json->>'customerIdStr') customer_id_str + , (json_array_elements(data::json)::json->>'externalId') external_id + , (json_array_elements(data::json)::json->>'idStr') id_str + ,CURRENT_TIMESTAMP(0) etl_tx_dt +from (select * from data_api.api_data +WHERE api_id='747d33fa-a0e0-421d-aa9b-4ca4517b' and is_loaded = '0' order by request_tm desc limit 1) p )p; + +update data_api.api_data +set is_loaded = '1' , + status = '1', + request_tm = current_timestamp(0) +where api_id='747d33fa-a0e0-421d-aa9b-4ca4517b'; +\q \ No newline at end of file diff --git a/dev/workflow/TK_Cust/tk_api_2/获取线索事件_open_content_page/custom_events_open_content_page_feign.py b/dev/workflow/TK_Cust/tk_api_2/获取线索事件_open_content_page/custom_events_open_content_page_feign.py new file mode 100644 index 0000000..5b88694 --- /dev/null +++ b/dev/workflow/TK_Cust/tk_api_2/获取线索事件_open_content_page/custom_events_open_content_page_feign.py @@ -0,0 +1,72 @@ +# coding: utf-8 +import requests +import json +import psycopg2 +import uuid +import datetime +import time +import hashlib +import time + +#全局变量,便于参数使用的预设值 +current_date = datetime.date.today() # 获取当前日期 +previous_date = current_date - datetime.timedelta(days=1) # 获取前一天日期 +formatted_current_date = current_date.strftime("%Y-%m-%dT%H:%M:%SZ") # 获取当前日期 - 标准化 +formatted_previous_date = previous_date.strftime("%Y-%m-%dT%H:%M:%SZ") # 获取前一天日期 - 标准化 +timestamp = time.time() # 为Unix time,即从"1970-01-01 00:00:00"至今的秒数; +sign_version = 'v2' # 签名版本号,固定值v2 +nonce = str(uuid.uuid4()) + +#获取签名令牌 +def sign_data(email, open_api_token, timestamp, nonce, sign_version): + # 按照指定的格式拼接字符串 + data_to_sign = f"{email}&{open_api_token}&{timestamp}&{nonce}&{sign_version}" + # 使用SHA256算法计算哈希值 + sha256_hash = hashlib.sha256(data_to_sign.encode()).hexdigest() + return sha256_hash + +#获取鉴权token +def get_token(url): + #请求鉴权接口 + authRequest=requests.get(url) + #解析结果 + if not authRequest: #若为空时,返回空 + return + auth=json.loads(authRequest.text) + return auth + +print('开始加载数据:custom_events_open_content_page:获取线索事件_open_content_page') +authUrl='https://api.huiju.cool/v2/oauth2/token?app_id=cl037b184ebccd97c&secret=deaa83fdd8d385eec0d1d04d34282c4ec8c33b60&grant_type=client_credentials' + +print('开始请求令牌。') +#authRequest=requests.get(authUrl) +#auth=json.loads(authRequest.text) +auth = get_token(authUrl) +#循环判断auth是否为空,若为空,等待30s后重新请求 +i = 0 +while not auth['access_token'] and i < 60: + time.sleep(60) + auth = get_token(authUrl) + i = i + 1 +print('开始请求数据总数。') +url='https://api.huiju.cool/v2/customerEvents' +header={} +body={'access_token':auth['access_token'],'limit':'2000','event':'open_content_page','sort':'date','date[ge]':formatted_previous_date,'date[le]':formatted_current_date,} +dataReqL=requests.get(url,headers=header,params=body) +resL=json.loads(dataReqL.text) +# print(resL) +dataList=resL['data'] +total=len(dataList) +conn = psycopg2.connect(database="dataops_db", user="dbuser_dops", password="MIgTi3jA", + host="172.17.0.8", port="5432") +print('数据库连接成功') +dataId=str(uuid.uuid4()) +print('临时id:'+dataId) +json_object = json.dumps(dataList) +cur=conn.cursor() +sql="update data_api.api_data set is_loaded = '1' where api_id = '43edfb13-8a42-4152-9dc8-d5feb3c8';INSERT INTO data_api.api_data (id,api_id,data,total_num,is_loaded,status,request_tm,execute_tm,remark) values (%s,%s,%s,%s,'0','0',current_timestamp(0),current_timestamp(0),'')" +cur.execute(sql,[dataId,'43edfb13-8a42-4152-9dc8-d5feb3c8',json_object,total]) +conn.commit() +cur.close() +conn.close() +print('加载数据结束:custom_events_open_content_page:获取线索事件_open_content_page') \ No newline at end of file diff --git a/dev/workflow/TK_Cust/tk_api_2/获取线索事件_open_content_page/custom_events_open_content_page_load.sql b/dev/workflow/TK_Cust/tk_api_2/获取线索事件_open_content_page/custom_events_open_content_page_load.sql new file mode 100644 index 0000000..6526288 --- /dev/null +++ b/dev/workflow/TK_Cust/tk_api_2/获取线索事件_open_content_page/custom_events_open_content_page_load.sql @@ -0,0 +1,102 @@ +/*******Main Section**************************************************************************/ +\set ON_ERROR_STOP on +\set AUTOCOMMIT on +\timing on + +DELETE FROM data_api.customer_events_open_content_page; + +insert into data_api.customer_events_open_content_page ( + id + , customer_id + , event + , last_updated + , date + , browser + , browser_version + , customer_id_str + , device + , external_id + , id_str + , ip + , ip_city + , ip_country + , ip_province + , os + , os_version + , page_id + , page_type + , platform + , screen_height + , screen_width + , short_id + , target_id + , target_name + , url + ,etl_tx_dt +) +select + case when trim(both from id)='' then null else id::text end id + , case when trim(both from customer_id)='' then null else customer_id::text end customer_id + , case when trim(both from event)='' then null else event::text end event + , case when trim(both from last_updated)='' then null else last_updated::text end last_updated + , case when trim(both from date)='' then null else date::text end date + , case when trim(both from browser)='' then null else browser::text end browser + , case when trim(both from browser_version)='' then null else browser_version::text end browser_version + , case when trim(both from customer_id_str)='' then null else customer_id_str::text end customer_id_str + , case when trim(both from device)='' then null else device::text end device + , case when trim(both from external_id)='' then null else external_id::text end external_id + , case when trim(both from id_str)='' then null else id_str::text end id_str + , case when trim(both from ip)='' then null else ip::text end ip + , case when trim(both from ip_city)='' then null else ip_city::text end ip_city + , case when trim(both from ip_country)='' then null else ip_country::text end ip_country + , case when trim(both from ip_province)='' then null else ip_province::text end ip_province + , case when trim(both from os)='' then null else os::text end os + , case when trim(both from os_version)='' then null else os_version::text end os_version + , case when trim(both from page_id)='' then null else page_id::text end page_id + , case when trim(both from page_type)='' then null else page_type::text end page_type + , case when trim(both from platform)='' then null else platform::text end platform + , case when trim(both from screen_height)='' then null else screen_height::text end screen_height + , case when trim(both from screen_width)='' then null else screen_width::text end screen_width + , case when trim(both from short_id)='' then null else short_id::text end short_id + , case when trim(both from target_id)='' then null else target_id::text end target_id + , case when trim(both from target_name)='' then null else target_name::text end target_name + , case when trim(both from url)='' then null else url::text end url +,etl_tx_dt +from ( +select + (json_array_elements(data::json)::json->>'id') id + , (json_array_elements(data::json)::json->>'customerId') customer_id + , (json_array_elements(data::json)::json->>'event') event + , (json_array_elements(data::json)::json->>'lastUpdated') last_updated + , (json_array_elements(data::json)::json->>'date') date + , (json_array_elements(data::json)::json->>'browser') browser + , (json_array_elements(data::json)::json->>'browserVersion') browser_version + , (json_array_elements(data::json)::json->>'customerIdStr') customer_id_str + , (json_array_elements(data::json)::json->>'device') device + , (json_array_elements(data::json)::json->>'externalId') external_id + , (json_array_elements(data::json)::json->>'idStr') id_str + , (json_array_elements(data::json)::json->>'ip') ip + , (json_array_elements(data::json)::json->>'ipCity') ip_city + , (json_array_elements(data::json)::json->>'ipCountry') ip_country + , (json_array_elements(data::json)::json->>'ipProvince') ip_province + , (json_array_elements(data::json)::json->>'os') os + , (json_array_elements(data::json)::json->>'osVersion') os_version + , (json_array_elements(data::json)::json->>'pageId') page_id + , (json_array_elements(data::json)::json->>'pageType') page_type + , (json_array_elements(data::json)::json->>'platform') platform + , (json_array_elements(data::json)::json->>'screenHeight') screen_height + , (json_array_elements(data::json)::json->>'screenWidth') screen_width + , (json_array_elements(data::json)::json->>'shortId') short_id + , (json_array_elements(data::json)::json->>'targetId') target_id + , (json_array_elements(data::json)::json->>'targetName') target_name + , (json_array_elements(data::json)::json->>'url') url + ,CURRENT_TIMESTAMP(0) etl_tx_dt +from (select * from data_api.api_data +WHERE api_id='43edfb13-8a42-4152-9dc8-d5feb3c8' and is_loaded = '0' order by request_tm desc limit 1) p )p; + +update data_api.api_data +set is_loaded = '1' , + status = '1', + request_tm = current_timestamp(0) +where api_id='43edfb13-8a42-4152-9dc8-d5feb3c8'; +\q \ No newline at end of file diff --git a/dev/workflow/TK_Cust/tk_api_2/获取线索事件_open_page/custom_events_open_page_feign.py b/dev/workflow/TK_Cust/tk_api_2/获取线索事件_open_page/custom_events_open_page_feign.py new file mode 100644 index 0000000..30b2e5c --- /dev/null +++ b/dev/workflow/TK_Cust/tk_api_2/获取线索事件_open_page/custom_events_open_page_feign.py @@ -0,0 +1,72 @@ +# coding: utf-8 +import requests +import json +import psycopg2 +import uuid +import datetime +import time +import hashlib +import time + +#全局变量,便于参数使用的预设值 +current_date = datetime.date.today() # 获取当前日期 +previous_date = current_date - datetime.timedelta(days=1) # 获取前一天日期 +formatted_current_date = current_date.strftime("%Y-%m-%dT%H:%M:%SZ") # 获取当前日期 - 标准化 +formatted_previous_date = previous_date.strftime("%Y-%m-%dT%H:%M:%SZ") # 获取前一天日期 - 标准化 +timestamp = time.time() # 为Unix time,即从"1970-01-01 00:00:00"至今的秒数; +sign_version = 'v2' # 签名版本号,固定值v2 +nonce = str(uuid.uuid4()) + +#获取签名令牌 +def sign_data(email, open_api_token, timestamp, nonce, sign_version): + # 按照指定的格式拼接字符串 + data_to_sign = f"{email}&{open_api_token}&{timestamp}&{nonce}&{sign_version}" + # 使用SHA256算法计算哈希值 + sha256_hash = hashlib.sha256(data_to_sign.encode()).hexdigest() + return sha256_hash + +#获取鉴权token +def get_token(url): + #请求鉴权接口 + authRequest=requests.get(url) + #解析结果 + if not authRequest: #若为空时,返回空 + return + auth=json.loads(authRequest.text) + return auth + +print('开始加载数据:custom_events_open_page:获取线索事件_open_page') +authUrl='https://api.huiju.cool/v2/oauth2/token?app_id=cl037b184ebccd97c&secret=deaa83fdd8d385eec0d1d04d34282c4ec8c33b60&grant_type=client_credentials' + +print('开始请求令牌。') +#authRequest=requests.get(authUrl) +#auth=json.loads(authRequest.text) +auth = get_token(authUrl) +#循环判断auth是否为空,若为空,等待30s后重新请求 +i = 0 +while not auth['access_token'] and i < 60: + time.sleep(60) + auth = get_token(authUrl) + i = i + 1 +print('开始请求数据总数。') +url='https://api.huiju.cool/v2/customerEvents' +header={} +body={'access_token':auth['access_token'],'limit':'2000','sort':'date','event':'open_page','date[le]':formatted_current_date,'date[ge]':formatted_previous_date,} +dataReqL=requests.get(url,headers=header,params=body) +resL=json.loads(dataReqL.text) +# print(resL) +dataList=resL['data'] +total=len(dataList) +conn = psycopg2.connect(database="dataops_db", user="dbuser_dops", password="MIgTi3jA", + host="172.17.0.8", port="5432") +print('数据库连接成功') +dataId=str(uuid.uuid4()) +print('临时id:'+dataId) +json_object = json.dumps(dataList) +cur=conn.cursor() +sql="update data_api.api_data set is_loaded = '1' where api_id = 'a2b284c3-322f-4bc0-89ff-414aa66a';INSERT INTO data_api.api_data (id,api_id,data,total_num,is_loaded,status,request_tm,execute_tm,remark) values (%s,%s,%s,%s,'0','0',current_timestamp(0),current_timestamp(0),'')" +cur.execute(sql,[dataId,'a2b284c3-322f-4bc0-89ff-414aa66a',json_object,total]) +conn.commit() +cur.close() +conn.close() +print('加载数据结束:custom_events_open_page:获取线索事件_open_page') \ No newline at end of file diff --git a/dev/workflow/TK_Cust/tk_api_2/获取线索事件_open_page/custom_events_open_page_load.sql b/dev/workflow/TK_Cust/tk_api_2/获取线索事件_open_page/custom_events_open_page_load.sql new file mode 100644 index 0000000..26f01bb --- /dev/null +++ b/dev/workflow/TK_Cust/tk_api_2/获取线索事件_open_page/custom_events_open_page_load.sql @@ -0,0 +1,123 @@ +/*******Main Section**************************************************************************/ +\set ON_ERROR_STOP on +\set AUTOCOMMIT on +\timing on + +DELETE FROM data_api.customer_events_open_page; + +insert into data_api.customer_events_open_page ( + id + , customer_id + , event + , last_updated + , date + , browser + , browser_version + , campaign + , customer_id_str + , device + , domain + , external_id + , id_str + , ip + , ip_city + , ip_country + , ip_county + , ip_province + , target_name + , os + , os_version + , page_id + , page_type + , path + , platform + , ref_domain + , root_domain + , score + , screen_height + , screen_width + , short_id + , target_id + , url + ,etl_tx_dt +) +select + case when trim(both from id)='' then null else id::text end id + , case when trim(both from customer_id)='' then null else customer_id::text end customer_id + , case when trim(both from event)='' then null else event::text end event + , case when trim(both from last_updated)='' then null else last_updated::text end last_updated + , case when trim(both from date)='' then null else date::text end date + , case when trim(both from browser)='' then null else browser::text end browser + , case when trim(both from browser_version)='' then null else browser_version::text end browser_version + , case when trim(both from campaign)='' then null else campaign::text end campaign + , case when trim(both from customer_id_str)='' then null else customer_id_str::text end customer_id_str + , case when trim(both from device)='' then null else device::text end device + , case when trim(both from domain)='' then null else domain::text end domain + , case when trim(both from external_id)='' then null else external_id::text end external_id + , case when trim(both from id_str)='' then null else id_str::text end id_str + , case when trim(both from ip)='' then null else ip::text end ip + , case when trim(both from ip_city)='' then null else ip_city::text end ip_city + , case when trim(both from ip_country)='' then null else ip_country::text end ip_country + , case when trim(both from ip_county)='' then null else ip_county::text end ip_county + , case when trim(both from ip_province)='' then null else ip_province::text end ip_province + , case when trim(both from target_name)='' then null else target_name::text end target_name + , case when trim(both from os)='' then null else os::text end os + , case when trim(both from os_version)='' then null else os_version::text end os_version + , case when trim(both from page_id)='' then null else page_id::text end page_id + , case when trim(both from page_type)='' then null else page_type::text end page_type + , case when trim(both from path)='' then null else path::text end path + , case when trim(both from platform)='' then null else platform::text end platform + , case when trim(both from ref_domain)='' then null else ref_domain::text end ref_domain + , case when trim(both from root_domain)='' then null else root_domain::text end root_domain + , case when trim(both from score)='' then null else score::text end score + , case when trim(both from screen_height)='' then null else screen_height::text end screen_height + , case when trim(both from screen_width)='' then null else screen_width::text end screen_width + , case when trim(both from short_id)='' then null else short_id::text end short_id + , case when trim(both from target_id)='' then null else target_id::text end target_id + , case when trim(both from url)='' then null else url::text end url +,etl_tx_dt +from ( +select + (json_array_elements(data::json)::json->>'id') id + , (json_array_elements(data::json)::json->>'customerId') customer_id + , (json_array_elements(data::json)::json->>'event') event + , (json_array_elements(data::json)::json->>'lastUpdated') last_updated + , (json_array_elements(data::json)::json->>'date') date + , (json_array_elements(data::json)::json->>'browser') browser + , (json_array_elements(data::json)::json->>'browserVersion') browser_version + , (json_array_elements(data::json)::json->>'campaign') campaign + , (json_array_elements(data::json)::json->>'customerIdStr') customer_id_str + , (json_array_elements(data::json)::json->>'device') device + , (json_array_elements(data::json)::json->>'domain') domain + , (json_array_elements(data::json)::json->>'externalId') external_id + , (json_array_elements(data::json)::json->>'idStr') id_str + , (json_array_elements(data::json)::json->>'ip') ip + , (json_array_elements(data::json)::json->>'ipCity') ip_city + , (json_array_elements(data::json)::json->>'ipCountry') ip_country + , (json_array_elements(data::json)::json->>'ipCounty') ip_county + , (json_array_elements(data::json)::json->>'ipProvince') ip_province + , (json_array_elements(data::json)::json->>'targetName') target_name + , (json_array_elements(data::json)::json->>'os') os + , (json_array_elements(data::json)::json->>'osVersion') os_version + , (json_array_elements(data::json)::json->>'pageId') page_id + , (json_array_elements(data::json)::json->>'pageType') page_type + , (json_array_elements(data::json)::json->>'path') path + , (json_array_elements(data::json)::json->>'platform') platform + , (json_array_elements(data::json)::json->>'refDomain') ref_domain + , (json_array_elements(data::json)::json->>'rootDomain') root_domain + , (json_array_elements(data::json)::json->>'score') score + , (json_array_elements(data::json)::json->>'screenHeight') screen_height + , (json_array_elements(data::json)::json->>'screenWidth') screen_width + , (json_array_elements(data::json)::json->>'shortId') short_id + , (json_array_elements(data::json)::json->>'targetId') target_id + , (json_array_elements(data::json)::json->>'url') url + ,CURRENT_TIMESTAMP(0) etl_tx_dt +from (select * from data_api.api_data +WHERE api_id='a2b284c3-322f-4bc0-89ff-414aa66a' and is_loaded = '0' order by request_tm desc limit 1) p )p; + +update data_api.api_data +set is_loaded = '1' , + status = '1', + request_tm = current_timestamp(0) +where api_id='a2b284c3-322f-4bc0-89ff-414aa66a'; +\q \ No newline at end of file diff --git a/dev/workflow/TK_Cust/tk_api_2/获取线索事件元数据/customer_event_meta_feign.py b/dev/workflow/TK_Cust/tk_api_2/获取线索事件元数据/customer_event_meta_feign.py new file mode 100644 index 0000000..38ee490 --- /dev/null +++ b/dev/workflow/TK_Cust/tk_api_2/获取线索事件元数据/customer_event_meta_feign.py @@ -0,0 +1,72 @@ +# coding: utf-8 +import requests +import json +import psycopg2 +import uuid +import datetime +import time +import hashlib +import time + +#全局变量,便于参数使用的预设值 +current_date = datetime.date.today() # 获取当前日期 +previous_date = current_date - datetime.timedelta(days=1) # 获取前一天日期 +formatted_current_date = current_date.strftime("%Y-%m-%dT%H:%M:%SZ") # 获取当前日期 - 标准化 +formatted_previous_date = previous_date.strftime("%Y-%m-%dT%H:%M:%SZ") # 获取前一天日期 - 标准化 +timestamp = time.time() # 为Unix time,即从"1970-01-01 00:00:00"至今的秒数; +sign_version = 'v2' # 签名版本号,固定值v2 +nonce = str(uuid.uuid4()) + +#获取签名令牌 +def sign_data(email, open_api_token, timestamp, nonce, sign_version): + # 按照指定的格式拼接字符串 + data_to_sign = f"{email}&{open_api_token}&{timestamp}&{nonce}&{sign_version}" + # 使用SHA256算法计算哈希值 + sha256_hash = hashlib.sha256(data_to_sign.encode()).hexdigest() + return sha256_hash + +#获取鉴权token +def get_token(url): + #请求鉴权接口 + authRequest=requests.get(url) + #解析结果 + if not authRequest: #若为空时,返回空 + return + auth=json.loads(authRequest.text) + return auth + +print('开始加载数据:customer_event_meta:获取线索事件元数据') +authUrl='https://api.huiju.cool/v2/oauth2/token?app_id=cl037b184ebccd97c&secret=deaa83fdd8d385eec0d1d04d34282c4ec8c33b60&grant_type=client_credentials' + +print('开始请求令牌。') +#authRequest=requests.get(authUrl) +#auth=json.loads(authRequest.text) +auth = get_token(authUrl) +#循环判断auth是否为空,若为空,等待30s后重新请求 +i = 0 +while not auth['access_token'] and i < 60: + time.sleep(60) + auth = get_token(authUrl) + i = i + 1 +print('开始请求数据总数。') +url='https://api.huiju.cool/v2/metaService/getCustomerEventMeta' +header={} +body={'access_token':auth['access_token'],} +dataReqL=requests.get(url,headers=header,params=body) +resL=json.loads(dataReqL.text) +# print(resL) +dataList=resL['data'] +total=len(dataList) +conn = psycopg2.connect(database="dataops_db", user="dbuser_dops", password="MIgTi3jA", + host="172.17.0.8", port="5432") +print('数据库连接成功') +dataId=str(uuid.uuid4()) +print('临时id:'+dataId) +json_object = json.dumps(dataList) +cur=conn.cursor() +sql="update data_api.api_data set is_loaded = '1' where api_id = '057169de-4aa7-4f73-8894-221b4fd0';INSERT INTO data_api.api_data (id,api_id,data,total_num,is_loaded,status,request_tm,execute_tm,remark) values (%s,%s,%s,%s,'0','0',current_timestamp(0),current_timestamp(0),'')" +cur.execute(sql,[dataId,'057169de-4aa7-4f73-8894-221b4fd0',json_object,total]) +conn.commit() +cur.close() +conn.close() +print('加载数据结束:customer_event_meta:获取线索事件元数据') \ No newline at end of file diff --git a/dev/workflow/TK_Cust/tk_api_2/获取线索事件元数据/customer_event_meta_load.sql b/dev/workflow/TK_Cust/tk_api_2/获取线索事件元数据/customer_event_meta_load.sql new file mode 100644 index 0000000..fb99ba0 --- /dev/null +++ b/dev/workflow/TK_Cust/tk_api_2/获取线索事件元数据/customer_event_meta_load.sql @@ -0,0 +1,33 @@ +/*******Main Section**************************************************************************/ +\set ON_ERROR_STOP on +\set AUTOCOMMIT on +\timing on + +DELETE FROM data_api.customer_event_meta; + +insert into data_api.customer_event_meta ( + group_name + , group_label + , events + ,etl_tx_dt +) +select + case when trim(both from group_name)='' then null else group_name::text end group_name + , case when trim(both from group_label)='' then null else group_label::text end group_label + , case when trim(both from events)='' then null else events::text end events +,etl_tx_dt +from ( +select + (json_array_elements(data::json)::json->>'groupName') group_name + , (json_array_elements(data::json)::json->>'groupLabel') group_label + , (json_array_elements(data::json)::json->>'events') events + ,CURRENT_TIMESTAMP(0) etl_tx_dt +from (select * from data_api.api_data +WHERE api_id='057169de-4aa7-4f73-8894-221b4fd0' and is_loaded = '0' order by request_tm desc limit 1) p )p; + +update data_api.api_data +set is_loaded = '1' , + status = '1', + request_tm = current_timestamp(0) +where api_id='057169de-4aa7-4f73-8894-221b4fd0'; +\q \ No newline at end of file