diff --git a/TK_Cust/dev/tk_api/customer_event_meta/S98_S_customer_event_meta.sql b/TK_Cust/dev/tk_api/customer_event_meta/S98_S_customer_event_meta.sql deleted file mode 100644 index f487eb0..0000000 --- a/TK_Cust/dev/tk_api/customer_event_meta/S98_S_customer_event_meta.sql +++ /dev/null @@ -1,34 +0,0 @@ -/*******Main Section**************************************************************************/ -\set ON_ERROR_STOP on -\set AUTOCOMMIT on -\timing on -delete from p10_sa.S98_S_customer_event_meta -; -insert into p10_sa.S98_S_customer_event_meta -( group_name - , group_label - , events - , etl_tx_dt ) - select - group_name - , group_label - , events - , etl_tx_dt - from p00_tal.S98_S_customer_event_meta - ; - delete from p12_sfull.S98_S_customer_event_meta -; -; -insert into p12_sfull.S98_S_customer_event_meta -( group_name - , group_label - , events - , etl_tx_dt ) - select - group_name - , group_label - , events - , etl_tx_dt - from p10_sa.S98_S_customer_event_meta -; -\q \ No newline at end of file diff --git a/TK_Cust/dev/tk_api/customer_event_meta/sa_foreign_tables.sql b/TK_Cust/dev/tk_api/customer_event_meta/sa_foreign_tables.sql deleted file mode 100644 index aaf7b8e..0000000 --- a/TK_Cust/dev/tk_api/customer_event_meta/sa_foreign_tables.sql +++ /dev/null @@ -1,15 +0,0 @@ - -CREATE FOREIGN TABLE if not exists p00_tal.S98_S_customer_event_meta ( - group_name TEXT - , group_label TEXT - , events TEXT - , etl_tx_dt TIMESTAMP -) - - -SERVER pgsql_server_S98_S OPTIONS(schema_name 'data_api', table_name 'customer_event_meta' ); - - - - - diff --git a/TK_Cust/dev/tk_api/customer_event_meta/sa_tables.sql b/TK_Cust/dev/tk_api/customer_event_meta/sa_tables.sql deleted file mode 100644 index ad1ef68..0000000 --- a/TK_Cust/dev/tk_api/customer_event_meta/sa_tables.sql +++ /dev/null @@ -1,31 +0,0 @@ - -create table if not exists p10_sa.S98_S_customer_event_meta ( - group_name TEXT - , group_label TEXT - , events TEXT - , etl_tx_dt TIMESTAMP -) ; - - COMMENT ON COLUMN p10_sa.S98_S_customer_event_meta.group_name IS ''; - COMMENT ON COLUMN p10_sa.S98_S_customer_event_meta.group_label IS ''; - COMMENT ON COLUMN p10_sa.S98_S_customer_event_meta.events IS ''; - COMMENT ON COLUMN p10_sa.S98_S_customer_event_meta.etl_tx_dt IS ''; - -COMMENT ON TABLE p10_sa.S98_S_customer_event_meta IS ''; - - - -create table if not exists p12_sfull.S98_S_customer_event_meta ( - group_name TEXT - , group_label TEXT - , events TEXT - , etl_tx_dt TIMESTAMP -) ; - - COMMENT ON COLUMN p12_sfull.S98_S_customer_event_meta.group_name IS ''; - COMMENT ON COLUMN p12_sfull.S98_S_customer_event_meta.group_label IS ''; - COMMENT ON COLUMN p12_sfull.S98_S_customer_event_meta.events IS ''; - COMMENT ON COLUMN p12_sfull.S98_S_customer_event_meta.etl_tx_dt IS ''; - -COMMENT ON TABLE p12_sfull.S98_S_customer_event_meta IS ''; - diff --git a/TK_Cust/dev/tk_api/customer_events/S98_S_customer_events.sql b/TK_Cust/dev/tk_api/customer_events/S98_S_customer_events.sql deleted file mode 100644 index b3ec31b..0000000 --- a/TK_Cust/dev/tk_api/customer_events/S98_S_customer_events.sql +++ /dev/null @@ -1,66 +0,0 @@ -/*******Main Section**************************************************************************/ -\set ON_ERROR_STOP on -\set AUTOCOMMIT on -\timing on -delete from p10_sa.S98_S_customer_events -; -insert into p10_sa.S98_S_customer_events -( id_str - , date - , last_updated - , customer_id_str - , create_method - , customer_id - , external_id - , id - , source - , event - , content_name - , etl_tx_dt ) - select - id_str - , date - , last_updated - , customer_id_str - , create_method - , customer_id - , external_id - , id - , source - , event - , content_name - , etl_tx_dt - from p00_tal.S98_S_customer_events - ; - delete from p12_sfull.S98_S_customer_events -; -; -insert into p12_sfull.S98_S_customer_events -( id_str - , date - , last_updated - , customer_id_str - , create_method - , customer_id - , external_id - , id - , source - , event - , content_name - , etl_tx_dt ) - select - id_str - , date - , last_updated - , customer_id_str - , create_method - , customer_id - , external_id - , id - , source - , event - , content_name - , etl_tx_dt - from p10_sa.S98_S_customer_events -; -\q \ No newline at end of file diff --git a/TK_Cust/dev/tk_api/customer_events/sa_foreign_tables.sql b/TK_Cust/dev/tk_api/customer_events/sa_foreign_tables.sql deleted file mode 100644 index 1e17fd2..0000000 --- a/TK_Cust/dev/tk_api/customer_events/sa_foreign_tables.sql +++ /dev/null @@ -1,23 +0,0 @@ - -CREATE FOREIGN TABLE if not exists p00_tal.S98_S_customer_events ( - id_str TEXT - , date TEXT - , last_updated TEXT - , customer_id_str TEXT - , create_method TEXT - , customer_id TEXT - , external_id TEXT - , id TEXT - , source TEXT - , event TEXT - , content_name TEXT - , etl_tx_dt TIMESTAMP -) - - -SERVER pgsql_server_S98_S OPTIONS(schema_name 'data_api', table_name 'customer_events' ); - - - - - diff --git a/TK_Cust/dev/tk_api/customer_events/sa_tables.sql b/TK_Cust/dev/tk_api/customer_events/sa_tables.sql deleted file mode 100644 index 3c75d1e..0000000 --- a/TK_Cust/dev/tk_api/customer_events/sa_tables.sql +++ /dev/null @@ -1,63 +0,0 @@ - -create table if not exists p10_sa.S98_S_customer_events ( - id_str TEXT - , date TEXT - , last_updated TEXT - , customer_id_str TEXT - , create_method TEXT - , customer_id TEXT - , external_id TEXT - , id TEXT - , source TEXT - , event TEXT - , content_name TEXT - , etl_tx_dt TIMESTAMP -) ; - - COMMENT ON COLUMN p10_sa.S98_S_customer_events.id_str IS ''; - COMMENT ON COLUMN p10_sa.S98_S_customer_events.date IS ''; - COMMENT ON COLUMN p10_sa.S98_S_customer_events.last_updated IS ''; - COMMENT ON COLUMN p10_sa.S98_S_customer_events.customer_id_str IS ''; - COMMENT ON COLUMN p10_sa.S98_S_customer_events.create_method IS ''; - COMMENT ON COLUMN p10_sa.S98_S_customer_events.customer_id IS ''; - COMMENT ON COLUMN p10_sa.S98_S_customer_events.external_id IS ''; - COMMENT ON COLUMN p10_sa.S98_S_customer_events.id IS ''; - COMMENT ON COLUMN p10_sa.S98_S_customer_events.source IS ''; - COMMENT ON COLUMN p10_sa.S98_S_customer_events.event IS ''; - COMMENT ON COLUMN p10_sa.S98_S_customer_events.content_name IS ''; - COMMENT ON COLUMN p10_sa.S98_S_customer_events.etl_tx_dt IS ''; - -COMMENT ON TABLE p10_sa.S98_S_customer_events IS ''; - - - -create table if not exists p12_sfull.S98_S_customer_events ( - id_str TEXT - , date TEXT - , last_updated TEXT - , customer_id_str TEXT - , create_method TEXT - , customer_id TEXT - , external_id TEXT - , id TEXT - , source TEXT - , event TEXT - , content_name TEXT - , etl_tx_dt TIMESTAMP -) ; - - COMMENT ON COLUMN p12_sfull.S98_S_customer_events.id_str IS ''; - COMMENT ON COLUMN p12_sfull.S98_S_customer_events.date IS ''; - COMMENT ON COLUMN p12_sfull.S98_S_customer_events.last_updated IS ''; - COMMENT ON COLUMN p12_sfull.S98_S_customer_events.customer_id_str IS ''; - COMMENT ON COLUMN p12_sfull.S98_S_customer_events.create_method IS ''; - COMMENT ON COLUMN p12_sfull.S98_S_customer_events.customer_id IS ''; - COMMENT ON COLUMN p12_sfull.S98_S_customer_events.external_id IS ''; - COMMENT ON COLUMN p12_sfull.S98_S_customer_events.id IS ''; - COMMENT ON COLUMN p12_sfull.S98_S_customer_events.source IS ''; - COMMENT ON COLUMN p12_sfull.S98_S_customer_events.event IS ''; - COMMENT ON COLUMN p12_sfull.S98_S_customer_events.content_name IS ''; - COMMENT ON COLUMN p12_sfull.S98_S_customer_events.etl_tx_dt IS ''; - -COMMENT ON TABLE p12_sfull.S98_S_customer_events IS ''; - diff --git a/TK_Cust/dev/tk_api/customer_events_click_link_in_page/S98_S_customer_events_click_link_in_page.sql b/TK_Cust/dev/tk_api/customer_events_click_link_in_page/S98_S_customer_events_click_link_in_page.sql deleted file mode 100644 index 51a6d63..0000000 --- a/TK_Cust/dev/tk_api/customer_events_click_link_in_page/S98_S_customer_events_click_link_in_page.sql +++ /dev/null @@ -1,126 +0,0 @@ -/*******Main Section**************************************************************************/ -\set ON_ERROR_STOP on -\set AUTOCOMMIT on -\timing on -delete from p10_sa.S98_S_customer_events_click_link_in_page -; -insert into p10_sa.S98_S_customer_events_click_link_in_page -( date - , screen_width - , target_name - , ip_city - , ip_country - , platform - , score - , last_updated - , page_type - , os_version - , browser - , browser_version - , customer_id - , id - , event - , id_str - , os - , target_id - , customer_id_str - , ip - , screen_height - , external_id - , page_id - , url - , ip_province - , device - , etl_tx_dt ) - select - date - , screen_width - , target_name - , ip_city - , ip_country - , platform - , score - , last_updated - , page_type - , os_version - , browser - , browser_version - , customer_id - , id - , event - , id_str - , os - , target_id - , customer_id_str - , ip - , screen_height - , external_id - , page_id - , url - , ip_province - , device - , etl_tx_dt - from p00_tal.S98_S_customer_events_click_link_in_page - ; - delete from p12_sfull.S98_S_customer_events_click_link_in_page -; -; -insert into p12_sfull.S98_S_customer_events_click_link_in_page -( date - , screen_width - , target_name - , ip_city - , ip_country - , platform - , score - , last_updated - , page_type - , os_version - , browser - , browser_version - , customer_id - , id - , event - , id_str - , os - , target_id - , customer_id_str - , ip - , screen_height - , external_id - , page_id - , url - , ip_province - , device - , etl_tx_dt ) - select - date - , screen_width - , target_name - , ip_city - , ip_country - , platform - , score - , last_updated - , page_type - , os_version - , browser - , browser_version - , customer_id - , id - , event - , id_str - , os - , target_id - , customer_id_str - , ip - , screen_height - , external_id - , page_id - , url - , ip_province - , device - , etl_tx_dt - from p10_sa.S98_S_customer_events_click_link_in_page -; -\q \ No newline at end of file diff --git a/TK_Cust/dev/tk_api/customer_events_click_link_in_page/sa_foreign_tables.sql b/TK_Cust/dev/tk_api/customer_events_click_link_in_page/sa_foreign_tables.sql deleted file mode 100644 index 121153d..0000000 --- a/TK_Cust/dev/tk_api/customer_events_click_link_in_page/sa_foreign_tables.sql +++ /dev/null @@ -1,38 +0,0 @@ - -CREATE FOREIGN TABLE if not exists p00_tal.S98_S_customer_events_click_link_in_page ( - date TEXT - , screen_width TEXT - , target_name TEXT - , ip_city TEXT - , ip_country TEXT - , platform TEXT - , score TEXT - , last_updated TEXT - , page_type TEXT - , os_version TEXT - , browser TEXT - , browser_version TEXT - , customer_id TEXT - , id TEXT - , event TEXT - , id_str TEXT - , os TEXT - , target_id TEXT - , customer_id_str TEXT - , ip TEXT - , screen_height TEXT - , external_id TEXT - , page_id TEXT - , url TEXT - , ip_province TEXT - , device TEXT - , etl_tx_dt TIMESTAMP -) - - -SERVER pgsql_server_S98_S OPTIONS(schema_name 'data_api', table_name 'customer_events_click_link_in_page' ); - - - - - diff --git a/TK_Cust/dev/tk_api/customer_events_click_link_in_page/sa_tables.sql b/TK_Cust/dev/tk_api/customer_events_click_link_in_page/sa_tables.sql deleted file mode 100644 index ca5b166..0000000 --- a/TK_Cust/dev/tk_api/customer_events_click_link_in_page/sa_tables.sql +++ /dev/null @@ -1,123 +0,0 @@ - -create table if not exists p10_sa.S98_S_customer_events_click_link_in_page ( - date TEXT - , screen_width TEXT - , target_name TEXT - , ip_city TEXT - , ip_country TEXT - , platform TEXT - , score TEXT - , last_updated TEXT - , page_type TEXT - , os_version TEXT - , browser TEXT - , browser_version TEXT - , customer_id TEXT - , id TEXT - , event TEXT - , id_str TEXT - , os TEXT - , target_id TEXT - , customer_id_str TEXT - , ip TEXT - , screen_height TEXT - , external_id TEXT - , page_id TEXT - , url TEXT - , ip_province TEXT - , device TEXT - , etl_tx_dt TIMESTAMP -) ; - - COMMENT ON COLUMN p10_sa.S98_S_customer_events_click_link_in_page.date IS ''; - COMMENT ON COLUMN p10_sa.S98_S_customer_events_click_link_in_page.screen_width IS ''; - COMMENT ON COLUMN p10_sa.S98_S_customer_events_click_link_in_page.target_name IS ''; - COMMENT ON COLUMN p10_sa.S98_S_customer_events_click_link_in_page.ip_city IS ''; - COMMENT ON COLUMN p10_sa.S98_S_customer_events_click_link_in_page.ip_country IS ''; - COMMENT ON COLUMN p10_sa.S98_S_customer_events_click_link_in_page.platform IS ''; - COMMENT ON COLUMN p10_sa.S98_S_customer_events_click_link_in_page.score IS ''; - COMMENT ON COLUMN p10_sa.S98_S_customer_events_click_link_in_page.last_updated IS ''; - COMMENT ON COLUMN p10_sa.S98_S_customer_events_click_link_in_page.page_type IS ''; - COMMENT ON COLUMN p10_sa.S98_S_customer_events_click_link_in_page.os_version IS ''; - COMMENT ON COLUMN p10_sa.S98_S_customer_events_click_link_in_page.browser IS ''; - COMMENT ON COLUMN p10_sa.S98_S_customer_events_click_link_in_page.browser_version IS ''; - COMMENT ON COLUMN p10_sa.S98_S_customer_events_click_link_in_page.customer_id IS ''; - COMMENT ON COLUMN p10_sa.S98_S_customer_events_click_link_in_page.id IS ''; - COMMENT ON COLUMN p10_sa.S98_S_customer_events_click_link_in_page.event IS ''; - COMMENT ON COLUMN p10_sa.S98_S_customer_events_click_link_in_page.id_str IS ''; - COMMENT ON COLUMN p10_sa.S98_S_customer_events_click_link_in_page.os IS ''; - COMMENT ON COLUMN p10_sa.S98_S_customer_events_click_link_in_page.target_id IS ''; - COMMENT ON COLUMN p10_sa.S98_S_customer_events_click_link_in_page.customer_id_str IS ''; - COMMENT ON COLUMN p10_sa.S98_S_customer_events_click_link_in_page.ip IS ''; - COMMENT ON COLUMN p10_sa.S98_S_customer_events_click_link_in_page.screen_height IS ''; - COMMENT ON COLUMN p10_sa.S98_S_customer_events_click_link_in_page.external_id IS ''; - COMMENT ON COLUMN p10_sa.S98_S_customer_events_click_link_in_page.page_id IS ''; - COMMENT ON COLUMN p10_sa.S98_S_customer_events_click_link_in_page.url IS ''; - COMMENT ON COLUMN p10_sa.S98_S_customer_events_click_link_in_page.ip_province IS ''; - COMMENT ON COLUMN p10_sa.S98_S_customer_events_click_link_in_page.device IS ''; - COMMENT ON COLUMN p10_sa.S98_S_customer_events_click_link_in_page.etl_tx_dt IS ''; - -COMMENT ON TABLE p10_sa.S98_S_customer_events_click_link_in_page IS ''; - - - -create table if not exists p12_sfull.S98_S_customer_events_click_link_in_page ( - date TEXT - , screen_width TEXT - , target_name TEXT - , ip_city TEXT - , ip_country TEXT - , platform TEXT - , score TEXT - , last_updated TEXT - , page_type TEXT - , os_version TEXT - , browser TEXT - , browser_version TEXT - , customer_id TEXT - , id TEXT - , event TEXT - , id_str TEXT - , os TEXT - , target_id TEXT - , customer_id_str TEXT - , ip TEXT - , screen_height TEXT - , external_id TEXT - , page_id TEXT - , url TEXT - , ip_province TEXT - , device TEXT - , etl_tx_dt TIMESTAMP -) ; - - COMMENT ON COLUMN p12_sfull.S98_S_customer_events_click_link_in_page.date IS ''; - COMMENT ON COLUMN p12_sfull.S98_S_customer_events_click_link_in_page.screen_width IS ''; - COMMENT ON COLUMN p12_sfull.S98_S_customer_events_click_link_in_page.target_name IS ''; - COMMENT ON COLUMN p12_sfull.S98_S_customer_events_click_link_in_page.ip_city IS ''; - COMMENT ON COLUMN p12_sfull.S98_S_customer_events_click_link_in_page.ip_country IS ''; - COMMENT ON COLUMN p12_sfull.S98_S_customer_events_click_link_in_page.platform IS ''; - COMMENT ON COLUMN p12_sfull.S98_S_customer_events_click_link_in_page.score IS ''; - COMMENT ON COLUMN p12_sfull.S98_S_customer_events_click_link_in_page.last_updated IS ''; - COMMENT ON COLUMN p12_sfull.S98_S_customer_events_click_link_in_page.page_type IS ''; - COMMENT ON COLUMN p12_sfull.S98_S_customer_events_click_link_in_page.os_version IS ''; - COMMENT ON COLUMN p12_sfull.S98_S_customer_events_click_link_in_page.browser IS ''; - COMMENT ON COLUMN p12_sfull.S98_S_customer_events_click_link_in_page.browser_version IS ''; - COMMENT ON COLUMN p12_sfull.S98_S_customer_events_click_link_in_page.customer_id IS ''; - COMMENT ON COLUMN p12_sfull.S98_S_customer_events_click_link_in_page.id IS ''; - COMMENT ON COLUMN p12_sfull.S98_S_customer_events_click_link_in_page.event IS ''; - COMMENT ON COLUMN p12_sfull.S98_S_customer_events_click_link_in_page.id_str IS ''; - COMMENT ON COLUMN p12_sfull.S98_S_customer_events_click_link_in_page.os IS ''; - COMMENT ON COLUMN p12_sfull.S98_S_customer_events_click_link_in_page.target_id IS ''; - COMMENT ON COLUMN p12_sfull.S98_S_customer_events_click_link_in_page.customer_id_str IS ''; - COMMENT ON COLUMN p12_sfull.S98_S_customer_events_click_link_in_page.ip IS ''; - COMMENT ON COLUMN p12_sfull.S98_S_customer_events_click_link_in_page.screen_height IS ''; - COMMENT ON COLUMN p12_sfull.S98_S_customer_events_click_link_in_page.external_id IS ''; - COMMENT ON COLUMN p12_sfull.S98_S_customer_events_click_link_in_page.page_id IS ''; - COMMENT ON COLUMN p12_sfull.S98_S_customer_events_click_link_in_page.url IS ''; - COMMENT ON COLUMN p12_sfull.S98_S_customer_events_click_link_in_page.ip_province IS ''; - COMMENT ON COLUMN p12_sfull.S98_S_customer_events_click_link_in_page.device IS ''; - COMMENT ON COLUMN p12_sfull.S98_S_customer_events_click_link_in_page.etl_tx_dt IS ''; - -COMMENT ON TABLE p12_sfull.S98_S_customer_events_click_link_in_page IS ''; - diff --git a/TK_Cust/dev/tk_api/customer_events_open_app/S98_S_customer_events_open_app.sql b/TK_Cust/dev/tk_api/customer_events_open_app/S98_S_customer_events_open_app.sql deleted file mode 100644 index 4bb96ba..0000000 --- a/TK_Cust/dev/tk_api/customer_events_open_app/S98_S_customer_events_open_app.sql +++ /dev/null @@ -1,126 +0,0 @@ -/*******Main Section**************************************************************************/ -\set ON_ERROR_STOP on -\set AUTOCOMMIT on -\timing on -delete from p10_sa.S98_S_customer_events_open_app -; -insert into p10_sa.S98_S_customer_events_open_app -( date - , screen_width - , target_name - , ip_city - , ip_country - , platform - , last_updated - , page_type - , os_version - , browser - , browser_version - , customer_id - , id - , event - , id_str - , os - , target_id - , customer_id_str - , ip - , screen_height - , external_id - , ip_county - , page_id - , url - , ip_province - , device - , etl_tx_dt ) - select - date - , screen_width - , target_name - , ip_city - , ip_country - , platform - , last_updated - , page_type - , os_version - , browser - , browser_version - , customer_id - , id - , event - , id_str - , os - , target_id - , customer_id_str - , ip - , screen_height - , external_id - , ip_county - , page_id - , url - , ip_province - , device - , etl_tx_dt - from p00_tal.S98_S_customer_events_open_app - ; - delete from p12_sfull.S98_S_customer_events_open_app -; -; -insert into p12_sfull.S98_S_customer_events_open_app -( date - , screen_width - , target_name - , ip_city - , ip_country - , platform - , last_updated - , page_type - , os_version - , browser - , browser_version - , customer_id - , id - , event - , id_str - , os - , target_id - , customer_id_str - , ip - , screen_height - , external_id - , ip_county - , page_id - , url - , ip_province - , device - , etl_tx_dt ) - select - date - , screen_width - , target_name - , ip_city - , ip_country - , platform - , last_updated - , page_type - , os_version - , browser - , browser_version - , customer_id - , id - , event - , id_str - , os - , target_id - , customer_id_str - , ip - , screen_height - , external_id - , ip_county - , page_id - , url - , ip_province - , device - , etl_tx_dt - from p10_sa.S98_S_customer_events_open_app -; -\q \ No newline at end of file diff --git a/TK_Cust/dev/tk_api/customer_events_open_app/sa_foreign_tables.sql b/TK_Cust/dev/tk_api/customer_events_open_app/sa_foreign_tables.sql deleted file mode 100644 index 8efad6a..0000000 --- a/TK_Cust/dev/tk_api/customer_events_open_app/sa_foreign_tables.sql +++ /dev/null @@ -1,38 +0,0 @@ - -CREATE FOREIGN TABLE if not exists p00_tal.S98_S_customer_events_open_app ( - date TEXT - , screen_width TEXT - , target_name TEXT - , ip_city TEXT - , ip_country TEXT - , platform TEXT - , last_updated TEXT - , page_type TEXT - , os_version TEXT - , browser TEXT - , browser_version TEXT - , customer_id TEXT - , id TEXT - , event TEXT - , id_str TEXT - , os TEXT - , target_id TEXT - , customer_id_str TEXT - , ip TEXT - , screen_height TEXT - , external_id TEXT - , ip_county TEXT - , page_id TEXT - , url TEXT - , ip_province TEXT - , device TEXT - , etl_tx_dt TIMESTAMP -) - - -SERVER pgsql_server_S98_S OPTIONS(schema_name 'data_api', table_name 'customer_events_open_app' ); - - - - - diff --git a/TK_Cust/dev/tk_api/customer_events_open_app/sa_tables.sql b/TK_Cust/dev/tk_api/customer_events_open_app/sa_tables.sql deleted file mode 100644 index 88b635b..0000000 --- a/TK_Cust/dev/tk_api/customer_events_open_app/sa_tables.sql +++ /dev/null @@ -1,123 +0,0 @@ - -create table if not exists p10_sa.S98_S_customer_events_open_app ( - date TEXT - , screen_width TEXT - , target_name TEXT - , ip_city TEXT - , ip_country TEXT - , platform TEXT - , last_updated TEXT - , page_type TEXT - , os_version TEXT - , browser TEXT - , browser_version TEXT - , customer_id TEXT - , id TEXT - , event TEXT - , id_str TEXT - , os TEXT - , target_id TEXT - , customer_id_str TEXT - , ip TEXT - , screen_height TEXT - , external_id TEXT - , ip_county TEXT - , page_id TEXT - , url TEXT - , ip_province TEXT - , device TEXT - , etl_tx_dt TIMESTAMP -) ; - - COMMENT ON COLUMN p10_sa.S98_S_customer_events_open_app.date IS ''; - COMMENT ON COLUMN p10_sa.S98_S_customer_events_open_app.screen_width IS ''; - COMMENT ON COLUMN p10_sa.S98_S_customer_events_open_app.target_name IS ''; - COMMENT ON COLUMN p10_sa.S98_S_customer_events_open_app.ip_city IS ''; - COMMENT ON COLUMN p10_sa.S98_S_customer_events_open_app.ip_country IS ''; - COMMENT ON COLUMN p10_sa.S98_S_customer_events_open_app.platform IS ''; - COMMENT ON COLUMN p10_sa.S98_S_customer_events_open_app.last_updated IS ''; - COMMENT ON COLUMN p10_sa.S98_S_customer_events_open_app.page_type IS ''; - COMMENT ON COLUMN p10_sa.S98_S_customer_events_open_app.os_version IS ''; - COMMENT ON COLUMN p10_sa.S98_S_customer_events_open_app.browser IS ''; - COMMENT ON COLUMN p10_sa.S98_S_customer_events_open_app.browser_version IS ''; - COMMENT ON COLUMN p10_sa.S98_S_customer_events_open_app.customer_id IS ''; - COMMENT ON COLUMN p10_sa.S98_S_customer_events_open_app.id IS ''; - COMMENT ON COLUMN p10_sa.S98_S_customer_events_open_app.event IS ''; - COMMENT ON COLUMN p10_sa.S98_S_customer_events_open_app.id_str IS ''; - COMMENT ON COLUMN p10_sa.S98_S_customer_events_open_app.os IS ''; - COMMENT ON COLUMN p10_sa.S98_S_customer_events_open_app.target_id IS ''; - COMMENT ON COLUMN p10_sa.S98_S_customer_events_open_app.customer_id_str IS ''; - COMMENT ON COLUMN p10_sa.S98_S_customer_events_open_app.ip IS ''; - COMMENT ON COLUMN p10_sa.S98_S_customer_events_open_app.screen_height IS ''; - COMMENT ON COLUMN p10_sa.S98_S_customer_events_open_app.external_id IS ''; - COMMENT ON COLUMN p10_sa.S98_S_customer_events_open_app.ip_county IS ''; - COMMENT ON COLUMN p10_sa.S98_S_customer_events_open_app.page_id IS ''; - COMMENT ON COLUMN p10_sa.S98_S_customer_events_open_app.url IS ''; - COMMENT ON COLUMN p10_sa.S98_S_customer_events_open_app.ip_province IS ''; - COMMENT ON COLUMN p10_sa.S98_S_customer_events_open_app.device IS ''; - COMMENT ON COLUMN p10_sa.S98_S_customer_events_open_app.etl_tx_dt IS ''; - -COMMENT ON TABLE p10_sa.S98_S_customer_events_open_app IS ''; - - - -create table if not exists p12_sfull.S98_S_customer_events_open_app ( - date TEXT - , screen_width TEXT - , target_name TEXT - , ip_city TEXT - , ip_country TEXT - , platform TEXT - , last_updated TEXT - , page_type TEXT - , os_version TEXT - , browser TEXT - , browser_version TEXT - , customer_id TEXT - , id TEXT - , event TEXT - , id_str TEXT - , os TEXT - , target_id TEXT - , customer_id_str TEXT - , ip TEXT - , screen_height TEXT - , external_id TEXT - , ip_county TEXT - , page_id TEXT - , url TEXT - , ip_province TEXT - , device TEXT - , etl_tx_dt TIMESTAMP -) ; - - COMMENT ON COLUMN p12_sfull.S98_S_customer_events_open_app.date IS ''; - COMMENT ON COLUMN p12_sfull.S98_S_customer_events_open_app.screen_width IS ''; - COMMENT ON COLUMN p12_sfull.S98_S_customer_events_open_app.target_name IS ''; - COMMENT ON COLUMN p12_sfull.S98_S_customer_events_open_app.ip_city IS ''; - COMMENT ON COLUMN p12_sfull.S98_S_customer_events_open_app.ip_country IS ''; - COMMENT ON COLUMN p12_sfull.S98_S_customer_events_open_app.platform IS ''; - COMMENT ON COLUMN p12_sfull.S98_S_customer_events_open_app.last_updated IS ''; - COMMENT ON COLUMN p12_sfull.S98_S_customer_events_open_app.page_type IS ''; - COMMENT ON COLUMN p12_sfull.S98_S_customer_events_open_app.os_version IS ''; - COMMENT ON COLUMN p12_sfull.S98_S_customer_events_open_app.browser IS ''; - COMMENT ON COLUMN p12_sfull.S98_S_customer_events_open_app.browser_version IS ''; - COMMENT ON COLUMN p12_sfull.S98_S_customer_events_open_app.customer_id IS ''; - COMMENT ON COLUMN p12_sfull.S98_S_customer_events_open_app.id IS ''; - COMMENT ON COLUMN p12_sfull.S98_S_customer_events_open_app.event IS ''; - COMMENT ON COLUMN p12_sfull.S98_S_customer_events_open_app.id_str IS ''; - COMMENT ON COLUMN p12_sfull.S98_S_customer_events_open_app.os IS ''; - COMMENT ON COLUMN p12_sfull.S98_S_customer_events_open_app.target_id IS ''; - COMMENT ON COLUMN p12_sfull.S98_S_customer_events_open_app.customer_id_str IS ''; - COMMENT ON COLUMN p12_sfull.S98_S_customer_events_open_app.ip IS ''; - COMMENT ON COLUMN p12_sfull.S98_S_customer_events_open_app.screen_height IS ''; - COMMENT ON COLUMN p12_sfull.S98_S_customer_events_open_app.external_id IS ''; - COMMENT ON COLUMN p12_sfull.S98_S_customer_events_open_app.ip_county IS ''; - COMMENT ON COLUMN p12_sfull.S98_S_customer_events_open_app.page_id IS ''; - COMMENT ON COLUMN p12_sfull.S98_S_customer_events_open_app.url IS ''; - COMMENT ON COLUMN p12_sfull.S98_S_customer_events_open_app.ip_province IS ''; - COMMENT ON COLUMN p12_sfull.S98_S_customer_events_open_app.device IS ''; - COMMENT ON COLUMN p12_sfull.S98_S_customer_events_open_app.etl_tx_dt IS ''; - -COMMENT ON TABLE p12_sfull.S98_S_customer_events_open_app IS ''; - diff --git a/TK_Cust/dev/tk_api/customer_events_open_page/S98_S_customer_events_open_page.sql b/TK_Cust/dev/tk_api/customer_events_open_page/S98_S_customer_events_open_page.sql deleted file mode 100644 index e1ff9a6..0000000 --- a/TK_Cust/dev/tk_api/customer_events_open_page/S98_S_customer_events_open_page.sql +++ /dev/null @@ -1,150 +0,0 @@ -/*******Main Section**************************************************************************/ -\set ON_ERROR_STOP on -\set AUTOCOMMIT on -\timing on -delete from p10_sa.S98_S_customer_events_open_page -; -insert into p10_sa.S98_S_customer_events_open_page -( date - , short_id - , screen_width - , target_name - , ip_city - , ip_country - , source1 - , source - , platform - , path - , score - , last_updated - , page_type - , os_version - , browser - , browser_version - , customer_id - , id - , event - , id_str - , os - , target_id - , customer_id_str - , ip - , screen_height - , root_domain - , external_id - , page_id - , url - , domain - , ip_province - , device - , etl_tx_dt ) - select - date - , short_id - , screen_width - , target_name - , ip_city - , ip_country - , source1 - , source - , platform - , path - , score - , last_updated - , page_type - , os_version - , browser - , browser_version - , customer_id - , id - , event - , id_str - , os - , target_id - , customer_id_str - , ip - , screen_height - , root_domain - , external_id - , page_id - , url - , domain - , ip_province - , device - , etl_tx_dt - from p00_tal.S98_S_customer_events_open_page - ; - delete from p12_sfull.S98_S_customer_events_open_page -; -; -insert into p12_sfull.S98_S_customer_events_open_page -( date - , short_id - , screen_width - , target_name - , ip_city - , ip_country - , source1 - , source - , platform - , path - , score - , last_updated - , page_type - , os_version - , browser - , browser_version - , customer_id - , id - , event - , id_str - , os - , target_id - , customer_id_str - , ip - , screen_height - , root_domain - , external_id - , page_id - , url - , domain - , ip_province - , device - , etl_tx_dt ) - select - date - , short_id - , screen_width - , target_name - , ip_city - , ip_country - , source1 - , source - , platform - , path - , score - , last_updated - , page_type - , os_version - , browser - , browser_version - , customer_id - , id - , event - , id_str - , os - , target_id - , customer_id_str - , ip - , screen_height - , root_domain - , external_id - , page_id - , url - , domain - , ip_province - , device - , etl_tx_dt - from p10_sa.S98_S_customer_events_open_page -; -\q \ No newline at end of file diff --git a/TK_Cust/dev/tk_api/customer_events_open_page/sa_foreign_tables.sql b/TK_Cust/dev/tk_api/customer_events_open_page/sa_foreign_tables.sql deleted file mode 100644 index 0a63b5e..0000000 --- a/TK_Cust/dev/tk_api/customer_events_open_page/sa_foreign_tables.sql +++ /dev/null @@ -1,44 +0,0 @@ - -CREATE FOREIGN TABLE if not exists p00_tal.S98_S_customer_events_open_page ( - date TEXT - , short_id TEXT - , screen_width TEXT - , target_name TEXT - , ip_city TEXT - , ip_country TEXT - , source1 TEXT - , source TEXT - , platform TEXT - , path TEXT - , score TEXT - , last_updated TEXT - , page_type TEXT - , os_version TEXT - , browser TEXT - , browser_version TEXT - , customer_id TEXT - , id TEXT - , event TEXT - , id_str TEXT - , os TEXT - , target_id TEXT - , customer_id_str TEXT - , ip TEXT - , screen_height TEXT - , root_domain TEXT - , external_id TEXT - , page_id TEXT - , url TEXT - , domain TEXT - , ip_province TEXT - , device TEXT - , etl_tx_dt TIMESTAMP -) - - -SERVER pgsql_server_S98_S OPTIONS(schema_name 'data_api', table_name 'customer_events_open_page' ); - - - - - diff --git a/TK_Cust/dev/tk_api/customer_events_open_page/sa_tables.sql b/TK_Cust/dev/tk_api/customer_events_open_page/sa_tables.sql deleted file mode 100644 index 6329bcc..0000000 --- a/TK_Cust/dev/tk_api/customer_events_open_page/sa_tables.sql +++ /dev/null @@ -1,147 +0,0 @@ - -create table if not exists p10_sa.S98_S_customer_events_open_page ( - date TEXT - , short_id TEXT - , screen_width TEXT - , target_name TEXT - , ip_city TEXT - , ip_country TEXT - , source1 TEXT - , source TEXT - , platform TEXT - , path TEXT - , score TEXT - , last_updated TEXT - , page_type TEXT - , os_version TEXT - , browser TEXT - , browser_version TEXT - , customer_id TEXT - , id TEXT - , event TEXT - , id_str TEXT - , os TEXT - , target_id TEXT - , customer_id_str TEXT - , ip TEXT - , screen_height TEXT - , root_domain TEXT - , external_id TEXT - , page_id TEXT - , url TEXT - , domain TEXT - , ip_province TEXT - , device TEXT - , etl_tx_dt TIMESTAMP -) ; - - COMMENT ON COLUMN p10_sa.S98_S_customer_events_open_page.date IS ''; - COMMENT ON COLUMN p10_sa.S98_S_customer_events_open_page.short_id IS ''; - COMMENT ON COLUMN p10_sa.S98_S_customer_events_open_page.screen_width IS ''; - COMMENT ON COLUMN p10_sa.S98_S_customer_events_open_page.target_name IS ''; - COMMENT ON COLUMN p10_sa.S98_S_customer_events_open_page.ip_city IS ''; - COMMENT ON COLUMN p10_sa.S98_S_customer_events_open_page.ip_country IS ''; - COMMENT ON COLUMN p10_sa.S98_S_customer_events_open_page.source1 IS ''; - COMMENT ON COLUMN p10_sa.S98_S_customer_events_open_page.source IS ''; - COMMENT ON COLUMN p10_sa.S98_S_customer_events_open_page.platform IS ''; - COMMENT ON COLUMN p10_sa.S98_S_customer_events_open_page.path IS ''; - COMMENT ON COLUMN p10_sa.S98_S_customer_events_open_page.score IS ''; - COMMENT ON COLUMN p10_sa.S98_S_customer_events_open_page.last_updated IS ''; - COMMENT ON COLUMN p10_sa.S98_S_customer_events_open_page.page_type IS ''; - COMMENT ON COLUMN p10_sa.S98_S_customer_events_open_page.os_version IS ''; - COMMENT ON COLUMN p10_sa.S98_S_customer_events_open_page.browser IS ''; - COMMENT ON COLUMN p10_sa.S98_S_customer_events_open_page.browser_version IS ''; - COMMENT ON COLUMN p10_sa.S98_S_customer_events_open_page.customer_id IS ''; - COMMENT ON COLUMN p10_sa.S98_S_customer_events_open_page.id IS ''; - COMMENT ON COLUMN p10_sa.S98_S_customer_events_open_page.event IS ''; - COMMENT ON COLUMN p10_sa.S98_S_customer_events_open_page.id_str IS ''; - COMMENT ON COLUMN p10_sa.S98_S_customer_events_open_page.os IS ''; - COMMENT ON COLUMN p10_sa.S98_S_customer_events_open_page.target_id IS ''; - COMMENT ON COLUMN p10_sa.S98_S_customer_events_open_page.customer_id_str IS ''; - COMMENT ON COLUMN p10_sa.S98_S_customer_events_open_page.ip IS ''; - COMMENT ON COLUMN p10_sa.S98_S_customer_events_open_page.screen_height IS ''; - COMMENT ON COLUMN p10_sa.S98_S_customer_events_open_page.root_domain IS ''; - COMMENT ON COLUMN p10_sa.S98_S_customer_events_open_page.external_id IS ''; - COMMENT ON COLUMN p10_sa.S98_S_customer_events_open_page.page_id IS ''; - COMMENT ON COLUMN p10_sa.S98_S_customer_events_open_page.url IS ''; - COMMENT ON COLUMN p10_sa.S98_S_customer_events_open_page.domain IS ''; - COMMENT ON COLUMN p10_sa.S98_S_customer_events_open_page.ip_province IS ''; - COMMENT ON COLUMN p10_sa.S98_S_customer_events_open_page.device IS ''; - COMMENT ON COLUMN p10_sa.S98_S_customer_events_open_page.etl_tx_dt IS ''; - -COMMENT ON TABLE p10_sa.S98_S_customer_events_open_page IS ''; - - - -create table if not exists p12_sfull.S98_S_customer_events_open_page ( - date TEXT - , short_id TEXT - , screen_width TEXT - , target_name TEXT - , ip_city TEXT - , ip_country TEXT - , source1 TEXT - , source TEXT - , platform TEXT - , path TEXT - , score TEXT - , last_updated TEXT - , page_type TEXT - , os_version TEXT - , browser TEXT - , browser_version TEXT - , customer_id TEXT - , id TEXT - , event TEXT - , id_str TEXT - , os TEXT - , target_id TEXT - , customer_id_str TEXT - , ip TEXT - , screen_height TEXT - , root_domain TEXT - , external_id TEXT - , page_id TEXT - , url TEXT - , domain TEXT - , ip_province TEXT - , device TEXT - , etl_tx_dt TIMESTAMP -) ; - - COMMENT ON COLUMN p12_sfull.S98_S_customer_events_open_page.date IS ''; - COMMENT ON COLUMN p12_sfull.S98_S_customer_events_open_page.short_id IS ''; - COMMENT ON COLUMN p12_sfull.S98_S_customer_events_open_page.screen_width IS ''; - COMMENT ON COLUMN p12_sfull.S98_S_customer_events_open_page.target_name IS ''; - COMMENT ON COLUMN p12_sfull.S98_S_customer_events_open_page.ip_city IS ''; - COMMENT ON COLUMN p12_sfull.S98_S_customer_events_open_page.ip_country IS ''; - COMMENT ON COLUMN p12_sfull.S98_S_customer_events_open_page.source1 IS ''; - COMMENT ON COLUMN p12_sfull.S98_S_customer_events_open_page.source IS ''; - COMMENT ON COLUMN p12_sfull.S98_S_customer_events_open_page.platform IS ''; - COMMENT ON COLUMN p12_sfull.S98_S_customer_events_open_page.path IS ''; - COMMENT ON COLUMN p12_sfull.S98_S_customer_events_open_page.score IS ''; - COMMENT ON COLUMN p12_sfull.S98_S_customer_events_open_page.last_updated IS ''; - COMMENT ON COLUMN p12_sfull.S98_S_customer_events_open_page.page_type IS ''; - COMMENT ON COLUMN p12_sfull.S98_S_customer_events_open_page.os_version IS ''; - COMMENT ON COLUMN p12_sfull.S98_S_customer_events_open_page.browser IS ''; - COMMENT ON COLUMN p12_sfull.S98_S_customer_events_open_page.browser_version IS ''; - COMMENT ON COLUMN p12_sfull.S98_S_customer_events_open_page.customer_id IS ''; - COMMENT ON COLUMN p12_sfull.S98_S_customer_events_open_page.id IS ''; - COMMENT ON COLUMN p12_sfull.S98_S_customer_events_open_page.event IS ''; - COMMENT ON COLUMN p12_sfull.S98_S_customer_events_open_page.id_str IS ''; - COMMENT ON COLUMN p12_sfull.S98_S_customer_events_open_page.os IS ''; - COMMENT ON COLUMN p12_sfull.S98_S_customer_events_open_page.target_id IS ''; - COMMENT ON COLUMN p12_sfull.S98_S_customer_events_open_page.customer_id_str IS ''; - COMMENT ON COLUMN p12_sfull.S98_S_customer_events_open_page.ip IS ''; - COMMENT ON COLUMN p12_sfull.S98_S_customer_events_open_page.screen_height IS ''; - COMMENT ON COLUMN p12_sfull.S98_S_customer_events_open_page.root_domain IS ''; - COMMENT ON COLUMN p12_sfull.S98_S_customer_events_open_page.external_id IS ''; - COMMENT ON COLUMN p12_sfull.S98_S_customer_events_open_page.page_id IS ''; - COMMENT ON COLUMN p12_sfull.S98_S_customer_events_open_page.url IS ''; - COMMENT ON COLUMN p12_sfull.S98_S_customer_events_open_page.domain IS ''; - COMMENT ON COLUMN p12_sfull.S98_S_customer_events_open_page.ip_province IS ''; - COMMENT ON COLUMN p12_sfull.S98_S_customer_events_open_page.device IS ''; - COMMENT ON COLUMN p12_sfull.S98_S_customer_events_open_page.etl_tx_dt IS ''; - -COMMENT ON TABLE p12_sfull.S98_S_customer_events_open_page IS ''; - diff --git a/TK_Cust/dev/tk_api/customer_events_submit_form/S98_S_customer_events_submit_form.sql b/TK_Cust/dev/tk_api/customer_events_submit_form/S98_S_customer_events_submit_form.sql deleted file mode 100644 index b2a745f..0000000 --- a/TK_Cust/dev/tk_api/customer_events_submit_form/S98_S_customer_events_submit_form.sql +++ /dev/null @@ -1,118 +0,0 @@ -/*******Main Section**************************************************************************/ -\set ON_ERROR_STOP on -\set AUTOCOMMIT on -\timing on -delete from p10_sa.S98_S_customer_events_submit_form -; -insert into p10_sa.S98_S_customer_events_submit_form -( id_str - , date - , target_name - , os - , target_id - , customer_id_str - , ip_city - , ip - , external_id - , ip_country - , ip_county - , page_id - , platform - , score - , last_updated - , page_type - , os_version - , browser - , browser_version - , customer_id - , ip_province - , id - , event - , submit_id - , etl_tx_dt ) - select - id_str - , date - , target_name - , os - , target_id - , customer_id_str - , ip_city - , ip - , external_id - , ip_country - , ip_county - , page_id - , platform - , score - , last_updated - , page_type - , os_version - , browser - , browser_version - , customer_id - , ip_province - , id - , event - , submit_id - , etl_tx_dt - from p00_tal.S98_S_customer_events_submit_form - ; - delete from p12_sfull.S98_S_customer_events_submit_form -; -; -insert into p12_sfull.S98_S_customer_events_submit_form -( id_str - , date - , target_name - , os - , target_id - , customer_id_str - , ip_city - , ip - , external_id - , ip_country - , ip_county - , page_id - , platform - , score - , last_updated - , page_type - , os_version - , browser - , browser_version - , customer_id - , ip_province - , id - , event - , submit_id - , etl_tx_dt ) - select - id_str - , date - , target_name - , os - , target_id - , customer_id_str - , ip_city - , ip - , external_id - , ip_country - , ip_county - , page_id - , platform - , score - , last_updated - , page_type - , os_version - , browser - , browser_version - , customer_id - , ip_province - , id - , event - , submit_id - , etl_tx_dt - from p10_sa.S98_S_customer_events_submit_form -; -\q \ No newline at end of file diff --git a/TK_Cust/dev/tk_api/customer_events_submit_form/sa_foreign_tables.sql b/TK_Cust/dev/tk_api/customer_events_submit_form/sa_foreign_tables.sql deleted file mode 100644 index ed9452a..0000000 --- a/TK_Cust/dev/tk_api/customer_events_submit_form/sa_foreign_tables.sql +++ /dev/null @@ -1,36 +0,0 @@ - -CREATE FOREIGN TABLE if not exists p00_tal.S98_S_customer_events_submit_form ( - id_str TEXT - , date TEXT - , target_name TEXT - , os TEXT - , target_id TEXT - , customer_id_str TEXT - , ip_city TEXT - , ip TEXT - , external_id TEXT - , ip_country TEXT - , ip_county TEXT - , page_id TEXT - , platform TEXT - , score TEXT - , last_updated TEXT - , page_type TEXT - , os_version TEXT - , browser TEXT - , browser_version TEXT - , customer_id TEXT - , ip_province TEXT - , id TEXT - , event TEXT - , submit_id TEXT - , etl_tx_dt TIMESTAMP -) - - -SERVER pgsql_server_S98_S OPTIONS(schema_name 'data_api', table_name 'customer_events_submit_form' ); - - - - - diff --git a/TK_Cust/dev/tk_api/customer_events_submit_form/sa_tables.sql b/TK_Cust/dev/tk_api/customer_events_submit_form/sa_tables.sql deleted file mode 100644 index f997ba6..0000000 --- a/TK_Cust/dev/tk_api/customer_events_submit_form/sa_tables.sql +++ /dev/null @@ -1,115 +0,0 @@ - -create table if not exists p10_sa.S98_S_customer_events_submit_form ( - id_str TEXT - , date TEXT - , target_name TEXT - , os TEXT - , target_id TEXT - , customer_id_str TEXT - , ip_city TEXT - , ip TEXT - , external_id TEXT - , ip_country TEXT - , ip_county TEXT - , page_id TEXT - , platform TEXT - , score TEXT - , last_updated TEXT - , page_type TEXT - , os_version TEXT - , browser TEXT - , browser_version TEXT - , customer_id TEXT - , ip_province TEXT - , id TEXT - , event TEXT - , submit_id TEXT - , etl_tx_dt TIMESTAMP -) ; - - COMMENT ON COLUMN p10_sa.S98_S_customer_events_submit_form.id_str IS ''; - COMMENT ON COLUMN p10_sa.S98_S_customer_events_submit_form.date IS ''; - COMMENT ON COLUMN p10_sa.S98_S_customer_events_submit_form.target_name IS ''; - COMMENT ON COLUMN p10_sa.S98_S_customer_events_submit_form.os IS ''; - COMMENT ON COLUMN p10_sa.S98_S_customer_events_submit_form.target_id IS ''; - COMMENT ON COLUMN p10_sa.S98_S_customer_events_submit_form.customer_id_str IS ''; - COMMENT ON COLUMN p10_sa.S98_S_customer_events_submit_form.ip_city IS ''; - COMMENT ON COLUMN p10_sa.S98_S_customer_events_submit_form.ip IS ''; - COMMENT ON COLUMN p10_sa.S98_S_customer_events_submit_form.external_id IS ''; - COMMENT ON COLUMN p10_sa.S98_S_customer_events_submit_form.ip_country IS ''; - COMMENT ON COLUMN p10_sa.S98_S_customer_events_submit_form.ip_county IS ''; - COMMENT ON COLUMN p10_sa.S98_S_customer_events_submit_form.page_id IS ''; - COMMENT ON COLUMN p10_sa.S98_S_customer_events_submit_form.platform IS ''; - COMMENT ON COLUMN p10_sa.S98_S_customer_events_submit_form.score IS ''; - COMMENT ON COLUMN p10_sa.S98_S_customer_events_submit_form.last_updated IS ''; - COMMENT ON COLUMN p10_sa.S98_S_customer_events_submit_form.page_type IS ''; - COMMENT ON COLUMN p10_sa.S98_S_customer_events_submit_form.os_version IS ''; - COMMENT ON COLUMN p10_sa.S98_S_customer_events_submit_form.browser IS ''; - COMMENT ON COLUMN p10_sa.S98_S_customer_events_submit_form.browser_version IS ''; - COMMENT ON COLUMN p10_sa.S98_S_customer_events_submit_form.customer_id IS ''; - COMMENT ON COLUMN p10_sa.S98_S_customer_events_submit_form.ip_province IS ''; - COMMENT ON COLUMN p10_sa.S98_S_customer_events_submit_form.id IS ''; - COMMENT ON COLUMN p10_sa.S98_S_customer_events_submit_form.event IS ''; - COMMENT ON COLUMN p10_sa.S98_S_customer_events_submit_form.submit_id IS ''; - COMMENT ON COLUMN p10_sa.S98_S_customer_events_submit_form.etl_tx_dt IS ''; - -COMMENT ON TABLE p10_sa.S98_S_customer_events_submit_form IS ''; - - - -create table if not exists p12_sfull.S98_S_customer_events_submit_form ( - id_str TEXT - , date TEXT - , target_name TEXT - , os TEXT - , target_id TEXT - , customer_id_str TEXT - , ip_city TEXT - , ip TEXT - , external_id TEXT - , ip_country TEXT - , ip_county TEXT - , page_id TEXT - , platform TEXT - , score TEXT - , last_updated TEXT - , page_type TEXT - , os_version TEXT - , browser TEXT - , browser_version TEXT - , customer_id TEXT - , ip_province TEXT - , id TEXT - , event TEXT - , submit_id TEXT - , etl_tx_dt TIMESTAMP -) ; - - COMMENT ON COLUMN p12_sfull.S98_S_customer_events_submit_form.id_str IS ''; - COMMENT ON COLUMN p12_sfull.S98_S_customer_events_submit_form.date IS ''; - COMMENT ON COLUMN p12_sfull.S98_S_customer_events_submit_form.target_name IS ''; - COMMENT ON COLUMN p12_sfull.S98_S_customer_events_submit_form.os IS ''; - COMMENT ON COLUMN p12_sfull.S98_S_customer_events_submit_form.target_id IS ''; - COMMENT ON COLUMN p12_sfull.S98_S_customer_events_submit_form.customer_id_str IS ''; - COMMENT ON COLUMN p12_sfull.S98_S_customer_events_submit_form.ip_city IS ''; - COMMENT ON COLUMN p12_sfull.S98_S_customer_events_submit_form.ip IS ''; - COMMENT ON COLUMN p12_sfull.S98_S_customer_events_submit_form.external_id IS ''; - COMMENT ON COLUMN p12_sfull.S98_S_customer_events_submit_form.ip_country IS ''; - COMMENT ON COLUMN p12_sfull.S98_S_customer_events_submit_form.ip_county IS ''; - COMMENT ON COLUMN p12_sfull.S98_S_customer_events_submit_form.page_id IS ''; - COMMENT ON COLUMN p12_sfull.S98_S_customer_events_submit_form.platform IS ''; - COMMENT ON COLUMN p12_sfull.S98_S_customer_events_submit_form.score IS ''; - COMMENT ON COLUMN p12_sfull.S98_S_customer_events_submit_form.last_updated IS ''; - COMMENT ON COLUMN p12_sfull.S98_S_customer_events_submit_form.page_type IS ''; - COMMENT ON COLUMN p12_sfull.S98_S_customer_events_submit_form.os_version IS ''; - COMMENT ON COLUMN p12_sfull.S98_S_customer_events_submit_form.browser IS ''; - COMMENT ON COLUMN p12_sfull.S98_S_customer_events_submit_form.browser_version IS ''; - COMMENT ON COLUMN p12_sfull.S98_S_customer_events_submit_form.customer_id IS ''; - COMMENT ON COLUMN p12_sfull.S98_S_customer_events_submit_form.ip_province IS ''; - COMMENT ON COLUMN p12_sfull.S98_S_customer_events_submit_form.id IS ''; - COMMENT ON COLUMN p12_sfull.S98_S_customer_events_submit_form.event IS ''; - COMMENT ON COLUMN p12_sfull.S98_S_customer_events_submit_form.submit_id IS ''; - COMMENT ON COLUMN p12_sfull.S98_S_customer_events_submit_form.etl_tx_dt IS ''; - -COMMENT ON TABLE p12_sfull.S98_S_customer_events_submit_form IS ''; - diff --git a/TK_Cust/dev/tk_api/customer_events_subscribe/S98_S_customer_events_subscribe.sql b/TK_Cust/dev/tk_api/customer_events_subscribe/S98_S_customer_events_subscribe.sql deleted file mode 100644 index f640a7c..0000000 --- a/TK_Cust/dev/tk_api/customer_events_subscribe/S98_S_customer_events_subscribe.sql +++ /dev/null @@ -1,70 +0,0 @@ -/*******Main Section**************************************************************************/ -\set ON_ERROR_STOP on -\set AUTOCOMMIT on -\timing on -delete from p10_sa.S98_S_customer_events_subscribe -; -insert into p10_sa.S98_S_customer_events_subscribe -( id_str - , date - , score - , last_updated - , target_name - , target_id - , customer_id_str - , channel_account - , customer_id - , external_id - , id - , event - , etl_tx_dt ) - select - id_str - , date - , score - , last_updated - , target_name - , target_id - , customer_id_str - , channel_account - , customer_id - , external_id - , id - , event - , etl_tx_dt - from p00_tal.S98_S_customer_events_subscribe - ; - delete from p12_sfull.S98_S_customer_events_subscribe -; -; -insert into p12_sfull.S98_S_customer_events_subscribe -( id_str - , date - , score - , last_updated - , target_name - , target_id - , customer_id_str - , channel_account - , customer_id - , external_id - , id - , event - , etl_tx_dt ) - select - id_str - , date - , score - , last_updated - , target_name - , target_id - , customer_id_str - , channel_account - , customer_id - , external_id - , id - , event - , etl_tx_dt - from p10_sa.S98_S_customer_events_subscribe -; -\q \ No newline at end of file diff --git a/TK_Cust/dev/tk_api/customer_events_subscribe/sa_foreign_tables.sql b/TK_Cust/dev/tk_api/customer_events_subscribe/sa_foreign_tables.sql deleted file mode 100644 index 350b654..0000000 --- a/TK_Cust/dev/tk_api/customer_events_subscribe/sa_foreign_tables.sql +++ /dev/null @@ -1,24 +0,0 @@ - -CREATE FOREIGN TABLE if not exists p00_tal.S98_S_customer_events_subscribe ( - id_str TEXT - , date TEXT - , score TEXT - , last_updated TEXT - , target_name TEXT - , target_id TEXT - , customer_id_str TEXT - , channel_account TEXT - , customer_id TEXT - , external_id TEXT - , id TEXT - , event TEXT - , etl_tx_dt TIMESTAMP -) - - -SERVER pgsql_server_S98_S OPTIONS(schema_name 'data_api', table_name 'customer_events_subscribe' ); - - - - - diff --git a/TK_Cust/dev/tk_api/customer_events_subscribe/sa_tables.sql b/TK_Cust/dev/tk_api/customer_events_subscribe/sa_tables.sql deleted file mode 100644 index 8c14c70..0000000 --- a/TK_Cust/dev/tk_api/customer_events_subscribe/sa_tables.sql +++ /dev/null @@ -1,67 +0,0 @@ - -create table if not exists p10_sa.S98_S_customer_events_subscribe ( - id_str TEXT - , date TEXT - , score TEXT - , last_updated TEXT - , target_name TEXT - , target_id TEXT - , customer_id_str TEXT - , channel_account TEXT - , customer_id TEXT - , external_id TEXT - , id TEXT - , event TEXT - , etl_tx_dt TIMESTAMP -) ; - - COMMENT ON COLUMN p10_sa.S98_S_customer_events_subscribe.id_str IS ''; - COMMENT ON COLUMN p10_sa.S98_S_customer_events_subscribe.date IS ''; - COMMENT ON COLUMN p10_sa.S98_S_customer_events_subscribe.score IS ''; - COMMENT ON COLUMN p10_sa.S98_S_customer_events_subscribe.last_updated IS ''; - COMMENT ON COLUMN p10_sa.S98_S_customer_events_subscribe.target_name IS ''; - COMMENT ON COLUMN p10_sa.S98_S_customer_events_subscribe.target_id IS ''; - COMMENT ON COLUMN p10_sa.S98_S_customer_events_subscribe.customer_id_str IS ''; - COMMENT ON COLUMN p10_sa.S98_S_customer_events_subscribe.channel_account IS ''; - COMMENT ON COLUMN p10_sa.S98_S_customer_events_subscribe.customer_id IS ''; - COMMENT ON COLUMN p10_sa.S98_S_customer_events_subscribe.external_id IS ''; - COMMENT ON COLUMN p10_sa.S98_S_customer_events_subscribe.id IS ''; - COMMENT ON COLUMN p10_sa.S98_S_customer_events_subscribe.event IS ''; - COMMENT ON COLUMN p10_sa.S98_S_customer_events_subscribe.etl_tx_dt IS ''; - -COMMENT ON TABLE p10_sa.S98_S_customer_events_subscribe IS ''; - - - -create table if not exists p12_sfull.S98_S_customer_events_subscribe ( - id_str TEXT - , date TEXT - , score TEXT - , last_updated TEXT - , target_name TEXT - , target_id TEXT - , customer_id_str TEXT - , channel_account TEXT - , customer_id TEXT - , external_id TEXT - , id TEXT - , event TEXT - , etl_tx_dt TIMESTAMP -) ; - - COMMENT ON COLUMN p12_sfull.S98_S_customer_events_subscribe.id_str IS ''; - COMMENT ON COLUMN p12_sfull.S98_S_customer_events_subscribe.date IS ''; - COMMENT ON COLUMN p12_sfull.S98_S_customer_events_subscribe.score IS ''; - COMMENT ON COLUMN p12_sfull.S98_S_customer_events_subscribe.last_updated IS ''; - COMMENT ON COLUMN p12_sfull.S98_S_customer_events_subscribe.target_name IS ''; - COMMENT ON COLUMN p12_sfull.S98_S_customer_events_subscribe.target_id IS ''; - COMMENT ON COLUMN p12_sfull.S98_S_customer_events_subscribe.customer_id_str IS ''; - COMMENT ON COLUMN p12_sfull.S98_S_customer_events_subscribe.channel_account IS ''; - COMMENT ON COLUMN p12_sfull.S98_S_customer_events_subscribe.customer_id IS ''; - COMMENT ON COLUMN p12_sfull.S98_S_customer_events_subscribe.external_id IS ''; - COMMENT ON COLUMN p12_sfull.S98_S_customer_events_subscribe.id IS ''; - COMMENT ON COLUMN p12_sfull.S98_S_customer_events_subscribe.event IS ''; - COMMENT ON COLUMN p12_sfull.S98_S_customer_events_subscribe.etl_tx_dt IS ''; - -COMMENT ON TABLE p12_sfull.S98_S_customer_events_subscribe IS ''; - diff --git a/TK_Cust/dev/tk_api/customer_events_wechat_scan/S98_S_customer_events_wechat_scan.sql b/TK_Cust/dev/tk_api/customer_events_wechat_scan/S98_S_customer_events_wechat_scan.sql deleted file mode 100644 index 37c4b39..0000000 --- a/TK_Cust/dev/tk_api/customer_events_wechat_scan/S98_S_customer_events_wechat_scan.sql +++ /dev/null @@ -1,70 +0,0 @@ -/*******Main Section**************************************************************************/ -\set ON_ERROR_STOP on -\set AUTOCOMMIT on -\timing on -delete from p10_sa.S98_S_customer_events_wechat_scan -; -insert into p10_sa.S98_S_customer_events_wechat_scan -( id_str - , date - , score - , last_updated - , target_name - , target_id - , customer_id_str - , channel_account - , customer_id - , external_id - , id - , event - , etl_tx_dt ) - select - id_str - , date - , score - , last_updated - , target_name - , target_id - , customer_id_str - , channel_account - , customer_id - , external_id - , id - , event - , etl_tx_dt - from p00_tal.S98_S_customer_events_wechat_scan - ; - delete from p12_sfull.S98_S_customer_events_wechat_scan -; -; -insert into p12_sfull.S98_S_customer_events_wechat_scan -( id_str - , date - , score - , last_updated - , target_name - , target_id - , customer_id_str - , channel_account - , customer_id - , external_id - , id - , event - , etl_tx_dt ) - select - id_str - , date - , score - , last_updated - , target_name - , target_id - , customer_id_str - , channel_account - , customer_id - , external_id - , id - , event - , etl_tx_dt - from p10_sa.S98_S_customer_events_wechat_scan -; -\q \ No newline at end of file diff --git a/TK_Cust/dev/tk_api/customer_events_wechat_scan/sa_foreign_tables.sql b/TK_Cust/dev/tk_api/customer_events_wechat_scan/sa_foreign_tables.sql deleted file mode 100644 index 72b1a71..0000000 --- a/TK_Cust/dev/tk_api/customer_events_wechat_scan/sa_foreign_tables.sql +++ /dev/null @@ -1,24 +0,0 @@ - -CREATE FOREIGN TABLE if not exists p00_tal.S98_S_customer_events_wechat_scan ( - id_str TEXT - , date TEXT - , score TEXT - , last_updated TEXT - , target_name TEXT - , target_id TEXT - , customer_id_str TEXT - , channel_account TEXT - , customer_id TEXT - , external_id TEXT - , id TEXT - , event TEXT - , etl_tx_dt TIMESTAMP -) - - -SERVER pgsql_server_S98_S OPTIONS(schema_name 'data_api', table_name 'customer_events_wechat_scan' ); - - - - - diff --git a/TK_Cust/dev/tk_api/customer_events_wechat_scan/sa_tables.sql b/TK_Cust/dev/tk_api/customer_events_wechat_scan/sa_tables.sql deleted file mode 100644 index 0537b2b..0000000 --- a/TK_Cust/dev/tk_api/customer_events_wechat_scan/sa_tables.sql +++ /dev/null @@ -1,67 +0,0 @@ - -create table if not exists p10_sa.S98_S_customer_events_wechat_scan ( - id_str TEXT - , date TEXT - , score TEXT - , last_updated TEXT - , target_name TEXT - , target_id TEXT - , customer_id_str TEXT - , channel_account TEXT - , customer_id TEXT - , external_id TEXT - , id TEXT - , event TEXT - , etl_tx_dt TIMESTAMP -) ; - - COMMENT ON COLUMN p10_sa.S98_S_customer_events_wechat_scan.id_str IS ''; - COMMENT ON COLUMN p10_sa.S98_S_customer_events_wechat_scan.date IS ''; - COMMENT ON COLUMN p10_sa.S98_S_customer_events_wechat_scan.score IS ''; - COMMENT ON COLUMN p10_sa.S98_S_customer_events_wechat_scan.last_updated IS ''; - COMMENT ON COLUMN p10_sa.S98_S_customer_events_wechat_scan.target_name IS ''; - COMMENT ON COLUMN p10_sa.S98_S_customer_events_wechat_scan.target_id IS ''; - COMMENT ON COLUMN p10_sa.S98_S_customer_events_wechat_scan.customer_id_str IS ''; - COMMENT ON COLUMN p10_sa.S98_S_customer_events_wechat_scan.channel_account IS ''; - COMMENT ON COLUMN p10_sa.S98_S_customer_events_wechat_scan.customer_id IS ''; - COMMENT ON COLUMN p10_sa.S98_S_customer_events_wechat_scan.external_id IS ''; - COMMENT ON COLUMN p10_sa.S98_S_customer_events_wechat_scan.id IS ''; - COMMENT ON COLUMN p10_sa.S98_S_customer_events_wechat_scan.event IS ''; - COMMENT ON COLUMN p10_sa.S98_S_customer_events_wechat_scan.etl_tx_dt IS ''; - -COMMENT ON TABLE p10_sa.S98_S_customer_events_wechat_scan IS ''; - - - -create table if not exists p12_sfull.S98_S_customer_events_wechat_scan ( - id_str TEXT - , date TEXT - , score TEXT - , last_updated TEXT - , target_name TEXT - , target_id TEXT - , customer_id_str TEXT - , channel_account TEXT - , customer_id TEXT - , external_id TEXT - , id TEXT - , event TEXT - , etl_tx_dt TIMESTAMP -) ; - - COMMENT ON COLUMN p12_sfull.S98_S_customer_events_wechat_scan.id_str IS ''; - COMMENT ON COLUMN p12_sfull.S98_S_customer_events_wechat_scan.date IS ''; - COMMENT ON COLUMN p12_sfull.S98_S_customer_events_wechat_scan.score IS ''; - COMMENT ON COLUMN p12_sfull.S98_S_customer_events_wechat_scan.last_updated IS ''; - COMMENT ON COLUMN p12_sfull.S98_S_customer_events_wechat_scan.target_name IS ''; - COMMENT ON COLUMN p12_sfull.S98_S_customer_events_wechat_scan.target_id IS ''; - COMMENT ON COLUMN p12_sfull.S98_S_customer_events_wechat_scan.customer_id_str IS ''; - COMMENT ON COLUMN p12_sfull.S98_S_customer_events_wechat_scan.channel_account IS ''; - COMMENT ON COLUMN p12_sfull.S98_S_customer_events_wechat_scan.customer_id IS ''; - COMMENT ON COLUMN p12_sfull.S98_S_customer_events_wechat_scan.external_id IS ''; - COMMENT ON COLUMN p12_sfull.S98_S_customer_events_wechat_scan.id IS ''; - COMMENT ON COLUMN p12_sfull.S98_S_customer_events_wechat_scan.event IS ''; - COMMENT ON COLUMN p12_sfull.S98_S_customer_events_wechat_scan.etl_tx_dt IS ''; - -COMMENT ON TABLE p12_sfull.S98_S_customer_events_wechat_scan IS ''; - diff --git a/TK_Cust/dev/tk_api/scrm_contact/S98_S_scrm_contact.sql b/TK_Cust/dev/tk_api/scrm_contact/S98_S_scrm_contact.sql deleted file mode 100644 index ea55866..0000000 --- a/TK_Cust/dev/tk_api/scrm_contact/S98_S_scrm_contact.sql +++ /dev/null @@ -1,130 +0,0 @@ -/*******Main Section**************************************************************************/ -\set ON_ERROR_STOP on -\set AUTOCOMMIT on -\timing on -delete from p10_sa.S98_S_scrm_contact -; -insert into p10_sa.S98_S_scrm_contact -( img - , not_acceptsms - , gender - , create_method - , mobile_verified - , language - , is_member - , source - , is_referrer - , create_from - , last_updated - , date_created - , acceptsms - , id - , subscribe_scene - , id_str - , is_employee - , version - , not_accept_email - , date_join - , content_name - , update_method - , email_verified - , is_anonymous - , accept_email - , create_from_name - , status - , etl_tx_dt ) - select - img - , not_acceptsms - , gender - , create_method - , mobile_verified - , language - , is_member - , source - , is_referrer - , create_from - , last_updated - , date_created - , acceptsms - , id - , subscribe_scene - , id_str - , is_employee - , version - , not_accept_email - , date_join - , content_name - , update_method - , email_verified - , is_anonymous - , accept_email - , create_from_name - , status - , etl_tx_dt - from p00_tal.S98_S_scrm_contact - where etl_tx_dt >= CURRENT_DATE; - delete from p12_sfull.S98_S_scrm_contact - where (id) in (select id from p10_sa.S98_S_scrm_contact) ; -; -insert into p12_sfull.S98_S_scrm_contact -( img - , not_acceptsms - , gender - , create_method - , mobile_verified - , language - , is_member - , source - , is_referrer - , create_from - , last_updated - , date_created - , acceptsms - , id - , subscribe_scene - , id_str - , is_employee - , version - , not_accept_email - , date_join - , content_name - , update_method - , email_verified - , is_anonymous - , accept_email - , create_from_name - , status - , etl_tx_dt ) - select - img - , not_acceptsms - , gender - , create_method - , mobile_verified - , language - , is_member - , source - , is_referrer - , create_from - , last_updated - , date_created - , acceptsms - , id - , subscribe_scene - , id_str - , is_employee - , version - , not_accept_email - , date_join - , content_name - , update_method - , email_verified - , is_anonymous - , accept_email - , create_from_name - , status - , etl_tx_dt - from p10_sa.S98_S_scrm_contact -; -\q \ No newline at end of file diff --git a/TK_Cust/dev/tk_api/scrm_contact/sa_foreign_tables.sql b/TK_Cust/dev/tk_api/scrm_contact/sa_foreign_tables.sql deleted file mode 100644 index 44716fe..0000000 --- a/TK_Cust/dev/tk_api/scrm_contact/sa_foreign_tables.sql +++ /dev/null @@ -1,39 +0,0 @@ - -CREATE FOREIGN TABLE if not exists p00_tal.S98_S_scrm_contact ( - img TEXT - , not_acceptsms TEXT - , gender TEXT - , create_method TEXT - , mobile_verified TEXT - , language TEXT - , is_member TEXT - , source TEXT - , is_referrer TEXT - , create_from TEXT - , last_updated TEXT - , date_created TEXT - , acceptsms TEXT - , id TEXT - , subscribe_scene TEXT - , id_str TEXT - , is_employee TEXT - , version TEXT - , not_accept_email TEXT - , date_join TEXT - , content_name TEXT - , update_method TEXT - , email_verified TEXT - , is_anonymous TEXT - , accept_email TEXT - , create_from_name TEXT - , status TEXT - , etl_tx_dt TIMESTAMP -) - - -SERVER pgsql_server_S98_S OPTIONS(schema_name 'data_api', table_name 'scrm_contact' ); - - - - - diff --git a/TK_Cust/dev/tk_api/scrm_contact/sa_tables.sql b/TK_Cust/dev/tk_api/scrm_contact/sa_tables.sql deleted file mode 100644 index 21d308d..0000000 --- a/TK_Cust/dev/tk_api/scrm_contact/sa_tables.sql +++ /dev/null @@ -1,127 +0,0 @@ - -create table if not exists p10_sa.S98_S_scrm_contact ( - img TEXT - , not_acceptsms TEXT - , gender TEXT - , create_method TEXT - , mobile_verified TEXT - , language TEXT - , is_member TEXT - , source TEXT - , is_referrer TEXT - , create_from TEXT - , last_updated TEXT - , date_created TEXT - , acceptsms TEXT - , id TEXT - , subscribe_scene TEXT - , id_str TEXT - , is_employee TEXT - , version TEXT - , not_accept_email TEXT - , date_join TEXT - , content_name TEXT - , update_method TEXT - , email_verified TEXT - , is_anonymous TEXT - , accept_email TEXT - , create_from_name TEXT - , status TEXT - , etl_tx_dt TIMESTAMP -) ; - - COMMENT ON COLUMN p10_sa.S98_S_scrm_contact.img IS ''; - COMMENT ON COLUMN p10_sa.S98_S_scrm_contact.not_acceptsms IS ''; - COMMENT ON COLUMN p10_sa.S98_S_scrm_contact.gender IS ''; - COMMENT ON COLUMN p10_sa.S98_S_scrm_contact.create_method IS ''; - COMMENT ON COLUMN p10_sa.S98_S_scrm_contact.mobile_verified IS ''; - COMMENT ON COLUMN p10_sa.S98_S_scrm_contact.language IS ''; - COMMENT ON COLUMN p10_sa.S98_S_scrm_contact.is_member IS ''; - COMMENT ON COLUMN p10_sa.S98_S_scrm_contact.source IS ''; - COMMENT ON COLUMN p10_sa.S98_S_scrm_contact.is_referrer IS ''; - COMMENT ON COLUMN p10_sa.S98_S_scrm_contact.create_from IS ''; - COMMENT ON COLUMN p10_sa.S98_S_scrm_contact.last_updated IS ''; - COMMENT ON COLUMN p10_sa.S98_S_scrm_contact.date_created IS ''; - COMMENT ON COLUMN p10_sa.S98_S_scrm_contact.acceptsms IS ''; - COMMENT ON COLUMN p10_sa.S98_S_scrm_contact.id IS ''; - COMMENT ON COLUMN p10_sa.S98_S_scrm_contact.subscribe_scene IS ''; - COMMENT ON COLUMN p10_sa.S98_S_scrm_contact.id_str IS ''; - COMMENT ON COLUMN p10_sa.S98_S_scrm_contact.is_employee IS ''; - COMMENT ON COLUMN p10_sa.S98_S_scrm_contact.version IS ''; - COMMENT ON COLUMN p10_sa.S98_S_scrm_contact.not_accept_email IS ''; - COMMENT ON COLUMN p10_sa.S98_S_scrm_contact.date_join IS ''; - COMMENT ON COLUMN p10_sa.S98_S_scrm_contact.content_name IS ''; - COMMENT ON COLUMN p10_sa.S98_S_scrm_contact.update_method IS ''; - COMMENT ON COLUMN p10_sa.S98_S_scrm_contact.email_verified IS ''; - COMMENT ON COLUMN p10_sa.S98_S_scrm_contact.is_anonymous IS ''; - COMMENT ON COLUMN p10_sa.S98_S_scrm_contact.accept_email IS ''; - COMMENT ON COLUMN p10_sa.S98_S_scrm_contact.create_from_name IS ''; - COMMENT ON COLUMN p10_sa.S98_S_scrm_contact.status IS ''; - COMMENT ON COLUMN p10_sa.S98_S_scrm_contact.etl_tx_dt IS ''; - -COMMENT ON TABLE p10_sa.S98_S_scrm_contact IS ''; - - - -create table if not exists p12_sfull.S98_S_scrm_contact ( - img TEXT - , not_acceptsms TEXT - , gender TEXT - , create_method TEXT - , mobile_verified TEXT - , language TEXT - , is_member TEXT - , source TEXT - , is_referrer TEXT - , create_from TEXT - , last_updated TEXT - , date_created TEXT - , acceptsms TEXT - , id TEXT - , subscribe_scene TEXT - , id_str TEXT - , is_employee TEXT - , version TEXT - , not_accept_email TEXT - , date_join TEXT - , content_name TEXT - , update_method TEXT - , email_verified TEXT - , is_anonymous TEXT - , accept_email TEXT - , create_from_name TEXT - , status TEXT - , etl_tx_dt TIMESTAMP -) ; - - COMMENT ON COLUMN p12_sfull.S98_S_scrm_contact.img IS ''; - COMMENT ON COLUMN p12_sfull.S98_S_scrm_contact.not_acceptsms IS ''; - COMMENT ON COLUMN p12_sfull.S98_S_scrm_contact.gender IS ''; - COMMENT ON COLUMN p12_sfull.S98_S_scrm_contact.create_method IS ''; - COMMENT ON COLUMN p12_sfull.S98_S_scrm_contact.mobile_verified IS ''; - COMMENT ON COLUMN p12_sfull.S98_S_scrm_contact.language IS ''; - COMMENT ON COLUMN p12_sfull.S98_S_scrm_contact.is_member IS ''; - COMMENT ON COLUMN p12_sfull.S98_S_scrm_contact.source IS ''; - COMMENT ON COLUMN p12_sfull.S98_S_scrm_contact.is_referrer IS ''; - COMMENT ON COLUMN p12_sfull.S98_S_scrm_contact.create_from IS ''; - COMMENT ON COLUMN p12_sfull.S98_S_scrm_contact.last_updated IS ''; - COMMENT ON COLUMN p12_sfull.S98_S_scrm_contact.date_created IS ''; - COMMENT ON COLUMN p12_sfull.S98_S_scrm_contact.acceptsms IS ''; - COMMENT ON COLUMN p12_sfull.S98_S_scrm_contact.id IS ''; - COMMENT ON COLUMN p12_sfull.S98_S_scrm_contact.subscribe_scene IS ''; - COMMENT ON COLUMN p12_sfull.S98_S_scrm_contact.id_str IS ''; - COMMENT ON COLUMN p12_sfull.S98_S_scrm_contact.is_employee IS ''; - COMMENT ON COLUMN p12_sfull.S98_S_scrm_contact.version IS ''; - COMMENT ON COLUMN p12_sfull.S98_S_scrm_contact.not_accept_email IS ''; - COMMENT ON COLUMN p12_sfull.S98_S_scrm_contact.date_join IS ''; - COMMENT ON COLUMN p12_sfull.S98_S_scrm_contact.content_name IS ''; - COMMENT ON COLUMN p12_sfull.S98_S_scrm_contact.update_method IS ''; - COMMENT ON COLUMN p12_sfull.S98_S_scrm_contact.email_verified IS ''; - COMMENT ON COLUMN p12_sfull.S98_S_scrm_contact.is_anonymous IS ''; - COMMENT ON COLUMN p12_sfull.S98_S_scrm_contact.accept_email IS ''; - COMMENT ON COLUMN p12_sfull.S98_S_scrm_contact.create_from_name IS ''; - COMMENT ON COLUMN p12_sfull.S98_S_scrm_contact.status IS ''; - COMMENT ON COLUMN p12_sfull.S98_S_scrm_contact.etl_tx_dt IS ''; - -COMMENT ON TABLE p12_sfull.S98_S_scrm_contact IS ''; - diff --git a/TK_Cust/dev/tk_api/scrm_contact_lastupdated/S98_S_scrm_contact_lastupdated.sql b/TK_Cust/dev/tk_api/scrm_contact_lastupdated/S98_S_scrm_contact_lastupdated.sql deleted file mode 100644 index d3e4537..0000000 --- a/TK_Cust/dev/tk_api/scrm_contact_lastupdated/S98_S_scrm_contact_lastupdated.sql +++ /dev/null @@ -1,138 +0,0 @@ -/*******Main Section**************************************************************************/ -\set ON_ERROR_STOP on -\set AUTOCOMMIT on -\timing on -delete from p10_sa.S98_S_scrm_contact_lastupdated -; -insert into p10_sa.S98_S_scrm_contact_lastupdated -( img - , not_acceptsms - , gender - , display_name - , create_method - , mobile_verified - , language - , is_member - , source - , is_referrer - , create_from - , last_updated - , date_created - , acceptsms - , id - , subscribe_scene - , id_str - , is_employee - , nick_name - , version - , not_accept_email - , date_join - , content_name - , update_method - , email_verified - , is_anonymous - , accept_email - , create_from_name - , status - , etl_tx_dt ) - select - img - , not_acceptsms - , gender - , display_name - , create_method - , mobile_verified - , language - , is_member - , source - , is_referrer - , create_from - , last_updated - , date_created - , acceptsms - , id - , subscribe_scene - , id_str - , is_employee - , nick_name - , version - , not_accept_email - , date_join - , content_name - , update_method - , email_verified - , is_anonymous - , accept_email - , create_from_name - , status - , etl_tx_dt - from p00_tal.S98_S_scrm_contact_lastupdated - ; - delete from p12_sfull.S98_S_scrm_contact_lastupdated -; -; -insert into p12_sfull.S98_S_scrm_contact_lastupdated -( img - , not_acceptsms - , gender - , display_name - , create_method - , mobile_verified - , language - , is_member - , source - , is_referrer - , create_from - , last_updated - , date_created - , acceptsms - , id - , subscribe_scene - , id_str - , is_employee - , nick_name - , version - , not_accept_email - , date_join - , content_name - , update_method - , email_verified - , is_anonymous - , accept_email - , create_from_name - , status - , etl_tx_dt ) - select - img - , not_acceptsms - , gender - , display_name - , create_method - , mobile_verified - , language - , is_member - , source - , is_referrer - , create_from - , last_updated - , date_created - , acceptsms - , id - , subscribe_scene - , id_str - , is_employee - , nick_name - , version - , not_accept_email - , date_join - , content_name - , update_method - , email_verified - , is_anonymous - , accept_email - , create_from_name - , status - , etl_tx_dt - from p10_sa.S98_S_scrm_contact_lastupdated -; -\q \ No newline at end of file diff --git a/TK_Cust/dev/tk_api/scrm_contact_lastupdated/sa_foreign_tables.sql b/TK_Cust/dev/tk_api/scrm_contact_lastupdated/sa_foreign_tables.sql deleted file mode 100644 index b8ea69b..0000000 --- a/TK_Cust/dev/tk_api/scrm_contact_lastupdated/sa_foreign_tables.sql +++ /dev/null @@ -1,41 +0,0 @@ - -CREATE FOREIGN TABLE if not exists p00_tal.S98_S_scrm_contact_lastupdated ( - img TEXT - , not_acceptsms TEXT - , gender TEXT - , display_name TEXT - , create_method TEXT - , mobile_verified TEXT - , language TEXT - , is_member TEXT - , source TEXT - , is_referrer TEXT - , create_from TEXT - , last_updated TEXT - , date_created TEXT - , acceptsms TEXT - , id TEXT - , subscribe_scene TEXT - , id_str TEXT - , is_employee TEXT - , nick_name TEXT - , version TEXT - , not_accept_email TEXT - , date_join TEXT - , content_name TEXT - , update_method TEXT - , email_verified TEXT - , is_anonymous TEXT - , accept_email TEXT - , create_from_name TEXT - , status TEXT - , etl_tx_dt TIMESTAMP -) - - -SERVER pgsql_server_S98_S OPTIONS(schema_name 'data_api', table_name 'scrm_contact_lastupdated' ); - - - - - diff --git a/TK_Cust/dev/tk_api/scrm_contact_lastupdated/sa_tables.sql b/TK_Cust/dev/tk_api/scrm_contact_lastupdated/sa_tables.sql deleted file mode 100644 index 2e113be..0000000 --- a/TK_Cust/dev/tk_api/scrm_contact_lastupdated/sa_tables.sql +++ /dev/null @@ -1,135 +0,0 @@ - -create table if not exists p10_sa.S98_S_scrm_contact_lastupdated ( - img TEXT - , not_acceptsms TEXT - , gender TEXT - , display_name TEXT - , create_method TEXT - , mobile_verified TEXT - , language TEXT - , is_member TEXT - , source TEXT - , is_referrer TEXT - , create_from TEXT - , last_updated TEXT - , date_created TEXT - , acceptsms TEXT - , id TEXT - , subscribe_scene TEXT - , id_str TEXT - , is_employee TEXT - , nick_name TEXT - , version TEXT - , not_accept_email TEXT - , date_join TEXT - , content_name TEXT - , update_method TEXT - , email_verified TEXT - , is_anonymous TEXT - , accept_email TEXT - , create_from_name TEXT - , status TEXT - , etl_tx_dt TIMESTAMP -) ; - - COMMENT ON COLUMN p10_sa.S98_S_scrm_contact_lastupdated.img IS ''; - COMMENT ON COLUMN p10_sa.S98_S_scrm_contact_lastupdated.not_acceptsms IS ''; - COMMENT ON COLUMN p10_sa.S98_S_scrm_contact_lastupdated.gender IS ''; - COMMENT ON COLUMN p10_sa.S98_S_scrm_contact_lastupdated.display_name IS ''; - COMMENT ON COLUMN p10_sa.S98_S_scrm_contact_lastupdated.create_method IS ''; - COMMENT ON COLUMN p10_sa.S98_S_scrm_contact_lastupdated.mobile_verified IS ''; - COMMENT ON COLUMN p10_sa.S98_S_scrm_contact_lastupdated.language IS ''; - COMMENT ON COLUMN p10_sa.S98_S_scrm_contact_lastupdated.is_member IS ''; - COMMENT ON COLUMN p10_sa.S98_S_scrm_contact_lastupdated.source IS ''; - COMMENT ON COLUMN p10_sa.S98_S_scrm_contact_lastupdated.is_referrer IS ''; - COMMENT ON COLUMN p10_sa.S98_S_scrm_contact_lastupdated.create_from IS ''; - COMMENT ON COLUMN p10_sa.S98_S_scrm_contact_lastupdated.last_updated IS ''; - COMMENT ON COLUMN p10_sa.S98_S_scrm_contact_lastupdated.date_created IS ''; - COMMENT ON COLUMN p10_sa.S98_S_scrm_contact_lastupdated.acceptsms IS ''; - COMMENT ON COLUMN p10_sa.S98_S_scrm_contact_lastupdated.id IS ''; - COMMENT ON COLUMN p10_sa.S98_S_scrm_contact_lastupdated.subscribe_scene IS ''; - COMMENT ON COLUMN p10_sa.S98_S_scrm_contact_lastupdated.id_str IS ''; - COMMENT ON COLUMN p10_sa.S98_S_scrm_contact_lastupdated.is_employee IS ''; - COMMENT ON COLUMN p10_sa.S98_S_scrm_contact_lastupdated.nick_name IS ''; - COMMENT ON COLUMN p10_sa.S98_S_scrm_contact_lastupdated.version IS ''; - COMMENT ON COLUMN p10_sa.S98_S_scrm_contact_lastupdated.not_accept_email IS ''; - COMMENT ON COLUMN p10_sa.S98_S_scrm_contact_lastupdated.date_join IS ''; - COMMENT ON COLUMN p10_sa.S98_S_scrm_contact_lastupdated.content_name IS ''; - COMMENT ON COLUMN p10_sa.S98_S_scrm_contact_lastupdated.update_method IS ''; - COMMENT ON COLUMN p10_sa.S98_S_scrm_contact_lastupdated.email_verified IS ''; - COMMENT ON COLUMN p10_sa.S98_S_scrm_contact_lastupdated.is_anonymous IS ''; - COMMENT ON COLUMN p10_sa.S98_S_scrm_contact_lastupdated.accept_email IS ''; - COMMENT ON COLUMN p10_sa.S98_S_scrm_contact_lastupdated.create_from_name IS ''; - COMMENT ON COLUMN p10_sa.S98_S_scrm_contact_lastupdated.status IS ''; - COMMENT ON COLUMN p10_sa.S98_S_scrm_contact_lastupdated.etl_tx_dt IS ''; - -COMMENT ON TABLE p10_sa.S98_S_scrm_contact_lastupdated IS ''; - - - -create table if not exists p12_sfull.S98_S_scrm_contact_lastupdated ( - img TEXT - , not_acceptsms TEXT - , gender TEXT - , display_name TEXT - , create_method TEXT - , mobile_verified TEXT - , language TEXT - , is_member TEXT - , source TEXT - , is_referrer TEXT - , create_from TEXT - , last_updated TEXT - , date_created TEXT - , acceptsms TEXT - , id TEXT - , subscribe_scene TEXT - , id_str TEXT - , is_employee TEXT - , nick_name TEXT - , version TEXT - , not_accept_email TEXT - , date_join TEXT - , content_name TEXT - , update_method TEXT - , email_verified TEXT - , is_anonymous TEXT - , accept_email TEXT - , create_from_name TEXT - , status TEXT - , etl_tx_dt TIMESTAMP -) ; - - COMMENT ON COLUMN p12_sfull.S98_S_scrm_contact_lastupdated.img IS ''; - COMMENT ON COLUMN p12_sfull.S98_S_scrm_contact_lastupdated.not_acceptsms IS ''; - COMMENT ON COLUMN p12_sfull.S98_S_scrm_contact_lastupdated.gender IS ''; - COMMENT ON COLUMN p12_sfull.S98_S_scrm_contact_lastupdated.display_name IS ''; - COMMENT ON COLUMN p12_sfull.S98_S_scrm_contact_lastupdated.create_method IS ''; - COMMENT ON COLUMN p12_sfull.S98_S_scrm_contact_lastupdated.mobile_verified IS ''; - COMMENT ON COLUMN p12_sfull.S98_S_scrm_contact_lastupdated.language IS ''; - COMMENT ON COLUMN p12_sfull.S98_S_scrm_contact_lastupdated.is_member IS ''; - COMMENT ON COLUMN p12_sfull.S98_S_scrm_contact_lastupdated.source IS ''; - COMMENT ON COLUMN p12_sfull.S98_S_scrm_contact_lastupdated.is_referrer IS ''; - COMMENT ON COLUMN p12_sfull.S98_S_scrm_contact_lastupdated.create_from IS ''; - COMMENT ON COLUMN p12_sfull.S98_S_scrm_contact_lastupdated.last_updated IS ''; - COMMENT ON COLUMN p12_sfull.S98_S_scrm_contact_lastupdated.date_created IS ''; - COMMENT ON COLUMN p12_sfull.S98_S_scrm_contact_lastupdated.acceptsms IS ''; - COMMENT ON COLUMN p12_sfull.S98_S_scrm_contact_lastupdated.id IS ''; - COMMENT ON COLUMN p12_sfull.S98_S_scrm_contact_lastupdated.subscribe_scene IS ''; - COMMENT ON COLUMN p12_sfull.S98_S_scrm_contact_lastupdated.id_str IS ''; - COMMENT ON COLUMN p12_sfull.S98_S_scrm_contact_lastupdated.is_employee IS ''; - COMMENT ON COLUMN p12_sfull.S98_S_scrm_contact_lastupdated.nick_name IS ''; - COMMENT ON COLUMN p12_sfull.S98_S_scrm_contact_lastupdated.version IS ''; - COMMENT ON COLUMN p12_sfull.S98_S_scrm_contact_lastupdated.not_accept_email IS ''; - COMMENT ON COLUMN p12_sfull.S98_S_scrm_contact_lastupdated.date_join IS ''; - COMMENT ON COLUMN p12_sfull.S98_S_scrm_contact_lastupdated.content_name IS ''; - COMMENT ON COLUMN p12_sfull.S98_S_scrm_contact_lastupdated.update_method IS ''; - COMMENT ON COLUMN p12_sfull.S98_S_scrm_contact_lastupdated.email_verified IS ''; - COMMENT ON COLUMN p12_sfull.S98_S_scrm_contact_lastupdated.is_anonymous IS ''; - COMMENT ON COLUMN p12_sfull.S98_S_scrm_contact_lastupdated.accept_email IS ''; - COMMENT ON COLUMN p12_sfull.S98_S_scrm_contact_lastupdated.create_from_name IS ''; - COMMENT ON COLUMN p12_sfull.S98_S_scrm_contact_lastupdated.status IS ''; - COMMENT ON COLUMN p12_sfull.S98_S_scrm_contact_lastupdated.etl_tx_dt IS ''; - -COMMENT ON TABLE p12_sfull.S98_S_scrm_contact_lastupdated IS ''; - diff --git a/TK_Cust/dev/tk_api/合并SCRM CONTACT/custom_events_update_feign.py b/TK_Cust/dev/tk_api/合并SCRM CONTACT/custom_events_update_feign.py deleted file mode 100644 index 96af1cc..0000000 --- a/TK_Cust/dev/tk_api/合并SCRM CONTACT/custom_events_update_feign.py +++ /dev/null @@ -1,41 +0,0 @@ -# coding: utf-8 -import requests -import json -import psycopg2 -import uuid -import datetime - -# 获取当前日期 -current_date = datetime.date.today() -previous_date = current_date - datetime.timedelta(days=2) -formatted_current_date = current_date.strftime("%Y-%m-%dT%H:%M:%SZ") -formatted_previous_date = previous_date.strftime("%Y-%m-%dT%H:%M:%SZ") - -print('开始加载数据：custom_events_update:合并SCRM CONTACT') -authUrl='https://api.huiju.cool/v2/oauth2/token?app_id=cl037b184ebccd97c&secret=deaa83fdd8d385eec0d1d04d34282c4ec8c33b60&grant_type=client_credentials' - -print('开始请求令牌。') -authRequest=requests.get(authUrl) -auth=json.loads(authRequest.text) -print('开始请求数据总数。') -url='https://api.huiju.cool/v2/customerEvents' -header={} -body={'access_token':auth['access_token'],'sort':'lastUpdated','lastUpdated[ge]':formatted_previous_date,'lastUpdated[le]':formatted_current_date,'limit':'',} -dataReqL=requests.get(url,headers=header,params=body) -resL=json.loads(dataReqL.text) -# print(resL) -dataList=resL['data'] -total=len(dataList) -conn = psycopg2.connect(database="dataops_db", user="dbuser_dops", password="MIgTi3jA", - host="172.17.0.8", port="5432") -print('数据库连接成功') -dataId=str(uuid.uuid4()) -print('临时id：'+dataId) -json_object = json.dumps(dataList) -cur=conn.cursor() -sql="update data_api.api_data set is_loaded = '1' where api_id = 'ff5c7bf6-0d18-4201-9501-8fdd6152';INSERT INTO data_api.api_data (id,api_id,data,total_num,is_loaded,status,request_tm,execute_tm,remark) values (%s,%s,%s,%s,'0','0',current_timestamp(0),current_timestamp(0),'')" -cur.execute(sql,[dataId,'ff5c7bf6-0d18-4201-9501-8fdd6152',json_object,total]) -conn.commit() -cur.close() -conn.close() -print('加载数据结束：custom_events_update:合并SCRM CONTACT') \ No newline at end of file diff --git a/TK_Cust/dev/tk_api/合并SCRM CONTACT/custom_events_update_load.sql b/TK_Cust/dev/tk_api/合并SCRM CONTACT/custom_events_update_load.sql deleted file mode 100644 index 1217982..0000000 --- a/TK_Cust/dev/tk_api/合并SCRM CONTACT/custom_events_update_load.sql +++ /dev/null @@ -1,105 +0,0 @@ -/*******Main Section**************************************************************************/ -\set ON_ERROR_STOP on -\set AUTOCOMMIT on -\timing on - -DELETE FROM data_api.customer_events_lastupdated; - -insert into data_api.customer_events_lastupdated ( - date - , screen_width - , target_name - , ip_city - , ip_country - , platform - , last_updated - , page_type - , os_version - , browser - , browser_version - , customer_id - , id - , tag - , event - , id_str - , os - , target_id - , customer_id_str - , ip - , screen_height - , external_id - , ip_county - , page_id - , url - , ip_province - , device - ,etl_tx_dt -) -select - case when trim(both from date)='' then null else date::text end date - , case when trim(both from screen_width)='' then null else screen_width::text end screen_width - , case when trim(both from target_name)='' then null else target_name::text end target_name - , case when trim(both from ip_city)='' then null else ip_city::text end ip_city - , case when trim(both from ip_country)='' then null else ip_country::text end ip_country - , case when trim(both from platform)='' then null else platform::text end platform - , case when trim(both from last_updated)='' then null else last_updated::text end last_updated - , case when trim(both from page_type)='' then null else page_type::text end page_type - , case when trim(both from os_version)='' then null else os_version::text end os_version - , case when trim(both from browser)='' then null else browser::text end browser - , case when trim(both from browser_version)='' then null else browser_version::text end browser_version - , case when trim(both from customer_id)='' then null else customer_id::text end customer_id - , case when trim(both from id)='' then null else id::text end id - , case when trim(both from tag)='' then null else tag::text end tag - , case when trim(both from event)='' then null else event::text end event - , case when trim(both from id_str)='' then null else id_str::text end id_str - , case when trim(both from os)='' then null else os::text end os - , case when trim(both from target_id)='' then null else target_id::text end target_id - , case when trim(both from customer_id_str)='' then null else customer_id_str::text end customer_id_str - , case when trim(both from ip)='' then null else ip::text end ip - , case when trim(both from screen_height)='' then null else screen_height::text end screen_height - , case when trim(both from external_id)='' then null else external_id::text end external_id - , case when trim(both from ip_county)='' then null else ip_county::text end ip_county - , case when trim(both from page_id)='' then null else page_id::text end page_id - , case when trim(both from url)='' then null else url::text end url - , case when trim(both from ip_province)='' then null else ip_province::text end ip_province - , case when trim(both from device)='' then null else device::text end device -,etl_tx_dt -from ( -select - (json_array_elements(data::json)::json->>'date') date - , (json_array_elements(data::json)::json->>'screenWidth') screen_width - , (json_array_elements(data::json)::json->>'targetName') target_name - , (json_array_elements(data::json)::json->>'ipCity') ip_city - , (json_array_elements(data::json)::json->>'ipCountry') ip_country - , (json_array_elements(data::json)::json->>'platform') platform - , (json_array_elements(data::json)::json->>'lastUpdated') last_updated - , (json_array_elements(data::json)::json->>'pageType') page_type - , (json_array_elements(data::json)::json->>'osVersion') os_version - , (json_array_elements(data::json)::json->>'browser') browser - , (json_array_elements(data::json)::json->>'browserVersion') browser_version - , (json_array_elements(data::json)::json->>'customerId') customer_id - , (json_array_elements(data::json)::json->>'id') id - , (json_array_elements(data::json)::json->>'tag') tag - , (json_array_elements(data::json)::json->>'event') event - , (json_array_elements(data::json)::json->>'idStr') id_str - , (json_array_elements(data::json)::json->>'os') os - , (json_array_elements(data::json)::json->>'targetId') target_id - , (json_array_elements(data::json)::json->>'customerIdStr') customer_id_str - , (json_array_elements(data::json)::json->>'ip') ip - , (json_array_elements(data::json)::json->>'screenHeight') screen_height - , (json_array_elements(data::json)::json->>'externalId') external_id - , (json_array_elements(data::json)::json->>'ipCounty') ip_county - , (json_array_elements(data::json)::json->>'pageId') page_id - , (json_array_elements(data::json)::json->>'url') url - , (json_array_elements(data::json)::json->>'ipProvince') ip_province - , (json_array_elements(data::json)::json->>'device') device - ,CURRENT_TIMESTAMP(0) etl_tx_dt -from (select * from data_api.api_data -WHERE api_id='ff5c7bf6-0d18-4201-9501-8fdd6152' and is_loaded = '0' order by request_tm desc limit 1) p )p; - -update data_api.api_data -set is_loaded = '1' , - status = '1', - request_tm = current_timestamp(0) -where api_id='ff5c7bf6-0d18-4201-9501-8fdd6152'; -\q \ No newline at end of file diff --git a/TK_Cust/dev/tk_api/更新scrm CONTACT/update_scrm_contact_feign.py b/TK_Cust/dev/tk_api/更新scrm CONTACT/update_scrm_contact_feign.py deleted file mode 100644 index aff3b26..0000000 --- a/TK_Cust/dev/tk_api/更新scrm CONTACT/update_scrm_contact_feign.py +++ /dev/null @@ -1,41 +0,0 @@ -# coding: utf-8 -import requests -import json -import psycopg2 -import uuid -import datetime - -# 获取当前日期 -current_date = datetime.date.today() -previous_date = current_date - datetime.timedelta(days=2) -formatted_current_date = current_date.strftime("%Y-%m-%dT%H:%M:%SZ") -formatted_previous_date = previous_date.strftime("%Y-%m-%dT%H:%M:%SZ") - -print('开始加载数据：update_scrm_contact:更新scrm CONTACT') -authUrl='https://api.huiju.cool/v2/oauth2/token?app_id=cl037b184ebccd97c&secret=deaa83fdd8d385eec0d1d04d34282c4ec8c33b60&grant_type=client_credentials' - -print('开始请求令牌。') -authRequest=requests.get(authUrl) -auth=json.loads(authRequest.text) -print('开始请求数据总数。') -url='https://api.huiju.cool/v2/customers' -header={} -body={'access_token':auth['access_token'],'sort':'lastUpdated','lastUpdated[ge]':formatted_previous_date,'lastUpdated[le]':formatted_current_date,'limit':'',} -dataReqL=requests.get(url,headers=header,params=body) -resL=json.loads(dataReqL.text) -# print(resL) -dataList=resL['data'] -total=len(dataList) -conn = psycopg2.connect(database="dataops_db", user="dbuser_dops", password="MIgTi3jA", - host="172.17.0.8", port="5432") -print('数据库连接成功') -dataId=str(uuid.uuid4()) -print('临时id：'+dataId) -json_object = json.dumps(dataList) -cur=conn.cursor() -sql="update data_api.api_data set is_loaded = '1' where api_id = '03d918ea-00d1-448a-8b9b-6a0b1a33';INSERT INTO data_api.api_data (id,api_id,data,total_num,is_loaded,status,request_tm,execute_tm,remark) values (%s,%s,%s,%s,'0','0',current_timestamp(0),current_timestamp(0),'')" -cur.execute(sql,[dataId,'03d918ea-00d1-448a-8b9b-6a0b1a33',json_object,total]) -conn.commit() -cur.close() -conn.close() -print('加载数据结束：update_scrm_contact:更新scrm CONTACT') \ No newline at end of file diff --git a/TK_Cust/dev/tk_api/更新scrm CONTACT/update_scrm_contact_load.sql b/TK_Cust/dev/tk_api/更新scrm CONTACT/update_scrm_contact_load.sql deleted file mode 100644 index 1fa8598..0000000 --- a/TK_Cust/dev/tk_api/更新scrm CONTACT/update_scrm_contact_load.sql +++ /dev/null @@ -1,111 +0,0 @@ -/*******Main Section**************************************************************************/ -\set ON_ERROR_STOP on -\set AUTOCOMMIT on -\timing on - -DELETE FROM data_api.scrm_contact_lastUpdated; - -insert into data_api.scrm_contact_lastUpdated ( - img - , not_acceptsms - , gender - , display_name - , create_method - , mobile_verified - , language - , is_member - , source - , is_referrer - , create_from - , last_updated - , date_created - , acceptsms - , id - , subscribe_scene - , id_str - , is_employee - , nick_name - , version - , not_accept_email - , date_join - , content_name - , update_method - , email_verified - , is_anonymous - , accept_email - , create_from_name - , status - ,etl_tx_dt -) -select - case when trim(both from img)='' then null else img::text end img - , case when trim(both from not_acceptsms)='' then null else not_acceptsms::text end not_acceptsms - , case when trim(both from gender)='' then null else gender::text end gender - , case when trim(both from display_name)='' then null else display_name::text end display_name - , case when trim(both from create_method)='' then null else create_method::text end create_method - , case when trim(both from mobile_verified)='' then null else mobile_verified::text end mobile_verified - , case when trim(both from language)='' then null else language::text end language - , case when trim(both from is_member)='' then null else is_member::text end is_member - , case when trim(both from source)='' then null else source::text end source - , case when trim(both from is_referrer)='' then null else is_referrer::text end is_referrer - , case when trim(both from create_from)='' then null else create_from::text end create_from - , case when trim(both from last_updated)='' then null else last_updated::text end last_updated - , case when trim(both from date_created)='' then null else date_created::text end date_created - , case when trim(both from acceptsms)='' then null else acceptsms::text end acceptsms - , case when trim(both from id)='' then null else id::text end id - , case when trim(both from subscribe_scene)='' then null else subscribe_scene::text end subscribe_scene - , case when trim(both from id_str)='' then null else id_str::text end id_str - , case when trim(both from is_employee)='' then null else is_employee::text end is_employee - , case when trim(both from nick_name)='' then null else nick_name::text end nick_name - , case when trim(both from version)='' then null else version::text end version - , case when trim(both from not_accept_email)='' then null else not_accept_email::text end not_accept_email - , case when trim(both from date_join)='' then null else date_join::text end date_join - , case when trim(both from content_name)='' then null else content_name::text end content_name - , case when trim(both from update_method)='' then null else update_method::text end update_method - , case when trim(both from email_verified)='' then null else email_verified::text end email_verified - , case when trim(both from is_anonymous)='' then null else is_anonymous::text end is_anonymous - , case when trim(both from accept_email)='' then null else accept_email::text end accept_email - , case when trim(both from create_from_name)='' then null else create_from_name::text end create_from_name - , case when trim(both from status)='' then null else status::text end status -,etl_tx_dt -from ( -select - (json_array_elements(data::json)::json->>'img') img - , (json_array_elements(data::json)::json->>'notAcceptsms') not_acceptsms - , (json_array_elements(data::json)::json->>'gender') gender - , (json_array_elements(data::json)::json->>'displayName') display_name - , (json_array_elements(data::json)::json->>'createMethod') create_method - , (json_array_elements(data::json)::json->>'mobileVerified') mobile_verified - , (json_array_elements(data::json)::json->>'language') language - , (json_array_elements(data::json)::json->>'isMember') is_member - , (json_array_elements(data::json)::json->>'source') source - , (json_array_elements(data::json)::json->>'isReferrer') is_referrer - , (json_array_elements(data::json)::json->>'createFrom') create_from - , (json_array_elements(data::json)::json->>'lastUpdated') last_updated - , (json_array_elements(data::json)::json->>'dateCreated') date_created - , (json_array_elements(data::json)::json->>'acceptsms') acceptsms - , (json_array_elements(data::json)::json->>'id') id - , (json_array_elements(data::json)::json->>'subscribeScene') subscribe_scene - , (json_array_elements(data::json)::json->>'idStr') id_str - , (json_array_elements(data::json)::json->>'isEmployee') is_employee - , (json_array_elements(data::json)::json->>'nickName') nick_name - , (json_array_elements(data::json)::json->>'version') version - , (json_array_elements(data::json)::json->>'notAcceptEmail') not_accept_email - , (json_array_elements(data::json)::json->>'dateJoin') date_join - , (json_array_elements(data::json)::json->>'contentName') content_name - , (json_array_elements(data::json)::json->>'updateMethod') update_method - , (json_array_elements(data::json)::json->>'emailVerified') email_verified - , (json_array_elements(data::json)::json->>'isAnonymous') is_anonymous - , (json_array_elements(data::json)::json->>'acceptEmail') accept_email - , (json_array_elements(data::json)::json->>'createFromName') create_from_name - , (json_array_elements(data::json)::json->>'status') status - ,CURRENT_TIMESTAMP(0) etl_tx_dt -from (select * from data_api.api_data -WHERE api_id='03d918ea-00d1-448a-8b9b-6a0b1a33' and is_loaded = '0' order by request_tm desc limit 1) p )p; - -update data_api.api_data -set is_loaded = '1' , - status = '1', - request_tm = current_timestamp(0) -where api_id='03d918ea-00d1-448a-8b9b-6a0b1a33'; -\q \ No newline at end of file diff --git a/TK_Cust/dev/tk_api/荟聚API/wf_dag_tk_api.py b/TK_Cust/dev/tk_api/荟聚API/wf_dag_tk_api.py deleted file mode 100644 index 893a1a5..0000000 --- a/TK_Cust/dev/tk_api/荟聚API/wf_dag_tk_api.py +++ /dev/null @@ -1,447 +0,0 @@ -#!/usr/bin/python -# -*- encoding=utf-8 -*- -from airflow import DAG -from datetime import datetime, timedelta -from airflow.contrib.hooks.ssh_hook import SSHHook -from airflow.contrib.operators.ssh_operator import SSHOperator -from airflow.sensors.external_task_sensor import ExternalTaskSensor -import json - -from airflow.operators.email_operator import EmailOperator -from airflow.utils.trigger_rule import TriggerRule - - -sshHook = SSHHook(ssh_conn_id ='ssh_air') -default_args = { -'owner': 'info@idgvalue.com', -'email': [''], -'email_on_failure': True, -'email_on_retry':True, -'start_date': datetime(2022, 9, 12), -'depends_on_past': False, -'retries': 6, -'retry_delay': timedelta(minutes=10), -} - -dag = DAG('wf_dag_tk_api', default_args=default_args, -schedule_interval="0 0 * * *", -catchup=False, -dagrun_timeout=timedelta(minutes=160), -max_active_runs=3) - -task_failed = EmailOperator ( - dag=dag, - trigger_rule=TriggerRule.ONE_FAILED, - task_id="task_failed", - to=["info@idgvalue.com"], - cc=[""], - subject="tk_api_failed", - html_content='

您好，tk_api作业失败，请及时处理"

') - -scrm_contact_feign = SSHOperator( -ssh_hook=sshHook, -task_id='scrm_contact_feign', -command='python3 /data/airflow/etl/API/scrm_contact_feign.py', -depends_on_past=False, -retries=3, -dag=dag) - -scrm_contact_load = SSHOperator( -ssh_hook=sshHook, -task_id='scrm_contact_load', -command='/data/airflow/etl/API/run_psql.sh {{ ds_nodash }} {{params.my_param }}', -params={'my_param':"scrm_contact_load"}, -depends_on_past=False, -retries=3, -dag=dag) - -scrm_contact_feign >> scrm_contact_load - -scrm_contact_1129 = SSHOperator( -ssh_hook=sshHook, -task_id='scrm_contact_1129', -command='/data/airflow/etl/SA/run_sa.sh {{ ds_nodash }} {{ params.my_param }} >>/data/airflow/logs/run_psql_{{ds_nodash}}.log 2>&1 ', -params={'my_param':"S98_S_scrm_contact"}, -depends_on_past=False, -retries=3, -dag=dag) - -custom_events_update_feign = SSHOperator( -ssh_hook=sshHook, -task_id='custom_events_update_feign', -command='python3 /data/airflow/etl/API/custom_events_update_feign.py', -depends_on_past=False, -retries=3, -dag=dag) - -custom_events_update_load = SSHOperator( -ssh_hook=sshHook, -task_id='custom_events_update_load', -command='/data/airflow/etl/API/run_psql.sh {{ ds_nodash }} {{params.my_param }}', -params={'my_param':"custom_events_update_load"}, -depends_on_past=False, -retries=3, -dag=dag) - -custom_events_update_feign >> custom_events_update_load - -update_scrm_contact_feign = SSHOperator( -ssh_hook=sshHook, -task_id='update_scrm_contact_feign', -command='python3 /data/airflow/etl/API/update_scrm_contact_feign.py', -depends_on_past=False, -retries=3, -dag=dag) - -update_scrm_contact_load = SSHOperator( -ssh_hook=sshHook, -task_id='update_scrm_contact_load', -command='/data/airflow/etl/API/run_psql.sh {{ ds_nodash }} {{params.my_param }}', -params={'my_param':"update_scrm_contact_load"}, -depends_on_past=False, -retries=3, -dag=dag) - -update_scrm_contact_feign >> update_scrm_contact_load - -customer_events_feign = SSHOperator( -ssh_hook=sshHook, -task_id='customer_events_feign', -command='python3 /data/airflow/etl/API/customer_events_feign.py', -depends_on_past=False, -retries=3, -dag=dag) - -customer_events_load = SSHOperator( -ssh_hook=sshHook, -task_id='customer_events_load', -command='/data/airflow/etl/API/run_psql.sh {{ ds_nodash }} {{params.my_param }}', -params={'my_param':"customer_events_load"}, -depends_on_past=False, -retries=3, -dag=dag) - -customer_events_feign >> customer_events_load - -customer_event_meta_feign = SSHOperator( -ssh_hook=sshHook, -task_id='customer_event_meta_feign', -command='python3 /data/airflow/etl/API/customer_event_meta_feign.py', -depends_on_past=False, -retries=3, -dag=dag) - -customer_event_meta_load = SSHOperator( -ssh_hook=sshHook, -task_id='customer_event_meta_load', -command='/data/airflow/etl/API/run_psql.sh {{ ds_nodash }} {{params.my_param }}', -params={'my_param':"customer_event_meta_load"}, -depends_on_past=False, -retries=3, -dag=dag) - -customer_event_meta_feign >> customer_event_meta_load - -scrm_contact_lastupdated_4112 = SSHOperator( -ssh_hook=sshHook, -task_id='scrm_contact_lastupdated_4112', -command='/data/airflow/etl/SA/run_sa.sh {{ ds_nodash }} {{ params.my_param }} >>/data/airflow/logs/run_psql_{{ds_nodash}}.log 2>&1 ', -params={'my_param':"S98_S_scrm_contact_lastupdated"}, -depends_on_past=False, -retries=3, -dag=dag) - -customer_event_meta_2268 = SSHOperator( -ssh_hook=sshHook, -task_id='customer_event_meta_2268', -command='/data/airflow/etl/SA/run_sa.sh {{ ds_nodash }} {{ params.my_param }} >>/data/airflow/logs/run_psql_{{ds_nodash}}.log 2>&1 ', -params={'my_param':"S98_S_customer_event_meta"}, -depends_on_past=False, -retries=3, -dag=dag) - -customer_events_lastupdated_8895 = SSHOperator( -ssh_hook=sshHook, -task_id='customer_events_lastupdated_8895', -command='/data/airflow/etl/SA/run_sa.sh {{ ds_nodash }} {{ params.my_param }} >>/data/airflow/logs/run_psql_{{ds_nodash}}.log 2>&1 ', -params={'my_param':"S98_S_customer_events_lastupdated"}, -depends_on_past=False, -retries=3, -dag=dag) - -customer_events_3292 = SSHOperator( -ssh_hook=sshHook, -task_id='customer_events_3292', -command='/data/airflow/etl/SA/run_sa.sh {{ ds_nodash }} {{ params.my_param }} >>/data/airflow/logs/run_psql_{{ds_nodash}}.log 2>&1 ', -params={'my_param':"S98_S_customer_events"}, -depends_on_past=False, -retries=3, -dag=dag) - -custom_events_9_feign = SSHOperator( -ssh_hook=sshHook, -task_id='custom_events_9_feign', -command='python3 /data/airflow/etl/API/custom_events_9_feign.py', -depends_on_past=False, -retries=3, -dag=dag) - -custom_events_9_load = SSHOperator( -ssh_hook=sshHook, -task_id='custom_events_9_load', -command='/data/airflow/etl/API/run_psql.sh {{ ds_nodash }} {{params.my_param }}', -params={'my_param':"custom_events_9_load"}, -depends_on_past=False, -retries=3, -dag=dag) - -custom_events_9_feign >> custom_events_9_load - -custom_events_6_feign = SSHOperator( -ssh_hook=sshHook, -task_id='custom_events_6_feign', -command='python3 /data/airflow/etl/API/custom_events_6_feign.py', -depends_on_past=False, -retries=3, -dag=dag) - -custom_events_6_load = SSHOperator( -ssh_hook=sshHook, -task_id='custom_events_6_load', -command='/data/airflow/etl/API/run_psql.sh {{ ds_nodash }} {{params.my_param }}', -params={'my_param':"custom_events_6_load"}, -depends_on_past=False, -retries=3, -dag=dag) - -custom_events_6_feign >> custom_events_6_load - -custom_events_4_feign = SSHOperator( -ssh_hook=sshHook, -task_id='custom_events_4_feign', -command='python3 /data/airflow/etl/API/custom_events_4_feign.py', -depends_on_past=False, -retries=3, -dag=dag) - -custom_events_4_load = SSHOperator( -ssh_hook=sshHook, -task_id='custom_events_4_load', -command='/data/airflow/etl/API/run_psql.sh {{ ds_nodash }} {{params.my_param }}', -params={'my_param':"custom_events_4_load"}, -depends_on_past=False, -retries=3, -dag=dag) - -custom_events_4_feign >> custom_events_4_load - -custom_events_8_feign = SSHOperator( -ssh_hook=sshHook, -task_id='custom_events_8_feign', -command='python3 /data/airflow/etl/API/custom_events_8_feign.py', -depends_on_past=False, -retries=3, -dag=dag) - -custom_events_8_load = SSHOperator( -ssh_hook=sshHook, -task_id='custom_events_8_load', -command='/data/airflow/etl/API/run_psql.sh {{ ds_nodash }} {{params.my_param }}', -params={'my_param':"custom_events_8_load"}, -depends_on_past=False, -retries=3, -dag=dag) - -custom_events_8_feign >> custom_events_8_load - -custom_events_3_feign = SSHOperator( -ssh_hook=sshHook, -task_id='custom_events_3_feign', -command='python3 /data/airflow/etl/API/custom_events_3_feign.py', -depends_on_past=False, -retries=3, -dag=dag) - -custom_events_3_load = SSHOperator( -ssh_hook=sshHook, -task_id='custom_events_3_load', -command='/data/airflow/etl/API/run_psql.sh {{ ds_nodash }} {{params.my_param }}', -params={'my_param':"custom_events_3_load"}, -depends_on_past=False, -retries=3, -dag=dag) - -custom_events_3_feign >> custom_events_3_load - -custom_events_1_feign = SSHOperator( -ssh_hook=sshHook, -task_id='custom_events_1_feign', -command='python3 /data/airflow/etl/API/custom_events_1_feign.py', -depends_on_past=False, -retries=3, -dag=dag) - -custom_events_1_load = SSHOperator( -ssh_hook=sshHook, -task_id='custom_events_1_load', -command='/data/airflow/etl/API/run_psql.sh {{ ds_nodash }} {{params.my_param }}', -params={'my_param':"custom_events_1_load"}, -depends_on_past=False, -retries=3, -dag=dag) - -custom_events_1_feign >> custom_events_1_load - -custom_events_7_feign = SSHOperator( -ssh_hook=sshHook, -task_id='custom_events_7_feign', -command='python3 /data/airflow/etl/API/custom_events_7_feign.py', -depends_on_past=False, -retries=3, -dag=dag) - -custom_events_7_load = SSHOperator( -ssh_hook=sshHook, -task_id='custom_events_7_load', -command='/data/airflow/etl/API/run_psql.sh {{ ds_nodash }} {{params.my_param }}', -params={'my_param':"custom_events_7_load"}, -depends_on_past=False, -retries=3, -dag=dag) - -custom_events_7_feign >> custom_events_7_load - -custom_events_2_feign = SSHOperator( -ssh_hook=sshHook, -task_id='custom_events_2_feign', -command='python3 /data/airflow/etl/API/custom_events_2_feign.py', -depends_on_past=False, -retries=3, -dag=dag) - -custom_events_2_load = SSHOperator( -ssh_hook=sshHook, -task_id='custom_events_2_load', -command='/data/airflow/etl/API/run_psql.sh {{ ds_nodash }} {{params.my_param }}', -params={'my_param':"custom_events_2_load"}, -depends_on_past=False, -retries=3, -dag=dag) - -custom_events_2_feign >> custom_events_2_load - -custom_events_5_feign = SSHOperator( -ssh_hook=sshHook, -task_id='custom_events_5_feign', -command='python3 /data/airflow/etl/API/custom_events_5_feign.py', -depends_on_past=False, -retries=3, -dag=dag) - -custom_events_5_load = SSHOperator( -ssh_hook=sshHook, -task_id='custom_events_5_load', -command='/data/airflow/etl/API/run_psql.sh {{ ds_nodash }} {{params.my_param }}', -params={'my_param':"custom_events_5_load"}, -depends_on_past=False, -retries=3, -dag=dag) - -custom_events_5_feign >> custom_events_5_load - -customer_events_submit_form_4283 = SSHOperator( -ssh_hook=sshHook, -task_id='customer_events_submit_form_4283', -command='/data/airflow/etl/SA/run_sa.sh {{ ds_nodash }} {{ params.my_param }} >>/data/airflow/logs/run_psql_{{ds_nodash}}.log 2>&1 ', -params={'my_param':"S98_S_customer_events_submit_form"}, -depends_on_past=False, -retries=3, -dag=dag) - -customer_events_wechat_scan_2105 = SSHOperator( -ssh_hook=sshHook, -task_id='customer_events_wechat_scan_2105', -command='/data/airflow/etl/SA/run_sa.sh {{ ds_nodash }} {{ params.my_param }} >>/data/airflow/logs/run_psql_{{ds_nodash}}.log 2>&1 ', -params={'my_param':"S98_S_customer_events_wechat_scan"}, -depends_on_past=False, -retries=3, -dag=dag) - -customer_events_subscribe_5997 = SSHOperator( -ssh_hook=sshHook, -task_id='customer_events_subscribe_5997', -command='/data/airflow/etl/SA/run_sa.sh {{ ds_nodash }} {{ params.my_param }} >>/data/airflow/logs/run_psql_{{ds_nodash}}.log 2>&1 ', -params={'my_param':"S98_S_customer_events_subscribe"}, -depends_on_past=False, -retries=3, -dag=dag) - -customer_events_click_link_in_page_2034 = SSHOperator( -ssh_hook=sshHook, -task_id='customer_events_click_link_in_page_2034', -command='/data/airflow/etl/SA/run_sa.sh {{ ds_nodash }} {{ params.my_param }} >>/data/airflow/logs/run_psql_{{ds_nodash}}.log 2>&1 ', -params={'my_param':"S98_S_customer_events_click_link_in_page"}, -depends_on_past=False, -retries=3, -dag=dag) - -customer_events_open_content_page_9684 = SSHOperator( -ssh_hook=sshHook, -task_id='customer_events_open_content_page_9684', -command='/data/airflow/etl/SA/run_sa.sh {{ ds_nodash }} {{ params.my_param }} >>/data/airflow/logs/run_psql_{{ds_nodash}}.log 2>&1 ', -params={'my_param':"S98_S_customer_events_open_content_page"}, -depends_on_past=False, -retries=3, -dag=dag) - -customer_events_open_app_5878 = SSHOperator( -ssh_hook=sshHook, -task_id='customer_events_open_app_5878', -command='/data/airflow/etl/SA/run_sa.sh {{ ds_nodash }} {{ params.my_param }} >>/data/airflow/logs/run_psql_{{ds_nodash}}.log 2>&1 ', -params={'my_param':"S98_S_customer_events_open_app"}, -depends_on_past=False, -retries=3, -dag=dag) - -customer_events_open_page_7519 = SSHOperator( -ssh_hook=sshHook, -task_id='customer_events_open_page_7519', -command='/data/airflow/etl/SA/run_sa.sh {{ ds_nodash }} {{ params.my_param }} >>/data/airflow/logs/run_psql_{{ds_nodash}}.log 2>&1 ', -params={'my_param':"S98_S_customer_events_open_page"}, -depends_on_past=False, -retries=3, -dag=dag) - -customer_events_c_minipro_page_view_9309 = SSHOperator( -ssh_hook=sshHook, -task_id='customer_events_c_minipro_page_view_9309', -command='/data/airflow/etl/SA/run_sa.sh {{ ds_nodash }} {{ params.my_param }} >>/data/airflow/logs/run_psql_{{ds_nodash}}.log 2>&1 ', -params={'my_param':"S98_S_customer_events_c_minipro_page_view"}, -depends_on_past=False, -retries=3, -dag=dag) - -customer_events_add_user_9393 = SSHOperator( -ssh_hook=sshHook, -task_id='customer_events_add_user_9393', -command='/data/airflow/etl/SA/run_sa.sh {{ ds_nodash }} {{ params.my_param }} >>/data/airflow/logs/run_psql_{{ds_nodash}}.log 2>&1 ', -params={'my_param':"S98_S_customer_events_add_user"}, -depends_on_past=False, -retries=3, -dag=dag) - -scrm_contact_load >> scrm_contact_1129 -update_scrm_contact_load >> scrm_contact_lastupdated_4112 -customer_event_meta_load >> customer_event_meta_2268 -custom_events_update_load >> customer_events_lastupdated_8895 -customer_events_load >> customer_events_3292 -custom_events_9_load >> customer_events_wechat_scan_2105 -custom_events_6_load >> customer_events_submit_form_4283 -custom_events_4_load >> customer_events_subscribe_5997 -custom_events_8_load >> customer_events_click_link_in_page_2034 -custom_events_1_load >> customer_events_open_app_5878 -custom_events_2_load >> customer_events_open_content_page_9684 -custom_events_3_load >> customer_events_open_page_7519 -custom_events_5_load >> customer_events_c_minipro_page_view_9309 -custom_events_7_load >> customer_events_add_user_9393 -customer_events_add_user_9393 >> task_failed diff --git a/TK_Cust/dev/tk_api/获取SCRM contact/scrm_contact_feign.py b/TK_Cust/dev/tk_api/获取SCRM contact/scrm_contact_feign.py deleted file mode 100644 index 6d5f4d0..0000000 --- a/TK_Cust/dev/tk_api/获取SCRM contact/scrm_contact_feign.py +++ /dev/null @@ -1,41 +0,0 @@ -# coding: utf-8 -import requests -import json -import psycopg2 -import uuid -import datetime - -# 获取当前日期 -current_date = datetime.date.today() -previous_date = current_date - datetime.timedelta(days=2) -formatted_current_date = current_date.strftime("%Y-%m-%dT%H:%M:%SZ") -formatted_previous_date = previous_date.strftime("%Y-%m-%dT%H:%M:%SZ") - -print('开始加载数据：scrm_contact:获取SCRM contact') -authUrl='https://api.huiju.cool/v2/oauth2/token?app_id=cl037b184ebccd97c&secret=deaa83fdd8d385eec0d1d04d34282c4ec8c33b60&grant_type=client_credentials' - -print('开始请求令牌。') -authRequest=requests.get(authUrl) -auth=json.loads(authRequest.text) -print('开始请求数据总数。') -url='https://api.huiju.cool/v2/customers' -header={} -body={'access_token':auth['access_token'],'sort':'dateJoin','dateJoin[ge]':formatted_previous_date,'dateJoin[le]':formatted_current_date,'limit':'',} -dataReqL=requests.get(url,headers=header,params=body) -resL=json.loads(dataReqL.text) -# print(resL) -dataList=resL['data'] -total=len(dataList) -conn = psycopg2.connect(database="dataops_db", user="dbuser_dops", password="MIgTi3jA", - host="172.17.0.8", port="5432") -print('数据库连接成功') -dataId=str(uuid.uuid4()) -print('临时id：'+dataId) -json_object = json.dumps(dataList) -cur=conn.cursor() -sql="update data_api.api_data set is_loaded = '1' where api_id = 'cab1a761-b8e5-4db4-a769-88db3ec1';INSERT INTO data_api.api_data (id,api_id,data,total_num,is_loaded,status,request_tm,execute_tm,remark) values (%s,%s,%s,%s,'0','0',current_timestamp(0),current_timestamp(0),'')" -cur.execute(sql,[dataId,'cab1a761-b8e5-4db4-a769-88db3ec1',json_object,total]) -conn.commit() -cur.close() -conn.close() -print('加载数据结束：scrm_contact:获取SCRM contact') \ No newline at end of file diff --git a/TK_Cust/dev/tk_api/获取SCRM contact/scrm_contact_load.sql b/TK_Cust/dev/tk_api/获取SCRM contact/scrm_contact_load.sql deleted file mode 100644 index 8079f7e..0000000 --- a/TK_Cust/dev/tk_api/获取SCRM contact/scrm_contact_load.sql +++ /dev/null @@ -1,105 +0,0 @@ -/*******Main Section**************************************************************************/ -\set ON_ERROR_STOP on -\set AUTOCOMMIT on -\timing on - -DELETE FROM data_api.scrm_contact; - -insert into data_api.scrm_contact ( - img - , not_acceptsms - , gender - , create_method - , mobile_verified - , language - , is_member - , source - , is_referrer - , create_from - , last_updated - , date_created - , acceptsms - , id - , subscribe_scene - , id_str - , is_employee - , version - , not_accept_email - , date_join - , content_name - , update_method - , email_verified - , is_anonymous - , accept_email - , create_from_name - , status - ,etl_tx_dt -) -select - case when trim(both from img)='' then null else img::text end img - , case when trim(both from not_acceptsms)='' then null else not_acceptsms::text end not_acceptsms - , case when trim(both from gender)='' then null else gender::text end gender - , case when trim(both from create_method)='' then null else create_method::text end create_method - , case when trim(both from mobile_verified)='' then null else mobile_verified::text end mobile_verified - , case when trim(both from language)='' then null else language::text end language - , case when trim(both from is_member)='' then null else is_member::text end is_member - , case when trim(both from source)='' then null else source::text end source - , case when trim(both from is_referrer)='' then null else is_referrer::text end is_referrer - , case when trim(both from create_from)='' then null else create_from::text end create_from - , case when trim(both from last_updated)='' then null else last_updated::text end last_updated - , case when trim(both from date_created)='' then null else date_created::text end date_created - , case when trim(both from acceptsms)='' then null else acceptsms::text end acceptsms - , case when trim(both from id)='' then null else id::text end id - , case when trim(both from subscribe_scene)='' then null else subscribe_scene::text end subscribe_scene - , case when trim(both from id_str)='' then null else id_str::text end id_str - , case when trim(both from is_employee)='' then null else is_employee::text end is_employee - , case when trim(both from version)='' then null else version::text end version - , case when trim(both from not_accept_email)='' then null else not_accept_email::text end not_accept_email - , case when trim(both from date_join)='' then null else date_join::text end date_join - , case when trim(both from content_name)='' then null else content_name::text end content_name - , case when trim(both from update_method)='' then null else update_method::text end update_method - , case when trim(both from email_verified)='' then null else email_verified::text end email_verified - , case when trim(both from is_anonymous)='' then null else is_anonymous::text end is_anonymous - , case when trim(both from accept_email)='' then null else accept_email::text end accept_email - , case when trim(both from create_from_name)='' then null else create_from_name::text end create_from_name - , case when trim(both from status)='' then null else status::text end status -,etl_tx_dt -from ( -select - (json_array_elements(data::json)::json->>'img') img - , (json_array_elements(data::json)::json->>'notAcceptsms') not_acceptsms - , (json_array_elements(data::json)::json->>'gender') gender - , (json_array_elements(data::json)::json->>'createMethod') create_method - , (json_array_elements(data::json)::json->>'mobileVerified') mobile_verified - , (json_array_elements(data::json)::json->>'language') language - , (json_array_elements(data::json)::json->>'isMember') is_member - , (json_array_elements(data::json)::json->>'source') source - , (json_array_elements(data::json)::json->>'isReferrer') is_referrer - , (json_array_elements(data::json)::json->>'createFrom') create_from - , (json_array_elements(data::json)::json->>'lastUpdated') last_updated - , (json_array_elements(data::json)::json->>'dateCreated') date_created - , (json_array_elements(data::json)::json->>'acceptsms') acceptsms - , (json_array_elements(data::json)::json->>'id') id - , (json_array_elements(data::json)::json->>'subscribeScene') subscribe_scene - , (json_array_elements(data::json)::json->>'idStr') id_str - , (json_array_elements(data::json)::json->>'isEmployee') is_employee - , (json_array_elements(data::json)::json->>'version') version - , (json_array_elements(data::json)::json->>'notAcceptEmail') not_accept_email - , (json_array_elements(data::json)::json->>'dateJoin') date_join - , (json_array_elements(data::json)::json->>'contentName') content_name - , (json_array_elements(data::json)::json->>'updateMethod') update_method - , (json_array_elements(data::json)::json->>'emailVerified') email_verified - , (json_array_elements(data::json)::json->>'isAnonymous') is_anonymous - , (json_array_elements(data::json)::json->>'acceptEmail') accept_email - , (json_array_elements(data::json)::json->>'createFromName') create_from_name - , (json_array_elements(data::json)::json->>'status') status - ,CURRENT_TIMESTAMP(0) etl_tx_dt -from (select * from data_api.api_data -WHERE api_id='cab1a761-b8e5-4db4-a769-88db3ec1' and is_loaded = '0' order by request_tm desc limit 1) p )p; - -update data_api.api_data -set is_loaded = '1' , - status = '1', - request_tm = current_timestamp(0) -where api_id='cab1a761-b8e5-4db4-a769-88db3ec1'; -\q \ No newline at end of file diff --git a/TK_Cust/dev/tk_api/获取线索事件/customer_events_feign.py b/TK_Cust/dev/tk_api/获取线索事件/customer_events_feign.py deleted file mode 100644 index 848c849..0000000 --- a/TK_Cust/dev/tk_api/获取线索事件/customer_events_feign.py +++ /dev/null @@ -1,41 +0,0 @@ -# coding: utf-8 -import requests -import json -import psycopg2 -import uuid -import datetime - -# 获取当前日期 -current_date = datetime.date.today() -previous_date = current_date - datetime.timedelta(days=2) -formatted_current_date = current_date.strftime("%Y-%m-%dT%H:%M:%SZ") -formatted_previous_date = previous_date.strftime("%Y-%m-%dT%H:%M:%SZ") - -print('开始加载数据：customer_events:获取线索事件') -authUrl='https://api.huiju.cool/v2/oauth2/token?app_id=cl037b184ebccd97c&secret=deaa83fdd8d385eec0d1d04d34282c4ec8c33b60&grant_type=client_credentials' - -print('开始请求令牌。') -authRequest=requests.get(authUrl) -auth=json.loads(authRequest.text) -print('开始请求数据总数。') -url='https://api.huiju.cool/v2/customerEvents' -header={} -body={'access_token':auth['access_token'],'limit':'','sort':'date','event':'create_customer','date[ge]':formatted_previous_date,'date[le]':formatted_current_date,} -dataReqL=requests.get(url,headers=header,params=body) -resL=json.loads(dataReqL.text) -# print(resL) -dataList=resL['data'] -total=len(dataList) -conn = psycopg2.connect(database="dataops_db", user="dbuser_dops", password="MIgTi3jA", - host="172.17.0.8", port="5432") -print('数据库连接成功') -dataId=str(uuid.uuid4()) -print('临时id：'+dataId) -json_object = json.dumps(dataList) -cur=conn.cursor() -sql="update data_api.api_data set is_loaded = '1' where api_id = '747d33fa-a0e0-421d-aa9b-4ca4517b';INSERT INTO data_api.api_data (id,api_id,data,total_num,is_loaded,status,request_tm,execute_tm,remark) values (%s,%s,%s,%s,'0','0',current_timestamp(0),current_timestamp(0),'')" -cur.execute(sql,[dataId,'747d33fa-a0e0-421d-aa9b-4ca4517b',json_object,total]) -conn.commit() -cur.close() -conn.close() -print('加载数据结束：customer_events:获取线索事件') \ No newline at end of file diff --git a/TK_Cust/dev/tk_api/获取线索事件/customer_events_load.sql b/TK_Cust/dev/tk_api/获取线索事件/customer_events_load.sql deleted file mode 100644 index 1633753..0000000 --- a/TK_Cust/dev/tk_api/获取线索事件/customer_events_load.sql +++ /dev/null @@ -1,57 +0,0 @@ -/*******Main Section**************************************************************************/ -\set ON_ERROR_STOP on -\set AUTOCOMMIT on -\timing on - -DELETE FROM data_api.customer_events; - -insert into data_api.customer_events ( - id_str - , date - , last_updated - , customer_id_str - , create_method - , customer_id - , external_id - , id - , source - , event - , content_name - ,etl_tx_dt -) -select - case when trim(both from id_str)='' then null else id_str::text end id_str - , case when trim(both from date)='' then null else date::text end date - , case when trim(both from last_updated)='' then null else last_updated::text end last_updated - , case when trim(both from customer_id_str)='' then null else customer_id_str::text end customer_id_str - , case when trim(both from create_method)='' then null else create_method::text end create_method - , case when trim(both from customer_id)='' then null else customer_id::text end customer_id - , case when trim(both from external_id)='' then null else external_id::text end external_id - , case when trim(both from id)='' then null else id::text end id - , case when trim(both from source)='' then null else source::text end source - , case when trim(both from event)='' then null else event::text end event - , case when trim(both from content_name)='' then null else content_name::text end content_name -,etl_tx_dt -from ( -select - (json_array_elements(data::json)::json->>'idStr') id_str - , (json_array_elements(data::json)::json->>'date') date - , (json_array_elements(data::json)::json->>'lastUpdated') last_updated - , (json_array_elements(data::json)::json->>'customerIdStr') customer_id_str - , (json_array_elements(data::json)::json->>'createMethod') create_method - , (json_array_elements(data::json)::json->>'customerId') customer_id - , (json_array_elements(data::json)::json->>'externalId') external_id - , (json_array_elements(data::json)::json->>'id') id - , (json_array_elements(data::json)::json->>'source') source - , (json_array_elements(data::json)::json->>'event') event - , (json_array_elements(data::json)::json->>'contentName') content_name - ,CURRENT_TIMESTAMP(0) etl_tx_dt -from (select * from data_api.api_data -WHERE api_id='747d33fa-a0e0-421d-aa9b-4ca4517b' and is_loaded = '0' order by request_tm desc limit 1) p )p; - -update data_api.api_data -set is_loaded = '1' , - status = '1', - request_tm = current_timestamp(0) -where api_id='747d33fa-a0e0-421d-aa9b-4ca4517b'; -\q \ No newline at end of file diff --git a/TK_Cust/dev/tk_api/获取线索事件_c_minipro_page_view/custom_events_5_feign.py b/TK_Cust/dev/tk_api/获取线索事件_c_minipro_page_view/custom_events_5_feign.py deleted file mode 100644 index 6edebaa..0000000 --- a/TK_Cust/dev/tk_api/获取线索事件_c_minipro_page_view/custom_events_5_feign.py +++ /dev/null @@ -1,41 +0,0 @@ -# coding: utf-8 -import requests -import json -import psycopg2 -import uuid -import datetime - -# 获取当前日期 -current_date = datetime.date.today() -previous_date = current_date - datetime.timedelta(days=2) -formatted_current_date = current_date.strftime("%Y-%m-%dT%H:%M:%SZ") -formatted_previous_date = previous_date.strftime("%Y-%m-%dT%H:%M:%SZ") - -print('开始加载数据：custom_events_5:获取线索事件_c_minipro_page_view') -authUrl='https://api.huiju.cool/v2/oauth2/token?app_id=cl037b184ebccd97c&secret=deaa83fdd8d385eec0d1d04d34282c4ec8c33b60&grant_type=client_credentials' - -print('开始请求令牌。') -authRequest=requests.get(authUrl) -auth=json.loads(authRequest.text) -print('开始请求数据总数。') -url='https://api.huiju.cool/v2/customerEvents' -header={} -body={'access_token':auth['access_token'],'limit':'','event':'c_minipro_page_view','sort':'date','date[ge]':formatted_previous_date,'date[le]':formatted_current_date,} -dataReqL=requests.get(url,headers=header,params=body) -resL=json.loads(dataReqL.text) -# print(resL) -dataList=resL['data'] -total=len(dataList) -conn = psycopg2.connect(database="dataops_db", user="dbuser_dops", password="MIgTi3jA", - host="172.17.0.8", port="5432") -print('数据库连接成功') -dataId=str(uuid.uuid4()) -print('临时id：'+dataId) -json_object = json.dumps(dataList) -cur=conn.cursor() -sql="update data_api.api_data set is_loaded = '1' where api_id = 'a71a43ff-efcd-4427-8dbc-ed2686e4';INSERT INTO data_api.api_data (id,api_id,data,total_num,is_loaded,status,request_tm,execute_tm,remark) values (%s,%s,%s,%s,'0','0',current_timestamp(0),current_timestamp(0),'')" -cur.execute(sql,[dataId,'a71a43ff-efcd-4427-8dbc-ed2686e4',json_object,total]) -conn.commit() -cur.close() -conn.close() -print('加载数据结束：custom_events_5:获取线索事件_c_minipro_page_view') \ No newline at end of file diff --git a/TK_Cust/dev/tk_api/获取线索事件_c_minipro_page_view/custom_events_5_load.sql b/TK_Cust/dev/tk_api/获取线索事件_c_minipro_page_view/custom_events_5_load.sql deleted file mode 100644 index cfea3d5..0000000 --- a/TK_Cust/dev/tk_api/获取线索事件_c_minipro_page_view/custom_events_5_load.sql +++ /dev/null @@ -1,96 +0,0 @@ -/*******Main Section**************************************************************************/ -\set ON_ERROR_STOP on -\set AUTOCOMMIT on -\timing on - -DELETE FROM data_api.customer_events_c_minipro_page_view; - -insert into data_api.customer_events_c_minipro_page_view ( - id_str - , c_source_content - , date - , target_name - , c_source - , os - , target_id - , customer_id_str - , ip_city - , ip - , external_id - , ip_country - , platform - , url - , last_updated - , c_url - , os_version - , browser - , browser_version - , customer_id - , ip_province - , c_name - , id - , event - ,etl_tx_dt -) -select - case when trim(both from id_str)='' then null else id_str::text end id_str - , case when trim(both from c_source_content)='' then null else c_source_content::text end c_source_content - , case when trim(both from date)='' then null else date::text end date - , case when trim(both from target_name)='' then null else target_name::text end target_name - , case when trim(both from c_source)='' then null else c_source::text end c_source - , case when trim(both from os)='' then null else os::text end os - , case when trim(both from target_id)='' then null else target_id::text end target_id - , case when trim(both from customer_id_str)='' then null else customer_id_str::text end customer_id_str - , case when trim(both from ip_city)='' then null else ip_city::text end ip_city - , case when trim(both from ip)='' then null else ip::text end ip - , case when trim(both from external_id)='' then null else external_id::text end external_id - , case when trim(both from ip_country)='' then null else ip_country::text end ip_country - , case when trim(both from platform)='' then null else platform::text end platform - , case when trim(both from url)='' then null else url::text end url - , case when trim(both from last_updated)='' then null else last_updated::text end last_updated - , case when trim(both from c_url)='' then null else c_url::text end c_url - , case when trim(both from os_version)='' then null else os_version::text end os_version - , case when trim(both from browser)='' then null else browser::text end browser - , case when trim(both from browser_version)='' then null else browser_version::text end browser_version - , case when trim(both from customer_id)='' then null else customer_id::text end customer_id - , case when trim(both from ip_province)='' then null else ip_province::text end ip_province - , case when trim(both from c_name)='' then null else c_name::text end c_name - , case when trim(both from id)='' then null else id::text end id - , case when trim(both from event)='' then null else event::text end event -,etl_tx_dt -from ( -select - (json_array_elements(data::json)::json->>'idStr') id_str - , (json_array_elements(data::json)::json->>'c_sourceContent') c_source_content - , (json_array_elements(data::json)::json->>'date') date - , (json_array_elements(data::json)::json->>'targetName') target_name - , (json_array_elements(data::json)::json->>'c_source') c_source - , (json_array_elements(data::json)::json->>'os') os - , (json_array_elements(data::json)::json->>'targetId') target_id - , (json_array_elements(data::json)::json->>'customerIdStr') customer_id_str - , (json_array_elements(data::json)::json->>'ipCity') ip_city - , (json_array_elements(data::json)::json->>'ip') ip - , (json_array_elements(data::json)::json->>'externalId') external_id - , (json_array_elements(data::json)::json->>'ipCountry') ip_country - , (json_array_elements(data::json)::json->>'platform') platform - , (json_array_elements(data::json)::json->>'url') url - , (json_array_elements(data::json)::json->>'lastUpdated') last_updated - , (json_array_elements(data::json)::json->>'c_url') c_url - , (json_array_elements(data::json)::json->>'osVersion') os_version - , (json_array_elements(data::json)::json->>'browser') browser - , (json_array_elements(data::json)::json->>'browserVersion') browser_version - , (json_array_elements(data::json)::json->>'customerId') customer_id - , (json_array_elements(data::json)::json->>'ipProvince') ip_province - , (json_array_elements(data::json)::json->>'c_name') c_name - , (json_array_elements(data::json)::json->>'id') id - , (json_array_elements(data::json)::json->>'event') event - ,CURRENT_TIMESTAMP(0) etl_tx_dt -from (select * from data_api.api_data -WHERE api_id='a71a43ff-efcd-4427-8dbc-ed2686e4' and is_loaded = '0' order by request_tm desc limit 1) p )p; - -update data_api.api_data -set is_loaded = '1' , - status = '1', - request_tm = current_timestamp(0) -where api_id='a71a43ff-efcd-4427-8dbc-ed2686e4'; -\q \ No newline at end of file diff --git a/TK_Cust/dev/tk_api/获取线索事件_click_link_in_page/custom_events_8_feign.py b/TK_Cust/dev/tk_api/获取线索事件_click_link_in_page/custom_events_8_feign.py deleted file mode 100644 index e349eb7..0000000 --- a/TK_Cust/dev/tk_api/获取线索事件_click_link_in_page/custom_events_8_feign.py +++ /dev/null @@ -1,41 +0,0 @@ -# coding: utf-8 -import requests -import json -import psycopg2 -import uuid -import datetime - -# 获取当前日期 -current_date = datetime.date.today() -previous_date = current_date - datetime.timedelta(days=2) -formatted_current_date = current_date.strftime("%Y-%m-%dT%H:%M:%SZ") -formatted_previous_date = previous_date.strftime("%Y-%m-%dT%H:%M:%SZ") - -print('开始加载数据：custom_events_8:获取线索事件_click_link_in_page') -authUrl='https://api.huiju.cool/v2/oauth2/token?app_id=cl037b184ebccd97c&secret=deaa83fdd8d385eec0d1d04d34282c4ec8c33b60&grant_type=client_credentials' - -print('开始请求令牌。') -authRequest=requests.get(authUrl) -auth=json.loads(authRequest.text) -print('开始请求数据总数。') -url='https://api.huiju.cool/v2/customerEvents' -header={} -body={'sort':'date','date[le]':formatted_current_date,'limit':'','event':'click_link_in_page','access_token':auth['access_token'],'date[ge]':formatted_previous_date,} -dataReqL=requests.get(url,headers=header,params=body) -resL=json.loads(dataReqL.text) -# print(resL) -dataList=resL['data'] -total=len(dataList) -conn = psycopg2.connect(database="dataops_db", user="dbuser_dops", password="MIgTi3jA", - host="172.17.0.8", port="5432") -print('数据库连接成功') -dataId=str(uuid.uuid4()) -print('临时id：'+dataId) -json_object = json.dumps(dataList) -cur=conn.cursor() -sql="update data_api.api_data set is_loaded = '1' where api_id = 'b20bb19a-6ace-4d10-ab2b-4cdb3c0a';INSERT INTO data_api.api_data (id,api_id,data,total_num,is_loaded,status,request_tm,execute_tm,remark) values (%s,%s,%s,%s,'0','0',current_timestamp(0),current_timestamp(0),'')" -cur.execute(sql,[dataId,'b20bb19a-6ace-4d10-ab2b-4cdb3c0a',json_object,total]) -conn.commit() -cur.close() -conn.close() -print('加载数据结束：custom_events_8:获取线索事件_click_link_in_page') \ No newline at end of file diff --git a/TK_Cust/dev/tk_api/获取线索事件_click_link_in_page/custom_events_8_load.sql b/TK_Cust/dev/tk_api/获取线索事件_click_link_in_page/custom_events_8_load.sql deleted file mode 100644 index 7be6a83..0000000 --- a/TK_Cust/dev/tk_api/获取线索事件_click_link_in_page/custom_events_8_load.sql +++ /dev/null @@ -1,102 +0,0 @@ -/*******Main Section**************************************************************************/ -\set ON_ERROR_STOP on -\set AUTOCOMMIT on -\timing on - -DELETE FROM data_api.customer_events_click_link_in_page; - -insert into data_api.customer_events_click_link_in_page ( - date - , screen_width - , target_name - , ip_city - , ip_country - , platform - , score - , last_updated - , page_type - , os_version - , browser - , browser_version - , customer_id - , id - , event - , id_str - , os - , target_id - , customer_id_str - , ip - , screen_height - , external_id - , page_id - , url - , ip_province - , device - ,etl_tx_dt -) -select - case when trim(both from date)='' then null else date::text end date - , case when trim(both from screen_width)='' then null else screen_width::text end screen_width - , case when trim(both from target_name)='' then null else target_name::text end target_name - , case when trim(both from ip_city)='' then null else ip_city::text end ip_city - , case when trim(both from ip_country)='' then null else ip_country::text end ip_country - , case when trim(both from platform)='' then null else platform::text end platform - , case when trim(both from score)='' then null else score::text end score - , case when trim(both from last_updated)='' then null else last_updated::text end last_updated - , case when trim(both from page_type)='' then null else page_type::text end page_type - , case when trim(both from os_version)='' then null else os_version::text end os_version - , case when trim(both from browser)='' then null else browser::text end browser - , case when trim(both from browser_version)='' then null else browser_version::text end browser_version - , case when trim(both from customer_id)='' then null else customer_id::text end customer_id - , case when trim(both from id)='' then null else id::text end id - , case when trim(both from event)='' then null else event::text end event - , case when trim(both from id_str)='' then null else id_str::text end id_str - , case when trim(both from os)='' then null else os::text end os - , case when trim(both from target_id)='' then null else target_id::text end target_id - , case when trim(both from customer_id_str)='' then null else customer_id_str::text end customer_id_str - , case when trim(both from ip)='' then null else ip::text end ip - , case when trim(both from screen_height)='' then null else screen_height::text end screen_height - , case when trim(both from external_id)='' then null else external_id::text end external_id - , case when trim(both from page_id)='' then null else page_id::text end page_id - , case when trim(both from url)='' then null else url::text end url - , case when trim(both from ip_province)='' then null else ip_province::text end ip_province - , case when trim(both from device)='' then null else device::text end device -,etl_tx_dt -from ( -select - (json_array_elements(data::json)::json->>'date') date - , (json_array_elements(data::json)::json->>'screenWidth') screen_width - , (json_array_elements(data::json)::json->>'targetName') target_name - , (json_array_elements(data::json)::json->>'ipCity') ip_city - , (json_array_elements(data::json)::json->>'ipCountry') ip_country - , (json_array_elements(data::json)::json->>'platform') platform - , (json_array_elements(data::json)::json->>'score') score - , (json_array_elements(data::json)::json->>'lastUpdated') last_updated - , (json_array_elements(data::json)::json->>'pageType') page_type - , (json_array_elements(data::json)::json->>'osVersion') os_version - , (json_array_elements(data::json)::json->>'browser') browser - , (json_array_elements(data::json)::json->>'browserVersion') browser_version - , (json_array_elements(data::json)::json->>'customerId') customer_id - , (json_array_elements(data::json)::json->>'id') id - , (json_array_elements(data::json)::json->>'event') event - , (json_array_elements(data::json)::json->>'idStr') id_str - , (json_array_elements(data::json)::json->>'os') os - , (json_array_elements(data::json)::json->>'targetId') target_id - , (json_array_elements(data::json)::json->>'customerIdStr') customer_id_str - , (json_array_elements(data::json)::json->>'ip') ip - , (json_array_elements(data::json)::json->>'screenHeight') screen_height - , (json_array_elements(data::json)::json->>'externalId') external_id - , (json_array_elements(data::json)::json->>'pageId') page_id - , (json_array_elements(data::json)::json->>'url') url - , (json_array_elements(data::json)::json->>'ipProvince') ip_province - , (json_array_elements(data::json)::json->>'device') device - ,CURRENT_TIMESTAMP(0) etl_tx_dt -from (select * from data_api.api_data -WHERE api_id='b20bb19a-6ace-4d10-ab2b-4cdb3c0a' and is_loaded = '0' order by request_tm desc limit 1) p )p; - -update data_api.api_data -set is_loaded = '1' , - status = '1', - request_tm = current_timestamp(0) -where api_id='b20bb19a-6ace-4d10-ab2b-4cdb3c0a'; -\q \ No newline at end of file diff --git a/TK_Cust/dev/tk_api/获取线索事件_cp_employee_tools_huiju_event_add/custom_events_7_feign.py b/TK_Cust/dev/tk_api/获取线索事件_cp_employee_tools_huiju_event_add/custom_events_7_feign.py deleted file mode 100644 index be35ba0..0000000 --- a/TK_Cust/dev/tk_api/获取线索事件_cp_employee_tools_huiju_event_add/custom_events_7_feign.py +++ /dev/null @@ -1,41 +0,0 @@ -# coding: utf-8 -import requests -import json -import psycopg2 -import uuid -import datetime - -# 获取当前日期 -current_date = datetime.date.today() -previous_date = current_date - datetime.timedelta(days=2) -formatted_current_date = current_date.strftime("%Y-%m-%dT%H:%M:%SZ") -formatted_previous_date = previous_date.strftime("%Y-%m-%dT%H:%M:%SZ") - -print('开始加载数据：custom_events_7:获取线索事件_cp_employee_tools_huiju_event_add') -authUrl='https://api.huiju.cool/v2/oauth2/token?app_id=cl037b184ebccd97c&secret=deaa83fdd8d385eec0d1d04d34282c4ec8c33b60&grant_type=client_credentials' - -print('开始请求令牌。') -authRequest=requests.get(authUrl) -auth=json.loads(authRequest.text) -print('开始请求数据总数。') -url='https://api.huiju.cool/v2/customerEvents' -header={} -body={'event':'cp_employee_tools_huiju_event_add_user','access_token':auth['access_token'],'limit':'','sort':'','date[ge]':formatted_previous_date,'date[le]':formatted_current_date,} -dataReqL=requests.get(url,headers=header,params=body) -resL=json.loads(dataReqL.text) -# print(resL) -dataList=resL['data'] -total=len(dataList) -conn = psycopg2.connect(database="dataops_db", user="dbuser_dops", password="MIgTi3jA", - host="172.17.0.8", port="5432") -print('数据库连接成功') -dataId=str(uuid.uuid4()) -print('临时id：'+dataId) -json_object = json.dumps(dataList) -cur=conn.cursor() -sql="update data_api.api_data set is_loaded = '1' where api_id = 'd3b5116a-08c3-431f-8d96-9780221b';INSERT INTO data_api.api_data (id,api_id,data,total_num,is_loaded,status,request_tm,execute_tm,remark) values (%s,%s,%s,%s,'0','0',current_timestamp(0),current_timestamp(0),'')" -cur.execute(sql,[dataId,'d3b5116a-08c3-431f-8d96-9780221b',json_object,total]) -conn.commit() -cur.close() -conn.close() -print('加载数据结束：custom_events_7:获取线索事件_cp_employee_tools_huiju_event_add') \ No newline at end of file diff --git a/TK_Cust/dev/tk_api/获取线索事件_cp_employee_tools_huiju_event_add/custom_events_7_load.sql b/TK_Cust/dev/tk_api/获取线索事件_cp_employee_tools_huiju_event_add/custom_events_7_load.sql deleted file mode 100644 index 620f3be..0000000 --- a/TK_Cust/dev/tk_api/获取线索事件_cp_employee_tools_huiju_event_add/custom_events_7_load.sql +++ /dev/null @@ -1,69 +0,0 @@ -/*******Main Section**************************************************************************/ -\set ON_ERROR_STOP on -\set AUTOCOMMIT on -\timing on - -DELETE FROM data_api.customer_events_add_user; - -insert into data_api.customer_events_add_user ( - id_str - , date - , target_id - , customer_id_str - , attr6 - , attr5 - , external_id - , attr7 - , source - , attr2 - , attr1 - , last_updated - , customer_id - , id - , event - ,etl_tx_dt -) -select - case when trim(both from id_str)='' then null else id_str::text end id_str - , case when trim(both from date)='' then null else date::text end date - , case when trim(both from target_id)='' then null else target_id::text end target_id - , case when trim(both from customer_id_str)='' then null else customer_id_str::text end customer_id_str - , case when trim(both from attr6)='' then null else attr6::text end attr6 - , case when trim(both from attr5)='' then null else attr5::text end attr5 - , case when trim(both from external_id)='' then null else external_id::text end external_id - , case when trim(both from attr7)='' then null else attr7::text end attr7 - , case when trim(both from source)='' then null else source::text end source - , case when trim(both from attr2)='' then null else attr2::text end attr2 - , case when trim(both from attr1)='' then null else attr1::text end attr1 - , case when trim(both from last_updated)='' then null else last_updated::text end last_updated - , case when trim(both from customer_id)='' then null else customer_id::text end customer_id - , case when trim(both from id)='' then null else id::text end id - , case when trim(both from event)='' then null else event::text end event -,etl_tx_dt -from ( -select - (json_array_elements(data::json)::json->>'idStr') id_str - , (json_array_elements(data::json)::json->>'date') date - , (json_array_elements(data::json)::json->>'targetId') target_id - , (json_array_elements(data::json)::json->>'customerIdStr') customer_id_str - , (json_array_elements(data::json)::json->>'attr6') attr6 - , (json_array_elements(data::json)::json->>'attr5') attr5 - , (json_array_elements(data::json)::json->>'externalId') external_id - , (json_array_elements(data::json)::json->>'attr7') attr7 - , (json_array_elements(data::json)::json->>'source') source - , (json_array_elements(data::json)::json->>'attr2') attr2 - , (json_array_elements(data::json)::json->>'attr1') attr1 - , (json_array_elements(data::json)::json->>'lastUpdated') last_updated - , (json_array_elements(data::json)::json->>'customerId') customer_id - , (json_array_elements(data::json)::json->>'id') id - , (json_array_elements(data::json)::json->>'event') event - ,CURRENT_TIMESTAMP(0) etl_tx_dt -from (select * from data_api.api_data -WHERE api_id='d3b5116a-08c3-431f-8d96-9780221b' and is_loaded = '0' order by request_tm desc limit 1) p )p; - -update data_api.api_data -set is_loaded = '1' , - status = '1', - request_tm = current_timestamp(0) -where api_id='d3b5116a-08c3-431f-8d96-9780221b'; -\q \ No newline at end of file diff --git a/TK_Cust/dev/tk_api/获取线索事件_open_app/custom_events_1_feign.py b/TK_Cust/dev/tk_api/获取线索事件_open_app/custom_events_1_feign.py deleted file mode 100644 index 3f66be9..0000000 --- a/TK_Cust/dev/tk_api/获取线索事件_open_app/custom_events_1_feign.py +++ /dev/null @@ -1,41 +0,0 @@ -# coding: utf-8 -import requests -import json -import psycopg2 -import uuid -import datetime - -# 获取当前日期 -current_date = datetime.date.today() -previous_date = current_date - datetime.timedelta(days=2) -formatted_current_date = current_date.strftime("%Y-%m-%dT%H:%M:%SZ") -formatted_previous_date = previous_date.strftime("%Y-%m-%dT%H:%M:%SZ") - -print('开始加载数据：custom_events_1:获取线索事件_open_app') -authUrl='https://api.huiju.cool/v2/oauth2/token?app_id=cl037b184ebccd97c&secret=deaa83fdd8d385eec0d1d04d34282c4ec8c33b60&grant_type=client_credentials' - -print('开始请求令牌。') -authRequest=requests.get(authUrl) -auth=json.loads(authRequest.text) -print('开始请求数据总数。') -url='https://api.huiju.cool/v2/customerEvents' -header={} -body={'access_token':auth['access_token'],'sort':'date','limit':'','date[ge]':formatted_previous_date,'date[le]':formatted_current_date,'event':'open_app',} -dataReqL=requests.get(url,headers=header,params=body) -resL=json.loads(dataReqL.text) -# print(resL) -dataList=resL['data'] -total=len(dataList) -conn = psycopg2.connect(database="dataops_db", user="dbuser_dops", password="MIgTi3jA", - host="172.17.0.8", port="5432") -print('数据库连接成功') -dataId=str(uuid.uuid4()) -print('临时id：'+dataId) -json_object = json.dumps(dataList) -cur=conn.cursor() -sql="update data_api.api_data set is_loaded = '1' where api_id = '2433868c-3416-4475-884d-11e687a6';INSERT INTO data_api.api_data (id,api_id,data,total_num,is_loaded,status,request_tm,execute_tm,remark) values (%s,%s,%s,%s,'0','0',current_timestamp(0),current_timestamp(0),'')" -cur.execute(sql,[dataId,'2433868c-3416-4475-884d-11e687a6',json_object,total]) -conn.commit() -cur.close() -conn.close() -print('加载数据结束：custom_events_1:获取线索事件_open_app') \ No newline at end of file diff --git a/TK_Cust/dev/tk_api/获取线索事件_open_app/custom_events_1_load.sql b/TK_Cust/dev/tk_api/获取线索事件_open_app/custom_events_1_load.sql deleted file mode 100644 index c632fc6..0000000 --- a/TK_Cust/dev/tk_api/获取线索事件_open_app/custom_events_1_load.sql +++ /dev/null @@ -1,102 +0,0 @@ -/*******Main Section**************************************************************************/ -\set ON_ERROR_STOP on -\set AUTOCOMMIT on -\timing on - -DELETE FROM data_api.customer_events_open_app; - -insert into data_api.customer_events_open_app ( - date - , screen_width - , target_name - , ip_city - , ip_country - , platform - , last_updated - , page_type - , os_version - , browser - , browser_version - , customer_id - , id - , event - , id_str - , os - , target_id - , customer_id_str - , ip - , screen_height - , external_id - , ip_county - , page_id - , url - , ip_province - , device - ,etl_tx_dt -) -select - case when trim(both from date)='' then null else date::text end date - , case when trim(both from screen_width)='' then null else screen_width::text end screen_width - , case when trim(both from target_name)='' then null else target_name::text end target_name - , case when trim(both from ip_city)='' then null else ip_city::text end ip_city - , case when trim(both from ip_country)='' then null else ip_country::text end ip_country - , case when trim(both from platform)='' then null else platform::text end platform - , case when trim(both from last_updated)='' then null else last_updated::text end last_updated - , case when trim(both from page_type)='' then null else page_type::text end page_type - , case when trim(both from os_version)='' then null else os_version::text end os_version - , case when trim(both from browser)='' then null else browser::text end browser - , case when trim(both from browser_version)='' then null else browser_version::text end browser_version - , case when trim(both from customer_id)='' then null else customer_id::text end customer_id - , case when trim(both from id)='' then null else id::text end id - , case when trim(both from event)='' then null else event::text end event - , case when trim(both from id_str)='' then null else id_str::text end id_str - , case when trim(both from os)='' then null else os::text end os - , case when trim(both from target_id)='' then null else target_id::text end target_id - , case when trim(both from customer_id_str)='' then null else customer_id_str::text end customer_id_str - , case when trim(both from ip)='' then null else ip::text end ip - , case when trim(both from screen_height)='' then null else screen_height::text end screen_height - , case when trim(both from external_id)='' then null else external_id::text end external_id - , case when trim(both from ip_county)='' then null else ip_county::text end ip_county - , case when trim(both from page_id)='' then null else page_id::text end page_id - , case when trim(both from url)='' then null else url::text end url - , case when trim(both from ip_province)='' then null else ip_province::text end ip_province - , case when trim(both from device)='' then null else device::text end device -,etl_tx_dt -from ( -select - (json_array_elements(data::json)::json->>'date') date - , (json_array_elements(data::json)::json->>'screenWidth') screen_width - , (json_array_elements(data::json)::json->>'targetName') target_name - , (json_array_elements(data::json)::json->>'ipCity') ip_city - , (json_array_elements(data::json)::json->>'ipCountry') ip_country - , (json_array_elements(data::json)::json->>'platform') platform - , (json_array_elements(data::json)::json->>'lastUpdated') last_updated - , (json_array_elements(data::json)::json->>'pageType') page_type - , (json_array_elements(data::json)::json->>'osVersion') os_version - , (json_array_elements(data::json)::json->>'browser') browser - , (json_array_elements(data::json)::json->>'browserVersion') browser_version - , (json_array_elements(data::json)::json->>'customerId') customer_id - , (json_array_elements(data::json)::json->>'id') id - , (json_array_elements(data::json)::json->>'event') event - , (json_array_elements(data::json)::json->>'idStr') id_str - , (json_array_elements(data::json)::json->>'os') os - , (json_array_elements(data::json)::json->>'targetId') target_id - , (json_array_elements(data::json)::json->>'customerIdStr') customer_id_str - , (json_array_elements(data::json)::json->>'ip') ip - , (json_array_elements(data::json)::json->>'screenHeight') screen_height - , (json_array_elements(data::json)::json->>'externalId') external_id - , (json_array_elements(data::json)::json->>'ipCounty') ip_county - , (json_array_elements(data::json)::json->>'pageId') page_id - , (json_array_elements(data::json)::json->>'url') url - , (json_array_elements(data::json)::json->>'ipProvince') ip_province - , (json_array_elements(data::json)::json->>'device') device - ,CURRENT_TIMESTAMP(0) etl_tx_dt -from (select * from data_api.api_data -WHERE api_id='2433868c-3416-4475-884d-11e687a6' and is_loaded = '0' order by request_tm desc limit 1) p )p; - -update data_api.api_data -set is_loaded = '1' , - status = '1', - request_tm = current_timestamp(0) -where api_id='2433868c-3416-4475-884d-11e687a6'; -\q \ No newline at end of file diff --git a/TK_Cust/dev/tk_api/获取线索事件_open_content_page/custom_events_2_feign.py b/TK_Cust/dev/tk_api/获取线索事件_open_content_page/custom_events_2_feign.py deleted file mode 100644 index 0e2748c..0000000 --- a/TK_Cust/dev/tk_api/获取线索事件_open_content_page/custom_events_2_feign.py +++ /dev/null @@ -1,41 +0,0 @@ -# coding: utf-8 -import requests -import json -import psycopg2 -import uuid -import datetime - -# 获取当前日期 -current_date = datetime.date.today() -previous_date = current_date - datetime.timedelta(days=2) -formatted_current_date = current_date.strftime("%Y-%m-%dT%H:%M:%SZ") -formatted_previous_date = previous_date.strftime("%Y-%m-%dT%H:%M:%SZ") - -print('开始加载数据：custom_events_2:获取线索事件_open_content_page') -authUrl='https://api.huiju.cool/v2/oauth2/token?app_id=cl037b184ebccd97c&secret=deaa83fdd8d385eec0d1d04d34282c4ec8c33b60&grant_type=client_credentials' - -print('开始请求令牌。') -authRequest=requests.get(authUrl) -auth=json.loads(authRequest.text) -print('开始请求数据总数。') -url='https://api.huiju.cool/v2/customerEvents' -header={} -body={'access_token':auth['access_token'],'limit':'','event':'open_content_page','sort':'date','date[ge]':formatted_previous_date,'date[le]':formatted_current_date,} -dataReqL=requests.get(url,headers=header,params=body) -resL=json.loads(dataReqL.text) -# print(resL) -dataList=resL['data'] -total=len(dataList) -conn = psycopg2.connect(database="dataops_db", user="dbuser_dops", password="MIgTi3jA", - host="172.17.0.8", port="5432") -print('数据库连接成功') -dataId=str(uuid.uuid4()) -print('临时id：'+dataId) -json_object = json.dumps(dataList) -cur=conn.cursor() -sql="update data_api.api_data set is_loaded = '1' where api_id = '43edfb13-8a42-4152-9dc8-d5feb3c8';INSERT INTO data_api.api_data (id,api_id,data,total_num,is_loaded,status,request_tm,execute_tm,remark) values (%s,%s,%s,%s,'0','0',current_timestamp(0),current_timestamp(0),'')" -cur.execute(sql,[dataId,'43edfb13-8a42-4152-9dc8-d5feb3c8',json_object,total]) -conn.commit() -cur.close() -conn.close() -print('加载数据结束：custom_events_2:获取线索事件_open_content_page') \ No newline at end of file diff --git a/TK_Cust/dev/tk_api/获取线索事件_open_content_page/custom_events_2_load.sql b/TK_Cust/dev/tk_api/获取线索事件_open_content_page/custom_events_2_load.sql deleted file mode 100644 index 7234405..0000000 --- a/TK_Cust/dev/tk_api/获取线索事件_open_content_page/custom_events_2_load.sql +++ /dev/null @@ -1,102 +0,0 @@ -/*******Main Section**************************************************************************/ -\set ON_ERROR_STOP on -\set AUTOCOMMIT on -\timing on - -DELETE FROM data_api.customer_events_open_content_page; - -insert into data_api.customer_events_open_content_page ( - date - , short_id - , screen_width - , target_name - , ip_city - , ip_country - , platform - , last_updated - , page_type - , os_version - , browser - , browser_version - , customer_id - , id - , event - , id_str - , os - , target_id - , customer_id_str - , ip - , screen_height - , external_id - , page_id - , url - , ip_province - , device - ,etl_tx_dt -) -select - case when trim(both from date)='' then null else date::text end date - , case when trim(both from short_id)='' then null else short_id::text end short_id - , case when trim(both from screen_width)='' then null else screen_width::text end screen_width - , case when trim(both from target_name)='' then null else target_name::text end target_name - , case when trim(both from ip_city)='' then null else ip_city::text end ip_city - , case when trim(both from ip_country)='' then null else ip_country::text end ip_country - , case when trim(both from platform)='' then null else platform::text end platform - , case when trim(both from last_updated)='' then null else last_updated::text end last_updated - , case when trim(both from page_type)='' then null else page_type::text end page_type - , case when trim(both from os_version)='' then null else os_version::text end os_version - , case when trim(both from browser)='' then null else browser::text end browser - , case when trim(both from browser_version)='' then null else browser_version::text end browser_version - , case when trim(both from customer_id)='' then null else customer_id::text end customer_id - , case when trim(both from id)='' then null else id::text end id - , case when trim(both from event)='' then null else event::text end event - , case when trim(both from id_str)='' then null else id_str::text end id_str - , case when trim(both from os)='' then null else os::text end os - , case when trim(both from target_id)='' then null else target_id::text end target_id - , case when trim(both from customer_id_str)='' then null else customer_id_str::text end customer_id_str - , case when trim(both from ip)='' then null else ip::text end ip - , case when trim(both from screen_height)='' then null else screen_height::text end screen_height - , case when trim(both from external_id)='' then null else external_id::text end external_id - , case when trim(both from page_id)='' then null else page_id::text end page_id - , case when trim(both from url)='' then null else url::text end url - , case when trim(both from ip_province)='' then null else ip_province::text end ip_province - , case when trim(both from device)='' then null else device::text end device -,etl_tx_dt -from ( -select - (json_array_elements(data::json)::json->>'date') date - , (json_array_elements(data::json)::json->>'shortId') short_id - , (json_array_elements(data::json)::json->>'screenWidth') screen_width - , (json_array_elements(data::json)::json->>'targetName') target_name - , (json_array_elements(data::json)::json->>'ipCity') ip_city - , (json_array_elements(data::json)::json->>'ipCountry') ip_country - , (json_array_elements(data::json)::json->>'platform') platform - , (json_array_elements(data::json)::json->>'lastUpdated') last_updated - , (json_array_elements(data::json)::json->>'pageType') page_type - , (json_array_elements(data::json)::json->>'osVersion') os_version - , (json_array_elements(data::json)::json->>'browser') browser - , (json_array_elements(data::json)::json->>'browserVersion') browser_version - , (json_array_elements(data::json)::json->>'customerId') customer_id - , (json_array_elements(data::json)::json->>'id') id - , (json_array_elements(data::json)::json->>'event') event - , (json_array_elements(data::json)::json->>'idStr') id_str - , (json_array_elements(data::json)::json->>'os') os - , (json_array_elements(data::json)::json->>'targetId') target_id - , (json_array_elements(data::json)::json->>'customerIdStr') customer_id_str - , (json_array_elements(data::json)::json->>'ip') ip - , (json_array_elements(data::json)::json->>'screenHeight') screen_height - , (json_array_elements(data::json)::json->>'externalId') external_id - , (json_array_elements(data::json)::json->>'pageId') page_id - , (json_array_elements(data::json)::json->>'url') url - , (json_array_elements(data::json)::json->>'ipProvince') ip_province - , (json_array_elements(data::json)::json->>'device') device - ,CURRENT_TIMESTAMP(0) etl_tx_dt -from (select * from data_api.api_data -WHERE api_id='43edfb13-8a42-4152-9dc8-d5feb3c8' and is_loaded = '0' order by request_tm desc limit 1) p )p; - -update data_api.api_data -set is_loaded = '1' , - status = '1', - request_tm = current_timestamp(0) -where api_id='43edfb13-8a42-4152-9dc8-d5feb3c8'; -\q \ No newline at end of file diff --git a/TK_Cust/dev/tk_api/获取线索事件_open_page/custom_events_3_feign.py b/TK_Cust/dev/tk_api/获取线索事件_open_page/custom_events_3_feign.py deleted file mode 100644 index 3633c98..0000000 --- a/TK_Cust/dev/tk_api/获取线索事件_open_page/custom_events_3_feign.py +++ /dev/null @@ -1,41 +0,0 @@ -# coding: utf-8 -import requests -import json -import psycopg2 -import uuid -import datetime - -# 获取当前日期 -current_date = datetime.date.today() -previous_date = current_date - datetime.timedelta(days=2) -formatted_current_date = current_date.strftime("%Y-%m-%dT%H:%M:%SZ") -formatted_previous_date = previous_date.strftime("%Y-%m-%dT%H:%M:%SZ") - -print('开始加载数据：custom_events_3:获取线索事件_open_page') -authUrl='https://api.huiju.cool/v2/oauth2/token?app_id=cl037b184ebccd97c&secret=deaa83fdd8d385eec0d1d04d34282c4ec8c33b60&grant_type=client_credentials' - -print('开始请求令牌。') -authRequest=requests.get(authUrl) -auth=json.loads(authRequest.text) -print('开始请求数据总数。') -url='https://api.huiju.cool/v2/customerEvents' -header={} -body={'date[le]':formatted_current_date,'event':'open_page','access_token':auth['access_token'],'limit':'','sort':'date','date[ge]':formatted_previous_date,} -dataReqL=requests.get(url,headers=header,params=body) -resL=json.loads(dataReqL.text) -# print(resL) -dataList=resL['data'] -total=len(dataList) -conn = psycopg2.connect(database="dataops_db", user="dbuser_dops", password="MIgTi3jA", - host="172.17.0.8", port="5432") -print('数据库连接成功') -dataId=str(uuid.uuid4()) -print('临时id：'+dataId) -json_object = json.dumps(dataList) -cur=conn.cursor() -sql="update data_api.api_data set is_loaded = '1' where api_id = 'a2b284c3-322f-4bc0-89ff-414aa66a';INSERT INTO data_api.api_data (id,api_id,data,total_num,is_loaded,status,request_tm,execute_tm,remark) values (%s,%s,%s,%s,'0','0',current_timestamp(0),current_timestamp(0),'')" -cur.execute(sql,[dataId,'a2b284c3-322f-4bc0-89ff-414aa66a',json_object,total]) -conn.commit() -cur.close() -conn.close() -print('加载数据结束：custom_events_3:获取线索事件_open_page') \ No newline at end of file diff --git a/TK_Cust/dev/tk_api/获取线索事件_open_page/custom_events_3_load.sql b/TK_Cust/dev/tk_api/获取线索事件_open_page/custom_events_3_load.sql deleted file mode 100644 index a9427e3..0000000 --- a/TK_Cust/dev/tk_api/获取线索事件_open_page/custom_events_3_load.sql +++ /dev/null @@ -1,120 +0,0 @@ -/*******Main Section**************************************************************************/ -\set ON_ERROR_STOP on -\set AUTOCOMMIT on -\timing on - -DELETE FROM data_api.customer_events_open_page; - -insert into data_api.customer_events_open_page ( - date - , short_id - , screen_width - , target_name - , ip_city - , ip_country - , source1 - , source - , platform - , path - , score - , last_updated - , page_type - , os_version - , browser - , browser_version - , customer_id - , id - , event - , id_str - , os - , target_id - , customer_id_str - , ip - , screen_height - , root_domain - , external_id - , page_id - , url - , domain - , ip_province - , device - ,etl_tx_dt -) -select - case when trim(both from date)='' then null else date::text end date - , case when trim(both from short_id)='' then null else short_id::text end short_id - , case when trim(both from screen_width)='' then null else screen_width::text end screen_width - , case when trim(both from target_name)='' then null else target_name::text end target_name - , case when trim(both from ip_city)='' then null else ip_city::text end ip_city - , case when trim(both from ip_country)='' then null else ip_country::text end ip_country - , case when trim(both from source1)='' then null else source1::text end source1 - , case when trim(both from source)='' then null else source::text end source - , case when trim(both from platform)='' then null else platform::text end platform - , case when trim(both from path)='' then null else path::text end path - , case when trim(both from score)='' then null else score::text end score - , case when trim(both from last_updated)='' then null else last_updated::text end last_updated - , case when trim(both from page_type)='' then null else page_type::text end page_type - , case when trim(both from os_version)='' then null else os_version::text end os_version - , case when trim(both from browser)='' then null else browser::text end browser - , case when trim(both from browser_version)='' then null else browser_version::text end browser_version - , case when trim(both from customer_id)='' then null else customer_id::text end customer_id - , case when trim(both from id)='' then null else id::text end id - , case when trim(both from event)='' then null else event::text end event - , case when trim(both from id_str)='' then null else id_str::text end id_str - , case when trim(both from os)='' then null else os::text end os - , case when trim(both from target_id)='' then null else target_id::text end target_id - , case when trim(both from customer_id_str)='' then null else customer_id_str::text end customer_id_str - , case when trim(both from ip)='' then null else ip::text end ip - , case when trim(both from screen_height)='' then null else screen_height::text end screen_height - , case when trim(both from root_domain)='' then null else root_domain::text end root_domain - , case when trim(both from external_id)='' then null else external_id::text end external_id - , case when trim(both from page_id)='' then null else page_id::text end page_id - , case when trim(both from url)='' then null else url::text end url - , case when trim(both from domain)='' then null else domain::text end domain - , case when trim(both from ip_province)='' then null else ip_province::text end ip_province - , case when trim(both from device)='' then null else device::text end device -,etl_tx_dt -from ( -select - (json_array_elements(data::json)::json->>'date') date - , (json_array_elements(data::json)::json->>'shortId') short_id - , (json_array_elements(data::json)::json->>'screenWidth') screen_width - , (json_array_elements(data::json)::json->>'targetName') target_name - , (json_array_elements(data::json)::json->>'ipCity') ip_city - , (json_array_elements(data::json)::json->>'ipCountry') ip_country - , (json_array_elements(data::json)::json->>'source1') source1 - , (json_array_elements(data::json)::json->>'source') source - , (json_array_elements(data::json)::json->>'platform') platform - , (json_array_elements(data::json)::json->>'path') path - , (json_array_elements(data::json)::json->>'score') score - , (json_array_elements(data::json)::json->>'lastUpdated') last_updated - , (json_array_elements(data::json)::json->>'pageType') page_type - , (json_array_elements(data::json)::json->>'osVersion') os_version - , (json_array_elements(data::json)::json->>'browser') browser - , (json_array_elements(data::json)::json->>'browserVersion') browser_version - , (json_array_elements(data::json)::json->>'customerId') customer_id - , (json_array_elements(data::json)::json->>'id') id - , (json_array_elements(data::json)::json->>'event') event - , (json_array_elements(data::json)::json->>'idStr') id_str - , (json_array_elements(data::json)::json->>'os') os - , (json_array_elements(data::json)::json->>'targetId') target_id - , (json_array_elements(data::json)::json->>'customerIdStr') customer_id_str - , (json_array_elements(data::json)::json->>'ip') ip - , (json_array_elements(data::json)::json->>'screenHeight') screen_height - , (json_array_elements(data::json)::json->>'rootDomain') root_domain - , (json_array_elements(data::json)::json->>'externalId') external_id - , (json_array_elements(data::json)::json->>'pageId') page_id - , (json_array_elements(data::json)::json->>'url') url - , (json_array_elements(data::json)::json->>'domain') domain - , (json_array_elements(data::json)::json->>'ipProvince') ip_province - , (json_array_elements(data::json)::json->>'device') device - ,CURRENT_TIMESTAMP(0) etl_tx_dt -from (select * from data_api.api_data -WHERE api_id='a2b284c3-322f-4bc0-89ff-414aa66a' and is_loaded = '0' order by request_tm desc limit 1) p )p; - -update data_api.api_data -set is_loaded = '1' , - status = '1', - request_tm = current_timestamp(0) -where api_id='a2b284c3-322f-4bc0-89ff-414aa66a'; -\q \ No newline at end of file diff --git a/TK_Cust/dev/tk_api/获取线索事件_submit_form/custom_events_6_feign.py b/TK_Cust/dev/tk_api/获取线索事件_submit_form/custom_events_6_feign.py deleted file mode 100644 index 728c04e..0000000 --- a/TK_Cust/dev/tk_api/获取线索事件_submit_form/custom_events_6_feign.py +++ /dev/null @@ -1,41 +0,0 @@ -# coding: utf-8 -import requests -import json -import psycopg2 -import uuid -import datetime - -# 获取当前日期 -current_date = datetime.date.today() -previous_date = current_date - datetime.timedelta(days=2) -formatted_current_date = current_date.strftime("%Y-%m-%dT%H:%M:%SZ") -formatted_previous_date = previous_date.strftime("%Y-%m-%dT%H:%M:%SZ") - -print('开始加载数据：custom_events_6:获取线索事件_submit_form') -authUrl='https://api.huiju.cool/v2/oauth2/token?app_id=cl037b184ebccd97c&secret=deaa83fdd8d385eec0d1d04d34282c4ec8c33b60&grant_type=client_credentials' - -print('开始请求令牌。') -authRequest=requests.get(authUrl) -auth=json.loads(authRequest.text) -print('开始请求数据总数。') -url='https://api.huiju.cool/v2/customerEvents' -header={} -body={'access_token':auth['access_token'],'event':'submit_form','limit':'','sort':'date','date[ge]':formatted_previous_date,'date[le]':formatted_current_date,} -dataReqL=requests.get(url,headers=header,params=body) -resL=json.loads(dataReqL.text) -# print(resL) -dataList=resL['data'] -total=len(dataList) -conn = psycopg2.connect(database="dataops_db", user="dbuser_dops", password="MIgTi3jA", - host="172.17.0.8", port="5432") -print('数据库连接成功') -dataId=str(uuid.uuid4()) -print('临时id：'+dataId) -json_object = json.dumps(dataList) -cur=conn.cursor() -sql="update data_api.api_data set is_loaded = '1' where api_id = 'c6e5b45f-35f5-442b-84be-35ff2e76';INSERT INTO data_api.api_data (id,api_id,data,total_num,is_loaded,status,request_tm,execute_tm,remark) values (%s,%s,%s,%s,'0','0',current_timestamp(0),current_timestamp(0),'')" -cur.execute(sql,[dataId,'c6e5b45f-35f5-442b-84be-35ff2e76',json_object,total]) -conn.commit() -cur.close() -conn.close() -print('加载数据结束：custom_events_6:获取线索事件_submit_form') \ No newline at end of file diff --git a/TK_Cust/dev/tk_api/获取线索事件_submit_form/custom_events_6_load.sql b/TK_Cust/dev/tk_api/获取线索事件_submit_form/custom_events_6_load.sql deleted file mode 100644 index c9b2c1a..0000000 --- a/TK_Cust/dev/tk_api/获取线索事件_submit_form/custom_events_6_load.sql +++ /dev/null @@ -1,96 +0,0 @@ -/*******Main Section**************************************************************************/ -\set ON_ERROR_STOP on -\set AUTOCOMMIT on -\timing on - -DELETE FROM data_api.customer_events_submit_form; - -insert into data_api.customer_events_submit_form ( - id_str - , date - , target_name - , os - , target_id - , customer_id_str - , ip_city - , ip - , external_id - , ip_country - , ip_county - , page_id - , platform - , score - , last_updated - , page_type - , os_version - , browser - , browser_version - , customer_id - , ip_province - , id - , event - , submit_id - ,etl_tx_dt -) -select - case when trim(both from id_str)='' then null else id_str::text end id_str - , case when trim(both from date)='' then null else date::text end date - , case when trim(both from target_name)='' then null else target_name::text end target_name - , case when trim(both from os)='' then null else os::text end os - , case when trim(both from target_id)='' then null else target_id::text end target_id - , case when trim(both from customer_id_str)='' then null else customer_id_str::text end customer_id_str - , case when trim(both from ip_city)='' then null else ip_city::text end ip_city - , case when trim(both from ip)='' then null else ip::text end ip - , case when trim(both from external_id)='' then null else external_id::text end external_id - , case when trim(both from ip_country)='' then null else ip_country::text end ip_country - , case when trim(both from ip_county)='' then null else ip_county::text end ip_county - , case when trim(both from page_id)='' then null else page_id::text end page_id - , case when trim(both from platform)='' then null else platform::text end platform - , case when trim(both from score)='' then null else score::text end score - , case when trim(both from last_updated)='' then null else last_updated::text end last_updated - , case when trim(both from page_type)='' then null else page_type::text end page_type - , case when trim(both from os_version)='' then null else os_version::text end os_version - , case when trim(both from browser)='' then null else browser::text end browser - , case when trim(both from browser_version)='' then null else browser_version::text end browser_version - , case when trim(both from customer_id)='' then null else customer_id::text end customer_id - , case when trim(both from ip_province)='' then null else ip_province::text end ip_province - , case when trim(both from id)='' then null else id::text end id - , case when trim(both from event)='' then null else event::text end event - , case when trim(both from submit_id)='' then null else submit_id::text end submit_id -,etl_tx_dt -from ( -select - (json_array_elements(data::json)::json->>'idStr') id_str - , (json_array_elements(data::json)::json->>'date') date - , (json_array_elements(data::json)::json->>'targetName') target_name - , (json_array_elements(data::json)::json->>'os') os - , (json_array_elements(data::json)::json->>'targetId') target_id - , (json_array_elements(data::json)::json->>'customerIdStr') customer_id_str - , (json_array_elements(data::json)::json->>'ipCity') ip_city - , (json_array_elements(data::json)::json->>'ip') ip - , (json_array_elements(data::json)::json->>'externalId') external_id - , (json_array_elements(data::json)::json->>'ipCountry') ip_country - , (json_array_elements(data::json)::json->>'ipCounty') ip_county - , (json_array_elements(data::json)::json->>'pageId') page_id - , (json_array_elements(data::json)::json->>'platform') platform - , (json_array_elements(data::json)::json->>'score') score - , (json_array_elements(data::json)::json->>'lastUpdated') last_updated - , (json_array_elements(data::json)::json->>'pageType') page_type - , (json_array_elements(data::json)::json->>'osVersion') os_version - , (json_array_elements(data::json)::json->>'browser') browser - , (json_array_elements(data::json)::json->>'browserVersion') browser_version - , (json_array_elements(data::json)::json->>'customerId') customer_id - , (json_array_elements(data::json)::json->>'ipProvince') ip_province - , (json_array_elements(data::json)::json->>'id') id - , (json_array_elements(data::json)::json->>'event') event - , (json_array_elements(data::json)::json->>'submitId') submit_id - ,CURRENT_TIMESTAMP(0) etl_tx_dt -from (select * from data_api.api_data -WHERE api_id='c6e5b45f-35f5-442b-84be-35ff2e76' and is_loaded = '0' order by request_tm desc limit 1) p )p; - -update data_api.api_data -set is_loaded = '1' , - status = '1', - request_tm = current_timestamp(0) -where api_id='c6e5b45f-35f5-442b-84be-35ff2e76'; -\q \ No newline at end of file diff --git a/TK_Cust/dev/tk_api/获取线索事件_subscribe/custom_events_4_feign.py b/TK_Cust/dev/tk_api/获取线索事件_subscribe/custom_events_4_feign.py deleted file mode 100644 index c369560..0000000 --- a/TK_Cust/dev/tk_api/获取线索事件_subscribe/custom_events_4_feign.py +++ /dev/null @@ -1,41 +0,0 @@ -# coding: utf-8 -import requests -import json -import psycopg2 -import uuid -import datetime - -# 获取当前日期 -current_date = datetime.date.today() -previous_date = current_date - datetime.timedelta(days=2) -formatted_current_date = current_date.strftime("%Y-%m-%dT%H:%M:%SZ") -formatted_previous_date = previous_date.strftime("%Y-%m-%dT%H:%M:%SZ") - -print('开始加载数据：custom_events_4:获取线索事件_subscribe') -authUrl='https://api.huiju.cool/v2/oauth2/token?app_id=cl037b184ebccd97c&secret=deaa83fdd8d385eec0d1d04d34282c4ec8c33b60&grant_type=client_credentials' - -print('开始请求令牌。') -authRequest=requests.get(authUrl) -auth=json.loads(authRequest.text) -print('开始请求数据总数。') -url='https://api.huiju.cool/v2/customerEvents' -header={} -body={'access_token':auth['access_token'],'limit':'','event':'subscribe','sort':'date','date[ge]':formatted_previous_date,'date[le]':formatted_current_date,} -dataReqL=requests.get(url,headers=header,params=body) -resL=json.loads(dataReqL.text) -# print(resL) -dataList=resL['data'] -total=len(dataList) -conn = psycopg2.connect(database="dataops_db", user="dbuser_dops", password="MIgTi3jA", - host="172.17.0.8", port="5432") -print('数据库连接成功') -dataId=str(uuid.uuid4()) -print('临时id：'+dataId) -json_object = json.dumps(dataList) -cur=conn.cursor() -sql="update data_api.api_data set is_loaded = '1' where api_id = '36308871-eea2-49b0-8268-03ab68e3';INSERT INTO data_api.api_data (id,api_id,data,total_num,is_loaded,status,request_tm,execute_tm,remark) values (%s,%s,%s,%s,'0','0',current_timestamp(0),current_timestamp(0),'')" -cur.execute(sql,[dataId,'36308871-eea2-49b0-8268-03ab68e3',json_object,total]) -conn.commit() -cur.close() -conn.close() -print('加载数据结束：custom_events_4:获取线索事件_subscribe') \ No newline at end of file diff --git a/TK_Cust/dev/tk_api/获取线索事件_subscribe/custom_events_4_load.sql b/TK_Cust/dev/tk_api/获取线索事件_subscribe/custom_events_4_load.sql deleted file mode 100644 index da68d3a..0000000 --- a/TK_Cust/dev/tk_api/获取线索事件_subscribe/custom_events_4_load.sql +++ /dev/null @@ -1,60 +0,0 @@ -/*******Main Section**************************************************************************/ -\set ON_ERROR_STOP on -\set AUTOCOMMIT on -\timing on - -DELETE FROM data_api.customer_events_subscribe; - -insert into data_api.customer_events_subscribe ( - id_str - , date - , score - , last_updated - , target_name - , target_id - , customer_id_str - , channel_account - , customer_id - , external_id - , id - , event - ,etl_tx_dt -) -select - case when trim(both from id_str)='' then null else id_str::text end id_str - , case when trim(both from date)='' then null else date::text end date - , case when trim(both from score)='' then null else score::text end score - , case when trim(both from last_updated)='' then null else last_updated::text end last_updated - , case when trim(both from target_name)='' then null else target_name::text end target_name - , case when trim(both from target_id)='' then null else target_id::text end target_id - , case when trim(both from customer_id_str)='' then null else customer_id_str::text end customer_id_str - , case when trim(both from channel_account)='' then null else channel_account::text end channel_account - , case when trim(both from customer_id)='' then null else customer_id::text end customer_id - , case when trim(both from external_id)='' then null else external_id::text end external_id - , case when trim(both from id)='' then null else id::text end id - , case when trim(both from event)='' then null else event::text end event -,etl_tx_dt -from ( -select - (json_array_elements(data::json)::json->>'idStr') id_str - , (json_array_elements(data::json)::json->>'date') date - , (json_array_elements(data::json)::json->>'score') score - , (json_array_elements(data::json)::json->>'lastUpdated') last_updated - , (json_array_elements(data::json)::json->>'targetName') target_name - , (json_array_elements(data::json)::json->>'targetId') target_id - , (json_array_elements(data::json)::json->>'customerIdStr') customer_id_str - , (json_array_elements(data::json)::json->>'channelAccount') channel_account - , (json_array_elements(data::json)::json->>'customerId') customer_id - , (json_array_elements(data::json)::json->>'externalId') external_id - , (json_array_elements(data::json)::json->>'id') id - , (json_array_elements(data::json)::json->>'event') event - ,CURRENT_TIMESTAMP(0) etl_tx_dt -from (select * from data_api.api_data -WHERE api_id='36308871-eea2-49b0-8268-03ab68e3' and is_loaded = '0' order by request_tm desc limit 1) p )p; - -update data_api.api_data -set is_loaded = '1' , - status = '1', - request_tm = current_timestamp(0) -where api_id='36308871-eea2-49b0-8268-03ab68e3'; -\q \ No newline at end of file diff --git a/TK_Cust/dev/tk_api/获取线索事件_wechat_scan/custom_events_9_feign.py b/TK_Cust/dev/tk_api/获取线索事件_wechat_scan/custom_events_9_feign.py deleted file mode 100644 index bc182a6..0000000 --- a/TK_Cust/dev/tk_api/获取线索事件_wechat_scan/custom_events_9_feign.py +++ /dev/null @@ -1,41 +0,0 @@ -# coding: utf-8 -import requests -import json -import psycopg2 -import uuid -import datetime - -# 获取当前日期 -current_date = datetime.date.today() -previous_date = current_date - datetime.timedelta(days=2) -formatted_current_date = current_date.strftime("%Y-%m-%dT%H:%M:%SZ") -formatted_previous_date = previous_date.strftime("%Y-%m-%dT%H:%M:%SZ") - -print('开始加载数据：custom_events_9:获取线索事件_wechat_scan') -authUrl='https://api.huiju.cool/v2/oauth2/token?app_id=cl037b184ebccd97c&secret=deaa83fdd8d385eec0d1d04d34282c4ec8c33b60&grant_type=client_credentials' - -print('开始请求令牌。') -authRequest=requests.get(authUrl) -auth=json.loads(authRequest.text) -print('开始请求数据总数。') -url='https://api.huiju.cool/v2/customerEvents' -header={} -body={'access_token':auth['access_token'],'limit':'','sort':'date','event':'wechat_scan','date[le]':formatted_current_date,'date[ge]':formatted_previous_date,} -dataReqL=requests.get(url,headers=header,params=body) -resL=json.loads(dataReqL.text) -# print(resL) -dataList=resL['data'] -total=len(dataList) -conn = psycopg2.connect(database="dataops_db", user="dbuser_dops", password="MIgTi3jA", - host="172.17.0.8", port="5432") -print('数据库连接成功') -dataId=str(uuid.uuid4()) -print('临时id：'+dataId) -json_object = json.dumps(dataList) -cur=conn.cursor() -sql="update data_api.api_data set is_loaded = '1' where api_id = '14f55a0e-1c01-4bd5-9eec-7e0d0bdf';INSERT INTO data_api.api_data (id,api_id,data,total_num,is_loaded,status,request_tm,execute_tm,remark) values (%s,%s,%s,%s,'0','0',current_timestamp(0),current_timestamp(0),'')" -cur.execute(sql,[dataId,'14f55a0e-1c01-4bd5-9eec-7e0d0bdf',json_object,total]) -conn.commit() -cur.close() -conn.close() -print('加载数据结束：custom_events_9:获取线索事件_wechat_scan') \ No newline at end of file diff --git a/TK_Cust/dev/tk_api/获取线索事件_wechat_scan/custom_events_9_load.sql b/TK_Cust/dev/tk_api/获取线索事件_wechat_scan/custom_events_9_load.sql deleted file mode 100644 index 0d87b50..0000000 --- a/TK_Cust/dev/tk_api/获取线索事件_wechat_scan/custom_events_9_load.sql +++ /dev/null @@ -1,60 +0,0 @@ -/*******Main Section**************************************************************************/ -\set ON_ERROR_STOP on -\set AUTOCOMMIT on -\timing on - -DELETE FROM data_api.customer_events_wechat_scan; - -insert into data_api.customer_events_wechat_scan ( - id_str - , date - , score - , last_updated - , target_name - , target_id - , customer_id_str - , channel_account - , customer_id - , external_id - , id - , event - ,etl_tx_dt -) -select - case when trim(both from id_str)='' then null else id_str::text end id_str - , case when trim(both from date)='' then null else date::text end date - , case when trim(both from score)='' then null else score::text end score - , case when trim(both from last_updated)='' then null else last_updated::text end last_updated - , case when trim(both from target_name)='' then null else target_name::text end target_name - , case when trim(both from target_id)='' then null else target_id::text end target_id - , case when trim(both from customer_id_str)='' then null else customer_id_str::text end customer_id_str - , case when trim(both from channel_account)='' then null else channel_account::text end channel_account - , case when trim(both from customer_id)='' then null else customer_id::text end customer_id - , case when trim(both from external_id)='' then null else external_id::text end external_id - , case when trim(both from id)='' then null else id::text end id - , case when trim(both from event)='' then null else event::text end event -,etl_tx_dt -from ( -select - (json_array_elements(data::json)::json->>'idStr') id_str - , (json_array_elements(data::json)::json->>'date') date - , (json_array_elements(data::json)::json->>'score') score - , (json_array_elements(data::json)::json->>'lastUpdated') last_updated - , (json_array_elements(data::json)::json->>'targetName') target_name - , (json_array_elements(data::json)::json->>'targetId') target_id - , (json_array_elements(data::json)::json->>'customerIdStr') customer_id_str - , (json_array_elements(data::json)::json->>'channelAccount') channel_account - , (json_array_elements(data::json)::json->>'customerId') customer_id - , (json_array_elements(data::json)::json->>'externalId') external_id - , (json_array_elements(data::json)::json->>'id') id - , (json_array_elements(data::json)::json->>'event') event - ,CURRENT_TIMESTAMP(0) etl_tx_dt -from (select * from data_api.api_data -WHERE api_id='14f55a0e-1c01-4bd5-9eec-7e0d0bdf' and is_loaded = '0' order by request_tm desc limit 1) p )p; - -update data_api.api_data -set is_loaded = '1' , - status = '1', - request_tm = current_timestamp(0) -where api_id='14f55a0e-1c01-4bd5-9eec-7e0d0bdf'; -\q \ No newline at end of file diff --git a/TK_Cust/dev/tk_api/获取线索事件元数据/customer_event_meta_feign.py b/TK_Cust/dev/tk_api/获取线索事件元数据/customer_event_meta_feign.py deleted file mode 100644 index 9029921..0000000 --- a/TK_Cust/dev/tk_api/获取线索事件元数据/customer_event_meta_feign.py +++ /dev/null @@ -1,41 +0,0 @@ -# coding: utf-8 -import requests -import json -import psycopg2 -import uuid -import datetime - -# 获取当前日期 -current_date = datetime.date.today() -previous_date = current_date - datetime.timedelta(days=2) -formatted_current_date = current_date.strftime("%Y-%m-%dT%H:%M:%SZ") -formatted_previous_date = previous_date.strftime("%Y-%m-%dT%H:%M:%SZ") - -print('开始加载数据：customer_event_meta:获取线索事件元数据') -authUrl='https://api.huiju.cool/v2/oauth2/token?app_id=cl037b184ebccd97c&secret=deaa83fdd8d385eec0d1d04d34282c4ec8c33b60&grant_type=client_credentials' - -print('开始请求令牌。') -authRequest=requests.get(authUrl) -auth=json.loads(authRequest.text) -print('开始请求数据总数。') -url='https://api.huiju.cool/v2/metaService/getCustomerEventMeta' -header={} -body={'access_token':auth['access_token'],} -dataReqL=requests.get(url,headers=header,params=body) -resL=json.loads(dataReqL.text) -# print(resL) -dataList=resL['data'] -total=len(dataList) -conn = psycopg2.connect(database="dataops_db", user="dbuser_dops", password="MIgTi3jA", - host="172.17.0.8", port="5432") -print('数据库连接成功') -dataId=str(uuid.uuid4()) -print('临时id：'+dataId) -json_object = json.dumps(dataList) -cur=conn.cursor() -sql="update data_api.api_data set is_loaded = '1' where api_id = '057169de-4aa7-4f73-8894-221b4fd0';INSERT INTO data_api.api_data (id,api_id,data,total_num,is_loaded,status,request_tm,execute_tm,remark) values (%s,%s,%s,%s,'0','0',current_timestamp(0),current_timestamp(0),'')" -cur.execute(sql,[dataId,'057169de-4aa7-4f73-8894-221b4fd0',json_object,total]) -conn.commit() -cur.close() -conn.close() -print('加载数据结束：customer_event_meta:获取线索事件元数据') \ No newline at end of file diff --git a/TK_Cust/dev/tk_api/获取线索事件元数据/customer_event_meta_load.sql b/TK_Cust/dev/tk_api/获取线索事件元数据/customer_event_meta_load.sql deleted file mode 100644 index fb99ba0..0000000 --- a/TK_Cust/dev/tk_api/获取线索事件元数据/customer_event_meta_load.sql +++ /dev/null @@ -1,33 +0,0 @@ -/*******Main Section**************************************************************************/ -\set ON_ERROR_STOP on -\set AUTOCOMMIT on -\timing on - -DELETE FROM data_api.customer_event_meta; - -insert into data_api.customer_event_meta ( - group_name - , group_label - , events - ,etl_tx_dt -) -select - case when trim(both from group_name)='' then null else group_name::text end group_name - , case when trim(both from group_label)='' then null else group_label::text end group_label - , case when trim(both from events)='' then null else events::text end events -,etl_tx_dt -from ( -select - (json_array_elements(data::json)::json->>'groupName') group_name - , (json_array_elements(data::json)::json->>'groupLabel') group_label - , (json_array_elements(data::json)::json->>'events') events - ,CURRENT_TIMESTAMP(0) etl_tx_dt -from (select * from data_api.api_data -WHERE api_id='057169de-4aa7-4f73-8894-221b4fd0' and is_loaded = '0' order by request_tm desc limit 1) p )p; - -update data_api.api_data -set is_loaded = '1' , - status = '1', - request_tm = current_timestamp(0) -where api_id='057169de-4aa7-4f73-8894-221b4fd0'; -\q \ No newline at end of file