add workflow 荟聚API_2,dev
This commit is contained in:
root
parent
a5c4730a86
commit
2396efdf98
|
|
@ -0,0 +1,34 @@
|
|||
/*******Main Section**************************************************************************/
|
||||
\set ON_ERROR_STOP on
|
||||
\set AUTOCOMMIT on
|
||||
\timing on
|
||||
delete from p10_sa.S98_S_customer_event_meta
|
||||
;
|
||||
insert into p10_sa.S98_S_customer_event_meta
|
||||
( events
|
||||
, group_label
|
||||
, group_name
|
||||
, etl_tx_dt )
|
||||
select
|
||||
events
|
||||
, group_label
|
||||
, group_name
|
||||
, etl_tx_dt
|
||||
from p00_tal.S98_S_customer_event_meta
|
||||
;
|
||||
delete from p12_sfull.S98_S_customer_event_meta
|
||||
;
|
||||
;
|
||||
insert into p12_sfull.S98_S_customer_event_meta
|
||||
( events
|
||||
, group_label
|
||||
, group_name
|
||||
, etl_tx_dt )
|
||||
select
|
||||
events
|
||||
, group_label
|
||||
, group_name
|
||||
, etl_tx_dt
|
||||
from p10_sa.S98_S_customer_event_meta
|
||||
;
|
||||
\q
|
||||
|
|
@ -0,0 +1,15 @@
|
|||
|
||||
CREATE FOREIGN TABLE if not exists p00_tal.S98_S_customer_event_meta (
|
||||
events TEXT
|
||||
, group_label TEXT
|
||||
, group_name TEXT
|
||||
, etl_tx_dt TIMESTAMP
|
||||
)
|
||||
|
||||
|
||||
SERVER pgsql_server_S98_S OPTIONS(schema_name 'data_api', table_name 'customer_event_meta' );
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
|
@ -0,0 +1,31 @@
|
|||
|
||||
create table if not exists p10_sa.S98_S_customer_event_meta (
|
||||
events TEXT
|
||||
, group_label TEXT
|
||||
, group_name TEXT
|
||||
, etl_tx_dt TIMESTAMP
|
||||
) ;
|
||||
|
||||
COMMENT ON COLUMN p10_sa.S98_S_customer_event_meta.events IS '';
|
||||
COMMENT ON COLUMN p10_sa.S98_S_customer_event_meta.group_label IS '';
|
||||
COMMENT ON COLUMN p10_sa.S98_S_customer_event_meta.group_name IS '';
|
||||
COMMENT ON COLUMN p10_sa.S98_S_customer_event_meta.etl_tx_dt IS '';
|
||||
|
||||
COMMENT ON TABLE p10_sa.S98_S_customer_event_meta IS '';
|
||||
|
||||
|
||||
|
||||
create table if not exists p12_sfull.S98_S_customer_event_meta (
|
||||
events TEXT
|
||||
, group_label TEXT
|
||||
, group_name TEXT
|
||||
, etl_tx_dt TIMESTAMP
|
||||
) ;
|
||||
|
||||
COMMENT ON COLUMN p12_sfull.S98_S_customer_event_meta.events IS '';
|
||||
COMMENT ON COLUMN p12_sfull.S98_S_customer_event_meta.group_label IS '';
|
||||
COMMENT ON COLUMN p12_sfull.S98_S_customer_event_meta.group_name IS '';
|
||||
COMMENT ON COLUMN p12_sfull.S98_S_customer_event_meta.etl_tx_dt IS '';
|
||||
|
||||
COMMENT ON TABLE p12_sfull.S98_S_customer_event_meta IS '';
|
||||
|
||||
|
|
@ -0,0 +1,66 @@
|
|||
/*******Main Section**************************************************************************/
|
||||
\set ON_ERROR_STOP on
|
||||
\set AUTOCOMMIT on
|
||||
\timing on
|
||||
delete from p10_sa.S98_S_customer_events
|
||||
;
|
||||
insert into p10_sa.S98_S_customer_events
|
||||
( id
|
||||
, customer_id
|
||||
, event
|
||||
, last_updated
|
||||
, source
|
||||
, content_name
|
||||
, date
|
||||
, create_method
|
||||
, customer_id_str
|
||||
, external_id
|
||||
, id_str
|
||||
, etl_tx_dt )
|
||||
select
|
||||
id
|
||||
, customer_id
|
||||
, event
|
||||
, last_updated
|
||||
, source
|
||||
, content_name
|
||||
, date
|
||||
, create_method
|
||||
, customer_id_str
|
||||
, external_id
|
||||
, id_str
|
||||
, etl_tx_dt
|
||||
from p00_tal.S98_S_customer_events
|
||||
;
|
||||
delete from p12_sfull.S98_S_customer_events
|
||||
;
|
||||
;
|
||||
insert into p12_sfull.S98_S_customer_events
|
||||
( id
|
||||
, customer_id
|
||||
, event
|
||||
, last_updated
|
||||
, source
|
||||
, content_name
|
||||
, date
|
||||
, create_method
|
||||
, customer_id_str
|
||||
, external_id
|
||||
, id_str
|
||||
, etl_tx_dt )
|
||||
select
|
||||
id
|
||||
, customer_id
|
||||
, event
|
||||
, last_updated
|
||||
, source
|
||||
, content_name
|
||||
, date
|
||||
, create_method
|
||||
, customer_id_str
|
||||
, external_id
|
||||
, id_str
|
||||
, etl_tx_dt
|
||||
from p10_sa.S98_S_customer_events
|
||||
;
|
||||
\q
|
||||
|
|
@ -0,0 +1,23 @@
|
|||
|
||||
CREATE FOREIGN TABLE if not exists p00_tal.S98_S_customer_events (
|
||||
id TEXT
|
||||
, customer_id TEXT
|
||||
, event TEXT
|
||||
, last_updated TEXT
|
||||
, source TEXT
|
||||
, content_name TEXT
|
||||
, date TEXT
|
||||
, create_method TEXT
|
||||
, customer_id_str TEXT
|
||||
, external_id TEXT
|
||||
, id_str TEXT
|
||||
, etl_tx_dt TIMESTAMP
|
||||
)
|
||||
|
||||
|
||||
SERVER pgsql_server_S98_S OPTIONS(schema_name 'data_api', table_name 'customer_events' );
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
|
@ -0,0 +1,63 @@
|
|||
|
||||
create table if not exists p10_sa.S98_S_customer_events (
|
||||
id TEXT
|
||||
, customer_id TEXT
|
||||
, event TEXT
|
||||
, last_updated TEXT
|
||||
, source TEXT
|
||||
, content_name TEXT
|
||||
, date TEXT
|
||||
, create_method TEXT
|
||||
, customer_id_str TEXT
|
||||
, external_id TEXT
|
||||
, id_str TEXT
|
||||
, etl_tx_dt TIMESTAMP
|
||||
) ;
|
||||
|
||||
COMMENT ON COLUMN p10_sa.S98_S_customer_events.id IS '';
|
||||
COMMENT ON COLUMN p10_sa.S98_S_customer_events.customer_id IS '';
|
||||
COMMENT ON COLUMN p10_sa.S98_S_customer_events.event IS '';
|
||||
COMMENT ON COLUMN p10_sa.S98_S_customer_events.last_updated IS '';
|
||||
COMMENT ON COLUMN p10_sa.S98_S_customer_events.source IS '';
|
||||
COMMENT ON COLUMN p10_sa.S98_S_customer_events.content_name IS '';
|
||||
COMMENT ON COLUMN p10_sa.S98_S_customer_events.date IS '';
|
||||
COMMENT ON COLUMN p10_sa.S98_S_customer_events.create_method IS '';
|
||||
COMMENT ON COLUMN p10_sa.S98_S_customer_events.customer_id_str IS '';
|
||||
COMMENT ON COLUMN p10_sa.S98_S_customer_events.external_id IS '';
|
||||
COMMENT ON COLUMN p10_sa.S98_S_customer_events.id_str IS '';
|
||||
COMMENT ON COLUMN p10_sa.S98_S_customer_events.etl_tx_dt IS '';
|
||||
|
||||
COMMENT ON TABLE p10_sa.S98_S_customer_events IS '';
|
||||
|
||||
|
||||
|
||||
create table if not exists p12_sfull.S98_S_customer_events (
|
||||
id TEXT
|
||||
, customer_id TEXT
|
||||
, event TEXT
|
||||
, last_updated TEXT
|
||||
, source TEXT
|
||||
, content_name TEXT
|
||||
, date TEXT
|
||||
, create_method TEXT
|
||||
, customer_id_str TEXT
|
||||
, external_id TEXT
|
||||
, id_str TEXT
|
||||
, etl_tx_dt TIMESTAMP
|
||||
) ;
|
||||
|
||||
COMMENT ON COLUMN p12_sfull.S98_S_customer_events.id IS '';
|
||||
COMMENT ON COLUMN p12_sfull.S98_S_customer_events.customer_id IS '';
|
||||
COMMENT ON COLUMN p12_sfull.S98_S_customer_events.event IS '';
|
||||
COMMENT ON COLUMN p12_sfull.S98_S_customer_events.last_updated IS '';
|
||||
COMMENT ON COLUMN p12_sfull.S98_S_customer_events.source IS '';
|
||||
COMMENT ON COLUMN p12_sfull.S98_S_customer_events.content_name IS '';
|
||||
COMMENT ON COLUMN p12_sfull.S98_S_customer_events.date IS '';
|
||||
COMMENT ON COLUMN p12_sfull.S98_S_customer_events.create_method IS '';
|
||||
COMMENT ON COLUMN p12_sfull.S98_S_customer_events.customer_id_str IS '';
|
||||
COMMENT ON COLUMN p12_sfull.S98_S_customer_events.external_id IS '';
|
||||
COMMENT ON COLUMN p12_sfull.S98_S_customer_events.id_str IS '';
|
||||
COMMENT ON COLUMN p12_sfull.S98_S_customer_events.etl_tx_dt IS '';
|
||||
|
||||
COMMENT ON TABLE p12_sfull.S98_S_customer_events IS '';
|
||||
|
||||
|
|
@ -0,0 +1,126 @@
|
|||
/*******Main Section**************************************************************************/
|
||||
\set ON_ERROR_STOP on
|
||||
\set AUTOCOMMIT on
|
||||
\timing on
|
||||
delete from p10_sa.S98_S_customer_events_open_content_page
|
||||
;
|
||||
insert into p10_sa.S98_S_customer_events_open_content_page
|
||||
( id
|
||||
, customer_id
|
||||
, event
|
||||
, last_updated
|
||||
, date
|
||||
, browser
|
||||
, browser_version
|
||||
, customer_id_str
|
||||
, device
|
||||
, external_id
|
||||
, id_str
|
||||
, ip
|
||||
, ip_city
|
||||
, ip_country
|
||||
, ip_province
|
||||
, os
|
||||
, os_version
|
||||
, page_id
|
||||
, page_type
|
||||
, platform
|
||||
, screen_height
|
||||
, screen_width
|
||||
, short_id
|
||||
, target_id
|
||||
, target_name
|
||||
, url
|
||||
, etl_tx_dt )
|
||||
select
|
||||
id
|
||||
, customer_id
|
||||
, event
|
||||
, last_updated
|
||||
, date
|
||||
, browser
|
||||
, browser_version
|
||||
, customer_id_str
|
||||
, device
|
||||
, external_id
|
||||
, id_str
|
||||
, ip
|
||||
, ip_city
|
||||
, ip_country
|
||||
, ip_province
|
||||
, os
|
||||
, os_version
|
||||
, page_id
|
||||
, page_type
|
||||
, platform
|
||||
, screen_height
|
||||
, screen_width
|
||||
, short_id
|
||||
, target_id
|
||||
, target_name
|
||||
, url
|
||||
, etl_tx_dt
|
||||
from p00_tal.S98_S_customer_events_open_content_page
|
||||
;
|
||||
delete from p12_sfull.S98_S_customer_events_open_content_page
|
||||
;
|
||||
;
|
||||
insert into p12_sfull.S98_S_customer_events_open_content_page
|
||||
( id
|
||||
, customer_id
|
||||
, event
|
||||
, last_updated
|
||||
, date
|
||||
, browser
|
||||
, browser_version
|
||||
, customer_id_str
|
||||
, device
|
||||
, external_id
|
||||
, id_str
|
||||
, ip
|
||||
, ip_city
|
||||
, ip_country
|
||||
, ip_province
|
||||
, os
|
||||
, os_version
|
||||
, page_id
|
||||
, page_type
|
||||
, platform
|
||||
, screen_height
|
||||
, screen_width
|
||||
, short_id
|
||||
, target_id
|
||||
, target_name
|
||||
, url
|
||||
, etl_tx_dt )
|
||||
select
|
||||
id
|
||||
, customer_id
|
||||
, event
|
||||
, last_updated
|
||||
, date
|
||||
, browser
|
||||
, browser_version
|
||||
, customer_id_str
|
||||
, device
|
||||
, external_id
|
||||
, id_str
|
||||
, ip
|
||||
, ip_city
|
||||
, ip_country
|
||||
, ip_province
|
||||
, os
|
||||
, os_version
|
||||
, page_id
|
||||
, page_type
|
||||
, platform
|
||||
, screen_height
|
||||
, screen_width
|
||||
, short_id
|
||||
, target_id
|
||||
, target_name
|
||||
, url
|
||||
, etl_tx_dt
|
||||
from p10_sa.S98_S_customer_events_open_content_page
|
||||
;
|
||||
\q
|
||||
|
|
@ -0,0 +1,38 @@
|
|||
|
||||
CREATE FOREIGN TABLE if not exists p00_tal.S98_S_customer_events_open_content_page (
|
||||
id TEXT
|
||||
, customer_id TEXT
|
||||
, event TEXT
|
||||
, last_updated TEXT
|
||||
, date TEXT
|
||||
, browser TEXT
|
||||
, browser_version TEXT
|
||||
, customer_id_str TEXT
|
||||
, device TEXT
|
||||
, external_id TEXT
|
||||
, id_str TEXT
|
||||
, ip TEXT
|
||||
, ip_city TEXT
|
||||
, ip_country TEXT
|
||||
, ip_province TEXT
|
||||
, os TEXT
|
||||
, os_version TEXT
|
||||
, page_id TEXT
|
||||
, page_type TEXT
|
||||
, platform TEXT
|
||||
, screen_height TEXT
|
||||
, screen_width TEXT
|
||||
, short_id TEXT
|
||||
, target_id TEXT
|
||||
, target_name TEXT
|
||||
, url TEXT
|
||||
, etl_tx_dt TIMESTAMP
|
||||
)
|
||||
|
||||
|
||||
SERVER pgsql_server_S98_S OPTIONS(schema_name 'data_api', table_name 'customer_events_open_content_page' );
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
|
@ -0,0 +1,123 @@
|
|||
|
||||
create table if not exists p10_sa.S98_S_customer_events_open_content_page (
|
||||
id TEXT
|
||||
, customer_id TEXT
|
||||
, event TEXT
|
||||
, last_updated TEXT
|
||||
, date TEXT
|
||||
, browser TEXT
|
||||
, browser_version TEXT
|
||||
, customer_id_str TEXT
|
||||
, device TEXT
|
||||
, external_id TEXT
|
||||
, id_str TEXT
|
||||
, ip TEXT
|
||||
, ip_city TEXT
|
||||
, ip_country TEXT
|
||||
, ip_province TEXT
|
||||
, os TEXT
|
||||
, os_version TEXT
|
||||
, page_id TEXT
|
||||
, page_type TEXT
|
||||
, platform TEXT
|
||||
, screen_height TEXT
|
||||
, screen_width TEXT
|
||||
, short_id TEXT
|
||||
, target_id TEXT
|
||||
, target_name TEXT
|
||||
, url TEXT
|
||||
, etl_tx_dt TIMESTAMP
|
||||
) ;
|
||||
|
||||
COMMENT ON COLUMN p10_sa.S98_S_customer_events_open_content_page.id IS '';
|
||||
COMMENT ON COLUMN p10_sa.S98_S_customer_events_open_content_page.customer_id IS '';
|
||||
COMMENT ON COLUMN p10_sa.S98_S_customer_events_open_content_page.event IS '';
|
||||
COMMENT ON COLUMN p10_sa.S98_S_customer_events_open_content_page.last_updated IS '';
|
||||
COMMENT ON COLUMN p10_sa.S98_S_customer_events_open_content_page.date IS '';
|
||||
COMMENT ON COLUMN p10_sa.S98_S_customer_events_open_content_page.browser IS '';
|
||||
COMMENT ON COLUMN p10_sa.S98_S_customer_events_open_content_page.browser_version IS '';
|
||||
COMMENT ON COLUMN p10_sa.S98_S_customer_events_open_content_page.customer_id_str IS '';
|
||||
COMMENT ON COLUMN p10_sa.S98_S_customer_events_open_content_page.device IS '';
|
||||
COMMENT ON COLUMN p10_sa.S98_S_customer_events_open_content_page.external_id IS '';
|
||||
COMMENT ON COLUMN p10_sa.S98_S_customer_events_open_content_page.id_str IS '';
|
||||
COMMENT ON COLUMN p10_sa.S98_S_customer_events_open_content_page.ip IS '';
|
||||
COMMENT ON COLUMN p10_sa.S98_S_customer_events_open_content_page.ip_city IS '';
|
||||
COMMENT ON COLUMN p10_sa.S98_S_customer_events_open_content_page.ip_country IS '';
|
||||
COMMENT ON COLUMN p10_sa.S98_S_customer_events_open_content_page.ip_province IS '';
|
||||
COMMENT ON COLUMN p10_sa.S98_S_customer_events_open_content_page.os IS '';
|
||||
COMMENT ON COLUMN p10_sa.S98_S_customer_events_open_content_page.os_version IS '';
|
||||
COMMENT ON COLUMN p10_sa.S98_S_customer_events_open_content_page.page_id IS '';
|
||||
COMMENT ON COLUMN p10_sa.S98_S_customer_events_open_content_page.page_type IS '';
|
||||
COMMENT ON COLUMN p10_sa.S98_S_customer_events_open_content_page.platform IS '';
|
||||
COMMENT ON COLUMN p10_sa.S98_S_customer_events_open_content_page.screen_height IS '';
|
||||
COMMENT ON COLUMN p10_sa.S98_S_customer_events_open_content_page.screen_width IS '';
|
||||
COMMENT ON COLUMN p10_sa.S98_S_customer_events_open_content_page.short_id IS '';
|
||||
COMMENT ON COLUMN p10_sa.S98_S_customer_events_open_content_page.target_id IS '';
|
||||
COMMENT ON COLUMN p10_sa.S98_S_customer_events_open_content_page.target_name IS '';
|
||||
COMMENT ON COLUMN p10_sa.S98_S_customer_events_open_content_page.url IS '';
|
||||
COMMENT ON COLUMN p10_sa.S98_S_customer_events_open_content_page.etl_tx_dt IS '';
|
||||
|
||||
COMMENT ON TABLE p10_sa.S98_S_customer_events_open_content_page IS '';
|
||||
|
||||
|
||||
|
||||
create table if not exists p12_sfull.S98_S_customer_events_open_content_page (
|
||||
id TEXT
|
||||
, customer_id TEXT
|
||||
, event TEXT
|
||||
, last_updated TEXT
|
||||
, date TEXT
|
||||
, browser TEXT
|
||||
, browser_version TEXT
|
||||
, customer_id_str TEXT
|
||||
, device TEXT
|
||||
, external_id TEXT
|
||||
, id_str TEXT
|
||||
, ip TEXT
|
||||
, ip_city TEXT
|
||||
, ip_country TEXT
|
||||
, ip_province TEXT
|
||||
, os TEXT
|
||||
, os_version TEXT
|
||||
, page_id TEXT
|
||||
, page_type TEXT
|
||||
, platform TEXT
|
||||
, screen_height TEXT
|
||||
, screen_width TEXT
|
||||
, short_id TEXT
|
||||
, target_id TEXT
|
||||
, target_name TEXT
|
||||
, url TEXT
|
||||
, etl_tx_dt TIMESTAMP
|
||||
) ;
|
||||
|
||||
COMMENT ON COLUMN p12_sfull.S98_S_customer_events_open_content_page.id IS '';
|
||||
COMMENT ON COLUMN p12_sfull.S98_S_customer_events_open_content_page.customer_id IS '';
|
||||
COMMENT ON COLUMN p12_sfull.S98_S_customer_events_open_content_page.event IS '';
|
||||
COMMENT ON COLUMN p12_sfull.S98_S_customer_events_open_content_page.last_updated IS '';
|
||||
COMMENT ON COLUMN p12_sfull.S98_S_customer_events_open_content_page.date IS '';
|
||||
COMMENT ON COLUMN p12_sfull.S98_S_customer_events_open_content_page.browser IS '';
|
||||
COMMENT ON COLUMN p12_sfull.S98_S_customer_events_open_content_page.browser_version IS '';
|
||||
COMMENT ON COLUMN p12_sfull.S98_S_customer_events_open_content_page.customer_id_str IS '';
|
||||
COMMENT ON COLUMN p12_sfull.S98_S_customer_events_open_content_page.device IS '';
|
||||
COMMENT ON COLUMN p12_sfull.S98_S_customer_events_open_content_page.external_id IS '';
|
||||
COMMENT ON COLUMN p12_sfull.S98_S_customer_events_open_content_page.id_str IS '';
|
||||
COMMENT ON COLUMN p12_sfull.S98_S_customer_events_open_content_page.ip IS '';
|
||||
COMMENT ON COLUMN p12_sfull.S98_S_customer_events_open_content_page.ip_city IS '';
|
||||
COMMENT ON COLUMN p12_sfull.S98_S_customer_events_open_content_page.ip_country IS '';
|
||||
COMMENT ON COLUMN p12_sfull.S98_S_customer_events_open_content_page.ip_province IS '';
|
||||
COMMENT ON COLUMN p12_sfull.S98_S_customer_events_open_content_page.os IS '';
|
||||
COMMENT ON COLUMN p12_sfull.S98_S_customer_events_open_content_page.os_version IS '';
|
||||
COMMENT ON COLUMN p12_sfull.S98_S_customer_events_open_content_page.page_id IS '';
|
||||
COMMENT ON COLUMN p12_sfull.S98_S_customer_events_open_content_page.page_type IS '';
|
||||
COMMENT ON COLUMN p12_sfull.S98_S_customer_events_open_content_page.platform IS '';
|
||||
COMMENT ON COLUMN p12_sfull.S98_S_customer_events_open_content_page.screen_height IS '';
|
||||
COMMENT ON COLUMN p12_sfull.S98_S_customer_events_open_content_page.screen_width IS '';
|
||||
COMMENT ON COLUMN p12_sfull.S98_S_customer_events_open_content_page.short_id IS '';
|
||||
COMMENT ON COLUMN p12_sfull.S98_S_customer_events_open_content_page.target_id IS '';
|
||||
COMMENT ON COLUMN p12_sfull.S98_S_customer_events_open_content_page.target_name IS '';
|
||||
COMMENT ON COLUMN p12_sfull.S98_S_customer_events_open_content_page.url IS '';
|
||||
COMMENT ON COLUMN p12_sfull.S98_S_customer_events_open_content_page.etl_tx_dt IS '';
|
||||
|
||||
COMMENT ON TABLE p12_sfull.S98_S_customer_events_open_content_page IS '';
|
||||
|
||||
|
|
@ -0,0 +1,142 @@
|
|||
/*******Main Section**************************************************************************/
|
||||
\set ON_ERROR_STOP on
|
||||
\set AUTOCOMMIT on
|
||||
\timing on
|
||||
delete from p10_sa.S98_S_customer_events_open_page
|
||||
;
|
||||
insert into p10_sa.S98_S_customer_events_open_page
|
||||
( id
|
||||
, customer_id
|
||||
, event
|
||||
, last_updated
|
||||
, date
|
||||
, browser
|
||||
, browser_version
|
||||
, customer_id_str
|
||||
, device
|
||||
, domain
|
||||
, external_id
|
||||
, id_str
|
||||
, ip
|
||||
, ip_city
|
||||
, ip_country
|
||||
, ip_province
|
||||
, target_name
|
||||
, os
|
||||
, os_version
|
||||
, page_id
|
||||
, page_type
|
||||
, path
|
||||
, platform
|
||||
, root_domain
|
||||
, score
|
||||
, screen_height
|
||||
, screen_width
|
||||
, short_id
|
||||
, target_id
|
||||
, url
|
||||
, etl_tx_dt )
|
||||
select
|
||||
id
|
||||
, customer_id
|
||||
, event
|
||||
, last_updated
|
||||
, date
|
||||
, browser
|
||||
, browser_version
|
||||
, customer_id_str
|
||||
, device
|
||||
, domain
|
||||
, external_id
|
||||
, id_str
|
||||
, ip
|
||||
, ip_city
|
||||
, ip_country
|
||||
, ip_province
|
||||
, target_name
|
||||
, os
|
||||
, os_version
|
||||
, page_id
|
||||
, page_type
|
||||
, path
|
||||
, platform
|
||||
, root_domain
|
||||
, score
|
||||
, screen_height
|
||||
, screen_width
|
||||
, short_id
|
||||
, target_id
|
||||
, url
|
||||
, etl_tx_dt
|
||||
from p00_tal.S98_S_customer_events_open_page
|
||||
;
|
||||
delete from p12_sfull.S98_S_customer_events_open_page
|
||||
;
|
||||
;
|
||||
insert into p12_sfull.S98_S_customer_events_open_page
|
||||
( id
|
||||
, customer_id
|
||||
, event
|
||||
, last_updated
|
||||
, date
|
||||
, browser
|
||||
, browser_version
|
||||
, customer_id_str
|
||||
, device
|
||||
, domain
|
||||
, external_id
|
||||
, id_str
|
||||
, ip
|
||||
, ip_city
|
||||
, ip_country
|
||||
, ip_province
|
||||
, target_name
|
||||
, os
|
||||
, os_version
|
||||
, page_id
|
||||
, page_type
|
||||
, path
|
||||
, platform
|
||||
, root_domain
|
||||
, score
|
||||
, screen_height
|
||||
, screen_width
|
||||
, short_id
|
||||
, target_id
|
||||
, url
|
||||
, etl_tx_dt )
|
||||
select
|
||||
id
|
||||
, customer_id
|
||||
, event
|
||||
, last_updated
|
||||
, date
|
||||
, browser
|
||||
, browser_version
|
||||
, customer_id_str
|
||||
, device
|
||||
, domain
|
||||
, external_id
|
||||
, id_str
|
||||
, ip
|
||||
, ip_city
|
||||
, ip_country
|
||||
, ip_province
|
||||
, target_name
|
||||
, os
|
||||
, os_version
|
||||
, page_id
|
||||
, page_type
|
||||
, path
|
||||
, platform
|
||||
, root_domain
|
||||
, score
|
||||
, screen_height
|
||||
, screen_width
|
||||
, short_id
|
||||
, target_id
|
||||
, url
|
||||
, etl_tx_dt
|
||||
from p10_sa.S98_S_customer_events_open_page
|
||||
;
|
||||
\q
|
||||
|
|
@ -0,0 +1,42 @@
|
|||
|
||||
CREATE FOREIGN TABLE if not exists p00_tal.S98_S_customer_events_open_page (
|
||||
id TEXT
|
||||
, customer_id TEXT
|
||||
, event TEXT
|
||||
, last_updated TEXT
|
||||
, date TEXT
|
||||
, browser TEXT
|
||||
, browser_version TEXT
|
||||
, customer_id_str TEXT
|
||||
, device TEXT
|
||||
, domain TEXT
|
||||
, external_id TEXT
|
||||
, id_str TEXT
|
||||
, ip TEXT
|
||||
, ip_city TEXT
|
||||
, ip_country TEXT
|
||||
, ip_province TEXT
|
||||
, target_name TEXT
|
||||
, os TEXT
|
||||
, os_version TEXT
|
||||
, page_id TEXT
|
||||
, page_type TEXT
|
||||
, path TEXT
|
||||
, platform TEXT
|
||||
, root_domain TEXT
|
||||
, score TEXT
|
||||
, screen_height TEXT
|
||||
, screen_width TEXT
|
||||
, short_id TEXT
|
||||
, target_id TEXT
|
||||
, url TEXT
|
||||
, etl_tx_dt TIMESTAMP
|
||||
)
|
||||
|
||||
|
||||
SERVER pgsql_server_S98_S OPTIONS(schema_name 'data_api', table_name 'customer_events_open_page' );
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
|
@ -0,0 +1,139 @@
|
|||
|
||||
create table if not exists p10_sa.S98_S_customer_events_open_page (
|
||||
id TEXT
|
||||
, customer_id TEXT
|
||||
, event TEXT
|
||||
, last_updated TEXT
|
||||
, date TEXT
|
||||
, browser TEXT
|
||||
, browser_version TEXT
|
||||
, customer_id_str TEXT
|
||||
, device TEXT
|
||||
, domain TEXT
|
||||
, external_id TEXT
|
||||
, id_str TEXT
|
||||
, ip TEXT
|
||||
, ip_city TEXT
|
||||
, ip_country TEXT
|
||||
, ip_province TEXT
|
||||
, target_name TEXT
|
||||
, os TEXT
|
||||
, os_version TEXT
|
||||
, page_id TEXT
|
||||
, page_type TEXT
|
||||
, path TEXT
|
||||
, platform TEXT
|
||||
, root_domain TEXT
|
||||
, score TEXT
|
||||
, screen_height TEXT
|
||||
, screen_width TEXT
|
||||
, short_id TEXT
|
||||
, target_id TEXT
|
||||
, url TEXT
|
||||
, etl_tx_dt TIMESTAMP
|
||||
) ;
|
||||
|
||||
COMMENT ON COLUMN p10_sa.S98_S_customer_events_open_page.id IS '';
|
||||
COMMENT ON COLUMN p10_sa.S98_S_customer_events_open_page.customer_id IS '';
|
||||
COMMENT ON COLUMN p10_sa.S98_S_customer_events_open_page.event IS '';
|
||||
COMMENT ON COLUMN p10_sa.S98_S_customer_events_open_page.last_updated IS '';
|
||||
COMMENT ON COLUMN p10_sa.S98_S_customer_events_open_page.date IS '';
|
||||
COMMENT ON COLUMN p10_sa.S98_S_customer_events_open_page.browser IS '';
|
||||
COMMENT ON COLUMN p10_sa.S98_S_customer_events_open_page.browser_version IS '';
|
||||
COMMENT ON COLUMN p10_sa.S98_S_customer_events_open_page.customer_id_str IS '';
|
||||
COMMENT ON COLUMN p10_sa.S98_S_customer_events_open_page.device IS '';
|
||||
COMMENT ON COLUMN p10_sa.S98_S_customer_events_open_page.domain IS '';
|
||||
COMMENT ON COLUMN p10_sa.S98_S_customer_events_open_page.external_id IS '';
|
||||
COMMENT ON COLUMN p10_sa.S98_S_customer_events_open_page.id_str IS '';
|
||||
COMMENT ON COLUMN p10_sa.S98_S_customer_events_open_page.ip IS '';
|
||||
COMMENT ON COLUMN p10_sa.S98_S_customer_events_open_page.ip_city IS '';
|
||||
COMMENT ON COLUMN p10_sa.S98_S_customer_events_open_page.ip_country IS '';
|
||||
COMMENT ON COLUMN p10_sa.S98_S_customer_events_open_page.ip_province IS '';
|
||||
COMMENT ON COLUMN p10_sa.S98_S_customer_events_open_page.target_name IS '';
|
||||
COMMENT ON COLUMN p10_sa.S98_S_customer_events_open_page.os IS '';
|
||||
COMMENT ON COLUMN p10_sa.S98_S_customer_events_open_page.os_version IS '';
|
||||
COMMENT ON COLUMN p10_sa.S98_S_customer_events_open_page.page_id IS '';
|
||||
COMMENT ON COLUMN p10_sa.S98_S_customer_events_open_page.page_type IS '';
|
||||
COMMENT ON COLUMN p10_sa.S98_S_customer_events_open_page.path IS '';
|
||||
COMMENT ON COLUMN p10_sa.S98_S_customer_events_open_page.platform IS '';
|
||||
COMMENT ON COLUMN p10_sa.S98_S_customer_events_open_page.root_domain IS '';
|
||||
COMMENT ON COLUMN p10_sa.S98_S_customer_events_open_page.score IS '';
|
||||
COMMENT ON COLUMN p10_sa.S98_S_customer_events_open_page.screen_height IS '';
|
||||
COMMENT ON COLUMN p10_sa.S98_S_customer_events_open_page.screen_width IS '';
|
||||
COMMENT ON COLUMN p10_sa.S98_S_customer_events_open_page.short_id IS '';
|
||||
COMMENT ON COLUMN p10_sa.S98_S_customer_events_open_page.target_id IS '';
|
||||
COMMENT ON COLUMN p10_sa.S98_S_customer_events_open_page.url IS '';
|
||||
COMMENT ON COLUMN p10_sa.S98_S_customer_events_open_page.etl_tx_dt IS '';
|
||||
|
||||
COMMENT ON TABLE p10_sa.S98_S_customer_events_open_page IS '';
|
||||
|
||||
|
||||
|
||||
create table if not exists p12_sfull.S98_S_customer_events_open_page (
|
||||
id TEXT
|
||||
, customer_id TEXT
|
||||
, event TEXT
|
||||
, last_updated TEXT
|
||||
, date TEXT
|
||||
, browser TEXT
|
||||
, browser_version TEXT
|
||||
, customer_id_str TEXT
|
||||
, device TEXT
|
||||
, domain TEXT
|
||||
, external_id TEXT
|
||||
, id_str TEXT
|
||||
, ip TEXT
|
||||
, ip_city TEXT
|
||||
, ip_country TEXT
|
||||
, ip_province TEXT
|
||||
, target_name TEXT
|
||||
, os TEXT
|
||||
, os_version TEXT
|
||||
, page_id TEXT
|
||||
, page_type TEXT
|
||||
, path TEXT
|
||||
, platform TEXT
|
||||
, root_domain TEXT
|
||||
, score TEXT
|
||||
, screen_height TEXT
|
||||
, screen_width TEXT
|
||||
, short_id TEXT
|
||||
, target_id TEXT
|
||||
, url TEXT
|
||||
, etl_tx_dt TIMESTAMP
|
||||
) ;
|
||||
|
||||
COMMENT ON COLUMN p12_sfull.S98_S_customer_events_open_page.id IS '';
|
||||
COMMENT ON COLUMN p12_sfull.S98_S_customer_events_open_page.customer_id IS '';
|
||||
COMMENT ON COLUMN p12_sfull.S98_S_customer_events_open_page.event IS '';
|
||||
COMMENT ON COLUMN p12_sfull.S98_S_customer_events_open_page.last_updated IS '';
|
||||
COMMENT ON COLUMN p12_sfull.S98_S_customer_events_open_page.date IS '';
|
||||
COMMENT ON COLUMN p12_sfull.S98_S_customer_events_open_page.browser IS '';
|
||||
COMMENT ON COLUMN p12_sfull.S98_S_customer_events_open_page.browser_version IS '';
|
||||
COMMENT ON COLUMN p12_sfull.S98_S_customer_events_open_page.customer_id_str IS '';
|
||||
COMMENT ON COLUMN p12_sfull.S98_S_customer_events_open_page.device IS '';
|
||||
COMMENT ON COLUMN p12_sfull.S98_S_customer_events_open_page.domain IS '';
|
||||
COMMENT ON COLUMN p12_sfull.S98_S_customer_events_open_page.external_id IS '';
|
||||
COMMENT ON COLUMN p12_sfull.S98_S_customer_events_open_page.id_str IS '';
|
||||
COMMENT ON COLUMN p12_sfull.S98_S_customer_events_open_page.ip IS '';
|
||||
COMMENT ON COLUMN p12_sfull.S98_S_customer_events_open_page.ip_city IS '';
|
||||
COMMENT ON COLUMN p12_sfull.S98_S_customer_events_open_page.ip_country IS '';
|
||||
COMMENT ON COLUMN p12_sfull.S98_S_customer_events_open_page.ip_province IS '';
|
||||
COMMENT ON COLUMN p12_sfull.S98_S_customer_events_open_page.target_name IS '';
|
||||
COMMENT ON COLUMN p12_sfull.S98_S_customer_events_open_page.os IS '';
|
||||
COMMENT ON COLUMN p12_sfull.S98_S_customer_events_open_page.os_version IS '';
|
||||
COMMENT ON COLUMN p12_sfull.S98_S_customer_events_open_page.page_id IS '';
|
||||
COMMENT ON COLUMN p12_sfull.S98_S_customer_events_open_page.page_type IS '';
|
||||
COMMENT ON COLUMN p12_sfull.S98_S_customer_events_open_page.path IS '';
|
||||
COMMENT ON COLUMN p12_sfull.S98_S_customer_events_open_page.platform IS '';
|
||||
COMMENT ON COLUMN p12_sfull.S98_S_customer_events_open_page.root_domain IS '';
|
||||
COMMENT ON COLUMN p12_sfull.S98_S_customer_events_open_page.score IS '';
|
||||
COMMENT ON COLUMN p12_sfull.S98_S_customer_events_open_page.screen_height IS '';
|
||||
COMMENT ON COLUMN p12_sfull.S98_S_customer_events_open_page.screen_width IS '';
|
||||
COMMENT ON COLUMN p12_sfull.S98_S_customer_events_open_page.short_id IS '';
|
||||
COMMENT ON COLUMN p12_sfull.S98_S_customer_events_open_page.target_id IS '';
|
||||
COMMENT ON COLUMN p12_sfull.S98_S_customer_events_open_page.url IS '';
|
||||
COMMENT ON COLUMN p12_sfull.S98_S_customer_events_open_page.etl_tx_dt IS '';
|
||||
|
||||
COMMENT ON TABLE p12_sfull.S98_S_customer_events_open_page IS '';
|
||||
|
||||
|
|
@ -0,0 +1,156 @@
|
|||
#!/usr/bin/python
|
||||
# -*- encoding=utf-8 -*-
|
||||
from airflow import DAG
|
||||
from datetime import datetime, timedelta
|
||||
from airflow.contrib.hooks.ssh_hook import SSHHook
|
||||
from airflow.contrib.operators.ssh_operator import SSHOperator
|
||||
from airflow.sensors.external_task_sensor import ExternalTaskSensor
|
||||
import json
|
||||
|
||||
from airflow.operators.email_operator import EmailOperator
|
||||
from airflow.utils.trigger_rule import TriggerRule
|
||||
|
||||
|
||||
sshHook = SSHHook(ssh_conn_id ='ssh_air')
|
||||
default_args = {
|
||||
'owner': 'info@idgvalue.com',
|
||||
'email_on_failure': True,
|
||||
'email_on_retry':True,
|
||||
'start_date': datetime(2024, 1, 1),
|
||||
'depends_on_past': False,
|
||||
'retries': 6,
|
||||
'retry_delay': timedelta(minutes=10),
|
||||
}
|
||||
|
||||
dag = DAG('wf_dag_tk_api_2', default_args=default_args,
|
||||
schedule_interval="10 0 * * *",
|
||||
catchup=False,
|
||||
dagrun_timeout=timedelta(minutes=160),
|
||||
max_active_runs=3)
|
||||
|
||||
task_failed = EmailOperator (
|
||||
dag=dag,
|
||||
trigger_rule=TriggerRule.ONE_FAILED,
|
||||
task_id="task_failed",
|
||||
to=["info@idgvalue.com"],
|
||||
cc=[""],
|
||||
subject="tk_api_2_failed",
|
||||
html_content='<h3>您好，tk_api_2作业失败，请及时处理" </h3>')
|
||||
|
||||
customer_events_feign = SSHOperator(
|
||||
ssh_hook=sshHook,
|
||||
task_id='customer_events_feign',
|
||||
command='python3 /data/airflow/etl/API/customer_events_feign.py',
|
||||
depends_on_past=False,
|
||||
retries=3,
|
||||
dag=dag)
|
||||
|
||||
customer_events_load = SSHOperator(
|
||||
ssh_hook=sshHook,
|
||||
task_id='customer_events_load',
|
||||
command='/data/airflow/etl/API/run_psql.sh {{ ds_nodash }} {{params.my_param }}',
|
||||
params={'my_param':"customer_events_load"},
|
||||
depends_on_past=False,
|
||||
retries=3,
|
||||
dag=dag)
|
||||
|
||||
customer_events_feign >> customer_events_load
|
||||
|
||||
custom_events_open_page_feign = SSHOperator(
|
||||
ssh_hook=sshHook,
|
||||
task_id='custom_events_open_page_feign',
|
||||
command='python3 /data/airflow/etl/API/custom_events_open_page_feign.py',
|
||||
depends_on_past=False,
|
||||
retries=3,
|
||||
dag=dag)
|
||||
|
||||
custom_events_open_page_load = SSHOperator(
|
||||
ssh_hook=sshHook,
|
||||
task_id='custom_events_open_page_load',
|
||||
command='/data/airflow/etl/API/run_psql.sh {{ ds_nodash }} {{params.my_param }}',
|
||||
params={'my_param':"custom_events_open_page_load"},
|
||||
depends_on_past=False,
|
||||
retries=3,
|
||||
dag=dag)
|
||||
|
||||
custom_events_open_page_feign >> custom_events_open_page_load
|
||||
|
||||
custom_events_open_content_page_feign = SSHOperator(
|
||||
ssh_hook=sshHook,
|
||||
task_id='custom_events_open_content_page_feign',
|
||||
command='python3 /data/airflow/etl/API/custom_events_open_content_page_feign.py',
|
||||
depends_on_past=False,
|
||||
retries=3,
|
||||
dag=dag)
|
||||
|
||||
custom_events_open_content_page_load = SSHOperator(
|
||||
ssh_hook=sshHook,
|
||||
task_id='custom_events_open_content_page_load',
|
||||
command='/data/airflow/etl/API/run_psql.sh {{ ds_nodash }} {{params.my_param }}',
|
||||
params={'my_param':"custom_events_open_content_page_load"},
|
||||
depends_on_past=False,
|
||||
retries=3,
|
||||
dag=dag)
|
||||
|
||||
custom_events_open_content_page_feign >> custom_events_open_content_page_load
|
||||
|
||||
customer_event_meta_feign = SSHOperator(
|
||||
ssh_hook=sshHook,
|
||||
task_id='customer_event_meta_feign',
|
||||
command='python3 /data/airflow/etl/API/customer_event_meta_feign.py',
|
||||
depends_on_past=False,
|
||||
retries=3,
|
||||
dag=dag)
|
||||
|
||||
customer_event_meta_load = SSHOperator(
|
||||
ssh_hook=sshHook,
|
||||
task_id='customer_event_meta_load',
|
||||
command='/data/airflow/etl/API/run_psql.sh {{ ds_nodash }} {{params.my_param }}',
|
||||
params={'my_param':"customer_event_meta_load"},
|
||||
depends_on_past=False,
|
||||
retries=3,
|
||||
dag=dag)
|
||||
|
||||
customer_event_meta_feign >> customer_event_meta_load
|
||||
|
||||
customer_events_open_content_page_9684 = SSHOperator(
|
||||
ssh_hook=sshHook,
|
||||
task_id='customer_events_open_content_page_9684',
|
||||
command='/data/airflow/etl/SA/run_sa.sh {{ ds_nodash }} {{ params.my_param }} >>/data/airflow/logs/run_psql_{{ds_nodash}}.log 2>&1 ',
|
||||
params={'my_param':"S98_S_customer_events_open_content_page"},
|
||||
depends_on_past=False,
|
||||
retries=3,
|
||||
dag=dag)
|
||||
|
||||
customer_event_meta_2268 = SSHOperator(
|
||||
ssh_hook=sshHook,
|
||||
task_id='customer_event_meta_2268',
|
||||
command='/data/airflow/etl/SA/run_sa.sh {{ ds_nodash }} {{ params.my_param }} >>/data/airflow/logs/run_psql_{{ds_nodash}}.log 2>&1 ',
|
||||
params={'my_param':"S98_S_customer_event_meta"},
|
||||
depends_on_past=False,
|
||||
retries=3,
|
||||
dag=dag)
|
||||
|
||||
customer_events_open_page_7519 = SSHOperator(
|
||||
ssh_hook=sshHook,
|
||||
task_id='customer_events_open_page_7519',
|
||||
command='/data/airflow/etl/SA/run_sa.sh {{ ds_nodash }} {{ params.my_param }} >>/data/airflow/logs/run_psql_{{ds_nodash}}.log 2>&1 ',
|
||||
params={'my_param':"S98_S_customer_events_open_page"},
|
||||
depends_on_past=False,
|
||||
retries=3,
|
||||
dag=dag)
|
||||
|
||||
customer_events_3292 = SSHOperator(
|
||||
ssh_hook=sshHook,
|
||||
task_id='customer_events_3292',
|
||||
command='/data/airflow/etl/SA/run_sa.sh {{ ds_nodash }} {{ params.my_param }} >>/data/airflow/logs/run_psql_{{ds_nodash}}.log 2>&1 ',
|
||||
params={'my_param':"S98_S_customer_events"},
|
||||
depends_on_past=False,
|
||||
retries=3,
|
||||
dag=dag)
|
||||
|
||||
customer_events_3292 >> customer_events_feign
|
||||
customer_events_open_content_page_9684 >> custom_events_open_content_page_feign
|
||||
customer_event_meta_2268 >> customer_event_meta_feign
|
||||
customer_events_open_page_7519 >> custom_events_open_page_feign
|
||||
custom_events_open_page_load >> task_failed
|
||||
|
|
@ -0,0 +1,72 @@
|
|||
# coding: utf-8
|
||||
import requests
|
||||
import json
|
||||
import psycopg2
|
||||
import uuid
|
||||
import datetime
|
||||
import time
|
||||
import hashlib
|
||||
import time
|
||||
|
||||
#全局变量，便于参数使用的预设值
|
||||
current_date = datetime.date.today() # 获取当前日期
|
||||
previous_date = current_date - datetime.timedelta(days=1) # 获取前一天日期
|
||||
formatted_current_date = current_date.strftime("%Y-%m-%dT%H:%M:%SZ") # 获取当前日期 - 标准化
|
||||
formatted_previous_date = previous_date.strftime("%Y-%m-%dT%H:%M:%SZ") # 获取前一天日期 - 标准化
|
||||
timestamp = time.time() # 为Unix time，即从"1970-01-01 00:00:00"至今的秒数；
|
||||
sign_version = 'v2' # 签名版本号，固定值v2
|
||||
nonce = str(uuid.uuid4())
|
||||
|
||||
#获取签名令牌
|
||||
def sign_data(email, open_api_token, timestamp, nonce, sign_version):
|
||||
# 按照指定的格式拼接字符串
|
||||
data_to_sign = f"{email}&{open_api_token}&{timestamp}&{nonce}&{sign_version}"
|
||||
# 使用SHA256算法计算哈希值
|
||||
sha256_hash = hashlib.sha256(data_to_sign.encode()).hexdigest()
|
||||
return sha256_hash
|
||||
|
||||
#获取鉴权token
|
||||
def get_token(url):
|
||||
#请求鉴权接口
|
||||
authRequest=requests.get(url)
|
||||
#解析结果
|
||||
if not authRequest: #若为空时，返回空
|
||||
return
|
||||
auth=json.loads(authRequest.text)
|
||||
return auth
|
||||
|
||||
print('开始加载数据：customer_events:获取线索事件')
|
||||
authUrl='https://api.huiju.cool/v2/oauth2/token?app_id=cl037b184ebccd97c&secret=deaa83fdd8d385eec0d1d04d34282c4ec8c33b60&grant_type=client_credentials'
|
||||
|
||||
print('开始请求令牌。')
|
||||
#authRequest=requests.get(authUrl)
|
||||
#auth=json.loads(authRequest.text)
|
||||
auth = get_token(authUrl)
|
||||
#循环判断auth是否为空，若为空，等待30s后重新请求
|
||||
i = 0
|
||||
while not auth['access_token'] and i < 60:
|
||||
time.sleep(60)
|
||||
auth = get_token(authUrl)
|
||||
i = i + 1
|
||||
print('开始请求数据总数。')
|
||||
url='https://api.huiju.cool/v2/customerEvents'
|
||||
header={}
|
||||
body={'access_token':auth['access_token'],'limit':'2000','sort':'date','event':'create_customer','date[ge]':formatted_previous_date,'date[le]':formatted_current_date,}
|
||||
dataReqL=requests.get(url,headers=header,params=body)
|
||||
resL=json.loads(dataReqL.text)
|
||||
# print(resL)
|
||||
dataList=resL['data']
|
||||
total=len(dataList)
|
||||
conn = psycopg2.connect(database="dataops_db", user="dbuser_dops", password="MIgTi3jA",
|
||||
host="172.17.0.8", port="5432")
|
||||
print('数据库连接成功')
|
||||
dataId=str(uuid.uuid4())
|
||||
print('临时id：'+dataId)
|
||||
json_object = json.dumps(dataList)
|
||||
cur=conn.cursor()
|
||||
sql="update data_api.api_data set is_loaded = '1' where api_id = '747d33fa-a0e0-421d-aa9b-4ca4517b';INSERT INTO data_api.api_data (id,api_id,data,total_num,is_loaded,status,request_tm,execute_tm,remark) values (%s,%s,%s,%s,'0','0',current_timestamp(0),current_timestamp(0),'')"
|
||||
cur.execute(sql,[dataId,'747d33fa-a0e0-421d-aa9b-4ca4517b',json_object,total])
|
||||
conn.commit()
|
||||
cur.close()
|
||||
conn.close()
|
||||
print('加载数据结束：customer_events:获取线索事件')
|
||||
|
|
@ -0,0 +1,57 @@
|
|||
/*******Main Section**************************************************************************/
|
||||
\set ON_ERROR_STOP on
|
||||
\set AUTOCOMMIT on
|
||||
\timing on
|
||||
|
||||
DELETE FROM data_api.customer_events;
|
||||
|
||||
insert into data_api.customer_events (
|
||||
id
|
||||
, customer_id
|
||||
, event
|
||||
, last_updated
|
||||
, source
|
||||
, content_name
|
||||
, date
|
||||
, create_method
|
||||
, customer_id_str
|
||||
, external_id
|
||||
, id_str
|
||||
,etl_tx_dt
|
||||
)
|
||||
select
|
||||
case when trim(both from id)='' then null else id::text end id
|
||||
, case when trim(both from customer_id)='' then null else customer_id::text end customer_id
|
||||
, case when trim(both from event)='' then null else event::text end event
|
||||
, case when trim(both from last_updated)='' then null else last_updated::text end last_updated
|
||||
, case when trim(both from source)='' then null else source::text end source
|
||||
, case when trim(both from content_name)='' then null else content_name::text end content_name
|
||||
, case when trim(both from date)='' then null else date::text end date
|
||||
, case when trim(both from create_method)='' then null else create_method::text end create_method
|
||||
, case when trim(both from customer_id_str)='' then null else customer_id_str::text end customer_id_str
|
||||
, case when trim(both from external_id)='' then null else external_id::text end external_id
|
||||
, case when trim(both from id_str)='' then null else id_str::text end id_str
|
||||
,etl_tx_dt
|
||||
from (
|
||||
select
|
||||
(json_array_elements(data::json)::json->>'id') id
|
||||
, (json_array_elements(data::json)::json->>'customerId') customer_id
|
||||
, (json_array_elements(data::json)::json->>'event') event
|
||||
, (json_array_elements(data::json)::json->>'lastUpdated') last_updated
|
||||
, (json_array_elements(data::json)::json->>'source') source
|
||||
, (json_array_elements(data::json)::json->>'contentName') content_name
|
||||
, (json_array_elements(data::json)::json->>'date') date
|
||||
, (json_array_elements(data::json)::json->>'createMethod') create_method
|
||||
, (json_array_elements(data::json)::json->>'customerIdStr') customer_id_str
|
||||
, (json_array_elements(data::json)::json->>'externalId') external_id
|
||||
, (json_array_elements(data::json)::json->>'idStr') id_str
|
||||
,CURRENT_TIMESTAMP(0) etl_tx_dt
|
||||
from (select * from data_api.api_data
|
||||
WHERE api_id='747d33fa-a0e0-421d-aa9b-4ca4517b' and is_loaded = '0' order by request_tm desc limit 1) p )p;
|
||||
|
||||
update data_api.api_data
|
||||
set is_loaded = '1' ,
|
||||
status = '1',
|
||||
request_tm = current_timestamp(0)
|
||||
where api_id='747d33fa-a0e0-421d-aa9b-4ca4517b';
|
||||
\q
|
||||
|
|
@ -0,0 +1,72 @@
|
|||
# coding: utf-8
|
||||
import requests
|
||||
import json
|
||||
import psycopg2
|
||||
import uuid
|
||||
import datetime
|
||||
import time
|
||||
import hashlib
|
||||
import time
|
||||
|
||||
#全局变量，便于参数使用的预设值
|
||||
current_date = datetime.date.today() # 获取当前日期
|
||||
previous_date = current_date - datetime.timedelta(days=1) # 获取前一天日期
|
||||
formatted_current_date = current_date.strftime("%Y-%m-%dT%H:%M:%SZ") # 获取当前日期 - 标准化
|
||||
formatted_previous_date = previous_date.strftime("%Y-%m-%dT%H:%M:%SZ") # 获取前一天日期 - 标准化
|
||||
timestamp = time.time() # 为Unix time，即从"1970-01-01 00:00:00"至今的秒数；
|
||||
sign_version = 'v2' # 签名版本号，固定值v2
|
||||
nonce = str(uuid.uuid4())
|
||||
|
||||
#获取签名令牌
|
||||
def sign_data(email, open_api_token, timestamp, nonce, sign_version):
|
||||
# 按照指定的格式拼接字符串
|
||||
data_to_sign = f"{email}&{open_api_token}&{timestamp}&{nonce}&{sign_version}"
|
||||
# 使用SHA256算法计算哈希值
|
||||
sha256_hash = hashlib.sha256(data_to_sign.encode()).hexdigest()
|
||||
return sha256_hash
|
||||
|
||||
#获取鉴权token
|
||||
def get_token(url):
|
||||
#请求鉴权接口
|
||||
authRequest=requests.get(url)
|
||||
#解析结果
|
||||
if not authRequest: #若为空时，返回空
|
||||
return
|
||||
auth=json.loads(authRequest.text)
|
||||
return auth
|
||||
|
||||
print('开始加载数据：custom_events_open_content_page:获取线索事件_open_content_page')
|
||||
authUrl='https://api.huiju.cool/v2/oauth2/token?app_id=cl037b184ebccd97c&secret=deaa83fdd8d385eec0d1d04d34282c4ec8c33b60&grant_type=client_credentials'
|
||||
|
||||
print('开始请求令牌。')
|
||||
#authRequest=requests.get(authUrl)
|
||||
#auth=json.loads(authRequest.text)
|
||||
auth = get_token(authUrl)
|
||||
#循环判断auth是否为空，若为空，等待30s后重新请求
|
||||
i = 0
|
||||
while not auth['access_token'] and i < 60:
|
||||
time.sleep(60)
|
||||
auth = get_token(authUrl)
|
||||
i = i + 1
|
||||
print('开始请求数据总数。')
|
||||
url='https://api.huiju.cool/v2/customerEvents'
|
||||
header={}
|
||||
body={'access_token':auth['access_token'],'limit':'2000','event':'open_content_page','sort':'date','date[ge]':formatted_previous_date,'date[le]':formatted_current_date,}
|
||||
dataReqL=requests.get(url,headers=header,params=body)
|
||||
resL=json.loads(dataReqL.text)
|
||||
# print(resL)
|
||||
dataList=resL['data']
|
||||
total=len(dataList)
|
||||
conn = psycopg2.connect(database="dataops_db", user="dbuser_dops", password="MIgTi3jA",
|
||||
host="172.17.0.8", port="5432")
|
||||
print('数据库连接成功')
|
||||
dataId=str(uuid.uuid4())
|
||||
print('临时id：'+dataId)
|
||||
json_object = json.dumps(dataList)
|
||||
cur=conn.cursor()
|
||||
sql="update data_api.api_data set is_loaded = '1' where api_id = '43edfb13-8a42-4152-9dc8-d5feb3c8';INSERT INTO data_api.api_data (id,api_id,data,total_num,is_loaded,status,request_tm,execute_tm,remark) values (%s,%s,%s,%s,'0','0',current_timestamp(0),current_timestamp(0),'')"
|
||||
cur.execute(sql,[dataId,'43edfb13-8a42-4152-9dc8-d5feb3c8',json_object,total])
|
||||
conn.commit()
|
||||
cur.close()
|
||||
conn.close()
|
||||
print('加载数据结束：custom_events_open_content_page:获取线索事件_open_content_page')
|
||||
|
|
@ -0,0 +1,102 @@
|
|||
/*******Main Section**************************************************************************/
|
||||
\set ON_ERROR_STOP on
|
||||
\set AUTOCOMMIT on
|
||||
\timing on
|
||||
|
||||
DELETE FROM data_api.customer_events_open_content_page;
|
||||
|
||||
insert into data_api.customer_events_open_content_page (
|
||||
id
|
||||
, customer_id
|
||||
, event
|
||||
, last_updated
|
||||
, date
|
||||
, browser
|
||||
, browser_version
|
||||
, customer_id_str
|
||||
, device
|
||||
, external_id
|
||||
, id_str
|
||||
, ip
|
||||
, ip_city
|
||||
, ip_country
|
||||
, ip_province
|
||||
, os
|
||||
, os_version
|
||||
, page_id
|
||||
, page_type
|
||||
, platform
|
||||
, screen_height
|
||||
, screen_width
|
||||
, short_id
|
||||
, target_id
|
||||
, target_name
|
||||
, url
|
||||
,etl_tx_dt
|
||||
)
|
||||
select
|
||||
case when trim(both from id)='' then null else id::text end id
|
||||
, case when trim(both from customer_id)='' then null else customer_id::text end customer_id
|
||||
, case when trim(both from event)='' then null else event::text end event
|
||||
, case when trim(both from last_updated)='' then null else last_updated::text end last_updated
|
||||
, case when trim(both from date)='' then null else date::text end date
|
||||
, case when trim(both from browser)='' then null else browser::text end browser
|
||||
, case when trim(both from browser_version)='' then null else browser_version::text end browser_version
|
||||
, case when trim(both from customer_id_str)='' then null else customer_id_str::text end customer_id_str
|
||||
, case when trim(both from device)='' then null else device::text end device
|
||||
, case when trim(both from external_id)='' then null else external_id::text end external_id
|
||||
, case when trim(both from id_str)='' then null else id_str::text end id_str
|
||||
, case when trim(both from ip)='' then null else ip::text end ip
|
||||
, case when trim(both from ip_city)='' then null else ip_city::text end ip_city
|
||||
, case when trim(both from ip_country)='' then null else ip_country::text end ip_country
|
||||
, case when trim(both from ip_province)='' then null else ip_province::text end ip_province
|
||||
, case when trim(both from os)='' then null else os::text end os
|
||||
, case when trim(both from os_version)='' then null else os_version::text end os_version
|
||||
, case when trim(both from page_id)='' then null else page_id::text end page_id
|
||||
, case when trim(both from page_type)='' then null else page_type::text end page_type
|
||||
, case when trim(both from platform)='' then null else platform::text end platform
|
||||
, case when trim(both from screen_height)='' then null else screen_height::text end screen_height
|
||||
, case when trim(both from screen_width)='' then null else screen_width::text end screen_width
|
||||
, case when trim(both from short_id)='' then null else short_id::text end short_id
|
||||
, case when trim(both from target_id)='' then null else target_id::text end target_id
|
||||
, case when trim(both from target_name)='' then null else target_name::text end target_name
|
||||
, case when trim(both from url)='' then null else url::text end url
|
||||
,etl_tx_dt
|
||||
from (
|
||||
select
|
||||
(json_array_elements(data::json)::json->>'id') id
|
||||
, (json_array_elements(data::json)::json->>'customerId') customer_id
|
||||
, (json_array_elements(data::json)::json->>'event') event
|
||||
, (json_array_elements(data::json)::json->>'lastUpdated') last_updated
|
||||
, (json_array_elements(data::json)::json->>'date') date
|
||||
, (json_array_elements(data::json)::json->>'browser') browser
|
||||
, (json_array_elements(data::json)::json->>'browserVersion') browser_version
|
||||
, (json_array_elements(data::json)::json->>'customerIdStr') customer_id_str
|
||||
, (json_array_elements(data::json)::json->>'device') device
|
||||
, (json_array_elements(data::json)::json->>'externalId') external_id
|
||||
, (json_array_elements(data::json)::json->>'idStr') id_str
|
||||
, (json_array_elements(data::json)::json->>'ip') ip
|
||||
, (json_array_elements(data::json)::json->>'ipCity') ip_city
|
||||
, (json_array_elements(data::json)::json->>'ipCountry') ip_country
|
||||
, (json_array_elements(data::json)::json->>'ipProvince') ip_province
|
||||
, (json_array_elements(data::json)::json->>'os') os
|
||||
, (json_array_elements(data::json)::json->>'osVersion') os_version
|
||||
, (json_array_elements(data::json)::json->>'pageId') page_id
|
||||
, (json_array_elements(data::json)::json->>'pageType') page_type
|
||||
, (json_array_elements(data::json)::json->>'platform') platform
|
||||
, (json_array_elements(data::json)::json->>'screenHeight') screen_height
|
||||
, (json_array_elements(data::json)::json->>'screenWidth') screen_width
|
||||
, (json_array_elements(data::json)::json->>'shortId') short_id
|
||||
, (json_array_elements(data::json)::json->>'targetId') target_id
|
||||
, (json_array_elements(data::json)::json->>'targetName') target_name
|
||||
, (json_array_elements(data::json)::json->>'url') url
|
||||
,CURRENT_TIMESTAMP(0) etl_tx_dt
|
||||
from (select * from data_api.api_data
|
||||
WHERE api_id='43edfb13-8a42-4152-9dc8-d5feb3c8' and is_loaded = '0' order by request_tm desc limit 1) p )p;
|
||||
|
||||
update data_api.api_data
|
||||
set is_loaded = '1' ,
|
||||
status = '1',
|
||||
request_tm = current_timestamp(0)
|
||||
where api_id='43edfb13-8a42-4152-9dc8-d5feb3c8';
|
||||
\q
|
||||
|
|
@ -0,0 +1,72 @@
|
|||
# coding: utf-8
|
||||
import requests
|
||||
import json
|
||||
import psycopg2
|
||||
import uuid
|
||||
import datetime
|
||||
import time
|
||||
import hashlib
|
||||
import time
|
||||
|
||||
#全局变量，便于参数使用的预设值
|
||||
current_date = datetime.date.today() # 获取当前日期
|
||||
previous_date = current_date - datetime.timedelta(days=1) # 获取前一天日期
|
||||
formatted_current_date = current_date.strftime("%Y-%m-%dT%H:%M:%SZ") # 获取当前日期 - 标准化
|
||||
formatted_previous_date = previous_date.strftime("%Y-%m-%dT%H:%M:%SZ") # 获取前一天日期 - 标准化
|
||||
timestamp = time.time() # 为Unix time，即从"1970-01-01 00:00:00"至今的秒数；
|
||||
sign_version = 'v2' # 签名版本号，固定值v2
|
||||
nonce = str(uuid.uuid4())
|
||||
|
||||
#获取签名令牌
|
||||
def sign_data(email, open_api_token, timestamp, nonce, sign_version):
|
||||
# 按照指定的格式拼接字符串
|
||||
data_to_sign = f"{email}&{open_api_token}&{timestamp}&{nonce}&{sign_version}"
|
||||
# 使用SHA256算法计算哈希值
|
||||
sha256_hash = hashlib.sha256(data_to_sign.encode()).hexdigest()
|
||||
return sha256_hash
|
||||
|
||||
#获取鉴权token
|
||||
def get_token(url):
|
||||
#请求鉴权接口
|
||||
authRequest=requests.get(url)
|
||||
#解析结果
|
||||
if not authRequest: #若为空时，返回空
|
||||
return
|
||||
auth=json.loads(authRequest.text)
|
||||
return auth
|
||||
|
||||
print('开始加载数据：custom_events_open_page:获取线索事件_open_page')
|
||||
authUrl='https://api.huiju.cool/v2/oauth2/token?app_id=cl037b184ebccd97c&secret=deaa83fdd8d385eec0d1d04d34282c4ec8c33b60&grant_type=client_credentials'
|
||||
|
||||
print('开始请求令牌。')
|
||||
#authRequest=requests.get(authUrl)
|
||||
#auth=json.loads(authRequest.text)
|
||||
auth = get_token(authUrl)
|
||||
#循环判断auth是否为空，若为空，等待30s后重新请求
|
||||
i = 0
|
||||
while not auth['access_token'] and i < 60:
|
||||
time.sleep(60)
|
||||
auth = get_token(authUrl)
|
||||
i = i + 1
|
||||
print('开始请求数据总数。')
|
||||
url='https://api.huiju.cool/v2/customerEvents'
|
||||
header={}
|
||||
body={'access_token':auth['access_token'],'limit':'2000','sort':'date','event':'open_page','date[le]':formatted_current_date,'date[ge]':formatted_previous_date,}
|
||||
dataReqL=requests.get(url,headers=header,params=body)
|
||||
resL=json.loads(dataReqL.text)
|
||||
# print(resL)
|
||||
dataList=resL['data']
|
||||
total=len(dataList)
|
||||
conn = psycopg2.connect(database="dataops_db", user="dbuser_dops", password="MIgTi3jA",
|
||||
host="172.17.0.8", port="5432")
|
||||
print('数据库连接成功')
|
||||
dataId=str(uuid.uuid4())
|
||||
print('临时id：'+dataId)
|
||||
json_object = json.dumps(dataList)
|
||||
cur=conn.cursor()
|
||||
sql="update data_api.api_data set is_loaded = '1' where api_id = 'a2b284c3-322f-4bc0-89ff-414aa66a';INSERT INTO data_api.api_data (id,api_id,data,total_num,is_loaded,status,request_tm,execute_tm,remark) values (%s,%s,%s,%s,'0','0',current_timestamp(0),current_timestamp(0),'')"
|
||||
cur.execute(sql,[dataId,'a2b284c3-322f-4bc0-89ff-414aa66a',json_object,total])
|
||||
conn.commit()
|
||||
cur.close()
|
||||
conn.close()
|
||||
print('加载数据结束：custom_events_open_page:获取线索事件_open_page')
|
||||
|
|
@ -0,0 +1,123 @@
|
|||
/*******Main Section**************************************************************************/
|
||||
\set ON_ERROR_STOP on
|
||||
\set AUTOCOMMIT on
|
||||
\timing on
|
||||
|
||||
DELETE FROM data_api.customer_events_open_page;
|
||||
|
||||
insert into data_api.customer_events_open_page (
|
||||
id
|
||||
, customer_id
|
||||
, event
|
||||
, last_updated
|
||||
, date
|
||||
, browser
|
||||
, browser_version
|
||||
, campaign
|
||||
, customer_id_str
|
||||
, device
|
||||
, domain
|
||||
, external_id
|
||||
, id_str
|
||||
, ip
|
||||
, ip_city
|
||||
, ip_country
|
||||
, ip_county
|
||||
, ip_province
|
||||
, target_name
|
||||
, os
|
||||
, os_version
|
||||
, page_id
|
||||
, page_type
|
||||
, path
|
||||
, platform
|
||||
, ref_domain
|
||||
, root_domain
|
||||
, score
|
||||
, screen_height
|
||||
, screen_width
|
||||
, short_id
|
||||
, target_id
|
||||
, url
|
||||
,etl_tx_dt
|
||||
)
|
||||
select
|
||||
case when trim(both from id)='' then null else id::text end id
|
||||
, case when trim(both from customer_id)='' then null else customer_id::text end customer_id
|
||||
, case when trim(both from event)='' then null else event::text end event
|
||||
, case when trim(both from last_updated)='' then null else last_updated::text end last_updated
|
||||
, case when trim(both from date)='' then null else date::text end date
|
||||
, case when trim(both from browser)='' then null else browser::text end browser
|
||||
, case when trim(both from browser_version)='' then null else browser_version::text end browser_version
|
||||
, case when trim(both from campaign)='' then null else campaign::text end campaign
|
||||
, case when trim(both from customer_id_str)='' then null else customer_id_str::text end customer_id_str
|
||||
, case when trim(both from device)='' then null else device::text end device
|
||||
, case when trim(both from domain)='' then null else domain::text end domain
|
||||
, case when trim(both from external_id)='' then null else external_id::text end external_id
|
||||
, case when trim(both from id_str)='' then null else id_str::text end id_str
|
||||
, case when trim(both from ip)='' then null else ip::text end ip
|
||||
, case when trim(both from ip_city)='' then null else ip_city::text end ip_city
|
||||
, case when trim(both from ip_country)='' then null else ip_country::text end ip_country
|
||||
, case when trim(both from ip_county)='' then null else ip_county::text end ip_county
|
||||
, case when trim(both from ip_province)='' then null else ip_province::text end ip_province
|
||||
, case when trim(both from target_name)='' then null else target_name::text end target_name
|
||||
, case when trim(both from os)='' then null else os::text end os
|
||||
, case when trim(both from os_version)='' then null else os_version::text end os_version
|
||||
, case when trim(both from page_id)='' then null else page_id::text end page_id
|
||||
, case when trim(both from page_type)='' then null else page_type::text end page_type
|
||||
, case when trim(both from path)='' then null else path::text end path
|
||||
, case when trim(both from platform)='' then null else platform::text end platform
|
||||
, case when trim(both from ref_domain)='' then null else ref_domain::text end ref_domain
|
||||
, case when trim(both from root_domain)='' then null else root_domain::text end root_domain
|
||||
, case when trim(both from score)='' then null else score::text end score
|
||||
, case when trim(both from screen_height)='' then null else screen_height::text end screen_height
|
||||
, case when trim(both from screen_width)='' then null else screen_width::text end screen_width
|
||||
, case when trim(both from short_id)='' then null else short_id::text end short_id
|
||||
, case when trim(both from target_id)='' then null else target_id::text end target_id
|
||||
, case when trim(both from url)='' then null else url::text end url
|
||||
,etl_tx_dt
|
||||
from (
|
||||
select
|
||||
(json_array_elements(data::json)::json->>'id') id
|
||||
, (json_array_elements(data::json)::json->>'customerId') customer_id
|
||||
, (json_array_elements(data::json)::json->>'event') event
|
||||
, (json_array_elements(data::json)::json->>'lastUpdated') last_updated
|
||||
, (json_array_elements(data::json)::json->>'date') date
|
||||
, (json_array_elements(data::json)::json->>'browser') browser
|
||||
, (json_array_elements(data::json)::json->>'browserVersion') browser_version
|
||||
, (json_array_elements(data::json)::json->>'campaign') campaign
|
||||
, (json_array_elements(data::json)::json->>'customerIdStr') customer_id_str
|
||||
, (json_array_elements(data::json)::json->>'device') device
|
||||
, (json_array_elements(data::json)::json->>'domain') domain
|
||||
, (json_array_elements(data::json)::json->>'externalId') external_id
|
||||
, (json_array_elements(data::json)::json->>'idStr') id_str
|
||||
, (json_array_elements(data::json)::json->>'ip') ip
|
||||
, (json_array_elements(data::json)::json->>'ipCity') ip_city
|
||||
, (json_array_elements(data::json)::json->>'ipCountry') ip_country
|
||||
, (json_array_elements(data::json)::json->>'ipCounty') ip_county
|
||||
, (json_array_elements(data::json)::json->>'ipProvince') ip_province
|
||||
, (json_array_elements(data::json)::json->>'targetName') target_name
|
||||
, (json_array_elements(data::json)::json->>'os') os
|
||||
, (json_array_elements(data::json)::json->>'osVersion') os_version
|
||||
, (json_array_elements(data::json)::json->>'pageId') page_id
|
||||
, (json_array_elements(data::json)::json->>'pageType') page_type
|
||||
, (json_array_elements(data::json)::json->>'path') path
|
||||
, (json_array_elements(data::json)::json->>'platform') platform
|
||||
, (json_array_elements(data::json)::json->>'refDomain') ref_domain
|
||||
, (json_array_elements(data::json)::json->>'rootDomain') root_domain
|
||||
, (json_array_elements(data::json)::json->>'score') score
|
||||
, (json_array_elements(data::json)::json->>'screenHeight') screen_height
|
||||
, (json_array_elements(data::json)::json->>'screenWidth') screen_width
|
||||
, (json_array_elements(data::json)::json->>'shortId') short_id
|
||||
, (json_array_elements(data::json)::json->>'targetId') target_id
|
||||
, (json_array_elements(data::json)::json->>'url') url
|
||||
,CURRENT_TIMESTAMP(0) etl_tx_dt
|
||||
from (select * from data_api.api_data
|
||||
WHERE api_id='a2b284c3-322f-4bc0-89ff-414aa66a' and is_loaded = '0' order by request_tm desc limit 1) p )p;
|
||||
|
||||
update data_api.api_data
|
||||
set is_loaded = '1' ,
|
||||
status = '1',
|
||||
request_tm = current_timestamp(0)
|
||||
where api_id='a2b284c3-322f-4bc0-89ff-414aa66a';
|
||||
\q
|
||||
|
|
@ -0,0 +1,72 @@
|
|||
# coding: utf-8
|
||||
import requests
|
||||
import json
|
||||
import psycopg2
|
||||
import uuid
|
||||
import datetime
|
||||
import time
|
||||
import hashlib
|
||||
import time
|
||||
|
||||
#全局变量，便于参数使用的预设值
|
||||
current_date = datetime.date.today() # 获取当前日期
|
||||
previous_date = current_date - datetime.timedelta(days=1) # 获取前一天日期
|
||||
formatted_current_date = current_date.strftime("%Y-%m-%dT%H:%M:%SZ") # 获取当前日期 - 标准化
|
||||
formatted_previous_date = previous_date.strftime("%Y-%m-%dT%H:%M:%SZ") # 获取前一天日期 - 标准化
|
||||
timestamp = time.time() # 为Unix time，即从"1970-01-01 00:00:00"至今的秒数；
|
||||
sign_version = 'v2' # 签名版本号，固定值v2
|
||||
nonce = str(uuid.uuid4())
|
||||
|
||||
#获取签名令牌
|
||||
def sign_data(email, open_api_token, timestamp, nonce, sign_version):
|
||||
# 按照指定的格式拼接字符串
|
||||
data_to_sign = f"{email}&{open_api_token}&{timestamp}&{nonce}&{sign_version}"
|
||||
# 使用SHA256算法计算哈希值
|
||||
sha256_hash = hashlib.sha256(data_to_sign.encode()).hexdigest()
|
||||
return sha256_hash
|
||||
|
||||
#获取鉴权token
|
||||
def get_token(url):
|
||||
#请求鉴权接口
|
||||
authRequest=requests.get(url)
|
||||
#解析结果
|
||||
if not authRequest: #若为空时，返回空
|
||||
return
|
||||
auth=json.loads(authRequest.text)
|
||||
return auth
|
||||
|
||||
print('开始加载数据：customer_event_meta:获取线索事件元数据')
|
||||
authUrl='https://api.huiju.cool/v2/oauth2/token?app_id=cl037b184ebccd97c&secret=deaa83fdd8d385eec0d1d04d34282c4ec8c33b60&grant_type=client_credentials'
|
||||
|
||||
print('开始请求令牌。')
|
||||
#authRequest=requests.get(authUrl)
|
||||
#auth=json.loads(authRequest.text)
|
||||
auth = get_token(authUrl)
|
||||
#循环判断auth是否为空，若为空，等待30s后重新请求
|
||||
i = 0
|
||||
while not auth['access_token'] and i < 60:
|
||||
time.sleep(60)
|
||||
auth = get_token(authUrl)
|
||||
i = i + 1
|
||||
print('开始请求数据总数。')
|
||||
url='https://api.huiju.cool/v2/metaService/getCustomerEventMeta'
|
||||
header={}
|
||||
body={'access_token':auth['access_token'],}
|
||||
dataReqL=requests.get(url,headers=header,params=body)
|
||||
resL=json.loads(dataReqL.text)
|
||||
# print(resL)
|
||||
dataList=resL['data']
|
||||
total=len(dataList)
|
||||
conn = psycopg2.connect(database="dataops_db", user="dbuser_dops", password="MIgTi3jA",
|
||||
host="172.17.0.8", port="5432")
|
||||
print('数据库连接成功')
|
||||
dataId=str(uuid.uuid4())
|
||||
print('临时id：'+dataId)
|
||||
json_object = json.dumps(dataList)
|
||||
cur=conn.cursor()
|
||||
sql="update data_api.api_data set is_loaded = '1' where api_id = '057169de-4aa7-4f73-8894-221b4fd0';INSERT INTO data_api.api_data (id,api_id,data,total_num,is_loaded,status,request_tm,execute_tm,remark) values (%s,%s,%s,%s,'0','0',current_timestamp(0),current_timestamp(0),'')"
|
||||
cur.execute(sql,[dataId,'057169de-4aa7-4f73-8894-221b4fd0',json_object,total])
|
||||
conn.commit()
|
||||
cur.close()
|
||||
conn.close()
|
||||
print('加载数据结束：customer_event_meta:获取线索事件元数据')
|
||||
|
|
@ -0,0 +1,33 @@
|
|||
/*******Main Section**************************************************************************/
|
||||
\set ON_ERROR_STOP on
|
||||
\set AUTOCOMMIT on
|
||||
\timing on
|
||||
|
||||
DELETE FROM data_api.customer_event_meta;
|
||||
|
||||
insert into data_api.customer_event_meta (
|
||||
group_name
|
||||
, group_label
|
||||
, events
|
||||
,etl_tx_dt
|
||||
)
|
||||
select
|
||||
case when trim(both from group_name)='' then null else group_name::text end group_name
|
||||
, case when trim(both from group_label)='' then null else group_label::text end group_label
|
||||
, case when trim(both from events)='' then null else events::text end events
|
||||
,etl_tx_dt
|
||||
from (
|
||||
select
|
||||
(json_array_elements(data::json)::json->>'groupName') group_name
|
||||
, (json_array_elements(data::json)::json->>'groupLabel') group_label
|
||||
, (json_array_elements(data::json)::json->>'events') events
|
||||
,CURRENT_TIMESTAMP(0) etl_tx_dt
|
||||
from (select * from data_api.api_data
|
||||
WHERE api_id='057169de-4aa7-4f73-8894-221b4fd0' and is_loaded = '0' order by request_tm desc limit 1) p )p;
|
||||
|
||||
update data_api.api_data
|
||||
set is_loaded = '1' ,
|
||||
status = '1',
|
||||
request_tm = current_timestamp(0)
|
||||
where api_id='057169de-4aa7-4f73-8894-221b4fd0';
|
||||
\q
|
||||
Loading…
Reference in New Issue