add workflow 荟聚API,dev
This commit is contained in:
root
parent
81ef45ac5d
commit
dfbb29e94c
|
|
@ -1,28 +1,23 @@
|
|||
|
||||
drop foreign table if exists p00_tal.S98_S_customer_events_add_user;
|
||||
CREATE FOREIGN TABLE if not exists p00_tal.S98_S_customer_events_add_user (
|
||||
attr1 TEXT
|
||||
, attr2 TEXT
|
||||
, attr5 TEXT
|
||||
, attr6 TEXT
|
||||
, attr7 TEXT
|
||||
, customer_id TEXT
|
||||
, customer_id_str TEXT
|
||||
, date TEXT
|
||||
, event TEXT
|
||||
, external_id TEXT
|
||||
, id TEXT
|
||||
, id_str TEXT
|
||||
, last_updated TEXT
|
||||
, source TEXT
|
||||
, summary TEXT
|
||||
, target_id TEXT
|
||||
attr1 TEXT
|
||||
, attr2 TEXT
|
||||
, attr5 TEXT
|
||||
, attr6 TEXT
|
||||
, attr7 TEXT
|
||||
, customer_id TEXT
|
||||
, customer_id_str TEXT
|
||||
, date TEXT
|
||||
, event TEXT
|
||||
, external_id TEXT
|
||||
, id TEXT
|
||||
, id_str TEXT
|
||||
, last_updated TEXT
|
||||
, source TEXT
|
||||
, summary TEXT
|
||||
, target_id TEXT
|
||||
, etl_tx_dt TIMESTAMP
|
||||
)
|
||||
|
||||
|
||||
SERVER pgsql_server_S98_S OPTIONS(schema_name 'data_api', table_name 'customer_events_add_user' );
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
SERVER pgsql_server_S98_S OPTIONS(schema_name 'data_api', table_name 'customer_events_add_user' );
|
||||
|
|
@ -1,83 +1,82 @@
|
|||
|
||||
drop table if exists p10_sa.S98_S_customer_events_add_user;
|
||||
create table if not exists p10_sa.S98_S_customer_events_add_user (
|
||||
attr1 TEXT
|
||||
, attr2 TEXT
|
||||
, attr5 TEXT
|
||||
, attr6 TEXT
|
||||
, attr7 TEXT
|
||||
, customer_id TEXT
|
||||
, customer_id_str TEXT
|
||||
, date TEXT
|
||||
, event TEXT
|
||||
, external_id TEXT
|
||||
, id TEXT
|
||||
, id_str TEXT
|
||||
, last_updated TEXT
|
||||
, source TEXT
|
||||
, summary TEXT
|
||||
, target_id TEXT
|
||||
attr1 TEXT
|
||||
, attr2 TEXT
|
||||
, attr5 TEXT
|
||||
, attr6 TEXT
|
||||
, attr7 TEXT
|
||||
, customer_id TEXT
|
||||
, customer_id_str TEXT
|
||||
, date TEXT
|
||||
, event TEXT
|
||||
, external_id TEXT
|
||||
, id TEXT
|
||||
, id_str TEXT
|
||||
, last_updated TEXT
|
||||
, source TEXT
|
||||
, summary TEXT
|
||||
, target_id TEXT
|
||||
, etl_tx_dt TIMESTAMP
|
||||
) ;
|
||||
|
||||
COMMENT ON COLUMN p10_sa.S98_S_customer_events_add_user.attr1 IS '';
|
||||
COMMENT ON COLUMN p10_sa.S98_S_customer_events_add_user.attr2 IS '';
|
||||
COMMENT ON COLUMN p10_sa.S98_S_customer_events_add_user.attr5 IS '';
|
||||
COMMENT ON COLUMN p10_sa.S98_S_customer_events_add_user.attr6 IS '';
|
||||
COMMENT ON COLUMN p10_sa.S98_S_customer_events_add_user.attr7 IS '';
|
||||
COMMENT ON COLUMN p10_sa.S98_S_customer_events_add_user.customer_id IS '';
|
||||
COMMENT ON COLUMN p10_sa.S98_S_customer_events_add_user.customer_id_str IS '';
|
||||
COMMENT ON COLUMN p10_sa.S98_S_customer_events_add_user.date IS '';
|
||||
COMMENT ON COLUMN p10_sa.S98_S_customer_events_add_user.event IS '';
|
||||
COMMENT ON COLUMN p10_sa.S98_S_customer_events_add_user.external_id IS '';
|
||||
COMMENT ON COLUMN p10_sa.S98_S_customer_events_add_user.id IS '';
|
||||
COMMENT ON COLUMN p10_sa.S98_S_customer_events_add_user.id_str IS '';
|
||||
COMMENT ON COLUMN p10_sa.S98_S_customer_events_add_user.last_updated IS '';
|
||||
COMMENT ON COLUMN p10_sa.S98_S_customer_events_add_user.source IS '';
|
||||
COMMENT ON COLUMN p10_sa.S98_S_customer_events_add_user.summary IS '';
|
||||
COMMENT ON COLUMN p10_sa.S98_S_customer_events_add_user.target_id IS '';
|
||||
COMMENT ON COLUMN p10_sa.S98_S_customer_events_add_user.etl_tx_dt IS '';
|
||||
COMMENT ON COLUMN p10_sa.S98_S_customer_events_add_user.attr1 IS '';
|
||||
COMMENT ON COLUMN p10_sa.S98_S_customer_events_add_user.attr2 IS '';
|
||||
COMMENT ON COLUMN p10_sa.S98_S_customer_events_add_user.attr5 IS '';
|
||||
COMMENT ON COLUMN p10_sa.S98_S_customer_events_add_user.attr6 IS '';
|
||||
COMMENT ON COLUMN p10_sa.S98_S_customer_events_add_user.attr7 IS '';
|
||||
COMMENT ON COLUMN p10_sa.S98_S_customer_events_add_user.customer_id IS '';
|
||||
COMMENT ON COLUMN p10_sa.S98_S_customer_events_add_user.customer_id_str IS '';
|
||||
COMMENT ON COLUMN p10_sa.S98_S_customer_events_add_user.date IS '';
|
||||
COMMENT ON COLUMN p10_sa.S98_S_customer_events_add_user.event IS '';
|
||||
COMMENT ON COLUMN p10_sa.S98_S_customer_events_add_user.external_id IS '';
|
||||
COMMENT ON COLUMN p10_sa.S98_S_customer_events_add_user.id IS '';
|
||||
COMMENT ON COLUMN p10_sa.S98_S_customer_events_add_user.id_str IS '';
|
||||
COMMENT ON COLUMN p10_sa.S98_S_customer_events_add_user.last_updated IS '';
|
||||
COMMENT ON COLUMN p10_sa.S98_S_customer_events_add_user.source IS '';
|
||||
COMMENT ON COLUMN p10_sa.S98_S_customer_events_add_user.summary IS '';
|
||||
COMMENT ON COLUMN p10_sa.S98_S_customer_events_add_user.target_id IS '';
|
||||
COMMENT ON COLUMN p10_sa.S98_S_customer_events_add_user.etl_tx_dt IS '';
|
||||
|
||||
COMMENT ON TABLE p10_sa.S98_S_customer_events_add_user IS '';
|
||||
|
||||
|
||||
|
||||
drop table if exists p12_sfull.S98_S_customer_events_add_user;
|
||||
create table if not exists p12_sfull.S98_S_customer_events_add_user (
|
||||
attr1 TEXT
|
||||
, attr2 TEXT
|
||||
, attr5 TEXT
|
||||
, attr6 TEXT
|
||||
, attr7 TEXT
|
||||
, customer_id TEXT
|
||||
, customer_id_str TEXT
|
||||
, date TEXT
|
||||
, event TEXT
|
||||
, external_id TEXT
|
||||
, id TEXT
|
||||
, id_str TEXT
|
||||
, last_updated TEXT
|
||||
, source TEXT
|
||||
, summary TEXT
|
||||
, target_id TEXT
|
||||
attr1 TEXT
|
||||
, attr2 TEXT
|
||||
, attr5 TEXT
|
||||
, attr6 TEXT
|
||||
, attr7 TEXT
|
||||
, customer_id TEXT
|
||||
, customer_id_str TEXT
|
||||
, date TEXT
|
||||
, event TEXT
|
||||
, external_id TEXT
|
||||
, id TEXT
|
||||
, id_str TEXT
|
||||
, last_updated TEXT
|
||||
, source TEXT
|
||||
, summary TEXT
|
||||
, target_id TEXT
|
||||
, etl_tx_dt TIMESTAMP
|
||||
) ;
|
||||
|
||||
COMMENT ON COLUMN p12_sfull.S98_S_customer_events_add_user.attr1 IS '';
|
||||
COMMENT ON COLUMN p12_sfull.S98_S_customer_events_add_user.attr2 IS '';
|
||||
COMMENT ON COLUMN p12_sfull.S98_S_customer_events_add_user.attr5 IS '';
|
||||
COMMENT ON COLUMN p12_sfull.S98_S_customer_events_add_user.attr6 IS '';
|
||||
COMMENT ON COLUMN p12_sfull.S98_S_customer_events_add_user.attr7 IS '';
|
||||
COMMENT ON COLUMN p12_sfull.S98_S_customer_events_add_user.customer_id IS '';
|
||||
COMMENT ON COLUMN p12_sfull.S98_S_customer_events_add_user.customer_id_str IS '';
|
||||
COMMENT ON COLUMN p12_sfull.S98_S_customer_events_add_user.date IS '';
|
||||
COMMENT ON COLUMN p12_sfull.S98_S_customer_events_add_user.event IS '';
|
||||
COMMENT ON COLUMN p12_sfull.S98_S_customer_events_add_user.external_id IS '';
|
||||
COMMENT ON COLUMN p12_sfull.S98_S_customer_events_add_user.id IS '';
|
||||
COMMENT ON COLUMN p12_sfull.S98_S_customer_events_add_user.id_str IS '';
|
||||
COMMENT ON COLUMN p12_sfull.S98_S_customer_events_add_user.last_updated IS '';
|
||||
COMMENT ON COLUMN p12_sfull.S98_S_customer_events_add_user.source IS '';
|
||||
COMMENT ON COLUMN p12_sfull.S98_S_customer_events_add_user.summary IS '';
|
||||
COMMENT ON COLUMN p12_sfull.S98_S_customer_events_add_user.target_id IS '';
|
||||
COMMENT ON COLUMN p12_sfull.S98_S_customer_events_add_user.etl_tx_dt IS '';
|
||||
COMMENT ON COLUMN p12_sfull.S98_S_customer_events_add_user.attr1 IS '';
|
||||
COMMENT ON COLUMN p12_sfull.S98_S_customer_events_add_user.attr2 IS '';
|
||||
COMMENT ON COLUMN p12_sfull.S98_S_customer_events_add_user.attr5 IS '';
|
||||
COMMENT ON COLUMN p12_sfull.S98_S_customer_events_add_user.attr6 IS '';
|
||||
COMMENT ON COLUMN p12_sfull.S98_S_customer_events_add_user.attr7 IS '';
|
||||
COMMENT ON COLUMN p12_sfull.S98_S_customer_events_add_user.customer_id IS '';
|
||||
COMMENT ON COLUMN p12_sfull.S98_S_customer_events_add_user.customer_id_str IS '';
|
||||
COMMENT ON COLUMN p12_sfull.S98_S_customer_events_add_user.date IS '';
|
||||
COMMENT ON COLUMN p12_sfull.S98_S_customer_events_add_user.event IS '';
|
||||
COMMENT ON COLUMN p12_sfull.S98_S_customer_events_add_user.external_id IS '';
|
||||
COMMENT ON COLUMN p12_sfull.S98_S_customer_events_add_user.id IS '';
|
||||
COMMENT ON COLUMN p12_sfull.S98_S_customer_events_add_user.id_str IS '';
|
||||
COMMENT ON COLUMN p12_sfull.S98_S_customer_events_add_user.last_updated IS '';
|
||||
COMMENT ON COLUMN p12_sfull.S98_S_customer_events_add_user.source IS '';
|
||||
COMMENT ON COLUMN p12_sfull.S98_S_customer_events_add_user.summary IS '';
|
||||
COMMENT ON COLUMN p12_sfull.S98_S_customer_events_add_user.target_id IS '';
|
||||
COMMENT ON COLUMN p12_sfull.S98_S_customer_events_add_user.etl_tx_dt IS '';
|
||||
|
||||
COMMENT ON TABLE p12_sfull.S98_S_customer_events_add_user IS '';
|
||||
|
||||
COMMENT ON TABLE p12_sfull.S98_S_customer_events_add_user IS '';
|
||||
|
|
@ -1,137 +1,137 @@
|
|||
DROP FOREIGN TABLE if exists p00_tal.S98_S_scrm_contact;
|
||||
|
||||
CREATE FOREIGN TABLE if not exists p00_tal.S98_S_scrm_contact (
|
||||
accept_email TEXT
|
||||
, acceptsms TEXT
|
||||
, annual_revenue TEXT
|
||||
, attr_id TEXT
|
||||
, attr1 TEXT
|
||||
, attr10 TEXT
|
||||
, attr11 TEXT
|
||||
, attr12 TEXT
|
||||
, attr13 TEXT
|
||||
, attr14 TEXT
|
||||
, attr15 TEXT
|
||||
, attr16 TEXT
|
||||
, attr17 TEXT
|
||||
, attr18 TEXT
|
||||
, attr19 TEXT
|
||||
, attr2 TEXT
|
||||
, attr20 TEXT
|
||||
, attr21 TEXT
|
||||
, attr22 TEXT
|
||||
, attr23 TEXT
|
||||
, attr24 TEXT
|
||||
, attr25 TEXT
|
||||
, attr26 TEXT
|
||||
, attr27 TEXT
|
||||
, attr28 TEXT
|
||||
, attr3 TEXT
|
||||
, attr4 TEXT
|
||||
, attr5 TEXT
|
||||
, attr6 TEXT
|
||||
, attr7 TEXT
|
||||
, attr8 TEXT
|
||||
, attr9 TEXT
|
||||
, birthday TEXT
|
||||
, campaign_id TEXT
|
||||
, campaign_name TEXT
|
||||
, city TEXT
|
||||
, company TEXT
|
||||
, content_name TEXT
|
||||
, convert_date TEXT
|
||||
, country TEXT
|
||||
, county TEXT
|
||||
, create_from TEXT
|
||||
, create_from_name TEXT
|
||||
, create_method TEXT
|
||||
, date_created TEXT
|
||||
, date_join TEXT
|
||||
, date_member_reg TEXT
|
||||
, display_name TEXT
|
||||
, email TEXT
|
||||
, email_verified TEXT
|
||||
, employee_number TEXT
|
||||
, external_id TEXT
|
||||
, extra_convert_content_name TEXT
|
||||
, extra_convert_date TEXT
|
||||
, extra_convert_source TEXT
|
||||
, extra_convert_source1 TEXT
|
||||
, extra_convert_source2 TEXT
|
||||
, extra_convert_source3 TEXT
|
||||
, extra_convert_source4 TEXT
|
||||
, extra_convert_source5 TEXT
|
||||
, extra_convert_source6 TEXT
|
||||
, extra_convert_source7 TEXT
|
||||
, extra_convert_source8 TEXT
|
||||
, extra_regist_content_name TEXT
|
||||
, extra_regist_date TEXT
|
||||
, extra_regist_source TEXT
|
||||
, extra_regist_source1 TEXT
|
||||
, extra_regist_source2 TEXT
|
||||
, extra_regist_source3 TEXT
|
||||
, extra_regist_source4 TEXT
|
||||
, extra_regist_source5 TEXT
|
||||
, extra_regist_source6 TEXT
|
||||
, extra_regist_source7 TEXT
|
||||
, extra_regist_source8 TEXT
|
||||
, first_assign_date TEXT
|
||||
, gender TEXT
|
||||
, home_address TEXT
|
||||
, id TEXT
|
||||
, id_str TEXT
|
||||
, img TEXT
|
||||
, industry TEXT
|
||||
, is_anonymous TEXT
|
||||
, is_employee TEXT
|
||||
, is_member TEXT
|
||||
, is_referrer TEXT
|
||||
, language TEXT
|
||||
, last_updated TEXT
|
||||
, latest_engage_date TEXT
|
||||
, mobile TEXT
|
||||
, mobile_verified TEXT
|
||||
, name TEXT
|
||||
, nick_name TEXT
|
||||
, not_accept_email TEXT
|
||||
, not_acceptsms TEXT
|
||||
, office_address TEXT
|
||||
, owner_date TEXT
|
||||
, owner_employee_id TEXT
|
||||
, owner_employee_name TEXT
|
||||
, owner_id TEXT
|
||||
, owner_name TEXT
|
||||
, province TEXT
|
||||
, qq TEXT
|
||||
, referrer TEXT
|
||||
, referrer_name TEXT
|
||||
, refund_date TEXT
|
||||
, region TEXT
|
||||
, remark TEXT
|
||||
, source TEXT
|
||||
, source1 TEXT
|
||||
, source2 TEXT
|
||||
, source3 TEXT
|
||||
, source4 TEXT
|
||||
, source5 TEXT
|
||||
, source6 TEXT
|
||||
, source7 TEXT
|
||||
, source8 TEXT
|
||||
, stage TEXT
|
||||
, status TEXT
|
||||
, street TEXT
|
||||
, subscribe_scene TEXT
|
||||
, telephone TEXT
|
||||
, title TEXT
|
||||
, type TEXT
|
||||
, update_method TEXT
|
||||
, utm_medium TEXT
|
||||
, utm_term TEXT
|
||||
, version TEXT
|
||||
, website TEXT
|
||||
, wechat TEXT
|
||||
, wechat_city TEXT
|
||||
, wechat_nick_name TEXT
|
||||
, weibo TEXT
|
||||
accept_email TEXT
|
||||
, acceptsms TEXT
|
||||
, annual_revenue TEXT
|
||||
, attr_id TEXT
|
||||
, attr1 TEXT
|
||||
, attr10 TEXT
|
||||
, attr11 TEXT
|
||||
, attr12 TEXT
|
||||
, attr13 TEXT
|
||||
, attr14 TEXT
|
||||
, attr15 TEXT
|
||||
, attr16 TEXT
|
||||
, attr17 TEXT
|
||||
, attr18 TEXT
|
||||
, attr19 TEXT
|
||||
, attr2 TEXT
|
||||
, attr20 TEXT
|
||||
, attr21 TEXT
|
||||
, attr22 TEXT
|
||||
, attr23 TEXT
|
||||
, attr24 TEXT
|
||||
, attr25 TEXT
|
||||
, attr26 TEXT
|
||||
, attr27 TEXT
|
||||
, attr28 TEXT
|
||||
, attr3 TEXT
|
||||
, attr4 TEXT
|
||||
, attr5 TEXT
|
||||
, attr6 TEXT
|
||||
, attr7 TEXT
|
||||
, attr8 TEXT
|
||||
, attr9 TEXT
|
||||
, birthday TEXT
|
||||
, campaign_id TEXT
|
||||
, campaign_name TEXT
|
||||
, city TEXT
|
||||
, company TEXT
|
||||
, content_name TEXT
|
||||
, convert_date TEXT
|
||||
, country TEXT
|
||||
, county TEXT
|
||||
, create_from TEXT
|
||||
, create_from_name TEXT
|
||||
, create_method TEXT
|
||||
, date_created TEXT
|
||||
, date_join TEXT
|
||||
, date_member_reg TEXT
|
||||
, display_name TEXT
|
||||
, email TEXT
|
||||
, email_verified TEXT
|
||||
, employee_number TEXT
|
||||
, external_id TEXT
|
||||
, extra_convert_content_name TEXT
|
||||
, extra_convert_date TEXT
|
||||
, extra_convert_source TEXT
|
||||
, extra_convert_source1 TEXT
|
||||
, extra_convert_source2 TEXT
|
||||
, extra_convert_source3 TEXT
|
||||
, extra_convert_source4 TEXT
|
||||
, extra_convert_source5 TEXT
|
||||
, extra_convert_source6 TEXT
|
||||
, extra_convert_source7 TEXT
|
||||
, extra_convert_source8 TEXT
|
||||
, extra_regist_content_name TEXT
|
||||
, extra_regist_date TEXT
|
||||
, extra_regist_source TEXT
|
||||
, extra_regist_source1 TEXT
|
||||
, extra_regist_source2 TEXT
|
||||
, extra_regist_source3 TEXT
|
||||
, extra_regist_source4 TEXT
|
||||
, extra_regist_source5 TEXT
|
||||
, extra_regist_source6 TEXT
|
||||
, extra_regist_source7 TEXT
|
||||
, extra_regist_source8 TEXT
|
||||
, first_assign_date TEXT
|
||||
, gender TEXT
|
||||
, home_address TEXT
|
||||
, id TEXT
|
||||
, id_str TEXT
|
||||
, img TEXT
|
||||
, industry TEXT
|
||||
, is_anonymous TEXT
|
||||
, is_employee TEXT
|
||||
, is_member TEXT
|
||||
, is_referrer TEXT
|
||||
, language TEXT
|
||||
, last_updated TEXT
|
||||
, latest_engage_date TEXT
|
||||
, mobile TEXT
|
||||
, mobile_verified TEXT
|
||||
, name TEXT
|
||||
, nick_name TEXT
|
||||
, not_accept_email TEXT
|
||||
, not_acceptsms TEXT
|
||||
, office_address TEXT
|
||||
, owner_date TEXT
|
||||
, owner_employee_id TEXT
|
||||
, owner_employee_name TEXT
|
||||
, owner_id TEXT
|
||||
, owner_name TEXT
|
||||
, province TEXT
|
||||
, qq TEXT
|
||||
, referrer TEXT
|
||||
, referrer_name TEXT
|
||||
, refund_date TEXT
|
||||
, region TEXT
|
||||
, remark TEXT
|
||||
, source TEXT
|
||||
, source1 TEXT
|
||||
, source2 TEXT
|
||||
, source3 TEXT
|
||||
, source4 TEXT
|
||||
, source5 TEXT
|
||||
, source6 TEXT
|
||||
, source7 TEXT
|
||||
, source8 TEXT
|
||||
, stage TEXT
|
||||
, status TEXT
|
||||
, street TEXT
|
||||
, subscribe_scene TEXT
|
||||
, telephone TEXT
|
||||
, title TEXT
|
||||
, type TEXT
|
||||
, update_method TEXT
|
||||
, utm_medium TEXT
|
||||
, utm_term TEXT
|
||||
, version TEXT
|
||||
, website TEXT
|
||||
, wechat TEXT
|
||||
, wechat_city TEXT
|
||||
, wechat_nick_name TEXT
|
||||
, weibo TEXT
|
||||
, etl_tx_dt TIMESTAMP
|
||||
)
|
||||
|
||||
|
|
@ -141,4 +141,4 @@ SERVER pgsql_server_S98_S OPTIONS(schema_name 'data_api', table_name 'scrm_conta
|
|||
|
||||
|
||||
|
||||
|
||||
|
||||
|
|
|
|||
File diff suppressed because it is too large
Load Diff
|
|
@ -0,0 +1,158 @@
|
|||
/*******Main Section**************************************************************************/
|
||||
\set ON_ERROR_STOP on
|
||||
\set AUTOCOMMIT on
|
||||
\timing on
|
||||
delete from p10_sa.S98_S_scrm_contact_merge
|
||||
;
|
||||
insert into p10_sa.S98_S_scrm_contact_merge
|
||||
( mergedcustomerid
|
||||
, id
|
||||
, customer_id
|
||||
, date
|
||||
, event
|
||||
, target_name
|
||||
, last_updated
|
||||
, c_name
|
||||
, c_type
|
||||
, contentname
|
||||
, source
|
||||
, tag
|
||||
, c_keyword
|
||||
, attr2
|
||||
, screen_width
|
||||
, ip_city
|
||||
, ip_country
|
||||
, platform
|
||||
, page_type
|
||||
, os_version
|
||||
, browser
|
||||
, browser_version
|
||||
, id_str
|
||||
, os
|
||||
, target_id
|
||||
, customer_id_str
|
||||
, ip
|
||||
, screen_height
|
||||
, external_id
|
||||
, ip_county
|
||||
, page_id
|
||||
, url
|
||||
, ip_province
|
||||
, device
|
||||
, etl_tx_dt )
|
||||
select
|
||||
mergedcustomerid
|
||||
, id
|
||||
, customer_id
|
||||
, date
|
||||
, event
|
||||
, target_name
|
||||
, last_updated
|
||||
, c_name
|
||||
, c_type
|
||||
, contentname
|
||||
, source
|
||||
, tag
|
||||
, c_keyword
|
||||
, attr2
|
||||
, screen_width
|
||||
, ip_city
|
||||
, ip_country
|
||||
, platform
|
||||
, page_type
|
||||
, os_version
|
||||
, browser
|
||||
, browser_version
|
||||
, id_str
|
||||
, os
|
||||
, target_id
|
||||
, customer_id_str
|
||||
, ip
|
||||
, screen_height
|
||||
, external_id
|
||||
, ip_county
|
||||
, page_id
|
||||
, url
|
||||
, ip_province
|
||||
, device
|
||||
, etl_tx_dt
|
||||
from p00_tal.S98_S_scrm_contact_merge
|
||||
;
|
||||
delete from p12_sfull.S98_S_scrm_contact_merge
|
||||
;
|
||||
;
|
||||
insert into p12_sfull.S98_S_scrm_contact_merge
|
||||
( mergedcustomerid
|
||||
, id
|
||||
, customer_id
|
||||
, date
|
||||
, event
|
||||
, target_name
|
||||
, last_updated
|
||||
, c_name
|
||||
, c_type
|
||||
, contentname
|
||||
, source
|
||||
, tag
|
||||
, c_keyword
|
||||
, attr2
|
||||
, screen_width
|
||||
, ip_city
|
||||
, ip_country
|
||||
, platform
|
||||
, page_type
|
||||
, os_version
|
||||
, browser
|
||||
, browser_version
|
||||
, id_str
|
||||
, os
|
||||
, target_id
|
||||
, customer_id_str
|
||||
, ip
|
||||
, screen_height
|
||||
, external_id
|
||||
, ip_county
|
||||
, page_id
|
||||
, url
|
||||
, ip_province
|
||||
, device
|
||||
, etl_tx_dt )
|
||||
select
|
||||
mergedcustomerid
|
||||
, id
|
||||
, customer_id
|
||||
, date
|
||||
, event
|
||||
, target_name
|
||||
, last_updated
|
||||
, c_name
|
||||
, c_type
|
||||
, contentname
|
||||
, source
|
||||
, tag
|
||||
, c_keyword
|
||||
, attr2
|
||||
, screen_width
|
||||
, ip_city
|
||||
, ip_country
|
||||
, platform
|
||||
, page_type
|
||||
, os_version
|
||||
, browser
|
||||
, browser_version
|
||||
, id_str
|
||||
, os
|
||||
, target_id
|
||||
, customer_id_str
|
||||
, ip
|
||||
, screen_height
|
||||
, external_id
|
||||
, ip_county
|
||||
, page_id
|
||||
, url
|
||||
, ip_province
|
||||
, device
|
||||
, etl_tx_dt
|
||||
from p10_sa.S98_S_scrm_contact_merge
|
||||
;
|
||||
\q
|
||||
|
|
@ -0,0 +1,46 @@
|
|||
|
||||
CREATE FOREIGN TABLE if not exists p00_tal.S98_S_scrm_contact_merge (
|
||||
mergedcustomerid TEXT
|
||||
, id TEXT
|
||||
, customer_id TEXT
|
||||
, date TEXT
|
||||
, event TEXT
|
||||
, target_name TEXT
|
||||
, last_updated TEXT
|
||||
, c_name TEXT
|
||||
, c_type TEXT
|
||||
, contentname TEXT
|
||||
, source TEXT
|
||||
, tag TEXT
|
||||
, c_keyword TEXT
|
||||
, attr2 TEXT
|
||||
, screen_width TEXT
|
||||
, ip_city TEXT
|
||||
, ip_country TEXT
|
||||
, platform TEXT
|
||||
, page_type TEXT
|
||||
, os_version TEXT
|
||||
, browser TEXT
|
||||
, browser_version TEXT
|
||||
, id_str TEXT
|
||||
, os TEXT
|
||||
, target_id TEXT
|
||||
, customer_id_str TEXT
|
||||
, ip TEXT
|
||||
, screen_height TEXT
|
||||
, external_id TEXT
|
||||
, ip_county TEXT
|
||||
, page_id TEXT
|
||||
, url TEXT
|
||||
, ip_province TEXT
|
||||
, device TEXT
|
||||
, etl_tx_dt TIMESTAMP
|
||||
)
|
||||
|
||||
|
||||
SERVER pgsql_server_S98_S OPTIONS(schema_name 'data_api', table_name 'scrm_contact_merge' );
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
|
@ -0,0 +1,155 @@
|
|||
|
||||
create table if not exists p10_sa.S98_S_scrm_contact_merge (
|
||||
mergedcustomerid TEXT
|
||||
, id TEXT
|
||||
, customer_id TEXT
|
||||
, date TEXT
|
||||
, event TEXT
|
||||
, target_name TEXT
|
||||
, last_updated TEXT
|
||||
, c_name TEXT
|
||||
, c_type TEXT
|
||||
, contentname TEXT
|
||||
, source TEXT
|
||||
, tag TEXT
|
||||
, c_keyword TEXT
|
||||
, attr2 TEXT
|
||||
, screen_width TEXT
|
||||
, ip_city TEXT
|
||||
, ip_country TEXT
|
||||
, platform TEXT
|
||||
, page_type TEXT
|
||||
, os_version TEXT
|
||||
, browser TEXT
|
||||
, browser_version TEXT
|
||||
, id_str TEXT
|
||||
, os TEXT
|
||||
, target_id TEXT
|
||||
, customer_id_str TEXT
|
||||
, ip TEXT
|
||||
, screen_height TEXT
|
||||
, external_id TEXT
|
||||
, ip_county TEXT
|
||||
, page_id TEXT
|
||||
, url TEXT
|
||||
, ip_province TEXT
|
||||
, device TEXT
|
||||
, etl_tx_dt TIMESTAMP
|
||||
) ;
|
||||
|
||||
COMMENT ON COLUMN p10_sa.S98_S_scrm_contact_merge.mergedcustomerid IS '';
|
||||
COMMENT ON COLUMN p10_sa.S98_S_scrm_contact_merge.id IS '';
|
||||
COMMENT ON COLUMN p10_sa.S98_S_scrm_contact_merge.customer_id IS '';
|
||||
COMMENT ON COLUMN p10_sa.S98_S_scrm_contact_merge.date IS '';
|
||||
COMMENT ON COLUMN p10_sa.S98_S_scrm_contact_merge.event IS '';
|
||||
COMMENT ON COLUMN p10_sa.S98_S_scrm_contact_merge.target_name IS '';
|
||||
COMMENT ON COLUMN p10_sa.S98_S_scrm_contact_merge.last_updated IS '';
|
||||
COMMENT ON COLUMN p10_sa.S98_S_scrm_contact_merge.c_name IS '';
|
||||
COMMENT ON COLUMN p10_sa.S98_S_scrm_contact_merge.c_type IS '';
|
||||
COMMENT ON COLUMN p10_sa.S98_S_scrm_contact_merge.contentname IS '';
|
||||
COMMENT ON COLUMN p10_sa.S98_S_scrm_contact_merge.source IS '';
|
||||
COMMENT ON COLUMN p10_sa.S98_S_scrm_contact_merge.tag IS '';
|
||||
COMMENT ON COLUMN p10_sa.S98_S_scrm_contact_merge.c_keyword IS '';
|
||||
COMMENT ON COLUMN p10_sa.S98_S_scrm_contact_merge.attr2 IS '';
|
||||
COMMENT ON COLUMN p10_sa.S98_S_scrm_contact_merge.screen_width IS '';
|
||||
COMMENT ON COLUMN p10_sa.S98_S_scrm_contact_merge.ip_city IS '';
|
||||
COMMENT ON COLUMN p10_sa.S98_S_scrm_contact_merge.ip_country IS '';
|
||||
COMMENT ON COLUMN p10_sa.S98_S_scrm_contact_merge.platform IS '';
|
||||
COMMENT ON COLUMN p10_sa.S98_S_scrm_contact_merge.page_type IS '';
|
||||
COMMENT ON COLUMN p10_sa.S98_S_scrm_contact_merge.os_version IS '';
|
||||
COMMENT ON COLUMN p10_sa.S98_S_scrm_contact_merge.browser IS '';
|
||||
COMMENT ON COLUMN p10_sa.S98_S_scrm_contact_merge.browser_version IS '';
|
||||
COMMENT ON COLUMN p10_sa.S98_S_scrm_contact_merge.id_str IS '';
|
||||
COMMENT ON COLUMN p10_sa.S98_S_scrm_contact_merge.os IS '';
|
||||
COMMENT ON COLUMN p10_sa.S98_S_scrm_contact_merge.target_id IS '';
|
||||
COMMENT ON COLUMN p10_sa.S98_S_scrm_contact_merge.customer_id_str IS '';
|
||||
COMMENT ON COLUMN p10_sa.S98_S_scrm_contact_merge.ip IS '';
|
||||
COMMENT ON COLUMN p10_sa.S98_S_scrm_contact_merge.screen_height IS '';
|
||||
COMMENT ON COLUMN p10_sa.S98_S_scrm_contact_merge.external_id IS '';
|
||||
COMMENT ON COLUMN p10_sa.S98_S_scrm_contact_merge.ip_county IS '';
|
||||
COMMENT ON COLUMN p10_sa.S98_S_scrm_contact_merge.page_id IS '';
|
||||
COMMENT ON COLUMN p10_sa.S98_S_scrm_contact_merge.url IS '';
|
||||
COMMENT ON COLUMN p10_sa.S98_S_scrm_contact_merge.ip_province IS '';
|
||||
COMMENT ON COLUMN p10_sa.S98_S_scrm_contact_merge.device IS '';
|
||||
COMMENT ON COLUMN p10_sa.S98_S_scrm_contact_merge.etl_tx_dt IS '';
|
||||
|
||||
COMMENT ON TABLE p10_sa.S98_S_scrm_contact_merge IS '';
|
||||
|
||||
|
||||
|
||||
create table if not exists p12_sfull.S98_S_scrm_contact_merge (
|
||||
mergedcustomerid TEXT
|
||||
, id TEXT
|
||||
, customer_id TEXT
|
||||
, date TEXT
|
||||
, event TEXT
|
||||
, target_name TEXT
|
||||
, last_updated TEXT
|
||||
, c_name TEXT
|
||||
, c_type TEXT
|
||||
, contentname TEXT
|
||||
, source TEXT
|
||||
, tag TEXT
|
||||
, c_keyword TEXT
|
||||
, attr2 TEXT
|
||||
, screen_width TEXT
|
||||
, ip_city TEXT
|
||||
, ip_country TEXT
|
||||
, platform TEXT
|
||||
, page_type TEXT
|
||||
, os_version TEXT
|
||||
, browser TEXT
|
||||
, browser_version TEXT
|
||||
, id_str TEXT
|
||||
, os TEXT
|
||||
, target_id TEXT
|
||||
, customer_id_str TEXT
|
||||
, ip TEXT
|
||||
, screen_height TEXT
|
||||
, external_id TEXT
|
||||
, ip_county TEXT
|
||||
, page_id TEXT
|
||||
, url TEXT
|
||||
, ip_province TEXT
|
||||
, device TEXT
|
||||
, etl_tx_dt TIMESTAMP
|
||||
) ;
|
||||
|
||||
COMMENT ON COLUMN p12_sfull.S98_S_scrm_contact_merge.mergedcustomerid IS '';
|
||||
COMMENT ON COLUMN p12_sfull.S98_S_scrm_contact_merge.id IS '';
|
||||
COMMENT ON COLUMN p12_sfull.S98_S_scrm_contact_merge.customer_id IS '';
|
||||
COMMENT ON COLUMN p12_sfull.S98_S_scrm_contact_merge.date IS '';
|
||||
COMMENT ON COLUMN p12_sfull.S98_S_scrm_contact_merge.event IS '';
|
||||
COMMENT ON COLUMN p12_sfull.S98_S_scrm_contact_merge.target_name IS '';
|
||||
COMMENT ON COLUMN p12_sfull.S98_S_scrm_contact_merge.last_updated IS '';
|
||||
COMMENT ON COLUMN p12_sfull.S98_S_scrm_contact_merge.c_name IS '';
|
||||
COMMENT ON COLUMN p12_sfull.S98_S_scrm_contact_merge.c_type IS '';
|
||||
COMMENT ON COLUMN p12_sfull.S98_S_scrm_contact_merge.contentname IS '';
|
||||
COMMENT ON COLUMN p12_sfull.S98_S_scrm_contact_merge.source IS '';
|
||||
COMMENT ON COLUMN p12_sfull.S98_S_scrm_contact_merge.tag IS '';
|
||||
COMMENT ON COLUMN p12_sfull.S98_S_scrm_contact_merge.c_keyword IS '';
|
||||
COMMENT ON COLUMN p12_sfull.S98_S_scrm_contact_merge.attr2 IS '';
|
||||
COMMENT ON COLUMN p12_sfull.S98_S_scrm_contact_merge.screen_width IS '';
|
||||
COMMENT ON COLUMN p12_sfull.S98_S_scrm_contact_merge.ip_city IS '';
|
||||
COMMENT ON COLUMN p12_sfull.S98_S_scrm_contact_merge.ip_country IS '';
|
||||
COMMENT ON COLUMN p12_sfull.S98_S_scrm_contact_merge.platform IS '';
|
||||
COMMENT ON COLUMN p12_sfull.S98_S_scrm_contact_merge.page_type IS '';
|
||||
COMMENT ON COLUMN p12_sfull.S98_S_scrm_contact_merge.os_version IS '';
|
||||
COMMENT ON COLUMN p12_sfull.S98_S_scrm_contact_merge.browser IS '';
|
||||
COMMENT ON COLUMN p12_sfull.S98_S_scrm_contact_merge.browser_version IS '';
|
||||
COMMENT ON COLUMN p12_sfull.S98_S_scrm_contact_merge.id_str IS '';
|
||||
COMMENT ON COLUMN p12_sfull.S98_S_scrm_contact_merge.os IS '';
|
||||
COMMENT ON COLUMN p12_sfull.S98_S_scrm_contact_merge.target_id IS '';
|
||||
COMMENT ON COLUMN p12_sfull.S98_S_scrm_contact_merge.customer_id_str IS '';
|
||||
COMMENT ON COLUMN p12_sfull.S98_S_scrm_contact_merge.ip IS '';
|
||||
COMMENT ON COLUMN p12_sfull.S98_S_scrm_contact_merge.screen_height IS '';
|
||||
COMMENT ON COLUMN p12_sfull.S98_S_scrm_contact_merge.external_id IS '';
|
||||
COMMENT ON COLUMN p12_sfull.S98_S_scrm_contact_merge.ip_county IS '';
|
||||
COMMENT ON COLUMN p12_sfull.S98_S_scrm_contact_merge.page_id IS '';
|
||||
COMMENT ON COLUMN p12_sfull.S98_S_scrm_contact_merge.url IS '';
|
||||
COMMENT ON COLUMN p12_sfull.S98_S_scrm_contact_merge.ip_province IS '';
|
||||
COMMENT ON COLUMN p12_sfull.S98_S_scrm_contact_merge.device IS '';
|
||||
COMMENT ON COLUMN p12_sfull.S98_S_scrm_contact_merge.etl_tx_dt IS '';
|
||||
|
||||
COMMENT ON TABLE p12_sfull.S98_S_scrm_contact_merge IS '';
|
||||
|
||||
|
|
@ -51,7 +51,7 @@ while not auth and i < 60:
|
|||
print('开始请求数据总数。')
|
||||
url='https://api.huiju.cool/v2/customerEvents'
|
||||
header={}
|
||||
body={'access_token':auth['access_token'],'sort':'lastUpdated','lastUpdated[ge]':formatted_previous_date,'lastUpdated[le]':formatted_current_date,'limit':'',}
|
||||
body={'access_token':auth['access_token'],'limit':'','sort':'lastUpdated','lastUpdated[le]':formatted_current_date,'lastUpdated[ge]':formatted_previous_date,}
|
||||
dataReqL=requests.get(url,headers=header,params=body)
|
||||
resL=json.loads(dataReqL.text)
|
||||
# print(resL)
|
||||
|
|
|
|||
|
|
@ -3,24 +3,31 @@
|
|||
\set AUTOCOMMIT on
|
||||
\timing on
|
||||
|
||||
DELETE FROM data_api.customer_events_lastupdated;
|
||||
DELETE FROM data_api.scrm_contact_merge;
|
||||
|
||||
insert into data_api.customer_events_lastupdated (
|
||||
date
|
||||
, screen_width
|
||||
insert into data_api.scrm_contact_merge (
|
||||
mergedCustomerId
|
||||
, id
|
||||
, customer_id
|
||||
, date
|
||||
, event
|
||||
, target_name
|
||||
, last_updated
|
||||
, c_name
|
||||
, c_type
|
||||
, contentName
|
||||
, source
|
||||
, tag
|
||||
, c_keyword
|
||||
, attr2
|
||||
, screen_width
|
||||
, ip_city
|
||||
, ip_country
|
||||
, platform
|
||||
, last_updated
|
||||
, page_type
|
||||
, os_version
|
||||
, browser
|
||||
, browser_version
|
||||
, customer_id
|
||||
, id
|
||||
, tag
|
||||
, event
|
||||
, id_str
|
||||
, os
|
||||
, target_id
|
||||
|
|
@ -36,21 +43,28 @@ insert into data_api.customer_events_lastupdated (
|
|||
,etl_tx_dt
|
||||
)
|
||||
select
|
||||
case when trim(both from date)='' then null else date::text end date
|
||||
, case when trim(both from screen_width)='' then null else screen_width::text end screen_width
|
||||
case when trim(both from mergedCustomerId)='' then null else mergedCustomerId::text end mergedCustomerId
|
||||
, case when trim(both from id)='' then null else id::text end id
|
||||
, case when trim(both from customer_id)='' then null else customer_id::text end customer_id
|
||||
, case when trim(both from date)='' then null else date::text end date
|
||||
, case when trim(both from event)='' then null else event::text end event
|
||||
, case when trim(both from target_name)='' then null else target_name::text end target_name
|
||||
, case when trim(both from last_updated)='' then null else last_updated::text end last_updated
|
||||
, case when trim(both from c_name)='' then null else c_name::text end c_name
|
||||
, case when trim(both from c_type)='' then null else c_type::text end c_type
|
||||
, case when trim(both from contentName)='' then null else contentName::text end contentName
|
||||
, case when trim(both from source)='' then null else source::text end source
|
||||
, case when trim(both from tag)='' then null else tag::text end tag
|
||||
, case when trim(both from c_keyword)='' then null else c_keyword::text end c_keyword
|
||||
, case when trim(both from attr2)='' then null else attr2::text end attr2
|
||||
, case when trim(both from screen_width)='' then null else screen_width::text end screen_width
|
||||
, case when trim(both from ip_city)='' then null else ip_city::text end ip_city
|
||||
, case when trim(both from ip_country)='' then null else ip_country::text end ip_country
|
||||
, case when trim(both from platform)='' then null else platform::text end platform
|
||||
, case when trim(both from last_updated)='' then null else last_updated::text end last_updated
|
||||
, case when trim(both from page_type)='' then null else page_type::text end page_type
|
||||
, case when trim(both from os_version)='' then null else os_version::text end os_version
|
||||
, case when trim(both from browser)='' then null else browser::text end browser
|
||||
, case when trim(both from browser_version)='' then null else browser_version::text end browser_version
|
||||
, case when trim(both from customer_id)='' then null else customer_id::text end customer_id
|
||||
, case when trim(both from id)='' then null else id::text end id
|
||||
, case when trim(both from tag)='' then null else tag::text end tag
|
||||
, case when trim(both from event)='' then null else event::text end event
|
||||
, case when trim(both from id_str)='' then null else id_str::text end id_str
|
||||
, case when trim(both from os)='' then null else os::text end os
|
||||
, case when trim(both from target_id)='' then null else target_id::text end target_id
|
||||
|
|
@ -66,21 +80,28 @@ select
|
|||
,etl_tx_dt
|
||||
from (
|
||||
select
|
||||
(json_array_elements(data::json)::json->>'date') date
|
||||
, (json_array_elements(data::json)::json->>'screenWidth') screen_width
|
||||
(json_array_elements(data::json)::json->>'date') mergedCustomerId
|
||||
, (json_array_elements(data::json)::json->>'id') id
|
||||
, (json_array_elements(data::json)::json->>'customerId') customer_id
|
||||
, (json_array_elements(data::json)::json->>'date') date
|
||||
, (json_array_elements(data::json)::json->>'event') event
|
||||
, (json_array_elements(data::json)::json->>'targetName') target_name
|
||||
, (json_array_elements(data::json)::json->>'lastUpdated') last_updated
|
||||
, (json_array_elements(data::json)::json->>'pageId') c_name
|
||||
, (json_array_elements(data::json)::json->>'date') c_type
|
||||
, (json_array_elements(data::json)::json->>'date') contentName
|
||||
, (json_array_elements(data::json)::json->>'date') source
|
||||
, (json_array_elements(data::json)::json->>'tag') tag
|
||||
, (json_array_elements(data::json)::json->>'date') c_keyword
|
||||
, (json_array_elements(data::json)::json->>'date') attr2
|
||||
, (json_array_elements(data::json)::json->>'screenWidth') screen_width
|
||||
, (json_array_elements(data::json)::json->>'ipCity') ip_city
|
||||
, (json_array_elements(data::json)::json->>'ipCountry') ip_country
|
||||
, (json_array_elements(data::json)::json->>'platform') platform
|
||||
, (json_array_elements(data::json)::json->>'lastUpdated') last_updated
|
||||
, (json_array_elements(data::json)::json->>'pageType') page_type
|
||||
, (json_array_elements(data::json)::json->>'osVersion') os_version
|
||||
, (json_array_elements(data::json)::json->>'browser') browser
|
||||
, (json_array_elements(data::json)::json->>'browserVersion') browser_version
|
||||
, (json_array_elements(data::json)::json->>'customerId') customer_id
|
||||
, (json_array_elements(data::json)::json->>'id') id
|
||||
, (json_array_elements(data::json)::json->>'tag') tag
|
||||
, (json_array_elements(data::json)::json->>'event') event
|
||||
, (json_array_elements(data::json)::json->>'idStr') id_str
|
||||
, (json_array_elements(data::json)::json->>'os') os
|
||||
, (json_array_elements(data::json)::json->>'targetId') target_id
|
||||
|
|
|
|||
|
|
@ -51,7 +51,7 @@ while not auth and i < 60:
|
|||
print('开始请求数据总数。')
|
||||
url='https://api.huiju.cool/v2/customers'
|
||||
header={}
|
||||
body={'access_token':auth['access_token'],'sort':'lastUpdated','lastUpdated[ge]':formatted_previous_date,'lastUpdated[le]':formatted_current_date,'limit':'2000',}
|
||||
body={'access_token':auth['access_token'],'limit':'2000','sort':'lastUpdated','lastUpdated[le]':formatted_current_date,'lastUpdated[ge]':formatted_previous_date,}
|
||||
dataReqL=requests.get(url,headers=header,params=body)
|
||||
resL=json.loads(dataReqL.text)
|
||||
# print(resL)
|
||||
|
|
|
|||
|
|
@ -6,408 +6,39 @@
|
|||
DELETE FROM data_api.scrm_contact_lastUpdated;
|
||||
|
||||
insert into data_api.scrm_contact_lastUpdated (
|
||||
extra_convert_source1
|
||||
, referrer
|
||||
, accept_email
|
||||
, acceptsms
|
||||
, annual_revenue
|
||||
, attr_id
|
||||
, attr1
|
||||
, attr10
|
||||
, attr11
|
||||
, attr12
|
||||
, attr13
|
||||
, attr14
|
||||
, attr15
|
||||
, attr16
|
||||
, attr17
|
||||
, attr18
|
||||
, attr19
|
||||
, attr2
|
||||
, attr20
|
||||
, attr21
|
||||
, attr22
|
||||
, attr23
|
||||
, attr24
|
||||
, attr25
|
||||
, attr26
|
||||
, attr27
|
||||
, attr28
|
||||
, attr3
|
||||
, attr4
|
||||
, attr5
|
||||
, attr6
|
||||
, attr7
|
||||
, attr8
|
||||
, attr9
|
||||
, birthday
|
||||
, campaign_id
|
||||
, campaign_name
|
||||
id
|
||||
, city
|
||||
, company
|
||||
, content_name
|
||||
, convert_date
|
||||
, country
|
||||
, county
|
||||
, create_from
|
||||
, create_from_name
|
||||
, create_method
|
||||
, date_created
|
||||
, date_join
|
||||
, date_member_reg
|
||||
, display_name
|
||||
, email
|
||||
, email_verified
|
||||
, employee_number
|
||||
, external_id
|
||||
, extra_convert_content_name
|
||||
, extra_convert_date
|
||||
, extra_convert_source
|
||||
, extra_convert_source2
|
||||
, extra_convert_source3
|
||||
, extra_convert_source4
|
||||
, extra_convert_source5
|
||||
, extra_convert_source6
|
||||
, extra_convert_source7
|
||||
, extra_convert_source8
|
||||
, extra_regist_content_name
|
||||
, extra_regist_date
|
||||
, extra_regist_source
|
||||
, extra_regist_source1
|
||||
, extra_regist_source2
|
||||
, extra_regist_source3
|
||||
, extra_regist_source4
|
||||
, extra_regist_source5
|
||||
, extra_regist_source6
|
||||
, extra_regist_source7
|
||||
, extra_regist_source8
|
||||
, first_assign_date
|
||||
, gender
|
||||
, home_address
|
||||
, id
|
||||
, id_str
|
||||
, img
|
||||
, industry
|
||||
, is_anonymous
|
||||
, is_employee
|
||||
, is_member
|
||||
, is_referrer
|
||||
, language
|
||||
, last_updated
|
||||
, latest_engage_date
|
||||
, mobile
|
||||
, mobile_verified
|
||||
, name
|
||||
, nick_name
|
||||
, not_accept_email
|
||||
, not_acceptsms
|
||||
, office_address
|
||||
, owner_date
|
||||
, owner_employee_id
|
||||
, owner_employee_name
|
||||
, owner_id
|
||||
, owner_name
|
||||
, province
|
||||
, qq
|
||||
, referrer_name
|
||||
, refund_date
|
||||
, region
|
||||
, remark
|
||||
, source
|
||||
, source1
|
||||
, source2
|
||||
, source3
|
||||
, source4
|
||||
, source5
|
||||
, source6
|
||||
, source7
|
||||
, source8
|
||||
, stage
|
||||
, status
|
||||
, street
|
||||
, subscribe_scene
|
||||
, telephone
|
||||
, title
|
||||
, type
|
||||
, update_method
|
||||
, utm_medium
|
||||
, utm_term
|
||||
, version
|
||||
, website
|
||||
, wechat
|
||||
, wechat_city
|
||||
, wechat_nick_name
|
||||
, weibo
|
||||
, date_join
|
||||
,etl_tx_dt
|
||||
)
|
||||
select
|
||||
case when trim(both from extra_convert_source1)='' then null else extra_convert_source1::text end extra_convert_source1
|
||||
, case when trim(both from referrer)='' then null else referrer::text end referrer
|
||||
, case when trim(both from accept_email)='' then null else accept_email::text end accept_email
|
||||
, case when trim(both from acceptsms)='' then null else acceptsms::text end acceptsms
|
||||
, case when trim(both from annual_revenue)='' then null else annual_revenue::text end annual_revenue
|
||||
, case when trim(both from attr_id)='' then null else attr_id::text end attr_id
|
||||
, case when trim(both from attr1)='' then null else attr1::text end attr1
|
||||
, case when trim(both from attr10)='' then null else attr10::text end attr10
|
||||
, case when trim(both from attr11)='' then null else attr11::text end attr11
|
||||
, case when trim(both from attr12)='' then null else attr12::text end attr12
|
||||
, case when trim(both from attr13)='' then null else attr13::text end attr13
|
||||
, case when trim(both from attr14)='' then null else attr14::text end attr14
|
||||
, case when trim(both from attr15)='' then null else attr15::text end attr15
|
||||
, case when trim(both from attr16)='' then null else attr16::text end attr16
|
||||
, case when trim(both from attr17)='' then null else attr17::text end attr17
|
||||
, case when trim(both from attr18)='' then null else attr18::text end attr18
|
||||
, case when trim(both from attr19)='' then null else attr19::text end attr19
|
||||
, case when trim(both from attr2)='' then null else attr2::text end attr2
|
||||
, case when trim(both from attr20)='' then null else attr20::text end attr20
|
||||
, case when trim(both from attr21)='' then null else attr21::text end attr21
|
||||
, case when trim(both from attr22)='' then null else attr22::text end attr22
|
||||
, case when trim(both from attr23)='' then null else attr23::text end attr23
|
||||
, case when trim(both from attr24)='' then null else attr24::text end attr24
|
||||
, case when trim(both from attr25)='' then null else attr25::text end attr25
|
||||
, case when trim(both from attr26)='' then null else attr26::text end attr26
|
||||
, case when trim(both from attr27)='' then null else attr27::text end attr27
|
||||
, case when trim(both from attr28)='' then null else attr28::text end attr28
|
||||
, case when trim(both from attr3)='' then null else attr3::text end attr3
|
||||
, case when trim(both from attr4)='' then null else attr4::text end attr4
|
||||
, case when trim(both from attr5)='' then null else attr5::text end attr5
|
||||
, case when trim(both from attr6)='' then null else attr6::text end attr6
|
||||
, case when trim(both from attr7)='' then null else attr7::text end attr7
|
||||
, case when trim(both from attr8)='' then null else attr8::text end attr8
|
||||
, case when trim(both from attr9)='' then null else attr9::text end attr9
|
||||
, case when trim(both from birthday)='' then null else birthday::text end birthday
|
||||
, case when trim(both from campaign_id)='' then null else campaign_id::text end campaign_id
|
||||
, case when trim(both from campaign_name)='' then null else campaign_name::text end campaign_name
|
||||
case when trim(both from id)='' then null else id::text end id
|
||||
, case when trim(both from city)='' then null else city::text end city
|
||||
, case when trim(both from company)='' then null else company::text end company
|
||||
, case when trim(both from content_name)='' then null else content_name::text end content_name
|
||||
, case when trim(both from convert_date)='' then null else convert_date::text end convert_date
|
||||
, case when trim(both from country)='' then null else country::text end country
|
||||
, case when trim(both from county)='' then null else county::text end county
|
||||
, case when trim(both from create_from)='' then null else create_from::text end create_from
|
||||
, case when trim(both from create_from_name)='' then null else create_from_name::text end create_from_name
|
||||
, case when trim(both from create_method)='' then null else create_method::text end create_method
|
||||
, case when trim(both from date_created)='' then null else date_created::text end date_created
|
||||
, case when trim(both from date_join)='' then null else date_join::text end date_join
|
||||
, case when trim(both from date_member_reg)='' then null else date_member_reg::text end date_member_reg
|
||||
, case when trim(both from display_name)='' then null else display_name::text end display_name
|
||||
, case when trim(both from email)='' then null else email::text end email
|
||||
, case when trim(both from email_verified)='' then null else email_verified::text end email_verified
|
||||
, case when trim(both from employee_number)='' then null else employee_number::text end employee_number
|
||||
, case when trim(both from external_id)='' then null else external_id::text end external_id
|
||||
, case when trim(both from extra_convert_content_name)='' then null else extra_convert_content_name::text end extra_convert_content_name
|
||||
, case when trim(both from extra_convert_date)='' then null else extra_convert_date::text end extra_convert_date
|
||||
, case when trim(both from extra_convert_source)='' then null else extra_convert_source::text end extra_convert_source
|
||||
, case when trim(both from extra_convert_source2)='' then null else extra_convert_source2::text end extra_convert_source2
|
||||
, case when trim(both from extra_convert_source3)='' then null else extra_convert_source3::text end extra_convert_source3
|
||||
, case when trim(both from extra_convert_source4)='' then null else extra_convert_source4::text end extra_convert_source4
|
||||
, case when trim(both from extra_convert_source5)='' then null else extra_convert_source5::text end extra_convert_source5
|
||||
, case when trim(both from extra_convert_source6)='' then null else extra_convert_source6::text end extra_convert_source6
|
||||
, case when trim(both from extra_convert_source7)='' then null else extra_convert_source7::text end extra_convert_source7
|
||||
, case when trim(both from extra_convert_source8)='' then null else extra_convert_source8::text end extra_convert_source8
|
||||
, case when trim(both from extra_regist_content_name)='' then null else extra_regist_content_name::text end extra_regist_content_name
|
||||
, case when trim(both from extra_regist_date)='' then null else extra_regist_date::text end extra_regist_date
|
||||
, case when trim(both from extra_regist_source)='' then null else extra_regist_source::text end extra_regist_source
|
||||
, case when trim(both from extra_regist_source1)='' then null else extra_regist_source1::text end extra_regist_source1
|
||||
, case when trim(both from extra_regist_source2)='' then null else extra_regist_source2::text end extra_regist_source2
|
||||
, case when trim(both from extra_regist_source3)='' then null else extra_regist_source3::text end extra_regist_source3
|
||||
, case when trim(both from extra_regist_source4)='' then null else extra_regist_source4::text end extra_regist_source4
|
||||
, case when trim(both from extra_regist_source5)='' then null else extra_regist_source5::text end extra_regist_source5
|
||||
, case when trim(both from extra_regist_source6)='' then null else extra_regist_source6::text end extra_regist_source6
|
||||
, case when trim(both from extra_regist_source7)='' then null else extra_regist_source7::text end extra_regist_source7
|
||||
, case when trim(both from extra_regist_source8)='' then null else extra_regist_source8::text end extra_regist_source8
|
||||
, case when trim(both from first_assign_date)='' then null else first_assign_date::text end first_assign_date
|
||||
, case when trim(both from gender)='' then null else gender::text end gender
|
||||
, case when trim(both from home_address)='' then null else home_address::text end home_address
|
||||
, case when trim(both from id)='' then null else id::text end id
|
||||
, case when trim(both from id_str)='' then null else id_str::text end id_str
|
||||
, case when trim(both from img)='' then null else img::text end img
|
||||
, case when trim(both from industry)='' then null else industry::text end industry
|
||||
, case when trim(both from is_anonymous)='' then null else is_anonymous::text end is_anonymous
|
||||
, case when trim(both from is_employee)='' then null else is_employee::text end is_employee
|
||||
, case when trim(both from is_member)='' then null else is_member::text end is_member
|
||||
, case when trim(both from is_referrer)='' then null else is_referrer::text end is_referrer
|
||||
, case when trim(both from language)='' then null else language::text end language
|
||||
, case when trim(both from last_updated)='' then null else last_updated::text end last_updated
|
||||
, case when trim(both from latest_engage_date)='' then null else latest_engage_date::text end latest_engage_date
|
||||
, case when trim(both from mobile)='' then null else mobile::text end mobile
|
||||
, case when trim(both from mobile_verified)='' then null else mobile_verified::text end mobile_verified
|
||||
, case when trim(both from name)='' then null else name::text end name
|
||||
, case when trim(both from nick_name)='' then null else nick_name::text end nick_name
|
||||
, case when trim(both from not_accept_email)='' then null else not_accept_email::text end not_accept_email
|
||||
, case when trim(both from not_acceptsms)='' then null else not_acceptsms::text end not_acceptsms
|
||||
, case when trim(both from office_address)='' then null else office_address::text end office_address
|
||||
, case when trim(both from owner_date)='' then null else owner_date::text end owner_date
|
||||
, case when trim(both from owner_employee_id)='' then null else owner_employee_id::text end owner_employee_id
|
||||
, case when trim(both from owner_employee_name)='' then null else owner_employee_name::text end owner_employee_name
|
||||
, case when trim(both from owner_id)='' then null else owner_id::text end owner_id
|
||||
, case when trim(both from owner_name)='' then null else owner_name::text end owner_name
|
||||
, case when trim(both from province)='' then null else province::text end province
|
||||
, case when trim(both from qq)='' then null else qq::text end qq
|
||||
, case when trim(both from referrer_name)='' then null else referrer_name::text end referrer_name
|
||||
, case when trim(both from refund_date)='' then null else refund_date::text end refund_date
|
||||
, case when trim(both from region)='' then null else region::text end region
|
||||
, case when trim(both from remark)='' then null else remark::text end remark
|
||||
, case when trim(both from source)='' then null else source::text end source
|
||||
, case when trim(both from source1)='' then null else source1::text end source1
|
||||
, case when trim(both from source2)='' then null else source2::text end source2
|
||||
, case when trim(both from source3)='' then null else source3::text end source3
|
||||
, case when trim(both from source4)='' then null else source4::text end source4
|
||||
, case when trim(both from source5)='' then null else source5::text end source5
|
||||
, case when trim(both from source6)='' then null else source6::text end source6
|
||||
, case when trim(both from source7)='' then null else source7::text end source7
|
||||
, case when trim(both from source8)='' then null else source8::text end source8
|
||||
, case when trim(both from stage)='' then null else stage::text end stage
|
||||
, case when trim(both from status)='' then null else status::text end status
|
||||
, case when trim(both from street)='' then null else street::text end street
|
||||
, case when trim(both from subscribe_scene)='' then null else subscribe_scene::text end subscribe_scene
|
||||
, case when trim(both from telephone)='' then null else telephone::text end telephone
|
||||
, case when trim(both from title)='' then null else title::text end title
|
||||
, case when trim(both from type)='' then null else type::text end type
|
||||
, case when trim(both from update_method)='' then null else update_method::text end update_method
|
||||
, case when trim(both from utm_medium)='' then null else utm_medium::text end utm_medium
|
||||
, case when trim(both from utm_term)='' then null else utm_term::text end utm_term
|
||||
, case when trim(both from version)='' then null else version::text end version
|
||||
, case when trim(both from website)='' then null else website::text end website
|
||||
, case when trim(both from wechat)='' then null else wechat::text end wechat
|
||||
, case when trim(both from wechat_city)='' then null else wechat_city::text end wechat_city
|
||||
, case when trim(both from wechat_nick_name)='' then null else wechat_nick_name::text end wechat_nick_name
|
||||
, case when trim(both from weibo)='' then null else weibo::text end weibo
|
||||
, case when trim(both from date_join)='' then null else date_join::text end date_join
|
||||
,etl_tx_dt
|
||||
from (
|
||||
select
|
||||
(json_array_elements(data::json)::json->>'extraConvertSource1') extra_convert_source1
|
||||
, (json_array_elements(data::json)::json->>'referrer') referrer
|
||||
, (json_array_elements(data::json)::json->>'acceptEmail') accept_email
|
||||
, (json_array_elements(data::json)::json->>'acceptsms') acceptsms
|
||||
, (json_array_elements(data::json)::json->>'annualRevenue') annual_revenue
|
||||
, (json_array_elements(data::json)::json->>'attrId') attr_id
|
||||
, (json_array_elements(data::json)::json->>'attr1') attr1
|
||||
, (json_array_elements(data::json)::json->>'attr10') attr10
|
||||
, (json_array_elements(data::json)::json->>'attr11') attr11
|
||||
, (json_array_elements(data::json)::json->>'attr12') attr12
|
||||
, (json_array_elements(data::json)::json->>'attr13') attr13
|
||||
, (json_array_elements(data::json)::json->>'attr14') attr14
|
||||
, (json_array_elements(data::json)::json->>'attr15') attr15
|
||||
, (json_array_elements(data::json)::json->>'attr16') attr16
|
||||
, (json_array_elements(data::json)::json->>'attr17') attr17
|
||||
, (json_array_elements(data::json)::json->>'attr18') attr18
|
||||
, (json_array_elements(data::json)::json->>'attr19') attr19
|
||||
, (json_array_elements(data::json)::json->>'attr2') attr2
|
||||
, (json_array_elements(data::json)::json->>'attr20') attr20
|
||||
, (json_array_elements(data::json)::json->>'attr21') attr21
|
||||
, (json_array_elements(data::json)::json->>'attr22') attr22
|
||||
, (json_array_elements(data::json)::json->>'attr23') attr23
|
||||
, (json_array_elements(data::json)::json->>'attr24') attr24
|
||||
, (json_array_elements(data::json)::json->>'attr25') attr25
|
||||
, (json_array_elements(data::json)::json->>'attr26') attr26
|
||||
, (json_array_elements(data::json)::json->>'attr27') attr27
|
||||
, (json_array_elements(data::json)::json->>'attr28') attr28
|
||||
, (json_array_elements(data::json)::json->>'attr3') attr3
|
||||
, (json_array_elements(data::json)::json->>'attr4') attr4
|
||||
, (json_array_elements(data::json)::json->>'attr5') attr5
|
||||
, (json_array_elements(data::json)::json->>'attr6') attr6
|
||||
, (json_array_elements(data::json)::json->>'attr7') attr7
|
||||
, (json_array_elements(data::json)::json->>'attr8') attr8
|
||||
, (json_array_elements(data::json)::json->>'attr9') attr9
|
||||
, (json_array_elements(data::json)::json->>'birthday') birthday
|
||||
, (json_array_elements(data::json)::json->>'campaignId') campaign_id
|
||||
, (json_array_elements(data::json)::json->>'campaignName') campaign_name
|
||||
(json_array_elements(data::json)::json->>'id') id
|
||||
, (json_array_elements(data::json)::json->>'city') city
|
||||
, (json_array_elements(data::json)::json->>'company') company
|
||||
, (json_array_elements(data::json)::json->>'contentName') content_name
|
||||
, (json_array_elements(data::json)::json->>'convertDate') convert_date
|
||||
, (json_array_elements(data::json)::json->>'country') country
|
||||
, (json_array_elements(data::json)::json->>'county') county
|
||||
, (json_array_elements(data::json)::json->>'createFrom') create_from
|
||||
, (json_array_elements(data::json)::json->>'createFromName') create_from_name
|
||||
, (json_array_elements(data::json)::json->>'createMethod') create_method
|
||||
, (json_array_elements(data::json)::json->>'dateCreated') date_created
|
||||
, (json_array_elements(data::json)::json->>'dateJoin') date_join
|
||||
, (json_array_elements(data::json)::json->>'dateMemberReg') date_member_reg
|
||||
, (json_array_elements(data::json)::json->>'displayName') display_name
|
||||
, (json_array_elements(data::json)::json->>'email') email
|
||||
, (json_array_elements(data::json)::json->>'emailVerified') email_verified
|
||||
, (json_array_elements(data::json)::json->>'employeeNumber') employee_number
|
||||
, (json_array_elements(data::json)::json->>'externalId') external_id
|
||||
, (json_array_elements(data::json)::json->>'extraConvertContentName') extra_convert_content_name
|
||||
, (json_array_elements(data::json)::json->>'extraConvertDate') extra_convert_date
|
||||
, (json_array_elements(data::json)::json->>'extraConvertSource') extra_convert_source
|
||||
, (json_array_elements(data::json)::json->>'extraConvertSource2') extra_convert_source2
|
||||
, (json_array_elements(data::json)::json->>'extraConvertSource3') extra_convert_source3
|
||||
, (json_array_elements(data::json)::json->>'extraConvertSource4') extra_convert_source4
|
||||
, (json_array_elements(data::json)::json->>'extraConvertSource5') extra_convert_source5
|
||||
, (json_array_elements(data::json)::json->>'extraConvertSource6') extra_convert_source6
|
||||
, (json_array_elements(data::json)::json->>'extraConvertSource7') extra_convert_source7
|
||||
, (json_array_elements(data::json)::json->>'extraConvertSource8') extra_convert_source8
|
||||
, (json_array_elements(data::json)::json->>'extraRegistContentName') extra_regist_content_name
|
||||
, (json_array_elements(data::json)::json->>'extraRegistDate') extra_regist_date
|
||||
, (json_array_elements(data::json)::json->>'extraRegistSource') extra_regist_source
|
||||
, (json_array_elements(data::json)::json->>'extraRegistSource1') extra_regist_source1
|
||||
, (json_array_elements(data::json)::json->>'extraRegistSource2') extra_regist_source2
|
||||
, (json_array_elements(data::json)::json->>'extraRegistSource3') extra_regist_source3
|
||||
, (json_array_elements(data::json)::json->>'extraRegistSource4') extra_regist_source4
|
||||
, (json_array_elements(data::json)::json->>'extraRegistSource5') extra_regist_source5
|
||||
, (json_array_elements(data::json)::json->>'extraRegistSource6') extra_regist_source6
|
||||
, (json_array_elements(data::json)::json->>'extraRegistSource7') extra_regist_source7
|
||||
, (json_array_elements(data::json)::json->>'extraRegistSource8') extra_regist_source8
|
||||
, (json_array_elements(data::json)::json->>'firstAssignDate') first_assign_date
|
||||
, (json_array_elements(data::json)::json->>'gender') gender
|
||||
, (json_array_elements(data::json)::json->>'homeAddress') home_address
|
||||
, (json_array_elements(data::json)::json->>'id') id
|
||||
, (json_array_elements(data::json)::json->>'idStr') id_str
|
||||
, (json_array_elements(data::json)::json->>'img') img
|
||||
, (json_array_elements(data::json)::json->>'industry') industry
|
||||
, (json_array_elements(data::json)::json->>'isAnonymous') is_anonymous
|
||||
, (json_array_elements(data::json)::json->>'isEmployee') is_employee
|
||||
, (json_array_elements(data::json)::json->>'isMember') is_member
|
||||
, (json_array_elements(data::json)::json->>'isReferrer') is_referrer
|
||||
, (json_array_elements(data::json)::json->>'language') language
|
||||
, (json_array_elements(data::json)::json->>'lastUpdated') last_updated
|
||||
, (json_array_elements(data::json)::json->>'latestEngageDate') latest_engage_date
|
||||
, (json_array_elements(data::json)::json->>'mobile') mobile
|
||||
, (json_array_elements(data::json)::json->>'mobileVerified') mobile_verified
|
||||
, (json_array_elements(data::json)::json->>'name') name
|
||||
, (json_array_elements(data::json)::json->>'nickName') nick_name
|
||||
, (json_array_elements(data::json)::json->>'notAcceptEmail') not_accept_email
|
||||
, (json_array_elements(data::json)::json->>'notAcceptsms') not_acceptsms
|
||||
, (json_array_elements(data::json)::json->>'officeAddress') office_address
|
||||
, (json_array_elements(data::json)::json->>'ownerDate') owner_date
|
||||
, (json_array_elements(data::json)::json->>'ownerEmployeeId') owner_employee_id
|
||||
, (json_array_elements(data::json)::json->>'ownerEmployeeName') owner_employee_name
|
||||
, (json_array_elements(data::json)::json->>'ownerId') owner_id
|
||||
, (json_array_elements(data::json)::json->>'ownerName') owner_name
|
||||
, (json_array_elements(data::json)::json->>'province') province
|
||||
, (json_array_elements(data::json)::json->>'qq') qq
|
||||
, (json_array_elements(data::json)::json->>'referrerName') referrer_name
|
||||
, (json_array_elements(data::json)::json->>'refundDate') refund_date
|
||||
, (json_array_elements(data::json)::json->>'region') region
|
||||
, (json_array_elements(data::json)::json->>'remark') remark
|
||||
, (json_array_elements(data::json)::json->>'source') source
|
||||
, (json_array_elements(data::json)::json->>'source1') source1
|
||||
, (json_array_elements(data::json)::json->>'source2') source2
|
||||
, (json_array_elements(data::json)::json->>'source3') source3
|
||||
, (json_array_elements(data::json)::json->>'source4') source4
|
||||
, (json_array_elements(data::json)::json->>'source5') source5
|
||||
, (json_array_elements(data::json)::json->>'source6') source6
|
||||
, (json_array_elements(data::json)::json->>'source7') source7
|
||||
, (json_array_elements(data::json)::json->>'source8') source8
|
||||
, (json_array_elements(data::json)::json->>'stage') stage
|
||||
, (json_array_elements(data::json)::json->>'status') status
|
||||
, (json_array_elements(data::json)::json->>'street') street
|
||||
, (json_array_elements(data::json)::json->>'subscribeScene') subscribe_scene
|
||||
, (json_array_elements(data::json)::json->>'telephone') telephone
|
||||
, (json_array_elements(data::json)::json->>'title') title
|
||||
, (json_array_elements(data::json)::json->>'type') type
|
||||
, (json_array_elements(data::json)::json->>'updateMethod') update_method
|
||||
, (json_array_elements(data::json)::json->>'utmMedium') utm_medium
|
||||
, (json_array_elements(data::json)::json->>'utmTerm') utm_term
|
||||
, (json_array_elements(data::json)::json->>'version') version
|
||||
, (json_array_elements(data::json)::json->>'website') website
|
||||
, (json_array_elements(data::json)::json->>'wechat') wechat
|
||||
, (json_array_elements(data::json)::json->>'wechatCity') wechat_city
|
||||
, (json_array_elements(data::json)::json->>'wechatNickName') wechat_nick_name
|
||||
, (json_array_elements(data::json)::json->>'weibo') weibo
|
||||
, (json_array_elements(data::json)::json->>'dateJoin') date_join
|
||||
,CURRENT_TIMESTAMP(0) etl_tx_dt
|
||||
from (select * from data_api.api_data
|
||||
WHERE api_id='03d918ea-00d1-448a-8b9b-6a0b1a33' and is_loaded = '0' order by request_tm desc limit 1) p )p;
|
||||
|
|
|
|||
|
|
@ -14,10 +14,9 @@ from airflow.utils.trigger_rule import TriggerRule
|
|||
sshHook = SSHHook(ssh_conn_id ='ssh_air')
|
||||
default_args = {
|
||||
'owner': 'info@idgvalue.com',
|
||||
'email': [''],
|
||||
'email_on_failure': True,
|
||||
'email_on_retry':True,
|
||||
'start_date': datetime(2024, 2, 12),
|
||||
'start_date': datetime(2024, 1, 1),
|
||||
'depends_on_past': False,
|
||||
'retries': 6,
|
||||
'retry_delay': timedelta(minutes=10),
|
||||
|
|
@ -37,7 +36,7 @@ task_failed = EmailOperator (
|
|||
cc=[""],
|
||||
subject="tk_api_failed",
|
||||
html_content='<h3>您好，tk_api作业失败，请及时处理" </h3>')
|
||||
|
||||
|
||||
scrm_contact_feign = SSHOperator(
|
||||
ssh_hook=sshHook,
|
||||
task_id='scrm_contact_feign',
|
||||
|
|
@ -56,7 +55,7 @@ retries=3,
|
|||
dag=dag)
|
||||
|
||||
scrm_contact_feign >> scrm_contact_load
|
||||
|
||||
|
||||
scrm_contact_1129 = SSHOperator(
|
||||
ssh_hook=sshHook,
|
||||
task_id='scrm_contact_1129',
|
||||
|
|
@ -65,7 +64,7 @@ params={'my_param':"S98_S_scrm_contact"},
|
|||
depends_on_past=False,
|
||||
retries=3,
|
||||
dag=dag)
|
||||
|
||||
|
||||
custom_events_update_feign = SSHOperator(
|
||||
ssh_hook=sshHook,
|
||||
task_id='custom_events_update_feign',
|
||||
|
|
@ -84,7 +83,7 @@ retries=3,
|
|||
dag=dag)
|
||||
|
||||
custom_events_update_feign >> custom_events_update_load
|
||||
|
||||
|
||||
update_scrm_contact_feign = SSHOperator(
|
||||
ssh_hook=sshHook,
|
||||
task_id='update_scrm_contact_feign',
|
||||
|
|
@ -103,7 +102,7 @@ retries=3,
|
|||
dag=dag)
|
||||
|
||||
update_scrm_contact_feign >> update_scrm_contact_load
|
||||
|
||||
|
||||
customer_events_feign = SSHOperator(
|
||||
ssh_hook=sshHook,
|
||||
task_id='customer_events_feign',
|
||||
|
|
@ -122,7 +121,7 @@ retries=3,
|
|||
dag=dag)
|
||||
|
||||
customer_events_feign >> customer_events_load
|
||||
|
||||
|
||||
customer_event_meta_feign = SSHOperator(
|
||||
ssh_hook=sshHook,
|
||||
task_id='customer_event_meta_feign',
|
||||
|
|
@ -141,7 +140,7 @@ retries=3,
|
|||
dag=dag)
|
||||
|
||||
customer_event_meta_feign >> customer_event_meta_load
|
||||
|
||||
|
||||
scrm_contact_lastupdated_4112 = SSHOperator(
|
||||
ssh_hook=sshHook,
|
||||
task_id='scrm_contact_lastupdated_4112',
|
||||
|
|
@ -150,7 +149,7 @@ params={'my_param':"S98_S_scrm_contact_lastupdated"},
|
|||
depends_on_past=False,
|
||||
retries=3,
|
||||
dag=dag)
|
||||
|
||||
|
||||
customer_event_meta_2268 = SSHOperator(
|
||||
ssh_hook=sshHook,
|
||||
task_id='customer_event_meta_2268',
|
||||
|
|
@ -159,16 +158,7 @@ params={'my_param':"S98_S_customer_event_meta"},
|
|||
depends_on_past=False,
|
||||
retries=3,
|
||||
dag=dag)
|
||||
|
||||
customer_events_lastupdated_8895 = SSHOperator(
|
||||
ssh_hook=sshHook,
|
||||
task_id='customer_events_lastupdated_8895',
|
||||
command='/data/airflow/etl/SA/run_sa.sh {{ ds_nodash }} {{ params.my_param }} >>/data/airflow/logs/run_psql_{{ds_nodash}}.log 2>&1 ',
|
||||
params={'my_param':"S98_S_customer_events_lastupdated"},
|
||||
depends_on_past=False,
|
||||
retries=3,
|
||||
dag=dag)
|
||||
|
||||
|
||||
customer_events_3292 = SSHOperator(
|
||||
ssh_hook=sshHook,
|
||||
task_id='customer_events_3292',
|
||||
|
|
@ -177,7 +167,7 @@ params={'my_param':"S98_S_customer_events"},
|
|||
depends_on_past=False,
|
||||
retries=3,
|
||||
dag=dag)
|
||||
|
||||
|
||||
custom_events_9_feign = SSHOperator(
|
||||
ssh_hook=sshHook,
|
||||
task_id='custom_events_9_feign',
|
||||
|
|
@ -196,7 +186,7 @@ retries=3,
|
|||
dag=dag)
|
||||
|
||||
custom_events_9_feign >> custom_events_9_load
|
||||
|
||||
|
||||
custom_events_6_feign = SSHOperator(
|
||||
ssh_hook=sshHook,
|
||||
task_id='custom_events_6_feign',
|
||||
|
|
@ -215,7 +205,7 @@ retries=3,
|
|||
dag=dag)
|
||||
|
||||
custom_events_6_feign >> custom_events_6_load
|
||||
|
||||
|
||||
custom_events_4_feign = SSHOperator(
|
||||
ssh_hook=sshHook,
|
||||
task_id='custom_events_4_feign',
|
||||
|
|
@ -234,7 +224,7 @@ retries=3,
|
|||
dag=dag)
|
||||
|
||||
custom_events_4_feign >> custom_events_4_load
|
||||
|
||||
|
||||
custom_events_8_feign = SSHOperator(
|
||||
ssh_hook=sshHook,
|
||||
task_id='custom_events_8_feign',
|
||||
|
|
@ -253,7 +243,7 @@ retries=3,
|
|||
dag=dag)
|
||||
|
||||
custom_events_8_feign >> custom_events_8_load
|
||||
|
||||
|
||||
custom_events_3_feign = SSHOperator(
|
||||
ssh_hook=sshHook,
|
||||
task_id='custom_events_3_feign',
|
||||
|
|
@ -272,7 +262,7 @@ retries=3,
|
|||
dag=dag)
|
||||
|
||||
custom_events_3_feign >> custom_events_3_load
|
||||
|
||||
|
||||
custom_events_1_feign = SSHOperator(
|
||||
ssh_hook=sshHook,
|
||||
task_id='custom_events_1_feign',
|
||||
|
|
@ -291,7 +281,7 @@ retries=3,
|
|||
dag=dag)
|
||||
|
||||
custom_events_1_feign >> custom_events_1_load
|
||||
|
||||
|
||||
custom_events_7_feign = SSHOperator(
|
||||
ssh_hook=sshHook,
|
||||
task_id='custom_events_7_feign',
|
||||
|
|
@ -310,7 +300,7 @@ retries=3,
|
|||
dag=dag)
|
||||
|
||||
custom_events_7_feign >> custom_events_7_load
|
||||
|
||||
|
||||
custom_events_2_feign = SSHOperator(
|
||||
ssh_hook=sshHook,
|
||||
task_id='custom_events_2_feign',
|
||||
|
|
@ -329,7 +319,7 @@ retries=3,
|
|||
dag=dag)
|
||||
|
||||
custom_events_2_feign >> custom_events_2_load
|
||||
|
||||
|
||||
custom_events_5_feign = SSHOperator(
|
||||
ssh_hook=sshHook,
|
||||
task_id='custom_events_5_feign',
|
||||
|
|
@ -348,7 +338,7 @@ retries=3,
|
|||
dag=dag)
|
||||
|
||||
custom_events_5_feign >> custom_events_5_load
|
||||
|
||||
|
||||
customer_events_submit_form_4283 = SSHOperator(
|
||||
ssh_hook=sshHook,
|
||||
task_id='customer_events_submit_form_4283',
|
||||
|
|
@ -357,7 +347,7 @@ params={'my_param':"S98_S_customer_events_submit_form"},
|
|||
depends_on_past=False,
|
||||
retries=3,
|
||||
dag=dag)
|
||||
|
||||
|
||||
customer_events_wechat_scan_2105 = SSHOperator(
|
||||
ssh_hook=sshHook,
|
||||
task_id='customer_events_wechat_scan_2105',
|
||||
|
|
@ -366,7 +356,7 @@ params={'my_param':"S98_S_customer_events_wechat_scan"},
|
|||
depends_on_past=False,
|
||||
retries=3,
|
||||
dag=dag)
|
||||
|
||||
|
||||
customer_events_subscribe_5997 = SSHOperator(
|
||||
ssh_hook=sshHook,
|
||||
task_id='customer_events_subscribe_5997',
|
||||
|
|
@ -375,7 +365,7 @@ params={'my_param':"S98_S_customer_events_subscribe"},
|
|||
depends_on_past=False,
|
||||
retries=3,
|
||||
dag=dag)
|
||||
|
||||
|
||||
customer_events_click_link_in_page_2034 = SSHOperator(
|
||||
ssh_hook=sshHook,
|
||||
task_id='customer_events_click_link_in_page_2034',
|
||||
|
|
@ -384,7 +374,7 @@ params={'my_param':"S98_S_customer_events_click_link_in_page"},
|
|||
depends_on_past=False,
|
||||
retries=3,
|
||||
dag=dag)
|
||||
|
||||
|
||||
customer_events_open_content_page_9684 = SSHOperator(
|
||||
ssh_hook=sshHook,
|
||||
task_id='customer_events_open_content_page_9684',
|
||||
|
|
@ -393,7 +383,7 @@ params={'my_param':"S98_S_customer_events_open_content_page"},
|
|||
depends_on_past=False,
|
||||
retries=3,
|
||||
dag=dag)
|
||||
|
||||
|
||||
customer_events_open_app_5878 = SSHOperator(
|
||||
ssh_hook=sshHook,
|
||||
task_id='customer_events_open_app_5878',
|
||||
|
|
@ -402,7 +392,7 @@ params={'my_param':"S98_S_customer_events_open_app"},
|
|||
depends_on_past=False,
|
||||
retries=3,
|
||||
dag=dag)
|
||||
|
||||
|
||||
customer_events_open_page_7519 = SSHOperator(
|
||||
ssh_hook=sshHook,
|
||||
task_id='customer_events_open_page_7519',
|
||||
|
|
@ -411,7 +401,7 @@ params={'my_param':"S98_S_customer_events_open_page"},
|
|||
depends_on_past=False,
|
||||
retries=3,
|
||||
dag=dag)
|
||||
|
||||
|
||||
customer_events_c_minipro_page_view_9309 = SSHOperator(
|
||||
ssh_hook=sshHook,
|
||||
task_id='customer_events_c_minipro_page_view_9309',
|
||||
|
|
@ -420,7 +410,7 @@ params={'my_param':"S98_S_customer_events_c_minipro_page_view"},
|
|||
depends_on_past=False,
|
||||
retries=3,
|
||||
dag=dag)
|
||||
|
||||
|
||||
customer_events_add_user_9393 = SSHOperator(
|
||||
ssh_hook=sshHook,
|
||||
task_id='customer_events_add_user_9393',
|
||||
|
|
@ -429,33 +419,28 @@ params={'my_param':"S98_S_customer_events_add_user"},
|
|||
depends_on_past=False,
|
||||
retries=3,
|
||||
dag=dag)
|
||||
|
||||
scrm_contact_load >> scrm_contact_1129
|
||||
|
||||
update_scrm_contact_load >> scrm_contact_lastupdated_4112
|
||||
customer_event_meta_load >> customer_event_meta_2268
|
||||
custom_events_update_load >> customer_events_lastupdated_8895
|
||||
customer_events_load >> customer_events_3292
|
||||
custom_events_9_load >> customer_events_wechat_scan_2105
|
||||
custom_events_6_load >> customer_events_submit_form_4283
|
||||
custom_events_4_load >> customer_events_subscribe_5997
|
||||
custom_events_8_load >> customer_events_click_link_in_page_2034
|
||||
custom_events_1_load >> customer_events_open_app_5878
|
||||
custom_events_2_load >> customer_events_open_content_page_9684
|
||||
custom_events_3_load >> customer_events_open_page_7519
|
||||
custom_events_5_load >> customer_events_c_minipro_page_view_9309
|
||||
custom_events_7_load >> customer_events_add_user_9393
|
||||
|
||||
customer_events_add_user_9393 >> task_failed
|
||||
scrm_contact_lastupdated_4112 >> task_failed
|
||||
customer_event_meta_2268 >> task_failed
|
||||
customer_events_lastupdated_8895 >> task_failed
|
||||
customer_events_3292 >> task_failed
|
||||
customer_events_wechat_scan_2105 >> task_failed
|
||||
customer_events_submit_form_4283 >> task_failed
|
||||
customer_events_subscribe_5997 >> task_failed
|
||||
customer_events_click_link_in_page_2034 >> task_failed
|
||||
customer_events_open_app_5878 >> task_failed
|
||||
customer_events_open_content_page_9684 >> task_failed
|
||||
customer_events_open_page_7519 >> task_failed
|
||||
customer_events_c_minipro_page_view_9309 >> task_failed
|
||||
|
||||
scrm_contact_merge_6671 = SSHOperator(
|
||||
ssh_hook=sshHook,
|
||||
task_id='scrm_contact_merge_6671',
|
||||
command='/data/airflow/etl/SA/run_sa.sh {{ ds_nodash }} {{ params.my_param }} >>/data/airflow/logs/run_psql_{{ds_nodash}}.log 2>&1 ',
|
||||
params={'my_param':"S98_S_scrm_contact_merge"},
|
||||
depends_on_past=False,
|
||||
retries=3,
|
||||
dag=dag)
|
||||
|
||||
scrm_contact_load >> scrm_contact_1129
|
||||
update_scrm_contact_load >> scrm_contact_lastupdated_4112
|
||||
customer_event_meta_load >> customer_event_meta_2268
|
||||
customer_events_load >> customer_events_3292
|
||||
custom_events_9_load >> customer_events_wechat_scan_2105
|
||||
custom_events_6_load >> customer_events_submit_form_4283
|
||||
custom_events_4_load >> customer_events_subscribe_5997
|
||||
custom_events_8_load >> customer_events_click_link_in_page_2034
|
||||
custom_events_1_load >> customer_events_open_app_5878
|
||||
custom_events_2_load >> customer_events_open_content_page_9684
|
||||
custom_events_3_load >> customer_events_open_page_7519
|
||||
custom_events_5_load >> customer_events_c_minipro_page_view_9309
|
||||
custom_events_7_load >> customer_events_add_user_9393
|
||||
custom_events_update_load >> scrm_contact_merge_6671
|
||||
scrm_contact_merge_6671 >> task_failed
|
||||
|
|
|
|||
|
|
@ -6,408 +6,39 @@
|
|||
DELETE FROM data_api.scrm_contact;
|
||||
|
||||
insert into data_api.scrm_contact (
|
||||
accept_email
|
||||
, acceptsms
|
||||
, annual_revenue
|
||||
, attr_id
|
||||
, attr1
|
||||
, attr10
|
||||
, attr11
|
||||
, attr12
|
||||
, attr13
|
||||
, attr14
|
||||
, attr15
|
||||
, attr16
|
||||
, attr17
|
||||
, attr18
|
||||
, attr19
|
||||
, attr2
|
||||
, attr20
|
||||
, attr21
|
||||
, attr22
|
||||
, attr23
|
||||
, attr24
|
||||
, attr25
|
||||
, attr26
|
||||
, attr27
|
||||
, attr28
|
||||
, attr3
|
||||
, attr4
|
||||
, attr5
|
||||
, attr6
|
||||
, attr7
|
||||
, attr8
|
||||
, attr9
|
||||
, birthday
|
||||
, campaign_id
|
||||
, campaign_name
|
||||
id
|
||||
, city
|
||||
, company
|
||||
, content_name
|
||||
, convert_date
|
||||
, country
|
||||
, county
|
||||
, create_from
|
||||
, create_from_name
|
||||
, create_method
|
||||
, date_created
|
||||
, date_join
|
||||
, date_member_reg
|
||||
, display_name
|
||||
, email
|
||||
, email_verified
|
||||
, employee_number
|
||||
, external_id
|
||||
, extra_convert_content_name
|
||||
, extra_convert_date
|
||||
, extra_convert_source
|
||||
, extra_convert_source1
|
||||
, extra_convert_source2
|
||||
, extra_convert_source3
|
||||
, extra_convert_source4
|
||||
, extra_convert_source5
|
||||
, extra_convert_source6
|
||||
, extra_convert_source7
|
||||
, extra_convert_source8
|
||||
, extra_regist_content_name
|
||||
, extra_regist_date
|
||||
, extra_regist_source
|
||||
, extra_regist_source1
|
||||
, extra_regist_source2
|
||||
, extra_regist_source3
|
||||
, extra_regist_source4
|
||||
, extra_regist_source5
|
||||
, extra_regist_source6
|
||||
, extra_regist_source7
|
||||
, extra_regist_source8
|
||||
, first_assign_date
|
||||
, gender
|
||||
, home_address
|
||||
, id
|
||||
, id_str
|
||||
, img
|
||||
, industry
|
||||
, is_anonymous
|
||||
, is_employee
|
||||
, is_member
|
||||
, is_referrer
|
||||
, language
|
||||
, last_updated
|
||||
, latest_engage_date
|
||||
, mobile
|
||||
, mobile_verified
|
||||
, name
|
||||
, nick_name
|
||||
, not_accept_email
|
||||
, not_acceptsms
|
||||
, office_address
|
||||
, owner_date
|
||||
, owner_employee_id
|
||||
, owner_employee_name
|
||||
, owner_id
|
||||
, owner_name
|
||||
, province
|
||||
, qq
|
||||
, referrer
|
||||
, referrer_name
|
||||
, refund_date
|
||||
, region
|
||||
, remark
|
||||
, source
|
||||
, source1
|
||||
, source2
|
||||
, source3
|
||||
, source4
|
||||
, source5
|
||||
, source6
|
||||
, source7
|
||||
, source8
|
||||
, stage
|
||||
, status
|
||||
, street
|
||||
, subscribe_scene
|
||||
, telephone
|
||||
, title
|
||||
, type
|
||||
, update_method
|
||||
, utm_medium
|
||||
, utm_term
|
||||
, version
|
||||
, website
|
||||
, wechat
|
||||
, wechat_city
|
||||
, wechat_nick_name
|
||||
, weibo
|
||||
, date_join
|
||||
,etl_tx_dt
|
||||
)
|
||||
select
|
||||
case when trim(both from accept_email)='' then null else accept_email::text end accept_email
|
||||
, case when trim(both from acceptsms)='' then null else acceptsms::text end acceptsms
|
||||
, case when trim(both from annual_revenue)='' then null else annual_revenue::text end annual_revenue
|
||||
, case when trim(both from attr_id)='' then null else attr_id::text end attr_id
|
||||
, case when trim(both from attr1)='' then null else attr1::text end attr1
|
||||
, case when trim(both from attr10)='' then null else attr10::text end attr10
|
||||
, case when trim(both from attr11)='' then null else attr11::text end attr11
|
||||
, case when trim(both from attr12)='' then null else attr12::text end attr12
|
||||
, case when trim(both from attr13)='' then null else attr13::text end attr13
|
||||
, case when trim(both from attr14)='' then null else attr14::text end attr14
|
||||
, case when trim(both from attr15)='' then null else attr15::text end attr15
|
||||
, case when trim(both from attr16)='' then null else attr16::text end attr16
|
||||
, case when trim(both from attr17)='' then null else attr17::text end attr17
|
||||
, case when trim(both from attr18)='' then null else attr18::text end attr18
|
||||
, case when trim(both from attr19)='' then null else attr19::text end attr19
|
||||
, case when trim(both from attr2)='' then null else attr2::text end attr2
|
||||
, case when trim(both from attr20)='' then null else attr20::text end attr20
|
||||
, case when trim(both from attr21)='' then null else attr21::text end attr21
|
||||
, case when trim(both from attr22)='' then null else attr22::text end attr22
|
||||
, case when trim(both from attr23)='' then null else attr23::text end attr23
|
||||
, case when trim(both from attr24)='' then null else attr24::text end attr24
|
||||
, case when trim(both from attr25)='' then null else attr25::text end attr25
|
||||
, case when trim(both from attr26)='' then null else attr26::text end attr26
|
||||
, case when trim(both from attr27)='' then null else attr27::text end attr27
|
||||
, case when trim(both from attr28)='' then null else attr28::text end attr28
|
||||
, case when trim(both from attr3)='' then null else attr3::text end attr3
|
||||
, case when trim(both from attr4)='' then null else attr4::text end attr4
|
||||
, case when trim(both from attr5)='' then null else attr5::text end attr5
|
||||
, case when trim(both from attr6)='' then null else attr6::text end attr6
|
||||
, case when trim(both from attr7)='' then null else attr7::text end attr7
|
||||
, case when trim(both from attr8)='' then null else attr8::text end attr8
|
||||
, case when trim(both from attr9)='' then null else attr9::text end attr9
|
||||
, case when trim(both from birthday)='' then null else birthday::text end birthday
|
||||
, case when trim(both from campaign_id)='' then null else campaign_id::text end campaign_id
|
||||
, case when trim(both from campaign_name)='' then null else campaign_name::text end campaign_name
|
||||
case when trim(both from id)='' then null else id::text end id
|
||||
, case when trim(both from city)='' then null else city::text end city
|
||||
, case when trim(both from company)='' then null else company::text end company
|
||||
, case when trim(both from content_name)='' then null else content_name::text end content_name
|
||||
, case when trim(both from convert_date)='' then null else convert_date::text end convert_date
|
||||
, case when trim(both from country)='' then null else country::text end country
|
||||
, case when trim(both from county)='' then null else county::text end county
|
||||
, case when trim(both from create_from)='' then null else create_from::text end create_from
|
||||
, case when trim(both from create_from_name)='' then null else create_from_name::text end create_from_name
|
||||
, case when trim(both from create_method)='' then null else create_method::text end create_method
|
||||
, case when trim(both from date_created)='' then null else date_created::text end date_created
|
||||
, case when trim(both from date_join)='' then null else date_join::text end date_join
|
||||
, case when trim(both from date_member_reg)='' then null else date_member_reg::text end date_member_reg
|
||||
, case when trim(both from display_name)='' then null else display_name::text end display_name
|
||||
, case when trim(both from email)='' then null else email::text end email
|
||||
, case when trim(both from email_verified)='' then null else email_verified::text end email_verified
|
||||
, case when trim(both from employee_number)='' then null else employee_number::text end employee_number
|
||||
, case when trim(both from external_id)='' then null else external_id::text end external_id
|
||||
, case when trim(both from extra_convert_content_name)='' then null else extra_convert_content_name::text end extra_convert_content_name
|
||||
, case when trim(both from extra_convert_date)='' then null else extra_convert_date::text end extra_convert_date
|
||||
, case when trim(both from extra_convert_source)='' then null else extra_convert_source::text end extra_convert_source
|
||||
, case when trim(both from extra_convert_source1)='' then null else extra_convert_source1::text end extra_convert_source1
|
||||
, case when trim(both from extra_convert_source2)='' then null else extra_convert_source2::text end extra_convert_source2
|
||||
, case when trim(both from extra_convert_source3)='' then null else extra_convert_source3::text end extra_convert_source3
|
||||
, case when trim(both from extra_convert_source4)='' then null else extra_convert_source4::text end extra_convert_source4
|
||||
, case when trim(both from extra_convert_source5)='' then null else extra_convert_source5::text end extra_convert_source5
|
||||
, case when trim(both from extra_convert_source6)='' then null else extra_convert_source6::text end extra_convert_source6
|
||||
, case when trim(both from extra_convert_source7)='' then null else extra_convert_source7::text end extra_convert_source7
|
||||
, case when trim(both from extra_convert_source8)='' then null else extra_convert_source8::text end extra_convert_source8
|
||||
, case when trim(both from extra_regist_content_name)='' then null else extra_regist_content_name::text end extra_regist_content_name
|
||||
, case when trim(both from extra_regist_date)='' then null else extra_regist_date::text end extra_regist_date
|
||||
, case when trim(both from extra_regist_source)='' then null else extra_regist_source::text end extra_regist_source
|
||||
, case when trim(both from extra_regist_source1)='' then null else extra_regist_source1::text end extra_regist_source1
|
||||
, case when trim(both from extra_regist_source2)='' then null else extra_regist_source2::text end extra_regist_source2
|
||||
, case when trim(both from extra_regist_source3)='' then null else extra_regist_source3::text end extra_regist_source3
|
||||
, case when trim(both from extra_regist_source4)='' then null else extra_regist_source4::text end extra_regist_source4
|
||||
, case when trim(both from extra_regist_source5)='' then null else extra_regist_source5::text end extra_regist_source5
|
||||
, case when trim(both from extra_regist_source6)='' then null else extra_regist_source6::text end extra_regist_source6
|
||||
, case when trim(both from extra_regist_source7)='' then null else extra_regist_source7::text end extra_regist_source7
|
||||
, case when trim(both from extra_regist_source8)='' then null else extra_regist_source8::text end extra_regist_source8
|
||||
, case when trim(both from first_assign_date)='' then null else first_assign_date::text end first_assign_date
|
||||
, case when trim(both from gender)='' then null else gender::text end gender
|
||||
, case when trim(both from home_address)='' then null else home_address::text end home_address
|
||||
, case when trim(both from id)='' then null else id::text end id
|
||||
, case when trim(both from id_str)='' then null else id_str::text end id_str
|
||||
, case when trim(both from img)='' then null else img::text end img
|
||||
, case when trim(both from industry)='' then null else industry::text end industry
|
||||
, case when trim(both from is_anonymous)='' then null else is_anonymous::text end is_anonymous
|
||||
, case when trim(both from is_employee)='' then null else is_employee::text end is_employee
|
||||
, case when trim(both from is_member)='' then null else is_member::text end is_member
|
||||
, case when trim(both from is_referrer)='' then null else is_referrer::text end is_referrer
|
||||
, case when trim(both from language)='' then null else language::text end language
|
||||
, case when trim(both from last_updated)='' then null else last_updated::text end last_updated
|
||||
, case when trim(both from latest_engage_date)='' then null else latest_engage_date::text end latest_engage_date
|
||||
, case when trim(both from mobile)='' then null else mobile::text end mobile
|
||||
, case when trim(both from mobile_verified)='' then null else mobile_verified::text end mobile_verified
|
||||
, case when trim(both from name)='' then null else name::text end name
|
||||
, case when trim(both from nick_name)='' then null else nick_name::text end nick_name
|
||||
, case when trim(both from not_accept_email)='' then null else not_accept_email::text end not_accept_email
|
||||
, case when trim(both from not_acceptsms)='' then null else not_acceptsms::text end not_acceptsms
|
||||
, case when trim(both from office_address)='' then null else office_address::text end office_address
|
||||
, case when trim(both from owner_date)='' then null else owner_date::text end owner_date
|
||||
, case when trim(both from owner_employee_id)='' then null else owner_employee_id::text end owner_employee_id
|
||||
, case when trim(both from owner_employee_name)='' then null else owner_employee_name::text end owner_employee_name
|
||||
, case when trim(both from owner_id)='' then null else owner_id::text end owner_id
|
||||
, case when trim(both from owner_name)='' then null else owner_name::text end owner_name
|
||||
, case when trim(both from province)='' then null else province::text end province
|
||||
, case when trim(both from qq)='' then null else qq::text end qq
|
||||
, case when trim(both from referrer)='' then null else referrer::text end referrer
|
||||
, case when trim(both from referrer_name)='' then null else referrer_name::text end referrer_name
|
||||
, case when trim(both from refund_date)='' then null else refund_date::text end refund_date
|
||||
, case when trim(both from region)='' then null else region::text end region
|
||||
, case when trim(both from remark)='' then null else remark::text end remark
|
||||
, case when trim(both from source)='' then null else source::text end source
|
||||
, case when trim(both from source1)='' then null else source1::text end source1
|
||||
, case when trim(both from source2)='' then null else source2::text end source2
|
||||
, case when trim(both from source3)='' then null else source3::text end source3
|
||||
, case when trim(both from source4)='' then null else source4::text end source4
|
||||
, case when trim(both from source5)='' then null else source5::text end source5
|
||||
, case when trim(both from source6)='' then null else source6::text end source6
|
||||
, case when trim(both from source7)='' then null else source7::text end source7
|
||||
, case when trim(both from source8)='' then null else source8::text end source8
|
||||
, case when trim(both from stage)='' then null else stage::text end stage
|
||||
, case when trim(both from status)='' then null else status::text end status
|
||||
, case when trim(both from street)='' then null else street::text end street
|
||||
, case when trim(both from subscribe_scene)='' then null else subscribe_scene::text end subscribe_scene
|
||||
, case when trim(both from telephone)='' then null else telephone::text end telephone
|
||||
, case when trim(both from title)='' then null else title::text end title
|
||||
, case when trim(both from type)='' then null else type::text end type
|
||||
, case when trim(both from update_method)='' then null else update_method::text end update_method
|
||||
, case when trim(both from utm_medium)='' then null else utm_medium::text end utm_medium
|
||||
, case when trim(both from utm_term)='' then null else utm_term::text end utm_term
|
||||
, case when trim(both from version)='' then null else version::text end version
|
||||
, case when trim(both from website)='' then null else website::text end website
|
||||
, case when trim(both from wechat)='' then null else wechat::text end wechat
|
||||
, case when trim(both from wechat_city)='' then null else wechat_city::text end wechat_city
|
||||
, case when trim(both from wechat_nick_name)='' then null else wechat_nick_name::text end wechat_nick_name
|
||||
, case when trim(both from weibo)='' then null else weibo::text end weibo
|
||||
, case when trim(both from date_join)='' then null else date_join::text end date_join
|
||||
,etl_tx_dt
|
||||
from (
|
||||
select
|
||||
(json_array_elements(data::json)::json->>'acceptEmail') accept_email
|
||||
, (json_array_elements(data::json)::json->>'acceptsms') acceptsms
|
||||
, (json_array_elements(data::json)::json->>'annualRevenue') annual_revenue
|
||||
, (json_array_elements(data::json)::json->>'attrId') attr_id
|
||||
, (json_array_elements(data::json)::json->>'attr1') attr1
|
||||
, (json_array_elements(data::json)::json->>'attr10') attr10
|
||||
, (json_array_elements(data::json)::json->>'attr11') attr11
|
||||
, (json_array_elements(data::json)::json->>'attr12') attr12
|
||||
, (json_array_elements(data::json)::json->>'attr13') attr13
|
||||
, (json_array_elements(data::json)::json->>'attr14') attr14
|
||||
, (json_array_elements(data::json)::json->>'attr15') attr15
|
||||
, (json_array_elements(data::json)::json->>'attr16') attr16
|
||||
, (json_array_elements(data::json)::json->>'attr17') attr17
|
||||
, (json_array_elements(data::json)::json->>'attr18') attr18
|
||||
, (json_array_elements(data::json)::json->>'attr19') attr19
|
||||
, (json_array_elements(data::json)::json->>'attr2') attr2
|
||||
, (json_array_elements(data::json)::json->>'attr20') attr20
|
||||
, (json_array_elements(data::json)::json->>'attr21') attr21
|
||||
, (json_array_elements(data::json)::json->>'attr22') attr22
|
||||
, (json_array_elements(data::json)::json->>'attr23') attr23
|
||||
, (json_array_elements(data::json)::json->>'attr24') attr24
|
||||
, (json_array_elements(data::json)::json->>'attr25') attr25
|
||||
, (json_array_elements(data::json)::json->>'attr26') attr26
|
||||
, (json_array_elements(data::json)::json->>'attr27') attr27
|
||||
, (json_array_elements(data::json)::json->>'attr28') attr28
|
||||
, (json_array_elements(data::json)::json->>'attr3') attr3
|
||||
, (json_array_elements(data::json)::json->>'attr4') attr4
|
||||
, (json_array_elements(data::json)::json->>'attr5') attr5
|
||||
, (json_array_elements(data::json)::json->>'attr6') attr6
|
||||
, (json_array_elements(data::json)::json->>'attr7') attr7
|
||||
, (json_array_elements(data::json)::json->>'attr8') attr8
|
||||
, (json_array_elements(data::json)::json->>'attr9') attr9
|
||||
, (json_array_elements(data::json)::json->>'birthday') birthday
|
||||
, (json_array_elements(data::json)::json->>'campaignId') campaign_id
|
||||
, (json_array_elements(data::json)::json->>'campaignName') campaign_name
|
||||
(json_array_elements(data::json)::json->>'id') id
|
||||
, (json_array_elements(data::json)::json->>'city') city
|
||||
, (json_array_elements(data::json)::json->>'company') company
|
||||
, (json_array_elements(data::json)::json->>'contentName') content_name
|
||||
, (json_array_elements(data::json)::json->>'convertDate') convert_date
|
||||
, (json_array_elements(data::json)::json->>'country') country
|
||||
, (json_array_elements(data::json)::json->>'county') county
|
||||
, (json_array_elements(data::json)::json->>'createFrom') create_from
|
||||
, (json_array_elements(data::json)::json->>'createFromName') create_from_name
|
||||
, (json_array_elements(data::json)::json->>'createMethod') create_method
|
||||
, (json_array_elements(data::json)::json->>'dateCreated') date_created
|
||||
, (json_array_elements(data::json)::json->>'dateJoin') date_join
|
||||
, (json_array_elements(data::json)::json->>'dateMemberReg') date_member_reg
|
||||
, (json_array_elements(data::json)::json->>'displayName') display_name
|
||||
, (json_array_elements(data::json)::json->>'email') email
|
||||
, (json_array_elements(data::json)::json->>'emailVerified') email_verified
|
||||
, (json_array_elements(data::json)::json->>'employeeNumber') employee_number
|
||||
, (json_array_elements(data::json)::json->>'externalId') external_id
|
||||
, (json_array_elements(data::json)::json->>'extraConvertContentName') extra_convert_content_name
|
||||
, (json_array_elements(data::json)::json->>'extraConvertDate') extra_convert_date
|
||||
, (json_array_elements(data::json)::json->>'extraConvertSource') extra_convert_source
|
||||
, (json_array_elements(data::json)::json->>'extraConvertSource1') extra_convert_source1
|
||||
, (json_array_elements(data::json)::json->>'extraConvertSource2') extra_convert_source2
|
||||
, (json_array_elements(data::json)::json->>'extraConvertSource3') extra_convert_source3
|
||||
, (json_array_elements(data::json)::json->>'extraConvertSource4') extra_convert_source4
|
||||
, (json_array_elements(data::json)::json->>'extraConvertSource5') extra_convert_source5
|
||||
, (json_array_elements(data::json)::json->>'extraConvertSource6') extra_convert_source6
|
||||
, (json_array_elements(data::json)::json->>'extraConvertSource7') extra_convert_source7
|
||||
, (json_array_elements(data::json)::json->>'extraConvertSource8') extra_convert_source8
|
||||
, (json_array_elements(data::json)::json->>'extraRegistContentName') extra_regist_content_name
|
||||
, (json_array_elements(data::json)::json->>'extraRegistDate') extra_regist_date
|
||||
, (json_array_elements(data::json)::json->>'extraRegistSource') extra_regist_source
|
||||
, (json_array_elements(data::json)::json->>'extraRegistSource1') extra_regist_source1
|
||||
, (json_array_elements(data::json)::json->>'extraRegistSource2') extra_regist_source2
|
||||
, (json_array_elements(data::json)::json->>'extraRegistSource3') extra_regist_source3
|
||||
, (json_array_elements(data::json)::json->>'extraRegistSource4') extra_regist_source4
|
||||
, (json_array_elements(data::json)::json->>'extraRegistSource5') extra_regist_source5
|
||||
, (json_array_elements(data::json)::json->>'extraRegistSource6') extra_regist_source6
|
||||
, (json_array_elements(data::json)::json->>'extraRegistSource7') extra_regist_source7
|
||||
, (json_array_elements(data::json)::json->>'extraRegistSource8') extra_regist_source8
|
||||
, (json_array_elements(data::json)::json->>'firstAssignDate') first_assign_date
|
||||
, (json_array_elements(data::json)::json->>'gender') gender
|
||||
, (json_array_elements(data::json)::json->>'homeAddress') home_address
|
||||
, (json_array_elements(data::json)::json->>'id') id
|
||||
, (json_array_elements(data::json)::json->>'idStr') id_str
|
||||
, (json_array_elements(data::json)::json->>'img') img
|
||||
, (json_array_elements(data::json)::json->>'industry') industry
|
||||
, (json_array_elements(data::json)::json->>'isAnonymous') is_anonymous
|
||||
, (json_array_elements(data::json)::json->>'isEmployee') is_employee
|
||||
, (json_array_elements(data::json)::json->>'isMember') is_member
|
||||
, (json_array_elements(data::json)::json->>'isReferrer') is_referrer
|
||||
, (json_array_elements(data::json)::json->>'language') language
|
||||
, (json_array_elements(data::json)::json->>'lastUpdated') last_updated
|
||||
, (json_array_elements(data::json)::json->>'latestEngageDate') latest_engage_date
|
||||
, (json_array_elements(data::json)::json->>'mobile') mobile
|
||||
, (json_array_elements(data::json)::json->>'mobileVerified') mobile_verified
|
||||
, (json_array_elements(data::json)::json->>'name') name
|
||||
, (json_array_elements(data::json)::json->>'nickName') nick_name
|
||||
, (json_array_elements(data::json)::json->>'notAcceptEmail') not_accept_email
|
||||
, (json_array_elements(data::json)::json->>'notAcceptsms') not_acceptsms
|
||||
, (json_array_elements(data::json)::json->>'officeAddress') office_address
|
||||
, (json_array_elements(data::json)::json->>'ownerDate') owner_date
|
||||
, (json_array_elements(data::json)::json->>'ownerEmployeeId') owner_employee_id
|
||||
, (json_array_elements(data::json)::json->>'ownerEmployeeName') owner_employee_name
|
||||
, (json_array_elements(data::json)::json->>'ownerId') owner_id
|
||||
, (json_array_elements(data::json)::json->>'ownerName') owner_name
|
||||
, (json_array_elements(data::json)::json->>'province') province
|
||||
, (json_array_elements(data::json)::json->>'qq') qq
|
||||
, (json_array_elements(data::json)::json->>'referrer') referrer
|
||||
, (json_array_elements(data::json)::json->>'referrerName') referrer_name
|
||||
, (json_array_elements(data::json)::json->>'refundDate') refund_date
|
||||
, (json_array_elements(data::json)::json->>'region') region
|
||||
, (json_array_elements(data::json)::json->>'remark') remark
|
||||
, (json_array_elements(data::json)::json->>'source') source
|
||||
, (json_array_elements(data::json)::json->>'source1') source1
|
||||
, (json_array_elements(data::json)::json->>'source2') source2
|
||||
, (json_array_elements(data::json)::json->>'source3') source3
|
||||
, (json_array_elements(data::json)::json->>'source4') source4
|
||||
, (json_array_elements(data::json)::json->>'source5') source5
|
||||
, (json_array_elements(data::json)::json->>'source6') source6
|
||||
, (json_array_elements(data::json)::json->>'source7') source7
|
||||
, (json_array_elements(data::json)::json->>'source8') source8
|
||||
, (json_array_elements(data::json)::json->>'stage') stage
|
||||
, (json_array_elements(data::json)::json->>'status') status
|
||||
, (json_array_elements(data::json)::json->>'street') street
|
||||
, (json_array_elements(data::json)::json->>'subscribeScene') subscribe_scene
|
||||
, (json_array_elements(data::json)::json->>'telephone') telephone
|
||||
, (json_array_elements(data::json)::json->>'title') title
|
||||
, (json_array_elements(data::json)::json->>'type') type
|
||||
, (json_array_elements(data::json)::json->>'updateMethod') update_method
|
||||
, (json_array_elements(data::json)::json->>'utmMedium') utm_medium
|
||||
, (json_array_elements(data::json)::json->>'utmTerm') utm_term
|
||||
, (json_array_elements(data::json)::json->>'version') version
|
||||
, (json_array_elements(data::json)::json->>'website') website
|
||||
, (json_array_elements(data::json)::json->>'wechat') wechat
|
||||
, (json_array_elements(data::json)::json->>'wechatCity') wechat_city
|
||||
, (json_array_elements(data::json)::json->>'wechatNickName') wechat_nick_name
|
||||
, (json_array_elements(data::json)::json->>'weibo') weibo
|
||||
, (json_array_elements(data::json)::json->>'dateJoin') date_join
|
||||
,CURRENT_TIMESTAMP(0) etl_tx_dt
|
||||
from (select * from data_api.api_data
|
||||
WHERE api_id='cab1a761-b8e5-4db4-a769-88db3ec1' and is_loaded = '0' order by request_tm desc limit 1) p )p;
|
||||
|
|
|
|||
Loading…
Reference in New Issue