add workflow partner1site,dev

This commit is contained in:
root 2025-12-02 16:24:11 +08:00
parent 41d45dffb5
commit 34f4c09c23
11 changed files with 347 additions and 300 deletions

View File

@ -18,6 +18,7 @@ insert into p10_sa.S98_S_partner_summary_pos
, contact_phone , contact_phone
, contact_title , contact_title
, customer_chanel_name , customer_chanel_name
, customer_id
, customer_name , customer_name
, distributor_name , distributor_name
, distributor_name2 , distributor_name2
@ -25,12 +26,15 @@ insert into p10_sa.S98_S_partner_summary_pos
, id , id
, industry , industry
, invoice_number , invoice_number
, net_usd
, online_name , online_name
, online_or_offline , online_or_offline
, pos_insert_date , pos_insert_date
, pos_invoice_date , pos_invoice_date
, pos_update_date , pos_update_date
, product_ap , product_ap
, product_family_code
, product_family_name
, product_qty , product_qty
, province , province
, sales_name , sales_name
@ -56,6 +60,7 @@ insert into p10_sa.S98_S_partner_summary_pos
, contact_phone , contact_phone
, contact_title , contact_title
, customer_chanel_name , customer_chanel_name
, customer_id
, customer_name , customer_name
, distributor_name , distributor_name
, distributor_name2 , distributor_name2
@ -63,12 +68,15 @@ insert into p10_sa.S98_S_partner_summary_pos
, id , id
, industry , industry
, invoice_number , invoice_number
, net_usd
, online_name , online_name
, online_or_offline , online_or_offline
, pos_insert_date , pos_insert_date
, pos_invoice_date , pos_invoice_date
, pos_update_date , pos_update_date
, product_ap , product_ap
, product_family_code
, product_family_name
, product_qty , product_qty
, province , province
, sales_name , sales_name
@ -99,6 +107,7 @@ insert into p12_sfull.S98_S_partner_summary_pos
, contact_phone , contact_phone
, contact_title , contact_title
, customer_chanel_name , customer_chanel_name
, customer_id
, customer_name , customer_name
, distributor_name , distributor_name
, distributor_name2 , distributor_name2
@ -106,12 +115,15 @@ insert into p12_sfull.S98_S_partner_summary_pos
, id , id
, industry , industry
, invoice_number , invoice_number
, net_usd
, online_name , online_name
, online_or_offline , online_or_offline
, pos_insert_date , pos_insert_date
, pos_invoice_date , pos_invoice_date
, pos_update_date , pos_update_date
, product_ap , product_ap
, product_family_code
, product_family_name
, product_qty , product_qty
, province , province
, sales_name , sales_name
@ -137,6 +149,7 @@ insert into p12_sfull.S98_S_partner_summary_pos
, contact_phone , contact_phone
, contact_title , contact_title
, customer_chanel_name , customer_chanel_name
, customer_id
, customer_name , customer_name
, distributor_name , distributor_name
, distributor_name2 , distributor_name2
@ -144,12 +157,15 @@ insert into p12_sfull.S98_S_partner_summary_pos
, id , id
, industry , industry
, invoice_number , invoice_number
, net_usd
, online_name , online_name
, online_or_offline , online_or_offline
, pos_insert_date , pos_insert_date
, pos_invoice_date , pos_invoice_date
, pos_update_date , pos_update_date
, product_ap , product_ap
, product_family_code
, product_family_name
, product_qty , product_qty
, province , province
, sales_name , sales_name

View File

@ -13,6 +13,7 @@ CREATE FOREIGN TABLE if not exists p00_tal.S98_S_partner_summary_pos (
, contact_phone TEXT , contact_phone TEXT
, contact_title TEXT , contact_title TEXT
, customer_chanel_name TEXT , customer_chanel_name TEXT
, customer_id TEXT
, customer_name TEXT , customer_name TEXT
, distributor_name TEXT , distributor_name TEXT
, distributor_name2 TEXT , distributor_name2 TEXT
@ -20,12 +21,15 @@ CREATE FOREIGN TABLE if not exists p00_tal.S98_S_partner_summary_pos (
, id TEXT , id TEXT
, industry TEXT , industry TEXT
, invoice_number TEXT , invoice_number TEXT
, net_usd TEXT
, online_name TEXT , online_name TEXT
, online_or_offline TEXT , online_or_offline TEXT
, pos_insert_date TEXT , pos_insert_date TEXT
, pos_invoice_date TEXT , pos_invoice_date TEXT
, pos_update_date TEXT , pos_update_date TEXT
, product_ap TEXT , product_ap TEXT
, product_family_code TEXT
, product_family_name TEXT
, product_qty TEXT , product_qty TEXT
, province TEXT , province TEXT
, sales_name TEXT , sales_name TEXT

View File

@ -13,6 +13,7 @@ create table if not exists p10_sa.S98_S_partner_summary_pos (
, contact_phone TEXT , contact_phone TEXT
, contact_title TEXT , contact_title TEXT
, customer_chanel_name TEXT , customer_chanel_name TEXT
, customer_id TEXT
, customer_name TEXT , customer_name TEXT
, distributor_name TEXT , distributor_name TEXT
, distributor_name2 TEXT , distributor_name2 TEXT
@ -20,12 +21,15 @@ create table if not exists p10_sa.S98_S_partner_summary_pos (
, id TEXT , id TEXT
, industry TEXT , industry TEXT
, invoice_number TEXT , invoice_number TEXT
, net_usd TEXT
, online_name TEXT , online_name TEXT
, online_or_offline TEXT , online_or_offline TEXT
, pos_insert_date TEXT , pos_insert_date TEXT
, pos_invoice_date TEXT , pos_invoice_date TEXT
, pos_update_date TEXT , pos_update_date TEXT
, product_ap TEXT , product_ap TEXT
, product_family_code TEXT
, product_family_name TEXT
, product_qty TEXT , product_qty TEXT
, province TEXT , province TEXT
, sales_name TEXT , sales_name TEXT
@ -52,6 +56,7 @@ create table if not exists p10_sa.S98_S_partner_summary_pos (
COMMENT ON COLUMN p10_sa.S98_S_partner_summary_pos.contact_phone IS ''; COMMENT ON COLUMN p10_sa.S98_S_partner_summary_pos.contact_phone IS '';
COMMENT ON COLUMN p10_sa.S98_S_partner_summary_pos.contact_title IS ''; COMMENT ON COLUMN p10_sa.S98_S_partner_summary_pos.contact_title IS '';
COMMENT ON COLUMN p10_sa.S98_S_partner_summary_pos.customer_chanel_name IS ''; COMMENT ON COLUMN p10_sa.S98_S_partner_summary_pos.customer_chanel_name IS '';
COMMENT ON COLUMN p10_sa.S98_S_partner_summary_pos.customer_id IS '';
COMMENT ON COLUMN p10_sa.S98_S_partner_summary_pos.customer_name IS ''; COMMENT ON COLUMN p10_sa.S98_S_partner_summary_pos.customer_name IS '';
COMMENT ON COLUMN p10_sa.S98_S_partner_summary_pos.distributor_name IS ''; COMMENT ON COLUMN p10_sa.S98_S_partner_summary_pos.distributor_name IS '';
COMMENT ON COLUMN p10_sa.S98_S_partner_summary_pos.distributor_name2 IS ''; COMMENT ON COLUMN p10_sa.S98_S_partner_summary_pos.distributor_name2 IS '';
@ -59,12 +64,15 @@ create table if not exists p10_sa.S98_S_partner_summary_pos (
COMMENT ON COLUMN p10_sa.S98_S_partner_summary_pos.id IS ''; COMMENT ON COLUMN p10_sa.S98_S_partner_summary_pos.id IS '';
COMMENT ON COLUMN p10_sa.S98_S_partner_summary_pos.industry IS ''; COMMENT ON COLUMN p10_sa.S98_S_partner_summary_pos.industry IS '';
COMMENT ON COLUMN p10_sa.S98_S_partner_summary_pos.invoice_number IS ''; COMMENT ON COLUMN p10_sa.S98_S_partner_summary_pos.invoice_number IS '';
COMMENT ON COLUMN p10_sa.S98_S_partner_summary_pos.net_usd IS '';
COMMENT ON COLUMN p10_sa.S98_S_partner_summary_pos.online_name IS ''; COMMENT ON COLUMN p10_sa.S98_S_partner_summary_pos.online_name IS '';
COMMENT ON COLUMN p10_sa.S98_S_partner_summary_pos.online_or_offline IS ''; COMMENT ON COLUMN p10_sa.S98_S_partner_summary_pos.online_or_offline IS '';
COMMENT ON COLUMN p10_sa.S98_S_partner_summary_pos.pos_insert_date IS ''; COMMENT ON COLUMN p10_sa.S98_S_partner_summary_pos.pos_insert_date IS '';
COMMENT ON COLUMN p10_sa.S98_S_partner_summary_pos.pos_invoice_date IS ''; COMMENT ON COLUMN p10_sa.S98_S_partner_summary_pos.pos_invoice_date IS '';
COMMENT ON COLUMN p10_sa.S98_S_partner_summary_pos.pos_update_date IS ''; COMMENT ON COLUMN p10_sa.S98_S_partner_summary_pos.pos_update_date IS '';
COMMENT ON COLUMN p10_sa.S98_S_partner_summary_pos.product_ap IS ''; COMMENT ON COLUMN p10_sa.S98_S_partner_summary_pos.product_ap IS '';
COMMENT ON COLUMN p10_sa.S98_S_partner_summary_pos.product_family_code IS '';
COMMENT ON COLUMN p10_sa.S98_S_partner_summary_pos.product_family_name IS '';
COMMENT ON COLUMN p10_sa.S98_S_partner_summary_pos.product_qty IS ''; COMMENT ON COLUMN p10_sa.S98_S_partner_summary_pos.product_qty IS '';
COMMENT ON COLUMN p10_sa.S98_S_partner_summary_pos.province IS ''; COMMENT ON COLUMN p10_sa.S98_S_partner_summary_pos.province IS '';
COMMENT ON COLUMN p10_sa.S98_S_partner_summary_pos.sales_name IS ''; COMMENT ON COLUMN p10_sa.S98_S_partner_summary_pos.sales_name IS '';
@ -95,6 +103,7 @@ create table if not exists p12_sfull.S98_S_partner_summary_pos (
, contact_phone TEXT , contact_phone TEXT
, contact_title TEXT , contact_title TEXT
, customer_chanel_name TEXT , customer_chanel_name TEXT
, customer_id TEXT
, customer_name TEXT , customer_name TEXT
, distributor_name TEXT , distributor_name TEXT
, distributor_name2 TEXT , distributor_name2 TEXT
@ -102,12 +111,15 @@ create table if not exists p12_sfull.S98_S_partner_summary_pos (
, id TEXT , id TEXT
, industry TEXT , industry TEXT
, invoice_number TEXT , invoice_number TEXT
, net_usd TEXT
, online_name TEXT , online_name TEXT
, online_or_offline TEXT , online_or_offline TEXT
, pos_insert_date TEXT , pos_insert_date TEXT
, pos_invoice_date TEXT , pos_invoice_date TEXT
, pos_update_date TEXT , pos_update_date TEXT
, product_ap TEXT , product_ap TEXT
, product_family_code TEXT
, product_family_name TEXT
, product_qty TEXT , product_qty TEXT
, province TEXT , province TEXT
, sales_name TEXT , sales_name TEXT
@ -134,6 +146,7 @@ create table if not exists p12_sfull.S98_S_partner_summary_pos (
COMMENT ON COLUMN p12_sfull.S98_S_partner_summary_pos.contact_phone IS ''; COMMENT ON COLUMN p12_sfull.S98_S_partner_summary_pos.contact_phone IS '';
COMMENT ON COLUMN p12_sfull.S98_S_partner_summary_pos.contact_title IS ''; COMMENT ON COLUMN p12_sfull.S98_S_partner_summary_pos.contact_title IS '';
COMMENT ON COLUMN p12_sfull.S98_S_partner_summary_pos.customer_chanel_name IS ''; COMMENT ON COLUMN p12_sfull.S98_S_partner_summary_pos.customer_chanel_name IS '';
COMMENT ON COLUMN p12_sfull.S98_S_partner_summary_pos.customer_id IS '';
COMMENT ON COLUMN p12_sfull.S98_S_partner_summary_pos.customer_name IS ''; COMMENT ON COLUMN p12_sfull.S98_S_partner_summary_pos.customer_name IS '';
COMMENT ON COLUMN p12_sfull.S98_S_partner_summary_pos.distributor_name IS ''; COMMENT ON COLUMN p12_sfull.S98_S_partner_summary_pos.distributor_name IS '';
COMMENT ON COLUMN p12_sfull.S98_S_partner_summary_pos.distributor_name2 IS ''; COMMENT ON COLUMN p12_sfull.S98_S_partner_summary_pos.distributor_name2 IS '';
@ -141,12 +154,15 @@ create table if not exists p12_sfull.S98_S_partner_summary_pos (
COMMENT ON COLUMN p12_sfull.S98_S_partner_summary_pos.id IS ''; COMMENT ON COLUMN p12_sfull.S98_S_partner_summary_pos.id IS '';
COMMENT ON COLUMN p12_sfull.S98_S_partner_summary_pos.industry IS ''; COMMENT ON COLUMN p12_sfull.S98_S_partner_summary_pos.industry IS '';
COMMENT ON COLUMN p12_sfull.S98_S_partner_summary_pos.invoice_number IS ''; COMMENT ON COLUMN p12_sfull.S98_S_partner_summary_pos.invoice_number IS '';
COMMENT ON COLUMN p12_sfull.S98_S_partner_summary_pos.net_usd IS '';
COMMENT ON COLUMN p12_sfull.S98_S_partner_summary_pos.online_name IS ''; COMMENT ON COLUMN p12_sfull.S98_S_partner_summary_pos.online_name IS '';
COMMENT ON COLUMN p12_sfull.S98_S_partner_summary_pos.online_or_offline IS ''; COMMENT ON COLUMN p12_sfull.S98_S_partner_summary_pos.online_or_offline IS '';
COMMENT ON COLUMN p12_sfull.S98_S_partner_summary_pos.pos_insert_date IS ''; COMMENT ON COLUMN p12_sfull.S98_S_partner_summary_pos.pos_insert_date IS '';
COMMENT ON COLUMN p12_sfull.S98_S_partner_summary_pos.pos_invoice_date IS ''; COMMENT ON COLUMN p12_sfull.S98_S_partner_summary_pos.pos_invoice_date IS '';
COMMENT ON COLUMN p12_sfull.S98_S_partner_summary_pos.pos_update_date IS ''; COMMENT ON COLUMN p12_sfull.S98_S_partner_summary_pos.pos_update_date IS '';
COMMENT ON COLUMN p12_sfull.S98_S_partner_summary_pos.product_ap IS ''; COMMENT ON COLUMN p12_sfull.S98_S_partner_summary_pos.product_ap IS '';
COMMENT ON COLUMN p12_sfull.S98_S_partner_summary_pos.product_family_code IS '';
COMMENT ON COLUMN p12_sfull.S98_S_partner_summary_pos.product_family_name IS '';
COMMENT ON COLUMN p12_sfull.S98_S_partner_summary_pos.product_qty IS ''; COMMENT ON COLUMN p12_sfull.S98_S_partner_summary_pos.product_qty IS '';
COMMENT ON COLUMN p12_sfull.S98_S_partner_summary_pos.province IS ''; COMMENT ON COLUMN p12_sfull.S98_S_partner_summary_pos.province IS '';
COMMENT ON COLUMN p12_sfull.S98_S_partner_summary_pos.sales_name IS ''; COMMENT ON COLUMN p12_sfull.S98_S_partner_summary_pos.sales_name IS '';

View File

@ -33,6 +33,7 @@ insert into p10_sa.S98_S_partner_summary_visit
, like_number , like_number
, sub_industry , sub_industry
, tsm_names_by_alias , tsm_names_by_alias
, update_date
, visit_remark , visit_remark
, visitor , visitor
, etl_tx_dt ) , etl_tx_dt )
@ -65,6 +66,7 @@ insert into p10_sa.S98_S_partner_summary_visit
, like_number , like_number
, sub_industry , sub_industry
, tsm_names_by_alias , tsm_names_by_alias
, update_date
, visit_remark , visit_remark
, visitor , visitor
, etl_tx_dt , etl_tx_dt
@ -102,6 +104,7 @@ insert into p12_sfull.S98_S_partner_summary_visit
, like_number , like_number
, sub_industry , sub_industry
, tsm_names_by_alias , tsm_names_by_alias
, update_date
, visit_remark , visit_remark
, visitor , visitor
, etl_tx_dt ) , etl_tx_dt )
@ -134,6 +137,7 @@ insert into p12_sfull.S98_S_partner_summary_visit
, like_number , like_number
, sub_industry , sub_industry
, tsm_names_by_alias , tsm_names_by_alias
, update_date
, visit_remark , visit_remark
, visitor , visitor
, etl_tx_dt , etl_tx_dt

View File

@ -28,6 +28,7 @@ CREATE FOREIGN TABLE if not exists p00_tal.S98_S_partner_summary_visit (
, like_number TEXT , like_number TEXT
, sub_industry TEXT , sub_industry TEXT
, tsm_names_by_alias TEXT , tsm_names_by_alias TEXT
, update_date TEXT
, visit_remark TEXT , visit_remark TEXT
, visitor TEXT , visitor TEXT
, etl_tx_dt TIMESTAMP , etl_tx_dt TIMESTAMP

View File

@ -28,6 +28,7 @@ create table if not exists p10_sa.S98_S_partner_summary_visit (
, like_number TEXT , like_number TEXT
, sub_industry TEXT , sub_industry TEXT
, tsm_names_by_alias TEXT , tsm_names_by_alias TEXT
, update_date TEXT
, visit_remark TEXT , visit_remark TEXT
, visitor TEXT , visitor TEXT
, etl_tx_dt TIMESTAMP , etl_tx_dt TIMESTAMP
@ -61,6 +62,7 @@ create table if not exists p10_sa.S98_S_partner_summary_visit (
COMMENT ON COLUMN p10_sa.S98_S_partner_summary_visit.like_number IS ''; COMMENT ON COLUMN p10_sa.S98_S_partner_summary_visit.like_number IS '';
COMMENT ON COLUMN p10_sa.S98_S_partner_summary_visit.sub_industry IS ''; COMMENT ON COLUMN p10_sa.S98_S_partner_summary_visit.sub_industry IS '';
COMMENT ON COLUMN p10_sa.S98_S_partner_summary_visit.tsm_names_by_alias IS ''; COMMENT ON COLUMN p10_sa.S98_S_partner_summary_visit.tsm_names_by_alias IS '';
COMMENT ON COLUMN p10_sa.S98_S_partner_summary_visit.update_date IS '';
COMMENT ON COLUMN p10_sa.S98_S_partner_summary_visit.visit_remark IS ''; COMMENT ON COLUMN p10_sa.S98_S_partner_summary_visit.visit_remark IS '';
COMMENT ON COLUMN p10_sa.S98_S_partner_summary_visit.visitor IS ''; COMMENT ON COLUMN p10_sa.S98_S_partner_summary_visit.visitor IS '';
COMMENT ON COLUMN p10_sa.S98_S_partner_summary_visit.etl_tx_dt IS ''; COMMENT ON COLUMN p10_sa.S98_S_partner_summary_visit.etl_tx_dt IS '';
@ -98,6 +100,7 @@ create table if not exists p12_sfull.S98_S_partner_summary_visit (
, like_number TEXT , like_number TEXT
, sub_industry TEXT , sub_industry TEXT
, tsm_names_by_alias TEXT , tsm_names_by_alias TEXT
, update_date TEXT
, visit_remark TEXT , visit_remark TEXT
, visitor TEXT , visitor TEXT
, etl_tx_dt TIMESTAMP , etl_tx_dt TIMESTAMP
@ -131,6 +134,7 @@ create table if not exists p12_sfull.S98_S_partner_summary_visit (
COMMENT ON COLUMN p12_sfull.S98_S_partner_summary_visit.like_number IS ''; COMMENT ON COLUMN p12_sfull.S98_S_partner_summary_visit.like_number IS '';
COMMENT ON COLUMN p12_sfull.S98_S_partner_summary_visit.sub_industry IS ''; COMMENT ON COLUMN p12_sfull.S98_S_partner_summary_visit.sub_industry IS '';
COMMENT ON COLUMN p12_sfull.S98_S_partner_summary_visit.tsm_names_by_alias IS ''; COMMENT ON COLUMN p12_sfull.S98_S_partner_summary_visit.tsm_names_by_alias IS '';
COMMENT ON COLUMN p12_sfull.S98_S_partner_summary_visit.update_date IS '';
COMMENT ON COLUMN p12_sfull.S98_S_partner_summary_visit.visit_remark IS ''; COMMENT ON COLUMN p12_sfull.S98_S_partner_summary_visit.visit_remark IS '';
COMMENT ON COLUMN p12_sfull.S98_S_partner_summary_visit.visitor IS ''; COMMENT ON COLUMN p12_sfull.S98_S_partner_summary_visit.visitor IS '';
COMMENT ON COLUMN p12_sfull.S98_S_partner_summary_visit.etl_tx_dt IS ''; COMMENT ON COLUMN p12_sfull.S98_S_partner_summary_visit.etl_tx_dt IS '';

View File

@ -20,14 +20,6 @@ import psycopg2
ACCESS_KEY = "75c4ab4d-6a67-4aed-8b1d-5bb64fd36afc" ACCESS_KEY = "75c4ab4d-6a67-4aed-8b1d-5bb64fd36afc"
SECRET_KEY = "117347a7dd066a50a4d2973c5f3d5ba9101094c5" SECRET_KEY = "117347a7dd066a50a4d2973c5f3d5ba9101094c5"
# URL 用占位符(外部替换)
BASE_URLS = {
"visits": "http://onesite.tek.cn/api/summary/pos_datas", # 客户拜访数据
"reports": "http://onesite.tek.cn/api/summary/pos_datas", # 报备数据
"pos_datas": "http://onesite.tek.cn/api/summary/pos_datas", # POS数据
"customer_and_contact_datas": "http://onesite.tek.cn/api/summary/pos_datas" # 客户及联系人数据
}
PG_DSN = dict( PG_DSN = dict(
database="dataops_db", database="dataops_db",
user="dbuser_dba", user="dbuser_dba",
@ -80,7 +72,7 @@ class Partner1SiteClient:
"page": page_num "page": page_num
}) })
resp = requests.get(base_url, params=params_with_paging, timeout=120) resp = requests.get(base_url, params=params_with_paging, timeout=30)
resp.raise_for_status() resp.raise_for_status()
data_json = resp.json() data_json = resp.json()
@ -139,10 +131,8 @@ def main():
client = Partner1SiteClient(ACCESS_KEY, SECRET_KEY) client = Partner1SiteClient(ACCESS_KEY, SECRET_KEY)
data = client.fetch_all_pages( data = client.fetch_all_pages(
api_name="pos_datas", api_name="visits",2000-01-01
params={'startPosInsertDate':get_previous_date(7)} params={'startPosUpdateDate':'2000-01-01',} # 拉全量,若有默认时间限制可改成 params={'startPosUpdateDate':get_previous_date(7),}
# params={'startPosInsertDate':'2000-1-1'}
)
save_json_to_pg(data, API_ID) save_json_to_pg(data, API_ID)

View File

@ -19,6 +19,7 @@ insert into data_api.partner_summary_pos (
, contact_phone , contact_phone
, contact_title , contact_title
, customer_chanel_name , customer_chanel_name
, customer_id
, customer_name , customer_name
, distributor_name , distributor_name
, distributor_name2 , distributor_name2
@ -26,12 +27,15 @@ insert into data_api.partner_summary_pos (
, id , id
, industry , industry
, invoice_number , invoice_number
, net_usd
, online_name , online_name
, online_or_offline , online_or_offline
, pos_insert_date , pos_insert_date
, pos_invoice_date , pos_invoice_date
, pos_update_date , pos_update_date
, product_ap , product_ap
, product_family_code
, product_family_name
, product_qty , product_qty
, province , province
, sales_name , sales_name
@ -58,6 +62,7 @@ select
, case when trim(both from contact_phone)='' then null else contact_phone::text end contact_phone , case when trim(both from contact_phone)='' then null else contact_phone::text end contact_phone
, case when trim(both from contact_title)='' then null else contact_title::text end contact_title , case when trim(both from contact_title)='' then null else contact_title::text end contact_title
, case when trim(both from customer_chanel_name)='' then null else customer_chanel_name::text end customer_chanel_name , case when trim(both from customer_chanel_name)='' then null else customer_chanel_name::text end customer_chanel_name
, case when trim(both from customer_id)='' then null else customer_id::text end customer_id
, case when trim(both from customer_name)='' then null else customer_name::text end customer_name , case when trim(both from customer_name)='' then null else customer_name::text end customer_name
, case when trim(both from distributor_name)='' then null else distributor_name::text end distributor_name , case when trim(both from distributor_name)='' then null else distributor_name::text end distributor_name
, case when trim(both from distributor_name2)='' then null else distributor_name2::text end distributor_name2 , case when trim(both from distributor_name2)='' then null else distributor_name2::text end distributor_name2
@ -65,12 +70,15 @@ select
, case when trim(both from id)='' then null else id::text end id , case when trim(both from id)='' then null else id::text end id
, case when trim(both from industry)='' then null else industry::text end industry , case when trim(both from industry)='' then null else industry::text end industry
, case when trim(both from invoice_number)='' then null else invoice_number::text end invoice_number , case when trim(both from invoice_number)='' then null else invoice_number::text end invoice_number
, case when trim(both from net_usd)='' then null else net_usd::text end net_usd
, case when trim(both from online_name)='' then null else online_name::text end online_name , case when trim(both from online_name)='' then null else online_name::text end online_name
, case when trim(both from online_or_offline)='' then null else online_or_offline::text end online_or_offline , case when trim(both from online_or_offline)='' then null else online_or_offline::text end online_or_offline
, case when trim(both from pos_insert_date)='' then null else pos_insert_date::text end pos_insert_date , case when trim(both from pos_insert_date)='' then null else pos_insert_date::text end pos_insert_date
, case when trim(both from pos_invoice_date)='' then null else pos_invoice_date::text end pos_invoice_date , case when trim(both from pos_invoice_date)='' then null else pos_invoice_date::text end pos_invoice_date
, case when trim(both from pos_update_date)='' then null else pos_update_date::text end pos_update_date , case when trim(both from pos_update_date)='' then null else pos_update_date::text end pos_update_date
, case when trim(both from product_ap)='' then null else product_ap::text end product_ap , case when trim(both from product_ap)='' then null else product_ap::text end product_ap
, case when trim(both from product_family_code)='' then null else product_family_code::text end product_family_code
, case when trim(both from product_family_name)='' then null else product_family_name::text end product_family_name
, case when trim(both from product_qty)='' then null else product_qty::text end product_qty , case when trim(both from product_qty)='' then null else product_qty::text end product_qty
, case when trim(both from province)='' then null else province::text end province , case when trim(both from province)='' then null else province::text end province
, case when trim(both from sales_name)='' then null else sales_name::text end sales_name , case when trim(both from sales_name)='' then null else sales_name::text end sales_name
@ -97,6 +105,7 @@ select
, (json_array_elements(data::json)::json->>'contactPhone') contact_phone , (json_array_elements(data::json)::json->>'contactPhone') contact_phone
, (json_array_elements(data::json)::json->>'contactTitle') contact_title , (json_array_elements(data::json)::json->>'contactTitle') contact_title
, (json_array_elements(data::json)::json->>'customerChanelName') customer_chanel_name , (json_array_elements(data::json)::json->>'customerChanelName') customer_chanel_name
, (json_array_elements(data::json)::json->>'customerId') customer_id
, (json_array_elements(data::json)::json->>'customerName') customer_name , (json_array_elements(data::json)::json->>'customerName') customer_name
, (json_array_elements(data::json)::json->>'distributorName') distributor_name , (json_array_elements(data::json)::json->>'distributorName') distributor_name
, (json_array_elements(data::json)::json->>'distributorName2') distributor_name2 , (json_array_elements(data::json)::json->>'distributorName2') distributor_name2
@ -104,12 +113,15 @@ select
, (json_array_elements(data::json)::json->>'id') id , (json_array_elements(data::json)::json->>'id') id
, (json_array_elements(data::json)::json->>'industry') industry , (json_array_elements(data::json)::json->>'industry') industry
, (json_array_elements(data::json)::json->>'invoiceNumber') invoice_number , (json_array_elements(data::json)::json->>'invoiceNumber') invoice_number
, (json_array_elements(data::json)::json->>'netUsd') net_usd
, (json_array_elements(data::json)::json->>'onlineName') online_name , (json_array_elements(data::json)::json->>'onlineName') online_name
, (json_array_elements(data::json)::json->>'onlineOrOffline') online_or_offline , (json_array_elements(data::json)::json->>'onlineOrOffline') online_or_offline
, (json_array_elements(data::json)::json->>'posInsertDate') pos_insert_date , (json_array_elements(data::json)::json->>'posInsertDate') pos_insert_date
, (json_array_elements(data::json)::json->>'posInvoiceDate') pos_invoice_date , (json_array_elements(data::json)::json->>'posInvoiceDate') pos_invoice_date
, (json_array_elements(data::json)::json->>'posUpdateDate') pos_update_date , (json_array_elements(data::json)::json->>'posUpdateDate') pos_update_date
, (json_array_elements(data::json)::json->>'productAp') product_ap , (json_array_elements(data::json)::json->>'productAp') product_ap
, (json_array_elements(data::json)::json->>'productFamilyCode') product_family_code
, (json_array_elements(data::json)::json->>'productFamilyName') product_family_name
, (json_array_elements(data::json)::json->>'productQty') product_qty , (json_array_elements(data::json)::json->>'productQty') product_qty
, (json_array_elements(data::json)::json->>'province') province , (json_array_elements(data::json)::json->>'province') province
, (json_array_elements(data::json)::json->>'salesName') sales_name , (json_array_elements(data::json)::json->>'salesName') sales_name

View File

@ -132,8 +132,7 @@ def main():
data = client.fetch_all_pages( data = client.fetch_all_pages(
api_name="visits", api_name="visits",
# params={'startInsertDate':get_previous_date(7),} # 拉全量,若有默认时间限制可改成 {"startInsertDate":"2000-01-01","endInsertDate":get_previous_date(0)} params={'startUpdateDate':'2025-8-20',} # 拉全量,若有默认时间限制可改成 {"startInsertDate":"2000-01-01","endInsertDate":get_previous_date(0)}
params={'startInsertDate':'2000-01-01',}
) )
save_json_to_pg(data, API_ID) save_json_to_pg(data, API_ID)

View File

@ -34,6 +34,7 @@ insert into data_api.partner_summary_visit (
, like_number , like_number
, sub_industry , sub_industry
, tsm_names_by_alias , tsm_names_by_alias
, update_date
, visit_remark , visit_remark
, visitor , visitor
,etl_tx_dt ,etl_tx_dt
@ -67,6 +68,7 @@ select
, case when trim(both from like_number)='' then null else like_number::text end like_number , case when trim(both from like_number)='' then null else like_number::text end like_number
, case when trim(both from sub_industry)='' then null else sub_industry::text end sub_industry , case when trim(both from sub_industry)='' then null else sub_industry::text end sub_industry
, case when trim(both from tsm_names_by_alias)='' then null else tsm_names_by_alias::text end tsm_names_by_alias , case when trim(both from tsm_names_by_alias)='' then null else tsm_names_by_alias::text end tsm_names_by_alias
, case when trim(both from update_date)='' then null else update_date::text end update_date
, case when trim(both from visit_remark)='' then null else visit_remark::text end visit_remark , case when trim(both from visit_remark)='' then null else visit_remark::text end visit_remark
, case when trim(both from visitor)='' then null else visitor::text end visitor , case when trim(both from visitor)='' then null else visitor::text end visitor
,etl_tx_dt ,etl_tx_dt
@ -100,6 +102,7 @@ select
, (json_array_elements(data::json)::json->>'likeNumber') like_number , (json_array_elements(data::json)::json->>'likeNumber') like_number
, (json_array_elements(data::json)::json->>'subIndustry') sub_industry , (json_array_elements(data::json)::json->>'subIndustry') sub_industry
, (json_array_elements(data::json)::json->>'tsmNamesByAlias') tsm_names_by_alias , (json_array_elements(data::json)::json->>'tsmNamesByAlias') tsm_names_by_alias
, (json_array_elements(data::json)::json->>'updateDate') update_date
, (json_array_elements(data::json)::json->>'visitRemark') visit_remark , (json_array_elements(data::json)::json->>'visitRemark') visit_remark
, (json_array_elements(data::json)::json->>'visitor') visitor , (json_array_elements(data::json)::json->>'visitor') visitor
,CURRENT_TIMESTAMP(0) etl_tx_dt ,CURRENT_TIMESTAMP(0) etl_tx_dt

View File

@ -132,10 +132,8 @@ def main():
data = client.fetch_all_pages( data = client.fetch_all_pages(
api_name="visits", api_name="visits",
#params={'startApplyDate':get_previous_date(7),} # 拉全量,若有默认时间限制可改成 {"startInsertDate":"2000-01-01","endInsertDate":get_previous_date(0)} params={'startUpdateDate':'2000-01-01',} # 拉全量,若有默认时间限制可改成 {"startInsertDate":"2000-01-01","endInsertDate":get_previous_date(0)}
params={'startApplyDate':'2000-01-01',}
) )
save_json_to_pg(data, API_ID) save_json_to_pg(data, API_ID)