clear workflow
This commit is contained in:
root
parent
53cdd36c8d
commit
482b74325d
|
|
@ -1,41 +0,0 @@
|
|||
# coding: utf-8
|
||||
import requests
|
||||
import json
|
||||
import psycopg2
|
||||
import uuid
|
||||
import datetime
|
||||
|
||||
# 获取当前日期
|
||||
current_date = datetime.date.today()
|
||||
previous_date = current_date - datetime.timedelta(days=2)
|
||||
formatted_current_date = current_date.strftime("%Y-%m-%dT%H:%M:%SZ")
|
||||
formatted_previous_date = previous_date.strftime("%Y-%m-%dT%H:%M:%SZ")
|
||||
|
||||
print('开始加载数据：custom_events_update:合并SCRM CONTACT')
|
||||
authUrl='https://api.huiju.cool/v2/oauth2/token?app_id=cl037b184ebccd97c&secret=deaa83fdd8d385eec0d1d04d34282c4ec8c33b60&grant_type=client_credentials'
|
||||
|
||||
print('开始请求令牌。')
|
||||
authRequest=requests.get(authUrl)
|
||||
auth=json.loads(authRequest.text)
|
||||
print('开始请求数据总数。')
|
||||
url='https://api.huiju.cool/v2/customerEvents'
|
||||
header={}
|
||||
body={'access_token':auth['access_token'],'sort':'lastUpdated','lastUpdated[le]':formatted_current_date,'limit':'2000',}
|
||||
dataReqL=requests.get(url,headers=header,params=body)
|
||||
resL=json.loads(dataReqL.text)
|
||||
# print(resL)
|
||||
dataList=resL['data']
|
||||
total=len(dataList)
|
||||
conn = psycopg2.connect(database="dataops_db", user="dbuser_dops", password="MIgTi3jA",
|
||||
host="172.17.0.8", port="5432")
|
||||
print('数据库连接成功')
|
||||
dataId=str(uuid.uuid4())
|
||||
print('临时id：'+dataId)
|
||||
json_object = json.dumps(dataList)
|
||||
cur=conn.cursor()
|
||||
sql="update data_api.api_data set is_loaded = '1' where api_id = 'ff5c7bf6-0d18-4201-9501-8fdd6152';INSERT INTO data_api.api_data (id,api_id,data,total_num,is_loaded,status,request_tm,execute_tm,remark) values (%s,%s,%s,%s,'0','0',current_timestamp(0),current_timestamp(0),'')"
|
||||
cur.execute(sql,[dataId,'ff5c7bf6-0d18-4201-9501-8fdd6152',json_object,total])
|
||||
conn.commit()
|
||||
cur.close()
|
||||
conn.close()
|
||||
print('加载数据结束：custom_events_update:合并SCRM CONTACT')
|
||||
|
|
@ -1,105 +0,0 @@
|
|||
/*******Main Section**************************************************************************/
|
||||
\set ON_ERROR_STOP on
|
||||
\set AUTOCOMMIT on
|
||||
\timing on
|
||||
|
||||
DELETE FROM data_api.customer_events_lastupdated;
|
||||
|
||||
insert into data_api.customer_events_lastupdated (
|
||||
date
|
||||
, screen_width
|
||||
, target_name
|
||||
, ip_city
|
||||
, ip_country
|
||||
, platform
|
||||
, last_updated
|
||||
, page_type
|
||||
, os_version
|
||||
, browser
|
||||
, browser_version
|
||||
, customer_id
|
||||
, id
|
||||
, tag
|
||||
, event
|
||||
, id_str
|
||||
, os
|
||||
, target_id
|
||||
, customer_id_str
|
||||
, ip
|
||||
, screen_height
|
||||
, external_id
|
||||
, ip_county
|
||||
, page_id
|
||||
, url
|
||||
, ip_province
|
||||
, device
|
||||
,etl_tx_dt
|
||||
)
|
||||
select
|
||||
case when trim(both from date)='' then null else date::text end date
|
||||
, case when trim(both from screen_width)='' then null else screen_width::text end screen_width
|
||||
, case when trim(both from target_name)='' then null else target_name::text end target_name
|
||||
, case when trim(both from ip_city)='' then null else ip_city::text end ip_city
|
||||
, case when trim(both from ip_country)='' then null else ip_country::text end ip_country
|
||||
, case when trim(both from platform)='' then null else platform::text end platform
|
||||
, case when trim(both from last_updated)='' then null else last_updated::text end last_updated
|
||||
, case when trim(both from page_type)='' then null else page_type::text end page_type
|
||||
, case when trim(both from os_version)='' then null else os_version::text end os_version
|
||||
, case when trim(both from browser)='' then null else browser::text end browser
|
||||
, case when trim(both from browser_version)='' then null else browser_version::text end browser_version
|
||||
, case when trim(both from customer_id)='' then null else customer_id::text end customer_id
|
||||
, case when trim(both from id)='' then null else id::text end id
|
||||
, case when trim(both from tag)='' then null else tag::text end tag
|
||||
, case when trim(both from event)='' then null else event::text end event
|
||||
, case when trim(both from id_str)='' then null else id_str::text end id_str
|
||||
, case when trim(both from os)='' then null else os::text end os
|
||||
, case when trim(both from target_id)='' then null else target_id::text end target_id
|
||||
, case when trim(both from customer_id_str)='' then null else customer_id_str::text end customer_id_str
|
||||
, case when trim(both from ip)='' then null else ip::text end ip
|
||||
, case when trim(both from screen_height)='' then null else screen_height::text end screen_height
|
||||
, case when trim(both from external_id)='' then null else external_id::text end external_id
|
||||
, case when trim(both from ip_county)='' then null else ip_county::text end ip_county
|
||||
, case when trim(both from page_id)='' then null else page_id::text end page_id
|
||||
, case when trim(both from url)='' then null else url::text end url
|
||||
, case when trim(both from ip_province)='' then null else ip_province::text end ip_province
|
||||
, case when trim(both from device)='' then null else device::text end device
|
||||
,etl_tx_dt
|
||||
from (
|
||||
select
|
||||
(json_array_elements(data::json)::json->>'date') date
|
||||
, (json_array_elements(data::json)::json->>'screenWidth') screen_width
|
||||
, (json_array_elements(data::json)::json->>'targetName') target_name
|
||||
, (json_array_elements(data::json)::json->>'ipCity') ip_city
|
||||
, (json_array_elements(data::json)::json->>'ipCountry') ip_country
|
||||
, (json_array_elements(data::json)::json->>'platform') platform
|
||||
, (json_array_elements(data::json)::json->>'lastUpdated') last_updated
|
||||
, (json_array_elements(data::json)::json->>'pageType') page_type
|
||||
, (json_array_elements(data::json)::json->>'osVersion') os_version
|
||||
, (json_array_elements(data::json)::json->>'browser') browser
|
||||
, (json_array_elements(data::json)::json->>'browserVersion') browser_version
|
||||
, (json_array_elements(data::json)::json->>'customerId') customer_id
|
||||
, (json_array_elements(data::json)::json->>'id') id
|
||||
, (json_array_elements(data::json)::json->>'tag') tag
|
||||
, (json_array_elements(data::json)::json->>'event') event
|
||||
, (json_array_elements(data::json)::json->>'idStr') id_str
|
||||
, (json_array_elements(data::json)::json->>'os') os
|
||||
, (json_array_elements(data::json)::json->>'targetId') target_id
|
||||
, (json_array_elements(data::json)::json->>'customerIdStr') customer_id_str
|
||||
, (json_array_elements(data::json)::json->>'ip') ip
|
||||
, (json_array_elements(data::json)::json->>'screenHeight') screen_height
|
||||
, (json_array_elements(data::json)::json->>'externalId') external_id
|
||||
, (json_array_elements(data::json)::json->>'ipCounty') ip_county
|
||||
, (json_array_elements(data::json)::json->>'pageId') page_id
|
||||
, (json_array_elements(data::json)::json->>'url') url
|
||||
, (json_array_elements(data::json)::json->>'ipProvince') ip_province
|
||||
, (json_array_elements(data::json)::json->>'device') device
|
||||
,CURRENT_TIMESTAMP(0) etl_tx_dt
|
||||
from (select * from data_api.api_data
|
||||
WHERE api_id='ff5c7bf6-0d18-4201-9501-8fdd6152' and is_loaded = '0' order by request_tm desc limit 1) p )p;
|
||||
|
||||
update data_api.api_data
|
||||
set is_loaded = '1' ,
|
||||
status = '1',
|
||||
request_tm = current_timestamp(0)
|
||||
where api_id='ff5c7bf6-0d18-4201-9501-8fdd6152';
|
||||
\q
|
||||
|
|
@ -1,41 +0,0 @@
|
|||
# coding: utf-8
|
||||
import requests
|
||||
import json
|
||||
import psycopg2
|
||||
import uuid
|
||||
import datetime
|
||||
|
||||
# 获取当前日期
|
||||
current_date = datetime.date.today()
|
||||
previous_date = current_date - datetime.timedelta(days=2)
|
||||
formatted_current_date = current_date.strftime("%Y-%m-%dT%H:%M:%SZ")
|
||||
formatted_previous_date = previous_date.strftime("%Y-%m-%dT%H:%M:%SZ")
|
||||
|
||||
print('开始加载数据：update_scrm_contact:更新scrm CONTACT')
|
||||
authUrl='https://api.huiju.cool/v2/oauth2/token?app_id=cl037b184ebccd97c&secret=deaa83fdd8d385eec0d1d04d34282c4ec8c33b60&grant_type=client_credentials'
|
||||
|
||||
print('开始请求令牌。')
|
||||
authRequest=requests.get(authUrl)
|
||||
auth=json.loads(authRequest.text)
|
||||
print('开始请求数据总数。')
|
||||
url='https://api.huiju.cool/v2/customers'
|
||||
header={}
|
||||
body={'access_token':auth['access_token'],'sort':'lastUpdated','lastUpdated[le]':formatted_current_date,'limit':'2000',}
|
||||
dataReqL=requests.get(url,headers=header,params=body)
|
||||
resL=json.loads(dataReqL.text)
|
||||
# print(resL)
|
||||
dataList=resL['data']
|
||||
total=len(dataList)
|
||||
conn = psycopg2.connect(database="dataops_db", user="dbuser_dops", password="MIgTi3jA",
|
||||
host="172.17.0.8", port="5432")
|
||||
print('数据库连接成功')
|
||||
dataId=str(uuid.uuid4())
|
||||
print('临时id：'+dataId)
|
||||
json_object = json.dumps(dataList)
|
||||
cur=conn.cursor()
|
||||
sql="update data_api.api_data set is_loaded = '1' where api_id = '03d918ea-00d1-448a-8b9b-6a0b1a33';INSERT INTO data_api.api_data (id,api_id,data,total_num,is_loaded,status,request_tm,execute_tm,remark) values (%s,%s,%s,%s,'0','0',current_timestamp(0),current_timestamp(0),'')"
|
||||
cur.execute(sql,[dataId,'03d918ea-00d1-448a-8b9b-6a0b1a33',json_object,total])
|
||||
conn.commit()
|
||||
cur.close()
|
||||
conn.close()
|
||||
print('加载数据结束：update_scrm_contact:更新scrm CONTACT')
|
||||
|
|
@ -1,420 +0,0 @@
|
|||
/*******Main Section**************************************************************************/
|
||||
\set ON_ERROR_STOP on
|
||||
\set AUTOCOMMIT on
|
||||
\timing on
|
||||
|
||||
DELETE FROM data_api.scrm_contact_lastUpdated;
|
||||
|
||||
insert into data_api.scrm_contact_lastUpdated (
|
||||
accept_email
|
||||
, acceptsms
|
||||
, annual_revenue
|
||||
, attr_id
|
||||
, attr1
|
||||
, attr10
|
||||
, attr11
|
||||
, attr12
|
||||
, attr13
|
||||
, attr14
|
||||
, attr15
|
||||
, attr16
|
||||
, attr17
|
||||
, attr18
|
||||
, attr19
|
||||
, attr2
|
||||
, attr20
|
||||
, attr21
|
||||
, attr22
|
||||
, attr23
|
||||
, attr24
|
||||
, attr25
|
||||
, attr26
|
||||
, attr27
|
||||
, attr28
|
||||
, attr3
|
||||
, attr4
|
||||
, attr5
|
||||
, attr6
|
||||
, attr7
|
||||
, attr8
|
||||
, attr9
|
||||
, birthday
|
||||
, campaign_id
|
||||
, campaign_name
|
||||
, city
|
||||
, company
|
||||
, content_name
|
||||
, convert_date
|
||||
, country
|
||||
, county
|
||||
, create_from
|
||||
, create_from_name
|
||||
, create_method
|
||||
, date_created
|
||||
, date_join
|
||||
, date_member_reg
|
||||
, display_name
|
||||
, email
|
||||
, email_verified
|
||||
, employee_number
|
||||
, external_id
|
||||
, extra_convert_content_name
|
||||
, extra_convert_date
|
||||
, extra_convert_source
|
||||
, extra_convert_source1
|
||||
, extra_convert_source2
|
||||
, extra_convert_source3
|
||||
, extra_convert_source4
|
||||
, extra_convert_source5
|
||||
, extra_convert_source6
|
||||
, extra_convert_source7
|
||||
, extra_convert_source8
|
||||
, extra_regist_content_name
|
||||
, extra_regist_date
|
||||
, extra_regist_source
|
||||
, extra_regist_source1
|
||||
, extra_regist_source2
|
||||
, extra_regist_source3
|
||||
, extra_regist_source4
|
||||
, extra_regist_source5
|
||||
, extra_regist_source6
|
||||
, extra_regist_source7
|
||||
, extra_regist_source8
|
||||
, first_assign_date
|
||||
, gender
|
||||
, home_address
|
||||
, id
|
||||
, id_str
|
||||
, img
|
||||
, industry
|
||||
, is_anonymous
|
||||
, is_employee
|
||||
, is_member
|
||||
, is_referrer
|
||||
, language
|
||||
, last_updated
|
||||
, latest_engage_date
|
||||
, mobile
|
||||
, mobile_verified
|
||||
, name
|
||||
, nick_name
|
||||
, not_accept_email
|
||||
, not_acceptsms
|
||||
, office_address
|
||||
, owner_date
|
||||
, owner_employee_id
|
||||
, owner_employee_name
|
||||
, owner_id
|
||||
, owner_name
|
||||
, province
|
||||
, qq
|
||||
, referrer
|
||||
, referrer_name
|
||||
, refund_date
|
||||
, region
|
||||
, remark
|
||||
, source
|
||||
, source1
|
||||
, source2
|
||||
, source3
|
||||
, source4
|
||||
, source5
|
||||
, source6
|
||||
, source7
|
||||
, source8
|
||||
, stage
|
||||
, status
|
||||
, street
|
||||
, subscribe_scene
|
||||
, telephone
|
||||
, title
|
||||
, type
|
||||
, update_method
|
||||
, utm_medium
|
||||
, utm_term
|
||||
, version
|
||||
, website
|
||||
, wechat
|
||||
, wechat_city
|
||||
, wechat_nick_name
|
||||
, weibo
|
||||
,etl_tx_dt
|
||||
)
|
||||
select
|
||||
case when trim(both from accept_email)='' then null else accept_email::text end accept_email
|
||||
, case when trim(both from acceptsms)='' then null else acceptsms::text end acceptsms
|
||||
, case when trim(both from annual_revenue)='' then null else annual_revenue::text end annual_revenue
|
||||
, case when trim(both from attr_id)='' then null else attr_id::text end attr_id
|
||||
, case when trim(both from attr1)='' then null else attr1::text end attr1
|
||||
, case when trim(both from attr10)='' then null else attr10::text end attr10
|
||||
, case when trim(both from attr11)='' then null else attr11::text end attr11
|
||||
, case when trim(both from attr12)='' then null else attr12::text end attr12
|
||||
, case when trim(both from attr13)='' then null else attr13::text end attr13
|
||||
, case when trim(both from attr14)='' then null else attr14::text end attr14
|
||||
, case when trim(both from attr15)='' then null else attr15::text end attr15
|
||||
, case when trim(both from attr16)='' then null else attr16::text end attr16
|
||||
, case when trim(both from attr17)='' then null else attr17::text end attr17
|
||||
, case when trim(both from attr18)='' then null else attr18::text end attr18
|
||||
, case when trim(both from attr19)='' then null else attr19::text end attr19
|
||||
, case when trim(both from attr2)='' then null else attr2::text end attr2
|
||||
, case when trim(both from attr20)='' then null else attr20::text end attr20
|
||||
, case when trim(both from attr21)='' then null else attr21::text end attr21
|
||||
, case when trim(both from attr22)='' then null else attr22::text end attr22
|
||||
, case when trim(both from attr23)='' then null else attr23::text end attr23
|
||||
, case when trim(both from attr24)='' then null else attr24::text end attr24
|
||||
, case when trim(both from attr25)='' then null else attr25::text end attr25
|
||||
, case when trim(both from attr26)='' then null else attr26::text end attr26
|
||||
, case when trim(both from attr27)='' then null else attr27::text end attr27
|
||||
, case when trim(both from attr28)='' then null else attr28::text end attr28
|
||||
, case when trim(both from attr3)='' then null else attr3::text end attr3
|
||||
, case when trim(both from attr4)='' then null else attr4::text end attr4
|
||||
, case when trim(both from attr5)='' then null else attr5::text end attr5
|
||||
, case when trim(both from attr6)='' then null else attr6::text end attr6
|
||||
, case when trim(both from attr7)='' then null else attr7::text end attr7
|
||||
, case when trim(both from attr8)='' then null else attr8::text end attr8
|
||||
, case when trim(both from attr9)='' then null else attr9::text end attr9
|
||||
, case when trim(both from birthday)='' then null else birthday::text end birthday
|
||||
, case when trim(both from campaign_id)='' then null else campaign_id::text end campaign_id
|
||||
, case when trim(both from campaign_name)='' then null else campaign_name::text end campaign_name
|
||||
, case when trim(both from city)='' then null else city::text end city
|
||||
, case when trim(both from company)='' then null else company::text end company
|
||||
, case when trim(both from content_name)='' then null else content_name::text end content_name
|
||||
, case when trim(both from convert_date)='' then null else convert_date::text end convert_date
|
||||
, case when trim(both from country)='' then null else country::text end country
|
||||
, case when trim(both from county)='' then null else county::text end county
|
||||
, case when trim(both from create_from)='' then null else create_from::text end create_from
|
||||
, case when trim(both from create_from_name)='' then null else create_from_name::text end create_from_name
|
||||
, case when trim(both from create_method)='' then null else create_method::text end create_method
|
||||
, case when trim(both from date_created)='' then null else date_created::text end date_created
|
||||
, case when trim(both from date_join)='' then null else date_join::text end date_join
|
||||
, case when trim(both from date_member_reg)='' then null else date_member_reg::text end date_member_reg
|
||||
, case when trim(both from display_name)='' then null else display_name::text end display_name
|
||||
, case when trim(both from email)='' then null else email::text end email
|
||||
, case when trim(both from email_verified)='' then null else email_verified::text end email_verified
|
||||
, case when trim(both from employee_number)='' then null else employee_number::text end employee_number
|
||||
, case when trim(both from external_id)='' then null else external_id::text end external_id
|
||||
, case when trim(both from extra_convert_content_name)='' then null else extra_convert_content_name::text end extra_convert_content_name
|
||||
, case when trim(both from extra_convert_date)='' then null else extra_convert_date::text end extra_convert_date
|
||||
, case when trim(both from extra_convert_source)='' then null else extra_convert_source::text end extra_convert_source
|
||||
, case when trim(both from extra_convert_source1)='' then null else extra_convert_source1::text end extra_convert_source1
|
||||
, case when trim(both from extra_convert_source2)='' then null else extra_convert_source2::text end extra_convert_source2
|
||||
, case when trim(both from extra_convert_source3)='' then null else extra_convert_source3::text end extra_convert_source3
|
||||
, case when trim(both from extra_convert_source4)='' then null else extra_convert_source4::text end extra_convert_source4
|
||||
, case when trim(both from extra_convert_source5)='' then null else extra_convert_source5::text end extra_convert_source5
|
||||
, case when trim(both from extra_convert_source6)='' then null else extra_convert_source6::text end extra_convert_source6
|
||||
, case when trim(both from extra_convert_source7)='' then null else extra_convert_source7::text end extra_convert_source7
|
||||
, case when trim(both from extra_convert_source8)='' then null else extra_convert_source8::text end extra_convert_source8
|
||||
, case when trim(both from extra_regist_content_name)='' then null else extra_regist_content_name::text end extra_regist_content_name
|
||||
, case when trim(both from extra_regist_date)='' then null else extra_regist_date::text end extra_regist_date
|
||||
, case when trim(both from extra_regist_source)='' then null else extra_regist_source::text end extra_regist_source
|
||||
, case when trim(both from extra_regist_source1)='' then null else extra_regist_source1::text end extra_regist_source1
|
||||
, case when trim(both from extra_regist_source2)='' then null else extra_regist_source2::text end extra_regist_source2
|
||||
, case when trim(both from extra_regist_source3)='' then null else extra_regist_source3::text end extra_regist_source3
|
||||
, case when trim(both from extra_regist_source4)='' then null else extra_regist_source4::text end extra_regist_source4
|
||||
, case when trim(both from extra_regist_source5)='' then null else extra_regist_source5::text end extra_regist_source5
|
||||
, case when trim(both from extra_regist_source6)='' then null else extra_regist_source6::text end extra_regist_source6
|
||||
, case when trim(both from extra_regist_source7)='' then null else extra_regist_source7::text end extra_regist_source7
|
||||
, case when trim(both from extra_regist_source8)='' then null else extra_regist_source8::text end extra_regist_source8
|
||||
, case when trim(both from first_assign_date)='' then null else first_assign_date::text end first_assign_date
|
||||
, case when trim(both from gender)='' then null else gender::text end gender
|
||||
, case when trim(both from home_address)='' then null else home_address::text end home_address
|
||||
, case when trim(both from id)='' then null else id::text end id
|
||||
, case when trim(both from id_str)='' then null else id_str::text end id_str
|
||||
, case when trim(both from img)='' then null else img::text end img
|
||||
, case when trim(both from industry)='' then null else industry::text end industry
|
||||
, case when trim(both from is_anonymous)='' then null else is_anonymous::text end is_anonymous
|
||||
, case when trim(both from is_employee)='' then null else is_employee::text end is_employee
|
||||
, case when trim(both from is_member)='' then null else is_member::text end is_member
|
||||
, case when trim(both from is_referrer)='' then null else is_referrer::text end is_referrer
|
||||
, case when trim(both from language)='' then null else language::text end language
|
||||
, case when trim(both from last_updated)='' then null else last_updated::text end last_updated
|
||||
, case when trim(both from latest_engage_date)='' then null else latest_engage_date::text end latest_engage_date
|
||||
, case when trim(both from mobile)='' then null else mobile::text end mobile
|
||||
, case when trim(both from mobile_verified)='' then null else mobile_verified::text end mobile_verified
|
||||
, case when trim(both from name)='' then null else name::text end name
|
||||
, case when trim(both from nick_name)='' then null else nick_name::text end nick_name
|
||||
, case when trim(both from not_accept_email)='' then null else not_accept_email::text end not_accept_email
|
||||
, case when trim(both from not_acceptsms)='' then null else not_acceptsms::text end not_acceptsms
|
||||
, case when trim(both from office_address)='' then null else office_address::text end office_address
|
||||
, case when trim(both from owner_date)='' then null else owner_date::text end owner_date
|
||||
, case when trim(both from owner_employee_id)='' then null else owner_employee_id::text end owner_employee_id
|
||||
, case when trim(both from owner_employee_name)='' then null else owner_employee_name::text end owner_employee_name
|
||||
, case when trim(both from owner_id)='' then null else owner_id::text end owner_id
|
||||
, case when trim(both from owner_name)='' then null else owner_name::text end owner_name
|
||||
, case when trim(both from province)='' then null else province::text end province
|
||||
, case when trim(both from qq)='' then null else qq::text end qq
|
||||
, case when trim(both from referrer)='' then null else referrer::text end referrer
|
||||
, case when trim(both from referrer_name)='' then null else referrer_name::text end referrer_name
|
||||
, case when trim(both from refund_date)='' then null else refund_date::text end refund_date
|
||||
, case when trim(both from region)='' then null else region::text end region
|
||||
, case when trim(both from remark)='' then null else remark::text end remark
|
||||
, case when trim(both from source)='' then null else source::text end source
|
||||
, case when trim(both from source1)='' then null else source1::text end source1
|
||||
, case when trim(both from source2)='' then null else source2::text end source2
|
||||
, case when trim(both from source3)='' then null else source3::text end source3
|
||||
, case when trim(both from source4)='' then null else source4::text end source4
|
||||
, case when trim(both from source5)='' then null else source5::text end source5
|
||||
, case when trim(both from source6)='' then null else source6::text end source6
|
||||
, case when trim(both from source7)='' then null else source7::text end source7
|
||||
, case when trim(both from source8)='' then null else source8::text end source8
|
||||
, case when trim(both from stage)='' then null else stage::text end stage
|
||||
, case when trim(both from status)='' then null else status::text end status
|
||||
, case when trim(both from street)='' then null else street::text end street
|
||||
, case when trim(both from subscribe_scene)='' then null else subscribe_scene::text end subscribe_scene
|
||||
, case when trim(both from telephone)='' then null else telephone::text end telephone
|
||||
, case when trim(both from title)='' then null else title::text end title
|
||||
, case when trim(both from type)='' then null else type::text end type
|
||||
, case when trim(both from update_method)='' then null else update_method::text end update_method
|
||||
, case when trim(both from utm_medium)='' then null else utm_medium::text end utm_medium
|
||||
, case when trim(both from utm_term)='' then null else utm_term::text end utm_term
|
||||
, case when trim(both from version)='' then null else version::text end version
|
||||
, case when trim(both from website)='' then null else website::text end website
|
||||
, case when trim(both from wechat)='' then null else wechat::text end wechat
|
||||
, case when trim(both from wechat_city)='' then null else wechat_city::text end wechat_city
|
||||
, case when trim(both from wechat_nick_name)='' then null else wechat_nick_name::text end wechat_nick_name
|
||||
, case when trim(both from weibo)='' then null else weibo::text end weibo
|
||||
,etl_tx_dt
|
||||
from (
|
||||
select
|
||||
(json_array_elements(data::json)::json->>'acceptEmail') accept_email
|
||||
, (json_array_elements(data::json)::json->>'acceptsms') acceptsms
|
||||
, (json_array_elements(data::json)::json->>'annualRevenue') annual_revenue
|
||||
, (json_array_elements(data::json)::json->>'attrId') attr_id
|
||||
, (json_array_elements(data::json)::json->>'attr1') attr1
|
||||
, (json_array_elements(data::json)::json->>'attr10') attr10
|
||||
, (json_array_elements(data::json)::json->>'attr11') attr11
|
||||
, (json_array_elements(data::json)::json->>'attr12') attr12
|
||||
, (json_array_elements(data::json)::json->>'attr13') attr13
|
||||
, (json_array_elements(data::json)::json->>'attr14') attr14
|
||||
, (json_array_elements(data::json)::json->>'attr15') attr15
|
||||
, (json_array_elements(data::json)::json->>'attr16') attr16
|
||||
, (json_array_elements(data::json)::json->>'attr17') attr17
|
||||
, (json_array_elements(data::json)::json->>'attr18') attr18
|
||||
, (json_array_elements(data::json)::json->>'attr19') attr19
|
||||
, (json_array_elements(data::json)::json->>'attr2') attr2
|
||||
, (json_array_elements(data::json)::json->>'attr20') attr20
|
||||
, (json_array_elements(data::json)::json->>'attr21') attr21
|
||||
, (json_array_elements(data::json)::json->>'attr22') attr22
|
||||
, (json_array_elements(data::json)::json->>'attr23') attr23
|
||||
, (json_array_elements(data::json)::json->>'attr24') attr24
|
||||
, (json_array_elements(data::json)::json->>'attr25') attr25
|
||||
, (json_array_elements(data::json)::json->>'attr26') attr26
|
||||
, (json_array_elements(data::json)::json->>'attr27') attr27
|
||||
, (json_array_elements(data::json)::json->>'attr28') attr28
|
||||
, (json_array_elements(data::json)::json->>'attr3') attr3
|
||||
, (json_array_elements(data::json)::json->>'attr4') attr4
|
||||
, (json_array_elements(data::json)::json->>'attr5') attr5
|
||||
, (json_array_elements(data::json)::json->>'attr6') attr6
|
||||
, (json_array_elements(data::json)::json->>'attr7') attr7
|
||||
, (json_array_elements(data::json)::json->>'attr8') attr8
|
||||
, (json_array_elements(data::json)::json->>'attr9') attr9
|
||||
, (json_array_elements(data::json)::json->>'birthday') birthday
|
||||
, (json_array_elements(data::json)::json->>'campaignId') campaign_id
|
||||
, (json_array_elements(data::json)::json->>'campaignName') campaign_name
|
||||
, (json_array_elements(data::json)::json->>'city') city
|
||||
, (json_array_elements(data::json)::json->>'company') company
|
||||
, (json_array_elements(data::json)::json->>'contentName') content_name
|
||||
, (json_array_elements(data::json)::json->>'convertDate') convert_date
|
||||
, (json_array_elements(data::json)::json->>'country') country
|
||||
, (json_array_elements(data::json)::json->>'county') county
|
||||
, (json_array_elements(data::json)::json->>'createFrom') create_from
|
||||
, (json_array_elements(data::json)::json->>'createFromName') create_from_name
|
||||
, (json_array_elements(data::json)::json->>'createMethod') create_method
|
||||
, (json_array_elements(data::json)::json->>'dateCreated') date_created
|
||||
, (json_array_elements(data::json)::json->>'dateJoin') date_join
|
||||
, (json_array_elements(data::json)::json->>'dateMemberReg') date_member_reg
|
||||
, (json_array_elements(data::json)::json->>'displayName') display_name
|
||||
, (json_array_elements(data::json)::json->>'email') email
|
||||
, (json_array_elements(data::json)::json->>'emailVerified') email_verified
|
||||
, (json_array_elements(data::json)::json->>'employeeNumber') employee_number
|
||||
, (json_array_elements(data::json)::json->>'externalId') external_id
|
||||
, (json_array_elements(data::json)::json->>'extraConvertContentName') extra_convert_content_name
|
||||
, (json_array_elements(data::json)::json->>'extraConvertDate') extra_convert_date
|
||||
, (json_array_elements(data::json)::json->>'extraConvertSource') extra_convert_source
|
||||
, (json_array_elements(data::json)::json->>'extraConvertSource1') extra_convert_source1
|
||||
, (json_array_elements(data::json)::json->>'extraConvertSource2') extra_convert_source2
|
||||
, (json_array_elements(data::json)::json->>'extraConvertSource3') extra_convert_source3
|
||||
, (json_array_elements(data::json)::json->>'extraConvertSource4') extra_convert_source4
|
||||
, (json_array_elements(data::json)::json->>'extraConvertSource5') extra_convert_source5
|
||||
, (json_array_elements(data::json)::json->>'extraConvertSource6') extra_convert_source6
|
||||
, (json_array_elements(data::json)::json->>'extraConvertSource7') extra_convert_source7
|
||||
, (json_array_elements(data::json)::json->>'extraConvertSource8') extra_convert_source8
|
||||
, (json_array_elements(data::json)::json->>'extraRegistContentName') extra_regist_content_name
|
||||
, (json_array_elements(data::json)::json->>'extraRegistDate') extra_regist_date
|
||||
, (json_array_elements(data::json)::json->>'extraRegistSource') extra_regist_source
|
||||
, (json_array_elements(data::json)::json->>'extraRegistSource1') extra_regist_source1
|
||||
, (json_array_elements(data::json)::json->>'extraRegistSource2') extra_regist_source2
|
||||
, (json_array_elements(data::json)::json->>'extraRegistSource3') extra_regist_source3
|
||||
, (json_array_elements(data::json)::json->>'extraRegistSource4') extra_regist_source4
|
||||
, (json_array_elements(data::json)::json->>'extraRegistSource5') extra_regist_source5
|
||||
, (json_array_elements(data::json)::json->>'extraRegistSource6') extra_regist_source6
|
||||
, (json_array_elements(data::json)::json->>'extraRegistSource7') extra_regist_source7
|
||||
, (json_array_elements(data::json)::json->>'extraRegistSource8') extra_regist_source8
|
||||
, (json_array_elements(data::json)::json->>'firstAssignDate') first_assign_date
|
||||
, (json_array_elements(data::json)::json->>'gender') gender
|
||||
, (json_array_elements(data::json)::json->>'homeAddress') home_address
|
||||
, (json_array_elements(data::json)::json->>'id') id
|
||||
, (json_array_elements(data::json)::json->>'idStr') id_str
|
||||
, (json_array_elements(data::json)::json->>'img') img
|
||||
, (json_array_elements(data::json)::json->>'industry') industry
|
||||
, (json_array_elements(data::json)::json->>'isAnonymous') is_anonymous
|
||||
, (json_array_elements(data::json)::json->>'isEmployee') is_employee
|
||||
, (json_array_elements(data::json)::json->>'isMember') is_member
|
||||
, (json_array_elements(data::json)::json->>'isReferrer') is_referrer
|
||||
, (json_array_elements(data::json)::json->>'language') language
|
||||
, (json_array_elements(data::json)::json->>'lastUpdated') last_updated
|
||||
, (json_array_elements(data::json)::json->>'latestEngageDate') latest_engage_date
|
||||
, (json_array_elements(data::json)::json->>'mobile') mobile
|
||||
, (json_array_elements(data::json)::json->>'mobileVerified') mobile_verified
|
||||
, (json_array_elements(data::json)::json->>'name') name
|
||||
, (json_array_elements(data::json)::json->>'nickName') nick_name
|
||||
, (json_array_elements(data::json)::json->>'notAcceptEmail') not_accept_email
|
||||
, (json_array_elements(data::json)::json->>'notAcceptsms') not_acceptsms
|
||||
, (json_array_elements(data::json)::json->>'officeAddress') office_address
|
||||
, (json_array_elements(data::json)::json->>'ownerDate') owner_date
|
||||
, (json_array_elements(data::json)::json->>'ownerEmployeeId') owner_employee_id
|
||||
, (json_array_elements(data::json)::json->>'ownerEmployeeName') owner_employee_name
|
||||
, (json_array_elements(data::json)::json->>'ownerId') owner_id
|
||||
, (json_array_elements(data::json)::json->>'ownerName') owner_name
|
||||
, (json_array_elements(data::json)::json->>'province') province
|
||||
, (json_array_elements(data::json)::json->>'qq') qq
|
||||
, (json_array_elements(data::json)::json->>'referrer') referrer
|
||||
, (json_array_elements(data::json)::json->>'referrerName') referrer_name
|
||||
, (json_array_elements(data::json)::json->>'refundDate') refund_date
|
||||
, (json_array_elements(data::json)::json->>'region') region
|
||||
, (json_array_elements(data::json)::json->>'remark') remark
|
||||
, (json_array_elements(data::json)::json->>'source') source
|
||||
, (json_array_elements(data::json)::json->>'source1') source1
|
||||
, (json_array_elements(data::json)::json->>'source2') source2
|
||||
, (json_array_elements(data::json)::json->>'source3') source3
|
||||
, (json_array_elements(data::json)::json->>'source4') source4
|
||||
, (json_array_elements(data::json)::json->>'source5') source5
|
||||
, (json_array_elements(data::json)::json->>'source6') source6
|
||||
, (json_array_elements(data::json)::json->>'source7') source7
|
||||
, (json_array_elements(data::json)::json->>'source8') source8
|
||||
, (json_array_elements(data::json)::json->>'stage') stage
|
||||
, (json_array_elements(data::json)::json->>'status') status
|
||||
, (json_array_elements(data::json)::json->>'street') street
|
||||
, (json_array_elements(data::json)::json->>'subscribeScene') subscribe_scene
|
||||
, (json_array_elements(data::json)::json->>'telephone') telephone
|
||||
, (json_array_elements(data::json)::json->>'title') title
|
||||
, (json_array_elements(data::json)::json->>'type') type
|
||||
, (json_array_elements(data::json)::json->>'updateMethod') update_method
|
||||
, (json_array_elements(data::json)::json->>'utmMedium') utm_medium
|
||||
, (json_array_elements(data::json)::json->>'utmTerm') utm_term
|
||||
, (json_array_elements(data::json)::json->>'version') version
|
||||
, (json_array_elements(data::json)::json->>'website') website
|
||||
, (json_array_elements(data::json)::json->>'wechat') wechat
|
||||
, (json_array_elements(data::json)::json->>'wechatCity') wechat_city
|
||||
, (json_array_elements(data::json)::json->>'wechatNickName') wechat_nick_name
|
||||
, (json_array_elements(data::json)::json->>'weibo') weibo
|
||||
,CURRENT_TIMESTAMP(0) etl_tx_dt
|
||||
from (select * from data_api.api_data
|
||||
WHERE api_id='03d918ea-00d1-448a-8b9b-6a0b1a33' and is_loaded = '0' order by request_tm desc limit 1) p )p;
|
||||
|
||||
update data_api.api_data
|
||||
set is_loaded = '1' ,
|
||||
status = '1',
|
||||
request_tm = current_timestamp(0)
|
||||
where api_id='03d918ea-00d1-448a-8b9b-6a0b1a33';
|
||||
\q
|
||||
|
|
@ -1,41 +0,0 @@
|
|||
# coding: utf-8
|
||||
import requests
|
||||
import json
|
||||
import psycopg2
|
||||
import uuid
|
||||
import datetime
|
||||
|
||||
# 获取当前日期
|
||||
current_date = datetime.date.today()
|
||||
previous_date = current_date - datetime.timedelta(days=2)
|
||||
formatted_current_date = current_date.strftime("%Y-%m-%dT%H:%M:%SZ")
|
||||
formatted_previous_date = previous_date.strftime("%Y-%m-%dT%H:%M:%SZ")
|
||||
|
||||
print('开始加载数据：scrm_contact:获取SCRM contact')
|
||||
authUrl='https://api.huiju.cool/v2/oauth2/token?app_id=cl037b184ebccd97c&secret=deaa83fdd8d385eec0d1d04d34282c4ec8c33b60&grant_type=client_credentials'
|
||||
|
||||
print('开始请求令牌。')
|
||||
authRequest=requests.get(authUrl)
|
||||
auth=json.loads(authRequest.text)
|
||||
print('开始请求数据总数。')
|
||||
url='https://api.huiju.cool/v2/customers'
|
||||
header={}
|
||||
body={'access_token':auth['access_token'],'sort':'dateJoin','limit':'2000','dateJoin[le]':formatted_current_date,}
|
||||
dataReqL=requests.get(url,headers=header,params=body)
|
||||
resL=json.loads(dataReqL.text)
|
||||
# print(resL)
|
||||
dataList=resL['data']
|
||||
total=len(dataList)
|
||||
conn = psycopg2.connect(database="dataops_db", user="dbuser_dops", password="MIgTi3jA",
|
||||
host="172.17.0.8", port="5432")
|
||||
print('数据库连接成功')
|
||||
dataId=str(uuid.uuid4())
|
||||
print('临时id：'+dataId)
|
||||
json_object = json.dumps(dataList)
|
||||
cur=conn.cursor()
|
||||
sql="update data_api.api_data set is_loaded = '1' where api_id = 'cab1a761-b8e5-4db4-a769-88db3ec1';INSERT INTO data_api.api_data (id,api_id,data,total_num,is_loaded,status,request_tm,execute_tm,remark) values (%s,%s,%s,%s,'0','0',current_timestamp(0),current_timestamp(0),'')"
|
||||
cur.execute(sql,[dataId,'cab1a761-b8e5-4db4-a769-88db3ec1',json_object,total])
|
||||
conn.commit()
|
||||
cur.close()
|
||||
conn.close()
|
||||
print('加载数据结束：scrm_contact:获取SCRM contact')
|
||||
|
|
@ -1,420 +0,0 @@
|
|||
/*******Main Section**************************************************************************/
|
||||
\set ON_ERROR_STOP on
|
||||
\set AUTOCOMMIT on
|
||||
\timing on
|
||||
|
||||
DELETE FROM data_api.scrm_contact;
|
||||
|
||||
insert into data_api.scrm_contact (
|
||||
accept_email
|
||||
, acceptsms
|
||||
, annual_revenue
|
||||
, attr_id
|
||||
, attr1
|
||||
, attr10
|
||||
, attr11
|
||||
, attr12
|
||||
, attr13
|
||||
, attr14
|
||||
, attr15
|
||||
, attr16
|
||||
, attr17
|
||||
, attr18
|
||||
, attr19
|
||||
, attr2
|
||||
, attr20
|
||||
, attr21
|
||||
, attr22
|
||||
, attr23
|
||||
, attr24
|
||||
, attr25
|
||||
, attr26
|
||||
, attr27
|
||||
, attr28
|
||||
, attr3
|
||||
, attr4
|
||||
, attr5
|
||||
, attr6
|
||||
, attr7
|
||||
, attr8
|
||||
, attr9
|
||||
, birthday
|
||||
, campaign_id
|
||||
, campaign_name
|
||||
, city
|
||||
, company
|
||||
, content_name
|
||||
, convert_date
|
||||
, country
|
||||
, county
|
||||
, create_from
|
||||
, create_from_name
|
||||
, create_method
|
||||
, date_created
|
||||
, date_join
|
||||
, date_member_reg
|
||||
, display_name
|
||||
, email
|
||||
, email_verified
|
||||
, employee_number
|
||||
, external_id
|
||||
, extra_convert_content_name
|
||||
, extra_convert_date
|
||||
, extra_convert_source
|
||||
, extra_convert_source1
|
||||
, extra_convert_source2
|
||||
, extra_convert_source3
|
||||
, extra_convert_source4
|
||||
, extra_convert_source5
|
||||
, extra_convert_source6
|
||||
, extra_convert_source7
|
||||
, extra_convert_source8
|
||||
, extra_regist_content_name
|
||||
, extra_regist_date
|
||||
, extra_regist_source
|
||||
, extra_regist_source1
|
||||
, extra_regist_source2
|
||||
, extra_regist_source3
|
||||
, extra_regist_source4
|
||||
, extra_regist_source5
|
||||
, extra_regist_source6
|
||||
, extra_regist_source7
|
||||
, extra_regist_source8
|
||||
, first_assign_date
|
||||
, gender
|
||||
, home_address
|
||||
, id
|
||||
, id_str
|
||||
, img
|
||||
, industry
|
||||
, is_anonymous
|
||||
, is_employee
|
||||
, is_member
|
||||
, is_referrer
|
||||
, language
|
||||
, last_updated
|
||||
, latest_engage_date
|
||||
, mobile
|
||||
, mobile_verified
|
||||
, name
|
||||
, nick_name
|
||||
, not_accept_email
|
||||
, not_acceptsms
|
||||
, office_address
|
||||
, owner_date
|
||||
, owner_employee_id
|
||||
, owner_employee_name
|
||||
, owner_id
|
||||
, owner_name
|
||||
, province
|
||||
, qq
|
||||
, referrer
|
||||
, referrer_name
|
||||
, refund_date
|
||||
, region
|
||||
, remark
|
||||
, source
|
||||
, source1
|
||||
, source2
|
||||
, source3
|
||||
, source4
|
||||
, source5
|
||||
, source6
|
||||
, source7
|
||||
, source8
|
||||
, stage
|
||||
, status
|
||||
, street
|
||||
, subscribe_scene
|
||||
, telephone
|
||||
, title
|
||||
, type
|
||||
, update_method
|
||||
, utm_medium
|
||||
, utm_term
|
||||
, version
|
||||
, website
|
||||
, wechat
|
||||
, wechat_city
|
||||
, wechat_nick_name
|
||||
, weibo
|
||||
,etl_tx_dt
|
||||
)
|
||||
select
|
||||
case when trim(both from accept_email)='' then null else accept_email::text end accept_email
|
||||
, case when trim(both from acceptsms)='' then null else acceptsms::text end acceptsms
|
||||
, case when trim(both from annual_revenue)='' then null else annual_revenue::text end annual_revenue
|
||||
, case when trim(both from attr_id)='' then null else attr_id::text end attr_id
|
||||
, case when trim(both from attr1)='' then null else attr1::text end attr1
|
||||
, case when trim(both from attr10)='' then null else attr10::text end attr10
|
||||
, case when trim(both from attr11)='' then null else attr11::text end attr11
|
||||
, case when trim(both from attr12)='' then null else attr12::text end attr12
|
||||
, case when trim(both from attr13)='' then null else attr13::text end attr13
|
||||
, case when trim(both from attr14)='' then null else attr14::text end attr14
|
||||
, case when trim(both from attr15)='' then null else attr15::text end attr15
|
||||
, case when trim(both from attr16)='' then null else attr16::text end attr16
|
||||
, case when trim(both from attr17)='' then null else attr17::text end attr17
|
||||
, case when trim(both from attr18)='' then null else attr18::text end attr18
|
||||
, case when trim(both from attr19)='' then null else attr19::text end attr19
|
||||
, case when trim(both from attr2)='' then null else attr2::text end attr2
|
||||
, case when trim(both from attr20)='' then null else attr20::text end attr20
|
||||
, case when trim(both from attr21)='' then null else attr21::text end attr21
|
||||
, case when trim(both from attr22)='' then null else attr22::text end attr22
|
||||
, case when trim(both from attr23)='' then null else attr23::text end attr23
|
||||
, case when trim(both from attr24)='' then null else attr24::text end attr24
|
||||
, case when trim(both from attr25)='' then null else attr25::text end attr25
|
||||
, case when trim(both from attr26)='' then null else attr26::text end attr26
|
||||
, case when trim(both from attr27)='' then null else attr27::text end attr27
|
||||
, case when trim(both from attr28)='' then null else attr28::text end attr28
|
||||
, case when trim(both from attr3)='' then null else attr3::text end attr3
|
||||
, case when trim(both from attr4)='' then null else attr4::text end attr4
|
||||
, case when trim(both from attr5)='' then null else attr5::text end attr5
|
||||
, case when trim(both from attr6)='' then null else attr6::text end attr6
|
||||
, case when trim(both from attr7)='' then null else attr7::text end attr7
|
||||
, case when trim(both from attr8)='' then null else attr8::text end attr8
|
||||
, case when trim(both from attr9)='' then null else attr9::text end attr9
|
||||
, case when trim(both from birthday)='' then null else birthday::text end birthday
|
||||
, case when trim(both from campaign_id)='' then null else campaign_id::text end campaign_id
|
||||
, case when trim(both from campaign_name)='' then null else campaign_name::text end campaign_name
|
||||
, case when trim(both from city)='' then null else city::text end city
|
||||
, case when trim(both from company)='' then null else company::text end company
|
||||
, case when trim(both from content_name)='' then null else content_name::text end content_name
|
||||
, case when trim(both from convert_date)='' then null else convert_date::text end convert_date
|
||||
, case when trim(both from country)='' then null else country::text end country
|
||||
, case when trim(both from county)='' then null else county::text end county
|
||||
, case when trim(both from create_from)='' then null else create_from::text end create_from
|
||||
, case when trim(both from create_from_name)='' then null else create_from_name::text end create_from_name
|
||||
, case when trim(both from create_method)='' then null else create_method::text end create_method
|
||||
, case when trim(both from date_created)='' then null else date_created::text end date_created
|
||||
, case when trim(both from date_join)='' then null else date_join::text end date_join
|
||||
, case when trim(both from date_member_reg)='' then null else date_member_reg::text end date_member_reg
|
||||
, case when trim(both from display_name)='' then null else display_name::text end display_name
|
||||
, case when trim(both from email)='' then null else email::text end email
|
||||
, case when trim(both from email_verified)='' then null else email_verified::text end email_verified
|
||||
, case when trim(both from employee_number)='' then null else employee_number::text end employee_number
|
||||
, case when trim(both from external_id)='' then null else external_id::text end external_id
|
||||
, case when trim(both from extra_convert_content_name)='' then null else extra_convert_content_name::text end extra_convert_content_name
|
||||
, case when trim(both from extra_convert_date)='' then null else extra_convert_date::text end extra_convert_date
|
||||
, case when trim(both from extra_convert_source)='' then null else extra_convert_source::text end extra_convert_source
|
||||
, case when trim(both from extra_convert_source1)='' then null else extra_convert_source1::text end extra_convert_source1
|
||||
, case when trim(both from extra_convert_source2)='' then null else extra_convert_source2::text end extra_convert_source2
|
||||
, case when trim(both from extra_convert_source3)='' then null else extra_convert_source3::text end extra_convert_source3
|
||||
, case when trim(both from extra_convert_source4)='' then null else extra_convert_source4::text end extra_convert_source4
|
||||
, case when trim(both from extra_convert_source5)='' then null else extra_convert_source5::text end extra_convert_source5
|
||||
, case when trim(both from extra_convert_source6)='' then null else extra_convert_source6::text end extra_convert_source6
|
||||
, case when trim(both from extra_convert_source7)='' then null else extra_convert_source7::text end extra_convert_source7
|
||||
, case when trim(both from extra_convert_source8)='' then null else extra_convert_source8::text end extra_convert_source8
|
||||
, case when trim(both from extra_regist_content_name)='' then null else extra_regist_content_name::text end extra_regist_content_name
|
||||
, case when trim(both from extra_regist_date)='' then null else extra_regist_date::text end extra_regist_date
|
||||
, case when trim(both from extra_regist_source)='' then null else extra_regist_source::text end extra_regist_source
|
||||
, case when trim(both from extra_regist_source1)='' then null else extra_regist_source1::text end extra_regist_source1
|
||||
, case when trim(both from extra_regist_source2)='' then null else extra_regist_source2::text end extra_regist_source2
|
||||
, case when trim(both from extra_regist_source3)='' then null else extra_regist_source3::text end extra_regist_source3
|
||||
, case when trim(both from extra_regist_source4)='' then null else extra_regist_source4::text end extra_regist_source4
|
||||
, case when trim(both from extra_regist_source5)='' then null else extra_regist_source5::text end extra_regist_source5
|
||||
, case when trim(both from extra_regist_source6)='' then null else extra_regist_source6::text end extra_regist_source6
|
||||
, case when trim(both from extra_regist_source7)='' then null else extra_regist_source7::text end extra_regist_source7
|
||||
, case when trim(both from extra_regist_source8)='' then null else extra_regist_source8::text end extra_regist_source8
|
||||
, case when trim(both from first_assign_date)='' then null else first_assign_date::text end first_assign_date
|
||||
, case when trim(both from gender)='' then null else gender::text end gender
|
||||
, case when trim(both from home_address)='' then null else home_address::text end home_address
|
||||
, case when trim(both from id)='' then null else id::text end id
|
||||
, case when trim(both from id_str)='' then null else id_str::text end id_str
|
||||
, case when trim(both from img)='' then null else img::text end img
|
||||
, case when trim(both from industry)='' then null else industry::text end industry
|
||||
, case when trim(both from is_anonymous)='' then null else is_anonymous::text end is_anonymous
|
||||
, case when trim(both from is_employee)='' then null else is_employee::text end is_employee
|
||||
, case when trim(both from is_member)='' then null else is_member::text end is_member
|
||||
, case when trim(both from is_referrer)='' then null else is_referrer::text end is_referrer
|
||||
, case when trim(both from language)='' then null else language::text end language
|
||||
, case when trim(both from last_updated)='' then null else last_updated::text end last_updated
|
||||
, case when trim(both from latest_engage_date)='' then null else latest_engage_date::text end latest_engage_date
|
||||
, case when trim(both from mobile)='' then null else mobile::text end mobile
|
||||
, case when trim(both from mobile_verified)='' then null else mobile_verified::text end mobile_verified
|
||||
, case when trim(both from name)='' then null else name::text end name
|
||||
, case when trim(both from nick_name)='' then null else nick_name::text end nick_name
|
||||
, case when trim(both from not_accept_email)='' then null else not_accept_email::text end not_accept_email
|
||||
, case when trim(both from not_acceptsms)='' then null else not_acceptsms::text end not_acceptsms
|
||||
, case when trim(both from office_address)='' then null else office_address::text end office_address
|
||||
, case when trim(both from owner_date)='' then null else owner_date::text end owner_date
|
||||
, case when trim(both from owner_employee_id)='' then null else owner_employee_id::text end owner_employee_id
|
||||
, case when trim(both from owner_employee_name)='' then null else owner_employee_name::text end owner_employee_name
|
||||
, case when trim(both from owner_id)='' then null else owner_id::text end owner_id
|
||||
, case when trim(both from owner_name)='' then null else owner_name::text end owner_name
|
||||
, case when trim(both from province)='' then null else province::text end province
|
||||
, case when trim(both from qq)='' then null else qq::text end qq
|
||||
, case when trim(both from referrer)='' then null else referrer::text end referrer
|
||||
, case when trim(both from referrer_name)='' then null else referrer_name::text end referrer_name
|
||||
, case when trim(both from refund_date)='' then null else refund_date::text end refund_date
|
||||
, case when trim(both from region)='' then null else region::text end region
|
||||
, case when trim(both from remark)='' then null else remark::text end remark
|
||||
, case when trim(both from source)='' then null else source::text end source
|
||||
, case when trim(both from source1)='' then null else source1::text end source1
|
||||
, case when trim(both from source2)='' then null else source2::text end source2
|
||||
, case when trim(both from source3)='' then null else source3::text end source3
|
||||
, case when trim(both from source4)='' then null else source4::text end source4
|
||||
, case when trim(both from source5)='' then null else source5::text end source5
|
||||
, case when trim(both from source6)='' then null else source6::text end source6
|
||||
, case when trim(both from source7)='' then null else source7::text end source7
|
||||
, case when trim(both from source8)='' then null else source8::text end source8
|
||||
, case when trim(both from stage)='' then null else stage::text end stage
|
||||
, case when trim(both from status)='' then null else status::text end status
|
||||
, case when trim(both from street)='' then null else street::text end street
|
||||
, case when trim(both from subscribe_scene)='' then null else subscribe_scene::text end subscribe_scene
|
||||
, case when trim(both from telephone)='' then null else telephone::text end telephone
|
||||
, case when trim(both from title)='' then null else title::text end title
|
||||
, case when trim(both from type)='' then null else type::text end type
|
||||
, case when trim(both from update_method)='' then null else update_method::text end update_method
|
||||
, case when trim(both from utm_medium)='' then null else utm_medium::text end utm_medium
|
||||
, case when trim(both from utm_term)='' then null else utm_term::text end utm_term
|
||||
, case when trim(both from version)='' then null else version::text end version
|
||||
, case when trim(both from website)='' then null else website::text end website
|
||||
, case when trim(both from wechat)='' then null else wechat::text end wechat
|
||||
, case when trim(both from wechat_city)='' then null else wechat_city::text end wechat_city
|
||||
, case when trim(both from wechat_nick_name)='' then null else wechat_nick_name::text end wechat_nick_name
|
||||
, case when trim(both from weibo)='' then null else weibo::text end weibo
|
||||
,etl_tx_dt
|
||||
from (
|
||||
select
|
||||
(json_array_elements(data::json)::json->>'acceptEmail') accept_email
|
||||
, (json_array_elements(data::json)::json->>'acceptsms') acceptsms
|
||||
, (json_array_elements(data::json)::json->>'annualRevenue') annual_revenue
|
||||
, (json_array_elements(data::json)::json->>'attrId') attr_id
|
||||
, (json_array_elements(data::json)::json->>'attr1') attr1
|
||||
, (json_array_elements(data::json)::json->>'attr10') attr10
|
||||
, (json_array_elements(data::json)::json->>'attr11') attr11
|
||||
, (json_array_elements(data::json)::json->>'attr12') attr12
|
||||
, (json_array_elements(data::json)::json->>'attr13') attr13
|
||||
, (json_array_elements(data::json)::json->>'attr14') attr14
|
||||
, (json_array_elements(data::json)::json->>'attr15') attr15
|
||||
, (json_array_elements(data::json)::json->>'attr16') attr16
|
||||
, (json_array_elements(data::json)::json->>'attr17') attr17
|
||||
, (json_array_elements(data::json)::json->>'attr18') attr18
|
||||
, (json_array_elements(data::json)::json->>'attr19') attr19
|
||||
, (json_array_elements(data::json)::json->>'attr2') attr2
|
||||
, (json_array_elements(data::json)::json->>'attr20') attr20
|
||||
, (json_array_elements(data::json)::json->>'attr21') attr21
|
||||
, (json_array_elements(data::json)::json->>'attr22') attr22
|
||||
, (json_array_elements(data::json)::json->>'attr23') attr23
|
||||
, (json_array_elements(data::json)::json->>'attr24') attr24
|
||||
, (json_array_elements(data::json)::json->>'attr25') attr25
|
||||
, (json_array_elements(data::json)::json->>'attr26') attr26
|
||||
, (json_array_elements(data::json)::json->>'attr27') attr27
|
||||
, (json_array_elements(data::json)::json->>'attr28') attr28
|
||||
, (json_array_elements(data::json)::json->>'attr3') attr3
|
||||
, (json_array_elements(data::json)::json->>'attr4') attr4
|
||||
, (json_array_elements(data::json)::json->>'attr5') attr5
|
||||
, (json_array_elements(data::json)::json->>'attr6') attr6
|
||||
, (json_array_elements(data::json)::json->>'attr7') attr7
|
||||
, (json_array_elements(data::json)::json->>'attr8') attr8
|
||||
, (json_array_elements(data::json)::json->>'attr9') attr9
|
||||
, (json_array_elements(data::json)::json->>'birthday') birthday
|
||||
, (json_array_elements(data::json)::json->>'campaignId') campaign_id
|
||||
, (json_array_elements(data::json)::json->>'campaignName') campaign_name
|
||||
, (json_array_elements(data::json)::json->>'city') city
|
||||
, (json_array_elements(data::json)::json->>'company') company
|
||||
, (json_array_elements(data::json)::json->>'contentName') content_name
|
||||
, (json_array_elements(data::json)::json->>'convertDate') convert_date
|
||||
, (json_array_elements(data::json)::json->>'country') country
|
||||
, (json_array_elements(data::json)::json->>'county') county
|
||||
, (json_array_elements(data::json)::json->>'createFrom') create_from
|
||||
, (json_array_elements(data::json)::json->>'createFromName') create_from_name
|
||||
, (json_array_elements(data::json)::json->>'createMethod') create_method
|
||||
, (json_array_elements(data::json)::json->>'dateCreated') date_created
|
||||
, (json_array_elements(data::json)::json->>'dateJoin') date_join
|
||||
, (json_array_elements(data::json)::json->>'dateMemberReg') date_member_reg
|
||||
, (json_array_elements(data::json)::json->>'displayName') display_name
|
||||
, (json_array_elements(data::json)::json->>'email') email
|
||||
, (json_array_elements(data::json)::json->>'emailVerified') email_verified
|
||||
, (json_array_elements(data::json)::json->>'employeeNumber') employee_number
|
||||
, (json_array_elements(data::json)::json->>'externalId') external_id
|
||||
, (json_array_elements(data::json)::json->>'extraConvertContentName') extra_convert_content_name
|
||||
, (json_array_elements(data::json)::json->>'extraConvertDate') extra_convert_date
|
||||
, (json_array_elements(data::json)::json->>'extraConvertSource') extra_convert_source
|
||||
, (json_array_elements(data::json)::json->>'extraConvertSource1') extra_convert_source1
|
||||
, (json_array_elements(data::json)::json->>'extraConvertSource2') extra_convert_source2
|
||||
, (json_array_elements(data::json)::json->>'extraConvertSource3') extra_convert_source3
|
||||
, (json_array_elements(data::json)::json->>'extraConvertSource4') extra_convert_source4
|
||||
, (json_array_elements(data::json)::json->>'extraConvertSource5') extra_convert_source5
|
||||
, (json_array_elements(data::json)::json->>'extraConvertSource6') extra_convert_source6
|
||||
, (json_array_elements(data::json)::json->>'extraConvertSource7') extra_convert_source7
|
||||
, (json_array_elements(data::json)::json->>'extraConvertSource8') extra_convert_source8
|
||||
, (json_array_elements(data::json)::json->>'extraRegistContentName') extra_regist_content_name
|
||||
, (json_array_elements(data::json)::json->>'extraRegistDate') extra_regist_date
|
||||
, (json_array_elements(data::json)::json->>'extraRegistSource') extra_regist_source
|
||||
, (json_array_elements(data::json)::json->>'extraRegistSource1') extra_regist_source1
|
||||
, (json_array_elements(data::json)::json->>'extraRegistSource2') extra_regist_source2
|
||||
, (json_array_elements(data::json)::json->>'extraRegistSource3') extra_regist_source3
|
||||
, (json_array_elements(data::json)::json->>'extraRegistSource4') extra_regist_source4
|
||||
, (json_array_elements(data::json)::json->>'extraRegistSource5') extra_regist_source5
|
||||
, (json_array_elements(data::json)::json->>'extraRegistSource6') extra_regist_source6
|
||||
, (json_array_elements(data::json)::json->>'extraRegistSource7') extra_regist_source7
|
||||
, (json_array_elements(data::json)::json->>'extraRegistSource8') extra_regist_source8
|
||||
, (json_array_elements(data::json)::json->>'firstAssignDate') first_assign_date
|
||||
, (json_array_elements(data::json)::json->>'gender') gender
|
||||
, (json_array_elements(data::json)::json->>'homeAddress') home_address
|
||||
, (json_array_elements(data::json)::json->>'id') id
|
||||
, (json_array_elements(data::json)::json->>'idStr') id_str
|
||||
, (json_array_elements(data::json)::json->>'img') img
|
||||
, (json_array_elements(data::json)::json->>'industry') industry
|
||||
, (json_array_elements(data::json)::json->>'isAnonymous') is_anonymous
|
||||
, (json_array_elements(data::json)::json->>'isEmployee') is_employee
|
||||
, (json_array_elements(data::json)::json->>'isMember') is_member
|
||||
, (json_array_elements(data::json)::json->>'isReferrer') is_referrer
|
||||
, (json_array_elements(data::json)::json->>'language') language
|
||||
, (json_array_elements(data::json)::json->>'lastUpdated') last_updated
|
||||
, (json_array_elements(data::json)::json->>'latestEngageDate') latest_engage_date
|
||||
, (json_array_elements(data::json)::json->>'mobile') mobile
|
||||
, (json_array_elements(data::json)::json->>'mobileVerified') mobile_verified
|
||||
, (json_array_elements(data::json)::json->>'name') name
|
||||
, (json_array_elements(data::json)::json->>'nickName') nick_name
|
||||
, (json_array_elements(data::json)::json->>'notAcceptEmail') not_accept_email
|
||||
, (json_array_elements(data::json)::json->>'notAcceptsms') not_acceptsms
|
||||
, (json_array_elements(data::json)::json->>'officeAddress') office_address
|
||||
, (json_array_elements(data::json)::json->>'ownerDate') owner_date
|
||||
, (json_array_elements(data::json)::json->>'ownerEmployeeId') owner_employee_id
|
||||
, (json_array_elements(data::json)::json->>'ownerEmployeeName') owner_employee_name
|
||||
, (json_array_elements(data::json)::json->>'ownerId') owner_id
|
||||
, (json_array_elements(data::json)::json->>'ownerName') owner_name
|
||||
, (json_array_elements(data::json)::json->>'province') province
|
||||
, (json_array_elements(data::json)::json->>'qq') qq
|
||||
, (json_array_elements(data::json)::json->>'referrer') referrer
|
||||
, (json_array_elements(data::json)::json->>'referrerName') referrer_name
|
||||
, (json_array_elements(data::json)::json->>'refundDate') refund_date
|
||||
, (json_array_elements(data::json)::json->>'region') region
|
||||
, (json_array_elements(data::json)::json->>'remark') remark
|
||||
, (json_array_elements(data::json)::json->>'source') source
|
||||
, (json_array_elements(data::json)::json->>'source1') source1
|
||||
, (json_array_elements(data::json)::json->>'source2') source2
|
||||
, (json_array_elements(data::json)::json->>'source3') source3
|
||||
, (json_array_elements(data::json)::json->>'source4') source4
|
||||
, (json_array_elements(data::json)::json->>'source5') source5
|
||||
, (json_array_elements(data::json)::json->>'source6') source6
|
||||
, (json_array_elements(data::json)::json->>'source7') source7
|
||||
, (json_array_elements(data::json)::json->>'source8') source8
|
||||
, (json_array_elements(data::json)::json->>'stage') stage
|
||||
, (json_array_elements(data::json)::json->>'status') status
|
||||
, (json_array_elements(data::json)::json->>'street') street
|
||||
, (json_array_elements(data::json)::json->>'subscribeScene') subscribe_scene
|
||||
, (json_array_elements(data::json)::json->>'telephone') telephone
|
||||
, (json_array_elements(data::json)::json->>'title') title
|
||||
, (json_array_elements(data::json)::json->>'type') type
|
||||
, (json_array_elements(data::json)::json->>'updateMethod') update_method
|
||||
, (json_array_elements(data::json)::json->>'utmMedium') utm_medium
|
||||
, (json_array_elements(data::json)::json->>'utmTerm') utm_term
|
||||
, (json_array_elements(data::json)::json->>'version') version
|
||||
, (json_array_elements(data::json)::json->>'website') website
|
||||
, (json_array_elements(data::json)::json->>'wechat') wechat
|
||||
, (json_array_elements(data::json)::json->>'wechatCity') wechat_city
|
||||
, (json_array_elements(data::json)::json->>'wechatNickName') wechat_nick_name
|
||||
, (json_array_elements(data::json)::json->>'weibo') weibo
|
||||
,CURRENT_TIMESTAMP(0) etl_tx_dt
|
||||
from (select * from data_api.api_data
|
||||
WHERE api_id='cab1a761-b8e5-4db4-a769-88db3ec1' and is_loaded = '0' order by request_tm desc limit 1) p )p;
|
||||
|
||||
update data_api.api_data
|
||||
set is_loaded = '1' ,
|
||||
status = '1',
|
||||
request_tm = current_timestamp(0)
|
||||
where api_id='cab1a761-b8e5-4db4-a769-88db3ec1';
|
||||
\q
|
||||
|
|
@ -1,41 +0,0 @@
|
|||
# coding: utf-8
|
||||
import requests
|
||||
import json
|
||||
import psycopg2
|
||||
import uuid
|
||||
import datetime
|
||||
|
||||
# 获取当前日期
|
||||
current_date = datetime.date.today()
|
||||
previous_date = current_date - datetime.timedelta(days=2)
|
||||
formatted_current_date = current_date.strftime("%Y-%m-%dT%H:%M:%SZ")
|
||||
formatted_previous_date = previous_date.strftime("%Y-%m-%dT%H:%M:%SZ")
|
||||
|
||||
print('开始加载数据：customer_events:获取线索事件')
|
||||
authUrl='https://api.huiju.cool/v2/oauth2/token?app_id=cl037b184ebccd97c&secret=deaa83fdd8d385eec0d1d04d34282c4ec8c33b60&grant_type=client_credentials'
|
||||
|
||||
print('开始请求令牌。')
|
||||
authRequest=requests.get(authUrl)
|
||||
auth=json.loads(authRequest.text)
|
||||
print('开始请求数据总数。')
|
||||
url='https://api.huiju.cool/v2/customerEvents'
|
||||
header={}
|
||||
body={'access_token':auth['access_token'],'limit':'2000','sort':'date','event':'create_customer','date[le]':formatted_current_date,}
|
||||
dataReqL=requests.get(url,headers=header,params=body)
|
||||
resL=json.loads(dataReqL.text)
|
||||
# print(resL)
|
||||
dataList=resL['data']
|
||||
total=len(dataList)
|
||||
conn = psycopg2.connect(database="dataops_db", user="dbuser_dops", password="MIgTi3jA",
|
||||
host="172.17.0.8", port="5432")
|
||||
print('数据库连接成功')
|
||||
dataId=str(uuid.uuid4())
|
||||
print('临时id：'+dataId)
|
||||
json_object = json.dumps(dataList)
|
||||
cur=conn.cursor()
|
||||
sql="update data_api.api_data set is_loaded = '1' where api_id = '747d33fa-a0e0-421d-aa9b-4ca4517b';INSERT INTO data_api.api_data (id,api_id,data,total_num,is_loaded,status,request_tm,execute_tm,remark) values (%s,%s,%s,%s,'0','0',current_timestamp(0),current_timestamp(0),'')"
|
||||
cur.execute(sql,[dataId,'747d33fa-a0e0-421d-aa9b-4ca4517b',json_object,total])
|
||||
conn.commit()
|
||||
cur.close()
|
||||
conn.close()
|
||||
print('加载数据结束：customer_events:获取线索事件')
|
||||
|
|
@ -1,57 +0,0 @@
|
|||
/*******Main Section**************************************************************************/
|
||||
\set ON_ERROR_STOP on
|
||||
\set AUTOCOMMIT on
|
||||
\timing on
|
||||
|
||||
DELETE FROM data_api.customer_events;
|
||||
|
||||
insert into data_api.customer_events (
|
||||
content_name
|
||||
, create_method
|
||||
, customer_id
|
||||
, customer_id_str
|
||||
, date
|
||||
, event
|
||||
, external_id
|
||||
, id
|
||||
, id_str
|
||||
, last_updated
|
||||
, source
|
||||
,etl_tx_dt
|
||||
)
|
||||
select
|
||||
case when trim(both from content_name)='' then null else content_name::text end content_name
|
||||
, case when trim(both from create_method)='' then null else create_method::text end create_method
|
||||
, case when trim(both from customer_id)='' then null else customer_id::text end customer_id
|
||||
, case when trim(both from customer_id_str)='' then null else customer_id_str::text end customer_id_str
|
||||
, case when trim(both from date)='' then null else date::text end date
|
||||
, case when trim(both from event)='' then null else event::text end event
|
||||
, case when trim(both from external_id)='' then null else external_id::text end external_id
|
||||
, case when trim(both from id)='' then null else id::text end id
|
||||
, case when trim(both from id_str)='' then null else id_str::text end id_str
|
||||
, case when trim(both from last_updated)='' then null else last_updated::text end last_updated
|
||||
, case when trim(both from source)='' then null else source::text end source
|
||||
,etl_tx_dt
|
||||
from (
|
||||
select
|
||||
(json_array_elements(data::json)::json->>'contentName') content_name
|
||||
, (json_array_elements(data::json)::json->>'createMethod') create_method
|
||||
, (json_array_elements(data::json)::json->>'customerId') customer_id
|
||||
, (json_array_elements(data::json)::json->>'customerIdStr') customer_id_str
|
||||
, (json_array_elements(data::json)::json->>'date') date
|
||||
, (json_array_elements(data::json)::json->>'event') event
|
||||
, (json_array_elements(data::json)::json->>'externalId') external_id
|
||||
, (json_array_elements(data::json)::json->>'id') id
|
||||
, (json_array_elements(data::json)::json->>'idStr') id_str
|
||||
, (json_array_elements(data::json)::json->>'lastUpdated') last_updated
|
||||
, (json_array_elements(data::json)::json->>'source') source
|
||||
,CURRENT_TIMESTAMP(0) etl_tx_dt
|
||||
from (select * from data_api.api_data
|
||||
WHERE api_id='747d33fa-a0e0-421d-aa9b-4ca4517b' and is_loaded = '0' order by request_tm desc limit 1) p )p;
|
||||
|
||||
update data_api.api_data
|
||||
set is_loaded = '1' ,
|
||||
status = '1',
|
||||
request_tm = current_timestamp(0)
|
||||
where api_id='747d33fa-a0e0-421d-aa9b-4ca4517b';
|
||||
\q
|
||||
|
|
@ -1,41 +0,0 @@
|
|||
# coding: utf-8
|
||||
import requests
|
||||
import json
|
||||
import psycopg2
|
||||
import uuid
|
||||
import datetime
|
||||
|
||||
# 获取当前日期
|
||||
current_date = datetime.date.today()
|
||||
previous_date = current_date - datetime.timedelta(days=365)
|
||||
formatted_current_date = current_date.strftime("%Y-%m-%dT%H:%M:%SZ")
|
||||
formatted_previous_date = previous_date.strftime("%Y-%m-%dT%H:%M:%SZ")
|
||||
|
||||
print('开始加载数据：custom_events_7:获取线索事件_cp_employee_tools_huiju_event_add')
|
||||
authUrl='https://api.huiju.cool/v2/oauth2/token?app_id=cl037b184ebccd97c&secret=deaa83fdd8d385eec0d1d04d34282c4ec8c33b60&grant_type=client_credentials'
|
||||
|
||||
print('开始请求令牌。')
|
||||
authRequest=requests.get(authUrl)
|
||||
auth=json.loads(authRequest.text)
|
||||
print('开始请求数据总数。')
|
||||
url='https://api.huiju.cool/v2/customerEvents'
|
||||
header={}
|
||||
body={'event':'cp_employee_tools_huiju_event_add_user','access_token':auth['access_token'],'limit':'2000','sort':'date','date[le]':formatted_current_date,}
|
||||
dataReqL=requests.get(url,headers=header,params=body)
|
||||
resL=json.loads(dataReqL.text)
|
||||
# print(resL)
|
||||
dataList=resL['data']
|
||||
total=len(dataList)
|
||||
conn = psycopg2.connect(database="dataops_db", user="dbuser_dops", password="MIgTi3jA",
|
||||
host="172.17.0.8", port="5432")
|
||||
print('数据库连接成功')
|
||||
dataId=str(uuid.uuid4())
|
||||
print('临时id：'+dataId)
|
||||
json_object = json.dumps(dataList)
|
||||
cur=conn.cursor()
|
||||
sql="update data_api.api_data set is_loaded = '1' where api_id = 'd3b5116a-08c3-431f-8d96-9780221b';INSERT INTO data_api.api_data (id,api_id,data,total_num,is_loaded,status,request_tm,execute_tm,remark) values (%s,%s,%s,%s,'0','0',current_timestamp(0),current_timestamp(0),'')"
|
||||
cur.execute(sql,[dataId,'d3b5116a-08c3-431f-8d96-9780221b',json_object,total])
|
||||
conn.commit()
|
||||
cur.close()
|
||||
conn.close()
|
||||
print('加载数据结束：custom_events_7:获取线索事件_cp_employee_tools_huiju_event_add')
|
||||
|
|
@ -1,72 +0,0 @@
|
|||
/*******Main Section**************************************************************************/
|
||||
\set ON_ERROR_STOP on
|
||||
\set AUTOCOMMIT on
|
||||
\timing on
|
||||
|
||||
DELETE FROM data_api.customer_events_add_user;
|
||||
|
||||
insert into data_api.customer_events_add_user (
|
||||
attr1
|
||||
, attr2
|
||||
, attr5
|
||||
, attr6
|
||||
, attr7
|
||||
, customer_id
|
||||
, customer_id_str
|
||||
, date
|
||||
, event
|
||||
, external_id
|
||||
, id
|
||||
, id_str
|
||||
, last_updated
|
||||
, source
|
||||
, summary
|
||||
, target_id
|
||||
,etl_tx_dt
|
||||
)
|
||||
select
|
||||
case when trim(both from attr1)='' then null else attr1::text end attr1
|
||||
, case when trim(both from attr2)='' then null else attr2::text end attr2
|
||||
, case when trim(both from attr5)='' then null else attr5::text end attr5
|
||||
, case when trim(both from attr6)='' then null else attr6::text end attr6
|
||||
, case when trim(both from attr7)='' then null else attr7::text end attr7
|
||||
, case when trim(both from customer_id)='' then null else customer_id::text end customer_id
|
||||
, case when trim(both from customer_id_str)='' then null else customer_id_str::text end customer_id_str
|
||||
, case when trim(both from date)='' then null else date::text end date
|
||||
, case when trim(both from event)='' then null else event::text end event
|
||||
, case when trim(both from external_id)='' then null else external_id::text end external_id
|
||||
, case when trim(both from id)='' then null else id::text end id
|
||||
, case when trim(both from id_str)='' then null else id_str::text end id_str
|
||||
, case when trim(both from last_updated)='' then null else last_updated::text end last_updated
|
||||
, case when trim(both from source)='' then null else source::text end source
|
||||
, case when trim(both from summary)='' then null else summary::text end summary
|
||||
, case when trim(both from target_id)='' then null else target_id::text end target_id
|
||||
,etl_tx_dt
|
||||
from (
|
||||
select
|
||||
(json_array_elements(data::json)::json->>'attr1') attr1
|
||||
, (json_array_elements(data::json)::json->>'attr2') attr2
|
||||
, (json_array_elements(data::json)::json->>'attr5') attr5
|
||||
, (json_array_elements(data::json)::json->>'attr6') attr6
|
||||
, (json_array_elements(data::json)::json->>'attr7') attr7
|
||||
, (json_array_elements(data::json)::json->>'customerId') customer_id
|
||||
, (json_array_elements(data::json)::json->>'customerIdStr') customer_id_str
|
||||
, (json_array_elements(data::json)::json->>'date') date
|
||||
, (json_array_elements(data::json)::json->>'event') event
|
||||
, (json_array_elements(data::json)::json->>'externalId') external_id
|
||||
, (json_array_elements(data::json)::json->>'id') id
|
||||
, (json_array_elements(data::json)::json->>'idStr') id_str
|
||||
, (json_array_elements(data::json)::json->>'lastUpdated') last_updated
|
||||
, (json_array_elements(data::json)::json->>'source') source
|
||||
, (json_array_elements(data::json)::json->>'summary') summary
|
||||
, (json_array_elements(data::json)::json->>'targetId') target_id
|
||||
,CURRENT_TIMESTAMP(0) etl_tx_dt
|
||||
from (select * from data_api.api_data
|
||||
WHERE api_id='d3b5116a-08c3-431f-8d96-9780221b' and is_loaded = '0' order by request_tm desc limit 1) p )p;
|
||||
|
||||
update data_api.api_data
|
||||
set is_loaded = '1' ,
|
||||
status = '1',
|
||||
request_tm = current_timestamp(0)
|
||||
where api_id='d3b5116a-08c3-431f-8d96-9780221b';
|
||||
\q
|
||||
Loading…
Reference in New Issue