clear workflow
This commit is contained in:
root
parent
61789ce8fc
commit
ef53aa7083
|
|
@ -1,41 +0,0 @@
|
||||||
# coding: utf-8
|
|
||||||
import requests
|
|
||||||
import json
|
|
||||||
import psycopg2
|
|
||||||
import uuid
|
|
||||||
import datetime
|
|
||||||
|
|
||||||
# 获取当前日期
|
|
||||||
current_date = datetime.date.today()
|
|
||||||
previous_date = current_date - datetime.timedelta(days=365)
|
|
||||||
formatted_current_date = current_date.strftime("%Y-%m-%dT%H:%M:%SZ")
|
|
||||||
formatted_previous_date = previous_date.strftime("%Y-%m-%dT%H:%M:%SZ")
|
|
||||||
|
|
||||||
print('开始加载数据：custom_events_7:获取线索事件_cp_employee_tools_huiju_event_add')
|
|
||||||
authUrl='https://api.huiju.cool/v2/oauth2/token?app_id=cl037b184ebccd97c&secret=deaa83fdd8d385eec0d1d04d34282c4ec8c33b60&grant_type=client_credentials'
|
|
||||||
|
|
||||||
print('开始请求令牌。')
|
|
||||||
authRequest=requests.get(authUrl)
|
|
||||||
auth=json.loads(authRequest.text)
|
|
||||||
print('开始请求数据总数。')
|
|
||||||
url='https://api.huiju.cool/v2/customerEvents'
|
|
||||||
header={}
|
|
||||||
body={'event':'cp_employee_tools_huiju_event_add_user','access_token':auth['access_token'],'limit':'2000','sort':'date','date[ge]':formatted_previous_date,'date[le]':formatted_current_date,}
|
|
||||||
dataReqL=requests.get(url,headers=header,params=body)
|
|
||||||
resL=json.loads(dataReqL.text)
|
|
||||||
# print(resL)
|
|
||||||
dataList=resL['data']
|
|
||||||
total=len(dataList)
|
|
||||||
conn = psycopg2.connect(database="dataops_db", user="dbuser_dops", password="MIgTi3jA",
|
|
||||||
host="172.17.0.8", port="5432")
|
|
||||||
print('数据库连接成功')
|
|
||||||
dataId=str(uuid.uuid4())
|
|
||||||
print('临时id：'+dataId)
|
|
||||||
json_object = json.dumps(dataList)
|
|
||||||
cur=conn.cursor()
|
|
||||||
sql="update data_api.api_data set is_loaded = '1' where api_id = 'd3b5116a-08c3-431f-8d96-9780221b';INSERT INTO data_api.api_data (id,api_id,data,total_num,is_loaded,status,request_tm,execute_tm,remark) values (%s,%s,%s,%s,'0','0',current_timestamp(0),current_timestamp(0),'')"
|
|
||||||
cur.execute(sql,[dataId,'d3b5116a-08c3-431f-8d96-9780221b',json_object,total])
|
|
||||||
conn.commit()
|
|
||||||
cur.close()
|
|
||||||
conn.close()
|
|
||||||
print('加载数据结束：custom_events_7:获取线索事件_cp_employee_tools_huiju_event_add')
|
|
||||||
|
|
@ -1,72 +0,0 @@
|
||||||
/*******Main Section**************************************************************************/
|
|
||||||
\set ON_ERROR_STOP on
|
|
||||||
\set AUTOCOMMIT on
|
|
||||||
\timing on
|
|
||||||
|
|
||||||
DELETE FROM data_api.customer_events_add_user;
|
|
||||||
|
|
||||||
insert into data_api.customer_events_add_user (
|
|
||||||
attr1
|
|
||||||
, attr2
|
|
||||||
, attr5
|
|
||||||
, attr6
|
|
||||||
, attr7
|
|
||||||
, customer_id
|
|
||||||
, customer_id_str
|
|
||||||
, date
|
|
||||||
, event
|
|
||||||
, external_id
|
|
||||||
, id
|
|
||||||
, id_str
|
|
||||||
, last_updated
|
|
||||||
, source
|
|
||||||
, summary
|
|
||||||
, target_id
|
|
||||||
,etl_tx_dt
|
|
||||||
)
|
|
||||||
select
|
|
||||||
case when trim(both from attr1)='' then null else attr1::text end attr1
|
|
||||||
, case when trim(both from attr2)='' then null else attr2::text end attr2
|
|
||||||
, case when trim(both from attr5)='' then null else attr5::text end attr5
|
|
||||||
, case when trim(both from attr6)='' then null else attr6::text end attr6
|
|
||||||
, case when trim(both from attr7)='' then null else attr7::text end attr7
|
|
||||||
, case when trim(both from customer_id)='' then null else customer_id::text end customer_id
|
|
||||||
, case when trim(both from customer_id_str)='' then null else customer_id_str::text end customer_id_str
|
|
||||||
, case when trim(both from date)='' then null else date::text end date
|
|
||||||
, case when trim(both from event)='' then null else event::text end event
|
|
||||||
, case when trim(both from external_id)='' then null else external_id::text end external_id
|
|
||||||
, case when trim(both from id)='' then null else id::text end id
|
|
||||||
, case when trim(both from id_str)='' then null else id_str::text end id_str
|
|
||||||
, case when trim(both from last_updated)='' then null else last_updated::text end last_updated
|
|
||||||
, case when trim(both from source)='' then null else source::text end source
|
|
||||||
, case when trim(both from summary)='' then null else summary::text end summary
|
|
||||||
, case when trim(both from target_id)='' then null else target_id::text end target_id
|
|
||||||
,etl_tx_dt
|
|
||||||
from (
|
|
||||||
select
|
|
||||||
(json_array_elements(data::json)::json->>'attr1') attr1
|
|
||||||
, (json_array_elements(data::json)::json->>'attr2') attr2
|
|
||||||
, (json_array_elements(data::json)::json->>'attr5') attr5
|
|
||||||
, (json_array_elements(data::json)::json->>'attr6') attr6
|
|
||||||
, (json_array_elements(data::json)::json->>'attr7') attr7
|
|
||||||
, (json_array_elements(data::json)::json->>'customerId') customer_id
|
|
||||||
, (json_array_elements(data::json)::json->>'customerIdStr') customer_id_str
|
|
||||||
, (json_array_elements(data::json)::json->>'date') date
|
|
||||||
, (json_array_elements(data::json)::json->>'event') event
|
|
||||||
, (json_array_elements(data::json)::json->>'externalId') external_id
|
|
||||||
, (json_array_elements(data::json)::json->>'id') id
|
|
||||||
, (json_array_elements(data::json)::json->>'idStr') id_str
|
|
||||||
, (json_array_elements(data::json)::json->>'lastUpdated') last_updated
|
|
||||||
, (json_array_elements(data::json)::json->>'source') source
|
|
||||||
, (json_array_elements(data::json)::json->>'summary') summary
|
|
||||||
, (json_array_elements(data::json)::json->>'targetId') target_id
|
|
||||||
,CURRENT_TIMESTAMP(0) etl_tx_dt
|
|
||||||
from (select * from data_api.api_data
|
|
||||||
WHERE api_id='d3b5116a-08c3-431f-8d96-9780221b' and is_loaded = '0' order by request_tm desc limit 1) p )p;
|
|
||||||
|
|
||||||
update data_api.api_data
|
|
||||||
set is_loaded = '1' ,
|
|
||||||
status = '1',
|
|
||||||
request_tm = current_timestamp(0)
|
|
||||||
where api_id='d3b5116a-08c3-431f-8d96-9780221b';
|
|
||||||
\q
|
|
||||||
Loading…
Reference in New Issue