dtgitea
/
dataops
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Compare commits

base: dtgitea:20617b311236024235f185cc305cfab4b8760912
Branches Tags
dtgitea:master
...
compare: dtgitea:ed5fac1949484999a4d07a81f66a724dccf7a7dc
Branches Tags
dtgitea:master

2 Commits
20617b3112 ... ed5fac1949

Author SHA1 Message Date
root ed5fac1949 add workflow 荟聚API,dev 2024-02-05 14:25:07 +08:00
root 07ff31622f clear workflow 2024-02-05 14:25:07 +08:00
2 changed files with 144 additions and 0 deletions
Show Stats Expand all files Collapse all files

42
TK_Cust/dev/tk_api/获取线索事件_open_app/custom_events_1_feign.py Normal file
View File

@ -0,0 +1,42 @@
# coding: utf-8
import requests
import json
import psycopg2
import uuid
import datetime
# 获取当前日期
current_date = datetime.date.today()
previous_date1 = current_date - datetime.timedelta(days=1)
previous_date2 = current_date - datetime.timedelta(days=3)
formatted_current_date = previous_date1.strftime("%Y-%m-%dT%H:%M:%SZ")
formatted_previous_date = previous_date2.strftime("%Y-%m-%dT%H:%M:%SZ")
print('开始加载数据：custom_events_1:获取线索事件_open_app')
authUrl='https://api.huiju.cool/v2/oauth2/token?app_id=cl037b184ebccd97c&secret=deaa83fdd8d385eec0d1d04d34282c4ec8c33b60&grant_type=client_credentials'
print('开始请求令牌。')
authRequest=requests.get(authUrl)
auth=json.loads(authRequest.text)
print('开始请求数据总数。')
url='https://api.huiju.cool/v2/customerEvents'
header={}
body={'access_token':auth['access_token'],'sort':'date','limit':'2000','date[le]':formatted_current_date,'date[ge]':formatted_previous_date,'event':'open_app',}
dataReqL=requests.get(url,headers=header,params=body)
resL=json.loads(dataReqL.text)
# print(resL)
dataList=resL['data']
total=len(dataList)
conn = psycopg2.connect(database="dataops_db", user="dbuser_dops", password="MIgTi3jA",
host="172.17.0.8", port="5432")
print('数据库连接成功')
dataId=str(uuid.uuid4())
print('临时id：'+dataId)
json_object = json.dumps(dataList)
cur=conn.cursor()
sql="update data_api.api_data set is_loaded = '1' where api_id = '2433868c-3416-4475-884d-11e687a6';INSERT INTO data_api.api_data (id,api_id,data,total_num,is_loaded,status,request_tm,execute_tm,remark) values (%s,%s,%s,%s,'0','0',current_timestamp(0),current_timestamp(0),'')"
cur.execute(sql,[dataId,'2433868c-3416-4475-884d-11e687a6',json_object,total])
conn.commit()
cur.close()
conn.close()
print('加载数据结束：custom_events_1:获取线索事件_open_app')

102
TK_Cust/dev/tk_api/获取线索事件_open_app/custom_events_1_load.sql Normal file
View File

@ -0,0 +1,102 @@
/*******Main Section**************************************************************************/
\set ON_ERROR_STOP on
\set AUTOCOMMIT on
\timing on
DELETE FROM data_api.customer_events_open_app;
insert into data_api.customer_events_open_app (
browser
, browser_version
, customer_id
, customer_id_str
, date
, device
, event
, external_id
, id
, id_str
, ip
, ip_city
, ip_country
, ip_county
, ip_province
, last_updated
, os
, os_version
, page_id
, page_type
, platform
, screen_height
, screen_width
, target_id
, target_name
, url
,etl_tx_dt
)
select
case when trim(both from browser)='' then null else browser::text end browser
, case when trim(both from browser_version)='' then null else browser_version::text end browser_version
, case when trim(both from customer_id)='' then null else customer_id::text end customer_id
, case when trim(both from customer_id_str)='' then null else customer_id_str::text end customer_id_str
, case when trim(both from date)='' then null else date::text end date
, case when trim(both from device)='' then null else device::text end device
, case when trim(both from event)='' then null else event::text end event
, case when trim(both from external_id)='' then null else external_id::text end external_id
, case when trim(both from id)='' then null else id::text end id
, case when trim(both from id_str)='' then null else id_str::text end id_str
, case when trim(both from ip)='' then null else ip::text end ip
, case when trim(both from ip_city)='' then null else ip_city::text end ip_city
, case when trim(both from ip_country)='' then null else ip_country::text end ip_country
, case when trim(both from ip_county)='' then null else ip_county::text end ip_county
, case when trim(both from ip_province)='' then null else ip_province::text end ip_province
, case when trim(both from last_updated)='' then null else last_updated::text end last_updated
, case when trim(both from os)='' then null else os::text end os
, case when trim(both from os_version)='' then null else os_version::text end os_version
, case when trim(both from page_id)='' then null else page_id::text end page_id
, case when trim(both from page_type)='' then null else page_type::text end page_type
, case when trim(both from platform)='' then null else platform::text end platform
, case when trim(both from screen_height)='' then null else screen_height::text end screen_height
, case when trim(both from screen_width)='' then null else screen_width::text end screen_width
, case when trim(both from target_id)='' then null else target_id::text end target_id
, case when trim(both from target_name)='' then null else target_name::text end target_name
, case when trim(both from url)='' then null else url::text end url
,etl_tx_dt
from (
select
(json_array_elements(data::json)::json->>'browser') browser
, (json_array_elements(data::json)::json->>'browserVersion') browser_version
, (json_array_elements(data::json)::json->>'customerId') customer_id
, (json_array_elements(data::json)::json->>'customerIdStr') customer_id_str
, (json_array_elements(data::json)::json->>'date') date
, (json_array_elements(data::json)::json->>'device') device
, (json_array_elements(data::json)::json->>'event') event
, (json_array_elements(data::json)::json->>'externalId') external_id
, (json_array_elements(data::json)::json->>'id') id
, (json_array_elements(data::json)::json->>'idStr') id_str
, (json_array_elements(data::json)::json->>'ip') ip
, (json_array_elements(data::json)::json->>'ipCity') ip_city
, (json_array_elements(data::json)::json->>'ipCountry') ip_country
, (json_array_elements(data::json)::json->>'ipCounty') ip_county
, (json_array_elements(data::json)::json->>'ipProvince') ip_province
, (json_array_elements(data::json)::json->>'lastUpdated') last_updated
, (json_array_elements(data::json)::json->>'os') os
, (json_array_elements(data::json)::json->>'osVersion') os_version
, (json_array_elements(data::json)::json->>'pageId') page_id
, (json_array_elements(data::json)::json->>'pageType') page_type
, (json_array_elements(data::json)::json->>'platform') platform
, (json_array_elements(data::json)::json->>'screenHeight') screen_height
, (json_array_elements(data::json)::json->>'screenWidth') screen_width
, (json_array_elements(data::json)::json->>'targetId') target_id
, (json_array_elements(data::json)::json->>'targetName') target_name
, (json_array_elements(data::json)::json->>'url') url
,CURRENT_TIMESTAMP(0) etl_tx_dt
from (select * from data_api.api_data
WHERE api_id='2433868c-3416-4475-884d-11e687a6' and is_loaded = '0' order by request_tm desc limit 1) p )p;
update data_api.api_data
set is_loaded = '1' ,
status = '1',
request_tm = current_timestamp(0)
where api_id='2433868c-3416-4475-884d-11e687a6';
\q