From ad4865b4fd5e76c26f2b116cfa1f86bca53e38ed Mon Sep 17 00:00:00 2001 From: root Date: Mon, 5 Feb 2024 14:34:45 +0800 Subject: [PATCH] clear workflow --- .../custom_events_5_feign.py | 41 -------- .../custom_events_5_load.sql | 93 ------------------- 2 files changed, 134 deletions(-) delete mode 100644 TK_Cust/dev/tk_api/获取线索事件_c_minipro_page_view/custom_events_5_feign.py delete mode 100644 TK_Cust/dev/tk_api/获取线索事件_c_minipro_page_view/custom_events_5_load.sql diff --git a/TK_Cust/dev/tk_api/获取线索事件_c_minipro_page_view/custom_events_5_feign.py b/TK_Cust/dev/tk_api/获取线索事件_c_minipro_page_view/custom_events_5_feign.py deleted file mode 100644 index 31ce9fc..0000000 --- a/TK_Cust/dev/tk_api/获取线索事件_c_minipro_page_view/custom_events_5_feign.py +++ /dev/null @@ -1,41 +0,0 @@ -# coding: utf-8 -import requests -import json -import psycopg2 -import uuid -import datetime - -# 获取当前日期 -current_date = datetime.date.today() -previous_date = current_date - datetime.timedelta(days=2) -formatted_current_date = current_date.strftime("%Y-%m-%dT%H:%M:%SZ") -formatted_previous_date = previous_date.strftime("%Y-%m-%dT%H:%M:%SZ") - -print('开始加载数据：custom_events_5:获取线索事件_c_minipro_page_view') -authUrl='https://api.huiju.cool/v2/oauth2/token?app_id=cl037b184ebccd97c&secret=deaa83fdd8d385eec0d1d04d34282c4ec8c33b60&grant_type=client_credentials' - -print('开始请求令牌。') -authRequest=requests.get(authUrl) -auth=json.loads(authRequest.text) -print('开始请求数据总数。') -url='https://api.huiju.cool/v2/customerEvents' -header={} -body={'access_token':auth['access_token'],'limit':'2000','event':'c_minipro_page_view','sort':'date','date[ge]':formatted_previous_date,'date[le]':formatted_current_date,} -dataReqL=requests.get(url,headers=header,params=body) -resL=json.loads(dataReqL.text) -# print(resL) -dataList=resL['data'] -total=len(dataList) -conn = psycopg2.connect(database="dataops_db", user="dbuser_dops", password="MIgTi3jA", - host="172.17.0.8", port="5432") -print('数据库连接成功') -dataId=str(uuid.uuid4()) -print('临时id：'+dataId) -json_object = json.dumps(dataList) -cur=conn.cursor() -sql="update data_api.api_data set is_loaded = '1' where api_id = 'a71a43ff-efcd-4427-8dbc-ed2686e4';INSERT INTO data_api.api_data (id,api_id,data,total_num,is_loaded,status,request_tm,execute_tm,remark) values (%s,%s,%s,%s,'0','0',current_timestamp(0),current_timestamp(0),'')" -cur.execute(sql,[dataId,'a71a43ff-efcd-4427-8dbc-ed2686e4',json_object,total]) -conn.commit() -cur.close() -conn.close() -print('加载数据结束：custom_events_5:获取线索事件_c_minipro_page_view') \ No newline at end of file diff --git a/TK_Cust/dev/tk_api/获取线索事件_c_minipro_page_view/custom_events_5_load.sql b/TK_Cust/dev/tk_api/获取线索事件_c_minipro_page_view/custom_events_5_load.sql deleted file mode 100644 index 5c3b5d4..0000000 --- a/TK_Cust/dev/tk_api/获取线索事件_c_minipro_page_view/custom_events_5_load.sql +++ /dev/null @@ -1,93 +0,0 @@ -/*******Main Section**************************************************************************/ -\set ON_ERROR_STOP on -\set AUTOCOMMIT on -\timing on - -DELETE FROM data_api.customer_events_c_minipro_page_view; - -insert into data_api.customer_events_c_minipro_page_view ( - browser - , browser_version - , c_name - , c_source_content - , c_url - , customer_id - , customer_id_str - , date - , event - , external_id - , id - , id_str - , ip - , ip_city - , ip_country - , ip_province - , last_updated - , os - , os_version - , platform - , target_id - , target_name - , url - ,etl_tx_dt -) -select - case when trim(both from browser)='' then null else browser::text end browser - , case when trim(both from browser_version)='' then null else browser_version::text end browser_version - , case when trim(both from c_name)='' then null else c_name::text end c_name - , case when trim(both from c_source_content)='' then null else c_source_content::text end c_source_content - , case when trim(both from c_url)='' then null else c_url::text end c_url - , case when trim(both from customer_id)='' then null else customer_id::text end customer_id - , case when trim(both from customer_id_str)='' then null else customer_id_str::text end customer_id_str - , case when trim(both from date)='' then null else date::text end date - , case when trim(both from event)='' then null else event::text end event - , case when trim(both from external_id)='' then null else external_id::text end external_id - , case when trim(both from id)='' then null else id::text end id - , case when trim(both from id_str)='' then null else id_str::text end id_str - , case when trim(both from ip)='' then null else ip::text end ip - , case when trim(both from ip_city)='' then null else ip_city::text end ip_city - , case when trim(both from ip_country)='' then null else ip_country::text end ip_country - , case when trim(both from ip_province)='' then null else ip_province::text end ip_province - , case when trim(both from last_updated)='' then null else last_updated::text end last_updated - , case when trim(both from os)='' then null else os::text end os - , case when trim(both from os_version)='' then null else os_version::text end os_version - , case when trim(both from platform)='' then null else platform::text end platform - , case when trim(both from target_id)='' then null else target_id::text end target_id - , case when trim(both from target_name)='' then null else target_name::text end target_name - , case when trim(both from url)='' then null else url::text end url -,etl_tx_dt -from ( -select - (json_array_elements(data::json)::json->>'browser') browser - , (json_array_elements(data::json)::json->>'browserVersion') browser_version - , (json_array_elements(data::json)::json->>'c_name') c_name - , (json_array_elements(data::json)::json->>'c_sourceContent') c_source_content - , (json_array_elements(data::json)::json->>'c_url') c_url - , (json_array_elements(data::json)::json->>'customerId') customer_id - , (json_array_elements(data::json)::json->>'customerIdStr') customer_id_str - , (json_array_elements(data::json)::json->>'date') date - , (json_array_elements(data::json)::json->>'event') event - , (json_array_elements(data::json)::json->>'externalId') external_id - , (json_array_elements(data::json)::json->>'id') id - , (json_array_elements(data::json)::json->>'idStr') id_str - , (json_array_elements(data::json)::json->>'ip') ip - , (json_array_elements(data::json)::json->>'ipCity') ip_city - , (json_array_elements(data::json)::json->>'ipCountry') ip_country - , (json_array_elements(data::json)::json->>'ipProvince') ip_province - , (json_array_elements(data::json)::json->>'lastUpdated') last_updated - , (json_array_elements(data::json)::json->>'os') os - , (json_array_elements(data::json)::json->>'osVersion') os_version - , (json_array_elements(data::json)::json->>'platform') platform - , (json_array_elements(data::json)::json->>'targetId') target_id - , (json_array_elements(data::json)::json->>'targetName') target_name - , (json_array_elements(data::json)::json->>'url') url - ,CURRENT_TIMESTAMP(0) etl_tx_dt -from (select * from data_api.api_data -WHERE api_id='a71a43ff-efcd-4427-8dbc-ed2686e4' and is_loaded = '0' order by request_tm desc limit 1) p )p; - -update data_api.api_data -set is_loaded = '1' , - status = '1', - request_tm = current_timestamp(0) -where api_id='a71a43ff-efcd-4427-8dbc-ed2686e4'; -\q \ No newline at end of file