diff --git a/TK_Cust/dev/tk_api/合并SCRM CONTACT/custom_events_update_feign.py b/TK_Cust/dev/tk_api/合并SCRM CONTACT/custom_events_update_feign.py new file mode 100644 index 0000000..bbfff96 --- /dev/null +++ b/TK_Cust/dev/tk_api/合并SCRM CONTACT/custom_events_update_feign.py @@ -0,0 +1,43 @@ +# coding: utf-8 +import requests +import json +import psycopg2 +import uuid +import datetime + +# 获取当前日期 +current_date = datetime.date.today() +previous_date = current_date - datetime.timedelta(days=2) + +formatted_current_date = current_date.strftime("%Y-%m-%dT%H:%M:%SZ") +formatted_previous_date = previous_date.strftime("%Y-%m-%dT%H:%M:%SZ") + + +print('开始加载数据：custom_events_update:合并SCRM CONTACT') +authUrl='https://api.huiju.cool/v2/oauth2/token?app_id=cl037b184ebccd97c&secret=deaa83fdd8d385eec0d1d04d34282c4ec8c33b60&grant_type=client_credentials' + +print('开始请求令牌。') +authRequest=requests.get(authUrl) +auth=json.loads(authRequest.text) +print('开始请求数据总数。') +url='https://api.huiju.cool/v2/customerEvents' +header={} +body={'access_token':auth['access_token'],'sort':'lastUpdated','lastUpdated[ge]':formatted_current_date,'lastUpdated[le]':formatted_previous_date,'limit':'2000',} +dataReqL=requests.get(url,headers=header,params=body) +resL=json.loads(dataReqL.text) +# print(resL) +dataList=resL['data'] +total=len(dataList) +conn = psycopg2.connect(database="dataops_db", user="dbuser_dops", password="MIgTi3jA", + host="172.17.0.8", port="5432") +print('数据库连接成功') +dataId=str(uuid.uuid4()) +print('临时id：'+dataId) +json_object = json.dumps(dataList) +cur=conn.cursor() +sql="update data_api.api_data set is_loaded = '1' where api_id = 'ff5c7bf6-0d18-4201-9501-8fdd6152';INSERT INTO data_api.api_data (id,api_id,data,total_num,is_loaded,status,request_tm,execute_tm,remark) values (%s,%s,%s,%s,'0','0',current_timestamp(0),current_timestamp(0),'')" +cur.execute(sql,[dataId,'ff5c7bf6-0d18-4201-9501-8fdd6152',json_object,total]) +conn.commit() +cur.close() +conn.close() +print('加载数据结束：custom_events_update:合并SCRM CONTACT') \ No newline at end of file diff --git a/TK_Cust/dev/tk_api/合并SCRM CONTACT/custom_events_update_load.sql b/TK_Cust/dev/tk_api/合并SCRM CONTACT/custom_events_update_load.sql new file mode 100644 index 0000000..1217982 --- /dev/null +++ b/TK_Cust/dev/tk_api/合并SCRM CONTACT/custom_events_update_load.sql @@ -0,0 +1,105 @@ +/*******Main Section**************************************************************************/ +\set ON_ERROR_STOP on +\set AUTOCOMMIT on +\timing on + +DELETE FROM data_api.customer_events_lastupdated; + +insert into data_api.customer_events_lastupdated ( + date + , screen_width + , target_name + , ip_city + , ip_country + , platform + , last_updated + , page_type + , os_version + , browser + , browser_version + , customer_id + , id + , tag + , event + , id_str + , os + , target_id + , customer_id_str + , ip + , screen_height + , external_id + , ip_county + , page_id + , url + , ip_province + , device + ,etl_tx_dt +) +select + case when trim(both from date)='' then null else date::text end date + , case when trim(both from screen_width)='' then null else screen_width::text end screen_width + , case when trim(both from target_name)='' then null else target_name::text end target_name + , case when trim(both from ip_city)='' then null else ip_city::text end ip_city + , case when trim(both from ip_country)='' then null else ip_country::text end ip_country + , case when trim(both from platform)='' then null else platform::text end platform + , case when trim(both from last_updated)='' then null else last_updated::text end last_updated + , case when trim(both from page_type)='' then null else page_type::text end page_type + , case when trim(both from os_version)='' then null else os_version::text end os_version + , case when trim(both from browser)='' then null else browser::text end browser + , case when trim(both from browser_version)='' then null else browser_version::text end browser_version + , case when trim(both from customer_id)='' then null else customer_id::text end customer_id + , case when trim(both from id)='' then null else id::text end id + , case when trim(both from tag)='' then null else tag::text end tag + , case when trim(both from event)='' then null else event::text end event + , case when trim(both from id_str)='' then null else id_str::text end id_str + , case when trim(both from os)='' then null else os::text end os + , case when trim(both from target_id)='' then null else target_id::text end target_id + , case when trim(both from customer_id_str)='' then null else customer_id_str::text end customer_id_str + , case when trim(both from ip)='' then null else ip::text end ip + , case when trim(both from screen_height)='' then null else screen_height::text end screen_height + , case when trim(both from external_id)='' then null else external_id::text end external_id + , case when trim(both from ip_county)='' then null else ip_county::text end ip_county + , case when trim(both from page_id)='' then null else page_id::text end page_id + , case when trim(both from url)='' then null else url::text end url + , case when trim(both from ip_province)='' then null else ip_province::text end ip_province + , case when trim(both from device)='' then null else device::text end device +,etl_tx_dt +from ( +select + (json_array_elements(data::json)::json->>'date') date + , (json_array_elements(data::json)::json->>'screenWidth') screen_width + , (json_array_elements(data::json)::json->>'targetName') target_name + , (json_array_elements(data::json)::json->>'ipCity') ip_city + , (json_array_elements(data::json)::json->>'ipCountry') ip_country + , (json_array_elements(data::json)::json->>'platform') platform + , (json_array_elements(data::json)::json->>'lastUpdated') last_updated + , (json_array_elements(data::json)::json->>'pageType') page_type + , (json_array_elements(data::json)::json->>'osVersion') os_version + , (json_array_elements(data::json)::json->>'browser') browser + , (json_array_elements(data::json)::json->>'browserVersion') browser_version + , (json_array_elements(data::json)::json->>'customerId') customer_id + , (json_array_elements(data::json)::json->>'id') id + , (json_array_elements(data::json)::json->>'tag') tag + , (json_array_elements(data::json)::json->>'event') event + , (json_array_elements(data::json)::json->>'idStr') id_str + , (json_array_elements(data::json)::json->>'os') os + , (json_array_elements(data::json)::json->>'targetId') target_id + , (json_array_elements(data::json)::json->>'customerIdStr') customer_id_str + , (json_array_elements(data::json)::json->>'ip') ip + , (json_array_elements(data::json)::json->>'screenHeight') screen_height + , (json_array_elements(data::json)::json->>'externalId') external_id + , (json_array_elements(data::json)::json->>'ipCounty') ip_county + , (json_array_elements(data::json)::json->>'pageId') page_id + , (json_array_elements(data::json)::json->>'url') url + , (json_array_elements(data::json)::json->>'ipProvince') ip_province + , (json_array_elements(data::json)::json->>'device') device + ,CURRENT_TIMESTAMP(0) etl_tx_dt +from (select * from data_api.api_data +WHERE api_id='ff5c7bf6-0d18-4201-9501-8fdd6152' and is_loaded = '0' order by request_tm desc limit 1) p )p; + +update data_api.api_data +set is_loaded = '1' , + status = '1', + request_tm = current_timestamp(0) +where api_id='ff5c7bf6-0d18-4201-9501-8fdd6152'; +\q \ No newline at end of file