add workflow 荟聚API,dev

TK_Cust/dev/tk_api/获取线索事件_c_minipro_page_view/custom_events_5_feign.py

@@ -0,0 +1,41 @@
+# coding: utf-8
+import requests
+import json
+import psycopg2
+import uuid
+import datetime  
+
+# 获取当前日期  
+current_date = datetime.date.today() 
+previous_date = current_date - datetime.timedelta(days=365)
+formatted_current_date = current_date.strftime("%Y-%m-%dT%H:%M:%SZ")  
+formatted_previous_date = previous_date.strftime("%Y-%m-%dT%H:%M:%SZ") 
+
+print('开始加载数据：custom_events_5:获取线索事件_c_minipro_page_view')
+authUrl='https://api.huiju.cool/v2/oauth2/token?app_id=cl037b184ebccd97c&secret=deaa83fdd8d385eec0d1d04d34282c4ec8c33b60&grant_type=client_credentials'
+ 
+print('开始请求令牌。')
+authRequest=requests.get(authUrl)
+auth=json.loads(authRequest.text)
+print('开始请求数据总数。')
+url='https://api.huiju.cool/v2/customerEvents'
+header={}
+body={'access_token':auth['access_token'],'limit':'2000','event':'c_minipro_page_view','sort':'date','date[le]':formatted_current_date,}
+dataReqL=requests.get(url,headers=header,params=body)
+resL=json.loads(dataReqL.text)
+# print(resL)
+dataList=resL['data']
+total=len(dataList)
+conn = psycopg2.connect(database="dataops_db", user="dbuser_dops", password="MIgTi3jA",
+                                host="172.17.0.8", port="5432")
+print('数据库连接成功')
+dataId=str(uuid.uuid4())
+print('临时id：'+dataId)
+json_object = json.dumps(dataList)
+cur=conn.cursor()
+sql="update data_api.api_data set is_loaded = '1' where api_id = 'a71a43ff-efcd-4427-8dbc-ed2686e4';INSERT INTO data_api.api_data (id,api_id,data,total_num,is_loaded,status,request_tm,execute_tm,remark) values (%s,%s,%s,%s,'0','0',current_timestamp(0),current_timestamp(0),'')"
+cur.execute(sql,[dataId,'a71a43ff-efcd-4427-8dbc-ed2686e4',json_object,total])
+conn.commit()
+cur.close()
+conn.close()
+print('加载数据结束：custom_events_5:获取线索事件_c_minipro_page_view')

TK_Cust/dev/tk_api/获取线索事件_c_minipro_page_view/custom_events_5_load.sql

@@ -0,0 +1,93 @@
+/*******Main Section**************************************************************************/
+\set ON_ERROR_STOP on
+\set AUTOCOMMIT on
+\timing on
+ 
+DELETE FROM data_api.customer_events_c_minipro_page_view;
+
+insert into  data_api.customer_events_c_minipro_page_view ( 
+	 browser 
+	, browser_version 
+	, c_name 
+	, c_source_content 
+	, c_url 
+	, customer_id 
+	, customer_id_str 
+	, date 
+	, event 
+	, external_id 
+	, id 
+	, id_str 
+	, ip 
+	, ip_city 
+	, ip_country 
+	, ip_province 
+	, last_updated 
+	, os 
+	, os_version 
+	, platform 
+	, target_id 
+	, target_name 
+	, url 
+	,etl_tx_dt
+)
+select 
+	 case when trim(both from browser)='' then null else  browser::text  end  browser 
+	, case when trim(both from browser_version)='' then null else  browser_version::text  end  browser_version 
+	, case when trim(both from c_name)='' then null else  c_name::text  end  c_name 
+	, case when trim(both from c_source_content)='' then null else  c_source_content::text  end  c_source_content 
+	, case when trim(both from c_url)='' then null else  c_url::text  end  c_url 
+	, case when trim(both from customer_id)='' then null else  customer_id::text  end  customer_id 
+	, case when trim(both from customer_id_str)='' then null else  customer_id_str::text  end  customer_id_str 
+	, case when trim(both from date)='' then null else  date::text  end  date 
+	, case when trim(both from event)='' then null else  event::text  end  event 
+	, case when trim(both from external_id)='' then null else  external_id::text  end  external_id 
+	, case when trim(both from id)='' then null else  id::text  end  id 
+	, case when trim(both from id_str)='' then null else  id_str::text  end  id_str 
+	, case when trim(both from ip)='' then null else  ip::text  end  ip 
+	, case when trim(both from ip_city)='' then null else  ip_city::text  end  ip_city 
+	, case when trim(both from ip_country)='' then null else  ip_country::text  end  ip_country 
+	, case when trim(both from ip_province)='' then null else  ip_province::text  end  ip_province 
+	, case when trim(both from last_updated)='' then null else  last_updated::text  end  last_updated 
+	, case when trim(both from os)='' then null else  os::text  end  os 
+	, case when trim(both from os_version)='' then null else  os_version::text  end  os_version 
+	, case when trim(both from platform)='' then null else  platform::text  end  platform 
+	, case when trim(both from target_id)='' then null else  target_id::text  end  target_id 
+	, case when trim(both from target_name)='' then null else  target_name::text  end  target_name 
+	, case when trim(both from url)='' then null else  url::text  end  url 
+,etl_tx_dt
+from (
+select 
+	 (json_array_elements(data::json)::json->>'browser')  browser 
+	, (json_array_elements(data::json)::json->>'browserVersion')  browser_version 
+	, (json_array_elements(data::json)::json->>'c_name')  c_name 
+	, (json_array_elements(data::json)::json->>'c_sourceContent')  c_source_content 
+	, (json_array_elements(data::json)::json->>'c_url')  c_url 
+	, (json_array_elements(data::json)::json->>'customerId')  customer_id 
+	, (json_array_elements(data::json)::json->>'customerIdStr')  customer_id_str 
+	, (json_array_elements(data::json)::json->>'date')  date 
+	, (json_array_elements(data::json)::json->>'event')  event 
+	, (json_array_elements(data::json)::json->>'externalId')  external_id 
+	, (json_array_elements(data::json)::json->>'id')  id 
+	, (json_array_elements(data::json)::json->>'idStr')  id_str 
+	, (json_array_elements(data::json)::json->>'ip')  ip 
+	, (json_array_elements(data::json)::json->>'ipCity')  ip_city 
+	, (json_array_elements(data::json)::json->>'ipCountry')  ip_country 
+	, (json_array_elements(data::json)::json->>'ipProvince')  ip_province 
+	, (json_array_elements(data::json)::json->>'lastUpdated')  last_updated 
+	, (json_array_elements(data::json)::json->>'os')  os 
+	, (json_array_elements(data::json)::json->>'osVersion')  os_version 
+	, (json_array_elements(data::json)::json->>'platform')  platform 
+	, (json_array_elements(data::json)::json->>'targetId')  target_id 
+	, (json_array_elements(data::json)::json->>'targetName')  target_name 
+	, (json_array_elements(data::json)::json->>'url')  url 
+	,CURRENT_TIMESTAMP(0)  etl_tx_dt
+from (select * from data_api.api_data 
+WHERE api_id='a71a43ff-efcd-4427-8dbc-ed2686e4' and is_loaded = '0' order by request_tm desc limit 1) p )p;
+
+update  data_api.api_data 
+set is_loaded = '1' ,
+	status = '1',
+	request_tm = current_timestamp(0)
+where api_id='a71a43ff-efcd-4427-8dbc-ed2686e4';
+\q