diff --git a/TK_Cust/dev/tk_api/获取SCRM contact/scrm_contact_feign.py b/TK_Cust/dev/tk_api/获取SCRM contact/scrm_contact_feign.py new file mode 100644 index 0000000..5c8700b --- /dev/null +++ b/TK_Cust/dev/tk_api/获取SCRM contact/scrm_contact_feign.py @@ -0,0 +1,43 @@ +# coding: utf-8 +import requests +import json +import psycopg2 +import uuid +import datetime + +# 获取当前日期 +current_date = datetime.date.today() +previous_date = current_date - datetime.timedelta(days=1) +previous_date2 = current_date - datetime.timedelta(days=3) +formatted_current_date = current_date.strftime("%Y-%m-%dT%H:%M:%SZ") +formatted_previous_date = previous_date.strftime("%Y-%m-%dT%H:%M:%SZ") +formatted_previous_date2 = previous_date2.strftime("%Y-%m-%dT%H:%M:%SZ") + +print('开始加载数据：scrm_contact:获取SCRM contact') +authUrl='https://api.huiju.cool/v2/oauth2/token?app_id=cl037b184ebccd97c&secret=deaa83fdd8d385eec0d1d04d34282c4ec8c33b60&grant_type=client_credentials' + +print('开始请求令牌。') +authRequest=requests.get(authUrl) +auth=json.loads(authRequest.text) +print('开始请求数据总数。') +url='https://api.huiju.cool/v2/customers' +header={} +body={'access_token':auth['access_token'],'sort':'dateJoin','limit':'2000','dateJoin[le]':formatted_previous_date,'dateJoin[ge]':formatted_previous_date2,} +dataReqL=requests.get(url,headers=header,params=body) +resL=json.loads(dataReqL.text) +# print(resL) +dataList=resL['data'] +total=len(dataList) +conn = psycopg2.connect(database="dataops_db", user="dbuser_dops", password="MIgTi3jA", + host="172.17.0.8", port="5432") +print('数据库连接成功') +dataId=str(uuid.uuid4()) +print('临时id：'+dataId) +json_object = json.dumps(dataList) +cur=conn.cursor() +sql="update data_api.api_data set is_loaded = '1' where api_id = 'cab1a761-b8e5-4db4-a769-88db3ec1';INSERT INTO data_api.api_data (id,api_id,data,total_num,is_loaded,status,request_tm,execute_tm,remark) values (%s,%s,%s,%s,'0','0',current_timestamp(0),current_timestamp(0),'')" +cur.execute(sql,[dataId,'cab1a761-b8e5-4db4-a769-88db3ec1',json_object,total]) +conn.commit() +cur.close() +conn.close() +print('加载数据结束：scrm_contact:获取SCRM contact') \ No newline at end of file diff --git a/TK_Cust/dev/tk_api/获取SCRM contact/scrm_contact_load.sql b/TK_Cust/dev/tk_api/获取SCRM contact/scrm_contact_load.sql new file mode 100644 index 0000000..ab7fc02 --- /dev/null +++ b/TK_Cust/dev/tk_api/获取SCRM contact/scrm_contact_load.sql @@ -0,0 +1,420 @@ +/*******Main Section**************************************************************************/ +\set ON_ERROR_STOP on +\set AUTOCOMMIT on +\timing on + +DELETE FROM data_api.scrm_contact; + +insert into data_api.scrm_contact ( + accept_email + , acceptsms + , annual_revenue + , attr_id + , attr1 + , attr10 + , attr11 + , attr12 + , attr13 + , attr14 + , attr15 + , attr16 + , attr17 + , attr18 + , attr19 + , attr2 + , attr20 + , attr21 + , attr22 + , attr23 + , attr24 + , attr25 + , attr26 + , attr27 + , attr28 + , attr3 + , attr4 + , attr5 + , attr6 + , attr7 + , attr8 + , attr9 + , birthday + , campaign_id + , campaign_name + , city + , company + , content_name + , convert_date + , country + , county + , create_from + , create_from_name + , create_method + , date_created + , date_join + , date_member_reg + , display_name + , email + , email_verified + , employee_number + , external_id + , extra_convert_content_name + , extra_convert_date + , extra_convert_source + , extra_convert_source1 + , extra_convert_source2 + , extra_convert_source3 + , extra_convert_source4 + , extra_convert_source5 + , extra_convert_source6 + , extra_convert_source7 + , extra_convert_source8 + , extra_regist_content_name + , extra_regist_date + , extra_regist_source + , extra_regist_source1 + , extra_regist_source2 + , extra_regist_source3 + , extra_regist_source4 + , extra_regist_source5 + , extra_regist_source6 + , extra_regist_source7 + , extra_regist_source8 + , first_assign_date + , gender + , home_address + , id + , id_str + , img + , industry + , is_anonymous + , is_employee + , is_member + , is_referrer + , language + , last_updated + , latest_engage_date + , mobile + , mobile_verified + , name + , nick_name + , not_accept_email + , not_acceptsms + , office_address + , owner_date + , owner_employee_id + , owner_employee_name + , owner_id + , owner_name + , province + , qq + , referrer + , referrer_name + , refund_date + , region + , remark + , source + , source1 + , source2 + , source3 + , source4 + , source5 + , source6 + , source7 + , source8 + , stage + , status + , street + , subscribe_scene + , telephone + , title + , type + , update_method + , utm_medium + , utm_term + , version + , website + , wechat + , wechat_city + , wechat_nick_name + , weibo + ,etl_tx_dt +) +select + case when trim(both from accept_email)='' then null else accept_email::text end accept_email + , case when trim(both from acceptsms)='' then null else acceptsms::text end acceptsms + , case when trim(both from annual_revenue)='' then null else annual_revenue::text end annual_revenue + , case when trim(both from attr_id)='' then null else attr_id::text end attr_id + , case when trim(both from attr1)='' then null else attr1::text end attr1 + , case when trim(both from attr10)='' then null else attr10::text end attr10 + , case when trim(both from attr11)='' then null else attr11::text end attr11 + , case when trim(both from attr12)='' then null else attr12::text end attr12 + , case when trim(both from attr13)='' then null else attr13::text end attr13 + , case when trim(both from attr14)='' then null else attr14::text end attr14 + , case when trim(both from attr15)='' then null else attr15::text end attr15 + , case when trim(both from attr16)='' then null else attr16::text end attr16 + , case when trim(both from attr17)='' then null else attr17::text end attr17 + , case when trim(both from attr18)='' then null else attr18::text end attr18 + , case when trim(both from attr19)='' then null else attr19::text end attr19 + , case when trim(both from attr2)='' then null else attr2::text end attr2 + , case when trim(both from attr20)='' then null else attr20::text end attr20 + , case when trim(both from attr21)='' then null else attr21::text end attr21 + , case when trim(both from attr22)='' then null else attr22::text end attr22 + , case when trim(both from attr23)='' then null else attr23::text end attr23 + , case when trim(both from attr24)='' then null else attr24::text end attr24 + , case when trim(both from attr25)='' then null else attr25::text end attr25 + , case when trim(both from attr26)='' then null else attr26::text end attr26 + , case when trim(both from attr27)='' then null else attr27::text end attr27 + , case when trim(both from attr28)='' then null else attr28::text end attr28 + , case when trim(both from attr3)='' then null else attr3::text end attr3 + , case when trim(both from attr4)='' then null else attr4::text end attr4 + , case when trim(both from attr5)='' then null else attr5::text end attr5 + , case when trim(both from attr6)='' then null else attr6::text end attr6 + , case when trim(both from attr7)='' then null else attr7::text end attr7 + , case when trim(both from attr8)='' then null else attr8::text end attr8 + , case when trim(both from attr9)='' then null else attr9::text end attr9 + , case when trim(both from birthday)='' then null else birthday::text end birthday + , case when trim(both from campaign_id)='' then null else campaign_id::text end campaign_id + , case when trim(both from campaign_name)='' then null else campaign_name::text end campaign_name + , case when trim(both from city)='' then null else city::text end city + , case when trim(both from company)='' then null else company::text end company + , case when trim(both from content_name)='' then null else content_name::text end content_name + , case when trim(both from convert_date)='' then null else convert_date::text end convert_date + , case when trim(both from country)='' then null else country::text end country + , case when trim(both from county)='' then null else county::text end county + , case when trim(both from create_from)='' then null else create_from::text end create_from + , case when trim(both from create_from_name)='' then null else create_from_name::text end create_from_name + , case when trim(both from create_method)='' then null else create_method::text end create_method + , case when trim(both from date_created)='' then null else date_created::text end date_created + , case when trim(both from date_join)='' then null else date_join::text end date_join + , case when trim(both from date_member_reg)='' then null else date_member_reg::text end date_member_reg + , case when trim(both from display_name)='' then null else display_name::text end display_name + , case when trim(both from email)='' then null else email::text end email + , case when trim(both from email_verified)='' then null else email_verified::text end email_verified + , case when trim(both from employee_number)='' then null else employee_number::text end employee_number + , case when trim(both from external_id)='' then null else external_id::text end external_id + , case when trim(both from extra_convert_content_name)='' then null else extra_convert_content_name::text end extra_convert_content_name + , case when trim(both from extra_convert_date)='' then null else extra_convert_date::text end extra_convert_date + , case when trim(both from extra_convert_source)='' then null else extra_convert_source::text end extra_convert_source + , case when trim(both from extra_convert_source1)='' then null else extra_convert_source1::text end extra_convert_source1 + , case when trim(both from extra_convert_source2)='' then null else extra_convert_source2::text end extra_convert_source2 + , case when trim(both from extra_convert_source3)='' then null else extra_convert_source3::text end extra_convert_source3 + , case when trim(both from extra_convert_source4)='' then null else extra_convert_source4::text end extra_convert_source4 + , case when trim(both from extra_convert_source5)='' then null else extra_convert_source5::text end extra_convert_source5 + , case when trim(both from extra_convert_source6)='' then null else extra_convert_source6::text end extra_convert_source6 + , case when trim(both from extra_convert_source7)='' then null else extra_convert_source7::text end extra_convert_source7 + , case when trim(both from extra_convert_source8)='' then null else extra_convert_source8::text end extra_convert_source8 + , case when trim(both from extra_regist_content_name)='' then null else extra_regist_content_name::text end extra_regist_content_name + , case when trim(both from extra_regist_date)='' then null else extra_regist_date::text end extra_regist_date + , case when trim(both from extra_regist_source)='' then null else extra_regist_source::text end extra_regist_source + , case when trim(both from extra_regist_source1)='' then null else extra_regist_source1::text end extra_regist_source1 + , case when trim(both from extra_regist_source2)='' then null else extra_regist_source2::text end extra_regist_source2 + , case when trim(both from extra_regist_source3)='' then null else extra_regist_source3::text end extra_regist_source3 + , case when trim(both from extra_regist_source4)='' then null else extra_regist_source4::text end extra_regist_source4 + , case when trim(both from extra_regist_source5)='' then null else extra_regist_source5::text end extra_regist_source5 + , case when trim(both from extra_regist_source6)='' then null else extra_regist_source6::text end extra_regist_source6 + , case when trim(both from extra_regist_source7)='' then null else extra_regist_source7::text end extra_regist_source7 + , case when trim(both from extra_regist_source8)='' then null else extra_regist_source8::text end extra_regist_source8 + , case when trim(both from first_assign_date)='' then null else first_assign_date::text end first_assign_date + , case when trim(both from gender)='' then null else gender::text end gender + , case when trim(both from home_address)='' then null else home_address::text end home_address + , case when trim(both from id)='' then null else id::text end id + , case when trim(both from id_str)='' then null else id_str::text end id_str + , case when trim(both from img)='' then null else img::text end img + , case when trim(both from industry)='' then null else industry::text end industry + , case when trim(both from is_anonymous)='' then null else is_anonymous::text end is_anonymous + , case when trim(both from is_employee)='' then null else is_employee::text end is_employee + , case when trim(both from is_member)='' then null else is_member::text end is_member + , case when trim(both from is_referrer)='' then null else is_referrer::text end is_referrer + , case when trim(both from language)='' then null else language::text end language + , case when trim(both from last_updated)='' then null else last_updated::text end last_updated + , case when trim(both from latest_engage_date)='' then null else latest_engage_date::text end latest_engage_date + , case when trim(both from mobile)='' then null else mobile::text end mobile + , case when trim(both from mobile_verified)='' then null else mobile_verified::text end mobile_verified + , case when trim(both from name)='' then null else name::text end name + , case when trim(both from nick_name)='' then null else nick_name::text end nick_name + , case when trim(both from not_accept_email)='' then null else not_accept_email::text end not_accept_email + , case when trim(both from not_acceptsms)='' then null else not_acceptsms::text end not_acceptsms + , case when trim(both from office_address)='' then null else office_address::text end office_address + , case when trim(both from owner_date)='' then null else owner_date::text end owner_date + , case when trim(both from owner_employee_id)='' then null else owner_employee_id::text end owner_employee_id + , case when trim(both from owner_employee_name)='' then null else owner_employee_name::text end owner_employee_name + , case when trim(both from owner_id)='' then null else owner_id::text end owner_id + , case when trim(both from owner_name)='' then null else owner_name::text end owner_name + , case when trim(both from province)='' then null else province::text end province + , case when trim(both from qq)='' then null else qq::text end qq + , case when trim(both from referrer)='' then null else referrer::text end referrer + , case when trim(both from referrer_name)='' then null else referrer_name::text end referrer_name + , case when trim(both from refund_date)='' then null else refund_date::text end refund_date + , case when trim(both from region)='' then null else region::text end region + , case when trim(both from remark)='' then null else remark::text end remark + , case when trim(both from source)='' then null else source::text end source + , case when trim(both from source1)='' then null else source1::text end source1 + , case when trim(both from source2)='' then null else source2::text end source2 + , case when trim(both from source3)='' then null else source3::text end source3 + , case when trim(both from source4)='' then null else source4::text end source4 + , case when trim(both from source5)='' then null else source5::text end source5 + , case when trim(both from source6)='' then null else source6::text end source6 + , case when trim(both from source7)='' then null else source7::text end source7 + , case when trim(both from source8)='' then null else source8::text end source8 + , case when trim(both from stage)='' then null else stage::text end stage + , case when trim(both from status)='' then null else status::text end status + , case when trim(both from street)='' then null else street::text end street + , case when trim(both from subscribe_scene)='' then null else subscribe_scene::text end subscribe_scene + , case when trim(both from telephone)='' then null else telephone::text end telephone + , case when trim(both from title)='' then null else title::text end title + , case when trim(both from type)='' then null else type::text end type + , case when trim(both from update_method)='' then null else update_method::text end update_method + , case when trim(both from utm_medium)='' then null else utm_medium::text end utm_medium + , case when trim(both from utm_term)='' then null else utm_term::text end utm_term + , case when trim(both from version)='' then null else version::text end version + , case when trim(both from website)='' then null else website::text end website + , case when trim(both from wechat)='' then null else wechat::text end wechat + , case when trim(both from wechat_city)='' then null else wechat_city::text end wechat_city + , case when trim(both from wechat_nick_name)='' then null else wechat_nick_name::text end wechat_nick_name + , case when trim(both from weibo)='' then null else weibo::text end weibo +,etl_tx_dt +from ( +select + (json_array_elements(data::json)::json->>'acceptEmail') accept_email + , (json_array_elements(data::json)::json->>'acceptsms') acceptsms + , (json_array_elements(data::json)::json->>'annualRevenue') annual_revenue + , (json_array_elements(data::json)::json->>'attrId') attr_id + , (json_array_elements(data::json)::json->>'attr1') attr1 + , (json_array_elements(data::json)::json->>'attr10') attr10 + , (json_array_elements(data::json)::json->>'attr11') attr11 + , (json_array_elements(data::json)::json->>'attr12') attr12 + , (json_array_elements(data::json)::json->>'attr13') attr13 + , (json_array_elements(data::json)::json->>'attr14') attr14 + , (json_array_elements(data::json)::json->>'attr15') attr15 + , (json_array_elements(data::json)::json->>'attr16') attr16 + , (json_array_elements(data::json)::json->>'attr17') attr17 + , (json_array_elements(data::json)::json->>'attr18') attr18 + , (json_array_elements(data::json)::json->>'attr19') attr19 + , (json_array_elements(data::json)::json->>'attr2') attr2 + , (json_array_elements(data::json)::json->>'attr20') attr20 + , (json_array_elements(data::json)::json->>'attr21') attr21 + , (json_array_elements(data::json)::json->>'attr22') attr22 + , (json_array_elements(data::json)::json->>'attr23') attr23 + , (json_array_elements(data::json)::json->>'attr24') attr24 + , (json_array_elements(data::json)::json->>'attr25') attr25 + , (json_array_elements(data::json)::json->>'attr26') attr26 + , (json_array_elements(data::json)::json->>'attr27') attr27 + , (json_array_elements(data::json)::json->>'attr28') attr28 + , (json_array_elements(data::json)::json->>'attr3') attr3 + , (json_array_elements(data::json)::json->>'attr4') attr4 + , (json_array_elements(data::json)::json->>'attr5') attr5 + , (json_array_elements(data::json)::json->>'attr6') attr6 + , (json_array_elements(data::json)::json->>'attr7') attr7 + , (json_array_elements(data::json)::json->>'attr8') attr8 + , (json_array_elements(data::json)::json->>'attr9') attr9 + , (json_array_elements(data::json)::json->>'birthday') birthday + , (json_array_elements(data::json)::json->>'campaignId') campaign_id + , (json_array_elements(data::json)::json->>'campaignName') campaign_name + , (json_array_elements(data::json)::json->>'city') city + , (json_array_elements(data::json)::json->>'company') company + , (json_array_elements(data::json)::json->>'contentName') content_name + , (json_array_elements(data::json)::json->>'convertDate') convert_date + , (json_array_elements(data::json)::json->>'country') country + , (json_array_elements(data::json)::json->>'county') county + , (json_array_elements(data::json)::json->>'createFrom') create_from + , (json_array_elements(data::json)::json->>'createFromName') create_from_name + , (json_array_elements(data::json)::json->>'createMethod') create_method + , (json_array_elements(data::json)::json->>'dateCreated') date_created + , (json_array_elements(data::json)::json->>'dateJoin') date_join + , (json_array_elements(data::json)::json->>'dateMemberReg') date_member_reg + , (json_array_elements(data::json)::json->>'displayName') display_name + , (json_array_elements(data::json)::json->>'email') email + , (json_array_elements(data::json)::json->>'emailVerified') email_verified + , (json_array_elements(data::json)::json->>'employeeNumber') employee_number + , (json_array_elements(data::json)::json->>'externalId') external_id + , (json_array_elements(data::json)::json->>'extraConvertContentName') extra_convert_content_name + , (json_array_elements(data::json)::json->>'extraConvertDate') extra_convert_date + , (json_array_elements(data::json)::json->>'extraConvertSource') extra_convert_source + , (json_array_elements(data::json)::json->>'extraConvertSource1') extra_convert_source1 + , (json_array_elements(data::json)::json->>'extraConvertSource2') extra_convert_source2 + , (json_array_elements(data::json)::json->>'extraConvertSource3') extra_convert_source3 + , (json_array_elements(data::json)::json->>'extraConvertSource4') extra_convert_source4 + , (json_array_elements(data::json)::json->>'extraConvertSource5') extra_convert_source5 + , (json_array_elements(data::json)::json->>'extraConvertSource6') extra_convert_source6 + , (json_array_elements(data::json)::json->>'extraConvertSource7') extra_convert_source7 + , (json_array_elements(data::json)::json->>'extraConvertSource8') extra_convert_source8 + , (json_array_elements(data::json)::json->>'extraRegistContentName') extra_regist_content_name + , (json_array_elements(data::json)::json->>'extraRegistDate') extra_regist_date + , (json_array_elements(data::json)::json->>'extraRegistSource') extra_regist_source + , (json_array_elements(data::json)::json->>'extraRegistSource1') extra_regist_source1 + , (json_array_elements(data::json)::json->>'extraRegistSource2') extra_regist_source2 + , (json_array_elements(data::json)::json->>'extraRegistSource3') extra_regist_source3 + , (json_array_elements(data::json)::json->>'extraRegistSource4') extra_regist_source4 + , (json_array_elements(data::json)::json->>'extraRegistSource5') extra_regist_source5 + , (json_array_elements(data::json)::json->>'extraRegistSource6') extra_regist_source6 + , (json_array_elements(data::json)::json->>'extraRegistSource7') extra_regist_source7 + , (json_array_elements(data::json)::json->>'extraRegistSource8') extra_regist_source8 + , (json_array_elements(data::json)::json->>'firstAssignDate') first_assign_date + , (json_array_elements(data::json)::json->>'gender') gender + , (json_array_elements(data::json)::json->>'homeAddress') home_address + , (json_array_elements(data::json)::json->>'id') id + , (json_array_elements(data::json)::json->>'idStr') id_str + , (json_array_elements(data::json)::json->>'img') img + , (json_array_elements(data::json)::json->>'industry') industry + , (json_array_elements(data::json)::json->>'isAnonymous') is_anonymous + , (json_array_elements(data::json)::json->>'isEmployee') is_employee + , (json_array_elements(data::json)::json->>'isMember') is_member + , (json_array_elements(data::json)::json->>'isReferrer') is_referrer + , (json_array_elements(data::json)::json->>'language') language + , (json_array_elements(data::json)::json->>'lastUpdated') last_updated + , (json_array_elements(data::json)::json->>'latestEngageDate') latest_engage_date + , (json_array_elements(data::json)::json->>'mobile') mobile + , (json_array_elements(data::json)::json->>'mobileVerified') mobile_verified + , (json_array_elements(data::json)::json->>'name') name + , (json_array_elements(data::json)::json->>'nickName') nick_name + , (json_array_elements(data::json)::json->>'notAcceptEmail') not_accept_email + , (json_array_elements(data::json)::json->>'notAcceptsms') not_acceptsms + , (json_array_elements(data::json)::json->>'officeAddress') office_address + , (json_array_elements(data::json)::json->>'ownerDate') owner_date + , (json_array_elements(data::json)::json->>'ownerEmployeeId') owner_employee_id + , (json_array_elements(data::json)::json->>'ownerEmployeeName') owner_employee_name + , (json_array_elements(data::json)::json->>'ownerId') owner_id + , (json_array_elements(data::json)::json->>'ownerName') owner_name + , (json_array_elements(data::json)::json->>'province') province + , (json_array_elements(data::json)::json->>'qq') qq + , (json_array_elements(data::json)::json->>'referrer') referrer + , (json_array_elements(data::json)::json->>'referrerName') referrer_name + , (json_array_elements(data::json)::json->>'refundDate') refund_date + , (json_array_elements(data::json)::json->>'region') region + , (json_array_elements(data::json)::json->>'remark') remark + , (json_array_elements(data::json)::json->>'source') source + , (json_array_elements(data::json)::json->>'source1') source1 + , (json_array_elements(data::json)::json->>'source2') source2 + , (json_array_elements(data::json)::json->>'source3') source3 + , (json_array_elements(data::json)::json->>'source4') source4 + , (json_array_elements(data::json)::json->>'source5') source5 + , (json_array_elements(data::json)::json->>'source6') source6 + , (json_array_elements(data::json)::json->>'source7') source7 + , (json_array_elements(data::json)::json->>'source8') source8 + , (json_array_elements(data::json)::json->>'stage') stage + , (json_array_elements(data::json)::json->>'status') status + , (json_array_elements(data::json)::json->>'street') street + , (json_array_elements(data::json)::json->>'subscribeScene') subscribe_scene + , (json_array_elements(data::json)::json->>'telephone') telephone + , (json_array_elements(data::json)::json->>'title') title + , (json_array_elements(data::json)::json->>'type') type + , (json_array_elements(data::json)::json->>'updateMethod') update_method + , (json_array_elements(data::json)::json->>'utmMedium') utm_medium + , (json_array_elements(data::json)::json->>'utmTerm') utm_term + , (json_array_elements(data::json)::json->>'version') version + , (json_array_elements(data::json)::json->>'website') website + , (json_array_elements(data::json)::json->>'wechat') wechat + , (json_array_elements(data::json)::json->>'wechatCity') wechat_city + , (json_array_elements(data::json)::json->>'wechatNickName') wechat_nick_name + , (json_array_elements(data::json)::json->>'weibo') weibo + ,CURRENT_TIMESTAMP(0) etl_tx_dt +from (select * from data_api.api_data +WHERE api_id='cab1a761-b8e5-4db4-a769-88db3ec1' and is_loaded = '0' order by request_tm desc limit 1) p )p; + +update data_api.api_data +set is_loaded = '1' , + status = '1', + request_tm = current_timestamp(0) +where api_id='cab1a761-b8e5-4db4-a769-88db3ec1'; +\q \ No newline at end of file