add workflow 天润Smart-ccc会话数据,dev
This commit is contained in:
root
parent
5cf4f99821
commit
0f7c25c650
|
|
@ -0,0 +1,78 @@
|
|||
/*******Main Section**************************************************************************/
|
||||
\set ON_ERROR_STOP on
|
||||
\set AUTOCOMMIT on
|
||||
\timing on
|
||||
delete from p10_sa.S98_S_tr_chat_messages_new
|
||||
;
|
||||
insert into p10_sa.S98_S_tr_chat_messages_new
|
||||
( main_unique_id
|
||||
, detail_unique_id
|
||||
, visitor_id
|
||||
, visitor_name
|
||||
, content
|
||||
, sender_name
|
||||
, sender
|
||||
, create_time
|
||||
, file_key
|
||||
, file_name
|
||||
, file_url
|
||||
, send_type_name
|
||||
, message_type_name
|
||||
, bot_provider_name
|
||||
, etl_tx_dt )
|
||||
select
|
||||
main_unique_id
|
||||
, detail_unique_id
|
||||
, visitor_id
|
||||
, visitor_name
|
||||
, content
|
||||
, sender_name
|
||||
, sender
|
||||
, create_time
|
||||
, file_key
|
||||
, file_name
|
||||
, file_url
|
||||
, send_type_name
|
||||
, message_type_name
|
||||
, bot_provider_name
|
||||
, etl_tx_dt
|
||||
from p00_tal.S98_S_tr_chat_messages_new
|
||||
;
|
||||
delete from p12_sfull.S98_S_tr_chat_messages_new
|
||||
;
|
||||
;
|
||||
insert into p12_sfull.S98_S_tr_chat_messages_new
|
||||
( main_unique_id
|
||||
, detail_unique_id
|
||||
, visitor_id
|
||||
, visitor_name
|
||||
, content
|
||||
, sender_name
|
||||
, sender
|
||||
, create_time
|
||||
, file_key
|
||||
, file_name
|
||||
, file_url
|
||||
, send_type_name
|
||||
, message_type_name
|
||||
, bot_provider_name
|
||||
, etl_tx_dt )
|
||||
select
|
||||
main_unique_id
|
||||
, detail_unique_id
|
||||
, visitor_id
|
||||
, visitor_name
|
||||
, content
|
||||
, sender_name
|
||||
, sender
|
||||
, create_time
|
||||
, file_key
|
||||
, file_name
|
||||
, file_url
|
||||
, send_type_name
|
||||
, message_type_name
|
||||
, bot_provider_name
|
||||
, etl_tx_dt
|
||||
from p10_sa.S98_S_tr_chat_messages_new
|
||||
;
|
||||
\q
|
||||
|
|
@ -0,0 +1,26 @@
|
|||
|
||||
CREATE FOREIGN TABLE if not exists p00_tal.S98_S_tr_chat_messages_new (
|
||||
main_unique_id TEXT
|
||||
, detail_unique_id TEXT
|
||||
, visitor_id TEXT
|
||||
, visitor_name TEXT
|
||||
, content TEXT
|
||||
, sender_name TEXT
|
||||
, sender TEXT
|
||||
, create_time TEXT
|
||||
, file_key TEXT
|
||||
, file_name TEXT
|
||||
, file_url TEXT
|
||||
, send_type_name TEXT
|
||||
, message_type_name TEXT
|
||||
, bot_provider_name TEXT
|
||||
, etl_tx_dt TIMESTAMP
|
||||
)
|
||||
|
||||
|
||||
SERVER pgsql_server_S98_S OPTIONS(schema_name 'data_api', table_name 'tr_chat_messages_new' );
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
|
@ -0,0 +1,75 @@
|
|||
|
||||
create table if not exists p10_sa.S98_S_tr_chat_messages_new (
|
||||
main_unique_id TEXT
|
||||
, detail_unique_id TEXT
|
||||
, visitor_id TEXT
|
||||
, visitor_name TEXT
|
||||
, content TEXT
|
||||
, sender_name TEXT
|
||||
, sender TEXT
|
||||
, create_time TEXT
|
||||
, file_key TEXT
|
||||
, file_name TEXT
|
||||
, file_url TEXT
|
||||
, send_type_name TEXT
|
||||
, message_type_name TEXT
|
||||
, bot_provider_name TEXT
|
||||
, etl_tx_dt TIMESTAMP
|
||||
) ;
|
||||
|
||||
COMMENT ON COLUMN p10_sa.S98_S_tr_chat_messages_new.main_unique_id IS '会话ID';
|
||||
COMMENT ON COLUMN p10_sa.S98_S_tr_chat_messages_new.detail_unique_id IS '从会话ID';
|
||||
COMMENT ON COLUMN p10_sa.S98_S_tr_chat_messages_new.visitor_id IS '访客ID';
|
||||
COMMENT ON COLUMN p10_sa.S98_S_tr_chat_messages_new.visitor_name IS '访客姓名';
|
||||
COMMENT ON COLUMN p10_sa.S98_S_tr_chat_messages_new.content IS '消息内容';
|
||||
COMMENT ON COLUMN p10_sa.S98_S_tr_chat_messages_new.sender_name IS '发送者名称';
|
||||
COMMENT ON COLUMN p10_sa.S98_S_tr_chat_messages_new.sender IS '发送者';
|
||||
COMMENT ON COLUMN p10_sa.S98_S_tr_chat_messages_new.create_time IS '消息创建时间';
|
||||
COMMENT ON COLUMN p10_sa.S98_S_tr_chat_messages_new.file_key IS '文件key';
|
||||
COMMENT ON COLUMN p10_sa.S98_S_tr_chat_messages_new.file_name IS '文件名称';
|
||||
COMMENT ON COLUMN p10_sa.S98_S_tr_chat_messages_new.file_url IS '文件链接';
|
||||
COMMENT ON COLUMN p10_sa.S98_S_tr_chat_messages_new.send_type_name IS '发送者类型名称';
|
||||
COMMENT ON COLUMN p10_sa.S98_S_tr_chat_messages_new.message_type_name IS '消息类型名称';
|
||||
COMMENT ON COLUMN p10_sa.S98_S_tr_chat_messages_new.bot_provider_name IS '机器人提供者名称';
|
||||
COMMENT ON COLUMN p10_sa.S98_S_tr_chat_messages_new.etl_tx_dt IS '';
|
||||
|
||||
COMMENT ON TABLE p10_sa.S98_S_tr_chat_messages_new IS '';
|
||||
|
||||
|
||||
|
||||
create table if not exists p12_sfull.S98_S_tr_chat_messages_new (
|
||||
main_unique_id TEXT
|
||||
, detail_unique_id TEXT
|
||||
, visitor_id TEXT
|
||||
, visitor_name TEXT
|
||||
, content TEXT
|
||||
, sender_name TEXT
|
||||
, sender TEXT
|
||||
, create_time TEXT
|
||||
, file_key TEXT
|
||||
, file_name TEXT
|
||||
, file_url TEXT
|
||||
, send_type_name TEXT
|
||||
, message_type_name TEXT
|
||||
, bot_provider_name TEXT
|
||||
, etl_tx_dt TIMESTAMP
|
||||
) ;
|
||||
|
||||
COMMENT ON COLUMN p12_sfull.S98_S_tr_chat_messages_new.main_unique_id IS '会话ID';
|
||||
COMMENT ON COLUMN p12_sfull.S98_S_tr_chat_messages_new.detail_unique_id IS '从会话ID';
|
||||
COMMENT ON COLUMN p12_sfull.S98_S_tr_chat_messages_new.visitor_id IS '访客ID';
|
||||
COMMENT ON COLUMN p12_sfull.S98_S_tr_chat_messages_new.visitor_name IS '访客姓名';
|
||||
COMMENT ON COLUMN p12_sfull.S98_S_tr_chat_messages_new.content IS '消息内容';
|
||||
COMMENT ON COLUMN p12_sfull.S98_S_tr_chat_messages_new.sender_name IS '发送者名称';
|
||||
COMMENT ON COLUMN p12_sfull.S98_S_tr_chat_messages_new.sender IS '发送者';
|
||||
COMMENT ON COLUMN p12_sfull.S98_S_tr_chat_messages_new.create_time IS '消息创建时间';
|
||||
COMMENT ON COLUMN p12_sfull.S98_S_tr_chat_messages_new.file_key IS '文件key';
|
||||
COMMENT ON COLUMN p12_sfull.S98_S_tr_chat_messages_new.file_name IS '文件名称';
|
||||
COMMENT ON COLUMN p12_sfull.S98_S_tr_chat_messages_new.file_url IS '文件链接';
|
||||
COMMENT ON COLUMN p12_sfull.S98_S_tr_chat_messages_new.send_type_name IS '发送者类型名称';
|
||||
COMMENT ON COLUMN p12_sfull.S98_S_tr_chat_messages_new.message_type_name IS '消息类型名称';
|
||||
COMMENT ON COLUMN p12_sfull.S98_S_tr_chat_messages_new.bot_provider_name IS '机器人提供者名称';
|
||||
COMMENT ON COLUMN p12_sfull.S98_S_tr_chat_messages_new.etl_tx_dt IS '';
|
||||
|
||||
COMMENT ON TABLE p12_sfull.S98_S_tr_chat_messages_new IS '';
|
||||
|
||||
|
|
@ -0,0 +1,300 @@
|
|||
# coding: utf-8
|
||||
import requests
|
||||
import json
|
||||
import psycopg2
|
||||
import uuid
|
||||
import datetime
|
||||
import time
|
||||
import hashlib
|
||||
import time
|
||||
import hmac
|
||||
import base64
|
||||
import urllib.parse
|
||||
import hashlib
|
||||
from collections import OrderedDict
|
||||
from urllib.parse import quote_plus
|
||||
|
||||
#全局变量，便于参数使用的预设值
|
||||
current_date = datetime.date.today() # 获取当前日期
|
||||
previous_date = current_date - datetime.timedelta(days=1) # 获取前一天日期
|
||||
formatted_current_date = current_date.strftime("%Y-%m-%dT%H:%M:%SZ") # 获取当前日期 - 标准化
|
||||
formatted_previous_date = previous_date.strftime("%Y-%m-%dT%H:%M:%SZ") # 获取前一天日期 - 标准化
|
||||
timestamp = time.time() # 为Unix time，即从"1970-01-01 00:00:00"至今的秒数；
|
||||
sign_version = 'v2' # 签名版本号，固定值v2
|
||||
nonce = str(uuid.uuid4())
|
||||
current_time_utc =( datetime.datetime.now() - datetime.timedelta(hours=8)).strftime("%Y-%m-%dT%H:%M:%SZ")
|
||||
|
||||
formatted2_current_date = current_date.strftime("%Y-%m-%d %H:%M:%S") # 获取当前日期 - 标准化
|
||||
|
||||
# database="dataops_db"
|
||||
# user="dbuser_dba"
|
||||
# password="EmBRxnmmjnE3"
|
||||
# host="124.221.232.219"
|
||||
# port="5432"
|
||||
|
||||
database="dataops_db"
|
||||
user="dbuser_dops"
|
||||
password="MIgTi3jA"
|
||||
host="172.17.0.8"
|
||||
port="5432"
|
||||
|
||||
def formatted2_previous_date(d):
|
||||
if d==0:
|
||||
return datetime.date.today().strftime("%Y%m%d")
|
||||
# 减去一个小时，得到前一个小时的开始时间
|
||||
start_of_previous_date = datetime.date.today() - datetime.timedelta(days=d)
|
||||
return start_of_previous_date.strftime("%Y%m%d")
|
||||
|
||||
|
||||
def formatted2_previous_hour(h):
|
||||
if h==0:
|
||||
return datetime.datetime.now().strftime("%Y-%m-%d %H:%M")
|
||||
start_of_current_hour = datetime.datetime.now().replace(minute=0, second=0, microsecond=0)
|
||||
# 减去一个小时，得到前一个小时的开始时间
|
||||
start_of_previous_hour = start_of_current_hour - datetime.timedelta(hours=h)
|
||||
return start_of_previous_hour.strftime("%Y-%m-%d %H:%M")
|
||||
|
||||
def previous_hour_timestamp(h):
|
||||
if h==0:
|
||||
return int(time.time())
|
||||
start_of_current_hour = datetime.datetime.now().replace(minute=0, second=0, microsecond=0)
|
||||
# 减去一个小时，得到前一个小时的开始时间
|
||||
start_of_previous_hour = start_of_current_hour - datetime.timedelta(hours=h)
|
||||
return int(start_of_previous_hour.timestamp())
|
||||
|
||||
#计算签名
|
||||
def generate_signature(str, private_key):
|
||||
signature = hmac.new(private_key.encode(), (str).encode(), hashlib.sha1)
|
||||
signature_b64 = base64.b64encode(signature.digest()).decode()
|
||||
return signature_b64
|
||||
|
||||
#构建查询链接
|
||||
def build_query_string(params):
|
||||
# 使用OrderedDict来保持排序
|
||||
sorted_params = OrderedDict(sorted(params.items()))
|
||||
|
||||
# 拼接属性名和属性值，并使用&连接
|
||||
query_string = '&'.join('{}={}'.format(
|
||||
urllib.parse.quote_plus(k),
|
||||
urllib.parse.quote_plus(str(v))
|
||||
) for k, v in sorted_params.items())
|
||||
return query_string
|
||||
|
||||
""" 查询record 列表--新 """
|
||||
def request_data_signature_post(pageIndex=1):
|
||||
print(f'开始请求工单列表数据：{formatted2_previous_hour(0)}')
|
||||
url='https://api-bj.clink.cn/livechat/record_session_page'
|
||||
header={'Content-Type':'application/x-www-form-urlencoded'}
|
||||
param={'AccessKeyId':'b17759d3a36fba9a2cf522fbf4cbf177','Timestamp':current_time_utc,'Expires':86400,'endTime':formatted2_previous_hour(0),'startTime':formatted2_previous_hour(1),'pageSize':1000,'pageIndex':pageIndex}
|
||||
print(f'param: {param}')
|
||||
url_path = build_query_string(param)
|
||||
url_param = build_query_string(param)
|
||||
print(f'url_param: {url_param}')
|
||||
url_param = f'POSTapi-bj.clink.cn/livechat/record_session_page?{url_param}'
|
||||
print(f'待计算字符串: {url_param}')
|
||||
signature= generate_signature(url_param,'5g027B6w06630Y5240c1')
|
||||
print(f'计算签名: {signature}')
|
||||
print(f'编码后签名: {urllib.parse.quote_plus(signature)}')
|
||||
url = f'{url}?{url_path}&Signature={urllib.parse.quote_plus(signature)}'
|
||||
print(f'url: {url}')
|
||||
|
||||
dataReqL=requests.post(url,headers=header,data=None)
|
||||
resText = dataReqL.text
|
||||
i = 0
|
||||
while 'error' in resText and 'data' not in resText and i < 5:
|
||||
print(f'请求工单列表失败，再次请求第{i+1}次')
|
||||
time.sleep(1)
|
||||
dataReqL=requests.post(url,headers=header,data=None)
|
||||
resText = dataReqL.text
|
||||
i = i + 1
|
||||
resL=json.loads(resText)
|
||||
print(dataReqL)
|
||||
return resL
|
||||
|
||||
""" 查询消息详情 -- 新 """
|
||||
def request_detail_signature_post(id, pageIndex=1):
|
||||
print(f'开始请求工单列表数据：{formatted2_previous_hour(0)}')
|
||||
url='https://api-bj.clink.cn/livechat/record_session_message_list'
|
||||
header={'Content-Type':'application/x-www-form-urlencoded'}
|
||||
param={'AccessKeyId':'b17759d3a36fba9a2cf522fbf4cbf177','Timestamp':current_time_utc,'Expires':86400,'mainUniqueId':id,'pageSize':1000,'pageIndex':pageIndex}
|
||||
print(f'param: {param}')
|
||||
url_path = build_query_string(param)
|
||||
url_param = build_query_string(param)
|
||||
print(f'url_param: {url_param}')
|
||||
url_param = f'POSTapi-bj.clink.cn/livechat/record_session_message_list?{url_param}'
|
||||
print(f'待计算字符串: {url_param}')
|
||||
signature= generate_signature(url_param,'5g027B6w06630Y5240c1')
|
||||
print(f'计算签名: {signature}')
|
||||
print(f'编码后签名: {urllib.parse.quote_plus(signature)}')
|
||||
url = f'{url}?{url_path}&Signature={urllib.parse.quote_plus(signature)}'
|
||||
print(f'url: {url}')
|
||||
|
||||
dataReqL=requests.post(url,headers=header,data=None)
|
||||
resText = dataReqL.text
|
||||
i = 0
|
||||
while 'error' in resText and 'data' not in resText and i < 5:
|
||||
print(f'请求工单列表失败，再次请求第{i+1}次')
|
||||
time.sleep(1)
|
||||
dataReqL=requests.post(url,headers=header,data=None)
|
||||
resText = dataReqL.text
|
||||
i = i + 1
|
||||
resL=json.loads(resText)
|
||||
print(dataReqL)
|
||||
return resL
|
||||
|
||||
def get_data_from_db():
|
||||
conn = psycopg2.connect(database=database, user=user, password=password,host=host, port=port)
|
||||
cur=conn.cursor()
|
||||
sql = "select * from data_api.tr_chat_record_ids_new"
|
||||
cur.execute(sql)
|
||||
records = cur.fetchall()
|
||||
conn.commit()
|
||||
cur.close()
|
||||
conn.close()
|
||||
return records
|
||||
|
||||
def load_data_to_db(dataList):
|
||||
conn = psycopg2.connect(database=database, user=user, password=password,host=host, port=port)
|
||||
print('数据库连接成功')
|
||||
dataId=str(uuid.uuid4())
|
||||
total=len(dataList)
|
||||
print('临时id：'+dataId)
|
||||
json_object = json.dumps(dataList)
|
||||
cur=conn.cursor()
|
||||
sql="INSERT INTO data_api.api_data (id,api_id,data,total_num,is_loaded,status,request_tm,execute_tm,remark) values (%s,%s,%s,%s,'0','0',current_timestamp(0),current_timestamp(0),'')"
|
||||
cur.execute(sql,[dataId,'8224925b5df649349807d1aa540192da', json_object, total])
|
||||
conn.commit()
|
||||
cur.close()
|
||||
conn.close()
|
||||
print('加载数据结束：chat_records:查询会话记录列表')
|
||||
|
||||
def load_detail_data_to_db(ids, dataList):
|
||||
conn = psycopg2.connect(database=database, user=user, password=password,host=host, port=port)
|
||||
print('数据库连接成功')
|
||||
dataId=str(uuid.uuid4())
|
||||
total=len(dataList)
|
||||
print('临时id：'+dataId)
|
||||
json_object = json.dumps(dataList)
|
||||
idstr = ','.join(ids)
|
||||
cur=conn.cursor()
|
||||
sql='''update data_api.cc_details_ids_exp set is_loaded = '1' where api_id = '63bb5ebbf3bd4839816ea3380082e1e7' and id in (%s);
|
||||
update data_api.cc_message_details set is_loaded = '1' where api_id = '63bb5ebbf3bd4839816ea3380082e1e7';
|
||||
INSERT INTO data_api.cc_message_details (id,api_id,data,total_num,is_loaded,status,request_tm,execute_tm,remark)
|
||||
values (%s,%s,%s,%s,'0','0',current_timestamp(0),current_timestamp(0),'')'''
|
||||
cur.execute(sql,[idstr,dataId,'63bb5ebbf3bd4839816ea3380082e1e7', json_object, total])
|
||||
conn.commit()
|
||||
cur.close()
|
||||
conn.close()
|
||||
print('加载数据结束：message_detail:获取会话详情')
|
||||
|
||||
def load_detail_exp_to_db(id):
|
||||
try:
|
||||
print(f'添加消息异常记录:{id}')
|
||||
conn = psycopg2.connect(database=database, user=user, password=password,host=host, port=port)
|
||||
print('数据库连接成功')
|
||||
dataId=str(uuid.uuid4())
|
||||
print('临时id：'+dataId)
|
||||
cur=conn.cursor()
|
||||
sql=" INSERT INTO data_api.cc_details_ids_exp (id,api_id,is_loaded,status,request_tm,execute_tm,remark) values (%s,%s,'0','0',current_timestamp(0),current_timestamp(0),'')"
|
||||
cur.execute(sql,[id, '63bb5ebbf3bd4839816ea3380082e1e7'])
|
||||
conn.commit()
|
||||
cur.close()
|
||||
conn.close()
|
||||
print(f'添加消息异常记录:{id} 结束')
|
||||
except Exception as e:
|
||||
print(f'添加消息异常记录:{id}失败, 错误信息：{e}')
|
||||
|
||||
def fetch_records_all(d):
|
||||
"""
|
||||
Fetches all chat records from a remote server.
|
||||
|
||||
This function sends a series of POST requests to retrieve chat records.
|
||||
It handles pagination by checking the 'hasNextPage' flag in the response and
|
||||
continues to request the next page of records until all records are fetched.
|
||||
|
||||
Parameters:
|
||||
d (any): This parameter is not used in the function. It seems to be part of a document reference or database entry.
|
||||
|
||||
Returns:
|
||||
list: A list of chat records fetched from the server.
|
||||
|
||||
Raises:
|
||||
None: The function prints error messages but does not raise exceptions.
|
||||
"""
|
||||
def fetch_records_all(d):
|
||||
resL = request_data_signature_post(1)
|
||||
# print(resL)
|
||||
list = []
|
||||
print(f"resl:{resL}")
|
||||
if 'error' in resL:
|
||||
error = resL['error']
|
||||
print(f'请求会话列表失败，失败原因：{error}')
|
||||
else:
|
||||
data = resL['data']
|
||||
list = data['list']
|
||||
flag = data['hasNextPage']
|
||||
nextList = []
|
||||
i = 1
|
||||
while flag is True:
|
||||
i = i+1
|
||||
resN = request_data_signature_post(i)
|
||||
if 'records' not in resN:
|
||||
error = resL['error']
|
||||
print(f'请求会话列表失败，失败原因：{error}')
|
||||
break
|
||||
nextData = resN['data']
|
||||
nextList = nextData['records']
|
||||
flag = nextData['hasNextPage']
|
||||
if len(nextList) == 0:
|
||||
break
|
||||
list = list + nextList
|
||||
if len(nextList) < 100:
|
||||
break
|
||||
print(f'records会话记录数为：{len(list)}, 共请求{i}次会话')
|
||||
return list
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
print(f'{formatted2_previous_hour(0)}开始请求会话信息')
|
||||
hour_delta = 1
|
||||
previous_time = previous_hour_timestamp(hour_delta)
|
||||
list = fetch_records_all(0)
|
||||
|
||||
if datetime.datetime.now().hour < 3:
|
||||
pre_list = fetch_records_all(1)
|
||||
list = list + pre_list
|
||||
load_data_to_db(list)
|
||||
records = get_data_from_db()
|
||||
if len(records) > 0:
|
||||
print(f'此处需处理{len(records)}条会话信息信息')
|
||||
detailDataList = []
|
||||
ids = []
|
||||
j = 0
|
||||
k = 0
|
||||
l = 0
|
||||
for data in records:
|
||||
try:
|
||||
id = data[0]
|
||||
j = j + 1
|
||||
print(f'{j}. 开始请求会话详情数据：{id}')
|
||||
resD = request_detail_signature_post(id)
|
||||
if 'data' in resD:
|
||||
k = k + 1
|
||||
ids.append(id)
|
||||
dataList = resD['data']
|
||||
detailDataList = detailDataList+dataList
|
||||
print(f'{j}. deltail: {len(dataList)}')
|
||||
else:
|
||||
l = l+1
|
||||
error = resD['error']
|
||||
print(f"请求消息详情（id:{id}）失败，错误信息：{error}")
|
||||
load_detail_exp_to_db(id)
|
||||
except Exception as e:
|
||||
print(f'请求消息详情（id:{id}）异常, ）异常信息：{e}')
|
||||
load_detail_exp_to_db(data['id'])
|
||||
print(f'{j}. exp end')
|
||||
if len(ids) > 0:
|
||||
ids_str = [str(item) for item in ids]
|
||||
load_detail_data_to_db(ids_str,detailDataList)
|
||||
print(f'实际加载{j}-{len(records)}，{len(detailDataList)} - {k}-{l} 条记录')
|
||||
print(f'{formatted2_previous_hour(0)}请求会话信息结束')
|
||||
|
|
@ -0,0 +1,162 @@
|
|||
/*******Main Section**************************************************************************/
|
||||
\set ON_ERROR_STOP on
|
||||
\set AUTOCOMMIT on
|
||||
\timing on
|
||||
|
||||
DELETE FROM data_api.tr_chat_records_new;
|
||||
|
||||
insert into data_api.tr_chat_records_new (
|
||||
main_unique_id
|
||||
, app_name
|
||||
, start_time
|
||||
, end_time
|
||||
, session_tags
|
||||
, is_valid
|
||||
, first_qno
|
||||
, first_qname
|
||||
, first_cno
|
||||
, first_cname
|
||||
, has_comment
|
||||
, has_agent_ticket
|
||||
, open_type_name
|
||||
, contact_type_name
|
||||
, receive_type_name
|
||||
, close_reason_name
|
||||
, close_status_name
|
||||
, repeat_visit_name
|
||||
, is_robot_valid_name
|
||||
, is_valid_name
|
||||
, total_duration_pretty
|
||||
, total_duration
|
||||
, queue_duration_pretty
|
||||
, queue_duration
|
||||
, visitor_id
|
||||
, visitor_name
|
||||
, customer_name
|
||||
, ip
|
||||
, province
|
||||
, city
|
||||
, phone_type_name
|
||||
, visitor_extra_info
|
||||
, device_type
|
||||
, browser
|
||||
, operating_system
|
||||
, search_word
|
||||
, market_keyword
|
||||
, first_visit_page_url
|
||||
, initiation_page_url
|
||||
, referer_url
|
||||
, search_engine_name
|
||||
, medium
|
||||
, plan
|
||||
, unit
|
||||
, account
|
||||
, source
|
||||
,etl_tx_dt
|
||||
)
|
||||
select
|
||||
case when trim(both from main_unique_id)='' then null else main_unique_id::text end main_unique_id
|
||||
, case when trim(both from app_name)='' then null else app_name::text end app_name
|
||||
, case when trim(both from start_time)='' then null else start_time::text end start_time
|
||||
, case when trim(both from end_time)='' then null else end_time::text end end_time
|
||||
, case when trim(both from session_tags)='' then null else session_tags::text end session_tags
|
||||
, case when trim(both from is_valid)='' then null else is_valid::text end is_valid
|
||||
, case when trim(both from first_qno)='' then null else first_qno::text end first_qno
|
||||
, case when trim(both from first_qname)='' then null else first_qname::text end first_qname
|
||||
, case when trim(both from first_cno)='' then null else first_cno::text end first_cno
|
||||
, case when trim(both from first_cname)='' then null else first_cname::text end first_cname
|
||||
, case when trim(both from has_comment)='' then null else has_comment::text end has_comment
|
||||
, case when trim(both from has_agent_ticket)='' then null else has_agent_ticket::text end has_agent_ticket
|
||||
, case when trim(both from open_type_name)='' then null else open_type_name::text end open_type_name
|
||||
, case when trim(both from contact_type_name)='' then null else contact_type_name::text end contact_type_name
|
||||
, case when trim(both from receive_type_name)='' then null else receive_type_name::text end receive_type_name
|
||||
, case when trim(both from close_reason_name)='' then null else close_reason_name::text end close_reason_name
|
||||
, case when trim(both from close_status_name)='' then null else close_status_name::text end close_status_name
|
||||
, case when trim(both from repeat_visit_name)='' then null else repeat_visit_name::text end repeat_visit_name
|
||||
, case when trim(both from is_robot_valid_name)='' then null else is_robot_valid_name::text end is_robot_valid_name
|
||||
, case when trim(both from is_valid_name)='' then null else is_valid_name::text end is_valid_name
|
||||
, case when trim(both from total_duration_pretty)='' then null else total_duration_pretty::text end total_duration_pretty
|
||||
, case when trim(both from total_duration)='' then null else total_duration::text end total_duration
|
||||
, case when trim(both from queue_duration_pretty)='' then null else queue_duration_pretty::text end queue_duration_pretty
|
||||
, case when trim(both from queue_duration)='' then null else queue_duration::text end queue_duration
|
||||
, case when trim(both from visitor_id)='' then null else visitor_id::text end visitor_id
|
||||
, case when trim(both from visitor_name)='' then null else visitor_name::text end visitor_name
|
||||
, case when trim(both from customer_name)='' then null else customer_name::text end customer_name
|
||||
, case when trim(both from ip)='' then null else ip::text end ip
|
||||
, case when trim(both from province)='' then null else province::text end province
|
||||
, case when trim(both from city)='' then null else city::text end city
|
||||
, case when trim(both from phone_type_name)='' then null else phone_type_name::text end phone_type_name
|
||||
, case when trim(both from visitor_extra_info)='' then null else visitor_extra_info::text end visitor_extra_info
|
||||
, case when trim(both from device_type)='' then null else device_type::text end device_type
|
||||
, case when trim(both from browser)='' then null else browser::text end browser
|
||||
, case when trim(both from operating_system)='' then null else operating_system::text end operating_system
|
||||
, case when trim(both from search_word)='' then null else search_word::text end search_word
|
||||
, case when trim(both from market_keyword)='' then null else market_keyword::text end market_keyword
|
||||
, case when trim(both from first_visit_page_url)='' then null else first_visit_page_url::text end first_visit_page_url
|
||||
, case when trim(both from initiation_page_url)='' then null else initiation_page_url::text end initiation_page_url
|
||||
, case when trim(both from referer_url)='' then null else referer_url::text end referer_url
|
||||
, case when trim(both from search_engine_name)='' then null else search_engine_name::text end search_engine_name
|
||||
, case when trim(both from medium)='' then null else medium::text end medium
|
||||
, case when trim(both from plan)='' then null else plan::text end plan
|
||||
, case when trim(both from unit)='' then null else unit::text end unit
|
||||
, case when trim(both from account)='' then null else account::text end account
|
||||
, case when trim(both from source)='' then null else source::text end source
|
||||
,etl_tx_dt
|
||||
from (
|
||||
select
|
||||
(json_array_elements(data::json)::json -> 'sessionInfoApiDto' ->>'mainUniqueId') main_unique_id
|
||||
, (json_array_elements(data::json)::json -> 'sessionInfoApiDto' ->>'appName') app_name
|
||||
, (json_array_elements(data::json)::json -> 'sessionInfoApiDto' ->>'startTime') start_time
|
||||
, (json_array_elements(data::json)::json -> 'sessionInfoApiDto' ->>'endTime') end_time
|
||||
, (json_array_elements(data::json)::json -> 'sessionInfoApiDto' ->>'sessionTags') session_tags
|
||||
, (json_array_elements(data::json)::json -> 'sessionInfoApiDto' ->>'isValid') is_valid
|
||||
, (json_array_elements(data::json)::json -> 'sessionInfoApiDto' ->>'firstQno') first_qno
|
||||
, (json_array_elements(data::json)::json -> 'sessionInfoApiDto' ->>'firstQname') first_qname
|
||||
, (json_array_elements(data::json)::json -> 'sessionInfoApiDto' ->>'firstCno') first_cno
|
||||
, (json_array_elements(data::json)::json -> 'sessionInfoApiDto' ->>'firstCname') first_cname
|
||||
, (json_array_elements(data::json)::json -> 'sessionInfoApiDto' ->>'hasComment') has_comment
|
||||
, (json_array_elements(data::json)::json -> 'sessionInfoApiDto' ->>'hasAgentTicket') has_agent_ticket
|
||||
, (json_array_elements(data::json)::json -> 'sessionInfoApiDto' ->>'openTypeName') open_type_name
|
||||
, (json_array_elements(data::json)::json -> 'sessionInfoApiDto' ->>'contactTypeName') contact_type_name
|
||||
, (json_array_elements(data::json)::json -> 'sessionInfoApiDto' ->>'receiveTypeName') receive_type_name
|
||||
, (json_array_elements(data::json)::json -> 'sessionInfoApiDto' ->>'closeReasonName') close_reason_name
|
||||
, (json_array_elements(data::json)::json -> 'sessionInfoApiDto' ->>'closeStatusName') close_status_name
|
||||
, (json_array_elements(data::json)::json -> 'sessionInfoApiDto' ->>'repeatVisitName') repeat_visit_name
|
||||
, (json_array_elements(data::json)::json -> 'sessionInfoApiDto' ->>'isRobotValidName') is_robot_valid_name
|
||||
, (json_array_elements(data::json)::json -> 'sessionInfoApiDto' ->>'isValidName') is_valid_name
|
||||
, (json_array_elements(data::json)::json -> 'sessionInfoApiDto' ->>'totalDurationPretty') total_duration_pretty
|
||||
, (json_array_elements(data::json)::json -> 'sessionInfoApiDto' ->>'totalDuration') total_duration
|
||||
, (json_array_elements(data::json)::json -> 'sessionInfoApiDto' ->>'queueDurationPretty') queue_duration_pretty
|
||||
, (json_array_elements(data::json)::json -> 'sessionInfoApiDto' ->>'queueDuration') queue_duration
|
||||
, (json_array_elements(data::json)::json -> 'sessionVisitorApiDto' ->>'visitorId') visitor_id
|
||||
, (json_array_elements(data::json)::json -> 'sessionVisitorApiDto' ->>'visitorName') visitor_name
|
||||
, (json_array_elements(data::json)::json -> 'sessionVisitorApiDto' ->>'customerName') customer_name
|
||||
, (json_array_elements(data::json)::json -> 'sessionVisitorApiDto' ->>'ip') ip
|
||||
, (json_array_elements(data::json)::json -> 'sessionVisitorApiDto' ->>'province') province
|
||||
, (json_array_elements(data::json)::json -> 'sessionVisitorApiDto' ->>'city') city
|
||||
, (json_array_elements(data::json)::json -> 'sessionVisitorApiDto' ->>'phoneTypeName') phone_type_name
|
||||
, (json_array_elements(data::json)::json -> 'sessionVisitorApiDto' ->>'visitorExtraInfo') visitor_extra_info
|
||||
, (json_array_elements(data::json)::json -> 'sessionVisitorApiDto' ->>'deviceType') device_type
|
||||
, (json_array_elements(data::json)::json -> 'sessionVisitorApiDto' ->>'browser') browser
|
||||
, (json_array_elements(data::json)::json -> 'sessionVisitorApiDto' ->>'operatingSystem') operating_system
|
||||
, (json_array_elements(data::json)::json -> 'sessionVisitorApiDto' ->>'searchWord') search_word
|
||||
, (json_array_elements(data::json)::json -> 'sessionVisitorApiDto' ->>'marketKeyword') market_keyword
|
||||
, (json_array_elements(data::json)::json -> 'sessionVisitorApiDto' ->>'firstVisitPageUrl') first_visit_page_url
|
||||
, (json_array_elements(data::json)::json -> 'sessionVisitorApiDto' ->>'initiationPageUrl') initiation_page_url
|
||||
, (json_array_elements(data::json)::json -> 'sessionVisitorApiDto' ->>'refererUrl') referer_url
|
||||
, (json_array_elements(data::json)::json -> 'sessionVisitorApiDto' ->>'searchEngineName') search_engine_name
|
||||
, (json_array_elements(data::json)::json -> 'sessionUtmApiDto' ->>'medium') medium
|
||||
, (json_array_elements(data::json)::json -> 'sessionUtmApiDto' ->>'plan') plan
|
||||
, (json_array_elements(data::json)::json -> 'sessionUtmApiDto' ->>'unit') unit
|
||||
, (json_array_elements(data::json)::json -> 'sessionUtmApiDto' ->>'account') account
|
||||
, (json_array_elements(data::json)::json -> 'sessionUtmApiDto' ->>'source') source
|
||||
,CURRENT_TIMESTAMP(0) etl_tx_dt
|
||||
from (select * from data_api.api_data
|
||||
WHERE api_id='8224925b5df649349807d1aa540192da' and is_loaded = '0' order by request_tm desc limit 1) p )p;
|
||||
|
||||
update data_api.api_data
|
||||
set is_loaded = '1' ,
|
||||
status = '1',
|
||||
request_tm = current_timestamp(0)
|
||||
where api_id='8224925b5df649349807d1aa540192da';
|
||||
\q
|
||||
Loading…
Reference in New Issue