From e769cc604c2648527e4b01c2578cde51c758a156 Mon Sep 17 00:00:00 2001
From: root <root@81c39a89d7ce>
Date: Tue, 26 Dec 2023 09:42:36 +0800
Subject: [PATCH] =?UTF-8?q?add=20workflow=20=E6=B3=B0=E5=85=8B=E5=AE=A2?=
 =?UTF-8?q?=E6=88=B7,dev?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 TK_Cust/dev/tk_cust/livechat/S98_livechat.sql | 102 ++++++++++++++++++
 .../tk_cust/livechat/sa_foreign_tables.sql    |  32 ++++++
 TK_Cust/dev/tk_cust/livechat/sa_tables.sql    |  99 +++++++++++++++++
 .../dev/tk_cust/泰克客户/wf_dag_tk_cust.py    |  61 +++++++++++
 4 files changed, 294 insertions(+)
 create mode 100644 TK_Cust/dev/tk_cust/livechat/S98_livechat.sql
 create mode 100644 TK_Cust/dev/tk_cust/livechat/sa_foreign_tables.sql
 create mode 100644 TK_Cust/dev/tk_cust/livechat/sa_tables.sql
 create mode 100644 TK_Cust/dev/tk_cust/泰克客户/wf_dag_tk_cust.py

diff --git a/TK_Cust/dev/tk_cust/livechat/S98_livechat.sql b/TK_Cust/dev/tk_cust/livechat/S98_livechat.sql
new file mode 100644
index 0000000..de9fdee
--- /dev/null
+++ b/TK_Cust/dev/tk_cust/livechat/S98_livechat.sql
@@ -0,0 +1,102 @@
+/*******Main Section**************************************************************************/
+\set ON_ERROR_STOP on
+\set AUTOCOMMIT on
+\timing on
+delete from p10_sa.S98_livechat
+;
+insert into p10_sa.S98_livechat
+(	 conference_id  
+	, create_date  
+	, start_date  
+	, start_url  
+	, referrer  
+	, chat_dura  
+	, queue_dura  
+	, visitor_id  
+	, visit_nick  
+	, visitor_ip  
+	, visitor_email  
+	, operator_id  
+	, group_id  
+	, group_name  
+	, rate  
+	, last_rate_comment  
+	, operator_1_nick  
+	, operator_2_nick  
+	, operator_3_nick  
+	, operator_4_nick  
+	, etl_tx_dt   )
+  select 
+	 conference_id  
+	, create_date  
+	, start_date  
+	, start_url  
+	, referrer  
+	, chat_dura  
+	, queue_dura  
+	, visitor_id  
+	, visit_nick  
+	, visitor_ip  
+	, visitor_email  
+	, operator_id  
+	, group_id  
+	, group_name  
+	, rate  
+	, last_rate_comment  
+	, operator_1_nick  
+	, operator_2_nick  
+	, operator_3_nick  
+	, operator_4_nick  
+	, etl_tx_dt  
+  from p00_tal.S98_livechat
+  ;
+  delete from p12_sfull.S98_livechat
+;
+;
+insert into p12_sfull.S98_livechat
+(	 conference_id  
+	, create_date  
+	, start_date  
+	, start_url  
+	, referrer  
+	, chat_dura  
+	, queue_dura  
+	, visitor_id  
+	, visit_nick  
+	, visitor_ip  
+	, visitor_email  
+	, operator_id  
+	, group_id  
+	, group_name  
+	, rate  
+	, last_rate_comment  
+	, operator_1_nick  
+	, operator_2_nick  
+	, operator_3_nick  
+	, operator_4_nick  
+	, etl_tx_dt   )
+  select 
+	 conference_id  
+	, create_date  
+	, start_date  
+	, start_url  
+	, referrer  
+	, chat_dura  
+	, queue_dura  
+	, visitor_id  
+	, visit_nick  
+	, visitor_ip  
+	, visitor_email  
+	, operator_id  
+	, group_id  
+	, group_name  
+	, rate  
+	, last_rate_comment  
+	, operator_1_nick  
+	, operator_2_nick  
+	, operator_3_nick  
+	, operator_4_nick  
+	, etl_tx_dt  
+  from p10_sa.S98_livechat
+;
+\q
\ No newline at end of file
diff --git a/TK_Cust/dev/tk_cust/livechat/sa_foreign_tables.sql b/TK_Cust/dev/tk_cust/livechat/sa_foreign_tables.sql
new file mode 100644
index 0000000..7b3e6a4
--- /dev/null
+++ b/TK_Cust/dev/tk_cust/livechat/sa_foreign_tables.sql
@@ -0,0 +1,32 @@
+
+CREATE FOREIGN TABLE if not exists p00_tal.S98_livechat (
+ conference_id  TEXT 
+	, create_date  TEXT 
+	, start_date  TEXT 
+	, start_url  TEXT 
+	, referrer  TEXT 
+	, chat_dura  TEXT 
+	, queue_dura  TEXT 
+	, visitor_id  TEXT 
+	, visit_nick  TEXT 
+	, visitor_ip  TEXT 
+	, visitor_email  TEXT 
+	, operator_id  TEXT 
+	, group_id  TEXT 
+	, group_name  TEXT 
+	, rate  TEXT 
+	, last_rate_comment  TEXT 
+	, operator_1_nick  TEXT 
+	, operator_2_nick  TEXT 
+	, operator_3_nick  TEXT 
+	, operator_4_nick  TEXT 
+	, etl_tx_dt  TIMESTAMP 
+) 
+
+
+SERVER pgsql_server_S98 OPTIONS(schema_name 'data_api', table_name 'livechat' ); 
+
+
+
+
+
diff --git a/TK_Cust/dev/tk_cust/livechat/sa_tables.sql b/TK_Cust/dev/tk_cust/livechat/sa_tables.sql
new file mode 100644
index 0000000..b4a1c2b
--- /dev/null
+++ b/TK_Cust/dev/tk_cust/livechat/sa_tables.sql
@@ -0,0 +1,99 @@
+
+create table if not exists p10_sa.S98_livechat (
+	 conference_id  TEXT 
+	, create_date  TEXT 
+	, start_date  TEXT 
+	, start_url  TEXT 
+	, referrer  TEXT 
+	, chat_dura  TEXT 
+	, queue_dura  TEXT 
+	, visitor_id  TEXT 
+	, visit_nick  TEXT 
+	, visitor_ip  TEXT 
+	, visitor_email  TEXT 
+	, operator_id  TEXT 
+	, group_id  TEXT 
+	, group_name  TEXT 
+	, rate  TEXT 
+	, last_rate_comment  TEXT 
+	, operator_1_nick  TEXT 
+	, operator_2_nick  TEXT 
+	, operator_3_nick  TEXT 
+	, operator_4_nick  TEXT 
+	, etl_tx_dt  TIMESTAMP 
+) ; 
+
+ COMMENT ON COLUMN p10_sa.S98_livechat.conference_id IS 'conferenceId'; 
+ COMMENT ON COLUMN p10_sa.S98_livechat.create_date IS 'chat creation date Asia/Shanghai'; 
+ COMMENT ON COLUMN p10_sa.S98_livechat.start_date IS 'chat start date Asia/Shanghai'; 
+ COMMENT ON COLUMN p10_sa.S98_livechat.start_url IS 'chat start url'; 
+ COMMENT ON COLUMN p10_sa.S98_livechat.referrer IS 'referrer'; 
+ COMMENT ON COLUMN p10_sa.S98_livechat.chat_dura IS 'chat duration in seconds'; 
+ COMMENT ON COLUMN p10_sa.S98_livechat.queue_dura IS 'queue duration in seconds'; 
+ COMMENT ON COLUMN p10_sa.S98_livechat.visitor_id IS 'visitor livechat id'; 
+ COMMENT ON COLUMN p10_sa.S98_livechat.visit_nick IS 'visitor nick'; 
+ COMMENT ON COLUMN p10_sa.S98_livechat.visitor_ip IS 'visitor ip'; 
+ COMMENT ON COLUMN p10_sa.S98_livechat.visitor_email IS 'visitor email'; 
+ COMMENT ON COLUMN p10_sa.S98_livechat.operator_id IS 'last operator id'; 
+ COMMENT ON COLUMN p10_sa.S98_livechat.group_id IS 'group'; 
+ COMMENT ON COLUMN p10_sa.S98_livechat.group_name IS 'group name'; 
+ COMMENT ON COLUMN p10_sa.S98_livechat.rate IS 'rate'; 
+ COMMENT ON COLUMN p10_sa.S98_livechat.last_rate_comment IS 'last rate comment'; 
+ COMMENT ON COLUMN p10_sa.S98_livechat.operator_1_nick IS 'operator 1 nick'; 
+ COMMENT ON COLUMN p10_sa.S98_livechat.operator_2_nick IS 'operator 2 nick'; 
+ COMMENT ON COLUMN p10_sa.S98_livechat.operator_3_nick IS 'operator 3 nick'; 
+ COMMENT ON COLUMN p10_sa.S98_livechat.operator_4_nick IS 'operator 4 nick'; 
+ COMMENT ON COLUMN p10_sa.S98_livechat.etl_tx_dt IS ''; 
+
+COMMENT ON TABLE p10_sa.S98_livechat IS '';
+
+
+
+create table if not exists p12_sfull.S98_livechat (
+	 conference_id  TEXT 
+	, create_date  TEXT 
+	, start_date  TEXT 
+	, start_url  TEXT 
+	, referrer  TEXT 
+	, chat_dura  TEXT 
+	, queue_dura  TEXT 
+	, visitor_id  TEXT 
+	, visit_nick  TEXT 
+	, visitor_ip  TEXT 
+	, visitor_email  TEXT 
+	, operator_id  TEXT 
+	, group_id  TEXT 
+	, group_name  TEXT 
+	, rate  TEXT 
+	, last_rate_comment  TEXT 
+	, operator_1_nick  TEXT 
+	, operator_2_nick  TEXT 
+	, operator_3_nick  TEXT 
+	, operator_4_nick  TEXT 
+	, etl_tx_dt  TIMESTAMP 
+) ; 
+
+ COMMENT ON COLUMN p12_sfull.S98_livechat.conference_id IS 'conferenceId'; 
+ COMMENT ON COLUMN p12_sfull.S98_livechat.create_date IS 'chat creation date Asia/Shanghai'; 
+ COMMENT ON COLUMN p12_sfull.S98_livechat.start_date IS 'chat start date Asia/Shanghai'; 
+ COMMENT ON COLUMN p12_sfull.S98_livechat.start_url IS 'chat start url'; 
+ COMMENT ON COLUMN p12_sfull.S98_livechat.referrer IS 'referrer'; 
+ COMMENT ON COLUMN p12_sfull.S98_livechat.chat_dura IS 'chat duration in seconds'; 
+ COMMENT ON COLUMN p12_sfull.S98_livechat.queue_dura IS 'queue duration in seconds'; 
+ COMMENT ON COLUMN p12_sfull.S98_livechat.visitor_id IS 'visitor livechat id'; 
+ COMMENT ON COLUMN p12_sfull.S98_livechat.visit_nick IS 'visitor nick'; 
+ COMMENT ON COLUMN p12_sfull.S98_livechat.visitor_ip IS 'visitor ip'; 
+ COMMENT ON COLUMN p12_sfull.S98_livechat.visitor_email IS 'visitor email'; 
+ COMMENT ON COLUMN p12_sfull.S98_livechat.operator_id IS 'last operator id'; 
+ COMMENT ON COLUMN p12_sfull.S98_livechat.group_id IS 'group'; 
+ COMMENT ON COLUMN p12_sfull.S98_livechat.group_name IS 'group name'; 
+ COMMENT ON COLUMN p12_sfull.S98_livechat.rate IS 'rate'; 
+ COMMENT ON COLUMN p12_sfull.S98_livechat.last_rate_comment IS 'last rate comment'; 
+ COMMENT ON COLUMN p12_sfull.S98_livechat.operator_1_nick IS 'operator 1 nick'; 
+ COMMENT ON COLUMN p12_sfull.S98_livechat.operator_2_nick IS 'operator 2 nick'; 
+ COMMENT ON COLUMN p12_sfull.S98_livechat.operator_3_nick IS 'operator 3 nick'; 
+ COMMENT ON COLUMN p12_sfull.S98_livechat.operator_4_nick IS 'operator 4 nick'; 
+ COMMENT ON COLUMN p12_sfull.S98_livechat.etl_tx_dt IS ''; 
+
+COMMENT ON TABLE p12_sfull.S98_livechat IS '';
+
diff --git a/TK_Cust/dev/tk_cust/泰克客户/wf_dag_tk_cust.py b/TK_Cust/dev/tk_cust/泰克客户/wf_dag_tk_cust.py
new file mode 100644
index 0000000..ee1267c
--- /dev/null
+++ b/TK_Cust/dev/tk_cust/泰克客户/wf_dag_tk_cust.py
@@ -0,0 +1,61 @@
+#!/usr/bin/python
+# -*- encoding=utf-8 -*-
+from airflow import DAG
+from datetime import datetime, timedelta
+from airflow.contrib.hooks.ssh_hook import SSHHook
+from airflow.contrib.operators.ssh_operator import SSHOperator
+from airflow.sensors.external_task_sensor import ExternalTaskSensor
+import json
+
+from airflow.operators.email_operator import EmailOperator
+from airflow.utils.trigger_rule import TriggerRule
+
+
+sshHook = SSHHook(ssh_conn_id ='ssh_air')
+default_args = {
+'owner': 'info@idgvalue.com',
+'email': [''],
+'email_on_failure': True,
+'email_on_retry':True,
+'start_date': datetime(2022, 9, 12),
+'depends_on_past': False,
+'retries': 6,
+'retry_delay': timedelta(minutes=10),
+}
+
+dag = DAG('wf_dag_tk_cust', default_args=default_args,
+schedule_interval="0 0 * * *",
+catchup=False,
+dagrun_timeout=timedelta(minutes=160),
+max_active_runs=3)
+
+task_failed = EmailOperator (
+    dag=dag,
+    trigger_rule=TriggerRule.ONE_FAILED,
+    task_id="task_failed",
+    to=["info@idgvalue.com"],
+    cc=[""],
+    subject="tk_cust_failed",
+    html_content='<h3>您好,tk_cust作业失败,请及时处理" </h3>')
+
+file_Tk01 = SSHOperator(
+ssh_hook=sshHook,
+task_id='file_Tk01',
+command='python /data/airflow/bin/FILELOD.py {{ params.my_param }} {{ ds_nodash }}  >>/data/airflow/logs/file_load/file_load_{{ds_nodash}}.log 2>&1 ',
+params={'my_param':"Tk01"},
+depends_on_past=False, 
+retries=3, 
+dag=dag)
+
+
+livechat_6381 = SSHOperator(
+ssh_hook=sshHook,
+task_id='livechat_6381',
+command='/data/airflow/etl/SA/run_sa.sh {{ ds_nodash }} {{ params.my_param }} >>/data/airflow/logs/run_psql_{{ds_nodash}}.log 2>&1 ',
+params={'my_param':"S98_S_livechat"},
+depends_on_past=False, 
+retries=3, 
+dag=dag)
+
+file_Tk01 >> livechat_6381
+livechat_6381 >> task_failed