add workflow 市场News-Agents调度,dev

dev/workflow/TK_Cust/agents_market_newsletter/市场News-Agents调度/wf_dag_agents_market_newsletter.py

@@ -0,0 +1,119 @@
+#!/usr/bin/python
+# -*- encoding=utf-8 -*-
+from airflow import DAG
+from datetime import datetime, timedelta
+from airflow.contrib.hooks.ssh_hook import SSHHook
+from airflow.contrib.operators.ssh_operator import SSHOperator
+from airflow.sensors.external_task_sensor import ExternalTaskSensor
+import json
+
+from airflow.operators.trigger_dagrun import TriggerDagRunOperator
+from airflow.operators.email_operator import EmailOperator
+from airflow.utils.trigger_rule import TriggerRule
+
+
+sshHook = SSHHook(ssh_conn_id ='ssh_air')
+default_args = {
+'owner': 'tek_newsletter@163.com',
+'email_on_failure': True,
+'email_on_retry':True,
+'start_date': datetime(2024, 1, 1),
+'depends_on_past': False,
+'retries': 6,
+'retry_delay': timedelta(minutes=10),
+}
+
+dag = DAG('wf_dag_agents_market_newsletter', default_args=default_args,
+schedule_interval="0 10 * * 5,1,3",
+catchup=False,
+dagrun_timeout=timedelta(minutes=600),
+max_active_runs=3,
+tags=['64cbd1bbced14209b5e3a879f89e8ab1','TK_Cust','市场News-Agents调度']
+)
+
+task_failed = EmailOperator (
+    dag=dag,
+    trigger_rule=TriggerRule.ONE_FAILED,
+    task_id="task_failed",
+    to=["tek_newsletter@163.com"],
+    cc=[""],
+    subject="agents_market_newsletter_failed",
+    html_content='<h3>您好，agents_market_newsletter作业失败，请及时处理" </h3>')
+
+
+agents_recuriments = SSHOperator(
+ssh_hook=sshHook,
+task_id='agents_recuriments',
+command='cd /data/airflow/etl/agents && python agents_recuriments.py',
+params={'my_param':"agents_recuriments"},
+depends_on_past=False, 
+retries=1, 
+dag=dag)
+
+
+agents_market_summar = SSHOperator(
+ssh_hook=sshHook,
+task_id='agents_market_summar',
+command='cd /data/airflow/etl/agents && python agents_market_summar.py',
+params={'my_param':"agents_market_summar"},
+depends_on_past=False, 
+retries=1, 
+dag=dag)
+
+
+tk_product_prefer = SSHOperator(
+ssh_hook=sshHook,
+task_id='tk_product_prefer',
+command='cd /data/airflow/etl/agents && python tk_product_prefer.py',
+params={'my_param':"tk_product_prefer"},
+depends_on_past=False, 
+retries=1, 
+dag=dag)
+
+
+batch_gen_crawer = SSHOperator(
+ssh_hook=sshHook,
+task_id='batch_gen_crawer',
+command='cd /data/airflow/etl/agents && python batch_gen_crawer.py',
+params={'my_param':"batch_gen_crawer"},
+depends_on_past=False, 
+retries=1, 
+dag=dag)
+
+
+batch_gen_recruiment = SSHOperator(
+ssh_hook=sshHook,
+task_id='batch_gen_recruiment',
+command='cd /data/airflow/etl/agents && python batch_gen_recruiment.py',
+params={'my_param':"batch_gen_recruiment"},
+depends_on_past=False, 
+retries=1, 
+dag=dag)
+
+a_market_Intelligence_newsletter = SSHOperator(
+ssh_hook=sshHook,
+task_id='a_market_Intelligence_newsletter',
+command='/data/airflow/etl/COM/run_psql.sh {{ ds_nodash }} {{params.my_param}}  ',
+params={'my_param':"a_market_Intelligence_newsletter_agi"},
+depends_on_past=False, 
+retries=3, 
+dag=dag)
+
+dysql_news_letter_info = SSHOperator(
+ssh_hook=sshHook,
+task_id='dysql_news_letter_info',
+command='/data/airflow/etl/MART/run_psql.sh {{ ds_nodash }} {{params.my_param}}  ',
+params={'my_param':"dysql_news_letter_info"},
+depends_on_past=False, 
+retries=3, 
+dag=dag)
+
+
+
+agents_market_summar >> tk_product_prefer
+agents_market_summar >> batch_gen_crawer
+agents_recuriments >> batch_gen_recruiment
+a_market_Intelligence_newsletter >> tk_product_prefer
+tk_product_prefer >> dysql_news_letter_info
+agents_market_summar >> a_market_Intelligence_newsletter
+a_market_Intelligence_newsletter >> task_failed