add workflow 荟聚API,dev
This commit is contained in:
root
parent
b4e42dab1f
commit
a630221ac9
|
|
@ -0,0 +1,447 @@
|
|||
#!/usr/bin/python
|
||||
# -*- encoding=utf-8 -*-
|
||||
from airflow import DAG
|
||||
from datetime import datetime, timedelta
|
||||
from airflow.contrib.hooks.ssh_hook import SSHHook
|
||||
from airflow.contrib.operators.ssh_operator import SSHOperator
|
||||
from airflow.sensors.external_task_sensor import ExternalTaskSensor
|
||||
import json
|
||||
|
||||
from airflow.operators.email_operator import EmailOperator
|
||||
from airflow.utils.trigger_rule import TriggerRule
|
||||
|
||||
|
||||
sshHook = SSHHook(ssh_conn_id ='ssh_air')
|
||||
default_args = {
|
||||
'owner': 'info@idgvalue.com',
|
||||
'email': [''],
|
||||
'email_on_failure': True,
|
||||
'email_on_retry':True,
|
||||
'start_date': datetime(2022, 9, 12),
|
||||
'depends_on_past': False,
|
||||
'retries': 6,
|
||||
'retry_delay': timedelta(minutes=10),
|
||||
}
|
||||
|
||||
dag = DAG('wf_dag_tk_api', default_args=default_args,
|
||||
schedule_interval="0 0 * * *",
|
||||
catchup=False,
|
||||
dagrun_timeout=timedelta(minutes=160),
|
||||
max_active_runs=3)
|
||||
|
||||
task_failed = EmailOperator (
|
||||
dag=dag,
|
||||
trigger_rule=TriggerRule.ONE_FAILED,
|
||||
task_id="task_failed",
|
||||
to=["info@idgvalue.com"],
|
||||
cc=[""],
|
||||
subject="tk_api_failed",
|
||||
html_content='<h3>您好，tk_api作业失败，请及时处理" </h3>')
|
||||
|
||||
scrm_contact_feign = SSHOperator(
|
||||
ssh_hook=sshHook,
|
||||
task_id='scrm_contact_feign',
|
||||
command='python3 /data/airflow/etl/API/scrm_contact_feign.py',
|
||||
depends_on_past=False,
|
||||
retries=3,
|
||||
dag=dag)
|
||||
|
||||
scrm_contact_load = SSHOperator(
|
||||
ssh_hook=sshHook,
|
||||
task_id='scrm_contact_load',
|
||||
command='/data/airflow/etl/API/run_psql.sh {{ ds_nodash }} {{params.my_param }}',
|
||||
params={'my_param':"scrm_contact_load"},
|
||||
depends_on_past=False,
|
||||
retries=3,
|
||||
dag=dag)
|
||||
|
||||
scrm_contact_feign >> scrm_contact_load
|
||||
|
||||
scrm_contact_1129 = SSHOperator(
|
||||
ssh_hook=sshHook,
|
||||
task_id='scrm_contact_1129',
|
||||
command='/data/airflow/etl/SA/run_sa.sh {{ ds_nodash }} {{ params.my_param }} >>/data/airflow/logs/run_psql_{{ds_nodash}}.log 2>&1 ',
|
||||
params={'my_param':"S98_S_scrm_contact"},
|
||||
depends_on_past=False,
|
||||
retries=3,
|
||||
dag=dag)
|
||||
|
||||
custom_events_update_feign = SSHOperator(
|
||||
ssh_hook=sshHook,
|
||||
task_id='custom_events_update_feign',
|
||||
command='python3 /data/airflow/etl/API/custom_events_update_feign.py',
|
||||
depends_on_past=False,
|
||||
retries=3,
|
||||
dag=dag)
|
||||
|
||||
custom_events_update_load = SSHOperator(
|
||||
ssh_hook=sshHook,
|
||||
task_id='custom_events_update_load',
|
||||
command='/data/airflow/etl/API/run_psql.sh {{ ds_nodash }} {{params.my_param }}',
|
||||
params={'my_param':"custom_events_update_load"},
|
||||
depends_on_past=False,
|
||||
retries=3,
|
||||
dag=dag)
|
||||
|
||||
custom_events_update_feign >> custom_events_update_load
|
||||
|
||||
update_scrm_contact_feign = SSHOperator(
|
||||
ssh_hook=sshHook,
|
||||
task_id='update_scrm_contact_feign',
|
||||
command='python3 /data/airflow/etl/API/update_scrm_contact_feign.py',
|
||||
depends_on_past=False,
|
||||
retries=3,
|
||||
dag=dag)
|
||||
|
||||
update_scrm_contact_load = SSHOperator(
|
||||
ssh_hook=sshHook,
|
||||
task_id='update_scrm_contact_load',
|
||||
command='/data/airflow/etl/API/run_psql.sh {{ ds_nodash }} {{params.my_param }}',
|
||||
params={'my_param':"update_scrm_contact_load"},
|
||||
depends_on_past=False,
|
||||
retries=3,
|
||||
dag=dag)
|
||||
|
||||
update_scrm_contact_feign >> update_scrm_contact_load
|
||||
|
||||
customer_events_feign = SSHOperator(
|
||||
ssh_hook=sshHook,
|
||||
task_id='customer_events_feign',
|
||||
command='python3 /data/airflow/etl/API/customer_events_feign.py',
|
||||
depends_on_past=False,
|
||||
retries=3,
|
||||
dag=dag)
|
||||
|
||||
customer_events_load = SSHOperator(
|
||||
ssh_hook=sshHook,
|
||||
task_id='customer_events_load',
|
||||
command='/data/airflow/etl/API/run_psql.sh {{ ds_nodash }} {{params.my_param }}',
|
||||
params={'my_param':"customer_events_load"},
|
||||
depends_on_past=False,
|
||||
retries=3,
|
||||
dag=dag)
|
||||
|
||||
customer_events_feign >> customer_events_load
|
||||
|
||||
customer_event_meta_feign = SSHOperator(
|
||||
ssh_hook=sshHook,
|
||||
task_id='customer_event_meta_feign',
|
||||
command='python3 /data/airflow/etl/API/customer_event_meta_feign.py',
|
||||
depends_on_past=False,
|
||||
retries=3,
|
||||
dag=dag)
|
||||
|
||||
customer_event_meta_load = SSHOperator(
|
||||
ssh_hook=sshHook,
|
||||
task_id='customer_event_meta_load',
|
||||
command='/data/airflow/etl/API/run_psql.sh {{ ds_nodash }} {{params.my_param }}',
|
||||
params={'my_param':"customer_event_meta_load"},
|
||||
depends_on_past=False,
|
||||
retries=3,
|
||||
dag=dag)
|
||||
|
||||
customer_event_meta_feign >> customer_event_meta_load
|
||||
|
||||
scrm_contact_lastupdated_4112 = SSHOperator(
|
||||
ssh_hook=sshHook,
|
||||
task_id='scrm_contact_lastupdated_4112',
|
||||
command='/data/airflow/etl/SA/run_sa.sh {{ ds_nodash }} {{ params.my_param }} >>/data/airflow/logs/run_psql_{{ds_nodash}}.log 2>&1 ',
|
||||
params={'my_param':"S98_S_scrm_contact_lastupdated"},
|
||||
depends_on_past=False,
|
||||
retries=3,
|
||||
dag=dag)
|
||||
|
||||
customer_event_meta_2268 = SSHOperator(
|
||||
ssh_hook=sshHook,
|
||||
task_id='customer_event_meta_2268',
|
||||
command='/data/airflow/etl/SA/run_sa.sh {{ ds_nodash }} {{ params.my_param }} >>/data/airflow/logs/run_psql_{{ds_nodash}}.log 2>&1 ',
|
||||
params={'my_param':"S98_S_customer_event_meta"},
|
||||
depends_on_past=False,
|
||||
retries=3,
|
||||
dag=dag)
|
||||
|
||||
customer_events_lastupdated_8895 = SSHOperator(
|
||||
ssh_hook=sshHook,
|
||||
task_id='customer_events_lastupdated_8895',
|
||||
command='/data/airflow/etl/SA/run_sa.sh {{ ds_nodash }} {{ params.my_param }} >>/data/airflow/logs/run_psql_{{ds_nodash}}.log 2>&1 ',
|
||||
params={'my_param':"S98_S_customer_events_lastupdated"},
|
||||
depends_on_past=False,
|
||||
retries=3,
|
||||
dag=dag)
|
||||
|
||||
customer_events_3292 = SSHOperator(
|
||||
ssh_hook=sshHook,
|
||||
task_id='customer_events_3292',
|
||||
command='/data/airflow/etl/SA/run_sa.sh {{ ds_nodash }} {{ params.my_param }} >>/data/airflow/logs/run_psql_{{ds_nodash}}.log 2>&1 ',
|
||||
params={'my_param':"S98_S_customer_events"},
|
||||
depends_on_past=False,
|
||||
retries=3,
|
||||
dag=dag)
|
||||
|
||||
custom_events_9_feign = SSHOperator(
|
||||
ssh_hook=sshHook,
|
||||
task_id='custom_events_9_feign',
|
||||
command='python3 /data/airflow/etl/API/custom_events_9_feign.py',
|
||||
depends_on_past=False,
|
||||
retries=3,
|
||||
dag=dag)
|
||||
|
||||
custom_events_9_load = SSHOperator(
|
||||
ssh_hook=sshHook,
|
||||
task_id='custom_events_9_load',
|
||||
command='/data/airflow/etl/API/run_psql.sh {{ ds_nodash }} {{params.my_param }}',
|
||||
params={'my_param':"custom_events_9_load"},
|
||||
depends_on_past=False,
|
||||
retries=3,
|
||||
dag=dag)
|
||||
|
||||
custom_events_9_feign >> custom_events_9_load
|
||||
|
||||
custom_events_6_feign = SSHOperator(
|
||||
ssh_hook=sshHook,
|
||||
task_id='custom_events_6_feign',
|
||||
command='python3 /data/airflow/etl/API/custom_events_6_feign.py',
|
||||
depends_on_past=False,
|
||||
retries=3,
|
||||
dag=dag)
|
||||
|
||||
custom_events_6_load = SSHOperator(
|
||||
ssh_hook=sshHook,
|
||||
task_id='custom_events_6_load',
|
||||
command='/data/airflow/etl/API/run_psql.sh {{ ds_nodash }} {{params.my_param }}',
|
||||
params={'my_param':"custom_events_6_load"},
|
||||
depends_on_past=False,
|
||||
retries=3,
|
||||
dag=dag)
|
||||
|
||||
custom_events_6_feign >> custom_events_6_load
|
||||
|
||||
custom_events_4_feign = SSHOperator(
|
||||
ssh_hook=sshHook,
|
||||
task_id='custom_events_4_feign',
|
||||
command='python3 /data/airflow/etl/API/custom_events_4_feign.py',
|
||||
depends_on_past=False,
|
||||
retries=3,
|
||||
dag=dag)
|
||||
|
||||
custom_events_4_load = SSHOperator(
|
||||
ssh_hook=sshHook,
|
||||
task_id='custom_events_4_load',
|
||||
command='/data/airflow/etl/API/run_psql.sh {{ ds_nodash }} {{params.my_param }}',
|
||||
params={'my_param':"custom_events_4_load"},
|
||||
depends_on_past=False,
|
||||
retries=3,
|
||||
dag=dag)
|
||||
|
||||
custom_events_4_feign >> custom_events_4_load
|
||||
|
||||
custom_events_8_feign = SSHOperator(
|
||||
ssh_hook=sshHook,
|
||||
task_id='custom_events_8_feign',
|
||||
command='python3 /data/airflow/etl/API/custom_events_8_feign.py',
|
||||
depends_on_past=False,
|
||||
retries=3,
|
||||
dag=dag)
|
||||
|
||||
custom_events_8_load = SSHOperator(
|
||||
ssh_hook=sshHook,
|
||||
task_id='custom_events_8_load',
|
||||
command='/data/airflow/etl/API/run_psql.sh {{ ds_nodash }} {{params.my_param }}',
|
||||
params={'my_param':"custom_events_8_load"},
|
||||
depends_on_past=False,
|
||||
retries=3,
|
||||
dag=dag)
|
||||
|
||||
custom_events_8_feign >> custom_events_8_load
|
||||
|
||||
custom_events_3_feign = SSHOperator(
|
||||
ssh_hook=sshHook,
|
||||
task_id='custom_events_3_feign',
|
||||
command='python3 /data/airflow/etl/API/custom_events_3_feign.py',
|
||||
depends_on_past=False,
|
||||
retries=3,
|
||||
dag=dag)
|
||||
|
||||
custom_events_3_load = SSHOperator(
|
||||
ssh_hook=sshHook,
|
||||
task_id='custom_events_3_load',
|
||||
command='/data/airflow/etl/API/run_psql.sh {{ ds_nodash }} {{params.my_param }}',
|
||||
params={'my_param':"custom_events_3_load"},
|
||||
depends_on_past=False,
|
||||
retries=3,
|
||||
dag=dag)
|
||||
|
||||
custom_events_3_feign >> custom_events_3_load
|
||||
|
||||
custom_events_1_feign = SSHOperator(
|
||||
ssh_hook=sshHook,
|
||||
task_id='custom_events_1_feign',
|
||||
command='python3 /data/airflow/etl/API/custom_events_1_feign.py',
|
||||
depends_on_past=False,
|
||||
retries=3,
|
||||
dag=dag)
|
||||
|
||||
custom_events_1_load = SSHOperator(
|
||||
ssh_hook=sshHook,
|
||||
task_id='custom_events_1_load',
|
||||
command='/data/airflow/etl/API/run_psql.sh {{ ds_nodash }} {{params.my_param }}',
|
||||
params={'my_param':"custom_events_1_load"},
|
||||
depends_on_past=False,
|
||||
retries=3,
|
||||
dag=dag)
|
||||
|
||||
custom_events_1_feign >> custom_events_1_load
|
||||
|
||||
custom_events_7_feign = SSHOperator(
|
||||
ssh_hook=sshHook,
|
||||
task_id='custom_events_7_feign',
|
||||
command='python3 /data/airflow/etl/API/custom_events_7_feign.py',
|
||||
depends_on_past=False,
|
||||
retries=3,
|
||||
dag=dag)
|
||||
|
||||
custom_events_7_load = SSHOperator(
|
||||
ssh_hook=sshHook,
|
||||
task_id='custom_events_7_load',
|
||||
command='/data/airflow/etl/API/run_psql.sh {{ ds_nodash }} {{params.my_param }}',
|
||||
params={'my_param':"custom_events_7_load"},
|
||||
depends_on_past=False,
|
||||
retries=3,
|
||||
dag=dag)
|
||||
|
||||
custom_events_7_feign >> custom_events_7_load
|
||||
|
||||
custom_events_2_feign = SSHOperator(
|
||||
ssh_hook=sshHook,
|
||||
task_id='custom_events_2_feign',
|
||||
command='python3 /data/airflow/etl/API/custom_events_2_feign.py',
|
||||
depends_on_past=False,
|
||||
retries=3,
|
||||
dag=dag)
|
||||
|
||||
custom_events_2_load = SSHOperator(
|
||||
ssh_hook=sshHook,
|
||||
task_id='custom_events_2_load',
|
||||
command='/data/airflow/etl/API/run_psql.sh {{ ds_nodash }} {{params.my_param }}',
|
||||
params={'my_param':"custom_events_2_load"},
|
||||
depends_on_past=False,
|
||||
retries=3,
|
||||
dag=dag)
|
||||
|
||||
custom_events_2_feign >> custom_events_2_load
|
||||
|
||||
custom_events_5_feign = SSHOperator(
|
||||
ssh_hook=sshHook,
|
||||
task_id='custom_events_5_feign',
|
||||
command='python3 /data/airflow/etl/API/custom_events_5_feign.py',
|
||||
depends_on_past=False,
|
||||
retries=3,
|
||||
dag=dag)
|
||||
|
||||
custom_events_5_load = SSHOperator(
|
||||
ssh_hook=sshHook,
|
||||
task_id='custom_events_5_load',
|
||||
command='/data/airflow/etl/API/run_psql.sh {{ ds_nodash }} {{params.my_param }}',
|
||||
params={'my_param':"custom_events_5_load"},
|
||||
depends_on_past=False,
|
||||
retries=3,
|
||||
dag=dag)
|
||||
|
||||
custom_events_5_feign >> custom_events_5_load
|
||||
|
||||
customer_events_submit_form_4283 = SSHOperator(
|
||||
ssh_hook=sshHook,
|
||||
task_id='customer_events_submit_form_4283',
|
||||
command='/data/airflow/etl/SA/run_sa.sh {{ ds_nodash }} {{ params.my_param }} >>/data/airflow/logs/run_psql_{{ds_nodash}}.log 2>&1 ',
|
||||
params={'my_param':"S98_S_customer_events_submit_form"},
|
||||
depends_on_past=False,
|
||||
retries=3,
|
||||
dag=dag)
|
||||
|
||||
customer_events_wechat_scan_2105 = SSHOperator(
|
||||
ssh_hook=sshHook,
|
||||
task_id='customer_events_wechat_scan_2105',
|
||||
command='/data/airflow/etl/SA/run_sa.sh {{ ds_nodash }} {{ params.my_param }} >>/data/airflow/logs/run_psql_{{ds_nodash}}.log 2>&1 ',
|
||||
params={'my_param':"S98_S_customer_events_wechat_scan"},
|
||||
depends_on_past=False,
|
||||
retries=3,
|
||||
dag=dag)
|
||||
|
||||
customer_events_subscribe_5997 = SSHOperator(
|
||||
ssh_hook=sshHook,
|
||||
task_id='customer_events_subscribe_5997',
|
||||
command='/data/airflow/etl/SA/run_sa.sh {{ ds_nodash }} {{ params.my_param }} >>/data/airflow/logs/run_psql_{{ds_nodash}}.log 2>&1 ',
|
||||
params={'my_param':"S98_S_customer_events_subscribe"},
|
||||
depends_on_past=False,
|
||||
retries=3,
|
||||
dag=dag)
|
||||
|
||||
customer_events_click_link_in_page_2034 = SSHOperator(
|
||||
ssh_hook=sshHook,
|
||||
task_id='customer_events_click_link_in_page_2034',
|
||||
command='/data/airflow/etl/SA/run_sa.sh {{ ds_nodash }} {{ params.my_param }} >>/data/airflow/logs/run_psql_{{ds_nodash}}.log 2>&1 ',
|
||||
params={'my_param':"S98_S_customer_events_click_link_in_page"},
|
||||
depends_on_past=False,
|
||||
retries=3,
|
||||
dag=dag)
|
||||
|
||||
customer_events_open_content_page_9684 = SSHOperator(
|
||||
ssh_hook=sshHook,
|
||||
task_id='customer_events_open_content_page_9684',
|
||||
command='/data/airflow/etl/SA/run_sa.sh {{ ds_nodash }} {{ params.my_param }} >>/data/airflow/logs/run_psql_{{ds_nodash}}.log 2>&1 ',
|
||||
params={'my_param':"S98_S_customer_events_open_content_page"},
|
||||
depends_on_past=False,
|
||||
retries=3,
|
||||
dag=dag)
|
||||
|
||||
customer_events_open_app_5878 = SSHOperator(
|
||||
ssh_hook=sshHook,
|
||||
task_id='customer_events_open_app_5878',
|
||||
command='/data/airflow/etl/SA/run_sa.sh {{ ds_nodash }} {{ params.my_param }} >>/data/airflow/logs/run_psql_{{ds_nodash}}.log 2>&1 ',
|
||||
params={'my_param':"S98_S_customer_events_open_app"},
|
||||
depends_on_past=False,
|
||||
retries=3,
|
||||
dag=dag)
|
||||
|
||||
customer_events_open_app_5878 = SSHOperator(
|
||||
ssh_hook=sshHook,
|
||||
task_id='customer_events_open_app_5878',
|
||||
command='/data/airflow/etl/SA/run_sa.sh {{ ds_nodash }} {{ params.my_param }} >>/data/airflow/logs/run_psql_{{ds_nodash}}.log 2>&1 ',
|
||||
params={'my_param':"S98_S_customer_events_open_app"},
|
||||
depends_on_past=False,
|
||||
retries=3,
|
||||
dag=dag)
|
||||
|
||||
customer_events_open_page_7519 = SSHOperator(
|
||||
ssh_hook=sshHook,
|
||||
task_id='customer_events_open_page_7519',
|
||||
command='/data/airflow/etl/SA/run_sa.sh {{ ds_nodash }} {{ params.my_param }} >>/data/airflow/logs/run_psql_{{ds_nodash}}.log 2>&1 ',
|
||||
params={'my_param':"S98_S_customer_events_open_page"},
|
||||
depends_on_past=False,
|
||||
retries=3,
|
||||
dag=dag)
|
||||
|
||||
customer_events_c_minipro_page_view_9309 = SSHOperator(
|
||||
ssh_hook=sshHook,
|
||||
task_id='customer_events_c_minipro_page_view_9309',
|
||||
command='/data/airflow/etl/SA/run_sa.sh {{ ds_nodash }} {{ params.my_param }} >>/data/airflow/logs/run_psql_{{ds_nodash}}.log 2>&1 ',
|
||||
params={'my_param':"S98_S_customer_events_c_minipro_page_view"},
|
||||
depends_on_past=False,
|
||||
retries=3,
|
||||
dag=dag)
|
||||
|
||||
scrm_contact_load >> scrm_contact_1129
|
||||
update_scrm_contact_load >> scrm_contact_lastupdated_4112
|
||||
customer_event_meta_load >> customer_event_meta_2268
|
||||
custom_events_update_load >> customer_events_lastupdated_8895
|
||||
customer_events_load >> customer_events_3292
|
||||
custom_events_9_load >> customer_events_wechat_scan_2105
|
||||
custom_events_6_load >> customer_events_submit_form_4283
|
||||
custom_events_4_load >> customer_events_subscribe_5997
|
||||
custom_events_8_load >> customer_events_click_link_in_page_2034
|
||||
custom_events_1_load >> customer_events_open_app_5878
|
||||
custom_events_7_load >> customer_events_open_app_5878
|
||||
custom_events_2_load >> customer_events_open_content_page_9684
|
||||
custom_events_3_load >> customer_events_open_page_7519
|
||||
custom_events_5_load >> customer_events_c_minipro_page_view_9309
|
||||
customer_events_c_minipro_page_view_9309 >> task_failed
|
||||
Loading…
Reference in New Issue