From ac642df3efbe19d41ea4c0f8e0edba0b8dca94fb Mon Sep 17 00:00:00 2001 From: Atalya Alon <20992625+atalyaalon@users.noreply.github.com> Date: Thu, 27 Apr 2023 13:25:26 +0300 Subject: [PATCH] Add kwargs (#40) * add kwarg * add kwarg * add kwarg --- .../dags/import_email_to_s3_and_update_data.py | 12 ++++-------- airflow_server/dags/infographics_generate_image.py | 3 ++- 2 files changed, 6 insertions(+), 9 deletions(-) diff --git a/airflow_server/dags/import_email_to_s3_and_update_data.py b/airflow_server/dags/import_email_to_s3_and_update_data.py index 674bf8f..ba77892 100644 --- a/airflow_server/dags/import_email_to_s3_and_update_data.py +++ b/airflow_server/dags/import_email_to_s3_and_update_data.py @@ -14,17 +14,13 @@ with DAG('import-email-to-s3-and-update-data', **dag_kwargs) as import_email_to_s3_and_update_data: - CliBashOperator( - 'anyway-etl anyway-kubectl-exec python3 main.py scripts importemail', + CliBashOperator(cmd='anyway-etl anyway-kubectl-exec python3 main.py scripts importemail', task_id='import-email-to-s3' - ) >> CliBashOperator( - 'anyway-etl anyway-kubectl-exec python3 main.py process cbs --source s3' + ) >> CliBashOperator(cmd='anyway-etl anyway-kubectl-exec python3 main.py process cbs --source s3' '{% if dag_run.conf.get("load_start_year") %} --load_start_year {{ dag_run.conf["load_start_year"] }}{% endif %}', task_id='cbs-import-from-s3' - ) >> CliBashOperator( - 'anyway-etl anyway-kubectl-exec python3 main.py process infographics-data-cache-for-road-segments', + ) >> CliBashOperator(cmd='anyway-etl anyway-kubectl-exec python3 main.py process infographics-data-cache-for-road-segments', task_id='fill-infographics-cache-for-road-segments' - ) >> CliBashOperator( - 'anyway-etl anyway-kubectl-exec python3 main.py process cache update-street', + ) >> CliBashOperator(cmd='anyway-etl anyway-kubectl-exec python3 main.py process cache update-street', task_id='fill-infographics-cache-for-streets' ) diff --git a/airflow_server/dags/infographics_generate_image.py b/airflow_server/dags/infographics_generate_image.py index 73635fa..cdeddce 100644 --- a/airflow_server/dags/infographics_generate_image.py +++ b/airflow_server/dags/infographics_generate_image.py @@ -1,6 +1,6 @@ from airflow import DAG from airflow.models.param import Param - +import pendulum from anyway_etl_airflow.operators.cli_bash_operator import CliBashOperator @@ -8,6 +8,7 @@ default_args={ 'owner': 'airflow', }, + start_date=pendulum.datetime(2023, 4, 1, tz="Asia/Jerusalem") )