From 49a5c518c9a8389bb88bc2c4eb8bbea07a6e3ac6 Mon Sep 17 00:00:00 2001 From: Eduardo Alves Date: Wed, 3 Jul 2024 17:23:44 -0300 Subject: [PATCH 1/3] Added dbt monitoring packages dag --- dags/dbt_monitoring_packages_dag.py | 33 +++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) create mode 100644 dags/dbt_monitoring_packages_dag.py diff --git a/dags/dbt_monitoring_packages_dag.py b/dags/dbt_monitoring_packages_dag.py new file mode 100644 index 00000000..eedf99b7 --- /dev/null +++ b/dags/dbt_monitoring_packages_dag.py @@ -0,0 +1,33 @@ +from datetime import datetime + +from airflow import DAG +from airflow.operators.empty import EmptyOperator +from kubernetes.client import models as k8s +from stellar_etl_airflow.build_dbt_task import dbt_task +from stellar_etl_airflow.default import get_default_dag_args, init_sentry + +init_sentry() + +with DAG( + "dbt_monitoring_packages", + default_args=get_default_dag_args(), + start_date=datetime(2024, 7, 3, 0, 0), + description="This DAG runs dbt build related to data observability and monitoring tag", + schedule="0 8 * * *", + user_defined_filters={ + "container_resources": lambda s: k8s.V1ResourceRequirements(requests=s), + }, + max_active_runs=1, + catchup=False, + tags=["monitoring"], +) as dag: + + # DBT tests to run + monitoring_task = dbt_task( + dag, + tag="monitoring_packages", + ) + start_task = EmptyOperator(task_id="start_monitoring_task") + + # DAG task graph + start_task >> monitoring_task From ad42e78dfce375a74662fd4243a5124bc238a1c1 Mon Sep 17 00:00:00 2001 From: Eduardo Alves Date: Wed, 17 Jul 2024 17:20:20 -0300 Subject: [PATCH 2/3] Updated stellar-dbt image --- airflow_variables_dev.json | 2 +- airflow_variables_prod.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/airflow_variables_dev.json b/airflow_variables_dev.json index 496e1e3a..31889329 100644 --- a/airflow_variables_dev.json +++ b/airflow_variables_dev.json @@ -122,7 +122,7 @@ "partnership_assets__account_holders_activity_fact": false, "partnership_assets__asset_activity_fact": false }, - "dbt_image_name": "stellar/stellar-dbt:1a777f9", + "dbt_image_name": "stellar/stellar-dbt:5c7c924", "dbt_internal_source_db": "test-hubble-319619", "dbt_internal_source_schema": "test_crypto_stellar_internal", "dbt_job_execution_timeout_seconds": 300, diff --git a/airflow_variables_prod.json b/airflow_variables_prod.json index b05d2b70..1c6bb2e6 100644 --- a/airflow_variables_prod.json +++ b/airflow_variables_prod.json @@ -123,7 +123,7 @@ "partnership_assets__asset_activity_fact": false, "trade_agg": false }, - "dbt_image_name": "stellar/stellar-dbt:1a777f9", + "dbt_image_name": "stellar/stellar-dbt:5c7c924", "dbt_internal_source_db": "hubble-261722", "dbt_internal_source_schema": "crypto_stellar_internal_2", "dbt_job_execution_timeout_seconds": 1800, From 8da396c614402d1ab1e34353c5fa6e34f3bc019e Mon Sep 17 00:00:00 2001 From: Eduardo Alves Date: Wed, 17 Jul 2024 17:38:49 -0300 Subject: [PATCH 3/3] Added missing variables --- airflow_variables_dev.json | 1 + airflow_variables_prod.json | 1 + 2 files changed, 2 insertions(+) diff --git a/airflow_variables_dev.json b/airflow_variables_dev.json index 31889329..1a1724e4 100644 --- a/airflow_variables_dev.json +++ b/airflow_variables_dev.json @@ -359,6 +359,7 @@ "liquidity_pools_value_history": 600, "liquidity_providers": 720, "mgi": 660, + "monitoring_packages": 720, "network_stats": 720, "ohlc": 720, "partnership_assets": 660, diff --git a/airflow_variables_prod.json b/airflow_variables_prod.json index 1c6bb2e6..0a7c3ab5 100644 --- a/airflow_variables_prod.json +++ b/airflow_variables_prod.json @@ -357,6 +357,7 @@ "liquidity_pools_value_history": 360, "liquidity_providers": 720, "mgi": 1020, + "monitoring_packages": 720, "network_stats": 360, "ohlc": 960, "partnership_assets": 1380,