diff --git a/dags/external_data_dag.py b/dags/external_data_dag.py index 16720102..1fb7954f 100644 --- a/dags/external_data_dag.py +++ b/dags/external_data_dag.py @@ -22,6 +22,7 @@ get_airflow_metadata, ) from stellar_etl_airflow.default import get_default_dag_args, init_sentry +from stellar_etl_airflow.utils import retrieve_secret init_sentry() @@ -65,7 +66,10 @@ "{{ subtract_data_interval(dag, data_interval_end).isoformat() }}", ], use_gcs=True, - env_vars={"RETOOL_API_KEY": "{{ var.value.retool_api_key }}"}, + env_vars={ + "RETOOL_API_KEY": "{{ var.value.retool_api_key }}", + "AIRFLOW_SECRET": retrieve_secret("retool-api-key-test"), + }, ) diff --git a/dags/stellar_etl_airflow/utils.py b/dags/stellar_etl_airflow/utils.py index a260fd9b..48698717 100644 --- a/dags/stellar_etl_airflow/utils.py +++ b/dags/stellar_etl_airflow/utils.py @@ -4,6 +4,9 @@ from airflow.configuration import conf from airflow.models import Variable +from airflow.providers.google.cloud.hooks.secret_manager import ( + GoogleCloudSecretManagerHook, +) from airflow.utils.state import TaskInstanceState base_log_folder = conf.get("logging", "base_log_folder") @@ -100,3 +103,14 @@ def skip_retry_dbt_errors(context) -> None: return else: return + + +def retrieve_secret(secret_name: str): + """ + Extracts secret from GCS secrets + """ + + hook = GoogleCloudSecretManagerHook() + + secret_value = hook.access_secret(secret_id=secret_name) + return secret_value