From 17e16437f7d0f912c2fb3522e95a18c0d57e844b Mon Sep 17 00:00:00 2001 From: Ran Lu Date: Sat, 23 Mar 2024 14:29:47 -0400 Subject: [PATCH] No longer need a separate config file for celery --- config/airflow.cfg | 4 ++-- config/celeryconfig.py | 14 -------------- 2 files changed, 2 insertions(+), 16 deletions(-) delete mode 100644 config/celeryconfig.py diff --git a/config/airflow.cfg b/config/airflow.cfg index 3cf5308e..47c32fd2 100644 --- a/config/airflow.cfg +++ b/config/airflow.cfg @@ -375,7 +375,7 @@ celeryd_concurrency = 1 # information. #broker_url = redis://redis:6379/1 #broker_url = amqp://172.31.31.249:5672 - +task_acks_late = True # Another key Celery setting #celery_result_backend = db+mysql://airflow:airflow@172.31.31.249:3306/airflow #celery_result_backend = db+postgresql://airflow:airflow@172.31.31.249/airflow @@ -388,7 +388,7 @@ flower_host = 0.0.0.0 flower_port = 5555 # Import path for celery configuration options -celery_config_options = celeryconfig.CELERY_CONFIG +#celery_config_options = celeryconfig.CELERY_CONFIG #celery_config_options = airflow.config_templates.default_celery.DEFAULT_CELERY_CONFIG ssl_active = False diff --git a/config/celeryconfig.py b/config/celeryconfig.py deleted file mode 100644 index 6075fec3..00000000 --- a/config/celeryconfig.py +++ /dev/null @@ -1,14 +0,0 @@ -from airflow import configuration - -# Broker settings. -CELERY_CONFIG = { - 'worker_prefetch_multiplier': 1, - 'task_acks_late': True, - 'broker_url': configuration.get('celery', 'broker_url'), - 'result_backend': configuration.get('celery', 'celery_result_backend'), - 'worker_concurrency': - configuration.getint('celery', 'celeryd_concurrency'), - 'task_default_queue': configuration.get('operators', 'default_queue'), - 'task_default_exchange': configuration.get('operators', 'default_queue'), - 'worker_send_task_events': False -}