upgrade to Celery 4.1.0

mozilla · Mar 5, 2018 · de4a5a1 · de4a5a1
1 parent 357a8d4
commit de4a5a1
Show file tree

Hide file tree

Showing 4 changed files with 23 additions and 13 deletions.
diff --git a/redash/metrics/celery.py b/redash/metrics/celery.py
@@ -12,7 +12,7 @@
 
 
 @task_prerun.connect
-def task_prerun_handler(signal, sender, task_id, task, args, kwargs):
+def task_prerun_handler(signal, sender, task_id, task, args, kwargs, **kw):
     try:
         tasks_start_time[task_id] = time.time()
     except Exception:
@@ -30,7 +30,7 @@ def metric_name(name, tags):
 
 
 @task_postrun.connect
-def task_postrun_handler(signal, sender, task_id, task, args, kwargs, retval, state):
+def task_postrun_handler(signal, sender, task_id, task, args, kwargs, retval, state, **kw):
     try:
         run_time = 1000 * (time.time() - tasks_start_time.pop(task_id))
 

diff --git a/redash/settings/__init__.py b/redash/settings/__init__.py
@@ -32,8 +32,12 @@ def all_settings():
 
 # Celery related settings
 CELERY_BROKER = os.environ.get("REDASH_CELERY_BROKER", REDIS_URL)
-CELERY_BACKEND = os.environ.get("REDASH_CELERY_BACKEND", CELERY_BROKER)
-CELERY_TASK_RESULT_EXPIRES = int(os.environ.get('REDASH_CELERY_TASK_RESULT_EXPIRES', 3600 * 4))
+CELERY_RESULT_BACKEND = os.environ.get(
+    "REDASH_CELERY_RESULT_BACKEND",
+    os.environ.get("REDASH_CELERY_BACKEND", CELERY_BROKER))
+CELERY_RESULT_EXPIRES = int(os.environ.get(
+    "REDASH_CELERY_RESULT_EXPIRES",
+    os.environ.get("REDASH_CELERY_TASK_RESULT_EXPIRES", 3600 * 4)))
 
 # The following enables periodic job (every 5 minutes) of removing unused query results.
 QUERY_RESULTS_CLEANUP_ENABLED = parse_boolean(os.environ.get("REDASH_QUERY_RESULTS_CLEANUP_ENABLED", "true"))
@@ -107,8 +111,14 @@ def all_settings():
 LOG_STDOUT = parse_boolean(os.environ.get('REDASH_LOG_STDOUT', 'false'))
 LOG_PREFIX = os.environ.get('REDASH_LOG_PREFIX', '')
 LOG_FORMAT = os.environ.get('REDASH_LOG_FORMAT', LOG_PREFIX + '[%(asctime)s][PID:%(process)d][%(levelname)s][%(name)s] %(message)s')
-CELERYD_LOG_FORMAT = os.environ.get('REDASH_CELERYD_LOG_FORMAT', LOG_PREFIX + '[%(asctime)s][PID:%(process)d][%(levelname)s][%(processName)s] %(message)s')
-CELERYD_TASK_LOG_FORMAT = os.environ.get('REDASH_CELERYD_TASK_LOG_FORMAT', LOG_PREFIX + '[%(asctime)s][PID:%(process)d][%(levelname)s][%(processName)s] task_name=%(task_name)s taks_id=%(task_id)s %(message)s')
+CELERYD_WORKER_LOG_FORMAT = os.environ.get(
+    "REDASH_CELERYD_WORKER_LOG_FORMAT",
+    os.environ.get('REDASH_CELERYD_LOG_FORMAT',
+                   LOG_PREFIX + '[%(asctime)s][PID:%(process)d][%(levelname)s][%(processName)s] %(message)s'))
+CELERYD_WORKER_TASK_LOG_FORMAT = os.environ.get(
+    "REDASH_CELERYD_WORKER_TASK_LOG_FORMAT",
+    os.environ.get('REDASH_CELERYD_TASK_LOG_FORMAT',
+                   LOG_PREFIX + '[%(asctime)s][PID:%(process)d][%(levelname)s][%(processName)s] task_name=%(task_name)s taks_id=%(task_id)s %(message)s'))
 
 # Mail settings:
 MAIL_SERVER = os.environ.get('REDASH_MAIL_SERVER', 'localhost')

diff --git a/redash/worker.py b/redash/worker.py
@@ -44,12 +44,12 @@
         'schedule': timedelta(minutes=5)
     }
 
-celery.conf.update(CELERY_RESULT_BACKEND=settings.CELERY_BACKEND,
-                   CELERYBEAT_SCHEDULE=celery_schedule,
-                   CELERY_TIMEZONE='UTC',
-                   CELERY_TASK_RESULT_EXPIRES=settings.CELERY_TASK_RESULT_EXPIRES,
-                   CELERYD_LOG_FORMAT=settings.CELERYD_LOG_FORMAT,
-                   CELERYD_TASK_LOG_FORMAT=settings.CELERYD_TASK_LOG_FORMAT)
+celery.conf.update(result_backend=settings.CELERY_RESULT_BACKEND,
+                   beat_schedule=celery_schedule,
+                   timezone='UTC',
+                   result_expires=settings.CELERY_RESULT_EXPIRES,
+                   worker_log_format=settings.CELERYD_WORKER_LOG_FORMAT,
+                   worker_task_log_format=settings.CELERYD_WORKER_TASK_LOG_FORMAT)
 
 if settings.SENTRY_DSN:
     from raven import Client

diff --git a/requirements.txt b/requirements.txt
@@ -32,7 +32,7 @@ wsgiref==0.1.2
 honcho==0.5.0
 statsd==2.1.2
 gunicorn==19.7.1
-celery==3.1.25
+celery==4.1.0
 jsonschema==2.4.0
 RestrictedPython==3.6.0
 pysaml2==4.5.0