diff --git a/redash/metrics/celery.py b/redash/metrics/celery.py index 25b63af136..ef7ed68cf2 100644 --- a/redash/metrics/celery.py +++ b/redash/metrics/celery.py @@ -12,7 +12,7 @@ @task_prerun.connect -def task_prerun_handler(signal, sender, task_id, task, args, kwargs): +def task_prerun_handler(signal, sender, task_id, task, args, kwargs, **kw): try: tasks_start_time[task_id] = time.time() except Exception: @@ -30,7 +30,7 @@ def metric_name(name, tags): @task_postrun.connect -def task_postrun_handler(signal, sender, task_id, task, args, kwargs, retval, state): +def task_postrun_handler(signal, sender, task_id, task, args, kwargs, retval, state, **kw): try: run_time = 1000 * (time.time() - tasks_start_time.pop(task_id)) diff --git a/redash/settings/__init__.py b/redash/settings/__init__.py index eb5f43aadd..664b60f908 100644 --- a/redash/settings/__init__.py +++ b/redash/settings/__init__.py @@ -32,8 +32,12 @@ def all_settings(): # Celery related settings CELERY_BROKER = os.environ.get("REDASH_CELERY_BROKER", REDIS_URL) -CELERY_BACKEND = os.environ.get("REDASH_CELERY_BACKEND", CELERY_BROKER) -CELERY_TASK_RESULT_EXPIRES = int(os.environ.get('REDASH_CELERY_TASK_RESULT_EXPIRES', 3600 * 4)) +CELERY_RESULT_BACKEND = os.environ.get( + "REDASH_CELERY_RESULT_BACKEND", + os.environ.get("REDASH_CELERY_BACKEND", CELERY_BROKER)) +CELERY_RESULT_EXPIRES = int(os.environ.get( + "REDASH_CELERY_RESULT_EXPIRES", + os.environ.get("REDASH_CELERY_TASK_RESULT_EXPIRES", 3600 * 4))) # The following enables periodic job (every 5 minutes) of removing unused query results. QUERY_RESULTS_CLEANUP_ENABLED = parse_boolean(os.environ.get("REDASH_QUERY_RESULTS_CLEANUP_ENABLED", "true")) @@ -107,8 +111,14 @@ def all_settings(): LOG_STDOUT = parse_boolean(os.environ.get('REDASH_LOG_STDOUT', 'false')) LOG_PREFIX = os.environ.get('REDASH_LOG_PREFIX', '') LOG_FORMAT = os.environ.get('REDASH_LOG_FORMAT', LOG_PREFIX + '[%(asctime)s][PID:%(process)d][%(levelname)s][%(name)s] %(message)s') -CELERYD_LOG_FORMAT = os.environ.get('REDASH_CELERYD_LOG_FORMAT', LOG_PREFIX + '[%(asctime)s][PID:%(process)d][%(levelname)s][%(processName)s] %(message)s') -CELERYD_TASK_LOG_FORMAT = os.environ.get('REDASH_CELERYD_TASK_LOG_FORMAT', LOG_PREFIX + '[%(asctime)s][PID:%(process)d][%(levelname)s][%(processName)s] task_name=%(task_name)s taks_id=%(task_id)s %(message)s') +CELERYD_WORKER_LOG_FORMAT = os.environ.get( + "REDASH_CELERYD_WORKER_LOG_FORMAT", + os.environ.get('REDASH_CELERYD_LOG_FORMAT', + LOG_PREFIX + '[%(asctime)s][PID:%(process)d][%(levelname)s][%(processName)s] %(message)s')) +CELERYD_WORKER_TASK_LOG_FORMAT = os.environ.get( + "REDASH_CELERYD_WORKER_TASK_LOG_FORMAT", + os.environ.get('REDASH_CELERYD_TASK_LOG_FORMAT', + LOG_PREFIX + '[%(asctime)s][PID:%(process)d][%(levelname)s][%(processName)s] task_name=%(task_name)s taks_id=%(task_id)s %(message)s')) # Mail settings: MAIL_SERVER = os.environ.get('REDASH_MAIL_SERVER', 'localhost') diff --git a/redash/worker.py b/redash/worker.py index ecab48ec20..629180b1f1 100644 --- a/redash/worker.py +++ b/redash/worker.py @@ -44,12 +44,12 @@ 'schedule': timedelta(minutes=5) } -celery.conf.update(CELERY_RESULT_BACKEND=settings.CELERY_BACKEND, - CELERYBEAT_SCHEDULE=celery_schedule, - CELERY_TIMEZONE='UTC', - CELERY_TASK_RESULT_EXPIRES=settings.CELERY_TASK_RESULT_EXPIRES, - CELERYD_LOG_FORMAT=settings.CELERYD_LOG_FORMAT, - CELERYD_TASK_LOG_FORMAT=settings.CELERYD_TASK_LOG_FORMAT) +celery.conf.update(result_backend=settings.CELERY_RESULT_BACKEND, + beat_schedule=celery_schedule, + timezone='UTC', + result_expires=settings.CELERY_RESULT_EXPIRES, + worker_log_format=settings.CELERYD_WORKER_LOG_FORMAT, + worker_task_log_format=settings.CELERYD_WORKER_TASK_LOG_FORMAT) if settings.SENTRY_DSN: from raven import Client diff --git a/requirements.txt b/requirements.txt index 7d6dcd7354..1387521e64 100644 --- a/requirements.txt +++ b/requirements.txt @@ -32,7 +32,7 @@ wsgiref==0.1.2 honcho==0.5.0 statsd==2.1.2 gunicorn==19.7.1 -celery==3.1.25 +celery==4.1.0 jsonschema==2.4.0 RestrictedPython==3.6.0 pysaml2==4.5.0