diff --git a/.gitignore b/.gitignore index 350e1f610d..ec0a379187 100644 --- a/.gitignore +++ b/.gitignore @@ -9,7 +9,6 @@ venv/ coverage.xml client/dist .DS_Store -celerybeat-schedule* .#* \#*# *~ diff --git a/redash/extensions.py b/redash/extensions.py index 8d7481ebb2..56452d1d13 100644 --- a/redash/extensions.py +++ b/redash/extensions.py @@ -6,11 +6,11 @@ # The global Redash extension registry extensions = odict() -# The periodic Celery tasks as provided by Redash extensions. -# This is separate from the internal periodic Celery tasks in -# celery_schedule since the extension task discovery phase is +# The periodic RQ jobs as provided by Redash extensions. +# This is separate from the internal periodic RQ jobs +# since the extension job discovery phase is # after the configuration has already happened. -periodic_tasks = odict() +periodic_jobs = odict() extension_logger = logging.getLogger(__name__) @@ -69,30 +69,30 @@ def extension(app): entry_point_loader("redash.extensions", extensions, logger=app.logger, app=app) -def load_periodic_tasks(logger=None): - """Load the periodic tasks as defined in Redash extensions. +def load_periodic_jobs(logger=None): + """Load the periodic jobs as defined in Redash extensions. The periodic task entry point needs to return a set of parameters - that can be passed to Celery's add_periodic_task: + that can be passed to RQ Scheduler API: - https://docs.celeryproject.org/en/latest/userguide/periodic-tasks.html#entries + https://github.com/rq/rq-scheduler#periodic--repeated-jobs E.g.:: def add_two_and_two(): return { - 'name': 'add 2 and 2 every 10 seconds' - 'sig': add.s(2, 2), - 'schedule': 10.0, # in seconds or a timedelta + "func": add, + "args": [2, 2] + "interval": 10, # in seconds or as a timedelta } - and then registered with an entry point under the "redash.periodic_tasks" + and then registered with an entry point under the "redash.periodic_jobs" group, e.g. in your setup.py:: setup( # ... entry_points={ - "redash.periodic_tasks": [ + "redash.periodic_jobs": [ "add_two_and_two = calculus.addition:add_two_and_two", ] # ... @@ -100,7 +100,7 @@ def add_two_and_two(): # ... ) """ - entry_point_loader("redash.periodic_tasks", periodic_tasks, logger=logger) + entry_point_loader("redash.periodic_jobs", periodic_jobs, logger=logger) def init_app(app): diff --git a/redash/tasks/schedule.py b/redash/tasks/schedule.py index 4b3e5e1644..bbce3b5024 100644 --- a/redash/tasks/schedule.py +++ b/redash/tasks/schedule.py @@ -3,13 +3,11 @@ import hashlib import json from datetime import datetime, timedelta -from functools import partial -from random import randint from rq.job import Job from rq_scheduler import Scheduler -from redash import settings, rq_redis_connection +from redash import extensions, settings, rq_redis_connection from redash.tasks import ( sync_user_details, refresh_queries, @@ -79,6 +77,10 @@ def periodic_job_definitions(): # Add your own custom periodic jobs in your dynamic_settings module. jobs.extend(settings.dynamic_settings.periodic_jobs() or []) + # Add periodic jobs that are shipped as part of Redash extensions + extensions.load_periodic_jobs(logger) + jobs.extend(list(extensions.periodic_jobs.values())) + return jobs diff --git a/tests/extensions/redash-dummy/redash_dummy.egg-info/entry_points.txt b/tests/extensions/redash-dummy/redash_dummy.egg-info/entry_points.txt index 6b28f51d18..3baf150ef9 100644 --- a/tests/extensions/redash-dummy/redash_dummy.egg-info/entry_points.txt +++ b/tests/extensions/redash-dummy/redash_dummy.egg-info/entry_points.txt @@ -5,6 +5,6 @@ not_findable_extension = redash_dummy:missing_attribute not_importable_extension = missing_extension_module:extension working_extension = redash_dummy:extension -[redash.periodic_tasks] -dummy_periodic_task = redash_dummy:periodic_task +[redash.periodic_jobs] +dummy_periodic_job = redash_dummy:periodic_job diff --git a/tests/extensions/redash-dummy/redash_dummy.py b/tests/extensions/redash-dummy/redash_dummy.py index 15276040b5..ee6f1571be 100644 --- a/tests/extensions/redash-dummy/redash_dummy.py +++ b/tests/extensions/redash-dummy/redash_dummy.py @@ -11,5 +11,5 @@ def assertive_extension(app): assert False -def periodic_task(*args, **kwargs): - """This periodic task will successfully load""" +def periodic_job(*args, **kwargs): + """This periodic job will successfully load""" diff --git a/tests/extensions/redash-dummy/setup.py b/tests/extensions/redash-dummy/setup.py index e1e79a96cb..496f03c41d 100644 --- a/tests/extensions/redash-dummy/setup.py +++ b/tests/extensions/redash-dummy/setup.py @@ -15,7 +15,7 @@ "not_importable_extension = missing_extension_module:extension", "assertive_extension = redash_dummy:assertive_extension", ], - "redash.periodic_tasks": ["dummy_periodic_task = redash_dummy:periodic_task"], + "redash.periodic_jobs": ["dummy_periodic_job = redash_dummy:periodic_job"], }, py_modules=["redash_dummy"], ) diff --git a/tests/extensions/test_extensions.py b/tests/extensions/test_extensions.py index dff1dbca6a..685959fa8d 100644 --- a/tests/extensions/test_extensions.py +++ b/tests/extensions/test_extensions.py @@ -41,5 +41,5 @@ def test_dummy_periodic_task(self): # need to load the periodic tasks manually since this isn't # done automatically on test suite start but only part of # the worker configuration - extensions.load_periodic_tasks(logger) - self.assertIn("dummy_periodic_task", extensions.periodic_tasks.keys()) + extensions.load_periodic_jobs(logger) + self.assertIn("dummy_periodic_job", extensions.periodic_jobs.keys())