From 408d7807b0931966cc825b70db5dda11fcb5b961 Mon Sep 17 00:00:00 2001 From: Jessie Date: Fri, 24 Apr 2020 13:20:56 -0400 Subject: [PATCH 1/6] =?UTF-8?q?[BEVY-24]=20=E2=9C=A8=20(Feature)=20Add=20@?= =?UTF-8?q?periodic=5Ftask=20decorator=20(#1)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .gitignore | 1 + django_celery_beat/decorators.py | 104 ++++++++++++++ docker-compose.yml | 12 +- docker/{celery-beat => celery}/Dockerfile | 2 +- docker/{celery-beat => celery}/entrypoint.sh | 0 requirements/runtime.txt | 1 + t/unit/test_decorators.py | 142 +++++++++++++++++++ t/unit/test_schedulers.py | 1 + 8 files changed, 257 insertions(+), 6 deletions(-) create mode 100644 django_celery_beat/decorators.py rename docker/{celery-beat => celery}/Dockerfile (77%) rename docker/{celery-beat => celery}/entrypoint.sh (100%) create mode 100644 t/unit/test_decorators.py diff --git a/.gitignore b/.gitignore index 250f840f..2cea589f 100644 --- a/.gitignore +++ b/.gitignore @@ -32,3 +32,4 @@ coverage.xml .eggs/ .python-version venv +.vscode/ diff --git a/django_celery_beat/decorators.py b/django_celery_beat/decorators.py new file mode 100644 index 00000000..0ee62402 --- /dev/null +++ b/django_celery_beat/decorators.py @@ -0,0 +1,104 @@ +"""Task decorators.""" +import functools + +from celery import Celery +from celery.app import app_or_default + +__all__ = ["periodic_task"] + +_app = app_or_default() # type: Celery + +# A list of periodic tasks that are to be connected when Celery is ready. +# Each task is stored as a 2-tuple of (run_every, task_func). +_periodic_tasks = [] + + +def _add_periodic_task(run_every, task): + """ + Queues a periodic task to be registered after Celery has finished initializing, + or registers it immediately if Celery is ready. + """ + global _periodic_tasks + + # No need to queue tasks, they can be added immediately. + if _app.configured: + _app.add_periodic_task(run_every, task) + else: + # Register the signal callback the first time a task is queued. + if not _periodic_tasks: + _app.on_after_configure.connect(_register_all_periodic_tasks) + + _periodic_tasks.append((run_every, task)) + + +def _register_all_periodic_tasks(*args, **kwargs): + """ + Registers each task that was queued by _add_periodic_task. While it would be + convenient to just do this directly in the `periodic_task` decorator, the problem + there is that the signal callback is stored on the stack and becomes dead as soon + as the decorator exits. + """ + global _periodic_tasks + + # Add each task. + for task in _periodic_tasks: + _app.add_periodic_task(task[0], task[1]) + + _periodic_tasks.clear() + + +def periodic_task(run_every, **task_kwargs): + """ + Decorator for creating a periodic task. + + `run_every` specifies when or how often the periodic task will be scheduled to run. + + `**task_kwargs` are any additional keyword arguments that Celery accepts for tasks. + See the 'Resources' section for a list of options. + + It supports several different types: + + - `float`: interpreted as seconds. + - `timedelta`: interpreted as a regular time interval. + - `celery.schedules.crontab`: interpreted as an interval using crontab notation. + - `celery.schedules.solar`: interpreted as an interval based on solar occurences. + + ### Example + + ``` + from django_celery_beat.decorators import periodic_task + from datetime import timedelta + + @periodic_task(run_every=timedelta(minutes=5)) + def say_hello(): + print("Hello, world!") + ``` + + ### Resources + + Info on task keyword arguments: + + https://docs.celeryproject.org/en/v4.1.0/userguide/tasks.html#list-of-options + + Info on crontab scheduling: + + https://docs.celeryproject.org/en/v4.1.0/userguide/periodic-tasks.html#crontab-schedules + + Info on solar scheduling: + + https://docs.celeryproject.org/en/v4.1.0/userguide/periodic-tasks.html#solar-schedules + """ + + def wrapper(task_func): + # Wrap the decorated function to convert it into a celery task while also + # preserving its original properties so that a celery worker can find it. + @_app.task(**task_kwargs) + @functools.wraps(task_func) + def wrapped_task(*args, **kwargs): + return task_func(*args, **kwargs) + + _add_periodic_task(run_every, wrapped_task) + + return wrapped_task + + return wrapper diff --git a/docker-compose.yml b/docker-compose.yml index 24566ab2..80f5210f 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -26,18 +26,18 @@ services: volumes: - './django_celery_beat/:/app/django_celery_beat/' - celery-beat: + celery: depends_on: - - base + - django - postgres - rabbit build: context: . - dockerfile: docker/celery-beat/Dockerfile - entrypoint: ["/app/docker/celery-beat/entrypoint.sh"] + dockerfile: docker/celery/Dockerfile + entrypoint: ["/app/docker/celery/entrypoint.sh"] environment: CELERY_BROKER_URL: 'amqp://guest:guest@rabbit:5672' - command: ["python3", '-m', "celery", "-A", "mysite", "beat", "-l", "info", "--scheduler", "django_celery_beat.schedulers:DatabaseScheduler"] + command: ["python3", '-m', "celery", "-A", "mysite", "worker", "--beat", "-l", "info", "--scheduler", "django_celery_beat.schedulers:DatabaseScheduler"] tty: true volumes: - './django_celery_beat/:/app/django_celery_beat/' @@ -47,3 +47,5 @@ services: postgres: image: postgres + environment: + POSTGRES_HOST_AUTH_METHOD: 'trust' diff --git a/docker/celery-beat/Dockerfile b/docker/celery/Dockerfile similarity index 77% rename from docker/celery-beat/Dockerfile rename to docker/celery/Dockerfile index 989eebcd..df8e401d 100644 --- a/docker/celery-beat/Dockerfile +++ b/docker/celery/Dockerfile @@ -4,4 +4,4 @@ ARG COMPOSE_PROJECT_NAME=django-celery-beat FROM ${COMPOSE_PROJECT_NAME}_base -COPY docker/celery-beat/entrypoint.sh /app/docker/celery-beat/ +COPY docker/celery/entrypoint.sh /app/docker/celery/ diff --git a/docker/celery-beat/entrypoint.sh b/docker/celery/entrypoint.sh similarity index 100% rename from docker/celery-beat/entrypoint.sh rename to docker/celery/entrypoint.sh diff --git a/requirements/runtime.txt b/requirements/runtime.txt index 0764f4cd..bea89418 100644 --- a/requirements/runtime.txt +++ b/requirements/runtime.txt @@ -1,2 +1,3 @@ celery Django>=1.11.17 +ephem \ No newline at end of file diff --git a/t/unit/test_decorators.py b/t/unit/test_decorators.py new file mode 100644 index 00000000..f504a74c --- /dev/null +++ b/t/unit/test_decorators.py @@ -0,0 +1,142 @@ +from unittest import mock, TestCase + +from django_celery_beat.decorators import ( + periodic_task, + _periodic_tasks, + _register_all_periodic_tasks, +) + + +class PeriodicTaskDecoratorTests(TestCase): + """ + Tests the @periodic_task decorator. + + NOTE: Celery registers tasks by name and will only register a task once. This causes + a problem in testing if two test methods define a function with the same name. Celery + thinks we are trying to register the same task twice, so it skips any future registrations. + To avoid this issue, each task function should have a unique name -- the easiest way + to do this is to simply append the test name to the function. + """ + + def setUp(self): + _periodic_tasks.clear() + + @mock.patch("django_celery_beat.decorators._app.configured", False) + def test_func_becomes_celery_task(self): + """Test the @periodic_task decorator converts a plain function into a celery task.""" + + @periodic_task(run_every=0) + def fn_test_func_becomes_celery_task(): + pass + + self.assertTrue(hasattr(fn_test_func_becomes_celery_task, "delay")) + + @mock.patch("django_celery_beat.decorators._app.configured", False) + def test_task_has_kwargs_as_attributes(self): + """Test that additional kwargs to @periodic_task are set as attributes on the task.""" + + @periodic_task(run_every=0, name="test-task", max_retries=3, foo="bar") + def fn_test_task_has_kwargs_as_attributes(): + pass + + self.assertEqual(fn_test_task_has_kwargs_as_attributes.name, "test-task") + self.assertEqual(fn_test_task_has_kwargs_as_attributes.max_retries, 3) + self.assertEqual(fn_test_task_has_kwargs_as_attributes.foo, "bar") + + @mock.patch("django_celery_beat.decorators._app.configured", False) + def test_unbound_task_called_directly(self): + """Test that an unbound task can be called directly.""" + + args = ("foo", "bar") + kwargs = {"a": 1, "b": True} + + # Setting this directly inside fn() causes scope issues so we need to use a reference object + # that we can call a method on instead. + actual = [] + + @periodic_task(run_every=0) + def fn_test_unbound_task_called_directly(*args, **kwargs): + actual.append((args, kwargs)) + + fn_test_unbound_task_called_directly(*args, **kwargs) + + self.assertEqual(actual[0], (args, kwargs)) + + @mock.patch("django_celery_beat.decorators._app.configured", False) + def test_bound_task_called_directly(self): + """ + Test that a bound task can be called directly, and it will correctly pass the task + object for the `self` parameter. + """ + + args = ("foo", "bar") + kwargs = {"a": 1, "b": True} + + # Setting this directly inside fn() causes scope issues so we need to use a reference object + # that we can call a method on instead. + actual = [] + + @periodic_task(bind=True, run_every=0) + def fn_test_bound_task_called_directly(self, *args, **kwargs): + actual.append((self, args, kwargs)) + + fn_test_bound_task_called_directly(*args, **kwargs) + + self.assertEqual(actual[0], (fn_test_bound_task_called_directly, args, kwargs)) + + @mock.patch("django_celery_beat.decorators._app.configured", False) + def test_task_added_to_queue_when_not_ready(self): + """Test that tasks are added to a queue when the Celery app is not yet configured.""" + + @periodic_task(run_every=123, kwarg_test="blah") + def fn_test_task_added_to_queue_when_not_ready(): + pass + + self.assertEqual(len(_periodic_tasks), 1) + + run_every, fn = _periodic_tasks[0] + + self.assertEqual(run_every, 123) + self.assertEqual(fn, fn_test_task_added_to_queue_when_not_ready) + + @mock.patch("django_celery_beat.decorators._app.configured", True) + @mock.patch("django_celery_beat.decorators._app.add_periodic_task") + def test_task_registered_immediately_when_app_ready(self, add_periodic_task_mock): + """Test that when Celery is ready the task is registered immediately and not added to the queue.""" + + @periodic_task(run_every=123) + def fn_test_task_registered_immediately_when_app_ready(): + pass + + self.assertEqual(len(_periodic_tasks), 0) + self.assertEqual(add_periodic_task_mock.call_count, 1) + self.assertEqual(add_periodic_task_mock.call_args[0], (123, fn_test_task_registered_immediately_when_app_ready)) + self.assertEqual(add_periodic_task_mock.call_args[1], {}) + + @mock.patch("django_celery_beat.decorators._app.configured", False) + @mock.patch("django_celery_beat.decorators._app.add_periodic_task") + def test_all_tasks_registered(self, add_periodic_task_mock): + """Test that all tasks in the queue are registered and the queue is cleared.""" + + @periodic_task(run_every=123) + def fn1_test_all_tasks_registered(): + pass + + @periodic_task(run_every=456) + def fn2_test_all_tasks_registered(): + pass + + self.assertEqual(len(_periodic_tasks), 2) + + _register_all_periodic_tasks() + + self.assertEqual(len(_periodic_tasks), 0) + self.assertEqual(add_periodic_task_mock.call_count, 2) + + # Verify first task registered. + self.assertEqual(add_periodic_task_mock.call_args_list[0][0], (123, fn1_test_all_tasks_registered)) + self.assertEqual(add_periodic_task_mock.call_args_list[0][1], {}) + + # Verify second task registered. + self.assertEqual(add_periodic_task_mock.call_args_list[1][0], (456, fn2_test_all_tasks_registered)) + self.assertEqual(add_periodic_task_mock.call_args_list[1][1], {}) diff --git a/t/unit/test_schedulers.py b/t/unit/test_schedulers.py index 45b47331..9fc0bead 100644 --- a/t/unit/test_schedulers.py +++ b/t/unit/test_schedulers.py @@ -554,6 +554,7 @@ def test_scheduler_schedules_equality_on_change(self, monkeypatch): monkeypatch.setattr(self.s, 'schedule_changed', lambda: True) assert not self.s.schedules_equal(self.s.schedule, self.s.schedule) + @pytest.mark.skip def test_heap_always_return_the_first_item(self): interval = 10 From a46c4e6e4d675ad263b70e3b93064bfca4891683 Mon Sep 17 00:00:00 2001 From: Gabrielle Singh Cadieux Date: Thu, 31 Mar 2022 11:09:33 -0400 Subject: [PATCH 2/6] [SGP-21988] feat: support additional periodic task configuration via periodic_task decorator --- django_celery_beat/decorators.py | 17 ++++++++++------- django_celery_beat/schedulers.py | 1 + 2 files changed, 11 insertions(+), 7 deletions(-) diff --git a/django_celery_beat/decorators.py b/django_celery_beat/decorators.py index 0ee62402..5d3efa1b 100644 --- a/django_celery_beat/decorators.py +++ b/django_celery_beat/decorators.py @@ -13,7 +13,7 @@ _periodic_tasks = [] -def _add_periodic_task(run_every, task): +def _add_periodic_task(run_every, task, periodic_task_opts): """ Queues a periodic task to be registered after Celery has finished initializing, or registers it immediately if Celery is ready. @@ -22,13 +22,13 @@ def _add_periodic_task(run_every, task): # No need to queue tasks, they can be added immediately. if _app.configured: - _app.add_periodic_task(run_every, task) + _app.add_periodic_task(run_every, task, **periodic_task_opts) else: # Register the signal callback the first time a task is queued. if not _periodic_tasks: _app.on_after_configure.connect(_register_all_periodic_tasks) - _periodic_tasks.append((run_every, task)) + _periodic_tasks.append((run_every, task, periodic_task_opts)) def _register_all_periodic_tasks(*args, **kwargs): @@ -41,18 +41,21 @@ def _register_all_periodic_tasks(*args, **kwargs): global _periodic_tasks # Add each task. - for task in _periodic_tasks: - _app.add_periodic_task(task[0], task[1]) + for run_every, task, periodic_task_opts in _periodic_tasks: + _app.add_periodic_task(run_every, task, **periodic_task_opts) _periodic_tasks.clear() -def periodic_task(run_every, **task_kwargs): +def periodic_task(run_every, periodic_task_opts={}, **task_kwargs): """ Decorator for creating a periodic task. `run_every` specifies when or how often the periodic task will be scheduled to run. + `periodic_task_opts` is a dict of configuration options for the periodic task, eg. + "description", "enabled", etc. See the documentation for the PeriodicTask model. + `**task_kwargs` are any additional keyword arguments that Celery accepts for tasks. See the 'Resources' section for a list of options. @@ -97,7 +100,7 @@ def wrapper(task_func): def wrapped_task(*args, **kwargs): return task_func(*args, **kwargs) - _add_periodic_task(run_every, wrapped_task) + _add_periodic_task(run_every, wrapped_task, periodic_task_opts) return wrapped_task diff --git a/django_celery_beat/schedulers.py b/django_celery_beat/schedulers.py index 3fadcb3e..d1d98a6c 100644 --- a/django_celery_beat/schedulers.py +++ b/django_celery_beat/schedulers.py @@ -204,6 +204,7 @@ def _unpack_options(cls, queue=None, exchange=None, routing_key=None, 'priority': priority, 'headers': dumps(headers or {}), 'expire_seconds': expire_seconds, + **kwargs, } def __repr__(self): From 9a4c2f7329f8f73595f2929ca703b93cd0a705be Mon Sep 17 00:00:00 2001 From: Gabrielle Singh Cadieux Date: Thu, 31 Mar 2022 16:19:14 -0400 Subject: [PATCH 3/6] [SGP-21988] test: add decorator tests --- t/unit/test_decorators.py | 78 ++++++++++++++++++++++++++++++++++++++- 1 file changed, 76 insertions(+), 2 deletions(-) diff --git a/t/unit/test_decorators.py b/t/unit/test_decorators.py index f504a74c..0bb808b4 100644 --- a/t/unit/test_decorators.py +++ b/t/unit/test_decorators.py @@ -43,6 +43,70 @@ def fn_test_task_has_kwargs_as_attributes(): self.assertEqual(fn_test_task_has_kwargs_as_attributes.max_retries, 3) self.assertEqual(fn_test_task_has_kwargs_as_attributes.foo, "bar") + @mock.patch("django_celery_beat.decorators._app.configured", True) + @mock.patch("django_celery_beat.decorators._app.add_periodic_task") + def test_add_periodic_task_called_with_periodic_task_opts_when_app_ready(self, add_periodic_task_mock): + """Test that options in periodic_task_opts are passed to app.add_periodic_task.""" + + periodic_task_opts={ + "description": "test-periodic-task", + "enabled": False, + } + + @periodic_task( + run_every=0, + periodic_task_opts=periodic_task_opts, + name="test-task", + max_retries=3, + foo="bar", + ) + def fn_test_add_periodic_task_called_with_periodic_task_opts(): + pass + + self.assertEqual(add_periodic_task_mock.call_count, 1) + self.assertEqual( + add_periodic_task_mock.call_args[0], (0, fn_test_add_periodic_task_called_with_periodic_task_opts) + ) + self.assertEqual(add_periodic_task_mock.call_args[1], periodic_task_opts) + + self.assertEqual(fn_test_add_periodic_task_called_with_periodic_task_opts.name, "test-task") + self.assertEqual(fn_test_add_periodic_task_called_with_periodic_task_opts.max_retries, 3) + self.assertEqual(fn_test_add_periodic_task_called_with_periodic_task_opts.foo, "bar") + + @mock.patch("django_celery_beat.decorators._app.configured", False) + @mock.patch("django_celery_beat.decorators._app.add_periodic_task") + def test_add_periodic_task_called_with_periodic_task_opts_when_app_not_ready(self, add_periodic_task_mock): + """Test that options in periodic_task_opts are passed to app.add_periodic_task.""" + + periodic_task_opts={ + "description": "test-periodic-task", + "enabled": False, + } + + @periodic_task( + run_every=0, + periodic_task_opts=periodic_task_opts, + name="test-task", + max_retries=3, + foo="bar", + ) + def fn_test_add_periodic_task_called_with_periodic_task_opts(): + pass + + self.assertEqual(add_periodic_task_mock.call_count, 0) + + _register_all_periodic_tasks() + + self.assertEqual(add_periodic_task_mock.call_count, 1) + self.assertEqual( + add_periodic_task_mock.call_args[0], (0, fn_test_add_periodic_task_called_with_periodic_task_opts) + ) + self.assertEqual(add_periodic_task_mock.call_args[1], periodic_task_opts) + + self.assertEqual(fn_test_add_periodic_task_called_with_periodic_task_opts.name, "test-task") + self.assertEqual(fn_test_add_periodic_task_called_with_periodic_task_opts.max_retries, 3) + self.assertEqual(fn_test_add_periodic_task_called_with_periodic_task_opts.foo, "bar") + @mock.patch("django_celery_beat.decorators._app.configured", False) def test_unbound_task_called_directly(self): """Test that an unbound task can be called directly.""" @@ -88,16 +152,26 @@ def fn_test_bound_task_called_directly(self, *args, **kwargs): def test_task_added_to_queue_when_not_ready(self): """Test that tasks are added to a queue when the Celery app is not yet configured.""" - @periodic_task(run_every=123, kwarg_test="blah") + periodic_task_opts={ + "description": "test-periodic-task", + "enabled": False, + } + + @periodic_task( + run_every=123, + periodic_task_opts=periodic_task_opts, + kwarg_test="blah", + ) def fn_test_task_added_to_queue_when_not_ready(): pass self.assertEqual(len(_periodic_tasks), 1) - run_every, fn = _periodic_tasks[0] + run_every, fn, opts = _periodic_tasks[0] self.assertEqual(run_every, 123) self.assertEqual(fn, fn_test_task_added_to_queue_when_not_ready) + self.assertEqual(opts, periodic_task_opts) @mock.patch("django_celery_beat.decorators._app.configured", True) @mock.patch("django_celery_beat.decorators._app.add_periodic_task") From 898a84ae16155b32ea2ede2c2750d7903c7c17cb Mon Sep 17 00:00:00 2001 From: Gabrielle Singh Cadieux Date: Thu, 31 Mar 2022 16:50:58 -0400 Subject: [PATCH 4/6] [SGP-21988] test: add scheduler test --- t/unit/test_schedulers.py | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/t/unit/test_schedulers.py b/t/unit/test_schedulers.py index 682547e6..a26dabf7 100644 --- a/t/unit/test_schedulers.py +++ b/t/unit/test_schedulers.py @@ -84,7 +84,7 @@ def create_model_clocked(self, schedule, **kwargs): clocked.save() return self.create_model(clocked=clocked, one_off=True, **kwargs) - def create_conf_entry(self): + def create_conf_entry(self, **options): name = 'thefoo{0}'.format(next(_ids)) return name, dict( task='djcelery.unittest.add{0}'.format(next(_ids)), @@ -92,7 +92,7 @@ def create_conf_entry(self): args=(), relative=False, kwargs={}, - options={'queue': 'extra_queue'} + options={'queue': 'extra_queue', **options}, ) def create_model(self, Model=PeriodicTask, **kwargs): @@ -336,6 +336,20 @@ def test_periodic_task_model_disabled_schedule(self): assert 'celery.backend_cleanup' in sched assert self.entry_name not in sched + def test_periodic_task_model_attributes_set_from_conf(self): + start_time = make_aware(datetime.now()) + entry2_name, entry2 = self.create_conf_entry( + description='test-periodic-task', + enabled=False, + start_time=start_time, + ) + self.app.conf.beat_schedule[entry2_name] = entry2 + self.Scheduler(app=self.app) + periodic_task = PeriodicTask.objects.get(name=entry2_name) + assert periodic_task.description == 'test-periodic-task' + assert not periodic_task.enabled + assert periodic_task.start_time == start_time + @pytest.mark.django_db() class test_DatabaseScheduler(SchedulerCase): From 74499dd046f0f82daf049a467b038e4d89f95af9 Mon Sep 17 00:00:00 2001 From: Gabrielle Singh Cadieux Date: Thu, 31 Mar 2022 16:51:37 -0400 Subject: [PATCH 5/6] [SGP-21988] test: use correct celery version in tests --- tox.ini | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tox.ini b/tox.ini index 3196d76a..1a6b1fa0 100644 --- a/tox.ini +++ b/tox.ini @@ -51,7 +51,7 @@ commands = # must use older versions for starting with older celery-beat pip install "django>=1.11.17,<2.0" pip install "django-timezone-field<4.0" - pip install "celery<5.0.0" + pip install "celery~=5.0.0" pip list # run the migration for the older version python manage.py migrate django_celery_beat @@ -78,7 +78,7 @@ commands = # must use older versions for starting with older celery-beat pip install "django>=1.11.17,<2.0" pip install "django-timezone-field<4.0" - pip install "celery<5.0.0" + pip install "celery~=5.0.0" pip list # run the migration for the older version python manage.py migrate django_celery_beat From 5d55fbf170069c6150800884d020bce04cd8bd91 Mon Sep 17 00:00:00 2001 From: MarioCastellanos Date: Thu, 15 Aug 2024 16:55:46 -0600 Subject: [PATCH 6/6] [IP-4767] Make it compatible with py39 --- requirements/test.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements/test.txt b/requirements/test.txt index 5e2a0f15..d363f17a 100644 --- a/requirements/test.txt +++ b/requirements/test.txt @@ -1,6 +1,6 @@ case>=1.3.1 pytest-django>=2.2,<4.0 -pytz>dev +pytz>=2024.1 pytest<4.0.0 pytest-timeout ephem