From 6730a6b651c1977a34fb79a03d27a306a6570629 Mon Sep 17 00:00:00 2001 From: Jamie Matthews Date: Mon, 20 Jan 2025 10:29:48 +0000 Subject: [PATCH 01/10] Add Python 3.13 and Django 5.1 to test matrix, drop unsupported versions --- .github/workflows/ci.yml | 20 ++++++-------------- README.md | 4 ++-- django_dbq/tests.py | 15 ++++----------- 3 files changed, 12 insertions(+), 27 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 5c2a216..0747581 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -9,21 +9,13 @@ jobs: strategy: matrix: - python: ["3.8", "3.9", "3.10", "3.11", "3.12"] - django: ["3.2", "4.0", "4.1", "4.2", "5.0"] + python: ["3.9", "3.10", "3.11", "3.12", "3.13"] + django: ["4.2", "5.0", "5.1"] exclude: - - python: "3.11" - django: "3.2" - - python: "3.12" - django: "3.2" - - python: "3.11" - django: "4.0" - - python: "3.12" - django: "4.0" - - python: "3.8" - django: "5.0" - python: "3.9" django: "5.0" + - python: "3.9" + django: "5.1" database_url: - postgres://runner:password@localhost/project - mysql://root:root@127.0.0.1/project @@ -45,11 +37,11 @@ jobs: steps: - name: Start MySQL run: sudo systemctl start mysql.service - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 - name: Install system Python build deps for psycopg2 run: sudo apt-get install python3-dev python3.11-dev - name: Set up Python ${{ matrix.python }} - uses: actions/setup-python@v2 + uses: actions/setup-python@v5 with: python-version: ${{ matrix.python }} - name: Upgraded pip diff --git a/README.md b/README.md index c4e698a..d589707 100644 --- a/README.md +++ b/README.md @@ -7,8 +7,8 @@ Simple database-backed job queue. Jobs are defined in your settings, and are pro Asynchronous tasks are run via a *job queue*. This system is designed to support multi-step job workflows. Supported and tested against: -- Django 3.2, 4.0, 4.1, 4.2, 5.0 -- Python 3.8, 3.9, 3.10, 3.11, 3.12 +- Django 4.2, 5.0, 5.1 +- Python 3.9, 3.10, 3.11, 3.12, 3.13 ## Getting Started diff --git a/django_dbq/tests.py b/django_dbq/tests.py index dd83540..ba80b7f 100644 --- a/django_dbq/tests.py +++ b/django_dbq/tests.py @@ -1,8 +1,8 @@ -from datetime import datetime, timedelta +from datetime import datetime, timedelta, timezone as datetime_timezone import mock import freezegun -from django.core.management import call_command, CommandError +from django.core.management import call_command from django.test import TestCase from django.test.utils import override_settings from django.utils import timezone @@ -13,14 +13,6 @@ from io import StringIO -try: - utc = timezone.utc -except AttributeError: - from datetime import timezone as datetime_timezone - - utc = datetime_timezone.utc - - def test_task(job=None): pass # pragma: no cover @@ -253,7 +245,8 @@ def test_gets_jobs_in_priority_and_date_order(self): def test_ignores_jobs_until_run_after_is_in_the_past(self): job_1 = Job.objects.create(name="testjob") job_2 = Job.objects.create( - name="testjob", run_after=datetime(2021, 11, 4, 8, tzinfo=utc) + name="testjob", + run_after=datetime(2021, 11, 4, 8, tzinfo=datetime_timezone.utc), ) with freezegun.freeze_time(datetime(2021, 11, 4, 7)): From 2de272a1a44fec0c1b74539241b236eb9a6c0774 Mon Sep 17 00:00:00 2001 From: Jamie Matthews Date: Mon, 20 Jan 2025 10:31:25 +0000 Subject: [PATCH 02/10] Remove Python 3.11 from apt packages --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 0747581..2a3dc58 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -39,7 +39,7 @@ jobs: run: sudo systemctl start mysql.service - uses: actions/checkout@v4 - name: Install system Python build deps for psycopg2 - run: sudo apt-get install python3-dev python3.11-dev + run: sudo apt-get install python3-dev - name: Set up Python ${{ matrix.python }} uses: actions/setup-python@v5 with: From 3dd1843f72a7c9abdd0a29b208c8ac755750342e Mon Sep 17 00:00:00 2001 From: Jamie Matthews Date: Mon, 20 Jan 2025 10:35:02 +0000 Subject: [PATCH 03/10] Exclude 3.13 builds from unsupported Django versions --- .github/workflows/ci.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 2a3dc58..02ec500 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -16,6 +16,10 @@ jobs: django: "5.0" - python: "3.9" django: "5.1" + - python: "3.13" + django: "4.2" + - python: "3.13" + django: "5.0" database_url: - postgres://runner:password@localhost/project - mysql://root:root@127.0.0.1/project From d797d37b10251026bc73974a93652b4df166e348 Mon Sep 17 00:00:00 2001 From: Jamie Matthews Date: Mon, 20 Jan 2025 10:41:52 +0000 Subject: [PATCH 04/10] Install the latest point release of each Django and Python version --- .github/workflows/ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 02ec500..fbef003 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -47,13 +47,13 @@ jobs: - name: Set up Python ${{ matrix.python }} uses: actions/setup-python@v5 with: - python-version: ${{ matrix.python }} + python-version: ${{ matrix.python }}.* - name: Upgraded pip run: pip install --upgrade pip - name: Install dependencies run: pip install -r test-requirements.txt - name: Install Django - run: pip install -U django==${{ matrix.django }} + run: pip install -U django~=${{ matrix.django }}.0 - name: Run tests run: python manage.py test - name: Run black From e9f234798ff1d367833e2689bfe53341331e3d9b Mon Sep 17 00:00:00 2001 From: Jamie Matthews Date: Mon, 20 Jan 2025 10:46:20 +0000 Subject: [PATCH 05/10] Upgrade test requirements --- django_dbq/tests.py | 2 +- test-requirements.txt | 11 +++++------ 2 files changed, 6 insertions(+), 7 deletions(-) diff --git a/django_dbq/tests.py b/django_dbq/tests.py index ba80b7f..f755666 100644 --- a/django_dbq/tests.py +++ b/django_dbq/tests.py @@ -1,5 +1,5 @@ from datetime import datetime, timedelta, timezone as datetime_timezone -import mock +from unittest import mock import freezegun from django.core.management import call_command diff --git a/test-requirements.txt b/test-requirements.txt index f5b2998..93545b8 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -1,6 +1,5 @@ -mysqlclient==1.4.6 -freezegun==0.3.12 -mock==3.0.5 -dj-database-url==0.5.0 -psycopg2==2.9.5 -black==24.3.0 +mysqlclient==2.2.7 +freezegun==1.5.1 +dj-database-url==2.3.0 +psycopg2==2.9.10 +black==24.10.0 From e0c7f18e07a34b421e180df37653f1baa130bfad Mon Sep 17 00:00:00 2001 From: Jamie Matthews Date: Mon, 20 Jan 2025 10:47:32 +0000 Subject: [PATCH 06/10] Bump versions in setup.py --- setup.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/setup.py b/setup.py index 5bb0f76..fda66ff 100644 --- a/setup.py +++ b/setup.py @@ -16,7 +16,7 @@ author_email = "contact@dabapps.com" license = "BSD" install_requires = [ - "Django>=3.1", + "Django>=4.2", ] long_description = """Simple database-backed job queue system""" @@ -82,5 +82,5 @@ def get_package_data(package): package_data=get_package_data(package), install_requires=install_requires, classifiers=[], - python_requires=">=3.6" + python_requires=">=3.9" ) From f28e85e124a71d32f51bcaf997bed52d5eb69867 Mon Sep 17 00:00:00 2001 From: George Waller <54817483+George9Waller@users.noreply.github.com> Date: Fri, 21 Feb 2025 13:28:19 +0100 Subject: [PATCH 07/10] Add exclude_future_jobs arg to get_queue_depths New arg exclude_future_jobs filters the READY and NEW jobs to only be ones without a run_after or the run_after is now/in the past --- django_dbq/models.py | 15 +++++++++++---- django_dbq/tests.py | 25 ++++++++++++++++++++++++- 2 files changed, 35 insertions(+), 5 deletions(-) diff --git a/django_dbq/models.py b/django_dbq/models.py index b58eef4..a90354f 100644 --- a/django_dbq/models.py +++ b/django_dbq/models.py @@ -8,7 +8,7 @@ get_failure_hook_name, get_creation_hook_name, ) -from django.db.models import JSONField, UUIDField, Count, TextChoices +from django.db.models import JSONField, UUIDField, Count, TextChoices, Q import datetime import logging import uuid @@ -173,10 +173,17 @@ def run_creation_hook(self): creation_hook_function(self) @staticmethod - def get_queue_depths(): + def get_queue_depths(*, exclude_future_jobs=False): + jobs_waiting_in_queue = Job.objects.filter( + state__in=(Job.STATES.READY, Job.STATES.NEW) + ) + if exclude_future_jobs: + jobs_waiting_in_queue = jobs_waiting_in_queue.filter( + Q(run_after__isnull=True) | Q(run_after__lte=timezone.now()) + ) + annotation_dicts = ( - Job.objects.filter(state__in=(Job.STATES.READY, Job.STATES.NEW)) - .values("queue_name") + jobs_waiting_in_queue.values("queue_name") .order_by("queue_name") .annotate(Count("queue_name")) ) diff --git a/django_dbq/tests.py b/django_dbq/tests.py index dd83540..8709eb5 100644 --- a/django_dbq/tests.py +++ b/django_dbq/tests.py @@ -71,12 +71,17 @@ def test_worker_with_queue_name(self): self.assertTrue("test_queue" in output) +@freezegun.freeze_time("2025-01-01T12:00:00Z") @override_settings(JOBS={"testjob": {"tasks": ["a"]}}) class JobModelMethodTestCase(TestCase): def test_get_queue_depths(self): Job.objects.create(name="testjob", queue_name="default") Job.objects.create(name="testjob", queue_name="testworker") - Job.objects.create(name="testjob", queue_name="testworker") + Job.objects.create( + name="testjob", + queue_name="testworker", + run_after=timezone.make_aware(datetime(2025, 1, 1, 13, 0, 0)), + ) Job.objects.create( name="testjob", queue_name="testworker", state=Job.STATES.FAILED ) @@ -87,6 +92,24 @@ def test_get_queue_depths(self): queue_depths = Job.get_queue_depths() self.assertDictEqual(queue_depths, {"default": 1, "testworker": 2}) + def test_get_queue_depths_exclude_future_jobs(self): + Job.objects.create(name="testjob", queue_name="default") + Job.objects.create(name="testjob", queue_name="testworker") + Job.objects.create( + name="testjob", + queue_name="testworker", + run_after=timezone.make_aware(datetime(2025, 1, 1, 13, 0, 0)), + ) + Job.objects.create( + name="testjob", queue_name="testworker", state=Job.STATES.FAILED + ) + Job.objects.create( + name="testjob", queue_name="testworker", state=Job.STATES.COMPLETE + ) + + queue_depths = Job.get_queue_depths(exclude_future_jobs=True) + self.assertDictEqual(queue_depths, {"default": 1, "testworker": 1}) + @override_settings(JOBS={"testjob": {"tasks": ["a"]}}) class QueueDepthTestCase(TestCase): From 738be6db1603c96429023f48d29022c154757481 Mon Sep 17 00:00:00 2001 From: George Waller <54817483+George9Waller@users.noreply.github.com> Date: Fri, 21 Feb 2025 13:29:43 +0100 Subject: [PATCH 08/10] Add the exclude_future_jobs arg to the queue_depth command The queue_depth command can call get_queue_depths with the option to exclude jobs in the future --- django_dbq/management/commands/queue_depth.py | 5 +++- django_dbq/tests.py | 30 +++++++++++++++++-- 2 files changed, 32 insertions(+), 3 deletions(-) diff --git a/django_dbq/management/commands/queue_depth.py b/django_dbq/management/commands/queue_depth.py index 3419601..cb8b6fd 100644 --- a/django_dbq/management/commands/queue_depth.py +++ b/django_dbq/management/commands/queue_depth.py @@ -8,10 +8,13 @@ class Command(BaseCommand): def add_arguments(self, parser): parser.add_argument("queue_name", nargs="*", default=["default"], type=str) + parser.add_argument("--exclude_future_jobs", default=False, type=bool) def handle(self, *args, **options): queue_names = options["queue_name"] - queue_depths = Job.get_queue_depths() + queue_depths = Job.get_queue_depths( + exclude_future_jobs=options["exclude_future_jobs"] + ) queue_depths_string = " ".join( [ diff --git a/django_dbq/tests.py b/django_dbq/tests.py index 8709eb5..20c4029 100644 --- a/django_dbq/tests.py +++ b/django_dbq/tests.py @@ -111,15 +111,19 @@ def test_get_queue_depths_exclude_future_jobs(self): self.assertDictEqual(queue_depths, {"default": 1, "testworker": 1}) +@freezegun.freeze_time("2025-01-01T12:00:00Z") @override_settings(JOBS={"testjob": {"tasks": ["a"]}}) class QueueDepthTestCase(TestCase): def test_queue_depth(self): - Job.objects.create(name="testjob", state=Job.STATES.FAILED) Job.objects.create(name="testjob", state=Job.STATES.NEW) Job.objects.create(name="testjob", state=Job.STATES.FAILED) Job.objects.create(name="testjob", state=Job.STATES.COMPLETE) - Job.objects.create(name="testjob", state=Job.STATES.READY) + Job.objects.create( + name="testjob", + state=Job.STATES.READY, + run_after=timezone.make_aware(datetime(2025, 1, 1, 13, 0, 0)), + ) Job.objects.create( name="testjob", queue_name="testqueue", state=Job.STATES.READY ) @@ -132,6 +136,28 @@ def test_queue_depth(self): output = stdout.getvalue() self.assertEqual(output.strip(), "event=queue_depths default=2") + def test_queue_depth_exclude_future_jobs(self): + Job.objects.create(name="testjob", state=Job.STATES.FAILED) + Job.objects.create(name="testjob", state=Job.STATES.NEW) + Job.objects.create(name="testjob", state=Job.STATES.FAILED) + Job.objects.create(name="testjob", state=Job.STATES.COMPLETE) + Job.objects.create( + name="testjob", + state=Job.STATES.READY, + run_after=timezone.make_aware(datetime(2025, 1, 1, 13, 0, 0)), + ) + Job.objects.create( + name="testjob", queue_name="testqueue", state=Job.STATES.READY + ) + Job.objects.create( + name="testjob", queue_name="testqueue", state=Job.STATES.READY + ) + + stdout = StringIO() + call_command("queue_depth", exclude_future_jobs=True, stdout=stdout) + output = stdout.getvalue() + self.assertEqual(output.strip(), "event=queue_depths default=1") + def test_queue_depth_multiple_queues(self): Job.objects.create(name="testjob", state=Job.STATES.FAILED) From 73228276f58dcd4502503a2255a4580828eb0ab2 Mon Sep 17 00:00:00 2001 From: George Waller <54817483+George9Waller@users.noreply.github.com> Date: Fri, 21 Feb 2025 13:30:09 +0100 Subject: [PATCH 09/10] Include new arg in readme documentation --- README.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/README.md b/README.md index c4e698a..04f0caf 100644 --- a/README.md +++ b/README.md @@ -288,6 +288,11 @@ queue_depths = Job.get_queue_depths() print(queue_depths) # {"default": 1, "other_queue": 1} ``` +You can also exclude jobs which exist but are scheduled to be run in the future from the queue depths, where `run_after` is set to a future time from now. To do this set the `exclude_future_jobs` kwarg like so: +```python +queue_depths = Job.get_queue_depths(exclude_future_jobs=True) +``` + **Important:** When checking queue depths, do not assume that the key for your queue will always be available. Queue depths of zero won't be included in the dict returned by this method. @@ -312,6 +317,8 @@ manage.py worker [queue_name] [--rate_limit] If you'd like to check your queue depth from the command line, you can run `manage.py queue_depth [queue_name [queue_name ...]]` and any jobs in the "NEW" or "READY" states will be returned. +If you wish to exclude jobs which are scheduled to be run in the future you can add `--exclude_future_jobs` to the command. + **Important:** If you misspell or provide a queue name which does not have any jobs, a depth of 0 will always be returned. ### Gotcha: `bulk_create` From fd09d21bb9e4f5c4935ccbb76f15c50a2fccb186 Mon Sep 17 00:00:00 2001 From: Jamie Matthews Date: Tue, 25 Feb 2025 12:53:40 +0000 Subject: [PATCH 10/10] Version 3.3.0 --- django_dbq/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/django_dbq/__init__.py b/django_dbq/__init__.py index 1173108..88c513e 100644 --- a/django_dbq/__init__.py +++ b/django_dbq/__init__.py @@ -1 +1 @@ -__version__ = "3.2.0" +__version__ = "3.3.0"