diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 5c2a216..fbef003 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -9,20 +9,16 @@ jobs: strategy: matrix: - python: ["3.8", "3.9", "3.10", "3.11", "3.12"] - django: ["3.2", "4.0", "4.1", "4.2", "5.0"] + python: ["3.9", "3.10", "3.11", "3.12", "3.13"] + django: ["4.2", "5.0", "5.1"] exclude: - - python: "3.11" - django: "3.2" - - python: "3.12" - django: "3.2" - - python: "3.11" - django: "4.0" - - python: "3.12" - django: "4.0" - - python: "3.8" + - python: "3.9" django: "5.0" - python: "3.9" + django: "5.1" + - python: "3.13" + django: "4.2" + - python: "3.13" django: "5.0" database_url: - postgres://runner:password@localhost/project @@ -45,19 +41,19 @@ jobs: steps: - name: Start MySQL run: sudo systemctl start mysql.service - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 - name: Install system Python build deps for psycopg2 - run: sudo apt-get install python3-dev python3.11-dev + run: sudo apt-get install python3-dev - name: Set up Python ${{ matrix.python }} - uses: actions/setup-python@v2 + uses: actions/setup-python@v5 with: - python-version: ${{ matrix.python }} + python-version: ${{ matrix.python }}.* - name: Upgraded pip run: pip install --upgrade pip - name: Install dependencies run: pip install -r test-requirements.txt - name: Install Django - run: pip install -U django==${{ matrix.django }} + run: pip install -U django~=${{ matrix.django }}.0 - name: Run tests run: python manage.py test - name: Run black diff --git a/README.md b/README.md index c4e698a..fb01036 100644 --- a/README.md +++ b/README.md @@ -7,8 +7,8 @@ Simple database-backed job queue. Jobs are defined in your settings, and are pro Asynchronous tasks are run via a *job queue*. This system is designed to support multi-step job workflows. Supported and tested against: -- Django 3.2, 4.0, 4.1, 4.2, 5.0 -- Python 3.8, 3.9, 3.10, 3.11, 3.12 +- Django 4.2, 5.0, 5.1 +- Python 3.9, 3.10, 3.11, 3.12, 3.13 ## Getting Started @@ -288,6 +288,11 @@ queue_depths = Job.get_queue_depths() print(queue_depths) # {"default": 1, "other_queue": 1} ``` +You can also exclude jobs which exist but are scheduled to be run in the future from the queue depths, where `run_after` is set to a future time from now. To do this set the `exclude_future_jobs` kwarg like so: +```python +queue_depths = Job.get_queue_depths(exclude_future_jobs=True) +``` + **Important:** When checking queue depths, do not assume that the key for your queue will always be available. Queue depths of zero won't be included in the dict returned by this method. @@ -312,6 +317,8 @@ manage.py worker [queue_name] [--rate_limit] If you'd like to check your queue depth from the command line, you can run `manage.py queue_depth [queue_name [queue_name ...]]` and any jobs in the "NEW" or "READY" states will be returned. +If you wish to exclude jobs which are scheduled to be run in the future you can add `--exclude_future_jobs` to the command. + **Important:** If you misspell or provide a queue name which does not have any jobs, a depth of 0 will always be returned. ### Gotcha: `bulk_create` diff --git a/django_dbq/__init__.py b/django_dbq/__init__.py index 1173108..88c513e 100644 --- a/django_dbq/__init__.py +++ b/django_dbq/__init__.py @@ -1 +1 @@ -__version__ = "3.2.0" +__version__ = "3.3.0" diff --git a/django_dbq/management/commands/queue_depth.py b/django_dbq/management/commands/queue_depth.py index 3419601..cb8b6fd 100644 --- a/django_dbq/management/commands/queue_depth.py +++ b/django_dbq/management/commands/queue_depth.py @@ -8,10 +8,13 @@ class Command(BaseCommand): def add_arguments(self, parser): parser.add_argument("queue_name", nargs="*", default=["default"], type=str) + parser.add_argument("--exclude_future_jobs", default=False, type=bool) def handle(self, *args, **options): queue_names = options["queue_name"] - queue_depths = Job.get_queue_depths() + queue_depths = Job.get_queue_depths( + exclude_future_jobs=options["exclude_future_jobs"] + ) queue_depths_string = " ".join( [ diff --git a/django_dbq/models.py b/django_dbq/models.py index b58eef4..a90354f 100644 --- a/django_dbq/models.py +++ b/django_dbq/models.py @@ -8,7 +8,7 @@ get_failure_hook_name, get_creation_hook_name, ) -from django.db.models import JSONField, UUIDField, Count, TextChoices +from django.db.models import JSONField, UUIDField, Count, TextChoices, Q import datetime import logging import uuid @@ -173,10 +173,17 @@ def run_creation_hook(self): creation_hook_function(self) @staticmethod - def get_queue_depths(): + def get_queue_depths(*, exclude_future_jobs=False): + jobs_waiting_in_queue = Job.objects.filter( + state__in=(Job.STATES.READY, Job.STATES.NEW) + ) + if exclude_future_jobs: + jobs_waiting_in_queue = jobs_waiting_in_queue.filter( + Q(run_after__isnull=True) | Q(run_after__lte=timezone.now()) + ) + annotation_dicts = ( - Job.objects.filter(state__in=(Job.STATES.READY, Job.STATES.NEW)) - .values("queue_name") + jobs_waiting_in_queue.values("queue_name") .order_by("queue_name") .annotate(Count("queue_name")) ) diff --git a/django_dbq/tests.py b/django_dbq/tests.py index dd83540..200623b 100644 --- a/django_dbq/tests.py +++ b/django_dbq/tests.py @@ -1,8 +1,8 @@ -from datetime import datetime, timedelta -import mock +from datetime import datetime, timedelta, timezone as datetime_timezone +from unittest import mock import freezegun -from django.core.management import call_command, CommandError +from django.core.management import call_command from django.test import TestCase from django.test.utils import override_settings from django.utils import timezone @@ -13,14 +13,6 @@ from io import StringIO -try: - utc = timezone.utc -except AttributeError: - from datetime import timezone as datetime_timezone - - utc = datetime_timezone.utc - - def test_task(job=None): pass # pragma: no cover @@ -71,12 +63,17 @@ def test_worker_with_queue_name(self): self.assertTrue("test_queue" in output) +@freezegun.freeze_time("2025-01-01T12:00:00Z") @override_settings(JOBS={"testjob": {"tasks": ["a"]}}) class JobModelMethodTestCase(TestCase): def test_get_queue_depths(self): Job.objects.create(name="testjob", queue_name="default") Job.objects.create(name="testjob", queue_name="testworker") - Job.objects.create(name="testjob", queue_name="testworker") + Job.objects.create( + name="testjob", + queue_name="testworker", + run_after=timezone.make_aware(datetime(2025, 1, 1, 13, 0, 0)), + ) Job.objects.create( name="testjob", queue_name="testworker", state=Job.STATES.FAILED ) @@ -87,16 +84,38 @@ def test_get_queue_depths(self): queue_depths = Job.get_queue_depths() self.assertDictEqual(queue_depths, {"default": 1, "testworker": 2}) + def test_get_queue_depths_exclude_future_jobs(self): + Job.objects.create(name="testjob", queue_name="default") + Job.objects.create(name="testjob", queue_name="testworker") + Job.objects.create( + name="testjob", + queue_name="testworker", + run_after=timezone.make_aware(datetime(2025, 1, 1, 13, 0, 0)), + ) + Job.objects.create( + name="testjob", queue_name="testworker", state=Job.STATES.FAILED + ) + Job.objects.create( + name="testjob", queue_name="testworker", state=Job.STATES.COMPLETE + ) + + queue_depths = Job.get_queue_depths(exclude_future_jobs=True) + self.assertDictEqual(queue_depths, {"default": 1, "testworker": 1}) + +@freezegun.freeze_time("2025-01-01T12:00:00Z") @override_settings(JOBS={"testjob": {"tasks": ["a"]}}) class QueueDepthTestCase(TestCase): def test_queue_depth(self): - Job.objects.create(name="testjob", state=Job.STATES.FAILED) Job.objects.create(name="testjob", state=Job.STATES.NEW) Job.objects.create(name="testjob", state=Job.STATES.FAILED) Job.objects.create(name="testjob", state=Job.STATES.COMPLETE) - Job.objects.create(name="testjob", state=Job.STATES.READY) + Job.objects.create( + name="testjob", + state=Job.STATES.READY, + run_after=timezone.make_aware(datetime(2025, 1, 1, 13, 0, 0)), + ) Job.objects.create( name="testjob", queue_name="testqueue", state=Job.STATES.READY ) @@ -109,6 +128,28 @@ def test_queue_depth(self): output = stdout.getvalue() self.assertEqual(output.strip(), "event=queue_depths default=2") + def test_queue_depth_exclude_future_jobs(self): + Job.objects.create(name="testjob", state=Job.STATES.FAILED) + Job.objects.create(name="testjob", state=Job.STATES.NEW) + Job.objects.create(name="testjob", state=Job.STATES.FAILED) + Job.objects.create(name="testjob", state=Job.STATES.COMPLETE) + Job.objects.create( + name="testjob", + state=Job.STATES.READY, + run_after=timezone.make_aware(datetime(2025, 1, 1, 13, 0, 0)), + ) + Job.objects.create( + name="testjob", queue_name="testqueue", state=Job.STATES.READY + ) + Job.objects.create( + name="testjob", queue_name="testqueue", state=Job.STATES.READY + ) + + stdout = StringIO() + call_command("queue_depth", exclude_future_jobs=True, stdout=stdout) + output = stdout.getvalue() + self.assertEqual(output.strip(), "event=queue_depths default=1") + def test_queue_depth_multiple_queues(self): Job.objects.create(name="testjob", state=Job.STATES.FAILED) @@ -253,7 +294,8 @@ def test_gets_jobs_in_priority_and_date_order(self): def test_ignores_jobs_until_run_after_is_in_the_past(self): job_1 = Job.objects.create(name="testjob") job_2 = Job.objects.create( - name="testjob", run_after=datetime(2021, 11, 4, 8, tzinfo=utc) + name="testjob", + run_after=datetime(2021, 11, 4, 8, tzinfo=datetime_timezone.utc), ) with freezegun.freeze_time(datetime(2021, 11, 4, 7)): diff --git a/setup.py b/setup.py index 5bb0f76..fda66ff 100644 --- a/setup.py +++ b/setup.py @@ -16,7 +16,7 @@ author_email = "contact@dabapps.com" license = "BSD" install_requires = [ - "Django>=3.1", + "Django>=4.2", ] long_description = """Simple database-backed job queue system""" @@ -82,5 +82,5 @@ def get_package_data(package): package_data=get_package_data(package), install_requires=install_requires, classifiers=[], - python_requires=">=3.6" + python_requires=">=3.9" ) diff --git a/test-requirements.txt b/test-requirements.txt index f5b2998..93545b8 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -1,6 +1,5 @@ -mysqlclient==1.4.6 -freezegun==0.3.12 -mock==3.0.5 -dj-database-url==0.5.0 -psycopg2==2.9.5 -black==24.3.0 +mysqlclient==2.2.7 +freezegun==1.5.1 +dj-database-url==2.3.0 +psycopg2==2.9.10 +black==24.10.0