From 3f6c8292254e8192f47440425ab1c0ed801be5fb Mon Sep 17 00:00:00 2001 From: Max Hurl Date: Mon, 25 Jan 2016 14:13:02 +0000 Subject: [PATCH 001/158] Added note to add to installed apps to readme --- README.md | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/README.md b/README.md index 55dad27..52160da 100644 --- a/README.md +++ b/README.md @@ -13,6 +13,19 @@ re-run `manage.py migrate`. ## Getting Started +### Installation + +Install from PIP + + pip install django-db-queue + +Add `django_dbq` to your installed apps + + INSTALLED_APPS = ( + ... + 'django_dbq', + ) + ### Describe your job In e.g. project.common.jobs: From 535a868a2b025976c8e3702ac9ea02cd60bdf4ff Mon Sep 17 00:00:00 2001 From: Max Hurl Date: Tue, 25 Apr 2017 17:32:00 +0100 Subject: [PATCH 002/158] Test against newer django versions --- .travis.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.travis.yml b/.travis.yml index 7142c9b..11839aa 100644 --- a/.travis.yml +++ b/.travis.yml @@ -6,6 +6,9 @@ python: env: - DJANGO_VERSION=1.7 - DJANGO_VERSION=1.8 +- DJANGO_VERSION=1.9 +- DJANGO_VERSION=1.10 +- DJANGO_VERSION=1.11 install: - pip install -r test-requirements.txt - pip install -U django==$DJANGO_VERSION From 01e952c5b649c9a43df2dd070f0b7c3c927f7e20 Mon Sep 17 00:00:00 2001 From: Max Hurl Date: Wed, 26 Apr 2017 10:51:40 +0100 Subject: [PATCH 003/158] Fix for Django 1.9+ and drop support for 1.7 --- .travis.yml | 1 - README.md | 4 +--- django_dbq/__init__.py | 2 +- django_dbq/management/commands/create_job.py | 23 ++++++++++++++------ django_dbq/management/commands/worker.py | 17 ++++++++------- django_dbq/models.py | 5 ++--- django_dbq/tests.py | 3 +-- test-requirements.txt | 2 +- 8 files changed, 31 insertions(+), 26 deletions(-) diff --git a/.travis.yml b/.travis.yml index 11839aa..12b0dda 100644 --- a/.travis.yml +++ b/.travis.yml @@ -4,7 +4,6 @@ python: - '2.7' - '3.4' env: -- DJANGO_VERSION=1.7 - DJANGO_VERSION=1.8 - DJANGO_VERSION=1.9 - DJANGO_VERSION=1.10 diff --git a/README.md b/README.md index 55dad27..58ab204 100644 --- a/README.md +++ b/README.md @@ -7,9 +7,7 @@ Simple databased-backed job queue. Jobs are defined in your settings, and are pr Asynchronous tasks are run via a *job queue*. This system is designed to support multi-step job workflows. -**NOTE**: This module uses differing implementations of UUIDField on Django 1.7 and 1.8 - a Python 3 shimmed django-uuidfield version on 1.7, and the built-in implementation on Django 1.8 and above. The simplest way to upgrade it is to drop the existing -`django_dbq_job` table, delete the migration from `django_migrations`, and then -re-run `manage.py migrate`. +Tested against Django 1.8, 1.9, 1.10, 1.11 ## Getting Started diff --git a/django_dbq/__init__.py b/django_dbq/__init__.py index cd7ca49..58d478a 100644 --- a/django_dbq/__init__.py +++ b/django_dbq/__init__.py @@ -1 +1 @@ -__version__ = '1.0.1' +__version__ = '1.2.0' diff --git a/django_dbq/management/commands/create_job.py b/django_dbq/management/commands/create_job.py index 93dbb01..7cb3d3b 100644 --- a/django_dbq/management/commands/create_job.py +++ b/django_dbq/management/commands/create_job.py @@ -1,7 +1,6 @@ from django.conf import settings from django.core.management.base import BaseCommand, CommandError from django_dbq.models import Job -from optparse import make_option import json import logging @@ -14,12 +13,22 @@ class Command(BaseCommand): help = "Create a job" args = '' - option_list = BaseCommand.option_list + ( - make_option('--workspace', - help='JSON-formatted initial command workspace'), - make_option('--queue_name', - help='A specific queue to add this job to'), - ) + def add_arguments(self, parser): + parser.add_argument('args', nargs='+') + parser.add_argument( + '--workspace', + action='store_true', + dest='workspace', + default=None, + help="JSON-formatted initial commandworkspace." + ) + parser.add_argument( + '--queue_name', + action='store_true', + dest='queue_name', + default=None, + help="A specific queue to add this job to" + ) def handle(self, *args, **options): if len(args) != 1: diff --git a/django_dbq/management/commands/worker.py b/django_dbq/management/commands/worker.py index 326a030..5c14c83 100644 --- a/django_dbq/management/commands/worker.py +++ b/django_dbq/management/commands/worker.py @@ -1,8 +1,7 @@ from django.db import transaction from django.core.management.base import BaseCommand, CommandError -from django.utils.module_loading import import_by_path +from django.utils.module_loading import import_string from django_dbq.models import Job -from optparse import make_option from simplesignals.process import WorkerProcessBase from time import sleep import logging @@ -27,7 +26,7 @@ def process_job(queue_name): job.save() try: - task_function = import_by_path(job.next_task) + task_function = import_string(job.next_task) task_function(job) job.update_next_task() if not job.next_task: @@ -41,7 +40,7 @@ def process_job(queue_name): failure_hook_name = job.get_failure_hook_name() if failure_hook_name: logger.info("Running failure hook %s for job id=%s", failure_hook_name, job.pk) - failure_hook_function = import_by_path(failure_hook_name) + failure_hook_function = import_string(failure_hook_name) failure_hook_function(job, exception) else: logger.info("No failure hook for job id=%s", job.pk) @@ -72,12 +71,14 @@ class Command(BaseCommand): help = "Run a queue worker process" - option_list = BaseCommand.option_list + ( - make_option('--dry-run', + def add_arguments(self, parser): + parser.add_argument('queue_name', nargs='?', default='default', type=str) + parser.add_argument( + '--dry-run', action='store_true', dest='dry_run', default=False, - help="Don't actually start the worker. Used for testing."), + help="Don't actually start the worker. Used for testing." ) def handle(self, *args, **options): @@ -87,7 +88,7 @@ def handle(self, *args, **options): if len(args) != 1: raise CommandError("Please supply a single queue job name") - queue_name = args[0] + queue_name = options['queue_name'] self.stdout.write("Starting job worker for queue \"%s\"" % queue_name) diff --git a/django_dbq/models.py b/django_dbq/models.py index f9778a1..f926fa5 100644 --- a/django_dbq/models.py +++ b/django_dbq/models.py @@ -1,5 +1,5 @@ from django.db import models -from django.utils.module_loading import import_by_path +from django.utils.module_loading import import_string from django_dbq.tasks import get_next_task_name, get_failure_hook_name, get_creation_hook_name from jsonfield import JSONField from model_utils import Choices @@ -103,6 +103,5 @@ def run_creation_hook(self): creation_hook_name = self.get_creation_hook_name() if creation_hook_name: logger.info("Running creation hook %s for new job", creation_hook_name) - creation_hook_function = import_by_path(creation_hook_name) + creation_hook_function = import_string(creation_hook_name) creation_hook_function(self) - diff --git a/django_dbq/tests.py b/django_dbq/tests.py index 654e6ea..2af866a 100644 --- a/django_dbq/tests.py +++ b/django_dbq/tests.py @@ -1,6 +1,5 @@ from datetime import datetime, timedelta from django.core.management import call_command, CommandError -from django.core.urlresolvers import reverse from django.test import TestCase from django.test.utils import override_settings from django_dbq.management.commands.worker import process_job @@ -73,7 +72,7 @@ def test_worker_no_args(self): def test_worker_with_queue_name(self): stdout = StringIO() - call_command('worker', 'test_queue', dry_run=True, stdout=stdout) + call_command('worker', queue_name='test_queue', dry_run=True, stdout=stdout) output = stdout.getvalue() self.assertTrue('test_queue' in output) diff --git a/test-requirements.txt b/test-requirements.txt index 31ea0da..74a791b 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -1,2 +1,2 @@ -r requirements.txt -pymysql==0.6.7 +pymysql==0.7.11 From 4a06b2c212af545639a4a3bfc8c7e77a2462331c Mon Sep 17 00:00:00 2001 From: Max Hurl Date: Wed, 26 Apr 2017 10:55:10 +0100 Subject: [PATCH 004/158] added tests for python 3.6 --- .travis.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.travis.yml b/.travis.yml index 12b0dda..dd161c2 100644 --- a/.travis.yml +++ b/.travis.yml @@ -3,6 +3,7 @@ sudo: false python: - '2.7' - '3.4' +- '3.6' env: - DJANGO_VERSION=1.8 - DJANGO_VERSION=1.9 From 08372c9a05a0f63d75e7883eb72d4b13d756caa1 Mon Sep 17 00:00:00 2001 From: Jordan Brown Date: Tue, 12 Sep 2017 09:41:50 +0100 Subject: [PATCH 005/158] Update docs --- README.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 09e327a..780e937 100644 --- a/README.md +++ b/README.md @@ -43,7 +43,9 @@ In project.settings: ```python JOBS = { - 'my_job': ['project.common.jobs.my_task'], + 'my_job': { + 'tasks': ['project.common.jobs.my_task'] + }, } ``` From b814137a5ab9387eb486fca9370ee78600d10e49 Mon Sep 17 00:00:00 2001 From: Jake Howard Date: Thu, 26 Apr 2018 15:28:33 +0100 Subject: [PATCH 006/158] Add priority field --- .../migrations/0003_auto_20180426_0922.py | 24 +++++++++++++++++++ django_dbq/models.py | 9 ++++--- 2 files changed, 30 insertions(+), 3 deletions(-) create mode 100644 django_dbq/migrations/0003_auto_20180426_0922.py diff --git a/django_dbq/migrations/0003_auto_20180426_0922.py b/django_dbq/migrations/0003_auto_20180426_0922.py new file mode 100644 index 0000000..af42720 --- /dev/null +++ b/django_dbq/migrations/0003_auto_20180426_0922.py @@ -0,0 +1,24 @@ +# -*- coding: utf-8 -*- +# Generated by Django 1.11 on 2018-04-26 09:22 +from __future__ import unicode_literals + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('django_dbq', '0002_auto_20151016_1027'), + ] + + operations = [ + migrations.AlterModelOptions( + name='job', + options={'ordering': ['priority', 'created']}, + ), + migrations.AddField( + model_name='job', + name='priority', + field=models.PositiveSmallIntegerField(default=0), + ), + ] diff --git a/django_dbq/models.py b/django_dbq/models.py index f926fa5..7e7fcaf 100644 --- a/django_dbq/models.py +++ b/django_dbq/models.py @@ -20,7 +20,6 @@ class JobManager(models.Manager): - def get_ready_or_none(self, queue_name, max_retries=3): """ Get a job in state READY or NEW for a given queue. Supports retrying in case of database deadlock @@ -40,7 +39,7 @@ def get_ready_or_none(self, queue_name, max_retries=3): retries_left = max_retries while True: try: - return self.select_for_update().filter(queue_name=queue_name, state__in=(Job.STATES.READY, Job.STATES.NEW)).first() + return self.to_process(queue_name).first() except Exception as e: if retries_left == 0: raise @@ -56,6 +55,9 @@ def delete_old(self): logger.info("Deleting all job in states %s created before %s", ", ".join(delete_jobs_in_states), delete_jobs_created_before.isoformat()) Job.objects.filter(state__in=delete_jobs_in_states, created__lte=delete_jobs_created_before).delete() + def to_process(self, queue_name): + return self.select_for_update().filter(queue_name=queue_name, state__in=(Job.STATES.READY, Job.STATES.NEW)) + class Job(models.Model): @@ -69,9 +71,10 @@ class Job(models.Model): next_task = models.CharField(max_length=100, blank=True) workspace = JSONField(null=True) queue_name = models.CharField(max_length=20, default='default', db_index=True) + priority = models.PositiveSmallIntegerField(default=0) class Meta: - ordering = ['created'] + ordering = ['priority', 'created'] objects = JobManager() From b8ef8458d595d900d7538dc948961ae6ae3af65f Mon Sep 17 00:00:00 2001 From: Jake Howard Date: Thu, 26 Apr 2018 15:31:59 +0100 Subject: [PATCH 007/158] Test priority order is actually used --- django_dbq/tests.py | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/django_dbq/tests.py b/django_dbq/tests.py index 2af866a..db212d7 100644 --- a/django_dbq/tests.py +++ b/django_dbq/tests.py @@ -100,6 +100,32 @@ def test_get_next_ready_job(self): self.assertEqual(Job.objects.get_ready_or_none('default'), expected) + def test_gets_jobs_in_priority_order(self): + job_1 = Job.objects.create(name='testjob') + job_2 = Job.objects.create(name='testjob', state=Job.STATES.PROCESSING) + job_3 = Job.objects.create(name='testjob', priority=3) + job_4 = Job.objects.create(name='testjob', priority=2) + self.assertEqual({ + job for job in Job.objects.to_process('default') + }, { + job_3, job_4, job_1 + }) + self.assertEqual(Job.objects.get_ready_or_none('default'), job_3) + self.assertFalse(Job.objects.to_process('default').filter(id=job_2.id).exists()) + + def test_gets_jobs_in_priority_and_date_order(self): + job_1 = Job.objects.create(name='testjob', priority=3) + job_2 = Job.objects.create(name='testjob', state=Job.STATES.PROCESSING, priority=3) + job_3 = Job.objects.create(name='testjob', priority=3) + job_4 = Job.objects.create(name='testjob', priority=3) + self.assertEqual({ + job for job in Job.objects.to_process('default') + }, { + job_1, job_3, job_4 + }) + self.assertEqual(Job.objects.get_ready_or_none('default'), job_1) + self.assertFalse(Job.objects.to_process('default').filter(id=job_2.id).exists()) + def test_get_next_ready_job_created(self): """ Created jobs should be picked too. From c2c2a86b580a5218767791abddddc5ba0b6cd6e0 Mon Sep 17 00:00:00 2001 From: Jake Howard Date: Thu, 26 Apr 2018 15:41:23 +0100 Subject: [PATCH 008/158] Correct ordering 0 is meant to be LOWEST priority --- django_dbq/migrations/0003_auto_20180426_0922.py | 2 +- django_dbq/models.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/django_dbq/migrations/0003_auto_20180426_0922.py b/django_dbq/migrations/0003_auto_20180426_0922.py index af42720..e7dc5b0 100644 --- a/django_dbq/migrations/0003_auto_20180426_0922.py +++ b/django_dbq/migrations/0003_auto_20180426_0922.py @@ -14,7 +14,7 @@ class Migration(migrations.Migration): operations = [ migrations.AlterModelOptions( name='job', - options={'ordering': ['priority', 'created']}, + options={'ordering': ['-priority', 'created']}, ), migrations.AddField( model_name='job', diff --git a/django_dbq/models.py b/django_dbq/models.py index 7e7fcaf..b41ed20 100644 --- a/django_dbq/models.py +++ b/django_dbq/models.py @@ -74,7 +74,7 @@ class Job(models.Model): priority = models.PositiveSmallIntegerField(default=0) class Meta: - ordering = ['priority', 'created'] + ordering = ['-priority', 'created'] objects = JobManager() From fad0c1c624eab6777eda7b2349bbdcc3f6484ffd Mon Sep 17 00:00:00 2001 From: Jake Howard Date: Thu, 26 Apr 2018 16:10:10 +0100 Subject: [PATCH 009/158] Add db index to test performance --- .../migrations/0004_auto_20180426_1009.py | 20 +++++++++++++++++++ django_dbq/models.py | 2 +- 2 files changed, 21 insertions(+), 1 deletion(-) create mode 100644 django_dbq/migrations/0004_auto_20180426_1009.py diff --git a/django_dbq/migrations/0004_auto_20180426_1009.py b/django_dbq/migrations/0004_auto_20180426_1009.py new file mode 100644 index 0000000..3f63d15 --- /dev/null +++ b/django_dbq/migrations/0004_auto_20180426_1009.py @@ -0,0 +1,20 @@ +# -*- coding: utf-8 -*- +# Generated by Django 1.11 on 2018-04-26 10:09 +from __future__ import unicode_literals + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('django_dbq', '0003_auto_20180426_0922'), + ] + + operations = [ + migrations.AlterField( + model_name='job', + name='priority', + field=models.PositiveSmallIntegerField(db_index=True, default=0), + ), + ] diff --git a/django_dbq/models.py b/django_dbq/models.py index b41ed20..3c5131a 100644 --- a/django_dbq/models.py +++ b/django_dbq/models.py @@ -71,7 +71,7 @@ class Job(models.Model): next_task = models.CharField(max_length=100, blank=True) workspace = JSONField(null=True) queue_name = models.CharField(max_length=20, default='default', db_index=True) - priority = models.PositiveSmallIntegerField(default=0) + priority = models.PositiveSmallIntegerField(default=0, db_index=True) class Meta: ordering = ['-priority', 'created'] From 017ae940c880871e0694d2d160194cd99950ed64 Mon Sep 17 00:00:00 2001 From: Jake Howard Date: Thu, 26 Apr 2018 16:24:58 +0100 Subject: [PATCH 010/158] Squash migrations --- ...426_0922.py => 0003_auto_20180426_1024.py} | 4 ++-- .../migrations/0004_auto_20180426_1009.py | 20 ------------------- 2 files changed, 2 insertions(+), 22 deletions(-) rename django_dbq/migrations/{0003_auto_20180426_0922.py => 0003_auto_20180426_1024.py} (79%) delete mode 100644 django_dbq/migrations/0004_auto_20180426_1009.py diff --git a/django_dbq/migrations/0003_auto_20180426_0922.py b/django_dbq/migrations/0003_auto_20180426_1024.py similarity index 79% rename from django_dbq/migrations/0003_auto_20180426_0922.py rename to django_dbq/migrations/0003_auto_20180426_1024.py index e7dc5b0..da7b438 100644 --- a/django_dbq/migrations/0003_auto_20180426_0922.py +++ b/django_dbq/migrations/0003_auto_20180426_1024.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Generated by Django 1.11 on 2018-04-26 09:22 +# Generated by Django 1.11 on 2018-04-26 10:24 from __future__ import unicode_literals from django.db import migrations, models @@ -19,6 +19,6 @@ class Migration(migrations.Migration): migrations.AddField( model_name='job', name='priority', - field=models.PositiveSmallIntegerField(default=0), + field=models.PositiveSmallIntegerField(db_index=True, default=0), ), ] diff --git a/django_dbq/migrations/0004_auto_20180426_1009.py b/django_dbq/migrations/0004_auto_20180426_1009.py deleted file mode 100644 index 3f63d15..0000000 --- a/django_dbq/migrations/0004_auto_20180426_1009.py +++ /dev/null @@ -1,20 +0,0 @@ -# -*- coding: utf-8 -*- -# Generated by Django 1.11 on 2018-04-26 10:09 -from __future__ import unicode_literals - -from django.db import migrations, models - - -class Migration(migrations.Migration): - - dependencies = [ - ('django_dbq', '0003_auto_20180426_0922'), - ] - - operations = [ - migrations.AlterField( - model_name='job', - name='priority', - field=models.PositiveSmallIntegerField(db_index=True, default=0), - ), - ] From 544ee2191655665572e5e9ca890d73ec432b0db4 Mon Sep 17 00:00:00 2001 From: Jake Howard Date: Fri, 13 Jul 2018 16:00:33 +0100 Subject: [PATCH 011/158] Allow negative priorities This allows both promoting and demoting of jobs --- ..._20180426_1024.py => 0003_auto_20180713_1000.py} | 4 ++-- django_dbq/models.py | 2 +- django_dbq/tests.py | 13 +++++++++++++ 3 files changed, 16 insertions(+), 3 deletions(-) rename django_dbq/migrations/{0003_auto_20180426_1024.py => 0003_auto_20180713_1000.py} (79%) diff --git a/django_dbq/migrations/0003_auto_20180426_1024.py b/django_dbq/migrations/0003_auto_20180713_1000.py similarity index 79% rename from django_dbq/migrations/0003_auto_20180426_1024.py rename to django_dbq/migrations/0003_auto_20180713_1000.py index da7b438..4763455 100644 --- a/django_dbq/migrations/0003_auto_20180426_1024.py +++ b/django_dbq/migrations/0003_auto_20180713_1000.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Generated by Django 1.11 on 2018-04-26 10:24 +# Generated by Django 1.11 on 2018-07-13 10:00 from __future__ import unicode_literals from django.db import migrations, models @@ -19,6 +19,6 @@ class Migration(migrations.Migration): migrations.AddField( model_name='job', name='priority', - field=models.PositiveSmallIntegerField(db_index=True, default=0), + field=models.SmallIntegerField(db_index=True, default=0), ), ] diff --git a/django_dbq/models.py b/django_dbq/models.py index 3c5131a..33f89a1 100644 --- a/django_dbq/models.py +++ b/django_dbq/models.py @@ -71,7 +71,7 @@ class Job(models.Model): next_task = models.CharField(max_length=100, blank=True) workspace = JSONField(null=True) queue_name = models.CharField(max_length=20, default='default', db_index=True) - priority = models.PositiveSmallIntegerField(default=0, db_index=True) + priority = models.SmallIntegerField(default=0, db_index=True) class Meta: ordering = ['-priority', 'created'] diff --git a/django_dbq/tests.py b/django_dbq/tests.py index db212d7..dbba1c7 100644 --- a/django_dbq/tests.py +++ b/django_dbq/tests.py @@ -113,6 +113,19 @@ def test_gets_jobs_in_priority_order(self): self.assertEqual(Job.objects.get_ready_or_none('default'), job_3) self.assertFalse(Job.objects.to_process('default').filter(id=job_2.id).exists()) + def test_gets_jobs_in_negative_priority_order(self): + job_1 = Job.objects.create(name='testjob') + job_2 = Job.objects.create(name='testjob', state=Job.STATES.PROCESSING) + job_3 = Job.objects.create(name='testjob', priority=-2) + job_4 = Job.objects.create(name='testjob', priority=1) + self.assertEqual({ + job for job in Job.objects.to_process('default') + }, { + job_4, job_3, job_1 + }) + self.assertEqual(Job.objects.get_ready_or_none('default'), job_4) + self.assertFalse(Job.objects.to_process('default').filter(id=job_2.id).exists()) + def test_gets_jobs_in_priority_and_date_order(self): job_1 = Job.objects.create(name='testjob', priority=3) job_2 = Job.objects.create(name='testjob', state=Job.STATES.PROCESSING, priority=3) From 5184b3a9f99034a650dddea7936e3af526698c28 Mon Sep 17 00:00:00 2001 From: Timothy Stimpson <32487390+timstimpson@users.noreply.github.com> Date: Thu, 29 Nov 2018 16:18:53 +0000 Subject: [PATCH 012/158] Update README.md --- README.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/README.md b/README.md index 780e937..86a3cb7 100644 --- a/README.md +++ b/README.md @@ -141,3 +141,7 @@ To start a worker: ## Testing It may be necessary to supply a DATABASE_PORT environment variable. + +## Code of conduct + +For guidelines regarding the code of conduct when contributing to this repository please review [https://www.dabapps.com/open-source/code-of-conduct/](https://www.dabapps.com/open-source/code-of-conduct/) From 64538824f27444e58bf5c48d408b49ee874be527 Mon Sep 17 00:00:00 2001 From: Jordan Brown Date: Thu, 7 Mar 2019 11:29:18 +0000 Subject: [PATCH 013/158] Document hooks in readme --- README.md | 52 +++++++++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 49 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 86a3cb7..feace92 100644 --- a/README.md +++ b/README.md @@ -12,11 +12,11 @@ Tested against Django 1.8, 1.9, 1.10, 1.11 ## Getting Started ### Installation - + Install from PIP - + pip install django-db-queue - + Add `django_dbq` to your installed apps INSTALLED_APPS = ( @@ -49,6 +49,52 @@ JOBS = { } ``` +### Hooks + + +#### Failure Hooks +When an unhandled exception is raised by a job, a failure hook will be called if one exists enabling +you to clean up any state left behind by your failed job. + +A failure hook receives the failed `Job` instance along with the unhandled exception raised by your failed job as its arguments. Here's an example: + +```python +def my_task_failure_hook(job, e): + # delete some temporary files on the filesystem +``` + +To ensure this hook gets run, simply add a `failure_hook` key to your job config like so: + +```python +JOBS = { + 'my_job': { + 'tasks': ['project.common.jobs.my_task'], + 'failure_hook': 'project.common.jobs.my_task_failure_hook' + }, +} +``` + +#### Creation Hooks +You can also run creation hooks, which are run just before a job begins. + +A creation hook receives your `Job` instance as its only argument. Here's an example: + +```python +def my_task_creation_hook(job): + # configure something before running your job +``` + +To ensure this hook gets run, simply add a `creation_hook` key to your job config like so: + +```python +JOBS = { + 'my_job': { + 'tasks': ['project.common.jobs.my_task'], + 'creation_hook': 'project.common.jobs.my_task_creation_hook' + }, +} +``` + ### Start the worker In another terminal: From 6138f4ebfac29d53231b44b9e9ef3af4776492ca Mon Sep 17 00:00:00 2001 From: Jordan Brown Date: Thu, 7 Mar 2019 11:39:23 +0000 Subject: [PATCH 014/158] Add which process hooks run in --- README.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index feace92..34a6b9d 100644 --- a/README.md +++ b/README.md @@ -54,7 +54,7 @@ JOBS = { #### Failure Hooks When an unhandled exception is raised by a job, a failure hook will be called if one exists enabling -you to clean up any state left behind by your failed job. +you to clean up any state left behind by your failed job. Failure hook are run in your worker process (if your job fails). A failure hook receives the failed `Job` instance along with the unhandled exception raised by your failed job as its arguments. Here's an example: @@ -75,7 +75,8 @@ JOBS = { ``` #### Creation Hooks -You can also run creation hooks, which are run just before a job begins. +You can also run creation hooks, which happen just after the creation of your `Job` instances and are executed in the process +in which the job was created, _not the worker process_. A creation hook receives your `Job` instance as its only argument. Here's an example: From 95f4f0746b4837b44e1d722e8e807e15b0555b45 Mon Sep 17 00:00:00 2001 From: Jordan Brown Date: Thu, 7 Mar 2019 11:41:59 +0000 Subject: [PATCH 015/158] Fix typo --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 34a6b9d..c53fc4d 100644 --- a/README.md +++ b/README.md @@ -54,7 +54,7 @@ JOBS = { #### Failure Hooks When an unhandled exception is raised by a job, a failure hook will be called if one exists enabling -you to clean up any state left behind by your failed job. Failure hook are run in your worker process (if your job fails). +you to clean up any state left behind by your failed job. Failure hooks are run in your worker process (if your job fails). A failure hook receives the failed `Job` instance along with the unhandled exception raised by your failed job as its arguments. Here's an example: From d0fe290f95e398b33714510ac4ed66a9ee813238 Mon Sep 17 00:00:00 2001 From: Jordan Brown Date: Tue, 19 Nov 2019 09:47:39 +0000 Subject: [PATCH 016/158] Add rate limit to queues --- .python-version | 1 + django_dbq/management/commands/worker.py | 15 +++++++-- django_dbq/tests.py | 40 +++++++++++++++++++++++- test-requirements.txt | 1 + 4 files changed, 53 insertions(+), 4 deletions(-) create mode 100644 .python-version diff --git a/.python-version b/.python-version new file mode 100644 index 0000000..0833a98 --- /dev/null +++ b/.python-version @@ -0,0 +1 @@ +3.7.4 diff --git a/django_dbq/management/commands/worker.py b/django_dbq/management/commands/worker.py index 5c14c83..0aeaa97 100644 --- a/django_dbq/management/commands/worker.py +++ b/django_dbq/management/commands/worker.py @@ -1,5 +1,6 @@ from django.db import transaction from django.core.management.base import BaseCommand, CommandError +from django.utils import timezone from django.utils.module_loading import import_string from django_dbq.models import Job from simplesignals.process import WorkerProcessBase @@ -58,13 +59,19 @@ class Worker(WorkerProcessBase): process_title = "jobworker" - def __init__(self, name): + def __init__(self, name, rate_limit_in_seconds): self.queue_name = name + self.rate_limit_in_seconds = rate_limit_in_seconds + self.last_job_finished = None super(Worker, self).__init__() def do_work(self): sleep(1) + if self.last_job_finished and (self.last_job_finished - timezone.now()).seconds < self.rate_limit_in_seconds: + return + process_job(self.queue_name) + self.last_job_finished = timezone.now() class Command(BaseCommand): @@ -73,6 +80,7 @@ class Command(BaseCommand): def add_arguments(self, parser): parser.add_argument('queue_name', nargs='?', default='default', type=str) + parser.add_argument('rate_limit', help='The rate limit in seconds. The default rate limit is 1 job per second.', nargs='?', default=1, type=int) parser.add_argument( '--dry-run', action='store_true', @@ -89,10 +97,11 @@ def handle(self, *args, **options): raise CommandError("Please supply a single queue job name") queue_name = options['queue_name'] + rate_limit_in_seconds = options['rate_limit'] - self.stdout.write("Starting job worker for queue \"%s\"" % queue_name) + self.stdout.write("Starting job worker for queue \"%s\" with rate limit %s/s" % (queue_name, rate_limit_in_seconds)) - worker = Worker(queue_name) + worker = Worker(queue_name, rate_limit_in_seconds) if options['dry_run']: return diff --git a/django_dbq/tests.py b/django_dbq/tests.py index 2af866a..93de2df 100644 --- a/django_dbq/tests.py +++ b/django_dbq/tests.py @@ -1,8 +1,13 @@ from datetime import datetime, timedelta +from unittest import mock + +import freezegun from django.core.management import call_command, CommandError from django.test import TestCase from django.test.utils import override_settings -from django_dbq.management.commands.worker import process_job +from django.utils import timezone + +from django_dbq.management.commands.worker import process_job, Worker from django_dbq.models import Job try: from StringIO import StringIO @@ -77,6 +82,39 @@ def test_worker_with_queue_name(self): self.assertTrue('test_queue' in output) +@freezegun.freeze_time() +@mock.patch('django_dbq.management.commands.worker.time.sleep') +@mock.patch('django_dbq.management.commands.worker.process_job') +class WorkerProcessDoWorkTestCase(TestCase): + + def setUp(self): + super().setUp() + self.MockWorker = mock.MagicMock() + self.MockWorker.queue_name = 'default' + self.MockWorker.rate_limit_in_seconds = 5 + self.MockWorker.last_job_finished = None + + def test_do_work_no_previous_job_run(self, mock_process_job, mock_sleep): + Worker.do_work(self.MockWorker) + self.assertEqual(mock_sleep.call_count, 1) + self.assertEqual(mock_process_job.call_count, 1) + self.assertEqual(self.MockWorker.last_job_finished, timezone.now()) + + def test_do_work_previous_job_too_soon(self, mock_process_job, mock_sleep): + self.MockWorker.last_job_finished = timezone.now() - timezone.timedelta(seconds=2) + Worker.do_work(self.MockWorker) + self.assertEqual(mock_sleep.call_count, 1) + self.assertEqual(mock_process_job.call_count, 0) + self.assertEqual(self.MockWorker.last_job_finished, timezone.now() - timezone.timedelta(seconds=2)) + + def test_do_work_previous_job_long_time_ago(self, mock_process_job, mock_sleep): + self.MockWorker.last_job_finished = timezone.now() - timezone.timedelta(seconds=7) + Worker.do_work(self.MockWorker) + self.assertEqual(mock_sleep.call_count, 1) + self.assertEqual(mock_process_job.call_count, 1) + self.assertEqual(self.MockWorker.last_job_finished, timezone.now()) + + @override_settings(JOBS={'testjob': {'tasks': ['a']}}) class JobTestCase(TestCase): diff --git a/test-requirements.txt b/test-requirements.txt index 74a791b..55b7282 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -1,2 +1,3 @@ -r requirements.txt pymysql==0.7.11 +freezegun==0.3.12 From d445da96c362a7d7b2cec96382fb299dc1a24e1a Mon Sep 17 00:00:00 2001 From: Jordan Brown Date: Thu, 21 Nov 2019 11:41:16 +0000 Subject: [PATCH 017/158] Add MySQL service to .travis.yml --- .travis.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.travis.yml b/.travis.yml index dd161c2..6c4bbe0 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,5 +1,7 @@ language: python sudo: false +services: + - mysql python: - '2.7' - '3.4' From acf9ae9fc5fd780277400dccd43c3f686c07b8f2 Mon Sep 17 00:00:00 2001 From: Jordan Brown Date: Thu, 21 Nov 2019 12:17:59 +0000 Subject: [PATCH 018/158] Fix unit tests --- django_dbq/management/commands/worker.py | 2 +- django_dbq/tests.py | 4 ++-- test-requirements.txt | 1 + 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/django_dbq/management/commands/worker.py b/django_dbq/management/commands/worker.py index 0aeaa97..3eb5bcd 100644 --- a/django_dbq/management/commands/worker.py +++ b/django_dbq/management/commands/worker.py @@ -67,7 +67,7 @@ def __init__(self, name, rate_limit_in_seconds): def do_work(self): sleep(1) - if self.last_job_finished and (self.last_job_finished - timezone.now()).seconds < self.rate_limit_in_seconds: + if self.last_job_finished and (timezone.now() - self.last_job_finished).seconds < self.rate_limit_in_seconds: return process_job(self.queue_name) diff --git a/django_dbq/tests.py b/django_dbq/tests.py index 93de2df..1b02b56 100644 --- a/django_dbq/tests.py +++ b/django_dbq/tests.py @@ -1,5 +1,5 @@ from datetime import datetime, timedelta -from unittest import mock +import mock import freezegun from django.core.management import call_command, CommandError @@ -83,7 +83,7 @@ def test_worker_with_queue_name(self): @freezegun.freeze_time() -@mock.patch('django_dbq.management.commands.worker.time.sleep') +@mock.patch('django_dbq.management.commands.worker.sleep') @mock.patch('django_dbq.management.commands.worker.process_job') class WorkerProcessDoWorkTestCase(TestCase): diff --git a/test-requirements.txt b/test-requirements.txt index 55b7282..a496570 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -1,3 +1,4 @@ -r requirements.txt pymysql==0.7.11 freezegun==0.3.12 +mock==3.0.5 From b8eb107cf0d6b262fd99bb0fdba9d8698d425bc2 Mon Sep 17 00:00:00 2001 From: Jordan Brown Date: Thu, 21 Nov 2019 13:37:36 +0000 Subject: [PATCH 019/158] Provide arguments to super() for compatibility with Python 2.7 --- django_dbq/tests.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/django_dbq/tests.py b/django_dbq/tests.py index 1b02b56..3b2907e 100644 --- a/django_dbq/tests.py +++ b/django_dbq/tests.py @@ -88,7 +88,7 @@ def test_worker_with_queue_name(self): class WorkerProcessDoWorkTestCase(TestCase): def setUp(self): - super().setUp() + super(WorkerProcessDoWorkTestCase, self).setUp() self.MockWorker = mock.MagicMock() self.MockWorker.queue_name = 'default' self.MockWorker.rate_limit_in_seconds = 5 From c536841dfc12e647a6ac0fc411dbab2db5b9a847 Mon Sep 17 00:00:00 2001 From: Jordan Brown Date: Thu, 21 Nov 2019 13:40:00 +0000 Subject: [PATCH 020/158] Drop support for Python 2.7 and add support for Python 3.7 --- .travis.yml | 2 +- django_dbq/tests.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 6c4bbe0..9f05a39 100644 --- a/.travis.yml +++ b/.travis.yml @@ -3,9 +3,9 @@ sudo: false services: - mysql python: -- '2.7' - '3.4' - '3.6' +- '3.7' env: - DJANGO_VERSION=1.8 - DJANGO_VERSION=1.9 diff --git a/django_dbq/tests.py b/django_dbq/tests.py index 3b2907e..1b02b56 100644 --- a/django_dbq/tests.py +++ b/django_dbq/tests.py @@ -88,7 +88,7 @@ def test_worker_with_queue_name(self): class WorkerProcessDoWorkTestCase(TestCase): def setUp(self): - super(WorkerProcessDoWorkTestCase, self).setUp() + super().setUp() self.MockWorker = mock.MagicMock() self.MockWorker.queue_name = 'default' self.MockWorker.rate_limit_in_seconds = 5 From 8555f5d0b4a13159e290943851f9f2d5da40ff25 Mon Sep 17 00:00:00 2001 From: Jordan Brown Date: Thu, 21 Nov 2019 13:52:47 +0000 Subject: [PATCH 021/158] Document the new flag --- README.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index c53fc4d..cb9d00e 100644 --- a/README.md +++ b/README.md @@ -181,9 +181,10 @@ The `workspace` flag is optional. If supplied, it must be a valid JSON string. To start a worker: - manage.py worker [queue_name] + manage.py worker [queue_name] [--rate_limit] -`queue_name` is optional, and will default to `default` +- `queue_name` is optional, and will default to `default` +- The `--rate_limit` flag is optional, and will default to `1`. It is the minimum number of seconds that must have elapsed before a subsequent job can be run. ## Testing From f5b674bfc071310044a2f2418461584ea53186d5 Mon Sep 17 00:00:00 2001 From: Jordan Brown Date: Thu, 21 Nov 2019 13:59:37 +0000 Subject: [PATCH 022/158] Use total_seconds over .seconds --- django_dbq/management/commands/worker.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/django_dbq/management/commands/worker.py b/django_dbq/management/commands/worker.py index 3eb5bcd..cf6b19d 100644 --- a/django_dbq/management/commands/worker.py +++ b/django_dbq/management/commands/worker.py @@ -67,7 +67,7 @@ def __init__(self, name, rate_limit_in_seconds): def do_work(self): sleep(1) - if self.last_job_finished and (timezone.now() - self.last_job_finished).seconds < self.rate_limit_in_seconds: + if self.last_job_finished and (timezone.now() - self.last_job_finished).total_seconds() < self.rate_limit_in_seconds: return process_job(self.queue_name) From 096e7e05c67ec99fa454002f31694e9a7b8c233f Mon Sep 17 00:00:00 2001 From: Jordan Brown Date: Thu, 21 Nov 2019 15:01:08 +0000 Subject: [PATCH 023/158] Clean up dependencies, add Python 3.7, Django 2.2, PostgresSQL testing to Travis matrix --- .travis.yml | 5 ++++ README.md | 4 +++- django_dbq/management/commands/worker.py | 8 +++---- django_dbq/models.py | 21 +++++++++++++---- django_dbq/tests.py | 30 ++++++++++++------------ setup.py | 4 +--- test-requirements.txt | 1 + testsettings.py | 7 ++---- 8 files changed, 48 insertions(+), 32 deletions(-) diff --git a/.travis.yml b/.travis.yml index 9f05a39..efaba87 100644 --- a/.travis.yml +++ b/.travis.yml @@ -2,15 +2,20 @@ language: python sudo: false services: - mysql + - postgresql python: - '3.4' - '3.6' - '3.7' +- '3.8' env: - DJANGO_VERSION=1.8 - DJANGO_VERSION=1.9 - DJANGO_VERSION=1.10 - DJANGO_VERSION=1.11 +- DJANGO_VERSION=2.2 +- DATABASE_URL=mysql://root@127.0.0.1/dbq +- DATABASE_URL=postgres://postgres@127.0.0.1/dbq install: - pip install -r test-requirements.txt - pip install -U django==$DJANGO_VERSION diff --git a/README.md b/README.md index cb9d00e..d1b15da 100644 --- a/README.md +++ b/README.md @@ -7,7 +7,9 @@ Simple databased-backed job queue. Jobs are defined in your settings, and are pr Asynchronous tasks are run via a *job queue*. This system is designed to support multi-step job workflows. -Tested against Django 1.8, 1.9, 1.10, 1.11 +Supported and tested against: +- Django 1.8, 1.9, 1.10, 1.11 and 2.2 +- Python 3.4, 3.6, 3.7 and 3.8 ## Getting Started diff --git a/django_dbq/management/commands/worker.py b/django_dbq/management/commands/worker.py index cf6b19d..2cf97af 100644 --- a/django_dbq/management/commands/worker.py +++ b/django_dbq/management/commands/worker.py @@ -23,7 +23,7 @@ def process_job(queue_name): return logger.info('Processing job: name="%s" queue="%s" id=%s state=%s next_task=%s', job.name, queue_name, job.pk, job.state, job.next_task) - job.state = Job.STATES.PROCESSING + job.state = Job.State.PROCESSING job.save() try: @@ -31,12 +31,12 @@ def process_job(queue_name): task_function(job) job.update_next_task() if not job.next_task: - job.state = Job.STATES.COMPLETE + job.state = Job.State.COMPLETE else: - job.state = Job.STATES.READY + job.state = Job.State.READY except Exception as exception: logger.exception("Job id=%s failed", job.pk) - job.state = Job.STATES.FAILED + job.state = Job.State.FAILED failure_hook_name = job.get_failure_hook_name() if failure_hook_name: diff --git a/django_dbq/models.py b/django_dbq/models.py index f926fa5..52a9b7a 100644 --- a/django_dbq/models.py +++ b/django_dbq/models.py @@ -40,7 +40,7 @@ def get_ready_or_none(self, queue_name, max_retries=3): retries_left = max_retries while True: try: - return self.select_for_update().filter(queue_name=queue_name, state__in=(Job.STATES.READY, Job.STATES.NEW)).first() + return self.select_for_update().filter(queue_name=queue_name, state__in=(Job.State.READY, Job.State.NEW)).first() except Exception as e: if retries_left == 0: raise @@ -51,7 +51,7 @@ def delete_old(self): """ Delete all jobs older than DELETE_JOBS_AFTER_HOURS """ - delete_jobs_in_states = [Job.STATES.FAILED, Job.STATES.COMPLETE] + delete_jobs_in_states = [Job.State.FAILED, Job.State.COMPLETE] delete_jobs_created_before = datetime.datetime.utcnow() - datetime.timedelta(hours=DELETE_JOBS_AFTER_HOURS) logger.info("Deleting all job in states %s created before %s", ", ".join(delete_jobs_in_states), delete_jobs_created_before.isoformat()) Job.objects.filter(state__in=delete_jobs_in_states, created__lte=delete_jobs_created_before).delete() @@ -59,13 +59,26 @@ def delete_old(self): class Job(models.Model): - STATES = Choices("NEW", "READY", "PROCESSING", "FAILED", "COMPLETE") + class State: + NEW = 'NEW' + READY = 'READY' + PROCESSING = 'PROCESSING' + FAILED = 'FAILED' + COMPLETE = 'COMPLETE' + + STATES = [ + (State.NEW, "NEW"), + (State.READY, "READY"), + (State.PROCESSING, "PROCESSING"), + (State.FAILED, "FAILED"), + (State.COMPLETE, "COMPLETE"), + ] id = UUIDField(primary_key=True, default=uuid.uuid4, editable=False) created = models.DateTimeField(auto_now_add=True, db_index=True) modified = models.DateTimeField(auto_now=True) name = models.CharField(max_length=100) - state = models.CharField(max_length=20, choices=STATES, default=STATES.NEW, db_index=True) + state = models.CharField(max_length=20, choices=STATES, default=State.NEW, db_index=True) next_task = models.CharField(max_length=100, blank=True) workspace = JSONField(null=True) queue_name = models.CharField(max_length=20, default='default', db_index=True) diff --git a/django_dbq/tests.py b/django_dbq/tests.py index 1b02b56..e976f51 100644 --- a/django_dbq/tests.py +++ b/django_dbq/tests.py @@ -120,19 +120,19 @@ class JobTestCase(TestCase): def test_create_job(self): job = Job(name='testjob') - self.assertEqual(job.state, Job.STATES.NEW) + self.assertEqual(job.state, Job.State.NEW) def test_create_job_with_queue(self): job = Job(name='testjob', queue_name='lol') - self.assertEqual(job.state, Job.STATES.NEW) + self.assertEqual(job.state, Job.State.NEW) self.assertEqual(job.queue_name, 'lol') def test_get_next_ready_job(self): self.assertTrue(Job.objects.get_ready_or_none('default') is None) - Job.objects.create(name='testjob', state=Job.STATES.READY) - Job.objects.create(name='testjob', state=Job.STATES.PROCESSING) - expected = Job.objects.create(name='testjob', state=Job.STATES.READY) + Job.objects.create(name='testjob', state=Job.State.READY) + Job.objects.create(name='testjob', state=Job.State.PROCESSING) + expected = Job.objects.create(name='testjob', state=Job.State.READY) expected.created = datetime.now() - timedelta(minutes=1) expected.save() @@ -148,9 +148,9 @@ def test_get_next_ready_job_created(self): """ self.assertTrue(Job.objects.get_ready_or_none('default') is None) - Job.objects.create(name='testjob', state=Job.STATES.NEW) - Job.objects.create(name='testjob', state=Job.STATES.PROCESSING) - expected = Job.objects.create(name='testjob', state=Job.STATES.NEW) + Job.objects.create(name='testjob', state=Job.State.NEW) + Job.objects.create(name='testjob', state=Job.State.PROCESSING) + expected = Job.objects.create(name='testjob', state=Job.State.NEW) expected.created = datetime.now() - timedelta(minutes=1) expected.save() @@ -178,7 +178,7 @@ def test_process_job(self): job = Job.objects.create(name='testjob') process_job('default') job = Job.objects.get() - self.assertEqual(job.state, Job.STATES.COMPLETE) + self.assertEqual(job.state, Job.State.COMPLETE) def test_process_job_wrong_queue(self): """ @@ -187,7 +187,7 @@ def test_process_job_wrong_queue(self): job = Job.objects.create(name='testjob', queue_name='lol') process_job('default') job = Job.objects.get() - self.assertEqual(job.state, Job.STATES.NEW) + self.assertEqual(job.state, Job.State.NEW) @override_settings(JOBS={'testjob': {'tasks': ['django_dbq.tests.test_task'], 'creation_hook': 'django_dbq.tests.creation_hook'}}) @@ -214,7 +214,7 @@ def test_failure_hook(self): job = Job.objects.create(name='testjob') process_job('default') job = Job.objects.get() - self.assertEqual(job.state, Job.STATES.FAILED) + self.assertEqual(job.state, Job.State.FAILED) self.assertEqual(job.workspace['output'], 'failure hook ran') @@ -224,19 +224,19 @@ class DeleteOldJobsTestCase(TestCase): def test_delete_old_jobs(self): two_days_ago = datetime.utcnow() - timedelta(days=2) - j1 = Job.objects.create(name='testjob', state=Job.STATES.COMPLETE) + j1 = Job.objects.create(name='testjob', state=Job.State.COMPLETE) j1.created = two_days_ago j1.save() - j2 = Job.objects.create(name='testjob', state=Job.STATES.FAILED) + j2 = Job.objects.create(name='testjob', state=Job.State.FAILED) j2.created = two_days_ago j2.save() - j3 = Job.objects.create(name='testjob', state=Job.STATES.NEW) + j3 = Job.objects.create(name='testjob', state=Job.State.NEW) j3.created = two_days_ago j3.save() - j4 = Job.objects.create(name='testjob', state=Job.STATES.COMPLETE) + j4 = Job.objects.create(name='testjob', state=Job.State.COMPLETE) Job.objects.delete_old() diff --git a/setup.py b/setup.py index 80c966b..2fd5d92 100644 --- a/setup.py +++ b/setup.py @@ -16,9 +16,7 @@ author_email = 'contact@dabapps.com' license = 'BSD' install_requires = [ - "django-model-utils==2.3.1", - "django-uuidfield==0.5.0", - "jsonfield==1.0.3", + "jsonfield==2.0.2", "Django>=1.7", "simplesignals==0.3.0", ] diff --git a/test-requirements.txt b/test-requirements.txt index a496570..8ef10c7 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -2,3 +2,4 @@ pymysql==0.7.11 freezegun==0.3.12 mock==3.0.5 +dj-database-url==0.5.0 diff --git a/testsettings.py b/testsettings.py index 868c9e9..97e0b6c 100644 --- a/testsettings.py +++ b/testsettings.py @@ -1,13 +1,10 @@ import os import pymysql +import dj_database_url pymysql.install_as_MySQLdb() DATABASES = { - 'default': { - 'ENGINE': 'django.db.backends.mysql', - 'NAME': 'django_db_queue', - 'PORT': os.getenv('DATABASE_PORT', 3306), - }, + 'default': dj_database_url.parse(os.environ['DATABASE_URL']), } INSTALLED_APPS = ( From c184dc3b58b0679576bc209909563eb7958286e7 Mon Sep 17 00:00:00 2001 From: Jordan Brown Date: Thu, 21 Nov 2019 15:49:38 +0000 Subject: [PATCH 024/158] Remove UUIDField and always use Djangos one --- django_dbq/fields.py | 7 ------- django_dbq/models.py | 7 +------ 2 files changed, 1 insertion(+), 13 deletions(-) delete mode 100644 django_dbq/fields.py diff --git a/django_dbq/fields.py b/django_dbq/fields.py deleted file mode 100644 index 6991a8d..0000000 --- a/django_dbq/fields.py +++ /dev/null @@ -1,7 +0,0 @@ -from uuidfield import UUIDField -from django.db.models import SubfieldBase -from django.utils import six - - -class UUIDField(six.with_metaclass(SubfieldBase, UUIDField)): - pass diff --git a/django_dbq/models.py b/django_dbq/models.py index 61617bb..f91fe8f 100644 --- a/django_dbq/models.py +++ b/django_dbq/models.py @@ -2,16 +2,11 @@ from django.utils.module_loading import import_string from django_dbq.tasks import get_next_task_name, get_failure_hook_name, get_creation_hook_name from jsonfield import JSONField -from model_utils import Choices +from django.db.models import UUIDField import datetime import logging import uuid -try: - from django.db.models import UUIDField -except ImportError: - from django_dbq.fields import UUIDField - logger = logging.getLogger(__name__) From 126efb01d2eb02d330eea8cdd696c8f2e616e56e Mon Sep 17 00:00:00 2001 From: Jordan Brown Date: Thu, 21 Nov 2019 15:49:48 +0000 Subject: [PATCH 025/158] Add database URLs to testing matrix --- .travis.yml | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/.travis.yml b/.travis.yml index efaba87..c6a3a39 100644 --- a/.travis.yml +++ b/.travis.yml @@ -9,13 +9,16 @@ python: - '3.7' - '3.8' env: -- DJANGO_VERSION=1.8 -- DJANGO_VERSION=1.9 -- DJANGO_VERSION=1.10 -- DJANGO_VERSION=1.11 -- DJANGO_VERSION=2.2 -- DATABASE_URL=mysql://root@127.0.0.1/dbq -- DATABASE_URL=postgres://postgres@127.0.0.1/dbq +- DJANGO_VERSION=1.8 DATABASE_URL=postgres://postgres@127.0.0.1/dbq +- DJANGO_VERSION=1.9 DATABASE_URL=postgres://postgres@127.0.0.1/dbq +- DJANGO_VERSION=1.10 DATABASE_URL=postgres://postgres@127.0.0.1/dbq +- DJANGO_VERSION=1.11 DATABASE_URL=postgres://postgres@127.0.0.1/dbq +- DJANGO_VERSION=2.2 DATABASE_URL=postgres://postgres@127.0.0.1/dbq +- DJANGO_VERSION=1.8 DATABASE_URL=mysql://root@127.0.0.1/dbq +- DJANGO_VERSION=1.9 DATABASE_URL=mysql://root@127.0.0.1/dbq +- DJANGO_VERSION=1.10 DATABASE_URL=mysql://root@127.0.0.1/dbq +- DJANGO_VERSION=1.11 DATABASE_URL=mysql://root@127.0.0.1/dbq +- DJANGO_VERSION=2.2 DATABASE_URL=mysql://root@127.0.0.1/dbq install: - pip install -r test-requirements.txt - pip install -U django==$DJANGO_VERSION From 693b629866ce67f00c78103b2185cd57886aa6e5 Mon Sep 17 00:00:00 2001 From: Jordan Brown Date: Thu, 21 Nov 2019 15:51:55 +0000 Subject: [PATCH 026/158] Add psycopg2 to test requirements --- test-requirements.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/test-requirements.txt b/test-requirements.txt index 8ef10c7..909b056 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -3,3 +3,4 @@ pymysql==0.7.11 freezegun==0.3.12 mock==3.0.5 dj-database-url==0.5.0 +psycopg2==2.8.4 From 152b73b91bea3fe83de9b937564be777f76e4f43 Mon Sep 17 00:00:00 2001 From: Jordan Brown Date: Thu, 21 Nov 2019 16:00:39 +0000 Subject: [PATCH 027/158] Replace Job.STATES with Job.State --- django_dbq/tests.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/django_dbq/tests.py b/django_dbq/tests.py index 6b413a7..354e1aa 100644 --- a/django_dbq/tests.py +++ b/django_dbq/tests.py @@ -140,7 +140,7 @@ def test_get_next_ready_job(self): def test_gets_jobs_in_priority_order(self): job_1 = Job.objects.create(name='testjob') - job_2 = Job.objects.create(name='testjob', state=Job.STATES.PROCESSING) + job_2 = Job.objects.create(name='testjob', state=Job.State.PROCESSING) job_3 = Job.objects.create(name='testjob', priority=3) job_4 = Job.objects.create(name='testjob', priority=2) self.assertEqual({ @@ -153,7 +153,7 @@ def test_gets_jobs_in_priority_order(self): def test_gets_jobs_in_negative_priority_order(self): job_1 = Job.objects.create(name='testjob') - job_2 = Job.objects.create(name='testjob', state=Job.STATES.PROCESSING) + job_2 = Job.objects.create(name='testjob', state=Job.State.PROCESSING) job_3 = Job.objects.create(name='testjob', priority=-2) job_4 = Job.objects.create(name='testjob', priority=1) self.assertEqual({ @@ -166,7 +166,7 @@ def test_gets_jobs_in_negative_priority_order(self): def test_gets_jobs_in_priority_and_date_order(self): job_1 = Job.objects.create(name='testjob', priority=3) - job_2 = Job.objects.create(name='testjob', state=Job.STATES.PROCESSING, priority=3) + job_2 = Job.objects.create(name='testjob', state=Job.State.PROCESSING, priority=3) job_3 = Job.objects.create(name='testjob', priority=3) job_4 = Job.objects.create(name='testjob', priority=3) self.assertEqual({ From 75d2e5ba8114220cb4d35f03a282d81c7330440e Mon Sep 17 00:00:00 2001 From: Jordan Brown Date: Thu, 21 Nov 2019 16:07:19 +0000 Subject: [PATCH 028/158] Drop Python 3.4 support and add Python 3.5 support --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index c6a3a39..b49ed7c 100644 --- a/.travis.yml +++ b/.travis.yml @@ -4,7 +4,7 @@ services: - mysql - postgresql python: -- '3.4' +- '3.5' - '3.6' - '3.7' - '3.8' From f6d900dd64f17ff2a56468dc848dc1e971a59f82 Mon Sep 17 00:00:00 2001 From: Jordan Brown Date: Thu, 21 Nov 2019 16:23:44 +0000 Subject: [PATCH 029/158] Use mysqlclient instead of PyMySQL --- test-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test-requirements.txt b/test-requirements.txt index 909b056..3499577 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -1,5 +1,5 @@ -r requirements.txt -pymysql==0.7.11 +mysqlclient==1.4.6 freezegun==0.3.12 mock==3.0.5 dj-database-url==0.5.0 From befa501a7cd3816bf5aba1b565413bbaffaf71b4 Mon Sep 17 00:00:00 2001 From: Jordan Brown Date: Thu, 21 Nov 2019 16:30:19 +0000 Subject: [PATCH 030/158] Remove reference to PyMySQL --- testsettings.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/testsettings.py b/testsettings.py index 97e0b6c..a700428 100644 --- a/testsettings.py +++ b/testsettings.py @@ -1,7 +1,5 @@ import os -import pymysql import dj_database_url -pymysql.install_as_MySQLdb() DATABASES = { 'default': dj_database_url.parse(os.environ['DATABASE_URL']), From 438a4f25ad8b88baa1d7e445d97a638eaf2f4fb5 Mon Sep 17 00:00:00 2001 From: Jordan Brown Date: Thu, 21 Nov 2019 16:33:20 +0000 Subject: [PATCH 031/158] Remove unsupported versions of Django --- .travis.yml | 8 -------- 1 file changed, 8 deletions(-) diff --git a/.travis.yml b/.travis.yml index b49ed7c..38938a7 100644 --- a/.travis.yml +++ b/.travis.yml @@ -4,19 +4,11 @@ services: - mysql - postgresql python: -- '3.5' -- '3.6' - '3.7' - '3.8' env: -- DJANGO_VERSION=1.8 DATABASE_URL=postgres://postgres@127.0.0.1/dbq -- DJANGO_VERSION=1.9 DATABASE_URL=postgres://postgres@127.0.0.1/dbq -- DJANGO_VERSION=1.10 DATABASE_URL=postgres://postgres@127.0.0.1/dbq - DJANGO_VERSION=1.11 DATABASE_URL=postgres://postgres@127.0.0.1/dbq - DJANGO_VERSION=2.2 DATABASE_URL=postgres://postgres@127.0.0.1/dbq -- DJANGO_VERSION=1.8 DATABASE_URL=mysql://root@127.0.0.1/dbq -- DJANGO_VERSION=1.9 DATABASE_URL=mysql://root@127.0.0.1/dbq -- DJANGO_VERSION=1.10 DATABASE_URL=mysql://root@127.0.0.1/dbq - DJANGO_VERSION=1.11 DATABASE_URL=mysql://root@127.0.0.1/dbq - DJANGO_VERSION=2.2 DATABASE_URL=mysql://root@127.0.0.1/dbq install: From 1a82961dc981a3cb22ae39667e95af08b990c78f Mon Sep 17 00:00:00 2001 From: Jordan Brown Date: Thu, 21 Nov 2019 16:39:51 +0000 Subject: [PATCH 032/158] Use the more proper way of doing global env vars --- .travis.yml | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/.travis.yml b/.travis.yml index 38938a7..e5b15ed 100644 --- a/.travis.yml +++ b/.travis.yml @@ -7,10 +7,14 @@ python: - '3.7' - '3.8' env: -- DJANGO_VERSION=1.11 DATABASE_URL=postgres://postgres@127.0.0.1/dbq -- DJANGO_VERSION=2.2 DATABASE_URL=postgres://postgres@127.0.0.1/dbq -- DJANGO_VERSION=1.11 DATABASE_URL=mysql://root@127.0.0.1/dbq -- DJANGO_VERSION=2.2 DATABASE_URL=mysql://root@127.0.0.1/dbq + global: + - DATABASE_URL=postgres://postgres@127.0.0.1/dbq + - DATABASE_URL=mysql://root@127.0.0.1/dbq + jobs: + - DJANGO_VERSION=1.11 + - DJANGO_VERSION=2.2 + - DJANGO_VERSION=1.11 + - DJANGO_VERSION=2.2 install: - pip install -r test-requirements.txt - pip install -U django==$DJANGO_VERSION From d7f3ea41bc3d252d786a339fc34337f01e1cc3eb Mon Sep 17 00:00:00 2001 From: Jordan Brown Date: Thu, 21 Nov 2019 16:48:48 +0000 Subject: [PATCH 033/158] Remove reference to old UUIDfield in django migration --- django_dbq/migrations/0001_initial.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/django_dbq/migrations/0001_initial.py b/django_dbq/migrations/0001_initial.py index 4075219..19c42b4 100644 --- a/django_dbq/migrations/0001_initial.py +++ b/django_dbq/migrations/0001_initial.py @@ -5,10 +5,7 @@ import jsonfield.fields import uuid -try: - from django.db.models import UUIDField -except ImportError: - from django_dbq.fields import UUIDField +from django.db.models import UUIDField class Migration(migrations.Migration): From 5fca58ea30d6a39a2d5383e95b688922b3b58d9e Mon Sep 17 00:00:00 2001 From: Jordan Brown Date: Thu, 21 Nov 2019 16:50:24 +0000 Subject: [PATCH 034/158] Update readme --- README.md | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index d1b15da..5d52848 100644 --- a/README.md +++ b/README.md @@ -8,8 +8,10 @@ Simple databased-backed job queue. Jobs are defined in your settings, and are pr Asynchronous tasks are run via a *job queue*. This system is designed to support multi-step job workflows. Supported and tested against: -- Django 1.8, 1.9, 1.10, 1.11 and 2.2 -- Python 3.4, 3.6, 3.7 and 3.8 +- Django 1.11 and 2.2 +- Python 3.7 and 3.8 + +This package may still work with older versions of Django and Python but they aren't explicitly supported. ## Getting Started From 1280cf236b449acff47db033b2c32f3106e949af Mon Sep 17 00:00:00 2001 From: Jordan Brown Date: Thu, 21 Nov 2019 16:53:03 +0000 Subject: [PATCH 035/158] Undo using global env vars --- .travis.yml | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/.travis.yml b/.travis.yml index e5b15ed..38938a7 100644 --- a/.travis.yml +++ b/.travis.yml @@ -7,14 +7,10 @@ python: - '3.7' - '3.8' env: - global: - - DATABASE_URL=postgres://postgres@127.0.0.1/dbq - - DATABASE_URL=mysql://root@127.0.0.1/dbq - jobs: - - DJANGO_VERSION=1.11 - - DJANGO_VERSION=2.2 - - DJANGO_VERSION=1.11 - - DJANGO_VERSION=2.2 +- DJANGO_VERSION=1.11 DATABASE_URL=postgres://postgres@127.0.0.1/dbq +- DJANGO_VERSION=2.2 DATABASE_URL=postgres://postgres@127.0.0.1/dbq +- DJANGO_VERSION=1.11 DATABASE_URL=mysql://root@127.0.0.1/dbq +- DJANGO_VERSION=2.2 DATABASE_URL=mysql://root@127.0.0.1/dbq install: - pip install -r test-requirements.txt - pip install -U django==$DJANGO_VERSION From b478dc65f22d78df2663024cc6a15dd3b99909c4 Mon Sep 17 00:00:00 2001 From: Jordan Brown Date: Thu, 21 Nov 2019 17:12:41 +0000 Subject: [PATCH 036/158] Add Python 3.5 and Python 3.6 support again --- .travis.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.travis.yml b/.travis.yml index 38938a7..5ae53b1 100644 --- a/.travis.yml +++ b/.travis.yml @@ -4,6 +4,8 @@ services: - mysql - postgresql python: +- '3.5' +- '3.6' - '3.7' - '3.8' env: From 67d296a86ed8518f84a416452b13ef087bd13253 Mon Sep 17 00:00:00 2001 From: Jordan Brown Date: Thu, 21 Nov 2019 17:14:11 +0000 Subject: [PATCH 037/158] Make STATES backwards compatible --- django_dbq/management/commands/worker.py | 8 +++--- django_dbq/models.py | 20 ++++++------- django_dbq/tests.py | 36 ++++++++++++------------ 3 files changed, 32 insertions(+), 32 deletions(-) diff --git a/django_dbq/management/commands/worker.py b/django_dbq/management/commands/worker.py index 2cf97af..cf6b19d 100644 --- a/django_dbq/management/commands/worker.py +++ b/django_dbq/management/commands/worker.py @@ -23,7 +23,7 @@ def process_job(queue_name): return logger.info('Processing job: name="%s" queue="%s" id=%s state=%s next_task=%s', job.name, queue_name, job.pk, job.state, job.next_task) - job.state = Job.State.PROCESSING + job.state = Job.STATES.PROCESSING job.save() try: @@ -31,12 +31,12 @@ def process_job(queue_name): task_function(job) job.update_next_task() if not job.next_task: - job.state = Job.State.COMPLETE + job.state = Job.STATES.COMPLETE else: - job.state = Job.State.READY + job.state = Job.STATES.READY except Exception as exception: logger.exception("Job id=%s failed", job.pk) - job.state = Job.State.FAILED + job.state = Job.STATES.FAILED failure_hook_name = job.get_failure_hook_name() if failure_hook_name: diff --git a/django_dbq/models.py b/django_dbq/models.py index f91fe8f..881f718 100644 --- a/django_dbq/models.py +++ b/django_dbq/models.py @@ -45,37 +45,37 @@ def delete_old(self): """ Delete all jobs older than DELETE_JOBS_AFTER_HOURS """ - delete_jobs_in_states = [Job.State.FAILED, Job.State.COMPLETE] + delete_jobs_in_states = [Job.STATES.FAILED, Job.STATES.COMPLETE] delete_jobs_created_before = datetime.datetime.utcnow() - datetime.timedelta(hours=DELETE_JOBS_AFTER_HOURS) logger.info("Deleting all job in states %s created before %s", ", ".join(delete_jobs_in_states), delete_jobs_created_before.isoformat()) Job.objects.filter(state__in=delete_jobs_in_states, created__lte=delete_jobs_created_before).delete() def to_process(self, queue_name): - return self.select_for_update().filter(queue_name=queue_name, state__in=(Job.State.READY, Job.State.NEW)) + return self.select_for_update().filter(queue_name=queue_name, state__in=(Job.STATES.READY, Job.STATES.NEW)) class Job(models.Model): - class State: + class STATES: NEW = 'NEW' READY = 'READY' PROCESSING = 'PROCESSING' FAILED = 'FAILED' COMPLETE = 'COMPLETE' - STATES = [ - (State.NEW, "NEW"), - (State.READY, "READY"), - (State.PROCESSING, "PROCESSING"), - (State.FAILED, "FAILED"), - (State.COMPLETE, "COMPLETE"), + STATE_CHOICES = [ + (STATES.NEW, "NEW"), + (STATES.READY, "READY"), + (STATES.PROCESSING, "PROCESSING"), + (STATES.FAILED, "FAILED"), + (STATES.COMPLETE, "COMPLETE"), ] id = UUIDField(primary_key=True, default=uuid.uuid4, editable=False) created = models.DateTimeField(auto_now_add=True, db_index=True) modified = models.DateTimeField(auto_now=True) name = models.CharField(max_length=100) - state = models.CharField(max_length=20, choices=STATES, default=State.NEW, db_index=True) + state = models.CharField(max_length=20, choices=STATE_CHOICES, default=STATES.NEW, db_index=True) next_task = models.CharField(max_length=100, blank=True) workspace = JSONField(null=True) queue_name = models.CharField(max_length=20, default='default', db_index=True) diff --git a/django_dbq/tests.py b/django_dbq/tests.py index 354e1aa..ee39c53 100644 --- a/django_dbq/tests.py +++ b/django_dbq/tests.py @@ -120,19 +120,19 @@ class JobTestCase(TestCase): def test_create_job(self): job = Job(name='testjob') - self.assertEqual(job.state, Job.State.NEW) + self.assertEqual(job.state, Job.STATES.NEW) def test_create_job_with_queue(self): job = Job(name='testjob', queue_name='lol') - self.assertEqual(job.state, Job.State.NEW) + self.assertEqual(job.state, Job.STATES.NEW) self.assertEqual(job.queue_name, 'lol') def test_get_next_ready_job(self): self.assertTrue(Job.objects.get_ready_or_none('default') is None) - Job.objects.create(name='testjob', state=Job.State.READY) - Job.objects.create(name='testjob', state=Job.State.PROCESSING) - expected = Job.objects.create(name='testjob', state=Job.State.READY) + Job.objects.create(name='testjob', state=Job.STATES.READY) + Job.objects.create(name='testjob', state=Job.STATES.PROCESSING) + expected = Job.objects.create(name='testjob', state=Job.STATES.READY) expected.created = datetime.now() - timedelta(minutes=1) expected.save() @@ -140,7 +140,7 @@ def test_get_next_ready_job(self): def test_gets_jobs_in_priority_order(self): job_1 = Job.objects.create(name='testjob') - job_2 = Job.objects.create(name='testjob', state=Job.State.PROCESSING) + job_2 = Job.objects.create(name='testjob', state=Job.STATES.PROCESSING) job_3 = Job.objects.create(name='testjob', priority=3) job_4 = Job.objects.create(name='testjob', priority=2) self.assertEqual({ @@ -153,7 +153,7 @@ def test_gets_jobs_in_priority_order(self): def test_gets_jobs_in_negative_priority_order(self): job_1 = Job.objects.create(name='testjob') - job_2 = Job.objects.create(name='testjob', state=Job.State.PROCESSING) + job_2 = Job.objects.create(name='testjob', state=Job.STATES.PROCESSING) job_3 = Job.objects.create(name='testjob', priority=-2) job_4 = Job.objects.create(name='testjob', priority=1) self.assertEqual({ @@ -166,7 +166,7 @@ def test_gets_jobs_in_negative_priority_order(self): def test_gets_jobs_in_priority_and_date_order(self): job_1 = Job.objects.create(name='testjob', priority=3) - job_2 = Job.objects.create(name='testjob', state=Job.State.PROCESSING, priority=3) + job_2 = Job.objects.create(name='testjob', state=Job.STATES.PROCESSING, priority=3) job_3 = Job.objects.create(name='testjob', priority=3) job_4 = Job.objects.create(name='testjob', priority=3) self.assertEqual({ @@ -187,9 +187,9 @@ def test_get_next_ready_job_created(self): """ self.assertTrue(Job.objects.get_ready_or_none('default') is None) - Job.objects.create(name='testjob', state=Job.State.NEW) - Job.objects.create(name='testjob', state=Job.State.PROCESSING) - expected = Job.objects.create(name='testjob', state=Job.State.NEW) + Job.objects.create(name='testjob', state=Job.STATES.NEW) + Job.objects.create(name='testjob', state=Job.STATES.PROCESSING) + expected = Job.objects.create(name='testjob', state=Job.STATES.NEW) expected.created = datetime.now() - timedelta(minutes=1) expected.save() @@ -217,7 +217,7 @@ def test_process_job(self): job = Job.objects.create(name='testjob') process_job('default') job = Job.objects.get() - self.assertEqual(job.state, Job.State.COMPLETE) + self.assertEqual(job.state, Job.STATES.COMPLETE) def test_process_job_wrong_queue(self): """ @@ -226,7 +226,7 @@ def test_process_job_wrong_queue(self): job = Job.objects.create(name='testjob', queue_name='lol') process_job('default') job = Job.objects.get() - self.assertEqual(job.state, Job.State.NEW) + self.assertEqual(job.state, Job.STATES.NEW) @override_settings(JOBS={'testjob': {'tasks': ['django_dbq.tests.test_task'], 'creation_hook': 'django_dbq.tests.creation_hook'}}) @@ -253,7 +253,7 @@ def test_failure_hook(self): job = Job.objects.create(name='testjob') process_job('default') job = Job.objects.get() - self.assertEqual(job.state, Job.State.FAILED) + self.assertEqual(job.state, Job.STATES.FAILED) self.assertEqual(job.workspace['output'], 'failure hook ran') @@ -263,19 +263,19 @@ class DeleteOldJobsTestCase(TestCase): def test_delete_old_jobs(self): two_days_ago = datetime.utcnow() - timedelta(days=2) - j1 = Job.objects.create(name='testjob', state=Job.State.COMPLETE) + j1 = Job.objects.create(name='testjob', state=Job.STATES.COMPLETE) j1.created = two_days_ago j1.save() - j2 = Job.objects.create(name='testjob', state=Job.State.FAILED) + j2 = Job.objects.create(name='testjob', state=Job.STATES.FAILED) j2.created = two_days_ago j2.save() - j3 = Job.objects.create(name='testjob', state=Job.State.NEW) + j3 = Job.objects.create(name='testjob', state=Job.STATES.NEW) j3.created = two_days_ago j3.save() - j4 = Job.objects.create(name='testjob', state=Job.State.COMPLETE) + j4 = Job.objects.create(name='testjob', state=Job.STATES.COMPLETE) Job.objects.delete_old() From 458c9ef955a2a2e31a04ad9d946695c7b471aa0e Mon Sep 17 00:00:00 2001 From: Jordan Brown Date: Thu, 21 Nov 2019 17:30:45 +0000 Subject: [PATCH 038/158] Add and run black --- .travis.yml | 2 +- django_dbq/__init__.py | 2 +- django_dbq/management/commands/create_job.py | 35 +-- .../management/commands/delete_old_jobs.py | 2 +- django_dbq/management/commands/worker.py | 64 +++-- django_dbq/migrations/0001_initial.py | 50 ++-- .../migrations/0002_auto_20151016_1027.py | 7 +- .../migrations/0003_auto_20180713_1000.py | 9 +- django_dbq/models.py | 55 +++-- django_dbq/serializers.py | 8 +- django_dbq/tasks.py | 6 +- django_dbq/tests.py | 227 +++++++++--------- setup.py | 47 ++-- test-requirements.txt | 1 + testsettings.py | 43 ++-- 15 files changed, 316 insertions(+), 242 deletions(-) diff --git a/.travis.yml b/.travis.yml index 5ae53b1..0a098a2 100644 --- a/.travis.yml +++ b/.travis.yml @@ -16,7 +16,7 @@ env: install: - pip install -r test-requirements.txt - pip install -U django==$DJANGO_VERSION -script: python manage.py test +script: black . --check && python manage.py test deploy: provider: pypi user: dabapps diff --git a/django_dbq/__init__.py b/django_dbq/__init__.py index 58d478a..c68196d 100644 --- a/django_dbq/__init__.py +++ b/django_dbq/__init__.py @@ -1 +1 @@ -__version__ = '1.2.0' +__version__ = "1.2.0" diff --git a/django_dbq/management/commands/create_job.py b/django_dbq/management/commands/create_job.py index 7cb3d3b..c060218 100644 --- a/django_dbq/management/commands/create_job.py +++ b/django_dbq/management/commands/create_job.py @@ -11,23 +11,23 @@ class Command(BaseCommand): help = "Create a job" - args = '' + args = "" def add_arguments(self, parser): - parser.add_argument('args', nargs='+') + parser.add_argument("args", nargs="+") parser.add_argument( - '--workspace', - action='store_true', - dest='workspace', + "--workspace", + action="store_true", + dest="workspace", default=None, - help="JSON-formatted initial commandworkspace." + help="JSON-formatted initial commandworkspace.", ) parser.add_argument( - '--queue_name', - action='store_true', - dest='queue_name', + "--queue_name", + action="store_true", + dest="queue_name", default=None, - help="A specific queue to add this job to" + help="A specific queue to add this job to", ) def handle(self, *args, **options): @@ -38,19 +38,22 @@ def handle(self, *args, **options): if name not in settings.JOBS: raise CommandError('"%s" is not a valid job name' % name) - workspace = options['workspace'] + workspace = options["workspace"] if workspace: workspace = json.loads(workspace) - queue_name = options['queue_name'] + queue_name = options["queue_name"] kwargs = { - 'name': name, - 'workspace': workspace, + "name": name, + "workspace": workspace, } if queue_name: - kwargs['queue_name'] = queue_name + kwargs["queue_name"] = queue_name job = Job.objects.create(**kwargs) - self.stdout.write('Created job: "%s", id=%s for queue "%s"' % (job.name, job.pk, queue_name if queue_name else 'default')) + self.stdout.write( + 'Created job: "%s", id=%s for queue "%s"' + % (job.name, job.pk, queue_name if queue_name else "default") + ) diff --git a/django_dbq/management/commands/delete_old_jobs.py b/django_dbq/management/commands/delete_old_jobs.py index 4c512d9..15d4cc8 100644 --- a/django_dbq/management/commands/delete_old_jobs.py +++ b/django_dbq/management/commands/delete_old_jobs.py @@ -8,4 +8,4 @@ class Command(BaseCommand): def handle(self, *args, **options): Job.objects.delete_old() - self.stdout.write('Deleted old jobs') + self.stdout.write("Deleted old jobs") diff --git a/django_dbq/management/commands/worker.py b/django_dbq/management/commands/worker.py index cf6b19d..b0c6724 100644 --- a/django_dbq/management/commands/worker.py +++ b/django_dbq/management/commands/worker.py @@ -11,7 +11,7 @@ logger = logging.getLogger(__name__) -DEFAULT_QUEUE_NAME = 'default' +DEFAULT_QUEUE_NAME = "default" def process_job(queue_name): @@ -22,7 +22,14 @@ def process_job(queue_name): if not job: return - logger.info('Processing job: name="%s" queue="%s" id=%s state=%s next_task=%s', job.name, queue_name, job.pk, job.state, job.next_task) + logger.info( + 'Processing job: name="%s" queue="%s" id=%s state=%s next_task=%s', + job.name, + queue_name, + job.pk, + job.state, + job.next_task, + ) job.state = Job.STATES.PROCESSING job.save() @@ -40,18 +47,30 @@ def process_job(queue_name): failure_hook_name = job.get_failure_hook_name() if failure_hook_name: - logger.info("Running failure hook %s for job id=%s", failure_hook_name, job.pk) + logger.info( + "Running failure hook %s for job id=%s", failure_hook_name, job.pk + ) failure_hook_function = import_string(failure_hook_name) failure_hook_function(job, exception) else: logger.info("No failure hook for job id=%s", job.pk) - logger.info('Updating job: name="%s" id=%s state=%s next_task=%s', job.name, job.pk, job.state, job.next_task or 'none') + logger.info( + 'Updating job: name="%s" id=%s state=%s next_task=%s', + job.name, + job.pk, + job.state, + job.next_task or "none", + ) try: job.save() except: - logger.error('Failed to save job: id=%s org=%s', job.pk, job.workspace.get('organisation_id')) + logger.error( + "Failed to save job: id=%s org=%s", + job.pk, + job.workspace.get("organisation_id"), + ) raise @@ -67,7 +86,11 @@ def __init__(self, name, rate_limit_in_seconds): def do_work(self): sleep(1) - if self.last_job_finished and (timezone.now() - self.last_job_finished).total_seconds() < self.rate_limit_in_seconds: + if ( + self.last_job_finished + and (timezone.now() - self.last_job_finished).total_seconds() + < self.rate_limit_in_seconds + ): return process_job(self.queue_name) @@ -79,14 +102,20 @@ class Command(BaseCommand): help = "Run a queue worker process" def add_arguments(self, parser): - parser.add_argument('queue_name', nargs='?', default='default', type=str) - parser.add_argument('rate_limit', help='The rate limit in seconds. The default rate limit is 1 job per second.', nargs='?', default=1, type=int) + parser.add_argument("queue_name", nargs="?", default="default", type=str) parser.add_argument( - '--dry-run', - action='store_true', - dest='dry_run', + "rate_limit", + help="The rate limit in seconds. The default rate limit is 1 job per second.", + nargs="?", + default=1, + type=int, + ) + parser.add_argument( + "--dry-run", + action="store_true", + dest="dry_run", default=False, - help="Don't actually start the worker. Used for testing." + help="Don't actually start the worker. Used for testing.", ) def handle(self, *args, **options): @@ -96,14 +125,17 @@ def handle(self, *args, **options): if len(args) != 1: raise CommandError("Please supply a single queue job name") - queue_name = options['queue_name'] - rate_limit_in_seconds = options['rate_limit'] + queue_name = options["queue_name"] + rate_limit_in_seconds = options["rate_limit"] - self.stdout.write("Starting job worker for queue \"%s\" with rate limit %s/s" % (queue_name, rate_limit_in_seconds)) + self.stdout.write( + 'Starting job worker for queue "%s" with rate limit %s/s' + % (queue_name, rate_limit_in_seconds) + ) worker = Worker(queue_name, rate_limit_in_seconds) - if options['dry_run']: + if options["dry_run"]: return worker.run() diff --git a/django_dbq/migrations/0001_initial.py b/django_dbq/migrations/0001_initial.py index 19c42b4..0ec8c73 100644 --- a/django_dbq/migrations/0001_initial.py +++ b/django_dbq/migrations/0001_initial.py @@ -10,24 +10,46 @@ class Migration(migrations.Migration): - dependencies = [ - ] + dependencies = [] operations = [ migrations.CreateModel( - name='Job', + name="Job", fields=[ - ('id', UUIDField(serialize=False, editable=False, default=uuid.uuid4, primary_key=True)), - ('created', models.DateTimeField(db_index=True, auto_now_add=True)), - ('modified', models.DateTimeField(auto_now=True)), - ('name', models.CharField(max_length=100)), - ('state', models.CharField(db_index=True, max_length=20, default='NEW', choices=[('NEW', 'NEW'), ('READY', 'READY'), ('PROCESSING', 'PROCESSING'), ('FAILED', 'FAILED'), ('COMPLETE', 'COMPLETE')])), - ('next_task', models.CharField(max_length=100, blank=True)), - ('workspace', jsonfield.fields.JSONField(null=True)), - ('queue_name', models.CharField(db_index=True, max_length=20, default='default')), + ( + "id", + UUIDField( + serialize=False, + editable=False, + default=uuid.uuid4, + primary_key=True, + ), + ), + ("created", models.DateTimeField(db_index=True, auto_now_add=True)), + ("modified", models.DateTimeField(auto_now=True)), + ("name", models.CharField(max_length=100)), + ( + "state", + models.CharField( + db_index=True, + max_length=20, + default="NEW", + choices=[ + ("NEW", "NEW"), + ("READY", "READY"), + ("PROCESSING", "PROCESSING"), + ("FAILED", "FAILED"), + ("COMPLETE", "COMPLETE"), + ], + ), + ), + ("next_task", models.CharField(max_length=100, blank=True)), + ("workspace", jsonfield.fields.JSONField(null=True)), + ( + "queue_name", + models.CharField(db_index=True, max_length=20, default="default"), + ), ], - options={ - 'ordering': ['-created'], - }, + options={"ordering": ["-created"],}, ), ] diff --git a/django_dbq/migrations/0002_auto_20151016_1027.py b/django_dbq/migrations/0002_auto_20151016_1027.py index a9cf0b4..d0a72c4 100644 --- a/django_dbq/migrations/0002_auto_20151016_1027.py +++ b/django_dbq/migrations/0002_auto_20151016_1027.py @@ -7,12 +7,9 @@ class Migration(migrations.Migration): dependencies = [ - ('django_dbq', '0001_initial'), + ("django_dbq", "0001_initial"), ] operations = [ - migrations.AlterModelOptions( - name='job', - options={'ordering': ['created']}, - ), + migrations.AlterModelOptions(name="job", options={"ordering": ["created"]},), ] diff --git a/django_dbq/migrations/0003_auto_20180713_1000.py b/django_dbq/migrations/0003_auto_20180713_1000.py index 4763455..4d959f3 100644 --- a/django_dbq/migrations/0003_auto_20180713_1000.py +++ b/django_dbq/migrations/0003_auto_20180713_1000.py @@ -8,17 +8,16 @@ class Migration(migrations.Migration): dependencies = [ - ('django_dbq', '0002_auto_20151016_1027'), + ("django_dbq", "0002_auto_20151016_1027"), ] operations = [ migrations.AlterModelOptions( - name='job', - options={'ordering': ['-priority', 'created']}, + name="job", options={"ordering": ["-priority", "created"]}, ), migrations.AddField( - model_name='job', - name='priority', + model_name="job", + name="priority", field=models.SmallIntegerField(db_index=True, default=0), ), ] diff --git a/django_dbq/models.py b/django_dbq/models.py index 881f718..e483b57 100644 --- a/django_dbq/models.py +++ b/django_dbq/models.py @@ -1,6 +1,10 @@ from django.db import models from django.utils.module_loading import import_string -from django_dbq.tasks import get_next_task_name, get_failure_hook_name, get_creation_hook_name +from django_dbq.tasks import ( + get_next_task_name, + get_failure_hook_name, + get_creation_hook_name, +) from jsonfield import JSONField from django.db.models import UUIDField import datetime @@ -39,29 +43,42 @@ def get_ready_or_none(self, queue_name, max_retries=3): if retries_left == 0: raise retries_left -= 1 - logger.warn("Caught %s when looking for a READY job, retrying %s more times", str(e), retries_left) + logger.warn( + "Caught %s when looking for a READY job, retrying %s more times", + str(e), + retries_left, + ) def delete_old(self): """ Delete all jobs older than DELETE_JOBS_AFTER_HOURS """ delete_jobs_in_states = [Job.STATES.FAILED, Job.STATES.COMPLETE] - delete_jobs_created_before = datetime.datetime.utcnow() - datetime.timedelta(hours=DELETE_JOBS_AFTER_HOURS) - logger.info("Deleting all job in states %s created before %s", ", ".join(delete_jobs_in_states), delete_jobs_created_before.isoformat()) - Job.objects.filter(state__in=delete_jobs_in_states, created__lte=delete_jobs_created_before).delete() + delete_jobs_created_before = datetime.datetime.utcnow() - datetime.timedelta( + hours=DELETE_JOBS_AFTER_HOURS + ) + logger.info( + "Deleting all job in states %s created before %s", + ", ".join(delete_jobs_in_states), + delete_jobs_created_before.isoformat(), + ) + Job.objects.filter( + state__in=delete_jobs_in_states, created__lte=delete_jobs_created_before + ).delete() def to_process(self, queue_name): - return self.select_for_update().filter(queue_name=queue_name, state__in=(Job.STATES.READY, Job.STATES.NEW)) + return self.select_for_update().filter( + queue_name=queue_name, state__in=(Job.STATES.READY, Job.STATES.NEW) + ) class Job(models.Model): - class STATES: - NEW = 'NEW' - READY = 'READY' - PROCESSING = 'PROCESSING' - FAILED = 'FAILED' - COMPLETE = 'COMPLETE' + NEW = "NEW" + READY = "READY" + PROCESSING = "PROCESSING" + FAILED = "FAILED" + COMPLETE = "COMPLETE" STATE_CHOICES = [ (STATES.NEW, "NEW"), @@ -75,14 +92,16 @@ class STATES: created = models.DateTimeField(auto_now_add=True, db_index=True) modified = models.DateTimeField(auto_now=True) name = models.CharField(max_length=100) - state = models.CharField(max_length=20, choices=STATE_CHOICES, default=STATES.NEW, db_index=True) + state = models.CharField( + max_length=20, choices=STATE_CHOICES, default=STATES.NEW, db_index=True + ) next_task = models.CharField(max_length=100, blank=True) workspace = JSONField(null=True) - queue_name = models.CharField(max_length=20, default='default', db_index=True) + queue_name = models.CharField(max_length=20, default="default", db_index=True) priority = models.SmallIntegerField(default=0, db_index=True) class Meta: - ordering = ['-priority', 'created'] + ordering = ["-priority", "created"] objects = JobManager() @@ -96,13 +115,15 @@ def save(self, *args, **kwargs): try: self.run_creation_hook() except Exception as exception: # noqa - logger.exception("Failed to create new job, creation hook raised an exception") + logger.exception( + "Failed to create new job, creation hook raised an exception" + ) return # cancel the save return super(Job, self).save(*args, **kwargs) def update_next_task(self): - self.next_task = get_next_task_name(self.name, self.next_task) or '' + self.next_task = get_next_task_name(self.name, self.next_task) or "" def get_failure_hook_name(self): return get_failure_hook_name(self.name) diff --git a/django_dbq/serializers.py b/django_dbq/serializers.py index 1a8a7e7..12b40d5 100644 --- a/django_dbq/serializers.py +++ b/django_dbq/serializers.py @@ -10,17 +10,17 @@ class JobSerializer(serializers.Serializer): modified = serializers.DateTimeField(read_only=True) state = serializers.CharField(read_only=True) workspace = serializers.WritableField(required=False) - url = serializers.HyperlinkedIdentityField(view_name='job_detail') + url = serializers.HyperlinkedIdentityField(view_name="job_detail") def __init__(self, *args, **kwargs): super(JobSerializer, self).__init__(*args, **kwargs) - self.fields['name'].choices = ((key, key) for key in settings.JOBS) + self.fields["name"].choices = ((key, key) for key in settings.JOBS) def validate_workspace(self, attrs, source): - workspace = attrs.get('workspace') + workspace = attrs.get("workspace") if workspace and isinstance(workspace, basestring): try: - attrs['workspace'] = json.loads(workspace) + attrs["workspace"] = json.loads(workspace) except ValueError: raise serializers.ValidationError("Invalid JSON") return attrs diff --git a/django_dbq/tasks.py b/django_dbq/tasks.py index b7a7ad2..3e43da3 100644 --- a/django_dbq/tasks.py +++ b/django_dbq/tasks.py @@ -1,9 +1,9 @@ from django.conf import settings -TASK_LIST_KEY = 'tasks' -FAILURE_HOOK_KEY = 'failure_hook' -CREATION_HOOK_KEY = 'creation_hook' +TASK_LIST_KEY = "tasks" +FAILURE_HOOK_KEY = "failure_hook" +CREATION_HOOK_KEY = "creation_hook" def get_next_task_name(job_name, current_task=None): diff --git a/django_dbq/tests.py b/django_dbq/tests.py index ee39c53..c566461 100644 --- a/django_dbq/tests.py +++ b/django_dbq/tests.py @@ -9,6 +9,7 @@ from django_dbq.management.commands.worker import process_job, Worker from django_dbq.models import Job + try: from StringIO import StringIO except ImportError: @@ -20,8 +21,8 @@ def test_task(job=None): def workspace_test_task(job): - input = job.workspace['input'] - job.workspace['output'] = input + '-output' + input = job.workspace["input"] + job.workspace["output"] = input + "-output" def failing_task(job): @@ -29,68 +30,65 @@ def failing_task(job): def failure_hook(job, exception): - job.workspace['output'] = 'failure hook ran' + job.workspace["output"] = "failure hook ran" def creation_hook(job): - job.workspace['output'] = 'creation hook ran' + job.workspace["output"] = "creation hook ran" -@override_settings(JOBS={'testjob': {'tasks': ['a']}}) +@override_settings(JOBS={"testjob": {"tasks": ["a"]}}) class JobManagementCommandTestCase(TestCase): - def test_create_job(self): - call_command('create_job', 'testjob', stdout=StringIO()) + call_command("create_job", "testjob", stdout=StringIO()) job = Job.objects.get() - self.assertEqual(job.name, 'testjob') - self.assertEqual(job.queue_name, 'default') + self.assertEqual(job.name, "testjob") + self.assertEqual(job.queue_name, "default") def test_create_job_with_workspace(self): workspace = '{"test": "test"}' - call_command('create_job', 'testjob', workspace=workspace, stdout=StringIO()) + call_command("create_job", "testjob", workspace=workspace, stdout=StringIO()) job = Job.objects.get() - self.assertEqual(job.workspace, {'test': 'test'}) + self.assertEqual(job.workspace, {"test": "test"}) def test_create_job_with_queue_name(self): - call_command('create_job', 'testjob', queue_name='lol', stdout=StringIO()) + call_command("create_job", "testjob", queue_name="lol", stdout=StringIO()) job = Job.objects.get() - self.assertEqual(job.name, 'testjob') - self.assertEqual(job.queue_name, 'lol') + self.assertEqual(job.name, "testjob") + self.assertEqual(job.queue_name, "lol") def test_errors_raised_correctly(self): with self.assertRaises(CommandError): - call_command('create_job', stdout=StringIO()) + call_command("create_job", stdout=StringIO()) with self.assertRaises(CommandError): - call_command('create_job', 'some_other_job', stdout=StringIO()) + call_command("create_job", "some_other_job", stdout=StringIO()) -@override_settings(JOBS={'testjob': {'tasks': ['a']}}) +@override_settings(JOBS={"testjob": {"tasks": ["a"]}}) class WorkerManagementCommandTestCase(TestCase): - def test_worker_no_args(self): stdout = StringIO() - call_command('worker', dry_run=True, stdout=stdout) + call_command("worker", dry_run=True, stdout=stdout) output = stdout.getvalue() - self.assertTrue('Starting job worker' in output) - self.assertTrue('default' in output) + self.assertTrue("Starting job worker" in output) + self.assertTrue("default" in output) def test_worker_with_queue_name(self): stdout = StringIO() - call_command('worker', queue_name='test_queue', dry_run=True, stdout=stdout) + call_command("worker", queue_name="test_queue", dry_run=True, stdout=stdout) output = stdout.getvalue() - self.assertTrue('test_queue' in output) + self.assertTrue("test_queue" in output) @freezegun.freeze_time() -@mock.patch('django_dbq.management.commands.worker.sleep') -@mock.patch('django_dbq.management.commands.worker.process_job') +@mock.patch("django_dbq.management.commands.worker.sleep") +@mock.patch("django_dbq.management.commands.worker.process_job") class WorkerProcessDoWorkTestCase(TestCase): - def setUp(self): super().setUp() self.MockWorker = mock.MagicMock() - self.MockWorker.queue_name = 'default' + self.MockWorker.queue_name = "default" self.MockWorker.rate_limit_in_seconds = 5 self.MockWorker.last_job_finished = None @@ -101,81 +99,83 @@ def test_do_work_no_previous_job_run(self, mock_process_job, mock_sleep): self.assertEqual(self.MockWorker.last_job_finished, timezone.now()) def test_do_work_previous_job_too_soon(self, mock_process_job, mock_sleep): - self.MockWorker.last_job_finished = timezone.now() - timezone.timedelta(seconds=2) + self.MockWorker.last_job_finished = timezone.now() - timezone.timedelta( + seconds=2 + ) Worker.do_work(self.MockWorker) self.assertEqual(mock_sleep.call_count, 1) self.assertEqual(mock_process_job.call_count, 0) - self.assertEqual(self.MockWorker.last_job_finished, timezone.now() - timezone.timedelta(seconds=2)) + self.assertEqual( + self.MockWorker.last_job_finished, + timezone.now() - timezone.timedelta(seconds=2), + ) def test_do_work_previous_job_long_time_ago(self, mock_process_job, mock_sleep): - self.MockWorker.last_job_finished = timezone.now() - timezone.timedelta(seconds=7) + self.MockWorker.last_job_finished = timezone.now() - timezone.timedelta( + seconds=7 + ) Worker.do_work(self.MockWorker) self.assertEqual(mock_sleep.call_count, 1) self.assertEqual(mock_process_job.call_count, 1) self.assertEqual(self.MockWorker.last_job_finished, timezone.now()) -@override_settings(JOBS={'testjob': {'tasks': ['a']}}) +@override_settings(JOBS={"testjob": {"tasks": ["a"]}}) class JobTestCase(TestCase): - def test_create_job(self): - job = Job(name='testjob') + job = Job(name="testjob") self.assertEqual(job.state, Job.STATES.NEW) def test_create_job_with_queue(self): - job = Job(name='testjob', queue_name='lol') + job = Job(name="testjob", queue_name="lol") self.assertEqual(job.state, Job.STATES.NEW) - self.assertEqual(job.queue_name, 'lol') + self.assertEqual(job.queue_name, "lol") def test_get_next_ready_job(self): - self.assertTrue(Job.objects.get_ready_or_none('default') is None) + self.assertTrue(Job.objects.get_ready_or_none("default") is None) - Job.objects.create(name='testjob', state=Job.STATES.READY) - Job.objects.create(name='testjob', state=Job.STATES.PROCESSING) - expected = Job.objects.create(name='testjob', state=Job.STATES.READY) + Job.objects.create(name="testjob", state=Job.STATES.READY) + Job.objects.create(name="testjob", state=Job.STATES.PROCESSING) + expected = Job.objects.create(name="testjob", state=Job.STATES.READY) expected.created = datetime.now() - timedelta(minutes=1) expected.save() - self.assertEqual(Job.objects.get_ready_or_none('default'), expected) + self.assertEqual(Job.objects.get_ready_or_none("default"), expected) def test_gets_jobs_in_priority_order(self): - job_1 = Job.objects.create(name='testjob') - job_2 = Job.objects.create(name='testjob', state=Job.STATES.PROCESSING) - job_3 = Job.objects.create(name='testjob', priority=3) - job_4 = Job.objects.create(name='testjob', priority=2) - self.assertEqual({ - job for job in Job.objects.to_process('default') - }, { - job_3, job_4, job_1 - }) - self.assertEqual(Job.objects.get_ready_or_none('default'), job_3) - self.assertFalse(Job.objects.to_process('default').filter(id=job_2.id).exists()) + job_1 = Job.objects.create(name="testjob") + job_2 = Job.objects.create(name="testjob", state=Job.STATES.PROCESSING) + job_3 = Job.objects.create(name="testjob", priority=3) + job_4 = Job.objects.create(name="testjob", priority=2) + self.assertEqual( + {job for job in Job.objects.to_process("default")}, {job_3, job_4, job_1} + ) + self.assertEqual(Job.objects.get_ready_or_none("default"), job_3) + self.assertFalse(Job.objects.to_process("default").filter(id=job_2.id).exists()) def test_gets_jobs_in_negative_priority_order(self): - job_1 = Job.objects.create(name='testjob') - job_2 = Job.objects.create(name='testjob', state=Job.STATES.PROCESSING) - job_3 = Job.objects.create(name='testjob', priority=-2) - job_4 = Job.objects.create(name='testjob', priority=1) - self.assertEqual({ - job for job in Job.objects.to_process('default') - }, { - job_4, job_3, job_1 - }) - self.assertEqual(Job.objects.get_ready_or_none('default'), job_4) - self.assertFalse(Job.objects.to_process('default').filter(id=job_2.id).exists()) + job_1 = Job.objects.create(name="testjob") + job_2 = Job.objects.create(name="testjob", state=Job.STATES.PROCESSING) + job_3 = Job.objects.create(name="testjob", priority=-2) + job_4 = Job.objects.create(name="testjob", priority=1) + self.assertEqual( + {job for job in Job.objects.to_process("default")}, {job_4, job_3, job_1} + ) + self.assertEqual(Job.objects.get_ready_or_none("default"), job_4) + self.assertFalse(Job.objects.to_process("default").filter(id=job_2.id).exists()) def test_gets_jobs_in_priority_and_date_order(self): - job_1 = Job.objects.create(name='testjob', priority=3) - job_2 = Job.objects.create(name='testjob', state=Job.STATES.PROCESSING, priority=3) - job_3 = Job.objects.create(name='testjob', priority=3) - job_4 = Job.objects.create(name='testjob', priority=3) - self.assertEqual({ - job for job in Job.objects.to_process('default') - }, { - job_1, job_3, job_4 - }) - self.assertEqual(Job.objects.get_ready_or_none('default'), job_1) - self.assertFalse(Job.objects.to_process('default').filter(id=job_2.id).exists()) + job_1 = Job.objects.create(name="testjob", priority=3) + job_2 = Job.objects.create( + name="testjob", state=Job.STATES.PROCESSING, priority=3 + ) + job_3 = Job.objects.create(name="testjob", priority=3) + job_4 = Job.objects.create(name="testjob", priority=3) + self.assertEqual( + {job for job in Job.objects.to_process("default")}, {job_1, job_3, job_4} + ) + self.assertEqual(Job.objects.get_ready_or_none("default"), job_1) + self.assertFalse(Job.objects.to_process("default").filter(id=job_2.id).exists()) def test_get_next_ready_job_created(self): """ @@ -185,37 +185,35 @@ def test_get_next_ready_job_created(self): selected by get_ready_or_none (the model is ordered by 'created' and the query picks the .first()) """ - self.assertTrue(Job.objects.get_ready_or_none('default') is None) + self.assertTrue(Job.objects.get_ready_or_none("default") is None) - Job.objects.create(name='testjob', state=Job.STATES.NEW) - Job.objects.create(name='testjob', state=Job.STATES.PROCESSING) - expected = Job.objects.create(name='testjob', state=Job.STATES.NEW) + Job.objects.create(name="testjob", state=Job.STATES.NEW) + Job.objects.create(name="testjob", state=Job.STATES.PROCESSING) + expected = Job.objects.create(name="testjob", state=Job.STATES.NEW) expected.created = datetime.now() - timedelta(minutes=1) expected.save() - self.assertEqual(Job.objects.get_ready_or_none('default'), expected) + self.assertEqual(Job.objects.get_ready_or_none("default"), expected) -@override_settings(JOBS={'testjob': {'tasks': ['a', 'b', 'c']}}) +@override_settings(JOBS={"testjob": {"tasks": ["a", "b", "c"]}}) class JobTaskTestCase(TestCase): - def test_task_sequence(self): - job = Job.objects.create(name='testjob') - self.assertEqual(job.next_task, 'a') + job = Job.objects.create(name="testjob") + self.assertEqual(job.next_task, "a") job.update_next_task() - self.assertEqual(job.next_task, 'b') + self.assertEqual(job.next_task, "b") job.update_next_task() - self.assertEqual(job.next_task, 'c') + self.assertEqual(job.next_task, "c") job.update_next_task() - self.assertEqual(job.next_task, '') + self.assertEqual(job.next_task, "") -@override_settings(JOBS={'testjob': {'tasks': ['django_dbq.tests.test_task']}}) +@override_settings(JOBS={"testjob": {"tasks": ["django_dbq.tests.test_task"]}}) class ProcessJobTestCase(TestCase): - def test_process_job(self): - job = Job.objects.create(name='testjob') - process_job('default') + job = Job.objects.create(name="testjob") + process_job("default") job = Job.objects.get() self.assertEqual(job.state, Job.STATES.COMPLETE) @@ -223,59 +221,70 @@ def test_process_job_wrong_queue(self): """ Processing a different queue shouldn't touch our other job """ - job = Job.objects.create(name='testjob', queue_name='lol') - process_job('default') + job = Job.objects.create(name="testjob", queue_name="lol") + process_job("default") job = Job.objects.get() self.assertEqual(job.state, Job.STATES.NEW) -@override_settings(JOBS={'testjob': {'tasks': ['django_dbq.tests.test_task'], 'creation_hook': 'django_dbq.tests.creation_hook'}}) +@override_settings( + JOBS={ + "testjob": { + "tasks": ["django_dbq.tests.test_task"], + "creation_hook": "django_dbq.tests.creation_hook", + } + } +) class JobCreationHookTestCase(TestCase): - def test_creation_hook(self): - job = Job.objects.create(name='testjob') + job = Job.objects.create(name="testjob") job = Job.objects.get() - self.assertEqual(job.workspace['output'], 'creation hook ran') + self.assertEqual(job.workspace["output"], "creation hook ran") def test_creation_hook_only_runs_on_create(self): - job = Job.objects.create(name='testjob') + job = Job.objects.create(name="testjob") job = Job.objects.get() - job.workspace['output'] = 'creation hook output removed' + job.workspace["output"] = "creation hook output removed" job.save() job = Job.objects.get() - self.assertEqual(job.workspace['output'], 'creation hook output removed') + self.assertEqual(job.workspace["output"], "creation hook output removed") -@override_settings(JOBS={'testjob': {'tasks': ['django_dbq.tests.failing_task'], 'failure_hook': 'django_dbq.tests.failure_hook'}}) +@override_settings( + JOBS={ + "testjob": { + "tasks": ["django_dbq.tests.failing_task"], + "failure_hook": "django_dbq.tests.failure_hook", + } + } +) class JobFailureHookTestCase(TestCase): - def test_failure_hook(self): - job = Job.objects.create(name='testjob') - process_job('default') + job = Job.objects.create(name="testjob") + process_job("default") job = Job.objects.get() self.assertEqual(job.state, Job.STATES.FAILED) - self.assertEqual(job.workspace['output'], 'failure hook ran') + self.assertEqual(job.workspace["output"], "failure hook ran") -@override_settings(JOBS={'testjob': {'tasks': ['a']}}) +@override_settings(JOBS={"testjob": {"tasks": ["a"]}}) class DeleteOldJobsTestCase(TestCase): - def test_delete_old_jobs(self): two_days_ago = datetime.utcnow() - timedelta(days=2) - j1 = Job.objects.create(name='testjob', state=Job.STATES.COMPLETE) + j1 = Job.objects.create(name="testjob", state=Job.STATES.COMPLETE) j1.created = two_days_ago j1.save() - j2 = Job.objects.create(name='testjob', state=Job.STATES.FAILED) + j2 = Job.objects.create(name="testjob", state=Job.STATES.FAILED) j2.created = two_days_ago j2.save() - j3 = Job.objects.create(name='testjob', state=Job.STATES.NEW) + j3 = Job.objects.create(name="testjob", state=Job.STATES.NEW) j3.created = two_days_ago j3.save() - j4 = Job.objects.create(name='testjob', state=Job.STATES.COMPLETE) + j4 = Job.objects.create(name="testjob", state=Job.STATES.COMPLETE) Job.objects.delete_old() diff --git a/setup.py b/setup.py index 2fd5d92..f5d5a90 100644 --- a/setup.py +++ b/setup.py @@ -8,13 +8,13 @@ import sys -name = 'django-db-queue' -package = 'django_dbq' -description = 'Simple database-backed job queue system' -url = 'http://www.dabapps.com' -author = 'DabApps' -author_email = 'contact@dabapps.com' -license = 'BSD' +name = "django-db-queue" +package = "django_dbq" +description = "Simple database-backed job queue system" +url = "http://www.dabapps.com" +author = "DabApps" +author_email = "contact@dabapps.com" +license = "BSD" install_requires = [ "jsonfield==2.0.2", "Django>=1.7", @@ -23,21 +23,26 @@ long_description = """Simple database-backed job queue system""" + def get_version(package): """ Return package version as listed in `__version__` in `init.py`. """ - init_py = open(os.path.join(package, '__init__.py')).read() - return re.search("^__version__ = ['\"]([^'\"]+)['\"]", init_py, re.MULTILINE).group(1) + init_py = open(os.path.join(package, "__init__.py")).read() + return re.search("^__version__ = ['\"]([^'\"]+)['\"]", init_py, re.MULTILINE).group( + 1 + ) def get_packages(package): """ Return root package and all sub-packages. """ - return [dirpath - for dirpath, dirnames, filenames in os.walk(package) - if os.path.exists(os.path.join(dirpath, '__init__.py'))] + return [ + dirpath + for dirpath, dirnames, filenames in os.walk(package) + if os.path.exists(os.path.join(dirpath, "__init__.py")) + ] def get_package_data(package): @@ -45,20 +50,21 @@ def get_package_data(package): Return all files under the root package, that are not in a package themselves. """ - walk = [(dirpath.replace(package + os.sep, '', 1), filenames) - for dirpath, dirnames, filenames in os.walk(package) - if not os.path.exists(os.path.join(dirpath, '__init__.py'))] + walk = [ + (dirpath.replace(package + os.sep, "", 1), filenames) + for dirpath, dirnames, filenames in os.walk(package) + if not os.path.exists(os.path.join(dirpath, "__init__.py")) + ] filepaths = [] for base, filenames in walk: - filepaths.extend([os.path.join(base, filename) - for filename in filenames]) + filepaths.extend([os.path.join(base, filename) for filename in filenames]) return {package: filepaths} -if sys.argv[-1] == 'publish': +if sys.argv[-1] == "publish": os.system("python setup.py sdist upload") - args = {'version': get_version(package)} + args = {"version": get_version(package)} print("You probably want to also tag the version now:") print(" git tag -a %(version)s -m 'version %(version)s'" % args) print(" git push --tags") @@ -77,6 +83,5 @@ def get_package_data(package): packages=get_packages(package), package_data=get_package_data(package), install_requires=install_requires, - classifiers=[ - ] + classifiers=[], ) diff --git a/test-requirements.txt b/test-requirements.txt index 3499577..0d388af 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -1,4 +1,5 @@ -r requirements.txt +black==19.10b0 mysqlclient==1.4.6 freezegun==0.3.12 mock==3.0.5 diff --git a/testsettings.py b/testsettings.py index a700428..6cb7863 100644 --- a/testsettings.py +++ b/testsettings.py @@ -2,41 +2,26 @@ import dj_database_url DATABASES = { - 'default': dj_database_url.parse(os.environ['DATABASE_URL']), + "default": dj_database_url.parse(os.environ["DATABASE_URL"]), } -INSTALLED_APPS = ( - 'django_dbq', -) +INSTALLED_APPS = ("django_dbq",) MIDDLEWARE_CLASSES = ( - 'django.contrib.sessions.middleware.SessionMiddleware', - 'django.middleware.common.CommonMiddleware', - 'django.middleware.csrf.CsrfViewMiddleware', - 'django.contrib.auth.middleware.AuthenticationMiddleware', - 'django.contrib.messages.middleware.MessageMiddleware', - 'django.middleware.clickjacking.XFrameOptionsMiddleware', + "django.contrib.sessions.middleware.SessionMiddleware", + "django.middleware.common.CommonMiddleware", + "django.middleware.csrf.CsrfViewMiddleware", + "django.contrib.auth.middleware.AuthenticationMiddleware", + "django.contrib.messages.middleware.MessageMiddleware", + "django.middleware.clickjacking.XFrameOptionsMiddleware", ) -SECRET_KEY = 'abcde12345' +SECRET_KEY = "abcde12345" LOGGING = { - 'version': 1, - 'disable_existing_loggers': True, - 'handlers': { - 'console': { - 'level': 'DEBUG', - 'class': 'logging.StreamHandler', - }, - }, - 'root': { - 'handlers': ['console'], - 'level': 'INFO', - }, - 'loggers': { - 'django_dbq': { - 'level': 'CRITICAL', - 'propagate': True, - }, - } + "version": 1, + "disable_existing_loggers": True, + "handlers": {"console": {"level": "DEBUG", "class": "logging.StreamHandler",},}, + "root": {"handlers": ["console"], "level": "INFO",}, + "loggers": {"django_dbq": {"level": "CRITICAL", "propagate": True,},}, } From 9d87bf3c3383c1413f642cd2c3a41928f20f7ce9 Mon Sep 17 00:00:00 2001 From: Jordan Brown Date: Thu, 21 Nov 2019 17:33:29 +0000 Subject: [PATCH 039/158] Turn script: into a list --- .travis.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 0a098a2..666fbf7 100644 --- a/.travis.yml +++ b/.travis.yml @@ -16,7 +16,9 @@ env: install: - pip install -r test-requirements.txt - pip install -U django==$DJANGO_VERSION -script: black . --check && python manage.py test +script: + - black --check django_dbq + - python manage.py test deploy: provider: pypi user: dabapps From f75c4773d7bbc943e260da8dee44b10e44eae647 Mon Sep 17 00:00:00 2001 From: Jordan Brown Date: Fri, 22 Nov 2019 10:26:28 +0000 Subject: [PATCH 040/158] Do not run black on Python 3.5 --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 666fbf7..4b950a3 100644 --- a/.travis.yml +++ b/.travis.yml @@ -17,7 +17,7 @@ install: - pip install -r test-requirements.txt - pip install -U django==$DJANGO_VERSION script: - - black --check django_dbq + - if [ "$TRAVIS_PYTHON_VERSION" != "3.5" ]; then black --check django_dbq; fi - python manage.py test deploy: provider: pypi From d9e0eb2ae109064faf4f9e5668b7094af4eaeec4 Mon Sep 17 00:00:00 2001 From: Jordan Brown Date: Fri, 22 Nov 2019 10:39:35 +0000 Subject: [PATCH 041/158] Install black in travis yaml --- .travis.yml | 1 + test-requirements.txt | 1 - 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 4b950a3..d837d65 100644 --- a/.travis.yml +++ b/.travis.yml @@ -16,6 +16,7 @@ env: install: - pip install -r test-requirements.txt - pip install -U django==$DJANGO_VERSION +- if [ "$TRAVIS_PYTHON_VERSION" != "3.5" ]; then pip install black==19.10b0; fi script: - if [ "$TRAVIS_PYTHON_VERSION" != "3.5" ]; then black --check django_dbq; fi - python manage.py test diff --git a/test-requirements.txt b/test-requirements.txt index 0d388af..3499577 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -1,5 +1,4 @@ -r requirements.txt -black==19.10b0 mysqlclient==1.4.6 freezegun==0.3.12 mock==3.0.5 From 4ee8ddd1c8d9511eb2b3074d24ed84c83268a889 Mon Sep 17 00:00:00 2001 From: Jordan Brown Date: Fri, 22 Nov 2019 13:24:51 +0000 Subject: [PATCH 042/158] Add model method to get queue depths as a dict --- django_dbq/models.py | 13 ++++++++++++- django_dbq/test_task.py | 10 ++++++++++ django_dbq/tests.py | 17 +++++++++++++++++ 3 files changed, 39 insertions(+), 1 deletion(-) create mode 100644 django_dbq/test_task.py diff --git a/django_dbq/models.py b/django_dbq/models.py index e483b57..120def0 100644 --- a/django_dbq/models.py +++ b/django_dbq/models.py @@ -6,7 +6,7 @@ get_creation_hook_name, ) from jsonfield import JSONField -from django.db.models import UUIDField +from django.db.models import UUIDField, Count import datetime import logging import uuid @@ -137,3 +137,14 @@ def run_creation_hook(self): logger.info("Running creation hook %s for new job", creation_hook_name) creation_hook_function = import_string(creation_hook_name) creation_hook_function(self) + + @staticmethod + def get_queue_depths_dict(): + annotation_dicts = Job.objects.filter(state__in=(Job.STATES.READY, Job.STATES.NEW)).values('queue_name').order_by().annotate( + Count('queue_name')) + + return { + annotation_dict['queue_name']: annotation_dict['queue_name__count'] + for annotation_dict + in annotation_dicts + } diff --git a/django_dbq/test_task.py b/django_dbq/test_task.py new file mode 100644 index 0000000..dcaa04f --- /dev/null +++ b/django_dbq/test_task.py @@ -0,0 +1,10 @@ +from time import sleep + +from django_dbq.models import Job + + +def test_task(job): + print('going to sleep') + sleep(45) + print('running job') + Job.objects.filter(id=1) diff --git a/django_dbq/tests.py b/django_dbq/tests.py index c566461..25563e2 100644 --- a/django_dbq/tests.py +++ b/django_dbq/tests.py @@ -81,6 +81,23 @@ def test_worker_with_queue_name(self): self.assertTrue("test_queue" in output) +@override_settings(JOBS={"testjob": {"tasks": ["a"]}}) +class JobModelMethodTestCase(TestCase): + + def test_get_queue_depths_dict(self): + Job.objects.create(name='testjob', queue_name='default') + Job.objects.create(name='testjob', queue_name='testworker') + Job.objects.create(name='testjob', queue_name='testworker') + Job.objects.create(name='testjob', queue_name='testworker', state=Job.STATES.FAILED) + Job.objects.create(name='testjob', queue_name='testworker', state=Job.STATES.COMPLETE) + + queue_depths = Job.get_queue_depths_dict() + self.assertDictEqual(queue_depths, { + 'default': 1, + 'testworker': 2 + }) + + @freezegun.freeze_time() @mock.patch("django_dbq.management.commands.worker.sleep") @mock.patch("django_dbq.management.commands.worker.process_job") From 1b0a771a5afb031f6d660aabd8eaf5f0ec856c83 Mon Sep 17 00:00:00 2001 From: Jordan Brown Date: Thu, 28 Nov 2019 15:33:36 +0000 Subject: [PATCH 043/158] Add get_queue_depths_dict method to Job model --- django_dbq/models.py | 13 ++++++++----- django_dbq/tests.py | 20 ++++++++++---------- 2 files changed, 18 insertions(+), 15 deletions(-) diff --git a/django_dbq/models.py b/django_dbq/models.py index 120def0..f667554 100644 --- a/django_dbq/models.py +++ b/django_dbq/models.py @@ -140,11 +140,14 @@ def run_creation_hook(self): @staticmethod def get_queue_depths_dict(): - annotation_dicts = Job.objects.filter(state__in=(Job.STATES.READY, Job.STATES.NEW)).values('queue_name').order_by().annotate( - Count('queue_name')) + annotation_dicts = ( + Job.objects.filter(state__in=(Job.STATES.READY, Job.STATES.NEW)) + .values("queue_name") + .order_by() + .annotate(Count("queue_name")) + ) return { - annotation_dict['queue_name']: annotation_dict['queue_name__count'] - for annotation_dict - in annotation_dicts + annotation_dict["queue_name"]: annotation_dict["queue_name__count"] + for annotation_dict in annotation_dicts } diff --git a/django_dbq/tests.py b/django_dbq/tests.py index 25563e2..af254ba 100644 --- a/django_dbq/tests.py +++ b/django_dbq/tests.py @@ -83,19 +83,19 @@ def test_worker_with_queue_name(self): @override_settings(JOBS={"testjob": {"tasks": ["a"]}}) class JobModelMethodTestCase(TestCase): - def test_get_queue_depths_dict(self): - Job.objects.create(name='testjob', queue_name='default') - Job.objects.create(name='testjob', queue_name='testworker') - Job.objects.create(name='testjob', queue_name='testworker') - Job.objects.create(name='testjob', queue_name='testworker', state=Job.STATES.FAILED) - Job.objects.create(name='testjob', queue_name='testworker', state=Job.STATES.COMPLETE) + Job.objects.create(name="testjob", queue_name="default") + Job.objects.create(name="testjob", queue_name="testworker") + Job.objects.create(name="testjob", queue_name="testworker") + Job.objects.create( + name="testjob", queue_name="testworker", state=Job.STATES.FAILED + ) + Job.objects.create( + name="testjob", queue_name="testworker", state=Job.STATES.COMPLETE + ) queue_depths = Job.get_queue_depths_dict() - self.assertDictEqual(queue_depths, { - 'default': 1, - 'testworker': 2 - }) + self.assertDictEqual(queue_depths, {"default": 1, "testworker": 2}) @freezegun.freeze_time() From 8cd9516225665b6942602e772f3cf4bdf70ddbe9 Mon Sep 17 00:00:00 2001 From: Jordan Brown Date: Thu, 28 Nov 2019 15:58:32 +0000 Subject: [PATCH 044/158] Add queue_depth management command --- django_dbq/management/commands/queue_depth.py | 16 +++++++++++++ django_dbq/tests.py | 23 +++++++++++++++++++ 2 files changed, 39 insertions(+) create mode 100644 django_dbq/management/commands/queue_depth.py diff --git a/django_dbq/management/commands/queue_depth.py b/django_dbq/management/commands/queue_depth.py new file mode 100644 index 0000000..4cb2626 --- /dev/null +++ b/django_dbq/management/commands/queue_depth.py @@ -0,0 +1,16 @@ +from django.core.management.base import BaseCommand +from django_dbq.models import Job + + +class Command(BaseCommand): + + help = "Print the current depth of the given queue" + + def add_arguments(self, parser): + parser.add_argument('queue_name', nargs='?', default='default', type=str) + + def handle(self, *args, **options): + queue_name = options['queue_name'] + queue_depths = Job.get_queue_depths_dict() + + self.stdout.write('queue_name={queue_name} queue_depth={depth}'.format(queue_name=queue_name, depth=queue_depths.get(queue_name, 0))) diff --git a/django_dbq/tests.py b/django_dbq/tests.py index af254ba..10769fb 100644 --- a/django_dbq/tests.py +++ b/django_dbq/tests.py @@ -98,6 +98,29 @@ def test_get_queue_depths_dict(self): self.assertDictEqual(queue_depths, {"default": 1, "testworker": 2}) +@override_settings(JOBS={'testjob': {'tasks': ['a']}}) +class QueueDepthTestCase(TestCase): + + def test_queue_depth(self): + + Job.objects.create(name='testjob', state=Job.STATES.FAILED) + Job.objects.create(name='testjob', state=Job.STATES.NEW) + Job.objects.create(name='testjob', state=Job.STATES.FAILED) + Job.objects.create(name='testjob', state=Job.STATES.COMPLETE) + Job.objects.create(name='testjob', state=Job.STATES.READY) + + stdout = StringIO() + call_command('queue_depth', stdout=stdout) + output = stdout.getvalue() + self.assertEqual(output.strip(), 'queue_name=default queue_depth=2') + + def test_queue_depth_for_queue_with_zero_jobs(self): + stdout = StringIO() + call_command('queue_depth', queue_name='otherqueue', stdout=stdout) + output = stdout.getvalue() + self.assertEqual(output.strip(), 'queue_name=otherqueue queue_depth=0') + + @freezegun.freeze_time() @mock.patch("django_dbq.management.commands.worker.sleep") @mock.patch("django_dbq.management.commands.worker.process_job") From ad555b5d13cb380b02165e1b1e77edfbf2c28d11 Mon Sep 17 00:00:00 2001 From: Jordan Brown Date: Thu, 28 Nov 2019 15:58:55 +0000 Subject: [PATCH 045/158] Add documentation for queue depth management command and model method --- README.md | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/README.md b/README.md index 5d52848..609b6d9 100644 --- a/README.md +++ b/README.md @@ -168,13 +168,34 @@ Jobs have a `state` field which can have one of the following values: ### API +#### Model methods + +##### Job.get_queue_depths_dict +If you need to programatically get the depth of any queue you can run the following: +```python +from django_dbq.models import Job + +... + +Job.objects.create(name='do_work', workspace={}) +Job.objects.create(name='do_other_work', queue_name='other_queue', workspace={}) + +queue_depths = Job.get_queue_depths_dict() +print(queue_depths) # {"default": 1, "other_queue": 1} +``` + +**Important:** When checking queue depths, do not assume that the key for your queue will always be available. Queue depths of zero won't be included +in the dict returned by this method. + #### Management commands +##### manage.py delete_old_jobs There is a management command, `manage.py delete_old_jobs`, which deletes any jobs from the database which are in state `COMPLETE` or `FAILED` and were created more than 24 hours ago. This could be run, for example, as a cron task, to ensure the jobs table remains at a reasonable size. +##### manage.py create_job For debugging/development purposes, a simple management command is supplied to create jobs: manage.py create_job --queue_name 'my_queue_name' --workspace '{"key": "value"}' @@ -183,6 +204,7 @@ The `workspace` flag is optional. If supplied, it must be a valid JSON string. `queue_name` is optional and defaults to `default` +##### manage.py worker To start a worker: manage.py worker [queue_name] [--rate_limit] @@ -190,6 +212,12 @@ To start a worker: - `queue_name` is optional, and will default to `default` - The `--rate_limit` flag is optional, and will default to `1`. It is the minimum number of seconds that must have elapsed before a subsequent job can be run. +##### manage.py queue_depth +If you'd like to check your queue depth from the command line, you can run `manage.py queue_depth [queue_name]` and any +jobs in the "NEW" or "READY" states will be returned. + +**Important:** If you misspell or provide a queue name which does not have any jobs, a depth of 0 will always be returned. + ## Testing It may be necessary to supply a DATABASE_PORT environment variable. From 5fcd1a86ced6d74d429a729936c3038d270bcee2 Mon Sep 17 00:00:00 2001 From: Jordan Brown Date: Thu, 28 Nov 2019 16:07:59 +0000 Subject: [PATCH 046/158] Black --- django_dbq/management/commands/queue_depth.py | 10 ++++++--- django_dbq/test_task.py | 4 ++-- django_dbq/tests.py | 21 +++++++++---------- 3 files changed, 19 insertions(+), 16 deletions(-) diff --git a/django_dbq/management/commands/queue_depth.py b/django_dbq/management/commands/queue_depth.py index 4cb2626..6e3a725 100644 --- a/django_dbq/management/commands/queue_depth.py +++ b/django_dbq/management/commands/queue_depth.py @@ -7,10 +7,14 @@ class Command(BaseCommand): help = "Print the current depth of the given queue" def add_arguments(self, parser): - parser.add_argument('queue_name', nargs='?', default='default', type=str) + parser.add_argument("queue_name", nargs="?", default="default", type=str) def handle(self, *args, **options): - queue_name = options['queue_name'] + queue_name = options["queue_name"] queue_depths = Job.get_queue_depths_dict() - self.stdout.write('queue_name={queue_name} queue_depth={depth}'.format(queue_name=queue_name, depth=queue_depths.get(queue_name, 0))) + self.stdout.write( + "queue_name={queue_name} queue_depth={depth}".format( + queue_name=queue_name, depth=queue_depths.get(queue_name, 0) + ) + ) diff --git a/django_dbq/test_task.py b/django_dbq/test_task.py index dcaa04f..6b4d9ee 100644 --- a/django_dbq/test_task.py +++ b/django_dbq/test_task.py @@ -4,7 +4,7 @@ def test_task(job): - print('going to sleep') + print("going to sleep") sleep(45) - print('running job') + print("running job") Job.objects.filter(id=1) diff --git a/django_dbq/tests.py b/django_dbq/tests.py index 10769fb..c6058cb 100644 --- a/django_dbq/tests.py +++ b/django_dbq/tests.py @@ -98,27 +98,26 @@ def test_get_queue_depths_dict(self): self.assertDictEqual(queue_depths, {"default": 1, "testworker": 2}) -@override_settings(JOBS={'testjob': {'tasks': ['a']}}) +@override_settings(JOBS={"testjob": {"tasks": ["a"]}}) class QueueDepthTestCase(TestCase): - def test_queue_depth(self): - Job.objects.create(name='testjob', state=Job.STATES.FAILED) - Job.objects.create(name='testjob', state=Job.STATES.NEW) - Job.objects.create(name='testjob', state=Job.STATES.FAILED) - Job.objects.create(name='testjob', state=Job.STATES.COMPLETE) - Job.objects.create(name='testjob', state=Job.STATES.READY) + Job.objects.create(name="testjob", state=Job.STATES.FAILED) + Job.objects.create(name="testjob", state=Job.STATES.NEW) + Job.objects.create(name="testjob", state=Job.STATES.FAILED) + Job.objects.create(name="testjob", state=Job.STATES.COMPLETE) + Job.objects.create(name="testjob", state=Job.STATES.READY) stdout = StringIO() - call_command('queue_depth', stdout=stdout) + call_command("queue_depth", stdout=stdout) output = stdout.getvalue() - self.assertEqual(output.strip(), 'queue_name=default queue_depth=2') + self.assertEqual(output.strip(), "queue_name=default queue_depth=2") def test_queue_depth_for_queue_with_zero_jobs(self): stdout = StringIO() - call_command('queue_depth', queue_name='otherqueue', stdout=stdout) + call_command("queue_depth", queue_name="otherqueue", stdout=stdout) output = stdout.getvalue() - self.assertEqual(output.strip(), 'queue_name=otherqueue queue_depth=0') + self.assertEqual(output.strip(), "queue_name=otherqueue queue_depth=0") @freezegun.freeze_time() From eff2fbaadfefb3470a3507010dece6ff4025244f Mon Sep 17 00:00:00 2001 From: Jordan Brown Date: Thu, 28 Nov 2019 16:26:36 +0000 Subject: [PATCH 047/158] Remove test task file --- django_dbq/test_task.py | 10 ---------- 1 file changed, 10 deletions(-) delete mode 100644 django_dbq/test_task.py diff --git a/django_dbq/test_task.py b/django_dbq/test_task.py deleted file mode 100644 index 6b4d9ee..0000000 --- a/django_dbq/test_task.py +++ /dev/null @@ -1,10 +0,0 @@ -from time import sleep - -from django_dbq.models import Job - - -def test_task(job): - print("going to sleep") - sleep(45) - print("running job") - Job.objects.filter(id=1) From 4c23d4c87e075d2658725818589f5e0ee8446956 Mon Sep 17 00:00:00 2001 From: Jordan Brown Date: Thu, 28 Nov 2019 16:26:55 +0000 Subject: [PATCH 048/158] Specify name in order_by --- django_dbq/models.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/django_dbq/models.py b/django_dbq/models.py index f667554..4198946 100644 --- a/django_dbq/models.py +++ b/django_dbq/models.py @@ -143,7 +143,7 @@ def get_queue_depths_dict(): annotation_dicts = ( Job.objects.filter(state__in=(Job.STATES.READY, Job.STATES.NEW)) .values("queue_name") - .order_by() + .order_by('queue_name') .annotate(Count("queue_name")) ) From cb3ed1627c27bd687050fa418b5691a0c2f30063 Mon Sep 17 00:00:00 2001 From: Jordan Brown Date: Thu, 28 Nov 2019 17:14:18 +0000 Subject: [PATCH 049/158] Blak --- django_dbq/models.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/django_dbq/models.py b/django_dbq/models.py index 4198946..634a9fa 100644 --- a/django_dbq/models.py +++ b/django_dbq/models.py @@ -143,7 +143,7 @@ def get_queue_depths_dict(): annotation_dicts = ( Job.objects.filter(state__in=(Job.STATES.READY, Job.STATES.NEW)) .values("queue_name") - .order_by('queue_name') + .order_by("queue_name") .annotate(Count("queue_name")) ) From f4aa96134f213d8ec48306bec1b1b768ce0ea1c8 Mon Sep 17 00:00:00 2001 From: Jordan Brown Date: Mon, 2 Dec 2019 11:20:45 +0000 Subject: [PATCH 050/158] Fix definition of --rate-limit flag to match docs --- django_dbq/management/commands/worker.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/django_dbq/management/commands/worker.py b/django_dbq/management/commands/worker.py index b0c6724..a09a0a9 100644 --- a/django_dbq/management/commands/worker.py +++ b/django_dbq/management/commands/worker.py @@ -104,7 +104,7 @@ class Command(BaseCommand): def add_arguments(self, parser): parser.add_argument("queue_name", nargs="?", default="default", type=str) parser.add_argument( - "rate_limit", + "--rate_limit", help="The rate limit in seconds. The default rate limit is 1 job per second.", nargs="?", default=1, From bbed85754558e822713d58aaf6038f4fe7766c55 Mon Sep 17 00:00:00 2001 From: Jordan Brown Date: Mon, 2 Dec 2019 16:44:35 +0000 Subject: [PATCH 051/158] Rename Job.get_queue_depths_dict to Job.get_queue_depths --- django_dbq/management/commands/queue_depth.py | 2 +- django_dbq/models.py | 2 +- django_dbq/tests.py | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/django_dbq/management/commands/queue_depth.py b/django_dbq/management/commands/queue_depth.py index 6e3a725..7def792 100644 --- a/django_dbq/management/commands/queue_depth.py +++ b/django_dbq/management/commands/queue_depth.py @@ -11,7 +11,7 @@ def add_arguments(self, parser): def handle(self, *args, **options): queue_name = options["queue_name"] - queue_depths = Job.get_queue_depths_dict() + queue_depths = Job.get_queue_depths() self.stdout.write( "queue_name={queue_name} queue_depth={depth}".format( diff --git a/django_dbq/models.py b/django_dbq/models.py index 634a9fa..65e7150 100644 --- a/django_dbq/models.py +++ b/django_dbq/models.py @@ -139,7 +139,7 @@ def run_creation_hook(self): creation_hook_function(self) @staticmethod - def get_queue_depths_dict(): + def get_queue_depths(): annotation_dicts = ( Job.objects.filter(state__in=(Job.STATES.READY, Job.STATES.NEW)) .values("queue_name") diff --git a/django_dbq/tests.py b/django_dbq/tests.py index c6058cb..d6a5bc5 100644 --- a/django_dbq/tests.py +++ b/django_dbq/tests.py @@ -83,7 +83,7 @@ def test_worker_with_queue_name(self): @override_settings(JOBS={"testjob": {"tasks": ["a"]}}) class JobModelMethodTestCase(TestCase): - def test_get_queue_depths_dict(self): + def test_get_queue_depths(self): Job.objects.create(name="testjob", queue_name="default") Job.objects.create(name="testjob", queue_name="testworker") Job.objects.create(name="testjob", queue_name="testworker") @@ -94,7 +94,7 @@ def test_get_queue_depths_dict(self): name="testjob", queue_name="testworker", state=Job.STATES.COMPLETE ) - queue_depths = Job.get_queue_depths_dict() + queue_depths = Job.get_queue_depths() self.assertDictEqual(queue_depths, {"default": 1, "testworker": 2}) From d119331cd84efe1b208cd6704da347c2fd5864d9 Mon Sep 17 00:00:00 2001 From: Jordan Brown Date: Mon, 2 Dec 2019 16:46:19 +0000 Subject: [PATCH 052/158] Log all_queues_depth in manage.py queue_depth --- django_dbq/management/commands/queue_depth.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/django_dbq/management/commands/queue_depth.py b/django_dbq/management/commands/queue_depth.py index 7def792..2258f75 100644 --- a/django_dbq/management/commands/queue_depth.py +++ b/django_dbq/management/commands/queue_depth.py @@ -12,9 +12,12 @@ def add_arguments(self, parser): def handle(self, *args, **options): queue_name = options["queue_name"] queue_depths = Job.get_queue_depths() + all_queues_depth = sum([queue_depth for _, queue_depth in queue_depths.items()]) self.stdout.write( - "queue_name={queue_name} queue_depth={depth}".format( - queue_name=queue_name, depth=queue_depths.get(queue_name, 0) + "all_queues_depth= queue_name={queue_name} queue_depth={depth}".format( + all_queues_depth=all_queues_depth, + queue_name=queue_name, + depth=queue_depths.get(queue_name, 0), ) ) From 99b382cc198a45ec2f28525f6e9102d7e47072ca Mon Sep 17 00:00:00 2001 From: Jordan Brown Date: Mon, 2 Dec 2019 16:52:05 +0000 Subject: [PATCH 053/158] Update documented supported Python versions --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 5d52848..c60159b 100644 --- a/README.md +++ b/README.md @@ -9,7 +9,7 @@ Asynchronous tasks are run via a *job queue*. This system is designed to support Supported and tested against: - Django 1.11 and 2.2 -- Python 3.7 and 3.8 +- Python 3.5, 3.6, 3.7 and 3.8 This package may still work with older versions of Django and Python but they aren't explicitly supported. From 0ea57ec8023e51217fdd85bb7979151f02b227f4 Mon Sep 17 00:00:00 2001 From: Jordan Brown Date: Mon, 2 Dec 2019 17:05:01 +0000 Subject: [PATCH 054/158] Document prioritising jobs --- README.md | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/README.md b/README.md index c60159b..4246283 100644 --- a/README.md +++ b/README.md @@ -114,6 +114,20 @@ Using the name you configured for your job in your settings, create an instance Job.objects.create(name='my_job') ``` +### Prioritising jobs +Sometimes it is necessary for certain jobs to take precedence over others. For example; you may have a worker which has a primary purpose of dispatching somewhat +important emails to users. However, once an hour, you may need to run a _really_ important job which needs to be done on time and cannot wait in the queue for dozens +of emails to be dispatched before it can begin. + +In order to make sure that an important job is run before others, you can set the `priority` field to an integer higher than `0` (the default). For example: +```python +Job.objects.create(name='normal_job') +Job.objects.create(name='important_job', priority=1) +Job.objects.create(name='critical_job', priority=2) +``` + +Jobs will be ordered by their `priority` (highest to lowest) and then the time which they were created (oldest to newest) and processed in that order. + ## Terminology ### Job From c8878d706cb60ce9d27fa3c5a1b91494248b6104 Mon Sep 17 00:00:00 2001 From: Jordan Brown Date: Mon, 2 Dec 2019 17:11:45 +0000 Subject: [PATCH 055/158] Update django_dbq/management/commands/queue_depth.py Co-Authored-By: Jake Howard --- django_dbq/management/commands/queue_depth.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/django_dbq/management/commands/queue_depth.py b/django_dbq/management/commands/queue_depth.py index 2258f75..e71b15e 100644 --- a/django_dbq/management/commands/queue_depth.py +++ b/django_dbq/management/commands/queue_depth.py @@ -12,7 +12,7 @@ def add_arguments(self, parser): def handle(self, *args, **options): queue_name = options["queue_name"] queue_depths = Job.get_queue_depths() - all_queues_depth = sum([queue_depth for _, queue_depth in queue_depths.items()]) + all_queues_depth = sum(queue_depths.values()) self.stdout.write( "all_queues_depth= queue_name={queue_name} queue_depth={depth}".format( From 89973ed75031fb4e697b66d9f7d150e11978bce0 Mon Sep 17 00:00:00 2001 From: Jordan Brown Date: Mon, 2 Dec 2019 17:11:55 +0000 Subject: [PATCH 056/158] Update django_dbq/management/commands/queue_depth.py Co-Authored-By: Jake Howard --- django_dbq/management/commands/queue_depth.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/django_dbq/management/commands/queue_depth.py b/django_dbq/management/commands/queue_depth.py index e71b15e..7b1dd85 100644 --- a/django_dbq/management/commands/queue_depth.py +++ b/django_dbq/management/commands/queue_depth.py @@ -15,7 +15,7 @@ def handle(self, *args, **options): all_queues_depth = sum(queue_depths.values()) self.stdout.write( - "all_queues_depth= queue_name={queue_name} queue_depth={depth}".format( + "queue_name={queue_name} queue_depth={depth} all_queues_depth={all_queue_depths}".format( all_queues_depth=all_queues_depth, queue_name=queue_name, depth=queue_depths.get(queue_name, 0), From 8ea8d810b62bb37c3c848540ba866d20d09f8671 Mon Sep 17 00:00:00 2001 From: Jordan Brown Date: Mon, 2 Dec 2019 17:14:49 +0000 Subject: [PATCH 057/158] Update tests --- django_dbq/management/commands/queue_depth.py | 2 +- django_dbq/tests.py | 14 ++++++++++++-- 2 files changed, 13 insertions(+), 3 deletions(-) diff --git a/django_dbq/management/commands/queue_depth.py b/django_dbq/management/commands/queue_depth.py index 7b1dd85..b695581 100644 --- a/django_dbq/management/commands/queue_depth.py +++ b/django_dbq/management/commands/queue_depth.py @@ -15,7 +15,7 @@ def handle(self, *args, **options): all_queues_depth = sum(queue_depths.values()) self.stdout.write( - "queue_name={queue_name} queue_depth={depth} all_queues_depth={all_queue_depths}".format( + "queue_name={queue_name} queue_depth={depth} all_queues_depth={all_queues_depth}".format( all_queues_depth=all_queues_depth, queue_name=queue_name, depth=queue_depths.get(queue_name, 0), diff --git a/django_dbq/tests.py b/django_dbq/tests.py index d6a5bc5..ebb6dd1 100644 --- a/django_dbq/tests.py +++ b/django_dbq/tests.py @@ -107,17 +107,27 @@ def test_queue_depth(self): Job.objects.create(name="testjob", state=Job.STATES.FAILED) Job.objects.create(name="testjob", state=Job.STATES.COMPLETE) Job.objects.create(name="testjob", state=Job.STATES.READY) + Job.objects.create( + name="testjob", queue_name="testqueue", state=Job.STATES.READY + ) + Job.objects.create( + name="testjob", queue_name="testqueue", state=Job.STATES.READY + ) stdout = StringIO() call_command("queue_depth", stdout=stdout) output = stdout.getvalue() - self.assertEqual(output.strip(), "queue_name=default queue_depth=2") + self.assertEqual( + output.strip(), "queue_name=default queue_depth=2 all_queues_depth=4" + ) def test_queue_depth_for_queue_with_zero_jobs(self): stdout = StringIO() call_command("queue_depth", queue_name="otherqueue", stdout=stdout) output = stdout.getvalue() - self.assertEqual(output.strip(), "queue_name=otherqueue queue_depth=0") + self.assertEqual( + output.strip(), "queue_name=otherqueue queue_depth=0 all_queues_depth=0" + ) @freezegun.freeze_time() From 3025c744402b6a5101147f65b500933d5541f045 Mon Sep 17 00:00:00 2001 From: Jordan Brown Date: Mon, 2 Dec 2019 17:20:49 +0000 Subject: [PATCH 058/158] Remove all_queues_depth from queue_depth command --- django_dbq/management/commands/queue_depth.py | 7 ++----- django_dbq/tests.py | 8 ++------ 2 files changed, 4 insertions(+), 11 deletions(-) diff --git a/django_dbq/management/commands/queue_depth.py b/django_dbq/management/commands/queue_depth.py index b695581..99d5280 100644 --- a/django_dbq/management/commands/queue_depth.py +++ b/django_dbq/management/commands/queue_depth.py @@ -12,12 +12,9 @@ def add_arguments(self, parser): def handle(self, *args, **options): queue_name = options["queue_name"] queue_depths = Job.get_queue_depths() - all_queues_depth = sum(queue_depths.values()) self.stdout.write( - "queue_name={queue_name} queue_depth={depth} all_queues_depth={all_queues_depth}".format( - all_queues_depth=all_queues_depth, - queue_name=queue_name, - depth=queue_depths.get(queue_name, 0), + "queue_name={queue_name} queue_depth={depth}".format( + queue_name=queue_name, depth=queue_depths.get(queue_name, 0), ) ) diff --git a/django_dbq/tests.py b/django_dbq/tests.py index ebb6dd1..672984a 100644 --- a/django_dbq/tests.py +++ b/django_dbq/tests.py @@ -117,17 +117,13 @@ def test_queue_depth(self): stdout = StringIO() call_command("queue_depth", stdout=stdout) output = stdout.getvalue() - self.assertEqual( - output.strip(), "queue_name=default queue_depth=2 all_queues_depth=4" - ) + self.assertEqual(output.strip(), "queue_name=default queue_depth=2") def test_queue_depth_for_queue_with_zero_jobs(self): stdout = StringIO() call_command("queue_depth", queue_name="otherqueue", stdout=stdout) output = stdout.getvalue() - self.assertEqual( - output.strip(), "queue_name=otherqueue queue_depth=0 all_queues_depth=0" - ) + self.assertEqual(output.strip(), "queue_name=otherqueue queue_depth=0") @freezegun.freeze_time() From 36da3f6aacc96a3d3fe2dee965c8c5105a91d185 Mon Sep 17 00:00:00 2001 From: Jordan Brown Date: Mon, 2 Dec 2019 17:38:48 +0000 Subject: [PATCH 059/158] Have manage.py queue_depths take multiple queues and return their depths --- django_dbq/management/commands/queue_depth.py | 11 +++++--- django_dbq/tests.py | 25 ++++++++++++++++--- 2 files changed, 29 insertions(+), 7 deletions(-) diff --git a/django_dbq/management/commands/queue_depth.py b/django_dbq/management/commands/queue_depth.py index 99d5280..7dcaa77 100644 --- a/django_dbq/management/commands/queue_depth.py +++ b/django_dbq/management/commands/queue_depth.py @@ -7,14 +7,17 @@ class Command(BaseCommand): help = "Print the current depth of the given queue" def add_arguments(self, parser): - parser.add_argument("queue_name", nargs="?", default="default", type=str) + parser.add_argument("queue_name", nargs="*", default=["default"], type=str) def handle(self, *args, **options): - queue_name = options["queue_name"] + queue_names = options["queue_name"] queue_depths = Job.get_queue_depths() self.stdout.write( - "queue_name={queue_name} queue_depth={depth}".format( - queue_name=queue_name, depth=queue_depths.get(queue_name, 0), + " ".join( + [ + f"{queue_name}={queue_depths.get(queue_name, 0)}" + for queue_name in queue_names + ] ) ) diff --git a/django_dbq/tests.py b/django_dbq/tests.py index 672984a..1eb117d 100644 --- a/django_dbq/tests.py +++ b/django_dbq/tests.py @@ -117,13 +117,32 @@ def test_queue_depth(self): stdout = StringIO() call_command("queue_depth", stdout=stdout) output = stdout.getvalue() - self.assertEqual(output.strip(), "queue_name=default queue_depth=2") + self.assertEqual(output.strip(), "default=2") + + def test_queue_depth_multiple_queues(self): + + Job.objects.create(name="testjob", state=Job.STATES.FAILED) + Job.objects.create(name="testjob", state=Job.STATES.NEW) + Job.objects.create(name="testjob", state=Job.STATES.FAILED) + Job.objects.create(name="testjob", state=Job.STATES.COMPLETE) + Job.objects.create(name="testjob", state=Job.STATES.READY) + Job.objects.create( + name="testjob", queue_name="testqueue", state=Job.STATES.READY + ) + Job.objects.create( + name="testjob", queue_name="testqueue", state=Job.STATES.READY + ) + + stdout = StringIO() + call_command("queue_depth", queue_name=("default", "testqueue",), stdout=stdout) + output = stdout.getvalue() + self.assertEqual(output.strip(), "default=2 testqueue=2") def test_queue_depth_for_queue_with_zero_jobs(self): stdout = StringIO() - call_command("queue_depth", queue_name="otherqueue", stdout=stdout) + call_command("queue_depth", queue_name=("otherqueue",), stdout=stdout) output = stdout.getvalue() - self.assertEqual(output.strip(), "queue_name=otherqueue queue_depth=0") + self.assertEqual(output.strip(), "otherqueue=0") @freezegun.freeze_time() From c4f87b38484fac1a9dcb73896df092c3baad83dd Mon Sep 17 00:00:00 2001 From: Jordan Brown Date: Tue, 3 Dec 2019 08:37:40 +0000 Subject: [PATCH 060/158] Update readme --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 609b6d9..2f3b083 100644 --- a/README.md +++ b/README.md @@ -170,7 +170,7 @@ Jobs have a `state` field which can have one of the following values: #### Model methods -##### Job.get_queue_depths_dict +##### Job.get_queue_depths If you need to programatically get the depth of any queue you can run the following: ```python from django_dbq.models import Job @@ -180,7 +180,7 @@ from django_dbq.models import Job Job.objects.create(name='do_work', workspace={}) Job.objects.create(name='do_other_work', queue_name='other_queue', workspace={}) -queue_depths = Job.get_queue_depths_dict() +queue_depths = Job.get_queue_depths() print(queue_depths) # {"default": 1, "other_queue": 1} ``` From bc0193a3a32521512b77e6149e91eab805836f8d Mon Sep 17 00:00:00 2001 From: Jordan Brown Date: Tue, 3 Dec 2019 08:39:40 +0000 Subject: [PATCH 061/158] Convert f-string to .format call --- django_dbq/management/commands/queue_depth.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/django_dbq/management/commands/queue_depth.py b/django_dbq/management/commands/queue_depth.py index 7dcaa77..2ba005e 100644 --- a/django_dbq/management/commands/queue_depth.py +++ b/django_dbq/management/commands/queue_depth.py @@ -16,7 +16,10 @@ def handle(self, *args, **options): self.stdout.write( " ".join( [ - f"{queue_name}={queue_depths.get(queue_name, 0)}" + "{queue_name}={queue_depth}".format( + queue_name=queue_name, + queue_depth=queue_depths.get(queue_name, 0) + ) for queue_name in queue_names ] ) From 939928a2e7b051a123b73c7edbee772ba484381d Mon Sep 17 00:00:00 2001 From: Jordan Brown Date: Tue, 3 Dec 2019 08:39:52 +0000 Subject: [PATCH 062/158] Black --- django_dbq/management/commands/queue_depth.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/django_dbq/management/commands/queue_depth.py b/django_dbq/management/commands/queue_depth.py index 2ba005e..8ae135c 100644 --- a/django_dbq/management/commands/queue_depth.py +++ b/django_dbq/management/commands/queue_depth.py @@ -18,7 +18,7 @@ def handle(self, *args, **options): [ "{queue_name}={queue_depth}".format( queue_name=queue_name, - queue_depth=queue_depths.get(queue_name, 0) + queue_depth=queue_depths.get(queue_name, 0), ) for queue_name in queue_names ] From 7ea4e51ad581aad9c320cdf8546088be188b6ed9 Mon Sep 17 00:00:00 2001 From: Jordan Brown Date: Tue, 3 Dec 2019 08:57:59 +0000 Subject: [PATCH 063/158] Add event key to queue depths --- README.md | 2 +- django_dbq/management/commands/queue_depth.py | 19 ++++++++++--------- django_dbq/tests.py | 6 +++--- 3 files changed, 14 insertions(+), 13 deletions(-) diff --git a/README.md b/README.md index 2f3b083..b00d1ea 100644 --- a/README.md +++ b/README.md @@ -213,7 +213,7 @@ To start a worker: - The `--rate_limit` flag is optional, and will default to `1`. It is the minimum number of seconds that must have elapsed before a subsequent job can be run. ##### manage.py queue_depth -If you'd like to check your queue depth from the command line, you can run `manage.py queue_depth [queue_name]` and any +If you'd like to check your queue depth from the command line, you can run `manage.py queue_depth [queue_name [queue_name ...]]` and any jobs in the "NEW" or "READY" states will be returned. **Important:** If you misspell or provide a queue name which does not have any jobs, a depth of 0 will always be returned. diff --git a/django_dbq/management/commands/queue_depth.py b/django_dbq/management/commands/queue_depth.py index 8ae135c..483ddc5 100644 --- a/django_dbq/management/commands/queue_depth.py +++ b/django_dbq/management/commands/queue_depth.py @@ -13,14 +13,15 @@ def handle(self, *args, **options): queue_names = options["queue_name"] queue_depths = Job.get_queue_depths() + queue_depths_string = " ".join( + [ + "{queue_name}={queue_depth}".format( + queue_name=queue_name, queue_depth=queue_depths.get(queue_name, 0), + ) + for queue_name in queue_names + ] + ) + self.stdout.write( - " ".join( - [ - "{queue_name}={queue_depth}".format( - queue_name=queue_name, - queue_depth=queue_depths.get(queue_name, 0), - ) - for queue_name in queue_names - ] - ) + "event=queue_depths {queue_depths}".format(queue_depths=queue_depths_string) ) diff --git a/django_dbq/tests.py b/django_dbq/tests.py index 1eb117d..f354551 100644 --- a/django_dbq/tests.py +++ b/django_dbq/tests.py @@ -117,7 +117,7 @@ def test_queue_depth(self): stdout = StringIO() call_command("queue_depth", stdout=stdout) output = stdout.getvalue() - self.assertEqual(output.strip(), "default=2") + self.assertEqual(output.strip(), "event=queue_depths default=2") def test_queue_depth_multiple_queues(self): @@ -136,13 +136,13 @@ def test_queue_depth_multiple_queues(self): stdout = StringIO() call_command("queue_depth", queue_name=("default", "testqueue",), stdout=stdout) output = stdout.getvalue() - self.assertEqual(output.strip(), "default=2 testqueue=2") + self.assertEqual(output.strip(), "event=queue_depths default=2 testqueue=2") def test_queue_depth_for_queue_with_zero_jobs(self): stdout = StringIO() call_command("queue_depth", queue_name=("otherqueue",), stdout=stdout) output = stdout.getvalue() - self.assertEqual(output.strip(), "otherqueue=0") + self.assertEqual(output.strip(), "event=queue_depths otherqueue=0") @freezegun.freeze_time() From 810aa6c5f14e786215af3d2d3696152dc7384059 Mon Sep 17 00:00:00 2001 From: Jordan Brown Date: Tue, 3 Dec 2019 09:58:49 +0000 Subject: [PATCH 064/158] Bump package version --- django_dbq/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/django_dbq/__init__.py b/django_dbq/__init__.py index c68196d..67bc602 100644 --- a/django_dbq/__init__.py +++ b/django_dbq/__init__.py @@ -1 +1 @@ -__version__ = "1.2.0" +__version__ = "1.3.0" From 81d85b1a61c4f029d6a4e702e02705fc48535d7b Mon Sep 17 00:00:00 2001 From: Harley Date: Wed, 19 Feb 2020 12:38:32 +0000 Subject: [PATCH 065/158] Fiz timezone unaware datetime --- django_dbq/models.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/django_dbq/models.py b/django_dbq/models.py index 65e7150..24ee6aa 100644 --- a/django_dbq/models.py +++ b/django_dbq/models.py @@ -1,4 +1,5 @@ from django.db import models +from django.utils import timezone from django.utils.module_loading import import_string from django_dbq.tasks import ( get_next_task_name, @@ -54,7 +55,7 @@ def delete_old(self): Delete all jobs older than DELETE_JOBS_AFTER_HOURS """ delete_jobs_in_states = [Job.STATES.FAILED, Job.STATES.COMPLETE] - delete_jobs_created_before = datetime.datetime.utcnow() - datetime.timedelta( + delete_jobs_created_before = timezone.now() - datetime.timedelta( hours=DELETE_JOBS_AFTER_HOURS ) logger.info( From 2e7e503cf663842acafa0d8e95c5fa437005cb7f Mon Sep 17 00:00:00 2001 From: Jake Howard Date: Thu, 20 Feb 2020 09:30:04 +0000 Subject: [PATCH 066/158] Bump version to 1.3.1 --- django_dbq/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/django_dbq/__init__.py b/django_dbq/__init__.py index 67bc602..9c73af2 100644 --- a/django_dbq/__init__.py +++ b/django_dbq/__init__.py @@ -1 +1 @@ -__version__ = "1.3.0" +__version__ = "1.3.1" From 2bdd71753aee21e4578c481a5089a2017969c112 Mon Sep 17 00:00:00 2001 From: Jake Howard Date: Mon, 5 Oct 2020 16:53:08 +0100 Subject: [PATCH 067/158] Add GitHub actions config --- .github/workflows/ci.yml | 53 ++++++++++++++++++++++++++++++++++++++++ .travis.yml | 30 ----------------------- test-requirements.txt | 1 + 3 files changed, 54 insertions(+), 30 deletions(-) create mode 100644 .github/workflows/ci.yml delete mode 100644 .travis.yml diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..6eac603 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,53 @@ +name: CI + +on: [pull_request] + +jobs: + build: + + runs-on: ubuntu-18.04 + + strategy: + matrix: + python: [3.6, 3.7, 3.8] + django: [2.2, 3.0, 3.1] + database_url: + - postgres://runner:password@localhost/project + - mysql://runner:password@localhost/project + + services: + postgres: + image: postgres:11 + ports: + - 5432:5432 + env: + POSTGRES_DB: project + POSTGRES_USER: runner + POSTGRES_PASSWORD: password + + mysql: + image: mysql:5.6 + ports: + - 3306:3306 + env: + MYSQL_DATABASE: project + MYSQL_USER: runner + MYSQL_PASSWORD: password + + env: + DATABASE_URL: ${{ matrix.python }} + + steps: + - uses: actions/checkout@v2 + - name: Set up Python ${{ matrix.python }} + uses: actions/setup-python@v2 + with: + python-version: ${{ matrix.python }} + - name: Install dependencies + run: pip install -r test-requirements.txt + - name: Install Django + run: pip install -U django==${{ matrix.django }} + - name: Run tests + run: python manage.py test + - name: Run black + run: black --check django_dbq diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index d837d65..0000000 --- a/.travis.yml +++ /dev/null @@ -1,30 +0,0 @@ -language: python -sudo: false -services: - - mysql - - postgresql -python: -- '3.5' -- '3.6' -- '3.7' -- '3.8' -env: -- DJANGO_VERSION=1.11 DATABASE_URL=postgres://postgres@127.0.0.1/dbq -- DJANGO_VERSION=2.2 DATABASE_URL=postgres://postgres@127.0.0.1/dbq -- DJANGO_VERSION=1.11 DATABASE_URL=mysql://root@127.0.0.1/dbq -- DJANGO_VERSION=2.2 DATABASE_URL=mysql://root@127.0.0.1/dbq -install: -- pip install -r test-requirements.txt -- pip install -U django==$DJANGO_VERSION -- if [ "$TRAVIS_PYTHON_VERSION" != "3.5" ]; then pip install black==19.10b0; fi -script: - - if [ "$TRAVIS_PYTHON_VERSION" != "3.5" ]; then black --check django_dbq; fi - - python manage.py test -deploy: - provider: pypi - user: dabapps - password: - secure: YkRDJO+QK2Rr3AJwmxoghTWCCTZFLfXUlqky/my6g7oeMI5Q/F2WzNBmvr84v069fckobHhlN4hhH/JFEaRnCNuYmPhFdiNQh5M5cP/qhUqqh7LsMiX5mJSfM6yCp+rAL6F+yb5r59t3IQKmXKiFzRm/AuS4nHINDFHXwaPjWTw= - on: - tags: true - repo: dabapps/django-db-queue diff --git a/test-requirements.txt b/test-requirements.txt index 3499577..dd2ea67 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -4,3 +4,4 @@ freezegun==0.3.12 mock==3.0.5 dj-database-url==0.5.0 psycopg2==2.8.4 +black==19.10b0 From 3aa62befbe2c803428d13ca8954c38ce7bc1d279 Mon Sep 17 00:00:00 2001 From: Jake Howard Date: Mon, 5 Oct 2020 17:02:19 +0100 Subject: [PATCH 068/158] Remove 3.1 support for now Will add back later --- .github/workflows/ci.yml | 2 +- requirements.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 6eac603..350b387 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -10,7 +10,7 @@ jobs: strategy: matrix: python: [3.6, 3.7, 3.8] - django: [2.2, 3.0, 3.1] + django: [2.2] database_url: - postgres://runner:password@localhost/project - mysql://runner:password@localhost/project diff --git a/requirements.txt b/requirements.txt index 8ed66d8..96ce7c6 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,5 @@ django-model-utils==2.3.1 django-uuidfield==0.5.0 jsonfield==1.0.3 -Django>=1.8 +Django>=2.2<3.0 simplesignals==0.3.0 From 51a6d9a8b8bd295502985ac108a850a056de4eb5 Mon Sep 17 00:00:00 2001 From: Jake Howard Date: Mon, 5 Oct 2020 17:04:18 +0100 Subject: [PATCH 069/158] Correctly set db url --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 350b387..bc10543 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -35,7 +35,7 @@ jobs: MYSQL_PASSWORD: password env: - DATABASE_URL: ${{ matrix.python }} + DATABASE_URL: ${{ matrix.database_url }} steps: - uses: actions/checkout@v2 From 84fa09440e06eb9895ff0f3bac2475b56d2a918f Mon Sep 17 00:00:00 2001 From: Jake Howard Date: Mon, 5 Oct 2020 17:04:35 +0100 Subject: [PATCH 070/158] Specify in setup.py that it requires 3.6+ --- setup.py | 1 + 1 file changed, 1 insertion(+) diff --git a/setup.py b/setup.py index f5d5a90..7d06963 100644 --- a/setup.py +++ b/setup.py @@ -84,4 +84,5 @@ def get_package_data(package): package_data=get_package_data(package), install_requires=install_requires, classifiers=[], + python_requires=">=3.6" ) From a4fdd455037293c52d5ab5f60304c39e749bd2dd Mon Sep 17 00:00:00 2001 From: Jake Howard Date: Mon, 5 Oct 2020 17:06:57 +0100 Subject: [PATCH 071/158] Use 127.0.0.1 rather than localhost To MySQL, they're different --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index bc10543..87ec09f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -13,7 +13,7 @@ jobs: django: [2.2] database_url: - postgres://runner:password@localhost/project - - mysql://runner:password@localhost/project + - mysql://runner:password@127.0.0.1/project services: postgres: From 2b595b83b5fd7447e69d6e07a0270b6fda2a5096 Mon Sep 17 00:00:00 2001 From: Jake Howard Date: Mon, 5 Oct 2020 17:16:09 +0100 Subject: [PATCH 072/158] Use GitHub Actions' built-in mysql --- .github/workflows/ci.yml | 13 +++---------- 1 file changed, 3 insertions(+), 10 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 87ec09f..44f01fe 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -13,7 +13,7 @@ jobs: django: [2.2] database_url: - postgres://runner:password@localhost/project - - mysql://runner:password@127.0.0.1/project + - mysql://root:root@127.0.0.1/project services: postgres: @@ -25,19 +25,12 @@ jobs: POSTGRES_USER: runner POSTGRES_PASSWORD: password - mysql: - image: mysql:5.6 - ports: - - 3306:3306 - env: - MYSQL_DATABASE: project - MYSQL_USER: runner - MYSQL_PASSWORD: password - env: DATABASE_URL: ${{ matrix.database_url }} steps: + - name: Start MySQL + run: sudo systemctl start mysql.service - uses: actions/checkout@v2 - name: Set up Python ${{ matrix.python }} uses: actions/setup-python@v2 From c503c30003823b74bd39bf34caab6ae905450fa2 Mon Sep 17 00:00:00 2001 From: Jake Howard Date: Fri, 9 Oct 2020 11:04:37 +0100 Subject: [PATCH 073/158] Specify it requires Django>=2.2 --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 7d06963..25cc24a 100644 --- a/setup.py +++ b/setup.py @@ -17,7 +17,7 @@ license = "BSD" install_requires = [ "jsonfield==2.0.2", - "Django>=1.7", + "Django>=2.2", "simplesignals==0.3.0", ] From 97312b55525f81ecba3c0df1d6c2e55fa217ce83 Mon Sep 17 00:00:00 2001 From: Jamie Matthews Date: Mon, 5 Apr 2021 20:18:38 +0100 Subject: [PATCH 074/158] Make tests run on sqlite by default --- testsettings.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/testsettings.py b/testsettings.py index 6cb7863..516bfcb 100644 --- a/testsettings.py +++ b/testsettings.py @@ -1,8 +1,11 @@ import os import dj_database_url + +DATABASE_URL = os.environ.get("DATABASE_URL", "sqlite:///:memory:") + DATABASES = { - "default": dj_database_url.parse(os.environ["DATABASE_URL"]), + "default": dj_database_url.parse(DATABASE_URL), } INSTALLED_APPS = ("django_dbq",) From 40ebd8176d48040683edc66f0dfe80e1fc0c2bb7 Mon Sep 17 00:00:00 2001 From: Jamie Matthews Date: Mon, 5 Apr 2021 20:20:02 +0100 Subject: [PATCH 075/158] Remove unnecessary setting --- testsettings.py | 9 --------- 1 file changed, 9 deletions(-) diff --git a/testsettings.py b/testsettings.py index 516bfcb..040eade 100644 --- a/testsettings.py +++ b/testsettings.py @@ -10,15 +10,6 @@ INSTALLED_APPS = ("django_dbq",) -MIDDLEWARE_CLASSES = ( - "django.contrib.sessions.middleware.SessionMiddleware", - "django.middleware.common.CommonMiddleware", - "django.middleware.csrf.CsrfViewMiddleware", - "django.contrib.auth.middleware.AuthenticationMiddleware", - "django.contrib.messages.middleware.MessageMiddleware", - "django.middleware.clickjacking.XFrameOptionsMiddleware", -) - SECRET_KEY = "abcde12345" LOGGING = { From ae52e3b824791700f0f88698430129e88cf5efc2 Mon Sep 17 00:00:00 2001 From: Jamie Matthews Date: Mon, 5 Apr 2021 20:41:07 +0100 Subject: [PATCH 076/158] Remove jsonfield dependency and switch to enum choices --- django_dbq/migrations/0001_initial.py | 43 +++++++++---------- .../migrations/0002_auto_20151016_1027.py | 15 ------- .../migrations/0003_auto_20180713_1000.py | 23 ---------- django_dbq/models.py | 15 ++----- 4 files changed, 24 insertions(+), 72 deletions(-) delete mode 100644 django_dbq/migrations/0002_auto_20151016_1027.py delete mode 100644 django_dbq/migrations/0003_auto_20180713_1000.py diff --git a/django_dbq/migrations/0001_initial.py b/django_dbq/migrations/0001_initial.py index 0ec8c73..6ed4e08 100644 --- a/django_dbq/migrations/0001_initial.py +++ b/django_dbq/migrations/0001_initial.py @@ -1,15 +1,13 @@ -# -*- coding: utf-8 -*- -from __future__ import unicode_literals +# Generated by Django 3.2rc1 on 2021-04-05 14:31 -from django.db import models, migrations -import jsonfield.fields +from django.db import migrations, models import uuid -from django.db.models import UUIDField - class Migration(migrations.Migration): + initial = True + dependencies = [] operations = [ @@ -18,38 +16,39 @@ class Migration(migrations.Migration): fields=[ ( "id", - UUIDField( - serialize=False, - editable=False, + models.UUIDField( default=uuid.uuid4, + editable=False, primary_key=True, + serialize=False, ), ), - ("created", models.DateTimeField(db_index=True, auto_now_add=True)), + ("created", models.DateTimeField(auto_now_add=True, db_index=True)), ("modified", models.DateTimeField(auto_now=True)), ("name", models.CharField(max_length=100)), ( "state", models.CharField( - db_index=True, - max_length=20, - default="NEW", choices=[ - ("NEW", "NEW"), - ("READY", "READY"), - ("PROCESSING", "PROCESSING"), - ("FAILED", "FAILED"), - ("COMPLETE", "COMPLETE"), + ("NEW", "New"), + ("READY", "Ready"), + ("PROCESSING", "Processing"), + ("FAILED", "Failed"), + ("COMPLETE", "Complete"), ], + db_index=True, + default="NEW", + max_length=20, ), ), - ("next_task", models.CharField(max_length=100, blank=True)), - ("workspace", jsonfield.fields.JSONField(null=True)), + ("next_task", models.CharField(blank=True, max_length=100)), + ("workspace", models.JSONField(null=True)), ( "queue_name", - models.CharField(db_index=True, max_length=20, default="default"), + models.CharField(db_index=True, default="default", max_length=20), ), + ("priority", models.SmallIntegerField(db_index=True, default=0)), ], - options={"ordering": ["-created"],}, + options={"ordering": ["-priority", "created"],}, ), ] diff --git a/django_dbq/migrations/0002_auto_20151016_1027.py b/django_dbq/migrations/0002_auto_20151016_1027.py deleted file mode 100644 index d0a72c4..0000000 --- a/django_dbq/migrations/0002_auto_20151016_1027.py +++ /dev/null @@ -1,15 +0,0 @@ -# -*- coding: utf-8 -*- -from __future__ import unicode_literals - -from django.db import models, migrations - - -class Migration(migrations.Migration): - - dependencies = [ - ("django_dbq", "0001_initial"), - ] - - operations = [ - migrations.AlterModelOptions(name="job", options={"ordering": ["created"]},), - ] diff --git a/django_dbq/migrations/0003_auto_20180713_1000.py b/django_dbq/migrations/0003_auto_20180713_1000.py deleted file mode 100644 index 4d959f3..0000000 --- a/django_dbq/migrations/0003_auto_20180713_1000.py +++ /dev/null @@ -1,23 +0,0 @@ -# -*- coding: utf-8 -*- -# Generated by Django 1.11 on 2018-07-13 10:00 -from __future__ import unicode_literals - -from django.db import migrations, models - - -class Migration(migrations.Migration): - - dependencies = [ - ("django_dbq", "0002_auto_20151016_1027"), - ] - - operations = [ - migrations.AlterModelOptions( - name="job", options={"ordering": ["-priority", "created"]}, - ), - migrations.AddField( - model_name="job", - name="priority", - field=models.SmallIntegerField(db_index=True, default=0), - ), - ] diff --git a/django_dbq/models.py b/django_dbq/models.py index 24ee6aa..0a9d51b 100644 --- a/django_dbq/models.py +++ b/django_dbq/models.py @@ -6,8 +6,7 @@ get_failure_hook_name, get_creation_hook_name, ) -from jsonfield import JSONField -from django.db.models import UUIDField, Count +from django.db.models import JSONField, UUIDField, Count, TextChoices import datetime import logging import uuid @@ -74,27 +73,19 @@ def to_process(self, queue_name): class Job(models.Model): - class STATES: + class STATES(TextChoices): NEW = "NEW" READY = "READY" PROCESSING = "PROCESSING" FAILED = "FAILED" COMPLETE = "COMPLETE" - STATE_CHOICES = [ - (STATES.NEW, "NEW"), - (STATES.READY, "READY"), - (STATES.PROCESSING, "PROCESSING"), - (STATES.FAILED, "FAILED"), - (STATES.COMPLETE, "COMPLETE"), - ] - id = UUIDField(primary_key=True, default=uuid.uuid4, editable=False) created = models.DateTimeField(auto_now_add=True, db_index=True) modified = models.DateTimeField(auto_now=True) name = models.CharField(max_length=100) state = models.CharField( - max_length=20, choices=STATE_CHOICES, default=STATES.NEW, db_index=True + max_length=20, choices=STATES.choices, default=STATES.NEW, db_index=True ) next_task = models.CharField(max_length=100, blank=True) workspace = JSONField(null=True) From 7df7d404896c339f0f338610cd6d972daf63bbc5 Mon Sep 17 00:00:00 2001 From: Jamie Matthews Date: Mon, 5 Apr 2021 20:48:23 +0100 Subject: [PATCH 077/158] Remove dependency on simplesignals --- django_dbq/management/commands/worker.py | 24 +++++++++++++++++------- django_dbq/tests.py | 14 +++++++------- 2 files changed, 24 insertions(+), 14 deletions(-) diff --git a/django_dbq/management/commands/worker.py b/django_dbq/management/commands/worker.py index a09a0a9..e67c425 100644 --- a/django_dbq/management/commands/worker.py +++ b/django_dbq/management/commands/worker.py @@ -3,9 +3,9 @@ from django.utils import timezone from django.utils.module_loading import import_string from django_dbq.models import Job -from simplesignals.process import WorkerProcessBase from time import sleep import logging +import signal logger = logging.getLogger(__name__) @@ -74,17 +74,27 @@ def process_job(queue_name): raise -class Worker(WorkerProcessBase): - - process_title = "jobworker" - +class Worker: def __init__(self, name, rate_limit_in_seconds): self.queue_name = name self.rate_limit_in_seconds = rate_limit_in_seconds + self.alive = True self.last_job_finished = None - super(Worker, self).__init__() + self.init_signals() + + def init_signals(self): + signal.signal(signal.SIGINT, self.shutdown) + signal.signal(signal.SIGQUIT, self.shutdown) + signal.signal(signal.SIGTERM, self.shutdown) + + def shutdown(self): + self.alive = False + + def run(self): + while self.alive: + self.process_job() - def do_work(self): + def process_job(self): sleep(1) if ( self.last_job_finished diff --git a/django_dbq/tests.py b/django_dbq/tests.py index f354551..9012b44 100644 --- a/django_dbq/tests.py +++ b/django_dbq/tests.py @@ -148,7 +148,7 @@ def test_queue_depth_for_queue_with_zero_jobs(self): @freezegun.freeze_time() @mock.patch("django_dbq.management.commands.worker.sleep") @mock.patch("django_dbq.management.commands.worker.process_job") -class WorkerProcessDoWorkTestCase(TestCase): +class WorkerProcessProcessJobTestCase(TestCase): def setUp(self): super().setUp() self.MockWorker = mock.MagicMock() @@ -156,17 +156,17 @@ def setUp(self): self.MockWorker.rate_limit_in_seconds = 5 self.MockWorker.last_job_finished = None - def test_do_work_no_previous_job_run(self, mock_process_job, mock_sleep): - Worker.do_work(self.MockWorker) + def test_process_job_no_previous_job_run(self, mock_process_job, mock_sleep): + Worker.process_job(self.MockWorker) self.assertEqual(mock_sleep.call_count, 1) self.assertEqual(mock_process_job.call_count, 1) self.assertEqual(self.MockWorker.last_job_finished, timezone.now()) - def test_do_work_previous_job_too_soon(self, mock_process_job, mock_sleep): + def test_process_job_previous_job_too_soon(self, mock_process_job, mock_sleep): self.MockWorker.last_job_finished = timezone.now() - timezone.timedelta( seconds=2 ) - Worker.do_work(self.MockWorker) + Worker.process_job(self.MockWorker) self.assertEqual(mock_sleep.call_count, 1) self.assertEqual(mock_process_job.call_count, 0) self.assertEqual( @@ -174,11 +174,11 @@ def test_do_work_previous_job_too_soon(self, mock_process_job, mock_sleep): timezone.now() - timezone.timedelta(seconds=2), ) - def test_do_work_previous_job_long_time_ago(self, mock_process_job, mock_sleep): + def test_process_job_previous_job_long_time_ago(self, mock_process_job, mock_sleep): self.MockWorker.last_job_finished = timezone.now() - timezone.timedelta( seconds=7 ) - Worker.do_work(self.MockWorker) + Worker.process_job(self.MockWorker) self.assertEqual(mock_sleep.call_count, 1) self.assertEqual(mock_process_job.call_count, 1) self.assertEqual(self.MockWorker.last_job_finished, timezone.now()) From 3b5f5b818d88f317d66fd17c9b7fb49828e7d258 Mon Sep 17 00:00:00 2001 From: Jamie Matthews Date: Mon, 5 Apr 2021 20:52:28 +0100 Subject: [PATCH 078/158] Remove requirements.txt and stuff from install_requires --- requirements.txt | 5 ----- setup.py | 4 +--- test-requirements.txt | 1 - 3 files changed, 1 insertion(+), 9 deletions(-) delete mode 100644 requirements.txt diff --git a/requirements.txt b/requirements.txt deleted file mode 100644 index 96ce7c6..0000000 --- a/requirements.txt +++ /dev/null @@ -1,5 +0,0 @@ -django-model-utils==2.3.1 -django-uuidfield==0.5.0 -jsonfield==1.0.3 -Django>=2.2<3.0 -simplesignals==0.3.0 diff --git a/setup.py b/setup.py index 25cc24a..5bb0f76 100644 --- a/setup.py +++ b/setup.py @@ -16,9 +16,7 @@ author_email = "contact@dabapps.com" license = "BSD" install_requires = [ - "jsonfield==2.0.2", - "Django>=2.2", - "simplesignals==0.3.0", + "Django>=3.1", ] long_description = """Simple database-backed job queue system""" diff --git a/test-requirements.txt b/test-requirements.txt index dd2ea67..249c8b3 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -1,4 +1,3 @@ --r requirements.txt mysqlclient==1.4.6 freezegun==0.3.12 mock==3.0.5 From 9f78d383da928f5cac92d863f17476f1241626d1 Mon Sep 17 00:00:00 2001 From: Jamie Matthews Date: Mon, 5 Apr 2021 20:54:40 +0100 Subject: [PATCH 079/158] Add supported django versions and sqlite url to CI --- .github/workflows/ci.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 44f01fe..2b04815 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -9,11 +9,12 @@ jobs: strategy: matrix: - python: [3.6, 3.7, 3.8] - django: [2.2] + python: [3.6, 3.7, 3.8, 3.9] + django: [3.1, 3.2rc1] database_url: - postgres://runner:password@localhost/project - mysql://root:root@127.0.0.1/project + - sqlite:///:memory: services: postgres: From 85acafdd031fa42db0893bd16d3c84fd7c8bb6a7 Mon Sep 17 00:00:00 2001 From: Jamie Matthews Date: Mon, 5 Apr 2021 20:56:24 +0100 Subject: [PATCH 080/158] Remove unused create_job management command --- README.md | 9 --- django_dbq/management/commands/create_job.py | 59 -------------------- django_dbq/tests.py | 28 ---------- 3 files changed, 96 deletions(-) delete mode 100644 django_dbq/management/commands/create_job.py diff --git a/README.md b/README.md index 75969ad..bfe5034 100644 --- a/README.md +++ b/README.md @@ -209,15 +209,6 @@ jobs from the database which are in state `COMPLETE` or `FAILED` and were created more than 24 hours ago. This could be run, for example, as a cron task, to ensure the jobs table remains at a reasonable size. -##### manage.py create_job -For debugging/development purposes, a simple management command is supplied to create jobs: - - manage.py create_job --queue_name 'my_queue_name' --workspace '{"key": "value"}' - -The `workspace` flag is optional. If supplied, it must be a valid JSON string. - -`queue_name` is optional and defaults to `default` - ##### manage.py worker To start a worker: diff --git a/django_dbq/management/commands/create_job.py b/django_dbq/management/commands/create_job.py deleted file mode 100644 index c060218..0000000 --- a/django_dbq/management/commands/create_job.py +++ /dev/null @@ -1,59 +0,0 @@ -from django.conf import settings -from django.core.management.base import BaseCommand, CommandError -from django_dbq.models import Job -import json -import logging - - -logger = logging.getLogger(__name__) - - -class Command(BaseCommand): - - help = "Create a job" - args = "" - - def add_arguments(self, parser): - parser.add_argument("args", nargs="+") - parser.add_argument( - "--workspace", - action="store_true", - dest="workspace", - default=None, - help="JSON-formatted initial commandworkspace.", - ) - parser.add_argument( - "--queue_name", - action="store_true", - dest="queue_name", - default=None, - help="A specific queue to add this job to", - ) - - def handle(self, *args, **options): - if len(args) != 1: - raise CommandError("Please supply a single job name") - - name = args[0] - if name not in settings.JOBS: - raise CommandError('"%s" is not a valid job name' % name) - - workspace = options["workspace"] - if workspace: - workspace = json.loads(workspace) - - queue_name = options["queue_name"] - - kwargs = { - "name": name, - "workspace": workspace, - } - - if queue_name: - kwargs["queue_name"] = queue_name - - job = Job.objects.create(**kwargs) - self.stdout.write( - 'Created job: "%s", id=%s for queue "%s"' - % (job.name, job.pk, queue_name if queue_name else "default") - ) diff --git a/django_dbq/tests.py b/django_dbq/tests.py index 9012b44..41ec958 100644 --- a/django_dbq/tests.py +++ b/django_dbq/tests.py @@ -37,34 +37,6 @@ def creation_hook(job): job.workspace["output"] = "creation hook ran" -@override_settings(JOBS={"testjob": {"tasks": ["a"]}}) -class JobManagementCommandTestCase(TestCase): - def test_create_job(self): - call_command("create_job", "testjob", stdout=StringIO()) - job = Job.objects.get() - self.assertEqual(job.name, "testjob") - self.assertEqual(job.queue_name, "default") - - def test_create_job_with_workspace(self): - workspace = '{"test": "test"}' - call_command("create_job", "testjob", workspace=workspace, stdout=StringIO()) - job = Job.objects.get() - self.assertEqual(job.workspace, {"test": "test"}) - - def test_create_job_with_queue_name(self): - call_command("create_job", "testjob", queue_name="lol", stdout=StringIO()) - job = Job.objects.get() - self.assertEqual(job.name, "testjob") - self.assertEqual(job.queue_name, "lol") - - def test_errors_raised_correctly(self): - with self.assertRaises(CommandError): - call_command("create_job", stdout=StringIO()) - - with self.assertRaises(CommandError): - call_command("create_job", "some_other_job", stdout=StringIO()) - - @override_settings(JOBS={"testjob": {"tasks": ["a"]}}) class WorkerManagementCommandTestCase(TestCase): def test_worker_no_args(self): From dd283b353402268851377268e515f6c8e3262147 Mon Sep 17 00:00:00 2001 From: Jamie Matthews Date: Mon, 5 Apr 2021 20:58:29 +0100 Subject: [PATCH 081/158] Remove unused JobSerializer --- django_dbq/serializers.py | 29 ----------------------------- 1 file changed, 29 deletions(-) delete mode 100644 django_dbq/serializers.py diff --git a/django_dbq/serializers.py b/django_dbq/serializers.py deleted file mode 100644 index 12b40d5..0000000 --- a/django_dbq/serializers.py +++ /dev/null @@ -1,29 +0,0 @@ -from django.conf import settings -from django_dbq.models import Job -from rest_framework import serializers -import json - - -class JobSerializer(serializers.Serializer): - name = serializers.ChoiceField() - created = serializers.DateTimeField(read_only=True) - modified = serializers.DateTimeField(read_only=True) - state = serializers.CharField(read_only=True) - workspace = serializers.WritableField(required=False) - url = serializers.HyperlinkedIdentityField(view_name="job_detail") - - def __init__(self, *args, **kwargs): - super(JobSerializer, self).__init__(*args, **kwargs) - self.fields["name"].choices = ((key, key) for key in settings.JOBS) - - def validate_workspace(self, attrs, source): - workspace = attrs.get("workspace") - if workspace and isinstance(workspace, basestring): - try: - attrs["workspace"] = json.loads(workspace) - except ValueError: - raise serializers.ValidationError("Invalid JSON") - return attrs - - def restore_object(self, attrs, instance=None): - return Job(**attrs) From 9efebb2c818474a0121fe79fdc693b74d8f7971a Mon Sep 17 00:00:00 2001 From: Jamie Matthews Date: Mon, 5 Apr 2021 20:59:26 +0100 Subject: [PATCH 082/158] Remove unused python 2 compat import --- django_dbq/tests.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/django_dbq/tests.py b/django_dbq/tests.py index 41ec958..33df2f5 100644 --- a/django_dbq/tests.py +++ b/django_dbq/tests.py @@ -10,10 +10,7 @@ from django_dbq.management.commands.worker import process_job, Worker from django_dbq.models import Job -try: - from StringIO import StringIO -except ImportError: - from io import StringIO +from io import StringIO def test_task(job=None): From 54664b29d1b5cf152cc0d473aef5c688cf64a0ec Mon Sep 17 00:00:00 2001 From: Jamie Matthews Date: Mon, 5 Apr 2021 21:08:37 +0100 Subject: [PATCH 083/158] Add upgrade notes to README --- README.md | 20 ++++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index bfe5034..c404c58 100644 --- a/README.md +++ b/README.md @@ -1,17 +1,14 @@ django-db-queue ========== -[![Build Status](https://travis-ci.org/dabapps/django-db-queue.svg)](https://travis-ci.org/dabapps/django-db-queue) [![pypi release](https://img.shields.io/pypi/v/django-db-queue.svg)](https://pypi.python.org/pypi/django-db-queue) -Simple databased-backed job queue. Jobs are defined in your settings, and are processed by management commands. +Simple database-backed job queue. Jobs are defined in your settings, and are processed by management commands. Asynchronous tasks are run via a *job queue*. This system is designed to support multi-step job workflows. Supported and tested against: -- Django 1.11 and 2.2 -- Python 3.5, 3.6, 3.7 and 3.8 - -This package may still work with older versions of Django and Python but they aren't explicitly supported. +- Django 3.1 and 3.2 +- Python 3.6, 3.7, 3.8 and 3.9 ## Getting Started @@ -28,6 +25,17 @@ Add `django_dbq` to your installed apps 'django_dbq', ) +### Upgrading from 1.x to 2.x + +This library underwent significant changes in its 2.x release, which meant migrating automatically was not practical. Before upgrading, you should open a database shell and run the following commands. Note that this will **delete all your existing jobs**, so you should ensure all jobs are completed before proceeding, and make sure you don't need any data at all from the jobs table. + +``` +DROP TABLE django_dbq_job; +DELETE FROM django_migrations WHERE app='django_dbq'; +``` + +Then, run `python manage.py migrate` to recreate the jobs table. + ### Describe your job In e.g. project.common.jobs: From ec2b86010f6e8cb33b561ec4f2594d74e86341fe Mon Sep 17 00:00:00 2001 From: Jamie Matthews Date: Mon, 5 Apr 2021 21:08:58 +0100 Subject: [PATCH 084/158] Bump version --- django_dbq/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/django_dbq/__init__.py b/django_dbq/__init__.py index 9c73af2..8c0d5d5 100644 --- a/django_dbq/__init__.py +++ b/django_dbq/__init__.py @@ -1 +1 @@ -__version__ = "1.3.1" +__version__ = "2.0.0" From 27fd0962340eb402c847434824114367e9ef432f Mon Sep 17 00:00:00 2001 From: Jamie Matthews Date: Mon, 5 Apr 2021 21:09:39 +0100 Subject: [PATCH 085/158] Add pypi publishing workflow --- .github/workflows/pypi.yml | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) create mode 100644 .github/workflows/pypi.yml diff --git a/.github/workflows/pypi.yml b/.github/workflows/pypi.yml new file mode 100644 index 0000000..fd2d1c3 --- /dev/null +++ b/.github/workflows/pypi.yml @@ -0,0 +1,26 @@ +name: Upload Python Package + +on: + release: + types: [created] + +jobs: + deploy: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - name: Set up Python + uses: actions/setup-python@v2 + with: + python-version: '3.8' + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install setuptools wheel twine + - name: Build and publish + env: + TWINE_USERNAME: ${{ secrets.PYPI_USERNAME }} + TWINE_PASSWORD: ${{ secrets.PYPI_PASSWORD }} + run: | + python setup.py sdist bdist_wheel + twine upload dist/* From 72c2994e4b57a2d3dbb5404d4068e6d8658b74ec Mon Sep 17 00:00:00 2001 From: Jamie Matthews Date: Mon, 5 Apr 2021 21:16:25 +0100 Subject: [PATCH 086/158] Quote YAML properly --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 2b04815..bcb2059 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -14,7 +14,7 @@ jobs: database_url: - postgres://runner:password@localhost/project - mysql://root:root@127.0.0.1/project - - sqlite:///:memory: + - `sqlite:///:memory:` services: postgres: From 30444b37e95ea251d3156657e4c3040f9a7f7285 Mon Sep 17 00:00:00 2001 From: Jamie Matthews Date: Mon, 5 Apr 2021 21:17:11 +0100 Subject: [PATCH 087/158] Really quote YAML properly --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index bcb2059..555c241 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -14,7 +14,7 @@ jobs: database_url: - postgres://runner:password@localhost/project - mysql://root:root@127.0.0.1/project - - `sqlite:///:memory:` + - 'sqlite:///:memory:' services: postgres: From f809529790bf91f78a4160f8057345884b76b5e6 Mon Sep 17 00:00:00 2001 From: Jamie Matthews Date: Mon, 5 Apr 2021 21:22:45 +0100 Subject: [PATCH 088/158] Use newer way of determining if instance is new, saves a query --- django_dbq/models.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/django_dbq/models.py b/django_dbq/models.py index 0a9d51b..fe1cffa 100644 --- a/django_dbq/models.py +++ b/django_dbq/models.py @@ -98,9 +98,7 @@ class Meta: objects = JobManager() def save(self, *args, **kwargs): - is_new = not Job.objects.filter(pk=self.pk).exists() - - if is_new: + if self._state.adding: self.next_task = get_next_task_name(self.name) self.workspace = self.workspace or {} From 9e008f0559515d53b72203629ad7275c8ca098c3 Mon Sep 17 00:00:00 2001 From: Jamie Matthews Date: Mon, 5 Apr 2021 21:22:56 +0100 Subject: [PATCH 089/158] Remove python2 compat syntax --- django_dbq/models.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/django_dbq/models.py b/django_dbq/models.py index fe1cffa..2f51a78 100644 --- a/django_dbq/models.py +++ b/django_dbq/models.py @@ -110,7 +110,7 @@ def save(self, *args, **kwargs): ) return # cancel the save - return super(Job, self).save(*args, **kwargs) + return super().save(*args, **kwargs) def update_next_task(self): self.next_task = get_next_task_name(self.name, self.next_task) or "" From ec3822faa554ce5d9c757fa3b0158941c44a58c0 Mon Sep 17 00:00:00 2001 From: Jamie Matthews Date: Tue, 6 Apr 2021 11:55:03 +0100 Subject: [PATCH 090/158] Add note on Django version support --- README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.md b/README.md index c404c58..917e600 100644 --- a/README.md +++ b/README.md @@ -36,6 +36,8 @@ DELETE FROM django_migrations WHERE app='django_dbq'; Then, run `python manage.py migrate` to recreate the jobs table. +Note that version 2.x only supports Django 3.1 or newer. If you need support for Django 2.2, please stick with the latest 1.x release. + ### Describe your job In e.g. project.common.jobs: From 31e0e0065c87a2b13cdef1d297d3bc607375e50e Mon Sep 17 00:00:00 2001 From: Jamie Matthews Date: Tue, 6 Apr 2021 11:57:08 +0100 Subject: [PATCH 091/158] Django 3.2 final --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 555c241..7be073c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -10,7 +10,7 @@ jobs: strategy: matrix: python: [3.6, 3.7, 3.8, 3.9] - django: [3.1, 3.2rc1] + django: [3.1, 3.2] database_url: - postgres://runner:password@localhost/project - mysql://root:root@127.0.0.1/project From 908823c693b2bfe1a2a945a4575b454be43174a7 Mon Sep 17 00:00:00 2001 From: Jamie Matthews Date: Fri, 14 May 2021 09:51:15 +0100 Subject: [PATCH 092/158] Ensure signal handler takes correct arguments --- django_dbq/management/commands/worker.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/django_dbq/management/commands/worker.py b/django_dbq/management/commands/worker.py index e67c425..92e72e4 100644 --- a/django_dbq/management/commands/worker.py +++ b/django_dbq/management/commands/worker.py @@ -87,7 +87,7 @@ def init_signals(self): signal.signal(signal.SIGQUIT, self.shutdown) signal.signal(signal.SIGTERM, self.shutdown) - def shutdown(self): + def shutdown(self, signum, frame): self.alive = False def run(self): From f5bb1dc1a47bafb88342db8c54c0d6ca27dcd432 Mon Sep 17 00:00:00 2001 From: Jamie Matthews Date: Wed, 18 Aug 2021 08:50:38 +0100 Subject: [PATCH 093/158] Reinstate migrations for upgrading from 1.x to 2.0 --- README.md | 9 ---- django_dbq/migrations/0001_initial.py | 42 +++++++++---------- .../migrations/0002_auto_20151016_1027.py | 15 +++++++ .../migrations/0003_auto_20180713_1000.py | 23 ++++++++++ .../migrations/0004_auto_20210818_0247.py | 32 ++++++++++++++ 5 files changed, 91 insertions(+), 30 deletions(-) create mode 100644 django_dbq/migrations/0002_auto_20151016_1027.py create mode 100644 django_dbq/migrations/0003_auto_20180713_1000.py create mode 100644 django_dbq/migrations/0004_auto_20210818_0247.py diff --git a/README.md b/README.md index 917e600..5a55d00 100644 --- a/README.md +++ b/README.md @@ -27,15 +27,6 @@ Add `django_dbq` to your installed apps ### Upgrading from 1.x to 2.x -This library underwent significant changes in its 2.x release, which meant migrating automatically was not practical. Before upgrading, you should open a database shell and run the following commands. Note that this will **delete all your existing jobs**, so you should ensure all jobs are completed before proceeding, and make sure you don't need any data at all from the jobs table. - -``` -DROP TABLE django_dbq_job; -DELETE FROM django_migrations WHERE app='django_dbq'; -``` - -Then, run `python manage.py migrate` to recreate the jobs table. - Note that version 2.x only supports Django 3.1 or newer. If you need support for Django 2.2, please stick with the latest 1.x release. ### Describe your job diff --git a/django_dbq/migrations/0001_initial.py b/django_dbq/migrations/0001_initial.py index 6ed4e08..d5114d3 100644 --- a/django_dbq/migrations/0001_initial.py +++ b/django_dbq/migrations/0001_initial.py @@ -1,12 +1,13 @@ -# Generated by Django 3.2rc1 on 2021-04-05 14:31 +# -*- coding: utf-8 -*- +from __future__ import unicode_literals -from django.db import migrations, models +from django.db import models, migrations import uuid +from django.db.models import UUIDField -class Migration(migrations.Migration): - initial = True +class Migration(migrations.Migration): dependencies = [] @@ -16,39 +17,38 @@ class Migration(migrations.Migration): fields=[ ( "id", - models.UUIDField( - default=uuid.uuid4, + UUIDField( + serialize=False, editable=False, + default=uuid.uuid4, primary_key=True, - serialize=False, ), ), - ("created", models.DateTimeField(auto_now_add=True, db_index=True)), + ("created", models.DateTimeField(db_index=True, auto_now_add=True)), ("modified", models.DateTimeField(auto_now=True)), ("name", models.CharField(max_length=100)), ( "state", models.CharField( - choices=[ - ("NEW", "New"), - ("READY", "Ready"), - ("PROCESSING", "Processing"), - ("FAILED", "Failed"), - ("COMPLETE", "Complete"), - ], db_index=True, - default="NEW", max_length=20, + default="NEW", + choices=[ + ("NEW", "NEW"), + ("READY", "READY"), + ("PROCESSING", "PROCESSING"), + ("FAILED", "FAILED"), + ("COMPLETE", "COMPLETE"), + ], ), ), - ("next_task", models.CharField(blank=True, max_length=100)), - ("workspace", models.JSONField(null=True)), + ("next_task", models.CharField(max_length=100, blank=True)), + ("workspace", models.TextField(null=True)), ( "queue_name", - models.CharField(db_index=True, default="default", max_length=20), + models.CharField(db_index=True, max_length=20, default="default"), ), - ("priority", models.SmallIntegerField(db_index=True, default=0)), ], - options={"ordering": ["-priority", "created"],}, + options={"ordering": ["-created"],}, ), ] diff --git a/django_dbq/migrations/0002_auto_20151016_1027.py b/django_dbq/migrations/0002_auto_20151016_1027.py new file mode 100644 index 0000000..d0a72c4 --- /dev/null +++ b/django_dbq/migrations/0002_auto_20151016_1027.py @@ -0,0 +1,15 @@ +# -*- coding: utf-8 -*- +from __future__ import unicode_literals + +from django.db import models, migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ("django_dbq", "0001_initial"), + ] + + operations = [ + migrations.AlterModelOptions(name="job", options={"ordering": ["created"]},), + ] diff --git a/django_dbq/migrations/0003_auto_20180713_1000.py b/django_dbq/migrations/0003_auto_20180713_1000.py new file mode 100644 index 0000000..4d959f3 --- /dev/null +++ b/django_dbq/migrations/0003_auto_20180713_1000.py @@ -0,0 +1,23 @@ +# -*- coding: utf-8 -*- +# Generated by Django 1.11 on 2018-07-13 10:00 +from __future__ import unicode_literals + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ("django_dbq", "0002_auto_20151016_1027"), + ] + + operations = [ + migrations.AlterModelOptions( + name="job", options={"ordering": ["-priority", "created"]}, + ), + migrations.AddField( + model_name="job", + name="priority", + field=models.SmallIntegerField(db_index=True, default=0), + ), + ] diff --git a/django_dbq/migrations/0004_auto_20210818_0247.py b/django_dbq/migrations/0004_auto_20210818_0247.py new file mode 100644 index 0000000..a1ff5ff --- /dev/null +++ b/django_dbq/migrations/0004_auto_20210818_0247.py @@ -0,0 +1,32 @@ +# Generated by Django 3.2rc1 on 2021-08-18 02:47 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ("django_dbq", "0003_auto_20180713_1000"), + ] + + operations = [ + migrations.AlterField( + model_name="job", + name="state", + field=models.CharField( + choices=[ + ("NEW", "New"), + ("READY", "Ready"), + ("PROCESSING", "Processing"), + ("FAILED", "Failed"), + ("COMPLETE", "Complete"), + ], + db_index=True, + default="NEW", + max_length=20, + ), + ), + migrations.AlterField( + model_name="job", name="workspace", field=models.JSONField(null=True), + ), + ] From 9fce8506236b0e6f27e97d58e83db68e5b035f50 Mon Sep 17 00:00:00 2001 From: Jamie Matthews Date: Thu, 4 Nov 2021 08:52:37 +0000 Subject: [PATCH 094/158] Add support for scheduling jobs to run in the future --- README.md | 9 +++++++++ django_dbq/migrations/0005_job_run_after.py | 18 ++++++++++++++++++ django_dbq/models.py | 8 +++++++- django_dbq/tests.py | 14 ++++++++++++++ 4 files changed, 48 insertions(+), 1 deletion(-) create mode 100644 django_dbq/migrations/0005_job_run_after.py diff --git a/README.md b/README.md index 5a55d00..869798f 100644 --- a/README.md +++ b/README.md @@ -129,6 +129,15 @@ Job.objects.create(name='critical_job', priority=2) Jobs will be ordered by their `priority` (highest to lowest) and then the time which they were created (oldest to newest) and processed in that order. +### Scheduling jobs +If you'd like to create a job but have it run at some time in the future, you can use the `run_after` field on the Job model: + +```python +Job.objects.create(name='scheduled_job', run_after=timezone.now() + timedelta(minutes=10)) +``` + +Of course, the job will only be run if your `python manage.py worker` process is running at the time when the job is scheduled to run. Otherwise, it will run the next time you start your worker process after that time has passed. + ## Terminology ### Job diff --git a/django_dbq/migrations/0005_job_run_after.py b/django_dbq/migrations/0005_job_run_after.py new file mode 100644 index 0000000..ad86b3b --- /dev/null +++ b/django_dbq/migrations/0005_job_run_after.py @@ -0,0 +1,18 @@ +# Generated by Django 3.2rc1 on 2021-11-04 03:32 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('django_dbq', '0004_auto_20210818_0247'), + ] + + operations = [ + migrations.AddField( + model_name='job', + name='run_after', + field=models.DateTimeField(db_index=True, null=True), + ), + ] diff --git a/django_dbq/models.py b/django_dbq/models.py index 2f51a78..a0075da 100644 --- a/django_dbq/models.py +++ b/django_dbq/models.py @@ -68,7 +68,12 @@ def delete_old(self): def to_process(self, queue_name): return self.select_for_update().filter( - queue_name=queue_name, state__in=(Job.STATES.READY, Job.STATES.NEW) + models.Q(queue_name=queue_name) & + models.Q(state__in=(Job.STATES.READY, Job.STATES.NEW)) & + models.Q( + models.Q(run_after__isnull=True) | + models.Q(run_after__lte=timezone.now()) + ) ) @@ -91,6 +96,7 @@ class STATES(TextChoices): workspace = JSONField(null=True) queue_name = models.CharField(max_length=20, default="default", db_index=True) priority = models.SmallIntegerField(default=0, db_index=True) + run_after = models.DateTimeField(null=True, db_index=True) class Meta: ordering = ["-priority", "created"] diff --git a/django_dbq/tests.py b/django_dbq/tests.py index 33df2f5..2c8e0e9 100644 --- a/django_dbq/tests.py +++ b/django_dbq/tests.py @@ -210,6 +210,20 @@ def test_gets_jobs_in_priority_and_date_order(self): self.assertEqual(Job.objects.get_ready_or_none("default"), job_1) self.assertFalse(Job.objects.to_process("default").filter(id=job_2.id).exists()) + def test_ignores_jobs_until_run_after_is_in_the_past(self): + job_1 = Job.objects.create(name="testjob") + job_2 = Job.objects.create(name="testjob", run_after=datetime(2021, 11, 4, 8)) + + with freezegun.freeze_time(datetime(2021, 11, 4, 7)): + self.assertEqual( + {job for job in Job.objects.to_process("default")}, {job_1} + ) + + with freezegun.freeze_time(datetime(2021, 11, 4, 9)): + self.assertEqual( + {job for job in Job.objects.to_process("default")}, {job_1, job_2} + ) + def test_get_next_ready_job_created(self): """ Created jobs should be picked too. From e17d9bd24b24c4bdcbd4290dedaf3f08dce17e81 Mon Sep 17 00:00:00 2001 From: Jamie Matthews Date: Thu, 4 Nov 2021 08:57:18 +0000 Subject: [PATCH 095/158] Format with black --- django_dbq/migrations/0005_job_run_after.py | 6 +++--- django_dbq/models.py | 10 +++++----- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/django_dbq/migrations/0005_job_run_after.py b/django_dbq/migrations/0005_job_run_after.py index ad86b3b..67a2c0d 100644 --- a/django_dbq/migrations/0005_job_run_after.py +++ b/django_dbq/migrations/0005_job_run_after.py @@ -6,13 +6,13 @@ class Migration(migrations.Migration): dependencies = [ - ('django_dbq', '0004_auto_20210818_0247'), + ("django_dbq", "0004_auto_20210818_0247"), ] operations = [ migrations.AddField( - model_name='job', - name='run_after', + model_name="job", + name="run_after", field=models.DateTimeField(db_index=True, null=True), ), ] diff --git a/django_dbq/models.py b/django_dbq/models.py index a0075da..5669861 100644 --- a/django_dbq/models.py +++ b/django_dbq/models.py @@ -68,11 +68,11 @@ def delete_old(self): def to_process(self, queue_name): return self.select_for_update().filter( - models.Q(queue_name=queue_name) & - models.Q(state__in=(Job.STATES.READY, Job.STATES.NEW)) & - models.Q( - models.Q(run_after__isnull=True) | - models.Q(run_after__lte=timezone.now()) + models.Q(queue_name=queue_name) + & models.Q(state__in=(Job.STATES.READY, Job.STATES.NEW)) + & models.Q( + models.Q(run_after__isnull=True) + | models.Q(run_after__lte=timezone.now()) ) ) From d1610281d617e5d790974462ee929691b9ff5598 Mon Sep 17 00:00:00 2001 From: Jamie Matthews Date: Thu, 4 Nov 2021 09:06:15 +0000 Subject: [PATCH 096/158] Add note to README on precision of scheduled job timing --- README.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 869798f..8556a3f 100644 --- a/README.md +++ b/README.md @@ -136,7 +136,9 @@ If you'd like to create a job but have it run at some time in the future, you ca Job.objects.create(name='scheduled_job', run_after=timezone.now() + timedelta(minutes=10)) ``` -Of course, the job will only be run if your `python manage.py worker` process is running at the time when the job is scheduled to run. Otherwise, it will run the next time you start your worker process after that time has passed. +Of course, the scheduled job will only be run if your `python manage.py worker` process is running at the time when the job is scheduled to run. Otherwise, it will run the next time you start your worker process after that time has passed. + +It's also worth noting that, by default, scheduled jobs run as part of the same queue as all other jobs, and so if a job is already being processed at the time when your scheduled job is due to run, it won't run until that job has finished. If increased precision is important, you might consider using the `queue_name` feature to run a separate worker dedicated to only running scheduled jobs. ## Terminology From d27fd08a9b11271dca23e42b37772f29727cef7c Mon Sep 17 00:00:00 2001 From: Jamie Matthews Date: Tue, 23 Nov 2021 20:48:32 +0000 Subject: [PATCH 097/158] Bump version --- django_dbq/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/django_dbq/__init__.py b/django_dbq/__init__.py index 8c0d5d5..9aa3f90 100644 --- a/django_dbq/__init__.py +++ b/django_dbq/__init__.py @@ -1 +1 @@ -__version__ = "2.0.0" +__version__ = "2.1.0" From 305d9f4388a78f366555984012d4ac9334f54a14 Mon Sep 17 00:00:00 2001 From: Jamie Matthews Date: Mon, 29 Nov 2021 10:36:30 +0000 Subject: [PATCH 098/158] Refactor to move process_job function onto Worker class --- django_dbq/management/commands/worker.py | 116 +++++++++++------------ django_dbq/tests.py | 45 +++++---- 2 files changed, 77 insertions(+), 84 deletions(-) diff --git a/django_dbq/management/commands/worker.py b/django_dbq/management/commands/worker.py index 92e72e4..7fb1a95 100644 --- a/django_dbq/management/commands/worker.py +++ b/django_dbq/management/commands/worker.py @@ -14,66 +14,6 @@ DEFAULT_QUEUE_NAME = "default" -def process_job(queue_name): - """This function grabs the next available job for a given queue, and runs its next task.""" - - with transaction.atomic(): - job = Job.objects.get_ready_or_none(queue_name) - if not job: - return - - logger.info( - 'Processing job: name="%s" queue="%s" id=%s state=%s next_task=%s', - job.name, - queue_name, - job.pk, - job.state, - job.next_task, - ) - job.state = Job.STATES.PROCESSING - job.save() - - try: - task_function = import_string(job.next_task) - task_function(job) - job.update_next_task() - if not job.next_task: - job.state = Job.STATES.COMPLETE - else: - job.state = Job.STATES.READY - except Exception as exception: - logger.exception("Job id=%s failed", job.pk) - job.state = Job.STATES.FAILED - - failure_hook_name = job.get_failure_hook_name() - if failure_hook_name: - logger.info( - "Running failure hook %s for job id=%s", failure_hook_name, job.pk - ) - failure_hook_function = import_string(failure_hook_name) - failure_hook_function(job, exception) - else: - logger.info("No failure hook for job id=%s", job.pk) - - logger.info( - 'Updating job: name="%s" id=%s state=%s next_task=%s', - job.name, - job.pk, - job.state, - job.next_task or "none", - ) - - try: - job.save() - except: - logger.error( - "Failed to save job: id=%s org=%s", - job.pk, - job.workspace.get("organisation_id"), - ) - raise - - class Worker: def __init__(self, name, rate_limit_in_seconds): self.queue_name = name @@ -103,9 +43,63 @@ def process_job(self): ): return - process_job(self.queue_name) + self._process_job() + self.last_job_finished = timezone.now() + def _process_job(self): + with transaction.atomic(): + job = Job.objects.get_ready_or_none(self.queue_name) + if not job: + return + + logger.info( + 'Processing job: name="%s" queue="%s" id=%s state=%s next_task=%s', + job.name, + self.queue_name, + job.pk, + job.state, + job.next_task, + ) + job.state = Job.STATES.PROCESSING + job.save() + + try: + task_function = import_string(job.next_task) + task_function(job) + job.update_next_task() + if not job.next_task: + job.state = Job.STATES.COMPLETE + else: + job.state = Job.STATES.READY + except Exception as exception: + logger.exception("Job id=%s failed", job.pk) + job.state = Job.STATES.FAILED + + failure_hook_name = job.get_failure_hook_name() + if failure_hook_name: + logger.info( + "Running failure hook %s for job id=%s", failure_hook_name, job.pk + ) + failure_hook_function = import_string(failure_hook_name) + failure_hook_function(job, exception) + else: + logger.info("No failure hook for job id=%s", job.pk) + + logger.info( + 'Updating job: name="%s" id=%s state=%s next_task=%s', + job.name, + job.pk, + job.state, + job.next_task or "none", + ) + + try: + job.save() + except: + logger.exception("Failed to save job: id=%s", job.pk) + raise + class Command(BaseCommand): diff --git a/django_dbq/tests.py b/django_dbq/tests.py index 33df2f5..2de026a 100644 --- a/django_dbq/tests.py +++ b/django_dbq/tests.py @@ -7,7 +7,7 @@ from django.test.utils import override_settings from django.utils import timezone -from django_dbq.management.commands.worker import process_job, Worker +from django_dbq.management.commands.worker import Worker from django_dbq.models import Job from io import StringIO @@ -116,41 +116,40 @@ def test_queue_depth_for_queue_with_zero_jobs(self): @freezegun.freeze_time() @mock.patch("django_dbq.management.commands.worker.sleep") -@mock.patch("django_dbq.management.commands.worker.process_job") class WorkerProcessProcessJobTestCase(TestCase): def setUp(self): super().setUp() - self.MockWorker = mock.MagicMock() - self.MockWorker.queue_name = "default" - self.MockWorker.rate_limit_in_seconds = 5 - self.MockWorker.last_job_finished = None + self.mock_worker = mock.MagicMock() + self.mock_worker.queue_name = "default" + self.mock_worker.rate_limit_in_seconds = 5 + self.mock_worker.last_job_finished = None - def test_process_job_no_previous_job_run(self, mock_process_job, mock_sleep): - Worker.process_job(self.MockWorker) + def test_process_job_no_previous_job_run(self, mock_sleep): + Worker.process_job(self.mock_worker) self.assertEqual(mock_sleep.call_count, 1) - self.assertEqual(mock_process_job.call_count, 1) - self.assertEqual(self.MockWorker.last_job_finished, timezone.now()) + self.assertEqual(self.mock_worker._process_job.call_count, 1) + self.assertEqual(self.mock_worker.last_job_finished, timezone.now()) - def test_process_job_previous_job_too_soon(self, mock_process_job, mock_sleep): - self.MockWorker.last_job_finished = timezone.now() - timezone.timedelta( + def test_process_job_previous_job_too_soon(self, mock_sleep): + self.mock_worker.last_job_finished = timezone.now() - timezone.timedelta( seconds=2 ) - Worker.process_job(self.MockWorker) + Worker.process_job(self.mock_worker) self.assertEqual(mock_sleep.call_count, 1) - self.assertEqual(mock_process_job.call_count, 0) + self.assertEqual(self.mock_worker._process_job.call_count, 0) self.assertEqual( - self.MockWorker.last_job_finished, + self.mock_worker.last_job_finished, timezone.now() - timezone.timedelta(seconds=2), ) - def test_process_job_previous_job_long_time_ago(self, mock_process_job, mock_sleep): - self.MockWorker.last_job_finished = timezone.now() - timezone.timedelta( + def test_process_job_previous_job_long_time_ago(self, mock_sleep): + self.mock_worker.last_job_finished = timezone.now() - timezone.timedelta( seconds=7 ) - Worker.process_job(self.MockWorker) + Worker.process_job(self.mock_worker) self.assertEqual(mock_sleep.call_count, 1) - self.assertEqual(mock_process_job.call_count, 1) - self.assertEqual(self.MockWorker.last_job_finished, timezone.now()) + self.assertEqual(self.mock_worker._process_job.call_count, 1) + self.assertEqual(self.mock_worker.last_job_finished, timezone.now()) @override_settings(JOBS={"testjob": {"tasks": ["a"]}}) @@ -246,7 +245,7 @@ def test_task_sequence(self): class ProcessJobTestCase(TestCase): def test_process_job(self): job = Job.objects.create(name="testjob") - process_job("default") + Worker("default", 1)._process_job() job = Job.objects.get() self.assertEqual(job.state, Job.STATES.COMPLETE) @@ -255,7 +254,7 @@ def test_process_job_wrong_queue(self): Processing a different queue shouldn't touch our other job """ job = Job.objects.create(name="testjob", queue_name="lol") - process_job("default") + Worker("default", 1)._process_job() job = Job.objects.get() self.assertEqual(job.state, Job.STATES.NEW) @@ -294,7 +293,7 @@ def test_creation_hook_only_runs_on_create(self): class JobFailureHookTestCase(TestCase): def test_failure_hook(self): job = Job.objects.create(name="testjob") - process_job("default") + Worker("default", 1)._process_job() job = Job.objects.get() self.assertEqual(job.state, Job.STATES.FAILED) self.assertEqual(job.workspace["output"], "failure hook ran") From 5a88fa764586f5e031a719fd370af26808758503 Mon Sep 17 00:00:00 2001 From: Jamie Matthews Date: Mon, 29 Nov 2021 10:37:51 +0000 Subject: [PATCH 099/158] Add current_job attribute to Worker --- django_dbq/management/commands/worker.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/django_dbq/management/commands/worker.py b/django_dbq/management/commands/worker.py index 7fb1a95..18db2d2 100644 --- a/django_dbq/management/commands/worker.py +++ b/django_dbq/management/commands/worker.py @@ -20,6 +20,7 @@ def __init__(self, name, rate_limit_in_seconds): self.rate_limit_in_seconds = rate_limit_in_seconds self.alive = True self.last_job_finished = None + self.current_job = None self.init_signals() def init_signals(self): @@ -63,6 +64,7 @@ def _process_job(self): ) job.state = Job.STATES.PROCESSING job.save() + self.current_job = job try: task_function = import_string(job.next_task) @@ -100,6 +102,8 @@ def _process_job(self): logger.exception("Failed to save job: id=%s", job.pk) raise + self.current_job = None + class Command(BaseCommand): From 2619a49122b1375acc411b2bf849d31632f72612 Mon Sep 17 00:00:00 2001 From: Jamie Matthews Date: Mon, 29 Nov 2021 10:44:12 +0000 Subject: [PATCH 100/158] Add STOPPING state and update when shutdown is requested --- django_dbq/management/commands/worker.py | 3 ++ django_dbq/migrations/0005_alter_job_state.py | 30 +++++++++++++++++++ django_dbq/models.py | 1 + django_dbq/tests.py | 13 ++++++++ 4 files changed, 47 insertions(+) create mode 100644 django_dbq/migrations/0005_alter_job_state.py diff --git a/django_dbq/management/commands/worker.py b/django_dbq/management/commands/worker.py index 18db2d2..7409dd3 100644 --- a/django_dbq/management/commands/worker.py +++ b/django_dbq/management/commands/worker.py @@ -30,6 +30,9 @@ def init_signals(self): def shutdown(self, signum, frame): self.alive = False + if self.current_job: + self.current_job.state = Job.STATES.STOPPING + self.current_job.save(update_fields=["state"]) def run(self): while self.alive: diff --git a/django_dbq/migrations/0005_alter_job_state.py b/django_dbq/migrations/0005_alter_job_state.py new file mode 100644 index 0000000..5a39976 --- /dev/null +++ b/django_dbq/migrations/0005_alter_job_state.py @@ -0,0 +1,30 @@ +# Generated by Django 3.2rc1 on 2021-11-29 04:43 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ("django_dbq", "0004_auto_20210818_0247"), + ] + + operations = [ + migrations.AlterField( + model_name="job", + name="state", + field=models.CharField( + choices=[ + ("NEW", "New"), + ("READY", "Ready"), + ("PROCESSING", "Processing"), + ("STOPPING", "Stopping"), + ("FAILED", "Failed"), + ("COMPLETE", "Complete"), + ], + db_index=True, + default="NEW", + max_length=20, + ), + ), + ] diff --git a/django_dbq/models.py b/django_dbq/models.py index 2f51a78..7186ae6 100644 --- a/django_dbq/models.py +++ b/django_dbq/models.py @@ -77,6 +77,7 @@ class STATES(TextChoices): NEW = "NEW" READY = "READY" PROCESSING = "PROCESSING" + STOPPING = "STOPPING" FAILED = "FAILED" COMPLETE = "COMPLETE" diff --git a/django_dbq/tests.py b/django_dbq/tests.py index 2de026a..1e05343 100644 --- a/django_dbq/tests.py +++ b/django_dbq/tests.py @@ -152,6 +152,19 @@ def test_process_job_previous_job_long_time_ago(self, mock_sleep): self.assertEqual(self.mock_worker.last_job_finished, timezone.now()) +@override_settings(JOBS={"testjob": {"tasks": ["a"]}}) +class ShutdownTestCase(TestCase): + def test_shutdown_sets_state_to_stopping(self): + job = Job.objects.create(name="testjob") + worker = Worker("default", 1) + worker.current_job = job + + worker.shutdown(None, None) + + job.refresh_from_db() + self.assertEqual(job.state, Job.STATES.STOPPING) + + @override_settings(JOBS={"testjob": {"tasks": ["a"]}}) class JobTestCase(TestCase): def test_create_job(self): From 8214131bd7bb46ea15291a73488d5ad7947d1789 Mon Sep 17 00:00:00 2001 From: Jamie Matthews Date: Mon, 29 Nov 2021 10:46:13 +0000 Subject: [PATCH 101/158] Delete old jobs in state STOPPING --- django_dbq/models.py | 6 +++++- django_dbq/tests.py | 10 +++++++--- 2 files changed, 12 insertions(+), 4 deletions(-) diff --git a/django_dbq/models.py b/django_dbq/models.py index 7186ae6..c3c5782 100644 --- a/django_dbq/models.py +++ b/django_dbq/models.py @@ -53,7 +53,11 @@ def delete_old(self): """ Delete all jobs older than DELETE_JOBS_AFTER_HOURS """ - delete_jobs_in_states = [Job.STATES.FAILED, Job.STATES.COMPLETE] + delete_jobs_in_states = [ + Job.STATES.FAILED, + Job.STATES.COMPLETE, + Job.STATES.STOPPING, + ] delete_jobs_created_before = timezone.now() - datetime.timedelta( hours=DELETE_JOBS_AFTER_HOURS ) diff --git a/django_dbq/tests.py b/django_dbq/tests.py index 1e05343..216210f 100644 --- a/django_dbq/tests.py +++ b/django_dbq/tests.py @@ -325,14 +325,18 @@ def test_delete_old_jobs(self): j2.created = two_days_ago j2.save() - j3 = Job.objects.create(name="testjob", state=Job.STATES.NEW) + j3 = Job.objects.create(name="testjob", state=Job.STATES.STOPPING) j3.created = two_days_ago j3.save() - j4 = Job.objects.create(name="testjob", state=Job.STATES.COMPLETE) + j4 = Job.objects.create(name="testjob", state=Job.STATES.NEW) + j4.created = two_days_ago + j4.save() + + j5 = Job.objects.create(name="testjob", state=Job.STATES.COMPLETE) Job.objects.delete_old() self.assertEqual(Job.objects.count(), 2) - self.assertTrue(j3 in Job.objects.all()) self.assertTrue(j4 in Job.objects.all()) + self.assertTrue(j5 in Job.objects.all()) From a93403b3ca986c5345660fda48590c642688de71 Mon Sep 17 00:00:00 2001 From: Jamie Matthews Date: Mon, 29 Nov 2021 10:49:07 +0000 Subject: [PATCH 102/158] Rebase migrations --- .../{0005_alter_job_state.py => 0006_alter_job_state.py} | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) rename django_dbq/migrations/{0005_alter_job_state.py => 0006_alter_job_state.py} (87%) diff --git a/django_dbq/migrations/0005_alter_job_state.py b/django_dbq/migrations/0006_alter_job_state.py similarity index 87% rename from django_dbq/migrations/0005_alter_job_state.py rename to django_dbq/migrations/0006_alter_job_state.py index 5a39976..e7c51cb 100644 --- a/django_dbq/migrations/0005_alter_job_state.py +++ b/django_dbq/migrations/0006_alter_job_state.py @@ -1,4 +1,4 @@ -# Generated by Django 3.2rc1 on 2021-11-29 04:43 +# Generated by Django 3.2rc1 on 2021-11-29 04:48 from django.db import migrations, models @@ -6,7 +6,7 @@ class Migration(migrations.Migration): dependencies = [ - ("django_dbq", "0004_auto_20210818_0247"), + ("django_dbq", "0005_job_run_after"), ] operations = [ From 631dc032cfba0becf519b2ceb85209898a8ae877 Mon Sep 17 00:00:00 2001 From: Jamie Matthews Date: Mon, 29 Nov 2021 14:37:29 +0000 Subject: [PATCH 103/158] Document STOPPING state and add state diagram --- README.md | 5 +++++ states.png | Bin 0 -> 101513 bytes 2 files changed, 5 insertions(+) create mode 100644 states.png diff --git a/README.md b/README.md index 8556a3f..298bde6 100644 --- a/README.md +++ b/README.md @@ -189,9 +189,14 @@ Jobs have a `state` field which can have one of the following values: * `NEW` (has been created, waiting for a worker process to run the next task) * `READY` (has run a task before, awaiting a worker process to run the next task) * `PROCESSING` (a task is currently being processed by a worker) +* `STOPPING` (the worker process has received a signal from the OS requesting it to exit) * `COMPLETED` (all job tasks have completed successfully) * `FAILED` (a job task failed) +#### State diagram + +![state diagram](states.png) + ### API #### Model methods diff --git a/states.png b/states.png new file mode 100644 index 0000000000000000000000000000000000000000..acc285820187d2522077d16c2d70aa3e8123a3d4 GIT binary patch literal 101513 zcmeEug2pxpu!fU1r-HpkS-B0=8xGSWD&wx~*Yv|{m|2%m{AX#+4envn3DUh1UB)odLkYZ^Z0p93!zd(lPUK4M zsrKXqO3{p2M1&JKS8vbR^A5!>PhD4dBj z1mAF~o&+I^`4N|8>L*kZq z^mp$^`o2%P1Mw-8QaIV=UkK;T&UXJ&CVQ{Q`HJb@~5>#mzoa4!|t z_s_#cULw>6szThHWCn&oO)Z$kYz&^x;Wrb|*~Xa`jpQ%VUok)rTEH{Xf4~qQ){FC; zLDx(y{Ph=wr7pX4bT33b)u(MmHHKf@cTpud54c&l5-Akdg z-Nfj5BqhHYT@*dTY=s|>JWV*Z{FY%0Z|~;4C{MQ(OIBUWG`Z&I$}^tgAHUr<_AF8q zSQj~AC0Tth()sYpw~L>84bM5u;N5L{U1ivXIs5t=q6_PR{QXpWCYcw+&AF>~c%q`u z?%JRhXXf!dD7!4nh*<5{l80tc5apbyg!v-(hkTe6~lLzUNb zG2}i;Pr$*_qQR^ZK}k@KYebl%a~QZvqg|gS48jn(Q_5_lr-Blk@~b0K#m(s)FM3G} zI^$NOVi2Av=S`mv^XR$FVv`DuL|GSqHt~}XP=!{Y5VV|p7$SW9!CXy_D6RJsmLNMP z7SnV6W;ugvYzbx&BR}fJ;Ar_#!6NsJ84m=b_|^W?$E>s}YOTzbYBt20v`qTG6V}5@uWo#&{J% zj1?g$$}Y~(j=0RGwz)>5XWJ}5LLX3*TSJZ#>=L%Mv10YHfghvC=4VFyHa`k#+tKkc zp=X~sCAwZAp};S!4@;}DC}z3nw(B&q%uFiFwk`fBr`I`W9&lh}G+Y{Pd@}CKy@}OG zbN-$$Pvm*_1|BuztK$m_tT z{o_O*j*cJICc(~C?7Mf(uqoN@bzZP&zGx>Id!Ob~Kr_wr`{!Qzf1@?H7cM3Cod)rV z-3b3ZPPX_SE!GR7Pj2rhtHK2CRY!7vq9+#1&SHM-H=0Z*cccAoT91+A?YIz`dyl%S z7HLay^F>`!@fUG+Xx%B-M08W@NE`auJUb~IT4PDcLz>u&DWrnGG_)5#%sZFc7B}Ni zAMB(zF)W!7)Ms~Y{k)6qwIsjo&F2T_pIog!Hx$FJG&ryY&3%o@OecK**J5fmx;5yR&?=*0Xj7{6E-=WA4#j zc}ejA^&QWp7^Qm*bon>du2IryUv+vZl$D+PCc7%vKi4?t`M|>gh5_d+O68i=J*oVN zJn`}y$1mnSJ=r4OGT6Fo&vAIE>eaYpPsXqU+Re|e1==kiBGQxWw zM2?e1hirSM%{IAA9xGXJnrZd8rau@g9W*iz@4MZT*FRq9_Mk*FB+odlWpMWKm+qWF zzAPujT-Er3wG7Hk?W{UEkH9^*wFSOwl;nx5l=WY=62I`#J5lwqRbNk(Ug%lquIQHS z<9B*qU0zKwAuxeEpM2fGliJpIc{OZ+>iHxI+#tgR%MyZH6 z=QllH(7xO=VeAZlQ>VAcrWT4OftVvsYa1UH=+&~W|6Kc}w(H{{ZA)suK}Cz_qu}d( z>}~e`#v#TLQEvur=9;TGskr2Q9+a|3*=1NM8Qie4wCc6|KEgRFP{CC+KO8+gI5Jt1 zHi9`KU7A~5VI=V1cSn^@jhtHK4n=*%znKw0jGMu)}DSS-^gI<@uSTU?m+BAow zqiVS7>tRYr{iD!F*^kmXaxbdTu}j{jv~zA&Rb?4M8)7POnQ$CW9e+3xH!)esU%5S= zJ5lJ?E$~s$#GTWvcF%E#*tKB4c3ym+Y@Vwtw`C=yBBG+dX0oQEt)0HZG`~wagP9|# zD$?z1%ryBdf8T!5hNQ5~2igywUfxIi#|y_Bo8Cgb&hPZTMZbtZzl~Oj8jr?@cEwlS zmmHnnuk&f8D3@r2AJfyrM$M;hi&xd%@)Gk@uNGYGjYPk?G2*qlv^tlfW_Hia%}g^^ zM(yEnToKmH<(aA(ry0SEIu{k`!X+=!rqRVnOn!^CbxqKak$o-Ozmuid)4%v`k*g=d zfWqJ%+n2O%dHJ;Hh-wlGO2%-jP;|S5%6Yr}IjcGPR_cc|H=5~{=mn%5A02-HKtAAR+O1cw-^5jM zmAE(#FGi*Dx4DMhB6l#%TO0mt*cHJ+rpmhh$@R;R@gK(}*y`r${O^Sv<|?_yCoN_6 zMpvn*;wyspUs8AKHf~(Pz>daAn#cHdyTdJsJhDch;t}b!JYwgN;Z&~FA-5N97$Y{ zm7d~6fAc~x%{XJevxw7Q7!$?)qDZ8@DX@*ZPxqIUiRoFZ>ec4H>=c!J_lw zS>?UjZ>_z_>0>eGER5=Vu7_Fk_>82C?W!kb_9yMzQAttcD)K6yRh^Vtl(_QCHR_CZ z7q~uh4dsgExD;~f)ab1pJDoVG4q_D+PU*c*F=)yROAV{IBX0RPt2jG*Txwja_HZe> zpUKo)|Cd6|fSE87>~Yz8OEk5l){R!!FnPX~^M%CHu(+Q}mWZs7}teQXWS2+i1gDvzZZ_cNZ5 z!k)Kut>|!Ona*AEGIcsdqDSee+UMfQxtDVj^!Bm>)>o$N_l=G|TV!66DN76&@?4iX zc=O6s-83Q*kH_VpcIxmcNh}E&i@UC++3sV(&G;c3tkK#rGUh|A$JMJIRaKrw$32Xq zsxn&mb?)EE7DiiEiigH*-i=o}Idpyr*F4eevLPAw^2|JtnVL}F$KG$>u$U-(qIZ(~ z-Q{@7#l_Wy!j1H(Xji^BttP@PLRaX8;Ndv${;f6hd5#j(vaX4@)a$q#)h7l!Azx!& z#&}K`dz+mW??p}I>YkqeRZ>P@m$_ZG<8&JDZ-YN&ecZ8K7(IC~iM4?{wOhw>s&}A& zEcv~yH)Num!-{)_-Vkqv(#?*N+*BXN zt}%i~fStOki*?~ZR&9iZAcz}fck@^n`XZ#q;inEV}3IWC`6zu(8?$6KQD_26&E*f`Lkgf(g%1;YSFS z{6EhnP;a83|9%|}1triN1>?`p$iO4=UnKk>zw^f<`pW-FlKhOvPB*TMZlF0o?00@>eIMdSz0^rxeC(${sbR9N50HLOY{39 zjuwKnYSMBvVm9{1Gl&%4F?8_s2p0I?g>~2Sa;vTSs#nYZ~Nn^$l#C z90h4zq^S0myS&;9ruraf;{O8#4RRQFyd~)Wl#+K^$%&lO| z;2c6+yqp5RKY!+(|DN%0-&FhWH`zE?dH(*@zrFR(uPQqj+l$#)!8sj;{ySfPe);cj z{`o}#7UZq}Z7BX2=ijfwKnvjtu>5DzgmAC_dhG}ENMU|YRtX+ql_CG3-hy8@|9C_` zuVQKP)v-cB5kYxyPgKbjb#($~Tx$3D^hd|#;LFC(A3FEazWemjtS9yU1EWV8rE)1j z@1m3#qFyrHeR!P2;vbnV{r(yg@ymxHrHdzB$J@tm%}v(dR4tgSS5*ux{2+Dsp1@)1 z_SLSdlk07l6JqPcgcIX}2ny=iUo@C#{@6SjA}^$Ad{EG^&;0TsqM82W|F3VMpxuHu zW7!|xdG>GLL_x!#s5S1`ynsfe*+ENYx(zA&rUV+ zB4IMQ{{4{uH<`$h|G&xnzsUT*$VB`9Ce3~l^4+LE{wj2O;@~n?r2q8iPM>VK=dttg ze2nh6=fTckzUG84h3m9$`IOi3Nd578{7k*n8+Mgq1Myt>w167r0xjNSf+QxAn@|HV zm*>A5UHoTxp{c;TtzF%(QlMpSF;w8x*s{AaG?t^ByBZ<<>%Qya^+(Z;qT-mOPGc5@ zb(Ul0Rw+K;Y(D&q-aOQBZN4b^3@sA(-e9(3TO5}~^wPE(s;@1aX@*RS?rfSizbxnT zBhh8DxXNK+wAO8Vb$`mc%N7IoWFSwy@9|K9)j+Ol#p8jTkzdCf)VQoU^oHzAi-_#W z$ZF5S`IvI+2|mpd6X}}`RfcWh=i~X7-o28lh-T8%*#7>77PMON2g_e4hXq&$MUU_;RP+kHejYJ5WYHK7d@G*K zFu-c${q<6AOd>?7MNg`Knf1izywZ85PH`gPUk7P;kqmM+>;i<<(ewG_%{Sr zdecMrBR=}$Eu_*d$l2couppCFB73G7e%ePPD>(V~O14@gEx0broX64Cr}(Zvu3N9D zSP2(~0JcbhTJc9r8s%phikULZ2Wo{nbpmeI6gx?H7i`BJ#LCak4mOYPTkl^ZH)NA6 z`!Z!KU~ZX5FtldNJ=QwMWRC!h1GSxX3stg|B9 zP~RQl<5dpLI`=Zh&D5)3gikh~u8mi5%Zo&&ReB_;?Y9q$*{xD$X+eiu`}v>@QwTicH~%m-O0MsuxU z(!O7f;@wRdXgp4T?vKGTu<^%5yoYm*1)RTPfKPr$%9$fCCJ+GL_Y{&4G4X#!n{NVQ zxZbk{pN#Vp{*}Q^ef&rDN6Wcu)hh)KeV-{3aSO5xb~(BP3?XVmT!RBRxvfU~Cy0j? zS;UlmWlR+}Br$0aC;e6sqbaqajdq-7-3_A^HZ~JAF0$3{%iUEUXYFRmr%4Hl4e&WG z>g`XsyrpzsOli!h>cWEf-);=~bn6T#71_geO)`}g`0)(y4ugw9mHm94?gy{IOj&A1 z?P|viepT7l7+tTC45>E;Fo@Ml#iKKU)Yf0(xExR=CM<3`SX=6P?nW`G6*XW|BYc#Y zAG|Re-g`iwXhfp>q!^DXnn9VNuSqqc`@yRlBS9DGRzq0oZR@T_P5n4pDV&>Mx0$NHV>w*(E&#!JKa(cSog>FIO#{-C zbk%v~`{^cM;@n<}M`dv(JQ*!D|0+%||K8{U|Ngjx_U4a%McFc1@!)F$+`AV%r+u*! z(gU=9?iG9XBhDPL1)DnvX<5c2$;s;z~i z7*(B}1wHmUONVB^e~H(&>AAvTQn`@iJZgT&c~x!bm}86}4&O8vz`o_Y@!XltBytF& ze|!CfBvo;WSYRA(u2P;_&wAAY70GQWN9NO$?TD2xSvdp9dL@=WFefY8swl&Iy^qTK zd3mC0bar+5mLkJ+wbP0U|H1UrwE%MKBF2KskGnonj#TZYBQ-8IP)Ne?6!tczra1dG zlq<~L9>JhYy4Ue&oE;PiU#NG32r>2dM$ZvU8vTf8SA?EaUT>E^fUGon@MF&E+oueys(kUvn_O4v zFfuO7!94W>SDEm)#8{D~Fc|N$)h?c0A0-W|BPIDz_4ei>gsW$2yM&JOvdJ7$_fEFE zYDD7D!j9K#3OhEkX6fTH-rOS1)n9a(aEh@SE&Z(Iuh|!)-qg)T=HJ@UzHo+*4Ln@n7bideWEg zvOAzU2B}ADqt%kHQ=o1HDq7RLalh^M#)s3%<)K0wNF@&YKV+opC~c~jZq)2gdQ7aO zrYz+U7oc2YazQ5Gi9$ljz`y6--^*a+X0qS)g+!MmCaubDia$mfvXh*5e*qwWI=*Ot0O`UNf zzsnuOk8H)vaB_ou4~=yMH2ogB#>5riG%I#pAk zEVb9cOaaNa&-8C8Ts$u~lN>CMzj?`N#c^ zL`X;u3o9Sx&$PiJcW|>_+C#b0W~u35`7kaKLAAy}*ETvIPRXxjD-{(JLc(IrJ?WZ8 zzMN}5x{hadP`$<9t~k&Tf5+-^d%Sx6lfzi@fum)7w+^O?*Z@sJ@;jZM@G?M_ zkvtsXJ;N`&)71O;3@1gukBDBT)Ow=2MerF!=_v0)f?@l}aIKp|pF^&4?q#?+BW*pX zH5oaR03FJn67YQ@9tI#|JdST3v|F{1WYLc2)`ZVJEVw^TZ3&g?D}_2{Byr_L}|jxne{Cge=yXP}<%__gGR{kR6m z+PJsX!P2%SUIB10^De!fU~C%`izLw40DRG~Nn%YRM;|6Yw8(SOldn=p;4PM$5E|*d{l}C8<68cXJfy z&GA#L`!b{nD41$OFRPVXzUde^fSfgoP}R~W6h5)-q}hc0U~T%=v;0dU|Ilc;l|`5E zX)S_Ti^ttTK;{+PDyL`(`I%e2U)V5QY^WWX1MNWuEghK*Wp4JuIN*Ts?j>YCBD-Tr zYQgn=KGKKbS*}Rvx-y$q(>Q`O+{`2*{W+K^pg&*fQkz5)PXN~9? z@Unv=^;T$t&t=`xxFect_hp~$^5PzVpcM}}9 zyUJmXjd-S+G|MYLil&aL-$|8~ya^hfz_+uyBYmW#6{t$U>F zre*k+t{qQ5CDbb9owrnivvFfE*X<~9YgH?xKfGfzDbN>rxR^0B1Q@qA+w@(49yAPx zJ-M4aqoV`xaL&ST~03;+x(5k5WX!*3j1k$Dt>j0T<(k~4So zNdgt~N`%mn*)engmH>?)9cP~R7%Q`gA(zOX>BE(%KbQ?sV*i@_+QsCiUx$50t&hk%|Ta&WUW^|51wJI zX~NK`y-$i^NbJf1TnuNv^?J;3?b3uqD5*ZO3LzVGOmaej&4xu~*MvaKUq>q}76UoI z5izWJ1L(g#+uoTGTSvTBihLRE3Fj&4ie@^~2twI=hV_E;-Vp%RFGbt{FyxjY^~N!% ztt3UCVmLCDn)Q5Cx+&2(U$b7#*z|yb$MYi&y;`wB%cq4{QTj3tV3h}7<)1C5aTtF3 z?;#@U08=qE=^*qhd*3~eVo+|MUsoiQ)cA!MF1L!$Rru9HqFbqz2;f0K;FafUaoZ}? z2<|OjljH|W=Yb@jH!?gs+XTL_B!`jUtJ+L z-qz@)63nGV2)!rcdThj#3YC1c^9{!u6uGjVG)Ysnuc3k@G}Dy}0hXv|kVo&$BK@Xw zB=;ErWRBN{0dy;dac??DNM%_u<>f&Z5w1O7+7Nn$BM)javgJjFDZ5hoWU*l&BNdc$ zzL@DKmcPNb>*un8(f$#0i&Dx z0!$?It8Vuzbt5eB4; zCA7Yrhd)P5-;P`jq_~8-33W{kFx*5flGPH$q9iU(z^Y*pX|UU0bG>mEBgyL>Y|c0X zrfO!|vzw1n9fkRP%sp=Vbi$!a&|HAsIZ6|=HH_S}&SSq^C0}C%a5)EsiQJQ{M={KI zjN4zI?|4(}0R!XG6=r(o7DUJ}{qC}*KZ?1tE79las#$E{&xiq;711a6c=vJIRT^J_ z2{jMrA&=k=%zW7v;NCfUB`cKTBgNB6B|~I8E+&$6marIMxj-B&&%(JYfoFqDmW}f1 zy>W--*96`{c-(A}BC$oq%!=Td#RifjG+({shFDP?8hiE!5eTKp*G#nkB!Wb6F_GCO zjGVjthIHJp$3|U@eg_#O(0(N3mVPTF*ko@)!|J#wSF=_=hHRsqr#$zKQL4VZd}Ij# zbA2#PG6II-1sC84^=$*IGwb7iNdsUy6Pvo-Xe2~}UV*yU1@Q}tc>pZ1j==(L3usq@ z)V`u)T)UuSyNE2KfTJWEvM#&`m{Tp#8o~d}qE%_zY)@q*4fSoXR*n2$0xJ1&KQ4RH zldaFpgt|Kg1LRh{?;bbv4N)HkvL7)=w?B)5!2*V*UHx`t+rr*A7~gBdMX zhl}?hLybUEC$5WaQwP~87W1~sS*l|P7o&DZpX{dfE!Whl4t9^N5&j zHt$4F_pbpyhVBU?cxU3(NOT%llVTjA8^b$GebR!xAII9G7$@W`lrwJ(=~SNY`wSB# zYKWO9e6&c%9r^+InY~Gmjq5cJ2Jt}qB@Jl2cIKGhY8#NPX7W>l_ef|COibjA4-yap zt}Ovb5tm_(ZKSLR4Usc6_v%dn_*P53=~t2_tDP)V5!!+@$ik@4%W(->T&dM2g|h=v z((wA0VjhqecWkFcYM^(g+bg(Bt(>b;hD7p%g}OG7io=oX|0RM4`hqp+DV%*iQ@%5%JYGzL~LSDIYGyI+ z=33Q(5&lTR7A6vnflU5P8l|k-HV~64%?<+{DuR{+eS3giKSwUG85mO;AtS)|*_E@! zQ4tN0Y8lX1H5Be0F)R^0p)}Y^@9tkbUdnu%2Zb9FvyX@hrqcWgbYu>q;=6J9Hb~Np zyj<92U5g($e7K0j=K!^&eAGhEkO50Xq`da;$R7&2@0iu>jatw-3pjk2S{W`*%TDej zk`=gE9Eyd#8ZKg6H{o@>cE@#dI!M~AvMfM*f3c?t!5Xag>FG0A^+a1m7tW@LTXLB) zl$tfp`Bsy)Bkj>ldzS%~i>d^Za$Va8Ywhpo>8% zOOM}qm36n433??+L(MP;q*DRc&km1-FQL#u0~x28o@t^}blu5T$3>4{Fr9fI(lBFZ z)5xdX^{0*AM)93M$TgB;77HJKBjR7qN~67#_ECHpGdpK7kt3(AT|+SN)%B)xJ;!K$KJ+$v}Lu>yO!sSnfwlnPadnu9Vn2 zH3tib(1f5>PjWud?YN~MsU95EV}hX!`)GWL>>oh z#`Z#2#Bec#0wd|WnF3&0^qbkKXd-b@5p)XjsUQrD{b-}6F8+*^3aGqK>_|3ME?&|2 z4#UC0ZGivJ3%>Bl#8>ar4pYUr0R(2q@K}Gg;@WHoOF7Vpqxlt+WXSdM?(TGUW)hNb z7>>LuW(gJlVtoIFQ#_X1Zql#RjW0J*QQMvg+`f0FIdsH!Tp9cVHRVA(w@nh4#o%p& z7$g#aa2f$!DGyr4iyf{$A`BRy#PR2X*iZ&|cG7JjF-RJC)CgywWJhu66ZAqZGa)B+ z81BNbhLz;((!42G|HN$!X!MRLHNeC{fHR#~1>nM^s2>X;O>lPl>7?=UZq{T3 zC&v3nfAuTKb{Sa{FV-xLRTK(tHlPA&Q`-duLy9Xs-$4zx`OK3q36wmSSmQ$fvcn}Rw&AY<#WgT*GZ5u#1lV;``)TKlA?pBpuhg3^OG`CJti5h0%W7b zO8>;XXcZb{=cb&d7|8z4*w9q8I@Tdmlk=dusrahF&SDSLpbxPfv?||xFWP>iJ)?5^ z-o|@|Me&qAH**f!7A?E5pZin7v5A|%qEvN?kxCIz7D6PQIJ$>rLpmi$T8Vg3UgTAP zh%+9>kXGKxj{#L(C(%!UEr%dMRj&%Y7O3-tR`_UsZu);D3*S>BpnU?78laF={14dcj>JF7RIz@A3&io1n=`2 zYD_wWmP#M%O1#I@0muCFGvdkt>06P0u6F|I#}B)ZuT(v{PMb(C?jU7!Gs@;ic0hnFRn?;iahCL4XZq1)8ejyVEAO< z*IV+(K<%;bq=pWOu@7t`I`OS>x6bv=?9XgBWtN3g0oI<%aKNUL6jmKlbH2079!j9Z zK$?AsMPcg`bxW}o$ovo8hF|cam+CD?-rEt%Hb#S*->OYz;p@)Fu$e{6G3wP#9Qgz! zK55^i#r6-GN$xp#N1qgj81Q`^v61y-?K^O7UyEv%2XbrngQer|7^}60o&q`i%e0aB z@5t_(o$p((qt8h7`aJiSQ&9KrZVzm;y4C`^eVtI|$Cdst>Y=M*ktAyJ;w~o(-+fXH zLcEXN9V1s}(vKHW z^lIC1NgoI0S~v17UzZT0Da1a3m6w51Ykl+~%8bvw3hj0y>ylC;GBG3V4;e|JHuZ-l z%_dQ*P+n38f&L#tXE)F%5R#+%034Oc5;lWMo*58*PdXPmfx7=Dm#vT<+wt@b*<`MW zO`dbVyyOVX)+z|>5oN5@Y^g%$67NcB`1Fuh(NrE{#dZ6oF$IBKv;98Pq+3okQknIYLOBwVJHg)*}iYZ#70 z8Sj^FuK~R}(uCp+;=w4SergRWDKpZlfH;?>-j;Q!_Il2p9%~k0GAZD*@8&oLp3^?6 zZa`6~(GcHhsA!jlHG56RdBk`Sy0@Va&hG&~-+WGZ6|TJSjwazNG!-&nBjAj2pM{XA zm}v?e^TWg&XC@BqkM!8h`3d+%YeN|kp`dqIyZzOn(A=@B(X}m_Xgd%Dd7y`C$iGfO zixnhRx&4*jYIC}AulEs4)$;O1Jhuy%U?rvMq5U0UyhO;c(0hvF6Vgb?+5qxp8X_^o ze=gO()50O3M=FJd`f=KB$CEGE@EV_W^i?@IOz;Vmx1mbTf3^Tp4H$rl{S*(GBJHRxy%e;fSM;*ZL!ChV&jq80Q40$dsBhLLR8zAVJj1)mct20i&Q)!xE^7V(r+F|7JoFUn02+@{Nr&wxk&6pSI2HYsl7VU*ht#8)U6sj1L$P@1i4A zp?wg2HS!q4nS0t5Nt@|};?8E7z71~4GV7G)o-kVGy*wFxTYWa*Dx)5^`_AH~c$L8s z5RB(B)B2kVL1AKxc;qih%$X?Y;T$ke#$2G6{_B?z^zzOv2vs9BdiT~(FX*hM8+@&u z^91~0xjE>xhh!%t!QLERdQ8fJ4i+P9;?OcDO(=pHd8`s#MP=Y3cG3p+nicD zRTq@;`6{^TF6TrF8MhM;9dn)>0A*tYy^<}eJ!bpA#nPdSh(8_Yd6~LLe)09#Sbb0w zwR=UI)kK>X-FAdJU1BcKKNGo6GvbpLaD_2B3ngtmE#Cd7x!NX@Er+f%SWUq6D+Nht z_2Kt;pnt4G)S}6$ZklDF0lC-(Zewv1T`=-aA0 z)U&lBO3$_fzFOB|*G5krQQFhs9|UTGl1i?kx&%*;H_dUgnqL_TpX!q(bk>@2-77%YzYKjlIqU&`yUqHJD;YLw^Q&lPui(%sc4L;9coF?^>;UoCDnJy4UqV4#? zLwfJpbk^Ffg?RvF6Bo9?I54W?v6enkm73kQ41L=)gWnXOwu^1=+UG#=EbaKgW6~K5 zc>F-t$7jy_vee&ti^zO5vfjJ|Rot8BxOX*_lI8b{<8s(7K4}BepmnD|hbn>Zl!5%b zo3e<%0TBJG!GQ^2kGYtcBGLtjRIiu;Q!+1BjPN6RIKva=clTd(zW|KRlF{DzxCd2o zcU^I#6$(c{4yT1SzOd@!P@vD=v3?I=ePpHaO zhh=i-WHLWC!`lycsW>NdMQwnw1o)c_H1>a6d=r~L=aWDd(u};vB;))+=VQWaQ6b6;i5(YzQyFtmXsw& zOp3Iw2Uy7>2|9SYLKA&d;mfypJ&zor7c27c?v(a#-O(wa-UAV8mO(ji^CwQ201|N| zf=-h`@$TR5Di)1w4}o#%>DBJ)A?c>%r<2r;uavEnc!@0r^Okbl2D9XwCRU2k!XDPj zB3B`9jt`n8Zs~Pg9)`@6t)=<$=$q4|6uItJLO_LHM3qc@1WaH)^mvVr*S(;NO!>%* zTi$*{rQ@?tTDU223sCZ(3KH-PLN9@&wvm#fd91zlV>Yygj-*A3N4Z2O%G z>qS`KY1E-AA9B}Xisk462hG2b;P5+*8KH%{PjwSYi!y5qT8~%SjZ?+9uVhhef%&26 z_#&feXBjM?}} zt&ZB^q`{#F64<_bXuvi(&}MGNj}Y*}Q{ zxsud*9+@S4&!gptOhB{CUAp%Xk0n&CfabOR;w{;vt$7%AppuSaoeJ7_vkZ8PN6Bq! z^+6=F3qg`GAR&OXg!*m_=-$)h>wqO~yK0EDc( zh-0H4=6aPVjSL~Q7{;KJxooGM_AH{#uPCu4!btS2Lm^#Mwm&Hj_+i0Q8xX-bW#X#1 zyAJrg`sxPRkxU5Pv~K6$cTjEl`v~9b*!H;Y>Q#Te>-5sxVI94|2l~_nLgPNt#i01N zp;Jh#HjtL_4tmaG5byA<##qRR;;02@7C81!X4xhie9^^1qM5bH+zTcZ4Y}4z zKsK|kgNoaROvW{5&D1>-h0$2+~5gu+Xsu{r=ig$kY1T!#b(0{zLmB2qqCPgj}+W zcS#evYGc)rAnL^Xm4X`P*r?p~pC_+U_7i{1wb2WdBVB^q4wu~lncS~Tx?B&Ax0E&b zv|!zWKexB5o7rKic*sYRD)SGfNt)~XpY~63dSNVVyt_7(7rnGRpi0#Nk?sKnVhhl5 zVK*P{$+?XbuU%W&E}S^OP-q@^h(q~jo)L|^1M5X_*~80bpxYr3pZt5cCTa_GzizNs zpkS!PRMrMe3415CuXRA{s)JH_eP?|9&W!4e^TI5lMVGA^(G4lvj+3jWSEAYGgi!;O zj!$2i{TDw~sZN9UzX-e(pXLLOz~^>%A8A=`7G*5t+{-abgRl;F#1au2F3y z7n~xLlIT(6UPb*UG-*giBAtO31c_Co0>z-5^K7sq zjEGJ$oKqNkJr6cMklO_zh3Z35+x?@rE|YFR{cR$>S*N#S|GOvelRm1lml8qNbOuUV zj#GfQodQ7%Y^Df63pq;7nNAI^TnwSWM|NX08`uYe-?em_z$Xy5g%iN7ET&Rsg6Tv& z4pJMlujtcrRpT$cx}?B_>(*@j^93SfJj`oD2E-+0AWBc@MH+qx<7NDJara(J*13#( zb)z&15#lsVlLll>lYb#odO)>g1` zoeyX;E`Cz+1;o&cVDH5aN%nIo5<{>mL?|M5~no4(K&0?nsU?P0~ml??E%6TQ5OWtWxP*-{0D$F z)}*y>7uXGiVR<<;byrx97(k^90gC(#nbSapdw?^o4PK2TTz)e1wk~V|EpTL zP)G0yBI7rMh#Tpn>(dMw@VotaiOg0r$YvhG@?8gu#+X4tiqa(_K&^gZDKZ4~^QFji z9x(x%Ys42`el0RUeJk2f?4yg40OL&tUCn7DY;%h(Vp{W1onYSjV7t|{Vgtz;(IxPG|3 zAi2t{mT^~zp;Rmc3<+%Y_>cZOo4-Ez1efQ5T2EVe&VvhCHc3^o}9q+AOCI7iq*xo(UC@#pXQEFyJdE<|ZyDuTqcyGQ1`1)t~ zq^vZvw~#D89w)<1Ufb!0KN;*Kk9A(vE1vyX^2k~%xk}qpMg}TCs1F?p>R)3j*MRA~ zI(!lLHUQ!I8@O@yCQC8DHm6f@t5<1-oK;{c4s($e`6}K$}Dd)9)xiFANwC~nxR41mO!Ys&0klrro|7NEVEDz5=DqY zDz!vz$f&vX#v93XcD(t6Ur~NC5idg!xd^lkz)>873uM_{M2`bv%Mp!Em z0$cZktDm8>h5ZH&YiKzqHFk3n$3WI`tUl^MHbxbp{ghW+V_z7cYTGdQk>s*h1LKf- z1mv=G__EYE4vz_1v%z0o6%7kcR%Hih-gIc8M`$FRp7Q*A!-it0t)S^@!ZruLmOlqF zk`AqXChNo%cc*zp8ENGMw5uLYLCw!ZiUxB?hg0iwT1q)&FJ_*{J~Ys*zxNTO;LSi58Vd*=mAd4{h$9bJT+%Tg=c`A zwytsQtr>=NCl&~0?^*wQ1&L_USAH zhAz(4YM{e!P1tyilcKhr9OT%`{JE+kO@ABjxNjeU^jBUn>E0o9vMJUV!&(MAztoXZ z6L<*LGf?Kh;*eJ|Unlc!lU>{6r|LJOqvD6=_Jn}6jeAM(P1<6hA0kJBPkb|3QPP)R@Mqx+| zyJY?ppXUh^0e~>6zi$o)pGA-~a;FI7bK}DLqnt#$)+@*)stN1DlH~h;TsyajkXGka z*g=55YoHX-nw%Y68W4cl^bXutL3$VW|CmK$9$yI8c)yT#eUKApg0_VLa*Q7@!HZ%0 z9oS@4t0_MAcNmiMUtEJM|yd(q1 zw4^ukG3MHTrm3Xf!%>~Sj?D^8H)|6~&cp;u7qXcJ*ineL1L&(Yg4*Wvc96@OuKLW=-8zuq*GUovai;lodxmwWg4~vK*x+4clzVdZcyKGp4vKgv}?VdvE z*ezP{vM4ya2WH&|w`Fk~x#v_%KX%iKp;FMSn+6<~$4GY>G-ooft(^^(Amp#|=RBe8lY}O%a(14GxqO`3}iEcsRKNM6_9j{hgAS&t%0;u{w1EfMt|svik^fNaL8=x zvCsb+03%|#T{D>H$RALzI6Ou0ksfU=A9>MMSOiJ>y{PB+vA=Qnf{Hr~ZYGoJDKMb^ zgdCZnqgs7Ej#>TvJET4J_3@+q9~n_YW$wGn%)*JmhTRu#2B9sBXFMDLEsq0}v?+el zdsinnWdOF*;ORDVqSait1#mz5TeIXlf_`D*J^~RezoDt|4|~^5A2g&|lKVQFW^TO7 zL9B`t!P(7f?zzN!yjmhXkPe6?(i=kv3@1o70go99@s~##YcTE?QCpB8aZ*E6PhlUN z*ADFQ?;&+nO{d`+J)1%rsNN-YJ;eSd?hBUw?%g|mW~gwXny!TR5iz$Q#T*y_lziV% z+(u<_bWD|tW%#FQ*|zSna!XuvEiB$|`D%Y9b6g*vfQKyF)tsD48VH=H8CSV*S(m}Q z_z0=u`iVm-7ooZ=_mvreCijc^=Ud$xXRCk`C{V#GZfa=~_xT?3T$0a`2L{m1gYpNM358iPq#l44}S44J0UM6VL!yC_rDEe?VN6f-A} zX+*kAmftjIgf#S;BMGx5vfqAZf$KNr-t80oWJ))dh7tdp@pG%F0c^Qz5c}m|=xha9 z#(+#nJ4^2Bz9`3HOP0^m?>&(FZ15I4=8&dH*x*cvaP2*{!WUDqmFM>K?Ub#$5H+?A z@*5#04Wutiepk|m(|>sw=a;Pkfi^pUy7w(7oiQ;$L96>cGi0+B69M!;;nY^7k6$P*&mwxd zU+808KC@w?Kmy8DpaN~FZ*F(rvdhAzxgaSbm`ZGvAmH}i6AV}>WyD5GHd)|bBF;M9 zSYG1+%_5%9Dd+zY^_5Xotf}mo6fPg5i zGzbEMbVzr8bDw+f_z#8xHP@#r87)Q#r51=CA9b^5*O$yQb8!%N-9x~9 z_wiZ4TBX~9qSp=;{Tc|898qXE?d#$+Tv1RN;vmMfvdfO1tT5&S&=-KZd+)U605Quk z@#CF!4t%hJ*E#-InE|v37?ve!@`?k5`7J;8JgI-Sf43htRe)AVWNAWnZi{SBEvttiw-m6{n1!5Yh*Zs>A)O;BPVh zsqUEj_Gy<-x&2-rrLUAKqlTqvu%h~mTlr2TNzCN9)4x0W2Bw`)g1`T#OGFEIAse7f zv90$?-LTAhkyWIvs7COC8=M^<{}mCyC47o$jJ5>ezHc*)n2LF{KDoz0>H~sGW9FRd zkMuG=^`*jieq2r@&u!vnl0E$~rRF1;iX)g50Uw~MGj!s{QQ|(Pnj&8q(O1C4JWw?U zIwS@i?3+WF=KB&`fRrYFxc$|C*NB83)Ahjqhn~X4!kAfjZd)X_On(%uEi*zBkIS!O zrY+zPVM`@v?$di?`&XxMSUs;eN3KaWtzNVMWb)EF`Lu;rPELp2PSV)dxzuJLE z6O3-G`XfdcAe$BsE}Vi<|Llz6ErKW=sKXpl3ZDOM;gD{`@FxY7qHQ7sZl1D~B8Wji z4tBxo_T!$+c%JyOa>HT}YG%?L!+cJs{z@!#47HO)q zTSYk(NbVTdel2ibv#QJw2*6a-z{G3M-vx;tc(PyL>373as+%&|jag;(Iv=7St_AKk5Gpuv#5c%H*is=UePZ&#%8Z5i^ zpL`FT4Ovc@x^l9tRUgO)`G6rekf|8&15C~3>){nYDzZx=Y@rgq2kAEZhkGW@rUy$X zuM|Tfo;V0%i#8i*v;gLnqpUV+)y=B%3YU?O`wX?#-+Ri- z`!Q(nXlGzx-54TtDh$BQP-4FmyNX1HvX08!X?Drxw^|G6!0Pq7f+4hiD(SYKzk6Bi zR%~sr7bzk{BH%yV>%UQZ9RmVH)=oyzn*ngFs(#gYa?;lX6OMn#+_iqjkev18+-pi` z;Ul=mS&nj{aSkh2Em!P=d$BG4U^mwrlG)i42ngxt>c1MInoMT2-9JYw8~@~PvlUHg zy?uW7o2I*fMD0qb!QpvpnJ@T9{6Yps_ChmppVj@rpF-)CmycHVFQL9X*Hd0UQ|uk^ z;Wn$;kOMV9;U8tuRo6FmZ4}#&xngUc%?8Wz9EH3Q`g^quo#f`n@`qBUfgl{6@@zpf ztp47o+`1RY_xa(%tE!rW%DE#KuimqxEyl-EB6S={qE5Dmch&#LzO0fA2&yY|%_dM| zjNUg1noAV<=^;m*RphFN$Nqeo_ZR5tH{!!$>a9SNQzDyinL;QIjrT+7)?!NB-yW8o zTdt9y_@TI0AJpj{MS1_59Hmgb2?mVn@}(;O9wImf+WAnE*6G(mb-b3#t?jb!c*Zxf zuNULB(XgOwc%NO5!`1g9Wqd$CvA`pu%kcpbIMge&GC?)dt}T}())xx8wvV=L2pKRh zu@ah(*B6#Y=&k*zo&m57fr3zm4%}pC!>-*aoLaH|(=(3h_{`F7ugBN~wu#sG=Rswv z_J1q>zo^o4^VTib`Iu|moEZ^G+y%u!(x;`xvth4ZzB1%NwOyaW64bfg90L@Qd&w^f zxf60Iv#KFy4f8N68=}gzwzw=0ud+fC`N?g1qkca5mb1*sBlh65LV)n_<(=y*^YBKbc0LAcbhhct<-^=n>cq@-zM9gyEDC>VZsH5Q40u;ce z`{ccBs~h0)Xk<S;TE2(CwXV&3~FE7oVT zwbeaS8f(xwnSaF^@p}dgth5i`me0MLEE?h%AO;v#yUU6!1WlGBZbsef$MN^C?=pQ@ zmO65%NsD94LQSAA`=7y7?mrw#=b7{?tlm?w}`UeWc@}L zc=R_C+ADT_@5}XGxM$XIrUtxt<@+kZui{f#fZP!Llz2@Q>eYtNg8;D1EoEDqpt@WD zK*j=&r%p4iJw@CE%4mmi02f|5mu zf(_ANg+iBRp#Pvh%^v_ftDo^QXZ7xx)KnALT>N03^a{OoUaL3*3al-ZTPIgOo8m7% zfLhpvMXbYn!xoZe2ja~z+BBhvpUaHPE7Gf2wp!5Rm5k4C)b$^ppF8Htr%>$HhFuC& zc|5rcU0b?BBhDjSQvJg@TqktL1%O+(e4y(Hi|F$XuiYvP_yf#a-n!6p!)=aziM}hL zZ&o~)9;-QMfZ_S}UNGLa<9A%z#2;uMyt<{1f*I0D25KuWiS>Pl--pPJ>S#_xuFlE_ z6xv7)9gyp%5KQmmTME)o#rE$h&{+UUp^dxg&13T=NI14j8fSq9ebHvGD*y9LVoxvC zf{_ss+v2?q@b9x*Rx^zkkYmHSzDZ)<5?m>B z^2@To2j_mNv5*us{c#O(S`f7U`cqy2grxqv#R_Y2Xz3DYiu_F?IY>0SsPBTC{PQ}?dx z4y(N{3LW{KDj6+Ja@zX)9}JG#&dU{@_#4ea`{6Yn8?$m2Trx$3i_)&OsaeT5gReA! zI&9{dt$lH(1v>TX@_t5<`+XkcHQZIhc*(7-DYAcuT%05iP+qx`>Xkb8@tS9}K36Mc zDU+bhWPpn2O|v^Wf#+piXDIss_C@bx45a77(j!i*aC2w`o7`y zh$p#=G6^#^4C~kYiG!|h37w!CcYDKGqfQFrtm}VbWJZJov~KlnHQ+eRkwSX&MJ;nf*;<*w+(8s+z13Jr%MdVX>j+gOl}o)1VvswOpw}gOF;52#Hg)p;qy6SKo94# z_V#6=o%5R5%W?Hpo4;ZgRNi3Y_&n|gMkLR6Y7_Ffr$?CktYI-TXA~9F-7B%j;n%P~ zGatmui^LvzYOB66kU_y{`1eCW6V9J?3(yJ?v~`iX0G0JR5H;_iOp-rP7Uq<&8q5Rd zk>;XLgXaZv)VjJwcO$KL;g3IN?Gȉe{(7yBvyF&bSLh!7$*cQ8E^7PYj?pn44@ z+bexZac>?0Oo#I})<*@I!fEm#;q4Ju1{1`A0;#jS5!kuU*K^*2{n` zA&ATDP&2fbAbl!X@7tPK2Q>YMB|ty?l1hZ&`SyGQW#&%k#X$|RCggL;g;Kp=UbedB z=NHni-mgdb_wviJLa~1cp?Lq50pHwX^Bj5fP~*7eE6H_h#?Z{@@~`v(&&WQ23chDg zJ)(ZO04DkwAnJ3Ryq_-Gg#)bq!zjFbw(h;n#5l#Ho(&otg%|CFGJSiINrhKsp~o9? zdDftT0l~L{UZ}{qR=d!rMKgP%3o8rollOV&ag>v_nX_^V7`eS$CvsUVa8 z=vy<2k1)+`UUY8#SUG7a`6<$Lm_H?pHVRdcst`KC2VFg$O}sG;5peH~42= z3v;wTo(hNa>7BnsmR{@Yv4edzu`0jqEKV^CLofXWj<2fw(Joi0`}JO=uM*+BO&-~z z$mP5GbWw7(c?#m%?*tKNa$uuB)tx<>O!D^Y>upDb)H>_cw$vCXjMpwxeo62b4+$Xp zlw71Px#W5OB?=H<&Huo3e>|+)3?0F2%kLMJX_ZUu|4j^Xk+2-A6T?uZ!}S$_e8y zSb#nQ8M{*RpM8mT#+en@>Ca+wBxlidQ-5x}-Xh9BoB@m~YQ0X-7iVS93}7bZqUO{6 z)%g?nvKR`UTKO+#xdn$hRY^dU*fgH`_i1!|L+;|4u1n%m{bKVgg$w42!pqR*u_dBs z-s9Oe4&0l}C{xVqvf@?fyGecJZY=PjV^WSr-X5!#@MQ;v)I5*CZ_fqL;Uxl(Wz{_f zp%uc=5!JjR_WrO>q~B|O3plevL`p6}!_R#*bLM}Dkhau;l-!WbhXQqTuTBf*%=mq3 z8*R*E`sp5oeka}156|z;;0#=Acy-nL`J};2PJz<;FXvLki!}=R#El(%itn;47fFK} zMYBzRF<;ur=jJw;w)XR^TN@pCjHIrb|L9-bI#DH$xceof?2{^ z-SD}v=b-BRizdqEhwga&(TgVGAcJn2*PfeAz_S(Fl>FGs2#S?AcmDOEh;C*t!^P1x zviey$=6M8RO=op~W+0-WN?GTTqp}YF~Itblm#B z0|a9LfMj0ltB?8k@$18lv7ebr^uwD$}@uhWR5s6jP}rYan8u| z5?bF_d8eLJAgg@$PK1eLvidnF%uN|+F>&|v)*}O8T;~$b?6-E*#SY&f9mqa#3z_4( zxX{=84lPs1bk;7`*{G`(k;bC4u|aM%c|ly`-5U#d=r7z2n1!tHA#+Y~s!&bZoS+-p zu8)>Skl3^hP+}p^&yr&LsNI>$H@5oKnzhe%|FEBi7W_^!4$=y>F;?NQPO&_Q81^pT zGwETzJwvX4o2xknFVh`(!X?cW?1SReK9n`|G5!7fsC1c)NB62=+CZRVw(fJ365$81 z4(eo)h~2q`$?G)1`6lEdzk!DIgUzNrT}0DFXW=1VRczX>2v+S|?uKBv_kC3!=op`~TN|ZeEqaZ7#?$@+0P>FMy%2^Q15V^70%Pmd{jS{l zl(m%-hQL@)eeU^F*@_0w{*>7w+I0WQv^L9d-RavC>z4C@jzsGDpF9TPR92yiLUb>D zXFCs?4tjXpt81(X%Jj(1O-;OB`@)2`{Fuaq8{pE4}{2NaHV`fVur9*9|~G)T^m={4W8Y26taUb}MwSeQr&H|rEY$Bf`Z&Hdl(1@JV8*R(Xq)uhd)}>|Q ztiyR0(LY!s4- z!>2fki&Y0ibg~|baWh*RSxXYQk7O$^V&BK{maROOu&E7Btx;~@oDZHG7kh-lc^|6C z4MCC1zY_F3P9)brUHzZHktnkae(K)fn%=FVx-4O}&_Ls3wf0VXG0Zm*e6}?{zqdc{2Tx$?Vb)5sdg#N5zeqgk-7gs;wT%B4q(;n9jC?A8eI+tQo;VYgV&lb< zN)eO!gyVo&o1zAAZug@A5~`o70=kLHgRxGFOV?vQtzP|#=PX>Q%w|RD%YM4m`EmE&>K37Lg|E7Fs4Qnt+=mG10GotSa;B9(EiH}KBWJo4_Z|fD7pEy- z!@7=>L}!?+bJ1#}AK`pzU=n?I-**LEQb)$oGN!`^zN;^?KOYdv?E|iVc%0}xf#@&wG#gwuU;tv3w^e=uGTnEWm_2LKTjFb7%Szh8U z1;pBfIrJ+>5~?RNq{03~uT1wA<*u488v{A!jn1U~IQcqBkhH1RDc#)F)GmW)SEv!wpP z5al%pT^}XmvNlR7we_p^{B|R=(0E5S=PDF2*FYoEnwGY83YN`S z(S(;?Rh=qcJ+*YZxvj9W9;l|Db((3Ut9jhas&TDkP!_oMzDfMn$(xio{pdkpI%1m& z5sb2~q(1(gmdeeC4@o=GxzCuw!Z1XQ72RhQ)18}1R?Ie3l;5WI${9^jMl6B)Swza$fYp|Sk;o8mO4Vc}7@~8j>NmYSDg;ige*9MRHomjo zjGw%eSP?`ech}iq;N^tvZew&WX2}MD`hEtwN{d!VtEe)O^OX{WKkkm9l2wQXR?~M5l)MEGMk*9Jd_!1 z*!=>6#d^+Ppj7vjTNPnb(C@rIMDzq4BnDinX#vq|*tSh}fzgy)hC~}G&9mZeQ&gvy zrE^U%jXl^%5|&s*tWh)WYnU*AJS%B@e`dJ;k^Rvpr6Y`qdCeE0@(^#9U#E_Nm)Qzo z6mr4lkmMB?NTz?8slv#MPp&JkFacwQKhW2{wT%J4D@zZ61j*~~mu@aUg5Jmvh180S z4LrC&=%sY*f5F-y2#;VA1YGyXcM0r_@n@?1O{s|il53ruG>CYN{dtE0G68Er#Wu~w zhrLJO&Z0A1HT{EHjr|9ED%3b9V^^0?rf&O-90#(uca=6Bnv= z$%J3tt8Il$h*d-Mm8)PNFG(-`K7b%z-}>%>$3in_piO?Y+p{ox)9>btfDPTyujN^E z9LbNK>2vJMN!x#HA#{bKje`Bw2fg)yL#>GZ(c!O?{1*p!$3iYuSmyX+I)Uc?6)8Vg`}BjCfbfmPM#yHt<0pI?DH? zT!b#V5Cp+NkMJca9IE<$21C^APlNG4fD4e;)bv{`mZ(VUvpt{u7afpdfUb36V@5ax zgf-jT$B~A)RAs_P>p+5M92G|0ma2P`>a*TbjFRCq-9r=wALgEPFQ9Uomx#7RTf*W_ zq#UXNM7;SMo3_UiCP@QYLEEqs8{U28>=QXgS5R&)5$SbXa;CSduM;gkx++@gFWO3e z4mzeKGb2$2A_GE3i_4j2h3ZaV)$BsH1--T)OYpVjr%6!sZMX%{X|K;*Ma!{soi?t9 zlHg6B3&@l_P++V9*0${&uXSnpo43U4Tgpr(2ypHQE412&7zhFklG`|D=h_G(?NX?P z@7BesjGi(R58{$&WQk}(B9!Y51jleKdB1F_L?y^vVbkoX|NI4NzIX4Q1ih`fOqmW? ze=Qz(=;AAnGEAW|xX%X(PX3;IuCaMtoP6i~!I*R}WKhmG_`A%FD{Pe`34`&HBBKE} zFee|Kj4^hTZQbnCQ$qeBI9D=QT7LK*V(JFZXFk6?DpDzOhfrpd^{Ixs)xB{D^61kW z&-zwq&OVK5SNVQk5_c^lP=f8iy6w;;@nGUS;d=><{mbP)RxcC|e`3tM^1rxBS18JuRuT5exXEUyLe8$to)%a1#4XOSOO}UQgk43LtlN<{?@f%>wg$Le*4cB>@3?_K9eot)WcXQt}JdqYOVLZ@Mw1W z*dY6cU(V=swmfPv=>KRwY&caxJi?DuhE1+*MKCDmCMAvszc(Mi%zC~ zfEfs~PGMwOL7DsrtY=WTpIUN->b zyr4~ubE5jNooH7&_=BW^2eJ0x{)PI%7}){FC&vd~jM_^TFY_xp>z$OSc^LVO;#)>( zt!tWgwHjNTiq2D_e@S>7dX)l_O9z6YYh(wQ<(V4HEtf*y4Q=I^yd)v5{OjPqu! zymF^Co6259hMie7)3^%XZCHGo0XPrM$Nj9oi9$=%js?tc=}`1|)QApo)>H`GLPO|E za?G;%QP72JA;T1RAi?M+;2!mY17B83pSZ*fU^8TS_~8GJc~j;Cu-p?4a^lhZZrx-m0 zi?|VNcO}_o^QXG}e@*IPfLk^z!FrkyU>E{y!&mH5@_ziwW!l=mE*(r-%BhBo8X6Qo zv-v*$Qu;XMd5cxO&UL-F>Kghc2?v3gR3g&HfmTB#{CP>iN7!r0TP=;YYkC6{eRL!P zT+x#_A3&_aacJ>PB)F5__kBFGY{4Or`K2T5xH718!fJ&JP?A`BgB(?2oooYS8JZQ= z-$%nMFm=AzGdx!zoB{}72vTaR|3wwhi8`fLIAmtE-SPzhUui$w(hXxJQs_qO2^wkI z3a9J>vqv5E7Dtz=j+%3Zt+uECdt>qU>2IfSI~@wJYol59$8ZG6reEBm+#|IcGa<9c zkBv`>y=Z{}PzvJ~JB)BmZH{kAH(#!{0T6q=lJ%XCrDmMI_Ed zkm?YBFUu4>#6`$%V@ipE zQ}u0F^FxuU810Y8PX)&-P+HM`ZX-S!`1+1i)Px@jr+ySqdpfEj@p7CI$Xn4w*2C?Iy9TWQ-hO9ETWxx{MIE7 zToAWR=1E;;<4qVKY8r`~f^4=#{2QUQmW>@!fb!~J{M-j(_Xfh)OxYZFv{$c`joQih zEC`g2Us+PDgx%CndXLonMI-sr{xdkUv8F$J<9G4P-2IBtZmhX;^>5^gClCW{n8q?! z4$b}dSK}Q<$Yd;qY&LO5sL^M9??&W63Y+x$p)|AOc%`nFnMx^6(j^XuZUL1`-4F3G ze$xB<>*E3gxTV!p#|eL#i_=Nbg9T?op?$t?Z2e*^a#ckKiJ%#^`sgaU#ys%G<`$jV zdW^~Y^aVRswL~0IrEh|E=M~>veuhxnV{T&pjE>RAUh}`LT9FlMBzMdEq)6F4oK>F` zLv(}b7wG;r=4!18!au*ye$Egw&;xOicGsjGy*P|BxWaHMIU<`l^N-YMT{AzI8=p&i zuyEoqi5N5T>$_iLr4)NUA*jXKHo`MlLn~;Mwu8)AHy&OzdjB!qf>t;^o*nCQHFt1$ zXZZ@W5ds4q6-ebC!~x;k@VbCy^4D%(=Kn&ZFPl&3gSg-^?Nv~};2KAY>UX2}wpthD z9a~pXqaTFfF_HRzL~1yQa1?MohD*VbC;fPlxY{A-enjF&WJN`cRq@>C_}ai1X8L5` zC)ntgRbLpp{1rCKpHOQyFq)_~X_2xeReyoz>3aL^x`|IO$IWgG!l$XLAT=Wp#bb5M zhrWo!>07jGx%5|bi^}O9d~{B_-t{1VSNd=jrebQ(f8Tp=Dc8j=9(>hM`QA(#uZAQm zdw%~=?7diyh@)=lx@AQVDyCtRb(52#mr(TV{^-xRD#E*HbK%RS2_MUw@LM^~1x0>J z%Wg2StWGtG@guyB%XRYaTc^>CIb0Jah}U{nwtXl1&-EMIqS54G=HvR)9k1Uo@(EyU z)7?j;q3;5mOV&uRoV-lFSnw+)x{LA#4!QRRxAscd`!blydf-fBxyi_o~p~dE zw6)b=_{9)weYq{bzhLso_bUhBq4wjHB{#CId@R1(p|yPm{j|B|Y8*X$VJ2da@gImi z7-C~B5hZiuPLySkUA= ztLazcjE?Om){upT{uSNi(_o=G_avhw?g9>&W$xa~;d=BKBOxiwS#FuFEw|u#ORGcX zS{p^97Qu`{ztn>LcHgoTP8w#ep=%mF0oJ=T-)%5}V80S@UQT>80O6Q7oJc0ZT08WHqhL}PZLu~5-NI>R(BT-o4>8`XY<^;rT? zE1zyP?B&TIk`mBV1|Tcgi751b1x~m({6x{MR=(2|)$fP2e3~B|?j;Oi-Gg)#n!Ie{ zXO*CZ!35yf^KG$qUkLD)Qo|D9!H%>Oqug;G?TZ|{saj-^kg7NHHKsq0m6z;S=Y~~urXZ*F#AWFoX>u4@ zGOmK6wVXN=E)FsknXrwxo0^ldq(-{-&zs8~L-e=Do5iP=lisnmz76ZtLlbPgoaDV@ z8!by8%w`;KemRV*BHIi?3Cht@9o0`A;S#L-#tH%#&25OU$@Es2VL-RZ6F{)-&V51l zS3(9lOvBdLJqX|A`o=>RxpER)m^f9|#JauGm*v1oUj<2> zsCnHNp z2HI*X%#vim3%*gRJ@c4AXski4Js8 zUX4Ond^A{{EDX_j2Er8`cg2j24yPXLO+X^P3fYsm0L^jpEd{!8FAUMCnawOz#PN(A zq#$KMN+MHp+(m7q91M<06;Izd9d3;Hswvu}QlKBykh0ULa^6std;J1NU+R0Pw%jeRFFrNbC`Y-4=m+v!WX@T1hVTTcQ`7lV-< z7Br)*TF6OnS~ra=ea=ah7rrX(39pH&9q!bBHfSk%V%#!*>c4-mviwLV_N8WrikyUM z8KuK~k^~{;^110via`vSvCeYAi=cx;-jnA1y1&-Nx;T<^=A89SZ&>xMaQccAY`m7V zj>m{4azSR4xpvRYsw?o}Fo|t($=%UXnvi$jZ&;egs*C)6NvXMU#ZBn;t@OU`jUQ{y z^zT4x9Uo)J{_*FDabhyZv!BpTs(EqDXq4>#qoUf~7}yqPr4o~s5B_kqg=$aYT;kM6 z6)3ti;@(KIkRpUH+L`o~f5A!xtv@RO90@6@$f++DJpS43#LxB)QeMHy6{sbBsL+D;(d9@ zbXKN47B0&(F0&W6`-U_4jk^1jlPtnP;iB$q$={COO@Ah1~YnXW!(!Y!>GHk822NhE|6X7S{>EEloi|XgcYzDrWB$@hl zvCN$18CZ($M5mRd9@Aj)%SKLdGooi9NC`fOyQ|**4Hk((kATOM6?*GpMs=#mNaSvE zB-%GxxqK8CUm6yNkr1*1+uVKclE$ZSG*#~8e{n7#(R`B@(B}SO)KraMNpPsH4K8^6xUM=f7Bn74?gq_Qsc z%l8maPdZZQ7skrrUfO!DNzQa-MT)VKL3+~zz^Er{OZP^?1oDY-f1zJ;^-k&UHD?la>+^DHXOU(8}Bp|JfX0(282+we|FKfZEb9hd_ob zMuCU+jMNT|6oOqf%p(kFsRTTPC5~9h2wd;!#I3z=!NRnp8vD{|3fVw@5IrloSwGL4 z_$2%%sC|>@__FnXf;YlMG~+!FVoWNro8BJ*~^{HSf>q0Ocf$r zKI2u!V8T{@mlQ8>-90c)T_@r`iOhHBryS3O1Ud%F(&QNySS}>KJeiPCZP^EpTq=mu zoIV? zCA$Jb^2mEi6U1*GP`;n+Ae1n9n3@SDv3)3}j7W>1q06((zRBCG}S_ z5XNjx6SZ0|g~*CQi&X6DObt`n|ppj z#(2j}e6VjXoQ*@$rT%rZSI!WF8oktxiDp$#SSZYqGvoG3w)5ed%fRujmO7F#^M58m z48SkG|6nx-xCZfqW+_e*5wDIEb4C0sp75@+;JIaJbKrslinROQvu7YMzo{v==Bdw4 zagALi64C&$Uai`DFVBp>$wfLAT^s#^XI%G|zj6OvGE$YPm`$}fAT>PT3R4=1mzjBM znsJJio0FL95sA~&6QpwDwb3T-UY~We@kwvM`@fCZU?+aGeqmxaex`)#gDS)et;)-v z`P_E#HfHD=tI~Z?yrAxq`;iWkKnL$o{F4*$aRo+dMw-6KO=qQ2T>j`##Fn( zA1Zj-A6Pw58oK;5>Gf)R>h?QEWs|jWyAKo!vHU25V2jk!YL~XNJ=LTXB5#xR@!&{b z{&yfX>iWCW?w7uotjKu;k*7laj|+R7^uXdwv=_0^IASOs<;2<;`SJ)1{C$sePVJlY zR-WuJECieMdHi!RKiyqGeH5PkN#lJaPUILkI_!6}Oq&~<(7kyT; zI$Un~alvl1=qH7yEodAcSa3=4pXwj{_+?fd@XkgLEb+Qc+4D`KWry zZu7pn*|Qo^j4qlyU1Z44cgji*Qu&yU2O3Dyk5l@;0ZU5KXI|&O)(~SJkaP3Y*C9KF zCWEs}ITa&P&;TywZXD#0M+=Ft&WhK_m&?T`xcaAcnr7CZ8Vbdo!bU}=U&Ii z@qV$V91)9J8pJhGwTkg~G|rdxSrrH-jKP1Absg=}5})7^NgS@Mbk|?{^u!g-zu$2N zSC0Hkb#u*L_L0~kyXwM;whZ=3l)k z3Z1WXZYkgW=Pv?=dBDy;+7H!y9+R+-{nAR}gn^;q*7a=?>@!YVD=WP0xh|K z&wYbVAkuaH*N!Kzah$z!i9?u57p2lP)QQ??rFZ=RU4F$;i&JZQJ_Qp2%+ekL!jji< zHNO6&8oA4c(MpDaNGBrYXmEiTP+{Pacw#t(&p6)`Dm}01XPi0EmgT+uzKcWWuWWNp zBGdDf)A`EPC7jG_2)}!CI@e1g)J^wiT6rFV*!2B61F6F1WO<^qU@t77N*u7b8%>ee zWZZ1pUtvi|shBlN;@;kJc~C11$?%QVI~r`S*xAUKiT8xk`4sh)wEvd;ftXTOO6D64 zjs*_=*s1~#`34 z$jJ;t@W$fRYypiGcIYC`CvHLsj@JUgVF|6JR=iayTZ9c-b57Myqt})tY{T zbgjIL#EA<53Yl=+&t!_GZ3?EEBC{TC3B58mEUr0obMn{FCzWf4 zt(+gLz<7$od@m}vwq8^^^MhPXo(BWfAv7PbY}foflX% z@1ml*S$CO^$6)Xa5*N^-mgOTM>bw@OueFB5JO}`{6vS^mhZ3zSRkJlh;(?Ly1dWUZ z)nkwf7a(WjqO8lFsbVPc%PgcSyX*?msO5d3XL#Z#C(_oXVn4EdyKYf;)BT`7HvkbM*%1ftNb8w+@w98UP6z zt&q>~rgeD}?k}YzN(ihSjz(9ognBDHExK?P=3;sh)PGbUFftvk?+cMum>E;NAIo3C z!Ds{H9(hGDP=B0>0HT(6erwF)0)BX%_hJ;Ixu(@gg?o%-LAReb;f=aM8aC?i{i_5`z1wH-IkTVQemFP1^GLP(kU4|Rg4I76QjCV0 z#=5#TP5p7U=}?XT+Nj{ym*D4owri}S7VRL0T7lyNqnOy?38Y4J!=6$uX(?8GN`7ru zqq5X14~!lhbb5|n@!u}2Z0ZYnosAOH09;}}7^}|Jd|9vVE-if70>lG=ABwHGoquQ7 z0=*&Fuoa|%d@}#uXGGr;TKVNQtV3obQ5(lPK!t;xRUEOMBDyZFk%y71PW2bY8b|7z zqr;r;uRE1S^1}x~pl_G^rpWIv?wkY$X|97F@C6eN89I26sQ#$4G6@2a>kkauz0IkH zo9St(za&|u+O&9HXDag6y!ndv)mHLZ;`LDF>)$QFR+SVK6~p;FONIMz`Z{8 z79mR>VBTFbC{yFQl$gG^ezPCZ$O-?37cb1;xbbeq`RX4|COZPPoXTZgBL!Auu$K{_ zmW336i{hnOf?H+&f4^-qatC!W`29L9qz_ML-G(>w1FNL(uaZlqVM@({tsmwq8gx<- zj|&vD>IsqGy$~tK$7T@mJXi3pQbMZB_0$3F_WVaMJb?zN=ojWM+VX>e9Q6n)~Wa64IBT&ZW?WD|8M!$+Y{2a_f*Va*pa_2LDb zb4c!Z3O*MjDsttTL@J@u1@B!vNyH0KB^{$uR)lrugMDCRynVmtPPYS|iR0Hs6j946 zNxhNrAdh$0q06iGy;8}Yf(yY15JjZE0n00#>7t_OpCG{UGo7d0DwR1nJ;t+kMd82 z)r(#J%l=sa%fLsrhZ-+bbiAJ}f+Z^pwZeL)eemG&H|jEhTV4=_HBiw73nU}|5;NV2 zdP9Qs6>br)pVyn?ABLHAk#Gqj@={e{+m8CMNPGJmy&`!`d8MUI$tCcbJ-jsRTB#=D2S13-Y}t)a->4D55T+wOQeX2( zd@~Jj#O-`0J+WJXJ3N2YyW-Kt|G@Ifvw}sg*1acs1#=T|)jhBUgbzaBJd;>JE+#tE z0Q`w3Ol9Nd%Yf5rDI)7W=11WfV5m^45X4iQ_G))Asm}f1q4Y!jZn7`GsH+!0$06p% zD?xPcKt!$O$1mxoT|}EWY}e6cgS*bP?Jzemw6`P!R~c)=j^X z<7XyNn`;;B?KQ8HH@`IrLX)+`Ed$ACW7@I-*Wt9UjX!+&d~GBa{ZOd2RA#~VC{AxQ z*-^)TS`!<&GIPy|8A5y61Y*eb50rHMq%-Qgwxwfcv$_Ac+LI5>U=gCX-Fao zIXw%o%uh^s8?3j^9%4Ie46yGcTCc1~tDfb^25hr@tdAmzW@WhW5Po5b#E%Jzqj4rs znHH>nwnHi%=B2;Sr)_+@(?Zql29fFr4Q@HOpbWyXaq# z2MCkK2tI6T1n^Z$Eey8>6_YPJ-aS_62EM)*vu}S!;R}=h58fb&h*pOJ@YQ}T&Gu8qY-K_BBqraFlLF*i@v8DN0D1xV!w+?+GgT)DuL)mhT8ZPTbFT zbpX@&jhgz6cfCt>v;sy53??h4NvgQN6uM6hHN;7AUEXzbuvZv7Ad-TF()LI}&uYjb zdFqmwxM@&ZJTx|vQizXU=;}~#IlUF)SW;?`Vp4MkpP3OV@@dAZ;cywcbWgQe;PNxA zd?vn^(hL|S)8u_bsZJiqkR8na-=X^dA55_Wj$jxEM(P0a+nXI@VASZsH*io!$EOB< z@PkucI;Iw;q6F#hHUhN_POt>~f^EeyM^pJBmH%*@j8^Txn+TEo=2p z<(cm@^nPolM;ekWB@2ECk+elq2AtFtoSR{w2FO2hdq2h|7XbHvQX*=DsMm)p9}I3n z^wV~yaZ=MHAJ@;!d~o}zJ2cx2^`2h+Z&T-udiI~~mn;G1QDGW!uzM4ios2qH3u?A- z8{RmbsxlK$wEZI9{y0uhRmnpEx(EE0?;@TcPkaYaW1}A{YeD1UL`mx06V(UDz-9TF z_M(HEPgQ&?80C_3P>o77h6XCUX4Uek{nvs#zN&K@?EnzXiJ2Cq5j&GI8DQ1&Ln)yl zu7@qhi>SCS6*q9@RSi0Dm=4_TV9i-BROc88>kR#`nhe;&hwCC3_2rs+@jgy!aYVSt z9Mg5KQfA{9wI$=NBpMtDzb020kDk6gh0V;uQV~u%gNLjcJcW;(mTgj`&suP)r3DhQ`uIp!se)iIM3U;D6|64BQYpT1t+XZ=pwr zZqtbgW*>3lR+s!Td{hXL#VCjnzL<4NP|0he*bO z`^pp9k*$BOE`5i8|2@rv?cButi>RY^!is#k_RMFp8*7#LkU3XLiDaLenKq5v9JUVd z;a;EMl_M7l4Z3AQnxykJy!%Vv<99&MdjgVmD)WJX6C-$bra2J*+7Tp7-nM%Otm+4b zz+Ih$c;w1k`fSlnxK+i3(=WM>A|F1!v8R3fB_jsy9jAjk%IYXQgN_PcX=Vfo?A;6= z&8o)n&j0#qPc=A5ScP);Gf+~|J?=n=vC`>(l5J7q3jeP7E=kfC*v!}C*7%R3s`{N) z3DNu+k21ypBdtP66S1%-tfqK+7r&_|8M#V=>NXP-zdm^jTjri3Bex%5Ek~shQVAmK z$IlxqPakLL5%@`b&{}Dsd{G8MT%!xCMxIvBNi$&Q;VxaoHBfxOJE@)`@tzsqhhY9! zGD6`S(5#pghgMRS@A@&l3J=rKdfo2L1gz+|Tgu>jBTz z9J4w0=}G|Mca0?Z41Qf4>ttv`(;;nHN}gRUZ21Nmi*{-)<@`62q1NV_S70WI}DdZ7aEq)xa zT*4t|ME~lp2b-FEO@m+!dsj&;pk zTJ*xuAEuGKD10Y`&&ohs+D&GJ%I^(mpKhEcOPGDWfTFFYCcL8~`2hgPB%BTSs)S)7 z39W*VMUmUw*%d5V6x#Z?@BP2yMl$jO%B2v#52TVlg`V-Z^q`+e3|3n^c8XG5hapWa z>&`-P1~ofVH_{Il#b~0ygO%n?0MaS3!$AjXOenjJ1VZS^6trdErQ)_mNh(+{n5sWc z`&bSueSd<#_2|#yN($0l67Mzp{ol_w_i?y|UtMdhbDfbTAI80?9N&I|&BNVijyw}#WJ31w3S3!k zp=zp)$Ykkn8NUmkvoJ9IoVohn8GeR5!_nOX`CbBFAw^++hVaNo8G>m1Te77<(-FfJ zdqRk(+|2t|Z51N~wi?kp0!-Zlc_DQJGQRE97(wVq_rTfwRZfI(QFyQ?2bCWIgA61E z08+DPj~UI6yD*ug=sc+&^FO421-!gT*u{XwkI(N7KjZAsW%H;fhe(Gwx@k4v7`E9` z7uYY*!f~0TVw~8%Bp!Wv`lP&RUyP2S{J%#N-QW#Qg(Q$HWbhb3nb_#jOSGpHJj=iD z&g;l`y#4-fKw1RP=cqLq>5nQb5MCf0%nf@$CbjX)qxv7&9Z4A2eOPHyU>$oI&?&@JYmWJ^q!Lm36|qo(K8StGZ75afX`-Pu z`H2N1XC9DSJ#2qmE>F<*5!?hh|72bj?BBq}CWk4?5J2`^yD4z~8x6%_Kn5Q~qkOFy zEr##!iP3`PA+J6@rg4~dy7Gx4e8I1&0P9}DQF{N}WgQh3?%^J;p zGyq~32>O<5x&>yrJkb{bU@qr=1mx+ahxTxbj?!v$WP+FCWZaP(pcf(8mlJvgs*$)U z=!G5wZ25g4t;@bzY|_aD6q1JrO5hDlg>B8%6!+tI+kO2FMsDOpnnuFA#`r%x1wxx; zimGb|6zW(Y#zPbR0TaaoZ`6kr!6r_eg4jHotNdf_8d(%kegwg_PUpX)qi{b2fLG+n z)=c@!tjyPb_TiIFk-)$NUNk%z(r9lz{o!Er2`3JOg5=dqJhLL}vxheYkut=z&R`|< z%iA6FLprh!gXJMUSA#EiZ%RP0wV(R8gqIK4>w8o^3|m!MG#_LPl`bbT=i3$K*j)sC zxUtqzt!a5f?JWhaW)5b?*-fT=MkSCnrt;A^f8D!B>?IO}K#r->(4vvfm8(CgxVmoswKX|Lk4S)z-2A}JnU$=L?16cZy_KqK(deE7GG^IaGv7fN-rBl%H zYk>!gKvQD9RDI%qDE1*b05I{~%PVcFQ9VtYm>&26V^Lj@;mv-s&#V&ewZpR3oIB&6 z>lMgCpzBvAm}=+PEpGg&fUJFe`2piPz~2FHH+UJUj8;`a|B--f$ zGvoD7UDxDE(p*txglN)SY`kW4zw-+ z>3jq^3+v+kS~|8i=4|(KUGwNzMw0NPNP;3N^@@Kig=b!Gar*%d1||qH%a(aqe=?_J zNvvJPR}Y-I?Y9KBQAEikJe;!_bxo%<%ZFtMV+@SJ7Mse};JxX*XLgUgjJTBhC&&7g z;2!Y;ntoV;D^w68c=7$#L_-3uA2Pet^A~9QAWY?|`Y>tN444P#6asYh47*3ZT342x zoVUpCq1z&`I`vq-3Sin>MV0;x*$8(7OREAs=)2W{eXQz>oNsh9wYYm9|99#HGtlDv zR&c#Q2~lJrozOgUfZ+5F7|u4l-9$_6o;4L7_>BgRe05&e^*$MSEmz01KnQ+b(I{EL zl0gv;QYEMp#b|S;qrva{I(L!rhbM4yaCKaJNbv}D>J{eX10l_WIT=ZJ%FV1#MBoNi zs8D9+ZyWAjn1+;n22KjUyi|l@B*zUyMr7sjfFbE98nPgH1E-CjV~bbkcIbgt&B4#} zxo$^Wi^ktv;O6EcU$vV@sIh_k)1s?i82D-13*j|34j{8n88Fh(+q&cwiJ}Re0jqTcntzF}q4W4n-=qvicoLp1nKmx5pN4 zQ0qj8NQlwZ_e4^s71x(LL7~B*#BGlMD4{@oEgi^!FYBv9xrDXYAN|P2_?h2y?b@?VWXJAm3?(&}u$w z??r|fWaA6UYyl~!0j=t!jn3DLNcHGnjl_Bebe2DO{`XlufVi!(#JvFcRVxyOpiHaT z27#aIPEy0?RJKaP1BBd9>?U%po>ZXLSPJ8n5)5a)+0%txpm-0?h4a3)zB+ZS;(d#^ zQHoQ#T8h<)F$%!NRqC#bT~Ll{=*N)3sU+;?+76;G4auM$(x2(-#ZdKTg+U^{a3cZE=Th7Dr0^}L9j@QzkScp{64bK z$#~-zBu8(*b46PJl;}EOt%e7(Z&%t}N2Vr-j?MnQUeRzaNi&Bg!K+iG+w zw@EE`X)z4F=94zYrY|*RKeHSLKP&vFD5X_a(%)Mdc2effrnJPdL&}Z9$%<$Pk zF7PWy0Dp5TvJI@S-drf93x6N*{3Vt!U*w+LeEVNRC`jg4#X<$#jQ4abh++|w*6)Ut z=>u$Xwgu4w5jx*v4A9CIzQE923b296D^-3-;|&n6V7@9=$ApOg5Q4PJTn%Jkny$NL*rb@U=e09Z_9_AyG$2({ zv?e-#7%7uLCnKk|kJl5R#end0gAX+zYY6za1+o3A)G6c%h;WEwfJeA0h_w~dI#BeM zwxS4m4g~j%peQ~$x&nI)rFh#xPJ_IsfY8APgpFmk*7i@i--10vFZ~ht3XC8r^Nvl! z1jiEKP^<#&doSSQ;wZ`dP=6I6G+hBFHrYX9ews$mQ#S{dJ{|jxCGpI_?263*Mn3i{ zx@~VBF;r#>y%)b8TrLrOZUa@56QIY%TUHeS6cU%?C$GDKjd4yQ9vdN+!Y4p#Bn9Q& z29FQy>~R8^#q{5G)c1(pub}IQ|EEy=F)B3Jv;g*Ri=(csQhkV zti~Nj;Ag~ByZ@5)FqdIG2O5#usqJo?2V_;;4szt40IhokxUi92#=}^;o$$%D5wvkY zk1i>wwVYNf0ltz%{Ll8mAd1KiQq<8CaPPK9W+CtZtcCb&ff#3pb=v%6EAoOLL4lVN zl~xp80}yTm+2jGK8LjaalBB9uJ(MYFELhUWD}Mfi@v$G06#JnzRJE=T@NcTlB?*2< z6X-aD6hhvizG}+hZ^ey~=ji~(uiIT>mp!|Qj+8;b#=5K!{d{lyIgG|jYv)v@fYZ@0 z>>0|2M~rLtd$f*!z>7ytWcX>WYsyrSC;+gODE6F&3b6|Bpnyy@J6*TIT`PDR_xw zJ$tnyM>K?36gjm)t&ZP&^IJfJwq|5{oI^oC-1_{bkrPyh=Q%)BMBAIjRgOP&Zc&Os z9xC|$(_&)XUF5gy3HRS`GJ5~*n0cc1jq!MR7Qgin**JFg6sE#dZCU~0H4%~$1H0D6 zf>TqLsgziO)&23P&z*qf(_s@X_FjuXmH6->)?F+rKX$koSWFT3e8E50vRvpB&-8Kt zd&jV{^Z#w8J?S;w1J{7<_bZ^hjRiJGvO4G>uy7;s*S8VS$h!abet+VtlkG!?Z6ILU z1K!Ddc^Lt^G^H)DJFr+n^Wg0XdUoP!ZB0F@C08MoGk@>CdM`wVIu#Xz?DNTZ@f-hJ zW%bLN0s0w1W=v|}5F=jf^Y6zcDrDE5U;)z)qGNS3BMyz8prazR4xL}rJ7*822q=NI zPcPbDxsk9b*jNuU#AB=#@bN(m{h@GC@?%LgD*@CaFtGIj8y%a!@4=n0dRu{SqFREsggFu9%lo-IdKnV;3NTc_ZgYe9N;+Zba*g~L1AFX-3W1!Jn z?-lP+??3-TzBQl}%&4>V#&PQkk~@$$_Rhb|KR?{IBQ-|lKS5G@2kb75Uz5K~rbgx0 z11i$HU>Xby9R`6~UjG9znF`nKR}{uQq|@E07-q{Fo`TFs|A!i*jeY517&{ zW|-Wv6`3E3WfP8ZCmding5G}_IEkW91LBiRh)9#v~u(arC(E|Biwz zT=aO|M0@^_qr;hs-@c?~>4i(dKQ8+|cz~(4oA~IJCoM|KQzb|<9kpAcsCzNkk~{=z z{W@3nMk?g<0mdBJwdzG9zBAsIzdJzX4-NJGAP>c6oLc)<&|4Us@qT|{c8TV$r7ZY+ zMpJ}UP~LCU!h=Lg)C%o`reFRP83YBmFLoPqxv{@KiuU-6X7}y!NB1`cli{Tn{ORWc zM-RwJQ$L{2xlR`VG}Jf2DLiN@f;6()^JHK+^_PKOIeA*`Ntc ziun*VTj@cOY3w{EID0iCMkcaiwnY(k0u%4l7bt>JGVH`chUfviBT|MRhW@BWimdSd zDh0TrOYR)1xxR(Upg3_i&tG=MP)1JNmA7&(CYWDJPAj8HXq$^GAG^^V(x}Tx>LT+W(mSKKslS3u&^ z2hnG{Nkv}Qu6LUTbatOfvx4pOLziKFQb(cT6pw#X^t{;-f;RWq4}7P_MRjopFcDdZ zD!EkAdm8{*_X~EiDS33@HuGYpb_QxkzVnlG8QM{!IQd+ZT*Oaf!Ha5CYg!Vy zW0_%2$9d?L|1Cw+M>lzjD8#&9m&jNE%(Sn6#v_@9vL*580XRSN1HSzrc=^g zU0>1K9E0RJonBKxfg1BdmB^x;_9a!NENoaZSg0=Mdkoc2QKO>Wsp0egx`?i6T+{D2 z4yP8M-seJaE>tmO^;=-)1ISS%2Y(!P+K$px&1Yc2s^`8+#w z{o49Rdj_gm{86Dy>5p3m7m0J~7L2pz5A5SYH1* z_fhU~*)90|O40m9R%#lmDPqGaup~gkr2eBv%MTt?b(_hZZP2Z&z-f;H8KA)UaBnv# zD@4VD?*{CPICBG&6k9*tS`1!xTr;$(2|SKm4(qiP76+qc$q*pte@?V>S zLEMkrbJ*R3S&}K#?uLMV+Bqz?BtY2StGOb125*Sg>6(|a51jjX7edIQsDI`3A6`Y} zCq^6X(E-c0czjvomMS;?5}MS>8zN@5tZ*XZ>r5c`M1-_Ca4q~WQ~G8FPwx)^!y-6V z2VH5GO`nUR98X2c_~VNJ((In+1(<{w3-tQSG2s|6tiYW-JVe~fx8>%RbiSV{O%3ts z{?ZjhxZSvli~uRK-6)O3If5iMX5$YN1%dOaka2svg ziY=KJcL(1j=6b5SMkI0VleEta{Mu%+r<#o$PerP54)TBB{CQIQwVsjRCD2m2%VoD7 z|M;{q{^TC?PJAX?UP{jd9OuO}mdF%(02dS4Mgm&XGm&8Gk-2x2o#d(=m{m-%g_Mk@ zVw8}fXMS`!@(N93hrzfgRK^7qk^A1edh zQ!$mKV#zCl$iI!D4d>)#pD%Mv;n}&yZvq*X{Il;EX5KYUDT>6(iglsl^}j2|TIq9V z492+VuKf!W1lQ1p!~_>)R_?y_!7bOIe=)mCbklx%Q|bf*%0_K0VVx z$z{*eeCS^~n5ZjiB&E`NB#bQyk&g|3JyiXKW-!SaP7rWQ*z&bJ=18>Ud1IudvPi7V z(f3g)ERT`w+ujEA@D5>+`P4rqf`_3L+sj+1(I-IH416w&Ui`EA@fi5#DxPhtic~!y zfCalr18UFnf8eboc;T)F}kzAz+=5e+|3ty{}xyXQZl>5zF4^>&L>P= zX7-%=4<1scLf#lKo!NU~blLs%rT?B0Xgo$fTt^ryaynQoXD-~Lo*~v9TtRpTCnMVY z7Hgx3<$VoY7QLJP463aE7$dv91DsEJGqBRG`Iorc=-2?0d6X>6GmZgd4%|>=@J5p< z0827W6p9^hk>sRbUHuDLgnizMKBXoR%2B-JjBm0vLh?LwWh%l6;d~+5y<4wHm_ak^2lrvL92RBG{ zU$CLD+R1!ak}Ke0^uSRxV3}@#tqoI=C@JbO>jvQG@k@By|DKr9zjU=U322)J<2Z z7?BSkx;$eLG^$=elQltp(uf3VMCqfIJC1-YY|ZBz04!PZDL4cqwE?!_s=06_I}8W$n)I=ZGnw^B>SLfagz7(j(}7hX44yU>F!R}q z^0t2K_~r2<4fp`z;|#|QIT7tm5JO0x;75U{-Pd88xb3pmlULCI%)2MjC_pHyhExi> zMhC)`PaEBb_MEn!8~CBU~ynY#J#E8SzA^ePCxqQSA9C z9aDkUC6K}V(L44r9l)@8$#2$M^EHrhh@PH%PfrZm2OsVXdt{t`q-lkgaI%r2yoO)Y zfPJed_I)2_L`ZtSfzC*dD&ah~Hz0(r?n+6KGbbk7T;J|iDh8F)>y;ae>9Nn zM zqfb>y<{(cJeDS8tNzmQ}dk(N!!=KZsOanOGS{pAq0-ieg%BS_;;(^+}-zM*<_^Afc zu+3Mh0?5;_{JLO~z)0hJ8r!^@`~BVR8}IagUS&Pk*BuPtQf$O&z{bdUccfE2*{iy% zBhKiHrU(ySht+?OSTp`FfT--<-GTt!f#TH&gLwdZUp6(tj>@X#gR=8ys}I>re_T~> zmews*d4+y*nN&8B`-%V&Ipew^w-JxIC-`SMKL+#0yf7xn{)nLzNf?{#akXBR6Dc;p zo0q9&A-KE4%#@otwA8?u|KWVsZf`{_gdoxk=}`D>dNi3YN6%v*!5O=p?Nl?TrylJ! za=ieU;d|&rM5s#+Y8K1c=(2roq!;GYkMFlS09n2U^@3?>_q7IsoyvQAXZbsz^}JoT zsM-cE0K5f8SY~MwZR${%gRDV|)R^2x>@AYe0Dh zwY3kfnjbcjVk`jxPdC7G=ujO<)we_2fV2PZ<3R<)q+9ETNhStvrZ~Qm*1ulqJM$f6 zR{T~`*PRqFI2&SMk^J}e!z&H(JwXS;6okX~F#(*{$cvi`MWZSP>2|fhMJCu5l5aTy z<(MR>!+^=&B#FuiOq*e4MQ2aoEc37l1JZv#zusBRzZ$HFfP*R0+kTNLQw0qEmABx9 zh;Ga_l>#JtY3uLroW@--3vpF+*D(+z=YvPw}X#@FC_H^TF7Q9k_$nifRb?PUK z>kX0Lqz|N;spHzbc*o`Ze*$W!2$F5LOCB&|1^Cj>&+@5}CPXp$Rz9Q5O?t`_-Y@%= zz#X+LSo8obpAD^2GTQ?ZZmU!1LTrGH$S@SexvAabKc29Q}qR%u!oefI`n6x zO`C=XeReOtgNx!X7muNj7*Eg+l>b&x$5PdJzWGKI=L%5X$~TWhVK{a-AUxrX&sN>W z9EpfKiwv?qx|3_CsFaN!65lX+Ix@5jve$XRjxJGXBPe9eEjxuq-e*P$5bR2G`{kUqX^cr6od!_GT z-{2>(wg}x*Q_qx`E^vV#CNq{4^&d)1 z1Cf{^iv|F0pgZ8N`LvnwK9WhR^TVMe@M8Gs@$@AL zpal&A<^qdMFAk4-Wd9W{eMF1oN_=#U$8=&`5dVulLO9n)CqHl(H1(k*AOr0JN`P(N z!LhH;jBzMIBl+RoAs{D$PzyInJ%YhCj7T`zsB<;r&##6a)@wfu(`Mcg%&)r@NkaxJ z3QeI*b%1d-fW&e^*f{=FdVj$v609A#33C^Qi$eidMb-{Brsjm8qeAdFjT@$h57=%# zMbgyAXK}_~myz!jd-}P*g9`+Oj6{cxW){3$*TG4Ygcu|LVCBO>@JG*%U$-sC}3Z=X#vj9rhA za}d_cNUp(IDinU0m$*v^=%}j1FB6=#eXkW%thv7$^%(%83fp!+ix@9ShO&KlXW}Ut zh+fRRIg0WbGJF&qzD>*qkl*h%qL-|NX>XfyNi>}}$2k`e*Y6Qsb`kc$Ya}@Nq)Y}5 zB_}`zNCi1{0b(o0@DMi(wMlVfLGl9IUzG)Q4FyYe1>t@L0qP5Xh<~U__6NNep_%$z zeMgHkt-`m47b&ueu61x8+LM(^nS|bmA~{G)6*~;H+kfeHtjVFpa641m1vJ`6*jflL(@PFjMks`Hsn^U-ySC_ z`rGS79{S-&{L?Fk&jLtwkC16?=f%;zBf1TW-YJHzqK4pjL;-0*!9{IM6p}U1YyL-1 z6rIuEc{17OsqYwIK#xAC^G#!+hL*5V@MU97F&*R zQIl5OpqHW1>eS+*@GbCl6;j3}o=%cH|CJ;A%yy}{VR@s#Rhayr-C|=+fk}*Qvcvz8 zX6V_K%(_0?zpxf*3bZ7_(IxF81xf)pioY5Y6%fhC0rFnU~p1Y6~{UjH~I*h+iRCg>i0lIlDCo2I)gg3#%ln$>+7mJ{waUg@_lp-FHN=yCc-fRo>boCEZk{xZ zzC9S(-yL#tw+1QIC*pB%`m%^)z)P(MI;O=^P(t>B(=#huKe%y!07YRR7{-~`fp#oD zci!(co%5VseghaCW|6sGj$q@=!M=cu)3Z*sW4-{oz&UU5@hjLZ-dF-+olBg$Sm8f_ z4>XaZ6*tjEdqKZ_{q&)?x2Nu2cmwpWnlyI!^>NBMXK3nW-N(ss_1tE=vGy(lO;#;`l*!9GJ1ZpV&{bUX=UX4n7l_ z(ox8#0-?)d9t>4meIu$0uc^qe3-BYM&!_l0=y?tOcr}M*uR#z0o%yivukjte?Y^+xEqy!0teMUv4Y}e=`p9#bZY77929FPQ=|qq`dHY2k>#pEozZG+Ge#;a#>Y z6rc%+&lg?yxjXr~1&+h8vCY@u=Zs?u3~B1rflf()K*7B0;~`+a7Lq^F5>~WHcTr^Q z5l!V03kKv%X7EBL*^dlz+tNVFyzA}`GG1*8rvhOK0(clV=CUH~L8MHO?pZ3S)P39# zz(0y)rv@>AxzQGdMQ@PqZSlEoYpApBS`uKvI_g-`GBu08#R^T<;q&_lcGF@-re}KA zfVQV2VWr=^#JMXbp9Mt~F8FKXsb>XfL2E{jT9->M?g0c{a76_os2?)rKH3Fu*<|(I z?Z1EW=fClJp?9BU8t#sOq|@Oyj6=Ay7z-xD*Ygqg7Haw3qYMtXi62kF5j7QvrrHB+ zOBI0)LG_=tUeDusAhpZ^sFwH#P+}P3-81M{z44JM>llL>C9r*cAvpi~JMk2fsvOm& z%7>JhI20X&fEoN3&g~ClJVc0&=fN9C+?vW?6*uoMV@w>mikpKs%$)X`phSY=wHOOt znaU^pdf+}?vq)wY*O4`_|495Xv(~#sx0yVPSC?dvV3`(td{ULYQ6P);io%L~NI0#w z6kFI!UP~8XXpHNFkV6McZH;_Ss+k&$9*C)Fuq>N;+x84J* z42kz1b)GMR`vLU>SuA!m*oG)xYv@aVnHK|4^xhi3sY#$1=4H0j{p?1q=RSqGX$r<> zSkDu=q`XnGtE`_$A<7iL#zI{2a2dmhykD5386f$F`~mCkjwU=T3tpINWzkN^1;=zO zEBCF}bCbFpWYX`p#BSF#!#n%Q=h8N*G+USlfXFVr^eu0!`>v<;X}a5 z6891fW90(#+xwsG;0#l#tVbSFg>!oP@>vWpGCqsIbDNdx!s1V?bjcrw0a6|mLX@cK zNyMBSeqtb+N(PBgl)21$TRwCimX~>RG5y$1RpF0Cht`bkC6}~3?FBlOa~tY)BxZy4 z2Qdh^>WnB9G|eT3fFm;YCnbL?ii#CA{TG26*?#;Y?B$KzSB`~>f2V&5B>{5)rsu;f z%>b^S2Mz=VPJ)Z?ezNH`=L-YNDWQ%l(BglRsvLTPojk^XF`D;Bw&`~NeQVfI&Npn^ zH){699WG!4Kk=E-s(ov*a7BQ(p9Yc~`typ9MI*G%m*{xWN_*z^oTnb?Q?j$|NGuE$ z!$BqC4)=0C)!{Jh??c0xVV3sP6#=GhIy)Ucgz{PA=&wP6z5Zz4o%9N1PBL1* zjgL{UC#c>I!Md{3`T0HksE4Q_&RbrN3jVb+5A^nournPmF@6l5f_ad7uW_er8~~7S zKg;WZAh7Zr5Zjo$^&`Z%5q@0O_{Bg%ZlhED04Xtl2heMM@6wE4CeXQW&ZjpK1|30G zg?WkEudM2hr~)hc*7!>F{vN_FFfsNud)CPQ3gG^23ZVn%JQ}24L@t(^KxBIPQITTh z@@Ri?N6d>)h>zf9AC~bAqV|S9&9xpxUZM_1H(1_D{Q6p1m4){jIA6fZ(m*xMY+T@b z;?IvR3c{`fX0BUk?rF3IuRkEck<`(y5W?H+j(gxUqxSPPor!N(3mL?IO8LIyp#bCI z&@2U0pxRxOH$V-ygXF3$Gf_=E<(E9NQwf&4y$)Tt2Dp08jYJ;HU(HooKz~I?DIJ^I zj5hckwj3uW}j__DG+?1vZ*pkscxTQb@Lazg^}WEmmR{Q_%MZ!7^$1hl(&w zFQ#rUc4Y6<&-gFH66*Zpl|;Hr54l@2ym08Igo2J1!%yYL184M{DanN3U2i$Fw0J&0 zq*)%^Vv>kQD=xkC#J5^P2l|`_lC5hzfS_OCbALAuD4z!dPih5H{blBfB9*A!;@uaJ z1nFFh_X{qczmfY`VhA>f$(*{Ng^Z2B#N~2@=u;iaWI~9ijcus_A-OW=d& z1Ceh@Sr*QUbptP4x`)QdhG^}vVEw9$fqRDx*jk_qVxe-{7N6mpJ@EZZkLng|~hw zu*tjO`XL4dN%nWPfS|-t{b1<+-+4dh>m{2avHyWs3`f)&*%{E72dK8}&!!MW99v>+mNs$YQ9kbD#K1@_Uf37HMo1 zqY!F_&tb#zPE#G5AzhPz2weC_y0&EY%=rD>ICWPG;!Qme8O<^&tj_s+KriqDbF>ZE zrRHz)sXwJ9U}+zqvaC_@!jF!4|GtOZMr6fp5^~s*7$MA(sqhL*5I9M*v{NmD=T@?b zK^!IyVv#d6zsCOd@Wb)Hv+rnklhVajrnR9RMkCtvHGXFajfH`W4|`Q?pg;z zBTJU)l6}DT=4P+1=V<`dXmk(MiO3|cCzw2vXfk|9?~xVriJX}msB5^W%(bhHD}fbr z-as;ABppx3d9wqfqvt7J58DiZ4iRILg=$1L^kW;EYE(F5o3HhGKxZsCmjIU+VebmH!$K{+^lrmBVCXd!GEaN z|6ByWpQ(uBOKd+!1sRXTn(b_%0(hUUJVrJv5LNvu#ouUI0;YX6Ye3H&8jK`?*GVDfhm8~cqJ9FgH0MJ7PSD0U2+0 zlCmkTLrXocOvU#K&|Qj~ZZEPTT{7annCt`4))`oTJpX@p&}GUM2!gwY$4f!ZByO=1 zT@yDe%@3yMHN>N_dD2aT8-WpwQm@Ih9Y<^)jCbMNqCq;+3Z`*8NcXzBi{OP{HV{x_ zyP^aOjh!Hc3TLAkt8ubMmbYvew)i^i7X5zRmBT`EeY2I0@+f%{dJ!I`D}D(?@iB7k z%bM$9|K9M=e%eGq}HB~s;KiRsJGtZ4wj1U0Yw`5>YIt)C>rl$+VCmNQ4Ufn8WOsfEN=F9=v%tVTbONFhZcIx)U_q} zE?fvYnhS!HB~G$-CxnlS^8y5Q0>3A%(dss8sm*?O+baO^;NQ@Feo4|urb>t*x;KvS8z8ep$5^WMEQDv0(R5a@RtiY zdO@CZ^~XTyvy;08^rdd~tt;1lX($&qpGiowqMdy(+?UDpT?k>Bgu3f*!|z6VaJ9=y zD?YT>RKV6=&gJd{4(N9aRndxXofMBu$S+t~2<3W21EwhI5Xl09_L-nf zj12~wIikXzW4@=DN-vXC?@s6DlhDdrl~<9-x9BSo{Fnuw7cm| zy-$8`R8Y(JpCb~=A`tohAj?yll{Jo=NFP|4EXz1QHP)+Z*Vu2&pSahC0QwJ;*0~%p zQ2Bb=aml=(a%Z#B3RUfgH(8X@qNb!rUx=@;a>kcawt8$hC>Qo64|=sZ1-@gD(zB6o zV?fzMS>q~8t{?8{=3oe_t6Ba;u2FX|tAvuuKCRdBd(&14gJ0WS(yuCX6XgNxDc;y> zI{kDs_^sW}IUJaD^gZJ{J3LFo9{zcIy5~CGPax4F`+Jj3Ig4+lfi1s4u)yn!uR&^! ziPcfbdIr7)i-L%?;4_k${@ic(Bw+(vweyZ)Rvl%p@NI=Kb+>x(V?v3Q;G&{YUxwfW z@%TeyM9kx$0vp!jPPkU$>a03z6~Osv z6<5ERD3d-NSB{T{!6?@tcCC0r3}I2q?v6$DeG!}EUk>1fB;C(>IS^0%sbMRPTI)dt zXDs~IrV8WPX}PI<|fu}1k|U4f(80cw4N z^s%zi^pJw&ex=o8?RxILu)|U~BJ1ZT)<4HFyG9%htLee-X(~o0^?XNUX;R zo^rxM=vfKEVwBwn=quq7R{V(Z5!NJwA=`$+Cg;y~ym+(J9Fq$1K{561B-f3cUusOl zQI{Dor#fzLND+(U8bK_~1S8_qPXR-Z+7HX}DIhrA76#qJ?HHVRzwyNM0^MF&eHcW8 zFHmO|t33^TCM^Vz*Z%MLOa<+4PA*RF+b5%f;O`QWJ_-V}v~Cgt+X|A49%asZdabgn zLDzpYt`2i4Up(v}Fws$Ei304i&Bk?>sAPPV=uJ_M4zHADl=%L&r(aioJZ;VWisAi~ zasPSDp++K66U)JA>=3~t)+El?^ebd)xkUFwOm6VpA)#r59L+~s_S?T`Nn9ylR2J{H z>6hZPOeV$iXK?xaL3J;tBkQP@mzdwU6vxMr5%*{gxS1Wn5Ls- zj1f8l?cZEd@L9Tfra_Ryq?PvIC&J;-<0JDiOofT_AfycfeWgdJTug8X+13eLQm^bI z>fDU3&3>QC({YFHZTe3Jtzo)mj5lVf+_`_5{@$UeYPlFscmI%{-oF2IsKIT;`Y4p{ zkdEFY6WYxm`k07K5=P=Z`IR-+la$^}n>6zY$rDop#_we3b}k$2nD5G+&HboIS4=~| z=`HyKH;69uTrehsYcJn|k2EC6wBN2dQ1f;6n2K?-wl$G={MWm10%^8%20g}9yt5Du zH7k9S!2y@}Pj(%D{0s<4n10@|;;{KpiL2Aov#Y9OROr0Jk@moCprIsp$^K2_xgL{B z!X9QYNir}%mJqKP;j{j9K5JbTl|9j?-}tYSu)JoGHfhg&GZv-Rq=LV>t}b!;$Mw{) z%_Ayy^S|E>bsQ`DAN9QU=ezR=xghUBsWdxRQMmUyR_7UDJrBUa9bhCmsLc8MQcLW5 zGq8n_bX)dRx0S8q&$GFOfC#kmb?de7Ce=!ruPU>`yNKC7HP!mugB@|MK}Gh|UdJC& z$}x*v1+W;ZqxSQx69J^JC&YR3U6ir9BQdKFFb&gJiyBf@?; zhdGUG=1=QQ;6n(c9 z!a7XfWJ)&++hm=uy^)!XbGLg~Z-?D+BBuT?vAVen6m6-aE}HhGNYsgn%?vx^DM!2; z>joo%-4>A~GkokJhZ+XIrt4!K#{B?SHIu5R6KjuvyP=xo6Gx2h5%!sr!fJuYGkp`r zsshs_YuM)rb(6fcq^IQL@+c0U0b|wv61>{`}{APaP5de!|cwqoUujZ!7O0Kcf@)9v)(< zloZDe_)LH&zImMsR$fI}rzatq}}j zIl%7j`YTL8T+d!5t{Xzf5 z!gG!g{fQhk5cJH0nQtjK>YwUr_Qph|g4bF}uxzU{_NVlt1ft0Pmi-ruz;pBGa%mw8 ziLwhSmPt&5P3k3i0?Tlbpeaqu&uO~ii*--K`@PaK zImg4P#6=F8*`a})ju@M`_tw);duV3AU{DTphV?XJOdNTd&L8<_36j2caC8__I29NoTO{hLbAkw_? zL8B)xR&CFhE?NM}-MB?hEl>f~z@*FxX9*udsCSdT*ZQ789KzlB>+Ma~P`HR^K8A17*&o5ERLUP+;lo~3++Y;Eh z<8SLMyl3`Nr<%N1Ra1$|$A;5l>7K198(-pu)vFFD9XO64*bcN+R<`Vj+E^Ufo`XL% zbQ(mn5Z^h6cs8$12iqASx{fwG_qHDHkVOiZV{i@JdLG@g60dm|Qv@-bVmU`LDL}%v zZVPkZPoFaHI4wxVdM`8uC~HRQ7(qhfO=4fxVD@H)`(!FTByF{SAGNZAz4Bb!FPz3# zBWbhHxU_65fAS^d@vE{tb~dUXy4)Lhh7ZqAGaf&J=1H7l5(s{4W9#Am$W*AeqAbN) zT!=*@>@C+3m|hw6w%+GsdiyHLjEAsc6>GiUb4R+*qZUU-rp&@8Z47V92K>V=f&o-h zik_vG8x<&Fag&-GyIN#iIcW6b2MiSx5l#A{xw*Q~XSA~aYq%G_^i0cp63CG#$#fVi z!P-f>)tm9b<-}1DKfb4Z^Gu!*w`Jin#zc{azsI9hp3;QwG%Wv$c$3sc9BDw{iN!b& z9?QTcsU`CG?GPVDA6*Jd>usdq#k2h$6`fN5Ai}>YSaKv3&~n@NN=MKkD2WPY(_uXI zE1k?-a1+mtY1D3JgPyXRess=~n360l0UtoisKBpY7uI}dxa=kuo;;Sz!DEb(Def_qWXZMt+3rw%M@;%J3|UX5202+Pa8DwE>OO%U`_soh!iP}tk>tZ~A@ASQ zf3P{YO83Ix1-P&iJd_r*wuc%SSV}qqV3cZQF$%6tlERmTKdXKf*3Yu#O^%n z7ze8RnbV>bW-z#=)NC8Imix2!44Yp)n;?qhgs-L1(+!$!6X=Uf|5In7+5h}hzqy>d zW>Pzp(z`Kz65-7FH9^lv<>b}(&wLDwoNdXLV~(tc2F?cnHi6w#8Imq>X*wj|kfLhH z6BwzPCumT1mcFq-e+R-Bo*&(OZzqU#ZMTx!oUfc_@YQX)z_iG`0bTYt690?jmzFX) z1o^V6Kl%v7C{X{HKz~|NnNjyia6dbgIz6Gkd+-0m!svyu%SQr7`nEb9xbbcm{V0ZK zG2!zqgx8?eJVyRk!o8|ER0M0Ou5?#bd%s0nKX+)HO5>6qy{TxW@>|4)dsWWrC0E%{ zDs$M!I$+u@W@tuCtc7S7ThaV!*&(}W^*%3}d)io?`TiPwg6fy^n07}ztFgOao5?H5 zNZqZXc#KD$wSWw8%b_3WQvAxJr6y8&xua!jSbqub;#!1=2>Q{aIgK)=tA~e-?+2Y+7#Yg=V^OPKa3h(z^q6|)i*6nBBoa6>F zjJkBrv7OPd#MxLmeH6M;?N^?Rp3VJ-AN~&Sn_QvpMd3fHT$nn(K=`Ir@&=t<+`DRf zns#^f;pBWK?hE`S$!47u1ip#5Yh#|rD;ifC8!htP##4>sUpI{HuhqTM_*KyIHu}@) z3j(IR-J+-JzHiEuFySoXW%rXTACDUxA13}XYY z3>WPjmrN9o^9_&*Rx*Eex@Osvo&O%%EtO+j!j>s|1m6QJ6uelcYoPu+Fk&{7E2eb? zs=ORLQg+72xTkb~3gFPJ&6ueO>*{gQq%SWziE#&|1|_QDE_jR$FV&mlWIim_aR9c- zH?XGtUEREaK16DQX`9#I4y(?boT)9aTp7-h{pghk&DiJZC)Hiiv*S;_HUMVP=WTgn z-&`n*dAGjdi*%6;r4T0kdm8AiTBI-S?_y;fA8s8L^fbilhG@)C$jX#1ReAv9lRWg= zK19(%Pdy02M#Plbgx_Dm#R`{>mQ-7WJx#^;%2l+^41QR-qvn#0n&6Ap&v^R=L!m&9 zMgvbW(hm+p`P8Y&1=-*9rW7Xnrb6lK3k3eG=)Z9*M62<|c(=5wZt74m_)N^@f;D}P zrYo^Ojfgt06Qh%cz9~`l(qSk6`)tjDeqVU|{4cWD`Wsfp6L|Dlr|Xk>T(|i|ty4$G zM%RmZJ<)$73^K*!t2Wc-NEDg38|31TkooFs)z ztebi*`OfBv&EeJ^#~x?BtrQp!%@$7%M@Glu^_g4rtzqva1@7V za<9|(F96ewxWJ8I9}(ZVZjI zlZ2TLguR%gZXAN<9umSFFBuLeoaP?*!fCJZK2fuaZV#1_lk>?L9BR9B7;CNX9b?~iVWSW5Mx3#HrIU718&B2DTV4{CH&;uh-Zicl=qzcxCvnUg`ML`Kw zZeHfy>hqLIfev0ipi;aibAmV8+eVGglyL9(T<*#BTXYPOt{K^}So7H9zGT zY$$wJK)GP%_s?zo1gNY<8{s9&% z{SVW<<;#Cbvv1i*?ND#4HyKNeR!iVpM#GpJ1`> zE76k{b!ETtuGzhyIkGHY&tn<2_%xQ?t6D7DRl}zn)CW9De&7qoj_Mqt&MAu5;&@`J zrOC|m3YaKkxBfTPTxIi9-)Dl|{u=u=ngu>?b6j0WnE66K&fa6_LK+n@amJPsoRpJ-b~A!} z#QnGT8yuW#?-k7r3O(m%-S;`dsJrX@ncg*ZS4AuN-t@Le@)TmC<39{1%Az$5e)(4J znuASQ!_03Vq~I@1gVwcuKHR!VNiEzZ}v0M9=&*RS^WkB?FXyaqJU8Om}7l$AM)r#nmro?Vz5AEDQ9Ja@K2;F}QuFrtBB8lPa*AZCXD_mc| zf2GHrI#~3e3NUq?am4Eig`nVJ)UMMm+e-PghqLY0c304?)`y;N#`;qBV9{%Ja3(q6 zC!C(ujYzJ+LUh6gvEFiw$|Q!ms^0Zgh%vpK!?yj`N+$q{Y3(3~L5Vb`WlB{U#yPUM8)C#1nuWPX}^t@`eMx4OQ_PcWt=BiAe|MLYH$ z?S%?mIDeE|_W?#Lg8k7QQM&qp zvnK>C>RR)?F5nwMzmrrM`MS8}d)3wmTSJMq5qCo5(Z65K6Gn}Y$YZ?kOFp@MU*L!q zAqhjC<6pHgw=$PyYMtpP(}ve#8HuLuE#|H3`}1_Hi~_1tygvsATr_Da-A(v zGy52iidn8; zxOEfjb?PC}!=d2eLY4~pChzhZTdgW)u)0F`bvWu%pQ6%=_Tz!^x&m3YIb zvC4d>N~g+DhGauhR}JB9eD)?;JQYG=mJe}k!oo7BpgxC+Cx6_~5+3{P!!Oo|q{rz? zk2SW9uH;8EmChvd8dw3JEu@ zFi}<=%P9ONmTu!qs}yeg2u8rPxp}li`tFc1^XI8o zh|xomaFRj}x*1yBAk3|tj^ zZF=*GVl3sdpN6vOYbDx5<}uMoeI0y_;3u1;jC)c+`-~xRoo~>rrEqbVRldBiePZd2 z_p@Kc|Hf`r)6f)$gk2>g+2~Vm&;EdVbK0@FnlyG;^J$gX&d>_5AGM}8eI4ikp~fGa zvIXAXg%gMkGDJvzpjo>mar^uvJ``ZErvrQ6roN&Ik_w_MG$0FGichI`gXBlN)jvz% z=Z5-)Ae~Eg2@JjAS2xcbzXmv=$(PuyCU34%_R{`2gL5=$024>mIduc|!Qpl^WnOTj zDNK3(44&SY;v=V`xGrSTjr29LwWL1^wOeu|N;;KE$@Q^oLYT2e2L-3vxc3;!mmqR+ zm80*vgT`hN_|>Z{9HnupC^V1I+`?(w$QqeQHV3Z#%cvcKZ+v}kzLf^O%+hGYAg+Ye znQi6pVR5h_8Q!~Np~ofh`~_ckIx5dK(+rr>>0OKIf?|;LW0M*qyMnk;&9IId5#2Qt z!{Xmw2ZDjuuYV)7e3v5CYPi@l$8~<2tE<}CL26i+WMz|moYpqt4GA8J2d(r zd5&vz(znD!_z^g--j&9&%Ka7o9_)C_B~M?bwMls^O)p8gZnlQ2Z2`~g)seqA{wZOY zxBNFkHRle2iD_YN(-*qi4Sl!}Eg_0r;;0HSXu9iGwcl;nY<@dGEhyby^Xgx1i&Tde z7DH7im|`-X@!eYa8YsCAT{|U_f8HQWdbL+P{8H+%Hys%cFeFB{I`Mw7YBU^^U6|-< z79NKPFiW?s1vs0{A=LWvL)&}6ND~Bbc4P0rSMBOOtkin zac(lpMat2i39|`7*iU1JwES1dC(s^2X&&=Nq+`(Gbq#isL1wAC^F~pldh9j<+f1=z zTnAcjn9p~l*zr*XZ>hw!9WabMlh)*t@Ll*9u+Geu1xWVVAxQKn%EhmdUuErCOX27X zSxJ2d@>FRq^e)#m31xWAi)c#5X%2?1ZK!uGfF$x?;^SMoGT18J_cFi8-$_p{R*W&M zkf~>5dR2t7$IM)pcw_9$rLrpO*yVQHM>>kiu$FC`+_zOt-h}cUtsr;c`Vj-sZl1fk z5`mWVt@~Q3RdPN)1HAzn4{}dEs+oM+8z{Tw%iMv$C^oTd>}5_SvJMC)y#z-OlWg4= z*=T(I_M##outQECn$4Vg7-~&K()j0n|8|)t*@9mT!*vz@8-;^(5Py*R2u)7-KPE-5ep>MfV;1b!6KCx)# zlr(d^92RxtM9ZTcHXMXg&F;ncHP`v-7FaGlV;#P$*^k1ZHvH&R0=WLdi_qY%^lLiz zAfJbA@*S)=UIRIv)#8Pr(N;ono}0At57-4Vo&H10f-3mzO5^Ots5 zh7nMC+xfXgimpwUz2}g6!k|KIm z<5@2jQH-3>k6NbQ>2gAqVxyzzlpYMiM1xlw-$vvWmX0`fy8z|qAI}r=0Sve!Q`(>H z`G(3~+-$N~YWV(4@UT&^v*KJMY>S_z|nB8KJ2BakxpJJep9meUEM@g4HBWEb&=sq$=1Dfrgl_1z&}ZAE}nXyYcz_U<*8Uh<~9DP^kNleess_^wEKqcYX|9=yv9(d zR-_U!q5{;)<2DGRi757``Rbk*9_yrV9)vUsWs`#?!EqH0>&Ex>s9~@Pzo*Uht&Q;r z5nL6_r=Tn8&D$#9#=CnX`~nysX)<9q;PMs*UPl!!tt&8L9eCVIPGNWqwT-0r&-(sCx@zm^%4`Egs4dMpZ z!&Y^jE(uu|h-NKT>V%)P9W*yFpxDrHUOCV#Fw`8T{X{VQt+=fEqtwe&X8UkJQBY)J z@oMw!YAN_}&kM?l&!aABiK`G~6+Q3~4PlbT-MlZ4l77Md1chokhe5C3aE0ykgIXLP zTJ&eVWfjncLY@OI_h-Im&h)4ng7~-wBCpvg=a|N)Ht&~uFb^$OV3LL*3ef6L_T)o8 ze|(Jag*LywDRFK->=nT|;bzoid4Jkb5_eJkkBM0=2A&mG$n)`w$&1|)rAd6oBOm%} zGvIm8YaSJ{*JX^&Oz{v+!iM&j&7yJUr+(a0U&e?yNGXl?fq4jJpA~tMhKDsdj?+%L*<9!KBY*{8AmMCLxw|etb z9@|~)^kF1*k^+>DIPMLQ-Y(p0P%>WYH4DNxI z==!PX?R-3MB}V@`^_!2eXP+rAX{pR`umX@avE9Rg^u2?Q_d7nWq}C7V=GA;nRADX6 zjJ%hqn8CI*-69kL;jY`F#fPtZ8%|`XJB=SO!|( zAVl=(%NGFCJKumKs5>&^MfDkWLm{iWohprl@ykQ&sqeQG&+=y~&5cMPp zEYEi*d#UEv58oTftVA1H%FgSC@!uygz0c2sX~%$mi4anvsJI>FJooH8s|I_JxP-ez zyg6v+7#RT&2=QsJ(kB`fx+-;lSJE$L~<6upBdecX0FS?g)VW7*ne z|HLZ#@QeVhv-o($?<_fLP~C@|VjE{3R}h4X2o=w0Vl?=jeRjkQR))cLbBtJ&kgAkF zczL!m$uC>QvhRfL3|an7CMBPXK%QH@(_qP_KWHGEM{L=@Al&efw=hZG59K; zXeW3ox3>pD$dA6OI0HL6!q4N*`;~8IBKitw$+;nMls?WI;ca6aZ3xvqhXG`<)D@Rl z@6vd$MHL$CWp74(yK_~%7n0;KNHN?N)K6Peayr{qyb59!_hMCuhSus%z@OGB!Yu z=#^<6=&SW>(P@ZB3O{AELpN=~Mq(}8a_H!=>E}^>j2h`anAImgX9Sq{b zX{`k_;{4a_RPlT^*X#%VM!~DV2U(RVGVcmhw^Jg`4;b2xv>61hsL}vjR(ZpIqFF7I zt~}@~58Pt?-POTDGic)9VKfK=!h0srd)@@DOQQ6gMpQFY*L17H*yk=9P!2`5yfwct zN`hcof&zt`#dvJ~IF1`@H}+tQhAL^3d7$WTH};%;?kVO6+*1^U38072!|9eLt(F&i zO!qPEhh-#%atuRm$z5JwO+q98a-u2w;rG^WV0WL~>?&)}N52h{TQtAcW@$2}{`SVg z5dsp^#=l!OCfY_?@(xC%g_DkFQr#?i#jvbI{tOv+TRze=?Pgl626F4795hUgP^YHBgB~M_0`Dn$^fZc-;wJ>%w zi+50CG3YgkV}}gEos=s4$#)=+_Wk28;DUAXu`lpH8;Rn7GM8{cE!MuLP!jd(rsFJ+ zdg{Htkv;uD6t@_ik7|^ZHs1(csw@K5w1+Qh3u~Z2=8lPdU*PBPx761$Mzm3Us6LzM z0_yOF-pC;6ON*|~5RmqnqEak&FqPN4{;|)I72;6({Af|Z0@!(GuTt|bQhNP*cfdPR zNmum)OEJxPfZ3^bUqGy~6Cro^0U=F|5|oI}p;BIZor#bNJ~Ws}Y3?3C{L|0}K>VKb zn>!K_L(*|9+$PR1fi_4_WOn)?vKt{Lw*$^Ut+HXD8r2HHAY;G{)+bW5veX4qUCiK0 znxGWr%Gdl+x^}AF16pkUmy6Cj{<&1V0M(YL$;Bmp3V%8R+r+ptZ^8)n>kfM|&Ucz- z+#8(f6+@ld}{Y=AHXebd=lox zK%*z4c%OsNqGR|;L{!&VM@WjoarT7*s%LJJPd7}hiR#G<)un`lb zAQ+-Eh2W~vtfv@bf+Qu(RJRMIoB@Ym#iw9){C}~Sc(}Fb%qyu2i!L(VIKu)E+-&tf zym$v<=z28%r1ZTSWOdD4+)&bJq-xc6!T5|aFWpVOxZY5baSPyE)br=;yxFj5rQn0s zhjn9cwiivX@|lYLR9erEsK=(w@Y3|B3mvye3MMF&?KmPW%Q+GLgu>!o&M3iqIaCC(3E=b8xIl?R!qk;+Cao62336Fv8AT~itlcOCN&3fEL{#t-97R1_P z`JI%5ui3{j86mFEU&W$$E4h&f!Z}+??+T-JoNrx#4~ooaJLK^gmOQP4h+b zgDY`mzo`E!0Obiq$_T^#C|9`4*D2AAul1N5Xsw;tuMxgEDSqj$cVhlPqE`?VuPoLl zVmN2`lA_8bC!9#x(uZt4Ufpp>eQ;<$!v$Kg?Or;;*}7L^O43w(h_28ZK7(X1K#$@m z#31(htAMeqcAD7Fz$QxMw77;{BgZNNwm8|>(-#OjPu8jv>S8=f4PVNEb6?-SBW#NV zIojxhRMDA9qjL?W9Q zVkp=BMi&ggyn1MQz5g)hpSs10bLtmX@n{H`#I`NN>t3NO- zDy)H+p}B7!z>JfLPx#roBt_;u`O+Fw+@+CxG0Jze^sUO% z?`!)fvST;!DrvzaM|eOKs{l(|^>}2t!a~N`zARWx$q||2|NVRXcET#zktOwmYZk)j$+sFf z@gr(Iyy}T#$h`VDVu&j(?`r9(u+@l)D{I^@2Gy*jx@o~CAt!wy?Przic2o?KN|xR+ z4Pa5uoc*GUHv);S+k-{~ulkWv3kV@-WY<#8xsU4xy7!OG9=%%|yGDzGGDU4T7VF(@ z=tq{R4>U-IUwoMtW+oM(YY!!b>9MbuhW~iJHY;!!aFRiiqczCHIGY@~F<7aOMT%iQ zexExSjI|A!I0mSTyp@>ynis`Xyfdo2YO<0sV(y)A`y2rwKv@cPTJt44eF5GO&i}na zoG{(K-6(zJ04u=g7*%N`)&8c+L|zy1=$`kR5UoKupdmQ6wyZy19~qZA4soAWVni8h z0R9I@QBVx$g7fhi;>cm;Fn9!>lL$$1I5Dqn+?92SCir#;PXm4?a7S+KhJyO(BOdY{ z(Yc%txtd=Dl3pBQlCzObDES^-^*LCSpP~`ZhXE%K7D#ttCH6u2WEw|VyYYnN^NX^l z#jnA+psCT0R0B-psx9G>qfaFsXY$`zCqIo*j=!Yc#BPrVwOXvX!_Ozd%QQ%Zv}A|= z(yNbNW%xXu2QmttrpwC4wQNlx9-JQY9*%urE7#SI1cqHzbGF=9^-2hn!ea<9n037~ zw9S??C6m@tM?|E?i_0_eGRs>4aPpo12RCMdLZ>cZ5Tc5Ob&vG+u4NcIk64{9=`m0i zS%J61(U|1-V16(Xo+p!GkuAb7De_L+sd@l#&pelvF{t{Mok5oSth=$lK?$ zalOKMDw<^t1v;C<1LFk{Y9Qt`axIb%HB7h)a0KZ}9(F089M{!&P)Q=tAHB6>3-qVc-1k zzc*uEl>%n4303zNq_Ygz)_91vojj>KjNco+{X%;C zt+lYQ=qGU9=RdC#n16hB2N(cG63y-rVcqd;32_tLVF?tDD3JD-9ka`ILr=-tw?WBE z$=z#RbyAm;MsotPNtwk7_uk=?iE2|o z3|QdFg!_ac@N7Q9U|_{p{TCrU2`v>;$*?5RQmfU>Y^4GLr*~ktd3{6OJD!^k^UJRsLF#;5Wn56q;0ib~fX-c+R|cu}!q`B;a94c; z>aIS<1udB(Oo;cY?}sby%|9%Uw6Rk`_zLO(vC4E3aiKMgKy(fr+%QgI_0a>O$2oju zzO185`)~X~RX>Hsm{)YbwLkIl@8x_V*J^o#aAW9ip2gvW+Ib3b#PIgf8}iWFg&qbP%_lCK!2D9)Q(E+xO8J!_y58Dk zMiJkIaz7$CYYKSd2|!$vzLesL*!k|=12nqAn`86Y(sYQ$og_CZg@Lp~wF`^`(E82d@ai}nn{iU4Il^TdhEm(@4 zRt(Oj14R}YentYrHaw^czmnxpF=Jvt`w6m4Q9A4%R8e>0Mz+~DM!r9mS1`up*(hD3 z`8j?r^P3xE+ymY1zKJ!s&RraZTi177+E3-B+){UPe%+EI#@e}&-hDDtgCbVtd%qeG z%xnv#jrMrZSGk|$Ig2U=J;(GwP(3I_-=4;r+quq_KBq=kf}Hr+ynC4Nf9yPD1f{2U zB%a$V_HIXsDMwKgZPLM!%xnZ?H%H1oZBe?t_Pxk{-e0;SdA;zWrXS9tPFY}D1a)If z;Sb#aCTq*lY%1xm5_{g{+_4(5{nG|> zD=|p9zKBqA@s}Xm&+r1|y)nYVcb&p8R!WJHKURa2nXt}qGd&vUi;Tkbm%ysWNN8tf zYxD%(Fdk3Rij8rdT|A1NFf6{Fm|uPTB<%JuP|%C)_#2k zyGGEJ#<~t$CEOLMZj$^KSRM3RIs+pqJywMp-8>)~CJ=oPy`aQUx^ZVkGke9vG;?#; zk?v;{7omJ>+)AC~4fRLL27s3Lr7U~TCo@&Sq-pBrbW>$obX}5xAOT55mhy2IpA+Z% z9|&}dy6ov&L3;`a80@xwc_Nxf-Q-P}63bQ}RbcI}GTVef&MJ1!G^2*NpebSZ08TJ- zJqb?u%tT02OWJD6ACN}q?5_0t%2yHr^Pj$^S$@G#B~wHSzMhPS+77$sfrm~}!d#=Q z_v#4eH@6QvhM3W(0uh?!{&APQQnu3QgInDAROFv7IwKfU&Jk7?3uJ*64;?iGV~lGB zixiTdfJAl#5H`Nh03${3kXh~kI}n-qTak(9lWRYVGr|skLJ4+UR5w{4r+rT3EC}qU z{h|J)LTMfLrM=>s;~+C^<{sj?K(SC3?l@5SR;1FhN`xZ1yX)U2X8?)v~Z};Kh_e>!t=(>Lj$o-67{|j*qy)4jo{QysAwapLO%LDeh-z_-Y!$ zwb(<7g8;!@`u4sSaLNxRdjKqws$CKn>nCgfj&)t~RlBM#PRSFAA>q&_r@8eL!HA_; zx3D+Fs4lQ)qH}7n-<^MsmWZN)lks=@z_VBLEm2F|E9ZsYI&9K)k$W;-$N3g!Jh;1X zuP}2iP;a36*~A|lPKbt?2B2E9LpTd|GI@S_3nB6ixcv6ZKb)DuSv#mFQD9;KVz6YM zD6|p9@zH6sVxh^!eBf*49BHII<4(i%>YD?19*|z4UTMa$4}A{RQ*Ty&f9kn;v`@Ra z@z{|Jt;KFhm2mn^Q?aV*=3}|?a2{NGLak&q2Zc5LuDkRNeWk@L)uA&!X2e3cJwasu zH zF>Uhq=hVS{p;0f}ivYQ3eUjO$5&}Ww0B?8$bG@F>ObX&qT!Gy@2?5j9FKtXMtzC(F zO5JFcvTB&YbP%Sdh2{R40iUAOy@snMb7)V;9iX+Kx?q(-W0`i|PBLT>gY=)E9~=`C zuRedx7_}xek6p;A(F=Qc(>Wwuu3CBV!&oGwt?9*!&z1?oJDScr;IXKoN8w?;#8vz) znMqswRp7^-(2$=5vk&Z~ApGDS%*_=UcJI7*2oip$T>*11*)BjPVTU~F8sZsad$+sb ztdCRjl%n*}jiTGhW$6ea^2fuO0w_w+;8Z`Sedp*mh%}Zj)q4XM{$mo75 z1eJe{sod~o0D@K-V23 zXL|cUPGh}$_$?+rn^jIU)Bk>GEInJ*8`YW{>)jZHk#svP^oV{20 z1=}UxjbJNeRLv^NF!Jfle!Wm2F0=L$ig2YJ3Kl+Zb0JjD7T zNe%jC$$|Co0|?Hw6u`&|)s80@*5)>OneX{p(}>jy$mT=h(`Nt%h$!6T!bt0Jn*uSi z7#9dCBs%;I=G@Nvc{nz70Xe?oU=Xvc75kox$E)EfR8#s8+(37EMQOc%1=ReL@G8%z zOY}aNIu5;+4{AnrtZK{%B(?mdNcqy5nhp^H{e}H8Gvf1{@Ine6C?+P8a1Qf3!?(RZ z4Abe#vZP}qKX$cJ+n|$-BLl&wc$C*Eue-pjubyUo1mJ95R{Mwjd2L(dqM9Vg*#;ui zSKflluA!-!DR>Dyj2J;F7&fS5RSnb1>Lp>KTIiySzHGunSl<(ZT&ab%#6%2Tg`@BF z-aPgS2gnh=C&68R4_(ylu$Mt3`u<=X$xOo^51GM|W>0B`V~`Xg&q&C&qG|KVngn{a zV2#a4Hh5GL%kdf@VqLR&*~=+lQ7OaGO|f{`m`wwO#?64zGZW1%sif6EBe$CboJ6U{ zK&9as$YkO|zq=TGBrD0bobQ_50)Bon8Z5BHc^UPdNpXRwzP*0*>1erjJWfGIDdy1K z#hwCME`aYSQI$peedA+!bPj&wdm3{;D*wLEwk5p=NL~lg!3n%aRBCwEMQeU5g^mwT zf}LfNRz?sjGJuQaZvu%Rgi=%ynCg2fd9q&f?lHLMceENgI$rv3AO8IO1UwNU(`Jp2YaeCU+62*ucw19~#6P{5Rrq$&K!$i1f?}cYeTc;p=79@A z*!8a_Kq{7}cH@#Z5v8OpXxkFNTKM6PhPcF>ni{CyEmr5Jf5Wj8ham7lSIJ-k{mh4#yL!e$>jZV?+>1+)WA>@R4YJ$z z8cG9bsP6jcABMnCc`YNTAtO3b$p&h0avM$eFhdx0s_2$4|H_D@Y^RW#uob?{qWT~6 zgUT2-{H^rtQ|W1z&8LP%PBlPHzRd3gGp(0hwR*kpsZPKSL%#VrPZP!|DgXBS4Eq&W z?K+zt$&y(*q`%0c$aH{*9mpUv_d7qBX1#^>({LC8u*HS%0)Rf!hSZcPi6xDLsc^1W z4u#7jh*e9alh0mpw6|xH`a%qQwkp&(+!pIhZV#QPHTSrAS0Fk#Fd7 zve@}h?udp1Dz5pDkd4f)dz%*pWViO4@)^3zp6>u<&GgSL@c5h0;gl{(^2Di?d zhFhtk3?y!FrnrRl!UuSyPT4{{1D9GD=?79GeJc~A`~g0s)K*`GD*L-CTX8weB2OAe zzmiynX1Q%?2mz)CV{u;4p*~Qrh4x<^t%ydUMLcQe4Br&XD)~dVR`~CN(Q~RB z9aFuEkGk~?8I`;!9x-cH_@c=skM5m`z!#-ZiP^ZtX+Uea4x#aHxN#Tn!e=J{iVg=& z%j*leZ5hz2<0BKQONSMV8h!pdxumr?B;PQC`VnH@%4#wW* z4rK;9uRto2Uo|u(TJu%*3+!O5t z(za4=%6n0Hmq11LfCR+qtvt zc<9fu($DaqpRpD=<0ByP_4HfytH}9;uUQU{`f=!>oqBh|go)q|kcoUZlRx_gl#>l_ zXTE`%hl6ZTEnhi(lB&^DuXcVIjw(LJP?Gcwn8(cpnsQ|6Ix5?Hb3*~#1rcf)V7t~G zOpg-L>kMY_v%=a&NIz3zdHkrZ(G^@NC^2vl7vOvOnw@3v*l-%z6^A#tt$Y90dwm9{ zmB(+^T?p8+DP$0UE6TjT9V!HoXaRa-D2fh&%)ha~O@{<&My~hq15s@FD$IGYu)S>4AaFV!d$hK@=Rxn#G;my^aGzzZy4M zJ8l<07wsgi&Q(3t2r^Q-Z8>Bd+6`tYRewD}oI2*JD?s(ldZc|t+_2z^WFsriHSj+j z2xY{}iuQgXeAm4eP9nMhM_1sA2dXz*PWCml-Cp6(fWU13s9q(s%~Lp>Y~CZLDDP6M z)3+NO#uxSe@d9wWw_`OR+hz(((+Sk_gzEl=*WnUFOk=u19(rlpU0flJ|A6&{E2I$~^tQ;9?57za^3i$!u|J|=v2KF+Iv zP?#-_O2*5T>JffVzu6qKeG$zrwT6&ad_i{`wXDNJJqIp~`IPzvP(-UsS%9>v*+2sH z-hS>u1<1&bp@{stA}tP;ZNBH+R7B5VlXehs$m?-nTlEu9{*NXp)?~9-Z_OX_I{D;4 ztl9N(JFo-W)L2H2Dx`gmmNfjBlLI3_a`FOjQNc^`j22KIl+xCMzqX?UIlqJMol?d$ z7Zfygb4+J zSqTbr5C*ajV^wONS33(wsy+?;T02PUsJl!TIRQ5d*l7Z7s8J`&kMOeaLN~-}^V1ei zyWm^QcniJaG=4bN%5qqbJRhev>7)zlh$Z9DQ}^*ah7IMCy48q z?SY`4w3G&kQ)o`TL2wW$UFai-LjPt5CeyR$Xl&NGqUDpeEWvfR-d%t4zmRS6kIL*+ z*@uKIqMns@h}wTm*V|PW5(ia+llAg@nUF@2fR({ZFp(U`QB*)e5x1uHkEgziMt~9j z5R#YsrYGnKP2(`l)b4B!puUCOOKzJ5n)G4Jw2|^!SW!o#yr3T-pSRWW@u{N;O3J?r z>H{Uv^QjypR;_WbmO%!{qP=1BxLqok`n>p{sfR05q=F-~l_b;fumVsxrrMjobQNgCLM``a=)AzlpcNjZ=LG`T1Rgxhw{}Aa? z>V3SrRLdu3TiJ}Jt8H+1@Z&FKpz?6eREkdE4IixZ9NfnGY^1?B_s$J!P?p}=X}qij zfM)lY0+he0%c)so^!Am?19&$}q+$hIit18mH&|7FIOeS^0MS-sSNOe0OSI4;yPjtS z*gcb~tr*B@_{|1=xe1N(Wy1UzY&$>4trzcYF=2**9z-#V<%eJus3)t$4?6Og@X#;l znZ8RG0I#T<*bqash?XrCB+1=C5p13E8UVQ+NdJCEtP~s+kVtXw^o_#2=cr#TZ9f+= zT337rjwd3xR+hbNGh;Pab5|^(>nDLS%ZyRMB-T?7q+Icg(4$Cff;}tcA?ODy(n{K8 z|GjIU&HB=Zh9p6iDpmaqs9T+AG-q|Dms0I$ml%(eD~w`uR=(owkw7$^Cyzml`PApQ z-}_I&jU87@a&_JH(4pYFI8Y_G24r09u8oE`MRQh#Wo-kdmFqX56(xB^7X3Oy=l8`y z4S@KM;}}Rv7`;k{nFFW!Xtk%Qe$u=I66`OZITYw@CyhFj9;gb$a@nGx^%!L{hKHDi zeQUBt9KUhhpiccH@#MROx%6A?cZ70 z6c2UxrPlZ6r{c1N#aSI29Mf3TEP`#}I8GB>eV8JXI$sd+!=#`tpVewwlpAyibIkru zlBq0e^tts7E;1IY!V=gta6xktkpdD>aC_KQmlBb39oj!-7&(+jc^4^N!F@o*=;@4H zg`ua#uo5r1Jdo4kSjIG06uypBAR;Dn@s`H#%p|)33we*N$mY!dy2I}h-Kdpb1!_uw zVYl>E>_f`l)^5|T3kV9JP(lAnN=8f6oPij%f#a#D%%T!=Dhd;rwPz+zx3pTN*Z0UC zS0UE+l}`p$&cX9t$qYk9w%b_d5ySsSAG3jtL3|Ya$B`C=h{q%;3h4`|JBT3NH-FwS zi2*GZjY-KT0tVO9){|!!LHfCc`SYaE_H^qgnKFL*+BSfX3q{(xj{6Mwb2ityB7OV= zPw)mFIrR=w&Ius=-fD)GR%=e}+Zo37#+s~#V71LL#{~a1Gp@vnHW!b;-d?>OL~74m z;1jfe3Yp~oyN>;q37Xu01F?=&mbM_CGgx&WHTlR&opIq%B&$unX-Z5DxE$VF*y%UBvr4$Anu=eU+K{4VRiO2l}J<%ccdouqh z{ZE2Z2hvhAJMegal-%1uj%D5UUqot2)c-I}Gdmgr%;dwc%_&nbPw{hc({d{r_Q18Q zUbe}|fJ1!*0I$}W5#9Y7)SG+fX~`vq+?c@$(vr*Fd7Ab@gZ6hh&5546f6q)v)`Q%E zK*qcMr2^3X^>1)b%>Ej#$GC1h_|a6;`6(=SIQsQR+5`La3QA-&@-0L+oUan=nU-pi z=nN#&-o58pIpqdA!gXo0?ugCLGmYiX6xJapH6P4gqBT;VfMJI4>5+D2Po6jR0mC|x zd!^K9v_X{!cdsjDz(sO(Xm#s8i?MauBQ{j~jc>oet{Bd^8oc!Wlm_GNJJGXYT2E7@ z2jJAe48Fs`=@UTE!ka;XGz5H0HgefqSQz|8scIsO5q5I+EjP&u?Dn*yn5g(B;lJBe zWSA8;LHSS9EbovGCLh#(0LJ$Y`O#x0jS;X)qxcK{CbRw{ZH$yAwgei{d$#mTt&W!#*O$4J5*~*lO*11^^x5GYhKBb(LRO{S^Xxc zjaBJ4iSm8CN_6?nu$8#8`jbfVUORo$1t5qguODnaX2r`_`+)&?DFD9G?}_28Jf9BSrhRg#lhG1J;@E zr#Z~D)E!i$*W#)%sohT>O;}P%7W=c<6srV5@{ZXH0!N zo91qQMtJSWC_CW{dZS7aN>Of6n8kL~CEHv@Vc?y;OhM}J@86WIU=5VWcMAYdy(eo~ zGPiTcnVe1))(!5Ut^Sn>fH5`NfVl2A_88(~)^8;xGl;1C3+nEZZmSLuK? zM4(9tWJbROweW<6sB94%$eBJ;Ad9IFB~ToM?WlWdpOQylQB;^A#nt$rZYk%y8yYvS zE@{?!D(kgfikyD0XVx%^{fOGRAdF%FRN>E-Kr%B1bmI3@YtPAQDnZ{~PHda-hp62& zDs)6eK4}0Y{`1QmQSDmuepx0>K$2qYKQw*!$5QLj#5<(}I1YV40}zHdc2U=oxU&E` ziyqVr+>a2Acdud;>bs}A6KA(*brneQQ|fZDGiVIVM+Rk0S5;=0d24pL)O%Gp8&FA! zzgo-x4F8-3Tc3?$V!{Y!+yyWZon5`LT)7@4x*sNl|RD9^S}eYOgl_Sa}e$@(mYqHKb2%R-@3}& zscTi^Tm=h~oVY_qqV@)a)r=%YLRld#5$v=&U`3drJnM*Yy|@mdf_iLI>BWoOv=;dH zj$pYVdn|Px5V%RAXPsuz$Nk0%w+{@4Y6qo+_vVE*Ov{j&wHEYZxpUpDMn3u$EfCa{ zN1%?6^CYb33fOh{==F!xl`dI@SK9Z^7^c{wB5o}&C@f6=ZR!F|Pwaq$`TRpqKPDw# zxV7@8Vo?U{kSA81#^?_bZ{Xzy4}K5D%#qS1kcbg-5975MEoY+JgbCUMy+>@8m~NX& zHYAkd8y6%{6IVYHGjloySea3g@o#a1WN96H6>L%{p|z_&fqMDk+3=sFs4t-N1=3rN zsP|b`=-YjRvAsa?iVxY)COiaZM0gAPxeMBz!DV~K?J=V40m5p9cV&Nk#pIy~fP0$4 z(@MWu0*A6aFvjQHXiMuBtKjN)Pz%x5%V$48N1N8M!$Y$m=>P-z=jJYqJJ`th#s7Su zLeOTZ#jRo?{a>m9p;x{2R{yBH1QezeW7m?NXINs@hJYkxzv^m3LRPf# z9i-ZO3vAyM6IlqzHBbuzarCbj8Yu$r_VKy#Dl zgDjNY+=Ad;hLrNx@|y+T+8{igt*4tqj$FY&HZnHS`#IRVwh}W~8w#SuJ-vY>OL);@ zz(9I;v3|W~>sCjN|M@QzW(C+6Ms#_`xfJzE^4M=Q7Vjo`grff3c1=oS5R|<qOg%3nbUkXBWS0uKwhyW!w+a~^2^ zkD_Q3CM||G)=4!+udY=;$+&AagM1&BR`Bg;e1&Lyi&-uC+oPxVpZt;BU_+Oq_m04S z0Q#qBS*7ep9+gp5l4}5rGV@ZcB(Z73%L^bnXyHkALraOZO90y=9p(vJ){EP2-jmLI zy5#g*kxQgVyFK?W2ltUhGgeNc^#m#5{1|Pbm^QoOy6`|-pPjK^M7gP{C9b6)84Dv;#%->$;>5g?)$`d|#R^SKSXgq!cWh>aL^(*U(obH04Y{ zxXN9imH)e=&dPrhLxEtv?&w>GRh5y+ZPt&N5Bcl`PJ1Tt#=n*}|C zAZ!evpx0J$W0PyQFy!E9ML05!z*9E0;*U1IjX>K70zt;z%Ne)l7x6^rVDNXwg7Vxim?9EBBm7yuth`r! zc9EG(?`B1YKkonW_10loW?lQR+=3DYh=52)OCt?ROG$Tw3W{_~Nh?Z&NC`-Hr*ugv zAzjiSrF04ieCu+ad7t;4`5oUM^BnUWGvj?-``UZ$bDis4=UV>s;Lf{P%0?ZV)zM-z z7Wxjs+oFd)E|{9tx*!<&)6qeb6?}RICN^ssz=XIf%0bflWAb_}pAD`#yU^SxonMdK z8PsHaue!6*M4&No*}iy=c(5SVmHV0M0{dN@J$%|Tcm02?)oK#B7^(-PS3WqPw3+k2 zT@A+D`uPDXy0%SXS}6;6$GQg2wz!UwTHhY^Ce%@WID34;x?Kt5Gl-t9%(mW2*U|lL?2R13j~C)&)cAKE|2EC?78^&t;!2Na`IB2kUC} z+ujHxJ_#x_C||vP*(T!ir1FE~w=c8%6mHdD2G*R~aB1NB;k3rwTqY=MEVYIwf6~|pEPj1IyYa6 z&a2E+_otRt(-3t9rv(PK{kT{Uoe%Nro3trj;@TGv3N@+dSP19amBn0Tr3+t)n=VI& z|9{M`R3xR2>I?pN3m-wXCWZNU@!chM`wO z1O2n(xaAT}Sauufoh#E+CaZ5KdyUYYFLpu6Pk~-sW*ZXhbcgdn2QrF0@iH7%b7-o3 zJa~GY-*8Q6QD-oC%wZ{x?5duT%`FhsT!KN{TFizgR037&z#S?@*Y1{jDm1x=`oqBZ z2rW!jORYYbd8%P1aj-MC&mxPNByivMbsWVzZo=Q6Jac81h@Zu?%Su?vYA@qw#@hu5 zO$#i^ITw5AzyyJD74n*FN+odGzGb-O7q|)P+CuFj(K0BeSA&K(H}FoJxK@Be^_@lU zCeWkqQBS;B2ShPwhEHz^R@BPoN;8kG>NRZxb(xPTW%*Md

xm4R@f_{t#VI7H?)&G2^!X}h z#@vDY{t+UHJ%Sw|#cW=5y@?7V@By<~-Whk{iEAvUp5>0-?X@p<-g!$P=_81FKn1oXN z9)YhqjdugrLn_^?H|{i2>GGl`=oUqv=zjB;9_BC&Y)PTM3*;2HiVMDV-21$z#W%26 zyoNsU>+NBn4ExYC?|jQ%ihQO4jhwGH~Z? z4Bj@eX0h}VGX1MwfIp!7Pnpr|)(g<7$Z+8QbeBg|#t(EfuEzKpu?~LHkiMVfif$9x zWQkegNu0%8{b;Xu?7hCvIe{RXLrW-pt%Ewk#3Ua}$!Cc4ZGLge%ZI+_IN_$U4t+_) zRhAT`jfbkt*8>6EIuwtTndqq|uKD5g{|2R@B>zwV>*uYjj$NQ^O#U5|a0~O4$5{?U zemkTiT>(~@YN6*yLxEY&L#qBBHr#|y^(fE#$bbwrYt`-d0WvAw*IwrYL(|r01eX-| z==|9k3@c)~jxl1JcR<|mD-XPz%$oH1nqaPnIt4{s@m=fLGb9moX#Lx~Kb?*rr2m2a zaC-kN1@m^XSOZi6BKL16%mX2m_7jVlY^VQBZuW+?DBKYam?&ndSx=EM@0w*8vNb@- zU21j%C3yeb!CgY5&AsABFOwRGQD5$+{f6Nq2Ht0w_0reDP@~m%1aU2LYLmP?VGU<8 z`glqBESt5pQZL2rRq8wGQWyNJ^DTDD)a0LzU*d|KuOQ=_*#2}rf_Ean_E7X(Y%mme z#Ueu?M~ry^tkX;1X9f}!H&Y~DJf#ukJjOXQz_zlbrLRDct^I>H3|OSY&P4H{;r0yP5S;MT%qV1VIJL>xR-=*%LH?dmP4PT zmQYIk%j2M{IK|AY>n?4xM0>Yv@zleV9a6a#4iRU(d#_7b;}pXp8GA5`#=KG#C#1+o z)OQ(<@Dh;*y_}q4{#k-BdK%(mWs5d~K=FzK$r%Ij;$`8@ylt}9+j#O&9g?J?C>T^& zzPMYp#2?tn-cCCI6$AteSgipN*YBX;3`8c}=Cp}9kK(3ru)^rEx_Zr5SK23G2HF38 zVN$eNYL#eZy}SOSkUltYq7Yw-3s?C7(N6S9-I}zcM>At}awovf znz<5kolL#0ye(q5t<+YmT5W9@nzAVosb1pA!5EfIzVylH?$BQKOzo`@$UHvnEOa)d zgq^$7i<$Hb+NOZy!V@u-enc1%im-n=)ULEtfui-PMKlzVXWi`eoL^I41?h^nH5_wA z_=)=;V7EaBQPnAGJo2?jFy@TpPSvLxQsCs*3NBS zN^MB_eEZk53{!sE)TK}FGD#eEJHzf=d33q^-Y=wkN`(!RsKg-jw!8{->!L<~Ad*-!mg=l)UPi) z<0mkRW6cV!rh&3o96jcGDemIJ^|{BUbNMXh_bF5{!CQjm$F;z@1z7&)wRz*TS~_!6 z*gdI<%te(nC41f%$(Lc`-&;q5?sZ)Z)T%%z3GdgY!?riA*b`L-MLtjZeD)X%qTPNe z)a=vP@drr!8js0pfHc|njQ;-or_vTGTm#87cAP9eu|KoJ5Q=~lY~H7!^r(Q!ja2Gc zg*uT8adVB|#7z`en&?@Z`6rC!BpHX-kr5Ef(2Wd$sxlDC|X&jXWA_1U(FHX3!_aG*ekf+k_g_ zxiZLKCBJ+--bPF`88OvZnK-ZD`7J;mmruXgV9%}FBBtoU1O3tKqK&OPA<^TQSM{0D zs^ZaS2i$(0ufru-rO(Vhej2U62x9YFx4q>FUV_!9T<2%*Yh{YO^2JB5w|y_`bp%Q9 zFSHwB=8m&`98F2<8k(w5Dhl#!O1)r#%zbX9(bmgbGL19KUwb!F@!Lrwbxf-YSrm56 z&tR4u*zG6nyG>;wmk||68AgrXf=wGvHYM$7Mve77JUcvaj`14Kcj@cOH=irep}mzO zr=Il;5@&y(U83{2ACKZFSME2p@=d8vi6s85D#?PT1HYu`0{TGnNcd%r<{Ir2ntS_i zwW=lO63G13C1|*AywzehDbr5SF3zK`7D|Y=X_qMR^4o{=D^0ik)@DD0s9y!14I$<) zc{AeTI?20lU>4eb8Jk)yEDRh=esiqXFp`EUvP@`t%a?AF;_8+!qX|l9>~$Pw`@= zQO!g$haV2WieB&k^+{kbT7ri+u%Gc{bA%(5Ktg-dL77xCMI`7|sM5ewd%;-G(?e=3 zLrn)Ed&-VtsJRo65&0U?>~iVer1aSX+HECK)7^G*Py`4-zZq#Y=vY zCg0%(U5H#7ZR|6es7rkad6o}z)81;4OU_xTE76yj;2Bhi#dTisq_N_9ySgrASci)x ztV?6X`Bv+p_@}BHh_}PrN@oe9wjlo&GYZcnEq{==B8$us>ZSBb+)$K&Ia6zF<(}+> z(90EVq0h%aBW`|^nmbYdm_=W~$7_{GIqdXS>)sDrp4XjUpo61ui411b(PpwQQCl}8 z)30~ZF40_-4#S;{`F>YOtmmBoEkeJM-FkdFN$LVoQC+pX!yS;v{0tUaI#EwBPh~Nz$6UAlqTgy|xD_AjUwc zWAF74ZF4-PP|98!bQp)pMHXL_g0-f_v+^-2pR$(+{S}*Re*0*8JwU;48X-E?%y=r) z{(gE-_WTIqDj!SDRqL8NI@SYn0Hq3or2uA7do2hO?o;CBT+vtjZpq@^9(^E$OWvV0 z6cF~`HJI-TJCM;ivP9zaT?%#gv%IVVG2(d2F_am*o-8TJ5#(mP`H2XLtQDv75 zgSMj)sGVG!Q%Dc>9j*u$)zh#EQTnDZMrD}(w~xrzjD+rE`}Ot`{@=Y20muc4xUMFas+-yu0T``In(h9H1yOemt(el3~25Rr=e0?qQ6Hlay z74O>~Kat(eFi{egQ;YncNt(PA!eiG<_Dx`Lw7Oz^oDy?qKN2AR+QN4)q4Dt2+SwIp z&{NqSbiKR7w!7Y$_wvVEWDIap9tic0b2>itNF?w1GfmWfdTYn7_|$-RCGDN$wY&_P zhXT4ZD$eMFgxBwRT+s^sv;{OfgI;KPTC`b&)9J+bGqVC;Lb{fx@4IsQu-nZx;+Ulf zJzk_6eh%02IW@UQ$A70(X2>1X5onzzILvwzt4`eBso@%a61yvFB->mFh z_Eeyaj;?+DO1B#-St(%9S%1egjjj1ApT%fV4xNKbZfWGmf8R<63-gC3w( znX4@u3G}41?)wFmHjswJ<|^Mj+NL>cFO_92Q_>!R322g1xg~)fo1am!) zoc9yotEM)`M9!^ab!GGRRr*gz6@~W}m?Zo(f6Dv!mx7`jRpW&BVGGn(h-&ey5-eO7 zANZl8kzwKoa~o0xl}?^A3* z-u`TkeMHY*0{^x`*Nf*ltipjW-%?9Sf-M6JZpzE^aoD)2_w_=?LlA2Cf1=6)?q)|3 za4#j_xe9}5!t@iqL(pcRuZKzMe$@f>ppCn}ofUQd+PcJ|ksCRTdHLin*ydqHcy)jK zY`g~FIWFhyZ2lH8r#L19UPna4cU>xma4L>84t13W8T5?l+Ttpd>7r?uXk){|Op>lq z`HB%UJ`E}iI2PX6JKi`J9H?7wt#jJ2fB18YFVz0))Ny?2zPePZeG66d2vukZ5? zCmPibE8l&*SFre^7*yNT`3v-_?7}sEXLXcEJ$;e*{x@avMT6c=6Qbb0p1>$8^9#=; zF-Yh4`0El$tY)Lz^=llT3-fLnv+u7Aw6E!wnY?{i9cLzt$(Pa5tbl$^SES){gr<0f zOERe~7rGDQ9rIykA^~BTu8Asbp`gw?SR0S#GVP~{cGH`VctKPhi;-=;N5p0jk9Vu{ z1(nx16efk22ny5W^@jqoITk^U^v7%F(65*D)nT&w`InLVIytUAp1-a~`CDSwCBBRU`?~oP zhv~W(P6ws<)`as=0$JN<(JE?G)K@5GLnl>DdwdZ!c3(twJu*v6mK{NttG4~|PsOm7 zNrW&)?3TWMNVE1gJ-eM0c#oD})6UdAdq#gZ=SR66-qyy4#Ovs|BInT1aWRRM3QY9b z%1rvG+)7{Mv2mBPr7sM#vu%F4(Cs`h&Hn@X!^srlcr10cpASYbsP^v?PN|q;;5F;k zdLg}n?|+|yc+VkxTY->OkMBVhQ61r+SL(KA25n|Fk%J`+a(C)rn98J;jveli-@0F7 zBUNEFIni^!*8F+8YJbo6CtsY>qBo{zCa4h8iaBZYYZ@YvUw3b5`d_0BkEo4#Euw@T z#r7~E4Tt~aH)Ew_AX^g6HyDHL8XkpiN|;sapt@Ns6-xe~^jVLBK?(jD*=WF@FLMs% z&pOj>SPO?|J|TyWAs!Hs?ZKf-s*hIny7Jk4_sx51(*%AjDrtT%Ld(8;t35YO|JC;_ zN9Vb7Ad!ull3J^?LoK8C&u?<26QAN9qC#dxi^|YXKlM=F^?dTHK9$VolIO;t5btlk z27&d7s?uX^rTYEl;nHV{71m$EY5Qv24&%L2+1~gnP=+0%j(Fr7zVBM4kwFK0V)f$B`eiCDiKlAax>er08wR`!4nq=KkVl25;h5h<_8Pp_ z6Ja!r1+$R?vV)dD0w%}NQuO)y$8?rzjoBjJsCCyRXc~~&$y}eT;XNEhx&iA9MiQ`Z zW(*rR&tf>`VQ0T$%N#bZefkr6zeAmI^gCTgo>bW_$n>jhZZ5v@O0BQ0hnhfmO=N;s zs$D+Q*&`X2R8+{Dqlt*4cah`I1q%{=;hJE{Ao+0jx!v*Hf>RLx=tUAtzN1ID%P5ow zUN0NoiBD%7FPra^-yHP2)CFGQNIX z$MHM>DmtO!3z;X>{HhOnFSSH`+g}@>#J;Alx?-KX{LyvK?0_7&3fUG`7H!QY;W}1V z7+|l~q_}ZczNu$9!)=nmn6qk_Dgiu3O~)C7sPH68wlJ! zls`Rt2{M3zJ!%#gp3uc5$%E#rJzy~ldH%Ukt#;(_Y24|@EtO6#%Wqr{N-s3W8FMiX zsevM4!Ypjl-8C;hVC`GE{fJaG>(Pei8Huf*imG~^R$q698|XW!_FWV~So+i83_25C z|@_gsy98ILh2oW4t&$wqyE z`SA`-O5N*&GQ@>0g+qHO1z(!aKk1-7h2jKFFuCA248}-BJ=y(vZho`K4^y|11sh&! z`x$ua;|EZttUseScvzcM(ME?E>}7;2=}*jZf&6bT03oWdFQ^=g#RUee!jPyhlQZ+U zet0+dY?5kjC{MMe%SN}YK5Op72ijG5f_H}(revzT=Xe^S zTU}d?>0X!yo=Z>jMjWs8`V?#W2%|}My=kl7ub;bGLpiDAUq9TNykwJ;HGhG)sOe_qVk&RSVkPk3($=>Ar`@Tr>DmdFa7UCl57R!A=^_8NmYcxaQgPCrH zReA6Bw4b}^ixl(DUXI5&=^?gDH)G$ML^5f9RHESM&b-#qY?5sD`J>B@F-~_^sZ!Es z2YvDTGt%f{Rl<|g_mUskca6u}bubWilv^d!{$fA=UFq$WdP~Yzh;jzsn??7d7X6!7 z48LyVSeYkhpAjAnciyf`B^i?tA+om||5y&DaJuDYLl$4-`89IXvRCbn>IHgwD{=@s zPGU&C^lF`pPmVVl3JQ;esdnYdKR}U~Fz3mN)zi+Xy#<$EhX1^u-}OqE7HZiM!9d99 zXqY@_hj*{WVZ|_+wrSn^Wv|&AKX(DqbKcZMN)!UmzQ$`*Sn2B)JreCAVWdc4dniQY z5Hz;Zd>Nh8osZrD3r!g7RXCk4z2a+zB-~hRYovPTo3VPF9k@&Djp4X4wdQdOEx4RS zmDme6*TZ z*Zmpjp!}#5`IOhb5Ml)02W)lvkHG4Fa$mWT!$?{j3jddqigmp#+$@17Q${qOH{1PIk_G)~K^J?3( zI{h#?MVzohn|SN>_(w}lcmG=*r9bG-ORU1&cvOfG*^O#nuW9r$y^{NHuIz*VOeJv$ zLfQI;6CCHq?cNrJf0PN3fK~WGmWlsy#)o*mCUQNDu)3Mlb#5FEweCBIX%1)$EX~7p ztQlfhL+=iD{))F~=o|)(IE>JAT(cfHy{QOt{0QHGwyK7U5$msCpF>G$Qm9=YSdiQp zE6wzU&Y0T6Z{_BbX^9&7I#qf(c=z0oUU{V|UVLpuIirbo(mqo7Q1}=&oIYMCh&_pEXjly5 z{3?A#jPcfY=y6|PTO|KaOydxN$5e>9(*(ayw1ov|s5;Ij>FE&Q{ov7dd~{qsJ8w`; zX=UB46ogJC9-Em)KVH3(C;qasgSnb)BtmY^EkX@&sFp|GCO^ddrS)j6|E>9&)w}cz@{~6?vk%bLq~s-nJC4YWNn2kq&mL}hFPsfrOjhRcA=+y zlOUr{{xJP|d>Z8+WE}PZ!@y;Mz-bLi=R{T0k$XQ!gqg1_m}{O^ioU#{4c%_kQ};Xb zeh%e6`*6?6vDJvmaT^^CJw)r#=P*huyB=sSG|^cgm_i2J4y?9;r%pdIF|)q2A{2}7 z$kAl^2I`csca?^v7>almlh@4kzn57}ew+S~;Z#rH41IBVFAml}I&B!JK5qdf%-Hr) z&&K#^e{mOHZ_ms6?mc5sM$ALd(Mh;fZoB>nyozzF?$HytM}HmqE6rj3f%YjlCX$2D zk$b~7_NZR<9zGecN(%9fR^TRqTmr865Z^G-ZE zs?iy#@AG{RZ4$~PU@ekMbt|Hw28%Ca=b~3Lo(P4CNEKs7UwP_W_FxU*f+wdvJ#Ppi$i(ZoK}wmRMKU}Krs8X#%be< zCvhmA$P7IueP|Un8)N0eBoS{vCyVL&IuOO1TX_P6px!^+!$Om42)T43Mj?V?(kyTo zs42f5MLUdkvyyq;?ANs)5O1o|4JpmPqjFA6X|ei zN2lzJVj=OsgdhBqY@~_kphEcVz89sXrTOi^pBs-OWVWAbr4x`(+Cjo&;YXr|^vhUI z)w=8hH=+Ys0M%3=62_jO7-O``Oq9nfZH>A}KG=Wd-`^W~IKJ2&hYSM@8rx`xsC&Z4 zxQpd|4%;L{XQV)X^14m8nris?z4gjwyM+#d#?s_YVumE2`!6qE1%_Z1oWD>VgacO%lN5aH8beiUh^qd9t294?g2Z=HjXlmZtM4deR2_#6hMBFw!% z2DzNP%XgiR7R2m0+t7Ve0U38JD*jE}ZH_wk@6PBYFB_v+bn_fm`iJ3(79fS>5sMPl7-ZX64-sJ8hp5 zVq>|G`;j;)2k2fwwL30%04;RpEkX$tUX8OjCW}SJxaR#|iyk*qO-XZxaFg(x9I9gR zNnWB>#1)w_!K81G#rDeTD2feN_~|1t9o1iw**Ki9tIu5yz^fUJzHiG9k9wlQc}(?H z>Ha&fdgYKSV&L@<%HH`kf<%Srg#m>MC*_`2WJ#+1qr;Wo%Q%0|lp(eWVM>(dkAs+k zhDaRU(!GUmhAs4UFOq6Taft;%$S~n#De_D2F=F7JB{nUybEeU3FWuyh0EKq zxpMmVB$5p|I`A{yY-OU;D5RZL!&DRPAu7cd?3uz|c0j*AF%o?NwA$~po#mV%AXmm6 zW;tAleM}l5*~m^$Eg_7DndP;eB?wEpp5f-6<9q-XYU4oO6ZdkajGGelw&p~Q{ETy4 z*sBGlLB92LebjBmgtB^!cFLK1XzCltgtPFMVel0W#w)BtR|-`wieIE82vXbpKw81B zXMvo?l1ML<|CHrGhP>6slmH2n;gt2gbobhfU2?`%JCOO&POFId*lI{PSq}0&!RnG=^17Yca}f9{rVzB^Z9pxN1tZc2ylHwZqoM@NoN(@$GteD zt_Ro@uAzGvYP&@81|wYdN}Tw0k5blC6QdynW02R5r0tzTGn3=P-3&*!Ooa~<{iX{v z3i&HX@|<+g(6xqr;(O}_SX+$26u8IsOJB<`T)z2gV~!n#2_|3BIZ5id8a#kYVw{v% z(DP&ah=@ha;O)3{b_w9^dMlC+qaA(vz%gS3xpM#Uxv*#Jy>}67gk8q z7insOklTFHkc(nII@9DuzMSQe4|jTomxwb(-wRT5r9r#zL1qYN!hSAhSR{3XcVT+i zj3!?}rM_)5LnI9qelTGl%EXFaAKH#d5$53y+4LA(4;!ZmD(i!ri|Scc0~x{c2?G3X z(Hpn$ZFWy^`$W;NX^I>ouC)(HfP(2%PY&cdx%dLUqe|c1bIk=biyb}LRCi*V(igUL zcnSrA)o89Zp6i#G##tS&=;^+oxydeYV+G^(!Ri?P0cqKwWYgzOiZr75K^}Jok^9K` zdg*0M{rVzKUHeoWk=h`4Hrc z)Y{J_cr8Hzhb!Kzv`I6Oj9OR8?he8|v}6}yYF3p)!eq* z)ZKtc60XQ<{^qr!6osTnXZha6ih7F8?Esg9DIUQQ<_>3hqPp1ZRSdel_KydAzYT~! z>&(d`-|UK}$cSf$OBS1QM25t{Y{jxAHcti*2|J{E>t}@#K5u5nOI;=BW@FR$?-GRc z?b%n||863;CRpo!nS{qdJ%{^3Uu)Oy!VY+PuXgiJmjMKDWt#=y-p_zII#l~89XaJT z_@>5zTJM_nI|m;}XlG`3<#Gu*hpNYQ2GO{Fg4D^yyo@uidrs>X&wmc@Nt~6a-ril? z8!a}{Pf}jWw;o=HaaT2vq?Y2JUkVTr!!|JvAo7!2F_1`7F*|BD;G`pnACyP?JV^di z%X(j4X5)(^(7Xj+^{bmczJ1=GB2WR)K(Co|J5RW%?(A9ZnNjVr^V!DcnOsi%%D3$J zpGR|^kp2z>0P!0E7`o&Lf7LkL1+&$VKk0>Z4Kn>ipXEGDwiUpv7My4n-Nn=WYET8O z)g4R)juO3s-T3m3%r2g{1@k6ID5?a!;cl?S?f`>TBE&}}TZM_m2%RfVdAbNeI16C| znR?J-1-y)K4Row?EZ%p~M91As_lg=a^>>s(dtfM5v-8byKggi%wjHA6_x@nt z4V+I#fZ1-)9{HU^eRN#YQQc#8-zoJtalLKVgh5ofPzwyib0UE|WbeIBi@>3ecNJ=@ z-0tT287Ta<@d_t21HlX=pl_kqYRkv=MMT9J-(^U$&055upYJxrzAEP$iX}%X6LB*3 z;L*YQWC495w%kg_`&UQ$8yHugC#syYg_T}O**2zNBuY6YlRS&t`}i|-Wm!0DU{91Z zJ{x7Aqm$sD1>( zKUXzVq06Jenqp@8058Q8FfL`1bvIH^<=s)}ytgaw3OGDpyVRKV7XrDk$;AEAPsRDe zxC9@HN=`yQ-Zbl0MU<>L9fRzs>VXj(+$9NKr)-)t_aVGn$;yUtl*d~duyNtYwB|Yt zVEmq)ijiONw@V~Obv$0iz-Znept$5oe#)2bKD7^a^yGFKk9Vt9TxB?~MzYBrn7Zqw zNxyAeAP5w4O2}mHt#z@FV>f!;iIwM8Kf4B4sF+;PH6~|y3Y>&yYY>obimO%h$a|>p z;dHgr#(IvDR6`no>GdaVP_p<+aI|vne0|HM zKa)wA#|+Ur*9ufy?QIk7;~6rQ-wH(NSw>N_q78%OOf#9AvVBb$5*h=_Zdt#`8y7e4j)wqri= z71@}oTLZ9T=Q;4hx@hyG&8%3iW?_OU-By|Dz$Yh+MRv6I#!Ccb66d@J(RZbmz_?l; zcP+QD-tNs0U9AO@!3xRh>>N8s%W4X>EuxK&Pdf>vlxrLvvvl0hq5@-Jptm@CkezNgdoUZCdLSXJxT2;0W6(l#^3$ys zbqZ%k_o64DUE-sFlXSr>>YF+GG5;gKu)u^ zhS|E722b0pieVC2`+&kW7 zSY7e_1NnDU?(4Gh8LsGC+?K=rZo=pe(kFRPG_79c`$HE{*kh>R$e0Z$*sYoI zB$59V`fY1At};$^NkKQ`b-|GO zgMHIBzo~)o4ayEjZg@Vm$Jbf7^%FIW;=Jy_hUN|49c)W{@zCm=Ide?RGGaDYi{D#SQ;raJvUWg%4UZ+a^ z;pW!+sf->BL}{^6{E=)Ar^>j}3|j-%4MLx*F5Eex3|E);+|9mQk@LJzw}N$UAM(a( zV$L6Nr_w5WR@Q{6tf#wg2rvJKF+WjDMJJ-BAGyEW6xjgI@7)Z4n@WLTp> z&omY*JNikpSP!TOOzs93 z6X~fHu~(}D(Z;Br^Cb3t_QLE?j=Jpf)49H-9{c~0(D1ljc=W0nK{W>IHsYKwq0l32 zM))0EMvQa}kqtA5QTFQSNC__kY0(0EP*QsvpkgMbQ)P^j-iL@Q3DY6#|SEJmHZSFgUX*YAa%5mVNfWsK|33sHR9{PP@nfdb-yg8oW%rESm_ zOx5n#{*47n)op3jh1s-A(kv$`*^NJHv@`(~j#lwXM>d_ikM{1u5a70aWjYsZuX^R^V$>RR`yx&UCHB8>03a+24*V0pQ3EaQ3{ zc|&X3n4wY{!D4?sg$M-`M#Lv6)7P=w=Ej`HU4APvzzSP#fB(YzusTW$3Lg~pUfZ0E zd=FD4FRR+HS0mXR!DClopYkE^>8m-v7w=`1^ctGd{xFmO{U4~ymYa`!1p4$FkM)!V ztxSa9aw7;ge)_ycI7tlSzc|To3^4OEJUQCqTrjN#(N=iH&!(oA7bnx+k==8w^nBK_ zbK^VQEuT}RrvD8*eV(5}M$`VXDo(GY`(dk7Q(bp*-96-Q|*S z6a@BXXbaEOeiY>ER#ba(&JrS30)3Z-NLEYlej@ZbJex~LnEjrAFUQ{t!=d%)cz=ym zB;}LuR+JB<=>rK+%X+b-o}=u`x>nQ1p2;{r2KZ` zt4mm@G5r}Zv?~GgiE*xX#ij!pc%(X#`Dz$Cl6CNZUeVW!H?kZ0xxnH7$Dh50u69Eb z!w{UojNoB|`M>{-{0FmD^Wo>%kH@@390`=d3`$=TWSs%MW^_X`lpL7}5C=L4_kFO% zZC$-OcL6XNDfiJIe(vw*hMy84#JtX+($Zaj1k<$h0RrF&Dp5#?Mk39H8hKiHNMYp3 zGjGSWZ)3raCx=VKCx zu#oc#XIx}-O{PulP|nnQ4#OsD8iCi(b35GDNEX95hJ`l*ko3MhN|))45!Lm;t+qI? z)Z0yh|6CQ!G{&Sk_S+vE!s8~hL^=>3aNFO&UYkMzus|Q&x#7nDgw&rX?qN>==-CIq z<8PI=^La+?A?)bTP#5A3(aY+!5mdnp_b)y_kJg?PLV6=-ONK`2I0WL$EmgeW(Fq;+PW)5sDg2i?Nbc1sDrTl=iU3F@K>Wl4)-6Sx#P`B1W#m`f$k$st3`s`o(U)&p1vy%!!hfg$>HA_iTU()X^c9#OVKZ?*r z*jc@}p9@fvHBJ>!AsGhw5kJD}8Q$Q+{{>>oa_E)pFdNPzE*1WaY*LrqWh9*ZukU;g z1)DUWIw$Y=6AoKURFEVuOYZRY4_etMt$dyG6sH|h?^i^lA1&kJ#FB_<8Xad>GP;PtEf)e01OZ?1cXD z_?S<88WEZT`*AH$?~YJ7Zgn$+0H#kF1PUQ-&{-Jnm!f%#6Z`_kaTT^^1quUu6ef@w z1u$hx2L-g65SK@(-ZrG3iSbMA;Qy=k^4|ptYeEVC-h8Q^tV2fcqCASWudy`cq^c6G@7NHnrA*zz@C}?zF{C!(W#vaaI&!{l?BC2 z3})(Q{{Q<0AiiUukn$@mY|YaF>=f||a9D1mlaGD!HwV43$dAIhjS#an&>uE&Npj&% z@Rb&(Uj<_gaMsHSm^4~e6|du>Li{xoRt9e)fkY&Knj-6Ip@W@z(*e4Jjaew5`SA;?+l;JEBeeDPBE>-n)yh&) zJO9yG{P&i@N5AW^{0OEh7C~?y2NQaN9e+i^y@v!{?M1&{SyK))t20~@s% zsbT>my$b$>!Tc-)N@imv>A4g2pdonmVKNK<_RI~=-+x%v|FfmM%djoRKX!LSvsquf zLRtjZ&o}?u2{*P>EJ0>lI=S5k+7lgc4W`=PLH8EN{RN=|QV|>?R*RfDgQ7=Gegplh zQVX2wqVECcRlRS*yGSz%HBl^{r1&fUS~r$W=vPIKcOK9PJWo3!VnWY=MP5Vn5Z+&4 z<+UNZGYWZp9wkbsM7s|?QC-l7VGY@8C_jPq{yaomW6*m9jU^&?#Q%HymWZgRgo72D zHmmhyJiq;FxGbyxifA;ez6FSMpztVy%y0X%hm)|Om$?rrme1hot9J)$4ED_A-{~Fp zIb)%&dRa=zF-F%cA!3P>D1W{9KcD5VljzNit0XZ6|80$S5NzTQ28vRn_AB=QZ^_Hb za(60s7MWtA>p$vA5QNapKMs|sY`^d*c`4pPrLO~yNEW8 z*@eLT#SsKWZ4NVtMgsH~H48(rSW%EO#~6)ziG-nlD>I!BphPH6f*}9(2sU0 zR2e`~YIpsWyFw%T<2Om3@E7@qytQCTmOtA6k#dI zc%w(qULh8C&?F||4AWmE zzwDiaNpGF%vj~N}BBTIaYTRvDeF}#Z(PgZ{xMLe%!w|v!QTWs@$fqXP|N6fW`k&7N zpSpLIO+BAgIpvkEj82|bag2cgeCXjCC+l40)Ymu)W*IjwdsASV?5~4P)(E03w14WEkoSxOFK8%a3j`0V4MHBhfZ}z&%g!Qudl4F11E$GhfiWvo zG)KS)R+mZlqlMfEBCW~M%a0F%I%?_f!<}~0cVs-cHeFXzyZHYFXfaUWyYZ&eD3h@c zhmg4ly67k}h_E}5u_mCQ6skavbPHh@x7hR+(b?uH(61Rz$3)n(PWhZ4Lu2K5HeChO zJucM$y+8i^T^JTBkdUPW_E#@KkCII>RK?g6@7jHrt^v2&dnCtU_UVs(r&nDuE7LjJ0;KpSbx1#}K=3I>KR-#%KJceEmf|8H<) ztoiL1gI0VGc5+edW8Bt8eAsC1HkUzR@(&=|s7$2Py9hDM_K-BPQ<#9=(vv+)C2=81mFU9cYl|(y_(wzmAw!8* zbk7z5V|(>n^}4&&CkR(|rwSMwbEvZe*xQf=_;`TLFRdM}}>@sEcOdyFy=YKMH}>w=lu*S6)1jv*-D*s6PhxYHW0kN}}q zOPEh88((4J=fOn`QfgCOkbu;Lo+@GV0qto@!fWNwnN=PiyYGoYCp~=d-?ZgFf9EZS z?+gtxYhYxmz#X;&<@i`Te#Wq!)zTgvxcWHqJ>76ooX@_NWyr_rB#Ze@bV+vG1X)1_ z9Ss1eM?F=;`vyTk@`<9C*Zdi?^Q(U>{y7vebCMe{G>%-|6a$x}y2(`^-L8ft=qthd zzH6(HvS@?qIOQq`Dy8%vZOyd;E_FYs?-Q%c<5xr$aY*@W8sQh@N`TO=jt|*F@gNaUv|aGU{%D?dsb1tv*9J>sj6OODc@{j# zVZ!!$0HHnzo@^CQKq=`cn+3Y=$H>UF2``rkBIDFxbkfqkNA@}Y{3gF7W)O7`LKvx0 zECOu^v|Pp*^o8ON2(D@Rns*1e0Wq8hS2uFrVLYT#Q#4??@kGy4Q;_bL!EhL!bzR27 zW$1UvhwLF^EDZ9kz!!Rs)6$LXHD~BHr~mxPCvQcZy<_0A8i(@0{JyxXMjWKcpnWN4 zH)_*@dw7n#T@w9@)FTkZtuM){_ammf0PgfBDu}@X7a=z^QpB+|-&V`L=dcYmN+m{< z^1%K-m<8F}pxyd}(~?D1ZXDKX4)@b@PAYZcUywwW!D$nodTenY$SKtfIoncVAOshT z$9Ou#x1bY{)0ls z6A^#}B`^!9oR!8cWQ`%x5x*DABzSxgjSzl8)?j7NeD7aQV8N&|75|gLDu~7O+EYaTiU9qzGu<@z)VyO5mY}gG zMItZ{Fk^;@k~a#Ye@~kZ61DDUeaemVk^ zI1W>uq+4g4l`Lclm7Y4N^PkPUyt)b%HoMMLi0!ep!oR&Wr59lz7DM4QT}ztoH@CoP z;yJYTGwn^xfBq!88%2>}D>iTGD#W)^aJU@;W^BGAZ0~BXFvbgtAQ1#MJ-)KN zg#85$y3~*H0(caIRr^i4XhX$sMg(50)**)$T8l|MBEBF98gxEsa{{@Dpf$IvbYF~)6C6qQDp->weWe8)_QrFpApUgdY+GF|dw&F(;tI8%&fj^Hf!uD$V|#iRpahH z&sl)X$m$s(n}LPpx3Xf$eXB*%edK^I427*&`o;_5HTE8)!>cI2qxY52aiA*TLYCVx z#7L7m5dyTF%$Am88Y>`?=ePF%d1nJxfxS9aH_qa#Zq*JKcH_b{lUkP21bKGl8*HDh z81`nFDCQMZ1VH#;_%(dp88~rH!o1ZZ94}ZZMoNyI*BZUgBZC#cL34&)nDQZsDkBa4 z#-2qW0maM>^G}5T+W(txy|A71z#MiToS3v>%TKC|T%(mq8@+GsUygbJDOCf)u`sZ< zD{RS+;%On?-%YOjtEREDqkzIlz!x+?HVZJ!Pt-eX4zt6ub+|+vpTH#&+7+t#uS-^h z4wGJZ9vY1dvFScI^gqa0D29vb!PdLxXf5-o)F>nKVQLmz47IUZP$jHCyD!>&pC#F# z)HoQ2#{+!Btyb!>TfRfYw)N)Ng6r0piszQ^KY~0+H0U;g5*mZ0m<-}3@T-D81{JsF zP1<2V3bmJSa(%X{KQvBFk_&j^J()aK2<}8Bu-gyAWV4=!L+Eu7#zC&F&XYsv8`Xta zdH%(z$4@>`zy)KT0LS@%*U(!HW6I;$5{&F$no8=vqmxI<@*2=VbN%5wOYw03Do~Zx z+Tqoi7!O6%yTHgD3Cb`T`<5Bj9<=f1LwH_)MxmG>P+(YYaQ|L4vT_BtH-Jh1$7Xy_ zktCw}b2$G`4M#SQyEz{yD+Z5kD5xVD`o~92sOTPLQO0g(0ccxNal3r)H&asMEKuR= zRvkmYfpEV7%iA^KvqQyz55w1fAWGGoCCH;WF$&;Ldpoza%IAg#EgshGO#cM0KoemA zbbt7Bq%aYoFInEG+lGOXj(Uik0o3zC?yJB|u@fW^-*w1}V|o80aCC*q8`9Tmy%9S3(8?x+GHE1NLAYmx{%JlXwK%2Lc@9h7%z!W6dT_8eM zFP^G*uhBKkcXP{Dzp3$b4x-p7BG>q%s@)@CoM#Z{D}(h^?HJ3OVGE6gS+#Sv zK*NAIY3sV87$;@<6EHsLE1(GS8Hk;f>C}R^CH~o2^Z(kV2-yXCK2z8jRUbfHgrv-B z(OS8(@CEES=OfM(G43PzD{RU{=&E4HckPl&Q=b82)aSC9?Lovwz{lEmCY8G#fhoy@ zJr>9xZfp9w!WE9tfN?_Z)w-@9V4YmlkkCUQIR%2(f84YsI^Yg}ejnV8M~y-S7o$cH zURO|fuwQ#&dw`jbrB84%#%RLg3&enFC?7-&Wk%!+KNNiuya3m~yvt4K3Np@-^T?8! z*D)&`%hfT)$S+nFc$@1so9~ zh$|40OAv5T5e!9mbsy1=>G$`XtTb;(H_ZXGN3*%%n|vx{J-sR9OjfL(Oy2oYdM^5@8rMz{lZ4MX7&<6*=6Pdrk#$44DGK zjEtOyEtuwgDH5?lhj6u&!C2E4V}Mf+9y}CFxh{IkUk)(ov{0bR7P#M;y6{kyo`*^cC`Ny zd4Sb>is~51A`XO9f9uGm++Fq z0G^_aWarr97JqqNOGrWg++M^C=)HynXA=;5$ME4(W7t8sxu0PcEu zUsJB;0)-EEOk~k>R&5^w60_z?bRgX4gR7|{2=T|ji_H)*`lz1bq`$Z^Q8hp?qU`+vQDg!B zUEJn?05AbowKs`O@}L!{T0yFc`2TC~T6mf~qcBA)vlw9n2FpadAtGfI9lFUXO6ArY zHz_D|D59gRI=1t& z#bk8;0_YED)AOG9oaa2xdA@Hw5gHu7dDl9(t@?6_Vckhf>%{Si$qv%&+GVX;i5c9> z6wtjEMYu;{0%02)7n{YpFRZA z@zM3<5HPqBL4V`=ArlD)A!3b&uWvLGm z+=(y5d9F~wBA{{hM@ndR3h#H`wIeU#t|UMdrM;(Gn`J3Fw3+|F+1>wv4ckL0qjf>O zAytoYR_PHTWF3AW5x`iZ^O*4xmJVrMb341w6!fiHETgDliE2MMhUFABWE2+-N|3ry z;(*+xq?2V;o*v-!#o~wV3ian#a3l95xdAujXWeN@D+LtwB-8O)3shMaf6dyBH4Oj{ zFSLge13Z*T(DDCaW2nzgfiqV#c#PT2X)Dq!p}rvm>d~%jkK)OOi;LkHdj%biCdX(f zR3f7Qz?Sm>b?>=5ZAvnfv2Z$h3~8I0fA`t^3OKBG@ImY*xv2)f*DC21t6=n!h7zgjL&2ce{emR8vP_!=6ES_M58<@E zj;WP^_d}hk0?^#pLKrD;?fUe-?ZOx!lNh04e}!!O!d2?aLLL`-V#3 zrIYHLAy{(LXPyrFeOKB(_|20PrR7F>^tIOUB%q7s*4b&9sELwqCDmDV@o^cJ_IV${ z0|myZE+qr;TM05SygCCN9uVD^wHsWrFzNZ;T4a(MKKSkCpq4R8n}@MZR?9a!e9Y8T zQ?e8bx!DwwXPW%-{qv8Z9+v#E(r-10pFWLj?k{ zl|=?|wtO(dU|KT){LTOH`?$`BdR7w6C)aHa1B2)je5N*z`-#MXlFoO@(KiE&cB{uDE)@x7M literal 0 HcmV?d00001 From e8069eed2a28f3b944ff238f0285730e0e1da17f Mon Sep 17 00:00:00 2001 From: John Macy <36553266+johncmacy@users.noreply.github.com> Date: Sun, 5 Dec 2021 07:42:19 -0600 Subject: [PATCH 104/158] Add instructions for migrating --- README.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/README.md b/README.md index 8556a3f..f783118 100644 --- a/README.md +++ b/README.md @@ -25,6 +25,10 @@ Add `django_dbq` to your installed apps 'django_dbq', ) +Run migrations + + manage.py migrate + ### Upgrading from 1.x to 2.x Note that version 2.x only supports Django 3.1 or newer. If you need support for Django 2.2, please stick with the latest 1.x release. From 61fb30e0e121a49cfca3c50e236e40828c242611 Mon Sep 17 00:00:00 2001 From: John Macy <36553266+johncmacy@users.noreply.github.com> Date: Sun, 5 Dec 2021 08:28:56 -0600 Subject: [PATCH 105/158] Test if `signal` has attribute `SIGQUIT` Add support for Windows users, since Windows doesn't support the SIGQUIT signal. --- django_dbq/management/commands/worker.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/django_dbq/management/commands/worker.py b/django_dbq/management/commands/worker.py index 92e72e4..aaefadf 100644 --- a/django_dbq/management/commands/worker.py +++ b/django_dbq/management/commands/worker.py @@ -84,7 +84,11 @@ def __init__(self, name, rate_limit_in_seconds): def init_signals(self): signal.signal(signal.SIGINT, self.shutdown) - signal.signal(signal.SIGQUIT, self.shutdown) + + # for Windows, which doesn't support the SIGQUIT signal + if hasattr(signal, "SIGQUIT"): + signal.signal(signal.SIGQUIT, self.shutdown) + signal.signal(signal.SIGTERM, self.shutdown) def shutdown(self, signum, frame): From b609b93b620b757ad9c7a7c7e8b88dbbbcf40a30 Mon Sep 17 00:00:00 2001 From: John Macy <36553266+johncmacy@users.noreply.github.com> Date: Mon, 6 Dec 2021 07:32:13 -0600 Subject: [PATCH 106/158] formatted w/black; added note about Windows support to README --- README.md | 4 ++++ django_dbq/management/commands/queue_depth.py | 3 ++- django_dbq/management/commands/worker.py | 4 ++-- django_dbq/migrations/0001_initial.py | 4 +++- django_dbq/migrations/0002_auto_20151016_1027.py | 5 ++++- django_dbq/migrations/0003_auto_20180713_1000.py | 3 ++- django_dbq/migrations/0004_auto_20210818_0247.py | 4 +++- django_dbq/tests.py | 9 ++++++++- 8 files changed, 28 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index 8556a3f..50afb79 100644 --- a/README.md +++ b/README.md @@ -239,6 +239,10 @@ jobs in the "NEW" or "READY" states will be returned. It may be necessary to supply a DATABASE_PORT environment variable. +## Windows support + +Windows is supported on a best-effort basis only, and is not covered by automated or manual testing. + ## Code of conduct For guidelines regarding the code of conduct when contributing to this repository please review [https://www.dabapps.com/open-source/code-of-conduct/](https://www.dabapps.com/open-source/code-of-conduct/) diff --git a/django_dbq/management/commands/queue_depth.py b/django_dbq/management/commands/queue_depth.py index 483ddc5..3419601 100644 --- a/django_dbq/management/commands/queue_depth.py +++ b/django_dbq/management/commands/queue_depth.py @@ -16,7 +16,8 @@ def handle(self, *args, **options): queue_depths_string = " ".join( [ "{queue_name}={queue_depth}".format( - queue_name=queue_name, queue_depth=queue_depths.get(queue_name, 0), + queue_name=queue_name, + queue_depth=queue_depths.get(queue_name, 0), ) for queue_name in queue_names ] diff --git a/django_dbq/management/commands/worker.py b/django_dbq/management/commands/worker.py index aaefadf..9d72871 100644 --- a/django_dbq/management/commands/worker.py +++ b/django_dbq/management/commands/worker.py @@ -84,11 +84,11 @@ def __init__(self, name, rate_limit_in_seconds): def init_signals(self): signal.signal(signal.SIGINT, self.shutdown) - + # for Windows, which doesn't support the SIGQUIT signal if hasattr(signal, "SIGQUIT"): signal.signal(signal.SIGQUIT, self.shutdown) - + signal.signal(signal.SIGTERM, self.shutdown) def shutdown(self, signum, frame): diff --git a/django_dbq/migrations/0001_initial.py b/django_dbq/migrations/0001_initial.py index d5114d3..4d63fb3 100644 --- a/django_dbq/migrations/0001_initial.py +++ b/django_dbq/migrations/0001_initial.py @@ -49,6 +49,8 @@ class Migration(migrations.Migration): models.CharField(db_index=True, max_length=20, default="default"), ), ], - options={"ordering": ["-created"],}, + options={ + "ordering": ["-created"], + }, ), ] diff --git a/django_dbq/migrations/0002_auto_20151016_1027.py b/django_dbq/migrations/0002_auto_20151016_1027.py index d0a72c4..9769061 100644 --- a/django_dbq/migrations/0002_auto_20151016_1027.py +++ b/django_dbq/migrations/0002_auto_20151016_1027.py @@ -11,5 +11,8 @@ class Migration(migrations.Migration): ] operations = [ - migrations.AlterModelOptions(name="job", options={"ordering": ["created"]},), + migrations.AlterModelOptions( + name="job", + options={"ordering": ["created"]}, + ), ] diff --git a/django_dbq/migrations/0003_auto_20180713_1000.py b/django_dbq/migrations/0003_auto_20180713_1000.py index 4d959f3..78a09ed 100644 --- a/django_dbq/migrations/0003_auto_20180713_1000.py +++ b/django_dbq/migrations/0003_auto_20180713_1000.py @@ -13,7 +13,8 @@ class Migration(migrations.Migration): operations = [ migrations.AlterModelOptions( - name="job", options={"ordering": ["-priority", "created"]}, + name="job", + options={"ordering": ["-priority", "created"]}, ), migrations.AddField( model_name="job", diff --git a/django_dbq/migrations/0004_auto_20210818_0247.py b/django_dbq/migrations/0004_auto_20210818_0247.py index a1ff5ff..b62ab02 100644 --- a/django_dbq/migrations/0004_auto_20210818_0247.py +++ b/django_dbq/migrations/0004_auto_20210818_0247.py @@ -27,6 +27,8 @@ class Migration(migrations.Migration): ), ), migrations.AlterField( - model_name="job", name="workspace", field=models.JSONField(null=True), + model_name="job", + name="workspace", + field=models.JSONField(null=True), ), ] diff --git a/django_dbq/tests.py b/django_dbq/tests.py index 2c8e0e9..c1f828d 100644 --- a/django_dbq/tests.py +++ b/django_dbq/tests.py @@ -103,7 +103,14 @@ def test_queue_depth_multiple_queues(self): ) stdout = StringIO() - call_command("queue_depth", queue_name=("default", "testqueue",), stdout=stdout) + call_command( + "queue_depth", + queue_name=( + "default", + "testqueue", + ), + stdout=stdout, + ) output = stdout.getvalue() self.assertEqual(output.strip(), "event=queue_depths default=2 testqueue=2") From da7fe87f08e775f8fcc3a86e29582a98a66ac6a7 Mon Sep 17 00:00:00 2001 From: Jamie Matthews Date: Mon, 6 Dec 2021 13:48:28 +0000 Subject: [PATCH 107/158] Fix black formatting --- django_dbq/management/commands/queue_depth.py | 3 +-- django_dbq/migrations/0001_initial.py | 4 +--- django_dbq/migrations/0002_auto_20151016_1027.py | 5 +---- django_dbq/migrations/0003_auto_20180713_1000.py | 3 +-- django_dbq/migrations/0004_auto_20210818_0247.py | 4 +--- django_dbq/tests.py | 7 +------ 6 files changed, 6 insertions(+), 20 deletions(-) diff --git a/django_dbq/management/commands/queue_depth.py b/django_dbq/management/commands/queue_depth.py index 3419601..483ddc5 100644 --- a/django_dbq/management/commands/queue_depth.py +++ b/django_dbq/management/commands/queue_depth.py @@ -16,8 +16,7 @@ def handle(self, *args, **options): queue_depths_string = " ".join( [ "{queue_name}={queue_depth}".format( - queue_name=queue_name, - queue_depth=queue_depths.get(queue_name, 0), + queue_name=queue_name, queue_depth=queue_depths.get(queue_name, 0), ) for queue_name in queue_names ] diff --git a/django_dbq/migrations/0001_initial.py b/django_dbq/migrations/0001_initial.py index 4d63fb3..d5114d3 100644 --- a/django_dbq/migrations/0001_initial.py +++ b/django_dbq/migrations/0001_initial.py @@ -49,8 +49,6 @@ class Migration(migrations.Migration): models.CharField(db_index=True, max_length=20, default="default"), ), ], - options={ - "ordering": ["-created"], - }, + options={"ordering": ["-created"],}, ), ] diff --git a/django_dbq/migrations/0002_auto_20151016_1027.py b/django_dbq/migrations/0002_auto_20151016_1027.py index 9769061..d0a72c4 100644 --- a/django_dbq/migrations/0002_auto_20151016_1027.py +++ b/django_dbq/migrations/0002_auto_20151016_1027.py @@ -11,8 +11,5 @@ class Migration(migrations.Migration): ] operations = [ - migrations.AlterModelOptions( - name="job", - options={"ordering": ["created"]}, - ), + migrations.AlterModelOptions(name="job", options={"ordering": ["created"]},), ] diff --git a/django_dbq/migrations/0003_auto_20180713_1000.py b/django_dbq/migrations/0003_auto_20180713_1000.py index 78a09ed..4d959f3 100644 --- a/django_dbq/migrations/0003_auto_20180713_1000.py +++ b/django_dbq/migrations/0003_auto_20180713_1000.py @@ -13,8 +13,7 @@ class Migration(migrations.Migration): operations = [ migrations.AlterModelOptions( - name="job", - options={"ordering": ["-priority", "created"]}, + name="job", options={"ordering": ["-priority", "created"]}, ), migrations.AddField( model_name="job", diff --git a/django_dbq/migrations/0004_auto_20210818_0247.py b/django_dbq/migrations/0004_auto_20210818_0247.py index b62ab02..a1ff5ff 100644 --- a/django_dbq/migrations/0004_auto_20210818_0247.py +++ b/django_dbq/migrations/0004_auto_20210818_0247.py @@ -27,8 +27,6 @@ class Migration(migrations.Migration): ), ), migrations.AlterField( - model_name="job", - name="workspace", - field=models.JSONField(null=True), + model_name="job", name="workspace", field=models.JSONField(null=True), ), ] diff --git a/django_dbq/tests.py b/django_dbq/tests.py index c1f828d..35c3357 100644 --- a/django_dbq/tests.py +++ b/django_dbq/tests.py @@ -104,12 +104,7 @@ def test_queue_depth_multiple_queues(self): stdout = StringIO() call_command( - "queue_depth", - queue_name=( - "default", - "testqueue", - ), - stdout=stdout, + "queue_depth", queue_name=("default", "testqueue",), stdout=stdout, ) output = stdout.getvalue() self.assertEqual(output.strip(), "event=queue_depths default=2 testqueue=2") From 485e5b5d41334f214a92b7eabf43fdd97bd45ca7 Mon Sep 17 00:00:00 2001 From: John Macy <36553266+johncmacy@users.noreply.github.com> Date: Mon, 6 Dec 2021 07:50:50 -0600 Subject: [PATCH 108/158] Clarify workspace docs Added verbiage to clarify that workspaces can be used for passing args/kwargs to task functions, and that the workspace field can be queried as a JSONField. --- README.md | 27 ++++++++++++++++++++++++--- 1 file changed, 24 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index f783118..f746fb4 100644 --- a/README.md +++ b/README.md @@ -160,7 +160,22 @@ Jobs are processed to completion by *tasks*. These are simply Python functions, ### Workspace -The *workspace* is an area that tasks within a single job can use to communicate with each other. It is implemented as a Python dictionary, available on the `job` instance passed to tasks as `job.workspace`. The initial workspace of a job can be empty, or can contain some parameters that the tasks require (for example, API access tokens, account IDs etc). A single task can edit the workspace, and the modified workspace will be passed on to the next task in the sequence. For example: +The *workspace* is an area that can be used 1) to provide additional arg/kwargs to task functions, and 2) to categorize jobs with additional metadata. It is implemented as a Python dictionary, available on the `job` instance passed to tasks as `job.workspace`. The initial workspace of a job can be empty, or can contain some parameters that the tasks require (for example, API access tokens, account IDs etc). + +When creating a Job, the workspace is passed as a keyword argument: + +```python +Job.objects.create(name='my_job', workspace={'key': value}) +``` + +Then, the task function can access the workspace to get the data it needs to perform its task: + +```python +def my_task(job): + cats_import = CatsImport.objects.get(pk=job.workspace['cats_import_id'] +``` + +Tasks within a single job can use the workspace to communicate with each other. A single task can edit the workspace, and the modified workspace will be passed on to the next task in the sequence. For example: def my_first_task(job): job.workspace['message'] = 'Hello, task 2!' @@ -168,10 +183,16 @@ The *workspace* is an area that tasks within a single job can use to communicate def my_second_task(job): logger.info("Task 1 says: %s" % job.workspace['message']) -When creating a Job, the workspace is passed as a keyword argument: +The workspace can be queried like any [JSONField](https://docs.djangoproject.com/en/3.2/topics/db/queries/#querying-jsonfield). For instance, if you wanted to display a list of jobs that a certain user had initiated, add `user_id` to the workspace when creating the job: ```python -Job.objects.create(name='my_job', workspace={'key': value}) +Job.objects.create(name='foo', workspace={'user_id': request.user.id}) +``` + +Then filter the query with it in the view that renders the list: + +```python +user_jobs = Job.objects.filter(workspace__user_id=request.user.id) ``` ### Worker process From da50d39502b5bae6acf68fabdcc77a08ee08b8c7 Mon Sep 17 00:00:00 2001 From: Jamie Matthews Date: Mon, 6 Dec 2021 14:42:32 +0000 Subject: [PATCH 109/158] Blacken docs and reword slightly --- README.md | 55 ++++++++++++++++++++++++------------------- test-requirements.txt | 2 +- 2 files changed, 32 insertions(+), 25 deletions(-) diff --git a/README.md b/README.md index b9b32a6..eccb0a1 100644 --- a/README.md +++ b/README.md @@ -40,6 +40,7 @@ In e.g. project.common.jobs: ```python import time + def my_task(job): logger.info("Working hard...") time.sleep(10) @@ -52,8 +53,8 @@ In project.settings: ```python JOBS = { - 'my_job': { - 'tasks': ['project.common.jobs.my_task'] + "my_job": { + "tasks": ["project.common.jobs.my_task"], }, } ``` @@ -69,16 +70,16 @@ A failure hook receives the failed `Job` instance along with the unhandled excep ```python def my_task_failure_hook(job, e): - # delete some temporary files on the filesystem + ... # clean up after failed job ``` To ensure this hook gets run, simply add a `failure_hook` key to your job config like so: ```python JOBS = { - 'my_job': { - 'tasks': ['project.common.jobs.my_task'], - 'failure_hook': 'project.common.jobs.my_task_failure_hook' + "my_job": { + "tasks": ["project.common.jobs.my_task"], + "failure_hook": "project.common.jobs.my_task_failure_hook", }, } ``` @@ -91,16 +92,16 @@ A creation hook receives your `Job` instance as its only argument. Here's an exa ```python def my_task_creation_hook(job): - # configure something before running your job + ... # configure something before running your job ``` To ensure this hook gets run, simply add a `creation_hook` key to your job config like so: ```python JOBS = { - 'my_job': { - 'tasks': ['project.common.jobs.my_task'], - 'creation_hook': 'project.common.jobs.my_task_creation_hook' + "my_job": { + "tasks": ["project.common.jobs.my_task"], + "creation_hook": "project.common.jobs.my_task_creation_hook", }, } ``` @@ -116,7 +117,7 @@ In another terminal: Using the name you configured for your job in your settings, create an instance of Job. ```python -Job.objects.create(name='my_job') +Job.objects.create(name="my_job") ``` ### Prioritising jobs @@ -125,10 +126,11 @@ important emails to users. However, once an hour, you may need to run a _really_ of emails to be dispatched before it can begin. In order to make sure that an important job is run before others, you can set the `priority` field to an integer higher than `0` (the default). For example: + ```python -Job.objects.create(name='normal_job') -Job.objects.create(name='important_job', priority=1) -Job.objects.create(name='critical_job', priority=2) +Job.objects.create(name="normal_job") +Job.objects.create(name="important_job", priority=1) +Job.objects.create(name="critical_job", priority=2) ``` Jobs will be ordered by their `priority` (highest to lowest) and then the time which they were created (oldest to newest) and processed in that order. @@ -137,7 +139,10 @@ Jobs will be ordered by their `priority` (highest to lowest) and then the time w If you'd like to create a job but have it run at some time in the future, you can use the `run_after` field on the Job model: ```python -Job.objects.create(name='scheduled_job', run_after=timezone.now() + timedelta(minutes=10)) +Job.objects.create( + name="scheduled_job", + run_after=(timezone.now() + timedelta(minutes=10)), +) ``` Of course, the scheduled job will only be run if your `python manage.py worker` process is running at the time when the job is scheduled to run. Otherwise, it will run the next time you start your worker process after that time has passed. @@ -154,25 +159,27 @@ The top-level abstraction of a standalone piece of work. Jobs are stored in the Jobs are processed to completion by *tasks*. These are simply Python functions, which must take a single argument - the `Job` instance being processed. A single job will often require processing by more than one task to be completed fully. Creating the task functions is the responsibility of the developer. For example: - def my_task(job): - logger.info("Doing some hard work") - do_some_hard_work() +```python +def my_task(job): + logger.info("Doing some hard work") + do_some_hard_work() +``` ### Workspace -The *workspace* is an area that can be used 1) to provide additional arg/kwargs to task functions, and 2) to categorize jobs with additional metadata. It is implemented as a Python dictionary, available on the `job` instance passed to tasks as `job.workspace`. The initial workspace of a job can be empty, or can contain some parameters that the tasks require (for example, API access tokens, account IDs etc). +The *workspace* is an area that can be used 1) to provide additional arguments to task functions, and 2) to categorize jobs with additional metadata. It is implemented as a Python dictionary, available on the `job` instance passed to tasks as `job.workspace`. The initial workspace of a job can be empty, or can contain some parameters that the tasks require (for example, API access tokens, account IDs etc). When creating a Job, the workspace is passed as a keyword argument: ```python -Job.objects.create(name='my_job', workspace={'key': value}) +Job.objects.create(name="my_job", workspace={"key": value}) ``` Then, the task function can access the workspace to get the data it needs to perform its task: ```python def my_task(job): - cats_import = CatsImport.objects.get(pk=job.workspace['cats_import_id'] + cats_import = CatsImport.objects.get(pk=job.workspace["cats_import_id"]) ``` Tasks within a single job can use the workspace to communicate with each other. A single task can edit the workspace, and the modified workspace will be passed on to the next task in the sequence. For example: @@ -186,7 +193,7 @@ Tasks within a single job can use the workspace to communicate with each other. The workspace can be queried like any [JSONField](https://docs.djangoproject.com/en/3.2/topics/db/queries/#querying-jsonfield). For instance, if you wanted to display a list of jobs that a certain user had initiated, add `user_id` to the workspace when creating the job: ```python -Job.objects.create(name='foo', workspace={'user_id': request.user.id}) +Job.objects.create(name="foo", workspace={"user_id": request.user.id}) ``` Then filter the query with it in the view that renders the list: @@ -228,8 +235,8 @@ from django_dbq.models import Job ... -Job.objects.create(name='do_work', workspace={}) -Job.objects.create(name='do_other_work', queue_name='other_queue', workspace={}) +Job.objects.create(name="do_work", workspace={}) +Job.objects.create(name="do_other_work", queue_name="other_queue", workspace={}) queue_depths = Job.get_queue_depths() print(queue_depths) # {"default": 1, "other_queue": 1} diff --git a/test-requirements.txt b/test-requirements.txt index 249c8b3..a1452df 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -3,4 +3,4 @@ freezegun==0.3.12 mock==3.0.5 dj-database-url==0.5.0 psycopg2==2.8.4 -black==19.10b0 +black==21.12b0 From b1ee1eef49cdaed9025d60b1faadee57eeb3e789 Mon Sep 17 00:00:00 2001 From: Jamie Matthews Date: Mon, 6 Dec 2021 14:42:50 +0000 Subject: [PATCH 110/158] Bump version --- django_dbq/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/django_dbq/__init__.py b/django_dbq/__init__.py index 9aa3f90..58039f5 100644 --- a/django_dbq/__init__.py +++ b/django_dbq/__init__.py @@ -1 +1 @@ -__version__ = "2.1.0" +__version__ = "2.1.1" From f8df0f0d7fc5d0b85ef9133c85c4f90ebf4c6108 Mon Sep 17 00:00:00 2001 From: Jamie Matthews Date: Mon, 6 Dec 2021 14:44:02 +0000 Subject: [PATCH 111/158] Blacken INSTALLED_APPS docs --- README.md | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index eccb0a1..2bad113 100644 --- a/README.md +++ b/README.md @@ -20,10 +20,12 @@ Install from PIP Add `django_dbq` to your installed apps - INSTALLED_APPS = ( - ... - 'django_dbq', - ) +```python +INSTALLED_APPS = [ + ..., + "django_dbq", +] +``` Run migrations From cfbcb353feed57b4f3b0c18089f39c100f993bd5 Mon Sep 17 00:00:00 2001 From: Jamie Matthews Date: Mon, 6 Dec 2021 14:45:13 +0000 Subject: [PATCH 112/158] Reformat to match new Black version --- django_dbq/management/commands/queue_depth.py | 3 ++- django_dbq/migrations/0001_initial.py | 4 +++- django_dbq/migrations/0002_auto_20151016_1027.py | 5 ++++- django_dbq/migrations/0003_auto_20180713_1000.py | 3 ++- django_dbq/migrations/0004_auto_20210818_0247.py | 4 +++- django_dbq/tests.py | 7 ++++++- 6 files changed, 20 insertions(+), 6 deletions(-) diff --git a/django_dbq/management/commands/queue_depth.py b/django_dbq/management/commands/queue_depth.py index 483ddc5..3419601 100644 --- a/django_dbq/management/commands/queue_depth.py +++ b/django_dbq/management/commands/queue_depth.py @@ -16,7 +16,8 @@ def handle(self, *args, **options): queue_depths_string = " ".join( [ "{queue_name}={queue_depth}".format( - queue_name=queue_name, queue_depth=queue_depths.get(queue_name, 0), + queue_name=queue_name, + queue_depth=queue_depths.get(queue_name, 0), ) for queue_name in queue_names ] diff --git a/django_dbq/migrations/0001_initial.py b/django_dbq/migrations/0001_initial.py index d5114d3..4d63fb3 100644 --- a/django_dbq/migrations/0001_initial.py +++ b/django_dbq/migrations/0001_initial.py @@ -49,6 +49,8 @@ class Migration(migrations.Migration): models.CharField(db_index=True, max_length=20, default="default"), ), ], - options={"ordering": ["-created"],}, + options={ + "ordering": ["-created"], + }, ), ] diff --git a/django_dbq/migrations/0002_auto_20151016_1027.py b/django_dbq/migrations/0002_auto_20151016_1027.py index d0a72c4..9769061 100644 --- a/django_dbq/migrations/0002_auto_20151016_1027.py +++ b/django_dbq/migrations/0002_auto_20151016_1027.py @@ -11,5 +11,8 @@ class Migration(migrations.Migration): ] operations = [ - migrations.AlterModelOptions(name="job", options={"ordering": ["created"]},), + migrations.AlterModelOptions( + name="job", + options={"ordering": ["created"]}, + ), ] diff --git a/django_dbq/migrations/0003_auto_20180713_1000.py b/django_dbq/migrations/0003_auto_20180713_1000.py index 4d959f3..78a09ed 100644 --- a/django_dbq/migrations/0003_auto_20180713_1000.py +++ b/django_dbq/migrations/0003_auto_20180713_1000.py @@ -13,7 +13,8 @@ class Migration(migrations.Migration): operations = [ migrations.AlterModelOptions( - name="job", options={"ordering": ["-priority", "created"]}, + name="job", + options={"ordering": ["-priority", "created"]}, ), migrations.AddField( model_name="job", diff --git a/django_dbq/migrations/0004_auto_20210818_0247.py b/django_dbq/migrations/0004_auto_20210818_0247.py index a1ff5ff..b62ab02 100644 --- a/django_dbq/migrations/0004_auto_20210818_0247.py +++ b/django_dbq/migrations/0004_auto_20210818_0247.py @@ -27,6 +27,8 @@ class Migration(migrations.Migration): ), ), migrations.AlterField( - model_name="job", name="workspace", field=models.JSONField(null=True), + model_name="job", + name="workspace", + field=models.JSONField(null=True), ), ] diff --git a/django_dbq/tests.py b/django_dbq/tests.py index 35c3357..c1f828d 100644 --- a/django_dbq/tests.py +++ b/django_dbq/tests.py @@ -104,7 +104,12 @@ def test_queue_depth_multiple_queues(self): stdout = StringIO() call_command( - "queue_depth", queue_name=("default", "testqueue",), stdout=stdout, + "queue_depth", + queue_name=( + "default", + "testqueue", + ), + stdout=stdout, ) output = stdout.getvalue() self.assertEqual(output.strip(), "event=queue_depths default=2 testqueue=2") From be3c935549dd7cfdedd740b932823e08b826f5d3 Mon Sep 17 00:00:00 2001 From: Jamie Matthews Date: Tue, 7 Dec 2021 08:38:25 +0000 Subject: [PATCH 113/158] Remove .python-version --- .python-version | 1 - 1 file changed, 1 deletion(-) delete mode 100644 .python-version diff --git a/.python-version b/.python-version deleted file mode 100644 index 0833a98..0000000 --- a/.python-version +++ /dev/null @@ -1 +0,0 @@ -3.7.4 From 2d1fb9478662d953ba2fbd42db652a9b801847a1 Mon Sep 17 00:00:00 2001 From: Jamie Matthews Date: Tue, 7 Dec 2021 21:00:17 +0000 Subject: [PATCH 114/158] Add Django 4.0 and Python 3.10 to test matrix, remove unsupported versions --- .github/workflows/ci.yml | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 7be073c..baf944d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -9,8 +9,15 @@ jobs: strategy: matrix: - python: [3.6, 3.7, 3.8, 3.9] - django: [3.1, 3.2] + python: [3.6, 3.7, 3.8, 3.9, "3.10"] + django: [2.2, 3.2, 4.0] + exclude: + - python: 3.6 + django: 4.0 + - python: 3.7 + django: 4.0 + - python: "3.10" + django: 2.2 database_url: - postgres://runner:password@localhost/project - mysql://root:root@127.0.0.1/project From 0f24d1c4682b591c24be3e867696c3a8f2fdb122 Mon Sep 17 00:00:00 2001 From: Jamie Matthews Date: Tue, 7 Dec 2021 21:01:08 +0000 Subject: [PATCH 115/158] Remove Django 2.2 --- .github/workflows/ci.yml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index baf944d..3bf9fd5 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -10,14 +10,12 @@ jobs: strategy: matrix: python: [3.6, 3.7, 3.8, 3.9, "3.10"] - django: [2.2, 3.2, 4.0] + django: [3.2, 4.0] exclude: - python: 3.6 django: 4.0 - python: 3.7 django: 4.0 - - python: "3.10" - django: 2.2 database_url: - postgres://runner:password@localhost/project - mysql://root:root@127.0.0.1/project From d1628a13a4102e8fd0f2a223d9b7884f485b43ad Mon Sep 17 00:00:00 2001 From: Jamie Matthews Date: Tue, 7 Dec 2021 21:03:58 +0000 Subject: [PATCH 116/158] Correct support versions in README --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 2bad113..11516d0 100644 --- a/README.md +++ b/README.md @@ -7,8 +7,8 @@ Simple database-backed job queue. Jobs are defined in your settings, and are pro Asynchronous tasks are run via a *job queue*. This system is designed to support multi-step job workflows. Supported and tested against: -- Django 3.1 and 3.2 -- Python 3.6, 3.7, 3.8 and 3.9 +- Django 3.2 and 4.0 +- Python 3.6, 3.7, 3.8, 3.9 and 3.10 ## Getting Started From 1d26ae9e85163f862cad16505e9a963d3e5a8827 Mon Sep 17 00:00:00 2001 From: Jamie Matthews Date: Wed, 8 Dec 2021 08:23:40 +0000 Subject: [PATCH 117/158] Quote all versions in ci.yml --- .github/workflows/ci.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 3bf9fd5..2d73bf9 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -9,13 +9,13 @@ jobs: strategy: matrix: - python: [3.6, 3.7, 3.8, 3.9, "3.10"] - django: [3.2, 4.0] + python: ["3.6", "3.7", "3.8", "3.9", "3.10"] + django: ["3.2", "4.0"] exclude: - - python: 3.6 - django: 4.0 - - python: 3.7 - django: 4.0 + - python: "3.6" + django: "4.0" + - python: "3.7" + django: "4.0" database_url: - postgres://runner:password@localhost/project - mysql://root:root@127.0.0.1/project From 4fe3e7ee15ad42508c14eae3f9bb847d52661faf Mon Sep 17 00:00:00 2001 From: Jamie Matthews Date: Thu, 6 Jan 2022 21:15:07 +0000 Subject: [PATCH 118/158] Version 3.0.0 --- django_dbq/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/django_dbq/__init__.py b/django_dbq/__init__.py index 58039f5..528787c 100644 --- a/django_dbq/__init__.py +++ b/django_dbq/__init__.py @@ -1 +1 @@ -__version__ = "2.1.1" +__version__ = "3.0.0" From 2a8fab73b0f64c1582a699a9e2dbd8e48eef2cbe Mon Sep 17 00:00:00 2001 From: Nik Kantar Date: Thu, 3 Feb 2022 19:58:07 -0800 Subject: [PATCH 119/158] Clean up code block formatting in README --- README.md | 34 +++++++++++++++++++++++----------- 1 file changed, 23 insertions(+), 11 deletions(-) diff --git a/README.md b/README.md index 03f099f..1baaad3 100644 --- a/README.md +++ b/README.md @@ -16,7 +16,9 @@ Supported and tested against: Install from PIP - pip install django-db-queue +``` +pip install django-db-queue +``` Add `django_dbq` to your installed apps @@ -29,7 +31,9 @@ INSTALLED_APPS = [ Run migrations - manage.py migrate +``` +manage.py migrate +``` ### Upgrading from 1.x to 2.x @@ -112,7 +116,9 @@ JOBS = { In another terminal: -`python manage.py worker` +``` +python manage.py worker +``` ### Create a job @@ -186,11 +192,13 @@ def my_task(job): Tasks within a single job can use the workspace to communicate with each other. A single task can edit the workspace, and the modified workspace will be passed on to the next task in the sequence. For example: - def my_first_task(job): - job.workspace['message'] = 'Hello, task 2!' +```python +def my_first_task(job): + job.workspace['message'] = 'Hello, task 2!' - def my_second_task(job): - logger.info("Task 1 says: %s" % job.workspace['message']) +def my_second_task(job): + logger.info("Task 1 says: %s" % job.workspace['message']) +``` The workspace can be queried like any [JSONField](https://docs.djangoproject.com/en/3.2/topics/db/queries/#querying-jsonfield). For instance, if you wanted to display a list of jobs that a certain user had initiated, add `user_id` to the workspace when creating the job: @@ -212,9 +220,11 @@ A *worker process* is a long-running process, implemented as a Django management Jobs are configured in the Django `settings.py` file. The `JOBS` setting is a dictionary mapping a *job name* (eg `import_cats`) to a *list* of one or more task function paths. For example: - JOBS = { - 'import_cats': ['apps.cat_importer.import_cats.step_one', 'apps.cat_importer.import_cats.step_two'], - } +```python +JOBS = { + 'import_cats': ['apps.cat_importer.import_cats.step_one', 'apps.cat_importer.import_cats.step_two'], +} +``` ### Job states @@ -263,7 +273,9 @@ to ensure the jobs table remains at a reasonable size. ##### manage.py worker To start a worker: - manage.py worker [queue_name] [--rate_limit] +``` +manage.py worker [queue_name] [--rate_limit] +``` - `queue_name` is optional, and will default to `default` - The `--rate_limit` flag is optional, and will default to `1`. It is the minimum number of seconds that must have elapsed before a subsequent job can be run. From f623a664092d68d4bc5450d057473a7809fbc6d7 Mon Sep 17 00:00:00 2001 From: collinr3 Date: Tue, 16 Aug 2022 14:53:17 +0100 Subject: [PATCH 120/158] Fixes #53 Creating test database for alias 'default'... System check identified no issues (0 silenced). ......................... Ran 25 tests in 0.123s OK --- django_dbq/management/commands/worker.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/django_dbq/management/commands/worker.py b/django_dbq/management/commands/worker.py index 5f8d7b2..9215aad 100644 --- a/django_dbq/management/commands/worker.py +++ b/django_dbq/management/commands/worker.py @@ -144,7 +144,7 @@ def handle(self, *args, **options): rate_limit_in_seconds = options["rate_limit"] self.stdout.write( - 'Starting job worker for queue "%s" with rate limit %s/s' + 'Starting job worker for queue "%s" with rate limit of one job per %s second(s)' % (queue_name, rate_limit_in_seconds) ) From ce51bfbb29639442c3c7c687d5fafb8976b1ecde Mon Sep 17 00:00:00 2001 From: Jamie Matthews Date: Wed, 17 Aug 2022 13:01:34 +0100 Subject: [PATCH 121/158] Bump black version to fix issue with unpinned sub-dependency --- test-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test-requirements.txt b/test-requirements.txt index a1452df..a40e03b 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -3,4 +3,4 @@ freezegun==0.3.12 mock==3.0.5 dj-database-url==0.5.0 psycopg2==2.8.4 -black==21.12b0 +black==22.6.0 From 831fa58761e89a8ca05c158520fa7cef69f81c29 Mon Sep 17 00:00:00 2001 From: Jamie Matthews Date: Sat, 21 Jan 2023 19:50:19 +0000 Subject: [PATCH 122/158] Add Django 4.1 and Python 3.11 to test matrix, remove Python 3.6 --- .github/workflows/ci.yml | 8 ++++++-- README.md | 4 ++-- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 2d73bf9..49931a3 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -9,13 +9,17 @@ jobs: strategy: matrix: - python: ["3.6", "3.7", "3.8", "3.9", "3.10"] - django: ["3.2", "4.0"] + python: ["3.7", "3.8", "3.9", "3.10", "3.11"] + django: ["3.2", "4.0", "4.1"] exclude: - python: "3.6" django: "4.0" - python: "3.7" django: "4.0" + - python: "3.6" + django: "4.1" + - python: "3.7" + django: "4.1" database_url: - postgres://runner:password@localhost/project - mysql://root:root@127.0.0.1/project diff --git a/README.md b/README.md index 1baaad3..f8e2831 100644 --- a/README.md +++ b/README.md @@ -7,8 +7,8 @@ Simple database-backed job queue. Jobs are defined in your settings, and are pro Asynchronous tasks are run via a *job queue*. This system is designed to support multi-step job workflows. Supported and tested against: -- Django 3.2 and 4.0 -- Python 3.6, 3.7, 3.8, 3.9 and 3.10 +- Django 3.2, 4.0, 4.1 +- Python 3.7, 3.8, 3.9, 3.10, 3.11 ## Getting Started From 2d4028dec7558934a3833d42279dc22ae93f441d Mon Sep 17 00:00:00 2001 From: Jamie Matthews Date: Sat, 21 Jan 2023 20:47:04 +0000 Subject: [PATCH 123/158] Upgrade pip in ci --- .github/workflows/ci.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 49931a3..cd56dc4 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -46,6 +46,8 @@ jobs: uses: actions/setup-python@v2 with: python-version: ${{ matrix.python }} + - name: Upgraded pip + run: pip install --upgrade pip - name: Install dependencies run: pip install -r test-requirements.txt - name: Install Django From b5799ac3490151a4beeeed660a14cf4de5c49f0c Mon Sep 17 00:00:00 2001 From: Jamie Matthews Date: Sat, 21 Jan 2023 20:50:52 +0000 Subject: [PATCH 124/158] Install libpython3.x-dev --- .github/workflows/ci.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index cd56dc4..161364f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -42,6 +42,8 @@ jobs: - name: Start MySQL run: sudo systemctl start mysql.service - uses: actions/checkout@v2 + - name: Install system Python build deps for psycopg2 + run: sudo apt-get install libpython3.x-dev - name: Set up Python ${{ matrix.python }} uses: actions/setup-python@v2 with: From 9374a09b8c96ebb85734745717833616021f0509 Mon Sep 17 00:00:00 2001 From: Jamie Matthews Date: Sat, 21 Jan 2023 20:52:26 +0000 Subject: [PATCH 125/158] Run on ubuntu-latest --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 161364f..820d728 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -5,7 +5,7 @@ on: [pull_request] jobs: build: - runs-on: ubuntu-18.04 + runs-on: ubuntu-latest strategy: matrix: From a3cfb48f5b25400526f6cf48e395e2317eb0b840 Mon Sep 17 00:00:00 2001 From: Jamie Matthews Date: Sat, 21 Jan 2023 20:54:54 +0000 Subject: [PATCH 126/158] Try psycopg2-binary instead --- .github/workflows/ci.yml | 2 -- test-requirements.txt | 2 +- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 820d728..3de6a51 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -42,8 +42,6 @@ jobs: - name: Start MySQL run: sudo systemctl start mysql.service - uses: actions/checkout@v2 - - name: Install system Python build deps for psycopg2 - run: sudo apt-get install libpython3.x-dev - name: Set up Python ${{ matrix.python }} uses: actions/setup-python@v2 with: diff --git a/test-requirements.txt b/test-requirements.txt index a40e03b..138fc05 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -2,5 +2,5 @@ mysqlclient==1.4.6 freezegun==0.3.12 mock==3.0.5 dj-database-url==0.5.0 -psycopg2==2.8.4 +psycopg2-binary==2.8.4 black==22.6.0 From c11c950601bce734e74aed312e5af93f66cd2c92 Mon Sep 17 00:00:00 2001 From: Jamie Matthews Date: Sat, 21 Jan 2023 20:56:29 +0000 Subject: [PATCH 127/158] Revert "Try psycopg2-binary instead" This reverts commit a3cfb48f5b25400526f6cf48e395e2317eb0b840. --- .github/workflows/ci.yml | 2 ++ test-requirements.txt | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 3de6a51..820d728 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -42,6 +42,8 @@ jobs: - name: Start MySQL run: sudo systemctl start mysql.service - uses: actions/checkout@v2 + - name: Install system Python build deps for psycopg2 + run: sudo apt-get install libpython3.x-dev - name: Set up Python ${{ matrix.python }} uses: actions/setup-python@v2 with: diff --git a/test-requirements.txt b/test-requirements.txt index 138fc05..a40e03b 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -2,5 +2,5 @@ mysqlclient==1.4.6 freezegun==0.3.12 mock==3.0.5 dj-database-url==0.5.0 -psycopg2-binary==2.8.4 +psycopg2==2.8.4 black==22.6.0 From d52bea440c46a0831c648846f3ff22753753b208 Mon Sep 17 00:00:00 2001 From: Jamie Matthews Date: Sat, 21 Jan 2023 20:57:05 +0000 Subject: [PATCH 128/158] All the python dev --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 820d728..057e4d5 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -43,7 +43,7 @@ jobs: run: sudo systemctl start mysql.service - uses: actions/checkout@v2 - name: Install system Python build deps for psycopg2 - run: sudo apt-get install libpython3.x-dev + run: sudo apt-get install python-dev python3-dev python3.11-dev - name: Set up Python ${{ matrix.python }} uses: actions/setup-python@v2 with: From e483e764f3fb14bdb2373c98ec14827c20431b7a Mon Sep 17 00:00:00 2001 From: Jamie Matthews Date: Sat, 21 Jan 2023 20:58:19 +0000 Subject: [PATCH 129/158] One less of the python dev --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 057e4d5..a493cf0 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -43,7 +43,7 @@ jobs: run: sudo systemctl start mysql.service - uses: actions/checkout@v2 - name: Install system Python build deps for psycopg2 - run: sudo apt-get install python-dev python3-dev python3.11-dev + run: sudo apt-get install python3-dev python3.11-dev - name: Set up Python ${{ matrix.python }} uses: actions/setup-python@v2 with: From 56dc33d1288f453ada9bf43070825495b7297fea Mon Sep 17 00:00:00 2001 From: Jamie Matthews Date: Sat, 21 Jan 2023 21:04:23 +0000 Subject: [PATCH 130/158] Try upgrading psycopg2 --- test-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test-requirements.txt b/test-requirements.txt index a40e03b..3797a76 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -2,5 +2,5 @@ mysqlclient==1.4.6 freezegun==0.3.12 mock==3.0.5 dj-database-url==0.5.0 -psycopg2==2.8.4 +psycopg2==2.9.5 black==22.6.0 From c98548d31f5d63265f3cbe02884c7f197c069740 Mon Sep 17 00:00:00 2001 From: Jamie Matthews Date: Mon, 17 Jul 2023 14:17:32 +0100 Subject: [PATCH 131/158] Add Django 4.2 to test matrix, remove Python 3.7 --- .github/workflows/ci.yml | 14 +++++--------- 1 file changed, 5 insertions(+), 9 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a493cf0..853e3a5 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -9,17 +9,13 @@ jobs: strategy: matrix: - python: ["3.7", "3.8", "3.9", "3.10", "3.11"] - django: ["3.2", "4.0", "4.1"] + python: ["3.8", "3.9", "3.10", "3.11"] + django: ["3.2", "4.0", "4.1", "4.2"] exclude: - - python: "3.6" + - python: "3.11" + django: "3.2" + - python: "3.11" django: "4.0" - - python: "3.7" - django: "4.0" - - python: "3.6" - django: "4.1" - - python: "3.7" - django: "4.1" database_url: - postgres://runner:password@localhost/project - mysql://root:root@127.0.0.1/project From 535d80922799cf5254f256c762a2e747b949f782 Mon Sep 17 00:00:00 2001 From: Jamie Matthews Date: Mon, 17 Jul 2023 14:26:50 +0100 Subject: [PATCH 132/158] Use postgres 12 --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 853e3a5..bbd0ea7 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -23,7 +23,7 @@ jobs: services: postgres: - image: postgres:11 + image: postgres:12 ports: - 5432:5432 env: From cbc1accb9c17c1935286f7d123df7b6d95f7a049 Mon Sep 17 00:00:00 2001 From: Jamie Matthews Date: Mon, 17 Jul 2023 14:31:23 +0100 Subject: [PATCH 133/158] Correct versions in README --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index f8e2831..d814caa 100644 --- a/README.md +++ b/README.md @@ -7,8 +7,8 @@ Simple database-backed job queue. Jobs are defined in your settings, and are pro Asynchronous tasks are run via a *job queue*. This system is designed to support multi-step job workflows. Supported and tested against: -- Django 3.2, 4.0, 4.1 -- Python 3.7, 3.8, 3.9, 3.10, 3.11 +- Django 3.2, 4.0, 4.1, 4.2 +- Python 3.8, 3.9, 3.10, 3.11 ## Getting Started From 69402195bf93191781824ec9654ea78cd600b76f Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 20 Mar 2024 17:08:02 +0000 Subject: [PATCH 134/158] Bump black from 22.6.0 to 24.3.0 Bumps [black](https://github.com/psf/black) from 22.6.0 to 24.3.0. - [Release notes](https://github.com/psf/black/releases) - [Changelog](https://github.com/psf/black/blob/main/CHANGES.md) - [Commits](https://github.com/psf/black/compare/22.6.0...24.3.0) --- updated-dependencies: - dependency-name: black dependency-type: direct:production ... Signed-off-by: dependabot[bot] --- test-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test-requirements.txt b/test-requirements.txt index 3797a76..f5b2998 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -3,4 +3,4 @@ freezegun==0.3.12 mock==3.0.5 dj-database-url==0.5.0 psycopg2==2.9.5 -black==22.6.0 +black==24.3.0 From e4c939e634cf2cc54f4c0d5fbbad690bdf359c1c Mon Sep 17 00:00:00 2001 From: Jamie Matthews Date: Tue, 9 Apr 2024 08:47:50 +0100 Subject: [PATCH 135/158] Add Django 5 and Python 3.12 to test matrix --- .github/workflows/ci.yml | 8 ++++++-- README.md | 4 ++-- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index bbd0ea7..0b5155c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -9,13 +9,17 @@ jobs: strategy: matrix: - python: ["3.8", "3.9", "3.10", "3.11"] - django: ["3.2", "4.0", "4.1", "4.2"] + python: ["3.8", "3.9", "3.10", "3.11", "3.12"] + django: ["3.2", "4.0", "4.1", "4.2", "5.0"] exclude: - python: "3.11" django: "3.2" + - python: "3.12" + django: "3.2" - python: "3.11" django: "4.0" + - python: "3.12" + django: "4.0" database_url: - postgres://runner:password@localhost/project - mysql://root:root@127.0.0.1/project diff --git a/README.md b/README.md index d814caa..e78b62f 100644 --- a/README.md +++ b/README.md @@ -7,8 +7,8 @@ Simple database-backed job queue. Jobs are defined in your settings, and are pro Asynchronous tasks are run via a *job queue*. This system is designed to support multi-step job workflows. Supported and tested against: -- Django 3.2, 4.0, 4.1, 4.2 -- Python 3.8, 3.9, 3.10, 3.11 +- Django 3.2, 4.0, 4.1, 4.2, 5.0 +- Python 3.8, 3.9, 3.10, 3.11, 3.12 ## Getting Started From 705dc4b2cd1186ba4dd82c0cebb4b0c3e395119e Mon Sep 17 00:00:00 2001 From: Jamie Matthews Date: Tue, 9 Apr 2024 08:48:20 +0100 Subject: [PATCH 136/158] Unpin postgres version --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 0b5155c..acb2a03 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -27,7 +27,7 @@ jobs: services: postgres: - image: postgres:12 + image: postgres ports: - 5432:5432 env: From 8d88eb2fef3b3a4e6936517079a6aef53368dfe4 Mon Sep 17 00:00:00 2001 From: Jamie Matthews Date: Tue, 9 Apr 2024 09:22:48 +0100 Subject: [PATCH 137/158] Fix timezone issues --- django_dbq/tests.py | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/django_dbq/tests.py b/django_dbq/tests.py index 3ae7ab9..5ded5d7 100644 --- a/django_dbq/tests.py +++ b/django_dbq/tests.py @@ -13,6 +13,13 @@ from io import StringIO +try: + utc = timezone.utc +except AttributeError: + from datetime import timezone as datetime_timezone + utc = datetime_timezone.utc + + def test_task(job=None): pass # pragma: no cover @@ -189,7 +196,7 @@ def test_get_next_ready_job(self): Job.objects.create(name="testjob", state=Job.STATES.READY) Job.objects.create(name="testjob", state=Job.STATES.PROCESSING) expected = Job.objects.create(name="testjob", state=Job.STATES.READY) - expected.created = datetime.now() - timedelta(minutes=1) + expected.created = timezone.now() - timedelta(minutes=1) expected.save() self.assertEqual(Job.objects.get_ready_or_none("default"), expected) @@ -231,7 +238,7 @@ def test_gets_jobs_in_priority_and_date_order(self): def test_ignores_jobs_until_run_after_is_in_the_past(self): job_1 = Job.objects.create(name="testjob") - job_2 = Job.objects.create(name="testjob", run_after=datetime(2021, 11, 4, 8)) + job_2 = Job.objects.create(name="testjob", run_after=datetime(2021, 11, 4, 8, tzinfo=utc)) with freezegun.freeze_time(datetime(2021, 11, 4, 7)): self.assertEqual( @@ -256,7 +263,7 @@ def test_get_next_ready_job_created(self): Job.objects.create(name="testjob", state=Job.STATES.NEW) Job.objects.create(name="testjob", state=Job.STATES.PROCESSING) expected = Job.objects.create(name="testjob", state=Job.STATES.NEW) - expected.created = datetime.now() - timedelta(minutes=1) + expected.created = timezone.now() - timedelta(minutes=1) expected.save() self.assertEqual(Job.objects.get_ready_or_none("default"), expected) @@ -336,7 +343,7 @@ def test_failure_hook(self): @override_settings(JOBS={"testjob": {"tasks": ["a"]}}) class DeleteOldJobsTestCase(TestCase): def test_delete_old_jobs(self): - two_days_ago = datetime.utcnow() - timedelta(days=2) + two_days_ago = timezone.now() - timedelta(days=2) j1 = Job.objects.create(name="testjob", state=Job.STATES.COMPLETE) j1.created = two_days_ago From 8d5e7f637c80e3e8f373f397a4dab5c2ff9bf48b Mon Sep 17 00:00:00 2001 From: Jamie Matthews Date: Tue, 9 Apr 2024 09:25:03 +0100 Subject: [PATCH 138/158] Exclude unsupported Python/Django version combinations --- .github/workflows/ci.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index acb2a03..5c2a216 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -20,6 +20,10 @@ jobs: django: "4.0" - python: "3.12" django: "4.0" + - python: "3.8" + django: "5.0" + - python: "3.9" + django: "5.0" database_url: - postgres://runner:password@localhost/project - mysql://root:root@127.0.0.1/project From c12e62d176eff86aff9e1d30941d320a0db41b35 Mon Sep 17 00:00:00 2001 From: Jamie Matthews Date: Tue, 9 Apr 2024 09:27:05 +0100 Subject: [PATCH 139/158] Set USE_TZ to True in test settings --- testsettings.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/testsettings.py b/testsettings.py index 040eade..2d0d0ba 100644 --- a/testsettings.py +++ b/testsettings.py @@ -19,3 +19,5 @@ "root": {"handlers": ["console"], "level": "INFO",}, "loggers": {"django_dbq": {"level": "CRITICAL", "propagate": True,},}, } + +USE_TZ = True From 90f48fe4c142da458b5d81949ec6b664095832b6 Mon Sep 17 00:00:00 2001 From: Jamie Matthews Date: Tue, 9 Apr 2024 09:27:48 +0100 Subject: [PATCH 140/158] Black formatting --- django_dbq/tests.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/django_dbq/tests.py b/django_dbq/tests.py index 5ded5d7..5ed868c 100644 --- a/django_dbq/tests.py +++ b/django_dbq/tests.py @@ -17,6 +17,7 @@ utc = timezone.utc except AttributeError: from datetime import timezone as datetime_timezone + utc = datetime_timezone.utc @@ -238,7 +239,9 @@ def test_gets_jobs_in_priority_and_date_order(self): def test_ignores_jobs_until_run_after_is_in_the_past(self): job_1 = Job.objects.create(name="testjob") - job_2 = Job.objects.create(name="testjob", run_after=datetime(2021, 11, 4, 8, tzinfo=utc)) + job_2 = Job.objects.create( + name="testjob", run_after=datetime(2021, 11, 4, 8, tzinfo=utc) + ) with freezegun.freeze_time(datetime(2021, 11, 4, 7)): self.assertEqual( From 4261e8d7e63e8cdb6585312c12b69645681739d9 Mon Sep 17 00:00:00 2001 From: Jamie Matthews Date: Tue, 9 Apr 2024 10:02:47 +0100 Subject: [PATCH 141/158] Add support for customising the age of jobs that are deleted by the delete_old_jobs command --- django_dbq/management/commands/delete_old_jobs.py | 11 ++++++++++- django_dbq/models.py | 8 ++++---- django_dbq/tests.py | 14 ++++++++++++++ 3 files changed, 28 insertions(+), 5 deletions(-) diff --git a/django_dbq/management/commands/delete_old_jobs.py b/django_dbq/management/commands/delete_old_jobs.py index 15d4cc8..1bdc072 100644 --- a/django_dbq/management/commands/delete_old_jobs.py +++ b/django_dbq/management/commands/delete_old_jobs.py @@ -6,6 +6,15 @@ class Command(BaseCommand): help = "Delete old jobs" + def add_arguments(self, parser): + parser.add_argument( + "--hours", + help="Delete jobs older than this many hours", + default=None, + required=False, + type=int, + ) + def handle(self, *args, **options): - Job.objects.delete_old() + Job.objects.delete_old(hours=options["hours"]) self.stdout.write("Deleted old jobs") diff --git a/django_dbq/models.py b/django_dbq/models.py index 3e12289..d93a05b 100644 --- a/django_dbq/models.py +++ b/django_dbq/models.py @@ -15,7 +15,7 @@ logger = logging.getLogger(__name__) -DELETE_JOBS_AFTER_HOURS = 24 +DEFAULT_DELETE_JOBS_AFTER_HOURS = 24 class JobManager(models.Manager): @@ -49,9 +49,9 @@ def get_ready_or_none(self, queue_name, max_retries=3): retries_left, ) - def delete_old(self): + def delete_old(self, hours=None): """ - Delete all jobs older than DELETE_JOBS_AFTER_HOURS + Delete all jobs older than hours, or DEFAULT_DELETE_JOBS_AFTER_HOURS """ delete_jobs_in_states = [ Job.STATES.FAILED, @@ -59,7 +59,7 @@ def delete_old(self): Job.STATES.STOPPING, ] delete_jobs_created_before = timezone.now() - datetime.timedelta( - hours=DELETE_JOBS_AFTER_HOURS + hours=hours or DEFAULT_DELETE_JOBS_AFTER_HOURS ) logger.info( "Deleting all job in states %s created before %s", diff --git a/django_dbq/tests.py b/django_dbq/tests.py index 5ed868c..ad376cd 100644 --- a/django_dbq/tests.py +++ b/django_dbq/tests.py @@ -371,3 +371,17 @@ def test_delete_old_jobs(self): self.assertEqual(Job.objects.count(), 2) self.assertTrue(j4 in Job.objects.all()) self.assertTrue(j5 in Job.objects.all()) + + def test_delete_old_jobs_with_custom_hours_argument(self): + j1 = Job.objects.create(name="testjob", state=Job.STATES.COMPLETE) + j1.created = timezone.now() - timedelta(days=5) + j1.save() + + j2 = Job.objects.create(name="testjob", state=Job.STATES.COMPLETE) + j2.created = timezone.now() - timedelta(days=3) + j2.save() + + Job.objects.delete_old(hours=24 * 4) + + self.assertEqual(Job.objects.count(), 1) + self.assertTrue(j2 in Job.objects.all()) From 9bc282ee48ed4bbd86c2c3a178483f03494f6b94 Mon Sep 17 00:00:00 2001 From: Jamie Matthews Date: Tue, 9 Apr 2024 10:05:08 +0100 Subject: [PATCH 142/158] Document --hours argument --- README.md | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/README.md b/README.md index e78b62f..81770ab 100644 --- a/README.md +++ b/README.md @@ -267,8 +267,7 @@ in the dict returned by this method. ##### manage.py delete_old_jobs There is a management command, `manage.py delete_old_jobs`, which deletes any jobs from the database which are in state `COMPLETE` or `FAILED` and were -created more than 24 hours ago. This could be run, for example, as a cron task, -to ensure the jobs table remains at a reasonable size. +created more than (by default) 24 hours ago. This could be run, for example, as a cron task, to ensure the jobs table remains at a reasonable size. Use the `--hours` argument to control the age of jobs that will be deleted. ##### manage.py worker To start a worker: From 13c1bdaefef581d5baedd5b250b1009869f0d2c2 Mon Sep 17 00:00:00 2001 From: Jamie Matthews Date: Tue, 9 Apr 2024 10:26:46 +0100 Subject: [PATCH 143/158] Version 3.1.0 --- django_dbq/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/django_dbq/__init__.py b/django_dbq/__init__.py index 528787c..f5f41e5 100644 --- a/django_dbq/__init__.py +++ b/django_dbq/__init__.py @@ -1 +1 @@ -__version__ = "3.0.0" +__version__ = "3.1.0" From 9a8facdf506e80d65943df9d0b621bcded89bc49 Mon Sep 17 00:00:00 2001 From: Jamie Matthews Date: Wed, 1 May 2024 09:04:25 +0100 Subject: [PATCH 144/158] Add note about bulk_create to README --- README.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/README.md b/README.md index 81770ab..5562651 100644 --- a/README.md +++ b/README.md @@ -285,6 +285,12 @@ jobs in the "NEW" or "READY" states will be returned. **Important:** If you misspell or provide a queue name which does not have any jobs, a depth of 0 will always be returned. +### Gotcha: `bulk_create` + +Because the `Job` model has logic in its `save` method, and because `save` doesn't get called when using `bulk_create`, you can't easily use `bulk_create` to create multiple `Job` instances at the same time. + +If you really need to do this, you should be able to get it to work by using `django_dbq.tasks.get_next_task_name` to compute the next task name from the `name` of the job, and then use that value to populate the `next_task` field on each of the unsaved `Job` instances before calling `bulk_create`. Note that if you use the approach, the job's `creation_hook` will not be called. + ## Testing It may be necessary to supply a DATABASE_PORT environment variable. From 71f53be102e8ce86bb1982385399d757bdf69b90 Mon Sep 17 00:00:00 2001 From: James Addison Date: Thu, 20 Jun 2024 09:25:18 -0700 Subject: [PATCH 145/158] Add pre and post task hooks. --- README.md | 23 +++++++++++ django_dbq/__init__.py | 2 +- django_dbq/management/commands/worker.py | 4 ++ django_dbq/models.py | 22 ++++++++++ django_dbq/tasks.py | 12 ++++++ django_dbq/tests.py | 52 ++++++++++++++++++++++++ 6 files changed, 114 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 5562651..cb77aef 100644 --- a/README.md +++ b/README.md @@ -112,6 +112,29 @@ JOBS = { } ``` +#### Pre & Post Task Hooks +You can also run pre task or post task hooks, which happen in the normal processing of your `Job` instances and are executed in the worker process. + +Both pre and post task hooks receive your `Job` instance as their only argument. Here's an example: + +```python +def my_pre_task_hook(job): + ... # configure something before running your task +``` + +To ensure these hooks gets run, simply add a `pre_task_hook` or `post_task_hook` key (or both, if needed) to your job config like so: + +```python +JOBS = { + "my_job": { + "tasks": ["project.common.jobs.my_task"], + "pre_task_hook": "project.common.jobs.my_pre_task_hook", + "post_task_hook": "project.common.jobs.my_post_task_hook", + }, +} +``` + + ### Start the worker In another terminal: diff --git a/django_dbq/__init__.py b/django_dbq/__init__.py index f5f41e5..1173108 100644 --- a/django_dbq/__init__.py +++ b/django_dbq/__init__.py @@ -1 +1 @@ -__version__ = "3.1.0" +__version__ = "3.2.0" diff --git a/django_dbq/management/commands/worker.py b/django_dbq/management/commands/worker.py index 9215aad..7434981 100644 --- a/django_dbq/management/commands/worker.py +++ b/django_dbq/management/commands/worker.py @@ -61,6 +61,8 @@ def _process_job(self): if not job: return + job.run_pre_task_hook() + logger.info( 'Processing job: name="%s" queue="%s" id=%s state=%s next_task=%s', job.name, @@ -109,6 +111,8 @@ def _process_job(self): logger.exception("Failed to save job: id=%s", job.pk) raise + job.run_post_task_hook() + self.current_job = None diff --git a/django_dbq/models.py b/django_dbq/models.py index d93a05b..31c4aef 100644 --- a/django_dbq/models.py +++ b/django_dbq/models.py @@ -3,6 +3,8 @@ from django.utils.module_loading import import_string from django_dbq.tasks import ( get_next_task_name, + get_pre_task_hook_name, + get_post_task_hook_name, get_failure_hook_name, get_creation_hook_name, ) @@ -126,12 +128,32 @@ def save(self, *args, **kwargs): def update_next_task(self): self.next_task = get_next_task_name(self.name, self.next_task) or "" + def get_pre_task_hook_name(self): + return get_pre_task_hook_name(self.name) + + def get_post_task_hook_name(self): + return get_post_task_hook_name(self.name) + def get_failure_hook_name(self): return get_failure_hook_name(self.name) def get_creation_hook_name(self): return get_creation_hook_name(self.name) + def run_pre_task_hook(self): + pre_task_hook_name = self.get_pre_task_hook_name() + if pre_task_hook_name: + logger.info("Running pre_task hook %s for new job", pre_task_hook_name) + pre_task_hook_function = import_string(pre_task_hook_name) + pre_task_hook_function(self) + + def run_post_task_hook(self): + post_task_hook_name = self.get_post_task_hook_name() + if post_task_hook_name: + logger.info("Running post_task hook %s for new job", post_task_hook_name) + post_task_hook_function = import_string(post_task_hook_name) + post_task_hook_function(self) + def run_creation_hook(self): creation_hook_name = self.get_creation_hook_name() if creation_hook_name: diff --git a/django_dbq/tasks.py b/django_dbq/tasks.py index 3e43da3..a95b4a5 100644 --- a/django_dbq/tasks.py +++ b/django_dbq/tasks.py @@ -2,6 +2,8 @@ TASK_LIST_KEY = "tasks" +PRE_TASK_HOOK_KEY = "pre_task_hook" +POST_TASK_HOOK_KEY = "post_task_hook" FAILURE_HOOK_KEY = "failure_hook" CREATION_HOOK_KEY = "creation_hook" @@ -24,6 +26,16 @@ def get_next_task_name(job_name, current_task=None): return None +def get_pre_task_hook_name(job_name): + """Return the name of the pre task hook for the given job (as a string) or None""" + return settings.JOBS[job_name].get(PRE_TASK_HOOK_KEY) + + +def get_post_task_hook_name(job_name): + """Return the name of the post_task hook for the given job (as a string) or None""" + return settings.JOBS[job_name].get(POST_TASK_HOOK_KEY) + + def get_failure_hook_name(job_name): """Return the name of the failure hook for the given job (as a string) or None""" return settings.JOBS[job_name].get(FAILURE_HOOK_KEY) diff --git a/django_dbq/tests.py b/django_dbq/tests.py index ad376cd..39c0ba5 100644 --- a/django_dbq/tests.py +++ b/django_dbq/tests.py @@ -34,12 +34,25 @@ def failing_task(job): raise Exception("uh oh") +def pre_task_hook(job): + job.workspace["output"] = "pre task hook ran" + job.workspace["job_id"] = str(job.id) + + +def post_task_hook(job): + job.workspace["output"] = "post task hook ran" + job.workspace["job_id"] = str(job.id) + + def failure_hook(job, exception): job.workspace["output"] = "failure hook ran" + job.workspace["exception"] = str(exception) + job.workspace["job_id"] = str(job.id) def creation_hook(job): job.workspace["output"] = "creation hook ran" + job.workspace["job_id"] = str(job.id) @override_settings(JOBS={"testjob": {"tasks": ["a"]}}) @@ -316,6 +329,7 @@ def test_creation_hook(self): job = Job.objects.create(name="testjob") job = Job.objects.get() self.assertEqual(job.workspace["output"], "creation hook ran") + self.assertEqual(job.workspace["job_id"], str(job.id)) def test_creation_hook_only_runs_on_create(self): job = Job.objects.create(name="testjob") @@ -326,6 +340,42 @@ def test_creation_hook_only_runs_on_create(self): self.assertEqual(job.workspace["output"], "creation hook output removed") +@override_settings( + JOBS={ + "testjob": { + "tasks": ["django_dbq.tests.failing_task"], + "pre_task_hook": "django_dbq.tests.pre_task_hook", + } + } +) +class JobPreTaskHookTestCase(TestCase): + def test_pre_task_hook(self): + job = Job.objects.create(name="testjob") + Worker("default", 1)._process_job() + job = Job.objects.get() + self.assertEqual(job.state, Job.STATES.FAILED) + self.assertEqual(job.workspace["output"], "failure hook ran") + self.assertEqual(job.workspace["job_id"], str(job.id)) + + +@override_settings( + JOBS={ + "testjob": { + "tasks": ["django_dbq.tests.failing_task"], + "post_task_hook": "django_dbq.tests.post_task_hook", + } + } +) +class JobPostTaskHookTestCase(TestCase): + def test_post_task_hook(self): + job = Job.objects.create(name="testjob") + Worker("default", 1)._process_job() + job = Job.objects.get() + self.assertEqual(job.state, Job.STATES.FAILED) + self.assertEqual(job.workspace["output"], "post task hook ran") + self.assertEqual(job.workspace["job_id"], str(job.id)) + + @override_settings( JOBS={ "testjob": { @@ -341,6 +391,8 @@ def test_failure_hook(self): job = Job.objects.get() self.assertEqual(job.state, Job.STATES.FAILED) self.assertEqual(job.workspace["output"], "failure hook ran") + self.assertIn("uh oh", job.workspace["exception"]) + self.assertEqual(job.workspace["job_id"], str(job.id)) @override_settings(JOBS={"testjob": {"tasks": ["a"]}}) From dba1e9159354869adf5b6f214c73698aa354cc82 Mon Sep 17 00:00:00 2001 From: Jamie Matthews Date: Fri, 21 Jun 2024 14:54:57 +0100 Subject: [PATCH 146/158] Rework where pre and post task hooks are run, and fix tests --- django_dbq/management/commands/worker.py | 12 ++++++++---- django_dbq/tests.py | 10 +++++----- 2 files changed, 13 insertions(+), 9 deletions(-) diff --git a/django_dbq/management/commands/worker.py b/django_dbq/management/commands/worker.py index 7434981..fcafd14 100644 --- a/django_dbq/management/commands/worker.py +++ b/django_dbq/management/commands/worker.py @@ -61,8 +61,6 @@ def _process_job(self): if not job: return - job.run_pre_task_hook() - logger.info( 'Processing job: name="%s" queue="%s" id=%s state=%s next_task=%s', job.name, @@ -77,7 +75,10 @@ def _process_job(self): try: task_function = import_string(job.next_task) + + job.run_pre_task_hook() task_function(job) + job.update_next_task() if not job.next_task: job.state = Job.STATES.COMPLETE @@ -96,6 +97,11 @@ def _process_job(self): failure_hook_function(job, exception) else: logger.info("No failure hook for job id=%s", job.pk) + finally: + try: + job.run_post_task_hook() + except: + logger.exception("Job id=%s post_task_hook failed", job.pk) logger.info( 'Updating job: name="%s" id=%s state=%s next_task=%s', @@ -111,8 +117,6 @@ def _process_job(self): logger.exception("Failed to save job: id=%s", job.pk) raise - job.run_post_task_hook() - self.current_job = None diff --git a/django_dbq/tests.py b/django_dbq/tests.py index 39c0ba5..dd83540 100644 --- a/django_dbq/tests.py +++ b/django_dbq/tests.py @@ -343,7 +343,7 @@ def test_creation_hook_only_runs_on_create(self): @override_settings( JOBS={ "testjob": { - "tasks": ["django_dbq.tests.failing_task"], + "tasks": ["django_dbq.tests.test_task"], "pre_task_hook": "django_dbq.tests.pre_task_hook", } } @@ -353,15 +353,15 @@ def test_pre_task_hook(self): job = Job.objects.create(name="testjob") Worker("default", 1)._process_job() job = Job.objects.get() - self.assertEqual(job.state, Job.STATES.FAILED) - self.assertEqual(job.workspace["output"], "failure hook ran") + self.assertEqual(job.state, Job.STATES.COMPLETE) + self.assertEqual(job.workspace["output"], "pre task hook ran") self.assertEqual(job.workspace["job_id"], str(job.id)) @override_settings( JOBS={ "testjob": { - "tasks": ["django_dbq.tests.failing_task"], + "tasks": ["django_dbq.tests.test_task"], "post_task_hook": "django_dbq.tests.post_task_hook", } } @@ -371,7 +371,7 @@ def test_post_task_hook(self): job = Job.objects.create(name="testjob") Worker("default", 1)._process_job() job = Job.objects.get() - self.assertEqual(job.state, Job.STATES.FAILED) + self.assertEqual(job.state, Job.STATES.COMPLETE) self.assertEqual(job.workspace["output"], "post task hook ran") self.assertEqual(job.workspace["job_id"], str(job.id)) From c52941f6fe2aa41e21c1df96bb294e2db9cb9be0 Mon Sep 17 00:00:00 2001 From: Jamie Matthews Date: Fri, 21 Jun 2024 16:14:18 +0100 Subject: [PATCH 147/158] Refactor to pull task and hook running into Job model --- django_dbq/management/commands/worker.py | 17 +++-------------- django_dbq/models.py | 17 ++++++++++++++--- 2 files changed, 17 insertions(+), 17 deletions(-) diff --git a/django_dbq/management/commands/worker.py b/django_dbq/management/commands/worker.py index fcafd14..d166b8d 100644 --- a/django_dbq/management/commands/worker.py +++ b/django_dbq/management/commands/worker.py @@ -74,12 +74,10 @@ def _process_job(self): self.current_job = job try: - task_function = import_string(job.next_task) - job.run_pre_task_hook() - task_function(job) - + job.run_next_task() job.update_next_task() + if not job.next_task: job.state = Job.STATES.COMPLETE else: @@ -87,16 +85,7 @@ def _process_job(self): except Exception as exception: logger.exception("Job id=%s failed", job.pk) job.state = Job.STATES.FAILED - - failure_hook_name = job.get_failure_hook_name() - if failure_hook_name: - logger.info( - "Running failure hook %s for job id=%s", failure_hook_name, job.pk - ) - failure_hook_function = import_string(failure_hook_name) - failure_hook_function(job, exception) - else: - logger.info("No failure hook for job id=%s", job.pk) + job.run_failure_hook(exception) finally: try: job.run_post_task_hook() diff --git a/django_dbq/models.py b/django_dbq/models.py index 31c4aef..b58eef4 100644 --- a/django_dbq/models.py +++ b/django_dbq/models.py @@ -128,6 +128,10 @@ def save(self, *args, **kwargs): def update_next_task(self): self.next_task = get_next_task_name(self.name, self.next_task) or "" + def run_next_task(self): + next_task_function = import_string(self.next_task) + next_task_function(self) + def get_pre_task_hook_name(self): return get_pre_task_hook_name(self.name) @@ -143,21 +147,28 @@ def get_creation_hook_name(self): def run_pre_task_hook(self): pre_task_hook_name = self.get_pre_task_hook_name() if pre_task_hook_name: - logger.info("Running pre_task hook %s for new job", pre_task_hook_name) + logger.info("Running pre_task hook %s for job", pre_task_hook_name) pre_task_hook_function = import_string(pre_task_hook_name) pre_task_hook_function(self) def run_post_task_hook(self): post_task_hook_name = self.get_post_task_hook_name() if post_task_hook_name: - logger.info("Running post_task hook %s for new job", post_task_hook_name) + logger.info("Running post_task hook %s for job", post_task_hook_name) post_task_hook_function = import_string(post_task_hook_name) post_task_hook_function(self) + def run_failure_hook(self, exception): + failure_hook_name = self.get_failure_hook_name() + if failure_hook_name: + logger.info("Running failure hook %s for job", failure_hook_name) + failure_hook_function = import_string(failure_hook_name) + failure_hook_function(self, exception) + def run_creation_hook(self): creation_hook_name = self.get_creation_hook_name() if creation_hook_name: - logger.info("Running creation hook %s for new job", creation_hook_name) + logger.info("Running creation hook %s for job", creation_hook_name) creation_hook_function = import_string(creation_hook_name) creation_hook_function(self) From 32b1ac9a353cc1277c5d29d87d6f6fd192008c95 Mon Sep 17 00:00:00 2001 From: Jamie Matthews Date: Tue, 25 Jun 2024 10:53:31 +0100 Subject: [PATCH 148/158] Add notes on pre/post hook behaviour --- README.md | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index cb77aef..c4e698a 100644 --- a/README.md +++ b/README.md @@ -113,7 +113,7 @@ JOBS = { ``` #### Pre & Post Task Hooks -You can also run pre task or post task hooks, which happen in the normal processing of your `Job` instances and are executed in the worker process. +You can also run pre task or post task hooks, which happen in the normal processing of your `Job` instances and are executed inside the worker process. Both pre and post task hooks receive your `Job` instance as their only argument. Here's an example: @@ -122,7 +122,7 @@ def my_pre_task_hook(job): ... # configure something before running your task ``` -To ensure these hooks gets run, simply add a `pre_task_hook` or `post_task_hook` key (or both, if needed) to your job config like so: +To ensure these hooks are run, simply add a `pre_task_hook` or `post_task_hook` key (or both, if needed) to your job config like so: ```python JOBS = { @@ -134,6 +134,12 @@ JOBS = { } ``` +Notes: + +* If the `pre_task_hook` fails (raises an exception), the task function is not run, and django-db-queue behaves as if the task function itself had failed: the failure hook is called, and the job is goes into the `FAILED` state. +* The `post_task_hook` is always run, even if the job fails. In this case, it runs after the `failure_hook`. +* If the `post_task_hook` raises an exception, this is logged but the the job is **not marked as failed** and the failure hook does not run. This is because the `post_task_hook` might need to perform cleanup that always happens after the task, no matter whether it succeeds or fails. + ### Start the worker From 6730a6b651c1977a34fb79a03d27a306a6570629 Mon Sep 17 00:00:00 2001 From: Jamie Matthews Date: Mon, 20 Jan 2025 10:29:48 +0000 Subject: [PATCH 149/158] Add Python 3.13 and Django 5.1 to test matrix, drop unsupported versions --- .github/workflows/ci.yml | 20 ++++++-------------- README.md | 4 ++-- django_dbq/tests.py | 15 ++++----------- 3 files changed, 12 insertions(+), 27 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 5c2a216..0747581 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -9,21 +9,13 @@ jobs: strategy: matrix: - python: ["3.8", "3.9", "3.10", "3.11", "3.12"] - django: ["3.2", "4.0", "4.1", "4.2", "5.0"] + python: ["3.9", "3.10", "3.11", "3.12", "3.13"] + django: ["4.2", "5.0", "5.1"] exclude: - - python: "3.11" - django: "3.2" - - python: "3.12" - django: "3.2" - - python: "3.11" - django: "4.0" - - python: "3.12" - django: "4.0" - - python: "3.8" - django: "5.0" - python: "3.9" django: "5.0" + - python: "3.9" + django: "5.1" database_url: - postgres://runner:password@localhost/project - mysql://root:root@127.0.0.1/project @@ -45,11 +37,11 @@ jobs: steps: - name: Start MySQL run: sudo systemctl start mysql.service - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 - name: Install system Python build deps for psycopg2 run: sudo apt-get install python3-dev python3.11-dev - name: Set up Python ${{ matrix.python }} - uses: actions/setup-python@v2 + uses: actions/setup-python@v5 with: python-version: ${{ matrix.python }} - name: Upgraded pip diff --git a/README.md b/README.md index c4e698a..d589707 100644 --- a/README.md +++ b/README.md @@ -7,8 +7,8 @@ Simple database-backed job queue. Jobs are defined in your settings, and are pro Asynchronous tasks are run via a *job queue*. This system is designed to support multi-step job workflows. Supported and tested against: -- Django 3.2, 4.0, 4.1, 4.2, 5.0 -- Python 3.8, 3.9, 3.10, 3.11, 3.12 +- Django 4.2, 5.0, 5.1 +- Python 3.9, 3.10, 3.11, 3.12, 3.13 ## Getting Started diff --git a/django_dbq/tests.py b/django_dbq/tests.py index dd83540..ba80b7f 100644 --- a/django_dbq/tests.py +++ b/django_dbq/tests.py @@ -1,8 +1,8 @@ -from datetime import datetime, timedelta +from datetime import datetime, timedelta, timezone as datetime_timezone import mock import freezegun -from django.core.management import call_command, CommandError +from django.core.management import call_command from django.test import TestCase from django.test.utils import override_settings from django.utils import timezone @@ -13,14 +13,6 @@ from io import StringIO -try: - utc = timezone.utc -except AttributeError: - from datetime import timezone as datetime_timezone - - utc = datetime_timezone.utc - - def test_task(job=None): pass # pragma: no cover @@ -253,7 +245,8 @@ def test_gets_jobs_in_priority_and_date_order(self): def test_ignores_jobs_until_run_after_is_in_the_past(self): job_1 = Job.objects.create(name="testjob") job_2 = Job.objects.create( - name="testjob", run_after=datetime(2021, 11, 4, 8, tzinfo=utc) + name="testjob", + run_after=datetime(2021, 11, 4, 8, tzinfo=datetime_timezone.utc), ) with freezegun.freeze_time(datetime(2021, 11, 4, 7)): From 2de272a1a44fec0c1b74539241b236eb9a6c0774 Mon Sep 17 00:00:00 2001 From: Jamie Matthews Date: Mon, 20 Jan 2025 10:31:25 +0000 Subject: [PATCH 150/158] Remove Python 3.11 from apt packages --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 0747581..2a3dc58 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -39,7 +39,7 @@ jobs: run: sudo systemctl start mysql.service - uses: actions/checkout@v4 - name: Install system Python build deps for psycopg2 - run: sudo apt-get install python3-dev python3.11-dev + run: sudo apt-get install python3-dev - name: Set up Python ${{ matrix.python }} uses: actions/setup-python@v5 with: From 3dd1843f72a7c9abdd0a29b208c8ac755750342e Mon Sep 17 00:00:00 2001 From: Jamie Matthews Date: Mon, 20 Jan 2025 10:35:02 +0000 Subject: [PATCH 151/158] Exclude 3.13 builds from unsupported Django versions --- .github/workflows/ci.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 2a3dc58..02ec500 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -16,6 +16,10 @@ jobs: django: "5.0" - python: "3.9" django: "5.1" + - python: "3.13" + django: "4.2" + - python: "3.13" + django: "5.0" database_url: - postgres://runner:password@localhost/project - mysql://root:root@127.0.0.1/project From d797d37b10251026bc73974a93652b4df166e348 Mon Sep 17 00:00:00 2001 From: Jamie Matthews Date: Mon, 20 Jan 2025 10:41:52 +0000 Subject: [PATCH 152/158] Install the latest point release of each Django and Python version --- .github/workflows/ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 02ec500..fbef003 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -47,13 +47,13 @@ jobs: - name: Set up Python ${{ matrix.python }} uses: actions/setup-python@v5 with: - python-version: ${{ matrix.python }} + python-version: ${{ matrix.python }}.* - name: Upgraded pip run: pip install --upgrade pip - name: Install dependencies run: pip install -r test-requirements.txt - name: Install Django - run: pip install -U django==${{ matrix.django }} + run: pip install -U django~=${{ matrix.django }}.0 - name: Run tests run: python manage.py test - name: Run black From e9f234798ff1d367833e2689bfe53341331e3d9b Mon Sep 17 00:00:00 2001 From: Jamie Matthews Date: Mon, 20 Jan 2025 10:46:20 +0000 Subject: [PATCH 153/158] Upgrade test requirements --- django_dbq/tests.py | 2 +- test-requirements.txt | 11 +++++------ 2 files changed, 6 insertions(+), 7 deletions(-) diff --git a/django_dbq/tests.py b/django_dbq/tests.py index ba80b7f..f755666 100644 --- a/django_dbq/tests.py +++ b/django_dbq/tests.py @@ -1,5 +1,5 @@ from datetime import datetime, timedelta, timezone as datetime_timezone -import mock +from unittest import mock import freezegun from django.core.management import call_command diff --git a/test-requirements.txt b/test-requirements.txt index f5b2998..93545b8 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -1,6 +1,5 @@ -mysqlclient==1.4.6 -freezegun==0.3.12 -mock==3.0.5 -dj-database-url==0.5.0 -psycopg2==2.9.5 -black==24.3.0 +mysqlclient==2.2.7 +freezegun==1.5.1 +dj-database-url==2.3.0 +psycopg2==2.9.10 +black==24.10.0 From e0c7f18e07a34b421e180df37653f1baa130bfad Mon Sep 17 00:00:00 2001 From: Jamie Matthews Date: Mon, 20 Jan 2025 10:47:32 +0000 Subject: [PATCH 154/158] Bump versions in setup.py --- setup.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/setup.py b/setup.py index 5bb0f76..fda66ff 100644 --- a/setup.py +++ b/setup.py @@ -16,7 +16,7 @@ author_email = "contact@dabapps.com" license = "BSD" install_requires = [ - "Django>=3.1", + "Django>=4.2", ] long_description = """Simple database-backed job queue system""" @@ -82,5 +82,5 @@ def get_package_data(package): package_data=get_package_data(package), install_requires=install_requires, classifiers=[], - python_requires=">=3.6" + python_requires=">=3.9" ) From f28e85e124a71d32f51bcaf997bed52d5eb69867 Mon Sep 17 00:00:00 2001 From: George Waller <54817483+George9Waller@users.noreply.github.com> Date: Fri, 21 Feb 2025 13:28:19 +0100 Subject: [PATCH 155/158] Add exclude_future_jobs arg to get_queue_depths New arg exclude_future_jobs filters the READY and NEW jobs to only be ones without a run_after or the run_after is now/in the past --- django_dbq/models.py | 15 +++++++++++---- django_dbq/tests.py | 25 ++++++++++++++++++++++++- 2 files changed, 35 insertions(+), 5 deletions(-) diff --git a/django_dbq/models.py b/django_dbq/models.py index b58eef4..a90354f 100644 --- a/django_dbq/models.py +++ b/django_dbq/models.py @@ -8,7 +8,7 @@ get_failure_hook_name, get_creation_hook_name, ) -from django.db.models import JSONField, UUIDField, Count, TextChoices +from django.db.models import JSONField, UUIDField, Count, TextChoices, Q import datetime import logging import uuid @@ -173,10 +173,17 @@ def run_creation_hook(self): creation_hook_function(self) @staticmethod - def get_queue_depths(): + def get_queue_depths(*, exclude_future_jobs=False): + jobs_waiting_in_queue = Job.objects.filter( + state__in=(Job.STATES.READY, Job.STATES.NEW) + ) + if exclude_future_jobs: + jobs_waiting_in_queue = jobs_waiting_in_queue.filter( + Q(run_after__isnull=True) | Q(run_after__lte=timezone.now()) + ) + annotation_dicts = ( - Job.objects.filter(state__in=(Job.STATES.READY, Job.STATES.NEW)) - .values("queue_name") + jobs_waiting_in_queue.values("queue_name") .order_by("queue_name") .annotate(Count("queue_name")) ) diff --git a/django_dbq/tests.py b/django_dbq/tests.py index dd83540..8709eb5 100644 --- a/django_dbq/tests.py +++ b/django_dbq/tests.py @@ -71,12 +71,17 @@ def test_worker_with_queue_name(self): self.assertTrue("test_queue" in output) +@freezegun.freeze_time("2025-01-01T12:00:00Z") @override_settings(JOBS={"testjob": {"tasks": ["a"]}}) class JobModelMethodTestCase(TestCase): def test_get_queue_depths(self): Job.objects.create(name="testjob", queue_name="default") Job.objects.create(name="testjob", queue_name="testworker") - Job.objects.create(name="testjob", queue_name="testworker") + Job.objects.create( + name="testjob", + queue_name="testworker", + run_after=timezone.make_aware(datetime(2025, 1, 1, 13, 0, 0)), + ) Job.objects.create( name="testjob", queue_name="testworker", state=Job.STATES.FAILED ) @@ -87,6 +92,24 @@ def test_get_queue_depths(self): queue_depths = Job.get_queue_depths() self.assertDictEqual(queue_depths, {"default": 1, "testworker": 2}) + def test_get_queue_depths_exclude_future_jobs(self): + Job.objects.create(name="testjob", queue_name="default") + Job.objects.create(name="testjob", queue_name="testworker") + Job.objects.create( + name="testjob", + queue_name="testworker", + run_after=timezone.make_aware(datetime(2025, 1, 1, 13, 0, 0)), + ) + Job.objects.create( + name="testjob", queue_name="testworker", state=Job.STATES.FAILED + ) + Job.objects.create( + name="testjob", queue_name="testworker", state=Job.STATES.COMPLETE + ) + + queue_depths = Job.get_queue_depths(exclude_future_jobs=True) + self.assertDictEqual(queue_depths, {"default": 1, "testworker": 1}) + @override_settings(JOBS={"testjob": {"tasks": ["a"]}}) class QueueDepthTestCase(TestCase): From 738be6db1603c96429023f48d29022c154757481 Mon Sep 17 00:00:00 2001 From: George Waller <54817483+George9Waller@users.noreply.github.com> Date: Fri, 21 Feb 2025 13:29:43 +0100 Subject: [PATCH 156/158] Add the exclude_future_jobs arg to the queue_depth command The queue_depth command can call get_queue_depths with the option to exclude jobs in the future --- django_dbq/management/commands/queue_depth.py | 5 +++- django_dbq/tests.py | 30 +++++++++++++++++-- 2 files changed, 32 insertions(+), 3 deletions(-) diff --git a/django_dbq/management/commands/queue_depth.py b/django_dbq/management/commands/queue_depth.py index 3419601..cb8b6fd 100644 --- a/django_dbq/management/commands/queue_depth.py +++ b/django_dbq/management/commands/queue_depth.py @@ -8,10 +8,13 @@ class Command(BaseCommand): def add_arguments(self, parser): parser.add_argument("queue_name", nargs="*", default=["default"], type=str) + parser.add_argument("--exclude_future_jobs", default=False, type=bool) def handle(self, *args, **options): queue_names = options["queue_name"] - queue_depths = Job.get_queue_depths() + queue_depths = Job.get_queue_depths( + exclude_future_jobs=options["exclude_future_jobs"] + ) queue_depths_string = " ".join( [ diff --git a/django_dbq/tests.py b/django_dbq/tests.py index 8709eb5..20c4029 100644 --- a/django_dbq/tests.py +++ b/django_dbq/tests.py @@ -111,15 +111,19 @@ def test_get_queue_depths_exclude_future_jobs(self): self.assertDictEqual(queue_depths, {"default": 1, "testworker": 1}) +@freezegun.freeze_time("2025-01-01T12:00:00Z") @override_settings(JOBS={"testjob": {"tasks": ["a"]}}) class QueueDepthTestCase(TestCase): def test_queue_depth(self): - Job.objects.create(name="testjob", state=Job.STATES.FAILED) Job.objects.create(name="testjob", state=Job.STATES.NEW) Job.objects.create(name="testjob", state=Job.STATES.FAILED) Job.objects.create(name="testjob", state=Job.STATES.COMPLETE) - Job.objects.create(name="testjob", state=Job.STATES.READY) + Job.objects.create( + name="testjob", + state=Job.STATES.READY, + run_after=timezone.make_aware(datetime(2025, 1, 1, 13, 0, 0)), + ) Job.objects.create( name="testjob", queue_name="testqueue", state=Job.STATES.READY ) @@ -132,6 +136,28 @@ def test_queue_depth(self): output = stdout.getvalue() self.assertEqual(output.strip(), "event=queue_depths default=2") + def test_queue_depth_exclude_future_jobs(self): + Job.objects.create(name="testjob", state=Job.STATES.FAILED) + Job.objects.create(name="testjob", state=Job.STATES.NEW) + Job.objects.create(name="testjob", state=Job.STATES.FAILED) + Job.objects.create(name="testjob", state=Job.STATES.COMPLETE) + Job.objects.create( + name="testjob", + state=Job.STATES.READY, + run_after=timezone.make_aware(datetime(2025, 1, 1, 13, 0, 0)), + ) + Job.objects.create( + name="testjob", queue_name="testqueue", state=Job.STATES.READY + ) + Job.objects.create( + name="testjob", queue_name="testqueue", state=Job.STATES.READY + ) + + stdout = StringIO() + call_command("queue_depth", exclude_future_jobs=True, stdout=stdout) + output = stdout.getvalue() + self.assertEqual(output.strip(), "event=queue_depths default=1") + def test_queue_depth_multiple_queues(self): Job.objects.create(name="testjob", state=Job.STATES.FAILED) From 73228276f58dcd4502503a2255a4580828eb0ab2 Mon Sep 17 00:00:00 2001 From: George Waller <54817483+George9Waller@users.noreply.github.com> Date: Fri, 21 Feb 2025 13:30:09 +0100 Subject: [PATCH 157/158] Include new arg in readme documentation --- README.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/README.md b/README.md index c4e698a..04f0caf 100644 --- a/README.md +++ b/README.md @@ -288,6 +288,11 @@ queue_depths = Job.get_queue_depths() print(queue_depths) # {"default": 1, "other_queue": 1} ``` +You can also exclude jobs which exist but are scheduled to be run in the future from the queue depths, where `run_after` is set to a future time from now. To do this set the `exclude_future_jobs` kwarg like so: +```python +queue_depths = Job.get_queue_depths(exclude_future_jobs=True) +``` + **Important:** When checking queue depths, do not assume that the key for your queue will always be available. Queue depths of zero won't be included in the dict returned by this method. @@ -312,6 +317,8 @@ manage.py worker [queue_name] [--rate_limit] If you'd like to check your queue depth from the command line, you can run `manage.py queue_depth [queue_name [queue_name ...]]` and any jobs in the "NEW" or "READY" states will be returned. +If you wish to exclude jobs which are scheduled to be run in the future you can add `--exclude_future_jobs` to the command. + **Important:** If you misspell or provide a queue name which does not have any jobs, a depth of 0 will always be returned. ### Gotcha: `bulk_create` From fd09d21bb9e4f5c4935ccbb76f15c50a2fccb186 Mon Sep 17 00:00:00 2001 From: Jamie Matthews Date: Tue, 25 Feb 2025 12:53:40 +0000 Subject: [PATCH 158/158] Version 3.3.0 --- django_dbq/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/django_dbq/__init__.py b/django_dbq/__init__.py index 1173108..88c513e 100644 --- a/django_dbq/__init__.py +++ b/django_dbq/__init__.py @@ -1 +1 @@ -__version__ = "3.2.0" +__version__ = "3.3.0"