Thanks to visit codestin.com
Credit goes to github.com

Skip to content

Add STOPPING state #44

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 8 commits into from
Jan 6, 2022
Merged

Add STOPPING state #44

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
8 commits
Select commit Hold shift + click to select a range
305d9f4
Refactor to move process_job function onto Worker class
j4mie Nov 29, 2021
5a88fa7
Add current_job attribute to Worker
j4mie Nov 29, 2021
2619a49
Add STOPPING state and update when shutdown is requested
j4mie Nov 29, 2021
8214131
Delete old jobs in state STOPPING
j4mie Nov 29, 2021
0661ef2
Merge remote-tracking branch 'origin/master' into stopping-state
j4mie Nov 29, 2021
a93403b
Rebase migrations
j4mie Nov 29, 2021
631dc03
Document STOPPING state and add state diagram
j4mie Nov 29, 2021
e870812
Merge remote-tracking branch 'origin/master' into stopping-state
j4mie Dec 8, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Next Next commit
Refactor to move process_job function onto Worker class
  • Loading branch information
@j4mie
j4mie committed Nov 29, 2021
commit 305d9f4388a78f366555984012d4ac9334f54a14
116 changes: 55 additions & 61 deletions django_dbq/management/commands/worker.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,66 +14,6 @@
DEFAULT_QUEUE_NAME = "default"


def process_job(queue_name):
"""This function grabs the next available job for a given queue, and runs its next task."""

with transaction.atomic():
job = Job.objects.get_ready_or_none(queue_name)
if not job:
return

logger.info(
'Processing job: name="%s" queue="%s" id=%s state=%s next_task=%s',
job.name,
queue_name,
job.pk,
job.state,
job.next_task,
)
job.state = Job.STATES.PROCESSING
job.save()

try:
task_function = import_string(job.next_task)
task_function(job)
job.update_next_task()
if not job.next_task:
job.state = Job.STATES.COMPLETE
else:
job.state = Job.STATES.READY
except Exception as exception:
logger.exception("Job id=%s failed", job.pk)
job.state = Job.STATES.FAILED

failure_hook_name = job.get_failure_hook_name()
if failure_hook_name:
logger.info(
"Running failure hook %s for job id=%s", failure_hook_name, job.pk
)
failure_hook_function = import_string(failure_hook_name)
failure_hook_function(job, exception)
else:
logger.info("No failure hook for job id=%s", job.pk)

logger.info(
'Updating job: name="%s" id=%s state=%s next_task=%s',
job.name,
job.pk,
job.state,
job.next_task or "none",
)

try:
job.save()
except:
logger.error(
"Failed to save job: id=%s org=%s",
job.pk,
job.workspace.get("organisation_id"),
)
raise


class Worker:
def __init__(self, name, rate_limit_in_seconds):
self.queue_name = name
Expand Down Expand Up @@ -103,9 +43,63 @@ def process_job(self):
):
return

process_job(self.queue_name)
self._process_job()

self.last_job_finished = timezone.now()

def _process_job(self):
with transaction.atomic():
job = Job.objects.get_ready_or_none(self.queue_name)
if not job:
return

logger.info(
'Processing job: name="%s" queue="%s" id=%s state=%s next_task=%s',
job.name,
self.queue_name,
job.pk,
job.state,
job.next_task,
)
job.state = Job.STATES.PROCESSING
job.save()

try:
task_function = import_string(job.next_task)
task_function(job)
job.update_next_task()
if not job.next_task:
job.state = Job.STATES.COMPLETE
else:
job.state = Job.STATES.READY
except Exception as exception:
logger.exception("Job id=%s failed", job.pk)
job.state = Job.STATES.FAILED

failure_hook_name = job.get_failure_hook_name()
if failure_hook_name:
logger.info(
"Running failure hook %s for job id=%s", failure_hook_name, job.pk
)
failure_hook_function = import_string(failure_hook_name)
failure_hook_function(job, exception)
else:
logger.info("No failure hook for job id=%s", job.pk)

logger.info(
'Updating job: name="%s" id=%s state=%s next_task=%s',
job.name,
job.pk,
job.state,
job.next_task or "none",
)

try:
job.save()
except:
logger.exception("Failed to save job: id=%s", job.pk)
raise


class Command(BaseCommand):

Expand Down
45 changes: 22 additions & 23 deletions django_dbq/tests.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@
from django.test.utils import override_settings
from django.utils import timezone

from django_dbq.management.commands.worker import process_job, Worker
from django_dbq.management.commands.worker import Worker
from django_dbq.models import Job

from io import StringIO
Expand Down Expand Up @@ -116,41 +116,40 @@ def test_queue_depth_for_queue_with_zero_jobs(self):

@freezegun.freeze_time()
@mock.patch("django_dbq.management.commands.worker.sleep")
@mock.patch("django_dbq.management.commands.worker.process_job")
class WorkerProcessProcessJobTestCase(TestCase):
def setUp(self):
super().setUp()
self.MockWorker = mock.MagicMock()
self.MockWorker.queue_name = "default"
self.MockWorker.rate_limit_in_seconds = 5
self.MockWorker.last_job_finished = None
self.mock_worker = mock.MagicMock()
self.mock_worker.queue_name = "default"
self.mock_worker.rate_limit_in_seconds = 5
self.mock_worker.last_job_finished = None

def test_process_job_no_previous_job_run(self, mock_process_job, mock_sleep):
Worker.process_job(self.MockWorker)
def test_process_job_no_previous_job_run(self, mock_sleep):
Worker.process_job(self.mock_worker)
self.assertEqual(mock_sleep.call_count, 1)
self.assertEqual(mock_process_job.call_count, 1)
self.assertEqual(self.MockWorker.last_job_finished, timezone.now())
self.assertEqual(self.mock_worker._process_job.call_count, 1)
self.assertEqual(self.mock_worker.last_job_finished, timezone.now())

def test_process_job_previous_job_too_soon(self, mock_process_job, mock_sleep):
self.MockWorker.last_job_finished = timezone.now() - timezone.timedelta(
def test_process_job_previous_job_too_soon(self, mock_sleep):
self.mock_worker.last_job_finished = timezone.now() - timezone.timedelta(
seconds=2
)
Worker.process_job(self.MockWorker)
Worker.process_job(self.mock_worker)
self.assertEqual(mock_sleep.call_count, 1)
self.assertEqual(mock_process_job.call_count, 0)
self.assertEqual(self.mock_worker._process_job.call_count, 0)
self.assertEqual(
self.MockWorker.last_job_finished,
self.mock_worker.last_job_finished,
timezone.now() - timezone.timedelta(seconds=2),
)

def test_process_job_previous_job_long_time_ago(self, mock_process_job, mock_sleep):
self.MockWorker.last_job_finished = timezone.now() - timezone.timedelta(
def test_process_job_previous_job_long_time_ago(self, mock_sleep):
self.mock_worker.last_job_finished = timezone.now() - timezone.timedelta(
seconds=7
)
Worker.process_job(self.MockWorker)
Worker.process_job(self.mock_worker)
self.assertEqual(mock_sleep.call_count, 1)
self.assertEqual(mock_process_job.call_count, 1)
self.assertEqual(self.MockWorker.last_job_finished, timezone.now())
self.assertEqual(self.mock_worker._process_job.call_count, 1)
self.assertEqual(self.mock_worker.last_job_finished, timezone.now())


@override_settings(JOBS={"testjob": {"tasks": ["a"]}})
Expand Down Expand Up @@ -246,7 +245,7 @@ def test_task_sequence(self):
class ProcessJobTestCase(TestCase):
def test_process_job(self):
job = Job.objects.create(name="testjob")
process_job("default")
Worker("default", 1)._process_job()
job = Job.objects.get()
self.assertEqual(job.state, Job.STATES.COMPLETE)

Expand All @@ -255,7 +254,7 @@ def test_process_job_wrong_queue(self):
Processing a different queue shouldn't touch our other job
"""
job = Job.objects.create(name="testjob", queue_name="lol")
process_job("default")
Worker("default", 1)._process_job()
job = Job.objects.get()
self.assertEqual(job.state, Job.STATES.NEW)

Expand Down Expand Up @@ -294,7 +293,7 @@ def test_creation_hook_only_runs_on_create(self):
class JobFailureHookTestCase(TestCase):
def test_failure_hook(self):
job = Job.objects.create(name="testjob")
process_job("default")
Worker("default", 1)._process_job()
job = Job.objects.get()
self.assertEqual(job.state, Job.STATES.FAILED)
self.assertEqual(job.workspace["output"], "failure hook ran")
Expand Down