chore: improve debugging for non-running query refresh job (#42)

* Handle unsupported data source schema refreshes as skips * Check and log missing periodic jobs
stacklet · fwereade · May 21, 2021 · May 24, 2021 · May 24, 2021 · May 25, 2021
commit 8a80dc3d9af18a644123b50e7bdcedfc029eebaa
diff --git a/redash/cli/rq.py b/redash/cli/rq.py
@@ -15,6 +15,7 @@
     periodic_job_definitions,
     rq_scheduler,
     schedule_periodic_jobs,
+    check_periodic_jobs,
 )
 from redash.tasks.worker import Worker
 from redash.worker import default_queues
@@ -69,19 +70,25 @@ def __call__(self, process_spec):
         total_jobs_in_watched_queues = sum([len(q.jobs) for q in worker.queues])
         has_nothing_to_do = total_jobs_in_watched_queues == 0
 
-        is_healthy = is_busy or seen_lately or has_nothing_to_do
+        pjobs_ok, num_pjobs, num_missing_pjobs = check_periodic_jobs()
+
+        is_healthy = (is_busy or seen_lately or has_nothing_to_do) and pjobs_ok
 
         self._log(
             "Worker %s healthcheck: Is busy? %s. "
             "Seen lately? %s (%d seconds ago). "
             "Has nothing to do? %s (%d jobs in watched queues). "
+            "Periodic jobs ok? %s (%s missing of %s). "
             "==> Is healthy? %s",
             worker.key,
             is_busy,
             seen_lately,
             time_since_seen.seconds,
             has_nothing_to_do,
             total_jobs_in_watched_queues,
+            pjobs_ok,
+            num_missing_pjobs,
+            num_pjobs,
             is_healthy,
         )
 

diff --git a/redash/tasks/__init__.py b/redash/tasks/__init__.py
@@ -22,8 +22,13 @@
     periodic_job_definitions,
     rq_scheduler,
     schedule_periodic_jobs,
+    check_periodic_jobs
 )
 from redash.tasks.worker import Job, Queue, Worker
+from .alerts import check_alerts_for_query
+from .failure_report import send_aggregated_errors
+from redash import rq_redis_connection
+from rq.connections import push_connection, pop_connection
 
 
 def init_app(app):

diff --git a/redash/tasks/queries/maintenance.py b/redash/tasks/queries/maintenance.py
@@ -11,7 +11,8 @@
 from redash.monitor import rq_job_ids
 from redash.tasks.failure_report import track_failure
 from redash.utils import json_dumps, sentry
-from redash.worker import get_job_logger, job
+from redash.worker import job, get_job_logger
+from redash.query_runner import NotSupported
 
 from .execution import enqueue_query
 
@@ -160,7 +161,7 @@ def remove_ghost_locks():
 @job("schemas")
 def refresh_schema(data_source_id):
     ds = models.DataSource.get_by_id(data_source_id)
-    logger.info("task=refresh_schema state=start ds_id=%s", ds.id)
+    logger.info(u"task=refresh_schema state=start ds_id=%s ds_name=%s", ds.id, ds.name)
     start_time = time.time()
     try:
         ds.get_schema(refresh=True)
@@ -170,6 +171,11 @@ def refresh_schema(data_source_id):
             time.time() - start_time,
         )
         statsd_client.incr("refresh_schema.success")
+    except NotSupported:
+        logger.info(
+            u"task=refresh_schema state=skip ds_id=%s reason=not_supported",
+            ds.id
+        )
     except JobTimeoutException:
         logger.info(
             "task=refresh_schema state=timeout ds_id=%s runtime=%.2f",

diff --git a/redash/tasks/schedule.py b/redash/tasks/schedule.py
@@ -101,11 +101,18 @@ def schedule_periodic_jobs(jobs):
 
     jobs_to_schedule = [job for job in job_definitions if job_id(job) not in rq_scheduler]
 
+    logger.info("Current jobs: %s", ", ".join([
+        job.func_name.rsplit('.', 1)[-1]
+        for job in rq_scheduler.get_jobs()
+    ]))
+
     for job in jobs_to_clean_up:
         logger.info("Removing %s (%s) from schedule.", job.id, job.func_name)
         rq_scheduler.cancel(job)
         job.delete()
 
+    if not jobs_to_schedule:
+        logger.info("No jobs to schedule")
     for job in jobs_to_schedule:
         logger.info(
             "Scheduling %s (%s) with interval %s.",
@@ -114,3 +121,17 @@ def schedule_periodic_jobs(jobs):
             job.get("interval"),
         )
         schedule(job)
+
+
+def check_periodic_jobs():
+    job_definitions = [prep(job) for job in periodic_job_definitions()]
+    missing_jobs = [
+        job["func"].__name__
+        for job in job_definitions
+        if job_id(job) not in rq_scheduler
+    ]
+    if not job_definitions:
+        logger.warn("No periodic jobs defined")
+    if missing_jobs:
+        logger.warn("Missing periodic jobs: %s", ", ".join(missing_jobs))
+    return job_definitions and not missing_jobs, len(job_definitions), len(missing_jobs)