version 1.8.6b1, support use stop_spider method to stop spider

Boris-code · Boris-code · commit ae507ed2e4b2 · 2023-06-28T17:14:32.000+08:00
diff --git a/feapder/VERSION b/feapder/VERSION
@@ -1 +1 @@
-1.8.5
+1.8.6-beta1
diff --git a/feapder/core/scheduler.py b/feapder/core/scheduler.py
@@ -155,6 +155,8 @@ def __init__(
         # 重置丢失的任务
         self.reset_task()
 
+        self._stop = False
+
     def init_metrics(self):
         """
         初始化打点系统
@@ -176,7 +178,7 @@ def run(self):
 
         while True:
             try:
-                if self.all_thread_is_done():
+                if self._stop or self.all_thread_is_done():
                     if not self._is_notify_end:
                         self.spider_end()  # 跑完一轮
                         self._is_notify_end = True
@@ -586,3 +588,6 @@ def reset_task(self, heartbeat_interval=10):
             lose_count = len(datas)
             if lose_count:
                 log.info("重置丢失任务完毕，共{}条".format(len(datas)))
+
+    def stop_spider(self):
+        self._stop = True
diff --git a/feapder/core/spiders/air_spider.py b/feapder/core/spiders/air_spider.py
@@ -46,6 +46,7 @@ def __init__(self, thread_count=None):
             db=self._memory_db, dedup_name=self.name
         )
 
+        self._stop = False
         metrics.init(**setting.METRICS_OTHER_ARGS)
 
     def distribute_task(self):
@@ -97,7 +98,7 @@ def run(self):
 
         while True:
             try:
-                if self.all_thread_is_done():
+                if self._stop or self.all_thread_is_done():
                     # 停止 parser_controls
                     for parser_control in self._parser_controls:
                         parser_control.stop()
@@ -108,7 +109,10 @@ def run(self):
                     # 关闭webdirver
                     Request.render_downloader and Request.render_downloader.close_all()
 
-                    log.info("无任务，爬虫结束")
+                    if self._stop:
+                        log.info("爬虫被停止")
+                    else:
+                        log.info("无任务，爬虫结束")
                     break
 
             except Exception as e:
@@ -130,3 +134,6 @@ def join(self, timeout=None):
             return
 
         super().join()
+
+    def stop_spider(self):
+        self._stop = True
diff --git a/feapder/core/spiders/batch_spider.py b/feapder/core/spiders/batch_spider.py
@@ -1002,7 +1002,7 @@ def run(self):
 
             while True:
                 try:
-                    if (
+                    if self._stop or (
                         self.task_is_done() and self.all_thread_is_done()
                     ):  # redis全部的任务已经做完 并且mysql中的任务已经做完（检查各个线程all_thread_is_done，防止任务没做完，就更新任务状态，导致程序结束的情况）
                         if not self._is_notify_end:
diff --git a/feapder/core/spiders/spider.py b/feapder/core/spiders/spider.py
@@ -184,7 +184,7 @@ def run(self):
 
         while True:
             try:
-                if self.all_thread_is_done():
+                if self._stop or self.all_thread_is_done():
                     if not self._is_notify_end:
                         self.spider_end()  # 跑完一轮
                         self._is_notify_end = True
diff --git a/feapder/core/spiders/task_spider.py b/feapder/core/spiders/task_spider.py
@@ -516,7 +516,7 @@ def run(self):
 
             while True:
                 try:
-                    if (
+                    if self._stop or (
                         self.all_thread_is_done()
                         and self.task_is_done()
                         and self.related_spider_is_done()
