下载器抽取出来

Boris-code · Boris-code · commit db26be234f1e · 2022-04-21T20:05:29.000+08:00
diff --git a/feapder/network/downloader/__init__.py b/feapder/network/downloader/__init__.py
@@ -1,8 +1,3 @@
-from feapder.network.response import Response
-import abc
-
-
-class Downloader:
-    @abc.abstractmethod
-    def download(self, method, url, **kwargs) -> Response:
-        raise NotImplementedError
+from .base import Downloader
+from ._requests import RequestsDownloader
+from ._requests import RequestsSessionDownloader
diff --git a/feapder/network/downloader/_requests.py b/feapder/network/downloader/_requests.py
@@ -8,6 +8,35 @@
 @email: boris_liu@foxmail.com
 """
 
+import requests
+from requests.adapters import HTTPAdapter
 
-class _Requests:
-    pass
+from feapder.network.downloader import Downloader
+from feapder.network.response import Response
+
+
+class RequestsDownloader(Downloader):
+    def download(self, method, url, **kwargs) -> Response:
+        response = requests.request(method, url, **kwargs)
+        response = Response(response)
+        return response
+
+
+class RequestsSessionDownloader(Downloader):
+    session = None
+
+    @property
+    def _session(self):
+        if not self.__class__.session:
+            self.__class__.session = requests.Session()
+            # pool_connections – 缓存的 urllib3 连接池个数  pool_maxsize – 连接池中保存的最大连接数
+            http_adapter = HTTPAdapter(pool_connections=1000, pool_maxsize=1000)
+            # 任何使用该session会话的 HTTP 请求，只要其 URL 是以给定的前缀开头，该传输适配器就会被使用到。
+            self.__class__.session.mount("http", http_adapter)
+
+        return self.__class__.session
+
+    def download(self, method, url, **kwargs) -> Response:
+        response = self._session.request(method, url, **kwargs)
+        response = Response(response)
+        return response
diff --git a/feapder/network/downloader/base.py b/feapder/network/downloader/base.py
@@ -0,0 +1,9 @@
+import abc
+
+from feapder.network.response import Response
+
+
+class Downloader:
+    @abc.abstractmethod
+    def download(self, method, url, **kwargs) -> Response:
+        raise NotImplementedError
diff --git a/feapder/network/request.py b/feapder/network/request.py
@@ -8,15 +8,17 @@
 @email:  boris_liu@foxmail.com
 """
 
+import importlib
+
 import requests
-from requests.adapters import HTTPAdapter
 from requests.cookies import RequestsCookieJar
 from requests.packages.urllib3.exceptions import InsecureRequestWarning
 
 import feapder.setting as setting
 import feapder.utils.tools as tools
 from feapder.db.redisdb import RedisDB
 from feapder.network import user_agent
+from feapder.network.downloader import Downloader
 from feapder.network.proxy_pool import ProxyPool
 from feapder.network.response import Response
 from feapder.utils.log import log
@@ -26,8 +28,13 @@
 requests.packages.urllib3.disable_warnings(InsecureRequestWarning)
 
 
+def import_cls(cls_info) -> Downloader:
+    module, class_name = cls_info.rsplit(".", 1)
+    cls = importlib.import_module(module).__getattribute__(class_name)
+    return cls()
+
+
 class Request(object):
-    session = None
     webdriver_pool: WebDriverPool = None
     user_agent_pool = user_agent
     proxies_pool: ProxyPool = None
@@ -36,8 +43,9 @@ class Request(object):
     cached_redis_key = None  # 缓存response的文件文件夹 response_cached:cached_redis_key:md5
     cached_expire_time = 1200  # 缓存过期时间
 
-    local_filepath = None
-    oss_handler = None
+    # 下载器
+    downloader = import_cls(setting.DOWNLOADER)
+    session_downloader = import_cls(setting.SESSION_DOWNLOADER)
 
     __REQUEST_ATTRS__ = {
         # 'method', 'url', 必须传递 不加入**kwargs中
@@ -175,20 +183,6 @@ def __setattr__(self, key, value):
     def __lt__(self, other):
         return self.priority < other.priority
 
-    @property
-    def _session(self):
-        use_session = (
-            setting.USE_SESSION if self.use_session is None else self.use_session
-        )  # self.use_session 优先级高
-        if use_session and not self.__class__.session:
-            self.__class__.session = requests.Session()
-            # pool_connections – 缓存的 urllib3 连接池个数  pool_maxsize – 连接池中保存的最大连接数
-            http_adapter = HTTPAdapter(pool_connections=1000, pool_maxsize=1000)
-            # 任何使用该session会话的 HTTP 请求，只要其 URL 是以给定的前缀开头，该传输适配器就会被使用到。
-            self.__class__.session.mount("http", http_adapter)
-
-        return self.__class__.session
-
     @property
     def _webdriver_pool(self):
         if not self.__class__.webdriver_pool:
@@ -392,11 +386,13 @@ def get_response(self, save_cached=False):
                 raise e
 
         elif use_session:
-            response = self._session.request(method, self.url, **self.requests_kwargs)
-            response = Response(response)
+            response = self.session_downloader.download(
+                method, self.url, **self.requests_kwargs
+            )
         else:
-            response = requests.request(method, self.url, **self.requests_kwargs)
-            response = Response(response)
+            response = self.downloader.download(
+                method, self.url, **self.requests_kwargs
+            )
 
         if save_cached:
             self.save_cached(response, expire_time=self.__class__.cached_expire_time)
diff --git a/feapder/setting.py b/feapder/setting.py
@@ -112,6 +112,10 @@
 # requests 使用session
 USE_SESSION = False
 
+# 下载
+DOWNLOADER = "feapder.network.downloader.RequestsDownloader"
+SESSION_DOWNLOADER = "feapder.network.downloader.RequestsSessionDownloader"
+
 # 去重
 ITEM_FILTER_ENABLE = False  # item 去重
 ITEM_FILTER_SETTING = dict(
diff --git a/tests/air-spider/test_air_spider.py b/tests/air-spider/test_air_spider.py
@@ -12,9 +12,9 @@
 
 
 class TestAirSpider(feapder.AirSpider):
-    # __custom_setting__ = dict(
-    #     LOG_LEVEL = "INFO"
-    # )
+    __custom_setting__ = dict(
+        USE_SESSION = True
+    )
 
     def start_callback(self):
         print("爬虫开始")
