当前位置：首页 > news >正文

Scrapy源码解析：DownloadHandlers设计与解析

news 2025/7/10 0:31:40

1、源码解析

代码路径：scrapy/core/downloader/__init__.py
详细代码解析，请看代码注释

"""Download handlers for different schemes"""import logging
from typing import TYPE_CHECKING, Any, Callable, Dict, Generator, Union, castfrom twisted.internet import defer
from twisted.internet.defer import Deferredfrom scrapy import Request, Spider, signals
from scrapy.exceptions import NotConfigured, NotSupported
from scrapy.utils.httpobj import urlparse_cached
from scrapy.utils.misc import create_instance, load_object
from scrapy.utils.python import without_none_valuesif TYPE_CHECKING:from scrapy.crawler import Crawlerlogger = logging.getLogger(__name__)class DownloadHandlers:def __init__(self, crawler: "Crawler"):self._crawler: "Crawler" = crawlerself._schemes: Dict[str, Union[str, Callable]] = {}                               # self._schemes 用于存放协议：协议处理器路径的字典，注意是协议处理器的路径，还没有实例化这个协议处理器。self._handlers: Dict[str, Any] = {}  # stores instanced handlers for schemes 用于存放各个协议的下载器的实例self._notconfigured: Dict[str, str] = {}  # remembers failed handlers  # 记录不在设置中设置的协议handlers: Dict[str, Union[str, Callable]] = without_none_values(crawler.settings.getwithbase("DOWNLOAD_HANDLERS") # 读取配置中，设置的什么协议用什么下载器)for scheme, clspath in handlers.items():self._schemes[scheme] = clspath           # 把协议：协议处理器存入self._schemesself._load_handler(scheme, skip_lazy=True)# 在这个循环里就加载了大部分协议下载器。只有s3没有被加载。在调用self._load_handler的时候# 这里传的skip_lazy为True，在self._load_handler内部需要与各个协议下载器中的lazy属性结合判断，最终决定要不要加载crawler.signals.connect(self._close, signals.engine_stopped)def _get_handler(self, scheme: str) -> Any: # 就是根据协议名，获取对应的下载器实例"""Lazy-load the downloadhandler for a schemeonly on the first request for that scheme."""if scheme in self._handlers:       # 如果已经取过对应协议的处理器了，就直接返回该处理器return self._handlers[scheme]if scheme in self._notconfigured:  # 如果是已知的未配置的协议，就直接返回Nonereturn Noneif scheme not in self._schemes:self._notconfigured[scheme] = "no handler available for that scheme" # 第一次出现一个未配置下载器的协议的时候，记录一下这个协议return Nonereturn self._load_handler(scheme)def _load_handler(self, scheme: str, skip_lazy: bool = False) -> Any: # 根据协议名，实例化下载器path = self._schemes[scheme]try:dhcls = load_object(path) # 加载下载器类if skip_lazy and getattr(dhcls, "lazy", True): # 判断要不要惰性加载return Nonedh = create_instance(   # 实例化协议下载器objcls=dhcls,settings=self._crawler.settings,crawler=self._crawler,)except NotConfigured as ex:self._notconfigured[scheme] = str(ex)return Noneexcept Exception as ex:logger.error('Loading "%(clspath)s" for scheme "%(scheme)s"',{"clspath": path, "scheme": scheme},exc_info=True,extra={"crawler": self._crawler},)self._notconfigured[scheme] = str(ex)return Noneelse:self._handlers[scheme] = dhreturn dhdef download_request(self, request: Request, spider: Spider) -> Deferred: # 从request中取出协议，然后取出下载器，下载scheme = urlparse_cached(request).schemehandler = self._get_handler(scheme)if not handler:raise NotSupported(f"Unsupported URL scheme '{scheme}': {self._notconfigured[scheme]}")return cast(Deferred, handler.download_request(request, spider))@defer.inlineCallbacksdef _close(self, *_a: Any, **_kw: Any) -> Generator[Deferred, Any, None]: # 当收到引擎的engine_stopped的信号的时候，挨个的停止各个协议的下载器for dh in self._handlers.values():if hasattr(dh, "close"):yield dh.close()