Python—scrapy redis超全源碼解析！

Scrapy-redis的源碼解析

Scrapy-redis的官方文檔寫的比較簡潔，沒有提及其運行原理，所以如果想全面的理解分佈式爬蟲的運行原理，還是得看scrapy-redis的源代碼纔行。

connection.py

負責根據setting中配置實例化redis連接。被dupefilter和scheduler調用，總之涉及到redis存取的都要使用到這個模塊。Connection提供了一個很重要的函數。

import six
from scrapy.utils.misc import load_object

from . import defaults

# Shortcut maps 'setting name' -> 'parmater name'.
# redis數據庫的關係映射
SETTINGS_PARAMS_MAP = {
    'REDIS_URL': 'url',
    'REDIS_HOST': 'host',
    'REDIS_PORT': 'port',
    'REDIS_ENCODING': 'encoding',
}


def get_redis_from_settings(settings):
    # 獲取一個redis連接實例
    # 生成連接redis參數
    """Returns a redis client instance from given Scrapy settings object.

    This function uses ``get_client`` to instantiate the client and uses
    ``defaults.REDIS_PARAMS`` global as defaults values for the parameters. You
    can override them using the ``REDIS_PARAMS`` setting.

    Parameters
    ----------
    settings : Settings
        A scrapy settings object. See the supported settings below.

    Returns
    -------
    server
        Redis client instance.

    Other Parameters
    ----------------
    REDIS_URL : str, optional
        Server connection URL.
    REDIS_HOST : str, optional
        Server host.
    REDIS_PORT : str, optional
        Server port.
    REDIS_ENCODING : str, optional
        Data encoding.
    REDIS_PARAMS : dict, optional
        Additional client parameters.

    """
    # 淺拷貝，是爲了防止params改變，會導致默認的REDIS_PARAMS被改變
    params = defaults.REDIS_PARAMS.copy()
    # 將settings中的參數更新到params
    params.update(settings.getdict('REDIS_PARAMS'))
    # XXX: Deprecate REDIS_* settings.
    # 遍歷映射表，獲取指定的參數
    for source, dest in SETTINGS_PARAMS_MAP.items():
        # 優先使用settings中的參數
        val = settings.get(source)
        # 如果settings中沒有進行設置，則params不更新
        if val:
            params[dest] = val

    # Allow ``redis_cls`` to be a path to a class.
    if isinstance(params.get('redis_cls'), six.string_types):
        params['redis_cls'] = load_object(params['redis_cls'])

    return get_redis(**params)


# Backwards compatible alias.
from_settings = get_redis_from_settings


def get_redis(**kwargs):
    """Returns a redis client instance.

    Parameters
    ----------
    redis_cls : class, optional
        Defaults to ``redis.StrictRedis``.
    url : str, optional
        If given, ``redis_cls.from_url`` is used to instantiate the class.
    **kwargs
        Extra parameters to be passed to the ``redis_cls`` class.

    Returns
    -------
    server
        Redis client instance.

    """
    # 沒有redis_cli，則默認redis連接
    redis_cls = kwargs.pop('redis_cls', defaults.REDIS_CLS)
    url = kwargs.pop('url', None) # 判斷kwargs有沒有url
    if url:
        #用url鏈接redis，優先使用url連接redis
        return redis_cls.from_url(url, **kwargs)
    else:
        #用字典的方式連接redis
        return redis_cls(**kwargs)

defaults.py

import redis


# For standalone use.
# 去重的鍵名
DUPEFILTER_KEY = 'dupefilter:%(timestamp)s'
# 定義的存儲items的鍵名（key），spider是爬蟲的名稱
PIPELINE_KEY = '%(spider)s:items'
# Redis的連接對象，用於連接redis
REDIS_CLS = redis.StrictRedis
# 字符集編碼
REDIS_ENCODING = 'utf-8'
# Sane connection defaults.
# redis數據庫的連接參數
REDIS_PARAMS = {
    'socket_timeout': 30,
    'socket_connect_timeout': 30,
    'retry_on_timeout': True,
    'encoding': REDIS_ENCODING,
}
# 隊列的變量名，用於存儲爬取的url隊列
SCHEDULER_QUEUE_KEY = '%(spider)s:requests'
# 優先級隊列，用於規定隊列的進出方式
SCHEDULER_QUEUE_CLASS = 'scrapy_redis.queue.PriorityQueue'
# 用於去重的key值，給request加指紋存儲的地方
SCHEDULER_DUPEFILTER_KEY = '%(spider)s:dupefilter'
# 用於生成指紋的類
SCHEDULER_DUPEFILTER_CLASS = 'scrapy_redis.dupefilter.RFPDupeFilter'
#起始url對應的類（key）
START_URLS_KEY = '%(name)s:start_urls'
#起始url的類型
START_URLS_AS_SET = False

dupefilter.py

分佈式爬蟲url去重原理： 　　通過分析可以知道self.server爲redis實例，使用一個key來向redis的一個set中插入fingerprint（這個key對於同一個spider是相同的，redis是一個key-value的數據庫，如果key是相同的，訪問到的值就是相同的，默認使用spider名字 + fingerpoint的key就是爲了區分在不同主機上的不同spider實例，只要數據是同一個spider，就會訪問到redis中的同一個spider-set而這個set就是url的判重池）。

import logging
import time

from scrapy.dupefilters import BaseDupeFilter
from scrapy.utils.request import request_fingerprint

from . import defaults
from .connection import get_redis_from_settings

logger = logging.getLogger(__name__)

# scrapy去重是利用集合實現的
# TODO: Rename class to RedisDupeFilter.
class RFPDupeFilter(BaseDupeFilter):
    """Redis-based request duplicates filter.

    This class can also be used with default Scrapy's scheduler.

    """

    logger = logger

    def __init__(self, server, key, debug=False):
        """Initialize the duplicates filter.

        Parameters
        ----------
        server : redis.StrictRedis
            The redis server instance.
            redis 連接實例

        key : str  存儲requests指紋的地方
            Redis key Where to store fingerprints.
        debug : bool, optional
            Whether to log filtered requests.
            是否記錄過濾的requests

        """
        #看server是如何生成的，因爲我們通過server就可以獲取redis中的隊列或者set
        self.server = server
        self.key = key
        self.debug = debug
        self.logdupes = True
    # 類方法傳遞當前的方法
    @classmethod
    def from_settings(cls, settings):
        """Returns an instance from given settings.

        This uses by default the key ``dupefilter:<timestamp>``. When using the
        ``scrapy_redis.scheduler.Scheduler`` class, this method is not used as
        it needs to pass the spider name in the key.

        Parameters
        ----------
        settings : scrapy.settings.Settings

        Returns
        -------
        RFPDupeFilter
            A RFPDupeFilter instance.


        """
        # 獲取redis的連接實例
        server = get_redis_from_settings(settings)
        # XXX: This creates one-time key. needed to support to use this
        # class as standalone dupefilter with scrapy's default scheduler
        # if scrapy passes spider on open() method this wouldn't be needed
        # TODO: Use SCRAPY_JOB env as default and fallback to timestamp.
        # 存取指紋的key
        key = defaults.DUPEFILTER_KEY % {'timestamp': int(time.time())}
        debug = settings.getbool('DUPEFILTER_DEBUG') # 默認值是false
        # 傳給當前類，並把參數傳遞給init函數
        return cls(server, key=key, debug=debug)

    @classmethod
    def from_crawler(cls, crawler):
        """Returns instance from crawler.

        Parameters
        ----------
        crawler : scrapy.crawler.Crawler

        Returns
        -------
        RFPDupeFilter
            Instance of RFPDupeFilter.

        """
        return cls.from_settings(crawler.settings)

    def request_seen(self, request):
        """Returns True if request was already seen.

        Parameters
        ----------
        request : scrapy.http.Request

        Returns
        -------
        bool

        """
        fp = self.request_fingerprint(request) # 生成一個指紋
        # This returns the number of values added, zero if already exists.
        # 將 指紋加入redis  是一個集合類型
        # self.server redis連接實例
        # self.key 存儲指紋的key
        # fp  就是指紋
        added = self.server.sadd(self.key, fp)
        # 當added爲0，說明指紋已經存在，返回True，否則返回False
        return added == 0

    def request_fingerprint(self, request):
        """Returns a fingerprint for a given request.

        Parameters
        ----------
        request : scrapy.http.Request

        Returns
        -------
        str

        """
        return request_fingerprint(request)

    @classmethod
    def from_spider(cls, spider):
        settings = spider.settings
        server = get_redis_from_settings(settings)
        dupefilter_key = settings.get("SCHEDULER_DUPEFILTER_KEY", defaults.SCHEDULER_DUPEFILTER_KEY)
        key = dupefilter_key % {'spider': spider.name}
        debug = settings.getbool('DUPEFILTER_DEBUG')
        return cls(server, key=key, debug=debug)

    def close(self, reason=''):
        # 當爬蟲結束時，清空指紋的地方
        """Delete data on close. Called by Scrapy's scheduler.

        Parameters
        ----------
        reason : str, optional

        """
        self.clear()

    def clear(self):
        """Clears fingerprints data."""
        self.server.delete(self.key)

    # 生成日誌的地方
    def log(self, request, spider):
        """Logs given request.

        Parameters
        ----------
        request : scrapy.http.Request
        spider : scrapy.spiders.Spider

        """
        if self.debug:
            msg = "Filtered duplicate request: %(request)s"
            self.logger.debug(msg, {'request': request}, extra={'spider': spider})
        elif self.logdupes:
            msg = ("Filtered duplicate request %(request)s"
                   " - no more duplicates will be shown"
                   " (see DUPEFILTER_DEBUG to show all duplicates)")
            self.logger.debug(msg, {'request': request}, extra={'spider': spider})
            self.logdupes = False

picklecompat.py

這裏實現了loads和dumps兩個函數，其實就是實現了一個serializer： 　　1、因爲redis數據庫不能存儲複雜對象（value部分只能是字符串，字符串列表，字符串集合和hash，key部分只能是字符串），所以我們存啥都要先串行化成文本纔行。這裏使用的就是python的pickle模塊，一個兼容py2和py3的串行化工具。

"""A pickle wrapper module with protocol=-1 by default."""

try:
    import cPickle as pickle  # PY2
except ImportError:
    import pickle #PY3用的包

#反序列化就是將字符串數據轉化成json數據
def loads(s):
    return pickle.loads(s)

#序列化 就是將json數據轉化成字符串
def dumps(obj):
    return pickle.dumps(obj, protocol=-1)

pipelines.py

pipelines.py中類的作用： 　　pipeline.py文件用來實現數據分佈式處理。它通過從settings中拿到我們配置的REDIS_ITEMS_KEY作爲key，把item串行化之後存入redis數據庫對應的value中（這個value可以看出是個list，我們的每個item是這個list中的一個結點），這個pipeline把提取出的item存起來，主要是爲了方便我們延後處理數據。

from scrapy.utils.misc import load_object
from scrapy.utils.serialize import ScrapyJSONEncoder
from twisted.internet.threads import deferToThread

from . import connection, defaults

# 序列化的字符串
default_serialize = ScrapyJSONEncoder().encode

# 用於處理爬蟲爬取的數據 將數據序列化到redis中
class RedisPipeline(object):
    """Pushes serialized item into a redis list/queue

    Settings
    --------
    REDIS_ITEMS_KEY : str
        Redis key where to store items.
    REDIS_ITEMS_SERIALIZER : str
        Object path to serializer function.

    """

    def __init__(self, server,
                 key=defaults.PIPELINE_KEY,
                 serialize_func=default_serialize):
        """Initialize pipeline.

        Parameters
        ----------
        server : StrictRedis
            Redis client instance.
        key : str
            Redis key where to store items.
        serialize_func : callable
            Items serializer function.

        """
        self.server = server
        self.key = key
        self.serialize = serialize_func
    # 將類本身傳入函數，用來生成參數和redis連接實例
    @classmethod
    def from_settings(cls, settings):
        # 生成redis連接實例
        params = {
            'server': connection.from_settings(settings),
        }
        # 如果設置中有REDIS_ITEMS_KEY，我們就用設置中
        if settings.get('REDIS_ITEMS_KEY'):
            params['key'] = settings['REDIS_ITEMS_KEY']
        # 如果設置中有序列化的函數，則優先使用設置中的
        if settings.get('REDIS_ITEMS_SERIALIZER'):
            params['serialize_func'] = load_object(
                settings['REDIS_ITEMS_SERIALIZER']
            )

        return cls(**params)

    @classmethod
    def from_crawler(cls, crawler):
        return cls.from_settings(crawler.settings)
    # 將item傳遞過來，自動觸發這個函數
    def process_item(self, item, spider):
        # 創建一個線程，用於存儲item，也就是說上一個iten還沒有存儲完，下一個item就可以同時存儲
        return deferToThread(self._process_item, item, spider)
    # 實現存儲的函數
    def _process_item(self, item, spider):
        key = self.item_key(item, spider) # 生成item_key
        data = self.serialize(item)  # 使用默認的序列化函數，將item序列化爲字符串
        self.server.rpush(key, data) # 把數據放到redis裏面----self.server是redis的連接實例
        return item
    # 用於存儲item
    def item_key(self, item, spider):
        """Returns redis key based on given spider.

        Override this function to use a different key depending on the item
        and/or spider.

        """
        #根據spider的name生成一個redis_key
        return self.key % {'spider': spider.name}

queue.py

這是個隊列類，它會作爲scheduler調度request的容器來維護一個秩序： 　　1、 scheduler在每個主機上都會實例化一個，並且和spider一一對應，所以分佈式運行時會有一個spider的多個實例和一個scheduler的多個實例存在於不同的主機上。 　　2、因爲scheduler都是用相同的容器，而這些容器都連接同一個 redis服務器，又都使用spider名 + queue來作爲key 讀寫數據，所以不同主機上的不同爬蟲實例公用一個request調度池，實現了分佈式爬蟲之間的統一調度。

from scrapy.utils.reqser import request_to_dict, request_from_dict

from . import picklecompat


class Base(object):
    """Per-spider base queue class"""

    def __init__(self, server, spider, key, serializer=None):
        """Initialize per-spider redis queue.

        Parameters
        ----------
        server : StrictRedis
            Redis client instance.
        spider : Spider
            Scrapy spider instance.
        key: str
            Redis key where to put and get messages.
        serializer : object
            Serializer object with ``loads`` and ``dumps`` methods.

        """
        if serializer is None:
            # Backward compatibility.
            # TODO: deprecate pickle.
            serializer = picklecompat
        # 當序列化時沒有laods函數時，就會拋出異常
        # 拋出異常的目的就是爲了使傳過來的序列化必須函數loads函數
        if not hasattr(serializer, 'loads'):
            raise TypeError("serializer does not implement 'loads' function: %r"
                            % serializer)
        # 當序列化時沒有dumps函數時，就會拋出異常
        if not hasattr(serializer, 'dumps'):
            raise TypeError("serializer '%s' does not implement 'dumps' function: %r"
                            % serializer)
        # 下面的這些函數當類的所有函數，都可以使用
        self.server = server
        self.spider = spider
        self.key = key % {'spider': spider.name}
        self.serializer = serializer
    # 將requests進行編碼成字符串
    def _encode_request(self, request):
        """Encode a request object"""
        # 將requests轉化成字典
        obj = request_to_dict(request, self.spider)
        # 將字典轉化爲字符串並返回
        return self.serializer.dumps(obj)
    # 將已經編碼的ncode_request解碼爲字典
    def _decode_request(self, encoded_request):
        """Decode an request previously encoded"""
        # 將dict轉換爲requests object取出，直接通過下載器進行下載
        obj = self.serializer.loads(encoded_request)
        return request_from_dict(obj, self.spider)
    # 下面的len方法 push方法 pop方法 必須重載 否則不能使用
    def __len__(self):
        """Return the length of the queue"""
        raise NotImplementedError

    def push(self, request):
        """Push a request"""
        raise NotImplementedError

    def pop(self, timeout=0):
        """Pop a request"""
        raise NotImplementedError
    # 刪除指定的self.key的值
    def clear(self):
        """Clear queue/stack"""
        self.server.delete(self.key)

# 先進先出 針對有序隊列
class FifoQueue(Base):
    """Per-spider FIFO queue"""
    # 返回隊列長度
    def __len__(self):
        """Return the length of the queue"""
        return self.server.llen(self.key)
    # 從頭部插入request
    def push(self, request):
        """Push a request"""
        self.server.lpush(self.key, self._encode_request(request))

    def pop(self, timeout=0):
        """Pop a request"""
        # timeout超時，一般默認爲0
        if timeout > 0:
            #pop出來的時候是隊尾彈出
            data = self.server.brpop(self.key, timeout)
            if isinstance(data, tuple):
                data = data[1]
        else:
            # 這個是從尾部刪除
            data = self.server.rpop(self.key)
        if data:
            # 彈出元素再解碼爲request直接給下載器進行下載
            return self._decode_request(data)

# 優先級隊列 每次放出的打一個分數，對於有序集合，彈出的時候優先彈出
class PriorityQueue(Base):
    """Per-spider priority queue abstraction using redis' sorted set"""

    def __len__(self):
        """Return the length of the queue"""
        return self.server.zcard(self.key)

    def push(self, request):
        """Push a request"""
        data = self._encode_request(request)
        score = -request.priority
        # We don't use zadd method as the order of arguments change depending on
        # whether the class is Redis or StrictRedis, and the option of using
        # kwargs only accepts strings, not bytes.
        # 使用有序集合實現優先級隊列
        self.server.execute_command('ZADD', self.key, score, data)

    def pop(self, timeout=0):
        """
        Pop a request
        timeout not support in this queue class
        """
        # use atomic range/remove using multi/exec
        # pipeline其實就是self.server的一個方法
        # pipe相當於實例化的函數
        pipe = self.server.pipeline()
        pipe.multi()
        # zrange是從小到大排序後返回第一個值
        # zremrangebyrank是刪除第一個request
        pipe.zrange(self.key, 0, 0).zremrangebyrank(self.key, 0, 0)
        # 執行上面的語句，刪除的同時返回被刪除的數據
        # results接收的是第一條數據
        # count 刪除的元素，返回值是1或0
        results, count = pipe.execute()
        if results: # 只要有一個元素results是真值
            # 將獲取的第一個元素（返回的是一個列表），拿出來，進行解碼
            return self._decode_request(results[0])

# 後進先出
class LifoQueue(Base):
    """Per-spider LIFO queue."""

    def __len__(self):
        """Return the length of the stack"""
        return self.server.llen(self.key)

    def push(self, request):
        """Push a request"""
        self.server.lpush(self.key, self._encode_request(request))

    def pop(self, timeout=0):
        """Pop a request"""
        if timeout > 0:
            data = self.server.blpop(self.key, timeout)
            if isinstance(data, tuple):
                data = data[1]
        else:
            data = self.server.lpop(self.key)

        if data:
            return self._decode_request(data)


# TODO: Deprecate the use of these names.
SpiderQueue = FifoQueue
SpiderStack = LifoQueue
SpiderPriorityQueue = PriorityQueue

scheduler.py

這個文件重寫了scheduler類，用來代替scrapy.core.scheduler的原有調度器。實現原理是使用指定的一個redis內存作爲數據存儲的媒介，以達到各個爬蟲之間的統一調度。 　　1、scheduler負責調度各個spider的request請求，scheduler初始化時，通過settings文件讀取queue和dupefilters（url去重）的類型，配置queue和dupefilters使用的key（一般就是spider name加上queue或者dupefilters，這樣對於同一種spider的不同實例，就會使用相同的數據塊了）。 　　2、每當一個request要被調度時，enqueue_request被調用，scheduler使用dupefilters來判斷這個url是否重複，如果不重複，就添加到queue的容器中（三種隊列方式：先進先出，先進後出和優先級都可以，可以在settings中配置）。 　　3、當調度完成時，next_request被調用，scheduler就通過queue容器的接口，取出一個request，把他發送給相應的spider，讓spider進行爬取工作。

import importlib

import six
from scrapy.utils.misc import load_object

from . import connection, defaults


# TODO: add SCRAPY_JOB support.
# FIXME
class Scheduler(object):
    """Redis-based scheduler

    Settings
    --------
    SCHEDULER_PERSIST : bool (default: False)
        Whether to persist or clear redis queue.
    SCHEDULER_FLUSH_ON_START : bool (default: False)
        Whether to flush redis queue on start.
    SCHEDULER_IDLE_BEFORE_CLOSE : int (default: 0)
        How many seconds to wait before closing if no message is received.
    SCHEDULER_QUEUE_KEY : str
        Scheduler redis key.
    SCHEDULER_QUEUE_CLASS : str
        Scheduler queue class.
    SCHEDULER_DUPEFILTER_KEY : str
        Scheduler dupefilter redis key.
    SCHEDULER_DUPEFILTER_CLASS : str
        Scheduler dupefilter class.
    SCHEDULER_SERIALIZER : str
        Scheduler serializer.

    """

    def __init__(self, server,
                 persist=False,
                 flush_on_start=False,
                 queue_key=defaults.SCHEDULER_QUEUE_KEY,
                 queue_cls=defaults.SCHEDULER_QUEUE_CLASS,
                 dupefilter_key=defaults.SCHEDULER_DUPEFILTER_KEY,
                 dupefilter_cls=defaults.SCHEDULER_DUPEFILTER_CLASS,
                 idle_before_close=0,
                 serializer=None):
        """Initialize scheduler.

        Parameters
        ----------
        server : Redis
            The redis server instance.
        persist : bool
            Whether to flush requests when closing. Default is False.
        flush_on_start : bool
            Whether to flush requests on start. Default is False.
        queue_key : str
            Requests queue key.
        queue_cls : str
            Importable path to the queue class.
        dupefilter_key : str
            Duplicates filter key.
        dupefilter_cls : str
            Importable path to the dupefilter class.
        idle_before_close : int
            Timeout before giving up.

        """
        if idle_before_close < 0:
            raise TypeError("idle_before_close cannot be negative")

        self.server = server
        self.persist = persist
        self.flush_on_start = flush_on_start
        self.queue_key = queue_key
        self.queue_cls = queue_cls
        self.dupefilter_cls = dupefilter_cls
        self.dupefilter_key = dupefilter_key
        self.idle_before_close = idle_before_close
        self.serializer = serializer
        self.stats = None

    def __len__(self):
        return len(self.queue)

    @classmethod
    def from_settings(cls, settings): #作爲入口
        kwargs = {
            #是否將隊列持久化
            'persist': settings.getbool('SCHEDULER_PERSIST'),
            #是否將隊列中的數據清空
            'flush_on_start': settings.getbool('SCHEDULER_FLUSH_ON_START'),
            'idle_before_close': settings.getint('SCHEDULER_IDLE_BEFORE_CLOSE'),
        }

        # If these values are missing, it means we want to use the defaults.
        optional = {
            # TODO: Use custom prefixes for this settings to note that are
            # specific to scrapy-redis.
            'queue_key': 'SCHEDULER_QUEUE_KEY',
            'queue_cls': 'SCHEDULER_QUEUE_CLASS', #默認作爲優先隊列
            'dupefilter_key': 'SCHEDULER_DUPEFILTER_KEY', #去重
            # We use the default setting name to keep compatibility.
            'dupefilter_cls': 'DUPEFILTER_CLASS',
            'serializer': 'SCHEDULER_SERIALIZER',
        }
        for name, setting_name in optional.items():
            val = settings.get(setting_name)
            if val:
                kwargs[name] = val

        # Support serializer as a path to a module.
        if isinstance(kwargs.get('serializer'), six.string_types):
            kwargs['serializer'] = importlib.import_module(kwargs['serializer'])
        # redis的連接實例
        server = connection.from_settings(settings)
        # 驗證
        # Ensure the connection is working.
        server.ping()

        return cls(server=server, **kwargs)

    @classmethod
    def from_crawler(cls, crawler):
        instance = cls.from_settings(crawler.settings)
        # FIXME: for now, stats are only supported from this constructor
        instance.stats = crawler.stats
        return instance

    def open(self, spider):
        self.spider = spider

        try:
            self.queue = load_object(self.queue_cls)(
                server=self.server,
                spider=spider,
                key=self.queue_key % {'spider': spider.name},
                serializer=self.serializer,
            )
        except TypeError as e:
            raise ValueError("Failed to instantiate queue class '%s': %s",
                             self.queue_cls, e)

        self.df = load_object(self.dupefilter_cls).from_spider(spider)

        if self.flush_on_start:
            self.flush()
        # notice if there are requests already in the queue to resume the crawl
        if len(self.queue):
            spider.log("Resuming crawl (%d requests scheduled)" % len(self.queue))

    def close(self, reason):
        if not self.persist:
            self.flush()

    def flush(self):
        self.df.clear()
        self.queue.clear()
    # 入隊函數
    def enqueue_request(self, request):
        # self.df.request_seen(request) 返回的bool值，返回True代表request存在
        # not request.dont_filter 默認返回是true
        # 當我們選擇是過濾而且request 已經進入隊列，我們返回一個false
        if not request.dont_filter and self.df.request_seen(request):
            self.df.log(request, self.spider)
            return False
        # 一般用不到，默認是none
        if self.stats:
            self.stats.inc_value('scheduler/enqueued/redis', spider=self.spider)
        self.queue.push(request)
        return True
    # 出隊函數
    def next_request(self):
        block_pop_timeout = self.idle_before_close
        # 彈出一條數據
        request = self.queue.pop(block_pop_timeout)
        if request and self.stats:
            self.stats.inc_value('scheduler/dequeued/redis', spider=self.spider)
        # 返回request給引擎，引擎給下載器，進行下載網頁
        return request

    def has_pending_requests(self):
        return len(self) > 0

spiders.py

spider.py文件是分佈式爬蟲的入口代碼： 　　1、通過connection接口，spider初始化時，通過setup_redis()函數初始化好和redis的連接。 　　2、通過next_requests函數從redis中取出strat url，spider使用少量的start url + LinkExtractor，可以發展出很多新的url，這些url會進入scheduler進行判重和調度。直到spider跑到調度池內沒有url的時候，會觸發spider_idle信號，從而觸發spider的next_requests函數。 　　3、再次從redis的start url池中讀取一些url。

from scrapy import signals
from scrapy.exceptions import DontCloseSpider
from scrapy.spiders import Spider, CrawlSpider

from . import connection, defaults
from .utils import bytes_to_str


class RedisMixin(object):
    """Mixin class to implement reading urls from a redis queue."""
    redis_key = None # 在redis裏起始url對應的key
    redis_batch_size = None # 容量
    redis_encoding = None # 字符集編碼

    # Redis client placeholder.
    server = None
    #重寫start_request方法調用next_requests
    def start_requests(self):
        """Returns a batch of start requests from redis."""
        return self.next_requests()

    def setup_redis(self, crawler=None):
        """Setup redis connection and idle signal.

        This should be called after the spider has set its crawler object.
        """
        if self.server is not None:
            return

        if crawler is None:
            # We allow optional crawler argument to keep backwards
            # compatibility.
            # XXX: Raise a deprecation warning.
            crawler = getattr(self, 'crawler', None)

        if crawler is None:
            raise ValueError("crawler is required")

        settings = crawler.settings

        if self.redis_key is None:
            self.redis_key = settings.get(
                'REDIS_START_URLS_KEY', defaults.START_URLS_KEY,
            )

        self.redis_key = self.redis_key % {'name': self.name}

        if not self.redis_key.strip():
            raise ValueError("redis_key must not be empty")

        if self.redis_batch_size is None:
            # TODO: Deprecate this setting (REDIS_START_URLS_BATCH_SIZE).
            self.redis_batch_size = settings.getint(
                'REDIS_START_URLS_BATCH_SIZE',
                settings.getint('CONCURRENT_REQUESTS'),
            )

        try:
            self.redis_batch_size = int(self.redis_batch_size)
        except (TypeError, ValueError):
            raise ValueError("redis_batch_size must be an integer")

        if self.redis_encoding is None:
            self.redis_encoding = settings.get('REDIS_ENCODING', defaults.REDIS_ENCODING)

        self.logger.info("Reading start URLs from redis key '%(redis_key)s' "
                         "(batch size: %(redis_batch_size)s, encoding: %(redis_encoding)s",
                         self.__dict__)
        #redis連接實例
        self.server = connection.from_settings(crawler.settings)
        # The idle signal is called when the spider has no requests left,
        # that's when we will schedule new requests from redis queue
        crawler.signals.connect(self.spider_idle, signal=signals.spider_idle)

    def next_requests(self):
        """Returns a request to be scheduled or none."""
        # 默認使用redis_keys是一個列表，否則是集合
        use_set = self.settings.getbool('REDIS_START_URLS_AS_SET', defaults.START_URLS_AS_SET)
        #從redis數據庫裏取出起始url
        #use_set=false 返回self.server.lpop(列表數據類型）
        #use_set=true 返回self.server.spop(集合類型)
        fetch_one = self.server.spop if use_set else self.server.lpop
        # XXX: Do we need to use a timeout here?
        found = 0
        # TODO: Use redis pipeline execution.
        while found < self.redis_batch_size:
            #從數據庫中取出起始url數據，返回一個列表
            data = fetch_one(self.redis_key)
            if not data:
                # Queue empty.
                break
            #取出的url是一個bytes類型，需要轉換爲str兼容python3
            req = self.make_request_from_data(data)
            if req:
                yield req # 把req給Request
                found += 1
            else:
                self.logger.debug("Request not made from data: %r", data)

        if found:
            self.logger.debug("Read %s requests from '%s'", found, self.redis_key)
    #返回的是requestss實例，是通過來自redis的data數據
    def make_request_from_data(self, data):
        """Returns a Request instance from data coming from Redis.

        By default, ``data`` is an encoded URL. You can override this method to
        provide your own message decoding.

        Parameters
        ----------
        data : bytes
            Message from redis.

        """
        url = bytes_to_str(data, self.redis_encoding)
        return self.make_requests_from_url(url)

    def schedule_next_requests(self):
        """Schedules a request if available"""
        # TODO: While there is capacity, schedule a batch of redis requests.
        for req in self.next_requests():
            self.crawler.engine.crawl(req, spider=self)

    def spider_idle(self):
        """Schedules a request if available, otherwise waits."""
        # XXX: Handle a sentinel to close the spider.
        self.schedule_next_requests()
        raise DontCloseSpider


class RedisSpider(RedisMixin, Spider):
    """Spider that reads urls from redis queue when idle.

    Attributes
    ----------
    redis_key : str (default: REDIS_START_URLS_KEY)
        Redis key where to fetch start URLs from..
    redis_batch_size : int (default: CONCURRENT_REQUESTS)
        Number of messages to fetch from redis on each attempt.
    redis_encoding : str (default: REDIS_ENCODING)
        Encoding to use when decoding messages from redis queue.

    Settings
    --------
    REDIS_START_URLS_KEY : str (default: "<spider.name>:start_urls")
        Default Redis key where to fetch start URLs from..
    REDIS_START_URLS_BATCH_SIZE : int (deprecated by CONCURRENT_REQUESTS)
        Default number of messages to fetch from redis on each attempt.
    REDIS_START_URLS_AS_SET : bool (default: False)
        Use SET operations to retrieve messages from the redis queue. If False,
        the messages are retrieve using the LPOP command.
    REDIS_ENCODING : str (default: "utf-8")
        Default encoding to use when decoding messages from redis queue.

    """

    @classmethod
    def from_crawler(self, crawler, *args, **kwargs):
        obj = super(RedisSpider, self).from_crawler(crawler, *args, **kwargs)
        obj.setup_redis(crawler)
        return obj


class RedisCrawlSpider(RedisMixin, CrawlSpider):
    """Spider that reads urls from redis queue when idle.

    Attributes
    ----------
    redis_key : str (default: REDIS_START_URLS_KEY)
        Redis key where to fetch start URLs from..
    redis_batch_size : int (default: CONCURRENT_REQUESTS)
        Number of messages to fetch from redis on each attempt.
    redis_encoding : str (default: REDIS_ENCODING)
        Encoding to use when decoding messages from redis queue.

    Settings
    --------
    REDIS_START_URLS_KEY : str (default: "<spider.name>:start_urls")
        Default Redis key where to fetch start URLs from..
    REDIS_START_URLS_BATCH_SIZE : int (deprecated by CONCURRENT_REQUESTS)
        Default number of messages to fetch from redis on each attempt.
    REDIS_START_URLS_AS_SET : bool (default: True)
        Use SET operations to retrieve messages from the redis queue.
    REDIS_ENCODING : str (default: "utf-8")
        Default encoding to use when decoding messages from redis queue.

    """

    @classmethod
    def from_crawler(self, crawler, *args, **kwargs):
        obj = super(RedisCrawlSpider, self).from_crawler(crawler, *args, **kwargs)
        obj.setup_redis(crawler)
        return obj

utils.py

import six


def bytes_to_str(s, encoding='utf-8'):
    """Returns a str if a bytes object is given."""
    #將我們的bytes類型轉化爲字符串
    if six.PY3 and isinstance(s, bytes):
        return s.decode(encoding)
    return s

**最後總結一下scrapy-redis的總體思路** 這個工程通過重寫scheduler和spider類，實現了scheduler調度、spider啓動和固定redis的交互。 實現新的dupefilter和queue類，達到了去重和調度容器和redis的交互，因爲每個主機上的爬蟲進程都訪問同一個redis數據庫，所以調度和去重都統一進行統一管理，達到了分佈式爬蟲的目的。 當spider被初始化時，同時會初始化一個對應的scheduler對象，這個調度器對象通過讀取settings，配置好自己的調度容器queue和判重工具dupefilter。 每當一個spider產出一個request的時候，scrapy內核會把這個reuqest遞交給這個spider對應的scheduler對象進行調度，scheduler對象通過訪問redis對request進行判重，如果不重複就把他添加進redis中的調度池。 當調度條件滿足時，scheduler對象就從redis的調度池中取出一個request發送給spider，讓他爬取。 當spider爬取的所有暫時可用url之後，scheduler發現這個spider對應的redis的調度池空了，於是觸發信號spider_idle，spider收到這個信號之後，直接連接redis讀取strart url池，拿去新的一批url入口，然後再次重複上邊的工作。