将 Scrapy 项目改造为 scrapy-redis

修改文件

setting.py
SCHEDULER = "scrapy_redis.scheduler.Scheduler"
DUPEFILTER_CLASS = "scrapy_redis.dupefilter.RFPDupeFilter"
ITEM_PIPELINES = {
    'scrapy_redis.pipelines.RedisPipeline': 300
}
spider.py
from scrapy_redis.spiders import RedisSpider

class MySpider(RedisSpider):
    ...
    redis_key = "myspider:start_urls"
    #start_urls = [...]
    ...