tkit_scrapy_mongo
use mongodb instead of redis and rewrite scrapy_redis py3 module
install
pip install tkit-scrapy-mongo
use
# settings.py or custom_settings
# 这里覆盖默认的settings配置,custom_settings settings.py
custom_settings = dict(
#MONGODB_URI="mongodb://192.168.1.18:27017/",
#REDIS_URL = 'redis://192.168.1.18:6379', # 指定redis的地址
#POOL_PROXY = "http://192.168.1.18:5010"
CONCURRENT_REQUESTS_PER_IP = 4,
CONCURRENT_REQUESTS_PER_DOMAIN = 4,
DUPEFILTER_CLASS = 'tkit_scrapy_mongo.dupefilter.RFPDupeFilter',
# 去重复
SPIDER_MIDDLEWARES={
# "scrapy_splash.SplashDeduplicateArgsMiddleware": 100,
"tkit_scrapy_mongo.dupefilter.RFPDupeFilter":101
}
)