|
|
@@ -24,7 +24,7 @@ USER_AGENT = {
|
|
|
ROBOTSTXT_OBEY = False
|
|
|
|
|
|
# Configure maximum concurrent requests performed by Scrapy (default: 16)
|
|
|
-CONCURRENT_REQUESTS = 100
|
|
|
+CONCURRENT_REQUESTS = 32
|
|
|
|
|
|
# Configure a delay for requests for the same website (default: 0)
|
|
|
# See https://doc.scrapy.org/en/latest/topics/settings.html#download-delay
|
|
|
@@ -32,8 +32,8 @@ CONCURRENT_REQUESTS = 100
|
|
|
#DOWNLOAD_DELAY = 3
|
|
|
DOWNLOAD_DELAY = 0
|
|
|
# The download delay setting will honor only one of:
|
|
|
-CONCURRENT_REQUESTS_PER_DOMAIN = 100
|
|
|
-CONCURRENT_REQUESTS_PER_IP = 100
|
|
|
+CONCURRENT_REQUESTS_PER_DOMAIN = 32
|
|
|
+CONCURRENT_REQUESTS_PER_IP = 32
|
|
|
|
|
|
# Disable cookies (enabled by default)
|
|
|
#COOKIES_ENABLED = False
|
|
|
@@ -77,20 +77,20 @@ DOWNLOADER_MIDDLEWARES = {
|
|
|
# Configure item pipelines
|
|
|
# See https://doc.scrapy.org/en/latest/topics/item-pipeline.html
|
|
|
ITEM_PIPELINES = {
|
|
|
- # 篮球
|
|
|
- 'hg3535.pipelines.Lanqiupipeline': 300,
|
|
|
- # 冠军
|
|
|
- 'hg3535.pipelines.Guanjunpipeline': 300,
|
|
|
- # 联赛
|
|
|
- 'hg3535.pipelines.Liansaipipeline': 300,
|
|
|
- # 足球
|
|
|
- 'hg3535.pipelines.Zuqiupipeline': 300,
|
|
|
- # 网球
|
|
|
- "hg3535.pipelines.Wangqiupipeline": 300,
|
|
|
- # 网球波胆
|
|
|
- 'hg3535.pipelines.Wqbodanpipeline': 300,
|
|
|
- # 棒球让球&大小盘
|
|
|
- "hg3535.pipelines.Bangqiupipeline": 300,
|
|
|
+ # # 篮球
|
|
|
+ # 'hg3535.pipelines.Lanqiupipeline': 300,
|
|
|
+ # # 冠军
|
|
|
+ # 'hg3535.pipelines.Guanjunpipeline': 300,
|
|
|
+ # # 联赛
|
|
|
+ # 'hg3535.pipelines.Liansaipipeline': 300,
|
|
|
+ # # 足球
|
|
|
+ # 'hg3535.pipelines.Zuqiupipeline': 300,
|
|
|
+ # # 网球
|
|
|
+ # "hg3535.pipelines.Wangqiupipeline": 300,
|
|
|
+ # # 网球波胆
|
|
|
+ # 'hg3535.pipelines.Wqbodanpipeline': 300,
|
|
|
+ # # 棒球让球&大小盘
|
|
|
+ # "hg3535.pipelines.Bangqiupipeline": 300,
|
|
|
}
|
|
|
|
|
|
# Enable and configure the AutoThrottle extension (disabled by default)
|
Your Name