'更新'

hg3535/settings.py

@@ -24,7 +24,7 @@ USER_AGENT = {
 ROBOTSTXT_OBEY = False
 
 # Configure maximum concurrent requests performed by Scrapy (default: 16)
-CONCURRENT_REQUESTS = 100
+CONCURRENT_REQUESTS = 32
 
 # Configure a delay for requests for the same website (default: 0)
 # See https://doc.scrapy.org/en/latest/topics/settings.html#download-delay
@@ -32,8 +32,8 @@ CONCURRENT_REQUESTS = 100
 #DOWNLOAD_DELAY = 3
 DOWNLOAD_DELAY = 0
 # The download delay setting will honor only one of:
-CONCURRENT_REQUESTS_PER_DOMAIN = 100
-CONCURRENT_REQUESTS_PER_IP = 100
+CONCURRENT_REQUESTS_PER_DOMAIN = 32
+CONCURRENT_REQUESTS_PER_IP = 32
 
 # Disable cookies (enabled by default)
 #COOKIES_ENABLED = False
@@ -77,20 +77,20 @@ DOWNLOADER_MIDDLEWARES = {
 # Configure item pipelines
 # See https://doc.scrapy.org/en/latest/topics/item-pipeline.html
 ITEM_PIPELINES = {
-   # 篮球
-   'hg3535.pipelines.Lanqiupipeline': 300,
-   # 冠军
-   'hg3535.pipelines.Guanjunpipeline': 300,
-   # 联赛
-   'hg3535.pipelines.Liansaipipeline': 300,
-   # 足球
-   'hg3535.pipelines.Zuqiupipeline': 300,
-   # 网球
-   "hg3535.pipelines.Wangqiupipeline": 300,
-   # 网球波胆
-   'hg3535.pipelines.Wqbodanpipeline': 300,
-   # 棒球让球&大小盘
-   "hg3535.pipelines.Bangqiupipeline": 300,
+   # # 篮球
+   # 'hg3535.pipelines.Lanqiupipeline': 300,
+   # # 冠军
+   # 'hg3535.pipelines.Guanjunpipeline': 300,
+   # # 联赛
+   # 'hg3535.pipelines.Liansaipipeline': 300,
+   # # 足球
+   # 'hg3535.pipelines.Zuqiupipeline': 300,
+   # # 网球
+   # "hg3535.pipelines.Wangqiupipeline": 300,
+   # # 网球波胆
+   # 'hg3535.pipelines.Wqbodanpipeline': 300,
+   # # 棒球让球&大小盘
+   # "hg3535.pipelines.Bangqiupipeline": 300,
 }
 
 # Enable and configure the AutoThrottle extension (disabled by default)