update

hgg070_spider/pipelines/lanqiu.py

@@ -5,6 +5,7 @@ from ..settings import LEAGUE_URL,MATCH_URL
 class ZuqiuPipeline(object):
     @defer.inlineCallbacks
     def process_item(self,item,spider):
+        print('---------------------------------------------------------------------')
         logger=logging.getLogger(__name__)
         logger.info("进入管道")
         out=defer.Deferred()
@@ -28,6 +29,7 @@ class ZuqiuPipeline(object):
         obj={"uuid":uuid,"league_id":item['id'],"league_name":league_name}
         res=Helper.async_post(LEAGUE_URL,obj)
         if res:
+            print('提交联赛成功888888888888888888888888888*******************')
             if res.get('status')==1:
                 logging.warning("联赛提交成功,{}".format(res))
                 #提交赛事
@@ -37,6 +39,7 @@ class ZuqiuPipeline(object):
                 }
                 lres=Helper.async_post(MATCH_URL,childer)
                 if lres.get('status')==1:
+                    print("提交赛事成功666666666666666666666666666666666666666666666")
                     logging.warning("联赛提交成功,{}".format(res))
                 else:
                     logging.warning("联赛提交失败,{}".format(res))

hgg070_spider/spiders/lanqiu.py

@@ -8,11 +8,11 @@ class LanqiuSpider(scrapy.Spider):
     allowed_domains = ['m.hgg070.com/']
     start_urls = ['http://m.hgg070.com//']
     remath=re.compile("篮球")
-    # custom_settings={
-    #     "ITEM_PIPELINES": {
-    #         "collectSports.pipelines.zuqiu.ZuqiuPipeline": 200,
-    #     },
-    # }
+    custom_settings={
+        "ITEM_PIPELINES": {
+            "hgg070_spider.pipelines.lanqiu.ZuqiuPipeline": 200,
+        },
+    }
     def start_requests(self):
         #今日，早盘
         h_types=[('FT'),('FU')]
@@ -33,7 +33,7 @@ class LanqiuSpider(scrapy.Spider):
         for item in h_types:
             showtype = item
             data={
-                'uid': '9b9f0dea34ee4e5225990433370a88fea2b37b79c98e266b338152c619a71d2b',
+                'uid': '257853bc6f4166ca4e84f4d75d1cfc3540c6eab54b34898f4ad405cb2412402f',
                 'langx': 'zh-cn',
                 'ltype': '3',
                 'gtype': 'BK',
@@ -71,7 +71,6 @@ class LanqiuSpider(scrapy.Spider):
         for g in game:
             gid=g.xpath("./gid/text()").extract_first()
             data["gid"]=gid
-            print('wwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwww',data)
             yield scrapy.FormRequest(url=url,formdata=data,callback=self.getItem,dont_filter=True)
 
 
@@ -79,7 +78,7 @@ class LanqiuSpider(scrapy.Spider):
         data=response.xpath("//game")
         for item in data:
             obj=LanqiuItem()
-            obj['id']=item.xpath("./gid")
+            obj['id']=item.xpath("./gid/text()").extract_first()
             obj['league'] = item.xpath("./league/text()").extract_first()
             obj['team_h'] = item.xpath("./team_h/text()").extract_first()
             obj['team_c'] = item.xpath("./team_c/text()").extract_first()

hgg070_spider/spiders/zuqiu.py

@@ -4,7 +4,7 @@ import lxml
 
 import scrapy
 
-from items import ZuqiuItem
+from ..items import ZuqiuItem
 
 
 class ZuqiuSpider(scrapy.Spider):