hgg070_spider/main.py

@@ -6,7 +6,7 @@ from scrapy.cmdline import execute
 sys.path.append(os.path.dirname(os.path.abspath(__file__)))
 execute(["scrapy", "crawl", "zuqiu"])
 # execute(["scrapy", "crawl", "lanqiu"])
-execute(["scrapy", "crawl", "lq_sports"])
+# execute(["scrapy", "crawl", "lq_sports"])
 # execute(["scrapy", "crawl", "guanjun"])
 # execute(["scrapy", "crawl", "wangqiu"])
 # execute(["scrapy", "crawl", "wqbodan"])

hgg070_spider/pipelines/zuqiu.py

@@ -35,11 +35,11 @@ class ZuqiuPipeline(object):
         league_list = []
         last_time = '{}-12-31 23:59:59'.format(datetime.datetime.now().year)
         match_date, match_time, time3 = Helper.change_time(us_time)
-        if self.db.zq_league35.find({'lg_id': league_id}).count() < 1:
+        if self.db.zq_league070.find({'lg_id': league_id}).count() < 1:
             # if self.db.zq_league35.find({'uuid': uuid}).count() < 1:
-            league_dict = {"game_code": "zq", "title": "league", "source": "hg3535"}
+            league_dict = {"game_code": "zq", "title": "league", "source": "hgg070"}
             league_key = ["name_chinese", "kind", "match_mode", "if_stop", "last_time", "lg_id", "source", "uuid"]
-            league_value = [league_name, "1", "1", "0", last_time, league_id, "hg3535", uuid]
+            league_value = [league_name, "1", "1", "0", last_time, league_id, "hgg070", uuid]
             league_data = dict(zip(league_key, league_value))
             league_list.append(league_data)
             league_dict['data'] = league_list
@@ -72,9 +72,9 @@ class ZuqiuPipeline(object):
 
         match_list = []
         match_identity = Helper.genearte_uuid(team_h + team_c + match_date)
-        if self.db.zq_competition35.find({'match_id': match_id, pt_status: 1}).count() < 1:
+        if self.db.zq_competition070.find({'match_id': match_id, pt_status: 1}).count() < 1:
             # if self.db.zq_competition35.find({'match_identity': match_identity, pt_status: 1}).count() < 1:
-            match_dict = {"game_code": "zq", "title": "match", "source": "hg3535"}
+            match_dict = {"game_code": "zq", "title": "match", "source": "hgg070"}
             match_kay = ["home_team", "guest_team", "lg_id", "status", "match_id", "match_date", "match_time",
                          "tag", "source", "is_rollball", "is_morningplate", "is_stringscene", "us_time", "uuid",
                          "half_match_id", "is_today", "is_horn", 'match_identity']
@@ -97,4 +97,5 @@ class ZuqiuPipeline(object):
                 # logger.warning(match_dict)
         else:
             logger.info('足球赛事已存在，不提交')
+
         # reactor.callFromThread(out.callback, item)

hgg070_spider/spiders/zuqiu.py

@@ -3,6 +3,7 @@ import logging
 import lxml
 
 import scrapy
+import xmltodict
 
 from ..items import ZuqiuItem
 
@@ -132,27 +133,21 @@ class ZuqiuSpider(scrapy.Spider):
                                          meta={'index': index, 'tag': tag}, dont_filter=True)
 
     def parse_odds(self, response):
+        logger = logging.getLogger(__name__)
         index = response.meta['index']
         tag = response.meta['tag']
-        try:
-            game = response.xpath('//serverresponse/game')[0]
-        except:
+        game = xmltodict.parse(response.text)
+        gopen = game['serverresponse']['game']['gopen']
+        if gopen == 'Y':
+            try:
+                game_odds = game['serverresponse']['game'][0]
+            except:
+                game_odds = game['serverresponse']['game']
+        else:
+            logger.info('gopen == "N", 详细赔率盘口未开启')
             return
-        logger = logging.getLogger(__name__)
-        if game:
-            game_odds = {}
-            gopen = game.xpath('//game/gopen/text()').extract_first()
-            if gopen == 'Y':
-                game = lxml.etree.fromstring(game.extract())
-                for i in game.getchildren():
-                    if i.text == None:
-                        game_odds[i.tag] = ""
-                    else:
-                        game_odds[i.tag] = i.text
-            else:
-                logger.info('gopen == "N", 详细赔率盘口未开启')
-            item = ZuqiuItem()
-            item['data'] = game_odds
-            item['index'] = index
-            item['tag'] = tag
-            yield item
+        item = ZuqiuItem()
+        item['data'] = game_odds
+        item['index'] = index
+        item['tag'] = tag
+        yield item