hgg070_spider/items.py

@@ -21,23 +21,13 @@ class LanqiuItem(scrapy.Field):
     team_c = scrapy.Field()
     showtype=scrapy.Field()
     datetime=scrapy.Field()
+    match_identity=scrapy.Field()
+    more_count=scrapy.Field()
+    match_id=scrapy.Field()
+    source=scrapy.Field()
+    updata=scrapy.Field()
+    content=scrapy.Field()
+    league_id=scrapy.Field()
+
 
 
-class Odds(scrapy.Item):
-    # zq_odds = scrapy.Field()  #足球详细赔率
-    match_id = scrapy.Field()
-    uuid = scrapy.Field()
-    source = scrapy.Field()
-    updata = scrapy.Field()
-    content = scrapy.Field()  # 足球详细玩法赔率
-    gidm = scrapy.Field()
-    tag = scrapy.Field()
-    league = scrapy.Field()
-    match_uid = scrapy.Field()
-    process = scrapy.Field()
-    score_h = scrapy.Field()
-    score_c = scrapy.Field()
-    team_h = scrapy.Field()
-    team_c = scrapy.Field()
-    score_dict = scrapy.Field()
-    datetime = scrapy.Field()

hgg070_spider/main.py

@@ -6,7 +6,7 @@ from scrapy.cmdline import execute
 sys.path.append(os.path.dirname(os.path.abspath(__file__)))
 # execute(["scrapy", "crawl", "zuqiu"])
 # execute(["scrapy", "crawl", "lanqiu"])
-# execute(["scrapy", "crawl", "lq_sports"])
+execute(["scrapy", "crawl", "lq_sports"])
 # execute(["scrapy", "crawl", "guanjun"])
 # execute(["scrapy", "crawl", "wangqiu"])
 # execute(["scrapy", "crawl", "wqbodan"])

hgg070_spider/pipelines/lanqiu.py

@@ -2,7 +2,14 @@ import logging
 from twisted.internet import defer,reactor
 from ..utils.helper import Helper
 from ..settings import LEAGUE_URL,MATCH_URL
+import pymongo
+from ..settings import M_HOST,M_USER,M_PASSWORD,M_POST,M_DB
 class ZuqiuPipeline(object):
+    def open_spider(self, spider):
+        self.mongo = pymongo.MongoClient(host=M_HOST, username=M_USER, password=M_PASSWORD, port=M_POST,
+                                         authSource=M_DB)
+        self.db = self.mongo[M_DB]
+
     @defer.inlineCallbacks
     def process_item(self,item,spider):
         logger=logging.getLogger(__name__)
@@ -33,6 +40,7 @@ class ZuqiuPipeline(object):
         #联赛
         obj = {"game_code": "lq", "title": "league", "source": "hgg070","data":[childer]}
         res=Helper.async_post(LEAGUE_URL,obj)
+
         if res:
             if res.get('status')==1:
                 logging.warning("联赛提交成功,{}".format(res))

hgg070_spider/pipelines/lq_sports.py

@@ -2,9 +2,11 @@ import logging
 from twisted.internet import defer,reactor
 from ..utils.helper import Helper
 from ..settings import LEAGUE_URL,MATCH_URL
-class ZuqiuPipeline(object):
+import time
+class LqSportsPipeline(object):
     @defer.inlineCallbacks
     def process_item(self,item,spider):
+        print('555555555555555555555555555555555555555555555555555555555555555555555')
         logger=logging.getLogger(__name__)
         logger.info("进入管道")
         out=defer.Deferred()
@@ -16,33 +18,23 @@ class ZuqiuPipeline(object):
         #先保存联赛
         league_name = item['league']
         uuid = Helper.genearte_uuid(league_name)
-        type=item['showtype']
-        is_rollball,is_today,is_morningplate = 0,0,0
-        if type=="FT":
-            is_today=1
-        elif type=="FU":
-            is_morningplate=1
-        elif type=="RB":
-            is_rollball=1
+        #是否串场
+        if item['isP'] == 'P':
+            ris_stringscene = 1
         else:
-            is_stringscene=1
-        league_key = ["name_chinese", "kind", "match_mode", "if_stop", "last_time", "lg_id", "source", "uuid","is_rollball","is_today","is_morningplate","is_stringscene"]
-        league_value = [league_name, "1", "1", "0", item['datetime'], item['id'], "hgg070", uuid,is_rollball,is_today,is_morningplate,is_stringscene]
+            ris_stringscene = 0
+        # 现在时间，时间戳
+        utime = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime())
+        odds_key = ["game_code", "title", "match_id", "lg_id","data", "source", "odds_only", "tag", "uuid",
+                    "is_stringscene", "utime", "pt", 'match_identity']
+        odds_value = ["lq", "odds", item['match_id'], item['league_id'], item["content"],"hgg070", [], item['more_count'], uuid,
+                      ris_stringscene, utime, item['isP'], item["match_identity"]]
         #赛事
-        childer = dict(zip(league_key, league_value))
-        #联赛
-        obj = {"game_code": "lq", "title": "league", "source": "hgg070","data":[childer]}
-        res=Helper.async_post(LEAGUE_URL,obj)
+        childer = dict(zip(odds_key, odds_value))
+        res=Helper.async_post(LEAGUE_URL,childer)
         if res:
             if res.get('status')==1:
                 logging.warning("联赛提交成功,{}".format(res))
-                #提交赛事
-                lres=Helper.async_post(MATCH_URL,childer)
-                if lres.get('status')==1:
-                    logging.warning("联赛提交成功,{}".format(res))
-                else:
-                    logging.warning("联赛提交失败,{}".format(res))
-
             else:
                 logging.warning("联赛提交失败,{}".format(res))
         else:

hgg070_spider/spiders/lq_sports.py

@@ -6,18 +6,19 @@ import lxml.etree
 import re,os,json
 from ..utils.helper import Helper
 import time
-from ..items import Odds
+from ..items import LanqiuItem
+import xmltodict
 
 class LqSportsSpider(scrapy.Spider):
     name = 'lq_sports'
     allowed_domains = ['m.hgg070.com/']
     start_urls = ['http://m.hgg070.com//']
     remath = re.compile("篮球")
-    custom_settings={
-        "ITEM_PIPELINES": {
-            "hgg070_spider.pipelines.lanqiu.ZuqiuPipeline": 200,
-        },
-    }
+    # custom_settings={
+    #     "ITEM_PIPELINES": {
+    #         "hgg070_spider.pipelines.lq_sports.LqSportsPipeline": 200,
+    #     },
+    # }
     def start_requests(self):
         #今日，早盘
         h_types=[('FT'),('FU')]
@@ -38,7 +39,7 @@ class LqSportsSpider(scrapy.Spider):
         for item in h_types:
             showtype = item
             data={
-                'uid': 'ab179dc88196ff82fbb13c259575332f01fbad2c52b465f5def15a4876c10410',
+                'uid': '1c721b7a80b72068ee7217348356d79160e0d97f2e34c98b3e12ea33355066b6',
                 'langx': 'zh-cn',
                 'ltype': '3',
                 'gtype': 'BK',
@@ -75,90 +76,81 @@ class LqSportsSpider(scrapy.Spider):
         game=response.xpath("//game")
         for g in game:
             gid=g.xpath("./gid/text()").extract_first()
+            more_count = g.xpath("./more_count/text()").extract_first()
             data["gid"]=gid
-            yield scrapy.FormRequest(url=url,formdata=data,callback=self.getItem,dont_filter=True)
+            yield scrapy.FormRequest(url=url,formdata=data,callback=self.getItem,meta={"more_count":more_count,"isP":data["isP"]},dont_filter=True)
 
 
     def getItem(self,response):
-        game_lists = []
-        data=response.xpath("//game")
-        if data:
-            uid_list = []
-            for game in data:
-                game_odds = {}
-                gopen = game.xpath('//game/gopen/text()').extract_first()
-                if gopen == 'Y':
-                    game = lxml.etree.fromstring(game.extract())
-                    for i in game.getchildren():
-                        if i.text == None:
-                            game_odds[i.tag] = ""
-                        else:
-                            game_odds[i.tag] = i.text.replace(' ', '')
-                    game_lists.append(game_odds)
-                else:
-                    print('gopen == N, 详细赔率盘口未开启')
+        more_count = response.meta["more_count"]
+        isP = response.meta["isP"]
+        data= xmltodict.parse(response.text)['serverresponse']['game']
+        game_lists=[i for i in data if i['gopen']=='Y']
+
         if game_lists:
             for gl in game_lists:
                 cpath=os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
                 with open(cpath+"/conf/hgg070.json",encoding='utf8') as hg:
                     hgg=json.load(hg)['bk']
-                odd_list = []
                 datetime = gl['datetime'][:-8] + " " + gl['datetime'][-8:]
                 team_h = gl['team_h']
                 team_c = gl['team_c']
-                tag = 0
                 league_id = gl['gidm']
                 match_id = gl.get('gid', '')
-                match_uid = Helper.genearte_MD5(team_h + team_c + datetime)
-                for x in hgg:
-                    try:
-                        enabled = gl[x['prodds']]
-                        if enabled == 'Y':
-                            x['enabled'] = 1
-                            tag += 1
-                        else:
-                            x['enabled'] = 0
-                    except:
-                        enabled = ''
-                        x['enabled'] = 0
-                    items = x['items']
-                    new_items = []
-                    for y in items:
-                        try:
-                            y['oddsv'] = gl[y['rodds']]
-                        except:
-                            y['oddsv'] = 0
-                        try:
-                            y['ratio'] = gl[y['ratio_name']]
-                        except:
-                            y['ratio'] = ""
-                        y['data'] = gl
-                        uid = str(x['plodds']) + str(y['lodds']) + str(y['rodds']) + str(y['ratio']) + str(
-                            y['ratio_name']) + str(y['oddsv']) + str(match_id) + str(league_id)
-                        sl = str(y['lodds']) + str(y['rodds']) + str(y['ratio']) + str(y['ratio_name']) + str(
-                            x['plodds']) + str(x['prodds'])
-                        odds_only = Helper.genearte_MD5(uid)
-                        sole = Helper.genearte_MD5(sl)
-                        y['uid'] = odds_only
-                        y['sole'] = sole
-                        if enabled == 'Y':
-                            uid_list.append(odds_only)
-                        new_items.append(y)
-                    n_i = copy.deepcopy(x)
-                    n_i['items'] = new_items
-                    odd_list.append(n_i)
-                item = Odds()
+                match_uid = Helper.genearte_uuid(team_h + team_c + datetime)
+                data = []
+                for hg in hgg:
+                    items=hg['items']
+                    for x in items:
+                        print('*****************', gl)
+                        print('*****************',x)
+                        odds_code = gl[x['rodds']]
+                        p_code = gl[x['prodds']]
+                        odds=gl["ior_OUH"]
+
+                        #有两个条件，加两条数据
+                        if x['ratio_name']:      #大的
+                            condition_u=gl[x['ratio_name']]
+                            odds_only = gl[x["prodds"]] + gl[x["rodds"]] + '0' + condition_u + str(odds) + "hg3535" + str(match_id)
+                            sole = gl[x["prodds"]] + gl[x["rodds"]] + '0' + str(match_id) + "hg3535"
+                            tobj = {"match_id": match_id, "lg_id": league_id, "odds_code": odds_code, "status": 0,
+                                    "sort": 0, "p_code": p_code,
+                                    "odds": odds, "condition": condition_u, "odds_only": odds_only, "sole": sole,
+                                    "source": "hgg070", "type": 0, "team": ""}
+                            data.append(tobj)
+
+                        if x['latio']:   #小的
+                            condition_s = gl[x['latio']]
+                            odds_only = gl[x["prodds"]] + gl[x["rodds"]] + '0' +condition_s + str(odds) + "hg3535" + str(match_id)
+                            sole = gl[x["prodds"]] + gl[x["rodds"]] + '0' + str(match_id) + "hg3535"
+                            tobj = {"match_id": match_id, "lg_id": league_id, "odds_code": odds_code, "status": 0,
+                                    "sort": 0, "p_code": p_code,
+                                    "odds": odds,"condition": condition_s, "odds_only": odds_only, "sole": sole,
+                                    "source": "hgg070", "type": 0, "team": ""}
+                            data.append(tobj)
+
+                        if not x['latio'] and not x['ratio_name']:
+                            condition_s = ''
+                            odds_only = gl[x["prodds"]] + gl[x["rodds"]] + '0' +condition_s + str(odds) + "hg3535" + str(match_id)
+                            sole = gl[x["prodds"]] + gl[x["rodds"]] + '0' + str(match_id) + "hg3535"
+                            tobj = {"match_id": match_id, "lg_id": league_id, "odds_code": odds_code, "status": 0,
+                                    "sort": 0, "p_code": p_code,
+                                    "odds": odds,"condition": condition_s, "odds_only": odds_only, "sole": sole,
+                                    "source": "hgg070", "type": 0, "team": ""}
+                            data.append(tobj)
+
+                item = LanqiuItem()
                 item['match_id'] = match_id
-                item['uuid'] = uid_list
                 item['source'] = "hg0088"
                 item['updata'] = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime())
-                item['content'] = odd_list
-                item['gidm'] = league_id
-                item['tag'] = tag
+                item['content'] = data
+                item['league_id'] = league_id
+                item['more_count'] = more_count
                 item['league'] = gl["league"]
-                item['match_uid'] = match_uid
+                item['match_identity'] = match_uid
                 item['datetime'] = datetime
                 item['team_h'] = team_h
                 item['team_c'] = team_c
-                print('最后#######################################################',item)
+                item['isP'] = isP
+                print('2222222222222222222222222222222222222222222222222222222:',item)
                 yield item

main.py

@@ -3,5 +3,5 @@ import os
 from scrapy.cmdline import execute
 
 sys.path.append(os.path.dirname(os.path.abspath(__file__)))
-execute(["scrapy", "crawl", "lanqiu"])
+execute(["scrapy", "crawl", "lq_sports"])
 # execute(["scrapy", "crawl", "zuqiu"])