updata

hg3535/main.py

@@ -11,10 +11,10 @@ sys.path.append(os.path.dirname(os.path.abspath(__file__)))
 # execute(["scrapy", "crawl", "wangqiu"])
 # execute(["scrapy", "crawl", "wqbodan"])
 # execute(["scrapy", "crawl", "bangqiu"])
-execute(["scrapy", "crawl", "roll_zuqiu"]) # 滚球足球 回来要解开这个注释 其他全部解封
+# execute(["scrapy", "crawl", "roll_zuqiu"]) # 滚球足球 回来要解开这个注释 其他全部解封
 # execute(["scrapy", "crawl", "roll_lanqiu"]) #滚球篮球
 # execute(["scrapy", "crawl", "roll_wangqiu"]) #滚球网球
-# execute(["scrapy", "crawl", "roll_bangqiu"])  # 滚球棒球
+execute(["scrapy", "crawl", "roll_bangqiu"])  # 滚球棒球
 # execute(["scrapy", "crawl", "ball_status"]) #滚球id本地存
 # execute(["scrapy", "crawl", "ball_status_update"]) #滚球id结束时间更新状态
 # execute(["scrapy", "crawl", "saiguo"]) #滚球id结束时间更新状态

hg3535/pipeline/roll_bangqiu.py

@@ -3,7 +3,6 @@ import pymongo
 import time
 import logging
 # import json
-
 from twisted.internet import defer, reactor
 
 # from .ball_func import fuhao
@@ -20,13 +19,14 @@ class Roll_Banqiupipeline(object):
         self.db = self.mongo[M_DB]
 
     # @defer.inlineCallbacks
-    def process_item(self, item, spider):
-        out = defer.Deferred()
-        reactor.callInThread(self._do_calculation, item, out)
-        yield out
-        defer.returnValue(item)
+    # def process_item(self, item, spider):
+    #     out = defer.Deferred()
+    #     reactor.callInThread(self._do_calculation, item, out)
+    #     yield out
+    #     defer.returnValue(item)
 
-    def _do_calculation(self, item, out):
+    def process_item(self, item, spider):
+    # def _do_calculation(self, item, out):
         # 使用twisted将mysql插入变成异步执行
         logger = logging.getLogger(__name__)
         # 联赛id
@@ -45,7 +45,7 @@ class Roll_Banqiupipeline(object):
         # 数量(97>)
         tag_number = item['number']
         # 新增 match_score_dict
-        match_score_dict = item['match_score_dict']
+        # match_score_dict = item['match_score_dict']
         # 比赛状态
         zhuangtai = item['zhuangtai']
         # 日期
@@ -68,8 +68,6 @@ class Roll_Banqiupipeline(object):
         score_guest = item['score_guest']
         # 第几节
         jijie = item['jijie']
-        # 球队得分
-        qiudui = item['qiudui']
         pt = 4
         # 滚球这个位置获取不到这些字段
         # match_date, match_time = new_time(ctime)
@@ -87,7 +85,7 @@ class Roll_Banqiupipeline(object):
         # 赛事失效时间
         # new_match_score_dict = json.dumps(match_score_dict)
         # 具体比分
-        team_source_dict = item['team_source_dict']
+        source_dict = item['source_dict']
         uuid = Helper.genearte_uuid(league_name + 'hg3535')
         """联赛"""
         last_time = '{}-12-31 23:59:59'.format(datetime.datetime.now().year)
@@ -224,11 +222,11 @@ class Roll_Banqiupipeline(object):
         bq_rball = {
             "lg_id": league_id, "home_team": team_home, "guest_team": team_guest, "home_rate": '', "guest_rate": '',
             "home_score": score_home, "guest_score": score_guest, "status": 1, "first_score": '',
-            "last_score": '', "all_inning": team_source_dict['total_inning'],
-            "match_score": team_source_dict['total_source'], "match_winer": '', "updata_time": '',
+            "last_score": '', "all_inning": '',
+            "match_score": '', "match_winer": '', "updata_time": '',
             "match_time": time_game, "match_process": jijie,
             "tag": tag_number, "match_id": match_id, "source": 'hg3535',
-            "ctime": ctime, "uuid": uuid, "result_mark": qiudui, "start_time": '', 'warn_more': '',
+            "ctime": ctime, "uuid": uuid, "result_mark": source_dict, "start_time": '', 'warn_more': '',
         }
         da_list.append(bq_rball)
         r_data_dict = {
@@ -243,15 +241,15 @@ class Roll_Banqiupipeline(object):
                 res = Helper.async_post(MATCH_RESULT, r_data_dict)
                 if "成功" in res:
                     logger.info('棒球滚球结果记录提交成功, {}'.format(res))
-                    logger.info(odds_dict)
+                    logger.info(r_data_dict)
                 else:
                     logger.warning('棒球滚球结果记录提交失败, {}'.format(res))
-                    logger.warning(odds_dict)
+                    logger.warning(r_data_dict)
 
             except Exception as e:
                 logger.warning("滚球接口数据异常, 提交失败, {}".format(e))
                 logging.warning(r_data_dict)
         else:
             logger.info('棒球滚球结果记录为空, 不提交')
-        reactor.callFromThread(out.callback, item)
+        # reactor.callFromThread(out.callback, item)
 

hg3535/spiders/roll_bangqiu.py

@@ -29,7 +29,6 @@ class BqrangqiuSpider(scrapy.Spider):
         # 'LOG_FILE': "../hg3535/log/roll_bangqiu_{}_{}_{}.log".format(to_day.year, to_day.month, to_day.day)
     }
 
-    # start_urls = ['https://hg3535z.com/odds2/d/getodds?sid=4&pt=4&ubt=am&pn=0&sb=2&dc=null&pid=0']  # 滚球菜单 棒球滚球列url
     start_urls = [
         'https://odata.jiushan6688.com/odds6i/d/getodds/zh-cn/sid/4/pt/4/ubt/am/pn/0/sb/2/dc/null/pid/0']  # 滚球菜单 棒球滚球列url
     rls = redis.Redis(host='192.168.2.200', port=6379, db=1, password=123456)
@@ -47,6 +46,7 @@ class BqrangqiuSpider(scrapy.Spider):
         datas = json.loads(response.text)
         # 倒三角形  sa向下，sh向上
         triangle = datas['i'][5]
+        rule = {'sa': 'Bottom', 'sh': 'Top'}
         try:
             new_datas = datas['eg']
         except:
@@ -85,31 +85,36 @@ class BqrangqiuSpider(scrapy.Spider):
                     # 第几盘
                     # jijie = result['i'][12]
                     # 球队得分
-                    qiudui = result['pci'].get('ctn', "")
+                    # qiudui = result['pci'].get('ctn', "")
                     # 是否出局 不一定存在
                     ot = result['sb'].get("ot", "")
 
                     ps_list = result['sb']['ps']
                     # 右菱形bool值
                     fb = result['sb']['fb']
+                    if not fb:
+                        baseman_1 = 'N'
+                    else:
+                        baseman_1 = 'Y'
                     # 左菱形bool值
                     tb = result['sb']['tb']
+                    if not tb:
+                        baseman_3 = 'N'
+                    else:
+                        baseman_3 = 'Y'
                     # 上菱形bool值
                     sb = result['sb']['tb']
+                    if not sb:
+                        baseman_2 = 'N'
+                    else:
+                        baseman_2 = 'Y'
                     # 局数或者节或者分
                     cp = result['sb']['cp']
                     # 第一个队伍
                     a = result['sb']['a']
                     # 第二个队伍
                     h = result['sb']['h']
-                    d = {}
-                    d['ps'] = ps_list
-                    d['cp'] = cp
-                    d['ot'] = ot
-                    d['triangle'] = triangle
-                    d['fb'] = fb
-                    d['tb'] = tb
-                    d['sb'] = sb
+
                     # --------------------------------------------------让球分割线----------------------------------------
                     concedes_dict = {}
                     concedes_dict_rule = {}
@@ -190,50 +195,34 @@ class BqrangqiuSpider(scrapy.Spider):
                         odd_evens_dict['tsd'] = ""
                         odd_evens_dict_rule['tss'] = ""
                         odd_evens_dict_rule['tsd'] = ""
-
-                    team_source_dict = {}
-                    try:
-                        team_sources = result['sb']['ps']
-                        for team_source in team_sources:
-                            if team_source['p'] == 'i1':
-                                team_source_dict['one_score'] = str(team_source['h']) + '-' + str(team_source['a'])
-                            if team_source['p'] == 'i2':
-                                team_source_dict['two_score'] = str(team_source['h']) + '-' + str(team_source['a'])
-                            if team_source['p'] == 'i3':
-                                team_source_dict['three_score'] = str(team_source['h']) + '-' + str(team_source['a'])
-                            if team_source['p'] == 'i4':
-                                team_source_dict['four_score'] = str(team_source['h']) + '-' + str(team_source['a'])
-                            if team_source['p'] == 'i5':
-                                team_source_dict['five_score'] = str(team_source['h']) + '-' + str(team_source['a'])
-                            if team_source['p'] == 'i6':
-                                team_source_dict['six_score'] = str(team_source['h']) + '-' + str(team_source['a'])
-                            if team_source['p'] == 'i7':
-                                team_source_dict['seven_score'] = str(team_source['h']) + '-' + str(team_source['a'])
-                            if team_source['p'] == 'i8':
-                                team_source_dict['eight_score'] = str(team_source['h']) + '-' + str(team_source['a'])
-                            if team_source['p'] == 'i9':
-                                team_source_dict['nine_score'] = str(team_source['h']) + '-' + str(team_source['a'])
-                            # 加时
-                            if team_source['p'] == 'ot':
-                                team_source_dict['overtime_score'] = str(team_source['h']) + '-' + str(team_source['a'])
-                            # 总局数比分
-                            if team_source['p'] == 'ft':
-                                team_source_dict['total_inning'] = int(team_source['h']) + int(team_source['a'])
-                                team_source_dict['total_source'] = str(team_source['h']) + '-' + str(team_source['a'])
-                    except:
-                        team_source_dict['one_score'] = ''
-                        team_source_dict['two_score'] = ''
-                        team_source_dict['three_score'] = ''
-                        team_source_dict['four_score'] = ''
-                        team_source_dict['five_score'] = ''
-                        team_source_dict['six_score'] = ''
-                        team_source_dict['seven_score'] = ''
-                        team_source_dict['eight_score'] = ''
-                        team_source_dict['nine_score'] = ''
-                        team_source_dict['overtime_score'] = ''
-                        team_source_dict['total_inning'] = 0
-                        team_source_dict['total_source'] = ''
-
+                    source_dict = {}
+                    source_dict['sc_1th_H'] = ps_list[2]['h']
+                    source_dict['sc_1th_C'] = ps_list[2]['a']
+                    source_dict['sc_2th_H'] = ps_list[3]['h']
+                    source_dict['sc_2th_C'] = ps_list[3]['a']
+                    source_dict['sc_3th_H'] = ps_list[4]['h']
+                    source_dict['sc_3th_C'] = ps_list[4]['a']
+                    source_dict['sc_4th_H'] = ps_list[5]['h']
+                    source_dict['sc_4th_C'] = ps_list[5]['a']
+                    source_dict['sc_5th_H'] = ps_list[6]['h']
+                    source_dict['sc_5th_C'] = ps_list[6]['a']
+                    source_dict['sc_6th_H'] = ps_list[7]['h']
+                    source_dict['sc_6th_C'] = ps_list[7]['a']
+                    source_dict['sc_7th_H'] = ps_list[8]['h']
+                    source_dict['sc_7th_C'] = ps_list[8]['a']
+                    source_dict['sc_8th_H'] = ps_list[9]['h']
+                    source_dict['sc_8th_C'] = ps_list[9]['a']
+                    source_dict['sc_9th_H'] = ps_list[10]['h']
+                    source_dict['sc_9th_C'] = ps_list[10]['a']
+                    source_dict['OT_H'] = ps_list[1]['h']
+                    source_dict['OT_C'] = ps_list[1]['a']
+                    source_dict['game_num_H'] = ps_list[0]['h']
+                    source_dict['game_num_C'] = ps_list[0]['a']
+                    source_dict['schedule'] = rule[triangle] + cp
+                    source_dict['out_game_num'] = ot
+                    source_dict['baseman_1'] = baseman_1
+                    source_dict['baseman_2'] = baseman_2
+                    source_dict['baseman_3'] = baseman_3
                     # 联赛id
                     item['league_id'] = league_id
                     # 联赛名
@@ -272,6 +261,6 @@ class BqrangqiuSpider(scrapy.Spider):
                     # 总得分：单/双
                     item['odd_evens_dict'] = odd_evens_dict
                     item['odd_evens_dict_rule'] = odd_evens_dict_rule
-                    item['match_score_dict'] = d
-                    item['team_source_dict'] = team_source_dict
+                    # item['match_score_dict'] = d
+                    item['source_dict'] = source_dict
                     yield item