足球,篮球赛果抓取规则更新

hg3535/items.py

@@ -471,8 +471,20 @@ class Roll_Bangqiu(scrapy.Field):
 # -----------------------------------------------------------比赛结果-----------------------------------------------------
 class Hgsaiguo(scrapy.Field):
     # 足球比赛结果元组
-    id_score = scrapy.Field()
+    league_id = scrapy.Field()
+    league_name = scrapy.Field()
+    match_date = scrapy.Field()
+    match_time = scrapy.Field()
+    home_team = scrapy.Field()
+    guest_team = scrapy.Field()
+    score_half = scrapy.Field()
+    score_full = scrapy.Field()
+    play_data = scrapy.Field()
+    match_id = scrapy.Field()
     pt = scrapy.Field()
+    page = scrapy.Field()
+    score_below = scrapy.Field()
+    score_result = scrapy.Field()
 
 
 class Hgjieshu(scrapy.Field):

hg3535/pipeline/bangqiu.py

@@ -116,7 +116,7 @@ class Bangqiupipeline(object):
             league_dict['data'] = league_list
             res = Helper.async_post(LEAGUE_URL, league_dict)
             if res:
-                if '成功' in res:
+                if res.get('status') == 1:
                     self.db.bq_league35.insert(league_data)
                     # self.db.bq_league35.update({'lg_id': league_id}, {'$set': league_data},
                     #                                 upsert=True)
@@ -162,7 +162,7 @@ class Bangqiupipeline(object):
             match_dict['data'] = match_list
             res = Helper.async_post(MATCH_URL, match_dict)
             if res:
-                if '成功' in res:
+                if res.get('status') == 1:
                     self.db.bq_competition35.insert(match_data)
                     # self.db.bq_competition35.update({"match_id": match_id, pt_status: 1}, {'$set': match_data},
                     #                                 upsert=True)
@@ -286,7 +286,7 @@ class Bangqiupipeline(object):
         if data_list:
             res = Helper.async_post(ODDS_URL, odds_dict)
             if res:
-                if "成功" in res:
+                if res.get('status') == 1:
                     logger.info('棒球赔率提交成功, {}'.format(res))
                     logger.info(odds_dict)
                 else:

hg3535/pipeline/guanjun.py

@@ -77,7 +77,7 @@ class Guanjunpipeline(object):
             try:
                 res = Helper.async_post(LEAGUE_URL, l_payload)
                 if res:
-                    if "成功" in res:
+                    if res.get('status') == 1:
                         logger.info('{}, {}冠军联赛, 提交成功, {}'.format(game_code, league_name, res))
                         logger.info(league_list)
                         self.db[league_table].insert(league_dict)
@@ -122,7 +122,7 @@ class Guanjunpipeline(object):
             try:
                 res = Helper.async_post(ODDSCH, chain_dict)
                 if res:
-                    if "成功" in res:
+                    if res.get('status') == 1:
                         logger.info('{}冠军赔率，提交成功, {}'.format(game_code, res))
                         logger.info(chain_dict)
                         self.db[odds_table].insert(chain_data)

hg3535/pipeline/jieshu.py

@@ -53,7 +53,7 @@ class Jieshuqiupipeline(object):
         status_dict['data'] = data_list
         res = Helper.async_post(MATCH_STATUS, status_dict)
         if res:
-            if "成功" in res:
+            if res.get('status') == 1:
                 self.db.match_status35.insert(data)
                 # self.db.match_status35.update({'match_id': match_id}, {'$set': data}, upsert=True)
                 if ostatus == 2:

hg3535/pipeline/lanqiu.py

@@ -89,7 +89,7 @@ class Lanqiupipeline(object):
             league_dict['data'] = league_list
             res = Helper.async_post(LEAGUE_URL, league_dict)
             if res:
-                if '成功' in res:
+                if res.get('status') == 1:
                     logger.info('篮球联赛提交成功, {}'.format(res))
                     self.db.lq_league35.insert(league_data)
                     # self.db.lq_league35.update({'lg_id': league_id}, {'$set': league_data}, upsert=True)
@@ -133,7 +133,7 @@ class Lanqiupipeline(object):
             match_dict['data'] = match_list
             res = Helper.async_post(MATCH_URL, match_dict)
             if res:
-                if '成功' in res:
+                if res.get('status') == 1:
                     logger.info('篮球赛事提交成功, {}'.format(res))
                     self.db.lq_competition35.insert(match_data)
                     # self.db.lq_competition35.update({'match_id': match_id, pt_status: 1}, {'$set': match_data}, upsert=True)
@@ -270,7 +270,7 @@ class Lanqiupipeline(object):
         if data_list:
             res = Helper.async_post(ODDS_URL, odds_dict)
             if res:
-                if '成功' in res:
+                if res.get('status') == 1:
                     logger.info('篮球详细赔率提交成功, {}'.format(res))
                     logger.info(odds_dict)
                 else:

hg3535/pipeline/roll_bangqiu.py

@@ -84,7 +84,7 @@ class Roll_Banqiupipeline(object):
             league_dict['data'] = league_list
             res = Helper.async_post(LEAGUE_URL, league_dict)
             if res:
-                if '成功' in res:
+                if res.get('status') == 1:
                     self.db.bq_league35.insert(league_data)
                     # self.db.bq_competition35.update({'lg_id': league_id}, {'$set': league_data},
                     #                                 upsert=True)
@@ -111,7 +111,7 @@ class Roll_Banqiupipeline(object):
             match_dict['data'] = match_list
             res = Helper.async_post(MATCH_URL, match_dict)
             if res:
-                if '成功' in res:
+                if res.get('status') == 1:
                     self.db.bq_competition35.insert(match_data)
                     # self.db.bq_competition35.update({"match_id": match_id, 'is_rollball': 1}, {'$set': match_data},
                     #                                 upsert=True)
@@ -195,7 +195,7 @@ class Roll_Banqiupipeline(object):
         if data_list:
             res = Helper.async_post(ODDS_URL, odds_dict)
             if res:
-                if "成功" in res:
+                if res.get('status') == 1:
                     logger.info('棒球滚球赔率提交成功, {}'.format(res))
                     logger.info(odds_dict)
                 else:
@@ -227,7 +227,7 @@ class Roll_Banqiupipeline(object):
         if da_list:
             try:
                 res = Helper.async_post(MATCH_RESULT, r_data_dict)
-                if "成功" in res:
+                if res.get('status') == 1:
                     logger.info('棒球滚球结果记录提交成功, {}'.format(res))
                     logger.info(r_data_dict)
                 else:
@@ -247,7 +247,7 @@ class Roll_Banqiupipeline(object):
         # status_dict['data'] = data_list
         # res = Helper.async_post(MATCH_STATUS, status_dict)
         # if res:
-        #     if "成功" in res:
+        #     if res.get('status') == 1
         #         self.db.match_status35.insert(status_dict)
         #         logger.info('{},赛事结果状态交成功, {}'.format('bq', res))
         #         logger.info(status_dict)

hg3535/pipeline/roll_lanqiu.py

@@ -5,10 +5,8 @@ import time
 import logging
 
 from twisted.internet import defer, reactor
-
-# from .ball_func import
 from ..utils.helper import Helper
-from ..settings import M_HOST, M_USER, M_PASSWORD, M_POST, M_DB, LEAGUE_URL, ODDS_URL, MATCH_URL, MATCH_RESULT, MATCH_STATUS
+from ..settings import M_HOST, M_USER, M_PASSWORD, M_POST, M_DB, LEAGUE_URL, ODDS_URL, MATCH_URL, MATCH_RESULT
 
 
 class Roll_Lanqiupipeline(object):
@@ -93,7 +91,7 @@ class Roll_Lanqiupipeline(object):
             league_dict['data'] = league_list
             res = Helper.async_post(LEAGUE_URL, league_dict)
             if res:
-                if '成功' in res:
+                if res.get('status') == 1:
                     logger.info('篮球滚球联赛提交成功, {}'.format(res))
                     # self.db.lq_league35.update({'lg_id': league_id}, {'$set': league_data},
                     #                                 upsert=True)
@@ -120,7 +118,7 @@ class Roll_Lanqiupipeline(object):
             match_dict['data'] = match_list
             res = Helper.async_post(MATCH_URL, match_dict)
             if res:
-                if '成功' in res:
+                if res.get('status') == 1:
                     logger.info('篮球滚球赛事提交成功, {}'.format(res))
                     self.db.lq_competition35.insert(match_data)
                     # self.db.lq_competition35.update({"match_id": match_id, 'is_rollball': 1}, {'$set': match_data},
@@ -259,7 +257,7 @@ class Roll_Lanqiupipeline(object):
         if data_list:
             res = Helper.async_post(ODDS_URL, odds_dict)
             if res:
-                if "成功" in res:
+                if res.get('status') == 1:
                         logger.info('篮球滚球详细赔率提交成功, {}'.format(res))
                 else:
                     logger.warning('篮球滚球详细赔率提交失败, {}'.format(res))
@@ -288,7 +286,7 @@ class Roll_Lanqiupipeline(object):
         if data_list:
             try:
                 res = Helper.async_post(MATCH_RESULT, r_data_dict)
-                if "成功" in res:
+                if res.get('status') == 1:
                     logger.info('篮球滚球赛事结果记录提交成功, {}'.format(res))
                 else:
                     logger.warning('篮球滚球赛事结果记录提交失败, {}'.format(res))
@@ -305,7 +303,7 @@ class Roll_Lanqiupipeline(object):
         # status_dict['data'] = data_list
         # res = Helper.async_post(MATCH_STATUS, status_dict)
         # if res:
-        #     if "成功" in res:
+        #     if res.get('status') == 1:
         #         self.db.match_status35.insert(status_dict)
         #         logger.info('{},赛事结果状态交成功, {}'.format('lq', res))
         #         logger.info(status_dict)

hg3535/pipeline/roll_wangqiu.py

@@ -106,7 +106,7 @@ class Roll_Wangqiupipeline(object):
             league_dict['data'] = league_list
             res = Helper.async_post(LEAGUE_URL, league_dict)
             if res:
-                if '成功' in res:
+                if res.get('status') == 1:
                     self.db.wq_league35.insert(league_data)
                     # self.db.wq_league35.update({'lg_id': league_id}, {'$set': league_data},
                     #                                 upsert=True)
@@ -134,7 +134,7 @@ class Roll_Wangqiupipeline(object):
             match_dict['data'] = match_list
             res = Helper.async_post(MATCH_URL, match_dict)
             if res:
-                if '成功' in res:
+                if res.get('status') == 1:
                     self.db.wq_competition35.insert(match_data)
                     # self.db.wq_competition35.update({'match_id': match_id, 'is_rollball': 1}, {'$set': match_data},
                     #                                 upsert=True)
@@ -239,7 +239,7 @@ class Roll_Wangqiupipeline(object):
         if data_list:
             res = Helper.async_post(ODDS_URL, odds_dict)
             if res:
-                if "成功" in res:
+                if res.get('status') == 1:
                     logger.info('网球滚球赔率提交成功, {}'.format(res))
                     logger.info(odds_dict)
                 else:
@@ -276,7 +276,7 @@ class Roll_Wangqiupipeline(object):
         if da_list:
             try:
                 res = Helper.async_post(MATCH_RESULT, r_data_dict)
-                if "成功" in res:
+                if res.get('status') == 1:
                     logger.info('网球滚球结果记录提交成功, {}'.format(res))
                     logger.info(r_data_dict)
                 else:

hg3535/pipeline/roll_zuqiu.py

@@ -6,11 +6,8 @@ import time
 import logging
 
 from twisted.internet import defer, reactor
-
-# from .ball_func import fuhao
 from ..utils.helper import Helper
-# from .ball_func import get_pcode
-from ..settings import M_HOST, M_USER, M_PASSWORD, M_POST, M_DB, LEAGUE_URL, ODDS_URL, MATCH_URL, MATCH_RESULT, MATCH_STATUS
+from ..settings import M_HOST, M_USER, M_PASSWORD, M_POST, M_DB, LEAGUE_URL, ODDS_URL, MATCH_URL, MATCH_RESULT
 
 
 # 滚球足球 插入
@@ -68,7 +65,7 @@ class Roll_Zuqiupipeline(object):
             league_dict['data'] = league_list
             res = Helper.async_post(LEAGUE_URL, league_dict)
             if res:
-                if "成功" in res:
+                if res.get('status') == 1:
                     self.db.zq_league35.insert(league_data)
                     # self.db.zq_league35.update({'lg_id': league_id}, {'$set': league_data},
                     #                                 upsert=True)
@@ -96,7 +93,7 @@ class Roll_Zuqiupipeline(object):
             match_dict['data'] = match_list
             res = Helper.async_post(MATCH_URL, match_dict)
             if res:
-                if "成功" in res:
+                if res.get('status') == 1:
                     self.db.zq_competition35.insert(match_data)
                     # self.db.zq_competition35.update({'match_id': match_id, 'is_rollball': 1}, {'$set': match_data},
                     #                                 upsert=True)
@@ -541,7 +538,7 @@ class Roll_Zuqiupipeline(object):
         if data_list:
             res = Helper.async_post(ODDS_URL, odds_dict)
             if res:
-                if "成功" in res:
+                if res.get('status') == 1:
                     logger.info('足球滚球详细赔率提交成功, {}'.format(res))
                     logger.info(odds_dict)
                 else:
@@ -572,7 +569,7 @@ class Roll_Zuqiupipeline(object):
             try:
                 res = Helper.async_post(MATCH_RESULT, r_data_dict)
                 if res:
-                    if "成功" in res:
+                    if res.get('status') == 1:
                         logger.info('足球滚球结果记录提交成功, {}'.format(res))
                         logger.info(r_data_dict)
                     else:
@@ -591,7 +588,7 @@ class Roll_Zuqiupipeline(object):
         # status_dict['data'] = data_list
         # res = Helper.async_post(MATCH_STATUS, status_dict)
         # if res:
-        #     if "成功" in res:
+        #     if res.get('status') == 1:
         #         self.db.match_status35.insert(status_dict)
         #         logger.info('{},赛事结果状态交成功, {}'.format('zq', res))
         #         logger.info(status_dict)
@@ -625,7 +622,7 @@ class Roll_Zuqiupipeline(object):
                 match_dict['data'] = match_list
                 res = Helper.async_post(MATCH_URL, match_dict)
                 if res:
-                    if "成功" in res:
+                    if res.get('status') == 1:
                         self.db.zq_competition35.insert(match_data)
                         # self.db.zq_competition35.update({'match_id': match_id, 'is_rollball': 1}, {'$set': match_data},
                         #                                 upsert=True)
@@ -1069,7 +1066,7 @@ class Roll_Zuqiupipeline(object):
             if data_list:
                 res = Helper.async_post(ODDS_URL, odds_dict)
                 if res:
-                    if "成功" in res:
+                    if res.get('status') == 1:
                         logger.info('足球滚球, 角球详细赔率提交成功, {}'.format(res))
                         logger.info(odds_dict)
                     else:
@@ -1100,7 +1097,7 @@ class Roll_Zuqiupipeline(object):
                 try:
                     res = Helper.async_post(MATCH_RESULT, r_data_dict)
                     if res:
-                        if "成功" in res:
+                        if res.get('status') == 1:
                             logger.info('足球滚球, 角球结果记录提交成功, {}'.format(res))
                             logger.info(r_data_dict)
                         else:

hg3535/pipeline/saiguo.py

@@ -6,9 +6,10 @@ import logging
 # import redis
 from twisted.internet import defer, reactor
 
-from ..settings import M_HOST, M_USER, M_PASSWORD, M_POST, M_DB, MATCH_STATUS
+from ..settings import M_HOST, M_USER, M_PASSWORD, M_POST, M_DB, SAIGUO_RESULT
 from ..utils.helper import Helper
 
+
 # 滚球足球 插入
 class Jieshuqiupipeline(object):
 
@@ -17,44 +18,104 @@ class Jieshuqiupipeline(object):
         self.db = self.mongo[M_DB]
         # cls = redis.Redis(host='192.168.0.110', port=6379, db=0)
 
-    @defer.inlineCallbacks
-    def process_item(self, item, spider):
-        out = defer.Deferred()
-        reactor.callInThread(self._do_calculation, item, out)
-        yield out
-        defer.returnValue(item)
-
+    # @defer.inlineCallbacks
     # def process_item(self, item, spider):
-    def _do_calculation(self, item, out):
+    #     out = defer.Deferred()
+    #     reactor.callInThread(self._do_calculation, item, out)
+    #     yield out
+        # defer.returnValue(item)
+
+    def process_item(self, item, spider):
+    # def _do_calculation(self, item, out):
         logger = logging.getLogger(__name__)
-        match_id, score_other = item['id_score']
         pt = item['pt']
+        league_id = item["league_id"]
+        league_name = item["league_name"]
+        match_id = item["match_id"]
+        match_date = item["match_date"]
+        match_time = item["match_time"]
+        h_name = item["home_team"]
+        a_name = item["guest_team"]
+        play_data = item["play_data"]
         if pt == 1:
-            game_code = "zq"
+            h_score = item["score_half"]
+            f_score = item["score_full"]
+            post_data = {'game_code': 'zq', "title": "result", "source": "hg3535", "data": []}
+            data = {"league_id": league_id, "league_name": league_name, "match_date": match_date,
+                    "match_time": match_time, "home_team": h_name, "guest_team": a_name, "score_half": h_score,
+                    "score_full": f_score, "play_data": play_data, "match_id": match_id}
+            post_data['data'].append(data)
+            res = Helper.async_post(SAIGUO_RESULT, post_data)
+            if res:
+                if res.get('status') == 1:
+                    self.db.zq_saiguo35.insert(post_data)
+                    logger.info('{},赛事结果状态交成功, {}'.format('zq', res))
+                    logger.info(post_data)
+                else:
+                    logger.warning('{},赛事结果状态交失败, {}'.format('zq', res))
+                    logger.warning(post_data)
+            else:
+                logger.warning('{},赛事结果状态接口异常提交失败, {}'.format('zq', res))
+                logger.warning(post_data)
         elif pt == 2:
-            game_code = "lq"
+            h_score = item["score_half"]
+            f_score = item["score_result"]
+            x_score = item["score_below"]
+            post_data = {'game_code': 'lq', "title": "result", "source": "hg3535", "data": []}
+            data = {"league_id": league_id, "league_name": league_name, "match_date": match_date,
+                    "match_time": match_time, "home_team": h_name, "match_id": match_id,
+                    "guest_team": a_name, "score_half": h_score, "score_below": x_score, "score_result": f_score,
+                    "play_data": play_data}
+            post_data['data'].append(data)
+            res = Helper.async_post(SAIGUO_RESULT, post_data)
+            if res:
+                if res.get('status') == 1:
+                    self.db.lq_saiguo35.insert(post_data)
+                    logger.info('{},赛事结果状态交成功, {}'.format('lq', res))
+                    logger.info(post_data)
+                else:
+                    logger.warning('{},赛事结果状态交失败, {}'.format('lq', res))
+                    logger.warning(post_data)
+            else:
+                logger.warning('{},赛事结果状态接口异常提交失败, {}'.format('lq', res))
+                logger.warning(post_data)
         elif pt == 3:
-            game_code = "wq"
-        else:
-            game_code = "bq"
-        if self.db.match_status35.find({'match_id': match_id}).count() < 1:
-            status_dict = {"game_code": game_code, "title": "match_status", "source": "hg3535"}
-            data_list = []
-            data = {'match_id': match_id, 'status': 2, "is_rollball": 0, "is_today": 0, "is_morningplate": 0, "is_stringscene": 0, "is_horn": 0}
-            data_list.append(data)
-            status_dict['data'] = data_list
-            res = Helper.async_post(MATCH_STATUS, status_dict)
+            f_score = item["score_result"]
+            post_data = {'game_code': 'wq', "title": "result", "source": "hg3535", "data": []}
+            data = {"league_id": league_id, "league_name": league_name, "match_date": match_date,
+                    "match_time": match_time, "home_team": h_name, "guest_team": a_name, "score_result": f_score,
+                    "play_data": play_data}
+            post_data['data'].append(data)
+            res = Helper.async_post(SAIGUO_RESULT, post_data)
             if res:
-                if "成功" in res:
-                    self.db.match_status35.insert(status_dict)
-                    logger.info('{},赛事结果状态交成功, {}'.format(game_code, res))
-                    logger.info(status_dict)
+                if res.get('status') == 1:
+                    self.db.wq_saiguo35.insert(post_data)
+                    logger.info('{},赛事结果状态交成功, {}'.format('wq', res))
+                    logger.info(post_data)
                 else:
-                    logger.warning('{},赛事结果状态交失败, {}'.format(game_code, res))
-                    logger.warning(status_dict)
+                    logger.warning('{},赛事结果状态交失败, {}'.format('wq', res))
+                    logger.warning(post_data)
             else:
-                logger.warning('{},赛事结果状态接口异常提交失败, {}'.format(game_code, res))
-                logger.warning(status_dict)
+                logger.warning('{},赛事结果状态接口异常提交失败, {}'.format('wq', res))
+                logger.warning(post_data)
         else:
-            logger.info('足球赛事结果状态已存在，不提交')
-        reactor.callFromThread(out.callback, item)
+            h_score = item["score_half"]
+            f_score = item["score_full"]
+            post_data = {'game_code': 'bq', "title": "result", "source": "hg3535", "data": []}
+            data = {"league_id": league_id, "league_name": league_name, "match_date": match_date,
+                    "match_time": match_time, "home_team": h_name, "guest_team": a_name, "score_half": h_score,
+                    "score_full": f_score, "play_data": play_data}
+            post_data['data'].append(data)
+            res = Helper.async_post(SAIGUO_RESULT, post_data)
+            if res:
+                if res.get('status') == 1:
+                    self.db.bq_saiguo35.insert(post_data)
+                    logger.info('{},赛事结果状态交成功, {}'.format('bq', res))
+                    logger.info(post_data)
+                else:
+                    logger.warning('{},赛事结果状态交失败, {}'.format('bq', res))
+                    logger.warning(post_data)
+            else:
+                logger.warning('{},赛事结果状态接口异常提交失败, {}'.format('bq', res))
+                logger.warning(post_data)
+        # reactor.callFromThread(out.callback, item)

hg3535/pipeline/wangqiu.py

@@ -8,7 +8,7 @@ from twisted.internet import defer, reactor
 # from .ball_func import fuhao
 from ..utils.helper import Helper
 from .ball_func import new_time, out_time
-from ..settings import M_HOST, M_USER, M_PASSWORD, M_POST, M_DB, LEAGUE_URL, ODDS_URL, MATCH_URL, MATCH_RESULT
+from ..settings import M_HOST, M_USER, M_PASSWORD, M_POST, M_DB, LEAGUE_URL, ODDS_URL, MATCH_URL
 
 
 class Wangqiupipeline(object):
@@ -108,7 +108,7 @@ class Wangqiupipeline(object):
             league_dict['data'] = league_list
             res = Helper.async_post(LEAGUE_URL, league_dict)
             if res:
-                if '成功' in res:
+                if res.get('status') == 1:
                     self.db.wq_league35.insert(league_data)
                     # self.db.wq_league35.update({'lg_id': league_id}, {'$set': league_data},
                     #                                 upsert=True)
@@ -153,7 +153,7 @@ class Wangqiupipeline(object):
             match_dict['data'] = match_list
             res = Helper.async_post(MATCH_URL, match_dict)
             if res:
-                if '成功' in res:
+                if res.get('status') == 1:
                     self.db.wq_competition35.insert(match_data)
                     # self.db.wq_competition35.update({'match_id': match_id, pt_status: 1}, {'$set': match_data},
                     #                                 upsert=True)
@@ -263,7 +263,7 @@ class Wangqiupipeline(object):
         if data_list:
             res = Helper.async_post(ODDS_URL, odds_dict)
             if res:
-                if "成功" in res:
+                if res.get('status') == 1:
                     logger.info('网球赔率提交成功, {}'.format(res))
                     logger.info(odds_dict)
                 else:

hg3535/pipeline/zuqiu.py

@@ -72,7 +72,7 @@ class Zuqiupipeline(object):
             league_dict['data'] = league_list
             res = Helper.async_post(LEAGUE_URL, league_dict)
             if res:
-                if "成功" in res:
+                if res.get('status') == 1:
                     self.db.zq_league35.insert(league_data)
                     # self.db.zq_league35.update({'lg_id': league_id}, {'$set': league_data},
                     #                                 upsert=True)
@@ -111,7 +111,7 @@ class Zuqiupipeline(object):
             match_dict['data'] = match_list
             res = Helper.async_post(MATCH_URL, match_dict)
             if res:
-                if "成功" in res:
+                if res.get('status') == 1:
                     self.db.zq_competition35.insert(match_data)
                     # self.db.zq_competition35.update({'match_id': match_id, pt_status: 1}, {'$set': match_data},
                     #                                 upsert=True)
@@ -557,7 +557,7 @@ class Zuqiupipeline(object):
         if data_list:
             res = Helper.async_post(ODDS_URL, odds_dict)
             if res:
-                if "成功" in res:
+                if res.get('status') == 1:
                     logger.info('足球详细赔率提交成功, {}'.format(res))
                     logger.info(odds_dict)
                 else:
@@ -587,7 +587,7 @@ class Zuqiupipeline(object):
                 match_dict['data'] = match_list
                 res = Helper.async_post(MATCH_URL, match_dict)
                 if res:
-                    if "成功" in res:
+                    if res.get('status') == 1:
                         self.db.zq_competition35.insert(match_data)
                         # self.db.zq_competition35.update({'match_id': match_id, pt_status: 1}, {'$set': match_data},
                         #                                 upsert=True)
@@ -1046,7 +1046,7 @@ class Zuqiupipeline(object):
             if data_list:
                 res = Helper.async_post(ODDS_URL, odds_dict)
                 if res:
-                    if "成功" in res:
+                    if res.get('status') == 1:
                         logger.info('足球角球, 详细赔率提交成功, {}'.format(res))
                         logger.info(odds_dict)
                     else:

hg3535/settings.py

@@ -153,6 +153,7 @@ TOKEN_URL = "http://stadmin.bocai108.com/getToken"
 MATCH_RESULT = "http://stadmin.bocai108.com:19093/setMatchResult"
 MATCH_STATUS = "http://stadmin.bocai108.com:19093/upMatch"
 ODDSCH = "http://stadmin.bocai108.com:19093/setOddsCH"
+SAIGUO_RESULT = "http://stadmin.bocai108.com/setResultExpress"
 
 
 R_HOST = '192.168.2.200'

hg3535/spiders/saiguo.py

@@ -4,7 +4,8 @@ import re
 import scrapy
 from lxml import etree
 
-# from ..items import Hgsaiguo
+from ..items import Hgsaiguo
+
 
 
 class HgjieshuSpider(scrapy.Spider):
@@ -21,12 +22,19 @@ class HgjieshuSpider(scrapy.Spider):
 
     def start_requests(self):
         for y in range(1, 5):
-            url = 'https://www.hg3535.cn/zh-cn/info-centre/sportsbook-info/results/{}/normal/3'.format(y)
-            yield scrapy.Request(url=url, callback=self.parse, dont_filter=True, meta={'pt': y})
+            for z in range(1, 3):
+                url = 'https://www.hg3535.cn/zh-cn/info-centre/sportsbook-info/results/{}/normal/{}'.format(y, z)
+                yield scrapy.Request(url=url, callback=self.parse, dont_filter=True, meta={'pt': y, "page": z})
 
     def parse(self, response):
         if response.status == 200:
             pt = response.meta['pt']
+            page = response.meta['page']
+            if page == 1:
+                us_datetime = datetime.datetime.now() - datetime.timedelta(hours=12)
+            else:
+                us_datetime = datetime.datetime.now() - datetime.timedelta(hours=36)
+            match_date = us_datetime.strftime("%Y-%m-%d")
             # 足球赛果
             if pt == 1:
                 league_ids = response.xpath('//div[@class="rt-l-bar football"]/@id').extract()
@@ -35,7 +43,6 @@ class HgjieshuSpider(scrapy.Spider):
                     league_id = league_ids[index]
                     league_name = league_names[index]
                     response_data = response.xpath('//div[@id="dt-{}"]'.format(league_id)).extract_first()
-                    # response_data = response.xpath('//div[@id="dt-{}"]'.format('cmp-36254')).extract_first()
                     data = etree.HTML(response_data)
                     # 球队名
                     team_names = data.xpath('//div[@class="rt-event"]/@title')
@@ -63,14 +70,14 @@ class HgjieshuSpider(scrapy.Spider):
                         h_name = team_name[0]
                         # 客队
                         a_name = team_name[1]
-                        print(h_name, a_name)
                         # 上半场
                         h_score = h_scores[y].xpath('string(.)').replace(' ', '').replace('\r\n', '')
                         # 全场
                         f_score = f_scores[y].xpath('string(.)').replace(' ', '').replace('\r\n', '')
                         # 正则时间匹配规则
                         pattern = re.compile(r"\d{1,3}:\d{1,3}")
-                        stime = pattern.findall(stimes[y])[0]
+                        match_time = pattern.findall(stimes[y])[0]
+                        play_datas = []
                         if odd_names:
                             for i in range(len(odd_names)):
                                 name = odd_names[i].text
@@ -79,8 +86,21 @@ class HgjieshuSpider(scrapy.Spider):
                                     play = '{}&&{}'.format(plays[0], plays[1])
                                 else:
                                     play = plays[0]
-                        else:
-                            pass
+                                play_datas.append({'play_name': name, 'play_result': play})
+                        item = Hgsaiguo()
+                        item["league_id"] = league_id
+                        item["league_name"] = league_name
+                        item["match_id"] = match_id
+                        item["match_date"] = match_date
+                        item["match_time"] = match_time
+                        item["home_team"] = h_name
+                        item["guest_team"] = a_name
+                        item["score_half"] = h_score
+                        item["score_full"] = f_score
+                        item["play_data"] = play_datas
+                        item["pt"] = pt
+                        item["page"] = page
+                        yield item
 
             # 篮球赛果
             if pt == 2:
@@ -90,7 +110,6 @@ class HgjieshuSpider(scrapy.Spider):
                     league_id = league_ids[index]
                     league_name = league_names[index]
                     response_data = response.xpath('//div[@id="dt-{}"]'.format(league_id)).extract_first()
-                    # response_data = response.xpath('//div[@id="dt-{}"]'.format('cmp-26405')).extract_first()
                     data = etree.HTML(response_data)
                     # 球队名
                     team_names = data.xpath('//div[@class="rt-event"]/@title')
@@ -114,7 +133,6 @@ class HgjieshuSpider(scrapy.Spider):
                         h_name = team_name[0]
                         # 客队
                         a_name = team_name[1]
-                        print(h_name, a_name)
                         # 上半场
                         h_score = h_scores[y].xpath('string(.)').replace(' ', '').replace('\r\n', '')
                         # 全场
@@ -123,25 +141,47 @@ class HgjieshuSpider(scrapy.Spider):
                         x_score = x_scores[y].xpath('string(.)').replace(' ', '').replace('\r\n', '')
                         # 正则时间匹配规则
                         pattern = re.compile(r"\d{1,3}:\d{1,3}")
-                        stime = pattern.findall(stimes[y])[0]
+                        match_time = pattern.findall(stimes[y])[0]
+                        play_datas = []
                         if child_data:
+                            h_dict = {'team_name': h_name}
+                            a_dict = {'team_name': a_name}
                             for i in range(len(child_data)):
                                 if i == 0:
                                     h_datas = child_data[i].xpath('.//td/table/tbody/tr[3]/td[@class="r-odds"]')
                                     a_datas = child_data[i].xpath('.//td/table/tbody/tr[4]/td[@class="r-odds"]')
+                                    rule = {0: "sc_1th", 1: "sc_2th", 2: "sc_3th", 3: "sc_4th", 4: "sc_other"}
                                     if h_datas and a_datas:
                                         for x in range(len(h_datas)):
                                             # 主队节得分
                                             h_data = h_datas[x].text.replace(' ', '').replace('\r\n', '')
+                                            h_dict[rule[x]] = h_data
                                             # 客队节得分
                                             a_data = a_datas[x].text.replace(' ', '').replace('\r\n', '')
+                                            a_dict[rule[x]] = a_data
                                 else:
                                     # 子玩法名
                                     child_name = child_data[i].xpath('.//td[contains(@class, "r-bt ")]/text()')[0].replace(' ', '').replace('\r\n', '')
                                     # 子玩法结果
                                     child_play = child_data[i].xpath('.//td[@class="r-odds"]/span[@class="prop"]/text()')[0]
-                        else:
-                            pass
+                                    play_datas.append({"play_name": child_name, "play_result": child_play})
+                            play_datas.append(h_dict)
+                            play_datas.append(a_dict)
+                        item = Hgsaiguo()
+                        item["league_id"] = league_id
+                        item["league_name"] = league_name
+                        item["match_id"] = match_id
+                        item["match_date"] = match_date
+                        item["match_time"] = match_time
+                        item["home_team"] = h_name
+                        item["guest_team"] = a_name
+                        item["score_half"] = h_score
+                        item["score_result"] = f_score
+                        item["play_data"] = play_datas
+                        item["pt"] = pt
+                        item["page"] = page
+                        item["score_below"] = x_score
+                        yield item
 
             # 网球赛果
             if pt == 3:
@@ -151,7 +191,6 @@ class HgjieshuSpider(scrapy.Spider):
                     league_id = league_ids[index]
                     league_name = league_names[index]
                     response_data = response.xpath('//div[@id="dt-{}"]'.format(league_id)).extract_first()
-                    # response_data = response.xpath('//div[@id="dt-{}"]'.format('cmp-26405')).extract_first()
                     data = etree.HTML(response_data)
                     # 球队名
                     team_names = data.xpath('//div[@class="rt-event"]/@title')
@@ -173,13 +212,16 @@ class HgjieshuSpider(scrapy.Spider):
                         h_name = team_name[0]
                         # 客队
                         a_name = team_name[1]
-                        print(h_name, a_name)
                         # 全场
                         f_score = f_scores[y].xpath('string(.)').replace(' ', '').replace('\r\n', '')
                         # 正则时间匹配规则
                         pattern = re.compile(r"\d{1,3}:\d{1,3}")
-                        stime = pattern.findall(stimes[y])[0]
+                        match_time = pattern.findall(stimes[y])[0]
+                        play_datas = []
                         if child_data:
+                            rule = {0: "sc_1th", 1: "sc_2th", 2: "sc_3th", 3: "sc_4th", 4: "sc_5th", 5: "game_num", 6: "disc_num"}
+                            h_dict = {'team_name': h_name}
+                            a_dict = {'team_name': a_name}
                             for i in range(len(child_data)):
                                 if i == 0:
                                     h_datas = child_data[i].xpath('.//tbody/tr[3]/td[contains(@class, "r-odds")]')
@@ -188,17 +230,32 @@ class HgjieshuSpider(scrapy.Spider):
                                         for x in range(len(h_datas)):
                                             # 主队节得分
                                             h_data = h_datas[x].text.replace(' ', '').replace('\r\n', '')
+                                            h_dict[rule[x]] = h_data
                                             # 客队节得分
                                             a_data = a_datas[x].text.replace(' ', '').replace('\r\n', '')
+                                            a_dict[rule[x]] = a_data
                                 else:
                                     # 子玩法名
                                     child_name = child_data[i].xpath('.//td[contains(@class, "r-bt ")]/text()')[0].replace(' ', '').replace('\r\n', '')
                                     # 子玩法结果
                                     child_play = child_data[i].xpath('.//td[@class="r-odds"]/span[@class="prop"]')[0]
                                     play = child_play.xpath('string(.)')
-                                    print(child_name, play)
-                        else:
-                            pass
+                                    play_datas.append({"play_name": child_name, "play_result": play})
+                            play_datas.append(h_dict)
+                            play_datas.append(a_dict)
+                        item = Hgsaiguo()
+                        item["league_id"] = league_id
+                        item["league_name"] = league_name
+                        item["match_id"] = match_id
+                        item["match_date"] = match_date
+                        item["match_time"] = match_time
+                        item["home_team"] = h_name
+                        item["guest_team"] = a_name
+                        item["score_result"] = f_score
+                        item["play_data"] = play_datas
+                        item["pt"] = pt
+                        item["page"] = page
+                        yield item
 
             # 棒球赛果
             if pt == 4:
@@ -208,7 +265,6 @@ class HgjieshuSpider(scrapy.Spider):
                     league_id = league_ids[index]
                     league_name = league_names[index]
                     response_data = response.xpath('//div[@id="dt-{}"]'.format(league_id)).extract_first()
-                    # response_data = response.xpath('//div[@id="dt-{}"]'.format('cmp-26753')).extract_first()
                     data = etree.HTML(response_data)
                     # 球队名
                     team_names = data.xpath('//div[@class="rt-event"]/@title')
@@ -236,19 +292,32 @@ class HgjieshuSpider(scrapy.Spider):
                         h_name = team_name[0]
                         # 客队
                         a_name = team_name[1]
-                        print(h_name, a_name)
                         # 上半场
                         h_score = h_scores[y].xpath('string(.)').replace(' ', '').replace('\r\n', '')
                         # 全场
                         f_score = f_scores[y].xpath('string(.)').replace(' ', '').replace('\r\n', '')
                         # 正则时间匹配规则
                         pattern = re.compile(r"\d{1,3}:\d{1,3}")
-                        stime = pattern.findall(stimes[y])[0]
+                        match_time = pattern.findall(stimes[y])[0]
+                        play_datas = []
                         if odd_names:
                             for i in range(len(odd_names)):
                                 # 子玩法名
                                 name = odd_names[i].text.replace(' ', '').replace('\r\n', '')
                                 # 子玩法赛果
                                 play = odd_plays[i].xpath('string(.)').replace(' ', '').replace('\r\n', '')
-                        else:
-                            pass
+                                play_datas.append({"play_name": name, "play_result": play})
+                        item = Hgsaiguo()
+                        item["league_id"] = league_id
+                        item["league_name"] = league_name
+                        item["match_id"] = match_id
+                        item["match_date"] = match_date
+                        item["match_time"] = match_time
+                        item["home_team"] = h_name
+                        item["guest_team"] = a_name
+                        item["score_half"] = h_score
+                        item["score_full"] = f_score
+                        item["play_data"] = play_datas
+                        item["pt"] = pt
+                        item["page"] = page
+                        yield item

hg3535/utils/helper.py

@@ -11,7 +11,7 @@ class Helper(object):
             fs_session = FuturesSession()
             data = fs_session.post(url, data={"data": json.dumps(params), "token": "u4Gdf015662654065d5b503ea2517"}, timeout=180).result()
             if data:
-                return data.content.decode('utf-8')
+                return data.json()
         except requests.exceptions.RequestException as e:
             print(e)