juan 6 years ago
parent
commit
0ee2f50d0a

+ 1 - 662
hgg070_spider/pipelines/zuqiu.py

@@ -1,4 +1,3 @@
-<<<<<<< HEAD
 from twisted.internet import defer,reactor
 from ..utils.helper import Helper
 
@@ -16,664 +15,4 @@ class ZuqiuPipeline(object):
                "team_c":item["team_c"],"ior_RTS2Y":item["ior_RTS2Y"],"ior_RTS2N":item["ior_RTS2N"],
                "ior_REH":item["ior_REH"],"ior_REC":item["ior_REC"],"ior_ROUC":item["ior_ROUC"],
                "ior_HROUH":item["ior_HROUH"],"ratio_rouo":item["ratio_rouo"],"ratio_rouu":item["ratio_rouu"]}
-        data=Helper.async_post("")
-=======
-import datetime
-import time
-import logging
-import pymongo
-from twisted.internet import defer, reactor
-from ..utils.helper import Helper
-# from .ball_func import new_time
-from ..settings import M_HOST, M_USER, M_PASSWORD, M_POST, M_DB, LEAGUE_URL, ODDS_URL, MATCH_URL
-
-
-class Zuqiupipeline(object):
-    def open_spider(self, spider):
-        self.mongo = pymongo.MongoClient(host=M_HOST, username=M_USER, password=M_PASSWORD, port=M_POST, authSource=M_DB)
-        self.db = self.mongo[M_DB]
-
-    # @defer.inlineCallbacks
-    # def process_item(self, item, spider):
-    #     out = defer.Deferred()
-    #     reactor.callInThread(self._do_calculation, item, out)
-    #     yield out
-    #     # defer.returnValue(item)
-
-    def process_item(self, item, spider):
-    # def _do_calculation(self, item, out):
-        logger = logging.getLogger(__name__)
-        # 比赛日期
-        try:
-            data_game = item['data_game'].split("/")
-            month = str(data_game[1].strip())
-            day = str(data_game[0].strip())
-        except Exception as e:
-            logger.warning(e)
-            data_game = item['data_game'].split(" ")
-            months = str(data_game[1].strip())
-            month_dict = {'Jan': '01', 'Feb': '02', 'Mar': '03', 'Apr': '04', 'May': '05', 'Jun': '06', 'Jul': '07',
-                          'Aug': '08', 'Sep': '09', 'Oct': '10', 'Nov': '11', 'Dec': '12'}
-            month = month_dict[months]
-            day = str(data_game[0]).strip()
-
-        # 比赛时间
-        time_game = str(item['time_game'])
-        # 比赛时间,时间戳
-        us_time = str(datetime.datetime.now().year) + "-" + month + "-" + day + " " + time_game + ":00"
-        # 美东日期
-        # new_us_time = us_time.split(' ')[0]
-        # 现在时间,时间戳
-        utime = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime())
-        # 比赛id
-        match_id = item['game_id']
-        # 联赛id
-        league_id = item['league_id']
-        # 联赛name
-        league_name = item['league_name']
-        # # 主队
-        team_home = item['team_home']
-        # 客队
-        team_guest = item['team_guest']
-        # number
-        tag_number = item['number']
-        pt = str(item['pt'])
-        # match_date, match_time, time3 = new_time(us_time)
-        # 联赛接口写入
-        last_time = '{}-12-31 23:59:59'.format(datetime.datetime.now().year)
-        league_list = []
-        # uuid = Helper.genearte_uuid(league_name + 'hg3535')
-        uuid = Helper.genearte_uuid(league_name)
-        if self.db.zq_league35.find({'lg_id': league_id}).count() < 1:
-        # if self.db.zq_league35.find({'uuid': uuid}).count() < 1:
-            league_dict = {"game_code": "zq", "title": "league", "source": "hg3535"}
-            league_key = ["name_chinese", "kind", "match_mode", "if_stop", "last_time", "lg_id", "source", "uuid"]
-            league_value = [league_name, "1", "1", "0", last_time, league_id, "hg3535", uuid]
-            league_data = dict(zip(league_key, league_value))
-            league_list.append(league_data)
-            league_dict['data'] = league_list
-            res = Helper.async_post(LEAGUE_URL, league_dict)
-            if res:
-                if res.get('status') == 1:
-                    self.db.zq_league35.insert(league_data)
-            else:
-                logging.warning('足球详细赔率接口异常, {}'.format(res))
-        else:
-            logging.info('{},联赛已存在, 不提交'.format(league_name))
-        pt_dict = {'1': 'is_today', '2': 'is_morningplate', '3': 'is_stringscene', '4': 'is_rollball'}
-        pt_status = pt_dict[pt]
-        if pt == '3':
-            is_rollball = 0
-            is_today = 0
-            is_morningplate = 0
-            is_stringscene = 1
-        elif pt == '2':
-            is_rollball = 0
-            is_today = 0
-            is_morningplate = 1
-            is_stringscene = 0
-        else:
-            is_today = 1
-            is_rollball = 0
-            is_morningplate = 0
-            is_stringscene = 0
-
-        match_list = []
-        match_identity = Helper.genearte_uuid(team_home + team_guest + match_date)
-        if self.db.zq_competition35.find({'match_id': match_id, pt_status: 1}).count() < 1:
-        # if self.db.zq_competition35.find({'match_identity': match_identity, pt_status: 1}).count() < 1:
-            match_dict = {"game_code": "zq", "title": "match", "source": "hg3535"}
-            match_kay = ["home_team", "guest_team", "lg_id", "status", "match_id", "match_date", "match_time",
-                         "tag", "source", "is_rollball", "is_morningplate", "is_stringscene", "us_time", "uuid",
-                         "half_match_id", "is_today", "is_horn", 'match_identity']
-            match_value = [team_home, team_guest, league_id, 0, match_id, match_date, match_time, tag_number,
-                           "hg3535", is_rollball, is_morningplate, is_stringscene, us_time, uuid, 0, is_today, 0, match_identity]
-            match_data = dict(zip(match_kay, match_value))
-            match_list.append(match_data)
-            match_dict['data'] = match_list
-            res = Helper.async_post(MATCH_URL, match_dict)
-            if res:
-                if res.get('status') == 1:
-                    self.db.zq_competition35.insert(match_data)
-                else:
-                    logger.warning('足球赛事表提交失败, {}'.format(res))
-                    # logger.warning(match_dict)
-            else:
-                logger.warning('足球赛事接口异常提交失败, {}'.format(res))
-                # logger.warning(match_dict)
-        else:
-            logger.info('足球赛事已存在,不提交')
-        #
-        # p_code = "GS"
-        # half_size_guest = item["half_size_guest"]
-        # half_size_guest_rule = item["half_size_guest_rule"]
-        # half_size_home = item["half_size_home"]
-        # half_size_home_rule = item["half_size_home_rule"]
-        # data_list = []
-        # odds_onlys = []
-        # # half_size_guest
-        # for index, value in enumerate(half_size_guest):
-        #     hash_str = p_code + "gss_h" + str(index) + str(half_size_guest_rule[index]) + str(
-        #         value) + "hg3535" + str(match_id)
-        #     sole_str = p_code + "gss_h" + str(index) + str(match_id) + "hg3535"
-        #     odds_only = Helper.genearte_MD5(hash_str, pt)
-        #     sole = Helper.genearte_MD5(sole_str, pt)
-        #     half_size_guest_key = ["match_id", "lg_id", "odds_code", "status", "sort", "p_code", "odds", "condition",
-        #                            "odds_only", "sole", "source", "type", "team"]
-        #     condition = half_size_guest_rule[index]
-        #     half_size_guest_value = [match_id, league_id, "gss_h", "0", index, p_code, value, condition,
-        #                              odds_only, sole, "hg3535", "0", ""]
-        #     half_size_guest_data = dict(zip(half_size_guest_key, half_size_guest_value))
-        #     data_list.append(half_size_guest_data)
-        # # half_size_home
-        # for index, value in enumerate(half_size_home):
-        #     hash_str = p_code + "gsb_h" + str(index) + str(half_size_home_rule[index]) + str(
-        #         value) + "hg3535" + str(match_id)
-        #     sole_str = p_code + "gsb_h" + str(index) + str(match_id) + "hg3535"
-        #     odds_only = Helper.genearte_MD5(hash_str, pt)
-        #     sole = Helper.genearte_MD5(sole_str, pt)
-        #     half_size_home_key = ["match_id", "lg_id", "odds_code", "status", "sort", "p_code", "odds", "condition",
-        #                           "odds_only", "sole", "source", "type", "team"]
-        #     condition = half_size_home_rule[index]
-        #     half_size_home_value = [match_id, league_id, "gsb_h", "0", index, p_code, value, condition,
-        #                             odds_only, sole, "hg3535", "0", ""]
-        #     half_size_home_data = dict(zip(half_size_home_key, half_size_home_value))
-        #     data_list.append(half_size_home_data)
-        #
-        # # 全场场大小
-        # size_guest = item["size_guest"]
-        # size_guest_rule = item["size_guest_rule"]
-        # size_home = item["size_home"]
-        # size_home_rule = item["size_home_rule"]
-        # # size_home
-        # for index, value in enumerate(size_home):
-        #     hash_str = p_code + "gsb" + str(index) + str(size_home_rule[index]) + str(value) + "hg3535" + str(
-        #         match_id)
-        #     sole_str = p_code + "gsb" + str(index) + str(match_id) + "hg3535"
-        #     odds_only = Helper.genearte_MD5(hash_str, pt)
-        #     sole = Helper.genearte_MD5(sole_str, pt)
-        #     size_home_key = ["match_id", "lg_id", "odds_code", "status", "sort", "p_code", "odds", "condition",
-        #                      "odds_only", "sole", "source", "type", "team"]
-        #     condition = size_home_rule[index]
-        #     size_home_value = [match_id, league_id, "gsb", "0", index, p_code, value, condition, odds_only,
-        #                        sole, "hg3535", "0", ""]
-        #     size_home_data = dict(zip(size_home_key, size_home_value))
-        #     data_list.append(size_home_data)
-        # # size_guest
-        # for index, value in enumerate(size_guest):
-        #     hash_str = p_code + "gss" + str(index) + str(size_guest_rule[index]) + str(value) + "hg3535" + str(
-        #         match_id)
-        #     sole_str = p_code + "gss" + str(index) + str(match_id) + "hg3535"
-        #     odds_only = Helper.genearte_MD5(hash_str, pt)
-        #     sole = Helper.genearte_MD5(sole_str, pt)
-        #     size_guest_key = ["match_id", "lg_id", "odds_code", "status", "sort", "p_code", "odds", "condition",
-        #                       "odds_only", "sole", "source", "type", "team"]
-        #     condition = size_guest_rule[index]
-        #     size_guest_value = [match_id, league_id, "gss", "0", index, p_code, value, condition, odds_only,
-        #                         sole, "hg3535", "0", ""]
-        #     size_home_data = dict(zip(size_guest_key, size_guest_value))
-        #     data_list.append(size_home_data)
-        #
-        # p_code = 'CO'
-        # half_concede_home_rule = item["half_concede_home_rule"]
-        # half_concede_home = item["half_concede_home"]
-        # half_concede_guest_rule = item["half_concede_guest_rule"]
-        # half_concede_guest = item["half_concede_guest"]
-        # # half_concede_home
-        # if half_concede_guest:
-        #     for index, value in enumerate(half_concede_guest):
-        #         hash_str = p_code + "cog_h" + str(index) + str(half_concede_guest_rule[index]) + str(
-        #             value) + "hg3535" + str(match_id)
-        #         sole_str = p_code + "cog_h" + str(index) + str(match_id) + "hg3535"
-        #         odds_only = Helper.genearte_MD5(hash_str, pt)
-        #         sole = Helper.genearte_MD5(sole_str, pt)
-        #         half_concede_guest_key = ["match_id", "lg_id", "odds_code", "status", "sort", "p_code", "odds", "condition",
-        #                                   "odds_only", "sole", "source", "type", "team"]
-        #         condition = half_concede_guest_rule[index]
-        #         half_concede_guest_value = [match_id, league_id, "cog_h", "0", index, p_code, value, condition,
-        #                                     odds_only, sole, "hg3535", "0", ""]
-        #         half_concede_guest_data = dict(zip(half_concede_guest_key, half_concede_guest_value))
-        #         data_list.append(half_concede_guest_data)
-        #
-        # # half_concede_home
-        # if half_concede_home:
-        #     for index, value in enumerate(half_concede_home):
-        #         hash_str = p_code + "coh_h" + str(index) + str(half_concede_home_rule[index]) + str(
-        #             value) + "hg3535" + str(match_id)
-        #         sole_str = p_code + "coh_h" + str(index) + str(match_id) + "hg3535"
-        #         odds_only = Helper.genearte_MD5(hash_str, pt)
-        #         sole = Helper.genearte_MD5(sole_str, pt)
-        #         half_concede_home_key = ["match_id", "lg_id", "odds_code", "status", "sort", "p_code", "odds", "condition",
-        #                                  "odds_only", "sole", "source", "type", "team"]
-        #         condition = half_concede_home_rule[index]
-        #         half_concede_home_value = [match_id, league_id, "coh_h", "0", index, p_code, value, condition,
-        #                                    odds_only, sole, "hg3535", "0", ""]
-        #         half_concede_home_data = dict(zip(half_concede_home_key, half_concede_home_value))
-        #         data_list.append(half_concede_home_data)
-        #
-        # concede_guest = item["concede_guest"]
-        # concede_guest_rule = item["concede_guest_rule"]
-        # concede_home = item["concede_home"]
-        # concede_home_rule = item["concede_home_rule"]
-        # # concede_guest
-        # if concede_guest:
-        #     for index, value in enumerate(concede_guest):
-        #         hash_str = p_code + "cog" + str(index) + str(concede_guest_rule[index]) + str(
-        #             value) + "hg3535" + str(match_id)
-        #         sole_str = p_code + "cog" + str(index) + str(match_id) + "hg3535"
-        #         odds_only = Helper.genearte_MD5(hash_str, pt)
-        #         sole = Helper.genearte_MD5(sole_str, pt)
-        #         concede_guest_key = ["match_id", "lg_id", "odds_code", "status", "sort", "p_code", "odds", "condition",
-        #                              "odds_only", "sole", "source", "type", "team"]
-        #         condition = concede_guest_rule[index]
-        #         concede_guest_value = [match_id, league_id, "cog", "0", index, p_code, value, condition,
-        #                                odds_only, sole, "hg3535", "0", ""]
-        #         concede_guest_data = dict(zip(concede_guest_key, concede_guest_value))
-        #         data_list.append(concede_guest_data)
-        # # concede_home
-        # if concede_home:
-        #     for index, value in enumerate(concede_home):
-        #         hash_str = p_code + "coh" + str(index) + str(concede_home_rule[index]) + str(
-        #             value) + "hg3535" + str(match_id)
-        #         sole_str = p_code + "coh" + str(index) + str(match_id) + "hg3535"
-        #         odds_only = Helper.genearte_MD5(hash_str, pt)
-        #         sole = Helper.genearte_MD5(sole_str, pt)
-        #         concede_home_key = ["match_id", "lg_id", "odds_code", "status", "sort", "p_code", "odds", "condition",
-        #                             "odds_only", "sole", "source", "type", "team"]
-        #         condition = concede_home_rule[index]
-        #         concede_home_value = [match_id, league_id, "coh", "0", index, p_code, value, condition, odds_only,
-        #                               sole, "hg3535", "0", ""]
-        #         concede_guest_data = dict(zip(concede_home_key, concede_home_value))
-        #         data_list.append(concede_guest_data)
-        #
-        # # 独赢----------------------------------------------------------------------------------------------------------
-        # p_code = 'C'
-        # half_capot_home = item["half_capot_home"]
-        # half_capot_guest = item["half_capot_guest"]
-        # half_capot_dogfall = item["half_capot_dogfall"]
-        # capot_home = item["capot_home"]
-        # capot_guest = item["capot_guest"]
-        # capot_dogfall = item["capot_dogfall"]
-        #
-        # # half_capot_home
-        # hash_str = p_code + "ch_h" + '0' + '1' + str(half_capot_home) + "hg3535" + str(match_id)
-        # sole_str = p_code + "ch_h" + '0' + str(match_id) + "hg3535"
-        # odds_only = Helper.genearte_MD5(hash_str, pt)
-        # sole = Helper.genearte_MD5(sole_str, pt)
-        # half_capot_home_key = ["match_id", "lg_id", "odds_code", "status", "sort", "p_code", "odds", "condition",
-        #                        "odds_only", "sole", "source", "type", "team"]
-        # half_capot_home_value = [match_id, league_id, "ch_h", "0", 0, p_code, half_capot_home, '1',
-        #                          odds_only, sole, "hg3535", "0", ""]
-        # half_capot_homet_data = dict(zip(half_capot_home_key, half_capot_home_value))
-        # data_list.append(half_capot_homet_data)
-        #
-        # # half_capot_guest
-        # hash_str = p_code + "cg_h" + '0' + '2' + str(half_capot_guest) + "hg3535" + str(match_id)
-        # sole_str = p_code + "cg_h" + '0' + str(match_id) + "hg3535"
-        # odds_only = Helper.genearte_MD5(hash_str, pt)
-        # sole = Helper.genearte_MD5(sole_str, pt)
-        # half_capot_guest_key = ["match_id", "lg_id", "odds_code", "status", "sort", "p_code", "odds", "condition",
-        #                         "odds_only", "sole", "source", "type", "team"]
-        # half_capot_guest_value = [match_id, league_id, "cg_h", "0", 0, p_code, half_capot_guest, '2',
-        #                           odds_only, sole, "hg3535", "0", ""]
-        # half_capot_guest_data = dict(zip(half_capot_guest_key, half_capot_guest_value))
-        # data_list.append(half_capot_guest_data)
-        #
-        # # half_capot_dogfall
-        # hash_str = p_code + "cd_h" + '0' + 'X' + str(half_capot_dogfall) + "hg3535" + str(match_id)
-        # sole_str = p_code + "cd_h" + '0' + str(match_id) + "hg3535"
-        # odds_only = Helper.genearte_MD5(hash_str, pt)
-        # sole = Helper.genearte_MD5(sole_str, pt)
-        # half_capot_dogfall_key = ["match_id", "lg_id", "odds_code", "status", "sort", "p_code", "odds", "condition",
-        #                           "odds_only", "sole", "source", "type", "team"]
-        # half_capot_dogfall_value = [match_id, league_id, "cd_h", "0", 0, p_code, half_capot_dogfall, 'X',
-        #                             odds_only, sole, "hg3535", "0", ""]
-        # half_capot_dogfall_data = dict(zip(half_capot_dogfall_key, half_capot_dogfall_value))
-        # data_list.append(half_capot_dogfall_data)
-        #
-        # # capot_dogfall
-        # hash_str = p_code + "cd" + '0' + 'X' + str(capot_dogfall) + "hg3535" + str(match_id)
-        # sole_str = p_code + "cd" + '0' + str(match_id) + "hg3535"
-        # odds_only = Helper.genearte_MD5(hash_str, pt)
-        # sole = Helper.genearte_MD5(sole_str, pt)
-        # capot_dogfall_key = ["match_id", "lg_id", "odds_code", "status", "sort", "p_code", "odds", "condition",
-        #                      "odds_only", "sole", "source", "type", "team"]
-        # capot_dogfall_value = [match_id, league_id, "cd", "0", 0, p_code, capot_dogfall, 'X', odds_only,
-        #                        sole, "hg3535", "0", ""]
-        # capot_dogfall_data = dict(zip(capot_dogfall_key, capot_dogfall_value))
-        # data_list.append(capot_dogfall_data)
-        #
-        # # capot_home
-        # hash_str = p_code + "ch" + '0' + '1' + str(capot_home) + "hg3535" + str(match_id)
-        # sole_str = p_code + "ch" + '0' + str(match_id) + "hg3535"
-        # odds_only = Helper.genearte_MD5(hash_str, pt)
-        # sole = Helper.genearte_MD5(sole_str, pt)
-        # capot_home_key = ["match_id", "lg_id", "odds_code", "status", "sort", "p_code", "odds", "condition",
-        #                   "odds_only", "sole", "source", "type", "team"]
-        # capot_home_value = [match_id, league_id, "ch", "0", 0, p_code, capot_home, '1',
-        #                     odds_only, sole, "hg3535", "0", ""]
-        # capot_homet_data = dict(zip(capot_home_key, capot_home_value))
-        # data_list.append(capot_homet_data)
-        #
-        # # capot_guest
-        # hash_str = p_code + "cg" + '0' + '2' + str(capot_guest) + "hg3535" + str(match_id)
-        # sole_str = p_code + "cg" + '0' + str(match_id) + "hg3535"
-        # odds_only = Helper.genearte_MD5(hash_str, pt)
-        # sole = Helper.genearte_MD5(sole_str, pt)
-        # capot_guest_key = ["match_id", "lg_id", "odds_code", "status", "sort", "p_code", "odds", "condition",
-        #                    "odds_only", "sole", "source", "type", "team"]
-        # capot_guest_value = [match_id, league_id, "cg", "0", 0, p_code, capot_guest, '2',
-        #                      odds_only, sole, "hg3535", "0", ""]
-        # capot_guest_data = dict(zip(capot_guest_key, capot_guest_value))
-        # data_list.append(capot_guest_data)
-        #
-        # # 入球数单双-----------------------------------------------------------------------------------------------------
-        # # p_code, p_id = get_pcode(corner_ball, 'two_sides')
-        # p_code = 'TS'
-        # odd_even_odd = item["odd_even_odd"]
-        # odd_even_even = item["odd_even_even"]
-        # half_odd_even_odd = item["half_odd_even_odd"]
-        # half_odd_even_even = item["half_odd_even_even"]
-        #
-        # # odd_even_odd
-        # hash_str = p_code + "tss" + '0' + '单' + str(odd_even_odd) + "hg3535" + str(match_id)
-        # sole_str = p_code + "tss" + '0' + str(match_id) + "hg3535"
-        # odds_only = Helper.genearte_MD5(hash_str, pt)
-        # sole = Helper.genearte_MD5(sole_str, pt)
-        # single_key = ["match_id", "lg_id", "odds_code", "status", "sort", "p_code", "odds", "condition",
-        #               "odds_only", "sole", "source", "type", "team"]
-        # single_value = [match_id, league_id, "tss", "0", 0, p_code, odd_even_odd, '单',
-        #                 odds_only, sole, "hg3535", "0", ""]
-        # single_data = dict(zip(single_key, single_value))
-        # data_list.append(single_data)
-        #
-        # # odd_even_even
-        # hash_str = p_code + "tsd" + '0' + '双' + str(odd_even_even) + "hg3535" + str(match_id)
-        # sole_str = p_code + "tsd" + '0' + str(match_id) + "hg3535"
-        # odds_only = Helper.genearte_MD5(hash_str, pt)
-        # sole = Helper.genearte_MD5(sole_str, pt)
-        # double_key = ["match_id", "lg_id", "odds_code", "status", "sort", "p_code", "odds", "condition",
-        #               "odds_only", "sole", "source", "type", "team"]
-        # double_value = [match_id, league_id, "tsd", "0", 0, p_code, odd_even_even, '双',
-        #                 odds_only, sole, "hg3535", "0", ""]
-        # double_data = dict(zip(double_key, double_value))
-        # data_list.append(double_data)
-        #
-        # # half_odd_even_even
-        # hash_str = p_code + "tsd_h" + '0' + '双' + str(half_odd_even_even) + "hg3535" + str(match_id)
-        # sole_str = p_code + "tsd_h" + '0' + str(match_id) + "hg3535"
-        # odds_only = Helper.genearte_MD5(hash_str, pt)
-        # sole = Helper.genearte_MD5(sole_str, pt)
-        # half_double_key = ["match_id", "lg_id", "odds_code", "status", "sort", "p_code", "odds", "condition",
-        #                    "odds_only", "sole", "source", "type", "team"]
-        # half_double_value = [match_id, league_id, "tsd_h", "0", 0, p_code, half_odd_even_even, '双',
-        #                      odds_only, sole, "hg3535", "0", ""]
-        # half_double_data = dict(zip(half_double_key, half_double_value))
-        # data_list.append(half_double_data)
-        #
-        # # half_odd_even_odd
-        # hash_str = p_code + "tss_h" + '0' + '单' + str(half_odd_even_odd) + "hg3535" + str(match_id)
-        # sole_str = p_code + "tss_h" + '0' + str(match_id) + "hg3535"
-        # odds_only = Helper.genearte_MD5(hash_str, pt)
-        # sole = Helper.genearte_MD5(sole_str, pt)
-        # half_single_key = ["match_id", "lg_id", "odds_code", "status", "sort", "p_code", "odds", "condition",
-        #                    "odds_only", "sole", "source", "type", "team"]
-        # half_single_value = [match_id, league_id, "tss_h", "0", 0, p_code, half_odd_even_odd, '单', odds_only,
-        #                      sole, "hg3535", "0", ""]
-        # half_single_data = dict(zip(half_single_key, half_single_value))
-        # data_list.append(half_single_data)
-        #
-        # # 总入球数 ------------------------------------------------------------------------------------------------------
-        # p_code = 'TG'
-        # total_goals = item['total_goal']
-        # total_dict = {'tg0': '0-1', 'tg1': '2-3', 'tg2': '4-6','tg3': '7或以上', 'tg0_h': '0', "tg1_h": '1',
-        #               "tg2_h": '2', "tg3_h": '3或以上'}
-        # # 全场入球数 单双
-        # # 上半场入球数 单双
-        # for key, value in total_goals.items():
-        #     hash_str = p_code + key + '0' + total_dict[key] + str(value) + "hg3535" + str(match_id)
-        #     sole_str = p_code + key + '0' + str(match_id) + "hg3535"
-        #     odds_only = Helper.genearte_MD5(hash_str, pt)
-        #     sole = Helper.genearte_MD5(sole_str, pt)
-        #     total_goals_key = ["match_id", "lg_id", "odds_code", "status", "sort", "p_code", "odds", "condition",
-        #                        "odds_only", "sole", "source", "type", "team"]
-        #     total_goals_value = [match_id, league_id, key, "0", 0, p_code, value, total_dict[key], odds_only,
-        #                          sole, "hg3535", "0", ""]
-        #     total_goals_data = dict(zip(total_goals_key, total_goals_value))
-        #     data_list.append(total_goals_data)
-        #
-        # # 全场半场 ------------------------------------------------------------------------------------------------------
-        # half_fulls = item['half_full']
-        # p_code = 'HF'
-        # full_dict = {"hfhh": "主主", "hfhd": "主和", "hfhg": "主客", "hfdh": "和主",
-        #              "hfdd": "和和", "hfdg": "和客", "hfgh": "客主", "hfgd": "客和", "hfgg": "客客"}
-        # if half_fulls:
-        #     for key, value in half_fulls.items():
-        #         hash_str = p_code + key + '0' + full_dict[key] + str(value) + "hg3535" + str(match_id)
-        #         sole_str = p_code + key + '0' + str(match_id) + "hg3535"
-        #         odds_only = Helper.genearte_MD5(hash_str, pt)
-        #         sole = Helper.genearte_MD5(sole_str, pt)
-        #         half_fulls_key = ["match_id", "lg_id", "odds_code", "status", "sort", "p_code", "odds", "condition",
-        #                            "odds_only", "sole", "source", "type", "team"]
-        #         half_fulls_value = [match_id, league_id, key, "0", 0, p_code, value, full_dict[key], odds_only,
-        #                              sole, "hg3535", "0", ""]
-        #         half_fulls_data = dict(zip(half_fulls_key, half_fulls_value))
-        #         data_list.append(half_fulls_data)
-        #
-        # # 波胆-----------------------------------------------------------------------------------------------------------
-        # bodan_datas = item['bodan_data']
-        # p_code = 'B'
-        # bodan_dict = {"b10": "1-0", "b20": "2-0", "b21": "2-1", "b30": "3-0", "b31": "3-1", "b32": "3-2",
-        #               "b40": "4-0", "b41": "4-1", "b42": "4-2", "b43": "4-3", "b01": "0-1", "b02": "0-2",
-        #               "b12": "1-2", "b03": "0-3", "b13": "1-3", "b23": "2-3", "b04": "0-4", "b14": "1-4",
-        #               "b24": "2-4", "b34": "3-4", "b00": "0-0", "b11": "1-1", "b22": "2-2", "b33": "3-3",
-        #               "b44": "4-4", "bo": "其他", "b10_h": "1-0", "b20_h": "2-0", "b21_h": "2-1", "b30_h": "3-0",
-        #               "b31_h": "3-1", "b32_h": "3-2", "b01_h": "0-1", "b02_h": "0-2", "b12_h": "1-2", "b03_h": "0-3",
-        #               "b13_h": "1-3", "b23_h": "2-3", "b00_h": "0-0", "b11_h": "1-1", "b22_h": "2-2", "b33_h": "3-3",
-        #               "bo_h": "其他"}
-        # if bodan_datas:
-        #     for key, value in bodan_datas.items():
-        #         hash_str = p_code + key + '0' + bodan_dict[key] + str(value) + "hg3535" + str(match_id)
-        #         sole_str = p_code + key + '0' + str(match_id) + "hg3535"
-        #         odds_only = Helper.genearte_MD5(hash_str, pt)
-        #         sole = Helper.genearte_MD5(sole_str, pt)
-        #         bodan_dict_key = ["match_id", "lg_id", "odds_code", "status", "sort", "p_code", "odds", "condition",
-        #                           "odds_only", "sole", "source", "type", "team"]
-        #         bodan_dict_value = [match_id, league_id, key, "0", 0, p_code, value, bodan_dict[key], odds_only,
-        #                             sole, "hg3535", "0", ""]
-        #         bodan_dict_data = dict(zip(bodan_dict_key, bodan_dict_value))
-        #         data_list.append(bodan_dict_data)
-        #
-        # # 最先进球/最后进球 ----------------------------------------------------------------------------------------------
-        # first_last_balls = item['first_last_ball']
-        # # p_code, p_id = get_pcode(corner_ball, 'first_last_ball')
-        # p_code = 'FLB'
-        # first_last_dict = {"flbfh": "最先进球", "flbfg": "最先进球", "flblh": "最后进球", "flblg": "最后进球", "flbn": "没有进球"}
-        # if first_last_balls:
-        #     for key, value in first_last_balls.items():
-        #         hash_str = p_code + key + '0' + first_last_dict[key] + str(value) + "hg3535" + str(match_id)
-        #         sole_str = p_code + key + '0' + str(match_id) + "hg3535"
-        #         odds_only = Helper.genearte_MD5(hash_str, pt)
-        #         sole = Helper.genearte_MD5(sole_str, pt)
-        #         first_last_balls_key = ["match_id", "lg_id", "odds_code", "status", "sort", "p_code", "odds", "condition",
-        #                           "odds_only", "sole", "source", "type", "team"]
-        #         first_last_balls_value = [match_id, league_id, key, "0", 0, p_code, value, first_last_dict[key], odds_only,
-        #                             sole, "hg3535", "0", ""]
-        #         first_last_balls_data = dict(zip(first_last_balls_key, first_last_balls_value))
-        #         data_list.append(first_last_balls_data)
-        #
-        # # 球队得分大小
-        # p_code = 'TB'
-        # # 全场
-        # full_dicts = item['full_data']
-        # # 上半场
-        # half_dicts = item['half_data']
-        # full_dict_rules = item['full_data_rule']
-        # half_dict_rules = item['half_data_rule']
-        # if full_dicts:
-        #     for key, value in full_dicts.items():
-        #         hash_str = p_code + key + '0' + full_dict_rules[key] + str(value) + "hg3535" + str(match_id)
-        #         sole_str = p_code + key + '0' + str(match_id) + "hg3535"
-        #         odds_only = Helper.genearte_MD5(hash_str, pt)
-        #         sole = Helper.genearte_MD5(sole_str, pt)
-        #         full_dict_rules_key = ["match_id", "lg_id", "odds_code", "status", "sort", "p_code", "odds",
-        #                                 "condition", "odds_only", "sole", "source", "type", "team"]
-        #         full_dict_rules_value = [match_id, league_id, key, "0", 0, p_code, value, full_dict_rules[key],
-        #                                   odds_only,sole, "hg3535", "0", ""]
-        #         full_dict_rules_data = dict(zip(full_dict_rules_key, full_dict_rules_value))
-        #         data_list.append(full_dict_rules_data)
-        #
-        # if half_dicts:
-        #     for key, value in half_dicts.items():
-        #         hash_str = p_code + key + '0' + half_dict_rules[key] + str(value) + "hg3535" + str(match_id)
-        #         sole_str = p_code + key + '0' + str(match_id) + "hg3535"
-        #         odds_only = Helper.genearte_MD5(hash_str, pt)
-        #         sole = Helper.genearte_MD5(sole_str, pt)
-        #         half_dicts_key = ["match_id", "lg_id", "odds_code", "status", "sort", "p_code", "odds",
-        #                         "condition","odds_only", "sole", "source", "type", "team"]
-        #         half_dicts_value = [match_id, league_id, key, "0", 0, p_code, value, half_dict_rules[key],
-        #                                  odds_only, sole, "hg3535", "0", ""]
-        #         half_dicts_data = dict(zip(half_dicts_key, half_dicts_value))
-        #         data_list.append(half_dicts_data)
-        #
-        # p_code = 'CB'
-        # horn_ou_dict = item['horn_ou_dict']
-        # horn_ou_dict_rule = item['horn_ou_dict_rule']
-        # horn_oe_dict = item['horn_oe_dict']
-        # horn_oe_dict_rule = item['horn_oe_dict_rule']
-        # if horn_ou_dict:
-        #     for key, value in horn_ou_dict.items():
-        #         hash_str = p_code + key + '0' + horn_ou_dict_rule[key] + str(value) + "hg3535" + str(match_id)
-        #         sole_str = p_code + key + '0' + str(match_id) + "hg3535"
-        #         odds_only = Helper.genearte_MD5(hash_str, pt)
-        #         sole = Helper.genearte_MD5(sole_str, pt)
-        #         horn_ou_key = ["match_id", "lg_id", "odds_code", "status", "sort", "p_code", "odds",
-        #                        "condition", "odds_only", "sole", "source", "type", "team"]
-        #         horn_ou_value = [match_id, league_id, key, "0", 0, p_code, value, horn_ou_dict_rule[key],
-        #                          odds_only, sole, "hg3535", "0", ""]
-        #         horn_ou_data = dict(zip(horn_ou_key, horn_ou_value))
-        #         data_list.append(horn_ou_data)
-        #
-        # if horn_oe_dict:
-        #     for key, value in horn_oe_dict.items():
-        #         hash_str = p_code + key + '0' + horn_oe_dict_rule[key] + str(value) + "hg3535" + str(match_id)
-        #         sole_str = p_code + key + '0' + str(match_id) + "hg3535"
-        #         odds_only = Helper.genearte_MD5(hash_str, pt)
-        #         sole = Helper.genearte_MD5(sole_str, pt)
-        #         horn_oe_key = ["match_id", "lg_id", "odds_code", "status", "sort", "p_code", "odds",
-        #                        "condition", "odds_only", "sole", "source", "type", "team"]
-        #         horn_oe_value = [match_id, league_id, key, "0", 0, p_code, value, horn_oe_dict_rule[key],
-        #                          odds_only, sole, "hg3535", "0", ""]
-        #         horn_oe_data = dict(zip(horn_oe_key, horn_oe_value))
-        #         data_list.append(horn_oe_data)
-        #
-        # if pt == '3':
-        #     ris_stringscene = 1
-        # else:
-        #     ris_stringscene = 0
-        # odds_key = ["game_code", "title", "match_id", "lg_id", "data", "source", "odds_only", "tag", "uuid", "is_stringscene", "utime", "pt", 'match_identity']
-        # odds_value = ["zq", "odds", match_id, league_id, data_list, "hg3535", odds_onlys, tag_number, uuid, ris_stringscene, utime, pt, match_identity]
-        # odds_dict = dict(zip(odds_key, odds_value))
-        # if data_list:
-        #     res = Helper.async_post(ODDS_URL, odds_dict)
-        #     if res:
-        #         if res.get('status') == 1:
-        #             logger.info('足球详细赔率提交成功, {}'.format(res))
-        #             # logger.info(odds_dict)
-        #         else:
-        #             logger.warning('足球相信赔率提交失败, {}'.format(res))
-        #             # logger.warning(odds_dict)
-        #     else:
-        #         logging.warning('足球详细赔率接口异常, {}'.format(res))
-        # else:
-        #     logger.info('足球详细赔率列表为空')
-        # # # 角球处理分割线---------------------------------------------------------------------------------------------------
-        # # horn_team = item['horn_team']
-        # # if horn_team:
-        # #     team_home = horn_team['horn_home']
-        # #     team_guest = horn_team['horn_guest']
-        # #     match_id = horn_team['horn_id']
-        # #     match_list = []
-        # #     match_identity = Helper.genearte_uuid(team_home + team_guest + new_us_time)
-        # #     # if self.db.zq_competition35.find({'match_id': match_id, pt_status: 1}).count() < 1:
-        # #     if self.db.zq_competition35.find({'match_identity': match_identity, pt_status: 1}).count() < 1:
-        # #         match_dict = {"game_code": "zq", "title": "match", "source": "hg3535"}
-        # #         match_kay = ["home_team", "guest_team", "lg_id", "status", "match_id", "match_date", "match_time",
-        # #                      "tag", "source", "is_rollball", "is_morningplate", "is_stringscene", "us_time", "uuid",
-        # #                      "half_match_id", "is_today", 'is_horn', 'match_identity']
-        # #         match_value = [team_home, team_guest, league_id, 0, match_id, match_date, match_time, tag_number,
-        # #                        "hg3535", is_rollball, is_morningplate, is_stringscene, us_time, uuid, 0, is_today, 1, match_identity]
-        # #         match_data = dict(zip(match_kay, match_value))
-        # #         match_list.append(match_data)
-        # #         match_dict['data'] = match_list
-        # #         res = Helper.async_post(MATCH_URL, match_dict)
-        # #         if res:
-        # #             if res.get('status') == 1:
-        # #                 self.db.zq_competition35.insert(match_data)
-        # #                 logger.info('足球角球, 赛事表提交失败, {}'.format(res))
-        # #                 # logger.info(match_dict)
-        # #             else:
-        # #                 logger.warning('足球角球, 赛事表提交失败, {}'.format(res))
-        # #                 # logger.warning(match_dict)
-        # #         else:
-        # #             logger.warning('足球角球, 赛事接口异常提交失败, {}'.format(res))
-        # #             # logger.warning(match_dict)
-        # #     else:
-        # #         logger.info('足球角球, 赛事已存在,不提交')
-        # #     data_list = []
-        # #     odds_onlys = []
-        # #     p_code = 'CB'
-        # #     horn_ou_dict = item['horn_ou_dict']
-        # #     horn_ou_dict_rule = item['horn_ou_dict_rule']
-        # #     horn_oe_dict = item['horn_oe_dict']
-        # #     horn_oe_dict_rule = item['horn_oe_dict_rule']
-        # #     if horn_ou_dict:
-        # #         for key, value in horn_ou_dict.items():
-        # #             hash_str = p_code + key + '0' + horn_ou_dict_rule[key] + str(value) + "hg3535" + str(match_id)
-        # #             sole_str = p_code + key + '0' + str(match_id) + "hg3535"
-        # #             odds_only = Helper.genearte_MD5(hash_str, pt)
-        # #             sole = Helper.genearte_MD5(sole_str, pt)
-        # #             horn_ou_key = ["match_id", "lg_id", "odds_code", "status", "sort", "p_code", "odds",
-        # #                            "condition", "odds_only", "sole", "source", "type", "team"]
-        # #             horn_ou_value = [match_id, league_id, key, "0", 0, p_code, value, horn_ou_dict_rule[key],
-        # #                              odds_only, sole, "hg3535", "0", ""]
-        # #             horn_ou_data = dict(zip(horn_ou_key, horn_ou_value))
-        # #             data_list.append(horn_ou_data)
-        # #
-        # #     if horn_oe_dict:
-        # #         for key, value in horn_oe_dict.items():
-        # #             hash_str = p_code + key + '0' + horn_oe_dict_rule[key] + str(value) + "hg3535" + str(match_id)
-        # #             sole_str = p_code + key + '0' + str(match_id) + "hg3535"
-        # #             odds_only = Helper.genearte_MD5(hash_str, pt)
-        # #             sole = Helper.genearte_MD5(sole_str, pt)
-        # #             horn_oe_key = ["match_id", "lg_id", "odds_code", "status", "sort", "p_code", "odds",
-        # #                            "condition", "odds_only", "sole", "source", "type", "team"]
-        # #             horn_oe_value = [match_id, league_id, key, "0", 0, p_code, value, horn_oe_dict_rule[key],
-        # #                              odds_only, sole, "hg3535", "0", ""]
-        # #             horn_oe_data = dict(zip(horn_oe_key, horn_oe_value))
-        # #             data_list.append(horn_oe_data)
-        # #
-        # #     if pt == '3':
-        # #         ris_stringscene = 1
-        # #     else:
-        # #         ris_stringscene = 0
-        # #     odds_key = ["game_code", "title", "match_id", "lg_id", "data", "source", "odds_only", "tag", "uuid",
-        # #                 "is_stringscene", "utime", "pt", 'match_identity']
-        # #     odds_value = ["zq", "odds", match_id, league_id, data_list, "hg3535", odds_onlys, tag_number, uuid,
-        # #                   ris_stringscene, utime, pt, match_identity]
-        # #     odds_dict = dict(zip(odds_key, odds_value))
-        # #     if data_list:
-        # #         res = Helper.async_post(ODDS_URL, odds_dict)
-        # #         if res:
-        # #             if res.get('status') == 1:
-        # #                 logger.info('足球角球, 详细赔率提交成功, {}'.format(res))
-        # #                 # logger.info(odds_dict)
-        # #             else:
-        # #                 logger.warning('足球角球, 详细赔率提交失败, {}'.format(res))
-        # #                 # logger.warning(odds_dict)
-        # #         else:
-        # #             logging.warning('足球角球, 详细赔率接口异常, {}'.format(res))
-        # #     else:
-        # #         logger.info('足球详细赔率列表为空')
-        # reactor.callFromThread(out.callback, item)
->>>>>>> 10c979a4fcc4f2d36f17fa2ecd6de7bad38358f3
+        data=Helper.async_post("")

+ 1 - 12
hgg070_spider/settings.py

@@ -88,16 +88,6 @@ ROBOTSTXT_OBEY = False
 #HTTPCACHE_DIR = 'httpcache'
 #HTTPCACHE_IGNORE_HTTP_CODES = []
 #HTTPCACHE_STORAGE = 'scrapy.extensions.httpcache.FilesystemCacheStorage'
-<<<<<<< HEAD
-
-R_HOST = '103.108.41.110'
-R_POST = 10432
-R_DB = "kaiyou"
-R_USER = "kaiyou"
-R_PASSWORD = 123456
-
-
-=======
 M_HOST = '192.168.2.200'
 # M_HOST = '127.0.0.1'
 M_POST = 27017
@@ -123,5 +113,4 @@ MATCHWARN = "http://stadmin.bocai108.com:19093/setMatchWarn"
 R_HOST = '192.168.2.200'
 R_POST = 6379
 R_DB = 1
-R_PASSWORD = 123456
->>>>>>> 10c979a4fcc4f2d36f17fa2ecd6de7bad38358f3
+R_PASSWORD = 123456

+ 0 - 79
hgg070_spider/spiders/zuqiu.py

@@ -1,83 +1,5 @@
 # -*- coding: utf-8 -*-
 import scrapy
-<<<<<<< HEAD
-from ..items import ZuqiuItem
-
-class ZuqiuSpider(scrapy.Spider):
-    name = 'zuqiu'
-    allowed_domains = ['m.hgg070.com/']
-    headers = {
-        'Accept': '*/*',
-        'Accept-Encoding': 'gzip, deflate',
-        'Accept-Language': 'zh-CN,zh;q=0.9',
-        'Content-Length': '130',
-        'Content-type': 'application/x-www-form-urlencoded',
-        'Cookie': '_ga=GA1.2.471918301.1572059707; _gid=GA1.2.2109447865.1572059707; _gat=1',
-        'Host': 'm.hgg070.com',
-        'Origin': 'http://m.hgg070.com',
-        'Proxy-Connection': 'keep-alive',
-        'Referer': 'http://m.hgg070.com/',
-        'User-Agent': 'Mozilla/5.0 (Linux; Android 6.0; Nexus 5 Build/MRA58N) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/78.0.3904.70 Mobile Safari/537.36'
-    }
-    custom_settings={
-        "ITEM_PIPELINES": {
-            "collectSports.pipelines.zuqiu.ZuqiuPipeline": 200,
-        },
-    }
-    #读取今日足球
-    def start_requests(self):
-        url="http://m.hgg070.com/app/member/get_league_list.php"
-        from_data={
-            'uid': '7c70e73f576d42d9f6d9fb1fcaa08c47b04bb9279584caedfe65858afb26722d',
-            'langx': 'zh-cn',
-            'ltype': '3',
-            'gtype': 'FT',
-            'showtype': 'RB',
-            'sorttype': '',
-            'date': '',
-            'isP': ''
-        }
-        yield scrapy.FormRequest(url=url, formdata=from_data, callback=self.parse, headers=self.headers,meta={'data': from_data}, dont_filter=True)
-
-
-    #解析今日足球,获取所有的联赛,并请求联赛详情页
-    def parse(self, response):
-        url='http://m.hgg070.com/app/member/get_game_list.php'
-        data=response.xpath("//league")
-        from_data=response.meta['data']
-        for item in data:
-            lid = item.xpath('./league_id/text()').extract_first()
-            from_data['lid'] = lid
-            yield scrapy.FormRequest(url=url, formdata=from_data, callback=self.detailtedMsg,meta={'data': response.meta['data']},dont_filter=True)
-
-    # 获取所有玩法
-    def detailtedMsg(self,response):
-        url = 'http://m.hgg070.com/app/member/get_game_more.php'
-        data=response.xpath("//game")
-        from_data=response.meta['data']
-        for item in data:
-            lid = item.xpath('./gid/text()').extract_first()
-            from_data['gid'] = lid
-            yield scrapy.FormRequest(url=url, formdata=from_data, callback=self.getitem,dont_filter=True)
-
-    #获取更多
-    def getitem(self,response):
-        data=response.xpath("//game")
-        for obj in data:
-            item=ZuqiuItem()
-            item.id=obj.xpath('./gid')
-            item.team_h=obj.xpath('./team_h')
-            item.team_c = obj.xpath('./team_c')
-            item.ior_RTS2Y = obj.xpath('./ior_RTS2Y')
-            item.ior_RTS2N = obj.xpath('./ior_RTS2N')
-            item.ior_REH = obj.xpath('./ior_REH')
-            item.ior_REC = obj.xpath('./ior_REC')
-            item.ior_ROUC = obj.xpath('./ior_ROUC')
-            item.ior_HROUH = obj.xpath('./ior_HROUH')
-            item.ratio_rouo = obj.xpath('./ratio_rouo')
-            item.ratio_rouu = obj.xpath('./ratio_rouu')
-            yield item
-=======
 
 
 class ZuqiuSpider(scrapy.Spider):
@@ -172,4 +94,3 @@ class ZuqiuSpider(scrapy.Spider):
         print(response.text)
         pass
 
->>>>>>> 10c979a4fcc4f2d36f17fa2ecd6de7bad38358f3

+ 1 - 6
hgg070_spider/utils/helper.py

@@ -1,4 +1,3 @@
-<<<<<<< HEAD
 import datetime
 import hashlib
 import json
@@ -13,18 +12,16 @@ from hgg070_spider.conf.uid import UID as u_id, UID
 
 
 fs_session = FuturesSession()
-=======
 import hashlib
 import json
 from requests_futures.sessions import FuturesSession
 from .langconv import *
 from .LocalToken import token
->>>>>>> 10c979a4fcc4f2d36f17fa2ecd6de7bad38358f3
+
 
 
 class Helper(object):
     @staticmethod
-<<<<<<< HEAD
     def changetime(params):
         if params.endswith('p'):
             p_time = params[-6:-1]
@@ -124,7 +121,6 @@ if __name__ == '__main__':
     import os
     print(os.path.abspath(os.path.join(os.getcwd(), "..")) + "\\conf\\uid.py")
     print(os.path.abspath(__file__))
-=======
     def async_post(url, params):
         fs_session = FuturesSession()
         t_url, t_user, t_password, t_token = token['token_url'], token['username'], token['password'], token['token']
@@ -164,4 +160,3 @@ if __name__ == '__main__':
         hl = hashlib.md5()
         hl.update(line.encode(encoding='utf-8'))
         return hl.hexdigest()
->>>>>>> 10c979a4fcc4f2d36f17fa2ecd6de7bad38358f3