sports_scrapy/hg3535/pipeline/roll_lanqiu.py

import pymongo
import time
import logging
from ball_func import fuhao
from ..utils.helper import Helper
# from .ball_func import get_pcode, new_time


class Roll_Lanqiupipeline(object):

    def open_spider(self, spider):
        self.mongo = pymongo.MongoClient(host="192.168.2.200", username="kaiyou", password="kaiyou", port=27017)
        self.db = self.mongo['kaiyou']

    def process_item(self, item, spider):
        url = 'http://admin.5gogo.com'
        roll_lq = []
        # 使用twisted将mysql插入变成异步执行
        logger = logging.getLogger(__name__)
        # 联赛id
        league_id = item['league_id']
        # 联赛名
        league_name = item['league_name']
        # result = item['result']
        # 比赛id
        match_id = item['game_id']
        # 球队1 #home_team
        team_home = item['team_home']
        # 球队2 # guest_team
        team_guest = item['team_guest']
        # 数量(97>)
        number = item['number']
        # 比赛状态
        zhuangtai = item['zhuangtai']
        # 日期
        # data_game = item['data_game']
        try:
            data_game = item['data_game'].split("/")
            month = str(data_game[1].strip())
            day = str(data_game[0])
        except Exception as e:
            logger.warning(e)
            data_game = item['data_game'].split(" ")
            months = str(data_game[1].strip())
            month_dict = {'Jan': '01', 'Feb': '02', 'Mar': '03', 'Apr': '04', 'May': '05', 'Jun': '06', 'Jul': '07',
                          'Aug': '08', 'Sep': '09', 'Oct': '10', 'Nov': '11', 'Dec': '12'}
            month = month_dict[months]
            day = str(data_game[0])
        # 比赛时间
        time_game = str(item['time_game'])
        # 比赛时间，时间戳
        ctime = "2019" + "-" + month + "-" + day + "" + time_game + ":00"
        r_ctime = "2019" + "-" + month + "-" + day
        expire_time = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(time.time() + 60))  # 过期时间
        # 现在时间，时间戳
        utime = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime())
        # 队1分数
        score_home = item['score_home']
        # 队2分数
        score_guest = item['score_guest']
        # 第几节
        jijie = item['jijie']
        # 球队得分
        qiudui = item['qiudui']
        pt = 4
        match_score = "{}:{}".format(score_home, score_guest)
        concedes_dict = item['concede']
        concedes_dict_rule = item['concede_rule']
        odd_evens_dict = item['odd_even']
        odd_evens_dict_rule = item['odd_even_rule']
        total_sizes_dict = item['total_size']
        total_sizes_dict_rule = item['total_size_rule']
        last_numbers_dict = item['last_number']
        capots_dict = item['capot']
        team_scores_dict = item['team_score']
        team_scores_dict_rule = item['team_score_rule']

        last_time = "2019-12-31 23:59:59"
        league_dict = {"game_code": "lq", "title": "league", "source": "hg3535"}
        league_list = []
        uuid = Helper.genearte_MD5(league_name + 'hg3535')

        league_key = ["name_chinese", "kind", "match_mode", "if_stop", "last_time", "lg_id", "source", "uuid"]
        league_value = [league_name, "1", "1", "0", last_time, league_id, "hg3535", uuid]
        league_data = dict(zip(league_key, league_value))
        league_list.append(league_data)
        league_dict['data'] = league_list
        if self.db.hg3535_lq_league.find({'lg_id': league_id}).count() < 1:
            response_data = Helper.async_post(url + '/setLeague', league_dict)
            print(response_data)
            self.db.hg3535_lq_league.insert(league_list)
        competition = self.db.hg3535_lq_competition.find_one({"match_id": match_id})
        match_list = []
        if competition:
            print('篮球赛事已存在，修改篮球赛事四状态')
            is_morningplate = competition['is_morningplate']
            is_rollball = competition['is_rollball']
            is_stringscene = competition['is_stringscene']
            is_today = competition['is_today']
            status = competition['status']
            match_date = competition['match_date']
            match_time = competition['match_time']
            us_time = competition['us_time']
            if is_rollball != 1 or status != 1:
                match_dict = {"game_code": "lq", "title": "match", "source": "hg3535"}
                match_kay = ["home_team", "guest_team", "lg_id", "status", "match_id", "match_date", "match_time",
                             "tag", "source", "is_rollball", "is_morningplate", "is_stringscene", "us_time", "uuid",
                             "half_match_id", "is_today"]
                match_value = [team_home, team_guest, league_id, 1, match_id, match_date, match_time, number,
                               "hg3535", 1, is_morningplate, is_stringscene, us_time, uuid, 0, is_today]
                match_data = dict(zip(match_kay, match_value))
                match_list.append(match_data)
                match_dict['data'] = match_list
                response_data = Helper.async_post(url + '/setMatch', match_dict)
                self.db.hg3535_lq_competition.insert(match_list)
                print(response_data)
                self.db.hg3535_lq_competition.update({'match_id': str(match_id)},
                                                     {"$set": {"is_rollball": 1, "status": 1}},
                                                     upsert=True)
            else:
                print("篮球赛事已存在，篮球赛事状态时间未改变，不做修改")
        else:
            is_rollball = 1
            is_today = 0
            is_morningplate = 0
            is_stringscene = 0
            # 赛事接口写入
            match_dict = {"game_code": "lq", "title": "match", "source": "hg3535"}
            match_kay = ["home_team", "guest_team", "lg_id", "status", "match_id", "match_date", "match_time", "tag",
                         "source", "is_rollball", "is_morningplate", "is_stringscene", "us_time", "uuid",
                         "half_match_id", "is_today"]
            match_value = [team_home, team_guest, league_id, 1, match_id, r_ctime, "00:00", number, "hg3535",
                           is_rollball, is_morningplate, is_stringscene, utime, uuid, 0, is_today]
            match_data = dict(zip(match_kay, match_value))
            match_list.append(match_data)
            match_dict['data'] = match_list
            response_data = Helper.async_post(url + '/setMatch', match_dict)
            self.db.hg3535_lq_competition.insert(match_list)
            print(response_data)
        # 赔率
        try:
            result = self.db.hg3535_lq_odds.find_one({'match_id': match_id, 'pt': pt})
            uuid_list = result['odds_only']
        except:
            uuid_list = []
            result = ''
        data_list = []
        odds_onlys = []
        if concedes_dict:
            for key, value in concedes_dict.items():
                if value:
                    for index, concede_value in enumerate(value):
                        hash_str = "concede" + key + str(index) + str(concedes_dict_rule[key][index]) + str(
                            concede_value) + "hg3535" + str(match_id)
                        sole_str = "concede" + "key" + str(index) + str(match_id) + "hg3535"
                        odds_only = Helper.genearte_MD5(hash_str)
                        sole = Helper.genearte_MD5(sole_str)
                        odds_onlys.append(odds_only)
                        concede_key = ["match_id", "lg_id", "odds_code", "status", "sort", "p_code", "odds",
                                       "condition", "odds_only", "sole", "source", "type", "team"]
                        condition = fuhao(concedes_dict_rule[key][index])
                        concede_value = [match_id, league_id, key, "0", index, 'concede', concede_value, condition,
                                         odds_only, sole, "hg3535", "0", ""]
                        concede_data = dict(zip(concede_key, concede_value))
                        if odds_only not in uuid_list:
                            data_list.append(concede_data)
        if total_sizes_dict:
            for key, value in total_sizes_dict.items():
                if value:
                    for index, total_sizes_value in enumerate(value):
                        condition = fuhao(total_sizes_dict_rule[key][index])
                        hash_str = "total_size" + key + str(index) + str(condition) + str(
                            total_sizes_value) + "hg3535" + str(match_id)
                        sole_str = "total_size" + "key" + str(index) + str(match_id) + "hg3535"
                        odds_only = Helper.genearte_MD5(hash_str)
                        sole = Helper.genearte_MD5(sole_str)
                        odds_onlys.append(odds_only)
                        total_sizes_key = ["match_id", "lg_id", "odds_code", "status", "sort", "p_code", "odds",
                                           "condition", "odds_only", "sole", "source", "type", "team"]
                        total_sizes_value = [match_id, league_id, key, "0", index, 'total_size', total_sizes_value,
                                             condition,
                                             odds_only, sole, "hg3535", "0", ""]
                        total_sizes_data = dict(zip(total_sizes_key, total_sizes_value))
                        if odds_only not in uuid_list:
                            data_list.append(total_sizes_data)
        if odd_evens_dict:
            for key, value in odd_evens_dict.items():
                if value:
                    condition = odd_evens_dict_rule[key]
                    hash_str = "two_sides" + key + '0' + str(condition) + str(
                        value) + "hg3535" + str(match_id)
                    sole_str = "two_sides" + "key" + '0' + str(match_id) + "hg3535"
                    odds_only = Helper.genearte_MD5(hash_str)
                    sole = Helper.genearte_MD5(sole_str)
                    odds_onlys.append(odds_only)
                    odd_evens_key = ["match_id", "lg_id", "odds_code", "status", "sort", "p_code", "odds",
                                     "condition", "odds_only", "sole", "source", "type", "team"]
                    odd_evens_value = [match_id, league_id, key, "0", 0, 'two_sides', value,
                                       condition, odds_only, sole, "hg3535", "0", ""]
                    odd_evens_data = dict(zip(odd_evens_key, odd_evens_value))
                    if odds_only not in uuid_list:
                        data_list.append(odd_evens_data)

        lastnumber_dict = {'0或5': '_zero_five', '1或6': '_one_six', '2或7': '_two_seven', '3或8': '_three_eight',
                           '4或9': '_four_nine'}
        last_home = last_numbers_dict['last_home']
        if last_home:
            for key, value in last_home.items():
                # pass
                odds_code = 'last_home' + lastnumber_dict[key]
                condition = lastnumber_dict[key]
                hash_str = "last_number" + odds_code + '0' + str(condition) + str(
                    value) + "hg3535" + str(match_id)
                sole_str = "last_number" + "key" + '0' + str(match_id) + "hg3535"
                odds_only = Helper.genearte_MD5(hash_str)
                sole = Helper.genearte_MD5(sole_str)
                odds_onlys.append(odds_only)
                last_home_key = ["match_id", "lg_id", "odds_code", "status", "sort", "p_code", "odds",
                                 "condition", "odds_only", "sole", "source", "type", "team"]
                last_home_value = [match_id, league_id, odds_code, "0", 0, 'last_number', value,
                                   condition, odds_only, sole, "hg3535", "0", ""]
                last_home_data = dict(zip(last_home_key, last_home_value))
                if odds_only not in uuid_list:
                    data_list.append(last_home_data)

        last_guest = last_numbers_dict['last_guest']
        if last_guest:
            for key, value in last_guest.items():
                odds_code = 'last_guest' + lastnumber_dict[key]
                condition = lastnumber_dict[key]
                hash_str = "last_number" + odds_code + '0' + str(condition) + str(value) + "hg3535" + str(match_id)
                sole_str = "last_number" + "key" + '0' + str(match_id) + "hg3535"
                odds_only = Helper.genearte_MD5(hash_str)
                sole = Helper.genearte_MD5(sole_str)
                odds_onlys.append(odds_only)
                last_guest_key = ["match_id", "lg_id", "odds_code", "status", "sort", "p_code", "odds",
                                  "condition", "odds_only", "sole", "source", "type", "team"]
                last_guest_value = [match_id, league_id, odds_code, "0", 0, 'last_number', value,
                                    condition, odds_only, sole, "hg3535", "0", ""]
                last_guest_data = dict(zip(last_guest_key, last_guest_value))
                if odds_only not in uuid_list:
                    data_list.append(last_guest_data)

        if capots_dict:
            for key, value in capots_dict.items():
                if value:
                    hash_str = "capot" + key + '0' + '0' + str(value) + "hg3535" + str(match_id)
                    sole_str = "capot" + "key" + '0' + str(match_id) + "hg3535"
                    odds_only = Helper.genearte_MD5(hash_str)
                    sole = Helper.genearte_MD5(sole_str)
                    odds_onlys.append(odds_only)
                    capots_key = ["match_id", "lg_id", "odds_code", "status", "sort", "p_code", "odds",
                                  "condition", "odds_only", "sole", "source", "type", "team"]
                    capots_value = [match_id, league_id, key, "0", 0, 'capot', value,
                                    "", odds_only, sole, "hg3535", "0", ""]
                    capots_data = dict(zip(capots_key, capots_value))
                    if odds_only not in uuid_list:
                        data_list.append(capots_data)

        if team_scores_dict:
            for key, team_scores_value in team_scores_dict.items():
                if team_scores_value:
                    for index, value in enumerate(team_scores_value):
                        hash_str = "team_score" + key + str(index) + str(team_scores_dict_rule[key][index]) + str(
                            value) + "hg3535" + str(match_id)
                        sole_str = "team_score" + "key" + str(index) + str(match_id) + "hg3535"
                        odds_only = Helper.genearte_MD5(hash_str)
                        sole = Helper.genearte_MD5(sole_str)
                        odds_onlys.append(odds_only)
                        team_scores_key = ["match_id", "lg_id", "odds_code", "status", "sort", "p_code", "odds",
                                           "condition", "odds_only", "sole", "source", "type", "team"]
                        condition = team_scores_dict_rule[key][index]
                        team_scores_value = [match_id, league_id, key, "0", index, 'team_score', value, condition,
                                             odds_only, sole, "hg3535", "0", ""]
                        team_scores_data = dict(zip(team_scores_key, team_scores_value))
                        if odds_only not in uuid_list:
                            data_list.append(team_scores_data)

        odds_key = ["game_code", "title", "match_id", "lg_id", "data", "source", "odds_only", "tag", "uuid",
                    "is_stringscene", "utime", "pt"]
        odds_value = ["lq", "odds", match_id, league_id, data_list, "hg3535", odds_onlys, number, uuid,
                      0, utime, pt]
        odds_dict = dict(zip(odds_key, odds_value))
        if data_list:
            # print(odds_dict)
            response_data = Helper.async_post(url + '/setOdds', odds_dict)
            # print(response_data)
            print('篮球球赔率请求成功', response_data)
            if "成功" in response_data:
                if result:
                    self.db.hg3535_lq_odds.update({'match_id': match_id, 'pt': pt}, {
                        '$set': {"data": data_list, "odds_only": odds_onlys, "utime": utime}}, upsert=True)
                else:
                    self.db.hg3535_lq_odds.insert(odds_dict)
        data_list = []
        lq_rball = {"home_team": team_home, "guest_team": team_guest,
                    "lg_id": league_id, "home_rate": 0,
                    "guest_rate": 0, "home_score": score_home,
                    "guest_score": score_guest, "all_goal": "", "status": 1,
                    "first_score": "", "last_score": "", "match_score": match_score, "uuid": uuid,
                    "match_winer": "", "match_time": time_game,"u_home_score": "", "u_guest_score": "",
                    "match_process": jijie, "tag": number,
                    "match_id": match_id, "p_code": ""}
        data_list.append(lq_rball)
        r_data_dict = {
            "game_code": "lq",
            "title": "match_result_r",
            "source": "hg3535",
            "data": data_list
        }
        if data_list:
            try:
                response_data = Helper.async_post(url + '/setMatchResult', r_data_dict)
                # print(r_data_dict)
                print("滚球篮球赛事结果post提交", response_data)
            except Exception as e:
                logger.warning('滚球数据接口异常，提交失败')
                logger.warning(e)