sports_scrapy/hg3535/pipeline/bangqiu.py

import datetime
import pymongo
import time
import logging
from .ball_func import fuhao
from ..utils.helper import Helper
from .ball_func import get_pcode, new_time, out_time
from concurrent.futures import ThreadPoolExecutor
from requests_futures.sessions import FuturesSession


class Bangqiupipeline(object):
    def open_spider(self, spider):
        self.mongo = pymongo.MongoClient(host="192.168.2.200", username="kaiyou", password="kaiyou", port=27017)
        self.db = self.mongo['kaiyou']
        self.session = FuturesSession(executor=ThreadPoolExecutor(max_workers=10))

    def process_item(self, item, spider):
        # print(item)
        logger = logging.getLogger(__name__)
        url = 'http://admin.5gogo.com'
        # logger.warning(query.addErrback(self.handle_error, item, spider))
        # 联赛id
        league_id = item['league_id']
        # 联赛名
        league_name = item['league_name']
        # result = item['result']
        # 比赛id
        match_id = item['game_id']
        # 球队1
        team_home = item['team_home']
        # 球队2
        team_guest = item['team_guest']
        # 数量(97>)
        tag_number = item['number']
        # 比赛状态
        zhuangtai = item['zhuangtai']
        # 日期
        # data_game = item['data_game']
        try:
            data_game = item['data_game'].split("/")
            month = str(data_game[1].strip())
            day = str(data_game[0])
        except Exception as e:
            # logger.warning('data_game字段分割错误')
            # logger.warning(e)
            data_game = item['data_game'].split(" ")
            months = str(data_game[1].strip())
            month_dict = {'Jan': '01', 'Feb': '02', 'Mar': '03', 'Apr': '04', 'May': '05', 'Jun': '06', 'Jul': '07',
                          'Aug': '08', 'Sep': '09', 'Oct': '10', 'Nov': '11', 'Dec': '12'}
            month = month_dict[months]
            day = str(data_game[0])
        # 比赛时间
        time_game = str(item['time_game'])
        # 比赛时间，时间戳
        ctime = "2019" + "-" + month + "-" + day + " " + time_game + ":00"
        r_ctime = "2019" + "-" + month + "-" + day
        # 现在时间，时间戳
        utime = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime())
        expire_time = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(time.time() + 60))
        # 队1分数
        score_home = item['score_home']
        # 队2分数
        score_guest = item['score_guest']
        # 第几节
        jijie = item['jijie']
        # 球队得分
        qiudui = item['qiudui']
        pt = item['pt']
        # 让球
        concedes_dict = item['concedes_dict']
        concedes_dict_rule = item['concedes_dict_rule']
        # 独赢
        capots_dict = item['capots_dict']
        # 总得分大小
        total_size_dict = item['total_size_dict']
        total_size_dict_rule = item['total_size_dict_rule']
        # 总得分单双
        odd_evens_dict = item['odd_evens_dict']
        odd_evens_dict_rule = item['odd_evens_dict_rule']
        # # 球队得分 主队
        # try:
        #     team_score_home_dict = item['team_score_home_dict']
        #     team_score_home_rule_dict = item['team_score_home_rule_dict']
        # except:
        #     team_score_home_dict = ""
        #     team_score_home_rule_dict = ""
        # # 球队得分 客队
        # try:
        #     team_score_guest_dict = item['team_score_guest_dict']
        #     team_score_guest_rule_dict = item['team_score_guest_rule_dict']
        # except:
        #     team_score_guest_dict = ""
        #     team_score_guest_rule_dict = ""
        bangqiu = item['bangqiu']
        match_date, match_time, time3 = new_time(ctime)
        us_time = ctime
        uuid = Helper.genearte_MD5(league_name + 'hg3535')
        """联赛表"""
        last_time = '2019-12-31 23:59:59'
        league_dict = {"game_code": "bq", "title": "league", "source": "hg3535"}
        league_list = []

        league_key = ["name_chinese", "kind", "match_mode", "if_stop", "last_time", "lg_id", "source", "uuid"]
        league_value = [league_name, "1", "1", "0", last_time, league_id, "hg3535", uuid]
        league_data = dict(zip(league_key, league_value))
        league_list.append(league_data)
        league_dict['data'] = league_list
        # print(league_list)
        # print(league_dict)
        if self.db.hg3535_bq_league.find({'lg_id': league_id}).count() < 1:
            res = Helper.async_post(url + '/setLeague', league_dict)
            print(res)
            self.db.hg3535_bq_league.insert(league_list)

        """插入赛事表"""
        competition = self.db.hg3535_bq_competition.find_one({"match_id": str(match_id)})
        match_list = []
        if competition:
            print('赛事已存在，修改赛事四状态')
            is_morningplate = competition['is_morningplate']
            is_rollball = competition['is_rollball']
            is_stringscene = competition['is_stringscene']
            is_today = competition['is_today']
            lmtime = competition['match_time']
            if pt == 3:
                if is_stringscene != 1 or lmtime != match_time:
                    match_dict = {"game_code": "bq", "title": "match", "source": "hg3535"}
                    match_key = ["home_team", "guest_team", "lg_id", "status", "match_id", "match_date", "match_time",
                                 "tag", "source", "is_rollball", "is_morningplate", "is_stringscene", "us_time", "uuid",
                                 "half_match_id", "is_today"]
                    match_value = [team_home, team_guest, league_id, 0, match_id, match_date, match_time, tag_number,
                                   "hg3535", is_rollball, is_morningplate, 1, us_time, uuid, 0, is_today]
                    match_data = dict(zip(match_key, match_value))
                    match_list.append(match_data)
                    match_dict['data'] = match_list
                    # print(match_list)
                    # print(match_dict)
                    res = Helper.async_post(url + '/setMatch', match_dict)
                    print(res)
                    self.db.hg3535_bq_competition.update({'match_id': match_id},
                                                         {"$set": {"is_stringscene": 1, 'match_time': match_time}},
                                                         upsert=True)
            if pt == 2:
                if is_morningplate != 1 or lmtime != match_time:
                    match_dict = {"game_code": "bq", "title": "match", "source": "hg3535"}
                    match_key = ["home_team", "guest_team", "lg_id", "status", "match_id", "match_date", "match_time",
                                 "tag", "source", "is_rollball", "is_morningplate", "is_stringscene", "us_time", "uuid",
                                 "half_match_id", "is_today"]
                    match_value = [team_home, team_guest, league_id, 0, match_id, match_date, match_time, tag_number,
                                   "hg3535", is_rollball, 1, is_stringscene, us_time, uuid, 0, is_today]
                    match_data = dict(zip(match_key, match_value))
                    match_list.append(match_data)
                    match_dict['data'] = match_list
                    # print(match_list)
                    # print(match_dict)
                    res = Helper.async_post(url + '/setMatch', match_dict)
                    print(res)
                    self.db.hg3535_bq_competition.update({'match_id': match_id},
                                                         {"$set": {"is_morningplate": 1, 'match_time': match_time}},
                                                         upsert=True)
            if pt == 1:
                if is_today != 1 or lmtime != match_time:
                    match_dict = {"game_code": "bq", "title": "match", "source": "hg3535"}
                    match_kay = ["home_team", "guest_team", "lg_id", "status", "match_id", "match_date", "match_time",
                                 "tag", "source", "is_rollball", "is_morningplate", "is_stringscene", "us_time", "uuid",
                                 "half_match_id", "is_today"]
                    match_value = [team_home, team_guest, league_id, 0, match_id, match_date, match_time, tag_number,
                                   "hg3535", is_rollball, is_morningplate, is_stringscene, us_time, uuid, 0, 1]
                    match_data = dict(zip(match_kay, match_value))
                    match_list.append(match_data)
                    match_dict['data'] = match_list
                    # print(match_list)
                    # print(match_dict)
                    res = Helper.async_post(url + '/setMatch', match_dict)
                    print(res)
                    self.db.hg3535_bq_competition.update({'match_id': match_id},
                                                         {"$set": {"is_today": 1, 'match_time': match_time}},
                                                         upsert=True)
            else:
                print("赛事已存在，赛事状态时间未改变，不做修改")
        else:
            print('赛事不存在, 插入新数据')
            if pt == 3:
                is_rollball = 0
                is_today = 0
                is_morningplate = 0
                is_stringscene = 1
            elif pt == 2:
                is_rollball = 0
                is_today = 0
                is_morningplate = 1
                is_stringscene = 0
            else:
                is_today = 1
                is_rollball = 0
                is_morningplate = 0
                is_stringscene = 0
            match_dict = {"game_code": "bq", "title": "match", "source": "hg3535"}
            match_kay = ["home_team", "guest_team", "lg_id", "status", "match_id", "match_date", "match_time", "tag",
                         "source", "is_rollball", "is_morningplate", "is_stringscene", "us_time", "uuid",
                         "half_match_id", "is_today"]
            match_value = [team_home, team_guest, league_id, 0, match_id, match_date, match_time, tag_number, "hg3535",
                           is_rollball, is_morningplate, is_stringscene, us_time, uuid, 0, is_today]
            match_data = dict(zip(match_kay, match_value))
            match_list.append(match_data)
            match_dict['data'] = match_list
            res = Helper.async_post(url + '/setMatch', match_dict)
            print(res)
            self.db.hg3535_bq_competition.insert(match_list)

        """赔率"""
        try:
            result = self.db.hg3535_bq_odds.find_one({'match_id': str(match_id), 'pt': pt})
            odds_only_list = result['odds_only']
        except:
            result = ''
            odds_only_list = []
        data_list = []
        odds_onlys = []
        # 让球
        if concedes_dict:
            for key, value in concedes_dict.items():
                sole_str = 'CO' + str(key) + str(match_id) + 'hg3535'
                sole = Helper.genearte_MD5(sole_str)
                odds_str = 'CO' + str(key) + str(match_id) + 'hg3535' + str(concedes_dict_rule[key]) + str(
                    value)
                odds_only = Helper.genearte_MD5(odds_str)
                condition = fuhao(str(concedes_dict_rule[key]))
                concedes_dict_key = ["match_id", "lg_id", "odds_code", "status", "sort", "p_code", "odds", "condition",
                                     "odds_only", "sole", "source", "type", "team"]
                concedes_dict_value = [match_id, league_id, key, '0', 0, 'CO', value, condition,
                                       odds_only, sole, 'hg3535', '0', '']
                concedes_dict_data = dict(zip(concedes_dict_key, concedes_dict_value))
                odds_onlys.append(odds_only)
                if odds_only not in odds_only_list:
                    data_list.append(concedes_dict_data)
        # 总得分大/小
        if total_size_dict:
            for key, value in total_size_dict.items():
                sole_str = 'TN' + str(key) + str(match_id) + 'hg3535'
                sole = Helper.genearte_MD5(sole_str)
                odds_str = 'TN' + str(key) + str(match_id) + 'hg3535' + str(
                    total_size_dict_rule[key]) + str(value)
                odds_only = Helper.genearte_MD5(odds_str)
                condition = fuhao(str(total_size_dict_rule[key]))
                total_size_dict_key = ["match_id", "lg_id", "odds_code", "status", "sort", "p_code", "odds",
                                       "condition", "odds_only", "sole", "source", "type", "team"]
                total_size_dict_value = [match_id, league_id, key, '0', 0, 'TN', value, condition,
                                         odds_only, sole, 'hg3535', '0', '']
                total_size_dict_data = dict(zip(total_size_dict_key, total_size_dict_value))
                odds_onlys.append(odds_only)
                if odds_only not in odds_only_list:
                    data_list.append(total_size_dict_data)
        # 球队得分大/小 主队
        # if team_score_home_dict:
        #     for key, value in team_score_home_dict.items():
        #         sole_str = 'team_score_home' + str(key) + str(match_id) + 'hg3535'
        #         sole = Helper.genearte_MD5(sole_str)
        #         odds_str = 'team_score_home' + str(key) + str(match_id) + 'hg3535' + str(
        #             team_score_home_rule_dict[key]) + str(value)
        #         odds_only = Helper.genearte_MD5(odds_str)
        #         condition = fuhao(str(team_score_home_rule_dict[key]))
        #         team_score_home_dict_key = ["match_id", "lg_id", "odds_code", "status", "sort", "p_code", "odds",
        #                                     "condition", "odds_only", "sole", "source", "type", "team"]
        #         team_score_home_dict_value = [match_id, league_id, key, '0', '0', 'team_score_home', value, condition,
        #                                       odds_only, sole, 'hg3535', '0', '']
        #         team_score_home_dict_data = dict(zip(team_score_home_dict_key, team_score_home_dict_value))
        #         odds_onlys.append(odds_only)
        #         if odds_only not in odds_only_list:
        #             data_list.append(team_score_home_dict_data)
        # # 球队得分大/小 客队
        # if team_score_guest_dict:
        #     for key, value in team_score_guest_dict.items():
        #         sole_str = 'team_score_guest' + str(key) + str(match_id) + 'hg3535'
        #         sole = Helper.genearte_MD5(sole_str)
        #         odds_str = 'team_score_guest' + str(key) + str(match_id) + 'hg3535' + str(
        #             team_score_guest_rule_dict[key]) + str(value)
        #         odds_only = Helper.genearte_MD5(odds_str)
        #         condition = fuhao(str(team_score_guest_rule_dict[key]))
        #         team_score_guest_dict_key = ["match_id", "lg_id", "odds_code", "status", "sort", "p_code", "odds",
        #                                      "condition", "odds_only", "sole", "source", "type", "team"]
        #         team_score_guest_dict_value = [match_id, league_id, key, '0', '0', 'team_score_guest', value, condition,
        #                                        odds_only, sole, 'hg3535', '0', '']
        #         team_score_guest_dict_data = dict(zip(team_score_guest_dict_key, team_score_guest_dict_value))
        #         odds_onlys.append(odds_only)
        #         if odds_only not in odds_only_list:
        #             data_list.append(team_score_guest_dict_data)
        # 独赢
        if capots_dict:
            for key, value in capots_dict.items():
                sole_str = 'C' + str(key) + str(match_id) + 'hg3535'
                sole = Helper.genearte_MD5(sole_str)
                odds_str = 'C' + str(key) + str(match_id) + 'hg3535' + str(value)
                odds_only = Helper.genearte_MD5(odds_str)
                capots_dict_key = ["match_id", "lg_id", "odds_code", "status", "sort", "p_code", "odds",
                                   "condition", "odds_only", "sole", "source", "type", "team"]
                capots_dict_value = [match_id, league_id, key, '0', 0, 'C', value, '',
                                     odds_only, sole, 'hg3535', '0', '']
                capots_dict_data = dict(zip(capots_dict_key, capots_dict_value))
                odds_onlys.append(odds_only)
                if odds_only not in odds_only_list:
                    data_list.append(capots_dict_data)
        # 总得分单双
        if odd_evens_dict:
            for key, value in odd_evens_dict.items():
                sole_str = 'TS' + str(key) + str(match_id) + 'hg3535'
                sole = Helper.genearte_MD5(sole_str)
                odds_str = 'TS' + str(key) + str(match_id) + 'hg3535' + str(
                    odd_evens_dict_rule[key]) + str(value)
                odds_only = Helper.genearte_MD5(odds_str)
                condition = fuhao(str(odd_evens_dict_rule[key]))
                odd_evens_dict_key = ["match_id", "lg_id", "odds_code", "status", "sort", "p_code", "odds",
                                      "condition", "odds_only", "sole", "source", "type", "team"]
                odd_evens_dict_value = [match_id, league_id, key, '0', 0, 'TS', value, condition,
                                        odds_only, sole, 'hg3535', '0', '']
                odd_evens_dict_data = dict(zip(odd_evens_dict_key, odd_evens_dict_value))
                odds_onlys.append(odds_only)
                if odds_only not in odds_only_list:
                    data_list.append(odd_evens_dict_data)
        # 让球 前五局
        # 总得分大/小 前五局
        if pt == 3:
            ris_stringscene = 1
        else:
            ris_stringscene = 0
        odds_key = ["game_code", "title", "match_id", "lg_id", "data", "source", "odds_only", "tag", "uuid",
                    "is_stringscene", "utime", "pt"]
        odds_value = ["bq", "odds", match_id, league_id, data_list, "hg3535", odds_onlys, tag_number, uuid,
                      ris_stringscene, utime, pt]
        odds_dict = dict(zip(odds_key, odds_value))
        # print(odds_dict)
        # print(123)
        if data_list:
            res = Helper.async_post(url + '/setOdds', odds_dict)
            print(res)
            if '成功' in res:
                if result:
                    self.db.hg3535_bq_odds.update({'match_id': match_id, 'pt': pt}, {
                        '$set': {"data": data_list, "odds_only": odds_onlys, "utime": utime}}, upsert=True)
                else:
                    self.db.hg3535_bq_odds.insert(odds_dict)