sports_scrapy/hg3535/pipeline/roll_wangqiu.py

import datetime
import pymongo
import time
import logging
import json
from .ball_func import fuhao
from ..utils.helper import Helper
from .ball_func import get_pcode, new_time, out_time
from concurrent.futures import ThreadPoolExecutor
from requests_futures.sessions import FuturesSession


class Roll_Wangqiupipeline(object):
    def open_spider(self, spider):
        self.mongo = pymongo.MongoClient(host="192.168.2.200", username="kaiyou", password="kaiyou", port=27017)
        self.db = self.mongo['kaiyou']
        self.session = FuturesSession(executor=ThreadPoolExecutor(max_workers=10))

    def process_item(self, item, spider):
        url = 'http://admin.5gogo.com'
        logger = logging.getLogger(__name__)
        # 联赛id
        league_id = item['league_id']
        # 联赛名
        league_name = item['league_name']
        # result = item['result']
        # 比赛id
        match_id = item['game_id']
        # 球队1
        team_home = item['team_home']
        # 球队2
        team_guest = item['team_guest']
        # 数量(97>)
        tag_number = item['number']
        # 比赛状态
        zhuangtai = item['zhuangtai']
        # 日期
        # data_game = item['data_game']
        try:
            data_game = item['data_game'].split("/")
            month = str(data_game[1].strip())
            day = str(data_game[0])
        except Exception as e:
            logger.warning(e)
            data_game = item['data_game'].split(" ")
            months = str(data_game[1].strip())
            month_dict = {'Jan': '01', 'Feb': '02', 'Mar': '03', 'Apr': '04', 'May': '05', 'Jun': '06', 'Jul': '07',
                          'Aug': '08', 'Sep': '09', 'Oct': '10', 'Nov': '11', 'Dec': '12'}
            month = month_dict[months]
            day = str(data_game[0])
        # 比赛时间
        time_game = str(item['time_game'])
        # 比赛时间，时间戳
        ctime = "2019" + "-" + month + "-" + day + "" + time_game + ":00"
        r_ctime = "2019" + "-" + month + "-" + day
        # 现在时间，时间戳
        utime = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime())
        # 队1分数
        score_home = item['score_home']
        # 队2分数
        score_guest = item['score_guest']
        # 第几节
        jijie = item['jijie']
        # 球队得分
        qiudui = item['qiudui']
        pt = 4
        # 取不到 暂时注掉
        # match_date, match_time = new_times(ctime)
        # 每局比分
        score_dict = item['score_dict']
        # 让盘
        concedes_dict = item['concedes_dict']
        concedes_dict_rule = item['concedes_dict_rule']
        # 冠军 独赢
        kemps_dict = item['kemps_dict']
        # 让局
        bureaus_dict = item['bureaus_dict']
        bureaus_dict_rule = item['bureaus_dict_rule']
        # 总局数大小
        total_number_dict = item['total_number_dict']
        total_number_dict_rule = item['total_number_dict_rule']
        # 总局数单双
        odd_evens_dict = item['odd_evens_dict']
        odd_evens_dict_rule = item['odd_evens_dict_rule']
        sq_dict = item['sq_dict']
        new_sq_dict = json.dumps(sq_dict)
        uuid = Helper.genearte_MD5(league_name + 'hg3535')
        """联赛"""
        # last_time = '2019-12-31 23:59:59'
        # league_dict = {"game_code": "wq", "title": "league", "source": "hg3535"}
        # league_list = []
        #
        # league_key = ["name_chinese", "kind", "match_mode", "if_stop", "last_time", "lg_id", "source", "uuid"]
        # league_value = [league_name, "1", "1", "0", last_time, league_id, "hg3535", uuid]
        # league_data = dict(zip(league_key, league_value))
        # league_list.append(league_data)
        # league_dict['data'] = league_list
        # if self.db.hg3535_wq_league.find({'lg_id': league_id}).count() < 1:
        #     res = Helper.async_post(url + '/setLeague', league_dict)
        #     print(res)
        #     self.db.hg3535_wq_league.insert(league_list)

        """赛事"""
        # competition = self.db.hg3535_wq_competition.find_one({"match_id": match_id})
        # match_list = []
        # if competition:
        #     print('网球赛事已存在，修改网球赛事四状态')
        #     is_morningplate = competition['is_morningplate']
        #     is_rollball = competition['is_rollball']
        #     is_stringscene = competition['is_stringscene']
        #     is_today = competition['is_today']
        #     status = competition['status']
        #     match_date = competition['match_date']
        #     match_time = competition['match_time']
        #     us_time = competition['us_time']
        #     if is_rollball != 1 or status != 1:
        #         match_dict = {"game_code": "wq", "title": "match", "source": "hg3535"}
        #         match_key = ["home_team", "guest_team", "lg_id", "status", "match_id", "match_date", "match_time",
        #                      "tag", "source", "is_rollball", "is_morningplate", "is_stringscene", "us_time", "uuid",
        #                      "half_match_id", "is_today", 'rule']
        #         match_value = [team_home, team_guest, league_id, 1, match_id, match_date, match_time, tag_number,
        #                        "hg3535", 1, is_morningplate, is_stringscene, us_time, uuid, 0, is_today, '']
        #         match_data = dict(zip(match_key, match_value))
        #         match_list.append(match_data)
        #         match_dict['data'] = match_list
        #         res = Helper.async_post(url + '/setMatch', match_dict)
        #         print(res)
        #         self.db.hg3535_wq_competition.insert(match_list)
        #         self.db.hg3535_wq_competition.update({'match_id': str(match_id)},
        #                                              {"$set": {"is_rollball": 1, "status": 1}},
        #                                              upsert=True)
        #     else:
        #         print("网球赛事已存在，网球赛事状态时间未改变，不做修改")
        # else:
        #     is_rollball = 1
        #     is_today = 0
        #     is_morningplate = 0
        #     is_stringscene = 0
        #     match_dict = {"game_code": "wq", "title": "match", "source": "hg3535"}
        #     match_key = ["home_team", "guest_team", "lg_id", "status", "match_id", "match_date", "match_time",
        #                  "tag", "source", "is_rollball", "is_morningplate", "is_stringscene", "us_time", "uuid",
        #                  "half_match_id", "is_today", 'rule']
        #     match_value = [team_home, team_guest, league_id, 1, match_id, r_ctime, "00:00", tag_number,
        #                    "hg3535", is_rollball, is_morningplate, is_stringscene, utime, uuid, 0, is_today, '']
        #     match_data = dict(zip(match_key, match_value))
        #     match_list.append(match_data)
        #     match_dict['data'] = match_list
        #     res = Helper.async_post(url + '/setMatch', match_dict)
        #     print(res)
        #     self.db.hg3535_wq_competition.insert(match_list)

        """赔率"""
        try:
            res = self.db.hg3535_wq_odds.find_one({'match_id': str(match_id), 'pt': pt})
            odds_only_list = res['odds_only']
        except:
            res = ''
            odds_only_list = []
        data_list = []
        odds_onlys = []
        # 滚球独赢
        if kemps_dict:
            for key, value in kemps_dict.items():
                if value:
                    sole_str = 'kemps' + 'key' + '0' + str(match_id) + 'hg3535'
                    sole = Helper.genearte_MD5(sole_str)
                    odds_str = 'kemps' + str(key) + '0' + '0' + str(value) + 'hg3535' + str(match_id)
                    odds_only = Helper.genearte_MD5(odds_str)
                    kemps_key = ["match_id", "lg_id", "odds_code", "status", "sort", "p_code", "odds",
                                 "condition", "odds_only", "sole", "source", "type", "team"]
                    kemps_value = [match_id, league_id, key, "0", 0, 'kemps', value, '',
                                   odds_only, sole, "hg3535", "0", ""]
                    kemps_data = dict(zip(kemps_key, kemps_value))
                    odds_onlys.append(odds_only)
                    if odds_only not in odds_only_list:
                        data_list.append(kemps_data)
        # 滚球让盘
        if concedes_dict:
            for key, value in concedes_dict.items():
                if value:
                    sole_str = 'concedes' + 'key' + '0' + str(match_id) + 'hg3535'
                    sole = Helper.genearte_MD5(sole_str)
                    odds_str = 'concedes' + str(key) + '0' + str(concedes_dict_rule[key]) + str(
                        value) + 'hg3535' + str(match_id)
                    odds_only = Helper.genearte_MD5(odds_str)
                    condition = fuhao(concedes_dict_rule[key])
                    concedes_key = ["match_id", "lg_id", "odds_code", "status", "sort", "p_code", "odds",
                                    "condition", "odds_only", "sole", "source", "type", "team"]
                    concedes_value = [match_id, league_id, key, "0", 0, 'concede', value, condition,
                                      odds_only, sole, "hg3535", "0", ""]
                    concedes_data = dict(zip(concedes_key, concedes_value))
                    odds_onlys.append(odds_only)
                    if odds_only not in odds_only_list:
                        data_list.append(concedes_data)
        # 滚球让局
        if bureaus_dict:
            for key, value in bureaus_dict.items():
                if value:
                    sole_str = 'bureaus' + 'key' + '0' + str(match_id) + 'hg3535'
                    sole = Helper.genearte_MD5(sole_str)
                    odds_str = 'bureaus' + str(key) + '0' + str(bureaus_dict_rule[key]) + str(
                        value) + 'hg3535' + str(match_id)
                    odds_only = Helper.genearte_MD5(odds_str)
                    condition = fuhao(bureaus_dict_rule[key])
                    bureaus_key = ["match_id", "lg_id", "odds_code", "status", "sort", "p_code", "odds",
                                   "condition", "odds_only", "sole", "source", "type", "team"]
                    bureaus_value = [match_id, league_id, key, "0", 0, 'bureaus', value, condition,
                                     odds_only, sole, "hg3535", "0", ""]
                    bureaus_data = dict(zip(bureaus_key, bureaus_value))
                    odds_onlys.append(odds_only)
                    if odds_only not in odds_only_list:
                        data_list.append(bureaus_data)
        # 滚球总局数大小
        if total_number_dict:
            for key, value in total_number_dict.items():
                if value:
                    sole_str = 'total_number' + 'key' + '0' + str(match_id) + 'hg3535'
                    sole = Helper.genearte_MD5(sole_str)
                    odds_str = 'total_number' + str(key) + '0' + str(
                        total_number_dict_rule[key]) + str(value) + 'hg3535' + str(match_id)
                    odds_only = Helper.genearte_MD5(odds_str)
                    condition = fuhao(total_number_dict_rule[key])
                    total_number_key = ["match_id", "lg_id", "odds_code", "status", "sort", "p_code", "odds",
                                        "condition", "odds_only", "sole", "source", "type", "team"]
                    total_number_value = [match_id, league_id, key, "0", 0, 'bureaus', value,
                                          condition, odds_only, sole, "hg3535", "0", ""]
                    total_number_data = dict(zip(total_number_key, total_number_value))
                    odds_onlys.append(odds_only)
                    if odds_only not in odds_only_list:
                        data_list.append(total_number_data)
        # 滚球总局数单双
        if odd_evens_dict:
            for key, value in odd_evens_dict.items():
                if value:
                    condition = odd_evens_dict_rule[key]
                    odds_str = "odd_evens" + key + '0' + str(odd_evens_dict_rule[key]) + str(
                        value) + "hg3535" + str(match_id)
                    sole_str = "odd_evens" + "key" + '0' + str(match_id) + "hg3535"
                    odds_only = Helper.genearte_MD5(odds_str)
                    sole = Helper.genearte_MD5(sole_str)
                    odds_onlys.append(odds_only)
                    odd_evens_key = ["match_id", "lg_id", "odds_code", "status", "sort", "p_code", "odds",
                                     "condition", "odds_only", "sole", "source", "type", "team"]
                    odd_evens_value = [match_id, league_id, key, "0", 0, 'odd_evens', value,
                                       condition, odds_only, sole, "hg3535", "0", ""]
                    odd_evens_data = dict(zip(odd_evens_key, odd_evens_value))
                    if odds_only not in odds_only_list:
                        data_list.append(odd_evens_data)
        odds_key = ["game_code", "title", "match_id", "lg_id", "data", "source", "odds_only", "tag", "uuid",
                    "is_stringscene", "utime", "pt"]
        odds_value = ["wq", "odds", match_id, league_id, data_list, "hg3535", odds_onlys, tag_number, uuid,
                      0, utime, pt]
        odds_dict = dict(zip(odds_key, odds_value))
        # print(odds_dict)
        if data_list:
            res_data = Helper.async_post(url + '/setOdds', odds_dict)
            print(res_data)
            if "成功" in res_data:
                if res:
                    self.db.hg3535_wq_odds.update({'match_id': match_id, 'pt': pt}, {
                        '$set': {"data": data_list, "odds_only": odds_onlys, "utime": utime}}, upsert=True)
                else:
                    self.db.hg3535_wq_odds.insert(odds_dict)

        # 赛事结果
        da_list = []
        wq_rball = {"lg_id": league_id, "home_player_name": team_home, "guest_player_name": team_guest,
                    "home_player_let_plate": '', "guest_player_let_plate": '', "home_player_let_inning": "",
                    "guest_player_let_inning": "", "all_inning": "", "home_player_score": score_home,
                    "guest_player_score": score_guest,
                    "status": 1, "first_score_player": "", "last_score_player": "",
                    "first_inning_score": score_dict['one_score'],
                    "second_inning_score": score_dict['second_score'], "third_inning_score": score_dict['third_score'],
                    "match_winer_player": "", "update_time": "",
                    "match_time": time_game, "match_process": jijie, "tag": tag_number, "match_id": match_id,
                    "source": "hg3535",
                    "match_score": score_dict['current_point_home_guest'], "ctime": ctime, "inning": "",
                    "strat_time": "", "uuid": uuid, "result_mark": score_dict
                    }

        da_list.append(wq_rball)
        r_data_dict = {
            "game_code": "wq",
            "title": "match_result_r",
            "source": "hg3535",
            "data": da_list
        }
        if da_list:
            try:
                res = Helper.async_post(url + '/setMatchResult', r_data_dict)
                print(res)
            except Exception as e:
                logger.warning('滚球接口数据异常, 提交失败')
                logger.warning(e)