sports_scrapy/hg3535/pipeline/wangqiu.py

import datetime
import pymongo
import time
import logging

from twisted.internet import defer, reactor

# from .ball_func import fuhao
from ..utils.helper import Helper
from .ball_func import new_time, out_time
from ..settings import M_HOST, M_USER, M_PASSWORD, M_POST, M_DB, LEAGUE_URL, ODDS_URL, MATCH_URL


class Wangqiupipeline(object):
    def open_spider(self, spider):
        # self.connection = AsyncIOMotorClient("mongodb://{}:{}@{}:{}/database?authSource={}".format('kaiyou', 'kaiyou', '192.168.2.200', 27017, 'kaiyou'))
        # session = aiohttp.ClientSession()
        self.mongo = pymongo.MongoClient(host=M_HOST, username=M_USER, password=M_PASSWORD, port=M_POST, authSource='kaiyou')
        self.db = self.mongo[M_DB]

    @defer.inlineCallbacks
    def process_item(self, item, spider):
        out = defer.Deferred()
        reactor.callInThread(self._do_calculation, item, out)
        yield out
        # defer.returnValue(item)

    def _do_calculation(self, item, out):
        logger = logging.getLogger(__name__)
        # logger.warning(query.addErrback(self.handle_error, item, spider))
        # 联赛id
        league_id = item['league_id']
        # 联赛名
        league_name = item['league_name']
        # result = item['result']
        # 比赛id
        match_id = item['game_id']
        # print(game_id)
        # 球队1
        team_home = item['team_home']
        # 球队2
        team_guest = item['team_guest']
        # 数量(97>)
        tag_number = item['number']
        # 比赛状态
        # zhuangtai = item['zhuangtai']
        # 日期
        # data_game = item['data_game']
        try:
            data_game = item['data_game'].split("/")
            month = str(data_game[1].strip())
            day = str(data_game[0]).strip()
        except Exception as e:
            logger.warning(e)
            data_game = item['data_game'].split(" ")
            months = str(data_game[1].strip())
            month_dict = {'Jan': '01', 'Feb': '02', 'Mar': '03', 'Apr': '04', 'May': '05', 'Jun': '06', 'Jul': '07',
                          'Aug': '08', 'Sep': '09', 'Oct': '10', 'Nov': '11', 'Dec': '12'}
            month = month_dict[months]
            day = str(data_game[0]).strip()
        year = datetime.datetime.now().year
        # 比赛时间
        time_game = str(item['time_game'])
        # 比赛时间，时间戳
        ctime = str(year) + "-" + month + "-" + day + " " + time_game + ":00"
        # r_ctime = str(year) + "-" + month + "-" + day
        # 现在时间，时间戳
        utime = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime())
        # expire_time = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(time.time() + 60))
        # 队1分数
        # score_home = item['score_home']
        # 队2分数
        # score_guest = item['score_guest']
        # 第几节
        # jijie = item['jijie']
        # 球队得分
        # qiudui = item['qiudui']
        pt = str(item['pt'])
        # 让盘
        concedes_dict = item['concedes_dict']
        concedes_dict_rule = item['concedes_dict_rule']
        # 冠军
        kemps_dict = item['kemps_dict']
        # 让局
        bureaus_dict = item['bureaus_dict']
        bureaus_dict_rule = item['bureaus_dict_rule']
        # 总局数大小
        total_number_dict = item['total_number_dict']
        total_number_dict_rule = item['total_number_dict_rule']
        # 总决赛单双
        odd_evens_dict = item['odd_evens_dict']
        odd_evens_dict_rule = item['odd_evens_dict_rule']

        us_time = ctime
        match_date, match_time, time3 = new_time(ctime)
        # n_time = out_time(time3, 3)
        uuid = Helper.genearte_uuid(league_name)
        """联赛"""
        last_time = '{}-12-31 23:59:59'.format(year)

        if self.db.wq_league35.find({'lg_id': league_id}).count() < 1:
        # if self.db.wq_league35.find({'uuid': uuid}).count() < 1:
            league_dict = {"game_code": "wq", "title": "league", "source": "hg3535"}
            league_list = []
            league_key = ["name_chinese", "kind", "match_mode", "if_stop", "last_time", "lg_id", "source", "uuid"]
            league_value = [league_name, "1", "1", "0", last_time, league_id, "hg3535", uuid]
            league_data = dict(zip(league_key, league_value))
            league_list.append(league_data)
            league_dict['data'] = league_list
            res = Helper.async_post(LEAGUE_URL, league_dict)
            if res:
                if res.get('status') == 1:
                    self.db.wq_league35.insert(league_data)
                    # self.db.wq_league35.update({'lg_id': league_id}, {'$set': league_data},
                    #                                 upsert=True)
                    logger.info('网球联赛提交成功, {}'.format(res))
                else:
                    logger.warning('网球联赛提交失败, {}'.format(res))
                    # logger.warning(league_dict)
            else:
                logger.warning('网球联赛接口异常,提交失败, {}'.format(res))
                # logger.warning(league_dict)
        else:
            logger.info('网球联赛已存在, 不提交')

        """赛事"""
        pt_dict = {'1': 'is_today', '2': 'is_morningplate', '3': 'is_stringscene', '4': 'is_rollball'}
        pt_status = pt_dict[pt]
        if pt == '3':
            is_rollball = 0
            is_today = 0
            is_morningplate = 0
            is_stringscene = 1
        elif pt == '2':
            is_rollball = 0
            is_today = 0
            is_morningplate = 1
            is_stringscene = 0
        else:
            is_today = 1
            is_rollball = 0
            is_morningplate = 0
            is_stringscene = 0
        match_identity = Helper.genearte_uuid(team_home + team_guest + match_date)
        if self.db.wq_competition35.find({'match_id': match_id, pt_status: 1}).count() < 1:
        # if self.db.wq_competition35.find({'match_identity': match_identity, pt_status: 1}).count() < 1:
            match_list = []
            match_dict = {"game_code": "wq", "title": "match", "source": "hg3535"}
            match_key = ["home_team", "guest_team", "lg_id", "status", "match_id", "match_date", "match_time",
                         "tag", "source", "is_rollball", "is_morningplate", "is_stringscene", "us_time", "uuid",
                         "half_match_id", "is_today", 'rule', "is_horn", "match_identity"]
            match_value = [team_home, team_guest, league_id, 0, match_id, match_date, match_time, tag_number,
                           "hg3535", is_rollball, is_morningplate, is_stringscene, us_time, uuid, 0, is_today, '', 0, match_identity]
            match_data = dict(zip(match_key, match_value))
            match_list.append(match_data)
            match_dict['data'] = match_list
            res = Helper.async_post(MATCH_URL, match_dict)
            if res:
                if res.get('status') == 1:
                    self.db.wq_competition35.insert(match_data)
                    # self.db.wq_competition35.update({'match_id': match_id, pt_status: 1}, {'$set': match_data},
                    #                                 upsert=True)
                    logger.info('网球赛事提交成功, {}'.format(res))
                else:
                    logger.warning('网球赛事提交失败, {}'.format(res))
                    # logger.warning(match_data)
            else:
                logger.warning('网球赛事接口异常,提交失败, {}'.format(res))
                # logger.warning(match_dict)
        else:
            logging.info('网球赛事已存在, 不提交')

        """赔率"""
        data_list = []
        odds_onlys = []
        # 让盘
        if concedes_dict:
            for key, value in concedes_dict.items():
                sole_str = 'LD' + str(key) + '0' + str(match_id) + 'hg3535'
                sole = Helper.genearte_MD5(sole_str, pt)
                odds_str = 'LD' + str(key) + str(match_id) + 'hg3535' + str(concedes_dict_rule[key]) + str(
                    value)
                odds_only = Helper.genearte_MD5(odds_str, pt)
                condition = concedes_dict_rule[key]
                concedes_dict_key = ["match_id", "lg_id", "odds_code", "status", "sort", "p_code", "odds", "condition",
                                     "odds_only", "sole", "source", "type", "team"]
                concedes_dict_value = [match_id, league_id, key, '0', 0, 'LD', value, condition,
                                       odds_only, sole, 'hg3535', '0', '']
                concedes_dict_data = dict(zip(concedes_dict_key, concedes_dict_value))
                data_list.append(concedes_dict_data)
        # 让局
        if bureaus_dict:
            for key, value in bureaus_dict.items():
                sole_str = 'LB' + str(key) + str(match_id) + 'hg3535'
                sole = Helper.genearte_MD5(sole_str, pt)
                odds_str = 'LB' + str(key) + str(match_id) + 'hg3535' + str(bureaus_dict_rule[key]) + str(
                    value)
                odds_only = Helper.genearte_MD5(odds_str, pt)
                condition = bureaus_dict_rule[key]
                bureaus_dict_key = ["match_id", "lg_id", "odds_code", "status", "sort", "p_code", "odds", "condition",
                                    "odds_only", "sole", "source", "type", "team"]
                bureaus_dict_value = [match_id, league_id, key, '0', 0, 'LB', value, condition,
                                      odds_only, sole, 'hg3535', '0', '']
                bureaus_dict_data = dict(zip(bureaus_dict_key, bureaus_dict_value))
                data_list.append(bureaus_dict_data)
        # 总局数大小
        if total_number_dict:
            for key, value in total_number_dict.items():
                sole_str = 'TN' + str(key) + str(match_id) + 'hg3535'
                sole = Helper.genearte_MD5(sole_str, pt)
                odds_str = 'TN' + str(key) + str(match_id) + 'hg3535' + str(
                    total_number_dict_rule[key]) + str(value)
                odds_only = Helper.genearte_MD5(odds_str, pt)
                condition = total_number_dict_rule[key]
                total_number_dict_key = ["match_id", "lg_id", "odds_code", "status", "sort", "p_code", "odds",
                                         "condition",
                                         "odds_only", "sole", "source", "type", "team"]
                total_number_dict_value = [match_id, league_id, key, '0', 0, 'TN', value, condition,
                                           odds_only, sole, 'hg3535', '0', '']
                total_number_dict_data = dict(zip(total_number_dict_key, total_number_dict_value))
                data_list.append(total_number_dict_data)
        # 总局数单双
        if odd_evens_dict:
            for key, value in odd_evens_dict.items():
                sole_str = 'TS' + str(key) + str(match_id) + 'hg3535'
                sole = Helper.genearte_MD5(sole_str, pt)
                odds_str = 'TS' + str(key) + str(match_id) + 'hg3535' + str(odd_evens_dict_rule[key]) + str(
                    value)
                odds_only = Helper.genearte_MD5(odds_str, pt)
                condition = odd_evens_dict_rule[key]
                odd_evens_dict_key = ["match_id", "lg_id", "odds_code", "status", "sort", "p_code", "odds", "condition",
                                      "odds_only", "sole", "source", "type", "team"]
                odd_evens_dict_value = [match_id, league_id, key, '0', 0, 'TS', value, condition,
                                        odds_only, sole, 'hg3535', '0', '']
                odd_evens_dict_data = dict(zip(odd_evens_dict_key, odd_evens_dict_value))
                data_list.append(odd_evens_dict_data)
        # 冠军
        if kemps_dict:
            for key, value in kemps_dict.items():
                sole_str = 'C' + str(key) + str(match_id) + 'hg3535'
                sole = Helper.genearte_MD5(sole_str, pt)
                odds_str = 'C' + str(key) + str(match_id) + 'hg3535' + '2' + str(value)
                odds_only = Helper.genearte_MD5(odds_str, pt)
                kemps_dict_key = ["match_id", "lg_id", "odds_code", "status", "sort", "p_code", "odds", "condition",
                                  "odds_only", "sole", "source", "type", "team"]
                kemps_dict_value = [match_id, league_id, key, '0', 0, 'C', value, '',
                                    odds_only, sole, 'hg3535', '0', '']
                kemps_dict_data = dict(zip(kemps_dict_key, kemps_dict_value))
                data_list.append(kemps_dict_data)
        if pt == '3':
            ris_stringscene = 1
        else:
            ris_stringscene = 0
        odds_key = ["game_code", "title", "match_id", "lg_id", "data", "source", "odds_only", "tag", "uuid",
                    "is_stringscene", "utime", "pt", 'match_identity']
        odds_value = ["wq", "odds", match_id, league_id, data_list, "hg3535", odds_onlys, tag_number, uuid,
                      ris_stringscene, utime, pt, match_identity]
        odds_dict = dict(zip(odds_key, odds_value))
        if data_list:
            res = Helper.async_post(ODDS_URL, odds_dict)
            if res:
                if res.get('status') == 1:
                    logger.info('网球赔率提交成功, {}'.format(res))
                    # logger.info(odds_dict)
                else:
                    logger.warning('网球赔率提交失败, {}'.format(res))
                    # logger.warning(odds_dict)
            else:
                logger.warning('网球详细接口赔率返回异常, {}'.format(res))
                # logger.warning(odds_dict)
        else:
            logger.info('网球详细赔率列表为空, 不提交')

        reactor.callFromThread(out.callback, item)