sports_scrapy/hg3535/pipeline/guanjun.py

# import datetime
import pymongo
import time
import logging

from twisted.internet import defer, reactor

from ..utils.helper import Helper
from ..settings import M_HOST, M_USER, M_PASSWORD, M_POST, M_DB, LEAGUE_URL, ODDSCH


class Guanjunpipeline(object):
    def open_spider(self, spider):
        self.mongo = pymongo.MongoClient(host=M_HOST, username=M_USER, password=M_PASSWORD, port=M_POST)
        self.db = self.mongo[M_DB]

    @defer.inlineCallbacks
    def process_item(self, item, spider):
        out = defer.Deferred()
        reactor.callInThread(self._do_calculation, item, out)
        yield out
        defer.returnValue(item)

    # def process_item(self, item, spider):
    def _do_calculation(self, item, out):
        logger = logging.getLogger(__name__)
        data_game = item['data_game'].replace('年', "-").replace('月', "-").replace('日', "")
        time_game = item['time_game']
        ctime = data_game + " " + time_game + ":00"
        # uptime = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime())
        league_name = item['league_name']
        # tema_home = item['tema_home']
        league_id = item['league_id']
        # match_id = item['game_id']
        new_league_name = item['new_league_name']
        # 冠军赔率
        # champion_team = round(item['champion_team'], 2)
        # 冠军玩法
        champion_datas = item['champion_data']
        # 构建唯一哈希索引
        time1 = time.mktime(time.strptime(ctime, '%Y-%m-%d %H:%M:%S')) + 43200
        time2 = time.localtime(time1)
        time3 = time.strftime('%Y-%m-%d %H:%M:%S', time2)
        ball = item['ball']
        if ball == 1:
            game_code = 'zq'
            league_table = 'zq_cleague35'
            odds_table = "zq_chain35"
        elif ball == 2:
            game_code = 'lq'
            league_table = 'lq_cleague35'
            odds_table = "lq_chain35"
        elif ball == 3:
            game_code = 'wq'
            league_table = "wq_cleague35"
            odds_table = "wq_chain35"
        elif ball == 4:
            game_code = 'bq'
            league_table = "bq_cleague35"
            odds_table = "bq_chain35"
        else:
            print(ball, "冠军数据球类错误")
            return
        uuid = Helper.genearte_uuid(league_name + 'hg3535')
        if self.db[league_table].find({'lg_id': league_id}).count() < 1:
            league_list = []
            payload_key = ['game_code', 'title', 'lg_id', 'source']
            payload_value = [game_code, 'league', league_id, 'hg3535']
            l_payload = dict(zip(payload_key, payload_value))

            league_key = ['name_chinese', 'kind', 'match_mode', 'if_stop', 'identity', 'belong', 'last_time',
                          'lg_id', 'source', 'uuid']
            league_value = [new_league_name, '0', '0', '0', uuid, '0', time3, league_id, 'hg3535', uuid]
            league_dict = dict(zip(league_key, league_value))
            league_list.append(league_dict)
            l_payload['data'] = league_list
            try:
                res = Helper.async_post(LEAGUE_URL, l_payload)
                if res:
                    if res.get('status') == 1:
                        logger.info('{}, {}冠军联赛, 提交成功, {}'.format(game_code, league_name, res))
                        logger.info(league_list)
                        self.db[league_table].insert(league_dict)
                        # self.db[league_table].update({'lg_id': league_id}, {'$set': league_dict},
                        #                                 upsert=True)
                    else:
                        logger.warning('{}, {}, 冠军联赛, 提交失败, {}'.format(game_code, league_name, res))
                        logger.warning(l_payload)
                else:
                    logger.warning('{}, {}, 冠军联赛, 接口异常, 提交失败, {}'.format(game_code, league_name, res))
                    logger.warning(l_payload)
            except Exception as e:
                logger.warning('冠军联赛接口异常, 提交失败, {}'.format(e))
                logger.warning(e)
        else:
            logging.info('{},冠军联赛数据已存在'.format(league_name))
        payload_key1 = ['game_code', 'title', 'source', 'lg_id', 'uuid']
        payload_value1 = [game_code, 'odds_ch', 'hg3535', league_id, uuid]
        chain_dict = dict(zip(payload_key1, payload_value1))
        data_list = []
        only_list = []
        set_id = item['set_id']
        for champion_data in champion_datas:
            mn = champion_data['mn']
            odd_datas = champion_data['o']
            for odd_data in odd_datas:
                tema = odd_data[0]
                odd = round(float(odd_data[2]) - 1, 2)
                sole = Helper.genearte_MD5(mn + str(league_id) + league_name + new_league_name + tema, 0)
                odds_only = Helper.genearte_MD5(
                    mn + str(league_id) + league_name + new_league_name + str(odd) + tema, 0)
                # if self.db[odds_table].find({'odds_only': odds_only}).count() < 1:
                data_key = ['match_id', 'lg_id', 'odds_code', 'status', 'sort', 'p_code', 'odds',
                            'condition', 'odds_only', 'sole', 'source', 'type', 'team', 'uuid']
                data_value = [set_id, league_id, mn, '0', '0', new_league_name, odd, '0', odds_only,
                              sole, 'hg3535', '1', tema, uuid]
                chain_data = dict(zip(data_key, data_value))
                data_list.append(chain_data)
        if data_list:
            chain_dict['data'] = data_list
            chain_dict['odds_only'] = only_list
            try:
                res = Helper.async_post(ODDSCH, chain_dict)
                if res:
                    if res.get('status') == 1:
                        logger.info('{}冠军赔率，提交成功, {}'.format(game_code, res))
                        logger.info(chain_dict)
                        self.db[odds_table].insert(chain_data)
                    else:
                        logger.warning('{}, 冠军赔率接口, 提交失败, {}'.format(game_code, res))
                        logger.warning(chain_dict)
                else:
                    logger.warning('{}, 冠军赔率接口, 接口异常, 提交失败, {}'.format(game_code, res))
                    logger.warning(chain_dict)
            except Exception as e:
                logger.warning('冠军赔率接口异常，提交失败')
                logger.warning(e)

        else:
            logger.info('足球冠军赔率, 列表为空, 不提交过')
        reactor.callFromThread(out.callback, item)