sports_scrapy/hg3535/pipeline/guanjun.py

# import datetime
import pymongo
import time
import logging
from ..utils.helper import Helper


class Guanjunpipeline(object):
    def open_spider(self, spider):
        self.mongo = pymongo.MongoClient(host="192.168.2.200", username="kaiyou", password="kaiyou", port=27017)
        self.db = self.mongo['kaiyou']

    def process_item(self, item, spider):
        gj_odds = []
        url = 'http://admin.5gogo.com/'
        logger = logging.getLogger(__name__)
        data_game = item['data_game'].replace('年', "-").replace('月', "-").replace('日', "")
        time_game = item['time_game']
        ctime = data_game + " " + time_game + ":00"
        uptime = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime())
        # expire_time = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(time.time() + 60))
        league_name = item['league_name']
        tema_home = item['tema_home']
        league_id = item['league_id']
        match_id = item['game_id']
        new_league_name = item['new_league_name']
        # 冠军赔率
        champion_team = round(item['champion_team'], 2)
        # 冠军玩法
        new_champion = item['new_champion']
        # 构建唯一哈希索引
        time1 = time.mktime(time.strptime(ctime, '%Y-%m-%d %H:%M:%S')) + 43200
        time2 = time.localtime(time1)
        time3 = time.strftime('%Y-%m-%d %H:%M:%S', time2)
        # pt = item['pt']
        ball = item['ball']
        if ball == '足球':
            game_code = 'zq'
            league_table = 'hg3535_zq_league'
            odds_table = "hg3535_zq_chain_odds"
        elif ball == '篮球':
            game_code = 'lq'
            league_table = 'hg3535_lq_league'
            odds_table = "hg3535_lq_chain_odds"
        elif ball == '网球':
            game_code = 'wq'
            league_table = "hg3535_wq_league"
            odds_table = "hg3535_wq_chain_odds"
        elif ball == '棒球':
            game_code = 'bq'
            league_table = "hg3535_bq_league"
            odds_table = "hg3535_bq_chain_odds"
        else:
            print(ball, "冠军数据球类错误")
            return
        league_list = []
        uuid = Helper.genearte_MD5(league_name + 'hg0088')
        payload_key = ['game_code', 'title', 'match_id', 'lg_id', 'source']
        payload_value = [game_code, 'league', match_id, league_id, 'hg3535']
        l_payload = dict(zip(payload_key, payload_value))

        league_key = ['name_chinese', 'kind', 'match_mode', 'if_stop', 'identity', 'belong', 'last_time',
                      'lg_id', 'source', 'uuid']
        league_value = [new_league_name, '0', '0', '0', uuid, '0', time3, league_id, 'hg3535', uuid]
        league_dict = dict(zip(league_key, league_value))
        if self.db[league_table].find({'uuid': uuid}).count() < 1:
            league_list.append(league_dict)
            l_payload['data'] = league_list
            try:
                print("冠军联赛数据不为空，数据提交")
                response_data = Helper.async_post(url + '/setLeague', l_payload)
                print(response_data)
                print('请求冠军联赛成功')
                logger.info('请求足球冠军联赛成功')
            except Exception as e:
                logger.warning('足球冠军联赛数据数据接口异常，提交失败')
                logger.warning(e)
            self.db[league_table].insert(league_list)
        else:
            print('冠军联赛数据已存在')

        payload_key1 = ['game_code', 'title', 'source']
        payload_value1 = [game_code, 'odds_ch', 'hg3535']
        only_list = []
        chain_dict = dict(zip(payload_key1, payload_value1))
        sole = Helper.genearte_MD5(new_champion + str(league_id) + league_name + new_league_name + tema_home)
        odds_only = Helper.genearte_MD5(new_champion + str(league_id) + league_name + new_league_name + str(champion_team) + tema_home + str(match_id))
        only_list.append(odds_only)
        data_key = ['match_id', 'lg_id', 'odds_code', 'status', 'sort', 'p_code', 'odds',
                    'condition', 'odds_only', 'sole', 'source', 'type', 'team', 'uuid']
        data_value = [match_id, league_id, new_champion, '0', '0', new_league_name, champion_team, '0', odds_only, sole, 'hg3535', '1', tema_home, uuid]
        chain_data = dict(zip(data_key, data_value))
        data_list = []
        if self.db[odds_table].find({'odds_only': odds_only}).count() < 1:
            data_list.append(chain_data)
            chain_dict['data'] = data_list
            chain_dict['odds_only'] = only_list
            try:
                response_data = Helper.async_post(url + '/setOddsCH', chain_dict)
                # logger.warning(response_data.text)
                print(odds_table, response_data)
                print(odds_table, '请求冠军赔率成功')
            except Exception as e:
                logger.warning('冠军赔率接口异常，提交失败')
                logger.warning(e)
            self.db[odds_table].insert(chain_data)
        else:
            print('足球冠军已经提交过了')
            logger.info('足球冠军已经提交过了')