# import datetime import pymongo import time import logging from twisted.internet import defer, reactor from ..utils.helper import Helper from ..settings import M_HOST, M_USER, M_PASSWORD, M_POST, M_DB, LEAGUE_URL, ODDSCH class Guanjunpipeline(object): def open_spider(self, spider): self.mongo = pymongo.MongoClient(host=M_HOST, username=M_USER, password=M_PASSWORD, port=M_POST) self.db = self.mongo[M_DB] @defer.inlineCallbacks def process_item(self, item, spider): out = defer.Deferred() reactor.callInThread(self._do_calculation, item, out) yield out defer.returnValue(item) # def process_item(self, item, spider): def _do_calculation(self, item, out): # def process_item(self, item, spider): logger = logging.getLogger(__name__) data_game = item['data_game'].replace('年', "-").replace('月', "-").replace('日', "") time_game = item['time_game'] ctime = data_game + " " + time_game + ":00" # uptime = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime()) league_name = item['league_name'] tema_home = item['tema_home'] league_id = item['league_id'] match_id = item['game_id'] new_league_name = item['new_league_name'] # 冠军赔率 champion_team = round(item['champion_team'], 2) # 冠军玩法 new_champion = item['new_champion'] # 构建唯一哈希索引 time1 = time.mktime(time.strptime(ctime, '%Y-%m-%d %H:%M:%S')) + 43200 time2 = time.localtime(time1) time3 = time.strftime('%Y-%m-%d %H:%M:%S', time2) ball = item['ball'] if ball == '足球': game_code = 'zq' league_table = 'zq_league35' odds_table = "zq_chain35" elif ball == '篮球': game_code = 'lq' league_table = 'lq_league35' odds_table = "lq_chain35" elif ball == '网球': game_code = 'wq' league_table = "wq_league35" odds_table = "wq_chain35" elif ball == '棒球': game_code = 'bq' league_table = "bq_league35" odds_table = "bq_chain35" else: print(ball, "冠军数据球类错误") return uuid = Helper.genearte_uuid(league_name + 'hg3535') if self.db[league_table].find({'uuid': uuid}).count() < 1: league_list = [] payload_key = ['game_code', 'title', 'match_id', 'lg_id', 'source'] payload_value = [game_code, 'league', match_id, league_id, 'hg3535'] l_payload = dict(zip(payload_key, payload_value)) league_key = ['name_chinese', 'kind', 'match_mode', 'if_stop', 'identity', 'belong', 'last_time', 'lg_id', 'source', 'uuid'] league_value = [new_league_name, '0', '0', '0', uuid, '0', time3, league_id, 'hg3535', uuid] league_dict = dict(zip(league_key, league_value)) league_list.append(league_dict) l_payload['data'] = league_list try: res = Helper.async_post(LEAGUE_URL, league_list) if res: if "成功" in res: logger.info('{}冠军联赛, 提交成功, {}'.format(game_code, res)) logger.info(league_list) self.db[league_table].insert(league_list) else: logger.warning('{}, 冠军联赛, 提交失败, {}'.format(game_code, res)) logger.warning(l_payload) else: logger.warning('{}, 冠军联赛, 接口异常, 提交失败, {}'.format(game_code, res)) logger.warning(l_payload) except Exception as e: logger.warning('冠军联赛接口异常, 提交失败, {}'.format(e)) logger.warning(e) else: logging.info('冠军联赛数据已存在') sole = Helper.genearte_MD5(new_champion + str(league_id) + league_name + new_league_name + tema_home, 0) odds_only = Helper.genearte_MD5( new_champion + str(league_id) + league_name + new_league_name + str(champion_team) + tema_home + str( match_id), 0) if self.db[odds_table].find({'odds_only': odds_only}).count() < 1: data_list = [] payload_key1 = ['game_code', 'title', 'source'] payload_value1 = [game_code, 'odds_ch', 'hg3535'] only_list = [] chain_dict = dict(zip(payload_key1, payload_value1)) only_list.append(odds_only) data_key = ['match_id', 'lg_id', 'odds_code', 'status', 'sort', 'p_code', 'odds', 'condition', 'odds_only', 'sole', 'source', 'type', 'team', 'uuid'] data_value = [match_id, league_id, new_champion, '0', '0', new_league_name, champion_team, '0', odds_only, sole, 'hg3535', '1', tema_home, uuid] chain_data = dict(zip(data_key, data_value)) data_list.append(chain_data) chain_dict['data'] = data_list chain_dict['odds_only'] = only_list try: res = Helper.async_post(ODDSCH, chain_dict) if res: if "成功" in res: logger.info('{}冠军联赛, 提交成功, {}'.format(game_code, res)) logger.info(chain_dict) self.db[odds_table].insert(chain_data) else: logger.warning('{}, 冠军联赛, 提交失败, {}'.format(game_code, res)) logger.warning(chain_dict) else: logger.warning('{}, 冠军联赛, 接口异常, 提交失败, {}'.format(game_code, res)) logger.warning(chain_dict) except Exception as e: logger.warning('冠军赔率接口异常,提交失败') logger.warning(e) else: logger.info('足球冠军已经提交过了') reactor.callFromThread(out.callback, item)