import datetime import json import time import logging import pymongo from twisted.internet import defer, reactor from ..utils.helper import Helper from ..settings import M_HOST, M_USER, M_PASSWORD, M_POST, M_DB, LEAGUE_URL, ODDS_URL, MATCH_URL, MATCH_RESULT class RollPipeline(object): def open_spider(self, spider): self.mongo = pymongo.MongoClient(host=M_HOST, username=M_USER, password=M_PASSWORD, port=M_POST, authSource=M_DB) self.db = self.mongo[M_DB] with open('./conf/zuqiu.json', 'r', encoding='utf8') as zq: hgg070 = json.load(zq) self.hgg070 = hgg070 # @defer.inlineCallbacks # def process_item(self,item,spider): # out=defer.Deferred() # reactor.callInThread(self._do_calculation,item,out) # yield out # def _do_calculation(self,item,out): def process_item(self, item, spider): logger = logging.getLogger(__name__) match_all = item['data'] team_h, team_c = match_all['team_h'], match_all['team_c'] league_name, league_id = match_all['league'], match_all['gidm'] us_time, re_time = match_all['datetime'], match_all['re_time'] match_id = match_all['gid'] tag_number = item['tag'] uuid = Helper.genearte_uuid(league_name) league_list = [] pt = 4 last_time = '{}-12-31 23:59:59'.format(datetime.datetime.now().year) match_date, match_time, time3 = Helper.change_time(us_time) utime = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime()) if self.db.zq_league070.find({'lg_id': league_id}).count() < 1: league_dict = {"game_code": "zq", "title": "league", "source": "hgg070"} league_key = ["name_chinese", "kind", "match_mode", "if_stop", "last_time", "lg_id", "source", "uuid"] league_value = [league_name, "1", "1", "0", last_time, league_id, "hgg070", uuid] league_data = dict(zip(league_key, league_value)) league_list.append(league_data) league_dict['data'] = league_list res = Helper.async_post(LEAGUE_URL, league_dict) if res: if res.get('status') == 1: self.db.zq_league070.insert(league_data) logging.info('足球联赛提交, {}'.format(res)) else: logging.warning('足球联赛接口异常, {}'.format(res)) else: logging.info('{},联赛已存在, 不提交'.format(league_name)) json_key = 'zq_rollball' match_list = [] match_identity = Helper.genearte_uuid(team_h + team_c + match_date) ptype_id = match_all.get('ptype_id') is_horn = 0 if ptype_id == '146': ptype = match_all['ptype'] team_h = team_h.replace(ptype, '').strip() team_c = team_c.replace(ptype, '').strip() is_horn = 1 if self.db.zq_competition070.find({'match_id': match_id, 'is_rollball': 1}).count() < 1: match_dict = {"game_code": "zq", "title": "match", "source": "hgg070"} match_kay = ["home_team", "guest_team", "lg_id", "status", "match_id", "match_date", "match_time", "tag", "source", "is_rollball", "is_morningplate", "is_stringscene", "us_time", "uuid", "half_match_id", "is_today", "is_horn", 'match_identity'] match_value = [team_h, team_c, league_id, 0, match_id, match_date, match_time, tag_number, "hgg070", 1, 0, 0, us_time, uuid, 0, 0, is_horn, match_identity] match_data = dict(zip(match_kay, match_value)) match_list.append(match_data) match_dict['data'] = match_list res = Helper.async_post(MATCH_URL, match_dict) if res: if res.get('status') == 1: self.db.zq_competition070.insert(match_data) logging.info('足球赛事提交, {}'.format(res)) else: logger.warning('足球赛事表提交失败, {}'.format(res)) # logger.warning(match_dict) else: logger.warning('足球赛事接口异常提交失败, {}'.format(res)) # logger.warning(match_dict) else: logger.info('足球赛事已存在，不提交') data_list = [] for i in self.hgg070[json_key]: r_code = i['prodds'] p_code = i['plodds'] if match_all.get(r_code) == 'Y': for y in i['items']: odd = match_all.get(y['rodds']) if odd: code = y['lodds'] sole_str = p_code + code + '0' + str(match_id) + 'hgg070' sole = Helper.genearte_MD5(sole_str, pt) ratio_name = y['ratio_name'] if ratio_name: condition = match_all[ratio_name] else: condition = y['latio'] hash_str = p_code + code + '0' + str(odd) + "hgg070" + str(match_id) odds_only = Helper.genearte_MD5(hash_str, pt) odd_key = ["match_id", "lg_id", "odds_code", "status", "sort", "p_code", "odds", "condition", "odds_only", "sole", "source", "type", "team"] odd_value = [match_id, league_id, code, '0', '0', code, odd, condition, odds_only, sole, 'hgg070', '0', ''] odd_dict = dict(zip(odd_key, odd_value)) data_list.append(odd_dict) odds_key = ["game_code", "title", "match_id", "lg_id", "data", "source", "odds_only", "tag", "uuid", "is_stringscene", "utime", "pt", 'match_identity'] odds_value = ["zq", "odds", match_id, league_id, data_list, "hgg070", '', tag_number, uuid, 0, utime, pt, match_identity] odds_dict = dict(zip(odds_key, odds_value)) if data_list: res = Helper.async_post(ODDS_URL, odds_dict) if res: if res.get('status') == 1: logger.info('足球详细赔率提交成功, {}'.format(res)) # logger.info(odds_dict) else: logger.warning('足球详细赔率提交失败, {}'.format(res)) # logger.warning(odds_dict) else: logging.warning('足球详细赔率接口异常, {}'.format(res)) else: logger.info('足球详细赔率列表为空') data_result = [] process = {"HT": "半场", "1H": "上半场", "2H": "下半场"} match_process = process[re_time.split('^')[0]] time_game = re_time.split('^')[1][:-1] score_h = match_all.get('score_h') score_c = match_all.get('score_c') # all_goal = score_h + score_c # match_score = '{}:{}'.format(score_h, score_c) zq_rball = {"home_team": team_h, "guest_team": team_c, "lg_id": league_id, "home_rate": 0, "guest_rate": 0, "home_score": score_h, "guest_score": score_c, "all_goal": '', "status": 1, "first_score": "", "last_score": "", "match_score": '', "uuid": uuid, "match_winer": "", "match_time": time_game, 'match_identity': match_identity, "match_process": match_process, "tag": tag_number, "match_id": match_id, "p_code": ""} data_result.append(zq_rball) r_data_dict = { "game_code": "zq", "title": "match_result_r", "source": "hgg070", "data": data_result } if data_result: try: res = Helper.async_post(MATCH_RESULT, r_data_dict) if res: if res.get('status') == 1: logger.info('足球滚球结果记录提交成功, {}'.format(res)) self.db.zq_match_result35.insert(zq_rball) # logger.info(r_data_dict) else: logger.warning('足球滚球结果记录提交失败, {}'.format(res)) # logger.warning(r_data_dict) else: logger.warning('足球滚球结果记录接口异常，提交失败, {}'.format(res)) # logger.warning(r_data_dict) except Exception as e: logger.warning('滚球数据接口异常，提交失败, {}'.format(e)) # reactor.callFromThread(out.callback, item)