import datetime import pymongo import time import logging # import json from twisted.internet import defer, reactor from ..utils.helper import Helper from ..settings import M_HOST, M_USER, M_PASSWORD, M_POST, M_DB, LEAGUE_URL, ODDS_URL, MATCH_URL, MATCH_RESULT, MATCH_STATUS class Roll_Banqiupipeline(object): def open_spider(self, spider): self.mongo = pymongo.MongoClient(host=M_HOST, username=M_USER, password=M_PASSWORD, port=M_POST, authSource='kaiyou') self.db = self.mongo[M_DB] @defer.inlineCallbacks def process_item(self, item, spider): out = defer.Deferred() reactor.callInThread(self._do_calculation, item, out) yield out defer.returnValue(item) # def process_item(self, item, spider): def _do_calculation(self, item, out): # 使用twisted将mysql插入变成异步执行 logger = logging.getLogger(__name__) # 联赛id league_id = item['league_id'] # 联赛名 league_name = item['league_name'] # 比赛id match_id = item['game_id'] # 球队1 team_home = item['team_home'] # 球队2 team_guest = item['team_guest'] # 数量(97>) tag_number = item['number'] # 日期 data_game = item['data_game'].split("/") month = str(data_game[1].strip()) day = str(data_game[0]) # 比赛时间 time_game = str(item['time_game']) # 比赛时间，时间戳 ctime = "2019" + "-" + month + "-" + day + "" + time_game + ":00".strip() # 现在时间，时间戳 utime = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime()) cdate = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S") match_date = cdate.split(' ')[0] match_time = cdate.split(' ')[1] # 队1分数 score_home = item['score_home'] # 队2分数 score_guest = item['score_guest'] # 第几节 jijie = item['jijie'] pt = 4 # 滚球这个位置获取不到这些字段 # match_date, match_time = new_time(ctime) # 让球 concedes_dict = item['concedes_dict'] concedes_dict_rule = item['concedes_dict_rule'] # 独赢 capots_dict = item['capots_dict'] # 总得分大小 total_size_dict = item['total_size_dict'] total_size_dict_rule = item['total_size_dict_rule'] # 总得分单双 odd_evens_dict = item['odd_evens_dict'] odd_evens_dict_rule = item['odd_evens_dict_rule'] # 具体比分 source_dict = item['source_dict'] uuid = Helper.genearte_uuid(league_name + 'hg3535') """联赛""" last_time = '{}-12-31 23:59:59'.format(datetime.datetime.now().year) if self.db.bq_league35.find({'lg_id': league_id}).count() < 1: league_dict = {"game_code": "bq", "title": "league", "source": "hg3535"} league_list = [] league_key = ["name_chinese", "kind", "match_mode", "if_stop", "last_time", "lg_id", "source", "uuid"] league_value = [league_name, "1", "1", "0", last_time, league_id, "hg3535", uuid] league_data = dict(zip(league_key, league_value)) league_list.append(league_data) league_dict['data'] = league_list res = Helper.async_post(LEAGUE_URL, league_dict) if res: if res.get('status') == 1: self.db.bq_league35.insert(league_data) # self.db.bq_competition35.update({'lg_id': league_id}, {'$set': league_data}, # upsert=True) logger.info('棒球滚球联赛提交成功, {}'.format(res)) else: logger.warning('棒球滚球联赛提交失败, {}'.format(res)) logger.warning(league_dict) else: logger.warning('棒球滚球联赛,接口异常,提交失败, {}'.format(res)) logger.warning(league_dict) else: logger.info('棒球滚球联赛, 已存在不提交') """赛事""" if self.db.bq_competition35.find({"match_id": match_id, 'is_rollball': 1}).count() < 1: match_list = [] match_dict = {"game_code": "bq", "title": "match", "source": "hg3535"} match_key = ["home_team", "guest_team", "lg_id", "status", "match_id", "match_date", "match_time", "tag", "source", "is_rollball", "is_morningplate", "is_stringscene", "us_time", "uuid", "half_match_id", "is_today", "is_horn"] match_value = [team_home, team_guest, league_id, 1, match_id, match_date, match_time, tag_number, "hg3535", 1, 0, 0, utime, uuid, 0, 0, 0] match_data = dict(zip(match_key, match_value)) match_list.append(match_data) match_dict['data'] = match_list res = Helper.async_post(MATCH_URL, match_dict) if res: if res.get('status') == 1: self.db.bq_competition35.insert(match_data) # self.db.bq_competition35.update({"match_id": match_id, 'is_rollball': 1}, {'$set': match_data}, # upsert=True) else: logger.warning('棒球滚球赛事提交失败, {}'.format(res)) logger.warning(match_data) else: logger.warning('棒球滚球赛事,接口异常,提交失败, {}'.format(res)) logger.warning(match_data) else: logger.info("棒球赛事已存在，棒球赛事状态时间未改变，不做修改") """赔率""" data_list = [] odds_onlys = [] # 让球 if concedes_dict: for key, value in concedes_dict.items(): if value: sole_str = 'CO' + str(key) + '0' + str(match_id) + 'hg3535' sole = Helper.genearte_MD5(sole_str, pt) odds_str = 'CO' + str(key) + '0' + str(concedes_dict_rule[key]) + str( value) + 'hg3535' + str(match_id) odds_only = Helper.genearte_MD5(odds_str, pt) condition = concedes_dict_rule[key] concedes_key = ["match_id", "lg_id", "odds_code", "status", "sort", "p_code", "odds", "condition", "odds_only", "sole", "source", "type", "team"] concedes_value = [match_id, league_id, key, "0", 0, 'CO', value, condition, odds_only, sole, "hg3535", "0", ""] concedes_data = dict(zip(concedes_key, concedes_value)) data_list.append(concedes_data) # 独赢 if capots_dict: for key, value in capots_dict.items(): if value: sole_str = 'C' + str(key) + '0' + str(match_id) + 'hg3535' sole = Helper.genearte_MD5(sole_str, pt) odds_str = 'C' + str(key) + '0' + '0' + str(value) + 'hg3535' + str(match_id) odds_only = Helper.genearte_MD5(odds_str, pt) capots_key = ["match_id", "lg_id", "odds_code", "status", "sort", "p_code", "odds", "condition", "odds_only", "sole", "source", "type", "team"] capots_value = [match_id, league_id, key, "0", 0, 'C', value, '', odds_only, sole, "hg3535", "0", ""] capots_data = dict(zip(capots_key, capots_value)) data_list.append(capots_data) # 总得分大小 if total_size_dict: for key, value in total_size_dict.items(): if value: sole_str = 'TN' + str(key) + '0' + str(match_id) + 'hg3535' sole = Helper.genearte_MD5(sole_str, pt) odds_str = 'TN' + str(key) + '0' + str(total_size_dict_rule[key]) + str( value) + 'hg3535' + str(match_id) odds_only = Helper.genearte_MD5(odds_str, pt) condition = total_size_dict_rule[key] total_size_key = ["match_id", "lg_id", "odds_code", "status", "sort", "p_code", "odds", "condition", "odds_only", "sole", "source", "type", "team"] total_size_value = [match_id, league_id, key, "0", 0, 'TN', value, condition, odds_only, sole, "hg3535", "0", ""] total_size_data = dict(zip(total_size_key, total_size_value)) data_list.append(total_size_data) # 总得分单双 if odd_evens_dict: for key, value in odd_evens_dict.items(): sole_str = 'TS' + str(key) + '0' + str(match_id) + 'hg3535' sole = Helper.genearte_MD5(sole_str, pt) odds_str = 'TS' + str(key) + '0' + str(odd_evens_dict_rule[key]) + str(value) + 'hg3535' + str( match_id) odds_only = Helper.genearte_MD5(odds_str, pt) condition = odd_evens_dict_rule[key] odd_evens_key = ["match_id", "lg_id", "odds_code", "status", "sort", "p_code", "odds", "condition", "odds_only", "sole", "source", "type", "team"] odd_evens_value = [match_id, league_id, key, "0", 0, 'TS', value, condition, odds_only, sole, "hg3535", "0", ""] odd_evens_data = dict(zip(odd_evens_key, odd_evens_value)) data_list.append(odd_evens_data) odds_key = ["game_code", "title", "match_id", "lg_id", "data", "source", "odds_only", "tag", "uuid", "is_stringscene", "utime", "pt"] odds_value = ["bq", "odds", match_id, league_id, data_list, "hg3535", odds_onlys, tag_number, uuid, 0, utime, pt] odds_dict = dict(zip(odds_key, odds_value)) if data_list: res = Helper.async_post(ODDS_URL, odds_dict) if res: if res.get('status') == 1: logger.info('棒球滚球赔率提交成功, {}'.format(res)) logger.info(odds_dict) else: logger.warning('棒球滚球赔率提交失败, {}'.format(res)) logger.warning(odds_dict) else: logger.warning('棒球滚球赔率接口异常,提交失败, {}'.format(res)) logger.warning(odds_dict) else: logger.info('棒球滚球详细赔率列表为空, 不提交') """赛事结果""" da_list = [] bq_rball = { "lg_id": league_id, "home_team": team_home, "guest_team": team_guest, "home_rate": '', "guest_rate": '', "home_score": score_home, "guest_score": score_guest, "status": 1, "first_score": '', "last_score": '', "all_inning": '', "match_score": '', "match_winer": '', "updata_time": '', "match_time": time_game, "match_process": jijie, "tag": tag_number, "match_id": match_id, "source": 'hg3535', "ctime": ctime, "uuid": uuid, "result_mark": source_dict, "start_time": '', 'warn_more': '', } da_list.append(bq_rball) r_data_dict = { "game_code": "bq", "title": "match_result_r", "source": "hg3535", "data": da_list, } if da_list: try: res = Helper.async_post(MATCH_RESULT, r_data_dict) if res.get('status') == 1: logger.info('棒球滚球结果记录提交成功, {}'.format(res)) logger.info(r_data_dict) else: logger.warning('棒球滚球结果记录提交失败, {}'.format(res)) logger.warning(r_data_dict) except Exception as e: logger.warning("滚球接口数据异常, 提交失败, {}".format(e)) logging.warning(r_data_dict) else: logger.info('棒球滚球结果记录为空, 不提交') # status_dict = {"game_code": 'bq', "title": "match_status", "source": "hg3535"} # data_list = [] # data = {'match_id': match_id, 'status': 1, "is_rollball": 0, "is_today": 0, "is_morningplate": 0, # "is_stringscene": 0, "is_horn": 0} # data_list.append(data) # status_dict['data'] = data_list # res = Helper.async_post(MATCH_STATUS, status_dict) # if res: # if res.get('status') == 1 # self.db.match_status35.insert(status_dict) # logger.info('{},赛事结果状态交成功, {}'.format('bq', res)) # logger.info(status_dict) # else: # logger.warning('{},赛事结果状态交失败, {}'.format('bq', res)) # logger.warning(status_dict) # else: # logger.warning('{},赛事结果状态接口异常提交失败, {}'.format('bq', res)) # logger.warning(status_dict) reactor.callFromThread(out.callback, item)