import datetime import pymongo import time import logging from twisted.internet import defer, reactor # from .ball_func import fuhao from ..utils.helper import Helper from .ball_func import new_time from ..settings import M_HOST, M_USER, M_PASSWORD, M_POST, M_DB, LEAGUE_URL, ODDS_URL, MATCH_URL, MATCH_RESULT class Lanqiupipeline(object): def open_spider(self, spider): self.mongo = pymongo.MongoClient(host=M_HOST, username=M_USER, password=M_PASSWORD, port=M_POST) self.db = self.mongo[M_DB] @defer.inlineCallbacks def process_item(self, item, spider): out = defer.Deferred() reactor.callInThread(self._do_calculation, item, out) yield out defer.returnValue(item) # self._do_calculation(item) # return # def process_item(self, item, spider): def _do_calculation(self, item, out): logger = logging.getLogger(__name__) # 联赛id league_id = item['league_id'] # 联赛名 league_name = item['league_name'] # result = item['result'] # 比赛id match_id = item['game_id'] # 球队1 team_home = item['team_home'] # 球队2 team_guest = item['team_guest'] # 数量(97>) tag_number = item['number'] # 比赛状态 # zhuangtai = item['zhuangtai'] # 日期 try: data_game = item['data_game'].split("/") month = str(data_game[1].strip()) day = str(data_game[0]) except Exception as e: logger.warning(e) data_game = item['data_game'].split(" ") months = str(data_game[1].strip()) month_dict = {'Jan': '01', 'Feb': '02', 'Mar': '03', 'Apr': '04', 'May': '05', 'Jun': '06', 'Jul': '07', 'Aug': '08', 'Sep': '09', 'Oct': '10', 'Nov': '11', 'Dec': '12'} month = month_dict[months] day = str(data_game[0]) # 比赛时间 time_game = str(item['time_game']) # 比赛时间，时间戳 us_time = "2019" + "-" + month + "-" + day + " " + time_game + ":00" # r_ctime = "2019" + "-" + month + "-" + day # 现在时间，时间戳 utime = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime()) # expire_time = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(time.time() + 60)) # # 队1分数 # score_home = item['score_home'] # # 队2分数 # score_guest = item['score_guest'] # # 第几节 # jijie = item['jijie'] # # 球队得分 # qiudui = item['qiudui'] pt = item['pt'] # 让球 concedes_dict = item['concede'] concedes_dict_rule = item['concede_rule'] # 总得分单双 odd_evens_dict = item['odd_even'] odd_evens_dict_rule = item['odd_even_rule'] # 总比分大小 total_sizes_dict = item['total_size'] total_sizes_dict_rule = item['total_size_rule'] # 球队得分最后一位数 last_numbers_dict = item['last_number'] # 独赢 capots_dict = item['capot'] # 球队得分大小 team_scores_dict = item['team_score'] team_scores_dict_rule = item['team_score_rule'] # 让球 # lanqiu = item['lanqiu'] match_date, match_time, time3 = new_time(us_time) uuid = Helper.genearte_uuid(league_name + 'hg3535') # 让球 数据插入数据库 """联赛""" last_time = '{}-12-31 23:59:59'.format(datetime.datetime.now().year) if self.db.hg3535_lq_league.find({'lg_id': league_id}).count() < 1: league_dict = {"game_code": "lq", "title": "league", "source": "hg3535"} league_list = [] league_key = ["name_chinese", "kind", "match_mode", "if_stop", "last_time", "lg_id", "source", "uuid"] league_value = [league_name, "1", "1", "0", last_time, league_id, "hg3535", uuid] league_data = dict(zip(league_key, league_value)) league_list.append(league_data) league_dict['data'] = league_list res = Helper.async_post(LEAGUE_URL, league_dict) if res: if '成功' in res: logger.info('篮球联赛提交成功, {}'.format(res)) self.db.hg3535_lq_league.insert(league_list) logger.info(league_dict) else: logger.warning('篮球联赛提交失败, {}'.format(res)) else: logger.warning('篮球联赛接口, 提交失败, {}'.format(res)) logger.warning(league_dict) else: logger.info('篮球联赛已存在') """赛事""" if pt == 3: is_rollball = 0 is_today = 0 is_morningplate = 0 is_stringscene = 1 elif pt == 2: is_rollball = 0 is_today = 0 is_morningplate = 1 is_stringscene = 0 else: is_today = 1 is_rollball = 0 is_morningplate = 0 is_stringscene = 0 pt_dict = {'1': 'is_today', '2': 'is_morningplate', '3': 'is_stringscene', '4': 'is_rollball'} pt_status = pt_dict[str(pt)] if self.db.lq_competition35.find({'match_id': match_id, pt_status: 1}).count() < 1: match_list = [] match_dict = {"game_code": "lq", "title": "match", "source": "hg3535"} match_kay = ["home_team", "guest_team", "lg_id", "status", "match_id", "match_date", "match_time", "tag", "source", "is_rollball", "is_morningplate", "is_stringscene", "us_time", "uuid", "half_match_id", "is_today", 'pt'] match_value = [team_home, team_guest, league_id, 0, match_id, match_date, match_time, tag_number, "hg3535", is_rollball, is_morningplate, is_stringscene, us_time, uuid, 0, is_today, pt] match_data = dict(zip(match_kay, match_value)) match_list.append(match_data) match_dict['data'] = match_list res = Helper.async_post(MATCH_URL, match_dict) if res: if '成功' in res: logger.info('篮球赛事提交成功, {}'.format(res)) self.db.hg3535_lq_competition.insert(match_list) logger.info(match_dict) else: logger.warning('篮球赛事提交失败, {}'.format(res)) logger.warning(match_dict) else: logger.warning('篮球赛事接口异常,提交失败, {}'.format(res)) logger.warning(match_dict) else: logger.info('篮球赛事已存在') """赔率""" data_list = [] odds_onlys = [] # 让球 if concedes_dict: for key, value in concedes_dict.items(): if value: for index, concedes_value in enumerate(value): sole_str = 'CO' + str(index) + str(match_id) + 'hg3535' sole = Helper.genearte_MD5(sole_str, pt) odds_str = 'CO' + str(index) + str(match_id) + 'hg3535' + str( concedes_dict_rule[key][index]) + str(concedes_value) odds_only = Helper.genearte_MD5(odds_str, pt) condition = str(concedes_dict_rule[key][index]) concedes_dict_key = ["match_id", "lg_id", "odds_code", "status", "sort", "p_code", "odds", "condition", "odds_only", "sole", "source", "type", "team"] concedes_dict_value = [match_id, league_id, key, '0', index, 'CO', concedes_value, condition, odds_only, sole, 'hg3535', '0', ''] concedes_dict_data = dict(zip(concedes_dict_key, concedes_dict_value)) data_list.append(concedes_dict_data) else: sole_str = 'CO' + str(key) + str(match_id) + 'hg3535' sole = Helper.genearte_MD5(sole_str, pt) odds_str = 'CO' + str(key) + str(match_id) + 'hg3535' + str( concedes_dict_rule[key]) + str(value) odds_only = Helper.genearte_MD5(odds_str, pt) condition = str(concedes_dict_rule[key]) concedes_dict_key = ["match_id", "lg_id", "odds_code", "status", "sort", "p_code", "odds", "condition", "odds_only", "sole", "source", "type", "team"] concedes_dict_value = [match_id, league_id, key, '0', 0, 'CO', value, condition, odds_only, sole, 'hg3535', '0', ''] concedes_dict_data = dict(zip(concedes_dict_key, concedes_dict_value)) data_list.append(concedes_dict_data) # 总得分单双 if odd_evens_dict: for key, value in odd_evens_dict.items(): sole_str = 'TS' + str(key) + str(match_id) + 'hg3535' sole = Helper.genearte_MD5(sole_str, pt) odds_str = 'TS' + str(key) + str(match_id) + 'hg3535' + str( odd_evens_dict_rule[key]) + str(value) odds_only = Helper.genearte_MD5(odds_str, pt) condition = str(odd_evens_dict_rule[key]) odd_evens_dict_key = ["match_id", "lg_id", "odds_code", "status", "sort", "p_code", "odds", "condition", "odds_only", "sole", "source", "type", "team"] odd_evens_dict_value = [match_id, league_id, key, '0', 0, 'TS', value, condition, odds_only, sole, 'hg3535', '0', ''] odd_evens_dict_data = dict(zip(odd_evens_dict_key, odd_evens_dict_value)) data_list.append(odd_evens_dict_data) # 总比分大小 if total_sizes_dict: for key, value in total_sizes_dict.items(): if value: for index, total_sizes_value in enumerate(value): sole_str = 'TN' + str(key) + str(match_id) + 'hg3535' sole = Helper.genearte_MD5(sole_str, pt) odds_str = 'TN' + str(key) + str(match_id) + 'hg3535' + str( total_sizes_dict_rule[key][index]) + str(total_sizes_value) odds_only = Helper.genearte_MD5(odds_str, pt) condition = str(total_sizes_dict_rule[key][index]) total_sizes_key = ["match_id", "lg_id", "odds_code", "status", "sort", "p_code", "odds", "condition", "odds_only", "sole", "source", "type", "team"] total_sizes_value = [match_id, league_id, key, '0', index, 'TN', total_sizes_value, condition, odds_only, sole, 'hg3535', '0', ''] total_sizes_data = dict(zip(total_sizes_key, total_sizes_value)) data_list.append(total_sizes_data) else: sole_str = 'TN' + str(key) + str(match_id) + 'hg3535' sole = Helper.genearte_MD5(sole_str, pt) odds_str = 'TN' + str(key) + str(match_id) + 'hg3535' + str( total_sizes_dict_rule[key]) + str(value) odds_only = Helper.genearte_MD5(odds_str, pt) condition = str(total_sizes_dict_rule[key]) total_sizes_key = ["match_id", "lg_id", "odds_code", "status", "sort", "p_code", "odds", "condition", "odds_only", "sole", "source", "type", "team"] total_sizes_value = [match_id, league_id, key, '0', 0, 'TN', value, condition, odds_only, sole, 'hg3535', '0', ''] total_sizes_data = dict(zip(total_sizes_key, total_sizes_value)) data_list.append(total_sizes_data) # 球队得分大小 if team_scores_dict: for key, value in team_scores_dict.items(): sole_str = 'TB' + str(key) + str(match_id) + 'hg3535' sole = Helper.genearte_MD5(sole_str, pt) odds_str = 'TB' + str(key) + str(match_id) + 'hg3535' + str( team_scores_dict_rule[key]) + str(value) odds_only = Helper.genearte_MD5(odds_str, pt) condition = str(team_scores_dict_rule[key]) team_scores_key = ["match_id", "lg_id", "odds_code", "status", "sort", "p_code", "odds", "condition", "odds_only", "sole", "source", "type", "team"] team_scores_value = [match_id, league_id, key, '0', 0, 'TB', value, condition, odds_only, sole, 'hg3535', '0', ''] team_scores_data = dict(zip(team_scores_key, team_scores_value)) data_list.append(team_scores_data) # 球队得分最后一位数 last_home_dict = {'0或5': 'lnh0', '1或6': 'lnh1', '2或7': 'lnh2', '3或8': 'lnh3', '4或9': 'lnh4'} last_home = last_numbers_dict['last_home'] if last_home: for key, value in last_home.items(): sole_str = 'LN' + last_home_dict[key] + str(match_id) + 'hg3535' sole = Helper.genearte_MD5(sole_str, pt) odds_str = 'LN' + last_home_dict[key] + str(match_id) + 'hg3535' + '0' + str(value) odds_only = Helper.genearte_MD5(odds_str, pt) last_home_key = ["match_id", "lg_id", "odds_code", "status", "sort", "p_code", "odds", "condition", "odds_only", "sole", "source", "type", "team"] last_home_value = [match_id, league_id, last_home_dict[key], '0', 0, 'LN', value, key, odds_only, sole, 'hg3535', '0', ''] last_home_data = dict(zip(last_home_key, last_home_value)) data_list.append(last_home_data) last_guest = last_numbers_dict['last_guest'] last_guest_dict = {'0或5': 'lng0', '1或6': 'lng1', '2或7': 'lng2', '3或8': 'lng3', '4或9': 'lng4'} if last_guest: for key, value in last_guest.items(): sole_str = 'LN' + last_guest_dict[key] + str(match_id) + 'hg3535' sole = Helper.genearte_MD5(sole_str, pt) odds_str = 'LN' + last_guest_dict[key] + str(match_id) + 'hg3535' + '0' + str(value) odds_only = Helper.genearte_MD5(odds_str, pt) last_guest_key = ["match_id", "lg_id", "odds_code", "status", "sort", "p_code", "odds", "condition", "odds_only", "sole", "source", "type", "team"] last_guest_value = [match_id, league_id, key, '0', 0, 'LN', value, key, odds_only, sole, 'hg3535', '0', ''] last_guest_data = dict(zip(last_guest_key, last_guest_value)) data_list.append(last_guest_data) # 独赢 if capots_dict: for key, value in capots_dict.items(): sole_str = 'C' + str(key) + str(match_id) + 'hg3535' sole = Helper.genearte_MD5(sole_str, pt) odds_str = 'C' + str(key) + str(match_id) + 'hg3535' + str(value) odds_only = Helper.genearte_MD5(odds_str, pt) capots_key = ["match_id", "lg_id", "odds_code", "status", "sort", "p_code", "odds", "condition", "odds_only", "sole", "source", "type", "team"] capots_value = [match_id, league_id, key, '0', 0, 'C', value, '', odds_only, sole, 'hg3535', '0', ''] capots_data = dict(zip(capots_key, capots_value)) data_list.append(capots_data) if pt == 3: ris_stringscene = 1 else: ris_stringscene = 0 odds_key = ["game_code", "title", "match_id", "lg_id", "data", "source", "odds_only", "tag", "uuid", "is_stringscene", "utime", "pt"] odds_value = ["lq", "odds", match_id, league_id, data_list, "hg3535", odds_onlys, tag_number, uuid, ris_stringscene, utime, pt] odds_dict = dict(zip(odds_key, odds_value)) if data_list: res = Helper.async_post(ODDS_URL, odds_dict) if res: if '成功' in res: logger.info('篮球详细赔率提交成功, {}'.format(res)) logger.info(odds_dict) else: logger.warning('篮球详细赔率提交成功, {}'.format(res)) logger.warning(odds_dict) else: logger.warning('篮球详细赔率接口异常, 提交成功, {}'.format(res)) logger.warning(odds_dict) else: logger.info('详细赔率列表为空') reactor.callFromThread(out.callback, item)