import datetime import pymongo import time import logging from .ball_func import fuhao from ..utils.helper import Helper from .ball_func import get_pcode, new_time, out_time from concurrent.futures import ThreadPoolExecutor from requests_futures.sessions import FuturesSession class Bangqiupipeline(object): def open_spider(self, spider): self.mongo = pymongo.MongoClient(host="192.168.2.200", username="kaiyou", password="kaiyou", port=27017) self.db = self.mongo['kaiyou'] self.session = FuturesSession(executor=ThreadPoolExecutor(max_workers=10)) def process_item(self, item, spider): # print(item) logger = logging.getLogger(__name__) url = 'http://admin.5gogo.com' # logger.warning(query.addErrback(self.handle_error, item, spider)) # 联赛id league_id = item['league_id'] # 联赛名 league_name = item['league_name'] # result = item['result'] # 比赛id match_id = item['game_id'] # 球队1 team_home = item['team_home'] # 球队2 team_guest = item['team_guest'] # 数量(97>) tag_number = item['number'] # 比赛状态 zhuangtai = item['zhuangtai'] # 日期 # data_game = item['data_game'] try: data_game = item['data_game'].split("/") month = str(data_game[1].strip()) day = str(data_game[0]) except Exception as e: # logger.warning('data_game字段分割错误') # logger.warning(e) data_game = item['data_game'].split(" ") months = str(data_game[1].strip()) month_dict = {'Jan': '01', 'Feb': '02', 'Mar': '03', 'Apr': '04', 'May': '05', 'Jun': '06', 'Jul': '07', 'Aug': '08', 'Sep': '09', 'Oct': '10', 'Nov': '11', 'Dec': '12'} month = month_dict[months] day = str(data_game[0]) # 比赛时间 time_game = str(item['time_game']) # 比赛时间，时间戳 ctime = "2019" + "-" + month + "-" + day + " " + time_game + ":00" r_ctime = "2019" + "-" + month + "-" + day # 现在时间，时间戳 utime = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime()) expire_time = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(time.time() + 60)) # 队1分数 score_home = item['score_home'] # 队2分数 score_guest = item['score_guest'] # 第几节 jijie = item['jijie'] # 球队得分 qiudui = item['qiudui'] pt = item['pt'] # 让球 concedes_dict = item['concedes_dict'] concedes_dict_rule = item['concedes_dict_rule'] # 独赢 capots_dict = item['capots_dict'] # 总得分大小 total_size_dict = item['total_size_dict'] total_size_dict_rule = item['total_size_dict_rule'] # 总得分单双 odd_evens_dict = item['odd_evens_dict'] odd_evens_dict_rule = item['odd_evens_dict_rule'] # # 球队得分 主队 # try: # team_score_home_dict = item['team_score_home_dict'] # team_score_home_rule_dict = item['team_score_home_rule_dict'] # except: # team_score_home_dict = "" # team_score_home_rule_dict = "" # # 球队得分 客队 # try: # team_score_guest_dict = item['team_score_guest_dict'] # team_score_guest_rule_dict = item['team_score_guest_rule_dict'] # except: # team_score_guest_dict = "" # team_score_guest_rule_dict = "" bangqiu = item['bangqiu'] match_date, match_time, time3 = new_time(ctime) us_time = ctime uuid = Helper.genearte_MD5(league_name + 'hg3535') """联赛表""" last_time = '2019-12-31 23:59:59' league_dict = {"game_code": "bq", "title": "league", "source": "hg3535"} league_list = [] league_key = ["name_chinese", "kind", "match_mode", "if_stop", "last_time", "lg_id", "source", "uuid"] league_value = [league_name, "1", "1", "0", last_time, league_id, "hg3535", uuid] league_data = dict(zip(league_key, league_value)) league_list.append(league_data) league_dict['data'] = league_list # print(league_list) # print(league_dict) if self.db.hg3535_bq_league.find({'lg_id': league_id}).count() < 1: res = Helper.async_post(url + '/setLeague', league_dict) print(res) self.db.hg3535_bq_league.insert(league_list) """插入赛事表""" competition = self.db.hg3535_bq_competition.find_one({"match_id": str(match_id)}) match_list = [] if competition: print('赛事已存在，修改赛事四状态') is_morningplate = competition['is_morningplate'] is_rollball = competition['is_rollball'] is_stringscene = competition['is_stringscene'] is_today = competition['is_today'] lmtime = competition['match_time'] if pt == 3: if is_stringscene != 1 or lmtime != match_time: match_dict = {"game_code": "bq", "title": "match", "source": "hg3535"} match_key = ["home_team", "guest_team", "lg_id", "status", "match_id", "match_date", "match_time", "tag", "source", "is_rollball", "is_morningplate", "is_stringscene", "us_time", "uuid", "half_match_id", "is_today"] match_value = [team_home, team_guest, league_id, 0, match_id, match_date, match_time, tag_number, "hg3535", is_rollball, is_morningplate, 1, us_time, uuid, 0, is_today] match_data = dict(zip(match_key, match_value)) match_list.append(match_data) match_dict['data'] = match_list # print(match_list) # print(match_dict) res = Helper.async_post(url + '/setMatch', match_dict) print(res) self.db.hg3535_bq_competition.update({'match_id': match_id}, {"$set": {"is_stringscene": 1, 'match_time': match_time}}, upsert=True) if pt == 2: if is_morningplate != 1 or lmtime != match_time: match_dict = {"game_code": "bq", "title": "match", "source": "hg3535"} match_key = ["home_team", "guest_team", "lg_id", "status", "match_id", "match_date", "match_time", "tag", "source", "is_rollball", "is_morningplate", "is_stringscene", "us_time", "uuid", "half_match_id", "is_today"] match_value = [team_home, team_guest, league_id, 0, match_id, match_date, match_time, tag_number, "hg3535", is_rollball, 1, is_stringscene, us_time, uuid, 0, is_today] match_data = dict(zip(match_key, match_value)) match_list.append(match_data) match_dict['data'] = match_list # print(match_list) # print(match_dict) res = Helper.async_post(url + '/setMatch', match_dict) print(res) self.db.hg3535_bq_competition.update({'match_id': match_id}, {"$set": {"is_morningplate": 1, 'match_time': match_time}}, upsert=True) if pt == 1: if is_today != 1 or lmtime != match_time: match_dict = {"game_code": "bq", "title": "match", "source": "hg3535"} match_kay = ["home_team", "guest_team", "lg_id", "status", "match_id", "match_date", "match_time", "tag", "source", "is_rollball", "is_morningplate", "is_stringscene", "us_time", "uuid", "half_match_id", "is_today"] match_value = [team_home, team_guest, league_id, 0, match_id, match_date, match_time, tag_number, "hg3535", is_rollball, is_morningplate, is_stringscene, us_time, uuid, 0, 1] match_data = dict(zip(match_kay, match_value)) match_list.append(match_data) match_dict['data'] = match_list # print(match_list) # print(match_dict) res = Helper.async_post(url + '/setMatch', match_dict) print(res) self.db.hg3535_bq_competition.update({'match_id': match_id}, {"$set": {"is_today": 1, 'match_time': match_time}}, upsert=True) else: print("赛事已存在，赛事状态时间未改变，不做修改") else: print('赛事不存在, 插入新数据') if pt == 3: is_rollball = 0 is_today = 0 is_morningplate = 0 is_stringscene = 1 elif pt == 2: is_rollball = 0 is_today = 0 is_morningplate = 1 is_stringscene = 0 else: is_today = 1 is_rollball = 0 is_morningplate = 0 is_stringscene = 0 match_dict = {"game_code": "bq", "title": "match", "source": "hg3535"} match_kay = ["home_team", "guest_team", "lg_id", "status", "match_id", "match_date", "match_time", "tag", "source", "is_rollball", "is_morningplate", "is_stringscene", "us_time", "uuid", "half_match_id", "is_today"] match_value = [team_home, team_guest, league_id, 0, match_id, match_date, match_time, tag_number, "hg3535", is_rollball, is_morningplate, is_stringscene, us_time, uuid, 0, is_today] match_data = dict(zip(match_kay, match_value)) match_list.append(match_data) match_dict['data'] = match_list res = Helper.async_post(url + '/setMatch', match_dict) print(res) self.db.hg3535_bq_competition.insert(match_list) """赔率""" try: result = self.db.hg3535_bq_odds.find_one({'match_id': str(match_id), 'pt': pt}) odds_only_list = result['odds_only'] except: result = '' odds_only_list = [] data_list = [] odds_onlys = [] # 让球 if concedes_dict: for key, value in concedes_dict.items(): sole_str = 'CO' + str(key) + str(match_id) + 'hg3535' sole = Helper.genearte_MD5(sole_str) odds_str = 'CO' + str(key) + str(match_id) + 'hg3535' + str(concedes_dict_rule[key]) + str( value) odds_only = Helper.genearte_MD5(odds_str) condition = fuhao(str(concedes_dict_rule[key])) concedes_dict_key = ["match_id", "lg_id", "odds_code", "status", "sort", "p_code", "odds", "condition", "odds_only", "sole", "source", "type", "team"] concedes_dict_value = [match_id, league_id, key, '0', 0, 'CO', value, condition, odds_only, sole, 'hg3535', '0', ''] concedes_dict_data = dict(zip(concedes_dict_key, concedes_dict_value)) odds_onlys.append(odds_only) if odds_only not in odds_only_list: data_list.append(concedes_dict_data) # 总得分大/小 if total_size_dict: for key, value in total_size_dict.items(): sole_str = 'TN' + str(key) + str(match_id) + 'hg3535' sole = Helper.genearte_MD5(sole_str) odds_str = 'TN' + str(key) + str(match_id) + 'hg3535' + str( total_size_dict_rule[key]) + str(value) odds_only = Helper.genearte_MD5(odds_str) condition = fuhao(str(total_size_dict_rule[key])) total_size_dict_key = ["match_id", "lg_id", "odds_code", "status", "sort", "p_code", "odds", "condition", "odds_only", "sole", "source", "type", "team"] total_size_dict_value = [match_id, league_id, key, '0', 0, 'TN', value, condition, odds_only, sole, 'hg3535', '0', ''] total_size_dict_data = dict(zip(total_size_dict_key, total_size_dict_value)) odds_onlys.append(odds_only) if odds_only not in odds_only_list: data_list.append(total_size_dict_data) # 球队得分大/小 主队 # if team_score_home_dict: # for key, value in team_score_home_dict.items(): # sole_str = 'team_score_home' + str(key) + str(match_id) + 'hg3535' # sole = Helper.genearte_MD5(sole_str) # odds_str = 'team_score_home' + str(key) + str(match_id) + 'hg3535' + str( # team_score_home_rule_dict[key]) + str(value) # odds_only = Helper.genearte_MD5(odds_str) # condition = fuhao(str(team_score_home_rule_dict[key])) # team_score_home_dict_key = ["match_id", "lg_id", "odds_code", "status", "sort", "p_code", "odds", # "condition", "odds_only", "sole", "source", "type", "team"] # team_score_home_dict_value = [match_id, league_id, key, '0', '0', 'team_score_home', value, condition, # odds_only, sole, 'hg3535', '0', ''] # team_score_home_dict_data = dict(zip(team_score_home_dict_key, team_score_home_dict_value)) # odds_onlys.append(odds_only) # if odds_only not in odds_only_list: # data_list.append(team_score_home_dict_data) # # 球队得分大/小 客队 # if team_score_guest_dict: # for key, value in team_score_guest_dict.items(): # sole_str = 'team_score_guest' + str(key) + str(match_id) + 'hg3535' # sole = Helper.genearte_MD5(sole_str) # odds_str = 'team_score_guest' + str(key) + str(match_id) + 'hg3535' + str( # team_score_guest_rule_dict[key]) + str(value) # odds_only = Helper.genearte_MD5(odds_str) # condition = fuhao(str(team_score_guest_rule_dict[key])) # team_score_guest_dict_key = ["match_id", "lg_id", "odds_code", "status", "sort", "p_code", "odds", # "condition", "odds_only", "sole", "source", "type", "team"] # team_score_guest_dict_value = [match_id, league_id, key, '0', '0', 'team_score_guest', value, condition, # odds_only, sole, 'hg3535', '0', ''] # team_score_guest_dict_data = dict(zip(team_score_guest_dict_key, team_score_guest_dict_value)) # odds_onlys.append(odds_only) # if odds_only not in odds_only_list: # data_list.append(team_score_guest_dict_data) # 独赢 if capots_dict: for key, value in capots_dict.items(): sole_str = 'C' + str(key) + str(match_id) + 'hg3535' sole = Helper.genearte_MD5(sole_str) odds_str = 'C' + str(key) + str(match_id) + 'hg3535' + str(value) odds_only = Helper.genearte_MD5(odds_str) capots_dict_key = ["match_id", "lg_id", "odds_code", "status", "sort", "p_code", "odds", "condition", "odds_only", "sole", "source", "type", "team"] capots_dict_value = [match_id, league_id, key, '0', 0, 'C', value, '', odds_only, sole, 'hg3535', '0', ''] capots_dict_data = dict(zip(capots_dict_key, capots_dict_value)) odds_onlys.append(odds_only) if odds_only not in odds_only_list: data_list.append(capots_dict_data) # 总得分单双 if odd_evens_dict: for key, value in odd_evens_dict.items(): sole_str = 'TS' + str(key) + str(match_id) + 'hg3535' sole = Helper.genearte_MD5(sole_str) odds_str = 'TS' + str(key) + str(match_id) + 'hg3535' + str( odd_evens_dict_rule[key]) + str(value) odds_only = Helper.genearte_MD5(odds_str) condition = fuhao(str(odd_evens_dict_rule[key])) odd_evens_dict_key = ["match_id", "lg_id", "odds_code", "status", "sort", "p_code", "odds", "condition", "odds_only", "sole", "source", "type", "team"] odd_evens_dict_value = [match_id, league_id, key, '0', 0, 'TS', value, condition, odds_only, sole, 'hg3535', '0', ''] odd_evens_dict_data = dict(zip(odd_evens_dict_key, odd_evens_dict_value)) odds_onlys.append(odds_only) if odds_only not in odds_only_list: data_list.append(odd_evens_dict_data) # 让球 前五局 # 总得分大/小 前五局 if pt == 3: ris_stringscene = 1 else: ris_stringscene = 0 odds_key = ["game_code", "title", "match_id", "lg_id", "data", "source", "odds_only", "tag", "uuid", "is_stringscene", "utime", "pt"] odds_value = ["bq", "odds", match_id, league_id, data_list, "hg3535", odds_onlys, tag_number, uuid, ris_stringscene, utime, pt] odds_dict = dict(zip(odds_key, odds_value)) # print(odds_dict) # print(123) if data_list: res = Helper.async_post(url + '/setOdds', odds_dict) print(res) if '成功' in res: if result: self.db.hg3535_bq_odds.update({'match_id': match_id, 'pt': pt}, { '$set': {"data": data_list, "odds_only": odds_onlys, "utime": utime}}, upsert=True) else: self.db.hg3535_bq_odds.insert(odds_dict)