import datetime import json import pymongo import time import logging import requests from .ball_func import fuhao from ..utils.helper import Helper from .ball_func import get_pcode, new_time, out_time from concurrent.futures import ThreadPoolExecutor from requests_futures.sessions import FuturesSession class Lanqiupipeline(object): def open_spider(self, spider): self.mongo = pymongo.MongoClient(host="192.168.2.200", username="kaiyou", password="kaiyou", port=27017) self.db = self.mongo['kaiyou'] self.session = FuturesSession(executor=ThreadPoolExecutor(max_workers=10)) def process_item(self, item, spider): # 使用twisted将mysql插入变成异步执行 lq_odds = [] url = 'http://admin.5gogo.com' logger = logging.getLogger(__name__) # # logger.warning(query.addErrback(self.handle_error, item, spider)) # 联赛id league_id = item['league_id'] # 联赛名 league_name = item['league_name'] # result = item['result'] # 比赛id match_id = item['game_id'] # 球队1 team_home = item['team_home'] # 球队2 team_guest = item['team_guest'] # 数量(97>) tag_number = item['number'] # 比赛状态 zhuangtai = item['zhuangtai'] # 日期 # data_game = item['data_game'] try: data_game = item['data_game'].split("/") month = str(data_game[1].strip()) day = str(data_game[0]) except Exception as e: logger.warning(e) data_game = item['data_game'].split(" ") months = str(data_game[1].strip()) month_dict = {'Jan': '01', 'Feb': '02', 'Mar': '03', 'Apr': '04', 'May': '05', 'Jun': '06', 'Jul': '07', 'Aug': '08', 'Sep': '09', 'Oct': '10', 'Nov': '11', 'Dec': '12'} month = month_dict[months] day = str(data_game[0]) # 比赛时间 time_game = str(item['time_game']) # 比赛时间，时间戳 us_time = "2019" + "-" + month + "-" + day + " " + time_game + ":00" r_ctime = "2019" + "-" + month + "-" + day # 现在时间，时间戳 utime = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime()) expire_time = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(time.time() + 60)) # 队1分数 score_home = item['score_home'] # 队2分数 score_guest = item['score_guest'] # 第几节 jijie = item['jijie'] # 球队得分 qiudui = item['qiudui'] pt = item['pt'] concedes_dict = item['concede'] concedes_dict_rule = item['concede_rule'] odd_evens_dict = item['odd_even'] odd_evens_dict_rule = item['odd_even_rule'] total_sizes_dict = item['total_size'] total_sizes_dict_rule = item['total_size_rule'] last_numbers_dict = item['last_number'] capots_dict = item['capot'] team_scores_dict = item['team_score'] team_scores_dict_rule = item['team_score_rule'] # 让球 lanqiu = item['lanqiu'] match_date, match_time, time3 = new_time(us_time) uuid = Helper.genearte_MD5(league_name + 'hg3535') # 让球 数据插入数据库 """联赛""" # last_time = '2019-12-31 23:59:59' # league_dict = {"game_code": "lq", "title": "league", "source": "hg3535"} # league_list = [] # league_key = ["name_chinese", "kind", "match_mode", "if_stop", "last_time", "lg_id", "source", "uuid"] # league_value = [league_name, "1", "1", "0", last_time, league_id, "hg3535", uuid] # league_data = dict(zip(league_key, league_value)) # league_list.append(league_data) # league_dict['data'] = league_list # # print(league_list) # # print(league_dict) # if self.db.hg3535_lq_league.find({'lg_id': league_id}).count() < 1: # res = Helper.async_post(url + '/setLeague', league_dict) # print(res) # self.db.hg3535_lq_league.insert(league_list) """赛事""" competition = self.db.hg3535_lq_competition.find_one({"match_id": str(match_id)}) match_list = [] if competition: print('赛事已存在，修改赛事四状态') is_morningplate = competition['is_morningplate'] is_rollball = competition['is_rollball'] is_stringscene = competition['is_stringscene'] is_today = competition['is_today'] lmtime = competition['match_time'] if pt == 3: if is_stringscene != 1 or lmtime != match_time: match_dict = {"game_code": "lq", "title": "match", "source": "hg3535"} match_key = ["home_team", "guest_team", "lg_id", "status", "match_id", "match_date", "match_time", "tag", "source", "is_rollball", "is_morningplate", "is_stringscene", "us_time", "uuid", "half_match_id", "is_today", 'pt'] match_value = [team_home, team_guest, league_id, 0, match_id, match_date, match_time, tag_number, "hg3535", is_rollball, is_morningplate, 1, us_time, uuid, 0, is_today, pt] match_data = dict(zip(match_key, match_value)) match_list.append(match_data) match_dict['data'] = match_list # print(match_list) # print(match_dict) res = Helper.async_post(url + '/setMatch', match_dict) print(res) self.db.hg3535_lq_competition.update({'match_id': str(match_id)}, {"$set": {"is_stringscene": 1, 'match_time': match_time}}, upsert=True) if pt == 2: if is_morningplate != 1 or lmtime != match_time: match_dict = {"game_code": "lq", "title": "match", "source": "hg3535"} match_key = ["home_team", "guest_team", "lg_id", "status", "match_id", "match_date", "match_time", "tag", "source", "is_rollball", "is_morningplate", "is_stringscene", "us_time", "uuid", "half_match_id", "is_today", 'pt'] match_value = [team_home, team_guest, league_id, 0, match_id, match_date, match_time, tag_number, "hg3535", is_rollball, 1, is_stringscene, us_time, uuid, 0, is_today, pt] match_data = dict(zip(match_key, match_value)) match_list.append(match_data) match_dict['data'] = match_list # print(match_list) # print(match_dict) res = Helper.async_post(url + '/setMatch', match_dict) print(res) self.db.hg3535_lq_competition.update({'match_id': str(match_id)}, {"$set": {"is_morningplate": 1, 'match_time': match_time}}, upsert=True) if pt == 1: if is_today != 1 or lmtime != match_time: match_dict = {"game_code": "lq", "title": "match", "source": "hg3535"} match_kay = ["home_team", "guest_team", "lg_id", "status", "match_id", "match_date", "match_time", "tag", "source", "is_rollball", "is_morningplate", "is_stringscene", "us_time", "uuid", "half_match_id", "is_today", 'pt'] match_value = [team_home, team_guest, league_id, 0, match_id, match_date, match_time, tag_number, "hg3535", is_rollball, is_morningplate, is_stringscene, us_time, uuid, 0, 1, pt] match_data = dict(zip(match_kay, match_value)) match_list.append(match_data) match_dict['data'] = match_list # print(match_list) # print(match_dict) res = Helper.async_post(url + '/setMatch', match_dict) print(res) self.db.hg3535_lq_competition.update({'match_id': str(match_id)}, {"$set": {"is_today": 1, 'match_time': match_time}}, upsert=True) else: print("赛事已存在，赛事状态时间未改变，不做修改") else: print('赛事不存在,插入新赛事') if pt == 3: is_rollball = 0 is_today = 0 is_morningplate = 0 is_stringscene = 1 elif pt == 2: is_rollball = 0 is_today = 0 is_morningplate = 1 is_stringscene = 0 else: is_today = 1 is_rollball = 0 is_morningplate = 0 is_stringscene = 0 match_dict = {"game_code": "lq", "title": "match", "source": "hg3535"} match_kay = ["home_team", "guest_team", "lg_id", "status", "match_id", "match_date", "match_time", "tag", "source", "is_rollball", "is_morningplate", "is_stringscene", "us_time", "uuid", "half_match_id", "is_today", 'pt'] match_value = [team_home, team_guest, league_id, 0, match_id, match_date, match_time, tag_number, "hg3535", is_rollball, is_morningplate, is_stringscene, us_time, uuid, 0, is_today, pt] match_data = dict(zip(match_kay, match_value)) match_list.append(match_data) match_dict['data'] = match_list res = Helper.async_post(url + '/setMatch', match_dict) print(res) self.db.hg3535_lq_competition.insert(match_list)