# -*- coding: utf-8 -*- # import copy import datetime import json # import jsonpath import time import redis import scrapy from scrapy.http import Request from ..settings import R_HOST, R_PASSWORD, R_POST, R_DB from ..items import Roll_Zuqiu class ZuqiuSpider(scrapy.Spider): name = 'roll_zuqiu' to_day = datetime.datetime.now() allowed_domains = ['hg3535z.com'] custom_settings = { "ITEM_PIPELINES":{ 'hg3535.pipeline.roll_zuqiu.Roll_Zuqiupipeline': 200, }, # 'LOG_LEVEL': 'DEBUG', # 'LOG_FILE': "../hg3535/log/roll_zuqiu_{}_{}_{}.log".format(to_day.year, to_day.month,to_day.day) } start_urls = ['https://odata.yonghuai5515.com/odds6i/d/getodds/zh-cn/sid/1/pt/4/ubt/am/pn/0/sb/2/dc/null/pid/0'] rls = redis.Redis(host=R_HOST, port=R_POST, db=R_DB, password=R_PASSWORD) def parse(self, response): responses = json.loads(response.text) try: datas = responses["i-ot"] except: print("暂无滚球篮球数据") return if datas: for data in datas: egs = data.get('egs') if egs: for es in egs: es = es['es'] for e in es: match_id = e['k'] up_time = str(time.time()) + '&' + 'zq' self.rls.sadd('hg3535.gunqiu.ids', match_id) self.rls.hset('hg3535.ball.ids', match_id, up_time) url = 'https://odata.yonghuai5515.com/odds6i/d/getamodds/zh-cn/eid/{}/iip/true/ubt/am/isp/false'.format(match_id) yield Request(url=url, callback=self.parse_each, dont_filter=True) else: print('足球滚球数据为空') else: print("暂无滚球足球数据") return def parse_each(self, response): # 球队进球数 大小 try: datas = json.loads(response.text)['eg']['es'] league_id = json.loads(response.text)['eg']['c']['k'] # 联赛名 league_name = json.loads(response.text)['eg']['c']['n'] except: datas = "" league_id = "" # 联赛名 league_name = "" if datas: item = Roll_Zuqiu() full_dict = {} half_dict = {} full_dict_rule = {} half_dict_rule = {} horn_ou_dict = {} horn_ou_dict_rule = {} horn_oe_dict = {} horn_oe_dict_rule = {} horn_team = {} for data in datas: try: ctid = data['pci']['ctid'] except: ctid = "" if ctid == 0: # 比赛id game_id = str(data['k']) # 球队1 team_home = data['i'][0] # 球队2 team_guest = data['i'][1] # 数量(97>) number = data['i'][2] # 下半场 half_way = data['i'][12] # 角球或者其他 # corner_ball = data['pci'].get('ctn', "") # 日期 data_game = data['i'][4] # 开赛时间 time_game = data['i'][5] score_home = data['i'][10] # 队一分数 score_guest = data['i'][11] # 队二分数 # 让球------------------------------------------------------------------------------------------------------------------ try: concedes = data['o']['ah']['v'] new_concedes = [concedes[i] for i in range(len(concedes)) if i % 2 == 1] concede_homes = [new_concedes[i] for i in range(len(new_concedes)) if i % 2 == 0] concede_home_rule = [concede_homes[i] for i in range(len(concede_homes)) if i % 2 == 0] concede_home = [concede_homes[i] for i in range(len(concede_homes)) if i % 2 is 1] concede_guests = [new_concedes[i] for i in range(len(new_concedes)) if i % 2 is 1] concede_guest_rule = [concede_guests[i] for i in range(len(concede_guests)) if i % 2 == 0] concede_guest = [concede_guests[i] for i in range(len(concede_guests)) if i % 2 is 1] except: concede_guest = "" concede_guest_rule = "" concede_home = "" concede_home_rule = "" # 上半场让球half_concede------------------------------------------------------------------------------------------------- try: half_concedes = data['o']['ah1st']['v'] new_half_concedes = [half_concedes[i] for i in range(len(half_concedes)) if i % 2 is 1] new_half_concede_homes = [new_half_concedes[i] for i in range(len(new_half_concedes)) if i % 2 is 0] half_concede_guest_rule = [new_half_concede_homes[i] for i in range(len(new_half_concede_homes)) if i % 2 == 0] # concede_home half_concede_home = [new_half_concede_homes[i] for i in range(len(new_half_concede_homes)) if i % 2 is 1] half_concede_guests = [new_half_concedes[i] for i in range(len(new_half_concedes)) if i % 2 is 1] half_concede_home_rule = [half_concede_guests[i] for i in range(len(half_concede_guests)) if i % 2 == 0] # concede_guest half_concede_guest = [half_concede_guests[i] for i in range(len(half_concede_guests)) if i % 2 is 1] except: half_concede_home_rule = "" half_concede_home = "" half_concede_guest_rule = "" half_concede_guest = "" # 全场大小size 进球大小--------------------------------------------------------------------------------------------------- try: sizes = data['o']['ou']['v'] new_sizes = [sizes[i] for i in range(len(sizes)) if i % 2 is 1] size_homes = [new_sizes[i] for i in range(len(new_sizes)) if i % 2 is 0] # size_home_rule size_home_rule = [size_homes[i] for i in range(len(size_homes)) if i % 2 is 0] # size_home size_home = [size_homes[i] for i in range(len(size_homes)) if i % 2 is 1] size_guests = [new_sizes[i] for i in range(len(new_sizes)) if i % 2 is 1] # size_guest_rule size_guest_rule = [size_guests[i] for i in range(len(size_guests)) if i % 2 is 0] # csize_guest size_guest = [size_guests[i] for i in range(len(size_guests)) if i % 2 is 1] except: size_guest = "" size_guest_rule = "" size_home = "" size_home_rule = "" # 上半场大小 进球大小 half_size------------------------------------------------------------------------------------------- try: half_sizes = data['o']['ou1st']['v'] new_half_sizes = [half_sizes[i] for i in range(len(half_sizes)) if i % 2 is 1] half_size_homes = [new_half_sizes[i] for i in range(len(new_half_sizes)) if i % 2 is 0] # size_home_rule half_size_home_rule = [half_size_homes[i] for i in range(len(half_size_homes)) if i % 2 is 0] # half_size_home 主队 half_size_home = [half_size_homes[i] for i in range(len(half_size_homes)) if i % 2 is 1] half_size_guests = [new_half_sizes[i] for i in range(len(new_half_sizes)) if i % 2 is 1] # half_size_guest_rule 客队 half_size_guest_rule = [half_size_guests[i] for i in range(len(half_size_guests)) if i % 2 is 0] # half_size_guest half_size_guest = [half_size_guests[i] for i in range(len(half_size_guests)) if i % 2 is 1] except: half_size_guest = "" half_size_guest_rule = "" half_size_home = "" half_size_home_rule = "" # 全场总进球 total_goal-------------------------------------------------------------------------------------------------- total_goal = {} try: total_goals = data['o']['tg']['v'] new_total_goals = [total_goals[i] for i in range(len(total_goals)) if i % 2 is 1] try: total_goal["tg0"] = round(float(new_total_goals[0]) - 1, 2) except: total_goal["tg0"] = 0 try: total_goal["tg1"] = round(float(new_total_goals[1]) - 1, 2) except: total_goal["tg1"] = 0 try: total_goal["tg2"] = round(float(new_total_goals[2]) - 1, 2) except: total_goal["tg2"] = 0 try: total_goal["tg3"] = round(float(new_total_goals[3]) - 1, 2) except: total_goal["tg3"] = 0 except: total_goal["tg0"] = "" total_goal["tg1"] = "" total_goal["tg2"] = "" total_goal["tg3"] = "" # 总进球上半场 half_total_goal------------------------------------------------------------------------------------------- try: half_total_goals = data['o']['tg1st']['v'] new_half_total_goals = [half_total_goals[i] for i in range(len(half_total_goals)) if i % 2 is 1] try: total_goal["tg0_h"] = round(float(new_half_total_goals[0]) - 1, 2) except: total_goal["tg0_h"] = 0 try: total_goal["tg1_h"] = round(float(new_half_total_goals[1]) - 1, 2) except: total_goal["tg1_h"] = 0 try: total_goal["tg2_h"] = round(float(new_half_total_goals[2]) - 1, 2) except: total_goal["tg2_h"] = 0 try: total_goal["tg3_h"] = round(float(new_half_total_goals[3]) - 1, 2) except: total_goal["tg3_h"] = 0 except: total_goal["tg0_h"] = "" total_goal["tg1_h"] = "" total_goal["tg2_h"] = "" total_goal["tg3_h"] = "" # 早盘 半场/全场--------------------------------------------------------------------------------------------------------- half_full = {} new_lists = ["hfhh", "hfhd", "hfhg", "hfdh", "hfdd", "hfdg", "hfgh", "hfgd", "hfgg"] # 早盘 半场/全场 try: half_fulls = data['o']['hf']['v'] new_half_fulls = [half_fulls[i] for i in range(len(half_fulls)) if i % 2 is 1] for index, value in enumerate(new_lists): try: half_full[value] = round(float(new_half_fulls[index]) - 1, 2) except: half_full[value] = "" except: for index, value in enumerate(new_lists): half_full[value] = "" # 早盘 最先/最后进球 最先进球 --------------------------------------------------------------------------------------------- first_last_ball = {} try: first_balls = data['o']['ttslast']['v'] first_ball = [first_balls[i] for i in range(len(first_balls)) if i % 2 is 1] try: first_last_ball['flbfh'] = round(float(first_ball[0]) - 1, 2) except: first_last_ball['flbfh'] = 0 try: first_last_ball['flbfg'] = round(float(first_ball[1]) - 1, 2) except: first_last_ball['flbfg'] = 0 except: pass # 早盘 最先/最后进球 最后进球 try: last_balls = data['o']['tts1st']['v'] last_ball = [last_balls[i] for i in range(len(last_balls)) if i % 2 is 1] try: first_last_ball['flblh'] = round(float(last_ball[0]) - 1, 2) except: first_last_ball['flblh'] = 0 try: first_last_ball['flblg'] = round(float(last_ball[1]) - 1, 2) except: first_last_ball['flblg'] = 0 try: first_last_ball['flbn'] = round(float(last_ball[2]) - 1, 2) except: first_last_ball['flbn'] = 0 except: pass # 全场独赢capot --------------------------------------------------------------------------------------------------------- try: capots = data['o']['1x2']['v'] new_capots = [capots[i] for i in range(len(capots)) if i % 2 is 1] try: capot_home = round(float(new_capots[0]) - 1, 2) except: capot_home = "" try: capot_guest = round(float(new_capots[1]) - 1, 2) except: capot_guest = "" try: capot_dogfall = round(float(new_capots[2]) - 1, 2) except: capot_dogfall = "" except: capot_home = "" capot_guest = "" capot_dogfall = "" # 上半场独赢capot try: half_capots = data['o']['1x21st']['v'] new_half_capots = [half_capots[i] for i in range(len(half_capots)) if i % 2 is 1] try: half_capot_home = round(float(new_half_capots[0]) - 1, 2) except: half_capot_home = "" try: half_capot_guest = round(float(new_half_capots[1]) - 1, 2) except: half_capot_guest = "" try: half_capot_dogfall = round(float(new_half_capots[2]) - 1, 2) except: half_capot_dogfall = '' except: half_capot_home = "" half_capot_guest = "" half_capot_dogfall = "" # 全场入球：单/双 odd_even------------------------------------------------------------------------------------------------ try: odd_evens = data['o']['oe']['v'] new_odd_evens = [odd_evens[i] for i in range(len(odd_evens)) if i % 2 is 1] odd_even_odd = new_odd_evens[0] odd_even_even = new_odd_evens[1] except: odd_even_odd = "" odd_even_even = "" # 半场入球：单/双 half_odd_even try: half_odd_evens = data['o']['oe1st']['v'] new_half_odd_evens = [half_odd_evens[i] for i in range(len(half_odd_evens)) if i % 2 is 1] half_odd_even_odd = new_half_odd_evens[0] half_odd_even_even = new_half_odd_evens[1] except: half_odd_even_odd = "" half_odd_even_even = "" bodan_data = {} # 波胆------------------------------------------------------------------------------------------------------------------ try: bodans = data['o']['cs']['v'] one_list = ["b10", "b20", "b21", "b30", "b31", "b32", "b40", "b41", "b42", "b43"] two_list = ["b01", "b02", "b12", "b03", "b13", "b23", "b04", "b14", "b24", "b34"] three_list = ["b00", "b11", "b22", "b33", "b44"] new_bodans = [bodans[i] for i in range(len(bodans)) if i % 2 is 1] new_bodan = new_bodans[0:20] # 主队bodan_home bodan_home = [new_bodan[i] for i in range(len(new_bodan)) if i % 2 is 0] for index, t in enumerate(one_list): try: bodan_data[t] = round(float(bodan_home[index]) - 1, 2) except: bodan_data[t] = 0 # 客队bodan_guest bodan_guest = [new_bodan[i] for i in range(len(new_bodan)) if i % 2 is 1] for y, z in enumerate(two_list): try: bodan_data[z] = round(float(bodan_guest[y]) - 1, 2) except: bodan_data[z] = 0 bodan_dogfall = new_bodans[-6:-1] for a, b in enumerate(three_list): try: bodan_data[b] = round(float(bodan_dogfall[a]) - 1, 2) except: bodan_data[b] = 0 # 其他bodan_other try: bodan_data['bo'] = round(float(new_bodans[-1]) - 1, 2) except: bodan_data['bo'] = 0 except: pass try: half_bodans = data['o']['cs1st']['v'] new_half_bodans = [half_bodans[i] for i in range(len(half_bodans)) if i % 2 is 1] new_one = ["b10_h", "b20_h", "b21_h", "b30_h", "b31_h", "b32_h"] new_two = ["b01_h", "b02_h", "b12_h", "b03_h", "b13_h", "b23_h"] new_three = ["b00_h", "b11_h", "b22_h", "b33_h"] halfbodan = new_half_bodans[0:12] half_bodan_home = [halfbodan[i] for i in range(len(halfbodan)) if i % 2 is 0] # 队1 for index, t in enumerate(new_one): try: bodan_data[t] = round(float(half_bodan_home[index]) - 1, 2) except: bodan_data[t] = 0 # 队2 half_bodan_guest = [halfbodan[i] for i in range(len(halfbodan)) if i % 2 is 1] for y, z in enumerate(new_two): try: bodan_data[z] = round(float(half_bodan_guest[y]) - 1, 2) except: bodan_data[z] = 0 # 和/平 half_bodan_dogfall = new_half_bodans[-5:-1] for a, b in enumerate(new_three): try: bodan_data[b] = round(float(half_bodan_dogfall[a]) - 1, 2) except: bodan_data[b] = 0 # 其他 try: bodan_data['bo_h'] = round(float(new_half_bodans[-1]) - 1, 2) except: bodan_data['bo_h'] = 0 except Exception as e: pass if ctid == 1: # 角球数据 horn_team['horn_home'] = data['i'][0] horn_team['horn_guest'] = data['i'][1] horn_team['horn_scoreh'] = data['i'][10] horn_team['horn_scoreg'] = data['i'][11] horn_team['horn_id'] = str(data['k']) try: horn_ou = data['o']['ou']['v'] except: horn_ou = '' if horn_ou: horn_ou_dict['cbgb'] = horn_ou[5] horn_ou_dict['cbgs'] = horn_ou[7] horn_ou_dict_rule['cbgb'] = horn_ou[1] horn_ou_dict_rule['cbgs'] = horn_ou[3] try: horn_ou1st = data['o']['ou1st']['v'] except: horn_ou1st = '' if horn_ou1st: horn_ou_dict['cbgb_h'] = horn_ou1st[5] horn_ou_dict['cbgs_h'] = horn_ou1st[7] horn_ou_dict_rule['cbgb_h'] = horn_ou1st[1] horn_ou_dict_rule['cbgs_h'] = horn_ou1st[3] try: horn_oe = data['o']['oe']['v'] except: horn_oe = '' if horn_oe: horn_oe_dict['cbtss'] = horn_oe[1] horn_oe_dict['cbtsd'] = horn_oe[3] horn_oe_dict_rule['cbtss'] = '单' horn_oe_dict_rule['cbtsd'] = '双' try: horn_oe1st = data['o']['oe1st']['v'] except: horn_oe1st = '' if horn_oe1st: horn_oe_dict['cbtss_h'] = horn_oe1st[1] horn_oe_dict['cbtsd_h'] = horn_oe1st[3] horn_oe_dict_rule['cbtss_h'] = '单' horn_oe_dict_rule['cbtsd_h'] = '双' if ctid == 12: try: home_data = data['o']['ou']['v'] # 球队进球 大条件 full_dict_rule['tbhb'] = home_data[1] # 球队进球小条件 full_dict_rule['tbhs'] = home_data[3] # 球队进球大赔率 full_dict['tbhb'] = home_data[5] # 球队进球小赔率 full_dict['tbhs'] = home_data[7] except: full_dict['tbhs'] = "" full_dict_rule['tbhs'] = "" full_dict['tbhb'] = "" full_dict_rule['tbhb'] = "" try: half_home_data = data['o']['ou1st']['v'] half_dict_rule['tbhb_h'] = half_home_data[1] half_dict_rule['tbhs_h'] = half_home_data[3] half_dict['tbhb_h'] = half_home_data[5] half_dict['tbhs_h'] = half_home_data[7] except: half_dict_rule['tbhb_h'] = "" half_dict['tbhb_h'] = "" half_dict_rule['tbhs_h'] = "" half_dict['tbhs_h'] = "" if ctid == 13: try: guest_data = data['o']['ou']['v'] full_dict_rule['tbgb'] = guest_data[1] full_dict_rule['tbgs'] = guest_data[3] full_dict['tbgb'] = guest_data[5] full_dict['tbgs'] = guest_data[7] except: full_dict_rule['tbgb'] = "" full_dict['tbgb'] = "" full_dict_rule['tbgs'] = "" full_dict['tbgs'] = '' try: half_guest_data = data['o']['ou1st']['v'] half_dict_rule['tbgb_h'] = half_guest_data[1] half_dict_rule['tbgs_h'] = half_guest_data[3] half_dict['tbgb_h'] = half_guest_data[5] half_dict['tbgs_h'] = half_guest_data[7] except: half_dict_rule['tbgb_h'] = "" half_dict['tbgb_h'] = "" half_dict_rule['tbgs_h'] = "" half_dict['tbgs_h'] = "" item['league_id'] = league_id item['league_name'] = league_name item['game_id'] = game_id item['team_home'] = team_home item['team_guest'] = team_guest item['number'] = number item['data_game'] = data_game item['time_game'] = time_game # 波胆 item['bodan_data'] = bodan_data # 早盘入球数单双 item["odd_even_odd"] = odd_even_odd item["odd_even_even"] = odd_even_even item["half_odd_even_odd"] = half_odd_even_odd item["half_odd_even_even"] = half_odd_even_even item['total_goal'] = total_goal item["half_capot_home"] = half_capot_home item["half_capot_guest"] = half_capot_guest item["half_capot_dogfall"] = half_capot_dogfall item["capot_home"] = capot_home item["capot_guest"] = capot_guest item["capot_dogfall"] = capot_dogfall item["first_last_ball"] = first_last_ball item["half_full"] = half_full item["half_size_guest"] = half_size_guest item["half_size_guest_rule"] = half_size_guest_rule item["half_size_home"] = half_size_home item["half_size_home_rule"] = half_size_home_rule # 全场大小 item["size_guest"] = size_guest item["size_guest_rule"] = size_guest_rule item["size_home"] = size_home item["size_home_rule"] = size_home_rule # 上半场让球 item["half_concede_home_rule"] = half_concede_home_rule item["half_concede_home"] = half_concede_home item["half_concede_guest_rule"] = half_concede_guest_rule item["half_concede_guest"] = half_concede_guest # 全场让球 item["concede_guest"] = concede_guest item["concede_guest_rule"] = concede_guest_rule item["concede_home"] = concede_home item["concede_home_rule"] = concede_home_rule item['score_home'] = score_home item['score_guest'] = score_guest item['full_data'] = full_dict item['half_data'] = half_dict item['full_data_rule'] = full_dict_rule item['half_data_rule'] = half_dict_rule item['half_way'] = half_way item['horn_ou_dict'] = horn_ou_dict item['horn_ou_dict_rule'] = horn_ou_dict_rule item['horn_oe_dict'] = horn_oe_dict item['horn_oe_dict_rule'] = horn_oe_dict_rule item['horn_team'] = horn_team yield item