# -*- coding: utf-8 -*- import copy import datetime import json import jsonpath import scrapy from scrapy.http import Request from ..items import Roll_Zuqiu class ZuqiuSpider(scrapy.Spider): name = 'roll_zuqiu' to_day = datetime.datetime.now() allowed_domains = ['hg3535z.com'] custom_settings = { "ITEM_PIPELINES":{ 'hg3535.pipelines.Roll_Zuqiupipeline': 100, }, # 'LOG_LEVEL': 'DEBUG', # 'LOG_FILE': "../hg3535/log/roll_zuqiu_{}_{}_{}.log".format(to_day.year, to_day.month,to_day.day) } start_urls = ['https://hg3535z.com/odds2/d/getodds?sid=1&pt=4&ubt=am&pn=0&sb=2&dc=null&pid=0'] # 滚球菜单 足球滚球列url def parse(self, response): datas = json.loads(response.text) # item = Today_all() ids = jsonpath.jsonpath(datas, '$..i-ot[0]..egs..es..i[16]') # ids新列表 print(ids) if ids: ids = set(ids) for i in ids: urls = 'https://hg3535z.com/odds2/d/getamodds?eid={}&iip=true&ubt=am&isp=false'.format(i) print(urls) yield Request(url=urls, callback=self.parse_each,dont_filter=True) def parse_each(self, response): # 球队进球数 大小 try: datas = json.loads(response.text)['eg']['es'] league_id = json.loads(response.text)['eg']['c']['k'] # 联赛名 league_name = json.loads(response.text)['eg']['c']['n'] except: datas = "" league_id = "" # 联赛名 league_name = "" try: pt = response.meta['pt'] except: pt = 0 if datas: item = Roll_Zuqiu() full_dict = {} half_dict = {} full_dict_rule = {} half_dict_rule = {} for data in datas: try: new_data = data['pci']['ctid'] except: new_data = "" if new_data == 0: # 比赛id game_id = str(data['k']) # 球队1 team_home = data['i'][0] # 球队2 team_guest = data['i'][1] # 数量(97>) number = data['i'][2] # 下半场 half_way = data['i'][12] # 角球或者其他 corner_ball = data['pci'].get('ctn', "") # 日期 data_game = data['i'][4] # 开赛时间 time_game = data['i'][5] #这个地方的值不准确 需要再次确认 score_home = data['i'][10] # 队一分数 score_guest = data['i'][11] # 队二分数 # half_way = new_result['i'][12] #下半场 # 让球------------------------------------------------------------------------------------------------------------------ try: concedes = data['o']['ah']['v'] new_concedes = [concedes[i] for i in range(len(concedes)) if i % 2 == 1] concede_homes = [new_concedes[i] for i in range(len(new_concedes)) if i % 2 == 0] # concede_home_rule = [concede_homes[i] for i in range(len(concede_homes)) if i % 2 == 0] concede_home_rule = [] for i in range(len(concede_homes)): if i % 2 is 0: home_rule = str(concede_homes[i]) if home_rule.startswith('+'): n_home_rule = home_rule.replace('+', '-') concede_home_rule.append(n_home_rule) elif home_rule.startswith('-'): n_home_rule = home_rule.replace('-', '+') concede_home_rule.append(n_home_rule) else: concede_home_rule.append(home_rule) # concede_homes[i] # concede_home if pt is 3: concede_home = [float(concede_homes[i]) - 1 for i in range(len(concede_homes)) if i % 2 is 1] else: concede_home = [concede_homes[i] for i in range(len(concede_homes)) if i % 2 is 1] concede_guests = [new_concedes[i] for i in range(len(new_concedes)) if i % 2 is 1] # concede_guest_rule # concede_guest concede_guest_rule = [] for i in range(len(concede_guests)): if i % 2 is 0: guest_rule = str(concede_guests[i]) if guest_rule.startswith('+'): n_guest_rule = guest_rule.replace('+', '-') concede_guest_rule.append(n_guest_rule) elif guest_rule.startswith('-'): n_guest_rule = guest_rule.replace('-', '+') concede_guest_rule.append(n_guest_rule) else: concede_guest_rule.append(guest_rule) if pt is 3: concede_guest = [float(concede_guests[i]) -1 for i in range(len(concede_guests)) if i % 2 is 1] else: concede_guest = [concede_guests[i] for i in range(len(concede_guests)) if i % 2 is 1] except: concede_guest = "" concede_guest_rule = "" concede_home = "" concede_home_rule = "" # 上半场让球half_concede------------------------------------------------------------------------------------------------- try: half_concedes = data['o']['ah1st']['v'] new_half_concedes = [half_concedes[i] for i in range(len(half_concedes)) if i % 2 is 1] new_half_concede_homes = [new_half_concedes[i] for i in range(len(new_half_concedes)) if i % 2 is 0] half_concede_home_rule = [] for i in range(len(new_half_concede_homes)): if i % 2 == 0: home_rule = str(new_half_concede_homes[i]) if home_rule.startswith('+'): n_home_rule = home_rule.replace('+', '-') half_concede_home_rule.append(n_home_rule) if home_rule.startswith('-'): n_home_rule = home_rule.replace('-', '+') half_concede_home_rule.append(n_home_rule) else: half_concede_home_rule.append(home_rule) # concede_home if pt is 3: half_concede_home = [float(new_half_concede_homes[i]) - 1 for i in range(len(new_half_concede_homes)) if i % 2 is 1] else: half_concede_home = [new_half_concede_homes[i] for i in range(len(new_half_concede_homes)) if i % 2 is 1] half_concede_guests = [new_half_concedes[i] for i in range(len(new_half_concedes)) if i % 2 is 1] half_concede_guest_rule = [] for i in range(len(half_concede_guests)): if i % 2 == 0: guest_rule = str(half_concede_guests[i]) if guest_rule.startswith('+'): n_guest_rule = guest_rule.replace('+', '-') half_concede_guest_rule.append(n_guest_rule) if guest_rule.startswith('-'): n_guest_rule = guest_rule.replace('-', '+') half_concede_guest_rule.append(n_guest_rule) else: half_concede_guest_rule.append(guest_rule) # concede_guest if pt is 3: half_concede_guest = [float(half_concede_guests[i]) -1 for i in range(len(half_concede_guests)) if i % 2 is 1] else: half_concede_guest = [half_concede_guests[i] for i in range(len(half_concede_guests)) if i % 2 is 1] except: half_concede_home_rule = "" half_concede_home = "" half_concede_guest_rule = "" half_concede_guest = "" # 全场大小size 进球大小--------------------------------------------------------------------------------------------------- try: sizes = data['o']['ou']['v'] new_sizes = [sizes[i] for i in range(len(sizes)) if i % 2 is 1] size_homes = [new_sizes[i] for i in range(len(new_sizes)) if i % 2 is 0] # size_home_rule size_home_rule = [size_homes[i] for i in range(len(size_homes)) if i % 2 is 0] # size_home if pt is 3: size_home = [float(size_homes[i]) - 1 for i in range(len(size_homes)) if i % 2 is 1] else: size_home = [size_homes[i] for i in range(len(size_homes)) if i % 2 is 1] size_guests = [new_sizes[i] for i in range(len(new_sizes)) if i % 2 is 1] # size_guest_rule size_guest_rule = [size_guests[i] for i in range(len(size_guests)) if i % 2 is 0] # csize_guest if pt is 3: size_guest = [float(size_guests[i]) - 1 for i in range(len(size_guests)) if i % 2 is 1] else: size_guest = [size_guests[i] for i in range(len(size_guests)) if i % 2 is 1] except: size_guest = "" size_guest_rule = "" size_home = "" size_home_rule = "" # 上半场大小 进球大小 half_size------------------------------------------------------------------------------------------- try: half_sizes = data['o']['ou1st']['v'] new_half_sizes = [half_sizes[i] for i in range(len(half_sizes)) if i % 2 is 1] half_size_homes = [new_half_sizes[i] for i in range(len(new_half_sizes)) if i % 2 is 0] # size_home_rule half_size_home_rule = [half_size_homes[i] for i in range(len(half_size_homes)) if i % 2 is 0] # half_size_home 主队 if pt is 3: half_size_home = [float(half_size_homes[i]) -1 for i in range(len(half_size_homes)) if i % 2 is 1] else: half_size_home = [half_size_homes[i] for i in range(len(half_size_homes)) if i % 2 is 1] half_size_guests = [new_half_sizes[i] for i in range(len(new_half_sizes)) if i % 2 is 1] # half_size_guest_rule 客队 half_size_guest_rule = [half_size_guests[i] for i in range(len(half_size_guests)) if i % 2 is 0] # half_size_guest if pt is 3: half_size_guest = [float(half_size_guests[i]) - 1 for i in range(len(half_size_guests)) if i % 2 is 1] else: half_size_guest = [half_size_guests[i] for i in range(len(half_size_guests)) if i % 2 is 1] except: half_size_guest = "" half_size_guest_rule = "" half_size_home = "" half_size_home_rule = "" # 全场总进球 total_goal-------------------------------------------------------------------------------------------------- total_goal = {} try: total_goals = data['o']['tg']['v'] new_total_goals = [total_goals[i] for i in range(len(total_goals)) if i % 2 is 1] total_goal["total_goal_zero"] = float(new_total_goals[0]) - 1 total_goal["total_goal_two"] = float(new_total_goals[1]) - 1 total_goal["total_goal_four"] = float(new_total_goals[2]) - 1 total_goal["total_goal_seven"] = float(new_total_goals[3]) - 1 except: total_goal["total_goal_zero"] = "" total_goal["total_goal_two"] = "" total_goal["total_goal_four"] = "" total_goal["total_goal_seven"] = "" # 总进球上半场 half_total_goal------------------------------------------------------------------------------------------- try: half_total_goals = data['o']['tg1st']['v'] new_half_total_goals = [half_total_goals[i] for i in range(len(half_total_goals)) if i % 2 is 1] total_goal["half_total_goal_zero"] = float(new_half_total_goals[0]) - 1 total_goal["half_total_goal_one"] = float(new_half_total_goals[1]) - 1 total_goal["half_total_goal_two"] = float(new_half_total_goals[2]) - 1 total_goal["half_total_goal_three"] = float(new_half_total_goals[3]) - 1 except: total_goal["half_total_goal_zero"] = "" total_goal["half_total_goal_one"] = "" total_goal["half_total_goal_two"] = "" total_goal["half_total_goal_three"] = "" # 早盘 半场/全场--------------------------------------------------------------------------------------------------------- half_full = {} new_lists = ["half_full_home_home", "half_full_home_dogfall", "half_full_home_guest", "half_full_dogfall_home", "half_full_dogfall_dogfall", "half_full_dogfall_guest", "half_full_guest_home", "half_full_guest_dogfall", "half_full_guest_guest"] # 早盘 半场/全场 try: half_fulls = data['o']['hf']['v'] new_half_fulls = [half_fulls[i] for i in range(len(half_fulls)) if i % 2 is 1] for index, value in enumerate(new_lists): half_full[value] = float(new_half_fulls[index]) - 1 except: for index, value in enumerate(new_lists): half_full[value] = "" # 早盘 最先/最后进球 最先进球 --------------------------------------------------------------------------------------------- first_last_ball = {} try: first_balls = data['o']['ttslast']['v'] first_ball = [first_balls[i] for i in range(len(first_balls)) if i % 2 is 1] first_last_ball['first_ball_home'] = float(first_ball[0]) - 1 first_last_ball['first_ball_guest'] = float(first_ball[1]) - 1 except: pass # 早盘 最先/最后进球 最后进球 try: last_balls = data['o']['tts1st']['v'] last_ball = [last_balls[i] for i in range(len(last_balls)) if i % 2 is 1] first_last_ball['last_ball_home'] = float(last_ball[0]) - 1 first_last_ball['last_ball_guest'] = float(last_ball[1]) - 1 first_last_ball['not_ball'] = float(last_ball[2]) - 1 except: pass # 全场独赢capot --------------------------------------------------------------------------------------------------------- try: capots = data['o']['1x2']['v'] new_capots = [capots[i] for i in range(len(capots)) if i % 2 is 1] capot_home = float(new_capots[0]) - 1 capot_guest = float(new_capots[1]) - 1 capot_dogfall = float(new_capots[2]) - 1 except: capot_home = "" capot_guest = "" capot_dogfall = "" # 上半场独赢capot try: half_capots = data['o']['1x21st']['v'] new_half_capots = [half_capots[i] for i in range(len(half_capots)) if i % 2 is 1] half_capot_home = float(new_half_capots[0]) - 1 half_capot_guest = float(new_half_capots[1]) - 1 half_capot_dogfall = float(new_half_capots[2]) - 1 except: half_capot_home = "" half_capot_guest = "" half_capot_dogfall = "" # 全场入球：单/双 odd_even------------------------------------------------------------------------------------------------ try: odd_evens = data['o']['oe']['v'] new_odd_evens = [odd_evens[i] for i in range(len(odd_evens)) if i % 2 is 1] odd_even_odd = new_odd_evens[0] odd_even_even = new_odd_evens[1] except: odd_even_odd = "" odd_even_even = "" # 半场入球：单/双 half_odd_even try: half_odd_evens = data['o']['oe1st']['v'] new_half_odd_evens = [half_odd_evens[i] for i in range(len(half_odd_evens)) if i % 2 is 1] half_odd_even_odd = new_half_odd_evens[0] half_odd_even_even = new_half_odd_evens[1] except: half_odd_even_odd = "" half_odd_even_even = "" bodan_data = {} # 波胆------------------------------------------------------------------------------------------------------------------ try: bodans = data['o']['cs']['v'] one_list = ["bodanhome_one_zero", "bodanhome_two_zero", "bodanhome_two_one", "bodanhome_three_zero", "bodanhome_three_one", "bodanhome_three_two", "bodanhome_four_zero", "bodanhome_four_one", "bodanhome_four_two", "bodanhome_four_three"] two_list = ["bodanguest_one_zero", "bodanguest_two_zero", "bodanguest_two_one", "bodanguest_three_zero", "bodanguest_three_one", "bodanguest_three_two", "bodanguest_four_zero", "bodanguest_four_one", "bodanguest_four_two", "bodanguest_four_three"] three_list = ["bodandogfall_zero_zero", "bodandogfall_one_one", "bodandogfall_two_two", "bodandogfall_three_three", "bodandogfall_four_four"] new_bodans = [bodans[i] for i in range(len(bodans)) if i % 2 is 1] new_bodan = new_bodans[0:20] # 主队bodan_home bodan_home = [new_bodan[i] for i in range(len(new_bodan)) if i % 2 is 0] for index, t in enumerate(one_list): bodan_data[t] = float(bodan_home[index]) - 1 # 客队bodan_guest bodan_guest = [new_bodan[i] for i in range(len(new_bodan)) if i % 2 is 1] for y, z in enumerate(two_list): bodan_data[z] = float(bodan_guest[y]) - 1 bodan_dogfall = new_bodans[-7:-2] for a, b in enumerate(three_list): bodan_data[b] = float(bodan_dogfall[a]) - 1 # 其他bodan_other bodan_data['bodanother'] = float(new_bodans[-1]) - 1 except: pass try: half_bodans = data['o']['cs1st']['v'] new_half_bodans = [half_bodans[i] for i in range(len(half_bodans)) if i % 2 is 1] new_one = ["halfbodanhome_one_zero", "halfbodanhome_two_zero", "halfbodanhome_two_one", "halfbodanhome_three_zero", "halfbodanhome_three_one", "halfbodanhome_three_two"] new_two = ["halfbodanguest_one_zero", "halfbodanguest_two_zero", "halfbodanguest_two_one", "halfbodanguest_three_zero", "halfbodanguest_three_one", "halfbodanguest_three_two"] new_three = ["halfbodandogfall_zero_zero", "halfbodandogfall_one_one", "halfbodandogfall_two_two", "halfbodandogfall_three_three"] halfbodan = new_half_bodans[0:12] half_bodan_home = [halfbodan[i] for i in range(len(halfbodan)) if i % 2 is 0] # 队1 for index, t in enumerate(new_one): bodan_data[t] = float(half_bodan_home[index]) - 1 # 队2 half_bodan_guest = [halfbodan[i] for i in range(len(halfbodan)) if i % 2 is 1] for y, z in enumerate(new_two): bodan_data[z] = float(half_bodan_guest[y]) - 1 # 和/平 half_bodan_dogfall = new_half_bodans[-6:-2] for a, b in enumerate(new_three): bodan_data[b] = float(half_bodan_dogfall[a]) - 1 # 其他 bodan_data['halfbodanother'] = float(new_half_bodans[-1]) - 1 except: pass if new_data == 12: try: home_data = data['o']['ou']['v'] # 球队进球 大条件 home_size_big_rule = home_data[1] full_dict_rule['home_tema_ball_big"'] = home_size_big_rule # 球队进球大赔率 home_size_big = home_data[5] full_dict['home_tema_ball_big'] = home_size_big # 球队进球小条件 home_size_small_rule = home_data[3] full_dict_rule['home_tema_ball_small'] = home_size_small_rule # 球队进球小赔率 home_size_small = home_data[7] full_dict['home_tema_ball_small'] = home_size_small except: full_dict['home_tema_ball_small'] = "" full_dict_rule['home_tema_ball_small'] = "" full_dict['home_tema_ball_big'] = "" full_dict_rule['home_tema_ball_big'] = "" try: half_home_data = data['o']['ou1st']['v'] half_home_size_big_rule = half_home_data[1] half_dict_rule['half_home_tema_ball_big'] = half_home_size_big_rule half_home_size_big = half_home_data[5] half_dict['half_home_tema_ball_big'] = half_home_size_big half_home_size_small_rule = half_home_data[3] half_dict_rule['half_home_tema_ball_small'] = half_home_size_small_rule half_home_size_small = half_home_data[7] half_dict['half_home_tema_ball_small'] = half_home_size_small except: half_dict_rule['half_home_tema_ball_big'] = "" half_dict['half_home_tema_ball_big'] = "" half_dict_rule['half_home_tema_ball_small'] = "" half_dict['half_home_tema_ball_small'] = "" if new_data == 13: try: guest_data = data['o']['ou']['v'] guest_size_big_rule = guest_data[1] full_dict_rule['guest_tema_ball_big'] = guest_size_big_rule guest_size_big = guest_data[5] full_dict['guest_tema_ball_big'] = guest_size_big guest_size_small_rule = guest_data[3] full_dict_rule['guest_tema_ball_small'] = guest_size_small_rule guest_size_small = guest_data[7] full_dict['guest_tema_ball_small'] = guest_size_small except: full_dict_rule['guest_tema_ball_big'] = "" full_dict['guest_tema_ball_big'] = "" full_dict_rule['guest_tema_ball_small'] = "" full_dict['guest_tema_ball_small'] = '' try: half_guest_data = data['o']['ou1st']['v'] half_guest_size_big_rule = half_guest_data[1] half_dict_rule['half_guest_tema_ball_big'] = half_guest_size_big_rule half_guest_size_big = half_guest_data[5] half_dict['half_guest_tema_ball_big'] = half_guest_size_big half_guest_size_small_rule = half_guest_data[3] half_dict_rule['half_guest_tema_ball_small'] = half_guest_size_small_rule half_guest_size_small = half_guest_data[7] half_dict['half_guest_tema_ball_small'] = half_guest_size_small except: half_dict_rule['half_guest_tema_ball_big'] = "" half_dict['half_guest_tema_ball_big'] = "" half_dict_rule['half_guest_tema_ball_small'] = "" half_dict['half_guest_tema_ball_small'] = "" item['league_id'] = league_id item['league_name'] = league_name item['pt'] = pt item['game_id'] = game_id item['team_home'] = team_home item['team_guest'] = team_guest item['number'] = number item['data_game'] = data_game item['time_game'] = time_game item['corner_ball'] = corner_ball # 波胆 item['bodan_data'] = bodan_data # 早盘入球数单双 item["odd_even_odd"] = odd_even_odd item["odd_even_even"] = odd_even_even item["half_odd_even_odd"] = half_odd_even_odd item["half_odd_even_even"] = half_odd_even_even item['total_goal'] = total_goal item["half_capot_home"] = half_capot_home item["half_capot_guest"] = half_capot_guest item["half_capot_dogfall"] = half_capot_dogfall item["capot_home"] = capot_home item["capot_guest"] = capot_guest item["capot_dogfall"] = capot_dogfall item["first_last_ball"] = first_last_ball item["half_full"] = half_full item["half_size_guest"] = half_size_guest item["half_size_guest_rule"] = half_size_guest_rule item["half_size_home"] = half_size_home item["half_size_home_rule"] = half_size_home_rule # 全场大小 item["size_guest"] = size_guest item["size_guest_rule"] = size_guest_rule item["size_home"] = size_home item["size_home_rule"] = size_home_rule # 上半场让球 item["half_concede_home_rule"] = half_concede_home_rule item["half_concede_home"] = half_concede_home item["half_concede_guest_rule"] = half_concede_guest_rule item["half_concede_guest"] = half_concede_guest # 全场让球 item["concede_guest"] = concede_guest item["concede_guest_rule"] = concede_guest_rule item["concede_home"] = concede_home item["concede_home_rule"] = concede_home_rule item['score_home'] = score_home item['score_guest'] = score_guest item['full_data'] = full_dict item['half_data'] = half_dict item['full_data_rule'] = full_dict_rule item['half_data_rule'] = half_dict_rule item['half_way'] = half_way yield item