# -*- coding: utf-8 -*- import copy import json import scrapy from scrapy.http import Request from ..items import Zuqiu import datetime class ZuqiuSpider(scrapy.Spider): name = 'zuqiu' to_day = datetime.datetime.now() # print(to_day.year, to_day) allowed_domains = ['hg3535z.com'] custom_settings = { "ITEM_PIPELINES": { 'hg3535.pipeline.zuqiu.Zuqiupipeline': 300, }, # 'LOG_LEVEL': 'DEBUG', # 'LOG_FILE': "../hg3535/log/zuqiu_{}_{}_{}.log".format(to_day.year, to_day.month, to_day.day), # 'RETRY_ENABLED': False, # 'COOKIES_ENABLED': False, # 'DOWNLOAD_TIMEOUT': 1, # 'SCHEDULER_PERSIST': False, # 'SCHEDULER_FLUSH_ON_START': False, } def start_requests(self): for y in range(1, 4): for i in range(10): y = str(y) i = str(i) # yield y url = "https://odata.jiushan6688.com/odds6i/d/getodds/zh-cn/sid/1/pt/" + y + "/ubt/am/pn/" + i + "/sb/2/dc/null/pid/0" yield scrapy.Request(url=url, callback=self.parse, dont_filter=True, meta={'pt': y}) def parse(self, response): if response.status == 200: if response.text: try: datas = json.loads(response.text).get('n-ot', "").get('egs', "") except: # datas = '' print("为获取到数据") return pt = copy.copy(response.meta['pt']) for result in datas: new_results = result['es'] for new_result in new_results: game_id = str(new_result['i'][16]) if pt == "3": url = "https://odata.jiushan6688.com/odds6i/d/getamodds/zh-cn/eid/" + str(game_id) + "/iip/false/ubt/am/isp/true" yield Request(url=url, callback=self.parse_each, dont_filter=True, meta={'pt': pt}) if pt == "2": url = "https://odata.jiushan6688.com/odds6i/d/getamodds/zh-cn/eid/" + str(game_id) + "/iip/false/ubt/am/isp/false" yield Request(url=url, callback=self.parse_each, dont_filter=True, meta={'pt': pt}) if pt == "1": url = "https://odata.jiushan6688.com/odds6i/d/getamodds/zh-cn/eid/" + str(game_id) + "/iip/false/ubt/am/isp/false" yield Request(url=url, callback=self.parse_each, dont_filter=True, meta={'pt': pt}) # def parse_each(self, response): if response.status == 200: # 球队进球数 大小 try: datas = json.loads(response.text)['eg']['es'] league_id = json.loads(response.text)['eg']['c']['k'] # 联赛名 league_name = json.loads(response.text)['eg']['c']['n'] zuqiu = json.loads(response.text)['i'][31] except: datas = "" league_id = "" # 联赛名 league_name = "" zuqiu = "" try: pt = response.meta['pt'] except: pt = 0 if datas: item = Zuqiu() full_dict = {} half_dict = {} full_dict_rule = {} half_dict_rule = {} for data in datas: try: new_data = data['pci']['ctid'] except: new_data = "" if new_data == 0: # 比赛id game_id = str(data['k']) # 球队1 team_home = data['i'][0] # 球队2 team_guest = data['i'][1] # 数量(97>) number = data['i'][2] # 下半场 half_way = data['i'][12] # 角球或者其他 try: corner_ball = data['pci']['ctn'] except: corner_ball = 0 # 日期 data_game = data['i'][4] # 开赛时间 time_game = data['i'][5] # 让球------------------------------------------------------------------------------------------------------------------ try: concedes = data['o']['ah']['v'] new_concedes = [concedes[i] for i in range(len(concedes)) if i % 2 is 1] concede_homes = [new_concedes[i] for i in range(len(new_concedes)) if i % 2 is 0] concede_home_rule = [concede_homes[i] for i in range(len(concede_homes)) if i % 2 is 0] # concede_home concede_guests = [new_concedes[i] for i in range(len(new_concedes)) if i % 2 is 1] # concede_guest_rule concede_guest_rule = [concede_guests[i] for i in range(len(concede_guests)) if i % 2 == 0] if pt is 3: concede_guest = [round(float(concede_guests[i]) - 1, 2) for i in range(len(concede_guests)) if i % 2 is 1] concede_home = [round(float(concede_homes[i]) - 1, 2) for i in range(len(concede_homes)) if i % 2 is 1] else: concede_guest = [concede_guests[i] for i in range(len(concede_guests)) if i % 2 is 1] concede_home = [concede_homes[i] for i in range(len(concede_homes)) if i % 2 is 1] except: concede_guest = "" concede_guest_rule = "" concede_home = "" concede_home_rule = "" # 上半场让球half_concede------------------------------------------------------------------------------------------------- try: half_concedes = data['o']['ah1st']['v'] new_half_concedes = [half_concedes[i] for i in range(len(half_concedes)) if i % 2 is 1] new_half_concede_homes = [new_half_concedes[i] for i in range(len(new_half_concedes)) if i % 2 == 0] # half_concede_home_rule half_concede_home_rule = [new_half_concede_homes[i] for i in range(len(new_half_concede_homes)) if i % 2 == 0] # concede_home half_concede_guests = [new_half_concedes[i] for i in range(len(new_half_concedes)) if i % 2 is 1] # concede_guest_rule half_concede_guest_rule = [half_concede_guests[i] for i in range(len(half_concede_guests)) if i % 2 == 0] if pt is 3: half_concede_home = [round(float(new_half_concede_homes[i]) - 1, 2) for i in range(len(new_half_concede_homes)) if i % 2 is 1] half_concede_guest = [round(float(half_concede_guests[i]) - 1, 2) for i in range(len(half_concede_guests)) if i % 2 is 1] else: half_concede_home = [new_half_concede_homes[i] for i in range(len(new_half_concede_homes)) if i % 2 is 1] half_concede_guest = [half_concede_guests[i] for i in range(len(half_concede_guests)) if i % 2 is 1] except: half_concede_home_rule = "" half_concede_home = "" half_concede_guest_rule = "" half_concede_guest = "" # 全场大小size 进球大小--------------------------------------------------------------------------------------------------- try: sizes = data['o']['ou']['v'] new_sizes = [sizes[i] for i in range(len(sizes)) if i % 2 is 1] size_homes = [new_sizes[i] for i in range(len(new_sizes)) if i % 2 is 0] # size_home_rule size_home_rule = [size_homes[i] for i in range(len(size_homes)) if i % 2 is 0] # size_home size_guests = [new_sizes[i] for i in range(len(new_sizes)) if i % 2 is 1] # size_guest_rule size_guest_rule = [size_guests[i] for i in range(len(size_guests)) if i % 2 is 0] # csize_guest if pt is 3: size_guest = [round(float(size_guests[i]) - 1, 2) for i in range(len(size_guests)) if i % 2 is 1] size_home = [round(float(size_homes[i]) - 1, 2) for i in range(len(size_homes)) if i % 2 is 1] else: size_guest = [size_guests[i] for i in range(len(size_guests)) if i % 2 is 1] size_home = [size_homes[i] for i in range(len(size_homes)) if i % 2 is 1] except: size_guest = "" size_guest_rule = "" size_home = "" size_home_rule = "" # 上半场大小 进球大小 half_size------------------------------------------------------------------------------------------- try: half_sizes = data['o']['ou1st']['v'] new_half_sizes = [half_sizes[i] for i in range(len(half_sizes)) if i % 2 is 1] half_size_homes = [new_half_sizes[i] for i in range(len(new_half_sizes)) if i % 2 is 0] # size_home_rule half_size_home_rule = [half_size_homes[i] for i in range(len(half_size_homes)) if i % 2 is 0] half_size_guests = [new_half_sizes[i] for i in range(len(new_half_sizes)) if i % 2 is 1] # half_size_guest_rule 客队 half_size_guest_rule = [half_size_guests[i] for i in range(len(half_size_guests)) if i % 2 is 0] # half_size_guest if pt is 3: half_size_guest = [round(float(half_size_guests[i]) - 1, 2) for i in range(len(half_size_guests)) if i % 2 is 1] half_size_home = [round(float(half_size_homes[i]) - 1, 2) for i in range(len(half_size_homes)) if i % 2 is 1] else: half_size_guest = [half_size_guests[i] for i in range(len(half_size_guests)) if i % 2 is 1] half_size_home = [half_size_homes[i] for i in range(len(half_size_homes)) if i % 2 is 1] except: half_size_guest = "" half_size_guest_rule = "" half_size_home = "" half_size_home_rule = "" # 全场总进球 total_goal-------------------------------------------------------------------------------------------------- total_goal = {} try: total_goals = data['o']['tg']['v'] new_total_goals = [total_goals[i] for i in range(len(total_goals)) if i % 2 is 1] total_goal["tg0"] = round(float(new_total_goals[0]) - 1, 2) total_goal["tg1"] = round(float(new_total_goals[1]) - 1, 2) total_goal["tg2"] = round(float(new_total_goals[2]) - 1, 2) total_goal["tg3"] = round(float(new_total_goals[3]) - 1, 2) except: total_goal["tg0"] = "" total_goal["tg1"] = "" total_goal["tg2"] = "" total_goal["tg3"] = "" # 总进球上半场 half_total_goal------------------------------------------------------------------------------------------- try: half_total_goals = data['o']['tg1st']['v'] new_half_total_goals = [half_total_goals[i] for i in range(len(half_total_goals)) if i % 2 is 1] total_goal["tg0_h"] = round(float(new_half_total_goals[0]) - 1, 2) total_goal["tg1_h"] = round(float(new_half_total_goals[1]) - 1, 2) total_goal["tg2_h"] = round(float(new_half_total_goals[2]) - 1, 2) total_goal["tg3_h"] = round(float(new_half_total_goals[3]) - 1, 2) except: total_goal["tg0_h"] = "" total_goal["tg1_h"] = "" total_goal["tg2_h"] = "" total_goal["tg3_h"] = "" # 早盘 半场/全场--------------------------------------------------------------------------------------------------------- half_full = {} new_lists = ["hfhh", "hfhd", "hfhg", "hfdh", "hfdd", "hfdg", "hfgh", "hfgd", "hfgg"] # 早盘 半场/全场 try: half_fulls = data['o']['hf']['v'] new_half_fulls = [half_fulls[i] for i in range(len(half_fulls)) if i % 2 is 1] for index, value in enumerate(new_lists): half_full[value] = round(float(new_half_fulls[index]) - 1, 2) except: for index, value in enumerate(new_lists): half_full[value] = "" # 早盘 最先/最后进球 最先进球 --------------------------------------------------------------------------------------------- first_last_ball = {} try: first_balls = data['o']['ttslast']['v'] first_ball = [first_balls[i] for i in range(len(first_balls)) if i % 2 is 1] first_last_ball['flbfh'] = round(float(first_ball[0]) - 1, 2) first_last_ball['flbfg'] = round(float(first_ball[1]) - 1, 2) except: pass # 早盘 最先/最后进球 最后进球 try: last_balls = data['o']['tts1st']['v'] last_ball = [last_balls[i] for i in range(len(last_balls)) if i % 2 is 1] first_last_ball['flblh'] = round(float(last_ball[0]) - 1, 2) first_last_ball['flblg'] = round(float(last_ball[1]) - 1, 2) first_last_ball['flbn'] = round(float(last_ball[2]) - 1, 2) except: pass # 全场独赢capot --------------------------------------------------------------------------------------------------------- try: capots = data['o']['1x2']['v'] new_capots = [capots[i] for i in range(len(capots)) if i % 2 is 1] capot_home = round(float(new_capots[0]) - 1, 2) capot_guest = round(float(new_capots[1]) - 1, 2) capot_dogfall = round(float(new_capots[2]) - 1, 2) except: capot_home = "" capot_guest = "" capot_dogfall = "" # 上半场独赢capot try: half_capots = data['o']['1x21st']['v'] new_half_capots = [half_capots[i] for i in range(len(half_capots)) if i % 2 is 1] half_capot_home = round(float(new_half_capots[0]) - 1, 2) half_capot_guest = round(float(new_half_capots[1]) - 1, 2) half_capot_dogfall = round(float(new_half_capots[2]) - 1, 2) except: half_capot_home = "" half_capot_guest = "" half_capot_dogfall = "" # 全场入球：单/双 odd_even------------------------------------------------------------------------------------------------ try: odd_evens = data['o']['oe']['v'] new_odd_evens = [odd_evens[i] for i in range(len(odd_evens)) if i % 2 is 1] if pt is 3: odd_even_odd = round(float(new_odd_evens[0]) - 1, 2) odd_even_even = round(float(new_odd_evens[1]) - 1, 2) else: odd_even_odd = round(float(new_odd_evens[0]), 2) odd_even_even = round(float(new_odd_evens[1]), 2) except: odd_even_odd = "" odd_even_even = "" # 半场入球：单/双 half_odd_even try: half_odd_evens = data['o']['oe1st']['v'] new_half_odd_evens = [half_odd_evens[i] for i in range(len(half_odd_evens)) if i % 2 is 1] if pt is 3: half_odd_even_odd = round(float(new_half_odd_evens[0]) - 1, 2) half_odd_even_even = round(float(new_half_odd_evens[1]) - 1, 2) else: half_odd_even_odd = new_half_odd_evens[0] half_odd_even_even = new_half_odd_evens[1] except: half_odd_even_odd = "" half_odd_even_even = "" bodan_data = {} # 波胆------------------------------------------------------------------------------------------------------------------ try: bodans = data['o']['cs']['v'] one_list = ["b10", "b20", "b21", "b30", "b31", "b32", "b40", "b41", "b42", "b43"] two_list = ["b01", "b02", "b12", "b03", "b13", "b23", "b04", "b14", "b24", "b34"] three_list = ["b00", "b11", "b22", "b33", "b44"] new_bodans = [bodans[i] for i in range(len(bodans)) if i % 2 is 1] new_bodan = new_bodans[0:20] # 主队bodan_home bodan_home = [new_bodan[i] for i in range(len(new_bodan)) if i % 2 is 0] for index, t in enumerate(one_list): bodan_data[t] = round(float(bodan_home[index]) - 1, 2) # 客队bodan_guest bodan_guest = [new_bodan[i] for i in range(len(new_bodan)) if i % 2 is 1] for y, z in enumerate(two_list): bodan_data[z] = round(float(bodan_guest[y]) - 1, 2) bodan_dogfall = new_bodans[-7:-2] for a, b in enumerate(three_list): bodan_data[b] = round(float(bodan_dogfall[a]) - 1, 2) # 其他bodan_other bodan_data['bo'] = round(float(new_bodans[-1]) - 1, 2) except: pass try: half_bodans = data['o']['cs1st']['v'] new_half_bodans = [half_bodans[i] for i in range(len(half_bodans)) if i % 2 is 1] new_one = ["b10_h", "b20_h", "b21_h", "b30_h", "b31_h", "b32_h"] new_two = ["b01_h", "b02_h", "b12_h", "b03_h", "b13_h", "b23_h"] new_three = ["b00_h", "b11_h", "b22_h", "b33_h"] halfbodan = new_half_bodans[0:12] half_bodan_home = [halfbodan[i] for i in range(len(halfbodan)) if i % 2 is 0] # 队1 for index, t in enumerate(new_one): bodan_data[t] = round(float(half_bodan_home[index]) - 1, 2) # 队2 half_bodan_guest = [halfbodan[i] for i in range(len(halfbodan)) if i % 2 is 1] for y, z in enumerate(new_two): bodan_data[z] = round(float(half_bodan_guest[y]) - 1, 2) # 和/平 half_bodan_dogfall = new_half_bodans[-6:-2] for a, b in enumerate(new_three): bodan_data[b] = round(float(half_bodan_dogfall[a]) - 1, 2) # 其他 bodan_data['bo_h'] = round(float(new_half_bodans[-1]) - 1, 2) except: pass if new_data is 12: try: home_data = data['o']['ou']['v'] # 球队进球 大条件 full_dict_rule['tbhb'] = home_data[1] # 球队进球小条件 full_dict_rule['tbhs'] = home_data[3] if pt is 3: # 球队进球大赔率 full_dict['tbhb'] = round(float(home_data[5]) - 1, 2) # 球队进球小赔率 full_dict['tbhs'] = round(float(home_data[7]) - 1, 2) else: # 球队进球大赔率 full_dict['tbhb'] = home_data[5] # 球队进球小赔率 full_dict['tbhs'] = home_data[7] except: full_dict['tbhs'] = "" full_dict_rule['tbhs'] = "" full_dict['tbhb'] = "" full_dict_rule['tbhb'] = "" try: half_home_data = data['o']['ou1st']['v'] half_dict_rule['tbhb_h'] = half_home_data[1] half_dict_rule['tbhs_h'] = half_home_data[3] if pt is 3: half_dict['tbhb_h'] = round(float(half_home_data[5]) - 1, 2) half_dict['tbhs_h'] = round(float(half_home_data[7]) - 1, 2) else: half_dict['tbhb_h'] = half_home_data[5] half_dict['tbhs_h'] = half_home_data[7] except: half_dict_rule['tbhb_h'] = "" half_dict['tbhb_h'] = "" half_dict_rule['tbhs_h'] = "" half_dict['tbhs_h'] = "" if new_data is 13: try: guest_data = data['o']['ou']['v'] full_dict_rule['tbgb'] = guest_data[1] full_dict_rule['tbgs'] = guest_data[3] if pt is 3: full_dict['tbgb'] = round(float(guest_data[5]) - 1, 2) full_dict['tbgs'] = round(float(guest_data[7]) - 1, 2) else: full_dict['tbgb'] = guest_data[5] full_dict['tbgs'] = guest_data[7] except: full_dict_rule['tbgb'] = "" full_dict['tbgb'] = "" full_dict_rule['tbgs'] = "" full_dict['tbgs'] = '' try: half_guest_data = data['o']['ou1st']['v'] half_dict_rule['tbgb_h'] = half_guest_data[1] half_dict_rule['tbgs_h'] = half_guest_data[3] if pt is 3: half_dict['tbgb_h'] = round(float(half_guest_data[5]) - 1, 2) half_dict['tbgs_h'] = round(float(half_guest_data[7]) - 1, 2) else: half_dict['tbgb_h'] = half_guest_data[5] half_dict['tbgs_h'] = half_guest_data[7] except: half_dict_rule['tbgb_h'] = "" half_dict['tbgb_h'] = "" half_dict_rule['tbgs_h'] = "" half_dict['tbgs_h'] = "" item['league_id'] = league_id item['league_name'] = league_name item['pt'] = pt item['game_id'] = game_id item['team_home'] = team_home item['team_guest'] = team_guest item['number'] = number item['data_game'] = data_game item['time_game'] = time_game item['corner_ball'] = corner_ball # 波胆 item['bodan_data'] = bodan_data # 早盘入球数单双 item["odd_even_odd"] = odd_even_odd item["odd_even_even"] = odd_even_even item["half_odd_even_odd"] = half_odd_even_odd item["half_odd_even_even"] = half_odd_even_even item['total_goal'] = total_goal item["half_capot_home"] = half_capot_home item["half_capot_guest"] = half_capot_guest item["half_capot_dogfall"] = half_capot_dogfall item["capot_home"] = capot_home item["capot_guest"] = capot_guest item["capot_dogfall"] = capot_dogfall item["first_last_ball"] = first_last_ball item["half_full"] = half_full item["half_size_guest"] = half_size_guest item["half_size_guest_rule"] = half_size_guest_rule item["half_size_home"] = half_size_home item["half_size_home_rule"] = half_size_home_rule # 全场大小 item["size_guest"] = size_guest item["size_guest_rule"] = size_guest_rule item["size_home"] = size_home item["size_home_rule"] = size_home_rule # 上半场让球 item["half_concede_home_rule"] = half_concede_home_rule item["half_concede_home"] = half_concede_home item["half_concede_guest_rule"] = half_concede_guest_rule item["half_concede_guest"] = half_concede_guest # 全场让球 item["concede_guest"] = concede_guest item["concede_guest_rule"] = concede_guest_rule item["concede_home"] = concede_home item["concede_home_rule"] = concede_home_rule item['full_data'] = full_dict item['half_data'] = half_dict item['full_data_rule'] = full_dict_rule item['half_data_rule'] = half_dict_rule item['zuqiu'] = zuqiu yield item