# -*- coding: utf-8 -*- import datetime import json import jsonpath import scrapy from scrapy.http import Request # from ..items import Lanqiu from ..items import Roll_Wangqiu def rangqiu_daxiao(inner): concede = [inner[i] for i in range(len(inner)) if i % 2 == 1] # 主队让球条件赔率 concedehome = [concede[i] for i in range(len(concede)) if i % 2 == 0] # 客队让球条件赔率 concedeguest = [concede[i] for i in range(len(concede)) if i % 2 == 1] # 主队让球条件 concede_home_rule = [concedehome[i] for i in range(len(concedehome)) if i % 2 == 0] # 主队让球赔率 concede_home = [concedehome[i] for i in range(len(concedehome)) if i % 2 == 1] # 客队让球条件 concede_guest_rule = [concedeguest[i] for i in range(len(concedeguest)) if i % 2 == 0] # 客队让球赔率 concede_guest = [concedeguest[i] for i in range(len(concedeguest)) if i % 2 == 1] return concede_home_rule, concede_home, concede_guest_rule, concede_guest def danshaung_fun(inner): odd_even = [inner[i] for i in range(len(inner)) if i % 2 == 1] # 全场 总分单 odd_even_odd = [odd_even[i] for i in range(len(odd_even)) if i % 2 == 0] # 全场 总分双 odd_even_even = [odd_even[i] for i in range(len(odd_even)) if i % 2 == 1] return odd_even_odd, odd_even_even class WangqiuSpider(scrapy.Spider): name = "roll_wangqiu" to_day = datetime.datetime.now() allowed_domains = ['hg3535z.com'] start_urls = ['https://hg3535z.com/odds2/d/getodds?sid=3&pt=4&ubt=am&pn=0&sb=2&dc=null&pid=0'] # 滚球菜单 网球滚球列url custom_settings = { "ITEM_PIPELINES": { "hg3535.pipelines.Roll_Wangqiupipeline": 350, }, # 'LOG_LEVEL': 'DEBUG', # 'LOG_FILE': "../hg3535/log/roll_wangqiu_{}_{}_{}.log".format(to_day.year, to_day.month, to_day.day) } # start_urls = ['http://hg3535z.com/odds2/d/getodds?sid=2&pt=3&ubt=am&pn=0&sb=2&dc=null&pid=0'] # http: // hg3535z.com / odds2 / d / getamodds?eid = 3098030 & iip = false & ubt = am & isp = false # http://hg3535z.com/odds2/d/getodds?sid=2&pt=2&ubt=am&pn=0&sb=2&dc=null&pid=0 def parse(self, response): datas = json.loads(response.text) # item = Today_all() ids = jsonpath.jsonpath(datas, '$..i-ot[0]..egs..es..i[16]') # ids新列表 print(ids) if ids: ids = set(ids) for i in ids: urls = 'https://hg3535z.com/odds2/d/getamodds?eid={}&iip=true&ubt=am&isp=false'.format(i) print(urls) yield Request(url=urls, callback=self.parse_other) def parse_other(self,response): new_datas = json.loads(response.text).get('eg', "") try: pt = response.meta['pt'] except: pt = 0 item = Roll_Wangqiu() if new_datas: # 联赛id league_id = new_datas.get("c", "").get("k", "") # 联赛名 league_name = new_datas.get("c", "").get("n", "") new_data = new_datas.get("es", "") result = new_data[0] # 比赛id game_id = str(result['k']) # 球队1 team_home = result['i'][0] # 球队2 team_guest = result['i'][1] # 数量(97>) number = result['i'][2] # 比赛状态 zhuangtai = result['i'][3] # 日期 data_game = result['i'][4] # 开赛时间 滚球实际只有一个进行时间 # time_game = result['i'][5] try: time_game = result['sb']['ct'] except: time_game = '01:00' # time_game = '00:00:00' # time_game = result.get('sb',"").get('ct',"") # 队1分数 score_home1 = result['i'][10] # 队2分数 score_guest1 = result['i'][11] # 第几节 jijie = result['i'][12] # 球队得分 qiudui = result['pci'].get('ctn', "") # ----------------------------------------让球分割线--------------------------------------------------------------------------- concedes_dict = {} concedes_dict_rule = {} try: # 滚球让盘 concedes = result['o']["ah"]["v"] concedes_dict_rule['dishes_home'] = concedes[1] concedes_dict['dishes_home'] = concedes[5] concedes_dict_rule['dishes_guest'] = concedes[3] concedes_dict['dishes_guest'] = concedes[7] except: concedes_dict_rule['dishes_home'] = "" concedes_dict['dishes_home'] = "" concedes_dict_rule['dishes_guest'] = "" concedes_dict['dishes_guest'] = "" # print(concedes_dict) # ----------------------------------------第一节让球分割线--------------------------------------------------------------------------- #第一节让球 try: # 第一节让球 one_concedes = result['o']["ahs1"]["v"] one_concede_home_rule,one_concede_home,one_concede_guest_rule,one_concede_guest= rangqiu_daxiao(inner=one_concedes) concedes_dict_rule['one_dishes_home'] = ''.join(one_concede_home_rule) concedes_dict['one_dishes_home'] = ''.join(one_concede_home) concedes_dict_rule['one_dishes_guest'] = ''.join(one_concede_guest_rule) concedes_dict['one_dishes_guest'] = ''.join(one_concede_guest) except: one_concede_guest = "" one_concede_guest_rule = "" one_concede_home = "" one_concede_home_rule = "" concedes_dict_rule['one_dishes_home'] = one_concede_home_rule concedes_dict['one_dishes_home'] = one_concede_home concedes_dict_rule['one_dishes_guest'] = one_concede_guest_rule concedes_dict['one_dishes_guest'] = one_concede_guest # ----------------------------------------第二节让球分割线--------------------------------------------------------------------------- #第二节让球 # 第二节让球 try: two_concedes = result['o']["ahs2"]["v"] two_concede_home_rule,two_concede_home,two_concede_guest_rule,two_concede_guest = rangqiu_daxiao(inner=two_concedes) concedes_dict_rule['two_dishes_home'] = ''.join(two_concede_home_rule) concedes_dict['two_dishes_home'] = ''.join(two_concede_home) concedes_dict_rule['two_dishes_guest'] = ''.join(two_concede_guest_rule) concedes_dict['two_dishes_guest'] = ''.join(two_concede_guest) except: two_concede_guest = "" two_concede_guest_rule = "" two_concede_home = "" two_concede_home_rule = "" concedes_dict_rule['two_dishes_home'] = two_concede_home_rule concedes_dict['two_dishes_home'] = two_concede_home concedes_dict_rule['two_dishes_guest'] = two_concede_guest_rule concedes_dict['two_dishes_guest'] = two_concede_guest # ---------------------------------------------------冠军 滚球是独赢---------------------------------------------------------------- kemps_dict = {} try: kemps = result['o']["ml"]["v"] kemps_dict['kemp_home'] = kemps[1] kemps_dict['kemp_guest'] = kemps[3] except: kemps_dict['kemp_home'] = "" kemps_dict['kemp_guest'] = "" # 第一盘 冠军 第一盘独赢 滚球是独赢 try: kemps = result['o']["mls1"]["v"] kemps_dict['one_kemp_home'] = kemps[1] kemps_dict['one_kemp_guest'] = kemps[3] except: kemps_dict['one_kemp_home'] = "" kemps_dict['one_kemp_guest'] = "" try: # 第二盘 独赢 kemps = result['o']["mls2"]["v"] kemps_dict['two_kemp_home'] = kemps[1] kemps_dict['two_kemp_guest'] = kemps[3] except: kemps_dict['two_kemp_home'] = "" kemps_dict["two_kemp_guest"] = "" # ---------------------------------------------------让局--------------------------------------------------------------- bureaus_dict = {} bureaus_dict_rule = {} try: bureaus = result['o']["ahfts"]["v"] bureaus_dict_rule['concede_home'] = bureaus[1]# 添加前缀 bureaus_dict['concede_home'] = bureaus[5]# bureaus_dict_rule['concede_guest'] = bureaus[3]# bureaus_dict['concede_guest'] = bureaus[7]## except: bureaus_dict_rule['concede_home'] = "" bureaus_dict['concede_home'] = "" bureaus_dict_rule['concede_guest'] = "" bureaus_dict['concede_guest'] = "" # -----------------------------------------------------总局数 第一盘:大/小------------------------------------------------------ total_number_dict = {} total_number_dict_rule = {} try: #总局数 滚球大小 total_numbers = result['o']["oufts"]["v"] total_number_dict_rule['total_number_big'] = total_numbers[1] #添加前缀wp total_number_dict['total_number_big'] = total_numbers[5] total_number_dict_rule['total_number_small'] = total_numbers[3] total_number_dict['total_number_small'] = total_numbers[7] except: total_number_dict_rule['total_number_big'] = "" total_number_dict['total_number_big'] = "" total_number_dict_rule['total_number_small'] = "" total_number_dict['total_number_small'] = "" # 第一节盘 try: one_total_sizes = result['o']["ous1"]["v"] one_total_sizes_big_rule,one_total_sizes_big,one_total_sizes_small_rule,one_total_sizes_small = rangqiu_daxiao(inner=one_total_sizes) total_number_dict_rule["one_total_sizes_big"] = ''.join(one_total_sizes_big_rule) total_number_dict["one_total_sizes_big"] = ''.join(one_total_sizes_big) total_number_dict_rule["one_total_sizes_small"] = ''.join(one_total_sizes_small_rule) total_number_dict["one_total_sizes_small"] = ''.join(one_total_sizes_small) except: one_total_sizes_big_rule = "" one_total_sizes_big = "" one_total_sizes_small_rule = "" one_total_sizes_small = "" total_number_dict_rule["one_total_sizes_big"] = ''.join(one_total_sizes_big_rule) total_number_dict["one_total_sizes_big"] = ''.join(one_total_sizes_big) total_number_dict_rule["one_total_sizes_small"] = ''.join(one_total_sizes_small_rule) total_number_dict["one_total_sizes_small"] = ''.join(one_total_sizes_small) # 第二节盘 try: two_total_sizes = result['o']["ous2"]["v"] two_total_sizes_big_rule,two_total_sizes_big,two_total_sizes_small_rule,two_total_sizes_small = rangqiu_daxiao(inner=two_total_sizes) total_number_dict_rule["two_total_sizes_big"] = two_total_sizes_big_rule total_number_dict["two_total_sizes_big"] = "".join(two_total_sizes_big) total_number_dict_rule["two_total_sizes_small"] = "".join(two_total_sizes_small_rule) total_number_dict["two_total_sizes_small"] = "".join(two_total_sizes_small) except: two_total_sizes_big_rule = "" two_total_sizes_big = "" two_total_sizes_small_rule = "" two_total_sizes_small = "" total_number_dict_rule["two_total_sizes_big"] = two_total_sizes_big_rule total_number_dict["two_total_sizes_big"] = ''.join(two_total_sizes_big) total_number_dict_rule["two_total_sizes_small"] = ''.join(two_total_sizes_small_rule) total_number_dict["two_total_sizes_small"] = ''.join(two_total_sizes_small) # 第三节盘 try: three_total_sizes = result['o']["ous3"]["v"] three_total_sizes_big_rule,three_total_sizes_big,three_total_sizes_small_rule,three_total_sizes_small = rangqiu_daxiao(inner=three_total_sizes) total_number_dict_rule["three_total_sizes_big"] = ''.join(three_total_sizes_big_rule) total_number_dict["three_total_sizes_big"] = three_total_sizes_big total_number_dict_rule["three_total_sizes_small"] = ''.join(three_total_sizes_small_rule) total_number_dict["three_total_sizes_small"] = ''.join(three_total_sizes_small) except: three_total_sizes_big_rule = "" three_total_sizes_big = "" three_total_sizes_small_rule = "" three_total_sizes_small = "" total_number_dict_rule["three_total_sizes_big"] = ''.join(three_total_sizes_big_rule) total_number_dict["three_total_sizes_big"] = three_total_sizes_big total_number_dict_rule["three_total_sizes_small"] = ''.join(three_total_sizes_small_rule) total_number_dict["three_total_sizes_small"] = three_total_sizes_small # ------------------------------------------------------总局数:单/双----------------------------------------------------- odd_evens_dict = {} odd_evens_dict_rule = {} try: odd_evens = result['o']["oefts"]["v"] # 上半场 主队让球条件 odd_evens_dict['two_sides_single'] = odd_evens[1] # 之前是odd_even_odd odd_evens_dict_rule['two_sides_single'] = "单" odd_evens_dict['two_sides_double'] = odd_evens[3] odd_evens_dict_rule['two_sides_double'] = "双" except: odd_evens_dict['two_sides_single'] = "" odd_evens_dict['two_sides_double'] = "" odd_evens_dict_rule['two_sides_double'] = "" odd_evens_dict_rule['two_sides_single'] = "" # 联赛id item['league_id'] = league_id # 联赛名 item['league_name'] = league_name # 比赛id item['game_id'] = game_id # 球队1 item['team_home'] = team_home # 球队2 item['team_guest'] = team_guest # 数量(97>) item['number'] = number # 比赛状态 item['zhuangtai'] = zhuangtai # 日期 item['data_game'] = data_game # 开赛时间 item['time_game'] = time_game # 队1分数 item['score_home'] = score_home1 # 队2分数 item['score_guest'] = score_guest1 # 第几节 item['jijie'] = jijie item['pt'] = pt # 球队得分 item['qiudui'] = qiudui # 让盘 item['concedes_dict'] = concedes_dict item['concedes_dict_rule'] = concedes_dict_rule # 冠军 item['kemps_dict'] = kemps_dict # 让局 item['bureaus_dict'] = bureaus_dict item['bureaus_dict_rule'] = bureaus_dict_rule # 总局数大小 item['total_number_dict'] = total_number_dict item['total_number_dict_rule'] = total_number_dict_rule # 总局数单双 item['odd_evens_dict'] = odd_evens_dict item['odd_evens_dict_rule'] = odd_evens_dict_rule yield item