# -*- coding: utf-8 -*- import copy import datetime import json import scrapy from scrapy.http import Request from ..items import Wangqiu class LanqiuSpider(scrapy.Spider): name = "wangqiu" to_day = datetime.datetime.now() allowed_domains = ['hg3535z.com'] custom_settings = { "ITEM_PIPELINES": { "hg3535.pipeline.wangqiu.Wangqiupipeline": 300, }, 'LOG_LEVEL': 'DEBUG', 'LOG_FILE': "./log/wangqiu_{}_{}_{}.log".format(to_day.year, to_day.month, to_day.day) } # rls = redis.Redis(host=R_HOST, port=R_POST, db=R_DB, password=R_PASSWORD) def start_requests(self): for y in range(1, 4): url = 'https://odata.yonghuai5515.com/odds6i/d/getodds/zh-cn/sid/3/pt/{}/ubt/am/pn/0/sb/2/dc/null/pid/0'.format(y) yield scrapy.Request(url=url, callback=self.parse, meta={'pt': y}, dont_filter=True) def parse(self, response): if response.text: try: datas = json.loads(response.text).get('n-ot', "").get('egs', "") except: datas = "" try: pt = copy.copy(response.meta['pt']) except: pt = 0 if datas: for result in datas: new_results = result['es'] for new_result in new_results: game_id = new_result['i'][16] if pt == 1: url = "https://odata.yonghuai5515.com/odds6i/d/getamodds/zh-cn/eid/{}/iip/false/ubt/am/isp/false".format(game_id) yield Request(url=url, callback=self.parse_each, meta={'pt': pt}, dont_filter=True) if pt == 2: url = "https://odata.yonghuai5515.com/odds6i/d/getamodds/zh-cn/eid/{}/iip/false/ubt/am/isp/false".format(game_id) yield Request(url=url, callback=self.parse_each, meta={'pt': pt}, dont_filter=True) if pt == 3: url = "https://odata.yonghuai5515.com/odds6i/d/getamodds/zh-cn/eid/{}/iip/false/ubt/am/isp/true".format(game_id) yield Request(url=url, callback=self.parse_each, meta={'pt': pt}, dont_filter=True) def parse_each(self, response): try: new_datas = json.loads(response.text)['eg'] pt = response.meta['pt'] wangqiu = json.loads(response.text)['i'][31] except: new_datas = "" pt = 0 wangqiu = "" if new_datas: # 联赛id league_id = new_datas["c"]["k"] # 联赛名 league_name = new_datas["c"]["n"] new_data = new_datas["es"] for result in new_data: item = Wangqiu() # 比赛id game_id = str(result['k']) # 球队1 team_home = result['i'][0] # 球队2 team_guest = result['i'][1] # 数量(97>) number = result['i'][2] # 比赛状态 zhuangtai = result['i'][3] # 日期 data_game = result['i'][4] # 开赛时间 time_game = result['i'][5] # 队1分数 score_home1 = result['i'][10] # 队2分数 score_guest1 = result['i'][11] # 第几盘 jijie = result['i'][12] # 球队得分 qiudui = result['pci'].get('ctn', "") # --------------------------------------------------让盘分割线----------------------------------------------------------- concedes_dict = {} concedes_dict_rule = {} try: concedes = result['o']["ah"]["v"] concedes_dict_rule['ldh'] = concedes[1] # 主队 concedes_dict_rule['ldg'] = concedes[3] # 客队 if pt is 3: concedes_dict['ldh'] = float(concedes[5]) - 1 concedes_dict['ldg'] = float(concedes[7]) - 1 else: concedes_dict['ldg'] = concedes[7] concedes_dict['ldh'] = float(concedes[5]) except: concedes_dict_rule['ldh'] = "" concedes_dict['ldh'] = "" concedes_dict_rule['ldg'] = "" concedes_dict['ldg'] = "" # ---------------------------------------------------冠军---------------------------------------------------------------- kemps_dict = {} try: kemps = result['o']["ml"]["v"] kemps_dict['ch'] = kemps[1] # 主队 kemps_dict['cg'] = kemps[3] # 客队 except: kemps_dict['ch'] = "" kemps_dict['cg'] = "" # 第一盘 冠军 try: kemps = result['o']["mls1"]["v"] kemps_dict['ch_1'] = kemps[1] kemps_dict['cg_1'] = kemps[3] except: kemps_dict['ch_1'] = "" kemps_dict['cg_1'] = "" # ---------------------------------------------------让局--------------------------------------------------------------- bureaus_dict = {} bureaus_dict_rule = {} try: bureaus = result['o']["ahfts"]["v"] bureaus_dict_rule['lbh'] = bureaus[1] # 主队 bureaus_dict_rule['lbg'] = bureaus[3] # 客队 if pt is 3: bureaus_dict['lbh'] = float(bureaus[5]) - 1 bureaus_dict['lbg'] = float(bureaus[7]) - 1 else: bureaus_dict['lbh'] = bureaus[5] bureaus_dict['lbg'] = bureaus[7] except: bureaus_dict_rule['lbh'] = "" bureaus_dict['lbh'] = "" bureaus_dict_rule['lbg'] = "" bureaus_dict['lbg'] = "" # -----------------------------------------------------总局数:大/小------------------------------------------------------ # 总局数大小第一盘 未采集 total_number_dict = {} total_number_dict_rule = {} try: total_numbers = result['o']["oufts"]["v"] total_number_dict_rule['tnb'] = total_numbers[1] total_number_dict_rule['tns'] = total_numbers[3] if pt is 3: total_number_dict['tnb'] = float(total_numbers[5]) - 1 total_number_dict['tns'] = float(total_numbers[7]) - 1 else: total_number_dict['tnb'] = total_numbers[5] total_number_dict['tns'] = total_numbers[7] except: total_number_dict_rule['tnb'] = "" total_number_dict['tnb'] = "" total_number_dict_rule['tns'] = "" total_number_dict['tns'] = "" # 第一节盘 try: one_total_sizes = result['o']["ous1"]["v"] total_number_dict_rule["tnb_1"] = one_total_sizes[1] total_number_dict["tnb_1"] = one_total_sizes[5] total_number_dict_rule["tns_1"] = one_total_sizes[3] total_number_dict["tns_1"] = one_total_sizes[7] if pt is 3: total_number_dict['tnb_1'] = float(one_total_sizes[5]) - 1 total_number_dict['tns_1'] = float(one_total_sizes[7]) - 1 else: total_number_dict['tnb_1'] = one_total_sizes[5] total_number_dict['tns_1'] = one_total_sizes[7] except: total_number_dict_rule["tnb_1"] = "" total_number_dict["tnb_1"] = "" total_number_dict_rule["tns_1"] = "" total_number_dict["tns_1"] = "" # ------------------------------------------------------总局数:单/双----------------------------------------------------- odd_evens_dict = {} odd_evens_dict_rule = {} try: odd_evens = result['o']["oefts"]["v"] # 上半场 主队让球条件 odd_evens_dict['tss'] = odd_evens[1] odd_evens_dict_rule['tss'] = "单" odd_evens_dict['tsb'] = odd_evens[3] odd_evens_dict_rule['tsb'] = "双" except: odd_evens_dict['tss'] = "" odd_evens_dict['tsb'] = "" odd_evens_dict_rule['tss'] = "" odd_evens_dict_rule['tsb'] = "" # 联赛id item['league_id'] = league_id # 联赛名 item['league_name'] = league_name # 比赛id item['game_id'] = game_id # 球队1 item['team_home'] = team_home # 球队2 item['team_guest'] = team_guest # 数量(97>) item['number'] = number # 比赛状态 item['zhuangtai'] = zhuangtai # 日期 item['data_game'] = data_game # 开赛时间 item['time_game'] = time_game # 队1分数 item['score_home'] = score_home1 # 队2分数 item['score_guest'] = score_guest1 # 第几节 item['jijie'] = jijie item['pt'] = pt # 球队得分 item['qiudui'] = qiudui # 让盘 item['concedes_dict'] = concedes_dict item['concedes_dict_rule'] = concedes_dict_rule # 冠军 item['kemps_dict'] = kemps_dict # 让局 item['bureaus_dict'] = bureaus_dict item['bureaus_dict_rule'] = bureaus_dict_rule # 总局数大小 item['total_number_dict'] = total_number_dict item['total_number_dict_rule'] = total_number_dict_rule # 总局数单双 item['odd_evens_dict'] = odd_evens_dict item['odd_evens_dict_rule'] = odd_evens_dict_rule item['wangqiu'] = wangqiu yield item