#!/usr/bin/env python # -*- coding: utf-8 -*- import datetime import time import jsonpath import redis import json import scrapy from scrapy.http import Request from ..items import Roll_Bangqiu from ..settings import R_HOST, R_PASSWORD, R_POST, R_DB class BqrangqiuSpider(scrapy.Spider): name = 'roll_bangqiu' to_day = datetime.datetime.now() allowed_domains = ['hg3535z.com'] custom_settings = { "ITEM_PIPELINES": { "hg3535.pipeline.roll_bangqiu.Roll_Banqiupipeline": 350, }, # 'LOG_LEVEL': 'DEBUG', # 'LOG_FILE': "../hg3535/log/roll_bangqiu_{}_{}_{}.log".format(to_day.year, to_day.month, to_day.day) } start_urls = ['https://odata.yonghuai5515.com/odds6i/d/getodds/zh-cn/sid/4/pt/4/ubt/am/pn/0/sb/2/dc/null/pid/0'] # 滚球菜单 棒球滚球列url rls = redis.Redis(host=R_HOST, port=R_POST, db=R_DB, password=R_PASSWORD) def parse(self, response): datas = json.loads(response.text) ids = jsonpath.jsonpath(datas, '$..i-ot[0]..egs..es..i[16]') # ids新列表 if ids: for i in ids: urls = 'https://odata.yonghuai5515.com/odds6i/d/getamodds/zh-cn/eid/{}/iip/true/ubt/am/isp/false'.format( i) self.rls.sadd('hg3535.gunqiu.ids', i) up_time = str(time.time()) + '&' + 'bq' self.rls.hset('hg3535.ball.ids', i, up_time) yield Request(url=urls, callback=self.parse_each, dont_filter=True) def parse_each(self, response): datas = json.loads(response.text) try: new_datas = datas['eg'] except: new_datas = "" try: pt = response.meta['pt'] except: pt = 0 if new_datas: # 联赛id league_id = new_datas["c"]["k"] # 联赛名 league_name = new_datas["c"]["n"] new_data = new_datas["es"] for result in new_data: if result['pci']["ctid"] == 0: item = Roll_Bangqiu() # 比赛id game_id = str(result['k']) # 球队1 team_home = result['i'][0] # 球队2 team_guest = result['i'][1] # 数量(97>) number = result['i'][2] # 比赛状态 zhuangtai = result['i'][3] # 日期 data_game = result['i'][4] # 开赛时间 time_game = result['i'][5] # 队1分数 score_home1 = result['i'][10] # 队2分数 score_guest1 = result['i'][11] # 第几盘 # jijie = result['i'][12] # 球队得分 # qiudui = result['pci'].get('ctn', "") # 是否出局 不一定存在 ot = result['sb'].get("ot", "") ps_list = result['sb']['ps'] # 倒三角形 triangle = result['sb']['s'] if triangle: rule = 'Bottom' else: rule = 'Top' # 右菱形bool值 fb = result['sb']['fb'] if not fb: baseman_1 = 'N' else: baseman_1 = 'Y' # 左菱形bool值 tb = result['sb']['tb'] if not tb: baseman_3 = 'N' else: baseman_3 = 'Y' # 上菱形bool值 sb = result['sb']['tb'] if not sb: baseman_2 = 'N' else: baseman_2 = 'Y' # 局数或者节或者分 cp = result['sb']['cp'] # 第一个队伍 a = result['sb']['a'] # 第二个队伍 h = result['sb']['h'] # --------------------------------------------------让球分割线---------------------------------------- concedes_dict = {} concedes_dict_rule = {} try: concedes = result['o']["ah"]["v"] concedes_dict_rule['coh'] = concedes[1] concedes_dict['coh'] = concedes[5] concedes_dict_rule['cog'] = concedes[3] concedes_dict['cog'] = concedes[7] except: concedes_dict_rule['coh'] = "" concedes_dict['coh'] = "" concedes_dict_rule['cog'] = "" concedes_dict['cog'] = "" # 让球 前5局------------------------------------------------------------------------------------------ try: concedes = result['o']["ahf5in"]["v"] concedes_dict_rule['coh_5'] = concedes[1] concedes_dict['coh_5'] = concedes[5] concedes_dict_rule['cog_5'] = concedes[3] concedes_dict['cog_5'] = concedes[7] except: concedes_dict_rule['coh_5'] = "" concedes_dict['coh_5'] = "" concedes_dict_rule['cog_5'] = "" concedes_dict['cog_5'] = "" # ---------------------------------------------------独赢-------------------------------------------- capots_dict = {} try: capots = result['o']["ml"]["v"] capots_dict['ch'] = capots[1] capots_dict['cg'] = capots[3] except: capots_dict['ch'] = "" capots_dict['cg'] = "" # ---------------------------------------------------总得分:大/小------------------------------------- total_size_dict = {} total_size_dict_rule = {} try: total_sizes = result['o']["ou"]["v"] total_size_dict_rule['tnb'] = total_sizes[1] total_size_dict['tnb'] = total_sizes[5] total_size_dict_rule['tns'] = total_sizes[3] total_size_dict['tns'] = total_sizes[7] except: total_size_dict_rule['tnb'] = "" total_size_dict['tnb'] = "" total_size_dict_rule['tns'] = "" total_size_dict['tns'] = "" # 总得分: 大/小 前5局---------------------------------------------------------------------------------- try: total_sizes = result['o']["ouf5in"]["v"] total_size_dict_rule['tnb_5'] = total_sizes[1] total_size_dict['tnb_5'] = total_sizes[5] total_size_dict_rule['tns_5'] = total_sizes[3] total_size_dict['tns_5'] = total_sizes[7] except: total_size_dict_rule['tnb_5'] = "" total_size_dict['tnb_5'] = "" total_size_dict_rule['tns_5'] = "" total_size_dict['tns_5'] = "" # ------------------------------------------------------总得分:单/双---------------------------------- odd_evens_dict = {} odd_evens_dict_rule = {} try: odd_evens = result['o']["oe"]["v"] # 上半场 主队让球条件 odd_evens_dict['tss'] = odd_evens[1] odd_evens_dict_rule['tss'] = "单" odd_evens_dict['tsd'] = odd_evens[3] odd_evens_dict_rule['tsd'] = "双" except: odd_evens_dict['tss'] = "" odd_evens_dict['tsd'] = "" odd_evens_dict_rule['tss'] = "" odd_evens_dict_rule['tsd'] = "" source_dict = {} source_dict['sc_1th_H'] = ps_list[2]['h'] source_dict['sc_1th_C'] = ps_list[2]['a'] source_dict['sc_2th_H'] = ps_list[3]['h'] source_dict['sc_2th_C'] = ps_list[3]['a'] source_dict['sc_3th_H'] = ps_list[4]['h'] source_dict['sc_3th_C'] = ps_list[4]['a'] source_dict['sc_4th_H'] = ps_list[5]['h'] source_dict['sc_4th_C'] = ps_list[5]['a'] source_dict['sc_5th_H'] = ps_list[6]['h'] source_dict['sc_5th_C'] = ps_list[6]['a'] source_dict['sc_6th_H'] = ps_list[7]['h'] source_dict['sc_6th_C'] = ps_list[7]['a'] source_dict['sc_7th_H'] = ps_list[8]['h'] source_dict['sc_7th_C'] = ps_list[8]['a'] source_dict['sc_8th_H'] = ps_list[9]['h'] source_dict['sc_8th_C'] = ps_list[9]['a'] source_dict['sc_9th_H'] = ps_list[10]['h'] source_dict['sc_9th_C'] = ps_list[10]['a'] source_dict['OT_H'] = ps_list[1]['h'] source_dict['OT_C'] = ps_list[1]['a'] source_dict['game_num_H'] = ps_list[0]['h'] source_dict['game_num_C'] = ps_list[0]['a'] source_dict['schedule'] = rule + cp source_dict['out_game_num'] = ot source_dict['baseman_1'] = baseman_1 source_dict['baseman_2'] = baseman_2 source_dict['baseman_3'] = baseman_3 # 联赛id item['league_id'] = league_id # 联赛名 item['league_name'] = league_name # 比赛id item['game_id'] = game_id # 球队1 item['team_home'] = team_home # 球队2 item['team_guest'] = team_guest # 数量(97>) item['number'] = number # 比赛状态 item['zhuangtai'] = zhuangtai # 日期 item['data_game'] = data_game # 开赛时间 item['time_game'] = time_game # 队1分数 item['score_home'] = score_home1 # 队2分数 item['score_guest'] = score_guest1 # 第几节 item['jijie'] = cp item['pt'] = pt # 球队得分 item['qiudui'] = "{}:{}".format(a, h) # 让球 item['concedes_dict'] = concedes_dict item['concedes_dict_rule'] = concedes_dict_rule # 独赢 item['capots_dict'] = capots_dict # 总得分：大/小 item['total_size_dict'] = total_size_dict item['total_size_dict_rule'] = total_size_dict_rule # 总得分：单/双 item['odd_evens_dict'] = odd_evens_dict item['odd_evens_dict_rule'] = odd_evens_dict_rule # item['match_score_dict'] = d item['source_dict'] = source_dict yield item