# -*- coding: utf-8 -*- import datetime import json import redis import scrapy from scrapy.http import Request from ..settings import R_HOST, R_PASSWORD, R_POST, R_DB from ..items import FTzhibo class ZuqiuSpider(scrapy.Spider): name = 'zhibo' to_day = datetime.datetime.now() allowed_domains = ['hg3535z.com'] custom_settings = { "ITEM_PIPELINES":{ 'hg3535.pipeline.zhibo.Zuqiupipeline': 200, }, # 'LOG_LEVEL': 'DEBUG', # 'LOG_FILE': "./log/FTzhibo_{}_{}_{}.log".format(to_day.year, to_day.month,to_day.day) } start_urls = ['https://odata.yonghuai5515.com/odds6i/d/getodds/zh-cn/sid/1/pt/4/ubt/am/pn/0/sb/2/dc/null/pid/0'] rls = redis.Redis(host=R_HOST, port=R_POST, db=R_DB, password=R_PASSWORD) def parse(self, response): responses = json.loads(response.text) try: datas = responses["i-ot"] except: print("暂无滚球篮球数据") return if datas: for data in datas: egs = data.get('egs') if egs: for es in egs: es = es['es'] for e in es: match_id = e['k'] url = 'https://www.hg3535.cn/odds5/mid2rid/{}'.format(match_id) yield Request(url=url, callback=self.parse_one, dont_filter=True, meta={'match_id': match_id}) else: print('足球滚球数据为空') else: print("暂无滚球足球数据") return def parse_one(self, response): # 球队进球数 大小 r_data = json.loads(response.text) matchid = response.meta['match_id'] if 'df' in r_data: live_id = r_data['id'] live_url = 'https://erzhang1.com/188bet/zh/Etc:UTC/gismo/match_timelinedelta/{}'.format(live_id) yield Request(url=live_url, callback=self.parse_two, dont_filter=True, meta={'matchid': matchid}) else: return def parse_two(self, response): mid = response.meta['matchid'] response = json.loads(response.text) match_datas = response['doc'] for match_data in match_datas: try: match = match_data['data']['match'] except Exception as e: # print(e) return match_dict = match['teams'] date = match['_dt']['uts'] home = match_dict['home']['name'] away = match_dict['away']['name'] match_events = match_data['data']['events'] if match_events: list_ball = [] for match_event in match_events: event_type = match_event['type'] if event_type == 'card': print('有卡牌, 事件') event_team = match_event['team'] team_name = match_dict[event_team]['name'] # 事件分钟 # event_time = match_event['time'] event_name = match_event['name'] event_uts = match_event['uts'] # updated_uts = match_event['updated_uts'] # if match_event['card'] == 'red': # warn_type = 2 # else: # warn_type = '' warn_type = 2 list_ball.append((team_name, event_name, event_uts, warn_type)) elif event_type == 'goal': print('有进球, 事件') event_team = match_event['team'] team_name = match_dict[event_team]['name'] # 事件分钟 # event_time = match_event['time'] event_name = match_event['name'] event_uts = match_event['uts'] # updated_uts = match_event['updated_uts'] warn_type = 1 list_ball.append((team_name, event_name, event_uts, warn_type)) elif event_type == 'corner': print('有角球, 事件') event_team = match_event['team'] team_name = match_dict[event_team]['name'] # 事件分钟 # event_time = match_event['time'] event_name = match_event['name'] event_uts = match_event['uts'] warn_type = 3 list_ball.append((team_name, event_name, event_uts, warn_type)) if list_ball: item = FTzhibo() item['detail'] = list_ball item['mid'] = mid item['date'] = date item['home'] = home item['away'] = away yield item else: print('当前时间, 无事件') return