sports_scrapy/hg3535/spiders/zhibo.py

# -*- coding: utf-8 -*-
# import copy
import datetime
import json

# import jsonpath
# import time

import redis
import scrapy
from scrapy.http import Request
from ..settings import R_HOST, R_PASSWORD, R_POST, R_DB
from ..items import FTzhibo


class ZuqiuSpider(scrapy.Spider):
    name = 'zhibo'
    to_day = datetime.datetime.now()
    allowed_domains = ['hg3535z.com']
    custom_settings = {
        "ITEM_PIPELINES":{
            'hg3535.pipeline.zhibo.Zuqiupipeline': 200,
        },
        # 'LOG_LEVEL': 'DEBUG',
        # 'LOG_FILE': "./log/FTzhibo_{}_{}_{}.log".format(to_day.year, to_day.month,to_day.day)
    }
    start_urls = ['https://odata.yonghuai5515.com/odds6i/d/getodds/zh-cn/sid/1/pt/4/ubt/am/pn/0/sb/2/dc/null/pid/0']
    rls = redis.Redis(host=R_HOST, port=R_POST, db=R_DB, password=R_PASSWORD)

    def parse(self, response):
        responses = json.loads(response.text)
        try:
            datas = responses["i-ot"]
        except:
            print("暂无滚球篮球数据")
            return
        if datas:
            for data in datas:
                egs = data.get('egs')
                if egs:
                    for es in egs:
                        es = es['es']
                        for e in es:
                            match_id = e['k']
                            url = 'https://www.hg3535.cn/odds5/mid2rid/{}'.format(match_id)
                            yield Request(url=url, callback=self.parse_one, dont_filter=True, meta={'match_id': match_id})
                else:
                    print('足球滚球数据为空')
        else:
            print("暂无滚球足球数据")
            return

    def parse_one(self, response):
        # 球队进球数 大小
        r_data = json.loads(response.text)
        matchid = response.meta['match_id']
        if 'df' in r_data:
            live_id = r_data['id']
            live_url = 'https://erzhang1.com/188bet/zh/Etc:UTC/gismo/match_timelinedelta/{}'.format(live_id)
            yield Request(url=live_url, callback=self.parse_two, dont_filter=True, meta={'matchid': matchid})
        else:
            return

    def parse_two(self, response):
        mid = response.meta['matchid']
        response = json.loads(response.text)
        match_datas = response['doc']
        for match_data in match_datas:
            try:
                match = match_data['data']['match']
            except Exception as e:
                # print(e)
                return
            match_dict = match['teams']
            date = match['_dt']['uts']
            home = match_dict['home']['name']
            away = match_dict['away']['name']
            match_events = match_data['data']['events']
            if match_events:
                list_ball = []
                for match_event in match_events:
                    event_type = match_event['type']
                    if event_type == 'card':
                        print('有卡牌, 事件')
                        event_team = match_event['team']
                        team_name = match_dict[event_team]['name']
                        # 事件分钟
                        # event_time = match_event['time']
                        event_name = match_event['name']
                        event_uts = match_event['uts']
                        # updated_uts = match_event['updated_uts']
                        # if match_event['card'] == 'red':
                        #     warn_type = 2
                        # else:
                        #     warn_type = ''
                        warn_type = 2
                        list_ball.append((team_name, event_name, event_uts, warn_type))
                    elif event_type == 'goal':
                        print('有进球, 事件')
                        event_team = match_event['team']
                        team_name = match_dict[event_team]['name']
                        # 事件分钟
                        # event_time = match_event['time']
                        event_name = match_event['name']
                        event_uts = match_event['uts']
                        # updated_uts = match_event['updated_uts']
                        warn_type = 1
                        list_ball.append((team_name, event_name, event_uts, warn_type))
                    elif event_type == 'corner':
                        print('有角球, 事件')
                        event_team = match_event['team']
                        team_name = match_dict[event_team]['name']
                        # 事件分钟
                        # event_time = match_event['time']
                        event_name = match_event['name']
                        event_uts = match_event['uts']
                        warn_type = 3
                        # updated_uts = match_event['updated_uts']
                        list_ball.append((team_name, event_name, event_uts, warn_type))
                if list_ball:
                    item = FTzhibo()
                    item['detail'] = list_ball
                    item['mid'] = mid
                    item['date'] = date
                    item['home'] = home
                    item['away'] = away
                    yield item

            else:
                print('当前时间, 无事件')
                return