sports_scrapy/hg3535/spiders/roll_wangqiu.py

# -*- coding: utf-8 -*-
import datetime
import json

import jsonpath
import scrapy
from scrapy.http import Request

# from ..items import Lanqiu
from ..items import Roll_Wangqiu


def rangqiu_daxiao(inner):
    concede = [inner[i] for i in range(len(inner)) if i % 2 == 1]
    # 主队让球条件赔率
    concedehome = [concede[i] for i in range(len(concede)) if i % 2 == 0]
    # 客队让球条件赔率
    concedeguest = [concede[i] for i in range(len(concede)) if i % 2 == 1]
    # 主队让球条件
    concede_home_rule = [concedehome[i] for i in range(len(concedehome)) if i % 2 == 0]
    # 主队让球赔率
    concede_home = [concedehome[i] for i in range(len(concedehome)) if i % 2 == 1]
    # 客队让球条件
    concede_guest_rule = [concedeguest[i] for i in range(len(concedeguest)) if i % 2 == 0]
    # 客队让球赔率
    concede_guest = [concedeguest[i] for i in range(len(concedeguest)) if i % 2 == 1]
    return concede_home_rule, concede_home, concede_guest_rule, concede_guest


def danshaung_fun(inner):
    odd_even = [inner[i] for i in range(len(inner)) if i % 2 == 1]
    # 全场 总分单
    odd_even_odd = [odd_even[i] for i in range(len(odd_even)) if i % 2 == 0]
    # 全场 总分双
    odd_even_even = [odd_even[i] for i in range(len(odd_even)) if i % 2 == 1]
    return odd_even_odd, odd_even_even


class WangqiuSpider(scrapy.Spider):
    name = "roll_wangqiu"
    to_day = datetime.datetime.now()
    allowed_domains = ['hg3535z.com']
    # start_urls = ['https://hg3535z.com/odds2/d/getodds?sid=3&pt=4&ubt=am&pn=0&sb=2&dc=null&pid=0']  # 滚球菜单 网球滚球列url
    start_urls = [
        'https://odata.jiushan6688.com/odds6i/d/getodds/zh-cn/sid/3/pt/4/ubt/am/pn/0/sb/2/dc/null/pid/0']  # 滚球菜单 网球滚球列url
    custom_settings = {
        "ITEM_PIPELINES": {
            # "hg3535.pipeline.Roll_Wangqiupipeline": 350,
            "hg3535.pipeline.roll_wangqiu.Roll_Wangqiupipeline": 350,
        },
        # 'LOG_LEVEL': 'DEBUG',
        # 'LOG_FILE': "../hg3535/log/roll_wangqiu_{}_{}_{}.log".format(to_day.year, to_day.month, to_day.day)
    }

    # start_urls = ['http://hg3535z.com/odds2/d/getodds?sid=2&pt=3&ubt=am&pn=0&sb=2&dc=null&pid=0']
    # http: // hg3535z.com / odds2 / d / getamodds?eid = 3098030 & iip = false & ubt = am & isp = false
    # http://hg3535z.com/odds2/d/getodds?sid=2&pt=2&ubt=am&pn=0&sb=2&dc=null&pid=0

    def parse(self, response):
        datas = json.loads(response.text)
        # item = Today_all()
        ids = jsonpath.jsonpath(datas, '$..i-ot[0]..egs..es..i[16]')  # ids新列表
        # print(ids)
        if ids:
            ids = set(ids)
            for i in ids:
                # urls = 'https://www.hg3535z.com/odds2/d/getamodds?eid={}&iip=true&ubt=am&isp=false'.format(i)
                urls = 'https://odata.jiushan6688.com/odds6i/d/getamodds/zh-cn/eid/{}/iip/true/ubt/am/isp/false'.format(
                    i)
                # print(urls)
                yield Request(url=urls, callback=self.parse_other, dont_filter=True)

    def parse_other(self, response):
        new_datas = json.loads(response.text).get('eg', "")
        try:
            pt = response.meta['pt']
        except:
            pt = 0
        item = Roll_Wangqiu()
        if new_datas:
            # 联赛id
            league_id = new_datas.get("c", "").get("k", "")
            # 联赛名
            league_name = new_datas.get("c", "").get("n", "")
            new_data = new_datas.get("es", "")
            for result in new_data:
                # result = new_data[0]
                # 比赛id
                game_id = str(result['k'])
                # 球队1
                team_home = result['i'][0]
                # 球队2
                team_guest = result['i'][1]
                # 数量(97>)
                number = result['i'][2]
                # 比赛状态
                zhuangtai = result['i'][3]
                # 日期
                data_game = result['i'][4]
                # 开赛时间  滚球实际只有一个进行时间
                # time_game = result['i'][5]
                try:
                    time_game = result['sb']['ct']
                except:
                    time_game = '01:00'
                # time_game = '00:00:00'
                # time_game = result.get('sb',"").get('ct',"")
                # 队1分数
                # score_home1 = result['i'][10]
                # 队2分数
                # score_guest1 = result['i'][11]
                # 第几节
                # jijie = result['i'][12]
                # 球队得分
                qiudui = result['pci'].get('ctn', "")
                # ----------------------------------------让盘分割线---------------------------------------------------------------------------
                concedes_dict = {}
                concedes_dict_rule = {}
                try:
                    # 滚球让盘
                    concedes = result['o']["ah"]["v"]
                    concedes_dict_rule['ldh'] = concedes[1]  # 主队
                    concedes_dict['ldh'] = concedes[5]
                    concedes_dict_rule['ldg'] = concedes[3]  # 客队
                    concedes_dict['ldg'] = concedes[7]
                except:
                    concedes_dict_rule['ldh'] = ""
                    concedes_dict['ldh'] = ""
                    concedes_dict_rule['ldg'] = ""
                    concedes_dict['ldg'] = ""
                # print(concedes_dict)

                # ----------------------------------------第一节让球分割线----------------------------------------------------------------
                # 第一节让盘
                try:
                    # 第一节让球
                    one_concedes = result['o']["ahs1"]["v"]
                    concedes_dict_rule['one_dishes_home'] = one_concedes[1]
                    concedes_dict['one_dishes_home'] = one_concedes[5]
                    concedes_dict_rule['one_dishes_guest'] = one_concedes[3]
                    concedes_dict['one_dishes_guest'] = one_concedes[7]
                except:
                    concedes_dict_rule['one_dishes_home'] = ""
                    concedes_dict['one_dishes_home'] = ""
                    concedes_dict_rule['one_dishes_guest'] = ""
                    concedes_dict['one_dishes_guest'] = ""

                # ----------------------------------------第二节让球分割线----------------------------------------------------------------

                # 第二节让盘
                try:
                    two_concedes = result['o']["ahs2"]["v"]
                    concedes_dict_rule['two_dishes_home'] = two_concedes[1]
                    concedes_dict['two_dishes_home'] = two_concedes[5]
                    concedes_dict_rule['two_dishes_guest'] = two_concedes[3]
                    concedes_dict['two_dishes_guest'] = two_concedes[7]
                except:
                    concedes_dict_rule['two_dishes_home'] = ""
                    concedes_dict['two_dishes_home'] = ""
                    concedes_dict_rule['two_dishes_guest'] = ""
                    concedes_dict['two_dishes_guest'] = ""

                # ---------------------------------------------------冠军  滚球是独赢-----------------------------------------------------
                kemps_dict = {}
                try:
                    # 滚球独赢
                    kemps = result['o']["ml"]["v"]
                    kemps_dict['ch'] = kemps[1]  # 主队
                    kemps_dict['cg'] = kemps[3]  # 客队

                except:
                    kemps_dict['ch'] = ""
                    kemps_dict['cg'] = ""
                # 第一盘 冠军  第一盘独赢 滚球是独赢
                try:
                    kemps = result['o']["mls1"]["v"]
                    kemps_dict['ch_1'] = kemps[1]
                    kemps_dict['cg_1'] = kemps[3]

                except:
                    kemps_dict['ch_1'] = ""
                    kemps_dict['cg_1'] = ""

                try:  # 第二盘 独赢
                    kemps = result['o']["mls2"]["v"]
                    kemps_dict['ch_2'] = kemps[1]
                    kemps_dict['cg_2'] = kemps[3]
                except:
                    kemps_dict['ch_2'] = ""
                    kemps_dict["cg_2"] = ""

                # ---------------------------------------------------让局---------------------------------------------------------------

                bureaus_dict = {}
                bureaus_dict_rule = {}
                try:
                    bureaus = result['o']["ahfts"]["v"]
                    bureaus_dict_rule['lbh'] = bureaus[1]  # 主队
                    bureaus_dict['lbh'] = bureaus[5]
                    bureaus_dict_rule['lbg'] = bureaus[3]  # 客队
                    bureaus_dict['lbg'] = bureaus[7]
                except:
                    bureaus_dict_rule['lbh'] = ""
                    bureaus_dict['lbh'] = ""
                    bureaus_dict_rule['lbg'] = ""
                    bureaus_dict['lbg'] = ""

                # -----------------------------------------------------总局数 第一盘:大/小------------------------------------------------
                total_number_dict = {}
                total_number_dict_rule = {}
                try:  # 总局数 滚球大小
                    total_numbers = result['o']["oufts"]["v"]
                    total_number_dict_rule['tnb'] = total_numbers[1]
                    total_number_dict['tnb'] = total_numbers[5]
                    total_number_dict_rule['tns'] = total_numbers[3]
                    total_number_dict['tns'] = total_numbers[7]
                except:
                    total_number_dict_rule['tnb'] = ""
                    total_number_dict['tnb'] = ""
                    total_number_dict_rule['tns'] = ""
                    total_number_dict['tns'] = ""

                # 第一节盘
                try:
                    one_total_sizes = result['o']["ous1"]["v"]
                    total_number_dict_rule["tnb_1"] = one_total_sizes[1]
                    total_number_dict["tnb_1"] = one_total_sizes[5]
                    total_number_dict_rule["tns_1"] = one_total_sizes[3]
                    total_number_dict["tns_1"] = one_total_sizes[7]
                except:
                    total_number_dict_rule["tnb_1"] = ""
                    total_number_dict["tnb_1"] = ""
                    total_number_dict_rule["tns_1"] = ""
                    total_number_dict["tns_1"] = ""

                # 第二节盘
                try:
                    two_total_sizes = result['o']["ous2"]["v"]
                    total_number_dict_rule["tnb_2"] = two_total_sizes[1]
                    total_number_dict["tnb_2"] = two_total_sizes[5]
                    total_number_dict_rule["tns_2"] = two_total_sizes[3]
                    total_number_dict["tns_2"] = two_total_sizes[7]
                except:
                    total_number_dict_rule["tnb_2"] = ""
                    total_number_dict["tnb_2"] = ""
                    total_number_dict_rule["tns_2"] = ""
                    total_number_dict["tns_2"] = ""

                # 第三节盘
                try:
                    three_total_sizes = result['o']["ous3"]["v"]
                    total_number_dict_rule["tnb_3"] = three_total_sizes[1]
                    total_number_dict["tnb_3"] = three_total_sizes[5]
                    total_number_dict_rule["tns_3"] = three_total_sizes[3]
                    total_number_dict["tns_3"] = three_total_sizes[7]
                except:
                    total_number_dict_rule["tnb_3"] = ""
                    total_number_dict["tnb_3"] = ""
                    total_number_dict_rule["tns_3"] = ""
                    total_number_dict["tns_3"] = ""

                # ------------------------------------------------------总局数:单/双-----------------------------------------------------
                odd_evens_dict = {}
                odd_evens_dict_rule = {}
                try:
                    odd_evens = result['o']["oefts"]["v"]
                    # 上半场 主队让球条件
                    odd_evens_dict['tss'] = odd_evens[1]  # 之前是odd_even_odd
                    odd_evens_dict_rule['tss'] = "单"
                    odd_evens_dict['tsd'] = odd_evens[3]
                    odd_evens_dict_rule['tsd'] = "双"

                except:
                    odd_evens_dict['tss'] = ""
                    odd_evens_dict['tsd'] = ""
                    odd_evens_dict_rule['tsd'] = ""
                    odd_evens_dict_rule['tss'] = ""

                # 盘，局，点
                sq_dict = {}
                # 每局比分
                score_dict = {}
                try:
                    saiqing = []
                    # h为第一个队伍，a为第二个队伍
                    s = result['sb']['cp']  # 当前比赛是第几节
                    cps = result['sb']['ps']
                    # 队伍1 当前的大比分
                    h = result['sb']['h']
                    # 队伍2 当前的大比分
                    a = result['sb']['a']
                    for cp in cps:
                        # 当前比分
                        if cp['p'] == 'p':
                            score_dict['current_point_home_guest'] = str(cp['h']) + ':' + str(cp['a'])
                        # 每局比分
                        if cp['p'] == 's1':
                            score_dict['one_score'] = str(cp['h']) + '-' + str(cp['a'])
                        elif cp['p'] == 's2':
                            score_dict['second_score'] = str(cp['h']) + '-' + str(cp['a'])
                        elif cp['p'] == 's3':
                            score_dict['third_score'] = str(cp['h']) + '-' + str(cp['a'])
                        elif cp['p'] == 's4':
                            score_dict['four_score'] = str(cp['h']) + '-' + str(cp['a'])
                        elif cp['p'] == 's5':
                            score_dict['five_score'] = str(cp['h']) + '-' + str(cp['a'])
                        elif cp['p'] == 'ftg':
                            score_dict['total_inning'] = int(cp['h']) + int(cp['a'])
                        cp.pop('e')
                        cp.pop('adv')
                        cp['cp'] = s
                        saiqing.append(cp)
                    sq_dict['ps'] = saiqing
                    # print(sq_dict)
                except:
                    sq_dict['ps'] = None
                    # 队伍1
                    h = 0
                    # 队伍2
                    a = 0
                    s = 0
                # 联赛id
                item['league_id'] = league_id
                # 联赛名
                item['league_name'] = league_name
                # 比赛id
                item['game_id'] = game_id
                # 球队1
                item['team_home'] = team_home
                # 球队2
                item['team_guest'] = team_guest
                # 数量(97>)
                item['number'] = number
                # 比赛状态
                item['zhuangtai'] = zhuangtai
                # 日期
                item['data_game'] = data_game
                # 开赛时间
                item['time_game'] = time_game
                # 队1分数
                item['score_home'] = h
                # 队2分数
                item['score_guest'] = a
                # 当前第几节
                item['jijie'] = s
                item['pt'] = pt
                # 每局比分
                item['score_dict'] = score_dict
                # 球队得分
                item['qiudui'] = qiudui
                # 让盘
                item['concedes_dict'] = concedes_dict
                item['concedes_dict_rule'] = concedes_dict_rule
                # 冠军
                item['kemps_dict'] = kemps_dict
                # 让局
                item['bureaus_dict'] = bureaus_dict
                item['bureaus_dict_rule'] = bureaus_dict_rule
                # 总局数大小
                item['total_number_dict'] = total_number_dict
                item['total_number_dict_rule'] = total_number_dict_rule
                # 总局数单双
                item['odd_evens_dict'] = odd_evens_dict
                item['odd_evens_dict_rule'] = odd_evens_dict_rule
                item['sq_dict'] = sq_dict
                yield item