sports_scrapy/hg3535/spiders/roll_wangqiu.py

# -*- coding: utf-8 -*-
import datetime
import json

import jsonpath
import scrapy
from scrapy.http import Request

# from ..items import Lanqiu
from ..items import Roll_Wangqiu


def rangqiu_daxiao(inner):
    concede = [inner[i] for i in range(len(inner)) if i % 2 == 1]
    # 主队让球条件赔率
    concedehome = [concede[i] for i in range(len(concede)) if i % 2 == 0]
    # 客队让球条件赔率
    concedeguest = [concede[i] for i in range(len(concede)) if i % 2 == 1]
    # 主队让球条件
    concede_home_rule = [concedehome[i] for i in range(len(concedehome)) if i % 2 == 0]
    # 主队让球赔率
    concede_home = [concedehome[i] for i in range(len(concedehome)) if i % 2 == 1]
    # 客队让球条件
    concede_guest_rule = [concedeguest[i] for i in range(len(concedeguest)) if i % 2 == 0]
    # 客队让球赔率
    concede_guest = [concedeguest[i] for i in range(len(concedeguest)) if i % 2 == 1]
    return concede_home_rule, concede_home, concede_guest_rule, concede_guest

def danshaung_fun(inner):
    odd_even = [inner[i] for i in range(len(inner)) if i % 2 == 1]
    # 全场 总分单
    odd_even_odd = [odd_even[i] for i in range(len(odd_even)) if i % 2 == 0]
    # 全场 总分双
    odd_even_even = [odd_even[i] for i in range(len(odd_even)) if i % 2 == 1]
    return odd_even_odd, odd_even_even


class WangqiuSpider(scrapy.Spider):
    name = "roll_wangqiu"
    to_day = datetime.datetime.now()
    allowed_domains = ['hg3535z.com']
    start_urls = ['https://hg3535z.com/odds2/d/getodds?sid=3&pt=4&ubt=am&pn=0&sb=2&dc=null&pid=0'] # 滚球菜单 网球滚球列url
    custom_settings = {
        "ITEM_PIPELINES": {
            "hg3535.pipelines.Roll_Wangqiupipeline": 350,
        },
        'LOG_LEVEL': 'DEBUG',
        'LOG_FILE': "../hg3535/log/roll_wangqiu_{}_{}_{}.log".format(to_day.year, to_day.month, to_day.day)
    }
    # start_urls = ['http://hg3535z.com/odds2/d/getodds?sid=2&pt=3&ubt=am&pn=0&sb=2&dc=null&pid=0']
    # http: // hg3535z.com / odds2 / d / getamodds?eid = 3098030 & iip = false & ubt = am & isp = false
    # http://hg3535z.com/odds2/d/getodds?sid=2&pt=2&ubt=am&pn=0&sb=2&dc=null&pid=0

    def parse(self, response):
        datas = json.loads(response.text)
        # item = Today_all()
        ids = jsonpath.jsonpath(datas, '$..i-ot[0]..egs..es..i[16]') # ids新列表
        print(ids)
        if ids:
            ids = set(ids)
            for i in ids:
                urls = 'https://hg3535z.com/odds2/d/getamodds?eid={}&iip=true&ubt=am&isp=false'.format(i)
                print(urls)
                yield Request(url=urls, callback=self.parse_other)


    def parse_other(self,response):
        new_datas = json.loads(response.text).get('eg', "")
        try:
            pt = response.meta['pt']
        except:
            pt = 0
        item = Roll_Wangqiu()
        if new_datas:
            # 联赛id
            league_id = new_datas.get("c", "").get("k", "")
            # 联赛名
            league_name = new_datas.get("c", "").get("n", "")
            new_data = new_datas.get("es", "")
            result = new_data[0]
            # 比赛id
            game_id = str(result['k'])
            # 球队1
            team_home = result['i'][0]
            # 球队2
            team_guest = result['i'][1]
            # 数量(97>)
            number = result['i'][2]
            # 比赛状态
            zhuangtai = result['i'][3]
            # 日期
            data_game = result['i'][4]
            # 开赛时间  滚球实际只有一个进行时间
            # time_game = result['i'][5]
            try:
                time_game = result['sb']['ct']
            except:
                time_game = '01:00'
            # time_game = '00:00:00'
            # time_game = result.get('sb',"").get('ct',"")
            # 队1分数
            score_home1 = result['i'][10]
            # 队2分数
            score_guest1 = result['i'][11]
            # 第几节
            jijie = result['i'][12]
            # 球队得分
            qiudui = result['pci'].get('ctn', "")
# ----------------------------------------让球分割线---------------------------------------------------------------------------
            concedes_dict = {}
            concedes_dict_rule = {}
            try:
                # 滚球让盘
                concedes = result['o']["ah"]["v"]
                concedes_dict_rule['dishes_home'] = concedes[1]
                concedes_dict['dishes_home'] = concedes[5]
                concedes_dict_rule['dishes_guest'] = concedes[3]
                concedes_dict['dishes_guest'] = concedes[7]
            except:
                concedes_dict_rule['dishes_home'] = ""
                concedes_dict['dishes_home'] = ""
                concedes_dict_rule['dishes_guest'] = ""
                concedes_dict['dishes_guest'] = ""
            # print(concedes_dict)

# ----------------------------------------第一节让球分割线---------------------------------------------------------------------------

            #第一节让球
            try:
                # 第一节让球
                one_concedes = result['o']["ahs1"]["v"]
                one_concede_home_rule,one_concede_home,one_concede_guest_rule,one_concede_guest= rangqiu_daxiao(inner=one_concedes)
                concedes_dict_rule['one_concede_home'] = ''.join(one_concede_home_rule)
                concedes_dict['one_concede_home'] = ''.join(one_concede_home)
                concedes_dict_rule['one_concede_guest'] = ''.join(one_concede_guest_rule)
                concedes_dict['one_concede_guest'] = ''.join(one_concede_guest)
            except:
                one_concede_guest = ""
                one_concede_guest_rule = ""
                one_concede_home = ""
                one_concede_home_rule = ""
                concedes_dict['one_concede_home_rule'] = one_concede_home_rule
                concedes_dict['one_concede_home'] = one_concede_home
                concedes_dict['one_concede_guest_rule'] = one_concede_guest_rule
                concedes_dict['one_concede_guest'] = one_concede_guest

# ----------------------------------------第二节让球分割线---------------------------------------------------------------------------

            #第二节让球
            # 第二节让球
            try:
                two_concedes = result['o']["ahs2"]["v"]
                two_concede_home_rule,two_concede_home,two_concede_guest_rule,two_concede_guest = rangqiu_daxiao(inner=two_concedes)
                concedes_dict_rule['two_concede_home'] = ''.join(two_concede_home_rule)
                concedes_dict['two_concede_home'] = ''.join(two_concede_home)
                concedes_dict_rule['two_concede_guest'] = ''.join(two_concede_guest_rule)
                concedes_dict['two_concede_guest'] = ''.join(two_concede_guest)
            except:
                two_concede_guest = ""
                two_concede_guest_rule = ""
                two_concede_home = ""
                two_concede_home_rule = ""
                concedes_dict_rule['two_concede_home'] = two_concede_home_rule
                concedes_dict['two_concede_home'] = two_concede_home
                concedes_dict_rule['two_concede_guest'] = two_concede_guest_rule
                concedes_dict['two_concede_guest'] = two_concede_guest







# ---------------------------------------------------冠军  滚球是独赢----------------------------------------------------------------
            kemps_dict = {}
            try:
                kemps = result['o']["ml"]["v"]
                kemps_dict['kemp_home'] = kemps[1]
                kemps_dict['kemp_guest'] = kemps[3]

            except:
                kemps_dict['kemp_home'] = ""
                kemps_dict['kemp_guest'] = ""
            # 第一盘 冠军  第一盘独赢 滚球是独赢
            try:
                kemps = result['o']["mls1"]["v"]
                kemps_dict['one_kemp_home'] = kemps[1]
                kemps_dict['one_kemp_guest'] = kemps[3]

            except:
                kemps_dict['one_kemp_home'] = ""
                kemps_dict['one_kemp_guest'] = ""

            try: # 第二盘 独赢
                kemps = result['o']["mls2"]["v"]
                kemps_dict['two_kemp_home'] = kemps[1]
                kemps_dict['two_kemp_guest'] = kemps[3]
            except:
                kemps_dict['two_kemp_home'] = ""
                kemps_dict["two_kemp_guest"] = ""


# ---------------------------------------------------让局---------------------------------------------------------------

            bureaus_dict = {}
            bureaus_dict_rule = {}
            try:
                bureaus = result['o']["ahfts"]["v"]
                bureaus_dict_rule['concede_home'] = bureaus[1]# 添加前缀
                bureaus_dict['concede_home'] = bureaus[5]#
                bureaus_dict_rule['concede_guest'] = bureaus[3]#
                bureaus_dict['concede_guest'] = bureaus[7]##
            except:
                bureaus_dict_rule['concede_home'] = ""
                bureaus_dict['concede_home'] = ""
                bureaus_dict_rule['concede_guest'] = ""
                bureaus_dict['concede_guest'] = ""

# -----------------------------------------------------总局数 第一盘:大/小------------------------------------------------------
            total_number_dict = {}
            total_number_dict_rule = {}
            try:  #总局数 滚球大小
                total_numbers = result['o']["oufts"]["v"]
                total_number_dict_rule['total_number_big'] = total_numbers[1] #添加前缀wp
                total_number_dict['total_number_big'] = total_numbers[5]
                total_number_dict_rule['total_number_small'] = total_numbers[3]
                total_number_dict['total_number_small'] = total_numbers[7]
            except:
                total_number_dict_rule['total_number_big'] = ""
                total_number_dict['total_number_big'] = ""
                total_number_dict_rule['total_number_small'] = ""
                total_number_dict['total_number_small'] = ""

            # 第一节盘
            try:
                one_total_sizes = result['o']["ous1"]["v"]
                one_total_sizes_big_rule,one_total_sizes_big,one_total_sizes_small_rule,one_total_sizes_small = rangqiu_daxiao(inner=one_total_sizes)
                total_number_dict_rule["one_total_sizes_big"] = ''.join(one_total_sizes_big_rule)
                total_number_dict["one_total_sizes_big"] = ''.join(one_total_sizes_big)
                total_number_dict_rule["one_total_sizes_small"] = ''.join(one_total_sizes_small_rule)
                total_number_dict["one_total_sizes_small"] = ''.join(one_total_sizes_small)
            except:
                one_total_sizes_big_rule = ""
                one_total_sizes_big = ""
                one_total_sizes_small_rule = ""
                one_total_sizes_small = ""
                total_number_dict_rule["one_total_sizes_big"] = ''.join(one_total_sizes_big_rule)
                total_number_dict["one_total_sizes_big"] = ''.join(one_total_sizes_big)
                total_number_dict_rule["one_total_sizes_small"] = ''.join(one_total_sizes_small_rule)
                total_number_dict["one_total_sizes_small"] = ''.join(one_total_sizes_small)

            # 第二节盘
            try:
                two_total_sizes = result['o']["ous2"]["v"]
                two_total_sizes_big_rule,two_total_sizes_big,two_total_sizes_small_rule,two_total_sizes_small = rangqiu_daxiao(inner=two_total_sizes)
                total_number_dict_rule["two_total_sizes_big"] = two_total_sizes_big_rule
                total_number_dict["two_total_sizes_big"] = "".join(two_total_sizes_big)
                total_number_dict_rule["two_total_sizes_small"] = "".join(two_total_sizes_small_rule)
                total_number_dict["two_total_sizes_small"] = "".join(two_total_sizes_small)
            except:
                two_total_sizes_big_rule = ""
                two_total_sizes_big = ""
                two_total_sizes_small_rule = ""
                two_total_sizes_small = ""
                total_number_dict_rule["two_total_sizes_big"] = two_total_sizes_big_rule
                total_number_dict["two_total_sizes_big"] = ''.join(two_total_sizes_big)
                total_number_dict_rule["two_total_sizes_small"] = ''.join(two_total_sizes_small_rule)
                total_number_dict["two_total_sizes_small"] = ''.join(two_total_sizes_small)


            # 第三节盘
            try:
                three_total_sizes = result['o']["ous3"]["v"]
                three_total_sizes_big_rule,three_total_sizes_big,three_total_sizes_small_rule,three_total_sizes_small = rangqiu_daxiao(inner=three_total_sizes)
                total_number_dict_rule["three_total_sizes_big"] = ''.join(three_total_sizes_big_rule)
                total_number_dict["three_total_sizes_big"] = three_total_sizes_big
                total_number_dict_rule["three_total_sizes_small"] = ''.join(three_total_sizes_small_rule)
                total_number_dict["three_total_sizes_small"] = ''.join(three_total_sizes_small)
            except:
                three_total_sizes_big_rule = ""
                three_total_sizes_big = ""
                three_total_sizes_small_rule = ""
                three_total_sizes_small = ""
                total_number_dict_rule["three_total_sizes_big"] = ''.join(three_total_sizes_big_rule)
                total_number_dict["three_total_sizes_big"] = three_total_sizes_big
                total_number_dict_rule["three_total_sizes_small"] = ''.join(three_total_sizes_small_rule)
                total_number_dict["three_total_sizes_small"] = three_total_sizes_small



# ------------------------------------------------------总局数:单/双-----------------------------------------------------
            odd_evens_dict = {}
            odd_evens_dict_rule = {}
            try:
                odd_evens = result['o']["oefts"]["v"]
                # 上半场 主队让球条件
                odd_evens_dict['two_sides_single'] = odd_evens[1] # 之前是odd_even_odd
                odd_evens_dict_rule['two_sides_single'] = "单"
                odd_evens_dict['two_sides_double'] = odd_evens[3]
                odd_evens_dict_rule['two_sides_double'] = "双"

            except:
                odd_evens_dict['two_sides_single'] = ""
                odd_evens_dict['two_sides_single'] = ""
                odd_evens_dict_rule['two_sides_double'] = ""
                odd_evens_dict_rule['two_sides_double'] = ""

            # 联赛id
            item['league_id'] = league_id
            # 联赛名
            item['league_name'] = league_name
            # 比赛id
            item['game_id'] = game_id
            # 球队1
            item['team_home'] = team_home
            # 球队2
            item['team_guest'] = team_guest
            # 数量(97>)
            item['number'] = number
            # 比赛状态
            item['zhuangtai'] = zhuangtai
            # 日期
            item['data_game'] = data_game
            # 开赛时间
            item['time_game'] = time_game
            # 队1分数
            item['score_home'] = score_home1
            # 队2分数
            item['score_guest'] = score_guest1
            # 第几节
            item['jijie'] = jijie
            item['pt'] = pt
            # 球队得分
            item['qiudui'] = qiudui
            # 让盘
            item['concedes_dict'] = concedes_dict
            item['concedes_dict_rule'] = concedes_dict_rule
            # 冠军
            item['kemps_dict'] = kemps_dict
            # 让局
            item['bureaus_dict'] = bureaus_dict
            item['bureaus_dict_rule'] = bureaus_dict_rule
            # 总局数大小
            item['total_number_dict'] = total_number_dict
            item['total_number_dict_rule'] = total_number_dict_rule
            # 总局数单双
            item['odd_evens_dict'] = odd_evens_dict
            item['odd_evens_dict_rule'] = odd_evens_dict_rule
            yield item