sports_scrapy/hg3535/spiders/roll_zuqiu.py

# -*- coding: utf-8 -*-
import copy
import datetime
import json

import jsonpath
import scrapy
from scrapy.http import Request

from ..items import Roll_Zuqiu


class ZuqiuSpider(scrapy.Spider):
    name = 'roll_zuqiu'
    to_day = datetime.datetime.now()
    allowed_domains = ['hg3535z.com']
    custom_settings = {
        "ITEM_PIPELINES":{
            'hg3535.pipelines.Roll_Zuqiupipeline': 100,
        },
        # 'LOG_LEVEL': 'DEBUG',
        # 'LOG_FILE': "../hg3535/log/roll_zuqiu_{}_{}_{}.log".format(to_day.year, to_day.month,to_day.day)
    }
    start_urls = ['https://hg3535z.com/odds2/d/getodds?sid=1&pt=4&ubt=am&pn=0&sb=2&dc=null&pid=0'] # 滚球菜单 足球滚球列url



    def parse(self, response):
        datas = json.loads(response.text)
        # item = Today_all()
        ids = jsonpath.jsonpath(datas, '$..i-ot[0]..egs..es..i[16]') # ids新列表
        print(ids)
        if ids:
            ids = set(ids)
            for i in ids:
                urls = 'https://hg3535z.com/odds2/d/getamodds?eid={}&iip=true&ubt=am&isp=false'.format(i)
                print(urls)
                yield Request(url=urls, callback=self.parse_each,dont_filter=True)

    def parse_each(self, response):
        # 球队进球数 大小
        try:
            datas = json.loads(response.text)['eg']['es']
            league_id = json.loads(response.text)['eg']['c']['k']
            # 联赛名
            league_name = json.loads(response.text)['eg']['c']['n']
        except:
            datas = ""
            league_id = ""
            # 联赛名
            league_name = ""
        try:
            pt = response.meta['pt']
        except:
            pt = 0
        if datas:
            item = Roll_Zuqiu()
            full_dict = {}
            half_dict = {}
            full_dict_rule = {}
            half_dict_rule = {}
            for data in datas:
                try:
                    new_data = data['pci']['ctid']
                except:
                    new_data = ""
                if new_data == 0:
                    # 比赛id
                    game_id = str(data['k'])
                    # 球队1
                    team_home = data['i'][0]
                    # 球队2
                    team_guest = data['i'][1]
                    # 数量(97>)
                    number = data['i'][2]
                    # 下半场
                    half_way = data['i'][12]
                    # 角球或者其他
                    corner_ball = data['pci'].get('ctn', "")
                    # 日期
                    data_game = data['i'][4]
                    # 开赛时间
                    time_game = data['i'][5]
                    #这个地方的值不准确 需要再次确认
                    score_home = data['i'][10]  # 队一分数
                    score_guest = data['i'][11] # 队二分数
                    # half_way = new_result['i'][12] #下半场


# 让球------------------------------------------------------------------------------------------------------------------
                    try:
                        concedes = data['o']['ah']['v']
                        new_concedes = [concedes[i] for i in range(len(concedes)) if i % 2 == 1]
                        concede_homes = [new_concedes[i] for i in range(len(new_concedes)) if i % 2 == 0]
                        concede_home_rule = [concede_homes[i] for i in range(len(concede_homes)) if i % 2 == 0]
                        if pt is 3:
                            concede_home = [float(concede_homes[i]) - 1 for i in range(len(concede_homes)) if i % 2 is 1]
                        else:
                            concede_home = [concede_homes[i] for i in range(len(concede_homes)) if i % 2 is 1]

                        concede_guests = [new_concedes[i] for i in range(len(new_concedes)) if i % 2 is 1]
                        concede_guest_rule = [concede_guests[i] for i in range(len(concede_guests)) if i % 2 == 0]
                        if pt is 3:
                            concede_guest = [float(concede_guests[i]) -1 for i in range(len(concede_guests)) if i % 2 is 1]
                        else:
                            concede_guest = [concede_guests[i] for i in range(len(concede_guests)) if i % 2 is 1]
                    except:
                        concede_guest = ""
                        concede_guest_rule = ""
                        concede_home = ""
                        concede_home_rule = ""

# 上半场让球half_concede-------------------------------------------------------------------------------------------------
                    try:
                        half_concedes = data['o']['ah1st']['v']
                        new_half_concedes = [half_concedes[i] for i in range(len(half_concedes)) if i % 2 is 1]
                        new_half_concede_homes = [new_half_concedes[i] for i in range(len(new_half_concedes)) if
                                                  i % 2 is 0]
                        half_concede_home_rule = [new_half_concede_homes[i] for i in range(len(new_half_concede_homes)) if i % 2 == 0]
                        # concede_home
                        if pt is 3:
                            half_concede_home = [float(new_half_concede_homes[i]) - 1 for i in range(len(new_half_concede_homes)) if i % 2 is 1]
                        else:
                            half_concede_home = [new_half_concede_homes[i] for i in range(len(new_half_concede_homes)) if i % 2 is 1]
                        half_concede_guests = [new_half_concedes[i] for i in range(len(new_half_concedes)) if i % 2 is 1]
                        half_concede_guest_rule = [half_concede_guests[i] for i in range(len(half_concede_guests)) if i % 2 == 0]
                        # concede_guest
                        if pt is 3:
                            half_concede_guest = [float(half_concede_guests[i]) -1  for i in range(len(half_concede_guests)) if
                                                  i % 2 is 1]
                        else:
                            half_concede_guest = [half_concede_guests[i] for i in range(len(half_concede_guests)) if
                                                  i % 2 is 1]
                    except:
                        half_concede_home_rule = ""
                        half_concede_home = ""
                        half_concede_guest_rule = ""
                        half_concede_guest = ""

# 全场大小size 进球大小---------------------------------------------------------------------------------------------------
                    try:
                        sizes = data['o']['ou']['v']
                        new_sizes = [sizes[i] for i in range(len(sizes)) if i % 2 is 1]
                        size_homes = [new_sizes[i] for i in range(len(new_sizes)) if i % 2 is 0]
                        # size_home_rule
                        size_home_rule = [size_homes[i] for i in range(len(size_homes)) if i % 2 is 0]
                        # size_home
                        if pt is 3:
                            size_home = [float(size_homes[i]) - 1 for i in range(len(size_homes)) if i % 2 is 1]
                        else:
                            size_home = [size_homes[i] for i in range(len(size_homes)) if i % 2 is 1]

                        size_guests = [new_sizes[i] for i in range(len(new_sizes)) if i % 2 is 1]
                        # size_guest_rule
                        size_guest_rule = [size_guests[i] for i in range(len(size_guests)) if i % 2 is 0]
                        # csize_guest
                        if pt is 3:
                            size_guest = [float(size_guests[i]) - 1 for i in range(len(size_guests)) if i % 2 is 1]
                        else:
                            size_guest = [size_guests[i] for i in range(len(size_guests)) if i % 2 is 1]
                    except:
                        size_guest = ""
                        size_guest_rule = ""
                        size_home = ""
                        size_home_rule = ""

# 上半场大小 进球大小 half_size-------------------------------------------------------------------------------------------
                    try:
                        half_sizes = data['o']['ou1st']['v']
                        new_half_sizes = [half_sizes[i] for i in range(len(half_sizes)) if i % 2 is 1]
                        half_size_homes = [new_half_sizes[i] for i in range(len(new_half_sizes)) if i % 2 is 0]
                        # size_home_rule
                        half_size_home_rule = [half_size_homes[i] for i in range(len(half_size_homes)) if
                                               i % 2 is 0]
                        # half_size_home 主队
                        if pt is 3:
                            half_size_home = [float(half_size_homes[i]) -1 for i in range(len(half_size_homes)) if i % 2 is 1]
                        else:
                            half_size_home = [half_size_homes[i] for i in range(len(half_size_homes)) if i % 2 is 1]
                        half_size_guests = [new_half_sizes[i] for i in range(len(new_half_sizes)) if i % 2 is 1]
                        # half_size_guest_rule 客队
                        half_size_guest_rule = [half_size_guests[i] for i in range(len(half_size_guests)) if
                                                i % 2 is 0]
                        # half_size_guest
                        if pt is 3:
                            half_size_guest = [float(half_size_guests[i]) - 1 for i in range(len(half_size_guests)) if
                                               i % 2 is 1]
                        else:
                            half_size_guest = [half_size_guests[i] for i in range(len(half_size_guests)) if
                                               i % 2 is 1]
                    except:
                        half_size_guest = ""
                        half_size_guest_rule = ""
                        half_size_home = ""
                        half_size_home_rule = ""

# 全场总进球 total_goal--------------------------------------------------------------------------------------------------
                    total_goal = {}
                    try:
                        total_goals = data['o']['tg']['v']
                        new_total_goals = [total_goals[i] for i in range(len(total_goals)) if i % 2 is 1]
                        total_goal["total_goal_zero"] = float(new_total_goals[0]) - 1
                        total_goal["total_goal_two"] = float(new_total_goals[1]) - 1
                        total_goal["total_goal_four"] = float(new_total_goals[2]) - 1
                        total_goal["total_goal_seven"] = float(new_total_goals[3]) - 1
                    except:
                        total_goal["total_goal_zero"] = ""
                        total_goal["total_goal_two"] = ""
                        total_goal["total_goal_four"] = ""
                        total_goal["total_goal_seven"] = ""

# 总进球上半场 half_total_goal-------------------------------------------------------------------------------------------
                    try:
                        half_total_goals = data['o']['tg1st']['v']
                        new_half_total_goals = [half_total_goals[i] for i in range(len(half_total_goals)) if i % 2 is 1]
                        total_goal["half_total_goal_zero"] = float(new_half_total_goals[0]) - 1
                        total_goal["half_total_goal_one"] = float(new_half_total_goals[1]) - 1
                        total_goal["half_total_goal_two"] = float(new_half_total_goals[2]) - 1
                        total_goal["half_total_goal_three"] = float(new_half_total_goals[3]) - 1
                    except:
                        total_goal["half_total_goal_zero"] = ""
                        total_goal["half_total_goal_one"] = ""
                        total_goal["half_total_goal_two"] = ""
                        total_goal["half_total_goal_three"] = ""

# 早盘 半场/全场---------------------------------------------------------------------------------------------------------
                    half_full = {}
                    new_lists = ["half_full_home_home", "half_full_home_dogfall", "half_full_home_guest",
                                 "half_full_dogfall_home", "half_full_dogfall_dogfall", "half_full_dogfall_guest",
                                 "half_full_guest_home", "half_full_guest_dogfall", "half_full_guest_guest"]
                    # 早盘 半场/全场
                    try:
                        half_fulls = data['o']['hf']['v']
                        new_half_fulls = [half_fulls[i] for i in range(len(half_fulls)) if i % 2 is 1]
                        for index, value in enumerate(new_lists):
                            half_full[value] = float(new_half_fulls[index]) - 1
                    except:
                        for index, value in enumerate(new_lists):
                            half_full[value] = ""

# 早盘 最先/最后进球 最先进球 ---------------------------------------------------------------------------------------------
                    first_last_ball = {}
                    try:
                        first_balls = data['o']['ttslast']['v']
                        first_ball = [first_balls[i] for i in range(len(first_balls)) if i % 2 is 1]
                        first_last_ball['first_ball_home'] = float(first_ball[0]) - 1
                        first_last_ball['first_ball_guest'] = float(first_ball[1]) - 1
                    except:
                        pass

                    # 早盘 最先/最后进球 最后进球

                    try:
                        last_balls = data['o']['tts1st']['v']
                        last_ball = [last_balls[i] for i in range(len(last_balls)) if i % 2 is 1]
                        first_last_ball['last_ball_home'] = float(last_ball[0]) - 1
                        first_last_ball['last_ball_guest'] = float(last_ball[1]) - 1
                        first_last_ball['not_ball'] = float(last_ball[2]) - 1
                    except:
                        pass

# 全场独赢capot ---------------------------------------------------------------------------------------------------------

                    try:
                        capots = data['o']['1x2']['v']
                        new_capots = [capots[i] for i in range(len(capots)) if i % 2 is 1]
                        capot_home = float(new_capots[0]) - 1
                        capot_guest = float(new_capots[1]) - 1
                        capot_dogfall = float(new_capots[2]) - 1
                    except:
                        capot_home = ""
                        capot_guest = ""
                        capot_dogfall = ""

                    # 上半场独赢capot
                    try:
                        half_capots = data['o']['1x21st']['v']
                        new_half_capots = [half_capots[i] for i in range(len(half_capots)) if i % 2 is 1]
                        half_capot_home = float(new_half_capots[0]) - 1
                        half_capot_guest = float(new_half_capots[1]) - 1
                        half_capot_dogfall = float(new_half_capots[2]) - 1
                    except:
                        half_capot_home = ""
                        half_capot_guest = ""
                        half_capot_dogfall = ""

# 全场入球：单/双 odd_even------------------------------------------------------------------------------------------------
                    try:
                        odd_evens = data['o']['oe']['v']
                        new_odd_evens = [odd_evens[i] for i in range(len(odd_evens)) if i % 2 is 1]
                        odd_even_odd = new_odd_evens[0]
                        odd_even_even = new_odd_evens[1]
                    except:
                        odd_even_odd = ""
                        odd_even_even = ""
                    # 半场入球：单/双 half_odd_even
                    try:
                        half_odd_evens = data['o']['oe1st']['v']
                        new_half_odd_evens = [half_odd_evens[i] for i in range(len(half_odd_evens)) if i % 2 is 1]
                        half_odd_even_odd = new_half_odd_evens[0]
                        half_odd_even_even = new_half_odd_evens[1]
                    except:
                        half_odd_even_odd = ""
                        half_odd_even_even = ""

                    bodan_data = {}
# 波胆------------------------------------------------------------------------------------------------------------------
                    try:
                        bodans = data['o']['cs']['v']
                        one_list = ["bodanhome_one_zero", "bodanhome_two_zero", "bodanhome_two_one",
                                    "bodanhome_three_zero", "bodanhome_three_one", "bodanhome_three_two",
                                    "bodanhome_four_zero", "bodanhome_four_one", "bodanhome_four_two",
                                    "bodanhome_four_three"]
                        two_list = ["bodanguest_one_zero", "bodanguest_two_zero", "bodanguest_two_one",
                                    "bodanguest_three_zero", "bodanguest_three_one", "bodanguest_three_two",
                                    "bodanguest_four_zero", "bodanguest_four_one", "bodanguest_four_two",
                                    "bodanguest_four_three"]
                        three_list = ["bodandogfall_zero_zero", "bodandogfall_one_one", "bodandogfall_two_two",
                                      "bodandogfall_three_three", "bodandogfall_four_four"]
                        new_bodans = [bodans[i] for i in range(len(bodans)) if i % 2 is 1]
                        new_bodan = new_bodans[0:20]
                        # 主队bodan_home
                        bodan_home = [new_bodan[i] for i in range(len(new_bodan)) if i % 2 is 0]
                        for index, t in enumerate(one_list):
                            bodan_data[t] = float(bodan_home[index]) - 1
                        # 客队bodan_guest
                        bodan_guest = [new_bodan[i] for i in range(len(new_bodan)) if i % 2 is 1]
                        for y, z in enumerate(two_list):
                            bodan_data[z] = float(bodan_guest[y]) - 1
                        bodan_dogfall = new_bodans[-7:-2]
                        for a, b in enumerate(three_list):
                            bodan_data[b] = float(bodan_dogfall[a]) - 1
                        # 其他bodan_other
                        bodan_data['bodanother'] = float(new_bodans[-1]) - 1
                    except:
                        pass

                    try:
                        half_bodans = data['o']['cs1st']['v']
                        new_half_bodans = [half_bodans[i] for i in range(len(half_bodans)) if i % 2 is 1]
                        new_one = ["halfbodanhome_one_zero", "halfbodanhome_two_zero", "halfbodanhome_two_one",
                                   "halfbodanhome_three_zero", "halfbodanhome_three_one", "halfbodanhome_three_two"]
                        new_two = ["halfbodanguest_one_zero", "halfbodanguest_two_zero", "halfbodanguest_two_one",
                                   "halfbodanguest_three_zero", "halfbodanguest_three_one", "halfbodanguest_three_two"]
                        new_three = ["halfbodandogfall_zero_zero", "halfbodandogfall_one_one",
                                     "halfbodandogfall_two_two",
                                     "halfbodandogfall_three_three"]
                        halfbodan = new_half_bodans[0:12]
                        half_bodan_home = [halfbodan[i] for i in range(len(halfbodan)) if i % 2 is 0]
                        # 队1
                        for index, t in enumerate(new_one):
                            bodan_data[t] = float(half_bodan_home[index]) - 1
                        # 队2
                        half_bodan_guest = [halfbodan[i] for i in range(len(halfbodan)) if i % 2 is 1]
                        for y, z in enumerate(new_two):
                            bodan_data[z] = float(half_bodan_guest[y]) - 1
                        # 和/平
                        half_bodan_dogfall = new_half_bodans[-6:-2]
                        for a, b in enumerate(new_three):
                            bodan_data[b] = float(half_bodan_dogfall[a]) - 1
                        # 其他
                        bodan_data['halfbodanother'] = float(new_half_bodans[-1]) - 1
                    except:
                        pass
                if new_data == 12:
                    try:
                        home_data = data['o']['ou']['v']

                        # 球队进球 大条件
                        home_size_big_rule = home_data[1]
                        full_dict_rule['home_size_big'] = home_size_big_rule

                        # 球队进球大赔率
                        home_size_big = home_data[5]
                        full_dict['home_size_big'] = home_size_big

                        # 球队进球小条件
                        home_size_small_rule = home_data[3]
                        full_dict_rule['home_size_small'] = home_size_small_rule

                        # 球队进球小赔率
                        home_size_small = home_data[7]
                        full_dict['home_size_small'] = home_size_small
                    except:
                        full_dict['home_size_small'] = ""
                        full_dict_rule['home_size_small'] = ""

                        full_dict['home_size_big'] = ""
                        full_dict_rule['home_size_big'] = ""
                    try:
                        half_home_data = data['o']['ou1st']['v']

                        half_home_size_big_rule = half_home_data[1]
                        half_dict_rule['half_home_size_big'] = half_home_size_big_rule

                        half_home_size_big = half_home_data[5]
                        half_dict['half_home_size_big'] = half_home_size_big

                        half_home_size_small_rule = half_home_data[3]
                        half_dict_rule['half_home_size_small'] = half_home_size_small_rule

                        half_home_size_small = half_home_data[7]
                        half_dict['half_home_size_small'] = half_home_size_small

                    except:
                        half_dict_rule['half_home_size_big'] = ""
                        half_dict['half_home_size_big'] = ""

                        half_dict_rule['half_home_size_small'] = ""
                        half_dict['half_home_size_small'] = ""

                if new_data == 13:
                    try:
                        guest_data = data['o']['ou']['v']

                        guest_size_big_rule = guest_data[1]
                        full_dict_rule['guest_size_big'] = guest_size_big_rule

                        guest_size_big = guest_data[5]
                        full_dict['guest_size_big'] = guest_size_big

                        guest_size_small_rule = guest_data[3]
                        full_dict_rule['guest_size_small'] = guest_size_small_rule

                        guest_size_small = guest_data[7]
                        full_dict['guest_size_small'] = guest_size_small
                    except:
                        full_dict_rule['guest_size_big'] = ""
                        full_dict['guest_size_big'] = ""

                        full_dict_rule['guest_size_small'] = ""
                        full_dict['guest_size_small'] = ''
                    try:
                        half_guest_data = data['o']['ou1st']['v']

                        half_guest_size_big_rule = half_guest_data[1]
                        half_dict_rule['half_guest_size_big'] = half_guest_size_big_rule

                        half_guest_size_big = half_guest_data[5]
                        half_dict['half_guest_size_big'] = half_guest_size_big

                        half_guest_size_small_rule = half_guest_data[3]
                        half_dict_rule['half_guest_size_small'] = half_guest_size_small_rule

                        half_guest_size_small = half_guest_data[7]
                        half_dict['half_guest_size_small'] = half_guest_size_small
                    except:
                        half_dict_rule['half_guest_size_big'] = ""
                        half_dict['half_guest_size_big'] = ""

                        half_dict_rule['half_guest_size_small'] = ""
                        half_dict['half_guest_size_small'] = ""

            item['league_id'] = league_id
            item['league_name'] = league_name
            item['pt'] = pt
            item['game_id'] = game_id
            item['team_home'] = team_home
            item['team_guest'] = team_guest
            item['number'] = number
            item['data_game'] = data_game
            item['time_game'] = time_game
            item['corner_ball'] = corner_ball
            # 波胆
            item['bodan_data'] = bodan_data
            # 早盘入球数单双
            item["odd_even_odd"] = odd_even_odd
            item["odd_even_even"] = odd_even_even
            item["half_odd_even_odd"] = half_odd_even_odd
            item["half_odd_even_even"] = half_odd_even_even

            item['total_goal'] = total_goal

            item["half_capot_home"] = half_capot_home
            item["half_capot_guest"] = half_capot_guest
            item["half_capot_dogfall"] = half_capot_dogfall

            item["capot_home"] = capot_home
            item["capot_guest"] = capot_guest
            item["capot_dogfall"] = capot_dogfall

            item["first_last_ball"] = first_last_ball

            item["half_full"] = half_full

            item["half_size_guest"] = half_size_guest
            item["half_size_guest_rule"] = half_size_guest_rule
            item["half_size_home"] = half_size_home
            item["half_size_home_rule"] = half_size_home_rule
            # 全场大小
            item["size_guest"] = size_guest
            item["size_guest_rule"] = size_guest_rule
            item["size_home"] = size_home
            item["size_home_rule"] = size_home_rule
            # 上半场让球
            item["half_concede_home_rule"] = half_concede_home_rule
            item["half_concede_home"] = half_concede_home
            item["half_concede_guest_rule"] = half_concede_guest_rule
            item["half_concede_guest"] = half_concede_guest
            # 全场让球
            item["concede_guest"] = concede_guest
            item["concede_guest_rule"] = concede_guest_rule
            item["concede_home"] = concede_home
            item["concede_home_rule"] = concede_home_rule
            item['score_home'] = score_home
            item['score_guest'] = score_guest
            item['full_data'] = full_dict
            item['half_data'] = half_dict
            item['full_data_rule'] = full_dict_rule
            item['half_data_rule'] = half_dict_rule
            item['half_way'] = half_way
            yield item