sports_scrapy/hg3535/spiders/zuqiu.py

# -*- coding: utf-8 -*-
import copy
import json

import scrapy
from scrapy.http import Request

from .. items import Zuqiu
# from scrapy.spidermiddlewares.httperror import HttpError
# from twisted.internet.error import DNSLookupError
# from twisted.internet.error import TimeoutError

import datetime
class ZuqiuSpider(scrapy.Spider):
    name = 'zuqiu'
    to_day = datetime.datetime.now()
    # print(to_day.year, to_day)
    allowed_domains = ['hg3535z.com']
    custom_settings = {
        "ITEM_PIPELINES":{
            'hg3535.pipelines.Zuqiupipeline': 300,
        },
        'LOG_LEVEL': 'DEBUG',
        'LOG_FILE': "../hg3535/log/zuqiu_{}_{}_{}.log".format(to_day.year, to_day.month, to_day.day)
    }
    def start_requests(self):

        for y in range(1, 4):
            for i in range(10):
                url = 'https://hg3535z.com/odds2/d/getodds?sid=1&pt=' + str(y) + '&ubt=' + 'am' + '&pn=' + str(
                    i) + '&sb=2&dc=null&pid=0'
                yield scrapy.Request(url=url, callback=self.parse, meta={'pt': y}, dont_filter=True)
                # yield scrapy.Request(url=url, callback=self.parse, meta={'pt': y})

    def parse(self, response):
        if response.text:
            try:
                datas = json.loads(response.text).get('n-ot', "").get('egs', "")
            except:
                datas = ""
            pt = copy.copy(response.meta['pt'])
            if datas:
                for result in datas:
                    new_results = result['es']
                    for new_result in new_results:
                        game_id = str(new_result['i'][16])
                        if pt is 3:
                            url = "https://hg3535z.com/odds2/d/getamodds?eid="+game_id+"&iip=false&ubt=am&isp=true"
                            yield Request(url=url, callback=self.parse_each, meta={'pt': pt})
                            # yield Request(url=url, callback=self.parse_each, meta={'pt': pt}, dont_filter=True)
                        if pt is 2:
                            url = "https://hg3535z.com/odds2/d/getamodds?eid="+game_id+"&iip=false&ubt=am&isp=false"
                            yield Request(url=url, callback=self.parse_each, meta={'pt': pt})
                            # yield Request(url=url, callback=self.parse_each, meta={'pt': pt}, dont_filter=True)
                        if pt is 1:
                            url = "https://hg3535z.com/odds2/d/getamodds?eid="+game_id+"&iip=false&ubt=am&isp=false"
                            yield Request(url=url, callback=self.parse_each, meta={'pt': pt})
                            # yield Request(url=url, callback=self.parse_each, meta={'pt': pt}, dont_filter=True)

    def parse_each(self, response):
        # 球队进球数 大小
        try:
            datas = json.loads(response.text)['eg']['es']
            league_id = json.loads(response.text)['eg']['c']['k']
            # 联赛名
            league_name = json.loads(response.text)['eg']['c']['n']
            zuqiu = json.loads(response.text)['i'][31]
        except:
            datas = ""
            league_id = ""
            # 联赛名
            league_name = ""
            zuqiu = ""
        try:
            pt = response.meta['pt']
        except:
            pt = 0
        if datas:
            item = Zuqiu()
            full_dict = {}
            half_dict = {}
            full_dict_rule = {}
            half_dict_rule = {}
            for data in datas:
                try:
                    new_data = data['pci']['ctid']
                except:
                    new_data = ""
                if new_data == 0:
                    # 比赛id
                    game_id = str(data['k'])
                    # 球队1
                    team_home = data['i'][0]
                    # 球队2
                    team_guest = data['i'][1]
                    # 数量(97>)
                    number = data['i'][2]
                    # 下半场
                    half_way = data['i'][12]
                    # 角球或者其他
                    corner_ball = data['pci'].get('ctn', "")
                    # 日期
                    data_game = data['i'][4]
                    # 开赛时间
                    time_game = data['i'][5]

# 让球------------------------------------------------------------------------------------------------------------------
                    try:
                        concedes = data['o']['ah']['v']
                        new_concedes = [concedes[i] for i in range(len(concedes)) if i % 2 is 1]
                        concede_homes = [new_concedes[i] for i in range(len(new_concedes)) if i % 2 is 0]
                        concede_home_rule = [concede_homes[i] for i in range(len(concede_homes)) if i % 2 is 0]
                        # concede_home
                        concede_guests = [new_concedes[i] for i in range(len(new_concedes)) if i % 2 is 1]
                        # concede_guest_rule
                        concede_guest_rule = [concede_guests[i] for i in range(len(concede_guests)) if
                                              i % 2 == 0]
                        if pt is 3:
                            concede_guest = [round(float(concede_guests[i]) -1, 2) for i in range(len(concede_guests)) if i % 2 is 1]
                            concede_home = [round(float(concede_homes[i]) - 1, 2) for i in range(len(concede_homes)) if i % 2 is 1]
                        else:
                            concede_guest = [concede_guests[i] for i in range(len(concede_guests)) if i % 2 is 1]
                            concede_home = [concede_homes[i] for i in range(len(concede_homes)) if i % 2 is 1]
                    except:
                        concede_guest = ""
                        concede_guest_rule = ""
                        concede_home = ""
                        concede_home_rule = ""

# 上半场让球half_concede-------------------------------------------------------------------------------------------------
                    try:
                        half_concedes = data['o']['ah1st']['v']
                        new_half_concedes = [half_concedes[i] for i in range(len(half_concedes)) if i % 2 is 1]
                        new_half_concede_homes = [new_half_concedes[i] for i in range(len(new_half_concedes)) if i % 2 == 0]
                        # half_concede_home_rule
                        half_concede_home_rule = [new_half_concede_homes[i] for i in
                                                  range(len(new_half_concede_homes)) if i % 2 == 0]
                        # concede_home
                        half_concede_guests = [new_half_concedes[i] for i in range(len(new_half_concedes)) if i % 2 is 1]
                        # concede_guest_rule
                        half_concede_guest_rule = [half_concede_guests[i] for i in
                                                   range(len(half_concede_guests)) if i % 2 == 0]
                        if pt is 3:
                            half_concede_home = [round(float(new_half_concede_homes[i]) - 1, 2) for i in range(len(new_half_concede_homes)) if i % 2 is 1]
                            half_concede_guest = [round(float(half_concede_guests[i]) - 1, 2) for i in range(len(half_concede_guests)) if i % 2 is 1]
                        else:
                            half_concede_home = [new_half_concede_homes[i] for i in range(len(new_half_concede_homes)) if i % 2 is 1]
                            half_concede_guest = [half_concede_guests[i] for i in range(len(half_concede_guests)) if i % 2 is 1]
                    except:
                        half_concede_home_rule = ""
                        half_concede_home = ""
                        half_concede_guest_rule = ""
                        half_concede_guest = ""
# 全场大小size 进球大小---------------------------------------------------------------------------------------------------
                    try:
                        sizes = data['o']['ou']['v']
                        new_sizes = [sizes[i] for i in range(len(sizes)) if i % 2 is 1]
                        size_homes = [new_sizes[i] for i in range(len(new_sizes)) if i % 2 is 0]
                        # size_home_rule
                        size_home_rule = [size_homes[i] for i in range(len(size_homes)) if i % 2 is 0]
                        # size_home
                        size_guests = [new_sizes[i] for i in range(len(new_sizes)) if i % 2 is 1]
                        # size_guest_rule
                        size_guest_rule = [size_guests[i] for i in range(len(size_guests)) if i % 2 is 0]
                        # csize_guest
                        if pt is 3:
                            size_guest = [round(float(size_guests[i]) - 1, 2) for i in range(len(size_guests)) if i % 2 is 1]
                            size_home = [round(float(size_homes[i]) - 1, 2) for i in range(len(size_homes)) if i % 2 is 1]
                        else:
                            size_guest = [size_guests[i] for i in range(len(size_guests)) if i % 2 is 1]
                            size_home = [size_homes[i] for i in range(len(size_homes)) if i % 2 is 1]
                    except:
                        size_guest = ""
                        size_guest_rule = ""
                        size_home = ""
                        size_home_rule = ""

# 上半场大小 进球大小 half_size-------------------------------------------------------------------------------------------
                    try:
                        half_sizes = data['o']['ou1st']['v']
                        new_half_sizes = [half_sizes[i] for i in range(len(half_sizes)) if i % 2 is 1]
                        half_size_homes = [new_half_sizes[i] for i in range(len(new_half_sizes)) if i % 2 is 0]
                        # size_home_rule
                        half_size_home_rule = [half_size_homes[i] for i in range(len(half_size_homes)) if
                                               i % 2 is 0]

                        half_size_guests = [new_half_sizes[i] for i in range(len(new_half_sizes)) if i % 2 is 1]
                        # half_size_guest_rule 客队
                        half_size_guest_rule = [half_size_guests[i] for i in range(len(half_size_guests)) if
                                                i % 2 is 0]
                        # half_size_guest
                        if pt is 3:
                            half_size_guest = [round(float(half_size_guests[i]) - 1, 2) for i in range(len(half_size_guests)) if
                                               i % 2 is 1]
                            half_size_home = [round(float(half_size_homes[i]) - 1, 2) for i in range(len(half_size_homes)) if
                                              i % 2 is 1]
                        else:
                            half_size_guest = [half_size_guests[i] for i in range(len(half_size_guests)) if
                                               i % 2 is 1]
                            half_size_home = [half_size_homes[i] for i in range(len(half_size_homes)) if i % 2 is 1]
                    except:
                        half_size_guest = ""
                        half_size_guest_rule = ""
                        half_size_home = ""
                        half_size_home_rule = ""

# 全场总进球 total_goal--------------------------------------------------------------------------------------------------
                    total_goal = {}
                    try:
                        total_goals = data['o']['tg']['v']
                        new_total_goals = [total_goals[i] for i in range(len(total_goals)) if i % 2 is 1]
                        total_goal["total_goal_zero"] = round(float(new_total_goals[0]) - 1, 2)
                        total_goal["total_goal_two"] = round(float(new_total_goals[1]) - 1, 2)
                        total_goal["total_goal_four"] = round(float(new_total_goals[2]) - 1, 2)
                        total_goal["total_goal_seven"] = round(float(new_total_goals[3]) - 1, 2)
                    except:
                        total_goal["total_goal_zero"] = ""
                        total_goal["total_goal_two"] = ""
                        total_goal["total_goal_four"] = ""
                        total_goal["total_goal_seven"] = ""

# 总进球上半场 half_total_goal-------------------------------------------------------------------------------------------
                    try:
                        half_total_goals = data['o']['tg1st']['v']
                        new_half_total_goals = [half_total_goals[i] for i in range(len(half_total_goals)) if i % 2 is 1]
                        total_goal["half_total_goal_zero"] = round(float(new_half_total_goals[0]) - 1, 2)
                        total_goal["half_total_goal_one"] = round(float(new_half_total_goals[1]) - 1, 2)
                        total_goal["half_total_goal_two"] = round(float(new_half_total_goals[2]) - 1, 2)
                        total_goal["half_total_goal_three"] = round(float(new_half_total_goals[3]) - 1, 2)
                    except:
                        total_goal["half_total_goal_zero"] = ""
                        total_goal["half_total_goal_one"] = ""
                        total_goal["half_total_goal_two"] = ""
                        total_goal["half_total_goal_three"] = ""

# 早盘 半场/全场---------------------------------------------------------------------------------------------------------
                    half_full = {}
                    new_lists = ["half_full_home_home", "half_full_home_dogfall", "half_full_home_guest",
                                 "half_full_dogfall_home", "half_full_dogfall_dogfall", "half_full_dogfall_guest",
                                 "half_full_guest_home", "half_full_guest_dogfall", "half_full_guest_guest"]
                    # 早盘 半场/全场
                    try:
                        half_fulls = data['o']['hf']['v']
                        new_half_fulls = [half_fulls[i] for i in range(len(half_fulls)) if i % 2 is 1]
                        for index, value in enumerate(new_lists):
                            half_full[value] = round(float(new_half_fulls[index]) - 1, 2)
                    except:
                        for index, value in enumerate(new_lists):
                            half_full[value] = ""

# 早盘 最先/最后进球 最先进球 ---------------------------------------------------------------------------------------------
                    first_last_ball = {}
                    try:
                        first_balls = data['o']['ttslast']['v']
                        first_ball = [first_balls[i] for i in range(len(first_balls)) if i % 2 is 1]
                        first_last_ball['first_ball_home'] = round(float(first_ball[0]) - 1, 2)
                        first_last_ball['first_ball_guest'] = round(float(first_ball[1]) - 1, 2)
                    except:
                        pass

                    # 早盘 最先/最后进球 最后进球

                    try:
                        last_balls = data['o']['tts1st']['v']
                        last_ball = [last_balls[i] for i in range(len(last_balls)) if i % 2 is 1]
                        first_last_ball['last_ball_home'] = round(float(last_ball[0]) - 1, 2)
                        first_last_ball['last_ball_guest'] = round(float(last_ball[1]) - 1, 2)
                        first_last_ball['not_ball'] = round(float(last_ball[2]) - 1, 2)
                    except:
                        pass

# 全场独赢capot ---------------------------------------------------------------------------------------------------------

                    try:
                        capots = data['o']['1x2']['v']
                        new_capots = [capots[i] for i in range(len(capots)) if i % 2 is 1]
                        capot_home = round(float(new_capots[0]) - 1, 2)
                        capot_guest = round(float(new_capots[1]) - 1, 2)
                        capot_dogfall = round(float(new_capots[2]) - 1, 2)
                    except:
                        capot_home = ""
                        capot_guest = ""
                        capot_dogfall = ""

                    # 上半场独赢capot
                    try:
                        half_capots = data['o']['1x21st']['v']
                        new_half_capots = [half_capots[i] for i in range(len(half_capots)) if i % 2 is 1]
                        half_capot_home = round(float(new_half_capots[0]) - 1, 2)
                        half_capot_guest = round(float(new_half_capots[1]) - 1, 2)
                        half_capot_dogfall = round(float(new_half_capots[2]) - 1, 2)
                    except:
                        half_capot_home = ""
                        half_capot_guest = ""
                        half_capot_dogfall = ""

# 全场入球：单/双 odd_even------------------------------------------------------------------------------------------------
                    try:
                        odd_evens = data['o']['oe']['v']
                        new_odd_evens = [odd_evens[i] for i in range(len(odd_evens)) if i % 2 is 1]
                        if pt is 3:
                            odd_even_odd = round(float(new_odd_evens[0]) - 1, 2)
                            odd_even_even = round(float(new_odd_evens[1]) - 1, 2)
                        else:
                            odd_even_odd = round(float(new_odd_evens[0]), 2)
                            odd_even_even = round(float(new_odd_evens[1]), 2)
                    except:
                        odd_even_odd = ""
                        odd_even_even = ""
                    # 半场入球：单/双 half_odd_even
                    try:
                        half_odd_evens = data['o']['oe1st']['v']
                        new_half_odd_evens = [half_odd_evens[i] for i in range(len(half_odd_evens)) if i % 2 is 1]
                        if pt is 3:
                            half_odd_even_odd = round(float(new_half_odd_evens[0]) - 1, 2)
                            half_odd_even_even = round(float(new_half_odd_evens[1]) - 1, 2)
                        else:
                            half_odd_even_odd = new_half_odd_evens[0]
                            half_odd_even_even = new_half_odd_evens[1]
                    except:
                        half_odd_even_odd = ""
                        half_odd_even_even = ""

                    bodan_data = {}
# 波胆------------------------------------------------------------------------------------------------------------------
                    try:
                        bodans = data['o']['cs']['v']
                        one_list = ["bodanhome_one_zero", "bodanhome_two_zero", "bodanhome_two_one",
                                    "bodanhome_three_zero", "bodanhome_three_one", "bodanhome_three_two",
                                    "bodanhome_four_zero", "bodanhome_four_one", "bodanhome_four_two",
                                    "bodanhome_four_three"]
                        two_list = ["bodanguest_one_zero", "bodanguest_two_zero", "bodanguest_two_one",
                                    "bodanguest_three_zero", "bodanguest_three_one", "bodanguest_three_two",
                                    "bodanguest_four_zero", "bodanguest_four_one", "bodanguest_four_two",
                                    "bodanguest_four_three"]
                        three_list = ["bodandogfall_zero_zero", "bodandogfall_one_one", "bodandogfall_two_two",
                                      "bodandogfall_three_three", "bodandogfall_four_four"]
                        new_bodans = [bodans[i] for i in range(len(bodans)) if i % 2 is 1]
                        new_bodan = new_bodans[0:20]
                        # 主队bodan_home
                        bodan_home = [new_bodan[i] for i in range(len(new_bodan)) if i % 2 is 0]
                        for index, t in enumerate(one_list):
                            bodan_data[t] = round(float(bodan_home[index]) - 1, 2)
                        # 客队bodan_guest
                        bodan_guest = [new_bodan[i] for i in range(len(new_bodan)) if i % 2 is 1]
                        for y, z in enumerate(two_list):
                            bodan_data[z] = round(float(bodan_guest[y]) - 1, 2)
                        bodan_dogfall = new_bodans[-7:-2]
                        for a, b in enumerate(three_list):
                            bodan_data[b] = round(float(bodan_dogfall[a]) - 1, 2)
                        # 其他bodan_other
                        bodan_data['bodanother'] = round(float(new_bodans[-1]) - 1, 2)
                    except:
                        pass

                    try:
                        half_bodans = data['o']['cs1st']['v']
                        new_half_bodans = [half_bodans[i] for i in range(len(half_bodans)) if i % 2 is 1]
                        new_one = ["halfbodanhome_one_zero", "halfbodanhome_two_zero", "halfbodanhome_two_one",
                                   "halfbodanhome_three_zero", "halfbodanhome_three_one", "halfbodanhome_three_two"]
                        new_two = ["halfbodanguest_one_zero", "halfbodanguest_two_zero", "halfbodanguest_two_one",
                                   "halfbodanguest_three_zero", "halfbodanguest_three_one", "halfbodanguest_three_two"]
                        new_three = ["halfbodandogfall_zero_zero", "halfbodandogfall_one_one",
                                     "halfbodandogfall_two_two",
                                     "halfbodandogfall_three_three"]
                        halfbodan = new_half_bodans[0:12]
                        half_bodan_home = [halfbodan[i] for i in range(len(halfbodan)) if i % 2 is 0]
                        # 队1
                        for index, t in enumerate(new_one):
                            bodan_data[t] = round(float(half_bodan_home[index]) - 1, 2)
                        # 队2
                        half_bodan_guest = [halfbodan[i] for i in range(len(halfbodan)) if i % 2 is 1]
                        for y, z in enumerate(new_two):
                            bodan_data[z] = round(float(half_bodan_guest[y]) - 1, 2)
                        # 和/平
                        half_bodan_dogfall = new_half_bodans[-6:-2]
                        for a, b in enumerate(new_three):
                            bodan_data[b] = round(float(half_bodan_dogfall[a]) - 1, 2)
                        # 其他
                        bodan_data['halfbodanother'] = round(float(new_half_bodans[-1]) - 1, 2)
                    except:
                        pass
                if new_data is 12:
                    try:
                        home_data = data['o']['ou']['v']
                        # 球队进球 大条件
                        full_dict_rule['home_size_big'] = home_data[1]
                        # 球队进球小条件
                        full_dict_rule['home_size_small'] = home_data[3]

                        if pt is 3:
                            # 球队进球大赔率
                            full_dict['home_size_big'] = round(float(home_data[5]) - 1, 2)
                            # 球队进球小赔率
                            full_dict['home_size_small'] = round(float(home_data[7]) - 1, 2)
                        else:
                            # 球队进球大赔率
                            full_dict['home_size_big'] = home_data[5]
                            # 球队进球小赔率
                            full_dict['home_size_small'] = home_data[7]
                    except:
                        full_dict['home_size_small'] = ""
                        full_dict_rule['home_size_small'] = ""
                        full_dict['home_size_big'] = ""
                        full_dict_rule['home_size_big'] = ""
                    try:
                        half_home_data = data['o']['ou1st']['v']
                        half_dict_rule['half_home_size_big'] = half_home_data[1]
                        half_dict_rule['half_home_size_small'] = half_home_data[3]

                        if pt is 3:
                            half_dict['half_home_size_big'] = round(float(half_home_data[5]) - 1, 2)
                            half_dict['half_home_size_small'] = round(float(half_home_data[7]) - 1, 2)
                        else:
                            half_home_size_big = half_home_data[5]
                            half_dict['half_home_size_big'] = half_home_size_big
                            half_home_size_small = half_home_data[7]
                            half_dict['half_home_size_small'] = half_home_size_small

                    except:
                        half_dict_rule['half_home_size_big'] = ""
                        half_dict['half_home_size_big'] = ""
                        half_dict_rule['half_home_size_small'] = ""
                        half_dict['half_home_size_small'] = ""

                if new_data is 13:
                    try:
                        guest_data = data['o']['ou']['v']
                        full_dict_rule['guest_size_big'] = guest_data[1]
                        full_dict_rule['guest_size_small'] = guest_data[3]

                        if pt is 3:
                            full_dict['guest_size_big'] = round(float(guest_data[5]) - 1, 2)
                            full_dict['guest_size_small'] = round(float(guest_data[7]) - 1, 2)
                        else:
                            full_dict['guest_size_big'] = guest_data[5]
                            full_dict['guest_size_small'] = guest_data[7]
                    except:
                        full_dict_rule['guest_size_big'] = ""
                        full_dict['guest_size_big'] = ""
                        full_dict_rule['guest_size_small'] = ""
                        full_dict['guest_size_small'] = ''
                    try:
                        half_guest_data = data['o']['ou1st']['v']
                        half_dict_rule['half_guest_size_big'] = half_guest_data[1]
                        half_dict_rule['half_guest_size_small'] = half_guest_data[3]
                        if pt is 3:
                            half_dict['half_guest_size_big'] = round(float(half_guest_data[5]) - 1, 2)
                            half_dict['half_guest_size_small'] = round(float(half_guest_data[7]) - 1, 2)
                        else:
                            half_dict['half_guest_size_big'] = half_guest_data[5]
                            half_dict['half_guest_size_small'] = half_guest_data[7]
                    except:
                        half_dict_rule['half_guest_size_big'] = ""
                        half_dict['half_guest_size_big'] = ""

                        half_dict_rule['half_guest_size_small'] = ""
                        half_dict['half_guest_size_small'] = ""

            item['league_id'] = league_id
            item['league_name'] = league_name
            item['pt'] = pt
            item['game_id'] = game_id
            item['team_home'] = team_home
            item['team_guest'] = team_guest
            item['number'] = number
            item['data_game'] = data_game
            item['time_game'] = time_game
            item['corner_ball'] = corner_ball
            # 波胆
            item['bodan_data'] = bodan_data
            # 早盘入球数单双
            item["odd_even_odd"] = odd_even_odd
            item["odd_even_even"] = odd_even_even
            item["half_odd_even_odd"] = half_odd_even_odd
            item["half_odd_even_even"] = half_odd_even_even

            item['total_goal'] = total_goal

            item["half_capot_home"] = half_capot_home
            item["half_capot_guest"] = half_capot_guest
            item["half_capot_dogfall"] = half_capot_dogfall

            item["capot_home"] = capot_home
            item["capot_guest"] = capot_guest
            item["capot_dogfall"] = capot_dogfall

            item["first_last_ball"] = first_last_ball

            item["half_full"] = half_full

            item["half_size_guest"] = half_size_guest
            item["half_size_guest_rule"] = half_size_guest_rule
            item["half_size_home"] = half_size_home
            item["half_size_home_rule"] = half_size_home_rule
            # 全场大小
            item["size_guest"] = size_guest
            item["size_guest_rule"] = size_guest_rule
            item["size_home"] = size_home
            item["size_home_rule"] = size_home_rule
            # 上半场让球
            item["half_concede_home_rule"] = half_concede_home_rule
            item["half_concede_home"] = half_concede_home
            item["half_concede_guest_rule"] = half_concede_guest_rule
            item["half_concede_guest"] = half_concede_guest
            # 全场让球
            item["concede_guest"] = concede_guest
            item["concede_guest_rule"] = concede_guest_rule
            item["concede_home"] = concede_home
            item["concede_home_rule"] = concede_home_rule

            item['full_data'] = full_dict
            item['half_data'] = half_dict
            item['full_data_rule'] = full_dict_rule
            item['half_data_rule'] = half_dict_rule
            item['zuqiu'] = zuqiu
            yield item