sports_scrapy/hg3535/spiders/zuqiu.py

# -*- coding: utf-8 -*-
import copy
import json

# import redis
import scrapy
from scrapy.http import Request
from ..items import Zuqiu
import datetime


class ZuqiuSpider(scrapy.Spider):
    name = 'zuqiu'
    to_day = datetime.datetime.now()
    # print(to_day.year, to_day)
    allowed_domains = ['hg3535z.com']
    custom_settings = {
        "ITEM_PIPELINES": {
            'hg3535.pipeline.zuqiu.Zuqiupipeline': 300,
        },
        # 'LOG_LEVEL': 'DEBUG',
        # 'LOG_FILE': "../hg3535/log/zuqiu_{}_{}_{}.log".format(to_day.year, to_day.month, to_day.day),
        # 'RETRY_ENABLED': False,
        # 'COOKIES_ENABLED': False,
        # 'DOWNLOAD_TIMEOUT': 1,
        # 'SCHEDULER_PERSIST': False,
        # 'SCHEDULER_FLUSH_ON_START': False,
    }

    def start_requests(self):
        for y in range(1, 4):
            for i in range(10):
                y = str(y)
                i = str(i)
                # yield y
                url = "https://odata.jiushan6688.com/odds6i/d/getodds/zh-cn/sid/1/pt/" + y + "/ubt/am/pn/" + i + "/sb/2/dc/null/pid/0"
                yield scrapy.Request(url=url, callback=self.parse, dont_filter=True, meta={'pt': y})

    def parse(self, response):
        if response.status == 200:
            # cls = redis.Redis(host='192.168.0.110', port=6379, db=0)
            if response.text:
                try:
                    datas = json.loads(response.text).get('n-ot', "").get('egs', "")
                except:
                    # datas = ''
                    print("为获取到数据")
                    return
                pt = copy.copy(response.meta['pt'])
                for result in datas:
                    new_results = result['es']
                    for new_result in new_results:
                        game_id = str(new_result['i'][16])
                        # cls.sadd('hg3535.zq.ids', game_id)
                        # cls.expire('hg3535.zq.ids', 300)
                        if pt == "3":
                            url = "https://odata.jiushan6688.com/odds6i/d/getamodds/zh-cn/eid/" + str(game_id) + "/iip/false/ubt/am/isp/true"
                            yield Request(url=url, callback=self.parse_each, dont_filter=True, meta={'pt': pt})
                        if pt == "2":
                            url = "https://odata.jiushan6688.com/odds6i/d/getamodds/zh-cn/eid/" + str(game_id) + "/iip/false/ubt/am/isp/false"
                            yield Request(url=url, callback=self.parse_each, dont_filter=True, meta={'pt': pt})
                        if pt == "1":
                            url = "https://odata.jiushan6688.com/odds6i/d/getamodds/zh-cn/eid/" + str(game_id) + "/iip/false/ubt/am/isp/false"
                            yield Request(url=url, callback=self.parse_each, dont_filter=True, meta={'pt': pt})

    #
    def parse_each(self, response):
        if response.status == 200:
            # 球队进球数 大小
            try:
                datas = json.loads(response.text)['eg']['es']
                league_id = json.loads(response.text)['eg']['c']['k']
                # 联赛名
                league_name = json.loads(response.text)['eg']['c']['n']
                # zuqiu = json.loads(response.text)['i'][31]
            except:
                datas = ""
                league_id = ""
                # 联赛名
                league_name = ""
                # zuqiu = ""
            try:
                pt = response.meta['pt']
            except:
                pt = 0
            if datas:
                item = Zuqiu()
                full_dict = {}
                half_dict = {}
                full_dict_rule = {}
                half_dict_rule = {}
                for data in datas:
                    try:
                        new_data = data['pci']['ctid']
                    except:
                        new_data = ""
                    if new_data == 0:
                        # 比赛id
                        game_id = str(data['k'])
                        # 球队1
                        team_home = data['i'][0]
                        # 球队2
                        team_guest = data['i'][1]
                        # 数量(97>)
                        number = data['i'][2]
                        # 下半场
                        half_way = data['i'][12]
                        # 角球或者其他
                        try:
                            corner_ball = data['pci']['ctn']
                        except:
                            corner_ball = 0
                        # 日期
                        data_game = data['i'][4]
                        # 开赛时间
                        time_game = data['i'][5]

# 让球------------------------------------------------------------------------------------------------------------------
                        try:
                            concedes = data['o']['ah']['v']
                            new_concedes = [concedes[i] for i in range(len(concedes)) if i % 2 is 1]
                            concede_homes = [new_concedes[i] for i in range(len(new_concedes)) if i % 2 is 0]
                            concede_home_rule = [concede_homes[i] for i in range(len(concede_homes)) if i % 2 is 0]
                            # concede_home
                            concede_guests = [new_concedes[i] for i in range(len(new_concedes)) if i % 2 is 1]
                            # concede_guest_rule
                            concede_guest_rule = [concede_guests[i] for i in range(len(concede_guests)) if
                                                  i % 2 == 0]
                            if pt is 3:
                                concede_guest = [round(float(concede_guests[i]) - 1, 2) for i in range(len(concede_guests)) if i % 2 is 1]
                                concede_home = [round(float(concede_homes[i]) - 1, 2) for i in range(len(concede_homes)) if i % 2 is 1]
                            else:
                                concede_guest = [concede_guests[i] for i in range(len(concede_guests)) if i % 2 is 1]
                                concede_home = [concede_homes[i] for i in range(len(concede_homes)) if i % 2 is 1]
                        except:
                            concede_guest = ""
                            concede_guest_rule = ""
                            concede_home = ""
                            concede_home_rule = ""

# 上半场让球half_concede-------------------------------------------------------------------------------------------------
                        try:
                            half_concedes = data['o']['ah1st']['v']
                            new_half_concedes = [half_concedes[i] for i in range(len(half_concedes)) if i % 2 is 1]
                            new_half_concede_homes = [new_half_concedes[i] for i in range(len(new_half_concedes)) if i % 2 == 0]
                            # half_concede_home_rule
                            half_concede_home_rule = [new_half_concede_homes[i] for i in
                                                      range(len(new_half_concede_homes)) if i % 2 == 0]
                            # concede_home
                            half_concede_guests = [new_half_concedes[i] for i in range(len(new_half_concedes)) if i % 2 is 1]
                            # concede_guest_rule
                            half_concede_guest_rule = [half_concede_guests[i] for i in
                                                       range(len(half_concede_guests)) if i % 2 == 0]
                            if pt is 3:
                                half_concede_home = [round(float(new_half_concede_homes[i]) - 1, 2) for i in range(len(new_half_concede_homes)) if
                                                     i % 2 is 1]
                                half_concede_guest = [round(float(half_concede_guests[i]) - 1, 2) for i in range(len(half_concede_guests)) if
                                                      i % 2 is 1]
                            else:
                                half_concede_home = [new_half_concede_homes[i] for i in range(len(new_half_concede_homes)) if i % 2 is 1]
                                half_concede_guest = [half_concede_guests[i] for i in range(len(half_concede_guests)) if i % 2 is 1]
                        except:
                            half_concede_home_rule = ""
                            half_concede_home = ""
                            half_concede_guest_rule = ""
                            half_concede_guest = ""
# 全场大小size 进球大小---------------------------------------------------------------------------------------------------
                        try:
                            sizes = data['o']['ou']['v']
                            new_sizes = [sizes[i] for i in range(len(sizes)) if i % 2 is 1]
                            size_homes = [new_sizes[i] for i in range(len(new_sizes)) if i % 2 is 0]
                            # size_home_rule
                            size_home_rule = [size_homes[i] for i in range(len(size_homes)) if i % 2 is 0]
                            # size_home
                            size_guests = [new_sizes[i] for i in range(len(new_sizes)) if i % 2 is 1]
                            # size_guest_rule
                            size_guest_rule = [size_guests[i] for i in range(len(size_guests)) if i % 2 is 0]
                            # csize_guest
                            if pt is 3:
                                size_guest = [round(float(size_guests[i]) - 1, 2) for i in range(len(size_guests)) if i % 2 is 1]
                                size_home = [round(float(size_homes[i]) - 1, 2) for i in range(len(size_homes)) if i % 2 is 1]
                            else:
                                size_guest = [size_guests[i] for i in range(len(size_guests)) if i % 2 is 1]
                                size_home = [size_homes[i] for i in range(len(size_homes)) if i % 2 is 1]
                        except:
                            size_guest = ""
                            size_guest_rule = ""
                            size_home = ""
                            size_home_rule = ""

# 上半场大小 进球大小 half_size-------------------------------------------------------------------------------------------
                        try:
                            half_sizes = data['o']['ou1st']['v']
                            new_half_sizes = [half_sizes[i] for i in range(len(half_sizes)) if i % 2 is 1]
                            half_size_homes = [new_half_sizes[i] for i in range(len(new_half_sizes)) if i % 2 is 0]
                            # size_home_rule
                            half_size_home_rule = [half_size_homes[i] for i in range(len(half_size_homes)) if
                                                   i % 2 is 0]

                            half_size_guests = [new_half_sizes[i] for i in range(len(new_half_sizes)) if i % 2 is 1]
                            # half_size_guest_rule 客队
                            half_size_guest_rule = [half_size_guests[i] for i in range(len(half_size_guests)) if
                                                    i % 2 is 0]
                            # half_size_guest
                            if pt is 3:
                                half_size_guest = [round(float(half_size_guests[i]) - 1, 2) for i in range(len(half_size_guests)) if
                                                   i % 2 is 1]
                                half_size_home = [round(float(half_size_homes[i]) - 1, 2) for i in range(len(half_size_homes)) if
                                                  i % 2 is 1]
                            else:
                                half_size_guest = [half_size_guests[i] for i in range(len(half_size_guests)) if
                                                   i % 2 is 1]
                                half_size_home = [half_size_homes[i] for i in range(len(half_size_homes)) if i % 2 is 1]
                        except:
                            half_size_guest = ""
                            half_size_guest_rule = ""
                            half_size_home = ""
                            half_size_home_rule = ""

# 全场总进球 total_goal--------------------------------------------------------------------------------------------------
                        total_goal = {}
                        try:
                            total_goals = data['o']['tg']['v']
                            new_total_goals = [total_goals[i] for i in range(len(total_goals)) if i % 2 is 1]
                            total_goal["tg0"] = round(float(new_total_goals[0]) - 1, 2)
                            total_goal["tg1"] = round(float(new_total_goals[1]) - 1, 2)
                            total_goal["tg2"] = round(float(new_total_goals[2]) - 1, 2)
                            total_goal["tg3"] = round(float(new_total_goals[3]) - 1, 2)
                        except:
                            total_goal["tg0"] = ""
                            total_goal["tg1"] = ""
                            total_goal["tg2"] = ""
                            total_goal["tg3"] = ""

# 总进球上半场 half_total_goal-------------------------------------------------------------------------------------------
                        try:
                            half_total_goals = data['o']['tg1st']['v']
                            new_half_total_goals = [half_total_goals[i] for i in range(len(half_total_goals)) if i % 2 is 1]
                            total_goal["tg0_h"] = round(float(new_half_total_goals[0]) - 1, 2)
                            total_goal["tg1_h"] = round(float(new_half_total_goals[1]) - 1, 2)
                            total_goal["tg2_h"] = round(float(new_half_total_goals[2]) - 1, 2)
                            total_goal["tg3_h"] = round(float(new_half_total_goals[3]) - 1, 2)
                        except:
                            total_goal["tg0_h"] = ""
                            total_goal["tg1_h"] = ""
                            total_goal["tg2_h"] = ""
                            total_goal["tg3_h"] = ""

# 早盘 半场/全场---------------------------------------------------------------------------------------------------------
                        half_full = {}
                        new_lists = ["hfhh", "hfhd", "hfhg", "hfdh", "hfdd", "hfdg", "hfgh", "hfgd", "hfgg"]
                        # 早盘 半场/全场
                        try:
                            half_fulls = data['o']['hf']['v']
                            new_half_fulls = [half_fulls[i] for i in range(len(half_fulls)) if i % 2 is 1]
                            for index, value in enumerate(new_lists):
                                half_full[value] = round(float(new_half_fulls[index]) - 1, 2)
                        except:
                            for index, value in enumerate(new_lists):
                                half_full[value] = ""

# 早盘 最先/最后进球 最先进球 ---------------------------------------------------------------------------------------------
                        first_last_ball = {}
                        try:
                            first_balls = data['o']['ttslast']['v']
                            first_ball = [first_balls[i] for i in range(len(first_balls)) if i % 2 is 1]
                            first_last_ball['flbfh'] = round(float(first_ball[0]) - 1, 2)
                            first_last_ball['flbfg'] = round(float(first_ball[1]) - 1, 2)
                        except:
                            pass

                        # 早盘 最先/最后进球 最后进球

                        try:
                            last_balls = data['o']['tts1st']['v']
                            last_ball = [last_balls[i] for i in range(len(last_balls)) if i % 2 is 1]
                            first_last_ball['flblh'] = round(float(last_ball[0]) - 1, 2)
                            first_last_ball['flblg'] = round(float(last_ball[1]) - 1, 2)
                            first_last_ball['flbn'] = round(float(last_ball[2]) - 1, 2)
                        except:
                            pass

# 全场独赢capot ---------------------------------------------------------------------------------------------------------

                        try:
                            capots = data['o']['1x2']['v']
                            new_capots = [capots[i] for i in range(len(capots)) if i % 2 is 1]
                            capot_home = round(float(new_capots[0]) - 1, 2)
                            capot_guest = round(float(new_capots[1]) - 1, 2)
                            capot_dogfall = round(float(new_capots[2]) - 1, 2)
                        except:
                            capot_home = ""
                            capot_guest = ""
                            capot_dogfall = ""

                        # 上半场独赢capot
                        try:
                            half_capots = data['o']['1x21st']['v']
                            new_half_capots = [half_capots[i] for i in range(len(half_capots)) if i % 2 is 1]
                            half_capot_home = round(float(new_half_capots[0]) - 1, 2)
                            half_capot_guest = round(float(new_half_capots[1]) - 1, 2)
                            half_capot_dogfall = round(float(new_half_capots[2]) - 1, 2)
                        except:
                            half_capot_home = ""
                            half_capot_guest = ""
                            half_capot_dogfall = ""

# 全场入球：单/双 odd_even------------------------------------------------------------------------------------------------
                        try:
                            odd_evens = data['o']['oe']['v']
                            new_odd_evens = [odd_evens[i] for i in range(len(odd_evens)) if i % 2 is 1]
                            if pt is 3:
                                odd_even_odd = round(float(new_odd_evens[0]) - 1, 2)
                                odd_even_even = round(float(new_odd_evens[1]) - 1, 2)
                            else:
                                odd_even_odd = round(float(new_odd_evens[0]), 2)
                                odd_even_even = round(float(new_odd_evens[1]), 2)
                        except:
                            odd_even_odd = ""
                            odd_even_even = ""
                        # 半场入球：单/双 half_odd_even
                        try:
                            half_odd_evens = data['o']['oe1st']['v']
                            new_half_odd_evens = [half_odd_evens[i] for i in range(len(half_odd_evens)) if i % 2 is 1]
                            if pt is 3:
                                half_odd_even_odd = round(float(new_half_odd_evens[0]) - 1, 2)
                                half_odd_even_even = round(float(new_half_odd_evens[1]) - 1, 2)
                            else:
                                half_odd_even_odd = new_half_odd_evens[0]
                                half_odd_even_even = new_half_odd_evens[1]
                        except:
                            half_odd_even_odd = ""
                            half_odd_even_even = ""

                        bodan_data = {}
# 波胆------------------------------------------------------------------------------------------------------------------
                        try:
                            bodans = data['o']['cs']['v']
                            one_list = ["b10", "b20", "b21", "b30", "b31", "b32", "b40", "b41", "b42", "b43"]
                            two_list = ["b01", "b02", "b12", "b03", "b13", "b23", "b04", "b14", "b24", "b34"]
                            three_list = ["b00", "b11", "b22", "b33", "b44"]
                            new_bodans = [bodans[i] for i in range(len(bodans)) if i % 2 is 1]
                            new_bodan = new_bodans[0:20]
                            # 主队bodan_home
                            bodan_home = [new_bodan[i] for i in range(len(new_bodan)) if i % 2 is 0]
                            for index, t in enumerate(one_list):
                                bodan_data[t] = round(float(bodan_home[index]) - 1, 2)
                            # 客队bodan_guest
                            bodan_guest = [new_bodan[i] for i in range(len(new_bodan)) if i % 2 is 1]
                            for y, z in enumerate(two_list):
                                bodan_data[z] = round(float(bodan_guest[y]) - 1, 2)
                            bodan_dogfall = new_bodans[-6:-1]
                            for a, b in enumerate(three_list):
                                bodan_data[b] = round(float(bodan_dogfall[a]) - 1, 2)
                            # 其他bodan_other
                            bodan_data['bo'] = round(float(new_bodans[-1]) - 1, 2)
                        except:
                            pass

                        try:
                            half_bodans = data['o']['cs1st']['v']
                            new_half_bodans = [half_bodans[i] for i in range(len(half_bodans)) if i % 2 is 1]
                            new_one = ["b10_h", "b20_h", "b21_h", "b30_h", "b31_h", "b32_h"]
                            new_two = ["b01_h", "b02_h", "b12_h", "b03_h", "b13_h", "b23_h"]
                            new_three = ["b00_h", "b11_h", "b22_h", "b33_h"]
                            halfbodan = new_half_bodans[0:12]
                            half_bodan_home = [halfbodan[i] for i in range(len(halfbodan)) if i % 2 is 0]
                            # 队1
                            for index, t in enumerate(new_one):
                                bodan_data[t] = round(float(half_bodan_home[index]) - 1, 2)
                            # 队2
                            half_bodan_guest = [halfbodan[i] for i in range(len(halfbodan)) if i % 2 is 1]
                            for y, z in enumerate(new_two):
                                bodan_data[z] = round(float(half_bodan_guest[y]) - 1, 2)
                            # 和/平
                            half_bodan_dogfall = new_half_bodans[-6:-2]
                            for a, b in enumerate(new_three):
                                bodan_data[b] = round(float(half_bodan_dogfall[a]) - 1, 2)
                            # 其他
                            bodan_data['bo_h'] = round(float(new_half_bodans[-1]) - 1, 2)
                        except:
                            pass
                    if new_data is 12:
                        try:
                            home_data = data['o']['ou']['v']
                            # 球队进球 大条件
                            full_dict_rule['tbhb'] = home_data[1]
                            # 球队进球小条件
                            full_dict_rule['tbhs'] = home_data[3]

                            if pt is 3:
                                # 球队进球大赔率
                                full_dict['tbhb'] = round(float(home_data[5]) - 1, 2)
                                # 球队进球小赔率
                                full_dict['tbhs'] = round(float(home_data[7]) - 1, 2)
                            else:
                                # 球队进球大赔率
                                full_dict['tbhb'] = home_data[5]
                                # 球队进球小赔率
                                full_dict['tbhs'] = home_data[7]
                        except:
                            full_dict['tbhs'] = ""
                            full_dict_rule['tbhs'] = ""
                            full_dict['tbhb'] = ""
                            full_dict_rule['tbhb'] = ""
                        try:
                            half_home_data = data['o']['ou1st']['v']
                            half_dict_rule['tbhb_h'] = half_home_data[1]
                            half_dict_rule['tbhs_h'] = half_home_data[3]

                            if pt is 3:
                                half_dict['tbhb_h'] = round(float(half_home_data[5]) - 1, 2)
                                half_dict['tbhs_h'] = round(float(half_home_data[7]) - 1, 2)
                            else:
                                half_dict['tbhb_h'] = half_home_data[5]
                                half_dict['tbhs_h'] = half_home_data[7]

                        except:
                            half_dict_rule['tbhb_h'] = ""
                            half_dict['tbhb_h'] = ""
                            half_dict_rule['tbhs_h'] = ""
                            half_dict['tbhs_h'] = ""

                    if new_data is 13:
                        try:
                            guest_data = data['o']['ou']['v']
                            full_dict_rule['tbgb'] = guest_data[1]
                            full_dict_rule['tbgs'] = guest_data[3]

                            if pt is 3:
                                full_dict['tbgb'] = round(float(guest_data[5]) - 1, 2)
                                full_dict['tbgs'] = round(float(guest_data[7]) - 1, 2)
                            else:
                                full_dict['tbgb'] = guest_data[5]
                                full_dict['tbgs'] = guest_data[7]
                        except:
                            full_dict_rule['tbgb'] = ""
                            full_dict['tbgb'] = ""
                            full_dict_rule['tbgs'] = ""
                            full_dict['tbgs'] = ''
                        try:
                            half_guest_data = data['o']['ou1st']['v']
                            half_dict_rule['tbgb_h'] = half_guest_data[1]
                            half_dict_rule['tbgs_h'] = half_guest_data[3]
                            if pt is 3:
                                half_dict['tbgb_h'] = round(float(half_guest_data[5]) - 1, 2)
                                half_dict['tbgs_h'] = round(float(half_guest_data[7]) - 1, 2)
                            else:
                                half_dict['tbgb_h'] = half_guest_data[5]
                                half_dict['tbgs_h'] = half_guest_data[7]
                        except:
                            half_dict_rule['tbgb_h'] = ""
                            half_dict['tbgb_h'] = ""

                            half_dict_rule['tbgs_h'] = ""
                            half_dict['tbgs_h'] = ""

                item['league_id'] = league_id
                item['league_name'] = league_name
                item['pt'] = pt
                item['game_id'] = game_id
                item['team_home'] = team_home
                item['team_guest'] = team_guest
                item['number'] = number
                item['data_game'] = data_game
                item['time_game'] = time_game
                item['corner_ball'] = corner_ball
                # 波胆
                item['bodan_data'] = bodan_data
                # 早盘入球数单双
                item["odd_even_odd"] = odd_even_odd
                item["odd_even_even"] = odd_even_even
                item["half_odd_even_odd"] = half_odd_even_odd
                item["half_odd_even_even"] = half_odd_even_even

                item['total_goal'] = total_goal

                item["half_capot_home"] = half_capot_home
                item["half_capot_guest"] = half_capot_guest
                item["half_capot_dogfall"] = half_capot_dogfall

                item["capot_home"] = capot_home
                item["capot_guest"] = capot_guest
                item["capot_dogfall"] = capot_dogfall

                item["first_last_ball"] = first_last_ball

                item["half_full"] = half_full

                item["half_size_guest"] = half_size_guest
                item["half_size_guest_rule"] = half_size_guest_rule
                item["half_size_home"] = half_size_home
                item["half_size_home_rule"] = half_size_home_rule
                # 全场大小
                item["size_guest"] = size_guest
                item["size_guest_rule"] = size_guest_rule
                item["size_home"] = size_home
                item["size_home_rule"] = size_home_rule
                # 上半场让球
                item["half_concede_home_rule"] = half_concede_home_rule
                item["half_concede_home"] = half_concede_home
                item["half_concede_guest_rule"] = half_concede_guest_rule
                item["half_concede_guest"] = half_concede_guest
                # 全场让球
                item["concede_guest"] = concede_guest
                item["concede_guest_rule"] = concede_guest_rule
                item["concede_home"] = concede_home
                item["concede_home_rule"] = concede_home_rule

                item['full_data'] = full_dict
                item['half_data'] = half_dict
                item['full_data_rule'] = full_dict_rule
                item['half_data_rule'] = half_dict_rule
                # item['zuqiu'] = zuqiu
                yield item