sports_scrapy/hg3535/spiders/roll_zuqiu.py

# -*- coding: utf-8 -*-
# import copy
import datetime
import json
# import jsonpath
import time
import redis
import scrapy
from scrapy.http import Request
from ..settings import R_HOST, R_PASSWORD, R_POST, R_DB
from ..items import Roll_Zuqiu


class ZuqiuSpider(scrapy.Spider):
    name = 'roll_zuqiu'
    to_day = datetime.datetime.now()
    allowed_domains = ['hg3535z.com']
    custom_settings = {
        "ITEM_PIPELINES":{
            'hg3535.pipeline.roll_zuqiu.Roll_Zuqiupipeline': 200,
        },
        # 'LOG_LEVEL': 'DEBUG',
        # 'LOG_FILE': "../hg3535/log/roll_zuqiu_{}_{}_{}.log".format(to_day.year, to_day.month,to_day.day)
    }
    start_urls = ['https://odata.yonghuai5515.com/odds6i/d/getodds/zh-cn/sid/1/pt/4/ubt/am/pn/0/sb/2/dc/null/pid/0']
    rls = redis.Redis(host=R_HOST, port=R_POST, db=R_DB, password=R_PASSWORD)

    def parse(self, response):
        responses = json.loads(response.text)
        try:
            datas = responses["i-ot"]
        except:
            print("暂无滚球篮球数据")
            return
        if datas:
            for data in datas:
                egs = data.get('egs')
                if egs:
                    for es in egs:
                        es = es['es']
                        for e in es:
                            match_id = e['k']
                            # up_time = str(time.time()) + '&' + 'zq'
                            # self.rls.sadd('hg3535.gunqiu.ids', match_id)
                            creation = time.strftime('%Y-%m-%d', time.localtime())
                            # self.rls.hset('hg3535.ball.ids', match_id, up_time)
                            self.rls.hsetnx('hg3535.GunQiu.ids', match_id, creation)
                            url = 'https://odata.yonghuai5515.com/odds6i/d/getamodds/zh-cn/eid/{}/iip/true/ubt/am/isp/false'.format(match_id)
                            yield Request(url=url, callback=self.parse_each, dont_filter=True)
                else:
                    print('足球滚球数据为空')
        else:
            print("暂无滚球足球数据")
            return

    def parse_each(self, response):
        # 球队进球数 大小
        try:
            datas = json.loads(response.text)['eg']['es']
            league_id = json.loads(response.text)['eg']['c']['k']
            # 联赛名
            league_name = json.loads(response.text)['eg']['c']['n']
        except:
            datas = ""
            league_id = ""
            # 联赛名
            league_name = ""

        if datas:
            item = Roll_Zuqiu()
            full_dict = {}
            half_dict = {}
            full_dict_rule = {}
            half_dict_rule = {}
            horn_ou_dict = {}
            horn_ou_dict_rule = {}
            horn_oe_dict = {}
            horn_oe_dict_rule = {}
            horn_team = {}
            for data in datas:
                try:
                    ctid = data['pci']['ctid']
                except:
                    ctid = ""
                if ctid == 0:
                    # 比赛id
                    game_id = str(data['k'])
                    # 球队1
                    team_home = data['i'][0]
                    # 球队2
                    team_guest = data['i'][1]
                    # 数量(97>)
                    number = data['i'][2]
                    # 下半场
                    half_way = data['i'][12]
                    # 角球或者其他
                    # corner_ball = data['pci'].get('ctn', "")
                    # 日期
                    data_game = data['i'][4]
                    # 开赛时间
                    time_game = data['i'][5]
                    score_home = data['i'][10]  # 队一分数
                    score_guest = data['i'][11] # 队二分数

# 让球------------------------------------------------------------------------------------------------------------------
                    try:
                        concedes = data['o']['ah']['v']
                        new_concedes = [concedes[i] for i in range(len(concedes)) if i % 2 == 1]
                        concede_homes = [new_concedes[i] for i in range(len(new_concedes)) if i % 2 == 0]
                        concede_home_rule = [concede_homes[i] for i in range(len(concede_homes)) if i % 2 == 0]
                        concede_home = [concede_homes[i] for i in range(len(concede_homes)) if i % 2 is 1]

                        concede_guests = [new_concedes[i] for i in range(len(new_concedes)) if i % 2 is 1]
                        concede_guest_rule = [concede_guests[i] for i in range(len(concede_guests)) if i % 2 == 0]
                        concede_guest = [concede_guests[i] for i in range(len(concede_guests)) if i % 2 is 1]
                    except:
                        concede_guest = ""
                        concede_guest_rule = ""
                        concede_home = ""
                        concede_home_rule = ""

# 上半场让球half_concede-------------------------------------------------------------------------------------------------
                    try:
                        half_concedes = data['o']['ah1st']['v']
                        new_half_concedes = [half_concedes[i] for i in range(len(half_concedes)) if i % 2 is 1]
                        new_half_concede_homes = [new_half_concedes[i] for i in range(len(new_half_concedes)) if
                                                  i % 2 is 0]
                        half_concede_guest_rule = [new_half_concede_homes[i] for i in range(len(new_half_concede_homes)) if i % 2 == 0]
                        # concede_home
                        half_concede_home = [new_half_concede_homes[i] for i in range(len(new_half_concede_homes)) if i % 2 is 1]
                        half_concede_guests = [new_half_concedes[i] for i in range(len(new_half_concedes)) if i % 2 is 1]
                        half_concede_home_rule = [half_concede_guests[i] for i in range(len(half_concede_guests)) if i % 2 == 0]
                        # concede_guest
                        half_concede_guest = [half_concede_guests[i] for i in range(len(half_concede_guests)) if
                                              i % 2 is 1]
                    except:
                        half_concede_home_rule = ""
                        half_concede_home = ""
                        half_concede_guest_rule = ""
                        half_concede_guest = ""

# 全场大小size 进球大小---------------------------------------------------------------------------------------------------
                    try:
                        sizes = data['o']['ou']['v']
                        new_sizes = [sizes[i] for i in range(len(sizes)) if i % 2 is 1]
                        size_homes = [new_sizes[i] for i in range(len(new_sizes)) if i % 2 is 0]
                        # size_home_rule
                        size_home_rule = [size_homes[i] for i in range(len(size_homes)) if i % 2 is 0]
                        # size_home
                        size_home = [size_homes[i] for i in range(len(size_homes)) if i % 2 is 1]

                        size_guests = [new_sizes[i] for i in range(len(new_sizes)) if i % 2 is 1]
                        # size_guest_rule
                        size_guest_rule = [size_guests[i] for i in range(len(size_guests)) if i % 2 is 0]
                        # csize_guest
                        size_guest = [size_guests[i] for i in range(len(size_guests)) if i % 2 is 1]
                    except:
                        size_guest = ""
                        size_guest_rule = ""
                        size_home = ""
                        size_home_rule = ""

# 上半场大小 进球大小 half_size-------------------------------------------------------------------------------------------
                    try:
                        half_sizes = data['o']['ou1st']['v']
                        new_half_sizes = [half_sizes[i] for i in range(len(half_sizes)) if i % 2 is 1]
                        half_size_homes = [new_half_sizes[i] for i in range(len(new_half_sizes)) if i % 2 is 0]
                        # size_home_rule
                        half_size_home_rule = [half_size_homes[i] for i in range(len(half_size_homes)) if i % 2 is 0]
                        # half_size_home 主队

                        half_size_home = [half_size_homes[i] for i in range(len(half_size_homes)) if i % 2 is 1]
                        half_size_guests = [new_half_sizes[i] for i in range(len(new_half_sizes)) if i % 2 is 1]
                        # half_size_guest_rule 客队
                        half_size_guest_rule = [half_size_guests[i] for i in range(len(half_size_guests)) if i % 2 is 0]
                        # half_size_guest
                        half_size_guest = [half_size_guests[i] for i in range(len(half_size_guests)) if
                                           i % 2 is 1]
                    except:
                        half_size_guest = ""
                        half_size_guest_rule = ""
                        half_size_home = ""
                        half_size_home_rule = ""

# 全场总进球 total_goal--------------------------------------------------------------------------------------------------
                    total_goal = {}
                    try:
                        total_goals = data['o']['tg']['v']
                        new_total_goals = [total_goals[i] for i in range(len(total_goals)) if i % 2 is 1]
                        try:
                            total_goal["tg0"] = round(float(new_total_goals[0]) - 1, 2)
                        except:
                            total_goal["tg0"] = 0
                        try:
                            total_goal["tg1"] = round(float(new_total_goals[1]) - 1, 2)
                        except:
                            total_goal["tg1"] = 0
                        try:
                            total_goal["tg2"] = round(float(new_total_goals[2]) - 1, 2)
                        except:
                            total_goal["tg2"] = 0
                        try:
                            total_goal["tg3"] = round(float(new_total_goals[3]) - 1, 2)
                        except:
                            total_goal["tg3"] = 0
                    except:
                        total_goal["tg0"] = ""
                        total_goal["tg1"] = ""
                        total_goal["tg2"] = ""
                        total_goal["tg3"] = ""

# 总进球上半场 half_total_goal-------------------------------------------------------------------------------------------
                    try:
                        half_total_goals = data['o']['tg1st']['v']
                        new_half_total_goals = [half_total_goals[i] for i in range(len(half_total_goals)) if i % 2 is 1]
                        try:
                            total_goal["tg0_h"] = round(float(new_half_total_goals[0]) - 1, 2)
                        except:
                            total_goal["tg0_h"] = 0
                        try:
                            total_goal["tg1_h"] = round(float(new_half_total_goals[1]) - 1, 2)
                        except:
                            total_goal["tg1_h"] = 0
                        try:
                            total_goal["tg2_h"] = round(float(new_half_total_goals[2]) - 1, 2)
                        except:
                            total_goal["tg2_h"] = 0
                        try:
                            total_goal["tg3_h"] = round(float(new_half_total_goals[3]) - 1, 2)
                        except:
                            total_goal["tg3_h"] = 0
                    except:
                        total_goal["tg0_h"] = ""
                        total_goal["tg1_h"] = ""
                        total_goal["tg2_h"] = ""
                        total_goal["tg3_h"] = ""

# 早盘 半场/全场---------------------------------------------------------------------------------------------------------
                    half_full = {}
                    new_lists = ["hfhh", "hfhd", "hfhg", "hfdh", "hfdd", "hfdg", "hfgh", "hfgd", "hfgg"]
                    # 早盘 半场/全场
                    try:
                        half_fulls = data['o']['hf']['v']
                        new_half_fulls = [half_fulls[i] for i in range(len(half_fulls)) if i % 2 is 1]
                        for index, value in enumerate(new_lists):
                            try:
                                half_full[value] = round(float(new_half_fulls[index]) - 1, 2)
                            except:
                                half_full[value] = ""
                    except:
                        for index, value in enumerate(new_lists):
                            half_full[value] = ""

# 早盘 最先/最后进球 最先进球 ---------------------------------------------------------------------------------------------
                    first_last_ball = {}
                    try:
                        first_balls = data['o']['ttslast']['v']
                        first_ball = [first_balls[i] for i in range(len(first_balls)) if i % 2 is 1]
                        try:
                            first_last_ball['flbfh'] = round(float(first_ball[0]) - 1, 2)
                        except:
                            first_last_ball['flbfh'] = 0
                        try:
                            first_last_ball['flbfg'] = round(float(first_ball[1]) - 1, 2)
                        except:
                            first_last_ball['flbfg'] = 0

                    except:
                        pass

                    # 早盘 最先/最后进球 最后进球

                    try:
                        last_balls = data['o']['tts1st']['v']
                        last_ball = [last_balls[i] for i in range(len(last_balls)) if i % 2 is 1]
                        try:
                            first_last_ball['flblh'] = round(float(last_ball[0]) - 1, 2)
                        except:
                            first_last_ball['flblh'] = 0
                        try:
                            first_last_ball['flblg'] = round(float(last_ball[1]) - 1, 2)
                        except:
                            first_last_ball['flblg'] = 0
                        try:
                            first_last_ball['flbn'] = round(float(last_ball[2]) - 1, 2)
                        except:
                            first_last_ball['flbn'] = 0
                    except:
                        pass

# 全场独赢capot ---------------------------------------------------------------------------------------------------------

                    try:
                        capots = data['o']['1x2']['v']
                        new_capots = [capots[i] for i in range(len(capots)) if i % 2 is 1]
                        try:
                            capot_home = round(float(new_capots[0]) - 1, 2)
                        except:
                            capot_home = ""
                        try:
                            capot_guest = round(float(new_capots[1]) - 1, 2)
                        except:
                            capot_guest = ""
                        try:
                            capot_dogfall = round(float(new_capots[2]) - 1, 2)
                        except:
                            capot_dogfall = ""
                    except:
                        capot_home = ""
                        capot_guest = ""
                        capot_dogfall = ""

                    # 上半场独赢capot
                    try:
                        half_capots = data['o']['1x21st']['v']
                        new_half_capots = [half_capots[i] for i in range(len(half_capots)) if i % 2 is 1]
                        try:
                            half_capot_home = round(float(new_half_capots[0]) - 1, 2)
                        except:
                            half_capot_home = ""
                        try:
                            half_capot_guest = round(float(new_half_capots[1]) - 1, 2)
                        except:
                            half_capot_guest = ""
                        try:
                            half_capot_dogfall = round(float(new_half_capots[2]) - 1, 2)
                        except:
                            half_capot_dogfall = ''
                    except:
                        half_capot_home = ""
                        half_capot_guest = ""
                        half_capot_dogfall = ""

# 全场入球：单/双 odd_even------------------------------------------------------------------------------------------------
                    try:
                        odd_evens = data['o']['oe']['v']
                        new_odd_evens = [odd_evens[i] for i in range(len(odd_evens)) if i % 2 is 1]
                        odd_even_odd = new_odd_evens[0]
                        odd_even_even = new_odd_evens[1]
                    except:
                        odd_even_odd = ""
                        odd_even_even = ""
                    # 半场入球：单/双 half_odd_even
                    try:
                        half_odd_evens = data['o']['oe1st']['v']
                        new_half_odd_evens = [half_odd_evens[i] for i in range(len(half_odd_evens)) if i % 2 is 1]
                        half_odd_even_odd = new_half_odd_evens[0]
                        half_odd_even_even = new_half_odd_evens[1]
                    except:
                        half_odd_even_odd = ""
                        half_odd_even_even = ""

                    bodan_data = {}
# 波胆------------------------------------------------------------------------------------------------------------------
                    try:
                        bodans = data['o']['cs']['v']
                        one_list = ["b10", "b20", "b21", "b30", "b31", "b32", "b40", "b41", "b42", "b43"]
                        two_list = ["b01", "b02", "b12", "b03", "b13", "b23", "b04", "b14", "b24", "b34"]
                        three_list = ["b00", "b11", "b22", "b33", "b44"]
                        new_bodans = [bodans[i] for i in range(len(bodans)) if i % 2 is 1]
                        new_bodan = new_bodans[0:20]
                        # 主队bodan_home
                        bodan_home = [new_bodan[i] for i in range(len(new_bodan)) if i % 2 is 0]
                        for index, t in enumerate(one_list):
                            try:
                                bodan_data[t] = round(float(bodan_home[index]) - 1, 2)
                            except:
                                bodan_data[t] = 0
                        # 客队bodan_guest
                        bodan_guest = [new_bodan[i] for i in range(len(new_bodan)) if i % 2 is 1]
                        for y, z in enumerate(two_list):
                            try:
                                bodan_data[z] = round(float(bodan_guest[y]) - 1, 2)
                            except:
                                bodan_data[z] = 0
                        bodan_dogfall = new_bodans[-6:-1]
                        for a, b in enumerate(three_list):
                            try:
                                bodan_data[b] = round(float(bodan_dogfall[a]) - 1, 2)
                            except:
                                bodan_data[b] = 0
                        # 其他bodan_other
                        try:
                            bodan_data['bo'] = round(float(new_bodans[-1]) - 1, 2)
                        except:
                            bodan_data['bo'] = 0
                    except:
                        pass

                    try:
                        half_bodans = data['o']['cs1st']['v']
                        new_half_bodans = [half_bodans[i] for i in range(len(half_bodans)) if i % 2 is 1]
                        new_one = ["b10_h", "b20_h", "b21_h", "b30_h", "b31_h", "b32_h"]
                        new_two = ["b01_h", "b02_h", "b12_h", "b03_h", "b13_h", "b23_h"]
                        new_three = ["b00_h", "b11_h", "b22_h", "b33_h"]
                        halfbodan = new_half_bodans[0:12]
                        half_bodan_home = [halfbodan[i] for i in range(len(halfbodan)) if i % 2 is 0]
                        # 队1
                        for index, t in enumerate(new_one):
                            try:
                                bodan_data[t] = round(float(half_bodan_home[index]) - 1, 2)
                            except:
                                bodan_data[t] = 0
                        # 队2
                        half_bodan_guest = [halfbodan[i] for i in range(len(halfbodan)) if i % 2 is 1]
                        for y, z in enumerate(new_two):
                            try:
                                bodan_data[z] = round(float(half_bodan_guest[y]) - 1, 2)
                            except:
                                bodan_data[z] = 0
                        # 和/平
                        half_bodan_dogfall = new_half_bodans[-5:-1]
                        for a, b in enumerate(new_three):
                            try:
                                bodan_data[b] = round(float(half_bodan_dogfall[a]) - 1, 2)
                            except:
                                bodan_data[b] = 0
                        # 其他
                        try:
                            bodan_data['bo_h'] = round(float(new_half_bodans[-1]) - 1, 2)
                        except:
                            bodan_data['bo_h'] = 0
                    except Exception as e:
                        pass
                if ctid == 1:
                    # 角球数据
                    horn_team['horn_home'] = data['i'][0]
                    horn_team['horn_guest'] = data['i'][1]
                    horn_team['horn_scoreh'] = data['i'][10]
                    horn_team['horn_scoreg'] = data['i'][11]
                    horn_team['horn_id'] = str(data['k'])
                    try:
                        horn_ou = data['o']['ou']['v']
                    except:
                        horn_ou = ''
                    if horn_ou:
                        horn_ou_dict['cbgb'] = horn_ou[5]
                        horn_ou_dict['cbgs'] = horn_ou[7]
                        horn_ou_dict_rule['cbgb'] = horn_ou[1]
                        horn_ou_dict_rule['cbgs'] = horn_ou[3]
                    try:
                        horn_ou1st = data['o']['ou1st']['v']
                    except:
                        horn_ou1st = ''
                    if horn_ou1st:
                        horn_ou_dict['cbgb_h'] = horn_ou1st[5]
                        horn_ou_dict['cbgs_h'] = horn_ou1st[7]
                        horn_ou_dict_rule['cbgb_h'] = horn_ou1st[1]
                        horn_ou_dict_rule['cbgs_h'] = horn_ou1st[3]
                    try:
                        horn_oe = data['o']['oe']['v']
                    except:
                        horn_oe = ''
                    if horn_oe:
                        horn_oe_dict['cbtss'] = horn_oe[1]
                        horn_oe_dict['cbtsd'] = horn_oe[3]
                        horn_oe_dict_rule['cbtss'] = '单'
                        horn_oe_dict_rule['cbtsd'] = '双'
                    try:
                        horn_oe1st = data['o']['oe1st']['v']
                    except:
                        horn_oe1st = ''
                    if horn_oe1st:
                        horn_oe_dict['cbtss_h'] = horn_oe1st[1]
                        horn_oe_dict['cbtsd_h'] = horn_oe1st[3]
                        horn_oe_dict_rule['cbtss_h'] = '单'
                        horn_oe_dict_rule['cbtsd_h'] = '双'
                if ctid == 12:
                    try:
                        home_data = data['o']['ou']['v']
                        # 球队进球 大条件
                        full_dict_rule['tbhb'] = home_data[1]
                        # 球队进球小条件
                        full_dict_rule['tbhs'] = home_data[3]
                        # 球队进球大赔率
                        full_dict['tbhb'] = home_data[5]
                        # 球队进球小赔率
                        full_dict['tbhs'] = home_data[7]
                    except:
                        full_dict['tbhs'] = ""
                        full_dict_rule['tbhs'] = ""
                        full_dict['tbhb'] = ""
                        full_dict_rule['tbhb'] = ""
                    try:
                        half_home_data = data['o']['ou1st']['v']
                        half_dict_rule['tbhb_h'] = half_home_data[1]
                        half_dict_rule['tbhs_h'] = half_home_data[3]
                        half_dict['tbhb_h'] = half_home_data[5]
                        half_dict['tbhs_h'] = half_home_data[7]

                    except:
                        half_dict_rule['tbhb_h'] = ""
                        half_dict['tbhb_h'] = ""
                        half_dict_rule['tbhs_h'] = ""
                        half_dict['tbhs_h'] = ""
                if ctid == 13:
                    try:
                        guest_data = data['o']['ou']['v']
                        full_dict_rule['tbgb'] = guest_data[1]
                        full_dict_rule['tbgs'] = guest_data[3]
                        full_dict['tbgb'] = guest_data[5]
                        full_dict['tbgs'] = guest_data[7]
                    except:
                        full_dict_rule['tbgb'] = ""
                        full_dict['tbgb'] = ""
                        full_dict_rule['tbgs'] = ""
                        full_dict['tbgs'] = ''
                    try:
                        half_guest_data = data['o']['ou1st']['v']
                        half_dict_rule['tbgb_h'] = half_guest_data[1]
                        half_dict_rule['tbgs_h'] = half_guest_data[3]
                        half_dict['tbgb_h'] = half_guest_data[5]
                        half_dict['tbgs_h'] = half_guest_data[7]
                    except:
                        half_dict_rule['tbgb_h'] = ""
                        half_dict['tbgb_h'] = ""
                        half_dict_rule['tbgs_h'] = ""
                        half_dict['tbgs_h'] = ""
            item['league_id'] = league_id
            item['league_name'] = league_name
            item['game_id'] = game_id
            item['team_home'] = team_home
            item['team_guest'] = team_guest
            item['number'] = number
            item['data_game'] = data_game
            item['time_game'] = time_game
            # 波胆
            item['bodan_data'] = bodan_data
            # 早盘入球数单双
            item["odd_even_odd"] = odd_even_odd
            item["odd_even_even"] = odd_even_even
            item["half_odd_even_odd"] = half_odd_even_odd
            item["half_odd_even_even"] = half_odd_even_even

            item['total_goal'] = total_goal

            item["half_capot_home"] = half_capot_home
            item["half_capot_guest"] = half_capot_guest
            item["half_capot_dogfall"] = half_capot_dogfall

            item["capot_home"] = capot_home
            item["capot_guest"] = capot_guest
            item["capot_dogfall"] = capot_dogfall

            item["first_last_ball"] = first_last_ball

            item["half_full"] = half_full

            item["half_size_guest"] = half_size_guest
            item["half_size_guest_rule"] = half_size_guest_rule
            item["half_size_home"] = half_size_home
            item["half_size_home_rule"] = half_size_home_rule
            # 全场大小
            item["size_guest"] = size_guest
            item["size_guest_rule"] = size_guest_rule
            item["size_home"] = size_home
            item["size_home_rule"] = size_home_rule
            # 上半场让球
            item["half_concede_home_rule"] = half_concede_home_rule
            item["half_concede_home"] = half_concede_home
            item["half_concede_guest_rule"] = half_concede_guest_rule
            item["half_concede_guest"] = half_concede_guest
            # 全场让球
            item["concede_guest"] = concede_guest
            item["concede_guest_rule"] = concede_guest_rule
            item["concede_home"] = concede_home
            item["concede_home_rule"] = concede_home_rule
            item['score_home'] = score_home
            item['score_guest'] = score_guest
            item['full_data'] = full_dict
            item['half_data'] = half_dict
            item['full_data_rule'] = full_dict_rule
            item['half_data_rule'] = half_dict_rule
            item['half_way'] = half_way

            item['horn_ou_dict'] = horn_ou_dict
            item['horn_ou_dict_rule'] = horn_ou_dict_rule
            item['horn_oe_dict'] = horn_oe_dict
            item['horn_oe_dict_rule'] = horn_oe_dict_rule
            item['horn_team'] = horn_team
            yield item