sports_scrapy/hg3535/spiders/bangqiu.py

# -*- coding: utf-8 -*-
import copy
import datetime
import json

import scrapy
from scrapy.http import Request

from ..items import Bangqiu


class BqrangqiuSpider(scrapy.Spider):
    name = 'bangqiu'
    to_day = datetime.datetime.now()
    allowed_domains = ['hg3535z.com']
    custom_settings = {
        "ITEM_PIPELINES": {
            "hg3535.pipeline.bangqiu.Bangqiupipeline": 300,
        },
        # 'LOG_LEVEL': 'DEBUG',
        # 'LOG_FILE': "../hg3535/log/bangqiu_{}_{}_{}.log".format(to_day.year, to_day.month, to_day.day)
    }

    def start_requests(self):
        for y in range(1, 4):
            # url = 'https://hg3535z.com/odds2/d/getodds?sid=4&pt=' + str(y) + '&ubt=am&pn=0&sb=2&dc=null&pid=0'
            url = 'https://odata.jiushan6688.com/odds6i/d/getodds/zh-cn/sid/4/pt/' + str(
                y) + '/ubt/am/pn/0/sb/2/dc/null/pid/0'
            yield scrapy.Request(url=url, callback=self.parse, meta={'pt': y}, dont_filter=True)

    def parse(self, response):
        if response.text:
            try:
                datas = json.loads(response.text).get('n-ot', "").get('egs', "")
            except:
                datas = ""
            try:
                pt = copy.copy(response.meta['pt'])
            except:
                pt = 0
            if datas:
                for result in datas:
                    new_results = result['es']
                    for new_result in new_results:
                        game_id = str(new_result['i'][16])
                        if pt == 1:
                            # url = "https://hg3535z.com/odds2/d/getamodds?eid=" + game_id + "&iip=false&ubt=am&isp=false"
                            url = "https://odata.jiushan6688.com/odds6i/d/getamodds/zh-cn/eid/" + game_id + "/iip/false/ubt/am/isp/false"
                            yield Request(url=url, callback=self.parse_each, meta={'pt': pt}, dont_filter=True)
                        if pt == 2:
                            # url = "https://hg3535z.com/odds2/d/getamodds?eid=" + game_id + "&iip=false&ubt=am&isp=false"
                            url = "https://odata.jiushan6688.com/odds6i/d/getamodds/zh-cn/eid/" + game_id + "/iip/false/ubt/am/isp/false"
                            yield Request(url=url, callback=self.parse_each, meta={'pt': pt}, dont_filter=True)
                        if pt == 3:
                            # url = "https://hg3535z.com/odds2/d/getamodds?eid=" + game_id + "&iip=false&ubt=am&isp=true"
                            url = "https://odata.jiushan6688.com/odds6i/d/getamodds/zh-cn/eid/" + game_id + "/iip/false/ubt/am/isp/true"
                            yield Request(url=url, callback=self.parse_each, meta={'pt': pt}, dont_filter=True)

    def parse_each(self, response):
        try:
            new_datas = json.loads(response.text)['eg']
            # print(new_datas)
            pt = response.meta['pt']
            # print(pt)
            bangqiu = json.loads(response.text)['i'][31]
            # print(bangqiu)
        except:
            new_datas = ""
            pt = 0
            bangqiu = ""
            # print(bangqiu)
        if new_datas:
            # 联赛id
            league_id = new_datas["c"]["k"]
            # 联赛名
            league_name = new_datas["c"]["n"]
            new_data = new_datas["es"]
            item = Bangqiu()
            for result in new_data:
                if result['pci']["ctid"] is 0:

                    # 比赛id
                    game_id = str(result['k'])
                    # 球队1
                    team_home = result['i'][0]
                    # 球队2
                    team_guest = result['i'][1]
                    # 数量(97>)
                    number = result['i'][2]
                    # 比赛状态
                    zhuangtai = result['i'][3]
                    # 日期
                    data_game = result['i'][4]
                    # 开赛时间
                    time_game = result['i'][5]
                    # 队1分数
                    score_home1 = result['i'][10]
                    # 队2分数
                    score_guest1 = result['i'][11]
                    # 第几盘
                    jijie = result['i'][12]
                    # 球队得分
                    qiudui = result['pci'].get('ctn', "")
                    # --------------------------------------------------让球分割线-----------------------------------------------------------
                    concedes_dict = {}
                    concedes_dict_rule = {}
                    try:
                        concedes = result['o']["ah"]["v"]
                        concedes_dict_rule['coh'] = concedes[1]
                        concedes_dict_rule['cog'] = concedes[3]
                        if pt is 3:
                            concedes_dict['coh'] = float(concedes[5]) - 1
                            concedes_dict['cog'] = float(concedes[7]) - 1
                        else:
                            concedes_dict['coh'] = float(concedes[5])
                            concedes_dict['cog'] = float(concedes[7])
                    except:
                        concedes_dict_rule['coh'] = ""
                        concedes_dict['coh'] = ""
                        concedes_dict_rule['cog'] = ""
                        concedes_dict['cog'] = ""
                    # 让球 前5局-------------------------------------------------------------------------------------------------------------
                    try:
                        concedes = result['o']["ahf5in"]["v"]
                        concedes_dict_rule['coh_5'] = concedes[1]
                        concedes_dict_rule['cog_5'] = concedes[3]
                        if pt is 3:
                            concedes_dict['coh_5'] = float(concedes[5]) - 1
                            concedes_dict['cog_5'] = float(concedes[7]) - 1
                        else:
                            concedes_dict['coh_5'] = concedes[5]
                            concedes_dict['cog_5'] = concedes[7]
                    except:
                        concedes_dict_rule['coh_5'] = ""
                        concedes_dict['coh_5'] = ""
                        concedes_dict_rule['cog_5'] = ""
                        concedes_dict['cog_5'] = ""
                    # ---------------------------------------------------独赢----------------------------------------------------------------
                    capots_dict = {}
                    try:
                        capots = result['o']["ml"]["v"]
                        capots_dict['ch'] = capots[1]
                        capots_dict['cg'] = capots[3]

                    except:
                        capots_dict['ch'] = ""
                        capots_dict['cg'] = ""
                    # ---------------------------------------------------总得分:大/小--------------------------------------------------------

                    total_size_dict = {}
                    total_size_dict_rule = {}
                    try:
                        total_sizes = result['o']["ou"]["v"]
                        total_size_dict_rule['tnb'] = total_sizes[1]
                        total_size_dict_rule['tns'] = total_sizes[3]
                        if pt is 3:
                            total_size_dict['tnb'] = float(total_sizes[5]) - 1
                            total_size_dict['tns'] = float(total_sizes[7]) - 1
                        else:
                            total_size_dict['tnb'] = total_sizes[5]
                            total_size_dict['tns'] = total_sizes[7]
                    except:
                        total_size_dict_rule['tnb'] = ""
                        total_size_dict['tnb'] = ""
                        total_size_dict_rule['tns'] = ""
                        total_size_dict['tns'] = ""
                    # 总得分: 大/小 前5局-----------------------------------------------------------------------------------------------------
                    try:
                        total_sizes = result['o']["ouf5in"]["v"]
                        total_size_dict_rule['tnb_5'] = total_sizes[1]
                        total_size_dict_rule['tns_5'] = total_sizes[3]
                        if pt is 3:
                            total_size_dict['tnb_5'] = float(total_sizes[5]) - 1
                            total_size_dict['tns_5'] = float(total_sizes[7]) - 1
                        else:
                            total_size_dict['tnb_5'] = total_sizes[5]
                            total_size_dict['tns_5'] = total_sizes[7]
                    except:
                        total_size_dict_rule['tnb_5'] = ""
                        total_size_dict['tnb_5'] = ""
                        total_size_dict_rule['tns_5'] = ""
                        total_size_dict['tns_5'] = ""

                    # ------------------------------------------------------总得分:单/双-----------------------------------------------------
                    odd_evens_dict = {}
                    odd_evens_dict_rule = {}
                    try:
                        odd_evens = result['o']["oe"]["v"]
                        # 上半场 主队让球条件
                        odd_evens_dict_rule['tss'] = "单"
                        odd_evens_dict_rule['tsd'] = "双"
                        if pt is 3:
                            odd_evens_dict['tss'] = float(odd_evens[1]) - 1
                            odd_evens_dict['tsd'] = float(odd_evens[3]) - 1
                        else:
                            odd_evens_dict['tss'] = odd_evens[1]
                            odd_evens_dict['tsd'] = odd_evens[3]

                    except:
                        odd_evens_dict['tss'] = ""
                        odd_evens_dict['tsd'] = ""
                        odd_evens_dict_rule['tss'] = ""
                        odd_evens_dict_rule['tsd'] = ""
                    # 联赛id
                    item['league_id'] = league_id
                    # 联赛名
                    item['league_name'] = league_name
                    # 比赛id
                    item['game_id'] = game_id
                    # 球队1
                    item['team_home'] = team_home
                    # 球队2
                    item['team_guest'] = team_guest
                    # 数量(97>)
                    item['number'] = number
                    # 比赛状态
                    item['zhuangtai'] = zhuangtai
                    # 日期
                    item['data_game'] = data_game
                    # 开赛时间
                    item['time_game'] = time_game
                    # 队1分数
                    item['score_home'] = score_home1
                    # 队2分数
                    item['score_guest'] = score_guest1
                    # 第几节
                    item['jijie'] = jijie
                    item['pt'] = pt
                    # 球队得分
                    item['qiudui'] = qiudui
                    # 让球
                    item['concedes_dict'] = concedes_dict
                    item['concedes_dict_rule'] = concedes_dict_rule
                    # 独赢
                    item['capots_dict'] = capots_dict
                    # 总得分：大/小
                    item['total_size_dict'] = total_size_dict
                    item['total_size_dict_rule'] = total_size_dict_rule
                    # 总得分：单/双
                    item['odd_evens_dict'] = odd_evens_dict
                    item['odd_evens_dict_rule'] = odd_evens_dict_rule
                    item['bangqiu'] = bangqiu
                    yield item

            #     if result['pci']['ctid'] is 208:
            #         """球队得分大/小  主队"""
            #         team_score_home_dict = {}
            #         team_score_home_rule_dict = {}
            #         try:
            #             team_score_home = result['o']['ou']['v']
            #             team_score_home_rule_dict['team_score_big'] = team_score_home[1]
            #             team_score_home_rule_dict['team_score_small'] = team_score_home[3]
            #             if pt is 3:
            #                 team_score_home_dict['team_score_big'] = float(team_score_home[5]) - 1
            #                 team_score_home_dict['team_score_small'] = float(team_score_home[7]) - 1
            #             else:
            #                 team_score_home_dict['team_score_big'] = team_score_home[5]
            #                 team_score_home_dict['team_score_small'] = team_score_home[7]
            #         except:
            #             team_score_home_dict['team_score_big'] = ''
            #             team_score_home_rule_dict['team_score_big'] = ''
            #             team_score_home_dict['team_score_small'] = ''
            #             team_score_home_rule_dict['team_score_small'] = ''
            #         # 球队得分大/小  主队
            #         item['team_score_home_dict'] = team_score_home_dict
            #         item['team_score_home_rule_dict'] = team_score_home_rule_dict
            #
            #     if result['pci']['ctid'] is 209:
            #         """球队得分大/小  客队"""
            #         team_score_guest_dict = {}
            #         team_score_guest_rule_dict = {}
            #         try:
            #             team_score_guest = result['o']['ou']['v']
            #             team_score_guest_rule_dict['team_score_big'] = team_score_guest[1]
            #             team_score_guest_rule_dict['team_score_small'] = team_score_guest[3]
            #             if pt is 3:
            #                 team_score_guest_dict['team_score_big'] = float(team_score_guest[5]) - 1
            #                 team_score_guest_dict['team_score_small'] = float(team_score_guest[7]) - 1
            #             else:
            #                 team_score_guest_dict['team_score_big'] = team_score_guest[5]
            #                 team_score_guest_dict['team_score_small'] = team_score_guest[7]
            #         except:
            #             team_score_guest_dict['team_score_big'] = ''
            #             team_score_guest_rule_dict['team_score_big'] = ''
            #             team_score_guest_dict['team_score_small'] = ''
            #             team_score_guest_rule_dict['team_score_small'] = ''
            #         # 球队得分大/小  客队
            #         item['team_score_guest_dict'] = team_score_guest_dict
            #         item['team_score_guest_rule_dict'] = team_score_guest_rule_dict
            #
            # yield item