sports_scrapy/scrapy_yzd/scrapy_yzd/spiders/hg3535_bangqiu.py

#!/usr/bin/env python
# -*- coding: utf-8 -*-
import jsonpath

__author__ = 'ethereal'
__mtime__ = '2019/4/16'


# -*- coding: utf-8 -*-
import json

import scrapy
from scrapy.http import Request
from scrapy_yzd.items import  Bangqiu



class BqrangqiuSpider(scrapy.Spider):
    name = 'bangqiu'
    allowed_domains = ['hg3535z.com']
    custom_settings = {
        "ITEM_PIPELINES": {
            "scrapy_yzd.pipelines.Banqiupipeline": 300,
        }
    }

    start_urls = ['http://hg3535z.com/odds2/d/getodds?sid=4&pt=4&ubt=am&pn=0&sb=2&dc=null&pid=0'] # 滚球菜单 棒球滚球列url
    # start_urls = ['http://hg3535z.com/odds2/d/getodds?sid=3&pt=4&ubt=am&pn=0&sb=2&dc=null&pid=0'] # 滚球菜单 网球测试滚球列url


    def parse(self, response):
        datas = json.loads(response.text)
        # datas ={"i": ["true", 4, 3110520, 2, 1, "sh", 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, "14 / 04", "", "", "false", "true", "\u68d2\u7403", 0], "eg": {"c": {"k": 29400, "n": "\u7f8e\u56fd\u804c\u4e1a\u68d2\u7403"}, "es": [{"dbg": "false", "egid": 0, "g": "", "i": ["\u4e9a\u7279\u5170\u5927\u52c7\u58eb\u961f", "\u7ebd\u7ea6\u5927\u90fd\u4f1a\u961f", "9", "True", "14 / 04", "", "9", "False", "0", "0", "2", "1", "", "", "", "", 0, "", "", 16, "True"], "ibs": "true", "ibsc": "false", "k": 3110520, "o": {"ah": {"s": 1, "v": ["h3164784133", "-1.5", "h3164784134", "+1.5", "o3164784133", "1.33", "o3164784134", "0.55"], "n": "\u6eda\u7403 \u8ba9\u7403"}, "ou": {"s": 2, "v": ["h3164784135", "6.5", "h3164784136", "6.5", "o3164784135", "0.70", "o3164784136", "1.04"], "n": "\u603b\u5f97\u5206:\u6eda\u7403 \u5927 / \u5c0f"}, "oe": {"s": 6, "v": ["o3164784137", "0.51", "o3164784138", "1.58"], "n": "\u603b\u5f97\u5206:\u6eda\u7403 \u5355 / \u53cc"}, "ml": {"s": 5, "v": ["o3164784139", "0.00", "o3164784140", "0.00"], "n": "\u6eda\u7403 \u72ec\u8d62\u76d8"}}, "p-o": [{"n": "\u51c0\u80dc\u7403\u6570", "st": "ha", "mgk": 9, "s": 7, "k": 301, "o": [["\u4e9a\u7279\u5170\u5927\u52c7\u58eb\u961f-\u8d62\u5f971\u5206", "o3164784186", "3.15", "0"], ["\u4e9a\u7279\u5170\u5927\u52c7\u58eb\u961f-\u8d62\u5f972\u5206", "o3164784187", "5.5", "3"], ["\u4e9a\u7279\u5170\u5927\u52c7\u58eb\u961f-\u8d62\u5f973\u5206", "o3164784188", "9.5", "6"], ["\u4e9a\u7279\u5170\u5927\u52c7\u58eb\u961f-\u8d62\u5f974\u5206\u6216\u66f4\u591a", "o3164784189", "7.5", "9"], ["\u7ebd\u7ea6\u5927\u90fd\u4f1a\u961f-\u8d62\u5f971\u5206", "o3164784190", "5.0", "2"], ["\u7ebd\u7ea6\u5927\u90fd\u4f1a\u961f-\u8d62\u5f972\u5206", "o3164784191", "7.5", "5"], ["\u7ebd\u7ea6\u5927\u90fd\u4f1a\u961f-\u8d62\u5f973\u5206", "o3164784192", "11.5", "8"], ["\u7ebd\u7ea6\u5927\u90fd\u4f1a\u961f-\u8d62\u5f974\u5206\u6216\u66f4\u591a", "o3164784193", "10.0", "11"]]}, {"n": "\u7b2c6\u5c40\u83b7\u80dc\u8005", "st": "ha", "mgk": 0, "s": 3000, "k": 4000, "o": [["\u548c\u5c40", "o3164784213", "1.50"], ["\u4e9a\u7279\u5170\u5927\u52c7\u58eb\u961f", "o3164784214", "5.5"], ["\u7ebd\u7ea6\u5927\u90fd\u4f1a\u961f", "o3164784215", "3.80"]]}, {"n": "\u52a0\u65f6", "st": "ha", "mgk": 312, "s": 8, "k": 13520, "o": [["\u662f", "o3164784228", "6.0"], ["\u4e0d\u662f", "o3164784229", "1.09"]]}], "pci": {"ctid": 0}, "sb": {"fb": "false", "sb": "false", "tb": "false", "ot": 0, "ps": [{"adv": -1, "a": 1.0, "e": 3110520, "h": 2.0, "p": "ft"}, {"adv": -1, "a": 0.0, "e": 3110520, "h": 0.0, "p": "ot"}, {"adv": -1, "a": 0.0, "e": 3110520, "h": 0.0, "p": "i1"}, {"adv": -1, "a": 1.0, "e": 3110520, "h": 1.0, "p": "i2"}, {"adv": -1, "a": 0.0, "e": 3110520, "h": 1.0, "p": "i3"}, {"adv": -1, "a": 0.0, "e": 3110520, "h": 0.0, "p": "i4"}, {"adv": -1, "a": 0.0, "e": 3110520, "h": 0.0, "p": "i5"}, {"adv": -1, "a": 0.0, "e": 3110520, "h": 0.0, "p": "i6"}, {"adv": -1, "a": 0.0, "e": 3110520, "h": 0.0, "p": "i7"}, {"adv": -1, "a": 0.0, "e": 3110520, "h": 0.0, "p": "i8"}, {"adv": -1, "a": 0.0, "e": 3110520, "h": 0.0, "p": "i9"}], "a": 1, "h": 2, "k": 3110520, "iwd": "false", "ls": -1, "s": 1, "ts": 0, "tt": 0, "ct": "00:00", "cp": "i5"}, "egn": ""}, {"dbg": "false", "egid": 0, "g": "", "i": ["ATL Braves - Team Runs - Over", "ATL Braves - Team Runs - Under", "9", "True", "14 / 04", "", "9", "False", "0", "0", "2", "0", "", "", "", "", 0, "", "", 16, "True"], "ibs": "true", "ibsc": "false", "k": 3110522, "o": {"ou": {"s": 3, "v": ["h3164784242", "3.5", "h3164784243", "3.5", "o3164784242", "1.12", "o3164784243", "0.63"], "n": "\u7403\u961f\u5f97\u5206:\u4e9a\u7279\u5170\u5927\u52c7\u58eb\u961f-\u6eda\u7403 \u5927 / \u5c0f"}}, "pci": {"ctid": 208, "ctn": "\u7403\u961f\u5f97\u5206"}, "egn": ""}, {"dbg": "false", "egid": 0, "g": "", "i": ["NY Mets - Team Runs - Over", "NY Mets - Team Runs - Under", "9", "True", "14 / 04", "", "9", "False", "0", "0", "1", "0", "", "", "", "", 0, "", "", 16, "True"], "ibs": "true", "ibsc": "false", "k": 3110524, "o": {"ou": {"s": 4, "v": ["h3164784261", "2.5", "h3164784263", "2.5", "o3164784261", "0.55", "o3164784263", "1.23"], "n": "\u7403\u961f\u5f97\u5206:\u7ebd\u7ea6\u5927\u90fd\u4f1a\u961f-\u6eda\u7403 \u5927 / \u5c0f"}}, "pci": {"ctid": 209, "ctn": "\u7403\u961f\u5f97\u5206"}, "egn": ""}]}, "ot": 2, "sc": {"3110520": {"a": 1, "h": 2}, "3110522": {"a": 0, "h": 2}, "3110524": {"a": 0, "h": 1}}, "v": 40576}
        # item = Today_all()
        ids = jsonpath.jsonpath(datas, '$..i-ot[0]..egs..es..i[16]') # ids新列表
        print(ids)
        if ids:
            ids = set(ids)
            for i in ids:
                urls = 'http://hg3535z.com/odds2/d/getamodds?eid={}&iip=true&ubt=am&isp=false'.format(i)
                print(urls)
                yield Request(url=urls, callback=self.parse_each)


    def parse_each(self, response):
        datas = json.loads(response.text)
        # datas ={"i": ["true", 4, 3110520, 2, 1, "sh", 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, "14 / 04", "", "", "false", "true", "\u68d2\u7403", 0], "eg": {"c": {"k": 29400, "n": "\u7f8e\u56fd\u804c\u4e1a\u68d2\u7403"}, "es": [{"dbg": "false", "egid": 0, "g": "", "i": ["\u4e9a\u7279\u5170\u5927\u52c7\u58eb\u961f", "\u7ebd\u7ea6\u5927\u90fd\u4f1a\u961f", "9", "True", "14 / 04", "", "9", "False", "0", "0", "2", "1", "", "", "", "", 0, "", "", 16, "True"], "ibs": "true", "ibsc": "false", "k": 3110520, "o": {"ah": {"s": 1, "v": ["h3164784133", "-1.5", "h3164784134", "+1.5", "o3164784133", "1.33", "o3164784134", "0.55"], "n": "\u6eda\u7403 \u8ba9\u7403"}, "ou": {"s": 2, "v": ["h3164784135", "6.5", "h3164784136", "6.5", "o3164784135", "0.70", "o3164784136", "1.04"], "n": "\u603b\u5f97\u5206:\u6eda\u7403 \u5927 / \u5c0f"}, "oe": {"s": 6, "v": ["o3164784137", "0.51", "o3164784138", "1.58"], "n": "\u603b\u5f97\u5206:\u6eda\u7403 \u5355 / \u53cc"}, "ml": {"s": 5, "v": ["o3164784139", "0.00", "o3164784140", "0.00"], "n": "\u6eda\u7403 \u72ec\u8d62\u76d8"}}, "p-o": [{"n": "\u51c0\u80dc\u7403\u6570", "st": "ha", "mgk": 9, "s": 7, "k": 301, "o": [["\u4e9a\u7279\u5170\u5927\u52c7\u58eb\u961f-\u8d62\u5f971\u5206", "o3164784186", "3.15", "0"], ["\u4e9a\u7279\u5170\u5927\u52c7\u58eb\u961f-\u8d62\u5f972\u5206", "o3164784187", "5.5", "3"], ["\u4e9a\u7279\u5170\u5927\u52c7\u58eb\u961f-\u8d62\u5f973\u5206", "o3164784188", "9.5", "6"], ["\u4e9a\u7279\u5170\u5927\u52c7\u58eb\u961f-\u8d62\u5f974\u5206\u6216\u66f4\u591a", "o3164784189", "7.5", "9"], ["\u7ebd\u7ea6\u5927\u90fd\u4f1a\u961f-\u8d62\u5f971\u5206", "o3164784190", "5.0", "2"], ["\u7ebd\u7ea6\u5927\u90fd\u4f1a\u961f-\u8d62\u5f972\u5206", "o3164784191", "7.5", "5"], ["\u7ebd\u7ea6\u5927\u90fd\u4f1a\u961f-\u8d62\u5f973\u5206", "o3164784192", "11.5", "8"], ["\u7ebd\u7ea6\u5927\u90fd\u4f1a\u961f-\u8d62\u5f974\u5206\u6216\u66f4\u591a", "o3164784193", "10.0", "11"]]}, {"n": "\u7b2c6\u5c40\u83b7\u80dc\u8005", "st": "ha", "mgk": 0, "s": 3000, "k": 4000, "o": [["\u548c\u5c40", "o3164784213", "1.50"], ["\u4e9a\u7279\u5170\u5927\u52c7\u58eb\u961f", "o3164784214", "5.5"], ["\u7ebd\u7ea6\u5927\u90fd\u4f1a\u961f", "o3164784215", "3.80"]]}, {"n": "\u52a0\u65f6", "st": "ha", "mgk": 312, "s": 8, "k": 13520, "o": [["\u662f", "o3164784228", "6.0"], ["\u4e0d\u662f", "o3164784229", "1.09"]]}], "pci": {"ctid": 0}, "sb": {"fb": "false", "sb": "false", "tb": "false", "ot": 0, "ps": [{"adv": -1, "a": 1.0, "e": 3110520, "h": 2.0, "p": "ft"}, {"adv": -1, "a": 0.0, "e": 3110520, "h": 0.0, "p": "ot"}, {"adv": -1, "a": 0.0, "e": 3110520, "h": 0.0, "p": "i1"}, {"adv": -1, "a": 1.0, "e": 3110520, "h": 1.0, "p": "i2"}, {"adv": -1, "a": 0.0, "e": 3110520, "h": 1.0, "p": "i3"}, {"adv": -1, "a": 0.0, "e": 3110520, "h": 0.0, "p": "i4"}, {"adv": -1, "a": 0.0, "e": 3110520, "h": 0.0, "p": "i5"}, {"adv": -1, "a": 0.0, "e": 3110520, "h": 0.0, "p": "i6"}, {"adv": -1, "a": 0.0, "e": 3110520, "h": 0.0, "p": "i7"}, {"adv": -1, "a": 0.0, "e": 3110520, "h": 0.0, "p": "i8"}, {"adv": -1, "a": 0.0, "e": 3110520, "h": 0.0, "p": "i9"}], "a": 1, "h": 2, "k": 3110520, "iwd": "false", "ls": -1, "s": 1, "ts": 0, "tt": 0, "ct": "00:00", "cp": "i5"}, "egn": ""}, {"dbg": "false", "egid": 0, "g": "", "i": ["ATL Braves - Team Runs - Over", "ATL Braves - Team Runs - Under", "9", "True", "14 / 04", "", "9", "False", "0", "0", "2", "0", "", "", "", "", 0, "", "", 16, "True"], "ibs": "true", "ibsc": "false", "k": 3110522, "o": {"ou": {"s": 3, "v": ["h3164784242", "3.5", "h3164784243", "3.5", "o3164784242", "1.12", "o3164784243", "0.63"], "n": "\u7403\u961f\u5f97\u5206:\u4e9a\u7279\u5170\u5927\u52c7\u58eb\u961f-\u6eda\u7403 \u5927 / \u5c0f"}}, "pci": {"ctid": 208, "ctn": "\u7403\u961f\u5f97\u5206"}, "egn": ""}, {"dbg": "false", "egid": 0, "g": "", "i": ["NY Mets - Team Runs - Over", "NY Mets - Team Runs - Under", "9", "True", "14 / 04", "", "9", "False", "0", "0", "1", "0", "", "", "", "", 0, "", "", 16, "True"], "ibs": "true", "ibsc": "false", "k": 3110524, "o": {"ou": {"s": 4, "v": ["h3164784261", "2.5", "h3164784263", "2.5", "o3164784261", "0.55", "o3164784263", "1.23"], "n": "\u7403\u961f\u5f97\u5206:\u7ebd\u7ea6\u5927\u90fd\u4f1a\u961f-\u6eda\u7403 \u5927 / \u5c0f"}}, "pci": {"ctid": 209, "ctn": "\u7403\u961f\u5f97\u5206"}, "egn": ""}]}, "ot": 2, "sc": {"3110520": {"a": 1, "h": 2}, "3110522": {"a": 0, "h": 2}, "3110524": {"a": 0, "h": 1}}, "v": 40576}
        try:
            new_datas = json.loads(response.text)['eg']
            # new_datas = datas['eg']
        except:
            new_datas = ""
        try:
            pt = response.meta['pt']
        except:
            pt = 0
        if new_datas:
            # 联赛id
            league_id = new_datas["c"]["k"]
            # 联赛名
            league_name = new_datas["c"]["n"]
            new_data = new_datas["es"]
            for result in new_data:
                if result['pci']["ctid"] == 0:
                    item = Bangqiu()
                    # 比赛id
                    game_id = str(result['k'])
                    # 球队1
                    team_home = result['i'][0]
                    # 球队2
                    team_guest = result['i'][1]
                    # 数量(97>)
                    number = result['i'][2]
                    # 比赛状态
                    zhuangtai = result['i'][3]
                    # 日期
                    data_game = result['i'][4]
                    # 开赛时间
                    time_game = result['i'][5]
                    # 队1分数
                    score_home1 = result['i'][10]
                    # 队2分数
                    score_guest1 = result['i'][11]
                    # 第几盘
                    jijie = result['i'][12]
                    # 球队得分
                    qiudui = result['pci'].get('ctn', "")
# --------------------------------------------------让球分割线-----------------------------------------------------------
                    concedes_dict = {}
                    concedes_dict_rule = {}
                    try:
                        concedes = result['o']["ah"]["v"]
                        concedes_dict_rule['bq_concede_home'] = concedes[1]
                        concedes_dict['bq_concede_home'] = concedes[5]
                        concedes_dict_rule['bq_concede_guest'] = concedes[3]
                        concedes_dict['bq_concede_guest'] = concedes[7]
                    except:
                        concedes_dict_rule['bq_concede_home'] = ""
                        concedes_dict['bq_concede_home'] = ""
                        concedes_dict_rule['bq_concede_guest'] = ""
                        concedes_dict['bq_concede_guest'] = ""
# 让球 前5局-------------------------------------------------------------------------------------------------------------
                    try:
                        concedes = result['o']["ahf5in"]["v"]
                        concedes_dict_rule['bq_five_concede_home'] = concedes[1]
                        concedes_dict['bq_five_concede_home'] = concedes[5]
                        concedes_dict_rule['bq_five_concede_guest'] = concedes[3]
                        concedes_dict['bq_five_concede_guest'] = concedes[7]
                    except:
                        concedes_dict_rule['bq_five_concede_home'] = ""
                        concedes_dict['bq_five_concede_home'] = ""
                        concedes_dict_rule['bq_five_concede_guest'] = ""
                        concedes_dict['bq_five_concede_guest'] = ""
# ---------------------------------------------------独赢----------------------------------------------------------------
                    capots_dict = {}
                    try:
                        capots = result['o']["ml"]["v"]
                        capots_dict['bq_capot_home'] = capots[1]
                        capots_dict['bq_capot_guest'] = capots[3]

                    except:
                        capots_dict['bq_capot_home'] = ""
                        capots_dict['bq_capot_guest'] = ""
# ---------------------------------------------------总得分:大/小--------------------------------------------------------

                    total_size_dict = {}
                    total_size_dict_rule = {}
                    try:
                        total_sizes = result['o']["ou"]["v"]
                        total_size_dict_rule['bq_total_number_big'] = total_sizes[1]
                        total_size_dict['bq_total_number_big'] = total_sizes[5]
                        total_size_dict_rule['bq_total_number_small'] = total_sizes[3]
                        total_size_dict['bq_total_number_small'] = total_sizes[7]
                    except:
                        total_size_dict_rule['bq_total_number_big'] = ""
                        total_size_dict['bq_total_number_big'] = ""
                        total_size_dict_rule['bq_total_number_small'] = ""
                        total_size_dict['bq_total_number_small'] = ""
# 总得分: 大/小 前5局-----------------------------------------------------------------------------------------------------
                    try:
                        total_sizes = result['o']["ouf5in"]["v"]
                        total_size_dict_rule['bq_five_total_number_big'] = total_sizes[1]
                        total_size_dict['bq_five_total_number_big'] = total_sizes[5]
                        total_size_dict_rule['bq_five_total_number_small'] = total_sizes[3]
                        total_size_dict['bq_five_total_number_small'] = total_sizes[7]
                    except:
                        total_size_dict_rule['bq_five_total_number_big'] = ""
                        total_size_dict['bq_five_total_number_big'] = ""
                        total_size_dict_rule['bq_five_total_number_small'] = ""
                        total_size_dict['bq_five_total_number_small'] = ""

# ------------------------------------------------------总得分:单/双-----------------------------------------------------
                    odd_evens_dict = {}
                    odd_evens_dict_rule = {}
                    try:
                        odd_evens = result['o']["oe"]["v"]
                        # 上半场 主队让球条件
                        odd_evens_dict['bq_two_sides_single'] = odd_evens[1]
                        odd_evens_dict_rule['bq_two_sides_single'] = "single"
                        odd_evens_dict['bq_two_sides_double'] = odd_evens[3]
                        odd_evens_dict_rule['bq_two_sides_double'] = "double"

                    except:
                        odd_evens_dict['bq_two_sides_single'] = ""
                        odd_evens_dict['bq_two_sides_single'] = ""
                        odd_evens_dict_rule['bq_two_sides_double'] = ""
                        odd_evens_dict_rule['bq_two_sides_double'] = ""

                    # 联赛id
                    item['league_id'] = league_id
                    # 联赛名
                    item['league_name'] = league_name
                    # 比赛id
                    item['game_id'] = game_id
                    # 球队1
                    item['team_home'] = team_home
                    # 球队2
                    item['team_guest'] = team_guest
                    # 数量(97>)
                    item['number'] = number
                    # 比赛状态
                    item['zhuangtai'] = zhuangtai
                    # 日期
                    item['data_game'] = data_game
                    # 开赛时间
                    item['time_game'] = time_game
                    # 队1分数
                    item['score_home'] = score_home1
                    # 队2分数
                    item['score_guest'] = score_guest1
                    # 第几节
                    item['jijie'] = jijie
                    item['pt'] = pt
                    # 球队得分
                    item['qiudui'] = qiudui
                    # 让球
                    item['concedes_dict'] = concedes_dict
                    item['concedes_dict_rule'] = concedes_dict_rule
                    # 独赢
                    item['capots_dict'] = capots_dict
                    # 总得分：大/小
                    item['total_size_dict'] = total_size_dict
                    item['total_size_dict_rule'] = total_size_dict_rule
                    # 总得分：单/双
                    item['odd_evens_dict'] = odd_evens_dict
                    item['odd_evens_dict_rule'] = odd_evens_dict_rule
                    yield item