sports_scrapy/hg3535/spiders/roll_bangqiu.py

#!/usr/bin/env python
# -*- coding: utf-8 -*-
import datetime
import time

import jsonpath
import redis
import json
import scrapy
from scrapy.http import Request
from ..items import Roll_Bangqiu
from ..settings import R_HOST, R_PASSWORD, R_POST, R_DB

class BqrangqiuSpider(scrapy.Spider):
    name = 'roll_bangqiu'
    to_day = datetime.datetime.now()
    allowed_domains = ['hg3535z.com']
    custom_settings = {
        "ITEM_PIPELINES": {
            "hg3535.pipeline.roll_bangqiu.Roll_Banqiupipeline": 350,
        },
        # 'LOG_LEVEL': 'DEBUG',
        # 'LOG_FILE': "../hg3535/log/roll_bangqiu_{}_{}_{}.log".format(to_day.year, to_day.month, to_day.day)
    }

    start_urls = [
        'https://odata.jiushan6688.com/odds6i/d/getodds/zh-cn/sid/4/pt/4/ubt/am/pn/0/sb/2/dc/null/pid/0']  # 滚球菜单 棒球滚球列url
    rls = redis.Redis(host=R_HOST, port=R_POST, db=R_DB, password=R_PASSWORD)

    def parse(self, response):
        datas = json.loads(response.text)
        ids = jsonpath.jsonpath(datas, '$..i-ot[0]..egs..es..i[16]')  # ids新列表
        if ids:
            for i in ids:
                urls = 'https://odata.jiushan6688.com/odds6i/d/getamodds/zh-cn/eid/{}/iip/true/ubt/am/isp/false'.format(
                    i)
                self.rls.sadd('hg3535.gunqiu.ids', i)
                up_time = str(time.time()) + '&' + 'bq'
                self.rls.hset('hg3535.ball.ids', i, up_time)
                yield Request(url=urls, callback=self.parse_each, dont_filter=True)

    def parse_each(self, response):
        datas = json.loads(response.text)
        try:
            new_datas = datas['eg']
        except:
            new_datas = ""
        try:
            pt = response.meta['pt']
        except:
            pt = 0
        if new_datas:
            # 联赛id
            league_id = new_datas["c"]["k"]
            # 联赛名
            league_name = new_datas["c"]["n"]
            new_data = new_datas["es"]
            for result in new_data:
                if result['pci']["ctid"] == 0:
                    item = Roll_Bangqiu()
                    # 比赛id
                    game_id = str(result['k'])
                    # 球队1
                    team_home = result['i'][0]
                    # 球队2
                    team_guest = result['i'][1]
                    # 数量(97>)
                    number = result['i'][2]
                    # 比赛状态
                    zhuangtai = result['i'][3]
                    # 日期
                    data_game = result['i'][4]
                    # 开赛时间
                    time_game = result['i'][5]
                    # 队1分数
                    score_home1 = result['i'][10]
                    # 队2分数
                    score_guest1 = result['i'][11]
                    # 第几盘
                    # jijie = result['i'][12]
                    # 球队得分
                    # qiudui = result['pci'].get('ctn', "")
                    # 是否出局 不一定存在
                    ot = result['sb'].get("ot", "")

                    ps_list = result['sb']['ps']
                    # 倒三角形
                    triangle = result['sb']['s']
                    if triangle:
                        rule = 'Bottom'
                    else:
                        rule = 'Top'
                    # 右菱形bool值
                    fb = result['sb']['fb']
                    if not fb:
                        baseman_1 = 'N'
                    else:
                        baseman_1 = 'Y'
                    # 左菱形bool值
                    tb = result['sb']['tb']
                    if not tb:
                        baseman_3 = 'N'
                    else:
                        baseman_3 = 'Y'
                    # 上菱形bool值
                    sb = result['sb']['tb']
                    if not sb:
                        baseman_2 = 'N'
                    else:
                        baseman_2 = 'Y'
                    # 局数或者节或者分
                    cp = result['sb']['cp']
                    # 第一个队伍
                    a = result['sb']['a']
                    # 第二个队伍
                    h = result['sb']['h']

                    # --------------------------------------------------让球分割线----------------------------------------
                    concedes_dict = {}
                    concedes_dict_rule = {}
                    try:
                        concedes = result['o']["ah"]["v"]
                        concedes_dict_rule['coh'] = concedes[1]
                        concedes_dict['coh'] = concedes[5]
                        concedes_dict_rule['cog'] = concedes[3]
                        concedes_dict['cog'] = concedes[7]
                    except:
                        concedes_dict_rule['coh'] = ""
                        concedes_dict['coh'] = ""
                        concedes_dict_rule['cog'] = ""
                        concedes_dict['cog'] = ""
                    # 让球 前5局------------------------------------------------------------------------------------------
                    try:
                        concedes = result['o']["ahf5in"]["v"]
                        concedes_dict_rule['coh_5'] = concedes[1]
                        concedes_dict['coh_5'] = concedes[5]
                        concedes_dict_rule['cog_5'] = concedes[3]
                        concedes_dict['cog_5'] = concedes[7]
                    except:
                        concedes_dict_rule['coh_5'] = ""
                        concedes_dict['coh_5'] = ""
                        concedes_dict_rule['cog_5'] = ""
                        concedes_dict['cog_5'] = ""
                    # ---------------------------------------------------独赢--------------------------------------------
                    capots_dict = {}
                    try:
                        capots = result['o']["ml"]["v"]
                        capots_dict['ch'] = capots[1]
                        capots_dict['cg'] = capots[3]

                    except:
                        capots_dict['ch'] = ""
                        capots_dict['cg'] = ""
                    # ---------------------------------------------------总得分:大/小-------------------------------------

                    total_size_dict = {}
                    total_size_dict_rule = {}
                    try:
                        total_sizes = result['o']["ou"]["v"]
                        total_size_dict_rule['tnb'] = total_sizes[1]
                        total_size_dict['tnb'] = total_sizes[5]
                        total_size_dict_rule['tns'] = total_sizes[3]
                        total_size_dict['tns'] = total_sizes[7]
                    except:
                        total_size_dict_rule['tnb'] = ""
                        total_size_dict['tnb'] = ""
                        total_size_dict_rule['tns'] = ""
                        total_size_dict['tns'] = ""
                    # 总得分: 大/小 前5局----------------------------------------------------------------------------------
                    try:
                        total_sizes = result['o']["ouf5in"]["v"]
                        total_size_dict_rule['tnb_5'] = total_sizes[1]
                        total_size_dict['tnb_5'] = total_sizes[5]
                        total_size_dict_rule['tns_5'] = total_sizes[3]
                        total_size_dict['tns_5'] = total_sizes[7]
                    except:
                        total_size_dict_rule['tnb_5'] = ""
                        total_size_dict['tnb_5'] = ""
                        total_size_dict_rule['tns_5'] = ""
                        total_size_dict['tns_5'] = ""

                    # ------------------------------------------------------总得分:单/双----------------------------------
                    odd_evens_dict = {}
                    odd_evens_dict_rule = {}
                    try:
                        odd_evens = result['o']["oe"]["v"]
                        # 上半场 主队让球条件
                        odd_evens_dict['tss'] = odd_evens[1]
                        odd_evens_dict_rule['tss'] = "单"
                        odd_evens_dict['tsd'] = odd_evens[3]
                        odd_evens_dict_rule['tsd'] = "双"

                    except:
                        odd_evens_dict['tss'] = ""
                        odd_evens_dict['tsd'] = ""
                        odd_evens_dict_rule['tss'] = ""
                        odd_evens_dict_rule['tsd'] = ""
                    source_dict = {}
                    source_dict['sc_1th_H'] = ps_list[2]['h']
                    source_dict['sc_1th_C'] = ps_list[2]['a']
                    source_dict['sc_2th_H'] = ps_list[3]['h']
                    source_dict['sc_2th_C'] = ps_list[3]['a']
                    source_dict['sc_3th_H'] = ps_list[4]['h']
                    source_dict['sc_3th_C'] = ps_list[4]['a']
                    source_dict['sc_4th_H'] = ps_list[5]['h']
                    source_dict['sc_4th_C'] = ps_list[5]['a']
                    source_dict['sc_5th_H'] = ps_list[6]['h']
                    source_dict['sc_5th_C'] = ps_list[6]['a']
                    source_dict['sc_6th_H'] = ps_list[7]['h']
                    source_dict['sc_6th_C'] = ps_list[7]['a']
                    source_dict['sc_7th_H'] = ps_list[8]['h']
                    source_dict['sc_7th_C'] = ps_list[8]['a']
                    source_dict['sc_8th_H'] = ps_list[9]['h']
                    source_dict['sc_8th_C'] = ps_list[9]['a']
                    source_dict['sc_9th_H'] = ps_list[10]['h']
                    source_dict['sc_9th_C'] = ps_list[10]['a']
                    source_dict['OT_H'] = ps_list[1]['h']
                    source_dict['OT_C'] = ps_list[1]['a']
                    source_dict['game_num_H'] = ps_list[0]['h']
                    source_dict['game_num_C'] = ps_list[0]['a']
                    source_dict['schedule'] = rule + cp
                    source_dict['out_game_num'] = ot
                    source_dict['baseman_1'] = baseman_1
                    source_dict['baseman_2'] = baseman_2
                    source_dict['baseman_3'] = baseman_3
                    # 联赛id
                    item['league_id'] = league_id
                    # 联赛名
                    item['league_name'] = league_name
                    # 比赛id
                    item['game_id'] = game_id
                    # 球队1
                    item['team_home'] = team_home
                    # 球队2
                    item['team_guest'] = team_guest
                    # 数量(97>)
                    item['number'] = number
                    # 比赛状态
                    item['zhuangtai'] = zhuangtai
                    # 日期
                    item['data_game'] = data_game
                    # 开赛时间
                    item['time_game'] = time_game
                    # 队1分数
                    item['score_home'] = score_home1
                    # 队2分数
                    item['score_guest'] = score_guest1
                    # 第几节
                    item['jijie'] = cp
                    item['pt'] = pt
                    # 球队得分
                    item['qiudui'] = "{}:{}".format(a, h)
                    # 让球
                    item['concedes_dict'] = concedes_dict
                    item['concedes_dict_rule'] = concedes_dict_rule
                    # 独赢
                    item['capots_dict'] = capots_dict
                    # 总得分：大/小
                    item['total_size_dict'] = total_size_dict
                    item['total_size_dict_rule'] = total_size_dict_rule
                    # 总得分：单/双
                    item['odd_evens_dict'] = odd_evens_dict
                    item['odd_evens_dict_rule'] = odd_evens_dict_rule
                    # item['match_score_dict'] = d
                    item['source_dict'] = source_dict
                    yield item