sports_scrapy/hg3535/spiders/wangqiu.py

# -*- coding: utf-8 -*-

import copy
import datetime
import json
import scrapy
from scrapy.http import Request
from ..items import Wangqiu


class LanqiuSpider(scrapy.Spider):
    name = "wangqiu"
    to_day = datetime.datetime.now()
    allowed_domains = ['hg3535z.com']
    custom_settings = {
        "ITEM_PIPELINES": {
            "hg3535.pipeline.wangqiu.Wangqiupipeline": 300,
        },
        'LOG_LEVEL': 'DEBUG',
        'LOG_FILE': "./log/wangqiu_{}_{}_{}.log".format(to_day.year, to_day.month, to_day.day)
    }
    # rls = redis.Redis(host=R_HOST, port=R_POST, db=R_DB, password=R_PASSWORD)

    def start_requests(self):
        for y in range(1, 4):
            url = 'https://odata.yonghuai5515.com/odds6i/d/getodds/zh-cn/sid/3/pt/{}/ubt/am/pn/0/sb/2/dc/null/pid/0'.format(y)
            yield scrapy.Request(url=url, callback=self.parse, meta={'pt': y}, dont_filter=True)

    def parse(self, response):
        if response.text:
            try:
                datas = json.loads(response.text).get('n-ot', "").get('egs', "")
            except:
                datas = ""
            try:
                pt = copy.copy(response.meta['pt'])
            except:
                pt = 0
            if datas:
                for result in datas:
                    new_results = result['es']
                    for new_result in new_results:
                        game_id = new_result['i'][16]
                        if pt == 1:
                            url = "https://odata.yonghuai5515.com/odds6i/d/getamodds/zh-cn/eid/{}/iip/false/ubt/am/isp/false".format(game_id)
                            yield Request(url=url, callback=self.parse_each, meta={'pt': pt}, dont_filter=True)
                        if pt == 2:
                            url = "https://odata.yonghuai5515.com/odds6i/d/getamodds/zh-cn/eid/{}/iip/false/ubt/am/isp/false".format(game_id)
                            yield Request(url=url, callback=self.parse_each, meta={'pt': pt}, dont_filter=True)
                        if pt == 3:
                            url = "https://odata.yonghuai5515.com/odds6i/d/getamodds/zh-cn/eid/{}/iip/false/ubt/am/isp/true".format(game_id)
                            yield Request(url=url, callback=self.parse_each, meta={'pt': pt}, dont_filter=True)

    def parse_each(self, response):
        try:
            new_datas = json.loads(response.text)['eg']
            pt = response.meta['pt']
            wangqiu = json.loads(response.text)['i'][31]
        except:
            new_datas = ""
            pt = 0
            wangqiu = ""
        if new_datas:
            # 联赛id
            league_id = new_datas["c"]["k"]
            # 联赛名
            league_name = new_datas["c"]["n"]
            new_data = new_datas["es"]
            for result in new_data:
                item = Wangqiu()
                # 比赛id
                game_id = str(result['k'])
                # 球队1
                team_home = result['i'][0]
                # 球队2
                team_guest = result['i'][1]
                # 数量(97>)
                number = result['i'][2]
                # 比赛状态
                zhuangtai = result['i'][3]
                # 日期
                data_game = result['i'][4]
                # 开赛时间
                time_game = result['i'][5]
                # 队1分数
                score_home1 = result['i'][10]
                # 队2分数
                score_guest1 = result['i'][11]
                # 第几盘
                jijie = result['i'][12]
                # 球队得分
                qiudui = result['pci'].get('ctn', "")
                # --------------------------------------------------让盘分割线-----------------------------------------------------------
                concedes_dict = {}
                concedes_dict_rule = {}
                try:
                    concedes = result['o']["ah"]["v"]
                    concedes_dict_rule['ldh'] = concedes[1]  # 主队
                    concedes_dict_rule['ldg'] = concedes[3]  # 客队
                    if pt is 3:
                        concedes_dict['ldh'] = float(concedes[5]) - 1
                        concedes_dict['ldg'] = float(concedes[7]) - 1
                    else:
                        concedes_dict['ldg'] = concedes[7]
                        concedes_dict['ldh'] = float(concedes[5])
                except:
                    concedes_dict_rule['ldh'] = ""
                    concedes_dict['ldh'] = ""
                    concedes_dict_rule['ldg'] = ""
                    concedes_dict['ldg'] = ""
                # ---------------------------------------------------冠军----------------------------------------------------------------
                kemps_dict = {}
                try:
                    kemps = result['o']["ml"]["v"]
                    kemps_dict['ch'] = kemps[1]  # 主队
                    kemps_dict['cg'] = kemps[3]  # 客队

                except:
                    kemps_dict['ch'] = ""
                    kemps_dict['cg'] = ""
                # 第一盘 冠军
                try:
                    kemps = result['o']["mls1"]["v"]
                    kemps_dict['ch_1'] = kemps[1]
                    kemps_dict['cg_1'] = kemps[3]

                except:
                    kemps_dict['ch_1'] = ""
                    kemps_dict['cg_1'] = ""

                # ---------------------------------------------------让局---------------------------------------------------------------

                bureaus_dict = {}
                bureaus_dict_rule = {}
                try:
                    bureaus = result['o']["ahfts"]["v"]
                    bureaus_dict_rule['lbh'] = bureaus[1]  # 主队
                    bureaus_dict_rule['lbg'] = bureaus[3]  # 客队
                    if pt is 3:
                        bureaus_dict['lbh'] = float(bureaus[5]) - 1
                        bureaus_dict['lbg'] = float(bureaus[7]) - 1
                    else:
                        bureaus_dict['lbh'] = bureaus[5]
                        bureaus_dict['lbg'] = bureaus[7]
                except:
                    bureaus_dict_rule['lbh'] = ""
                    bureaus_dict['lbh'] = ""
                    bureaus_dict_rule['lbg'] = ""
                    bureaus_dict['lbg'] = ""

                # -----------------------------------------------------总局数:大/小------------------------------------------------------
                # 总局数大小第一盘 未采集
                total_number_dict = {}
                total_number_dict_rule = {}
                try:
                    total_numbers = result['o']["oufts"]["v"]
                    total_number_dict_rule['tnb'] = total_numbers[1]
                    total_number_dict_rule['tns'] = total_numbers[3]
                    if pt is 3:
                        total_number_dict['tnb'] = float(total_numbers[5]) - 1
                        total_number_dict['tns'] = float(total_numbers[7]) - 1
                    else:
                        total_number_dict['tnb'] = total_numbers[5]
                        total_number_dict['tns'] = total_numbers[7]
                except:
                    total_number_dict_rule['tnb'] = ""
                    total_number_dict['tnb'] = ""
                    total_number_dict_rule['tns'] = ""
                    total_number_dict['tns'] = ""
                # 第一节盘
                try:
                    one_total_sizes = result['o']["ous1"]["v"]
                    total_number_dict_rule["tnb_1"] = one_total_sizes[1]
                    total_number_dict["tnb_1"] = one_total_sizes[5]
                    total_number_dict_rule["tns_1"] = one_total_sizes[3]
                    total_number_dict["tns_1"] = one_total_sizes[7]
                    if pt is 3:
                        total_number_dict['tnb_1'] = float(one_total_sizes[5]) - 1
                        total_number_dict['tns_1'] = float(one_total_sizes[7]) - 1
                    else:
                        total_number_dict['tnb_1'] = one_total_sizes[5]
                        total_number_dict['tns_1'] = one_total_sizes[7]
                except:
                    total_number_dict_rule["tnb_1"] = ""
                    total_number_dict["tnb_1"] = ""
                    total_number_dict_rule["tns_1"] = ""
                    total_number_dict["tns_1"] = ""

                # ------------------------------------------------------总局数:单/双-----------------------------------------------------
                odd_evens_dict = {}
                odd_evens_dict_rule = {}
                try:
                    odd_evens = result['o']["oefts"]["v"]
                    # 上半场 主队让球条件
                    odd_evens_dict['tss'] = odd_evens[1]
                    odd_evens_dict_rule['tss'] = "单"
                    odd_evens_dict['tsd'] = odd_evens[3]
                    odd_evens_dict_rule['tsd'] = "双"

                except:
                    odd_evens_dict['tss'] = ""
                    odd_evens_dict['tsd'] = ""
                    odd_evens_dict_rule['tss'] = ""
                    odd_evens_dict_rule['tsd'] = ""

                # 联赛id
                item['league_id'] = league_id
                # 联赛名
                item['league_name'] = league_name
                # 比赛id
                item['game_id'] = game_id
                # 球队1
                item['team_home'] = team_home
                # 球队2
                item['team_guest'] = team_guest
                # 数量(97>)
                item['number'] = number
                # 比赛状态
                item['zhuangtai'] = zhuangtai
                # 日期
                item['data_game'] = data_game
                # 开赛时间
                item['time_game'] = time_game
                # 队1分数
                item['score_home'] = score_home1
                # 队2分数
                item['score_guest'] = score_guest1
                # 第几节
                item['jijie'] = jijie
                item['pt'] = pt
                # 球队得分
                item['qiudui'] = qiudui
                # 让盘
                item['concedes_dict'] = concedes_dict
                item['concedes_dict_rule'] = concedes_dict_rule
                # 冠军
                item['kemps_dict'] = kemps_dict
                # 让局
                item['bureaus_dict'] = bureaus_dict
                item['bureaus_dict_rule'] = bureaus_dict_rule
                # 总局数大小
                item['total_number_dict'] = total_number_dict
                item['total_number_dict_rule'] = total_number_dict_rule
                # 总局数单双
                item['odd_evens_dict'] = odd_evens_dict
                item['odd_evens_dict_rule'] = odd_evens_dict_rule
                item['wangqiu'] = wangqiu
                yield item