sports_scrapy/hg3535/spiders/wangqiu.py

# -*- coding: utf-8 -*-
# -*- coding: utf-8 -*-
import copy
import datetime
import json

import scrapy
from scrapy.http import Request

from ..items import Wangqiu


class LanqiuSpider(scrapy.Spider):

    name = "wangqiu"
    to_day = datetime.datetime.now()
    allowed_domains = ['hg3535z.com']
    custom_settings = {
        "ITEM_PIPELINES": {
            "hg3535.pipelines.Wangqiupipeline": 300,
        },
        'LOG_LEVEL': 'DEBUG',
        'LOG_FILE': "../hg3535/log/wangqiu_{}_{}_{}.log".format(to_day.year, to_day.month, to_day.day)
    }
    def start_requests(self):
        # newall_lists = ['am', 'ftht1x2', 'oe', 'tg', 'htft', 'cs', 'fglg', 'or']
        for y in range(1, 4):
            url = 'https://hg3535z.com/odds2/d/getodds?sid=3&pt='+str(y)+'&ubt=am&pn=0&sb=2&dc=null&pid=0'
            yield scrapy.Request(url=url, callback=self.parse, meta={'pt': y}, dont_filter=True)

    def parse(self, response):
        if response.text:
            try:
                datas = json.loads(response.text).get('n-ot', "").get('egs', "")
            except:
                datas = ""
            try:
                pt = copy.copy(response.meta['pt'])
            except:
                pt = 0
            if datas:
                for result in datas:
                    new_results = result['es']
                    for new_result in new_results:
                        game_id = str(new_result['i'][16])
                        if pt == 1:
                            url = "https://hg3535z.com/odds2/d/getamodds?eid=" + game_id + "&iip=false&ubt=am&isp=false"
                            yield Request(url=url, callback=self.parse_each, meta={'pt': pt}, dont_filter=True)
                        if pt == 2:
                            url = "https://hg3535z.com/odds2/d/getamodds?eid=" + game_id + "&iip=false&ubt=am&isp=false"
                            yield Request(url=url, callback=self.parse_each, meta={'pt': pt}, dont_filter=True)
                        if pt == 3:
                            url = "https://hg3535z.com/odds2/d/getamodds?eid=" + game_id + "&iip=false&ubt=am&isp=true"
                            yield Request(url=url, callback=self.parse_each, meta={'pt': pt}, dont_filter=True)
    def parse_each(self, response):
        try:
            new_datas = json.loads(response.text)['eg']
            pt = response.meta['pt']
            wangqiu = json.loads(response.text)['i'][31]
        except:
            new_datas = ""
            pt = 0
            wangqiu = ""
        if new_datas:
            # 联赛id
            league_id = new_datas["c"]["k"]
            # 联赛名
            league_name = new_datas["c"]["n"]
            new_data = new_datas["es"]
            for result in new_data:
                item = Wangqiu()
                # 比赛id
                game_id = str(result['k'])
                # 球队1
                team_home = result['i'][0]
                # 球队2
                team_guest = result['i'][1]
                # 数量(97>)
                number = result['i'][2]
                # 比赛状态
                zhuangtai = result['i'][3]
                # 日期
                data_game = result['i'][4]
                # 开赛时间
                time_game = result['i'][5]
                # 队1分数
                score_home1 = result['i'][10]
                # 队2分数
                score_guest1 = result['i'][11]
                # 第几盘
                jijie = result['i'][12]
                # 球队得分
                qiudui = result['pci'].get('ctn', "")
# --------------------------------------------------让盘分割线-----------------------------------------------------------
                concedes_dict = {}
                concedes_dict_rule = {}
                try:
                    concedes = result['o']["ah"]["v"]
                    concedes_dict_rule['dishes_home'] = concedes[1]
                    concedes_dict_rule['dishes_guest'] = concedes[3]
                    if pt is 3:
                        concedes_dict['dishes_home'] = float(concedes[5]) - 1
                        concedes_dict['dishes_guest'] = float(concedes[7]) - 1
                    else:
                        concedes_dict['dishes_guest'] = concedes[7]
                        concedes_dict['dishes_home'] = float(concedes[5])
                except:
                    concedes_dict_rule['dishes_home'] = ""
                    concedes_dict['dishes_home'] = ""
                    concedes_dict_rule['dishes_guest'] = ""
                    concedes_dict['dishes_guest'] = ""
# ---------------------------------------------------冠军----------------------------------------------------------------
                kemps_dict = {}
                try:
                    kemps = result['o']["ml"]["v"]
                    kemps_dict['kemp_home'] = kemps[1]
                    kemps_dict['kemp_guest'] = kemps[3]

                except:
                    kemps_dict['kemp_home'] = ""
                    kemps_dict['kemp_guest'] = ""
                # 第一盘 冠军
                try:
                    kemps = result['o']["mls1"]["v"]
                    kemps_dict['one_kemp_home'] = kemps[1]
                    kemps_dict['one_kemp_guest'] = kemps[3]

                except:
                    kemps_dict['one_kemp_home'] = ""
                    kemps_dict['one_kemp_guest'] = ""

# ---------------------------------------------------让局---------------------------------------------------------------

                bureaus_dict = {}
                bureaus_dict_rule = {}
                try:
                    bureaus = result['o']["ahfts"]["v"]
                    bureaus_dict_rule['concede_home'] = bureaus[1]
                    bureaus_dict_rule['concede_guest'] = bureaus[3]
                    if pt is 3:
                        bureaus_dict['concede_home'] = float(bureaus[5]) - 1
                        bureaus_dict['concede_guest'] = float(bureaus[7]) - 1
                    else:
                        bureaus_dict['concede_home'] = bureaus[5]
                        bureaus_dict['concede_guest'] = bureaus[7]
                except:
                    bureaus_dict_rule['concede_home'] = ""
                    bureaus_dict['concede_home'] = ""
                    bureaus_dict_rule['concede_guest'] = ""
                    bureaus_dict['concede_guest'] = ""

# -----------------------------------------------------总局数:大/小------------------------------------------------------
# 总局数大小第一盘 未采集
                total_number_dict = {}
                total_number_dict_rule = {}
                try:
                    total_numbers = result['o']["oufts"]["v"]
                    total_number_dict_rule['total_number_big'] = total_numbers[1]
                    total_number_dict_rule['total_number_small'] = total_numbers[3]
                    if pt is 3:
                        total_number_dict['total_number_big'] = float(total_numbers[5]) - 1
                        total_number_dict['total_number_small'] = float(total_numbers[7]) - 1
                    else:
                        total_number_dict['total_number_big'] = total_numbers[5]
                        total_number_dict['total_number_small'] = total_numbers[7]
                except:
                    total_number_dict_rule['total_number_big'] = ""
                    total_number_dict['total_number_big'] = ""
                    total_number_dict_rule['total_number_small'] = ""
                    total_number_dict['total_number_small'] = ""

# ------------------------------------------------------总局数:单/双-----------------------------------------------------
                odd_evens_dict = {}
                odd_evens_dict_rule = {}
                try:
                    odd_evens = result['o']["oefts"]["v"]
                    # 上半场 主队让球条件
                    odd_evens_dict['two_sides_single'] = odd_evens[1]
                    odd_evens_dict_rule['two_sides_single'] = "单"
                    odd_evens_dict['two_sides_double'] = odd_evens[3]
                    odd_evens_dict_rule['two_sides_double'] = "双"

                except:
                    odd_evens_dict['two_sides_single'] = ""
                    odd_evens_dict['two_sides_single'] = ""
                    odd_evens_dict_rule['two_sides_double'] = ""
                    odd_evens_dict_rule['two_sides_double'] = ""

                # 联赛id
                item['league_id'] = league_id
                # 联赛名
                item['league_name'] = league_name
                # 比赛id
                item['game_id'] = game_id
                # 球队1
                item['team_home'] = team_home
                # 球队2
                item['team_guest'] = team_guest
                # 数量(97>)
                item['number'] = number
                # 比赛状态
                item['zhuangtai'] = zhuangtai
                # 日期
                item['data_game'] = data_game
                # 开赛时间
                item['time_game'] = time_game
                # 队1分数
                item['score_home'] = score_home1
                # 队2分数
                item['score_guest'] = score_guest1
                # 第几节
                item['jijie'] = jijie
                item['pt'] = pt
                # 球队得分
                item['qiudui'] = qiudui
                # 让盘
                item['concedes_dict'] = concedes_dict
                item['concedes_dict_rule'] = concedes_dict_rule
                # 冠军
                item['kemps_dict'] = kemps_dict
                # 让局
                item['bureaus_dict'] = bureaus_dict
                item['bureaus_dict_rule'] = bureaus_dict_rule
                # 总局数大小
                item['total_number_dict'] = total_number_dict
                item['total_number_dict_rule'] = total_number_dict_rule
                # 总局数单双
                item['odd_evens_dict'] = odd_evens_dict
                item['odd_evens_dict_rule'] = odd_evens_dict_rule
                item['wangqiu'] = wangqiu
                yield item