sports_scrapy/hg3535/spiders/wqbodan.py

# -*- coding: utf-8 -*-
import copy
import datetime
import json

import scrapy

from scrapy.http import Request

from ..items import Wqbodan


class WgbodanSpider(scrapy.Spider):
    name = 'wqbodan'
    to_day = datetime.datetime.now()
    allowed_domains = ['hg3535z.com']
    custom_settings = {
        "ITEM_PIPELINES": {
            'hg3535.pipelines.Wqbodanpipeline': 300,
        },
        'LOG_LEVEL': 'DEBUG',
        'LOG_FILE': "../hg3535/log/wqbodan_{}_{}_{}.log".format(to_day.year, to_day.month, to_day.day)
    }

    def start_requests(self):
        url = 'https://hg3535z.com/odds2/d/getmenu?pid=0'
        yield scrapy.Request(url=url, callback=self.parse_one, dont_filter=True)

    def parse_one(self, response):
        for y in range(1, 4):
            url = 'https://hg3535z.com/odds2/d/getodds?sid=3&pt=' + str(y) + '&ubt=stcs&pn=0&sb=2&dc=null&pid=0'
            yield Request(url=url, callback=self.parse, meta={'pt': y}, dont_filter=True)

    def parse(self, response):
        try:
            datas = json.loads(response.text)
        except:
            datas = ""
        try:
            results = datas['n-ot']['egs']
        except:
            results = ""
        try:
            pt = copy.copy(response.meta['pt'])
        except:
            pt = 0
        if results:
            for result in results:
                # 联赛id
                league_id = result['c']['k']
                # 联赛名
                league_name = result['c']['n']
                new_results = result['es']
                for new_result in new_results:
                    item = Wqbodan()
                    # 比赛id
                    game_id = str(new_result['i'][16])
                    # 球队1
                    team_home = new_result['i'][0]
                    # 球队2
                    team_guest = new_result['i'][1]
                    # 数量(97>)
                    number = new_result['i'][2]
                    # 状态
                    zhuangtai = new_result['i'][3]
                    # 日期
                    data_game = new_result['i'][4]
                    # 开赛时间
                    time_game = new_result['i'][5]
                    # 队1分数
                    score_home = new_result['i'][10]
                    # 队2分数
                    score_guest = new_result['i'][11]
                    # 下半场
                    half_way = new_result['i'][12]
                    # 角球或者其他
                    corner_ball = new_result['pci'].get('ctn', "")
                    bodan_data = {}
                    # 波胆
                    bodans = new_result['o'].get('sb', "")
                    if bodans:
                        one_list = ["bodanhome_two_zero", "bodanhome_two_one", "bodanhome_three_zero",
                                    "bodanhome_three_one", "bodanhome_three_two", "bodanhome_four_zero",
                                    "bodanhome_four_one", "bodanhome_four_two", "bodanhome_four_three"]
                        two_list = ["bodanguest_two_zero", "bodanguest_two_one", "bodanguest_three_zero",
                                    "bodanguest_three_one", "bodanguest_three_two", "bodanguest_four_zero",
                                    "bodanguest_four_one", "bodanguest_four_two", "bodanguest_four_three"]

                        new_bodans = [bodans[i] for i in range(len(bodans)) if i % 2 == 1]
                        # 主队bodan_home
                        bodan_home = [new_bodans[i] for i in range(len(new_bodans)) if i % 2 == 0]
                        for index, t in enumerate(one_list):
                            bodan_data[t] = bodan_home[index]
                        # 客队bodan_guest
                        bodan_guest = [new_bodans[i] for i in range(len(new_bodans)) if i % 2 == 1]
                        for y, z in enumerate(two_list):
                            bodan_data[z] = bodan_guest[y]
                    item['league_id'] = league_id
                    item['league_name'] = league_name
                    item['game_id'] = game_id
                    item['team_home'] = team_home
                    item['team_guest'] = team_guest
                    item['number'] = number
                    item['data_game'] = data_game
                    item['time_game'] = time_game
                    item['corner_ball'] = corner_ball
                    item['score_home'] = score_home
                    item['score_guest'] = score_guest
                    item['half_way'] = half_way
                    item['pt'] = pt
                    item['zhuangtai'] = zhuangtai
                    item['bodan_data'] = bodan_data
                    yield item