sports_scrapy/hg3535/spiders/liansai.py

# -*- coding: utf-8 -*-
import datetime
import json

import requests
import scrapy

# from scrapy.http import Request

from .. items import Liansai


class LanqiulsSpider(scrapy.Spider):
    name = 'liansai'
    to_day = datetime.datetime.now()
    allowed_domains = ['hg3535z.com']
    custom_settings = {
        "ITEM_PIPELINES": {
            'hg3535.pipelines.Liansaipipeline': 300,
        },
        'LOG_LEVEL': 'DEBUG',
        'LOG_FILE': "../hg3535/log/liansai_{}_{}_{}.log".format(to_day.year, to_day.month, to_day.day)
    }

    def start_requests(self):
        for y in range(1, 5):
            for i in range(1, 5):
                url = 'https://hg3535z.com/odds2/d/getcomps?sid='+str(y)+'&pt=' + str(i) + '&ubt=am&dc=null&pn=0&pid=0'
                yield scrapy.Request(url=url, callback=self.parse, dont_filter=True)
                # yield scrapy.Request(url=url, callback=self.parse)
    def parse(self, response):
        try:
            datas = json.loads(response.text)['gs']
        except:
            datas = ""
        if datas:
            for data in datas:
                # 区域id，欧洲
                area_id = data['gid']
                # 区域名，欧洲
                area_name = data['gn']
                new_data = data['fc']
                if new_data:
                    for i in new_data:
                        item = Liansai()
                        ball = json.loads(response.text)['s']
                        st_league = i["id"]
                        name_chinese = i['nm']
                        item['area_id'] = area_id
                        item['area_name'] = area_name
                        item['st_league'] = st_league
                        item['name_chinese'] = name_chinese
                        item['ball'] = ball
                        yield item