# -*- coding: utf-8 -*- import json import scrapy # from scrapy.http import Request from .. items import Liansai class LanqiulsSpider(scrapy.Spider): name = 'liansai' allowed_domains = ['hg3535z.com'] custom_settings = { "ITEM_PIPELINES": { 'hg3535.pipelines.Liansaipipeline': 300, } } def start_requests(self): for y in range(1, 5): for i in range(1, 5): url = 'https://hg3535z.com/odds2/d/getcomps?sid='+str(y)+'&pt=' + str(i) + '&ubt=am&dc=null&pn=0&pid=0' yield scrapy.Request(url=url, callback=self.parse) def parse(self, response): try: datas = json.loads(response.text)['gs'] except: datas = "" if datas: for data in datas: # 区域id,欧洲 area_id = data['gid'] # 区域名,欧洲 area_name = data['gn'] new_data = data['fc'] if new_data: for i in new_data: item = Liansai() ball = json.loads(response.text)['s'] st_league = i["id"] name_chinese = i['nm'] item['area_id'] = area_id item['area_name'] = area_name item['st_league'] = st_league item['name_chinese'] = name_chinese item['ball'] = ball yield item