# -*- coding: utf-8 -*- import scrapy from ..items import ZuqiuItem class ZuqiuSpider(scrapy.Spider): name = 'zuqiu' allowed_domains = ['m.hgg070.com/'] headers = { 'Accept': '*/*', 'Accept-Encoding': 'gzip, deflate', 'Accept-Language': 'zh-CN,zh;q=0.9', 'Content-Length': '130', 'Content-type': 'application/x-www-form-urlencoded', 'Cookie': '_ga=GA1.2.471918301.1572059707; _gid=GA1.2.2109447865.1572059707; _gat=1', 'Host': 'm.hgg070.com', 'Origin': 'http://m.hgg070.com', 'Proxy-Connection': 'keep-alive', 'Referer': 'http://m.hgg070.com/', 'User-Agent': 'Mozilla/5.0 (Linux; Android 6.0; Nexus 5 Build/MRA58N) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/78.0.3904.70 Mobile Safari/537.36' } custom_settings={ "ITEM_PIPELINES": { "collectSports.pipelines.zuqiu.ZuqiuPipeline": 200, }, } #读取今日足球 def start_requests(self): url="http://m.hgg070.com/app/member/get_league_list.php" from_data={ 'uid': '7c70e73f576d42d9f6d9fb1fcaa08c47b04bb9279584caedfe65858afb26722d', 'langx': 'zh-cn', 'ltype': '3', 'gtype': 'FT', 'showtype': 'RB', 'sorttype': '', 'date': '', 'isP': '' } yield scrapy.FormRequest(url=url, formdata=from_data, callback=self.parse, headers=self.headers,meta={'data': from_data}, dont_filter=True) #解析今日足球,获取所有的联赛,并请求联赛详情页 def parse(self, response): url='http://m.hgg070.com/app/member/get_game_list.php' data=response.xpath("//league") from_data=response.meta['data'] for item in data: lid = item.xpath('./league_id/text()').extract_first() from_data['lid'] = lid yield scrapy.FormRequest(url=url, formdata=from_data, callback=self.detailtedMsg,meta={'data': response.meta['data']},dont_filter=True) # 获取所有玩法 def detailtedMsg(self,response): url = 'http://m.hgg070.com/app/member/get_game_more.php' data=response.xpath("//game") from_data=response.meta['data'] for item in data: lid = item.xpath('./gid/text()').extract_first() from_data['gid'] = lid yield scrapy.FormRequest(url=url, formdata=from_data, callback=self.getitem,dont_filter=True) #获取更多 def getitem(self,response): data=response.xpath("//game") for obj in data: item=ZuqiuItem() item.id=obj.xpath('./gid') item.team_h=obj.xpath('./team_h') item.team_c = obj.xpath('./team_c') item.ior_RTS2Y = obj.xpath('./ior_RTS2Y') item.ior_RTS2N = obj.xpath('./ior_RTS2N') item.ior_REH = obj.xpath('./ior_REH') item.ior_REC = obj.xpath('./ior_REC') item.ior_ROUC = obj.xpath('./ior_ROUC') item.ior_HROUH = obj.xpath('./ior_HROUH') item.ratio_rouo = obj.xpath('./ratio_rouo') item.ratio_rouu = obj.xpath('./ratio_rouu') yield item