# -*- coding: utf-8 -*- import scrapy class ZuqiuSpider(scrapy.Spider): name = 'zuqiu' allowed_domains = ['m.hgg070.com'] # 读取今日足球 def start_requests(self): headers = { 'Accept': '*/*', 'Accept-Encoding': 'gzip, deflate', 'Accept-Language': 'zh-CN,zh;q=0.9', 'Content-Length': '130', 'Content-type': 'application/x-www-form-urlencoded', 'Cookie': '_ga=GA1.2.1009358217.1572056223; _gid=GA1.2.97506800.1572056223; _gat=1', 'Host': 'm.hgg070.com', 'Origin': 'http://m.hgg070.com', 'Proxy-Connection': 'keep-alive', 'Referer': 'http://m.hgg070.com/', 'User-Agent': 'Mozilla/5.0 (iPhone; CPU iPhone OS 11_0 like Mac OS X) AppleWebKit/604.1.38 (KHTML, like Gecko) Version/11.0 Mobile/15A372 Safari/604.1' } url = "http://m.hgg070.com/app/member/get_league_list.php" h_types = [('FT', '', '130'), ('FU', 'P', '131'), ('FU', "", '130'), ('FU', '', '131')] for h_type in h_types: show_type, isp, length = h_type headers['Content-Length'] = length from_data = { 'uid': '4f6573b99cc31f7be1579b063888a8a2bcda122ce94228510cda9e1bb32a477f', 'langx': 'zh-cn', 'ltype': '3', 'gtype': 'FT', 'showtype': show_type, 'sorttype': '', 'date': '', 'isP': isp } yield scrapy.FormRequest(url=url, formdata=from_data, callback=self.parse, headers=headers, meta={'showtype': show_type, 'isp': isp}, dont_filter=True) def parse(self, response): leagues = response.xpath('//serverresponse/game/league') url = 'http://m.hgg070.com/app/member/get_game_list.php' headers = { 'Accept': '*/*', 'Accept-Encoding': 'gzip, deflate', 'Accept-Language': 'zh-CN,zh;q=0.9', 'Content-Length': '147', 'Content-type': 'application/x-www-form-urlencoded', 'Cookie': '_ga=GA1.2.1009358217.1572056223; _gid=GA1.2.97506800.1572056223; _gat=1', 'Host': 'm.hgg070.com', 'Origin': 'http://m.hgg070.com', 'Proxy-Connection': 'keep-alive', 'Referer': 'http://m.hgg070.com/', 'User-Agent': 'Mozilla/5.0 (iPhone; CPU iPhone OS 11_0 like Mac OS X) AppleWebKit/604.1.38 (KHTML, like Gecko) Version/11.0 Mobile/15A372 Safari/604.1' } if leagues: showtype = response.meta['showtype'] isp = response.meta['isp'] if showtype == 'FT' and isp == '': date = '' headers['Content-Length'] = '147' elif showtype == 'FU' and isp == 'P': date = 'all' headers['Content-Length'] = '151' elif showtype == 'FU' and isp == '': date = 'all' headers['Content-Length'] = '150' else: date = 'all' headers['Content-Length'] = '151' for league in leagues: lid = league.xpath('.//league_id/text()').extract_first() from_data = { 'uid': '4f6573b99cc31f7be1579b063888a8a2bcda122ce94228510cda9e1bb32a477f', 'langx': 'zh-cn', 'ltype': '3', 'gtype': 'FT', 'showtype': showtype, # 'showtype': "FT", # 'lid': '103391', 'lid': lid, 'sorttype': 'league', 'date': date, 'isP': isp # 'date': "", # 'isP': "" } yield scrapy.FormRequest(url=url, formdata=from_data, callback=self.parse_match, headers=headers, meta={'showtype': showtype, 'isp': isp}, dont_filter=True) def parse_match(self, response): print(response.text) pass