# -*- coding: utf-8 -*- import scrapy import lxml.etree import pycomm class SportsSpider(scrapy.Spider): name = 'sports' allowed_domains = ['hg0088.com'] # start_urls = ['http://hg0088.com/'] custom_settings = { "Accept":"text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8", "Accept-Encoding":"gzip, deflate", "Accept-Language":"zh-CN,zh;q=0.8", "Cache-Control":"max-age=0", "Connection":"keep-alive", "Cookie":"OddType@21627573=H; protocolstr=http; gamePoint_21627573=2019-05-10%2A0%2A0; _ga=GA1.4.601418716.1557495256; _gid=GA1.4.1118061739.1557495256", "Host":"199.26.100.178", "USER_AGENT": "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/55.0.2883.87 UBrowser/6.2.4098.3 Safari/537.36", } start_url= 'http://199.26.100.178/app/member/get_game_allbets.php' def start_requests(self): url=self.start_url form_data={ "uid":"x5cx6y9fm21627573l266800", "langx":"zh-cn", "gtype":"FT", "showtype":"FT", "gid":"3701122", "ltype":"4", "date":pycomm.gmdate() } request = scrapy.FormRequest(url, formdata=form_data, callback=self.parse) yield request def parse(self, response): doc = lxml.etree.fromstring(response.body) print(doc.xpath('/serverresponse/game/gid//text()')) print(pycomm.getCache('conf')) pass