sports_collect/collectSports/spiders/sports.py

# -*- coding: utf-8 -*-
import scrapy
import lxml.etree
import pycomm

class SportsSpider(scrapy.Spider):
    name = 'sports'
    allowed_domains = ['hg0088.com']
    # start_urls = ['http://hg0088.com/']
    custom_settings = {
        "Accept":"text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8",
        "Accept-Encoding":"gzip, deflate",
        "Accept-Language":"zh-CN,zh;q=0.8",
        "Cache-Control":"max-age=0",
        "Connection":"keep-alive",
        "Cookie":"OddType@21627573=H; protocolstr=http; gamePoint_21627573=2019-05-10%2A0%2A0; _ga=GA1.4.601418716.1557495256; _gid=GA1.4.1118061739.1557495256",
        "Host":"199.26.100.178",
        "USER_AGENT": "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/55.0.2883.87 UBrowser/6.2.4098.3 Safari/537.36",
    }

    start_url= 'http://199.26.100.178/app/member/get_game_allbets.php'

    def start_requests(self):
            url=self.start_url
            form_data={
                "uid":"x5cx6y9fm21627573l266800",
                "langx":"zh-cn",
                "gtype":"FT",
                "showtype":"FT",
                "gid":"3701122",
                "ltype":"4",
                "date":pycomm.gmdate()
            }
            request = scrapy.FormRequest(url, formdata=form_data, callback=self.parse)
            yield request

    def parse(self, response):
        doc = lxml.etree.fromstring(response.body)  
        print(doc.xpath('/serverresponse/game/gid//text()'))
        
        print(pycomm.getCache('conf'))
        pass