# encoding: utf-8 import datetime import json import logging import redis import scrapy from ..items import Hgjieshu from ..settings import R_HOST, R_PASSWORD, R_POST, R_DB class HgjieshuSpider(scrapy.Spider): name = 'jieshu' to_day = datetime.datetime.now() allowed_domains = ['hg3535z.com'] custom_settings = { "ITEM_PIPELINES": { 'hg3535.pipeline.jieshu.Jieshuqiupipeline': 300, }, # 'LOG_LEVEL': 'DEBUG', # 'LOG_FILE': "../hg3535/log/saiguo{}_{}_{}.log".format(to_day.year, to_day.month, to_day.day) } rls = redis.Redis(host=R_HOST, port=R_POST, db=R_DB, password=R_PASSWORD) def start_requests(self): match_ids = self.rls.hgetall("hg3535.GunQiu.ids") if match_ids: for match_id, ctime in match_ids.items(): match_id, ctime = match_id.decode(), ctime.decode() url = 'https://odata.yonghuai5515.com/odds6i/d/getamodds/zh-cn/eid/{}/iip/true/ubt/am/isp/false'.format(match_id) yield scrapy.Request(url=url, callback=self.parse, dont_filter=True, meta={'ctime': ctime}) def parse(self, response): logger = logging.getLogger(__name__) ctime = response.meta['ctime'] try: data = json.loads(response.text) status = data['i'][0] # if not status: ball = data['i'][1] match_id = data['i'][2] home = data['eg']['es'][0]['i'][0] away = data['eg']['es'][0]['i'][1] date = ctime item = Hgjieshu() item['ball'] = ball item['match_id'] = match_id item['status'] = status item['home'] = home item['away'] = away item['date'] = date yield item except Exception as e: logger.warning(e) return