import datetime import pymongo import logging from twisted.internet import defer, reactor import redis from ..settings import M_HOST, M_USER, M_PASSWORD, M_POST, M_DB, MATCH_STATUS from ..utils.helper import Helper from ..settings import R_HOST, R_PASSWORD, R_POST, R_DB # 滚球足球 插入 class Jieshuqiupipeline(object): def open_spider(self, spider): self.mongo = pymongo.MongoClient(host=M_HOST, username=M_USER, password=M_PASSWORD, port=M_POST) self.db = self.mongo[M_DB] self.rls = redis.Redis(host=R_HOST, port=R_POST, db=R_DB, password=R_PASSWORD) @defer.inlineCallbacks def process_item(self, item, spider): out = defer.Deferred() reactor.callInThread(self._do_calculation, item, out) yield out # defer.returnValue(item) def _do_calculation(self, item, out): # def process_item(self, item, spider): logger = logging.getLogger(__name__) ball = item['ball'] match_id = item['match_id'] status = item['status'] home = item['home'] away = item['away'] date = item['date'] if ball == 1: game_code = "zq" elif ball == 2: game_code = "lq" elif ball == 3: game_code = "wq" else: game_code = "bq" if status: ostatus = 1 dates = date.split("/") month = dates[1].strip() day = str(dates[0].strip()) else: ostatus = 2 month_dict = {'Jan': '01', 'Feb': '02', 'Mar': '03', 'Apr': '04', 'May': '05', 'Jun': '06', 'Jul': '07', 'Aug': '08', 'Sep': '09', 'Oct': '10', 'Nov': '11', 'Dec': '12'} dates = date.split(" ") month = month_dict[dates[1].strip()] day = str(dates[0].strip()) status_dict = {"game_code": game_code, "title": "match_status", "source": "hg3535"} update = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S") data_list = [] us_time = '{}-{}-{}'.format(datetime.datetime.now().year, month, day) match_identity = Helper.genearte_uuid(home + away + us_time) data = {'match_id': match_id, 'status': ostatus, "is_rollball": 0, "is_today": 0, "is_morningplate": 0, "is_stringscene": 0, "is_horn": 0, 'game_code': game_code, 'update': update, "match_identity": match_identity} data_list.append(data) status_dict['data'] = data_list res = Helper.async_post(MATCH_STATUS, status_dict) if res: if res.get('status') == 1: self.db.match_status35.insert(data) # self.db.match_status35.update({'match_id': match_id}, {'$set': data}, upsert=True) if ostatus == 2: self.rls.srem('hg3535.gunqiu.ids', match_id) logger.info('{},赛事结果状态交成功, {}'.format(game_code, res)) # logger.info(status_dict) # print(status_dict) else: logger.warning('{},赛事结果状态交失败, {}'.format(game_code, res)) # logger.warning(status_dict) else: logger.warning('{},赛事结果状态接口异常提交失败, {}'.format(game_code, res)) # logger.warning(status_dict) reactor.callFromThread(out.callback, item)