sports_scrapy/hg3535/pipeline/zhibo.py

import pymongo
import time
import logging

from twisted.internet import defer, reactor
from ..utils.helper import Helper
from ..settings import M_HOST, M_USER, M_PASSWORD, M_POST, M_DB, MATCHWARN


# 滚球足球 插入
class Zuqiupipeline(object):

    def open_spider(self, spider):
        self.mongo = pymongo.MongoClient(host=M_HOST, username=M_USER, password=M_PASSWORD, port=M_POST)
        self.db = self.mongo[M_DB]

    @defer.inlineCallbacks
    def process_item(self, item, spider):
        out = defer.Deferred()
        reactor.callInThread(self._do_calculation, item, out)
        yield out
        # defer.returnValue(item)

    # def process_item(self, item, spider):
    def _do_calculation(self, item, out):
        logger = logging.getLogger(__name__)
        detail_datas = item['detail']
        mid = item['mid']
        date = item['date']
        home = item['home']
        away = item['away']
        us_date = time.strftime('%Y-%m-%d', time.localtime(date))
        match_identity = Helper.genearte_uuid(home + away + us_date)
        warns = []
        up_time = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime())
        warn_dict = {"game_code": "zq", "title": "match_warn", "source": "hg3535", "match_id": mid, 'up_time': up_time, 'match_identity': match_identity}
        for detail_data in detail_datas:
            team_name, event_name, event_uts, warn_type = detail_data
            find_time = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(event_uts))
            detail_dict = {"find_time": find_time, "warn_name": event_name, 'warn_type': warn_type}
            warns.append(detail_dict)
        if warns:
            warn_dict['data'] = warns
            res = Helper.async_post(MATCHWARN, warn_dict)
            if res.get('status') == 1:
                # print('危险球提交成功')
                self.db.FT_warn35.insert(warn_dict)
                logger.info('足球直播危险球提交成功, {}'.format(res))
            else:
                logger.warning('足球直播危险球，提交失败, {}'.format(res))
        else:
            logger.info('足球直播危险球列表为空, 不提交')
        reactor.callFromThread(out.callback, item)