sports_scrapy/hg3535/pipeline/zhibo.py

# import datetime
import datetime

import pymongo
import time
import logging

from twisted.internet import defer, reactor
from ..utils.helper import Helper
from ..settings import M_HOST, M_USER, M_PASSWORD, M_POST, M_DB, MATCHWARN


# 滚球足球 插入
class Zuqiupipeline(object):

    def open_spider(self, spider):
        self.mongo = pymongo.MongoClient(host=M_HOST, username=M_USER, password=M_PASSWORD, port=M_POST)
        self.db = self.mongo[M_DB]

    # @defer.inlineCallbacks
    # def process_item(self, item, spider):
    #     out = defer.Deferred()
    #     reactor.callInThread(self._do_calculation, item, out)
    #     yield out
        # defer.returnValue(item)

    def process_item(self, item, spider):
    # def _do_calculation(self, item, out):
        logger = logging.getLogger(__name__)
        detail_datas = item['detail']
        mid = item['mid']
        warns = []
        warn_dict = {"game_code": "zq", "title": "match", "source": "hg3535", "match_id": mid}
        for detail_data in detail_datas:
            team_name, event_name, event_uts = detail_data
            find_time = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(event_uts))
            detail_dict = {"find_time": find_time, "warn_name": event_name}
            warns.append(detail_dict)
        if warns:
            warn_dict['data'] = warns
            res = Helper.async_post(MATCHWARN, warn_dict)
            if res.get('status') == 1:
                logger.info('足球直播危险球提交成功, {}'.format(res))
            else:
                logger.warning('足球直播危险球提交成功，提交失败, {}'.format(res))
        else:
            logger.info('足球直播危险球列表为空, 不提交')