sports_scrapy/hg3535/pipeline/saiguo.py

# import datetime
import datetime
import pymongo
import time
import logging

import redis
from twisted.internet import defer, reactor

from ..settings import M_HOST, M_USER, M_PASSWORD, M_POST, M_DB, MATCH_STATUS
from ..utils.helper import Helper

# 滚球足球 插入
class Jieshuqiupipeline(object):

    def open_spider(self, spider):
        self.mongo = pymongo.MongoClient(host=M_HOST, username=M_USER, password=M_PASSWORD, port=M_POST)
        self.db = self.mongo[M_DB]
        # cls = redis.Redis(host='192.168.0.110', port=6379, db=0)

    @defer.inlineCallbacks
    def process_item(self, item, spider):
        out = defer.Deferred()
        reactor.callInThread(self._do_calculation, item, out)
        yield out
        defer.returnValue(item)

    # def process_item(self, item, spider):
    def _do_calculation(self, item, out):
        logger = logging.getLogger(__name__)
        match_id, score_other = item['id_score']
        pt = item['pt']
        if pt == 1:
            game_code = "zq"
        elif pt == 2:
            game_code = "lq"
        elif pt == 3:
            game_code = "wq"
        else:
            game_code = "bq"
        if self.db.match_status35.find({'match_id': match_id}).count() < 1:
            status_dict = {"game_code": game_code, "title": "match_status", "source": "hg3535"}
            data_list = []
            data = {'match_id': match_id, 'status': 2, "is_rollball": 0, "is_today": 0, "is_morningplate": 0, "is_stringscene": 0, "is_horn": 0}
            data_list.append(data)
            status_dict['data'] = data_list
            res = Helper.async_post(MATCH_STATUS, status_dict)
            if res:
                if "成功" in res:
                    self.db.match_status35.insert(status_dict)
                    logger.info('{},赛事结果状态交成功, {}'.format(game_code, res))
                    logger.info(status_dict)
                else:
                    logger.warning('{},赛事结果状态交失败, {}'.format(game_code, res))
                    logger.warning(status_dict)
            else:
                logger.warning('{},赛事结果状态接口异常提交失败, {}'.format(game_code, res))
                logger.warning(status_dict)
        else:
            logger.info('足球赛事结果状态已存在，不提交')
        reactor.callFromThread(out.callback, item)