sports_scrapy/hg3535/pipeline/jieshu.py

import datetime
import pymongo
import logging
from twisted.internet import defer, reactor
import redis

from ..settings import M_HOST, M_USER, M_PASSWORD, M_POST, M_DB, MATCH_STATUS
from ..utils.helper import Helper
from ..settings import R_HOST, R_PASSWORD, R_POST, R_DB


# 滚球足球 插入
class Jieshuqiupipeline(object):

    def open_spider(self, spider):
        self.mongo = pymongo.MongoClient(host=M_HOST, username=M_USER, password=M_PASSWORD, port=M_POST)
        self.db = self.mongo[M_DB]
        self.rls = redis.Redis(host=R_HOST, port=R_POST, db=R_DB, password=R_PASSWORD)

    @defer.inlineCallbacks
    def process_item(self, item, spider):
        out = defer.Deferred()
        reactor.callInThread(self._do_calculation, item, out)
        yield out
        defer.returnValue(item)

    def _do_calculation(self, item, out):
        # def process_item(self, item, spider):
        logger = logging.getLogger(__name__)
        ball = item['ball']
        match_id = item['match_id']
        status = item['status']
        home = item['home']
        away = item['away']
        date = item['date']
        if ball == 1:
            game_code = "zq"
        elif ball == 2:
            game_code = "lq"
        elif ball == 3:
            game_code = "wq"
        else:
            game_code = "bq"

        dates = str(date.strip(" "))
        status_dict = {"game_code": game_code, "title": "match_status", "source": "hg3535"}
        update = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")
        month_dict = {'Jan': '01', 'Feb': '02', 'Mar': '03', 'Apr': '04', 'May': '05', 'Jun': '06', 'Jul': '07',
                      'Aug': '08', 'Sep': '09', 'Oct': '10', 'Nov': '11', 'Dec': '12'}
        month = month_dict[dates[1]]
        day = str(dates[0])
        data_list = []
        if status:
            ostatus = 1
        else:
            ostatus = 2
        us_time = '{}-{}-{}'.format(datetime.datetime.now().year, month, day)
        match_identity = Helper.genearte_uuid(home + away + us_time)
        data = {'match_id': match_id, 'status': ostatus, "is_rollball": 0, "is_today": 0, "is_morningplate": 0,
                "is_stringscene": 0, "is_horn": 0, 'game_code': game_code, 'update': update,
                "match_identity": match_identity}
        data_list.append(data)
        status_dict['data'] = data_list
        res = Helper.async_post(MATCH_STATUS, status_dict)
        if res:
            if res.get('status') == 1:
                self.db.match_status35.insert(data)
                # self.db.match_status35.update({'match_id': match_id}, {'$set': data}, upsert=True)
                if ostatus == 2:
                    self.rls.srem('hg3535.gunqiu.ids', match_id)
                logger.info('{},赛事结果状态交成功, {}'.format(game_code, res))
                logger.info(status_dict)
                print(status_dict)
            else:
                logger.warning('{},赛事结果状态交失败, {}'.format(game_code, res))
                logger.warning(status_dict)
        else:
            logger.warning('{},赛事结果状态接口异常提交失败, {}'.format(game_code, res))
            logger.warning(status_dict)
        reactor.callFromThread(out.callback, item)