import datetime import time import logging import pymongo from twisted.internet import defer, reactor from ..utils.helper import Helper from ..settings import M_HOST, M_USER, M_PASSWORD, M_POST, M_DB, LEAGUE_URL, ODDS_URL, MATCH_URL class ZuqiuPipeline(object): def open_spider(self, spider): self.mongo = pymongo.MongoClient(host=M_HOST, username=M_USER, password=M_PASSWORD, port=M_POST, authSource=M_DB) self.db = self.mongo[M_DB] # @defer.inlineCallbacks # def process_item(self,item,spider): # out=defer.Deferred() # reactor.callInThread(self._do_calculation,item,out) # yield out # def _do_calculation(self,item,out): # pass def process_item(self, item, spider): logger = logging.getLogger(__name__) match_all = item['data'] pt = str(item['index']) team_h, team_c = match_all['team_h'], match_all['team_c'] league_name, league_id = match_all['league'], match_all['gidm'] us_time, re_time = match_all['datetime'], match_all['re_time'] match_id = match_all['gid'] tag_number = item['tag'] uuid = Helper.genearte_uuid(league_name) league_list = [] last_time = '{}-12-31 23:59:59'.format(datetime.datetime.now().year) match_date, match_time, time3 = Helper.change_time(us_time) if self.db.zq_league35.find({'lg_id': league_id}).count() < 1: # if self.db.zq_league35.find({'uuid': uuid}).count() < 1: league_dict = {"game_code": "zq", "title": "league", "source": "hg3535"} league_key = ["name_chinese", "kind", "match_mode", "if_stop", "last_time", "lg_id", "source", "uuid"] league_value = [league_name, "1", "1", "0", last_time, league_id, "hg3535", uuid] league_data = dict(zip(league_key, league_value)) league_list.append(league_data) league_dict['data'] = league_list res = Helper.async_post(LEAGUE_URL, league_dict) if res: if res.get('status') == 1: self.db.zq_league070.insert(league_data) logging.info('足球联赛提交, {}'.format(res)) else: logging.warning('足球联赛接口异常, {}'.format(res)) else: logging.info('{},联赛已存在, 不提交'.format(league_name)) pt_dict = {'0': 'is_today', '1': 'is_morningplate', '2': 'is_stringscene', '3': 'is_rollball'} pt_status = pt_dict[pt] if pt == '0': is_rollball = 0 is_today = 1 is_morningplate = 0 is_stringscene = 0 elif pt == '2': is_rollball = 0 is_today = 0 is_morningplate = 1 is_stringscene = 0 else: is_today = 0 is_rollball = 0 is_morningplate = 0 is_stringscene = 1 match_list = [] match_identity = Helper.genearte_uuid(team_h + team_c + match_date) if self.db.zq_competition35.find({'match_id': match_id, pt_status: 1}).count() < 1: # if self.db.zq_competition35.find({'match_identity': match_identity, pt_status: 1}).count() < 1: match_dict = {"game_code": "zq", "title": "match", "source": "hg3535"} match_kay = ["home_team", "guest_team", "lg_id", "status", "match_id", "match_date", "match_time", "tag", "source", "is_rollball", "is_morningplate", "is_stringscene", "us_time", "uuid", "half_match_id", "is_today", "is_horn", 'match_identity'] match_value = [team_h, team_c, league_id, 0, match_id, match_date, match_time, tag_number, "hg3535", is_rollball, is_morningplate, is_stringscene, us_time, uuid, 0, is_today, 0, match_identity] match_data = dict(zip(match_kay, match_value)) match_list.append(match_data) match_dict['data'] = match_list res = Helper.async_post(MATCH_URL, match_dict) if res: if res.get('status') == 1: self.db.zq_competition070.insert(match_data) logging.info('足球赛事提交, {}'.format(res)) else: logger.warning('足球赛事表提交失败, {}'.format(res)) # logger.warning(match_dict) else: logger.warning('足球赛事接口异常提交失败, {}'.format(res)) # logger.warning(match_dict) else: logger.info('足球赛事已存在,不提交') # reactor.callFromThread(out.callback, item)