# -*- coding: utf-8 -*- # Define your item pipelines here # # Don't forget to add your pipeline to the ITEM_PIPELINES setting # See: https://doc.scrapy.org/en/latest/topics/item-pipeline.html import json import datetime import re import pymongo import requests from collectSports.biz import getMongo from collectSports.utils.helper import Helper class SportslstPipeline(object): def open_spider(self, spider): self.host = '192.168.2.200' self.port = 27017 self.dbname = 'kaiyou' # 数据库名 self.client = pymongo.MongoClient(host=self.host, port=self.port) self.tdb = self.client[self.dbname] self.port = self.tdb['zq_league'] # 表名 # competition self.port2 = self.tdb['zq_competition'] # 表名 def process_item(self, item, spider): mongo = getMongo() url = 'http://admin.5gogo.com/setSports' # print(mongo) # zaopan = item['csource'] # if zaopan == 'zaopan': # zq_leagues = set(item['zq_league']) # zq_matchs = set(item['zq_match']) # for zq_league in zq_leagues: # mongo.changeSet('zq_league').insert(dict(zq_league)) # for zq_match in zq_matchs: # mongo.changeSet('zq_competition').insert(dict(zq_match)) # return item zb = item['csource'] if zb == 'zhibo': zb_lists = item['zq_zhibo'] # print(zb_lists) list = [] for i in zb_lists: # print(i) dic = {} dic['title'] = 'broadcast' dic['game_code'] = 'zq' # dic['data'] = i dic['data'] = {} dic['data']['doing'] = i['doing'] dic['data']['game_type'] = i['game_type'] dic['data']['guest_team'] = i['guest_team'] dic['data']['host_team'] = i['host_team'] dic['data']['league_name'] = i['league_name'] dic['data']['shower'] = i['shower'] dic['data']['showid'] = i['showid'] dic['data']['start_time'] = i['start_time'] # print(dic) if dic['data']['showid'] == mongo.changeSet('zq_zhibo').find({'showid': dic['data']['showid']}): mongo.changeSet('zq_zhibo').update(dic) else: mongo.changeSet('zq_zhibo').insert(dic) list.append(dic) # print(list) r = requests.post(url='http://admin.5gogo.com/setSports', data={"data": json.dumps(list)}) print(r.text) elif zb == 'zaopan': zq_leagues = set(item['zq_league']) zq_matchs = set(item['zq_match']) current_time = datetime.datetime.now() # 联赛 for zq_league in zq_leagues: league_list = [] payload_key = ['game_code', 'title'] payload_value = ['zq', 'league'] payload = Helper.get_zip_data(payload_key, payload_value) data_key = ['name_chinese', 'kind', 'match_mode', 'if_stop', 'belong', 'last_time', 'lg_id', 'source', 'uuid'] data_value = ['name_chinese', '0', '0', '0', '0', current_time.strftime("%Y-%m-%d %H:%M:%S"), zq_league.get('league_id'), 'hg0088', str(zq_league.get('uuid'))] data = Helper.get_zip_data(data_key, data_value) payload['data'] = data league_list.append(payload) response_data = Helper.post(url, league_list) print(response_data.text) mongo.changeSet('zq_league').insert(dict(zq_league)) # 赛事 for zq_match in zq_matchs: match_list = [] payload_key = ['game_code', 'title'] payload_value = ['zq', 'competition'] payload = Helper.get_zip_data(payload_key, payload_value) match_date = zq_match.get('mdate') match_time = re.search(r'(\d{1,2}):(\d{1,2})', zq_match.get('mtime')) # 日期时间地区差异转换 if 0 <= int(match_time.group(1)) <= 12: if int(match_time.group(1)) == 12: r_match_time = '00:' + match_time.group(2) else: r_match_time = str(int(match_time.group(1)) + 12) + ':' + match_time.group(2) else: m_time = int(match_time.group(1)) + 12 - 24 if m_time < 10: r_match_time = '0' + str(m_time) + ':' + match_time.group(2) else: r_match_time = str(m_time) + ':' + match_time.group(2) if int(match_time.group(1)) + 12 >= 24: match_date = (datetime.datetime.strptime(match_date, '%Y-%m-%d') + datetime.timedelta( days=1)).strftime('%Y-%m-%d') old_time = match_date + ' ' + match_time.group(0) + ':00' if current_time - datetime.datetime.strptime(old_time, "%Y-%m-%d %H:%M:%S") >= datetime.timedelta( hours=12): status = 1 else: status = 0 us_time = match_date + ' ' + r_match_time data_key = ['home_team', 'guest_team', 'lg_id', 'status', 'match_id', 'match_date', 'match_time', 'tag', 'source', 'is_rollball', 'is_today', 'is_morningplate', 'is_stringscene', 'us_time', 'uuid'] data_value = [zq_match.get('host_name'), zq_match.get('guest_name'), zq_match.get('league_id'), status, zq_match.get('match_id'), match_date, r_match_time, '0', 'hg0088', zq_match.get('is_roll'), '0', '0', '0', us_time, str(zq_match.get('uuid'))] data = Helper.get_zip_data(data_key, data_value) payload['data'] = data match_list.append(payload) response_data = Helper.post(url, match_list) print(response_data.text) mongo.changeSet('zq_competition').insert(dict(zq_match)) # 冠军 elif zb == 'chain': zq_chains = set(item['zq_chain']) current_time = datetime.datetime.now() for zq_chain in zq_chains: chain_list = [] payload_key = ['game_code', 'title'] payload_value = ['zq', 'odds'] payload = Helper.get_zip_data(payload_key, payload_value) data_key = ['match_id', 'lg_id', 'odds_code', 'status', 'sort', 'p_code', 'odds', 'condition', 'odds_only', 'sole', 'source', 'type', 'team'] data_value = [zq_chain.get('league_id'), zq_chain.get('league_id'), zq_chain.get('league_subname'), zq_chain.get('enabled'), '0', zq_chain.get('game_type'), zq_chain.get('odds'), '0', '0', '0', 'c22123', '0', zq_chain.get('name') ] data = Helper.get_zip_data(data_key, data_value) payload['data'] = data chain_list.append(payload) response_data = Helper.post(url, chain_list) print(response_data.text) mongo.changeSet('zq_chain').insert(dict(zq_chain)) return item def close_spider(self, spider): self.client.close() # def open_spider(self, spider): # self.host = '192.168.2.200' # self.port = 27017 # self.dbname = 'kaiyou' # 数据库名 # self.client = pymongo.MongoClient(host=self.host, port=self.port) # self.tdb = self.client[self.dbname] # self.port = self.tdb['zq_league'] # 表名 # # competition # self.port2 = self.tdb['zq_competition'] # 表名