# -*- coding: utf-8 -*- # Define your item pipelines here # # Don't forget to add your pipeline to the ITEM_PIPELINES setting # See: https://doc.scrapy.org/en/latest/topics/item-pipeline.html import json import datetime import re import pymongo import requests from collectSports.biz import getMongo from collectSports.utils.helper import Helper from pycomm.mongo import DBMongo class SportslstPipeline(object): # def open_spider(self, spider): # self.host = '192.168.2.200' # self.port = 27017 # self.dbname = 'kaiyou' # 数据库名 # self.client = pymongo.MongoClient(host=self.host, port=self.port) # self.tdb = self.client[self.dbname] # self.port = self.tdb['zq_league'] # 表名 # # competition # self.port2 = self.tdb['zq_competition'] # 表名 def process_item(self, item, spider): mongo = getMongo() print(mongo) csource = item['csource'] url = 'http://admin.5gogo.com/setSports' if csource == 'zaopan': zq_leagues = set(item['zq_league']) zq_matchs = set(item['zq_match']) current_time = datetime.datetime.now() # 联赛 for zq_league in zq_leagues: league_list = [] payload_key = ['game_code', 'title'] payload_value = ['zq', 'league'] payload = Helper.get_zip_data(payload_key, payload_value) data_key = ['name_chinese', 'kind', 'match_mode', 'if_stop', 'belong', 'last_time', 'lg_id', 'source', 'uuid'] data_value = ['name_chinese', '0', '0', '0', '0', current_time.strftime("%Y-%m-%d %H:%M:%S"), zq_league.get('league_id'), 'hg0088', str(zq_league.get('uuid'))] data = Helper.get_zip_data(data_key, data_value) payload['data'] = data league_list.append(payload) response_data = Helper.post(url, league_list) print(response_data.text) mongo.changeSet('zq_league').insert(dict(zq_league)) # 赛事 for zq_match in zq_matchs: match_list = [] payload_key = ['game_code', 'title'] payload_value = ['zq', 'competition'] payload = Helper.get_zip_data(payload_key, payload_value) match_date = zq_match.get('mdate') match_time = re.search(r'(\d{1,2}):(\d{1,2})', zq_match.get('mtime')) # 日期时间地区差异转换 if 0 <= int(match_time.group(1)) <= 12: if int(match_time.group(1)) == 12: r_match_time = '00:' + match_time.group(2) else: r_match_time = str(int(match_time.group(1)) + 12) + ':' + match_time.group(2) else: m_time = int(match_time.group(1)) + 12 - 24 if m_time < 10: r_match_time = '0' + str(m_time) + ':' + match_time.group(2) else: r_match_time = str(m_time) + ':' + match_time.group(2) if int(match_time.group(1)) + 12 >= 24: match_date = (datetime.datetime.strptime(match_date, '%Y-%m-%d') + datetime.timedelta( days=1)).strftime('%Y-%m-%d') old_time = match_date + ' ' + match_time.group(0) + ':00' if current_time - datetime.datetime.strptime(old_time, "%Y-%m-%d %H:%M:%S") >= datetime.timedelta( hours=12): status = 1 else: status = 0 us_time = match_date + ' ' + r_match_time data_key = ['home_team', 'guest_team', 'lg_id', 'status', 'match_id', 'match_date', 'match_time', 'tag', 'source', 'is_rollball', 'is_today', 'is_morningplate', 'is_stringscene', 'us_time', 'uuid'] data_value = [zq_match.get('host_name'), zq_match.get('guest_name'), zq_match.get('league_id'), status, zq_match.get('match_id'), match_date, r_match_time, '0', 'hg0088', zq_match.get('is_roll'), '0', '0', '0', us_time, str(zq_match.get('uuid'))] data = Helper.get_zip_data(data_key, data_value) payload['data'] = data match_list.append(payload) response_data = Helper.post(url, match_list) print(response_data.text) mongo.changeSet('zq_competition').insert(dict(zq_match)) # 冠军 elif csource == 'chain': zq_chains = set(item['zq_chain']) current_time = datetime.datetime.now() for zq_chain in zq_chains: chain_list = [] payload_key = ['game_code', 'title'] payload_value = ['zq', 'odds'] payload = Helper.get_zip_data(payload_key, payload_value) data_key = ['match_id', 'lg_id', 'odds_code', 'status', 'sort', 'p_code', 'odds', 'condition', 'odds_only', 'sole', 'source', 'type', 'team'] data_value = [zq_chain.get('league_id'), zq_chain.get('league_id'), zq_chain.get('league_subname'), zq_chain.get('enabled'), '0', zq_chain.get('game_type'), zq_chain.get('odds'), '0', '0', '0', 'c22123', '0', zq_chain.get('name') ] data = Helper.get_zip_data(data_key, data_value) payload['data'] = data chain_list.append(payload) response_data = Helper.post(url, chain_list) print(response_data.text) mongo.changeSet('zq_chain').insert(dict(zq_chain)) return item # def close_spider(self, spider): # self.client.close()