# -*- coding: utf-8 -*- # Define your item pipelines here # # Don't forget to add your pipeline to the ITEM_PIPELINES setting # See: https://doc.scrapy.org/en/latest/topics/item-pipeline.html import json import pymongo import requests from collectSports.biz import getMongo from pycomm.mongo import DBMongo class SportslstPipeline(object): def open_spider(self, spider): self.host = '192.168.2.200' self.port = 27017 self.dbname = 'kaiyou' # 数据库名 self.client = pymongo.MongoClient(host=self.host, port=self.port) self.tdb = self.client[self.dbname] self.port = self.tdb['zq_league'] # 表名 # competition self.port2 = self.tdb['zq_competition'] # 表名 def process_item(self, item, spider): mongo = getMongo() # print(mongo) # zaopan = item['csource'] # if zaopan == 'zaopan': # zq_leagues = set(item['zq_league']) # zq_matchs = set(item['zq_match']) # for zq_league in zq_leagues: # mongo.changeSet('zq_league').insert(dict(zq_league)) # for zq_match in zq_matchs: # mongo.changeSet('zq_competition').insert(dict(zq_match)) # return item zb = item['csource'] if zb == 'zhibo': zb_lists = item['zq_zhibo'] # print(zb_lists) list = [] for i in zb_lists: # print(i) dic = {} dic['title'] = 'broadcast' dic['game_code'] = 'zq' # dic['data'] = i dic['data'] = {} dic['data']['doing'] = i['doing'] dic['data']['game_type'] = i['game_type'] dic['data']['guest_team'] = i['guest_team'] dic['data']['host_team'] = i['host_team'] dic['data']['league_name'] = i['league_name'] dic['data']['shower'] = i['shower'] dic['data']['showid'] = i['showid'] dic['data']['start_time'] = i['start_time'] # print(dic) if dic['data']['showid'] == mongo.changeSet('zq_zhibo').find({'showid': dic['data']['showid']}): mongo.changeSet('zq_zhibo').update(dic) else: mongo.changeSet('zq_zhibo').insert(dic) list.append(dic) # print(list) r = requests.post(url='http://admin.5gogo.com/setSports', data={"data": json.dumps(list)}) print(r.text) def close_spider(self, spider): self.client.close()