| 123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172 |
- # -*- coding: utf-8 -*-
- # Define your item pipelines here
- #
- # Don't forget to add your pipeline to the ITEM_PIPELINES setting
- # See: https://doc.scrapy.org/en/latest/topics/item-pipeline.html
- import json
- import pymongo
- import requests
- from collectSports.biz import getMongo
- from pycomm.mongo import DBMongo
- class SportslstPipeline(object):
- def open_spider(self, spider):
- self.host = '192.168.2.200'
- self.port = 27017
- self.dbname = 'kaiyou' # 数据库名
- self.client = pymongo.MongoClient(host=self.host, port=self.port)
- self.tdb = self.client[self.dbname]
- self.port = self.tdb['zq_league'] # 表名
- # competition
- self.port2 = self.tdb['zq_competition'] # 表名
- def process_item(self, item, spider):
- mongo = getMongo()
- # print(mongo)
- # zaopan = item['csource']
- # if zaopan == 'zaopan':
- # zq_leagues = set(item['zq_league'])
- # zq_matchs = set(item['zq_match'])
- # for zq_league in zq_leagues:
- # mongo.changeSet('zq_league').insert(dict(zq_league))
- # for zq_match in zq_matchs:
- # mongo.changeSet('zq_competition').insert(dict(zq_match))
- # return item
- zb = item['csource']
- if zb == 'zhibo':
- zb_lists = item['zq_zhibo']
- # print(zb_lists)
- list = []
- for i in zb_lists:
- # print(i)
- dic = {}
- dic['title'] = 'broadcast'
- dic['game_code'] = 'zq'
- # dic['data'] = i
- dic['data'] = {}
- dic['data']['doing'] = i['doing']
- dic['data']['game_type'] = i['game_type']
- dic['data']['guest_team'] = i['guest_team']
- dic['data']['host_team'] = i['host_team']
- dic['data']['league_name'] = i['league_name']
- dic['data']['shower'] = i['shower']
- dic['data']['showid'] = i['showid']
- dic['data']['start_time'] = i['start_time']
- # print(dic)
- if dic['data']['showid'] == mongo.changeSet('zq_zhibo').find({'showid': dic['data']['showid']}):
- mongo.changeSet('zq_zhibo').update(dic)
- else:
- mongo.changeSet('zq_zhibo').insert(dic)
- list.append(dic)
- # print(list)
- r = requests.post(url='http://admin.5gogo.com/setSports', data={"data": json.dumps(list)})
- print(r.text)
- def close_spider(self, spider):
- self.client.close()
|
