sports_collect/collectSports/pipelines/sportslst.py

# -*- coding: utf-8 -*-

# Define your item pipelines here
#
# Don't forget to add your pipeline to the ITEM_PIPELINES setting
# See: https://doc.scrapy.org/en/latest/topics/item-pipeline.html
import json
import pymongo

import requests

from collectSports.biz import getMongo
from pycomm.mongo import DBMongo


class SportslstPipeline(object):
    # def open_spider(self, spider):
    #     self.host = '192.168.2.200'
    #     self.port = 27017
    #     self.dbname = 'kaiyou'  # 数据库名
    #     self.client = pymongo.MongoClient(host=self.host, port=self.port)
    #     self.tdb = self.client[self.dbname]
    #     self.port = self.tdb['zq_league']  # 表名
    # #     competition
    #     self.port2 = self.tdb['zq_competition']  # 表名
    def process_item(self, item, spider):
        mongo = getMongo()
        print(mongo)
        zaopan = item['csource']
        if zaopan == 'zaopan':
            zq_leagues = set(item['zq_league'])
            zq_matchs = set(item['zq_match'])
            for zq_league in zq_leagues:
                mongo.changeSet('zq_league').insert(dict(zq_league))
            for zq_match in zq_matchs:
                mongo.changeSet('zq_competition').insert(dict(zq_match))
            # return item

    # def close_spider(self, spider):
    #     self.client.close()