sports_collect/collectSports/pipelines/sportslst.py

# -*- coding: utf-8 -*-

# Define your item pipelines here
#
# Don't forget to add your pipeline to the ITEM_PIPELINES setting
# See: https://doc.scrapy.org/en/latest/topics/item-pipeline.html
import json
import datetime
import re

import pymongo

import requests

from collectSports.biz import getMongo
from collectSports.utils.helper import Helper
from pycomm.mongo import DBMongo


class SportslstPipeline(object):
    # def open_spider(self, spider):
    #     self.host = '192.168.2.200'
    #     self.port = 27017
    #     self.dbname = 'kaiyou'  # 数据库名
    #     self.client = pymongo.MongoClient(host=self.host, port=self.port)
    #     self.tdb = self.client[self.dbname]
    #     self.port = self.tdb['zq_league']  # 表名
    # #     competition
    #     self.port2 = self.tdb['zq_competition']  # 表名
    def process_item(self, item, spider):
        mongo = getMongo()
        print(mongo)
        csource = item['csource']
        url = 'http://admin.5gogo.com/setSports'
        if csource == 'zaopan':
            zq_leagues = set(item['zq_league'])
            zq_matchs = set(item['zq_match'])
            current_time = datetime.datetime.now()
            # 联赛
            for zq_league in zq_leagues:
                league_list = []
                payload_key = ['game_code', 'title']
                payload_value = ['zq', 'league']
                payload = Helper.get_zip_data(payload_key, payload_value)
                data_key = ['name_chinese', 'kind', 'match_mode', 'if_stop', 'belong', 'last_time', 'lg_id',
                            'source', 'uuid']
                data_value = ['name_chinese', '0', '0', '0', '0', current_time.strftime("%Y-%m-%d %H:%M:%S"),
                              zq_league.get('league_id'), 'hg0088', str(zq_league.get('uuid'))]
                data = Helper.get_zip_data(data_key, data_value)
                payload['data'] = data
                league_list.append(payload)
                response_data = Helper.post(url, league_list)
                print(response_data.text)
                mongo.changeSet('zq_league').insert(dict(zq_league))
            # 赛事
            for zq_match in zq_matchs:
                match_list = []
                payload_key = ['game_code', 'title']
                payload_value = ['zq', 'competition']
                payload = Helper.get_zip_data(payload_key, payload_value)
                match_date = zq_match.get('mdate')
                match_time = re.search(r'(\d{1,2}):(\d{1,2})', zq_match.get('mtime'))
                # 日期时间地区差异转换
                if 0 <= int(match_time.group(1)) <= 12:
                    if int(match_time.group(1)) == 12:
                        r_match_time = '00:' + match_time.group(2)
                    else:
                        r_match_time = str(int(match_time.group(1)) + 12) + ':' + match_time.group(2)
                else:
                    m_time = int(match_time.group(1)) + 12 - 24
                    if m_time < 10:
                        r_match_time = '0' + str(m_time) + ':' + match_time.group(2)
                    else:
                        r_match_time = str(m_time) + ':' + match_time.group(2)
                if int(match_time.group(1)) + 12 >= 24:
                    match_date = (datetime.datetime.strptime(match_date, '%Y-%m-%d') + datetime.timedelta(
                        days=1)).strftime('%Y-%m-%d')
                old_time = match_date + ' ' + match_time.group(0) + ':00'
                if current_time - datetime.datetime.strptime(old_time, "%Y-%m-%d %H:%M:%S") >= datetime.timedelta(
                        hours=12):
                    status = 1
                else:
                    status = 0
                us_time = match_date + ' ' + r_match_time
                data_key = ['home_team', 'guest_team', 'lg_id', 'status', 'match_id', 'match_date', 'match_time',
                            'tag', 'source', 'is_rollball', 'is_today', 'is_morningplate', 'is_stringscene', 'us_time',
                            'uuid']
                data_value = [zq_match.get('host_name'), zq_match.get('guest_name'), zq_match.get('league_id'),
                              status, zq_match.get('match_id'), match_date, r_match_time,
                              '0', 'hg0088', zq_match.get('is_roll'), '0', '0', '0', us_time, str(zq_match.get('uuid'))]
                data = Helper.get_zip_data(data_key, data_value)
                payload['data'] = data
                match_list.append(payload)
                response_data = Helper.post(url, match_list)
                print(response_data.text)
                mongo.changeSet('zq_competition').insert(dict(zq_match))
        # 冠军
        elif csource == 'chain':
            zq_chains = set(item['zq_chain'])
            current_time = datetime.datetime.now()
            for zq_chain in zq_chains:
                chain_list = []
                payload_key = ['game_code', 'title']
                payload_value = ['zq', 'odds']
                payload = Helper.get_zip_data(payload_key, payload_value)
                data_key = ['match_id', 'lg_id', 'odds_code', 'status', 'sort', 'p_code', 'odds',
                            'condition', 'odds_only', 'sole', 'source', 'type', 'team']
                data_value = [zq_chain.get('league_id'), zq_chain.get('league_id'), zq_chain.get('league_subname'),
                              zq_chain.get('enabled'), '0', zq_chain.get('game_type'), zq_chain.get('odds'),
                              '0', '0', '0', 'c22123', '0', zq_chain.get('name')
                              ]
                data = Helper.get_zip_data(data_key, data_value)
                payload['data'] = data
                chain_list.append(payload)
                response_data = Helper.post(url, chain_list)
                print(response_data.text)

                mongo.changeSet('zq_chain').insert(dict(zq_chain))
        return item

    # def close_spider(self, spider):
    #     self.client.close()