sports_collect/collectSports/pipelines/sportslst.py

# -*- coding: utf-8 -*-

# Define your item pipelines here
#
# Don't forget to add your pipeline to the ITEM_PIPELINES setting
# See: https://doc.scrapy.org/en/latest/topics/item-pipeline.html
import json

import datetime
import os
import re

import pymongo

import requests

from collectSports.biz import getMongo

from collectSports.utils.helper import Helper


class SportslstPipeline(object):
    def process_item(self, item, spider):
        mongo = getMongo()
        url = 'http://admin.5gogo.com/setSports'
        zb = item['csource']

        if zb == 'zhibo':
            zb_lists = item['zq_zhibo']
            # print(zb_lists)
            list = []
            for i in zb_lists:
                # print(i)
                dic = {}
                dic['title'] = 'broadcast'
                dic['game_code'] = 'zq'
                # dic['data'] = i
                dic['data'] = {}
                dic['data']['doing'] = i['doing']
                dic['data']['game_type'] = i['game_type']
                dic['data']['guest_team'] = i['guest_team']
                dic['data']['host_team'] = i['host_team']
                dic['data']['league_name'] = i['league_name']
                dic['data']['shower'] = i['shower']
                dic['data']['showid'] = i['showid']
                dic['data']['start_time'] = i['start_time']
                # print(dic)
                if dic['data']['showid'] == mongo.changeSet('zq_zhibo').find({'showid': dic['data']['showid']}):
                    mongo.changeSet('zq_zhibo').update(dic)
                else:
                    mongo.changeSet('zq_zhibo').insert(dic)
                    list.append(dic)
            # print(list)
            r = requests.post(url='http://admin.5gogo.com/setSports', data={"data": json.dumps(list)})
            print(r.text)

        elif zb == 'zaopan':
            zq_leagues = set(item['zq_league'])
            zq_matchs = set(item['zq_match'])
            current_time = datetime.datetime.now()
            # 联赛
            league_list = []
            for zq_league in zq_leagues:
                payload_key = ['game_code', 'title']
                payload_value = ['zq', 'league']
                payload = Helper.get_zip_data(payload_key, payload_value)
                data_key = ['name_chinese', 'kind', 'match_mode', 'if_stop', 'belong', 'last_time', 'lg_id',
                            'source', 'uuid']
                data_value = [zq_league.get('league_name'), '0', '0', '0', '0',
                              current_time.strftime("%Y-%m-%d %H:%M:%S"),
                              zq_league.get('league_id'), 'hg0088', str(zq_league.get('uuid'))]
                data = Helper.get_zip_data(data_key, data_value)
                payload['data'] = data
                league_list.append(payload)
                mongo.changeSet('zq_league').insert(dict(zq_league))
            if league_list:
                response_data = Helper.post(url, league_list)
                print(response_data.text)
            # 赛事
            match_list = []
            for zq_match in zq_matchs:
                payload_key = ['game_code', 'title']
                payload_value = ['zq', 'competition']
                payload = Helper.get_zip_data(payload_key, payload_value)
                match_date = zq_match.get('mdate')
                match_time = re.search(r'(\d{1,2}):(\d{1,2})', zq_match.get('mtime'))
                # 日期时间地区差异转换
                if 0 <= int(match_time.group(1)) <= 12:
                    if int(match_time.group(1)) == 12:
                        r_match_time = '00:' + match_time.group(2)
                    else:
                        r_match_time = str(int(match_time.group(1)) + 12) + ':' + match_time.group(2)
                else:
                    m_time = int(match_time.group(1)) + 12 - 24
                    if m_time < 10:
                        r_match_time = '0' + str(m_time) + ':' + match_time.group(2)
                    else:
                        r_match_time = str(m_time) + ':' + match_time.group(2)
                if int(match_time.group(1)) + 12 >= 24:
                    match_date = (datetime.datetime.strptime(match_date, '%Y-%m-%d') + datetime.timedelta(
                        days=1)).strftime('%Y-%m-%d')
                old_time = match_date + ' ' + match_time.group(0) + ':00'
                if current_time - datetime.datetime.strptime(old_time, "%Y-%m-%d %H:%M:%S") >= datetime.timedelta(
                        hours=12):
                    status = 1
                else:
                    status = 0
                us_time = match_date + ' ' + r_match_time
                data_key = ['home_team', 'guest_team', 'lg_id', 'status', 'match_id', 'match_date', 'match_time',
                            'tag', 'source', 'is_rollball', 'is_today', 'is_morningplate', 'is_stringscene', 'us_time',
                            'uuid']
                data_value = [zq_match.get('host_name'), zq_match.get('guest_name'), zq_match.get('league_id'),
                              status, zq_match.get('match_id'), match_date, r_match_time,
                              '0', 'hg0088', zq_match.get('is_roll'), '0', '0', '0', us_time, str(zq_match.get('uuid'))]
                data = Helper.get_zip_data(data_key, data_value)
                payload['data'] = data
                match_list.append(payload)
                mongo.changeSet('zq_competition').insert(dict(zq_match))
            if match_list:
                response_data = Helper.post(url, match_list)
                print(response_data.text)
        # 冠军
        elif zb == 'chain':
            zq_chains = set(item['zq_chain'])
            current_time = datetime.datetime.now()
            league_list = []
            chain_list = []
            for zq_chain in zq_chains:
                # 联赛
                if 'game_type' in zq_chain:
                    payload_key = ['game_code', 'title']
                    payload_value = ['zq', 'league']
                    payload = Helper.get_zip_data(payload_key, payload_value)
                    data_key = ['name_chinese', 'kind', 'match_mode', 'if_stop', 'identity', 'belong', 'last_time',
                                'lg_id',
                                'source', 'uuid']
                    data_value = [zq_chain.get('league_name'), '0', '0', '0', '0', '0',
                                  current_time.strftime("%Y-%m-%d %H:%M:%S"),
                                  zq_chain.get('league_id'), 'hg0088', str(zq_chain.get('uuid', ''))]
                    data = Helper.get_zip_data(data_key, data_value)
                    payload['data'] = data
                    league_list.append(payload)
                    mongo.changeSet('zq_league').insert(dict(zq_chain))
                # 冠军赔率
                payload_key = ['game_code', 'title']
                payload_value = ['zq', 'odds']
                payload = Helper.get_zip_data(payload_key, payload_value)
                data_key = ['match_id', 'lg_id', 'odds_code', 'status', 'sort', 'p_code', 'odds',
                            'condition', 'odds_only', 'sole', 'source', 'type', 'team', 'uuid']
                data_value = [zq_chain.get('league_id'), zq_chain.get('league_id'), zq_chain.get('league_subname'),
                              zq_chain.get('enabled'), '0', zq_chain.get('game_type'), zq_chain.get('odds'),
                              '0', '0', '0', 'hg0088', '1', zq_chain.get('name'), str(zq_chain.get('uuid', ''))
                              ]
                data = Helper.get_zip_data(data_key, data_value)
                payload['data'] = data
                chain_list.append(payload)
                mongo.changeSet('zq_chain').insert(dict(zq_chain))
            if league_list:
                response_data = Helper.post(url, league_list)
                print(response_data.text)
                print('request league successful')
            if chain_list:
                response_data = Helper.post(url, chain_list)
                print(response_data.text)
                print('request guanjun successful')

        return item

    def close_spider(self, spider):
        self.client.close()