# -*- coding: utf-8 -*- import copy import datetime import json import scrapy from scrapy.http import Request from ..items import Wqbodan class WgbodanSpider(scrapy.Spider): name = 'wqbodan' to_day = datetime.datetime.now() allowed_domains = ['hg3535z.com'] custom_settings = { "ITEM_PIPELINES": { 'hg3535.pipeline.Wqbodanpipeline': 300, }, # 'LOG_LEVEL': 'DEBUG', # 'LOG_FILE': "../hg3535/log/wqbodan_{}_{}_{}.log".format(to_day.year, to_day.month, to_day.day) } def start_requests(self): url = 'https://hg3535z.com/odds2/d/getmenu?pid=0' yield scrapy.Request(url=url, callback=self.parse_one, dont_filter=True) def parse_one(self, response): for y in range(1, 4): url = 'https://hg3535z.com/odds2/d/getodds?sid=3&pt=' + str(y) + '&ubt=stcs&pn=0&sb=2&dc=null&pid=0' yield Request(url=url, callback=self.parse, meta={'pt': y}, dont_filter=True) def parse(self, response): try: datas = json.loads(response.text) except: datas = "" try: results = datas['n-ot']['egs'] except: results = "" try: pt = copy.copy(response.meta['pt']) except: pt = 0 if results: for result in results: # 联赛id league_id = result['c']['k'] # 联赛名 league_name = result['c']['n'] new_results = result['es'] for new_result in new_results: item = Wqbodan() # 比赛id game_id = str(new_result['i'][16]) # 球队1 team_home = new_result['i'][0] # 球队2 team_guest = new_result['i'][1] # 数量(97>) number = new_result['i'][2] # 状态 zhuangtai = new_result['i'][3] # 日期 data_game = new_result['i'][4] # 开赛时间 time_game = new_result['i'][5] # 队1分数 score_home = new_result['i'][10] # 队2分数 score_guest = new_result['i'][11] # 下半场 half_way = new_result['i'][12] # 角球或者其他 corner_ball = new_result['pci'].get('ctn', "") bodan_data = {} # 波胆 bodans = new_result['o'].get('sb', "") if bodans: one_list = ["bodanhome_two_zero", "bodanhome_two_one", "bodanhome_three_zero", "bodanhome_three_one", "bodanhome_three_two", "bodanhome_four_zero", "bodanhome_four_one", "bodanhome_four_two", "bodanhome_four_three"] two_list = ["bodanguest_two_zero", "bodanguest_two_one", "bodanguest_three_zero", "bodanguest_three_one", "bodanguest_three_two", "bodanguest_four_zero", "bodanguest_four_one", "bodanguest_four_two", "bodanguest_four_three"] new_bodans = [bodans[i] for i in range(len(bodans)) if i % 2 == 1] # 主队bodan_home bodan_home = [new_bodans[i] for i in range(len(new_bodans)) if i % 2 == 0] for index, t in enumerate(one_list): bodan_data[t] = bodan_home[index] # 客队bodan_guest bodan_guest = [new_bodans[i] for i in range(len(new_bodans)) if i % 2 == 1] for y, z in enumerate(two_list): bodan_data[z] = bodan_guest[y] item['league_id'] = league_id item['league_name'] = league_name item['game_id'] = game_id item['team_home'] = team_home item['team_guest'] = team_guest item['number'] = number item['data_game'] = data_game item['time_game'] = time_game item['corner_ball'] = corner_ball item['score_home'] = score_home item['score_guest'] = score_guest item['half_way'] = half_way item['pt'] = pt item['zhuangtai'] = zhuangtai item['bodan_data'] = bodan_data yield item