import datetime import json import time import hashlib import redis import logging from twisted.enterprise import adbapi from scrapy.conf import settings # 生成哈希索引 sole from .items import Zuqiustatus def hash_func(match_id, odds_code, sort, p_id): m = hashlib.md5() a = str(match_id) + str(odds_code) + str(sort) + str(p_id) m.update(a.encode('utf-8')) c = m.hexdigest() return c # 生成odds_only哈希 def r_func(match_id, odds_code, sort, p_id, odd): m = hashlib.md5() a = str(match_id) + str(odds_code) + str(sort) + str(p_id) + str(odd) m.update(a.encode('utf-8')) c = m.hexdigest() return c # 转换成本地时间 def new_time(ctime): time1 = time.mktime(time.strptime(ctime, '%Y-%m-%d %H:%M:%S')) + 43200 time2 = time.localtime(time1) time3 = time.strftime('%Y-%m-%d %H:%M:%S', time2) time4 = time3 data_time = str(time4).split(" ") match_date = data_time[0] match_time = data_time[1] return match_date, match_time, time3 def out_time(ctime, i): ctime1 = datetime.datetime.strptime(ctime, "%Y-%m-%d %H:%M:%S") n_ctime = (ctime1 + datetime.timedelta(hours=i)).strftime("%Y-%m-%d %H:%M:%S") return n_ctime def fuhao(f): if f.startswith('+'): f = f.replace('+', '-') return f elif f.startswith('-'): f = f.replace('-', '+') return f else: return f # 篮球插入赔率 ----------------------------------------------------------------------------------------------------------- def one_intodb(data1, data2, data3, cursor, redis_db): if data1: for key, value in data1.items(): if value: for x, y in enumerate(value): new_hash = hash_func(match_id=data2['game_id'], odds_code=key, sort=x, p_id=data2['p_id']) r_hash = r_func(match_id=data2['game_id'], odds_code=key, sort=x, p_id=data2['p_id'],odd=y) odds_only = r_hash l_f = fuhao(str(data3[key][x])) if redis_db.hexists("hg3535_lanqiu", r_hash): pass else: redis_db.hset("hg3535_lanqiu", r_hash, 0) sql1 = "insert into st_lq_odds(lg_id, odds_code, match_id, ctime, utime, odds, p_id, p_code, sort, source, sole, condition, expire_time,odds_only) values (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s,%s) on conflict(sole) do update set utime=%s,odds=%s,expire_time=%s,odds_only=%s,condition=%s;" cursor.execute(sql1, ( int(data2['league_id']), key, int(data2['game_id']), data2['utime'], data2['utime'], y, data2['p_id'], data2["p_code"], x, data2["source"], new_hash, l_f, data2['expire_time'], odds_only,data2['utime'], y, data2['expire_time'], odds_only, l_f)) # 更新主队st_zq_odds_record表 sql2 = "insert into st_lq_odds_record(lg_id, odds_code, match_id, ctime, utime, odds, p_id, p_code, sort, source, condition,odds_only) values (%s, %s, %s, %s, %s, %s,%s, %s, %s, %s, %s,%s);" cursor.execute(sql2, ( int(data2['league_id']), key, int(data2['game_id']), data2['utime'], data2['utime'], y, data2['p_id'], data2["p_code"], x, data2["source"], l_f, odds_only)) def two_intodb(data1, data2, data3, cursor, redis_db): if data1: for key, value in data1.items(): if value: new_hash = hash_func(match_id=data2['game_id'], odds_code=data3 + key, sort=data2['sort'], p_id=data2['p_id']) r_hash = r_func(match_id=data2['game_id'], odds_code=data3 + key, sort=data2['sort'], p_id=data2['p_id'],odd=value) odds_only = r_hash if redis_db.hexists("hg3535_lanqiu", r_hash): pass else: redis_db.hset("hg3535_lanqiu", r_hash, 0) sql1 = "insert into st_lq_odds(lg_id, odds_code, match_id, ctime, utime,odds, p_id, p_code, sort, source, sole, condition, expire_time,odds_only) values (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s,%s) on conflict(sole) do update set utime=%s,odds=%s,expire_time=%s,odds_only=%s,condition=%s;" cursor.execute(sql1, ( int(data2['league_id']), data3, int(data2['game_id']), data2['utime'], data2['utime'], float(value), data2['p_id'], data2["p_code"], data2['sort'], data2["source"], new_hash, key, data2['expire_time'], odds_only,data2['utime'], value, data2['expire_time'], odds_only,key)) # 更新主队st_zq_odds_record表 sql2 = "insert into st_lq_odds_record(lg_id, odds_code, match_id, ctime, utime,odds, p_id, p_code, sort, source, condition,odds_only) values (%s, %s, %s, %s, %s, %s,%s, %s, %s, %s, %s,%s);" cursor.execute(sql2, ( int(data2['league_id']), data3, int(data2['game_id']), data2['utime'], data2['utime'], float(value), data2['p_id'], data2["p_code"], data2['sort'], data2["source"], key, odds_only)) def three_intodb(data1, data2, cursor, redis_db): if data1: for key, value in data1.items(): if value: new_hash = hash_func(match_id=data2['game_id'], odds_code=key, sort=data2['sort'], p_id=data2['p_id']) r_hash = r_func(match_id=data2['game_id'], odds_code=key, sort=data2['sort'], p_id=data2['p_id'],odd=value) odds_only = r_hash if redis_db.hexists("hg3535_lanqiu", r_hash): pass else: redis_db.hset("hg3535_lanqiu", r_hash, 0) sql1 = "insert into st_lq_odds(lg_id, odds_code, match_id, ctime, utime,odds, p_id, p_code, sort, source, sole, expire_time,odds_only) values (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s,%s, %s,%s) on conflict(sole) do update set utime=%s,odds=%s,expire_time=%s,odds_only=%s;" cursor.execute(sql1, ( int(data2['league_id']), key, int(data2['game_id']), data2['utime'], data2['utime'], float(value), data2['p_id'], data2["p_code"], data2['sort'], data2["source"], new_hash, data2['expire_time'], odds_only,data2['utime'], value, data2['expire_time'], odds_only)) # 更新主队st_zq_odds_record表 sql2 = "insert into st_lq_odds_record(lg_id, odds_code, match_id, ctime, utime,odds, p_id, p_code, sort, source,odds_only) values (%s, %s, %s, %s, %s, %s,%s, %s, %s, %s, %s);" cursor.execute(sql2, ( int(data2['league_id']), key, int(data2['game_id']), data2['utime'], data2['utime'], float(value), data2['p_id'], data2["p_code"], data2['sort'], data2["source"], odds_only)) # 插入篮球赛事表 def into_competition(data, cursor): Competition_sql = "insert into st_lq_competition(home_team, guest_team, lg_id, match_id, match_date, match_time, ctime, utime, tag, type, source,expire_time,is_today,is_morningplate,is_stringscene) values (%s, %s, %s, %s, %s, %s, %s, %s,%s, %s, %s, %s,%s, %s, %s) on conflict(match_id) do update set tag = %s,expire_time=%s,is_today=%s,is_morningplate=%s,is_stringscene=%s;" cursor.execute(Competition_sql, ( data['team_home'], data['team_guest'], data['league_id'], data['game_id'], data['match_date'], data['match_time'], data['utime'], data['utime'], data['number'], data['pt'], data["source"], data['expire_time'], data['is_today'], data['is_morningplate'], data['is_stringscene'], data['number'], data['expire_time'], data['is_today'], data['is_morningplate'], data['is_stringscene'])) # 插入足球赛事表---------------------------------------------------------------------------------------------------------- def zqone_competition(data, cursor): Competition_sql = "insert into st_zq_competition(home_team, guest_team, lg_id, match_id, match_date, match_time, ctime, utime, tag,source, expire_time,is_today,us_time) values (%s, %s, %s, %s, %s, %s, %s,%s, %s, %s,%s,%s,%s) on conflict(match_id) do update set tag = %s,expire_time=%s,is_today=%s,us_time=%s;" cursor.execute(Competition_sql, ( data['team_home'], data['team_guest'], data['league_id'], data['game_id'], data['match_date'], data['match_time'], data['utime'], data['utime'], data['number'], data["source"], data['expire_time'], data['is_today'], data['us_time'], data['number'], data['expire_time'], data['is_today'],data['us_time'])) def zqtwo_competition(data, cursor): Competition_sql = "insert into st_zq_competition(home_team, guest_team, lg_id, match_id, match_date, match_time, ctime, utime, tag,source, expire_time,is_morningplate,us_time) values (%s, %s, %s, %s, %s, %s, %s,%s, %s, %s,%s,%s,%s) on conflict(match_id) do update set tag = %s,expire_time=%s,is_morningplate=%s,us_time=%s;" cursor.execute(Competition_sql, ( data['team_home'], data['team_guest'], data['league_id'], data['game_id'], data['match_date'], data['match_time'], data['utime'], data['utime'], data['number'], data["source"], data['expire_time'], data['is_morningplate'], data['us_time'], data['number'], data['expire_time'], data['is_morningplate'], data['us_time'])) def zqthree_competition(data, cursor): Competition_sql = "insert into st_zq_competition(home_team, guest_team, lg_id, match_id, match_date, match_time, ctime, utime, tag,source, expire_time,is_stringscene,us_time) values (%s, %s, %s, %s, %s, %s, %s,%s, %s, %s, %s,%s,%s) on conflict(match_id) do update set tag = %s,expire_time=%s,is_stringscene=%s,us_time=%s;" cursor.execute(Competition_sql, ( data['team_home'], data['team_guest'], data['league_id'], data['game_id'], data['match_date'], data['match_time'], data['utime'], data['utime'], data['number'], data["source"], data['expire_time'], data['is_stringscene'], data['us_time'], data['number'], data['expire_time'], data['is_stringscene'], data['us_time'])) # 足球 插入赔率 让球 大小------------------------------------------------------------------------------------------------- def zqone_intodb(data1, data2, data3, data4, cursor, redis_db): if data1: for index, value in enumerate(data1): if value: new_hash = hash_func(match_id=data2['game_id'], odds_code=data3, sort=index, p_id=data2['p_id']) r_hash = r_func(match_id=data2['game_id'], odds_code=data3, sort=index, p_id=data2['p_id'], odd=value) odds_only = r_hash f1 = fuhao(str(data4[index])) if redis_db.hexists("hg3535_zuqiu", r_hash): pass else: redis_db.hset("hg3535_zuqiu", r_hash, 0) sql1 = "insert into st_zq_odds(lg_id, odds_code, match_id, ctime, utime, odds, p_id, p_code, sort, source, sole, condition, expire_time,odds_only) values (%s,%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s) on conflict(sole) do update set utime=%s,odds=%s,expire_time=%s,odds_only=%s,condition=%s;" cursor.execute(sql1, ( int(data2['league_id']), data3, int(data2['game_id']), data2['utime'], data2['utime'], float(value), data2['p_id'], data2["p_code"], index, data2["source"], new_hash, f1, data2['expire_time'], odds_only,data2['utime'], value, data2['expire_time'], odds_only, f1)) # 更新主队st_zq_odds_record表 sql2 = "insert into st_zq_odds_record(lg_id, odds_code, match_id, ctime, utime, odds, p_id, p_code, sort, source, condition,odds_only) values (%s, %s, %s, %s, %s, %s, %s,%s, %s, %s, %s, %s);" cursor.execute(sql2, ( int(data2['league_id']), data3, int(data2['game_id']), data2['utime'], data2['utime'], float(value), data2['p_id'], data2["p_code"], index, data2["source"], f1, odds_only)) # 足球 插入赔率 独赢 单双 def zqtwo_intodb(data1, data2, data3, data4, cursor, redis_db): if data1: new_hash = hash_func(match_id=data2['game_id'], odds_code=data3, sort=data2['sort'], p_id=data2['p_id']) r_hash = r_func(match_id=data2['game_id'], odds_code=data3, sort=data2['sort'], p_id=data2['p_id'], odd=data1) odds_only = r_hash if redis_db.hexists("hg3535_zuqiu", r_hash): pass else: redis_db.hset("hg3535_zuqiu", r_hash, 0) sql1 = "insert into st_zq_odds(lg_id, odds_code, match_id, ctime, utime, odds, p_id, p_code, sort, source, sole, condition, expire_time, odds_only) values (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s) on conflict(sole) do update set utime=%s,odds=%s,expire_time=%s,odds_only=%s,condition=%s;" cursor.execute(sql1, ( int(data2['league_id']), data3, int(data2['game_id']), data2['utime'], data2['utime'], data1, data2['p_id'], data2["p_code"], data2['sort'], data2["source"], new_hash, data4, data2['expire_time'], odds_only, data2['utime'], data1, data2['expire_time'], odds_only,data4)) # 更新主队st_zq_odds_record表 sql2 = "insert into st_zq_odds_record(lg_id, odds_code, match_id, ctime, utime, odds, p_id, p_code, sort, source, condition,odds_only) values (%s,%s, %s, %s, %s, %s, %s,%s, %s, %s, %s, %s);" cursor.execute(sql2, ( int(data2['league_id']), data3, int(data2['game_id']), data2['utime'], data2['utime'], data1, data2['p_id'], data2["p_code"], data2['sort'], data2["source"], data4, odds_only)) def gjz_into(data, cursor, r_hash, s_hash, redis_db): if redis_db.hexists('hg3535_guanjun', r_hash): pass else: redis_db.hset('hg3535_guanjun', r_hash, 0) # if pt == 1: sql = "insert into st_zq_odds(lg_id, odds_code, match_id, ctime, utime,odds, p_id, p_code, team, type, source, sole, expire_time,odds_only) values (%s, %s, %s, %s, %s, %s, %s, %s,%s, %s,%s, %s, %s, %s) on conflict(sole) do update set utime = %s,odds = %s,expire_time=%s,odds_only=%s;" cursor.execute(sql, ( int(data['league_id']), data['new_champion'], int(data['game_id']), data['utime'], data['utime'], data['champion_team'], data['p_id'], data['new_league_name'], data['tema_home'], 1, "hg3535", data['new_hash'], data['expire_time'], data['odds_only'],data['utime'], data['champion_team'], data['expire_time'], data['odds_only'])) # 插入st_zq_odds_record表 new_sql = "insert into st_zq_odds_record(lg_id, odds_code, match_id, ctime, utime, odds, p_id, p_code, team, type, source,odds_only) values (%s, %s, %s, %s, %s, %s, %s, %s,%s, %s, %s, %s)" cursor.execute(new_sql, ( int(data['league_id']), data['new_champion'], int(data['game_id']), data['utime'], data['utime'], data['champion_team'], data['p_id'], data['new_league_name'], data['tema_home'], 1, "hg3535", data['odds_only'])) if redis_db.hexists('hg3535_liansai', s_hash): pass else: redis_db.hset('hg3535_liansai', s_hash, 0) league_sql = "insert into st_zq_league(name_chinese, lg_id,last_time) values (%s, %s, %s) on conflict(lg_id) do update set last_time = %s,source=%s;" cursor.execute(league_sql, (data['league_name'], data['league_id'], data['time3'], data['time3'], "hg3535")) def gjl_into(data, cursor, r_hash, s_hash, redis_db): if redis_db.hexists('hg3535_guanjun', r_hash): pass else: redis_db.hset('hg3535_guanjun', r_hash, 0) sql = "insert into st_lq_odds(lg_id, odds_code, match_id, ctime, utime,odds, p_id, p_code, team, type, source, sole, expire_time,odds_only) values (%s, %s, %s, %s, %s, %s, %s, %s,%s, %s,%s, %s, %s, %s) on conflict(sole) do update set utime = %s,odds = %s,expire_time=%s,odds_only=%s;" cursor.execute(sql, ( int(data['league_id']), data['new_champion'], int(data['game_id']), data['utime'], data['utime'], data['champion_team'], data['p_id'], data['new_league_name'], data['tema_home'], 1, "hg3535", data['new_hash'], data['expire_time'], data['odds_only'],data['utime'], data['champion_team'], data['expire_time'], data['odds_only'])) # 插入st_zq_odds_record表 new_sql = "insert into st_lq_odds_record(lg_id, odds_code, match_id, ctime, utime, odds, p_id, p_code, team, type, source,odds_only) values (%s, %s, %s, %s, %s, %s, %s, %s,%s, %s, %s, %s)" cursor.execute(new_sql, ( int(data['league_id']), data['new_champion'], int(data['game_id']), data['utime'], data['utime'], data['champion_team'], data['p_id'], data['new_league_name'], data['tema_home'], 1, "hg3535", data['odds_only'])) # 插入st_zq_league表 if redis_db.hexists('hg3535_liansai', s_hash): pass else: redis_db.hset('hg3535_liansai', s_hash, 0) league_sql = "insert into st_lq_league(name_chinese, lg_id,last_time) values (%s, %s, %s) on conflict(lg_id) do update set last_time = %s,source=%s;" cursor.execute(league_sql, (data['league_name'], data['league_id'], data['time3'], data['time3'], "hg3535")) def gjw_into(data, cursor, r_hash, s_hash, redis_db): if redis_db.hexists('hg3535_guanjun', r_hash): pass else: redis_db.hset('hg3535_guanjun', r_hash, 0) sql = "insert into st_wq_odds(lg_id, odds_code, match_id, ctime, utime,odds, p_id, p_code, team, type, source, sole, expire_time,odds_only) values (%s, %s, %s, %s, %s, %s, %s, %s,%s, %s,%s, %s, %s, %s) on conflict(sole) do update set utime = %s,odds = %s,expire_time=%s,odds_only=%s;" cursor.execute(sql, ( int(data['league_id']), data['new_champion'], int(data['game_id']), data['utime'], data['utime'], data['champion_team'], data['p_id'], data['new_league_name'], data['tema_home'], 1, "hg3535", data['new_hash'], data['expire_time'], data['odds_only'],data['utime'], data['champion_team'], data['expire_time'], data['odds_only'])) # 插入st_zq_odds_record表 new_sql = "insert into st_wq_odds_record(lg_id, odds_code, match_id, ctime, utime, odds, p_id, p_code, team, type, source,odds_only) values (%s,%s, %s, %s, %s, %s, %s, %s, %s,%s, %s, %s)" cursor.execute(new_sql, ( int(data['league_id']), data['new_champion'], int(data['game_id']), data['utime'], data['utime'], data['champion_team'], data['p_id'], data['new_league_name'], data['tema_home'], 1, "hg3535", data['odds_only'])) # 插入st_zq_league表 if redis_db.hexists('hg3535_liansai', s_hash): pass else: redis_db.hset('hg3535_liansai', s_hash, 0) league_sql = "insert into st_wq_league(name_chinese, lg_id,last_time) values (%s, %s, %s) on conflict(lg_id) do update set last_time = %s,source=%s;" cursor.execute(league_sql, (data['league_name'], data['league_id'], data['time3'], data['time3'], "hg3535")) def gjb_into(data, cursor, r_hash, s_hash, redis_db): if redis_db.hexists('hg3535_guanjun', r_hash): pass else: redis_db.hset('hg3535_guanjun', r_hash, 0) sql = "insert into st_bq_odds(lg_id, odds_code, match_id, ctime, utime,odds, p_id, p_code, team, type, source, sole, expire_time,odds_only) values (%s, %s, %s, %s, %s, %s, %s, %s,%s, %s,%s, %s, %s, %s) on conflict(sole) do update set utime = %s,odds = %s,expire_time=%s,odds_only=%s;" cursor.execute(sql, ( int(data['league_id']), data['new_champion'], int(data['game_id']), data['utime'], data['utime'], data['champion_team'], data['p_id'], data['new_league_name'], data['tema_home'], 1, "hg3535", data['new_hash'], data['expire_time'], data['odds_only'],data['utime'], data['champion_team'], data['expire_time'], data['odds_only'])) # 插入st_zq_odds_record表 new_sql = "insert into st_bq_odds_record(lg_id, odds_code, match_id, ctime, utime, odds, p_id, p_code, team, type, source,odds_only) values (%s, %s, %s, %s, %s, %s, %s, %s,%s, %s, %s, %s)" cursor.execute(new_sql, ( int(data['league_id']), data['new_champion'], int(data['game_id']), data['utime'], data['utime'], data['champion_team'], data['p_id'], data['new_league_name'], data['tema_home'], 1, "hg3535", data['odds_only'])) # 插入st_zq_league表 if redis_db.hexists('hg3535_liansai', s_hash): pass else: redis_db.hset('hg3535_liansai', s_hash, 0) league_sql = "insert into st_bq_league(name_chinese, lg_id,last_time) values (%s, %s, %s) on conflict(lg_id) do update set last_time = %s,source=%s;" cursor.execute(league_sql, (data['league_name'], data['league_id'], data['time3'], data['time3'], "hg3535")) # 网球 插入赔率---------------------------------------------------------------------------------------------------------- def wqone_intodb(data1, data2, data3, cursor, redis_db): if data1: for key, value in data1.items(): if value: new_hash = hash_func(match_id=data2['game_id'], odds_code=key, sort=data2['sort'], p_id=data2['p_id']) r_hash = r_func(match_id=data2['game_id'], odds_code=key, sort=data2['sort'], p_id=data2['p_id'], odd=value) odds_only = r_hash w_f = fuhao(str(data3[key])) if redis_db.hexists("hg3535_wangqiu", r_hash): pass else: redis_db.hset("hg3535_wangqiu", r_hash, 0) sql1 = "insert into st_wq_odds(lg_id, odds_code, match_id, ctime, utime, odds, p_id, p_code, sort, source, sole, condition, expire_time,odds_only) values (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s,%s) on conflict(sole) do update set utime=%s,odds=%s,expire_time=%s,odds_only=%s,condition=%s;" cursor.execute(sql1, ( int(data2['league_id']), key, int(data2['game_id']), data2['utime'], data2['utime'], value, data2['p_id'], data2["p_code"], data2['sort'], data2["source"], new_hash, w_f, data2['expire_time'], odds_only,data2['utime'], value, data2['expire_time'], odds_only, w_f)) # 更新主队st_zq_odds_record表 sql2 = "insert into st_wq_odds_record(lg_id, odds_code, match_id, ctime, utime, odds, p_id, p_code, sort, source, condition,odds_only) values (%s, %s, %s, %s, %s, %s, %s,%s, %s, %s, %s, %s);" cursor.execute(sql2, ( int(data2['league_id']), key, int(data2['game_id']), data2['utime'], data2['utime'], value, data2['p_id'], data2["p_code"], data2['sort'], data2["source"], w_f, odds_only)) # 棒球 插入赔率---------------------------------------------------------------------------------------------------------- def bqone_intodb(data1, data2, data3, cursor, redis_db): if data1: for key, value in data1.items(): if value: new_hash = hash_func(match_id=data2['game_id'], odds_code=key, sort=data2['sort'], p_id=data2['p_id']) r_hash = r_func(match_id=data2['game_id'], odds_code=key, sort=data2['sort'], p_id=data2['p_id'], odd=value) odds_only = r_hash b_f = fuhao(str(data3[key])) if redis_db.hexists("hg3535_bangqiu", r_hash): pass else: redis_db.hset("hg3535_bangqiu", r_hash, 0) sql1 = "insert into st_bq_odds(lg_id, odds_code, match_id, ctime, utime, odds, p_id, p_code, sort, source, sole, condition, expire_time,odds_only) values (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s,%s) on conflict(sole) do update set utime=%s,odds=%s,expire_time=%s,odds_only=%s,condition=%s" cursor.execute(sql1, ( int(data2['league_id']), key, int(data2['game_id']), data2['utime'], data2['utime'], value, data2['p_id'], data2["p_code"], data2['sort'], data2["source"], new_hash, b_f, data2['expire_time'], odds_only, data2['utime'], value,data2['expire_time'], odds_only, b_f)) # 更新主队st_zq_odds_record表 sql2 = "insert into st_bq_odds_record(lg_id, odds_code, match_id, ctime, utime, odds, p_id, p_code, sort, source, condition,odds_only) values (%s, %s, %s, %s, %s, %s,%s, %s, %s, %s, %s, %s);" cursor.execute(sql2, ( int(data2['league_id']), key, int(data2['game_id']), data2['utime'], data2['utime'], value, data2['p_id'], data2["p_code"], data2['sort'], data2["source"], b_f, odds_only)) def get_pcode(corner_ball, code): code_dict = {'concede_size': 1, 'capot': 2, 'two_sides': 3, 'total_goal': 4, 'half_full': 5, 'bodan': 6, 'first_last_ball': 7, 'temaball': 11} if corner_ball == "角球": p_code = "corner_ball" p_id = 9 elif corner_ball == "会晋级": p_code = "promotion" p_id = 10 elif corner_ball == "罚牌数": p_code = "Penalty_card" p_id = 12 else: p_code = code p_id = code_dict[code] return p_code, p_id # 足球 赔率 赛事--------------------------------------------------------------------------------------------------------- class Zuqiupipeline(object): def open_spider(self, spider): self.dbpool = adbapi.ConnectionPool("psycopg2",host=settings["POST_HOST"], port=settings['POST_PORT'], user=settings["POST_USER"], password=settings["POST_PASSWORD"], database=settings["POST_DATABASE"]) self.pool = redis.ConnectionPool(host=settings["R_HOST"], port=settings["R_POST"], password=settings["R_PASSWORD"]) self.redis_db = redis.StrictRedis(connection_pool=self.pool) def process_item(self, item, spider): # 使用twisted将mysql插入变成异步执行 # logger = logging.getLogger(__name__) # logger.warning(query.addErrback(self.handle_error, item, spider)) query = self.dbpool.runInteraction(self.do_insert, item) query.addErrback(self.handle_error, item, spider) # 处理异常 return item def handle_error(self, failure, item, spider): # 处理异步插入的异常 logger = logging.getLogger(__name__) logger.warning(failure) # print(failure) def do_insert(self, cursor, item): # 比赛日期 data_game = item['data_game'].split("/") month = str(data_game[1].strip()) day = str(data_game[0]) # 比赛时间 time_game = str(item['time_game']) # 比赛时间，时间戳 ctime = "2019" + "-" + month + "-" + day + "" + time_game + ":00" r_ctime = "2019" + "-" + month + "-" + day # 现在时间，时间戳 utime = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime()) expire_time = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(time.time() + 60)) # 比赛id competition_id = item['game_id'] # 联赛id league_id = item['league_id'] # 联赛name league_name = item['league_name'] # # 主队 team_home = item['team_home'] # 客队 team_guest = item['team_guest'] # number number = item['number'] pt = item['pt'] corner_ball = item['corner_ball'] p_code, p_id = get_pcode(corner_ball, 'concede_size') # 构建唯一索引 half_size_guest = item["half_size_guest"] half_size_guest_rule = item["half_size_guest_rule"] half_size_home = item["half_size_home"] half_size_home_rule = item["half_size_home_rule"] zuqiu = item['zuqiu'] size_data = {'league_id': league_id, 'game_id': competition_id, 'utime': utime, 'p_id': p_id, 'p_code': p_code, 'source': "hg3535", 'expire_time': expire_time, 'pt': pt} # 让球 数据插入数据库 # try: zqone_intodb(data1=half_size_home, data2=size_data, data3="half_size_home", data4=half_size_home_rule, cursor=cursor, redis_db=self.redis_db) # except: # print(item) # # 让球 数据插入数据库 # try: zqone_intodb(data1=half_size_guest, data2=size_data, data3="half_size_guest", data4=half_size_guest_rule, cursor=cursor, redis_db=self.redis_db) # except: # print(item) # 全场场大小 size_guest = item["size_guest"] size_guest_rule = item["size_guest_rule"] size_home = item["size_home"] size_home_rule = item["size_home_rule"] zqone_intodb(data1=size_guest, data2=size_data, data3="size_guest", data4=size_guest_rule, cursor=cursor, redis_db=self.redis_db) zqone_intodb(data1=size_home, data2=size_data, data3="size_home", data4=size_home_rule, cursor=cursor, redis_db=self.redis_db) # 上半场大小 half_concede_home_rule = item["half_concede_home_rule"] half_concede_home = item["half_concede_home"] half_concede_guest_rule = item["half_concede_guest_rule"] half_concede_guest = item["half_concede_guest"] # 上半场让球 zqone_intodb(data1=half_concede_home, data2=size_data, data3="half_concede_home", data4=half_concede_home_rule, cursor=cursor, redis_db=self.redis_db) zqone_intodb(data1=half_concede_guest, data2=size_data, data3="half_concede_guest", data4=half_concede_guest_rule, cursor=cursor, redis_db=self.redis_db) concede_guest = item["concede_guest"] concede_guest_rule = item["concede_guest_rule"] concede_home = item["concede_home"] concede_home_rule = item["concede_home_rule"] # 全场让球 zqone_intodb(data1=concede_guest, data2=size_data, data3="concede_guest", data4=concede_guest_rule, cursor=cursor, redis_db=self.redis_db) zqone_intodb(data1=concede_home, data2=size_data, data3="concede_home", data4=concede_home_rule, cursor=cursor,redis_db=self.redis_db) # p_code, p_id = get_pcode(corner_ball, 'capot') # 独赢---------------------------------------------------------------------------------------------------------- half_capot_home = item["half_capot_home"] half_capot_guest = item["half_capot_guest"] half_capot_dogfall = item["half_capot_dogfall"] capot_home = item["capot_home"] capot_guest = item["capot_guest"] capot_dogfall = item["capot_dogfall"] capot_data = {'league_id': league_id, 'game_id': competition_id, 'utime': utime, 'p_id': p_id, 'p_code': p_code, 'source': "hg3535", 'expire_time': expire_time, 'sort': 0} # 上半场独赢 主队 zqtwo_intodb(data1=half_capot_home, data2=capot_data, data3="half_capot_home", data4='1', cursor=cursor,redis_db=self.redis_db) # 上半场独赢 客队 zqtwo_intodb(data1=half_capot_guest, data2=capot_data, data3="half_capot_guest", data4='2', cursor=cursor,redis_db=self.redis_db) # 上半场独赢 和 zqtwo_intodb(data1=half_capot_dogfall, data2=capot_data, data3="half_capot_dogfall", data4='x', cursor=cursor,redis_db=self.redis_db) # # 全场独赢 主队 zqtwo_intodb(data1=capot_home, data2=capot_data, data3="capot_home", data4='1', cursor=cursor,redis_db=self.redis_db) # 全场独赢 客队 zqtwo_intodb(data1=capot_guest, data2=capot_data, data3="capot_guest", data4='2', cursor=cursor,redis_db=self.redis_db) # 全场独赢 和 zqtwo_intodb(data1=capot_dogfall, data2=capot_data, data3="capot_dogfall", data4='x', cursor=cursor,redis_db=self.redis_db) # # 入球数单双------------------------------------------------------------------------------------------------------------- p_code, p_id = get_pcode(corner_ball, 'two_sides') odd_even_odd = item["odd_even_odd"] odd_even_even = item["odd_even_even"] half_odd_even_odd = item["half_odd_even_odd"] half_odd_even_even = item["half_odd_even_even"] # 全场入球数 单双 two_sides_data = {'league_id': league_id, 'game_id': competition_id, 'utime': utime, 'p_id': p_id, 'p_code': p_code, 'source': "hg3535", 'expire_time': expire_time, 'sort': 0} # 上半场入球数 单双 zqtwo_intodb(data1=odd_even_odd, data2=two_sides_data, data3="two_sides_single", data4='单', cursor=cursor,redis_db=self.redis_db) zqtwo_intodb(data1=odd_even_even, data2=two_sides_data, data3="two_sides_double", data4='双', cursor=cursor,redis_db=self.redis_db) # 全场入球数 单双 zqtwo_intodb(data1=half_odd_even_odd, data2=two_sides_data, data3="half_two_sides_single", data4='单', cursor=cursor, redis_db=self.redis_db) zqtwo_intodb(data1=half_odd_even_even, data2=two_sides_data, data3="half_two_sides_double", data4='双', cursor=cursor, redis_db=self.redis_db) # 总入球数 -------------------------------------------------------------------------------------------------------------- p_code, p_id = get_pcode(corner_ball, 'total_goal') total_goals = item['total_goal'] total_dict = {'total_goal_zero': '0-1', 'total_goal_two': '2-3', 'total_goal_four': '4-6', 'total_goal_seven': '7或以上', 'half_total_goal_zero': '0', "half_total_goal_one": '1', "half_total_goal_two": '2', "half_total_goal_three": '3或以上'} # 全场入球数 单双 total_goal_data = {'league_id': league_id, 'game_id': competition_id, 'utime': utime, 'p_id': p_id, 'p_code': p_code, 'source': "hg3535", 'expire_time': expire_time, 'sort': 0} # 上半场入球数 单双 for key, value in total_goals.items(): if value: zqtwo_intodb(data1=value, data2=total_goal_data, data3=key, data4=total_dict[key], cursor=cursor, redis_db=self.redis_db) # 全场半场 -------------------------------------------------------------------------------------------------------------- half_fulls = item['half_full'] p_code, p_id = get_pcode(corner_ball, 'half_full') full_dict = {"half_full_home_home": "主主", "half_full_home_dogfall": "主和", "half_full_home_guest": "主客", "half_full_dogfall_home": "和主", "half_full_dogfall_dogfall": "和和", "half_full_dogfall_guest": "和客", "half_full_guest_home": "客主", "half_full_guest_dogfall": "客和", "half_full_guest_guest": "客客"} half_full_data = {'league_id': league_id, 'game_id': competition_id, 'utime': utime, 'p_id': p_id, 'p_code': p_code, 'source': "hg3535", 'expire_time': expire_time, 'sort': 0} if half_fulls: for key, value in half_fulls.items(): if value: zqtwo_intodb(data1=value, data2=half_full_data, data3=key, data4=full_dict[key], cursor=cursor, redis_db=self.redis_db) # 波胆------------------------------------------------------------------------------------------------------------------ bodan_datas = item['bodan_data'] p_code, p_id = get_pcode(corner_ball, 'bodan') bodan_dict = {"bodanhome_one_zero": "1-0", "bodanhome_two_zero": "2-0", "bodanhome_two_one": "2-1", "bodanhome_three_zero": "3-0", "bodanhome_three_one": "3-1", "bodanhome_three_two": "3-2", "bodanhome_four_zero": "4-0", "bodanhome_four_one": "4-1", "bodanhome_four_two": "4-2", "bodanhome_four_three": "4-3", "bodanguest_one_zero": "0-1", "bodanguest_two_zero": "0-2", "bodanguest_two_one": "1-2", "bodanguest_three_zero": "0-3", "bodanguest_three_one": "1-3", "bodanguest_three_two": "2-3", "bodanguest_four_zero": "0-4", "bodanguest_four_one": "1-4", "bodanguest_four_two": "2-4", "bodanguest_four_three": "3-4", "bodandogfall_zero_zero": "0-0", "bodandogfall_one_one": "1-1", "bodandogfall_two_two": "2-2", "bodandogfall_three_three": "3-3", "bodandogfall_four_four": "4-4", "bodanother": "其他", "halfbodanhome_one_zero": "1-0", "halfbodanhome_two_zero": "2-0", "halfbodanhome_two_one": "2-1", "halfbodanhome_three_zero": "3-0", "halfbodanhome_three_one": "3-1", "halfbodanhome_three_two": "3-2", "halfbodanguest_one_zero": "0-1", "halfbodanguest_two_zero": "0-2", "halfbodanguest_two_one": "1-2", "halfbodanguest_three_zero": "0-3", "halfbodanguest_three_one": "1-3", "halfbodanguest_three_two": "2-3", "halfbodandogfall_zero_zero": "0-0", "halfbodandogfall_one_one": "1-1", "halfbodandogfall_two_two": "2-2", "halfbodandogfall_three_three": "3-3", "halfbodanother": "其他"} bodan_data = {'league_id': league_id, 'game_id': competition_id, 'utime': utime, 'p_id': p_id, 'p_code': p_code, 'source': "hg3535", 'expire_time': expire_time, 'sort': 0} if bodan_datas: for key, value in bodan_datas.items(): zqtwo_intodb(data1=value, data2=bodan_data, data3=key, data4=bodan_dict[key], cursor=cursor, redis_db=self.redis_db) # 最先进球/最后进球 ------------------------------------------------------------------------------------------------------ first_last_balls = item['first_last_ball'] p_code, p_id = get_pcode(corner_ball, 'first_last_ball') first_last_dict = {"first_last_ball": "最先进球", "first_ball_home": "最先进球", "first_ball_guest": "最先进球", "last_ball_home": "最后进球", "last_ball_guest": "最后进球", "not_ball": "没有进球"} first_last_data = {'league_id': league_id, 'game_id': competition_id, 'utime': utime, 'p_id': p_id, 'p_code': p_code, 'source': "hg3535", 'expire_time': expire_time, 'sort': 0} if first_last_balls: for key, value in first_last_balls.items(): # 构建唯一索引 zqtwo_intodb(data1=value, data2=first_last_data, data3=key, data4=first_last_dict[key], cursor=cursor, redis_db=self.redis_db) p_code, p_id = get_pcode(corner_ball, 'temaball') full_dicts = item['full_data'] half_dicts = item['half_data'] full_dict_rules = item['full_data_rule'] half_dict_rules = item['half_data_rule'] data = {'league_id': league_id, 'game_id': competition_id, 'utime': utime, 'p_id': p_id, 'p_code': p_code, 'source': "hg3535", 'expire_time': expire_time, 'sort': 0} if full_dicts: for key, value in full_dicts.items(): zqtwo_intodb(data1=value, data2=data, data3=key, data4=full_dict_rules[key], cursor=cursor, redis_db=self.redis_db) if half_dicts: for key, value in half_dicts.items(): zqtwo_intodb(data1=value, data2=data, data3=key, data4=half_dict_rules[key], cursor=cursor, redis_db=self.redis_db) match_date, match_time, time3 = new_time(ctime) n_time = out_time(time3, 1.5) # 插入st_zq_competition表 if zuqiu == "足球": if pt == 1: data_competition = {'team_home': team_home, 'team_guest': team_guest, 'league_id': league_id, 'game_id': competition_id, 'match_date': match_date, 'match_time': match_time, 'utime': utime, 'number': number, 'source': "hg3535", "expire_time": n_time, 'is_today': 1, "us_time": ctime} zqone_competition(data=data_competition, cursor=cursor) if pt == 2: data_competition = {'team_home': team_home, 'team_guest': team_guest, 'league_id': league_id, 'game_id': competition_id, 'match_date': match_date, 'match_time': match_time, 'utime': utime, 'number': number, 'source': "hg3535", "expire_time": n_time, 'is_morningplate': 1, "us_time": ctime} zqtwo_competition(data=data_competition, cursor=cursor) if pt == 3: data_competition = {'team_home': team_home, 'team_guest': team_guest, 'league_id': league_id, 'game_id': competition_id, 'match_date': match_date, 'match_time': match_time, 'utime': utime, 'number': number, 'source': "hg3535", "expire_time": n_time, "is_stringscene": 1, "us_time": ctime} zqthree_competition(data=data_competition, cursor=cursor) def close_spider(self, spider): self.dbpool.close() # 篮球 让球大小---------------------------------------------------------------------------------------------------------- class Lanqiupipeline(object): def open_spider(self, spider): self.dbpool = adbapi.ConnectionPool("psycopg2",host=settings["POST_HOST"], port=settings['POST_PORT'], user=settings["POST_USER"], password=settings["POST_PASSWORD"], database=settings["POST_DATABASE"]) self.pool = redis.ConnectionPool(host=settings["R_HOST"], port=settings["R_POST"], password=settings["R_PASSWORD"]) self.redis_db = redis.StrictRedis(connection_pool=self.pool) def process_item(self, item, spider): # 使用twisted将mysql插入变成异步执行 # logger = logging.getLogger(__name__) # # logger.warning(query.addErrback(self.handle_error, item, spider)) query = self.dbpool.runInteraction(self.do_insert, item) query.addErrback(self.handle_error, item, spider) # 处理异常 return item def handle_error(self, failure, item, spider): # 处理异步插入的异常 logger = logging.getLogger(__name__) logger.warning(failure) # print(failure) def do_insert(self, cursor, item): # 联赛id league_id = item['league_id'] # 联赛名 league_name = item['league_name'] # result = item['result'] # 比赛id game_id = item['game_id'] # 球队1 team_home = item['team_home'] # 球队2 team_guest = item['team_guest'] # 数量(97>) number = item['number'] # 比赛状态 zhuangtai = item['zhuangtai'] # 日期 # data_game = item['data_game'] data_game = item['data_game'].split("/") month = str(data_game[1].strip()) day = str(data_game[0]) # 比赛时间 time_game = str(item['time_game']) # 比赛时间，时间戳 ctime = "2019" + "-" + month + "-" + day + "" + time_game + ":00" r_ctime = "2019" + "-" + month + "-" + day # 现在时间，时间戳 utime = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime()) expire_time = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(time.time() + 60)) # 队1分数 score_home = item['score_home'] # 队2分数 score_guest = item['score_guest'] # 第几节 jijie = item['jijie'] # 球队得分 qiudui = item['qiudui'] pt = item['pt'] concedes_dict = item['concede'] concedes_dict_rule = item['concede_rule'] odd_evens_dict = item['odd_even'] odd_evens_dict_rule = item['odd_even_rule'] total_sizes_dict = item['total_size'] total_sizes_dict_rule = item['total_size_rule'] last_numbers_dict = item['last_number'] capots_dict = item['capot'] team_scores_dict = item['team_score'] team_scores_dict_rule = item['team_score_rule'] # 让球 lanqiu = item['lanqiu'] concede = {'league_id': league_id, 'game_id': game_id, 'utime': utime, 'p_id': 1, 'p_code': "concede", 'source': "hg3535", 'expire_time': expire_time} # 让球 数据插入数据库 one_intodb(data1=concedes_dict, data2=concede, data3=concedes_dict_rule, cursor=cursor, redis_db=self.redis_db) two_sides = {'league_id': league_id, 'game_id': game_id, 'utime': utime, 'p_id': 2, 'p_code': "two_sides", 'source': "hg3535", 'expire_time': expire_time} # 总分单双 数据插入数据库 one_intodb(data1=odd_evens_dict, data2=two_sides, data3=odd_evens_dict_rule, cursor=cursor, redis_db=self.redis_db) total_size = {'league_id': league_id, 'game_id': game_id, 'utime': utime, 'p_id': 3, 'p_code': "total_size", 'source': "hg3535", 'expire_time': expire_time} # 全场总分大小 数据插入数据库 one_intodb(data1=total_sizes_dict, data2=total_size, data3=total_sizes_dict_rule, cursor=cursor, redis_db=self.redis_db) data4 = {'league_id': league_id, 'game_id': game_id, 'utime': utime, 'p_id': 6, 'p_code': "team_score", 'source': "hg3535", 'expire_time': expire_time} # 全场总分大小 数据插入数据库 one_intodb(data1=team_scores_dict, data2=data4, data3=team_scores_dict_rule, cursor=cursor, redis_db=self.redis_db) # 主队进球最后一位数 last_number_home = {'league_id': league_id, 'game_id': game_id, 'utime': utime, 'p_id': 4, 'p_code': "last_number", 'source': "hg3535", 'expire_time': expire_time, 'sort': 0} two_intodb(data1=last_numbers_dict['last_home'], data2=last_number_home, data3="last_home", cursor=cursor,redis_db=self.redis_db) # 客队进球最后一位数 last_number_guest = {'league_id': league_id, 'game_id': game_id, 'utime': utime, 'p_id': 4, 'p_code': "last_number", 'source': "hg3535", 'expire_time': expire_time, 'sort': 0} two_intodb(data1=last_numbers_dict['last_home'], data2=last_number_guest, data3="last_guest", cursor=cursor, redis_db=self.redis_db) # 独赢 capot_data = {'league_id': league_id, 'game_id': game_id, 'utime': utime, 'p_id': 5, 'p_code': "capot", 'source': "hg3535", 'expire_time': expire_time, 'sort': 0} three_intodb(data1=capots_dict, data2=capot_data, cursor=cursor, redis_db=self.redis_db) match_date, match_time, time3 = new_time(ctime) n_time = out_time(time3, 1.5) us_time = ctime # 插入赛事表 if lanqiu == "篮球": if pt == 1: Competition_sql = "insert into st_lq_competition(home_team, guest_team, lg_id, match_id, match_date, match_time,ctime, utime, tag,source,expire_time,is_today,us_time) values (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s,%s) on conflict(match_id) do update set tag = %s,expire_time=%s,is_today=%s;" cursor.execute(Competition_sql, ( team_home, team_guest, league_id, game_id, match_date, match_time, utime, utime, number, "hg3535", n_time,1, us_time, number, n_time, 1)) if pt == 2: Competition_sql = "insert into st_lq_competition(home_team, guest_team, lg_id, match_id, match_date, match_time,ctime, utime, tag,source,expire_time,is_morningplate,us_time) values (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s,%s) on conflict(match_id) do update set tag = %s,expire_time=%s,is_morningplate=%s;" cursor.execute(Competition_sql, ( team_home, team_guest, league_id, game_id, match_date, match_time, utime, utime, number, "hg3535", n_time,1, us_time, number, n_time, 1)) if pt == 3: Competition_sql = "insert into st_lq_competition(home_team, guest_team, lg_id, match_id, match_date, match_time,ctime, utime, tag,source,expire_time,is_stringscene,us_time) values (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s,%s) on conflict(match_id) do update set tag = %s,expire_time=%s,is_stringscene=%s;" cursor.execute(Competition_sql, ( team_home, team_guest, league_id, game_id, match_date, match_time, utime, utime, number, "hg3535", n_time,1, us_time, number, n_time, 1)) def close_spider(self, spider): self.dbpool.close() # 网球 让球&大小盘-------------------------------------------------------------------------------------------------------- class Wangqiupipeline(object): def open_spider(self, spider): self.dbpool = adbapi.ConnectionPool("psycopg2",host=settings["POST_HOST"], port=settings['POST_PORT'], user=settings["POST_USER"], password=settings["POST_PASSWORD"], database=settings["POST_DATABASE"]) self.pool = redis.ConnectionPool(host=settings["R_HOST"], port=settings["R_POST"], password=settings["R_PASSWORD"]) self.redis_db = redis.StrictRedis(connection_pool=self.pool) def process_item(self, item, spider): # logger = logging.getLogger(__name__) # logger.warning(query.addErrback(self.handle_error, item, spider)) # 使用twisted将mysql插入变成异步执行 query = self.dbpool.runInteraction(self.do_insert, item) query.addErrback(self.handle_error, item, spider) # 处理异常 return item def handle_error(self, failure, item, spider): # 处理异步插入的异常 logger = logging.getLogger(__name__) logger.warning(failure) # print(failure) def do_insert(self, cursor, item): # 联赛id league_id = item['league_id'] # 联赛名 league_name = item['league_name'] # result = item['result'] # 比赛id game_id = item['game_id'] # 球队1 team_home = item['team_home'] # 球队2 team_guest = item['team_guest'] # 数量(97>) number = item['number'] # 比赛状态 zhuangtai = item['zhuangtai'] # 日期 # data_game = item['data_game'] data_game = item['data_game'].split("/") month = str(data_game[1].strip()) day = str(data_game[0]) # 比赛时间 time_game = str(item['time_game']) # 比赛时间，时间戳 ctime = "2019" + "-" + month + "-" + day + "" + time_game + ":00" r_ctime = "2019" + "-" + month + "-" + day # 现在时间，时间戳 utime = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime()) expire_time = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(time.time() + 60)) # 队1分数 score_home = item['score_home'] # 队2分数 score_guest = item['score_guest'] # 第几节 jijie = item['jijie'] # 球队得分 qiudui = item['qiudui'] pt = item['pt'] # 让盘 concedes_dict = item['concedes_dict'] concedes_dict_rule = item['concedes_dict_rule'] # 冠军 kemps_dict = item['kemps_dict'] # 让局 bureaus_dict = item['bureaus_dict'] bureaus_dict_rule = item['bureaus_dict_rule'] # 总局数大小 total_number_dict = item['total_number_dict'] total_number_dict_rule = item['total_number_dict_rule'] # 总局数单双 odd_evens_dict = item['odd_evens_dict'] odd_evens_dict_rule = item['odd_evens_dict_rule'] wangqiu = item['wangqiu'] # 让盘 concede = {'league_id': league_id, 'game_id': game_id, 'utime': utime, 'p_id': 1, 'p_code': "dishes", 'source': "hg3535", 'expire_time': expire_time, 'sort': 0, 'pt': pt} wqone_intodb(data1=concedes_dict, data2=concede, data3=concedes_dict_rule, cursor=cursor, redis_db=self.redis_db) # 让局 bureaus = {'league_id': league_id, 'game_id': game_id, 'utime': utime, 'p_id': 3, 'p_code': "concede", 'source': "hg3535", 'expire_time': expire_time, 'sort': 0, 'pt': pt} wqone_intodb(data1=bureaus_dict, data2=bureaus, data3=bureaus_dict_rule, cursor=cursor, redis_db=self.redis_db) total_number = {'league_id': league_id, 'game_id': game_id, 'utime': utime, 'p_id': 4, 'p_code': "total_number", 'source': "hg3535", 'expire_time': expire_time, 'sort': 0, 'pt': pt} wqone_intodb(data1=total_number_dict, data2=total_number, data3=total_number_dict_rule, cursor=cursor, redis_db=self.redis_db) odd_evens = {'league_id': league_id, 'game_id': game_id, 'utime': utime, 'p_id': 5, 'p_code': "two_sides", 'source': "hg3535", 'expire_time': expire_time, 'sort': 0, 'pt': pt} wqone_intodb(data1=odd_evens_dict, data2=odd_evens, data3=odd_evens_dict_rule, cursor=cursor, redis_db=self.redis_db) match_date, match_time, time3 = new_time(ctime) n_time = out_time(time3, 3) if kemps_dict: for key, value in kemps_dict.items(): if value: new_hash = hash_func(match_id=game_id, odds_code=key, sort=0, p_id=2) r_hash = r_func(match_id=game_id, odds_code=key, sort=0, p_id=2, odd=value) odds_only = r_hash if self.redis_db.hexists("hg3535_wangqiu", r_hash): pass else: self.redis_db.hset("hg3535_wangqiu", r_hash, 0) sql1 = "insert into st_wq_odds(lg_id, odds_code, match_id, ctime, utime, odds, p_id, p_code, sort, source, sole,odds_only,expire_time) values (%s, %s, %s, %s, %s, %s, %s,%s, %s, %s, %s, %s,%s) on conflict(sole) do update set utime = %s,odds = %s,odds_only=%s;" cursor.execute(sql1, ( int(league_id), key, int(game_id), utime, utime, value, 2, "kemp", 0, "hg3535", new_hash, odds_only,expire_time, utime, value, odds_only)) # 更新主队st_zq_odds_record表 sql2 = "insert into st_wq_odds_record(lg_id, odds_code, match_id, ctime, utime,odds, p_id, p_code, sort, source,odds_only) values (%s, %s, %s, %s, %s, %s, %s,%s, %s, %s, %s);" cursor.execute(sql2, ( int(league_id), key, int(game_id), utime, utime, value, 2, "kemp", 0, "hg3535", odds_only)) us_time = ctime # 插入赛事表 if wangqiu == "网球": if pt is 1: Competition_sql = "insert into st_wq_competition(home_team, guest_team, lg_id, match_id, match_date, match_time,ctime, utime, tag,source,expire_time,is_today,us_time) values (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s,%s) on conflict(match_id) do update set tag = %s,expire_time=%s,is_today=%s;" cursor.execute(Competition_sql, ( team_home, team_guest, league_id, game_id, match_date, match_time, utime, utime, number, "hg3535", n_time,1, us_time, number, n_time, 1)) if pt is 2: Competition_sql = "insert into st_wq_competition(home_team, guest_team, lg_id, match_id, match_date, match_time,ctime, utime, tag,source,expire_time,is_morningplate,us_time) values (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s,%s) on conflict(match_id) do update set tag = %s,expire_time=%s,is_morningplate=%s;" cursor.execute(Competition_sql, ( team_home, team_guest, league_id, game_id, match_date, match_time, utime, utime, number, "hg3535", n_time,1, us_time, number, n_time, 1)) if pt is 3: Competition_sql = "insert into st_wq_competition(home_team, guest_team, lg_id, match_id, match_date, match_time,ctime, utime, tag,source,expire_time,is_stringscene,us_time) values (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s,%s) on conflict(match_id) do update set tag = %s,expire_time=%s,is_stringscene=%s;" cursor.execute(Competition_sql, ( team_home, team_guest, league_id, game_id, match_date, match_time, utime, utime, number, "hg3535", n_time,1, us_time, number, n_time, 1)) def close_spider(self, spider): self.dbpool.close() # 网球 波胆-------------------------------------------------------------------------------------------------------------- class Wqbodanpipeline(object): def open_spider(self, spider): self.dbpool = adbapi.ConnectionPool("psycopg2",host=settings["POST_HOST"], port=settings['POST_PORT'], user=settings["POST_USER"], password=settings["POST_PASSWORD"], database=settings["POST_DATABASE"]) self.pool = redis.ConnectionPool(host=settings["R_HOST"], port=settings["R_POST"], password=settings["R_PASSWORD"]) self.redis_db = redis.StrictRedis(connection_pool=self.pool) def process_item(self, item, spider): # 使用twisted将mysql插入变成异步执行 # logger = logging.getLogger(__name__) # logger.warning(query.addErrback(self.handle_error, item, spider)) query = self.dbpool.runInteraction(self.do_insert, item) query.addErrback(self.handle_error, item, spider) # 处理异常 return item def handle_error(self, failure, item, spider): # 处理异步插入的异常 logger = logging.getLogger(__name__) logger.warning(failure) # print(failure) def do_insert(self, cursor, item): # 比赛日期 data_game = item['data_game'].split("/") month = str(data_game[1].strip()) day = str(data_game[0]) # 比赛时间 time_game = str(item['time_game']) # 比赛时间，时间戳 ctime = "2019" + "-" + month + "-" + day + "" + time_game + ":00" r_ctime = "2019" + "-" + month + "-" + day # 现在时间，时间戳 utime = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime()) expire_time = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(time.time() + 60)) # 比赛id competition_id = item['game_id'] # 联赛id league_id = item['league_id'] # 联赛name league_name = item['league_name'] # 主队 team_home = item['team_home'] # 客队 team_guest = item['team_guest'] # 主队得分 score_home = item['score_home'] # 客队得分 score_guest = item['score_guest'] # number number = item['number'] corner_ball = item['corner_ball'] half_way = item['half_way'] # 类型早盘，今日，滚球，串场 pt = item['pt'] bodan_datas = item['bodan_data'] p_code = "bodan" p_id = 7 bodan_dict = {"bodanhome_two_zero": "2-0", "bodanhome_two_one": "2-1", "bodanhome_three_zero": "3-0", "bodanhome_three_one": "3-1", "bodanhome_three_two": "3-2", "bodanhome_four_zero": "4-0", "bodanhome_four_one": "4-1", "bodanhome_four_two": "4-2", "bodanhome_four_three": "4-3", "bodanguest_two_zero": "2-0", "bodanguest_two_one": "2-1", "bodanguest_three_zero": "3-0", "bodanguest_three_one": "3-1", "bodanguest_three_two": "3-2", "bodanguest_four_zero": "4-0", "bodanguest_four_one": "4-1", "bodanguest_four_two": "4-2", "bodanguest_four_three": "4-3"} if bodan_datas: for key, value in bodan_datas.items(): if value: new_hash = hash_func(match_id=competition_id, odds_code=key, sort=0, p_id=7) r_hash = r_func(match_id=competition_id, odds_code=key, sort=0, p_id=7, odd=value) odds_only = r_hash if self.redis_db.hexists("hg3535_wangqiu", r_hash): pass else: self.redis_db.hset("hg3535_wangqiu", r_hash, 0) sql1 = "insert into st_wq_odds(lg_id, odds_code, match_id, ctime, utime, odds, p_id, p_code, sort, source, sole, condition, expire_time,odds_only) values (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s,%s) on conflict(sole) do update set utime=%s,odds=%s,expire_time=%s,odds_only=%s;" cursor.execute(sql1, ( int(league_id), key, int(competition_id), utime, utime, value, p_id, p_code, 0, "hg3535", new_hash, bodan_dict[key], expire_time, odds_only,utime, value, expire_time, odds_only)) # 更新主队st_zq_odds_record表 sql2 = "insert into st_wq_odds_record(lg_id, odds_code, match_id, ctime, utime, odds, p_id, p_code, sort, source, condition,odds_only) values (%s, %s, %s, %s, %s, %s,%s, %s, %s, %s, %s, %s);" cursor.execute(sql2, ( int(league_id), key, int(competition_id), utime, utime, value, p_id, p_code, 0, "hg3535", key,odds_only)) def close_spider(self, spider): self.dbpool.close() # 棒球 让球&大小盘-------------------------------------------------------------------------------------------------------- class Bangqiupipeline(object): def open_spider(self, spider): self.dbpool = adbapi.ConnectionPool("psycopg2",host=settings["POST_HOST"], port=settings['POST_PORT'], user=settings["POST_USER"], password=settings["POST_PASSWORD"], database=settings["POST_DATABASE"]) self.pool = redis.ConnectionPool(host=settings["R_HOST"], port=settings["R_POST"], password=settings["R_PASSWORD"]) self.redis_db = redis.StrictRedis(connection_pool=self.pool) def process_item(self, item, spider): # logger = logging.getLogger(__name__) # logger.warning(query.addErrback(self.handle_error, item, spider)) # 使用twisted将mysql插入变成异步执行 query = self.dbpool.runInteraction(self.do_insert, item) query.addErrback(self.handle_error, item, spider) # 处理异常 return item def handle_error(self, failure, item, spider): # 处理异步插入的异常 logger = logging.getLogger(__name__) logger.warning(failure) # print(failure) def do_insert(self, cursor, item): # 联赛id league_id = item['league_id'] # 联赛名 league_name = item['league_name'] # result = item['result'] # 比赛id game_id = item['game_id'] # 球队1 team_home = item['team_home'] # 球队2 team_guest = item['team_guest'] # 数量(97>) number = item['number'] # 比赛状态 zhuangtai = item['zhuangtai'] # 日期 # data_game = item['data_game'] data_game = item['data_game'].split("/") month = str(data_game[1].strip()) day = str(data_game[0]) # 比赛时间 time_game = str(item['time_game']) # 比赛时间，时间戳 ctime = "2019" + "-" + month + "-" + day + "" + time_game + ":00" r_ctime = "2019" + "-" + month + "-" + day # 现在时间，时间戳 utime = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime()) expire_time = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(time.time() + 60)) # 队1分数 score_home = item['score_home'] # 队2分数 score_guest = item['score_guest'] # 第几节 jijie = item['jijie'] # 球队得分 qiudui = item['qiudui'] pt = item['pt'] # 让球 concedes_dict = item['concedes_dict'] concedes_dict_rule = item['concedes_dict_rule'] # 独赢 capots_dict = item['capots_dict'] # 总得分大小 total_size_dict = item['total_size_dict'] total_size_dict_rule = item['total_size_dict_rule'] # 总得分单双 odd_evens_dict = item['odd_evens_dict'] odd_evens_dict_rule = item['odd_evens_dict_rule'] bangqiu = item['bangqiu'] # 让球 concede = {'league_id': league_id, 'game_id': game_id, 'utime': utime, 'p_id': 1, 'p_code': "concede", 'source': "hg3535", 'expire_time': expire_time, 'sort': 0, 'pt': pt} bqone_intodb(data1=concedes_dict, data2=concede, data3=concedes_dict_rule, cursor=cursor, redis_db=self.redis_db) # 总得分：大/小 total_size = {'league_id': league_id, 'game_id': game_id, 'utime': utime, 'p_id': 2, 'p_code': "total_size", 'source': "hg3535", 'expire_time': expire_time, 'sort': 0, 'pt': pt} bqone_intodb(data1=total_size_dict, data2=total_size, data3=total_size_dict_rule, cursor=cursor, redis_db=self.redis_db) odd_even = {'league_id': league_id, 'game_id': game_id, 'utime': utime, 'p_id': 3, 'p_code': "two_sides", 'source': "hg3535", 'expire_time': expire_time, 'sort': 0, 'pt': pt} bqone_intodb(data1=odd_evens_dict, data2=odd_even, data3=odd_evens_dict_rule, cursor=cursor, redis_db=self.redis_db) # 赛事失效时间 match_date, match_time, time3 = new_time(ctime) n_time = out_time(time3, 3.5) # 插入独赢 if capots_dict: for key, value in capots_dict.items(): if value: new_hash = hash_func(match_id=game_id, odds_code=key, sort=0, p_id=4) r_hash = r_func(match_id=game_id, odds_code=key, sort=0, p_id=4, odd=value) odds_only = r_hash if self.redis_db.hexists("hg3535_bangqiu", r_hash): pass else: self.redis_db.hset("hg3535_bangqiu", r_hash, 0) sql1 = "insert into st_bq_odds(lg_id, odds_code, match_id, ctime, utime, odds, p_id, p_code, sort, source, sole,odds_only,expire_time) values (%s, %s, %s, %s, %s, %s, %s,%s, %s, %s, %s, %s,%s) on conflict(sole) do update set utime = %s,odds = %s,odds_only=%s,expire_time=%s;" cursor.execute(sql1, ( int(league_id), key, int(game_id), utime, utime, value, 4, "capot", 0, "hg3535", new_hash, odds_only, expire_time,utime, value, odds_only, expire_time)) # 更新主队st_zq_odds_record表 sql2 = "insert into st_bq_odds_record(lg_id, odds_code, match_id, ctime, utime,odds, p_id, p_code, sort, source,odds_only) values (%s, %s, %s, %s, %s, %s, %s,%s, %s, %s, %s);" cursor.execute(sql2, ( int(league_id), key, int(game_id), utime, utime, value, 4, "capot", 0, "hg3535", odds_only)) us_time = ctime # 插入赛事表 if bangqiu == '棒球': if pt == 1: Competition_sql = "insert into st_bq_competition(home_team, guest_team, lg_id, match_id, match_date, match_time,ctime, utime, tag,source,expire_time,is_today,us_time) values (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s,%s) on conflict(match_id) do update set tag = %s,expire_time=%s,is_today=%s;" cursor.execute(Competition_sql, ( team_home, team_guest, league_id, game_id, match_date, match_time, utime, utime, number, "hg3535", n_time, 1, us_time, number, n_time, 1)) if pt == 2: Competition_sql = "insert into st_bq_competition(home_team, guest_team, lg_id, match_id, match_date, match_time,ctime, utime, tag,source,expire_time,is_morningplate,us_time) values (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s,%s) on conflict(match_id) do update set tag = %s,expire_time=%s,is_morningplate=%s;" cursor.execute(Competition_sql, ( team_home, team_guest, league_id, game_id, match_date, match_time, utime, utime, number, "hg3535", n_time,1, us_time, number, n_time, 1)) if pt == 3: Competition_sql = "insert into st_bq_competition(home_team, guest_team, lg_id, match_id, match_date, match_time,ctime, utime, tag,source,expire_time,is_stringscene,us_time) values (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s,%s) on conflict(match_id) do update set tag = %s,expire_time=%s,is_stringscene=%s;" cursor.execute(Competition_sql, ( team_home, team_guest, league_id, game_id, match_date, match_time, utime, utime, number, "hg3535", n_time,1, us_time, number, n_time, 1)) def close_spider(self, spider): self.dbpool.close() # 足球 篮球 网球 棒球 冠军------------------------------------------------------------------------------------------------ class Guanjunpipeline(object): def open_spider(self, spider): self.dbpool = adbapi.ConnectionPool("psycopg2",host=settings["POST_HOST"], port=settings['POST_PORT'], user=settings["POST_USER"], password=settings["POST_PASSWORD"], database=settings["POST_DATABASE"]) self.pool = redis.ConnectionPool(host=settings["R_HOST"], port=settings["R_POST"], password=settings["R_PASSWORD"]) self.redis_db = redis.StrictRedis(connection_pool=self.pool) def process_item(self, item, spider): # logger = logging.getLogger(__name__) # logger.warning(query.addErrback(self.handle_error, item, spider)) # 使用twisted将mysql插入变成异步执行 query = self.dbpool.runInteraction(self.do_insert, item) query.addErrback(self.handle_error, item, spider) # 处理异常 return item def handle_error(self, failure, item, spider): # 处理异步插入的异常 logger = logging.getLogger(__name__) logger.warning(failure) # print(failure) def do_insert(self, cursor, item): # 执行具体的插入 # 根据不同的item 构建不同的sql语句并插入到mysql中 data_game = item['data_game'].replace('年', "-").replace('月', "-").replace('日', "") time_game = item['time_game'] ctime = data_game + " " + time_game + ":00" utime = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime()) expire_time = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(time.time() + 60)) league_name = item['league_name'] tema_home = item['tema_home'] league_id = item['league_id'] game_id = item['game_id'] new_league_name = item['new_league_name'] # 冠军赔率 champion_team = item['champion_team'] new_champion = item['new_champion'] # 构建唯一哈希索引 time1 = time.mktime(time.strptime(ctime, '%Y-%m-%d %H:%M:%S')) + 43200 time2 = time.localtime(time1) time3 = time.strftime('%Y-%m-%d %H:%M:%S', time2) # data_time = str(time3).split(" ") # match_date,match_time = new_time(ctime) pt = item['pt'] new_hash = hash_func(match_id=champion_team, odds_code=new_champion, sort=new_league_name, p_id=tema_home) ball = item['ball'] # 插入st_zq_odds表 r_hash = hash_func(league_id, new_champion, tema_home, champion_team) odds_only = r_hash s_hash = hash_func(league_id, league_name, ball, 0) # if game_id == '8947612': # print(ball, league_name) if ball == "足球": n_gameid = int('1' + str(league_id)) data_dict = {'league_id': league_id, 'new_champion': new_champion, 'utime': utime, 'champion_team': champion_team, "p_id": 8, 'new_league_name': new_league_name, 'tema_home': tema_home, 'new_hash': new_hash, 'expire_time': expire_time, "odds_only": odds_only, "league_name": league_name, 'time3': time3, 'game_id': n_gameid} gjz_into(data_dict, cursor, r_hash, s_hash, self.redis_db) Competition_sql = "insert into st_zq_competition(home_team, lg_id, match_id, ctime, utime, source, us_time) values (%s, %s, %s, %s, %s, %s, %s) on conflict(match_id) do update set utime=%s;" cursor.execute(Competition_sql, (league_name, league_id, n_gameid, utime, utime, 'hg3535', data_game, utime)) if ball == "篮球": n_gameid = int('2' + str(league_id)) data_dict = {'league_id': league_id, 'new_champion': new_champion, 'utime': utime, 'champion_team': champion_team, "p_id": 7, 'new_league_name': new_league_name, 'tema_home': tema_home, 'new_hash': new_hash, 'expire_time': expire_time, "odds_only": odds_only, "league_name": league_name, 'time3': time3, 'game_id': n_gameid} gjl_into(data_dict, cursor, r_hash, s_hash, self.redis_db) Competition_sql = "insert into st_lq_competition(home_team, lg_id, match_id, ctime, utime, source, us_time) values (%s, %s, %s, %s, %s, %s, %s) on conflict(match_id) do update set utime=%s;" cursor.execute(Competition_sql, (league_name, league_id, n_gameid, utime, utime, 'hg3535', data_game, utime)) if ball == "网球": n_gameid = int('3' + str(league_id)) data_dict = {'league_id': league_id, 'new_champion': new_champion, 'utime': utime, 'champion_team': champion_team, "p_id": 6, 'new_league_name': new_league_name, 'tema_home': tema_home, 'new_hash': new_hash, 'expire_time': expire_time, "odds_only": odds_only, "league_name": league_name, 'time3': time3, 'game_id': n_gameid} gjw_into(data_dict, cursor, r_hash, s_hash, self.redis_db) Competition_sql = "insert into st_wq_competition(home_team, lg_id, match_id, ctime, utime, source, us_time) values (%s, %s, %s, %s, %s, %s, %s) on conflict(match_id) do update set utime=%s;" cursor.execute(Competition_sql, (league_name, league_id, n_gameid, utime, utime, 'hg3535', data_game, utime)) if ball == "棒球": n_gameid = int('4' + str(league_id)) data_dict = {'league_id': league_id, 'new_champion': new_champion, 'utime': utime, 'champion_team': champion_team, "p_id": 5, 'new_league_name': new_league_name, 'tema_home': tema_home, 'new_hash': new_hash, 'expire_time': expire_time, "odds_only": odds_only, "league_name": league_name, 'time3': time3, 'game_id': n_gameid} gjb_into(data_dict, cursor, r_hash, s_hash, self.redis_db) Competition_sql = "insert into st_bq_competition(home_team, lg_id, match_id, ctime, utime, source, us_time) values (%s, %s, %s, %s, %s, %s, %s) on conflict(match_id) do update set utime=%s;" cursor.execute(Competition_sql, (league_name, league_id, n_gameid, utime, utime, 'hg3535', data_game, utime)) def close_spider(self, spider): # self.conn.close() self.dbpool.close() # 足球 篮球 网球 棒球 联赛------------------------------------------------------------------------------------------------ class Liansaipipeline(object): def open_spider(self, spider): self.dbpool = adbapi.ConnectionPool("psycopg2",host=settings["POST_HOST"], port=settings['POST_PORT'], user=settings["POST_USER"], password=settings["POST_PASSWORD"], database=settings["POST_DATABASE"]) self.pool = redis.ConnectionPool(host=settings["R_HOST"], port=settings["R_POST"], password=settings["R_PASSWORD"]) self.redis_db = redis.StrictRedis(connection_pool=self.pool) def process_item(self, item, spider): # logger = logging.getLogger(__name__) # logger.warning(query.addErrback(self.handle_error, item, spider)) # 使用twisted将mysql插入变成异步执行 query = self.dbpool.runInteraction(self.do_insert, item) query.addErrback(self.handle_error, item, spider) # 处理异常 return item def handle_error(self, failure, item, spider): # 处理异步插入的异常 logger = logging.getLogger(__name__) logger.warning(failure) # print(failure) def do_insert(self, cursor, item): area_id = item["area_id"] area_name = item["area_name"] st_league = item["st_league"] name_chinese = item["name_chinese"] ball = item['ball'] utime = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime()) area_dict = {"南美洲": 3, "北美洲": 4, "欧洲": 5, "大洋洲": 6, "南极洲": 7, "非洲": 1, "世界": 8, "亚洲": 2} redis_data_dict = "hg3535_liansai" r_hash = hash_func(st_league, name_chinese, ball, 0) if self.redis_db.hexists(redis_data_dict, r_hash): pass else: self.redis_db.hset(redis_data_dict, r_hash, 0) if ball == "足球": if area_name in area_dict: sql1 = "insert into st_zq_league(name_chinese, lg_id, area_id,source,utime) values (%s, %s, %s,%s,%s) on conflict(lg_id) do update set area_id = %s,source=%s;" cursor.execute(sql1, (name_chinese, st_league, area_dict[area_name], "hg3535",utime, area_dict[area_name], "hg3535")) else: if area_name == "足球": sql2 = "insert into st_zq_league(name_chinese, lg_id, country_id,source) values (%s, %s, %s,%s) on conflict(lg_id) do update set source=%s;" cursor.execute(sql2, (name_chinese, st_league, 0, "hg3535", "hg3535")) else: sql2 = "insert into st_zq_league(name_chinese, lg_id, country_id,source,utime) values (%s, %s, %s,%s,%s) on conflict(lg_id) do update set country_id = %s,source=%s;" cursor.execute(sql2, (name_chinese, st_league, area_id, "hg3535",utime, area_id, "hg3535")) sql3 = "insert into st_country(country_id, name_chinese, country_area, country_order,recommend_order, update_time,source) values (%s, %s, %s,%s, %s, %s,%s) on conflict(country_id) do update set update_time =%s,source=%s;" cursor.execute(sql3, (area_id, area_name, 0, 0, 0, utime, "hg3535", utime, "hg3535")) elif ball == "篮球": if area_name in area_dict: sql1 = "insert into st_lq_league(name_chinese, lg_id, area_id,source,utime) values (%s, %s, %s,%s,%s) on conflict(lg_id) do update set area_id = %s,source=%s;" cursor.execute(sql1, (name_chinese, st_league, area_dict[area_name], "hg3535",utime, area_dict[area_name],"hg3535")) else: if area_name == "篮球": sql2 = "insert into st_lq_league(name_chinese, lg_id, country_id,source) values (%s, %s, %s,%s) on conflict(lg_id) do update set source=%s;" cursor.execute(sql2, (name_chinese, st_league, 0, "hg3535", "hg3535")) else: sql2 = "insert into st_lq_league(name_chinese, lg_id, country_id,source,utime) values (%s, %s, %s,%s,%s) on conflict(lg_id) do update set country_id = %s,source=%s;" cursor.execute(sql2, (name_chinese, st_league, area_id, "hg3535",utime, area_id, "hg3535")) sql3 = "insert into st_country(country_id, name_chinese, country_area, country_order,recommend_order, update_time,source) values (%s, %s, %s,%s, %s, %s,%s) on conflict(country_id) do update set update_time =%s,source=%s;" cursor.execute(sql3, (area_id, area_name, 0, 0, 0, utime, "hg3535", utime, "hg3535")) elif ball == "网球": if area_name in area_dict: sql1 = "insert into st_wq_league(name_chinese, lg_id, area_id,source,utime) values (%s, %s, %s,%s,%s) on conflict(lg_id) do update set area_id = %s,source=%s;" cursor.execute(sql1, (name_chinese, st_league, area_dict[area_name], "hg3535",utime, area_dict[area_name],"hg3535")) else: if area_name == "网球": sql2 = "insert into st_wq_league(name_chinese, lg_id, country_id,source) values (%s, %s, %s,%s) on conflict(lg_id) do update set source=%s;" cursor.execute(sql2, (name_chinese, st_league, 0, "hg3535", "hg3535")) else: sql2 = "insert into st_wq_league(name_chinese, lg_id, country_id,source,utime) values (%s, %s, %s,%s,%s) on conflict(lg_id) do update set country_id = %s,source=%s;" cursor.execute(sql2, (name_chinese, st_league, area_id, "hg3535",utime, area_id, "hg3535")) sql3 = "insert into st_country(country_id, name_chinese, country_area, country_order,recommend_order, update_time,source) values (%s, %s, %s,%s, %s, %s,%s) on conflict(country_id) do update set update_time =%s,source=%s;" cursor.execute(sql3, (area_id, area_name, 0, 0, 0, utime, "hg3535", utime, "hg3535")) elif ball == "棒球": if area_name in area_dict: sql1 = "insert into st_bq_league(name_chinese, lg_id, area_id,source,utime) values (%s, %s, %s,%s,%s) on conflict(lg_id) do update set area_id = %s,source=%s;" cursor.execute(sql1,(name_chinese, st_league, area_dict[area_name], "hg3535",utime, area_dict[area_name],"hg3535")) else: if area_name == "棒球": sql2 = "insert into st_bq_league(name_chinese, lg_id, country_id,source) values (%s, %s, %s,%s) on conflict(lg_id) do update set source=%s;" cursor.execute(sql2, (name_chinese, st_league, 0, "hg3535", "hg3535")) else: sql2 = "insert into st_bq_league(name_chinese, lg_id, country_id,source,utime) values (%s, %s, %s,%s,%s) on conflict(lg_id) do update set country_id = %s,source=%s;" cursor.execute(sql2, (name_chinese, st_league, area_id, "hg3535",utime, area_id, "hg3535")) sql3 = "insert into st_country(country_id, name_chinese, country_area, country_order,recommend_order, update_time,source) values (%s, %s, %s,%s, %s, %s,%s) on conflict(country_id) do update set update_time =%s,source=%s;" cursor.execute(sql3, (area_id, area_name, 0, 0, 0, utime, "hg3535", utime, "hg3535")) def close_spider(self, spider): self.dbpool.close() # self.redis_db.disconnect() #************************************滚球------------------------------------------------------------------------------- #滚球方法都加了s def zqone_intodbs(data1, data2, data3, data4, cursor, redis_db): if data1: for index, value in enumerate(data1): if value: new_hash = hash_func(match_id=data2['game_id'], odds_code=data3, sort=index, p_id=data2['p_id']) r_hash = r_func(match_id=data2['game_id'], odds_code=data3, sort=index, p_id=data2['p_id'], odd=value) odds_only = r_hash if redis_db.hexists("hg3535_zuqiu", r_hash): pass else: redis_db.hset("hg3535_zuqiu", r_hash, 0) # if data2['pt'] == 0: sql1 = "insert into st_zq_odds(lg_id, odds_code, match_id, ctime, utime, odds, p_id, p_code, sort, source, sole, condition, expire_time,odds_only,is_rollball) values (%s,%s,%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s) on conflict(sole) do update set utime=%s,odds=%s,expire_time=%s,odds_only=%s,condition=%s;" cursor.execute(sql1, ( int(data2['league_id']), data3, int(data2['game_id']), data2['utime'], data2['utime'], float(value), data2['p_id'], data2["p_code"], index, data2["source"], new_hash, data4[index], data2['expire_time'], odds_only, 1, data2['utime'], value, data2['expire_time'], odds_only,data4[index])) # 更新主队st_zq_odds_record表 sql2 = "insert into st_zq_odds_record(lg_id, odds_code, match_id, ctime, utime, odds, p_id, p_code, sort, source, condition,odds_only,is_rollball) values (%s,%s, %s, %s, %s, %s, %s, %s,%s, %s, %s, %s, %s);" cursor.execute(sql2, ( int(data2['league_id']), data3, int(data2['game_id']), data2['utime'], data2['utime'], float(value), data2['p_id'], data2["p_code"], index, data2["source"], data4[index], odds_only,1)) #写入数据设置默认值竟然不生效 待看 sql3 = "insert into st_zq_result(match_id, match_score, home_team, guest_team, home_score, guest_score, all_goal, status, first_score, last_score, match_winer, update_time, match_time,match_process,tag,lg_id,u_home_score,u_guest_score,source,p_code) values (%s,%s,%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s,%s,%s,%s,%s,%s) on conflict(match_id) do update set update_time=%s,match_score=%s,match_time=%s,match_process=%s,home_score=%s,guest_score=%s;" cursor.execute(sql3, ( int(data2['game_id']), data2['match_score'],data2['team_home'],data2['team_guest'],data2['score_home'],data2['score_guest'],data2['number'],1,'','','',data2['utime'],data2['time_game'],data2['half_way'],data2['number'],int(data2['league_id']),0,0,'hg3535',data2["p_code"],data2['utime'],data2['match_score'],data2['time_game'],data2['half_way'],data2['score_home'],data2['score_guest'] )) sql4 = "insert into st_zq_result_record(lg_id, home_team, guest_team, home_score, guest_score, all_goal, status, first_score, last_score, match_score, match_winer, update_time, match_time,match_process,tag,match_id,source,p_code) values (%s,%s,%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s,%s,%s,%s);" cursor.execute(sql4, ( int(data2['league_id']), data2['team_home'],data2['team_guest'],data2['score_home'],data2['score_guest'],data2['number'],1,'','',data2['match_score'],'',data2['utime'],data2['time_game'],data2['half_way'],data2['number'],int(data2['game_id']),'hg3535',data2["p_code"] )) def zqtwo_intodbs(data1, data2, data3, data4, cursor, redis_db): if data1: new_hash = hash_func(match_id=data2['game_id'], odds_code=data3, sort=data2['sort'], p_id=data2['p_id']) r_hash = r_func(match_id=data2['game_id'], odds_code=data3, sort=data2['sort'], p_id=data2['p_id'], odd=data1) odds_only = r_hash if redis_db.hexists("hg3535_zuqiu", r_hash): pass else: redis_db.hset("hg3535_zuqiu", r_hash, 0) # if data2['pt'] == 1: sql1 = "insert into st_zq_odds(lg_id, odds_code, match_id, ctime, utime, odds, p_id, p_code, sort, source, sole, condition, expire_time, odds_only,is_today) values (%s,%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s) on conflict(sole) do update set utime=%s,odds=%s,expire_time=%s,odds_only=%s,condition=%s;" cursor.execute(sql1, ( int(data2['league_id']), data3, int(data2['game_id']), data2['utime'], data2['utime'], data1, data2['p_id'], data2["p_code"], data2['sort'], data2["source"], new_hash, data4, data2['expire_time'], odds_only, 1, data2['utime'], data1, data2['expire_time'], odds_only,data4)) # 更新主队st_zq_odds_record表 sql2 = "insert into st_zq_odds_record(lg_id, odds_code, match_id, ctime, utime, odds, p_id, p_code, sort, source, condition,odds_only, is_today) values (%s,%s,%s, %s, %s, %s, %s, %s,%s, %s, %s, %s, %s);" cursor.execute(sql2, ( int(data2['league_id']), data3, int(data2['game_id']), data2['utime'], data2['utime'], data1, data2['p_id'], data2["p_code"], data2['sort'], data2["source"], data4, odds_only, 1)) sql3 = "insert into st_zq_result(match_id, match_score, home_team, guest_team, home_score, guest_score, all_goal, status, first_score, last_score, match_winer, update_time, match_time,match_process,tag,lg_id,u_home_score,u_guest_score,source,p_code) values (%s,%s,%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s,%s,%s,%s,%s,%s) on conflict(match_id) do update set update_time=%s,match_score=%s,match_time=%s,match_process=%s,home_score=%s,guest_score=%s;" cursor.execute(sql3, ( int(data2['game_id']), data2['match_score'],data2['team_home'],data2['team_guest'],data2['score_home'],data2['score_guest'],data2['number'],1,'','','',data2['utime'],data2['time_game'],data2['half_way'],data2['number'],int(data2['league_id']),0,0,'hg3535',data2["p_code"],data2['utime'],data2['match_score'],data2['time_game'],data2['half_way'],data2['score_home'],data2['score_guest'] )) sql4 = "insert into st_zq_result_record(lg_id, home_team, guest_team, home_score, guest_score, all_goal, status, first_score, last_score, match_score, match_winer, update_time, match_time,match_process,tag,match_id,source,p_code) values (%s,%s,%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s,%s,%s,%s);" cursor.execute(sql4, ( int(data2['league_id']), data2['team_home'],data2['team_guest'],data2['score_home'],data2['score_guest'],data2['number'],1,'','',data2['match_score'],'',data2['utime'],data2['time_game'],data2['half_way'],data2['number'],int(data2['game_id']),'hg3535',data2["p_code"] )) def zqone_competitions(data, cursor): Competition_sql = "insert into st_zq_competition(home_team, guest_team, lg_id, status, match_id, match_date, match_time, ctime, utime,tag, source,type,expire_time,us_time,is_rollball) values (%s, %s, %s, %s, %s, %s, %s,%s, %s, %s,%s,%s,%s,%s,%s) on conflict(match_id) do update set utime = %s,status=%s,tag=%s,expire_time=%s;" cursor.execute(Competition_sql, ( data['team_home'], data['team_guest'], data['league_id'],1,data['game_id'], data['match_date'], data['match_time'], data['utime'], data['utime'], data['number'], data["source"], 4,data['expire_time'], data['us_time'],data['is_rollball'], data['utime'], 1, data['number'], data['expire_time'])) def get_pcodes(corner_ball, code): code_dict = {'concede_size': 1, 'capot': 2, 'two_sides': 3, 'total_goal': 4, 'half_full': 5, 'bodan': 6, 'first_last_ball': 7, 'temaball': 11} if corner_ball == "角球": p_code = "corner_ball" p_id = 9 elif corner_ball == "会晋级": p_code = "promotion" p_id = 10 elif corner_ball == "罚牌数": p_code = "Penalty_card" p_id = 12 else: p_code = code p_id = code_dict[code] return p_code, p_id def news_times(ctime): time1 = time.mktime(time.strptime(ctime, '%Y-%m-%d %H:%M:%S')) + 43200 time2 = time.localtime(time1) time3 = time.strftime('%Y-%m-%d %H:%M:%S', time2) time4 = time3 data_time = str(time4).split(" ") match_date = data_time[0] match_time = data_time[1] return match_date, match_time, time3 class Roll_Zuqiupipeline(object): def open_spider(self, spider): self.dbpool = adbapi.ConnectionPool("psycopg2",host=settings["POST_HOST"], port=settings['POST_PORT'], user=settings["POST_USER"], password=settings["POST_PASSWORD"], database=settings["POST_DATABASE"]) self.pool = redis.ConnectionPool(host=settings["R_HOST"], port=settings["R_POST"], password=settings["R_PASSWORD"]) self.redis_db = redis.StrictRedis(connection_pool=self.pool) def process_item(self, item, spider): logger = logging.getLogger(__name__) # logger.warning(query.addErrback(self.handle_error, item, spider)) # 使用twisted将mysql插入变成异步执行 query = self.dbpool.runInteraction(self.do_insert, item) query.addErrback(self.handle_error, item, spider) # 处理异常 return item def handle_error(self, failure, item, spider): # 处理异步插入的异常 logger = logging.getLogger(__name__) logger.warning(failure) # print(failure) def do_insert(self, cursor, item): # 比赛日期 data_game = item['data_game'].split("/") month = str(data_game[1].strip()) day = str(data_game[0]) # 比赛时间 time_game = str(item['time_game']) # 比赛时间，时间戳 ctime = "2019" + "-" + month + "-" + day + "" + time_game + ":00" r_ctime = "2019" + "-" + month + "-" + day # 现在时间，时间戳 utime = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime()) expire_time = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(time.time() + 60)) # 比赛id competition_id = item['game_id'] # 联赛id league_id = item['league_id'] # 联赛name league_name = item['league_name'] # 主队 team_home = item['team_home'] # 客队 team_guest = item['team_guest'] # number number = item['number'] score_home = item['score_home'] score_guest = item['score_guest'] half_way = item['half_way'] pt = item['pt'] match_score = "{}:{}".format(score_home,score_guest) corner_ball = item['corner_ball'] p_code, p_id = get_pcode(corner_ball, 'concede_size') # 构建唯一索引 half_size_guest = item["half_size_guest"] half_size_guest_rule = item["half_size_guest_rule"] half_size_home = item["half_size_home"] half_size_home_rule = item["half_size_home_rule"] size_data = {'league_id': league_id, 'game_id': competition_id, 'utime': utime, 'p_id': p_id, 'p_code': p_code, 'source': "hg3535", 'expire_time': expire_time, 'pt': pt,'match_score':match_score,'team_home':team_home,'team_guest':team_guest,'score_home':score_home,'score_guest':score_guest,'number':number,'time_game':time_game,'half_way':half_way} # 让球 数据插入数据库 zqone_intodbs(data1=half_size_home, data2=size_data, data3="half_size_home", data4=half_size_home_rule, cursor=cursor, redis_db=self.redis_db) # 让球 数据插入数据库 zqone_intodbs(data1=half_size_guest, data2=size_data, data3="half_size_guest", data4=half_size_guest_rule, cursor=cursor, redis_db=self.redis_db) # 全场场大小 size_guest = item["size_guest"] size_guest_rule = item["size_guest_rule"] size_home = item["size_home"] size_home_rule = item["size_home_rule"] zqone_intodbs(data1=size_guest, data2=size_data, data3="size_guest", data4=size_guest_rule, cursor=cursor, redis_db=self.redis_db) zqone_intodbs(data1=size_home, data2=size_data, data3="size_home", data4=size_home_rule, cursor=cursor, redis_db=self.redis_db) # 上半场大小 half_concede_home_rule = item["half_concede_home_rule"] half_concede_home = item["half_concede_home"] half_concede_guest_rule = item["half_concede_guest_rule"] half_concede_guest = item["half_concede_guest"] # 上半场让球 zqone_intodbs(data1=half_concede_home, data2=size_data, data3="half_concede_home", data4=half_concede_home_rule, cursor=cursor, redis_db=self.redis_db) zqone_intodbs(data1=half_concede_guest, data2=size_data, data3="half_concede_guest", data4=half_concede_guest_rule, cursor=cursor, redis_db=self.redis_db) concede_guest = item["concede_guest"] concede_guest_rule = item["concede_guest_rule"] concede_home = item["concede_home"] concede_home_rule = item["concede_home_rule"] # 全场让球 zqone_intodbs(data1=concede_guest, data2=size_data, data3="concede_guest", data4=concede_guest_rule, cursor=cursor, redis_db=self.redis_db) zqone_intodbs(data1=concede_home, data2=size_data, data3="concede_home", data4=concede_home_rule, cursor=cursor,redis_db=self.redis_db) # p_code, p_id = get_pcode(corner_ball, 'capot') # 独赢---------------------------------------------------------------------------------------------------------- half_capot_home = item["half_capot_home"] half_capot_guest = item["half_capot_guest"] half_capot_dogfall = item["half_capot_dogfall"] capot_home = item["capot_home"] capot_guest = item["capot_guest"] capot_dogfall = item["capot_dogfall"] capot_data = {'league_id': league_id, 'game_id': competition_id, 'utime': utime, 'p_id': p_id, 'p_code': p_code, 'source': "hg3535", 'expire_time': expire_time, 'sort': 0, 'pt': pt,'match_score':match_score,'team_home':team_home,'team_guest':team_guest,'score_home':score_home,'score_guest':score_guest,'number':number,'time_game':time_game,'half_way':half_way} # 上半场独赢 主队 zqtwo_intodbs(data1=half_capot_home, data2=capot_data, data3="half_capot_home", data4='1', cursor=cursor,redis_db=self.redis_db) # 上半场独赢 客队 zqtwo_intodbs(data1=half_capot_guest, data2=capot_data, data3="half_capot_guest", data4='2', cursor=cursor,redis_db=self.redis_db) # 上半场独赢 和 zqtwo_intodbs(data1=half_capot_dogfall, data2=capot_data, data3="half_capot_dogfall", data4='x', cursor=cursor,redis_db=self.redis_db) # # 全场独赢 主队 zqtwo_intodbs(data1=capot_home, data2=capot_data, data3="capot_home", data4='1', cursor=cursor,redis_db=self.redis_db) # 全场独赢 客队 zqtwo_intodbs(data1=capot_guest, data2=capot_data, data3="capot_guest", data4='2', cursor=cursor,redis_db=self.redis_db) # 全场独赢 和 zqtwo_intodbs(data1=capot_dogfall, data2=capot_data, data3="capot_dogfall", data4='x', cursor=cursor,redis_db=self.redis_db) # # 入球数单双------------------------------------------------------------------------------------------------------------- p_code, p_id = get_pcode(corner_ball, 'two_sides') odd_even_odd = item["odd_even_odd"] odd_even_even = item["odd_even_even"] half_odd_even_odd = item["half_odd_even_odd"] half_odd_even_even = item["half_odd_even_even"] # 全场入球数 单双 two_sides_data = {'league_id': league_id, 'game_id': competition_id, 'utime': utime, 'p_id': p_id, 'p_code': p_code, 'source': "hg3535", 'expire_time': expire_time, 'sort': 0, 'pt': pt,'match_score':match_score,'team_home':team_home,'team_guest':team_guest,'score_home':score_home,'score_guest':score_guest,'number':number,'time_game':time_game,'half_way':half_way} # 上半场入球数 单双 zqtwo_intodbs(data1=odd_even_odd, data2=two_sides_data, data3="two_sides_single", data4='单', cursor=cursor,redis_db=self.redis_db) zqtwo_intodbs(data1=odd_even_even, data2=two_sides_data, data3="two_sides_double", data4='双', cursor=cursor,redis_db=self.redis_db) # 全场入球数 单双 zqtwo_intodbs(data1=half_odd_even_odd, data2=two_sides_data, data3="half_two_sides_single", data4='单', cursor=cursor, redis_db=self.redis_db) zqtwo_intodbs(data1=half_odd_even_even, data2=two_sides_data, data3="half_two_sides_double", data4='双', cursor=cursor, redis_db=self.redis_db) # 总入球数 -------------------------------------------------------------------------------------------------------------- p_code, p_id = get_pcode(corner_ball, 'total_goal') total_goals = item['total_goal'] total_dict = {'total_goal_zero': '0-1', 'total_goal_two': '2-3', 'total_goal_four': '4-6', 'total_goal_seven': '7或以上', 'half_total_goal_zero': '0', "half_total_goal_one": '1', "half_total_goal_two": '2', "half_total_goal_three": '3或以上'} # 全场入球数 单双 total_goal_data = {'league_id': league_id, 'game_id': competition_id, 'utime': utime, 'p_id': p_id, 'p_code': p_code, 'source': "hg3535", 'expire_time': expire_time, 'sort': 0, 'pt': pt,'match_score':match_score,'team_home':team_home,'team_guest':team_guest,'score_home':score_home,'score_guest':score_guest,'number':number,'time_game':time_game,'half_way':half_way} # 上半场入球数 单双 for key, value in total_goals.items(): if value: zqtwo_intodbs(data1=value, data2=total_goal_data, data3=key, data4=total_dict[key], cursor=cursor, redis_db=self.redis_db) # 全场半场 -------------------------------------------------------------------------------------------------------------- half_fulls = item['half_full'] p_code, p_id = get_pcode(corner_ball, 'half_full') full_dict = {"half_full_home_home": "主主", "half_full_home_dogfall": "主和", "half_full_home_guest": "主客", "half_full_dogfall_home": "和主", "half_full_dogfall_dogfall": "和和", "half_full_dogfall_guest": "和客", "half_full_guest_home": "客主", "half_full_guest_dogfall": "客和", "half_full_guest_guest": "客客"} half_full_data = {'league_id': league_id, 'game_id': competition_id, 'utime': utime, 'p_id': p_id, 'p_code': p_code, 'source': "hg3535", 'expire_time': expire_time, 'sort': 0, 'pt': pt,'match_score':match_score,'team_home':team_home,'team_guest':team_guest,'score_home':score_home,'score_guest':score_guest,'number':number,'time_game':time_game,'half_way':half_way} if half_fulls: for key, value in half_fulls.items(): if value: zqtwo_intodbs(data1=value, data2=half_full_data, data3=key, data4=full_dict[key], cursor=cursor, redis_db=self.redis_db) # 波胆------------------------------------------------------------------------------------------------------------------ bodan_datas = item['bodan_data'] p_code, p_id = get_pcode(corner_ball, 'bodan') bodan_dict = {"bodanhome_one_zero": "1-0", "bodanhome_two_zero": "2-0", "bodanhome_two_one": "2-1", "bodanhome_three_zero": "3-0", "bodanhome_three_one": "3-1", "bodanhome_three_two": "3-2", "bodanhome_four_zero": "4-0", "bodanhome_four_one": "4-1", "bodanhome_four_two": "4-2", "bodanhome_four_three": "4-3", "bodanguest_one_zero": "0-1", "bodanguest_two_zero": "0-2", "bodanguest_two_one": "1-2", "bodanguest_three_zero": "0-3", "bodanguest_three_one": "1-3", "bodanguest_three_two": "2-3", "bodanguest_four_zero": "0-4", "bodanguest_four_one": "1-4", "bodanguest_four_two": "2-4", "bodanguest_four_three": "3-4", "bodandogfall_zero_zero": "0-0", "bodandogfall_one_one": "1-1", "bodandogfall_two_two": "2-2", "bodandogfall_three_three": "3-3", "bodandogfall_four_four": "4-4", "bodanother": "其他", "halfbodanhome_one_zero": "1-0", "halfbodanhome_two_zero": "2-0", "halfbodanhome_two_one": "2-1", "halfbodanhome_three_zero": "3-0", "halfbodanhome_three_one": "3-1", "halfbodanhome_three_two": "3-2", "halfbodanguest_one_zero": "0-1", "halfbodanguest_two_zero": "0-2", "halfbodanguest_two_one": "1-2", "halfbodanguest_three_zero": "0-3", "halfbodanguest_three_one": "1-3", "halfbodanguest_three_two": "2-3", "halfbodandogfall_zero_zero": "0-0", "halfbodandogfall_one_one": "1-1", "halfbodandogfall_two_two": "2-2", "halfbodandogfall_three_three": "3-3", "halfbodanother": "其他"} bodan_data = {'league_id': league_id, 'game_id': competition_id, 'utime': utime, 'p_id': p_id, 'p_code': p_code, 'source': "hg3535", 'expire_time': expire_time, 'sort': 0, 'pt': pt,'match_score':match_score,'team_home':team_home,'team_guest':team_guest,'score_home':score_home,'score_guest':score_guest,'number':number,'time_game':time_game,'half_way':half_way} if bodan_datas: for key, value in bodan_datas.items(): zqtwo_intodbs(data1=value, data2=bodan_data, data3=key, data4=bodan_dict[key], cursor=cursor, redis_db=self.redis_db) # 最先进球/最后进球 ------------------------------------------------------------------------------------------------------ first_last_balls = item['first_last_ball'] p_code, p_id = get_pcode(corner_ball, 'first_last_ball') first_last_dict = {"first_last_ball": "最先进球", "first_ball_home": "最先进球", "first_ball_guest": "最先进球", "last_ball_home": "最后进球", "last_ball_guest": "最后进球", "not_ball": "没有进球"} first_last_data = {'league_id': league_id, 'game_id': competition_id, 'utime': utime, 'p_id': p_id, 'p_code': p_code, 'source': "hg3535", 'expire_time': expire_time, 'sort': 0, 'pt': pt,'match_score':match_score,'team_home':team_home,'team_guest':team_guest,'score_home':score_home,'score_guest':score_guest,'number':number,'time_game':time_game,'half_way':half_way} if first_last_balls: for key, value in first_last_balls.items(): # 构建唯一索引 zqtwo_intodbs(data1=value, data2=first_last_data, data3=key, data4=first_last_dict[key], cursor=cursor, redis_db=self.redis_db) p_code, p_id = get_pcode(corner_ball, 'temaball') full_dicts = item['full_data'] half_dicts = item['half_data'] full_dict_rules = item['full_data_rule'] half_dict_rules = item['half_data_rule'] data = {'league_id': league_id, 'game_id': competition_id, 'utime': utime, 'p_id': p_id, 'p_code': p_code, 'source': "hg3535", 'expire_time': expire_time, 'sort': 0, 'pt': pt,'match_score':match_score,'team_home':team_home,'team_guest':team_guest,'score_home':score_home,'score_guest':score_guest,'number':number,'time_game':time_game,'half_way':half_way} if full_dicts: for key, value in full_dicts.items(): zqtwo_intodbs(data1=value, data2=data, data3=key, data4=full_dict_rules[key], cursor=cursor, redis_db=self.redis_db) if half_dicts: for key, value in half_dicts.items(): zqtwo_intodbs(data1=value, data2=data, data3=key, data4=half_dict_rules[key], cursor=cursor, redis_db=self.redis_db) #这日期在这里有问题 先行注释 # match_date, match_time, time3 = news_time(ctime) # n_time = out_time(time3, 1.5) # 插入st_zq_competition表 # if pt == 0: data_competition = {'team_home': team_home, 'team_guest': team_guest, 'league_id': league_id, 'game_id': competition_id, 'match_date': r_ctime, 'match_time': '00:00:00', 'utime': utime, 'number': number, 'source': "hg3535", "expire_time": expire_time, 'is_rollball': 1, "us_time":'2019-01-01 00:00:00','time_game':time_game} zqone_competitions(data=data_competition, cursor=cursor) def close_spider(self, spider): # self.conn.close() self.dbpool.close() class BallStatuspipeline(object): def open_spider(self, spider): self.dbpool = adbapi.ConnectionPool("psycopg2",host=settings["POST_HOST"], port=settings['POST_PORT'], user=settings["POST_USER"], password=settings["POST_PASSWORD"], database=settings["POST_DATABASE"]) self.pool = redis.ConnectionPool(host=settings["R_HOST"], port=settings["R_POST"], password=settings["R_PASSWORD"]) self.redis_db = redis.StrictRedis(connection_pool=self.pool) def process_item(self, item, spider): # logger.warning(query.addErrback(self.handle_error, item, spider)) # 使用twisted将mysql插入变成异步执行 query = self.dbpool.runInteraction(self.do_insert, item) query.addErrback(self.handle_error, item, spider) # 处理异常 return item def handle_error(self, failure, item, spider): # 处理异步插入的异常 logger = logging.getLogger(__name__) logger.warning(failure) # print(failure) def do_insert(self, cursor, item): # 联赛id if isinstance(item, Zuqiustatus): zuqiu_toal = item['zuqiu_total'] for i in zuqiu_toal: print(i) match_id = i['match_id'] create_time = i['create_time'] staus = i['status'] update_time = i['create_time'] ball_type = i['ball_type'] sql1 = "insert into st_ball_status(match_id, create_time, status,update_time, ball_type) values (%s,%s, %s, %s, %s) on conflict(match_id) do update set update_time = %s;" cursor.execute(sql1, (match_id, create_time,staus,update_time, ball_type,update_time)) sql2 = "update st_zq_result set status=1 where match_id={}".format(match_id) sql3 = "update st_zq_result_record set status=1 where match_id={}".format(match_id) sql4 = "update st_zq_competition set status=1 where match_id={}".format(match_id) cursor.execute(sql2) cursor.execute(sql3) cursor.execute(sql4) # 现在时间，时间戳 utime = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime()) def close_spider(self, spider): # self.conn.close() self.dbpool.close() class Roll_Lanqiupipeline(object): def open_spider(self, spider): self.dbpool = adbapi.ConnectionPool("psycopg2",host=settings["POST_HOST"], port=settings['POST_PORT'], user=settings["POST_USER"], password=settings["POST_PASSWORD"], database=settings["POST_DATABASE"]) self.pool = redis.ConnectionPool(host=settings["R_HOST"], port=settings["R_POST"], password=settings["R_PASSWORD"]) self.redis_db = redis.StrictRedis(connection_pool=self.pool) def process_item(self, item, spider): # 使用twisted将mysql插入变成异步执行 query = self.dbpool.runInteraction(self.do_insert, item) query.addErrback(self.handle_error, item, spider) # 处理异常 return item def handle_error(self, failure, item, spider): # 处理异步插入的异常 logger = logging.getLogger(__name__) logger.warning(failure) # print(failure) def do_insert(self, cursor, item): # 联赛id league_id = item['league_id'] # 联赛名 league_name = item['league_name'] # result = item['result'] # 比赛id game_id = item['game_id'] # 球队1 #home_team team_home = item['team_home'] # 球队2 # guest_team team_guest = item['team_guest'] # 数量(97>) number = item['number'] # 比赛状态 zhuangtai = item['zhuangtai'] # 日期 # data_game = item['data_game'] data_game = item['data_game'].split("/") month = str(data_game[1].strip()) day = str(data_game[0]) # 比赛时间 time_game = str(item['time_game']) # 比赛时间，时间戳 ctime = "2019" + "-" + month + "-" + day + "" + time_game + ":00" r_ctime = "2019" + "-" + month + "-" + day expire_time = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(time.time() + 60))#过期时间 # 现在时间，时间戳 utime = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime()) # 队1分数 score_home = item['score_home'] # 队2分数 score_guest = item['score_guest'] # 第几节 jijie = item['jijie'] # 球队得分 qiudui = item['qiudui'] pt = item['pt'] match_score = "{}:{}".format(score_home,score_guest) concedes_dict = item['concede'] concedes_dict_rule = item['concede_rule'] odd_evens_dict = item['odd_even'] odd_evens_dict_rule = item['odd_even_rule'] total_sizes_dict = item['total_size'] total_sizes_dict_rule = item['total_size_rule'] last_numbers_dict = item['last_number'] capots_dict = item['capot'] team_scores_dict = item['team_score'] team_scores_dict_rule = item['team_score_rule'] if concedes_dict: for key, value in concedes_dict.items(): if value: for x, y in enumerate(value): new_hash = hash_func(match_id=game_id, odds_code=key, sort=x, p_id=1) odds_only = r_func(match_id=game_id, odds_code=key, sort=x, p_id=1,odd=y) sql1 = "insert into st_lq_odds(lg_id, odds_code, match_id, ctime, utime, odds, p_id, p_code, sort, source, sole,condition,odds_only,is_rollball) values (%s, %s, %s, %s, %s, %s, %s,%s, %s, %s, %s, %s, %s,%s) on conflict(sole) do update set utime = %s,odds = %s,odds_only =%s;" cursor.execute(sql1, (int(league_id), key, int(game_id), utime, utime, y, 1, "concede",x, "hg3535", new_hash, concedes_dict_rule[key][x],odds_only,1,utime, y,odds_only)) # res = cursor.fetchone() # 更新主队st_zq_odds_record表 sql2 = "insert into st_lq_odds_record(lg_id, odds_code, match_id, ctime, utime, odds, p_id, p_code, sort, source, condition,odds_only,is_rollball) values (%s, %s, %s, %s, %s, %s,%s, %s, %s, %s, %s,%s,%s);" cursor.execute(sql2, (int(league_id), key, int(game_id), utime, utime,y, 1, "concede", x, "hg3535", concedes_dict_rule[key][x],odds_only,1)) #插入结果表 sql3 = "insert into st_lq_result(lg_id, home_team, guest_team, home_score, guest_score, all_goal, first_score, last_score,match_score,match_winer,update_time,match_time,match_process,tag,match_id,u_home_score,u_guest_score,source) values (%s, %s, %s, %s, %s, %s, %s,%s, %s, %s, %s, %s, %s,%s,%s,%s,%s,%s) on conflict(match_id) do update set update_time = %s,match_score = %s,match_time =%s,match_process =%s,home_score =%s,guest_score =%s;" cursor.execute(sql3, (int(league_id), team_home, team_guest,score_home, score_guest, number, '', '',match_score,'',utime,time_game,jijie,number,int(game_id),0,0,'hg3535',utime,match_score,time_game,jijie,score_home,score_guest)) #插入记录表 sql4 = "insert into st_lq_result_record(lg_id, home_team, guest_team, home_score, guest_score, first_score, last_score,match_score,match_winer,update_time,match_time,match_process,tag,match_id,source,status) values (%s, %s, %s, %s, %s, %s, %s,%s, %s, %s, %s, %s, %s,%s,%s,%s);" cursor.execute(sql4, (int(league_id), team_home, team_guest,score_home, score_guest, '', '',match_score,'',utime,time_game,jijie,number,int(game_id),'hg3535',0)) if odd_evens_dict: for key, value in odd_evens_dict.items(): if value: for x, y in enumerate(value): new_hash = hash_func(match_id=game_id, odds_code=key, sort=x, p_id=2) odds_only = r_func(match_id=game_id, odds_code=key, sort=x, p_id=2,odd=y) sql1 = "insert into st_lq_odds(lg_id, odds_code, match_id, ctime, utime,odds, p_id, p_code, sort, source, sole,condition,odds_only,is_rollball) values (%s, %s, %s, %s, %s, %s,%s, %s, %s, %s, %s, %s,%s,%s) on conflict(sole) do update set utime = %s,odds = %s,odds_only =%s;" cursor.execute(sql1, (int(league_id), key, int(game_id), utime, utime,y, 2, "two_sides",x, "hg3535", new_hash, odd_evens_dict_rule[key][x],odds_only,1,utime, y,odds_only)) # 更新主队st_zq_odds_record表 sql2 = "insert into st_lq_odds_record(lg_id, odds_code, match_id, ctime, utime,odds, p_id, p_code, sort, source, condition,odds_only,is_rollball) values (%s, %s, %s, %s, %s, %s, %s,%s, %s, %s, %s, %s,%s);" cursor.execute(sql2, (int(league_id), key, int(game_id), utime, utime,y, 2, "two_sides", x, "hg3535", odd_evens_dict_rule[key][x],odds_only,1)) sql3 = "insert into st_lq_result(lg_id, home_team, guest_team, home_score, guest_score, all_goal, first_score, last_score,match_score,match_winer,update_time,match_time,match_process,tag,match_id,u_home_score,u_guest_score,source) values (%s, %s, %s, %s, %s, %s, %s,%s, %s, %s, %s, %s, %s,%s,%s,%s,%s,%s) on conflict(match_id) do update set update_time = %s,match_score = %s,match_time =%s,match_process =%s,home_score =%s,guest_score =%s;" cursor.execute(sql3, (int(league_id), team_home, team_guest,score_home, score_guest, number, '', '',match_score,'',utime,time_game,jijie,number,int(game_id),0,0,'hg3535',utime,match_score,time_game,jijie,score_home,score_guest)) #插入记录表 sql4 = "insert into st_lq_result_record(lg_id, home_team, guest_team, home_score, guest_score, first_score, last_score,match_score,match_winer,update_time,match_time,match_process,tag,match_id,source,status) values (%s, %s, %s, %s, %s, %s, %s,%s, %s, %s, %s, %s, %s,%s,%s,%s);" cursor.execute(sql4, (int(league_id), team_home, team_guest,score_home, score_guest, '', '',match_score,'',utime,time_game,jijie,number,int(game_id),'hg3535',0)) if total_sizes_dict: for key, value in total_sizes_dict.items(): if value: for x, y in enumerate(value): new_hash = hash_func(match_id=game_id, odds_code=key, sort=x, p_id=3) odds_only = r_func(match_id=game_id, odds_code=key, sort=x, p_id=3,odd=y) sql1 = "insert into st_lq_odds(lg_id, odds_code, match_id, ctime, utime,odds, p_id, p_code, sort, source, sole,condition,odds_only,is_rollball) values (%s, %s, %s, %s, %s, %s,%s, %s, %s, %s, %s, %s,%s,%s) on conflict(sole) do update set utime = %s,odds = %s,odds_only =%s;" cursor.execute(sql1, (int(league_id), key, int(game_id), utime, utime,y, 3, "total_size",x, "hg3535", new_hash, total_sizes_dict_rule[key][x],odds_only,1,utime, y,odds_only)) # 更新主队st_zq_odds_record表 sql2 = "insert into st_lq_odds_record(lg_id, odds_code, match_id, ctime, utime,odds, p_id, p_code, sort, source, condition,odds_only,is_rollball) values (%s, %s, %s, %s, %s, %s,%s, %s, %s, %s, %s,%s,%s);" cursor.execute(sql2, (int(league_id), key, int(game_id), utime, utime, y, 3, "total_size", x, "hg3535", total_sizes_dict_rule[key][x],odds_only,1)) sql3 = "insert into st_lq_result(lg_id, home_team, guest_team, home_score, guest_score, all_goal, first_score, last_score,match_score,match_winer,update_time,match_time,match_process,tag,match_id,u_home_score,u_guest_score,source) values (%s, %s, %s, %s, %s, %s, %s,%s, %s, %s, %s, %s, %s,%s,%s,%s,%s,%s) on conflict(match_id) do update set update_time = %s,match_score = %s,match_time =%s,match_process =%s,home_score =%s,guest_score =%s;" cursor.execute(sql3, (int(league_id), team_home, team_guest,score_home, score_guest, number, '', '',match_score,'',utime,time_game,jijie,number,int(game_id),0,0,'hg3535',utime,match_score,time_game,jijie,score_home,score_guest)) #插入记录表 sql4 = "insert into st_lq_result_record(lg_id, home_team, guest_team, home_score, guest_score, first_score, last_score,match_score,match_winer,update_time,match_time,match_process,tag,match_id,source,status) values (%s, %s, %s, %s, %s, %s, %s,%s, %s, %s, %s, %s, %s,%s,%s,%s);" cursor.execute(sql4, (int(league_id), team_home, team_guest,score_home, score_guest, '', '',match_score,'',utime,time_game,jijie,number,int(game_id),'hg3535',0)) last_home = last_numbers_dict['last_home'] if last_home: for key, value in last_home.items(): if value: new_hash = hash_func(match_id=game_id, odds_code="last_home" + key, sort=0, p_id=4) odds_only = r_func(match_id=game_id, odds_code="last_home", sort=0, p_id=4,odd=value) sql1 = "insert into st_lq_odds(lg_id, odds_code, match_id, ctime, utime,odds, p_id, p_code, sort, source, sole,condition,odds_only,is_rollball) values (%s, %s, %s, %s, %s, %s,%s, %s, %s, %s, %s, %s,%s,%s) on conflict(sole) do update set utime = %s,odds = %s,odds_only=%s;" cursor.execute(sql1, (int(league_id), "last_home", int(game_id), utime, utime, value, 4, "last_number",0, "hg3535", new_hash, key,odds_only,1,utime, value,odds_only)) # 更新主队st_zq_odds_record表 sql2 = "insert into st_lq_odds_record(lg_id, odds_code, match_id, ctime, utime,odds, p_id, p_code, sort, source, condition,odds_only,is_rollball) values (%s, %s, %s, %s, %s, %s,%s, %s, %s, %s, %s,%s,%s);" cursor.execute(sql2, (int(league_id), "last_home", int(game_id), utime, utime,value, 4, "last_number", 0, "hg3535", key,odds_only,1)) sql3 = "insert into st_lq_result(lg_id, home_team, guest_team, home_score, guest_score, all_goal, first_score, last_score,match_score,match_winer,update_time,match_time,match_process,tag,match_id,u_home_score,u_guest_score,source) values (%s, %s, %s, %s, %s, %s, %s,%s, %s, %s, %s, %s, %s,%s,%s,%s,%s,%s) on conflict(match_id) do update set update_time = %s,match_score = %s,match_time =%s,match_process =%s,home_score =%s,guest_score =%s;" cursor.execute(sql3, (int(league_id), team_home, team_guest,score_home, score_guest, number, '', '',match_score,'',utime,time_game,jijie,number,int(game_id),0,0,'hg3535',utime,match_score,time_game,jijie,score_home,score_guest)) #插入记录表 sql4 = "insert into st_lq_result_record(lg_id, home_team, guest_team, home_score, guest_score, first_score, last_score,match_score,match_winer,update_time,match_time,match_process,tag,match_id,source,status) values (%s, %s, %s, %s, %s, %s, %s,%s, %s, %s, %s, %s, %s,%s,%s,%s);" cursor.execute(sql4, (int(league_id), team_home, team_guest,score_home, score_guest, '', '',match_score,'',utime,time_game,jijie,number,int(game_id),'hg3535',0)) last_guest = last_numbers_dict['last_guest'] if last_guest: for key, value in last_guest.items(): if value: new_hash = hash_func(match_id=game_id, odds_code="last_guest" + key, sort=0, p_id=4) odds_only = r_func(match_id=game_id, odds_code="last_guest", sort=0, p_id=4,odd=value) sql1 = "insert into st_lq_odds(lg_id, odds_code, match_id, ctime, utime,odds, p_id, p_code, sort, source, sole,condition,odds_only,is_rollball) values (%s, %s, %s, %s, %s, %s,%s, %s, %s, %s, %s, %s,%s,%s) on conflict(sole) do update set utime = %s,odds = %s,odds_only=%s;" cursor.execute(sql1, (int(league_id), "last_guest", int(game_id), utime, utime, value, 4, "last_number",0, "hg3535", new_hash, key,odds_only,1,utime, value,odds_only)) # 更新主队st_zq_odds_record表 sql2 = "insert into st_lq_odds_record(lg_id, odds_code, match_id, ctime, utime,odds, p_id, p_code, sort, source, condition,odds_only,is_rollball) values (%s, %s, %s, %s, %s, %s,%s, %s, %s, %s, %s,%s,%s);" cursor.execute(sql2, (int(league_id), "last_guest", int(game_id), utime, utime,value, 4, "last_number", 0, "hg3535", key,odds_only,1)) sql3 = "insert into st_lq_result(lg_id, home_team, guest_team, home_score, guest_score, all_goal, first_score, last_score,match_score,match_winer,update_time,match_time,match_process,tag,match_id,u_home_score,u_guest_score,source) values (%s, %s, %s, %s, %s, %s, %s,%s, %s, %s, %s, %s, %s,%s,%s,%s,%s,%s) on conflict(match_id) do update set update_time = %s,match_score = %s,match_time =%s,match_process =%s,home_score =%s,guest_score =%s;" cursor.execute(sql3, (int(league_id), team_home, team_guest,score_home, score_guest, number, '', '',match_score,'',utime,time_game,jijie,number,int(game_id),0,0,'hg3535',utime,match_score,time_game,jijie,score_home,score_guest)) #插入记录表 sql4 = "insert into st_lq_result_record(lg_id, home_team, guest_team, home_score, guest_score, first_score, last_score,match_score,match_winer,update_time,match_time,match_process,tag,match_id,source,status) values (%s, %s, %s, %s, %s, %s, %s,%s, %s, %s, %s, %s, %s,%s,%s,%s);" cursor.execute(sql4, (int(league_id), team_home, team_guest,score_home, score_guest, '', '',match_score,'',utime,time_game,jijie,number,int(game_id),'hg3535',0)) if capots_dict: for key, value in capots_dict.items(): if value: new_hash = hash_func(match_id=game_id, odds_code=key, sort=0, p_id=5) odds_only = r_func(match_id=game_id, odds_code=key, sort=0, p_id=5,odd=value) sql1 = "insert into st_lq_odds(lg_id, odds_code, match_id, ctime, utime, odds, p_id, p_code, sort, source, sole,odds_only,is_rollball) values (%s, %s, %s, %s, %s,%s, %s, %s, %s, %s, %s,%s,%s) on conflict(sole) do update set utime = %s,odds = %s,odds_only =%s;" cursor.execute(sql1, (int(league_id), key, int(game_id), utime, utime, value, 5, "capot",0, "hg3535", new_hash, odds_only,1,utime, value,odds_only)) # 更新主队st_zq_odds_record表 sql2 = "insert into st_lq_odds_record(lg_id, odds_code, match_id, ctime, utime,odds, p_id, p_code, sort, source,odds_only,is_rollball) values (%s, %s, %s, %s, %s,%s, %s, %s, %s, %s,%s,%s);" cursor.execute(sql2, (int(league_id), key, int(game_id), utime, utime, value, 5, "capot", 0, "hg3535",odds_only,1)) sql3 = "insert into st_lq_result(lg_id, home_team, guest_team, home_score, guest_score, all_goal, first_score, last_score,match_score,match_winer,update_time,match_time,match_process,tag,match_id,u_home_score,u_guest_score,source) values (%s, %s, %s, %s, %s, %s, %s,%s, %s, %s, %s, %s, %s,%s,%s,%s,%s,%s) on conflict(match_id) do update set update_time = %s,match_score = %s,match_time =%s,match_process =%s,home_score =%s,guest_score =%s;" cursor.execute(sql3, (int(league_id), team_home, team_guest,score_home, score_guest, number, '', '',match_score,'',utime,time_game,jijie,number,int(game_id),0,0,'hg3535',utime,match_score,time_game,jijie,score_home,score_guest)) #插入记录表 sql4 = "insert into st_lq_result_record(lg_id, home_team, guest_team, home_score, guest_score, first_score, last_score,match_score,match_winer,update_time,match_time,match_process,tag,match_id,source,status) values (%s, %s, %s, %s, %s, %s, %s,%s, %s, %s, %s, %s, %s,%s,%s,%s);" cursor.execute(sql4, (int(league_id), team_home, team_guest,score_home, score_guest, '', '',match_score,'',utime,time_game,jijie,number,int(game_id),'hg3535',0)) if team_scores_dict: for key, value in team_scores_dict.items(): if value: for x, y in enumerate(value): new_hash = hash_func(match_id=game_id, odds_code=key, sort=x, p_id=6) odds_only = r_func(match_id=game_id, odds_code=key, sort=x, p_id=6,odd=y) sql1 = "insert into st_lq_odds(lg_id, odds_code, match_id, ctime, utime,odds, p_id, p_code, sort, source, sole,condition,odds_only,is_rollball) values (%s, %s, %s, %s, %s, %s,%s, %s, %s, %s, %s, %s,%s,%s) on conflict(sole) do update set utime = %s,odds = %s,odds_only=%s;" cursor.execute(sql1, (int(league_id), key, int(game_id), utime, utime, y, 6, "team_score", x, "hg3535", new_hash,team_scores_dict_rule[key][x], odds_only,1,utime, y,odds_only)) # 更新主队st_zq_odds_record表 sql2 = "insert into st_lq_odds_record(lg_id, odds_code, match_id, ctime, utime,odds, p_id, p_code, sort, source, condition,odds_only,is_rollball) values (%s, %s, %s, %s, %s, %s,%s, %s, %s, %s, %s,%s,%s);" cursor.execute(sql2, (int(league_id), key, int(game_id), utime, utime,y, 6, "team_score", x, "hg3535",team_scores_dict_rule[key][x],odds_only,1)) sql3 = "insert into st_lq_result(lg_id, home_team, guest_team, home_score, guest_score, all_goal, first_score, last_score,match_score,match_winer,update_time,match_time,match_process,tag,match_id,u_home_score,u_guest_score,source) values (%s, %s, %s, %s, %s, %s, %s,%s, %s, %s, %s, %s, %s,%s,%s,%s,%s,%s) on conflict(match_id) do update set update_time = %s,match_score = %s,match_time =%s,match_process =%s,home_score =%s,guest_score =%s;" cursor.execute(sql3, (int(league_id), team_home, team_guest,score_home, score_guest, number, '', '',match_score,'',utime,time_game,jijie,number,int(game_id),0,0,'hg3535',utime,match_score,time_game,jijie,score_home,score_guest)) #插入记录表 sql4 = "insert into st_lq_result_record(lg_id, home_team, guest_team, home_score, guest_score, first_score, last_score,match_score,match_winer,update_time,match_time,match_process,tag,match_id,source,status) values (%s, %s, %s, %s, %s, %s, %s,%s, %s, %s, %s, %s, %s,%s,%s,%s);" cursor.execute(sql4, (int(league_id), team_home, team_guest,score_home, score_guest, '', '',match_score,'',utime,time_game,jijie,number,int(game_id),'hg3535',0)) # 插入赛事表 Competition_sql = "insert into st_lq_competition(home_team, guest_team, lg_id, match_id, match_date, match_time,ctime, utime, tag,type,source,expire_time,is_rollball) values (%s, %s, %s, %s, %s, %s, %s, %s,%s, %s, %s,%s,%s) on conflict(match_id) do update set tag = %s,expire_time = %s;" print(Competition_sql) cursor.execute(Competition_sql, (team_home, team_guest, league_id, game_id, r_ctime, time_game, utime, utime, number, pt, "hg3535",expire_time,1,number,expire_time)) def close_spider(self, spider): self.dbpool.close() def new_times(ctime): time1 = time.mktime(time.strptime(ctime, '%Y-%m-%d %H:%M:%S')) + 43200 time2 = time.localtime(time1) time3 = time.strftime('%Y-%m-%d %H:%M:%S', time2) data_time = str(time3).split(" ") match_date = data_time[0] match_time = data_time[1] return match_date,match_time class Roll_Wangqiupipeline(object): def open_spider(self, spider): self.dbpool = adbapi.ConnectionPool("psycopg2",host=settings["POST_HOST"], port=settings['POST_PORT'], user=settings["POST_USER"], password=settings["POST_PASSWORD"], database=settings["POST_DATABASE"]) self.pool = redis.ConnectionPool(host=settings["R_HOST"], port=settings["R_POST"], password=settings["R_PASSWORD"]) self.redis_db = redis.StrictRedis(connection_pool=self.pool) def process_item(self, item, spider): # 使用twisted将mysql插入变成异步执行 query = self.dbpool.runInteraction(self.do_insert, item) query.addErrback(self.handle_error, item, spider) # 处理异常 return item def handle_error(self, failure, item, spider): # 处理异步插入的异常 logger = logging.getLogger(__name__) logger.warning(failure) # print(failure) def do_insert(self, cursor, item): # 联赛id league_id = item['league_id'] # 联赛名 league_name = item['league_name'] # result = item['result'] # 比赛id game_id = item['game_id'] # 球队1 team_home = item['team_home'] # 球队2 team_guest = item['team_guest'] # 数量(97>) number = item['number'] # 比赛状态 zhuangtai = item['zhuangtai'] # 日期 # data_game = item['data_game'] data_game = item['data_game'].split("/") month = str(data_game[1].strip()) day = str(data_game[0]) # 比赛时间 time_game = str(item['time_game']) # 比赛时间，时间戳 ctime = "2019" + "-" + month + "-" + day + "" + time_game + ":00" r_ctime = "2019" + "-" + month + "-" + day # 现在时间，时间戳 utime = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime()) # 队1分数 score_home = item['score_home'] # 队2分数 score_guest = item['score_guest'] # 第几节 jijie = item['jijie'] # 球队得分 qiudui = item['qiudui'] pt = item['pt'] #取不到 暂时注掉 match_date, match_time = new_times(ctime) # 让盘 concedes_dict = item['concedes_dict'] concedes_dict_rule = item['concedes_dict_rule'] # 冠军 独赢 kemps_dict = item['kemps_dict'] # 让局 bureaus_dict = item['bureaus_dict'] bureaus_dict_rule = item['bureaus_dict_rule'] # 总局数大小 total_number_dict = item['total_number_dict'] total_number_dict_rule = item['total_number_dict_rule'] # 总局数单双 odd_evens_dict = item['odd_evens_dict'] odd_evens_dict_rule = item['odd_evens_dict_rule'] if concedes_dict: for key, value in concedes_dict.items(): if value: new_hash = hash_func(match_id=game_id, odds_code=key, sort=0, p_id=1) odds_only = r_func(match_id=game_id, odds_code=key, sort=0, p_id=1,odd=value) sql1 = "insert into st_wq_odds(lg_id, odds_code, match_id, ctime, utime, odds, p_id, p_code, sort, source, sole,condition,odds_only,is_rollball) values (%s, %s, %s, %s, %s, %s,%s, %s, %s, %s, %s, %s,%s,%s) on conflict(sole) do update set utime = %s,odds = %s,odds_only=%s;" cursor.execute(sql1, ( int(league_id), key, int(game_id), utime, utime,value, 1, "dishes_home", 0, "hg3535", new_hash, concedes_dict_rule[key],odds_only,1,utime, value,odds_only)) # 更新主队st_zq_odds_record表 sql2 = "insert into st_wq_odds_record(lg_id, odds_code, match_id, ctime, utime, odds, p_id, p_code, sort, source, condition,odds_only,is_rollball) values (%s, %s, %s, %s, %s, %s,%s, %s, %s, %s, %s,%s,%s);" cursor.execute(sql2, ( int(league_id), key, int(game_id), utime, utime, value, 1, "dishes_home", 0, "hg3535", concedes_dict_rule[key],odds_only,1)) #插入结果表 sql3 = "insert into st_wq_result(lg_id, home_player_name, guest_player_name, home_player_let_plate, guest_player_let_plate, all_inning,home_player_score,guest_player_score,status,first_score_player,last_score_player,first_inning_score,second_inning_score,third_inning_score,match_winer_player,update_time,match_process,tag,match_id,source,match_time) values (%s, %s, %s, %s, %s,%s, %s, %s, %s, %s, %s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s) on conflict(match_id) do update set update_time = %s,match_process =%s,home_player_score =%s,guest_player_score =%s;" cursor.execute(sql3, (int(league_id), team_home, team_guest,int(score_home), int(score_guest), 0, score_home,score_guest,1,'','','','','','',utime,jijie,number,game_id,'hg3535','00:00',utime,jijie,score_home,score_guest)) # 插入记录表 sql4 = "insert into st_wq_result_record(lg_id, home_player_name, guest_player_name, home_player_let_plate, guest_player_let_plate, all_inning,home_player_score,guest_player_score,status,first_score_player,last_score_player,first_inning_score,second_inning_score,third_inning_score,match_winer_player,update_time,match_process,tag,match_id,source,match_time) values (%s, %s, %s, %s, %s,%s, %s, %s, %s, %s, %s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s);" cursor.execute(sql4, (int(league_id), team_home, team_guest,int(score_home), int(score_guest), 0, score_home,score_guest,1,'','','','','','',utime,jijie,number,game_id,'hg3535','00:00')) if kemps_dict: for key, value in kemps_dict.items(): if value: new_hash = hash_func(match_id=game_id, odds_code=key, sort=0, p_id=2) odds_only = r_func(match_id=game_id, odds_code=key, sort=0, p_id=2,odd=value) sql1 = "insert into st_wq_odds(lg_id, odds_code, match_id, ctime, utime, odds, p_id, p_code, sort, source, sole,odds_only,is_rollball) values (%s, %s, %s, %s, %s, %s, %s,%s, %s, %s, %s, %s,%s) on conflict(sole) do update set utime = %s,odds = %s,odds_only=%s;" cursor.execute(sql1, ( int(league_id), key, int(game_id), utime, utime, value, 2, "kemp", 0, "hg3535", new_hash,odds_only,1,utime, value,odds_only)) # 更新主队st_zq_odds_record表 sql2 = "insert into st_wq_odds_record(lg_id, odds_code, match_id, ctime, utime, odds, p_id, p_code, sort, source,odds_only,is_rollball) values (%s, %s, %s, %s, %s, %s,%s, %s, %s, %s,%s,%s);" cursor.execute(sql2, ( int(league_id), key, int(game_id), utime, utime,value, 2, "kemp", 0, "hg3535",odds_only,1)) sql3 = "insert into st_wq_result(lg_id, home_player_name, guest_player_name, home_player_let_plate, guest_player_let_plate, all_inning,home_player_score,guest_player_score,status,first_score_player,last_score_player,first_inning_score,second_inning_score,third_inning_score,match_winer_player,update_time,match_process,tag,match_id,source,match_time) values (%s, %s, %s, %s, %s,%s, %s, %s, %s, %s, %s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s) on conflict(match_id) do update set update_time = %s,match_process =%s,home_player_score =%s,guest_player_score =%s;" cursor.execute(sql3, (int(league_id), team_home, team_guest,int(score_home), int(score_guest), 0, score_home,score_guest,1,'','','','','','',utime,jijie,number,game_id,'hg3535','00:00',utime,jijie,score_home,score_guest)) if bureaus_dict: for key, value in bureaus_dict.items(): if value: new_hash = hash_func(match_id=game_id, odds_code=key, sort=0, p_id=3) odds_only = r_func(match_id=game_id, odds_code=key, sort=0, p_id=3,odd=value) sql1 = "insert into st_wq_odds(lg_id, odds_code, match_id, ctime, utime, odds, p_id, p_code, sort, source, sole,condition,odds_only,is_rollball) values (%s, %s, %s, %s, %s, %s,%s, %s, %s, %s, %s, %s,%s,%s) on conflict(sole) do update set utime = %s,odds = %s,odds_only =%s;" cursor.execute(sql1, ( int(league_id), key, int(game_id), utime, utime, value, 3, "concede", 0, "hg3535", new_hash, bureaus_dict_rule[key],odds_only,1,utime, value,odds_only)) # 更新主队st_zq_odds_record表 sql2 = "insert into st_wq_odds_record(lg_id, odds_code, match_id, ctime, utime, odds, p_id, p_code, sort, source, condition,odds_only,is_rollball) values (%s, %s, %s, %s, %s, %s,%s, %s, %s, %s, %s,%s,%s);" cursor.execute(sql2, ( int(league_id), key, int(game_id), utime, utime,value, 3, "concede", 0, "hg3535", bureaus_dict_rule[key],odds_only,1)) sql3 = "insert into st_wq_result(lg_id, home_player_name, guest_player_name, home_player_let_plate, guest_player_let_plate, all_inning,home_player_score,guest_player_score,status,first_score_player,last_score_player,first_inning_score,second_inning_score,third_inning_score,match_winer_player,update_time,match_process,tag,match_id,source,match_time) values (%s, %s, %s, %s, %s,%s, %s, %s, %s, %s, %s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s) on conflict(match_id) do update set update_time = %s,match_process =%s,home_player_score =%s,guest_player_score =%s;" cursor.execute(sql3, (int(league_id), team_home, team_guest,int(score_home), int(score_guest), 0, score_home,score_guest,1,'','','','','','',utime,jijie,number,game_id,'hg3535','00:00',utime,jijie,score_home,score_guest)) if total_number_dict: for key, value in total_number_dict.items(): if value: new_hash = hash_func(match_id=game_id, odds_code=key, sort=0, p_id=4) odds_only = r_func(match_id=game_id, odds_code=key, sort=0, p_id=4,odd=value) sql1 = "insert into st_wq_odds(lg_id, odds_code, match_id, ctime, utime, odds, p_id, p_code, sort, source, sole,condition,odds_only,is_rollball) values (%s, %s, %s, %s, %s, %s,%s, %s, %s, %s, %s, %s,%s,%s) on conflict(sole) do update set utime = %s,odds = %s,odds_only=%s;" cursor.execute(sql1, ( int(league_id), key, int(game_id), utime, utime,value, 4, "total_number", 0, "hg3535", new_hash, total_number_dict_rule[key],odds_only,1,utime, value,odds_only)) # 更新主队st_zq_odds_record表 sql2 = "insert into st_wq_odds_record(lg_id, odds_code, match_id, ctime, utime, odds, p_id, p_code, sort, source, condition,odds_only,is_rollball) values (%s, %s, %s, %s, %s, %s,%s, %s, %s, %s, %s,%s,%s);" cursor.execute(sql2, ( int(league_id), key, int(game_id), utime, utime, value, 4, "total_number", 0, "hg3535", total_number_dict_rule[key],odds_only,1)) sql3 = "insert into st_wq_result(lg_id, home_player_name, guest_player_name, home_player_let_plate, guest_player_let_plate, all_inning,home_player_score,guest_player_score,status,first_score_player,last_score_player,first_inning_score,second_inning_score,third_inning_score,match_winer_player,update_time,match_process,tag,match_id,source,match_time) values (%s, %s, %s, %s, %s,%s, %s, %s, %s, %s, %s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s) on conflict(match_id) do update set update_time = %s,match_process =%s,home_player_score =%s,guest_player_score =%s;" cursor.execute(sql3, (int(league_id), team_home, team_guest,int(score_home), int(score_guest), 0, score_home,score_guest,1,'','','','','','',utime,jijie,number,game_id,'hg3535','00:00',utime,jijie,score_home,score_guest)) if odd_evens_dict: for key, value in odd_evens_dict.items(): if value: new_hash = hash_func(match_id=game_id, odds_code=key, sort=0, p_id=5) odds_only = r_func(match_id=game_id, odds_code=key, sort=0, p_id=5,odd=value) sql1 = "insert into st_wq_odds(lg_id, odds_code, match_id, ctime, utime, odds, p_id, p_code, sort, source, sole,condition,odds_only,is_rollball) values (%s, %s, %s, %s, %s, %s,%s, %s, %s, %s, %s, %s,%s,%s) on conflict(sole) do update set utime = %s,odds = %s,odds_only=%s;" cursor.execute(sql1, ( int(league_id), key, int(game_id), utime, utime,value, 5, "two_sides", 0, "hg3535", new_hash, odd_evens_dict_rule[key],odds_only,1,utime, value,odds_only)) # 更新主队st_zq_odds_record表 sql2 = "insert into st_wq_odds_record(lg_id, odds_code, match_id, ctime, utime, odds, p_id, p_code, sort, source, condition,odds_only,is_rollball) values (%s, %s, %s, %s, %s, %s,%s, %s, %s, %s, %s,%s,%s);" cursor.execute(sql2, ( int(league_id), key, int(game_id), utime, utime, value, 5, "two_sides", 0, "hg3535",odd_evens_dict_rule[key],odds_only,1)) sql3 = "insert into st_wq_result(lg_id, home_player_name, guest_player_name, home_player_let_plate, guest_player_let_plate, all_inning,home_player_score,guest_player_score,status,first_score_player,last_score_player,first_inning_score,second_inning_score,third_inning_score,match_winer_player,update_time,match_process,tag,match_id,source,match_time) values (%s, %s, %s, %s, %s,%s, %s, %s, %s, %s, %s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s) on conflict(match_id) do update set update_time = %s,match_process =%s,home_player_score =%s,guest_player_score =%s;" cursor.execute(sql3, (int(league_id), team_home, team_guest,int(score_home), int(score_guest), 0, score_home,score_guest,1,'','','','','','',utime,jijie,number,game_id,'hg3535','00:00',utime,jijie,score_home,score_guest)) # 插入赛事表 Competition_sql = "insert into st_wq_competition(home_team, guest_team, lg_id, match_id, match_date, match_time,ctime, utime, tag,type,source,is_rollball) values (%s, %s, %s, %s, %s, %s, %s, %s,%s, %s, %s,%s) on conflict(match_id) do update set tag = %s;" cursor.execute(Competition_sql, (team_home, team_guest, league_id, game_id, match_date, match_time, utime, utime, number, pt, "hg3535",1,number)) def close_spider(self, spider): self.dbpool.close() class Roll_Banqiupipeline(object): def open_spider(self, spider): self.dbpool = adbapi.ConnectionPool("psycopg2",host=settings["POST_HOST"], port=settings['POST_PORT'], user=settings["POST_USER"], password=settings["POST_PASSWORD"], database=settings["POST_DATABASE"]) self.pool = redis.ConnectionPool(host=settings["R_HOST"], port=settings["R_POST"], password=settings["R_PASSWORD"]) self.redis_db = redis.StrictRedis(connection_pool=self.pool) def process_item(self, item, spider): # 使用twisted将mysql插入变成异步执行 query = self.dbpool.runInteraction(self.do_insert, item) query.addErrback(self.handle_error, item, spider) # 处理异常 def handle_error(self, failure, item, spider): # 处理异步插入的异常 logger = logging.getLogger(__name__) logger.warning(failure) # print(failure) def do_insert(self, cursor, item): # 联赛id league_id = item['league_id'] # 联赛名 league_name = item['league_name'] # result = item['result'] # 比赛id game_id = item['game_id'] # 球队1 team_home = item['team_home'] # 球队2 team_guest = item['team_guest'] # 数量(97>) number = item['number'] #新增 match_score_dict match_score_dict = item['match_score_dict'] # 比赛状态 zhuangtai = item['zhuangtai'] # 日期 # data_game = item['data_game'] data_game = item['data_game'].split("/") month = str(data_game[1].strip()) day = str(data_game[0]) # 比赛时间 # time_game = str(item['time_game']) # 比赛时间，时间戳 # ctime = "2019" + "-" + month + "-" + day + "" + time_game + ":00".strip() r_ctime = "2019" + "-" + month + "-" + day # 现在时间，时间戳 utime = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime()) expire_time = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(time.time() + 60)) # 队1分数 score_home = item['score_home'] # 队2分数 score_guest = item['score_guest'] # 第几节 jijie = item['jijie'] # 球队得分 qiudui = item['qiudui'] pt = item['pt'] #滚球这个位置获取不到这些字段 # match_date, match_time = new_time(ctime) # 让球 concedes_dict = item['concedes_dict'] concedes_dict_rule = item['concedes_dict_rule'] # 独赢 capots_dict = item['capots_dict'] # 总得分大小 total_size_dict = item['total_size_dict'] total_size_dict_rule = item['total_size_dict_rule'] # 总得分单双 odd_evens_dict = item['odd_evens_dict'] odd_evens_dict_rule = item['odd_evens_dict_rule'] # 让球 concede = {'league_id': league_id, 'game_id': game_id, 'utime': utime, 'p_id': 1, 'p_code': "concede", 'source': "hg3535", 'expire_time': expire_time, 'sort': 0} # bqone_intodb(data1=concedes_dict, data2=concede, data3=concedes_dict_rule, cursor=cursor) # 总得分：大/小 total_size = {'league_id': league_id, 'game_id': game_id, 'utime': utime, 'p_id': 2, 'p_code': "total_size", 'source': "hg3535", 'expire_time': expire_time, 'sort': 0} # bqone_intodb(data1=total_size_dict, data2=total_size, data3=total_size_dict_rule, cursor=cursor) odd_even = {'league_id': league_id, 'game_id': game_id, 'utime': utime, 'p_id': 3, 'p_code': "two_sides", 'source': "hg3535", 'expire_time': expire_time, 'sort': 0} # bqone_intodb(data1=odd_evens_dict, data2=odd_even, data3=odd_evens_dict_rule, cursor=cursor) # 赛事失效时间 # n_time = out_time(ctime, 3.5) # 插入独赢 if capots_dict: for key, value in capots_dict.items(): if value: new_hash = hash_func(match_id=game_id, odds_code=key, sort=0, p_id=2) odds_only = r_func(match_id=game_id, odds_code=key, sort=0, p_id=2,odd=value) sql1 = "insert into st_bq_odds(lg_id, odds_code, match_id, ctime, utime, odds, p_id, p_code, sort, source, sole,odds_only,expire_time,is_rollball) values (%s, %s, %s, %s, %s, %s, %s,%s, %s, %s, %s, %s,%s,%s) on conflict(sole) do update set utime = %s,odds = %s,odds_only=%s,expire_time=%s;" cursor.execute(sql1, ( int(league_id), key, int(game_id), utime, utime, value, 4, "capot", 0, "hg3535", new_hash, odds_only, expire_time,1,utime, value, odds_only,expire_time)) # 更新主队st_zq_odds_record表 sql2 = "insert into st_bq_odds_record(lg_id, odds_code, match_id, ctime, utime,odds, p_id, p_code, sort, source,odds_only,is_rollball) values (%s, %s, %s, %s, %s, %s, %s,%s, %s, %s, %s,%s);" cursor.execute(sql2, ( int(league_id), key, int(game_id), utime, utime, value, 4, "capot", 0, "hg3535", odds_only,1)) res = json.dumps(match_score_dict) sql3 = "insert into st_bq_result(lg_id, home_team, guest_team, home_score, guest_score, all_goal, status,first_score, last_score,match_score,match_winer,update_time,match_time,match_process,tag,match_id,u_home_score,u_guest_score,source,match_score_t) values (%s,%s, %s, %s, %s, %s, %s, %s,%s, %s, %s, %s, %s, %s,%s,%s,%s,%s,%s,%s) on conflict(match_id) do update set update_time = %s,match_score = %s,match_time =%s,match_process =%s,home_score =%s,guest_score =%s;" cursor.execute(sql3, (int(league_id), team_home, team_guest,score_home, score_guest, number, 1,'','',qiudui,'',utime,r_ctime,jijie,number,int(game_id),0,0,'hg3535',res,utime,qiudui,r_ctime,jijie,score_home,score_guest)) # 插入赛事表 Competition_sql = "insert into st_bq_competition(home_team,guest_team,lg_id,status,match_id,ctime,utime,tag,source,type,is_rollball) values(%s, %s, %s, %s, %s, %s, %s,%s, %s,%s,%s) on conflict(match_id) do update set tag=%s,is_rollball=%s,utime=%s;" cursor.execute(Competition_sql, (team_home, team_guest, league_id, 1,game_id,utime, utime, number,"hg3535",1,1,number,1,utime)) #让球 if concedes_dict: for key, value in concedes_dict.items(): if value: new_hash = hash_func(match_id=game_id, odds_code=key, sort=0, p_id=1) odds_only = r_func(match_id=game_id, odds_code=key, sort=0, p_id=1,odd=value) sql1 = "insert into st_bq_odds(lg_id, odds_code, match_id, ctime, utime, odds, p_id, p_code, sort, source, sole,condition,odds_only,is_rollball) values (%s, %s, %s, %s, %s, %s,%s, %s, %s, %s, %s, %s,%s,%s) on conflict(sole) do update set utime = %s,odds = %s,odds_only=%s;" cursor.execute(sql1, ( int(league_id), key, int(game_id), utime, utime,value, 1, "concede", 0, "hg3535", new_hash, concedes_dict_rule[key],odds_only,1,utime, value,odds_only)) # 更新主队st_zq_odds_record表 sql2 = "insert into st_bq_odds_record(lg_id, odds_code, match_id, ctime, utime, odds, p_id, p_code, sort, source, condition,odds_only,is_rollball) values (%s, %s, %s, %s, %s, %s,%s, %s, %s, %s, %s,%s,%s);" cursor.execute(sql2, ( int(league_id), key, int(game_id), utime, utime, value, 1, "concede", 0, "hg3535", concedes_dict_rule[key],odds_only,1)) #插入结果表 res = json.dumps(match_score_dict) # r = pymysql.escape_string(res) sql3 = "insert into st_bq_result(lg_id, home_team, guest_team, home_score, guest_score, all_goal, status,first_score, last_score,match_score,match_winer,update_time,match_time,match_process,tag,match_id,u_home_score,u_guest_score,source,match_score_t) values (%s,%s,%s, %s, %s, %s, %s, %s, %s,%s, %s, %s, %s, %s, %s,%s,%s,%s,%s,%s) on conflict(match_id) do update set update_time = %s,match_score_t = %s,match_time =%s,match_process =%s,home_score =%s,guest_score =%s;" cursor.execute(sql3, (int(league_id), team_home, team_guest,score_home, score_guest, number, 1,'','','','',utime,r_ctime,jijie,number,int(game_id),0,0,'hg3535',res,utime,res,r_ctime,jijie,score_home,score_guest)) # 插入记录表 sql4 = "insert into st_bq_result_record(lg_id, home_team, guest_team, home_score, guest_score,first_score, last_score,match_score,match_winer,update_time,match_time,match_process,tag,match_id,source,status) values (%s, %s, %s, %s, %s, %s, %s,%s, %s, %s, %s, %s, %s,%s,%s,%s);" cursor.execute(sql4, (int(league_id), team_home, team_guest,score_home, score_guest, '', '','','',utime,r_ctime,jijie,number,int(game_id),'hg3535',0)) if total_size_dict: for key, value in total_size_dict.items(): if value: new_hash = hash_func(match_id=game_id, odds_code=key, sort=0, p_id=2) odds_only = r_func(match_id=game_id, odds_code=key, sort=0, p_id=2,odd=value) sql1 = "insert into st_bq_odds(lg_id, odds_code, match_id, ctime, utime, odds, p_id, p_code, sort, source, sole,condition,odds_only,is_rollball) values (%s, %s, %s, %s, %s, %s,%s, %s, %s, %s, %s, %s,%s,%s) on conflict(sole) do update set utime = %s,odds = %s,odds_only=%s;" cursor.execute(sql1, ( int(league_id), key, int(game_id), utime, utime,value, 2, "total_size", 0, "hg3535", new_hash, total_size_dict_rule[key],odds_only,1,utime, value,odds_only)) # 更新主队st_zq_odds_record表 sql2 = "insert into st_bq_odds_record(lg_id, odds_code, match_id, ctime, utime, odds, p_id, p_code, sort, source, condition,odds_only,is_rollball) values (%s, %s, %s, %s, %s, %s,%s, %s, %s, %s, %s,%s,%s);" cursor.execute(sql2, ( int(league_id), key, int(game_id), utime, utime, value, 2, "total_size", 0, "hg3535", total_size_dict_rule[key],odds_only,1)) #插入结果表 res = json.dumps(match_score_dict) # r = pymysql.escape_string(res) sql3 = "insert into st_bq_result(lg_id, home_team, guest_team, home_score, guest_score, all_goal, status,first_score, last_score,match_score,match_winer,update_time,match_time,match_process,tag,match_id,u_home_score,u_guest_score,source,match_score_t) values (%s,%s,%s, %s, %s, %s, %s, %s, %s,%s, %s, %s, %s, %s, %s,%s,%s,%s,%s,%s) on conflict(match_id) do update set update_time = %s,match_score_t = %s,match_time =%s,match_process =%s,home_score =%s,guest_score =%s;" cursor.execute(sql3, (int(league_id), team_home, team_guest,score_home, score_guest, number, 1,'','','','',utime,r_ctime,jijie,number,int(game_id),0,0,'hg3535',res,utime,res,r_ctime,jijie,score_home,score_guest)) # 插入记录表 sql4 = "insert into st_bq_result_record(lg_id, home_team, guest_team, home_score, guest_score,first_score, last_score,match_score,match_winer,update_time,match_time,match_process,tag,match_id,source,status) values (%s, %s, %s, %s, %s, %s, %s,%s, %s, %s, %s, %s, %s,%s,%s,%s);" cursor.execute(sql4, (int(league_id), team_home, team_guest,score_home, score_guest, '', '','','',utime,r_ctime,jijie,number,int(game_id),'hg3535',0)) if odd_evens_dict: for key, value in odd_evens_dict.items(): if value: new_hash = hash_func(match_id=game_id, odds_code=key, sort=0, p_id=3) odds_only = r_func(match_id=game_id, odds_code=key, sort=0, p_id=3,odd=value) sql1 = "insert into st_bq_odds(lg_id, odds_code, match_id, ctime, utime, odds, p_id, p_code, sort, source, sole,condition,odds_only,is_rollball) values (%s, %s, %s, %s, %s, %s,%s, %s, %s, %s, %s, %s,%s,%s) on conflict(sole) do update set utime = %s,odds = %s,odds_only=%s;" cursor.execute(sql1, ( int(league_id), key, int(game_id), utime, utime,value, 3, "two_sides", 0, "hg3535", new_hash, odd_evens_dict_rule[key],odds_only,1,utime, value,odds_only)) # 更新主队st_zq_odds_record表 sql2 = "insert into st_bq_odds_record(lg_id, odds_code, match_id, ctime, utime, odds, p_id, p_code, sort, source, condition,odds_only,is_rollball) values (%s, %s, %s, %s, %s, %s,%s, %s, %s, %s, %s,%s,%s);" cursor.execute(sql2, ( int(league_id), key, int(game_id), utime, utime, value, 3, "two_sides", 0, "hg3535", odd_evens_dict_rule[key],odds_only,1)) #插入结果表 res = json.dumps(match_score_dict) # r = pymysql.escape_string(res) sql3 = "insert into st_bq_result(lg_id, home_team, guest_team, home_score, guest_score, all_goal, status,first_score, last_score,match_score,match_winer,update_time,match_time,match_process,tag,match_id,u_home_score,u_guest_score,source,match_score_t) values (%s,%s,%s, %s, %s, %s, %s, %s, %s,%s, %s, %s, %s, %s, %s,%s,%s,%s,%s,%s) on conflict(match_id) do update set update_time = %s,match_score_t = %s,match_time =%s,match_process =%s,home_score =%s,guest_score =%s;" cursor.execute(sql3, (int(league_id), team_home, team_guest,score_home, score_guest, number, 1,'','','','',utime,r_ctime,jijie,number,int(game_id),0,0,'hg3535',res,utime,res,r_ctime,jijie,score_home,score_guest)) # 插入记录表 sql4 = "insert into st_bq_result_record(lg_id, home_team, guest_team, home_score, guest_score,first_score, last_score,match_score,match_winer,update_time,match_time,match_process,tag,match_id,source,status) values (%s, %s, %s, %s, %s, %s, %s,%s, %s, %s, %s, %s, %s,%s,%s,%s);" cursor.execute(sql4, (int(league_id), team_home, team_guest,score_home, score_guest, '', '','','',utime,r_ctime,jijie,number,int(game_id),'hg3535',0)) def close_spider(self, spider): # self.conn.close() self.dbpool.close()