|
|
@@ -3,44 +3,7 @@ import time
|
|
|
import hashlib
|
|
|
import redis
|
|
|
from twisted.enterprise import adbapi
|
|
|
-import psycopg2.extras
|
|
|
-
|
|
|
-# class MySnow:
|
|
|
-# def __init__(self,dataID):
|
|
|
-# self.start = int(time.mktime(time.strptime('2018-01-01 00:00:00', "%Y-%m-%d %H:%M:%S")))
|
|
|
-# self.last = int(time.time())
|
|
|
-# self.countID = 0
|
|
|
-# self.dataID = dataID # 数据ID，这个自定义或是映射
|
|
|
-#
|
|
|
-# def get_id(self):
|
|
|
-# # 时间差部分
|
|
|
-# now = int(time.time())
|
|
|
-# temp = now-self.start
|
|
|
-# if len(str(temp)) < 9: # 时间差不够9位的在前面补0
|
|
|
-# length = len(str(temp))
|
|
|
-# s = "0" * (9-length)
|
|
|
-# temp = s + str(temp)
|
|
|
-# if now == self.last:
|
|
|
-# self.countID += 1 # 同一时间差，序列号自增
|
|
|
-# else:
|
|
|
-# self.countID = 0 # 不同时间差，序列号重新置为0
|
|
|
-# self.last = now
|
|
|
-# # 标识ID部分
|
|
|
-# if len(str(self.dataID)) < 2:
|
|
|
-# length = len(str(self.dataID))
|
|
|
-# s = "0" * (2-length)
|
|
|
-# self.dataID = s + str(self.dataID)
|
|
|
-# # 自增序列号部分
|
|
|
-# if self.countID == 99999: # 序列号自增5位满了，睡眠一秒钟
|
|
|
-# time.sleep(1)
|
|
|
-# countIDdata = str(self.countID)
|
|
|
-# if len(countIDdata) < 5: # 序列号不够5位的在前面补0
|
|
|
-# length = len(countIDdata)
|
|
|
-# s = "0"*(5-length)
|
|
|
-# countIDdata = s + countIDdata
|
|
|
-# id = str(temp) + str(self.dataID) + countIDdata
|
|
|
-# return id
|
|
|
-
|
|
|
+from scrapy.conf import settings
|
|
|
|
|
|
# 生成哈希索引 sole
|
|
|
from .items import Zuqiustatus
|
|
|
@@ -433,26 +396,10 @@ def get_pcode(corner_ball, code):
|
|
|
|
|
|
# 足球 赔率 赛事---------------------------------------------------------------------------------------------------------
|
|
|
class Zuqiupipeline(object):
|
|
|
- def __init__(self, dbpool, conn, cursor, redis_db):
|
|
|
- self.dbpool = dbpool
|
|
|
- self.conn = conn
|
|
|
- self.cursor = cursor
|
|
|
- self.redis_db = redis_db
|
|
|
-
|
|
|
- @classmethod
|
|
|
- def from_settings(cls, settings):
|
|
|
- dbparms = dict(
|
|
|
- host=settings["POST_HOST"],
|
|
|
- user=settings["POST_USER"],
|
|
|
- password=settings["POST_PASSWORD"],
|
|
|
- port=settings['POST_PORT'],
|
|
|
- )
|
|
|
- dbpool = adbapi.ConnectionPool("psycopg2", **dbparms)
|
|
|
- conn = psycopg2.connect(host=settings["POST_HOST"], port=settings['POST_PORT'], user=settings["POST_USER"], password=settings["POST_PASSWORD"], database=settings["POST_DATABASE"])
|
|
|
- cursor = conn.cursor(cursor_factory=psycopg2.extras.DictCursor)
|
|
|
- pool = redis.ConnectionPool(host=settings["R_HOST"], port=settings["R_POST"], password=settings["R_PASSWORD"])
|
|
|
- redis_db = redis.StrictRedis(connection_pool=pool)
|
|
|
- return cls(dbpool, conn, cursor, redis_db)
|
|
|
+ def open_spider(self, spider):
|
|
|
+ self.dbpool = adbapi.ConnectionPool("psycopg2",host=settings["POST_HOST"], port=settings['POST_PORT'], user=settings["POST_USER"], password=settings["POST_PASSWORD"], database=settings["POST_DATABASE"])
|
|
|
+ self.pool = redis.ConnectionPool(host=settings["R_HOST"], port=settings["R_POST"], password=settings["R_PASSWORD"])
|
|
|
+ self.redis_db = redis.StrictRedis(connection_pool=self.pool)
|
|
|
|
|
|
def process_item(self, item, spider):
|
|
|
# 使用twisted将mysql插入变成异步执行
|
|
|
@@ -497,6 +444,7 @@ class Zuqiupipeline(object):
|
|
|
half_size_guest_rule = item["half_size_guest_rule"]
|
|
|
half_size_home = item["half_size_home"]
|
|
|
half_size_home_rule = item["half_size_home_rule"]
|
|
|
+ zuqiu = item['zuqiu']
|
|
|
|
|
|
size_data = {'league_id': league_id, 'game_id': competition_id, 'utime': utime, 'p_id': p_id, 'p_code': p_code,
|
|
|
'source': "hg3535", 'expire_time': expire_time, 'pt': pt}
|
|
|
@@ -689,52 +637,36 @@ class Zuqiupipeline(object):
|
|
|
match_date, match_time, time3 = new_time(ctime)
|
|
|
n_time = out_time(time3, 1.5)
|
|
|
# 插入st_zq_competition表
|
|
|
- if pt == 1:
|
|
|
- data_competition = {'team_home': team_home, 'team_guest': team_guest, 'league_id': league_id,
|
|
|
- 'game_id': competition_id,
|
|
|
- 'match_date': match_date, 'match_time': match_time, 'utime': utime, 'number': number,
|
|
|
- 'source': "hg3535", "expire_time": n_time, 'is_today': 1, "us_time": ctime}
|
|
|
- zqone_competition(data=data_competition, cursor=cursor)
|
|
|
- if pt == 2:
|
|
|
- data_competition = {'team_home': team_home, 'team_guest': team_guest, 'league_id': league_id,
|
|
|
- 'game_id': competition_id,
|
|
|
- 'match_date': match_date, 'match_time': match_time, 'utime': utime, 'number': number,
|
|
|
- 'source': "hg3535", "expire_time": n_time, 'is_morningplate': 1, "us_time": ctime}
|
|
|
- zqtwo_competition(data=data_competition, cursor=cursor)
|
|
|
- if pt == 3:
|
|
|
- data_competition = {'team_home': team_home, 'team_guest': team_guest, 'league_id': league_id,
|
|
|
- 'game_id': competition_id,
|
|
|
- 'match_date': match_date, 'match_time': match_time, 'utime': utime, 'number': number,
|
|
|
- 'source': "hg3535", "expire_time": n_time, "is_stringscene": 1, "us_time": ctime}
|
|
|
- zqthree_competition(data=data_competition, cursor=cursor)
|
|
|
+ if zuqiu == "足球":
|
|
|
+ if pt == 1:
|
|
|
+ data_competition = {'team_home': team_home, 'team_guest': team_guest, 'league_id': league_id,
|
|
|
+ 'game_id': competition_id,
|
|
|
+ 'match_date': match_date, 'match_time': match_time, 'utime': utime, 'number': number,
|
|
|
+ 'source': "hg3535", "expire_time": n_time, 'is_today': 1, "us_time": ctime}
|
|
|
+ zqone_competition(data=data_competition, cursor=cursor)
|
|
|
+ if pt == 2:
|
|
|
+ data_competition = {'team_home': team_home, 'team_guest': team_guest, 'league_id': league_id,
|
|
|
+ 'game_id': competition_id,
|
|
|
+ 'match_date': match_date, 'match_time': match_time, 'utime': utime, 'number': number,
|
|
|
+ 'source': "hg3535", "expire_time": n_time, 'is_morningplate': 1, "us_time": ctime}
|
|
|
+ zqtwo_competition(data=data_competition, cursor=cursor)
|
|
|
+ if pt == 3:
|
|
|
+ data_competition = {'team_home': team_home, 'team_guest': team_guest, 'league_id': league_id,
|
|
|
+ 'game_id': competition_id,
|
|
|
+ 'match_date': match_date, 'match_time': match_time, 'utime': utime, 'number': number,
|
|
|
+ 'source': "hg3535", "expire_time": n_time, "is_stringscene": 1, "us_time": ctime}
|
|
|
+ zqthree_competition(data=data_competition, cursor=cursor)
|
|
|
|
|
|
def close_spider(self, spider):
|
|
|
- self.conn.close()
|
|
|
self.dbpool.close()
|
|
|
|
|
|
|
|
|
# 篮球 让球大小----------------------------------------------------------------------------------------------------------
|
|
|
class Lanqiupipeline(object):
|
|
|
- def __init__(self, dbpool, conn, cursor, redis_db):
|
|
|
- self.dbpool = dbpool
|
|
|
- self.conn = conn
|
|
|
- self.cursor = cursor
|
|
|
- self.redis_db = redis_db
|
|
|
-
|
|
|
- @classmethod
|
|
|
- def from_settings(cls, settings):
|
|
|
- dbparms = dict(
|
|
|
- host=settings["POST_HOST"],
|
|
|
- user=settings["POST_USER"],
|
|
|
- password=settings["POST_PASSWORD"],
|
|
|
- port=settings['POST_PORT'],
|
|
|
- )
|
|
|
- dbpool = adbapi.ConnectionPool("psycopg2", **dbparms)
|
|
|
- conn = psycopg2.connect(host=settings["POST_HOST"], port=settings['POST_PORT'], user=settings["POST_USER"], password=settings["POST_PASSWORD"], database=settings["POST_DATABASE"])
|
|
|
- cursor = conn.cursor(cursor_factory=psycopg2.extras.DictCursor)
|
|
|
- pool = redis.ConnectionPool(host=settings["R_HOST"], port=settings["R_POST"], password=settings["R_PASSWORD"])
|
|
|
- redis_db = redis.StrictRedis(connection_pool=pool)
|
|
|
- return cls(dbpool, conn, cursor, redis_db)
|
|
|
+ def open_spider(self, spider):
|
|
|
+ self.dbpool = adbapi.ConnectionPool("psycopg2",host=settings["POST_HOST"], port=settings['POST_PORT'], user=settings["POST_USER"], password=settings["POST_PASSWORD"], database=settings["POST_DATABASE"])
|
|
|
+ self.pool = redis.ConnectionPool(host=settings["R_HOST"], port=settings["R_POST"], password=settings["R_PASSWORD"])
|
|
|
+ self.redis_db = redis.StrictRedis(connection_pool=self.pool)
|
|
|
|
|
|
def process_item(self, item, spider):
|
|
|
# 使用twisted将mysql插入变成异步执行
|
|
|
@@ -795,6 +727,7 @@ class Lanqiupipeline(object):
|
|
|
team_scores_dict = item['team_score']
|
|
|
team_scores_dict_rule = item['team_score_rule']
|
|
|
# 让球
|
|
|
+ lanqiu = item['lanqiu']
|
|
|
concede = {'league_id': league_id, 'game_id': game_id, 'utime': utime, 'p_id': 1, 'p_code': "concede",
|
|
|
'source': "hg3535", 'expire_time': expire_time}
|
|
|
# 让球 数据插入数据库
|
|
|
@@ -833,50 +766,33 @@ class Lanqiupipeline(object):
|
|
|
n_time = out_time(time3, 1.5)
|
|
|
us_time = ctime
|
|
|
# 插入赛事表
|
|
|
-
|
|
|
- if pt == 1:
|
|
|
- Competition_sql = "insert into st_lq_competition(home_team, guest_team, lg_id, match_id, match_date, match_time,ctime, utime, tag,source,expire_time,is_today,us_time) values (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s,%s) on conflict(match_id) do update set tag = %s,expire_time=%s,is_today=%s;"
|
|
|
- cursor.execute(Competition_sql, (
|
|
|
- team_home, team_guest, league_id, game_id, match_date, match_time, utime, utime, number, "hg3535",
|
|
|
- n_time,1, us_time, number, n_time, 1))
|
|
|
- if pt == 2:
|
|
|
- Competition_sql = "insert into st_lq_competition(home_team, guest_team, lg_id, match_id, match_date, match_time,ctime, utime, tag,source,expire_time,is_morningplate,us_time) values (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s,%s) on conflict(match_id) do update set tag = %s,expire_time=%s,is_morningplate=%s;"
|
|
|
- cursor.execute(Competition_sql, (
|
|
|
- team_home, team_guest, league_id, game_id, match_date, match_time, utime, utime, number, "hg3535",
|
|
|
- n_time,1, us_time, number, n_time, 1))
|
|
|
- if pt == 3:
|
|
|
- Competition_sql = "insert into st_lq_competition(home_team, guest_team, lg_id, match_id, match_date, match_time,ctime, utime, tag,source,expire_time,is_stringscene,us_time) values (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s,%s) on conflict(match_id) do update set tag = %s,expire_time=%s,is_stringscene=%s;"
|
|
|
- cursor.execute(Competition_sql, (
|
|
|
- team_home, team_guest, league_id, game_id, match_date, match_time, utime, utime, number, "hg3535",
|
|
|
- n_time,1, us_time, number, n_time, 1))
|
|
|
+ if lanqiu == "篮球":
|
|
|
+ if pt == 1:
|
|
|
+ Competition_sql = "insert into st_lq_competition(home_team, guest_team, lg_id, match_id, match_date, match_time,ctime, utime, tag,source,expire_time,is_today,us_time) values (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s,%s) on conflict(match_id) do update set tag = %s,expire_time=%s,is_today=%s;"
|
|
|
+ cursor.execute(Competition_sql, (
|
|
|
+ team_home, team_guest, league_id, game_id, match_date, match_time, utime, utime, number, "hg3535",
|
|
|
+ n_time,1, us_time, number, n_time, 1))
|
|
|
+ if pt == 2:
|
|
|
+ Competition_sql = "insert into st_lq_competition(home_team, guest_team, lg_id, match_id, match_date, match_time,ctime, utime, tag,source,expire_time,is_morningplate,us_time) values (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s,%s) on conflict(match_id) do update set tag = %s,expire_time=%s,is_morningplate=%s;"
|
|
|
+ cursor.execute(Competition_sql, (
|
|
|
+ team_home, team_guest, league_id, game_id, match_date, match_time, utime, utime, number, "hg3535",
|
|
|
+ n_time,1, us_time, number, n_time, 1))
|
|
|
+ if pt == 3:
|
|
|
+ Competition_sql = "insert into st_lq_competition(home_team, guest_team, lg_id, match_id, match_date, match_time,ctime, utime, tag,source,expire_time,is_stringscene,us_time) values (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s,%s) on conflict(match_id) do update set tag = %s,expire_time=%s,is_stringscene=%s;"
|
|
|
+ cursor.execute(Competition_sql, (
|
|
|
+ team_home, team_guest, league_id, game_id, match_date, match_time, utime, utime, number, "hg3535",
|
|
|
+ n_time,1, us_time, number, n_time, 1))
|
|
|
|
|
|
def close_spider(self, spider):
|
|
|
- self.conn.close()
|
|
|
self.dbpool.close()
|
|
|
|
|
|
|
|
|
# 网球 让球&大小盘--------------------------------------------------------------------------------------------------------
|
|
|
class Wangqiupipeline(object):
|
|
|
- def __init__(self, dbpool, conn, cursor, redis_db):
|
|
|
- self.dbpool = dbpool
|
|
|
- self.conn = conn
|
|
|
- self.cursor = cursor
|
|
|
- self.redis_db = redis_db
|
|
|
-
|
|
|
- @classmethod
|
|
|
- def from_settings(cls, settings):
|
|
|
- dbparms = dict(
|
|
|
- host=settings["POST_HOST"],
|
|
|
- user=settings["POST_USER"],
|
|
|
- password=settings["POST_PASSWORD"],
|
|
|
- port=settings['POST_PORT'],
|
|
|
- )
|
|
|
- dbpool = adbapi.ConnectionPool("psycopg2", **dbparms)
|
|
|
- conn = psycopg2.connect(host=settings["POST_HOST"], port=settings['POST_PORT'], user=settings["POST_USER"], password=settings["POST_PASSWORD"], database=settings["POST_DATABASE"])
|
|
|
- cursor = conn.cursor(cursor_factory=psycopg2.extras.DictCursor)
|
|
|
- pool = redis.ConnectionPool(host=settings["R_HOST"], port=settings["R_POST"], password=settings["R_PASSWORD"])
|
|
|
- redis_db = redis.StrictRedis(connection_pool=pool)
|
|
|
- return cls(dbpool, conn, cursor, redis_db)
|
|
|
+ def open_spider(self, spider):
|
|
|
+ self.dbpool = adbapi.ConnectionPool("psycopg2",host=settings["POST_HOST"], port=settings['POST_PORT'], user=settings["POST_USER"], password=settings["POST_PASSWORD"], database=settings["POST_DATABASE"])
|
|
|
+ self.pool = redis.ConnectionPool(host=settings["R_HOST"], port=settings["R_POST"], password=settings["R_PASSWORD"])
|
|
|
+ self.redis_db = redis.StrictRedis(connection_pool=self.pool)
|
|
|
|
|
|
def process_item(self, item, spider):
|
|
|
# 使用twisted将mysql插入变成异步执行
|
|
|
@@ -940,13 +856,14 @@ class Wangqiupipeline(object):
|
|
|
# 总局数单双
|
|
|
odd_evens_dict = item['odd_evens_dict']
|
|
|
odd_evens_dict_rule = item['odd_evens_dict_rule']
|
|
|
- #
|
|
|
- # 让球
|
|
|
- concede = {'league_id': league_id, 'game_id': game_id, 'utime': utime, 'p_id': 1, 'p_code': "concede",
|
|
|
+ wangqiu = item['wangqiu']
|
|
|
+
|
|
|
+ # 让盘
|
|
|
+ concede = {'league_id': league_id, 'game_id': game_id, 'utime': utime, 'p_id': 1, 'p_code': "dishes",
|
|
|
'source': "hg3535", 'expire_time': expire_time, 'sort': 0, 'pt': pt}
|
|
|
wqone_intodb(data1=concedes_dict, data2=concede, data3=concedes_dict_rule, cursor=cursor, redis_db=self.redis_db)
|
|
|
-
|
|
|
- bureaus = {'league_id': league_id, 'game_id': game_id, 'utime': utime, 'p_id': 3, 'p_code': "bureau",
|
|
|
+ # 让局
|
|
|
+ bureaus = {'league_id': league_id, 'game_id': game_id, 'utime': utime, 'p_id': 3, 'p_code': "concede",
|
|
|
'source': "hg3535", 'expire_time': expire_time, 'sort': 0, 'pt': pt}
|
|
|
wqone_intodb(data1=bureaus_dict, data2=bureaus, data3=bureaus_dict_rule, cursor=cursor, redis_db=self.redis_db)
|
|
|
|
|
|
@@ -979,51 +896,33 @@ class Wangqiupipeline(object):
|
|
|
int(league_id), key, int(game_id), utime, utime, value, 2, "kemp", 0, "hg3535", odds_only))
|
|
|
us_time = ctime
|
|
|
# 插入赛事表
|
|
|
- if pt is 1:
|
|
|
- Competition_sql = "insert into st_wq_competition(home_team, guest_team, lg_id, match_id, match_date, match_time,ctime, utime, tag,source,expire_time,is_today,us_time) values (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s,%s) on conflict(match_id) do update set tag = %s,expire_time=%s,is_today=%s;"
|
|
|
- cursor.execute(Competition_sql, (
|
|
|
- team_home, team_guest, league_id, game_id, match_date, match_time, utime, utime, number, "hg3535",
|
|
|
- n_time,1, us_time, number, n_time, 1))
|
|
|
- if pt is 2:
|
|
|
- Competition_sql = "insert into st_wq_competition(home_team, guest_team, lg_id, match_id, match_date, match_time,ctime, utime, tag,source,expire_time,is_morningplate,us_time) values (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s,%s) on conflict(match_id) do update set tag = %s,expire_time=%s,is_morningplate=%s;"
|
|
|
- cursor.execute(Competition_sql, (
|
|
|
- team_home, team_guest, league_id, game_id, match_date, match_time, utime, utime, number, "hg3535",
|
|
|
- n_time,1, us_time, number, n_time, 1))
|
|
|
- if pt is 3:
|
|
|
- Competition_sql = "insert into st_wq_competition(home_team, guest_team, lg_id, match_id, match_date, match_time,ctime, utime, tag,source,expire_time,is_stringscene,us_time) values (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s,%s) on conflict(match_id) do update set tag = %s,expire_time=%s,is_stringscene=%s;"
|
|
|
- cursor.execute(Competition_sql, (
|
|
|
- team_home, team_guest, league_id, game_id, match_date, match_time, utime, utime, number, "hg3535",
|
|
|
- n_time,1, us_time, number, n_time, 1))
|
|
|
+ if wangqiu == "网球":
|
|
|
+ if pt is 1:
|
|
|
+ Competition_sql = "insert into st_wq_competition(home_team, guest_team, lg_id, match_id, match_date, match_time,ctime, utime, tag,source,expire_time,is_today,us_time) values (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s,%s) on conflict(match_id) do update set tag = %s,expire_time=%s,is_today=%s;"
|
|
|
+ cursor.execute(Competition_sql, (
|
|
|
+ team_home, team_guest, league_id, game_id, match_date, match_time, utime, utime, number, "hg3535",
|
|
|
+ n_time,1, us_time, number, n_time, 1))
|
|
|
+ if pt is 2:
|
|
|
+ Competition_sql = "insert into st_wq_competition(home_team, guest_team, lg_id, match_id, match_date, match_time,ctime, utime, tag,source,expire_time,is_morningplate,us_time) values (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s,%s) on conflict(match_id) do update set tag = %s,expire_time=%s,is_morningplate=%s;"
|
|
|
+ cursor.execute(Competition_sql, (
|
|
|
+ team_home, team_guest, league_id, game_id, match_date, match_time, utime, utime, number, "hg3535",
|
|
|
+ n_time,1, us_time, number, n_time, 1))
|
|
|
+ if pt is 3:
|
|
|
+ Competition_sql = "insert into st_wq_competition(home_team, guest_team, lg_id, match_id, match_date, match_time,ctime, utime, tag,source,expire_time,is_stringscene,us_time) values (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s,%s) on conflict(match_id) do update set tag = %s,expire_time=%s,is_stringscene=%s;"
|
|
|
+ cursor.execute(Competition_sql, (
|
|
|
+ team_home, team_guest, league_id, game_id, match_date, match_time, utime, utime, number, "hg3535",
|
|
|
+ n_time,1, us_time, number, n_time, 1))
|
|
|
|
|
|
def close_spider(self, spider):
|
|
|
- self.conn.close()
|
|
|
self.dbpool.close()
|
|
|
|
|
|
|
|
|
# 网球 波胆--------------------------------------------------------------------------------------------------------------
|
|
|
class Wqbodanpipeline(object):
|
|
|
- def __init__(self, dbpool, conn, cursor, redis_db):
|
|
|
- self.dbpool = dbpool
|
|
|
- self.conn = conn
|
|
|
- self.cursor = cursor
|
|
|
- self.redis_db = redis_db
|
|
|
-
|
|
|
- @classmethod
|
|
|
- def from_settings(cls, settings):
|
|
|
- dbparms = dict(
|
|
|
- host=settings["POST_HOST"],
|
|
|
- # db = settings["POST_DATABASE"],
|
|
|
- user=settings["POST_USER"],
|
|
|
- password=settings["POST_PASSWORD"],
|
|
|
- port=settings['POST_PORT'],
|
|
|
- )
|
|
|
- dbpool = adbapi.ConnectionPool("psycopg2", **dbparms)
|
|
|
- conn = psycopg2.connect(host=settings["POST_HOST"], port=settings['POST_PORT'], user=settings["POST_USER"], password=settings["POST_PASSWORD"], database=settings["POST_DATABASE"])
|
|
|
- cursor = conn.cursor(cursor_factory=psycopg2.extras.DictCursor)
|
|
|
- pool = redis.ConnectionPool(host=settings["R_HOST"], port=settings["R_POST"], password=settings["R_PASSWORD"])
|
|
|
- redis_db = redis.StrictRedis(connection_pool=pool)
|
|
|
- return cls(dbpool, conn, cursor, redis_db)
|
|
|
-
|
|
|
+ def open_spider(self, spider):
|
|
|
+ self.dbpool = adbapi.ConnectionPool("psycopg2",host=settings["POST_HOST"], port=settings['POST_PORT'], user=settings["POST_USER"], password=settings["POST_PASSWORD"], database=settings["POST_DATABASE"])
|
|
|
+ self.pool = redis.ConnectionPool(host=settings["R_HOST"], port=settings["R_POST"], password=settings["R_PASSWORD"])
|
|
|
+ self.redis_db = redis.StrictRedis(connection_pool=self.pool)
|
|
|
def process_item(self, item, spider):
|
|
|
# 使用twisted将mysql插入变成异步执行
|
|
|
query = self.dbpool.runInteraction(self.do_insert, item)
|
|
|
@@ -1100,32 +999,15 @@ class Wqbodanpipeline(object):
|
|
|
cursor.execute(sql2, (
|
|
|
int(league_id), key, int(competition_id), utime, utime, value, p_id, p_code, 0, "hg3535", key,odds_only))
|
|
|
def close_spider(self, spider):
|
|
|
- self.conn.close()
|
|
|
self.dbpool.close()
|
|
|
|
|
|
|
|
|
# 棒球 让球&大小盘--------------------------------------------------------------------------------------------------------
|
|
|
class Bangqiupipeline(object):
|
|
|
- def __init__(self, dbpool, conn, cursor, redis_db):
|
|
|
- self.dbpool = dbpool
|
|
|
- self.conn = conn
|
|
|
- self.cursor = cursor
|
|
|
- self.redis_db = redis_db
|
|
|
-
|
|
|
- @classmethod
|
|
|
- def from_settings(cls, settings):
|
|
|
- dbparms = dict(
|
|
|
- host=settings["POST_HOST"],
|
|
|
- user=settings["POST_USER"],
|
|
|
- password=settings["POST_PASSWORD"],
|
|
|
- port=settings['POST_PORT'],
|
|
|
- )
|
|
|
- dbpool = adbapi.ConnectionPool("psycopg2", **dbparms)
|
|
|
- conn = psycopg2.connect(host=settings["POST_HOST"], port=settings['POST_PORT'], user=settings["POST_USER"], password=settings["POST_PASSWORD"], database=settings["POST_DATABASE"])
|
|
|
- cursor = conn.cursor(cursor_factory=psycopg2.extras.DictCursor)
|
|
|
- pool = redis.ConnectionPool(host=settings["R_HOST"], port=settings["R_POST"], password=settings["R_PASSWORD"])
|
|
|
- redis_db = redis.StrictRedis(connection_pool=pool)
|
|
|
- return cls(dbpool, conn, cursor, redis_db)
|
|
|
+ def open_spider(self, spider):
|
|
|
+ self.dbpool = adbapi.ConnectionPool("psycopg2",host=settings["POST_HOST"], port=settings['POST_PORT'], user=settings["POST_USER"], password=settings["POST_PASSWORD"], database=settings["POST_DATABASE"])
|
|
|
+ self.pool = redis.ConnectionPool(host=settings["R_HOST"], port=settings["R_POST"], password=settings["R_PASSWORD"])
|
|
|
+ self.redis_db = redis.StrictRedis(connection_pool=self.pool)
|
|
|
|
|
|
def process_item(self, item, spider):
|
|
|
# 使用twisted将mysql插入变成异步执行
|
|
|
@@ -1187,6 +1069,7 @@ class Bangqiupipeline(object):
|
|
|
# 总得分单双
|
|
|
odd_evens_dict = item['odd_evens_dict']
|
|
|
odd_evens_dict_rule = item['odd_evens_dict_rule']
|
|
|
+ bangqiu = item['bangqiu']
|
|
|
|
|
|
# 让球
|
|
|
concede = {'league_id': league_id, 'game_id': game_id, 'utime': utime, 'p_id': 1, 'p_code': "concede",
|
|
|
@@ -1224,50 +1107,33 @@ class Bangqiupipeline(object):
|
|
|
int(league_id), key, int(game_id), utime, utime, value, 4, "capot", 0, "hg3535", odds_only))
|
|
|
us_time = ctime
|
|
|
# 插入赛事表
|
|
|
- if pt == 1:
|
|
|
- Competition_sql = "insert into st_bq_competition(home_team, guest_team, lg_id, match_id, match_date, match_time,ctime, utime, tag,source,expire_time,is_today,us_time) values (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s,%s) on conflict(match_id) do update set tag = %s,expire_time=%s,is_today=%s;"
|
|
|
- cursor.execute(Competition_sql, (
|
|
|
- team_home, team_guest, league_id, game_id, match_date, match_time, utime, utime, number, "hg3535",
|
|
|
- n_time, 1, us_time, number, n_time, 1))
|
|
|
- if pt == 2:
|
|
|
- Competition_sql = "insert into st_bq_competition(home_team, guest_team, lg_id, match_id, match_date, match_time,ctime, utime, tag,source,expire_time,is_morningplate,us_time) values (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s,%s) on conflict(match_id) do update set tag = %s,expire_time=%s,is_morningplate=%s;"
|
|
|
- cursor.execute(Competition_sql, (
|
|
|
- team_home, team_guest, league_id, game_id, match_date, match_time, utime, utime, number, "hg3535",
|
|
|
- n_time,1, us_time, number, n_time, 1))
|
|
|
- if pt == 3:
|
|
|
- Competition_sql = "insert into st_bq_competition(home_team, guest_team, lg_id, match_id, match_date, match_time,ctime, utime, tag,source,expire_time,is_stringscene,us_time) values (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s,%s) on conflict(match_id) do update set tag = %s,expire_time=%s,is_stringscene=%s;"
|
|
|
- cursor.execute(Competition_sql, (
|
|
|
- team_home, team_guest, league_id, game_id, match_date, match_time, utime, utime, number, "hg3535",
|
|
|
- n_time,1, us_time, number, n_time, 1))
|
|
|
-
|
|
|
+ if bangqiu == '棒球':
|
|
|
+ if pt == 1:
|
|
|
+ Competition_sql = "insert into st_bq_competition(home_team, guest_team, lg_id, match_id, match_date, match_time,ctime, utime, tag,source,expire_time,is_today,us_time) values (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s,%s) on conflict(match_id) do update set tag = %s,expire_time=%s,is_today=%s;"
|
|
|
+ cursor.execute(Competition_sql, (
|
|
|
+ team_home, team_guest, league_id, game_id, match_date, match_time, utime, utime, number, "hg3535",
|
|
|
+ n_time, 1, us_time, number, n_time, 1))
|
|
|
+ if pt == 2:
|
|
|
+ Competition_sql = "insert into st_bq_competition(home_team, guest_team, lg_id, match_id, match_date, match_time,ctime, utime, tag,source,expire_time,is_morningplate,us_time) values (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s,%s) on conflict(match_id) do update set tag = %s,expire_time=%s,is_morningplate=%s;"
|
|
|
+ cursor.execute(Competition_sql, (
|
|
|
+ team_home, team_guest, league_id, game_id, match_date, match_time, utime, utime, number, "hg3535",
|
|
|
+ n_time,1, us_time, number, n_time, 1))
|
|
|
+ if pt == 3:
|
|
|
+ Competition_sql = "insert into st_bq_competition(home_team, guest_team, lg_id, match_id, match_date, match_time,ctime, utime, tag,source,expire_time,is_stringscene,us_time) values (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s,%s) on conflict(match_id) do update set tag = %s,expire_time=%s,is_stringscene=%s;"
|
|
|
+ cursor.execute(Competition_sql, (
|
|
|
+ team_home, team_guest, league_id, game_id, match_date, match_time, utime, utime, number, "hg3535",
|
|
|
+ n_time,1, us_time, number, n_time, 1))
|
|
|
|
|
|
def close_spider(self, spider):
|
|
|
- self.conn.close()
|
|
|
self.dbpool.close()
|
|
|
|
|
|
|
|
|
# 足球 篮球 网球 棒球 冠军------------------------------------------------------------------------------------------------
|
|
|
class Guanjunpipeline(object):
|
|
|
- def __init__(self, dbpool, conn, cursor, redis_db):
|
|
|
- self.dbpool = dbpool
|
|
|
- self.conn = conn
|
|
|
- self.cursor = cursor
|
|
|
- self.redis_db = redis_db
|
|
|
-
|
|
|
- @classmethod
|
|
|
- def from_settings(cls, settings):
|
|
|
- dbparms = dict(
|
|
|
- host=settings["POST_HOST"],
|
|
|
- user=settings["POST_USER"],
|
|
|
- password=settings["POST_PASSWORD"],
|
|
|
- port=settings['POST_PORT'],
|
|
|
- )
|
|
|
- dbpool = adbapi.ConnectionPool("psycopg2", **dbparms)
|
|
|
- conn = psycopg2.connect(host=settings["POST_HOST"], port=settings['POST_PORT'], user=settings["POST_USER"], password=settings["POST_PASSWORD"], database=settings["POST_DATABASE"])
|
|
|
- cursor = conn.cursor(cursor_factory=psycopg2.extras.DictCursor)
|
|
|
- pool = redis.ConnectionPool(host=settings["R_HOST"], port=settings["R_POST"], password=settings["R_PASSWORD"])
|
|
|
- redis_db = redis.StrictRedis(connection_pool=pool)
|
|
|
- return cls(dbpool, conn, cursor, redis_db)
|
|
|
+ def open_spider(self, spider):
|
|
|
+ self.dbpool = adbapi.ConnectionPool("psycopg2",host=settings["POST_HOST"], port=settings['POST_PORT'], user=settings["POST_USER"], password=settings["POST_PASSWORD"], database=settings["POST_DATABASE"])
|
|
|
+ self.pool = redis.ConnectionPool(host=settings["R_HOST"], port=settings["R_POST"], password=settings["R_PASSWORD"])
|
|
|
+ self.redis_db = redis.StrictRedis(connection_pool=self.pool)
|
|
|
|
|
|
def process_item(self, item, spider):
|
|
|
# 使用twisted将mysql插入变成异步执行
|
|
|
@@ -1309,6 +1175,8 @@ class Guanjunpipeline(object):
|
|
|
r_hash = hash_func(league_id, new_champion, tema_home, champion_team)
|
|
|
odds_only = r_hash
|
|
|
s_hash = hash_func(league_id, league_name, ball, 0)
|
|
|
+ # if game_id == '8947612':
|
|
|
+ # print(ball, league_name)
|
|
|
if ball == "足球":
|
|
|
n_gameid = int('1' + str(league_id))
|
|
|
data_dict = {'league_id': league_id, 'new_champion': new_champion, 'utime': utime,
|
|
|
@@ -1354,32 +1222,16 @@ class Guanjunpipeline(object):
|
|
|
cursor.execute(Competition_sql, (league_name, league_id, n_gameid, utime, utime, 'hg3535', data_game, utime))
|
|
|
|
|
|
def close_spider(self, spider):
|
|
|
- self.conn.close()
|
|
|
+ # self.conn.close()
|
|
|
self.dbpool.close()
|
|
|
|
|
|
|
|
|
# 足球 篮球 网球 棒球 联赛------------------------------------------------------------------------------------------------
|
|
|
class Liansaipipeline(object):
|
|
|
- def __init__(self, dbpool, conn, cursor, redis_db):
|
|
|
- self.dbpool = dbpool
|
|
|
- self.conn = conn
|
|
|
- self.cursor = cursor
|
|
|
- self.redis_db = redis_db
|
|
|
-
|
|
|
- @classmethod
|
|
|
- def from_settings(cls, settings):
|
|
|
- dbparms = dict(
|
|
|
- host=settings["POST_HOST"],
|
|
|
- user=settings["POST_USER"],
|
|
|
- password=settings["POST_PASSWORD"],
|
|
|
- port=settings['POST_PORT'],
|
|
|
- )
|
|
|
- dbpool = adbapi.ConnectionPool("psycopg2", **dbparms)
|
|
|
- conn = psycopg2.connect(host=settings["POST_HOST"], port=settings['POST_PORT'], user=settings["POST_USER"], password=settings["POST_PASSWORD"], database=settings["POST_DATABASE"])
|
|
|
- cursor = conn.cursor(cursor_factory=psycopg2.extras.DictCursor)
|
|
|
- pool = redis.ConnectionPool(host=settings["R_HOST"], port=settings["R_POST"], password=settings["R_PASSWORD"])
|
|
|
- redis_db = redis.StrictRedis(connection_pool=pool)
|
|
|
- return cls(dbpool, conn, cursor, redis_db)
|
|
|
+ def open_spider(self, spider):
|
|
|
+ self.dbpool = adbapi.ConnectionPool("psycopg2",host=settings["POST_HOST"], port=settings['POST_PORT'], user=settings["POST_USER"], password=settings["POST_PASSWORD"], database=settings["POST_DATABASE"])
|
|
|
+ self.pool = redis.ConnectionPool(host=settings["R_HOST"], port=settings["R_POST"], password=settings["R_PASSWORD"])
|
|
|
+ self.redis_db = redis.StrictRedis(connection_pool=self.pool)
|
|
|
|
|
|
def process_item(self, item, spider):
|
|
|
# 使用twisted将mysql插入变成异步执行
|
|
|
@@ -1464,15 +1316,11 @@ class Liansaipipeline(object):
|
|
|
cursor.execute(sql3, (area_id, area_name, 0, 0, 0, utime, "hg3535", utime, "hg3535"))
|
|
|
|
|
|
def close_spider(self, spider):
|
|
|
- self.conn.close()
|
|
|
self.dbpool.close()
|
|
|
# self.redis_db.disconnect()
|
|
|
|
|
|
#************************************滚球-------------------------------------------------------------------------------
|
|
|
|
|
|
-
|
|
|
-
|
|
|
-
|
|
|
#滚球方法都加了s
|
|
|
|
|
|
def zqone_intodbs(data1, data2, data3, data4, cursor, redis_db):
|
|
|
@@ -1582,26 +1430,10 @@ def news_times(ctime):
|
|
|
|
|
|
|
|
|
class Roll_Zuqiupipeline(object):
|
|
|
- def __init__(self, dbpool, conn, cursor, redis_db):
|
|
|
- self.dbpool = dbpool
|
|
|
- self.conn = conn
|
|
|
- self.cursor = cursor
|
|
|
- self.redis_db = redis_db
|
|
|
-
|
|
|
- @classmethod
|
|
|
- def from_settings(cls, settings):
|
|
|
- dbparms = dict(
|
|
|
- host=settings["POST_HOST"],
|
|
|
- user=settings["POST_USER"],
|
|
|
- password=settings["POST_PASSWORD"],
|
|
|
- port=settings['POST_PORT'],
|
|
|
- )
|
|
|
- dbpool = adbapi.ConnectionPool("psycopg2", **dbparms)
|
|
|
- conn = psycopg2.connect(host=settings["POST_HOST"], port=settings['POST_PORT'], user=settings["POST_USER"], password=settings["POST_PASSWORD"], database=settings["POST_DATABASE"])
|
|
|
- cursor = conn.cursor(cursor_factory=psycopg2.extras.DictCursor)
|
|
|
- pool = redis.ConnectionPool(host=settings["R_HOST"], port=settings["R_POST"], password=settings["R_PASSWORD"])
|
|
|
- redis_db = redis.StrictRedis(connection_pool=pool)
|
|
|
- return cls(dbpool, conn, cursor, redis_db)
|
|
|
+ def open_spider(self, spider):
|
|
|
+ self.dbpool = adbapi.ConnectionPool("psycopg2",host=settings["POST_HOST"], port=settings['POST_PORT'], user=settings["POST_USER"], password=settings["POST_PASSWORD"], database=settings["POST_DATABASE"])
|
|
|
+ self.pool = redis.ConnectionPool(host=settings["R_HOST"], port=settings["R_POST"], password=settings["R_PASSWORD"])
|
|
|
+ self.redis_db = redis.StrictRedis(connection_pool=self.pool)
|
|
|
|
|
|
def process_item(self, item, spider):
|
|
|
# 使用twisted将mysql插入变成异步执行
|
|
|
@@ -1830,28 +1662,16 @@ class Roll_Zuqiupipeline(object):
|
|
|
'match_date': r_ctime, 'match_time': '00:00:00', 'utime': utime, 'number': number,
|
|
|
'source': "hg3535", "expire_time": expire_time, 'is_rollball': 1, "us_time":'2019-01-01 00:00:00','time_game':time_game}
|
|
|
zqone_competitions(data=data_competition, cursor=cursor)
|
|
|
-
|
|
|
+ def close_spider(self, spider):
|
|
|
+ # self.conn.close()
|
|
|
+ self.dbpool.close()
|
|
|
|
|
|
|
|
|
class BallStatuspipeline(object):
|
|
|
- def __init__(self, dbpool, conn, cursor):
|
|
|
- self.dbpool = dbpool
|
|
|
- self.conn = conn
|
|
|
- self.cursor = cursor
|
|
|
-
|
|
|
- @classmethod
|
|
|
- def from_settings(cls, settings):
|
|
|
- dbparms = dict(
|
|
|
- host=settings["POST_HOST"],
|
|
|
- # db = settings["POST_DATABASE"],
|
|
|
- user=settings["POST_USER"],
|
|
|
- password=settings["POST_PASSWORD"],
|
|
|
- port=settings['POST_PORT'],
|
|
|
- )
|
|
|
- dbpool = adbapi.ConnectionPool("psycopg2", **dbparms)
|
|
|
- conn = psycopg2.connect(host=settings["POST_HOST"], port=settings['POST_PORT'], user=settings["POST_USER"],password=settings["POST_PASSWORD"],database=settings["POST_DATABASE"])
|
|
|
- cursor = conn.cursor(cursor_factory=psycopg2.extras.DictCursor)
|
|
|
- return cls(dbpool,conn,cursor)
|
|
|
+ def open_spider(self, spider):
|
|
|
+ self.dbpool = adbapi.ConnectionPool("psycopg2",host=settings["POST_HOST"], port=settings['POST_PORT'], user=settings["POST_USER"], password=settings["POST_PASSWORD"], database=settings["POST_DATABASE"])
|
|
|
+ self.pool = redis.ConnectionPool(host=settings["R_HOST"], port=settings["R_POST"], password=settings["R_PASSWORD"])
|
|
|
+ self.redis_db = redis.StrictRedis(connection_pool=self.pool)
|
|
|
|
|
|
def process_item(self, item, spider):
|
|
|
# 使用twisted将mysql插入变成异步执行
|
|
|
@@ -1885,5 +1705,5 @@ class BallStatuspipeline(object):
|
|
|
# 现在时间，时间戳
|
|
|
utime = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime())
|
|
|
def close_spider(self, spider):
|
|
|
- self.conn.close()
|
|
|
+ # self.conn.close()
|
|
|
self.dbpool.close()
|
Your Name