token更改

hg3535/change.py

@@ -1,50 +0,0 @@
-import time
-# from venv import logger
-import redis
-from utils.helper import Helper
-from settings import R_HOST, R_PASSWORD, R_POST, R_DB, MATCH_STATUS
-
-
-def get_redis(k):
-    rls = redis.Redis(host=R_HOST, port=R_POST, db=R_DB, password=R_PASSWORD)
-    ids = rls.hgetall(k)
-    return ids
-
-
-def Change_status(ids):
-    if ids:
-        for id, up_time in ids.items():
-            match_id = id.decode()
-            up_time = up_time.decode().split('&')
-            last_time = float(up_time[0])
-            game_code = up_time[1]
-            new_time = time.time() - last_time
-            if new_time > 900:
-                status_dict = {"game_code": game_code, "title": "match_status", "source": "hg3535"}
-                data_list = []
-                data = {'match_id': match_id, 'status': 2, "is_rollball": 0, "is_today": 0, "is_morningplate": 0,
-                        "is_stringscene": 0, "is_horn": 0}
-                data_list.append(data)
-                status_dict['data'] = data_list
-                res = Helper.async_post(MATCH_STATUS, status_dict)
-                if res:
-                    if "成功" in res:
-                        # self.db.match_status35.insert(status_dict)
-                        print('{},赛事结果状态交成功, {}'.format(game_code, res))
-                        print(status_dict)
-                    else:
-                        print('{},赛事结果状态交失败, {}'.format(game_code, res))
-                        print(status_dict)
-                else:
-                    print('{},赛事结果状态接口异常提交失败, {}'.format(game_code, res))
-                    print(status_dict)
-            else:
-                print('15分钟内, 暂无球类赛事结束或取消')
-
-
-if __name__ == '__main__':
-    while True:
-        ids = get_redis('hg3535.ball.ids')
-        Change_status(ids)
-        print('*****-------球类更新状态, 运行结束--------*****')
-        time.sleep(30)

hg3535/jianting.py

@@ -1,45 +0,0 @@
-import os
-import glob
-
-
-def search_file(file_path, file):
-    # file_path指定路径, file指定文件名
-    path = file_path
-    file_list = []
-    for root, dirs, files in os.walk(path):
-        file_pattern = os.path.join(root, file)
-        for f in glob.glob(file_pattern):
-            file_list.append(f)
-    return file_list
-
-
-def into(f):
-    pass
-    # with open(f, 'wb') as i:
-    #     pass
-        # f = str.encode(f)
-        # i.write(f)
-
-
-def func(file_list):
-    if not file_list:
-        print("没有这个文件")
-    else:
-        print("一共找出 %d 个文件" % len(file_list))
-        print("找出的文件如下: ")
-        for f in file_list:
-            print(f)
-            into(f)
-            # with open(f, 'wb') as i:
-            #     f = str.encode(f)
-            #     i.write(f)
-
-    print('查找文件结束')
-
-
-
-if __name__ == '__main__':
-    file_path = r'C:\Users\admin\Desktop\newscore'
-    file = 'id_json.txt'
-    file_list = search_file(file_path, file)
-    func(file_list)

hg3535/main.py

@@ -11,8 +11,8 @@ sys.path.append(os.path.dirname(os.path.abspath(__file__)))
 # execute(["scrapy", "crawl", "wqbodan"])
 # execute(["scrapy", "crawl", "bangqiu"])
 # execute(["scrapy", "crawl", "roll_zuqiu"]) # 滚球足球 回来要解开这个注释 其他全部解封
-execute(["scrapy", "crawl", "roll_lanqiu"]) #滚球篮球
-# execute(["scrapy", "crawl", "roll_wangqiu"]) #滚球网球
+# execute(["scrapy", "crawl", "roll_lanqiu"]) #滚球篮球
+execute(["scrapy", "crawl", "roll_wangqiu"]) #滚球网球
 # execute(["scrapy", "crawl", "roll_bangqiu"])  # 滚球棒球
 # execute(["scrapy", "crawl", "saiguo"]) #滚球id结束时间更新状态
 # execute(["scrapy", "crawl", "jieshu"]) #滚球id结束时间更新状态

hg3535/main_time.py

@@ -1,47 +0,0 @@
-import time
-import os
-
-import sys
-
-
-# print(sys.argv[1])
-
-can = sys.argv[1]
-# tag = sys.argv[2]
-# print(type(can))
-
-if can == 'roll':
-    while True:
-        os.system("scrapy crawl roll_zuqiu")
-        time.sleep(2)
-        # os.system("scrapy crawl zq_jieshu")
-        # time.sleep(2)
-        os.system("scrapy crawl roll_lanqiu")
-        time.sleep(2)
-        # os.system("scrapy crawl lq_jieshu")
-        # time.sleep(2)
-        os.system("scrapy crawl roll_wangqiu")
-        time.sleep(2)
-        # os.system("scrapy crawl wq_jieshu")
-        # time.sleep(2)
-        os.system("scrapy crawl roll_bangqiu")
-        # time.sleep(2)
-        # os.system("scrapy crawl bq_jieshu")
-        time.sleep(120)
-
-if can == 'notroll':
-    while True:
-        os.system("scrapy crawl zuqiu")
-        time.sleep(4)
-        os.system("scrapy crawl wqbodan")
-        time.sleep(4)
-        os.system("scrapy crawl wangqiu")
-        time.sleep(4)
-        os.system("scrapy crawl lanqiu")
-        time.sleep(4)
-        os.system("scrapy crawl liansai")
-        time.sleep(4)
-        os.system("scrapy crawl guanjun")
-        time.sleep(4)
-        os.system("scrapy crawl bangqiu")
-        time.sleep(300)

hg3535/not_roll.py

@@ -1,28 +0,0 @@
-import time
-import os
-# from subprocess import Popen
-from apscheduler.schedulers.background import BackgroundScheduler
-# pip install apscheduler
-
-
-def notroll():
-    os.system("scrapy crawl liansai")
-    os.system("scrapy crawl zuqiu")
-    os.system("scrapy crawl wqbodan")
-    os.system("scrapy crawl wangqiu")
-    os.system("scrapy crawl lanqiu")
-    os.system("scrapy crawl guanjun")
-    os.system("scrapy crawl bangqiu")
-
-
-if __name__ == "__main__":
-    scheduler = BackgroundScheduler()
-    # 每20分钟执行一次
-    scheduler.add_job(notroll, 'cron', minute="*/10")
-    scheduler.start()
-    print('Press Ctrl+{0} to exit'.format('Break' if os.name == 'nt' else 'C'))
-    try:
-        while True:
-            time.sleep(2)
-    except (KeyboardInterrupt, SystemExit):
-        scheduler.shutdown()

hg3535/pipeline/bangqiu.py

@@ -9,7 +9,7 @@ import logging
 from twisted.internet import defer, reactor
 from ..utils.helper import Helper
 from .ball_func import new_time
-from ..settings import M_HOST, M_USER, M_PASSWORD, M_POST, M_DB, LEAGUE_URL, ODDS_URL, MATCH_URL
+from ..settings import M_HOST, M_USER, M_PASSWORD, M_POST, M_DB, LEAGUE_URL, ODDS_URL, MATCH_URL, POST_URL
 
 
 class Bangqiupipeline(object):
@@ -90,7 +90,7 @@ class Bangqiupipeline(object):
             league_data = dict(zip(league_key, league_value))
             league_list.append(league_data)
             league_dict['data'] = league_list
-            res = Helper.async_post(LEAGUE_URL, league_dict)
+            res = Helper.async_post('{}/setLeague'.format(POST_URL), league_dict)
             if res:
                 if res.get('status') == 1:
                     self.db.bq_league35.insert(league_data)
@@ -133,7 +133,7 @@ class Bangqiupipeline(object):
             match_data = dict(zip(match_key, match_value))
             match_list.append(match_data)
             match_dict['data'] = match_list
-            res = Helper.async_post(MATCH_URL, match_dict)
+            res = Helper.async_post('{}/setMatch'.format(POST_URL), match_dict)
             if res:
                 if res.get('status') == 1:
                     self.db.bq_competition35.insert(match_data)
@@ -218,7 +218,7 @@ class Bangqiupipeline(object):
                       ris_stringscene, utime, pt, match_identity]
         odds_dict = dict(zip(odds_key, odds_value))
         if data_list:
-            res = Helper.async_post(ODDS_URL, odds_dict)
+            res = Helper.async_post('{}/setOdds'.format(POST_URL), odds_dict)
             if res:
                 if res.get('status') == 1:
                     logger.info('棒球赔率提交成功, {}, {}'.format(res, odds_dict))

hg3535/pipeline/guanjun.py

@@ -8,7 +8,7 @@ import logging
 from twisted.internet import defer, reactor
 
 from ..utils.helper import Helper
-from ..settings import M_HOST, M_USER, M_PASSWORD, M_POST, M_DB, LEAGUE_URL, ODDSCH
+from ..settings import M_HOST, M_USER, M_PASSWORD, M_POST, M_DB, LEAGUE_URL, ODDSCH, POST_URL
 
 
 class Guanjunpipeline(object):
@@ -76,7 +76,7 @@ class Guanjunpipeline(object):
             league_list.append(league_dict)
             l_payload['data'] = league_list
             try:
-                res = Helper.async_post(LEAGUE_URL, l_payload)
+                res = Helper.async_post('{}/setLeague'.format(POST_URL), l_payload)
                 if res:
                     if res.get('status') == 1:
                         logger.info('{}, {}冠军联赛, 提交成功, {}'.format(game_code, league_name, res))
@@ -114,7 +114,7 @@ class Guanjunpipeline(object):
             chain_dict['data'] = data_list
             chain_dict['odds_only'] = only_list
             try:
-                res = Helper.async_post(ODDSCH, chain_dict)
+                res = Helper.async_post('{}/setOdds'.format(POST_URL), chain_dict)
                 if res:
                     if res.get('status') == 1:
                         logger.info('{}冠军赔率，提交成功, {}'.format(game_code, res))
@@ -124,7 +124,7 @@ class Guanjunpipeline(object):
                 else:
                     logger.warning('{}, 冠军赔率接口, 接口异常, 提交失败, {}, {}'.format(game_code, res, chain_dict))
             except Exception as e:
-                logger.warning('冠军赔率接口异常，提交失败,{}'.format(e))
+                logger.warning('冠军赔率接口异常，提交失败,{}, {}'.format(e, chain_dict))
 
         else:
             logger.info('足球冠军赔率, 列表为空, 不提交过')

hg3535/pipeline/jieshu.py

@@ -7,7 +7,7 @@ import logging
 from twisted.internet import defer, reactor
 import redis
 
-from ..settings import M_HOST, M_USER, M_PASSWORD, M_POST, M_DB, MATCH_STATUS
+from ..settings import M_HOST, M_USER, M_PASSWORD, M_POST, M_DB, POST_URL
 from ..utils.helper import Helper
 from ..settings import R_HOST, R_PASSWORD, R_POST, R_DB
 
@@ -16,7 +16,7 @@ from ..settings import R_HOST, R_PASSWORD, R_POST, R_DB
 class Jieshuqiupipeline(object):
 
     def open_spider(self, spider):
-        self.mongo = pymongo.MongoClient(host=M_HOST, username=M_USER, password=M_PASSWORD, port=M_POST)
+        self.mongo = pymongo.MongoClient(host=M_HOST, username=M_USER, password=M_PASSWORD, port=M_POST, authSource=M_DB)
         self.db = self.mongo[M_DB]
         self.rls = redis.Redis(host=R_HOST, port=R_POST, db=R_DB, password=R_PASSWORD)
 
@@ -57,7 +57,7 @@ class Jieshuqiupipeline(object):
                 "match_identity": match_identity}
         data_list.append(data)
         status_dict['data'] = data_list
-        res = Helper.async_post(MATCH_STATUS, status_dict)
+        res = Helper.async_post('{}/upMatch'.format(POST_URL), status_dict)
         if res:
             if res.get('status') == 1:
                 self.db.match_status35.insert(data)

hg3535/pipeline/lanqiu.py

@@ -8,12 +8,12 @@ import logging
 from twisted.internet import defer, reactor
 from ..utils.helper import Helper
 from .ball_func import new_time
-from ..settings import M_HOST, M_USER, M_PASSWORD, M_POST, M_DB, LEAGUE_URL, ODDS_URL, MATCH_URL
+from ..settings import M_HOST, M_USER, M_PASSWORD, M_POST, M_DB, POST_URL
 
 
 class Lanqiupipeline(object):
     def open_spider(self, spider):
-        self.mongo = pymongo.MongoClient(host=M_HOST, username=M_USER, password=M_PASSWORD, port=M_POST)
+        self.mongo = pymongo.MongoClient(host=M_HOST, username=M_USER, password=M_PASSWORD, port=M_POST, authSource=M_DB)
         self.db = self.mongo[M_DB]
 
     @defer.inlineCallbacks
@@ -89,7 +89,7 @@ class Lanqiupipeline(object):
             league_data = dict(zip(league_key, league_value))
             league_list.append(league_data)
             league_dict['data'] = league_list
-            res = Helper.async_post(LEAGUE_URL, league_dict)
+            res = Helper.async_post('{}/setLeague'.format(POST_URL), league_dict)
             if res:
                 if res.get('status') == 1:
                     logger.info('篮球联赛提交成功, {}'.format(res))
@@ -131,7 +131,7 @@ class Lanqiupipeline(object):
             match_data = dict(zip(match_kay, match_value))
             match_list.append(match_data)
             match_dict['data'] = match_list
-            res = Helper.async_post(MATCH_URL, match_dict)
+            res = Helper.async_post('{}/setMatch'.format(POST_URL), match_dict)
             if res:
                 if res.get('status') == 1:
                     logger.info('篮球赛事提交成功, {}'.format(res))
@@ -261,7 +261,7 @@ class Lanqiupipeline(object):
                       ris_stringscene, utime, pt, match_identity]
         odds_dict = dict(zip(odds_key, odds_value))
         if data_list:
-            res = Helper.async_post(ODDS_URL, odds_dict)
+            res = Helper.async_post('{}/setOdds'.format(POST_URL), odds_dict)
             if res:
                 if res.get('status') == 1:
                     logger.info('篮球详细赔率提交成功, {}'.format(res))

hg3535/pipeline/roll_bangqiu.py

@@ -7,7 +7,7 @@ import time
 import logging
 from twisted.internet import defer, reactor
 from ..utils.helper import Helper
-from ..settings import M_HOST, M_USER, M_PASSWORD, M_POST, M_DB, LEAGUE_URL, ODDS_URL, MATCH_URL, MATCH_RESULT
+from ..settings import M_HOST, M_USER, M_PASSWORD, M_POST, M_DB, POST_URL
 
 
 class Roll_Banqiupipeline(object):
@@ -78,7 +78,7 @@ class Roll_Banqiupipeline(object):
             league_data = dict(zip(league_key, league_value))
             league_list.append(league_data)
             league_dict['data'] = league_list
-            res = Helper.async_post(LEAGUE_URL, league_dict)
+            res = Helper.async_post('{}/setLeague'.format(POST_URL), league_dict)
             if res:
                 if res.get('status') == 1:
                     self.db.bq_league35.insert(league_data)
@@ -102,16 +102,14 @@ class Roll_Banqiupipeline(object):
             match_data = dict(zip(match_key, match_value))
             match_list.append(match_data)
             match_dict['data'] = match_list
-            res = Helper.async_post(MATCH_URL, match_dict)
+            res = Helper.async_post('{}/setMatch'.format(POST_URL), match_dict)
             if res:
                 if res.get('status') == 1:
                     self.db.bq_competition35.insert(match_data)
                 else:
                     logger.warning('棒球滚球赛事提交失败, {}, {}'.format(res, match_data))
-                    # logger.warning(match_data)
             else:
                 logger.warning('棒球滚球赛事,接口异常,提交失败, {}, {}'.format(res, match_data))
-                # logger.warning(match_data)
         else:
             logger.info("棒球赛事已存在，棒球赛事状态时间未改变，不做修改")
         """赔率"""
@@ -184,17 +182,14 @@ class Roll_Banqiupipeline(object):
                       0, cdate, pt, match_identity]
         odds_dict = dict(zip(odds_key, odds_value))
         if data_list:
-            res = Helper.async_post(ODDS_URL, odds_dict)
+            res = Helper.async_post('{}/setOdds'.format(POST_URL), odds_dict)
             if res:
                 if res.get('status') == 1:
                     logger.info('棒球滚球赔率提交成功, {}'.format(res))
-                    # logger.info(odds_dict)
                 else:
                     logger.warning('棒球滚球赔率提交失败, {}, {}'.format(res, odds_dict))
-                    # logger.warning(odds_dict)
             else:
                 logger.warning('棒球滚球赔率接口异常,提交失败, {}, {}'.format(res, odds_dict))
-                # logger.warning(odds_dict)
         else:
             logger.info('棒球滚球详细赔率列表为空, 不提交')
         """赛事结果"""
@@ -217,18 +212,15 @@ class Roll_Banqiupipeline(object):
         }
         if da_list:
             try:
-                res = Helper.async_post(MATCH_RESULT, r_data_dict)
+                res = Helper.async_post('{}/setMatchResult'.format(POST_URL), r_data_dict)
                 if res.get('status') == 1:
                     logger.info('棒球滚球结果记录提交成功, {}'.format(res))
                     self.db.bq_match_result35.insert(bq_rball)
-                    # logger.info(r_data_dict)
                 else:
                     logger.warning('棒球滚球结果记录提交失败, {}, {}'.format(res, r_data_dict))
-                    # logger.warning(r_data_dict)
 
             except Exception as e:
-                logger.warning("滚球接口数据异常, 提交失败, {}".format(e))
-                # logging.warning(r_data_dict)
+                logger.warning("滚球接口数据异常, 提交失败, {}, {}".format(e, r_data_dict))
         else:
             logger.info('棒球滚球结果记录为空, 不提交')
         reactor.callFromThread(out.callback, item)

hg3535/pipeline/roll_lanqiu.py

@@ -8,12 +8,12 @@ import logging
 
 from twisted.internet import defer, reactor
 from ..utils.helper import Helper
-from ..settings import M_HOST, M_USER, M_PASSWORD, M_POST, M_DB, LEAGUE_URL, ODDS_URL, MATCH_URL, MATCH_RESULT
+from ..settings import M_HOST, M_USER, M_PASSWORD, M_POST, M_DB, POST_URL
 
 
 class Roll_Lanqiupipeline(object):
     def open_spider(self, spider):
-        self.mongo = pymongo.MongoClient(host=M_HOST, username=M_USER, password=M_PASSWORD, port=M_POST)
+        self.mongo = pymongo.MongoClient(host=M_HOST, username=M_USER, password=M_PASSWORD, port=M_POST, authSource=M_DB)
         self.db = self.mongo[M_DB]
 
     @defer.inlineCallbacks
@@ -76,7 +76,7 @@ class Roll_Lanqiupipeline(object):
             league_data = dict(zip(league_key, league_value))
             league_list.append(league_data)
             league_dict['data'] = league_list
-            res = Helper.async_post(LEAGUE_URL, league_dict)
+            res = Helper.async_post('{}/setLeague'.format(POST_URL), league_dict)
             if res:
                 if res.get('status') == 1:
                     logger.info('篮球滚球联赛提交成功, {}'.format(res))
@@ -99,7 +99,7 @@ class Roll_Lanqiupipeline(object):
             match_data = dict(zip(match_kay, match_value))
             match_list.append(match_data)
             match_dict['data'] = match_list
-            res = Helper.async_post(MATCH_URL, match_dict)
+            res = Helper.async_post('{}/setMatch'.format(POST_URL), match_dict)
             if res:
                 if res.get('status') == 1:
                     logger.info('篮球滚球赛事提交成功, {}'.format(res))
@@ -233,7 +233,7 @@ class Roll_Lanqiupipeline(object):
                       0, cdate, pt, match_identity]
         odds_dict = dict(zip(odds_key, odds_value))
         if data_list:
-            res = Helper.async_post(ODDS_URL, odds_dict)
+            res = Helper.async_post('{}/setOdds'.format(POST_URL), odds_dict)
             if res:
                 if res.get('status') == 1:
                         logger.info('篮球滚球详细赔率提交成功, {}'.format(res))
@@ -261,7 +261,7 @@ class Roll_Lanqiupipeline(object):
         }
         if data_list:
             try:
-                res = Helper.async_post(MATCH_RESULT, r_data_dict)
+                res = Helper.async_post('{}/setMatchResult'.format(POST_URL), r_data_dict)
                 if res.get('status') == 1:
                     logger.info('篮球滚球赛事结果记录提交成功, {}'.format(res))
                     self.db.lq_match_result35.insert(lq_rball)

hg3535/pipeline/roll_wangqiu.py

@@ -6,14 +6,14 @@ import pymongo
 import time
 import logging
 from ..utils.helper import Helper
-from ..settings import M_HOST, M_USER, M_PASSWORD, M_POST, M_DB, LEAGUE_URL, ODDS_URL, MATCH_URL, MATCH_RESULT
+from ..settings import M_HOST, M_USER, M_PASSWORD, M_POST, M_DB, POST_URL
 from twisted.internet import defer, reactor
 
 
 class Roll_Wangqiupipeline(object):
     def open_spider(self, spider):
         self.mongo = pymongo.MongoClient(host=M_HOST, username=M_USER, password=M_PASSWORD, port=M_POST,
-                                         authSource='kaiyou')
+                                         authSource=M_DB)
         self.db = self.mongo[M_DB]
 
     @defer.inlineCallbacks
@@ -91,7 +91,7 @@ class Roll_Wangqiupipeline(object):
             league_data = dict(zip(league_key, league_value))
             league_list.append(league_data)
             league_dict['data'] = league_list
-            res = Helper.async_post(LEAGUE_URL, league_dict)
+            res = Helper.async_post('{}/setLeague'.format(POST_URL), league_dict)
             if res:
                 if res.get('status') == 1:
                     self.db.wq_league35.insert(league_data)
@@ -116,7 +116,7 @@ class Roll_Wangqiupipeline(object):
             match_data = dict(zip(match_key, match_value))
             match_list.append(match_data)
             match_dict['data'] = match_list
-            res = Helper.async_post(MATCH_URL, match_dict)
+            res = Helper.async_post('{}/setMatch'.format(POST_URL), match_dict)
             if res:
                 if res.get('status') == 1:
                     self.db.wq_competition35.insert(match_data)
@@ -217,7 +217,7 @@ class Roll_Wangqiupipeline(object):
                       0, cdate, pt, match_identity]
         odds_dict = dict(zip(odds_key, odds_value))
         if data_list:
-            res = Helper.async_post(ODDS_URL, odds_dict)
+            res = Helper.async_post('{}/setOdds'.format(POST_URL), odds_dict)
             if res:
                 if res.get('status') == 1:
                     logger.info('网球滚球赔率提交成功, {}'.format(res))
@@ -253,14 +253,12 @@ class Roll_Wangqiupipeline(object):
         }
         if da_list:
             try:
-                res = Helper.async_post(MATCH_RESULT, r_data_dict)
+                res = Helper.async_post('{}/setMatchResult'.format(POST_URL), r_data_dict)
                 if res.get('status') == 1:
                     logger.info('网球滚球结果记录提交成功, {}'.format(res))
                     self.db.wq_match_result35.insert(wq_rball)
-                    # logger.info(r_data_dict)
                 else:
                     logger.warning('网球滚球结果记录提交失败, {}, {}'.format(res, r_data_dict))
-                    # logger.warning(r_data_dict)
             except Exception as e:
                 logger.warning("网球接口数据异常, 提交失败, {}".format(e))
         else:

hg3535/pipeline/roll_zuqiu.py

@@ -9,14 +9,14 @@ import logging
 
 from twisted.internet import defer, reactor
 from ..utils.helper import Helper
-from ..settings import M_HOST, M_USER, M_PASSWORD, M_POST, M_DB, LEAGUE_URL, ODDS_URL, MATCH_URL, MATCH_RESULT
+from ..settings import M_HOST, M_USER, M_PASSWORD, M_POST, M_DB, POST_URL
 
 
 # 滚球足球 插入
 class Roll_Zuqiupipeline(object):
 
     def open_spider(self, spider):
-        self.mongo = pymongo.MongoClient(host=M_HOST, username=M_USER, password=M_PASSWORD, port=M_POST)
+        self.mongo = pymongo.MongoClient(host=M_HOST, username=M_USER, password=M_PASSWORD, port=M_POST, authSource=M_DB)
         self.db = self.mongo[M_DB]
 
     @defer.inlineCallbacks
@@ -65,7 +65,7 @@ class Roll_Zuqiupipeline(object):
             league_data = dict(zip(league_key, league_value))
             league_list.append(league_data)
             league_dict['data'] = league_list
-            res = Helper.async_post(LEAGUE_URL, league_dict)
+            res = Helper.async_post('{}/setLeague'.format(POST_URL), league_dict)
             if res:
                 if res.get('status') == 1:
                     self.db.zq_league35.insert(league_data)
@@ -91,7 +91,7 @@ class Roll_Zuqiupipeline(object):
             match_data = dict(zip(match_kay, match_value))
             match_list.append(match_data)
             match_dict['data'] = match_list
-            res = Helper.async_post(MATCH_URL, match_dict)
+            res = Helper.async_post('{}/setMatch'.format(POST_URL), match_dict)
             if res:
                 if res.get('status') == 1:
                     self.db.zq_competition35.insert(match_data)
@@ -534,7 +534,7 @@ class Roll_Zuqiupipeline(object):
                       0, cdate, pt, match_identity]
         odds_dict = dict(zip(odds_key, odds_value))
         if data_list:
-            res = Helper.async_post(ODDS_URL, odds_dict)
+            res = Helper.async_post('{}/setOdds'.format(POST_URL), odds_dict)
             if res:
                 if res.get('status') == 1:
                     logger.info('足球滚球详细赔率提交成功, {}'.format(res))
@@ -565,7 +565,7 @@ class Roll_Zuqiupipeline(object):
         }
         if data_list:
             try:
-                res = Helper.async_post(MATCH_RESULT, r_data_dict)
+                res = Helper.async_post('{}/setMatchResult'.format(POST_URL), r_data_dict)
                 if res:
                     if res.get('status') == 1:
                         logger.info('足球滚球结果记录提交成功, {}'.format(res))

hg3535/pipeline/saiguo.py

@@ -5,7 +5,7 @@ import logging
 
 
 from twisted.internet import defer, reactor
-from ..settings import M_HOST, M_USER, M_PASSWORD, M_POST, M_DB, SAIGUO_RESULT
+from ..settings import M_HOST, M_USER, M_PASSWORD, M_POST, M_DB, POST_URL
 from ..utils.helper import Helper
 
 
@@ -13,7 +13,7 @@ from ..utils.helper import Helper
 class Jieshuqiupipeline(object):
 
     def open_spider(self, spider):
-        self.mongo = pymongo.MongoClient(host=M_HOST, username=M_USER, password=M_PASSWORD, port=M_POST)
+        self.mongo = pymongo.MongoClient(host=M_HOST, username=M_USER, password=M_PASSWORD, port=M_POST, authSource=M_DB)
         self.db = self.mongo[M_DB]
 
     @defer.inlineCallbacks
@@ -43,7 +43,7 @@ class Jieshuqiupipeline(object):
                     "match_time": match_time, "home_team": h_name, "guest_team": a_name, "score_half": h_score,
                     "score_full": f_score, "play_data": play_data, "match_id": match_id}
             post_data['data'].append(data)
-            res = Helper.async_post(SAIGUO_RESULT, post_data)
+            res = Helper.async_post('{}/setResultExpress'.format(POST_URL), post_data)
             if res:
                 if res.get('status') == 1:
                     self.db.zq_saiguo35.insert(post_data)
@@ -63,7 +63,7 @@ class Jieshuqiupipeline(object):
                     "guest_team": a_name, "score_half": h_score, "score_below": x_score, "score_result": f_score,
                     "play_data": play_data}
             post_data['data'].append(data)
-            res = Helper.async_post(SAIGUO_RESULT, post_data)
+            res = Helper.async_post('{}/setResultExpress'.format(POST_URL), post_data)
             if res:
                 if res.get('status') == 1:
                     self.db.lq_saiguo35.insert(post_data)
@@ -80,7 +80,7 @@ class Jieshuqiupipeline(object):
                     "match_time": match_time, "home_team": h_name, "guest_team": a_name, "score_result": f_score,
                     "play_data": play_data, "match_id": match_id}
             post_data['data'].append(data)
-            res = Helper.async_post(SAIGUO_RESULT, post_data)
+            res = Helper.async_post('{}/setResultExpress'.format(POST_URL), post_data)
             if res:
                 if res.get('status') == 1:
                     self.db.wq_saiguo35.insert(post_data)
@@ -99,7 +99,7 @@ class Jieshuqiupipeline(object):
                     "match_time": match_time, "home_team": h_name, "guest_team": a_name, "score_half": h_score,
                     "score_full": f_score, "play_data": play_data, "match_id": match_id}
             post_data['data'].append(data)
-            res = Helper.async_post(SAIGUO_RESULT, post_data)
+            res = Helper.async_post('{}/setResultExpress'.format(POST_URL), post_data)
             if res:
                 if res.get('status') == 1:
                     self.db.bq_saiguo35.insert(post_data)

hg3535/pipeline/wangqiu.py

@@ -9,12 +9,12 @@ import logging
 from twisted.internet import defer, reactor
 from ..utils.helper import Helper
 from .ball_func import new_time
-from ..settings import M_HOST, M_USER, M_PASSWORD, M_POST, M_DB, LEAGUE_URL, ODDS_URL, MATCH_URL
+from ..settings import M_HOST, M_USER, M_PASSWORD, M_POST, M_DB, POST_URL
 
 
 class Wangqiupipeline(object):
     def open_spider(self, spider):
-        self.mongo = pymongo.MongoClient(host=M_HOST, username=M_USER, password=M_PASSWORD, port=M_POST, authSource='kaiyou')
+        self.mongo = pymongo.MongoClient(host=M_HOST, username=M_USER, password=M_PASSWORD, port=M_POST, authSource=M_DB)
         self.db = self.mongo[M_DB]
 
     @defer.inlineCallbacks
@@ -90,7 +90,7 @@ class Wangqiupipeline(object):
             league_data = dict(zip(league_key, league_value))
             league_list.append(league_data)
             league_dict['data'] = league_list
-            res = Helper.async_post(LEAGUE_URL, league_dict)
+            res = Helper.async_post('{}/setLeague'.format(POST_URL), league_dict)
             if res:
                 if res.get('status') == 1:
                     self.db.wq_league35.insert(league_data)
@@ -136,7 +136,7 @@ class Wangqiupipeline(object):
             match_data = dict(zip(match_key, match_value))
             match_list.append(match_data)
             match_dict['data'] = match_list
-            res = Helper.async_post(MATCH_URL, match_dict)
+            res = Helper.async_post('{}/setMatch'.format(POST_URL), match_dict)
             if res:
                 if res.get('status') == 1:
                     self.db.wq_competition35.insert(match_data)
@@ -237,7 +237,7 @@ class Wangqiupipeline(object):
                       ris_stringscene, utime, pt, match_identity]
         odds_dict = dict(zip(odds_key, odds_value))
         if data_list:
-            res = Helper.async_post(ODDS_URL, odds_dict)
+            res = Helper.async_post('{}/setOdds'.format(POST_URL), odds_dict)
             if res:
                 if res.get('status') == 1:
                     logger.info('网球赔率提交成功, {}'.format(res))

hg3535/pipeline/zhibo.py

@@ -7,14 +7,14 @@ import logging
 
 from twisted.internet import defer, reactor
 from ..utils.helper import Helper
-from ..settings import M_HOST, M_USER, M_PASSWORD, M_POST, M_DB, MATCHWARN
+from ..settings import M_HOST, M_USER, M_PASSWORD, M_POST, M_DB, POST_URL
 
 
 # 滚球足球 插入
 class Zuqiupipeline(object):
 
     def open_spider(self, spider):
-        self.mongo = pymongo.MongoClient(host=M_HOST, username=M_USER, password=M_PASSWORD, port=M_POST)
+        self.mongo = pymongo.MongoClient(host=M_HOST, username=M_USER, password=M_PASSWORD, port=M_POST, authSource=M_DB)
         self.db = self.mongo[M_DB]
 
     @defer.inlineCallbacks
@@ -44,7 +44,7 @@ class Zuqiupipeline(object):
             warns.append(detail_dict)
         if warns:
             warn_dict['data'] = warns
-            res = Helper.async_post(MATCHWARN, warn_dict)
+            res = Helper.async_post('{}/setMatchWarn'.format(POST_URL), warn_dict)
             if res.get('status') == 1:
                 # print('危险球提交成功')
                 self.db.FT_warn35.insert(warn_dict)

hg3535/pipeline/zuqiu.py

@@ -6,7 +6,8 @@ import pymongo
 from twisted.internet import defer, reactor
 from ..utils.helper import Helper
 from .ball_func import new_time
-from ..settings import M_HOST, M_USER, M_PASSWORD, M_POST, M_DB, LEAGUE_URL, ODDS_URL, MATCH_URL
+# from ..settings import M_HOST, M_USER, M_PASSWORD, M_POST, M_DB, LEAGUE_URL, ODDS_URL, MATCH_URL
+from ..settings import M_HOST, M_USER, M_PASSWORD, M_POST, M_DB, POST_URL
 
 
 class Zuqiupipeline(object):
@@ -72,7 +73,7 @@ class Zuqiupipeline(object):
             league_data = dict(zip(league_key, league_value))
             league_list.append(league_data)
             league_dict['data'] = league_list
-            res = Helper.async_post(LEAGUE_URL, league_dict)
+            res = Helper.async_post('{}/setLeague'.format(POST_URL), league_dict)
             if res:
                 if res.get('status') == 1:
                     self.db.zq_league35.insert(league_data)
@@ -110,7 +111,7 @@ class Zuqiupipeline(object):
             match_data = dict(zip(match_kay, match_value))
             match_list.append(match_data)
             match_dict['data'] = match_list
-            res = Helper.async_post(MATCH_URL, match_dict)
+            res = Helper.async_post('{}/setMatch'.format(POST_URL), match_dict)
             if res:
                 if res.get('status') == 1:
                     self.db.zq_competition35.insert(match_data)
@@ -554,7 +555,7 @@ class Zuqiupipeline(object):
         odds_value = ["zq", "odds", match_id, league_id, data_list, "hg3535", odds_onlys, tag_number, uuid, ris_stringscene, utime, pt, match_identity]
         odds_dict = dict(zip(odds_key, odds_value))
         if data_list:
-            res = Helper.async_post(ODDS_URL, odds_dict)
+            res = Helper.async_post('{}/setOdds'.format(POST_URL), odds_dict)
             if res:
                 if res.get('status') == 1:
                     logger.info('足球详细赔率提交成功, {}'.format(res))

hg3535/settings.py

@@ -120,34 +120,6 @@ CONCURRENT_ITEMS = 1000
 # SCHEDULER_PERSIST = False
 # LOG_FILE = './log/'
 
-
-M_HOST = '192.168.2.200'
-# M_HOST = '127.0.0.1'
-M_POST = 27017
-M_USER = 'kaiyou'
-M_DB = 'kaiyou'
-M_PASSWORD = 'kaiyou'
-# M_PASSWORD = '123456'
-
-# LEAGUE_URL = 'http://stadmin.bocai108.com:19093/setLeague'
-LEAGUE_URL = 'http://stadmin.bocai108.com/setLeague'
-# MATCH_URL = 'http://stadmin.bocai108.com:19093/setMatch'
-MATCH_URL = 'http://stadmin.bocai108.com/setMatch'
-# ODDS_URL = 'http://stadmin.bocai108.com:19093/setOdds'
-ODDS_URL = 'http://stadmin.bocai108.com/setOdds'
-TOKEN_URL = "http://stadmin.bocai108.com/getToken"
-MATCH_RESULT = "http://stadmin.bocai108.com:19093/setMatchResult"
-MATCH_STATUS = "http://stadmin.bocai108.com:19093/upMatch"
-# MATCH_STATUS = "http://stadmin.bocai108.com/upMatch"
-ODDSCH = "http://stadmin.bocai108.com:19093/setOddsCH"
-SAIGUO_RESULT = "http://stadmin.bocai108.com:19093/setResultExpress"
-# SAIGUO_RESULT = "http://stadmin.bocai108.com/setResultExpress"
-MATCHWARN = "http://stadmin.bocai108.com:19093/setMatchWarn"
-
-R_HOST = '192.168.2.200'
-R_POST = 6379
-R_DB = 1
-R_PASSWORD = 123456
 # upMatch
 # POST_HOST = 'localhost'
 # POST_DATABASE = 'kaiyou'
@@ -156,13 +128,9 @@ R_PASSWORD = 123456
 # POST_PASSWORD = '123456'
 # SCHEDULER = "scrapy_redis.scheduler.Scheduler"
 # DUPEFILTER_CLASS = "scrapy_redis.dupefilter.RFPDupeFilter"
-
 # SCHEDULER_SERIALIZER = "scrapy_redis.picklecompat"
-
 # ＃不要清理redis队列，允许暂停/恢复抓取。
-
 # SCHEDULER_PERSIST = False
-
 # ＃使用优先级队列安排请求。（默认）
 # SCHEDULER_QUEUE_CLASS = 'scrapy_redis.queue.PriorityQueue'
 # REDIS_HOST = '192.168.2.200'
@@ -179,3 +147,26 @@ R_PASSWORD = 123456
 # DUPEFILTER_CLASS = "scrapy_redis.dupefilter.RFPDupeFilter"
 # REDIS_URL = 'redis://:123456@192.168.2.200:6379'
 # SCHEDULER_PERSIST = False
+
+# mongo
+M_HOST = '192.168.2.200'
+M_POST = 27017
+M_USER = 'kaiyou'
+M_DB = 'kaiyou'
+M_PASSWORD = 'kaiyou'
+
+# redis
+R_HOST = '192.168.2.200'
+R_POST = 6379
+R_DB = 1
+R_PASSWORD = 123456
+
+# token
+T_USER = 'python'
+T_PASSWORD = 'python888'
+T_URL = 'http://stadmin.bocai108.com/getToken'
+
+
+# 域名
+POST_URL = 'http://stadmin.bocai108.com:19093'
+

hg3535/utils/LocalToken.py

@@ -1 +1 @@
-token = {'token': 'mVUm5Y15717377705daed0aad4869', 'username': 'python', 'password': 'python888', 'token_url': 'http://stadmin.bocai108.com/getToken'}
+token = {'token': 'fa2V1A15717383585daed2f66dca8', 'username': 'python', 'password': 'python888', 'token_url': 'http://stadmin.bocai108.com/getToken'}

hg3535/utils/helper.py

@@ -3,18 +3,19 @@ import json
 from requests_futures.sessions import FuturesSession
 from .langconv import *
 from .LocalToken import token
+from ..settings import T_USER, T_PASSWORD, T_URL
 
 
 class Helper(object):
     @staticmethod
     def async_post(url, params):
         fs_session = FuturesSession()
-        t_url, t_user, t_password, t_token = token['token_url'], token['username'], token['password'], token['token']
+        t_url, t_user, t_password, t_token = T_URL, T_USER, T_PASSWORD, token['token']
         data = fs_session.post(url, data={"data": json.dumps(params), "token": t_token}, timeout=180).result()
         try:
             new_data = data.json()
             if new_data.get('status') == 6:
-                t_data = fs_session.post(url=t_url, data={'account': t_user, 'password': t_password}).result()
+                t_data = fs_session.post(url=T_URL, data={'account': T_USER, 'password': T_PASSWORD}).result()
                 if t_data.json().get('status') == 1:
                     g_token = t_data.json()['data']['token']
                     token['token'] = g_token