update

hgg070_spider/items.py

@@ -16,4 +16,6 @@ class LanqiuItem(scrapy.Field):
     id=scrapy.Field()
     league=scrapy.Field()
     team_h=scrapy.Field()
-    team_c = scrapy.Field()
+    team_c = scrapy.Field()
+    showtype=scrapy.Field()
+    datetime=scrapy.Field()

hgg070_spider/pipelines/lanqiu.py

@@ -1,2 +1,56 @@
+import logging
+from twisted.internet import defer,reactor
+from ..utils.helper import Helper
+from ..settings import LEAGUE_URL,MATCH_URL
 class ZuqiuPipeline(object):
-    pass
+    @defer.inlineCallbacks
+    def process_item(self,item,spider):
+        logger=logging.getLogger(__name__)
+        logger.info("进入管道")
+        out=defer.Deferred()
+        reactor.callInThread(self._do_calculation,item,out)
+        yield out
+
+
+    def _do_calculation(self,item,out):
+        #先保存联赛
+        league_name = item['league']
+        uuid = Helper.genearte_uuid(league_name)
+        type=item['showtype']
+        is_rollball,is_today,is_morningplate = 0,0,0
+        if type=="FT":
+            is_today=1
+        elif type=="":
+            is_morningplate=1
+        else:
+            is_rollball=1
+
+        obj={"uuid":uuid,"league_id":item['id'],"league_name":league_name}
+        res=Helper.async_post(LEAGUE_URL,obj)
+        if res:
+            if res.get('status')==1:
+                logging.warning("联赛提交成功,{}".format(res))
+                #提交赛事
+                match_identity = Helper.genearte_uuid(item['team_h'] + item['team_c'] + item['datetime'])
+                childer={"uuid":uuid,"match_identity":match_identity,"team_h":item["team_h"],"team_c":item["team_c"],
+                         "is_today":is_today,"is_morningplate": is_morningplate, "is_rollball": is_rollball
+                }
+                lres=Helper.async_post(MATCH_URL,childer)
+                if lres.get('status')==1:
+                    logging.warning("联赛提交成功,{}".format(res))
+                else:
+                    logging.warning("联赛提交失败,{}".format(res))
+
+            else:
+                logging.warning("联赛提交失败,{}".format(res))
+        else:
+            logging.warning("联赛提交失败,{}".format(res))
+
+
+
+
+
+
+
+
+

hgg070_spider/spiders/lanqiu.py

@@ -80,12 +80,11 @@ class LanqiuSpider(scrapy.Spider):
         for item in data:
             obj=LanqiuItem()
             obj['id']=item.xpath("./gid")
-            obj['league'] = item.xpath("./league")
-            obj['team_h'] = item.xpath("./team_h")
-            obj['team_c'] = item.xpath("./team_c")
-            # obj['ratio'] = item.xpath("./ratio")
-            # obj['ior_RH'] = item.xpath("./ior_RH")
-            # obj['ior_RC'] = item.xpath("./ior_RC")
+            obj['league'] = item.xpath("./league/text()").extract_first()
+            obj['team_h'] = item.xpath("./team_h/text()").extract_first()
+            obj['team_c'] = item.xpath("./team_c/text()").extract_first()
+            obj['showtype'] = item.xpath("./gtype/text()").extract_first()
+            obj['datetime'] = item.xpath("./datetime/text()").extract_first()
             yield obj
 
 

hgg070_spider/utils/LocalToken.py

@@ -1 +1 @@
-token = {'token': 'mVUm5Y15717377705daed0aad4869', 'username': 'python', 'password': 'python888', 'token_url': 'http://stadmin.bocai108.com/getToken'}
+token = {'token': 'mVUm5Y15717377705daed0aad4869', 'username': 'python', 'password': 'python888', 'token_url': 'http://admin.5gogo.com/getToken'}

hgg070_spider/utils/helper.py

@@ -1,17 +1,3 @@
-import datetime
-import hashlib
-import json
-import platform
-# import time
-import requests
-from requests_futures.sessions import FuturesSession
-# from scrapy import Selector
-# from selenium import webdriver
-from hgg070_spider.conf.uid import UID as u_id, UID
-# from selenium.webdriver import FirefoxOptions
-
-
-fs_session = FuturesSession()
 import hashlib
 import json
 from requests_futures.sessions import FuturesSession
@@ -19,108 +5,8 @@ from .langconv import *
 from .LocalToken import token
 
 
-
 class Helper(object):
     @staticmethod
-    def changetime(params):
-        if params.endswith('p'):
-            p_time = params[-6:-1]
-            if p_time.startswith('12'):
-                us_time = params[:-1] + ":00"
-                # print(us_time)
-                # start_time = datetime.datetime.strptime(us_time, "%Y-%m-%d %H:%M:%S") + datetime.timedelta(
-                #     hours=12)
-                # match_date = start_time.strftime("%Y-%m-%d %H:%M:%S").split(' ')[0]
-                # match_time = start_time.strftime("%Y-%m-%d %H:%M:%S").split(' ')[1]
-            else:
-                params = params[:-1] + ':00'
-                us_time = datetime.datetime.strptime(params, "%Y-%m-%d %H:%M:%S") + datetime.timedelta(
-                    hours=12)
-                # start_time = us_time + datetime.timedelta(hours=12)
-                # match_date = start_time.strftime("%Y-%m-%d %H:%M:%S").split(' ')[0]
-                # match_time = start_time.strftime("%Y-%m-%d %H:%M:%S").split(' ')[1]
-                us_time = us_time.strftime("%Y-%m-%d %H:%M:%S")
-            # pass
-        else:
-            us_time = params[:-1] + ':00'
-            # start_time = datetime.datetime.strptime(params, "%Y-%m-%d %H:%M:%S") + datetime.timedelta(
-            #     hours=12)
-            # match_date = start_time.strftime("%Y-%m-%d %H:%M:%S").split(' ')[0]
-            # match_time = start_time.strftime("%Y-%m-%d %H:%M:%S").split(' ')[1]
-        return us_time
-
-    @staticmethod
-    def get(url, params):
-        return requests.get(url, data={"data": json.dumps(params), "token": u_id['token']}, timeout=30)
-
-    @staticmethod
-    def post(url, params):
-        return requests.post(url, data={"data": json.dumps(params), "token": u_id['token']}, timeout=30)
-
-    @staticmethod
-    def async_post(url, params):
-        try:
-            # print(u_id['token'])
-            data = fs_session.post(url, data={"data": json.dumps(params), "token": u_id['token']}).result()
-            # data = fs_session.post(url, data={"data": json.dumps(params), "token": "agxrI115617094865d15cbaecca9f"}, timeout=30).result()
-            if data:
-                response_data = json.loads(data.content.decode('utf-8'))
-                # print(response_data)
-                # token异常重新获取
-                if response_data.get('status') == 10032:
-                    token = Helper.get_token()
-                    sys = platform.system()
-                    if sys == 'Windows':
-                        file_path = os.path.abspath(os.path.join(os.getcwd(), "..")) + "\\conf\\uid.py"
-                    else:
-                        file_path = os.path.abspath(os.path.join(os.getcwd(), "..")) + "/hgg070_spider/conf/uid.py"
-                    if os.path.exists(file_path):
-                        fs = open(file_path, 'w+')
-                        UID['token'] = token
-                        fs.write('UID={}'.format(UID))
-                        fs.close()
-                    data = fs_session.post(url, data={"data": json.dumps(params), "token": u_id['token']}).result()
-                    # data = fs_session.post(url, data={"data": json.dumps(params), "token": u_id['token']})
-                return data.content.decode('utf-8')
-                # return data
-        except requests.exceptions.RequestException as e:
-            print(e)
-
-    @staticmethod
-    def get_zip_data(list1, list2):
-        if list1 and list2:
-            if isinstance(list1, list) and isinstance(list2, list):
-                return dict(zip(list1, list2))
-
-    @staticmethod
-    def get_token():
-        cpath = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
-        with open(cpath + '/conf/settings.json', 'r', encoding='utf8') as f:
-            data = json.load(f)
-        token_url = data['token_url']
-        username1 = data['username1']
-        password1 = data['password1']
-        r = requests.post(token_url, data={'account': username1, 'password': password1})
-        token = r.json()['data']['token']
-        return token
-
-
-
-
-    @staticmethod
-    def genearte_MD5(params):
-        # 创建md5对象
-        hl = hashlib.md5()
-        hl.update(params.encode(encoding='utf-8'))
-        # print('MD5加密前为 ：' + params)
-        # print('MD5加密后为 ：' + hl.hexdigest())
-        return hl.hexdigest()
-
-
-if __name__ == '__main__':
-    import os
-    print(os.path.abspath(os.path.join(os.getcwd(), "..")) + "\\conf\\uid.py")
-    print(os.path.abspath(__file__))
     def async_post(url, params):
         fs_session = FuturesSession()
         t_url, t_user, t_password, t_token = token['token_url'], token['username'], token['password'], token['token']