updata

hg3535/.gitignore

@@ -9,5 +9,5 @@ log/
 pipeline/__pycache__/
 /__pycache__/
 spiders/__pycache__/
-.idea/
+/.idea/
 /utils/__pycache__/

hg3535/.idea/workspace.xml

@@ -1,8 +1,16 @@
 <?xml version="1.0" encoding="UTF-8"?>
 <project version="4">
   <component name="ChangeListManager">
-    <list default="true" id="7fa42e97-ddea-4404-91db-5a9ccc162649" name="Default Changelist" comment="">
+    <list default="true" id="7fa42e97-ddea-4404-91db-5a9ccc162649" name="Default Changelist" comment="updata">
+      <change afterPath="$PROJECT_DIR$/pipeline/__init__.py" afterDir="false" />
+      <change beforePath="$PROJECT_DIR$/.gitignore" beforeDir="false" afterPath="$PROJECT_DIR$/.gitignore" afterDir="false" />
       <change beforePath="$PROJECT_DIR$/.idea/workspace.xml" beforeDir="false" afterPath="$PROJECT_DIR$/.idea/workspace.xml" afterDir="false" />
+      <change beforePath="$PROJECT_DIR$/main.py" beforeDir="false" afterPath="$PROJECT_DIR$/main.py" afterDir="false" />
+      <change beforePath="$PROJECT_DIR$/pipeline/ball_func.py" beforeDir="false" afterPath="$PROJECT_DIR$/pipeline/ball_func.py" afterDir="false" />
+      <change beforePath="$PROJECT_DIR$/pipeline/guanjun.py" beforeDir="false" afterPath="$PROJECT_DIR$/pipeline/guanjun.py" afterDir="false" />
+      <change beforePath="$PROJECT_DIR$/pipeline/lanqiu.py" beforeDir="false" afterPath="$PROJECT_DIR$/pipeline/lanqiu.py" afterDir="false" />
+      <change beforePath="$PROJECT_DIR$/settings.py" beforeDir="false" afterPath="$PROJECT_DIR$/settings.py" afterDir="false" />
+      <change beforePath="$PROJECT_DIR$/spiders/guanjun.py" beforeDir="false" afterPath="$PROJECT_DIR$/spiders/guanjun.py" afterDir="false" />
     </list>
     <option name="EXCLUDED_CONVERTED_TO_IGNORED" value="true" />
     <option name="SHOW_DIALOG" value="false" />
@@ -12,7 +20,7 @@
   </component>
   <component name="CoverageDataManager">
     <SUITE FILE_PATH="coverage/hg3535$items.coverage" NAME="items Coverage Results" MODIFIED="1554291398794" SOURCE_PROVIDER="com.intellij.coverage.DefaultCoverageFileProvider" RUNNER="coverage.py" COVERAGE_BY_TEST_ENABLED="true" COVERAGE_TRACING_ENABLED="false" WORKING_DIRECTORY="$PROJECT_DIR$" />
-    <SUITE FILE_PATH="coverage/hg3535$main.coverage" NAME="main Coverage Results" MODIFIED="1564470189116" SOURCE_PROVIDER="com.intellij.coverage.DefaultCoverageFileProvider" RUNNER="coverage.py" COVERAGE_BY_TEST_ENABLED="true" COVERAGE_TRACING_ENABLED="false" WORKING_DIRECTORY="$PROJECT_DIR$" />
+    <SUITE FILE_PATH="coverage/hg3535$main.coverage" NAME="main Coverage Results" MODIFIED="1564480710470" SOURCE_PROVIDER="com.intellij.coverage.DefaultCoverageFileProvider" RUNNER="coverage.py" COVERAGE_BY_TEST_ENABLED="true" COVERAGE_TRACING_ENABLED="false" WORKING_DIRECTORY="$PROJECT_DIR$" />
     <SUITE FILE_PATH="coverage/hg3535$hgjieshu.coverage" NAME="hgjieshu Coverage Results" MODIFIED="1558494720094" SOURCE_PROVIDER="com.intellij.coverage.DefaultCoverageFileProvider" RUNNER="coverage.py" COVERAGE_BY_TEST_ENABLED="true" COVERAGE_TRACING_ENABLED="false" WORKING_DIRECTORY="$PROJECT_DIR$/spiders" />
     <SUITE FILE_PATH="coverage/hg3535$halffull.coverage" NAME="halffull Coverage Results" MODIFIED="1554544334424" SOURCE_PROVIDER="com.intellij.coverage.DefaultCoverageFileProvider" RUNNER="coverage.py" COVERAGE_BY_TEST_ENABLED="true" COVERAGE_TRACING_ENABLED="false" WORKING_DIRECTORY="$PROJECT_DIR$/spiders" />
     <SUITE FILE_PATH="coverage/hg3535$pipelines.coverage" NAME="pipelines Coverage Results" MODIFIED="1554290866137" SOURCE_PROVIDER="com.intellij.coverage.DefaultCoverageFileProvider" RUNNER="coverage.py" COVERAGE_BY_TEST_ENABLED="true" COVERAGE_TRACING_ENABLED="false" WORKING_DIRECTORY="$PROJECT_DIR$" />
@@ -40,44 +48,59 @@
   </component>
   <component name="FileEditorManager">
     <leaf SIDE_TABS_SIZE_LIMIT_KEY="300">
-      <file pinned="false" current-in-tab="true">
-        <entry file="file://$PROJECT_DIR$/.gitignore">
+      <file pinned="false" current-in-tab="false">
+        <entry file="file://$PROJECT_DIR$/settings.py">
           <provider selected="true" editor-type-id="text-editor">
-            <state relative-caret-position="234">
-              <caret line="13" lean-forward="true" selection-start-line="13" selection-end-line="13" />
+            <state relative-caret-position="346">
+              <caret line="180" lean-forward="true" selection-start-line="180" selection-end-line="180" />
             </state>
           </provider>
         </entry>
       </file>
       <file pinned="false" current-in-tab="false">
-        <entry file="file://$PROJECT_DIR$/main.py">
+        <entry file="file://$PROJECT_DIR$/spiders/guanjun.py">
           <provider selected="true" editor-type-id="text-editor">
-            <state relative-caret-position="108">
-              <caret line="8" column="37" lean-forward="true" selection-start-line="8" selection-start-column="37" selection-end-line="8" selection-end-column="37" />
+            <state relative-caret-position="180">
+              <caret line="10" lean-forward="true" selection-start-line="10" selection-end-line="10" />
+              <folding>
+                <element signature="e#38#53#0" expanded="true" />
+              </folding>
             </state>
           </provider>
         </entry>
       </file>
       <file pinned="false" current-in-tab="false">
-        <entry file="file://$PROJECT_DIR$/utils/helper.py">
-          <provider selected="true" editor-type-id="text-editor" />
-        </entry>
-      </file>
-      <file pinned="false" current-in-tab="false">
-        <entry file="file://$PROJECT_DIR$/pipeline/zuqiu.py">
-          <provider selected="true" editor-type-id="text-editor" />
+        <entry file="file://$PROJECT_DIR$/pipeline/lanqiu.py">
+          <provider selected="true" editor-type-id="text-editor">
+            <state relative-caret-position="156">
+              <caret line="150" selection-start-line="150" selection-end-line="218" selection-end-column="21" />
+              <folding>
+                <element signature="e#49#60#0" expanded="true" />
+              </folding>
+            </state>
+          </provider>
         </entry>
       </file>
-      <file pinned="false" current-in-tab="false">
-        <entry file="file://$PROJECT_DIR$/spiders/__init__.py">
-          <provider selected="true" editor-type-id="text-editor" />
+      <file pinned="false" current-in-tab="true">
+        <entry file="file://$PROJECT_DIR$/pipeline/guanjun.py">
+          <provider selected="true" editor-type-id="text-editor">
+            <state relative-caret-position="576">
+              <caret line="32" column="57" selection-start-line="32" selection-start-column="57" selection-end-line="32" selection-end-column="57" />
+              <folding>
+                <element signature="e#18#32#0" expanded="true" />
+              </folding>
+            </state>
+          </provider>
         </entry>
       </file>
       <file pinned="false" current-in-tab="false">
-        <entry file="file://$PROJECT_DIR$/spiders/zuqiu.py">
+        <entry file="file://$PROJECT_DIR$/pipeline/ball_func.py">
           <provider selected="true" editor-type-id="text-editor">
-            <state relative-caret-position="-8370">
-              <caret line="21" selection-start-line="21" selection-end-line="22" selection-end-column="35" />
+            <state relative-caret-position="7">
+              <caret line="4" lean-forward="true" selection-start-line="4" selection-end-line="4" />
+              <folding>
+                <element signature="e#0#15#0" expanded="true" />
+              </folding>
             </state>
           </provider>
         </entry>
@@ -170,9 +193,6 @@
   <component name="IdeDocumentHistory">
     <option name="CHANGED_PATHS">
       <list>
-        <option value="$PROJECT_DIR$/spiders/temaball.py" />
-        <option value="$PROJECT_DIR$/spiders/wqguanjun.py" />
-        <option value="$PROJECT_DIR$/spiders/lqguanjun.py" />
         <option value="$PROJECT_DIR$/spiders/lanqiuls.py" />
         <option value="$PROJECT_DIR$/spiders/testone.py" />
         <option value="$PROJECT_DIR$/spiders/wgbodan.py" />
@@ -195,7 +215,6 @@
         <option value="$PROJECT_DIR$/spiders/kaishi.py" />
         <option value="$PROJECT_DIR$/kaishi.py" />
         <option value="$PROJECT_DIR$/main_notroll.py" />
-        <option value="$PROJECT_DIR$/spiders/guanjun.py" />
         <option value="$PROJECT_DIR$/text.py" />
         <option value="$PROJECT_DIR$/middlewares.py" />
         <option value="$PROJECT_DIR$/spiders/hg3535_zq_status_up.py" />
@@ -218,16 +237,20 @@
         <option value="$PROJECT_DIR$/pipelines.py" />
         <option value="$PROJECT_DIR$/spiders/roll_wangqiu.py" />
         <option value="$PROJECT_DIR$/spiders/zuqiu.py" />
-        <option value="$PROJECT_DIR$/settings.py" />
-        <option value="$PROJECT_DIR$/main.py" />
         <option value="$PROJECT_DIR$/.gitignore" />
+        <option value="$PROJECT_DIR$/main.py" />
+        <option value="$PROJECT_DIR$/settings.py" />
+        <option value="$PROJECT_DIR$/spiders/guanjun.py" />
+        <option value="$PROJECT_DIR$/pipeline/ball_func.py" />
+        <option value="$PROJECT_DIR$/pipeline/lanqiu.py" />
+        <option value="$PROJECT_DIR$/pipeline/guanjun.py" />
       </list>
     </option>
   </component>
   <component name="ProjectFrameBounds" extendedState="6">
-    <option name="x" value="617" />
-    <option name="y" value="94" />
-    <option name="width" value="1058" />
+    <option name="x" value="381" />
+    <option name="y" value="17" />
+    <option name="width" value="1238" />
     <option name="height" value="1066" />
   </component>
   <component name="ProjectLevelVcsManager" settingsEditedManually="true" />
@@ -244,6 +267,16 @@
               <item name="hg3535" type="b2602c69:ProjectViewProjectNode" />
               <item name="hg3535" type="462c0819:PsiDirectoryNode" />
             </path>
+            <path>
+              <item name="hg3535" type="b2602c69:ProjectViewProjectNode" />
+              <item name="hg3535" type="462c0819:PsiDirectoryNode" />
+              <item name="pipeline" type="462c0819:PsiDirectoryNode" />
+            </path>
+            <path>
+              <item name="hg3535" type="b2602c69:ProjectViewProjectNode" />
+              <item name="hg3535" type="462c0819:PsiDirectoryNode" />
+              <item name="spiders" type="462c0819:PsiDirectoryNode" />
+            </path>
             <path>
               <item name="hg3535" type="b2602c69:ProjectViewProjectNode" />
               <item name="External Libraries" type="cb654da1:ExternalLibrariesNode" />
@@ -268,6 +301,7 @@
       <recent name="C:\Users\admin\Desktop\sports_scrapy\hg3535\spiders" />
     </key>
     <key name="CopyFile.RECENT_KEYS">
+      <recent name="C:\Users\admin\Desktop\sports_scrapy\hg3535\pipeline" />
       <recent name="C:\Users\admin\Desktop\sports_scrapy\hg3535" />
       <recent name="C:\Users\admin\Desktop\sports_scrapy\hg3535\spiders" />
     </key>
@@ -509,7 +543,7 @@
       <workItem from="1559817639837" duration="2439000" />
       <workItem from="1559820318166" duration="6091000" />
       <workItem from="1560128279184" duration="2486000" />
-      <workItem from="1564469892436" duration="541000" />
+      <workItem from="1564469892436" duration="10885000" />
     </task>
     <task id="LOCAL-00001" summary="修改后,版本二">
       <created>1555727493584</created>
@@ -707,11 +741,18 @@
       <option name="project" value="LOCAL" />
       <updated>1564470360034</updated>
     </task>
-    <option name="localTasksCounter" value="29" />
+    <task id="LOCAL-00029" summary="updata">
+      <created>1564470453693</created>
+      <option name="number" value="00029" />
+      <option name="presentableId" value="LOCAL-00029" />
+      <option name="project" value="LOCAL" />
+      <updated>1564470453693</updated>
+    </task>
+    <option name="localTasksCounter" value="30" />
     <servers />
   </component>
   <component name="TimeTrackingManager">
-    <option name="totallyTimeSpent" value="1583200000" />
+    <option name="totallyTimeSpent" value="1593544000" />
   </component>
   <component name="TodoView">
     <todo-panel id="selected-file">
@@ -724,13 +765,14 @@
   </component>
   <component name="ToolWindowManager">
     <frame x="-8" y="-8" width="1936" height="1066" extended-state="6" />
+    <editor active="true" />
     <layout>
       <window_info active="true" content_ui="combo" id="Project" order="0" sideWeight="0.71173847" visible="true" weight="0.15618336" />
       <window_info id="Structure" order="1" sideWeight="0.2882615" side_tool="true" weight="0.1108742" />
       <window_info id="Favorites" order="2" sideWeight="0.5013405" side_tool="true" weight="0.108208954" />
       <window_info anchor="bottom" id="Message" order="0" />
       <window_info anchor="bottom" id="Find" order="1" weight="0.32832617" />
-      <window_info anchor="bottom" id="Debug" order="2" sideWeight="0.49946696" weight="0.29399142" />
+      <window_info anchor="bottom" id="Debug" order="2" sideWeight="0.49946696" weight="0.37982833" />
       <window_info anchor="bottom" id="Cvs" order="3" weight="0.25" />
       <window_info anchor="bottom" id="Inspection" order="4" weight="0.4" />
       <window_info anchor="bottom" id="TODO" order="5" weight="0.32832617" />
@@ -738,9 +780,9 @@
       <window_info anchor="bottom" id="Version Control" order="7" weight="0.1555794" />
       <window_info anchor="bottom" id="Database Changes" order="8" />
       <window_info anchor="bottom" id="Event Log" order="9" sideWeight="0.50692964" side_tool="true" weight="0.3858521" />
-      <window_info anchor="bottom" x="22" y="376" width="1876" height="298" id="Terminal" order="10" sideWeight="0.49307036" weight="0.2274678" />
+      <window_info anchor="bottom" x="22" y="376" width="1876" height="298" id="Terminal" order="10" sideWeight="0.49307036" weight="0.444206" />
       <window_info anchor="bottom" id="Python Console" order="11" sideWeight="0.49946696" weight="0.4356223" />
-      <window_info anchor="bottom" id="Run" order="12" sideWeight="0.87473345" visible="true" weight="0.58798283" />
+      <window_info anchor="bottom" id="Run" order="12" sideWeight="0.87473345" visible="true" weight="0.29613733" />
       <window_info anchor="right" id="Commander" internal_type="SLIDING" order="0" type="SLIDING" weight="0.4" />
       <window_info anchor="right" id="Ant Build" order="1" weight="0.25" />
       <window_info anchor="right" content_ui="combo" id="Hierarchy" order="2" weight="0.25" />
@@ -880,54 +922,6 @@
     </expressions>
   </component>
   <component name="editorHistoryManager">
-    <entry file="file://$USER_HOME$/AppData/Local/Programs/Python/Python36/Lib/site-packages/w3lib/util.py">
-      <provider selected="true" editor-type-id="text-editor">
-        <state relative-caret-position="184">
-          <caret line="38" selection-start-line="38" selection-end-line="38" />
-        </state>
-      </provider>
-    </entry>
-    <entry file="file://$USER_HOME$/AppData/Local/Programs/Python/Python36/Lib/urllib/parse.py">
-      <provider selected="true" editor-type-id="text-editor">
-        <state relative-caret-position="295">
-          <caret line="820" selection-start-line="820" selection-end-line="820" />
-        </state>
-      </provider>
-    </entry>
-    <entry file="file://$USER_HOME$/AppData/Local/Programs/Python/Python36/Lib/site-packages/scrapy/http/request/__init__.py">
-      <provider selected="true" editor-type-id="text-editor">
-        <state relative-caret-position="151">
-          <caret line="57" selection-start-line="57" selection-end-line="57" />
-        </state>
-      </provider>
-    </entry>
-    <entry file="file://$USER_HOME$/AppData/Local/Programs/Python/Python36/Lib/site-packages/scrapy/http/response/__init__.py">
-      <provider selected="true" editor-type-id="text-editor">
-        <state relative-caret-position="159">
-          <caret line="28" selection-start-line="28" selection-end-line="28" />
-        </state>
-      </provider>
-    </entry>
-    <entry file="file://$PROJECT_DIR$/jianting.py">
-      <provider selected="true" editor-type-id="text-editor">
-        <state>
-          <folding>
-            <element signature="e#0#9#0" expanded="true" />
-          </folding>
-        </state>
-      </provider>
-    </entry>
-    <entry file="file://$PROJECT_DIR$/log/guanjun_2019_5_23.log" />
-    <entry file="file://$PROJECT_DIR$/log/wangqiu_2019_5_23.log" />
-    <entry file="file://$PROJECT_DIR$/kaishi.py" />
-    <entry file="file://$USER_HOME$/AppData/Local/Programs/Python/Python36/Lib/site-packages/parsel/selector.py">
-      <provider selected="true" editor-type-id="text-editor">
-        <state relative-caret-position="161">
-          <caret line="237" selection-start-line="237" selection-end-line="237" />
-        </state>
-      </provider>
-    </entry>
-    <entry file="file://$PROJECT_DIR$/log/bangqiu_2019_5_24.log" />
     <entry file="file://$PROJECT_DIR$/log/liansai_2019_5_24.log" />
     <entry file="file://$PROJECT_DIR$/log/lanqiu_2019_5_24.log" />
     <entry file="file://$USER_HOME$/AppData/Local/Programs/Python/Python36/Lib/site-packages/scrapy/spiderloader.py">
@@ -937,26 +931,9 @@
         </state>
       </provider>
     </entry>
-    <entry file="file://$PROJECT_DIR$/items.py">
-      <provider selected="true" editor-type-id="text-editor">
-        <state relative-caret-position="336">
-          <caret line="489" selection-start-line="489" selection-end-line="489" />
-        </state>
-      </provider>
-    </entry>
     <entry file="file://$PROJECT_DIR$/spiders/wqbodan.py">
       <provider selected="true" editor-type-id="text-editor" />
     </entry>
-    <entry file="file://$PROJECT_DIR$/spiders/guanjun.py">
-      <provider selected="true" editor-type-id="text-editor">
-        <state relative-caret-position="606">
-          <caret line="43" column="45" selection-start-line="43" selection-start-column="45" selection-end-line="43" selection-end-column="45" />
-          <folding>
-            <element signature="e#24#35#0" expanded="true" />
-          </folding>
-        </state>
-      </provider>
-    </entry>
     <entry file="file://$PROJECT_DIR$/spiders/hg3535_zq_status_up.py" />
     <entry file="file://$PROJECT_DIR$/spiders/hg3535_zq_status.py" />
     <entry file="file://$PROJECT_DIR$/main_notroll.py" />
@@ -1060,16 +1037,6 @@
         </state>
       </provider>
     </entry>
-    <entry file="file://$PROJECT_DIR$/middlewares.py">
-      <provider selected="true" editor-type-id="text-editor">
-        <state relative-caret-position="2664">
-          <caret line="151" selection-start-line="151" selection-end-line="151" />
-          <folding>
-            <element signature="e#168#181#0" expanded="true" />
-          </folding>
-        </state>
-      </provider>
-    </entry>
     <entry file="file://$PROJECT_DIR$/spiders/bangqiu.py">
       <provider selected="true" editor-type-id="text-editor">
         <state relative-caret-position="3132">
@@ -1097,13 +1064,6 @@
         </state>
       </provider>
     </entry>
-    <entry file="file://$PROJECT_DIR$/settings.py">
-      <provider selected="true" editor-type-id="text-editor">
-        <state relative-caret-position="2736">
-          <caret line="152" column="20" selection-start-line="152" selection-start-column="20" selection-end-line="152" selection-end-column="20" />
-        </state>
-      </provider>
-    </entry>
     <entry file="file://$PROJECT_DIR$/pipelines.py">
       <provider selected="true" editor-type-id="text-editor">
         <state relative-caret-position="31158">
@@ -1114,23 +1074,6 @@
         </state>
       </provider>
     </entry>
-    <entry file="file://$PROJECT_DIR$/main_time.py">
-      <provider selected="true" editor-type-id="text-editor">
-        <state relative-caret-position="144">
-          <caret line="11" selection-start-line="11" selection-end-line="11" />
-        </state>
-      </provider>
-    </entry>
-    <entry file="file://$PROJECT_DIR$/spiders/zq_jieshu.py">
-      <provider selected="true" editor-type-id="text-editor">
-        <state relative-caret-position="450">
-          <caret line="30" column="20" selection-start-line="30" selection-start-column="20" selection-end-line="30" selection-end-column="20" />
-          <folding>
-            <element signature="e#36#51#0" expanded="true" />
-          </folding>
-        </state>
-      </provider>
-    </entry>
     <entry file="file://$PROJECT_DIR$/spiders/wq_jieshu.py">
       <provider selected="true" editor-type-id="text-editor">
         <state relative-caret-position="882">
@@ -1174,31 +1117,143 @@
     <entry file="file://$PROJECT_DIR$/spiders/__init__.py">
       <provider selected="true" editor-type-id="text-editor" />
     </entry>
+    <entry file="file://$PROJECT_DIR$/.gitignore">
+      <provider selected="true" editor-type-id="text-editor">
+        <state relative-caret-position="198">
+          <caret line="11" column="1" selection-start-line="11" selection-start-column="1" selection-end-line="11" selection-end-column="1" />
+        </state>
+      </provider>
+    </entry>
+    <entry file="file://$PROJECT_DIR$/main.py">
+      <provider selected="true" editor-type-id="text-editor">
+        <state relative-caret-position="144">
+          <caret line="10" column="39" lean-forward="true" selection-start-line="10" selection-start-column="39" selection-end-line="10" selection-end-column="39" />
+        </state>
+      </provider>
+    </entry>
+    <entry file="file://$PROJECT_DIR$/requirements.txt">
+      <provider selected="true" editor-type-id="text-editor">
+        <state relative-caret-position="-530" />
+      </provider>
+    </entry>
+    <entry file="file://$PROJECT_DIR$/roll.py">
+      <provider selected="true" editor-type-id="text-editor">
+        <state>
+          <folding>
+            <element signature="e#0#11#0" expanded="true" />
+          </folding>
+        </state>
+      </provider>
+    </entry>
+    <entry file="file://$PROJECT_DIR$/__init__.py">
+      <provider selected="true" editor-type-id="text-editor" />
+    </entry>
+    <entry file="file://$PROJECT_DIR$/items.py">
+      <provider selected="true" editor-type-id="text-editor">
+        <state relative-caret-position="1426">
+          <caret line="472" selection-start-line="472" selection-end-line="472" />
+        </state>
+      </provider>
+    </entry>
+    <entry file="file://$PROJECT_DIR$/jianting.py">
+      <provider selected="true" editor-type-id="text-editor" />
+    </entry>
+    <entry file="file://$PROJECT_DIR$/main_time.py">
+      <provider selected="true" editor-type-id="text-editor">
+        <state relative-caret-position="-86">
+          <caret line="11" selection-start-line="11" selection-end-line="11" />
+        </state>
+      </provider>
+    </entry>
     <entry file="file://$PROJECT_DIR$/ball_func.py">
       <provider selected="true" editor-type-id="text-editor">
-        <state relative-caret-position="133">
+        <state relative-caret-position="511">
           <caret line="145" selection-start-line="145" selection-end-line="145" />
         </state>
       </provider>
     </entry>
     <entry file="file://$PROJECT_DIR$/spiders/zuqiu.py">
       <provider selected="true" editor-type-id="text-editor">
-        <state relative-caret-position="-8370">
+        <state relative-caret-position="270">
           <caret line="21" selection-start-line="21" selection-end-line="22" selection-end-column="35" />
         </state>
       </provider>
     </entry>
-    <entry file="file://$PROJECT_DIR$/main.py">
+    <entry file="file://$USER_HOME$/AppData/Local/Programs/Python/Python36/Lib/site-packages/scrapy/utils/trackref.py">
       <provider selected="true" editor-type-id="text-editor">
-        <state relative-caret-position="108">
-          <caret line="8" column="37" lean-forward="true" selection-start-line="8" selection-start-column="37" selection-end-line="8" selection-end-column="37" />
+        <state relative-caret-position="486">
+          <caret line="32" selection-start-line="32" selection-end-line="32" />
         </state>
       </provider>
     </entry>
-    <entry file="file://$PROJECT_DIR$/.gitignore">
+    <entry file="file://$USER_HOME$/AppData/Local/Programs/Python/Python36/Lib/site-packages/scrapy/http/request/__init__.py">
+      <provider selected="true" editor-type-id="text-editor">
+        <state relative-caret-position="202">
+          <caret line="78" selection-start-line="78" selection-end-line="78" />
+        </state>
+      </provider>
+    </entry>
+    <entry file="file://$PROJECT_DIR$/settings.py">
+      <provider selected="true" editor-type-id="text-editor">
+        <state relative-caret-position="346">
+          <caret line="180" lean-forward="true" selection-start-line="180" selection-end-line="180" />
+        </state>
+      </provider>
+    </entry>
+    <entry file="file://$PROJECT_DIR$/pipeline/ball_func.py">
       <provider selected="true" editor-type-id="text-editor">
-        <state relative-caret-position="234">
-          <caret line="13" lean-forward="true" selection-start-line="13" selection-end-line="13" />
+        <state relative-caret-position="7">
+          <caret line="4" lean-forward="true" selection-start-line="4" selection-end-line="4" />
+          <folding>
+            <element signature="e#0#15#0" expanded="true" />
+          </folding>
+        </state>
+      </provider>
+    </entry>
+    <entry file="file://$PROJECT_DIR$/middlewares.py">
+      <provider selected="true" editor-type-id="text-editor">
+        <state relative-caret-position="290">
+          <caret line="134" selection-start-line="134" selection-end-line="134" />
+        </state>
+      </provider>
+    </entry>
+    <entry file="file://$PROJECT_DIR$/spiders/zq_jieshu.py">
+      <provider selected="true" editor-type-id="text-editor">
+        <state relative-caret-position="180">
+          <caret line="21" column="14" selection-start-line="21" selection-start-column="4" selection-end-line="21" selection-end-column="14" />
+        </state>
+      </provider>
+    </entry>
+    <entry file="file://$PROJECT_DIR$/pipeline/__init__.py">
+      <provider selected="true" editor-type-id="text-editor" />
+    </entry>
+    <entry file="file://$PROJECT_DIR$/spiders/guanjun.py">
+      <provider selected="true" editor-type-id="text-editor">
+        <state relative-caret-position="180">
+          <caret line="10" lean-forward="true" selection-start-line="10" selection-end-line="10" />
+          <folding>
+            <element signature="e#38#53#0" expanded="true" />
+          </folding>
+        </state>
+      </provider>
+    </entry>
+    <entry file="file://$PROJECT_DIR$/pipeline/lanqiu.py">
+      <provider selected="true" editor-type-id="text-editor">
+        <state relative-caret-position="156">
+          <caret line="150" selection-start-line="150" selection-end-line="218" selection-end-column="21" />
+          <folding>
+            <element signature="e#49#60#0" expanded="true" />
+          </folding>
+        </state>
+      </provider>
+    </entry>
+    <entry file="file://$PROJECT_DIR$/pipeline/guanjun.py">
+      <provider selected="true" editor-type-id="text-editor">
+        <state relative-caret-position="576">
+          <caret line="32" column="57" selection-start-line="32" selection-start-column="57" selection-end-line="32" selection-end-column="57" />
+          <folding>
+            <element signature="e#18#32#0" expanded="true" />
+          </folding>
         </state>
       </provider>
     </entry>

hg3535/main.py

@@ -5,9 +5,9 @@ from scrapy.cmdline import execute
 # print(os.path.dirname(os.path.abspath(__file__)))
 sys.path.append(os.path.dirname(os.path.abspath(__file__)))
 # execute(["scrapy", "crawl", "liansai"])
-execute(["scrapy", "crawl", "zuqiu"])
+# execute(["scrapy", "crawl", "zuqiu"])
 # execute(["scrapy", "crawl", "lanqiu"])
-# execute(["scrapy", "crawl", "guanjun"])
+execute(["scrapy", "crawl", "guanjun"])
 # execute(["scrapy", "crawl", "wangqiu"])
 # execute(["scrapy", "crawl", "wqbodan"])
 # execute(["scrapy", "crawl", "bangqiu"])

hg3535/pipeline/__init__.py

@@ -0,0 +1,4 @@
+# This package will contain the spiders of your Scrapy project
+#
+# Please refer to the documentation for information on how to create and manage
+# your spiders.

hg3535/pipeline/ball_func.py

@@ -1,26 +1,8 @@
 import datetime
-import hashlib
+# import hashlib
 import time
 
 
-# # 生成哈希索引 sole
-# def hash_func(match_id, odds_code, sort, p_id):
-#     m = hashlib.md5()
-#     a = str(match_id) + str(odds_code) + str(sort) + str(p_id)
-#     m.update(a.encode('utf-8'))
-#     c = m.hexdigest()
-#     return c
-#
-#
-# # 生成odds_only哈希
-# def r_func(match_id, odds_code, sort, p_id, odd):
-#     m = hashlib.md5()
-#     a = str(match_id) + str(odds_code) + str(sort) + str(p_id) + str(odd)
-#     m.update(a.encode('utf-8'))
-#     c = m.hexdigest()
-#     return c
-
-
 # 转换成本地时间
 def new_time(ctime):
     time1 = time.mktime(time.strptime(ctime, '%Y-%m-%d %H:%M:%S')) + 43200
@@ -70,409 +52,3 @@ def get_pcode(corner_ball, code):
     else:
         return code, code_dict[code]
 
-
-# ----------------------------------------------------------------------------------------------------------------------
-# def new_times(ctime):
-#     time1 = time.mktime(time.strptime(ctime, '%Y-%m-%d %H:%M:%S')) + 43200
-#     time2 = time.localtime(time1)
-#     time3 = time.strftime('%Y-%m-%d %H:%M:%S', time2)
-#     data_time = str(time3).split(" ")
-#     match_date = data_time[0]
-#     match_time = data_time[1]
-#     return match_date,match_time
-
-
-# def news_times(ctime):
-#     time1 = time.mktime(time.strptime(ctime, '%Y-%m-%d %H:%M:%S')) + 43200
-#     time2 = time.localtime(time1)
-#     time3 = time.strftime('%Y-%m-%d %H:%M:%S', time2)
-#     time4 = time3
-#     data_time = str(time4).split(" ")
-#     match_date = data_time[0]
-#     match_time = data_time[1]
-#     return match_date, match_time, time3
-
-
-# def one_post(data, match_id, odds_code, p_id, league_id, p_code, condition, game_code, zq_odds):
-#     if data:
-#         new_hash = hash_func(match_id=match_id, odds_code=odds_code, sort=0, p_id=p_id)
-#         r_hash = r_func(match_id=match_id, odds_code=odds_code, sort=0, p_id=p_id, odd=data)
-#         payload = {
-#             "game_code": game_code,
-#             "title": "odds",
-#             "data": {
-#                 "match_id": match_id,
-#                 "lg_id": league_id,
-#                 "odds_code": odds_code,
-#                 "status": 0,
-#                 "sort": 0,
-#                 "p_code": p_code,
-#                 "odds": data,
-#                 "condition": condition,
-#                 "odds_only": r_hash,
-#                 "sole": new_hash,
-#                 "source": "hg3535",
-#                 "type": 0,
-#                 "team": ""
-#             }
-#         }
-#         zq_odds.append(payload)
-#         # return payload
-#
-#
-# def two_post(data, match_id, odds_code, p_id, league_id, p_code, condition, game_code, zq_odds):
-#     if data:
-#         # odds_list = []
-#         for index, value in enumerate(data):
-#             if value:
-#                 s_hash = hash_func(match_id=match_id, odds_code=odds_code, sort=index, p_id=p_id)
-#                 o_hash = r_func(match_id=match_id, odds_code=odds_code, sort=index, p_id=p_id, odd=value)
-#                 f1 = fuhao(str(condition[index]))
-#                 payload = {
-#                     "game_code": game_code,
-#                     "title": "odds",
-#                     "data": {
-#                         "match_id": match_id,
-#                         "lg_id": league_id,
-#                         "odds_code": odds_code,
-#                         "status": 0,
-#                         "sort": index,
-#                         "p_code": p_code,
-#                         "odds": value,
-#                         "condition": f1,
-#                         "odds_only": o_hash,
-#                         "sole": s_hash,
-#                         "source": "hg3535",
-#                         "type": 0,
-#                         "team": ""
-#                     }
-#                 }
-#                 zq_odds.append(payload)
-#         #         odds_list.append(payload)
-#         # return odds_list
-#
-#
-# def three_post(data, match_id, p_id, league_id, p_code, condition, game_code, zq_odds):
-#     if data:
-#         # odd_list = []
-#         for key, value in data.items():
-#             if value:
-#                 s_hash = hash_func(match_id=match_id, odds_code=key, sort=0, p_id=p_id)
-#                 o_hash = r_func(match_id=match_id, odds_code=key, sort=0, p_id=p_id, odd=value)
-#                 try:
-#                     f1 = fuhao(str(condition[key]))
-#                 except:
-#                     f1 = ""
-#                 payload = {
-#                     "game_code": game_code,
-#                     "title": "odds",
-#                     "data": {
-#                         "match_id": match_id,
-#                         "lg_id": league_id,
-#                         "odds_code": key,
-#                         "status": 0,
-#                         "sort": 0,
-#                         "p_code": p_code,
-#                         "odds": value,
-#                         "condition": f1,
-#                         "odds_only": o_hash,
-#                         "sole": s_hash,
-#                         "source": "hg3535",
-#                         "type": 0,
-#                         "team": ""
-#                     }
-#                 }
-#                 zq_odds.append(payload)
-#         #         odd_list.append(payload)
-#         # return odd_list
-#
-#
-# def gs_post(ball, league_id, league_name, data_game):
-#     if ball == "足球":
-#         n_gameid = int('1' + str(league_id))
-#         payload = {
-#             "game_code": "zq",
-#             "title": "competition",
-#             "data": {
-#                 "home_team": league_name,
-#                 "guest_team": "",
-#                 "lg_id": league_id,
-#                 "status": 0,
-#                 "match_id": n_gameid,
-#                 "source": "hg3535",
-#                 "us_time": data_game,
-#                 "match_date": "",
-#                 "match_time": "",
-#                 "tag": "",
-#                 "is_rollball": 0,
-#                 "is_today": 0,
-#                 "is_morningplate": 0,
-#                 "is_stringscene": 0,
-#                 }
-#         }
-#     elif ball == "篮球":
-#         n_gameid = int('1' + str(league_id))
-#         payload = {
-#             "game_code": "zq",
-#             "title": "competition",
-#             "data": {
-#                 "home_team": league_name,
-#                 "guest_team": "",
-#                 "lg_id": league_id,
-#                 "status": 0,
-#                 "match_id": n_gameid,
-#                 "source": "hg3535",
-#                 "us_time": data_game,
-#                 "match_date": "",
-#                 "match_time": "",
-#                 "tag": "",
-#                 "is_rollball": 0,
-#                 "is_today": 0,
-#                 "is_morningplate": 0,
-#                 "is_stringscene": 0,
-#             }
-#         }
-#     elif ball == "网球":
-#         n_gameid = int('1' + str(league_id))
-#         payload = {
-#             "game_code": "zq",
-#             "title": "competition",
-#             "data": {
-#                 "home_team": league_name,
-#                 "guest_team": "",
-#                 "lg_id": league_id,
-#                 "status": 0,
-#                 "match_id": n_gameid,
-#                 "source": "hg3535",
-#                 "us_time": data_game,
-#                 "match_date": "",
-#                 "match_time": "",
-#                 "tag": "",
-#                 "is_rollball": 0,
-#                 "is_today": 0,
-#                 "is_morningplate": 0,
-#                 "is_stringscene": 0,
-#             }
-#         }
-#     elif ball == "棒球":
-#         n_gameid = int('1' + str(league_id))
-#         payload = {
-#             "game_code": "zq",
-#             "title": "competition",
-#             "data": {
-#                 "home_team": league_name,
-#                 "guest_team": "",
-#                 "lg_id": league_id,
-#                 "status": 0,
-#                 "match_id": n_gameid,
-#                 "source": "hg3535",
-#                 "us_time": data_game,
-#                 "match_date": "",
-#                 "match_time": "",
-#                 "tag": "",
-#                 "is_rollball": 0,
-#                 "is_today": 0,
-#                 "is_morningplate": 0,
-#                 "is_stringscene": 0,
-#             }
-#         }
-#     return payload
-#
-#
-# def go_post(ball, league_id, new_champion, new_league_name, value, r_hash, new_hash, tema_home):
-#     game_id = int('1' + str(league_id))
-#     if ball == "足球":
-#         payload = {
-#             "game_code": "zq",
-#             "title": "odds",
-#             "data": {
-#                 "match_id": league_id,
-#                 "lg_id": game_id,
-#                 "odds_code": new_champion,
-#                 "status": 0,
-#                 "sort": 0,
-#                 "p_code": new_league_name,
-#                 "odds": value,
-#                 "condition": "",
-#                 "odds_only": r_hash,
-#                 "sole": new_hash,
-#                 "source": 'hg3535',
-#                 "type": 1,
-#                 "team": tema_home
-#             }
-#         }
-#     elif ball == "篮球":
-#         payload = {
-#             "game_code": "lq",
-#             "title": "odds",
-#             "data": {
-#                 "match_id": league_id,
-#                 "lg_id": game_id,
-#                 "odds_code": new_champion,
-#                 "status": 0,
-#                 "sort": 0,
-#                 "p_code": new_league_name,
-#                 "odds": value,
-#                 "condition": "",
-#                 "odds_only": r_hash,
-#                 "sole": new_hash,
-#                 "source": "hg3535",
-#                 "type": 1,
-#                 "team": tema_home
-#             }
-#         }
-#     elif ball == "网球":
-#         payload = {
-#             "game_code": "wq",
-#             "title": "odds",
-#             "data": {
-#                 "match_id": league_id,
-#                 "lg_id": game_id,
-#                 "odds_code": new_champion,
-#                 "status": 0,
-#                 "sort": 0,
-#                 "p_code": new_league_name,
-#                 "odds": value,
-#                 "condition": "",
-#                 "odds_only": r_hash,
-#                 "sole": new_hash,
-#                 "source": "hg3535",
-#                 "type": 1,
-#                 "team": tema_home
-#             }
-#         }
-#     elif ball == "棒球":
-#         payload = {
-#             "game_code": "bq",
-#             "title": "odds",
-#             "data": {
-#                 "match_id": league_id,
-#                 "lg_id": game_id,
-#                 "odds_code": new_champion,
-#                 "status": 0,
-#                 "sort": 0,
-#                 "p_code": new_league_name,
-#                 "odds": value,
-#                 "condition": "",
-#                 "odds_only": r_hash,
-#                 "sole": new_hash,
-#                 "source": "hg3535",
-#                 "type": 1,
-#                 "team": tema_home
-#             }
-#         }
-#     return payload
-#
-#
-# def guanl_post(ball, league_name, league_id, time3):
-#     if ball == "足球":
-#         payload = {
-#             "game_code": "zq",
-#             "title": "league",
-#             "data": {
-#                 "name_chinese": league_name,
-#                 "kind": "1",
-#                 "match_mode": "1",
-#                 "if_stop": "0",
-#                 # "belong": "",
-#                 "last_time": time3,
-#                 "lg_id": league_id,
-#                 "source": "hg3535"
-#                 }
-#             }
-#     elif ball == "篮球":
-#         payload = {
-#             "game_code": "zq",
-#             "title": "league",
-#             "data": {
-#                 "name_chinese": league_name,
-#                 "kind": "1",
-#                 "match_mode": "1",
-#                 "if_stop": "0",
-#                 # "belong": "",
-#                 "last_time": time3,
-#                 "lg_id": league_id,
-#                 "source": "hg3535"
-#             }
-#         }
-#     elif ball == "网球":
-#         payload = {
-#             "game_code": "zq",
-#             "title": "league",
-#             "data": {
-#                 "name_chinese": league_name,
-#                 "kind": "1",
-#                 "match_mode": "1",
-#                 "if_stop": "0",
-#                 # "belong": "",
-#                 "last_time": time3,
-#                 "lg_id": league_id,
-#                 "source": "hg3535"
-#             }
-#         }
-#     elif ball == "棒球":
-#         payload = {
-#             "game_code": "zq",
-#             "title": "league",
-#             "data": {
-#                 "name_chinese": league_name,
-#                 "kind": "1",
-#                 "match_mode": "1",
-#                 "if_stop": "0",
-#                 # "belong": "",
-#                 "last_time": time3,
-#                 "lg_id": league_id,
-#                 "source": "hg3535"
-#             }
-#         }
-#     return payload
-#
-#
-# def liansai(area_name, name_chinese, st_league, area_dict, game_code, qiu):
-#     if area_name in area_dict:
-#         payload = {
-#             "game_code": game_code,
-#             "title": "league",
-#             "data": {
-#                 "name_chinese": name_chinese,
-#                 "kind": "1",
-#                 "match_mode": "1",
-#                 "if_stop": "0",
-#                 "belong": area_name,
-#                 "last_time": '2019-12-31 23:59:59',
-#                 "lg_id": st_league,
-#                 "source": "hg3535"
-#             }
-#         }
-#     else:
-#         if area_name == qiu:
-#             payload = {
-#                 "game_code": game_code,
-#                 "title": "league",
-#                 "data": {
-#                     "name_chinese": name_chinese,
-#                     "kind": "1",
-#                     "match_mode": "1",
-#                     "if_stop": "0",
-#                     "belong": " ",
-#                     "last_time": '2019-12-31 23:59:59',
-#                     "lg_id": st_league,
-#                     "source": "hg3535"
-#                 }
-#             }
-#         else:
-#             payload = {
-#                 "game_code": game_code,
-#                 "title": "league",
-#                 "data": {
-#                     "name_chinese": name_chinese,
-#                     "kind": "1",
-#                     "match_mode": "1",
-#                     "if_stop": "0",
-#                     "belong": area_name,
-#                     "last_time": '2019-12-31 23:59:59',
-#                     "lg_id": st_league,
-#                     "source": "hg3535"
-#                 }}
-#     return payload
-

hg3535/pipeline/guanjun.py

@@ -11,14 +11,13 @@ class Guanjunpipeline(object):
         self.db = self.mongo['kaiyou']
 
     def process_item(self, item, spider):
-        gj_odds = []
+        # gj_odds = []
         url = 'http://admin.5gogo.com/'
         logger = logging.getLogger(__name__)
         data_game = item['data_game'].replace('年', "-").replace('月', "-").replace('日', "")
         time_game = item['time_game']
         ctime = data_game + " " + time_game + ":00"
         uptime = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime())
-        # expire_time = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(time.time() + 60))
         league_name = item['league_name']
         tema_home = item['tema_home']
         league_id = item['league_id']
@@ -32,7 +31,6 @@ class Guanjunpipeline(object):
         time1 = time.mktime(time.strptime(ctime, '%Y-%m-%d %H:%M:%S')) + 43200
         time2 = time.localtime(time1)
         time3 = time.strftime('%Y-%m-%d %H:%M:%S', time2)
-        # pt = item['pt']
         ball = item['ball']
         if ball == '足球':
             game_code = 'zq'

hg3535/pipeline/lanqiu.py

@@ -1,11 +1,11 @@
-import datetime
-import json
-import pymongo
+# import datetime
+# import json
+# import pymongo
 import time
 import logging
-import requests
+# import requests
 
-from ball_func import fuhao
+# from ball_func import fuhao
 from ..utils.helper import Helper
 from .ball_func import get_pcode, new_time, out_time
 

hg3535/settings.py

@@ -77,6 +77,7 @@ DOWNLOADER_MIDDLEWARES = {
 # Configure item pipeline
 # See https://doc.scrapy.org/en/latest/topics/item-pipeline.html
 ITEM_PIPELINES = {
+   'scrapy_redis.pipelines.RedisPipeline': 400,
    # # 篮球
    # 'hg3535.pipeline.Lanqiupipeline': 300,
    # # 冠军
@@ -123,7 +124,7 @@ DOWNLOAD_TIMEOUT = 1
 REDIRECT_ENABLED = False
 
 
-SCHEDULER_PERSIST = False  # 是否在关闭时候保留原来的调度器和去重记录，True=保留，False=清空
+# SCHEDULER_PERSIST = False  # 是否在关闭时候保留原来的调度器和去重记录，True=保留，False=清空
 SCHEDULER_FLUSH_ON_START = False
 TELNETCONSOLE_PORT = None
 # TELNETCONSOLE_ENABLED=False
@@ -164,12 +165,27 @@ TELNETCONSOLE_PORT = None
 # POST_PORT = '10432'
 # POST_PASSWORD = '123456'
 
-# R_HOST = 'localhost'
-# R_POST = '6379'
-# R_PASSWORD = '123456'
+R_HOST = '192.168.2.200'
+R_POST = '6379'
+R_PASSWORD = '123456'
 #
 # POST_HOST = 'localhost'
 # POST_DATABASE = 'kaiyou'
 # POST_USER = 'kaiyou'
 # POST_PORT = '10432'
 # POST_PASSWORD = '123456'
+SCHEDULER = "scrapy_redis.scheduler.Scheduler"
+DUPEFILTER_CLASS = "scrapy_redis.dupefilter.RFPDupeFilter"
+
+# SCHEDULER_SERIALIZER = "scrapy_redis.picklecompat"
+
+# ＃不要清理redis队列，允许暂停/恢复抓取。
+
+# SCHEDULER_PERSIST = False
+
+# ＃使用优先级队列安排请求。（默认）
+SCHEDULER_QUEUE_CLASS = 'scrapy_redis.queue.PriorityQueue'
+REDIS_HOST = '192.168.2.200'
+REDIS_PORT = 6379
+REDIS_PARAMS = {'password': 123456, 'db': 1}
+#

hg3535/spiders/guanjun.py

@@ -3,16 +3,18 @@
 import datetime
 import json
 
-import scrapy
+# import scrapy
 from scrapy.http import Request
 
 from .. items import Guanjun
+from scrapy_redis.spiders import RedisSpider
 
 
-class LqguanjunSpider(scrapy.Spider):
+class LqguanjunSpider(RedisSpider):
     name = 'guanjun'
     allowed_domains = ['hg3535z.com']
     to_day = datetime.datetime.now()
+    redis_key = "guanjun: start_urls"
     custom_settings = {
         "ITEM_PIPELINES": {
             'hg3535.pipeline.guanjun.Guanjunpipeline': 300,
@@ -26,6 +28,7 @@ class LqguanjunSpider(scrapy.Spider):
             for x in range(0, 4):
                 url = 'https://odata.jiushan6688.com/odds6i/d/getodds/zh-cn/sid/' + str(z) + '/pt/1/ubt/or/pn/' + str(
                     x) + '/sb/2/dc/null/pid/0'
+                # redis_key = "guanjun: start_urls"
                 yield Request(url=url, callback=self.parse, meta={'pt': 1, 'sid': z}, dont_filter=True)
 
     def parse(self, response):