updata

hg3535/.idea/workspace.xml

@@ -2,15 +2,14 @@
 <project version="4">
   <component name="ChangeListManager">
     <list default="true" id="7fa42e97-ddea-4404-91db-5a9ccc162649" name="Default Changelist" comment="">
+      <change afterPath="$PROJECT_DIR$/pipeline/saiguo.py" afterDir="false" />
+      <change afterPath="$PROJECT_DIR$/spiders/saiguo.py" afterDir="false" />
       <change beforePath="$PROJECT_DIR$/.idea/workspace.xml" beforeDir="false" afterPath="$PROJECT_DIR$/.idea/workspace.xml" afterDir="false" />
-      <change beforePath="$PROJECT_DIR$/pipeline/bangqiu.py" beforeDir="false" afterPath="$PROJECT_DIR$/pipeline/bangqiu.py" afterDir="false" />
-      <change beforePath="$PROJECT_DIR$/pipeline/lanqiu.py" beforeDir="false" afterPath="$PROJECT_DIR$/pipeline/lanqiu.py" afterDir="false" />
+      <change beforePath="$PROJECT_DIR$/items.py" beforeDir="false" afterPath="$PROJECT_DIR$/items.py" afterDir="false" />
+      <change beforePath="$PROJECT_DIR$/main.py" beforeDir="false" afterPath="$PROJECT_DIR$/main.py" afterDir="false" />
       <change beforePath="$PROJECT_DIR$/pipeline/roll_bangqiu.py" beforeDir="false" afterPath="$PROJECT_DIR$/pipeline/roll_bangqiu.py" afterDir="false" />
-      <change beforePath="$PROJECT_DIR$/pipeline/roll_lanqiu.py" beforeDir="false" afterPath="$PROJECT_DIR$/pipeline/roll_lanqiu.py" afterDir="false" />
-      <change beforePath="$PROJECT_DIR$/pipeline/roll_wangqiu.py" beforeDir="false" afterPath="$PROJECT_DIR$/pipeline/roll_wangqiu.py" afterDir="false" />
-      <change beforePath="$PROJECT_DIR$/pipeline/roll_zuqiu.py" beforeDir="false" afterPath="$PROJECT_DIR$/pipeline/roll_zuqiu.py" afterDir="false" />
-      <change beforePath="$PROJECT_DIR$/pipeline/wangqiu.py" beforeDir="false" afterPath="$PROJECT_DIR$/pipeline/wangqiu.py" afterDir="false" />
       <change beforePath="$PROJECT_DIR$/pipeline/zuqiu.py" beforeDir="false" afterPath="$PROJECT_DIR$/pipeline/zuqiu.py" afterDir="false" />
+      <change beforePath="$PROJECT_DIR$/settings.py" beforeDir="false" afterPath="$PROJECT_DIR$/settings.py" afterDir="false" />
     </list>
     <option name="EXCLUDED_CONVERTED_TO_IGNORED" value="true" />
     <option name="SHOW_DIALOG" value="false" />
@@ -20,7 +19,7 @@
   </component>
   <component name="CoverageDataManager">
     <SUITE FILE_PATH="coverage/hg3535$items.coverage" NAME="items Coverage Results" MODIFIED="1554291398794" SOURCE_PROVIDER="com.intellij.coverage.DefaultCoverageFileProvider" RUNNER="coverage.py" COVERAGE_BY_TEST_ENABLED="true" COVERAGE_TRACING_ENABLED="false" WORKING_DIRECTORY="$PROJECT_DIR$" />
-    <SUITE FILE_PATH="coverage/hg3535$main.coverage" NAME="main Coverage Results" MODIFIED="1567238781807" SOURCE_PROVIDER="com.intellij.coverage.DefaultCoverageFileProvider" RUNNER="coverage.py" COVERAGE_BY_TEST_ENABLED="true" COVERAGE_TRACING_ENABLED="false" WORKING_DIRECTORY="$PROJECT_DIR$" />
+    <SUITE FILE_PATH="coverage/hg3535$main.coverage" NAME="main Coverage Results" MODIFIED="1567407610967" SOURCE_PROVIDER="com.intellij.coverage.DefaultCoverageFileProvider" RUNNER="coverage.py" COVERAGE_BY_TEST_ENABLED="true" COVERAGE_TRACING_ENABLED="false" WORKING_DIRECTORY="$PROJECT_DIR$" />
     <SUITE FILE_PATH="coverage/hg3535$hgjieshu.coverage" NAME="hgjieshu Coverage Results" MODIFIED="1558494720094" SOURCE_PROVIDER="com.intellij.coverage.DefaultCoverageFileProvider" RUNNER="coverage.py" COVERAGE_BY_TEST_ENABLED="true" COVERAGE_TRACING_ENABLED="false" WORKING_DIRECTORY="$PROJECT_DIR$/spiders" />
     <SUITE FILE_PATH="coverage/hg3535$halffull.coverage" NAME="halffull Coverage Results" MODIFIED="1554544334424" SOURCE_PROVIDER="com.intellij.coverage.DefaultCoverageFileProvider" RUNNER="coverage.py" COVERAGE_BY_TEST_ENABLED="true" COVERAGE_TRACING_ENABLED="false" WORKING_DIRECTORY="$PROJECT_DIR$/spiders" />
     <SUITE FILE_PATH="coverage/hg3535$pipelines.coverage" NAME="pipelines Coverage Results" MODIFIED="1554290866137" SOURCE_PROVIDER="com.intellij.coverage.DefaultCoverageFileProvider" RUNNER="coverage.py" COVERAGE_BY_TEST_ENABLED="true" COVERAGE_TRACING_ENABLED="false" WORKING_DIRECTORY="$PROJECT_DIR$" />
@@ -51,8 +50,8 @@
       <file pinned="false" current-in-tab="false">
         <entry file="file://$PROJECT_DIR$/main.py">
           <provider selected="true" editor-type-id="text-editor">
-            <state relative-caret-position="180">
-              <caret line="9" column="38" lean-forward="true" selection-start-line="9" selection-start-column="38" selection-end-line="9" selection-end-column="38" />
+            <state relative-caret-position="160">
+              <caret line="8" column="37" lean-forward="true" selection-start-line="8" selection-start-column="37" selection-end-line="8" selection-end-column="37" />
               <folding>
                 <element signature="e#0#10#0" expanded="true" />
               </folding>
@@ -61,10 +60,10 @@
         </entry>
       </file>
       <file pinned="false" current-in-tab="true">
-        <entry file="file://$PROJECT_DIR$/pipeline/lanqiu.py">
+        <entry file="file://$PROJECT_DIR$/spiders/saiguo.py">
           <provider selected="true" editor-type-id="text-editor">
-            <state relative-caret-position="1157">
-              <caret line="157" column="17" lean-forward="true" selection-start-line="157" selection-start-column="17" selection-end-line="157" selection-end-column="17" />
+            <state relative-caret-position="112">
+              <caret line="26" column="34" lean-forward="true" selection-start-line="26" selection-start-column="34" selection-end-line="26" selection-end-column="34" />
               <folding>
                 <element signature="e#0#15#0" expanded="true" />
               </folding>
@@ -73,22 +72,31 @@
         </entry>
       </file>
       <file pinned="false" current-in-tab="false">
-        <entry file="file://$PROJECT_DIR$/pipeline/roll_lanqiu.py">
+        <entry file="file://$PROJECT_DIR$/pipeline/saiguo.py">
           <provider selected="true" editor-type-id="text-editor">
-            <state relative-caret-position="897">
-              <caret line="295" column="16" lean-forward="true" selection-start-line="295" selection-start-column="16" selection-end-line="295" selection-end-column="16" />
+            <state relative-caret-position="940">
+              <caret line="47" column="19" lean-forward="true" selection-start-line="47" selection-start-column="19" selection-end-line="47" selection-end-column="19" />
               <folding>
-                <element signature="e#0#15#0" expanded="true" />
+                <element signature="e#18#33#0" expanded="true" />
               </folding>
             </state>
           </provider>
         </entry>
       </file>
       <file pinned="false" current-in-tab="false">
-        <entry file="file://$PROJECT_DIR$/pipeline/roll_bangqiu.py">
+        <entry file="file://$PROJECT_DIR$/middlewares.py">
+          <provider selected="true" editor-type-id="text-editor">
+            <state relative-caret-position="799">
+              <caret line="152" selection-start-line="152" selection-end-line="152" />
+            </state>
+          </provider>
+        </entry>
+      </file>
+      <file pinned="false" current-in-tab="false">
+        <entry file="file://$PROJECT_DIR$/pipeline/zuqiu.py">
           <provider selected="true" editor-type-id="text-editor">
-            <state relative-caret-position="-2383">
-              <caret line="110" column="32" lean-forward="true" selection-start-line="109" selection-end-line="113" selection-end-column="41" />
+            <state relative-caret-position="396">
+              <caret line="22" column="4" selection-start-line="22" selection-start-column="4" selection-end-line="22" selection-end-column="4" />
               <folding>
                 <element signature="e#0#15#0" expanded="true" />
               </folding>
@@ -97,17 +105,62 @@
         </entry>
       </file>
       <file pinned="false" current-in-tab="false">
-        <entry file="file://$PROJECT_DIR$/pipeline/bangqiu.py">
+        <entry file="file://$PROJECT_DIR$/utils/helper.py">
           <provider selected="true" editor-type-id="text-editor">
-            <state relative-caret-position="277">
-              <caret line="14" lean-forward="true" selection-start-line="14" selection-end-line="14" />
+            <state relative-caret-position="200">
+              <caret line="13" selection-start-line="13" selection-end-line="13" />
               <folding>
-                <element signature="e#0#15#0" expanded="true" />
+                <element signature="e#0#14#0" expanded="true" />
+              </folding>
+            </state>
+          </provider>
+        </entry>
+      </file>
+      <file pinned="false" current-in-tab="false">
+        <entry file="file://$PROJECT_DIR$/spiders/zuqiu.py">
+          <provider selected="true" editor-type-id="text-editor">
+            <state relative-caret-position="-5140">
+              <caret line="40" column="21" lean-forward="true" selection-start-line="40" selection-start-column="14" selection-end-line="40" selection-end-column="70" />
+              <folding>
+                <element signature="e#24#35#0" expanded="true" />
+              </folding>
+            </state>
+          </provider>
+        </entry>
+      </file>
+      <file pinned="false" current-in-tab="false">
+        <entry file="file://$PROJECT_DIR$/spiders/lanqiu.py">
+          <provider selected="true" editor-type-id="text-editor">
+            <state relative-caret-position="302">
+              <caret line="56" column="39" lean-forward="true" selection-start-line="55" selection-end-line="59" selection-end-column="96" />
+              <folding>
+                <element signature="e#24#35#0" expanded="true" />
               </folding>
             </state>
           </provider>
         </entry>
       </file>
+      <file pinned="false" current-in-tab="false">
+        <entry file="file://$PROJECT_DIR$/pipeline/roll_zuqiu.py">
+          <provider selected="true" editor-type-id="text-editor">
+            <state relative-caret-position="295">
+              <caret line="97" column="31" selection-start-line="97" selection-start-column="31" selection-end-line="97" selection-end-column="31" />
+              <folding>
+                <element signature="e#18#33#0" expanded="true" />
+              </folding>
+            </state>
+          </provider>
+        </entry>
+      </file>
+      <file pinned="false" current-in-tab="false">
+        <entry file="file://$PROJECT_DIR$/settings.py">
+          <provider selected="true" editor-type-id="text-editor">
+            <state relative-caret-position="160">
+              <caret line="152" column="49" selection-start-line="152" selection-start-column="49" selection-end-line="152" selection-end-column="49" />
+            </state>
+          </provider>
+        </entry>
+      </file>
     </leaf>
   </component>
   <component name="FileTemplateManagerImpl">
@@ -202,8 +255,6 @@
   <component name="IdeDocumentHistory">
     <option name="CHANGED_PATHS">
       <list>
-        <option value="$PROJECT_DIR$/log/lanqiu.log" />
-        <option value="$PROJECT_DIR$/log/liansai.log" />
         <option value="$PROJECT_DIR$/log/roll_zuqiu.log" />
         <option value="$PROJECT_DIR$/log/wangqiu.log" />
         <option value="$PROJECT_DIR$/log/wqbodan.log" />
@@ -238,28 +289,29 @@
         <option value="$PROJECT_DIR$/spiders/roll_zuqiu.py" />
         <option value="$PROJECT_DIR$/spiders/roll_bangqiu.py" />
         <option value="$PROJECT_DIR$/utils/helper.py" />
-        <option value="$PROJECT_DIR$/settings.py" />
         <option value="$PROJECT_DIR$/pipeline/ball_func.py" />
         <option value="$PROJECT_DIR$/spiders/roll_wangqiu.py" />
         <option value="$PROJECT_DIR$/spiders/zuqiu.py" />
-        <option value="$PROJECT_DIR$/items.py" />
         <option value="$PROJECT_DIR$/spiders/roll_lanqiu.py" />
-        <option value="$PROJECT_DIR$/main.py" />
-        <option value="$PROJECT_DIR$/pipeline/zuqiu.py" />
         <option value="$PROJECT_DIR$/pipeline/roll_zuqiu.py" />
         <option value="$PROJECT_DIR$/pipeline/roll_wangqiu.py" />
         <option value="$PROJECT_DIR$/pipeline/wangqiu.py" />
         <option value="$PROJECT_DIR$/pipeline/lanqiu.py" />
         <option value="$PROJECT_DIR$/pipeline/roll_lanqiu.py" />
-        <option value="$PROJECT_DIR$/pipeline/roll_bangqiu.py" />
         <option value="$PROJECT_DIR$/pipeline/bangqiu.py" />
+        <option value="$PROJECT_DIR$/pipeline/roll_bangqiu.py" />
+        <option value="$PROJECT_DIR$/items.py" />
+        <option value="$PROJECT_DIR$/pipeline/saiguo.py" />
+        <option value="$PROJECT_DIR$/settings.py" />
+        <option value="$PROJECT_DIR$/main.py" />
+        <option value="$PROJECT_DIR$/pipeline/zuqiu.py" />
+        <option value="$PROJECT_DIR$/spiders/saiguo.py" />
       </list>
     </option>
   </component>
   <component name="ProjectFrameBounds" extendedState="6">
-    <option name="x" value="129" />
-    <option name="y" value="17" />
-    <option name="width" value="1336" />
+    <option name="x" value="-397" />
+    <option name="width" value="1458" />
     <option name="height" value="1017" />
   </component>
   <component name="ProjectLevelVcsManager" settingsEditedManually="true" />
@@ -564,7 +616,7 @@
       <workItem from="1566976369645" duration="4000" />
       <workItem from="1566976385269" duration="13184000" />
       <workItem from="1566990066292" duration="35762000" />
-      <workItem from="1567126891838" duration="60887000" />
+      <workItem from="1567126891838" duration="75798000" />
     </task>
     <task id="LOCAL-00001" summary="修改后,版本二">
       <created>1555727493584</created>
@@ -818,11 +870,18 @@
       <option name="project" value="LOCAL" />
       <updated>1567239137632</updated>
     </task>
-    <option name="localTasksCounter" value="37" />
+    <task id="LOCAL-00037" summary="updata">
+      <created>1567246291969</created>
+      <option name="number" value="00037" />
+      <option name="presentableId" value="LOCAL-00037" />
+      <option name="project" value="LOCAL" />
+      <updated>1567246291969</updated>
+    </task>
+    <option name="localTasksCounter" value="38" />
     <servers />
   </component>
   <component name="TimeTrackingManager">
-    <option name="totallyTimeSpent" value="1739561000" />
+    <option name="totallyTimeSpent" value="1754472000" />
   </component>
   <component name="TodoView">
     <todo-panel id="selected-file">
@@ -835,14 +894,13 @@
   </component>
   <component name="ToolWindowManager">
     <frame x="-8" y="-8" width="1936" height="1066" extended-state="6" />
-    <editor active="true" />
     <layout>
-      <window_info active="true" content_ui="combo" id="Project" order="0" sideWeight="0.71173847" visible="true" weight="0.1705757" />
+      <window_info content_ui="combo" id="Project" order="0" sideWeight="0.71173847" visible="true" weight="0.1705757" />
       <window_info id="Structure" order="1" sideWeight="0.2882615" side_tool="true" weight="0.1108742" />
       <window_info id="Favorites" order="2" sideWeight="0.5013405" side_tool="true" weight="0.108208954" />
       <window_info anchor="bottom" id="Message" order="0" />
       <window_info anchor="bottom" id="Find" order="1" weight="0.32832617" />
-      <window_info anchor="bottom" id="Debug" order="2" sideWeight="0.49946696" weight="0.32403433" />
+      <window_info anchor="bottom" id="Debug" order="2" sideWeight="0.49946696" weight="0.36802575" />
       <window_info anchor="bottom" id="Cvs" order="3" weight="0.25" />
       <window_info anchor="bottom" id="Inspection" order="4" weight="0.4" />
       <window_info anchor="bottom" id="TODO" order="5" weight="0.32832617" />
@@ -852,7 +910,7 @@
       <window_info anchor="bottom" id="Event Log" order="9" sideWeight="0.50692964" side_tool="true" weight="0.38519314" />
       <window_info anchor="bottom" x="22" y="376" width="1876" height="298" id="Terminal" order="10" sideWeight="0.49307036" weight="0.3530043" />
       <window_info anchor="bottom" id="Python Console" order="11" sideWeight="0.49946696" weight="0.4356223" />
-      <window_info anchor="bottom" id="Run" order="12" sideWeight="0.87473345" visible="true" weight="0.21995708" />
+      <window_info active="true" anchor="bottom" id="Run" order="12" sideWeight="0.87473345" visible="true" weight="0.6351931" />
       <window_info anchor="right" id="Commander" internal_type="SLIDING" order="0" type="SLIDING" weight="0.4" />
       <window_info anchor="right" id="Ant Build" order="1" weight="0.25" />
       <window_info anchor="right" content_ui="combo" id="Hierarchy" order="2" weight="0.25" />
@@ -973,7 +1031,7 @@
         </line-breakpoint>
         <line-breakpoint enabled="true" suspend="THREAD" type="python-line">
           <url>file://$PROJECT_DIR$/pipeline/zuqiu.py</url>
-          <line>248</line>
+          <line>251</line>
           <option name="timeStamp" value="46" />
         </line-breakpoint>
         <line-breakpoint enabled="true" suspend="THREAD" type="python-line">
@@ -998,12 +1056,12 @@
         </line-breakpoint>
         <line-breakpoint enabled="true" suspend="THREAD" type="python-line">
           <url>file://$PROJECT_DIR$/pipeline/zuqiu.py</url>
-          <line>271</line>
+          <line>274</line>
           <option name="timeStamp" value="62" />
         </line-breakpoint>
         <line-breakpoint enabled="true" suspend="THREAD" type="python-line">
           <url>file://$PROJECT_DIR$/pipeline/zuqiu.py</url>
-          <line>205</line>
+          <line>208</line>
           <option name="timeStamp" value="65" />
         </line-breakpoint>
         <line-breakpoint enabled="true" suspend="THREAD" type="python-line">
@@ -1073,30 +1131,6 @@
     </expressions>
   </component>
   <component name="editorHistoryManager">
-    <entry file="file://$USER_HOME$/AppData/Local/Programs/Python/Python36/Lib/site-packages/twisted/python/failure.py">
-      <provider selected="true" editor-type-id="text-editor">
-        <state relative-caret-position="4158">
-          <caret line="242" selection-start-line="242" selection-end-line="242" />
-        </state>
-      </provider>
-    </entry>
-    <entry file="file://$PROJECT_DIR$/spiders/bangqiu.py">
-      <provider selected="true" editor-type-id="text-editor">
-        <state relative-caret-position="3132">
-          <caret line="174" column="74" lean-forward="true" selection-start-line="174" selection-start-column="74" selection-end-line="174" selection-end-column="74" />
-          <folding>
-            <element signature="e#24#35#0" expanded="true" />
-          </folding>
-        </state>
-      </provider>
-    </entry>
-    <entry file="file://$USER_HOME$/AppData/Local/Programs/Python/Python36/Lib/site-packages/psycopg2/extras.py">
-      <provider selected="true" editor-type-id="text-editor">
-        <state relative-caret-position="1944">
-          <caret line="140" selection-start-line="140" selection-end-line="140" />
-        </state>
-      </provider>
-    </entry>
     <entry file="file://$PROJECT_DIR$/spiders/wq_jieshu.py" />
     <entry file="file://$PROJECT_DIR$/spiders/__init__.py">
       <provider selected="true" editor-type-id="text-editor" />
@@ -1226,16 +1260,6 @@
         </state>
       </provider>
     </entry>
-    <entry file="file://$PROJECT_DIR$/spiders/lanqiu.py">
-      <provider selected="true" editor-type-id="text-editor">
-        <state relative-caret-position="8310">
-          <caret line="513" column="35" selection-start-line="513" selection-start-column="24" selection-end-line="513" selection-end-column="35" />
-          <folding>
-            <element signature="e#24#35#0" expanded="true" />
-          </folding>
-        </state>
-      </provider>
-    </entry>
     <entry file="file://C:/venv/Lib/site-packages/twisted/internet/defer.py">
       <provider selected="true" editor-type-id="text-editor">
         <state relative-caret-position="-730">
@@ -1243,13 +1267,6 @@
         </state>
       </provider>
     </entry>
-    <entry file="file://$PROJECT_DIR$/middlewares.py">
-      <provider selected="true" editor-type-id="text-editor">
-        <state relative-caret-position="308">
-          <caret line="137" selection-start-line="137" selection-end-line="137" />
-        </state>
-      </provider>
-    </entry>
     <entry file="file://$PROJECT_DIR$/spiders/roll_bangqiu.py">
       <provider selected="true" editor-type-id="text-editor">
         <state relative-caret-position="100">
@@ -1274,13 +1291,6 @@
         </state>
       </provider>
     </entry>
-    <entry file="file://$PROJECT_DIR$/settings.py">
-      <provider selected="true" editor-type-id="text-editor">
-        <state relative-caret-position="296">
-          <caret line="146" lean-forward="true" selection-start-line="146" selection-end-line="146" />
-        </state>
-      </provider>
-    </entry>
     <entry file="file://$PROJECT_DIR$/pipeline/ball_func.py">
       <provider selected="true" editor-type-id="text-editor">
         <state relative-caret-position="241">
@@ -1303,154 +1313,203 @@
         </state>
       </provider>
     </entry>
-    <entry file="file://$PROJECT_DIR$/utils/helper.py">
+    <entry file="file://$PROJECT_DIR$/spiders/wangqiu.py">
       <provider selected="true" editor-type-id="text-editor">
-        <state relative-caret-position="245">
-          <caret line="13" selection-start-line="13" selection-end-line="13" />
+        <state relative-caret-position="-160">
+          <caret line="14" column="20" selection-start-line="14" selection-start-column="20" selection-end-line="14" selection-end-column="20" />
+        </state>
+      </provider>
+    </entry>
+    <entry file="file://$PROJECT_DIR$/spiders/roll_lanqiu.py">
+      <provider selected="true" editor-type-id="text-editor">
+        <state relative-caret-position="-12017">
+          <caret line="89" column="20" lean-forward="true" selection-start-line="89" selection-start-column="20" selection-end-line="89" selection-end-column="20" />
           <folding>
-            <element signature="e#0#14#0" expanded="true" />
+            <element signature="e#24#39#0" expanded="true" />
           </folding>
         </state>
       </provider>
     </entry>
-    <entry file="file://$PROJECT_DIR$/spiders/roll_wangqiu.py">
+    <entry file="file://$PROJECT_DIR$/pipeline/bangqiu.py">
       <provider selected="true" editor-type-id="text-editor">
-        <state relative-caret-position="40">
-          <caret line="62" selection-start-line="62" selection-end-line="62" />
+        <state relative-caret-position="277">
+          <caret line="14" lean-forward="true" selection-start-line="14" selection-end-line="14" />
           <folding>
-            <element signature="e#24#39#0" expanded="true" />
+            <element signature="e#0#15#0" expanded="true" />
           </folding>
         </state>
       </provider>
     </entry>
-    <entry file="file://$PROJECT_DIR$/spiders/roll_zuqiu.py">
+    <entry file="file://$PROJECT_DIR$/pipeline/roll_bangqiu.py">
       <provider selected="true" editor-type-id="text-editor">
-        <state relative-caret-position="8155">
-          <caret line="442" column="18" lean-forward="true" selection-start-line="442" selection-start-column="18" selection-end-line="442" selection-end-column="18" />
+        <state relative-caret-position="141">
+          <caret line="22" column="41" lean-forward="true" selection-start-line="22" selection-start-column="41" selection-end-line="22" selection-end-column="41" />
           <folding>
-            <element signature="e#24#35#0" expanded="true" />
+            <element signature="e#0#15#0" expanded="true" />
           </folding>
         </state>
       </provider>
     </entry>
-    <entry file="file://$PROJECT_DIR$/items.py">
+    <entry file="file://$PROJECT_DIR$/pipeline/roll_lanqiu.py">
       <provider selected="true" editor-type-id="text-editor">
-        <state relative-caret-position="1018">
-          <caret line="370" column="31" lean-forward="true" selection-start-line="370" selection-start-column="31" selection-end-line="370" selection-end-column="31" />
+        <state relative-caret-position="897">
+          <caret line="295" column="16" lean-forward="true" selection-start-line="295" selection-start-column="16" selection-end-line="295" selection-end-column="16" />
+          <folding>
+            <element signature="e#0#15#0" expanded="true" />
+          </folding>
         </state>
       </provider>
     </entry>
-    <entry file="file://$PROJECT_DIR$/spiders/zuqiu.py">
+    <entry file="file://$PROJECT_DIR$/pipeline/wangqiu.py">
       <provider selected="true" editor-type-id="text-editor">
-        <state relative-caret-position="200">
-          <caret line="10" lean-forward="true" selection-start-line="10" selection-end-line="10" />
+        <state relative-caret-position="243">
+          <caret line="150" column="43" lean-forward="true" selection-start-line="150" selection-start-column="43" selection-end-line="150" selection-end-column="43" />
           <folding>
-            <element signature="e#24#35#0" expanded="true" />
+            <element signature="e#0#15#0" expanded="true" />
           </folding>
         </state>
       </provider>
     </entry>
-    <entry file="file://$PROJECT_DIR$/spiders/wangqiu.py">
+    <entry file="file://$PROJECT_DIR$/spiders/roll_zuqiu.py">
       <provider selected="true" editor-type-id="text-editor">
-        <state relative-caret-position="-160">
-          <caret line="14" column="20" selection-start-line="14" selection-start-column="20" selection-end-line="14" selection-end-column="20" />
+        <state relative-caret-position="8600">
+          <caret line="442" column="18" lean-forward="true" selection-start-line="442" selection-start-column="18" selection-end-line="442" selection-end-column="18" />
+          <folding>
+            <element signature="e#24#35#0" expanded="true" />
+          </folding>
         </state>
       </provider>
     </entry>
-    <entry file="file://$PROJECT_DIR$/main.py">
+    <entry file="file://$PROJECT_DIR$/spiders/roll_wangqiu.py">
       <provider selected="true" editor-type-id="text-editor">
-        <state relative-caret-position="180">
-          <caret line="9" column="38" lean-forward="true" selection-start-line="9" selection-start-column="38" selection-end-line="9" selection-end-column="38" />
+        <state relative-caret-position="520">
+          <caret line="62" selection-start-line="62" selection-end-line="62" />
           <folding>
-            <element signature="e#0#10#0" expanded="true" />
+            <element signature="e#24#39#0" expanded="true" />
           </folding>
         </state>
       </provider>
     </entry>
     <entry file="file://$PROJECT_DIR$/pipeline/roll_wangqiu.py">
       <provider selected="true" editor-type-id="text-editor">
-        <state relative-caret-position="4937">
-          <caret line="249" column="13" lean-forward="true" selection-start-line="249" selection-start-column="13" selection-end-line="249" selection-end-column="13" />
+        <state relative-caret-position="280">
+          <caret line="14" selection-start-line="14" selection-end-line="17" selection-end-column="34" />
           <folding>
             <element signature="e#0#15#0" expanded="true" />
           </folding>
         </state>
       </provider>
     </entry>
-    <entry file="file://$PROJECT_DIR$/pipeline/wangqiu.py">
+    <entry file="file://$PROJECT_DIR$/items.py">
+      <provider selected="true" editor-type-id="text-editor">
+        <state relative-caret-position="356">
+          <caret line="458" lean-forward="true" selection-start-line="458" selection-end-line="458" />
+        </state>
+      </provider>
+    </entry>
+    <entry file="file://$PROJECT_DIR$/pipeline/lanqiu.py">
       <provider selected="true" editor-type-id="text-editor">
-        <state relative-caret-position="543">
-          <caret line="163" column="40" lean-forward="true" selection-start-line="163" selection-start-column="40" selection-end-line="163" selection-end-column="40" />
+        <state relative-caret-position="2717">
+          <caret line="157" column="17" lean-forward="true" selection-start-line="157" selection-start-column="17" selection-end-line="157" selection-end-column="17" />
           <folding>
             <element signature="e#0#15#0" expanded="true" />
           </folding>
         </state>
       </provider>
     </entry>
+    <entry file="file://$PROJECT_DIR$/spiders/liansai.py">
+      <provider selected="true" editor-type-id="text-editor">
+        <state relative-caret-position="-420" />
+      </provider>
+    </entry>
     <entry file="file://$PROJECT_DIR$/pipeline/roll_zuqiu.py">
       <provider selected="true" editor-type-id="text-editor">
-        <state relative-caret-position="463">
-          <caret line="86" column="17" lean-forward="true" selection-start-line="86" selection-end-line="88" selection-end-column="43" />
+        <state relative-caret-position="295">
+          <caret line="97" column="31" selection-start-line="97" selection-start-column="31" selection-end-line="97" selection-end-column="31" />
           <folding>
             <element signature="e#18#33#0" expanded="true" />
           </folding>
         </state>
       </provider>
     </entry>
-    <entry file="file://$PROJECT_DIR$/pipeline/zuqiu.py">
+    <entry file="file://$PROJECT_DIR$/utils/helper.py">
       <provider selected="true" editor-type-id="text-editor">
-        <state relative-caret-position="463">
-          <caret line="121" column="44" selection-start-line="121" selection-start-column="44" selection-end-line="121" selection-end-column="44" />
+        <state relative-caret-position="200">
+          <caret line="13" selection-start-line="13" selection-end-line="13" />
           <folding>
-            <element signature="e#0#15#0" expanded="true" />
+            <element signature="e#0#14#0" expanded="true" />
           </folding>
         </state>
       </provider>
     </entry>
-    <entry file="file://$PROJECT_DIR$/spiders/roll_lanqiu.py">
+    <entry file="file://$PROJECT_DIR$/middlewares.py">
       <provider selected="true" editor-type-id="text-editor">
-        <state relative-caret-position="-12017">
-          <caret line="89" column="20" lean-forward="true" selection-start-line="89" selection-start-column="20" selection-end-line="89" selection-end-column="20" />
+        <state relative-caret-position="799">
+          <caret line="152" selection-start-line="152" selection-end-line="152" />
+        </state>
+      </provider>
+    </entry>
+    <entry file="file://$PROJECT_DIR$/settings.py">
+      <provider selected="true" editor-type-id="text-editor">
+        <state relative-caret-position="160">
+          <caret line="152" column="49" selection-start-line="152" selection-start-column="49" selection-end-line="152" selection-end-column="49" />
+        </state>
+      </provider>
+    </entry>
+    <entry file="file://$PROJECT_DIR$/spiders/zuqiu.py">
+      <provider selected="true" editor-type-id="text-editor">
+        <state relative-caret-position="-5140">
+          <caret line="40" column="21" lean-forward="true" selection-start-line="40" selection-start-column="14" selection-end-line="40" selection-end-column="70" />
           <folding>
-            <element signature="e#24#39#0" expanded="true" />
+            <element signature="e#24#35#0" expanded="true" />
           </folding>
         </state>
       </provider>
     </entry>
-    <entry file="file://$PROJECT_DIR$/pipeline/roll_lanqiu.py">
+    <entry file="file://$PROJECT_DIR$/pipeline/saiguo.py">
       <provider selected="true" editor-type-id="text-editor">
-        <state relative-caret-position="897">
-          <caret line="295" column="16" lean-forward="true" selection-start-line="295" selection-start-column="16" selection-end-line="295" selection-end-column="16" />
+        <state relative-caret-position="940">
+          <caret line="47" column="19" lean-forward="true" selection-start-line="47" selection-start-column="19" selection-end-line="47" selection-end-column="19" />
           <folding>
-            <element signature="e#0#15#0" expanded="true" />
+            <element signature="e#18#33#0" expanded="true" />
           </folding>
         </state>
       </provider>
     </entry>
-    <entry file="file://$PROJECT_DIR$/pipeline/roll_bangqiu.py">
+    <entry file="file://$PROJECT_DIR$/main.py">
       <provider selected="true" editor-type-id="text-editor">
-        <state relative-caret-position="-2383">
-          <caret line="110" column="32" lean-forward="true" selection-start-line="109" selection-end-line="113" selection-end-column="41" />
+        <state relative-caret-position="160">
+          <caret line="8" column="37" lean-forward="true" selection-start-line="8" selection-start-column="37" selection-end-line="8" selection-end-column="37" />
           <folding>
-            <element signature="e#0#15#0" expanded="true" />
+            <element signature="e#0#10#0" expanded="true" />
           </folding>
         </state>
       </provider>
     </entry>
-    <entry file="file://$PROJECT_DIR$/pipeline/bangqiu.py">
+    <entry file="file://$PROJECT_DIR$/spiders/lanqiu.py">
       <provider selected="true" editor-type-id="text-editor">
-        <state relative-caret-position="277">
-          <caret line="14" lean-forward="true" selection-start-line="14" selection-end-line="14" />
+        <state relative-caret-position="302">
+          <caret line="56" column="39" lean-forward="true" selection-start-line="55" selection-end-line="59" selection-end-column="96" />
+          <folding>
+            <element signature="e#24#35#0" expanded="true" />
+          </folding>
+        </state>
+      </provider>
+    </entry>
+    <entry file="file://$PROJECT_DIR$/pipeline/zuqiu.py">
+      <provider selected="true" editor-type-id="text-editor">
+        <state relative-caret-position="396">
+          <caret line="22" column="4" selection-start-line="22" selection-start-column="4" selection-end-line="22" selection-end-column="4" />
           <folding>
             <element signature="e#0#15#0" expanded="true" />
           </folding>
         </state>
       </provider>
     </entry>
-    <entry file="file://$PROJECT_DIR$/pipeline/lanqiu.py">
+    <entry file="file://$PROJECT_DIR$/spiders/saiguo.py">
       <provider selected="true" editor-type-id="text-editor">
-        <state relative-caret-position="1157">
-          <caret line="157" column="17" lean-forward="true" selection-start-line="157" selection-start-column="17" selection-end-line="157" selection-end-column="17" />
+        <state relative-caret-position="112">
+          <caret line="26" column="34" lean-forward="true" selection-start-line="26" selection-start-column="34" selection-end-line="26" selection-end-column="34" />
           <folding>
             <element signature="e#0#15#0" expanded="true" />
           </folding>

hg3535/items.py

@@ -458,25 +458,8 @@ class Roll_Bangqiu(scrapy.Field):
     team_source_dict = scrapy.Field()
 
 
-# -----------------------------------------------------------足球比赛结果------------------------------------------------
-class Zujieguo(scrapy.Field):
+# -----------------------------------------------------------比赛结果-----------------------------------------------------
+class Hgsaiguo(scrapy.Field):
     # 足球比赛结果元组
     id_score = scrapy.Field()
-
-
-# -----------------------------------------------------------篮球比赛结果------------------------------------------------
-class Lanjieguo(scrapy.Field):
-    # 篮球比赛结果元组
-    id_score = scrapy.Field()
-
-
-# -----------------------------------------------------------网球比赛结果------------------------------------------------
-class Wangjieguo(scrapy.Field):
-    # 网球比赛结果元组
-    id_score = scrapy.Field()
-
-
-# -----------------------------------------------------------棒球比赛结果------------------------------------------------
-class Bangjieguo(scrapy.Field):
-    # 网球比赛结果元组
-    id_score = scrapy.Field()
+    pt = scrapy.Field()

hg3535/main.py

@@ -17,7 +17,7 @@ execute(["scrapy", "crawl", "lanqiu"])
 # execute(["scrapy", "crawl", "roll_bangqiu"])  # 滚球棒球
 # execute(["scrapy", "crawl", "ball_status"]) #滚球id本地存
 # execute(["scrapy", "crawl", "ball_status_update"]) #滚球id结束时间更新状态
-# execute(["scrapy", "crawl", "zq_jieshu"]) #滚球id结束时间更新状态
+# execute(["scrapy", "crawl", "saiguo"]) #滚球id结束时间更新状态
 # execute(["scrapy", "crawl", "lq_jieshu"]) #滚球id结束时间更新状态
 # execute(["scrapy", "crawl", "wq_jieshu"]) #滚球id结束时间更新状态
 # execute(["scrapy", "crawl", "bq_jieshu"]) #滚球id结束时间更新状态

hg3535/pipeline/roll_bangqiu.py

@@ -25,7 +25,7 @@ class Roll_Banqiupipeline(object):
         reactor.callInThread(self._do_calculation, item, out)
         yield out
         defer.returnValue(item)
-    #
+
     def _do_calculation(self, item, out):
         # 使用twisted将mysql插入变成异步执行
         logger = logging.getLogger(__name__)

hg3535/pipeline/saiguo.py

@@ -0,0 +1,61 @@
+# import datetime
+import datetime
+import pymongo
+import time
+import logging
+
+import redis
+from twisted.internet import defer, reactor
+
+from ..settings import M_HOST, M_USER, M_PASSWORD, M_POST, M_DB, MATCH_STATUS
+from ..utils.helper import Helper
+
+# 滚球足球 插入
+class Jieshuqiupipeline(object):
+
+    def open_spider(self, spider):
+        self.mongo = pymongo.MongoClient(host=M_HOST, username=M_USER, password=M_PASSWORD, port=M_POST)
+        self.db = self.mongo[M_DB]
+        # cls = redis.Redis(host='192.168.0.110', port=6379, db=0)
+
+    @defer.inlineCallbacks
+    def process_item(self, item, spider):
+        out = defer.Deferred()
+        reactor.callInThread(self._do_calculation, item, out)
+        yield out
+        defer.returnValue(item)
+
+    # def process_item(self, item, spider):
+    def _do_calculation(self, item, out):
+        logger = logging.getLogger(__name__)
+        match_id, score_other = item['id_score']
+        pt = item['pt']
+        if pt == 1:
+            game_code = "zq"
+        elif pt == 2:
+            game_code = "lq"
+        elif pt == 3:
+            game_code = "wq"
+        else:
+            game_code = "bq"
+        if self.db.match_status35.find({'match_id': match_id}).count() < 1:
+            status_dict = {"game_code": game_code, "title": "match_status", "source": "hg3535"}
+            data_list = []
+            data = {'match_id': match_id, 'status': 2, "is_rollball": 0, "is_today": 0, "is_morningplate": 0, "is_stringscene": 0, "is_horn": 0}
+            data_list.append(data)
+            status_dict['data'] = data_list
+            res = Helper.async_post(MATCH_STATUS, status_dict)
+            if res:
+                if "成功" in res:
+                    self.db.match_status35.insert(status_dict)
+                    logger.info('足球赛事结果状态交成功, {}'.format(res))
+                    logger.info(status_dict)
+                else:
+                    logger.warning('足球赛事结果状态交失败, {}'.format(res))
+                    logger.warning(status_dict)
+            else:
+                logger.warning('足球赛事结果状态接口异常提交失败, {}'.format(res))
+                logger.warning(status_dict)
+        else:
+            logger.info('足球赛事结果状态已存在，不提交')
+        reactor.callFromThread(out.callback, item)

hg3535/pipeline/zuqiu.py

@@ -1,4 +1,5 @@
 import datetime
+import threading
 import time
 import logging
 
@@ -17,6 +18,7 @@ class Zuqiupipeline(object):
         # session = aiohttp.ClientSession()
         self.mongo = pymongo.MongoClient(host=M_HOST, username=M_USER, password=M_PASSWORD, port=M_POST, authSource='kaiyou')
         self.db = self.mongo[M_DB]
+        # self.lock = threading.RLock()
 
     @defer.inlineCallbacks
     def process_item(self, item, spider):
@@ -27,6 +29,7 @@ class Zuqiupipeline(object):
 
     # def process_item(self, item, spider):
     def _do_calculation(self, item, out):
+        # with self.lock:
         logger = logging.getLogger(__name__)
         # 比赛日期
         try:
@@ -536,4 +539,5 @@ class Zuqiupipeline(object):
                 logging.warning('足球详细赔率接口异常, {}'.format(res))
         else:
             logger.info('足球详细赔率列表为空')
+        # self.lock.acquire()
         reactor.callFromThread(out.callback, item)

hg3535/settings.py

@@ -150,6 +150,8 @@ MATCH_URL = 'http://stadmin.bocai108.com:19093/setMatch'
 ODDS_URL = 'http://stadmin.bocai108.com:19093/setOdds'
 TOKEN_URL = "http://stadmin.bocai108.com/getToken"
 MATCH_RESULT = "http://stadmin.bocai108.com:19093/setMatchResult"
+MATCH_STATUS = "http://stadmin.bocai108.com:19093/upMatch"
+#upMatch
 # POST_HOST = 'localhost'
 # POST_DATABASE = 'kaiyou'
 # POST_USER = 'kaiyou'

hg3535/spiders/saiguo.py

@@ -0,0 +1,136 @@
+import datetime
+import re
+
+import scrapy
+
+from ..items import Hgsaiguo
+
+
+class HgjieshuSpider(scrapy.Spider):
+    name = 'saiguo'
+    to_day = datetime.datetime.now()
+    allowed_domains = ['hg3535z.com']
+    custom_settings = {
+        "ITEM_PIPELINES": {
+            'hg3535.pipeline.saiguo.Jieshuqiupipeline': 300,
+        },
+        # 'LOG_LEVEL': 'DEBUG',
+        # 'LOG_FILE': "../hg3535/log/zq_jieshu_{}_{}_{}.log".format(to_day.year, to_day.month, to_day.day)
+    }
+
+    def start_requests(self):
+        for y in range(1, 5):
+            url = 'https://www.hg3535.cn/zh-cn/info-centre/sportsbook-info/results/{}/normal/1'.format(y)
+            yield scrapy.Request(url=url, callback=self.parse, dont_filter=True, meta={'pt': y})
+
+    def parse(self, response):
+        if response.status == 200:
+            pt = response.meta['pt']
+            if pt == 1:
+                tema_score = response.xpath('//div[@class="flex-wrap"]/../div[5]')
+                # 获得所有比赛id对象
+                tema_id = response.xpath('//div[@class="flex-wrap"]/../div[1]/@id')
+                # 所有比赛id列表
+                temaid_list = [i.extract().replace('e-', "") for i in tema_id]
+                temascore_list = []
+                for score in tema_score:
+                    # 正则匹配规则
+                    p1 = r"\d{1,3}-\d{1,3}"
+                    pattern1 = re.compile(p1)
+                    try:
+                        # 获取正则匹配结果
+                        c = pattern1.findall(score.extract())[0]
+                        temascore_list.append(c)
+                    except:
+                        c = ""
+                        temascore_list.append(c)
+                # 赛事id，赛事比元组列表
+                tema_tupe = {(temaid_list[i], temascore_list[i]) for i in range(len(temaid_list))}
+                for y in tema_tupe:
+                    if y[1]:
+                        item = Hgsaiguo()
+                        item['id_score'] = y
+                        item['pt'] = pt
+                        yield item
+            if pt == 2:
+                tema_score = response.xpath('//div[@class="flex-wrap"]/../div[5]')
+                # 获得所有比赛id对象
+                tema_id = response.xpath('//div[@class="flex-wrap"]/../div[1]/@id')
+                # 所有比赛id列表
+                temaid_list = [i.extract().replace('e-', "") for i in tema_id]
+                temascore_list = []
+                for score in tema_score:
+                    # 正则匹配规则
+                    p1 = r"\d{1,3}-\d{1,3}"
+                    pattern1 = re.compile(p1)
+                    try:
+                        # 获取正则匹配结果
+                        c = pattern1.findall(score.extract())[0]
+                        temascore_list.append(c)
+                    except:
+                        c = ""
+                        temascore_list.append(c)
+                # 赛事id，赛事比元组列表
+                tema_tupe = {(temaid_list[i], temascore_list[i]) for i in range(len(temaid_list))}
+                for y in tema_tupe:
+                    if y[1]:
+                        item = Hgsaiguo()
+                        item['id_score'] = y
+                        item['pt'] = pt
+                        yield item
+
+            if pt == 3:
+                # 获得所有比赛获胜人，判断赛事是否结束
+                tema_score = response.xpath('//div[@class="flex-wrap"]/../div[4]/text()')
+                # 获得所有比赛id对象
+                tema_id = response.xpath('//div[@class="flex-wrap"]/../div[1]/@id')
+                # 所有比赛id列表
+                temaid_list = [i.extract().replace('e-', "") for i in tema_id]
+                temascore_list = []
+                for score in tema_score:
+                    # 正则匹配规则
+                    # p1 = r"\d{1,3}-\d{1,3}"
+                    # pattern1 = re.compile(p1)
+                    try:
+                        # 获取正则匹配结果
+                        c = score.extract()
+                        temascore_list.append(c)
+                    except:
+                        c = ""
+                        temascore_list.append(c)
+                # 赛事id，赛事比元组列表
+                tema_tupe = {(temaid_list[i], temascore_list[i]) for i in range(len(temaid_list))}
+                for y in tema_tupe:
+                    if y[1]:
+                        item = Hgsaiguo()
+                        item['id_score'] = y
+                        item['pt'] = pt
+                        yield item
+
+            if pt == 4:
+                tema_score = response.xpath('//div[@class="flex-wrap"]/../div[5]')
+                # 获得所有比赛id对象
+                tema_id = response.xpath('//div[@class="flex-wrap"]/../div[1]/@id')
+                # str.replace()
+                # 所有比赛id列表
+                temaid_list = [i.extract().replace('e-', "") for i in tema_id]
+                temascore_list = []
+                for score in tema_score:
+                    # 正则匹配规则
+                    p1 = r"\d{1,3}-\d{1,3}"
+                    pattern1 = re.compile(p1)
+                    try:
+                        # 获取正则匹配结果
+                        c = pattern1.findall(score.extract())[0]
+                        temascore_list.append(c)
+                    except:
+                        c = ""
+                        temascore_list.append(c)
+                # 赛事id，赛事比元组列表
+                tema_tupe = {(temaid_list[i], temascore_list[i]) for i in range(len(temaid_list))}
+                for y in tema_tupe:
+                    if y[1]:
+                        item = Hgsaiguo()
+                        item['id_score'] = y
+                        item['pt'] = pt
+                        yield item