'更新'

hg3535/.idea/vcs.xml

@@ -2,5 +2,6 @@
 <project version="4">
   <component name="VcsDirectoryMappings">
     <mapping directory="$PROJECT_DIR$/../.." vcs="Git" />
+    <mapping directory="$PROJECT_DIR$/.." vcs="Git" />
   </component>
 </project>

hg3535/.idea/workspace.xml

@@ -2,7 +2,11 @@
 <project version="4">
   <component name="ChangeListManager">
     <list default="true" id="7fa42e97-ddea-4404-91db-5a9ccc162649" name="Default Changelist" comment="">
-      <change beforePath="$PROJECT_DIR$/.idea/workspace.xml" beforeDir="false" afterPath="$PROJECT_DIR$/.idea/workspace.xml" afterDir="false" />
+      <change beforePath="$PROJECT_DIR$/.idea/vcs.xml" beforeDir="false" afterPath="$PROJECT_DIR$/.idea/vcs.xml" afterDir="false" />
+      <change beforePath="$PROJECT_DIR$/items.py" beforeDir="false" afterPath="$PROJECT_DIR$/items.py" afterDir="false" />
+      <change beforePath="$PROJECT_DIR$/main.py" beforeDir="false" afterPath="$PROJECT_DIR$/main.py" afterDir="false" />
+      <change beforePath="$PROJECT_DIR$/pipelines.py" beforeDir="false" afterPath="$PROJECT_DIR$/pipelines.py" afterDir="false" />
+      <change beforePath="$PROJECT_DIR$/settings.py" beforeDir="false" afterPath="$PROJECT_DIR$/settings.py" afterDir="false" />
     </list>
     <option name="EXCLUDED_CONVERTED_TO_IGNORED" value="true" />
     <option name="SHOW_DIALOG" value="false" />
@@ -12,7 +16,7 @@
   </component>
   <component name="CoverageDataManager">
     <SUITE FILE_PATH="coverage/hg3535$items.coverage" NAME="items Coverage Results" MODIFIED="1554291398794" SOURCE_PROVIDER="com.intellij.coverage.DefaultCoverageFileProvider" RUNNER="coverage.py" COVERAGE_BY_TEST_ENABLED="true" COVERAGE_TRACING_ENABLED="false" WORKING_DIRECTORY="$PROJECT_DIR$" />
-    <SUITE FILE_PATH="coverage/hg3535$main.coverage" NAME="main Coverage Results" MODIFIED="1557048948143" SOURCE_PROVIDER="com.intellij.coverage.DefaultCoverageFileProvider" RUNNER="coverage.py" COVERAGE_BY_TEST_ENABLED="true" COVERAGE_TRACING_ENABLED="false" WORKING_DIRECTORY="$PROJECT_DIR$" />
+    <SUITE FILE_PATH="coverage/hg3535$main.coverage" NAME="main Coverage Results" MODIFIED="1557310176053" SOURCE_PROVIDER="com.intellij.coverage.DefaultCoverageFileProvider" RUNNER="coverage.py" COVERAGE_BY_TEST_ENABLED="true" COVERAGE_TRACING_ENABLED="false" WORKING_DIRECTORY="$PROJECT_DIR$" />
     <SUITE FILE_PATH="coverage/hg3535$halffull.coverage" NAME="halffull Coverage Results" MODIFIED="1554544334424" SOURCE_PROVIDER="com.intellij.coverage.DefaultCoverageFileProvider" RUNNER="coverage.py" COVERAGE_BY_TEST_ENABLED="true" COVERAGE_TRACING_ENABLED="false" WORKING_DIRECTORY="$PROJECT_DIR$/spiders" />
     <SUITE FILE_PATH="coverage/hg3535$pipelines.coverage" NAME="pipelines Coverage Results" MODIFIED="1554290866137" SOURCE_PROVIDER="com.intellij.coverage.DefaultCoverageFileProvider" RUNNER="coverage.py" COVERAGE_BY_TEST_ENABLED="true" COVERAGE_TRACING_ENABLED="false" WORKING_DIRECTORY="$PROJECT_DIR$" />
     <SUITE FILE_PATH="coverage/hg3535$today_all.coverage" NAME="today_all Coverage Results" MODIFIED="1553172114927" SOURCE_PROVIDER="com.intellij.coverage.DefaultCoverageFileProvider" RUNNER="coverage.py" COVERAGE_BY_TEST_ENABLED="true" COVERAGE_TRACING_ENABLED="false" WORKING_DIRECTORY="$PROJECT_DIR$/spiders" />
@@ -38,12 +42,12 @@
     <favorites_list name="hg3535" />
   </component>
   <component name="FileEditorManager">
-    <leaf>
+    <leaf SIDE_TABS_SIZE_LIMIT_KEY="300">
       <file pinned="false" current-in-tab="false">
         <entry file="file://$PROJECT_DIR$/items.py">
           <provider selected="true" editor-type-id="text-editor">
-            <state relative-caret-position="-2932">
-              <caret line="50" lean-forward="true" selection-start-line="50" selection-end-line="50" />
+            <state relative-caret-position="252">
+              <caret line="246" column="32" lean-forward="true" selection-start-line="246" selection-start-column="32" selection-end-line="246" selection-end-column="32" />
               <folding>
                 <element signature="e#320#1230#0" />
                 <element signature="e#1382#1528#0" />
@@ -55,32 +59,8 @@
       <file pinned="false" current-in-tab="false">
         <entry file="file://$PROJECT_DIR$/spiders/zuqiu.py">
           <provider selected="true" editor-type-id="text-editor">
-            <state relative-caret-position="72">
-              <caret line="130" column="37" lean-forward="true" selection-start-line="130" selection-start-column="37" selection-end-line="130" selection-end-column="37" />
-              <folding>
-                <element signature="e#24#35#0" expanded="true" />
-              </folding>
-            </state>
-          </provider>
-        </entry>
-      </file>
-      <file pinned="false" current-in-tab="false">
-        <entry file="file://$PROJECT_DIR$/jianting.py">
-          <provider selected="true" editor-type-id="text-editor">
-            <state relative-caret-position="324">
-              <caret line="18" column="14" lean-forward="true" selection-start-line="18" selection-start-column="14" selection-end-line="18" selection-end-column="14" />
-              <folding>
-                <element signature="e#0#9#0" expanded="true" />
-              </folding>
-            </state>
-          </provider>
-        </entry>
-      </file>
-      <file pinned="false" current-in-tab="false">
-        <entry file="file://$PROJECT_DIR$/spiders/liansai.py">
-          <provider selected="true" editor-type-id="text-editor">
-            <state relative-caret-position="23">
-              <caret line="21" column="31" selection-start-line="21" selection-start-column="31" selection-end-line="21" selection-end-column="31" />
+            <state relative-caret-position="54">
+              <caret line="129" column="14" selection-start-line="129" selection-start-column="14" selection-end-line="129" selection-end-column="14" />
               <folding>
                 <element signature="e#24#35#0" expanded="true" />
               </folding>
@@ -91,43 +71,29 @@
       <file pinned="false" current-in-tab="false">
         <entry file="file://$PROJECT_DIR$/pipelines.py">
           <provider selected="true" editor-type-id="text-editor">
-            <state relative-caret-position="265">
-              <caret line="392" column="50" lean-forward="true" selection-start-line="392" selection-start-column="50" selection-end-line="392" selection-end-column="50" />
+            <state relative-caret-position="-144">
+              <caret line="45" column="29" lean-forward="true" selection-start-line="45" selection-start-column="29" selection-end-line="45" selection-end-column="29" />
               <folding>
-                <element signature="e#193#208#0" expanded="true" />
-                <element signature="e#18330#19014#0" />
-                <element signature="e#25714#29974#0" />
-                <element signature="e#34571#39061#0" />
-                <element signature="e#39126#43615#0" />
-                <element signature="e#43680#48166#0" />
-                <element signature="e#48346#53071#0" />
-                <element signature="e#53251#58051#0" />
-                <element signature="e#58092#58573#0" />
-                <element signature="e#58725#74468#0" />
-                <element signature="e#74621#82064#0" />
-                <element signature="e#82218#92209#0" />
-                <element signature="e#92365#99910#0" />
-                <element signature="e#100064#109753#0" />
-                <element signature="e#115021#121360#0" />
+                <element signature="e#0#15#0" expanded="true" />
               </folding>
             </state>
           </provider>
         </entry>
       </file>
-      <file pinned="false" current-in-tab="false">
+      <file pinned="false" current-in-tab="true">
         <entry file="file://$PROJECT_DIR$/settings.py">
           <provider selected="true" editor-type-id="text-editor">
-            <state relative-caret-position="153">
-              <caret line="117" lean-forward="true" selection-start-line="117" selection-end-line="117" />
+            <state relative-caret-position="226">
+              <caret line="129" column="28" lean-forward="true" selection-start-line="129" selection-start-column="28" selection-end-line="129" selection-end-column="28" />
             </state>
           </provider>
         </entry>
       </file>
-      <file pinned="false" current-in-tab="true">
+      <file pinned="false" current-in-tab="false">
         <entry file="file://$PROJECT_DIR$/main.py">
           <provider selected="true" editor-type-id="text-editor">
             <state relative-caret-position="234">
-              <caret line="13" lean-forward="true" selection-start-line="13" selection-end-line="13" />
+              <caret line="13" column="41" lean-forward="true" selection-start-line="13" selection-start-column="41" selection-end-line="13" selection-end-column="41" />
               <folding>
                 <element signature="e#0#10#0" expanded="true" />
               </folding>
@@ -176,7 +142,7 @@
     </findStrings>
   </component>
   <component name="Git.Settings">
-    <option name="RECENT_GIT_ROOT_PATH" value="$PROJECT_DIR$/../.." />
+    <option name="RECENT_GIT_ROOT_PATH" value="$PROJECT_DIR$/.." />
   </component>
   <component name="IdeDocumentHistory">
     <option name="CHANGED_PATHS">
@@ -212,7 +178,6 @@
         <option value="$PROJECT_DIR$/spiders/lqguanjun.py" />
         <option value="$PROJECT_DIR$/spiders/lanqiuls.py" />
         <option value="$PROJECT_DIR$/spiders/testone.py" />
-        <option value="$PROJECT_DIR$/items.py" />
         <option value="$PROJECT_DIR$/middlewares.py" />
         <option value="$PROJECT_DIR$/spiders/wangqiu.py" />
         <option value="$PROJECT_DIR$/spiders/guanjun.py" />
@@ -222,16 +187,18 @@
         <option value="$PROJECT_DIR$/spiders/lanqiu.py" />
         <option value="$PROJECT_DIR$/spiders/liansai.py" />
         <option value="$PROJECT_DIR$/spiders/zuqiu.py" />
-        <option value="$PROJECT_DIR$/settings.py" />
-        <option value="$PROJECT_DIR$/main.py" />
+        <option value="$PROJECT_DIR$/requirements.txt" />
+        <option value="$PROJECT_DIR$/items.py" />
         <option value="$PROJECT_DIR$/pipelines.py" />
+        <option value="$PROJECT_DIR$/main.py" />
+        <option value="$PROJECT_DIR$/settings.py" />
       </list>
     </option>
   </component>
   <component name="ProjectFrameBounds" extendedState="6">
-    <option name="x" value="-8" />
-    <option name="y" value="-8" />
-    <option name="width" value="1583" />
+    <option name="x" value="26" />
+    <option name="y" value="10" />
+    <option name="width" value="1309" />
     <option name="height" value="1064" />
   </component>
   <component name="ProjectLevelVcsManager" settingsEditedManually="true" />
@@ -240,6 +207,7 @@
       <foldersAlwaysOnTop value="true" />
     </navigator>
     <panes>
+      <pane id="Scope" />
       <pane id="ProjectPane">
         <subPane>
           <expand>
@@ -260,12 +228,11 @@
           <select />
         </subPane>
       </pane>
-      <pane id="Scope" />
     </panes>
   </component>
   <component name="PropertiesComponent">
     <property name="WebServerToolWindowFactoryState" value="false" />
-    <property name="last_opened_file_path" value="$PROJECT_DIR$/../../../onetest/onetest" />
+    <property name="last_opened_file_path" value="$PROJECT_DIR$/../../sports_scrapy-2/sports_scrapy-1/hg3535" />
     <property name="nodejs_interpreter_path.stuck_in_default_project" value="undefined stuck path" />
     <property name="nodejs_npm_path_reset_for_default_project" value="true" />
     <property name="settings.editor.selected.configurable" value="com.jetbrains.python.configuration.PyActiveSdkModuleConfigurable" />
@@ -478,6 +445,7 @@
       <workItem from="1556500244997" duration="22444000" />
       <workItem from="1556584868377" duration="15320000" />
       <workItem from="1557018228723" duration="19258000" />
+      <workItem from="1557293774618" duration="6910000" />
     </task>
     <task id="LOCAL-00001" summary="修改后,版本二">
       <created>1555727493584</created>
@@ -588,7 +556,7 @@
     <servers />
   </component>
   <component name="TimeTrackingManager">
-    <option name="totallyTimeSpent" value="1326611000" />
+    <option name="totallyTimeSpent" value="1333521000" />
   </component>
   <component name="TodoView">
     <todo-panel id="selected-file">
@@ -601,8 +569,9 @@
   </component>
   <component name="ToolWindowManager">
     <frame x="-8" y="-8" width="1936" height="1066" extended-state="6" />
+    <editor active="true" />
     <layout>
-      <window_info content_ui="combo" id="Project" order="0" sideWeight="0.71173847" visible="true" weight="0.16044776" />
+      <window_info content_ui="combo" id="Project" order="0" sideWeight="0.71173847" visible="true" weight="0.13219616" />
       <window_info id="Structure" order="1" sideWeight="0.2882615" side_tool="true" weight="0.1108742" />
       <window_info id="Favorites" order="2" sideWeight="0.5013405" side_tool="true" weight="0.108208954" />
       <window_info anchor="bottom" id="Message" order="0" />
@@ -614,10 +583,10 @@
       <window_info anchor="bottom" id="Docker" order="6" show_stripe_button="false" />
       <window_info anchor="bottom" id="Version Control" order="7" weight="0.2918455" />
       <window_info anchor="bottom" id="Database Changes" order="8" />
-      <window_info anchor="bottom" id="Event Log" order="9" sideWeight="0.5042644" side_tool="true" weight="0.4195279" />
+      <window_info anchor="bottom" id="Event Log" order="9" sideWeight="0.5053305" side_tool="true" weight="0.6630901" />
       <window_info anchor="bottom" x="22" y="376" width="1876" height="298" id="Terminal" order="10" sideWeight="0.49573562" weight="0.16021505" />
       <window_info anchor="bottom" id="Python Console" order="11" sideWeight="0.49946696" weight="0.41309014" />
-      <window_info anchor="bottom" id="Run" order="12" sideWeight="0.4946695" weight="0.45922747" />
+      <window_info active="true" anchor="bottom" id="Run" order="12" sideWeight="0.4946695" visible="true" weight="0.41416308" />
       <window_info anchor="right" id="Commander" internal_type="SLIDING" order="0" type="SLIDING" weight="0.4" />
       <window_info anchor="right" id="Ant Build" order="1" weight="0.25" />
       <window_info anchor="right" content_ui="combo" id="Hierarchy" order="2" weight="0.25" />
@@ -713,7 +682,6 @@
     </expressions>
   </component>
   <component name="editorHistoryManager">
-    <entry file="file://$PROJECT_DIR$/spiders/bangqiuls.py" />
     <entry file="file://$PROJECT_DIR$/spiders/bqguanjun.py" />
     <entry file="file://$PROJECT_DIR$/spiders/wangqiuls.py" />
     <entry file="file://$PROJECT_DIR$/spiders/bodan.py" />
@@ -805,23 +773,8 @@
         </state>
       </provider>
     </entry>
-    <entry file="file://$USER_HOME$/AppData/Local/Programs/Python/Python36/Lib/site-packages/scrapy_deltafetch/__init__.py">
-      <provider selected="true" editor-type-id="text-editor">
-        <state>
-          <caret column="12" selection-start-column="12" selection-end-column="12" />
-        </state>
-      </provider>
-    </entry>
-    <entry file="file://$USER_HOME$/AppData/Local/Programs/Python/Python36/Lib/site-packages/scrapy_deltafetch/middleware.py">
-      <provider selected="true" editor-type-id="text-editor">
-        <state relative-caret-position="924">
-          <caret line="93" column="28" lean-forward="true" selection-start-line="93" selection-start-column="28" selection-end-line="93" selection-end-column="28" />
-          <folding>
-            <element signature="e#0#14#0" expanded="true" />
-          </folding>
-        </state>
-      </provider>
-    </entry>
+    <entry file="file://$USER_HOME$/AppData/Local/Programs/Python/Python36/Lib/site-packages/scrapy_deltafetch/__init__.py" />
+    <entry file="file://$USER_HOME$/AppData/Local/Programs/Python/Python36/Lib/site-packages/scrapy_deltafetch/middleware.py" />
     <entry file="file://$USER_HOME$/AppData/Local/Programs/Python/Python36/Lib/site-packages/scrapy/spidermiddlewares/referer.py">
       <provider selected="true" editor-type-id="text-editor">
         <state relative-caret-position="151">
@@ -930,16 +883,7 @@
         </state>
       </provider>
     </entry>
-    <entry file="file://$PROJECT_DIR$/spiders/wgbodan.py">
-      <provider selected="true" editor-type-id="text-editor">
-        <state relative-caret-position="198">
-          <caret line="11" column="14" selection-start-line="11" selection-start-column="14" selection-end-line="11" selection-end-column="14" />
-          <folding>
-            <element signature="e#24#35#0" expanded="true" />
-          </folding>
-        </state>
-      </provider>
-    </entry>
+    <entry file="file://$PROJECT_DIR$/spiders/wgbodan.py" />
     <entry file="file://$PROJECT_DIR$/spiders/wangqiu.py">
       <provider selected="true" editor-type-id="text-editor">
         <state relative-caret-position="-846">
@@ -950,35 +894,27 @@
         </state>
       </provider>
     </entry>
-    <entry file="file://$PROJECT_DIR$/requirements.txt">
-      <provider selected="true" editor-type-id="text-editor">
-        <state relative-caret-position="-480" />
-      </provider>
-    </entry>
-    <entry file="file://$PROJECT_DIR$/jianting.py">
+    <entry file="file://$PROJECT_DIR$/spiders/liansai.py">
       <provider selected="true" editor-type-id="text-editor">
-        <state relative-caret-position="324">
-          <caret line="18" column="14" lean-forward="true" selection-start-line="18" selection-start-column="14" selection-end-line="18" selection-end-column="14" />
+        <state relative-caret-position="270">
+          <caret line="21" column="31" selection-start-line="21" selection-start-column="31" selection-end-line="21" selection-end-column="31" />
           <folding>
-            <element signature="e#0#9#0" expanded="true" />
+            <element signature="e#24#35#0" expanded="true" />
           </folding>
         </state>
       </provider>
     </entry>
-    <entry file="file://$PROJECT_DIR$/spiders/liansai.py">
+    <entry file="file://$USER_HOME$/AppData/Local/Programs/Python/Python36/Lib/site-packages/twisted/enterprise/adbapi.py">
       <provider selected="true" editor-type-id="text-editor">
-        <state relative-caret-position="23">
-          <caret line="21" column="31" selection-start-line="21" selection-start-column="31" selection-end-line="21" selection-end-column="31" />
-          <folding>
-            <element signature="e#24#35#0" expanded="true" />
-          </folding>
+        <state relative-caret-position="160">
+          <caret line="464" selection-start-line="464" selection-end-line="464" />
         </state>
       </provider>
     </entry>
     <entry file="file://$PROJECT_DIR$/items.py">
       <provider selected="true" editor-type-id="text-editor">
-        <state relative-caret-position="-2932">
-          <caret line="50" lean-forward="true" selection-start-line="50" selection-end-line="50" />
+        <state relative-caret-position="252">
+          <caret line="246" column="32" lean-forward="true" selection-start-line="246" selection-start-column="32" selection-end-line="246" selection-end-column="32" />
           <folding>
             <element signature="e#320#1230#0" />
             <element signature="e#1382#1528#0" />
@@ -986,17 +922,20 @@
         </state>
       </provider>
     </entry>
-    <entry file="file://$PROJECT_DIR$/settings.py">
+    <entry file="file://$PROJECT_DIR$/jianting.py">
       <provider selected="true" editor-type-id="text-editor">
-        <state relative-caret-position="153">
-          <caret line="117" lean-forward="true" selection-start-line="117" selection-end-line="117" />
+        <state relative-caret-position="306">
+          <caret line="17" column="30" lean-forward="true" selection-start-line="17" selection-start-column="30" selection-end-line="17" selection-end-column="30" />
+          <folding>
+            <element signature="e#0#9#0" expanded="true" />
+          </folding>
         </state>
       </provider>
     </entry>
     <entry file="file://$PROJECT_DIR$/spiders/zuqiu.py">
       <provider selected="true" editor-type-id="text-editor">
-        <state relative-caret-position="72">
-          <caret line="130" column="37" lean-forward="true" selection-start-line="130" selection-start-column="37" selection-end-line="130" selection-end-column="37" />
+        <state relative-caret-position="54">
+          <caret line="129" column="14" selection-start-line="129" selection-start-column="14" selection-end-line="129" selection-end-column="14" />
           <folding>
             <element signature="e#24#35#0" expanded="true" />
           </folding>
@@ -1005,24 +944,10 @@
     </entry>
     <entry file="file://$PROJECT_DIR$/pipelines.py">
       <provider selected="true" editor-type-id="text-editor">
-        <state relative-caret-position="265">
-          <caret line="392" column="50" lean-forward="true" selection-start-line="392" selection-start-column="50" selection-end-line="392" selection-end-column="50" />
+        <state relative-caret-position="-144">
+          <caret line="45" column="29" lean-forward="true" selection-start-line="45" selection-start-column="29" selection-end-line="45" selection-end-column="29" />
           <folding>
-            <element signature="e#193#208#0" expanded="true" />
-            <element signature="e#18330#19014#0" />
-            <element signature="e#25714#29974#0" />
-            <element signature="e#34571#39061#0" />
-            <element signature="e#39126#43615#0" />
-            <element signature="e#43680#48166#0" />
-            <element signature="e#48346#53071#0" />
-            <element signature="e#53251#58051#0" />
-            <element signature="e#58092#58573#0" />
-            <element signature="e#58725#74468#0" />
-            <element signature="e#74621#82064#0" />
-            <element signature="e#82218#92209#0" />
-            <element signature="e#92365#99910#0" />
-            <element signature="e#100064#109753#0" />
-            <element signature="e#115021#121360#0" />
+            <element signature="e#0#15#0" expanded="true" />
           </folding>
         </state>
       </provider>
@@ -1030,12 +955,26 @@
     <entry file="file://$PROJECT_DIR$/main.py">
       <provider selected="true" editor-type-id="text-editor">
         <state relative-caret-position="234">
-          <caret line="13" lean-forward="true" selection-start-line="13" selection-end-line="13" />
+          <caret line="13" column="41" lean-forward="true" selection-start-line="13" selection-start-column="41" selection-end-line="13" selection-end-column="41" />
           <folding>
             <element signature="e#0#10#0" expanded="true" />
           </folding>
         </state>
       </provider>
     </entry>
+    <entry file="file://$PROJECT_DIR$/requirements.txt">
+      <provider selected="true" editor-type-id="text-editor">
+        <state relative-caret-position="735">
+          <caret line="62" selection-start-line="62" selection-end-line="62" />
+        </state>
+      </provider>
+    </entry>
+    <entry file="file://$PROJECT_DIR$/settings.py">
+      <provider selected="true" editor-type-id="text-editor">
+        <state relative-caret-position="226">
+          <caret line="129" column="28" lean-forward="true" selection-start-line="129" selection-start-column="28" selection-end-line="129" selection-end-column="28" />
+        </state>
+      </provider>
+    </entry>
   </component>
 </project>

hg3535/items.py

@@ -250,3 +250,74 @@ class Bangqiu(scrapy.Field):
     odd_evens_dict = scrapy.Field()
     odd_evens_dict_rule = scrapy.Field()
 
+
+# 滚球------------------------------------------------------------------------------------------------------------------
+class Roll_Zuqiu(scrapy.Field):
+    league_id = scrapy.Field()
+    league_name = scrapy.Field()
+    game_id = scrapy.Field()
+    team_home = scrapy.Field()
+    team_guest = scrapy.Field()
+    number = scrapy.Field()
+    data_game = scrapy.Field()
+    time_game = scrapy.Field()
+    half_way = scrapy.Field()
+    pt = scrapy.Field()
+    corner_ball = scrapy.Field()
+    zhuangtai = scrapy.Field()
+    # 球队进球大小
+    full_data = scrapy.Field()
+    half_data = scrapy.Field()
+    full_data_rule = scrapy.Field()
+    half_data_rule = scrapy.Field()
+    # 足球 大小盘
+    half_size_guest = scrapy.Field()
+    half_size_guest_rule = scrapy.Field()
+    half_size_home = scrapy.Field()
+    half_size_home_rule = scrapy.Field()
+
+    size_guest = scrapy.Field()
+    size_guest_rule = scrapy.Field()
+    size_home = scrapy.Field()
+    size_home_rule = scrapy.Field()
+    # 足球 让球&大小盘
+    half_concede_home_rule = scrapy.Field()
+    half_concede_home = scrapy.Field()
+    half_concede_guest_rule = scrapy.Field()
+    half_concede_guest = scrapy.Field()
+    # 足球 独赢
+    concede_guest = scrapy.Field()
+    concede_guest_rule = scrapy.Field()
+    concede_home = scrapy.Field()
+    concede_home_rule = scrapy.Field()
+    # 独赢
+    half_capot_home = scrapy.Field()
+    half_capot_guest = scrapy.Field()
+    half_capot_dogfall = scrapy.Field()
+    capot_home = scrapy.Field()
+    capot_guest = scrapy.Field()
+    capot_dogfall = scrapy.Field()
+    # 足球 入球数单双
+    odd_even_odd = scrapy.Field()
+    odd_even_even = scrapy.Field()
+    half_odd_even_odd = scrapy.Field()
+    half_odd_even_even = scrapy.Field()
+    # 足球 半场/全场
+    half_full = scrapy.Field()
+    # 足球 总入球
+    total_goal = scrapy.Field()
+    # 足球 波胆
+    bodan_data = scrapy.Field()
+    # 足球 最先进球/最后进球
+    first_last_ball = scrapy.Field()
+    score_guest = scrapy.Field()
+    score_home = scrapy.Field()
+
+
+class Zuqiustatus(scrapy.Field):
+    match_id = scrapy.Field()
+    create_time = scrapy.Field()
+    status = scrapy.Field()
+    update_time = scrapy.Field()
+    ball_type = scrapy.Field()
+    zuqiu_toal = scrapy.Field()

hg3535/main.py

@@ -5,9 +5,12 @@ from scrapy.cmdline import execute
 # print(os.path.dirname(os.path.abspath(__file__)))
 sys.path.append(os.path.dirname(os.path.abspath(__file__)))
 # execute(["scrapy", "crawl", "liansai"])
-execute(["scrapy", "crawl", "zuqiu"])
+# execute(["scrapy", "crawl", "zuqiu"])
 # execute(["scrapy", "crawl", "lanqiu"])
 # execute(["scrapy", "crawl", "guanjun"])
 # execute(["scrapy", "crawl", "wangqiu"])
 # execute(["scrapy", "crawl", "wqbodan"])
 # execute(["scrapy", "crawl", "bangqiu"])
+# execute(["scrapy", "crawl", "roll_zuqiu"]) #滚球
+# execute(["scrapy", "crawl", "ball_status"]) #滚球id本地存
+# execute(["scrapy", "crawl", "ball_status_update"]) #滚球id结束时间更新状态

hg3535/pipelines.py

@@ -43,6 +43,9 @@ import psycopg2.extras
 
 
 # 生成哈希索引 sole
+from items import Zuqiustatus
+
+
 def hash_func(match_id, odds_code, sort, p_id):
     m = hashlib.md5()
     a = str(match_id) + str(odds_code) + str(sort) + str(p_id)
@@ -191,7 +194,7 @@ def zqtwo_competition(data, cursor):
         data['team_home'], data['team_guest'], data['league_id'], data['game_id'], data['match_date'],
         data['match_time'],
         data['utime'], data['utime'], data['number'], data["source"], data['expire_time'], data['is_morningplate'],
-        data['us_time'], data['number'], data['expire_time'], data['is_morningplate'],data['us_time']))
+        data['us_time'], data['number'], data['expire_time'], data['is_morningplate'], data['us_time']))
 
 
 def zqthree_competition(data, cursor):
@@ -200,7 +203,7 @@ def zqthree_competition(data, cursor):
         data['team_home'], data['team_guest'], data['league_id'], data['game_id'], data['match_date'],
         data['match_time'],
         data['utime'], data['utime'], data['number'], data["source"], data['expire_time'], data['is_stringscene'],
-        data['us_time'], data['number'], data['expire_time'], data['is_stringscene'],data['us_time']))
+        data['us_time'], data['number'], data['expire_time'], data['is_stringscene'], data['us_time']))
 
 
 # 足球 插入赔率 让球 大小-------------------------------------------------------------------------------------------------
@@ -1463,4 +1466,428 @@ class Liansaipipeline(object):
     def close_spider(self, spider):
         self.conn.close()
         self.dbpool.close()
-        # self.redis_db.disconnect()
+        # self.redis_db.disconnect()
+
+
+
+
+
+#************************************滚球-------------------------------------------------------------------------------
+
+
+
+
+#滚球方法都加了s
+
+def zqone_intodbs(data1, data2, data3, data4, cursor, redis_db):
+    if data1:
+        for index, value in enumerate(data1):
+            if value:
+                new_hash = hash_func(match_id=data2['game_id'], odds_code=data3, sort=index, p_id=data2['p_id'])
+                r_hash = r_func(match_id=data2['game_id'], odds_code=data3, sort=index, p_id=data2['p_id'], odd=value)
+                odds_only = r_hash
+                if redis_db.hexists("hg3535_zuqiu", r_hash):
+                    pass
+                else:
+                    redis_db.hset("hg3535_zuqiu", r_hash, 0)
+                    # if data2['pt'] == 0:
+                    sql1 = "insert into st_zq_odds(lg_id, odds_code, match_id, ctime, utime, odds, p_id, p_code, sort, source, sole, condition, expire_time,odds_only,is_rollball) values (%s,%s,%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s) on conflict(sole) do update set utime=%s,odds=%s,expire_time=%s,odds_only=%s,condition=%s;"
+                    cursor.execute(sql1, (
+                        int(data2['league_id']), data3, int(data2['game_id']), data2['utime'], data2['utime'], float(value),
+                        data2['p_id'], data2["p_code"], index, data2["source"], new_hash,
+                        data4[index], data2['expire_time'], odds_only, 1, data2['utime'], value, data2['expire_time'],
+                        odds_only,data4[index]))
+
+                    # 更新主队st_zq_odds_record表
+                    sql2 = "insert into st_zq_odds_record(lg_id, odds_code, match_id, ctime, utime, odds, p_id, p_code, sort, source, condition,odds_only,is_rollball) values (%s,%s, %s, %s, %s, %s, %s, %s,%s, %s, %s, %s, %s);"
+                    cursor.execute(sql2, (
+                        int(data2['league_id']), data3, int(data2['game_id']), data2['utime'], data2['utime'], float(value),
+                        data2['p_id'], data2["p_code"], index, data2["source"], data4[index], odds_only,1))
+                    #写入数据设置默认值竟然不生效 待看
+                    sql3 = "insert into st_zq_result(match_id, match_score, home_team, guest_team, home_score, guest_score, all_goal, status, first_score, last_score, match_winer, update_time, match_time,match_process,tag,lg_id,u_home_score,u_guest_score,source,p_code) values (%s,%s,%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s,%s,%s,%s,%s,%s) on conflict(match_id) do update set update_time=%s,match_score=%s,match_time=%s,match_process=%s,home_score=%s,guest_score=%s;"
+                    cursor.execute(sql3, (
+                        int(data2['game_id']), data2['match_score'],data2['team_home'],data2['team_guest'],data2['score_home'],data2['score_guest'],data2['number'],1,'','','',data2['utime'],data2['time_game'],data2['half_way'],data2['number'],int(data2['league_id']),0,0,'hg3535',data2["p_code"],data2['utime'],data2['match_score'],data2['time_game'],data2['half_way'],data2['score_home'],data2['score_guest']
+                    ))
+
+                    sql4 = "insert into st_zq_result_record(lg_id, home_team, guest_team, home_score, guest_score, all_goal, status, first_score, last_score, match_score, match_winer, update_time, match_time,match_process,tag,match_id,source,p_code) values (%s,%s,%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s,%s,%s,%s);"
+                    cursor.execute(sql4, (
+                        int(data2['league_id']), data2['team_home'],data2['team_guest'],data2['score_home'],data2['score_guest'],data2['number'],1,'','',data2['match_score'],'',data2['utime'],data2['time_game'],data2['half_way'],data2['number'],int(data2['game_id']),'hg3535',data2["p_code"]
+                    ))
+
+def zqtwo_intodbs(data1, data2, data3, data4, cursor, redis_db):
+    if data1:
+        new_hash = hash_func(match_id=data2['game_id'], odds_code=data3, sort=data2['sort'], p_id=data2['p_id'])
+        r_hash = r_func(match_id=data2['game_id'], odds_code=data3, sort=data2['sort'], p_id=data2['p_id'],
+                        odd=data1)
+        odds_only = r_hash
+        if redis_db.hexists("hg3535_zuqiu", r_hash):
+            pass
+        else:
+            redis_db.hset("hg3535_zuqiu", r_hash, 0)
+            # if data2['pt'] == 1:
+            sql1 = "insert into st_zq_odds(lg_id, odds_code, match_id, ctime, utime, odds, p_id, p_code, sort, source, sole, condition, expire_time, odds_only,is_today) values (%s,%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s) on conflict(sole) do update set utime=%s,odds=%s,expire_time=%s,odds_only=%s,condition=%s;"
+            cursor.execute(sql1, (
+                int(data2['league_id']), data3, int(data2['game_id']), data2['utime'], data2['utime'], data1,
+                data2['p_id'], data2["p_code"], data2['sort'], data2["source"], new_hash,
+                data4, data2['expire_time'], odds_only, 1, data2['utime'], data1, data2['expire_time'], odds_only,data4))
+            # 更新主队st_zq_odds_record表
+            sql2 = "insert into st_zq_odds_record(lg_id, odds_code, match_id, ctime, utime, odds, p_id, p_code, sort, source, condition,odds_only, is_today) values (%s,%s,%s, %s, %s, %s, %s, %s,%s, %s, %s, %s, %s);"
+            cursor.execute(sql2, (
+                int(data2['league_id']), data3, int(data2['game_id']), data2['utime'], data2['utime'], data1,
+                data2['p_id'], data2["p_code"], data2['sort'], data2["source"], data4, odds_only, 1))
+
+            sql3 = "insert into st_zq_result(match_id, match_score, home_team, guest_team, home_score, guest_score, all_goal, status, first_score, last_score, match_winer, update_time, match_time,match_process,tag,lg_id,u_home_score,u_guest_score,source,p_code) values (%s,%s,%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s,%s,%s,%s,%s,%s) on conflict(match_id) do update set update_time=%s,match_score=%s,match_time=%s,match_process=%s,home_score=%s,guest_score=%s;"
+            cursor.execute(sql3, (
+                    int(data2['game_id']), data2['match_score'],data2['team_home'],data2['team_guest'],data2['score_home'],data2['score_guest'],data2['number'],1,'','','',data2['utime'],data2['time_game'],data2['half_way'],data2['number'],int(data2['league_id']),0,0,'hg3535',data2["p_code"],data2['utime'],data2['match_score'],data2['time_game'],data2['half_way'],data2['score_home'],data2['score_guest']
+                ))
+
+            sql4 = "insert into st_zq_result_record(lg_id, home_team, guest_team, home_score, guest_score, all_goal, status, first_score, last_score, match_score, match_winer, update_time, match_time,match_process,tag,match_id,source,p_code) values (%s,%s,%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s,%s,%s,%s);"
+            cursor.execute(sql4, (
+                    int(data2['league_id']), data2['team_home'],data2['team_guest'],data2['score_home'],data2['score_guest'],data2['number'],1,'','',data2['match_score'],'',data2['utime'],data2['time_game'],data2['half_way'],data2['number'],int(data2['game_id']),'hg3535',data2["p_code"]
+                ))
+
+
+def zqone_competitions(data, cursor):
+    Competition_sql = "insert into st_zq_competition(home_team, guest_team, lg_id, status, match_id, match_date, match_time, ctime, utime,tag, source,type,expire_time,us_time,is_rollball) values (%s, %s, %s, %s, %s, %s, %s,%s, %s, %s,%s,%s,%s,%s,%s) on conflict(match_id) do update set utime = %s,status=%s,tag=%s,expire_time=%s;"
+    cursor.execute(Competition_sql, (
+        data['team_home'], data['team_guest'], data['league_id'],1,data['game_id'], data['match_date'],
+        data['match_time'],
+        data['utime'], data['utime'], data['number'], data["source"], 4,data['expire_time'], data['us_time'],data['is_rollball'],
+        data['utime'], 1, data['number'], data['expire_time']))
+
+
+def get_pcodes(corner_ball, code):
+    code_dict = {'concede_size': 1, 'capot': 2, 'two_sides': 3, 'total_goal': 4, 'half_full': 5, 'bodan': 6,
+                 'first_last_ball': 7, 'temaball': 11}
+    if corner_ball == "角球":
+        p_code = "corner_ball"
+        p_id = 9
+    elif corner_ball == "会晋级":
+        p_code = "promotion"
+        p_id = 10
+    elif corner_ball == "罚牌数":
+        p_code = "Penalty_card"
+        p_id = 12
+    else:
+        p_code = code
+        p_id = code_dict[code]
+    return p_code, p_id
+
+
+def news_times(ctime):
+    time1 = time.mktime(time.strptime(ctime, '%Y-%m-%d %H:%M:%S')) + 43200
+    time2 = time.localtime(time1)
+    time3 = time.strftime('%Y-%m-%d %H:%M:%S', time2)
+    time4 = time3
+    data_time = str(time4).split(" ")
+    match_date = data_time[0]
+    match_time = data_time[1]
+    return match_date, match_time, time3
+
+
+class Roll_Zuqiupipeline(object):
+    def __init__(self, dbpool, conn, cursor, redis_db):
+        self.dbpool = dbpool
+        self.conn = conn
+        self.cursor = cursor
+        self.redis_db = redis_db
+
+    @classmethod
+    def from_settings(cls, settings):
+        dbparms = dict(
+            host=settings["POST_HOST"],
+            user=settings["POST_USER"],
+            password=settings["POST_PASSWORD"],
+            port=settings['POST_PORT'],
+        )
+        dbpool = adbapi.ConnectionPool("psycopg2", **dbparms)
+        conn = psycopg2.connect(host=settings["POST_HOST"], port=settings['POST_PORT'], user=settings["POST_USER"], password=settings["POST_PASSWORD"], database=settings["POST_DATABASE"])
+        cursor = conn.cursor(cursor_factory=psycopg2.extras.DictCursor)
+        pool = redis.ConnectionPool(host=settings["R_HOST"], port=settings["R_POST"], password=settings["R_PASSWORD"])
+        redis_db = redis.StrictRedis(connection_pool=pool)
+        return cls(dbpool, conn, cursor, redis_db)
+
+    def process_item(self, item, spider):
+        # 使用twisted将mysql插入变成异步执行
+        query = self.dbpool.runInteraction(self.do_insert, item)
+        query.addErrback(self.handle_error, item, spider)  # 处理异常
+        # return item
+
+    def handle_error(self, failure, item, spider):
+        # 处理异步插入的异常
+        print(failure)
+
+    def do_insert(self, cursor, item):
+        # 比赛日期
+        data_game = item['data_game'].split("/")
+        month = str(data_game[1].strip())
+        day = str(data_game[0])
+        # 比赛时间
+        time_game = str(item['time_game'])
+        # 比赛时间，时间戳
+        ctime = "2019" + "-" + month + "-" + day + "" + time_game + ":00"
+        r_ctime = "2019" + "-" + month + "-" + day
+        # 现在时间，时间戳
+        utime = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime())
+        expire_time = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(time.time() + 60))
+        # 比赛id
+        competition_id = item['game_id']
+        # 联赛id
+        league_id = item['league_id']
+        # 联赛name
+        league_name = item['league_name']
+        # 主队
+        team_home = item['team_home']
+        # 客队
+        team_guest = item['team_guest']
+        # number
+        number = item['number']
+        score_home = item['score_home']
+        score_guest = item['score_guest']
+        half_way = item['half_way']
+        pt = item['pt']
+        match_score = "{}:{}".format(score_home,score_guest)
+        corner_ball = item['corner_ball']
+        p_code, p_id = get_pcode(corner_ball, 'concede_size')
+        # 构建唯一索引
+        half_size_guest = item["half_size_guest"]
+        half_size_guest_rule = item["half_size_guest_rule"]
+        half_size_home = item["half_size_home"]
+        half_size_home_rule = item["half_size_home_rule"]
+
+        size_data = {'league_id': league_id, 'game_id': competition_id, 'utime': utime, 'p_id': p_id, 'p_code': p_code,
+                     'source': "hg3535", 'expire_time': expire_time, 'pt': pt,'match_score':match_score,'team_home':team_home,'team_guest':team_guest,'score_home':score_home,'score_guest':score_guest,'number':number,'time_game':time_game,'half_way':half_way}
+        # 让球 数据插入数据库
+        zqone_intodbs(data1=half_size_home, data2=size_data, data3="half_size_home", data4=half_size_home_rule,
+                     cursor=cursor, redis_db=self.redis_db)
+        # 让球 数据插入数据库
+        zqone_intodbs(data1=half_size_guest, data2=size_data, data3="half_size_guest", data4=half_size_guest_rule,
+                     cursor=cursor, redis_db=self.redis_db)
+        # 全场场大小
+        size_guest = item["size_guest"]
+        size_guest_rule = item["size_guest_rule"]
+        size_home = item["size_home"]
+        size_home_rule = item["size_home_rule"]
+        zqone_intodbs(data1=size_guest, data2=size_data, data3="size_guest", data4=size_guest_rule,
+                     cursor=cursor, redis_db=self.redis_db)
+        zqone_intodbs(data1=size_home, data2=size_data, data3="size_home", data4=size_home_rule,
+                     cursor=cursor, redis_db=self.redis_db)
+        # 上半场大小
+        half_concede_home_rule = item["half_concede_home_rule"]
+        half_concede_home = item["half_concede_home"]
+        half_concede_guest_rule = item["half_concede_guest_rule"]
+        half_concede_guest = item["half_concede_guest"]
+        # 上半场让球
+        zqone_intodbs(data1=half_concede_home, data2=size_data, data3="half_concede_home", data4=half_concede_home_rule,
+                     cursor=cursor, redis_db=self.redis_db)
+        zqone_intodbs(data1=half_concede_guest, data2=size_data, data3="half_concede_guest", data4=half_concede_guest_rule, cursor=cursor, redis_db=self.redis_db)
+
+        concede_guest = item["concede_guest"]
+        concede_guest_rule = item["concede_guest_rule"]
+        concede_home = item["concede_home"]
+        concede_home_rule = item["concede_home_rule"]
+        # 全场让球
+        zqone_intodbs(data1=concede_guest, data2=size_data, data3="concede_guest", data4=concede_guest_rule,
+                     cursor=cursor, redis_db=self.redis_db)
+        zqone_intodbs(data1=concede_home, data2=size_data, data3="concede_home", data4=concede_home_rule,
+                     cursor=cursor,redis_db=self.redis_db)
+        #
+        p_code, p_id = get_pcode(corner_ball, 'capot')
+        # 独赢----------------------------------------------------------------------------------------------------------
+        half_capot_home = item["half_capot_home"]
+        half_capot_guest = item["half_capot_guest"]
+        half_capot_dogfall = item["half_capot_dogfall"]
+
+        capot_home = item["capot_home"]
+        capot_guest = item["capot_guest"]
+        capot_dogfall = item["capot_dogfall"]
+        capot_data = {'league_id': league_id, 'game_id': competition_id, 'utime': utime, 'p_id': p_id, 'p_code': p_code,
+                      'source': "hg3535", 'expire_time': expire_time, 'sort': 0, 'pt': pt,'match_score':match_score,'team_home':team_home,'team_guest':team_guest,'score_home':score_home,'score_guest':score_guest,'number':number,'time_game':time_game,'half_way':half_way}
+        # 上半场独赢 主队
+        zqtwo_intodbs(data1=half_capot_home, data2=capot_data, data3="half_capot_home", data4='1', cursor=cursor,redis_db=self.redis_db)
+        # 上半场独赢 客队
+        zqtwo_intodbs(data1=half_capot_guest, data2=capot_data, data3="half_capot_guest", data4='2', cursor=cursor,redis_db=self.redis_db)
+        # 上半场独赢 和
+        zqtwo_intodbs(data1=half_capot_dogfall, data2=capot_data, data3="half_capot_dogfall", data4='x', cursor=cursor,redis_db=self.redis_db)
+        #
+        # 全场独赢 主队
+        zqtwo_intodbs(data1=capot_home, data2=capot_data, data3="capot_home", data4='1', cursor=cursor,redis_db=self.redis_db)
+        # 全场独赢 客队
+        zqtwo_intodbs(data1=capot_guest, data2=capot_data, data3="capot_guest", data4='2', cursor=cursor,redis_db=self.redis_db)
+        # 全场独赢 和
+        zqtwo_intodbs(data1=capot_dogfall, data2=capot_data, data3="capot_dogfall", data4='x', cursor=cursor,redis_db=self.redis_db)
+        #
+# 入球数单双-------------------------------------------------------------------------------------------------------------
+        p_code, p_id = get_pcode(corner_ball, 'two_sides')
+        odd_even_odd = item["odd_even_odd"]
+        odd_even_even = item["odd_even_even"]
+        half_odd_even_odd = item["half_odd_even_odd"]
+        half_odd_even_even = item["half_odd_even_even"]
+        # 全场入球数 单双
+        two_sides_data = {'league_id': league_id, 'game_id': competition_id, 'utime': utime, 'p_id': p_id,
+                          'p_code': p_code,
+                          'source': "hg3535", 'expire_time': expire_time, 'sort': 0, 'pt': pt,'match_score':match_score,'team_home':team_home,'team_guest':team_guest,'score_home':score_home,'score_guest':score_guest,'number':number,'time_game':time_game,'half_way':half_way}
+        # 上半场入球数 单双
+        zqtwo_intodbs(data1=odd_even_odd, data2=two_sides_data, data3="two_sides_single", data4='单', cursor=cursor,redis_db=self.redis_db)
+        zqtwo_intodbs(data1=odd_even_even, data2=two_sides_data, data3="two_sides_double", data4='双', cursor=cursor,redis_db=self.redis_db)
+        # 全场入球数 单双
+        zqtwo_intodbs(data1=half_odd_even_odd, data2=two_sides_data, data3="half_two_sides_single", data4='单',
+                     cursor=cursor, redis_db=self.redis_db)
+        zqtwo_intodbs(data1=half_odd_even_even, data2=two_sides_data, data3="half_two_sides_double", data4='双',
+                     cursor=cursor, redis_db=self.redis_db)
+
+# 总入球数 --------------------------------------------------------------------------------------------------------------
+        p_code, p_id = get_pcode(corner_ball, 'total_goal')
+        total_goals = item['total_goal']
+        total_dict = {'total_goal_zero': '0-1', 'total_goal_two': '2-3', 'total_goal_four': '4-6',
+                      'total_goal_seven': '7或以上', 'half_total_goal_zero': '0', "half_total_goal_one": '1',
+                      "half_total_goal_two": '2', "half_total_goal_three": '3或以上'}
+        # 全场入球数 单双
+        total_goal_data = {'league_id': league_id, 'game_id': competition_id, 'utime': utime, 'p_id': p_id,
+                           'p_code': p_code,
+                           'source': "hg3535", 'expire_time': expire_time, 'sort': 0, 'pt': pt,'match_score':match_score,'team_home':team_home,'team_guest':team_guest,'score_home':score_home,'score_guest':score_guest,'number':number,'time_game':time_game,'half_way':half_way}
+        # 上半场入球数 单双
+        for key, value in total_goals.items():
+            if value:
+                zqtwo_intodbs(data1=value, data2=total_goal_data, data3=key, data4=total_dict[key], cursor=cursor, redis_db=self.redis_db)
+
+# 全场半场 --------------------------------------------------------------------------------------------------------------
+        half_fulls = item['half_full']
+        p_code, p_id = get_pcode(corner_ball, 'half_full')
+        full_dict = {"half_full_home_home": "主主", "half_full_home_dogfall": "主和",
+                     "half_full_home_guest": "主客", "half_full_dogfall_home": "和主",
+                     "half_full_dogfall_dogfall": "和和", "half_full_dogfall_guest": "和客",
+                     "half_full_guest_home": "客主", "half_full_guest_dogfall": "客和",
+                     "half_full_guest_guest": "客客"}
+        half_full_data = {'league_id': league_id, 'game_id': competition_id, 'utime': utime, 'p_id': p_id,
+                          'p_code': p_code,
+                          'source': "hg3535", 'expire_time': expire_time, 'sort': 0, 'pt': pt,'match_score':match_score,'team_home':team_home,'team_guest':team_guest,'score_home':score_home,'score_guest':score_guest,'number':number,'time_game':time_game,'half_way':half_way}
+        if half_fulls:
+            for key, value in half_fulls.items():
+                if value:
+                    zqtwo_intodbs(data1=value, data2=half_full_data, data3=key, data4=full_dict[key], cursor=cursor, redis_db=self.redis_db)
+# 波胆------------------------------------------------------------------------------------------------------------------
+        bodan_datas = item['bodan_data']
+        p_code, p_id = get_pcode(corner_ball, 'bodan')
+        bodan_dict = {"bodanhome_one_zero": "1-0", "bodanhome_two_zero": "2-0",
+                      "bodanhome_two_one": "2-1", "bodanhome_three_zero": "3-0",
+                      "bodanhome_three_one": "3-1", "bodanhome_three_two": "3-2",
+                      "bodanhome_four_zero": "4-0", "bodanhome_four_one": "4-1",
+                      "bodanhome_four_two": "4-2", "bodanhome_four_three": "4-3",
+                      "bodanguest_one_zero": "0-1", "bodanguest_two_zero": "0-2",
+                      "bodanguest_two_one": "1-2", "bodanguest_three_zero": "0-3",
+                      "bodanguest_three_one": "1-3", "bodanguest_three_two": "2-3",
+                      "bodanguest_four_zero": "0-4", "bodanguest_four_one": "1-4",
+                      "bodanguest_four_two": "2-4", "bodanguest_four_three": "3-4",
+                      "bodandogfall_zero_zero": "0-0", "bodandogfall_one_one": "1-1",
+                      "bodandogfall_two_two": "2-2", "bodandogfall_three_three": "3-3",
+                      "bodandogfall_four_four": "4-4", "bodanother": "其他",
+                      "halfbodanhome_one_zero": "1-0", "halfbodanhome_two_zero": "2-0",
+                      "halfbodanhome_two_one": "2-1", "halfbodanhome_three_zero": "3-0",
+                      "halfbodanhome_three_one": "3-1", "halfbodanhome_three_two": "3-2",
+                      "halfbodanguest_one_zero": "0-1", "halfbodanguest_two_zero": "0-2",
+                      "halfbodanguest_two_one": "1-2", "halfbodanguest_three_zero": "0-3",
+                      "halfbodanguest_three_one": "1-3", "halfbodanguest_three_two": "2-3",
+                      "halfbodandogfall_zero_zero": "0-0", "halfbodandogfall_one_one": "1-1",
+                      "halfbodandogfall_two_two": "2-2", "halfbodandogfall_three_three": "3-3",
+                      "halfbodanother": "其他"}
+        bodan_data = {'league_id': league_id, 'game_id': competition_id, 'utime': utime, 'p_id': p_id, 'p_code': p_code,
+                      'source': "hg3535", 'expire_time': expire_time, 'sort': 0, 'pt': pt,'match_score':match_score,'team_home':team_home,'team_guest':team_guest,'score_home':score_home,'score_guest':score_guest,'number':number,'time_game':time_game,'half_way':half_way}
+        if bodan_datas:
+            for key, value in bodan_datas.items():
+                zqtwo_intodbs(data1=value, data2=bodan_data, data3=key, data4=bodan_dict[key], cursor=cursor, redis_db=self.redis_db)
+
+# 最先进球/最后进球 ------------------------------------------------------------------------------------------------------
+        first_last_balls = item['first_last_ball']
+        p_code, p_id = get_pcode(corner_ball, 'first_last_ball')
+        first_last_dict = {"first_last_ball": "最先进球", "first_ball_home": "最先进球", "first_ball_guest": "最先进球",
+                           "last_ball_home": "最后进球", "last_ball_guest": "最后进球", "not_ball": "没有进球"}
+        first_last_data = {'league_id': league_id, 'game_id': competition_id, 'utime': utime, 'p_id': p_id,
+                           'p_code': p_code,
+                           'source': "hg3535", 'expire_time': expire_time, 'sort': 0, 'pt': pt,'match_score':match_score,'team_home':team_home,'team_guest':team_guest,'score_home':score_home,'score_guest':score_guest,'number':number,'time_game':time_game,'half_way':half_way}
+        if first_last_balls:
+            for key, value in first_last_balls.items():
+                # 构建唯一索引
+                zqtwo_intodbs(data1=value, data2=first_last_data, data3=key, data4=first_last_dict[key], cursor=cursor, redis_db=self.redis_db)
+
+        p_code, p_id = get_pcode(corner_ball, 'temaball')
+        full_dicts = item['full_data']
+        half_dicts = item['half_data']
+        full_dict_rules = item['full_data_rule']
+        half_dict_rules = item['half_data_rule']
+        data = {'league_id': league_id, 'game_id': competition_id, 'utime': utime, 'p_id': p_id, 'p_code': p_code,
+                'source': "hg3535", 'expire_time': expire_time, 'sort': 0, 'pt': pt,'match_score':match_score,'team_home':team_home,'team_guest':team_guest,'score_home':score_home,'score_guest':score_guest,'number':number,'time_game':time_game,'half_way':half_way}
+        if full_dicts:
+            for key, value in full_dicts.items():
+                zqtwo_intodbs(data1=value, data2=data, data3=key, data4=full_dict_rules[key], cursor=cursor, redis_db=self.redis_db)
+
+        if half_dicts:
+            for key, value in half_dicts.items():
+                zqtwo_intodbs(data1=value, data2=data, data3=key, data4=half_dict_rules[key], cursor=cursor, redis_db=self.redis_db)
+        #这日期在这里有问题 先行注释
+        # match_date, match_time, time3 = news_time(ctime)
+        # n_time = out_time(time3, 1.5)
+        # 插入st_zq_competition表
+        # if pt == 0:
+        data_competition = {'team_home': team_home, 'team_guest': team_guest, 'league_id': league_id,
+                            'game_id': competition_id,
+                            'match_date': r_ctime, 'match_time': '00:00:00', 'utime': utime, 'number': number,
+                            'source': "hg3535", "expire_time": expire_time, 'is_rollball': 1, "us_time":'2019-01-01 00:00:00','time_game':time_game}
+        zqone_competitions(data=data_competition, cursor=cursor)
+
+
+
+class BallStatuspipeline(object):
+    def __init__(self, dbpool, conn, cursor):
+        self.dbpool = dbpool
+        self.conn = conn
+        self.cursor = cursor
+
+    @classmethod
+    def from_settings(cls, settings):
+        dbparms = dict(
+            host=settings["POST_HOST"],
+            # db = settings["POST_DATABASE"],
+            user=settings["POST_USER"],
+            password=settings["POST_PASSWORD"],
+            port=settings['POST_PORT'],
+        )
+        dbpool = adbapi.ConnectionPool("psycopg2", **dbparms)
+        conn = psycopg2.connect(host=settings["POST_HOST"], port=settings['POST_PORT'], user=settings["POST_USER"],password=settings["POST_PASSWORD"],database=settings["POST_DATABASE"])
+        cursor = conn.cursor(cursor_factory=psycopg2.extras.DictCursor)
+        return cls(dbpool,conn,cursor)
+
+    def process_item(self, item, spider):
+        # 使用twisted将mysql插入变成异步执行
+        query = self.dbpool.runInteraction(self.do_insert, item)
+        query.addErrback(self.handle_error, item, spider)  # 处理异常
+
+    def handle_error(self, failure, item, spider):
+        # 处理异步插入的异常
+        print(failure)
+
+    def do_insert(self, cursor, item):
+        # 联赛id
+        if isinstance(item, Zuqiustatus):
+            zuqiu_toal = item['zuqiu_total']
+            for i in zuqiu_toal:
+                print(i)
+                match_id = i['match_id']
+                create_time = i['create_time']
+                staus = i['status']
+                update_time = i['create_time']
+                ball_type = i['ball_type']
+                sql1 = "insert into st_ball_status(match_id, create_time, status,update_time, ball_type) values (%s,%s, %s, %s, %s) on conflict(match_id) do update set update_time = %s;"
+                cursor.execute(sql1, (match_id, create_time,staus,update_time, ball_type,update_time))
+                sql2 = "update st_zq_result set status=1  where match_id={}".format(match_id)
+                sql3 = "update st_zq_result_record set status=1  where match_id={}".format(match_id)
+                sql4 = "update st_zq_competition set status=1  where match_id={}".format(match_id)
+                cursor.execute(sql2)
+                cursor.execute(sql3)
+                cursor.execute(sql4)
+
+        # 现在时间，时间戳
+        utime = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime())
+    def close_spider(self, spider):
+        self.conn.close()
+        self.dbpool.close()

hg3535/settings.py

@@ -126,15 +126,23 @@ REDIRECT_ENABLED = False
 # RETRY_TIMES = 2
 # RETRY_HTTP_CODES = [500, 502, 503, 504, 400, 408]
 
-POST_HOST = 'localhost'
-POST_DATABASE = 'postgres'
-POST_USER = 'postgres'
-POST_PORT = '5433'
-POST_PASSWORD = '123456'
+# POST_HOST = 'localhost'
+# POST_DATABASE = 'postgres'
+# POST_USER = 'postgres'
+# POST_PORT = '5433'
+# POST_PASSWORD = '123456'
+
+
+POST_HOST = '172.17.0.4'
+POST_DATABASE = 'kaiyou'
+POST_USER = 'kaiyou'
+POST_PORT = '5432'
+POST_PASSWORD = 'yjkj8888'
+
 
 R_HOST = 'localhost'
 R_POST = '6379'
-R_PASSWORD = '123456'
+#R_PASSWORD = '123456'
 
 # POST_HOST = '192.168.2.200'
 # POST_DATABASE = 'kaiyou'

hg3535/spiders/hg3535_zq_status.py

@@ -0,0 +1,233 @@
+# -*- coding: utf-8 -*-
+import json
+
+import jsonpath
+import scrapy
+import time
+from scrapy.http import Request
+import psycopg2
+import time
+from functools import wraps
+from contextlib import contextmanager
+import psycopg2.extras
+
+from ..items import Zuqiustatus
+
+import json
+from datetime import datetime
+from datetime import date
+import itertools
+
+
+# 测试一个函数的运行时间，使用方式：在待测函数直接添加此修饰器
+def timethis(func):
+    @wraps(func)
+    def wrapper(*args, **kwargs):
+        start = time.perf_counter()
+        r = func(*args, **kwargs)
+        end = time.perf_counter()
+        print('\n============================================================')
+        print('{}.{} : {}'.format(func.__module__, func.__name__, end - start))
+        print('============================================================\n')
+        return r
+    return wrapper
+
+
+# 测试一段代码运行的时间，使用方式：上下文管理器with
+# with timeblock('block_name'):
+#     your_code_block...
+@contextmanager
+def timeblock(label='Code'):
+    start = time.perf_counter()
+    try:
+        yield
+    finally:
+        end = time.perf_counter()
+        print('==============================================================')
+        print('{} run time: {}'.format(label, end - start))
+        print('==============================================================')
+
+
+class SqlConn():
+    '''
+    连接数据库，以及进行一些操作的封装
+    '''
+    sql_name = ''
+    database = ''
+    user = ''
+    password = ''
+    port = 0
+    host = ''
+
+    # 创建连接、游标
+    def __init__(self, *args, **kwargs):
+        if kwargs.get("sql_name"):
+            self.sql_name = kwargs.get("sql_name")
+        if kwargs.get("database"):
+            self.database = kwargs.get("database")
+        if kwargs.get("user"):
+            self.user = kwargs.get("user")
+        if kwargs.get("password"):
+            self.password = kwargs.get("password")
+        if kwargs.get("port"):
+            self.port = kwargs.get("port")
+        if kwargs.get("host"):
+            self.host = kwargs.get("host")
+
+        if not (self.host and self.port and self.user and
+                self.password and self.database):
+            raise Warning("conn_error, missing some params!")
+
+        sql_conn = {
+                    'postgresql': psycopg2,
+                    }
+
+        self.conn = sql_conn[self.sql_name].connect(host=self.host,
+                                                    port=self.port,
+                                                    user=self.user,
+                                                    password=self.password,
+                                                    database=self.database,
+                                                    )
+        self.cursor = self.conn.cursor(cursor_factory=psycopg2.extras.DictCursor)
+        if not self.cursor:
+            raise Warning("conn_error!")
+
+    # 测试连接
+    def test_conn(self):
+        if self.cursor:
+            print("conn success!")
+        else:
+            print('conn error!')
+
+    # 单条语句的并提交
+    def execute(self, sql_code):
+        self.cursor.execute(sql_code)
+        self.conn.commit()
+
+    # 单条语句的不提交
+    def execute_no_conmmit(self, sql_code):
+        self.cursor.execute(sql_code)
+
+    # 构造多条语句，使用%s参数化，对于每个list都进行替代构造
+    def excute_many(self, sql_base, param_list):
+        self.cursor.executemany(sql_base, param_list)
+
+    # 批量执行（待完善）
+    def batch_execute(self, sql_code):
+        pass
+
+    # 获取数据
+    def get_data(self, sql_code, count=0):
+        self.cursor.execute(sql_code)
+        if int(count):
+            return self.cursor.fetchmany(count)
+        else:
+            return self.cursor.fetchall()
+
+    # 更新数据
+    def updata_data(self, sql_code):
+        self.cursor(sql_code)
+
+    # 插入数据
+    def insert_data(self, sql_code):
+        self.cursor(sql_code)
+
+    # 滚动游标
+    def cursor_scroll(self, count, mode='relative'):
+        self.cursor.scroll(count, mode=mode)
+
+    # 提交
+    def commit(self):
+        self.conn.commit()
+
+    # 回滚
+    def rollback(self):
+        self.conn.rollback()
+
+    # 关闭连接
+    def close_conn(self):
+        self.cursor.close()
+        self.conn.close()
+
+class ComplexEncoder(json.JSONEncoder):
+    def default(self, obj):
+        if isinstance(obj, datetime):
+            return obj.strftime('%Y-%m-%d %H:%M:%S')
+        elif isinstance(obj, date):
+            return obj.strftime('%Y-%m-%d')
+        else:
+            return json.JSONEncoder.default(self, obj)
+
+
+
+
+
+
+
+
+class LanqiuSpider(scrapy.Spider):
+    name = "ball_status"
+    allowed_domains = ['hg3535z.com']
+    #sid要改为1 足球 现在测试改为4
+    start_urls = ['https://hg3535z.com/odds2/d/getodds?sid=1&pt=4&ubt=am&pn=0&sb=2&dc=null&pid=0'] # 滚球菜单 篮球滚球列url
+    custom_settings = {
+        "ITEM_PIPELINES": {
+            'hg3535.pipelines.BallStatuspipeline':200,
+        }
+    }
+    # start_urls = ['http://hg3535z.com/odds2/d/getodds?sid=2&pt=3&ubt=am&pn=0&sb=2&dc=null&pid=0']
+    # http: // hg3535z.com / odds2 / d / getamodds?eid = 3098030 & iip = false & ubt = am & isp = false
+    # http://hg3535z.com/odds2/d/getodds?sid=2&pt=2&ubt=am&pn=0&sb=2&dc=null&pid=0
+
+    def parse(self, response):
+        datas = json.loads(response.text)
+        ids = jsonpath.jsonpath(datas, '$..i-ot[0]..egs..es..i[16]') # ids新列表
+        item = Zuqiustatus()
+        utime = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime())
+        # zuqiu_total = {}
+        zuqiu_status_list = []
+        if ids:
+            ids = set(ids)
+            for i in ids:
+                zuqiu = {}
+                zuqiu['match_id'] = i
+                zuqiu['create_time'] = utime
+                zuqiu['status'] = 1
+                zuqiu['ball_type'] = datas['i-ot'][0]['s']['n']
+                # item['match_id'] = i
+                # item['create_time'] = utime
+                # item['status'] = 1
+                # item['ball_type'] = datas['i-ot'][0]['s']['n']
+                # item['zuqiu_toal'] = zuqiu
+                zuqiu_status_list.append(zuqiu)
+            item["zuqiu_total"] = zuqiu_status_list
+            yield item
+
+        # urls = ['http://hg3535z.com/odds2/d/getodds?sid=2&pt=4&ubt=am&pn=0&sb=2&dc=null&pid=0''http://hg3535z.com/odds2/d/getodds?sid=3&pt=4&ubt=am&pn=0&sb=2&dc=null&pid=0','http://hg3535z.com/odds2/d/getodds?sid=4&pt=4&ubt=am&pn=0&sb=2&dc=null&pid=0']
+        # for url in urls:
+        #     yield Request(url=url, callback=self.parse)
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+

hg3535/spiders/hg3535_zq_status_up.py

@@ -0,0 +1,325 @@
+# -*- coding: utf-8 -*-
+
+import jsonpath
+import scrapy
+from scrapy.http import Request
+import psycopg2
+import time
+from functools import wraps
+from contextlib import contextmanager
+import psycopg2.extras
+
+from scrapy.conf import settings
+import json
+from datetime import datetime
+from datetime import date
+import itertools
+import re
+from scrapy.xlib.pydispatch import dispatcher
+from scrapy import signals
+
+
+
+# 测试一个函数的运行时间，使用方式：在待测函数直接添加此修饰器
+def timethis(func):
+    @wraps(func)
+    def wrapper(*args, **kwargs):
+        start = time.perf_counter()
+        r = func(*args, **kwargs)
+        end = time.perf_counter()
+        print('\n============================================================')
+        print('{}.{} : {}'.format(func.__module__, func.__name__, end - start))
+        print('============================================================\n')
+        return r
+    return wrapper
+
+
+# 测试一段代码运行的时间，使用方式：上下文管理器with
+# with timeblock('block_name'):
+#     your_code_block...
+@contextmanager
+def timeblock(label='Code'):
+    start = time.perf_counter()
+    try:
+        yield
+    finally:
+        end = time.perf_counter()
+        print('==============================================================')
+        print('{} run time: {}'.format(label, end - start))
+        print('==============================================================')
+
+
+class SqlConn():
+    '''
+    连接数据库，以及进行一些操作的封装
+    '''
+    sql_name = ''
+    database = ''
+    user = ''
+    password = ''
+    port = 0
+    host = ''
+
+    # 创建连接、游标
+    def __init__(self, *args, **kwargs):
+        if kwargs.get("sql_name"):
+            self.sql_name = kwargs.get("sql_name")
+        if kwargs.get("database"):
+            self.database = kwargs.get("database")
+        if kwargs.get("user"):
+            self.user = kwargs.get("user")
+        if kwargs.get("password"):
+            self.password = kwargs.get("password")
+        if kwargs.get("port"):
+            self.port = kwargs.get("port")
+        if kwargs.get("host"):
+            self.host = kwargs.get("host")
+
+        if not (self.host and self.port and self.user and
+                self.password and self.database):
+            raise Warning("conn_error, missing some params!")
+
+        sql_conn = {
+                    'postgresql': psycopg2,
+                    }
+
+        self.conn = sql_conn[self.sql_name].connect(host=self.host,
+                                                    port=self.port,
+                                                    user=self.user,
+                                                    password=self.password,
+                                                    database=self.database,
+                                                    )
+        self.cursor = self.conn.cursor(cursor_factory=psycopg2.extras.DictCursor)
+        # self.cursor = self.conn.cursor()
+
+        if not self.cursor:
+            raise Warning("conn_error!")
+
+    # 测试连接
+    def test_conn(self):
+        if self.cursor:
+            print("conn success!")
+        else:
+            print('conn error!')
+
+    # 单条语句的并提交
+    def execute(self, sql_code):
+        self.cursor.execute(sql_code)
+        self.conn.commit()
+
+    # 单条语句的不提交
+    def execute_no_conmmit(self, sql_code):
+        self.cursor.execute(sql_code)
+
+    # 构造多条语句，使用%s参数化，对于每个list都进行替代构造
+    def excute_many(self, sql_base, param_list):
+        self.cursor.executemany(sql_base, param_list)
+
+    # 批量执行（待完善）
+    def batch_execute(self, sql_code):
+        pass
+
+    # 获取数据
+    def get_data(self, sql_code, count=0):
+        self.cursor.execute(sql_code)
+        if int(count):
+            return self.cursor.fetchmany(count)
+        else:
+            return self.cursor.fetchall()
+
+    # 更新数据
+    def updata_data(self, sql_code):
+        self.cursor.execute(sql_code)
+
+    # 插入数据
+    def insert_data(self, sql_code):
+        self.cursor(sql_code)
+
+    # 滚动游标
+    def cursor_scroll(self, count, mode='relative'):
+        self.cursor.scroll(count, mode=mode)
+
+    # 提交
+    def commit(self):
+        self.conn.commit()
+
+    # 回滚
+    def rollback(self):
+        self.conn.rollback()
+
+    # 关闭连接
+    def close_conn(self):
+        self.cursor.close()
+        self.conn.close()
+
+class ComplexEncoder(json.JSONEncoder):
+    def default(self, obj):
+        if isinstance(obj, datetime):
+            return obj.strftime('%Y-%m-%d %H:%M:%S')
+        elif isinstance(obj, date):
+            return obj.strftime('%Y-%m-%d')
+        else:
+            return json.JSONEncoder.default(self, obj)
+
+
+
+
+
+
+
+
+class LanqiuSpider(scrapy.Spider):
+    def __init__(self):
+        super(LanqiuSpider).__init__()
+        #信号量
+        # dispatcher.connect(self.spider_opened, signals.spider_opened)
+        dispatcher.connect(self.spider_closed, signals.spider_closed)
+        # self.conn = SqlConn(sql_name='postgresql',host='127.0.0.1',port=5432,user='postgres',password='9998877',database='postgres')
+        # database = PostgresqlDatabase('kaiyou',**{'host': '192.168.2.200', 'port': 10432, 'user': 'kaiyou', 'password': '123456'})
+        # self.conn = SqlConn(sql_name='postgresql',host='192.168.2.200',port=10432,user='kaiyou',password='123456',database='kaiyou')
+        self.conn = SqlConn(sql_name='postgresql',host=settings["POST_HOST"], port=settings['POST_PORT'], user=settings["POST_USER"],password=settings["POST_PASSWORD"],database=settings["POST_DATABASE"])
+
+
+
+    name = "ball_status_update"
+    allowed_domains = ['hg3535z.com']
+    #sid要改为1 足球 现在测试改为4
+    start_urls = ['https://hg3535z.com/odds2/d/getodds?sid=3&pt=4&ubt=am&pn=0&sb=2&dc=null&pid=0'] # 滚球菜单 篮球滚球列url
+    # custom_settings = {
+    #     "ITEM_PIPELINES": {
+    #         'scrapy_yzd.pipelines.BallStatuspipeline':1,
+    #     }
+    # }
+    # start_urls = ['http://hg3535z.com/odds2/d/getodds?sid=2&pt=3&ubt=am&pn=0&sb=2&dc=null&pid=0']
+    # http: // hg3535z.com / odds2 / d / getamodds?eid = 3098030 & iip = false & ubt = am & isp = false
+    # http://hg3535z.com/odds2/d/getodds?sid=2&pt=2&ubt=am&pn=0&sb=2&dc=null&pid=0
+
+    def parse(self, response):
+        # a = SqlConn(sql_name='postgresql',host='127.0.0.1',port=5432,user='postgres',password='9998877',database='postgres')
+        # a.test_conn()
+        b = self.conn.get_data("select match_id from st_ball_status where ball_type='足球'")
+        d = list(itertools.chain(*b))
+        # yield Request(url='http://hg3535z.com/odds2/d/getamodds?eid=3147927&iip=true&ubt=am&isp=false',callback=self.parse_each)
+        for i in d:
+            urls = 'https://hg3535z.com/odds2/d/getamodds?eid={}&iip=true&ubt=am&isp=false'.format(i)
+            yield Request(url=urls,callback=self.parse_each)
+
+    #取得url中的id字段
+    def re_str(self,url_str):
+        a = (re.findall(r"eid=(.+?)&",url_str))
+        result = "".join(a)
+        return result
+
+    def parse_each(self,response):
+        # print("URL: " + response.request.url)
+        if response.text != "null":
+            res = json.loads(response.text)
+            # res = {"i":['false',1,3149430,4,2,"sh",0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,"28 / 04","99:00","下半场",'false','true',"足球",0],"eg":{"c":{"k":27078,"n":"罗马尼亚乙组联赛"},"es":[{"dbg":'true',"egid":0,"g":"","i":["阿格斯","法乌尔","1","True","28 / 04","","1","False","0","0","0","1","FT","","","",0,"0","1",0,"False"],"ibs":'true',"ibsc":'true',"lcd":{"lcpid":5,"ilup":'false',"lcid":3147921,"p":0},"k":3147921,"o":{},"pci":{"ctid":0},"egn":""},{"dbg":'false',"egid":0,"g":"","i":["阿格斯-角球数","法乌尔-角球数","1","True","28 / 04","99:00","1","False","0","0","4","2","下半场","Second Half","","",0,"","",0,"False"],"ibs":'true',"ibsc":'false',"k":3149430,"o":{"oe":{"s":192,"v":["o3239030643","0.00","o3239030644","0.00"],"n":"角球:滚球 单 / 双"}},"pci":{"ctid":1,"ctn":"角球"},"egn":""}]},"ot":2,"sc":{"3147921":{"a":1,"h":0},"3149430":{"a":2,"h":4}},"v":567862}
+            # res1 = jsonpath.jsonpath(res,'$..eg..es[0]..o')
+            res1 = jsonpath.jsonpath(res,'$..eg..es[:]..o')
+            if len(res1) > 1:
+                print("这是有角球啊")
+                o_dict0 = res1[0]  # 递归取o字典
+                o_dict1 = res1[1]
+                re_url = response.request.url
+                res_id = self.re_str(re_url)
+                print("我是角球id是")
+                print(res_id)
+                if o_dict0 or o_dict1:
+                    print("这不是个空字典")
+                    print("我不做任何操作的啊")
+                    # re_url = response.request.url
+                    # res_id = self.re_str(re_url)
+                    # print(res_id)
+                    # utime = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime())
+                    # a = SqlConn(sql_name='postgresql',host='127.0.0.1',port=5432,user='postgres',password='9998877',database='postgres')
+                    # self.conn.updata_data("update st_ball_status  set status=0, update_time='{0}' where match_id={1}".format(utime,res_id))
+                    # sql1 = "insert into comendnotice(status, game_code, match_id,done_time) values (%s,%s, %s, %s) on conflict(match_id) do update set done_time = %s"
+                    # self.conn.cursor.execute(sql1,(4,'zq',res_id,utime,utime))
+                    # cursor.execute(sql1, (match_id, create_time,staus,update_time, ball_type,update_time))
+                    # self.conn.commit()
+            else:
+                o_dict0 = res1[0]
+                if not o_dict0:
+                    print("这是空字典我要改状态")
+                    re_url = response.request.url
+                    res_id = self.re_str(re_url)
+                    print(res_id)
+                    utime = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime())
+                    # a = SqlConn(sql_name='postgresql',host='127.0.0.1',port=5432,user='postgres',password='9998877',database='postgres')
+                    self.conn.updata_data("update st_ball_status  set status=0, update_time='{0}' where match_id={1}".format(utime,res_id))
+                    # sql1 = "insert into comendnotice(status, game_code, match_id,done_time) values (%s,%s, %s, %s) on conflict(match_id) do update set done_time = %s"
+                    self.conn.updata_data("update st_zq_result set status=2  where match_id={}".format(res_id))
+                    self.conn.updata_data("update st_zq_result_record set status=2  where match_id={}".format(res_id))
+                    self.conn.updata_data("update st_zq_competition set status=2  where match_id={}".format(res_id))
+
+                    # self.conn.cursor.execute(sql1,(4,'zq',res_id,utime,utime))
+                    # cursor.execute(sql1, (match_id, create_time,staus,update_time, ball_type,update_time))
+                    self.conn.commit()
+        # if o_dict:
+        #     print("有数据")
+        # else:
+        #     print("没数据")
+
+        if response.text == "null":
+            print("暂时没有数据")
+            re_url = response.request.url
+            res_id = self.re_str(re_url)
+            print(res_id)
+            utime = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime())
+            # a = SqlConn(sql_name='postgresql',host='127.0.0.1',port=5432,user='postgres',password='9998877',database='postgres')
+            self.conn.updata_data("update st_ball_status  set status=0, update_time='{0}' where match_id={1}".format(utime,res_id))
+            sql1 = "insert into comendnotice(status, game_code, match_id,done_time) values (%s,%s, %s, %s) on conflict(match_id) do update set done_time = %s"
+            self.conn.cursor.execute(sql1,(4,'zq',res_id,utime,utime))
+            self.conn.updata_data("update st_zq_result set status=2  where match_id={}".format(res_id))
+            self.conn.updata_data("update st_zq_result_record set status=2  where match_id={}".format(res_id))
+            self.conn.updata_data("update st_zq_competition set status=2  where match_id={}".format(res_id))
+            # cursor.execute(sql1, (match_id, create_time,staus,update_time, ball_type,update_time))
+
+            self.conn.commit()
+
+            # a.close_conn()
+
+    # def spider_opened(self, spider):
+
+
+    def spider_closed(self, spider):
+        print("我要关闭了")
+        self.conn.close_conn()
+
+
+        # new_datas = json.loads(response.text).get('eg', "")
+        # if new_datas:
+        #     new_data = new_datas.get("es", "")
+        #     result = new_data[0]
+        #     o_dic = result['o']
+        #     if response.text == "null"  or not o_dic:
+        #         print("没有数据")
+        # print(response.body)
+        # if response.text == "null":
+        #     print("这个网页没有数据的")
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+

hg3535/spiders/hg3535_zuqiu.py

@@ -0,0 +1,625 @@
+# -*- coding: utf-8 -*-
+import copy
+import json
+
+import jsonpath
+import scrapy
+from scrapy.http import Request
+
+from ..items import Roll_Zuqiu
+from scrapy.spidermiddlewares.httperror import HttpError
+from twisted.internet.error import DNSLookupError
+from twisted.internet.error import TimeoutError
+
+
+class ZuqiuSpider(scrapy.Spider):
+    name = 'roll_zuqiu'
+    allowed_domains = ['hg3535z.com']
+    custom_settings = {
+        "ITEM_PIPELINES":{
+            'hg3535.pipelines.Roll_Zuqiupipeline': 100,
+        }
+    }
+    # def start_requests(self):
+    #
+    #     for y in range(1, 4):
+    #         for i in range(10):
+    #             url = 'https://hg3535z.com/odds2/d/getodds?sid=1&pt=' + str(y) + '&ubt=' + 'am' + '&pn=' + str(
+    #                 i) + '&sb=2&dc=null&pid=0'
+    #             yield scrapy.Request(url=url, callback=self.parse, meta={'pt': y})
+    start_urls = ['https://hg3535z.com/odds2/d/getodds?sid=1&pt=4&ubt=am&pn=0&sb=2&dc=null&pid=0'] # 滚球菜单 足球滚球列url
+
+
+
+    def parse(self, response):
+        datas = json.loads(response.text)
+        # item = Today_all()
+        ids = jsonpath.jsonpath(datas, '$..i-ot[0]..egs..es..i[16]') # ids新列表
+        print(ids)
+        if ids:
+            ids = set(ids)
+            for i in ids:
+                urls = 'https://hg3535z.com/odds2/d/getamodds?eid={}&iip=true&ubt=am&isp=false'.format(i)
+                print(urls)
+                yield Request(url=urls, callback=self.parse_each)
+        # if response.text:
+        #     try:
+        #         datas = json.loads(response.text).get('n-ot', "").get('egs', "")
+        #     except:
+        #         datas = ""
+        #     pt = copy.copy(response.meta['pt'])
+        #     if datas:
+        #         for result in datas:
+        #             new_results = result['es']
+        #             for new_result in new_results:
+        #                 game_id = str(new_result['i'][16])
+        #                 if pt is 3:
+        #                     url = "https://hg3535z.com/odds2/d/getamodds?eid="+game_id+"&iip=false&ubt=am&isp=true"
+        #                     yield Request(url=url, callback=self.parse_each, meta={'pt': pt})
+        #                 if pt is 2:
+        #                     url = "https://hg3535z.com/odds2/d/getamodds?eid="+game_id+"&iip=false&ubt=am&isp=false"
+        #                     yield Request(url=url, callback=self.parse_each, meta={'pt': pt})
+        #                 if pt is 1:
+        #                     url = "https://hg3535z.com/odds2/d/getamodds?eid="+game_id+"&iip=false&ubt=am&isp=false"
+        #                     yield Request(url=url, callback=self.parse_each, meta={'pt': pt})
+
+
+    # def errback_httpbin(self, failure):
+    #     # log all errback failures,
+    #     # in case you want to do something special for some errors,
+    #     # you may need the failure's type
+    #     self.logger.error(repr(failure))
+    #
+    #     #if isinstance(failure.value, HttpError):
+    #     if failure.check(HttpError):
+    #         # you can get the response
+    #         response = failure.value.response
+    #         self.logger.error('HttpError on %s', response.url)
+    #
+    #     #elif isinstance(failure.value, DNSLookupError):
+    #     elif failure.check(DNSLookupError):
+    #         # this is the original request
+    #         request = failure.request
+    #         self.logger.error('DNSLookupError on %s', request.url)
+    #
+    #     #elif isinstance(failure.value, TimeoutError):
+    #     elif failure.check(TimeoutError):
+    #         request = failure.request
+    #         self.logger.error('TimeoutError on %s', request.url)
+
+
+    def parse_each(self, response):
+        # 球队进球数 大小
+        try:
+            datas = json.loads(response.text)['eg']['es']
+            league_id = json.loads(response.text)['eg']['c']['k']
+            # 联赛名
+            league_name = json.loads(response.text)['eg']['c']['n']
+        except:
+            datas = ""
+            league_id = ""
+            # 联赛名
+            league_name = ""
+        try:
+            pt = response.meta['pt']
+        except:
+            pt = 0
+        if datas:
+            item = Roll_Zuqiu()
+            full_dict = {}
+            half_dict = {}
+            full_dict_rule = {}
+            half_dict_rule = {}
+            for data in datas:
+                try:
+                    new_data = data['pci']['ctid']
+                except:
+                    new_data = ""
+                if new_data == 0:
+                    # 比赛id
+                    game_id = str(data['k'])
+                    # 球队1
+                    team_home = data['i'][0]
+                    # 球队2
+                    team_guest = data['i'][1]
+                    # 数量(97>)
+                    number = data['i'][2]
+                    # 下半场
+                    half_way = data['i'][12]
+                    # 角球或者其他
+                    corner_ball = data['pci'].get('ctn', "")
+                    # 日期
+                    data_game = data['i'][4]
+                    # 开赛时间
+                    time_game = data['i'][5]
+                    #这个地方的值不准确 需要再次确认
+                    score_home = data['i'][10]  # 队一分数
+                    score_guest = data['i'][11] # 队二分数
+                    # half_way = new_result['i'][12] #下半场
+
+
+# 让球------------------------------------------------------------------------------------------------------------------
+                    try:
+                        concedes = data['o']['ah']['v']
+                        new_concedes = [concedes[i] for i in range(len(concedes)) if i % 2 == 1]
+                        concede_homes = [new_concedes[i] for i in range(len(new_concedes)) if i % 2 == 0]
+                        # concede_home_rule = [concede_homes[i] for i in range(len(concede_homes)) if i % 2 == 0]
+                        concede_home_rule = []
+                        for i in range(len(concede_homes)):
+                            if i % 2 is 0:
+                                home_rule = str(concede_homes[i])
+                                if home_rule.startswith('+'):
+                                    n_home_rule = home_rule.replace('+', '-')
+                                    concede_home_rule.append(n_home_rule)
+                                elif home_rule.startswith('-'):
+                                    n_home_rule = home_rule.replace('-', '+')
+                                    concede_home_rule.append(n_home_rule)
+                                else:
+                                    concede_home_rule.append(home_rule)
+                                # concede_homes[i]
+                                # concede_home
+                        if pt is 3:
+                            concede_home = [float(concede_homes[i]) - 1 for i in range(len(concede_homes)) if i % 2 is 1]
+                        else:
+                            concede_home = [concede_homes[i] for i in range(len(concede_homes)) if i % 2 is 1]
+
+                        concede_guests = [new_concedes[i] for i in range(len(new_concedes)) if i % 2 is 1]
+                        # concede_guest_rule
+                        # concede_guest_rule = [concede_guests[i] for i in range(len(concede_guests)) if
+                        #                       i % 2 == 0]
+                        # concede_guest
+                        concede_guest_rule = []
+                        for i in range(len(concede_guests)):
+                            if i % 2 is 0:
+                                guest_rule = str(concede_guests[i])
+                                if guest_rule.startswith('+'):
+                                    n_guest_rule = guest_rule.replace('+', '-')
+                                    concede_guest_rule.append(n_guest_rule)
+                                elif guest_rule.startswith('-'):
+                                    n_guest_rule = guest_rule.replace('-', '+')
+                                    concede_guest_rule.append(n_guest_rule)
+                                else:
+                                    concede_guest_rule.append(guest_rule)
+                        if pt is 3:
+                            concede_guest = [float(concede_guests[i]) -1 for i in range(len(concede_guests)) if i % 2 is 1]
+                        else:
+                            concede_guest = [concede_guests[i] for i in range(len(concede_guests)) if i % 2 is 1]
+                    except:
+                        concede_guest = ""
+                        concede_guest_rule = ""
+                        concede_home = ""
+                        concede_home_rule = ""
+
+# 上半场让球half_concede-------------------------------------------------------------------------------------------------
+                    try:
+                        half_concedes = data['o']['ah1st']['v']
+                        new_half_concedes = [half_concedes[i] for i in range(len(half_concedes)) if i % 2 is 1]
+                        new_half_concede_homes = [new_half_concedes[i] for i in range(len(new_half_concedes)) if
+                                                  i % 2 is 0]
+                        # half_concede_home_rule
+                        # half_concede_home_rule = [new_half_concede_homes[i] for i in
+                        #                           range(len(new_half_concede_homes)) if i % 2 == 0]
+
+                        half_concede_home_rule = []
+                        for i in range(len(new_half_concede_homes)):
+                            if i % 2 == 0:
+                                home_rule = str(new_half_concede_homes[i])
+                                if home_rule.startswith('+'):
+                                    n_home_rule = home_rule.replace('+', '-')
+                                    half_concede_home_rule.append(n_home_rule)
+                                if home_rule.startswith('-'):
+                                    n_home_rule = home_rule.replace('-', '+')
+                                    half_concede_home_rule.append(n_home_rule)
+                                else:
+                                    half_concede_home_rule.append(home_rule)
+                        # concede_home
+                        if pt is 3:
+                            half_concede_home = [float(new_half_concede_homes[i]) - 1 for i in range(len(new_half_concede_homes)) if i % 2 is 1]
+                        else:
+                            half_concede_home = [new_half_concede_homes[i] for i in range(len(new_half_concede_homes)) if i % 2 is 1]
+                        half_concede_guests = [new_half_concedes[i] for i in range(len(new_half_concedes)) if i % 2 is 1]
+                        # concede_guest_rule
+                        # half_concede_guest_rule = [half_concede_guests[i] for i in
+                        #                            range(len(half_concede_guests)) if i % 2 == 0]
+                        half_concede_guest_rule = []
+                        for i in range(len(half_concede_guests)):
+                            if i % 2 == 0:
+                                guest_rule = str(half_concede_guests[i])
+                                if guest_rule.startswith('+'):
+                                    n_guest_rule = guest_rule.replace('+', '-')
+                                    half_concede_guest_rule.append(n_guest_rule)
+                                if guest_rule.startswith('-'):
+                                    n_guest_rule = guest_rule.replace('-', '+')
+                                    half_concede_guest_rule.append(n_guest_rule)
+                                else:
+                                    half_concede_guest_rule.append(guest_rule)
+                        # concede_guest
+                        if pt is 3:
+                            half_concede_guest = [float(half_concede_guests[i]) -1  for i in range(len(half_concede_guests)) if
+                                                  i % 2 is 1]
+                        else:
+                            half_concede_guest = [half_concede_guests[i] for i in range(len(half_concede_guests)) if
+                                                  i % 2 is 1]
+                    except:
+                        half_concede_home_rule = ""
+                        half_concede_home = ""
+                        half_concede_guest_rule = ""
+                        half_concede_guest = ""
+
+# 全场大小size 进球大小---------------------------------------------------------------------------------------------------
+                    try:
+                        sizes = data['o']['ou']['v']
+                        new_sizes = [sizes[i] for i in range(len(sizes)) if i % 2 is 1]
+                        size_homes = [new_sizes[i] for i in range(len(new_sizes)) if i % 2 is 0]
+                        # size_home_rule
+                        size_home_rule = [size_homes[i] for i in range(len(size_homes)) if i % 2 is 0]
+                        # size_home
+                        if pt is 3:
+                            size_home = [float(size_homes[i]) - 1 for i in range(len(size_homes)) if i % 2 is 1]
+                        else:
+                            size_home = [size_homes[i] for i in range(len(size_homes)) if i % 2 is 1]
+
+                        size_guests = [new_sizes[i] for i in range(len(new_sizes)) if i % 2 is 1]
+                        # size_guest_rule
+                        size_guest_rule = [size_guests[i] for i in range(len(size_guests)) if i % 2 is 0]
+                        # csize_guest
+                        if pt is 3:
+                            size_guest = [float(size_guests[i]) - 1 for i in range(len(size_guests)) if i % 2 is 1]
+                        else:
+                            size_guest = [size_guests[i] for i in range(len(size_guests)) if i % 2 is 1]
+                    except:
+                        size_guest = ""
+                        size_guest_rule = ""
+                        size_home = ""
+                        size_home_rule = ""
+
+# 上半场大小 进球大小 half_size-------------------------------------------------------------------------------------------
+                    try:
+                        half_sizes = data['o']['ou1st']['v']
+                        new_half_sizes = [half_sizes[i] for i in range(len(half_sizes)) if i % 2 is 1]
+                        half_size_homes = [new_half_sizes[i] for i in range(len(new_half_sizes)) if i % 2 is 0]
+                        # size_home_rule
+                        half_size_home_rule = [half_size_homes[i] for i in range(len(half_size_homes)) if
+                                               i % 2 is 0]
+                        # half_size_home 主队
+                        if pt is 3:
+                            half_size_home = [float(half_size_homes[i]) -1 for i in range(len(half_size_homes)) if i % 2 is 1]
+                        else:
+                            half_size_home = [half_size_homes[i] for i in range(len(half_size_homes)) if i % 2 is 1]
+                        half_size_guests = [new_half_sizes[i] for i in range(len(new_half_sizes)) if i % 2 is 1]
+                        # half_size_guest_rule 客队
+                        half_size_guest_rule = [half_size_guests[i] for i in range(len(half_size_guests)) if
+                                                i % 2 is 0]
+                        # half_size_guest
+                        if pt is 3:
+                            half_size_guest = [float(half_size_guests[i]) - 1 for i in range(len(half_size_guests)) if
+                                               i % 2 is 1]
+                        else:
+                            half_size_guest = [half_size_guests[i] for i in range(len(half_size_guests)) if
+                                               i % 2 is 1]
+                    except:
+                        half_size_guest = ""
+                        half_size_guest_rule = ""
+                        half_size_home = ""
+                        half_size_home_rule = ""
+
+# 全场总进球 total_goal--------------------------------------------------------------------------------------------------
+                    total_goal = {}
+                    try:
+                        total_goals = data['o']['tg']['v']
+                        new_total_goals = [total_goals[i] for i in range(len(total_goals)) if i % 2 is 1]
+                        total_goal["total_goal_zero"] = float(new_total_goals[0]) - 1
+                        total_goal["total_goal_two"] = float(new_total_goals[1]) - 1
+                        total_goal["total_goal_four"] = float(new_total_goals[2]) - 1
+                        total_goal["total_goal_seven"] = float(new_total_goals[3]) - 1
+                    except:
+                        total_goal["total_goal_zero"] = ""
+                        total_goal["total_goal_two"] = ""
+                        total_goal["total_goal_four"] = ""
+                        total_goal["total_goal_seven"] = ""
+
+# 总进球上半场 half_total_goal-------------------------------------------------------------------------------------------
+                    try:
+                        half_total_goals = data['o']['tg1st']['v']
+                        new_half_total_goals = [half_total_goals[i] for i in range(len(half_total_goals)) if i % 2 is 1]
+                        total_goal["half_total_goal_zero"] = float(new_half_total_goals[0]) - 1
+                        total_goal["half_total_goal_one"] = float(new_half_total_goals[1]) - 1
+                        total_goal["half_total_goal_two"] = float(new_half_total_goals[2]) - 1
+                        total_goal["half_total_goal_three"] = float(new_half_total_goals[3]) - 1
+                    except:
+                        total_goal["half_total_goal_zero"] = ""
+                        total_goal["half_total_goal_one"] = ""
+                        total_goal["half_total_goal_two"] = ""
+                        total_goal["half_total_goal_three"] = ""
+
+# 早盘 半场/全场---------------------------------------------------------------------------------------------------------
+                    half_full = {}
+                    new_lists = ["half_full_home_home", "half_full_home_dogfall", "half_full_home_guest",
+                                 "half_full_dogfall_home", "half_full_dogfall_dogfall", "half_full_dogfall_guest",
+                                 "half_full_guest_home", "half_full_guest_dogfall", "half_full_guest_guest"]
+                    # 早盘 半场/全场
+                    try:
+                        half_fulls = data['o']['hf']['v']
+                        new_half_fulls = [half_fulls[i] for i in range(len(half_fulls)) if i % 2 is 1]
+                        for index, value in enumerate(new_lists):
+                            half_full[value] = float(new_half_fulls[index]) - 1
+                    except:
+                        for index, value in enumerate(new_lists):
+                            half_full[value] = ""
+
+# 早盘 最先/最后进球 最先进球 ---------------------------------------------------------------------------------------------
+                    first_last_ball = {}
+                    try:
+                        first_balls = data['o']['ttslast']['v']
+                        first_ball = [first_balls[i] for i in range(len(first_balls)) if i % 2 is 1]
+                        first_last_ball['first_ball_home'] = float(first_ball[0]) - 1
+                        first_last_ball['first_ball_guest'] = float(first_ball[1]) - 1
+                    except:
+                        pass
+
+                    # 早盘 最先/最后进球 最后进球
+
+                    try:
+                        last_balls = data['o']['tts1st']['v']
+                        last_ball = [last_balls[i] for i in range(len(last_balls)) if i % 2 is 1]
+                        first_last_ball['last_ball_home'] = float(last_ball[0]) - 1
+                        first_last_ball['last_ball_guest'] = float(last_ball[1]) - 1
+                        first_last_ball['not_ball'] = float(last_ball[2]) - 1
+                    except:
+                        pass
+
+# 全场独赢capot ---------------------------------------------------------------------------------------------------------
+
+                    try:
+                        capots = data['o']['1x2']['v']
+                        new_capots = [capots[i] for i in range(len(capots)) if i % 2 is 1]
+                        capot_home = float(new_capots[0]) - 1
+                        capot_guest = float(new_capots[1]) - 1
+                        capot_dogfall = float(new_capots[2]) - 1
+                    except:
+                        capot_home = ""
+                        capot_guest = ""
+                        capot_dogfall = ""
+
+                    # 上半场独赢capot
+                    try:
+                        half_capots = data['o']['1x21st']['v']
+                        new_half_capots = [half_capots[i] for i in range(len(half_capots)) if i % 2 is 1]
+                        half_capot_home = float(new_half_capots[0]) - 1
+                        half_capot_guest = float(new_half_capots[1]) - 1
+                        half_capot_dogfall = float(new_half_capots[2]) - 1
+                    except:
+                        half_capot_home = ""
+                        half_capot_guest = ""
+                        half_capot_dogfall = ""
+
+# 全场入球：单/双 odd_even------------------------------------------------------------------------------------------------
+                    try:
+                        odd_evens = data['o']['oe']['v']
+                        new_odd_evens = [odd_evens[i] for i in range(len(odd_evens)) if i % 2 is 1]
+                        odd_even_odd = new_odd_evens[0]
+                        odd_even_even = new_odd_evens[1]
+                    except:
+                        odd_even_odd = ""
+                        odd_even_even = ""
+                    # 半场入球：单/双 half_odd_even
+                    try:
+                        half_odd_evens = data['o']['oe1st']['v']
+                        new_half_odd_evens = [half_odd_evens[i] for i in range(len(half_odd_evens)) if i % 2 is 1]
+                        half_odd_even_odd = new_half_odd_evens[0]
+                        half_odd_even_even = new_half_odd_evens[1]
+                    except:
+                        half_odd_even_odd = ""
+                        half_odd_even_even = ""
+
+                    bodan_data = {}
+# 波胆------------------------------------------------------------------------------------------------------------------
+                    try:
+                        bodans = data['o']['cs']['v']
+                        one_list = ["bodanhome_one_zero", "bodanhome_two_zero", "bodanhome_two_one",
+                                    "bodanhome_three_zero", "bodanhome_three_one", "bodanhome_three_two",
+                                    "bodanhome_four_zero", "bodanhome_four_one", "bodanhome_four_two",
+                                    "bodanhome_four_three"]
+                        two_list = ["bodanguest_one_zero", "bodanguest_two_zero", "bodanguest_two_one",
+                                    "bodanguest_three_zero", "bodanguest_three_one", "bodanguest_three_two",
+                                    "bodanguest_four_zero", "bodanguest_four_one", "bodanguest_four_two",
+                                    "bodanguest_four_three"]
+                        three_list = ["bodandogfall_zero_zero", "bodandogfall_one_one", "bodandogfall_two_two",
+                                      "bodandogfall_three_three", "bodandogfall_four_four"]
+                        new_bodans = [bodans[i] for i in range(len(bodans)) if i % 2 is 1]
+                        new_bodan = new_bodans[0:20]
+                        # 主队bodan_home
+                        bodan_home = [new_bodan[i] for i in range(len(new_bodan)) if i % 2 is 0]
+                        for index, t in enumerate(one_list):
+                            bodan_data[t] = float(bodan_home[index]) - 1
+                        # 客队bodan_guest
+                        bodan_guest = [new_bodan[i] for i in range(len(new_bodan)) if i % 2 is 1]
+                        for y, z in enumerate(two_list):
+                            bodan_data[z] = float(bodan_guest[y]) - 1
+                        bodan_dogfall = new_bodans[-7:-2]
+                        for a, b in enumerate(three_list):
+                            bodan_data[b] = float(bodan_dogfall[a]) - 1
+                        # 其他bodan_other
+                        bodan_data['bodanother'] = float(new_bodans[-1]) - 1
+                    except:
+                        pass
+
+                    try:
+                        half_bodans = data['o']['cs1st']['v']
+                        new_half_bodans = [half_bodans[i] for i in range(len(half_bodans)) if i % 2 is 1]
+                        new_one = ["halfbodanhome_one_zero", "halfbodanhome_two_zero", "halfbodanhome_two_one",
+                                   "halfbodanhome_three_zero", "halfbodanhome_three_one", "halfbodanhome_three_two"]
+                        new_two = ["halfbodanguest_one_zero", "halfbodanguest_two_zero", "halfbodanguest_two_one",
+                                   "halfbodanguest_three_zero", "halfbodanguest_three_one", "halfbodanguest_three_two"]
+                        new_three = ["halfbodandogfall_zero_zero", "halfbodandogfall_one_one",
+                                     "halfbodandogfall_two_two",
+                                     "halfbodandogfall_three_three"]
+                        halfbodan = new_half_bodans[0:12]
+                        half_bodan_home = [halfbodan[i] for i in range(len(halfbodan)) if i % 2 is 0]
+                        # 队1
+                        for index, t in enumerate(new_one):
+                            bodan_data[t] = float(half_bodan_home[index]) - 1
+                        # 队2
+                        half_bodan_guest = [halfbodan[i] for i in range(len(halfbodan)) if i % 2 is 1]
+                        for y, z in enumerate(new_two):
+                            bodan_data[z] = float(half_bodan_guest[y]) - 1
+                        # 和/平
+                        half_bodan_dogfall = new_half_bodans[-6:-2]
+                        for a, b in enumerate(new_three):
+                            bodan_data[b] = float(half_bodan_dogfall[a]) - 1
+                        # 其他
+                        bodan_data['halfbodanother'] = float(new_half_bodans[-1]) - 1
+                    except:
+                        pass
+                if new_data == 12:
+                    try:
+                        home_data = data['o']['ou']['v']
+
+                        # 球队进球 大条件
+                        home_size_big_rule = home_data[1]
+                        full_dict_rule['home_size_big'] = home_size_big_rule
+
+                        # 球队进球大赔率
+                        home_size_big = home_data[5]
+                        full_dict['home_size_big'] = home_size_big
+
+                        # 球队进球小条件
+                        home_size_small_rule = home_data[3]
+                        full_dict_rule['home_size_small'] = home_size_small_rule
+
+                        # 球队进球小赔率
+                        home_size_small = home_data[7]
+                        full_dict['home_size_small'] = home_size_small
+                    except:
+                        full_dict['home_size_small'] = ""
+                        full_dict_rule['home_size_small'] = ""
+
+                        full_dict['home_size_big'] = ""
+                        full_dict_rule['home_size_big'] = ""
+                    try:
+                        half_home_data = data['o']['ou1st']['v']
+
+                        half_home_size_big_rule = half_home_data[1]
+                        half_dict_rule['half_home_size_big'] = half_home_size_big_rule
+
+                        half_home_size_big = half_home_data[5]
+                        half_dict['half_home_size_big'] = half_home_size_big
+
+                        half_home_size_small_rule = half_home_data[3]
+                        half_dict_rule['half_home_size_small'] = half_home_size_small_rule
+
+                        half_home_size_small = half_home_data[7]
+                        half_dict['half_home_size_small'] = half_home_size_small
+
+                    except:
+                        half_dict_rule['half_home_size_big'] = ""
+                        half_dict['half_home_size_big'] = ""
+
+                        half_dict_rule['half_home_size_small'] = ""
+                        half_dict['half_home_size_small'] = ""
+
+                if new_data == 13:
+                    try:
+                        guest_data = data['o']['ou']['v']
+
+                        guest_size_big_rule = guest_data[1]
+                        full_dict_rule['guest_size_big'] = guest_size_big_rule
+
+                        guest_size_big = guest_data[5]
+                        full_dict['guest_size_big'] = guest_size_big
+
+                        guest_size_small_rule = guest_data[3]
+                        full_dict_rule['guest_size_small'] = guest_size_small_rule
+
+                        guest_size_small = guest_data[7]
+                        full_dict['guest_size_small'] = guest_size_small
+                    except:
+                        full_dict_rule['guest_size_big'] = ""
+                        full_dict['guest_size_big'] = ""
+
+                        full_dict_rule['guest_size_small'] = ""
+                        full_dict['guest_size_small'] = ''
+                    try:
+                        half_guest_data = data['o']['ou1st']['v']
+
+                        half_guest_size_big_rule = half_guest_data[1]
+                        half_dict_rule['half_guest_size_big'] = half_guest_size_big_rule
+
+                        half_guest_size_big = half_guest_data[5]
+                        half_dict['half_guest_size_big'] = half_guest_size_big
+
+                        half_guest_size_small_rule = half_guest_data[3]
+                        half_dict_rule['half_guest_size_small'] = half_guest_size_small_rule
+
+                        half_guest_size_small = half_guest_data[7]
+                        half_dict['half_guest_size_small'] = half_guest_size_small
+                    except:
+                        half_dict_rule['half_guest_size_big'] = ""
+                        half_dict['half_guest_size_big'] = ""
+
+                        half_dict_rule['half_guest_size_small'] = ""
+                        half_dict['half_guest_size_small'] = ""
+
+            item['league_id'] = league_id
+            item['league_name'] = league_name
+            item['pt'] = pt
+            item['game_id'] = game_id
+            item['team_home'] = team_home
+            item['team_guest'] = team_guest
+            item['number'] = number
+            item['data_game'] = data_game
+            item['time_game'] = time_game
+            item['corner_ball'] = corner_ball
+            # 波胆
+            item['bodan_data'] = bodan_data
+            # 早盘入球数单双
+            item["odd_even_odd"] = odd_even_odd
+            item["odd_even_even"] = odd_even_even
+            item["half_odd_even_odd"] = half_odd_even_odd
+            item["half_odd_even_even"] = half_odd_even_even
+
+            item['total_goal'] = total_goal
+
+            item["half_capot_home"] = half_capot_home
+            item["half_capot_guest"] = half_capot_guest
+            item["half_capot_dogfall"] = half_capot_dogfall
+
+            item["capot_home"] = capot_home
+            item["capot_guest"] = capot_guest
+            item["capot_dogfall"] = capot_dogfall
+
+            item["first_last_ball"] = first_last_ball
+
+            item["half_full"] = half_full
+
+            item["half_size_guest"] = half_size_guest
+            item["half_size_guest_rule"] = half_size_guest_rule
+            item["half_size_home"] = half_size_home
+            item["half_size_home_rule"] = half_size_home_rule
+            # 全场大小
+            item["size_guest"] = size_guest
+            item["size_guest_rule"] = size_guest_rule
+            item["size_home"] = size_home
+            item["size_home_rule"] = size_home_rule
+            # 上半场让球
+            item["half_concede_home_rule"] = half_concede_home_rule
+            item["half_concede_home"] = half_concede_home
+            item["half_concede_guest_rule"] = half_concede_guest_rule
+            item["half_concede_guest"] = half_concede_guest
+            # 全场让球
+            item["concede_guest"] = concede_guest
+            item["concede_guest_rule"] = concede_guest_rule
+            item["concede_home"] = concede_home
+            item["concede_home_rule"] = concede_home_rule
+            item['score_home'] = score_home
+            item['score_guest'] = score_guest
+            item['full_data'] = full_dict
+            item['half_data'] = half_dict
+            item['full_data_rule'] = full_dict_rule
+            item['half_data_rule'] = half_dict_rule
+            item['half_way'] = half_way
+            yield item
+
+
+
+