棒球赛果抓取规则更新

hg3535/.idea/workspace.xml

@@ -13,7 +13,7 @@
   </component>
   <component name="CoverageDataManager">
     <SUITE FILE_PATH="coverage/hg3535$items.coverage" NAME="items Coverage Results" MODIFIED="1554291398794" SOURCE_PROVIDER="com.intellij.coverage.DefaultCoverageFileProvider" RUNNER="coverage.py" COVERAGE_BY_TEST_ENABLED="true" COVERAGE_TRACING_ENABLED="false" WORKING_DIRECTORY="$PROJECT_DIR$" />
-    <SUITE FILE_PATH="coverage/hg3535$main.coverage" NAME="main Coverage Results" MODIFIED="1569405209751" SOURCE_PROVIDER="com.intellij.coverage.DefaultCoverageFileProvider" RUNNER="coverage.py" COVERAGE_BY_TEST_ENABLED="true" COVERAGE_TRACING_ENABLED="false" WORKING_DIRECTORY="$PROJECT_DIR$" />
+    <SUITE FILE_PATH="coverage/hg3535$main.coverage" NAME="main Coverage Results" MODIFIED="1569478865457" SOURCE_PROVIDER="com.intellij.coverage.DefaultCoverageFileProvider" RUNNER="coverage.py" COVERAGE_BY_TEST_ENABLED="true" COVERAGE_TRACING_ENABLED="false" WORKING_DIRECTORY="$PROJECT_DIR$" />
     <SUITE FILE_PATH="coverage/hg3535$change.coverage" NAME="change Coverage Results" MODIFIED="1567749757856" SOURCE_PROVIDER="com.intellij.coverage.DefaultCoverageFileProvider" RUNNER="coverage.py" COVERAGE_BY_TEST_ENABLED="true" COVERAGE_TRACING_ENABLED="false" WORKING_DIRECTORY="$PROJECT_DIR$" />
     <SUITE FILE_PATH="coverage/hg3535$hgjieshu.coverage" NAME="hgjieshu Coverage Results" MODIFIED="1558494720094" SOURCE_PROVIDER="com.intellij.coverage.DefaultCoverageFileProvider" RUNNER="coverage.py" COVERAGE_BY_TEST_ENABLED="true" COVERAGE_TRACING_ENABLED="false" WORKING_DIRECTORY="$PROJECT_DIR$/spiders" />
     <SUITE FILE_PATH="coverage/hg3535$halffull.coverage" NAME="halffull Coverage Results" MODIFIED="1554544334424" SOURCE_PROVIDER="com.intellij.coverage.DefaultCoverageFileProvider" RUNNER="coverage.py" COVERAGE_BY_TEST_ENABLED="true" COVERAGE_TRACING_ENABLED="false" WORKING_DIRECTORY="$PROJECT_DIR$/spiders" />
@@ -45,8 +45,8 @@
       <file pinned="false" current-in-tab="true">
         <entry file="file://$PROJECT_DIR$/spiders/saiguo.py">
           <provider selected="true" editor-type-id="text-editor">
-            <state relative-caret-position="140">
-              <caret line="7" lean-forward="true" selection-start-line="7" selection-end-line="7" />
+            <state relative-caret-position="-561">
+              <caret line="198" column="58" selection-start-line="198" selection-start-column="58" selection-end-line="198" selection-end-column="58" />
               <folding>
                 <element signature="e#0#15#0" expanded="true" />
               </folding>
@@ -54,6 +54,51 @@
           </provider>
         </entry>
       </file>
+      <file pinned="false" current-in-tab="false">
+        <entry file="file://$PROJECT_DIR$/spiders/roll_bangqiu.py">
+          <provider selected="true" editor-type-id="text-editor">
+            <state relative-caret-position="-4204">
+              <caret line="23" column="5" selection-start-line="23" selection-start-column="5" selection-end-line="23" selection-end-column="5" />
+              <folding>
+                <element signature="e#46#61#0" expanded="true" />
+              </folding>
+            </state>
+          </provider>
+        </entry>
+      </file>
+      <file pinned="false" current-in-tab="false">
+        <entry file="file://$PROJECT_DIR$/pipeline/zuqiu.py">
+          <provider selected="true" editor-type-id="text-editor">
+            <state relative-caret-position="420">
+              <caret line="63" column="8" lean-forward="true" selection-start-line="63" selection-start-column="8" selection-end-line="63" selection-end-column="14" />
+              <folding>
+                <element signature="e#0#15#0" expanded="true" />
+              </folding>
+            </state>
+          </provider>
+        </entry>
+      </file>
+      <file pinned="false" current-in-tab="false">
+        <entry file="file://$PROJECT_DIR$/spiders/roll_zuqiu.py">
+          <provider selected="true" editor-type-id="text-editor">
+            <state relative-caret-position="720">
+              <caret line="44" column="45" selection-start-line="44" selection-start-column="45" selection-end-line="44" selection-end-column="45" />
+              <folding>
+                <element signature="e#38#53#0" expanded="true" />
+              </folding>
+            </state>
+          </provider>
+        </entry>
+      </file>
+      <file pinned="false" current-in-tab="false">
+        <entry file="file://$PROJECT_DIR$/middlewares.py">
+          <provider selected="true" editor-type-id="text-editor">
+            <state relative-caret-position="163">
+              <caret line="137" selection-start-line="137" selection-end-line="137" />
+            </state>
+          </provider>
+        </entry>
+      </file>
     </leaf>
   </component>
   <component name="FileTemplateManagerImpl">
@@ -204,10 +249,10 @@
     </option>
   </component>
   <component name="ProjectFrameBounds" extendedState="6">
-    <option name="x" value="-143" />
-    <option name="y" value="201" />
-    <option name="width" value="1940" />
-    <option name="height" value="1024" />
+    <option name="x" value="-563" />
+    <option name="y" value="67" />
+    <option name="width" value="1609" />
+    <option name="height" value="1100" />
   </component>
   <component name="ProjectLevelVcsManager" settingsEditedManually="true" />
   <component name="ProjectView">
@@ -222,6 +267,16 @@
               <item name="hg3535" type="b2602c69:ProjectViewProjectNode" />
               <item name="hg3535" type="462c0819:PsiDirectoryNode" />
             </path>
+            <path>
+              <item name="hg3535" type="b2602c69:ProjectViewProjectNode" />
+              <item name="hg3535" type="462c0819:PsiDirectoryNode" />
+              <item name="pipeline" type="462c0819:PsiDirectoryNode" />
+            </path>
+            <path>
+              <item name="hg3535" type="b2602c69:ProjectViewProjectNode" />
+              <item name="hg3535" type="462c0819:PsiDirectoryNode" />
+              <item name="spiders" type="462c0819:PsiDirectoryNode" />
+            </path>
           </expand>
           <select />
         </subPane>
@@ -536,14 +591,7 @@
       <workItem from="1568604419513" duration="11095000" />
       <workItem from="1568970792599" duration="3014000" />
       <workItem from="1569036019518" duration="1144000" />
-      <workItem from="1569287788365" duration="46423000" />
-    </task>
-    <task id="LOCAL-00044" summary="updata">
-      <created>1567567386386</created>
-      <option name="number" value="00044" />
-      <option name="presentableId" value="LOCAL-00044" />
-      <option name="project" value="LOCAL" />
-      <updated>1567567386386</updated>
+      <workItem from="1569287788365" duration="57203000" />
     </task>
     <task id="LOCAL-00045" summary="updata">
       <created>1567588210355</created>
@@ -881,11 +929,18 @@
       <option name="project" value="LOCAL" />
       <updated>1569405412390</updated>
     </task>
-    <option name="localTasksCounter" value="93" />
+    <task id="LOCAL-00093" summary="足球,篮球赛果抓取规则更新">
+      <created>1569407801286</created>
+      <option name="number" value="00093" />
+      <option name="presentableId" value="LOCAL-00093" />
+      <option name="project" value="LOCAL" />
+      <updated>1569407801286</updated>
+    </task>
+    <option name="localTasksCounter" value="94" />
     <servers />
   </component>
   <component name="TimeTrackingManager">
-    <option name="totallyTimeSpent" value="1992039000" />
+    <option name="totallyTimeSpent" value="2002819000" />
   </component>
   <component name="TodoView">
     <todo-panel id="selected-file">
@@ -897,7 +952,7 @@
     </todo-panel>
   </component>
   <component name="ToolWindowManager">
-    <frame x="-8" y="-8" width="1936" height="1066" extended-state="7" />
+    <frame x="-8" y="-8" width="1936" height="1066" extended-state="6" />
     <editor active="true" />
     <layout>
       <window_info content_ui="combo" id="Project" order="0" sideWeight="0.71173847" visible="true" weight="0.16204691" />
@@ -905,7 +960,7 @@
       <window_info id="Favorites" order="2" sideWeight="0.5013405" side_tool="true" weight="0.108208954" />
       <window_info anchor="bottom" id="Message" order="0" />
       <window_info anchor="bottom" id="Find" order="1" weight="0.32832617" />
-      <window_info active="true" anchor="bottom" id="Debug" order="2" sideWeight="0.49573562" visible="true" weight="0.5343348" />
+      <window_info anchor="bottom" id="Debug" order="2" sideWeight="0.49573562" weight="0.30364805" />
       <window_info anchor="bottom" id="Cvs" order="3" weight="0.25" />
       <window_info anchor="bottom" id="Inspection" order="4" weight="0.4" />
       <window_info anchor="bottom" id="TODO" order="5" weight="0.32832617" />
@@ -915,7 +970,7 @@
       <window_info anchor="bottom" id="Event Log" order="9" sideWeight="0.5042644" side_tool="true" weight="0.2725322" />
       <window_info anchor="bottom" x="22" y="376" width="1876" height="298" id="Terminal" order="10" sideWeight="0.49307036" weight="0.15987125" />
       <window_info anchor="bottom" id="Python Console" order="11" sideWeight="0.49946696" weight="0.37017167" />
-      <window_info anchor="bottom" id="Run" order="12" sideWeight="0.87473345" weight="0.32403433" />
+      <window_info active="true" anchor="bottom" id="Run" order="12" sideWeight="0.87473345" visible="true" weight="0.2027897" />
       <window_info anchor="right" id="Commander" internal_type="SLIDING" order="0" type="SLIDING" weight="0.4" />
       <window_info anchor="right" id="Ant Build" order="1" weight="0.25" />
       <window_info anchor="right" content_ui="combo" id="Hierarchy" order="2" weight="0.25" />
@@ -1142,41 +1197,6 @@
           <line>120</line>
           <option name="timeStamp" value="279" />
         </line-breakpoint>
-        <line-breakpoint enabled="true" suspend="THREAD" type="python-line">
-          <url>file://$PROJECT_DIR$/spiders/saiguo.py</url>
-          <line>42</line>
-          <option name="timeStamp" value="310" />
-        </line-breakpoint>
-        <line-breakpoint enabled="true" suspend="THREAD" type="python-line">
-          <url>file://$PROJECT_DIR$/spiders/saiguo.py</url>
-          <line>37</line>
-          <option name="timeStamp" value="312" />
-        </line-breakpoint>
-        <line-breakpoint enabled="true" suspend="THREAD" type="python-line">
-          <url>file://$PROJECT_DIR$/spiders/saiguo.py</url>
-          <line>31</line>
-          <option name="timeStamp" value="313" />
-        </line-breakpoint>
-        <line-breakpoint enabled="true" suspend="THREAD" type="python-line">
-          <url>file://$PROJECT_DIR$/spiders/saiguo.py</url>
-          <line>99</line>
-          <option name="timeStamp" value="317" />
-        </line-breakpoint>
-        <line-breakpoint enabled="true" suspend="THREAD" type="python-line">
-          <url>file://$PROJECT_DIR$/spiders/saiguo.py</url>
-          <line>111</line>
-          <option name="timeStamp" value="318" />
-        </line-breakpoint>
-        <line-breakpoint enabled="true" suspend="THREAD" type="python-line">
-          <url>file://$PROJECT_DIR$/spiders/saiguo.py</url>
-          <line>134</line>
-          <option name="timeStamp" value="319" />
-        </line-breakpoint>
-        <line-breakpoint enabled="true" suspend="THREAD" type="python-line">
-          <url>file://$PROJECT_DIR$/spiders/saiguo.py</url>
-          <line>85</line>
-          <option name="timeStamp" value="321" />
-        </line-breakpoint>
       </breakpoints>
       <default-breakpoints>
         <breakpoint type="python-exception">
@@ -1388,16 +1408,6 @@
         </state>
       </provider>
     </entry>
-    <entry file="file://$PROJECT_DIR$/spiders/roll_bangqiu.py">
-      <provider selected="true" editor-type-id="text-editor">
-        <state relative-caret-position="160">
-          <caret line="23" column="5" selection-start-line="23" selection-start-column="5" selection-end-line="23" selection-end-column="5" />
-          <folding>
-            <element signature="e#46#61#0" expanded="true" />
-          </folding>
-        </state>
-      </provider>
-    </entry>
     <entry file="file://$PROJECT_DIR$/pipeline/guanjun.py">
       <provider selected="true" editor-type-id="text-editor">
         <state relative-caret-position="520">
@@ -1521,16 +1531,6 @@
         </state>
       </provider>
     </entry>
-    <entry file="file://$PROJECT_DIR$/spiders/roll_zuqiu.py">
-      <provider selected="true" editor-type-id="text-editor">
-        <state relative-caret-position="842">
-          <caret line="44" column="45" selection-start-line="44" selection-start-column="45" selection-end-line="44" selection-end-column="45" />
-          <folding>
-            <element signature="e#38#53#0" expanded="true" />
-          </folding>
-        </state>
-      </provider>
-    </entry>
     <entry file="file://$PROJECT_DIR$/utils/helper.py">
       <provider selected="true" editor-type-id="text-editor">
         <state relative-caret-position="260">
@@ -1558,13 +1558,6 @@
         </state>
       </provider>
     </entry>
-    <entry file="file://$PROJECT_DIR$/middlewares.py">
-      <provider selected="true" editor-type-id="text-editor">
-        <state relative-caret-position="461">
-          <caret line="152" selection-start-line="152" selection-end-line="152" />
-        </state>
-      </provider>
-    </entry>
     <entry file="file://C:/venv/Lib/site-packages/scrapy/http/response/text.py">
       <provider selected="true" editor-type-id="text-editor">
         <state relative-caret-position="145">
@@ -1572,16 +1565,6 @@
         </state>
       </provider>
     </entry>
-    <entry file="file://$PROJECT_DIR$/pipeline/zuqiu.py">
-      <provider selected="true" editor-type-id="text-editor">
-        <state relative-caret-position="642">
-          <caret line="63" column="8" lean-forward="true" selection-start-line="63" selection-start-column="8" selection-end-line="63" selection-end-column="14" />
-          <folding>
-            <element signature="e#0#15#0" expanded="true" />
-          </folding>
-        </state>
-      </provider>
-    </entry>
     <entry file="file://$PROJECT_DIR$/spiders/roll_lanqiu.py">
       <provider selected="true" editor-type-id="text-editor">
         <state relative-caret-position="165">
@@ -1602,10 +1585,47 @@
         </state>
       </provider>
     </entry>
+    <entry file="file://$PROJECT_DIR$/middlewares.py">
+      <provider selected="true" editor-type-id="text-editor">
+        <state relative-caret-position="163">
+          <caret line="137" selection-start-line="137" selection-end-line="137" />
+        </state>
+      </provider>
+    </entry>
+    <entry file="file://$PROJECT_DIR$/spiders/roll_bangqiu.py">
+      <provider selected="true" editor-type-id="text-editor">
+        <state relative-caret-position="-4204">
+          <caret line="23" column="5" selection-start-line="23" selection-start-column="5" selection-end-line="23" selection-end-column="5" />
+          <folding>
+            <element signature="e#46#61#0" expanded="true" />
+          </folding>
+        </state>
+      </provider>
+    </entry>
+    <entry file="file://$PROJECT_DIR$/spiders/roll_zuqiu.py">
+      <provider selected="true" editor-type-id="text-editor">
+        <state relative-caret-position="720">
+          <caret line="44" column="45" selection-start-line="44" selection-start-column="45" selection-end-line="44" selection-end-column="45" />
+          <folding>
+            <element signature="e#38#53#0" expanded="true" />
+          </folding>
+        </state>
+      </provider>
+    </entry>
+    <entry file="file://$PROJECT_DIR$/pipeline/zuqiu.py">
+      <provider selected="true" editor-type-id="text-editor">
+        <state relative-caret-position="420">
+          <caret line="63" column="8" lean-forward="true" selection-start-line="63" selection-start-column="8" selection-end-line="63" selection-end-column="14" />
+          <folding>
+            <element signature="e#0#15#0" expanded="true" />
+          </folding>
+        </state>
+      </provider>
+    </entry>
     <entry file="file://$PROJECT_DIR$/spiders/saiguo.py">
       <provider selected="true" editor-type-id="text-editor">
-        <state relative-caret-position="140">
-          <caret line="7" lean-forward="true" selection-start-line="7" selection-end-line="7" />
+        <state relative-caret-position="-561">
+          <caret line="198" column="58" selection-start-line="198" selection-start-column="58" selection-end-line="198" selection-end-column="58" />
           <folding>
             <element signature="e#0#15#0" expanded="true" />
           </folding>

hg3535/spiders/saiguo.py

@@ -21,7 +21,7 @@ class HgjieshuSpider(scrapy.Spider):
 
     def start_requests(self):
         for y in range(1, 5):
-            url = 'https://www.hg3535.cn/zh-cn/info-centre/sportsbook-info/results/{}/normal/1'.format(y)
+            url = 'https://www.hg3535.cn/zh-cn/info-centre/sportsbook-info/results/{}/normal/3'.format(y)
             yield scrapy.Request(url=url, callback=self.parse, dont_filter=True, meta={'pt': y})
 
     def parse(self, response):
@@ -145,8 +145,110 @@ class HgjieshuSpider(scrapy.Spider):
 
             # 网球赛果
             if pt == 3:
-                pass
+                league_ids = response.xpath('//div[@class="rt-l-bar tennis"]/@id').extract()
+                league_names = response.xpath('//div[@class="rt-l-bar tennis"]/span[@class="comp-txt"]/text()').extract()
+                for index in range(len(league_ids)):
+                    league_id = league_ids[index]
+                    league_name = league_names[index]
+                    response_data = response.xpath('//div[@id="dt-{}"]'.format(league_id)).extract_first()
+                    # response_data = response.xpath('//div[@id="dt-{}"]'.format('cmp-26405')).extract_first()
+                    data = etree.HTML(response_data)
+                    # 球队名
+                    team_names = data.xpath('//div[@class="rt-event"]/@title')
+                    # 赛果
+                    f_scores = data.xpath('.//div[@class="rt-set"]')
+                    # 时间
+                    stimes = data.xpath('//div[@class="rt-event"]/../div[1]/span/text()')
+                    # 赛事id
+                    match_ids = data.xpath('//div[@class="flex-wrap"]/../div[1]/@id')
+                    # 详细赛果信息, 比分等
+                    odd_datas = data.xpath('//div[contains(@class, "rt-sub ")]/table/tbody')
+                    for y in range(len(odd_datas)):
+                        match_id = match_ids[y].replace('e-', '')
+                        league_id = league_id.replace('cmp-', '')
+                        team_name = team_names[y].replace(' ', '').split('-')
+                        # 子集玩法
+                        child_data = odd_datas[y].xpath('./tr')
+                        # 主队
+                        h_name = team_name[0]
+                        # 客队
+                        a_name = team_name[1]
+                        print(h_name, a_name)
+                        # 全场
+                        f_score = f_scores[y].xpath('string(.)').replace(' ', '').replace('\r\n', '')
+                        # 正则时间匹配规则
+                        pattern = re.compile(r"\d{1,3}:\d{1,3}")
+                        stime = pattern.findall(stimes[y])[0]
+                        if child_data:
+                            for i in range(len(child_data)):
+                                if i == 0:
+                                    h_datas = child_data[i].xpath('.//tbody/tr[3]/td[contains(@class, "r-odds")]')
+                                    a_datas = child_data[i].xpath('.//tbody/tr[4]/td[contains(@class, "r-odds")]')
+                                    if h_datas and a_datas:
+                                        for x in range(len(h_datas)):
+                                            # 主队节得分
+                                            h_data = h_datas[x].text.replace(' ', '').replace('\r\n', '')
+                                            # 客队节得分
+                                            a_data = a_datas[x].text.replace(' ', '').replace('\r\n', '')
+                                else:
+                                    # 子玩法名
+                                    child_name = child_data[i].xpath('.//td[contains(@class, "r-bt ")]/text()')[0].replace(' ', '').replace('\r\n', '')
+                                    # 子玩法结果
+                                    child_play = child_data[i].xpath('.//td[@class="r-odds"]/span[@class="prop"]')[0]
+                                    play = child_play.xpath('string(.)')
+                                    print(child_name, play)
+                        else:
+                            pass
 
             # 棒球赛果
             if pt == 4:
-                pass
+                league_ids = response.xpath('//div[@class="rt-l-bar baseball"]/@id').extract()
+                league_names = response.xpath('//div[@class="rt-l-bar baseball"]/span[@class="comp-txt"]/text()').extract()
+                for index in range(len(league_ids)):
+                    league_id = league_ids[index]
+                    league_name = league_names[index]
+                    response_data = response.xpath('//div[@id="dt-{}"]'.format(league_id)).extract_first()
+                    # response_data = response.xpath('//div[@id="dt-{}"]'.format('cmp-26753')).extract_first()
+                    data = etree.HTML(response_data)
+                    # 球队名
+                    team_names = data.xpath('//div[@class="rt-event"]/@title')
+                    # 全场
+                    f_scores = data.xpath('.//div[@class="rt-ft"]')
+                    # 上半场
+                    h_scores = data.xpath('.//div[@class="rt-ht"]')
+                    # 时间
+                    stimes = data.xpath('//div[@class="rt-event"]/../div[1]/span/text()')
+                    # 子集玩法
+                    # odd_names = data.xpath('//div[@class="rt-sub rt-data-hide"]/table/tbody[2]/tr/td[2]')
+                    # 子集玩法结果
+                    # odd_plays = data.xpath('//div[@class="rt-sub rt-data-hide"]/table/tbody[2]/tr/td[3]/span')
+                    match_ids = data.xpath('//div[@class="flex-wrap"]/../div[1]/@id')
+                    odd_datas = data.xpath('//div[contains(@class, "rt-sub")]/table/tbody')
+                    for y in range(len(odd_datas)):
+                        match_id = match_ids[y].replace('e-', '')
+                        league_id = league_id.replace('cmp-', '')
+                        team_name = team_names[y].replace(' ', '').split('-')
+                        # 子集玩法
+                        odd_names = odd_datas[y].xpath('.//tr/td[2]')
+                        # 子集玩法结果
+                        odd_plays = odd_datas[y].xpath('.//tr/td[3]/span')
+                        # 主队
+                        h_name = team_name[0]
+                        # 客队
+                        a_name = team_name[1]
+                        print(h_name, a_name)
+                        # 上半场
+                        h_score = h_scores[y].xpath('string(.)').replace(' ', '').replace('\r\n', '')
+                        # 全场
+                        f_score = f_scores[y].xpath('string(.)').replace(' ', '').replace('\r\n', '')
+                        # 正则时间匹配规则
+                        pattern = re.compile(r"\d{1,3}:\d{1,3}")
+                        stime = pattern.findall(stimes[y])[0]
+                        if odd_names:
+                            for i in range(len(odd_names)):
+                                # 子玩法名
+                                name = odd_names[i].text.replace(' ', '').replace('\r\n', '')
+                                # 子玩法赛果
+                                play = odd_plays[i].xpath('string(.)').replace(' ', '').replace('\r\n', '')
+                        else:
+                            pass