add my dev

.idea/workspace.xml

@@ -1,29 +1,7 @@
 <?xml version="1.0" encoding="UTF-8"?>
 <project version="4">
   <component name="ChangeListManager">
-    <list default="true" id="87e036c0-2f0b-4473-b850-18ac24a30c0e" name="Default Changelist" comment="'更新'">
-      <change afterPath="$PROJECT_DIR$/collectSports/utils/__init__.py" afterDir="false" />
-      <change afterPath="$PROJECT_DIR$/collectSports/utils/helper.py" afterDir="false" />
-      <change beforePath="$PROJECT_DIR$/.idea/misc.xml" beforeDir="false" afterPath="$PROJECT_DIR$/.idea/misc.xml" afterDir="false" />
-      <change beforePath="$PROJECT_DIR$/.idea/sports_collect.iml" beforeDir="false" afterPath="$PROJECT_DIR$/.idea/sports_collect.iml" afterDir="false" />
-      <change beforePath="$PROJECT_DIR$/.idea/workspace.xml" beforeDir="false" afterPath="$PROJECT_DIR$/.idea/workspace.xml" afterDir="false" />
-      <change beforePath="$PROJECT_DIR$/collectSports/biz/__init__.py" beforeDir="false" afterPath="$PROJECT_DIR$/collectSports/biz/__init__.py" afterDir="false" />
-      <change beforePath="$PROJECT_DIR$/collectSports/conf/hg0088.json" beforeDir="false" afterPath="$PROJECT_DIR$/collectSports/conf/hg0088.json" afterDir="false" />
-      <change beforePath="$PROJECT_DIR$/collectSports/conf/settings.json" beforeDir="false" afterPath="$PROJECT_DIR$/collectSports/conf/settings.json" afterDir="false" />
-      <change beforePath="$PROJECT_DIR$/collectSports/items.py" beforeDir="false" afterPath="$PROJECT_DIR$/collectSports/items.py" afterDir="false" />
-      <change beforePath="$PROJECT_DIR$/collectSports/main.py" beforeDir="false" afterPath="$PROJECT_DIR$/collectSports/main.py" afterDir="false" />
-      <change beforePath="$PROJECT_DIR$/collectSports/mcollect/hg0088/LinkPool.py" beforeDir="false" afterPath="$PROJECT_DIR$/collectSports/mcollect/hg0088/LinkPool.py" afterDir="false" />
-      <change beforePath="$PROJECT_DIR$/collectSports/mcollect/hg0088/Resolver.py" beforeDir="false" afterPath="$PROJECT_DIR$/collectSports/mcollect/hg0088/Resolver.py" afterDir="false" />
-      <change beforePath="$PROJECT_DIR$/collectSports/mcollect/hg0088/Storage.py" beforeDir="false" afterPath="$PROJECT_DIR$/collectSports/mcollect/hg0088/Storage.py" afterDir="false" />
-      <change beforePath="$PROJECT_DIR$/collectSports/mcollect/hg0088/__init__.py" beforeDir="false" afterPath="$PROJECT_DIR$/collectSports/mcollect/hg0088/__init__.py" afterDir="false" />
-      <change beforePath="$PROJECT_DIR$/collectSports/mcollect/hg0088/fcache/hg0088.zaopan.js" beforeDir="false" afterPath="$PROJECT_DIR$/collectSports/mcollect/hg0088/fcache/hg0088.zaopan.js" afterDir="false" />
-      <change beforePath="$PROJECT_DIR$/collectSports/pipelines/sportslst.py" beforeDir="false" afterPath="$PROJECT_DIR$/collectSports/pipelines/sportslst.py" afterDir="false" />
-      <change beforePath="$PROJECT_DIR$/collectSports/settings.py" beforeDir="false" afterPath="$PROJECT_DIR$/collectSports/settings.py" afterDir="false" />
-      <change beforePath="$PROJECT_DIR$/collectSports/spiders/demo.py" beforeDir="false" afterPath="$PROJECT_DIR$/collectSports/spiders/demo.py" afterDir="false" />
-      <change beforePath="$PROJECT_DIR$/collectSports/spiders/sports.py" beforeDir="false" afterPath="$PROJECT_DIR$/collectSports/spiders/sports.py" afterDir="false" />
-      <change beforePath="$PROJECT_DIR$/collectSports/spiders/sportslst.py" beforeDir="false" afterPath="$PROJECT_DIR$/collectSports/spiders/sportslst.py" afterDir="false" />
-      <change beforePath="$PROJECT_DIR$/pycomm/mongo.py" beforeDir="false" afterPath="$PROJECT_DIR$/pycomm/mongo.py" afterDir="false" />
-    </list>
+    <list default="true" id="87e036c0-2f0b-4473-b850-18ac24a30c0e" name="Default Changelist" comment="'更新'" />
     <option name="EXCLUDED_CONVERTED_TO_IGNORED" value="true" />
     <option name="SHOW_DIALOG" value="false" />
     <option name="HIGHLIGHT_CONFLICTS" value="true" />
@@ -34,86 +12,7 @@
     <SUITE FILE_PATH="coverage/sports_collect$main.coverage" NAME="main Coverage Results" MODIFIED="1560765158051" SOURCE_PROVIDER="com.intellij.coverage.DefaultCoverageFileProvider" RUNNER="coverage.py" COVERAGE_BY_TEST_ENABLED="true" COVERAGE_TRACING_ENABLED="false" WORKING_DIRECTORY="$PROJECT_DIR$/collectSports" />
   </component>
   <component name="FileEditorManager">
-    <leaf SIDE_TABS_SIZE_LIMIT_KEY="300">
-      <file pinned="false" current-in-tab="false">
-        <entry file="file://$PROJECT_DIR$/collectSports/spiders/sports.py">
-          <provider selected="true" editor-type-id="text-editor">
-            <state relative-caret-position="-1406">
-              <caret line="39" column="26" lean-forward="true" selection-start-line="39" selection-start-column="26" selection-end-line="39" selection-end-column="26" />
-              <folding>
-                <element signature="e#24#35#0" expanded="true" />
-              </folding>
-            </state>
-          </provider>
-        </entry>
-      </file>
-      <file pinned="false" current-in-tab="true">
-        <entry file="file://$PROJECT_DIR$/collectSports/pipelines/sportslst.py">
-          <provider selected="true" editor-type-id="text-editor">
-            <state relative-caret-position="200">
-              <caret line="110" column="25" selection-start-line="110" selection-start-column="25" selection-end-line="110" selection-end-column="25" />
-              <folding>
-                <element signature="e#193#204#0" expanded="true" />
-              </folding>
-            </state>
-          </provider>
-        </entry>
-      </file>
-      <file pinned="false" current-in-tab="false">
-        <entry file="file://$PROJECT_DIR$/collectSports/mcollect/hg0088/Storage.py">
-          <provider selected="true" editor-type-id="text-editor">
-            <state relative-caret-position="147">
-              <caret line="79" column="33" lean-forward="true" selection-start-line="79" selection-start-column="6" selection-end-line="79" selection-end-column="33" />
-              <folding>
-                <element signature="e#0#33#0" expanded="true" />
-              </folding>
-            </state>
-          </provider>
-        </entry>
-      </file>
-      <file pinned="false" current-in-tab="false">
-        <entry file="file://$PROJECT_DIR$/collectSports/spiders/sportslst.py">
-          <provider selected="true" editor-type-id="text-editor">
-            <state relative-caret-position="-268">
-              <caret line="34" column="13" selection-start-line="34" selection-start-column="13" selection-end-line="34" selection-end-column="13" />
-            </state>
-          </provider>
-        </entry>
-      </file>
-      <file pinned="false" current-in-tab="false">
-        <entry file="file://$PROJECT_DIR$/collectSports/utils/helper.py">
-          <provider selected="true" editor-type-id="text-editor">
-            <state relative-caret-position="150">
-              <caret line="6" column="17" lean-forward="true" selection-start-line="6" selection-start-column="17" selection-end-line="6" selection-end-column="17" />
-              <folding>
-                <element signature="e#0#11#0" expanded="true" />
-              </folding>
-            </state>
-          </provider>
-        </entry>
-      </file>
-      <file pinned="false" current-in-tab="false">
-        <entry file="file://$PROJECT_DIR$/collectSports/mcollect/hg0088/__init__.py">
-          <provider selected="true" editor-type-id="text-editor">
-            <state relative-caret-position="133">
-              <caret line="12" lean-forward="true" selection-start-line="12" selection-end-line="12" />
-            </state>
-          </provider>
-        </entry>
-      </file>
-      <file pinned="false" current-in-tab="false">
-        <entry file="file://$PROJECT_DIR$/collectSports/main.py">
-          <provider selected="true" editor-type-id="text-editor">
-            <state relative-caret-position="158">
-              <caret line="9" column="41" lean-forward="true" selection-start-line="9" selection-start-column="41" selection-end-line="9" selection-end-column="41" />
-              <folding>
-                <element signature="e#0#10#0" expanded="true" />
-              </folding>
-            </state>
-          </provider>
-        </entry>
-      </file>
-    </leaf>
+    <leaf SIDE_TABS_SIZE_LIMIT_KEY="300" />
   </component>
   <component name="FileTemplateManagerImpl">
     <option name="RECENT_TEMPLATES">
@@ -196,43 +95,6 @@
               <item name="sports_collect" type="462c0819:PsiDirectoryNode" />
               <item name="collectSports" type="462c0819:PsiDirectoryNode" />
             </path>
-            <path>
-              <item name="sports_collect" type="b2602c69:ProjectViewProjectNode" />
-              <item name="sports_collect" type="462c0819:PsiDirectoryNode" />
-              <item name="collectSports" type="462c0819:PsiDirectoryNode" />
-              <item name="conf" type="462c0819:PsiDirectoryNode" />
-            </path>
-            <path>
-              <item name="sports_collect" type="b2602c69:ProjectViewProjectNode" />
-              <item name="sports_collect" type="462c0819:PsiDirectoryNode" />
-              <item name="collectSports" type="462c0819:PsiDirectoryNode" />
-              <item name="mcollect" type="462c0819:PsiDirectoryNode" />
-            </path>
-            <path>
-              <item name="sports_collect" type="b2602c69:ProjectViewProjectNode" />
-              <item name="sports_collect" type="462c0819:PsiDirectoryNode" />
-              <item name="collectSports" type="462c0819:PsiDirectoryNode" />
-              <item name="mcollect" type="462c0819:PsiDirectoryNode" />
-              <item name="hg0088" type="462c0819:PsiDirectoryNode" />
-            </path>
-            <path>
-              <item name="sports_collect" type="b2602c69:ProjectViewProjectNode" />
-              <item name="sports_collect" type="462c0819:PsiDirectoryNode" />
-              <item name="collectSports" type="462c0819:PsiDirectoryNode" />
-              <item name="pipelines" type="462c0819:PsiDirectoryNode" />
-            </path>
-            <path>
-              <item name="sports_collect" type="b2602c69:ProjectViewProjectNode" />
-              <item name="sports_collect" type="462c0819:PsiDirectoryNode" />
-              <item name="collectSports" type="462c0819:PsiDirectoryNode" />
-              <item name="spiders" type="462c0819:PsiDirectoryNode" />
-            </path>
-            <path>
-              <item name="sports_collect" type="b2602c69:ProjectViewProjectNode" />
-              <item name="sports_collect" type="462c0819:PsiDirectoryNode" />
-              <item name="collectSports" type="462c0819:PsiDirectoryNode" />
-              <item name="utils" type="462c0819:PsiDirectoryNode" />
-            </path>
           </expand>
           <select />
         </subPane>
@@ -310,7 +172,7 @@
       <workItem from="1560848283915" duration="215000" />
       <workItem from="1560848805159" duration="1220000" />
       <workItem from="1560906874868" duration="28010000" />
-      <workItem from="1560993078342" duration="14417000" />
+      <workItem from="1560993078342" duration="15258000" />
     </task>
     <task id="LOCAL-00001" summary="'更新'">
       <created>1560503640918</created>
@@ -323,11 +185,10 @@
     <servers />
   </component>
   <component name="TimeTrackingManager">
-    <option name="totallyTimeSpent" value="84684000" />
+    <option name="totallyTimeSpent" value="85525000" />
   </component>
   <component name="ToolWindowManager">
     <frame x="-7" y="-7" width="1295" height="695" extended-state="6" />
-    <editor active="true" />
     <layout>
       <window_info active="true" content_ui="combo" id="Project" order="0" visible="true" weight="0.18027486" />
       <window_info id="Structure" order="1" side_tool="true" weight="0.25" />
@@ -387,15 +248,6 @@
     <option name="LAST_COMMIT_MESSAGE" value="'更新'" />
   </component>
   <component name="XDebuggerManager">
-    <breakpoint-manager>
-      <breakpoints>
-        <line-breakpoint enabled="true" suspend="THREAD" type="python-line">
-          <url>file://$PROJECT_DIR$/collectSports/pipelines/sportslst.py</url>
-          <line>98</line>
-          <option name="timeStamp" value="33" />
-        </line-breakpoint>
-      </breakpoints>
-    </breakpoint-manager>
     <watches-manager>
       <configuration name="PythonConfigurationType">
         <watch expression="int(match_time.group(1))" language="Python" />

collectSports/requirements.txt

@@ -0,0 +1,74 @@
+aiopg==0.16.0
+APScheduler==3.6.0
+asn1crypto==0.24.0
+asyncio==3.4.3
+attrs==19.1.0
+Automat==0.7.0
+certifi==2019.3.9
+cffi==1.12.2
+chardet==3.0.4
+Click==7.0
+colorama==0.4.1
+constantly==15.1.0
+cryptography==2.6.1
+cssselect==1.0.3
+defusedxml==0.5.0
+et-xmlfile==1.0.1
+Flask==1.0.2
+Flask-Login==0.4.1
+hyperlink==18.0.0
+idna==2.8
+incremental==17.5.0
+inotify==0.2.10
+itsdangerous==1.1.0
+jdcal==1.4
+Jinja2==2.10
+jmespath==0.9.4
+jsmin==2.2.2
+jsonpath==0.81
+lxml==4.3.2
+MarkupSafe==1.1.1
+nose==1.3.7
+openpyxl==2.6.1
+parsel==1.5.1
+peewee==2.8.0
+psycopg2==2.7.7
+psycopg2-binary==2.8
+pyasn1==0.4.5
+pyasn1-modules==0.2.4
+pycparser==2.19
+pycurl==7.43.0.2
+pydispatch==1.1.0
+PyDispatcher==2.0.5
+PyHamcrest==1.9.0
+pymongo==3.8.0
+PyMySQL==0.9.3
+pyOpenSSL==19.0.0
+pypiwin32==223
+pyquery==1.4.0
+pyspider==0.3.10
+python-dateutil==2.8.0
+pytz==2018.9
+pywin32==224
+PyYAML==3.13
+queuelib==1.5.0
+redis==3.2.1
+requests==2.21.0
+schedule==0.6.0
+Scrapy==1.6.0
+scrapy-redis==0.6.8
+scrapy-splash==0.7.2
+scrapyd-client==1.1.0
+service-identity==18.1.0
+six==1.12.0
+tblib==1.3.2
+tornado==4.5.3
+Twisted==18.9.0
+tzlocal==1.5.1
+u-msgpack-python==2.5.1
+urllib3==1.24.1
+uuid==1.30
+w3lib==1.20.0
+Werkzeug==0.14.1
+WsgiDAV==2.4.1
+zope.interface==4.6.0