feats

.gitignore

@@ -0,0 +1,3 @@
+.idea/*
+.vscode/*
+__pycache__/*

.vscode/launch.json

@@ -10,7 +10,7 @@
       "request": "launch",
       "program": "${workspaceRoot}/crawl/run.py",
       "console": "integratedTerminal",
-      "pythonPath": "d:/Application/python-3.8.5/python.exe"
+      "pythonPath": "d:/python3.8.4/python.exe"
     }
   ]
 }

.vscode/settings.json

@@ -1,5 +1,5 @@
 {
-  "python.pythonPath": "d:/Application/python-3.8.5/python.exe",
+  "python.pythonPath": "D:\\Python3.8.4\\python.exe",
   "deploy": {
     "targets": [
       {

crawl/spiders/football.py

@@ -10,9 +10,6 @@ class FootballSpider(scrapy.Spider):
     url = 'https://info.sporttery.cn/football/match_list.php'
     yield scrapy.Request(url, self.parseMatch, 'GET')
     
-    # 赛果开奖
-    yield self.startPareseResult()
-
   def parseMatch(self, response):
     cssOfMatches = '.all-wrap > .match_list .m-tab tr'
 
@@ -183,57 +180,3 @@ class FootballSpider(scrapy.Spider):
 
     # 入库
     ftprice.persist()
-
-  def startPareseResult(self):
-    url = 'https://info.sporttery.cn/football/match_result.php'
-    self._pages = -1
-    yield scrapy.Request(url, self.parseResult, 'GET')
-
-    if self._pages > 0:
-      for i in range (2, self._pages):
-        url = 'https://info.sporttery.cn/football/match_result.php?page=' + str(i)
-        yield scrapy.Request(url, self.parseResult, 'GET')
-
-  def parseResult(self, response):
-    # 先处理页码
-    if self._pages == -1:
-      pgNodes = response.css('.m-page .u-pg2')
-      if pgNodes is None or len(pgNodes) == 0:
-        self._pages = 0
-      else:
-        self._pages = len(pgNodes) + 1
-
-    # 处理比赛结果
-    cssOfMatch = '.all-wrap > .match_list .m-tab tr'
-    matches = response.css(cssOfMatch)
-    if matches is None: return
-    for matchNode in matches:
-      tdNodeList = matchNode.css('td')
-
-      matchTime = tdNodeList[0].css('::text').get()
-      matchWeek = tdNodeList[1].css('::text').get()
-      league = tdNodeList[2].css('::text').get()
-      leagueFullName = tdNodeList[2].attrib.get('title')  # 联赛全称
-      leagueName = '|'.join((league if league is not None else '', leagueFullName if leagueFullName is not None else ''))
-      homeTeam = self.trimBrackets(tdNodeList[3].css('.zhu::text').get()) # 主队
-      awayTeam = self.trimBrackets(tdNodeList[3].css('.ke::text').get()) # 客队
-      half = tdNodeList[4].css('span::text').get()
-      whole = tdNodeList[5].css('span::text').get()
-      status = tdNodeList[9].css('span::text').get()
-
-      FTResult(
-        matchTime,
-        matchWeek,
-        leagueName,
-        homeTeam,
-        awayTeam,
-        '', # 不需要单固
-        half,
-        whole,
-        status
-      ).persist()
-
-  def trimBrackets(self, str):
-    if str is None: return ''
-
-    return re.sub(r'\(.*\)', '', str)

crawl/spiders/football_result.py

@@ -61,7 +61,7 @@ class FootballSpider(scrapy.Spider):
     # 下一页
     nextPG = int(current) + 1
     url = 'https://info.sporttery.cn/football/match_result.php?page=' + str(nextPG)
-    yield scrapy.Request(url, self.parseResult, 'GET')
+    return scrapy.Request(url, self.parseResult, 'GET')
 
 
   def trimBrackets(self, str):