feats

crawl/run.py

@@ -7,4 +7,4 @@ print(dirpath)
 # 添加环境变量
 sys.path.append(dirpath)
 # 启动爬虫,第三个参数为爬虫name
-execute(['scrapy','crawl','football-result'])
+execute(['scrapy','crawl','football'])

crawl/spiders/football.py

@@ -1,10 +1,14 @@
 import scrapy
 import re
+import logging
 from crawl.comm.football import FTMatch, FTPrice, FTResult, WDLOdds, WDLSpreadOdds, ScoreResult, PointsResult, DoubleResult
 
 class FootballSpider(scrapy.Spider):
   name = 'football'
 
+  # 所有比赛
+  _matchesMap = {}
+
   def start_requests(self):
     # 受注赛事
     url = 'https://info.sporttery.cn/football/match_list.php'
@@ -33,9 +37,12 @@ class FootballSpider(scrapy.Spider):
       score = self.parsePassWay(tdNodeList[8].css('div'))
       points = self.parsePassWay(tdNodeList[9].css('div'))
       double = self.parsePassWay(tdNodeList[10].css('div'))
+      
+      matchLink = tdNodeList[2].css('a').attrib.get('href')
+      matchId = re.sub(r'^.*m=', '', matchLink)
 
       # 入库
-      self._currentMatch = FTMatch(
+      match = FTMatch(
         matchWeek,
         matchTime,
         leagueName,
@@ -48,12 +55,13 @@ class FootballSpider(scrapy.Spider):
         points,
         double
       )
-      self._currentMatch.persist()
+      match.persist()
+
+      # 加入到临时字典里
+      self._matchesMap[matchId] = match
 
       # 请求比赛赔率
       if saleStatus == '已开售':
-        matchLink = tdNodeList[2].css('a').attrib.get('href')
-        matchId = re.sub(r'^.*m=', '', matchLink)
         priceAPI = 'https://i.sporttery.cn/api/fb_match_info/get_pool_rs?mid=' + matchId
         yield scrapy.Request(priceAPI, self.parsePrice, 'GET')
 
@@ -80,16 +88,20 @@ class FootballSpider(scrapy.Spider):
       return 'wait'
 
   def parsePrice(self, response):
+    logging.info("采集数据源 ---> %s" % response.url)
     data = response.json()
     res = data.get('result').get('odds_list')
 
+    matchId = re.sub(r'^.*get_pool_rs\?mid=', '', response.url)
+    match = self._matchesMap[matchId]
+
     ftprice = FTPrice(
       '', # 不需要 matchId
-      self._currentMatch.matchWeek,
-      self._currentMatch.matchTime,
-      self._currentMatch.league,
-      self._currentMatch.homeTeam,
-      self._currentMatch.awayTeam
+      match.matchWeek,
+      match.matchTime,
+      match.league,
+      match.homeTeam,
+      match.awayTeam
     )
 
     try:
@@ -178,5 +190,8 @@ class FootballSpider(scrapy.Spider):
     finally:
       pass
 
+    #
+    logging.info("采集到数据 --> %s" % ftprice.toString())
+
     # 入库
     ftprice.persist()