feats

.idea/.gitignore

@@ -0,0 +1,3 @@
+# Default ignored files
+/shelf/
+/workspace.xml

.idea/codeStyles/codeStyleConfig.xml

@@ -0,0 +1,5 @@
+<component name="ProjectCodeStyleConfiguration">
+  <state>
+    <option name="PREFERRED_PROJECT_CODE_STYLE" value="Default" />
+  </state>
+</component>

.idea/crawl.iml

@@ -0,0 +1,11 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<module type="PYTHON_MODULE" version="4">
+  <component name="NewModuleRootManager">
+    <content url="file://$MODULE_DIR$" />
+    <orderEntry type="inheritedJdk" />
+    <orderEntry type="sourceFolder" forTests="false" />
+  </component>
+  <component name="TestRunnerService">
+    <option name="PROJECT_TEST_RUNNER" value="Twisted Trial" />
+  </component>
+</module>

.idea/inspectionProfiles/profiles_settings.xml

@@ -0,0 +1,7 @@
+<component name="InspectionProjectProfileManager">
+  <settings>
+    <option name="PROJECT_PROFILE" value="Default" />
+    <option name="USE_PROJECT_PROFILE" value="false" />
+    <version value="1.0" />
+  </settings>
+</component>

.idea/misc.xml

@@ -0,0 +1,4 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project version="4">
+  <component name="ProjectRootManager" version="2" project-jdk-name="Python 3.8" project-jdk-type="Python SDK" />
+</project>

.idea/modules.xml

@@ -0,0 +1,8 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project version="4">
+  <component name="ProjectModuleManager">
+    <modules>
+      <module fileurl="file://$PROJECT_DIR$/.idea/crawl.iml" filepath="$PROJECT_DIR$/.idea/crawl.iml" />
+    </modules>
+  </component>
+</project>

.idea/vcs.xml

@@ -0,0 +1,6 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project version="4">
+  <component name="VcsDirectoryMappings">
+    <mapping directory="$PROJECT_DIR$" vcs="Git" />
+  </component>
+</project>

.vscode/launch.json

@@ -9,7 +9,8 @@
       "type": "python",
       "request": "launch",
       "program": "${workspaceRoot}/crawl/run.py",
-      "console": "integratedTerminal"
+      "console": "integratedTerminal",
+      "pythonPath": "d:/Application/python-3.8.5/python.exe"
     }
   ]
 }

.vscode/settings.json

@@ -0,0 +1,3 @@
+{
+  "python.pythonPath": "d:/Application/python-3.8.5/python.exe"
+}

crawl/comm/basketball.py

@@ -0,0 +1,320 @@
+from .mydb import MyDB
+
+# 比赛
+class BSTMatch:
+  def __init__(self,
+               # 比赛周编码
+               matchWeek,
+
+               # 比赛时间
+               matchTime,
+
+               # 联赛
+               league,
+
+               # 主队
+               homeTeam,
+
+               # 客队
+               awayTeam,
+
+               # 开售状态
+               status,
+
+               # 胜负
+               wl,
+
+               # 胜负(让)
+               wls,
+
+               # 大小分
+               score,
+
+               # 剩分差
+               points
+               ):
+    self.matchWeek = matchWeek
+    self.matchTime = matchTime
+    self.league = league
+    self.homeTeam = homeTeam
+    self.awayTeam = awayTeam
+    self.status = status
+    self.wl = wl
+    self.wls = wls
+    self.score = score
+    self.points = points
+
+  def toString(self):
+    return ";".join((
+      self.matchWeek,
+      self.matchTime,
+      self.league,
+      self.homeTeam,
+      self.awayTeam,
+      self.status,
+      self.wl,
+      self.wls,
+      self.score,
+      self.points
+    ))
+
+  def persist(self):
+    cursor = MyDB.getCursor()
+    if cursor is None:
+      return
+
+    sql = "insert into ta_crawl_ball(ball_type, data_type, content) values('basketball', 'match', %s)"
+    cursor.execute(sql, self.toString())
+    MyDB.commit()
+    cursor.close()
+
+
+# 赔率
+class BSTPrice:
+  def __init__(self,
+               # 比赛ID
+               matchId,
+
+               # 比赛周编码
+               matchWeek,
+
+               # 比赛时间
+               matchTime,
+
+               # 联赛
+               league,
+
+               # 主队
+               homeTeam,
+
+               # 客队
+               awayTeam,
+
+               # 胜负
+               wlOdds=None,
+
+               # 胜负(让) - [w, l, s]
+               wlsOdds=None,
+
+               # 大小分 - [大, 小, 总分]
+               scoreResult=None,
+
+               # 剩分差 - [主胜6 + 客胜 6]
+               pointsResult=None
+               ):
+    self.matchId = matchId
+    self.matchWeek = matchWeek
+    self.matchTime = matchTime
+    self.league = league
+    self.homeTeam = homeTeam
+    self.awayTeam = awayTeam
+    self.wlOdds = wlOdds
+    self.wlsOdds = wlsOdds
+    self.scoreResult = scoreResult
+    self.pointsResult = pointsResult
+
+  def toString(self):
+    return ";".join((
+      self.matchId,
+      self.matchWeek,
+      self.matchTime,
+      self.league,
+      self.homeTeam,
+      self.awayTeam,
+      self.wlOdds.toString(),
+      self.wlsOdds.toString(),
+      self.scoreResult.toString(),
+      self.pointsResult.toString()
+    ))
+
+  def persist(self):
+    cursor = MyDB.getCursor()
+    if cursor is None:
+      return
+
+    sql = "insert into ta_crawl_ball(ball_type, data_type, content) values('basketball', 'price', %s)"
+    cursor.execute(sql, self.toString())
+    MyDB.commit()
+    cursor.close()
+
+
+# 胜平负
+class WLOdds:
+  def __init__(self,
+               # 胜
+               win=None,
+
+               # 负
+               lose=None
+               ):
+    self.win = win
+    self.lose = lose
+
+  def datas(self, datas=[]):
+    self.win = datas[0]
+    self.lose = datas[1]
+
+  def toString(self):
+    return '|'.join((self.win, self.lose))
+
+
+# 胜平负(让)
+class WLSpreadOdds:
+  def __init__(self,
+               # 胜
+               win=None,
+
+               # 负
+               lose=None,
+
+               # 让球
+               spread=None
+               ):
+    self.win = win
+    self.lose = lose
+    self.spread = spread
+
+  def datas(self, datas=[]):
+    self.win = datas[0]
+    self.lose = datas[1]
+    self.spread = datas[2]
+
+  def toString(self):
+    return '|'.join((self.win, self.lose, self.spread))
+
+
+# 大小分
+class ScoreResult:
+  def __init__(self,
+               big = None, # 大
+               small = None, # 小
+               total = None # 预设总分
+               ):
+    self.big = big
+    self.small = small
+    self.total = total
+
+  def toString(self):
+    return '|'.join((
+      self.big,
+      self.small,
+      self.total)
+    )
+
+# 剩分差
+class PointsResult:
+  def __init__(self,
+               w15=None,  # 胜 1-5
+               w610=None,  # 胜 6-10
+               w1115=None,  # 胜 11-15
+               w1620=None,  # 胜 16-20
+               w2125=None,  # 胜 21-25
+               w99=None,  # 胜 26+
+               l15=None,  # 负 1-5
+               l610=None,  # 负 6-10
+               l1115=None,  # 负 11-15
+               l1620=None,  # 负 16-20
+               l2125=None,  # 负 21-25
+               l99=None  # 负 26+
+               ):
+    self.w15 = w15
+    self.w610 = w610
+    self.w1115 = w1115
+    self.w1620 = w1620
+    self.w2125 = w2125
+    self.w99 = w99
+    self.l15 = l15
+    self.l610 = l610
+    self.l1115 = l1115
+    self.l1620 = l1620
+    self.l2125 = l2125
+    self.l99 = l99
+
+  def datas(self, datas=[]):
+    self.w15 = datas[0]
+    self.w610 = datas[1]
+    self.w1115 = datas[2]
+    self.w1620 = datas[3]
+    self.w2125 = datas[4]
+    self.w99 = datas[5]
+    self.l15 = datas[6]
+    self.l610 = datas[7]
+    self.l1115 = datas[8]
+    self.l1620 = datas[9]
+    self.l2125 = datas[10]
+    self.l99 = datas[11]
+
+  def toString(self):
+    return '|'.join((
+      self.w15,
+      self.w610,
+      self.w1115,
+      self.w1620,
+      self.w2125,
+      self.w99,
+      self.l15,
+      self.l610,
+      self.l1115,
+      self.l1620,
+      self.l2125,
+      self.l99)
+    )
+
+
+# 结果
+class BSTResult:
+  def __init__(self,
+               # 比赛时间
+               matchTime,
+
+               # 周次
+               matchWeek,
+
+               # 联赛
+               league,
+
+               # 主队
+               homeTeam,
+
+               # 客队
+               awayTeam,
+
+               # 是否单固场次
+               single,
+
+               # 比分
+               score,
+
+               # 状态
+               status
+               ):
+    self.matchTime = matchTime
+    self.matchWeek = matchWeek
+    self.league = league
+    self.homeTeam = homeTeam
+    self.awayTeam = awayTeam
+    self.single = single
+    self.score = score
+    self.status = status
+
+  def toString(self):
+    return ';'.join((
+      self.matchTime,
+      self.matchWeek,
+      self.league,
+      self.homeTeam,
+      self.awayTeam,
+      self.single,
+      self.score,
+      self.status
+    ))
+
+  def persist(self):
+    cursor = MyDB.getCursor()
+    if cursor is None:
+      return
+
+    sql = "insert into ta_crawl_ball(ball_type, data_type, content) values('basketball', 'result', %s)"
+    cursor.execute(sql, self.toString())
+    MyDB.commit()
+    cursor.close()

crawl/comm/football.py

@@ -63,7 +63,7 @@ class FTMatch:
     if cursor is None:
       return
     
-    sql = "insert into ta_crawl_football(data_type, content) values('match', %s)"
+    sql = "insert into ta_crawl_ball(ball_type, data_type, content) values('football', 'match', %s)"
     cursor.execute(sql, self.toString())
     MyDB.commit()
     cursor.close()
@@ -136,7 +136,7 @@ class FTPrice:
     if cursor is None:
       return
     
-    sql = "insert into ta_crawl_football(data_type, content) values('price', %s)"
+    sql = "insert into ta_crawl_ball(ball_type, data_type, content) values('football', 'price', %s)"
     cursor.execute(sql, self.toString())
     MyDB.commit()
     cursor.close()
@@ -509,7 +509,7 @@ class FTResult:
     if cursor is None:
       return
     
-    sql = "insert into ta_crawl_football(data_type, content) values('result', %s)"
+    sql = "insert into ta_crawl_ball(ball_type, data_type, content) values('football', 'result', %s)"
     cursor.execute(sql, self.toString())
     MyDB.commit()
     cursor.close()

crawl/spiders/basketball_match.py

@@ -0,0 +1,69 @@
+import scrapy
+import time
+from crawl.comm.basketball import BSTMatch
+
+class BasketballSpider(scrapy.Spider):
+  name = "basketball-match"
+
+  def start_requests(self):
+    # 受注比赛
+    url = "https://www.lottery.gov.cn/basketball/match_list.jspx"
+    yield scrapy.Request(url, self.parseMatch)
+  
+  def parseMatch(self, response):
+    cssMain = ".xxsj table tr"
+    cssDetail = "td"
+
+    # 获取所有比赛
+    matches = response.css(cssMain)
+    for node in matches[1:]:  # 标题行忽略
+      prop = node.css(cssDetail)
+
+      # 小于 2 个 td 为无效行
+      if len(prop) < 2:
+        continue
+
+      matchWeek = prop[0].css('::text').get()
+      league = prop[1].css('::text').get()
+      team = prop[2].css('::text').get().split('VS')
+      homeTeam = team[0].strip()
+      awayTeam = team[1].strip()
+      matchTime = prop[3].css('::text').get()
+      status = prop[5].css('::text').get()
+      wl = self.parsePassWay(prop[6].css('img'))
+      wls = self.parsePassWay(prop[7].css('img'))
+      score = self.parsePassWay(prop[8].css('img'))
+      points = self.parsePassWay(prop[9].css('img'))
+
+      BSTMatch(
+        matchWeek,
+        matchTime,
+        league,
+        homeTeam,
+        awayTeam,
+        status,
+        wl,
+        wls,
+        score,
+        points
+      ).persist()
+  
+  def parsePassWay(self, img):
+    # 待开售
+    if img is None:
+      return 'wait'
+
+    # 图片地址
+    src = img.attrib['src']
+
+    # 开售单关方式和过关方式
+    if "ball2_11.png" in src:
+      return 'pass&single'
+
+    # 仅开售过关方式
+    elif "ball2_1.png" in src:
+      return 'pass'
+
+    # 未开售此玩法
+    else:
+      return 'no'

crawl/spiders/basketball_price.py

@@ -0,0 +1,72 @@
+import scrapy
+import time
+from crawl.comm.basketball import BSTPrice, WLOdds, WLSpreadOdds, ScoreResult, PointsResult
+
+class BasketballSpider(scrapy.Spider):
+  name = "basketball-price"
+
+  def start_requests(self):
+    # 赔率
+    url = "https://www.lottery.gov.cn/basketball/counter.jspx"
+    yield scrapy.Request(url, self.parsePrice)
+  
+  def parsePrice(self, response):
+    cssMain = "#content .article .articleCon .section"
+    cssDetail = ".saishi"
+    cssOther = ".saishiCon table td"
+
+    # 获取所有比赛
+    matches = response.css(cssMain)
+    for node in matches:
+
+      # 比赛ID
+      matchId = node.attrib['match_id']
+      matchTime = node.attrib['match_time']
+      matchWeek = node.attrib['match_week']
+      league = node.attrib['league_val']
+
+      # 其他相关属性
+      details = node.css(cssDetail).css("td")
+      homeTeam = details[3].css('::text').get()
+      awayTeam = details[4].css('::text').get()
+
+      match = BSTPrice(
+        matchId,
+        matchWeek,
+        matchTime,
+        league,
+        homeTeam,
+        awayTeam,
+        WLOdds(
+          details[6].css('strong::text').get(),
+          details[5].css('strong::text').get()
+        ),
+        WLSpreadOdds(
+          details[9].css('strong::text').get(),
+          details[7].css('strong::text').get(),
+          details[8].css('strong font::text').get()
+        ),
+        ScoreResult(
+          details[10].css('strong::text').get(),
+          details[12].css('strong::text').get(),
+          details[11].css('strong::text').get()
+        )
+      )
+
+      # 剩分差
+      pointsResult = PointsResult()
+      datas1 = []
+      lst = node.css(cssOther)
+      for it1 in lst[9:15]:
+        datas1 += [it1.css('strong::text').get()]
+
+      datas2 = []
+      for it2 in lst[16:22]:
+        datas2 += [it2.css('strong::text').get()]
+
+      pointsResult.datas(datas2 + datas1)
+      match.pointsResult = pointsResult
+
+      # 入库
+      match.persist()
+  

crawl/spiders/basketball_result.py

@@ -0,0 +1,56 @@
+import scrapy
+import time
+from crawl.comm.basketball import BSTResult
+
+class BasketballSpider(scrapy.Spider):
+  name = "basketball-result"
+
+  def start_requests(self):
+    # 开奖
+    today = time.strftime("%Y-%m-%d")
+    url = "https://www.lottery.gov.cn/basketball/result_99.jspx?startDate="+today+"&endDate="+today+"&f_league_id=0&f_league_name=%E5%85%A8%E9%83%A8%E8%81%94%E8%B5%9B&single=off"
+    yield scrapy.Request(url, self.parseResult)
+
+  def parseResult(self, response):
+    cssMain = ".xxsj table tr"
+
+    # 获取所有比赛
+    matches = response.css(cssMain)
+    for node in matches[1:-1]:  # 标题行忽略以及末尾一行
+      prop = node.css("td")
+      if len(prop) < 7:
+        continue
+
+      matchTime = prop[0].css('::text').get()
+      matchWeek = prop[1].css('::text').get()
+      league = prop[2].css('::text').get()
+      team = prop[3].css('a::text').getall()
+      if team is None or len(team) == 0:
+        team = prop[3].css('::text').get().split('VS')
+      homeTeam = team[0].strip()
+      awayTeam = team[1].strip()
+      single = self.isSingle(prop[3].attrib.get('class'))
+      tmp = prop[4].css('::text').get()
+      score = tmp.strip() if tmp is not None else ""
+      tmp = prop[5].css('::text').get()
+      status = tmp.strip() if tmp is not None else ""
+
+      BSTResult(
+        matchTime,
+        matchWeek,
+        league,
+        homeTeam,
+        awayTeam,
+        single,
+        score,
+        status
+      ).persist()
+
+  def isSingle(self, eleCls):
+    if eleCls is None:
+      return '0'
+    
+    if 'dan' in eleCls:
+      return '1'
+    else:
+      return '0'

main.py

@@ -0,0 +1,7 @@
+from scrapy.cmdline import execute
+import sys
+import os
+
+sys.path.append(os.path.dirname(os.path.abspath(__file__)))
+
+execute(['scrapy', 'crawl', 'basketball-result'])  # 你需要将此处的spider_name替换为你自己的爬虫名称