feats

.vscode/settings.json

@@ -1,3 +1,24 @@
 {
-  "python.pythonPath": "d:/Application/python-3.8.5/python.exe"
+  "python.pythonPath": "d:/Application/python-3.8.5/python.exe",
+  "deploy": {
+    "targets": [
+      {
+        "type": "sftp",
+        "name": "NiuCai Crawl Deploy",
+        "description": "Deploy crawler to 130",
+        "dir": "${workspaceRoot}",
+        "host": "47.101.36.130", "port": 22,
+        "user": "root", "password": "$w-<[n{d!(sz(A.:3&u5vY?sm%6c0e",
+
+        "checkBeforeDeploy": true,
+
+        "mappings": [
+            {
+                "source": "${workspaceRoot}",
+                "target": "/opt/niucai/crawl"
+            }
+        ]
+      }
+    ]
+  }
 }

basketball_job.sh

@@ -0,0 +1,14 @@
+#/bin/sh
+#
+# 足彩，每 5 分钟执行一次
+
+source /etc/profile
+
+cd /opt/niucai/crawl
+
+# 赛事
+scrapy crawl basketball-match >> logs/basketball-match.log
+# 赔率
+scrapy crawl basketball-price >> logs/basketball-price.log
+# 结果
+scrapy crawl basketball-result >> logs/basketball-result.log

crawl/comm/lottery.py

@@ -8,12 +8,21 @@ class LotteryResult:
                # 期数
                issueNo,
 
+               # 开奖日期
+               openingDate,
+
                # 号码
                result
                ):
     self.lotteryType = lotteryType
     self.issueNo = issueNo
+    self.openingDate = openingDate
     self.result = result
+    # 中奖详情
+    self.details = None
+
+  def setDetails(self, details):
+    self.details = details
 
   def persist(self):
     cursor = MyDB.getCursor()
@@ -26,7 +35,35 @@ class LotteryResult:
     if res[0] > 0:
       return
 
-    sql = "insert into ta_crawl_lottery_result(lottery_type, issue_no, result) values(%s, %s, %s)"
-    cursor.execute(sql, [self.lotteryType, self.issueNo, self.result])
+    sql = "insert into ta_crawl_lottery_result(lottery_type, issue_no, opening_date, result, detail) values(%s, %s, %s, %s, %s)"
+    detailStr = ""
+    if self.details is not None:
+      detailStr = ";".join(map(lambda x: x.toString(), self.details))
+
+    cursor.execute(sql, (self.lotteryType, self.issueNo, self.openingDate, self.result, detailStr))
     MyDB.commit()
     cursor.close()
+
+class LotteryResultDetail:
+  def __init__(self,
+               # 中奖级别
+               level,
+
+               # 注数
+               piece,
+
+               # 单注奖金
+               money):
+    self.level = level
+    self.piece = piece
+    self.money = money
+    # 额外追加
+    self.xPiece = "0"
+    self.xMoney = "0"
+
+  def add(self, xPiece, xMoney):
+    self.xPiece = xPiece
+    self.xMoney = xMoney
+
+  def toString(self):
+    return "|".join((self.level, self.piece, self.money, self.xPiece, self.xMoney))

crawl/run.py

@@ -7,4 +7,4 @@ print(dirpath)
 # 添加环境变量
 sys.path.append(dirpath)
 # 启动爬虫,第三个参数为爬虫name
-execute(['scrapy','crawl','football-price'])
+execute(['scrapy','crawl','lottery'])

crawl/spiders/lottery.py

@@ -1,60 +1,123 @@
 import scrapy
 
-from crawl.comm.lottery import LotteryResult
+from crawl.comm.lottery import LotteryResult, LotteryResultDetail
 
 
 class LotterySpider(scrapy.Spider):
   name = "lottery"
 
   def start_requests(self):
-    url = "https://www.lottery.gov.cn/tz_kj.json"
-    yield scrapy.Request(url, self.parseLottery)
+    # 大乐透
+    url = "https://www.lottery.gov.cn/api/lottery_kj_detail_new.jspx?_ltype=4"
+    yield scrapy.Request(url, self.parseLottery, 'POST')
+
+    # 排列三, 排列五
+    url = "https://www.lottery.gov.cn/api/lottery_kj_detail_new.jspx?_ltype=5"
+    yield scrapy.Request(url, self.parsePermutation, 'POST')
 
-    # 福彩
-    url = "http://www.cwl.gov.cn/cwl_admin/kjxx/findDrawNotice"
     # 双色球
-    yield scrapy.Request(url + "?name=ssq&issueCount=1", self.parseDraw)
+    url = "http://www.cwl.gov.cn/cwl_admin/kjxx/findDrawNotice?name=ssq&issueCount=1"
+    yield scrapy.Request(url, self.parseDraw)
 
-  # 体彩
+  # 大乐透
   def parseLottery(self, response):
     data = response.json()
     data = data[0]
 
     # 大乐透
-    key = 'dlt'
     typ = 'lottery'
-    LotteryResult(
+    result = LotteryResult(
       typ,
-      data[key]['term'],
-      '|'.join(data[key]['numberCode'])
-    ).persist()
+      data['lottery']['term'],
+      data['lottery']['fTime'][0:10],
+      '|'.join(data['codeNumber'])
+    )
+
+    details = []
+    for i, item in enumerate(data['details']):
+      detail = LotteryResultDetail(item.get('level'), item.get('piece'), item.get('money'))
+      detail.add(data['detailsAdd'][i].get('piece'), data['detailsAdd'][i].get('money'))
+      details.append(detail)
+    result.setDetails(details)
+
+    result.persist()
+
+  # 排列三, 排列五
+  def parsePermutation(self, response):
+    data = response.json()
+    data = data[0]
+
+    issueNo = data['lottery']['term']
+    openingDate = data['lottery']['fTime'][0:10]
 
     # 排列三
-    key = 'pls'
     typ = 'p3'
-    LotteryResult(
+    result = LotteryResult(
       typ,
-      data[key]['term'],
-      '|'.join(data[key]['numberCode'])
-    ).persist()
+      issueNo,
+      openingDate,
+      '|'.join(data['codeNumber'])
+    )
 
-    # 排列五w
-    key = 'plw'
+    details = []
+    for i, item in enumerate(data['details']):
+      detail = LotteryResultDetail(item.get('level'), item.get('piece'), item.get('money'))
+      details.append(detail)
+
+    result.setDetails(details)
+    result.persist()
+
+    # 排列五
     typ = 'p5'
-    LotteryResult(
+    result = LotteryResult(
       typ,
-      data[key]['term'],
-      '|'.join(data[key]['numberCode'])
-    ).persist()
+      issueNo,
+      openingDate,
+      '|'.join(data['codeNumberPlw'])
+    )
+
+    details = []
+    for i, item in enumerate(data['detailsPlw']):
+      detail = LotteryResultDetail(item.get('level'), item.get('piece'), item.get('money'))
+      details.append(detail)
+
+    result.setDetails(details)
+    result.persist()
 
   # 福彩 - 双色球
   def parseDraw(self, response):
     data = response.json()
-    result = data['result'][0]
+    data = data['result'][0]
 
     typ = 'double-color'
-    LotteryResult(
+    result = LotteryResult(
       typ,
-      result['code'],
-      '|'.join(result['red'].split(',') + [result['blue']])
-    ).persist()
+      data['code'],
+      data['date'][0:10],
+      '|'.join(data['red'].split(',') + [data['blue']])
+    )
+
+    details = []
+    for i, item in enumerate(data['prizegrades']):
+      level = item.get('type')
+      if level == 1:
+        level = '一等奖'
+      elif level == 2:
+        level = '二等奖'
+      elif level == 3:
+        level = '三等奖'
+      elif level == 4:
+        level = '四等奖'
+      elif level == 5:
+        level = '五等奖'
+      elif level == 6:
+        level = '六等奖'
+      else:
+        continue
+
+      detail = LotteryResultDetail(level, item.get('typenum'), item.get('typemoney'))
+      details.append(detail)
+
+    result.setDetails(details)
+    result.persist()
+

football_job.sh

@@ -0,0 +1,14 @@
+#/bin/sh
+#
+# 足彩，每 5 分钟执行一次
+
+source /etc/profile
+
+cd /opt/niucai/crawl
+
+# 赛事
+scrapy crawl football-match >> logs/football-match.log
+# 赔率
+scrapy crawl football-price >> logs/football-price.log
+# 结果
+scrapy crawl football-result >> logs/football-result.log

lottery_job.sh

@@ -0,0 +1,9 @@
+#/bin/sh
+#
+# 数字彩, 每天执行一次
+
+source /etc/profile
+
+cd /opt/niucai/crawl
+
+scrapy crawl lottery >> logs/lottery.log