feats

crawl/comm/lottery.py

@@ -28,19 +28,21 @@ class LotteryResult:
     cursor = MyDB.getCursor()
     if cursor is None:
       return
+      
+    detailStr = ""
+    if self.details is not None:
+      detailStr = ";".join(map(lambda x: x.toString(), self.details))
 
     query = "select count(*) from ta_crawl_lottery_result where lottery_type = %s and issue_no = %s and status = 1"
     cursor.execute(query,[self.lotteryType, self.issueNo])
     res = cursor.fetchone()
     if res[0] > 0:
-      return
-
-    sql = "insert into ta_crawl_lottery_result(lottery_type, issue_no, opening_date, result, detail) values(%s, %s, %s, %s, %s)"
-    detailStr = ""
-    if self.details is not None:
-      detailStr = ";".join(map(lambda x: x.toString(), self.details))
+      sql = "update ta_crawl_lottery_result set opening_date = %s , result = %s , detail = %s where lottery_type = %s and issue_no = %s"
+      cursor.execute(sql, (self.openingDate, self.result, detailStr, self.lotteryType, self.issueNo))
+    else:
+      sql = "insert into ta_crawl_lottery_result(lottery_type, issue_no, opening_date, result, detail) values(%s, %s, %s, %s, %s)"
+      cursor.execute(sql, (self.lotteryType, self.issueNo, self.openingDate, self.result, detailStr))
 
-    cursor.execute(sql, (self.lotteryType, self.issueNo, self.openingDate, self.result, detailStr))
     MyDB.commit()
     cursor.close()
 

crawl/run.py

@@ -7,4 +7,4 @@ print(dirpath)
 # 添加环境变量
 sys.path.append(dirpath)
 # 启动爬虫,第三个参数为爬虫name
-execute(['scrapy','crawl','basketball'])
+execute(['scrapy','crawl','lottery'])

crawl/settings.py

@@ -90,7 +90,7 @@ EXTENSIONS = {
 DATABASE = {
   'host': 'rm-8vb8r44l60dc5ik05ao.mysql.zhangbei.rds.aliyuncs.com',
   'port': 3306,
-  'name': 'niucai',
+  'name': 'niucai2',
   'user': 'niucai',
   'password': 'sseTv!lSWgQFTZR3'
 }

logs/lottery.log

@@ -0,0 +1,406 @@
+2020-11-17 09:36:51 [scrapy.extensions.telnet] INFO: Telnet Password: bf575cc8f5dc5fb5
+2020-11-17 09:36:51 [scrapy.middleware] INFO: Enabled extensions:
+['scrapy.extensions.corestats.CoreStats',
+ 'scrapy.extensions.telnet.TelnetConsole',
+ 'scrapy.extensions.logstats.LogStats',
+ 'crawl.comm.mydb.MyDB']
+2020-11-17 09:36:52 [scrapy.middleware] INFO: Enabled downloader middlewares:
+['scrapy.downloadermiddlewares.robotstxt.RobotsTxtMiddleware',
+ 'scrapy.downloadermiddlewares.httpauth.HttpAuthMiddleware',
+ 'scrapy.downloadermiddlewares.downloadtimeout.DownloadTimeoutMiddleware',
+ 'scrapy.downloadermiddlewares.defaultheaders.DefaultHeadersMiddleware',
+ 'scrapy.downloadermiddlewares.useragent.UserAgentMiddleware',
+ 'scrapy.downloadermiddlewares.retry.RetryMiddleware',
+ 'scrapy.downloadermiddlewares.redirect.MetaRefreshMiddleware',
+ 'scrapy.downloadermiddlewares.httpcompression.HttpCompressionMiddleware',
+ 'scrapy.downloadermiddlewares.redirect.RedirectMiddleware',
+ 'scrapy.downloadermiddlewares.cookies.CookiesMiddleware',
+ 'scrapy.downloadermiddlewares.httpproxy.HttpProxyMiddleware',
+ 'scrapy.downloadermiddlewares.stats.DownloaderStats']
+2020-11-17 09:36:52 [scrapy.middleware] INFO: Enabled spider middlewares:
+['scrapy.spidermiddlewares.httperror.HttpErrorMiddleware',
+ 'scrapy.spidermiddlewares.offsite.OffsiteMiddleware',
+ 'scrapy.spidermiddlewares.referer.RefererMiddleware',
+ 'scrapy.spidermiddlewares.urllength.UrlLengthMiddleware',
+ 'scrapy.spidermiddlewares.depth.DepthMiddleware']
+2020-11-17 09:36:52 [scrapy.middleware] INFO: Enabled item pipelines:
+[]
+2020-11-17 09:36:52 [scrapy.core.engine] INFO: Spider opened
+2020-11-17 09:36:52 [scrapy.extensions.logstats] INFO: Crawled 0 pages (at 0 pages/min), scraped 0 items (at 0 items/min)
+2020-11-17 09:36:52 [scrapy.extensions.telnet] INFO: Telnet console listening on 127.0.0.1:6023
+2020-11-17 09:36:52 [scrapy.robotstxt] WARNING: Failure while parsing robots.txt. File either contains garbage or is in an encoding other than UTF-8, treating it as an empty file.
+Traceback (most recent call last):
+  File "D:\Application\python-3.8.5\Lib\site-packages\twisted\internet\defer.py", line 1418, in _inlineCallbacks
+    result = g.send(result)
+StopIteration: <404 https://api.xinti.com/robots.txt>
+
+During handling of the above exception, another exception occurred:
+
+Traceback (most recent call last):
+  File "D:\Application\python-3.8.5\Lib\site-packages\scrapy\robotstxt.py", line 16, in decode_robotstxt
+    robotstxt_body = robotstxt_body.decode('utf-8')
+UnicodeDecodeError: 'utf-8' codec can't decode byte 0xd5 in position 248: invalid continuation byte
+2020-11-17 09:37:00 [scrapy.core.engine] INFO: Closing spider (finished)
+2020-11-17 09:37:00 [scrapy.statscollectors] INFO: Dumping Scrapy stats:
+{'downloader/request_bytes': 1739,
+ 'downloader/request_count': 6,
+ 'downloader/request_method_count/GET': 5,
+ 'downloader/request_method_count/POST': 1,
+ 'downloader/response_bytes': 10910,
+ 'downloader/response_count': 6,
+ 'downloader/response_status_count/200': 4,
+ 'downloader/response_status_count/404': 2,
+ 'elapsed_time_seconds': 8.002777,
+ 'finish_reason': 'finished',
+ 'finish_time': datetime.datetime(2020, 11, 17, 1, 37, 0, 186544),
+ 'log_count/INFO': 10,
+ 'log_count/WARNING': 1,
+ 'response_received_count': 6,
+ 'robotstxt/request_count': 2,
+ 'robotstxt/response_count': 2,
+ 'robotstxt/response_status_count/404': 2,
+ 'scheduler/dequeued': 4,
+ 'scheduler/dequeued/memory': 4,
+ 'scheduler/enqueued': 4,
+ 'scheduler/enqueued/memory': 4,
+ 'start_time': datetime.datetime(2020, 11, 17, 1, 36, 52, 183767)}
+2020-11-17 09:37:00 [scrapy.core.engine] INFO: Spider closed (finished)
+2020-11-17 09:38:07 [scrapy.extensions.telnet] INFO: Telnet Password: 295aef036b8b8ad9
+2020-11-17 09:38:08 [scrapy.middleware] INFO: Enabled extensions:
+['scrapy.extensions.corestats.CoreStats',
+ 'scrapy.extensions.telnet.TelnetConsole',
+ 'scrapy.extensions.logstats.LogStats',
+ 'crawl.comm.mydb.MyDB']
+2020-11-17 09:38:08 [scrapy.middleware] INFO: Enabled downloader middlewares:
+['scrapy.downloadermiddlewares.robotstxt.RobotsTxtMiddleware',
+ 'scrapy.downloadermiddlewares.httpauth.HttpAuthMiddleware',
+ 'scrapy.downloadermiddlewares.downloadtimeout.DownloadTimeoutMiddleware',
+ 'scrapy.downloadermiddlewares.defaultheaders.DefaultHeadersMiddleware',
+ 'scrapy.downloadermiddlewares.useragent.UserAgentMiddleware',
+ 'scrapy.downloadermiddlewares.retry.RetryMiddleware',
+ 'scrapy.downloadermiddlewares.redirect.MetaRefreshMiddleware',
+ 'scrapy.downloadermiddlewares.httpcompression.HttpCompressionMiddleware',
+ 'scrapy.downloadermiddlewares.redirect.RedirectMiddleware',
+ 'scrapy.downloadermiddlewares.cookies.CookiesMiddleware',
+ 'scrapy.downloadermiddlewares.httpproxy.HttpProxyMiddleware',
+ 'scrapy.downloadermiddlewares.stats.DownloaderStats']
+2020-11-17 09:38:08 [scrapy.middleware] INFO: Enabled spider middlewares:
+['scrapy.spidermiddlewares.httperror.HttpErrorMiddleware',
+ 'scrapy.spidermiddlewares.offsite.OffsiteMiddleware',
+ 'scrapy.spidermiddlewares.referer.RefererMiddleware',
+ 'scrapy.spidermiddlewares.urllength.UrlLengthMiddleware',
+ 'scrapy.spidermiddlewares.depth.DepthMiddleware']
+2020-11-17 09:38:08 [scrapy.middleware] INFO: Enabled item pipelines:
+[]
+2020-11-17 09:38:08 [scrapy.core.engine] INFO: Spider opened
+2020-11-17 09:38:08 [scrapy.extensions.logstats] INFO: Crawled 0 pages (at 0 pages/min), scraped 0 items (at 0 items/min)
+2020-11-17 09:38:08 [scrapy.extensions.telnet] INFO: Telnet console listening on 127.0.0.1:6023
+2020-11-17 09:38:08 [scrapy.robotstxt] WARNING: Failure while parsing robots.txt. File either contains garbage or is in an encoding other than UTF-8, treating it as an empty file.
+Traceback (most recent call last):
+  File "D:\Application\python-3.8.5\Lib\site-packages\twisted\internet\defer.py", line 1418, in _inlineCallbacks
+    result = g.send(result)
+StopIteration: <404 https://api.xinti.com/robots.txt>
+
+During handling of the above exception, another exception occurred:
+
+Traceback (most recent call last):
+  File "D:\Application\python-3.8.5\Lib\site-packages\scrapy\robotstxt.py", line 16, in decode_robotstxt
+    robotstxt_body = robotstxt_body.decode('utf-8')
+UnicodeDecodeError: 'utf-8' codec can't decode byte 0xd5 in position 248: invalid continuation byte
+2020-11-17 09:40:17 [scrapy.extensions.logstats] INFO: Crawled 6 pages (at 6 pages/min), scraped 0 items (at 0 items/min)
+2020-11-17 09:40:17 [scrapy.core.engine] INFO: Closing spider (finished)
+2020-11-17 09:40:17 [scrapy.statscollectors] INFO: Dumping Scrapy stats:
+{'downloader/request_bytes': 1739,
+ 'downloader/request_count': 6,
+ 'downloader/request_method_count/GET': 5,
+ 'downloader/request_method_count/POST': 1,
+ 'downloader/response_bytes': 10848,
+ 'downloader/response_count': 6,
+ 'downloader/response_status_count/200': 4,
+ 'downloader/response_status_count/404': 2,
+ 'elapsed_time_seconds': 129.262888,
+ 'finish_reason': 'finished',
+ 'finish_time': datetime.datetime(2020, 11, 17, 1, 40, 17, 488510),
+ 'log_count/INFO': 11,
+ 'log_count/WARNING': 1,
+ 'response_received_count': 6,
+ 'robotstxt/request_count': 2,
+ 'robotstxt/response_count': 2,
+ 'robotstxt/response_status_count/404': 2,
+ 'scheduler/dequeued': 4,
+ 'scheduler/dequeued/memory': 4,
+ 'scheduler/enqueued': 4,
+ 'scheduler/enqueued/memory': 4,
+ 'start_time': datetime.datetime(2020, 11, 17, 1, 38, 8, 225622)}
+2020-11-17 09:40:17 [scrapy.core.engine] INFO: Spider closed (finished)
+2020-11-17 09:43:18 [scrapy.extensions.telnet] INFO: Telnet Password: ce022468cacb8212
+2020-11-17 09:43:19 [scrapy.middleware] INFO: Enabled extensions:
+['scrapy.extensions.corestats.CoreStats',
+ 'scrapy.extensions.telnet.TelnetConsole',
+ 'scrapy.extensions.logstats.LogStats',
+ 'crawl.comm.mydb.MyDB']
+2020-11-17 09:43:19 [scrapy.middleware] INFO: Enabled downloader middlewares:
+['scrapy.downloadermiddlewares.robotstxt.RobotsTxtMiddleware',
+ 'scrapy.downloadermiddlewares.httpauth.HttpAuthMiddleware',
+ 'scrapy.downloadermiddlewares.downloadtimeout.DownloadTimeoutMiddleware',
+ 'scrapy.downloadermiddlewares.defaultheaders.DefaultHeadersMiddleware',
+ 'scrapy.downloadermiddlewares.useragent.UserAgentMiddleware',
+ 'scrapy.downloadermiddlewares.retry.RetryMiddleware',
+ 'scrapy.downloadermiddlewares.redirect.MetaRefreshMiddleware',
+ 'scrapy.downloadermiddlewares.httpcompression.HttpCompressionMiddleware',
+ 'scrapy.downloadermiddlewares.redirect.RedirectMiddleware',
+ 'scrapy.downloadermiddlewares.cookies.CookiesMiddleware',
+ 'scrapy.downloadermiddlewares.httpproxy.HttpProxyMiddleware',
+ 'scrapy.downloadermiddlewares.stats.DownloaderStats']
+2020-11-17 09:43:19 [scrapy.middleware] INFO: Enabled spider middlewares:
+['scrapy.spidermiddlewares.httperror.HttpErrorMiddleware',
+ 'scrapy.spidermiddlewares.offsite.OffsiteMiddleware',
+ 'scrapy.spidermiddlewares.referer.RefererMiddleware',
+ 'scrapy.spidermiddlewares.urllength.UrlLengthMiddleware',
+ 'scrapy.spidermiddlewares.depth.DepthMiddleware']
+2020-11-17 09:43:19 [scrapy.middleware] INFO: Enabled item pipelines:
+[]
+2020-11-17 09:43:19 [scrapy.core.engine] INFO: Spider opened
+2020-11-17 09:43:19 [scrapy.extensions.logstats] INFO: Crawled 0 pages (at 0 pages/min), scraped 0 items (at 0 items/min)
+2020-11-17 09:43:19 [scrapy.extensions.telnet] INFO: Telnet console listening on 127.0.0.1:6023
+2020-11-17 09:43:19 [scrapy.robotstxt] WARNING: Failure while parsing robots.txt. File either contains garbage or is in an encoding other than UTF-8, treating it as an empty file.
+Traceback (most recent call last):
+  File "D:\Application\python-3.8.5\Lib\site-packages\twisted\internet\defer.py", line 1418, in _inlineCallbacks
+    result = g.send(result)
+StopIteration: <404 https://api.xinti.com/robots.txt>
+
+During handling of the above exception, another exception occurred:
+
+Traceback (most recent call last):
+  File "D:\Application\python-3.8.5\Lib\site-packages\scrapy\robotstxt.py", line 16, in decode_robotstxt
+    robotstxt_body = robotstxt_body.decode('utf-8')
+UnicodeDecodeError: 'utf-8' codec can't decode byte 0xd5 in position 248: invalid continuation byte
+2020-11-17 09:44:27 [scrapy.extensions.logstats] INFO: Crawled 6 pages (at 6 pages/min), scraped 0 items (at 0 items/min)
+2020-11-17 09:44:28 [scrapy.core.engine] INFO: Closing spider (finished)
+2020-11-17 09:44:28 [scrapy.statscollectors] INFO: Dumping Scrapy stats:
+{'downloader/request_bytes': 1739,
+ 'downloader/request_count': 6,
+ 'downloader/request_method_count/GET': 5,
+ 'downloader/request_method_count/POST': 1,
+ 'downloader/response_bytes': 10881,
+ 'downloader/response_count': 6,
+ 'downloader/response_status_count/200': 4,
+ 'downloader/response_status_count/404': 2,
+ 'elapsed_time_seconds': 68.571785,
+ 'finish_reason': 'finished',
+ 'finish_time': datetime.datetime(2020, 11, 17, 1, 44, 28, 6481),
+ 'log_count/INFO': 11,
+ 'log_count/WARNING': 1,
+ 'response_received_count': 6,
+ 'robotstxt/request_count': 2,
+ 'robotstxt/response_count': 2,
+ 'robotstxt/response_status_count/404': 2,
+ 'scheduler/dequeued': 4,
+ 'scheduler/dequeued/memory': 4,
+ 'scheduler/enqueued': 4,
+ 'scheduler/enqueued/memory': 4,
+ 'start_time': datetime.datetime(2020, 11, 17, 1, 43, 19, 434696)}
+2020-11-17 09:44:28 [scrapy.core.engine] INFO: Spider closed (finished)
+2020-11-17 09:53:03 [scrapy.extensions.telnet] INFO: Telnet Password: 3dde26547467246c
+2020-11-17 09:53:03 [scrapy.middleware] INFO: Enabled extensions:
+['scrapy.extensions.corestats.CoreStats',
+ 'scrapy.extensions.telnet.TelnetConsole',
+ 'scrapy.extensions.logstats.LogStats',
+ 'crawl.comm.mydb.MyDB']
+2020-11-17 09:53:03 [scrapy.middleware] INFO: Enabled downloader middlewares:
+['scrapy.downloadermiddlewares.robotstxt.RobotsTxtMiddleware',
+ 'scrapy.downloadermiddlewares.httpauth.HttpAuthMiddleware',
+ 'scrapy.downloadermiddlewares.downloadtimeout.DownloadTimeoutMiddleware',
+ 'scrapy.downloadermiddlewares.defaultheaders.DefaultHeadersMiddleware',
+ 'scrapy.downloadermiddlewares.useragent.UserAgentMiddleware',
+ 'scrapy.downloadermiddlewares.retry.RetryMiddleware',
+ 'scrapy.downloadermiddlewares.redirect.MetaRefreshMiddleware',
+ 'scrapy.downloadermiddlewares.httpcompression.HttpCompressionMiddleware',
+ 'scrapy.downloadermiddlewares.redirect.RedirectMiddleware',
+ 'scrapy.downloadermiddlewares.cookies.CookiesMiddleware',
+ 'scrapy.downloadermiddlewares.httpproxy.HttpProxyMiddleware',
+ 'scrapy.downloadermiddlewares.stats.DownloaderStats']
+2020-11-17 09:53:03 [scrapy.middleware] INFO: Enabled spider middlewares:
+['scrapy.spidermiddlewares.httperror.HttpErrorMiddleware',
+ 'scrapy.spidermiddlewares.offsite.OffsiteMiddleware',
+ 'scrapy.spidermiddlewares.referer.RefererMiddleware',
+ 'scrapy.spidermiddlewares.urllength.UrlLengthMiddleware',
+ 'scrapy.spidermiddlewares.depth.DepthMiddleware']
+2020-11-17 09:53:03 [scrapy.middleware] INFO: Enabled item pipelines:
+[]
+2020-11-17 09:53:03 [scrapy.core.engine] INFO: Spider opened
+2020-11-17 09:53:03 [scrapy.extensions.logstats] INFO: Crawled 0 pages (at 0 pages/min), scraped 0 items (at 0 items/min)
+2020-11-17 09:53:03 [scrapy.extensions.telnet] INFO: Telnet console listening on 127.0.0.1:6023
+2020-11-17 09:53:04 [scrapy.robotstxt] WARNING: Failure while parsing robots.txt. File either contains garbage or is in an encoding other than UTF-8, treating it as an empty file.
+Traceback (most recent call last):
+  File "D:\Application\python-3.8.5\Lib\site-packages\twisted\internet\defer.py", line 1418, in _inlineCallbacks
+    result = g.send(result)
+StopIteration: <404 https://api.xinti.com/robots.txt>
+
+During handling of the above exception, another exception occurred:
+
+Traceback (most recent call last):
+  File "D:\Application\python-3.8.5\Lib\site-packages\scrapy\robotstxt.py", line 16, in decode_robotstxt
+    robotstxt_body = robotstxt_body.decode('utf-8')
+UnicodeDecodeError: 'utf-8' codec can't decode byte 0xd5 in position 248: invalid continuation byte
+2020-11-17 09:53:28 [scrapy.core.engine] INFO: Closing spider (finished)
+2020-11-17 09:53:28 [scrapy.statscollectors] INFO: Dumping Scrapy stats:
+{'downloader/request_bytes': 1739,
+ 'downloader/request_count': 6,
+ 'downloader/request_method_count/GET': 5,
+ 'downloader/request_method_count/POST': 1,
+ 'downloader/response_bytes': 10851,
+ 'downloader/response_count': 6,
+ 'downloader/response_status_count/200': 4,
+ 'downloader/response_status_count/404': 2,
+ 'elapsed_time_seconds': 24.955046,
+ 'finish_reason': 'finished',
+ 'finish_time': datetime.datetime(2020, 11, 17, 1, 53, 28, 832941),
+ 'log_count/INFO': 10,
+ 'log_count/WARNING': 1,
+ 'response_received_count': 6,
+ 'robotstxt/request_count': 2,
+ 'robotstxt/response_count': 2,
+ 'robotstxt/response_status_count/404': 2,
+ 'scheduler/dequeued': 4,
+ 'scheduler/dequeued/memory': 4,
+ 'scheduler/enqueued': 4,
+ 'scheduler/enqueued/memory': 4,
+ 'start_time': datetime.datetime(2020, 11, 17, 1, 53, 3, 877895)}
+2020-11-17 09:53:28 [scrapy.core.engine] INFO: Spider closed (finished)
+2020-11-17 09:53:51 [scrapy.extensions.telnet] INFO: Telnet Password: 366dfc97a237cf7c
+2020-11-17 09:53:51 [scrapy.middleware] INFO: Enabled extensions:
+['scrapy.extensions.corestats.CoreStats',
+ 'scrapy.extensions.telnet.TelnetConsole',
+ 'scrapy.extensions.logstats.LogStats',
+ 'crawl.comm.mydb.MyDB']
+2020-11-17 09:53:51 [scrapy.middleware] INFO: Enabled downloader middlewares:
+['scrapy.downloadermiddlewares.robotstxt.RobotsTxtMiddleware',
+ 'scrapy.downloadermiddlewares.httpauth.HttpAuthMiddleware',
+ 'scrapy.downloadermiddlewares.downloadtimeout.DownloadTimeoutMiddleware',
+ 'scrapy.downloadermiddlewares.defaultheaders.DefaultHeadersMiddleware',
+ 'scrapy.downloadermiddlewares.useragent.UserAgentMiddleware',
+ 'scrapy.downloadermiddlewares.retry.RetryMiddleware',
+ 'scrapy.downloadermiddlewares.redirect.MetaRefreshMiddleware',
+ 'scrapy.downloadermiddlewares.httpcompression.HttpCompressionMiddleware',
+ 'scrapy.downloadermiddlewares.redirect.RedirectMiddleware',
+ 'scrapy.downloadermiddlewares.cookies.CookiesMiddleware',
+ 'scrapy.downloadermiddlewares.httpproxy.HttpProxyMiddleware',
+ 'scrapy.downloadermiddlewares.stats.DownloaderStats']
+2020-11-17 09:53:51 [scrapy.middleware] INFO: Enabled spider middlewares:
+['scrapy.spidermiddlewares.httperror.HttpErrorMiddleware',
+ 'scrapy.spidermiddlewares.offsite.OffsiteMiddleware',
+ 'scrapy.spidermiddlewares.referer.RefererMiddleware',
+ 'scrapy.spidermiddlewares.urllength.UrlLengthMiddleware',
+ 'scrapy.spidermiddlewares.depth.DepthMiddleware']
+2020-11-17 09:53:51 [scrapy.middleware] INFO: Enabled item pipelines:
+[]
+2020-11-17 09:53:51 [scrapy.core.engine] INFO: Spider opened
+2020-11-17 09:53:52 [scrapy.extensions.logstats] INFO: Crawled 0 pages (at 0 pages/min), scraped 0 items (at 0 items/min)
+2020-11-17 09:53:52 [scrapy.extensions.telnet] INFO: Telnet console listening on 127.0.0.1:6023
+2020-11-17 09:53:52 [scrapy.robotstxt] WARNING: Failure while parsing robots.txt. File either contains garbage or is in an encoding other than UTF-8, treating it as an empty file.
+Traceback (most recent call last):
+  File "D:\Application\python-3.8.5\Lib\site-packages\twisted\internet\defer.py", line 1418, in _inlineCallbacks
+    result = g.send(result)
+StopIteration: <404 https://api.xinti.com/robots.txt>
+
+During handling of the above exception, another exception occurred:
+
+Traceback (most recent call last):
+  File "D:\Application\python-3.8.5\Lib\site-packages\scrapy\robotstxt.py", line 16, in decode_robotstxt
+    robotstxt_body = robotstxt_body.decode('utf-8')
+UnicodeDecodeError: 'utf-8' codec can't decode byte 0xd5 in position 248: invalid continuation byte
+2020-11-17 09:54:53 [scrapy.extensions.logstats] INFO: Crawled 6 pages (at 6 pages/min), scraped 0 items (at 0 items/min)
+2020-11-17 09:54:56 [scrapy.core.engine] INFO: Closing spider (finished)
+2020-11-17 09:54:56 [scrapy.statscollectors] INFO: Dumping Scrapy stats:
+{'downloader/request_bytes': 1739,
+ 'downloader/request_count': 6,
+ 'downloader/request_method_count/GET': 5,
+ 'downloader/request_method_count/POST': 1,
+ 'downloader/response_bytes': 10843,
+ 'downloader/response_count': 6,
+ 'downloader/response_status_count/200': 4,
+ 'downloader/response_status_count/404': 2,
+ 'elapsed_time_seconds': 64.547145,
+ 'finish_reason': 'finished',
+ 'finish_time': datetime.datetime(2020, 11, 17, 1, 54, 56, 548924),
+ 'log_count/INFO': 11,
+ 'log_count/WARNING': 1,
+ 'response_received_count': 6,
+ 'robotstxt/request_count': 2,
+ 'robotstxt/response_count': 2,
+ 'robotstxt/response_status_count/404': 2,
+ 'scheduler/dequeued': 4,
+ 'scheduler/dequeued/memory': 4,
+ 'scheduler/enqueued': 4,
+ 'scheduler/enqueued/memory': 4,
+ 'start_time': datetime.datetime(2020, 11, 17, 1, 53, 52, 1779)}
+2020-11-17 09:54:56 [scrapy.core.engine] INFO: Spider closed (finished)
+2020-11-17 10:50:15 [scrapy.extensions.telnet] INFO: Telnet Password: 2b6b900fab93768f
+2020-11-17 10:50:15 [scrapy.middleware] INFO: Enabled extensions:
+['scrapy.extensions.corestats.CoreStats',
+ 'scrapy.extensions.telnet.TelnetConsole',
+ 'scrapy.extensions.logstats.LogStats',
+ 'crawl.comm.mydb.MyDB']
+2020-11-17 10:50:15 [scrapy.middleware] INFO: Enabled downloader middlewares:
+['scrapy.downloadermiddlewares.robotstxt.RobotsTxtMiddleware',
+ 'scrapy.downloadermiddlewares.httpauth.HttpAuthMiddleware',
+ 'scrapy.downloadermiddlewares.downloadtimeout.DownloadTimeoutMiddleware',
+ 'scrapy.downloadermiddlewares.defaultheaders.DefaultHeadersMiddleware',
+ 'scrapy.downloadermiddlewares.useragent.UserAgentMiddleware',
+ 'scrapy.downloadermiddlewares.retry.RetryMiddleware',
+ 'scrapy.downloadermiddlewares.redirect.MetaRefreshMiddleware',
+ 'scrapy.downloadermiddlewares.httpcompression.HttpCompressionMiddleware',
+ 'scrapy.downloadermiddlewares.redirect.RedirectMiddleware',
+ 'scrapy.downloadermiddlewares.cookies.CookiesMiddleware',
+ 'scrapy.downloadermiddlewares.httpproxy.HttpProxyMiddleware',
+ 'scrapy.downloadermiddlewares.stats.DownloaderStats']
+2020-11-17 10:50:15 [scrapy.middleware] INFO: Enabled spider middlewares:
+['scrapy.spidermiddlewares.httperror.HttpErrorMiddleware',
+ 'scrapy.spidermiddlewares.offsite.OffsiteMiddleware',
+ 'scrapy.spidermiddlewares.referer.RefererMiddleware',
+ 'scrapy.spidermiddlewares.urllength.UrlLengthMiddleware',
+ 'scrapy.spidermiddlewares.depth.DepthMiddleware']
+2020-11-17 10:50:15 [scrapy.middleware] INFO: Enabled item pipelines:
+[]
+2020-11-17 10:50:15 [scrapy.core.engine] INFO: Spider opened
+2020-11-17 10:50:15 [scrapy.extensions.logstats] INFO: Crawled 0 pages (at 0 pages/min), scraped 0 items (at 0 items/min)
+2020-11-17 10:50:15 [scrapy.extensions.telnet] INFO: Telnet console listening on 127.0.0.1:6023
+2020-11-17 10:50:16 [scrapy.robotstxt] WARNING: Failure while parsing robots.txt. File either contains garbage or is in an encoding other than UTF-8, treating it as an empty file.
+Traceback (most recent call last):
+  File "D:\Application\python-3.8.5\Lib\site-packages\twisted\internet\defer.py", line 1418, in _inlineCallbacks
+    result = g.send(result)
+StopIteration: <404 https://api.xinti.com/robots.txt>
+
+During handling of the above exception, another exception occurred:
+
+Traceback (most recent call last):
+  File "D:\Application\python-3.8.5\Lib\site-packages\scrapy\robotstxt.py", line 16, in decode_robotstxt
+    robotstxt_body = robotstxt_body.decode('utf-8')
+UnicodeDecodeError: 'utf-8' codec can't decode byte 0xd5 in position 248: invalid continuation byte
+2020-11-17 10:51:39 [scrapy.extensions.logstats] INFO: Crawled 6 pages (at 6 pages/min), scraped 0 items (at 0 items/min)
+2020-11-17 10:51:39 [scrapy.core.engine] INFO: Closing spider (finished)
+2020-11-17 10:51:39 [scrapy.statscollectors] INFO: Dumping Scrapy stats:
+{'downloader/request_bytes': 1739,
+ 'downloader/request_count': 6,
+ 'downloader/request_method_count/GET': 5,
+ 'downloader/request_method_count/POST': 1,
+ 'downloader/response_bytes': 10860,
+ 'downloader/response_count': 6,
+ 'downloader/response_status_count/200': 4,
+ 'downloader/response_status_count/404': 2,
+ 'elapsed_time_seconds': 84.03754,
+ 'finish_reason': 'finished',
+ 'finish_time': datetime.datetime(2020, 11, 17, 2, 51, 39, 782671),
+ 'log_count/INFO': 11,
+ 'log_count/WARNING': 1,
+ 'response_received_count': 6,
+ 'robotstxt/request_count': 2,
+ 'robotstxt/response_count': 2,
+ 'robotstxt/response_status_count/404': 2,
+ 'scheduler/dequeued': 4,
+ 'scheduler/dequeued/memory': 4,
+ 'scheduler/enqueued': 4,
+ 'scheduler/enqueued/memory': 4,
+ 'start_time': datetime.datetime(2020, 11, 17, 2, 50, 15, 745131)}
+2020-11-17 10:51:39 [scrapy.core.engine] INFO: Spider closed (finished)

logs/main.log

@@ -0,0 +1,60 @@
+2020-11-17 09:36:51 [scrapy.utils.log] INFO: Scrapy 2.3.0 started (bot: crawl)
+2020-11-17 09:36:51 [scrapy.utils.log] INFO: Versions: lxml 4.5.2.0, libxml2 2.9.5, cssselect 1.1.0, parsel 1.6.0, w3lib 1.22.0, Twisted 20.3.0, Python 3.8.5 (tags/v3.8.5:580fbb0, Jul 20 2020, 15:57:54) [MSC v.1924 64 bit (AMD64)], pyOpenSSL 19.1.0 (OpenSSL 1.1.1g  21 Apr 2020), cryptography 3.0, Platform Windows-10-10.0.18362-SP0
+2020-11-17 09:36:51 [scrapy.utils.log] DEBUG: Using reactor: twisted.internet.selectreactor.SelectReactor
+2020-11-17 09:36:51 [scrapy.crawler] INFO: Overridden settings:
+{'BOT_NAME': 'crawl',
+ 'LOG_FILE': 'logs/lottery.log',
+ 'LOG_LEVEL': 20,
+ 'NEWSPIDER_MODULE': 'crawl.spiders',
+ 'ROBOTSTXT_OBEY': True,
+ 'SPIDER_MODULES': ['crawl.spiders']}
+2020-11-17 09:38:07 [scrapy.utils.log] INFO: Scrapy 2.3.0 started (bot: crawl)
+2020-11-17 09:38:07 [scrapy.utils.log] INFO: Versions: lxml 4.5.2.0, libxml2 2.9.5, cssselect 1.1.0, parsel 1.6.0, w3lib 1.22.0, Twisted 20.3.0, Python 3.8.5 (tags/v3.8.5:580fbb0, Jul 20 2020, 15:57:54) [MSC v.1924 64 bit (AMD64)], pyOpenSSL 19.1.0 (OpenSSL 1.1.1g  21 Apr 2020), cryptography 3.0, Platform Windows-10-10.0.18362-SP0
+2020-11-17 09:38:07 [scrapy.utils.log] DEBUG: Using reactor: twisted.internet.selectreactor.SelectReactor
+2020-11-17 09:38:07 [scrapy.crawler] INFO: Overridden settings:
+{'BOT_NAME': 'crawl',
+ 'LOG_FILE': 'logs/lottery.log',
+ 'LOG_LEVEL': 20,
+ 'NEWSPIDER_MODULE': 'crawl.spiders',
+ 'ROBOTSTXT_OBEY': True,
+ 'SPIDER_MODULES': ['crawl.spiders']}
+2020-11-17 09:43:18 [scrapy.utils.log] INFO: Scrapy 2.3.0 started (bot: crawl)
+2020-11-17 09:43:18 [scrapy.utils.log] INFO: Versions: lxml 4.5.2.0, libxml2 2.9.5, cssselect 1.1.0, parsel 1.6.0, w3lib 1.22.0, Twisted 20.3.0, Python 3.8.5 (tags/v3.8.5:580fbb0, Jul 20 2020, 15:57:54) [MSC v.1924 64 bit (AMD64)], pyOpenSSL 19.1.0 (OpenSSL 1.1.1g  21 Apr 2020), cryptography 3.0, Platform Windows-10-10.0.18362-SP0
+2020-11-17 09:43:18 [scrapy.utils.log] DEBUG: Using reactor: twisted.internet.selectreactor.SelectReactor
+2020-11-17 09:43:18 [scrapy.crawler] INFO: Overridden settings:
+{'BOT_NAME': 'crawl',
+ 'LOG_FILE': 'logs/lottery.log',
+ 'LOG_LEVEL': 20,
+ 'NEWSPIDER_MODULE': 'crawl.spiders',
+ 'ROBOTSTXT_OBEY': True,
+ 'SPIDER_MODULES': ['crawl.spiders']}
+2020-11-17 09:53:03 [scrapy.utils.log] INFO: Scrapy 2.3.0 started (bot: crawl)
+2020-11-17 09:53:03 [scrapy.utils.log] INFO: Versions: lxml 4.5.2.0, libxml2 2.9.5, cssselect 1.1.0, parsel 1.6.0, w3lib 1.22.0, Twisted 20.3.0, Python 3.8.5 (tags/v3.8.5:580fbb0, Jul 20 2020, 15:57:54) [MSC v.1924 64 bit (AMD64)], pyOpenSSL 19.1.0 (OpenSSL 1.1.1g  21 Apr 2020), cryptography 3.0, Platform Windows-10-10.0.18362-SP0
+2020-11-17 09:53:03 [scrapy.utils.log] DEBUG: Using reactor: twisted.internet.selectreactor.SelectReactor
+2020-11-17 09:53:03 [scrapy.crawler] INFO: Overridden settings:
+{'BOT_NAME': 'crawl',
+ 'LOG_FILE': 'logs/lottery.log',
+ 'LOG_LEVEL': 20,
+ 'NEWSPIDER_MODULE': 'crawl.spiders',
+ 'ROBOTSTXT_OBEY': True,
+ 'SPIDER_MODULES': ['crawl.spiders']}
+2020-11-17 09:53:51 [scrapy.utils.log] INFO: Scrapy 2.3.0 started (bot: crawl)
+2020-11-17 09:53:51 [scrapy.utils.log] INFO: Versions: lxml 4.5.2.0, libxml2 2.9.5, cssselect 1.1.0, parsel 1.6.0, w3lib 1.22.0, Twisted 20.3.0, Python 3.8.5 (tags/v3.8.5:580fbb0, Jul 20 2020, 15:57:54) [MSC v.1924 64 bit (AMD64)], pyOpenSSL 19.1.0 (OpenSSL 1.1.1g  21 Apr 2020), cryptography 3.0, Platform Windows-10-10.0.18362-SP0
+2020-11-17 09:53:51 [scrapy.utils.log] DEBUG: Using reactor: twisted.internet.selectreactor.SelectReactor
+2020-11-17 09:53:51 [scrapy.crawler] INFO: Overridden settings:
+{'BOT_NAME': 'crawl',
+ 'LOG_FILE': 'logs/lottery.log',
+ 'LOG_LEVEL': 20,
+ 'NEWSPIDER_MODULE': 'crawl.spiders',
+ 'ROBOTSTXT_OBEY': True,
+ 'SPIDER_MODULES': ['crawl.spiders']}
+2020-11-17 10:50:15 [scrapy.utils.log] INFO: Scrapy 2.3.0 started (bot: crawl)
+2020-11-17 10:50:15 [scrapy.utils.log] INFO: Versions: lxml 4.5.2.0, libxml2 2.9.5, cssselect 1.1.0, parsel 1.6.0, w3lib 1.22.0, Twisted 20.3.0, Python 3.8.5 (tags/v3.8.5:580fbb0, Jul 20 2020, 15:57:54) [MSC v.1924 64 bit (AMD64)], pyOpenSSL 19.1.0 (OpenSSL 1.1.1g  21 Apr 2020), cryptography 3.0, Platform Windows-10-10.0.18362-SP0
+2020-11-17 10:50:15 [scrapy.utils.log] DEBUG: Using reactor: twisted.internet.selectreactor.SelectReactor
+2020-11-17 10:50:15 [scrapy.crawler] INFO: Overridden settings:
+{'BOT_NAME': 'crawl',
+ 'LOG_FILE': 'logs/lottery.log',
+ 'LOG_LEVEL': 20,
+ 'NEWSPIDER_MODULE': 'crawl.spiders',
+ 'ROBOTSTXT_OBEY': True,
+ 'SPIDER_MODULES': ['crawl.spiders']}