From 2af89a2811b9d5b46e9808cc75d6ec2299d908dc Mon Sep 17 00:00:00 2001
From: chrfranke <authors@smartmontools.org>
Date: Tue, 12 Nov 2024 16:45:34 +0000
Subject: [PATCH] smartd.cpp: Add support for NVMe self-tests.

Support '-l selftest', '-l selfteststs' and '-s ...' for NVMe
(#1649, #1813, GH issues/218, GH pull/219).
Minor rework of ATA/SCSI self-test code to re-use its count/hours
heuristics which detects new self-test errors.

smartd.conf.5.in: Document this new feature.

diff --git smartmontools/smartd.conf.5.in smartmontools/smartd.conf.5.in
@@ -768,11 +768,12 @@ checks the maximum of both values.
 .\" %ENDIF OS Darwin FreeBSD Linux NetBSD Windows Cygwin
 .Sp
 .I selftest
-\- report if the number of failed tests reported in the SMART
-Self-Test Log has increased since the last check, or if the timestamp
-associated with the most recent failed test has increased.  Note that
-such errors will \fBonly\fP be logged if you run self-tests on the
-disk (and it fails a test!).  Self-Tests can be run automatically by
+\- [ATA] [SCSI] [NVMe: NEW EXPERIMENTAL SMARTD 7.5 FEATURE]
+report if the number of failed tests reported in the SMART Self-Test Log
+has increased since the last check, or if the timestamp associated with
+the most recent failed test has increased.
+Note that such errors will \fBonly\fP be logged if you run self-tests on
+the disk (and it fails a test!).  Self-Tests can be run automatically by
 \fBsmartd\fP: please see the \*(Aq\-s\*(Aq Directive below.
 Self-Tests can also be run manually by using the \*(Aq\-t short\*(Aq
 and \fB\*(Aq\-t\ long\*(Aq\fP options of \fBsmartctl\fP and the results of
@@ -781,10 +782,18 @@ command-line option.
 [Please see the \fBsmartctl \-l\fP and \fB\-t\fP command-line
 options.]
 .Sp
-[ATA only] Failed self-tests outdated by a newer successful extended
-self-test are ignored.  The warning email counter is reset if the
-number of failed self tests dropped to 0.  This typically happens when
-an extended self-test is run after all bad sectors have been reallocated.
+[ATA] [NVMe: NEW EXPERIMENTAL SMARTD 7.5 FEATURE]
+Failed self-tests outdated by a newer successful extended self-test
+are ignored.
+The warning email counter is reset if the number of failed self tests
+dropped to 0.
+This typically happens when an extended self-test is run after all bad
+blocks have been reallocated.
+.Sp
+[NVMe: NEW EXPERIMENTAL SMARTD 7.5 FEATURE]
+If the device name addresses an individual namespace, log entries with
+different namespace ids are ignored.
+Entries with unspecified or broadcast namespace id are always checked.
 .Sp
 .I offlinests[,ns]
 \- [ATA only] report if the Offline Data Collection status has changed
@@ -803,22 +812,23 @@ Data Collection is in progress.  See \*(Aq\-l selfteststs,ns\*(Aq below.
 .\" %ENDIF OS Cygwin Windows
 .Sp
 .I selfteststs[,ns]
-\- [ATA only] report if the Self-Test execution status has changed
-since the last check.  The report will be logged as LOG_CRIT if the new
-status indicates an error.
+\- [ATA] [NVMe: NEW EXPERIMENTAL SMARTD 7.5 FEATURE]
+report if the Self-Test execution status has changed since the last check.
+The report will be logged as LOG_CRIT if the new status indicates an error.
 .\" %IF NOT OS Cygwin Windows
 .\"! Appending ',ns' (no standby) to this directive is not implemented
 .\"! on OS_MAN_FILTER.
 .\" %ENDIF NOT OS Cygwin Windows
 .\" %IF OS Cygwin Windows
 .Sp
-[Windows and Cygwin only] If \*(Aq,ns\*(Aq (no standby) is appended to this
-directive, smartd disables system auto standby as long as a Self-Test
-is in progress.  This prevents that a Self-Test is aborted because the
-OS sets the system to a standby/sleep mode when idle.  Smartd check
-interval (\*(Aq\-i\*(Aq option) should be shorter than the configured idle
-timeout.  Auto standby is not disabled if the system is running on
-battery.
+[ATA only] [Windows and Cygwin only]
+If \*(Aq,ns\*(Aq (no standby) is appended to this directive, smartd disables
+system auto standby as long as a Self-Test is in progress.
+This prevents that a Self-Test is aborted because the OS sets the system to
+a standby/sleep mode when idle.
+Smartd check interval (\*(Aq\-i\*(Aq option) should be shorter than the
+configured idle timeout.
+Auto standby is not disabled if the system is running on battery.
 .\" %ENDIF OS Cygwin Windows
 .Sp
 .I scterc,READTIME,WRITETIME
@@ -871,6 +881,13 @@ only), and \*(AqO\*(Aq for an \fBO\fPffline Immediate Test (ATA only).  As
 soon as a match is found, the test will be started and no additional
 matches will be sought for that device and that polling cycle.
 .Sp
+[NVMe: NEW EXPERIMENTAL SMARTD 7.5 FEATURE]
+For NVMe devices with Self-Test support, use \*(AqL\*(Aq to start an
+"Extended" NVMe Self-Test and \*(AqS\*(Aq for a \fBS\fPhort NVMe Self-Test.
+If the device name addresses an individial namespace, the test will be started
+using this namespace id, otherwise using the broadcast namespace id.
+.Sp
+[ATA only]
 To run scheduled Selective Self-Tests, use \*(Aqn\*(Aq for \fBn\fPext span,
 \*(Aqr\*(Aq to \fBr\fPedo last span, or \*(Aqc\*(Aq to \fBc\fPontinue with
 next span or redo last span based on status of last test.
diff --git smartmontools/smartd.cpp smartmontools/smartd.cpp
index 10749e6f..eaaee48c 100644
--- smartmontools/smartd.cpp
+++ smartmontools/smartd.cpp
@@ -72,6 +72,7 @@ typedef int pid_t;
 #include "scsicmds.h"
 #include "nvmecmds.h"
 #include "utility.h"
+#include "sg_unaligned.h"
 
 #ifdef HAVE_POSIX_API
 #include "popen_as_ugid.h"
@@ -473,7 +474,8 @@ struct persistent_dev_state
   unsigned char tempmin{}, tempmax{};     // Min/Max Temperatures
 
   unsigned char selflogcount{};           // total number of self-test errors
-  unsigned short selfloghour{};           // lifetime hours of last self-test error
+  uint64_t selfloghour{};                 // lifetime hours of last self-test error
+                                          // (NVMe self-test log uses a 64 bit value)
 
   time_t scheduled_test_next_check{};     // Time of next check for scheduled self-tests
 
@@ -555,7 +557,13 @@ struct temp_dev_state
   ata_smart_values smartval{};            // SMART data
   ata_smart_thresholds_pvt smartthres{};  // SMART thresholds
   bool offline_started{};                 // true if offline data collection was started
+
+  // ATA and NVMe
   bool selftest_started{};                // true if self-test was started
+
+  // NVMe only
+  uint8_t selftest_op{};                  // last self-test operation
+  uint8_t selftest_compl{};               // last self-test completion
 };
 
 /// Runtime state data for a device.
@@ -669,7 +677,7 @@ static bool parse_dev_state_line(const char * line, persistent_dev_state & state
   else if (match[++m].rm_so >= 0)
     state.selflogcount = (unsigned char)val;
   else if (match[++m].rm_so >= 0)
-    state.selfloghour = (unsigned short)val;
+    state.selfloghour = val;
   else if (match[++m].rm_so >= 0)
     state.scheduled_test_next_check = (time_t)val;
   else if (match[++m].rm_so >= 0)
@@ -1768,13 +1776,15 @@ static int read_ata_error_count(ata_device * device, const char * name,
   }
 }
 
-// returns <0 if problem.  Otherwise, bottom 8 bits are the self test
-// error count, and top bits are the power-on hours of the last error.
-static int SelfTestErrorCount(ata_device * device, const char * name,
-                              firmwarebug_defs firmwarebugs)
+// Count error entries in ATA self-test log, set HOUR to power on hours of most
+// recent error.  Return error count or -1 on failure.
+static int check_ata_self_test_log(ata_device * device, const char * name,
+                                   firmwarebug_defs firmwarebugs,
+                                   unsigned & hour)
 {
   struct ata_smart_selftestlog log;
 
+  hour = 0;
   if (ataReadSelfTestLog(device, &log, firmwarebugs)){
     PrintOut(LOG_INFO,"Device: %s, Read SMART Self Test Log Failed\n",name);
     return -1;
@@ -1785,7 +1795,7 @@ static int SelfTestErrorCount(ata_device * device, const char * name,
     return 0;
 
   // Count failed self-tests
-  int errcnt = 0, hours = 0;
+  int errcnt = 0;
   for (int i = 20; i >= 0; i--) {
     int j = (i + log.mostrecenttest) % 21;
     const ata_smart_selftestlog_struct & entry = log.selftest_struct[j];
@@ -1801,17 +1811,14 @@ static int SelfTestErrorCount(ata_device * device, const char * name,
       // Self-test showed an error
       errcnt++;
       // Keep track of time of most recent error
-      if (!hours)
-        hours = entry.timestamp;
+      if (!hour)
+        hour = entry.timestamp;
     }
   }
 
-  return ((hours << 8) | errcnt);
+  return errcnt;
 }
 
-#define SELFTEST_ERRORCOUNT(x) (x & 0xff)
-#define SELFTEST_ERRORHOURS(x) ((x >> 8) & 0xffff)
-
 // Check offline data collection status
 static inline bool is_offl_coll_in_progress(unsigned char status)
 {
@@ -2224,20 +2231,20 @@ static int ATADeviceScan(dev_config & cfg, dev_state & state, ata_device * atade
   // capability check: self-test-log
   state.selflogcount = 0; state.selfloghour = 0;
   if (cfg.selftest) {
-    int retval;
+    int errcnt = 0; unsigned hour = 0;
     if (!(   cfg.permissive
           || ( smart_logdir_ok && smart_logdir.entry[0x06-1].numsectors)
           || (!smart_logdir_ok && smart_val_ok && isSmartTestLogCapable(&state.smartval, &drive)))) {
       PrintOut(LOG_INFO, "Device: %s, no SMART Self-test Log, ignoring -l selftest (override with -T permissive)\n", name);
       cfg.selftest = false;
     }
-    else if ((retval = SelfTestErrorCount(atadev, name, cfg.firmwarebugs)) < 0) {
+    else if ((errcnt = check_ata_self_test_log(atadev, name, cfg.firmwarebugs, hour)) < 0) {
       PrintOut(LOG_INFO, "Device: %s, no SMART Self-test Log, ignoring -l selftest\n", name);
       cfg.selftest = false;
     }
     else {
-      state.selflogcount=SELFTEST_ERRORCOUNT(retval);
-      state.selfloghour =SELFTEST_ERRORHOURS(retval);
+      state.selflogcount = (unsigned char)errcnt;
+      state.selfloghour  = hour;
     }
   }
   
@@ -2615,8 +2622,8 @@ static int SCSIDeviceScan(dev_config & cfg, dev_state & state, scsi_device * scs
     }
     else {
       // register starting values to watch for changes
-      state.selflogcount=SELFTEST_ERRORCOUNT(retval);
-      state.selfloghour =SELFTEST_ERRORHOURS(retval);
+      state.selflogcount = retval & 0xff;
+      state.selfloghour  = (retval >> 8) & 0xffff;
     }
   }
   
@@ -2639,6 +2646,9 @@ static int SCSIDeviceScan(dev_config & cfg, dev_state & state, scsi_device * scs
   // tell user we are registering device
   PrintOut(LOG_INFO, "Device: %s, is SMART capable. Adding to \"monitor\" list.\n", device);
 
+  // Disable ATA specific self-tests
+  state.not_cap_conveyance = state.not_cap_offline = state.not_cap_selective = true;
+
   // Make sure that init_standby_check() ignores SCSI devices
   cfg.offlinests_ns = cfg.selfteststs_ns = false;
 
@@ -2832,9 +2842,30 @@ static int NVMeDeviceScan(dev_config & cfg, dev_state & state, nvme_device * nvm
       state.nvme_err_log_entries = le128_to_uint64(smart_log.num_err_log_entries);
   }
 
+  // Check for self-test support
+  state.not_cap_short = state.not_cap_long = !(id_ctrl.oacs & 0x0010);
+  state.selflogcount = 0; state.selfloghour = 0;
+  if (cfg.selftest || cfg.selfteststs || !cfg.test_regex.empty()) {
+    nvme_self_test_log self_test_log;
+    if (   !state.not_cap_short
+        && !nvme_read_self_test_log(nvmedev, nvme_broadcast_nsid, self_test_log)) {
+      PrintOut(LOG_INFO, "Device: %s, Read NVMe Self-test Log failed: %s\n", name,
+               nvmedev->get_errmsg());
+      state.not_cap_short = state.not_cap_long = true;
+    }
+    if (state.not_cap_short) {
+      PrintOut(LOG_INFO, "Device: %s, does not support NVMe Self-tests, ignoring%s%s%s%s\n", name,
+               (cfg.selftest ? " -l selftest" : ""),
+               (cfg.selfteststs ? " -l selfteststs" : ""),
+               (!cfg.test_regex.empty() ? " -s " : ""), cfg.test_regex.get_pattern());
+      cfg.selftest = cfg.selfteststs = false; cfg.test_regex = {};
+    }
+  }
+
   // If no supported tests selected, return
   if (!(   cfg.smartcheck || cfg.errorlog || cfg.xerrorlog
-        || cfg.tempdiff   || cfg.tempinfo || cfg.tempcrit )) {
+        || cfg.selftest   || cfg.selfteststs || !cfg.test_regex.empty()
+        || cfg.tempdiff   || cfg.tempinfo || cfg.tempcrit )            ) {
     CloseDevice(nvmedev, name);
     return 3;
   }
@@ -2842,7 +2873,11 @@ static int NVMeDeviceScan(dev_config & cfg, dev_state & state, nvme_device * nvm
   // Tell user we are registering device
   PrintOut(LOG_INFO,"Device: %s, is SMART capable. Adding to \"monitor\" list.\n", name);
 
+  // Disable ATA specific self-tests
+  state.not_cap_conveyance = state.not_cap_offline = state.not_cap_selective = true;
+
   // Make sure that init_standby_check() ignores NVMe devices
+  // TODO: Implement '-l selfteststs,ns' for NVMe
   cfg.offlinests_ns = cfg.selfteststs_ns = false;
 
   CloseDevice(nvmedev, name);
@@ -2929,60 +2964,52 @@ static bool open_device(const dev_config & cfg, dev_state & state, smart_device
 
 // If the self-test log has got more self-test errors (or more recent
 // self-test errors) recorded, then notify user.
-static void CheckSelfTestLogs(const dev_config & cfg, dev_state & state, int newi)
+static void report_self_test_log_changes(const dev_config & cfg, dev_state & state,
+                                         int errcnt, uint64_t hour)
 {
   const char * name = cfg.name.c_str();
 
-  if (newi<0)
+  if (errcnt < 0)
     // command failed
+    // TODO: Move this to ATA/SCSICheckDevice()
     MailWarning(cfg, state, 8, "Device: %s, Read SMART Self-Test Log Failed", name);
   else {
     reset_warning_mail(cfg, state, 8, "Read SMART Self-Test Log worked again");
 
-    // old and new error counts
-    int oldc=state.selflogcount;
-    int newc=SELFTEST_ERRORCOUNT(newi);
-    
-    // old and new error timestamps in hours
-    int oldh=state.selfloghour;
-    int newh=SELFTEST_ERRORHOURS(newi);
-    
-    if (oldc<newc) {
+    if (state.selflogcount < errcnt) {
       // increase in error count
       PrintOut(LOG_CRIT, "Device: %s, Self-Test Log error count increased from %d to %d\n",
-               name, oldc, newc);
+               name, state.selflogcount, errcnt);
       MailWarning(cfg, state, 3, "Device: %s, Self-Test Log error count increased from %d to %d",
-                   name, oldc, newc);
+                  name, state.selflogcount, errcnt);
       state.must_write = true;
     }
-    else if (newc > 0 && oldh != newh) {
+    else if (errcnt > 0 && state.selfloghour != hour) {
       // more recent error
-      // a 'more recent' error might actually be a smaller hour number,
+      // ATA: a 'more recent' error might actually be a smaller hour number,
       // if the hour number has wrapped.
       // There's still a bug here.  You might just happen to run a new test
       // exactly 32768 hours after the previous failure, and have run exactly
       // 20 tests between the two, in which case smartd will miss the
       // new failure.
-      PrintOut(LOG_CRIT, "Device: %s, new Self-Test Log error at hour timestamp %d\n",
-               name, newh);
-      MailWarning(cfg, state, 3, "Device: %s, new Self-Test Log error at hour timestamp %d",
-                   name, newh);
+      PrintOut(LOG_CRIT, "Device: %s, new Self-Test Log error at hour timestamp %" PRIu64 "\n",
+               name, hour);
+      MailWarning(cfg, state, 3, "Device: %s, new Self-Test Log error at hour timestamp %" PRIu64 "\n",
+                   name, hour);
       state.must_write = true;
     }
 
     // Print info if error entries have disappeared
-    // or newer successful successful extended self-test exits
-    if (oldc > newc) {
+    // or newer successful extended self-test exists
+    if (state.selflogcount > errcnt) {
       PrintOut(LOG_INFO, "Device: %s, Self-Test Log error count decreased from %d to %d\n",
-               name, oldc, newc);
-      if (newc == 0)
+               name, state.selflogcount, errcnt);
+      if (errcnt == 0)
         reset_warning_mail(cfg, state, 3, "Self-Test Log does no longer report errors");
     }
 
-    // Needed since self-test error count may DECREASE.  Hour might
-    // also have changed.
-    state.selflogcount= newc;
-    state.selfloghour = newh;
+    state.selflogcount = errcnt;
+    state.selfloghour  = hour;
   }
   return;
 }
@@ -2993,15 +3020,15 @@ static const unsigned num_test_types = sizeof(test_type_chars)-1;
 
 // returns test type if time to do test of type testtype,
 // 0 if not time to do test.
-static char next_scheduled_test(const dev_config & cfg, dev_state & state, bool scsi, time_t usetime = 0)
+static char next_scheduled_test(const dev_config & cfg, dev_state & state, time_t usetime = 0)
 {
   // check that self-testing has been requested
   if (cfg.test_regex.empty())
     return 0;
 
   // Exit if drive not capable of any test
-  if ( state.not_cap_long && state.not_cap_short &&
-      (scsi || (state.not_cap_conveyance && state.not_cap_offline)))
+  if (   state.not_cap_long && state.not_cap_short
+      && state.not_cap_conveyance && state.not_cap_offline && state.not_cap_selective)
     return 0;
 
   // since we are about to call localtime(), be sure glibc is informed
@@ -3061,10 +3088,10 @@ static char next_scheduled_test(const dev_config & cfg, dev_state & state, bool
         switch (test_type_chars[j]) {
           case 'L': if (state.not_cap_long)       continue; break;
           case 'S': if (state.not_cap_short)      continue; break;
-          case 'C': if (scsi || state.not_cap_conveyance) continue; break;
-          case 'O': if (scsi || state.not_cap_offline)    continue; break;
+          case 'C': if (state.not_cap_conveyance) continue; break;
+          case 'O': if (state.not_cap_offline)    continue; break;
           case 'c': case 'n':
-          case 'r': if (scsi || state.not_cap_selective)  continue; break;
+          case 'r': if (state.not_cap_selective)  continue; break;
           default: continue;
         }
         // Try match of "T/MM/DD/d/HH[:NNN]"
@@ -3138,7 +3165,7 @@ static void PrintTestSchedule(const dev_config_vector & configs, dev_state_vecto
       const dev_config & cfg = configs.at(i);
       dev_state & state = states.at(i);
       const char * p;
-      char testtype = next_scheduled_test(cfg, state, devices.at(i)->is_scsi(), testtime);
+      char testtype = next_scheduled_test(cfg, state, testtime);
       if (testtype && (p = strchr(test_type_chars, testtype))) {
         unsigned t = (p - test_type_chars);
         // Report at most 5 tests of each type
@@ -3156,10 +3183,10 @@ static void PrintTestSchedule(const dev_config_vector & configs, dev_state_vecto
   PrintOut(LOG_INFO, "\nTotals [%s - %s]:\n", datenow, date);
   for (unsigned i = 0; i < numdev; i++) {
     const dev_config & cfg = configs.at(i);
-    bool scsi = devices.at(i)->is_scsi();
+    bool ata = devices.at(i)->is_ata();
     for (unsigned t = 0; t < num_test_types; t++) {
       int cnt = testcnts[i*num_test_types + t];
-      if (cnt == 0 && !strchr((scsi ? "LS" : "LSCO"), test_type_chars[t]))
+      if (cnt == 0 && !strchr((ata ? "LSCO" : "LS"), test_type_chars[t]))
         continue;
       PrintOut(LOG_INFO, "Device: %s, will do %3d test%s of type %c\n", cfg.name.c_str(),
         cnt, (cnt==1?"":"s"), test_type_chars[t]);
@@ -3234,6 +3261,7 @@ static int DoATASelfTest(const dev_config & cfg, dev_state & state, ata_device *
   const char *name = cfg.name.c_str();
 
   // Read current smart data and check status/capability
+  // TODO: Reuse smart data already read in ATACheckDevice()
   struct ata_smart_values data;
   if (ataReadSmartValues(device, &data) || !(data.offline_data_collection_capability)) {
     PrintOut(LOG_CRIT, "Device: %s, not capable of Offline or Self-Testing.\n", name);
@@ -3753,8 +3781,11 @@ static int ATACheckDevice(const dev_config & cfg, dev_state & state, ata_device
   state.offline_started = state.selftest_started = false;
   
   // check if number of selftest errors has increased (note: may also DECREASE)
-  if (cfg.selftest)
-    CheckSelfTestLogs(cfg, state, SelfTestErrorCount(atadev, name, cfg.firmwarebugs));
+  if (cfg.selftest) {
+    unsigned hour = 0;
+    int errcnt = check_ata_self_test_log(atadev, name, cfg.firmwarebugs, hour);
+    report_self_test_log_changes(cfg, state, errcnt, hour);
+  }
 
   // check if number of ATA errors has increased
   if (cfg.errorlog || cfg.xerrorlog) {
@@ -3839,11 +3870,13 @@ static int SCSICheckDevice(const dev_config & cfg, dev_state & state, scsi_devic
     CheckTemperature(cfg, state, currenttemp, triptemp);
 
   // check if number of selftest errors has increased (note: may also DECREASE)
-  if (cfg.selftest)
-    CheckSelfTestLogs(cfg, state, scsiCountFailedSelfTests(scsidev, 0));
+  if (cfg.selftest) {
+    int retval = scsiCountFailedSelfTests(scsidev, 0);
+    report_self_test_log_changes(cfg, state, (retval >= 0 ? (retval & 0xff) : -1), retval >> 8);
+  }
 
   if (allow_selftests && !cfg.test_regex.empty()) {
-    char testtype = next_scheduled_test(cfg, state, true/*scsi*/);
+    char testtype = next_scheduled_test(cfg, state);
     if (testtype)
       DoSCSISelfTest(cfg, state, scsidev, testtype);
   }
@@ -3883,7 +3916,147 @@ static int SCSICheckDevice(const dev_config & cfg, dev_state & state, scsi_devic
   return 0;
 }
 
-static int NVMeCheckDevice(const dev_config & cfg, dev_state & state, nvme_device * nvmedev)
+// Log NVMe self-test execution status changes
+static void log_nvme_self_test_exec_status(const char * name, dev_state & state, bool firstpass,
+                                           const nvme_self_test_log & self_test_log)
+{
+  uint8_t curr_op = self_test_log.current_operation & 0xf;
+  uint8_t curr_compl = self_test_log.current_completion & 0x7f;
+
+  // Return if no changes and log not forced
+  if (!(   curr_op != state.selftest_op
+        || curr_compl != state.selftest_compl
+        || state.selftest_started // test was started in previous call
+        || (firstpass && (debugmode || curr_op))))
+    return;
+
+  state.selftest_op = curr_op;
+  state.selftest_compl = curr_compl;
+
+  const nvme_self_test_result & r = self_test_log.results[0];
+  uint8_t op0 = r.self_test_status >> 4, res0 = r.self_test_status & 0xf;
+
+  uint8_t op = (curr_op ? curr_op : op0);
+  const char * t; char tb[32];
+  switch (op) {
+    case 0x0: t = ""; break;
+    case 0x1: t = "short"; break;
+    case 0x2: t = "extended"; break;
+    case 0xe: t = "vendor specific"; break;
+    default:  snprintf(tb, sizeof(tb), "unknown (0x%x)", op);
+              t = tb; break;
+  }
+
+  if (curr_op) {
+    PrintOut(LOG_INFO, "Device %s, %s self-test in progress, %d%% remaining\n",
+             name, t, 100 - curr_compl);
+  }
+  else if (!op0 || res0 == 0xf) { // First entry unused
+    PrintOut(LOG_INFO, "Device %s, no self-test has ever been run\n", name);
+  }
+  else {
+    // Report last test result from first log entry
+    const char * m; char mb[48];
+    switch (res0) {
+      case 0x0: m = "completed without error"; break;
+      case 0x1: m = "was aborted by a self-test command"; break;
+      case 0x2: m = "was aborted by a controller reset"; break;
+      case 0x3: m = "was aborted due to a namespace removal"; break;
+      case 0x4: m = "was aborted by a format NVM command"; break;
+      case 0x5: m = "completed with error (fatal or unknown error)"; break;
+      case 0x6: m = "completed with error (unknown failed segment)"; break;
+      case 0x7: m = "completed with error (failed segments)"; break;
+      case 0x8: m = "was aborted (unknown reason)"; break;
+      case 0x9: m = "was aborted due to a sanitize operation"; break;
+      default:  snprintf(mb, sizeof(mb), "returned an unknown result (0x%x)", res0);
+                m = mb; break;
+    }
+
+    char ns[32] = "";
+    if (r.valid & 0x01)
+      snprintf(ns, sizeof(ns), " of NSID 0x%x", r.nsid);
+
+    PrintOut((0x5 <= res0 && res0 <= 0x7 ? LOG_CRIT : LOG_INFO),
+             "Device %s, previous %s self-test%s %s\n", name, t, ns, m);
+  }
+}
+
+// Count error entries in NVMe self-test log, set HOUR to power on hours of most
+// recent error.  Return the error count.
+static int check_nvme_self_test_log(uint32_t nsid, const nvme_self_test_log & self_test_log,
+                                    uint64_t & hour)
+{
+  hour = 0;
+  int errcnt = 0;
+
+  for (unsigned i = 0; i < 20; i++) {
+    const nvme_self_test_result & r = self_test_log.results[i];
+    uint8_t op = r.self_test_status >> 4;
+    uint8_t res = r.self_test_status & 0xf;
+    if (!op || res == 0xf)
+      continue; // Unused entry
+
+    if (!(   nsid == nvme_broadcast_nsid
+          || !(r.valid & 0x01) /* No NSID */
+          || r.nsid == nvme_broadcast_nsid || r.nsid == nsid))
+      continue; // Different individual namespace
+
+    if (op == 0x2 /* Extended */ && !res /* Completed without error */)
+      break; // Stop count at first successful extended test
+
+    if (!(0x5 <= res && res <= 0x7))
+      continue; // No error or aborted
+
+    // Error found
+    if (++errcnt != 1)
+      continue; // Not most recent error
+
+    // Keep track of time of most recent error
+    hour = sg_get_unaligned_le64(r.power_on_hours);
+  }
+
+  return errcnt;
+}
+
+static int start_nvme_self_test(const dev_config & cfg, dev_state & state, nvme_device * device,
+                                char testtype, const nvme_self_test_log & self_test_log)
+{
+  const char *name = cfg.name.c_str();
+  unsigned nsid = device->get_nsid();
+
+  const char *testname; uint8_t stc;
+  switch (testtype) {
+    case 'S': testname = "Short";    stc = 1; break;
+    case 'L': testname = "Extended"; stc = 2; break;
+    default: // Should not happen
+      PrintOut(LOG_INFO, "Device: %s, not capable of %c Self-Test\n", name, testtype);
+      return 1;
+  }
+
+  // If currently running a self-test, do not try to start another.
+  if (self_test_log.current_operation & 0xf) {
+    PrintOut(LOG_INFO, "Device: %s, skip scheduled %s Self-Test (NSID 0x%x); %d%% remaining of current Self-Test.\n",
+             name, testname, nsid, 100 - (self_test_log.current_completion & 0x7f));
+    return 1;
+  }
+
+  if (!nvme_self_test(device, stc, nsid)) {
+    PrintOut(LOG_CRIT, "Device: %s, execute %s Self-Test failed (NSID 0x%x): %s.\n",
+             name, testname, nsid, device->get_errmsg());
+    return 1;
+  }
+
+  // Report recent test start to do_disable_standby_check()
+  // and force log of next test status
+  // TODO: Add NVMe support to do_disable_standby_check()
+  state.selftest_started = true;
+
+  PrintOut(LOG_INFO, "Device: %s, starting scheduled %s Self-Test (NSID 0x%x).\n",
+           name, testname, nsid);
+  return 0;
+}
+
+static int NVMeCheckDevice(const dev_config & cfg, dev_state & state, nvme_device * nvmedev, bool firstpass, bool allow_selftests)
 {
   if (!open_device(cfg, state, nvmedev, "NVMe"))
     return 1;
@@ -3939,6 +4112,37 @@ static int NVMeCheckDevice(const dev_config & cfg, dev_state & state, nvme_devic
     CheckTemperature(cfg, state, c, 0);
   }
 
+  // Check for test schedule
+  char testtype = (allow_selftests && !cfg.test_regex.empty()
+                   ? next_scheduled_test(cfg, state) : 0);
+
+  // Read the self-test log if required
+  nvme_self_test_log self_test_log{};
+  if (testtype || cfg.selftest || cfg.selfteststs) {
+    if (!nvme_read_self_test_log(nvmedev, nvme_broadcast_nsid, self_test_log)) {
+      PrintOut(LOG_CRIT, "Device: %s, Read Self-test Log failed: %s\n",
+               name, nvmedev->get_errmsg());
+      MailWarning(cfg, state, 8, "Device: %s, Read Self-test Log failed: %s\n",
+                  name, nvmedev->get_errmsg());
+      testtype = 0;
+    }
+    else {
+      reset_warning_mail(cfg, state, 8, "Read Self-Test Log worked again");
+
+      // Log changes of self-test execution status
+      if (cfg.selfteststs)
+        log_nvme_self_test_exec_status(name, state, firstpass, self_test_log);
+
+      // Check if number of selftest errors has increased (note: may also DECREASE)
+      if (cfg.selftest) {
+        uint64_t hour = 0;
+        int errcnt = check_nvme_self_test_log(nvmedev->get_nsid(), self_test_log, hour);
+        report_self_test_log_changes(cfg, state, errcnt, hour);
+      }
+    }
+  }
+  state.selftest_started = false;
+
   // Check if number of errors has increased
   if (cfg.errorlog || cfg.xerrorlog) {
     uint64_t newcnt = le128_to_uint64(smart_log.num_err_log_entries);
@@ -3949,6 +4153,10 @@ static int NVMeCheckDevice(const dev_config & cfg, dev_state & state, nvme_devic
     // else // TODO: Handle decrease of count?
   }
 
+  // Start self-test if scheduled
+  if (testtype)
+    start_nvme_self_test(cfg, state, nvmedev, testtype, self_test_log);
+
   CloseDevice(nvmedev, name);
   state.attrlog_dirty = true;
   return 0;
@@ -4051,7 +4259,7 @@ static void CheckDevicesOnce(const dev_config_vector & configs, dev_state_vector
     else if (dev->is_scsi())
       SCSICheckDevice(cfg, state, dev->to_scsi(), allow_selftests);
     else if (dev->is_nvme())
-      NVMeCheckDevice(cfg, state, dev->to_nvme());
+      NVMeCheckDevice(cfg, state, dev->to_nvme(), firstpass, allow_selftests);
 
     // Prevent systemd unit startup timeout when checking many devices on startup
     notify_extend_timeout();
-- 
2.35.3

