tests: replace config.json with CLI args in tap_runner; add --destructive flag

- tap_runner.py: add --controller, --namespace, --log-dir, --nvme-bin,
  --debug, --no-pci-validate, --destructive CLI args; each falls back to
  the corresponding NVME_TESTS_* env var; args are propagated as env vars
  for nvme_test.py to read; fix exit code to reflect test pass/fail

- nvme_test.py: load_config() now treats config.json as optional fallback;
  NVME_TESTS_* env vars (set by tap_runner) always override config.json;
  add destructive = False class attribute; skip destructive tests unless
  NVME_TESTS_DESTRUCTIVE=1; skip gracefully when no device is configured;
  drop unused mmap import

- nvme_{compare,copy,ctrl_reset,dsm,format,read_write,writeuncor,writezeros}
  _test.py: set destructive = True so they are skipped by default

- meson_options.txt: add nvme-tests-{controller,namespace,log-dir,bin,destructive}
  options for configuring hardware tests at meson setup time

- tests/meson.build: remove config.json from infra list; build extra args
  for tap_runner from the new meson options

- run-nightly-tests.yml: replace config.json generation with meson setup
  -D options; use absolute log dir path /nvme-cli/tests/nvmetests/ to fix
  artifact upload after the switch to meson (meson runs tests from the
  build dir, so a relative path resolved to .build-ci/tests/nvmetests/)

Agent-Logs-Url: https://github.com/linux-nvme/nvme-cli/sessions/9541e0d9-7570-4a69-9907-c871a0cd3cf9

Co-authored-by: igaw <[email protected]>

Author: Copilot

.github/workflows/run-nightly-tests.yml

@@ -100,17 +100,19 @@ jobs:
             cd /nvme-cli
             git checkout ${{ github.sha }}
             CONTROLLER=$(echo "${BDEV0}" | sed 's/n[0-9]*$//')
-            cat > tests/config.json << EOJ
-            {
-              "controller" : "${CONTROLLER}",
-              "ns1": "${BDEV0}",
-              "log_dir": "tests/nvmetests/",
-              "nvme_bin": "/nvme-cli/.build-ci/nvme"
-            }
-            EOJ
-            cat tests/config.json
-
-            scripts/build.sh -b release -c gcc tests
+
+            CC=gcc meson setup \
+              --werror \
+              --buildtype=release \
+              -Dnvme-tests=true \
+              -Dnvme-tests-controller="${CONTROLLER}" \
+              -Dnvme-tests-namespace="${BDEV0}" \
+              -Dnvme-tests-log-dir="/nvme-cli/tests/nvmetests/" \
+              -Dnvme-tests-bin="/nvme-cli/.build-ci/nvme" \
+              -Dnvme-tests-destructive=true \
+              .build-ci
+            meson compile -C .build-ci
+            meson test -C .build-ci
             EOF
 
             sudo chmod +x test.sh

meson_options.txt

@@ -65,6 +65,36 @@ option(
   value : false,
   description: 'Run tests against real hardware'
 )
+option(
+  'nvme-tests-controller',
+  type : 'string',
+  value : '',
+  description: 'NVMe controller device for tests (e.g. /dev/nvme0)'
+)
+option(
+  'nvme-tests-namespace',
+  type : 'string',
+  value : '',
+  description: 'NVMe namespace device for tests (e.g. /dev/nvme0n1)'
+)
+option(
+  'nvme-tests-log-dir',
+  type : 'string',
+  value : 'nvmetests/',
+  description: 'Log directory for nvme-cli hardware tests'
+)
+option(
+  'nvme-tests-bin',
+  type : 'string',
+  value : '',
+  description: 'Path to nvme binary used by tests (default: nvme from PATH)'
+)
+option(
+  'nvme-tests-destructive',
+  type : 'boolean',
+  value : false,
+  description: 'Enable destructive (write) tests against real hardware'
+)
 option(
   'tests',
   type : 'boolean',

tests/meson.build

@@ -6,7 +6,6 @@
 # Authors: Daniel Wagner <[email protected]>
 
 infra = [
-    'config.json',
     'nvme_test.py',
     'nvme_test_io.py',
     'nvme_test_logger.py',
@@ -41,16 +40,47 @@ python_module = import('python')
 
 python = python_module.find_installation('python3')
 
+# Build the list of extra arguments for tap_runner.py from meson options.
+# Callers configure device paths at setup time, e.g.:
+#   meson setup .build -Dnvme-tests=true \
+#       -Dnvme-tests-controller=/dev/nvme0 \
+#       -Dnvme-tests-namespace=/dev/nvme0n1 \
+#       -Dnvme-tests-destructive=true
+nvme_test_args = [
+    meson.current_source_dir() / 'tap_runner.py',
+    '--start-dir', meson.current_source_dir(),
+]
+
+nvme_tests_controller = get_option('nvme-tests-controller')
+if nvme_tests_controller != ''
+    nvme_test_args += ['--controller', nvme_tests_controller]
+endif
+
+nvme_tests_namespace = get_option('nvme-tests-namespace')
+if nvme_tests_namespace != ''
+    nvme_test_args += ['--namespace', nvme_tests_namespace]
+endif
+
+nvme_tests_log_dir = get_option('nvme-tests-log-dir')
+if nvme_tests_log_dir != ''
+    nvme_test_args += ['--log-dir', nvme_tests_log_dir]
+endif
+
+nvme_tests_bin = get_option('nvme-tests-bin')
+if nvme_tests_bin != ''
+    nvme_test_args += ['--nvme-bin', nvme_tests_bin]
+endif
+
+if get_option('nvme-tests-destructive')
+    nvme_test_args += ['--destructive']
+endif
+
 foreach t : tests
     t_name = t.split('.')[0]
     test(
         'nvme-cli - @0@'.format(t_name),
         python,
-        args: [
-            meson.current_source_dir() / 'tap_runner.py',
-            '--start-dir', meson.current_source_dir(),
-            t_name,
-        ],
+        args: nvme_test_args + [t_name],
         env: ['PATH=' + meson.project_build_root() + ':/usr/bin:/usr/sbin'],
         timeout: 500,
         protocol: 'tap',

tests/nvme_compare_test.py

@@ -46,6 +46,8 @@ class TestNVMeCompareCmd(TestNVMeIO):
               - test_log_dir : directory for logs, temp files.
     """
 
+    destructive = True
+
     def compare_cmd_supported(self):
         """ Wrapper for extracting optional NVM 'compare' command support
             - Args:

tests/nvme_copy_test.py

@@ -31,6 +31,8 @@ class TestNVMeCopy(TestNVMe):
               - test_log_dir :  directory for logs, temp files.
     """
 
+    destructive = True
+
     def setUp(self):
         """ Pre Section for TestNVMeCopy """
         super().setUp()

tests/nvme_ctrl_reset_test.py

@@ -24,6 +24,8 @@ class TestNVMeCtrlReset(TestNVMe):
               - test_log_dir :  directory for logs, temp files.
     """
 
+    destructive = True
+
     def setUp(self):
         """ Pre Section for TestNVMeCtrlReset """
         super().setUp()

tests/nvme_dsm_test.py

@@ -26,6 +26,8 @@ class TestNVMeDsm(TestNVMe):
               - test_log_dir :  directory for logs, temp files.
     """
 
+    destructive = True
+
     def setUp(self):
         """ Pre Section for TestNVMeDsm """
         super().setUp()

tests/nvme_format_test.py

@@ -59,6 +59,8 @@ class TestNVMeFormatCmd(TestNVMe):
               - test_log_dir : directory for logs, temp files.
     """
 
+    destructive = True
+
     def setUp(self):
         """ Pre Section for TestNVMeFormatCmd """
         super().setUp()

tests/nvme_read_write_test.py

@@ -44,6 +44,8 @@ class TestNVMeReadWriteTest(TestNVMeIO):
               - test_log_dir : directory for logs, temp files.
     """
 
+    destructive = True
+
     def setUp(self):
         """ Pre Section for TestNVMeReadWriteTest """
         super().setUp()

tests/nvme_test.py

@@ -23,7 +23,6 @@
 """
 
 import json
-import mmap
 import os
 import re
 import shutil
@@ -63,15 +62,18 @@ class or appropriate subclass which is a child of this class.
             - ctrl : NVMe Controller.
             - ns1 : default namespace.
             - default_nsid : default namespace id.
-            - config_file : configuration file.
+            - config_file : configuration file (optional fallback).
             - clear_log_dir : default log directory.
+            - destructive : set to True in subclasses that write device data.
     """
 
+    destructive = False
+
     def setUp(self):
         """ Pre Section for TestNVMe. """
         # common code used in various testcases.
-        self.ctrl = "XXX"
-        self.ns1 = "XXX"
+        self.ctrl = ""
+        self.ns1 = ""
         self.test_log_dir = "XXX"
         self.nvme_bin = "nvme"
         self.do_validate_pci_device = True
@@ -81,6 +83,16 @@ def setUp(self):
         self.config_file = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'config.json')
 
         self.load_config()
+
+        if not self.ctrl:
+            self.skipTest("no NVMe controller configured; use --controller or config.json")
+        if not self.ns1:
+            self.skipTest("no NVMe namespace configured; use --namespace or config.json")
+        if self.destructive and \
+                os.environ.get('NVME_TESTS_DESTRUCTIVE', '0') != '1':
+            self.skipTest("destructive tests are disabled; "
+                          "pass --destructive to tap_runner.py to enable")
+
         if self.do_validate_pci_device:
             self.validate_pci_device()
         self.ns_mgmt_supported = self.get_ns_mgmt_support()
@@ -136,30 +148,48 @@ def validate_pci_device(self):
         self.assertEqual(err, 0, "ERROR : Only NVMe PCI subsystem is supported")
 
     def load_config(self):
-        """ Load Basic test configuration.
+        """ Load test configuration from config.json (if present) and
+            environment variables. Environment variables always take
+            priority over config.json values.
             - Args:
                 - None
             - Returns:
                 - None
         """
-        with open(self.config_file) as data_file:
-            config = json.load(data_file)
-            self.ctrl = config['controller']
-            self.ns1 = config['ns1']
-            self.log_dir = config['log_dir']
-            self.nvme_bin = config.get('nvme_bin', self.nvme_bin)
-            self.debug = config.get('debug', False)
-            if self.debug:
-                print(f"Using nvme binary '{self.nvme_bin}'")
-            self.do_validate_pci_device = config.get(
-                'do_validate_pci_device', self.do_validate_pci_device)
-            self.clear_log_dir = False
-
-            if self.clear_log_dir is True:
-                shutil.rmtree(self.log_dir, ignore_errors=True)
-
-            if not os.path.exists(self.log_dir):
-                os.makedirs(self.log_dir)
+        if os.path.exists(self.config_file):
+            with open(self.config_file) as data_file:
+                config = json.load(data_file)
+                self.ctrl = config.get('controller', self.ctrl)
+                self.ns1 = config.get('ns1', self.ns1)
+                self.log_dir = config.get('log_dir', 'nvmetests/')
+                self.nvme_bin = config.get('nvme_bin', self.nvme_bin)
+                self.debug = config.get('debug', self.debug)
+                self.do_validate_pci_device = config.get(
+                    'do_validate_pci_device', self.do_validate_pci_device)
+        else:
+            self.log_dir = 'nvmetests/'
+
+        # Environment variables override config.json values
+        if os.environ.get('NVME_TESTS_CONTROLLER'):
+            self.ctrl = os.environ['NVME_TESTS_CONTROLLER']
+        if os.environ.get('NVME_TESTS_NAMESPACE'):
+            self.ns1 = os.environ['NVME_TESTS_NAMESPACE']
+        if os.environ.get('NVME_TESTS_LOG_DIR'):
+            self.log_dir = os.environ['NVME_TESTS_LOG_DIR']
+        if os.environ.get('NVME_TESTS_BIN'):
+            self.nvme_bin = os.environ['NVME_TESTS_BIN']
+        if os.environ.get('NVME_TESTS_DEBUG'):
+            self.debug = os.environ['NVME_TESTS_DEBUG'] == '1'
+        if os.environ.get('NVME_TESTS_SKIP_PCI_VALIDATE') == '1':
+            self.do_validate_pci_device = False
+
+        if self.debug:
+            print(f"Using nvme binary '{self.nvme_bin}'")
+
+        self.clear_log_dir = False
+
+        if not os.path.exists(self.log_dir):
+            os.makedirs(self.log_dir)
 
     def setup_log_dir(self, test_name):
         """ Set up the log directory for a testcase