[Concept,4/9] test: py: Add run_ut() helper for manual unit tests

Message ID 20251229160611.3899708-5-sjg@u-boot.org
State New
Headers
Series test: Various improvements to unit-test infrastructure |

Commit Message

Simon Glass Dec. 29, 2025, 4:06 p.m. UTC
  From: Simon Glass <simon.glass@canonical.com>

Running manual unit tests (those with _norun suffix) involves a common
pattern: building the ut command with the -f flag, running it, and
checking for failures. This is verbose and error-prone.

Add a run_ut() method to ConsoleBase that simplifies this. It handles
the command construction, test arguments and failure checking
automatically.

Before:
    output = ubman.run_command(
        f'ut -f fs fs_test_ext4l_probe_norun fs_image={ext4_image}')
    assert 'failures: 0' in output

After:
    ubman.run_ut('fs', 'fs_test_ext4l_probe', fs_image=ext4_image)

Co-developed-by: Claude Opus 4.5 <noreply@anthropic.com>
Signed-off-by: Simon Glass <simon.glass@canonical.com>
---

 test/py/console_base.py             | 27 +++++++++
 test/py/tests/test_fit_print.py     |  8 +--
 test/py/tests/test_fs/test_basic.py | 87 +++++++++--------------------
 test/py/tests/test_upl.py           |  3 +-
 test/py/tests/test_vbe.py           |  8 +--
 5 files changed, 58 insertions(+), 75 deletions(-)
  

Patch

diff --git a/test/py/console_base.py b/test/py/console_base.py
index 2bfd42746e4..86b6da1f5b0 100644
--- a/test/py/console_base.py
+++ b/test/py/console_base.py
@@ -485,6 +485,33 @@  class ConsoleBase():
             output.append(self.run_command(cmd))
         return output
 
+    def run_ut(self, suite, test, **kwargs):
+        """Run a manual unit test
+
+        Run a unit test that has the _norun suffix, meaning it requires
+        external setup (like creating a disk image) before it can run.
+
+        Args:
+            suite (str): Test suite name (e.g., 'fs')
+            test (str): Test name without _norun suffix
+                (e.g., 'fs_test_ext4l_probe')
+            **kwargs: Test arguments passed as key=value
+                (e.g., fs_image='/path/to/img')
+
+        Returns:
+            str: Command output
+
+        Raises:
+            AssertionError: If test reports failures
+        """
+        args = ' '.join(f'{k}={v}' for k, v in kwargs.items())
+        cmd = f'ut -f {suite} {test}_norun'
+        if args:
+            cmd += f' {args}'
+        output = self.run_command(cmd)
+        assert 'failures: 0' in output, f'Test {test} failed'
+        return output
+
     def send(self, msg):
         """Send characters without waiting for echo, etc."""
         self.run_command(msg, wait_for_prompt=False, wait_for_echo=False,
diff --git a/test/py/tests/test_fit_print.py b/test/py/tests/test_fit_print.py
index d8b034e9ce9..099a3a70591 100644
--- a/test/py/tests/test_fit_print.py
+++ b/test/py/tests/test_fit_print.py
@@ -261,9 +261,7 @@  def test_fit_print(ubman):
     build_test_fit(ubman, fit)
 
     # Run the C test which will load and verify this FIT
-    ubman.run_command('ut -f bootstd test_fit_print_norun')
-    result = ubman.run_command('echo $?')
-    assert '0' == result
+    ubman.run_ut('bootstd', 'test_fit_print')
 
 
 @pytest.mark.boardspec('sandbox')
@@ -279,9 +277,7 @@  def test_fit_print_no_desc(ubman):
     utils.run_and_log(ubman, ['fdtput', '-d', fit, '/', 'description'])
 
     # Run the C test to check the missing description
-    ubman.run_command('ut -f bootstd test_fit_print_no_desc_norun')
-    result = ubman.run_command('echo $?')
-    assert '0' == result
+    ubman.run_ut('bootstd', 'test_fit_print_no_desc')
 
 @pytest.mark.boardspec('sandbox')
 @pytest.mark.buildconfigspec('fit_print')
diff --git a/test/py/tests/test_fs/test_basic.py b/test/py/tests/test_fs/test_basic.py
index 7f805d04dd5..174e2e074f4 100644
--- a/test/py/tests/test_fs/test_basic.py
+++ b/test/py/tests/test_fs/test_basic.py
@@ -16,39 +16,6 @@  from fstest_defs import SMALL_FILE, BIG_FILE
 from fstest_helpers import assert_fs_integrity
 
 
-def run_c_test(ubman, fs_type, fs_img, test_name, small=None, big=None,
-               md5val=None):
-    """Run a C unit test with proper setup.
-
-    Args:
-        ubman (ConsoleBase): U-Boot console manager
-        fs_type (str): Filesystem type (ext4, fat, fs_generic, exfat)
-        fs_img (str): Path to filesystem image
-        test_name (str): Name of C test function (without _norun suffix)
-        small (str): Filename of small test file (optional)
-        big (str): Filename of big test file (optional)
-        md5val (str): Expected MD5 value for verification (optional)
-
-    Returns:
-        bool: True if test passed, False otherwise
-    """
-    # Build the command with arguments
-    cmd = f'ut -f fs {test_name}_norun fs_type={fs_type} fs_image={fs_img}'
-    if small:
-        cmd += f' small={small}'
-    if big:
-        cmd += f' big={big}'
-    if md5val:
-        cmd += f' md5val={md5val}'
-
-    # Run the C test
-    ubman.run_command(cmd)
-
-    # Check result
-    result = ubman.run_command('echo $?')
-    return result.strip() == '0'
-
-
 @pytest.mark.boardspec('sandbox')
 @pytest.mark.slow
 class TestFsBasic:
@@ -58,94 +25,92 @@  class TestFsBasic:
         """Test Case 1 - ls command, listing root and invalid directories"""
         fs_type, fs_img, _ = fs_obj_basic
         with ubman.log.section('Test Case 1 - ls'):
-            assert run_c_test(ubman, fs_type, fs_img, 'fs_test_ls',
-                              small=SMALL_FILE, big=BIG_FILE)
+            ubman.run_ut('fs', 'fs_test_ls', fs_type=fs_type, fs_image=fs_img,
+                         small=SMALL_FILE, big=BIG_FILE)
 
     def test_fs2(self, ubman, fs_obj_basic):
         """Test Case 2 - size command for a small file"""
         fs_type, fs_img, _ = fs_obj_basic
         with ubman.log.section('Test Case 2 - size (small)'):
-            assert run_c_test(ubman, fs_type, fs_img, 'fs_test_size_small',
-                              small=SMALL_FILE)
+            ubman.run_ut('fs', 'fs_test_size_small', fs_type=fs_type,
+                         fs_image=fs_img, small=SMALL_FILE)
 
     def test_fs3(self, ubman, fs_obj_basic):
         """Test Case 3 - size command for a large file"""
         fs_type, fs_img, _ = fs_obj_basic
         with ubman.log.section('Test Case 3 - size (large)'):
-            assert run_c_test(ubman, fs_type, fs_img, 'fs_test_size_big',
-                              big=BIG_FILE)
+            ubman.run_ut('fs', 'fs_test_size_big', fs_type=fs_type,
+                         fs_image=fs_img, big=BIG_FILE)
 
     def test_fs4(self, ubman, fs_obj_basic):
         """Test Case 4 - load a small file, 1MB"""
         fs_type, fs_img, md5val = fs_obj_basic
         with ubman.log.section('Test Case 4 - load (small)'):
-            assert run_c_test(ubman, fs_type, fs_img, 'fs_test_load_small',
-                              small=SMALL_FILE, md5val=md5val[0])
+            ubman.run_ut('fs', 'fs_test_load_small', fs_type=fs_type,
+                         fs_image=fs_img, small=SMALL_FILE, md5val=md5val[0])
 
     def test_fs5(self, ubman, fs_obj_basic):
         """Test Case 5 - load, reading first 1MB of 3GB file"""
         fs_type, fs_img, md5val = fs_obj_basic
         with ubman.log.section('Test Case 5 - load (first 1MB)'):
-            assert run_c_test(ubman, fs_type, fs_img, 'fs_test_load_big_first',
-                              big=BIG_FILE, md5val=md5val[1])
+            ubman.run_ut('fs', 'fs_test_load_big_first', fs_type=fs_type,
+                         fs_image=fs_img, big=BIG_FILE, md5val=md5val[1])
 
     def test_fs6(self, ubman, fs_obj_basic):
         """Test Case 6 - load, reading last 1MB of 3GB file"""
         fs_type, fs_img, md5val = fs_obj_basic
         with ubman.log.section('Test Case 6 - load (last 1MB)'):
-            assert run_c_test(ubman, fs_type, fs_img, 'fs_test_load_big_last',
-                              big=BIG_FILE, md5val=md5val[2])
+            ubman.run_ut('fs', 'fs_test_load_big_last', fs_type=fs_type,
+                         fs_image=fs_img, big=BIG_FILE, md5val=md5val[2])
 
     def test_fs7(self, ubman, fs_obj_basic):
         """Test Case 7 - load, 1MB from the last 1MB in 2GB"""
         fs_type, fs_img, md5val = fs_obj_basic
         with ubman.log.section('Test Case 7 - load (last 1MB in 2GB)'):
-            assert run_c_test(ubman, fs_type, fs_img,
-                              'fs_test_load_big_2g_last',
-                              big=BIG_FILE, md5val=md5val[3])
+            ubman.run_ut('fs', 'fs_test_load_big_2g_last', fs_type=fs_type,
+                         fs_image=fs_img, big=BIG_FILE, md5val=md5val[3])
 
     def test_fs8(self, ubman, fs_obj_basic):
         """Test Case 8 - load, reading first 1MB in 2GB"""
         fs_type, fs_img, md5val = fs_obj_basic
         with ubman.log.section('Test Case 8 - load (first 1MB in 2GB)'):
-            assert run_c_test(ubman, fs_type, fs_img,
-                              'fs_test_load_big_2g_first',
-                              big=BIG_FILE, md5val=md5val[4])
+            ubman.run_ut('fs', 'fs_test_load_big_2g_first', fs_type=fs_type,
+                         fs_image=fs_img, big=BIG_FILE, md5val=md5val[4])
 
     def test_fs9(self, ubman, fs_obj_basic):
         """Test Case 9 - load, 1MB crossing 2GB boundary"""
         fs_type, fs_img, md5val = fs_obj_basic
         with ubman.log.section('Test Case 9 - load (crossing 2GB boundary)'):
-            assert run_c_test(ubman, fs_type, fs_img,
-                              'fs_test_load_big_2g_cross',
-                              big=BIG_FILE, md5val=md5val[5])
+            ubman.run_ut('fs', 'fs_test_load_big_2g_cross', fs_type=fs_type,
+                         fs_image=fs_img, big=BIG_FILE, md5val=md5val[5])
 
     def test_fs10(self, ubman, fs_obj_basic):
         """Test Case 10 - load, reading beyond file end"""
         fs_type, fs_img, _ = fs_obj_basic
         with ubman.log.section('Test Case 10 - load (beyond file end)'):
-            assert run_c_test(ubman, fs_type, fs_img, 'fs_test_load_beyond',
-                              big=BIG_FILE)
+            ubman.run_ut('fs', 'fs_test_load_beyond', fs_type=fs_type,
+                         fs_image=fs_img, big=BIG_FILE)
 
     def test_fs11(self, ubman, fs_obj_basic):
         """Test Case 11 - write"""
         fs_type, fs_img, md5val = fs_obj_basic
         with ubman.log.section('Test Case 11 - write'):
-            assert run_c_test(ubman, fs_type, fs_img, 'fs_test_write',
-                              small=SMALL_FILE, md5val=md5val[0])
+            ubman.run_ut('fs', 'fs_test_write', fs_type=fs_type,
+                         fs_image=fs_img, small=SMALL_FILE, md5val=md5val[0])
             assert_fs_integrity(fs_type, fs_img)
 
     def test_fs12(self, ubman, fs_obj_basic):
         """Test Case 12 - write to "." directory"""
         fs_type, fs_img, _ = fs_obj_basic
         with ubman.log.section('Test Case 12 - write (".")'):
-            assert run_c_test(ubman, fs_type, fs_img, 'fs_test_write_dot')
+            ubman.run_ut('fs', 'fs_test_write_dot', fs_type=fs_type,
+                         fs_image=fs_img)
             assert_fs_integrity(fs_type, fs_img)
 
     def test_fs13(self, ubman, fs_obj_basic):
         """Test Case 13 - write to a file with '/./<filename>'"""
         fs_type, fs_img, md5val = fs_obj_basic
         with ubman.log.section('Test Case 13 - write  ("./<file>")'):
-            assert run_c_test(ubman, fs_type, fs_img, 'fs_test_write_dotpath',
-                              small=SMALL_FILE, md5val=md5val[0])
+            ubman.run_ut('fs', 'fs_test_write_dotpath', fs_type=fs_type,
+                         fs_image=fs_img, small=SMALL_FILE, md5val=md5val[0])
             assert_fs_integrity(fs_type, fs_img)
diff --git a/test/py/tests/test_upl.py b/test/py/tests/test_upl.py
index c79c32adf0b..f2b69078cf1 100644
--- a/test/py/tests/test_upl.py
+++ b/test/py/tests/test_upl.py
@@ -33,5 +33,4 @@  def test_upl_handoff(ubman):
     assert 'UPL state: active' == output
 
     # Check the FIT offsets look correct
-    output = ubman.run_command('ut upl -f upl_test_info_norun')
-    assert 'failures: 0' in output
+    ubman.run_ut('upl', 'upl_test_info')
diff --git a/test/py/tests/test_vbe.py b/test/py/tests/test_vbe.py
index 4ccf4fb937b..411ed429605 100644
--- a/test/py/tests/test_vbe.py
+++ b/test/py/tests/test_vbe.py
@@ -127,9 +127,7 @@  def test_vbe_extlinux_fit_no_oem(ubman):
     fname = os.path.join(ubman.config.persistent_data_dir, 'vbe0.img')
     ubman.run_command(f'host bind 0 {fname}')
 
-    ubman.run_command('ut -f bootstd vbe_test_abrec_no_oem_norun')
-    result = ubman.run_command('echo $?')
-    assert '0' == result
+    ubman.run_ut('bootstd', 'vbe_test_abrec_no_oem')
 
 @pytest.mark.boardspec('sandbox')
 def test_vbe_extlinux_fit_oem(ubman):
@@ -137,6 +135,4 @@  def test_vbe_extlinux_fit_oem(ubman):
     fname = os.path.join(ubman.config.persistent_data_dir, 'vbe1.img')
     ubman.run_command(f'host bind 0 {fname}')
 
-    ubman.run_command('ut -f bootstd vbe_test_abrec_oem_norun')
-    result = ubman.run_command('echo $?')
-    assert '0' == result
+    ubman.run_ut('bootstd', 'vbe_test_abrec_oem')