[Concept,13/18] buildman: Test _show_not_built() and toolchain errors

Message ID 20260109183116.3262115-14-sjg@u-boot.org
State New
Headers
Series buildman: Improve test coverage for builder.py |

Commit Message

Simon Glass Jan. 9, 2026, 6:31 p.m. UTC
  From: Simon Glass <simon.glass@canonical.com>

Add testToolchainErrors() functional test to verify that boards with
missing toolchains are reported in the summary display, grouped by
architecture.

Add TestShowNotBuilt unit test class with tests for:
- All boards built (no output)
- Some boards not built (reports missing boards)
- No boards built (reports all boards)

Update the docstring for _show_not_built() to clarify that it reports
boards missing from board_dict. Note that in practice this is unlikely
to trigger since get_result_summary() creates an outcome for every
board.

Co-developed-by: Claude Opus 4.5 <noreply@anthropic.com>
Signed-off-by: Simon Glass <simon.glass@canonical.com>
---

 tools/buildman/builder.py      |  5 ++++
 tools/buildman/func_test.py    | 25 ++++++++++++++++
 tools/buildman/main.py         |  1 +
 tools/buildman/test_builder.py | 53 ++++++++++++++++++++++++++++++++++
 4 files changed, 84 insertions(+)
  

Patch

diff --git a/tools/buildman/builder.py b/tools/buildman/builder.py
index 3cb14af279f..99aac80d95e 100644
--- a/tools/buildman/builder.py
+++ b/tools/buildman/builder.py
@@ -1938,6 +1938,11 @@  class Builder:
     def _show_not_built(board_selected, board_dict):
         """Show boards that were not built
 
+        This reports boards that are in board_selected but have no outcome in
+        board_dict. In practice this is unlikely to happen since
+        get_result_summary() creates an outcome for every board, even if just
+        OUTCOME_UNKNOWN.
+
         Args:
             board_selected (dict): Dict of selected boards, keyed by target
             board_dict (dict): Dict of boards that were built, keyed by target
diff --git a/tools/buildman/func_test.py b/tools/buildman/func_test.py
index efd66796a65..21c700aa073 100644
--- a/tools/buildman/func_test.py
+++ b/tools/buildman/func_test.py
@@ -590,6 +590,31 @@  Some images are invalid'''
                        f"No tool chain found for arch '{brd.arch}'"])
                   fd.close()
 
+    def testToolchainErrors(self):
+        """Test that toolchain errors are reported in the summary
+
+        When toolchains are missing, boards fail to build. The summary
+        should report which boards had errors, grouped by architecture.
+        """
+        self.setupToolchains()
+        # Build with missing toolchains - only sandbox will succeed
+        self._RunControl('-b', TEST_BRANCH, '-o', self._output_dir)
+
+        # Now show summary - should report boards with errors
+        terminal.get_print_test_lines()  # Clear
+        self._RunControl('-b', TEST_BRANCH, '-o', self._output_dir, '-s',
+                         clean_dir=False)
+        lines = terminal.get_print_test_lines()
+        text = '\n'.join(line.text for line in lines)
+
+        # Check that boards with missing toolchains are shown with errors
+        # The '+' prefix indicates new errors for these boards
+        self.assertIn('arm:', text)
+        self.assertIn('board0', text)
+        self.assertIn('board1', text)
+        self.assertIn('powerpc:', text)
+        self.assertIn('board2', text)
+
     def testBranch(self):
         """Test building a branch with all toolchains present"""
         self._RunControl('-b', TEST_BRANCH, '-o', self._output_dir)
diff --git a/tools/buildman/main.py b/tools/buildman/main.py
index 449263b48e8..18809d843c6 100755
--- a/tools/buildman/main.py
+++ b/tools/buildman/main.py
@@ -56,6 +56,7 @@  def run_tests(skip_net_tests, debug, verbose, args):
          test_builder.TestPrintFuncSizeDetail,
          test_builder.TestPrepareThread,
          test_builder.TestPrepareWorkingSpace,
+         test_builder.TestShowNotBuilt,
          test_builder.TestPrepareOutputSpace,
          'buildman.toolchain'])
 
diff --git a/tools/buildman/test_builder.py b/tools/buildman/test_builder.py
index b17d7942e1d..d31c0080863 100644
--- a/tools/buildman/test_builder.py
+++ b/tools/buildman/test_builder.py
@@ -339,6 +339,59 @@  class TestPrepareWorkingSpace(unittest.TestCase):
         mock_prepare_thread.assert_any_call(1, True)
 
 
+class TestShowNotBuilt(unittest.TestCase):
+    """Tests for Builder._show_not_built()"""
+
+    def setUp(self):
+        """Set up test fixtures"""
+        terminal.set_print_test_mode()
+
+    def tearDown(self):
+        """Clean up after tests"""
+        terminal.set_print_test_mode(False)
+
+    def test_all_boards_built(self):
+        """Test when all selected boards were built"""
+        board_selected = {'board1': None, 'board2': None}
+        board_dict = {'board1': None, 'board2': None}
+
+        terminal.get_print_test_lines()  # Clear
+        builder.Builder._show_not_built(board_selected, board_dict)
+        lines = terminal.get_print_test_lines()
+
+        # No output when all boards were built
+        self.assertEqual(len(lines), 0)
+
+    def test_some_boards_not_built(self):
+        """Test when some boards were not built"""
+        board_selected = {'board1': None, 'board2': None, 'board3': None}
+        board_dict = {'board1': None}  # Only board1 was built
+
+        terminal.get_print_test_lines()  # Clear
+        builder.Builder._show_not_built(board_selected, board_dict)
+        lines = terminal.get_print_test_lines()
+
+        self.assertEqual(len(lines), 1)
+        self.assertIn('Boards not built', lines[0].text)
+        self.assertIn('2', lines[0].text)  # Count of not-built boards
+        self.assertIn('board2', lines[0].text)
+        self.assertIn('board3', lines[0].text)
+
+    def test_no_boards_built(self):
+        """Test when no boards were built"""
+        board_selected = {'board1': None, 'board2': None}
+        board_dict = {}  # No boards built
+
+        terminal.get_print_test_lines()  # Clear
+        builder.Builder._show_not_built(board_selected, board_dict)
+        lines = terminal.get_print_test_lines()
+
+        self.assertEqual(len(lines), 1)
+        self.assertIn('Boards not built', lines[0].text)
+        self.assertIn('board1', lines[0].text)
+        self.assertIn('board2', lines[0].text)
+
+
 class TestPrepareOutputSpace(unittest.TestCase):
     """Tests for Builder._prepare_output_space() and _get_output_space_removals()"""