[Concept,4/5] buildman: Prioritise downloaded toolchains over system ones

Message ID 20260112225406.3274105-5-sjg@u-boot.org
State New
Headers
Series buildman: Improve toolchain selection and config adjustment |

Commit Message

Simon Glass Jan. 12, 2026, 10:54 p.m. UTC
  From: Simon Glass <simon.glass@canonical.com>

When a toolchain is downloaded via --fetch-arch for an architecture that
matches the host (e.g. aarch64 on aarch64), the system-installed gcc
from a distribution package may be selected instead of the downloaded
one. This happens because both toolchains have the same calculated
priority and the system one is scanned first.

Add a new PRIORITY_DOWNLOADED level (priority 3) that sits between
PRIORITY_PREFIX_GCC_PATH (2) and PRIORITY_CALC (4+). Track which paths
come from the 'download' key in the [toolchain] config section and use
the higher priority when scanning those paths.

The priority hierarchy is now:
  0: Explicit [toolchain-prefix] path exists as a file
  1: [toolchain-prefix] path + 'gcc' exists as a file
  2: [toolchain-prefix] path + 'gcc' found in PATH
  3: Downloaded toolchains (from --fetch-arch)
  4+: Toolchains from [toolchain] paths (calculated from filename)

Co-developed-by: Claude Opus 4.5 <noreply@anthropic.com>
Signed-off-by: Simon Glass <simon.glass@canonical.com>
---

 tools/buildman/buildman.rst | 29 +++++++++++++++++++++----
 tools/buildman/test.py      | 43 +++++++++++++++++++++++++++++++++++++
 tools/buildman/toolchain.py | 24 +++++++++++++++++++--
 3 files changed, 90 insertions(+), 6 deletions(-)
  

Patch

diff --git a/tools/buildman/buildman.rst b/tools/buildman/buildman.rst
index 379a6ab48ce..603b019540b 100644
--- a/tools/buildman/buildman.rst
+++ b/tools/buildman/buildman.rst
@@ -474,7 +474,10 @@  Setting up
       sudo mkdir -p /toolchains
       sudo mv ~/.buildman-toolchains/*/* /toolchains/
 
-   Buildman should now be set up to use your new toolchain.
+   Buildman should now be set up to use your new toolchain. Downloaded
+   toolchains are given priority over system-installed toolchains, so if you
+   have both a downloaded toolchain and one installed via your
+   distribution's package manager, the downloaded one will be used.
 
    At the time of writing, U-Boot has these architectures:
 
@@ -938,13 +941,31 @@  a set of (tag, value) pairs.
 
 '[toolchain-prefix]' section
     This can be used to provide the full toolchain-prefix for one or more
-    architectures. The full CROSS_COMPILE prefix must be provided. These
-    typically have a higher priority than matches in the '[toolchain]', due to
-    this prefix.
+    architectures. The full CROSS_COMPILE prefix must be provided.
 
     The tilde character ``~`` is supported in paths, to represent the home
     directory.
 
+Toolchain priority
+    When multiple toolchains are available for an architecture, buildman
+    selects the one with the highest priority (lowest priority number).
+
+    Note: Lower numbers indicate higher priority, so a toolchain with
+    priority 3 is preferred over one with priority 6.
+
+    The priority levels are:
+
+    - 0: Full prefix path from '[toolchain-prefix]' that exists as a file
+    - 1: Prefix from '[toolchain-prefix]' with 'gcc' appended that exists
+    - 2: Prefix from '[toolchain-prefix]' found in PATH
+    - 3: Downloaded toolchains (from ``--fetch-arch``)
+    - 4+: Toolchains found by scanning '[toolchain]' paths (priority
+      calculated from filename, e.g. '-linux' variants get priority 6)
+
+    This means that downloaded toolchains are preferred over system-installed
+    toolchains (e.g. from a distribution package), but explicit
+    '[toolchain-prefix]' entries take the highest priority.
+
 '[toolchain-alias]' section
     This converts toolchain architecture names to U-Boot names. For example,
     if an x86 toolchains is called i386-linux-gcc it will not normally be
diff --git a/tools/buildman/test.py b/tools/buildman/test.py
index b217b907176..da6df1f173c 100644
--- a/tools/buildman/test.py
+++ b/tools/buildman/test.py
@@ -701,6 +701,49 @@  class TestBuild(TestBuildBase):
                     'crosstool/files/bin/x86_64/.*/'
                     'x86_64-gcc-.*-nolibc[-_]arm-.*linux-gnueabi.tar.xz')
 
+    def test_toolchain_download_priority(self):
+        """Test that downloaded toolchains have priority over system ones"""
+        # Create a temp directory structure with two toolchains for same arch
+        with tempfile.TemporaryDirectory() as tmpdir:
+            # Create 'system' toolchain path (simulating /usr/bin)
+            system_path = os.path.join(tmpdir, 'system')
+            os.makedirs(os.path.join(system_path, 'bin'))
+            system_gcc = os.path.join(system_path, 'bin', 'aarch64-linux-gcc')
+            tools.write_file(system_gcc, b'#!/bin/sh\necho gcc')
+            os.chmod(system_gcc, 0o755)
+
+            # Create 'download' toolchain path
+            download_path = os.path.join(tmpdir, 'download')
+            os.makedirs(os.path.join(download_path, 'bin'))
+            download_gcc = os.path.join(download_path, 'bin',
+                                        'aarch64-linux-gcc')
+            tools.write_file(download_gcc, b'#!/bin/sh\necho gcc')
+            os.chmod(download_gcc, 0o755)
+
+            # Check system toolchain priority (not in download_paths)
+            sys_tc = toolchain.Toolchain(system_gcc, test=False)
+            self.assertEqual(toolchain.PRIORITY_CALC + 2, sys_tc.priority)
+
+            # Set up toolchains with download path tracked
+            tcs = toolchain.Toolchains()
+            tcs.paths = [system_path, download_path]
+            tcs.download_paths = {download_path}
+
+            # Scan and check which toolchain is selected
+            with terminal.capture():
+                tcs.scan(False, raise_on_error=False)
+
+            # The downloaded toolchain should be selected
+            tc = tcs.toolchains.get('aarch64')
+            self.assertIsNotNone(tc)
+            self.assertTrue(tc.gcc.startswith(download_path),
+                f"Expected downloaded toolchain from {download_path}, "
+                f"got {tc.gcc}")
+            self.assertEqual(toolchain.PRIORITY_DOWNLOADED, tc.priority)
+
+            # Verify downloaded priority beats system priority
+            self.assertLess(toolchain.PRIORITY_DOWNLOADED, sys_tc.priority)
+
     def test_get_env_args(self):
         """Test the GetEnvArgs() function"""
         tc = self.toolchains.select('arm')
diff --git a/tools/buildman/toolchain.py b/tools/buildman/toolchain.py
index 8ec1dbdebba..8f3d3ab3b0c 100644
--- a/tools/buildman/toolchain.py
+++ b/tools/buildman/toolchain.py
@@ -17,9 +17,17 @@  from u_boot_pylib import command
 from u_boot_pylib import terminal
 from u_boot_pylib import tools
 
+# Toolchain priority levels (lower number = higher priority):
+#   PRIORITY_FULL_PREFIX: Explicit [toolchain-prefix] path exists as a file
+#   PRIORITY_PREFIX_GCC: [toolchain-prefix] path + 'gcc' exists as a file
+#   PRIORITY_PREFIX_GCC_PATH: [toolchain-prefix] path + 'gcc' found in PATH
+#   PRIORITY_DOWNLOADED: Toolchain downloaded via --fetch-arch
+#   PRIORITY_CALC: Toolchain found by scanning [toolchain] paths; actual
+#       priority is PRIORITY_CALC + offset based on toolchain name
 (PRIORITY_FULL_PREFIX, PRIORITY_PREFIX_GCC, PRIORITY_PREFIX_GCC_PATH,
-    PRIORITY_CALC) = list(range(4))
+    PRIORITY_DOWNLOADED, PRIORITY_CALC) = list(range(5))
 
+# Environment variable / argument types for get_env_args()
 (VAR_CROSS_COMPILE, VAR_PATH, VAR_ARCH, VAR_MAKE_ARGS) = range(4)
 
 class MyHTMLParser(HTMLParser):
@@ -290,6 +298,7 @@  class Toolchains:
         self.toolchains = {}
         self.prefixes = {}
         self.paths = []
+        self.download_paths = set()
         self.override_toolchain = override_toolchain
         self._make_flags = dict(bsettings.get_items('make-flags'))
 
@@ -330,6 +339,15 @@  class Toolchains:
         self.prefixes = bsettings.get_items('toolchain-prefix')
         self.paths += self.get_path_list(show_warning)
 
+        # Track which paths are from downloaded toolchains
+        for name, value in bsettings.get_items('toolchain'):
+            if name == 'download':
+                fname = os.path.expanduser(value)
+                if '*' in value:
+                    self.download_paths.update(glob.glob(fname))
+                else:
+                    self.download_paths.add(fname)
+
     # pylint: disable=too-many-arguments,too-many-positional-arguments
     def add(self, fname, test=True, verbose=False, priority=PRIORITY_CALC,
             arch=None):
@@ -435,8 +453,10 @@  class Toolchains:
             if verbose:
                 print(f"   - scanning path '{path}'")
             fnames = self.scan_path(path, verbose)
+            priority = (PRIORITY_DOWNLOADED if path in self.download_paths
+                        else PRIORITY_CALC)
             for fname in fnames:
-                self.add(fname, True, verbose)
+                self.add(fname, True, verbose, priority)
 
     def list(self):
         """List out the selected toolchains for each architecture"""