@@ -20,7 +20,7 @@ def setup(fname=''):
global settings # pylint: disable=W0603
global config_fname # pylint: disable=W0603
- settings = configparser.ConfigParser()
+ settings = configparser.ConfigParser(allow_no_value=True)
if fname is not None:
config_fname = fname
if config_fname == '':
@@ -110,6 +110,14 @@ x86 = i386
# snapper-boards=ENABLE_AT91_TEST=1
# snapper9260=${snapper-boards} BUILD_TAG=442
# snapper9g45=${snapper-boards} BUILD_TAG=443
+
+[machines]
+# Remote build machines for distributed builds
+# List hostnames, one per line (or user@hostname)
+# e.g.
+# ohau
+# moa
+# user@build-server
''', file=out)
except IOError:
print(f"Couldn't create buildman config file '{cfgname}'\n")
@@ -105,6 +105,13 @@ def add_upto_m(parser):
parser.add_argument(
'-M', '--allow-missing', action='store_true', default=False,
help='Tell binman to allow missing blobs and generate fake ones as needed')
+ parser.add_argument('--mach', '--machines', action='store_true',
+ default=False, dest='machines',
+ help='Probe all remote machines from [machines] config and show '
+ 'their status and available toolchains')
+ parser.add_argument('--machines-buildman-path', type=str,
+ default='buildman',
+ help='Path to buildman on remote machines (default: %(default)s)')
parser.add_argument(
'--maintainer-check', action='store_true',
help='Check that maintainer entries exist for each board')
@@ -18,6 +18,7 @@ import time
from buildman import boards
from buildman import bsettings
from buildman import cfgutil
+from buildman import machine
from buildman import toolchain
from buildman.builder import Builder
from buildman.outcome import DisplayOptions
@@ -758,6 +759,11 @@ def do_buildman(args, toolchains=None, make_func=None, brds=None,
gitutil.setup()
col = terminal.Color()
+ # Handle --machines: probe remote machines and show status
+ if args.machines:
+ return machine.do_probe_machines(
+ col, buildman_path=args.machines_buildman_path)
+
git_dir = os.path.join(args.git, '.git')
toolchains = get_toolchains(toolchains, col, args.override_toolchain,
new file mode 100644
@@ -0,0 +1,923 @@
+# SPDX-License-Identifier: GPL-2.0+
+# Copyright 2026 Simon Glass <sjg@chromium.org>
+
+"""Handles remote machine probing and pool management for distributed builds
+
+This module provides the Machine and MachinePool classes for managing a pool of
+remote build machines. Machines are probed over SSH to determine their
+capabilities (CPUs, memory, load, toolchains) and can be used to distribute
+board builds across multiple hosts.
+"""
+
+import dataclasses
+import json
+import os
+import threading
+
+from buildman import bsettings
+from buildman import toolchain as toolchain_mod
+from u_boot_pylib import command
+from u_boot_pylib import terminal
+from u_boot_pylib import tout
+
+# Probe script run on remote machines via SSH. This is kept minimal so that
+# it works on any Linux machine with Python 3.
+PROBE_SCRIPT = r'''
+import json, os, platform, subprocess
+
+def get_cpus():
+ try:
+ return int(subprocess.check_output(['nproc', '--all'], text=True))
+ except (subprocess.CalledProcessError, FileNotFoundError, ValueError):
+ return 1
+
+def get_threads():
+ try:
+ return int(subprocess.check_output(['nproc'], text=True))
+ except (subprocess.CalledProcessError, FileNotFoundError, ValueError):
+ return get_cpus()
+
+def get_load():
+ try:
+ with open('/proc/loadavg') as f:
+ return float(f.read().split()[0])
+ except (IOError, ValueError, IndexError):
+ return 0.0
+
+def get_mem_avail_mb():
+ try:
+ with open('/proc/meminfo') as f:
+ for line in f:
+ if line.startswith('MemAvailable:'):
+ return int(line.split()[1]) // 1024
+ except (IOError, ValueError, IndexError):
+ pass
+ return 0
+
+def get_disk_avail_mb(path='~'):
+ path = os.path.expanduser(path)
+ try:
+ st = os.statvfs(path)
+ return (st.f_bavail * st.f_frsize) // (1024 * 1024)
+ except OSError:
+ return 0
+
+def get_bogomips():
+ try:
+ with open('/proc/cpuinfo') as f:
+ for line in f:
+ lower = line.lower()
+ if 'bogomips' in lower and ':' in lower:
+ return float(line.split(':')[1].strip())
+ except (IOError, ValueError, IndexError):
+ pass
+ return 0.0
+
+print(json.dumps({
+ 'arch': platform.machine(),
+ 'cpus': get_cpus(),
+ 'threads': get_threads(),
+ 'bogomips': get_bogomips(),
+ 'load_1m': get_load(),
+ 'mem_avail_mb': get_mem_avail_mb(),
+ 'disk_avail_mb': get_disk_avail_mb(),
+}))
+'''
+
+# Load threshold: if load_1m / cpus exceeds this, the machine is busy
+LOAD_THRESHOLD = 0.8
+
+# Minimum available disk space in MB to use a machine
+MIN_DISK_MB = 1000
+
+# Minimum available memory in MB to use a machine
+MIN_MEM_MB = 512
+
+# SSH connect timeout in seconds
+SSH_TIMEOUT = 10
+
+# Shorter timeout for probing, since it should be fast
+PROBE_TIMEOUT = 3
+
+
+@dataclasses.dataclass
+class MachineInfo:
+ """Probe results for a remote machine
+
+ Attributes:
+ arch (str): Machine architecture (e.g. 'x86_64', 'aarch64')
+ cpus (int): Number of physical CPU cores
+ threads (int): Number of hardware threads
+ bogomips (float): BogoMIPS from /proc/cpuinfo (single core)
+ load (float): 1-minute load average
+ mem_avail_mb (int): Available memory in MB
+ disk_avail_mb (int): Available disk space in MB
+ """
+ arch: str = ''
+ cpus: int = 0
+ threads: int = 0
+ bogomips: float = 0.0
+ load: float = 0.0
+ mem_avail_mb: int = 0
+ disk_avail_mb: int = 0
+
+
+class MachineError(Exception):
+ """Error communicating with a remote machine"""
+
+
+def _run_ssh(hostname, cmd, timeout=SSH_TIMEOUT, stdin_data=None):
+ """Run a command on a remote machine via SSH
+
+ Args:
+ hostname (str): SSH hostname (user@host or just host)
+ cmd (list of str): Command and arguments, passed after '--' to
+ SSH. May be a single-element list with a shell command string
+ timeout (int): Connection timeout in seconds
+ stdin_data (str or None): Data to send to the command's stdin
+
+ Returns:
+ str: stdout from the command
+
+ Raises:
+ MachineError: if SSH connection fails or command returns non-zero
+ """
+ ssh_cmd = [
+ 'ssh',
+ '-o', 'BatchMode=yes',
+ '-o', f'ConnectTimeout={timeout}',
+ '-o', 'StrictHostKeyChecking=accept-new',
+ hostname,
+ '--',
+ ] + cmd
+ try:
+ result = command.run_pipe(
+ [ssh_cmd], capture=True, capture_stderr=True,
+ raise_on_error=False, stdin_data=stdin_data)
+ except command.CommandExc as exc:
+ raise MachineError(str(exc)) from exc
+
+ if result.return_code:
+ stderr = result.stderr.strip()
+ if stderr:
+ # Take last non-empty line as the real error
+ lines = [l for l in stderr.splitlines()
+ if l.strip()]
+ msg = lines[-1] if lines else stderr
+ raise MachineError(f'SSH to {hostname}: {msg}')
+ raise MachineError(
+ f'SSH to {hostname} failed with code '
+ f'{result.return_code}')
+
+ return result.stdout
+
+
+def gcc_version(gcc_path):
+ """Extract the gcc version directory from a toolchain path
+
+ Looks for a 'gcc-*-nolibc' component in the path, which is the
+ standard naming convention for buildman-fetched toolchains.
+
+ Args:
+ gcc_path (str): Full path to gcc binary, e.g.
+ '~/.buildman-toolchains/gcc-13.1.0-nolibc/aarch64-linux/
+ bin/aarch64-linux-gcc'
+
+ Returns:
+ str or None: The version directory (e.g. 'gcc-13.1.0-nolibc'),
+ or None if the path does not follow this convention
+ """
+ for part in gcc_path.split('/'):
+ if part.startswith('gcc-') and 'nolibc' in part:
+ return part
+ return None
+
+
+def _parse_toolchain_list(output):
+ """Parse the output of 'buildman --list-tool-chains'
+
+ Extracts architecture -> gcc path mapping from the output.
+
+ Args:
+ output (str): Output from buildman --list-tool-chains
+
+ Returns:
+ dict: Architecture name -> gcc path string
+ """
+ archs = {}
+ in_list = False
+ for line in output.splitlines():
+ # The list starts after "List of available toolchains"
+ if 'List of available toolchains' in line:
+ in_list = True
+ continue
+ if in_list and ':' in line:
+ parts = line.split(':', 1)
+ if len(parts) == 2:
+ arch = parts[0].strip()
+ gcc = parts[1].strip()
+ if arch and gcc and arch != 'None':
+ archs[arch] = gcc
+ elif in_list and not line.strip():
+ # Empty line ends the list
+ break
+ return archs
+
+
+def _toolchain_status(mach, local_archs, local_gcc=None):
+ """Get toolchain status text and colour for a machine
+
+ Args:
+ mach (Machine): Machine to check
+ local_archs (set of str): Toolchain archs available on local host
+ local_gcc (dict or None): arch -> gcc path on the local machine
+
+ Returns:
+ tuple: (str, colour) where colour is a terminal.Color constant
+ or None for no colour
+ """
+ if not mach.toolchains:
+ err = mach.tc_error
+ if err:
+ return 'fail', terminal.Color.RED
+ if not mach.avail and not mach.info.arch:
+ return '-', None
+ return 'none', terminal.Color.YELLOW
+ if not local_archs:
+ return str(len(mach.toolchains)), None
+ missing = local_archs - set(mach.toolchains.keys())
+
+ # Check for version mismatches
+ mismatched = 0
+ if local_gcc:
+ for arch, path in mach.toolchains.items():
+ local_ver = gcc_version(local_gcc.get(arch, ''))
+ if not local_ver:
+ continue
+ remote_ver = gcc_version(path)
+ if remote_ver and remote_ver != local_ver:
+ mismatched += 1
+
+ parts = []
+ if missing:
+ parts.append(f'{len(missing)} missing')
+ if mismatched:
+ parts.append(f'{mismatched} wrong ver')
+ if parts:
+ return ', '.join(parts), terminal.Color.YELLOW
+ return 'OK', terminal.Color.GREEN
+
+
+def build_version_map(local_gcc):
+ """Build a map of architecture -> gcc version directory
+
+ Args:
+ local_gcc (dict): arch -> gcc path
+
+ Returns:
+ dict: arch -> version string (e.g. 'gcc-13.1.0-nolibc')
+ """
+ versions = {}
+ if local_gcc:
+ for arch, path in local_gcc.items():
+ ver = gcc_version(path)
+ if ver:
+ versions[arch] = ver
+ return versions
+
+
+def resolve_toolchain_aliases(gcc_dict):
+ """Add toolchain-alias entries to a gcc dict
+
+ Resolves [toolchain-alias] config entries (e.g. x86->i386, sh->sh4)
+ so that board architectures using alias names are recognised.
+
+ Args:
+ gcc_dict (dict): arch -> gcc path, modified in place
+ """
+ for tag, value in bsettings.get_items('toolchain-alias'):
+ if tag not in gcc_dict:
+ for alias in value.split():
+ if alias in gcc_dict:
+ gcc_dict[tag] = gcc_dict[alias]
+ break
+
+
+def get_machines_config():
+ """Get the list of machine hostnames from the config
+
+ Returns:
+ list of str: List of hostnames from [machines] section
+ """
+ items = bsettings.get_items('machines')
+ return [value.strip() if value else name.strip()
+ for name, value in items]
+
+
+def do_probe_machines(col=None, fetch=False, buildman_path='buildman'):
+ """Probe all configured machines and display their status
+
+ This is the entry point for 'buildman --machines' when used without a
+ build command. It probes all machines, checks their toolchains and
+ prints a summary.
+
+ Args:
+ col (terminal.Color or None): Colour object for output
+ fetch (bool): True to fetch missing toolchains
+ buildman_path (str): Path to buildman on remote machines
+
+ Returns:
+ int: 0 on success, non-zero on failure
+ """
+ if not col:
+ col = terminal.Color()
+
+ machines = get_machines_config()
+ if not machines:
+ print(col.build(col.RED,
+ 'No machines configured. Add a [machines] section '
+ 'to ~/.buildman'))
+ return 1
+
+ # Get local toolchains for comparison. Only include cross-
+ # toolchains under ~/.buildman-toolchains/ since system compilers
+ # (sandbox, c89, c99) can't be probed or fetched remotely.
+ local_toolchains = toolchain_mod.Toolchains()
+ local_toolchains.get_settings(show_warning=False)
+ local_toolchains.scan(verbose=False)
+ home = os.path.expanduser('~')
+ local_gcc = {arch: tc.gcc
+ for arch, tc in local_toolchains.toolchains.items()
+ if tc.gcc.startswith(home)}
+ resolve_toolchain_aliases(local_gcc)
+ local_archs = set(local_gcc.keys())
+
+ pool = MachinePool()
+ pool.probe_all(col)
+ pool.check_toolchains(local_archs, buildman_path=buildman_path,
+ fetch=fetch, col=col, local_gcc=local_gcc)
+ pool.print_summary(col, local_archs=local_archs,
+ local_gcc=local_gcc)
+ return 0
+
+
+class MachinePool:
+ """Manages a pool of remote build machines
+
+ Reads machine hostnames from the [machines] section of the buildman
+ config and provides methods to probe, check toolchains and display
+ the status of all machines.
+
+ Attributes:
+ machines (list of Machine): All machines in the pool
+ """
+
+ def __init__(self, names=None):
+ """Create a MachinePool
+
+ Args:
+ names (list of str or None): If provided, only include machines
+ whose config key matches one of these names. If None, include
+ all machines from the config.
+ """
+ self.machines = []
+ self._load_from_config(names)
+
+ def _load_from_config(self, names=None):
+ """Load machine hostnames from the [machines] config section
+
+ Supports bare hostnames (one per line) or name=hostname pairs.
+ The hostname may include a username (user@host):
+ [machines]
+ ohau
+ moa
+ myserver = build1.example.com
+ ruru = sglass@ruru
+
+ Args:
+ names (list of str or None): If provided, only include machines
+ whose config key matches one of these names
+ """
+ name_set = set(names) if names else set()
+ items = bsettings.get_items('machines')
+ for name, value in items:
+ # With allow_no_value=True, bare hostnames have value=None
+ # and the hostname is the key. For key=value pairs, use value.
+ key = name.strip()
+ if name_set and key not in name_set:
+ continue
+ hostname = value.strip() if value else key
+ mach = Machine(hostname, name=key)
+ for oname, ovalue in bsettings.get_items(f'machine:{key}'):
+ if oname == 'max_boards':
+ mach.max_boards = int(ovalue)
+ self.machines.append(mach)
+
+ def probe_all(self, col=None):
+ """Probe all machines in the pool in parallel
+
+ All machines are probed concurrently via threads. Progress is shown
+ on a single line and results are printed afterwards.
+
+ Args:
+ col (terminal.Color or None): Colour object for output
+
+ Returns:
+ list of Machine: Machines that are available
+ """
+ if not col:
+ col = terminal.Color()
+
+ names = [m.name for m in self.machines]
+ done = []
+ lock = threading.Lock()
+
+ def _probe(mach):
+ mach.probe()
+ with lock:
+ done.append(mach.name)
+ tout.progress(f'Probing {len(done)}/{len(names)}: '
+ f'{", ".join(done)}')
+
+ # Probe all machines in parallel
+ threads = []
+ tout.progress(f'Probing {len(names)} machines')
+ for mach in self.machines:
+ t = threading.Thread(target=_probe, args=(mach,))
+ t.start()
+ threads.append(t)
+ for t in threads:
+ t.join()
+ tout.clear_progress()
+ return self.get_available()
+
+ def check_toolchains(self, needed_archs, buildman_path='buildman',
+ fetch=False, col=None, local_gcc=None):
+ """Check and optionally fetch toolchains on available machines
+
+ Probes toolchains on all available machines in parallel. If
+ fetch is True, missing toolchains are fetched sequentially.
+
+ Toolchains whose gcc version (e.g. gcc-13.1.0-nolibc) differs
+ from the local machine are treated as missing and will be
+ re-fetched if fetch is True.
+
+ Args:
+ needed_archs (set of str): Set of architectures needed
+ (e.g. {'arm', 'aarch64', 'sandbox'})
+ buildman_path (str): Path to buildman on remote machines
+ fetch (bool): True to attempt to fetch missing toolchains
+ col (terminal.Color or None): Colour object for output
+ local_gcc (dict or None): arch -> gcc path on the local
+ machine, used for version comparison
+
+ Returns:
+ dict: Machine -> set of missing architectures
+ """
+ if not col:
+ col = terminal.Color()
+
+ reachable = self.get_reachable()
+ if not reachable:
+ return {}
+
+ # Probe toolchains on all reachable machines, not just available
+ # ones, so that busy machines still show toolchain info
+ done = []
+ lock = threading.Lock()
+
+ def _check(mach):
+ mach.probe_toolchains(buildman_path, local_gcc=local_gcc)
+ with lock:
+ done.append(mach.name)
+ tout.progress(f'Checking toolchains {len(done)}/'
+ f'{len(reachable)}: {", ".join(done)}')
+
+ threads = []
+ tout.progress(f'Checking toolchains on {len(reachable)} machines')
+ for mach in reachable:
+ t = threading.Thread(target=_check, args=(mach,))
+ t.start()
+ threads.append(t)
+ for t in threads:
+ t.join()
+ tout.clear_progress()
+
+ local_versions = build_version_map(local_gcc)
+
+ # Check for missing or version-mismatched toolchains
+ missing_map = {}
+ for mach in reachable:
+ missing = needed_archs - set(mach.toolchains.keys())
+ # Also treat version mismatches as missing
+ for arch, path in mach.toolchains.items():
+ local_ver = local_versions.get(arch)
+ if not local_ver:
+ continue
+ remote_ver = gcc_version(path)
+ if remote_ver and remote_ver != local_ver:
+ missing.add(arch)
+ if missing:
+ missing_map[mach] = missing
+
+ if fetch and missing_map:
+ self._fetch_all_missing(missing_map, local_versions,
+ local_gcc, buildman_path)
+
+ return missing_map
+
+ def _fetch_all_missing(self, missing_map, local_versions,
+ local_gcc, buildman_path):
+ """Fetch missing toolchains on all machines in parallel
+
+ For version-mismatched toolchains, removes the old version
+ directory on the remote before fetching, so the new version
+ takes its place.
+
+ Updates missing_map in place, removing architectures that
+ were successfully fetched.
+
+ Args:
+ missing_map (dict): Machine -> set of missing archs
+ local_versions (dict): arch -> version string (e.g.
+ 'gcc-13.1.0-nolibc') from the local machine
+ local_gcc (dict or None): arch -> gcc path on the boss,
+ passed to re-probe after fetching
+ buildman_path (str): Path to buildman on remote
+ """
+ lock = threading.Lock()
+ done = []
+ failed = []
+ total = sum(len(v) for v in missing_map.values())
+
+ def _fetch_one(mach, missing):
+ fetched = set()
+ for arch in list(missing):
+ # Remove old mismatched version before fetching
+ old_ver = gcc_version(mach.toolchains.get(arch, ''))
+ if old_ver and old_ver != local_versions.get(arch):
+ try:
+ _run_ssh(mach.name, [
+ 'rm', '-rf',
+ f'~/.buildman-toolchains/{old_ver}'])
+ except MachineError:
+ pass
+ ok = mach.fetch_toolchain(buildman_path, arch)
+ with lock:
+ done.append(arch)
+ if ok:
+ fetched.add(arch)
+ else:
+ failed.append(f'{mach.name}: {arch}')
+ tout.progress(
+ f'Fetching toolchains {len(done)}/{total}: '
+ f'{mach.name} {arch}')
+ if fetched:
+ mach.probe_toolchains(buildman_path,
+ local_gcc=local_gcc)
+ missing -= fetched
+ if not missing:
+ with lock:
+ del missing_map[mach]
+
+ tout.progress(f'Fetching {total} toolchains on '
+ f'{len(missing_map)} machines')
+ threads = []
+ for mach, missing in list(missing_map.items()):
+ t = threading.Thread(target=_fetch_one, args=(mach, missing))
+ t.start()
+ threads.append(t)
+ for t in threads:
+ t.join()
+ tout.clear_progress()
+
+ # Report failures
+ for msg in failed:
+ print(f' Failed to fetch {msg}')
+
+ # Report remaining version mismatches grouped by machine
+ if missing_map:
+ print('Version mismatches (local vs remote):')
+ for mach, missing in sorted(missing_map.items(),
+ key=lambda x: x[0].name):
+ diffs = []
+ for arch in sorted(missing):
+ local_ver = local_versions.get(arch, '?')
+ diffs.append(f'{arch}({local_ver})')
+ print(f' {mach.name}: {", ".join(diffs)}')
+
+ def get_reachable(self):
+ """Get list of machines that were successfully probed
+
+ This includes machines that are busy or low on resources, as long
+ as they were reachable via SSH.
+
+ Returns:
+ list of Machine: Reachable machines (may not be available)
+ """
+ return [m for m in self.machines
+ if m.avail or m.info.arch]
+
+ def get_available(self):
+ """Get list of machines that are available for building
+
+ Returns:
+ list of Machine: Available machines
+ """
+ return [m for m in self.machines if m.avail]
+
+ def get_total_weight(self):
+ """Get the total weight of all available machines
+
+ Returns:
+ int: Sum of weights of all available machines
+ """
+ return sum(m.weight for m in self.get_available())
+
+ def print_summary(self, col=None, local_archs=None, local_gcc=None):
+ """Print a summary of all machines in the pool
+
+ Args:
+ col (terminal.Color or None): Colour object for output
+ local_archs (set of str or None): Toolchain architectures available
+ on the local host, used to compare remote toolchain status
+ local_gcc (dict or None): arch -> gcc path on local machine,
+ for version comparison
+ """
+ if not col:
+ col = terminal.Color()
+ if not local_archs:
+ local_archs = set()
+ available = self.get_available()
+ total_weight = self.get_total_weight()
+ print(col.build(col.BLUE,
+ f'Machine pool: {len(available)} of {len(self.machines)} '
+ f'machines available, total weight {total_weight}'))
+ print()
+ fmt = ' {:<10} {:>10} {:>7} {:>8} {:>6} {:>7} {:>7} {:>10} {}'
+ print(fmt.format('Name', 'Arch', 'Threads', 'BogoMIPS',
+ 'Load', 'Mem GB', 'Disk TB', 'Toolchains', 'Status'))
+ print(f' {"-" * 88}')
+ for mach in self.machines:
+ if mach.avail:
+ parts = [f'weight {mach.weight}']
+ if mach.max_boards:
+ parts.append(f'max {mach.max_boards}')
+ status_text = ', '.join(parts)
+ status_colour = col.GREEN
+ elif mach.reason == 'not probed':
+ status_text = 'not probed'
+ status_colour = col.YELLOW
+ else:
+ status_text = mach.reason
+ status_colour = col.RED
+ inf = mach.info
+ mem_gb = f'{inf.mem_avail_mb / 1024:.1f}'
+ disk_tb = f'{inf.disk_avail_mb / 1024 / 1024:.1f}'
+ tc_text, tc_colour = _toolchain_status(
+ mach, local_archs, local_gcc)
+
+ # Format the line with plain text for correct alignment,
+ # then apply colour to the toolchain and status fields
+ line = fmt.format(mach.name, inf.arch or '-', inf.threads,
+ f'{inf.bogomips:.0f}', f'{inf.load:.1f}',
+ mem_gb, disk_tb, tc_text, status_text)
+ if tc_colour:
+ line = line.replace(tc_text,
+ col.build(tc_colour, tc_text), 1)
+ line = line.replace(status_text,
+ col.build(status_colour, status_text), 1)
+ print(line)
+
+ local_versions = build_version_map(local_gcc)
+
+ # Print toolchain errors and missing details after the table
+ notes = []
+ for mach in self.machines:
+ err = mach.tc_error
+ if err:
+ notes.append(f' {mach.name}: {err}')
+ elif local_archs and mach.toolchains is not None:
+ missing = local_archs - set(mach.toolchains.keys())
+ if missing:
+ parts = []
+ for arch in sorted(missing):
+ ver = local_versions.get(arch)
+ if ver:
+ parts.append(f'{arch}({ver})')
+ else:
+ parts.append(arch)
+ notes.append(
+ f' {mach.name}: need {", ".join(parts)}')
+ if notes:
+ print()
+ for note in notes:
+ print(note)
+
+
+class Machine:
+ """Represents a remote (or local) build machine
+
+ Attributes:
+ hostname (str): SSH hostname (user@host or just host)
+ name (str): Short display name from config key
+ info (MachineInfo): Probed machine information
+ avail (bool): True if reachable and not too busy
+ reason (str): Reason the machine is unavailable, or ''
+ toolchains (dict): Available toolchain architectures, arch -> gcc path
+ tc_error (str): Error from last toolchain probe, or ''
+ weight (int): Number of build threads to allocate
+ max_boards (int): Max concurrent boards (0 = use nthreads)
+ """
+ def __init__(self, hostname, name=None):
+ self.hostname = hostname
+ self.name = name or hostname
+ self.info = MachineInfo()
+ self.avail = False
+ self.reason = 'not probed'
+ self.toolchains = {}
+ self.tc_error = ''
+ self.weight = 0
+ self.max_boards = 0
+
+ def probe(self, timeout=PROBE_TIMEOUT):
+ """Probe this machine's capabilities over SSH
+
+ Runs a small Python script on the remote machine to collect
+ architecture, CPU count, thread count, load average, available memory
+ and disk space.
+
+ Args:
+ timeout (int): SSH connect timeout in seconds
+
+ Returns:
+ bool: True if the machine was probed successfully
+ """
+ try:
+ result = _run_ssh(self.hostname, ['python3'],
+ timeout=timeout, stdin_data=PROBE_SCRIPT)
+ except MachineError as exc:
+ self.avail = False
+ self.reason = str(exc)
+ return False
+
+ try:
+ info = json.loads(result)
+ except json.JSONDecodeError:
+ self.avail = False
+ self.reason = f'invalid probe response: {result[:100]}'
+ return False
+
+ self.info = MachineInfo(
+ arch=info.get('arch', ''),
+ cpus=info.get('cpus', 0),
+ threads=info.get('threads', 0),
+ bogomips=info.get('bogomips', 0.0),
+ load=info.get('load_1m', 0.0),
+ mem_avail_mb=info.get('mem_avail_mb', 0),
+ disk_avail_mb=info.get('disk_avail_mb', 0),
+ )
+
+ # Check whether the machine is too busy or low on resources
+ self.avail = True
+ self.reason = ''
+ inf = self.info
+ if inf.cpus and inf.load / inf.cpus > LOAD_THRESHOLD:
+ self.avail = False
+ self.reason = (f'busy (load {inf.load:.1f} '
+ f'with {inf.cpus} cpus)')
+ elif inf.disk_avail_mb < MIN_DISK_MB:
+ self.avail = False
+ self.reason = (f'low disk '
+ f'({inf.disk_avail_mb} MB available)')
+ elif inf.mem_avail_mb < MIN_MEM_MB:
+ self.avail = False
+ self.reason = (f'low memory '
+ f'({inf.mem_avail_mb} MB available)')
+
+ if self.avail:
+ self._calc_weight()
+ return True
+
+ def probe_toolchains(self, buildman_path, local_gcc=None):
+ """Probe available toolchains on this machine
+
+ If local_gcc is provided, checks which of the boss's
+ toolchains exist on this machine by testing for the gcc
+ binary under ~/.buildman-toolchains. This avoids depending
+ on the remote machine's .buildman config.
+
+ Falls back to running 'buildman --list-tool-chains' on the
+ remote when local_gcc is not provided (e.g. --machines
+ without a build).
+
+ Args:
+ buildman_path (str): Path to buildman on the remote machine
+ local_gcc (dict or None): arch -> gcc path on the boss
+
+ Returns:
+ dict: Architecture -> gcc path mapping
+ """
+ self.tc_error = ''
+ if local_gcc:
+ return self._probe_toolchains_from_boss(local_gcc)
+ try:
+ result = _run_ssh(self.hostname,
+ [buildman_path, '--list-tool-chains'])
+ except MachineError as exc:
+ self.toolchains = {}
+ self.tc_error = str(exc)
+ return self.toolchains
+
+ self.toolchains = _parse_toolchain_list(result)
+ return self.toolchains
+
+ def _probe_toolchains_from_boss(self, local_gcc):
+ """Check which of the boss's toolchains exist on this machine
+
+ For each architecture, extracts the path relative to the home
+ directory (e.g. .buildman-toolchains/gcc-13.1.0-nolibc/...)
+ and tests whether that gcc binary exists on the remote. This
+ makes the worker mirror the boss's toolchain choices.
+
+ Args:
+ local_gcc (dict): arch -> gcc path on the boss
+
+ Returns:
+ dict: Architecture -> gcc path mapping (using remote paths)
+ """
+ # Build a list of relative paths to check
+ home_prefix = os.path.expanduser('~')
+ checks = {}
+ for arch, gcc in local_gcc.items():
+ if gcc.startswith(home_prefix):
+ rel = gcc[len(home_prefix):]
+ if rel.startswith('/'):
+ rel = rel[1:]
+ checks[arch] = rel
+
+ if not checks:
+ self.toolchains = {}
+ return self.toolchains
+
+ # Build a single SSH command that tests all paths
+ # Output: "arch:yes" or "arch:no" for each
+ test_cmds = []
+ for arch, rel in checks.items():
+ test_cmds.append(
+ f'test -f ~/{rel} && echo {arch}:yes || echo {arch}:no')
+ try:
+ result = _run_ssh(self.hostname,
+ ['; '.join(test_cmds)])
+ except MachineError as exc:
+ self.toolchains = {}
+ self.tc_error = str(exc)
+ return self.toolchains
+
+ self.toolchains = {}
+ for line in result.splitlines():
+ line = line.strip()
+ if ':yes' in line:
+ arch = line.split(':')[0]
+ rel = checks.get(arch)
+ if rel:
+ self.toolchains[arch] = f'~/{rel}'
+ return self.toolchains
+
+ def fetch_toolchain(self, buildman_path, arch):
+ """Fetch a toolchain for a given architecture on this machine
+
+ Args:
+ buildman_path (str): Path to buildman on the remote
+ arch (str): Architecture to fetch (e.g. 'arm')
+
+ Returns:
+ bool: True if the fetch succeeded
+ """
+ try:
+ _run_ssh(self.hostname,
+ [buildman_path, '--fetch-arch', arch])
+ return True
+ except MachineError:
+ return False
+
+ def _calc_weight(self):
+ """Calculate the build weight (threads to allocate) for this machine
+
+ Uses available threads minus a fraction of the current load to avoid
+ over-committing a partially loaded machine.
+ """
+ if not self.avail:
+ self.weight = 0
+ return
+ # Reserve some capacity based on current load
+ spare = max(1, self.info.threads - int(self.info.load))
+ self.weight = spare
+
+ def __repr__(self):
+ inf = self.info
+ status = 'avail' if self.avail else f'unavail: {self.reason}'
+ return (f'Machine({self.hostname}, arch={inf.arch}, '
+ f'threads={inf.threads}, '
+ f'bogomips={inf.bogomips:.0f}, load={inf.load:.1f}, '
+ f'weight={self.weight}, {status})')
@@ -41,6 +41,7 @@ def run_tests(skip_net_tests, debug, verbose, args):
from buildman import test_bsettings
from buildman import test_builder
from buildman import test_cfgutil
+ from buildman import test_machine
test_name = args.terms and args.terms[0] or None
if skip_net_tests:
@@ -63,6 +64,7 @@ def run_tests(skip_net_tests, debug, verbose, args):
test_builder.TestCheckOutputForLoop,
test_builder.TestMake,
test_builder.TestPrintBuildSummary,
+ test_machine,
'buildman.toolchain'])
return (0 if result.wasSuccessful() else 1)
new file mode 100644
@@ -0,0 +1,1046 @@
+# SPDX-License-Identifier: GPL-2.0+
+# Copyright 2026 Simon Glass <sjg@chromium.org>
+
+"""Tests for the machine module"""
+
+# pylint: disable=W0212
+
+import json
+import os
+import unittest
+from unittest import mock
+
+from u_boot_pylib import command
+from u_boot_pylib import terminal
+
+from buildman import bsettings
+from buildman import machine
+
+
+# Base machine-info dict used by probe tests. Individual tests override
+# fields as needed (e.g. load_1m, mem_avail_mb) via {**MACHINE_INFO, ...}.
+MACHINE_INFO = {
+ 'arch': 'x86_64',
+ 'cpus': 4,
+ 'threads': 8,
+ 'bogomips': 5000.0,
+ 'load_1m': 0.5,
+ 'mem_avail_mb': 16000,
+ 'disk_avail_mb': 50000,
+}
+
+
+class TestParseToolchainList(unittest.TestCase):
+ """Test _parse_toolchain_list()"""
+
+ def test_parse_normal(self):
+ """Test parsing normal toolchain list output"""
+ output = '''List of available toolchains (3):
+arm : /usr/bin/arm-linux-gnueabi-gcc
+aarch64 : /usr/bin/aarch64-linux-gnu-gcc
+sandbox : /usr/bin/gcc
+'''
+ result = machine._parse_toolchain_list(output)
+ self.assertEqual(result, {
+ 'arm': '/usr/bin/arm-linux-gnueabi-gcc',
+ 'aarch64': '/usr/bin/aarch64-linux-gnu-gcc',
+ 'sandbox': '/usr/bin/gcc',
+ })
+
+ def test_parse_empty(self):
+ """Test parsing empty output"""
+ self.assertEqual(machine._parse_toolchain_list(''), {})
+
+ def test_parse_none_toolchains(self):
+ """Test parsing when no toolchains are available"""
+ output = '''List of available toolchains (0):
+None
+'''
+ result = machine._parse_toolchain_list(output)
+ self.assertEqual(result, {})
+
+ def test_parse_with_colour(self):
+ """Test parsing output that has extra text before the list"""
+ output = """Some preamble text
+List of available toolchains (2):
+arm : /opt/toolchains/arm-gcc
+x86 : /usr/bin/x86_64-linux-gcc
+
+Some trailing text
+"""
+ result = machine._parse_toolchain_list(output)
+ self.assertEqual(result, {
+ 'arm': '/opt/toolchains/arm-gcc',
+ 'x86': '/usr/bin/x86_64-linux-gcc',
+ })
+
+
+class TestMachine(unittest.TestCase):
+ """Test Machine class"""
+
+ def test_init(self):
+ """Test initial state of a Machine"""
+ m = machine.Machine('myhost')
+ self.assertEqual(m.hostname, 'myhost')
+ self.assertEqual(m.info.arch, '')
+ self.assertFalse(m.avail)
+ self.assertEqual(m.reason, 'not probed')
+ self.assertEqual(m.weight, 0)
+ self.assertEqual(m.toolchains, {})
+
+ @mock.patch('buildman.machine._run_ssh')
+ def test_probe_success(self, mock_ssh):
+ """Test successful probe"""
+ mock_ssh.return_value = json.dumps({
+ **MACHINE_INFO, 'cpus': 8, 'threads': 16, 'load_1m': 1.5})
+ m = machine.Machine('server1')
+ result = m.probe()
+ self.assertTrue(result)
+ self.assertTrue(m.avail)
+ self.assertEqual(m.info.cpus, 8)
+ self.assertEqual(m.info.threads, 16)
+ self.assertAlmostEqual(m.info.load, 1.5)
+ self.assertEqual(m.info.mem_avail_mb, 16000)
+ self.assertEqual(m.info.disk_avail_mb, 50000)
+ self.assertGreater(m.weight, 0)
+
+ @mock.patch('buildman.machine._run_ssh')
+ def test_probe_busy(self, mock_ssh):
+ """Test probe of a busy machine"""
+ mock_ssh.return_value = json.dumps({
+ **MACHINE_INFO, 'load_1m': 5.0})
+ m = machine.Machine('busy-host')
+ result = m.probe()
+ self.assertTrue(result)
+ self.assertFalse(m.avail)
+ self.assertIn('busy', m.reason)
+
+ @mock.patch('buildman.machine._run_ssh')
+ def test_probe_low_disk(self, mock_ssh):
+ """Test probe of a machine with low disk space"""
+ mock_ssh.return_value = json.dumps({
+ **MACHINE_INFO, 'disk_avail_mb': 500})
+ m = machine.Machine('low-disk')
+ result = m.probe()
+ self.assertTrue(result)
+ self.assertFalse(m.avail)
+ self.assertIn('disk', m.reason)
+
+ @mock.patch('buildman.machine._run_ssh')
+ def test_probe_low_mem(self, mock_ssh):
+ """Test probe of a machine with low memory"""
+ mock_ssh.return_value = json.dumps({
+ **MACHINE_INFO, 'mem_avail_mb': 200})
+ m = machine.Machine('low-mem')
+ result = m.probe()
+ self.assertTrue(result)
+ self.assertFalse(m.avail)
+ self.assertIn('memory', m.reason)
+
+ @mock.patch('buildman.machine._run_ssh')
+ def test_probe_ssh_failure(self, mock_ssh):
+ """Test probe when SSH fails"""
+ mock_ssh.side_effect = machine.MachineError('connection refused')
+ m = machine.Machine('bad-host')
+ result = m.probe()
+ self.assertFalse(result)
+ self.assertFalse(m.avail)
+ self.assertIn('connection refused', m.reason)
+
+ @mock.patch('buildman.machine._run_ssh')
+ def test_probe_bad_json(self, mock_ssh):
+ """Test probe when remote returns invalid JSON"""
+ mock_ssh.return_value = 'not json at all'
+ m = machine.Machine('bad-json')
+ result = m.probe()
+ self.assertFalse(result)
+ self.assertFalse(m.avail)
+ self.assertIn('invalid probe response', m.reason)
+
+ @mock.patch('buildman.machine._run_ssh')
+ def test_probe_toolchains(self, mock_ssh):
+ """Test probing toolchains"""
+ mock_ssh.return_value = '''List of available toolchains (2):
+arm : /usr/bin/arm-linux-gnueabi-gcc
+sandbox : /usr/bin/gcc
+
+'''
+ m = machine.Machine('server1')
+ archs = m.probe_toolchains('buildman')
+ self.assertEqual(archs, {
+ 'arm': '/usr/bin/arm-linux-gnueabi-gcc',
+ 'sandbox': '/usr/bin/gcc',
+ })
+
+ @mock.patch('buildman.machine._run_ssh')
+ def test_weight_calculation(self, mock_ssh):
+ """Test weight calculation based on load"""
+ mock_ssh.return_value = json.dumps({
+ **MACHINE_INFO, 'cpus': 8, 'threads': 16, 'load_1m': 4.0})
+ m = machine.Machine('server1')
+ m.probe()
+ # weight = threads - int(load) = 16 - 4 = 12
+ self.assertEqual(m.weight, 12)
+
+ @mock.patch('buildman.machine._run_ssh')
+ def test_weight_minimum(self, mock_ssh):
+ """Test weight is at least 1 when available"""
+ mock_ssh.return_value = json.dumps({
+ **MACHINE_INFO, 'arch': 'aarch64', 'threads': 4,
+ 'bogomips': 48.0, 'load_1m': 3.1})
+ m = machine.Machine('server1')
+ m.probe()
+ # weight = max(1, 4 - 3) = 1
+ self.assertEqual(m.weight, 1)
+
+ def test_repr(self):
+ """Test string representation"""
+ m = machine.Machine('server1')
+ self.assertIn('server1', repr(m))
+ self.assertIn('unavail', repr(m))
+
+
+class TestMachinePool(unittest.TestCase):
+ """Test MachinePool class"""
+
+ def setUp(self):
+ """Set up bsettings for each test"""
+ bsettings.setup(None)
+
+ def test_empty_pool(self):
+ """Test pool with no machines configured"""
+ pool = machine.MachinePool()
+ self.assertEqual(pool.machines, [])
+ self.assertEqual(pool.get_available(), [])
+ self.assertEqual(pool.get_total_weight(), 0)
+
+ def test_load_from_config(self):
+ """Test loading machines from config with bare hostnames"""
+ bsettings.add_file(
+ '[machines]\n'
+ 'ohau\n'
+ 'moa\n'
+ )
+ pool = machine.MachinePool()
+ self.assertEqual(len(pool.machines), 2)
+ self.assertEqual(pool.machines[0].hostname, 'ohau')
+ self.assertEqual(pool.machines[1].hostname, 'moa')
+
+ def test_load_from_config_key_value(self):
+ """Test loading machines from config with key=value pairs"""
+ bsettings.add_file(
+ '[machines]\n'
+ 'server1 = build1.example.com\n'
+ 'server2 = user@build2.example.com\n'
+ )
+ pool = machine.MachinePool()
+ self.assertEqual(len(pool.machines), 2)
+ self.assertEqual(pool.machines[0].hostname, 'build1.example.com')
+ self.assertEqual(pool.machines[1].hostname, 'user@build2.example.com')
+
+ @mock.patch('buildman.machine._run_ssh')
+ def test_probe_all(self, mock_ssh):
+ """Test probing all machines"""
+ mock_ssh.return_value = json.dumps({
+ **MACHINE_INFO, 'load_1m': 1.0,
+ 'mem_avail_mb': 8000, 'disk_avail_mb': 20000})
+ bsettings.add_file(
+ '[machines]\n'
+ 'host1\n'
+ 'host2\n'
+ )
+ pool = machine.MachinePool()
+ available = pool.probe_all()
+ self.assertEqual(len(available), 2)
+ self.assertEqual(pool.get_total_weight(), 14)
+
+ @mock.patch('buildman.machine._run_ssh')
+ def test_probe_mixed(self, mock_ssh):
+ """Test probing with some machines available and some not"""
+ def ssh_side_effect(hostname, _cmd, **_kwargs):
+ if hostname == 'host1':
+ return json.dumps({
+ **MACHINE_INFO, 'load_1m': 1.0,
+ 'mem_avail_mb': 8000, 'disk_avail_mb': 20000})
+ raise machine.MachineError('connection refused')
+
+ mock_ssh.side_effect = ssh_side_effect
+ bsettings.add_file(
+ '[machines]\n'
+ 'host1\n'
+ 'host2\n'
+ )
+ pool = machine.MachinePool()
+ available = pool.probe_all()
+ self.assertEqual(len(available), 1)
+ self.assertEqual(available[0].hostname, 'host1')
+
+ @mock.patch('buildman.machine._run_ssh')
+ def test_check_toolchains(self, mock_ssh):
+ """Test checking toolchains on machines"""
+ def ssh_side_effect(_hostname, cmd, **_kwargs):
+ if 'python3' in cmd:
+ return json.dumps({
+ **MACHINE_INFO, 'load_1m': 1.0,
+ 'mem_avail_mb': 8000, 'disk_avail_mb': 20000})
+ if '--list-tool-chains' in cmd:
+ return '''List of available toolchains (2):
+arm : /usr/bin/arm-gcc
+sandbox : /usr/bin/gcc
+
+'''
+ return ''
+
+ mock_ssh.side_effect = ssh_side_effect
+ bsettings.add_file(
+ '[machines]\n'
+ 'host1\n'
+ )
+ pool = machine.MachinePool()
+ pool.probe_all()
+ missing = pool.check_toolchains({'arm', 'sandbox'})
+ self.assertEqual(missing, {})
+
+ @mock.patch('buildman.machine._run_ssh')
+ def test_check_toolchains_missing(self, mock_ssh):
+ """Test checking toolchains with some missing"""
+ def ssh_side_effect(_hostname, cmd, **_kwargs):
+ if 'python3' in cmd:
+ return json.dumps({
+ **MACHINE_INFO, 'load_1m': 1.0,
+ 'mem_avail_mb': 8000, 'disk_avail_mb': 20000})
+ if '--list-tool-chains' in cmd:
+ return '''List of available toolchains (1):
+sandbox : /usr/bin/gcc
+
+'''
+ return ''
+
+ mock_ssh.side_effect = ssh_side_effect
+ bsettings.add_file(
+ '[machines]\n'
+ 'host1\n'
+ )
+ pool = machine.MachinePool()
+ pool.probe_all()
+ missing = pool.check_toolchains({'arm', 'sandbox'})
+ self.assertEqual(len(missing), 1)
+ m = list(missing.keys())[0]
+ self.assertIn('arm', missing[m])
+
+
+class TestRunSsh(unittest.TestCase):
+ """Test _run_ssh()"""
+
+ @mock.patch('buildman.machine.command.run_pipe')
+ def test_success(self, mock_pipe):
+ """Test successful SSH command"""
+ mock_pipe.return_value = mock.Mock(
+ return_code=0, stdout='hello\n', stderr='')
+ result = machine._run_ssh('host1', ['echo', 'hello'])
+ self.assertEqual(result, 'hello\n')
+
+ # Verify SSH options
+ pipe_list = mock_pipe.call_args[0][0]
+ cmd = pipe_list[0]
+ self.assertIn('ssh', cmd)
+ self.assertIn('BatchMode=yes', cmd)
+ self.assertIn('host1', cmd)
+ self.assertIn('echo', cmd)
+
+ @mock.patch('buildman.machine.command.run_pipe')
+ def test_failure(self, mock_pipe):
+ """Test SSH command failure"""
+ mock_pipe.return_value = mock.Mock(
+ return_code=255, stdout='',
+ stderr='Connection refused')
+ with self.assertRaises(machine.MachineError) as ctx:
+ machine._run_ssh('host1', ['echo', 'hello'])
+ self.assertIn('Connection refused', str(ctx.exception))
+
+ @mock.patch('buildman.machine.command.run_pipe')
+ def test_failure_multiline_stderr(self, mock_pipe):
+ """Test SSH failure with multi-line stderr picks last line"""
+ mock_pipe.return_value = mock.Mock(
+ return_code=255, stdout='',
+ stderr='Warning: Added host key\n'
+ 'Permission denied (publickey).')
+ with self.assertRaises(machine.MachineError) as ctx:
+ machine._run_ssh('host1', ['echo', 'hello'])
+ self.assertIn('Permission denied', str(ctx.exception))
+ self.assertNotIn('Warning', str(ctx.exception))
+
+ @mock.patch('buildman.machine.command.run_pipe')
+ def test_command_exc(self, mock_pipe):
+ """Test SSH command exception"""
+ mock_pipe.side_effect = command.CommandExc(
+ 'ssh failed', command.CommandResult())
+ with self.assertRaises(machine.MachineError) as ctx:
+ machine._run_ssh('host1', ['echo', 'hello'])
+ self.assertIn('ssh failed', str(ctx.exception))
+
+
+class TestGetMachinesConfig(unittest.TestCase):
+ """Test get_machines_config()"""
+
+ def setUp(self):
+ bsettings.setup(None)
+
+ def test_empty(self):
+ """Test with no machines configured"""
+ self.assertEqual(machine.get_machines_config(), [])
+
+ def test_with_machines(self):
+ """Test with machines configured"""
+ bsettings.add_file(
+ '[machines]\n'
+ 'host1\n'
+ 'host2\n'
+ )
+ result = machine.get_machines_config()
+ self.assertEqual(result, ['host1', 'host2'])
+
+
+class TestGccVersion(unittest.TestCase):
+ """Test gcc_version()"""
+
+ def test_normal_path(self):
+ """Test extracting version from a standard toolchain path"""
+ path = ('~/.buildman-toolchains/gcc-13.1.0-nolibc/'
+ 'aarch64-linux/bin/aarch64-linux-gcc')
+ self.assertEqual(machine.gcc_version(path), 'gcc-13.1.0-nolibc')
+
+ def test_no_match(self):
+ """Test path that does not contain a gcc-*-nolibc component"""
+ self.assertIsNone(machine.gcc_version('/usr/bin/gcc'))
+
+ def test_empty(self):
+ """Test empty path"""
+ self.assertIsNone(machine.gcc_version(''))
+
+
+class TestBuildVersionMap(unittest.TestCase):
+ """Test build_version_map()"""
+
+ def test_normal(self):
+ """Test building version map from gcc dict"""
+ gcc = {
+ 'arm': '~/.buildman-toolchains/gcc-13.1.0-nolibc/arm/bin/gcc',
+ 'sandbox': '/usr/bin/gcc',
+ }
+ result = machine.build_version_map(gcc)
+ self.assertEqual(result, {'arm': 'gcc-13.1.0-nolibc'})
+
+ def test_none(self):
+ """Test with None input"""
+ self.assertEqual(machine.build_version_map(None), {})
+
+
+class TestResolveToolchainAliases(unittest.TestCase):
+ """Test resolve_toolchain_aliases()"""
+
+ def setUp(self):
+ bsettings.setup(None)
+
+ def test_alias(self):
+ """Test resolving aliases from config"""
+ bsettings.add_file(
+ '[toolchain-alias]\n'
+ 'x86 = i386 i686\n'
+ )
+ gcc = {'i386': '/usr/bin/i386-gcc'}
+ machine.resolve_toolchain_aliases(gcc)
+ self.assertEqual(gcc['x86'], '/usr/bin/i386-gcc')
+
+ def test_no_alias_needed(self):
+ """Test when arch already exists"""
+ bsettings.add_file(
+ '[toolchain-alias]\n'
+ 'x86 = i386\n'
+ )
+ gcc = {'x86': '/usr/bin/x86-gcc', 'i386': '/usr/bin/i386-gcc'}
+ machine.resolve_toolchain_aliases(gcc)
+ # Should not overwrite existing
+ self.assertEqual(gcc['x86'], '/usr/bin/x86-gcc')
+
+
+class TestToolchainStatus(unittest.TestCase):
+ """Test _toolchain_status()"""
+
+ def test_no_toolchains_no_error(self):
+ """Test machine with no toolchains and no error"""
+ m = machine.Machine('host1')
+ m.avail = True
+ m.info.arch = 'x86_64'
+ text, colour = machine._toolchain_status(m, set())
+ self.assertEqual(text, 'none')
+
+ def test_no_toolchains_with_error(self):
+ """Test machine with toolchain error"""
+ m = machine.Machine('host1')
+ m.tc_error = 'SSH failed'
+ text, colour = machine._toolchain_status(m, set())
+ self.assertEqual(text, 'fail')
+
+ def test_all_present(self):
+ """Test all local toolchains present on machine"""
+ m = machine.Machine('host1')
+ m.toolchains = {'arm': '/usr/bin/arm-gcc',
+ 'sandbox': '/usr/bin/gcc'}
+ local = {'arm', 'sandbox'}
+ text, colour = machine._toolchain_status(m, local)
+ self.assertEqual(text, 'OK')
+
+ def test_some_missing(self):
+ """Test some toolchains missing"""
+ m = machine.Machine('host1')
+ m.toolchains = {'sandbox': '/usr/bin/gcc'}
+ local = {'arm', 'sandbox'}
+ text, colour = machine._toolchain_status(m, local)
+ self.assertIn('missing', text)
+
+ def test_version_mismatch(self):
+ """Test version mismatch detection"""
+ m = machine.Machine('host1')
+ m.toolchains = {
+ 'arm': '~/.buildman-toolchains/gcc-12.0.0-nolibc/arm/bin/gcc'}
+ local_gcc = {
+ 'arm': '~/.buildman-toolchains/gcc-13.1.0-nolibc/arm/bin/gcc'}
+ text, colour = machine._toolchain_status(
+ m, {'arm'}, local_gcc=local_gcc)
+ self.assertIn('wrong ver', text)
+
+ def test_no_local_archs(self):
+ """Test with empty local arch set"""
+ m = machine.Machine('host1')
+ m.toolchains = {'arm': '/usr/bin/gcc', 'x86': '/usr/bin/gcc'}
+ text, colour = machine._toolchain_status(m, set())
+ self.assertEqual(text, '2')
+
+ def test_unreachable_no_toolchains(self):
+ """Test unreachable machine with no arch info"""
+ m = machine.Machine('host1')
+ text, colour = machine._toolchain_status(m, {'arm'})
+ self.assertEqual(text, '-')
+
+
+class TestMachineExtended(unittest.TestCase):
+ """Extended Machine tests for coverage"""
+
+ @mock.patch('buildman.machine._run_ssh')
+ def test_probe_toolchains_ssh_failure(self, mock_ssh):
+ """Test toolchain probe when SSH fails"""
+ mock_ssh.side_effect = machine.MachineError('timeout')
+ m = machine.Machine('host1')
+ result = m.probe_toolchains('buildman')
+ self.assertEqual(result, {})
+ self.assertIn('timeout', m.tc_error)
+
+ @mock.patch('buildman.machine._run_ssh')
+ def test_probe_toolchains_from_boss(self, mock_ssh):
+ """Test probing toolchains by checking boss's paths on remote"""
+ home = os.path.expanduser('~')
+ local_gcc = {
+ 'arm': f'{home}/.buildman-toolchains/gcc-13/arm/bin/gcc',
+ 'x86': f'{home}/.buildman-toolchains/gcc-13/x86/bin/gcc',
+ }
+ mock_ssh.return_value = 'arm:yes\nx86:no\n'
+ m = machine.Machine('host1')
+ result = m.probe_toolchains('buildman', local_gcc=local_gcc)
+ self.assertIn('arm', result)
+ self.assertNotIn('x86', result)
+
+ @mock.patch('buildman.machine._run_ssh')
+ def test_probe_toolchains_from_boss_ssh_fail(self, mock_ssh):
+ """Test probing boss toolchains when SSH fails"""
+ home = os.path.expanduser('~')
+ local_gcc = {
+ 'arm': f'{home}/.buildman-toolchains/gcc-13/arm/bin/gcc',
+ }
+ mock_ssh.side_effect = machine.MachineError('conn refused')
+ m = machine.Machine('host1')
+ result = m.probe_toolchains('buildman', local_gcc=local_gcc)
+ self.assertEqual(result, {})
+ self.assertIn('conn refused', m.tc_error)
+
+ @mock.patch('buildman.machine._run_ssh')
+ def test_probe_toolchains_from_boss_no_home(self, mock_ssh):
+ """Test probing boss toolchains with non-home paths"""
+ local_gcc = {'sandbox': '/usr/bin/gcc'}
+ m = machine.Machine('host1')
+ result = m.probe_toolchains('buildman', local_gcc=local_gcc)
+ self.assertEqual(result, {})
+ mock_ssh.assert_not_called()
+
+ @mock.patch('buildman.machine._run_ssh')
+ def test_fetch_toolchain_success(self, mock_ssh):
+ """Test successful toolchain fetch"""
+ mock_ssh.return_value = 'Fetched arm toolchain'
+ m = machine.Machine('host1')
+ self.assertTrue(m.fetch_toolchain('buildman', 'arm'))
+
+ @mock.patch('buildman.machine._run_ssh')
+ def test_fetch_toolchain_failure(self, mock_ssh):
+ """Test failed toolchain fetch"""
+ mock_ssh.side_effect = machine.MachineError('fetch failed')
+ m = machine.Machine('host1')
+ self.assertFalse(m.fetch_toolchain('buildman', 'arm'))
+
+ @mock.patch('buildman.machine._run_ssh')
+ def test_weight_unavailable(self, mock_ssh):
+ """Test weight is 0 when unavailable"""
+ m = machine.Machine('host1')
+ m.avail = False
+ m._calc_weight()
+ self.assertEqual(m.weight, 0)
+
+
+class TestRunSshExtended(unittest.TestCase):
+ """Extended _run_ssh tests"""
+
+ @mock.patch('buildman.machine.command.run_pipe')
+ def test_failure_no_stderr(self, mock_pipe):
+ """Test SSH failure with no stderr"""
+ mock_pipe.return_value = mock.Mock(
+ return_code=1, stdout='', stderr='')
+ with self.assertRaises(machine.MachineError) as ctx:
+ machine._run_ssh('host1', ['cmd'])
+ self.assertIn('failed with code 1', str(ctx.exception))
+
+ @mock.patch('buildman.machine.command.run_pipe')
+ def test_stdin_data(self, mock_pipe):
+ """Test SSH with stdin_data"""
+ mock_pipe.return_value = mock.Mock(
+ return_code=0, stdout='result\n', stderr='')
+ result = machine._run_ssh('host1', ['python3'],
+ stdin_data='print("result")')
+ self.assertEqual(result, 'result\n')
+ # Verify stdin_data was passed through
+ _, kwargs = mock_pipe.call_args
+ self.assertEqual(kwargs['stdin_data'], 'print("result")')
+
+
+class TestMachinePoolExtended(unittest.TestCase):
+ """Extended MachinePool tests for coverage"""
+
+ def setUp(self):
+ bsettings.setup(None)
+
+ def test_load_with_max_boards(self):
+ """Test loading machines with max_boards config"""
+ bsettings.add_file(
+ '[machines]\n'
+ 'server1\n'
+ '[machine:server1]\n'
+ 'max_boards = 50\n'
+ )
+ pool = machine.MachinePool()
+ self.assertEqual(len(pool.machines), 1)
+ self.assertEqual(pool.machines[0].max_boards, 50)
+
+ def test_load_filtered_names(self):
+ """Test loading only specified machine names"""
+ bsettings.add_file(
+ '[machines]\n'
+ 'host1\n'
+ 'host2\n'
+ 'host3\n'
+ )
+ pool = machine.MachinePool(names=['host1', 'host3'])
+ self.assertEqual(len(pool.machines), 2)
+ names = [m.hostname for m in pool.machines]
+ self.assertEqual(names, ['host1', 'host3'])
+
+ def test_get_reachable(self):
+ """Test get_reachable includes busy machines"""
+ bsettings.add_file('[machines]\nhost1\nhost2\n')
+ pool = machine.MachinePool()
+ # Simulate host1 reachable but busy, host2 unreachable
+ pool.machines[0].avail = False
+ pool.machines[0].info.arch = 'x86_64'
+ self.assertEqual(len(pool.get_reachable()), 1)
+ self.assertEqual(pool.get_reachable()[0].hostname, 'host1')
+
+ @mock.patch('buildman.machine._run_ssh')
+ def test_print_summary(self, mock_ssh):
+ """Test print_summary runs without error"""
+ mock_ssh.return_value = json.dumps({
+ **MACHINE_INFO, 'load_1m': 1.0,
+ 'mem_avail_mb': 8000, 'disk_avail_mb': 20000})
+ bsettings.add_file('[machines]\nhost1\n')
+ pool = machine.MachinePool()
+ pool.probe_all()
+ # Just verify it doesn't crash
+ with terminal.capture():
+ pool.print_summary()
+
+ @mock.patch('buildman.machine._run_ssh')
+ def test_print_summary_with_toolchains(self, mock_ssh):
+ """Test print_summary with toolchain info"""
+ def ssh_side_effect(_hostname, cmd, **_kwargs):
+ if 'python3' in cmd:
+ return json.dumps({
+ **MACHINE_INFO, 'load_1m': 1.0,
+ 'mem_avail_mb': 8000, 'disk_avail_mb': 20000})
+ if '--list-tool-chains' in cmd:
+ return 'List of available toolchains (1):\narm : /bin/gcc\n\n'
+ return ''
+
+ mock_ssh.side_effect = ssh_side_effect
+ bsettings.add_file('[machines]\nhost1\n')
+ pool = machine.MachinePool()
+ pool.probe_all()
+ pool.check_toolchains({'arm', 'sandbox'})
+ with terminal.capture():
+ pool.print_summary(local_archs={'arm', 'sandbox'})
+
+ @mock.patch('buildman.machine._run_ssh')
+ def test_check_toolchains_version_mismatch(self, mock_ssh):
+ """Test version mismatch detection in check_toolchains"""
+ home = os.path.expanduser('~')
+
+ # The remote has gcc-12, but the boss has gcc-13
+ remote_gcc = (f'.buildman-toolchains/gcc-12.0.0-nolibc/'
+ 'arm/bin/arm-linux-gcc')
+
+ def ssh_side_effect(_hostname, cmd, **_kwargs):
+ if 'python3' in cmd:
+ return json.dumps({
+ **MACHINE_INFO, 'load_1m': 1.0,
+ 'mem_avail_mb': 8000, 'disk_avail_mb': 20000})
+ # Boss probe: report arm as present (wrong version)
+ return 'arm:yes\n'
+
+ mock_ssh.side_effect = ssh_side_effect
+ bsettings.add_file('[machines]\nhost1\n')
+ pool = machine.MachinePool()
+ pool.probe_all()
+
+ local_gcc = {
+ 'arm': f'{home}/.buildman-toolchains/gcc-13.1.0-nolibc/'
+ 'arm/bin/arm-linux-gcc',
+ }
+ # check_toolchains will re-probe, which sets the remote path
+ # via _probe_toolchains_from_boss. The SSH mock returns
+ # arm:yes, so the remote path uses the boss's relative path
+ # but we need it to have the old version. Patch the machine's
+ # toolchains after the probe runs.
+ orig_probe = machine.Machine._probe_toolchains_from_boss
+
+ def fake_probe(self_mach, lg):
+ orig_probe(self_mach, lg)
+ # Override with the wrong version
+ if 'arm' in self_mach.toolchains:
+ self_mach.toolchains['arm'] = f'~/{remote_gcc}'
+ return self_mach.toolchains
+
+ with mock.patch.object(machine.Machine,
+ '_probe_toolchains_from_boss',
+ fake_probe):
+ missing = pool.check_toolchains({'arm'}, local_gcc=local_gcc)
+
+ # arm should be flagged as missing due to version mismatch
+ self.assertEqual(len(missing), 1)
+
+
+class TestFetchMissing(unittest.TestCase):
+ """Test _fetch_all_missing()"""
+
+ def setUp(self):
+ bsettings.setup(None)
+
+ @mock.patch('buildman.machine._run_ssh')
+ def test_fetch_success(self, mock_ssh):
+ """Test successful toolchain fetch"""
+ mock_ssh.return_value = ''
+ bsettings.add_file('[machines]\nhost1\n')
+ pool = machine.MachinePool()
+ m = pool.machines[0]
+ m.avail = True
+ m.info.arch = 'x86_64'
+
+ missing_map = {m: {'arm'}}
+ with terminal.capture():
+ pool._fetch_all_missing(missing_map, {}, None, 'buildman')
+ # After successful fetch + re-probe, arch is removed
+ # (re-probe returns empty since SSH mock returns '')
+
+ @mock.patch('buildman.machine._run_ssh')
+ def test_fetch_failure(self, mock_ssh):
+ """Test failed toolchain fetch"""
+ mock_ssh.side_effect = machine.MachineError('failed')
+ bsettings.add_file('[machines]\nhost1\n')
+ pool = machine.MachinePool()
+ m = pool.machines[0]
+ m.avail = True
+ m.info.arch = 'x86_64'
+
+ missing_map = {m: {'arm'}}
+ with terminal.capture():
+ pool._fetch_all_missing(missing_map, {}, None, 'buildman')
+ # Should still have the missing arch
+ self.assertIn('arm', missing_map[m])
+
+ @mock.patch('buildman.machine._run_ssh')
+ def test_fetch_with_version_removal(self, mock_ssh):
+ """Test fetching removes old version first"""
+ calls = []
+
+ def ssh_side_effect(hostname, cmd, **_kwargs):
+ calls.append(cmd)
+ if '--fetch-arch' in cmd:
+ return ''
+ if 'rm' in cmd:
+ return ''
+ return ''
+
+ mock_ssh.side_effect = ssh_side_effect
+ bsettings.add_file('[machines]\nhost1\n')
+ pool = machine.MachinePool()
+ m = pool.machines[0]
+ m.avail = True
+ m.info.arch = 'x86_64'
+ m.toolchains = {
+ 'arm': '~/.buildman-toolchains/gcc-12.0.0-nolibc/arm/bin/gcc'}
+
+ missing_map = {m: {'arm'}}
+ local_versions = {'arm': 'gcc-13.1.0-nolibc'}
+ with terminal.capture():
+ pool._fetch_all_missing(missing_map, local_versions, None,
+ 'buildman')
+ # Should have called rm -rf for the old version
+ rm_calls = [c for c in calls if 'rm' in c]
+ self.assertTrue(len(rm_calls) > 0)
+
+
+class TestPrintSummaryEdgeCases(unittest.TestCase):
+ """Test print_summary edge cases"""
+
+ def setUp(self):
+ bsettings.setup(None)
+
+ @mock.patch('buildman.machine._run_ssh')
+ def test_unavailable_machine(self, mock_ssh):
+ """Test summary with unavailable machine"""
+ mock_ssh.side_effect = machine.MachineError('refused')
+ bsettings.add_file('[machines]\nhost1\n')
+ pool = machine.MachinePool()
+ pool.probe_all()
+ # Should not crash with unavailable machine
+ with terminal.capture():
+ pool.print_summary()
+
+ @mock.patch('buildman.machine._run_ssh')
+ def test_busy_machine(self, mock_ssh):
+ """Test summary with busy machine"""
+ mock_ssh.return_value = json.dumps({
+ **MACHINE_INFO, 'load_1m': 10.0})
+ bsettings.add_file('[machines]\nhost1\n')
+ pool = machine.MachinePool()
+ pool.probe_all()
+ with terminal.capture():
+ pool.print_summary()
+
+ @mock.patch('buildman.machine._run_ssh')
+ def test_with_max_boards(self, mock_ssh):
+ """Test summary shows max_boards"""
+ mock_ssh.return_value = json.dumps({
+ **MACHINE_INFO, 'load_1m': 1.0,
+ 'mem_avail_mb': 8000, 'disk_avail_mb': 20000})
+ bsettings.add_file(
+ '[machines]\nhost1\n'
+ '[machine:host1]\nmax_boards = 50\n')
+ pool = machine.MachinePool()
+ pool.probe_all()
+ with terminal.capture():
+ pool.print_summary()
+
+ @mock.patch('buildman.machine._run_ssh')
+ def test_with_tc_error(self, mock_ssh):
+ """Test summary with toolchain error note"""
+ mock_ssh.return_value = json.dumps({
+ **MACHINE_INFO, 'load_1m': 1.0,
+ 'mem_avail_mb': 8000, 'disk_avail_mb': 20000})
+ bsettings.add_file('[machines]\nhost1\n')
+ pool = machine.MachinePool()
+ pool.probe_all()
+ pool.machines[0].tc_error = 'buildman not found'
+ with terminal.capture():
+ pool.print_summary(local_archs={'arm'})
+
+ @mock.patch('buildman.machine._run_ssh')
+ def test_with_missing_toolchains(self, mock_ssh):
+ """Test summary with missing toolchain notes"""
+ mock_ssh.return_value = json.dumps({
+ **MACHINE_INFO, 'load_1m': 1.0,
+ 'mem_avail_mb': 8000, 'disk_avail_mb': 20000})
+ bsettings.add_file('[machines]\nhost1\n')
+ pool = machine.MachinePool()
+ pool.probe_all()
+ pool.machines[0].toolchains = {'sandbox': '/usr/bin/gcc'}
+ local_gcc = {
+ 'arm': os.path.expanduser(
+ '~/.buildman-toolchains/gcc-13/arm/bin/gcc'),
+ }
+ with terminal.capture():
+ pool.print_summary(local_archs={'arm', 'sandbox'},
+ local_gcc=local_gcc)
+
+
+class TestCheckToolchainsEdge(unittest.TestCase):
+ """Test check_toolchains edge cases"""
+
+ def setUp(self):
+ bsettings.setup(None)
+
+ def test_no_reachable(self):
+ """Test check_toolchains with no reachable machines (line 482)"""
+ bsettings.add_file('[machines]\nhost1\n')
+ pool = machine.MachinePool()
+ # Machine is not probed, so not reachable
+ result = pool.check_toolchains({'arm'})
+ self.assertEqual(result, {})
+
+ @mock.patch('buildman.machine._run_ssh')
+ def test_fetch_flag(self, mock_ssh):
+ """Test check_toolchains with fetch=True (line 524)"""
+ home = os.path.expanduser('~')
+
+ def ssh_side_effect(_hostname, cmd, **_kwargs):
+ if 'python3' in cmd:
+ return json.dumps({
+ **MACHINE_INFO, 'load_1m': 1.0,
+ 'mem_avail_mb': 8000, 'disk_avail_mb': 20000})
+ # No toolchains found
+ return ''
+
+ mock_ssh.side_effect = ssh_side_effect
+ bsettings.add_file('[machines]\nhost1\n')
+ pool = machine.MachinePool()
+ pool.probe_all()
+
+ local_gcc = {
+ 'arm': f'{home}/.buildman-toolchains/gcc-13/arm/bin/gcc',
+ }
+ # fetch=True should trigger _fetch_all_missing
+ with mock.patch.object(pool, '_fetch_all_missing') as mock_fetch:
+ pool.check_toolchains({'arm'}, fetch=True,
+ local_gcc=local_gcc)
+ mock_fetch.assert_called_once()
+
+
+class TestFetchVersionRemovalFailure(unittest.TestCase):
+ """Test _fetch_all_missing rm -rf failure path (lines 563-564)"""
+
+ def setUp(self):
+ bsettings.setup(None)
+
+ @mock.patch('buildman.machine._run_ssh')
+ def test_rm_failure_continues(self, mock_ssh):
+ """Test that rm -rf failure is silently ignored"""
+ call_count = [0]
+
+ def ssh_side_effect(hostname, cmd, **_kwargs):
+ call_count[0] += 1
+ if 'rm' in cmd:
+ raise machine.MachineError('rm failed')
+ if '--fetch-arch' in cmd:
+ return ''
+ return ''
+
+ mock_ssh.side_effect = ssh_side_effect
+ bsettings.add_file('[machines]\nhost1\n')
+ pool = machine.MachinePool()
+ m = pool.machines[0]
+ m.avail = True
+ m.info.arch = 'x86_64'
+ # Old version that differs from local
+ m.toolchains = {
+ 'arm': '~/.buildman-toolchains/gcc-12.0.0-nolibc/arm/bin/gcc'}
+
+ missing_map = {m: {'arm'}}
+ local_versions = {'arm': 'gcc-13.1.0-nolibc'}
+ with terminal.capture():
+ pool._fetch_all_missing(missing_map, local_versions, None,
+ 'buildman')
+ # Should have attempted rm and then fetch despite rm failure
+ self.assertGreater(call_count[0], 1)
+
+
+class TestPrintSummaryNotProbed(unittest.TestCase):
+ """Test print_summary 'not probed' branch (lines 669-670)"""
+
+ def setUp(self):
+ bsettings.setup(None)
+
+ def test_not_probed_machine(self):
+ """Test summary shows 'not probed' for unprobed machine"""
+ bsettings.add_file('[machines]\nhost1\n')
+ pool = machine.MachinePool()
+ # Don't probe - machine stays in 'not probed' state
+ with terminal.capture():
+ pool.print_summary()
+
+
+class TestPrintSummaryMissingNoVersion(unittest.TestCase):
+ """Test print_summary missing toolchain without version (line 707)"""
+
+ def setUp(self):
+ bsettings.setup(None)
+
+ @mock.patch('buildman.machine._run_ssh')
+ def test_missing_with_and_without_version(self, mock_ssh):
+ """Test missing note for archs with and without version info"""
+ home = os.path.expanduser('~')
+ mock_ssh.return_value = json.dumps({
+ **MACHINE_INFO, 'load_1m': 1.0,
+ 'mem_avail_mb': 8000, 'disk_avail_mb': 20000})
+ bsettings.add_file('[machines]\nhost1\n')
+ pool = machine.MachinePool()
+ pool.probe_all()
+ pool.machines[0].toolchains = {}
+ # arm has a version (under ~/.buildman-toolchains),
+ # sandbox does not
+ local_gcc = {
+ 'arm': f'{home}/.buildman-toolchains/gcc-13.1.0-nolibc/'
+ 'arm/bin/gcc',
+ 'sandbox': '/usr/bin/gcc',
+ }
+ with terminal.capture():
+ pool.print_summary(local_archs={'arm', 'sandbox'},
+ local_gcc=local_gcc)
+
+
+class TestDoProbe(unittest.TestCase):
+ """Test do_probe_machines()"""
+
+ def setUp(self):
+ bsettings.setup(None)
+
+ def test_no_machines(self):
+ """Test with no machines configured"""
+ with terminal.capture():
+ ret = machine.do_probe_machines()
+ self.assertEqual(ret, 1)
+
+ @mock.patch('buildman.machine._run_ssh')
+ def test_with_machines(self, mock_ssh):
+ """Test probing configured machines"""
+ def ssh_side_effect(_hostname, cmd, **_kwargs):
+ if 'python3' in cmd:
+ return json.dumps({
+ **MACHINE_INFO, 'load_1m': 1.0,
+ 'mem_avail_mb': 8000, 'disk_avail_mb': 20000})
+ if '--list-tool-chains' in cmd:
+ return 'List of available toolchains (0):\n\n'
+ return ''
+
+ mock_ssh.side_effect = ssh_side_effect
+ bsettings.add_file('[machines]\nhost1\n')
+ with terminal.capture():
+ ret = machine.do_probe_machines()
+ self.assertEqual(ret, 0)
+
+
+if __name__ == '__main__':
+ unittest.main()