@@ -24,14 +24,43 @@ You may need to install 'python3-asteval' for the 'asteval' module.
How does it work?
-----------------
-When building a database (`-b`), this tool runs configuration and builds
-include/autoconf.mk for every defconfig. The config options defined in Kconfig
-appear in the .config file (unless they are hidden because of unmet dependency.)
-On the other hand, the config options defined by board headers are seen
-in include/autoconf.mk.
+Building a database
+~~~~~~~~~~~~~~~~~~~
+
+When building a database (`-b`), this tool evaluates the Kconfig tree directly
+using kconfiglib (a pure-Python Kconfig implementation). For each defconfig, it
+loads the file with ``kconf.load_config()``, resolves all symbol dependencies,
+and collects the resulting CONFIG values. This is the same approach used by
+``buildman`` when scanning board parameters.
+
+Multiple worker processes run in parallel (one per CPU core by default,
+adjustable with ``-j``), each parsing the Kconfig tree once and then processing
+its share of defconfigs. This completes in a few seconds for all ~1500 boards,
+since no ``make`` subprocesses or cross-compiler toolchains are needed.
+
+Defconfig files containing ``#include`` directives are preprocessed with the
+C preprocessor before loading, matching the behaviour of the build system.
+
+There are two known cosmetic differences compared with the old make-based
+approach:
+
+- ``CONFIG_GCC_VERSION`` reflects the host compiler rather than each board's
+ cross-compiler, since no cross-compiler is invoked.
+
+- Backslash escape sequences in string values (e.g. ``\n``, ``\x1b``) may
+ differ slightly due to kconfiglib's unescape/escape round-trip. This affects
+ a handful of string CONFIGs such as ``CONFIG_AUTOBOOT_PROMPT``.
+
+Neither difference affects the usefulness of the database for finding CONFIG
+combinations or computing imply relationships.
+
+Resyncing defconfigs
+~~~~~~~~~~~~~~~~~~~~
When resyncing defconfigs (`-s`) the .config is synced by "make savedefconfig"
-and the defconfig is updated with it.
+and the defconfig is updated with it. This path still uses ``make``
+subprocesses and therefore requires appropriate cross-compiler toolchains (see
+below).
For faster processing, this tool is multi-threaded. It creates
separate build directories where the out-of-tree build is run. The
@@ -44,9 +73,12 @@ Note that `*.config` fragments are not supported.
Toolchains
----------
-Appropriate toolchains are necessary to generate include/autoconf.mk
-for all the architectures supported by U-Boot. Most of them are available
-at the kernel.org site. This tool uses the same tools as
+Toolchains are **not** needed for building the database (`-b`), since it
+uses kconfiglib to evaluate Kconfig files directly in Python.
+
+For resyncing defconfigs (`-s`), appropriate toolchains are necessary to run
+``make savedefconfig`` for all the architectures supported by U-Boot. Most of
+them are available at the kernel.org site. This tool uses the same tools as
:doc:`../build/buildman`, so you can use `buildman --fetch-arch` to fetch
toolchains.
@@ -283,6 +283,129 @@ def scan_kconfig():
return kconfiglib.Kconfig()
+def _scan_defconfigs_worker(srcdir, defconfigs, queue, error_queue):
+ """Worker process that scans defconfigs using kconfiglib
+
+ Each worker creates its own Kconfig instance (parsing is done once per
+ process) then loads each defconfig in turn, collecting all CONFIG values.
+
+ Args:
+ srcdir (str): Source-tree directory
+ defconfigs (list of str): Defconfig filenames to process, e.g.
+ ['sandbox_defconfig', 'snow_defconfig']
+ queue (multiprocessing.Queue): Output queue for (defconfig, configs)
+ error_queue (multiprocessing.Queue): Output queue for failed defconfigs
+ """
+ os.environ['srctree'] = srcdir
+ os.environ['UBOOTVERSION'] = 'dummy'
+ os.environ['KCONFIG_OBJDIR'] = ''
+ os.environ['CC'] = 'gcc'
+ kconf = kconfiglib.Kconfig(warn=False)
+
+ for defconfig in defconfigs:
+ fname = os.path.join(srcdir, 'configs', defconfig)
+ try:
+ if b'#include' in tools.read_file(fname):
+ cpp = os.getenv('CPP', 'cpp').split()
+ cmd = cpp + ['-nostdinc', '-P', '-I', srcdir,
+ '-undef', '-x', 'assembler-with-cpp', fname]
+ stdout = subprocess.check_output(cmd, stderr=subprocess.DEVNULL)
+ tmp = tempfile.NamedTemporaryFile(prefix='qconfig-', delete=False)
+ tmp.write(stdout)
+ tmp.close()
+ kconf.load_config(tmp.name)
+ os.unlink(tmp.name)
+ else:
+ kconf.load_config(fname)
+
+ configs = {}
+ for sym in kconf.unique_defined_syms:
+ conf = sym.config_string
+ if not conf or conf.startswith('#'):
+ continue
+ config, value = conf.rstrip('\n').split('=', 1)
+ configs[config] = value
+ queue.put((defconfig, configs))
+ except Exception as exc:
+ error_queue.put((defconfig, str(exc)))
+
+
+def do_build_db(args):
+ """Build the CONFIG database using kconfiglib instead of make
+
+ This evaluates the Kconfig tree directly in Python for each defconfig,
+ avoiding the overhead of spawning make subprocesses and the need for
+ cross-compiler toolchains.
+
+ Args:
+ args (Namespace): Program arguments (uses jobs, defconfigs,
+ defconfiglist, nocolour)
+
+ Returns:
+ tuple:
+ config_db (dict): configs for each defconfig
+ Progress: progress indicator
+ """
+ srcdir = os.getcwd()
+
+ if args.defconfigs:
+ defconfigs = [os.path.basename(d)
+ for d in get_matched_defconfigs(args.defconfigs)]
+ elif args.defconfiglist:
+ defconfigs = [os.path.basename(d)
+ for d in get_matched_defconfigs(args.defconfiglist)]
+ else:
+ defconfigs = get_all_defconfigs()
+
+ col = terminal.Color(terminal.COLOR_NEVER if args.nocolour
+ else terminal.COLOR_IF_TERMINAL)
+ progress = Progress(col, len(defconfigs))
+
+ jobs = args.jobs
+ total = len(defconfigs)
+ result_queue = multiprocessing.Queue()
+ error_queue = multiprocessing.Queue()
+ processes = []
+ for i in range(jobs):
+ chunk = defconfigs[total * i // jobs:total * (i + 1) // jobs]
+ if not chunk:
+ continue
+ proc = multiprocessing.Process(
+ target=_scan_defconfigs_worker,
+ args=(srcdir, chunk, result_queue, error_queue))
+ proc.start()
+ processes.append(proc)
+
+ config_db = {}
+ remaining = total
+ while remaining:
+ # Drain both queues without blocking forever
+ found = False
+ while not result_queue.empty():
+ defconfig, configs = result_queue.get()
+ config_db[defconfig] = configs
+ progress.inc(True)
+ progress.show()
+ remaining -= 1
+ found = True
+ while not error_queue.empty():
+ defconfig, msg = error_queue.get()
+ print(col.build(col.RED, f'{defconfig}: {msg}', bright=True),
+ file=sys.stderr)
+ progress.inc(False)
+ progress.show()
+ remaining -= 1
+ found = True
+ if not found:
+ time.sleep(SLEEP_TIME)
+
+ for proc in processes:
+ proc.join()
+
+ progress.completed()
+ return config_db, progress
+
+
# pylint: disable=R0903
class KconfigParser:
"""A parser of .config and include/autoconf.mk."""
@@ -1745,7 +1868,7 @@ def ensure_database(threads):
dry_run=False, exit_on_error=False, jobs=threads,
git_ref=None, defconfigs=None, defconfiglist=None,
nocolour=False)
- config_db, progress = move_config(args)
+ config_db, progress = do_build_db(args)
write_db(config_db, progress)
@@ -1772,13 +1895,14 @@ def main():
if args.find:
return do_find_config(args.configs, args.list)
+ if args.build_db:
+ config_db, progress = do_build_db(args)
+ return write_db(config_db, progress)
+
config_db, progress = move_config(args)
if args.commit:
add_commit(args.configs)
-
- if args.build_db:
- return write_db(config_db, progress)
return move_done(progress)