Snap for 6224475 from b95080ba9c72ea228d90023aa646cb4df25cf84c to rvc-release
Change-Id: I991a58bc99d73b5c9d439c50cb7105926a2adc5a
diff --git a/abi/kmi_defines.py b/abi/kmi_defines.py
new file mode 100755
index 0000000..a1e45a7
--- /dev/null
+++ b/abi/kmi_defines.py
@@ -0,0 +1,772 @@
+#!/usr/bin/env python3
+"""kmi_defines extract #define compile time constants from a Linux build.
+
+The kmi_defines tool is used to examine the output of a Linux build
+and extract from it C #define statements that define compile time
+constant expressions for the purpose of tracking them as part of the
+KMI (Kernel Module Interface) so that changes to their values can be
+prevented so as to ensure a constant KMI for kernel modules for the
+AOSP GKI Linux kernel project.
+
+This code is python3 only, it does not require any from __future__
+imports. This is a standalone program, it is not meant to be used as
+a module by other programs.
+
+This program runs under the multiprocessing module. Work done within
+a multiprocessing.Pool does not perform error logging or affects any
+state other than the value that it computes and returns via the function
+mapped through the pool's map() function. The reason that no external
+state is affected (for example error loggiing) is to avoid to have to
+even think about what concurrent updates would cause to shuch a facility.
+"""
+
+# TODO(pantin): per Matthias review feedback: "drop the .py from the
+# filename after(!) the review has completed. As last action. Until
+# then we can have the syntax highlighting here in Gerrit."
+
+import argparse
+import collections
+import logging
+import multiprocessing
+import os
+import pathlib
+import re
+import subprocess
+import sys
+from typing import List, Optional, Tuple
+from typing import Set # pytype needs this, pylint: disable=unused-import
+
+INDENT = 4 # number of spaces to indent for each depth level
+COMPILER = "clang" # TODO(pantin): should be determined at run-time
+
+# Dependency that is hidden by the transformation of the .o.d file into
+# the .o.cmd file as part of the Linux build environment. This header is
+# purposely removed and replaced by fictitious set of empty header files
+# that were never part of the actual compilation of the .o files. Those
+# fictitious empty files are generated under the build environment output
+# directory in this subdirectory:
+# include/config
+#
+# This is the actual header file that was part of the compilation of every
+# .o file, the HIDDEN_DEP are added to the dependencies of every .o file.
+#
+# It is important that this file be added because it is unknowable whether
+# the #defines in it were depended upon by a module to alter its behaviour
+# at compile time. For example to pass some flags or not pass some flags
+# to a function.
+
+HIDDEN_DEP = "include/generated/autoconf.h"
+
+
+class StopError(Exception):
+ """Exception raised to stop work when an unexpected error occurs."""
+
+
+def dump(this) -> None:
+ """Dump the data in this.
+
+ This is for debugging purposes, it does not handle every type, only
+ the types used by the underlying code are handled. This will not be
+ part of the final code, or if it is, it will be significantly enhanced
+ or replaced by some other introspection mechanism to serialize data.
+ """
+ def dump_this(this, name: str, depth: int) -> None:
+ """Dump the data in this."""
+ if name:
+ name += " = "
+ if isinstance(this, str):
+ indent = " " * (depth * INDENT)
+ print(indent + name + this)
+ elif isinstance(this, bool):
+ indent = " " * (depth * INDENT)
+ print(indent + name + str(this))
+ elif isinstance(this, List):
+ dump_list(this, name, depth)
+ elif isinstance(this, Set):
+ dump_set(this, name, depth)
+ else:
+ dump_object(this, name, depth)
+
+ def dump_list(lst: List[str], name: str, depth: int) -> None:
+ """Dump the data in lst."""
+ indent = " " * (depth * INDENT)
+ print(indent + name + "{")
+ index = 0
+ for entry in lst:
+ dump_this(entry, f"[{index}]", depth + 1)
+ index += 1
+ print(indent + "}")
+
+ def dump_set(aset: Set[str], name: str, depth: int) -> None:
+ """Dump the data in aset."""
+ lst = list(aset)
+ lst.sort()
+ dump_list(lst, name, depth)
+
+ def dump_object(this, name: str, depth: int) -> None:
+ """Dump the data in this."""
+ indent = " " * (depth * INDENT)
+ print(indent + name +
+ re.sub(r"(^<class '__main__\.|'>$)", "", str(type(this))) + " {")
+ for key, val in this.__dict__.items():
+ dump_this(val, key, depth + 1)
+ print(indent + "}")
+
+ dump_this(this, "", 0)
+
+
+def readfile(name: str) -> str:
+ """Open a file and return its contents in a string as its value."""
+ try:
+ with open(name) as file:
+ return file.read()
+ except OSError as os_error:
+ raise StopError("readfile() failed for: " + name + "\n"
+ "original OSError: " + str(os_error.args))
+
+
+def file_must_exist(file: str) -> None:
+ """If file is invalid print raise a StopError."""
+ if not os.path.exists(file):
+ raise StopError("file does not exist: " + file)
+ if not os.path.isfile(file):
+ raise StopError("file is not a regular file: " + file)
+
+
+def makefile_depends_get_dependencies(depends: str) -> List[str]:
+ """Return list with the dependencies of a makefile target.
+
+ Split the makefile depends specification, the name of the dependent is
+ followed by ":" its dependencies follow the ":". There could be spaces
+ around the ":". Line continuation characters, i.e. "\" are consumed by
+ the regular expression that splits the specification.
+
+ This results in a list with the dependent first, and its dependencies
+ in the remainder of the list, return everything in the list other than
+ the first element.
+ """
+ return re.split(r"[:\s\\]+", re.sub(r"[\s\\]*\Z", "", depends))[1:]
+
+
+def makefile_assignment_split(assignment: str) -> Tuple[str, str]:
+ """Split left:=right into a tuple with the left and right parts.
+
+ Spaces around the := are also removed.
+ """
+ result = re.split(r"\s*:=\s*", assignment, maxsplit=1)
+ if len(result) != 2:
+ raise StopError(
+ "expected: 'left<optional_spaces>:=<optional_spaces>right' in: " +
+ assignment)
+ return result[0], result[1] # left, right
+
+
+def get_src_ccline_deps(obj: str) -> Optional[Tuple[str, str, List[str]]]:
+ """Get the C source file, its cc_line, and non C source dependencies.
+
+ If the tool used to produce the object is not the compiler, or if the
+ source file is not a C source file None is returned.
+
+ Otherwise it returns a triplet with the C source file name, its cc_line,
+ the remaining dependencies.
+ """
+ o_cmd = os.path.join(os.path.dirname(obj),
+ "." + os.path.basename(obj) + ".cmd")
+
+ contents = readfile(o_cmd)
+ contents = re.sub(r"\$\(wildcard[^)]*\)", " ", contents)
+ contents = re.sub(r"[ \t]*\\\n[ \t]*", " ", contents)
+ lines = lines_to_list(contents)
+
+ cc_line = None
+ deps = None
+ source = None
+ for line in lines:
+ if line.startswith("cmd_"):
+ cc_line = line
+ elif line.startswith("deps_"):
+ deps = line
+ elif line.startswith("source_"):
+ source = line
+
+ if cc_line is None:
+ raise StopError("missing cmd_* variable in: " + o_cmd)
+ _, cc_line = makefile_assignment_split(cc_line)
+ if cc_line.split(maxsplit=1)[0] != COMPILER:
+ # The object file was made by strip, symbol renames, etc.
+ # i.e. it was not the result of running the compiler, thus
+ # it can not contribute to #define compile time constants.
+ return None
+
+ if source is None:
+ raise StopError("missing source_* variable in: " + o_cmd)
+ _, source = makefile_assignment_split(source)
+ source = source.strip()
+ if not source.endswith(".c"):
+ return None
+
+ if deps is None:
+ raise StopError("missing deps_* variable in: " + o_cmd)
+ _, deps = makefile_assignment_split(deps)
+ dependendencies = deps.split()
+ dependendencies.append(HIDDEN_DEP)
+
+ return source, cc_line, dependendencies
+
+
+def lines_to_list(lines: str) -> List[str]:
+ """Split a string into a list of non-empty lines."""
+ return [line for line in lines.strip().splitlines() if line]
+
+
+def lines_get_first_line(lines: str) -> str:
+ """Return the first non-empty line in lines."""
+ return lines.strip().splitlines()[0]
+
+
+def shell_line_to_o_files_list(line: str) -> List[str]:
+ """Return a list of .o files in the files list."""
+ return [entry for entry in line.split() if entry.endswith(".o")]
+
+
+def run(args: List[str],
+ raise_on_failure: bool = True) -> subprocess.CompletedProcess:
+ """Run the program specified in args[0] with the arguments in args[1:]."""
+ try:
+ # This argument does not always work for subprocess.run() below:
+ # check=False
+ # neither that nor:
+ # check=True
+ # prevents an exception from being raised if the program that
+ # will be executed is not found
+
+ completion = subprocess.run(args, capture_output=True, text=True)
+ if completion.returncode != 0 and raise_on_failure:
+ raise StopError("execution failed for: " + " ".join(args))
+ return completion
+ except OSError as os_error:
+ raise StopError("failure executing: " + " ".join(args) + "\n"
+ "original OSError: " + str(os_error.args))
+
+
+class KernelModule:
+ """A kernel module, i.e. a *.ko file."""
+ def __init__(self, kofile: str) -> None:
+ """Construct a KernelModule object."""
+ # An example argument is used below, assuming kofile is:
+ # possibly/empty/dirs/modname.ko
+ #
+ # Meant to refer to this module, shown here relative to the top of
+ # the build directory:
+ # drivers/usb/gadget/udc/modname.ko
+ # the values assigned to the members are shown in the comments below.
+
+ self._file = os.path.realpath(kofile) # /abs/dirs/modname.ko
+ self._base = os.path.basename(self._file) # modname.ko
+ self._directory = os.path.dirname(self._file) # /abs/dirs
+ self._cmd_file = os.path.join(self._directory,
+ "." + self._base + ".cmd")
+ self._cmd_text = readfile(self._cmd_file)
+
+ # Some builds append a '; true' to the .modname.ko.cmd, remove it
+
+ self._cmd_text = re.sub(r";\s*true\s*$", "", self._cmd_text)
+
+ # The modules .modname.ko.cmd file contains a makefile snippet,
+ # for example:
+ # cmd_drivers/usb/gadget/udc/dummy_hcd.ko := ld.lld -r ...
+ #
+ # Split the string prior to the spaces followed by ":=", and get
+ # the first element of the resulting list. If the string was not
+ # split (because it did not contain a ":=" then the input string
+ # is returned, by the re.sub() below, as the only element of the list.
+
+ left, _ = makefile_assignment_split(self._cmd_text)
+ self._rel_file = re.sub(r"^cmd_", "", left)
+ if self._rel_file == left:
+ raise StopError("expected: 'cmd_' at start of content of: " +
+ self._cmd_file)
+
+ base = os.path.basename(self._rel_file)
+ if base != self._base:
+ raise StopError("module name mismatch: " + base + " vs " +
+ self._base)
+
+ self._rel_dir = os.path.dirname(self._rel_file)
+
+ # The final step in the build of kernel modules is based on two .o
+ # files, one with the module name followed by .o and another followed
+ # by .mod.o
+ #
+ # The following test verifies that assumption, in case a module is
+ # built differently in the future.
+ #
+ # Even when there are multiple source files, the .o files that result
+ # from compiling them are all linked into a single .o file through an
+ # intermediate link step, that .o files is named:
+ # os.path.join(self._rel_dir, kofile_name + ".o")
+
+ kofile_name, _ = os.path.splitext(self._base)
+ objs = shell_line_to_o_files_list(self._cmd_text)
+ objs.sort()
+ expected = [ # sorted, i.e.: .mod.o < .o
+ os.path.join(self._rel_dir, kofile_name + ".mod.o"),
+ os.path.join(self._rel_dir, kofile_name + ".o")
+ ]
+ if objs != expected:
+ raise StopError("unexpected .o files in: " + self._cmd_file)
+
+ def get_build_dir(self) -> str:
+ """Return the top level build directory.
+
+ I.e. the directory where the output of the Linux build is stored.
+
+ Note that this, like pretty much all the code, can raise an exception,
+ by construction, if an exception is raised while an object is being
+ constructed, or after it is constructed, the object will not be used
+ thereafter (at least not any object explicitly created by this
+ program). Many other places, for example the ones that call readfile()
+ can raise exceptions, the code is located where it belongs.
+
+ In this specific case, the computation of index, and the derived
+ invariant that it be >= 0, is predicated by the condition checked
+ below, if the exception is not raised, then index is >= 0.
+ """
+ if not self._file.endswith(self._rel_file):
+ raise StopError("could not find: " + self._rel_file +
+ " at end of: " + self._file)
+ index = len(self._file) - len(self._rel_file)
+ if index > 0 and self._file[index - 1] == os.sep:
+ index -= 1
+ build_dir = self._file[0:index]
+ return build_dir
+
+ def get_object_files(self, build_dir: str) -> List[str]:
+ """Return a list object files that used to link the kernel module.
+
+ The ocmd_file is the file with extension ".o.cmd" (see below).
+ If the ocmd_file has a more than one line in it, its because the
+ module is made of a single source file and the ocmd_file has the
+ compilation rule and dependencies to build it. If it has a single
+ line single line it is because it builds the .o file by linking
+ multiple .o files.
+ """
+
+ kofile_name, _ = os.path.splitext(self._base)
+ ocmd_file = os.path.join(build_dir, self._rel_dir,
+ "." + kofile_name + ".o.cmd")
+ ocmd_content = readfile(ocmd_file)
+
+ olines = lines_to_list(ocmd_content)
+ if len(olines) > 1: # module made from a single .o file
+ return [os.path.join(build_dir, self._rel_dir, kofile_name + ".o")]
+
+ # Multiple .o files in the module
+
+ _, ldline = makefile_assignment_split(olines[0])
+ return [
+ os.path.realpath(os.path.join(build_dir, obj))
+ for obj in shell_line_to_o_files_list(ldline)
+ ]
+
+
+class Kernel:
+ """The Linux kernel component itself, i.e. vmlinux.o."""
+ def __init__(self, kernel: str) -> None:
+ """Construct a Kernel object."""
+ self._kernel = os.path.realpath(kernel)
+ self._build_dir = os.path.dirname(self._kernel)
+ libs = os.path.join(self._build_dir, "vmlinux.libs")
+ objs = os.path.join(self._build_dir, "vmlinux.objs")
+ file_must_exist(libs)
+ file_must_exist(objs)
+ contents = readfile(libs)
+ archives_and_objects = contents.split()
+ contents = readfile(objs)
+ archives_and_objects += contents.split()
+ self._archives_and_objects = [(os.path.join(self._build_dir, file)
+ if not os.path.isabs(file) else file)
+ for file in archives_and_objects]
+
+ def get_build_dir(self) -> str:
+ """Return the top level build directory.
+
+ I.e. the directory where the output of the Linux build is stored.
+ """
+ return self._build_dir
+
+ def get_object_files(self, build_dir: str) -> List[str]:
+ """Return a list object files that where used to link the kernel."""
+ olist = []
+ for file in self._archives_and_objects:
+ if file.endswith(".o"):
+ if not os.path.isabs(file):
+ file = os.path.join(build_dir, file)
+ olist.append(os.path.realpath(file))
+ continue
+
+ if not file.endswith(".a"):
+ raise StopError("unknown file type: " + file)
+
+ completion = run(["ar", "t", file])
+ objs = lines_to_list(completion.stdout)
+
+ for obj in objs:
+ if not os.path.isabs(obj):
+ obj = os.path.join(build_dir, obj)
+ olist.append(os.path.realpath(obj))
+
+ return olist
+
+
+class Target: # pylint: disable=too-few-public-methods
+ """Target of build and the information used to build it."""
+
+ # The compiler invocation has this form:
+ # clang -Wp,-MD,file.o.d ... -c -o file.o file.c
+ # these constants reflect that knowledge in the code, e.g.:
+ # - the "-Wp,_MD,file.o.d" is at WP_MD_FLAG_INDEX
+ # - the "-c" is at index C_FLAG_INDEX
+ # - the "-o" is at index O_FLAG_INDEX
+ # - the "file.o" is at index OBJ_INDEX
+ # - the "file.c" is at index SRC_INDEX
+ #
+ # There must be at least MIN_CC_LIST_LEN options in that command line.
+ # This knowledge is verified at run time in __init__(), see comments
+ # there.
+
+ MIN_CC_LIST_LEN = 6
+ WP_MD_FLAG_INDEX = 1
+ C_FLAG_INDEX = -4
+ O_FLAG_INDEX = -3
+ OBJ_INDEX = -2
+ SRC_INDEX = -1
+
+ def __init__(self, obj: str, src: str, cc_line: str,
+ deps: List[str]) -> None:
+ self._obj = obj
+ self._src = src
+ self._deps = deps
+
+ # The cc_line, eventually slightly modified, will be used to run
+ # the compiler in various ways. The cc_line could be fed through
+ # the shell to deal with the single-quotes in the cc_line that are
+ # there to quote the double-quotes meant to be part of a C string
+ # literal. Specifically, this occurs in to pass KBUILD_MODNAME and
+ # KBUILD_BASENAME, for example:
+ # -DKBUILD_MODNAME='"aes_ce_cipher"'
+ # -DKBUILD_BASENAME='"aes_cipher_glue"'
+ #
+ # Causing an extra execve(2) of the shell, just to deal with a few
+ # quotes is wasteful, so instead, here the quotes, in this specific
+ # case are removed. This can be done, easiest just by removing the
+ # single quotes with:
+ # cc_cmd = re.sub(r"'", "", cc_line)
+ #
+ # But this could mess up other quote usage in the future, for example
+ # using double quotes or backslash to quote a single quote meant to
+ # actually be seen by the compiler.
+ #
+ # As an alternative, and for this to be more robust, the specific
+ # cases that are known, i.e. the two -D shown above, are dealt with
+ # individually and if there are any single or double quotes, or
+ # backslashes the underlying work is stopped.
+ #
+ # Note that the cc_line comes from the .foo.o.cmd file which is a
+ # makefile snippet, so the actual syntax there is also subject to
+ # whatever other things make would want to do with them. Instead
+ # of doing the absolutely correct thing, which would actually be
+ # to run this through make to have make run then through the shell
+ # this program already has knowledge about these .cmd files and how
+ # they are formed. This compromise, or coupling of knowledge, is a
+ # source of fragility, but not expected to cause much trouble in the
+ # future as the Linux build evolves.
+
+ cc_cmd = re.sub(
+ r"""-D(KBUILD_BASENAME|KBUILD_MODNAME)='("[a-zA-Z0-9_.:]*")'""",
+ r"-D\1=\2", cc_line)
+ cc_list = cc_cmd.split()
+
+ # TODO(pantin): the handling of -D... arguments above is done better
+ # in a later commit by using shlex.split(). Please ignore for now.
+ # TODO(pantin): possibly use ArgumentParser to make this more robust.
+
+ # The compiler invocation has this form:
+ # clang -Wp,-MD,file.o.d ... -c -o file.o file.c
+ #
+ # The following checks are here to ensure that if this assumption is
+ # broken, failures occur. The indexes *_INDEX are hardcoded, they
+ # could in principle be determined at run time, the -o argument could
+ # be in a future update to the Linux build could changed to be a
+ # single argument with the object file name (as in: -ofile.o) which
+ # could also be detected in code at a later time.
+
+ if (len(cc_list) < Target.MIN_CC_LIST_LEN
+ or not cc_list[Target.WP_MD_FLAG_INDEX].startswith("-Wp,-MD,")
+ or cc_list[Target.C_FLAG_INDEX] != "-c"
+ or cc_list[Target.O_FLAG_INDEX] != "-o"):
+ raise StopError("unexpected or missing arguments for: " + obj +
+ " cc_line: " + cc_line)
+
+ # Instead of blindly normalizing the source and object arguments,
+ # they are only normalized if that allows the expected invariants
+ # to be verified, otherwise they are left undisturbed. Note that
+ # os.path.normpath() does not turn relative paths into absolute
+ # paths, it just removes up-down walks (e.g. a/b/../c -> a/c).
+
+ def verify_file(file: str, index: int, kind: str, cc_list: List[str],
+ target_file: str) -> None:
+ file_in_cc_list = cc_list[index]
+ if not file.endswith(file_in_cc_list):
+ file_normalized = os.path.normpath(file_in_cc_list)
+ if not file.endswith(file_normalized):
+ raise StopError(f"unexpected {kind} argument for: "
+ f"{target_file} value was: "
+ f"{file_in_cc_list}")
+ cc_list[index] = file_normalized
+
+ verify_file(obj, Target.OBJ_INDEX, "object", cc_list, obj)
+ verify_file(src, Target.SRC_INDEX, "source", cc_list, obj)
+
+ self._cc_list = cc_list
+
+
+class KernelComponentBase: # pylint: disable=too-few-public-methods
+ """Base class for KernelComponentCreationError and KernelComponent.
+
+ There is not much purpose for this class other than to satisfy the strong
+ typing checks of pytype, with looser typing, this could be removed but at
+ the risk of invoking member functions at run-time on objects that do not
+ provide them. Having this class makes the code more reliable.
+ """
+ def get_error(self) -> Optional[str]: # pylint: disable=no-self-use
+ """Return None for the error, means there was no error."""
+ return None
+
+ def get_deps_set(self) -> Set[str]: # pylint: disable=no-self-use
+ """Return the set of dependencies for the kernel component."""
+ return set()
+
+ def is_kernel(self) -> bool: # pylint: disable=no-self-use
+ """Is this the kernel?"""
+ return False
+
+
+class KernelComponentCreationError(KernelComponentBase): # pylint: disable=too-few-public-methods
+ """A KernelComponent creation error.
+
+ When a KernelComponent creation fails, or the creation of its subordinate
+ Kernel or KernelModule creation fails, a KernelComponentCreationError
+ object is created to store the information relevant to the failure.
+ """
+ def __init__(self, filename: str, error: str) -> None:
+ """Construct a KernelComponentCreationError object."""
+ self._error = error
+ self._filename = filename
+
+ def get_error(self) -> Optional[str]:
+ """Return the error."""
+ return self._filename + ": " + self._error
+
+
+class KernelComponent(KernelComponentBase):
+ """A kernel component, either vmlinux.o or a *.ko file.
+
+ Inspect a Linux kernel module (a *.ko file) or the Linux kernel to
+ determine what was used to build it: object filess, source files, header
+ files, and other information that is produced as a by-product of its build.
+ """
+ def __init__(self, filename: str) -> None:
+ """Construct a KernelComponent object."""
+ if filename.endswith("vmlinux.o"):
+ self._kernel = True
+ self._kind = Kernel(filename)
+ else:
+ self._kernel = False
+ self._kind = KernelModule(filename)
+ self._build_dir = self._kind.get_build_dir()
+ self._source_dir = self._get_source_dir()
+ self._files_o = self._kind.get_object_files(self._build_dir)
+ self._files_o.sort()
+
+ # using a set because there is no unique flag to list.sort()
+ deps_set = set()
+
+ self._targets = []
+ for obj in self._files_o:
+ file_must_exist(obj)
+ result = get_src_ccline_deps(obj)
+ if result is None:
+ continue
+ src, cc_line, dependendencies = result
+
+ file_must_exist(src)
+ depends = []
+ for dep in dependendencies:
+ if not os.path.isabs(dep):
+ dep = os.path.join(self._build_dir, dep)
+ dep = os.path.realpath(dep)
+ depends.append(dep)
+ deps_set.add(dep)
+
+ if not os.path.isabs(src):
+ src = os.path.join(self._build_dir, src)
+ src = os.path.realpath(src)
+ self._targets.append(Target(obj, src, cc_line, depends))
+
+ for dep in [dep for dep in list(deps_set) if not dep.endswith(".h")]:
+ deps_set.remove(dep)
+ self._deps_set = deps_set
+
+ def _get_source_dir(self) -> str:
+ """Return the top level Linux kernel source directory."""
+ source = os.path.join(self._build_dir, "source")
+ if not os.path.islink(source):
+ raise StopError("could not find source symlink: " + source)
+
+ if not os.path.isdir(source):
+ raise StopError("source symlink not a directory: " + source)
+
+ source_dir = os.path.realpath(source)
+ if not os.path.isdir(source_dir):
+ raise StopError("source directory not a directory: " + source_dir)
+
+ return source_dir
+
+ def get_deps_set(self) -> Set[str]:
+ """Return the set of dependencies for the kernel component."""
+ return self._deps_set
+
+ def is_kernel(self) -> bool:
+ """Is this the kernel?"""
+ return self._kernel
+
+
+def kernel_component_factory(filename: str) -> KernelComponentBase:
+ """Make an InfoKmod or an InfoKernel object for file and return it."""
+ try:
+ return KernelComponent(filename)
+ except StopError as stop_error:
+ return KernelComponentCreationError(filename,
+ " ".join([*stop_error.args]))
+
+
+class KernelComponentProcess(multiprocessing.Process):
+ """Process to make the KernelComponent concurrently."""
+ def __init__(self) -> None:
+ multiprocessing.Process.__init__(self)
+ self._queue = multiprocessing.Queue()
+ self.start()
+
+ def run(self) -> None:
+ """Create and save the KernelComponent."""
+ self._queue.put(kernel_component_factory("vmlinux.o"))
+
+ def get_component(self) -> KernelComponentBase:
+ """Return the kernel component."""
+ kernel_component = self._queue.get()
+ self.join() # must be after queue.get() otherwise it deadlocks
+ return kernel_component
+
+
+def work_on_all_components(options) -> List[KernelComponentBase]:
+ """Return a list of KernelComponentBase objects."""
+ files = [str(ko) for ko in pathlib.Path().rglob("*.ko")]
+ if options.sequential:
+ return [
+ kernel_component_factory(file) for file in ["vmlinux.o"] + files
+ ]
+
+ # There is significantly more work to be done for the vmlinux.o than
+ # the *.ko kernel modules. A dedicated process is started to do the
+ # work for vmlinux.o as soon as possible instead of leaving it to the
+ # vagaries of multiprocessing.Pool() and how it would spreads the work.
+ # This significantly reduces the elapsed time for this work.
+
+ kernel_component_process = KernelComponentProcess()
+
+ chunk_size = 128
+ processes = max(1, len(files) // (chunk_size * 3))
+ processes = min(processes, os.cpu_count())
+ with multiprocessing.Pool(processes) as pool:
+ components = pool.map(kernel_component_factory, files, chunk_size)
+
+ kernel_component = kernel_component_process.get_component()
+
+ return [kernel_component] + components
+
+
+def work_on_whole_build(options) -> int:
+ """Work on the whole build to extract the #define constants."""
+ if not os.path.isfile("vmlinux.o"):
+ logging.error("file not found: vmlinux.o")
+ return 1
+ components = work_on_all_components(options)
+ failed = False
+ header_count = collections.defaultdict(int)
+ for comp in components:
+ error = comp.get_error()
+ if error:
+ logging.error(error)
+ failed = True
+ continue
+ deps_set = comp.get_deps_set()
+ for header in deps_set:
+ header_count[header] += 1
+ if failed:
+ return 1
+ if options.dump:
+ dump(components)
+ if options.dump and options.includes:
+ print()
+ if options.includes:
+ for header, count in header_count.items():
+ if count >= 2:
+ print(header)
+ return 0
+
+
+def main() -> int:
+ """Extract #define compile time constants from a Linux build."""
+ def existing_file(file):
+ if not os.path.isfile(file):
+ raise argparse.ArgumentTypeError(
+ "{0} is not a valid file".format(file))
+ return file
+
+ parser = argparse.ArgumentParser()
+ parser.add_argument("-d",
+ "--dump",
+ action="store_true",
+ help="dump internal state")
+ parser.add_argument("-s",
+ "--sequential",
+ action="store_true",
+ help="execute without concurrency")
+ group = parser.add_mutually_exclusive_group()
+ group.add_argument("-i",
+ "--includes",
+ action="store_true",
+ help="show relevant include files")
+ group.add_argument("-c",
+ "--component",
+ type=existing_file,
+ help="show information for a component")
+ options = parser.parse_args()
+
+ if not options.component:
+ return work_on_whole_build(options)
+
+ comp = kernel_component_factory(options.component)
+
+ error = comp.get_error()
+ if error:
+ logging.error(error)
+ return 1
+ if options.dump:
+ dump([comp])
+ return 0
+
+
+if __name__ == "__main__":
+ sys.exit(main())
