RESTRICT AUTOMERGE Use the Kati stamp to include makefile dependencies. am: dba92766b7 am: ac43c7d853
Change-Id: I3e4f67789cf715caf4be62863c4ce5f09cccb1fb
diff --git a/split/manifest_split.py b/split/manifest_split.py
index 3e4698e..89f8dc5 100644
--- a/split/manifest_split.py
+++ b/split/manifest_split.py
@@ -39,11 +39,18 @@
--ninja-build <path>
Optional path to the combined-<target>.ninja file found in an out dir.
If not provided, the default file is used based on the lunch environment.
+ --ninja-binary <path>
+ Optional path to the ninja binary. Uses the standard binary by default.
--module-info <path>
Optional path to the module-info.json file found in an out dir.
If not provided, the default file is used based on the lunch environment.
- --ninja-binary <path>
- Optional path to the ninja binary. Uses the standard binary by default.
+ --kati-stamp <path>
+ Optional path to the .kati_stamp file found in an out dir.
+ If not provided, the default file is used based on the lunch environment.
+ --overlay <path>
+ Optional path(s) to treat as overlays when parsing the kati stamp file
+ and scanning for makefiles. See the tools/treble/build/sandbox directory
+ for more info about overlays. This flag can be passed more than once.
--debug
Print debug messages.
-h (--help)
@@ -172,6 +179,83 @@
return {path.strip() for path in inputs}
+def get_kati_makefiles(kati_stamp_file, overlays):
+ """Returns the set of makefile paths from the kati stamp file.
+
+ Uses the ckati_stamp_dump prebuilt binary.
+ Also includes symlink sources in the resulting set for any
+ makefiles that are symlinks.
+
+ Args:
+ kati_stamp_file: The path to a .kati_stamp file from a build.
+ overlays: A list of paths to treat as overlays when parsing the kati stamp
+ file.
+ """
+ # Get a set of all makefiles that were parsed by Kati during the build.
+ makefiles = set(
+ subprocess.check_output([
+ "prebuilts/build-tools/linux-x86/bin/ckati_stamp_dump",
+ "--files",
+ kati_stamp_file,
+ ]).decode().strip("\n").split("\n"))
+
+ def is_product_makefile(makefile):
+ """Returns True if the makefile path meets certain criteria."""
+ banned_prefixes = [
+ "out/",
+ # Ignore product makefiles for sample AOSP boards.
+ "device/amlogic",
+ "device/generic",
+ "device/google",
+ "device/linaro",
+ "device/sample",
+ ]
+ banned_suffixes = [
+ # All Android.mk files in the source are always parsed by Kati,
+ # so including them here would bring in lots of unnecessary projects.
+ "Android.mk",
+ # The ckati stamp file always includes a line for the ckati bin at
+ # the beginnning.
+ "bin/ckati",
+ ]
+ return (all([not makefile.startswith(p) for p in banned_prefixes]) and
+ all([not makefile.endswith(s) for s in banned_suffixes]))
+
+ # Limit the makefiles to only product makefiles.
+ product_makefiles = {
+ os.path.normpath(path) for path in makefiles if is_product_makefile(path)
+ }
+
+ def strip_overlay(makefile):
+ """Remove any overlays from a makefile path."""
+ for overlay in overlays:
+ if makefile.startswith(overlay):
+ return makefile[len(overlay):]
+ return makefile
+
+ makefiles_and_symlinks = set()
+ for makefile in product_makefiles:
+ # Search for the makefile, possibly scanning overlays as well.
+ for overlay in [""] + overlays:
+ makefile_with_overlay = os.path.join(overlay, makefile)
+ if os.path.exists(makefile_with_overlay):
+ makefile = makefile_with_overlay
+ break
+
+ if not os.path.exists(makefile):
+ logger.warning("Unknown kati makefile: %s" % makefile)
+ continue
+
+ # Ensure the project that contains the makefile is included, as well as
+ # the project that any makefile symlinks point to.
+ makefiles_and_symlinks.add(strip_overlay(makefile))
+ if os.path.islink(makefile):
+ makefiles_and_symlinks.add(
+ strip_overlay(os.path.relpath(os.path.realpath(makefile))))
+
+ return makefiles_and_symlinks
+
+
def scan_repo_projects(repo_projects, input_path):
"""Returns the project path of the given input path if it exists.
@@ -253,7 +337,8 @@
def create_split_manifest(targets, manifest_file, split_manifest_file,
config_files, repo_list_file, ninja_build_file,
- module_info_file, ninja_binary):
+ ninja_binary, module_info_file, kati_stamp_file,
+ overlays):
"""Creates and writes a split manifest by inspecting build inputs.
Args:
@@ -266,8 +351,11 @@
repo_list_file: Path to the output of the 'repo list' command.
ninja_build_file: Path to the combined-<target>.ninja file found in an out
dir.
- module_info_file: Path to the module-info.json file found in an out dir.
ninja_binary: Path to the ninja binary.
+ module_info_file: Path to the module-info.json file found in an out dir.
+ kati_stamp_file: The path to a .kati_stamp file from a build.
+ overlays: A list of paths to treat as overlays when parsing the kati stamp
+ file.
"""
remove_projects = set()
add_projects = set()
@@ -287,6 +375,15 @@
logger.info("%s projects needed for targets \"%s\"", len(input_projects),
" ".join(targets))
+ kati_makefiles = get_kati_makefiles(kati_stamp_file, overlays)
+ kati_makefiles_projects = get_input_projects(repo_projects, kati_makefiles)
+ if logger.isEnabledFor(logging.DEBUG):
+ for project in sorted(kati_makefiles_projects.difference(input_projects)):
+ logger.debug("Kati makefile dependency: %s", project)
+ input_projects = input_projects.union(kati_makefiles_projects)
+ logger.info("%s projects after including Kati makefiles projects.",
+ len(input_projects))
+
if logger.isEnabledFor(logging.DEBUG):
manual_projects = add_projects.difference(input_projects)
for project in sorted(manual_projects):
@@ -295,6 +392,11 @@
logger.info("%s projects after including manual additions.",
len(input_projects))
+ # Remove projects from our set of input projects before adding adjacent
+ # modules, so that no project is added only because of an adjacent
+ # dependency in a to-be-removed project.
+ input_projects = input_projects.difference(remove_projects)
+
# While we still have projects whose modules we haven't checked yet,
checked_projects = set()
projects_to_check = input_projects.difference(checked_projects)
@@ -339,8 +441,10 @@
"config=",
"repo-list=",
"ninja-build=",
- "module-info=",
"ninja-binary=",
+ "module-info=",
+ "kati-stamp=",
+ "overlay=",
])
except getopt.GetoptError as err:
print(__doc__, file=sys.stderr)
@@ -354,6 +458,8 @@
ninja_build_file = None
module_info_file = None
ninja_binary = "ninja"
+ kati_stamp_file = None
+ overlays = []
for o, a in opts:
if o in ("-h", "--help"):
@@ -371,10 +477,14 @@
repo_list_file = a
elif o in ("--ninja-build"):
ninja_build_file = a
- elif o in ("--module-info"):
- module_info_file = a
elif o in ("--ninja-binary"):
ninja_binary = a
+ elif o in ("--module-info"):
+ module_info_file = a
+ elif o in ("--kati-stamp"):
+ kati_stamp_file = a
+ elif o in ("--overlay"):
+ overlays.append(a)
else:
assert False, "unknown option \"%s\"" % o
@@ -393,6 +503,10 @@
if not module_info_file:
module_info_file = os.path.join(os.environ["ANDROID_PRODUCT_OUT"],
"module-info.json")
+ if not kati_stamp_file:
+ kati_stamp_file = os.path.join(
+ os.environ["ANDROID_BUILD_TOP"], "out",
+ ".kati_stamp-%s" % os.environ["TARGET_PRODUCT"])
if not ninja_build_file:
ninja_build_file = os.path.join(
os.environ["ANDROID_BUILD_TOP"], "out",
@@ -405,8 +519,10 @@
config_files=config_files,
repo_list_file=repo_list_file,
ninja_build_file=ninja_build_file,
+ ninja_binary=ninja_binary,
module_info_file=module_info_file,
- ninja_binary=ninja_binary)
+ kati_stamp_file=kati_stamp_file,
+ overlays=overlays)
if __name__ == "__main__":
diff --git a/split/manifest_split_test.py b/split/manifest_split_test.py
index 11b1607..2fdb9bb 100644
--- a/split/manifest_split_test.py
+++ b/split/manifest_split_test.py
@@ -15,6 +15,7 @@
import hashlib
import mock
+import os
import subprocess
import tempfile
import unittest
@@ -85,6 +86,52 @@
'Unknown module path for module target1'):
manifest_split.get_module_info(module_info_file.name, repo_projects)
+ @mock.patch.object(subprocess, 'check_output', autospec=True)
+ def test_get_kati_makefiles(self, mock_check_output):
+ with tempfile.TemporaryDirectory() as temp_dir:
+ os.chdir(temp_dir)
+
+ makefiles = [
+ 'device/oem1/product1.mk',
+ 'device/oem2/product2.mk',
+ 'device/google/google_product.mk',
+ 'overlays/oem_overlay/device/oem3/product3.mk',
+ 'packages/apps/Camera/Android.mk',
+ ]
+ for makefile in makefiles:
+ os.makedirs(os.path.dirname(makefile))
+ os.mknod(makefile)
+
+ symlink_src = os.path.join(temp_dir, 'vendor/oem4/symlink_src.mk')
+ os.makedirs(os.path.dirname(symlink_src))
+ os.mknod(symlink_src)
+ symlink_dest = 'device/oem4/symlink_dest.mk'
+ os.makedirs(os.path.dirname(symlink_dest))
+ os.symlink(symlink_src, symlink_dest)
+ # Only append the symlink destination, not where the symlink points to.
+ # (The Kati stamp file does not resolve symlink sources.)
+ makefiles.append(symlink_dest)
+
+ # Mock the output of ckati_stamp_dump:
+ mock_check_output.side_effect = [
+ '\n'.join(makefiles).encode(),
+ ]
+
+ kati_makefiles = manifest_split.get_kati_makefiles(
+ 'stamp-file', ['overlays/oem_overlay/'])
+ self.assertEqual(
+ kati_makefiles,
+ set([
+ # Regular product makefiles
+ 'device/oem1/product1.mk',
+ 'device/oem2/product2.mk',
+ # Product makefile remapped from an overlay
+ 'device/oem3/product3.mk',
+ # Product makefile symlink and its source
+ 'device/oem4/symlink_dest.mk',
+ 'vendor/oem4/symlink_src.mk',
+ ]))
+
def test_scan_repo_projects(self):
repo_projects = {
'system/project1': 'platform/project1',
@@ -208,6 +255,7 @@
mock_check_output.side_effect = [
ninja_inputs_droid,
+ b'', # Unused kati makefiles. This is tested in its own method.
ninja_inputs_target_b,
ninja_inputs_target_c,
]
@@ -219,12 +267,11 @@
</config>""")
config_file.flush()
- manifest_split.create_split_manifest(['droid'], manifest_file.name,
- split_manifest_file.name,
- [config_file.name],
- repo_list_file.name,
- 'build-target.ninja',
- module_info_file.name, 'ninja')
+ manifest_split.create_split_manifest(
+ ['droid'], manifest_file.name, split_manifest_file.name,
+ [config_file.name], repo_list_file.name, 'build-target.ninja',
+ 'ninja', module_info_file.name, 'unused kati stamp',
+ ['unused overlay'])
split_manifest = ET.parse(split_manifest_file.name)
split_manifest_projects = [
child.attrib['name']
