blob: 5114f2426e30008c79597c0a009e3547283e7556 [file] [log] [blame]
# Copyright (C) 2020 The Android Open Source Project
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Splits a manifest to the minimum set of projects needed to build the targets.
Usage: manifest_split [options] targets
targets: Space-separated list of targets that should be buildable
using the split manifest.
options:
--manifest <path>
Path to the repo manifest to split. [Required]
--split-manifest <path>
Path to write the resulting split manifest. [Required]
--config <path>
Optional path(s) to a config XML file containing projects to add or
remove. See default_config.xml for an example. This flag can be passed
more than once to use multiple config files.
Sample file my_config.xml:
<config>
<add_project name="vendor/my/needed/project" />
<remove_project name="vendor/my/unused/project" />
</config>
--ignore-default-config
If provided, don't include default_config.xml.
--installed-prebuilt
Specify the directory containing an installed prebuilt Android.bp file.
Supply this option zero or more times, once for each installed prebuilt
directory.
--repo-list <path>
Optional path to the output of the 'repo list' command. Used if the
output of 'repo list' needs pre-processing before being used by
this tool.
--ninja-build <path>
Optional path to the combined-<target>.ninja file found in an out dir.
If not provided, the default file is used based on the lunch environment.
--ninja-binary <path>
Optional path to the ninja binary. Uses the standard binary by default.
--module-info <path>
Optional path to the module-info.json file found in an out dir.
If not provided, the default file is used based on the lunch environment.
--skip-module-info
If provided, skip parsing module-info.json for direct and adjacent
dependencies. Overrides --module-info option.
--kati-stamp <path>
Optional path to the .kati_stamp file found in an out dir.
If not provided, the default file is used based on the lunch environment.
--skip-kati
If provided, skip Kati makefiles projects. Overrides --kati-stamp option.
--overlay <path>
Optional path(s) to treat as overlays when parsing the kati stamp file
and scanning for makefiles. See the tools/treble/build/sandbox directory
for more info about overlays. This flag can be passed more than once.
--debug-file <path>
If provided, debug info will be written to a JSON file at this path.
-h (--help)
Display this usage message and exit.
"""
from __future__ import print_function
import getopt
import json
import logging
import os
import pkgutil
import re
import subprocess
import sys
import tempfile
from typing import Dict, List, Pattern, Set, Tuple
import xml.etree.ElementTree as ET
import dataclasses
logging.basicConfig(
stream=sys.stdout,
level=logging.INFO,
format="%(asctime)s - %(filename)s - %(levelname)-8s: %(message)s",
datefmt="%Y-%m-%d %H:%M:%S")
logger = logging.getLogger(os.path.basename(__file__))
# Projects determined to be needed despite the dependency not being visible
# to ninja.
DEFAULT_CONFIG_XML = "default_config.xml"
# Pattern that matches a java dependency.
_JAVA_LIB_PATTERN = re.compile(
# pylint: disable=line-too-long
'^out/target/common/obj/JAVA_LIBRARIES/(.+)_intermediates/classes-header.jar$'
)
@dataclasses.dataclass
class PathMappingConfig:
pattern: Pattern[str]
sub: str
@dataclasses.dataclass
class ManifestSplitConfig:
"""Holds the configuration for the split manifest tool.
Attributes:
remove_projects: A Dict of project name to the config file that specified
this project, for projects that should be removed from the resulting
manifest.
add_projects: A Dict of project name to the config file that specified
this project, for projects that should be added to the resulting manifest.
path_mappings: A list of PathMappingConfigs to modify a path in the build
sandbox to the path in the manifest.
ignore_paths: Set of paths to ignore when parsing module_info_file
"""
remove_projects: Dict[str, str]
add_projects: Dict[str, str]
path_mappings: List[PathMappingConfig]
ignore_paths: Set[str]
@classmethod
def from_config_files(cls, config_files: List[str]):
"""Reads from a list of config XML files.
Args:
config_files: A list of config XML filenames.
Returns:
A ManifestSplitConfig from the files.
"""
remove_projects: Dict[str, str] = {}
add_projects: Dict[str, str] = {}
path_mappings = []
""" Always ignore paths in out/ directory. """
ignore_paths = set(["out/"])
for config_file in config_files:
root = ET.parse(config_file).getroot()
remove_projects.update({
c.attrib["name"]: config_file for c in root.findall("remove_project")
})
add_projects.update(
{c.attrib["name"]: config_file for c in root.findall("add_project")})
path_mappings.extend([
PathMappingConfig(
re.compile(child.attrib["pattern"]), child.attrib["sub"])
for child in root.findall("path_mapping")
])
ignore_paths.update(
{c.attrib["name"]: config_file for c in root.findall("ignore_path")})
return cls(remove_projects, add_projects, path_mappings, ignore_paths)
def get_repo_projects(repo_list_file, manifest, path_mappings):
"""Returns a dict of { project path : project name } using the manifest.
The path_mappings stop on the first match mapping. If the mapping results in
an empty string, that entry is removed.
Args:
repo_list_file: An optional filename to read instead of parsing the manifest.
manifest: The manifest object to scan for projects.
path_mappings: A list of PathMappingConfigs to modify a path in the build
sandbox to the path in the manifest.
"""
repo_list = []
if repo_list_file:
with open(repo_list_file) as repo_list_lines:
repo_list = [line.strip().split(" : ") for line in repo_list_lines if line.strip()]
else:
root = manifest.getroot()
repo_list = [(p.get("path", p.get("name")), p.get("name")) for p in root.findall("project")]
repo_dict = {}
for entry in repo_list:
path, project = entry
for mapping in path_mappings:
if mapping.pattern.fullmatch(path):
path = mapping.pattern.sub(mapping.sub, path)
break
# If the resulting path mapping is empty, then don't add entry
if path:
repo_dict[path] = project
return repo_dict
class ModuleInfo:
"""Contains various mappings to/from module/project"""
def __init__(self, module_info_file, repo_projects, ignore_paths):
"""Initialize a module info instance.
Builds various maps related to platform build system modules and how they
relate to each other and projects.
Args:
module_info_file: The path to a module-info.json file from a build.
repo_projects: The output of the get_repo_projects function.
ignore_paths: Set of paths to ignore from module_info_file data
Raises:
ValueError: A module from module-info.json belongs to a path not
known by the repo projects output.
"""
# Maps a project to the set of modules it contains.
self.project_modules = {}
# Maps a module to the project that contains it.
self.module_project = {}
# Maps a module to its class.
self.module_class = {}
# Maps a module to modules it depends on.
self.module_deps = {}
with open(module_info_file) as module_info_file:
module_info = json.load(module_info_file)
# Check that module contains a path and the path is not in set of
# ignore paths
def module_has_valid_path(module):
paths = module.get("path")
if not paths:
return False
return all(not paths[0].startswith(p) for p in ignore_paths)
module_paths = {
module: module_info[module]["path"][0]
for module in module_info
if module_has_valid_path(module_info[module])
}
module_project_paths = {
module: scan_repo_projects(repo_projects, module_paths[module])
for module in module_paths
}
for module, project_path in module_project_paths.items():
if not project_path:
raise ValueError("Unknown module path for module %s: %s" %
(module, module_info[module]))
repo_project = repo_projects[project_path]
self.project_modules.setdefault(repo_project, set()).add(module)
self.module_project[module] = repo_project
def dep_from_raw_dep(raw_dep):
match = re.search(_JAVA_LIB_PATTERN, raw_dep)
return match.group(1) if match else raw_dep
def deps_from_raw_deps(raw_deps):
return [dep_from_raw_dep(raw_dep) for raw_dep in raw_deps]
self.module_class = {
module: module_info[module]["class"][0]
for module in module_info
}
self.module_deps = {
module: deps_from_raw_deps(module_info[module]["dependencies"])
for module in module_info
}
def get_ninja_inputs(ninja_binary, ninja_build_file, modules):
"""Returns the set of input file path strings for the given modules.
Uses the `ninja -t inputs` tool.
Args:
ninja_binary: The path to a ninja binary.
ninja_build_file: The path to a .ninja file from a build.
modules: The list of modules to scan for inputs.
"""
inputs = set()
NINJA_SHARD_LIMIT = 20000
for i in range(0, len(modules), NINJA_SHARD_LIMIT):
modules_shard = modules[i:i + NINJA_SHARD_LIMIT]
inputs = inputs.union(set(
subprocess.check_output([
ninja_binary,
"-f",
ninja_build_file,
"-t",
"inputs",
"-d",
] + list(modules_shard)).decode().strip("\n").split("\n")))
def input_allowed(path):
path = path.strip()
if path.endswith("TEST_MAPPING") and "test_mapping" not in modules:
# Exclude projects that are only needed for TEST_MAPPING files, unless the
# user is asking to build 'test_mapping'.
return False
if path.endswith("MODULE_LICENSE_GPL"):
# Exclude projects that are included only due to having a
# MODULE_LICENSE_GPL file, if no other inputs from that project are used.
return False
return path
return {path.strip() for path in inputs if input_allowed(path)}
def get_kati_makefiles(kati_stamp_file, overlays):
"""Returns the set of makefile paths from the kati stamp file.
Uses the ckati_stamp_dump prebuilt binary.
Also includes symlink sources in the resulting set for any
makefiles that are symlinks.
Args:
kati_stamp_file: The path to a .kati_stamp file from a build.
overlays: A list of paths to treat as overlays when parsing the kati stamp
file.
"""
# Get a set of all makefiles that were parsed by Kati during the build.
makefiles = set(
subprocess.check_output([
"prebuilts/build-tools/linux-x86/bin/ckati_stamp_dump",
"--files",
kati_stamp_file,
]).decode().strip("\n").split("\n"))
def is_product_makefile(makefile):
"""Returns True if the makefile path meets certain criteria."""
banned_prefixes = [
"out/",
# Ignore product makefiles for sample AOSP boards.
"device/amlogic",
"device/generic",
"device/google",
"device/linaro",
"device/sample",
]
banned_suffixes = [
# All Android.mk files in the source are always parsed by Kati,
# so including them here would bring in lots of unnecessary projects.
"Android.mk",
# The ckati stamp file always includes a line for the ckati bin at
# the beginnning.
"bin/ckati",
]
return (all([not makefile.startswith(p) for p in banned_prefixes]) and
all([not makefile.endswith(s) for s in banned_suffixes]))
# Limit the makefiles to only product makefiles.
product_makefiles = {
os.path.normpath(path) for path in makefiles if is_product_makefile(path)
}
def strip_overlay(makefile):
"""Remove any overlays from a makefile path."""
for overlay in overlays:
if makefile.startswith(overlay):
return makefile[len(overlay):]
return makefile
makefiles_and_symlinks = set()
for makefile in product_makefiles:
# Search for the makefile, possibly scanning overlays as well.
for overlay in [""] + overlays:
makefile_with_overlay = os.path.join(overlay, makefile)
if os.path.exists(makefile_with_overlay):
makefile = makefile_with_overlay
break
if not os.path.exists(makefile):
logger.warning("Unknown kati makefile: %s" % makefile)
continue
# Ensure the project that contains the makefile is included, as well as
# the project that any makefile symlinks point to.
makefiles_and_symlinks.add(strip_overlay(makefile))
if os.path.islink(makefile):
makefiles_and_symlinks.add(
strip_overlay(os.path.relpath(os.path.realpath(makefile))))
return makefiles_and_symlinks
def scan_repo_projects(repo_projects, input_path):
"""Returns the project path of the given input path if it exists.
Args:
repo_projects: The output of the get_repo_projects function.
input_path: The path of an input file used in the build, as given by the
ninja inputs tool.
Returns:
The path string, or None if not found.
"""
parts = input_path.split("/")
for index in reversed(range(0, len(parts))):
project_path = os.path.join(*parts[:index + 1])
if project_path in repo_projects:
return project_path
return None
def get_input_projects(repo_projects, inputs):
"""Returns the collection of project names that contain the given input paths.
Args:
repo_projects: The output of the get_repo_projects function.
inputs: The paths of input files used in the build, as given by the ninja
inputs tool.
"""
input_project_paths = {}
for input_path in inputs:
if not input_path.startswith("out/") and not input_path.startswith("/"):
input_project_paths.setdefault(
scan_repo_projects(repo_projects, input_path), []).append(input_path)
return {
repo_projects[project_path]: inputs
for project_path, inputs in input_project_paths.items()
if project_path is not None
}
def update_manifest(manifest, input_projects, remove_projects):
"""Modifies and returns a manifest ElementTree by modifying its projects.
Args:
manifest: The manifest object to modify.
input_projects: A set of projects that should stay in the manifest.
remove_projects: A set of projects that should be removed from the manifest.
Projects in this set override input_projects.
Returns:
The modified manifest object.
"""
projects_to_keep = input_projects.difference(remove_projects)
root = manifest.getroot()
for child in root.findall("project"):
if child.attrib["name"] not in projects_to_keep:
root.remove(child)
return manifest
@dataclasses.dataclass
class DebugInfo:
"""Simple class to store structured debug info for a project."""
direct_input: bool = False
adjacent_input: bool = False
deps_input: bool = False
kati_makefiles: List[str] = dataclasses.field(default_factory=list)
manual_add_config: str = ""
manual_remove_config: str = ""
def create_split_manifest(targets, manifest_file, split_manifest_file,
config_files, repo_list_file, ninja_build_file,
ninja_binary, module_info_file, kati_stamp_file,
overlays, installed_prebuilts, debug_file):
"""Creates and writes a split manifest by inspecting build inputs.
Args:
targets: List of targets that should be buildable using the split manifest.
manifest_file: Path to the repo manifest to split.
split_manifest_file: Path to write the resulting split manifest.
config_files: Paths to a config XML file containing projects to add or
remove. See default_config.xml for an example. This flag can be passed
more than once to use multiple config files.
repo_list_file: Path to the output of the 'repo list' command.
ninja_build_file: Path to the combined-<target>.ninja file found in an out
dir.
ninja_binary: Path to the ninja binary.
module_info_file: Path to the module-info.json file found in an out dir.
kati_stamp_file: The path to a .kati_stamp file from a build.
overlays: A list of paths to treat as overlays when parsing the kati stamp
file.
installed_prebuilts: A list of paths for which to create "fake" repo
entries. These entries allow the tool to recognize modules that installed
rather than being sync'd via a manifest.
debug_file: If not None, the path to write JSON debug info.
"""
debug_info = {}
config = ManifestSplitConfig.from_config_files(config_files)
original_manifest = ET.parse(manifest_file)
repo_projects = get_repo_projects(repo_list_file, original_manifest,
config.path_mappings)
repo_projects.update({ip: ip for ip in installed_prebuilts})
inputs = get_ninja_inputs(ninja_binary, ninja_build_file, targets)
input_projects = set(get_input_projects(repo_projects, inputs).keys())
for project in input_projects:
debug_info.setdefault(project, DebugInfo()).direct_input = True
logger.info(
"%s projects needed for Ninja-graph direct dependencies of targets \"%s\"",
len(input_projects), " ".join(targets))
if kati_stamp_file:
kati_makefiles = get_kati_makefiles(kati_stamp_file, overlays)
kati_makefiles_projects = get_input_projects(repo_projects, kati_makefiles)
for project, makefiles in kati_makefiles_projects.items():
debug_info.setdefault(project, DebugInfo()).kati_makefiles = makefiles
input_projects = input_projects.union(kati_makefiles_projects.keys())
logger.info("%s projects after including Kati makefiles projects.",
len(input_projects))
else:
logger.info("Kati makefiles projects skipped.")
for project, cfile in config.add_projects.items():
debug_info.setdefault(project, DebugInfo()).manual_add_config = cfile
for project, cfile in config.remove_projects.items():
debug_info.setdefault(project, DebugInfo()).manual_remove_config = cfile
input_projects = input_projects.union(config.add_projects.keys())
logger.info("%s projects after including manual additions.",
len(input_projects))
# Remove projects from our set of input projects before adding adjacent
# modules, so that no project is added only because of an adjacent
# dependency in a to-be-removed project.
input_projects = input_projects.difference(config.remove_projects.keys())
# While we still have projects whose modules we haven't checked yet,
if module_info_file:
module_info = ModuleInfo(module_info_file, repo_projects,
config.ignore_paths)
checked_projects = set()
projects_to_check = input_projects.difference(checked_projects)
logger.info("Checking module-info dependencies for direct and adjacent modules...")
else:
logging.info("Direct and adjacent modules skipped.")
projects_to_check = None
iteration = 0
while projects_to_check:
iteration += 1
# check all modules in each project,
modules = []
deps_additions = set()
def process_deps(module):
for d in module_info.module_deps[module]:
if d in module_info.module_class:
if module_info.module_class[d] == "HEADER_LIBRARIES":
hla = module_info.module_project[d]
if hla not in input_projects:
deps_additions.add(hla)
for project in projects_to_check:
checked_projects.add(project)
if project not in module_info.project_modules:
continue
for module in module_info.project_modules[project]:
modules.append(module)
process_deps(module)
for project in deps_additions:
debug_info.setdefault(project, DebugInfo()).deps_input = True
input_projects = input_projects.union(deps_additions)
logger.info(
"pass %d - %d projects after including HEADER_LIBRARIES dependencies",
iteration, len(input_projects))
# adding those modules' input projects to our list of projects.
inputs = get_ninja_inputs(ninja_binary, ninja_build_file, modules)
adjacent_module_additions = set(
get_input_projects(repo_projects, inputs).keys())
for project in adjacent_module_additions:
debug_info.setdefault(project, DebugInfo()).adjacent_input = True
input_projects = input_projects.union(adjacent_module_additions)
logger.info(
"pass %d - %d projects after including adjacent-module Ninja-graph dependencies",
iteration, len(input_projects))
projects_to_check = input_projects.difference(checked_projects)
logger.info("%s projects - complete", len(input_projects))
split_manifest = update_manifest(original_manifest, input_projects,
config.remove_projects.keys())
split_manifest.write(split_manifest_file)
if debug_file:
with open(debug_file, "w") as debug_fp:
logger.info("Writing debug info to %s", debug_file)
json.dump(
debug_info,
fp=debug_fp,
sort_keys=True,
indent=2,
default=lambda info: info.__dict__)
def main(argv):
try:
opts, args = getopt.getopt(argv, "h", [
"help",
"debug-file=",
"manifest=",
"split-manifest=",
"config=",
"ignore-default-config",
"repo-list=",
"ninja-build=",
"ninja-binary=",
"module-info=",
"skip-module-info",
"kati-stamp=",
"skip-kati",
"overlay=",
"installed-prebuilt=",
])
except getopt.GetoptError as err:
print(__doc__, file=sys.stderr)
print("**%s**" % str(err), file=sys.stderr)
sys.exit(2)
debug_file = None
manifest_file = None
split_manifest_file = None
config_files = []
repo_list_file = None
ninja_build_file = None
module_info_file = None
ninja_binary = "prebuilts/build-tools/linux-x86/bin/ninja"
kati_stamp_file = None
overlays = []
installed_prebuilts = []
ignore_default_config = False
skip_kati = False
skip_module_info = False
for o, a in opts:
if o in ("-h", "--help"):
print(__doc__, file=sys.stderr)
sys.exit()
elif o in ("--debug-file"):
debug_file = a
elif o in ("--manifest"):
manifest_file = a
elif o in ("--split-manifest"):
split_manifest_file = a
elif o in ("--config"):
config_files.append(a)
elif o == "--ignore-default-config":
ignore_default_config = True
elif o in ("--repo-list"):
repo_list_file = a
elif o in ("--ninja-build"):
ninja_build_file = a
elif o in ("--ninja-binary"):
ninja_binary = a
elif o in ("--module-info"):
module_info_file = a
elif o == "--skip-module-info":
skip_module_info = True
elif o in ("--kati-stamp"):
kati_stamp_file = a
elif o == "--skip-kati":
skip_kati = True
elif o in ("--overlay"):
overlays.append(a)
elif o in ("--installed-prebuilt"):
installed_prebuilts.append(a)
else:
assert False, "unknown option \"%s\"" % o
if not args:
print(__doc__, file=sys.stderr)
print("**Missing targets**", file=sys.stderr)
sys.exit(2)
if not manifest_file:
print(__doc__, file=sys.stderr)
print("**Missing required flag --manifest**", file=sys.stderr)
sys.exit(2)
if not split_manifest_file:
print(__doc__, file=sys.stderr)
print("**Missing required flag --split-manifest**", file=sys.stderr)
sys.exit(2)
if skip_module_info:
if module_info_file:
logging.warning("User provided both --skip-module-info and --module-info args. Arg --module-info ignored.")
module_info_file = None
elif not module_info_file:
module_info_file = os.path.join(os.environ["ANDROID_PRODUCT_OUT"],
"module-info.json")
if skip_kati:
if kati_stamp_file:
logging.warning("User provided both --skip-kati and --kati-stamp args. Arg --kati-stamp ignored.")
kati_stamp_file = None
elif not kati_stamp_file:
kati_stamp_file = os.path.join(
os.environ["ANDROID_BUILD_TOP"], "out",
".kati_stamp-%s" % os.environ["TARGET_PRODUCT"])
if not ninja_build_file:
ninja_build_file = os.path.join(
os.environ["ANDROID_BUILD_TOP"], "out",
"combined-%s.ninja" % os.environ["TARGET_PRODUCT"])
with tempfile.NamedTemporaryFile() as default_config_file:
if not ignore_default_config:
default_config_file.write(pkgutil.get_data(__name__, DEFAULT_CONFIG_XML))
default_config_file.flush()
config_files.insert(0, default_config_file.name)
create_split_manifest(
targets=args,
manifest_file=manifest_file,
split_manifest_file=split_manifest_file,
config_files=config_files,
repo_list_file=repo_list_file,
ninja_build_file=ninja_build_file,
ninja_binary=ninja_binary,
module_info_file=module_info_file,
kati_stamp_file=kati_stamp_file,
overlays=overlays,
installed_prebuilts=installed_prebuilts,
debug_file=debug_file)
if __name__ == "__main__":
main(sys.argv[1:])