Merge changes Ice859e8e,I44915dcd
* changes:
Add typing info and documentation to ndk.builds.
Provide type annotations for ndk.paths.
diff --git a/ndk/abis.py b/ndk/abis.py
index 91bb998..76f36f1 100644
--- a/ndk/abis.py
+++ b/ndk/abis.py
@@ -14,6 +14,10 @@
# limitations under the License.
#
"""Constants and helper functions for NDK ABIs."""
+from typing import NewType
+
+
+Arch = NewType('Arch', str)
LP32_ABIS = (
@@ -32,10 +36,10 @@
ALL_ARCHITECTURES = (
- 'arm',
- 'arm64',
- 'x86',
- 'x86_64',
+ Arch('arm'),
+ Arch('arm64'),
+ Arch('x86'),
+ Arch('x86_64'),
)
diff --git a/ndk/builds.py b/ndk/builds.py
index c3d6b9b..a2c7cc6 100644
--- a/ndk/builds.py
+++ b/ndk/builds.py
@@ -19,12 +19,13 @@
"""
from __future__ import absolute_import
+from enum import auto, Enum, unique
import ntpath
import os
import shutil
import stat
import subprocess
-from typing import Iterable, Set
+from typing import Iterable, List, Optional, Set
import ndk.abis
import ndk.ext.shutil
@@ -33,21 +34,27 @@
class ModuleValidateError(RuntimeError):
+ """The error raised when module validation fails."""
pass
-class NoticeGroup:
+@unique
+class NoticeGroup(Enum):
"""An enum describing NOTICE file groupings.
The NDK ships two NOTICE files: one for the toolchain, and one for
everything else.
"""
- BASE = 1
- TOOLCHAIN = 2
+ BASE = auto()
+ TOOLCHAIN = auto()
class BuildContext:
- def __init__(self, out_dir, dist_dir, modules, host, arches, build_number):
+ """Class containing build context information."""
+
+ def __init__(self, out_dir: str, dist_dir: str, modules: List['Module'],
+ host: ndk.hosts.Host, arches: List[ndk.abis.Arch],
+ build_number: str) -> None:
self.out_dir = out_dir
self.dist_dir = dist_dir
self.modules = {m.name: m for m in modules}
@@ -57,6 +64,8 @@
class Module:
+ """Base module type for the build system."""
+
name: str
path: str
deps: Set[str] = set()
@@ -69,7 +78,7 @@
# interface is a single path, not a list. For the rare modules that have
# multiple notice files (such as yasm), the notices property should be
# overrided. By default this property will return `[self.notice]`.
- notice = None
+ notice: Optional[str] = None
# Not all components need a notice (stub scripts, basic things like the
# readme and changelog, etc), but this is opt-out.
@@ -95,29 +104,48 @@
self.validate()
@property
- def notices(self):
+ def notices(self) -> List[str]:
+ """Returns the list of notice files for this module."""
if self.no_notice:
return []
if self.notice is None:
return []
return [self.notice]
- def default_notice_path(self): # pylint: disable=no-self-use
+ def default_notice_path(self) -> Optional[str]:
+ """Returns the path to the default notice for this module, if any."""
return None
- def validate_error(self, msg):
- return ModuleValidateError('{}: {}'.format(self.name, msg))
+ def validate_error(self, msg: str) -> ModuleValidateError:
+ """Creates a validation error for this module.
- def validate(self):
+ Automatically includes the module name in the error string.
+
+ Args:
+ msg: Detailed error message.
+ """
+ return ModuleValidateError(f'{self.name}: {msg}')
+
+ def validate(self) -> None:
+ """Validates module config.
+
+ Raises:
+ ModuleValidateError: The module configuration is not valid.
+ """
if self.name is None:
- raise ModuleValidateError('{} has no name'.format(self.__class__))
+ raise ModuleValidateError(f'{self.__class__} has no name')
if self.path is None:
raise self.validate_error('path property not set')
- if self.notice_group not in (NoticeGroup.BASE, NoticeGroup.TOOLCHAIN):
+ if self.notice_group not in NoticeGroup:
raise self.validate_error('invalid notice group')
self.validate_notice()
- def validate_notice(self):
+ def validate_notice(self) -> None:
+ """Validates the notice files of this module.
+
+ Raises:
+ ModuleValidateError: The module configuration is not valid.
+ """
if self.no_notice:
return
@@ -126,36 +154,76 @@
for notice in self.notices:
if not os.path.exists(notice):
raise self.validate_error(
- 'notice file {} does not exist'.format(notice))
+ f'notice file {notice} does not exist')
- def get_dep(self, name):
+ def get_dep(self, name: str) -> 'Module':
+ """Returns the module object for the given dependency.
+
+ Returns:
+ The module object for the given dependency.
+
+ Raises:
+ KeyError: The given name does not match any of this module's
+ dependencies.
+ """
if name not in self.deps:
raise KeyError
return self.context.modules[name]
- def get_build_host_install(self, arch=None):
+ def get_build_host_install(self,
+ arch: Optional[ndk.abis.Arch] = None) -> str:
+ """Returns the module's install path for the current host.
+
+ In a cross-compiling context (i.e. building the Windows NDK from
+ Linux), this will return the install directory for the build OS rather
+ than the target OS.
+
+ Args:
+ arch: Architecture to fetch for architecture-specific modules.
+
+ Returns:
+ This module's install path for the build host.
+ """
return self.get_install_path(ndk.hosts.get_default_host(), arch)
@property
- def out_dir(self):
+ def out_dir(self) -> str:
+ """Base out directory for the current build."""
return self.context.out_dir
@property
- def dist_dir(self):
+ def dist_dir(self) -> str:
+ """Base dist directory for the current build."""
return self.context.dist_dir
@property
- def host(self):
+ def host(self) -> ndk.hosts.Host:
+ """Host for the current build."""
return self.context.host
@property
- def arches(self):
+ def arches(self) -> List[ndk.abis.Arch]:
+ """Architectures targeted by the current build."""
return self.context.arches
- def build(self):
+ def build(self) -> None:
+ """Builds the module.
+
+ A module's dependencies are guaranteed to have been installed before
+ its build begins.
+
+ The build phase should not modify the install directory.
+ """
raise NotImplementedError
- def install(self):
+ def install(self) -> None:
+ """Installs the module.
+
+ Install happens after the module has been built.
+
+ The install phase should only copy files, not create them. Compilation
+ should happen in the build phase.
+ """
package_installs = ndk.packaging.expand_packages(
self.name, self.path, self.host, self.arches)
@@ -168,12 +236,34 @@
shutil.rmtree(install_path)
ndk.packaging.extract_zip(package, install_path)
- def get_install_paths(self, host, arches):
+ def get_install_paths(
+ self, host: ndk.hosts.Host,
+ arches: Optional[Iterable[ndk.abis.Arch]]) -> List[str]:
+ """Returns the install paths for the given archiectures."""
install_subdirs = ndk.packaging.expand_paths(self.path, host, arches)
install_base = ndk.paths.get_install_path(self.out_dir, host)
return [os.path.join(install_base, d) for d in install_subdirs]
- def get_install_path(self, host=None, arch=None):
+ def get_install_path(self, host: Optional[ndk.hosts.Host] = None,
+ arch: Optional[ndk.abis.Arch] = None) -> str:
+ """Returns the install path for the given module config.
+
+ For an architecture-independent module, there should only ever be one
+ install path.
+
+ For an architecture-dependent module, the optional arch argument must
+ be provided to select between the install paths.
+
+ Args:
+ host: The host to use for a host-specific install path.
+ arch: The architecture to use for an architecure-dependent module.
+
+ Raises:
+ ValueError: This is an architecture-dependent module and no
+ architecture was provided.
+ RuntimeError: An architecture-independent module has non-unique
+ install paths.
+ """
if host is None:
host = self.host
@@ -194,7 +284,7 @@
arches = [self.build_arch]
elif arch_dependent:
raise ValueError(
- 'get_install_path for {} requires valid arch'.format(arch))
+ f'get_install_path for {arch} requires valid arch')
install_subdirs = self.get_install_paths(host, arches)
@@ -206,7 +296,7 @@
def __str__(self):
if self.split_build_by_arch and self.build_arch is not None:
- return '{} [{}]'.format(self.name, self.build_arch)
+ return f'{self.name} [{self.build_arch}]'
return self.name
def __hash__(self):
@@ -219,26 +309,36 @@
return str(self) == str(other)
@property
- def log_file(self):
+ def log_file(self) -> str:
+ """Returns the basename of the log file for this module."""
if self.split_build_by_arch and self.build_arch is not None:
- return '{}-{}.log'.format(self.name, self.build_arch)
+ return f'{self.name}-{self.build_arch}.log'
elif self.split_build_by_arch:
raise RuntimeError('Called log_file on unsplit module')
else:
- return '{}.log'.format(self.name)
+ return f'{self.name}.log'
- def log_path(self, log_dir):
+ def log_path(self, log_dir: str) -> str:
+ """Returns the path to the log file for this module."""
return os.path.join(log_dir, self.log_file)
class PackageModule(Module):
- src = None
+ """A directory to be installed to the NDK.
+
+ No transformation is performed on the installed directory.
+ """
+
+ #: The absolute path to the directory to be installed.
+ src: str
+
+ #: If true, create a repo.prop file for this module.
create_repo_prop = False
- def default_notice_path(self):
+ def default_notice_path(self) -> str:
return os.path.join(self.src, 'NOTICE')
- def validate(self):
+ def validate(self) -> None:
super().validate()
if ndk.packaging.package_varies_by(self.path, 'abi'):
@@ -254,10 +354,10 @@
raise self.validate_error(
'PackageModule cannot vary by triple')
- def build(self):
+ def build(self) -> None:
pass
- def install(self):
+ def install(self) -> None:
install_paths = self.get_install_paths(self.host,
ndk.abis.ALL_ARCHITECTURES)
assert len(install_paths) == 1
@@ -268,53 +368,78 @@
class InvokeExternalBuildModule(Module):
+ """A module that uses a build.py script.
+
+ These are legacy modules that have not yet been properly merged into
+ checkbuild.py.
+ """
+
+ #: The path to the build script relative to the top of the source tree.
script: str
+
+ #: True if the module can be built in parallel per-architecture.
arch_specific = False
- def build(self):
+ def build(self) -> None:
build_args = common_build_args(self.out_dir, self.dist_dir, self.host)
if self.split_build_by_arch:
- build_args.append('--arch={}'.format(self.build_arch))
+ build_args.append(f'--arch={self.build_arch}')
elif self.arch_specific and len(self.arches) == 1:
- build_args.append('--arch={}'.format(self.arches[0]))
+ build_args.append(f'--arch={self.arches[0]}')
elif self.arches == ndk.abis.ALL_ARCHITECTURES:
pass
else:
raise NotImplementedError(
- 'Module {} can only build all architectures or none'.format(
- self.name))
+ f'Module {self.name} can only build all architectures or none')
script = self.get_script_path()
invoke_external_build(script, build_args)
- def get_script_path(self):
+ def get_script_path(self) -> str:
+ """Returns the absolute path to the build script."""
return ndk.paths.android_path(self.script)
class InvokeBuildModule(InvokeExternalBuildModule):
- def get_script_path(self):
+ """A module that uses a build.py script within ndk/build/tools.
+
+ Identical to InvokeExternalBuildModule, but the script path is relative to
+ ndk/build/tools instead of the top of the source tree.
+ """
+
+ def get_script_path(self) -> str:
return ndk.paths.ndk_path('build/tools', self.script)
class FileModule(Module):
- src = None
+ """A module that installs a single file to the NDK."""
- # Used for things like the readme and the changelog. No notice needed.
+ #: Path to the file to be installed.
+ src: str
+
+ #: True if no notice file is needed for this module.
no_notice = True
- def build(self):
+ def build(self) -> None:
pass
- def install(self):
+ def install(self) -> None:
shutil.copy2(self.src, self.get_install_path())
class MultiFileModule(Module):
+ """A module that installs multiple files to the NDK.
+
+ This is similar to FileModule, but allows multiple files to be installed
+ with a single module.
+ """
+
+ #: List of absolute paths to files to be installed.
files: Iterable[str] = []
- def build(self):
+ def build(self) -> None:
pass
- def install(self):
+ def install(self) -> None:
install_dir = self.get_install_path()
ndk.ext.shutil.create_directory(install_dir)
for file_path in self.files:
@@ -322,13 +447,24 @@
class ScriptShortcutModule(Module):
+ """A module that installs a shortcut to another script in the NDK.
+
+ Some NDK tools are installed to a location other than the top of the NDK
+ (as a result of the modular NDK effort), but we want to make them
+ accessible from the top level because that is where they were Historically
+ installed.
+ """
+
+ #: The path to the installed NDK script, relative to the top of the NDK.
script: str
+
+ #: The file extension for the called script on Windows.
windows_ext: str
# These are all trivial shell scripts that we generated. No notice needed.
no_notice = True
- def validate(self):
+ def validate(self) -> None:
super().validate()
if ndk.packaging.package_varies_by(self.script, 'abi'):
@@ -347,16 +483,17 @@
raise self.validate_error(
'ScriptShortcutModule requires windows_ext')
- def build(self):
+ def build(self) -> None:
pass
- def install(self):
+ def install(self) -> None:
if self.host.startswith('windows'):
self.make_cmd_helper()
else:
self.make_sh_helper()
- def make_cmd_helper(self):
+ def make_cmd_helper(self) -> None:
+ """Makes a .cmd helper script for Windows."""
script = self.get_script_path()
full_path = ntpath.join(
'%~dp0', ntpath.normpath(script) + self.windows_ext)
@@ -368,7 +505,8 @@
full_path + ' %*\n',
])
- def make_sh_helper(self):
+ def make_sh_helper(self) -> None:
+ """Makes a bash helper script for POSIX systems."""
script = self.get_script_path()
full_path = os.path.join('$DIR', script)
@@ -383,7 +521,8 @@
os.chmod(install_path,
mode | stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH)
- def get_script_path(self):
+ def get_script_path(self) -> str:
+ """Returns the installed path of the script."""
scripts = ndk.packaging.expand_paths(
self.script, self.host, ndk.abis.ALL_ARCHITECTURES)
assert len(scripts) == 1
@@ -391,43 +530,85 @@
class PythonPackage(Module):
- def default_notice_path(self):
+ """A Python package that should be packaged for distribution.
+
+ These are not installed within the NDK, but are packaged (as a source
+ distribution) for archival on the build servers.
+
+ These are used to archive the NDK's build and test tools so test artifacts
+ may be regnenerated using only artifacts from the build server.
+ """
+
+ def default_notice_path(self) -> str:
# Assume there's a NOTICE file in the same directory as the setup.py.
return os.path.join(os.path.dirname(self.path), 'NOTICE')
- def build(self):
+ def build(self) -> None:
cwd = os.path.dirname(self.path)
subprocess.check_call(
['python3', self.path, 'sdist', '-d', self.out_dir], cwd=cwd)
- def install(self):
+ def install(self) -> None:
pass
-def _invoke_build(script, args):
- if args is None:
- args = []
+def invoke_external_build(script: str, args: List[str]) -> None:
+ """Invokes a build.py script rooted within the top level source tree.
+
+ Args:
+ script: Path to the script to be executed within the top level source
+ tree.
+ args: Command line arguments to be passed to the script.
+ """
subprocess.check_call(['python3', ndk.paths.android_path(script)] + args)
-def invoke_build(script, args=None):
- script_path = os.path.join('build/tools', script)
- _invoke_build(ndk.paths.ndk_path(script_path), args)
+def common_build_args(out_dir: str, dist_dir: str,
+ host: ndk.hosts.Host) -> List[str]:
+ """Returns a list of common arguments for build.py scripts.
+ Modules that have not been fully merged into checkbuild.py still use a
+ separately executed build.py script via InvokeBuildModule or
+ InvokeExternalBuildModule. These have a common command line interface for
+ determining out directories and target host.
-def invoke_external_build(script, args=None):
- _invoke_build(ndk.paths.android_path(script), args)
+ Args:
+ out_dir: Base out directory for the target host.
+ dist_dir: Distribution directory for archived artifacts.
+ host: Target host.
-
-def common_build_args(out_dir, dist_dir, host):
+ Returns:
+ List of command line arguments to be used with build.py.
+ """
return [
- '--out-dir={}'.format(os.path.join(out_dir, host)),
- '--dist-dir={}'.format(dist_dir),
- '--host={}'.format(host),
+ f'--out-dir={os.path.join(out_dir, host)}',
+ f'--dist-dir={dist_dir}',
+ f'--host={host}',
]
-def install_directory(src, dst):
+def install_directory(src: str, dst: str) -> None:
+ """Copies a directory to an install location, ignoring some file types.
+
+ The destination will be removed prior to copying if it exists, ensuring a
+ clean install.
+
+ Some file types (currently python intermediates, editor swap files, git
+ directories) will be removed from the install location.
+
+ Args:
+ src: Directory to install.
+ dst: Install location. Will be removed prior to installation. The
+ source directory will be copied *to* this path, not *into* this
+ path.
+ """
+ # TODO: Remove the ignore patterns in favor of purging the install
+ # directory after install, since not everything uses install_directory.
+ #
+ # We already do this to some extent with package_ndk, but we don't cover
+ # all these file types, and we should also do this before packaging since
+ # packaging only runs when a full NDK is built (fine for the build servers,
+ # could potentially be wrong for local testing).
if os.path.exists(dst):
shutil.rmtree(dst)
ignore_patterns = shutil.ignore_patterns(
@@ -435,7 +616,26 @@
shutil.copytree(src, dst, ignore=ignore_patterns)
-def make_repo_prop(out_dir):
+def make_repo_prop(out_dir: str) -> None:
+ """Installs a repro.prop file to the given directory.
+
+ A repo.prop file is a file listing all of the git projects used and their
+ checked out revisions. i.e.
+
+ platform/bionic 40538268d43d82409a93637960f2da3c1226840a
+ platform/development 688f15246399db98897e660889d9a202559fe5d8
+ ...
+
+ Historically we installed one of these per "module" (from the attempted
+ modular NDK), but since the same information can be retrieved from the
+ build number we do not install them for most things now.
+
+ If this build is happening on the build server then there will be a
+ repo.prop file in the DIST_DIR for us to copy, otherwise we generate our
+ own.
+ """
+ # TODO: Finish removing users of this in favor of installing a single
+ # manifest.xml file in the root of the NDK.
file_name = 'repo.prop'
dist_dir = os.environ.get('DIST_DIR')
diff --git a/ndk/hosts.py b/ndk/hosts.py
index 896bfa2..3c853fe 100644
--- a/ndk/hosts.py
+++ b/ndk/hosts.py
@@ -18,13 +18,17 @@
import os
import sys
+from typing import NewType
+
+
+Host = NewType('Host', str)
ALL_HOSTS = (
- 'darwin',
- 'linux',
- 'windows',
- 'windows64',
+ Host('darwin'),
+ Host('linux'),
+ Host('windows'),
+ Host('windows64'),
)
diff --git a/ndk/paths.py b/ndk/paths.py
index 66c9c5e..d763025 100644
--- a/ndk/paths.py
+++ b/ndk/paths.py
@@ -20,25 +20,28 @@
import os
import shutil
import sys
+from typing import Generator, Optional
import ndk.abis
import ndk.config
+import ndk.hosts
THIS_DIR = os.path.realpath(os.path.dirname(__file__))
-def android_path(*args):
+def android_path(*args: str) -> str:
"""Returns the absolute path rooted within the top level source tree."""
return os.path.normpath(os.path.join(THIS_DIR, '../../', *args))
-def ndk_path(*args):
+def ndk_path(*args: str) -> str:
"""Returns the absolute path rooted within the NDK source tree."""
return android_path('ndk', *args)
-def sysroot_path(toolchain):
+
+def sysroot_path(toolchain: str) -> str:
arch = ndk.abis.toolchain_to_arch(toolchain)
# Only ARM has more than one ABI, and they both have the same minimum
# platform level.
@@ -50,11 +53,11 @@
return android_path(prebuilt_ndk, sysroot_subpath)
-def toolchain_path(*args):
+def toolchain_path(*args: str) -> str:
return android_path('toolchain', *args)
-def _get_dir_from_env(default, env_var):
+def _get_dir_from_env(default: str, env_var: str) -> str:
"""Returns the path to a directory specified by the environment.
If the environment variable is not set, the default will be used. The
@@ -73,12 +76,12 @@
return path
-def get_out_dir():
+def get_out_dir() -> str:
"""Returns the out directory."""
return _get_dir_from_env(android_path('out'), 'OUT_DIR')
-def get_dist_dir(out_dir):
+def get_dist_dir(out_dir) -> str:
"""Returns the distribution directory.
The contents of the distribution directory are archived on the build
@@ -87,7 +90,7 @@
return _get_dir_from_env(os.path.join(out_dir, 'dist'), 'DIST_DIR')
-def path_in_out(dirname, out_dir=None):
+def path_in_out(dirname: str, out_dir: Optional[str] = None) -> str:
"""Returns a path within the out directory."
Args:
@@ -104,7 +107,8 @@
return os.path.join(out_dir, dirname)
-def get_install_path(out_dir=None, host=None):
+def get_install_path(out_dir: Optional[str] = None,
+ host: Optional[str] = None) -> str:
"""Returns the built NDK install path.
Note that the path returned might not actually contain the NDK. The NDK may
@@ -129,7 +133,7 @@
@contextlib.contextmanager
-def temp_dir_in_out(dirname, out_dir=None):
+def temp_dir_in_out(dirname: str, out_dir: Optional[str] = None) -> Generator:
"""Creates a well named temporary directory within the out directory.
If the directory exists on context entry, RuntimeError will be raised. The
@@ -159,7 +163,7 @@
shutil.rmtree(abspath)
-def to_posix_path(path):
+def to_posix_path(path: str) -> str:
if sys.platform == 'win32':
return path.replace('\\', '/')
else:
