Merge changes I2ae26513,Iddf3993c,I94bc5994 am: c788e51725 am: cd2360673f am: b24b5850aa
Original change: https://android-review.googlesource.com/c/platform/prebuilts/ndk/+/2067533
Change-Id: I4ad7e32e06fe18c5db92537f25564dce57a1808d
Signed-off-by: Automerger Merge Worker <android-build-automerger-merge-worker@system.gserviceaccount.com>
diff --git a/.gitignore b/.gitignore
new file mode 100644
index 0000000..a81c8ee
--- /dev/null
+++ b/.gitignore
@@ -0,0 +1,138 @@
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+share/python-wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+# PyInstaller
+# Usually these files are written by a python script from a template
+# before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.nox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+*.py,cover
+.hypothesis/
+.pytest_cache/
+cover/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+db.sqlite3
+db.sqlite3-journal
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+.pybuilder/
+target/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# IPython
+profile_default/
+ipython_config.py
+
+# pyenv
+# For a library or package, you might want to ignore these files since the code is
+# intended to run in multiple environments; otherwise, check them in:
+# .python-version
+
+# pipenv
+# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
+# However, in case of collaboration, if having platform-specific dependencies or dependencies
+# having no cross-platform support, pipenv may install dependencies that don't work, or not
+# install all needed dependencies.
+#Pipfile.lock
+
+# PEP 582; used by e.g. github.com/David-OConnor/pyflow
+__pypackages__/
+
+# Celery stuff
+celerybeat-schedule
+celerybeat.pid
+
+# SageMath parsed files
+*.sage.py
+
+# Environments
+.env
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+
+# Spyder project settings
+.spyderproject
+.spyproject
+
+# Rope project settings
+.ropeproject
+
+# mkdocs documentation
+/site
+
+# mypy
+.mypy_cache/
+.dmypy.json
+dmypy.json
+
+# Pyre type checker
+.pyre/
+
+# pytype static type analyzer
+.pytype/
+
+# Cython debug symbols
+cython_debug/
diff --git a/pylintrc b/pylintrc
index 0ce2669..9e03c7d 100644
--- a/pylintrc
+++ b/pylintrc
@@ -1,11 +1,22 @@
-[MESSAGES CONTROL]
-disable=missing-docstring,fixme,locally-disabled
+[MASTER]
+jobs = 0
+[MESSAGES CONTROL]
+disable=missing-docstring
+
+ignore=ndk-gdb.py
[BASIC]
-# Good variable names which should always be accepted, separated by a comma
good-names=i,j,k,ex,Run,_
+[VARIABLES]
+dummy-variables-rgx=_
+
+[SIMILARITIES]
+ignore-imports=yes
[FORMAT]
-max-line-length=79
+max-line-length=88
+ignore-long-lines=(?x)(
+ ^\s*(\#\ )?<?https?://\S+>?$|
+ ^\s*\#\ \S+$)
diff --git a/update.py b/update.py
index c8c821c..398a68b 100755
--- a/update.py
+++ b/update.py
@@ -15,16 +15,16 @@
# limitations under the License.
#
import argparse
-import glob
import logging
import os
+from pathlib import Path
import shutil
import subprocess
from tempfile import TemporaryDirectory
import textwrap
-THIS_DIR = os.path.realpath(os.path.dirname(__file__))
+THIS_DIR = Path(__file__).resolve().parent
def logger():
@@ -32,153 +32,212 @@
def check_call(cmd):
- logger().debug('Running `%s`', ' '.join(cmd))
+ logger().debug("Running `%s`", " ".join(cmd))
subprocess.check_call(cmd)
def remove(path):
- logger().debug('remove `%s`', path)
+ logger().debug("remove `%s`", path)
os.remove(path)
-def fetch_artifact(branch, build, pattern):
- fetch_artifact_path = '/google/data/ro/projects/android/fetch_artifact'
- cmd = [fetch_artifact_path, '--branch', branch, '--target=linux',
- '--bid', build, pattern]
+def fetch_artifact(branch: str, build: str, pattern: str) -> None:
+ """Fetches an artifact from the build server.
+
+ Use OAuth2 authentication and the gLinux android-fetch-artifact package,
+ which work with both on-corp and off-corp workstations."""
+ fetch_artifact_path = shutil.which("fetch_artifact")
+ if fetch_artifact_path is None:
+ raise RuntimeError(
+ "error: cannot find fetch_artifact in PATH. Install it using:\n"
+ " sudo glinux-add-repo android\n"
+ " sudo apt update\n"
+ " sudo apt install android-fetch-artifact\n"
+ )
+ cmd = [
+ fetch_artifact_path,
+ "--use_oauth2",
+ "--branch",
+ branch,
+ "--target=linux",
+ "--bid",
+ build,
+ pattern,
+ ]
check_call(cmd)
def api_str(api_level):
- return 'android-{}'.format(api_level)
+ return f"android-{api_level}"
def start_branch(build):
- branch_name = 'update-' + (build or 'latest')
- logger().info('Creating branch %s', branch_name)
- check_call(['repo', 'start', branch_name, '.'])
+ branch_name = "update-" + (build or "latest")
+ logger().info("Creating branch %s", branch_name)
+ check_call(["repo", "start", branch_name, "."])
-def remove_old_release(install_dir):
- if os.path.exists(os.path.join(install_dir, '.git')):
+def remove_old_release(install_dir: Path) -> None:
+ if (install_dir / ".git").exists():
logger().info('Removing old install directory "%s"', install_dir)
- check_call(['git', 'rm', '-rf', install_dir])
+ check_call(["git", "rm", "-rf", install_dir])
# Need to check again because git won't remove directories if they have
# non-git files in them.
- if os.path.exists(install_dir):
+ if install_dir.exists():
shutil.rmtree(install_dir)
-LIBUNWIND_GLOB = (
- 'toolchains/llvm/prebuilt/*/lib64/clang/*/lib/linux/*/libunwind.a'
+LIBUNWIND_GLOB = "toolchains/llvm/prebuilt/*/lib64/clang/*/lib/linux/*/libunwind.a"
+LIBCXX_SHARED_GLOB = (
+ "toolchains/llvm/prebuilt/linux-x86_64/sysroot/usr/lib/*/libc++_shared.so"
)
+LIBCXX_STATIC_GLOB = (
+ "toolchains/llvm/prebuilt/linux-x86_64/sysroot/usr/lib/*/libc++_static.a"
+)
+LIBCXXABI_GLOB = "toolchains/llvm/prebuilt/linux-x86_64/sysroot/usr/lib/*/libc++abi.a"
-def unzip_single_directory(artifact, destination):
+def unzip_single_directory(artifact: Path, destination: Path) -> None:
# Use cwd so that we can use rename without having to worry about crossing
# file systems.
with TemporaryDirectory(dir=os.getcwd()) as temp_dir:
cmd = [
- 'unzip',
- artifact,
- '-d',
+ "unzip",
+ str(artifact),
+ "-d",
temp_dir,
- '*/sources/android/cpufeatures/*',
- '*/sources/android/native_app_glue/*',
- '*/sources/android/support/*',
- '*/sources/cxx-stl/*',
- '*/source.properties',
- os.path.join('*', LIBUNWIND_GLOB),
+ "*/sources/android/cpufeatures/*",
+ "*/sources/android/native_app_glue/*",
+ "*/sources/android/support/*",
+ "*/sources/cxx-stl/*",
+ "*/source.properties",
+ os.path.join("*", LIBUNWIND_GLOB),
+ os.path.join("*", LIBCXX_SHARED_GLOB),
+ os.path.join("*", LIBCXX_STATIC_GLOB),
+ os.path.join("*", LIBCXXABI_GLOB),
]
check_call(cmd)
dirs = os.listdir(temp_dir)
assert len(dirs) == 1
- ndk_dir = os.path.join(temp_dir, dirs[0])
- for child in os.listdir(ndk_dir):
- os.rename(os.path.join(ndk_dir, child),
- os.path.join(destination, child))
+ ndk_dir = Path(temp_dir) / dirs[0]
+ for child in ndk_dir.iterdir():
+ child.rename(destination / child.name)
-def relocate_libunwind(install_dir):
- unwinds = glob.glob(os.path.join(install_dir, LIBUNWIND_GLOB))
- dest_base = os.path.join(install_dir, 'sources/cxx-stl/llvm-libc++/libs')
- for libunwind in unwinds:
- arch = os.path.basename(os.path.dirname(libunwind))
+def relocate_libcxx(install_dir: Path) -> None:
+ """Copies the libc++ libraries from the toolchain to sources.
+
+ New versions of the NDK have removed the libraries in the sources directory because
+ they are duplicates and they aren't needed in typical builds. Soong still expects to
+ find them in that directory though. We could fix Soong, but since this whole
+ directory should be dead soon we'll just fix-up the install for now.
+ """
+ dest_base = install_dir / "sources/cxx-stl/llvm-libc++/libs"
+ for glob in {LIBCXX_SHARED_GLOB, LIBCXX_STATIC_GLOB, LIBCXXABI_GLOB}:
+ file_name = Path(glob).name
+ for file_path in install_dir.glob(glob):
+ triple = file_path.parent.name
+ abi = {
+ "arm-linux-androideabi": "armeabi-v7a",
+ "aarch64-linux-android": "arm64-v8a",
+ "i686-linux-android": "x86",
+ "x86_64-linux-android": "x86_64",
+ }[triple]
+ dest_dir = dest_base / abi
+ dest_dir.mkdir(parents=True, exist_ok=True)
+ dest = dest_dir / file_name
+ logger().info("Relocating %s to %s", file_path, dest)
+ file_path.rename(dest)
+
+
+def relocate_libunwind(install_dir: Path) -> None:
+ dest_base = install_dir / "sources/cxx-stl/llvm-libc++/libs"
+ for libunwind in install_dir.glob(LIBUNWIND_GLOB):
+ arch = libunwind.parent.name
abi = {
- 'arm': 'armeabi-v7a',
- 'aarch64': 'arm64-v8a',
- 'i386': 'x86',
- 'x86_64': 'x86_64',
+ "arm": "armeabi-v7a",
+ "aarch64": "arm64-v8a",
+ "i386": "x86",
+ "x86_64": "x86_64",
}[arch]
- dest_dir = os.path.join(dest_base, abi)
- dest = os.path.join(dest_dir, 'libunwind.a')
- logger().info('Relocating %s to %s', libunwind, dest)
- os.rename(libunwind, dest)
+ dest_dir = dest_base / abi
+ dest = dest_dir / "libunwind.a"
+ logger().info("Relocating %s to %s", libunwind, dest)
+ libunwind.rename(dest)
-def install_new_release(branch, build, install_dir):
- os.makedirs(install_dir)
+def delete_android_mks(install_dir: Path) -> None:
+ for android_mk in install_dir.glob("**/Android.mk"):
+ android_mk.unlink()
- artifact_pattern = 'android-ndk-*.zip'
- logger().info('Fetching %s from %s (artifacts matching %s)', build, branch,
- artifact_pattern)
+
+def install_new_release(branch: str, build: str, install_dir: Path) -> None:
+ install_dir.mkdir()
+
+ artifact_pattern = "android-ndk-*.zip"
+ logger().info(
+ "Fetching %s from %s (artifacts matching %s)", build, branch, artifact_pattern
+ )
fetch_artifact(branch, build, artifact_pattern)
- artifacts = glob.glob('android-ndk-*.zip')
+ artifacts = list(Path().glob("android-ndk-*.zip"))
try:
assert len(artifacts) == 1
artifact = artifacts[0]
- logger().info('Extracting release')
+ logger().info("Extracting release")
unzip_single_directory(artifact, install_dir)
+ relocate_libcxx(install_dir)
relocate_libunwind(install_dir)
+ delete_android_mks(install_dir)
finally:
for artifact in artifacts:
- os.unlink(artifact)
+ artifact.unlink()
-def commit(branch, build, install_dir):
- logger().info('Making commit')
- check_call(['git', 'add', install_dir])
- message = textwrap.dedent("""\
+def commit(branch: str, build: str, install_dir: Path) -> None:
+ logger().info("Making commit")
+ check_call(["git", "add", str(install_dir)])
+ message = textwrap.dedent(
+ f"""\
Update NDK prebuilts to build {build}.
Taken from branch {branch}.
Bug: None
Test: treehugger
- """).format(branch=branch, build=build)
- check_call(['git', 'commit', '-m', message])
+ """
+ )
+ check_call(["git", "commit", "-m", message])
def get_args():
parser = argparse.ArgumentParser()
parser.add_argument(
- '-b', '--branch', default='master-ndk',
- help='Branch to pull build from.')
+ "-b", "--branch", default="master-ndk", help="Branch to pull build from."
+ )
+ parser.add_argument("--build", required=True, help="Build number to pull.")
parser.add_argument(
- 'major_release', help='Major release being installed, e.g. "r11".')
- parser.add_argument('--build', required=True, help='Build number to pull.')
+ "--use-current-branch",
+ action="store_true",
+ help="Perform the update in the current branch. Do not repo start.",
+ )
parser.add_argument(
- '--use-current-branch', action='store_true',
- help='Perform the update in the current branch. Do not repo start.')
- parser.add_argument(
- '-v', '--verbose', action='count', default=0,
- help='Increase output verbosity.')
+ "-v", "--verbose", action="count", default=0, help="Increase output verbosity."
+ )
return parser.parse_args()
-def main():
+def main() -> None:
os.chdir(THIS_DIR)
args = get_args()
verbose_map = (logging.WARNING, logging.INFO, logging.DEBUG)
- verbosity = args.verbose
- if verbosity > 2:
- verbosity = 2
+ verbosity = min(args.verbose, 2)
logging.basicConfig(level=verbose_map[verbosity])
- install_dir = os.path.realpath(args.major_release)
+ install_dir = THIS_DIR / "current"
if not args.use_current_branch:
start_branch(args.build)
@@ -187,5 +246,5 @@
commit(args.branch, args.build, install_dir)
-if __name__ == '__main__':
+if __name__ == "__main__":
main()
