| #!/usr/bin/env python |
| # |
| # Copyright (C) 2015 The Android Open Source Project |
| # |
| # Licensed under the Apache License, Version 2.0 (the "License"); |
| # you may not use this file except in compliance with the License. |
| # You may obtain a copy of the License at |
| # |
| # http://www.apache.org/licenses/LICENSE-2.0 |
| # |
| # Unless required by applicable law or agreed to in writing, software |
| # distributed under the License is distributed on an "AS IS" BASIS, |
| # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| # See the License for the specific language governing permissions and |
| # limitations under the License. |
| # |
| """Verifies that the build is sane. |
| |
| Cleans old build artifacts, configures the required environment, determines |
| build goals, and invokes the build scripts. |
| """ |
| from __future__ import print_function |
| |
| import argparse |
| import collections |
| import datetime |
| import inspect |
| import os |
| import shutil |
| import site |
| import subprocess |
| import sys |
| import tempfile |
| import textwrap |
| |
| site.addsitedir(os.path.join(os.path.dirname(__file__), 'build/lib')) |
| |
| import build_support # pylint: disable=import-error |
| |
| |
| ALL_MODULES = { |
| 'binutils', |
| 'build', |
| 'clang', |
| 'cpufeatures', |
| 'gabi++', |
| 'gcc', |
| 'gcclibs', |
| 'gdbserver', |
| 'gnustl', |
| 'gtest', |
| 'host-tools', |
| 'libandroid_support', |
| 'libc++', |
| 'libc++abi', |
| 'native_app_glue', |
| 'ndk_helper', |
| 'platforms', |
| 'python-packages', |
| 'stlport', |
| 'system-stl', |
| } |
| |
| |
| class ArgParser(argparse.ArgumentParser): |
| def __init__(self): |
| super(ArgParser, self).__init__( |
| description=inspect.getdoc(sys.modules[__name__])) |
| |
| self.add_argument( |
| '--arch', |
| choices=('arm', 'arm64', 'mips', 'mips64', 'x86', 'x86_64'), |
| help='Build for the given architecture. Build all by default.') |
| |
| package_group = self.add_mutually_exclusive_group() |
| package_group.add_argument( |
| '--package', action='store_true', dest='package', default=True, |
| help='Package the NDK when done building (default).') |
| package_group.add_argument( |
| '--no-package', action='store_false', dest='package', |
| help='Do not package the NDK when done building.') |
| |
| test_group = self.add_mutually_exclusive_group() |
| test_group.add_argument( |
| '--test', action='store_true', dest='test', default=True, |
| help=textwrap.dedent("""\ |
| Run host tests when finished. --package is required. Not supported |
| when targeting Windows. |
| """)) |
| test_group.add_argument( |
| '--no-test', action='store_false', dest='test', |
| help='Do not run host tests when finished.') |
| |
| self.add_argument( |
| '--release', |
| help='Release name. Package will be named android-ndk-RELEASE.') |
| |
| self.add_argument( |
| '--system', choices=('darwin', 'linux', 'windows', 'windows64'), |
| default=build_support.get_default_host(), |
| help='Build for the given OS.') |
| |
| module_group = self.add_mutually_exclusive_group() |
| |
| module_group.add_argument( |
| '--module', choices=sorted(ALL_MODULES), |
| help='NDK modules to build.') |
| |
| module_group.add_argument( |
| '--host-only', action='store_true', |
| help='Skip building target components.') |
| |
| |
| def _invoke_build(script, args): |
| if args is None: |
| args = [] |
| subprocess.check_call([build_support.android_path(script)] + args) |
| |
| |
| def invoke_build(script, args=None): |
| script_path = os.path.join('build/tools', script) |
| _invoke_build(build_support.ndk_path(script_path), args) |
| |
| |
| def invoke_external_build(script, args=None): |
| _invoke_build(build_support.android_path(script), args) |
| |
| |
| def package_ndk(out_dir, dist_dir, args): |
| package_args = common_build_args(out_dir, dist_dir, args) |
| package_args.append(dist_dir) |
| |
| if args.release is not None: |
| package_args.append('--release={}'.format(args.release)) |
| |
| if args.arch is not None: |
| package_args.append('--arch={}'.format(args.arch)) |
| |
| invoke_build('package.py', package_args) |
| |
| |
| def test_ndk(out_dir, args): |
| release = args.release |
| if args.release is None: |
| release = datetime.date.today().strftime('%Y%m%d') |
| |
| # The packaging step extracts all the modules to a known directory for |
| # packaging. This directory is not cleaned up after packaging, so we can |
| # reuse that for testing. |
| test_dir = os.path.join(out_dir, 'android-ndk-{}'.format(release)) |
| |
| test_env = dict(os.environ) |
| test_env['NDK'] = test_dir |
| |
| abis = build_support.ALL_ABIS |
| if args.arch is not None: |
| abis = build_support.arch_to_abis(args.arch) |
| |
| results = {} |
| for abi in abis: |
| cmd = [ |
| 'python', build_support.ndk_path('tests/run-all.py'), |
| '--abi', abi, '--suite', 'build' |
| ] |
| print('Running tests: {}'.format(' '.join(cmd))) |
| result = subprocess.call(cmd, env=test_env) |
| results[abi] = result == 0 |
| |
| print('Results:') |
| for abi, result in results.iteritems(): |
| print('{}: {}'.format(abi, 'PASS' if result else 'FAIL')) |
| return all(results.values()) |
| |
| |
| def common_build_args(out_dir, dist_dir, args): |
| build_args = ['--out-dir={}'.format(out_dir)] |
| build_args = ['--dist-dir={}'.format(dist_dir)] |
| build_args.append('--host={}'.format(args.system)) |
| return build_args |
| |
| |
| def fixup_toolchain_triple(toolchain): |
| """Maps toolchain names to their proper triple. |
| |
| The x86 toolchains are named stupidly and aren't a proper triple. |
| """ |
| return { |
| 'x86': 'i686-linux-android', |
| 'x86_64': 'x86_64-linux-android', |
| }.get(toolchain, toolchain) |
| |
| |
| def get_binutils_files(triple, has_gold, is_windows): |
| files = [ |
| 'ld.bfd', |
| 'nm', |
| 'as', |
| 'objcopy', |
| 'strip', |
| 'objdump', |
| 'ld', |
| 'ar', |
| 'ranlib', |
| ] |
| |
| if has_gold: |
| files.append('ld.gold') |
| |
| if is_windows: |
| files = [f + '.exe' for f in files] |
| |
| # binutils programs get installed to two locations: |
| # 1: $INSTALL_DIR/bin/$TRIPLE-$PROGRAM |
| # 2: $INSTALL_DIR/$TRIPLE/bin/$PROGRAM |
| # |
| # We need to copy both. |
| |
| prefixed_files = [] |
| for file_name in files: |
| prefixed_name = '-'.join([triple, file_name]) |
| prefixed_files.append(os.path.join('bin', prefixed_name)) |
| |
| dir_prefixed_files = [] |
| for file_name in files: |
| dir_prefixed_files.append(os.path.join(triple, 'bin', file_name)) |
| |
| ldscripts_dir = os.path.join(triple, 'lib/ldscripts') |
| return prefixed_files + dir_prefixed_files + [ldscripts_dir] |
| |
| |
| def install_file(file_name, src_dir, dst_dir): |
| src_file = os.path.join(src_dir, file_name) |
| dst_file = os.path.join(dst_dir, file_name) |
| |
| print('Copying {} to {}...'.format(src_file, dst_file)) |
| if os.path.isdir(src_file): |
| _install_dir(src_file, dst_file) |
| elif os.path.islink(src_file): |
| _install_symlink(src_file, dst_file) |
| else: |
| _install_file(src_file, dst_file) |
| |
| |
| def _install_dir(src_dir, dst_dir): |
| parent_dir = os.path.normpath(os.path.join(dst_dir, '..')) |
| if not os.path.exists(parent_dir): |
| os.makedirs(parent_dir) |
| shutil.copytree(src_dir, dst_dir, symlinks=True) |
| |
| |
| def _install_symlink(src_file, dst_file): |
| dirname = os.path.dirname(dst_file) |
| if not os.path.exists(dirname): |
| os.makedirs(dirname) |
| link_target = os.readlink(src_file) |
| os.symlink(link_target, dst_file) |
| |
| |
| def _install_file(src_file, dst_file): |
| dirname = os.path.dirname(dst_file) |
| if not os.path.exists(dirname): |
| os.makedirs(dirname) |
| # copy2 is just copy followed by copystat (preserves file metadata). |
| shutil.copy2(src_file, dst_file) |
| |
| |
| def pack_binutils(arch, host_tag, out_dir, binutils_path): |
| archive_name = '-'.join(['binutils', arch, host_tag]) |
| build_support.make_package(archive_name, binutils_path, out_dir) |
| |
| |
| def get_prebuilt_gcc(host, arch): |
| tag = build_support.host_to_tag(host) |
| system_subdir = 'prebuilts/ndk/current/toolchains/{}'.format(tag) |
| system_path = build_support.android_path(system_subdir) |
| toolchain = build_support.arch_to_toolchain(arch) |
| toolchain_dir = toolchain + '-4.9' |
| return os.path.join(system_path, toolchain_dir) |
| |
| |
| def build_binutils(out_dir, dist_dir, args): |
| print('Extracting binutils package from GCC...') |
| |
| arches = build_support.ALL_ARCHITECTURES |
| if args.arch is not None: |
| arches = [args.arch] |
| |
| host_tag = build_support.host_to_tag(args.system) |
| |
| for arch in arches: |
| toolchain = build_support.arch_to_toolchain(arch) |
| toolchain_path = get_prebuilt_gcc(args.system, arch) |
| |
| triple = fixup_toolchain_triple(toolchain) |
| |
| install_dir = os.path.join(out_dir, 'binutils', triple) |
| if os.path.exists(install_dir): |
| shutil.rmtree(install_dir) |
| os.makedirs(install_dir) |
| |
| has_gold = True |
| if host_tag == 'windows': |
| # Note: 64-bit Windows is fine. |
| has_gold = False |
| if arch in ('mips', 'mips64'): |
| has_gold = False |
| |
| is_windows = host_tag.startswith('windows') |
| for file_name in get_binutils_files(triple, has_gold, is_windows): |
| install_file(file_name, toolchain_path, install_dir) |
| |
| license_path = build_support.android_path( |
| 'toolchain/binutils/binutils-2.25/COPYING') |
| shutil.copy2(license_path, os.path.join(install_dir, 'NOTICE')) |
| |
| pack_binutils(arch, host_tag, dist_dir, install_dir) |
| |
| |
| def build_clang(out_dir, dist_dir, args): |
| print('Building Clang...') |
| invoke_build('build-llvm.py', common_build_args(out_dir, dist_dir, args)) |
| |
| |
| def build_gcc(out_dir, dist_dir, args): |
| print('Building GCC...') |
| build_args = common_build_args(out_dir, dist_dir, args) |
| if args.arch is not None: |
| build_args.append('--arch={}'.format(args.arch)) |
| invoke_build('build-gcc.py', build_args) |
| |
| |
| def build_gcc_libs(out_dir, dist_dir, args): |
| print('Packaging GCC libs...') |
| |
| arches = build_support.ALL_ARCHITECTURES |
| if args.arch is not None: |
| arches = [args.arch] |
| |
| for arch in arches: |
| toolchain = build_support.arch_to_toolchain(arch) |
| triple = fixup_toolchain_triple(toolchain) |
| libgcc_subdir = 'lib/gcc/{}/4.9'.format(triple) |
| is64 = arch.endswith('64') |
| libatomic_subdir = '{}/lib{}'.format(triple, '64' if is64 else '') |
| |
| lib_names = [ |
| (libatomic_subdir, 'libatomic.a'), |
| (libgcc_subdir, 'libgcc.a'), |
| ] |
| |
| lib_dirs = [''] |
| if arch == 'arm': |
| lib_dirs += [ |
| 'armv7-a', |
| 'armv7-a/hard', |
| 'armv7-a/thumb', |
| 'armv7-a/thumb/hard', |
| 'thumb', |
| ] |
| |
| libs = [] |
| for lib_dir in lib_dirs: |
| for subdir, lib in lib_names: |
| libs.append((subdir, os.path.join(lib_dir, lib))) |
| |
| install_dir = os.path.join(out_dir, 'gcclibs', triple) |
| if os.path.exists(install_dir): |
| shutil.rmtree(install_dir) |
| os.makedirs(install_dir) |
| |
| # These are target libraries, so the OS we use here is not |
| # important. We explicitly use Linux because for whatever reason |
| # the Windows aarch64 toolchain doesn't include libatomic. |
| gcc_path = get_prebuilt_gcc('linux', arch) |
| for gcc_subdir, lib in libs: |
| src = os.path.join(gcc_path, gcc_subdir, lib) |
| dst = os.path.join(install_dir, lib) |
| dst_dir = os.path.dirname(dst) |
| if not os.path.exists(dst_dir): |
| os.makedirs(dst_dir) |
| shutil.copy2(src, dst) |
| |
| shutil.copy2( |
| os.path.join(gcc_path, 'NOTICE'), |
| os.path.join(install_dir, 'NOTICE')) |
| |
| archive_name = os.path.join('gcclibs-' + arch) |
| build_support.make_package(archive_name, install_dir, dist_dir) |
| |
| |
| def build_host_tools(out_dir, dist_dir, args): |
| build_args = common_build_args(out_dir, dist_dir, args) |
| |
| print('Building ndk-stack...') |
| invoke_external_build( |
| 'ndk/sources/host-tools/ndk-stack/build.py', build_args) |
| |
| print('Building ndk-depends...') |
| invoke_external_build( |
| 'ndk/sources/host-tools/ndk-depends/build.py', build_args) |
| |
| print('Building awk...') |
| invoke_external_build( |
| 'ndk/sources/host-tools/nawk-20071023/build.py', build_args) |
| |
| print('Building make...') |
| invoke_external_build( |
| 'ndk/sources/host-tools/make-3.81/build.py', build_args) |
| |
| if args.system in ('windows', 'windows64'): |
| print('Building toolbox...') |
| invoke_external_build( |
| 'ndk/sources/host-tools/toolbox/build.py', build_args) |
| |
| print('Building Python...') |
| invoke_external_build('toolchain/python/build.py', build_args) |
| |
| print('Building GDB...') |
| invoke_external_build('toolchain/gdb/build.py', build_args) |
| |
| print('Building YASM...') |
| invoke_external_build('toolchain/yasm/build.py', build_args) |
| |
| package_host_tools(out_dir, dist_dir, args.system) |
| |
| |
| def merge_license_files(output_path, files): |
| licenses = [] |
| for license_path in files: |
| with open(license_path) as license_file: |
| licenses.append(license_file.read()) |
| |
| with open(output_path, 'w') as output_file: |
| output_file.write('\n'.join(licenses)) |
| |
| |
| def package_host_tools(out_dir, dist_dir, host): |
| packages = [ |
| 'gdb-multiarch-7.10', |
| 'ndk-awk', |
| 'ndk-depends', |
| 'ndk-make', |
| 'ndk-python', |
| 'ndk-stack', |
| 'ndk-yasm', |
| ] |
| |
| files = [ |
| 'ndk-gdb', |
| 'ndk-gdb.cmd', |
| 'ndk-gdb.py', |
| ] |
| |
| if host in ('windows', 'windows64'): |
| packages.append('toolbox') |
| |
| host_tag = build_support.host_to_tag(host) |
| |
| package_names = [p + '-' + host_tag + '.tar.bz2' for p in packages] |
| for package_name in package_names: |
| package_path = os.path.join(out_dir, package_name) |
| subprocess.check_call(['tar', 'xf', package_path, '-C', out_dir]) |
| |
| for f in files: |
| shutil.copy2(f, os.path.join(out_dir, 'host-tools/bin')) |
| |
| merge_license_files(os.path.join(out_dir, 'host-tools/NOTICE'), [ |
| build_support.android_path('toolchain/gdb/gdb-7.10/COPYING'), |
| build_support.ndk_path('sources/host-tools/nawk-20071023/NOTICE'), |
| build_support.ndk_path('sources/host-tools/ndk-depends/NOTICE'), |
| build_support.ndk_path('sources/host-tools/make-3.81/COPYING'), |
| build_support.android_path( |
| 'toolchain/python/Python-2.7.5/LICENSE'), |
| build_support.ndk_path('sources/host-tools/ndk-stack/NOTICE'), |
| build_support.ndk_path('sources/host-tools/toolbox/NOTICE'), |
| build_support.android_path('toolchain/yasm/COPYING'), |
| build_support.android_path('toolchain/yasm/BSD.txt'), |
| build_support.android_path('toolchain/yasm/Artistic.txt'), |
| build_support.android_path('toolchain/yasm/GNU_GPL-2.0'), |
| build_support.android_path('toolchain/yasm/GNU_LGPL-2.0'), |
| ]) |
| |
| package_name = 'host-tools-' + host_tag |
| path = os.path.join(out_dir, 'host-tools') |
| build_support.make_package(package_name, path, dist_dir) |
| |
| |
| def build_gdbserver(out_dir, dist_dir, args): |
| print('Building gdbserver...') |
| build_args = common_build_args(out_dir, dist_dir, args) |
| if args.arch is not None: |
| build_args.append('--arch={}'.format(args.arch)) |
| invoke_build('build-gdbserver.py', build_args) |
| |
| |
| def _build_stl(out_dir, dist_dir, args, stl): |
| build_args = common_build_args(out_dir, dist_dir, args) |
| if args.arch is not None: |
| build_args.append('--arch={}'.format(args.arch)) |
| script = 'ndk/sources/cxx-stl/{}/build.py'.format(stl) |
| invoke_external_build(script, build_args) |
| |
| |
| def build_gnustl(out_dir, dist_dir, args): |
| print('Building gnustl...') |
| _build_stl(out_dir, dist_dir, args, 'gnu-libstdc++') |
| |
| |
| def build_libcxx(out_dir, dist_dir, args): |
| print('Building libc++...') |
| _build_stl(out_dir, dist_dir, args, 'llvm-libc++') |
| |
| |
| def build_stlport(out_dir, dist_dir, args): |
| print('Building stlport...') |
| _build_stl(out_dir, dist_dir, args, 'stlport') |
| |
| |
| def build_platforms(out_dir, dist_dir, args): |
| print('Building platforms...') |
| build_args = common_build_args(out_dir, dist_dir, args) |
| invoke_build('build-platforms.py', build_args) |
| |
| |
| def build_cpufeatures(_, dist_dir, __): |
| path = build_support.ndk_path('sources/android/cpufeatures') |
| build_support.make_package('cpufeatures', path, dist_dir) |
| |
| |
| def build_native_app_glue(_, dist_dir, __): |
| path = build_support.android_path( |
| 'development/ndk/sources/android/native_app_glue') |
| build_support.make_package('native_app_glue', path, dist_dir) |
| |
| |
| def build_ndk_helper(_, dist_dir, __): |
| path = build_support.android_path( |
| 'development/ndk/sources/android/ndk_helper') |
| build_support.make_package('ndk_helper', path, dist_dir) |
| |
| |
| def build_gtest(_, dist_dir, __): |
| path = build_support.ndk_path('sources/third_party/googletest') |
| build_support.make_package('gtest', path, dist_dir) |
| |
| |
| def build_build(_, dist_dir, __): |
| path = build_support.ndk_path('build') |
| build_support.make_package('build', path, dist_dir) |
| |
| |
| def build_python_packages(_, dist_dir, __): |
| # Stage the files in a temporary directory to make things easier. |
| temp_dir = tempfile.mkdtemp() |
| try: |
| path = os.path.join(temp_dir, 'python-packages') |
| shutil.copytree( |
| build_support.android_path('development/python-packages'), path) |
| build_support.make_package('python-packages', path, dist_dir) |
| finally: |
| shutil.rmtree(temp_dir) |
| |
| |
| def build_gabixx(_out_dir, dist_dir, _args): |
| print('Building gabi++...') |
| path = build_support.ndk_path('sources/cxx-stl/gabi++') |
| build_support.make_package('gabixx', path, dist_dir) |
| |
| |
| def build_system_stl(_out_dir, dist_dir, _args): |
| print('Building system-stl...') |
| path = build_support.ndk_path('sources/cxx-stl/system') |
| build_support.make_package('system-stl', path, dist_dir) |
| |
| |
| def build_libandroid_support(_out_dir, dist_dir, _args): |
| print('Building libandroid_support...') |
| path = build_support.ndk_path('sources/android/support') |
| build_support.make_package('libandroid_support', path, dist_dir) |
| |
| |
| def build_libcxxabi(_out_dir, dist_dir, _args): |
| print('Building libc++abi...') |
| path = build_support.ndk_path('sources/cxx-stl/llvm-libc++abi') |
| build_support.make_package('libcxxabi', path, dist_dir) |
| |
| |
| def main(): |
| parser = ArgParser() |
| args = parser.parse_args() |
| |
| if args.module is None: |
| modules = ALL_MODULES |
| else: |
| modules = {args.module} |
| |
| if args.host_only: |
| modules = { |
| 'clang', |
| 'gcc', |
| 'host-tools', |
| } |
| |
| required_package_modules = ALL_MODULES |
| if args.package and required_package_modules <= modules: |
| do_package = True |
| else: |
| do_package = False |
| |
| # TODO(danalbert): wine? |
| # We're building the Windows packages from Linux, so we can't actually run |
| # any of the tests from here. |
| if args.system.startswith('windows') or not do_package: |
| args.test = False |
| |
| # Disable buffering on stdout so the build output doesn't hide all of our |
| # "Building..." messages. |
| sys.stdout = os.fdopen(sys.stdout.fileno(), 'w', 0) |
| |
| os.chdir(os.path.dirname(os.path.realpath(__file__))) |
| |
| # Set ANDROID_BUILD_TOP. |
| if 'ANDROID_BUILD_TOP' not in os.environ: |
| os.environ['ANDROID_BUILD_TOP'] = os.path.realpath('..') |
| |
| out_dir = build_support.get_out_dir() |
| dist_dir = build_support.get_dist_dir(out_dir) |
| |
| print('Cleaning up...') |
| invoke_build('dev-cleanup.sh') |
| |
| module_builds = collections.OrderedDict([ |
| ('binutils', build_binutils), |
| ('build', build_build), |
| ('clang', build_clang), |
| ('cpufeatures', build_cpufeatures), |
| ('gabi++', build_gabixx), |
| ('gcc', build_gcc), |
| ('gcclibs', build_gcc_libs), |
| ('gdbserver', build_gdbserver), |
| ('gnustl', build_gnustl), |
| ('gtest', build_gtest), |
| ('host-tools', build_host_tools), |
| ('libandroid_support', build_libandroid_support), |
| ('libc++', build_libcxx), |
| ('libc++abi', build_libcxxabi), |
| ('native_app_glue', build_native_app_glue), |
| ('ndk_helper', build_ndk_helper), |
| ('platforms', build_platforms), |
| ('python-packages', build_python_packages), |
| ('stlport', build_stlport), |
| ('system-stl', build_system_stl), |
| ]) |
| |
| print('Building modules: {}'.format(' '.join(modules))) |
| for module in modules: |
| module_builds[module](out_dir, dist_dir, args) |
| |
| if do_package: |
| package_ndk(out_dir, dist_dir, args) |
| |
| if args.test: |
| result = test_ndk(out_dir, args) |
| sys.exit(0 if result else 1) |
| |
| |
| if __name__ == '__main__': |
| main() |
