| #!/usr/bin/env python3 |
| # |
| # Copyright (C) 2020 The Android Open Source Project |
| # |
| # Licensed under the Apache License, Version 2.0 (the "License"); |
| # you may not use this file except in compliance with the License. |
| # You may obtain a copy of the License at |
| # |
| # http://www.apache.org/licenses/LICENSE-2.0 |
| # |
| # Unless required by applicable law or agreed to in writing, software |
| # distributed under the License is distributed on an "AS IS" BASIS, |
| # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| # See the License for the specific language governing permissions and |
| # limitations under the License. |
| # |
| """Unzips and installs the vendor snapshot.""" |
| |
| import argparse |
| import glob |
| import logging |
| import os |
| import re |
| import shutil |
| import subprocess |
| import sys |
| import tempfile |
| import textwrap |
| import json |
| |
| INDENT = ' ' * 4 |
| |
| |
| def get_notice_path(module_name): |
| return os.path.join('NOTICE_FILES', module_name + '.txt') |
| |
| |
| def get_target_arch(json_rel_path): |
| return json_rel_path.split('/')[0] |
| |
| |
| def get_arch(json_rel_path): |
| return json_rel_path.split('/')[1].split('-')[1] |
| |
| |
| def get_variation(json_rel_path): |
| return json_rel_path.split('/')[2] |
| |
| |
| # convert .bp prop dictionary to .bp prop string |
| def gen_bp_prop(prop, ind): |
| bp = '' |
| for key in prop: |
| val = prop[key] |
| |
| # Skip empty list or dict, rather than printing empty prop like |
| # "key: []," or "key: {}," |
| if type(val) == list or type(val) == dict: |
| if len(val) == 0: |
| continue |
| |
| bp += ind + key + ': ' |
| if type(val) == bool: |
| bp += 'true,\n' if val else 'false,\n' |
| elif type(val) == str: |
| bp += '"%s",\n' % val |
| elif type(val) == list: |
| bp += '[\n' |
| for elem in val: |
| bp += ind + INDENT + '"%s",\n' % elem |
| bp += ind + '],\n' |
| elif type(val) == dict: |
| bp += '{\n' |
| bp += gen_bp_prop(val, ind + INDENT) |
| bp += ind + '},\n' |
| else: |
| raise TypeError('unsupported type %s for gen_bp_prop' % type(val)) |
| return bp |
| |
| |
| # Remove non-existent dirs from given list. Emits warning for such dirs. |
| def remove_invalid_dirs(paths, bp_dir, module_name): |
| ret = [] |
| for path in paths: |
| if os.path.isdir(os.path.join(bp_dir, path)): |
| ret.append(path) |
| else: |
| logging.warning('Dir "%s" of module "%s" does not exist', path, |
| module_name) |
| return ret |
| |
| |
| JSON_TO_BP = { |
| 'ModuleName': 'name', |
| 'RelativeInstallPath': 'relative_install_path', |
| 'ExportedDirs': 'export_include_dirs', |
| 'ExportedSystemDirs': 'export_system_include_dirs', |
| 'ExportedFlags': 'export_flags', |
| 'Sanitize': 'sanitize', |
| 'SanitizeMinimalDep': 'sanitize_minimal_dep', |
| 'SanitizeUbsanDep': 'sanitize_ubsan_dep', |
| 'Symlinks': 'symlinks', |
| 'InitRc': 'init_rc', |
| 'VintfFragments': 'vintf_fragments', |
| 'SharedLibs': 'shared_libs', |
| 'RuntimeLibs': 'runtime_libs', |
| 'Required': 'required', |
| } |
| |
| SANITIZER_VARIANT_PROPS = { |
| 'export_include_dirs', |
| 'export_system_include_dirs', |
| 'export_flags', |
| 'sanitize_minimal_dep', |
| 'sanitize_ubsan_dep', |
| 'src', |
| } |
| |
| |
| # Converts parsed json dictionary (which is intermediate) to Android.bp prop |
| # dictionary. This validates paths such as include directories and init_rc |
| # files while converting. |
| def convert_json_to_bp_prop(json_path, bp_dir): |
| prop = json.load(json_path) |
| ret = {} |
| |
| module_name = prop['ModuleName'] |
| ret['name'] = module_name |
| |
| # Soong will complain about non-existing paths on Android.bp. There might |
| # be missing files among generated header files, so check all exported |
| # directories and filter out invalid ones. Emits warning for such dirs. |
| # TODO: fix soong to track all generated headers correctly |
| for key in {'ExportedDirs', 'ExportedSystemDirs'}: |
| if key in prop: |
| prop[key] = remove_invalid_dirs(prop[key], bp_dir, module_name) |
| |
| for key in prop: |
| if key in JSON_TO_BP: |
| ret[JSON_TO_BP[key]] = prop[key] |
| else: |
| logging.warning('Unknown prop "%s" of module "%s"', key, |
| module_name) |
| |
| return ret |
| |
| |
| def gen_bp_module(image, variation, name, version, target_arch, arch_props, bp_dir): |
| prop = { |
| # These three are common for all snapshot modules. |
| 'version': str(version), |
| 'target_arch': target_arch, |
| image: True, |
| 'arch': {}, |
| } |
| |
| # Factor out common prop among architectures to minimize Android.bp. |
| common_prop = None |
| for arch in arch_props: |
| if common_prop is None: |
| common_prop = dict() |
| for k in arch_props[arch]: |
| common_prop[k] = arch_props[arch][k] |
| continue |
| for k in list(common_prop.keys()): |
| if k not in arch_props[arch] or common_prop[k] != arch_props[arch][k]: |
| del common_prop[k] |
| |
| # Some keys has to be arch_props to prevent 32-bit only modules from being |
| # used as 64-bit modules, and vice versa. |
| for arch_prop_key in ['src', 'cfi']: |
| if arch_prop_key in common_prop: |
| del common_prop[arch_prop_key] |
| prop.update(common_prop) |
| |
| stem32 = stem64 = '' |
| |
| for arch in arch_props: |
| for k in common_prop: |
| if k in arch_props[arch]: |
| del arch_props[arch][k] |
| prop['arch'][arch] = arch_props[arch] |
| # Record stem for executable binary snapshots. |
| # We don't check existence of 'src'; src must exist for executables |
| if variation == 'binary': |
| if '64' in arch: # arm64, x86_64 |
| stem64 = os.path.basename(arch_props[arch]['src']) |
| else: |
| stem32 = os.path.basename(arch_props[arch]['src']) |
| |
| # For binary snapshots, compile_multilib must be assigned to 'both' |
| # in order to install both. Prefer 64bit if their stem collide and |
| # installing both is impossible |
| if variation == 'binary': |
| if stem32 and stem64: |
| if stem32 == stem64: |
| prop['compile_multilib'] = 'first' |
| else: |
| prop['compile_multilib'] = 'both' |
| elif stem32: |
| prop['compile_multilib'] = '32' |
| elif stem64: |
| prop['compile_multilib'] = '64' |
| |
| bp = '%s_snapshot_%s {\n' % (image, variation) |
| bp += gen_bp_prop(prop, INDENT) |
| bp += '}\n\n' |
| return bp |
| |
| |
| def build_props(install_dir): |
| # props[target_arch]["static"|"shared"|"binary"|"header"][name][arch] : json |
| props = dict() |
| |
| # {target_arch}/{arch}/{variation}/{module}.json |
| for root, _, files in os.walk(install_dir, followlinks = True): |
| for file_name in sorted(files): |
| if not file_name.endswith('.json'): |
| continue |
| full_path = os.path.join(root, file_name) |
| rel_path = os.path.relpath(full_path, install_dir) |
| |
| target_arch = get_target_arch(rel_path) |
| arch = get_arch(rel_path) |
| variation = get_variation(rel_path) |
| bp_dir = os.path.join(install_dir, target_arch) |
| |
| if not target_arch in props: |
| props[target_arch] = dict() |
| if not variation in props[target_arch]: |
| props[target_arch][variation] = dict() |
| |
| with open(full_path, 'r') as f: |
| prop = convert_json_to_bp_prop(f, bp_dir) |
| # Remove .json after parsing? |
| # os.unlink(full_path) |
| |
| if variation != 'header': |
| prop['src'] = os.path.relpath( |
| rel_path[:-5], # removing .json |
| target_arch) |
| |
| module_name = prop['name'] |
| |
| # Is this sanitized variant? |
| if 'sanitize' in prop: |
| sanitizer_type = prop['sanitize'] |
| # module_name is {name}.{sanitizer_type}; trim sanitizer_type |
| module_name = module_name[:-len(sanitizer_type) - 1] |
| # Only leave props for the sanitize variant |
| for k in list(prop.keys()): |
| if not k in SANITIZER_VARIANT_PROPS: |
| del prop[k] |
| prop = {'name': module_name, sanitizer_type: prop} |
| |
| notice_path = 'NOTICE_FILES/' + module_name + '.txt' |
| if os.path.exists(os.path.join(bp_dir, notice_path)): |
| prop['notice'] = notice_path |
| |
| variation_dict = props[target_arch][variation] |
| if not module_name in variation_dict: |
| variation_dict[module_name] = dict() |
| if not arch in variation_dict[module_name]: |
| variation_dict[module_name][arch] = prop |
| else: |
| variation_dict[module_name][arch].update(prop) |
| |
| return props |
| |
| |
| def gen_bp_files(image, install_dir, snapshot_version): |
| props = build_props(install_dir) |
| |
| for target_arch in sorted(props): |
| androidbp = '' |
| bp_dir = os.path.join(install_dir, target_arch) |
| for variation in sorted(props[target_arch]): |
| for name in sorted(props[target_arch][variation]): |
| androidbp += gen_bp_module(image, variation, name, |
| snapshot_version, target_arch, |
| props[target_arch][variation][name], |
| bp_dir) |
| with open(os.path.join(bp_dir, 'Android.bp'), 'w') as f: |
| logging.info('Generating Android.bp to: {}'.format(f.name)) |
| f.write(androidbp) |
| |
| |
| def check_call(cmd): |
| logging.debug('Running `{}`'.format(' '.join(cmd))) |
| subprocess.check_call(cmd) |
| |
| |
| def fetch_artifact(branch, build, target, pattern, destination): |
| """Fetches build artifacts from Android Build server. |
| |
| Args: |
| branch: string, branch to pull build artifacts from |
| build: string, build number to pull build artifacts from |
| target: string, target name to pull build artifacts from |
| pattern: string, pattern of build artifact file name |
| destination: string, destination to pull build artifact to |
| """ |
| fetch_artifact_path = '/google/data/ro/projects/android/fetch_artifact' |
| cmd = [ |
| fetch_artifact_path, '--branch', branch, '--target', target, '--bid', |
| build, pattern, destination |
| ] |
| check_call(cmd) |
| |
| def install_artifacts(image, branch, build, target, local_dir, symlink, |
| install_dir): |
| """Installs vendor snapshot build artifacts to {install_dir}/v{version}. |
| |
| 1) Fetch build artifacts from Android Build server or from local_dir |
| 2) Unzip or create symlinks to build artifacts |
| |
| Args: |
| image: string, img file for which the snapshot was created (vendor, |
| recovery, etc.) |
| branch: string or None, branch name of build artifacts |
| build: string or None, build number of build artifacts |
| target: string or None, target name of build artifacts |
| local_dir: string or None, local dir to pull artifacts from |
| symlink: boolean, whether to use symlinks instead of unzipping the |
| vendor snapshot zip |
| install_dir: string, directory to install vendor snapshot |
| temp_artifact_dir: string, temp directory to hold build artifacts fetched |
| from Android Build server. For 'local' option, is set to None. |
| """ |
| artifact_pattern = image + '-*.zip' |
| |
| def unzip_artifacts(artifact_dir): |
| artifacts = glob.glob(os.path.join(artifact_dir, artifact_pattern)) |
| for artifact in artifacts: |
| logging.info('Unzipping Vendor snapshot: {}'.format(artifact)) |
| check_call(['unzip', '-qn', artifact, '-d', install_dir]) |
| |
| if branch and build and target: |
| with tempfile.TemporaryDirectory() as tmpdir: |
| logging.info( |
| 'Fetching {pattern} from {branch} (bid: {build}, target: {target})' |
| .format( |
| pattern=artifact_pattern, |
| branch=branch, |
| build=build, |
| target=target)) |
| fetch_artifact(branch, build, target, artifact_pattern, tmpdir) |
| unzip_artifacts(tmpdir) |
| elif local_dir: |
| if symlink: |
| # This assumes local_dir is the location of vendor-snapshot in the |
| # build (e.g., out/soong/vendor-snapshot). |
| # |
| # Create the first level as proper directories and the next level |
| # as symlinks. |
| for item1 in os.listdir(local_dir): |
| dest_dir = os.path.join(install_dir, item1) |
| src_dir = os.path.join(local_dir, item1) |
| if os.path.isdir(src_dir): |
| check_call(['mkdir', '-p', dest_dir]) |
| # Create symlinks. |
| for item2 in os.listdir(src_dir): |
| src_item = os.path.join(src_dir, item2) |
| logging.info('Creating symlink from {} in {}'.format( |
| src_item, dest_dir)) |
| os.symlink(src_item, os.path.join(dest_dir, item2)) |
| else: |
| logging.info('Fetching local VNDK snapshot from {}'.format( |
| local_dir)) |
| unzip_artifacts(local_dir) |
| else: |
| raise RuntimeError('Neither local nor remote fetch information given.') |
| |
| def get_args(): |
| parser = argparse.ArgumentParser() |
| parser.add_argument( |
| 'snapshot_version', |
| type=int, |
| help='Vendor snapshot version to install, e.g. "30".') |
| parser.add_argument( |
| '--image', |
| help=('Image whose snapshot is being updated (e.g., vendor, ' |
| 'recovery , ramdisk, etc.)'), |
| default='vendor') |
| parser.add_argument('--branch', help='Branch to pull build from.') |
| parser.add_argument('--build', help='Build number to pull.') |
| parser.add_argument('--target', help='Target to pull.') |
| parser.add_argument( |
| '--local', |
| help=('Fetch local vendor snapshot artifacts from specified local ' |
| 'directory instead of Android Build server. ' |
| 'Example: --local /path/to/local/dir')) |
| parser.add_argument( |
| '--symlink', |
| action='store_true', |
| help='Use symlinks instead of unzipping vendor snapshot zip') |
| parser.add_argument( |
| '--install-dir', |
| help=( |
| 'Base directory to which vendor snapshot artifacts are installed. ' |
| 'Example: --install-dir vendor/<company name>/vendor_snapshot/v30')) |
| parser.add_argument( |
| '--overwrite', |
| action='store_true', |
| help=( |
| 'If provided, does not ask before overwriting the install-dir.')) |
| |
| parser.add_argument( |
| '-v', |
| '--verbose', |
| action='count', |
| default=0, |
| help='Increase output verbosity, e.g. "-v", "-vv".') |
| return parser.parse_args() |
| |
| |
| def main(): |
| """Program entry point.""" |
| args = get_args() |
| |
| local = None |
| if args.local: |
| local = os.path.expanduser(args.local) |
| |
| if local: |
| if args.build or args.branch or args.target: |
| raise ValueError( |
| 'When --local option is set, --branch, --build or --target cannot be ' |
| 'specified.') |
| elif not os.path.isdir(local): |
| raise RuntimeError( |
| 'The specified local directory, {}, does not exist.'.format( |
| local)) |
| else: |
| if not (args.build and args.branch and args.target): |
| raise ValueError( |
| 'Please provide --branch, --build and --target. Or set --local ' |
| 'option.') |
| |
| if not args.install_dir: |
| raise ValueError('Please provide --install-dir option.') |
| |
| snapshot_version = args.snapshot_version |
| |
| verbose_map = (logging.WARNING, logging.INFO, logging.DEBUG) |
| verbosity = min(args.verbose, 2) |
| logging.basicConfig( |
| format='%(levelname)-8s [%(filename)s:%(lineno)d] %(message)s', |
| level=verbose_map[verbosity]) |
| |
| install_dir = os.path.expanduser(args.install_dir) |
| if os.path.exists(install_dir): |
| def remove_dir(): |
| logging.info('Removing {}'.format(install_dir)) |
| check_call(['rm', '-rf', install_dir]) |
| if args.overwrite: |
| remove_dir() |
| else: |
| resp = input('Directory {} already exists. IT WILL BE REMOVED.\n' |
| 'Are you sure? (yes/no): '.format(install_dir)) |
| if resp == 'yes': |
| remove_dir() |
| elif resp == 'no': |
| logging.info('Cancelled snapshot install.') |
| return |
| else: |
| raise ValueError('Did not understand: ' + resp) |
| check_call(['mkdir', '-p', install_dir]) |
| |
| install_artifacts( |
| image=args.image, |
| branch=args.branch, |
| build=args.build, |
| target=args.target, |
| local_dir=local, |
| symlink=args.symlink, |
| install_dir=install_dir) |
| gen_bp_files(args.image, install_dir, snapshot_version) |
| |
| if __name__ == '__main__': |
| main() |
