| # |
| # Copyright (C) 2020 The Android Open Source Project |
| # |
| # Licensed under the Apache License, Version 2.0 (the "License"); |
| # you may not use this file except in compliance with the License. |
| # You may obtain a copy of the License at |
| # |
| # http://www.apache.org/licenses/LICENSE-2.0 |
| # |
| # Unless required by applicable law or agreed to in writing, software |
| # distributed under the License is distributed on an "AS IS" BASIS, |
| # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| # See the License for the specific language governing permissions and |
| # limitations under the License. |
| |
| """ |
| Package to manage LLVM sources when building a toolchain. |
| """ |
| |
| import logging |
| from pathlib import Path |
| import os |
| import re |
| import shutil |
| import string |
| import subprocess |
| import sys |
| |
| import android_version |
| import hosts |
| import paths |
| import utils |
| |
| |
| def logger(): |
| """Returns the module level logger.""" |
| return logging.getLogger(__name__) |
| |
| |
| def apply_patches(source_dir, svn_version, patch_json, patch_dir, |
| failure_mode='fail'): |
| """Apply patches in $patch_dir/$patch_json to $source_dir. |
| |
| Invokes external/toolchain-utils/llvm_tools/patch_manager.py to apply the |
| patches. |
| """ |
| |
| patch_manager_cmd = [ |
| sys.executable, |
| str(paths.TOOLCHAIN_UTILS_DIR / 'llvm_tools' / 'patch_manager.py'), |
| '--svn_version', str(svn_version), |
| '--patch_metadata_file', str(patch_json), |
| '--filesdir_path', str(patch_dir), |
| '--src_path', str(source_dir), |
| '--use_src_head', |
| '--failure_mode', failure_mode |
| ] |
| |
| return utils.check_output(patch_manager_cmd) |
| |
| |
| def get_base_llvm_revision(source_dir: str) -> str: |
| message = utils.check_output([ |
| 'git', f'--git-dir={source_dir}/.git', 'log', '--pretty=format:%s', |
| '--merges', '-n', '1' |
| ]) |
| return message.split()[1] |
| |
| |
| def get_scripts_sha() -> str: |
| return utils.check_output( |
| ['git', f'--git-dir={paths.SCRIPTS_DIR}/.git', 'rev-parse', |
| 'HEAD']).strip() |
| |
| |
| def write_source_info(source_dir: str, patch_output: str) -> None: |
| lowercase_hexdigits = '0123456789abcdef' |
| url_prefix = 'https://android.googlesource.com/toolchain/llvm_android/+/' +\ |
| get_scripts_sha() |
| |
| def _get_subject(patch_file): |
| with open(patch_file) as pf: |
| contents = pf.read() |
| matches = re.search('Subject: (.*)\n', contents) |
| if not matches: |
| # Patch doesn't have a subject line. Fallback to the file's |
| # basename. |
| return patch_file.name |
| |
| subject = matches.groups()[0] |
| trim_str = '[PATCH] ' |
| if subject.startswith(trim_str): |
| subject = subject[len(trim_str):] |
| return subject |
| |
| |
| def _format_patch_line(patch): |
| if all(c in lowercase_hexdigits for c in patch[:-len('.patch')]): |
| url_suffix = '/patches/cherry/' + patch |
| link_text = _get_subject(paths.SCRIPTS_DIR / 'patches' / 'cherry' / patch) |
| else: |
| url_suffix = '/patches/' + patch |
| link_text = patch |
| return f'- [{link_text}]({url_prefix}{url_suffix})' |
| |
| output = [] |
| base_revision = get_base_llvm_revision(source_dir) |
| github_url = 'https://github.com/llvm/llvm-project/commits/' + base_revision |
| output.append(f'Base revision: [{base_revision}]({github_url})') |
| output.append('') |
| |
| patches = patch_output.strip().splitlines() |
| patches_iter = iter(patches) |
| assert next(patches_iter) == 'The following patches applied successfully:' |
| while True: |
| patch = next(patches_iter, None) |
| # We may optionally have an empty line followed by patches that were not |
| # applicable. |
| if patch == '': |
| assert next( |
| patches_iter) == 'The following patches were not applicable:' |
| break |
| elif patch is None: |
| break |
| assert patch.endswith('.patch') |
| output.append(_format_patch_line(patch)) |
| |
| with open(paths.OUT_DIR / 'clang_source_info.md', 'w') as outfile: |
| outfile.write('\n'.join(output)) |
| |
| |
| def setup_sources(source_dir): |
| """Setup toolchain sources into source_dir. |
| |
| Copy toolchain/llvm-project into source_dir. |
| Apply patches per the specification in |
| toolchain/llvm_android/patches/PATCHES.json. The function overwrites |
| source_dir only if necessary to avoid recompiles during incremental builds. |
| """ |
| |
| copy_from = paths.TOOLCHAIN_LLVM_PATH |
| |
| # Copy llvm source tree to a temporary directory. |
| tmp_source_dir = source_dir.parent / (source_dir.name + '.tmp') |
| if os.path.exists(tmp_source_dir): |
| shutil.rmtree(tmp_source_dir) |
| |
| # mkdir parent of tmp_source_dir if necessary - so we can call 'cp' below. |
| tmp_source_parent = os.path.dirname(tmp_source_dir) |
| if not os.path.exists(tmp_source_parent): |
| os.makedirs(tmp_source_parent) |
| |
| # Use 'cp' instead of shutil.copytree. The latter uses copystat and retains |
| # timestamps from the source. We instead use rsync below to only update |
| # changed files into source_dir. Using 'cp' will ensure all changed files |
| # get a newer timestamp than files in $source_dir. |
| # Note: Darwin builds don't copy symlinks with -r. Use -R instead. |
| reflink = '--reflink=auto' if hosts.build_host().is_linux else '-c' |
| try: |
| cmd = ['cp', '-Rf', reflink, copy_from, tmp_source_dir] |
| subprocess.check_call(cmd) |
| except subprocess.CalledProcessError: |
| # Fallback to normal copy. |
| cmd = ['cp', '-Rf', copy_from, tmp_source_dir] |
| subprocess.check_call(cmd) |
| |
| # patch source tree |
| patch_dir = paths.SCRIPTS_DIR / 'patches' |
| patch_json = os.path.join(patch_dir, 'PATCHES.json') |
| svn_version = android_version.get_svn_revision_number() |
| |
| patch_output = apply_patches(tmp_source_dir, svn_version, patch_json, |
| patch_dir) |
| logger().info(patch_output) |
| write_source_info(tmp_source_dir, patch_output) |
| |
| # Copy tmp_source_dir to source_dir if they are different. This avoids |
| # invalidating prior build outputs. |
| if not os.path.exists(source_dir): |
| os.rename(tmp_source_dir, source_dir) |
| else: |
| # Without a trailing '/' in $SRC, rsync copies $SRC to |
| # $DST/BASENAME($SRC) instead of $DST. |
| tmp_source_dir_str = str(tmp_source_dir) + '/' |
| |
| # rsync to update only changed files. Use '-c' to use checksums to find |
| # if files have changed instead of only modification time and size - |
| # which could have inconsistencies. Use '--delete' to ensure files not |
| # in tmp_source_dir are deleted from $source_dir. |
| subprocess.check_call(['rsync', '-r', '--delete', '--links', '-c', |
| tmp_source_dir_str, source_dir]) |
| |
| shutil.rmtree(tmp_source_dir) |
| remote, url = try_set_git_remote(source_dir) |
| logger().info(f'git remote url: remote: {remote} url: {url}') |
| |
| |
| def try_set_git_remote(source_dir): |
| AOSP_URL = 'https://android.googlesource.com/toolchain/llvm-project' |
| |
| def get_git_remote_url(remote=None): |
| try: |
| if not remote: |
| remote = utils.check_output([ |
| 'git', 'rev-parse', '--abbrev-ref', '--symbolic-full-name', |
| '@{upstream}' |
| ]).strip() |
| remote = remote.split('/')[0] |
| url = utils.check_output(['git', 'remote', 'get-url', |
| remote]).strip() |
| return (remote, url) |
| except: |
| return (remote, None) |
| |
| git_dir = source_dir / '.git' |
| if not git_dir.is_dir(): |
| return (None, None) |
| |
| with utils.chdir_context(git_dir): |
| remote, url = get_git_remote_url() |
| if url != AOSP_URL: |
| if not remote: |
| remote = 'origin' |
| try: |
| if Path('config').is_symlink(): |
| link = utils.check_output(['readlink', 'config']).strip() |
| utils.check_call(['rm', '-f', 'config']) |
| utils.check_call(['cp', link, 'config']) |
| if remote: |
| utils.check_call( |
| ['git', 'remote', 'set-url', remote, AOSP_URL]) |
| else: |
| utils.check_call(['git', 'remote', 'add', remote, AOSP_URL]) |
| remote, url = get_git_remote_url(remote) |
| except: |
| pass |
| return (remote, url) |