blob: 41a0b6df4d9bb43691bbfa302afd88522e189cf7 [file] [log] [blame]
#!/usr/bin/env python3
#
# Copyright (C) 2020 The Android Open Source Project
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
from __future__ import annotations
import argparse
import collections
import copy
import dataclasses
from dataclasses import dataclass
import json
import logging
import math
import os
from pathlib import Path
import re
import sys
from typing import Any, Dict, List, Optional, Tuple
import urllib.request
from android_version import get_svn_revision_number
from merge_from_upstream import fetch_upstream, sha_to_revision
import paths
import source_manager
from utils import check_call, check_output
def parse_args():
parser = argparse.ArgumentParser(description="Cherry pick upstream LLVM patches.",
formatter_class=argparse.ArgumentDefaultsHelpFormatter)
parser.add_argument('--sha', nargs='+', help='sha of patches to cherry pick')
parser.add_argument('--pr', help='Cherry pick from a GitHub PR, e.g., 84422')
parser.add_argument(
'--start-version', default='llvm',
help="""svn revision to start applying patches. 'llvm' can also be used.""")
parser.add_argument('--verify-merge', action='store_true',
help='check if patches can be applied cleanly')
parser.add_argument('--create-cl', action='store_true', help='create a CL')
parser.add_argument('--bug', help='bug to reference in CLs created (if any)')
parser.add_argument('--reason', help='issue/reason to mention in CL subject line')
parser.add_argument('--verbose', help='Enable logging')
parser.add_argument('--patch-file', help='Use custom patch file')
args = parser.parse_args()
return args
def parse_start_version(start_version: str) -> int:
if start_version == 'llvm':
return int(get_svn_revision_number())
m = re.match(r'r?(\d+)', start_version)
assert m, f'invalid start_version: {start_version}'
return int(m.group(1))
@dataclass
class PatchItem:
metadata: Dict[str, Any]
# info: Optional[List[str]]
# title: str
platforms: List[str]
rel_patch_path: str
version_range: Dict[str, Optional[int]]
# from: Optional[int]
# until: Optional[int]
@classmethod
def from_dict(cls, d: Dict[str, Any]) -> PatchItem:
return PatchItem(
metadata=d['metadata'],
platforms=d['platforms'],
rel_patch_path=d['rel_patch_path'],
version_range=d['version_range'])
def to_dict(self) -> Dict[str, Any]:
return dataclasses.asdict(self, dict_factory=collections.OrderedDict)
@property
def is_local_patch(self) -> bool:
return not self.rel_patch_path.startswith('cherry/')
@property
def sha(self) -> str:
m = re.match(r'cherry/(.+)\.patch', self.rel_patch_path)
assert m, self.rel_patch_path
return m.group(1)
@property
def pr_link(self) -> str:
m = next(re.match(r'Pull Request: (.+)', line)
for line in open(f'patches/{self.rel_patch_path}'))
assert m, f'No PR link found in: {self.rel_patch_path}'
return m.group(1)
@property
def end_version(self) -> Optional[int]:
return self.version_range.get('until', None)
@property
def start_version(self) -> Optional[int]:
return self.version_range.get('from', None)
@property
def sort_key(item: PatchItem) -> Tuple:
# Keep local patches at the end of the list, and don't change the
# relative order between two local patches.
if item.is_local_patch:
return (True,)
# Just before local patches, include patches with no end_version. Sort
# them by start_version.
if item.end_version is None:
return (False, math.inf, item.start_version)
# At the front of the list, sort upstream patches by ascending order of
# end_version. Don't reorder patches with the same end_version.
return (False, item.end_version)
def __lt__(self, other: PatchItem) -> bool:
"""Used to sort patches in PatchList"""
return self.sort_key < other.sort_key
class PatchList(list):
""" a list of PatchItem """
JSON_FILE_PATH = paths.SCRIPTS_DIR / 'patches' / 'PATCHES.json'
@classmethod
def load_from_file(cls) -> PatchList:
with open(cls.JSON_FILE_PATH, 'r') as fh:
array = json.load(fh)
return PatchList(PatchItem.from_dict(d) for d in array)
def save_to_file(self):
array = [patch.to_dict() for patch in self]
with open(self.JSON_FILE_PATH, 'w') as fh:
json.dump(array, fh, indent=4, separators=(',', ': '), sort_keys=True)
fh.write('\n')
def generate_patch_files(sha_list: List[str], start_version: int, patch_list: PatchList) -> PatchList:
""" generate upstream cherry-pick patch files """
upstream_dir = paths.TOOLCHAIN_LLVM_PATH
fetch_upstream()
result = PatchList()
for sha in sha_list:
if len(sha) < 40:
sha = get_full_sha(upstream_dir, sha)
version = find_version(sha, patch_list, start_version)
version_name = '' if version == 1 else f'-v{version}'
rel_patch_path = f'cherry/{sha}' + version_name + '.patch'
file_path = paths.SCRIPTS_DIR / 'patches' / rel_patch_path
with open(file_path, 'w') as fh:
check_call(f'git format-patch -1 {sha} --stdout',
stdout=fh, shell=True, cwd=upstream_dir)
commit_subject = check_output(
f'git log -n1 --format=%s {sha}', shell=True, cwd=upstream_dir)
info: Optional[List[str]] = []
title = '[UPSTREAM] ' + commit_subject.strip()
end_version = sha_to_revision(sha)
metadata = { 'info': info, 'title': title }
platforms = ['android']
version_range: Dict[str, Optional[int]] = {
'from': start_version,
'until': end_version,
}
result.append(PatchItem(metadata, platforms, rel_patch_path, version_range))
return result
def add_new_patch_for_sha(patch_file: Path, sha: str, start_version: int, patch_list: PatchList) -> PatchList:
""" add patch files for sha"""
upstream_dir = paths.TOOLCHAIN_LLVM_PATH
result = []
assert len(sha) >= 40, f'the length of {sha} is {len(sha)} and it is shorter than 40'
version = find_version(sha, patch_list, start_version)
version_name = '' if version == 1 else f'-v{version}'
rel_patch_path = f'cherry/{sha}' + version_name + '.patch'
file_path = paths.SCRIPTS_DIR / 'patches' / rel_patch_path
with open(patch_file, 'r') as source, open(file_path, 'w') as dest:
for line in source:
dest.write(line)
commit_subject = check_output(
f'git log -n1 --format=%s {sha}', shell=True, cwd=upstream_dir)
info: Optional[List[str]] = []
title = '[UPSTREAM] ' + commit_subject.strip()
end_version = sha_to_revision(sha)
metadata = { 'info': info, 'title': title }
platforms = ['android']
version_range: Dict[str, Optional[int]] = {
'from': start_version,
'until': end_version,
}
result.append(PatchItem(metadata, platforms, rel_patch_path, version_range))
return result
def get_full_sha(upstream_dir: Path, short_sha: str) -> str:
return check_output(['git', 'rev-parse', short_sha], cwd=upstream_dir).strip()
def create_cl(new_patches: PatchList, reason: str, bug: Optional[str], cherry: bool):
file_list = [
str(paths.SCRIPTS_DIR / 'patches' / p.rel_patch_path) for p in new_patches
]
file_list += ['patches/PATCHES.json']
check_call(['git', 'add'] + file_list)
subject = f'[patches] Cherry pick CLS for: {reason}'
commit_lines = [subject, '']
script = os.path.basename(sys.argv[0])
argv_deepcopy = copy.deepcopy(sys.argv[1:])
for i in range(len(argv_deepcopy)):
element = argv_deepcopy[i]
if element.startswith('--reason'):
del argv_deepcopy[i]
if element == '--reason':
del argv_deepcopy[i]
break
for patch in new_patches:
if cherry: # Add SHA and title for each cherry-pick.
sha = patch.sha[:11]
subject = patch.metadata['title']
if subject.startswith('[UPSTREAM] '):
subject = subject[len('[UPSTREAM] '):]
commit_line = sha + ' ' + subject
else: # Add link to differential revision.
commit_line = patch.pr_link
commit_lines.append(commit_line)
commit_lines.append('')
args = ' '.join(argv_deepcopy)
auto_msg = f'This change is generated automatically by the script:\n {script} {args}'
commit_lines += [auto_msg, '']
if bug:
if bug.isnumeric():
commit_lines += [f'Bug: http://b/{bug}', '']
else:
commit_lines += [f'Bug: {bug}', '']
commit_lines += ['', 'Test: N/A']
check_call(['git', 'commit', '-m', '\n'.join(commit_lines)])
def create_patch(pr, start_version) -> PatchList:
pr_url=f'https://api.github.com/repos/llvm/llvm-project/pulls/{pr}'
patch_url_req = urllib.request.Request(f'https://github.com/llvm/llvm-project/pull/{pr}.diff',
method="HEAD")
patch_url = urllib.request.urlopen(patch_url_req).url
# TODO: Add commit body and author details as well.
with urllib.request.urlopen(pr_url) as response:
data = json.load(response)
title=data['title']
assert title, f'Title not found for {pr}'
print(f'Creating a patch for {title}')
file_name=f'{pr}.patch'
abs_file_name= paths.SCRIPTS_DIR / 'patches' / file_name
# Download the file from `patch_url` and save in `abs_file_name`:
urllib.request.urlretrieve(patch_url, abs_file_name)
# Add link to Differential Revision at the beginning of the file
patch_prefix=f'Pull Request: {patch_url}\nSubject: {title}'
with open(abs_file_name, 'r+') as f:
content = f.read()
f.seek(0, 0)
f.write(patch_prefix + '\n\n---\n' + content)
# Extend the PATCHES.json
result = PatchList()
info: Optional[List[str]] = []
rel_patch_path = f'{file_name}'
end_version = None
metadata = { 'info': info, 'title': title }
platforms = ['android']
version_range: Dict[str, Optional[int]] = {
'from': start_version,
'until': end_version,
}
result.append(PatchItem(metadata, platforms, rel_patch_path, version_range))
return result
def find_version(sha, patch_list, start_version) -> int:
""" Return the next version for the given SHA and update end_revision if needed"""
target = f'cherry/{sha}'
last_idx = -1
version = 1
name = ''
# Find the latest version
for i, item in enumerate(patch_list):
if item.rel_patch_path.startswith(target):
last_idx = i
name = item.rel_patch_path.removesuffix('.patch')
# If this patch is not new, update the end_revision for Vn
if last_idx != -1:
patch_list[last_idx].version_range['until'] = start_version
if name == target:
return 2
prefix = target + f'-v'
version = int(name.removeprefix(prefix)) + 1
return version
def main():
args = parse_args()
level = logging.DEBUG if args.verbose else logging.INFO
logging.basicConfig(level=level)
patch_list = PatchList.load_from_file()
assert not (bool(args.sha) and bool(args.pr)), (
'Only one of cherry-pick or patch supported.'
)
if args.pr:
start_version = parse_start_version(args.start_version)
new_patches = create_patch(args.pr, start_version)
patch_list.extend(new_patches)
elif args.sha:
start_version = parse_start_version(args.start_version)
if args.patch_file:
assert len(args.sha) == 1, f'error: --patch-file only requires 1 sha, but the size of sha list is {len(args.sha)}'
new_patches = add_new_patch_for_sha(args.patch_file, args.sha[0], start_version, patch_list)
else:
new_patches = generate_patch_files(args.sha, start_version, patch_list)
patch_list.extend(new_patches)
patch_list.sort()
patch_list.save_to_file()
if args.verify_merge:
print('Verifying merge...')
print('Verifying merge with patch ...')
source_manager.setup_sources()
print('Verifying merge with git am ...')
source_manager.setup_sources(git_am=True)
if args.create_cl:
if not args.reason:
print('error: --create-cl requires --reason')
exit(1)
cherry = True if args.sha else False
create_cl(new_patches, args.reason, args.bug, cherry)
if __name__ == '__main__':
main()