| #!/usr/bin/python |
| # |
| # Copyright (C) 2012 The Android Open Source Project |
| # |
| # Licensed under the Apache License, Version 2.0 (the "License"); |
| # you may not use this file except in compliance with the License. |
| # You may obtain a copy of the License at |
| # |
| # http://www.apache.org/licenses/LICENSE-2.0 |
| # |
| # Unless required by applicable law or agreed to in writing, software |
| # distributed under the License is distributed on an "AS IS" BASIS, |
| # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| # See the License for the specific language governing permissions and |
| # limitations under the License. |
| |
| """Merge master-chromium to master within the Android tree.""" |
| |
| import logging |
| import optparse |
| import os |
| import re |
| import shutil |
| import subprocess |
| import sys |
| |
| import merge_common |
| |
| |
| AUTOGEN_MESSAGE = 'This commit was generated by merge_to_master.py.' |
| WEBVIEW_PROJECT = 'frameworks/webview' |
| |
| |
| def _GetAbsPath(project): |
| """Returns the full path to a given project (either Chromium or Android).""" |
| if project in merge_common.ALL_PROJECTS: |
| abs_path = os.path.join(merge_common.REPOSITORY_ROOT, project) |
| else: |
| abs_path = os.path.join(os.environ['ANDROID_BUILD_TOP'], project) |
| if not os.path.exists(abs_path): |
| raise merge_common.MergeError('Cannot find path ' + abs_path) |
| return abs_path |
| |
| |
| def _CheckoutSingleProject(project, target_branch): |
| """Checks out the tip of the target_branch into a local branch (merge-to-XXX). |
| |
| Args: |
| project: a Chromium project (., third_party/foo) or frameworks/webview. |
| target_branch: name of the target branch (in the goog remote). |
| """ |
| dest_dir = _GetAbsPath(project) |
| tracking_branch = 'goog/' + target_branch |
| logging.debug('Check out %-45s at %-16s', project, tracking_branch) |
| merge_common.GetCommandStdout(['git', 'remote', 'update', 'goog'], |
| cwd=dest_dir) |
| merge_common.GetCommandStdout(['git', 'checkout', |
| '-b', 'merge-to-' + target_branch, |
| '-t', tracking_branch], cwd=dest_dir) |
| |
| |
| def _FetchSingleProject(project, remote, remote_ref): |
| """Fetches a remote ref for the given project and returns the fetched SHA. |
| |
| Args: |
| project: a Chromium project (., third_party/foo) or frameworks/webview. |
| remote: Git remote name (goog for most projects, history for squashed ones). |
| remote_ref: the remote ref to fetch (e.g., refs/archive/chromium-XXX). |
| |
| Returns: |
| The SHA1 of the FETCH_HEAD. |
| """ |
| dest_dir = _GetAbsPath(project) |
| logging.debug('Fetch %-45s %s:%s', project, remote, remote_ref) |
| merge_common.GetCommandStdout(['git', 'fetch', remote, remote_ref], |
| cwd=dest_dir) |
| return merge_common.GetCommandStdout(['git', 'rev-parse', 'FETCH_HEAD'], |
| cwd=dest_dir).strip() |
| |
| |
| def _MergeSingleProject(project, merge_sha, revision, target_branch, flatten): |
| """Merges a single project at a given SHA. |
| |
| Args: |
| project: a Chromium project (., third_party/foo) or frameworks/webview. |
| merge_sha: the SHA to merge. |
| revision: Abbrev. commitish in the main Chromium repository. |
| target_branch: name of the target branch. |
| flatten: True: squash history while merging; False: perform a normal merge. |
| """ |
| dest_dir = _GetAbsPath(project) |
| if flatten: |
| # Make the previous merges into grafts so we can do a correct merge. |
| old_sha = merge_common.GetCommandStdout(['git', 'rev-parse', 'HEAD'], |
| cwd=dest_dir).strip() |
| merge_log = os.path.join(dest_dir, '.merged-revisions') |
| if os.path.exists(merge_log): |
| shutil.copyfile(merge_log, |
| os.path.join(dest_dir, '.git', 'info', 'grafts')) |
| |
| # Early out if there is nothing to merge. |
| if not merge_common.GetCommandStdout(['git', 'rev-list', '-1', |
| 'HEAD..' + merge_sha], cwd=dest_dir): |
| logging.debug('No new commits to merge in project %s', project) |
| return |
| |
| logging.debug('Merging project %s (flatten: %s)...', project, flatten) |
| merge_cmd = ['git', 'merge', '--no-commit'] |
| merge_cmd += ['--squash'] if flatten else ['--no-ff'] |
| merge_cmd += [merge_sha] |
| # Merge conflicts cause 'git merge' to return 1, so ignore errors |
| merge_common.GetCommandStdout(merge_cmd, cwd=dest_dir, ignore_errors=True) |
| |
| if flatten: |
| dirs_to_prune = merge_common.PRUNE_WHEN_FLATTENING.get(project, []) |
| if dirs_to_prune: |
| merge_common.GetCommandStdout(['git', 'rm', '--ignore-unmatch', '-rf'] + |
| dirs_to_prune, cwd=dest_dir) |
| |
| if project in merge_common.ALL_PROJECTS: |
| commit_msg = 'Merge from Chromium at DEPS revision %s' % revision |
| else: |
| commit_msg = 'Merge master-chromium into %s at %s' % (target_branch, |
| revision) |
| commit_msg += '\n\n' + AUTOGEN_MESSAGE |
| merge_common.CheckNoConflictsAndCommitMerge(commit_msg, cwd=dest_dir) |
| |
| if flatten: |
| # Generate the new grafts file and commit it on top of the merge. |
| new_sha = merge_common.GetCommandStdout(['git', 'rev-parse', 'HEAD'], |
| cwd=dest_dir).strip() |
| with open(merge_log, 'a+') as f: |
| f.write('%s %s %s\n' % (new_sha, old_sha, merge_sha)) |
| merge_common.GetCommandStdout(['git', 'add', '.merged-revisions'], |
| cwd=dest_dir) |
| merge_common.GetCommandStdout( |
| ['git', 'commit', '-m', |
| 'Record Chromium merge at DEPS revision %s\n\n%s' % |
| (revision, AUTOGEN_MESSAGE)], cwd=dest_dir) |
| |
| |
| def _IsAncestor(ref1, ref2, cwd): |
| """Checks whether ref1 is a ancestor of ref2 in the given Git repo.""" |
| cmd = ['git', 'merge-base', '--is-ancestor', ref1, ref2] |
| ret = subprocess.call(cmd, cwd=cwd) |
| if ret == 0: |
| return True |
| elif ret == 1: |
| return False |
| else: |
| raise merge_common.CommandError(ret, ' '.join(cmd), cwd, 'N/A', 'N/A') |
| |
| |
| def _MergeChromiumProjects(revision, target_branch, repo_shas=None, |
| force=False): |
| """Merges the Chromium projects from master-chromium to target_branch. |
| |
| The larger projects' histories are flattened in the process. |
| When repo_shas != None, it checks that the SHAs of the projects in the |
| archive match exactly the SHAs of the projects in repo.prop. |
| |
| Args: |
| revision: Abbrev. commitish in the main Chromium repository. |
| target_branch: target branch name to merge and push to. |
| repo_shas: optional dict. of expected revisions (only for --repo-prop). |
| force: True: merge anyways using the SHAs from repo.prop; False: bail out if |
| projects mismatch (archive vs repo.prop). |
| """ |
| # Sync and checkout ToT for all projects (creating the merge-to-XXX branch) |
| # and fetch the archive snapshot. |
| fetched_shas = {} |
| remote_ref = 'refs/archive/chromium-%s' % revision |
| for project in merge_common.PROJECTS_WITH_FLAT_HISTORY: |
| _CheckoutSingleProject(project, target_branch) |
| fetched_shas[project] = _FetchSingleProject(project, 'history', remote_ref) |
| for project in merge_common.PROJECTS_WITH_FULL_HISTORY: |
| _CheckoutSingleProject(project, target_branch) |
| fetched_shas[project] = _FetchSingleProject(project, 'goog', remote_ref) |
| |
| if repo_shas: |
| project_shas_mismatch = False |
| for project, merge_sha in fetched_shas.items(): # the dict can be modified. |
| expected_sha = repo_shas.get(project) |
| if expected_sha != merge_sha: |
| logging.warn('The SHA for project %s specified in the repo.prop (%s) ' |
| 'and the one in the archive (%s) differ.', |
| project, expected_sha, merge_sha) |
| dest_dir = _GetAbsPath(project) |
| if expected_sha is None: |
| reason = 'cannot find a SHA in the repo.pro for %s' % project |
| elif _IsAncestor(merge_sha, expected_sha, cwd=dest_dir): |
| reason = 'the SHA in repo.prop is ahead of the SHA in the archive. ' |
| log_cmd = ['git', 'log', '--oneline', '--graph', '--max-count=10', |
| '%s..%s' % (merge_sha, expected_sha)] |
| log_cmd_output = merge_common.GetCommandStdout(log_cmd, cwd=dest_dir) |
| reason += 'showing partial log (%s): \n %s' % (' '.join(log_cmd), |
| log_cmd_output) |
| elif _IsAncestor(expected_sha, merge_sha, cwd=dest_dir): |
| reason = 'The SHA is already merged in the archive' |
| else: |
| reason = 'The project history diverged. Consult your Git historian.' |
| |
| project_shas_mismatch = True |
| if force: |
| logging.debug('Merging the SHA in repo.prop anyways (due to --force)') |
| fetched_shas[project] = expected_sha |
| else: |
| logging.debug('Reason: %s', reason) |
| if not force and project_shas_mismatch: |
| raise merge_common.MergeError( |
| 'The revision of some projects in the archive is different from the ' |
| 'one provided in build.prop. See the log for more details. Re-run ' |
| 'with --force to continue.') |
| |
| for project in merge_common.PROJECTS_WITH_FLAT_HISTORY: |
| _MergeSingleProject(project, fetched_shas[project], revision, target_branch, |
| flatten=True) |
| for project in merge_common.PROJECTS_WITH_FULL_HISTORY: |
| _MergeSingleProject(project, fetched_shas[project], revision, target_branch, |
| flatten=False) |
| |
| |
| def _GetNearestUpstreamAbbrevSHA(reference='history/master-chromium'): |
| """Returns the abbrev. upstream SHA which closest to the given reference.""" |
| logging.debug('Getting upstream SHA for %s...', reference) |
| merge_common.GetCommandStdout(['git', 'remote', 'update', 'history']) |
| upstream_commit = merge_common.Abbrev(merge_common.GetCommandStdout([ |
| 'git', 'merge-base', 'history/upstream-master', reference])) |
| |
| # Pedantic check: look for the existence of a merge commit which contains the |
| # |upstream_commit| in its message and is its children. |
| merge_parents = merge_common.GetCommandStdout([ |
| 'git', 'rev-list', reference, '--grep', upstream_commit, '--merges', |
| '--parents', '-1']) |
| if upstream_commit not in merge_parents: |
| raise merge_common.MergeError( |
| 'Found upstream commit %s, but the merge child (%s) could not be found ' |
| 'or is not a parent of the upstream SHA') |
| logging.debug('Found nearest Chromium revision %s', upstream_commit) |
| return upstream_commit |
| |
| |
| def _MergeWithRepoProp(repo_prop_file, target_branch, force): |
| """Performs a merge using a repo.prop file (from Android build waterfall). |
| |
| This does NOT merge (unless forced with force=True) the pinned |
| revisions in repo.prop, as a repo.prop can snapshot an intermediate state |
| (between two automerger cycles). Instead, this looks up the archived snapshot |
| (generated by the chromium->master-chromium auto-merger) which is closest to |
| the given repo.prop (following the main Chromium project) and merges that one. |
| If the projects revisions don't match, it fails with detailed error messages. |
| |
| Args: |
| repo_prop_file: Path to a downloaded repo.prop file. |
| target_branch: name of the target branch to merget to. |
| force: ignores the aforementioned check and merged anyways. |
| """ |
| chromium_sha = None |
| webview_sha = None |
| repo_shas = {} # 'project/path' -> 'sha' |
| with open(repo_prop_file) as prop: |
| for line in prop: |
| repo, sha = line.split() |
| # Translate the Android repo paths into the relative project paths used in |
| # merge_common (e.g., platform/external/chromium_org/foo -> foo). |
| m = ( |
| re.match(r'^platform/(frameworks/.+)$', repo) or |
| re.match(r'^platform/external/chromium_org/?(.*?)(-history)?$', repo)) |
| if m: |
| project = m.group(1) if m.group(1) else '.' # '.' = Main project. |
| repo_shas[project] = sha |
| |
| chromium_sha = repo_shas.get('.') |
| webview_sha = repo_shas.get(WEBVIEW_PROJECT) |
| if not chromium_sha or not webview_sha: |
| raise merge_common.MergeError('SHAs for projects not found; ' |
| 'invalid build.prop?') |
| |
| # Check that the revisions in repo.prop and the on in the archive match. |
| archived_chromium_revision = _GetNearestUpstreamAbbrevSHA(chromium_sha) |
| logging.info('Merging Chromium at %s and WebView at %s', |
| archived_chromium_revision, webview_sha) |
| _MergeChromiumProjects(archived_chromium_revision, target_branch, repo_shas, |
| force) |
| |
| _CheckoutSingleProject(WEBVIEW_PROJECT, target_branch) |
| _MergeSingleProject(WEBVIEW_PROJECT, webview_sha, |
| archived_chromium_revision, target_branch, flatten=False) |
| |
| |
| def Push(target_branch): |
| """Push the finished snapshot to the Android repository. |
| |
| Creates first a CL for frameworks/webview (if the merge-to-XXX branch exists) |
| then wait for user confirmation and pushes the Chromium merges. This is to |
| give an opportunity to get a +2 for frameworks/webview and then push both |
| frameworks/webview and the Chromium projects atomically(ish). |
| |
| Args: |
| target_branch: name of the target branch (in the goog remote). |
| """ |
| merge_branch = 'merge-to-%s' % target_branch |
| |
| # Create a Gerrit CL for the frameworks/webview project (if needed). |
| dest_dir = _GetAbsPath(WEBVIEW_PROJECT) |
| did_upload_webview_cl = False |
| if merge_common.GetCommandStdout(['git', 'branch', '--list', merge_branch], |
| cwd=dest_dir): |
| # Check that there was actually something to merge. |
| merge_range = 'goog/%s..%s' % (target_branch, merge_branch) |
| if merge_common.GetCommandStdout(['git', 'rev-list', '-1', merge_range], |
| cwd=dest_dir): |
| logging.info('Uploading a merge CL for %s...', WEBVIEW_PROJECT) |
| refspec = '%s:refs/for/%s' % (merge_branch, target_branch) |
| upload = merge_common.GetCommandStdout(['git', 'push', 'goog', refspec], |
| cwd=dest_dir) |
| logging.info(upload) |
| did_upload_webview_cl = True |
| |
| prompt_msg = 'About push the Chromium projects merge. ' |
| if not did_upload_webview_cl: |
| logging.info('No merge CL needed for %s.', WEBVIEW_PROJECT) |
| else: |
| prompt_msg += ('At this point you should have the CL +2-ed and merge it ' |
| 'together with this push.') |
| prompt_msg += '\nPress "y" to continue: ' |
| if raw_input(prompt_msg) != 'y': |
| logging.warn('Push aborted by the user!') |
| return |
| |
| logging.debug('Pushing Chromium projects to %s ...', target_branch) |
| refspec = '%s:%s' % (merge_branch, target_branch) |
| for path in merge_common.ALL_PROJECTS: |
| logging.debug('Pushing %s', path) |
| dest_dir = _GetAbsPath(path) |
| # Delete the graft before pushing otherwise git will attempt to push all the |
| # grafted-in objects to the server as well as the ones we want. |
| graftfile = os.path.join(dest_dir, '.git', 'info', 'grafts') |
| if os.path.exists(graftfile): |
| os.remove(graftfile) |
| merge_common.GetCommandStdout(['git', 'push', 'goog', refspec], |
| cwd=dest_dir) |
| |
| |
| def main(): |
| parser = optparse.OptionParser(usage='%prog [options]') |
| parser.epilog = ('Takes the current master-chromium branch of the Chromium ' |
| 'projects in Android and merges them into master to publish ' |
| 'them.') |
| parser.add_option( |
| '', '--revision', |
| default=None, |
| help=('Merge to the specified archived master-chromium revision (abbrev. ' |
| 'SHA or release version) rather than using HEAD. e.g., ' |
| '--revision=a1b2c3d4e5f6 or --revision=38.0.2125.24')) |
| parser.add_option( |
| '', '--repo-prop', |
| default=None, metavar='FILE', |
| help=('Merge to the revisions specified in this repo.prop file.')) |
| parser.add_option( |
| '', '--force', |
| default=False, action='store_true', |
| help=('Skip history checks and merged anyways (only for --repo-prop).')) |
| parser.add_option( |
| '', '--push', |
| default=False, action='store_true', |
| help=('Push the result of a previous merge to the server.')) |
| parser.add_option( |
| '', '--target', |
| default='master', metavar='BRANCH', |
| help=('Target branch to push to. Defaults to master.')) |
| (options, args) = parser.parse_args() |
| if args: |
| parser.print_help() |
| return 1 |
| |
| logging.basicConfig(format='%(message)s', level=logging.DEBUG, |
| stream=sys.stdout) |
| |
| if options.push: |
| Push(options.target) |
| elif options.repo_prop: |
| _MergeWithRepoProp(os.path.expanduser(options.repo_prop), |
| options.target, options.force) |
| elif options.revision: |
| _MergeChromiumProjects(options.revision, options.target) |
| else: |
| first_upstream_sha = _GetNearestUpstreamAbbrevSHA() |
| _MergeChromiumProjects(first_upstream_sha, options.target) |
| |
| return 0 |
| |
| if __name__ == '__main__': |
| sys.exit(main()) |
