blob: 027b909a2876e0d152e6767baa28012a76048385 [file] [log] [blame]
# Copyright (C) 2012 The Android Open Source Project
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# See the License for the specific language governing permissions and
# limitations under the License.
"""Merge Chromium into the Android tree."""
import logging
import optparse
import os
import re
import sys
import merge_common
# We need to import this *after* merging from upstream to get the latest
# version. Set it to none here to catch uses before it's imported.
webview_licenses = None
AUTOGEN_MESSAGE = 'This commit was generated by'
SRC_GIT_BRANCH = 'refs/remotes/history/upstream-master'
def _ReadGitFile(sha1, path, git_url=None, git_branch=None):
"""Reads a file from a (possibly remote) git project at a specific revision.
sha1: The SHA1 at which to read.
path: The relative path of the file to read.
git_url: The URL of the git server, if reading a remote project.
git_branch: The branch to fetch, if reading a remote project.
The contents of the specified file.
if git_url:
merge_common.GetCommandStdout(['git', 'fetch', '-f', git_url, git_branch])
return merge_common.GetCommandStdout(['git', 'show', '%s:%s' % (sha1, path)])
def _ParseDEPS(deps_content):
"""Parses the DEPS file from Chromium and returns its contents.
deps_content: The contents of the DEPS file as text.
A dictionary of the contents of DEPS at the specified revision
class FromImpl(object):
"""Used to implement the From syntax."""
def __init__(self, module_name):
self.module_name = module_name
def __str__(self):
return 'From("%s")' % self.module_name
class _VarImpl(object):
def __init__(self, custom_vars, local_scope):
self._custom_vars = custom_vars
self._local_scope = local_scope
def Lookup(self, var_name):
"""Implements the Var syntax."""
if var_name in self._custom_vars:
return self._custom_vars[var_name]
elif var_name in self._local_scope.get('vars', {}):
return self._local_scope['vars'][var_name]
raise Exception('Var is not defined: %s' % var_name)
tmp_locals = {}
var = _VarImpl({}, tmp_locals)
tmp_globals = {'From': FromImpl, 'Var': var.Lookup, 'deps_os': {}}
exec(deps_content) in tmp_globals, tmp_locals # pylint: disable=W0122
return tmp_locals
def _GetProjectMergeInfo(projects, deps_vars):
"""Gets the git URL and SHA1 for each project based on DEPS.
projects: The list of projects to consider.
deps_vars: The dictionary of dependencies from DEPS.
A dictionary from project to git URL and SHA1 - 'path: (url, sha1)'
TemporaryMergeError: if a project to be merged is not found in DEPS.
deps_fallback_order = [
result = {}
for path in projects:
for deps in deps_fallback_order:
if path:
upstream_path = os.path.join('src', path)
upstream_path = 'src'
url_plus_sha1 = deps.get(upstream_path)
if url_plus_sha1:
raise merge_common.TemporaryMergeError(
'Could not find DEPS entry for project %s. This probably '
'means that the project list in needs to be '
'updated.' % path)
match = re.match('(.*?)@(.*)', url_plus_sha1)
url =
sha1 =
logging.debug(' Got URL %s and SHA1 %s for project %s', url, sha1, path)
result[path] = {'url': url, 'sha1': sha1}
return result
def _MergeProjects(version, root_sha1, target, unattended, buildspec_url):
"""Merges each required Chromium project into the Android repository.
DEPS is consulted to determine which revision each project must be merged
at. Only a whitelist of required projects are merged.
version: The version to mention in generated commit messages.
root_sha1: The git hash to merge in the root repository.
target: The target branch to merge to.
unattended: Run in unattended mode.
buildspec_url: URL for buildspec repository, when merging a branch.
TemporaryMergeError: If incompatibly licensed code is left after pruning.
# The logic for this step lives here, in the Android tree, as it makes no
# sense for a Chromium tree to know about this merge.
if unattended:
branch_create_flag = '-B'
branch_create_flag = '-b'
branch_name = 'merge-from-chromium-%s' % version
logging.debug('Parsing DEPS ...')
if root_sha1:
deps_content = _ReadGitFile(root_sha1, 'DEPS')
# TODO(primiano): At some point the release branches will use DEPS as well,
# instead of .DEPS.git. Rename below when that day will come.
deps_content = _ReadGitFile('FETCH_HEAD',
'releases/' + version + '/.DEPS.git',
deps_vars = _ParseDEPS(deps_content)
merge_info = _GetProjectMergeInfo(merge_common.THIRD_PARTY_PROJECTS,
for path in merge_info:
# webkit needs special handling as we have a local mirror
local_mirrored = path == 'third_party/WebKit'
url = merge_info[path]['url']
sha1 = merge_info[path]['sha1']
dest_dir = os.path.join(merge_common.REPOSITORY_ROOT, path)
if local_mirrored:
remote = 'history'
remote = 'goog'
merge_common.GetCommandStdout(['git', 'checkout',
branch_create_flag, branch_name,
'-t', remote + '/' + target],
if not local_mirrored or not root_sha1:
logging.debug('Fetching project %s at %s ...', path, sha1)
fetch_args = ['git', 'fetch', url, sha1]
merge_common.GetCommandStdout(fetch_args, cwd=dest_dir)
if merge_common.GetCommandStdout(['git', 'rev-list', '-1', 'HEAD..' + sha1],
logging.debug('Merging project %s at %s ...', path, sha1)
# Merge conflicts make git merge return 1, so ignore errors
merge_common.GetCommandStdout(['git', 'merge', '--no-commit', sha1],
cwd=dest_dir, ignore_errors=True)
'Merge %s from %s at %s\n\n%s' % (path, url, sha1, AUTOGEN_MESSAGE),
cwd=dest_dir, unattended=unattended)
logging.debug('No new commits to merge in project %s', path)
# Handle root repository separately.
merge_common.GetCommandStdout(['git', 'checkout',
branch_create_flag, branch_name,
'-t', 'history/' + target])
if not root_sha1:
merge_info = _GetProjectMergeInfo([''], deps_vars)
url = merge_info['']['url']
root_sha1 = merge_info['']['sha1']
merge_common.GetCommandStdout(['git', 'fetch', url, root_sha1])
merge_msg_version = '%s (%s)' % (version, root_sha1)
merge_msg_version = root_sha1
logging.debug('Merging Chromium at %s ...', root_sha1)
# Merge conflicts make git merge return 1, so ignore errors
merge_common.GetCommandStdout(['git', 'merge', '--no-commit', root_sha1],
'Merge Chromium at %s\n\n%s'
% (merge_msg_version, AUTOGEN_MESSAGE), unattended=unattended)
logging.debug('Getting directories to exclude ...')
# We import this now that we have merged the latest version.
# It imports to a global in order that it can be used to generate NOTICE
# later. We also disable writing bytecode to keep the source tree clean.
sys.path.append(os.path.join(merge_common.REPOSITORY_ROOT, 'android_webview',
sys.dont_write_bytecode = True
global webview_licenses # pylint: disable=W0602
import webview_licenses # pylint: disable=W0621,W0612,C6204
import known_issues # pylint: disable=C6204
for path, exclude_list in known_issues.KNOWN_INCOMPATIBLE.iteritems():
logging.debug(' %s', '\n '.join(os.path.join(path, x) for x in
dest_dir = os.path.join(merge_common.REPOSITORY_ROOT, path)
merge_common.GetCommandStdout(['git', 'rm', '-rf', '--ignore-unmatch'] +
exclude_list, cwd=dest_dir)
if _ModifiedFilesInIndex(dest_dir):
merge_common.GetCommandStdout(['git', 'commit', '-m',
'Exclude unwanted directories'],
def _CheckLicenses():
"""Check that no incompatibly licensed directories exist."""
directories_left_over = webview_licenses.GetIncompatibleDirectories()
if directories_left_over:
raise merge_common.TemporaryMergeError(
'Incompatibly licensed directories remain: ' +
def _GenerateMakefiles(version, unattended):
"""Run gyp to generate the Android build system makefiles.
version: The version to mention in generated commit messages.
unattended: Run in unattended mode.
logging.debug('Generating makefiles ...')
# TODO(torne): come up with a way to deal with hooks from DEPS properly
# TODO(torne): The .tmp files are generated by
# third_party/WebKit/Source/WebCore/WebCore.gyp/WebCore.gyp into the source
# tree. We should avoid this, or at least use a more specific name to avoid
# accidentally removing or adding other files.
for path in merge_common.ALL_PROJECTS:
dest_dir = os.path.join(merge_common.REPOSITORY_ROOT, path)
merge_common.GetCommandStdout(['git', 'rm', '--ignore-unmatch',
'GypAndroid.*.mk', '*.target.*.mk',
'*.host.*.mk', '*.tmp'], cwd=dest_dir)
merge_common.GetCommandStdout(['android_webview/tools/gyp_webview', 'all'])
except merge_common.MergeError as e:
if not unattended:
for path in merge_common.ALL_PROJECTS:
['git', 'reset', '--hard'],
cwd=os.path.join(merge_common.REPOSITORY_ROOT, path))
raise merge_common.TemporaryMergeError('Makefile generation failed: ' +
for path in merge_common.ALL_PROJECTS:
dest_dir = os.path.join(merge_common.REPOSITORY_ROOT, path)
# git add doesn't have an --ignore-unmatch so we have to do this instead:
merge_common.GetCommandStdout(['git', 'add', '-f', 'GypAndroid.*.mk'],
ignore_errors=True, cwd=dest_dir)
merge_common.GetCommandStdout(['git', 'add', '-f', '*.target.*.mk'],
ignore_errors=True, cwd=dest_dir)
merge_common.GetCommandStdout(['git', 'add', '-f', '*.host.*.mk'],
ignore_errors=True, cwd=dest_dir)
merge_common.GetCommandStdout(['git', 'add', '-f', '*.tmp'],
ignore_errors=True, cwd=dest_dir)
# Only try to commit the makefiles if something has actually changed.
if _ModifiedFilesInIndex(dest_dir):
['git', 'commit', '-m',
'Update makefiles after merge of Chromium at %s\n\n%s' %
(version, AUTOGEN_MESSAGE)], cwd=dest_dir)
def _ModifiedFilesInIndex(cwd=merge_common.REPOSITORY_ROOT):
"""Returns true if git's index contains any changes."""
status = merge_common.GetCommandStdout(['git', 'status', '--porcelain'],
return'^[MADRC]', status, flags=re.MULTILINE) is not None
def _GenerateNoticeFile(version):
"""Generates and commits a NOTICE file containing code licenses.
This covers all third-party code (from Android's perspective) that lives in
the Chromium tree.
version: The version to mention in generated commit messages.
logging.debug('Regenerating NOTICE file ...')
contents = webview_licenses.GenerateNoticeFile()
with open(os.path.join(merge_common.REPOSITORY_ROOT, 'NOTICE'), 'w') as f:
merge_common.GetCommandStdout(['git', 'add', 'NOTICE'])
# Only try to commit the NOTICE update if the file has actually changed.
if _ModifiedFilesInIndex():
'git', 'commit', '-m',
'Update NOTICE file after merge of Chromium at %s\n\n%s'
% (version, AUTOGEN_MESSAGE)])
def _GenerateLastChange(version, root_sha1):
"""Write a build/util/LASTCHANGE file containing the current revision.
The revision number is compiled into the binary at build time from this file.
version: The version to mention in generated commit messages.
root_sha1: The SHA1 of the main project (before the merge).
logging.debug('Updating LASTCHANGE ...')
with open(os.path.join(merge_common.REPOSITORY_ROOT, 'build/util/LASTCHANGE'),
'w') as f:
f.write('LASTCHANGE=%s\n' % merge_common.Abbrev(root_sha1))
merge_common.GetCommandStdout(['git', 'add', '-f', 'build/util/LASTCHANGE'])
logging.debug('Updating LASTCHANGE.blink ...')
with open(os.path.join(merge_common.REPOSITORY_ROOT,
'build/util/LASTCHANGE.blink'), 'w') as f:
f.write('LASTCHANGE=%s\n' % _GetBlinkRevision())
merge_common.GetCommandStdout(['git', 'add', '-f',
if _ModifiedFilesInIndex():
'git', 'commit', '-m',
'Update LASTCHANGE file after merge of Chromium at %s\n\n%s'
% (version, AUTOGEN_MESSAGE)])
def GetHEAD():
"""Fetch the latest HEAD revision from the Chromium Git mirror.
The latest HEAD revision (A Git abbrev SHA1).
return _GetGitAbbrevSHA1(SRC_GIT_BRANCH, 'HEAD')
def _ParseSvnRevisionFromGitCommitMessage(commit_message):
return'^git-svn-id: .*@([0-9]+)', commit_message,
def _GetGitAbbrevSHA1(git_branch, revision):
"""Returns an abbrev. SHA for the given revision (or branch, if HEAD)."""
assert revision
logging.debug('Getting Git revision for %s ...', revision)
upstream = git_branch if revision == 'HEAD' else revision
# Make sure the |upstream| Git object has been mirrored.
'git', 'merge-base', '--is-ancestor', upstream, git_branch])
except merge_common.CommandError:
raise merge_common.TemporaryMergeError(
'Upstream object (%s) not reachable from %s' % (upstream, git_branch))
abbrev_sha = merge_common.Abbrev(merge_common.GetCommandStdout(
['git', 'rev-list', '--max-count=1', upstream]).split()[0])
return abbrev_sha
def _GetBlinkRevision():
# TODO(primiano): Switch to Git as soon as Blink gets migrated as well.
commit = merge_common.GetCommandStdout(
['git', 'log', '-n1', '--grep=git-svn-id:', '--format=%H%n%b'],
cwd=os.path.join(merge_common.REPOSITORY_ROOT, 'third_party', 'WebKit'))
return _ParseSvnRevisionFromGitCommitMessage(commit)
def Snapshot(root_sha1, release, target, unattended, buildspec_url):
"""Takes a snapshot of the Chromium tree and merges it into Android.
Android makefiles and a top-level NOTICE file are generated and committed
after the merge.
root_sha1: The abbrev sha1 in the Chromium git mirror to merge from.
release: The Chromium release version to merge from (e.g. "30.0.1599.20").
Only one of root_sha1 and release should be specified.
target: The target branch to merge to.
unattended: Run in unattended mode.
buildspec_url: URL for buildspec repository, used when merging a release.
True if new commits were merged; False if no new commits were present.
if release:
root_sha1 = None
version = release
root_sha1 = _GetGitAbbrevSHA1(SRC_GIT_BRANCH, root_sha1)
version = root_sha1
assert (root_sha1 is not None and len(root_sha1) > 6) or version == release
if root_sha1 and not merge_common.GetCommandStdout(
['git', 'rev-list', '-1', 'HEAD..' + root_sha1]):'No new commits to merge at %s (%s)', version, root_sha1)
return False'Snapshotting Chromium at %s (%s)', version, root_sha1)
# 1. Merge, accounting for excluded directories
_MergeProjects(version, root_sha1, target, unattended, buildspec_url)
# 2. Generate Android makefiles
_GenerateMakefiles(version, unattended)
# 3. Check for incompatible licenses
# 4. Generate Android NOTICE file
# 5. Generate LASTCHANGE file
_GenerateLastChange(version, root_sha1)
return True
def Push(version, target):
"""Push the finished snapshot to the Android repository."""
src = 'merge-from-chromium-%s' % version
# Use forced pushes ('+' prefix) for the temporary and archive branches in
# case they already got updated by a previous (possibly failed?) merge, but
# do not force push to the real master-chromium branch as this could erase
# downstream changes.
refspecs = ['%s:%s' % (src, target),
'+%s:refs/archive/chromium-%s' % (src, version)]
if target == 'master-chromium':
refspecs.insert(0, '+%s:master-chromium-merge' % src)
for refspec in refspecs:
logging.debug('Pushing to server (%s) ...', refspec)
for path in merge_common.ALL_PROJECTS:
if path in merge_common.PROJECTS_WITH_FLAT_HISTORY:
remote = 'history'
remote = 'goog'
logging.debug('Pushing %s', path)
dest_dir = os.path.join(merge_common.REPOSITORY_ROOT, path)
merge_common.GetCommandStdout(['git', 'push', remote, refspec],
def main():
parser = optparse.OptionParser(usage='%prog [options]')
parser.epilog = ('Takes a snapshot of the Chromium tree at the specified '
'Chromium Git revision and merges it into this repository. '
'Paths marked as excluded for license reasons are removed '
'as part of the merge. Also generates Android makefiles and '
'generates a top-level NOTICE file suitable for use in the '
'Android build.')
'', '--sha1',
help=('Merge to the specified chromium sha1 revision from ' +
SRC_GIT_BRANCH + ' branch. Default is HEAD, to merge from ToT.'))
'', '--release',
help=('Merge to the specified chromium release buildspec (e.g., "30.0.'
'1599.20"). Only one of --sha1 and --release should be specified'))
'', '--buildspec_url',
help=('Git URL for buildspec repository.'))
'', '--target',
default='master-chromium', metavar='BRANCH',
help=('Target branch to push to. Defaults to master-chromium.'))
'', '--push',
default=False, action='store_true',
help=('Push the result of a previous merge to the server. Note '
'--sha1 must be given.'))
'', '--get_head',
default=False, action='store_true',
help=('Just print the current HEAD revision on stdout and exit.'))
'', '--unattended',
default=False, action='store_true',
help=('Run in unattended mode.'))
'', '--no_changes_exit',
default=0, type='int',
help=('Exit code to use if there are no changes to merge, for scripts.'))
(options, args) = parser.parse_args()
if args:
return 1
if 'ANDROID_BUILD_TOP' not in os.environ:
print >>sys.stderr, 'You need to run the Android and lunch.'
return 1
logging.basicConfig(format='%(message)s', level=logging.DEBUG,
if options.get_head:
logging.disable(logging.CRITICAL) # Prevent log messages
print GetHEAD()
elif options.push:
if options.release:
elif options.sha1:
print >>sys.stderr, 'You need to pass the version to push.'
return 1
if not Snapshot(options.sha1, options.release,,
options.unattended, options.buildspec_url):
return options.no_changes_exit
return 0
if __name__ == '__main__':