| #!/usr/bin/env python |
| # Copyright (c) 2015 The WebRTC project authors. All Rights Reserved. |
| # |
| # Use of this source code is governed by a BSD-style license |
| # that can be found in the LICENSE file in the root of the source |
| # tree. An additional intellectual property rights grant can be found |
| # in the file PATENTS. All contributing project authors may |
| # be found in the AUTHORS file in the root of the source tree. |
| |
| import argparse |
| import collections |
| import getpass |
| import json |
| import logging |
| import os |
| import re |
| import shutil |
| import stat |
| import subprocess |
| import sys |
| import tempfile |
| import time |
| import urllib2 |
| |
| |
| SCRIPT_DIR = os.path.dirname(os.path.realpath(__file__)) |
| ROOT_DIR = os.path.abspath(os.path.join(SCRIPT_DIR, os.pardir, os.pardir)) |
| sys.path.insert(1, os.path.join(ROOT_DIR, 'tools')) |
| import find_depot_tools |
| find_depot_tools.add_depot_tools_to_path() |
| import rietveld |
| import roll_dep |
| from gclient import GClientKeywords |
| from third_party import upload |
| |
| # Avoid depot_tools/third_party/upload.py print verbose messages. |
| upload.verbosity = 0 # Errors only. |
| |
| CHROMIUM_GIT_URL = 'https://chromium.googlesource.com/chromium/src.git' |
| COMMIT_POSITION_RE = re.compile('^Cr-Original-Commit-Position: .*#([0-9]+).*$') |
| CL_ISSUE_RE = re.compile('^Issue number: ([0-9]+) \((.*)\)$') |
| RIETVELD_URL_RE = re.compile('^https?://(.*)/(.*)') |
| ROLL_BRANCH_NAME = 'special_webrtc_roll_branch' |
| TRYJOB_STATUS_SLEEP_SECONDS = 30 |
| |
| # Use a shell for subcommands on Windows to get a PATH search. |
| USE_SHELL = sys.platform.startswith('win') |
| WEBRTC_PATH = 'third_party/webrtc' |
| LIBJINGLE_PATH = 'third_party/libjingle/source/talk' |
| LIBJINGLE_README = 'third_party/libjingle/README.chromium' |
| |
| # Result codes from build/third_party/buildbot_8_4p1/buildbot/status/results.py |
| # plus the -1 code which is used when there's no result yet. |
| TRYJOB_STATUS = { |
| -1: 'RUNNING', |
| 0: 'SUCCESS', |
| 1: 'WARNINGS', |
| 2: 'FAILURE', |
| 3: 'SKIPPED', |
| 4: 'EXCEPTION', |
| 5: 'RETRY', |
| } |
| SUCCESS_STATUS = (0, 1, 3) |
| FAILURE_STATUS = (2, 4, 5) |
| |
| CommitInfo = collections.namedtuple('CommitInfo', ['commit_position', |
| 'git_commit', |
| 'git_repo_url']) |
| CLInfo = collections.namedtuple('CLInfo', ['issue', 'url', 'rietveld_server']) |
| |
| |
| def _ParseGitCommitPosition(description): |
| for line in reversed(description.splitlines()): |
| m = COMMIT_POSITION_RE.match(line.strip()) |
| if m: |
| return m.group(1) |
| logging.error('Failed to parse svn revision id from:\n%s\n', description) |
| sys.exit(-1) |
| |
| |
| def _ParseGitCommitHash(description): |
| for line in description.splitlines(): |
| if line.startswith('commit '): |
| return line.split()[1] |
| logging.error('Failed to parse git commit id from:\n%s\n', description) |
| sys.exit(-1) |
| return None |
| |
| |
| def _ParseDepsFile(filename): |
| with open(filename, 'rb') as f: |
| deps_content = f.read() |
| return _ParseDepsDict(deps_content) |
| |
| |
| def _ParseDepsDict(deps_content): |
| local_scope = {} |
| var = GClientKeywords.VarImpl({}, local_scope) |
| global_scope = { |
| 'File': GClientKeywords.FileImpl, |
| 'From': GClientKeywords.FromImpl, |
| 'Var': var.Lookup, |
| 'deps_os': {}, |
| } |
| exec(deps_content, global_scope, local_scope) |
| return local_scope |
| |
| |
| def _WaitForTrybots(issue, rietveld_server): |
| """Wait until all trybots have passed or at least one have failed. |
| |
| Returns: |
| An exit code of 0 if all trybots passed or non-zero otherwise. |
| """ |
| assert type(issue) is int |
| print 'Trybot status for https://%s/%d:' % (rietveld_server, issue) |
| remote = rietveld.Rietveld('https://' + rietveld_server, None, None) |
| |
| attempt = 0 |
| max_tries = 60*60/TRYJOB_STATUS_SLEEP_SECONDS # Max one hour |
| while attempt < max_tries: |
| # Get patches for the issue so we can use the latest one. |
| data = remote.get_issue_properties(issue, messages=False) |
| patchsets = data['patchsets'] |
| |
| # Get trybot status for the latest patch set. |
| data = remote.get_patchset_properties(issue, patchsets[-1]) |
| |
| tryjob_results = data['try_job_results'] |
| if len(tryjob_results) == 0: |
| logging.debug('No trybots have yet been triggered for https://%s/%d' , |
| rietveld_server, issue) |
| else: |
| _PrintTrybotsStatus(tryjob_results) |
| if any(r['result'] in FAILURE_STATUS for r in tryjob_results): |
| logging.error('Found failing tryjobs (see above)') |
| return 1 |
| if all(r['result'] in SUCCESS_STATUS for r in tryjob_results): |
| return 0 |
| |
| logging.debug('Waiting for %d seconds before next check...', |
| TRYJOB_STATUS_SLEEP_SECONDS) |
| time.sleep(TRYJOB_STATUS_SLEEP_SECONDS) |
| attempt += 1 |
| |
| |
| def _PrintTrybotsStatus(tryjob_results): |
| status_to_name = {} |
| for trybot_result in tryjob_results: |
| status = TRYJOB_STATUS.get(trybot_result['result'], 'UNKNOWN') |
| status_to_name.setdefault(status, []) |
| status_to_name[status].append(trybot_result['builder']) |
| |
| print '\n========== TRYJOBS STATUS ==========' |
| for status,name_list in status_to_name.iteritems(): |
| print '%s: %s' % (status, ','.join(sorted(name_list))) |
| |
| |
| def _GenerateCLDescription(webrtc_current, libjingle_current, |
| webrtc_new, libjingle_new): |
| delim = '' |
| webrtc_str = '' |
| def GetChangeLogURL(git_repo_url, current_hash, new_hash): |
| return '%s/+log/%s..%s' % (git_repo_url, current_hash[0:7], new_hash[0:7]) |
| |
| if webrtc_current.git_commit != webrtc_new.git_commit: |
| webrtc_str = 'WebRTC %s:%s' % (webrtc_current.commit_position, |
| webrtc_new.commit_position) |
| webrtc_changelog_url = GetChangeLogURL(webrtc_current.git_repo_url, |
| webrtc_current.git_commit, |
| webrtc_new.git_commit) |
| |
| libjingle_str = '' |
| if libjingle_current.git_commit != libjingle_new.git_commit: |
| if webrtc_str: |
| delim += ', ' |
| libjingle_str = 'Libjingle %s:%s' % (libjingle_current.commit_position, |
| libjingle_new.commit_position) |
| libjingle_changelog_url = GetChangeLogURL(libjingle_current.git_repo_url, |
| libjingle_current.git_commit, |
| libjingle_new.git_commit) |
| |
| description = 'Roll ' + webrtc_str + delim + libjingle_str + '\n\n' |
| if webrtc_str: |
| description += webrtc_str + '\n' |
| description += 'Changes: %s\n\n' % webrtc_changelog_url |
| if libjingle_str: |
| description += libjingle_str + '\n' |
| description += 'Changes: %s\n' % libjingle_changelog_url |
| description += '\nTBR=' |
| return description |
| |
| |
| def _IsChromiumCheckout(checkout_dir): |
| """Checks if the provided directory path is a Chromium checkout.""" |
| # Look for DEPS file and a chromium directory. |
| return (os.path.isfile(os.path.join(checkout_dir, 'DEPS')) and |
| os.path.isdir(os.path.join(checkout_dir, 'chrome'))) |
| |
| |
| class AutoRoller(object): |
| def __init__(self, chromium_src): |
| self._chromium_src = chromium_src |
| |
| def _RunCommand(self, command, working_dir=None, ignore_exit_code=False, |
| extra_env=None): |
| """Runs a command and returns the stdout from that command. |
| |
| If the command fails (exit code != 0), the function will exit the process. |
| """ |
| working_dir = working_dir or self._chromium_src |
| logging.debug('cmd: %s cwd: %s', ' '.join(command), working_dir) |
| env = os.environ.copy() |
| if extra_env: |
| logging.debug('extra env: %s', extra_env) |
| env.update(extra_env) |
| p = subprocess.Popen(command, stdout=subprocess.PIPE, |
| stderr=subprocess.PIPE, shell=USE_SHELL, env=env, |
| cwd=working_dir, universal_newlines=True) |
| output = p.stdout.read() |
| p.wait() |
| p.stdout.close() |
| p.stderr.close() |
| |
| if not ignore_exit_code and p.returncode != 0: |
| logging.error('Command failed: %s\n%s', str(command), output) |
| sys.exit(p.returncode) |
| return output |
| |
| def _GetCommitInfo(self, path_below_src, git_hash=None, git_repo_url=None): |
| working_dir = os.path.join(self._chromium_src, path_below_src) |
| self._RunCommand(['git', 'fetch', 'origin'], working_dir=working_dir) |
| revision_range = git_hash or 'origin' |
| ret = self._RunCommand( |
| ['git', '--no-pager', 'log', revision_range, '--pretty=full', '-1'], |
| working_dir=working_dir) |
| return CommitInfo(_ParseGitCommitPosition(ret), _ParseGitCommitHash(ret), |
| git_repo_url) |
| |
| def _GetDepsCommitInfo(self, deps_dict, path_below_src): |
| entry = deps_dict['deps']['src/%s' % path_below_src] |
| at_index = entry.find('@') |
| git_repo_url = entry[:at_index] |
| git_hash = entry[at_index + 1:] |
| return self._GetCommitInfo(path_below_src, git_hash, git_repo_url) |
| |
| def _GetCLInfo(self): |
| cl_output = self._RunCommand(['git', 'cl', 'issue']) |
| m = CL_ISSUE_RE.match(cl_output.strip()) |
| if not m: |
| logging.error('Cannot find any CL info. Output was:\n%s', cl_output) |
| sys.exit(-1) |
| issue_number = int(m.group(1)) |
| url = m.group(2) |
| |
| # Parse the Rietveld host from the URL. |
| m = RIETVELD_URL_RE.match(url) |
| if not m: |
| logging.error('Cannot parse Rietveld host from URL: %s', url) |
| sys.exit(-1) |
| rietveld_server = m.group(1) |
| return CLInfo(issue_number, url, rietveld_server) |
| |
| def _GetCurrentBranchName(self): |
| return self._RunCommand( |
| ['git', 'rev-parse', '--abbrev-ref', 'HEAD']).splitlines()[0] |
| |
| def _IsTreeClean(self): |
| lines = self._RunCommand(['git', 'status', '--porcelain']).splitlines() |
| if len(lines) == 0: |
| return True |
| |
| logging.debug('Dirty/unversioned files:\n%s', '\n'.join(lines)) |
| return False |
| |
| def _UpdateReadmeFile(self, readme_path, new_revision): |
| readme = open(os.path.join(self._chromium_src, readme_path), 'r+') |
| txt = readme.read() |
| m = re.sub(re.compile('.*^Revision\: ([0-9]*).*', re.MULTILINE), |
| ('Revision: %s' % new_revision), txt) |
| readme.seek(0) |
| readme.write(m) |
| readme.truncate() |
| |
| def PrepareRoll(self, dry_run, ignore_checks, no_commit, close_previous_roll): |
| # TODO(kjellander): use os.path.normcase, os.path.join etc for all paths for |
| # cross platform compatibility. |
| |
| if not ignore_checks: |
| if self._GetCurrentBranchName() != 'master': |
| logging.error('Please checkout the master branch.') |
| return -1 |
| if not self._IsTreeClean(): |
| logging.error('Please make sure you don\'t have any modified files.') |
| return -1 |
| |
| logging.debug('Checking for a previous roll branch.') |
| if close_previous_roll: |
| self.Abort() |
| |
| logging.debug('Pulling latest changes') |
| if not ignore_checks: |
| self._RunCommand(['git', 'pull']) |
| |
| self._RunCommand(['git', 'checkout', '-b', ROLL_BRANCH_NAME]) |
| |
| # Modify Chromium's DEPS file. |
| |
| # Parse current hashes. |
| deps_filename = os.path.join(self._chromium_src, 'DEPS') |
| deps = _ParseDepsFile(deps_filename) |
| webrtc_current = self._GetDepsCommitInfo(deps, WEBRTC_PATH) |
| libjingle_current = self._GetDepsCommitInfo(deps, LIBJINGLE_PATH) |
| |
| # Find ToT revisions. |
| webrtc_latest = self._GetCommitInfo(WEBRTC_PATH) |
| libjingle_latest = self._GetCommitInfo(LIBJINGLE_PATH) |
| |
| self._UpdateDep(deps_filename, WEBRTC_PATH, webrtc_latest) |
| self._UpdateDep(deps_filename, LIBJINGLE_PATH, libjingle_latest) |
| |
| if self._IsTreeClean(): |
| logging.debug('Tree is clean - no changes detected.') |
| self._DeleteRollBranch() |
| else: |
| self._UpdateReadmeFile(LIBJINGLE_README, libjingle_latest.commit_position) |
| description = _GenerateCLDescription(webrtc_current, libjingle_current, |
| webrtc_latest, libjingle_latest) |
| logging.debug('Committing changes locally.') |
| self._RunCommand(['git', 'add', '--update', '.']) |
| self._RunCommand(['git', 'commit', '-m', description]) |
| logging.debug('Uploading changes...') |
| self._RunCommand(['git', 'cl', 'upload', '-m', description], |
| extra_env={'EDITOR': 'true'}) |
| cl_info = self._GetCLInfo() |
| logging.debug('Issue: %d URL: %s', cl_info.issue, cl_info.url) |
| |
| if not dry_run and not no_commit: |
| logging.debug('Sending the CL to the CQ...') |
| self._RunCommand(['git', 'cl', 'set_commit']) |
| logging.debug('Sent the CL to the CQ. Monitor here: %s', cl_info.url) |
| |
| # TODO(kjellander): Checkout masters/previous branches again. |
| return 0 |
| |
| def _UpdateDep(self, deps_filename, dep_relative_to_src, commit_info): |
| dep_name = os.path.join('src', dep_relative_to_src) |
| comment = 'commit position %s' % commit_info.commit_position |
| |
| # roll_dep.py relies on cwd being the Chromium checkout, so let's |
| # temporarily change the working directory and then change back. |
| cwd = os.getcwd() |
| os.chdir(os.path.dirname(deps_filename)) |
| roll_dep.update_deps(deps_filename, dep_relative_to_src, dep_name, |
| commit_info.git_commit, comment) |
| os.chdir(cwd) |
| |
| def _DeleteRollBranch(self): |
| self._RunCommand(['git', 'checkout', 'master']) |
| self._RunCommand(['git', 'branch', '-D', ROLL_BRANCH_NAME]) |
| logging.debug('Deleted the local roll branch (%s)', ROLL_BRANCH_NAME) |
| |
| |
| def _GetBranches(self): |
| """Returns a tuple of active,branches. |
| |
| The 'active' is the name of the currently active branch and 'branches' is a |
| list of all branches. |
| """ |
| lines = self._RunCommand(['git', 'branch']).split('\n') |
| branches = [] |
| active = '' |
| for l in lines: |
| if '*' in l: |
| # The assumption is that the first char will always be the '*'. |
| active = l[1:].strip() |
| branches.append(active) |
| else: |
| b = l.strip() |
| if b: |
| branches.append(b) |
| return (active, branches) |
| |
| def Abort(self): |
| active_branch, branches = self._GetBranches() |
| if active_branch == ROLL_BRANCH_NAME: |
| active_branch = 'master' |
| if ROLL_BRANCH_NAME in branches: |
| print 'Aborting pending roll.' |
| self._RunCommand(['git', 'checkout', ROLL_BRANCH_NAME]) |
| # Ignore an error here in case an issue wasn't created for some reason. |
| self._RunCommand(['git', 'cl', 'set_close'], ignore_exit_code=True) |
| self._RunCommand(['git', 'checkout', active_branch]) |
| self._RunCommand(['git', 'branch', '-D', ROLL_BRANCH_NAME]) |
| return 0 |
| |
| def WaitForTrybots(self): |
| active_branch, _ = self._GetBranches() |
| if active_branch != ROLL_BRANCH_NAME: |
| self._RunCommand(['git', 'checkout', ROLL_BRANCH_NAME]) |
| cl_info = self._GetCLInfo() |
| return _WaitForTrybots(cl_info.issue, cl_info.rietveld_server) |
| |
| |
| def main(): |
| if sys.platform in ('win32', 'cygwin'): |
| logging.error('Only Linux and Mac platforms are supported right now.') |
| return -1 |
| |
| parser = argparse.ArgumentParser( |
| description='Find webrtc and libjingle revisions for roll.') |
| parser.add_argument('--chromium-checkout', type=str, |
| help=('Path to the Chromium checkout (src/ dir) to perform the roll in ' |
| '(optional if the current working dir is a Chromium checkout).')) |
| parser.add_argument('--abort', |
| help=('Aborts a previously prepared roll. ' |
| 'Closes any associated issues and deletes the roll branches'), |
| action='store_true') |
| parser.add_argument('--no-commit', |
| help=('Don\'t send the CL to the CQ. This is useful if additional changes ' |
| 'are needed to the CL (like for API changes).'), |
| action='store_true') |
| parser.add_argument('--wait-for-trybots', |
| help=('Waits until all trybots from a previously created roll are either ' |
| 'successful or at least one has failed. This is useful to be able to ' |
| 'continuously run this script but not initiating new rolls until a ' |
| 'previous one is known to have passed or failed.'), |
| action='store_true') |
| parser.add_argument('--close-previous-roll', action='store_true', |
| help='Abort a previous roll if one exists.') |
| parser.add_argument('--dry-run', action='store_true', default=False, |
| help='Create branches and CLs but doesn\'t send tryjobs or commit.') |
| parser.add_argument('--ignore-checks', action='store_true', default=False, |
| help=('Skips checks for being on the master branch, dirty workspaces and ' |
| 'the updating of the checkout. Will still delete and create local ' |
| 'Git branches.')) |
| parser.add_argument('-v', '--verbose', action='store_true', default=False, |
| help='Be extra verbose in printing of log messages.') |
| args = parser.parse_args() |
| |
| if args.verbose: |
| logging.basicConfig(level=logging.DEBUG) |
| else: |
| logging.basicConfig(level=logging.ERROR) |
| |
| if args.chromium_checkout: |
| if not _IsChromiumCheckout(args.chromium_checkout): |
| logging.error('Cannot find the specified Chromium checkout at: %s', |
| args.chromium_checkout) |
| return -2 |
| else: |
| # Try fallback on current working directory: |
| cwd = os.getcwd() |
| if _IsChromiumCheckout(cwd): |
| args.chromium_checkout = cwd |
| else: |
| logging.error( |
| '--chromium-checkout not specified and the current working directory ' |
| ' is not a Chromium checkout. Fix either and try again.') |
| return -2 |
| |
| autoroller = AutoRoller(args.chromium_checkout) |
| if args.abort: |
| return autoroller.Abort() |
| elif args.wait_for_trybots: |
| return autoroller.WaitForTrybots() |
| else: |
| return autoroller.PrepareRoll(args.dry_run, args.ignore_checks, |
| args.no_commit, args.close_previous_roll) |
| |
| if __name__ == '__main__': |
| sys.exit(main()) |
