blob: ddc05fdc5cd71ae364c9d24a8f858f0a4d70583a [file] [log] [blame]
#!/usr/bin/env python
# Copyright (c) 2013 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
"""Runs all the buildbot steps for ChromeDriver except for update/compile."""
import bisect
import csv
import datetime
import json
import optparse
import os
import platform as platform_module
import re
import shutil
import StringIO
import subprocess
import sys
import tempfile
import time
import urllib2
import zipfile
import archive
import chrome_paths
import util
_THIS_DIR = os.path.abspath(os.path.dirname(__file__))
GS_ARCHIVE_BUCKET = 'gs://chromedriver-prebuilts'
GS_ZIP_PREFIX = 'chromedriver2_prebuilts'
GS_RC_BUCKET = 'gs://chromedriver-rc'
GS_RELEASE_PATH = GS_RC_BUCKET + '/releases'
RC_LOG_FORMAT = '%s_log.json'
RC_ZIP_FORMAT = 'chromedriver_%s_%s_%s.zip'
RC_ZIP_GLOB = 'chromedriver_%s_%s_*'
RELEASE_ZIP_FORMAT = 'chromedriver_%s_%s.zip'
SCRIPT_DIR = os.path.join(_THIS_DIR, os.pardir, os.pardir, os.pardir, os.pardir,
os.pardir, os.pardir, os.pardir, 'scripts')
SITE_CONFIG_DIR = os.path.join(_THIS_DIR, os.pardir, os.pardir, os.pardir,
os.pardir, os.pardir, os.pardir, os.pardir,
'site_config')
sys.path.append(SCRIPT_DIR)
sys.path.append(SITE_CONFIG_DIR)
from slave import gsutil_download
from slave import slave_utils
def ArchivePrebuilts(revision):
"""Uploads the prebuilts to google storage."""
util.MarkBuildStepStart('archive')
prebuilts = ['chromedriver2_server',
'chromedriver2_unittests', 'chromedriver2_tests']
build_dir = chrome_paths.GetBuildDir(prebuilts[0:1])
zip_name = '%s_r%s.zip' % (GS_ZIP_PREFIX, revision)
temp_dir = util.MakeTempDir()
zip_path = os.path.join(temp_dir, zip_name)
print 'Zipping prebuilts %s' % zip_path
f = zipfile.ZipFile(zip_path, 'w', zipfile.ZIP_DEFLATED)
for prebuilt in prebuilts:
f.write(os.path.join(build_dir, prebuilt), prebuilt)
f.close()
if slave_utils.GSUtilCopyFile(zip_path, GS_ARCHIVE_BUCKET):
util.MarkBuildStepError()
def DownloadPrebuilts():
"""Downloads the most recent prebuilts from google storage."""
util.MarkBuildStepStart('Download chromedriver prebuilts')
temp_dir = util.MakeTempDir()
zip_path = os.path.join(temp_dir, 'chromedriver2_prebuilts.zip')
if gsutil_download.DownloadLatestFile(GS_ARCHIVE_BUCKET, GS_ZIP_PREFIX,
zip_path):
util.MarkBuildStepError()
build_dir = chrome_paths.GetBuildDir(['host_forwarder'])
print 'Unzipping prebuilts %s to %s' % (zip_path, build_dir)
f = zipfile.ZipFile(zip_path, 'r')
f.extractall(build_dir)
f.close()
# Workaround for Python bug: http://bugs.python.org/issue15795
os.chmod(os.path.join(build_dir, 'chromedriver2_server'), 0700)
def GetDownloads():
"""Gets the chromedriver google code downloads page."""
result, output = slave_utils.GSUtilListBucket(GS_RELEASE_PATH + '/*', ['-l'])
if result:
return ''
return output
def GetTestResultsLog(platform):
"""Gets the test results log for the given platform.
Returns:
A dictionary where the keys are SVN revisions and the values are booleans
indicating whether the tests passed.
"""
temp_log = tempfile.mkstemp()[1]
log_name = RC_LOG_FORMAT % platform
result = slave_utils.GSUtilDownloadFile(
'%s/%s' % (GS_RC_BUCKET, log_name), temp_log)
if result:
return {}
with open(temp_log, 'rb') as log_file:
json_dict = json.load(log_file)
# Workaround for json encoding dictionary keys as strings.
return dict([(int(v[0]), v[1]) for v in json_dict.items()])
def _PutTestResultsLog(platform, test_results_log):
"""Pushes the given test results log to google storage."""
temp_dir = util.MakeTempDir()
log_name = RC_LOG_FORMAT % platform
log_path = os.path.join(temp_dir, log_name)
with open(log_path, 'wb') as log_file:
json.dump(test_results_log, log_file)
if slave_utils.GSUtilCopyFile(log_path, GS_RC_BUCKET):
raise Exception('Failed to upload test results log to google storage')
def UpdateTestResultsLog(platform, revision, passed):
"""Updates the test results log for the given platform.
Args:
platform: The platform name.
revision: The SVN revision number.
passed: Boolean indicating whether the tests passed at this revision.
"""
assert isinstance(revision, int), 'The revision must be an integer'
log = GetTestResultsLog(platform)
if len(log) > 500:
del log[min(log.keys())]
assert revision not in log, 'Results already exist for revision %s' % revision
log[revision] = bool(passed)
_PutTestResultsLog(platform, log)
def GetVersion():
"""Get the current chromedriver version."""
with open(os.path.join(_THIS_DIR, 'VERSION'), 'r') as f:
return f.read().strip()
def GetSupportedChromeVersions():
"""Get the minimum and maximum supported Chrome versions.
Returns:
A tuple of the form (min_version, max_version).
"""
# Minimum supported Chrome version is embedded as:
# const int kMinimumSupportedChromeVersion[] = {27, 0, 1453, 0};
with open(os.path.join(_THIS_DIR, 'chrome', 'version.cc'), 'r') as f:
lines = f.readlines()
chrome_min_version_line = filter(
lambda x: 'kMinimumSupportedChromeVersion' in x, lines)
chrome_min_version = chrome_min_version_line[0].split('{')[1].split(',')[0]
with open(os.path.join(chrome_paths.GetSrc(), 'chrome', 'VERSION'), 'r') as f:
chrome_max_version = f.readlines()[0].split('=')[1].strip()
return (chrome_min_version, chrome_max_version)
def GetReleaseCandidateUrl(platform):
"""Get the google storage url for the current version's release candidate."""
rc_glob = RC_ZIP_GLOB % (platform, GetVersion())
rc_url = '%s/%s' % (GS_RC_BUCKET, rc_glob)
result, output = slave_utils.GSUtilListBucket(rc_url, ['-l'])
assert result == 0 and output, 'Release candidate doesn\'t exist'
rc_list = [b.split()[2] for b in output.split('\n') if GS_RC_BUCKET in b]
assert len(rc_list) == 1, 'Only one release candidate should exist'
return rc_list[0]
def ReleaseCandidateExists(platform):
"""Return whether a release candidate exists for the current version."""
try:
GetReleaseCandidateUrl(platform)
return True
except AssertionError:
return False
def DeleteReleaseCandidate(platform):
"""Delete the release candidate for the current version."""
rc_url = GetReleaseCandidateUrl(platform)
result = slave_utils.GSUtilDeleteFile(rc_url)
assert result == 0, 'Failed to delete %s' % rc_url
def DownloadReleaseCandidate(platform):
"""Download the release candidate for the current version."""
release_name = RELEASE_ZIP_FORMAT % (platform, GetVersion())
rc_url = GetReleaseCandidateUrl(platform)
temp_dir = util.MakeTempDir()
zip_path = os.path.join(temp_dir, release_name)
slave_utils.GSUtilDownloadFile(rc_url, zip_path)
return zip_path
def GetReleaseCandidateRevision(platform):
"""Get the SVN revision for the current release candidate."""
rc_url = GetReleaseCandidateUrl(platform)
match = re.search(RC_ZIP_FORMAT % (platform, GetVersion(), '(\d+)'), rc_url)
assert match, 'Bad format for release candidate zip name: %s' % rc_url
return int(match.group(1))
def _RevisionState(test_results_log, revision):
"""Check the state of tests at a given SVN revision.
Considers tests as having passed at a revision if they passed at revisons both
before and after.
Args:
test_results_log: A test results log dictionary from GetTestResultsLog().
revision: The revision to check at.
Returns:
'passed', 'failed', or 'unknown'
"""
assert isinstance(revision, int), 'The revision must be an integer'
keys = sorted(test_results_log.keys())
# Return passed if the exact revision passed on Android.
if revision in test_results_log:
return 'passed' if test_results_log[revision] else 'failed'
# Tests were not run on this exact revision on Android.
index = bisect.bisect_right(keys, revision)
# Tests have not yet run on Android at or above this revision.
if index == len(test_results_log):
return 'unknown'
# No log exists for any prior revision, assume it failed.
if index == 0:
return 'failed'
# Return passed if the revisions on both sides passed.
if test_results_log[keys[index]] and test_results_log[keys[index - 1]]:
return 'passed'
return 'failed'
def _MaybeUpdateReleaseCandidate(platform, revision):
"""Uploads a release candidate if one doesn't already exist."""
assert platform != 'android'
version = GetVersion()
if not ReleaseCandidateExists(platform):
rc_file = _ConstructReleaseCandidate(platform, revision)
util.MarkBuildStepStart('upload release candidate')
if slave_utils.GSUtilCopyFile(rc_file, GS_RC_BUCKET):
util.MarkBuildStepError()
else:
print 'Release candidate already exists'
def _MaybeRelease(platform):
"""Releases the release candidate if conditions are right."""
assert platform != 'android'
if not ReleaseCandidateExists(platform):
return
revision = GetReleaseCandidateRevision(platform)
android_tests = _RevisionState(GetTestResultsLog('android'), revision)
if android_tests == 'failed':
print 'Android tests did not pass at revision %s' % revision
version = GetVersion()
DeleteReleaseCandidate(platform)
elif android_tests == 'passed':
print 'Android tests passed at revision %s' % revision
Release(platform, revision)
else:
print 'Android tests have not run at a revision as recent as %s' % revision
def _ConstructReleaseCandidate(platform, revision):
"""Constructs a release candidate zip from the current build."""
zip_name = RC_ZIP_FORMAT % (platform, GetVersion(), revision)
if util.IsWindows():
server_orig_name = 'chromedriver2_server.exe'
server_name = 'chromedriver.exe'
else:
server_orig_name = 'chromedriver2_server'
server_name = 'chromedriver'
server = os.path.join(chrome_paths.GetBuildDir([server_orig_name]),
server_orig_name)
print 'Zipping ChromeDriver server', server
temp_dir = util.MakeTempDir()
zip_path = os.path.join(temp_dir, zip_name)
f = zipfile.ZipFile(zip_path, 'w', zipfile.ZIP_DEFLATED)
f.write(server, server_name)
f.close()
return zip_path
def Release(platform, revision):
"""Releases the current release candidate if it hasn't been released yet."""
version = GetVersion()
chrome_min_version, chrome_max_version = GetSupportedChromeVersions()
zip_name = RELEASE_ZIP_FORMAT % (platform, GetVersion())
if zip_name in GetDownloads():
print 'chromedriver version %s has already been released for %s' % (
version, platform)
return 0
util.MarkBuildStepStart('releasing %s' % zip_name)
zip_path = DownloadReleaseCandidate(platform)
if slave_utils.GSUtilCopyFile(zip_path, GS_RELEASE_PATH):
util.MarkBuildStepError()
_MaybeUploadReleaseNotes(version)
def _MaybeUploadReleaseNotes(version):
"""Upload release notes if conditions are right."""
name_template = 'release_notes_%s.txt'
new_name = name_template % version
prev_version = '.'.join([version.split('.')[0],
str(int(version.split('.')[1]) - 1)])
old_name = name_template % prev_version
fixed_issues = []
query = ('https://code.google.com/p/chromedriver/issues/csv?'
'q=status%3AToBeReleased&colspec=ID%20Summary')
issues = StringIO.StringIO(urllib2.urlopen(query).read().split('\n', 1)[1])
for issue in csv.reader(issues):
if not issue:
continue
id = issue[0]
desc = issue[1]
labels = issue[2]
fixed_issues += ['Resolved issue %s: %s [%s]' % (id, desc, labels)]
old_notes = ''
temp_notes_fname = tempfile.mkstemp()[1]
if not slave_utils.GSUtilDownloadFile('%s/%s' % (GS_RELEASE_PATH, old_name),
temp_notes_fname):
with open(temp_notes_fname, 'rb') as f:
old_notes = f.read()
new_notes = '----------ChromeDriver v%s (%s)----------\n%s\n\n%s' % (
version, datetime.date.today().isoformat(),
'\n'.join(fixed_issues),
old_notes)
release_notes_txt = os.path.join(util.MakeTempDir(), new_name)
with open(release_notes_txt, 'w') as f:
f.write(new_notes)
if new_name in GetDownloads():
return
if slave_utils.GSUtilCopyFile(release_notes_txt, GS_RELEASE_PATH):
util.MarkBuildStepError()
def KillChromes():
chrome_map = {
'win': 'chrome.exe',
'mac': 'Chromium',
'linux': 'chrome',
}
if util.IsWindows():
cmd = ['taskkill', '/F', '/IM']
else:
cmd = ['killall', '-9']
cmd.append(chrome_map[util.GetPlatformName()])
util.RunCommand(cmd)
def CleanTmpDir():
tmp_dir = tempfile.gettempdir()
print 'cleaning temp directory:', tmp_dir
for file_name in os.listdir(tmp_dir):
if os.path.isdir(os.path.join(tmp_dir, file_name)):
print 'deleting sub-directory', file_name
shutil.rmtree(os.path.join(tmp_dir, file_name), True)
def WaitForLatestSnapshot(revision):
util.MarkBuildStepStart('wait_for_snapshot')
while True:
snapshot_revision = archive.GetLatestRevision(archive.Site.SNAPSHOT)
if int(snapshot_revision) >= int(revision):
break
util.PrintAndFlush('Waiting for snapshot >= %s, found %s' %
(revision, snapshot_revision))
time.sleep(60)
util.PrintAndFlush('Got snapshot revision %s' % snapshot_revision)
def main():
parser = optparse.OptionParser()
parser.add_option(
'', '--android-packages',
help='Comma separated list of application package names, '
'if running tests on Android.')
parser.add_option(
'-r', '--revision', type='int', default=None,
help='Chromium revision')
options, _ = parser.parse_args()
bitness = '32'
if util.IsLinux() and platform_module.architecture()[0] == '64bit':
bitness = '64'
platform = '%s%s' % (util.GetPlatformName(), bitness)
if options.android_packages:
platform = 'android'
if platform != 'android':
KillChromes()
CleanTmpDir()
if platform == 'android':
DownloadPrebuilts()
else:
if not options.revision:
parser.error('Must supply a --revision')
if platform == 'linux64':
ArchivePrebuilts(options.revision)
WaitForLatestSnapshot(options.revision)
cmd = [
sys.executable,
os.path.join(_THIS_DIR, 'test', 'run_all_tests.py'),
]
if platform == 'android':
cmd.append('--android-packages=' + options.android_packages)
passed = (util.RunCommand(cmd) == 0)
if platform == 'android':
UpdateTestResultsLog(platform, options.revision, passed)
elif passed:
_MaybeUpdateReleaseCandidate(platform, options.revision)
_MaybeRelease(platform)
if __name__ == '__main__':
main()