# Copyright (c) 2013 The Chromium Authors. All rights reserved. | |
# Use of this source code is governed by a BSD-style license that can be | |
# found in the LICENSE file. | |
"""Set of operations/utilities related to checking out the depot, and | |
outputting annotations on the buildbot waterfall. These are intended to be | |
used by the bisection scripts.""" | |
import errno | |
import os | |
import shutil | |
import stat | |
import subprocess | |
import sys | |
DEFAULT_GCLIENT_CUSTOM_DEPS = { | |
"src/data/page_cycler": "https://chrome-internal.googlesource.com/" | |
"chrome/data/page_cycler/.git", | |
"src/data/dom_perf": "https://chrome-internal.googlesource.com/" | |
"chrome/data/dom_perf/.git", | |
"src/data/mach_ports": "https://chrome-internal.googlesource.com/" | |
"chrome/data/mach_ports/.git", | |
"src/tools/perf/data": "https://chrome-internal.googlesource.com/" | |
"chrome/tools/perf/data/.git", | |
"src/third_party/adobe/flash/binaries/ppapi/linux": | |
"https://chrome-internal.googlesource.com/" | |
"chrome/deps/adobe/flash/binaries/ppapi/linux/.git", | |
"src/third_party/adobe/flash/binaries/ppapi/linux_x64": | |
"https://chrome-internal.googlesource.com/" | |
"chrome/deps/adobe/flash/binaries/ppapi/linux_x64/.git", | |
"src/third_party/adobe/flash/binaries/ppapi/mac": | |
"https://chrome-internal.googlesource.com/" | |
"chrome/deps/adobe/flash/binaries/ppapi/mac/.git", | |
"src/third_party/adobe/flash/binaries/ppapi/mac_64": | |
"https://chrome-internal.googlesource.com/" | |
"chrome/deps/adobe/flash/binaries/ppapi/mac_64/.git", | |
"src/third_party/adobe/flash/binaries/ppapi/win": | |
"https://chrome-internal.googlesource.com/" | |
"chrome/deps/adobe/flash/binaries/ppapi/win/.git", | |
"src/third_party/adobe/flash/binaries/ppapi/win_x64": | |
"https://chrome-internal.googlesource.com/" | |
"chrome/deps/adobe/flash/binaries/ppapi/win_x64/.git",} | |
GCLIENT_SPEC_DATA = [ | |
{ "name" : "src", | |
"url" : "https://chromium.googlesource.com/chromium/src.git", | |
"deps_file" : ".DEPS.git", | |
"managed" : True, | |
"custom_deps" : {}, | |
"safesync_url": "", | |
}, | |
] | |
GCLIENT_SPEC_ANDROID = "\ntarget_os = ['android']" | |
GCLIENT_CUSTOM_DEPS_V8 = {"src/v8_bleeding_edge": "git://github.com/v8/v8.git"} | |
FILE_DEPS_GIT = '.DEPS.git' | |
REPO_PARAMS = [ | |
'https://chrome-internal.googlesource.com/chromeos/manifest-internal/', | |
'--repo-url', | |
'https://git.chromium.org/external/repo.git' | |
] | |
REPO_SYNC_COMMAND = 'git checkout -f $(git rev-list --max-count=1 '\ | |
'--before=%d remotes/m/master)' | |
ORIGINAL_ENV = {} | |
def OutputAnnotationStepStart(name): | |
"""Outputs appropriate annotation to signal the start of a step to | |
a trybot. | |
Args: | |
name: The name of the step. | |
""" | |
print '@@@SEED_STEP %s@@@' % name | |
print '@@@STEP_CURSOR %s@@@' % name | |
print '@@@STEP_STARTED@@@' | |
sys.stdout.flush() | |
def OutputAnnotationStepClosed(): | |
"""Outputs appropriate annotation to signal the closing of a step to | |
a trybot.""" | |
print '@@@STEP_CLOSED@@@' | |
sys.stdout.flush() | |
def OutputAnnotationStepLink(label, url): | |
"""Outputs appropriate annotation to print a link. | |
Args: | |
label: The name to print. | |
url: The url to print. | |
""" | |
print '@@@STEP_LINK@%s@%s@@@' % (label, url) | |
sys.stdout.flush() | |
def CreateAndChangeToSourceDirectory(working_directory): | |
"""Creates a directory 'bisect' as a subdirectory of 'working_directory'. If | |
the function is successful, the current working directory will change to that | |
of the new 'bisect' directory. | |
Returns: | |
True if the directory was successfully created (or already existed). | |
""" | |
cwd = os.getcwd() | |
os.chdir(working_directory) | |
try: | |
os.mkdir('bisect') | |
except OSError, e: | |
if e.errno != errno.EEXIST: | |
return False | |
os.chdir('bisect') | |
return True | |
def SubprocessCall(cmd, cwd=None): | |
"""Runs a subprocess with specified parameters. | |
Args: | |
params: A list of parameters to pass to gclient. | |
cwd: Working directory to run from. | |
Returns: | |
The return code of the call. | |
""" | |
if os.name == 'nt': | |
# "HOME" isn't normally defined on windows, but is needed | |
# for git to find the user's .netrc file. | |
if not os.getenv('HOME'): | |
os.environ['HOME'] = os.environ['USERPROFILE'] | |
shell = os.name == 'nt' | |
return subprocess.call(cmd, shell=shell, cwd=cwd) | |
def RunGClient(params, cwd=None): | |
"""Runs gclient with the specified parameters. | |
Args: | |
params: A list of parameters to pass to gclient. | |
cwd: Working directory to run from. | |
Returns: | |
The return code of the call. | |
""" | |
cmd = ['gclient'] + params | |
return SubprocessCall(cmd, cwd=cwd) | |
def RunRepo(params): | |
"""Runs cros repo command with specified parameters. | |
Args: | |
params: A list of parameters to pass to gclient. | |
Returns: | |
The return code of the call. | |
""" | |
cmd = ['repo'] + params | |
return SubprocessCall(cmd) | |
def RunRepoSyncAtTimestamp(timestamp): | |
"""Syncs all git depots to the timestamp specified using repo forall. | |
Args: | |
params: Unix timestamp to sync to. | |
Returns: | |
The return code of the call. | |
""" | |
repo_sync = REPO_SYNC_COMMAND % timestamp | |
cmd = ['forall', '-c', REPO_SYNC_COMMAND % timestamp] | |
return RunRepo(cmd) | |
def RunGClientAndCreateConfig(opts, custom_deps=None, cwd=None): | |
"""Runs gclient and creates a config containing both src and src-internal. | |
Args: | |
opts: The options parsed from the command line through parse_args(). | |
custom_deps: A dictionary of additional dependencies to add to .gclient. | |
cwd: Working directory to run from. | |
Returns: | |
The return code of the call. | |
""" | |
spec = GCLIENT_SPEC_DATA | |
if custom_deps: | |
for k, v in custom_deps.iteritems(): | |
spec[0]['custom_deps'][k] = v | |
# Cannot have newlines in string on windows | |
spec = 'solutions =' + str(spec) | |
spec = ''.join([l for l in spec.splitlines()]) | |
if opts.target_platform == 'android': | |
spec += GCLIENT_SPEC_ANDROID | |
return_code = RunGClient( | |
['config', '--spec=%s' % spec, '--git-deps'], cwd=cwd) | |
return return_code | |
def IsDepsFileBlink(): | |
"""Reads .DEPS.git and returns whether or not we're using blink. | |
Returns: | |
True if blink, false if webkit. | |
""" | |
locals = {'Var': lambda _: locals["vars"][_], | |
'From': lambda *args: None} | |
execfile(FILE_DEPS_GIT, {}, locals) | |
return 'blink.git' in locals['vars']['webkit_url'] | |
def RemoveThirdPartyWebkitDirectory(): | |
"""Removes third_party/WebKit. | |
Returns: | |
True on success. | |
""" | |
try: | |
path_to_dir = os.path.join(os.getcwd(), 'third_party', 'WebKit') | |
if os.path.exists(path_to_dir): | |
shutil.rmtree(path_to_dir) | |
except OSError, e: | |
if e.errno != errno.ENOENT: | |
return False | |
return True | |
def OnAccessError(func, path, exc_info): | |
""" | |
Source: http://stackoverflow.com/questions/2656322/python-shutil-rmtree-fails-on-windows-with-access-is-denied | |
Error handler for ``shutil.rmtree``. | |
If the error is due to an access error (read only file) | |
it attempts to add write permission and then retries. | |
If the error is for another reason it re-raises the error. | |
Args: | |
func: The function that raised the error. | |
path: The path name passed to func. | |
exc_info: Exception information returned by sys.exc_info(). | |
""" | |
if not os.access(path, os.W_OK): | |
# Is the error an access error ? | |
os.chmod(path, stat.S_IWUSR) | |
func(path) | |
else: | |
raise | |
def RemoveThirdPartyLibjingleDirectory(): | |
"""Removes third_party/libjingle. At some point, libjingle was causing issues | |
syncing when using the git workflow (crbug.com/266324). | |
Returns: | |
True on success. | |
""" | |
path_to_dir = os.path.join(os.getcwd(), 'third_party', 'libjingle') | |
try: | |
if os.path.exists(path_to_dir): | |
shutil.rmtree(path_to_dir, onerror=OnAccessError) | |
except OSError, e: | |
print 'Error #%d while running shutil.rmtree(%s): %s' % ( | |
e.errno, path_to_dir, str(e)) | |
if e.errno != errno.ENOENT: | |
return False | |
return True | |
def RunGClientAndSync(cwd=None): | |
"""Runs gclient and does a normal sync. | |
Args: | |
cwd: Working directory to run from. | |
Returns: | |
The return code of the call. | |
""" | |
params = ['sync', '--verbose', '--nohooks', '--reset', '--force'] | |
return RunGClient(params, cwd=cwd) | |
def SetupGitDepot(opts, custom_deps): | |
"""Sets up the depot for the bisection. The depot will be located in a | |
subdirectory called 'bisect'. | |
Args: | |
opts: The options parsed from the command line through parse_args(). | |
custom_deps: A dictionary of additional dependencies to add to .gclient. | |
Returns: | |
True if gclient successfully created the config file and did a sync, False | |
otherwise. | |
""" | |
name = 'Setting up Bisection Depot' | |
if opts.output_buildbot_annotations: | |
OutputAnnotationStepStart(name) | |
passed = False | |
if not RunGClientAndCreateConfig(opts, custom_deps): | |
passed_deps_check = True | |
if os.path.isfile(os.path.join('src', FILE_DEPS_GIT)): | |
cwd = os.getcwd() | |
os.chdir('src') | |
if not IsDepsFileBlink(): | |
passed_deps_check = RemoveThirdPartyWebkitDirectory() | |
else: | |
passed_deps_check = True | |
if passed_deps_check: | |
passed_deps_check = RemoveThirdPartyLibjingleDirectory() | |
os.chdir(cwd) | |
if passed_deps_check: | |
RunGClient(['revert']) | |
if not RunGClientAndSync(): | |
passed = True | |
if opts.output_buildbot_annotations: | |
OutputAnnotationStepClosed() | |
return passed | |
def SetupCrosRepo(): | |
"""Sets up cros repo for bisecting chromeos. | |
Returns: | |
Returns 0 on success. | |
""" | |
cwd = os.getcwd() | |
try: | |
os.mkdir('cros') | |
except OSError, e: | |
if e.errno != errno.EEXIST: | |
return False | |
os.chdir('cros') | |
cmd = ['init', '-u'] + REPO_PARAMS | |
passed = False | |
if not RunRepo(cmd): | |
if not RunRepo(['sync']): | |
passed = True | |
os.chdir(cwd) | |
return passed | |
def CopyAndSaveOriginalEnvironmentVars(): | |
"""Makes a copy of the current environment variables.""" | |
# TODO: Waiting on crbug.com/255689, will remove this after. | |
vars_to_remove = [] | |
for k, v in os.environ.iteritems(): | |
if 'ANDROID' in k: | |
vars_to_remove.append(k) | |
vars_to_remove.append('CHROME_SRC') | |
vars_to_remove.append('CHROMIUM_GYP_FILE') | |
vars_to_remove.append('GYP_CROSSCOMPILE') | |
vars_to_remove.append('GYP_DEFINES') | |
vars_to_remove.append('GYP_GENERATORS') | |
vars_to_remove.append('GYP_GENERATOR_FLAGS') | |
vars_to_remove.append('OBJCOPY') | |
for k in vars_to_remove: | |
if os.environ.has_key(k): | |
del os.environ[k] | |
global ORIGINAL_ENV | |
ORIGINAL_ENV = os.environ.copy() | |
def SetupAndroidBuildEnvironment(opts, path_to_src=None): | |
"""Sets up the android build environment. | |
Args: | |
opts: The options parsed from the command line through parse_args(). | |
path_to_src: Path to the src checkout. | |
Returns: | |
True if successful. | |
""" | |
# Revert the environment variables back to default before setting them up | |
# with envsetup.sh. | |
env_vars = os.environ.copy() | |
for k, _ in env_vars.iteritems(): | |
del os.environ[k] | |
for k, v in ORIGINAL_ENV.iteritems(): | |
os.environ[k] = v | |
path_to_file = os.path.join('build', 'android', 'envsetup.sh') | |
proc = subprocess.Popen(['bash', '-c', 'source %s && env' % path_to_file], | |
stdout=subprocess.PIPE, | |
stderr=subprocess.PIPE, | |
cwd=path_to_src) | |
(out, _) = proc.communicate() | |
for line in out.splitlines(): | |
(k, _, v) = line.partition('=') | |
os.environ[k] = v | |
return not proc.returncode | |
def SetupPlatformBuildEnvironment(opts): | |
"""Performs any platform specific setup. | |
Args: | |
opts: The options parsed from the command line through parse_args(). | |
Returns: | |
True if successful. | |
""" | |
if opts.target_platform == 'android': | |
CopyAndSaveOriginalEnvironmentVars() | |
return SetupAndroidBuildEnvironment(opts) | |
elif opts.target_platform == 'cros': | |
return SetupCrosRepo() | |
return True | |
def CheckIfBisectDepotExists(opts): | |
"""Checks if the bisect directory already exists. | |
Args: | |
opts: The options parsed from the command line through parse_args(). | |
Returns: | |
Returns True if it exists. | |
""" | |
path_to_dir = os.path.join(opts.working_directory, 'bisect', 'src') | |
return os.path.exists(path_to_dir) | |
def CreateBisectDirectoryAndSetupDepot(opts, custom_deps): | |
"""Sets up a subdirectory 'bisect' and then retrieves a copy of the depot | |
there using gclient. | |
Args: | |
opts: The options parsed from the command line through parse_args(). | |
custom_deps: A dictionary of additional dependencies to add to .gclient. | |
""" | |
if not CreateAndChangeToSourceDirectory(opts.working_directory): | |
raise RuntimeError('Could not create bisect directory.') | |
if not SetupGitDepot(opts, custom_deps): | |
raise RuntimeError('Failed to grab source.') |