blob: 84d5fe92348447c0966b46b2cb40a6a66b37796a [file] [log] [blame]
# Copyright 2014 The Chromium OS Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
"""Should be run on a GCE instance to set up the build environment."""
from __future__ import print_function
import getpass
import os
import shutil
from chromite.compute import compute_configs
from chromite.compute import bot_constants
from chromite.lib import cros_build_lib
from chromite.lib import osutils
# Make the script more readable.
RunCommand = cros_build_lib.RunCommand
SudoRunCommand = cros_build_lib.SudoRunCommand
# Most credentials are stored in the home directory.
HOME_DIR = osutils.ExpandPath('~')
def SetupPrerequisites():
"""Installs packages required for Chrome OS build."""
SudoRunCommand(['apt-get', 'update'])
SudoRunCommand(['apt-get', '-y', '--force-yes', 'upgrade'])
# Chrome OS pre-requisite packages.
packages = ['git', 'curl', 'pbzip2', 'gawk', 'gitk', 'subversion']
# Required for CIDB.
packages += ['python-sqlalchemy', 'python-mysqldb']
# Required for payload generation outside of the chroot.
packages += ['python-protobuf']
# Required to install python packages only available via pip.
packages += ['python-pip']
# Packages to monitor system performance and usage.
packages += ['sysstat']
SudoRunCommand(['apt-get', '-y', 'install'] + packages)
def SetupPipPrerequisites():
"""Installs python packages via pip.
This assumes that pip itself is installed already.
# dict of package to version. Provide version None if you don't care about the
# version installed.
packages = {'python-statsd': '1.7.0', 'google-api-python-client': '1.4.0'}
for package, version in packages.iteritems():
install_atom = package
if version is not None:
install_atom += ('==' + version)
SudoRunCommand(['pip', 'install', install_atom])
def InstallChromeDependencies():
"""Installs packages required to build Chromium."""
# The relies on some packages that are not in
# the base image. Install them first before invoking the script.
SudoRunCommand(['apt-get', '-y', 'install',
with osutils.TempDir(prefix='tmp-chrome-deps') as tempdir:
RunCommand(['git', 'clone', bot_constants.CHROMIUM_BUILD_URL], cwd=tempdir)
RunCommand([os.path.join(tempdir, 'build', ''),
'--syms', '--no-prompt'])
def SetMountCount():
"""Sets mount count to a large number."""
for drive in compute_configs.DRIVES:
SudoRunCommand(['tune2fs', '-c', '150', os.path.join('dev', drive)],
def _SetupSVN():
"""Sets up the chromium svn username/password."""
# Create a ~/.subversion directory.
RunCommand(['svn', 'ls', ''], redirect_stdout=True)
# Change the setting to store the svn password.
sed_str = ('s/# store-plaintext-passwords = '
'no/store-plaintext-passwords = yes/g')
RunCommand(['sed', '-i', '-e', sed_str,
osutils.ExpandPath(os.path.join('~', '.subversion', 'servers'))])
password_path = osutils.ExpandPath(
os.path.join(BOT_CREDS_PATH, bot_constants.SVN_PASSWORD_FILE))
password = osutils.ReadFile(password_path).strip()
# `svn ls` each repository to store the password in ~/.subversion.
for svn_host in bot_constants.CHROMIUM_SVN_HOSTS:
for svn_repo in bot_constants.CHROMIUM_SVN_REPOS:
RunCommand(['svn', 'ls', '--username', bot_constants.BUILDBOT_SVN_USER,
'--password', password, 'svn://%s/%s' % (svn_host, svn_repo)],
def _SetupGoB():
"""Sets up GoB credentials."""
RunCommand(['git', 'config', '--global', '',
RunCommand(['git', 'config', '--global', '',
RunCommand(['git', 'clone', bot_constants.GCOMPUTE_TOOLS_URL],
cwd=HOME_DIR, redirect_stdout=True)
# Run git-cookie-authdaemon at boot time by adding it to
# /etc/rc.local
rc_local_path = os.path.join(os.path.sep, 'etc', 'rc.local')
daemon_path = os.path.join(HOME_DIR, 'gcompute-tools',
daemon_cmd = ['su', bot_constants.BUILDBOT_USER, '-c', daemon_path]
content = osutils.ReadFile(rc_local_path).replace('exit 0', '')
content += (' '.join(daemon_cmd) + '\n')
content += 'exit 0\n'
with osutils.TempDir() as tempdir:
tmp_file = os.path.join(tempdir, 'rc.local')
osutils.WriteFile(tmp_file, content)
os.chmod(tmp_file, 755)
SudoRunCommand(['mv', tmp_file, rc_local_path])
# Also run the daemon now so that subsequent setup steps get credentials.
# NB: It's important to redirect all pipes because the daemonize code here is
# broken, it leaves open fds behind, causing ssh to hang.
mute_output=True, combine_stdout_stderr=True)
def _SetupCIDB():
"""Copies cidb credentials."""
shutil.copytree(os.path.join(BOT_CREDS_PATH, bot_constants.CIDB_CREDS_DIR),
os.path.join(HOME_DIR, bot_constants.CIDB_CREDS_DIR))
def _SetupTreeStatus():
"""Copies credentials for updating tree status."""
os.path.join(BOT_CREDS_PATH, bot_constants.TREE_STATUS_PASSWORD_FILE),
def _SetupGmail():
"""Copies credentials for accessing gmail API."""
os.path.join(BOT_CREDS_PATH, bot_constants.GMAIL_CREDENTIALS_FILE),
def SetupCredentials():
"""Sets up various credentials."""
def SetupBuildbotEnvironment():
"""Sets up the buildbot environment."""
# Append host entries to /etc/hosts. This includes the buildbot
# master IP address.
host_entries = RunCommand(
['cat', os.path.join(BOT_CREDS_PATH, bot_constants.HOST_ENTRIES)],
SudoRunCommand(['tee', '-a', '/etc/hosts'], input=host_entries)
# Create the buildbot directory.
SudoRunCommand(['mkdir', '-p', bot_constants.BUILDBOT_DIR])
SudoRunCommand(['chown', '-R', '%s:%s' % (bot_constants.BUILDBOT_USER,
with osutils.TempDir() as tempdir:
# Download depot tools to a temp directory to bootstrap. `gclient
# sync` will create depot_tools in BUILDBOT_DIR later.
tmp_depot_tools_path = os.path.join(tempdir, 'depot_tools')
RunCommand(['git', 'clone', bot_constants.DEPOT_TOOLS_URL],
cwd=tempdir, redirect_stdout=True)
# `gclient` relies on depot_tools in $PATH, pass the extra
# envinornment variable.
path_env = '%s:%s' % (os.getenv('PATH'), tmp_depot_tools_path)
RunCommand(['gclient', 'config', bot_constants.CHROME_INFRA_SLAVE_REPO,
'--deps-file', bot_constants.CHROME_INFRA_SLAVE_DEPS_FILE],
cwd=bot_constants.BUILDBOT_DIR, extra_env={'PATH': path_env})
RunCommand(['gclient', 'sync', '--jobs', '5'],
redirect_stdout=True, extra_env={'PATH': path_env})
# Set up buildbot password.
config_dir = os.path.join(bot_constants.BUILDBOT_DIR, 'build', 'site_config')
os.path.join(BOT_CREDS_PATH, bot_constants.BUILDBOT_PASSWORD_FILE),
# Update the environment variable.
depot_tools_path = os.path.join(bot_constants.BUILDBOT_DIR, 'depot_tools')
RunCommand(['bash', '-c', r'echo export PATH=\$PATH:%s >> ~/.bashrc'
% depot_tools_path])
# Start buildbot slave at startup.
crontab_content = ''
result = RunCommand(
['crontab', '-l'], capture_output=True, error_code_ok=True)
crontab_content = result.output if result.returncode == 0 else ''
crontab_content += ('SHELL=/bin/bash\nUSER=chrome-bot\n'
'@reboot cd /b/build/slave && make start\n')
RunCommand(['crontab', '-'], input=crontab_content)
def TuneSystemSettings():
"""Tune the system settings for our build environment."""
# Increase the user-level file descriptor limits.
entries = ('* soft nofile 65536\n'
'* hard nofile 65536\n')
SudoRunCommand(['tee', '-a', '/etc/security/limits.conf'], input=entries)
def main(_argv):
assert getpass.getuser() == bot_constants.BUILDBOT_USER, (
'This script should be run by %s instead of %s!' % (
bot_constants.BUILDBOT_USER, getpass.getuser()))