blob: ccb1dcc19fe595e9c887a273d4b32bae8105a67b [file] [log] [blame]
# Copyright (c) 2012 The Chromium OS Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
"""This script is used to upload host prebuilts as well as board BINHOSTS.
Prebuilts are uploaded using gsutil to Google Storage. After these prebuilts
are successfully uploaded, a file is updated with the proper BINHOST version.
To read more about prebuilts/binhost binary packages please refer to:
Example of uploading prebuilt amd64 host files to Google Storage:
upload_prebuilts -p /b/cbuild/build -s -u gs://chromeos-prebuilt
Example of uploading x86-dogfood binhosts to Google Storage:
upload_prebuilts -b x86-dogfood -p /b/cbuild/build/ -u gs://chromeos-prebuilt -g
from __future__ import print_function
import argparse
import datetime
import functools
import glob
import multiprocessing
import os
import sys
import tempfile
from chromite.cbuildbot import constants
from chromite.cbuildbot import commands
from chromite.lib import binpkg
from chromite.lib import commandline
from chromite.lib import cros_build_lib
from chromite.lib import cros_logging as logging
from chromite.lib import git
from chromite.lib import gs
from chromite.lib import osutils
from chromite.lib import parallel
from chromite.lib import portage_util
from chromite.lib import toolchain
# How many times to retry uploads.
# Multiplier for how long to sleep (in seconds) between retries; will delay
# (1*sleep) the first time, then (2*sleep), continuing via attempt * sleep.
# The length of time (in seconds) that Portage should wait before refetching
# binpkgs from the same binhost. We don't ever modify binhosts, so this should
# be something big.
_BINPKG_TTL = 60 * 60 * 24 * 365
_HOST_PACKAGES_PATH = 'chroot/var/lib/portage/pkgs'
_CATEGORIES_PATH = 'chroot/etc/portage/categories'
_PYM_PATH = 'chroot/usr/lib/portage/pym'
_HOST_ARCH = 'amd64'
_BOARD_PATH = 'chroot/build/%(board)s'
_REL_BOARD_PATH = 'board/%(target)s/%(version)s'
_REL_HOST_PATH = 'host/%(host_arch)s/%(target)s/%(version)s'
# Private overlays to look at for builds to filter
# relative to build path
_PRIVATE_OVERLAY_DIR = 'src/private-overlays'
_GOOGLESTORAGE_GSUTIL_FILE = 'googlestorage_acl.txt'
_BINHOST_BASE_URL = 'gs://chromeos-prebuilt'
_PREBUILT_BASE_DIR = 'src/third_party/chromiumos-overlay/chromeos/config/'
# Created in the event of new host targets becoming available
_PREBUILT_MAKE_CONF = {'amd64': os.path.join(_PREBUILT_BASE_DIR,
class BuildTarget(object):
"""A board/variant/profile tuple."""
def __init__(self, board_variant, profile=None):
self.board_variant = board_variant
self.board, _, self.variant = board_variant.partition('_')
self.profile = profile
def __str__(self):
if self.profile:
return '%s_%s' % (self.board_variant, self.profile)
return self.board_variant
def __eq__(self, other):
return str(other) == str(self)
def __hash__(self):
return hash(str(self))
def UpdateLocalFile(filename, value, key='PORTAGE_BINHOST'):
"""Update the key in file with the value passed.
File format:
Note quotes are added automatically
filename: Name of file to modify.
value: Value to write with the key.
key: The variable key to update. (Default: PORTAGE_BINHOST)
True if changes were made to the file.
if os.path.exists(filename):
file_fh = open(filename)
file_fh = open(filename, 'w+')
file_lines = []
found = False
made_changes = False
keyval_str = '%(key)s=%(value)s'
for line in file_fh:
# Strip newlines from end of line. We already add newlines below.
line = line.rstrip("\n")
if len(line.split('=')) != 2:
# Skip any line that doesn't fit key=val.
file_var, file_val = line.split('=')
if file_var == key:
found = True
print('Updating %s=%s to %s="%s"' % (file_var, file_val, key, value))
value = '"%s"' % value
made_changes |= (file_val != value)
file_lines.append(keyval_str % {'key': key, 'value': value})
file_lines.append(keyval_str % {'key': file_var, 'value': file_val})
if not found:
value = '"%s"' % value
made_changes = True
file_lines.append(keyval_str % {'key': key, 'value': value})
# write out new file
osutils.WriteFile(filename, '\n'.join(file_lines) + '\n')
return made_changes
def RevGitFile(filename, data, retries=5, dryrun=False):
"""Update and push the git file.
filename: file to modify that is in a git repo already
data: A dict of key/values to update in |filename|
retries: The number of times to retry before giving up, default: 5
dryrun: If True, do not actually commit the change.
prebuilt_branch = 'prebuilt_branch'
cwd = os.path.abspath(os.path.dirname(filename))
commit = git.RunGit(cwd, ['rev-parse', 'HEAD']).output.rstrip()
description = '%s: updating %s' % (os.path.basename(filename),
', '.join(data.keys()))
# UpdateLocalFile will print out the keys/values for us.
print('Revving git file %s' % filename)
git.CreatePushBranch(prebuilt_branch, cwd)
for key, value in data.iteritems():
UpdateLocalFile(filename, value, key)
git.RunGit(cwd, ['add', filename])
git.RunGit(cwd, ['commit', '-m', description])
git.PushWithRetry(prebuilt_branch, cwd, dryrun=dryrun, retries=retries)
git.RunGit(cwd, ['checkout', commit])
def GetVersion():
"""Get the version to put in LATEST and update the git version with."""
def _GsUpload(gs_context, acl, local_file, remote_file):
"""Upload to GS bucket.
gs_context: A instance.
acl: The ACL to use for uploading the file.
local_file: The local file to be uploaded.
remote_file: The remote location to upload to.
CANNED_ACLS = ['public-read', 'private', 'bucket-owner-read',
'authenticated-read', 'bucket-owner-full-control',
if acl in CANNED_ACLS:
gs_context.Copy(local_file, remote_file, acl=acl)
# For private uploads we assume that the overlay board is set up properly
# and a googlestore_acl.xml is present. Otherwise, this script errors.
# We set version=0 here to ensure that the ACL is set only once (see
# http://b/15883752#comment54).
gs_context.Copy(local_file, remote_file, version=0)
except gs.GSContextPreconditionFailed as ex:
# If we received a GSContextPreconditionFailed error, we know that the
# file exists now, but we don't know whether our specific update
# succeeded. See http://b/15883752#comment62
'Assuming upload succeeded despite PreconditionFailed errors: %s', ex)
if acl.endswith('.xml'):
# Apply the passed in ACL xml file to the uploaded object.
gs_context.SetACL(remote_file, acl=acl)
gs_context.ChangeACL(remote_file, acl_args_file=acl)
def RemoteUpload(gs_context, acl, files, pool=10):
"""Upload to google storage.
Create a pool of process and call _GsUpload with the proper arguments.
gs_context: A instance.
acl: The canned acl used for uploading. acl can be one of: "public-read",
"public-read-write", "authenticated-read", "bucket-owner-read",
"bucket-owner-full-control", or "private".
files: dictionary with keys to local files and values to remote path.
pool: integer of maximum proesses to have at the same time.
Return a set of tuple arguments of the failed uploads
upload = functools.partial(_GsUpload, gs_context, acl)
tasks = [[key, value] for key, value in files.iteritems()]
parallel.RunTasksInProcessPool(upload, tasks, pool)
def GenerateUploadDict(base_local_path, base_remote_path, pkgs):
"""Build a dictionary of local remote file key pairs to upload.
base_local_path: The base path to the files on the local hard drive.
base_remote_path: The base path to the remote paths.
pkgs: The packages to upload.
Returns a dictionary of local_path/remote_path pairs
upload_files = {}
for pkg in pkgs:
suffix = pkg['CPV'] + '.tbz2'
local_path = os.path.join(base_local_path, suffix)
assert os.path.exists(local_path), '%s does not exist' % local_path
upload_files[local_path] = os.path.join(base_remote_path, suffix)
if pkg.get('DEBUG_SYMBOLS') == 'yes':
debugsuffix = pkg['CPV'] + '.debug.tbz2'
local_path = os.path.join(base_local_path, debugsuffix)
assert os.path.exists(local_path)
upload_files[local_path] = os.path.join(base_remote_path, debugsuffix)
return upload_files
def GetBoardOverlay(build_path, target):
"""Get the path to the board variant.
build_path: The path to the root of the build directory
target: The target board as a BuildTarget object.
The last overlay configured for the given board as a string.
board = target.board_variant
overlays = portage_util.FindOverlays(constants.BOTH_OVERLAYS, board,
# We only care about the last entry.
return overlays[-1]
def DeterminePrebuiltConfFile(build_path, target):
"""Determine the prebuilt.conf file that needs to be updated for prebuilts.
build_path: The path to the root of the build directory
target: String representation of the board. This includes host and board
A string path to a prebuilt.conf file to be updated.
if _HOST_ARCH == target:
# We are host.
# Without more examples of hosts this is a kludge for now.
# TODO(Scottz): as new host targets come online expand this to
# work more like boards.
make_path = _PREBUILT_MAKE_CONF[target]
# We are a board
board = GetBoardOverlay(build_path, target)
make_path = os.path.join(board, 'prebuilt.conf')
return make_path
def UpdateBinhostConfFile(path, key, value):
"""Update binhost config file file with key=value.
path: Filename to update.
key: Key to update.
value: New value for key.
cwd, filename = os.path.split(os.path.abspath(path))
if not git.GetCurrentBranch(cwd):
git.CreatePushBranch(constants.STABLE_EBUILD_BRANCH, cwd, sync=False)
osutils.WriteFile(path, '', mode='a')
if UpdateLocalFile(path, value, key):
desc = '%s: %s %s' % (filename, 'updating' if value else 'clearing', key)
git.Commit(cwd, desc)
def GenerateHtmlIndex(files, index, board, version):
"""Given the list of |files|, generate an index.html at |index|.
files: The list of files to link to.
index: The path to the html index.
board: Name of the board this index is for.
version: Build version this index is for.
head = """<html>
<title>Package Prebuilt Index: %(board)s / %(version)s</title>
<h2>Package Prebuilt Index: %(board)s / %(version)s</h2>"""
head %= {
'board': board,
'version': version,
files = files + [
'.|Google Storage Index',
commands.GenerateHtmlIndex(index, files, head=head)
def _GrabAllRemotePackageIndexes(binhost_urls):
"""Grab all of the packages files associated with a list of binhost_urls.
binhost_urls: The URLs for the directories containing the Packages files we
want to grab.
A list of PackageIndex objects.
pkg_indexes = []
for url in binhost_urls:
pkg_index = binpkg.GrabRemotePackageIndex(url)
if pkg_index:
return pkg_indexes
class PrebuiltUploader(object):
"""Synchronize host and board prebuilts."""
def __init__(self, upload_location, acl, binhost_base_url, pkg_indexes,
build_path, packages, skip_upload, binhost_conf_dir, dryrun,
target, slave_targets, version):
"""Constructor for prebuilt uploader object.
This object can upload host or prebuilt files to Google Storage.
upload_location: The upload location.
acl: The canned acl used for uploading to Google Storage. acl can be one
of: "public-read", "public-read-write", "authenticated-read",
"bucket-owner-read", "bucket-owner-full-control", "project-private",
or "private" (see "gsutil help acls"). If we are not uploading to
Google Storage, this parameter is unused.
binhost_base_url: The URL used for downloading the prebuilts.
pkg_indexes: Old uploaded prebuilts to compare against. Instead of
uploading duplicate files, we just link to the old files.
build_path: The path to the directory containing the chroot.
packages: Packages to upload.
skip_upload: Don't actually upload the tarballs.
binhost_conf_dir: Directory where to store binhost.conf files.
dryrun: Don't push or upload prebuilts.
target: BuildTarget managed by this builder.
slave_targets: List of BuildTargets managed by slave builders.
version: A unique string, intended to be included in the upload path,
which identifies the version number of the uploaded prebuilts.
self._upload_location = upload_location
self._acl = acl
self._binhost_base_url = binhost_base_url
self._pkg_indexes = pkg_indexes
self._build_path = build_path
self._packages = set(packages)
self._found_packages = set()
self._skip_upload = skip_upload
self._binhost_conf_dir = binhost_conf_dir
self._dryrun = dryrun
self._target = target
self._slave_targets = slave_targets
self._version = version
self._gs_context = gs.GSContext(retries=_RETRIES, sleep=_SLEEP_TIME,
def _Upload(self, local_file, remote_file):
"""Wrapper around _GsUpload"""
_GsUpload(self._gs_context, self._acl, local_file, remote_file)
def _ShouldFilterPackage(self, pkg):
if not self._packages:
return False
pym_path = os.path.abspath(os.path.join(self._build_path, _PYM_PATH))
sys.path.insert(0, pym_path)
# pylint: disable=F0401
import portage.versions
cat, pkgname = portage.versions.catpkgsplit(pkg['CPV'])[0:2]
cp = '%s/%s' % (cat, pkgname)
return pkgname not in self._packages and cp not in self._packages
def _UploadPrebuilt(self, package_path, url_suffix):
"""Upload host or board prebuilt files to Google Storage space.
package_path: The path to the packages dir.
url_suffix: The remote subdirectory where we should upload the packages.
# Process Packages file, removing duplicates and filtered packages.
pkg_index = binpkg.GrabLocalPackageIndex(package_path)
pkg_index.SetUploadLocation(self._binhost_base_url, url_suffix)
uploads = pkg_index.ResolveDuplicateUploads(self._pkg_indexes)
unmatched_pkgs = self._packages - self._found_packages
if unmatched_pkgs:
logging.warning('unable to match packages: %r' % unmatched_pkgs)
# Write Packages file.
pkg_index.header['TTL'] = _BINPKG_TTL
tmp_packages_file = pkg_index.WriteToNamedTemporaryFile()
remote_location = '%s/%s' % (self._upload_location.rstrip('/'), url_suffix)
assert remote_location.startswith('gs://')
# Build list of files to upload. Manually include the dev-only files but
# skip them if not present.
# TODO(deymo): Upload dev-only-extras.tbz2 as dev-only-extras.tar.bz2
# outside packages/ directory. See for details.
if os.path.exists(os.path.join(package_path, 'dev-only-extras.tbz2')):
uploads.append({'CPV': 'dev-only-extras'})
upload_files = GenerateUploadDict(package_path, remote_location, uploads)
remote_file = '%s/Packages' % remote_location.rstrip('/')
upload_files[] = remote_file
RemoteUpload(self._gs_context, self._acl, upload_files)
with tempfile.NamedTemporaryFile(
prefix='chromite.upload_prebuilts.index.') as index:
[x[len(remote_location) + 1:] for x in upload_files.values()],, self._target, self._version)
self._Upload(, '%s/index.html' % remote_location.rstrip('/'))
link_name = 'Prebuilts[%s]: %s' % (self._target, self._version)
url = '%s%s/index.html' % (gs.PUBLIC_BASE_HTTPS_URL,
logging.PrintBuildbotLink(link_name, url)
def _UploadSdkTarball(self, board_path, url_suffix, prepackaged,
toolchain_tarballs, toolchain_upload_path):
"""Upload a tarball of the sdk at the specified path to Google Storage.
board_path: The path to the board dir.
url_suffix: The remote subdirectory where we should upload the packages.
prepackaged: If given, a tarball that has been packaged outside of this
script and should be used.
toolchains_overlay_tarballs: List of toolchains overlay tarball
specifications to upload. Items take the form
toolchains_overlay_upload_path: Path template under the bucket to place
toolchains overlay tarballs.
toolchain_tarballs: List of toolchain tarballs to upload.
toolchain_upload_path: Path under the bucket to place toolchain tarballs.
remote_location = '%s/%s' % (self._upload_location.rstrip('/'), url_suffix)
assert remote_location.startswith('gs://')
boardname = os.path.basename(board_path.rstrip('/'))
# We do not upload non SDK board tarballs,
assert boardname == constants.CHROOT_BUILDER_BOARD
assert prepackaged is not None
version_str = self._version[len('chroot-'):]
remote_tarfile = toolchain.GetSdkURL(
for_gsutil=True, suburl='cros-sdk-%s.tar.xz' % (version_str,))
# For SDK, also upload the manifest which is guaranteed to exist
# by the builderstage.
self._Upload(prepackaged + '.Manifest', remote_tarfile + '.Manifest')
self._Upload(prepackaged, remote_tarfile)
# Upload SDK toolchains overlays and toolchain tarballs, if given.
for tarball_list, upload_path, qualifier_name in (
(toolchains_overlay_tarballs, toolchains_overlay_upload_path,
(toolchain_tarballs, toolchain_upload_path, 'target')):
for tarball_spec in tarball_list:
qualifier_val, local_path = tarball_spec.split(':')
suburl = upload_path % {qualifier_name: qualifier_val}
remote_path = toolchain.GetSdkURL(for_gsutil=True, suburl=suburl)
self._Upload(local_path, remote_path)
# Finally, also update the pointer to the latest SDK on which polling
# scripts rely.
with osutils.TempDir() as tmpdir:
pointerfile = os.path.join(tmpdir, 'cros-sdk-latest.conf')
remote_pointerfile = toolchain.GetSdkURL(for_gsutil=True,
osutils.WriteFile(pointerfile, 'LATEST_SDK="%s"' % version_str)
self._Upload(pointerfile, remote_pointerfile)
def _GetTargets(self):
"""Retuns the list of targets to use."""
targets = self._slave_targets[:]
if self._target:
return targets
def SyncHostPrebuilts(self, key, git_sync, sync_binhost_conf):
"""Synchronize host prebuilt files.
This function will sync both the standard host packages, plus the host
packages associated with all targets that have been "setup" with the
current host's chroot. For instance, if this host has been used to build
x86-generic, it will sync the host packages associated with
'i686-pc-linux-gnu'. If this host has also been used to build arm-generic,
it will also sync the host packages associated with
key: The variable key to update in the git file.
git_sync: If set, update make.conf of target to reference the latest
prebuilt packages generated here.
sync_binhost_conf: If set, update binhost config file in
chromiumos-overlay for the host.
# Slave boards are listed before the master board so that the master board
# takes priority (i.e. x86-generic preflight host prebuilts takes priority
# over preflight host prebuilts from other builders.)
binhost_urls = []
for target in self._GetTargets():
url_suffix = _REL_HOST_PATH % {'version': self._version,
'host_arch': _HOST_ARCH,
'target': target}
packages_url_suffix = '%s/packages' % url_suffix.rstrip('/')
if self._target == target and not self._skip_upload:
# Upload prebuilts.
package_path = os.path.join(self._build_path, _HOST_PACKAGES_PATH)
self._UploadPrebuilt(package_path, packages_url_suffix)
# Record URL where prebuilts were uploaded.
binhost_urls.append('%s/%s/' % (self._binhost_base_url.rstrip('/'),
binhost = ' '.join(binhost_urls)
if git_sync:
git_file = os.path.join(self._build_path, _PREBUILT_MAKE_CONF[_HOST_ARCH])
RevGitFile(git_file, {key: binhost}, dryrun=self._dryrun)
if sync_binhost_conf:
binhost_conf = os.path.join(
self._binhost_conf_dir, 'host', '%s-%s.conf' % (_HOST_ARCH, key))
UpdateBinhostConfFile(binhost_conf, key, binhost)
def SyncBoardPrebuilts(self, key, git_sync, sync_binhost_conf,
upload_board_tarball, prepackaged_board,
toolchain_tarballs, toolchain_upload_path):
"""Synchronize board prebuilt files.
key: The variable key to update in the git file.
git_sync: If set, update make.conf of target to reference the latest
prebuilt packages generated here.
sync_binhost_conf: If set, update binhost config file in
chromiumos-overlay for the current board.
upload_board_tarball: Include a tarball of the board in our upload.
prepackaged_board: A tarball of the board built outside of this script.
toolchains_overlay_tarballs: List of toolchains overlay tarball
specifications to upload. Items take the form
toolchains_overlay_upload_path: Path template under the bucket to place
toolchains overlay tarballs.
toolchain_tarballs: A list of toolchain tarballs to upload.
toolchain_upload_path: Path under the bucket to place toolchain tarballs.
updated_binhosts = set()
for target in self._GetTargets():
board_path = os.path.join(self._build_path,
_BOARD_PATH % {'board': target.board_variant})
package_path = os.path.join(board_path, 'packages')
url_suffix = _REL_BOARD_PATH % {'target': target,
'version': self._version}
packages_url_suffix = '%s/packages' % url_suffix.rstrip('/')
# Process the target board differently if it is the main --board.
if self._target == target and not self._skip_upload:
# This strips "chroot" prefix because that is sometimes added as the
# --prepend-version argument (e.g. by chromiumos-sdk bot).
# TODO(build): Clean it up to be less hard-coded.
version_str = self._version[len('chroot-'):]
# Upload board tarballs in the background.
if upload_board_tarball:
if toolchain_upload_path:
toolchain_upload_path %= {'version': version_str}
if toolchains_overlay_upload_path:
toolchains_overlay_upload_path %= {'version': version_str}
tar_process = multiprocessing.Process(
args=(board_path, url_suffix, prepackaged_board,
toolchains_overlay_upload_path, toolchain_tarballs,
# Upload prebuilts.
self._UploadPrebuilt(package_path, packages_url_suffix)
# Make sure we finished uploading the board tarballs.
if upload_board_tarball:
assert tar_process.exitcode == 0
# TODO(zbehan): This should be done cleaner.
if target.board == constants.CHROOT_BUILDER_BOARD:
sdk_conf = os.path.join(self._binhost_conf_dir,
sdk_settings = {
'SDK_LATEST_VERSION': version_str,
'TC_PATH': toolchain_upload_path,
RevGitFile(sdk_conf, sdk_settings, dryrun=self._dryrun)
# Record URL where prebuilts were uploaded.
url_value = '%s/%s/' % (self._binhost_base_url.rstrip('/'),
if git_sync:
git_file = DeterminePrebuiltConfFile(self._build_path, target)
RevGitFile(git_file, {key: url_value}, dryrun=self._dryrun)
if sync_binhost_conf:
# Update the binhost configuration file in git.
binhost_conf = os.path.join(
self._binhost_conf_dir, 'target', '%s-%s.conf' % (target, key))
UpdateBinhostConfFile(binhost_conf, key, url_value)
if sync_binhost_conf:
# Clear all old binhosts. The files must be left empty in case anybody
# is referring to them.
all_binhosts = set(glob.glob(os.path.join(
self._binhost_conf_dir, 'target', '*-%s.conf' % key)))
for binhost_conf in all_binhosts - updated_binhosts:
UpdateBinhostConfFile(binhost_conf, key, '')
class _AddSlaveBoardAction(argparse.Action):
"""Callback that adds a slave board to the list of slave targets."""
def __call__(self, parser, namespace, values, option_string=None):
getattr(namespace, self.dest).append(BuildTarget(values))
class _AddSlaveProfileAction(argparse.Action):
"""Callback that adds a slave profile to the list of slave targets."""
def __call__(self, parser, namespace, values, option_string=None):
if not namespace.slave_targets:
parser.error('Must specify --slave-board before --slave-profile')
if namespace.slave_targets[-1].profile is not None:
parser.error('Cannot specify --slave-profile twice for same board')
namespace.slave_targets[-1].profile = values
def ParseOptions(argv):
"""Returns options given by the user and the target specified.
argv: The args to parse.
A tuple containing a parsed options object and BuildTarget.
The target instance is None if no board is specified.
parser = commandline.ArgumentParser()
parser.add_argument('-H', '--binhost-base-url', default=_BINHOST_BASE_URL,
help='Base URL to use for binhost in make.conf updates')
parser.add_argument('--previous-binhost-url', action='append', default=[],
help='Previous binhost URL')
parser.add_argument('-b', '--board',
help='Board type that was built on this machine')
parser.add_argument('-B', '--prepackaged-tarball', type='path',
help='Board tarball prebuilt outside of this script.')
action='append', default=[],
help='Toolchains overlay tarball specification to '
'upload. Takes the form '
parser.add_argument('--toolchains-overlay-upload-path', default='',
help='Path template for uploading toolchains overlays.')
parser.add_argument('--toolchain-tarball', dest='toolchain_tarballs',
action='append', default=[],
help='Redistributable toolchain tarball.')
parser.add_argument('--toolchain-upload-path', default='',
help='Path to place toolchain tarballs in the sdk tree.')
help='Profile that was built on this machine')
parser.add_argument('--slave-board', default=[], action=_AddSlaveBoardAction,
help='Board type that was built on a slave machine. To '
'add a profile to this board, use --slave-profile.')
parser.add_argument('--slave-profile', action=_AddSlaveProfileAction,
help='Board profile that was built on a slave machine. '
'Applies to previous slave board.')
parser.add_argument('-p', '--build-path', required=True,
help='Path to the directory containing the chroot')
parser.add_argument('--packages', action='append', default=[],
help='Only include the specified packages. '
'(Default is to include all packages.)')
parser.add_argument('-s', '--sync-host', default=False, action='store_true',
help='Sync host prebuilts')
parser.add_argument('-g', '--git-sync', default=False, action='store_true',
help='Enable git version sync (This commits to a repo.) '
'This is used by full builders to commit directly '
'to board overlays.')
parser.add_argument('-u', '--upload',
help='Upload location')
parser.add_argument('-V', '--prepend-version',
help='Add an identifier to the front of the version')
parser.add_argument('-f', '--filters', action='store_true', default=False,
help='Turn on filtering of private ebuild packages')
parser.add_argument('-k', '--key', default='PORTAGE_BINHOST',
help='Key to update in make.conf / binhost.conf')
help='Specify the version string')
parser.add_argument('--sync-binhost-conf', default=False, action='store_true',
help='Update binhost.conf in chromiumos-overlay or '
'chromeos-overlay. Commit the changes, but don\'t '
'push them. This is used for preflight binhosts.')
help='Directory to commit binhost config with '
parser.add_argument('-P', '--private', action='store_true', default=False,
help='Mark gs:// uploads as private.')
parser.add_argument('--skip-upload', action='store_true', default=False,
help='Skip upload step.')
parser.add_argument('--upload-board-tarball', action='store_true',
help='Upload board tarball to Google Storage.')
parser.add_argument('-n', '--dry-run', dest='dryrun',
action='store_true', default=False,
help='Don\'t push or upload prebuilts.')
options = parser.parse_args(argv)
if not options.upload and not options.skip_upload:
parser.error('you need to provide an upload location using -u')
if not options.set_version and options.skip_upload:
parser.error('If you are using --skip-upload, you must specify a '
'version number using --set-version.')
target = None
if options.board:
target = BuildTarget(options.board, options.profile)
if target in options.slave_targets:
parser.error('--board/--profile must not also be a slave target.')
if len(set(options.slave_targets)) != len(options.slave_targets):
parser.error('--slave-boards must not have duplicates.')
if options.slave_targets and options.git_sync:
parser.error('--slave-boards is not compatible with --git-sync')
if (options.upload_board_tarball and options.skip_upload and
options.board == 'amd64-host'):
parser.error('--skip-upload is not compatible with '
'--upload-board-tarball and --board=amd64-host')
if (options.upload_board_tarball and not options.skip_upload and
not options.upload.startswith('gs://')):
parser.error('--upload-board-tarball only works with gs:// URLs.\n'
'--upload must be a gs:// URL.')
if options.upload_board_tarball and options.prepackaged_tarball is None:
parser.error('--upload-board-tarball requires --prepackaged-tarball')
if options.private:
if options.sync_host:
parser.error('--private and --sync-host/-s cannot be specified '
'together; we do not support private host prebuilts')
if not options.upload or not options.upload.startswith('gs://'):
parser.error('--private is only valid for gs:// URLs; '
'--upload must be a gs:// URL.')
if options.binhost_base_url != _BINHOST_BASE_URL:
parser.error('when using --private the --binhost-base-url '
'is automatically derived.')
if options.sync_binhost_conf and not options.binhost_conf_dir:
parser.error('--sync-binhost-conf requires --binhost-conf-dir')
if (options.toolchains_overlay_tarballs and
not options.toolchains_overlay_upload_path):
parser.error('--toolchains-overlay-tarball requires '
return options, target
def main(argv):
# Set umask to a sane value so that files created as root are readable.
options, target = ParseOptions(argv)
# Calculate a list of Packages index files to compare against. Whenever we
# upload a package, we check to make sure it's not already stored in one of
# the packages files we uploaded. This list of packages files might contain
# both board and host packages.
pkg_indexes = _GrabAllRemotePackageIndexes(options.previous_binhost_url)
if options.set_version:
version = options.set_version
version = GetVersion()
if options.prepend_version:
version = '%s-%s' % (options.prepend_version, version)
acl = 'public-read'
binhost_base_url = options.binhost_base_url
if options.private:
binhost_base_url = options.upload
if target:
acl = portage_util.FindOverlayFile(_GOOGLESTORAGE_GSUTIL_FILE,
if acl is None:
cros_build_lib.Die('No Google Storage ACL file %s found in %s overlay.',
_GOOGLESTORAGE_GSUTIL_FILE, target.board_variant)
binhost_conf_dir = None
if options.binhost_conf_dir:
binhost_conf_dir = os.path.join(options.build_path,
uploader = PrebuiltUploader(options.upload, acl, binhost_base_url,
pkg_indexes, options.build_path,
options.packages, options.skip_upload,
binhost_conf_dir, options.dryrun,
target, options.slave_targets, version)
if options.sync_host:
uploader.SyncHostPrebuilts(options.key, options.git_sync,
if options.board or options.slave_targets:
uploader.SyncBoardPrebuilts(options.key, options.git_sync,