blob: 5f2d50b306e5d63e00f07ccdf94a4210f22f8014 [file] [log] [blame]
# Copyright (c) 2012 The Chromium OS Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
"""A library to generate and store the manifests for cros builders to use."""
from __future__ import print_function
import codecs
import os
import re
from xml.dom import minidom
from chromite.cbuildbot import config_lib
from chromite.cbuildbot import constants
from chromite.cbuildbot import manifest_version
from chromite.lib import cros_build_lib
from chromite.lib import cros_logging as logging
from chromite.lib import git
site_config = config_lib.GetConfig()
# Paladin constants for manifest names.
PALADIN_COMMIT_ELEMENT = 'pending_commit'
CHROME_ELEMENT = 'chrome'
CHROME_VERSION_ATTR = 'version'
LKGM_ELEMENT = 'lkgm'
LKGM_VERSION_ATTR = 'version'
class PromoteCandidateException(Exception):
"""Exception thrown for failure to promote manifest candidate."""
class _LKGMCandidateInfo(manifest_version.VersionInfo):
"""Class to encapsualte the chrome os lkgm candidate info
You can instantiate this class in two ways.
1)using a version file, specifically chromeos_version.sh,
which contains the version information.
2) just passing in the 4 version components (major, minor, sp, patch and
revision number),
Args:
You can instantiate this class in two ways.
1)using a version file, specifically chromeos_version.sh,
which contains the version information.
2) passing in a string with the 3 version components + revision e.g. 41.0.0-r1
Args:
version_string: Optional 3 component version string to parse. Contains:
build_number: release build number.
branch_build_number: current build number on a branch.
patch_number: patch number.
revision_number: version revision
chrome_branch: If version_string specified, specify chrome_branch i.e. 13.
version_file: version file location.
"""
LKGM_RE = r'(\d+\.\d+\.\d+)(?:-rc(\d+))?'
def __init__(self, version_string=None, chrome_branch=None, incr_type=None,
version_file=None):
self.revision_number = 1
if version_string:
match = re.search(self.LKGM_RE, version_string)
assert match, 'LKGM did not re %s' % self.LKGM_RE
super(_LKGMCandidateInfo, self).__init__(match.group(1), chrome_branch,
incr_type=incr_type)
if match.group(2):
self.revision_number = int(match.group(2))
else:
super(_LKGMCandidateInfo, self).__init__(version_file=version_file,
incr_type=incr_type)
def VersionString(self):
"""returns the full version string of the lkgm candidate"""
return '%s.%s.%s-rc%s' % (self.build_number, self.branch_build_number,
self.patch_number, self.revision_number)
def VersionComponents(self):
"""Return an array of ints of the version fields for comparing."""
return map(int, [self.build_number, self.branch_build_number,
self.patch_number, self.revision_number])
def IncrementVersion(self):
"""Increments the version by incrementing the revision #."""
self.revision_number += 1
return self.VersionString()
def UpdateVersionFile(self, *args, **kwargs):
"""Update the version file on disk.
For LKGMCandidateInfo there is no version file so this function is a no-op.
"""
class LKGMManager(manifest_version.BuildSpecsManager):
"""A Class to manage lkgm candidates and their states.
Vars:
lkgm_subdir: Subdirectory within manifest repo to store candidates.
"""
# Sub-directories for LKGM and Chrome LKGM's.
LKGM_SUBDIR = 'LKGM-candidates'
CHROME_PFQ_SUBDIR = 'chrome-LKGM-candidates'
COMMIT_QUEUE_SUBDIR = 'paladin'
def __init__(self, source_repo, manifest_repo, build_names, build_type,
incr_type, force, branch, manifest=constants.DEFAULT_MANIFEST,
dry_run=True, master=False,
lkgm_path_rel=constants.LKGM_MANIFEST):
"""Initialize an LKGM Manager.
Args:
source_repo: Repository object for the source code.
manifest_repo: Manifest repository for manifest versions/buildspecs.
build_names: Identifiers for the build. Must match config_lib
entries. If multiple identifiers are provided, the first item in the
list must be an identifier for the group.
build_type: Type of build. Must be a pfq type.
incr_type: How we should increment this version - build|branch|patch
force: Create a new manifest even if there are no changes.
branch: Branch this builder is running on.
manifest: Manifest to use for checkout. E.g. 'full' or 'buildtools'.
dry_run: Whether we actually commit changes we make or not.
master: Whether we are the master builder.
lkgm_path_rel: Path to the LKGM symlink, relative to manifest dir.
"""
super(LKGMManager, self).__init__(
source_repo=source_repo, manifest_repo=manifest_repo,
manifest=manifest, build_names=build_names, incr_type=incr_type,
force=force, branch=branch, dry_run=dry_run, master=master)
self.lkgm_path = os.path.join(self.manifest_dir, lkgm_path_rel)
self.compare_versions_fn = _LKGMCandidateInfo.VersionCompare
self.build_type = build_type
# Chrome PFQ and PFQ's exist at the same time and version separately so they
# must have separate subdirs in the manifest-versions repository.
if self.build_type == constants.CHROME_PFQ_TYPE:
self.rel_working_dir = self.CHROME_PFQ_SUBDIR
elif config_lib.IsCQType(self.build_type):
self.rel_working_dir = self.COMMIT_QUEUE_SUBDIR
else:
assert config_lib.IsPFQType(self.build_type)
self.rel_working_dir = self.LKGM_SUBDIR
def GetCurrentVersionInfo(self):
"""Returns the lkgm version info from the version file."""
version_info = super(LKGMManager, self).GetCurrentVersionInfo()
return _LKGMCandidateInfo(version_info.VersionString(),
chrome_branch=version_info.chrome_branch,
incr_type=self.incr_type)
def _WriteXml(self, dom_instance, file_path):
"""Wrapper function to write xml encoded in a proper way.
Args:
dom_instance: A DOM document instance contains contents to be written.
file_path: Path to the file to write into.
"""
with codecs.open(file_path, 'w+', 'utf-8') as f:
dom_instance.writexml(f)
def _AddLKGMToManifest(self, manifest):
"""Write the last known good version string to the manifest.
Args:
manifest: Path to the manifest.
"""
# Get the last known good version string.
try:
lkgm_filename = os.path.basename(os.readlink(self.lkgm_path))
lkgm_version, _ = os.path.splitext(lkgm_filename)
except OSError:
return
# Write the last known good version string to the manifest.
manifest_dom = minidom.parse(manifest)
lkgm_element = manifest_dom.createElement(LKGM_ELEMENT)
lkgm_element.setAttribute(LKGM_VERSION_ATTR, lkgm_version)
manifest_dom.documentElement.appendChild(lkgm_element)
self._WriteXml(manifest_dom, manifest)
def _AddChromeVersionToManifest(self, manifest, chrome_version):
"""Adds the chrome element with version |chrome_version| to |manifest|.
The manifest file should contain the Chrome version to build for
PFQ slaves.
Args:
manifest: Path to the manifest
chrome_version: A string representing the version of Chrome
(e.g. 35.0.1863.0).
"""
manifest_dom = minidom.parse(manifest)
chrome = manifest_dom.createElement(CHROME_ELEMENT)
chrome.setAttribute(CHROME_VERSION_ATTR, chrome_version)
manifest_dom.documentElement.appendChild(chrome)
self._WriteXml(manifest_dom, manifest)
def _AddPatchesToManifest(self, manifest, patches):
"""Adds list of |patches| to given |manifest|.
The manifest should have sufficient information for the slave
builders to fetch the patches from Gerrit and to print the CL link
(see cros_patch.GerritFetchOnlyPatch).
Args:
manifest: Path to the manifest.
patches: A list of cros_patch.GerritPatch objects.
"""
manifest_dom = minidom.parse(manifest)
for patch in patches:
pending_commit = manifest_dom.createElement(PALADIN_COMMIT_ELEMENT)
attr_dict = patch.GetAttributeDict()
for k, v in attr_dict.iteritems():
pending_commit.setAttribute(k, v)
manifest_dom.documentElement.appendChild(pending_commit)
self._WriteXml(manifest_dom, manifest)
def CreateNewCandidate(self, validation_pool=None,
chrome_version=None,
retries=manifest_version.NUM_RETRIES,
build_id=None):
"""Creates, syncs to, and returns the next candidate manifest.
Args:
validation_pool: Validation pool to apply to the manifest before
publishing.
chrome_version: The Chrome version to write in the manifest. Defaults
to None, in which case no version is written.
retries: Number of retries for updating the status. Defaults to
manifest_version.NUM_RETRIES.
build_id: Optional integer cidb id of the build that is creating
this candidate.
Raises:
GenerateBuildSpecException in case of failure to generate a buildspec
"""
self.CheckoutSourceCode()
# Refresh manifest logic from manifest_versions repository to grab the
# LKGM to generate the blamelist.
version_info = self.GetCurrentVersionInfo()
self.RefreshManifestCheckout()
self.InitializeManifestVariables(version_info)
self.GenerateBlameListSinceLKGM()
new_manifest = self.CreateManifest()
# For Chrome PFQ, add the version of Chrome to use.
if chrome_version:
self._AddChromeVersionToManifest(new_manifest, chrome_version)
# For the Commit Queue, apply the validation pool as part of checkout.
if validation_pool:
# If we have nothing that could apply from the validation pool and
# we're not also a pfq type, we got nothing to do.
assert self.cros_source.directory == validation_pool.build_root
if (not validation_pool.ApplyPoolIntoRepo() and
not config_lib.IsPFQType(self.build_type)):
return None
self._AddPatchesToManifest(new_manifest, validation_pool.changes)
# Add info about the last known good version to the manifest. This will
# be used by slaves to calculate what artifacts from old builds are safe
# to use.
self._AddLKGMToManifest(new_manifest)
last_error = None
for attempt in range(0, retries + 1):
try:
# Refresh manifest logic from manifest_versions repository.
# Note we don't need to do this on our first attempt as we needed to
# have done it to get the LKGM.
if attempt != 0:
self.RefreshManifestCheckout()
self.InitializeManifestVariables(version_info)
# If we don't have any valid changes to test, make sure the checkout
# is at least different.
if ((not validation_pool or not validation_pool.changes) and
not self.force and self.HasCheckoutBeenBuilt()):
return None
# Check whether the latest spec available in manifest-versions is
# newer than our current version number. If so, use it as the base
# version number. Otherwise, we default to 'rc1'.
if self.latest:
latest = max(self.latest, version_info.VersionString(),
key=self.compare_versions_fn)
version_info = _LKGMCandidateInfo(
latest, chrome_branch=version_info.chrome_branch,
incr_type=self.incr_type)
git.CreatePushBranch(manifest_version.PUSH_BRANCH, self.manifest_dir,
sync=False)
version = self.GetNextVersion(version_info)
self.PublishManifest(new_manifest, version, build_id=build_id)
self.current_version = version
return self.GetLocalManifest(version)
except cros_build_lib.RunCommandError as e:
err_msg = 'Failed to generate LKGM Candidate. error: %s' % e
logging.error(err_msg)
last_error = err_msg
raise manifest_version.GenerateBuildSpecException(last_error)
def CreateFromManifest(self, manifest, retries=manifest_version.NUM_RETRIES,
build_id=None):
"""Sets up an lkgm_manager from the given manifest.
This method sets up an LKGM manager and publishes a new manifest to the
manifest versions repo based on the passed in manifest but filtering
internal repositories and changes out of it.
Args:
manifest: A manifest that possibly contains private changes/projects. It
is named with the given version we want to create a new manifest from
i.e R20-1920.0.1-rc7.xml where R20-1920.0.1-rc7 is the version.
retries: Number of retries for updating the status.
build_id: Optional integer cidb build id of the build publishing the
manifest.
Returns:
Path to the manifest version file to use.
Raises:
GenerateBuildSpecException in case of failure to check-in the new
manifest because of a git error or the manifest is already checked-in.
"""
last_error = None
new_manifest = manifest_version.FilterManifest(
manifest, whitelisted_remotes=site_config.params.EXTERNAL_REMOTES)
version_info = self.GetCurrentVersionInfo()
for _attempt in range(0, retries + 1):
try:
self.RefreshManifestCheckout()
self.InitializeManifestVariables(version_info)
git.CreatePushBranch(manifest_version.PUSH_BRANCH, self.manifest_dir,
sync=False)
version = os.path.splitext(os.path.basename(manifest))[0]
logging.info('Publishing filtered build spec')
self.PublishManifest(new_manifest, version, build_id=build_id)
self.current_version = version
return self.GetLocalManifest(version)
except cros_build_lib.RunCommandError as e:
err_msg = 'Failed to generate LKGM Candidate. error: %s' % e
logging.error(err_msg)
last_error = err_msg
raise manifest_version.GenerateBuildSpecException(last_error)
def PromoteCandidate(self, retries=manifest_version.NUM_RETRIES):
"""Promotes the current LKGM candidate to be a real versioned LKGM."""
assert self.current_version, 'No current manifest exists.'
last_error = None
path_to_candidate = self.GetLocalManifest(self.current_version)
assert os.path.exists(path_to_candidate), 'Candidate not found locally.'
# This may potentially fail for not being at TOT while pushing.
for attempt in range(0, retries + 1):
try:
if attempt > 0:
self.RefreshManifestCheckout()
git.CreatePushBranch(manifest_version.PUSH_BRANCH,
self.manifest_dir, sync=False)
manifest_version.CreateSymlink(path_to_candidate, self.lkgm_path)
git.RunGit(self.manifest_dir, ['add', self.lkgm_path])
self.PushSpecChanges(
'Automatic: %s promoting %s to LKGM' % (self.build_names[0],
self.current_version))
return
except cros_build_lib.RunCommandError as e:
last_error = 'Failed to promote manifest. error: %s' % e
logging.info(last_error)
logging.info('Retrying to promote manifest: Retry %d/%d', attempt + 1,
retries)
raise PromoteCandidateException(last_error)
def _ShouldGenerateBlameListSinceLKGM(self):
"""Returns True if we should generate the blamelist."""
# We want to generate the blamelist only for valid pfq types and if we are
# building on the master branch i.e. revving the build number.
return (self.incr_type == 'build' and
config_lib.IsPFQType(self.build_type) and
self.build_type != constants.CHROME_PFQ_TYPE)
def GenerateBlameListSinceLKGM(self):
"""Prints out links to all CL's that have been committed since LKGM.
Add buildbot trappings to print <a href='url'>text</a> in the waterfall for
each CL committed since we last had a passing build.
"""
if not self._ShouldGenerateBlameListSinceLKGM():
logging.info('Not generating blamelist for lkgm as it is not appropriate '
'for this build type.')
return
# Suppress re-printing changes we tried ourselves on paladin
# builders since they are redundant.
only_print_chumps = self.build_type == constants.PALADIN_TYPE
GenerateBlameList(self.cros_source, self.lkgm_path,
only_print_chumps=only_print_chumps)
def GetLatestPassingSpec(self):
"""Get the last spec file that passed in the current branch."""
raise NotImplementedError()
def GenerateBlameList(source_repo, lkgm_path, only_print_chumps=False):
"""Generate the blamelist since the specified manifest.
Args:
source_repo: Repository object for the source code.
lkgm_path: Path to LKGM manifest.
only_print_chumps: If True, only print changes that were chumped.
"""
handler = git.Manifest(lkgm_path)
reviewed_on_re = re.compile(r'\s*Reviewed-on:\s*(\S+)')
author_re = re.compile(r'\s*Author:.*<(\S+)@\S+>\s*')
committer_re = re.compile(r'\s*Commit:.*<(\S+)@\S+>\s*')
for rel_src_path, checkout in handler.checkouts_by_path.iteritems():
project = checkout['name']
# Additional case in case the repo has been removed from the manifest.
src_path = source_repo.GetRelativePath(rel_src_path)
if not os.path.exists(src_path):
logging.info('Detected repo removed from manifest %s' % project)
continue
revision = checkout['revision']
cmd = ['log', '--pretty=full', '%s..HEAD' % revision]
try:
result = git.RunGit(src_path, cmd)
except cros_build_lib.RunCommandError as ex:
# Git returns 128 when the revision does not exist.
if ex.result.returncode != 128:
raise
logging.warning('Detected branch removed from local checkout.')
logging.PrintBuildbotStepWarnings()
return
current_author = None
current_committer = None
for line in unicode(result.output, 'ascii', 'ignore').splitlines():
author_match = author_re.match(line)
if author_match:
current_author = author_match.group(1)
committer_match = committer_re.match(line)
if committer_match:
current_committer = committer_match.group(1)
review_match = reviewed_on_re.match(line)
if review_match:
review = review_match.group(1)
_, _, change_number = review.rpartition('/')
if not current_author:
logging.notice('Failed to locate author before the line of review: '
'%s. Author name is set to <Unknown>', line)
current_author = '<Unknown>'
items = [
os.path.basename(project),
current_author,
change_number,
]
# TODO(phobbs) verify the domain of the email address as well.
if current_committer not in ('chrome-bot', 'chrome-internal-fetch',
'chromeos-commit-bot', '3su6n15k.default'):
items.insert(0, 'CHUMP')
elif only_print_chumps:
continue
logging.PrintBuildbotLink(' | '.join(items), review)