blob: 7b9dfd1396f0f2cc432bf66c1262b36edb0feb79 [file] [log] [blame]
# Copyright (c) 2012 The Chromium OS Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
"""Calculate what workon packages have changed since the last build.
A workon package is treated as changed if any of the below are true:
1) The package is not installed.
2) A file exists in the associated repository which has a newer modification
time than the installed package.
3) The source ebuild has a newer modification time than the installed package.
Some caveats:
- We do not look at eclasses. This replicates the existing behavior of the
commit queue, which also does not look at eclass changes.
- We do not try to fallback to the non-workon package if the local tree is
unmodified. This is probably a good thing, since developers who are
"working on" a package want to compile it locally.
- Portage only stores the time that a package finished building, so we
aren't able to detect when users modify source code during builds.
from __future__ import print_function
import errno
import multiprocessing
import os
import Queue
except ImportError:
# Python-3 renamed to "queue". We still use Queue to avoid collisions
# with naming variables as "queue". Maybe we'll transition at some point.
# pylint: disable=F0401
import queue as Queue
from chromite.cbuildbot import constants
from chromite.lib import commandline
from chromite.lib import cros_build_lib
from chromite.lib import cros_logging as logging
from chromite.lib import git
from chromite.lib import osutils
from chromite.lib import parallel
from chromite.lib import portage_util
from chromite.lib import sysroot_lib
from chromite.lib import workon_helper
class ModificationTimeMonitor(object):
"""Base class for monitoring last modification time of paths.
This takes a list of (keys, path) pairs and finds the latest mtime of an
object within each of the path's subtrees, populating a map from keys to
mtimes. Note that a key may be associated with multiple paths, in which case
the latest mtime among them will be returned.
_tasks: A list of (key, path) pairs to check.
_result_queue: A queue populated with corresponding (key, mtime) pairs.
def __init__(self, key_path_pairs):
self._tasks = list(key_path_pairs)
self._result_queue = multiprocessing.Queue(len(self._tasks))
def _EnqueueModificationTime(self, key, path):
"""Calculate the last modification time of |path| and enqueue it."""
if os.path.isdir(path):
self._result_queue.put((key, self._LastModificationTime(path)))
def _LastModificationTime(self, path):
"""Returns the latest modification time for anything under |path|."""
cmd = 'find . -name .git -prune -o -printf "%T@\n" | sort -nr | head -n1'
ret = cros_build_lib.RunCommand(cmd, cwd=path, shell=True, print_cmd=False,
return float(ret.output) if ret.output else 0
def GetModificationTimes(self):
"""Get the latest modification time for each of the queued keys."""
parallel.RunTasksInProcessPool(self._EnqueueModificationTime, self._tasks)
mtimes = {}
while True:
key, mtime = self._result_queue.get_nowait()
mtimes[key] = max((mtimes.get(key, 0), mtime))
except Queue.Empty:
return mtimes
class WorkonPackageInfo(object):
"""Class for getting information about workon packages.
cp: The package name (e.g. chromeos-base/power_manager).
mtime: The modification time of the installed package.
projects: The project(s) associated with the package.
full_srcpaths: The brick source path(s) associated with the package.
src_ebuild_mtime: The modification time of the source ebuild.
def __init__(self, cp, mtime, projects, full_srcpaths, src_ebuild_mtime):
self.cp = cp
self.pkg_mtime = int(mtime)
self.projects = projects
self.full_srcpaths = full_srcpaths
self.src_ebuild_mtime = src_ebuild_mtime
def ListWorkonPackages(sysroot, all_opt=False):
"""List the packages that are currently being worked on.
sysroot: sysroot_lib.Sysroot object.
all_opt: Pass --all to cros_workon. For testing purposes.
helper = workon_helper.WorkonHelper(sysroot.path)
return helper.ListAtoms(use_all=all_opt)
def ListWorkonPackagesInfo(sysroot):
"""Find the specified workon packages for the specified board.
sysroot: sysroot_lib.Sysroot object.
A list of WorkonPackageInfo objects for unique packages being worked on.
# Import portage late so that this script can be imported outside the chroot.
# pylint: disable=F0401
import portage.const
packages = ListWorkonPackages(sysroot)
if not packages:
return []
results = {}
if sysroot.path == '/':
overlays = portage_util.FindOverlays(constants.BOTH_OVERLAYS, None)
overlays = sysroot.GetStandardField('PORTDIR_OVERLAY').splitlines()
vdb_path = os.path.join(sysroot.path, portage.const.VDB_PATH)
for overlay in overlays:
for filename, projects, srcpaths in portage_util.GetWorkonProjectMap(
overlay, packages):
# chromeos-base/power_manager/power_manager-9999
# cp = chromeos-base/power_manager
# cpv = chromeos-base/power_manager-9999
category, pn, p = portage_util.SplitEbuildPath(filename)
cp = '%s/%s' % (category, pn)
cpv = '%s/%s' % (category, p)
# Get the time the package finished building. TODO(build): Teach Portage
# to store the time the package started building and use that here.
pkg_mtime_file = os.path.join(vdb_path, cpv, 'BUILD_TIME')
pkg_mtime = int(osutils.ReadFile(pkg_mtime_file))
except EnvironmentError as ex:
if ex.errno != errno.ENOENT:
pkg_mtime = 0
# Get the modificaton time of the ebuild in the overlay.
src_ebuild_mtime = os.lstat(os.path.join(overlay, filename)).st_mtime
# Write info into the results dictionary, overwriting any previous
# values. This ensures that overlays override appropriately.
results[cp] = WorkonPackageInfo(cp, pkg_mtime, projects, srcpaths,
return results.values()
def WorkonProjectsMonitor(projects):
"""Returns a monitor for project modification times."""
# TODO(garnold) In order for the mtime monitor to be as accurate as
# possible, this only needs to enqueue the checkout(s) relevant for the
# task at hand, e.g. the specific ebuild we want to emerge. In general, the
# CROS_WORKON_LOCALNAME variable in workon ebuilds defines the source path
# uniquely and can be used for this purpose.
project_path_pairs = []
manifest = git.ManifestCheckout.Cached(constants.SOURCE_ROOT)
for project in set(projects).intersection(manifest.checkouts_by_name):
for checkout in manifest.FindCheckouts(project):
project_path_pairs.append((project, checkout.GetPath(absolute=True)))
return ModificationTimeMonitor(project_path_pairs)
def WorkonSrcpathsMonitor(srcpaths):
"""Returns a monitor for srcpath modification times."""
return ModificationTimeMonitor(zip(srcpaths, srcpaths))
def ListModifiedWorkonPackages(sysroot):
"""List the workon packages that need to be rebuilt.
sysroot: sysroot_lib.Sysroot object.
packages = ListWorkonPackagesInfo(sysroot)
if not packages:
# Get mtimes for all projects and source paths associated with our packages.
all_projects = [p for info in packages for p in info.projects]
project_mtimes = WorkonProjectsMonitor(all_projects).GetModificationTimes()
all_srcpaths = [s for info in packages for s in info.full_srcpaths]
srcpath_mtimes = WorkonSrcpathsMonitor(all_srcpaths).GetModificationTimes()
for info in packages:
mtime = int(max([project_mtimes.get(p, 0) for p in info.projects] +
[srcpath_mtimes.get(s, 0) for s in info.full_srcpaths] +
if mtime >= info.pkg_mtime:
yield info.cp
def _ParseArguments(argv):
parser = commandline.ArgumentParser(description=__doc__)
target = parser.add_mutually_exclusive_group(required=True)
target.add_argument('--board', help='Board name')
target.add_argument('--host', default=False, action='store_true',
help='Look at host packages instead of board packages')
target.add_argument('--sysroot', help='Sysroot path.')
flags = parser.parse_args(argv)
return flags
def main(argv):
flags = _ParseArguments(argv)
sysroot = None
if flags.board:
sysroot = cros_build_lib.GetSysroot(flags.board)
sysroot = '/'
sysroot = flags.sysroot
modified = ListModifiedWorkonPackages(sysroot_lib.Sysroot(sysroot))
print(' '.join(sorted(modified)))