blob: d5f10be70a508762b19770c8026b6024c1eeb4c3 [file] [log] [blame]
# Copyright (c) 2011 The Chromium OS Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
"""Synchronize issues in Package Status spreadsheet with Issue Tracker."""
from __future__ import print_function
import optparse
import os
import sys
from chromite.lib import cros_build_lib
from chromite.lib import gdata_lib
from chromite.lib import operation
from chromite.lib import upgrade_table as utable
from chromite.scripts import upload_package_status as ups
PKGS_WS_NAME = 'Packages'
CR_ORG = ''
CHROMIUMOS_SITE = 'http://www.%s/chromium-os' % CR_ORG
PKG_UPGRADE_PAGE = '%s/gentoo-package-upgrade-process' % CHROMIUMOS_SITE
COL_PACKAGE = gdata_lib.PrepColNameForSS(utable.UpgradeTable.COL_PACKAGE)
COL_TEAM = gdata_lib.PrepColNameForSS('Team/Lead')
COL_OWNER = gdata_lib.PrepColNameForSS('Owner')
COL_TRACKER = gdata_lib.PrepColNameForSS('Tracker')
ARCHES = ('amd64', 'arm', 'x86')
DISABLING_TRACKER_VALUES = set(['n/a', 'disable', 'disabled'])
oper = operation.Operation('sync_package_status')
def _GetPkgSpreadsheetURL(ss_key):
return '%s/%s/spreadsheet/ccc?key=%s' % (DOCS_SITE, CR_ORG, ss_key)
class SyncError(RuntimeError):
"""Extend RuntimeError for use in this module."""
class PackageBlacklisted(RuntimeError):
"""Raised when package has disabled automatic tracker syncing."""
class Syncer(object):
"""Class to manage synchronizing between spreadsheet and Tracker."""
# Map spreadsheet team names to Tracker team labels.
VALID_TEAMS = {'build': 'Build',
'kernel': 'Cr-OS-Kernel',
'security': 'Security',
'system': 'Cr-OS-Systems',
'ui': 'Cr-UI'}
__slots__ = (
'default_owner', # Default owner to use when creating issues
'owners', # Set of owners to select (None means no filter)
'pretend', # If True, make no real changes
'scomm', # SpreadsheetComm
'tcomm', # TrackerComm
'teams', # Set of teams to select (None means no filter)
'tracker_col_ix', # Index of Tracker column in spreadsheet
'verbose', # Verbose boolean
def __init__(self, tcomm, scomm, pretend=False, verbose=False):
self.tcomm = tcomm
self.scomm = scomm
self.tracker_col_ix = None
self.teams = None
self.owners = None
self.default_owner = None
self.pretend = pretend
self.verbose = verbose
def _ReduceTeamName(self, team):
"""Translate |team| from spreadsheet/commandline name to short name.
For example: build/bdavirro --> build, build --> build
if team:
return team.lower().split('/')[0]
return None
def SetTeamFilter(self, teamarg):
"""Set team filter using colon-separated team names in |teamarg|
Resulting filter in self.teams is set of "reduced" team names.
'build:system:ui' --> set(['build', 'system', 'ui'])
'Build:system:UI' --> set(['build', 'system', 'ui'])
If an invalid team name is given oper.Die is called with explanation.
if teamarg:
teamlist = []
for team in teamarg.split(':'):
t = self._ReduceTeamName(team)
if t in self.VALID_TEAMS:
oper.Die('Invalid team name "%s". Choose from: %s' %
(team, ','.join(self.VALID_TEAMS.keys())))
self.teams = set(teamlist)
self.teams = None
def _ReduceOwnerName(self, owner):
"""Translate |owner| from spreadsheet/commandline name to short name.
For example: -> joe, joe --> joe
if owner:
return owner.lower().split('@')[0]
return None
def SetOwnerFilter(self, ownerarg):
"""Set owner filter using colon-separated owner names in |ownerarg|."""
if ownerarg:
self.owners = set([self._ReduceOwnerName(o) for o in ownerarg.split(':')])
self.owners = None
def SetDefaultOwner(self, default_owner):
"""Use |default_owner| as isue owner when none set in spreadsheet."""
if default_owner and default_owner == 'me':
self.default_owner = os.environ['USER']
self.default_owner = default_owner
def _RowPassesFilters(self, row):
"""Return true if |row| passes any team/owner filters."""
if self.teams:
team = self._ReduceTeamName(row[COL_TEAM])
if team not in self.teams:
return False
if self.owners:
owner = self._ReduceOwnerName(row[COL_OWNER])
if owner not in self.owners:
return False
return True
def Sync(self):
"""Do synchronization between Spreadsheet and Tracker."""
self.tracker_col_ix = self.scomm.GetColumnIndex(COL_TRACKER)
if None is self.tracker_col_ix:
raise SyncError('Unable to find "Tracker" column in spreadsheet')
errors = []
# Go over each row in Spreadsheet. Row index starts at 2
# because spreadsheet rows start at 1 and we don't want the header row.
rows = self.scomm.GetRows()
for rowIx, row in enumerate(rows, start=self.scomm.ROW_NUMBER_OFFSET):
if not self._RowPassesFilters(row):
oper.Info('\nSkipping row %d, pkg: %r (team=%s, owner=%s) ...' %
(rowIx, row[COL_PACKAGE], row[COL_TEAM], row[COL_OWNER]))
oper.Info('\nProcessing row %d, pkg: %r (team=%s, owner=%s) ...' %
(rowIx, row[COL_PACKAGE], row[COL_TEAM], row[COL_OWNER]))
new_issue = self._GenIssueForRow(row)
old_issue_id = self._GetRowTrackerId(row)
if new_issue and not old_issue_id:
self._CreateRowIssue(rowIx, row, new_issue)
elif not new_issue and old_issue_id:
self._ClearRowIssue(rowIx, row)
# Nothing to do for this package.
reason = 'already has issue' if old_issue_id else 'no upgrade needed'
oper.Notice('Nothing to do for row %d, package %r: %s.' %
(rowIx, row[COL_PACKAGE], reason))
except PackageBlacklisted:
oper.Notice('Tracker sync disabled for row %d, package %r: skipped.' %
(rowIx, row[COL_PACKAGE]))
except SyncError:
errors.append('Error processing row %d, pkg: %r. See above.' %
(rowIx, row[COL_PACKAGE]))
if errors:
raise SyncError('\n'.join(errors))
def _GetRowValue(self, row, colName, arch=None):
"""Get value from |row| at |colName|, adjusted for |arch|"""
if arch:
colName = utable.UpgradeTable.GetColumnName(colName, arch=arch)
colName = gdata_lib.PrepColNameForSS(colName)
return row[colName]
def _GenIssueForRow(self, row):
"""Generate an Issue object for |row| if applicable"""
# Row needs an issue if it "needs upgrade" on any platform.
statuses = {}
needs_issue = False
for arch in ARCHES:
state = self._GetRowValue(row, utable.UpgradeTable.COL_STATE, arch)
statuses[arch] = state
if state in self.UPGRADE_STATES:
needs_issue = True
if not needs_issue:
return None
pkg = row[COL_PACKAGE]
team = self._ReduceTeamName(row[COL_TEAM])
if not team:
oper.Error('Unable to create Issue for package "%s" because no '
'team value is specified.' % pkg)
raise SyncError()
labels = ['Type-Bug',
owner = self._ReduceOwnerName(row[COL_OWNER])
status = 'Untriaged'
if owner:
owner = owner + ''
status = 'Available'
elif self.default_owner:
owner = self.default_owner + ''
owner = None # Rather than ''
title = '%s package needs upgrade from upstream Portage' % pkg
lines = ['The %s package can be upgraded from upstream Portage' % pkg,
'At this moment the status on each arch is as follows:']
for arch in sorted(statuses):
arch_status = statuses[arch]
if arch_status:
# Get all versions for this arch right now.
curr_ver_col = utable.UpgradeTable.COL_CURRENT_VER
curr_ver = self._GetRowValue(row, curr_ver_col, arch)
stable_upst_ver_col = utable.UpgradeTable.COL_STABLE_UPSTREAM_VER
stable_upst_ver = self._GetRowValue(row, stable_upst_ver_col, arch)
latest_upst_ver_col = utable.UpgradeTable.COL_LATEST_UPSTREAM_VER
latest_upst_ver = self._GetRowValue(row, latest_upst_ver_col, arch)
arch_vers = ['Current version: %s' % curr_ver,
'Stable upstream version: %s' % stable_upst_ver,
'Latest upstream version: %s' % latest_upst_ver]
lines.append(' On %s: %s' % (arch, arch_status))
lines.append(' %s' % ', '.join(arch_vers))
lines.append(' On %s: not used' % arch)
lines.append('Check the latest status for this package, including '
'which upstream versions are available, at:\n %s' %
lines.append('For help upgrading see: %s' % PKG_UPGRADE_PAGE)
summary = '\n'.join(lines)
issue = gdata_lib.Issue(title=title,
return issue
def _GetRowTrackerId(self, row):
"""Get the tracker issue id in |row| if it exists, return None otherwise.
PackageBlacklisted if package has Tracker column value to disable syncing.
tracker_val = row[COL_TRACKER]
if tracker_val:
return int(tracker_val)
except ValueError:
# See if the unexpected value is one that disables tracker syncing.
if tracker_val.replace(' ', '').lower() in DISABLING_TRACKER_VALUES:
raise PackageBlacklisted()
return None
def _CreateRowIssue(self, rowIx, row, issue):
"""Create a Tracker issue for |issue|, insert into |row| at |rowIx|"""
pkg = row[COL_PACKAGE]
if not self.pretend:
oper.Info('Creating Tracker issue for package %s with details:\n%s' %
(pkg, issue))
# Before actually creating the Tracker issue, confirm that writing
# to this spreadsheet row is going to work.
self.scomm.ClearCellValue(rowIx, self.tracker_col_ix)
except gdata_lib.SpreadsheetError as ex:
oper.Error('Unable to write to row %d, package %r. Aborting issue'
' creation. Error was:\n%s' % (rowIx, pkg, ex))
raise SyncError
issue_id = self.tcomm.CreateTrackerIssue(issue)
except gdata_lib.TrackerInvalidUserError as ex:
oper.Warning('%s. Ignoring owner field for issue %d, package %r.' %
(ex, rowIx, pkg))
issue.summary += ('\n\nNote that the row for this package in'
' the spreadsheet at go/crospkgs has an "owner"\n'
'value that is not a valid Tracker user: "%s".' %
issue.owner = None
issue_id = self.tcomm.CreateTrackerIssue(issue)
oper.Info('Inserting new Tracker issue %d for package %s' %
(issue_id, pkg))
ss_issue_val = self._GenSSLinkToIssue(issue_id)
# This really should not fail since write access was checked before.
self.scomm.ReplaceCellValue(rowIx, self.tracker_col_ix, ss_issue_val)
oper.Notice('Created Tracker issue %d for row %d, package %r' %
(issue_id, rowIx, pkg))
except gdata_lib.SpreadsheetError as ex:
oper.Error('Failed to write link to new issue %d into'
' row %d, package %r:\n%s' %
(issue_id, rowIx, pkg, ex))
oper.Error('This means that the spreadsheet will have no record of'
' this Tracker Issue and will create one again next time'
' unless the spreadsheet is edited by hand!')
raise SyncError
oper.Notice('Would create and insert issue for row %d, package %r' %
(rowIx, pkg))
oper.Info('Issue would be as follows:\n%s' % issue)
def _GenSSLinkToIssue(self, issue_id):
"""Create the spreadsheet hyperlink format for |issue_id|"""
return '=hyperlink("";"%d")' % (issue_id, issue_id)
def _ClearRowIssue(self, rowIx, row):
"""Clear the Tracker cell for row at |rowIx|"""
pkg = row[COL_PACKAGE]
if not self.pretend:
self.scomm.ClearCellValue(rowIx, self.tracker_col_ix)
oper.Notice('Cleared Tracker issue from row %d, package %r' %
(rowIx, pkg))
except gdata_lib.SpreadsheetError as ex:
oper.Error('Error while clearing Tracker issue for'
' row %d, package %r:\n%s' % (rowIx, pkg, ex))
raise SyncError
oper.Notice('Would clear Tracker issue from row %d, package %r' %
(rowIx, pkg))
def PrepareCreds(cred_file, token_file, email):
"""Return a Creds object from given credentials.
If |email| is given, the Creds object will contain that |email|
and a password entered at a prompt.
Otherwise, if |token_file| is given then the Creds object will have
the auth_token from that file.
Otherwise, if |cred_file| is given then the Creds object will have
the email/password from that file.
creds = gdata_lib.Creds()
if email:
elif token_file and os.path.exists(token_file):
elif cred_file and os.path.exists(cred_file):
return creds
def _CreateOptParser():
"""Create the optparser.parser object for command-line args."""
usage = 'Usage: %prog [options]'
epilog = ("""
Use this script to synchronize tracker issues between the package status
spreadsheet and the chromium-os Tracker. It uses the "Tracker" column of the
package spreadsheet. If a package needs an upgrade and has no tracker issue in
that column then a tracker issue is created. If it does not need an upgrade
then that column is cleared.
Credentials must be specified using --auth-token-file, --cred-file or --email.
The first two have default values which you can rely on if valid, the latter
will prompt for your password. If you specify --email you will be given a
chance to save your email/password out as a credentials file for next time.
Uses spreadsheet key %(ss_key)s, worksheet "%(ws_name)s".
(if --test-spreadsheet is set then spreadsheet
%(test_ss_key)s is used).
Use the --team and --owner options to operate only on packages assigned to
particular owners or teams. Generally, running without a team or owner filter
is not intended, so use --team=all and/or --owner=all.
Issues will be assigned to the owner of the package in the spreadsheet, if
available. If not, the owner defaults to value given to --default-owner.
The --owner and --default-owner options accept "me" as an argument, which is
only useful if your username matches your chromium account name.
""" %
{'ss_key': ups.REAL_SS_KEY, 'ws_name': PKGS_WS_NAME,
'test_ss_key': ups.TEST_SS_KEY})
class MyOptParser(optparse.OptionParser):
"""Override default epilog formatter, which strips newlines."""
def format_epilog(self, formatter):
return self.epilog
teamhelp = '[%s]' % ', '.join(Syncer.VALID_TEAMS.keys())
parser = MyOptParser(usage=usage, epilog=epilog)
parser.add_option('--auth-token-file', dest='token_file', type='string',
action='store', default=gdata_lib.TOKEN_FILE,
help='File for reading/writing Docs auth token.'
' [default: "%default"]')
parser.add_option('--cred-file', dest='cred_file', type='string',
action='store', default=gdata_lib.CRED_FILE,
help='Path to gdata credentials file [default: "%default"]')
parser.add_option('--email', dest='email', type='string',
action='store', default=None,
help='Email for Google Doc/Tracker user')
parser.add_option('--pretend', dest='pretend', action='store_true',
help='Do not make any actual changes.')
parser.add_option('--team', dest='team', type='string', action='store',
help='Filter by team; colon-separated %s' % teamhelp)
parser.add_option('--default-owner', dest='default_owner',
type='string', action='store', default=None,
help='Specify issue owner to use when package has no owner')
parser.add_option('--owner', dest='owner', type='string', action='store',
help='Filter by package owner;'
' colon-separated accounts')
parser.add_option('--test-spreadsheet', dest='test_ss',
action='store_true', default=False,
help='Sync to the testing spreadsheet (implies --pretend).')
parser.add_option('--verbose', dest='verbose', action='store_true',
help='Enable verbose output (for debugging)')
return parser
def _CheckOptions(options):
"""Vet the options."""
me = os.environ['USER']
if not and not os.path.exists(options.cred_file): = me
oper.Notice('Assuming your chromium email is'
' Override with --email.' %
if not and not options.owner:
oper.Notice('Without --owner or --team filters this will run for all'
' packages in the spreadsheet (same as --team=all).')
if not cros_build_lib.BooleanPrompt(
'Are you sure you want to run for all packages?', False):
if and == 'all': = None
if options.owner and options.owner == 'all':
options.owner = None
if options.owner and options.owner == 'me':
options.owner = me
oper.Notice('Using %r for owner filter (from $USER envvar)' % options.owner)
if options.test_ss and not options.pretend:
oper.Notice('Running in --pretend mode because of --test-spreadsheet')
options.pretend = True
def main(argv):
"""Main function."""
parser = _CreateOptParser()
(options, _args) = parser.parse_args(argv)
oper.verbose = options.verbose
ss_key = ups.TEST_SS_KEY if options.test_ss else ups.REAL_SS_KEY
# Prepare credentials for Docs and Tracker access.
creds = PrepareCreds(options.cred_file, options.token_file,
scomm = gdata_lib.SpreadsheetComm()
scomm.Connect(creds, ss_key, PKGS_WS_NAME, source='Sync Package Status')
tcomm = gdata_lib.TrackerComm()
tcomm.Connect(creds, TRACKER_PROJECT_NAME, source='Sync Package Status')
oper.Notice('Syncing between Tracker and spreadsheet %s' % ss_key)
syncer = Syncer(tcomm, scomm,
pretend=options.pretend, verbose=options.verbose)
if options.owner:
if options.default_owner:
except SyncError as ex:
# If --email, which is only effective when run interactively (because the
# password must be entered), give the option of saving to a creds file for
# next time.
if and options.cred_file:
prompt = ('Do you want to save credentials for next time to %r?' %
if cros_build_lib.BooleanPrompt(prompt, False):
oper.Notice('Be sure to save the creds file to the same location'
' outside your chroot so it will also be used with'
' future chroots.')