blob: a072eae3eeb691da28bab15f2b78256cc2255ca3 [file] [log] [blame]
#!/usr/bin/env python
# Copyright 2014 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
import argparse
import datetime
import getpass
import json
import os
import smtplib
import sys
import time
import urllib
import urllib2
class Emailer:
DEFAULT_EMAIL_PASSWORD_FILE = '.email_password'
GMAIL_SMTP_SERVER = 'smtp.gmail.com:587'
SUBJECT = 'Chrome GPU Bots Notification'
def __init__(self, email_from, email_to, email_password_file):
self.email_from = email_from
self.email_to = email_to
self.email_password = Emailer._getEmailPassword(email_password_file)
@staticmethod
def format_email_body(time_str, offline_str, failed_str, noteworthy_str):
return '%s%s%s%s' % (time_str, offline_str, failed_str, noteworthy_str)
def send_email(self, body):
message = 'From: %s\r\nTo: %s\r\nSubject: %s\r\n\r\n%s' % (self.email_from,
','.join(self.email_to), Emailer.SUBJECT, body)
try:
server = smtplib.SMTP(Emailer.GMAIL_SMTP_SERVER)
server.starttls()
server.login(self.email_from, self.email_password)
server.sendmail(self.email_from, self.email_to, message)
server.quit()
except Exception as e:
print 'Error sending email: %s' % str(e)
def testEmailLogin(self):
server = smtplib.SMTP(Emailer.GMAIL_SMTP_SERVER)
server.starttls()
server.login(self.email_from, self.email_password)
server.quit()
@staticmethod
def _getEmailPassword(email_password_file):
password = ''
password_file = (email_password_file if email_password_file is not None
else Emailer.DEFAULT_EMAIL_PASSWORD_FILE)
if os.path.isfile(password_file):
with open(password_file, 'r') as f:
password = f.read().strip()
else:
password = getpass.getpass(
'Please enter email password for source email account: ')
return password
class GpuBot:
def __init__(self, waterfall_name, bot_name, bot_data):
self.waterfall_name = waterfall_name
self.bot_name = bot_name
self.bot_data = bot_data
self._end_time = None
self._hours_since_last_run = None
self.failure_string = None
self.bot_url = None
self.build_url = None
def getEndTime(self):
return self._end_time
def setEndTime(self, end_time):
self._end_time = end_time
self._hours_since_last_run = \
roughTimeDiffInHours(end_time, time.localtime())
def getHoursSinceLastRun(self):
return self._hours_since_last_run
def toDict(self):
dict = {'waterfall_name': self.waterfall_name, 'bot_name': self.bot_name}
if self._end_time is not None:
dict['end_time'] = serialTime(self._end_time)
dict['hours_since_last_run'] = self._hours_since_last_run
if self.failure_string is not None:
dict['failure_string'] = self.failure_string
if self.bot_url is not None:
dict['bot_url'] = self.bot_url
if self.build_url is not None:
dict['build_url'] = self.build_url
return dict
@staticmethod
def fromDict(dict):
gpu_bot = GpuBot(dict['waterfall_name'], dict['bot_name'], None)
if 'end_time' in dict:
gpu_bot._end_time = unserializeTime(dict['end_time'])
if 'hours_since_last_run' in dict:
gpu_bot._hours_since_last_run = dict['hours_since_last_run']
if 'failure_string' in dict:
gpu_bot.failure_string = dict['failure_string']
if 'bot_url' in dict:
gpu_bot.bot_url = dict['bot_url']
if 'build_url' in dict:
gpu_bot.build_url = dict['build_url']
return gpu_bot
def errorNoMostRecentBuild(waterfall_name, bot_name):
print 'No most recent build available: %s::%s' % (waterfall_name, bot_name)
class Waterfall:
BASE_URL = 'http://build.chromium.org/p/'
BASE_BUILD_URL = BASE_URL + '%s/builders/%s'
SPECIFIC_BUILD_URL = BASE_URL + '%s/builders/%s/builds/%s'
BASE_JSON_BUILDERS_URL = BASE_URL + '%s/json/builders'
BASE_JSON_BUILDS_URL = BASE_URL + '%s/json/builders/%s/builds'
REGULAR_WATERFALLS = ['chromium.gpu',
'tryserver.chromium.gpu',
'chromium.gpu.fyi']
WEBKIT_GPU_BOTS = ['GPU Win Builder',
'GPU Win Builder (dbg)',
'GPU Win7 (NVIDIA)',
'GPU Win7 (dbg) (NVIDIA)',
'GPU Mac Builder',
'GPU Mac Builder (dbg)',
'GPU Mac10.7',
'GPU Mac10.7 (dbg)',
'GPU Linux Builder',
'GPU Linux Builder (dbg)',
'GPU Linux (NVIDIA)',
'GPU Linux (dbg) (NVIDIA)']
FILTERED_WATERFALLS = [('chromium.webkit', WEBKIT_GPU_BOTS)]
@staticmethod
def getJsonFromUrl(url):
conn = urllib2.urlopen(url)
result = conn.read()
conn.close()
return json.loads(result)
@staticmethod
def getBuildersJsonForWaterfall(waterfall):
querystring = '?filter'
return (Waterfall.getJsonFromUrl((Waterfall.BASE_JSON_BUILDERS_URL + '%s')
% (waterfall, querystring)))
@staticmethod
def getLastNBuildsForBuilder(n, waterfall, builder):
if n <= 0:
return {}
querystring = '?'
for i in range(n):
querystring += 'select=-%d&' % (i + 1)
querystring += 'filter'
return Waterfall.getJsonFromUrl((Waterfall.BASE_JSON_BUILDS_URL + '%s') %
(waterfall, urllib.quote(builder), querystring))
@staticmethod
def getFilteredBuildersJsonForWaterfall(waterfall, filter):
querystring = '?'
for bot_name in filter:
querystring += 'select=%s&' % urllib.quote(bot_name)
querystring += 'filter'
return Waterfall.getJsonFromUrl((Waterfall.BASE_JSON_BUILDERS_URL + '%s')
% (waterfall, querystring))
@staticmethod
def getAllGpuBots():
allbots = {k: Waterfall.getBuildersJsonForWaterfall(k)
for k in Waterfall.REGULAR_WATERFALLS}
filteredbots = {k[0]:
Waterfall.getFilteredBuildersJsonForWaterfall(k[0], k[1])
for k in Waterfall.FILTERED_WATERFALLS}
allbots.update(filteredbots)
return allbots
@staticmethod
def getOfflineBots(bots):
offline_bots = []
for waterfall_name in bots:
waterfall = bots[waterfall_name]
for bot_name in waterfall:
bot = waterfall[bot_name]
if bot['state'] != 'offline':
continue
gpu_bot = GpuBot(waterfall_name, bot_name, bot)
gpu_bot.bot_url = Waterfall.BASE_BUILD_URL % (waterfall_name,
urllib.quote(bot_name))
most_recent_build = Waterfall.getMostRecentlyCompletedBuildForBot(
gpu_bot)
if (most_recent_build and 'times' in most_recent_build and
most_recent_build['times']):
gpu_bot.setEndTime(time.localtime(most_recent_build['times'][1]))
else:
errorNoMostRecentBuild(waterfall_name, bot_name)
offline_bots.append(gpu_bot)
return offline_bots
@staticmethod
def getMostRecentlyCompletedBuildForBot(bot):
if bot.bot_data is not None and 'most_recent_build' in bot.bot_data:
return bot.bot_data['most_recent_build']
# Unfortunately, the JSON API doesn't provide a "most recent completed
# build" call. We just have to get some number of the most recent (including
# current, in-progress builds) and give up if that's not enough.
NUM_BUILDS = 10
builds = Waterfall.getLastNBuildsForBuilder(NUM_BUILDS, bot.waterfall_name,
bot.bot_name)
for i in range(NUM_BUILDS):
current_build_name = '-%d' % (i + 1)
current_build = builds[current_build_name]
if 'results' in current_build and current_build['results'] is not None:
if bot.bot_data is not None:
bot.bot_data['most_recent_build'] = current_build
return current_build
return None
@staticmethod
def getFailedBots(bots):
failed_bots = []
for waterfall_name in bots:
waterfall = bots[waterfall_name]
for bot_name in waterfall:
bot = waterfall[bot_name]
gpu_bot = GpuBot(waterfall_name, bot_name, bot)
gpu_bot.bot_url = Waterfall.BASE_BUILD_URL % (waterfall_name,
urllib.quote(bot_name))
most_recent_build = Waterfall.getMostRecentlyCompletedBuildForBot(
gpu_bot)
if (most_recent_build and 'text' in most_recent_build and
'failed' in most_recent_build['text']):
gpu_bot.failure_string = ' '.join(most_recent_build['text'])
gpu_bot.build_url = Waterfall.SPECIFIC_BUILD_URL % (waterfall_name,
urllib.quote(bot_name), most_recent_build['number'])
failed_bots.append(gpu_bot)
elif not most_recent_build:
errorNoMostRecentBuild(waterfall_name, bot_name)
return failed_bots
def formatTime(t):
return time.strftime("%a, %d %b %Y %H:%M:%S", t)
def roughTimeDiffInHours(t1, t2):
datetimes = []
for t in [t1, t2]:
datetimes.append(datetime.datetime(t.tm_year, t.tm_mon, t.tm_mday,
t.tm_hour, t.tm_min, t.tm_sec))
datetime_diff = datetimes[0] - datetimes[1]
hours = float(datetime_diff.total_seconds()) / 3600.0
return abs(hours)
def getBotStr(bot):
s = ' %s::%s\n' % (bot.waterfall_name, bot.bot_name)
if bot.failure_string is not None:
s += ' failure: %s\n' % bot.failure_string
if bot.getEndTime() is not None:
s += (' last build end time: %s (roughly %f hours ago)\n' %
(formatTime(bot.getEndTime()), bot.getHoursSinceLastRun()))
if bot.bot_url is not None:
s += ' bot url: %s\n' % bot.bot_url
if bot.build_url is not None:
s += ' build url: %s\n' % bot.build_url
s += '\n'
return s
def getBotsStr(bots):
s = ''
for bot in bots:
s += getBotStr(bot)
s += '\n'
return s
def getOfflineBotsStr(offline_bots):
return 'Offline bots:\n%s' % getBotsStr(offline_bots)
def getFailedBotsStr(failed_bots):
return 'Failed bots:\n%s' % getBotsStr(failed_bots)
def getBotDicts(bots):
dicts = []
for bot in bots:
dicts.append(bot.toDict())
return dicts
def unserializeTime(t):
return time.struct_time((t['year'], t['mon'], t['day'], t['hour'], t['min'],
t['sec'], 0, 0, 0))
def serialTime(t):
return {'year': t.tm_year, 'mon': t.tm_mon, 'day': t.tm_mday,
'hour': t.tm_hour, 'min': t.tm_min, 'sec': t.tm_sec}
def getSummary(offline_bots, failed_bots):
offline_bot_dict = getBotDicts(offline_bots)
failed_bot_dict = getBotDicts(failed_bots)
return {'offline': offline_bot_dict, 'failed': failed_bot_dict}
def findBot(name, lst):
for bot in lst:
if bot.bot_name == name:
return bot
return None
def getNoteworthyEvents(offline_bots, failed_bots, previous_results):
CRITICAL_NUM_HOURS = 1.0
previous_offline = (previous_results['offline'] if 'offline'
in previous_results else [])
previous_failures = (previous_results['failed'] if 'failed'
in previous_results else [])
noteworthy_offline = []
for bot in offline_bots:
if bot.getHoursSinceLastRun() >= CRITICAL_NUM_HOURS:
previous_bot = findBot(bot.bot_name, previous_offline)
if (previous_bot is None or
previous_bot.getHoursSinceLastRun() < CRITICAL_NUM_HOURS):
noteworthy_offline.append(bot)
noteworthy_new_failures = []
for bot in failed_bots:
previous_bot = findBot(bot.bot_name, previous_failures)
if previous_bot is None:
noteworthy_new_failures.append(bot)
noteworthy_new_offline_recoveries = []
for bot in previous_offline:
if bot.getHoursSinceLastRun() < CRITICAL_NUM_HOURS:
continue
current_bot = findBot(bot.bot_name, offline_bots)
if current_bot is None:
noteworthy_new_offline_recoveries.append(bot)
noteworthy_new_failure_recoveries = []
for bot in previous_failures:
current_bot = findBot(bot.bot_name, failed_bots)
if current_bot is None:
noteworthy_new_failure_recoveries.append(bot)
return {'offline': noteworthy_offline, 'failed': noteworthy_new_failures,
'recovered_failures': noteworthy_new_failure_recoveries,
'recovered_offline': noteworthy_new_offline_recoveries}
def getNoteworthyStr(noteworthy_events):
s = ''
if noteworthy_events['offline']:
s += 'IMPORTANT bots newly offline for over an hour:\n'
for bot in noteworthy_events['offline']:
s += getBotStr(bot)
s += '\n'
if noteworthy_events['failed']:
s += 'IMPORTANT new failing bots:\n'
for bot in noteworthy_events['failed']:
s += getBotStr(bot)
s += '\n'
if noteworthy_events['recovered_offline']:
s += 'IMPORTANT newly recovered previously offline bots:\n'
for bot in noteworthy_events['recovered_offline']:
s += getBotStr(bot)
s += '\n'
if noteworthy_events['recovered_failures']:
s += 'IMPORTANT newly recovered failing bots:\n'
for bot in noteworthy_events['recovered_failures']:
s += getBotStr(bot)
s += '\n'
return s
def dictsToBots(bots):
offline_bots = []
for bot in bots['offline']:
offline_bots.append(GpuBot.fromDict(bot))
failed_bots = []
for bot in bots['failed']:
failed_bots.append(GpuBot.fromDict(bot))
return {'offline': offline_bots, 'failed': failed_bots}
class GpuBotPoller:
DEFAULT_PREVIOUS_RESULTS_FILE = '.check_gpu_bots_previous_results'
def __init__(self, emailer, send_email_for_recovered_offline_bots,
send_email_for_recovered_failing_bots, send_email_on_error,
previous_results_file):
self.emailer = emailer
self.send_email_for_recovered_offline_bots = \
send_email_for_recovered_offline_bots
self.send_email_for_recovered_failing_bots = \
send_email_for_recovered_failing_bots
self.send_email_on_error = send_email_on_error
self.previous_results_file = previous_results_file
def shouldEmail(self, noteworthy_events):
if noteworthy_events['offline'] or noteworthy_events['failed']:
return True
if (self.send_email_for_recovered_offline_bots and
noteworthy_events['recovered_offline']):
return True
if (self.send_email_for_recovered_failing_bots and
noteworthy_events['recovered_failures']):
return True
return False
def writeResults(self, summary):
results_file = (self.previous_results_file
if self.previous_results_file is not None
else GpuBotPoller.DEFAULT_PREVIOUS_RESULTS_FILE)
with open(results_file, 'w') as f:
f.write(json.dumps(summary))
def getPreviousResults(self):
previous_results_file = (self.previous_results_file
if self.previous_results_file is not None
else GpuBotPoller.DEFAULT_PREVIOUS_RESULTS_FILE)
previous_results = {}
if os.path.isfile(previous_results_file):
with open(previous_results_file, 'r') as f:
previous_results = dictsToBots(json.loads(f.read()))
return previous_results
def checkBots(self):
time_str = 'Current time: %s\n\n' % (formatTime(time.localtime()))
print time_str
try:
bots = Waterfall.getAllGpuBots()
offline_bots = Waterfall.getOfflineBots(bots)
offline_str = getOfflineBotsStr(offline_bots)
print offline_str
failed_bots = Waterfall.getFailedBots(bots)
failed_str = getFailedBotsStr(failed_bots)
print failed_str
previous_results = self.getPreviousResults()
noteworthy_events = getNoteworthyEvents(offline_bots, failed_bots,
previous_results)
noteworthy_str = getNoteworthyStr(noteworthy_events)
print noteworthy_str
summary = getSummary(offline_bots, failed_bots)
self.writeResults(summary)
if (self.emailer is not None and self.shouldEmail(noteworthy_events)):
self.emailer.send_email(Emailer.format_email_body(time_str, offline_str,
failed_str, noteworthy_str))
except Exception as e:
error_str = 'Error: %s' % str(e)
print error_str
if self.send_email_on_error:
self.emailer.send_email(error_str)
def parseArgs(sys_args):
parser = argparse.ArgumentParser(prog=sys_args[0],
description='Query the Chromium GPU Bots Waterfall, output ' +
'potential problems, and optionally repeat automatically and/or ' +
'email notifications of results.')
parser.add_argument('--repeat-delay', type=int, dest='repeat_delay',
required=False,
help='How often to automatically re-run the script, in minutes.')
parser.add_argument('--email-from', type=str, dest='email_from',
required=False,
help='Email address to send from. Requires also specifying ' +
'\'--email-to\'.')
parser.add_argument('--email-to', type=str, dest='email_to', required=False,
nargs='+',
help='Email address(es) to send to. Requires also specifying ' +
'\'--email-from\'')
parser.add_argument('--send-email-for-recovered-offline-bots',
dest='send_email_for_recovered_offline_bots', action='store_true',
default=False,
help='Send an email out when a bot which has been offline for more ' +
'than 1 hour goes back online.')
parser.add_argument('--send-email-for-recovered-failing-bots',
dest='send_email_for_recovered_failing_bots',
action='store_true', default=False,
help='Send an email when a failing bot recovers.')
parser.add_argument('--send-email-on-error',
dest='send_email_on_error',
action='store_true', default=False,
help='Send an email when the script has an error. For example, if ' +
'the server is unreachable.')
parser.add_argument('--email-password-file',
dest='email_password_file',
required=False,
help=(('File containing the plaintext password of the source email ' +
'account. By default, \'%s\' will be tried. If it does not exist, ' +
'you will be prompted. If you opt to store your password on disk ' +
'in plaintext, use of a dummy account is strongly recommended.')
% Emailer.DEFAULT_EMAIL_PASSWORD_FILE))
parser.add_argument('--previous-results-file',
dest='previous_results_file',
required=False,
help=(('File to store the results of the previous invocation of ' +
'this script. By default, \'%s\' will be used.')
% GpuBotPoller.DEFAULT_PREVIOUS_RESULTS_FILE))
args = parser.parse_args(sys_args[1:])
if args.email_from is not None and args.email_to is None:
parser.error('--email-from requires --email-to.')
elif args.email_to is not None and args.email_from is None:
parser.error('--email-to requires --email-from.')
elif args.email_from is None and args.send_email_for_recovered_offline_bots:
parser.error('--send-email-for-recovered-offline-bots requires ' +
'--email-to and --email-from.')
elif (args.email_from is None and args.send_email_for_recovered_failing_bots):
parser.error('--send-email-for-recovered-failing-bots ' +
'requires --email-to and --email-from.')
elif (args.email_from is None and args.send_email_on_error):
parser.error('--send-email-on-error ' +
'requires --email-to and --email-from.')
elif (args.email_password_file and
not os.path.isfile(args.email_password_file)):
parser.error('File does not exist: %s' % args.email_password_file)
return args
def main(sys_args):
args = parseArgs(sys_args)
emailer = None
if args.email_from is not None and args.email_to is not None:
emailer = Emailer(args.email_from, args.email_to, args.email_password_file)
try:
emailer.testEmailLogin()
except Exception as e:
print 'Error logging into email account: %s' % str(e)
return 1
poller = GpuBotPoller(emailer,
args.send_email_for_recovered_offline_bots,
args.send_email_for_recovered_failing_bots,
args.send_email_on_error,
args.previous_results_file)
while True:
poller.checkBots()
if args.repeat_delay is None:
break
print 'Will run again in %d minutes...\n' % args.repeat_delay
time.sleep(args.repeat_delay * 60)
return 0
if __name__ == '__main__':
sys.exit(main(sys.argv))