Tools/TestResultServer/handlers/buildershandler.py

# Copyright (C) 2013 Google Inc. All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are
# met:
#
#     * Redistributions of source code must retain the above copyright
# notice, this list of conditions and the following disclaimer.
#     * Redistributions in binary form must reproduce the above
# copyright notice, this list of conditions and the following disclaimer
# in the documentation and/or other materials provided with the
# distribution.
#     * Neither the name of Google Inc. nor the names of its
# contributors may be used to endorse or promote products derived from
# this software without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

import datetime
import json
import logging
import sys
import traceback
import urllib2
import webapp2

from google.appengine.api import memcache

MASTERS = [
    {'name': 'ChromiumWin', 'url': 'http://build.chromium.org/p/chromium.win', 'groups': ['@ToT Chromium']},
    {'name': 'ChromiumMac', 'url': 'http://build.chromium.org/p/chromium.mac', 'groups': ['@ToT Chromium']},
    {'name': 'ChromiumLinux', 'url': 'http://build.chromium.org/p/chromium.linux', 'groups': ['@ToT Chromium']},
    {'name': 'ChromiumChromiumOS', 'url': 'http://build.chromium.org/p/chromium.chromiumos', 'groups': ['@ToT ChromeOS']},
    {'name': 'ChromiumGPU', 'url': 'http://build.chromium.org/p/chromium.gpu', 'groups': ['@ToT Chromium']},
    {'name': 'ChromiumGPUFYI', 'url': 'http://build.chromium.org/p/chromium.gpu.fyi', 'groups': ['@ToT Chromium FYI']},
    {'name': 'ChromiumPerfAv', 'url': 'http://build.chromium.org/p/chromium.perf_av', 'groups': ['@ToT Chromium']},
    {'name': 'ChromiumWebkit', 'url': 'http://build.chromium.org/p/chromium.webkit', 'groups': ['@ToT Chromium', '@ToT Blink']},
    {'name': 'ChromiumFYI', 'url': 'http://build.chromium.org/p/chromium.fyi', 'groups': ['@ToT Chromium FYI']},
    {'name': 'V8', 'url': 'http://build.chromium.org/p/client.v8', 'groups': ['@ToT V8']},
]


class FetchBuildersException(Exception): pass


def master_json_url(master_url):
    return master_url + '/json/builders'


def builder_json_url(master_url, builder):
    return master_json_url(master_url) + '/' + urllib2.quote(builder)


def cached_build_json_url(master_url, builder, build_number):
    return builder_json_url(master_url, builder) + '/builds/' + str(build_number)


def fetch_json(url):
    logging.debug('Fetching %s' % url)
    fetched_json = {}
    try:
        resp = urllib2.urlopen(url)
    except:
        exc_info = sys.exc_info()
        logging.warning('Error while fetching %s: %s', url, exc_info[1])
        return fetched_json

    try:
        fetched_json = json.load(resp)
    except:
        exc_info = sys.exc_info()
        logging.warning('Unable to parse JSON response from %s: %s', url, exc_info[1])

    return fetched_json


def get_latest_build(build_data):
    cached_builds = []
    if 'cachedBuilds' in build_data:
        cached_builds = build_data['cachedBuilds']

    current_builds = build_data['currentBuilds']

    latest_cached_builds = set(cached_builds) - set(current_builds)
    if len(latest_cached_builds) != 0:
        latest_cached_builds = sorted(list(latest_cached_builds))
        latest_build = latest_cached_builds[-1]
    elif len(current_builds) != 0:
        latest_build = current_builds[0]
    else:
        basedir = build_data['basedir'] if 'basedir' in build_data else 'current builder'
        logging.info('No cached or current builds for %s', basedir)
        return None

    return latest_build


def dump_json(data):
    return json.dumps(data, separators=(',', ':'), sort_keys=True)


def fetch_buildbot_data(masters, force_update=False):
    if force_update:
        logging.info('Starting a forced buildbot update. Failure to fetch a master\'s data will not abort the fetch.')

    start_time = datetime.datetime.now()
    master_data = masters[:]
    for master in master_data:
        master_url = master['url']
        tests_object = master.setdefault('tests', {})
        master['tests'] = tests_object

        builders = fetch_json(master_json_url(master_url))
        if not builders:
            msg = 'Could not fetch builders from master "%s": %s.' % (master['name'], master_url)
            logging.warning(msg)
            if force_update:
                continue
            else:
                logging.warning('Aborting fetch.')
                raise FetchBuildersException(msg)

        for builder in builders:
            build_data = fetch_json(builder_json_url(master_url, builder))

            latest_build = get_latest_build(build_data)
            if not latest_build:
                logging.info('Skipping builder %s because it lacked cached or current builds.', builder)
                continue

            build = fetch_json(cached_build_json_url(master_url, builder, latest_build))
            if not build:
                logging.info('Skipping build %s on builder %s due to empty data', latest_build, builder)
            for step in build['steps']:
                step_name = step['name']
                is_test_step = 'test' in step_name and 'archive' not in step_name and 'Run tests' not in step_name
                if not is_test_step:
                    continue

                if step_name == 'webkit_tests':
                    step_name = 'layout-tests'

                tests_object.setdefault(step_name, {'builders': []})
                tests_object[step_name]['builders'].append(builder)

        for builders in tests_object.values():
            builders['builders'].sort()

    output_data = {'masters': master_data}

    delta = datetime.datetime.now() - start_time

    logging.info('Fetched buildbot data in %s seconds.', delta.seconds)

    return dump_json(output_data)


class UpdateBuilders(webapp2.RequestHandler):
    """Fetch and update the cached buildbot data."""
    def get(self):
        force_update = True if self.request.get('force') else False
        try:
            buildbot_data = fetch_buildbot_data(MASTERS, force_update)
            memcache.set('buildbot_data', buildbot_data)
            self.response.set_status(200)
            self.response.out.write("ok")
        except FetchBuildersException, ex:
            logging.error('Not updating builders because fetch failed: %s', str(ex))
            self.response.set_status(500)
            self.response.out.write(ex.message)



class GetBuilders(webapp2.RequestHandler):
    """Return a list of masters mapped to their respective builders, possibly using cached data."""
    def get(self):
        callback = self.request.get('callback')

        buildbot_data = memcache.get('buildbot_data')

        if not buildbot_data:
            logging.warning('No buildbot data in memcache. If this message repeats, something is probably wrong with memcache.')

            # Since we have no cached buildbot data, we would rather have missing masters than no data at all.
            buildbot_data = fetch_buildbot_data(MASTERS, True)
            try:
                memcache.set('buildbot_data', buildbot_data)
            except ValueError, err:
                logging.error(str(err))

        if callback:
            buildbot_data = callback + '(' + buildbot_data + ');'

        self.response.out.write(buildbot_data)