catapult/firefighter/update/handlers/builds.py

# Copyright 2015 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

import collections
import logging
import re
import time

from google.appengine.api import urlfetch
import webapp2

from base import bigquery
from base import constants
from common import buildbot


class Builds(webapp2.RequestHandler):

  def get(self):
    urlfetch.set_default_fetch_deadline(300)

    bq = bigquery.BigQuery()

    current_events = []
    events = []
    for master_name in constants.MASTER_NAMES:
      builders = buildbot.Builders(master_name)
      available_builds = _AvailableBuilds(builders)
      recorded_builds = _RecordedBuilds(bq, builders, available_builds)
      for builder in builders:
        # Filter out recorded builds from available builds.
        build_numbers = (available_builds[builder.name] -
                         recorded_builds[builder.name])
        builder_current_events, builder_events = _TraceEventsForBuilder(
            builder, build_numbers)
        current_events += builder_current_events
        events += builder_events

    jobs = []
    if current_events:
      jobs += bq.InsertRowsAsync(
          constants.DATASET, constants.CURRENT_BUILDS_TABLE,
          current_events, truncate=True)
    if events:
      jobs += bq.InsertRowsAsync(constants.DATASET, constants.BUILDS_TABLE,
                                 events)

    for job in jobs:
      bq.PollJob(job, 60 * 20)  # 20 minutes.


def _AvailableBuilds(builders):
  available_builds = {}
  for builder in builders:
    if not builder.cached_builds:
      available_builds[builder.name] = frozenset()
      continue

    max_build = max(builder.cached_builds)
    # Buildbot on tryserver.chromium.perf is occasionally including build 0 in
    # its list of cached builds. That results in more builds than we want.
    # Limit the list to the last 100 builds, because the urlfetch URL limit is
    # 2048 bytes, and "&select=100000" * 100 is 1400 bytes.
    builds = frozenset(build for build in builder.cached_builds
                       if build >= max_build - 100)
    available_builds[builder.name] = builds
  return available_builds


def _RecordedBuilds(bq, builders, available_builds):
  # 105 days / 15 weeks. Must be some number greater than 100 days, because
  # we request up to 100 builds (see above comment), and the slowest cron bots
  # run one job every day.
  start_time_ms = -1000 * 60 * 60 * 24 * 105
  table = '%s.%s@%d-' % (constants.DATASET, constants.BUILDS_TABLE,
                         start_time_ms)

  conditions = []
  for builder in builders:
    if not available_builds[builder.name]:
      continue
    max_build = max(available_builds[builder.name])
    min_build = min(available_builds[builder.name])
    conditions.append('WHEN builder = "%s" THEN build >= %d AND build <= %d' %
                      (builder.name, min_build, max_build))

  query = (
      'SELECT builder, build '
      'FROM [%s] ' % table +
      'WHERE CASE %s END ' % ' '.join(conditions) +
      'GROUP BY builder, build'
  )
  query_result = bq.QuerySync(query, 600)

  builds = collections.defaultdict(set)
  for row in query_result:
    builds[row['f'][0]['v']].add(int(row['f'][1]['v']))
  return builds


def _TraceEventsForBuilder(builder, build_numbers):
  if not build_numbers:
    return (), ()

  build_numbers_string = ', '.join(map(str, sorted(build_numbers)))
  logging.info('Getting %s: %s', builder.name, build_numbers_string)

  # Fetch build information and generate trace events.
  current_events = []
  events = []

  builder_builds = builder.builds.Fetch(build_numbers)
  query_time = time.time()
  for build in builder_builds:
    if build.complete:
      events += _TraceEventsFromBuild(builder, build, query_time)
    else:
      current_events += _TraceEventsFromBuild(builder, build, query_time)

  return current_events, events


def _TraceEventsFromBuild(builder, build, query_time):
  match = re.match(r'(.+) \(([0-9]+)\)', builder.name)
  if match:
    configuration, host_shard = match.groups()
    host_shard = int(host_shard)
  else:
    configuration = builder.name
    host_shard = 0

  # Build trace event.
  if build.end_time:
    build_end_time = build.end_time
  else:
    build_end_time = query_time
  os, os_version, role = _ParseBuilderName(builder.master_name, builder.name)
  yield {
      'name': 'Build %d' % build.number,
      'start_time': build.start_time,
      'end_time': build_end_time,

      'build': build.number,
      'builder': builder.name,
      'configuration': configuration,
      'host_shard': host_shard,
      'hostname': build.slave_name,
      'master': builder.master_name,
      'os': os,
      'os_version': os_version,
      'role': role,
      'status': build.status,
      'url': build.url,
  }

  # Step trace events.
  for step in build.steps:
    if not step.start_time:
      continue

    if step.name == 'steps':
      continue

    if step.end_time:
      step_end_time = step.end_time
    else:
      step_end_time = query_time
    yield {
        'name': step.name,
        'start_time': step.start_time,
        'end_time': step_end_time,

        'benchmark': step.name,  # TODO(dtu): This isn't always right.
        'build': build.number,
        'builder': builder.name,
        'configuration': configuration,
        'host_shard': host_shard,
        'hostname': build.slave_name,
        'master': builder.master_name,
        'os': os,
        'os_version': os_version,
        'role': role,
        'status': step.status,
        'url': step.url,
    }


def _ParseBuilderName(master_name, builder_name):
  if master_name == 'chromium.perf':
    match = re.match(r'^([A-Za-z]+)(?: ([0-9\.]+|XP))?([A-Za-z0-9-\. ]+)? '
                     r'(Builder|Perf)(?: \([0-9]+\))?$', builder_name).groups()
    os = match[0]
    if match[1]:
      os_version = match[1]
    else:
      os_version = None
    if match[3] == 'Builder':
      role = 'builder'
    elif match[3] == 'Perf':
      role = 'tester'
    else:
      raise NotImplementedError()
  elif master_name == 'client.catapult':
    match = re.match(r'^Catapult(?: ([A-Za-z])+)? ([A-Za-z]+)$',
                     builder_name).groups()
    os = match[1]
    os_version = None
    role = match[0]
    if not role:
      role = 'tester'
  elif master_name == 'tryserver.chromium.perf':
    match = re.match(r'^(android|linux|mac|win).*_([a-z]+)$',
                     builder_name).groups()
    os = match[0]
    os_version = None
    role = match[1]
  elif master_name == 'tryserver.client.catapult':
    match = re.match(r'^Catapult(?: (Android|Linux|Mac|Windows))? ([A-Za-z]+)$',
                     builder_name).groups()
    os = match[0]
    os_version = None
    role = match[1]
  else:
    raise NotImplementedError()

  if os:
    os = os.lower()
  if os == 'windows':
    os = 'win'
  if os_version:
    os_version = os_version.lower()
  role = role.lower()

  return (os, os_version, role)