catapult/base/util/perfbot_stats/chrome_perf_step_timings.py

#!/usr/bin/env python2.7
# Copyright 2015 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

"""Script to pull chromium.perf step timings from chrome-infra-stats API.

Currently this pulls the list of steps per builder. For each step, if it is not
a setup step, we get the step stats for the last 20 runs for that builder.

The API documentation for chrome-infra-stats is at:
https://apis-explorer.appspot.com/apis-explorer/?
   base=https://chrome-infra-stats.appspot.com/_ah/api#p/
"""

import calendar
import csv
import json
import sys
import urllib
import urllib2
from dateutil import parser as dateparser
from datetime import datetime, timedelta

BUILDER_STEPS_URL = ('https://chrome-infra-stats.appspot.com/_ah/api/stats/v1/'
                     'masters/chromium.perf/%s')

STEP_ACTIVE_URL = ('https://chrome-infra-stats.appspot.com/_ah/api/stats/v1/'
                   'steps/last/chromium.perf/%s/%s/1')

STEP_STATS_URL = ('https://chrome-infra-stats.appspot.com/_ah/api/stats/v1/'
                  'stats/last/chromium.perf/%s/%s/20')

IGNORED_STEPS = [
  'List Perf Tests',
  'Sharded Perf Tests',
  'authorize_adb_devices',
  'bot_update',
  'build__schedule__time__',
  'clean local files',
  'cleanup_temp',
  'device_status_check',
  'extract build',
  'gclient runhooks',
  'get compile targets for scripts',
  'get perf test list',
  'gsutil download_build_product',
  'host_info',
  'install ChromeShell.apk',
  'json.output cache',
  'json.output cache',
  'overall__build__result__',
  'overall__queued__time__',
  'provision_devices',
  'read test spec',
  'rmtree build directory',
  'setup_build',
  'spawn_logcat_monitor',
  'stack_tool_for_tombstones',
  'stack_tool_with_logcat_dump',
  'steps',
  'test_report',
  'unzip_build_product',
  'update_scripts'
]

KNOWN_TESTERS_LIST = [
    'Android Nexus4 Perf',
    'Android Nexus5 Perf',
    'Android Nexus6 Perf',
    'Android Nexus10 Perf',
    'Android Nexus7v2 Perf',
    'Android One Perf',
    'Linux Perf (1)',
    'Linux Perf (2)',
    'Linux Perf (3)',
    'Linux Perf (4)',
    'Linux Perf (5)',
    'Mac 10.8 Perf (1)',
    'Mac 10.8 Perf (2)',
    'Mac 10.8 Perf (3)',
    'Mac 10.8 Perf (4)',
    'Mac 10.8 Perf (5)',
    'Mac 10.9 Perf (1)',
    'Mac 10.9 Perf (2)',
    'Mac 10.9 Perf (3)',
    'Mac 10.9 Perf (4)',
    'Mac 10.9 Perf (5)',
    'Win 7 ATI GPU Perf',
    'Win 7 Intel GPU Perf',
    'Win 7 Low-End Perf (1)',
    'Win 7 Low-End Perf (2)',
    'Win 7 Nvidia GPU Perf',
    'Win 7 Perf (1)',
    'Win 7 Perf (2)',
    'Win 7 Perf (3)',
    'Win 7 Perf (4)',
    'Win 7 Perf (5)',
    'Win 7 x64 Perf (1)',
    'Win 7 x64 Perf (2)',
    'Win 8 Perf (1)',
    'Win 8 Perf (2)',
    'Win XP Perf (1)',
    'Win XP Perf (2)',
    'Win XP Perf (3)',
    'Win XP Perf (4)',
    'Win XP Perf (5)'
]

USAGE = 'Usage: chrome-perf-step-timings.py <outfilename>'

if len(sys.argv) != 2:
  print USAGE
  sys.exit(0)
outfilename = sys.argv[1]

threshold_time = datetime.now() - timedelta(days=2)

col_names = [('builder', 'step', 'run_count', 'stddev', 'mean', 'maximum',
             'median', 'seventyfive', 'ninety', 'ninetynine')]
with open(outfilename, 'wb') as f:
  writer = csv.writer(f)
  writer.writerows(col_names)

for builder in KNOWN_TESTERS_LIST:
  step_timings = []
  url = BUILDER_STEPS_URL % urllib.quote(builder)
  response = urllib2.urlopen(url)
  results = json.load(response)
  steps = results['steps']
  steps.sort() # to group tests and their references together.
  for step in steps:
    if step in IGNORED_STEPS:
      continue
    url = STEP_ACTIVE_URL % (urllib.quote(builder), urllib.quote(step))
    response = urllib2.urlopen(url)
    results = json.load(response)
    if ('step_records' not in results.keys() or
        len(results['step_records']) == 0):
      continue
    first_record = results['step_records'][0]
    last_step_time = dateparser.parse(first_record['step_start'])
    # ignore steps that did not run for more than 2 days
    if last_step_time < threshold_time:
      continue
    url = STEP_STATS_URL % (urllib.quote(builder), urllib.quote(step))
    response = urllib2.urlopen(url)
    results = json.load(response)
    step_timings.append([builder, step, results['count'], results['stddev'],
                         results['mean'], results['maximum'], results['median'],
                         results['seventyfive'], results['ninety'],
                         results['ninetynine']])
  with open(outfilename, 'ab') as f:
    writer = csv.writer(f)
    writer.writerows(step_timings)