| #!/usr/bin/env python |
| # Copyright 2014 the V8 project authors. All rights reserved. |
| # Use of this source code is governed by a BSD-style license that can be |
| # found in the LICENSE file. |
| |
| """ |
| Performance runner for d8. |
| |
| Call e.g. with tools/run-perf.py --arch ia32 some_suite.json |
| |
| The suite json format is expected to be: |
| { |
| "path": <relative path chunks to perf resources and main file>, |
| "name": <optional suite name, file name is default>, |
| "archs": [<architecture name for which this suite is run>, ...], |
| "binary": <name of binary to run, default "d8">, |
| "flags": [<flag to d8>, ...], |
| "run_count": <how often will this suite run (optional)>, |
| "run_count_XXX": <how often will this suite run for arch XXX (optional)>, |
| "resources": [<js file to be loaded before main>, ...] |
| "main": <main js perf runner file>, |
| "results_regexp": <optional regexp>, |
| "results_processor": <optional python results processor script>, |
| "units": <the unit specification for the performance dashboard>, |
| "tests": [ |
| { |
| "name": <name of the trace>, |
| "results_regexp": <optional more specific regexp>, |
| "results_processor": <optional python results processor script>, |
| "units": <the unit specification for the performance dashboard>, |
| }, ... |
| ] |
| } |
| |
| The tests field can also nest other suites in arbitrary depth. A suite |
| with a "main" file is a leaf suite that can contain one more level of |
| tests. |
| |
| A suite's results_regexp is expected to have one string place holder |
| "%s" for the trace name. A trace's results_regexp overwrites suite |
| defaults. |
| |
| A suite's results_processor may point to an optional python script. If |
| specified, it is called after running the tests like this (with a path |
| relatve to the suite level's path): |
| <results_processor file> <same flags as for d8> <suite level name> <output> |
| |
| The <output> is a temporary file containing d8 output. The results_regexp will |
| be applied to the output of this script. |
| |
| A suite without "tests" is considered a performance test itself. |
| |
| Full example (suite with one runner): |
| { |
| "path": ["."], |
| "flags": ["--expose-gc"], |
| "archs": ["ia32", "x64"], |
| "run_count": 5, |
| "run_count_ia32": 3, |
| "main": "run.js", |
| "results_regexp": "^%s: (.+)$", |
| "units": "score", |
| "tests": [ |
| {"name": "Richards"}, |
| {"name": "DeltaBlue"}, |
| {"name": "NavierStokes", |
| "results_regexp": "^NavierStokes: (.+)$"} |
| ] |
| } |
| |
| Full example (suite with several runners): |
| { |
| "path": ["."], |
| "flags": ["--expose-gc"], |
| "archs": ["ia32", "x64"], |
| "run_count": 5, |
| "units": "score", |
| "tests": [ |
| {"name": "Richards", |
| "path": ["richards"], |
| "main": "run.js", |
| "run_count": 3, |
| "results_regexp": "^Richards: (.+)$"}, |
| {"name": "NavierStokes", |
| "path": ["navier_stokes"], |
| "main": "run.js", |
| "results_regexp": "^NavierStokes: (.+)$"} |
| ] |
| } |
| |
| Path pieces are concatenated. D8 is always run with the suite's path as cwd. |
| """ |
| |
| from collections import OrderedDict |
| import json |
| import math |
| import optparse |
| import os |
| import re |
| import sys |
| |
| from testrunner.local import commands |
| from testrunner.local import utils |
| |
| ARCH_GUESS = utils.DefaultArch() |
| SUPPORTED_ARCHS = ["android_arm", |
| "android_arm64", |
| "android_ia32", |
| "arm", |
| "ia32", |
| "mips", |
| "mipsel", |
| "nacl_ia32", |
| "nacl_x64", |
| "x64", |
| "arm64"] |
| |
| GENERIC_RESULTS_RE = re.compile(r"^RESULT ([^:]+): ([^=]+)= ([^ ]+) ([^ ]*)$") |
| RESULT_STDDEV_RE = re.compile(r"^\{([^\}]+)\}$") |
| RESULT_LIST_RE = re.compile(r"^\[([^\]]+)\]$") |
| |
| |
| |
| def GeometricMean(values): |
| """Returns the geometric mean of a list of values. |
| |
| The mean is calculated using log to avoid overflow. |
| """ |
| values = map(float, values) |
| return str(math.exp(sum(map(math.log, values)) / len(values))) |
| |
| |
| class Results(object): |
| """Place holder for result traces.""" |
| def __init__(self, traces=None, errors=None): |
| self.traces = traces or [] |
| self.errors = errors or [] |
| |
| def ToDict(self): |
| return {"traces": self.traces, "errors": self.errors} |
| |
| def WriteToFile(self, file_name): |
| with open(file_name, "w") as f: |
| f.write(json.dumps(self.ToDict())) |
| |
| def __add__(self, other): |
| self.traces += other.traces |
| self.errors += other.errors |
| return self |
| |
| def __str__(self): # pragma: no cover |
| return str(self.ToDict()) |
| |
| |
| class Node(object): |
| """Represents a node in the suite tree structure.""" |
| def __init__(self, *args): |
| self._children = [] |
| |
| def AppendChild(self, child): |
| self._children.append(child) |
| |
| |
| class DefaultSentinel(Node): |
| """Fake parent node with all default values.""" |
| def __init__(self): |
| super(DefaultSentinel, self).__init__() |
| self.binary = "d8" |
| self.run_count = 10 |
| self.timeout = 60 |
| self.path = [] |
| self.graphs = [] |
| self.flags = [] |
| self.resources = [] |
| self.results_regexp = None |
| self.stddev_regexp = None |
| self.units = "score" |
| self.total = False |
| |
| |
| class Graph(Node): |
| """Represents a suite definition. |
| |
| Can either be a leaf or an inner node that provides default values. |
| """ |
| def __init__(self, suite, parent, arch): |
| super(Graph, self).__init__() |
| self._suite = suite |
| |
| assert isinstance(suite.get("path", []), list) |
| assert isinstance(suite["name"], basestring) |
| assert isinstance(suite.get("flags", []), list) |
| assert isinstance(suite.get("resources", []), list) |
| |
| # Accumulated values. |
| self.path = parent.path[:] + suite.get("path", []) |
| self.graphs = parent.graphs[:] + [suite["name"]] |
| self.flags = parent.flags[:] + suite.get("flags", []) |
| self.resources = parent.resources[:] + suite.get("resources", []) |
| |
| # Descrete values (with parent defaults). |
| self.binary = suite.get("binary", parent.binary) |
| self.run_count = suite.get("run_count", parent.run_count) |
| self.run_count = suite.get("run_count_%s" % arch, self.run_count) |
| self.timeout = suite.get("timeout", parent.timeout) |
| self.units = suite.get("units", parent.units) |
| self.total = suite.get("total", parent.total) |
| |
| # A regular expression for results. If the parent graph provides a |
| # regexp and the current suite has none, a string place holder for the |
| # suite name is expected. |
| # TODO(machenbach): Currently that makes only sense for the leaf level. |
| # Multiple place holders for multiple levels are not supported. |
| if parent.results_regexp: |
| regexp_default = parent.results_regexp % re.escape(suite["name"]) |
| else: |
| regexp_default = None |
| self.results_regexp = suite.get("results_regexp", regexp_default) |
| |
| # A similar regular expression for the standard deviation (optional). |
| if parent.stddev_regexp: |
| stddev_default = parent.stddev_regexp % re.escape(suite["name"]) |
| else: |
| stddev_default = None |
| self.stddev_regexp = suite.get("stddev_regexp", stddev_default) |
| |
| |
| class Trace(Graph): |
| """Represents a leaf in the suite tree structure. |
| |
| Handles collection of measurements. |
| """ |
| def __init__(self, suite, parent, arch): |
| super(Trace, self).__init__(suite, parent, arch) |
| assert self.results_regexp |
| self.results = [] |
| self.errors = [] |
| self.stddev = "" |
| |
| def ConsumeOutput(self, stdout): |
| try: |
| self.results.append( |
| re.search(self.results_regexp, stdout, re.M).group(1)) |
| except: |
| self.errors.append("Regexp \"%s\" didn't match for test %s." |
| % (self.results_regexp, self.graphs[-1])) |
| |
| try: |
| if self.stddev_regexp and self.stddev: |
| self.errors.append("Test %s should only run once since a stddev " |
| "is provided by the test." % self.graphs[-1]) |
| if self.stddev_regexp: |
| self.stddev = re.search(self.stddev_regexp, stdout, re.M).group(1) |
| except: |
| self.errors.append("Regexp \"%s\" didn't match for test %s." |
| % (self.stddev_regexp, self.graphs[-1])) |
| |
| def GetResults(self): |
| return Results([{ |
| "graphs": self.graphs, |
| "units": self.units, |
| "results": self.results, |
| "stddev": self.stddev, |
| }], self.errors) |
| |
| |
| class Runnable(Graph): |
| """Represents a runnable suite definition (i.e. has a main file). |
| """ |
| @property |
| def main(self): |
| return self._suite.get("main", "") |
| |
| def ChangeCWD(self, suite_path): |
| """Changes the cwd to to path defined in the current graph. |
| |
| The tests are supposed to be relative to the suite configuration. |
| """ |
| suite_dir = os.path.abspath(os.path.dirname(suite_path)) |
| bench_dir = os.path.normpath(os.path.join(*self.path)) |
| os.chdir(os.path.join(suite_dir, bench_dir)) |
| |
| def GetCommand(self, shell_dir): |
| # TODO(machenbach): This requires +.exe if run on windows. |
| return ( |
| [os.path.join(shell_dir, self.binary)] + |
| self.flags + |
| self.resources + |
| [self.main] |
| ) |
| |
| def Run(self, runner): |
| """Iterates over several runs and handles the output for all traces.""" |
| for stdout in runner(): |
| for trace in self._children: |
| trace.ConsumeOutput(stdout) |
| res = reduce(lambda r, t: r + t.GetResults(), self._children, Results()) |
| |
| if not res.traces or not self.total: |
| return res |
| |
| # Assume all traces have the same structure. |
| if len(set(map(lambda t: len(t["results"]), res.traces))) != 1: |
| res.errors.append("Not all traces have the same number of results.") |
| return res |
| |
| # Calculate the geometric means for all traces. Above we made sure that |
| # there is at least one trace and that the number of results is the same |
| # for each trace. |
| n_results = len(res.traces[0]["results"]) |
| total_results = [GeometricMean(t["results"][i] for t in res.traces) |
| for i in range(0, n_results)] |
| res.traces.append({ |
| "graphs": self.graphs + ["Total"], |
| "units": res.traces[0]["units"], |
| "results": total_results, |
| "stddev": "", |
| }) |
| return res |
| |
| class RunnableTrace(Trace, Runnable): |
| """Represents a runnable suite definition that is a leaf.""" |
| def __init__(self, suite, parent, arch): |
| super(RunnableTrace, self).__init__(suite, parent, arch) |
| |
| def Run(self, runner): |
| """Iterates over several runs and handles the output.""" |
| for stdout in runner(): |
| self.ConsumeOutput(stdout) |
| return self.GetResults() |
| |
| |
| class RunnableGeneric(Runnable): |
| """Represents a runnable suite definition with generic traces.""" |
| def __init__(self, suite, parent, arch): |
| super(RunnableGeneric, self).__init__(suite, parent, arch) |
| |
| def Run(self, runner): |
| """Iterates over several runs and handles the output.""" |
| traces = OrderedDict() |
| for stdout in runner(): |
| for line in stdout.strip().splitlines(): |
| match = GENERIC_RESULTS_RE.match(line) |
| if match: |
| stddev = "" |
| graph = match.group(1) |
| trace = match.group(2) |
| body = match.group(3) |
| units = match.group(4) |
| match_stddev = RESULT_STDDEV_RE.match(body) |
| match_list = RESULT_LIST_RE.match(body) |
| if match_stddev: |
| result, stddev = map(str.strip, match_stddev.group(1).split(",")) |
| results = [result] |
| elif match_list: |
| results = map(str.strip, match_list.group(1).split(",")) |
| else: |
| results = [body.strip()] |
| |
| trace_result = traces.setdefault(trace, Results([{ |
| "graphs": self.graphs + [graph, trace], |
| "units": (units or self.units).strip(), |
| "results": [], |
| "stddev": "", |
| }], [])) |
| trace_result.traces[0]["results"].extend(results) |
| trace_result.traces[0]["stddev"] = stddev |
| |
| return reduce(lambda r, t: r + t, traces.itervalues(), Results()) |
| |
| |
| def MakeGraph(suite, arch, parent): |
| """Factory method for making graph objects.""" |
| if isinstance(parent, Runnable): |
| # Below a runnable can only be traces. |
| return Trace(suite, parent, arch) |
| elif suite.get("main"): |
| # A main file makes this graph runnable. |
| if suite.get("tests"): |
| # This graph has subgraphs (traces). |
| return Runnable(suite, parent, arch) |
| else: |
| # This graph has no subgraphs, it's a leaf. |
| return RunnableTrace(suite, parent, arch) |
| elif suite.get("generic"): |
| # This is a generic suite definition. It is either a runnable executable |
| # or has a main js file. |
| return RunnableGeneric(suite, parent, arch) |
| elif suite.get("tests"): |
| # This is neither a leaf nor a runnable. |
| return Graph(suite, parent, arch) |
| else: # pragma: no cover |
| raise Exception("Invalid suite configuration.") |
| |
| |
| def BuildGraphs(suite, arch, parent=None): |
| """Builds a tree structure of graph objects that corresponds to the suite |
| configuration. |
| """ |
| parent = parent or DefaultSentinel() |
| |
| # TODO(machenbach): Implement notion of cpu type? |
| if arch not in suite.get("archs", ["ia32", "x64"]): |
| return None |
| |
| graph = MakeGraph(suite, arch, parent) |
| for subsuite in suite.get("tests", []): |
| BuildGraphs(subsuite, arch, graph) |
| parent.AppendChild(graph) |
| return graph |
| |
| |
| def FlattenRunnables(node): |
| """Generator that traverses the tree structure and iterates over all |
| runnables. |
| """ |
| if isinstance(node, Runnable): |
| yield node |
| elif isinstance(node, Node): |
| for child in node._children: |
| for result in FlattenRunnables(child): |
| yield result |
| else: # pragma: no cover |
| raise Exception("Invalid suite configuration.") |
| |
| |
| # TODO: Implement results_processor. |
| def Main(args): |
| parser = optparse.OptionParser() |
| parser.add_option("--arch", |
| help=("The architecture to run tests for, " |
| "'auto' or 'native' for auto-detect"), |
| default="x64") |
| parser.add_option("--buildbot", |
| help="Adapt to path structure used on buildbots", |
| default=False, action="store_true") |
| parser.add_option("--json-test-results", |
| help="Path to a file for storing json results.") |
| parser.add_option("--outdir", help="Base directory with compile output", |
| default="out") |
| (options, args) = parser.parse_args(args) |
| |
| if len(args) == 0: # pragma: no cover |
| parser.print_help() |
| return 1 |
| |
| if options.arch in ["auto", "native"]: # pragma: no cover |
| options.arch = ARCH_GUESS |
| |
| if not options.arch in SUPPORTED_ARCHS: # pragma: no cover |
| print "Unknown architecture %s" % options.arch |
| return 1 |
| |
| workspace = os.path.abspath(os.path.join(os.path.dirname(__file__), "..")) |
| |
| if options.buildbot: |
| shell_dir = os.path.join(workspace, options.outdir, "Release") |
| else: |
| shell_dir = os.path.join(workspace, options.outdir, |
| "%s.release" % options.arch) |
| |
| results = Results() |
| for path in args: |
| path = os.path.abspath(path) |
| |
| if not os.path.exists(path): # pragma: no cover |
| results.errors.append("Configuration file %s does not exist." % path) |
| continue |
| |
| with open(path) as f: |
| suite = json.loads(f.read()) |
| |
| # If no name is given, default to the file name without .json. |
| suite.setdefault("name", os.path.splitext(os.path.basename(path))[0]) |
| |
| for runnable in FlattenRunnables(BuildGraphs(suite, options.arch)): |
| print ">>> Running suite: %s" % "/".join(runnable.graphs) |
| runnable.ChangeCWD(path) |
| |
| def Runner(): |
| """Output generator that reruns several times.""" |
| for i in xrange(0, max(1, runnable.run_count)): |
| # TODO(machenbach): Allow timeout per arch like with run_count per |
| # arch. |
| output = commands.Execute(runnable.GetCommand(shell_dir), |
| timeout=runnable.timeout) |
| print ">>> Stdout (#%d):" % (i + 1) |
| print output.stdout |
| if output.stderr: # pragma: no cover |
| # Print stderr for debugging. |
| print ">>> Stderr (#%d):" % (i + 1) |
| print output.stderr |
| if output.timed_out: |
| print ">>> Test timed out after %ss." % runnable.timeout |
| yield output.stdout |
| |
| # Let runnable iterate over all runs and handle output. |
| results += runnable.Run(Runner) |
| |
| if options.json_test_results: |
| results.WriteToFile(options.json_test_results) |
| else: # pragma: no cover |
| print results |
| |
| return min(1, len(results.errors)) |
| |
| if __name__ == "__main__": # pragma: no cover |
| sys.exit(Main(sys.argv[1:])) |