blob: ce55f0660f32eb8bbd264d0004d471b273a0062a [file] [log] [blame]
<!DOCTYPE html>
<!--
Copyright 2016 The Chromium Authors. All rights reserved.
Use of this source code is governed by a BSD-style license that can be
found in the LICENSE file.
-->
<link rel="import" href="/tracing/base/iteration_helpers.html">
<link rel="import" href="/tracing/base/range.html">
<link rel="import" href="/tracing/base/running_statistics.html">
<link rel="import" href="/tracing/base/sorted_array_utils.html">
<link rel="import" href="/tracing/base/statistics.html">
<link rel="import" href="/tracing/base/unit.html">
<link rel="import" href="/tracing/value/diagnostics/diagnostic_map.html">
<link rel="import" href="/tracing/value/numeric.html">
<script>
'use strict';
tr.exportTo('tr.v', function() {
var MAX_DIAGNOSTIC_MAPS = 16;
// p-values less than this indicate statistical significance.
var DEFAULT_ALPHA = 0.05;
/** @enum */
var Significance = {
DONT_CARE: -1,
INSIGNIFICANT: 0,
SIGNIFICANT: 1
};
var DEFAULT_BOUNDARIES_FOR_UNIT = new Map();
class HistogramBin {
/**
* @param {!tr.b.Range} range
*/
constructor(range) {
this.range = range;
this.count = 0;
this.diagnosticMaps = [];
}
/**
* @param {*} value
*/
addSample(value) {
this.count += 1;
}
/**
* @param {!tr.v.d.DiagnosticMap} diagnostics
*/
addDiagnosticMap(diagnostics) {
tr.b.Statistics.uniformlySampleStream(
this.diagnosticMaps, this.count, diagnostics, MAX_DIAGNOSTIC_MAPS);
}
addBin(other) {
if (!this.range.equals(other.range))
throw new Error('Merging incompatible Histogram bins.');
tr.b.Statistics.mergeSampledStreams(this.diagnosticMaps, this.count,
other.diagnosticMaps, other.count, MAX_DIAGNOSTIC_MAPS);
this.count += other.count;
}
fromDict(d) {
this.count = d.count;
for (var map of d.diagnosticMaps)
this.diagnosticMaps.push(tr.v.d.DiagnosticMap.fromDict(map));
}
asDict() {
return {
count: this.count,
diagnosticMaps: this.diagnosticMaps.map(d => d.asDict())
};
}
}
/**
* This is basically a histogram, but so much more.
* Histogram is serializable using asDict/fromDict.
* Histogram computes several statistics of its contents.
* Histograms can be merged.
* getDifferenceSignificance() test whether one Histogram is statistically
* significantly different from another Histogram.
* Histogram stores a random sample of the exact number values added to it.
* Histogram stores a random sample of optional per-sample DiagnosticMaps.
* Histogram is visualized by <tr-v-ui-histogram-span>, which supports
* selecting bins, and visualizing the DiagnosticMaps of selected bins.
*
* @param {!tr.b.Unit} unit
* @param {!tr.v.HistogramBinBoundaries=} opt_binBoundaries
*/
class Histogram {
constructor(name, unit, opt_binBoundaries) {
var binBoundaries = opt_binBoundaries;
if (!binBoundaries) {
var baseUnit = unit.baseUnit ? unit.baseUnit : unit;
binBoundaries = DEFAULT_BOUNDARIES_FOR_UNIT.get(baseUnit.unitName);
}
// If this Histogram is being deserialized, then its guid will be set by
// fromDict().
// If this Histogram is being computed by a metric, then its guid will be
// allocated the first time the guid is gotten by asDict().
this.guid_ = undefined;
this.allBins = [];
this.centralBins = [];
this.description = '';
this.diagnostics = new tr.v.d.DiagnosticMap();
this.maxCount_ = 0;
this.name_ = name;
this.nanDiagnosticMaps = [];
this.numNans = 0;
this.running = new tr.b.RunningStatistics();
this.sampleValues_ = [];
this.shortName = undefined;
this.summaryOptions = {
count: true,
sum: true,
avg: true,
geometricMean: false,
std: true,
min: true,
max: true,
nans: false,
percentile: []
};
this.unit = unit;
this.underflowBin = new HistogramBin(tr.b.Range.fromExplicitRange(
-Number.MAX_VALUE, binBoundaries.minBinBoundary));
this.overflowBin = new HistogramBin(tr.b.Range.fromExplicitRange(
binBoundaries.maxBinBoundary, Number.MAX_VALUE));
for (var range of binBoundaries)
this.centralBins.push(new HistogramBin(range));
this.allBins.push(this.underflowBin);
for (var bin of this.centralBins)
this.allBins.push(bin);
this.allBins.push(this.overflowBin);
this.maxNumSampleValues = this.allBins.length * 10;
}
get name() {
return this.name_;
}
get guid() {
if (this.guid_ === undefined)
this.guid_ = tr.b.GUID.allocateUUID4();
return this.guid_;
}
set guid(guid) {
if (this.guid_ !== undefined)
throw new Error('Cannot reset guid');
this.guid_ = guid;
}
static fromDict(d) {
var boundaries = HistogramBinBoundaries.createWithBoundaries(
d.binBoundaries);
var n = new Histogram(d.name, tr.b.Unit.fromJSON(d.unit), boundaries);
n.guid = d.guid;
n.shortName = d.shortName;
n.description = d.description;
n.diagnostics.addDicts(d.diagnostics);
n.underflowBin.fromDict(d.underflowBin);
for (var i = 0; i < d.centralBins.length; ++i)
n.centralBins[i].fromDict(d.centralBins[i]);
n.overflowBin.fromDict(d.overflowBin);
for (var bin of n.allBins)
n.maxCount_ = Math.max(n.maxCount_, bin.count);
if (d.running)
n.running = tr.b.RunningStatistics.fromDict(d.running);
if (d.summaryOptions)
n.customizeSummaryOptions(d.summaryOptions);
n.maxNumSampleValues = d.maxNumSampleValues;
n.sampleValues_ = d.sampleValues;
n.numNans = d.numNans;
for (var map of d.nanDiagnosticMaps)
n.nanDiagnosticMaps.push(tr.v.d.DiagnosticMap.fromDict(map));
return n;
}
/**
* Build a Histogram from a set of samples in order to effectively merge a
* set of ScalarNumerics.
* The range of the resulting histogram is determined by the smallest and
* largest sample value, which is unpredictable.
* https://github.com/catapult-project/catapult/issues/2685
*
* @param {!tr.b.Unit} unit
* @param {!Array.<number>} samples
* @return {!Histogram}
*/
static buildFromSamples(unit, samples) {
var boundaries = HistogramBinBoundaries.createFromSamples(samples);
var result = new Histogram(unit, boundaries);
result.maxNumSampleValues = 1000;
// TODO(eakuefner): Propagate diagnosticMaps?
for (var sample of samples)
result.addSample(sample);
return result;
}
get numValues() {
return tr.b.Statistics.sum(this.allBins, function(e) {
return e.count;
});
}
get average() {
return this.running.mean;
}
get geometricMean() {
return this.running.geometricMean;
}
get sum() {
return this.running.sum;
}
get maxCount() {
return this.maxCount_;
}
/**
* Requires that units agree.
* Returns DONT_CARE if that is the units' improvementDirection.
* Returns SIGNIFICANT if the Mann-Whitney U test returns a
* p-value less than opt_alpha or DEFAULT_ALPHA. Returns INSIGNIFICANT if
* the p-value is greater than alpha.
*
* @param {!tr.v.Histogram} other
* @param {number=} opt_alpha
* @return {!tr.v.Significance}
*/
getDifferenceSignificance(other, opt_alpha) {
if (this.unit !== other.unit)
throw new Error('Cannot compare Numerics with different units');
if (this.unit.improvementDirection ===
tr.b.ImprovementDirection.DONT_CARE) {
return tr.v.Significance.DONT_CARE;
}
if (!(other instanceof Histogram))
throw new Error('Unable to compute a p-value');
var mwu = tr.b.Statistics.mwu.test(this.sampleValues, other.sampleValues);
if (mwu.p < (opt_alpha || DEFAULT_ALPHA))
return tr.v.Significance.SIGNIFICANT;
return tr.v.Significance.INSIGNIFICANT;
}
/*
* Compute an approximation of percentile based on the counts in the bins.
* If the real percentile lies within |this.range| then the result of
* the function will deviate from the real percentile by at most
* the maximum width of the bin(s) within which the point(s)
* from which the real percentile would be calculated lie.
* If the real percentile is outside |this.range| then the function
* returns the closest range limit: |this.range.min| or |this.range.max|.
*
* @param {number} percent The percent must be between 0.0 and 1.0.
*/
getApproximatePercentile(percent) {
if (!(percent >= 0 && percent <= 1))
throw new Error('percent must be [0,1]');
if (this.numValues == 0)
return 0;
var valuesToSkip = Math.floor((this.numValues - 1) * percent);
for (var i = 0; i < this.allBins.length; i++) {
var bin = this.allBins[i];
valuesToSkip -= bin.count;
if (valuesToSkip < 0) {
if (bin === this.underflowBin)
return bin.range.max;
else if (bin === this.overflowBin)
return bin.range.min;
else
return bin.range.center;
}
}
throw new Error('Unreachable');
}
getBinForValue(value) {
// Don't use subtraction to avoid arithmetic overflow.
var binIndex = tr.b.findHighIndexInSortedArray(
this.allBins, b => value < b.range.max ? -1 : 1);
return this.allBins[binIndex] || this.overflowBin;
}
/**
* @param {number|*} value
* @param {(!Object|!tr.v.d.DiagnosticMap)=} opt_diagnostics
*/
addSample(value, opt_diagnostics) {
if (opt_diagnostics &&
!(opt_diagnostics instanceof tr.v.d.DiagnosticMap))
opt_diagnostics = tr.v.d.DiagnosticMap.fromObject(opt_diagnostics);
if (typeof(value) !== 'number' || isNaN(value)) {
this.numNans++;
if (opt_diagnostics) {
tr.b.Statistics.uniformlySampleStream(this.nanDiagnosticMaps,
this.numNans, opt_diagnostics, MAX_DIAGNOSTIC_MAPS);
}
} else {
this.running.add(value);
var bin = this.getBinForValue(value);
bin.addSample(value);
if (opt_diagnostics)
bin.addDiagnosticMap(opt_diagnostics);
if (bin.count > this.maxCount_)
this.maxCount_ = bin.count;
}
tr.b.Statistics.uniformlySampleStream(this.sampleValues_,
this.numValues + this.numNans, value, this.maxNumSampleValues);
}
sampleValuesInto(samples) {
for (var sampleValue of this.sampleValues)
samples.push(sampleValue);
}
/**
* Return true if this Histogram can be added to |other|.
*
* @param {!tr.v.Histogram} other
* @return {boolean}
*/
canAddHistogram(other) {
if (this.unit !== other.unit)
return false;
if (this.allBins.length !== other.allBins.length)
return false;
for (var i = 0; i < this.allBins.length; ++i)
if (!this.allBins[i].range.equals(other.allBins[i].range))
return false;
return true;
}
/**
* Add |other| to this Histogram in-place if they can be added.
*
* @param {!tr.v.Histogram} other
*/
addHistogram(other) {
if (!this.canAddHistogram(other))
throw new Error('Merging incompatible Numerics.');
tr.b.Statistics.mergeSampledStreams(this.nanDiagnosticMaps, this.numNans,
other.nanDiagnosticMaps, other.numNans, MAX_DIAGNOSTIC_MAPS);
tr.b.Statistics.mergeSampledStreams(
this.sampleValues, this.numValues,
other.sampleValues, other.numValues, tr.b.Statistics.mean(
[this.maxNumSampleValues, other.maxNumSampleValues]));
this.numNans += other.numNans;
this.running = this.running.merge(other.running);
for (var i = 0; i < this.allBins.length; ++i) {
this.allBins[i].addBin(other.allBins[i]);
}
}
/**
* Controls which statistics are exported to dashboard for this numeric.
* The |summaryOptions| parameter is a dictionary with optional boolean
* fields |count|, |sum|, |avg|, |std|, |min|, |max| and an optional
* array field |percentile|.
* Each percentile should be a number between 0.0 and 1.0.
* The options not included in the |summaryOptions| will not change.
*/
customizeSummaryOptions(summaryOptions) {
tr.b.iterItems(summaryOptions, function(key, value) {
this.summaryOptions[key] = value;
}, this);
}
/**
* Returns a Map {statisticName: ScalarNumeric}.
*
* Each enabled summary option produces the corresponding value:
* min, max, count, sum, avg, or std.
* Each percentile 0.x produces pct_0x0.
* Each percentile 0.xx produces pct_0xx.
* Each percentile 0.xxy produces pct_0xx_y.
* Percentile 1.0 produces pct_100.
*
* @return {!Map.<string, ScalarNumeric>}
*/
get statisticsScalars() {
function statNameToKey(stat) {
switch (stat) {
case 'std':
return 'stddev';
case 'avg':
return 'mean';
}
return stat;
}
/**
* Converts the given percent to a string in the format specified above.
* @param {number} percent The percent must be between 0.0 and 1.0.
*/
function percentToString(percent) {
if (percent < 0 || percent > 1)
throw new Error('Percent must be between 0.0 and 1.0');
switch (percent) {
case 0:
return '000';
case 1:
return '100';
}
var str = percent.toString();
if (str[1] !== '.')
throw new Error('Unexpected percent');
// Pad short strings with zeros.
str = str + '0'.repeat(Math.max(4 - str.length, 0));
if (str.length > 4)
str = str.slice(0, 4) + '_' + str.slice(4);
return '0' + str.slice(2);
}
var results = new Map();
tr.b.iterItems(this.summaryOptions, function(stat, option) {
if (!option)
return;
if (stat === 'percentile') {
option.forEach(function(percent) {
var percentile = this.getApproximatePercentile(percent);
results.set('pct_' + percentToString(percent),
new tr.v.ScalarNumeric(this.unit, percentile));
}, this);
} else if (stat === 'nans') {
results.set('nans', new tr.v.ScalarNumeric(
tr.b.Unit.byName.count_smallerIsBetter, this.numNans));
} else {
var statUnit = stat === 'count' ?
tr.b.Unit.byName.count_smallerIsBetter : this.unit;
var key = statNameToKey(stat);
var statValue = this.running[key];
if (typeof(statValue) === 'number') {
results.set(stat, new tr.v.ScalarNumeric(statUnit, statValue));
}
}
}, this);
return results;
}
get sampleValues() {
return this.sampleValues_;
}
get binBoundaries() {
var boundaries = [];
for (var bin of this.centralBins)
boundaries.push(bin.range.min);
boundaries.push(this.overflowBin.range.min);
return boundaries;
}
clone() {
return Histogram.fromDict(this.asDict());
}
asDict() {
return {
name: this.name,
guid: this.guid,
shortName: this.shortName,
description: this.description,
diagnostics: this.diagnostics.asDict(),
unit: this.unit.asJSON(),
binBoundaries: this.binBoundaries,
underflowBin: this.underflowBin.asDict(),
centralBins: this.centralBins.map(bin => bin.asDict()),
overflowBin: this.overflowBin.asDict(),
running: this.running.asDict(),
summaryOptions: this.summaryOptions,
maxNumSampleValues: this.maxNumSampleValues,
sampleValues: this.sampleValues,
numNans: this.numNans,
nanDiagnosticMaps: this.nanDiagnosticMaps.map(dm => dm.asDict()),
};
}
}
/**
* Reusable builder for tr.v.Histogram objects.
*
* The bins of the numeric are specified by adding the desired boundaries
* between bins. Initially, the builder has only a single boundary:
*
* minBinBoundary=maxBinBoundary
* |
* |
* -MAX_INT <--------|------------------------------------------> +MAX_INT
* : resulting : resulting :
* : underflow : overflow :
* : bin : bin :
*
* More boundaries can be added (in increasing order) using addBinBoundary,
* addLinearBins and addExponentialBins:
*
* minBinBoundary maxBinBoundary
* | | | | |
* | | | | |
* -MAX_INT <--------|---------|---------|-----|---------|------> +MAX_INT
* : resulting : result. : result. : : result. : resulting :
* : underflow : central : central : ... : central : overflow :
* : bin : bin 0 : bin 1 : : bin N-1 : bin :
*
* An important feature of the builder is that it's reusable, i.e. it can be
* used to build multiple numerics with the same unit and bin structure.
*
* @constructor
* @param {!tr.b.Unit} unit Unit of the resulting Histogram(s).
* @param {number} minBinBoundary The minimum boundary between bins, namely
* the underflow bin and the first central bin (or the overflow bin if
* no other boundaries are added later).
*/
class HistogramBinBoundaries {
/**
* Create a linearly scaled tr.v.HistogramBinBoundaries with |numBins| bins
* ranging from |min| to |max|.
*
* @param {number} min
* @param {number} max
* @param {number} numBins
* @return {tr.v.HistogramBinBoundaries}
*/
static createLinear(min, max, numBins) {
return new HistogramBinBoundaries(min).addLinearBins(max, numBins);
}
/**
* Create an exponentially scaled tr.v.HistogramBinBoundaries with |numBins|
* bins ranging from |min| to |max|.
*
* @param {number} min
* @param {number} max
* @param {number} numBins
* @return {tr.v.HistogramBinBoundaries}
*/
static createExponential(min, max, numBins) {
return new HistogramBinBoundaries(min).addExponentialBins(max, numBins);
}
/**
* @param {Array.<number>} binBoundaries
*/
static createWithBoundaries(binBoundaries) {
var builder = new HistogramBinBoundaries(binBoundaries[0]);
for (var boundary of binBoundaries.slice(1))
builder.addBinBoundary(boundary);
return builder;
}
static createFromSamples(samples) {
var range = new tr.b.Range();
// Prevent non-numeric samples from introducing NaNs into the range.
for (var sample of samples)
if (!isNaN(Math.max(sample)))
range.addValue(sample);
// HistogramBinBoundaries.addLinearBins() requires this.
if (range.isEmpty)
range.addValue(1);
if (range.min === range.max)
range.addValue(range.min - 1);
// This optimizes the resolution when samples are uniformly distributed
// (which is almost never the case).
var numBins = Math.ceil(Math.sqrt(samples.length));
var builder = new HistogramBinBoundaries(range.min);
builder.addLinearBins(range.max, numBins);
return builder;
}
/**
* @param {number} minBinBoundary
*/
constructor(minBinBoundary) {
this.boundaries_ = [minBinBoundary];
}
get minBinBoundary() {
return this.boundaries_[0];
}
get maxBinBoundary() {
return this.boundaries_[this.boundaries_.length - 1];
}
/**
* Yield Ranges of adjacent boundaries.
*/
*[Symbol.iterator]() {
for (var i = 0; i < this.boundaries_.length - 1; ++i) {
yield tr.b.Range.fromExplicitRange(
this.boundaries_[i], this.boundaries_[i + 1]);
}
}
/**
* Add a bin boundary |nextMaxBinBoundary| to the builder.
*
* This operation effectively corresponds to appending a new central bin
* with the range [this.maxBinBoundary*, nextMaxBinBoundary].
*
* @param {number} nextMaxBinBoundary The added bin boundary (must be
* greater than |this.maxMinBoundary|).
*/
addBinBoundary(nextMaxBinBoundary) {
if (nextMaxBinBoundary <= this.maxBinBoundary) {
throw new Error('The added max bin boundary must be larger than ' +
'the current max boundary');
}
this.boundaries_.push(nextMaxBinBoundary);
return this;
}
/**
* Add |binCount| linearly scaled bin boundaries up to |nextMaxBinBoundary|
* to the builder.
*
* This operation corresponds to appending |binCount| central bins of
* constant range width
* W = ((|nextMaxBinBoundary| - |this.maxBinBoundary|) / |binCount|)
* with the following ranges:
*
* [|this.maxMinBoundary|, |this.maxMinBoundary| + W]
* [|this.maxMinBoundary| + W, |this.maxMinBoundary| + 2W]
* [|this.maxMinBoundary| + 2W, |this.maxMinBoundary| + 3W]
* ...
* [|this.maxMinBoundary| + (|binCount| - 2) * W,
* |this.maxMinBoundary| + (|binCount| - 2) * W]
* [|this.maxMinBoundary| + (|binCount| - 1) * W,
* |nextMaxBinBoundary|]
*
* @param {number} nextBinBoundary The last added bin boundary (must be
* greater than |this.maxMinBoundary|).
* @param {number} binCount Number of bins to be added (must be positive).
*/
addLinearBins(nextMaxBinBoundary, binCount) {
if (binCount <= 0)
throw new Error('Bin count must be positive');
var curMaxBinBoundary = this.maxBinBoundary;
if (curMaxBinBoundary >= nextMaxBinBoundary) {
throw new Error('The new max bin boundary must be greater than ' +
'the previous max bin boundary');
}
var binWidth = (nextMaxBinBoundary - curMaxBinBoundary) / binCount;
for (var i = 1; i < binCount; i++)
this.addBinBoundary(curMaxBinBoundary + i * binWidth);
this.addBinBoundary(nextMaxBinBoundary);
return this;
}
/**
* Add |binCount| exponentially scaled bin boundaries up to
* |nextMaxBinBoundary| to the builder.
*
* This operation corresponds to appending |binCount| central bins with
* a constant difference between the logarithms of their range min and max
* D = ((ln(|nextMaxBinBoundary|) - ln(|this.maxBinBoundary|)) / |binCount|)
* with the following ranges:
*
* [|this.maxMinBoundary|, |this.maxMinBoundary| * exp(D)]
* [|this.maxMinBoundary| * exp(D), |this.maxMinBoundary| * exp(2D)]
* [|this.maxMinBoundary| * exp(2D), |this.maxMinBoundary| * exp(3D)]
* ...
* [|this.maxMinBoundary| * exp((|binCount| - 2) * D),
* |this.maxMinBoundary| * exp((|binCount| - 2) * D)]
* [|this.maxMinBoundary| * exp((|binCount| - 1) * D),
* |nextMaxBinBoundary|]
*
* This method requires that the current max bin boundary is positive.
*
* @param {number} nextBinBoundary The last added bin boundary (must be
* greater than |this.maxMinBoundary|).
* @param {number} binCount Number of bins to be added (must be positive).
*/
addExponentialBins(nextMaxBinBoundary, binCount) {
if (binCount <= 0)
throw new Error('Bin count must be positive');
var curMaxBinBoundary = this.maxBinBoundary;
if (curMaxBinBoundary <= 0)
throw new Error('Current max bin boundary must be positive');
if (curMaxBinBoundary >= nextMaxBinBoundary) {
throw new Error('The last added max boundary must be greater than ' +
'the current max boundary boundary');
}
var binExponentWidth =
Math.log(nextMaxBinBoundary / curMaxBinBoundary) / binCount;
for (var i = 1; i < binCount; i++) {
this.addBinBoundary(
curMaxBinBoundary * Math.exp(i * binExponentWidth));
}
this.addBinBoundary(nextMaxBinBoundary);
return this;
}
}
DEFAULT_BOUNDARIES_FOR_UNIT.set(
tr.b.Unit.byName.timeDurationInMs.unitName,
HistogramBinBoundaries.createExponential(1e-3, 1e6, 1e2));
DEFAULT_BOUNDARIES_FOR_UNIT.set(
tr.b.Unit.byName.timeStampInMs.unitName,
HistogramBinBoundaries.createLinear(0, 1e10, 1e3));
DEFAULT_BOUNDARIES_FOR_UNIT.set(
tr.b.Unit.byName.normalizedPercentage.unitName,
HistogramBinBoundaries.createLinear(0, 1.0, 20));
DEFAULT_BOUNDARIES_FOR_UNIT.set(
tr.b.Unit.byName.sizeInBytes.unitName,
HistogramBinBoundaries.createExponential(1, 1e12, 1e2));
DEFAULT_BOUNDARIES_FOR_UNIT.set(
tr.b.Unit.byName.energyInJoules.unitName,
HistogramBinBoundaries.createExponential(1e-3, 1e3, 50));
DEFAULT_BOUNDARIES_FOR_UNIT.set(
tr.b.Unit.byName.powerInWatts.unitName,
HistogramBinBoundaries.createExponential(1e-3, 1, 50));
DEFAULT_BOUNDARIES_FOR_UNIT.set(
tr.b.Unit.byName.unitlessNumber.unitName,
HistogramBinBoundaries.createExponential(1e-3, 1e3, 50));
DEFAULT_BOUNDARIES_FOR_UNIT.set(
tr.b.Unit.byName.count.unitName,
HistogramBinBoundaries.createExponential(1, 1e3, 20));
return {
Significance: Significance,
Histogram: Histogram,
HistogramBinBoundaries: HistogramBinBoundaries,
};
});
</script>