Merge "Move regression classes to contrib"
diff --git a/res/config/regression.xml b/res/config/regression.xml
new file mode 100644
index 0000000..6b27344
--- /dev/null
+++ b/res/config/regression.xml
@@ -0,0 +1,24 @@
+<?xml version="1.0" encoding="utf-8"?>
+<!-- Copyright (C) 2018 The Android Open Source Project
+
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<configuration description="Runs a regression detection algorithm on two sets of metrics">
+
+ <option name="log-level" value="verbose" />
+ <option name="compress-files" value="false" />
+ <test class="com.android.regression.tests.DetectRegression" />
+ <logger class="com.android.tradefed.log.FileLogger" />
+ <result_reporter class="com.android.tradefed.result.ConsoleResultReporter" />
+
+</configuration>
diff --git a/src/com/android/regression/tests/DetectRegression.java b/src/com/android/regression/tests/DetectRegression.java
new file mode 100644
index 0000000..fee6a73
--- /dev/null
+++ b/src/com/android/regression/tests/DetectRegression.java
@@ -0,0 +1,286 @@
+/*
+ * Copyright (C) 2018 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.android.regression.tests;
+
+import com.android.ddmlib.Log;
+import com.android.regression.tests.MetricsXmlParser.ParseException;
+import com.android.tradefed.config.Option;
+import com.android.tradefed.config.OptionClass;
+import com.android.tradefed.log.LogUtil.CLog;
+import com.android.tradefed.result.ITestInvocationListener;
+import com.android.tradefed.result.TestDescription;
+import com.android.tradefed.testtype.IRemoteTest;
+import com.android.tradefed.testtype.suite.ModuleDefinition;
+import com.android.tradefed.util.FileUtil;
+import com.android.tradefed.util.MultiMap;
+import com.android.tradefed.util.Pair;
+import com.android.tradefed.util.TableBuilder;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.collect.ImmutableSet;
+import com.google.common.collect.Sets;
+import com.google.common.primitives.Doubles;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Random;
+import java.util.Set;
+import java.util.stream.Collectors;
+
+/** An algorithm to detect local metrics regression. */
+@OptionClass(alias = "regression")
+public class DetectRegression implements IRemoteTest {
+
+ @Option(
+ name = "pre-patch-metrics",
+ description = "Path to pre-patch metrics folder.",
+ mandatory = true
+ )
+ private File mPrePatchFolder;
+
+ @Option(
+ name = "post-patch-metrics",
+ description = "Path to post-patch metrics folder.",
+ mandatory = true
+ )
+ private File mPostPatchFolder;
+
+ @Option(
+ name = "strict-mode",
+ description = "When before/after metrics mismatch, true=throw exception, false=log error"
+ )
+ private boolean mStrict = false;
+
+ @Option(name = "blacklist-metrics", description = "Ignore metrics that match these names")
+ private Set<String> mBlacklistMetrics = new HashSet<>();
+
+ private static final String TITLE = "Metric Regressions";
+ private static final String PROLOG =
+ "\n====================Metrics Comparison Results====================\nTest Summary\n";
+ private static final String EPILOG =
+ "==================End Metrics Comparison Results==================\n";
+ private static final String[] TABLE_HEADER = {
+ "Metric Name", "Pre Avg", "Post Avg", "False Positive Probability"
+ };
+ /** Matches metrics xml filenames. */
+ private static final String METRICS_PATTERN = "metrics-.*\\.xml";
+
+ private static final int SAMPLES = 100000;
+ private static final double STD_DEV_THRESHOLD = 2.0;
+
+ private static final Set<String> DEFAULT_IGNORE =
+ ImmutableSet.of(
+ ModuleDefinition.PREPARATION_TIME,
+ ModuleDefinition.TEST_TIME,
+ ModuleDefinition.TEAR_DOWN_TIME);
+
+ @VisibleForTesting
+ public static class TableRow {
+ String name;
+ double preAvg;
+ double postAvg;
+ double probability;
+
+ public String[] toStringArray() {
+ return new String[] {
+ name,
+ String.format("%.2f", preAvg),
+ String.format("%.2f", postAvg),
+ String.format("%.3f", probability)
+ };
+ }
+ }
+
+ public DetectRegression() {
+ mBlacklistMetrics.addAll(DEFAULT_IGNORE);
+ }
+
+ @Override
+ public void run(ITestInvocationListener listener) {
+ try {
+ // Load metrics from files, and validate them.
+ Metrics before =
+ MetricsXmlParser.parse(
+ mBlacklistMetrics, mStrict, getMetricsFiles(mPrePatchFolder));
+ Metrics after =
+ MetricsXmlParser.parse(
+ mBlacklistMetrics, mStrict, getMetricsFiles(mPostPatchFolder));
+ before.crossValidate(after);
+ runRegressionDetection(before, after);
+ } catch (IOException | ParseException e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ /**
+ * Computes metrics regression between pre-patch and post-patch.
+ *
+ * @param before pre-patch metrics
+ * @param after post-patch metrics
+ */
+ @VisibleForTesting
+ void runRegressionDetection(Metrics before, Metrics after) {
+ Set<String> runMetricsToCompare =
+ Sets.intersection(before.getRunMetrics().keySet(), after.getRunMetrics().keySet());
+ List<TableRow> runMetricsResult = new ArrayList<>();
+ for (String name : runMetricsToCompare) {
+ List<Double> beforeMetrics = before.getRunMetrics().get(name);
+ List<Double> afterMetrics = after.getRunMetrics().get(name);
+ if (computeRegression(beforeMetrics, afterMetrics)) {
+ runMetricsResult.add(getTableRow(name, beforeMetrics, afterMetrics));
+ }
+ }
+
+ Set<Pair<TestDescription, String>> testMetricsToCompare =
+ Sets.intersection(
+ before.getTestMetrics().keySet(), after.getTestMetrics().keySet());
+ MultiMap<String, TableRow> testMetricsResult = new MultiMap<>();
+ for (Pair<TestDescription, String> id : testMetricsToCompare) {
+ List<Double> beforeMetrics = before.getTestMetrics().get(id);
+ List<Double> afterMetrics = after.getTestMetrics().get(id);
+ if (computeRegression(beforeMetrics, afterMetrics)) {
+ testMetricsResult.put(
+ id.first.toString(), getTableRow(id.second, beforeMetrics, afterMetrics));
+ }
+ }
+ logResult(before, after, runMetricsResult, testMetricsResult);
+ }
+
+ /** Prints results to the console. */
+ @VisibleForTesting
+ void logResult(
+ Metrics before,
+ Metrics after,
+ List<TableRow> runMetricsResult,
+ MultiMap<String, TableRow> testMetricsResult) {
+ TableBuilder table = new TableBuilder(TABLE_HEADER.length);
+ table.addTitle(TITLE).addLine(TABLE_HEADER).addDoubleLineSeparator();
+
+ int totalRunMetrics =
+ Sets.intersection(before.getRunMetrics().keySet(), after.getRunMetrics().keySet())
+ .size();
+ String runResult =
+ String.format(
+ "Run Metrics (%d compared, %d changed)",
+ totalRunMetrics, runMetricsResult.size());
+ table.addLine(runResult).addSingleLineSeparator();
+ runMetricsResult.stream().map(TableRow::toStringArray).forEach(table::addLine);
+ if (!runMetricsResult.isEmpty()) {
+ table.addSingleLineSeparator();
+ }
+
+ int totalTestMetrics =
+ Sets.intersection(before.getTestMetrics().keySet(), after.getTestMetrics().keySet())
+ .size();
+ int changedTestMetrics =
+ testMetricsResult
+ .keySet()
+ .stream()
+ .mapToInt(k -> testMetricsResult.get(k).size())
+ .sum();
+ String testResult =
+ String.format(
+ "Test Metrics (%d compared, %d changed)",
+ totalTestMetrics, changedTestMetrics);
+ table.addLine(testResult).addSingleLineSeparator();
+ for (String test : testMetricsResult.keySet()) {
+ table.addLine("> " + test);
+ testMetricsResult
+ .get(test)
+ .stream()
+ .map(TableRow::toStringArray)
+ .forEach(table::addLine);
+ table.addBlankLineSeparator();
+ }
+ table.addDoubleLineSeparator();
+
+ StringBuilder sb = new StringBuilder(PROLOG);
+ sb.append(
+ String.format(
+ "%d tests. %d sets of pre-patch metrics. %d sets of post-patch metrics.\n\n",
+ before.getNumTests(), before.getNumRuns(), after.getNumRuns()));
+ sb.append(table.build()).append('\n').append(EPILOG);
+
+ CLog.logAndDisplay(Log.LogLevel.INFO, sb.toString());
+ }
+
+ private List<File> getMetricsFiles(File folder) throws IOException {
+ CLog.i("Loading metrics from: %s", mPrePatchFolder.getAbsolutePath());
+ return FileUtil.findFiles(folder, METRICS_PATTERN)
+ .stream()
+ .map(File::new)
+ .collect(Collectors.toList());
+ }
+
+ private static TableRow getTableRow(String name, List<Double> before, List<Double> after) {
+ TableRow row = new TableRow();
+ row.name = name;
+ row.preAvg = calcMean(before);
+ row.postAvg = calcMean(after);
+ row.probability = probFalsePositive(before.size(), after.size());
+ return row;
+ }
+
+ /** @return true if there is regression from before to after, false otherwise */
+ @VisibleForTesting
+ static boolean computeRegression(List<Double> before, List<Double> after) {
+ final double mean = calcMean(before);
+ final double stdDev = calcStdDev(before);
+ int regCount = 0;
+ for (double value : after) {
+ if (Math.abs(value - mean) > stdDev * STD_DEV_THRESHOLD) {
+ regCount++;
+ }
+ }
+ return regCount > after.size() / 2;
+ }
+
+ @VisibleForTesting
+ static double calcMean(List<Double> list) {
+ return list.stream().collect(Collectors.averagingDouble(x -> x));
+ }
+
+ @VisibleForTesting
+ static double calcStdDev(List<Double> list) {
+ final double mean = calcMean(list);
+ return Math.sqrt(
+ list.stream().collect(Collectors.averagingDouble(x -> Math.pow(x - mean, 2))));
+ }
+
+ private static double probFalsePositive(int priorRuns, int postRuns) {
+ int failures = 0;
+ Random rand = new Random();
+ for (int run = 0; run < SAMPLES; run++) {
+ double[] prior = new double[priorRuns];
+ for (int x = 0; x < priorRuns; x++) {
+ prior[x] = rand.nextGaussian();
+ }
+ double estMu = calcMean(Doubles.asList(prior));
+ double estStd = calcStdDev(Doubles.asList(prior));
+ int count = 0;
+ for (int y = 0; y < postRuns; y++) {
+ if (Math.abs(rand.nextGaussian() - estMu) > estStd * STD_DEV_THRESHOLD) {
+ count++;
+ }
+ }
+ failures += count > postRuns / 2 ? 1 : 0;
+ }
+ return (double) failures / SAMPLES;
+ }
+}
diff --git a/src/com/android/regression/tests/Metrics.java b/src/com/android/regression/tests/Metrics.java
new file mode 100644
index 0000000..ce78b52
--- /dev/null
+++ b/src/com/android/regression/tests/Metrics.java
@@ -0,0 +1,229 @@
+/*
+ * Copyright (C) 2018 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.android.regression.tests;
+
+import com.android.tradefed.log.LogUtil.CLog;
+import com.android.tradefed.result.TestDescription;
+import com.android.tradefed.util.MultiMap;
+import com.android.tradefed.util.Pair;
+
+import com.google.common.annotations.VisibleForTesting;
+
+/** A metrics object to hold run metrics and test metrics parsed by {@link MetricsXmlParser} */
+public class Metrics {
+ private int mNumRuns;
+ private int mNumTests = -1;
+ private final boolean mStrictMode;
+ private final MultiMap<String, Double> mRunMetrics = new MultiMap<>();
+ private final MultiMap<Pair<TestDescription, String>, Double> mTestMetrics = new MultiMap<>();
+
+ /** Throw when metrics validation fails in strict mode. */
+ public static class MetricsException extends RuntimeException {
+ MetricsException(String cause) {
+ super(cause);
+ }
+ }
+
+ /**
+ * Constructs an empty Metrics object.
+ *
+ * @param strictMode whether exception should be thrown when validation fails
+ */
+ public Metrics(boolean strictMode) {
+ mStrictMode = strictMode;
+ }
+
+ /**
+ * Sets the number of tests. This method also checks if each call sets the same number of test,
+ * since this number should be consistent across multiple runs.
+ *
+ * @param numTests the number of tests
+ * @throws MetricsException if subsequent calls set a different number.
+ */
+ public void setNumTests(int numTests) {
+ if (mNumTests == -1) {
+ mNumTests = numTests;
+ } else {
+ if (mNumTests != numTests) {
+ String msg =
+ String.format(
+ "Number of test entries differ: expect #%d actual #%d",
+ mNumTests, numTests);
+ throw new MetricsException(msg);
+ }
+ }
+ }
+
+ /**
+ * Adds a run metric.
+ *
+ * @param name metric name
+ * @param value metric value
+ */
+ public void addRunMetric(String name, String value) {
+ try {
+ mRunMetrics.put(name, Double.parseDouble(value));
+ } catch (NumberFormatException e) {
+ // This is normal. We often get some string metrics like device name. Just log it.
+ CLog.w(String.format("Run metric \"%s\" is not a number: \"%s\"", name, value));
+ }
+ }
+
+ /**
+ * Adds a test metric.
+ *
+ * @param testId TestDescription of the metric
+ * @param name metric name
+ * @param value metric value
+ */
+ public void addTestMetric(TestDescription testId, String name, String value) {
+ Pair<TestDescription, String> metricId = new Pair<>(testId, name);
+ try {
+ mTestMetrics.put(metricId, Double.parseDouble(value));
+ } catch (NumberFormatException e) {
+ // This is normal. We often get some string metrics like device name. Just log it.
+ CLog.w(
+ String.format(
+ "Test %s metric \"%s\" is not a number: \"%s\"", testId, name, value));
+ }
+ }
+
+ /**
+ * Validates that the number of entries of each metric equals to the number of runs.
+ *
+ * @param numRuns number of runs
+ * @throws MetricsException when validation fails in strict mode
+ */
+ public void validate(int numRuns) {
+ mNumRuns = numRuns;
+ for (String name : mRunMetrics.keySet()) {
+ if (mRunMetrics.get(name).size() < mNumRuns) {
+ error(
+ String.format(
+ "Run metric \"%s\" too few entries: expected #%d actual #%d",
+ name, mNumRuns, mRunMetrics.get(name).size()));
+ }
+ }
+ for (Pair<TestDescription, String> id : mTestMetrics.keySet()) {
+ if (mTestMetrics.get(id).size() < mNumRuns) {
+ error(
+ String.format(
+ "Test %s metric \"%s\" too few entries: expected #%d actual #%d",
+ id.first, id.second, mNumRuns, mTestMetrics.get(id).size()));
+ }
+ }
+ }
+
+ /**
+ * Validates with after-patch Metrics object. Make sure two metrics object contain same run
+ * metric entries and test metric entries. Assume this object contains before-patch metrics.
+ *
+ * @param after a Metrics object containing after-patch metrics
+ * @throws MetricsException when cross validation fails in strict mode
+ */
+ public void crossValidate(Metrics after) {
+ if (mNumTests != after.mNumTests) {
+ error(
+ String.format(
+ "Number of test entries differ: before #%d after #%d",
+ mNumTests, after.mNumTests));
+ }
+
+ for (String name : mRunMetrics.keySet()) {
+ if (!after.mRunMetrics.containsKey(name)) {
+ warn(String.format("Run metric \"%s\" only in before-patch run.", name));
+ }
+ }
+
+ for (String name : after.mRunMetrics.keySet()) {
+ if (!mRunMetrics.containsKey(name)) {
+ warn(String.format("Run metric \"%s\" only in after-patch run.", name));
+ }
+ }
+
+ for (Pair<TestDescription, String> id : mTestMetrics.keySet()) {
+ if (!after.mTestMetrics.containsKey(id)) {
+ warn(
+ String.format(
+ "Test %s metric \"%s\" only in before-patch run.",
+ id.first, id.second));
+ }
+ }
+
+ for (Pair<TestDescription, String> id : after.mTestMetrics.keySet()) {
+ if (!mTestMetrics.containsKey(id)) {
+ warn(
+ String.format(
+ "Test %s metric \"%s\" only in after-patch run.",
+ id.first, id.second));
+ }
+ }
+ }
+
+ @VisibleForTesting
+ void error(String msg) {
+ if (mStrictMode) {
+ throw new MetricsException(msg);
+ } else {
+ CLog.e(msg);
+ }
+ }
+
+ @VisibleForTesting
+ void warn(String msg) {
+ if (mStrictMode) {
+ throw new MetricsException(msg);
+ } else {
+ CLog.w(msg);
+ }
+ }
+
+ /**
+ * Gets the number of test runs stored in this object.
+ *
+ * @return number of test runs
+ */
+ public int getNumRuns() {
+ return mNumRuns;
+ }
+
+ /**
+ * Gets the number of tests stored in this object.
+ *
+ * @return number of tests
+ */
+ public int getNumTests() {
+ return mNumTests;
+ }
+
+ /**
+ * Gets all run metrics stored in this object.
+ *
+ * @return a {@link MultiMap} from test name String to Double
+ */
+ public MultiMap<String, Double> getRunMetrics() {
+ return mRunMetrics;
+ }
+
+ /**
+ * Gets all test metrics stored in this object.
+ *
+ * @return a {@link MultiMap} from (TestDescription, test name) pair to Double
+ */
+ public MultiMap<Pair<TestDescription, String>, Double> getTestMetrics() {
+ return mTestMetrics;
+ }
+}
diff --git a/src/com/android/regression/tests/MetricsXmlParser.java b/src/com/android/regression/tests/MetricsXmlParser.java
new file mode 100644
index 0000000..b78a982
--- /dev/null
+++ b/src/com/android/regression/tests/MetricsXmlParser.java
@@ -0,0 +1,160 @@
+/*
+ * Copyright (C) 2018 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.android.regression.tests;
+
+import com.android.tradefed.result.MetricsXMLResultReporter;
+import com.android.tradefed.result.TestDescription;
+
+import com.google.common.annotations.VisibleForTesting;
+
+import org.xml.sax.Attributes;
+import org.xml.sax.SAXException;
+import org.xml.sax.helpers.DefaultHandler;
+
+import java.io.BufferedInputStream;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.util.List;
+import java.util.Set;
+
+import javax.xml.parsers.ParserConfigurationException;
+import javax.xml.parsers.SAXParser;
+import javax.xml.parsers.SAXParserFactory;
+
+/** Parser that extracts test metrics result data generated by {@link MetricsXMLResultReporter}. */
+public class MetricsXmlParser {
+
+ /** Thrown when MetricsXmlParser fails to parse a metrics xml file. */
+ public static class ParseException extends Exception {
+ public ParseException(Throwable cause) {
+ super(cause);
+ }
+
+ public ParseException(String msg, Throwable cause) {
+ super(msg, cause);
+ }
+ }
+
+ /*
+ * Parses the xml format. Expected tags/attributes are:
+ * testsuite name="runname" tests="X"
+ * runmetric name="metric1" value="1.0"
+ * testcase classname="FooTest" testname="testMethodName"
+ * testmetric name="metric2" value="1.0"
+ */
+ private static class MetricsXmlHandler extends DefaultHandler {
+
+ private static final String TESTSUITE_TAG = "testsuite";
+ private static final String TESTCASE_TAG = "testcase";
+ private static final String TIME_TAG = "time";
+ private static final String RUNMETRIC_TAG = "runmetric";
+ private static final String TESTMETRIC_TAG = "testmetric";
+
+ private TestDescription mCurrentTest = null;
+
+ private Metrics mMetrics;
+ private Set<String> mBlacklistMetrics;
+
+ public MetricsXmlHandler(Metrics metrics, Set<String> blacklistMetrics) {
+ mMetrics = metrics;
+ mBlacklistMetrics = blacklistMetrics;
+ }
+
+ @Override
+ public void startElement(String uri, String localName, String name, Attributes attributes)
+ throws SAXException {
+ if (TESTSUITE_TAG.equalsIgnoreCase(name)) {
+ // top level tag - maps to a test run in TF terminology
+ String testCount = getMandatoryAttribute(name, "tests", attributes);
+ mMetrics.setNumTests(Integer.parseInt(testCount));
+ mMetrics.addRunMetric(TIME_TAG, getMandatoryAttribute(name, TIME_TAG, attributes));
+ }
+ if (TESTCASE_TAG.equalsIgnoreCase(name)) {
+ // start of description of an individual test method
+ String testClassName = getMandatoryAttribute(name, "classname", attributes);
+ String methodName = getMandatoryAttribute(name, "testname", attributes);
+ mCurrentTest = new TestDescription(testClassName, methodName);
+ }
+ if (RUNMETRIC_TAG.equalsIgnoreCase(name)) {
+ String metricName = getMandatoryAttribute(name, "name", attributes);
+ String metricValue = getMandatoryAttribute(name, "value", attributes);
+ if (!mBlacklistMetrics.contains(metricName)) {
+ mMetrics.addRunMetric(metricName, metricValue);
+ }
+ }
+ if (TESTMETRIC_TAG.equalsIgnoreCase(name)) {
+ String metricName = getMandatoryAttribute(name, "name", attributes);
+ String metricValue = getMandatoryAttribute(name, "value", attributes);
+ if (!mBlacklistMetrics.contains(metricName)) {
+ mMetrics.addTestMetric(mCurrentTest, metricName, metricValue);
+ }
+ }
+ }
+
+ private String getMandatoryAttribute(String tagName, String attrName, Attributes attributes)
+ throws SAXException {
+ String value = attributes.getValue(attrName);
+ if (value == null) {
+ throw new SAXException(
+ String.format(
+ "Malformed XML, could not find '%s' attribute in '%s'",
+ attrName, tagName));
+ }
+ return value;
+ }
+ }
+
+ /**
+ * Parses xml data contained in given input files.
+ *
+ * @param blacklistMetrics ignore the metrics with these names
+ * @param strictMode whether to throw an exception when metric validation fails
+ * @param metricXmlFiles a list of metric xml files
+ * @return a Metric object containing metrics from all metric files
+ * @throws ParseException if input could not be parsed
+ */
+ public static Metrics parse(
+ Set<String> blacklistMetrics, boolean strictMode, List<File> metricXmlFiles)
+ throws ParseException {
+ Metrics metrics = new Metrics(strictMode);
+ for (File xml : metricXmlFiles) {
+ try (InputStream is = new BufferedInputStream(new FileInputStream(xml))) {
+ parse(metrics, blacklistMetrics, is);
+ } catch (Exception e) {
+ throw new ParseException("Unable to parse " + xml.getPath(), e);
+ }
+ }
+ metrics.validate(metricXmlFiles.size());
+ return metrics;
+ }
+
+ @VisibleForTesting
+ public static Metrics parse(Metrics metrics, Set<String> blacklistMetrics, InputStream is)
+ throws ParseException {
+ try {
+ SAXParserFactory parserFactory = SAXParserFactory.newInstance();
+ parserFactory.setNamespaceAware(true);
+ SAXParser parser = parserFactory.newSAXParser();
+ parser.parse(is, new MetricsXmlHandler(metrics, blacklistMetrics));
+ return metrics;
+ } catch (ParserConfigurationException | SAXException | IOException e) {
+ throw new ParseException(e);
+ }
+ }
+}
diff --git a/tests/.classpath b/tests/.classpath
index 328230f..3db2361 100644
--- a/tests/.classpath
+++ b/tests/.classpath
@@ -1,11 +1,15 @@
<?xml version="1.0" encoding="UTF-8"?>
<classpath>
<classpathentry kind="src" path="src"/>
+ <classpathentry kind="src" path="objenesis"/>
+ <classpathentry kind="src" path="mockito"/>
<classpathentry kind="src" path="easymock"/>
<classpathentry kind="con" path="org.eclipse.jdt.launching.JRE_CONTAINER"/>
<classpathentry combineaccessrules="false" kind="src" path="/tradefederation"/>
<classpathentry combineaccessrules="false" kind="src" path="/ddmlib"/>
<classpathentry kind="var" path="TRADEFED_ROOT/out/host/common/obj/JAVA_LIBRARIES/host-libprotobuf-java-full_intermediates/classes.jar"/>
<classpathentry combineaccessrules="false" kind="src" path="/tradefederation-contrib"/>
+ <classpathentry kind="var" path="TRADEFED_ROOT/out/soong/.intermediates/external/mockito/mockito-byte-buddy-agent/linux_glibc_common/combined/mockito-byte-buddy-agent.jar"/>
+ <classpathentry kind="var" path="TRADEFED_ROOT/out/soong/.intermediates/external/mockito/mockito-byte-buddy/linux_glibc_common/combined/mockito-byte-buddy.jar"/>
<classpathentry kind="output" path="bin"/>
</classpath>
diff --git a/tests/.project b/tests/.project
index 1d1f462..93dc9ae 100644
--- a/tests/.project
+++ b/tests/.project
@@ -18,7 +18,17 @@
<link>
<name>easymock</name>
<type>2</type>
- <location>TRADEFED_ROOT/external/easymock/src</location>
+ <locationURI>TRADEFED_ROOT/external/easymock/src</locationURI>
+ </link>
+ <link>
+ <name>mockito</name>
+ <type>2</type>
+ <locationURI>TRADEFED_ROOT/external/mockito/src/main/java</locationURI>
+ </link>
+ <link>
+ <name>objenesis</name>
+ <type>2</type>
+ <locationURI>TRADEFED_ROOT/external/objenesis/main/src/main/java</locationURI>
</link>
</linkedResources>
</projectDescription>
diff --git a/tests/build/Android.mk b/tests/build/Android.mk
index b1b51c5..89f2bab 100644
--- a/tests/build/Android.mk
+++ b/tests/build/Android.mk
@@ -25,7 +25,7 @@
LOCAL_MODULE := tf-contrib-tests
LOCAL_MODULE_TAGS := optional
-LOCAL_JAVA_LIBRARIES := tradefed tradefed-contrib easymock
+LOCAL_JAVA_LIBRARIES := tradefed tradefed-contrib easymock mockito objenesis
include $(BUILD_HOST_JAVA_LIBRARY)
diff --git a/tests/src/com/android/regression/tests/DetectRegressionTest.java b/tests/src/com/android/regression/tests/DetectRegressionTest.java
new file mode 100644
index 0000000..0434969
--- /dev/null
+++ b/tests/src/com/android/regression/tests/DetectRegressionTest.java
@@ -0,0 +1,169 @@
+/*
+ * Copyright (C) 2018 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.android.regression.tests;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.ArgumentMatchers.eq;
+import static org.mockito.Mockito.doNothing;
+import static org.mockito.Mockito.spy;
+import static org.mockito.Mockito.times;
+import static org.mockito.Mockito.verify;
+
+import com.android.regression.tests.DetectRegression.TableRow;
+import com.android.tradefed.result.TestDescription;
+import com.android.tradefed.util.MultiMap;
+
+import com.google.common.primitives.Doubles;
+
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.JUnit4;
+import org.mockito.ArgumentCaptor;
+
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.stream.Collectors;
+
+/** Unit tests for {@link DetectRegression}. */
+@RunWith(JUnit4.class)
+public class DetectRegressionTest {
+
+ private static final double EPS = 0.0001;
+
+ @Test
+ public void testCalcMean() {
+ Map<Double, double[]> data = new HashMap<>();
+ data.put(2.5, new double[] {1, 2, 3, 4});
+ data.put(
+ 4.5,
+ new double[] {
+ -11, 22, 5, 4, 2.5,
+ });
+ data.forEach(
+ (k, v) -> {
+ assertTrue(equal(k, DetectRegression.calcMean(Doubles.asList(v))));
+ });
+ }
+
+ @Test
+ public void testCalcStdDev() {
+ Map<Double, double[]> data = new HashMap<>();
+ data.put(36.331000536732, new double[] {12.3, 56.7, 45.6, 124, 56});
+ data.put(119.99906922093, new double[] {123.4, 22.5, 5.67, 4.56, 2.5, 333});
+ data.forEach(
+ (k, v) -> {
+ assertTrue(equal(k, DetectRegression.calcStdDev(Doubles.asList(v))));
+ });
+ }
+
+ @Test
+ public void testDetectRegression() {
+ List<List<Double>> befores =
+ Arrays.stream(
+ new double[][] {
+ {3, 3, 3, 3, 3},
+ {3, 5, 3, 5, 4},
+ {3, 4, 3, 4, 3},
+ {1, 2, 3, 2, 1},
+ {-1, -2, -3, 0, 1, 2, 3},
+ {5, 6, 5, 6, 6, 5, 7},
+ })
+ .map(Doubles::asList)
+ .collect(Collectors.toList());
+ List<List<Double>> afters =
+ Arrays.stream(
+ new double[][] {
+ {3, 3, 3, 3, 3},
+ {2, 3, 4, 5, 6},
+ {2, 5, 2, 5, 2},
+ {10, 11, 12, 13, 14},
+ {-10, -20, -30, 0, 10, 20, 30},
+ {5, 6, 5, 6, 6, 5, 700},
+ })
+ .map(Doubles::asList)
+ .collect(Collectors.toList());
+ boolean[] result = {false, false, true, true, true, false};
+
+ for (int i = 0; i < result.length; i++) {
+ assertEquals(
+ result[i], DetectRegression.computeRegression(befores.get(i), afters.get(i)));
+ }
+ }
+
+ @SuppressWarnings("unchecked")
+ @Test
+ public void testRunRegressionDetection() {
+ DetectRegression detector = spy(DetectRegression.class);
+ doNothing().when(detector).logResult(any(), any(), any(), any());
+ TestDescription id1 = new TestDescription("class", "test1");
+ TestDescription id2 = new TestDescription("class", "test2");
+ Metrics before = new Metrics(false);
+ Arrays.asList("3.0", "3.0", "3.0", "3.0", "3.0")
+ .forEach(e -> before.addRunMetric("metric-1", e));
+ Arrays.asList("3.1", "3.3", "3.1", "3.2", "3.3")
+ .forEach(e -> before.addRunMetric("metric-2", e));
+ Arrays.asList("5.1", "5.2", "5.1", "5.2", "5.1")
+ .forEach(e -> before.addRunMetric("metric-3", e));
+ Arrays.asList("3.0", "3.0", "3.0", "3.0", "3.0")
+ .forEach(e -> before.addTestMetric(id1, "metric-4", e));
+ Arrays.asList("3.1", "3.3", "3.1", "3.2", "3.3")
+ .forEach(e -> before.addTestMetric(id2, "metric-5", e));
+ Arrays.asList("5.1", "5.2", "5.1", "5.2", "5.1")
+ .forEach(e -> before.addTestMetric(id2, "metric-6", e));
+
+ Metrics after = new Metrics(false);
+ Arrays.asList("3.0", "3.0", "3.0", "3.0", "3.0")
+ .forEach(e -> after.addRunMetric("metric-1", e));
+ Arrays.asList("3.2", "3.2", "3.2", "3.2", "3.2")
+ .forEach(e -> after.addRunMetric("metric-2", e));
+ Arrays.asList("8.1", "8.2", "8.1", "8.2", "8.1")
+ .forEach(e -> after.addRunMetric("metric-3", e));
+ Arrays.asList("3.0", "3.0", "3.0", "3.0", "3.0")
+ .forEach(e -> after.addTestMetric(id1, "metric-4", e));
+ Arrays.asList("3.2", "3.2", "3.2", "3.2", "3.2")
+ .forEach(e -> after.addTestMetric(id2, "metric-5", e));
+ Arrays.asList("8.1", "8.2", "8.1", "8.2", "8.1")
+ .forEach(e -> after.addTestMetric(id2, "metric-6", e));
+
+ ArgumentCaptor<List<TableRow>> runResultCaptor = ArgumentCaptor.forClass(List.class);
+ ArgumentCaptor<MultiMap<String, TableRow>> testResultCaptor =
+ ArgumentCaptor.forClass(MultiMap.class);
+ detector.runRegressionDetection(before, after);
+ verify(detector, times(1))
+ .logResult(
+ eq(before),
+ eq(after),
+ runResultCaptor.capture(),
+ testResultCaptor.capture());
+
+ List<TableRow> runResults = runResultCaptor.getValue();
+ assertEquals(1, runResults.size());
+ assertEquals("metric-3", runResults.get(0).name);
+
+ MultiMap<String, TableRow> testResults = testResultCaptor.getValue();
+ assertEquals(1, testResults.size());
+ assertEquals(1, testResults.get(id2.toString()).size());
+ assertEquals("metric-6", testResults.get(id2.toString()).get(0).name);
+ }
+
+ private boolean equal(double d1, double d2) {
+ return Math.abs(d1 - d2) < EPS;
+ }
+}
diff --git a/tests/src/com/android/regression/tests/MetricsTest.java b/tests/src/com/android/regression/tests/MetricsTest.java
new file mode 100644
index 0000000..6d672db
--- /dev/null
+++ b/tests/src/com/android/regression/tests/MetricsTest.java
@@ -0,0 +1,135 @@
+/*
+ * Copyright (C) 2018 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.android.regression.tests;
+
+import static org.junit.Assert.assertEquals;
+import static org.mockito.ArgumentMatchers.anyString;
+import static org.mockito.Mockito.spy;
+import static org.mockito.Mockito.times;
+import static org.mockito.Mockito.verify;
+
+import com.android.tradefed.result.TestDescription;
+import com.android.tradefed.util.Pair;
+
+import com.google.common.primitives.Doubles;
+
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.JUnit4;
+
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+/** Unit tests for {@link Metrics}. */
+@RunWith(JUnit4.class)
+public class MetricsTest {
+
+ private Metrics mMetrics;
+
+ @Before
+ public void setUp() throws Exception {
+ mMetrics = spy(new Metrics(false));
+ }
+
+ @Test
+ public void testAddRunMetrics() {
+ Map<String, List<String>> data = new HashMap<>();
+ data.put("metric1", Arrays.asList("1.0", "1.1", "1.2"));
+ data.put("metric2", Arrays.asList("2.0", "2.1", "2.2"));
+ data.forEach((k, v) -> v.forEach(e -> mMetrics.addRunMetric(k, e)));
+ assertEquals(Doubles.asList(1.0, 1.1, 1.2), mMetrics.getRunMetrics().get("metric1"));
+ assertEquals(Doubles.asList(2.0, 2.1, 2.2), mMetrics.getRunMetrics().get("metric2"));
+ }
+
+ @Test
+ public void testAddTestMetrics() {
+ TestDescription id1 = new TestDescription("class", "test1");
+ Arrays.asList("1.0", "1.1", "1.2").forEach(e -> mMetrics.addTestMetric(id1, "metric1", e));
+ TestDescription id2 = new TestDescription("class", "test2");
+ Arrays.asList("2.0", "2.1", "2.2").forEach(e -> mMetrics.addTestMetric(id2, "metric1", e));
+ Arrays.asList("3.0", "3.1", "3.2").forEach(e -> mMetrics.addTestMetric(id2, "metric2", e));
+
+ assertEquals(
+ Doubles.asList(1.0, 1.1, 1.2),
+ mMetrics.getTestMetrics().get(new Pair<>(id1, "metric1")));
+ assertEquals(
+ Doubles.asList(2.0, 2.1, 2.2),
+ mMetrics.getTestMetrics().get(new Pair<>(id2, "metric1")));
+ assertEquals(
+ Doubles.asList(3.0, 3.1, 3.2),
+ mMetrics.getTestMetrics().get(new Pair<>(id2, "metric2")));
+ }
+
+ @Test
+ public void testValidate() {
+ Map<String, List<String>> data = new HashMap<>();
+ data.put("metric1", Arrays.asList("1.0", "1.1", "1.2"));
+ data.put("metric2", Arrays.asList("2.0", "2.1"));
+ data.forEach((k, v) -> v.forEach(e -> mMetrics.addRunMetric(k, e)));
+ TestDescription id1 = new TestDescription("class", "test1");
+ Arrays.asList("1.0", "1.1", "1.2").forEach(e -> mMetrics.addTestMetric(id1, "metric1", e));
+ TestDescription id2 = new TestDescription("class", "test2");
+ Arrays.asList("2.0", "2.1", "2.2").forEach(e -> mMetrics.addTestMetric(id2, "metric1", e));
+ Arrays.asList("3.0", "3.1").forEach(e -> mMetrics.addTestMetric(id2, "metric2", e));
+ mMetrics.validate(3);
+ verify(mMetrics, times(2)).error(anyString());
+ }
+
+ @Test
+ public void testCrossValidate() {
+ Metrics other = new Metrics(false);
+ Arrays.asList("1.0", "1.1", "1.2")
+ .forEach(
+ e -> {
+ mMetrics.addRunMetric("metric1", e);
+ other.addRunMetric("metric1", e);
+ });
+ Arrays.asList("2.0", "2.1", "2.2").forEach(e -> mMetrics.addRunMetric("metric2", e));
+ Arrays.asList("2.0", "2.1", "2.2").forEach(e -> other.addRunMetric("metric5", e));
+ TestDescription id1 = new TestDescription("class", "test1");
+ Arrays.asList("1.0", "1.1", "1.2")
+ .forEach(
+ e -> {
+ mMetrics.addTestMetric(id1, "metric1", e);
+ other.addTestMetric(id1, "metric1", e);
+ });
+ Arrays.asList("3.0", "3.1", "3.3").forEach(e -> mMetrics.addTestMetric(id1, "metric6", e));
+ TestDescription id2 = new TestDescription("class", "test2");
+ Arrays.asList("2.0", "2.1", "2.2")
+ .forEach(
+ e -> {
+ mMetrics.addTestMetric(id2, "metric1", e);
+ other.addTestMetric(id2, "metric1", e);
+ });
+ Arrays.asList("3.0", "3.1", "3.3").forEach(e -> other.addTestMetric(id2, "metric2", e));
+ mMetrics.crossValidate(other);
+ verify(mMetrics, times(1)).warn("Run metric \"metric2\" only in before-patch run.");
+ verify(mMetrics, times(1)).warn("Run metric \"metric5\" only in after-patch run.");
+ verify(mMetrics, times(1))
+ .warn(
+ String.format(
+ "Test %s metric \"metric6\" only in before-patch run.",
+ id1.toString()));
+ verify(mMetrics, times(1))
+ .warn(
+ String.format(
+ "Test %s metric \"metric2\" only in after-patch run.",
+ id2.toString()));
+ }
+}
diff --git a/tests/src/com/android/regression/tests/MetricsXmlParserTest.java b/tests/src/com/android/regression/tests/MetricsXmlParserTest.java
new file mode 100644
index 0000000..74596f6
--- /dev/null
+++ b/tests/src/com/android/regression/tests/MetricsXmlParserTest.java
@@ -0,0 +1,187 @@
+/*
+ * Copyright (C) 2018 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.android.regression.tests;
+
+import static org.junit.Assert.fail;
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.ArgumentMatchers.anyString;
+import static org.mockito.Mockito.doReturn;
+import static org.mockito.Mockito.times;
+import static org.mockito.Mockito.verify;
+
+import com.android.regression.tests.MetricsXmlParser.ParseException;
+import com.android.tradefed.build.BuildInfo;
+import com.android.tradefed.config.OptionSetter;
+import com.android.tradefed.invoker.IInvocationContext;
+import com.android.tradefed.invoker.InvocationContext;
+import com.android.tradefed.metrics.proto.MetricMeasurement.Metric;
+import com.android.tradefed.result.MetricsXMLResultReporter;
+import com.android.tradefed.result.TestDescription;
+import com.android.tradefed.util.proto.TfMetricProtoUtil;
+
+import com.google.common.collect.ImmutableSet;
+
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.JUnit4;
+import org.mockito.Mock;
+import org.mockito.Mockito;
+import org.mockito.MockitoAnnotations;
+import org.mockito.Spy;
+
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Set;
+
+/** Simple unit tests for {@link MetricsXmlParser}. */
+@RunWith(JUnit4.class)
+public class MetricsXmlParserTest {
+
+ @Spy private MetricsXMLResultReporter mResultReporter;
+ @Mock private Metrics mMetrics;
+ private ByteArrayOutputStream mOutputStream;
+
+ @Before
+ public void setUp() throws Exception {
+ mOutputStream = new ByteArrayOutputStream();
+ MockitoAnnotations.initMocks(this);
+ OptionSetter optionSetter = new OptionSetter(mResultReporter);
+ optionSetter.setOptionValue("metrics-folder", "/tmp");
+ doReturn(mOutputStream).when(mResultReporter).createOutputStream();
+ doReturn("ignore").when(mResultReporter).getTimeStamp();
+ }
+
+ /** Test behavior when data to parse is empty */
+ @Test
+ public void testEmptyParse() {
+ try {
+ MetricsXmlParser.parse(
+ mMetrics, Collections.emptySet(), new ByteArrayInputStream(new byte[0]));
+ fail("ParseException not thrown");
+ } catch (ParseException e) {
+ // expected
+ }
+ Mockito.verifyZeroInteractions(mMetrics);
+ }
+
+ /** Simple success test for xml parsing */
+ @Test
+ public void testSimpleParse() throws ParseException {
+ IInvocationContext context = new InvocationContext();
+ context.addDeviceBuildInfo("fakeDevice", new BuildInfo());
+ context.setTestTag("stub");
+ mResultReporter.invocationStarted(context);
+ mResultReporter.testRunStarted("run", 3);
+ final TestDescription testId0 = new TestDescription("Test", "pass1");
+ mResultReporter.testStarted(testId0);
+ mResultReporter.testEnded(testId0, new HashMap<String, Metric>());
+ final TestDescription testId1 = new TestDescription("Test", "pass2");
+ mResultReporter.testStarted(testId1);
+ mResultReporter.testEnded(testId1, new HashMap<String, Metric>());
+ final TestDescription testId2 = new TestDescription("Test", "pass3");
+ mResultReporter.testStarted(testId2);
+ mResultReporter.testEnded(testId2, new HashMap<String, Metric>());
+ mResultReporter.testRunEnded(3, new HashMap<String, Metric>());
+ mResultReporter.invocationEnded(5);
+
+ MetricsXmlParser.parse(
+ mMetrics, Collections.emptySet(), new ByteArrayInputStream(getOutput()));
+ verify(mMetrics).setNumTests(3);
+ verify(mMetrics).addRunMetric("time", "5");
+ verify(mMetrics, times(0)).addTestMetric(any(), anyString(), anyString());
+ Mockito.verifyNoMoreInteractions(mMetrics);
+ }
+
+ /** Test parsing a comprehensive document containing run metrics and test metrics */
+ @Test
+ public void testParse() throws ParseException {
+ IInvocationContext context = new InvocationContext();
+ context.addDeviceBuildInfo("fakeDevice", new BuildInfo());
+ context.setTestTag("stub");
+ mResultReporter.invocationStarted(context);
+ mResultReporter.testRunStarted("run", 2);
+
+ final TestDescription testId0 = new TestDescription("Test", "pass1");
+ mResultReporter.testStarted(testId0);
+ Map<String, String> testMetrics0 = new HashMap<>();
+ testMetrics0.put("metric1", "1.1");
+ mResultReporter.testEnded(testId0, TfMetricProtoUtil.upgradeConvert(testMetrics0));
+
+ final TestDescription testId1 = new TestDescription("Test", "pass2");
+ mResultReporter.testStarted(testId1);
+ Map<String, String> testMetrics1 = new HashMap<>();
+ testMetrics1.put("metric2", "5.5");
+ mResultReporter.testEnded(testId1, TfMetricProtoUtil.upgradeConvert(testMetrics1));
+
+ Map<String, String> runMetrics = new HashMap<>();
+ runMetrics.put("metric3", "8.8");
+ mResultReporter.testRunEnded(3, TfMetricProtoUtil.upgradeConvert(runMetrics));
+ mResultReporter.invocationEnded(5);
+
+ MetricsXmlParser.parse(
+ mMetrics, Collections.emptySet(), new ByteArrayInputStream(getOutput()));
+
+ verify(mMetrics).setNumTests(2);
+ verify(mMetrics).addRunMetric("metric3", "8.8");
+ verify(mMetrics).addTestMetric(testId0, "metric1", "1.1");
+ verify(mMetrics).addTestMetric(testId1, "metric2", "5.5");
+ }
+
+ /** Test parsing a document with blacklist metrics */
+ @Test
+ public void testParseBlacklist() throws ParseException {
+ IInvocationContext context = new InvocationContext();
+ context.addDeviceBuildInfo("fakeDevice", new BuildInfo());
+ context.setTestTag("stub");
+ mResultReporter.invocationStarted(context);
+ mResultReporter.testRunStarted("run", 3);
+
+ final TestDescription testId0 = new TestDescription("Test", "pass1");
+ mResultReporter.testStarted(testId0);
+ Map<String, String> testMetrics0 = new HashMap<>();
+ testMetrics0.put("metric1", "1.1");
+ mResultReporter.testEnded(testId0, TfMetricProtoUtil.upgradeConvert(testMetrics0));
+
+ final TestDescription testId1 = new TestDescription("Test", "pass2");
+ mResultReporter.testStarted(testId1);
+ Map<String, String> testMetrics1 = new HashMap<>();
+ testMetrics1.put("metric2", "5.5");
+ mResultReporter.testEnded(testId1, TfMetricProtoUtil.upgradeConvert(testMetrics1));
+
+ Map<String, String> runMetrics = new HashMap<>();
+ runMetrics.put("metric3", "8.8");
+ mResultReporter.testRunEnded(3, TfMetricProtoUtil.upgradeConvert(runMetrics));
+ mResultReporter.invocationEnded(5);
+
+ Set<String> blacklist = ImmutableSet.of("metric1", "metric3");
+
+ MetricsXmlParser.parse(mMetrics, blacklist, new ByteArrayInputStream(getOutput()));
+
+ verify(mMetrics, times(0)).addRunMetric("metric3", "8.8");
+ verify(mMetrics, times(0)).addTestMetric(testId0, "metric1", "1.1");
+ verify(mMetrics).addTestMetric(testId1, "metric2", "5.5");
+ }
+
+ /** Gets the output produced, stripping it of extraneous whitespace characters. */
+ private byte[] getOutput() {
+ return mOutputStream.toByteArray();
+ }
+}
diff --git a/tests/src/com/android/tradefed/prodtests/UnitTests.java b/tests/src/com/android/tradefed/prodtests/UnitTests.java
index d8ff12b..e74ca9c 100644
--- a/tests/src/com/android/tradefed/prodtests/UnitTests.java
+++ b/tests/src/com/android/tradefed/prodtests/UnitTests.java
@@ -18,6 +18,9 @@
import com.android.build.tests.ImageStatsTest;
import com.android.continuous.SmokeTestTest;
import com.android.monkey.MonkeyBaseTest;
+import com.android.regression.tests.DetectRegressionTest;
+import com.android.regression.tests.MetricsTest;
+import com.android.regression.tests.MetricsXmlParserTest;
import org.junit.runner.RunWith;
import org.junit.runners.Suite;
@@ -38,7 +41,12 @@
SmokeTestTest.class,
// monkey
- MonkeyBaseTest.class
+ MonkeyBaseTest.class,
+
+ // regression
+ DetectRegressionTest.class,
+ MetricsTest.class,
+ MetricsXmlParserTest.class,
})
public class UnitTests {
// empty of purpose
