Provide option to add iteration number microbenchmark runner.
Suffix with iteration number to uniquely identify the test results.
Bug: b/139395137
Test: MicrobenchmarkTest
Change-Id: Ibf1c9431c0e486b589dac18919b62e9683ae3675
(cherry picked from commit 85112086f22951249a2efd1e2c62bf7826d3b666)
diff --git a/libraries/health/runners/microbenchmark/src/android/platform/test/microbenchmark/Microbenchmark.java b/libraries/health/runners/microbenchmark/src/android/platform/test/microbenchmark/Microbenchmark.java
index 0cafd5c..980b11e 100644
--- a/libraries/health/runners/microbenchmark/src/android/platform/test/microbenchmark/Microbenchmark.java
+++ b/libraries/health/runners/microbenchmark/src/android/platform/test/microbenchmark/Microbenchmark.java
@@ -25,9 +25,13 @@
import java.lang.annotation.Retention;
import java.lang.annotation.RetentionPolicy;
import java.lang.annotation.Target;
+import java.util.HashMap;
import java.util.List;
+import java.util.Map;
import org.junit.rules.TestRule;
+import org.junit.runner.Description;
+import org.junit.runner.notification.RunNotifier;
import org.junit.runners.BlockJUnit4ClassRunner;
import org.junit.runners.model.InitializationError;
import org.junit.runners.model.FrameworkMethod;
@@ -41,6 +45,17 @@
public class Microbenchmark extends BlockJUnit4ClassRunner {
private Bundle mArguments;
+ @VisibleForTesting static final String ITERATION_SEP_OPTION = "iteration-separator";
+ @VisibleForTesting static final String ITERATION_SEP_DEFAULT = "$";
+ // A constant to indicate that the iteration number is not set.
+ @VisibleForTesting static final int ITERATION_NOT_SET = -1;
+ public static final String RENAME_ITERATION_OPTION = "rename-iterations";
+
+ private String mIterationSep = ITERATION_SEP_DEFAULT;
+
+ private boolean mRenameIterations;
+ private Map<Description, Integer> mIterations = new HashMap<>();
+
/**
* Called reflectively on classes annotated with {@code @RunWith(Microbenchmark.class)}.
*/
@@ -55,6 +70,12 @@
Microbenchmark(Class<?> klass, Bundle arguments) throws InitializationError {
super(klass);
mArguments = arguments;
+ // Parse out additional options.
+ mRenameIterations = Boolean.valueOf(arguments.getString(RENAME_ITERATION_OPTION));
+ mIterationSep =
+ arguments.containsKey(ITERATION_SEP_OPTION)
+ ? arguments.getString(ITERATION_SEP_OPTION)
+ : mIterationSep;
}
/**
@@ -115,4 +136,40 @@
@Retention(RetentionPolicy.RUNTIME)
@Target({ElementType.FIELD, ElementType.METHOD})
public @interface TightMethodRule { }
+
+
+ /**
+ * Rename the child class name to add iterations if the renaming iteration option is enabled.
+ *
+ * <p>Renaming the class here is chosen over renaming the method name because
+ *
+ * <ul>
+ * <li>Conceptually, the runner is running a class multiple times, as opposed to a method.
+ * <li>When instrumenting a suite in command line, by default the instrumentation command
+ * outputs the class name only. Renaming the class helps with interpretation in this case.
+ */
+ @Override
+ protected Description describeChild(FrameworkMethod method) {
+ Description original = super.describeChild(method);
+ if (!mRenameIterations) {
+ return original;
+ }
+ return Description.createTestDescription(
+ String.join(mIterationSep, original.getClassName(),
+ String.valueOf(mIterations.get(original))), original.getMethodName());
+ }
+
+ /**
+ * Keep track of the number of iterations for a particular method and
+ * set the current iteration count for changing the current description.
+ */
+ @Override
+ protected void runChild(final FrameworkMethod method, RunNotifier notifier) {
+ if (mRenameIterations) {
+ Description original = super.describeChild(method);
+ mIterations.computeIfPresent(original, (k, v) -> v + 1);
+ mIterations.computeIfAbsent(original, k -> 1);
+ }
+ super.runChild(method, notifier);
+ }
}
diff --git a/libraries/health/runners/microbenchmark/tests/src/android/platform/test/microbenchmark/MicrobenchmarkTest.java b/libraries/health/runners/microbenchmark/tests/src/android/platform/test/microbenchmark/MicrobenchmarkTest.java
index 0de79e3..c644fc8 100644
--- a/libraries/health/runners/microbenchmark/tests/src/android/platform/test/microbenchmark/MicrobenchmarkTest.java
+++ b/libraries/health/runners/microbenchmark/tests/src/android/platform/test/microbenchmark/MicrobenchmarkTest.java
@@ -80,6 +80,106 @@
}
/**
+ * Test iterations number are added to the test name with default suffix.
+ *
+ * Before --> TightBefore --> Trace (begin) with suffix @1 --> Test --> Trace(end)
+ * --> TightAfter --> After --> Before --> TightBefore --> Trace (begin) with suffix @2
+ * --> Test --> Trace(end) --> TightAfter --> After
+ */
+ @Test
+ public void testMultipleIterationsWithRename() throws InitializationError {
+ Bundle args = new Bundle();
+ args.putString("iterations", "2");
+ args.putString("rename-iterations", "true");
+ LoggingMicrobenchmark loggingRunner = new LoggingMicrobenchmark(LoggingTest.class, args);
+ loggingRunner.setOperationLog(new ArrayList<String>());
+ Result result = new JUnitCore().run(loggingRunner);
+ assertThat(result.wasSuccessful()).isTrue();
+ assertThat(loggingRunner.getOperationLog()).containsExactly(
+ "before",
+ "tight before",
+ "begin: testMethod("
+ + "android.platform.test.microbenchmark.MicrobenchmarkTest$LoggingTest$1)",
+ "test",
+ "end",
+ "tight after",
+ "after",
+ "before",
+ "tight before",
+ "begin: testMethod("
+ + "android.platform.test.microbenchmark.MicrobenchmarkTest$LoggingTest$2)",
+ "test",
+ "end",
+ "tight after",
+ "after")
+ .inOrder();
+ }
+
+ /**
+ * Test iterations number are added to the test name with custom suffix.
+ *
+ * Before --> TightBefore --> Trace (begin) with suffix --1 --> Test --> Trace(end)
+ * --> TightAfter --> After --> Before --> TightBefore --> Trace (begin) with suffix --2
+ * --> Test --> Trace(end) --> TightAfter --> After
+ */
+ @Test
+ public void testMultipleIterationsWithDifferentSuffix() throws InitializationError {
+ Bundle args = new Bundle();
+ args.putString("iterations", "2");
+ args.putString("rename-iterations", "true");
+ args.putString("iteration-separator", "--");
+ LoggingMicrobenchmark loggingRunner = new LoggingMicrobenchmark(LoggingTest.class, args);
+ loggingRunner.setOperationLog(new ArrayList<String>());
+ Result result = new JUnitCore().run(loggingRunner);
+ assertThat(result.wasSuccessful()).isTrue();
+ assertThat(loggingRunner.getOperationLog()).containsExactly(
+ "before",
+ "tight before",
+ "begin: testMethod("
+ + "android.platform.test.microbenchmark.MicrobenchmarkTest$LoggingTest--1)",
+ "test",
+ "end",
+ "tight after",
+ "after",
+ "before",
+ "tight before",
+ "begin: testMethod("
+ + "android.platform.test.microbenchmark.MicrobenchmarkTest$LoggingTest--2)",
+ "test",
+ "end",
+ "tight after",
+ "after")
+ .inOrder();
+ }
+
+ /**
+ * Test iteration number are not added to the test name when explictly disabled.
+ *
+ * Before --> TightBefore --> Trace (begin) --> Test --> Trace(end) --> TightAfter
+ * --> After
+ */
+ @Test
+ public void testMultipleIterationsWithoutRename() throws InitializationError {
+ Bundle args = new Bundle();
+ args.putString("iterations", "1");
+ args.putString("rename-iterations", "false");
+ LoggingMicrobenchmark loggingRunner = new LoggingMicrobenchmark(LoggingTest.class, args);
+ loggingRunner.setOperationLog(new ArrayList<String>());
+ Result result = new JUnitCore().run(loggingRunner);
+ assertThat(result.wasSuccessful()).isTrue();
+ assertThat(loggingRunner.getOperationLog()).containsExactly(
+ "before",
+ "tight before",
+ "begin: testMethod("
+ + "android.platform.test.microbenchmark.MicrobenchmarkTest$LoggingTest)",
+ "test",
+ "end",
+ "tight after",
+ "after")
+ .inOrder();
+ }
+
+ /**
* An extensions of the {@link Microbenchmark} runner that logs the start and end of collecting
* traces. It also passes the operation log to the provided test {@code Class}, if it is a
* {@link LoggingTest}. This is used for ensuring the proper order for evaluating test {@link
