Restrict when Caliper benchmarks are run.
Caliper benchmarks will only be run when --benchmark is
specified. They also will not be allowed when --mode=activity.
This removes the need for the client to send the host the name
of the runner that ran the tests. That was only used to check
that the CaliperRunner was always run with the --benchmark
option but that will always be true as the CaliperRunner is only
used when --benchmark option is specified. That allows us to
remove the runnerClass parameter to
TargetMonitor.outcomeStarted(...).
Bug: 26716337
Change-Id: I1a1e4a9e35a6dbd0424cc1ed086ee5f2b022865c
diff --git a/src/vogar/Run.java b/src/vogar/Run.java
index 25a2002..67b0593 100644
--- a/src/vogar/Run.java
+++ b/src/vogar/Run.java
@@ -33,8 +33,6 @@
import vogar.android.AndroidSdk;
import vogar.android.DeviceRuntime;
import vogar.android.HostRuntime;
-import vogar.commands.CommandFailedException;
-import vogar.commands.Jack;
import vogar.commands.Mkdir;
import vogar.commands.Rm;
import vogar.tasks.TaskQueue;
@@ -108,7 +106,7 @@
public final ClassFileIndex classFileIndex;
public final OutcomeStore outcomeStore;
public final TaskQueue taskQueue;
- public final boolean testOnly;
+ public final RunnerType runnerType;
public final boolean useJack;
public final boolean checkJni;
public final boolean debugging;
@@ -167,7 +165,7 @@
this.keystore = localFile("activity", "vogar.keystore");
this.classpath = Classpath.of(vogar.classpath);
this.classpath.addAll(vogarJar());
- this.testOnly = vogar.testOnly;
+ this.runnerType = vogar.runnerType;
this.androidSdk = androidSdk;
diff --git a/src/vogar/RunnerType.java b/src/vogar/RunnerType.java
new file mode 100644
index 0000000..dfc8c5a
--- /dev/null
+++ b/src/vogar/RunnerType.java
@@ -0,0 +1,64 @@
+/*
+ * Copyright (C) 2016 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package vogar;
+
+/**
+ * The type of runner to use for the test classes.
+ */
+public enum RunnerType {
+ /**
+ * Runs both JUnit classes and classes with a main(String[] args) method.
+ */
+ DEFAULT(false, true, true),
+
+ /**
+ * Runs only Caliper benchmarks.
+ */
+ CALIPER(true, false, false),
+
+ /**
+ * Runs only JUnit classes.
+ */
+ JUNIT(false, true, false),
+
+ /**
+ * Runs only classes with a main(String[] args) method.
+ */
+ MAIN(false, false, true);
+
+ private final boolean supportsCaliper;
+ private final boolean supportsJUnit;
+ private final boolean supportsMain;
+
+ RunnerType(boolean supportsCaliper, boolean supportsJUnit, boolean supportsMain) {
+ this.supportsCaliper = supportsCaliper;
+ this.supportsJUnit = supportsJUnit;
+ this.supportsMain = supportsMain;
+ }
+
+ public boolean supportsCaliper() {
+ return supportsCaliper;
+ }
+
+ public boolean supportsJUnit() {
+ return supportsJUnit;
+ }
+
+ public boolean supportsMain() {
+ return supportsMain;
+ }
+}
diff --git a/src/vogar/TestProperties.java b/src/vogar/TestProperties.java
index 626701c..50df237 100644
--- a/src/vogar/TestProperties.java
+++ b/src/vogar/TestProperties.java
@@ -75,9 +75,9 @@
public static final String PROFILE_THREAD_GROUP = "profileThreadGroup";
/**
- * Only run JUnit tests?
+ * The type of runner to use.
*/
- public static final String TEST_ONLY = "testOnly";
+ public static final String RUNNER_TYPE = "runnerType";
private TestProperties() {}
}
diff --git a/src/vogar/Vogar.java b/src/vogar/Vogar.java
index 29b64c8..5e16d3a 100644
--- a/src/vogar/Vogar.java
+++ b/src/vogar/Vogar.java
@@ -212,6 +212,9 @@
@Option(names = { "--check-jni" })
boolean checkJni = true;
+ @Option(names = {"--runner-type"})
+ RunnerType runnerType;
+
@VisibleForTesting public Vogar() {}
private void printUsage() {
@@ -258,7 +261,10 @@
System.out.println();
System.out.println(" --benchmark: for use with dalvikvm, this dexes all files together,");
System.out.println(" and is mandatory for running Caliper benchmarks, and a good idea");
- System.out.println(" other performance sensitive code.");
+ System.out.println(" for other performance sensitive code.");
+ System.out.println(" If you specify this without specifying --runner-type then it");
+ System.out.println(" assumes --runner-type="
+ + RunnerType.CALIPER.name().toLowerCase());
System.out.println();
System.out.println(" --profile: run with a profiler to produce an hprof file.");
System.out.println();
@@ -310,15 +316,25 @@
System.out.println(" --results-dir <directory>: read and write (if --record-results used)");
System.out.println(" results from and to this directory.");
System.out.println();
+ System.out.println(" --runner-type <default|caliper|main|junit>: specify which runner to use.");
+ System.out.println(" default: runs both JUnit tests and main() classes");
+ System.out.println(" caliper: runs Caliper benchmarks only");
+ System.out.println(" main: runs main() classes only");
+ System.out.println(" junit: runs JUnit tests only");
+ System.out.println(" Default is determined by --benchmark and --testonly, if they are");
+ System.out.println(" not specified then defaults to: default");
+ System.out.println();
System.out.println(" --test-only: only run JUnit tests.");
System.out.println(" Default is: " + testOnly);
+ System.out.println(" DEPRECATED: Use --runner-type="
+ + RunnerType.JUNIT.name().toLowerCase());
System.out.println();
System.out.println(" --verbose: turn on persistent verbose output.");
System.out.println();
System.out.println(" --check-jni: enable CheckJNI mode.");
System.out.println(" See http://developer.android.com/training/articles/perf-jni.html.");
System.out.println(" Default is: " + checkJni + ", but disabled for --benchmark.");
- System.out.println();
+ System.out.println("");
System.out.println("TARGET OPTIONS");
System.out.println();
System.out.println(" --debug <port>: enable Java debugging on the specified port.");
@@ -617,6 +633,40 @@
androidSdk = AndroidSdk.createAndroidSdk(console, mkdir, modeId, useJack);
}
+ if (runnerType == null) {
+ if (benchmark) {
+ if (testOnly) {
+ throw new IllegalStateException(
+ "--benchmark and --testOnly are mutually exclusive and deprecated,"
+ + " use --runner-type");
+ }
+ if (modeId == ModeId.ACTIVITY) {
+ throw new IllegalStateException(
+ "--benchmark and --mode activity are mutually exclusive");
+ }
+ runnerType = RunnerType.CALIPER;
+ } else if (testOnly) {
+ runnerType = RunnerType.JUNIT;
+ } else {
+ runnerType = RunnerType.DEFAULT;
+ }
+ } else {
+ if (testOnly) {
+ throw new IllegalStateException(
+ "--runnerType and --testOnly are mutually exclusive");
+ }
+
+ if (runnerType.supportsCaliper()) {
+ if (modeId == ModeId.ACTIVITY) {
+ throw new IllegalStateException(
+ "--runnerType caliper and --mode activity are mutually exclusive");
+ }
+
+ // Assume --benchmark
+ benchmark = true;
+ }
+ }
+
Run run = new Run(this, useJack, console, mkdir, androidSdk, rm, target, runnerDir);
if (configArgs.length > 0) {
run.console.verbose("loaded arguments from .vogarconfig: " +
diff --git a/src/vogar/monitor/HostMonitor.java b/src/vogar/monitor/HostMonitor.java
index cc9eb2d..72f32f2 100644
--- a/src/vogar/monitor/HostMonitor.java
+++ b/src/vogar/monitor/HostMonitor.java
@@ -16,7 +16,6 @@
package vogar.monitor;
-import com.google.gson.JsonElement;
import com.google.gson.JsonObject;
import java.io.BufferedInputStream;
import java.io.IOException;
@@ -125,9 +124,7 @@
if (jsonObject.get("outcome") != null) {
currentOutcome = jsonObject.get("outcome").getAsString();
handler.output(currentOutcome, "");
- JsonElement runner = jsonObject.get("runner");
- String runnerClass = runner != null ? runner.getAsString() : null;
- handler.start(currentOutcome, runnerClass);
+ handler.start(currentOutcome);
} else if (jsonObject.get("result") != null) {
Result currentResult = Result.valueOf(jsonObject.get("result").getAsString());
handler.finish(new Outcome(currentOutcome, currentResult, output.toString()));
@@ -151,10 +148,9 @@
public interface Handler {
/**
- * @param runnerClass can be null, indicating nothing is actually being run. This will
- * happen in the event of an impending error.
+ * Receive notification that an outcome is pending.
*/
- void start(String outcomeName, String runnerClass);
+ void start(String outcomeName);
/**
* Receive a completed outcome.
diff --git a/src/vogar/monitor/TargetMonitor.java b/src/vogar/monitor/TargetMonitor.java
index 9d79f9d..0286bcc 100644
--- a/src/vogar/monitor/TargetMonitor.java
+++ b/src/vogar/monitor/TargetMonitor.java
@@ -23,7 +23,6 @@
import java.net.ServerSocket;
import java.net.Socket;
import vogar.Result;
-import vogar.target.Runner;
/**
* Accepts a connection from the host process. Once connected, XML is sent over
@@ -64,12 +63,9 @@
}
}
- public void outcomeStarted(Class<? extends Runner> runnerClass, String outcomeName) {
+ public void outcomeStarted(String outcomeName) {
JsonObject jsonObject = new JsonObject();
jsonObject.addProperty("outcome", outcomeName);
- if (runnerClass != null) {
- jsonObject.addProperty("runner", runnerClass.getName());
- }
writer.print(marker + gson.toJson(jsonObject) + "\n");
}
diff --git a/src/vogar/target/CaliperRunner.java b/src/vogar/target/CaliperRunner.java
index f9a8002..1d760b7 100644
--- a/src/vogar/target/CaliperRunner.java
+++ b/src/vogar/target/CaliperRunner.java
@@ -41,7 +41,7 @@
}
public boolean run(Profiler profiler) {
- monitor.outcomeStarted(getClass(), testClass.getName());
+ monitor.outcomeStarted(testClass.getName());
ImmutableList.Builder<String> builder = ImmutableList.<String>builder()
.add(testClass.getName())
.add(args);
diff --git a/src/vogar/target/MainRunner.java b/src/vogar/target/MainRunner.java
index 79359f8..048f47c 100644
--- a/src/vogar/target/MainRunner.java
+++ b/src/vogar/target/MainRunner.java
@@ -43,7 +43,7 @@
}
public boolean run(Profiler profiler) {
- monitor.outcomeStarted(getClass(), mainClass.getName());
+ monitor.outcomeStarted(mainClass.getName());
try {
if (profiler != null) {
profiler.start();
diff --git a/src/vogar/target/TestRunner.java b/src/vogar/target/TestRunner.java
index 52ff05b..7aed594 100644
--- a/src/vogar/target/TestRunner.java
+++ b/src/vogar/target/TestRunner.java
@@ -31,6 +31,7 @@
import java.util.concurrent.atomic.AtomicReference;
import javax.annotation.Nullable;
import vogar.Result;
+import vogar.RunnerType;
import vogar.TestProperties;
import vogar.monitor.TargetMonitor;
import vogar.target.junit.JUnitRunnerFactory;
@@ -88,15 +89,20 @@
}
}
- boolean testOnly = Boolean.parseBoolean(properties.getProperty(TestProperties.TEST_ONLY));
- if (testOnly) {
- runnerFactory = new CompositeRunnerFactory(new JUnitRunnerFactory());
- } else {
- runnerFactory = new CompositeRunnerFactory(
- new JUnitRunnerFactory(),
- new CaliperRunnerFactory(argsList),
- new MainRunnerFactory());
+ // Select the RunnerFactory instances to use based on the selected runner type.
+ RunnerType runnerType =
+ RunnerType.valueOf(properties.getProperty(TestProperties.RUNNER_TYPE));
+ List<RunnerFactory> runnerFactories = new ArrayList<>();
+ if (runnerType.supportsCaliper()) {
+ runnerFactories.add(new CaliperRunnerFactory(argsList));
}
+ if (runnerType.supportsJUnit()) {
+ runnerFactories.add(new JUnitRunnerFactory());
+ }
+ if (runnerType.supportsMain()) {
+ runnerFactories.add(new MainRunnerFactory());
+ }
+ runnerFactory = new CompositeRunnerFactory(runnerFactories);
this.monitorPort = monitorPort;
this.skipPastReference = new AtomicReference<>(skipPast);
@@ -224,14 +230,14 @@
runner = runnerFactory.newRunner(monitor, qualification, klass,
skipPastReference, testEnvironment, timeoutSeconds, profile, args);
} catch (RuntimeException e) {
- monitor.outcomeStarted(null, qualifiedName);
+ monitor.outcomeStarted(qualifiedName);
e.printStackTrace();
monitor.outcomeFinished(Result.ERROR);
return;
}
if (runner == null) {
- monitor.outcomeStarted(null, klass.getName());
+ monitor.outcomeStarted(klass.getName());
System.out.println("Skipping " + klass.getName()
+ ": no associated runner class");
monitor.outcomeFinished(Result.UNSUPPORTED);
@@ -263,8 +269,8 @@
private final List<? extends RunnerFactory> runnerFactories;
- private CompositeRunnerFactory(RunnerFactory... runnerFactories) {
- this.runnerFactories = Arrays.asList(runnerFactories);
+ private CompositeRunnerFactory(List<RunnerFactory> factories) {
+ this.runnerFactories = factories;
}
@Override @Nullable
diff --git a/src/vogar/target/junit/JUnitRunner.java b/src/vogar/target/junit/JUnitRunner.java
index c8bf54c..3e9bffb 100644
--- a/src/vogar/target/junit/JUnitRunner.java
+++ b/src/vogar/target/junit/JUnitRunner.java
@@ -85,7 +85,7 @@
*/
private void runWithTimeout(final Profiler profiler, final VogarTest test) {
testEnvironment.reset();
- monitor.outcomeStarted(getClass(), test.toString());
+ monitor.outcomeStarted(test.toString());
// Start the test on a background thread.
final AtomicReference<Thread> executingThreadReference = new AtomicReference<Thread>();
diff --git a/src/vogar/tasks/BuildActionTask.java b/src/vogar/tasks/BuildActionTask.java
index e50378b..ef3ead0 100644
--- a/src/vogar/tasks/BuildActionTask.java
+++ b/src/vogar/tasks/BuildActionTask.java
@@ -194,6 +194,6 @@
properties.setProperty(TestProperties.PROFILE_FILE, run.profileFile.getName());
properties.setProperty(TestProperties.PROFILE_THREAD_GROUP,
Boolean.toString(run.profileThreadGroup));
- properties.setProperty(TestProperties.TEST_ONLY, Boolean.toString(run.testOnly));
+ properties.setProperty(TestProperties.RUNNER_TYPE, run.runnerType.toString());
}
}
diff --git a/src/vogar/tasks/RunActionTask.java b/src/vogar/tasks/RunActionTask.java
index 91e46d8..3c43d7b 100644
--- a/src/vogar/tasks/RunActionTask.java
+++ b/src/vogar/tasks/RunActionTask.java
@@ -23,10 +23,10 @@
import vogar.Outcome;
import vogar.Result;
import vogar.Run;
+import vogar.RunnerType;
import vogar.commands.Command;
import vogar.commands.VmCommandBuilder;
import vogar.monitor.HostMonitor;
-import vogar.target.CaliperRunner;
import vogar.target.TestRunner;
/**
@@ -175,15 +175,10 @@
: run.firstMonitorPort + runnerThreadId.get();
}
- @Override public void start(String outcomeName, String runnerClass) {
+ @Override public void start(String outcomeName) {
outcomeName = toQualifiedOutcomeName(outcomeName);
lastStartedOutcome = outcomeName;
- // TODO add to Outcome knowledge about what class was used to run it
- if (CaliperRunner.class.getName().equals(runnerClass)) {
- if (!run.benchmark) {
- throw new RuntimeException("you must use --benchmark when running Caliper "
- + "benchmarks.");
- }
+ if (run.runnerType.supportsCaliper()) {
run.console.verbose("running " + outcomeName + " with unlimited timeout");
Command command = currentCommand;
if (command != null && run.smallTimeoutSeconds != 0) {
diff --git a/test/vogar/target/JUnitRunnerTest.java b/test/vogar/target/JUnitRunnerTest.java
index 47afd29..ca41448 100644
--- a/test/vogar/target/JUnitRunnerTest.java
+++ b/test/vogar/target/JUnitRunnerTest.java
@@ -55,8 +55,7 @@
Runner runner = new JUnitRunner(monitor, skipPastReference, testEnvironment, 0, tests);
runner.run(null);
- verify(monitor).outcomeStarted(JUnitRunner.class,
- target.getName() + "#testSimple");
+ verify(monitor).outcomeStarted(target.getName() + "#testSimple");
verify(monitor).outcomeFinished(Result.SUCCESS);
}
@@ -66,13 +65,13 @@
Runner runner = new JUnitRunner(monitor, skipPastReference, testEnvironment, 0, tests);
runner.run(null);
- verify(monitor).outcomeStarted(JUnitRunner.class,
+ verify(monitor).outcomeStarted(
"vogar.target.junit3.SimpleTest#testSimple");
- verify(monitor).outcomeStarted(JUnitRunner.class,
+ verify(monitor).outcomeStarted(
"vogar.target.junit3.SimpleTest2#testSimple1");
- verify(monitor).outcomeStarted(JUnitRunner.class,
+ verify(monitor).outcomeStarted(
"vogar.target.junit3.SimpleTest2#testSimple2");
- verify(monitor).outcomeStarted(JUnitRunner.class,
+ verify(monitor).outcomeStarted(
"vogar.target.junit3.SimpleTest2#testSimple3");
verify(monitor, times(4)).outcomeFinished(Result.SUCCESS);
}
@@ -83,11 +82,11 @@
Runner runner = new JUnitRunner(monitor, skipPastReference, testEnvironment, 0, tests);
runner.run(null);
- verify(monitor).outcomeStarted(JUnitRunner.class,
+ verify(monitor).outcomeStarted(
target.getName() + "#testSimple1");
- verify(monitor).outcomeStarted(JUnitRunner.class,
+ verify(monitor).outcomeStarted(
target.getName() + "#testSimple2");
- verify(monitor).outcomeStarted(JUnitRunner.class,
+ verify(monitor).outcomeStarted(
target.getName() + "#testSimple3");
verify(monitor, times(3)).outcomeFinished(Result.SUCCESS);
}
@@ -99,7 +98,7 @@
Runner runner = new JUnitRunner(monitor, skipPastReference, testEnvironment, 0, tests);
runner.run(null);
- verify(monitor).outcomeStarted(JUnitRunner.class,
+ verify(monitor).outcomeStarted(
target.getName() + "#testSimple2");
verify(monitor).outcomeFinished(Result.SUCCESS);
}
@@ -111,9 +110,9 @@
Runner runner = new JUnitRunner(monitor, skipPastReference, testEnvironment, 0, tests);
runner.run(null);
- verify(monitor).outcomeStarted(JUnitRunner.class,
+ verify(monitor).outcomeStarted(
target.getName() + "#testSimple2");
- verify(monitor).outcomeStarted(JUnitRunner.class,
+ verify(monitor).outcomeStarted(
target.getName() + "#testSimple3");
verify(monitor, times(2)).outcomeFinished(Result.SUCCESS);
}
@@ -125,7 +124,7 @@
Runner runner = new JUnitRunner(monitor, skipPastReference, testEnvironment, 0, tests);
runner.run(null);
- verify(monitor).outcomeStarted(JUnitRunner.class,
+ verify(monitor).outcomeStarted(
target.getName() + "#testSimple2");
verify(monitor).outcomeFinished(Result.SUCCESS);
}
@@ -140,7 +139,7 @@
Runner runner = new JUnitRunner(monitor, skipPastReference, testEnvironment, 0, tests);
runner.run(null);
- verify(monitor).outcomeStarted(JUnitRunner.class,
+ verify(monitor).outcomeStarted(
target.getName() + "#testSimple5");
verify(monitor).outcomeFinished(Result.EXEC_FAILED);
@@ -157,9 +156,9 @@
Runner runner = new JUnitRunner(monitor, skipPastReference, testEnvironment, 0, tests);
runner.run(null);
- verify(monitor).outcomeStarted(JUnitRunner.class,
+ verify(monitor).outcomeStarted(
target.getName() + "#testSimple2");
- verify(monitor).outcomeStarted(JUnitRunner.class,
+ verify(monitor).outcomeStarted(
target.getName() + "#testSimple3");
verify(monitor, times(2)).outcomeFinished(Result.SUCCESS);
}
@@ -174,10 +173,10 @@
Runner runner = new JUnitRunner(monitor, skipPastReference, testEnvironment, 0, tests);
runner.run(null);
- verify(monitor).outcomeStarted(JUnitRunner.class,
+ verify(monitor).outcomeStarted(
target.getName() + "#testSuccess");
- verify(monitor).outcomeStarted(JUnitRunner.class, target.getName() + "#testFail");
- verify(monitor).outcomeStarted(JUnitRunner.class,
+ verify(monitor).outcomeStarted(target.getName() + "#testFail");
+ verify(monitor).outcomeStarted(
target.getName() + "#testThrowException");
verify(monitor).outcomeFinished(Result.SUCCESS);
verify(monitor, times(2)).outcomeFinished(Result.EXEC_FAILED);
@@ -198,7 +197,7 @@
Runner runner = new JUnitRunner(monitor, skipPastReference, testEnvironment, 1, tests);
runner.run(null);
- verify(monitor).outcomeStarted(JUnitRunner.class, target.getName() + "#test");
+ verify(monitor).outcomeStarted(target.getName() + "#test");
verify(monitor).outcomeFinished(Result.EXEC_FAILED);
String outStr = baos.toString();
diff --git a/test/vogar/target/TestRunnerProperties.java b/test/vogar/target/TestRunnerProperties.java
index 8fba605..58564b9 100644
--- a/test/vogar/target/TestRunnerProperties.java
+++ b/test/vogar/target/TestRunnerProperties.java
@@ -19,6 +19,7 @@
import java.lang.annotation.Retention;
import java.lang.annotation.RetentionPolicy;
import java.lang.annotation.Target;
+import vogar.RunnerType;
/**
* Allows test properties to be specified for tests.
@@ -45,7 +46,7 @@
String testClassOrPackage() default "";
- boolean testOnly() default false;
+ RunnerType runnerType() default RunnerType.DEFAULT;
int timeout() default 0;
diff --git a/test/vogar/target/TestRunnerRule.java b/test/vogar/target/TestRunnerRule.java
index 0f3c3e6..18b294b 100644
--- a/test/vogar/target/TestRunnerRule.java
+++ b/test/vogar/target/TestRunnerRule.java
@@ -57,7 +57,7 @@
}
}
setProperty(TestProperties.TEST_CLASS_OR_PACKAGE, testClassOrPackage);
- setProperty(TestProperties.TEST_ONLY, testRunnerProperties.testOnly());
+ setProperty(TestProperties.RUNNER_TYPE, testRunnerProperties.runnerType().toString());
setProperty(TestProperties.TIMEOUT, testRunnerProperties.timeout());
}
return base;
diff --git a/test/vogar/target/TestRunnerTest.java b/test/vogar/target/TestRunnerTest.java
index 7de4ef3..bcf9ca6 100644
--- a/test/vogar/target/TestRunnerTest.java
+++ b/test/vogar/target/TestRunnerTest.java
@@ -23,7 +23,7 @@
import org.junit.Test;
import org.junit.runner.RunWith;
import org.junit.runners.JUnit4;
-import vogar.target.junit.JUnitRunner;
+import vogar.RunnerType;
import vogar.testing.InterceptOutputStreams;
import vogar.testing.InterceptOutputStreams.Stream;
@@ -40,6 +40,20 @@
@Rule public TestRunnerRule testRunnerRule = new TestRunnerRule();
+ @TestRunnerProperties(testClass = JUnit3Test.class, runnerType = RunnerType.JUNIT)
+ @Test
+ public void testConstructor_JUnit3Test_RunnerType_JUNIT() throws Exception {
+ TestRunner runner = testRunnerRule.createTestRunner();
+ runner.run();
+
+ assertEquals(""
+ + "//00xx{\"outcome\":\"" + JUnit3Test.class.getName() + "#testMethodName\"}\n"
+ + "//00xx{\"result\":\"SUCCESS\"}\n"
+ + "//00xx{\"outcome\":\"" + JUnit3Test.class.getName() + "#testOtherName\"}\n"
+ + "//00xx{\"result\":\"SUCCESS\"}\n"
+ + "//00xx{\"completedNormally\":true}\n", ios.contents(Stream.OUT));
+ }
+
@TestRunnerProperties(testClass = JUnit3Test.class)
@Test
public void testConstructor_JUnit3Test() throws Exception {
@@ -47,11 +61,9 @@
runner.run();
assertEquals(""
- + "//00xx{\"outcome\":\"" + JUnit3Test.class.getName() + "#testMethodName\","
- + "\"runner\":\"" + JUnitRunner.class.getName() + "\"}\n"
+ + "//00xx{\"outcome\":\"" + JUnit3Test.class.getName() + "#testMethodName\"}\n"
+ "//00xx{\"result\":\"SUCCESS\"}\n"
- + "//00xx{\"outcome\":\"" + JUnit3Test.class.getName() + "#testOtherName\","
- + "\"runner\":\"" + JUnitRunner.class.getName() + "\"}\n"
+ + "//00xx{\"outcome\":\"" + JUnit3Test.class.getName() + "#testOtherName\"}\n"
+ "//00xx{\"result\":\"SUCCESS\"}\n"
+ "//00xx{\"completedNormally\":true}\n", ios.contents(Stream.OUT));
}
@@ -80,8 +92,7 @@
runner.run();
assertEquals(""
- + "//00xx{\"outcome\":\"" + JUnit3Test.class.getName() + "#testOtherName\","
- + "\"runner\":\"" + JUnitRunner.class.getName() + "\"}\n"
+ + "//00xx{\"outcome\":\"" + JUnit3Test.class.getName() + "#testOtherName\"}\n"
+ "//00xx{\"result\":\"SUCCESS\"}\n"
+ "//00xx{\"completedNormally\":true}\n", ios.contents(Stream.OUT));
}
@@ -94,7 +105,7 @@
}
}
- @TestRunnerProperties(testClass = CaliperBenchmark.class)
+ @TestRunnerProperties(testClass = CaliperBenchmark.class, runnerType = RunnerType.CALIPER)
@Test
public void testConstructor_CaliperBenchmark() throws Exception {
TestRunner runner = testRunnerRule.createTestRunner("-i", "runtime");
@@ -104,8 +115,7 @@
// Remove stack trace from output.
out = out.replaceAll("\t[^\n]+\\n", "");
assertEquals(""
- + "//00xx{\"outcome\":\"" + CaliperBenchmark.class.getName() + "\","
- + "\"runner\":\"" + CaliperRunner.class.getName() + "\"}\n"
+ + "//00xx{\"outcome\":\"" + CaliperBenchmark.class.getName() + "\"}\n"
+ "Experiment selection: \n"
+ " Benchmark Methods: [timeMethod]\n"
+ " Instruments: [runtime]\n"
@@ -128,7 +138,8 @@
* <p>Cannot check that profiling works because it will only work on Android and these tests
* do not run on android yet.
*/
- @TestRunnerProperties(testClass = CaliperBenchmark.class, profile = true)
+ @TestRunnerProperties(testClass = CaliperBenchmark.class, profile = true,
+ runnerType = RunnerType.CALIPER)
@Test
public void testConstructor_CaliperBenchmark_Profile() throws Exception {
TestRunner runner = testRunnerRule.createTestRunner("-i", "runtime");
@@ -146,8 +157,7 @@
out = out.replaceAll("\t[^\n]+\\n", "");
assertEquals(""
- + "//00xx{\"outcome\":\"" + CaliperBenchmark.class.getName() + "\","
- + "\"runner\":\"" + CaliperRunner.class.getName() + "\"}\n"
+ + "//00xx{\"outcome\":\"" + CaliperBenchmark.class.getName() + "\"}\n"
+ "Experiment selection: \n"
+ " Benchmark Methods: [timeMethod]\n"
+ " Instruments: [runtime]\n"
@@ -174,6 +184,18 @@
}
}
+ @TestRunnerProperties(testClass = Main.class, runnerType = RunnerType.MAIN)
+ @Test
+ public void testConstructor_Main_RunnerType_MAIN() throws Exception {
+ TestRunner runner = testRunnerRule.createTestRunner();
+ runner.run();
+
+ assertEquals(""
+ + "//00xx{\"outcome\":\"" + Main.class.getName() + "\"}\n"
+ + "//00xx{\"result\":\"SUCCESS\"}\n"
+ + "//00xx{\"completedNormally\":true}\n", ios.contents(Stream.OUT));
+ }
+
@TestRunnerProperties(testClass = Main.class)
@Test
public void testConstructor_Main() throws Exception {
@@ -181,8 +203,7 @@
runner.run();
assertEquals(""
- + "//00xx{\"outcome\":\"" + Main.class.getName() + "\","
- + "\"runner\":\"" + MainRunner.class.getName() + "\"}\n"
+ + "//00xx{\"outcome\":\"" + Main.class.getName() + "\"}\n"
+ "//00xx{\"result\":\"SUCCESS\"}\n"
+ "//00xx{\"completedNormally\":true}\n", ios.contents(Stream.OUT));
}
@@ -200,10 +221,7 @@
runner.run();
assertEquals(""
- + "Warning: Arguments are invalid for Caliper: "
- + "Extra stuff, did not expect non-option arguments: [" + methodName + "]\n"
- + "//00xx{\"outcome\":\"" + JUnit3Test.class.getName() + "#" + methodName + "\","
- + "\"runner\":\"" + JUnitRunner.class.getName() + "\"}\n"
+ + "//00xx{\"outcome\":\"" + JUnit3Test.class.getName() + "#" + methodName + "\"}\n"
+ "//00xx{\"result\":\"SUCCESS\"}\n"
+ "//00xx{\"completedNormally\":true}\n", ios.contents(Stream.OUT));
}
