Redefine compatibility test not_executed counts, improve setDone
Redefine not_executed in IModuleResult and IInvocationResult. Since multiple
test runs prevent true knowledge of the quantity of not-executed tests, this
number now indicates the maximum number of not-executed tests for the module,
assuming all test runs have started. For a module to be marked "done", it must
now complete every scheduled test run without failure or missing tests. Even
if all tests are ultimately collected in one invocation, a single test run
failure mandates an additional retry to mark the module done.
bug:33211104
bug:33380715
Test: run collect-tests-only, unplug during non dEQP module
Change-Id: I8675d269389d664a7019125d48fd39cf3ab11daf
diff --git a/common/host-side/tradefed/src/com/android/compatibility/common/tradefed/build/CompatibilityBuildHelper.java b/common/host-side/tradefed/src/com/android/compatibility/common/tradefed/build/CompatibilityBuildHelper.java
index 10fc053..235d71b 100644
--- a/common/host-side/tradefed/src/com/android/compatibility/common/tradefed/build/CompatibilityBuildHelper.java
+++ b/common/host-side/tradefed/src/com/android/compatibility/common/tradefed/build/CompatibilityBuildHelper.java
@@ -269,6 +269,13 @@
}
/**
+ * @return a {@link File} in the resultDir for counting expected test runs
+ */
+ public File getTestRunsFile() throws FileNotFoundException {
+ return new File(getResultDir(), "test_runs.txt");
+ }
+
+ /**
* @return a {@link String} to use for directory suffixes created from the given time.
*/
public static String getDirSuffix(long millis) {
diff --git a/common/host-side/tradefed/src/com/android/compatibility/common/tradefed/result/ConsoleReporter.java b/common/host-side/tradefed/src/com/android/compatibility/common/tradefed/result/ConsoleReporter.java
index 47c13e1..b9c0262 100644
--- a/common/host-side/tradefed/src/com/android/compatibility/common/tradefed/result/ConsoleReporter.java
+++ b/common/host-side/tradefed/src/com/android/compatibility/common/tradefed/result/ConsoleReporter.java
@@ -67,26 +67,17 @@
*/
@Override
public void testRunStarted(String id, int numTests) {
- if (mModuleId == null || !mModuleId.equals(id)) {
- mModuleId = id;
- mTotalTestsInModule = numTests;
- // Reset counters
- mCurrentTestNum = 0;
- mPassedTests = 0;
- mFailedTests = 0;
- mNotExecutedTests = 0;
- mTestFailed = false;
- logMessage("Starting %s with %d test%s",
- id, mTotalTestsInModule, (mTotalTestsInModule > 1) ? "s" : "");
- } else {
- if (mNotExecutedTests == 0) {
- mTotalTestsInModule += numTests;
- } else {
- mTotalTestsInModule += Math.max(0, numTests - mNotExecutedTests);
- }
- logMessage("Continuing %s with %d test%s",
- id, mTotalTestsInModule, (mTotalTestsInModule > 1) ? "s" : "");
- }
+ boolean isRepeatModule = (mModuleId != null && mModuleId.equals(id));
+ mModuleId = id;
+ mTotalTestsInModule = numTests;
+ // Reset counters
+ mCurrentTestNum = 0;
+ mPassedTests = 0;
+ mFailedTests = 0;
+ mNotExecutedTests = 0;
+ mTestFailed = false;
+ logMessage("%s %s with %d test%s", (isRepeatModule) ? "Continuing" : "Starting", id,
+ mTotalTestsInModule, (mTotalTestsInModule > 1) ? "s" : "");
}
/**
diff --git a/common/host-side/tradefed/src/com/android/compatibility/common/tradefed/result/ResultReporter.java b/common/host-side/tradefed/src/com/android/compatibility/common/tradefed/result/ResultReporter.java
index 93aa528..620fa0a 100644
--- a/common/host-side/tradefed/src/com/android/compatibility/common/tradefed/result/ResultReporter.java
+++ b/common/host-side/tradefed/src/com/android/compatibility/common/tradefed/result/ResultReporter.java
@@ -16,6 +16,8 @@
package com.android.compatibility.common.tradefed.result;
import com.android.compatibility.common.tradefed.build.CompatibilityBuildHelper;
+import com.android.compatibility.common.tradefed.result.InvocationFailureHandler;
+import com.android.compatibility.common.tradefed.result.TestRunHandler;
import com.android.compatibility.common.tradefed.testtype.CompatibilityTest;
import com.android.compatibility.common.tradefed.testtype.CompatibilityTest.RetryType;
import com.android.compatibility.common.util.ICaseResult;
@@ -251,27 +253,30 @@
*/
@Override
public void testRunStarted(String id, int numTests) {
- if (mCurrentModuleResult != null && mCurrentModuleResult.getId().equals(id)) {
- // In case we get another test run of a known module, update the complete
- // status to false to indicate it is not complete.
- if (mCurrentModuleResult.isDone()) {
- // modules run with HostTest treat each test class as a separate module.
- // TODO(aaronholden): remove this case when JarHostTest is no longer calls
- // testRunStarted for each test class.
- mTotalTestsInModule += numTests;
- } else {
- // treat new tests as not executed tests from current module
- mTotalTestsInModule +=
- Math.max(0, numTests - mCurrentModuleResult.getNotExecuted());
- }
+ if (mCurrentModuleResult != null && mCurrentModuleResult.getId().equals(id)
+ && mCurrentModuleResult.isDone()) {
+ // Modules run with JarHostTest treat each test class as a separate module,
+ // resulting in additional unexpected test runs.
+ // This case exists only for N
+ mTotalTestsInModule += numTests;
} else {
+ // Handle non-JarHostTest case
mCurrentModuleResult = mResult.getOrCreateModule(id);
- mTotalTestsInModule = numTests;
+ mModuleWasDone = mCurrentModuleResult.isDone();
+ if (!mModuleWasDone) {
+ // we only want to update testRun variables if the IModuleResult is not yet done
+ // otherwise leave testRun variables alone so isDone evaluates to true.
+ if (mCurrentModuleResult.getExpectedTestRuns() == 0) {
+ mCurrentModuleResult.setExpectedTestRuns(TestRunHandler.getTestRuns(
+ mBuildHelper, mCurrentModuleResult.getId()));
+ }
+ mCurrentModuleResult.addTestRun();
+ }
// Reset counters
+ mTotalTestsInModule = numTests;
mCurrentTestNum = 0;
}
- mModuleWasDone = mCurrentModuleResult.isDone();
- mCurrentModuleResult.setDone(false);
+ mCurrentModuleResult.inProgress(true);
}
/**
@@ -355,17 +360,30 @@
*/
@Override
public void testRunEnded(long elapsedTime, Map<String, String> metrics) {
+ mCurrentModuleResult.inProgress(false);
mCurrentModuleResult.addRuntime(elapsedTime);
- if (mCanMarkDone || mModuleWasDone) {
- // Only mark module done if status of the invocation allows it (mCanMarkDone) or module
- // was previously marked done (mModuleWasDone) and all expected tests are collected.
- // Expect mCurrentTestNum = mTotalTestsInModule, but use >= to be safe
- mCurrentModuleResult.setDone(mCurrentTestNum >= mTotalTestsInModule);
+ if (!mModuleWasDone) {
+ // Not executed count now represents an upper-bound for a fix to b/33211104.
+ // Only setNotExecuted this number if the module has already been completely executed.
+ int testCountDiff = Math.max(mTotalTestsInModule - mCurrentTestNum, 0);
+ if (isShardResultReporter()) {
+ // reset value, which is added to total count for master shard upon merge
+ mCurrentModuleResult.setNotExecuted(testCountDiff);
+ } else {
+ // increment value for master shard
+ mCurrentModuleResult.setNotExecuted(mCurrentModuleResult.getNotExecuted()
+ + testCountDiff);
+ }
+ if (mCanMarkDone) {
+ // Only mark module done if status of the invocation allows it (mCanMarkDone) and
+ // if module has not already been marked done.
+ mCurrentModuleResult.setDone(mCurrentTestNum >= mTotalTestsInModule);
+ }
}
- mCurrentModuleResult.setNotExecuted(Math.max(mTotalTestsInModule - mCurrentTestNum, 0));
if (isShardResultReporter()) {
// Forward module results to the master.
mMasterResultReporter.mergeModuleResult(mCurrentModuleResult);
+ mCurrentModuleResult.resetTestRuns();
}
}
diff --git a/common/host-side/tradefed/src/com/android/compatibility/common/tradefed/result/SubPlanCreator.java b/common/host-side/tradefed/src/com/android/compatibility/common/tradefed/result/SubPlanCreator.java
index 6b9b5e4..9dbbcbb 100644
--- a/common/host-side/tradefed/src/com/android/compatibility/common/tradefed/result/SubPlanCreator.java
+++ b/common/host-side/tradefed/src/com/android/compatibility/common/tradefed/result/SubPlanCreator.java
@@ -65,14 +65,6 @@
STATUS_MAP = Collections.unmodifiableMap(statusMap);
}
- // TODO(aaronholden): remove this temporary workaround for b/33090757
- private static final Set<String> MULTITEST_MODULES;
- static {
- Set<String> multiTestModuleSet = new HashSet<String>();
- multiTestModuleSet.add("CtsDeqpTestCases");
- MULTITEST_MODULES = Collections.unmodifiableSet(multiTestModuleSet);
- }
-
@Option (name = "name", shortName = 'n', description = "the name of the subplan to create",
importance=Importance.IF_UNSET)
private String mSubPlanName = null;
@@ -192,49 +184,7 @@
subPlan.addIncludeFilter(new TestFilter(mAbiName, mModuleName, mTestName).toString());
}
Set<TestStatus> statusesToRun = getStatusesToRun();
-
for (IModuleResult module : mResult.getModules()) {
-
- // TODO(aaronholden): remove this special case from SubPlanCreator, and filter
- // individual tests only when the module should run. Tracked by b/33211104
- if (MULTITEST_MODULES.contains(module.getName())) {
- // cannot check module.isDone() since this value is not accurate for modules
- // with multiple test configs. If we should run not-executed tests, include module
- // and exclude tests with status not in mResultTypes.
- TestFilter moduleFilter =
- new TestFilter(module.getAbi(), module.getName(), null /*test*/);
- if (mResultTypes.contains(NOT_EXECUTED)) {
- subPlan.addIncludeFilter(moduleFilter.toString());
- for (ICaseResult caseResult : module.getResults()) {
- for (ITestResult testResult : caseResult.getResults()) {
- if (!statusesToRun.contains(testResult.getResultStatus())) {
- TestFilter testExclude = new TestFilter(module.getAbi(),
- module.getName(), testResult.getFullName());
- subPlan.addExcludeFilter(testExclude.toString());
- }
- }
- }
- } else {
- // not running not-executed tests, only include executed tests
- if (shouldRunModule(module)) {
- // at least some executed tests will be included
- for (ICaseResult caseResult : module.getResults()) {
- for (ITestResult testResult : caseResult.getResults()) {
- if (statusesToRun.contains(testResult.getResultStatus())) {
- TestFilter testInclude = new TestFilter(module.getAbi(),
- module.getName(), testResult.getFullName());
- subPlan.addIncludeFilter(testInclude.toString());
- }
- }
- }
- } else {
- // no executed tests will be included, so exclude entire module
- subPlan.addExcludeFilter(moduleFilter.toString());
- }
- }
- continue;
- }
-
if (shouldRunModule(module)) {
TestFilter moduleInclude =
new TestFilter(module.getAbi(), module.getName(), null /*test*/);
diff --git a/common/host-side/tradefed/src/com/android/compatibility/common/tradefed/result/TestRunHandler.java b/common/host-side/tradefed/src/com/android/compatibility/common/tradefed/result/TestRunHandler.java
new file mode 100644
index 0000000..4473eb8
--- /dev/null
+++ b/common/host-side/tradefed/src/com/android/compatibility/common/tradefed/result/TestRunHandler.java
@@ -0,0 +1,97 @@
+/*
+ * Copyright (C) 2016 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.android.compatibility.common.tradefed.result;
+
+import com.android.compatibility.common.tradefed.build.CompatibilityBuildHelper;
+import com.android.tradefed.log.LogUtil.CLog;
+import com.android.tradefed.util.FileUtil;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.Map;
+
+
+/**
+ * A helper class for setting and checking the number of expected test runs.
+ */
+public class TestRunHandler {
+
+ private static final String MAP_DELIMITER = "->";
+
+ /**
+ * Determine the number of expected test runs for the module
+ *
+ * @param buildHelper the {@link CompatibilityBuildHelper} from which to retrieve invocation
+ * failure file
+ * @return the number of expected test runs, or 1 if module is not found
+ */
+ public static int getTestRuns(final CompatibilityBuildHelper buildHelper, String id) {
+ try {
+ File f = buildHelper.getTestRunsFile();
+ if (!f.exists() || f.length() == 0) {
+ return 1; // test runs file doesn't exist, expect one test run by default
+ }
+ String mapString = FileUtil.readStringFromFile(f);
+ Map<String, Integer> map = stringToMap(mapString);
+ Integer testRuns = map.get(id);
+ return (testRuns == null) ? 1 : testRuns;
+ } catch (IOException e) {
+ CLog.e("Could not read test run file for session %s",
+ buildHelper.getDirSuffix(buildHelper.getStartTime()));
+ CLog.e(e);
+ return 1;
+ }
+ }
+
+ /**
+ * Write the number of expected test runs to the result's test run file.
+ *
+ * @param buildHelper the {@link CompatibilityBuildHelper} used to write the
+ * test run file
+ * @param testRuns a mapping of module names to number of test runs expected
+ */
+ public static void setTestRuns(final CompatibilityBuildHelper buildHelper,
+ Map<String, Integer> testRuns) {
+ try {
+ File f = buildHelper.getTestRunsFile();
+ if (!f.exists()) {
+ f.createNewFile();
+ }
+ FileUtil.writeToFile(mapToString(testRuns), f);
+ } catch (IOException e) {
+ CLog.e("Exception while writing test runs file.");
+ CLog.e(e);
+ }
+ }
+
+ private static String mapToString(Map<String, Integer> map) {
+ StringBuilder sb = new StringBuilder("");
+ for (Map.Entry<String, Integer> entry : map.entrySet()) {
+ sb.append(String.format("%s%s%d\n", entry.getKey(), MAP_DELIMITER, entry.getValue()));
+ }
+ return sb.toString();
+ }
+
+ private static Map<String, Integer> stringToMap(String str) {
+ Map<String, Integer> map = new HashMap<>();
+ for (String entry : str.split("\n")) {
+ String[] parts = entry.split(MAP_DELIMITER);
+ map.put(parts[0], Integer.parseInt(parts[1]));
+ }
+ return map;
+ }
+}
diff --git a/common/host-side/tradefed/src/com/android/compatibility/common/tradefed/testtype/CompatibilityTest.java b/common/host-side/tradefed/src/com/android/compatibility/common/tradefed/testtype/CompatibilityTest.java
index 891a0ae..22d02ca 100644
--- a/common/host-side/tradefed/src/com/android/compatibility/common/tradefed/testtype/CompatibilityTest.java
+++ b/common/host-side/tradefed/src/com/android/compatibility/common/tradefed/testtype/CompatibilityTest.java
@@ -50,6 +50,7 @@
import com.android.tradefed.result.ITestInvocationListener;
import com.android.tradefed.result.InputStreamSource;
import com.android.tradefed.result.LogDataType;
+import com.android.tradefed.result.ResultForwarder;
import com.android.tradefed.targetprep.ITargetPreparer;
import com.android.tradefed.testtype.IAbi;
import com.android.tradefed.testtype.IBuildReceiver;
@@ -70,6 +71,7 @@
import java.io.PrintWriter;
import java.util.ArrayList;
import java.util.Collection;
+import java.util.Collections;
import java.util.HashSet;
import java.util.LinkedList;
import java.util.List;
@@ -422,8 +424,14 @@
// execute pre module execution checker
runPreModuleCheck(module.getName(), checkers, mDevice, listener);
+ // Workaround to b/34202787: Add result forwarder that ensures module is reported
+ // with 0 tests if test runner doesn't report anything in this case.
+ // Necessary for solution to b/33289177, in which completed modules may sometimes
+ // not be marked done until retried with 0 tests.
+ ModuleResultForwarder moduleListener = new ModuleResultForwarder(listener);
try {
- module.run(listener);
+ module.run(moduleListener);
+ moduleListener.finish(module.getId());
} catch (DeviceUnresponsiveException due) {
// being able to catch a DeviceUnresponsiveException here implies that recovery
// was successful, and test execution should proceed to next module
@@ -719,4 +727,31 @@
return shardQueue;
}
+ private class ModuleResultForwarder extends ResultForwarder {
+
+ private boolean mTestRunStarted = false;
+ private ITestInvocationListener mListener;
+
+ public ModuleResultForwarder(ITestInvocationListener listener) {
+ super(listener);
+ mListener = listener;
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ public void testRunStarted(String name, int numTests) {
+ mListener.testRunStarted(name, numTests);
+ mTestRunStarted = true;
+ }
+
+ public void finish(String moduleId) {
+ if (!mTestRunStarted) {
+ mListener.testRunStarted(moduleId, 0);
+ mListener.testRunEnded(0, Collections.emptyMap());
+ }
+ }
+ }
+
}
diff --git a/common/host-side/tradefed/src/com/android/compatibility/common/tradefed/testtype/ModuleRepo.java b/common/host-side/tradefed/src/com/android/compatibility/common/tradefed/testtype/ModuleRepo.java
index a2287d4..78995f0 100644
--- a/common/host-side/tradefed/src/com/android/compatibility/common/tradefed/testtype/ModuleRepo.java
+++ b/common/host-side/tradefed/src/com/android/compatibility/common/tradefed/testtype/ModuleRepo.java
@@ -15,6 +15,8 @@
*/
package com.android.compatibility.common.tradefed.testtype;
+import com.android.compatibility.common.tradefed.build.CompatibilityBuildHelper;
+import com.android.compatibility.common.tradefed.result.TestRunHandler;
import com.android.compatibility.common.util.AbiUtils;
import com.android.compatibility.common.util.TestFilter;
import com.android.ddmlib.Log.LogLevel;
@@ -271,6 +273,7 @@
throw new IllegalArgumentException(
String.format("No config files found in %s", testsDir.getAbsolutePath()));
}
+ Map<String, Integer> shardedTestCounts = new HashMap<>();
for (File configFile : configFiles) {
final String name = configFile.getName().replace(CONFIG_EXT, "");
final String[] pathArg = new String[] { configFile.getAbsolutePath() };
@@ -318,6 +321,9 @@
if (mShards > 1) {
shardedTests = splitShardableTests(tests, buildInfo);
}
+ if (shardedTests.size() > 1) {
+ shardedTestCounts.put(id, shardedTests.size());
+ }
for (IRemoteTest test : shardedTests) {
if (test instanceof IBuildReceiver) {
((IBuildReceiver)test).setBuild(buildInfo);
@@ -330,6 +336,7 @@
configFile.getName()), e);
}
}
+ TestRunHandler.setTestRuns(new CompatibilityBuildHelper(buildInfo), shardedTestCounts);
mModulesPerShard = mModuleCount / shards;
if (mModuleCount % shards != 0) {
mModulesPerShard++; // Round up
diff --git a/common/host-side/tradefed/tests/src/com/android/compatibility/common/tradefed/result/ConsoleReporterTest.java b/common/host-side/tradefed/tests/src/com/android/compatibility/common/tradefed/result/ConsoleReporterTest.java
index 928379e..942ae70 100644
--- a/common/host-side/tradefed/tests/src/com/android/compatibility/common/tradefed/result/ConsoleReporterTest.java
+++ b/common/host-side/tradefed/tests/src/com/android/compatibility/common/tradefed/result/ConsoleReporterTest.java
@@ -107,15 +107,6 @@
assertEquals(0, mReporter.getPassedTests());
assertEquals(0, mReporter.getCurrentTestNum());
assertEquals(3, mReporter.getTotalTestsInModule());
-
- runTests();
- // Same id, should not reset test counters, but aggregate total tests
- mReporter.testRunStarted(ID2, 5);
- assertEquals(ID2, mReporter.getModuleId());
- assertEquals(2, mReporter.getFailedTests());
- assertEquals(1, mReporter.getPassedTests());
- assertEquals(3, mReporter.getCurrentTestNum());
- assertEquals(8, mReporter.getTotalTestsInModule());
}
/** Run 4 test, but one is ignored */
diff --git a/common/host-side/tradefed/tests/src/com/android/compatibility/common/tradefed/result/ResultReporterTest.java b/common/host-side/tradefed/tests/src/com/android/compatibility/common/tradefed/result/ResultReporterTest.java
index 9e76e54..2db60bd 100644
--- a/common/host-side/tradefed/tests/src/com/android/compatibility/common/tradefed/result/ResultReporterTest.java
+++ b/common/host-side/tradefed/tests/src/com/android/compatibility/common/tradefed/result/ResultReporterTest.java
@@ -281,7 +281,7 @@
// Set up IInvocationResult with existing results from previous session
IInvocationResult invocationResult = mReporter.getResult();
IModuleResult moduleResult = invocationResult.getOrCreateModule(ID);
- moduleResult.setDone(false);
+ moduleResult.initializeDone(false);
ICaseResult caseResult = moduleResult.getOrCreateResult(CLASS);
ITestResult testResult1 = caseResult.getOrCreateResult(METHOD_1);
testResult1.setResultStatus(TestStatus.PASS);
diff --git a/common/host-side/tradefed/tests/src/com/android/compatibility/common/tradefed/testtype/ModuleRepoTest.java b/common/host-side/tradefed/tests/src/com/android/compatibility/common/tradefed/testtype/ModuleRepoTest.java
index d54277b..a0f248f 100644
--- a/common/host-side/tradefed/tests/src/com/android/compatibility/common/tradefed/testtype/ModuleRepoTest.java
+++ b/common/host-side/tradefed/tests/src/com/android/compatibility/common/tradefed/testtype/ModuleRepoTest.java
@@ -16,6 +16,7 @@
package com.android.compatibility.common.tradefed.testtype;
+import com.android.compatibility.common.tradefed.build.CompatibilityBuildHelper;
import com.android.compatibility.common.tradefed.build.CompatibilityBuildProvider;
import com.android.compatibility.common.tradefed.testtype.ModuleRepo.ConfigFilter;
import com.android.compatibility.common.tradefed.testtype.IModuleDef;
@@ -64,6 +65,9 @@
private static final Set<String> EXCLUDES = new HashSet<>();
private static final Set<String> FILES = new HashSet<>();
private static final String FILENAME = "%s.config";
+ private static final String ROOT_DIR_ATTR = "ROOT_DIR";
+ private static final String SUITE_NAME_ATTR = "SUITE_NAME";
+ private static final String START_TIME_MS_ATTR = "START_TIME_MS";
private static final String ABI_32 = "armeabi-v7a";
private static final String ABI_64 = "arm64-v8a";
private static final String MODULE_NAME_A = "FooModuleA";
@@ -105,6 +109,7 @@
}
private IModuleRepo mRepo;
private File mTestsDir;
+ private File mRootDir;
private IBuildInfo mBuild;
@Override
@@ -112,6 +117,15 @@
mTestsDir = setUpConfigs();
mRepo = new ModuleRepo();
mBuild = new CompatibilityBuildProvider().getBuild();
+ // Flesh out the result directory structure so ModuleRepo can write to the test runs file
+ mRootDir = FileUtil.createTempDir("root");
+ mBuild.addBuildAttribute(ROOT_DIR_ATTR, mRootDir.getAbsolutePath());
+ mBuild.addBuildAttribute(SUITE_NAME_ATTR, "suite");
+ mBuild.addBuildAttribute(START_TIME_MS_ATTR, Long.toString(0));
+ File subRootDir = new File(mRootDir, String.format("android-suite"));
+ File resultsDir = new File(subRootDir, "results");
+ File resultDir = new File(resultsDir, CompatibilityBuildHelper.getDirSuffix(0));
+ resultDir.mkdirs();
}
private File setUpConfigs() throws IOException {
@@ -138,6 +152,7 @@
@Override
public void tearDown() throws Exception {
tearDownConfigs(mTestsDir);
+ tearDownConfigs(mRootDir);
}
private void tearDownConfigs(File testsDir) {
diff --git a/common/host-side/util/src/com/android/compatibility/common/util/ResultHandler.java b/common/host-side/util/src/com/android/compatibility/common/util/ResultHandler.java
index b3ca245..89ec2d4 100644
--- a/common/host-side/util/src/com/android/compatibility/common/util/ResultHandler.java
+++ b/common/host-side/util/src/com/android/compatibility/common/util/ResultHandler.java
@@ -86,7 +86,7 @@
private static final String LOG_URL_ATTR = "log_url";
private static final String MESSAGE_ATTR = "message";
private static final String MODULE_TAG = "Module";
- private static final String MODULES_EXECUTED_ATTR = "modules_done";
+ private static final String MODULES_DONE_ATTR = "modules_done";
private static final String MODULES_TOTAL_ATTR = "modules_total";
private static final String NAME_ATTR = "name";
private static final String NOT_EXECUTED_ATTR = "not_executed";
@@ -185,9 +185,9 @@
String moduleId = AbiUtils.createId(abi, name);
boolean done = Boolean.parseBoolean(parser.getAttributeValue(NS, DONE_ATTR));
IModuleResult module = invocation.getOrCreateModule(moduleId);
- module.setDone(done);
- int notExecuted =
- Integer.parseInt(parser.getAttributeValue(NS, NOT_EXECUTED_ATTR));
+ module.initializeDone(done);
+ int notExecuted = Integer.parseInt(
+ parser.getAttributeValue(NS, NOT_EXECUTED_ATTR));
module.setNotExecuted(notExecuted);
long runtime = Long.parseLong(parser.getAttributeValue(NS, RUNTIME_ATTR));
module.addRuntime(runtime);
@@ -242,7 +242,7 @@
&& !checksumReporter.containsModuleResult(
module, invocation.getBuildFingerprint());
if (checksumMismatch) {
- module.setDone(false);
+ module.initializeDone(false);
}
}
parser.require(XmlPullParser.END_TAG, NS, RESULT_TAG);
@@ -355,7 +355,7 @@
serializer.attribute(NS, PASS_ATTR, Integer.toString(passed));
serializer.attribute(NS, FAILED_ATTR, Integer.toString(failed));
serializer.attribute(NS, NOT_EXECUTED_ATTR, Integer.toString(notExecuted));
- serializer.attribute(NS, MODULES_EXECUTED_ATTR,
+ serializer.attribute(NS, MODULES_DONE_ATTR,
Integer.toString(result.getModuleCompleteCount()));
serializer.attribute(NS, MODULES_TOTAL_ATTR,
Integer.toString(result.getModules().size()));
diff --git a/common/util/src/com/android/compatibility/common/util/IModuleResult.java b/common/util/src/com/android/compatibility/common/util/IModuleResult.java
index 7c6279c..06d66c0 100644
--- a/common/util/src/com/android/compatibility/common/util/IModuleResult.java
+++ b/common/util/src/com/android/compatibility/common/util/IModuleResult.java
@@ -32,17 +32,91 @@
long getRuntime();
- boolean isDone();
-
- void setDone(boolean done);
-
- boolean isPassed();
-
+ /**
+ * Get the estimate of not-executed tests for this module. This estimate is a maximum
+ * not-executed count, assuming all test runs have been started.
+ * @return estimate of not-executed tests
+ */
int getNotExecuted();
+ /**
+ * Set the estimate of not-executed tests for this module. This estimate is a maximum
+ * not-executed count, assuming all test runs have been started.
+ * @param estimate of not-executed tests
+ */
void setNotExecuted(int numTests);
/**
+ * Whether all expected tests have been executed and all expected test runs have been seen
+ * and completed.
+ *
+ * @return the comprehensive completeness status of the module
+ */
+ boolean isDone();
+
+ /**
+ * Whether all expected tests have been executed for the test runs seen so far.
+ *
+ * @return the completeness status of the module so far
+ */
+ boolean isDoneSoFar();
+
+ /**
+ * Explicitly sets the "done" status for this module. To be used when constructing this
+ * instance from an XML report. The done status for an {@link IModuleResult} can be changed
+ * indiscriminately by method setDone(boolean) immediately after a call to initializeDone,
+ * whereas the status may only be switched to false immediately after a call to setDone.
+ *
+ * @param done the initial completeness status of the module
+ */
+ void initializeDone(boolean done);
+
+ /**
+ * Sets the "done" status for this module. To be used after each test run for the module.
+ * After setDone is used once, subsequent calls to setDone will AND the given value with the
+ * existing done status value. Thus a module with "done" already set to false cannot be marked
+ * done unless re-initialized (see initializeDone).
+ *
+ * @param done the completeness status of the module for a test run
+ */
+ void setDone(boolean done);
+
+ /**
+ * Sets the "in-progress" status for this module. Useful for tracking completion of the module
+ * in the case that a test run begins but never ends.
+ *
+ * @param inProgress whether the module is currently in progress
+ */
+ void inProgress(boolean inProgress);
+
+ /**
+ * @return the number of expected test runs for this module in this invocation
+ */
+ int getExpectedTestRuns();
+
+ /**
+ * @param the number of expected test runs for this module in this invocation
+ */
+ void setExpectedTestRuns(int numRuns);
+
+ /**
+ * @return the number of test runs seen for this module in this invocation
+ */
+ int getTestRuns();
+
+ /**
+ * Adds to the count of test runs seen for this module in this invocation
+ */
+ void addTestRun();
+
+ /**
+ * Reset the count of test runs seen for this module in this invocation. Should be performed
+ * after merging the module into another module, so that future merges do not double-count the
+ * same test runs.
+ */
+ void resetTestRuns();
+
+ /**
* Gets a {@link ICaseResult} for the given testcase, creating it if it doesn't exist.
*
* @param caseName the name of the testcase eg <package-name><class-name>
diff --git a/common/util/src/com/android/compatibility/common/util/ModuleResult.java b/common/util/src/com/android/compatibility/common/util/ModuleResult.java
index 0c43e93..60038cf 100644
--- a/common/util/src/com/android/compatibility/common/util/ModuleResult.java
+++ b/common/util/src/com/android/compatibility/common/util/ModuleResult.java
@@ -28,7 +28,13 @@
private String mId;
private long mRuntime = 0;
+
+ /* Variables related to completion of the module */
private boolean mDone = false;
+ private boolean mHaveSetDone = false;
+ private boolean mInProgress = false;
+ private int mExpectedTestRuns = 0;
+ private int mActualTestRuns = 0;
private int mNotExecuted = 0;
private Map<String, ICaseResult> mResults = new HashMap<>();
@@ -46,7 +52,27 @@
*/
@Override
public boolean isDone() {
- return mDone;
+ return mDone && !mInProgress && (mActualTestRuns >= mExpectedTestRuns);
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ public boolean isDoneSoFar() {
+ return mDone && !mInProgress;
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ public void initializeDone(boolean done) {
+ mDone = done;
+ mHaveSetDone = false;
+ if (mDone) {
+ mNotExecuted = 0;
+ }
}
/**
@@ -54,15 +80,63 @@
*/
@Override
public void setDone(boolean done) {
- mDone = done;
+ if (mHaveSetDone) {
+ mDone &= done; // If we've already set done for this instance, AND the received value
+ } else {
+ mDone = done; // If done has only been initialized, overwrite the existing value
+ }
+ mHaveSetDone = true;
+ if (mDone) {
+ mNotExecuted = 0;
+ }
}
/**
* {@inheritDoc}
*/
@Override
- public boolean isPassed() {
- return mDone && countResults(TestStatus.FAIL) == 0;
+ public void inProgress(boolean inProgress) {
+ mInProgress = inProgress;
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ public int getExpectedTestRuns() {
+ return mExpectedTestRuns;
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ public void setExpectedTestRuns(int numRuns) {
+ mExpectedTestRuns = numRuns;
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ public int getTestRuns() {
+ return mActualTestRuns;
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ public void addTestRun() {
+ mActualTestRuns++;
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ public void resetTestRuns() {
+ mActualTestRuns = 0;
}
/**
@@ -184,7 +258,12 @@
}
this.mRuntime += otherModuleResult.getRuntime();
- this.mDone = otherModuleResult.isDone();
+ this.mNotExecuted += otherModuleResult.getNotExecuted();
+ this.setDone(otherModuleResult.isDoneSoFar());
+ this.mActualTestRuns += otherModuleResult.getTestRuns();
+ // expected test runs are the same across shards, except for shards that do not run this
+ // module at least once (for which the value is not yet set).
+ this.mExpectedTestRuns = otherModuleResult.getExpectedTestRuns();
for (ICaseResult otherCaseResult : otherModuleResult.getResults()) {
ICaseResult caseResult = getOrCreateResult(otherCaseResult.getName());
caseResult.mergeFrom(otherCaseResult);
