When a DNAE occurs report the non-executed tests
Allow test types to report their non-executed tests if all
the tests poller stops and some tests cannot be executed.
Test: unit tests
run cts --shard-count 2 then disconnect all devices
Bug: 111141078
Change-Id: If82bed881b6f774a53ca412ad2e6c4f1e55be787
Merged-In: If82bed881b6f774a53ca412ad2e6c4f1e55be787
diff --git a/src/com/android/tradefed/invoker/shard/TestsPoolPoller.java b/src/com/android/tradefed/invoker/shard/TestsPoolPoller.java
index e3a6cb5..2a50498 100644
--- a/src/com/android/tradefed/invoker/shard/TestsPoolPoller.java
+++ b/src/com/android/tradefed/invoker/shard/TestsPoolPoller.java
@@ -36,8 +36,10 @@
import com.android.tradefed.testtype.IInvocationContextReceiver;
import com.android.tradefed.testtype.IMultiDeviceTest;
import com.android.tradefed.testtype.IRemoteTest;
+import com.android.tradefed.testtype.IReportNotExecuted;
import com.android.tradefed.testtype.ITestCollector;
import com.android.tradefed.util.StreamUtil;
+import com.android.tradefed.util.TimeUtil;
import java.util.Collection;
import java.util.HashMap;
@@ -162,6 +164,11 @@
}
} finally {
mTracker.countDown();
+ if (mTracker.getCount() == 0) {
+ // If the last poller is also disconnected we want to know about the tests that
+ // did not execute.
+ reportNotExecuted(listener);
+ }
}
}
@@ -173,7 +180,9 @@
throws DeviceNotAvailableException {
try {
if (mTracker.getCount() > 1) {
- CLog.d("Wait 5 min for device to maybe coming back online.");
+ CLog.d(
+ "Wait %s for device to maybe come back online.",
+ TimeUtil.formatElapsedTime(WAIT_RECOVERY_TIME));
mDevice.waitForDeviceAvailable(WAIT_RECOVERY_TIME);
mDevice.reboot();
CLog.d("TestPoller was recovered after %s went offline", mDevice.getSerialNumber());
@@ -196,6 +205,21 @@
throw originalException;
}
+ /** Go through the remaining IRemoteTest and report them as not executed. */
+ private void reportNotExecuted(ITestInvocationListener listener) {
+ IRemoteTest test = poll();
+ while (test != null) {
+ if (test instanceof IReportNotExecuted) {
+ ((IReportNotExecuted) test).reportNotExecuted(listener);
+ } else {
+ CLog.e(
+ "Could not report not executed tests from %s.",
+ test.getClass().getCanonicalName());
+ }
+ test = poll();
+ }
+ }
+
/** Helper to log the device events. */
private void logDeviceEvent(EventType event, String serial, Throwable t) {
Map<String, String> args = new HashMap<>();
diff --git a/src/com/android/tradefed/testtype/IReportNotExecuted.java b/src/com/android/tradefed/testtype/IReportNotExecuted.java
new file mode 100644
index 0000000..941f7f3
--- /dev/null
+++ b/src/com/android/tradefed/testtype/IReportNotExecuted.java
@@ -0,0 +1,35 @@
+/*
+ * Copyright (C) 2018 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.android.tradefed.testtype;
+
+import com.android.tradefed.result.ITestInvocationListener;
+
+/**
+ * In case of an incomplete execution, {@link IRemoteTest} that implements this interface may report
+ * their non-executed tests for improved reporting.
+ */
+public interface IReportNotExecuted {
+
+ public static final String NOT_EXECUTED_FAILURE = "Test did not run. This is a placeholder.";
+
+ /**
+ * Report the non-executed tests to the main listener provided. They should be reported as
+ * failed with the {@link #NOT_EXECUTED_FAILURE} message.
+ *
+ * @param listener the main listener where to report the non-executed results.
+ */
+ public void reportNotExecuted(ITestInvocationListener listener);
+}
diff --git a/src/com/android/tradefed/testtype/suite/GranularRetriableTestWrapper.java b/src/com/android/tradefed/testtype/suite/GranularRetriableTestWrapper.java
index 4cdebe6..889d27b 100644
--- a/src/com/android/tradefed/testtype/suite/GranularRetriableTestWrapper.java
+++ b/src/com/android/tradefed/testtype/suite/GranularRetriableTestWrapper.java
@@ -270,7 +270,7 @@
* the IRemoteTest should already has the subset of testcases identified.
*/
@VisibleForTesting
- final void intraModuleRun() throws DeviceNotAvailableException, DeviceUnresponsiveException {
+ final void intraModuleRun() throws DeviceNotAvailableException {
ITestInvocationListener runListener = prepareRunListener();
try {
mTest.run(runListener);
@@ -288,8 +288,6 @@
CLog.w(due);
CLog.w("Proceeding to the next test.");
runListener.testRunFailed(due.getMessage());
- } catch (DeviceNotAvailableException dnae) {
- throw dnae;
} finally {
ModuleListener currentModuleListener =
mModuleListenerCollector.get(mModuleListenerCollector.size() - 1);
diff --git a/src/com/android/tradefed/testtype/suite/ITestSuite.java b/src/com/android/tradefed/testtype/suite/ITestSuite.java
index c72ba11..f6c97a4 100644
--- a/src/com/android/tradefed/testtype/suite/ITestSuite.java
+++ b/src/com/android/tradefed/testtype/suite/ITestSuite.java
@@ -50,6 +50,7 @@
import com.android.tradefed.testtype.IInvocationContextReceiver;
import com.android.tradefed.testtype.IMultiDeviceTest;
import com.android.tradefed.testtype.IRemoteTest;
+import com.android.tradefed.testtype.IReportNotExecuted;
import com.android.tradefed.testtype.IRuntimeHintProvider;
import com.android.tradefed.testtype.IShardableTest;
import com.android.tradefed.testtype.ITestCollector;
@@ -86,7 +87,8 @@
IInvocationContextReceiver,
IRuntimeHintProvider,
IMetricCollectorReceiver,
- IConfigurationReceiver {
+ IConfigurationReceiver,
+ IReportNotExecuted {
public static final String SKIP_SYSTEM_STATUS_CHECKER = "skip-system-status-check";
public static final String RUNNER_WHITELIST = "runner-whitelist";
@@ -792,6 +794,21 @@
mMainConfiguration = configuration;
}
+ /** {@inheritDoc} */
+ @Override
+ public void reportNotExecuted(ITestInvocationListener listener) {
+ List<ModuleDefinition> runModules = createExecutionList();
+
+ while (!runModules.isEmpty()) {
+ ModuleDefinition module = runModules.remove(0);
+ listener.testModuleStarted(module.getModuleInvocationContext());
+ listener.testRunStarted(module.getId(), 0);
+ listener.testRunFailed(IReportNotExecuted.NOT_EXECUTED_FAILURE);
+ listener.testRunEnded(0, new HashMap<String, Metric>());
+ listener.testModuleEnded();
+ }
+ }
+
/**
* Returns the {@link ModuleDefinition} to be executed directly, or null if none yet (when the
* ITestSuite has not been sharded yet).
diff --git a/tests/src/com/android/tradefed/invoker/shard/TestsPoolPollerTest.java b/tests/src/com/android/tradefed/invoker/shard/TestsPoolPollerTest.java
index 53cec94..8686306 100644
--- a/tests/src/com/android/tradefed/invoker/shard/TestsPoolPollerTest.java
+++ b/tests/src/com/android/tradefed/invoker/shard/TestsPoolPollerTest.java
@@ -29,7 +29,10 @@
import com.android.tradefed.metrics.proto.MetricMeasurement.Metric;
import com.android.tradefed.result.ITestInvocationListener;
import com.android.tradefed.testtype.IRemoteTest;
+import com.android.tradefed.testtype.IReportNotExecuted;
import com.android.tradefed.testtype.StubTest;
+import com.android.tradefed.testtype.suite.ITestSuite;
+import com.android.tradefed.testtype.suite.ITestSuiteTest.TestSuiteImpl;
import org.junit.Before;
import org.junit.Test;
@@ -213,6 +216,56 @@
}
/**
+ * Test that a runner that implements {@link IReportNotExecuted} can report the non-executed
+ * tests when the DNAE occurs.
+ */
+ @Test
+ public void testRun_dnae_reportNotExecuted() throws Exception {
+ List<IRemoteTest> testsList = new ArrayList<>();
+ // Add one bad test first that will throw an exception.
+ IRemoteTest badTest = new StubTest();
+ OptionSetter setter = new OptionSetter(badTest);
+ setter.setOptionValue("test-throw-not-available", "true");
+ testsList.add(badTest);
+ // Add tests that from a suite that can report their not executed tests.
+ int numTests = 5;
+ ITestSuite suite = new TestSuiteImpl(numTests);
+ testsList.addAll(suite.split(3));
+ CountDownLatch tracker = new CountDownLatch(1);
+ TestsPoolPoller poller = new TestsPoolPoller(testsList, tracker);
+ poller.setMetricCollectors(mMetricCollectors);
+ poller.setDevice(mDevice);
+ poller.setLogRegistry(mMockRegistry);
+ try {
+ poller.run(mListener);
+ fail("Should have thrown an exception.");
+ } catch (DeviceNotAvailableException expected) {
+ // expected
+ }
+ // We expect all the non-executed tests to be reported.
+ Mockito.verify(mListener, Mockito.times(1))
+ .testRunStarted(Mockito.eq("test"), Mockito.eq(0));
+ Mockito.verify(mListener, Mockito.times(1))
+ .testRunStarted(Mockito.eq("test1"), Mockito.eq(0));
+ Mockito.verify(mListener, Mockito.times(1))
+ .testRunStarted(Mockito.eq("test2"), Mockito.eq(0));
+ Mockito.verify(mListener, Mockito.times(1))
+ .testRunStarted(Mockito.eq("test3"), Mockito.eq(0));
+ Mockito.verify(mListener, Mockito.times(1))
+ .testRunStarted(Mockito.eq("test4"), Mockito.eq(0));
+ Mockito.verify(mListener, Mockito.times(5))
+ .testRunFailed(IReportNotExecuted.NOT_EXECUTED_FAILURE);
+ Mockito.verify(mListener, Mockito.times(5))
+ .testRunEnded(Mockito.anyLong(), (HashMap<String, Metric>) Mockito.any());
+ assertEquals(0, tracker.getCount());
+ Mockito.verify(mMockRegistry)
+ .logEvent(
+ Mockito.eq(LogLevel.DEBUG),
+ Mockito.eq(EventType.SHARD_POLLER_EARLY_TERMINATION),
+ Mockito.any());
+ }
+
+ /**
* If a device not available exception is thrown from a tests, and the poller is not the last
* one alive, we wait and attempt to recover the device. In case of success, execution will
* proceed.
diff --git a/tests/src/com/android/tradefed/testtype/suite/ITestSuiteTest.java b/tests/src/com/android/tradefed/testtype/suite/ITestSuiteTest.java
index cd73b60..25af8af 100644
--- a/tests/src/com/android/tradefed/testtype/suite/ITestSuiteTest.java
+++ b/tests/src/com/android/tradefed/testtype/suite/ITestSuiteTest.java
@@ -93,10 +93,13 @@
private IConfiguration mStubMainConfiguration;
private ILogSaver mMockLogSaver;
- /**
- * Very basic implementation of {@link ITestSuite} to test it.
- */
- static class TestSuiteImpl extends ITestSuite {
+ // Guice scope and objects for testing
+ private InvocationScope mScope;
+ private Injector mInjector;
+ private InvocationScopeModule mInvocationScope;
+
+ /** Very basic implementation of {@link ITestSuite} to test it. */
+ public static class TestSuiteImpl extends ITestSuite {
private int mNumTests = 1;
public TestSuiteImpl() {}
