When a DNAE occurs report the non-executed tests
Allow test types to report their non-executed tests if all
the tests poller stops and some tests cannot be executed.
Test: unit tests
run cts --shard-count 2 then disconnect all devices
Bug: 111141078
Bug: 120798810
Merged-In: If82bed881b6f774a53ca412ad2e6c4f1e55be787
Change-Id: Ia9ca3ed3e99d447c4c04f996d2c3c3de78d4fb77
diff --git a/src/com/android/tradefed/invoker/shard/TestsPoolPoller.java b/src/com/android/tradefed/invoker/shard/TestsPoolPoller.java
index 43e62ec..fbd1c9d 100644
--- a/src/com/android/tradefed/invoker/shard/TestsPoolPoller.java
+++ b/src/com/android/tradefed/invoker/shard/TestsPoolPoller.java
@@ -38,8 +38,10 @@
import com.android.tradefed.testtype.IInvocationContextReceiver;
import com.android.tradefed.testtype.IMultiDeviceTest;
import com.android.tradefed.testtype.IRemoteTest;
+import com.android.tradefed.testtype.IReportNotExecuted;
import com.android.tradefed.testtype.ITestCollector;
import com.android.tradefed.util.StreamUtil;
+import com.android.tradefed.util.TimeUtil;
import java.util.Collection;
import java.util.HashMap;
@@ -166,6 +168,11 @@
}
} finally {
mTracker.countDown();
+ if (mTracker.getCount() == 0) {
+ // If the last poller is also disconnected we want to know about the tests that
+ // did not execute.
+ reportNotExecuted(listener);
+ }
}
}
@@ -177,7 +184,9 @@
throws DeviceNotAvailableException {
try {
if (mTracker.getCount() > 1) {
- CLog.d("Wait 5 min for device to maybe coming back online.");
+ CLog.d(
+ "Wait %s for device to maybe come back online.",
+ TimeUtil.formatElapsedTime(WAIT_RECOVERY_TIME));
mDevice.waitForDeviceAvailable(WAIT_RECOVERY_TIME);
mDevice.reboot();
CLog.d("TestPoller was recovered after %s went offline", mDevice.getSerialNumber());
@@ -203,6 +212,21 @@
throw originalException;
}
+ /** Go through the remaining IRemoteTest and report them as not executed. */
+ private void reportNotExecuted(ITestInvocationListener listener) {
+ IRemoteTest test = poll();
+ while (test != null) {
+ if (test instanceof IReportNotExecuted) {
+ ((IReportNotExecuted) test).reportNotExecuted(listener);
+ } else {
+ CLog.e(
+ "Could not report not executed tests from %s.",
+ test.getClass().getCanonicalName());
+ }
+ test = poll();
+ }
+ }
+
/** Helper to log the device events. */
private void logDeviceEvent(EventType event, String serial, Throwable t) {
Map<String, String> args = new HashMap<>();
diff --git a/src/com/android/tradefed/testtype/IReportNotExecuted.java b/src/com/android/tradefed/testtype/IReportNotExecuted.java
new file mode 100644
index 0000000..941f7f3
--- /dev/null
+++ b/src/com/android/tradefed/testtype/IReportNotExecuted.java
@@ -0,0 +1,35 @@
+/*
+ * Copyright (C) 2018 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.android.tradefed.testtype;
+
+import com.android.tradefed.result.ITestInvocationListener;
+
+/**
+ * In case of an incomplete execution, {@link IRemoteTest} that implements this interface may report
+ * their non-executed tests for improved reporting.
+ */
+public interface IReportNotExecuted {
+
+ public static final String NOT_EXECUTED_FAILURE = "Test did not run. This is a placeholder.";
+
+ /**
+ * Report the non-executed tests to the main listener provided. They should be reported as
+ * failed with the {@link #NOT_EXECUTED_FAILURE} message.
+ *
+ * @param listener the main listener where to report the non-executed results.
+ */
+ public void reportNotExecuted(ITestInvocationListener listener);
+}
diff --git a/src/com/android/tradefed/testtype/suite/GranularRetriableTestWrapper.java b/src/com/android/tradefed/testtype/suite/GranularRetriableTestWrapper.java
index 4cdebe6..889d27b 100644
--- a/src/com/android/tradefed/testtype/suite/GranularRetriableTestWrapper.java
+++ b/src/com/android/tradefed/testtype/suite/GranularRetriableTestWrapper.java
@@ -270,7 +270,7 @@
* the IRemoteTest should already has the subset of testcases identified.
*/
@VisibleForTesting
- final void intraModuleRun() throws DeviceNotAvailableException, DeviceUnresponsiveException {
+ final void intraModuleRun() throws DeviceNotAvailableException {
ITestInvocationListener runListener = prepareRunListener();
try {
mTest.run(runListener);
@@ -288,8 +288,6 @@
CLog.w(due);
CLog.w("Proceeding to the next test.");
runListener.testRunFailed(due.getMessage());
- } catch (DeviceNotAvailableException dnae) {
- throw dnae;
} finally {
ModuleListener currentModuleListener =
mModuleListenerCollector.get(mModuleListenerCollector.size() - 1);
diff --git a/src/com/android/tradefed/testtype/suite/ITestSuite.java b/src/com/android/tradefed/testtype/suite/ITestSuite.java
index cbf1305..f6c97a4 100644
--- a/src/com/android/tradefed/testtype/suite/ITestSuite.java
+++ b/src/com/android/tradefed/testtype/suite/ITestSuite.java
@@ -50,6 +50,7 @@
import com.android.tradefed.testtype.IInvocationContextReceiver;
import com.android.tradefed.testtype.IMultiDeviceTest;
import com.android.tradefed.testtype.IRemoteTest;
+import com.android.tradefed.testtype.IReportNotExecuted;
import com.android.tradefed.testtype.IRuntimeHintProvider;
import com.android.tradefed.testtype.IShardableTest;
import com.android.tradefed.testtype.ITestCollector;
@@ -86,7 +87,8 @@
IInvocationContextReceiver,
IRuntimeHintProvider,
IMetricCollectorReceiver,
- IConfigurationReceiver {
+ IConfigurationReceiver,
+ IReportNotExecuted {
public static final String SKIP_SYSTEM_STATUS_CHECKER = "skip-system-status-check";
public static final String RUNNER_WHITELIST = "runner-whitelist";
@@ -535,7 +537,13 @@
continue;
}
- StatusCheckerResult result = checker.preExecutionCheck(device);
+ StatusCheckerResult result = new StatusCheckerResult(CheckStatus.FAILED);
+ try {
+ result = checker.preExecutionCheck(device);
+ } catch (RuntimeException e) {
+ // Catch RuntimeException to avoid leaking throws that go to the invocation.
+ result.setErrorMessage(e.getMessage());
+ }
if (!CheckStatus.SUCCESS.equals(result.getStatus())) {
String errorMessage =
(result.getErrorMessage() == null) ? "" : result.getErrorMessage();
@@ -577,7 +585,13 @@
continue;
}
- StatusCheckerResult result = checker.postExecutionCheck(device);
+ StatusCheckerResult result = new StatusCheckerResult(CheckStatus.FAILED);
+ try {
+ result = checker.postExecutionCheck(device);
+ } catch (RuntimeException e) {
+ // Catch RuntimeException to avoid leaking throws that go to the invocation.
+ result.setErrorMessage(e.getMessage());
+ }
if (!CheckStatus.SUCCESS.equals(result.getStatus())) {
String errorMessage =
(result.getErrorMessage() == null) ? "" : result.getErrorMessage();
@@ -780,6 +794,21 @@
mMainConfiguration = configuration;
}
+ /** {@inheritDoc} */
+ @Override
+ public void reportNotExecuted(ITestInvocationListener listener) {
+ List<ModuleDefinition> runModules = createExecutionList();
+
+ while (!runModules.isEmpty()) {
+ ModuleDefinition module = runModules.remove(0);
+ listener.testModuleStarted(module.getModuleInvocationContext());
+ listener.testRunStarted(module.getId(), 0);
+ listener.testRunFailed(IReportNotExecuted.NOT_EXECUTED_FAILURE);
+ listener.testRunEnded(0, new HashMap<String, Metric>());
+ listener.testModuleEnded();
+ }
+ }
+
/**
* Returns the {@link ModuleDefinition} to be executed directly, or null if none yet (when the
* ITestSuite has not been sharded yet).
diff --git a/tests/src/com/android/tradefed/invoker/shard/TestsPoolPollerTest.java b/tests/src/com/android/tradefed/invoker/shard/TestsPoolPollerTest.java
index 71cf2ed..509e708 100644
--- a/tests/src/com/android/tradefed/invoker/shard/TestsPoolPollerTest.java
+++ b/tests/src/com/android/tradefed/invoker/shard/TestsPoolPollerTest.java
@@ -32,7 +32,10 @@
import com.android.tradefed.metrics.proto.MetricMeasurement.Metric;
import com.android.tradefed.result.ITestInvocationListener;
import com.android.tradefed.testtype.IRemoteTest;
+import com.android.tradefed.testtype.IReportNotExecuted;
import com.android.tradefed.testtype.StubTest;
+import com.android.tradefed.testtype.suite.ITestSuite;
+import com.android.tradefed.testtype.suite.ITestSuiteTest.TestSuiteImpl;
import org.junit.Before;
import org.junit.Test;
@@ -224,6 +227,56 @@
}
/**
+ * Test that a runner that implements {@link IReportNotExecuted} can report the non-executed
+ * tests when the DNAE occurs.
+ */
+ @Test
+ public void testRun_dnae_reportNotExecuted() throws Exception {
+ List<IRemoteTest> testsList = new ArrayList<>();
+ // Add one bad test first that will throw an exception.
+ IRemoteTest badTest = new StubTest();
+ OptionSetter setter = new OptionSetter(badTest);
+ setter.setOptionValue("test-throw-not-available", "true");
+ testsList.add(badTest);
+ // Add tests that from a suite that can report their not executed tests.
+ int numTests = 5;
+ ITestSuite suite = new TestSuiteImpl(numTests);
+ testsList.addAll(suite.split(3));
+ CountDownLatch tracker = new CountDownLatch(1);
+ TestsPoolPoller poller = new TestsPoolPoller(testsList, tracker);
+ poller.setMetricCollectors(mMetricCollectors);
+ poller.setDevice(mDevice);
+ poller.setLogRegistry(mMockRegistry);
+ try {
+ poller.run(mListener);
+ fail("Should have thrown an exception.");
+ } catch (DeviceNotAvailableException expected) {
+ // expected
+ }
+ // We expect all the non-executed tests to be reported.
+ Mockito.verify(mListener, Mockito.times(1))
+ .testRunStarted(Mockito.eq("test"), Mockito.eq(0));
+ Mockito.verify(mListener, Mockito.times(1))
+ .testRunStarted(Mockito.eq("test1"), Mockito.eq(0));
+ Mockito.verify(mListener, Mockito.times(1))
+ .testRunStarted(Mockito.eq("test2"), Mockito.eq(0));
+ Mockito.verify(mListener, Mockito.times(1))
+ .testRunStarted(Mockito.eq("test3"), Mockito.eq(0));
+ Mockito.verify(mListener, Mockito.times(1))
+ .testRunStarted(Mockito.eq("test4"), Mockito.eq(0));
+ Mockito.verify(mListener, Mockito.times(5))
+ .testRunFailed(IReportNotExecuted.NOT_EXECUTED_FAILURE);
+ Mockito.verify(mListener, Mockito.times(5))
+ .testRunEnded(Mockito.anyLong(), (HashMap<String, Metric>) Mockito.any());
+ assertEquals(0, tracker.getCount());
+ Mockito.verify(mMockRegistry)
+ .logEvent(
+ Mockito.eq(LogLevel.DEBUG),
+ Mockito.eq(EventType.SHARD_POLLER_EARLY_TERMINATION),
+ Mockito.any());
+ }
+
+ /**
* If a device not available exception is thrown from a tests, and the poller is not the last
* one alive, we wait and attempt to recover the device. In case of success, execution will
* proceed.
diff --git a/tests/src/com/android/tradefed/testtype/suite/ITestSuiteTest.java b/tests/src/com/android/tradefed/testtype/suite/ITestSuiteTest.java
index f70f1f0..a096737 100644
--- a/tests/src/com/android/tradefed/testtype/suite/ITestSuiteTest.java
+++ b/tests/src/com/android/tradefed/testtype/suite/ITestSuiteTest.java
@@ -93,10 +93,8 @@
private IConfiguration mStubMainConfiguration;
private ILogSaver mMockLogSaver;
- /**
- * Very basic implementation of {@link ITestSuite} to test it.
- */
- static class TestSuiteImpl extends ITestSuite {
+ /** Very basic implementation of {@link ITestSuite} to test it. */
+ public static class TestSuiteImpl extends ITestSuite {
private int mNumTests = 1;
public TestSuiteImpl() {}
@@ -289,6 +287,35 @@
/**
* Test for {@link ITestSuite#run(ITestInvocationListener)} when the System status checker is
+ * failing with a runtime exception. RuntimeException is interpreted as a checker failure.
+ */
+ @Test
+ public void testRun_failedSystemChecker_runtimeException() throws Exception {
+ final byte[] fakeData = "fakeData".getBytes();
+ InputStreamSource fakeSource = new ByteArrayInputStreamSource(fakeData);
+ List<ISystemStatusChecker> sysChecker = new ArrayList<ISystemStatusChecker>();
+ sysChecker.add(mMockSysChecker);
+ mTestSuite.setSystemStatusChecker(sysChecker);
+
+ EasyMock.expect(mMockSysChecker.preExecutionCheck(EasyMock.eq(mMockDevice)))
+ .andThrow(new RuntimeException("I failed."));
+ EasyMock.expect(mMockDevice.getBugreport()).andReturn(fakeSource).times(2);
+ mMockListener.testLog(
+ (String) EasyMock.anyObject(),
+ EasyMock.eq(LogDataType.BUGREPORT),
+ EasyMock.eq(fakeSource));
+ EasyMock.expectLastCall().times(2);
+
+ EasyMock.expect(mMockSysChecker.postExecutionCheck(EasyMock.eq(mMockDevice)))
+ .andThrow(new RuntimeException("I failed post."));
+ expectTestRun(mMockListener);
+ replayMocks();
+ mTestSuite.run(mMockListener);
+ verifyMocks();
+ }
+
+ /**
+ * Test for {@link ITestSuite#run(ITestInvocationListener)} when the System status checker is
* passing pre-check but failing post-check and we enable reporting a failure for it.
*/
@Test