Allow the new retry.xml to be splitted
Retry.xml use new sharding too.
Prepare it to become the official retry mechanism when
CompatibilityTest is deprecated.
Test: unit tests
./cts-tradefed run retry --retry 63 --shard-count 2
./cts-tradefed run retry --retry 63 --shard-count 2 --retry-type
NOT_EXECUTED
Bug: 65013728
Bug: 36337428
Change-Id: I89bef352aae3f00af7bde36207e5b44ff19ab85f
diff --git a/common/host-side/tradefed/src/com/android/compatibility/common/tradefed/testtype/CompatibilityTest.java b/common/host-side/tradefed/src/com/android/compatibility/common/tradefed/testtype/CompatibilityTest.java
index 63fabeb..b722b3b 100644
--- a/common/host-side/tradefed/src/com/android/compatibility/common/tradefed/testtype/CompatibilityTest.java
+++ b/common/host-side/tradefed/src/com/android/compatibility/common/tradefed/testtype/CompatibilityTest.java
@@ -664,10 +664,10 @@
if (!failures.isEmpty()) {
CLog.w("There are failed system status checkers: %s capturing a bugreport",
failures.toString());
- InputStreamSource bugSource = device.getBugreport();
- logger.testLog(String.format("bugreport-checker-pre-module-%s", moduleName),
- LogDataType.BUGREPORT, bugSource);
- bugSource.cancel();
+ try (InputStreamSource bugSource = device.getBugreport()) {
+ logger.testLog(String.format("bugreport-checker-pre-module-%s", moduleName),
+ LogDataType.BUGREPORT, bugSource);
+ }
}
}
@@ -686,10 +686,10 @@
if (!failures.isEmpty()) {
CLog.w("There are failed system status checkers: %s capturing a bugreport",
failures.toString());
- InputStreamSource bugSource = device.getBugreport();
- logger.testLog(String.format("bugreport-checker-post-module-%s", moduleName),
- LogDataType.BUGREPORT, bugSource);
- bugSource.cancel();
+ try (InputStreamSource bugSource = device.getBugreport()) {
+ logger.testLog(String.format("bugreport-checker-post-module-%s", moduleName),
+ LogDataType.BUGREPORT, bugSource);
+ }
}
}
diff --git a/common/host-side/tradefed/src/com/android/compatibility/common/tradefed/testtype/FailureListener.java b/common/host-side/tradefed/src/com/android/compatibility/common/tradefed/testtype/FailureListener.java
index 1fac0f2..cd4829d 100644
--- a/common/host-side/tradefed/src/com/android/compatibility/common/tradefed/testtype/FailureListener.java
+++ b/common/host-side/tradefed/src/com/android/compatibility/common/tradefed/testtype/FailureListener.java
@@ -70,10 +70,10 @@
CLog.i("FailureListener.testFailed %s %b %b %b", test.toString(), mBugReportOnFailure, mLogcatOnFailure, mScreenshotOnFailure);
if (mScreenshotOnFailure) {
try {
- InputStreamSource screenSource = mDevice.getScreenshot();
- super.testLog(String.format("%s-screenshot", test.toString()), LogDataType.PNG,
- screenSource);
- screenSource.cancel();
+ try (InputStreamSource screenSource = mDevice.getScreenshot()) {
+ super.testLog(String.format("%s-screenshot", test.toString()), LogDataType.PNG,
+ screenSource);
+ }
} catch (DeviceNotAvailableException e) {
CLog.e(e);
CLog.e("Device %s became unavailable while capturing screenshot",
@@ -81,18 +81,18 @@
}
}
if (mBugReportOnFailure) {
- InputStreamSource bugSource = mDevice.getBugreportz();
- super.testLog(String.format("%s-bugreport", test.toString()), LogDataType.BUGREPORT,
- bugSource);
- bugSource.cancel();
+ try (InputStreamSource bugSource = mDevice.getBugreportz()) {
+ super.testLog(String.format("%s-bugreport", test.toString()), LogDataType.BUGREPORT,
+ bugSource);
+ }
}
if (mLogcatOnFailure) {
// sleep 2s to ensure test failure stack trace makes it into logcat capture
RunUtil.getDefault().sleep(2 * 1000);
- InputStreamSource logSource = mDevice.getLogcat(mMaxLogcatBytes);
- super.testLog(String.format("%s-logcat", test.toString()), LogDataType.LOGCAT,
- logSource);
- logSource.cancel();
+ try (InputStreamSource logSource = mDevice.getLogcat(mMaxLogcatBytes)) {
+ super.testLog(String.format("%s-logcat", test.toString()), LogDataType.LOGCAT,
+ logSource);
+ }
}
if (mRebootOnFailure) {
try {
diff --git a/common/host-side/tradefed/src/com/android/compatibility/common/tradefed/testtype/retry/RetryFactoryTest.java b/common/host-side/tradefed/src/com/android/compatibility/common/tradefed/testtype/retry/RetryFactoryTest.java
index 0276b3c..40bf633 100644
--- a/common/host-side/tradefed/src/com/android/compatibility/common/tradefed/testtype/retry/RetryFactoryTest.java
+++ b/common/host-side/tradefed/src/com/android/compatibility/common/tradefed/testtype/retry/RetryFactoryTest.java
@@ -17,6 +17,7 @@
import com.android.compatibility.common.tradefed.build.CompatibilityBuildHelper;
import com.android.compatibility.common.tradefed.testtype.CompatibilityTest;
+import com.android.compatibility.common.tradefed.testtype.suite.CompatibilityTestSuite;
import com.android.compatibility.common.tradefed.util.RetryFilterHelper;
import com.android.compatibility.common.tradefed.util.RetryType;
import com.android.tradefed.build.IBuildInfo;
@@ -28,6 +29,7 @@
import com.android.tradefed.device.DeviceNotAvailableException;
import com.android.tradefed.device.ITestDevice;
import com.android.tradefed.invoker.IInvocationContext;
+import com.android.tradefed.log.LogUtil.CLog;
import com.android.tradefed.result.ITestInvocationListener;
import com.android.tradefed.suite.checker.ISystemStatusChecker;
import com.android.tradefed.suite.checker.ISystemStatusCheckerReceiver;
@@ -35,9 +37,11 @@
import com.android.tradefed.testtype.IDeviceTest;
import com.android.tradefed.testtype.IInvocationContextReceiver;
import com.android.tradefed.testtype.IRemoteTest;
+import com.android.tradefed.testtype.IShardableTest;
import com.google.common.annotations.VisibleForTesting;
+import java.util.Collection;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
@@ -50,7 +54,7 @@
*/
@OptionClass(alias = "compatibility")
public class RetryFactoryTest implements IRemoteTest, IDeviceTest, IBuildReceiver,
- ISystemStatusCheckerReceiver, IInvocationContextReceiver {
+ ISystemStatusCheckerReceiver, IInvocationContextReceiver, IShardableTest {
/**
* Mirror the {@link CompatibilityTest} options in order to create it.
@@ -136,12 +140,35 @@
*/
@Override
public void run(ITestInvocationListener listener) throws DeviceNotAvailableException {
+ CompatibilityTestSuite test = loadSuite();
+ // run the retry run.
+ test.run(listener);
+ }
+
+ @Override
+ public Collection<IRemoteTest> split(int shardCountHint) {
+ try {
+ CompatibilityTestSuite test = loadSuite();
+ return test.split(shardCountHint);
+ } catch (DeviceNotAvailableException e) {
+ CLog.e("Failed to shard the retry run.");
+ CLog.e(e);
+ }
+ return null;
+ }
+
+ /**
+ * Helper to create a {@link CompatibilityTestSuite} from previous results.
+ */
+ private CompatibilityTestSuite loadSuite() throws DeviceNotAvailableException {
// Create a compatibility test and set it to run only what we want.
- CompatibilityTest test = createTest();
+ CompatibilityTestSuite test = createTest();
CompatibilityBuildHelper buildHelper = new CompatibilityBuildHelper(mBuildInfo);
// Create the helper with all the options needed.
RetryFilterHelper helper = createFilterHelper(buildHelper);
+ // TODO: we have access to the original command line, we should accommodate more re-run
+ // scenario like when the original cts.xml config was not used.
helper.validateBuildFingerprint(mDevice);
helper.setCommandLineOptionsFor(test);
helper.setCommandLineOptionsFor(this);
@@ -166,8 +193,7 @@
test.setInvocationContext(mContext);
// clean the helper
helper.tearDown();
- // run the retry run.
- test.run(listener);
+ return test;
}
@VisibleForTesting
@@ -177,7 +203,7 @@
}
@VisibleForTesting
- CompatibilityTest createTest() {
- return new CompatibilityTest();
+ CompatibilityTestSuite createTest() {
+ return new CompatibilityTestSuite();
}
}
diff --git a/common/host-side/tradefed/src/com/android/compatibility/common/tradefed/testtype/suite/CompatibilityTestSuite.java b/common/host-side/tradefed/src/com/android/compatibility/common/tradefed/testtype/suite/CompatibilityTestSuite.java
index 62b7af9..869c373 100644
--- a/common/host-side/tradefed/src/com/android/compatibility/common/tradefed/testtype/suite/CompatibilityTestSuite.java
+++ b/common/host-side/tradefed/src/com/android/compatibility/common/tradefed/testtype/suite/CompatibilityTestSuite.java
@@ -17,6 +17,7 @@
import com.android.compatibility.SuiteInfo;
import com.android.compatibility.common.tradefed.build.CompatibilityBuildHelper;
+import com.android.compatibility.common.tradefed.testtype.CompatibilityTest;
import com.android.compatibility.common.tradefed.testtype.ISubPlan;
import com.android.compatibility.common.tradefed.testtype.ModuleRepo;
import com.android.compatibility.common.tradefed.testtype.SubPlan;
@@ -68,6 +69,12 @@
private static final String PRIMARY_ABI_RUN = "primary-abi-only";
private static final String PRODUCT_CPU_ABI_KEY = "ro.product.cpu.abi";
+ // TODO: remove this option when CompatibilityTest goes away
+ @Option(name = CompatibilityTest.RETRY_OPTION,
+ shortName = 'r',
+ description = "Copy of --retry from CompatibilityTest to prevent using it.")
+ private Integer mRetrySessionId = null;
+
@Option(name = SUBPLAN_OPTION,
description = "the subplan to run",
importance = Importance.IF_UNSET)
@@ -147,6 +154,10 @@
*/
@Override
public LinkedHashMap<String, IConfiguration> loadTests() {
+ if (mRetrySessionId != null) {
+ throw new IllegalArgumentException("--retry cannot be specified with cts-suite.xml. "
+ + "Use 'run cts --retry <session id>' instead.");
+ }
try {
setupFilters();
Set<IAbi> abis = getAbis(getDevice());
@@ -282,4 +293,18 @@
filters.clear();
filters.addAll(cleanedFilters);
}
+
+ /**
+ * Sets include-filters for the compatibility test
+ */
+ public void setIncludeFilter(Set<String> includeFilters) {
+ mIncludeFilters.addAll(includeFilters);
+ }
+
+ /**
+ * Sets exclude-filters for the compatibility test
+ */
+ public void setExcludeFilter(Set<String> excludeFilters) {
+ mExcludeFilters.addAll(excludeFilters);
+ }
}
diff --git a/common/host-side/tradefed/tests/src/com/android/compatibility/common/tradefed/testtype/retry/RetryFactoryTestTest.java b/common/host-side/tradefed/tests/src/com/android/compatibility/common/tradefed/testtype/retry/RetryFactoryTestTest.java
index 01557e3..d67619a 100644
--- a/common/host-side/tradefed/tests/src/com/android/compatibility/common/tradefed/testtype/retry/RetryFactoryTestTest.java
+++ b/common/host-side/tradefed/tests/src/com/android/compatibility/common/tradefed/testtype/retry/RetryFactoryTestTest.java
@@ -15,14 +15,20 @@
*/
package com.android.compatibility.common.tradefed.testtype.retry;
+import static org.junit.Assert.*;
+
import com.android.compatibility.common.tradefed.build.CompatibilityBuildHelper;
import com.android.compatibility.common.tradefed.testtype.CompatibilityTest;
+import com.android.compatibility.common.tradefed.testtype.suite.CompatibilityTestSuite;
import com.android.compatibility.common.tradefed.util.RetryFilterHelper;
+import com.android.tradefed.config.IConfiguration;
import com.android.tradefed.config.OptionClass;
import com.android.tradefed.config.OptionSetter;
import com.android.tradefed.device.DeviceNotAvailableException;
import com.android.tradefed.device.ITestDevice;
import com.android.tradefed.result.ITestInvocationListener;
+import com.android.tradefed.testtype.IRemoteTest;
+import com.android.tradefed.testtype.StubTest;
import org.easymock.EasyMock;
import org.junit.Before;
@@ -30,6 +36,11 @@
import org.junit.runner.RunWith;
import org.junit.runners.JUnit4;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.LinkedHashMap;
+import java.util.List;
+
/**
* Unit tests for {@link RetryFactoryTest}.
*/
@@ -44,11 +55,19 @@
* A {@link CompatibilityTest} that does not run anything.
*/
@OptionClass(alias = "compatibility")
- public static class VoidCompatibilityTest extends CompatibilityTest {
+ public static class VoidCompatibilityTest extends CompatibilityTestSuite {
@Override
- public void run(ITestInvocationListener listener)
- throws DeviceNotAvailableException {
- // Do not run.
+ public LinkedHashMap<String, IConfiguration> loadTests() {
+ return new LinkedHashMap<>();
+ }
+
+ @Override
+ public Collection<IRemoteTest> split(int shardCountHint) {
+ List<IRemoteTest> tests = new ArrayList<>();
+ for (int i = 0; i < shardCountHint; i++) {
+ tests.add(new StubTest());
+ }
+ return tests;
}
}
@@ -75,7 +94,7 @@
return mSpyFilter;
}
@Override
- CompatibilityTest createTest() {
+ CompatibilityTestSuite createTest() {
return new VoidCompatibilityTest();
}
};
@@ -93,4 +112,16 @@
mFactory.run(mMockListener);
EasyMock.verify(mMockListener);
}
+
+ /**
+ * Assert that the {@link RetryFactoryTest#split(int)} calls the
+ * {@link CompatibilityTestSuite#split(int)} after applying all the filters.
+ */
+ @Test
+ public void testRetry_split() throws Exception {
+ EasyMock.replay(mMockListener);
+ Collection<IRemoteTest> res = mFactory.split(2);
+ assertEquals(2, res.size());
+ EasyMock.verify(mMockListener);
+ }
}
