add timeout per package for PTS
- Motivation: PTS tests can run for long time and needs customized timeout.
- each method can declare timeout using @TimeoutReq(minutes=value)
annotation.
- timeout is specified in minutes with 0 meaning no timeout.
- xml generator add timeout information like Class:method:10 if timeout
annotation is present.
- CTS parses the xml and read timeout per each method, but the current
architecture allows only one timeout for the whole package.
- Due to the restriction, the biggest timeout for the package is set.
- If timeout is not set, the current default of 10 mins will be used.
- The new annotation, TimeoutReq is defined in ctsutil.
CTS tests can use it too. But it is ** strongly recommended to use this
in a new package ** to prevent increasing test time too much.
- all file system tests and UI tests set to have timeout of 30 mins.
Change-Id: I9a19cb02b91376c581e13522d248872af8c7eee9
diff --git a/libs/util/src/android/cts/util/TimeoutReq.java b/libs/util/src/android/cts/util/TimeoutReq.java
new file mode 100644
index 0000000..943c3ce
--- /dev/null
+++ b/libs/util/src/android/cts/util/TimeoutReq.java
@@ -0,0 +1,29 @@
+/*
+ * Copyright (C) 2012 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package android.cts.util;
+
+import java.lang.annotation.Retention;
+import java.lang.annotation.RetentionPolicy;
+
+/**
+ * Annotation to specify necessary timeout for the test
+ * timeout is specified in minutes and 0 value means no timeout ( = infinite waiting).
+ */
+@Retention(RetentionPolicy.RUNTIME)
+public @interface TimeoutReq {
+ int minutes() default -1;
+}
diff --git a/suite/pts/deviceTests/filesystemperf/Android.mk b/suite/pts/deviceTests/filesystemperf/Android.mk
index 3709ecc..74e2c05 100644
--- a/suite/pts/deviceTests/filesystemperf/Android.mk
+++ b/suite/pts/deviceTests/filesystemperf/Android.mk
@@ -20,7 +20,7 @@
LOCAL_JAVA_LIBRARIES := android.test.runner
-LOCAL_STATIC_JAVA_LIBRARIES := ptsutil
+LOCAL_STATIC_JAVA_LIBRARIES := ptsutil ctsutil
LOCAL_SRC_FILES := $(call all-java-files-under, src)
diff --git a/suite/pts/deviceTests/filesystemperf/src/com/android/pts/filesystemperf/FullUpdateTest.java b/suite/pts/deviceTests/filesystemperf/src/com/android/pts/filesystemperf/FullUpdateTest.java
index 261735c..388e968 100644
--- a/suite/pts/deviceTests/filesystemperf/src/com/android/pts/filesystemperf/FullUpdateTest.java
+++ b/suite/pts/deviceTests/filesystemperf/src/com/android/pts/filesystemperf/FullUpdateTest.java
@@ -16,10 +16,7 @@
package com.android.pts.filesystemperf;
-import java.io.File;
-import java.io.FileOutputStream;
-import java.io.IOException;
-
+import android.cts.util.TimeoutReq;
import com.android.pts.util.MeasureRun;
import com.android.pts.util.MeasureTime;
import com.android.pts.util.PtsAndroidTestCase;
@@ -28,6 +25,10 @@
import android.util.Log;
+import java.io.File;
+import java.io.FileOutputStream;
+import java.io.IOException;
+
public class FullUpdateTest extends PtsAndroidTestCase {
private static final String DIR_INITIAL_FILL = "INITIAL_FILL";
private static final String DIR_WORK = "WORK";
@@ -42,6 +43,7 @@
// fill disk almost, update exceeding free space, then update some amount
// idea is to drain all free blocks and measure update performance
+ @TimeoutReq(minutes = 30)
public void testAlmostFilledUpdate() throws IOException {
long freeDisk = SystemUtil.getFreeDiskSize(getContext());
final long FREE_SPACE_TO_LEAVE = 500L * 1024L * 1024L; // leave this much
diff --git a/suite/pts/deviceTests/filesystemperf/src/com/android/pts/filesystemperf/RWTest.java b/suite/pts/deviceTests/filesystemperf/src/com/android/pts/filesystemperf/RWTest.java
index f44e797..ec4faba 100644
--- a/suite/pts/deviceTests/filesystemperf/src/com/android/pts/filesystemperf/RWTest.java
+++ b/suite/pts/deviceTests/filesystemperf/src/com/android/pts/filesystemperf/RWTest.java
@@ -16,17 +16,18 @@
package com.android.pts.filesystemperf;
-import java.io.File;
-import java.io.FileInputStream;
-import java.io.FileOutputStream;
-import java.io.IOException;
-
+import android.cts.util.TimeoutReq;
import com.android.pts.util.MeasureRun;
import com.android.pts.util.MeasureTime;
import com.android.pts.util.PtsAndroidTestCase;
import com.android.pts.util.ReportLog;
import com.android.pts.util.SystemUtil;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileOutputStream;
+import java.io.IOException;
+
public class RWTest extends PtsAndroidTestCase {
private static final String DIR_SEQ_WR = "SEQ_WR";
private static final String DIR_SEQ_UPD = "SEQ_UPD";
@@ -41,6 +42,7 @@
super.tearDown();
}
+ @TimeoutReq(minutes = 30)
public void testSingleSequentialWrite() throws IOException {
final int numberOfFiles = (int)(getFileSizeExceedingMemory() / BUFFER_SIZE);
getReportLog().printValue("files", numberOfFiles);
@@ -61,7 +63,7 @@
getReportLog().printArray("Wr amount", wrAmount, true);
}
-
+ @TimeoutReq(minutes = 30)
public void testSingleSequentialUpdate() throws IOException {
final long fileSize = getFileSizeExceedingMemory();
File file = FileUtil.createNewFilledFile(getContext(),
@@ -87,6 +89,7 @@
}
}
+ @TimeoutReq(minutes = 30)
public void testSingleSequentialRead() throws IOException {
final long fileSize = getFileSizeExceedingMemory();
long start = System.currentTimeMillis();
diff --git a/suite/pts/deviceTests/filesystemperf/src/com/android/pts/filesystemperf/TestTest.java b/suite/pts/deviceTests/filesystemperf/src/com/android/pts/filesystemperf/TestTest.java
index 32e1878..2373523 100644
--- a/suite/pts/deviceTests/filesystemperf/src/com/android/pts/filesystemperf/TestTest.java
+++ b/suite/pts/deviceTests/filesystemperf/src/com/android/pts/filesystemperf/TestTest.java
@@ -19,14 +19,20 @@
package com.android.pts.filesystemperf;
+import android.cts.util.TimeoutReq;
import com.android.pts.util.PtsAndroidTestCase;
+/**
+ * This class is for testing PTS itself. Will be disabled in release.
+ *
+ */
public class TestTest extends PtsAndroidTestCase {
@Override
protected void tearDown() throws Exception {
super.tearDown();
}
+ @TimeoutReq(minutes = 15)
public void testPass() {
double[] array = new double[] {1.0, 2.0, 3.0};
getReportLog().printArray(" ", array, true);
@@ -35,6 +41,7 @@
getReportLog().printValue(" ", 2.0);
}
+ @TimeoutReq(minutes = 10)
public void testFail() throws Exception {
getReportLog().printValue(" ", 1.0);
getReportLog().printValue(" ", 2.0);
diff --git a/suite/pts/deviceTests/ui/Android.mk b/suite/pts/deviceTests/ui/Android.mk
index f3c3944..ffb2979 100644
--- a/suite/pts/deviceTests/ui/Android.mk
+++ b/suite/pts/deviceTests/ui/Android.mk
@@ -22,7 +22,7 @@
LOCAL_JAVA_LIBRARIES := android.test.runner
-LOCAL_STATIC_JAVA_LIBRARIES := ptsutil
+LOCAL_STATIC_JAVA_LIBRARIES := ptsutil ctsutil
LOCAL_SRC_FILES := $(call all-java-files-under, src)
diff --git a/suite/pts/deviceTests/ui/src/com/android/pts/ui/ScrollingTest.java b/suite/pts/deviceTests/ui/src/com/android/pts/ui/ScrollingTest.java
index a985591..04ea562 100644
--- a/suite/pts/deviceTests/ui/src/com/android/pts/ui/ScrollingTest.java
+++ b/suite/pts/deviceTests/ui/src/com/android/pts/ui/ScrollingTest.java
@@ -15,13 +15,14 @@
*/
package com.android.pts.ui;
-import java.io.IOException;
-
+import android.cts.util.TimeoutReq;
import com.android.pts.util.MeasureRun;
import com.android.pts.util.MeasureTime;
import com.android.pts.util.PtsActivityInstrumentationTestCase2;
import com.android.pts.util.ReportLog;
+import java.io.IOException;
+
public class ScrollingTest extends PtsActivityInstrumentationTestCase2<ScrollingActivity> {
private ScrollingActivity mActivity;
@@ -51,6 +52,7 @@
super.tearDown();
}
+ @TimeoutReq(minutes = 30)
public void testFullScrolling() throws IOException {
final int NUMBER_REPEAT = 10;
final ScrollingActivity activity = mActivity;
diff --git a/tools/cts-java-scanner-doclet/src/com/android/cts/javascannerdoclet/CtsJavaScannerDoclet.java b/tools/cts-java-scanner-doclet/src/com/android/cts/javascannerdoclet/CtsJavaScannerDoclet.java
index 808473a..4086093 100644
--- a/tools/cts-java-scanner-doclet/src/com/android/cts/javascannerdoclet/CtsJavaScannerDoclet.java
+++ b/tools/cts-java-scanner-doclet/src/com/android/cts/javascannerdoclet/CtsJavaScannerDoclet.java
@@ -49,6 +49,7 @@
import com.sun.javadoc.MethodDoc;
import com.sun.javadoc.RootDoc;
import com.sun.javadoc.AnnotationDesc.ElementValuePair;
+import com.sun.javadoc.AnnotationTypeElementDoc;
/**
* Doclet that outputs in the following format:
@@ -56,7 +57,7 @@
* suite:android.holo.cts
* case:HoloTest
* test:testHolo
- * test:testHoloDialog
+ * test:testHoloDialog[:timeout_value]
*/
public class CtsJavaScannerDoclet extends Doclet {
@@ -81,7 +82,27 @@
if (!method.name().startsWith("test")) {
continue;
}
- writer.append("test:").println(method.name());
+ int timeout = -1;
+ AnnotationDesc[] annotations = method.annotations();
+ for (AnnotationDesc annot : annotations) {
+ AnnotationTypeDoc atype = annot.annotationType();
+ if (atype.toString().equals("android.cts.util.TimeoutReq")) {
+ ElementValuePair[] cpairs = annot.elementValues();
+ for (ElementValuePair pair: cpairs) {
+ AnnotationTypeElementDoc elem = pair.element();
+ AnnotationValue value = pair.value();
+ if (elem.name().equals("minutes")) {
+ timeout = ((Integer)value.value());
+ }
+ }
+ }
+ }
+ writer.append("test:");
+ if (timeout >= 0) {
+ writer.append(method.name()).println(":" + timeout);
+ } else {
+ writer.println(method.name());
+ }
}
}
}
diff --git a/tools/cts-java-scanner/src/com/android/cts/javascanner/DocletRunner.java b/tools/cts-java-scanner/src/com/android/cts/javascanner/DocletRunner.java
index 592b145..ba87156 100644
--- a/tools/cts-java-scanner/src/com/android/cts/javascanner/DocletRunner.java
+++ b/tools/cts-java-scanner/src/com/android/cts/javascanner/DocletRunner.java
@@ -71,6 +71,7 @@
sourcePath.add("./cts/tests/src");
// PTS adds PtsAndroidTestCase
sourcePath.add("./cts/suite/pts/deviceTests/ptsutil/src");
+ sourcePath.add("./cts/libs/util/src");
sourcePath.add(sourceDir.toString());
return join(sourcePath, ":");
}
diff --git a/tools/cts-xml-generator/src/com/android/cts/xmlgenerator/Test.java b/tools/cts-xml-generator/src/com/android/cts/xmlgenerator/Test.java
new file mode 100644
index 0000000..a9dd5e0
--- /dev/null
+++ b/tools/cts-xml-generator/src/com/android/cts/xmlgenerator/Test.java
@@ -0,0 +1,35 @@
+/*
+ * Copyright (C) 2012 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.android.cts.xmlgenerator;
+
+public class Test {
+ private String mName;
+ private int mTimeout;
+
+ public Test(String name, int timeout) {
+ mName = name;
+ mTimeout = timeout;
+ }
+
+ public String getName() {
+ return mName;
+ }
+
+ public int getTimeout() {
+ return mTimeout;
+ }
+}
diff --git a/tools/cts-xml-generator/src/com/android/cts/xmlgenerator/TestCase.java b/tools/cts-xml-generator/src/com/android/cts/xmlgenerator/TestCase.java
index bab3476..a9ea6e1 100644
--- a/tools/cts-xml-generator/src/com/android/cts/xmlgenerator/TestCase.java
+++ b/tools/cts-xml-generator/src/com/android/cts/xmlgenerator/TestCase.java
@@ -24,7 +24,7 @@
private final String mName;
- private final List<String> mTests = new ArrayList<String>();
+ private final List<Test> mTests = new ArrayList<Test>();
public TestCase(String name) {
mName = name;
@@ -34,11 +34,11 @@
return mName;
}
- public void addTest(String test) {
- mTests.add(test);
+ public void addTest(String testName, int timeout) {
+ mTests.add(new Test(testName, timeout));
}
- public Collection<String> getTests() {
+ public Collection<Test> getTests() {
return Collections.unmodifiableCollection(mTests);
}
}
diff --git a/tools/cts-xml-generator/src/com/android/cts/xmlgenerator/TestListParser.java b/tools/cts-xml-generator/src/com/android/cts/xmlgenerator/TestListParser.java
index 76e1437..c77d1a0 100644
--- a/tools/cts-xml-generator/src/com/android/cts/xmlgenerator/TestListParser.java
+++ b/tools/cts-xml-generator/src/com/android/cts/xmlgenerator/TestListParser.java
@@ -27,7 +27,7 @@
* suite:android.holo.cts
* case:HoloTest
* test:testHolo
- * test:testHoloDialog
+ * test:testHoloDialog[:timeout_value]
*/
class TestListParser {
@@ -41,7 +41,7 @@
while(scanner.hasNextLine()) {
String line = scanner.nextLine();
String[] tokens = line.split(":");
- if (tokens.length != 2) {
+ if (tokens.length < 2) {
continue;
}
@@ -52,7 +52,11 @@
} else if ("case".equals(key)) {
currentCase = handleCase(currentSuite, value);
} else if ("test".equals(key)) {
- handleTest(currentCase, value);
+ int timeout = -1;
+ if (tokens.length == 3) {
+ timeout = Integer.parseInt(tokens[2]);
+ }
+ handleTest(currentCase, value, timeout);
}
}
} finally {
@@ -95,7 +99,7 @@
return testCase;
}
- private void handleTest(TestCase testCase, String test) {
- testCase.addTest(test);
+ private void handleTest(TestCase testCase, String test, int timeout) {
+ testCase.addTest(test, timeout);
}
}
diff --git a/tools/cts-xml-generator/src/com/android/cts/xmlgenerator/XmlGenerator.java b/tools/cts-xml-generator/src/com/android/cts/xmlgenerator/XmlGenerator.java
index fda3acc..8a49cf3 100644
--- a/tools/cts-xml-generator/src/com/android/cts/xmlgenerator/XmlGenerator.java
+++ b/tools/cts-xml-generator/src/com/android/cts/xmlgenerator/XmlGenerator.java
@@ -174,17 +174,20 @@
}
}
- private void writeTests(PrintWriter writer, Collection<String> tests,
+ private void writeTests(PrintWriter writer, Collection<Test> tests,
StringBuilder nameCollector) {
- for (String test : tests) {
- nameCollector.append('#').append(test);
- writer.append("<Test name=\"").append(test).append("\"");
+ for (Test test : tests) {
+ nameCollector.append('#').append(test.getName());
+ writer.append("<Test name=\"").append(test.getName()).append("\"");
if (isKnownFailure(mExpectations, nameCollector.toString())) {
writer.append(" expectation=\"failure\"");
}
+ if (test.getTimeout() >= 0) {
+ writer.append(" timeout=\"" + test.getTimeout() + "\"");
+ }
writer.println(" />");
- nameCollector.delete(nameCollector.length() - test.length() - 1,
+ nameCollector.delete(nameCollector.length() - test.getName().length() - 1,
nameCollector.length());
}
}
diff --git a/tools/tradefed-host/res/report/cts_result.xsl b/tools/tradefed-host/res/report/cts_result.xsl
index 3dfd893..b924d2f 100644
--- a/tools/tradefed-host/res/report/cts_result.xsl
+++ b/tools/tradefed-host/res/report/cts_result.xsl
@@ -461,7 +461,7 @@
<TR>
<TH width="30%">Test</TH>
<TH width="5%">Result</TH>
- <TH>Failure Details / Details</TH>
+ <TH>Details</TH>
</TR>
<!-- test case -->
diff --git a/tools/tradefed-host/src/com/android/cts/tradefed/testtype/TestPackageDef.java b/tools/tradefed-host/src/com/android/cts/tradefed/testtype/TestPackageDef.java
index 3b345a7..06f940a 100644
--- a/tools/tradefed-host/src/com/android/cts/tradefed/testtype/TestPackageDef.java
+++ b/tools/tradefed-host/src/com/android/cts/tradefed/testtype/TestPackageDef.java
@@ -75,6 +75,10 @@
private TestFilter mExcludedTestFilter = new TestFilter();
private String mTargetBinaryName;
private String mTargetNameSpace;
+ // only timeout per package is supported. To change this to method granularity,
+ // test invocation should be done in method level.
+ // So for now, only max timeout for the package is used.
+ private int mTimeoutInMins = -1;
void setUri(String uri) {
mUri = uri;
@@ -238,6 +242,12 @@
} else {
CLog.d("Creating instrumentation test for %s", mName);
InstrumentationApkTest instrTest = new InstrumentationApkTest();
+ if (mTimeoutInMins >= 0) {
+ // as timeout cannot be set for each test,
+ // increase the time-out of the whole package
+ CLog.d("Setting new timeout to " + mTimeoutInMins + " mins");
+ instrTest.setTestTimeout(mTimeoutInMins * 60 * 1000);
+ }
return setInstrumentationTest(instrTest, testCaseDir);
}
}
@@ -300,10 +310,15 @@
* Add a {@link TestIdentifier} to the list of tests in this package.
*
* @param testDef
+ * @param timeout in mins
*/
- void addTest(TestIdentifier testDef) {
+ void addTest(TestIdentifier testDef, int timeout) {
mTests.add(testDef);
mTestClasses.add(testDef.getClassName());
+ // 0 means no timeout, so keep 0 if already is.
+ if ((timeout > mTimeoutInMins) && (mTimeoutInMins != 0)) {
+ mTimeoutInMins = timeout;
+ }
}
/**
diff --git a/tools/tradefed-host/src/com/android/cts/tradefed/testtype/TestPackageXmlParser.java b/tools/tradefed-host/src/com/android/cts/tradefed/testtype/TestPackageXmlParser.java
index 35bdfd2..675c986 100644
--- a/tools/tradefed-host/src/com/android/cts/tradefed/testtype/TestPackageXmlParser.java
+++ b/tools/tradefed-host/src/com/android/cts/tradefed/testtype/TestPackageXmlParser.java
@@ -124,12 +124,16 @@
classNameBuilder.append(".");
}
}
-
+ int timeout = -1;
+ String timeoutStr = attributes.getValue("timeout");
+ if (timeoutStr != null) {
+ timeout = Integer.parseInt(timeoutStr);
+ }
TestIdentifier testId = new TestIdentifier(classNameBuilder.toString(),
methodName);
boolean isKnownFailure = "failure".equals(attributes.getValue("expectation"));
if (!isKnownFailure || mIncludeKnownFailures) {
- mPackageDef.addTest(testId);
+ mPackageDef.addTest(testId, timeout);
}
}
}
diff --git a/tools/utils/buildCts.py b/tools/utils/buildCts.py
index 77e93b1..8e23f13 100755
--- a/tools/utils/buildCts.py
+++ b/tools/utils/buildCts.py
@@ -165,8 +165,9 @@
if __name__ == '__main__':
builder = CtsBuilder(sys.argv)
- result = builder.GenerateTestDescriptions()
- if result != 0:
- sys.exit(result)
+ if builder.isCts:
+ result = builder.GenerateTestDescriptions()
+ if result != 0:
+ sys.exit(result)
builder.GenerateTestPlans()
