Option for Running Known Failures

Since knownfailures.txt is required by the build system, we cannot
simply delete the file to run known failures. Therefore add a
"run-known-failures" option to run even known failures.

The XML Generator now writes even known failures but adds
expectation="failure" to indicate that the test is expected to fail.
Then TestPackageXmlParser will only include these tests like a
normal test if "run-known-failures" has been set.

CtsConsole builds a TestPackageRepo but doesn't have simple
access to the flag value, so just assume known failures are not
included. This should be ok, because it only uses a TestPackageRepo
to list the packages.

Change-Id: Ie77c8b3f69c2164cffa7d231e357e52338b21dad
diff --git a/tools/cts-xml-generator/src/com/android/cts/xmlgenerator/XmlGenerator.java b/tools/cts-xml-generator/src/com/android/cts/xmlgenerator/XmlGenerator.java
index 9951b8d..cd2275d 100644
--- a/tools/cts-xml-generator/src/com/android/cts/xmlgenerator/XmlGenerator.java
+++ b/tools/cts-xml-generator/src/com/android/cts/xmlgenerator/XmlGenerator.java
@@ -178,9 +178,12 @@
             StringBuilder nameCollector) {
         for (String test : tests) {
             nameCollector.append('#').append(test);
-            if (!isKnownFailure(mExpectations, nameCollector.toString())) {
-                writer.append("<Test name=\"").append(test).println("\" />");
+            writer.append("<Test name=\"").append(test).append("\"");
+            if (isKnownFailure(mExpectations, nameCollector.toString())) {
+                writer.append(" expectation=\"failure\"");
             }
+            writer.println(" />");
+
             nameCollector.delete(nameCollector.length() - test.length() - 1,
                     nameCollector.length());
         }
diff --git a/tools/tradefed-host/src/com/android/cts/tradefed/build/CtsBuildHelper.java b/tools/tradefed-host/src/com/android/cts/tradefed/build/CtsBuildHelper.java
index 5945cd8..8029b75 100644
--- a/tools/tradefed-host/src/com/android/cts/tradefed/build/CtsBuildHelper.java
+++ b/tools/tradefed-host/src/com/android/cts/tradefed/build/CtsBuildHelper.java
@@ -135,7 +135,7 @@
     /**
      * @return a {@link File} representing the test plan directory
      */
-    public File getTestPlansDir() throws FileNotFoundException {
+    public File getTestPlansDir() {
         return new File(getRepositoryDir(), "plans");
     }
 
diff --git a/tools/tradefed-host/src/com/android/cts/tradefed/command/CtsConsole.java b/tools/tradefed-host/src/com/android/cts/tradefed/command/CtsConsole.java
index 82fa0d7..b688bfe 100644
--- a/tools/tradefed-host/src/com/android/cts/tradefed/command/CtsConsole.java
+++ b/tools/tradefed-host/src/com/android/cts/tradefed/command/CtsConsole.java
@@ -183,18 +183,13 @@
                 return name.endsWith(".xml");
             }
         };
-        try {
-            for (File planFile : ctsBuild.getTestPlansDir().listFiles(xmlFilter)) {
-                printLine(FileUtil.getBaseName(planFile.getName()));
-            }
-        }
-        catch (FileNotFoundException e) {
-            printLine("Could not find CTS plan folder");
+        for (File planFile : ctsBuild.getTestPlansDir().listFiles(xmlFilter)) {
+            printLine(FileUtil.getBaseName(planFile.getName()));
         }
     }
 
     private void listPackages(CtsBuildHelper ctsBuild) {
-        ITestPackageRepo testCaseRepo = new TestPackageRepo(ctsBuild.getTestCasesDir());
+        ITestPackageRepo testCaseRepo = new TestPackageRepo(ctsBuild.getTestCasesDir(), false);
         for (String packageUri : testCaseRepo.getPackageNames()) {
             printLine(packageUri);
         }
diff --git a/tools/tradefed-host/src/com/android/cts/tradefed/result/PlanCreator.java b/tools/tradefed-host/src/com/android/cts/tradefed/result/PlanCreator.java
index 5e0dcb8..d263bc8 100644
--- a/tools/tradefed-host/src/com/android/cts/tradefed/result/PlanCreator.java
+++ b/tools/tradefed-host/src/com/android/cts/tradefed/result/PlanCreator.java
@@ -16,6 +16,7 @@
 package com.android.cts.tradefed.result;
 
 import com.android.cts.tradefed.build.CtsBuildHelper;
+import com.android.cts.tradefed.testtype.CtsTest;
 import com.android.cts.tradefed.testtype.ITestPackageDef;
 import com.android.cts.tradefed.testtype.ITestPackageRepo;
 import com.android.cts.tradefed.testtype.ITestPlan;
@@ -55,6 +56,9 @@
             importance=Importance.IF_UNSET)
     private String mResultFilterString = null;
 
+    @Option(name = CtsTest.RUN_KNOWN_FAILURES_OPTION)
+    private boolean mIncludeKnownFailures = false;
+
     private CtsTestStatus mResultFilter = null;
     private TestResults mResult = null;
 
@@ -63,7 +67,8 @@
     /**
      * Create an empty {@link PlanCreator}.
      * <p/>
-     * All {@link Option} fields must be populated via {@link ArgsOptionParser}
+     * All {@link Option} fields must be populated via
+     * {@link com.android.tradefed.config.ArgsOptionParser}
      */
     public PlanCreator() {
     }
@@ -102,12 +107,13 @@
      * {@link Option} values must all be set before this is called.
      *
      * @param build
-     * @return
+     * @return test plan
      * @throws ConfigurationException
      */
     public ITestPlan createDerivedPlan(CtsBuildHelper build) throws ConfigurationException {
         checkFields(build);
-        ITestPackageRepo pkgDefRepo = new TestPackageRepo(build.getTestCasesDir());
+        ITestPackageRepo pkgDefRepo = new TestPackageRepo(build.getTestCasesDir(),
+                mIncludeKnownFailures);
         ITestPlan derivedPlan = new TestPlan(mPlanName);
         for (TestPackageResult pkg : mResult.getPackages()) {
             Collection<TestIdentifier> filteredTests = pkg.getTestsWithStatus(mResultFilter);
diff --git a/tools/tradefed-host/src/com/android/cts/tradefed/testtype/CtsTest.java b/tools/tradefed-host/src/com/android/cts/tradefed/testtype/CtsTest.java
index b18d7cc..8142036 100644
--- a/tools/tradefed-host/src/com/android/cts/tradefed/testtype/CtsTest.java
+++ b/tools/tradefed-host/src/com/android/cts/tradefed/testtype/CtsTest.java
@@ -72,6 +72,7 @@
     private static final String CLASS_OPTION = "class";
     private static final String METHOD_OPTION = "method";
     public static final String CONTINUE_OPTION = "continue-session";
+    public static final String RUN_KNOWN_FAILURES_OPTION = "run-known-failures";
 
     public static final String PACKAGE_NAME_METRIC = "packageName";
     public static final String PACKAGE_DIGEST_METRIC = "packageDigest";
@@ -127,6 +128,10 @@
         "Warning: can potentially use a lot of disk space.")
     private boolean mBugreport = false;
 
+    @Option(name = RUN_KNOWN_FAILURES_OPTION, shortName = 'k', description =
+        "run tests including known failures")
+    private boolean mIncludeKnownFailures;
+
     /** data structure for a {@link IRemoteTest} and its known tests */
     class TestPackage {
         private final IRemoteTest mTestForPackage;
@@ -584,7 +589,7 @@
      * Exposed for unit testing
      */
     ITestPackageRepo createTestCaseRepo() {
-        return new TestPackageRepo(mCtsBuild.getTestCasesDir());
+        return new TestPackageRepo(mCtsBuild.getTestCasesDir(), mIncludeKnownFailures);
     }
 
     /**
diff --git a/tools/tradefed-host/src/com/android/cts/tradefed/testtype/TestPackageRepo.java b/tools/tradefed-host/src/com/android/cts/tradefed/testtype/TestPackageRepo.java
index 77fa996..0f3704d 100644
--- a/tools/tradefed-host/src/com/android/cts/tradefed/testtype/TestPackageRepo.java
+++ b/tools/tradefed-host/src/com/android/cts/tradefed/testtype/TestPackageRepo.java
@@ -38,19 +38,22 @@
 
     private static final String LOG_TAG = "TestCaseRepo";
 
-    private File mTestCaseDir;
+    private final File mTestCaseDir;
 
     /** mapping of uri to test definition */
-    private Map<String, TestPackageDef> mTestMap;
+    private final Map<String, TestPackageDef> mTestMap;
+
+    private final boolean mIncludeKnownFailures;
 
     /**
      * Creates a {@link TestPackageRepo}, initialized from provided repo files
      *
      * @param testCaseDir directory containing all test case definition xml and build files
      */
-    public TestPackageRepo(File testCaseDir) {
+    public TestPackageRepo(File testCaseDir, boolean includeKnownFailures) {
         mTestCaseDir = testCaseDir;
         mTestMap = new Hashtable<String, TestPackageDef>();
+        mIncludeKnownFailures = includeKnownFailures;
         parse(mTestCaseDir);
     }
 
@@ -64,12 +67,8 @@
         }
     }
 
-    /**
-     * @param xmlFile
-     * @throws ParseException
-     */
     private void parseTestFromXml(File xmlFile)  {
-        TestPackageXmlParser parser = new TestPackageXmlParser();
+        TestPackageXmlParser parser = new TestPackageXmlParser(mIncludeKnownFailures);
         try {
             parser.parse(createStreamFromFile(xmlFile));
             TestPackageDef def = parser.getTestPackageDef();
@@ -96,7 +95,7 @@
      * Exposed for unit testing
      *
      * @param xmlFile
-     * @return
+     * @return stream to read data
      *
      */
     InputStream createStreamFromFile(File xmlFile) throws FileNotFoundException {
@@ -108,6 +107,7 @@
         /**
          * {@inheritDoc}
          */
+        @Override
         public boolean accept(File dir, String name) {
             return name.endsWith(".xml");
         }
@@ -135,9 +135,9 @@
     }
 
     /**
-     * Return a list of all package names found in repo.
-     * @return
+     * @return list of all package names found in repo
      */
+    @Override
     public Collection<String> getPackageNames() {
         List<String> packageNames = new ArrayList<String>();
         packageNames.addAll(mTestMap.keySet());
diff --git a/tools/tradefed-host/src/com/android/cts/tradefed/testtype/TestPackageXmlParser.java b/tools/tradefed-host/src/com/android/cts/tradefed/testtype/TestPackageXmlParser.java
index d24c5bf..35bdfd2 100644
--- a/tools/tradefed-host/src/com/android/cts/tradefed/testtype/TestPackageXmlParser.java
+++ b/tools/tradefed-host/src/com/android/cts/tradefed/testtype/TestPackageXmlParser.java
@@ -35,8 +35,14 @@
 
     private static final String LOG_TAG = "TestPackageXmlParser";
 
+    private final boolean mIncludeKnownFailures;
+
     private TestPackageDef mPackageDef;
 
+    public TestPackageXmlParser(boolean includeKnownFailures) {
+        mIncludeKnownFailures = includeKnownFailures;
+    }
+
     /**
      * SAX callback object. Handles parsing data from the xml tags.
      * <p/>
@@ -118,9 +124,13 @@
                             classNameBuilder.append(".");
                         }
                     }
-                    TestIdentifier testdef = new TestIdentifier(classNameBuilder.toString(),
+
+                    TestIdentifier testId = new TestIdentifier(classNameBuilder.toString(),
                             methodName);
-                    mPackageDef.addTest(testdef);
+                    boolean isKnownFailure = "failure".equals(attributes.getValue("expectation"));
+                    if (!isKnownFailure || mIncludeKnownFailures) {
+                        mPackageDef.addTest(testId);
+                    }
                 }
             }
 
diff --git a/tools/tradefed-host/tests/src/com/android/cts/tradefed/testtype/TestPackageXmlParserTest.java b/tools/tradefed-host/tests/src/com/android/cts/tradefed/testtype/TestPackageXmlParserTest.java
index 7861449..ac3d394 100644
--- a/tools/tradefed-host/tests/src/com/android/cts/tradefed/testtype/TestPackageXmlParserTest.java
+++ b/tools/tradefed-host/tests/src/com/android/cts/tradefed/testtype/TestPackageXmlParserTest.java
@@ -42,7 +42,7 @@
         "        <TestSuite name=\"example\" >\n" +
         "            <TestCase name=\"ExampleTest\" >\n" +
         "                <Test name=\"testFoo\" />\n" +
-        "                <Test name=\"testFoo2\" />\n" +
+        "                <Test name=\"testFoo2\" expectation=\"failure\" />\n" +
         "            </TestCase>\n" +
         "        </TestSuite>\n" +
         "        <TestSuite name=\"example2\" >\n" +
@@ -68,7 +68,7 @@
      * Test parsing test case xml containing an instrumentation test definition.
      */
     public void testParse_instrPackage() throws ParseException  {
-        TestPackageXmlParser parser = new TestPackageXmlParser();
+        TestPackageXmlParser parser = new TestPackageXmlParser(true);
         parser.parse(getStringAsStream(INSTR_TEST_DATA));
         TestPackageDef def = parser.getTestPackageDef();
         assertEquals("com.example", def.getAppNameSpace());
@@ -80,7 +80,7 @@
      * Test parsing test case xml containing an host test attribute and test data.
      */
     public void testParse_hostTest() throws ParseException  {
-        TestPackageXmlParser parser = new TestPackageXmlParser();
+        TestPackageXmlParser parser = new TestPackageXmlParser(true);
         parser.parse(getStringAsStream(HOST_TEST_DATA));
         TestPackageDef def = parser.getTestPackageDef();
         assertEquals(TestPackageDef.HOST_SIDE_ONLY_TEST, def.getTestType());
@@ -98,13 +98,34 @@
         TestIdentifier thirdTest = iterator.next();
         assertEquals("com.example2.Example2Test", thirdTest.getClassName());
         assertEquals("testFoo", thirdTest.getTestName());
+
+        assertFalse(iterator.hasNext());
+    }
+
+    public void testParse_hostTest_noKnownFailures() throws ParseException  {
+        TestPackageXmlParser parser = new TestPackageXmlParser(false);
+        parser.parse(getStringAsStream(HOST_TEST_DATA));
+        TestPackageDef def = parser.getTestPackageDef();
+        assertEquals(TestPackageDef.HOST_SIDE_ONLY_TEST, def.getTestType());
+        assertEquals(2, def.getTests().size());
+        Iterator<TestIdentifier> iterator = def.getTests().iterator();
+
+        TestIdentifier firstTest = iterator.next();
+        assertEquals("com.example.ExampleTest", firstTest.getClassName());
+        assertEquals("testFoo", firstTest.getTestName());
+
+        TestIdentifier thirdTest = iterator.next();
+        assertEquals("com.example2.Example2Test", thirdTest.getClassName());
+        assertEquals("testFoo", thirdTest.getTestName());
+
+        assertFalse(iterator.hasNext());
     }
 
     /**
      * Test parsing test case xml containing an invalid host test attribute.
      */
     public void testParse_badHostTest() throws ParseException  {
-        TestPackageXmlParser parser = new TestPackageXmlParser();
+        TestPackageXmlParser parser = new TestPackageXmlParser(true);
         parser.parse(getStringAsStream(BAD_HOST_TEST_DATA));
         TestPackageDef def = parser.getTestPackageDef();
         assertFalse(TestPackageDef.HOST_SIDE_ONLY_TEST.equals(def.getTestType()));
@@ -119,7 +140,7 @@
     }
 
     private void assertTestType(String expectedType, String xml) throws ParseException {
-        TestPackageXmlParser parser = new TestPackageXmlParser();
+        TestPackageXmlParser parser = new TestPackageXmlParser(true);
         parser.parse(getStringAsStream(xml));
         TestPackageDef def = parser.getTestPackageDef();
         assertEquals(expectedType, def.getTestType());
@@ -129,7 +150,7 @@
      * Test parsing a test case xml with no test package data.
      */
     public void testParse_noData() throws ParseException  {
-        TestPackageXmlParser parser = new TestPackageXmlParser();
+        TestPackageXmlParser parser = new TestPackageXmlParser(true);
         parser.parse(getStringAsStream(NO_TEST_DATA));
         assertNull(parser.getTestPackageDef());
     }