test_framework/com/android/tradefed/testtype/PythonUnitTestResultParser.java - platform/tools/tradefederation - Git at Google

 /*
  * Copyright (C) 2016 The Android Open Source Project
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
  * You may obtain a copy of the License at
  *
  *      http://www.apache.org/licenses/LICENSE-2.0
  *
  * Unless required by applicable law or agreed to in writing, software
  * distributed under the License is distributed on an "AS IS" BASIS,
  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
 package com.android.tradefed.testtype;

 import com.android.ddmlib.MultiLineReceiver;
 import com.android.tradefed.error.HarnessRuntimeException;
 import com.android.tradefed.log.LogUtil.CLog;
 import com.android.tradefed.metrics.proto.MetricMeasurement.Metric;
 import com.android.tradefed.result.ITestInvocationListener;
 import com.android.tradefed.result.TestDescription;
 import com.android.tradefed.result.error.TestErrorIdentifier;

 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collection;
 import java.util.HashMap;
 import java.util.LinkedHashMap;
 import java.util.LinkedHashSet;
 import java.util.List;
 import java.util.Map;
 import java.util.Map.Entry;
 import java.util.Set;
 import java.util.regex.MatchResult;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;

 /**
  * Interprets the output of tests run with Python's unittest framework and translates it into calls
  * on a series of {@link ITestInvocationListener}s. Output from these tests follows this EBNF
  * grammar:
  *
  * <p>TestReport ::= TestResult* Line TimeMetric [FailMessage*] Status. TestResult ::= string
  * “(“string”)” “…” SingleStatus. FailMessage ::= EqLine “ERROR:” string “(“string”)” Line Traceback
  * Line. SingleStatus ::= “ok” | “ERROR”. TimeMetric ::= “Ran” integer “tests in” float ”s”. Status
  * ::= “OK” | “FAILED (errors=” int “)”. Traceback ::= string+.
  *
  * <p>Example output (passing): test_size (test_rangelib.RangeSetTest) ... ok test_str
  * (test_rangelib.RangeSetTest) ... ok test_subtract (test_rangelib.RangeSetTest) ... ok
  * test_to_string_raw (test_rangelib.RangeSetTest) ... ok test_union (test_rangelib.RangeSetTest)
  * ... ok
  *
  * <p>---------------------------------------------------------------------- Ran 5 tests in 0.002s
  *
  * <p>OK
  *
  * <p>Example output (failed) test_size (test_rangelib.RangeSetTest) ... ERROR
  *
  * <p>====================================================================== ERROR: test_size
  * (test_rangelib.RangeSetTest)
  * ---------------------------------------------------------------------- Traceback (most recent
  * call last): File "test_rangelib.py", line 129, in test_rangelib raise ValueError() ValueError
  * ---------------------------------------------------------------------- Ran 1 test in 0.001s
  * FAILED (errors=1)
  *
  * <p>Example output with several edge cases (failed): testError (foo.testFoo) ... ERROR
  * testExpectedFailure (foo.testFoo) ... expected failure testFail (foo.testFoo) ... FAIL
  * testFailWithDocString (foo.testFoo) foo bar ... FAIL testOk (foo.testFoo) ... ok
  * testOkWithDocString (foo.testFoo) foo bar ... ok testSkipped (foo.testFoo) ... skipped 'reason
  * foo' testUnexpectedSuccess (foo.testFoo) ... unexpected success
  *
  * <p>====================================================================== ERROR: testError
  * (foo.testFoo) ---------------------------------------------------------------------- Traceback
  * (most recent call last): File "foo.py", line 11, in testError self.assertEqual(2+2, 5/0)
  * ZeroDivisionError: integer division or modulo by zero
  *
  * <p>====================================================================== FAIL: testFail
  * (foo.testFoo) ---------------------------------------------------------------------- Traceback
  * (most recent call last): File "foo.py", line 8, in testFail self.assertEqual(2+2, 5)
  * AssertionError: 4 != 5
  *
  * <p>====================================================================== FAIL:
  * testFailWithDocString (foo.testFoo) foo bar
  * ---------------------------------------------------------------------- Traceback (most recent
  * call last): File "foo.py", line 31, in testFailWithDocString self.assertEqual(2+2, 5)
  * AssertionError: 4 != 5
  *
  * <p>---------------------------------------------------------------------- Ran 8 tests in 0.001s
  *
  * <p>FAILED (failures=2, errors=1, skipped=1, expected failures=1, unexpected successes=1)
  *
  * <p>TODO: Consider refactoring the full class, handling is quite messy right now.
  */
 public class PythonUnitTestResultParser extends MultiLineReceiver {

     private boolean mFinalizeWhenParsing = true;
     // Current test state
     private ParserState mCurrentParseState;
     private String mCurrentTestName;
     private String mCurrentTestClass;
     private String mCurrentTestStatus;
     private Matcher mCurrentMatcher;
     private StringBuilder mCurrentTraceback;
     private long mTotalElapsedTime;
     private int mTotalTestCount;
     private String mCurrentTestCaseString = null;

     // Filters used for parsing test methods.
     private Set<String> mIncludeFilters = new LinkedHashSet<>();
     private Set<String> mExcludeFilters = new LinkedHashSet<>();

     // General state
     private Collection<ITestInvocationListener> mListeners = new ArrayList<>();
     private final String mRunName;
     private Map<TestDescription, String> mTestResultCache;
     // Use a special entry to mark skipped test in mTestResultCache
     static final String SKIPPED_ENTRY = "Skipped";

     // Constant tokens that appear in the result grammar.
     static final String EQUAL_LINE =
             "======================================================================";
     static final String DASH_LINE =
             "----------------------------------------------------------------------";
     static final String TRACEBACK_LINE = "Traceback (most recent call last):";

     static final Pattern PATTERN_TEST_SUCCESS = Pattern.compile("ok|expected failure");
     static final Pattern PATTERN_TEST_FAILURE = Pattern.compile("FAIL|ERROR");
     static final Pattern PATTERN_TEST_SKIPPED = Pattern.compile("skipped '.*");
     static final Pattern PATTERN_TEST_UNEXPECTED_SUCCESS = Pattern.compile("unexpected success");

     static final Pattern PATTERN_ONE_LINE_RESULT =
             Pattern.compile(
                     "(\\S*) \\((\\S*)\\) \\.\\.\\. "
                             + "(ok|expected failure|FAIL|ERROR|skipped '.*'|unexpected success)?");
     static final Pattern PATTERN_TWO_LINE_RESULT_FIRST = Pattern.compile("(\\S*) \\((\\S*)\\)");
     static final Pattern PATTERN_TWO_LINE_RESULT_SECOND =
             Pattern.compile(
                     "(.*) \\.\\.\\. "
                             + "(ok|expected failure|FAIL|ERROR|skipped '.*'|unexpected success)");
     static final Pattern PATTERN_TWO_LINE_RESULT_SECOND_ERROR =
             Pattern.compile(
                     "(.*) \\.\\.\\. error: (.*)"
                             + "(ok|expected failure|FAIL|ERROR|skipped '.*'|unexpected success)",
                     Pattern.DOTALL);
     static final Pattern PATTERN_FAIL_MESSAGE =
             Pattern.compile("(FAIL|ERROR): (\\S*) \\((\\S*)\\)( \\(.*\\))?");

     static final Pattern PATTERN_RUN_SUMMARY =
             Pattern.compile("Ran (\\d+) tests? in (\\d+(.\\d*)?)s(.*)");

     /** In case of error spanning over multiple lines. */
     static final Pattern MULTILINE_RESULT_WITH_WARNING =
             Pattern.compile("(.*) \\.\\.\\. (.*)", Pattern.DOTALL);

     static final Pattern MULTILINE_FINAL_RESULT_WITH_WARNING =
             Pattern.compile("(.*) \\.\\.\\. (.*)ok(.*)", Pattern.DOTALL);

     /** Unexpected (multiline) text between ... and test status - likely corrupted */
     static final Pattern PATTERN_MULTILINE_RESULT_FIRST =
             Pattern.compile("(\\S*) \\((\\S*)\\) \\.\\.\\. .+");

     static final Pattern PATTERN_MULTILINE_RESULT_FIRST_NEGATIVE =
             Pattern.compile(
                     "(\\S*) \\((\\S*)\\) \\.\\.\\. (ok|expected failure|FAIL|ERROR|error|skipped"
                             + " '.*'|unexpected success).*");
     static final Pattern PATTERN_MULTILINE_RESULT_LAST =
             Pattern.compile("(ok|expected failure|FAIL|ERROR|skipped '.*'|unexpected success)");

     static final Pattern PATTERN_RUN_RESULT = Pattern.compile("(OK|FAILED).*");

     enum ParserState {
         TEST_CASE,
         FAIL_MESSAGE,
         TRACEBACK,
         SUMMARY,
         COMPLETE,
     }

     private class PythonUnitTestParseException extends Exception {
         static final long serialVersionUID = -3387516993124229948L;

         public PythonUnitTestParseException(String reason) {
             super(reason);
         }
     }

     /**
      * Create a new {@link PythonUnitTestResultParser} that reports to the given {@link
      * ITestInvocationListener}.
      */
     public PythonUnitTestResultParser(ITestInvocationListener listener, String runName) {
         this(Arrays.asList(listener), runName);
     }

     /**
      * Create a new {@link PythonUnitTestResultParser} that reports to the given {@link
      * ITestInvocationListener}s.
      */
     public PythonUnitTestResultParser(
             Collection<ITestInvocationListener> listeners, String runName) {
         this(listeners, runName, new LinkedHashSet<>(), new LinkedHashSet<>());
     }

     /**
      * Create a new {@link PythonUnitTestResultParser} that reports to the given {@link
      * ITestInvocationListener}s, with specified include and exclude filters.
      */
     public PythonUnitTestResultParser(
             Collection<ITestInvocationListener> listeners,
             String runName,
             Set<String> includeFilters,
             Set<String> excludeFilters) {
         mListeners.addAll(listeners);
         mRunName = runName;
         mTestResultCache = new LinkedHashMap<>();
         mIncludeFilters = includeFilters;
         mExcludeFilters = excludeFilters;
         mCurrentParseState = ParserState.TEST_CASE;
     }

     /**
      * Process Python unittest output and report parsed results.
      *
      * <p>This method should be called only once with the full output, unlike the base method in
      * {@link MultiLineReceiver}.
      */
     @Override
     public void processNewLines(String[] lines) {
         try {
             if (lines.length < 1 || isTracebackLine(lines[0])) {
                 throw new PythonUnitTestParseException("Test execution failed");
             }

             for (String line : lines) {
                 parse(line);
             }

             if (mFinalizeWhenParsing) {
                 finalizeParser();
             }
         } catch (PythonUnitTestParseException e) {
             throw new HarnessRuntimeException(
                     e.getMessage(), TestErrorIdentifier.OUTPUT_PARSER_ERROR);
         }
     }

     public void setFinalizeWhenParsing(boolean shouldFinalize) {
         mFinalizeWhenParsing = shouldFinalize;
     }

     public void finalizeParser() {
         if (mCurrentParseState != ParserState.COMPLETE) {
             throw new HarnessRuntimeException(
                     "Parser finished in unexpected state " + mCurrentParseState.toString(),
                     TestErrorIdentifier.OUTPUT_PARSER_ERROR);
         }
     }

     /** Parse the next result line according to current parser state. */
     void parse(String line) throws PythonUnitTestParseException {
         CLog.v(line);
         switch (mCurrentParseState) {
             case TEST_CASE:
                 processTestCase(line);
                 break;
             case TRACEBACK:
                 processTraceback(line);
                 break;
             case SUMMARY:
                 processRunSummary(line);
                 break;
             case FAIL_MESSAGE:
                 processFailMessage(line);
                 break;
             case COMPLETE:
                 break;
         }
     }

     /** Process a test case line and collect the test name, class, and status. */
     void processTestCase(String line) throws PythonUnitTestParseException {
         if (mCurrentTestCaseString != null) {
             mCurrentTestCaseString = mCurrentTestCaseString + "\n" + line;
             line = mCurrentTestCaseString;
         }

         if (isEqualLine(line)) {
             // equal line before fail message
             mCurrentParseState = ParserState.FAIL_MESSAGE;
             mCurrentTestCaseString = null;
         } else if (isDashLine(line)) {
             // dash line before run summary
             mCurrentParseState = ParserState.SUMMARY;
             mCurrentTestCaseString = null;
         } else if (lineStartswithPattern(line, PATTERN_ONE_LINE_RESULT)) {
             // The below parsing is involved due to output from tests that use Python's subTest
             // feature. In particular, a line could contain multiple test case summary lines such
             // as; a (T) ... b (T) ... ok.
             // TODO(hzalek): Consider adding a Python support library for writing the output of the
             // test in a structured format to avoid parsing string output.
             List<MatchResult> matchResults = new ArrayList<>();

             // Collect the results to avoid modifying state in case the entire line doesn't match.
             do {
                 matchResults.add(mCurrentMatcher.toMatchResult());
             } while (mCurrentMatcher.find());

             int lastMatchEnd = matchResults.get(matchResults.size() - 1).end();
             if (lastMatchEnd != line.length()) {
                 boolean canBeMultiline =
                         !lineMatchesPattern(line, PATTERN_MULTILINE_RESULT_FIRST_NEGATIVE);
                 if (canBeMultiline && lineMatchesPattern(line, PATTERN_MULTILINE_RESULT_FIRST)) {
                     mCurrentTestName = mCurrentMatcher.group(1);
                     mCurrentTestClass =
                             removeTestNameFromClassNameGroup(
                                     mCurrentMatcher.group(2), mCurrentTestName);
                     mCurrentTestCaseString = null;
                 }
                 return; // The entire line doesn't match so just ignore it.
             }

             for (MatchResult r : matchResults) {
                 mCurrentTestName = r.group(1);
                 mCurrentTestClass = removeTestNameFromClassNameGroup(r.group(2), mCurrentTestName);
                 mCurrentTestStatus = r.group(3);

                 // Tests with failed subtests have no status printed so we add an entry with 'FAIL'
                 // status. In any case, subsequent failed subtest assertions have the same test name
                 // and will clobber whatever status we set here. Passed subtest assertions don't
                 // appear in the output and do not risk overwriting the status.
                 if (mCurrentTestStatus == null) {
                     mCurrentTestStatus = "FAIL";
                 }

                 reportNonFailureTestResult();
             }

             mCurrentTestCaseString = null;
         } else if (lineMatchesPattern(line, PATTERN_MULTILINE_RESULT_LAST)) {
             mCurrentTestStatus = mCurrentMatcher.group(1);
             reportNonFailureTestResult();
             mCurrentTestCaseString = null;
         } else if (lineMatchesPattern(line, PATTERN_TWO_LINE_RESULT_FIRST)) {
             mCurrentTestName = mCurrentMatcher.group(1);
             mCurrentTestClass =
                     removeTestNameFromClassNameGroup(mCurrentMatcher.group(2), mCurrentTestName);
             mCurrentTestCaseString = null;
         } else if (lineMatchesPattern(line, PATTERN_TWO_LINE_RESULT_SECOND)) {
             mCurrentTestStatus = mCurrentMatcher.group(2);
             reportNonFailureTestResult();
             mCurrentTestCaseString = null;
         } else if (lineMatchesPattern(line, PATTERN_TWO_LINE_RESULT_SECOND_ERROR)) {
             // Skip that odd error message
             mCurrentTestCaseString = null;
         } else if (lineMatchesPattern(line, MULTILINE_FINAL_RESULT_WITH_WARNING)) {
             StringBuilder message = new StringBuilder("Test seems to pass but with Warnings:\n");
             message.append(mCurrentMatcher.group(2));
             mCurrentTraceback = message;
             reportFailureTestResult();
             mCurrentTestCaseString = null;
         } else if (lineMatchesPattern(line, MULTILINE_RESULT_WITH_WARNING)) {
             if (mCurrentTestCaseString == null) {
                 mCurrentTestCaseString = line;
             }
         }
     }

     String removeTestNameFromClassNameGroup(String classNameGroup, String testName) {
         if (!classNameGroup.endsWith(testName)) {
             return classNameGroup;
         }
         Pattern p = Pattern.compile("(.*)\\." + testName);
         Matcher matcher = p.matcher(classNameGroup);
         if (!matcher.matches()) {
             return classNameGroup;
         }
         return matcher.group(1);
     }

     /** Process a fail message line and collect the test name, class, and status. */
     void processFailMessage(String line) {
         if (isDashLine(line)) {
             // dash line before traceback
             mCurrentParseState = ParserState.TRACEBACK;
             mCurrentTraceback = new StringBuilder();
         } else if (lineMatchesPattern(line, PATTERN_FAIL_MESSAGE)) {
             mCurrentTestName = mCurrentMatcher.group(2);
             mCurrentTestClass =
                     removeTestNameFromClassNameGroup(mCurrentMatcher.group(3), mCurrentTestName);
             mCurrentTestStatus = mCurrentMatcher.group(1);
         }
         // optional docstring - do nothing
     }

     /** Process a traceback line and append it to the full traceback message. */
     void processTraceback(String line) {
         if (isDashLine(line)) {
             // dash line before run summary
             mCurrentParseState = ParserState.SUMMARY;
             reportFailureTestResult();
         } else if (isEqualLine(line)) {
             // equal line before another fail message followed by traceback
             mCurrentParseState = ParserState.FAIL_MESSAGE;
             reportFailureTestResult();
         } else {
             if (mCurrentTraceback.length() > 0) {
                 mCurrentTraceback.append(System.lineSeparator());
             }
             mCurrentTraceback.append(line);
         }
     }

     /** Process the run summary line and collect the test count and run time. */
     void processRunSummary(String line) {
         if (lineMatchesPattern(line, PATTERN_RUN_SUMMARY)) {
             mTotalTestCount = Integer.parseInt(mCurrentMatcher.group(1));
             double timeInSeconds = Double.parseDouble(mCurrentMatcher.group(2));
             mTotalElapsedTime = (long) (timeInSeconds * 1000D);
             reportToListeners();
             mCurrentParseState = ParserState.COMPLETE;
         }
         // ignore status message on the last line because Python consider "unexpected success"
         // passed while we consider it failed
     }

     boolean isEqualLine(String line) {
         return line.startsWith(EQUAL_LINE);
     }

     boolean isDashLine(String line) {
         return line.startsWith(DASH_LINE);
     }

     boolean isTracebackLine(String line) {
         return line.startsWith(TRACEBACK_LINE);
     }

     /** Check if the given line matches the given pattern and caches the matcher object */
     private boolean lineMatchesPattern(String line, Pattern p) {
         mCurrentMatcher = p.matcher(line);
         return mCurrentMatcher.matches();
     }

     private boolean lineStartswithPattern(String line, Pattern p) {
         mCurrentMatcher = p.matcher(line);
         return mCurrentMatcher.find();
     }

     /** Send recorded test results to all listeners. */
     private void reportToListeners() {
         for (ITestInvocationListener listener : mListeners) {
             listener.testRunStarted(mRunName, mTotalTestCount);

             for (Entry<TestDescription, String> test : mTestResultCache.entrySet()) {
                 listener.testStarted(test.getKey());
                 if (SKIPPED_ENTRY.equals(test.getValue())) {
                     listener.testIgnored(test.getKey());
                 } else if (test.getValue() != null) {
                     listener.testFailed(test.getKey(), test.getValue());
                 }
                 listener.testEnded(test.getKey(), new HashMap<String, Metric>());
             }
             listener.testRunEnded(mTotalElapsedTime, new HashMap<String, Metric>());
         }
     }

     /** Record a non-failure test case. */
     private void reportNonFailureTestResult() throws PythonUnitTestParseException {
         TestDescription testId = new TestDescription(mCurrentTestClass, mCurrentTestName);

         if (shouldSkipCurrentTest()) {
             // Force to skip any test not listed in include filters, or listed in exclude filters.
             mTestResultCache.put(testId, SKIPPED_ENTRY);
         } else if (PATTERN_TEST_SUCCESS.matcher(mCurrentTestStatus).matches()) {
             mTestResultCache.put(testId, null);
         } else if (PATTERN_TEST_SKIPPED.matcher(mCurrentTestStatus).matches()) {
             mTestResultCache.put(testId, SKIPPED_ENTRY);
         } else if (PATTERN_TEST_UNEXPECTED_SUCCESS.matcher(mCurrentTestStatus).matches()) {
             mTestResultCache.put(testId, "Test unexpected succeeded");
         } else if (PATTERN_TEST_FAILURE.matcher(mCurrentTestStatus).matches()) {
             // do nothing for now, report only after traceback is collected
         } else {
             throw new PythonUnitTestParseException("Unrecognized test status");
         }
     }

     /** Record a failed test case and its traceback message. */
     private void reportFailureTestResult() {
         TestDescription testId = new TestDescription(mCurrentTestClass, mCurrentTestName);
         if (shouldSkipCurrentTest()) {
             mTestResultCache.put(testId, SKIPPED_ENTRY);
         } else {
             mTestResultCache.put(testId, mCurrentTraceback.toString());
         }
     }

     /**
      * Check if current test should be skipped.
      *
      * @return true if the test should be skipped.
      */
     private boolean shouldSkipCurrentTest() {
         // Force to skip any test not listed in include filters, or listed in exclude filters.
         // exclude filters have highest priority.
         if (mExcludeFilters.contains(mCurrentTestClass + "#" + mCurrentTestName)
                 || mExcludeFilters.contains(mCurrentTestClass)) {
             return true;
         }
         if (!mIncludeFilters.isEmpty()) {
             if (mIncludeFilters.contains(mCurrentTestClass + "#" + mCurrentTestName)
                     || mIncludeFilters.contains(mCurrentTestClass)) {
                 return false;
             }
             for (String filter : mIncludeFilters) {
                 if ((mCurrentTestClass + "#" + mCurrentTestName).matches(filter)) {
                     return false;
                 }
             }
             return true;
         }
         return false;
     }

     @Override
     public boolean isCancelled() {
         return false;
     }
 }
	/*
	* Copyright (C) 2016 The Android Open Source Project
	*
	* Licensed under the Apache License, Version 2.0 (the "License");
	* you may not use this file except in compliance with the License.
	* You may obtain a copy of the License at
	*
	* http://www.apache.org/licenses/LICENSE-2.0
	*
	* Unless required by applicable law or agreed to in writing, software
	* distributed under the License is distributed on an "AS IS" BASIS,
	* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
	* See the License for the specific language governing permissions and
	* limitations under the License.
	*/
	package com.android.tradefed.testtype;

	import com.android.ddmlib.MultiLineReceiver;
	import com.android.tradefed.error.HarnessRuntimeException;
	import com.android.tradefed.log.LogUtil.CLog;
	import com.android.tradefed.metrics.proto.MetricMeasurement.Metric;
	import com.android.tradefed.result.ITestInvocationListener;
	import com.android.tradefed.result.TestDescription;
	import com.android.tradefed.result.error.TestErrorIdentifier;

	import java.util.ArrayList;
	import java.util.Arrays;
	import java.util.Collection;
	import java.util.HashMap;
	import java.util.LinkedHashMap;
	import java.util.LinkedHashSet;
	import java.util.List;
	import java.util.Map;
	import java.util.Map.Entry;
	import java.util.Set;
	import java.util.regex.MatchResult;
	import java.util.regex.Matcher;
	import java.util.regex.Pattern;

	/**
	* Interprets the output of tests run with Python's unittest framework and translates it into calls
	* on a series of {@link ITestInvocationListener}s. Output from these tests follows this EBNF
	* grammar:
	*
	* <p>TestReport ::= TestResult* Line TimeMetric [FailMessage*] Status. TestResult ::= string
	* “(“string”)” “…” SingleStatus. FailMessage ::= EqLine “ERROR:” string “(“string”)” Line Traceback
	* Line. SingleStatus ::= “ok” \| “ERROR”. TimeMetric ::= “Ran” integer “tests in” float ”s”. Status
	* ::= “OK” \| “FAILED (errors=” int “)”. Traceback ::= string+.
	*
	* <p>Example output (passing): test_size (test_rangelib.RangeSetTest) ... ok test_str
	* (test_rangelib.RangeSetTest) ... ok test_subtract (test_rangelib.RangeSetTest) ... ok
	* test_to_string_raw (test_rangelib.RangeSetTest) ... ok test_union (test_rangelib.RangeSetTest)
	* ... ok
	*
	* <p>---------------------------------------------------------------------- Ran 5 tests in 0.002s
	*
	* <p>OK
	*
	* <p>Example output (failed) test_size (test_rangelib.RangeSetTest) ... ERROR
	*
	* <p>====================================================================== ERROR: test_size
	* (test_rangelib.RangeSetTest)
	* ---------------------------------------------------------------------- Traceback (most recent
	* call last): File "test_rangelib.py", line 129, in test_rangelib raise ValueError() ValueError
	* ---------------------------------------------------------------------- Ran 1 test in 0.001s
	* FAILED (errors=1)
	*
	* <p>Example output with several edge cases (failed): testError (foo.testFoo) ... ERROR
	* testExpectedFailure (foo.testFoo) ... expected failure testFail (foo.testFoo) ... FAIL
	* testFailWithDocString (foo.testFoo) foo bar ... FAIL testOk (foo.testFoo) ... ok
	* testOkWithDocString (foo.testFoo) foo bar ... ok testSkipped (foo.testFoo) ... skipped 'reason
	* foo' testUnexpectedSuccess (foo.testFoo) ... unexpected success
	*
	* <p>====================================================================== ERROR: testError
	* (foo.testFoo) ---------------------------------------------------------------------- Traceback
	* (most recent call last): File "foo.py", line 11, in testError self.assertEqual(2+2, 5/0)
	* ZeroDivisionError: integer division or modulo by zero
	*
	* <p>====================================================================== FAIL: testFail
	* (foo.testFoo) ---------------------------------------------------------------------- Traceback
	* (most recent call last): File "foo.py", line 8, in testFail self.assertEqual(2+2, 5)
	* AssertionError: 4 != 5
	*
	* <p>====================================================================== FAIL:
	* testFailWithDocString (foo.testFoo) foo bar
	* ---------------------------------------------------------------------- Traceback (most recent
	* call last): File "foo.py", line 31, in testFailWithDocString self.assertEqual(2+2, 5)
	* AssertionError: 4 != 5
	*
	* <p>---------------------------------------------------------------------- Ran 8 tests in 0.001s
	*
	* <p>FAILED (failures=2, errors=1, skipped=1, expected failures=1, unexpected successes=1)
	*
	* <p>TODO: Consider refactoring the full class, handling is quite messy right now.
	*/
	public class PythonUnitTestResultParser extends MultiLineReceiver {

	private boolean mFinalizeWhenParsing = true;
	// Current test state
	private ParserState mCurrentParseState;
	private String mCurrentTestName;
	private String mCurrentTestClass;
	private String mCurrentTestStatus;
	private Matcher mCurrentMatcher;
	private StringBuilder mCurrentTraceback;
	private long mTotalElapsedTime;
	private int mTotalTestCount;
	private String mCurrentTestCaseString = null;

	// Filters used for parsing test methods.
	private Set<String> mIncludeFilters = new LinkedHashSet<>();
	private Set<String> mExcludeFilters = new LinkedHashSet<>();

	// General state
	private Collection<ITestInvocationListener> mListeners = new ArrayList<>();
	private final String mRunName;
	private Map<TestDescription, String> mTestResultCache;
	// Use a special entry to mark skipped test in mTestResultCache
	static final String SKIPPED_ENTRY = "Skipped";

	// Constant tokens that appear in the result grammar.
	static final String EQUAL_LINE =
	"======================================================================";
	static final String DASH_LINE =
	"----------------------------------------------------------------------";
	static final String TRACEBACK_LINE = "Traceback (most recent call last):";

	static final Pattern PATTERN_TEST_SUCCESS = Pattern.compile("ok\|expected failure");
	static final Pattern PATTERN_TEST_FAILURE = Pattern.compile("FAIL\|ERROR");
	static final Pattern PATTERN_TEST_SKIPPED = Pattern.compile("skipped '.*");
	static final Pattern PATTERN_TEST_UNEXPECTED_SUCCESS = Pattern.compile("unexpected success");

	static final Pattern PATTERN_ONE_LINE_RESULT =
	Pattern.compile(
	"(\\S) \\((\\S)\\) \\.\\.\\. "
	+ "(ok\|expected failure\|FAIL\|ERROR\|skipped '.*'\|unexpected success)?");
	static final Pattern PATTERN_TWO_LINE_RESULT_FIRST = Pattern.compile("(\\S) \\((\\S)\\)");
	static final Pattern PATTERN_TWO_LINE_RESULT_SECOND =
	Pattern.compile(
	"(.*) \\.\\.\\. "
	+ "(ok\|expected failure\|FAIL\|ERROR\|skipped '.*'\|unexpected success)");
	static final Pattern PATTERN_TWO_LINE_RESULT_SECOND_ERROR =
	Pattern.compile(
	"(.) \\.\\.\\. error: (.)"
	+ "(ok\|expected failure\|FAIL\|ERROR\|skipped '.*'\|unexpected success)",
	Pattern.DOTALL);
	static final Pattern PATTERN_FAIL_MESSAGE =
	Pattern.compile("(FAIL\|ERROR): (\\S) \\((\\S)\\)( \\(.*\\))?");

	static final Pattern PATTERN_RUN_SUMMARY =
	Pattern.compile("Ran (\\d+) tests? in (\\d+(.\\d)?)s(.)");

	/** In case of error spanning over multiple lines. */
	static final Pattern MULTILINE_RESULT_WITH_WARNING =
	Pattern.compile("(.) \\.\\.\\. (.)", Pattern.DOTALL);

	static final Pattern MULTILINE_FINAL_RESULT_WITH_WARNING =
	Pattern.compile("(.) \\.\\.\\. (.)ok(.*)", Pattern.DOTALL);

	/** Unexpected (multiline) text between ... and test status - likely corrupted */
	static final Pattern PATTERN_MULTILINE_RESULT_FIRST =
	Pattern.compile("(\\S) \\((\\S)\\) \\.\\.\\. .+");

	static final Pattern PATTERN_MULTILINE_RESULT_FIRST_NEGATIVE =
	Pattern.compile(
	"(\\S) \\((\\S)\\) \\.\\.\\. (ok\|expected failure\|FAIL\|ERROR\|error\|skipped"
	+ " '.'\|unexpected success).");
	static final Pattern PATTERN_MULTILINE_RESULT_LAST =
	Pattern.compile("(ok\|expected failure\|FAIL\|ERROR\|skipped '.*'\|unexpected success)");

	static final Pattern PATTERN_RUN_RESULT = Pattern.compile("(OK\|FAILED).*");

	enum ParserState {
	TEST_CASE,
	FAIL_MESSAGE,
	TRACEBACK,
	SUMMARY,
	COMPLETE,
	}

	private class PythonUnitTestParseException extends Exception {
	static final long serialVersionUID = -3387516993124229948L;

	public PythonUnitTestParseException(String reason) {
	super(reason);
	}
	}

	/**
	* Create a new {@link PythonUnitTestResultParser} that reports to the given {@link
	* ITestInvocationListener}.
	*/
	public PythonUnitTestResultParser(ITestInvocationListener listener, String runName) {
	this(Arrays.asList(listener), runName);
	}

	/**
	* Create a new {@link PythonUnitTestResultParser} that reports to the given {@link
	* ITestInvocationListener}s.
	*/
	public PythonUnitTestResultParser(
	Collection<ITestInvocationListener> listeners, String runName) {
	this(listeners, runName, new LinkedHashSet<>(), new LinkedHashSet<>());
	}

	/**
	* Create a new {@link PythonUnitTestResultParser} that reports to the given {@link
	* ITestInvocationListener}s, with specified include and exclude filters.
	*/
	public PythonUnitTestResultParser(
	Collection<ITestInvocationListener> listeners,
	String runName,
	Set<String> includeFilters,
	Set<String> excludeFilters) {
	mListeners.addAll(listeners);
	mRunName = runName;
	mTestResultCache = new LinkedHashMap<>();
	mIncludeFilters = includeFilters;
	mExcludeFilters = excludeFilters;
	mCurrentParseState = ParserState.TEST_CASE;
	}

	/**
	* Process Python unittest output and report parsed results.
	*
	* <p>This method should be called only once with the full output, unlike the base method in
	* {@link MultiLineReceiver}.
	*/
	@Override
	public void processNewLines(String[] lines) {
	try {
	if (lines.length < 1 \|\| isTracebackLine(lines[0])) {
	throw new PythonUnitTestParseException("Test execution failed");
	}

	for (String line : lines) {
	parse(line);
	}

	if (mFinalizeWhenParsing) {
	finalizeParser();
	}
	} catch (PythonUnitTestParseException e) {
	throw new HarnessRuntimeException(
	e.getMessage(), TestErrorIdentifier.OUTPUT_PARSER_ERROR);
	}
	}

	public void setFinalizeWhenParsing(boolean shouldFinalize) {
	mFinalizeWhenParsing = shouldFinalize;
	}

	public void finalizeParser() {
	if (mCurrentParseState != ParserState.COMPLETE) {
	throw new HarnessRuntimeException(
	"Parser finished in unexpected state " + mCurrentParseState.toString(),
	TestErrorIdentifier.OUTPUT_PARSER_ERROR);
	}
	}

	/** Parse the next result line according to current parser state. */
	void parse(String line) throws PythonUnitTestParseException {
	CLog.v(line);
	switch (mCurrentParseState) {
	case TEST_CASE:
	processTestCase(line);
	break;
	case TRACEBACK:
	processTraceback(line);
	break;
	case SUMMARY:
	processRunSummary(line);
	break;
	case FAIL_MESSAGE:
	processFailMessage(line);
	break;
	case COMPLETE:
	break;
	}
	}

	/** Process a test case line and collect the test name, class, and status. */
	void processTestCase(String line) throws PythonUnitTestParseException {
	if (mCurrentTestCaseString != null) {
	mCurrentTestCaseString = mCurrentTestCaseString + "\n" + line;
	line = mCurrentTestCaseString;
	}

	if (isEqualLine(line)) {
	// equal line before fail message
	mCurrentParseState = ParserState.FAIL_MESSAGE;
	mCurrentTestCaseString = null;
	} else if (isDashLine(line)) {
	// dash line before run summary
	mCurrentParseState = ParserState.SUMMARY;
	mCurrentTestCaseString = null;
	} else if (lineStartswithPattern(line, PATTERN_ONE_LINE_RESULT)) {
	// The below parsing is involved due to output from tests that use Python's subTest
	// feature. In particular, a line could contain multiple test case summary lines such
	// as; a (T) ... b (T) ... ok.
	// TODO(hzalek): Consider adding a Python support library for writing the output of the
	// test in a structured format to avoid parsing string output.
	List<MatchResult> matchResults = new ArrayList<>();

	// Collect the results to avoid modifying state in case the entire line doesn't match.
	do {
	matchResults.add(mCurrentMatcher.toMatchResult());
	} while (mCurrentMatcher.find());

	int lastMatchEnd = matchResults.get(matchResults.size() - 1).end();
	if (lastMatchEnd != line.length()) {
	boolean canBeMultiline =
	!lineMatchesPattern(line, PATTERN_MULTILINE_RESULT_FIRST_NEGATIVE);
	if (canBeMultiline && lineMatchesPattern(line, PATTERN_MULTILINE_RESULT_FIRST)) {
	mCurrentTestName = mCurrentMatcher.group(1);
	mCurrentTestClass =
	removeTestNameFromClassNameGroup(
	mCurrentMatcher.group(2), mCurrentTestName);
	mCurrentTestCaseString = null;
	}
	return; // The entire line doesn't match so just ignore it.
	}

	for (MatchResult r : matchResults) {
	mCurrentTestName = r.group(1);
	mCurrentTestClass = removeTestNameFromClassNameGroup(r.group(2), mCurrentTestName);
	mCurrentTestStatus = r.group(3);

	// Tests with failed subtests have no status printed so we add an entry with 'FAIL'
	// status. In any case, subsequent failed subtest assertions have the same test name
	// and will clobber whatever status we set here. Passed subtest assertions don't
	// appear in the output and do not risk overwriting the status.
	if (mCurrentTestStatus == null) {
	mCurrentTestStatus = "FAIL";
	}

	reportNonFailureTestResult();
	}

	mCurrentTestCaseString = null;
	} else if (lineMatchesPattern(line, PATTERN_MULTILINE_RESULT_LAST)) {
	mCurrentTestStatus = mCurrentMatcher.group(1);
	reportNonFailureTestResult();
	mCurrentTestCaseString = null;
	} else if (lineMatchesPattern(line, PATTERN_TWO_LINE_RESULT_FIRST)) {
	mCurrentTestName = mCurrentMatcher.group(1);
	mCurrentTestClass =
	removeTestNameFromClassNameGroup(mCurrentMatcher.group(2), mCurrentTestName);
	mCurrentTestCaseString = null;
	} else if (lineMatchesPattern(line, PATTERN_TWO_LINE_RESULT_SECOND)) {
	mCurrentTestStatus = mCurrentMatcher.group(2);
	reportNonFailureTestResult();
	mCurrentTestCaseString = null;
	} else if (lineMatchesPattern(line, PATTERN_TWO_LINE_RESULT_SECOND_ERROR)) {
	// Skip that odd error message
	mCurrentTestCaseString = null;
	} else if (lineMatchesPattern(line, MULTILINE_FINAL_RESULT_WITH_WARNING)) {
	StringBuilder message = new StringBuilder("Test seems to pass but with Warnings:\n");
	message.append(mCurrentMatcher.group(2));
	mCurrentTraceback = message;
	reportFailureTestResult();
	mCurrentTestCaseString = null;
	} else if (lineMatchesPattern(line, MULTILINE_RESULT_WITH_WARNING)) {
	if (mCurrentTestCaseString == null) {
	mCurrentTestCaseString = line;
	}
	}
	}

	String removeTestNameFromClassNameGroup(String classNameGroup, String testName) {
	if (!classNameGroup.endsWith(testName)) {
	return classNameGroup;
	}
	Pattern p = Pattern.compile("(.*)\\." + testName);
	Matcher matcher = p.matcher(classNameGroup);
	if (!matcher.matches()) {
	return classNameGroup;
	}
	return matcher.group(1);
	}

	/** Process a fail message line and collect the test name, class, and status. */
	void processFailMessage(String line) {
	if (isDashLine(line)) {
	// dash line before traceback
	mCurrentParseState = ParserState.TRACEBACK;
	mCurrentTraceback = new StringBuilder();
	} else if (lineMatchesPattern(line, PATTERN_FAIL_MESSAGE)) {
	mCurrentTestName = mCurrentMatcher.group(2);
	mCurrentTestClass =
	removeTestNameFromClassNameGroup(mCurrentMatcher.group(3), mCurrentTestName);
	mCurrentTestStatus = mCurrentMatcher.group(1);
	}
	// optional docstring - do nothing
	}

	/** Process a traceback line and append it to the full traceback message. */
	void processTraceback(String line) {
	if (isDashLine(line)) {
	// dash line before run summary
	mCurrentParseState = ParserState.SUMMARY;
	reportFailureTestResult();
	} else if (isEqualLine(line)) {
	// equal line before another fail message followed by traceback
	mCurrentParseState = ParserState.FAIL_MESSAGE;
	reportFailureTestResult();
	} else {
	if (mCurrentTraceback.length() > 0) {
	mCurrentTraceback.append(System.lineSeparator());
	}
	mCurrentTraceback.append(line);
	}
	}

	/** Process the run summary line and collect the test count and run time. */
	void processRunSummary(String line) {
	if (lineMatchesPattern(line, PATTERN_RUN_SUMMARY)) {
	mTotalTestCount = Integer.parseInt(mCurrentMatcher.group(1));
	double timeInSeconds = Double.parseDouble(mCurrentMatcher.group(2));
	mTotalElapsedTime = (long) (timeInSeconds * 1000D);
	reportToListeners();
	mCurrentParseState = ParserState.COMPLETE;
	}
	// ignore status message on the last line because Python consider "unexpected success"
	// passed while we consider it failed
	}

	boolean isEqualLine(String line) {
	return line.startsWith(EQUAL_LINE);
	}

	boolean isDashLine(String line) {
	return line.startsWith(DASH_LINE);
	}

	boolean isTracebackLine(String line) {
	return line.startsWith(TRACEBACK_LINE);
	}

	/** Check if the given line matches the given pattern and caches the matcher object */
	private boolean lineMatchesPattern(String line, Pattern p) {
	mCurrentMatcher = p.matcher(line);
	return mCurrentMatcher.matches();
	}

	private boolean lineStartswithPattern(String line, Pattern p) {
	mCurrentMatcher = p.matcher(line);
	return mCurrentMatcher.find();
	}

	/** Send recorded test results to all listeners. */
	private void reportToListeners() {
	for (ITestInvocationListener listener : mListeners) {
	listener.testRunStarted(mRunName, mTotalTestCount);

	for (Entry<TestDescription, String> test : mTestResultCache.entrySet()) {
	listener.testStarted(test.getKey());
	if (SKIPPED_ENTRY.equals(test.getValue())) {
	listener.testIgnored(test.getKey());
	} else if (test.getValue() != null) {
	listener.testFailed(test.getKey(), test.getValue());
	}
	listener.testEnded(test.getKey(), new HashMap<String, Metric>());
	}
	listener.testRunEnded(mTotalElapsedTime, new HashMap<String, Metric>());
	}
	}

	/** Record a non-failure test case. */
	private void reportNonFailureTestResult() throws PythonUnitTestParseException {
	TestDescription testId = new TestDescription(mCurrentTestClass, mCurrentTestName);

	if (shouldSkipCurrentTest()) {
	// Force to skip any test not listed in include filters, or listed in exclude filters.
	mTestResultCache.put(testId, SKIPPED_ENTRY);
	} else if (PATTERN_TEST_SUCCESS.matcher(mCurrentTestStatus).matches()) {
	mTestResultCache.put(testId, null);
	} else if (PATTERN_TEST_SKIPPED.matcher(mCurrentTestStatus).matches()) {
	mTestResultCache.put(testId, SKIPPED_ENTRY);
	} else if (PATTERN_TEST_UNEXPECTED_SUCCESS.matcher(mCurrentTestStatus).matches()) {
	mTestResultCache.put(testId, "Test unexpected succeeded");
	} else if (PATTERN_TEST_FAILURE.matcher(mCurrentTestStatus).matches()) {
	// do nothing for now, report only after traceback is collected
	} else {
	throw new PythonUnitTestParseException("Unrecognized test status");
	}
	}

	/** Record a failed test case and its traceback message. */
	private void reportFailureTestResult() {
	TestDescription testId = new TestDescription(mCurrentTestClass, mCurrentTestName);
	if (shouldSkipCurrentTest()) {
	mTestResultCache.put(testId, SKIPPED_ENTRY);
	} else {
	mTestResultCache.put(testId, mCurrentTraceback.toString());
	}
	}

	/**
	* Check if current test should be skipped.
	*
	* @return true if the test should be skipped.
	*/
	private boolean shouldSkipCurrentTest() {
	// Force to skip any test not listed in include filters, or listed in exclude filters.
	// exclude filters have highest priority.
	if (mExcludeFilters.contains(mCurrentTestClass + "#" + mCurrentTestName)
	\|\| mExcludeFilters.contains(mCurrentTestClass)) {
	return true;
	}
	if (!mIncludeFilters.isEmpty()) {
	if (mIncludeFilters.contains(mCurrentTestClass + "#" + mCurrentTestName)
	\|\| mIncludeFilters.contains(mCurrentTestClass)) {
	return false;
	}
	for (String filter : mIncludeFilters) {
	if ((mCurrentTestClass + "#" + mCurrentTestName).matches(filter)) {
	return false;
	}
	}
	return true;
	}
	return false;
	}

	@Override
	public boolean isCancelled() {
	return false;
	}
	}