Add support for skipped tests.
am: 849d24e8a4
Change-Id: Ie55fff4940a70430bb10a56a9e76598892215a54
diff --git a/Isolate.cpp b/Isolate.cpp
index 58283fd..227a875 100644
--- a/Isolate.cpp
+++ b/Isolate.cpp
@@ -267,7 +267,9 @@
test->AppendOutput(output);
test->set_result(TEST_FAIL);
} else {
- test->set_result(TEST_PASS);
+ // Set the result based on the output, since skipped tests and
+ // passing tests have the same exit status.
+ test->SetResultFromOutput();
}
}
} else if (test->result() == TEST_TIMEOUT) {
@@ -308,6 +310,9 @@
case TEST_XFAIL:
total_xfail_tests_++;
break;
+ case TEST_SKIPPED:
+ total_skipped_tests_++;
+ break;
case TEST_NONE:
LOG(FATAL) << "Test result is TEST_NONE, this should not be possible.";
}
@@ -381,6 +386,7 @@
total_xfail_tests_ = 0;
total_timeout_tests_ = 0;
total_slow_tests_ = 0;
+ total_skipped_tests_ = 0;
running_by_test_index_.clear();
@@ -415,7 +421,11 @@
void Isolate::PrintResults(size_t total, const ResultsType& results, std::string* footer) {
ColoredPrintf(results.color, results.prefix);
- printf(" %s %s, listed below:\n", PluralizeString(total, " test").c_str(), results.list_desc);
+ if (results.list_desc != nullptr) {
+ printf(" %s %s, listed below:\n", PluralizeString(total, " test").c_str(), results.list_desc);
+ } else {
+ printf(" %s, listed below:\n", PluralizeString(total, " test").c_str());
+ }
for (const auto& entry : finished_) {
const Test* test = entry.second.get();
if (results.match_func(*test)) {
@@ -427,6 +437,11 @@
printf("\n");
}
}
+
+ if (results.title == nullptr) {
+ return;
+ }
+
if (total < 10) {
*footer += ' ';
}
@@ -477,6 +492,15 @@
},
};
+Isolate::ResultsType Isolate::SkippedResults = {
+ .color = COLOR_GREEN,
+ .prefix = "[ SKIPPED ]",
+ .list_desc = nullptr,
+ .title = nullptr,
+ .match_func = [](const Test& test) { return test.result() == TEST_SKIPPED; },
+ .print_func = nullptr,
+};
+
void Isolate::PrintFooter(uint64_t elapsed_time_ns) {
ColoredPrintf(COLOR_GREEN, "[==========]");
printf(" %s from %s ran. (%" PRId64 " ms total)\n",
@@ -491,6 +515,12 @@
printf("\n");
std::string footer;
+
+ // Tests that were skipped.
+ if (total_skipped_tests_ != 0) {
+ PrintResults(total_skipped_tests_, SkippedResults, &footer);
+ }
+
// Tests that ran slow.
if (total_slow_tests_ != 0) {
PrintResults(total_slow_tests_, SlowResults, &footer);
@@ -699,7 +729,7 @@
WriteXmlResults(time_ns, start_time);
}
- if (total_pass_tests_ + total_xfail_tests_ != tests_.size()) {
+ if (total_pass_tests_ + total_skipped_tests_ + total_xfail_tests_ != tests_.size()) {
exit_code = 1;
}
}
diff --git a/Isolate.h b/Isolate.h
index fc5166a..3399ec5 100644
--- a/Isolate.h
+++ b/Isolate.h
@@ -90,6 +90,7 @@
size_t total_xfail_tests_;
size_t total_timeout_tests_;
size_t total_slow_tests_;
+ size_t total_skipped_tests_;
size_t cur_test_index_ = 0;
uint64_t slow_threshold_ns_;
@@ -110,6 +111,7 @@
static ResultsType XpassFailResults;
static ResultsType FailResults;
static ResultsType TimeoutResults;
+ static ResultsType SkippedResults;
};
} // namespace gtest_extras
diff --git a/Test.cpp b/Test.cpp
index 5fbaa67..171366b 100644
--- a/Test.cpp
+++ b/Test.cpp
@@ -59,6 +59,9 @@
case TEST_XFAIL:
ColoredPrintf(COLOR_GREEN, "[ OK ]");
break;
+ case TEST_SKIPPED:
+ ColoredPrintf(COLOR_GREEN, "[ SKIPPED ]");
+ break;
default:
ColoredPrintf(COLOR_RED, "[ FAILED ]");
break;
@@ -89,6 +92,9 @@
case TEST_TIMEOUT:
ColoredPrintf(COLOR_RED, "[ TIMEOUT ]");
break;
+ case TEST_SKIPPED:
+ ColoredPrintf(COLOR_GREEN, "[ SKIPPED ]");
+ break;
case TEST_NONE:
LOG(FATAL) << "Test result is TEST_NONE, this should not be possible.";
}
@@ -138,5 +144,32 @@
}
}
+void Test::SetResultFromOutput() {
+ result_ = TEST_PASS;
+
+ // Need to parse the output to determine if this test was skipped.
+ // Format of a skipped test:
+ // <filename>:(<line_number>) Failure in test <testname>
+ // Skipped
+ // <Skip Message>
+ size_t line_end = output_.find('\n');
+ if (line_end == std::string::npos) {
+ return;
+ }
+ std::string second_line(output_.substr(line_end, 9));
+ if (output_.substr(line_end, 9) != "\nSkipped\n") {
+ return;
+ }
+ size_t failure_index = output_.find(" Failure in test ");
+ if (failure_index == std::string::npos || failure_index >= line_end) {
+ return;
+ }
+
+ // Only leave the output from the skip message.
+ output_ = output_.substr(line_end + 9);
+
+ result_ = TEST_SKIPPED;
+}
+
} // namespace gtest_extras
} // namespace android
diff --git a/Test.h b/Test.h
index 95d5038..b33c9f0 100644
--- a/Test.h
+++ b/Test.h
@@ -31,6 +31,7 @@
TEST_FAIL,
TEST_XFAIL,
TEST_TIMEOUT,
+ TEST_SKIPPED,
};
class Test {
@@ -49,6 +50,8 @@
void CloseFd();
+ void SetResultFromOutput();
+
void AppendOutput(std::string& output) { output_ += output; }
void AppendOutput(const char* output) { output_ += output; }
diff --git a/tests/SystemTests.cpp b/tests/SystemTests.cpp
index b2c5920..3cfbcef 100644
--- a/tests/SystemTests.cpp
+++ b/tests/SystemTests.cpp
@@ -234,6 +234,115 @@
std::vector<const char*>{"--gtest_format", "--gtest_color=yes"}));
}
+TEST_F(SystemTests, verify_skip) {
+ std::string expected =
+ "Note: Google Test filter = *.DISABLED_skip_no_message\n"
+ "[==========] Running 1 test from 1 test case (20 jobs).\n"
+ "[ SKIPPED ] SystemTests.DISABLED_skip_no_message (XX ms)\n"
+ "[==========] 1 test from 1 test case ran. (XX ms total)\n"
+ "[ PASS ] 0 tests.\n"
+ "[ SKIPPED ] 1 test, listed below:\n"
+ "[ SKIPPED ] SystemTests.DISABLED_skip_no_message\n";
+ ASSERT_NO_FATAL_FAILURE(Verify("*.DISABLED_skip_no_message", expected, 0));
+}
+
+TEST_F(SystemTests, verify_skip_with_message) {
+ std::string expected =
+ "Note: Google Test filter = *.DISABLED_skip_with_message\n"
+ "[==========] Running 1 test from 1 test case (20 jobs).\n"
+ "[ SKIPPED ] SystemTests.DISABLED_skip_with_message (XX ms)\n"
+ "This is a skip message\n"
+ "[==========] 1 test from 1 test case ran. (XX ms total)\n"
+ "[ PASS ] 0 tests.\n"
+ "[ SKIPPED ] 1 test, listed below:\n"
+ "[ SKIPPED ] SystemTests.DISABLED_skip_with_message\n";
+ ASSERT_NO_FATAL_FAILURE(Verify("*.DISABLED_skip_with_message", expected, 0));
+}
+
+TEST_F(SystemTests, verify_skip_no_print_time) {
+ std::string expected =
+ "Note: Google Test filter = *.DISABLED_skip_no_message\n"
+ "[==========] Running 1 test from 1 test case (20 jobs).\n"
+ "[ SKIPPED ] SystemTests.DISABLED_skip_no_message\n"
+ "[==========] 1 test from 1 test case ran. (XX ms total)\n"
+ "[ PASS ] 0 tests.\n"
+ "[ SKIPPED ] 1 test, listed below:\n"
+ "[ SKIPPED ] SystemTests.DISABLED_skip_no_message\n";
+ ASSERT_NO_FATAL_FAILURE(Verify("*.DISABLED_skip_no_message", expected, 0,
+ std::vector<const char*>{"--gtest_print_time=0"}));
+}
+
+TEST_F(SystemTests, verify_skip_color) {
+ std::string expected =
+ "\x1B[0;33mNote: Google Test filter = *.DISABLED_skip_no_message\x1B[m\n"
+ "\x1B[0;32m[==========]\x1B[m Running 1 test from 1 test case (20 jobs).\n"
+ "\x1B[0;32m[ SKIPPED ]\x1B[m SystemTests.DISABLED_skip_no_message (XX ms)\n"
+ "\x1B[0;32m[==========]\x1B[m 1 test from 1 test case ran. (XX ms total)\n"
+ "\x1B[0;32m[ PASS ]\x1B[m 0 tests.\n"
+ "\x1B[0;32m[ SKIPPED ]\x1B[m 1 test, listed below:\n"
+ "\x1B[0;32m[ SKIPPED ]\x1B[m SystemTests.DISABLED_skip_no_message\n";
+ ASSERT_NO_FATAL_FAILURE(Verify("*.DISABLED_skip_no_message", expected, 0,
+ std::vector<const char*>{"--gtest_color=yes"}));
+}
+
+TEST_F(SystemTests, verify_skip_gtest_format) {
+ std::string expected =
+ "Note: Google Test filter = *.DISABLED_skip_no_message\n"
+ "[==========] Running 1 test from 1 test case (20 jobs).\n"
+ "[ RUN ] SystemTests.DISABLED_skip_no_message\n"
+ "[ SKIPPED ] SystemTests.DISABLED_skip_no_message (XX ms)\n"
+ "[==========] 1 test from 1 test case ran. (XX ms total)\n"
+ "[ PASS ] 0 tests.\n"
+ "[ SKIPPED ] 1 test, listed below:\n"
+ "[ SKIPPED ] SystemTests.DISABLED_skip_no_message\n";
+ ASSERT_NO_FATAL_FAILURE(Verify("*.DISABLED_skip_no_message", expected, 0,
+ std::vector<const char*>{"--gtest_format"}));
+}
+
+TEST_F(SystemTests, verify_skip_gtest_format_with_message) {
+ std::string expected =
+ "Note: Google Test filter = *.DISABLED_skip_with_message\n"
+ "[==========] Running 1 test from 1 test case (20 jobs).\n"
+ "[ RUN ] SystemTests.DISABLED_skip_with_message\n"
+ "This is a skip message\n"
+ "[ SKIPPED ] SystemTests.DISABLED_skip_with_message (XX ms)\n"
+ "[==========] 1 test from 1 test case ran. (XX ms total)\n"
+ "[ PASS ] 0 tests.\n"
+ "[ SKIPPED ] 1 test, listed below:\n"
+ "[ SKIPPED ] SystemTests.DISABLED_skip_with_message\n";
+ ASSERT_NO_FATAL_FAILURE(Verify("*.DISABLED_skip_with_message", expected, 0,
+ std::vector<const char*>{"--gtest_format"}));
+}
+
+TEST_F(SystemTests, verify_skip_gtest_format_no_print_time) {
+ std::string expected =
+ "Note: Google Test filter = *.DISABLED_skip_no_message\n"
+ "[==========] Running 1 test from 1 test case (20 jobs).\n"
+ "[ RUN ] SystemTests.DISABLED_skip_no_message\n"
+ "[ SKIPPED ] SystemTests.DISABLED_skip_no_message\n"
+ "[==========] 1 test from 1 test case ran. (XX ms total)\n"
+ "[ PASS ] 0 tests.\n"
+ "[ SKIPPED ] 1 test, listed below:\n"
+ "[ SKIPPED ] SystemTests.DISABLED_skip_no_message\n";
+ ASSERT_NO_FATAL_FAILURE(
+ Verify("*.DISABLED_skip_no_message", expected, 0,
+ std::vector<const char*>{"--gtest_format", "--gtest_print_time=0"}));
+}
+
+TEST_F(SystemTests, verify_skip_gtest_format_color) {
+ std::string expected =
+ "\x1B[0;33mNote: Google Test filter = *.DISABLED_skip_no_message\x1B[m\n"
+ "\x1B[0;32m[==========]\x1B[m Running 1 test from 1 test case (20 jobs).\n"
+ "\x1B[0;32m[ RUN ]\x1B[m SystemTests.DISABLED_skip_no_message\n"
+ "\x1B[0;32m[ SKIPPED ]\x1B[m SystemTests.DISABLED_skip_no_message (XX ms)\n"
+ "\x1B[0;32m[==========]\x1B[m 1 test from 1 test case ran. (XX ms total)\n"
+ "\x1B[0;32m[ PASS ]\x1B[m 0 tests.\n"
+ "\x1B[0;32m[ SKIPPED ]\x1B[m 1 test, listed below:\n"
+ "\x1B[0;32m[ SKIPPED ]\x1B[m SystemTests.DISABLED_skip_no_message\n";
+ ASSERT_NO_FATAL_FAILURE(Verify("*.DISABLED_skip_no_message", expected, 0,
+ std::vector<const char*>{"--gtest_format", "--gtest_color=yes"}));
+}
+
TEST_F(SystemTests, verify_xfail_fail_expect_to_fail) {
std::string expected =
"Note: Google Test filter = *.xfail_fail\n"
@@ -588,7 +697,7 @@
ASSERT_NE(0, exitcode_);
}
-TEST_F(SystemTests, verify_error_order) {
+TEST_F(SystemTests, verify_title_order) {
RunTest("*.DISABLED_all_*",
std::vector<const char*>{"--slow_threshold_ms=2000", "--deadline_threshold_ms=4000"});
// Verify the order of the output messages.
@@ -605,6 +714,9 @@
ASSERT_FALSE(footer.empty()) << "Test output:\n" << raw_output_;
ASSERT_EQ(
"[ PASS ] 4 tests.\n"
+ "[ SKIPPED ] 2 tests, listed below:\n"
+ "[ SKIPPED ] SystemTests.DISABLED_all_skip_1\n"
+ "[ SKIPPED ] SystemTests.DISABLED_all_skip_2\n"
"[ SLOW ] 2 tests slow, listed below:\n"
"[ SLOW ] SystemTests.DISABLED_all_slow_1 (XX ms, exceeded 2000 ms)\n"
"[ SLOW ] SystemTests.DISABLED_all_slow_2 (XX ms, exceeded 2000 ms)\n"
@@ -1156,6 +1268,14 @@
TEST_F(SystemTests, DISABLED_all_pass_2) {}
+TEST_F(SystemTests, DISABLED_all_skip_1) {
+ GTEST_SKIP();
+}
+
+TEST_F(SystemTests, DISABLED_all_skip_2) {
+ GTEST_SKIP() << "Skip message present";
+}
+
TEST_F(SystemTests, DISABLED_all_slow_1) {
sleep(3);
}
@@ -1192,6 +1312,14 @@
sleep(4);
}
+TEST_F(SystemTests, DISABLED_skip_no_message) {
+ GTEST_SKIP();
+}
+
+TEST_F(SystemTests, DISABLED_skip_with_message) {
+ GTEST_SKIP() << "This is a skip message";
+}
+
class DISABLED_SystemTestsXfail : public ::testing::Test {};
TEST_F(DISABLED_SystemTestsXfail, xfail_fail) {
