Stop ignoring untriaged ICU CTS test failures
This stops filtering tests that are known to fail but which have
not yet been investigated out of the set of tests to run. The
failing tests are still listed in the icu-knownn-failures.txt
file but are marked as being expected to succeed so they are
still run. That gives us a convenient list of the failures and
makes it easier to run just the failing tests.
It also updates the list of tests for old-cts to run to include
the failing tests.
Bug: 22023363
Change-Id: I2eda143182f65b3c144805db220c9587d552f866
diff --git a/tests/tests/icu/resources/android/icu/cts/expectations/icu-known-failures.txt b/tests/tests/icu/resources/android/icu/cts/expectations/icu-known-failures.txt
index 331f7f9..e667c08 100644
--- a/tests/tests/icu/resources/android/icu/cts/expectations/icu-known-failures.txt
+++ b/tests/tests/icu/resources/android/icu/cts/expectations/icu-known-failures.txt
@@ -34,6 +34,10 @@
description: "A completely empty test case",
name: "android.icu.dev.test.duration.PeriodBuilderTest"
},
+// The following are known to be broken but the reason is not yet known. They are expected to
+// succeed and so will be run and will fail but that is intentional so that they do not get
+// forgotten and are investigated to determine whether it is correct behavior on Android or a bug
+// that needs fixing.
{
description: "Broken and not yet triaged",
names: [
@@ -61,7 +65,8 @@
"android.icu.dev.test.timezone.TimeZoneBoundaryTest#TestStepwise",
"android.icu.dev.test.timezone.TimeZoneTest#TestObservesDaylightTime",
"android.icu.dev.test.util.CurrencyTest#TestGetKeywordValues"
- ]
+ ],
+ result: "SUCCESS"
},
{
description: "Broken due to warnings, not yet triaged",
@@ -72,6 +77,7 @@
"android.icu.dev.test.collator.CollationMiscTest#TestUCARules",
"android.icu.dev.test.collator.CollationTest#TestDataDriven",
"android.icu.dev.test.collator.G7CollationTest#TestG7Data"
- ]
+ ],
+ result: "SUCCESS"
}
]
diff --git a/tests/tests/icu/test-list.txt b/tests/tests/icu/test-list.txt
index ac200ec..93b9f89 100644
--- a/tests/tests/icu/test-list.txt
+++ b/tests/tests/icu/test-list.txt
@@ -3,6 +3,7 @@
android.icu.dev.test.bidi.TestBidi#testBidi
android.icu.dev.test.bidi.TestCharFromDirProp#testCharFromDirProp
android.icu.dev.test.bidi.TestClassOverride#testClassOverride
+android.icu.dev.test.bidi.TestCompatibility#testCompatibility
android.icu.dev.test.bidi.TestContext#testContext
android.icu.dev.test.bidi.TestFailureRecovery#testFailureRecovery
android.icu.dev.test.bidi.TestInverse#testInverse
@@ -80,7 +81,9 @@
android.icu.dev.test.calendar.CalendarRegression#TestYearJump3279
android.icu.dev.test.calendar.ChineseTest#Test6510
android.icu.dev.test.calendar.ChineseTest#TestAdd
+android.icu.dev.test.calendar.ChineseTest#TestCoverage
android.icu.dev.test.calendar.ChineseTest#TestFormat
+android.icu.dev.test.calendar.ChineseTest#TestInitWithCurrentTime
android.icu.dev.test.calendar.ChineseTest#TestJulianDayMapping
android.icu.dev.test.calendar.ChineseTest#TestLimits
android.icu.dev.test.calendar.ChineseTest#TestMapping
@@ -146,6 +149,7 @@
android.icu.dev.test.calendar.HolidayTest#TestEaster
android.icu.dev.test.calendar.HolidayTest#TestIsOn
android.icu.dev.test.calendar.IBMCalendarTest#TestAddAcrossZoneTransition
+android.icu.dev.test.calendar.IBMCalendarTest#TestAddRollEra0AndEraBounds
android.icu.dev.test.calendar.IBMCalendarTest#TestAmbiguousWallTimeAPIs
android.icu.dev.test.calendar.IBMCalendarTest#TestBuddhist
android.icu.dev.test.calendar.IBMCalendarTest#TestBuddhistCoverage
@@ -263,6 +267,7 @@
android.icu.dev.test.collator.CollationAPITest#TestSubClass
android.icu.dev.test.collator.CollationAPITest#TestVariableTopSetting
android.icu.dev.test.collator.CollationChineseTest#TestPinYin
+android.icu.dev.test.collator.CollationCreationMethodTest#TestRuleVsLocaleCreationMonkey
android.icu.dev.test.collator.CollationCurrencyTest#TestCurrency
android.icu.dev.test.collator.CollationDummyTest#TestExtra
android.icu.dev.test.collator.CollationDummyTest#TestIdentical
@@ -336,6 +341,8 @@
android.icu.dev.test.collator.CollationMiscTest#TestHungarianTailoring
android.icu.dev.test.collator.CollationMiscTest#TestIdenticalCompare
android.icu.dev.test.collator.CollationMiscTest#TestImplicitTailoring
+android.icu.dev.test.collator.CollationMiscTest#TestImport
+android.icu.dev.test.collator.CollationMiscTest#TestImportWithType
android.icu.dev.test.collator.CollationMiscTest#TestIncompleteCnt
android.icu.dev.test.collator.CollationMiscTest#TestIncrementalNormalize
android.icu.dev.test.collator.CollationMiscTest#TestInvalidListsAndRanges
@@ -377,6 +384,7 @@
android.icu.dev.test.collator.CollationMiscTest#TestSurrogates
android.icu.dev.test.collator.CollationMiscTest#TestTibetanConformance
android.icu.dev.test.collator.CollationMiscTest#TestUCAPrecontext
+android.icu.dev.test.collator.CollationMiscTest#TestUCARules
android.icu.dev.test.collator.CollationMiscTest#TestUnknownCollationKeyword
android.icu.dev.test.collator.CollationMiscTest#TestUpperFirstQuaternary
android.icu.dev.test.collator.CollationMiscTest#TestVariableTop
@@ -434,6 +442,7 @@
android.icu.dev.test.collator.CollationSpanishTest#TestPrimary
android.icu.dev.test.collator.CollationSpanishTest#TestTertiary
android.icu.dev.test.collator.CollationTest#TestCollationWeights
+android.icu.dev.test.collator.CollationTest#TestDataDriven
android.icu.dev.test.collator.CollationTest#TestFCD
android.icu.dev.test.collator.CollationTest#TestImplicits
android.icu.dev.test.collator.CollationTest#TestMinMax
@@ -453,6 +462,7 @@
android.icu.dev.test.collator.G7CollationTest#TestDemo2
android.icu.dev.test.collator.G7CollationTest#TestDemo3
android.icu.dev.test.collator.G7CollationTest#TestDemo4
+android.icu.dev.test.collator.G7CollationTest#TestG7Data
android.icu.dev.test.collator.LotusCollationKoreanTest#TestTertiary
android.icu.dev.test.collator.UCAConformanceTest#TestRulesNonIgnorable
android.icu.dev.test.collator.UCAConformanceTest#TestRulesShifted
@@ -476,6 +486,7 @@
android.icu.dev.test.duration.DataReadWriteTest#testStringTable
android.icu.dev.test.duration.ICUDurationTest#TestBadLocaleError
android.icu.dev.test.duration.ICUDurationTest#TestBadObjectError
+android.icu.dev.test.duration.ICUDurationTest#TestBasics
android.icu.dev.test.duration.ICUDurationTest#TestDurationFormat
android.icu.dev.test.duration.ICUDurationTest#TestFromNowTo
android.icu.dev.test.duration.ICUDurationTest#TestResourceWithCalendar
@@ -613,6 +624,8 @@
android.icu.dev.test.format.CompactDecimalFormatTest#TestSkLong
android.icu.dev.test.format.CompactDecimalFormatTest#TestSwahiliShort
android.icu.dev.test.format.CompactDecimalFormatTest#TestSwahiliShortNegative
+android.icu.dev.test.format.DataDrivenFormatTest#TestConsistentPivot
+android.icu.dev.test.format.DataDrivenFormatTest#TestDateFormatBasic
android.icu.dev.test.format.DataDrivenFormatTest#TestFormatRelative
android.icu.dev.test.format.DataDrivenFormatTest#TestHebrew
android.icu.dev.test.format.DataDrivenFormatTest#TestMoreDateParse
@@ -637,6 +650,7 @@
android.icu.dev.test.format.DateFormatRegressionTest#Test4061287
android.icu.dev.test.format.DateFormatRegressionTest#Test4065240
android.icu.dev.test.format.DateFormatRegressionTest#Test4071441
+android.icu.dev.test.format.DateFormatRegressionTest#Test4073003
android.icu.dev.test.format.DateFormatRegressionTest#Test4089106
android.icu.dev.test.format.DateFormatRegressionTest#Test4100302
android.icu.dev.test.format.DateFormatRegressionTest#Test4101483
@@ -648,7 +662,9 @@
android.icu.dev.test.format.DateFormatRegressionTest#Test4108407
android.icu.dev.test.format.DateFormatRegressionTest#Test4134203
android.icu.dev.test.format.DateFormatRegressionTest#Test4151631
+android.icu.dev.test.format.DateFormatRegressionTest#Test4151706
android.icu.dev.test.format.DateFormatRegressionTest#Test4162071
+android.icu.dev.test.format.DateFormatRegressionTest#Test4182066
android.icu.dev.test.format.DateFormatRegressionTest#Test4210209
android.icu.dev.test.format.DateFormatRegressionTest#Test5006GetShortMonths
android.icu.dev.test.format.DateFormatRegressionTest#Test714
@@ -666,6 +682,7 @@
android.icu.dev.test.format.DateFormatRoundTripTest#TestDateFormatRoundTrip
android.icu.dev.test.format.DateFormatTest#Test10632
android.icu.dev.test.format.DateFormatTest#Test6880
+android.icu.dev.test.format.DateFormatTest#TestBadInput135
android.icu.dev.test.format.DateFormatTest#TestBadInput135a
android.icu.dev.test.format.DateFormatTest#TestCalendarType
android.icu.dev.test.format.DateFormatTest#TestChineseDateFormatLocalizedPatternChars
@@ -679,6 +696,7 @@
android.icu.dev.test.format.DateFormatTest#TestDateFormatZone061
android.icu.dev.test.format.DateFormatTest#TestDateFormatZone146
android.icu.dev.test.format.DateFormatTest#TestDayOfYearPattern195
+android.icu.dev.test.format.DateFormatTest#TestDotAndAtLeniency
android.icu.dev.test.format.DateFormatTest#TestEquals
android.icu.dev.test.format.DateFormatTest#TestEras
android.icu.dev.test.format.DateFormatTest#TestErrorChecking
@@ -689,6 +707,7 @@
android.icu.dev.test.format.DateFormatTest#TestFormatToCharacterIterator
android.icu.dev.test.format.DateFormatTest#TestFormatToCharacterIteratorCoverage
android.icu.dev.test.format.DateFormatTest#TestGeneral
+android.icu.dev.test.format.DateFormatTest#TestGenericTime
android.icu.dev.test.format.DateFormatTest#TestGenericTimeZoneOrder
android.icu.dev.test.format.DateFormatTest#TestGetPatternInstance
android.icu.dev.test.format.DateFormatTest#TestGMTParsing
@@ -707,7 +726,9 @@
android.icu.dev.test.format.DateFormatTest#TestParseLeniencyAPIs
android.icu.dev.test.format.DateFormatTest#TestParseMultiPatternMatch
android.icu.dev.test.format.DateFormatTest#TestParsePosition
+android.icu.dev.test.format.DateFormatTest#TestPartialParse994
android.icu.dev.test.format.DateFormatTest#TestPatterns
+android.icu.dev.test.format.DateFormatTest#TestQuarters
android.icu.dev.test.format.DateFormatTest#TestQuotePattern161
android.icu.dev.test.format.DateFormatTest#TestRelativeDateFormat
android.icu.dev.test.format.DateFormatTest#TestRoundtripWithCalendar
@@ -715,12 +736,15 @@
android.icu.dev.test.format.DateFormatTest#TestRunTogetherPattern985
android.icu.dev.test.format.DateFormatTest#TestShortDays
android.icu.dev.test.format.DateFormatTest#TestSimpleDateFormatConstructor_String_String_ULocale
+android.icu.dev.test.format.DateFormatTest#TestSpaceParsing
android.icu.dev.test.format.DateFormatTest#TestStandAloneDays
android.icu.dev.test.format.DateFormatTest#TestStandAloneMonths
android.icu.dev.test.format.DateFormatTest#TestTimeZoneDisplayName
android.icu.dev.test.format.DateFormatTest#TestTwoDigitWOY
+android.icu.dev.test.format.DateFormatTest#TestTwoDigitYear
android.icu.dev.test.format.DateFormatTest#TestTwoDigitYearDSTParse
android.icu.dev.test.format.DateFormatTest#TestWallyWedel
+android.icu.dev.test.format.DateFormatTest#TestWhiteSpaceParsing
android.icu.dev.test.format.DateIntervalFormatTest#TestClone
android.icu.dev.test.format.DateIntervalFormatTest#TestCloseAsThawed
android.icu.dev.test.format.DateIntervalFormatTest#TestConstructor
@@ -941,6 +965,7 @@
android.icu.dev.test.format.NumberFormatTest#TestCurrFmtNegSameAsPositive
android.icu.dev.test.format.NumberFormatTest#TestCustomCurrencySignAndSeparator
android.icu.dev.test.format.NumberFormatTest#TestDataDrivenICU
+android.icu.dev.test.format.NumberFormatTest#TestDataDrivenJDK
android.icu.dev.test.format.NumberFormatTest#TestDecimalFormatCurrencyParse
android.icu.dev.test.format.NumberFormatTest#TestExplicitParents
android.icu.dev.test.format.NumberFormatTest#TestExponent
@@ -964,6 +989,7 @@
android.icu.dev.test.format.NumberFormatTest#TestMiscCurrencyParsing
android.icu.dev.test.format.NumberFormatTest#TestMultiCurrencySign
android.icu.dev.test.format.NumberFormatTest#TestNegZeroRounding
+android.icu.dev.test.format.NumberFormatTest#TestNonpositiveMultiplier
android.icu.dev.test.format.NumberFormatTest#TestNumberFormatFactory
android.icu.dev.test.format.NumberFormatTest#TestNumberFormatTestTupleToString
android.icu.dev.test.format.NumberFormatTest#TestNumberingSystems
@@ -1114,6 +1140,7 @@
android.icu.dev.test.format.PluralRulesTest#TestLocales
android.icu.dev.test.format.PluralRulesTest#testOperands
android.icu.dev.test.format.PluralRulesTest#TestOrdinal
+android.icu.dev.test.format.PluralRulesTest#testOverUnderflow
android.icu.dev.test.format.PluralRulesTest#TestParseDescription
android.icu.dev.test.format.PluralRulesTest#testParseEmpty
android.icu.dev.test.format.PluralRulesTest#testParsing
@@ -1684,6 +1711,7 @@
android.icu.dev.test.timescale.TimeScaleMonkeyTest#TestRoundTrip
android.icu.dev.test.timezone.TimeZoneBoundaryTest#TestBoundaries
android.icu.dev.test.timezone.TimeZoneBoundaryTest#TestNewRules
+android.icu.dev.test.timezone.TimeZoneBoundaryTest#TestStepwise
android.icu.dev.test.timezone.TimeZoneOffsetLocalTest#TestGetOffsetAroundTransition
android.icu.dev.test.timezone.TimeZoneRegression#Test4052967
android.icu.dev.test.timezone.TimeZoneRegression#Test4073209
@@ -1750,6 +1778,7 @@
android.icu.dev.test.timezone.TimeZoneTest#TestGetWindowsID
android.icu.dev.test.timezone.TimeZoneTest#TestHashCode
android.icu.dev.test.timezone.TimeZoneTest#TestMark
+android.icu.dev.test.timezone.TimeZoneTest#TestObservesDaylightTime
android.icu.dev.test.timezone.TimeZoneTest#TestOddTimeZoneNames
android.icu.dev.test.timezone.TimeZoneTest#TestPRTOffset
android.icu.dev.test.timezone.TimeZoneTest#TestRuleAPI
@@ -1955,6 +1984,7 @@
android.icu.dev.test.util.CurrencyTest#TestDeprecatedCurrencyFormat
android.icu.dev.test.util.CurrencyTest#TestGetAvailableCurrencies
android.icu.dev.test.util.CurrencyTest#TestGetDisplayName
+android.icu.dev.test.util.CurrencyTest#TestGetKeywordValues
android.icu.dev.test.util.CurrencyTest#TestGetNumericCode
android.icu.dev.test.util.CurrencyTest#TestIsAvailable
android.icu.dev.test.util.CurrencyTest#TestNames
