Create functional test results for FaaS
Allows us to have the results in ATH for triaging and tracking
Bug: 232515602
Test: Run CL and test using these changes through ABTD to check we get functional results for FaaS tests
Change-Id: I91904828cf9065d4f7544f342311f7e2c5dce21f
diff --git a/libraries/flicker/src/com/android/server/wm/flicker/FlickerBlockJUnit4ClassRunner.kt b/libraries/flicker/src/com/android/server/wm/flicker/FlickerBlockJUnit4ClassRunner.kt
index 594027d..88fe7fe 100644
--- a/libraries/flicker/src/com/android/server/wm/flicker/FlickerBlockJUnit4ClassRunner.kt
+++ b/libraries/flicker/src/com/android/server/wm/flicker/FlickerBlockJUnit4ClassRunner.kt
@@ -19,21 +19,31 @@
import android.platform.test.util.TestFilter
import android.util.Log
import androidx.test.platform.app.InstrumentationRegistry
+import com.android.server.wm.flicker.annotation.FlickerServiceCompatible
import com.android.server.wm.flicker.dsl.FlickerBuilder
+import com.android.server.wm.flicker.helpers.isShellTransitionsEnabled
+import com.android.server.wm.flicker.service.FlickerFrameworkMethod
+import com.android.server.wm.flicker.service.FlickerTestCase
+import com.android.server.wm.flicker.service.assertors.AssertionResult
+import com.android.server.wm.flicker.service.config.AssertionInvocationGroup
+import java.lang.reflect.Modifier
+import org.junit.Test
import org.junit.internal.runners.statements.RunAfters
import org.junit.runner.notification.RunNotifier
+import org.junit.runners.Parameterized
+import org.junit.runners.model.FrameworkField
import org.junit.runners.model.FrameworkMethod
import org.junit.runners.model.Statement
+import org.junit.runners.model.TestClass
import org.junit.runners.parameterized.BlockJUnit4ClassRunnerWithParameters
import org.junit.runners.parameterized.TestWithParameters
-import java.lang.reflect.Modifier
/**
* Implements the JUnit 4 standard test case class model, parsing from a flicker DSL.
*
* Supports both assertions in {@link org.junit.Test} and assertions defined in the DSL
*
- * When using this runnr the default `atest class#method` command doesn't work.
+ * When using this runner the default `atest class#method` command doesn't work.
* Instead use: -- --test-arg \
* com.android.tradefed.testtype.AndroidJUnitTest:instrumentation-arg:filter-tests:=<TEST_NAME>
*
@@ -47,9 +57,105 @@
test: TestWithParameters,
private val parameters: Array<Any> = test.parameters.toTypedArray(),
private val flickerTestParameter: FlickerTestParameter? =
- parameters.filterIsInstance(FlickerTestParameter::class.java).firstOrNull()
+ parameters.filterIsInstance<FlickerTestParameter>().firstOrNull()
) : BlockJUnit4ClassRunnerWithParameters(test) {
- private var flickerMethod: FrameworkMethod? = null
+ private var flickerBuilderProviderMethod: FrameworkMethod? = null
+
+ private val arguments = InstrumentationRegistry.getArguments()
+ // null parses to false (so defaults to running all FaaS tests)
+ private val isBlockingTest = arguments.getString("faas:blocking").toBoolean()
+
+ /**
+ * {@inheritDoc}
+ */
+ override fun validateInstanceMethods(errors: MutableList<Throwable>) {
+ validateFlickerObject(errors)
+ super.validateInstanceMethods(errors)
+ }
+
+ /**
+ * Returns the methods that run tests.
+ * Is ran after validateInstanceMethods, so flickerBuilderProviderMethod should be set.
+ */
+ override fun computeTestMethods(): List<FrameworkMethod> {
+ return computeTests()
+ }
+
+ private fun computeTests(): MutableList<FrameworkMethod> {
+ val tests = mutableListOf<FrameworkMethod>()
+ tests.addAll(super.computeTestMethods())
+
+ // Don't compute when called from validateInstanceMethods since this will fail
+ // as the parameters will not be set. And AndroidLogOnlyBuilder is a non-executing runner
+ // used to run tests in dry-run mode so we don't want to execute in flicker transition in
+ // that case either.
+ val stackTrace = Thread.currentThread().stackTrace
+ if (stackTrace.none { it.methodName == "validateInstanceMethods" } &&
+ stackTrace.none {
+ it.className == "androidx.test.internal.runner.AndroidLogOnlyBuilder"
+ }
+ ) {
+ require(flickerTestParameter != null) {
+ "Can't computeTests with null flickerTestParameter"
+ }
+
+ val hasFlickerServiceCompatibleAnnotation = TestClass(super.createTest()::class.java)
+ .annotations.filterIsInstance<FlickerServiceCompatible>().firstOrNull() != null
+
+ if (hasFlickerServiceCompatibleAnnotation && isShellTransitionsEnabled) {
+ if (!flickerTestParameter.isInitialized) {
+ Log.v(FLICKER_TAG, "Flicker object is not yet initialized")
+ val test = super.createTest()
+ injectFlickerOnTestParams(test)
+ }
+
+ tests.addAll(computeFlickerServiceTests(isBlockingTest))
+ }
+ }
+
+ return tests
+ }
+
+ /**
+ * Runs the flicker transition to collect the traces and run FaaS on them to get the FaaS
+ * results and then create functional test results for each of them.
+ */
+ private fun computeFlickerServiceTests(onlyBlockingAssertions: Boolean): List<FrameworkMethod> {
+ require(flickerTestParameter != null) {
+ "Can't computeFlickerServiceTests with null flickerTestParameter"
+ }
+
+ val flickerTestMethods = mutableListOf<FlickerFrameworkMethod>()
+
+ val flicker = flickerTestParameter.flicker
+ if (flicker.result == null) {
+ flicker.execute()
+ }
+
+ // TODO: Figure out how we can report this without aggregation to have more precise and
+ // granular data on the actual failure rate.
+ for (aggregatedResult in aggregateFaasResults(flicker.faas.assertionResults)
+ .entries.iterator()) {
+ val testName = aggregatedResult.key
+ var results = aggregatedResult.value
+ if (onlyBlockingAssertions) {
+ results = results.filter { it.invocationGroup == AssertionInvocationGroup.BLOCKING }
+ }
+ if (results.isEmpty()) {
+ continue
+ }
+
+ val injectedTestCase = FlickerTestCase(results)
+ val mockedTestMethod = TestClass(injectedTestCase.javaClass)
+ .getAnnotatedMethods(Test::class.java).first()
+ val mockedFrameworkMethod = FlickerFrameworkMethod(
+ mockedTestMethod.method, injectedTestCase, testName
+ )
+ flickerTestMethods.add(mockedFrameworkMethod)
+ }
+
+ return flickerTestMethods
+ }
/**
* {@inheritDoc}
@@ -87,14 +193,18 @@
val method = methods.first()
if (Modifier.isStatic(method.method.modifiers)) {
- errors.add(Exception("Method ${method.name}() show not be static"))
+ errors.add(Exception("Method ${method.name}() should not be static"))
}
if (!Modifier.isPublic(method.method.modifiers)) {
errors.add(Exception("Method ${method.name}() should be public"))
}
if (method.returnType != FlickerBuilder::class.java) {
- errors.add(Exception("Method ${method.name}() should return a " +
- "${FlickerBuilder::class.java.simpleName} object"))
+ errors.add(
+ Exception(
+ "Method ${method.name}() should return a " +
+ "${FlickerBuilder::class.java.simpleName} object"
+ )
+ )
}
if (method.method.parameterTypes.isNotEmpty()) {
errors.add(Exception("Method ${method.name} should have no parameters"))
@@ -102,7 +212,7 @@
}
if (errors.isEmpty()) {
- flickerMethod = methods.first()
+ flickerBuilderProviderMethod = methods.first()
}
}
@@ -115,6 +225,8 @@
Log.v(FLICKER_TAG, "Flicker object is not yet initialized")
injectFlickerOnTestParams(test)
}
+
+ val flicker = flickerTestParameter?.flicker
return test
}
@@ -123,30 +235,36 @@
*/
private fun injectFlickerOnTestParams(test: Any) {
val flickerTestParameter = flickerTestParameter
- val flickerMethod = flickerMethod
- if (flickerTestParameter != null && flickerMethod != null) {
- val testName = test::class.java.simpleName
- Log.v(FLICKER_TAG, "Creating flicker object for $testName and adding it into " +
- "test parameter")
- val builder = flickerMethod.invokeExplosively(test) as FlickerBuilder
+ val flickerBuilderProviderMethod = flickerBuilderProviderMethod
+ if (flickerTestParameter != null && flickerBuilderProviderMethod != null) {
+ val testClass = test::class.java
+ val testName = testClass.simpleName
+ Log.v(
+ FLICKER_TAG,
+ "Creating flicker object for $testName and adding it into " +
+ "test parameter"
+ )
+
+ val isFlickerServiceCompatible = TestClass(testClass).annotations
+ .filterIsInstance<FlickerServiceCompatible>().firstOrNull() != null
+ if (isFlickerServiceCompatible) {
+ flickerTestParameter.enableFaas()
+ }
+
+ val builder = flickerBuilderProviderMethod.invokeExplosively(test) as FlickerBuilder
flickerTestParameter.initialize(builder, testName)
} else {
- Log.v(FLICKER_TAG, "Missing flicker builder provider method " +
- "in ${test::class.java.simpleName}")
+ Log.v(
+ FLICKER_TAG,
+ "Missing flicker builder provider method " +
+ "in ${test::class.java.simpleName}"
+ )
}
}
/**
* {@inheritDoc}
*/
- override fun validateInstanceMethods(errors: MutableList<Throwable>) {
- validateFlickerObject(errors)
- super.validateInstanceMethods(errors)
- }
-
- /**
- * {@inheritDoc}
- */
override fun validateConstructor(errors: MutableList<Throwable>) {
super.validateConstructor(errors)
@@ -155,8 +273,12 @@
// validator will create an exception
val ctor = testClass.javaClass.constructors.first()
if (ctor.parameterTypes.none { it == FlickerTestParameter::class.java }) {
- errors.add(Exception("Constructor should have a parameter of type " +
- FlickerTestParameter::class.java.simpleName))
+ errors.add(
+ Exception(
+ "Constructor should have a parameter of type " +
+ FlickerTestParameter::class.java.simpleName
+ )
+ )
}
}
}
@@ -166,4 +288,34 @@
* necessary to release memory after a configuration is executed
*/
private fun getFlickerCleanUpMethod() = FlickerTestParameter::class.java.getMethod("clear")
+
+ private fun getAnnotatedFieldsByParameter(): List<FrameworkField?> {
+ return testClass.getAnnotatedFields(Parameterized.Parameter::class.java)
+ }
+
+ private fun getInjectionType(): String {
+ return if (fieldsAreAnnotated()) {
+ "FIELD"
+ } else {
+ "CONSTRUCTOR"
+ }
+ }
+
+ private fun fieldsAreAnnotated(): Boolean {
+ return !getAnnotatedFieldsByParameter().isEmpty()
+ }
+
+ private fun aggregateFaasResults(
+ assertionResults: MutableList<AssertionResult>
+ ): Map<String, List<AssertionResult>> {
+ val aggregatedResults = mutableMapOf<String, MutableList<AssertionResult>>()
+ for (result in assertionResults) {
+ val testName = "FaaS_${result.scenario.description}_${result.assertionName}"
+ if (!aggregatedResults.containsKey(testName)) {
+ aggregatedResults[testName] = mutableListOf()
+ }
+ aggregatedResults[testName]!!.add(result)
+ }
+ return aggregatedResults
+ }
}
diff --git a/libraries/flicker/src/com/android/server/wm/flicker/FlickerTestParameter.kt b/libraries/flicker/src/com/android/server/wm/flicker/FlickerTestParameter.kt
index ca050f0..0c8323c 100644
--- a/libraries/flicker/src/com/android/server/wm/flicker/FlickerTestParameter.kt
+++ b/libraries/flicker/src/com/android/server/wm/flicker/FlickerTestParameter.kt
@@ -28,6 +28,7 @@
import com.android.server.wm.flicker.dsl.AssertionTag
import com.android.server.wm.flicker.dsl.FlickerBuilder
import com.android.server.wm.flicker.helpers.SampleAppHelper
+import com.android.server.wm.flicker.helpers.isShellTransitionsEnabled
import com.android.server.wm.flicker.rules.ChangeDisplayOrientationRule
import com.android.server.wm.flicker.rules.LaunchAppRule
import com.android.server.wm.flicker.rules.RemoveAllTasksButHomeRule
@@ -50,7 +51,7 @@
) {
private var internalFlicker: Flicker? = null
- private val flicker: Flicker get() = internalFlicker ?: error("Flicker not initialized")
+ internal val flicker: Flicker get() = internalFlicker ?: error("Flicker not initialized")
private val name: String get() = nameOverride ?: defaultName(this)
internal val isInitialized: Boolean get() = internalFlicker != null
@@ -109,6 +110,13 @@
config[IS_TABLET] = isTablet
}
+ val isFaasEnabled get() = config.getOrDefault(FAAS_ENABLED, false) as Boolean &&
+ isShellTransitionsEnabled
+
+ fun enableFaas() {
+ config[FAAS_ENABLED] = true
+ }
+
/**
* Clean the internal flicker reference (cache)
*/
@@ -123,7 +131,7 @@
internalFlicker = builder
.withTestName { "${testName}_$name" }
.repeat { config.getOrDefault(REPETITIONS, 1) as Int }
- .withFlickerAsAService { config.getOrDefault(FAAS_ENABLED, false) as Boolean }
+ .withFlickerAsAService { isFaasEnabled }
.build(TransitionRunnerWithRules(getTestSetupRules(builder.instrumentation)))
}
diff --git a/libraries/flicker/src/com/android/server/wm/flicker/annotation/FlickerServiceCompatible.kt b/libraries/flicker/src/com/android/server/wm/flicker/annotation/FlickerServiceCompatible.kt
new file mode 100644
index 0000000..c602395
--- /dev/null
+++ b/libraries/flicker/src/com/android/server/wm/flicker/annotation/FlickerServiceCompatible.kt
@@ -0,0 +1,25 @@
+/*
+ * Copyright (C) 2022 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.android.server.wm.flicker.annotation
+
+/**
+ * Annotate your Flicker test class with this annotation to enable Flicker as a Service on the
+ * transition defined in the Flicker test class. It requires shell transitions to be enabled.
+ */
+@Target(AnnotationTarget.CLASS)
+@Retention(AnnotationRetention.RUNTIME)
+annotation class FlickerServiceCompatible
diff --git a/libraries/flicker/src/com/android/server/wm/flicker/service/FlickerFrameworkMethod.kt b/libraries/flicker/src/com/android/server/wm/flicker/service/FlickerFrameworkMethod.kt
new file mode 100644
index 0000000..5d43cad
--- /dev/null
+++ b/libraries/flicker/src/com/android/server/wm/flicker/service/FlickerFrameworkMethod.kt
@@ -0,0 +1,58 @@
+/*
+ * Copyright (C) 2022 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.android.server.wm.flicker.service
+
+import java.lang.reflect.Method
+import org.junit.runners.model.FrameworkMethod
+
+class FlickerFrameworkMethod(
+ method: Method?,
+ private val target: FlickerTestCase,
+ private val name: String
+) : FrameworkMethod(method) {
+ /**
+ * Returns the result of invoking this method on target with parameters params.
+ * Executes the test method on the supplied target (returned by the JUnitTestFactory)
+ * and not the instance generated by FrameworkMethod.
+ */
+ override fun invokeExplosively(target: Any?, vararg params: Any?): Any? {
+ return super.invokeExplosively(this.target, params)
+ }
+
+ /**
+ * Returns the method's name.
+ */
+ override fun getName(): String {
+ return name
+ }
+
+ /**
+ * We are reusing the same method with different parameters for each of our test results.
+ * So we can't use the parent definition of this which check for method equality, otherwise
+ * it would consider all FaaS test method as the same test which they are not.
+ */
+ override fun equals(other: Any?): Boolean {
+ return other is FlickerFrameworkMethod && name == other.name
+ }
+
+ /**
+ * @See equals
+ */
+ override fun hashCode(): Int {
+ return name.hashCode()
+ }
+}
diff --git a/libraries/flicker/src/com/android/server/wm/flicker/service/FlickerServiceResultsCollector.kt b/libraries/flicker/src/com/android/server/wm/flicker/service/FlickerServiceResultsCollector.kt
index 06872a1..155a632 100644
--- a/libraries/flicker/src/com/android/server/wm/flicker/service/FlickerServiceResultsCollector.kt
+++ b/libraries/flicker/src/com/android/server/wm/flicker/service/FlickerServiceResultsCollector.kt
@@ -22,6 +22,7 @@
import android.util.Log
import androidx.test.platform.app.InstrumentationRegistry
import com.android.server.wm.flicker.service.assertors.AssertionResult
+import com.android.server.wm.flicker.service.config.AssertionInvocationGroup
import java.nio.file.Path
import org.junit.runner.Description
import org.junit.runner.Result
@@ -39,9 +40,6 @@
private var criticalUserJourneyName: String = UNDEFINED_CUJ
private var collectMetricsPerTest = true
- // The flicker service results (aka metrics) we want to upload
- private val metrics = mutableMapOf<String, Int>()
-
private val _executionErrors = mutableListOf<Throwable>()
val executionErrors: List<Throwable> get() = _executionErrors
@@ -61,7 +59,6 @@
errorReportingBlock {
Log.i(LOG_TAG, "onTestRunStart :: collectMetricsPerTest = $collectMetricsPerTest")
if (!collectMetricsPerTest) {
- require(metrics.isEmpty())
tracesCollector.start()
}
}
@@ -70,7 +67,6 @@
override fun onTestStart(testData: DataRecord, description: Description) {
errorReportingBlock {
Log.i(LOG_TAG, "onTestStart :: collectMetricsPerTest = $collectMetricsPerTest")
- require(metrics.isEmpty())
if (collectMetricsPerTest) {
tracesCollector.start()
}
@@ -97,19 +93,10 @@
Log.i(LOG_TAG, "onTestRunEnd :: collectMetricsPerTest = $collectMetricsPerTest")
if (!collectMetricsPerTest) {
stopTracingAndCollectFlickerMetrics(runData)
- collectMetrics(runData)
}
}
}
- fun postFlickerResultsForCollection(results: Map<String, Int>) {
- for ((key, res) in results) {
- require(res == 1 || res == 0)
- // If a failure is posted for key then we fail
- metrics[key] = (metrics[key] ?: 1) and res
- }
- }
-
private fun stopTracingAndCollectFlickerMetrics(dataRecord: DataRecord) {
tracesCollector.stop()
val collectedTraces = tracesCollector.getCollectedTraces()
@@ -118,36 +105,72 @@
collectedTraces.wmTrace,
collectedTraces.layersTrace, collectedTraces.transitionsTrace
)
- processFlickerResults(results)
- collectMetrics(dataRecord)
+ val aggregatedResults = processFlickerResults(results)
+ collectMetrics(dataRecord, aggregatedResults)
}
- private fun collectMetrics(data: DataRecord) {
- val it = metrics.entries.iterator()
- while (it.hasNext()) {
- val (key, value) = it.next()
- data.addStringMetric(key, value.toString())
- it.remove()
- }
- }
-
- /**
- * Convert the assertions generated by the Flicker Service to specific metric key pairs that
- * contain enough information to later further and analyze in dashboards.
- */
- private fun processFlickerResults(results: List<AssertionResult>) {
+ private fun processFlickerResults(
+ results: List<AssertionResult>
+ ): Map<String, AggregatedFlickerResult> {
+ val aggregatedResults = mutableMapOf<String, AggregatedFlickerResult>()
for (result in results) {
- // 0 for pass, 1 for failure
- val metric = if (result.failed) 1 else 0
-
- // Add information about the CUJ we are running the assertions on
- val assertionName = "${result.assertionGroup}#${result.assertionName}"
- val key = "$FASS_METRICS_PREFIX::$criticalUserJourneyName::$assertionName"
- if (metrics.containsKey(key)) {
- Log.w(LOG_TAG, "overriding metric $key")
+ val key = getKeyForAssertionResult(result)
+ if (!aggregatedResults.containsKey(key)) {
+ aggregatedResults[key] = AggregatedFlickerResult()
}
- metrics[key] = metric
+ aggregatedResults[key]!!.addResult(result)
}
+ return aggregatedResults
+ }
+
+ private fun collectMetrics(
+ data: DataRecord,
+ aggregatedResults: Map<String, AggregatedFlickerResult>
+ ) {
+ val it = aggregatedResults.entries.iterator()
+
+ while (it.hasNext()) {
+ val (key, result) = it.next()
+ val resultString = "${result.passes}/${result.passes + result.failures}"
+ var color = ANSI_RESET
+ if (result.failures > 0) {
+ color = ANSI_RED
+ }
+ if (result.failures == 0 && result.passes > 0) {
+ color = ANSI_GREEN
+ }
+
+ val errorString = StringBuilder()
+ if (result.errors.isNotEmpty()) {
+ errorString.append("\n\t$ANSI_RED_BOLD$key$ANSI_RESET\n")
+ for ((index, error) in result.errors.withIndex()) {
+ errorString.append(
+ "$ANSI_RED\t ${index + 1}) ${error.lines()[0]}" +
+ "${error.substring(error.indexOf('\n') + 1)
+ .prependIndent("\t ")}$ANSI_RESET\n"
+ )
+ }
+ }
+
+ var blockingStatus = ""
+ if (result.failures > 0) {
+ blockingStatus = if (result.invocationGroup == AssertionInvocationGroup.BLOCKING) {
+ "$ANSI_RED_BOLD(BLOCKING)$ANSI_RESET"
+ } else {
+ "$ANSI_WHITE$ANSI_LOW_INTENSITY(non blocking)$ANSI_RESET"
+ }
+ }
+
+ data.addStringMetric(
+ key,
+ "$color$resultString$ANSI_RESET $blockingStatus$errorString"
+ )
+ }
+ }
+
+ private fun getKeyForAssertionResult(result: AssertionResult): String {
+ val assertionName = "${result.scenario}#${result.assertionName}"
+ return "$FASS_METRICS_PREFIX::$criticalUserJourneyName::$assertionName"
}
fun setCriticalUserJourneyName(className: String?) {
@@ -159,5 +182,29 @@
private const val FASS_METRICS_PREFIX = "FASS"
private const val UNDEFINED_CUJ = "UndefinedCUJ"
private val LOG_TAG = "FlickerResultsCollector"
+
+ class AggregatedFlickerResult {
+ var failures = 0
+ var passes = 0
+ val errors = mutableListOf<String>()
+ var invocationGroup: AssertionInvocationGroup? = null
+
+ fun addResult(result: AssertionResult) {
+ if (result.failed) {
+ failures++
+ errors.add(result.assertionError?.message ?: "FAILURE WITHOUT ERROR MESSAGE...")
+ } else {
+ passes++
+ }
+
+ if (invocationGroup == null) {
+ invocationGroup = result.invocationGroup
+ }
+
+ if (invocationGroup != result.invocationGroup) {
+ error("Unexpected assertion group mismatch")
+ }
+ }
+ }
}
}
diff --git a/libraries/flicker/src/com/android/server/wm/flicker/service/FlickerTestCase.kt b/libraries/flicker/src/com/android/server/wm/flicker/service/FlickerTestCase.kt
new file mode 100644
index 0000000..9615dac
--- /dev/null
+++ b/libraries/flicker/src/com/android/server/wm/flicker/service/FlickerTestCase.kt
@@ -0,0 +1,54 @@
+/*
+ * Copyright (C) 2022 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.android.server.wm.flicker.service
+
+import com.android.server.wm.flicker.service.assertors.AssertionResult
+import junit.framework.Assert
+import org.junit.Test
+
+class FlickerTestCase(val results: List<AssertionResult>) {
+
+ @Test
+ fun runTest(param: Any) {
+ if (containsFailures) {
+ Assert.fail(assertionMessage)
+ }
+ }
+
+ private val containsFailures: Boolean get() = results.any { it.failed }
+
+ private val assertionMessage: String get() {
+ if (!containsFailures) {
+ return "${results.size}/${results.size} PASSED"
+ }
+
+ if (results.size == 1) {
+ return results[0].assertionError!!.message
+ }
+
+ return buildString {
+ append("$failedCount/${results.size} FAILED\n")
+ for (result in results) {
+ if (result.failed) {
+ append("\n${result.assertionError!!.message.prependIndent(" ")}")
+ }
+ }
+ }
+ }
+
+ private val failedCount: Int get() = results.filter { it.failed }.size
+}
diff --git a/libraries/flicker/src/com/android/server/wm/flicker/service/assertors/AssertionData.kt b/libraries/flicker/src/com/android/server/wm/flicker/service/assertors/AssertionData.kt
index 564b82e..795f774 100644
--- a/libraries/flicker/src/com/android/server/wm/flicker/service/assertors/AssertionData.kt
+++ b/libraries/flicker/src/com/android/server/wm/flicker/service/assertors/AssertionData.kt
@@ -17,13 +17,13 @@
package com.android.server.wm.flicker.service.assertors
import com.android.server.wm.flicker.service.config.AssertionInvocationGroup
-import com.android.server.wm.flicker.service.config.FlickerServiceConfig.Companion.AssertionGroup
+import com.android.server.wm.flicker.service.config.FlickerServiceConfig.Companion.Scenario
/**
* Stores data for FASS assertions.
*/
data class AssertionData(
- val assertionGroup: AssertionGroup,
+ val scenario: Scenario,
val assertionBuilder: BaseAssertionBuilder,
val category: AssertionInvocationGroup
)
diff --git a/libraries/flicker/src/com/android/server/wm/flicker/service/assertors/AssertionResult.kt b/libraries/flicker/src/com/android/server/wm/flicker/service/assertors/AssertionResult.kt
index c20fcf1..3bbe0b3 100644
--- a/libraries/flicker/src/com/android/server/wm/flicker/service/assertors/AssertionResult.kt
+++ b/libraries/flicker/src/com/android/server/wm/flicker/service/assertors/AssertionResult.kt
@@ -17,7 +17,7 @@
package com.android.server.wm.flicker.service.assertors
import com.android.server.wm.flicker.service.config.AssertionInvocationGroup
-import com.android.server.wm.flicker.service.config.FlickerServiceConfig.Companion.AssertionGroup
+import com.android.server.wm.flicker.service.config.FlickerServiceConfig.Companion.Scenario
import com.android.server.wm.flicker.traces.FlickerSubjectException
import com.android.server.wm.flicker.traces.layers.LayerSubject
import com.android.server.wm.flicker.traces.layers.LayersTraceSubject
@@ -34,7 +34,7 @@
*/
data class AssertionResult(
val assertionName: String,
- val assertionGroup: AssertionGroup,
+ val scenario: Scenario,
val invocationGroup: AssertionInvocationGroup,
val assertionError: FlickerSubjectException?
) {
diff --git a/libraries/flicker/src/com/android/server/wm/flicker/service/assertors/BaseAssertionBuilder.kt b/libraries/flicker/src/com/android/server/wm/flicker/service/assertors/BaseAssertionBuilder.kt
index 2820058..0e3c805 100644
--- a/libraries/flicker/src/com/android/server/wm/flicker/service/assertors/BaseAssertionBuilder.kt
+++ b/libraries/flicker/src/com/android/server/wm/flicker/service/assertors/BaseAssertionBuilder.kt
@@ -18,7 +18,7 @@
import com.android.server.wm.flicker.service.config.AssertionInvocationGroup
import com.android.server.wm.flicker.service.config.AssertionInvocationGroup.NON_BLOCKING
-import com.android.server.wm.flicker.service.config.FlickerServiceConfig.Companion.AssertionGroup
+import com.android.server.wm.flicker.service.config.FlickerServiceConfig.Companion.Scenario
import com.android.server.wm.flicker.traces.FlickerSubjectException
import com.android.server.wm.flicker.traces.layers.LayersTraceSubject
import com.android.server.wm.flicker.traces.windowmanager.WindowManagerTraceSubject
@@ -31,7 +31,7 @@
internal var invocationGroup: AssertionInvocationGroup = NON_BLOCKING
// Assertion name
- val name: String = this::class.java.simpleName
+ open val name: String = this::class.java.simpleName
/**
* Evaluates assertions that require only WM traces.
@@ -67,7 +67,7 @@
transition: Transition,
wmSubject: WindowManagerTraceSubject?,
layerSubject: LayersTraceSubject?,
- assertionGroup: AssertionGroup
+ scenario: Scenario
): AssertionResult {
var assertionError: FlickerSubjectException? = null
try {
@@ -80,7 +80,7 @@
} catch (e: FlickerSubjectException) {
assertionError = e
}
- return AssertionResult(name, assertionGroup, invocationGroup, assertionError)
+ return AssertionResult(name, scenario, invocationGroup, assertionError)
}
infix fun runAs(invocationGroup: AssertionInvocationGroup): BaseAssertionBuilder {
diff --git a/libraries/flicker/src/com/android/server/wm/flicker/service/assertors/FaasData.kt b/libraries/flicker/src/com/android/server/wm/flicker/service/assertors/FaasData.kt
new file mode 100644
index 0000000..e341095
--- /dev/null
+++ b/libraries/flicker/src/com/android/server/wm/flicker/service/assertors/FaasData.kt
@@ -0,0 +1,60 @@
+/*
+ * Copyright (C) 2022 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.android.server.wm.flicker.service.assertors
+
+import com.android.server.wm.traces.common.layers.LayersTrace
+import com.android.server.wm.traces.common.prettyTimestamp
+import com.android.server.wm.traces.common.transition.Transition
+import com.android.server.wm.traces.common.windowmanager.WindowManagerTrace
+import com.google.common.truth.Fact
+
+data class FaasData(
+ val transition: Transition,
+ val entireWmTrace: WindowManagerTrace,
+ val entireLayersTrace: LayersTrace
+) {
+ fun toFacts(): Collection<Fact> {
+ val unclippedFirstTimestamp = entireWmTrace.firstOrNull()?.timestamp ?: 0L
+ val unclippedLastTimestamp = entireWmTrace.lastOrNull()?.timestamp ?: 0L
+ val unclippedTraceFirst = "${prettyTimestamp(unclippedFirstTimestamp)} " +
+ "(timestamp=$unclippedFirstTimestamp)"
+ val unclippedTraceLast = "${prettyTimestamp(unclippedLastTimestamp)} " +
+ "(timestamp=$unclippedLastTimestamp)"
+
+ return listOf(
+ Fact.fact("Transition type", transition.type),
+ Fact.fact(
+ "Transition start",
+ "${prettyTimestamp(transition.start)} (timestamp=${transition.start})"
+ ),
+ Fact.fact(
+ "Transition end",
+ "${prettyTimestamp(transition.end)} (timestamp=${transition.end})"
+ ),
+ Fact.fact("Transition type", transition.type),
+ Fact.fact(
+ "Transition changes",
+ transition.changes
+ .joinToString("\n -", "\n -") {
+ "${it.transitMode} ${it.windowName}"
+ }
+ ),
+ Fact.fact("Extracted from trace start", unclippedTraceFirst),
+ Fact.fact("Extracted from trace end", unclippedTraceLast)
+ )
+ }
+}
diff --git a/libraries/flicker/src/com/android/server/wm/flicker/service/assertors/TransitionAsserter.kt b/libraries/flicker/src/com/android/server/wm/flicker/service/assertors/TransitionAsserter.kt
index a5ef668..d0dacc2 100644
--- a/libraries/flicker/src/com/android/server/wm/flicker/service/assertors/TransitionAsserter.kt
+++ b/libraries/flicker/src/com/android/server/wm/flicker/service/assertors/TransitionAsserter.kt
@@ -32,52 +32,88 @@
/** {@inheritDoc} */
fun analyze(
transition: Transition,
- _wmTrace: WindowManagerTrace?,
- _layersTrace: LayersTrace?
+ wmTrace: WindowManagerTrace,
+ layersTrace: LayersTrace
): List<AssertionResult> {
- var wmTrace = _wmTrace
- if (wmTrace != null && wmTrace.entries.isEmpty()) {
- // Empty trace, nothing to assert on the wmTrace
- logger("Passed an empty wmTrace")
- wmTrace = null
- }
-
- var layersTrace = _layersTrace
- if (layersTrace != null && layersTrace.entries.isEmpty()) {
- layersTrace = null
- }
-
- require(wmTrace != null || layersTrace != null)
+ val (wmTraceForTransition, layersTraceForTransition) =
+ splitTraces(transition, wmTrace, layersTrace)
+ require(wmTraceForTransition != null || layersTraceForTransition != null)
logger.invoke("Running assertions...")
- return runAssertionsOnSubjects(transition, wmTrace, layersTrace, assertions)
+ return runAssertionsOnSubjects(
+ transition, wmTraceForTransition, layersTraceForTransition,
+ wmTrace, layersTrace, assertions
+ )
}
private fun runAssertionsOnSubjects(
transition: Transition,
- wmTrace: WindowManagerTrace?,
- layersTrace: LayersTrace?,
+ wmTraceToAssert: WindowManagerTrace?,
+ layersTraceToAssert: LayersTrace?,
+ entireWmTrace: WindowManagerTrace,
+ entireLayersTrace: LayersTrace,
assertions: List<AssertionData>
): List<AssertionResult> {
val results: MutableList<AssertionResult> = mutableListOf()
assertions.forEach {
val assertion = it.assertionBuilder
- logger.invoke("Running assertion $assertion for ${it.assertionGroup}")
- val wmSubject = if (wmTrace != null) {
- WindowManagerTraceSubject.assertThat(wmTrace)
+ logger.invoke("Running assertion $assertion for ${it.scenario}")
+ val faasFacts = FaasData(transition, entireWmTrace, entireLayersTrace).toFacts()
+ val wmSubject = if (wmTraceToAssert != null) {
+ WindowManagerTraceSubject.assertThat(wmTraceToAssert, facts = faasFacts)
} else {
null
}
- val layersSubject = if (layersTrace != null) {
- LayersTraceSubject.assertThat(layersTrace)
+ val layersSubject = if (layersTraceToAssert != null) {
+ LayersTraceSubject.assertThat(layersTraceToAssert, facts = faasFacts)
} else {
null
}
- val result = assertion.evaluate(transition, wmSubject, layersSubject, it.assertionGroup)
+ val result = assertion.evaluate(transition, wmSubject, layersSubject, it.scenario)
results.add(result)
}
return results
}
+
+ /**
+ * Splits a [WindowManagerTrace] and a [LayersTrace] by a [Transition].
+ *
+ * @param tag a list with all [TransitionTag]s
+ * @param wmTrace Window Manager trace
+ * @param layersTrace Surface Flinger trace
+ * @return a list with [WindowManagerTrace] blocks
+ */
+ private fun splitTraces(
+ transition: Transition,
+ wmTrace: WindowManagerTrace,
+ layersTrace: LayersTrace
+ ): FilteredTraces {
+ var filteredWmTrace: WindowManagerTrace? = wmTrace.transitionSlice(transition)
+ var filteredLayersTrace: LayersTrace? = layersTrace.transitionSlice(transition)
+
+ if (filteredWmTrace?.entries?.isEmpty() == true) {
+ // Empty trace, nothing to assert on the wmTrace
+ logger("Got an empty wmTrace for transition $transition")
+ filteredWmTrace = null
+ }
+
+ if (filteredLayersTrace?.entries?.isEmpty() == true) {
+ // Empty trace, nothing to assert on the layers trace
+ logger("Got an empty surface trace for transition $transition")
+ filteredLayersTrace = null
+ }
+
+ return FilteredTraces(filteredWmTrace, filteredLayersTrace)
+ }
+
+ data class FilteredTraces(
+ val wmTrace: WindowManagerTrace?,
+ val layersTrace: LayersTrace?
+ )
+}
+
+private fun WindowManagerTrace.transitionSlice(transition: Transition): WindowManagerTrace {
+ return slice(transition.start, transition.end)
}
diff --git a/libraries/flicker/src/com/android/server/wm/flicker/service/config/FlickerServiceConfig.kt b/libraries/flicker/src/com/android/server/wm/flicker/service/config/FlickerServiceConfig.kt
index f64faa3..343c3f2 100644
--- a/libraries/flicker/src/com/android/server/wm/flicker/service/config/FlickerServiceConfig.kt
+++ b/libraries/flicker/src/com/android/server/wm/flicker/service/config/FlickerServiceConfig.kt
@@ -16,7 +16,6 @@
package com.android.server.wm.flicker.service.config
-import android.view.WindowManager.TRANSIT_OPEN
import com.android.server.wm.flicker.service.assertors.AssertionData
import com.android.server.wm.flicker.service.assertors.BaseAssertionBuilder
import com.android.server.wm.flicker.service.assertors.Components
@@ -44,7 +43,7 @@
fun assertionsForTransition(transition: Transition): List<AssertionData> {
val assertions: MutableList<AssertionData> = mutableListOf()
- for (assertionGroup in AssertionGroup.values()) {
+ for (assertionGroup in Scenario.values()) {
assertionGroup.description
if (assertionGroup.executionCondition.shouldExecute(transition)) {
for (assertion in assertionGroup.assertions) {
@@ -59,7 +58,7 @@
}
companion object {
- enum class AssertionGroup(
+ enum class Scenario(
val description: String,
val executionCondition: AssertionExecutionCondition,
val assertions: List<BaseAssertionBuilder>
@@ -82,7 +81,7 @@
NEVER({ false }),
APP_LAUNCH({ t ->
t.type == Type.OPEN &&
- t.changes.any { it.transitMode == TRANSIT_OPEN }
+ t.changes.any { it.transitMode == Type.OPEN }
}),
}
diff --git a/libraries/flicker/src/com/android/server/wm/flicker/traces/FlickerSubjectException.kt b/libraries/flicker/src/com/android/server/wm/flicker/traces/FlickerSubjectException.kt
index 980d88e..9c6b914 100644
--- a/libraries/flicker/src/com/android/server/wm/flicker/traces/FlickerSubjectException.kt
+++ b/libraries/flicker/src/com/android/server/wm/flicker/traces/FlickerSubjectException.kt
@@ -28,10 +28,10 @@
) : AssertionError(cause.message, if (cause is FlickerSubjectException) null else cause) {
internal val timestamp = subject.timestamp
private val prettyTimestamp =
- if (timestamp > 0) "${prettyTimestamp(timestamp)} (timestamp=$timestamp)" else ""
+ if (timestamp > 0) "${prettyTimestamp(timestamp)} (timestamp=$timestamp)" else ""
internal val errorType: String =
- if (cause is AssertionError) "Flicker assertion error" else "Unknown error"
+ if (cause is AssertionError) "Flicker assertion error" else "Unknown error"
internal val errorDescription = buildString {
appendLine("Where? $prettyTimestamp")
@@ -49,7 +49,7 @@
internal val subjectInformation = buildString {
appendLine("Facts:")
- subject.completeFacts.forEach { append("\t").appendLine(it) }
+ subject.completeFacts.forEach { appendLine(it.toString().prependIndent("\t")) }
}
override val message: String
diff --git a/libraries/flicker/src/com/android/server/wm/flicker/traces/layers/LayersTraceSubject.kt b/libraries/flicker/src/com/android/server/wm/flicker/traces/layers/LayersTraceSubject.kt
index 9f910f0..73dade4 100644
--- a/libraries/flicker/src/com/android/server/wm/flicker/traces/layers/LayersTraceSubject.kt
+++ b/libraries/flicker/src/com/android/server/wm/flicker/traces/layers/LayersTraceSubject.kt
@@ -25,6 +25,7 @@
import com.android.server.wm.traces.common.layers.Layer
import com.android.server.wm.traces.common.layers.LayersTrace
import com.android.server.wm.traces.common.region.RegionTrace
+import com.google.common.truth.Fact
import com.google.common.truth.FailureMetadata
import com.google.common.truth.FailureStrategy
import com.google.common.truth.StandardSubjectBuilder
@@ -57,11 +58,17 @@
class LayersTraceSubject private constructor(
fm: FailureMetadata,
val trace: LayersTrace,
- override val parent: LayersTraceSubject?
+ override val parent: LayersTraceSubject?,
+ val facts: Collection<Fact>
) : FlickerTraceSubject<LayerTraceEntrySubject>(fm, trace),
ILayerSubject<LayersTraceSubject, RegionTraceSubject> {
- override val selfFacts
- get() = super.selfFacts.toMutableList()
+
+ override val selfFacts by lazy {
+ val allFacts = super.selfFacts.toMutableList()
+ allFacts.addAll(facts)
+ allFacts
+ }
+
override val subjects by lazy {
trace.entries.map { LayerTraceEntrySubject.assertThat(it, trace, this) }
}
@@ -308,8 +315,11 @@
/**
* Boilerplate Subject.Factory for LayersTraceSubject
*/
- private fun getFactory(parent: LayersTraceSubject?): Factory<Subject, LayersTrace> =
- Factory { fm, subject -> LayersTraceSubject(fm, subject, parent) }
+ private fun getFactory(
+ parent: LayersTraceSubject?,
+ facts: Collection<Fact> = emptyList()
+ ): Factory<Subject, LayersTrace> =
+ Factory { fm, subject -> LayersTraceSubject(fm, subject, parent, facts) }
/**
* Creates a [LayersTraceSubject] to representing a SurfaceFlinger trace,
@@ -321,11 +331,12 @@
@JvmOverloads
fun assertThat(
trace: LayersTrace,
- parent: LayersTraceSubject? = null
+ parent: LayersTraceSubject? = null,
+ facts: Collection<Fact> = emptyList()
): LayersTraceSubject {
val strategy = FlickerFailureStrategy()
val subject = StandardSubjectBuilder.forCustomFailureStrategy(strategy)
- .about(getFactory(parent))
+ .about(getFactory(parent, facts))
.that(trace) as LayersTraceSubject
strategy.init(subject)
return subject
@@ -335,8 +346,11 @@
* Static method for getting the subject factory (for use with assertAbout())
*/
@JvmStatic
- fun entries(parent: LayersTraceSubject?): Factory<Subject, LayersTrace> {
- return getFactory(parent)
+ fun entries(
+ parent: LayersTraceSubject?,
+ facts: Collection<Fact> = emptyList()
+ ): Factory<Subject, LayersTrace> {
+ return getFactory(parent, facts)
}
}
}
diff --git a/libraries/flicker/src/com/android/server/wm/flicker/traces/windowmanager/WindowManagerTraceSubject.kt b/libraries/flicker/src/com/android/server/wm/flicker/traces/windowmanager/WindowManagerTraceSubject.kt
index 04b6a7c..3d88999 100644
--- a/libraries/flicker/src/com/android/server/wm/flicker/traces/windowmanager/WindowManagerTraceSubject.kt
+++ b/libraries/flicker/src/com/android/server/wm/flicker/traces/windowmanager/WindowManagerTraceSubject.kt
@@ -26,8 +26,8 @@
import com.android.server.wm.traces.common.region.RegionTrace
import com.android.server.wm.traces.common.windowmanager.WindowManagerTrace
import com.android.server.wm.traces.common.windowmanager.windows.WindowState
+import com.google.common.truth.Fact
import com.google.common.truth.FailureMetadata
-import com.google.common.truth.FailureStrategy
import com.google.common.truth.StandardSubjectBuilder
import com.google.common.truth.Subject
import com.google.common.truth.Subject.Factory
@@ -58,11 +58,16 @@
class WindowManagerTraceSubject private constructor(
fm: FailureMetadata,
val trace: WindowManagerTrace,
- override val parent: WindowManagerTraceSubject?
+ override val parent: WindowManagerTraceSubject?,
+ private val facts: Collection<Fact>
) : FlickerTraceSubject<WindowManagerStateSubject>(fm, trace),
IWindowManagerSubject<WindowManagerTraceSubject, RegionTraceSubject> {
- override val selfFacts
- get() = super.selfFacts.toMutableList()
+
+ override val selfFacts by lazy {
+ val allFacts = super.selfFacts.toMutableList()
+ allFacts.addAll(facts)
+ allFacts
+ }
override val subjects by lazy {
trace.entries.map { WindowManagerStateSubject.assertThat(it, this, this) }
@@ -627,9 +632,10 @@
* Boilerplate Subject.Factory for WmTraceSubject
*/
private fun getFactory(
- parent: WindowManagerTraceSubject?
+ parent: WindowManagerTraceSubject?,
+ facts: Collection<Fact> = emptyList()
): Factory<Subject, WindowManagerTrace> =
- Factory { fm, subject -> WindowManagerTraceSubject(fm, subject, parent) }
+ Factory { fm, subject -> WindowManagerTraceSubject(fm, subject, parent, facts) }
/**
* Creates a [WindowManagerTraceSubject] representing a WindowManager trace,
@@ -641,11 +647,12 @@
@JvmOverloads
fun assertThat(
trace: WindowManagerTrace,
- parent: WindowManagerTraceSubject? = null
+ parent: WindowManagerTraceSubject? = null,
+ facts: Collection<Fact> = emptyList()
): WindowManagerTraceSubject {
val strategy = FlickerFailureStrategy()
val subject = StandardSubjectBuilder.forCustomFailureStrategy(strategy)
- .about(getFactory(parent))
+ .about(getFactory(parent, facts))
.that(trace) as WindowManagerTraceSubject
strategy.init(subject)
return subject
@@ -656,7 +663,8 @@
*/
@JvmStatic
fun entries(
- parent: WindowManagerTraceSubject?
- ): Factory<Subject, WindowManagerTrace> = getFactory(parent)
+ parent: WindowManagerTraceSubject?,
+ facts: Collection<Fact>
+ ): Factory<Subject, WindowManagerTrace> = getFactory(parent, facts)
}
}
