Produce a test per flicker assertion
Currently flicker runs all assertions for a transition and checks if the CUJ fails or passes.
For easier debugging, split the assertions into individual tests. Use a parameterized test runner to execute such tests until JUnit5 becomes available in the platform
Bug: 162923992
Test: atest FlickerTests
Change-Id: I21b062f60da745515a3f6280601d68f5afa17eb5
diff --git a/libraries/flicker/src/com/android/server/wm/flicker/Flicker.kt b/libraries/flicker/src/com/android/server/wm/flicker/Flicker.kt
index ad251e3..7116492 100644
--- a/libraries/flicker/src/com/android/server/wm/flicker/Flicker.kt
+++ b/libraries/flicker/src/com/android/server/wm/flicker/Flicker.kt
@@ -20,16 +20,12 @@
import android.support.test.launcherhelper.ILauncherStrategy
import androidx.annotation.VisibleForTesting
import androidx.test.uiautomator.UiDevice
-import com.android.server.wm.flicker.assertions.FlickerAssertionError
-import com.android.server.wm.flicker.dsl.AssertionTag
-import com.android.server.wm.flicker.dsl.AssertionTarget
-import com.android.server.wm.flicker.dsl.TestCommands
+import com.android.server.wm.flicker.assertions.AssertionData
import com.android.server.wm.flicker.monitor.ITransitionMonitor
import com.android.server.wm.flicker.monitor.WindowAnimationFrameStatsMonitor
-import com.android.server.wm.traces.parser.getCurrentState
-import com.google.common.truth.Truth
-import java.io.IOException
-import java.nio.file.Files
+import com.android.server.wm.traces.common.layers.LayersTrace
+import com.android.server.wm.traces.common.windowmanager.WindowManagerTrace
+import com.android.server.wm.traces.parser.windowmanager.WindowManagerStateHelper
import java.nio.file.Path
@DslMarker
@@ -41,189 +37,120 @@
* [WindowManagerTrace] and [LayersTrace]
*/
@FlickerDslMarker
-data class Flicker(
+class Flicker(
/**
* Instrumentation to run the tests
*/
- val instrumentation: Instrumentation,
+ @JvmField val instrumentation: Instrumentation,
/**
* Test automation component used to interact with the device
*/
- val device: UiDevice,
+ @JvmField val device: UiDevice,
/**
* Strategy used to interact with the launcher
*/
- val launcherStrategy: ILauncherStrategy,
+ @JvmField val launcherStrategy: ILauncherStrategy,
/**
* Output directory for test results
*/
- val outputDir: Path,
+ @JvmField val outputDir: Path,
/**
* Test name used to store the test results
*/
- private val testName: String,
+ @JvmField val testName: String,
/**
* Number of times the test should be executed
*/
- private var repetitions: Int,
+ @JvmField var repetitions: Int,
/**
* Monitor for janky frames, when filtering out janky runs
*/
- private val frameStatsMonitor: WindowAnimationFrameStatsMonitor?,
+ @JvmField val frameStatsMonitor: WindowAnimationFrameStatsMonitor?,
/**
* Enabled tracing monitors
*/
- private val traceMonitors: List<ITransitionMonitor>,
+ @JvmField val traceMonitors: List<ITransitionMonitor>,
+ /**
+ * Commands to be executed before each run
+ */
+ @JvmField val testSetup: List<Flicker.() -> Any>,
/**
* Commands to be executed before the test
*/
- private val setup: TestCommands,
+ @JvmField val runSetup: List<Flicker.() -> Any>,
/**
* Commands to be executed after the test
*/
- private val teardown: TestCommands,
+ @JvmField val testTeardown: List<Flicker.() -> Any>,
+ /**
+ * Commands to be executed after the run
+ */
+ @JvmField val runTeardown: List<Flicker.() -> Any>,
/**
* Test commands
*/
- private val transitions: List<Flicker.() -> Any>,
+ @JvmField val transitions: List<Flicker.() -> Any>,
/**
* Custom set of assertions
*/
- private val assertions: AssertionTarget
-) {
- private val results = mutableListOf<FlickerRunResult>()
- private val tags = AssertionTag.DEFAULT.map { it.tag }.toMutableSet()
@VisibleForTesting
- var error: Throwable? = null
- private set
-
+ @JvmField val assertions: List<AssertionData>,
/**
- * Iteration identifier during test run
+ * Runner to execute the test transitions
*/
- private var iteration = 0
+ @JvmField val runner: TransitionRunner,
+ /**
+ * Helper object for WM Synchronization
+ */
+ @JvmField val wmHelper: WindowManagerStateHelper
+) {
+ var result = FlickerResult()
+ private set
/**
* Executes the test.
*
- * The commands are executed in the following order:
- * 1) [setup] ([TestCommands.testCommands])
- * 2) [setup] ([TestCommands.runCommands])
- * 3) Start monitors
- * 4) [transitions]
- * 5) Stop monitors
- * 6) [teardown] ([TestCommands.runCommands])
- * 7) [teardown] ([TestCommands.testCommands])
- *
- * If the tests were already executed, reuse the previous results
- *
- * @throws IllegalArgumentException If the transitions
+ * @throws IllegalStateException If cannot execute the transition
*/
- fun execute() = apply {
- require(transitions.isNotEmpty()) { "A flicker test must include transitions to run" }
- if (results.isNotEmpty()) {
- Log.w(FLICKER_TAG, "Flicker test already executed. Reusing results.")
- return this
- }
- try {
- try {
- error = null
- setup.testCommands.forEach { it.invoke(this) }
- for (iteration in 0 until repetitions) {
- this.iteration = iteration
- try {
- setup.runCommands.forEach { it.invoke(this) }
- traceMonitors.forEach { it.start() }
- frameStatsMonitor?.run { start() }
- transitions.forEach { it.invoke(this) }
- } finally {
- traceMonitors.forEach { it.tryStop() }
- frameStatsMonitor?.run { tryStop() }
- teardown.runCommands.forEach { it.invoke(this) }
- }
- if (frameStatsMonitor?.jankyFramesDetected() == true) {
- Log.e(FLICKER_TAG, "Skipping iteration $iteration/${repetitions - 1} " +
- "for test $testName due to jank. $frameStatsMonitor")
- continue
- }
- saveResult(this.iteration)
- }
- } finally {
- teardown.testCommands.forEach { it.invoke(this) }
- }
- } catch (e: Throwable) {
- error = e
- throw RuntimeException(e)
+ fun execute(): Flicker = apply {
+ result = runner.execute(this)
+ val error = result.error
+ if (error != null) {
+ throw IllegalStateException("Unable to execute transition", error)
}
}
- private fun cleanUp(failures: List<FlickerAssertionError>) {
- results.forEach {
- if (it.canDelete(failures)) {
- it.cleanUp()
- }
- }
- }
-
- @Deprecated("Prefer checkAssertions", replaceWith = ReplaceWith("checkAssertions"))
- fun makeAssertions() = checkAssertions(includeFlakyAssertions = false)
+ /**
+ * Asserts if the transition of this flicker test has ben executed
+ */
+ fun checkIsExecuted() = result.checkIsExecuted()
/**
* Run the assertions on the trace
*
- * @param includeFlakyAssertions If true, checks the flaky assertion
+ * @param onlyFlaky Runs only the flaky assertions
* @throws AssertionError If the assertions fail or the transition crashed
*/
@JvmOverloads
- fun checkAssertions(includeFlakyAssertions: Boolean = false) {
- Truth.assertWithMessage(error?.message).that(error).isNull()
- Truth.assertWithMessage("Transition was not executed").that(results).isNotEmpty()
- val failures = results.flatMap { assertions.checkAssertions(it, includeFlakyAssertions) }
- this.cleanUp(failures)
+ fun checkAssertions(onlyFlaky: Boolean = false) {
+ if (result.isEmpty()) {
+ execute()
+ }
+ val failures = result.checkAssertions(assertions, onlyFlaky)
val failureMessage = failures.joinToString("\n") { it.message }
- Truth.assertWithMessage(failureMessage).that(failureMessage.isEmpty()).isTrue()
+
+ if (failureMessage.isNotEmpty()) {
+ throw AssertionError(failureMessage)
+ }
}
- private fun getTaggedFilePath(tag: String, file: String) =
- "${this.testName}_${this.iteration}_${tag}_$file"
-
/**
- * Captures a snapshot of the device state and associates it with a new tag.
- *
- * This tag can be used to make assertions about the state of the device when the
- * snapshot is collected.
- *
- * [tag] is used as part of the trace file name, thus, only valid letters and digits
- * can be used
- *
- * @throws IllegalArgumentException If [tag] contains invalid characters
+ * Deletes the traces files for successful assertions and clears the cached runner results
*/
- fun createTag(tag: String) {
- if (tag in tags) {
- throw IllegalArgumentException("Tag $tag has already been used")
- }
- tags.add(tag)
- val assertionTag = AssertionTag(tag)
-
- val deviceState = getCurrentState(instrumentation.uiAutomation)
- try {
- val wmTraceFile = outputDir.resolve(getTaggedFilePath(tag, "wm_trace"))
- Files.write(wmTraceFile, deviceState.wmTraceData)
-
- val layersTraceFile = outputDir.resolve(getTaggedFilePath(tag, "layers_trace"))
- Files.write(layersTraceFile, deviceState.layersTraceData)
-
- val result = FlickerRunResult(
- assertionTag,
- iteration = this.iteration,
- wmTraceFile = wmTraceFile,
- layersTraceFile = layersTraceFile,
- wmTrace = deviceState.wmTrace,
- layersTrace = deviceState.layersTrace
- )
- results.add(result)
- } catch (e: IOException) {
- throw RuntimeException("Unable to create trace file: ${e.message}", e)
- }
+ fun cleanUp() {
+ runner.cleanUp()
+ result.cleanUp()
+ result = FlickerResult()
}
/**
@@ -235,26 +162,25 @@
*/
fun withTag(tag: String, commands: Flicker.() -> Any) {
commands()
- createTag(tag)
+ runner.createTag(this, tag)
}
- private fun saveResult(iteration: Int) {
- val resultBuilder = FlickerRunResult.Builder()
- traceMonitors.forEach { it.save(testName, iteration, resultBuilder) }
-
- AssertionTag.DEFAULT.forEach { location ->
- results.add(resultBuilder.build(location))
- }
+ fun createTag(tag: String) {
+ withTag(tag) {}
}
- private fun ITransitionMonitor.tryStop() {
- this.run {
- try {
- stop()
- } catch (e: Exception) {
- Log.e(FLICKER_TAG, "Unable to stop $this")
- }
+ @JvmOverloads
+ fun copy(newAssertion: AssertionData?, newName: String = ""): Flicker {
+ val name = if (newName.isNotEmpty()) {
+ newName
+ } else {
+ testName
}
+ val assertion = newAssertion?.let { listOf(it) } ?: emptyList()
+ return Flicker(instrumentation, device, launcherStrategy, outputDir, name,
+ repetitions, frameStatsMonitor, traceMonitors, testSetup, runSetup,
+ testTeardown, runTeardown, transitions, assertion, runner, wmHelper
+ )
}
override fun toString(): String {
diff --git a/libraries/flicker/src/com/android/server/wm/flicker/FlickerResult.kt b/libraries/flicker/src/com/android/server/wm/flicker/FlickerResult.kt
new file mode 100644
index 0000000..e0bf86e
--- /dev/null
+++ b/libraries/flicker/src/com/android/server/wm/flicker/FlickerResult.kt
@@ -0,0 +1,89 @@
+/*
+ * Copyright (C) 2021 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.android.server.wm.flicker
+
+import com.android.server.wm.flicker.assertions.AssertionData
+import com.android.server.wm.flicker.assertions.FlickerAssertionError
+import com.google.common.truth.Truth
+
+/**
+ * Result of a flicker run, including transitions, errors and create tags
+ */
+data class FlickerResult(
+ /**
+ * Result of each transition run
+ */
+ @JvmField val runs: List<FlickerRunResult> = listOf(),
+ /**
+ * List of test created during the execution
+ */
+ @JvmField val tags: Set<String> = setOf(),
+ /**
+ * Error which happened during the transition
+ */
+ @JvmField val error: Throwable? = null
+) {
+ /**
+ * List of failures during assertion
+ */
+ private val failures: MutableList<FlickerAssertionError> = mutableListOf()
+
+ /**
+ * Asserts if the transition of this flicker test has ben executed
+ */
+ internal fun checkIsExecuted() {
+ Truth.assertWithMessage(error?.message).that(error).isNull()
+ Truth.assertWithMessage("Transition was not executed").that(runs).isNotEmpty()
+ }
+
+ /**
+ * Run the assertions on the trace
+ *
+ * @param onlyFlaky Runs only the flaky assertions
+ * @throws AssertionError If the assertions fail or the transition crashed
+ */
+ internal fun checkAssertions(
+ assertions: List<AssertionData>,
+ onlyFlaky: Boolean
+ ): List<FlickerAssertionError> {
+ checkIsExecuted()
+ val currFailures: List<FlickerAssertionError> = runs.flatMap { run ->
+ assertions.mapNotNull { assertion ->
+ try {
+ assertion.checkAssertion(run, onlyFlaky)
+ null
+ } catch (error: Throwable) {
+ FlickerAssertionError(error, assertion, run)
+ }
+ }
+ }
+ failures.addAll(currFailures)
+ return currFailures
+ }
+
+ fun cleanUp() {
+ runs.forEach {
+ if (it.canDelete(failures)) {
+ it.cleanUp()
+ }
+ }
+ }
+
+ fun isEmpty(): Boolean = error == null && runs.isEmpty()
+
+ fun isNotEmpty(): Boolean = !isEmpty()
+}
\ No newline at end of file
diff --git a/libraries/flicker/src/com/android/server/wm/flicker/FlickerRunResult.kt b/libraries/flicker/src/com/android/server/wm/flicker/FlickerRunResult.kt
index 715262d..b17383f 100644
--- a/libraries/flicker/src/com/android/server/wm/flicker/FlickerRunResult.kt
+++ b/libraries/flicker/src/com/android/server/wm/flicker/FlickerRunResult.kt
@@ -17,11 +17,18 @@
package com.android.server.wm.flicker
import android.util.Log
+import androidx.annotation.VisibleForTesting
import com.android.server.wm.flicker.assertions.FlickerAssertionError
-import com.android.server.wm.traces.common.windowmanager.WindowManagerTrace
+import com.android.server.wm.flicker.assertions.FlickerSubject
import com.android.server.wm.flicker.dsl.AssertionTag
+import com.android.server.wm.flicker.traces.eventlog.EventLogSubject
+import com.android.server.wm.traces.common.windowmanager.WindowManagerTrace
import com.android.server.wm.flicker.traces.eventlog.FocusEvent
+import com.android.server.wm.flicker.traces.layers.LayersTraceSubject
+import com.android.server.wm.flicker.traces.windowmanager.WindowManagerTraceSubject
+import com.android.server.wm.traces.common.layers.LayerTraceEntry
import com.android.server.wm.traces.common.layers.LayersTrace
+import com.android.server.wm.traces.common.windowmanager.WindowManagerState
import com.android.server.wm.traces.parser.layers.LayersTraceParser
import com.android.server.wm.traces.parser.windowmanager.WindowManagerTraceParser
import java.io.IOException
@@ -33,95 +40,40 @@
*/
class FlickerRunResult private constructor(
/**
- * Determines which assertions to run (e.g., start, end, all, or a custom tag)
- */
- var assertionTag: AssertionTag,
- /**
* Run identifier
*/
- val iteration: Int,
+ @JvmField val iteration: Int,
/**
- * Path to the WindowManager trace file, if collected
+ * Path to the trace files associated with the result (incl. screen recording)
*/
- @JvmField val wmTraceFile: Path?,
+ @JvmField val traceFiles: List<Path>,
/**
- * Path to the SurfaceFlinger trace file, if collected
+ * Determines which assertions to run (e.g., start, end, all, or a custom tag)
*/
- @JvmField val layersTraceFile: Path?,
+ @JvmField var assertionTag: String,
/**
- * Path to screen recording of the run, if collected
+ * Truth subject that corresponds to a [WindowManagerTrace] or [WindowManagerState]
*/
- @JvmField val screenRecording: Path?,
-
+ private val wmSubject: FlickerSubject?,
/**
- * List of focus events, if collected
+ * Truth subject that corresponds to a [LayersTrace] or [LayerTraceEntry]
*/
- val eventLog: List<FocusEvent>,
+ private val layersSubject: FlickerSubject?,
/**
- * Parse a [WindowManagerTrace]
+ * Truth subject that corresponds to a list of [FocusEvent]
*/
- val parseWmTrace: (() -> WindowManagerTrace?)?,
- /**
- * Parse a [WindowManagerTrace]
- */
- val parseLayersTrace: (() -> LayersTrace?)?
+ @VisibleForTesting
+ val eventLogSubject: EventLogSubject?
) {
- /**
- * [WindowManagerTrace] that corresponds to [wmTraceFile], or null if the
- * path is invalid
- */
- val wmTrace: WindowManagerTrace? get() = parseWmTrace?.invoke()
- /**
- * [LayersTrace] that corresponds to [layersTrace], or null if the
- * path is invalid
- */
- val layersTrace: LayersTrace? get() = parseLayersTrace?.invoke()
+ fun getSubjects(): List<FlickerSubject> {
+ val result = mutableListOf<FlickerSubject>()
- constructor(
- assertionTag: AssertionTag,
- iteration: Int,
- wmTraceFile: Path?,
- layersTraceFile: Path?,
- screenRecording: Path?,
- eventLog: List<FocusEvent>
- ) : this(
- assertionTag,
- iteration,
- wmTraceFile,
- layersTraceFile,
- screenRecording,
- eventLog,
- parseWmTrace = {
- wmTraceFile?.let {
- val traceData = Files.readAllBytes(it)
- WindowManagerTraceParser.parseFromTrace(traceData)
- }
- },
- parseLayersTrace = {
- layersTraceFile?.let {
- val traceData = Files.readAllBytes(it)
- LayersTraceParser.parseFromTrace(traceData)
- }
- }
- )
+ wmSubject?.run { result.add(this) }
+ layersSubject?.run { result.add(this) }
+ eventLogSubject?.run { result.add(this) }
- constructor(
- assertionTag: AssertionTag,
- iteration: Int,
- wmTraceFile: Path?,
- layersTraceFile: Path?,
- wmTrace: WindowManagerTrace?,
- layersTrace: LayersTrace?
- ) : this(
- assertionTag,
- iteration,
- wmTraceFile,
- layersTraceFile,
- screenRecording = null,
- eventLog = emptyList(),
- parseWmTrace = { wmTrace },
- parseLayersTrace = { layersTrace }
- )
+ return result
+ }
private fun Path?.tryDelete() {
try {
@@ -132,8 +84,8 @@
}
fun canDelete(failures: List<FlickerAssertionError>): Boolean {
- return failures.map { it.trace }.none {
- it == this.wmTraceFile || it == this.layersTraceFile
+ return failures.flatMap { it.traceFiles }.none { failureTrace ->
+ this.traceFiles.any { it == failureTrace }
}
}
@@ -141,9 +93,7 @@
* Delete the trace files collected
*/
fun cleanUp() {
- wmTraceFile.tryDelete()
- layersTraceFile.tryDelete()
- screenRecording.tryDelete()
+ this.traceFiles.forEach { it.tryDelete() }
}
class Builder @JvmOverloads constructor(private val iteration: Int = 0) {
@@ -165,23 +115,94 @@
/**
* List of focus events, if collected
*/
- var eventLog = listOf<FocusEvent>()
+ var eventLog: List<FocusEvent>? = null
- /**
- * Creates a new run result associated with an assertion tag
- *
- * By default assert all entries
- */
- @JvmOverloads
- fun build(assertionTag: AssertionTag = AssertionTag.ALL): FlickerRunResult {
- return FlickerRunResult(
- assertionTag,
- iteration,
- wmTraceFile,
- layersTraceFile,
- screenRecording,
- eventLog
+ private fun getTraceFiles() = listOfNotNull(wmTraceFile, layersTraceFile, screenRecording)
+
+ private fun buildResult(
+ assertionTag: String,
+ wmSubject: FlickerSubject?,
+ layersSubject: FlickerSubject?,
+ eventLogSubject: EventLogSubject? = null
+ ): FlickerRunResult {
+ return FlickerRunResult(iteration,
+ getTraceFiles(),
+ assertionTag,
+ wmSubject,
+ layersSubject,
+ eventLogSubject
)
}
+
+ /**
+ * Builds a new [FlickerRunResult] for a trace
+ *
+ * @param assertionTag Tag to associate with the result
+ * @param wmTrace WindowManager trace
+ * @param layersTrace Layers trace
+ */
+ fun buildStateResult(
+ assertionTag: String,
+ wmTrace: WindowManagerTrace?,
+ layersTrace: LayersTrace?
+ ): FlickerRunResult {
+ val wmSubject = wmTrace?.let { WindowManagerTraceSubject.assertThat(it).first() }
+ val layersSubject = layersTrace?.let { LayersTraceSubject.assertThat(it).first() }
+ return buildResult(assertionTag, wmSubject, layersSubject)
+ }
+
+ @VisibleForTesting
+ fun buildEventLogResult(): FlickerRunResult {
+ val events = eventLog ?: emptyList()
+ return buildResult(
+ AssertionTag.ALL,
+ wmSubject = null,
+ layersSubject = null,
+ eventLogSubject = EventLogSubject.assertThat(events)
+ )
+ }
+
+ @VisibleForTesting
+ fun buildTraceResults(): List<FlickerRunResult> {
+ var wmTrace: WindowManagerTrace? = null
+ var layersTrace: LayersTrace? = null
+
+ if (wmTrace == null && wmTraceFile != null) {
+ Log.v(FLICKER_TAG, "Parsing WM trace")
+ wmTrace = wmTraceFile?.let {
+ val traceData = Files.readAllBytes(it)
+ WindowManagerTraceParser.parseFromTrace(traceData)
+ }
+ }
+
+ if (layersTrace == null && layersTraceFile != null) {
+ Log.v(FLICKER_TAG, "Parsing Layers trace")
+ layersTrace = layersTraceFile?.let {
+ val traceData = Files.readAllBytes(it)
+ LayersTraceParser.parseFromTrace(traceData)
+ }
+ }
+
+ val wmSubject = wmTrace?.let { WindowManagerTraceSubject.assertThat(it) }
+ val layersSubject = layersTrace?.let { LayersTraceSubject.assertThat(it) }
+
+ val traceResult = buildResult(
+ AssertionTag.ALL, wmSubject, layersSubject)
+ val initialStateResult = buildResult(
+ AssertionTag.START, wmSubject?.first(), layersSubject?.first())
+ val finalStateResult = buildResult(
+ AssertionTag.END, wmSubject?.last(), layersSubject?.last())
+
+ return listOf(initialStateResult, finalStateResult, traceResult)
+ }
+
+ fun buildAll(): List<FlickerRunResult> {
+ val result = buildTraceResults().toMutableList()
+ if (eventLog != null) {
+ result.add(buildEventLogResult())
+ }
+
+ return result
+ }
}
-}
\ No newline at end of file
+}
diff --git a/libraries/flicker/src/com/android/server/wm/flicker/FlickerTestRunner.kt b/libraries/flicker/src/com/android/server/wm/flicker/FlickerTestRunner.kt
index e0eef46..4b7af23 100644
--- a/libraries/flicker/src/com/android/server/wm/flicker/FlickerTestRunner.kt
+++ b/libraries/flicker/src/com/android/server/wm/flicker/FlickerTestRunner.kt
@@ -1,5 +1,5 @@
/*
- * Copyright (C) 2020 The Android Open Source Project
+ * Copyright (C) 2021 The Android Open Source Project
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@@ -16,8 +16,9 @@
package com.android.server.wm.flicker
+import android.util.Log
import androidx.test.filters.FlakyTest
-import com.google.common.truth.Truth
+import org.junit.Assume
import org.junit.Rule
import org.junit.Test
@@ -29,25 +30,45 @@
*
* All the enabled assertions are created in a single test and all flaky assertions are created on
* a second test annotated with @FlakyTest
+ *
+ * @param testName Name of the test. Appears on log outputs and test dashboards
+ * @param flickerSpec Flicker test to execute
+ * @param cleanUp If this test should delete the traces and screen recording files if it passes
*/
-abstract class FlickerTestRunner(testName: String, private val flickerSpec: Flicker) {
+abstract class FlickerTestRunner(
+ testName: String,
+ private val flickerProvider: () -> Flicker,
+ private val cleanUp: Boolean
+) {
+ private val flickerSpec = flickerProvider.invoke()
+
@get:Rule
val flickerTestRule = FlickerTestRule(flickerSpec)
- /**
- * Tests if the transition executed successfully
- */
- @Test
- fun checkTransition() {
- Truth.assertWithMessage(flickerSpec.error?.message).that(flickerSpec.error).isNull()
+ private fun checkRequirements(onlyFlaky: Boolean) {
+ if (flickerSpec.assertions.size == 1) {
+ val isTestEnabled = flickerSpec.assertions.first().enabled
+ if (onlyFlaky) {
+ Assume.assumeFalse(isTestEnabled)
+ } else {
+ Assume.assumeTrue(isTestEnabled)
+ }
+ }
}
/**
* Run only the enabled assertions on the recorded traces.
*/
@Test
- fun checkAssertions() {
- flickerSpec.checkAssertions(includeFlakyAssertions = false)
+ fun test() {
+ checkRequirements(onlyFlaky = false)
+ flickerSpec.checkIsExecuted()
+ if (flickerSpec.hasAssertions()) {
+ flickerSpec.checkAssertions(includeFlakyAssertions = false)
+ if (cleanUp) {
+ flickerSpec.cleanUp()
+ }
+ }
}
/**
@@ -55,7 +76,14 @@
*/
@FlakyTest
@Test
- fun checkFlakyAssertions() {
- flickerSpec.checkAssertions(includeFlakyAssertions = true)
+ fun testFlaky() {
+ checkRequirements(onlyFlaky = true)
+ flickerSpec.checkIsExecuted()
+ if (flickerSpec.hasAssertions()) {
+ flickerSpec.checkAssertions(includeFlakyAssertions = true)
+ if (cleanUp) {
+ flickerSpec.cleanUp()
+ }
+ }
}
}
\ No newline at end of file
diff --git a/libraries/flicker/src/com/android/server/wm/flicker/FlickerTestRunnerFactory.kt b/libraries/flicker/src/com/android/server/wm/flicker/FlickerTestRunnerFactory.kt
index 5b7754d..7e44b3b 100644
--- a/libraries/flicker/src/com/android/server/wm/flicker/FlickerTestRunnerFactory.kt
+++ b/libraries/flicker/src/com/android/server/wm/flicker/FlickerTestRunnerFactory.kt
@@ -1,5 +1,5 @@
/*
- * Copyright (C) 2020 The Android Open Source Project
+ * Copyright (C) 2021 The Android Open Source Project
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@@ -22,6 +22,7 @@
import android.support.test.launcherhelper.LauncherStrategyFactory
import android.view.Surface
import com.android.server.wm.flicker.dsl.FlickerBuilder
+import com.android.server.wm.traces.parser.windowmanager.WindowManagerStateHelper
/**
* Factory for creating JUnit4 compatible tests based on the flicker DSL
@@ -33,7 +34,8 @@
private val supportedRotations: List<Int> = listOf(Surface.ROTATION_0, Surface.ROTATION_90),
private val repetitions: Int = 1,
private val launcherStrategy: ILauncherStrategy = LauncherStrategyFactory
- .getInstance(instrumentation).launcherStrategy
+ .getInstance(instrumentation).launcherStrategy,
+ private val wmHelper: WindowManagerStateHelper = WindowManagerStateHelper()
) {
/**
* Creates multiple instances of the same test, running on different device orientations
@@ -49,12 +51,12 @@
deviceConfigurations: List<Bundle> = getConfigNonRotationTests(),
testSpecification: FlickerBuilder.(Bundle) -> Any
): List<Array<Any>> {
- return deviceConfigurations.map {
- val builder = FlickerBuilder(instrumentation, launcherStrategy)
- val flickerTests = builder.apply { testSpecification(it) }.build()
+ return deviceConfigurations.flatMap {
+ val builder = FlickerBuilder(instrumentation, launcherStrategy, wmHelper = wmHelper)
+ val flickerTests = buildIndividualTests(builder.apply { testSpecification(it) })
flickerTests
- }.map { arrayOf(it.toString(), it) }
+ }.map { arrayOf(it.first, it.second, it.third) }
}
/**
@@ -72,12 +74,44 @@
deviceConfigurations: List<Bundle> = getConfigRotationTests(),
testSpecification: FlickerBuilder.(Bundle) -> Any
): List<Array<Any>> {
- return deviceConfigurations.map {
- val builder = FlickerBuilder(instrumentation, launcherStrategy)
- val flickerTests = builder.apply { testSpecification(it) }.build()
+ return deviceConfigurations.flatMap {
+ val builder = FlickerBuilder(instrumentation, launcherStrategy, wmHelper = wmHelper)
+ val flickerTests = buildIndividualTests(builder.apply { testSpecification(it) })
flickerTests
- }.map { arrayOf(it.toString(), it) }
+ }.map { arrayOf(it.first, it.second, it.third) }
+ }
+
+ /**
+ * Creates multiple flicker tests.
+ *
+ * Each test contains a single assertion, but all tests share the same setup, transition
+ * and results
+ */
+ private fun buildIndividualTests(
+ builder: FlickerBuilder
+ ): List<Triple<String, () -> Flicker, Boolean>> {
+ val transitionRunner = TransitionRunnerCached()
+ val flicker = builder.build(transitionRunner)
+ val assertionsList = flicker.assertions
+ val lastAssertionIdx = assertionsList.lastIndex
+ val onlyTransition = {
+ flicker.copy(newAssertion = null)
+ }
+
+ val result = mutableListOf(Triple(flicker.testName, onlyTransition, false))
+ result.addAll(
+ assertionsList.mapIndexed { idx, assertion ->
+ val newTestName = "${flicker.testName}_$assertion"
+ val newTest = {
+ flicker.copy(newAssertion = assertion, newName = newTestName)
+ }
+ val cleanUp = idx == lastAssertionIdx
+
+ Triple(newTestName, newTest, cleanUp)
+ }
+ )
+ return result
}
/**
diff --git a/libraries/flicker/src/com/android/server/wm/flicker/TransitionRunner.kt b/libraries/flicker/src/com/android/server/wm/flicker/TransitionRunner.kt
new file mode 100644
index 0000000..861576e
--- /dev/null
+++ b/libraries/flicker/src/com/android/server/wm/flicker/TransitionRunner.kt
@@ -0,0 +1,195 @@
+/*
+ * Copyright (C) 2021 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.android.server.wm.flicker
+
+import android.util.Log
+import com.android.server.wm.flicker.dsl.AssertionTag
+import com.android.server.wm.flicker.dsl.TestCommandsBuilder
+import com.android.server.wm.flicker.monitor.ITransitionMonitor
+import com.android.server.wm.traces.parser.DeviceStateDump
+import com.android.server.wm.traces.parser.getCurrentState
+import java.io.IOException
+import java.nio.file.Files
+
+/**
+ * Runner to execute the transitions of a flicker test
+ *
+ * The commands are executed in the following order:
+ * 1) [Flicker.testSetup]
+ * 2) [Flicker.runSetup
+ * 3) Start monitors
+ * 4) [Flicker.transitions]
+ * 5) Stop monitors
+ * 6) [Flicker.runTeardown]
+ * 7) [Flicker.testTeardown]
+ *
+ * If the tests were already executed, reuse the previous results
+ *
+ */
+open class TransitionRunner {
+ /**
+ * Iteration identifier during test run
+ */
+ private var iteration = 0
+ private val tags = mutableSetOf<String>()
+ private var tagsResults = mutableListOf<FlickerRunResult>()
+
+ /**
+ * Executes the setup, transitions and teardown defined in [flicker]
+ *
+ * @param flicker test specification
+ * @throws IllegalArgumentException If the transitions are empty or repetitions is set to 0
+ */
+ fun execute(flicker: Flicker): FlickerResult {
+ check(flicker)
+ return run(flicker)
+ }
+
+ /**
+ * Validate the [flicker] test specification before executing the transitions
+ *
+ * @param flicker test specification
+ * @throws IllegalArgumentException If the transitions are empty or repetitions is set to 0
+ */
+ private fun check(flicker: Flicker) {
+ require(flicker.transitions.isNotEmpty()) {
+ "A flicker test must include transitions to run" }
+ require(flicker.repetitions > 0) {
+ "Number of repetitions must be greater than 0" }
+ }
+
+ private fun internalCleanUp() {
+ tags.clear()
+ tagsResults.clear()
+ }
+
+ open fun cleanUp() {
+ internalCleanUp()
+ }
+
+ /**
+ * Runs the actual setup, transitions and teardown defined in [flicker]
+ *
+ * @param flicker test specification
+ */
+ internal open fun run(flicker: Flicker): FlickerResult {
+ val runs = mutableListOf<FlickerRunResult>()
+ var executionError: Throwable? = null
+ try {
+ try {
+ flicker.testSetup.forEach { it.invoke(flicker) }
+ for (iteration in 0 until flicker.repetitions) {
+ try {
+ flicker.runSetup.forEach { it.invoke(flicker) }
+ flicker.traceMonitors.forEach { it.start() }
+ flicker.frameStatsMonitor?.run { start() }
+ flicker.transitions.forEach { it.invoke(flicker) }
+ } finally {
+ flicker.traceMonitors.forEach { it.tryStop() }
+ flicker.frameStatsMonitor?.run { tryStop() }
+ flicker.runTeardown.forEach { it.invoke(flicker) }
+ }
+ if (flicker.frameStatsMonitor?.jankyFramesDetected() == true) {
+ Log.e(FLICKER_TAG, "Skipping iteration " +
+ "$iteration/${flicker.repetitions - 1} " +
+ "for test ${flicker.testName} due to jank. $flicker.frameStatsMonitor")
+ continue
+ }
+ val runResults = saveResult(flicker, iteration)
+ runs.addAll(runResults)
+ }
+ } finally {
+ flicker.testTeardown.forEach { it.invoke(flicker) }
+ }
+ } catch (e: Throwable) {
+ executionError = e
+ }
+
+ runs.addAll(tagsResults)
+ val result = FlickerResult(runs.toList(), tags.toSet(), executionError)
+ cleanUp()
+ return result
+ }
+
+ private fun saveResult(flicker: Flicker, iteration: Int): List<FlickerRunResult> {
+ val resultBuilder = FlickerRunResult.Builder(iteration)
+ flicker.traceMonitors.forEach {
+ it.save(flicker.testName, iteration, resultBuilder)
+ }
+
+ return resultBuilder.buildAll()
+ }
+
+ private fun ITransitionMonitor.tryStop() {
+ this.run {
+ try {
+ stop()
+ } catch (e: Exception) {
+ Log.e(FLICKER_TAG, "Unable to stop $this", e)
+ }
+ }
+ }
+
+ private fun getTaggedFilePath(flicker: Flicker, tag: String, file: String) =
+ "${flicker.testName}_${iteration}_${tag}_$file"
+
+ /**
+ * Captures a snapshot of the device state and associates it with a new tag.
+ *
+ * This tag can be used to make assertions about the state of the device when the
+ * snapshot is collected.
+ *
+ * [tag] is used as part of the trace file name, thus, only valid letters and digits
+ * can be used
+ *
+ * @throws IllegalArgumentException If [tag] contains invalid characters
+ */
+ fun createTag(flicker: Flicker, tag: String) {
+ require(!tag.contains(" ")) {
+ "The test tag $tag can not contain spaces since it is a part of the file name"
+ }
+ if (tag in tags) {
+ throw IllegalArgumentException("Tag $tag has already been used")
+ }
+ tags.add(tag)
+
+ val deviceStateBytes = getCurrentState(flicker.instrumentation.uiAutomation)
+ val deviceState = DeviceStateDump.fromDump(deviceStateBytes.first, deviceStateBytes.second)
+ try {
+ val wmTraceFile = flicker.outputDir.resolve(
+ getTaggedFilePath(flicker, tag, "wm_trace"))
+ Files.write(wmTraceFile, deviceStateBytes.first)
+
+ val layersTraceFile = flicker.outputDir.resolve(
+ getTaggedFilePath(flicker, tag, "layers_trace"))
+ Files.write(layersTraceFile, deviceStateBytes.second)
+
+ val builder = FlickerRunResult.Builder(iteration)
+ builder.wmTraceFile = wmTraceFile
+ builder.layersTraceFile = layersTraceFile
+
+ val result = builder.buildStateResult(
+ tag,
+ deviceState.wmTrace,
+ deviceState.layersTrace
+ )
+ tagsResults.add(result)
+ } catch (e: IOException) {
+ throw RuntimeException("Unable to create trace file: ${e.message}", e)
+ }
+ }
+}
\ No newline at end of file
diff --git a/libraries/flicker/src/com/android/server/wm/flicker/TransitionRunnerCached.kt b/libraries/flicker/src/com/android/server/wm/flicker/TransitionRunnerCached.kt
new file mode 100644
index 0000000..1d5216b
--- /dev/null
+++ b/libraries/flicker/src/com/android/server/wm/flicker/TransitionRunnerCached.kt
@@ -0,0 +1,48 @@
+/*
+ * Copyright (C) 2021 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.android.server.wm.flicker
+
+/**
+ * Execute the transitions of a flicker test and caches the results.
+ *
+ * Return cached results instead of re-executing the transitions if possible.
+ *
+ * @param runner Actual runner to execute the test
+ */
+class TransitionRunnerCached @JvmOverloads constructor(
+ private val runner: TransitionRunner = TransitionRunner()
+) : TransitionRunner() {
+ private var result = FlickerResult()
+
+ /**
+ * {@inheritDoc}
+ *
+ * @param flicker test specification
+ */
+ override fun run(flicker: Flicker): FlickerResult {
+ if (result.isEmpty()) {
+ result = runner.run(flicker)
+ }
+
+ return result
+ }
+
+ override fun cleanUp() {
+ result = FlickerResult()
+ runner.cleanUp()
+ }
+}
\ No newline at end of file
diff --git a/libraries/flicker/src/com/android/server/wm/flicker/assertions/FlickerAssertionError.kt b/libraries/flicker/src/com/android/server/wm/flicker/assertions/FlickerAssertionError.kt
index 9b483e8..07e6180 100644
--- a/libraries/flicker/src/com/android/server/wm/flicker/assertions/FlickerAssertionError.kt
+++ b/libraries/flicker/src/com/android/server/wm/flicker/assertions/FlickerAssertionError.kt
@@ -17,15 +17,21 @@
package com.android.server.wm.flicker.assertions
import com.android.server.wm.flicker.FlickerRunResult
+import com.android.server.wm.flicker.dsl.AssertionTag
+import com.android.server.wm.flicker.traces.FlickerSubjectException
import java.nio.file.Path
import kotlin.AssertionError
class FlickerAssertionError(
cause: Throwable,
- val assertion: AssertionData<*>,
- val run: FlickerRunResult,
- var trace: Path?
+ @JvmField val assertion: AssertionData,
+ @JvmField val iteration: Int,
+ @JvmField val assertionTag: String,
+ @JvmField val traceFiles: List<Path>
) : AssertionError(cause) {
+ constructor(cause: Throwable, assertion: AssertionData, run: FlickerRunResult)
+ : this(cause, assertion, run.iteration, run.assertionTag, run.traceFiles)
+
override val message: String
get() = buildString {
append("\n")
@@ -33,11 +39,19 @@
append(assertion.name)
append("\n")
append("Iteration: ")
- append(run.iteration)
+ append(iteration)
append("\n")
- append("Trace: ")
- append(trace)
+ append("Tag: ")
+ append(assertionTag)
append("\n")
- cause?.message?.let { append(it) }
+ // For subject exceptions, add the facts (layer/window/entry/etc)
+ // and the original cause of failure
+ if (cause is FlickerSubjectException) {
+ append(cause.facts)
+ append("\n")
+ cause.cause?.message?.let { append(it) }
+ } else {
+ cause?.message?.let { append(it) }
+ }
}
}
\ No newline at end of file
diff --git a/libraries/flicker/src/com/android/server/wm/flicker/assertions/FlickerSubject.kt b/libraries/flicker/src/com/android/server/wm/flicker/assertions/FlickerSubject.kt
index 6ffdbad..1928d38 100644
--- a/libraries/flicker/src/com/android/server/wm/flicker/assertions/FlickerSubject.kt
+++ b/libraries/flicker/src/com/android/server/wm/flicker/assertions/FlickerSubject.kt
@@ -62,7 +62,7 @@
* @param reason for the failure
*/
fun fail(reason: String): FlickerSubject = apply {
- fail(Fact.simpleFact(reason))
+ fail(Fact.fact("Reason", reason))
}
/**
diff --git a/libraries/flicker/src/com/android/server/wm/flicker/dsl/FlickerBuilder.kt b/libraries/flicker/src/com/android/server/wm/flicker/dsl/FlickerBuilder.kt
index cc3f287..c83f5a3 100644
--- a/libraries/flicker/src/com/android/server/wm/flicker/dsl/FlickerBuilder.kt
+++ b/libraries/flicker/src/com/android/server/wm/flicker/dsl/FlickerBuilder.kt
@@ -1,5 +1,5 @@
/*
- * Copyright (C) 2020 The Android Open Source Project
+ * Copyright (C) 2021 The Android Open Source Project
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@@ -22,6 +22,7 @@
import androidx.test.uiautomator.UiDevice
import com.android.server.wm.flicker.Flicker
import com.android.server.wm.flicker.FlickerDslMarker
+import com.android.server.wm.flicker.TransitionRunner
import com.android.server.wm.flicker.monitor.EventLogMonitor
import com.android.server.wm.flicker.getDefaultFlickerOutputDir
import com.android.server.wm.flicker.monitor.ITransitionMonitor
@@ -30,6 +31,10 @@
import com.android.server.wm.flicker.monitor.WindowAnimationFrameStatsMonitor
import com.android.server.wm.flicker.monitor.WindowManagerTraceMonitor
import com.android.server.wm.traces.common.layers.LayersTrace
+import com.android.server.wm.traces.common.layers.LayerTraceEntry
+import com.android.server.wm.traces.common.windowmanager.WindowManagerState
+import com.android.server.wm.traces.common.windowmanager.WindowManagerTrace
+import com.android.server.wm.traces.parser.windowmanager.WindowManagerStateHelper
import java.nio.file.Path
/**
@@ -41,12 +46,13 @@
private val launcherStrategy: ILauncherStrategy,
private val includeJankyRuns: Boolean,
private val outputDir: Path,
+ private val wmHelper: WindowManagerStateHelper,
private var testName: String,
private var iterations: Int,
- private val setupCommands: TestCommands,
- private val teardownCommands: TestCommands,
+ private val setupCommands: TestCommandsBuilder,
+ private val teardownCommands: TestCommandsBuilder,
private val transitionCommands: MutableList<Flicker.() -> Any>,
- private val assertions: AssertionTarget,
+ internal val assertions: AssertionTargetBuilder,
val device: UiDevice,
private val traceMonitors: MutableList<ITransitionMonitor>
) {
@@ -77,24 +83,29 @@
/**
* Output directory for the test results
*/
- outputDir: Path = getDefaultFlickerOutputDir()
+ outputDir: Path = getDefaultFlickerOutputDir(),
+ /**
+ * Helper object for WM Synchronization
+ */
+ wmHelper: WindowManagerStateHelper = WindowManagerStateHelper()
) : this(
instrumentation,
launcherStrategy,
includeJankyRuns,
outputDir,
+ wmHelper,
testName = "",
iterations = 1,
- setupCommands = TestCommands(),
- teardownCommands = TestCommands(),
+ setupCommands = TestCommandsBuilder(),
+ teardownCommands = TestCommandsBuilder(),
transitionCommands = mutableListOf(),
- assertions = AssertionTarget(),
+ assertions = AssertionTargetBuilder(),
device = UiDevice.getInstance(instrumentation),
traceMonitors = mutableListOf<ITransitionMonitor>()
.also {
it.add(WindowManagerTraceMonitor(outputDir))
it.add(LayersTraceMonitor(outputDir))
- it.add(ScreenRecorder(outputDir))
+ it.add(ScreenRecorder(outputDir, instrumentation.targetContext))
it.add(EventLogMonitor())
}
)
@@ -107,12 +118,13 @@
otherBuilder.launcherStrategy,
otherBuilder.includeJankyRuns,
otherBuilder.outputDir.toAbsolutePath(),
+ otherBuilder.wmHelper,
otherBuilder.testName,
otherBuilder.iterations,
- TestCommands(otherBuilder.setupCommands),
- TestCommands(otherBuilder.teardownCommands),
+ TestCommandsBuilder(otherBuilder.setupCommands),
+ TestCommandsBuilder(otherBuilder.teardownCommands),
otherBuilder.transitionCommands.toMutableList(),
- AssertionTarget(otherBuilder.assertions),
+ AssertionTargetBuilder(otherBuilder.assertions),
UiDevice.getInstance(otherBuilder.instrumentation),
otherBuilder.traceMonitors.toMutableList()
)
@@ -138,8 +150,8 @@
*
* By default the tracing is always active. To disable tracing return null
*
- * If this tracing is disabled, the assertions for [AssertionTarget.layerAssertions] will
- * not be executed
+ * If this tracing is disabled, the assertions for [WindowManagerTrace] and
+ * [WindowManagerState] will not be executed
*/
fun withWindowManagerTracing(traceMonitor: (Path) -> WindowManagerTraceMonitor?) {
traceMonitors.removeIf { it is WindowManagerTraceMonitor }
@@ -155,8 +167,8 @@
*
* By default the tracing is always active. To disable tracing return null
*
- * If this tracing is disabled, the assertions for [AssertionTarget.layerAssertions] will
- * not be executed
+ * If this tracing is disabled, the assertions for [LayersTrace] and [LayerTraceEntry]
+ * will not be executed
*/
fun withLayerTracing(traceMonitor: (Path) -> LayersTraceMonitor?) {
traceMonitors.removeIf { it is LayersTraceMonitor }
@@ -191,39 +203,40 @@
}
/**
- * Defines the test ([TestCommands.testCommands]) and run ([TestCommands.runCommands])
+ * Defines the test ([TestCommandsBuilder.testCommands]) and run ([TestCommandsBuilder.runCommands])
* commands executed before the [transitions] to test
*/
- fun setup(commands: TestCommands.() -> Unit) {
+ fun setup(commands: TestCommandsBuilder.() -> Unit) {
setupCommands.apply { commands() }
}
/**
- * Defines the test ([TestCommands.testCommands]) and run ([TestCommands.runCommands])
+ * Defines the test ([TestCommandsBuilder.testCommands]) and run ([TestCommandsBuilder.runCommands])
* commands executed after the [transitions] to test
*/
- fun teardown(commands: TestCommands.() -> Unit) {
+ fun teardown(commands: TestCommandsBuilder.() -> Unit) {
teardownCommands.apply { commands() }
}
/**
* Defines the commands that trigger the behavior to test
*/
- fun transitions(command: Flicker.() -> Any) {
+ fun transitions(command: Flicker.() -> Unit) {
transitionCommands.add(command)
}
/**
* Defines the assertions to check the recorded traces
*/
- fun assertions(assertion: AssertionTarget.() -> Unit) {
+ fun assertions(assertion: AssertionTargetBuilder.() -> Unit) {
assertions.apply { assertion() }
}
/**
* Creates a new Flicker runner based on the current builder configuration
*/
- fun build() = Flicker(
+ @JvmOverloads
+ fun build(runner: TransitionRunner = TransitionRunner()) = Flicker(
instrumentation,
device,
launcherStrategy,
@@ -232,10 +245,14 @@
iterations,
frameStatsMonitor,
traceMonitors,
- setupCommands,
- teardownCommands,
+ setupCommands.buildTestCommands(),
+ setupCommands.buildRunCommands(),
+ teardownCommands.buildTestCommands(),
+ teardownCommands.buildRunCommands(),
transitionCommands,
- assertions
+ assertions.build(),
+ runner,
+ wmHelper
)
/**
@@ -261,7 +278,7 @@
launcherStrategy: ILauncherStrategy = LauncherStrategyFactory
.getInstance(instrumentation).launcherStrategy,
configuration: FlickerBuilder.() -> Unit
-) = runFlicker(instrumentation, launcherStrategy, configuration)
+) = runFlicker(instrumentation, launcherStrategy, configuration = configuration)
/**
* Entry point for the Flicker DSL.
@@ -269,6 +286,8 @@
* Configures a builder, build the test runs, executes them and checks the configured assertions
*
* @param instrumentation to run the test (used to interact with the device)
+ * @param launcherStrategy to interact with the device's launcher
+ * @param runner to execute the transitions
* @param configuration Flicker DSL configuration
*/
@JvmOverloads
@@ -276,10 +295,11 @@
instrumentation: Instrumentation,
launcherStrategy: ILauncherStrategy = LauncherStrategyFactory
.getInstance(instrumentation).launcherStrategy,
+ runner: TransitionRunner = TransitionRunner(),
configuration: FlickerBuilder.() -> Unit
) {
val builder = FlickerBuilder(instrumentation, launcherStrategy)
- runWithFlicker(builder, configuration)
+ runWithFlicker(builder, runner, configuration)
}
/**
@@ -291,12 +311,14 @@
* The original builder object is not changed.
*
* @param builder to run the test (used to interact with the device
+ * @param runner to execute the transitions
* @param configuration Flicker DSL configuration
*/
@JvmOverloads
fun runWithFlicker(
builder: FlickerBuilder,
+ runner: TransitionRunner = TransitionRunner(),
configuration: FlickerBuilder.() -> Unit = {}
) {
- builder.copy(configuration).build().execute().checkAssertions()
+ builder.copy(configuration).build(runner).execute().checkAssertions()
}
\ No newline at end of file
diff --git a/libraries/flicker/src/com/android/server/wm/flicker/dsl/FlickerBuilderJava.java b/libraries/flicker/src/com/android/server/wm/flicker/dsl/FlickerBuilderJava.java
index 5e3a40c..36b91d9 100644
--- a/libraries/flicker/src/com/android/server/wm/flicker/dsl/FlickerBuilderJava.java
+++ b/libraries/flicker/src/com/android/server/wm/flicker/dsl/FlickerBuilderJava.java
@@ -1,5 +1,5 @@
/*
- * Copyright (C) 2020 The Android Open Source Project
+ * Copyright (C) 2021 The Android Open Source Project
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@@ -25,13 +25,17 @@
import androidx.test.uiautomator.UiDevice;
import com.android.server.wm.flicker.Flicker;
+import com.android.server.wm.flicker.TransitionRunner;
import com.android.server.wm.flicker.monitor.ITransitionMonitor;
import com.android.server.wm.flicker.monitor.LayersTraceMonitor;
import com.android.server.wm.flicker.monitor.ScreenRecorder;
import com.android.server.wm.flicker.monitor.WindowAnimationFrameStatsMonitor;
import com.android.server.wm.flicker.monitor.WindowManagerTraceMonitor;
+import com.android.server.wm.flicker.traces.layers.LayerTraceEntrySubject;
import com.android.server.wm.flicker.traces.layers.LayersTraceSubject;
-import com.android.server.wm.flicker.traces.windowmanager.WmTraceSubject;
+import com.android.server.wm.flicker.traces.windowmanager.WindowManagerStateSubject;
+import com.android.server.wm.flicker.traces.windowmanager.WindowManagerTraceSubject;
+import com.android.server.wm.traces.parser.windowmanager.WindowManagerStateHelper;
import org.junit.Assert;
@@ -127,10 +131,10 @@
private String testTag = "";
private Integer iterations = 1;
- private TestCommands setupCommands = new TestCommands();
- private TestCommands teardownCommands = new TestCommands();
+ private TestCommandsBuilder setupCommands = new TestCommandsBuilder();
+ private TestCommandsBuilder teardownCommands = new TestCommandsBuilder();
private List<Function1<? super Flicker, ?>> transitionCommands = new ArrayList<>();
- private AssertionTarget assertions = new AssertionTarget();
+ private AssertionTargetBuilder assertions = new AssertionTargetBuilder();
private List<ITransitionMonitor> traceMonitors = new ArrayList<>();
private WindowAnimationFrameStatsMonitor frameStatsMonitor;
@@ -264,7 +268,7 @@
/** Defines the assertions for the initial entry of the layers trace */
public FlickerBuilderJava addLayerTraceAssertionStart(
- FlickerAssertionJava<LayersTraceSubject> assertion) {
+ FlickerAssertionJava<LayerTraceEntrySubject> assertion) {
assertions.layersTrace(
assertionData -> {
assertionData.start(assertion::invoke);
@@ -275,7 +279,7 @@
/** Defines the assertions for the final entry of the layers trace */
public FlickerBuilderJava addLayerTraceAssertionEnd(
- FlickerAssertionJava<LayersTraceSubject> assertion) {
+ FlickerAssertionJava<LayerTraceEntrySubject> assertion) {
assertions.layersTrace(
assertionData -> {
assertionData.end(assertion::invoke);
@@ -297,7 +301,7 @@
/** Defines the assertions for the initial entry of the layers trace */
public FlickerBuilderJava addWindowManagerTraceAssertionStart(
- FlickerAssertionJava<WmTraceSubject> assertion) {
+ FlickerAssertionJava<WindowManagerStateSubject> assertion) {
assertions.windowManagerTrace(
assertionData -> {
assertionData.start(assertion::invoke);
@@ -308,7 +312,7 @@
/** Defines the assertions for the final entry of the layers trace */
public FlickerBuilderJava addWindowManagerTraceAssertionEnd(
- FlickerAssertionJava<WmTraceSubject> assertion) {
+ FlickerAssertionJava<WindowManagerStateSubject> assertion) {
assertions.windowManagerTrace(
assertionData -> {
assertionData.end(assertion::invoke);
@@ -319,7 +323,7 @@
/** Defines the assertions for all entries of the layers trace */
public FlickerBuilderJava addWindowManagerTraceAssertionAll(
- FlickerAssertionJava<WmTraceSubject> assertion) {
+ FlickerAssertionJava<WindowManagerTraceSubject> assertion) {
assertions.windowManagerTrace(
assertionData -> {
assertionData.all(assertion::invoke);
@@ -339,9 +343,13 @@
iterations,
frameStatsMonitor,
traceMonitors,
- setupCommands,
- teardownCommands,
+ setupCommands.buildTestCommands(),
+ setupCommands.buildRunCommands(),
+ teardownCommands.buildTestCommands(),
+ teardownCommands.buildRunCommands(),
transitionCommands,
- assertions);
+ assertions.build(),
+ new TransitionRunner(),
+ new WindowManagerStateHelper());
}
}
diff --git a/libraries/flicker/src/com/android/server/wm/flicker/dsl/TestCommands.kt b/libraries/flicker/src/com/android/server/wm/flicker/dsl/TestCommandsBuilder.kt
similarity index 83%
rename from libraries/flicker/src/com/android/server/wm/flicker/dsl/TestCommands.kt
rename to libraries/flicker/src/com/android/server/wm/flicker/dsl/TestCommandsBuilder.kt
index c05d1b2..e12ab64 100644
--- a/libraries/flicker/src/com/android/server/wm/flicker/dsl/TestCommands.kt
+++ b/libraries/flicker/src/com/android/server/wm/flicker/dsl/TestCommandsBuilder.kt
@@ -23,9 +23,9 @@
* Placeholder for test [Flicker.setup] and [Flicker.teardown] commands on the Flicker DSL
*/
@FlickerDslMarker
-class TestCommands private constructor(
- internal val testCommands: MutableList<Flicker.() -> Any>,
- internal val runCommands: MutableList<Flicker.() -> Any>
+class TestCommandsBuilder private constructor(
+ private val testCommands: MutableList<Flicker.() -> Any>,
+ private val runCommands: MutableList<Flicker.() -> Any>
) {
constructor() : this(
testCommands = mutableListOf<Flicker.() -> Any>(),
@@ -35,7 +35,7 @@
/**
* Copy constructor
*/
- constructor(otherCommands: TestCommands) : this(
+ constructor(otherCommands: TestCommandsBuilder) : this(
otherCommands.testCommands.toMutableList(),
otherCommands.runCommands.toMutableList()
)
@@ -51,7 +51,7 @@
*
* This command can be used multiple times, and the results are appended
*/
- fun test(command: Flicker.() -> Any) {
+ fun test(command: Flicker.() -> Unit) {
testCommands.add(command)
}
@@ -68,7 +68,11 @@
*
* This command can be used multiple times, and the results are appended
*/
- fun eachRun(command: Flicker.() -> Any) {
+ fun eachRun(command: Flicker.() -> Unit) {
runCommands.add(command)
}
+
+ fun buildTestCommands(): List<Flicker.() -> Any> = testCommands
+
+ fun buildRunCommands(): List<Flicker.() -> Any> = runCommands
}
\ No newline at end of file
diff --git a/libraries/flicker/src/com/android/server/wm/flicker/helpers/StandardAppHelper.kt b/libraries/flicker/src/com/android/server/wm/flicker/helpers/StandardAppHelper.kt
index 3dd26c7..4f1dc6c 100644
--- a/libraries/flicker/src/com/android/server/wm/flicker/helpers/StandardAppHelper.kt
+++ b/libraries/flicker/src/com/android/server/wm/flicker/helpers/StandardAppHelper.kt
@@ -18,9 +18,19 @@
import android.app.ActivityManager
import android.app.Instrumentation
+import android.content.ComponentName
+import android.content.Context
+import android.content.Intent
import android.platform.helpers.AbstractStandardAppHelper
import android.support.test.launcherhelper.ILauncherStrategy
import android.support.test.launcherhelper.LauncherStrategyFactory
+import androidx.test.uiautomator.By
+import androidx.test.uiautomator.BySelector
+import androidx.test.uiautomator.UiDevice
+import androidx.test.uiautomator.Until
+import com.android.server.wm.traces.parser.toActivityName
+import com.android.server.wm.traces.parser.toWindowName
+import com.android.server.wm.traces.parser.windowmanager.WindowManagerStateHelper
/**
* Class to take advantage of {@code IAppHelper} interface so the same test can be run against first
@@ -28,28 +38,57 @@
*/
open class StandardAppHelper @JvmOverloads constructor(
instr: Instrumentation,
- protected val packageName: String,
- protected val appName: String,
+ @JvmField val appName: String,
+ @JvmField val component: ComponentName,
protected val launcherStrategy: ILauncherStrategy =
- LauncherStrategyFactory.getInstance(instr).launcherStrategy
+ LauncherStrategyFactory.getInstance(instr).launcherStrategy
) : AbstractStandardAppHelper(instr) {
constructor(
instr: Instrumentation,
appName: String,
+ packageName: String,
+ activity: String,
launcherStrategy: ILauncherStrategy =
- LauncherStrategyFactory.getInstance(instr).launcherStrategy
- ) : this(instr, sFlickerPackage, appName, launcherStrategy)
+ LauncherStrategyFactory.getInstance(instr).launcherStrategy
+ ): this(instr, appName,
+ ComponentName.createRelative(packageName, ".$activity"), launcherStrategy)
+
+ val windowName: String = component.toWindowName()
+ val activityName: String = component.toActivityName()
private val activityManager: ActivityManager?
get() = mInstrumentation.context.getSystemService(ActivityManager::class.java)
+ protected val context: Context
+ get() = mInstrumentation.context
+
+ protected val uiDevice: UiDevice = UiDevice.getInstance(mInstrumentation)
+
+ private fun getAppSelector(expectedPackageName: String): BySelector {
+ val expected = if (expectedPackageName.isNotEmpty()) {
+ expectedPackageName
+ } else {
+ component.packageName
+ }
+ return By.pkg(expected).depth(0)
+ }
+
override fun open() {
- launcherStrategy.launch(appName, packageName)
+ launcherStrategy.launch(appName, component.packageName)
}
/** {@inheritDoc} */
override fun getPackage(): String {
- return packageName
+ return component.packageName
+ }
+
+ /** {@inheritDoc} */
+ override fun getOpenAppIntent(): Intent {
+ val intent = Intent()
+ intent.addCategory(Intent.CATEGORY_LAUNCHER)
+ intent.addFlags(Intent.FLAG_ACTIVITY_NEW_TASK)
+ intent.component = component
+ return intent
}
/** {@inheritDoc} */
@@ -65,10 +104,63 @@
super.exit()
// Ensure all testing components end up being closed.
- activityManager?.forceStopPackage(packageName)
+ activityManager?.forceStopPackage(component.packageName)
+ }
+
+ private fun launchAppViaIntent(
+ action: String? = null,
+ stringExtras: Map<String, String> = mapOf()
+ ) {
+ val intent = openAppIntent
+ intent.action = action
+ stringExtras.forEach {
+ intent.putExtra(it.key, it.value)
+ }
+ context.startActivity(intent)
+ }
+
+ /**
+ * Launches the app through an intent instead of interacting with the launcher.
+ *
+ * Uses UiAutomation to detect when the app is open
+ */
+ @JvmOverloads
+ fun launchViaIntent(
+ expectedPackageName: String = "",
+ action: String? = null,
+ stringExtras: Map<String, String> = mapOf()
+ ) {
+ launchAppViaIntent(action, stringExtras)
+ val appSelector = getAppSelector(expectedPackageName)
+ uiDevice.wait(Until.hasObject(appSelector), APP_LAUNCH_WAIT_TIME_MS)
+ }
+
+ /**
+ * Launches the app through an intent instead of interacting with the launcher and waits
+ * until the app window is visible
+ */
+ @JvmOverloads
+ fun launchViaIntent(
+ wmHelper: WindowManagerStateHelper,
+ expectedWindowName: String = "",
+ action: String? = null,
+ stringExtras: Map<String, String> = mapOf()
+ ) {
+ launchAppViaIntent(action, stringExtras)
+
+ val window = if (expectedWindowName.isNotEmpty()) {
+ expectedWindowName
+ } else {
+ windowName
+ }
+ wmHelper.waitFor("App is shown") {
+ it.wmState.isComplete() && it.wmState.isWindowVisible(window)
+ }
+ wmHelper.waitForNavBarStatusBarVisible()
+ wmHelper.waitForAppTransitionIdle()
}
companion object {
- private val sFlickerPackage = "com.android.server.wm.flicker.testapp"
+ private const val APP_LAUNCH_WAIT_TIME_MS = 10000L
}
}
diff --git a/libraries/flicker/src/com/android/server/wm/traces/parser/DeviceStateDump.kt b/libraries/flicker/src/com/android/server/wm/traces/parser/DeviceStateDump.kt
index 338c604..6534b31 100644
--- a/libraries/flicker/src/com/android/server/wm/traces/parser/DeviceStateDump.kt
+++ b/libraries/flicker/src/com/android/server/wm/traces/parser/DeviceStateDump.kt
@@ -29,31 +29,14 @@
*/
class DeviceStateDump(
/**
- * [WindowManagerTrace] content
- */
- val wmTraceData: ByteArray,
- /**
- * [LayersTrace] content
- */
- val layersTraceData: ByteArray,
- /**
- * Predicate to parse [wmTraceData] into a [WindowManagerTrace]
- */
- val wmTraceParser: (ByteArray) -> WindowManagerTrace?,
- /**
- * Predicate to parse [layersTraceData] into a [LayersTrace]
- */
- val layersTraceParser: (ByteArray) -> LayersTrace?
-) {
- /**
* Parsed [WindowManagerTrace]
*/
- val wmTrace: WindowManagerTrace? by lazy { wmTraceParser(wmTraceData) }
+ val wmTrace: WindowManagerTrace?,
/**
* Parsed [LayersTrace]
*/
- val layersTrace: LayersTrace? by lazy { layersTraceParser(layersTraceData) }
-
+ val layersTrace: LayersTrace?
+) {
companion object {
/**
* Creates a device state dump containing the [WindowManagerTrace] and [LayersTrace]
@@ -66,21 +49,15 @@
@JvmStatic
fun fromDump(wmTraceData: ByteArray, layersTraceData: ByteArray): DeviceStateDump {
return DeviceStateDump(
- wmTraceData,
- layersTraceData,
- {
- if (wmTraceData.isNotEmpty()) {
- WindowManagerTraceParser.parseFromDump(wmTraceData)
- } else {
- null
- }
+ wmTrace = if (wmTraceData.isNotEmpty()) {
+ WindowManagerTraceParser.parseFromDump(wmTraceData)
+ } else {
+ null
},
- {
- if (layersTraceData.isNotEmpty()) {
- LayersTraceParser.parseFromDump(layersTraceData)
- } else {
- null
- }
+ layersTrace = if (layersTraceData.isNotEmpty()) {
+ LayersTraceParser.parseFromDump(layersTraceData)
+ } else {
+ null
}
)
}
@@ -96,21 +73,15 @@
@JvmStatic
fun fromTrace(wmTraceData: ByteArray, layersTraceData: ByteArray): DeviceStateDump {
return DeviceStateDump(
- wmTraceData,
- layersTraceData,
- {
- if (wmTraceData.isNotEmpty()) {
- WindowManagerTraceParser.parseFromTrace(wmTraceData)
- } else {
- null
- }
+ wmTrace = if (wmTraceData.isNotEmpty()) {
+ WindowManagerTraceParser.parseFromTrace(wmTraceData)
+ } else {
+ null
},
- {
- if (layersTraceData.isNotEmpty()) {
- LayersTraceParser.parseFromTrace(layersTraceData)
- } else {
- null
- }
+ layersTrace = if (layersTraceData.isNotEmpty()) {
+ LayersTraceParser.parseFromTrace(layersTraceData)
+ } else {
+ null
}
)
}
diff --git a/libraries/flicker/src/com/android/server/wm/traces/parser/Extensions.kt b/libraries/flicker/src/com/android/server/wm/traces/parser/Extensions.kt
index 629d13a..3da3afa 100644
--- a/libraries/flicker/src/com/android/server/wm/traces/parser/Extensions.kt
+++ b/libraries/flicker/src/com/android/server/wm/traces/parser/Extensions.kt
@@ -56,7 +56,7 @@
fun getCurrentState(
uiAutomation: UiAutomation,
@WmStateDumpFlags dumpFlags: Int = FLAG_STATE_DUMP_FLAG_WM.or(FLAG_STATE_DUMP_FLAG_LAYERS)
-): DeviceStateDump {
+): Pair<ByteArray, ByteArray> {
if (dumpFlags == 0) {
throw IllegalArgumentException("No dump specified")
}
@@ -72,5 +72,17 @@
} else {
ByteArray(0)
}
+
+ return Pair(wmTraceData, layersTraceData)
+}
+
+@JvmOverloads
+fun getCurrentStateDump(
+ uiAutomation: UiAutomation,
+ @WmStateDumpFlags dumpFlags: Int = FLAG_STATE_DUMP_FLAG_WM.or(FLAG_STATE_DUMP_FLAG_LAYERS)
+): DeviceStateDump {
+ val currentStateDump = getCurrentState(uiAutomation, dumpFlags)
+ val wmTraceData = currentStateDump.first
+ val layersTraceData = currentStateDump.second
return DeviceStateDump.fromDump(wmTraceData, layersTraceData)
}
diff --git a/libraries/flicker/test/src/com/android/server/wm/flicker/FlickerDSLTest.kt b/libraries/flicker/test/src/com/android/server/wm/flicker/FlickerDSLTest.kt
index 8590371..2db00d1 100644
--- a/libraries/flicker/test/src/com/android/server/wm/flicker/FlickerDSLTest.kt
+++ b/libraries/flicker/test/src/com/android/server/wm/flicker/FlickerDSLTest.kt
@@ -17,10 +17,9 @@
package com.android.server.wm.flicker
import androidx.test.platform.app.InstrumentationRegistry
+import com.android.server.wm.flicker.dsl.AssertionTargetBuilder
import com.android.server.wm.flicker.dsl.FlickerBuilder
-import com.android.server.wm.flicker.dsl.WmAssertionBuilder
import com.android.server.wm.flicker.dsl.runWithFlicker
-import com.android.server.wm.flicker.traces.windowmanager.WindowManagerTraceSubject
import com.google.common.truth.Truth
import org.junit.Assert
import org.junit.FixMethodOrder
@@ -75,12 +74,6 @@
}
}
- private fun defaultAssertion(trace: WindowManagerTraceSubject): WindowManagerTraceSubject {
- return trace("Has dump") {
- it.isNotEmpty()
- }
- }
-
@Test
fun assertCreatedTags() {
val builder = FlickerBuilder(instrumentation)
@@ -93,11 +86,17 @@
}
assertions {
windowManagerTrace {
- tag(myTag) { defaultAssertion(this) }
+ tag(myTag) {
+ this.isNotEmpty()
+ }
- start { defaultAssertion(this) }
+ start {
+ this.isNotEmpty()
+ }
- end { defaultAssertion(this) }
+ end {
+ this.isNotEmpty()
+ }
tag("invalid") {
fail("`Invalid` tag was not created, so it should not " +
@@ -115,7 +114,9 @@
runWithFlicker(builder) {
assertions {
windowManagerTrace {
- tag("tag") { defaultAssertion(this) }
+ tag("tag") {
+ this.isNotEmpty()
+ }
}
}
}
@@ -146,7 +147,7 @@
}
}
- private fun detectFailedAssertion(assertion: WmAssertionBuilder.() -> Any): Throwable {
+ private fun detectFailedAssertion(assertions: AssertionTargetBuilder.() -> Any): Throwable {
val builder = FlickerBuilder(instrumentation)
return assertThrows(AssertionError::class.java) {
runWithFlicker(builder) {
@@ -154,23 +155,18 @@
device.pressHome()
}
assertions {
- windowManagerTrace {
- assertion()
- }
+ assertions()
}
}
}
}
@Test
- fun detectFailedAssertion_All() {
+ fun detectFailedWMAssertion_All() {
val error = detectFailedAssertion {
- all("fail") {
- fail("Correct error")
- }
-
- all("ignored", enabled = false) {
- fail("Ignored error")
+ windowManagerTrace {
+ all("fail") { fail("Correct error") }
+ all("ignored", enabled = false) { fail("Ignored error") }
}
}
assertFailure(error).hasMessageThat().contains("Correct error")
@@ -178,14 +174,11 @@
}
@Test
- fun detectFailedAssertion_Start() {
+ fun detectFailedWMAssertion_Start() {
val error = detectFailedAssertion {
- start("fail") {
- fail("Correct error")
- }
-
- start("ignored", enabled = false) {
- fail("Ignored error")
+ windowManagerTrace {
+ start("fail") { fail("Correct error") }
+ start("ignored", enabled = false) { fail("Ignored error") }
}
}
assertFailure(error).hasMessageThat().contains("Correct error")
@@ -193,14 +186,59 @@
}
@Test
- fun detectFailedAssertion_End() {
+ fun detectFailedWMAssertion_End() {
val error = detectFailedAssertion {
- end("fail") {
- fail("Correct error")
+ windowManagerTrace {
+ end("fail") { fail("Correct error") }
+ end("ignored", enabled = false) { fail("Ignored error") }
}
+ }
+ assertFailure(error).hasMessageThat().contains("Correct error")
+ assertFailure(error).hasMessageThat().doesNotContain("Ignored error")
+ }
- end("ignored", enabled = false) {
- fail("Ignored error")
+ @Test
+ fun detectFailedLayersAssertion_All() {
+ val error = detectFailedAssertion {
+ layersTrace {
+ all("fail") { fail("Correct error") }
+ all("ignored", enabled = false) { fail("Ignored error") }
+ }
+ }
+ assertFailure(error).hasMessageThat().contains("Correct error")
+ assertFailure(error).hasMessageThat().doesNotContain("Ignored error")
+ }
+
+ @Test
+ fun detectFailedLayersAssertion_Start() {
+ val error = detectFailedAssertion {
+ layersTrace {
+ start("fail") { fail("Correct error") }
+ start("ignored", enabled = false) { fail("Ignored error") }
+ }
+ }
+ assertFailure(error).hasMessageThat().contains("Correct error")
+ assertFailure(error).hasMessageThat().doesNotContain("Ignored error")
+ }
+
+ @Test
+ fun detectFailedLayersAssertion_End() {
+ val error = detectFailedAssertion {
+ layersTrace {
+ end("fail") { fail("Correct error") }
+ end("ignored", enabled = false) { fail("Ignored error") }
+ }
+ }
+ assertFailure(error).hasMessageThat().contains("Correct error")
+ assertFailure(error).hasMessageThat().doesNotContain("Ignored error")
+ }
+
+ @Test
+ fun detectFailedEventLogAssertion_All() {
+ val error = detectFailedAssertion {
+ eventLog {
+ all("fail") { fail("Correct error") }
+ all("ignored", enabled = false) { fail("Ignored error") }
}
}
assertFailure(error).hasMessageThat().contains("Correct error")
diff --git a/libraries/flicker/test/src/com/android/server/wm/flicker/FlickerTestFactoryRunnerTest.kt b/libraries/flicker/test/src/com/android/server/wm/flicker/FlickerTestFactoryRunnerTest.kt
index 38eb181..3684fee 100644
--- a/libraries/flicker/test/src/com/android/server/wm/flicker/FlickerTestFactoryRunnerTest.kt
+++ b/libraries/flicker/test/src/com/android/server/wm/flicker/FlickerTestFactoryRunnerTest.kt
@@ -1,5 +1,5 @@
/*
- * Copyright (C) 2020 The Android Open Source Project
+ * Copyright (C) 2021 The Android Open Source Project
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@@ -19,6 +19,7 @@
import android.os.Bundle
import android.view.Surface
import androidx.test.platform.app.InstrumentationRegistry
+import com.android.server.wm.flicker.dsl.FlickerBuilder
import com.google.common.truth.Truth.assertWithMessage
import org.junit.FixMethodOrder
import org.junit.Test
@@ -27,13 +28,19 @@
/**
* Contains [FlickerTestRunnerFactory] tests.
*
- * To run this test: `atest FlickerLibTest:FlickerTestFacroty`
+ * To run this test: `atest FlickerLibTest:FlickerTestFactoryRunnerTest`
*/
@FixMethodOrder(MethodSorters.NAME_ASCENDING)
class FlickerTestFactoryRunnerTest {
private val instrumentation = InstrumentationRegistry.getInstrumentation()
private val defaultRotations = listOf(Surface.ROTATION_0, Surface.ROTATION_90)
+ private fun FlickerBuilder.setDefaultTestCfg() = apply {
+ assertions {
+ layersTrace { all { fail("First assertion") } }
+ }
+ }
+
private fun validateRotationTest(actual: Bundle, rotations: List<Int> = defaultRotations) {
assertWithMessage("Rotation tests should not have the same start and end rotation")
.that(actual.startRotation).isNotEqualTo(actual.endRotation)
@@ -53,17 +60,25 @@
@Test
fun checkBuildTest() {
val factory = FlickerTestRunnerFactory(instrumentation)
- val actual = factory.buildTest { cfg -> validateTest(cfg) }
+ val actual = factory.buildTest { cfg ->
+ this.setDefaultTestCfg()
+ validateTest(cfg)
+ }
+ // Should have 1 test for transition and 1 for the assertions in each orientation
assertWithMessage("Flicker should create tests for 0 and 90 degrees")
- .that(actual).hasSize(2)
+ .that(actual).hasSize(4)
}
@Test
fun checkBuildRotationTest() {
val factory = FlickerTestRunnerFactory(instrumentation)
- val actual = factory.buildRotationTest { cfg -> validateRotationTest(cfg) }
+ val actual = factory.buildRotationTest { cfg ->
+ this.setDefaultTestCfg()
+ validateRotationTest(cfg)
+ }
+ // Should have 1 test for transition and 1 for the assertions in each orientation
assertWithMessage("Flicker should create tests for 0 and 90 degrees")
- .that(actual).hasSize(2)
+ .that(actual).hasSize(4)
}
@Test
@@ -71,9 +86,13 @@
val rotations = listOf(Surface.ROTATION_0, Surface.ROTATION_90, Surface.ROTATION_180,
Surface.ROTATION_270)
val factory = FlickerTestRunnerFactory(instrumentation, rotations)
- val actual = factory.buildRotationTest { cfg -> validateRotationTest(cfg, rotations) }
+ val actual = factory.buildRotationTest { cfg ->
+ this.setDefaultTestCfg()
+ validateRotationTest(cfg, rotations)
+ }
+ // Should have 1 test for transition and 1 for the assertions in each rotation
assertWithMessage("Flicker should create tests for 0/90/180/270 degrees")
- .that(actual).hasSize(12)
+ .that(actual).hasSize(24)
}
@Test
@@ -81,11 +100,97 @@
val factory = FlickerTestRunnerFactory(instrumentation)
val actual = listOf(Bundle().also { it.putBoolean("test", true) })
val tests = factory.buildTest(actual) { cfg ->
+ this.setDefaultTestCfg()
validateTest(cfg)
assertWithMessage("Could not find custom payload data")
.that(cfg.getBoolean("test", false)).isTrue()
}
- assertWithMessage("Flicker should create tests for 0 and 90 degrees")
- .that(tests).hasSize(1)
+ // Should have 1 test for transition and 1 for the assertions in each orientation
+ assertWithMessage("Flicker should create 1 test for transition and 1 for assertion")
+ .that(tests).hasSize(2)
+ }
+
+ private fun assertIsEmpty(producer: () -> Flicker) {
+ val spec = producer.invoke()
+ assertWithMessage("Should not have assertions")
+ .that(spec.assertions)
+ .isEmpty()
+ }
+
+ private fun assertHasSingleAssertion(producer: () -> Flicker) {
+ val spec = producer.invoke()
+ assertWithMessage("Should have 1 assertion")
+ .that(spec.assertions)
+ .hasSize(1)
+ }
+
+ @Test
+ fun checkBuildOneTestPerAssertion() {
+ val factory = FlickerTestRunnerFactory(instrumentation,
+ supportedRotations = listOf(Surface.ROTATION_0))
+ val tests = factory.buildTest {
+ assertions {
+ layersTrace { all { fail("First assertion") } }
+ windowManagerTrace { all { fail("Second assertion") } }
+ eventLog { all { fail("This assertion") } }
+ }
+ }
+
+ assertWithMessage("Factory should have created 4 tests, one for transition and " +
+ "3 with a single assertion each")
+ .that(tests)
+ .hasSize(4)
+
+ assertIsEmpty(tests.first()[1] as () -> Flicker)
+ tests.drop(1).forEach { (_, producer, _) ->
+ assertHasSingleAssertion(producer as () -> Flicker)
+ }
+ }
+
+ @Test
+ fun checkCleanUp() {
+ val factory = FlickerTestRunnerFactory(instrumentation)
+ val actual = factory.buildTest { cfg ->
+ this.setDefaultTestCfg()
+ validateTest(cfg)
+ }
+
+ actual.forEachIndexed { index, entry ->
+ val expectedCleanUp = index % 2 > 0
+ val actualCleanUp = entry[2]
+ val specProducer = entry[1] as () -> Flicker
+ val spec = specProducer.invoke()
+
+ assertWithMessage("Entry $index should${if (expectedCleanUp) "" else " not"} cleanup")
+ .that(actualCleanUp)
+ .isEqualTo(expectedCleanUp)
+ }
+ }
+
+ @Test
+ fun checkTransitionRunner() {
+ val factory = FlickerTestRunnerFactory(instrumentation)
+ val actual = factory.buildTest { cfg ->
+ this.setDefaultTestCfg()
+ validateTest(cfg)
+ }
+
+ val first = (actual[0][1] as () -> Flicker).invoke()
+ val second = (actual[1][1] as () -> Flicker).invoke()
+ val third = (actual[2][1] as () -> Flicker).invoke()
+ val fourth = (actual[3][1] as () -> Flicker).invoke()
+
+ assertWithMessage("First and second tests should share a runner")
+ .that(first.runner)
+ .isEqualTo(second.runner)
+
+ assertWithMessage("Third and fourth tests should share a runner")
+ .that(third.runner)
+ .isEqualTo(fourth.runner)
+
+ assertWithMessage("First and third tests should not share a runner")
+ .that(first.runner)
+ .isNotEqualTo(third.runner)
+
}
}
\ No newline at end of file
diff --git a/libraries/flicker/test/src/com/android/server/wm/flicker/TransitionRunnerTest.kt b/libraries/flicker/test/src/com/android/server/wm/flicker/TransitionRunnerTest.kt
new file mode 100644
index 0000000..e6b5883
--- /dev/null
+++ b/libraries/flicker/test/src/com/android/server/wm/flicker/TransitionRunnerTest.kt
@@ -0,0 +1,68 @@
+/*
+ * Copyright (C) 2021 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.android.server.wm.flicker
+
+import androidx.test.platform.app.InstrumentationRegistry
+import com.android.server.wm.flicker.dsl.FlickerBuilder
+import com.google.common.truth.Truth
+import org.junit.FixMethodOrder
+import org.junit.Test
+import org.junit.runners.MethodSorters
+
+/**
+ * Contains [TransitionRunnerTest] and [TransitionRunnerCached] tests.
+ *
+ * To run this test: `atest FlickerLibTest:TransitionRunnerTest`
+ */
+@FixMethodOrder(MethodSorters.NAME_ASCENDING)
+class TransitionRunnerTest {
+ private val instrumentation = InstrumentationRegistry.getInstrumentation()
+
+ @Test
+ fun canRunTransition() {
+ val runner = TransitionRunner()
+ var executed = false
+ val flicker = FlickerBuilder(instrumentation)
+ .apply {
+ transitions {
+ executed = true
+ }
+ }.build(runner)
+ Truth.assertThat(executed).isFalse()
+ val result = runner.execute(flicker)
+ Truth.assertThat(executed).isTrue()
+ Truth.assertThat(result.error).isNull()
+ Truth.assertThat(result.runs).hasSize(4)
+ }
+
+ @Test
+ fun canRunTransitionCached() {
+ val runner = TransitionRunnerCached()
+ var executed = false
+ val flicker = FlickerBuilder(instrumentation)
+ .apply {
+ transitions {
+ executed = true
+ }
+ }.build(runner)
+ val result = runner.execute(flicker)
+ executed = false
+ val cachedResult = runner.execute(flicker)
+ Truth.assertThat(executed).isFalse()
+ Truth.assertThat(cachedResult).isEqualTo(result)
+ }
+}
\ No newline at end of file
diff --git a/libraries/flicker/test/src/com/android/server/wm/flicker/monitor/EventLogMonitorTest.kt b/libraries/flicker/test/src/com/android/server/wm/flicker/monitor/EventLogMonitorTest.kt
index 1fcf81d..c6debc5 100644
--- a/libraries/flicker/test/src/com/android/server/wm/flicker/monitor/EventLogMonitorTest.kt
+++ b/libraries/flicker/test/src/com/android/server/wm/flicker/monitor/EventLogMonitorTest.kt
@@ -45,17 +45,18 @@
val result = FlickerRunResult.Builder()
monitor.save("test", result)
- assertEquals(2, result.eventLog.size)
+ assertEquals(2, result.eventLog?.size)
assertEquals(
"4749f88 com.android.phone/com.android.phone.settings.fdn.FdnSetting (server)",
- result.eventLog[0].window)
- assertEquals(FocusEvent.Focus.LOST, result.eventLog[0].focus)
+ result.eventLog?.get(0)?.window)
+ assertEquals(FocusEvent.Focus.LOST, result.eventLog?.get(0)?.focus)
assertEquals(
"7c01447 com.android.phone/com.android.phone.settings.fdn.FdnSetting (server)",
- result.eventLog[1].window)
- assertEquals(FocusEvent.Focus.GAINED, result.eventLog[1].focus)
- assertTrue(result.eventLog[0].timestamp <= result.eventLog[1].timestamp)
- assertEquals(result.eventLog[0].reason, "test")
+ result.eventLog?.get(1)?.window)
+ assertEquals(FocusEvent.Focus.GAINED, result.eventLog?.get(1)?.focus)
+ assertTrue(result.eventLog?.get(0)?.timestamp ?: 0
+ <= result.eventLog?.get(1)?.timestamp ?: 0)
+ assertEquals(result.eventLog?.get(0)?.reason, "test")
}
@Test
@@ -86,17 +87,18 @@
val result = FlickerRunResult.Builder()
monitor.save("test", result)
- assertEquals(2, result.eventLog.size)
+ assertEquals(2, result.eventLog?.size)
assertEquals("479f88 " +
"com.android.phone/" +
"com.android.phone.settings.fdn.FdnSetting (server)",
- result.eventLog[0].window)
- assertEquals(FocusEvent.Focus.LOST, result.eventLog[0].focus)
+ result.eventLog?.get(0)?.window)
+ assertEquals(FocusEvent.Focus.LOST, result.eventLog?.get(0)?.focus)
assertEquals("7c01447 com.android.phone/" +
"com.android.phone.settings.fdn.FdnSetting (server)",
- result.eventLog[1].window)
- assertEquals(FocusEvent.Focus.GAINED, result.eventLog[1].focus)
- assertTrue(result.eventLog[0].timestamp <= result.eventLog[1].timestamp)
+ result.eventLog?.get(1)?.window)
+ assertEquals(FocusEvent.Focus.GAINED, result.eventLog?.get(1)?.focus)
+ assertTrue(result.eventLog?.get(0)?.timestamp ?: 0
+ <= result.eventLog?.get(1)?.timestamp ?: 0)
}
@Test
@@ -120,17 +122,18 @@
val result = FlickerRunResult.Builder()
monitor.save("test", result)
- assertEquals(2, result.eventLog.size)
+ assertEquals(2, result.eventLog?.size)
assertEquals(
"4749f88 com.android.phone/com.android.phone.settings.fdn.FdnSetting (server)",
- result.eventLog[0].window)
- assertEquals(FocusEvent.Focus.LOST, result.eventLog[0].focus)
+ result.eventLog?.get(0)?.window)
+ assertEquals(FocusEvent.Focus.LOST, result.eventLog?.get(0)?.focus)
assertEquals(
"7c01447 com.android.phone/com.android.phone.settings.fdn.FdnSetting (server)",
- result.eventLog[1].window)
- assertEquals(FocusEvent.Focus.GAINED, result.eventLog[1].focus)
- assertTrue(result.eventLog[0].timestamp <= result.eventLog[1].timestamp)
- assertEquals(result.eventLog[0].reason, "test")
+ result.eventLog?.get(1)?.window)
+ assertEquals(FocusEvent.Focus.GAINED, result.eventLog?.get(1)?.focus)
+ assertTrue(result.eventLog?.get(0)?.timestamp ?: 0
+ <= result.eventLog?.get(1)?.timestamp ?: 0)
+ assertEquals(result.eventLog?.get(0)?.reason, "test")
}
private companion object {
diff --git a/libraries/flicker/test/src/com/android/server/wm/flicker/monitor/LayersTraceMonitorTest.kt b/libraries/flicker/test/src/com/android/server/wm/flicker/monitor/LayersTraceMonitorTest.kt
index b1513cf..8060511 100644
--- a/libraries/flicker/test/src/com/android/server/wm/flicker/monitor/LayersTraceMonitorTest.kt
+++ b/libraries/flicker/test/src/com/android/server/wm/flicker/monitor/LayersTraceMonitorTest.kt
@@ -42,7 +42,7 @@
}
override fun getTraceFile(result: FlickerRunResult): Path? {
- return result.layersTraceFile
+ return result.traceFiles.firstOrNull { it.toString().contains("layers_trace") }
}
@Test
diff --git a/libraries/flicker/test/src/com/android/server/wm/flicker/monitor/ScreenRecorderTest.kt b/libraries/flicker/test/src/com/android/server/wm/flicker/monitor/ScreenRecorderTest.kt
index addc50b..dce8e50 100644
--- a/libraries/flicker/test/src/com/android/server/wm/flicker/monitor/ScreenRecorderTest.kt
+++ b/libraries/flicker/test/src/com/android/server/wm/flicker/monitor/ScreenRecorderTest.kt
@@ -52,7 +52,9 @@
SystemClock.sleep(100)
mScreenRecorder.stop()
val file = mScreenRecorder.outputPath.toFile()
- Truth.assertThat(file.exists()).isTrue()
+ Truth.assertWithMessage("Screen recording file not found")
+ .that(file.exists())
+ .isTrue()
}
@Test
@@ -62,7 +64,16 @@
mScreenRecorder.stop()
val builder = FlickerRunResult.Builder()
mScreenRecorder.save("test", builder)
- val file = builder.build().screenRecording
- Truth.assertThat(Files.exists(file)).isTrue()
+ val traces = builder.buildTraceResults().mapNotNull { result ->
+ result.traceFiles.firstOrNull {
+ it.toString().contains("transition")
+ }
+ }
+ traces.forEach {
+ Truth.assertWithMessage("Trace file $it not found").that(Files.exists(it)).isTrue()
+ }
+ traces.forEach {
+ Files.deleteIfExists(it)
+ }
}
}
diff --git a/libraries/flicker/test/src/com/android/server/wm/flicker/monitor/TraceMonitorTest.kt b/libraries/flicker/test/src/com/android/server/wm/flicker/monitor/TraceMonitorTest.kt
index f9ed773..ec22368 100644
--- a/libraries/flicker/test/src/com/android/server/wm/flicker/monitor/TraceMonitorTest.kt
+++ b/libraries/flicker/test/src/com/android/server/wm/flicker/monitor/TraceMonitorTest.kt
@@ -74,7 +74,10 @@
traceMonitor.stop()
val builder = FlickerRunResult.Builder()
traceMonitor.save("capturedTrace", builder)
- savedTrace = getTraceFile(builder.build()) ?: error("Could not find saved trace file")
+ val results = builder.buildAll()
+ Truth.assertWithMessage("Expected 3 results for the trace").that(results).hasSize(3)
+ val result = results.first()
+ savedTrace = getTraceFile(result) ?: error("Could not find saved trace file")
val testFile = savedTrace.toFile()
Truth.assertThat(testFile.exists()).isTrue()
val calculatedChecksum = TraceMonitor.calculateChecksum(savedTrace)
diff --git a/libraries/flicker/test/src/com/android/server/wm/flicker/monitor/WindowManagerTraceMonitorTest.kt b/libraries/flicker/test/src/com/android/server/wm/flicker/monitor/WindowManagerTraceMonitorTest.kt
index d1ab6ba..1cdf679 100644
--- a/libraries/flicker/test/src/com/android/server/wm/flicker/monitor/WindowManagerTraceMonitorTest.kt
+++ b/libraries/flicker/test/src/com/android/server/wm/flicker/monitor/WindowManagerTraceMonitorTest.kt
@@ -17,8 +17,8 @@
package com.android.server.wm.flicker.monitor
import androidx.test.uiautomator.UiDevice
-import com.android.server.wm.flicker.FlickerRunResult
import com.android.server.wm.nano.WindowManagerTraceFileProto
+import com.android.server.wm.flicker.FlickerRunResult
import com.google.common.truth.Truth
import org.junit.FixMethodOrder
import org.junit.Test
@@ -38,7 +38,7 @@
}
override fun getTraceFile(result: FlickerRunResult): Path? {
- return result.wmTraceFile
+ return result.traceFiles.firstOrNull { it.toString().contains("wm_trace") }
}
@Test
