Optimize memory usage for new Flicker JUnit runner
Because we now pre-run all runs in the test computation stage we accumulate lots of memory from all the test results which we need to keep to avoid re-running the runs when it comes to checking the assertions. So we now keep track of the run results in one zip file per run and clear the results from memory between those two stages to only have the large results in memory when we actually need them. This avoids getting OOM exceptions when running many tests at once.
This also includes a refactor of the FlickerRunResult to simplify the logic of caching run results in storage. Now a run result includes all the results from an iteration instead of being split in multiple run results by tag.
Test: atest FlickerLibTest
Bug: 236592450
Change-Id: I9fc92d6012c78b08b9583ec4a3264fc4c9845b09
diff --git a/libraries/flicker/src/com/android/server/wm/flicker/Flicker.kt b/libraries/flicker/src/com/android/server/wm/flicker/Flicker.kt
index 2638854..c57b78f 100644
--- a/libraries/flicker/src/com/android/server/wm/flicker/Flicker.kt
+++ b/libraries/flicker/src/com/android/server/wm/flicker/Flicker.kt
@@ -106,7 +106,6 @@
)
var result: FlickerResult? = null
- private set
/**
* Executes the test transition.
@@ -114,8 +113,7 @@
* @throws IllegalStateException If cannot execute the transition
*/
fun execute(): Flicker = apply {
- val result = runner.execute(this)
- this.result = result
+ this.result = runner.execute(this, useCacheIfAvailable = true)
}
/**
@@ -139,8 +137,7 @@
if (result.executionErrors.isEmpty()) {
// If there are no execution errors we want to throw an error here since we won't
// fail later in the FlickerBlockJUnit4ClassRunner.
- throw Exception(
- "No transition runs were successful executed! Can't check assertion.")
+ throw Exception("No transition runs were executed! Can't check assertion.")
}
return
}
@@ -157,9 +154,10 @@
fun clear() {
Log.v(FLICKER_TAG, "Cleaning up spec $testName")
runner.cleanUp()
- result = null
+ result?.clearFromMemory()
faasTracesCollector.stop()
faasTracesCollector.clear()
+ Log.v(FLICKER_TAG, "Cleaned up spec $testName")
}
/**
diff --git a/libraries/flicker/src/com/android/server/wm/flicker/FlickerBlockJUnit4ClassRunner.kt b/libraries/flicker/src/com/android/server/wm/flicker/FlickerBlockJUnit4ClassRunner.kt
index 36ecb0d..31995b4 100644
--- a/libraries/flicker/src/com/android/server/wm/flicker/FlickerBlockJUnit4ClassRunner.kt
+++ b/libraries/flicker/src/com/android/server/wm/flicker/FlickerBlockJUnit4ClassRunner.kt
@@ -31,7 +31,6 @@
import java.util.concurrent.locks.Lock
import java.util.concurrent.locks.ReentrantLock
import org.junit.FixMethodOrder
-import org.junit.Test
import org.junit.internal.AssumptionViolatedException
import org.junit.internal.runners.model.EachTestNotifier
import org.junit.internal.runners.statements.RunAfters
@@ -140,7 +139,7 @@
* Implementation of ParentRunner based on BlockJUnit4ClassRunner.
* Modified to report Flicker execution errors in the test results.
*/
- override fun runChild(method: FrameworkMethod?, notifier: RunNotifier) {
+ override fun runChild(method: FrameworkMethod, notifier: RunNotifier) {
val description = describeChild(method)
if (isIgnored(method)) {
notifier.fireTestIgnored(description)
@@ -237,13 +236,16 @@
val injectedTestCase = FlickerTestCase(results)
val mockedTestMethod = TestClass(injectedTestCase.javaClass)
- .getAnnotatedMethods(Test::class.java).first()
+ .getAnnotatedMethods(FlickerTestCase.InjectedTest::class.java).first()
val mockedFrameworkMethod = FlickerFrameworkMethod(
mockedTestMethod.method, injectedTestCase, testName
)
flickerTestMethods.add(mockedFrameworkMethod)
}
+ // Flush flicker data to storage to save memory and avoid OOM exceptions
+ flicker.clear()
+
return flickerTestMethods
}
@@ -468,7 +470,7 @@
var children: List<FrameworkMethod> = getFilteredChildren()
// In theory, we could have duplicate Descriptions. De-dup them before ordering,
// and add them back at the end.
- val childMap: MutableMap<Description, MutableList<FrameworkMethod>?> = LinkedHashMap(
+ val childMap: MutableMap<Description, MutableList<FrameworkMethod>> = LinkedHashMap(
children.size)
for (child in children) {
val description = describeChild(child)
diff --git a/libraries/flicker/src/com/android/server/wm/flicker/FlickerResult.kt b/libraries/flicker/src/com/android/server/wm/flicker/FlickerResult.kt
index 40bfb3a..b0dd262 100644
--- a/libraries/flicker/src/com/android/server/wm/flicker/FlickerResult.kt
+++ b/libraries/flicker/src/com/android/server/wm/flicker/FlickerResult.kt
@@ -16,7 +16,7 @@
package com.android.server.wm.flicker
-import com.android.server.wm.flicker.FlickerRunResult.Companion.RunStatus
+import com.android.internal.annotations.VisibleForTesting
import com.android.server.wm.flicker.TransitionRunner.Companion.ExecutionError
import com.android.server.wm.flicker.assertions.AssertionData
import com.android.server.wm.flicker.assertions.FlickerAssertionError
@@ -29,7 +29,7 @@
/**
* Result of each transition run
*/
- @JvmField val runResults: List<FlickerRunResult> = listOf(),
+ @JvmField val runResults: Collection<FlickerRunResult>,
/**
* List of test created during the execution
*/
@@ -39,15 +39,6 @@
*/
@JvmField val executionErrors: List<ExecutionError> = listOf()
) {
- init {
- for (result in runResults) {
- require(result.status != RunStatus.UNDEFINED) {
- "Can't create FlickerResult from RunResult ${result.traceName} " +
- "(${result.assertionTag}) with UNDEFINED status."
- }
- }
- }
-
/** Successful runs on which we can run assertions */
val successfulRuns: List<FlickerRunResult> = runResults.filter { it.isSuccessfulRun }
/** Failed runs due to execution errors which we shouldn't run assertions on */
@@ -71,8 +62,7 @@
Truth.assertWithMessage("No transitions were not executed successful")
.that(successfulRuns).isNotEmpty()
- val filteredRuns = successfulRuns.filter { it.assertionTag == assertion.tag }
- val currFailures = filteredRuns.mapNotNull { run -> run.checkAssertion(assertion) }
+ val currFailures = successfulRuns.mapNotNull { run -> run.checkAssertion(assertion) }
failures.addAll(currFailures)
return currFailures
}
@@ -80,7 +70,8 @@
/**
* Asserts if there have been any execution errors while running the transitions
*/
- internal fun checkForExecutionErrors() {
+ @VisibleForTesting
+ fun checkForExecutionErrors() {
if (executionErrors.isNotEmpty()) {
if (executionErrors.size == 1) {
throw executionErrors[0]
@@ -93,6 +84,14 @@
fun isNotEmpty(): Boolean = !isEmpty()
+ fun clearFromMemory() {
+ runResults.forEach { it.clearFromMemory() }
+ }
+
+ fun lock() {
+ runResults.forEach { it.lock() }
+ }
+
companion object {
class CombinedExecutionError(val errors: List<Throwable>?) : Throwable() {
init {
diff --git a/libraries/flicker/src/com/android/server/wm/flicker/FlickerRunResult.kt b/libraries/flicker/src/com/android/server/wm/flicker/FlickerRunResult.kt
index f093226..78cf781 100644
--- a/libraries/flicker/src/com/android/server/wm/flicker/FlickerRunResult.kt
+++ b/libraries/flicker/src/com/android/server/wm/flicker/FlickerRunResult.kt
@@ -17,62 +17,38 @@
package com.android.server.wm.flicker
import androidx.annotation.VisibleForTesting
-import com.android.compatibility.common.util.ZipUtil
import com.android.server.wm.flicker.assertions.AssertionData
import com.android.server.wm.flicker.assertions.FlickerAssertionError
import com.android.server.wm.flicker.assertions.FlickerAssertionErrorBuilder
import com.android.server.wm.flicker.assertions.FlickerSubject
import com.android.server.wm.flicker.dsl.AssertionTag
+import com.android.server.wm.flicker.helpers.clearableLazy
import com.android.server.wm.flicker.traces.eventlog.EventLogSubject
import com.android.server.wm.flicker.traces.eventlog.FocusEvent
+import com.android.server.wm.flicker.traces.layers.LayerTraceEntrySubject
import com.android.server.wm.flicker.traces.layers.LayersTraceSubject
+import com.android.server.wm.flicker.traces.windowmanager.WindowManagerStateSubject
import com.android.server.wm.flicker.traces.windowmanager.WindowManagerTraceSubject
-import com.android.server.wm.traces.common.layers.BaseLayerTraceEntry
import com.android.server.wm.traces.common.layers.LayersTrace
import com.android.server.wm.traces.common.transactions.TransactionsTrace
import com.android.server.wm.traces.common.transition.TransitionsTrace
-import com.android.server.wm.traces.common.windowmanager.WindowManagerState
import com.android.server.wm.traces.common.windowmanager.WindowManagerTrace
+import com.android.server.wm.traces.parser.DeviceDumpParser
+import com.android.server.wm.traces.parser.layers.LayersTraceParser
+import com.android.server.wm.traces.parser.transaction.TransactionsTraceParser
+import com.android.server.wm.traces.parser.transition.TransitionsTraceParser
+import com.android.server.wm.traces.parser.windowmanager.WindowManagerTraceParser
import java.io.File
-import java.nio.file.Path
import kotlinx.coroutines.CoroutineScope
-import kotlinx.coroutines.Deferred
import kotlinx.coroutines.Dispatchers
import kotlinx.coroutines.SupervisorJob
-import kotlinx.coroutines.async
-import kotlinx.coroutines.runBlocking
+
+val CHAR_POOL: List<Char> = ('a'..'z') + ('A'..'Z') + ('0'..'9')
/**
* Defines the result of a flicker run
*/
-class FlickerRunResult private constructor(
- /**
- * The trace files associated with the result (incl. screen recording)
- */
- _traceFile: Path?,
- /**
- * Determines which assertions to run (e.g., start, end, all, or a custom tag)
- */
- @JvmField var assertionTag: String,
- /**
- * Truth subject that corresponds to a [WindowManagerTrace] or [WindowManagerState]
- */
- internal val wmSubject: FlickerSubject?,
- /**
- * Truth subject that corresponds to a [LayersTrace] or [BaseLayerTraceEntry]
- */
- internal val layersSubject: FlickerSubject?,
- /**
- * Truth subject that corresponds to a list of [FocusEvent]
- */
- @VisibleForTesting
- val eventLogSubject: EventLogSubject?,
- /**
- * A trace of all transitions that ran during the run that can be used by FaaS to determine
- * which assertion to run and on which parts of the run.
- */
- val transitionsTrace: TransitionsTrace?,
-) {
+class FlickerRunResult(testName: String, iteration: Int) {
/**
* The object responsible for managing the trace file associated with this result.
*
@@ -80,46 +56,66 @@
* derived or extracted from another RunResult then that other RunResult should be the trace
* file manager.
*/
- internal var mTraceFile: TraceFile? =
- if (_traceFile != null) TraceFile(_traceFile) else null
+ private val artifacts: RunResultArtifacts = RunResultArtifacts(getDefaultFlickerOutputDir()
+ .resolve("${testName}_$iteration.zip"))
+ /**
+ * Truth subject that corresponds to a [WindowManagerTrace]
+ */
+ internal var wmTraceSubject: WindowManagerTraceSubject?
+ by clearableLazy { buildWmTraceSubject() }
+ private set
+ /**
+ * Truth subject that corresponds to a [LayersTrace]
+ */
+ internal var layersTraceSubject: LayersTraceSubject?
+ by clearableLazy { buildLayersTraceSubject() }
+ private set
+ /**
+ * Truth subject that corresponds to a list of [FocusEvent]
+ */
+ @VisibleForTesting
+ var eventLogSubject: EventLogSubject? by clearableLazy { buildEventLog() }
+ private set
+ /**
+ * A trace of all transitions that ran during the run that can be used by FaaS to determine
+ * which assertion to run and on which parts of the run.
+ */
+ var transitionsTrace: TransitionsTrace? by clearableLazy { buildTransitionsTrace() }
+ private set
+ /**
+ * A collection of tagged states collected during the run. Stored as a mapping from tag to state
+ * entry subjects representing the dump.
+ */
+ var taggedStates: Map<String, List<StateDump>>? by clearableLazy { buildTaggedStates() }
+ private set
- internal val traceName = mTraceFile?.traceFile?.fileName ?: "UNNAMED_TRACE"
+ internal val traceName = this.artifacts.path.fileName ?: "UNNAMED_TRACE"
- var status: RunStatus = RunStatus.UNDEFINED
- internal set(value) {
- if (field != value) {
- require(value != RunStatus.UNDEFINED) {
- "Can't set status to UNDEFINED after being defined"
- }
- require(!field.isFailure) {
- "Status of run already set to a failed status $field " +
- "and can't be changed to $value."
- }
- field = value
- }
-
- mTraceFile?.status = status
+ val status: RunStatus
+ get() {
+ return this.artifacts.status
}
- fun setRunFailed() {
- status = RunStatus.RUN_FAILED
- }
-
val isSuccessfulRun: Boolean get() = !isFailedRun
val isFailedRun: Boolean get() {
require(status != RunStatus.UNDEFINED) {
- "RunStatus cannot be UNDEFINED for $traceName ($assertionTag)"
+ "RunStatus cannot be UNDEFINED for $traceName"
}
// Other types of failures can only happen if the run has succeeded
return status == RunStatus.RUN_FAILED
}
- fun getSubjects(): List<FlickerSubject> {
+ fun getSubjects(tag: String): List<FlickerSubject> {
val result = mutableListOf<FlickerSubject>()
- wmSubject?.run { result.add(this) }
- layersSubject?.run { result.add(this) }
- eventLogSubject?.run { result.add(this) }
+ if (tag == AssertionTag.ALL) {
+ wmTraceSubject?.run { result.add(this) }
+ layersTraceSubject?.run { result.add(this) }
+ eventLogSubject?.run { result.add(this) }
+ } else {
+ taggedStates!![tag]?.forEach { it.wmState?.run { result.add(this) } }
+ taggedStates!![tag]?.forEach { it.layersState?.run { result.add(this) } }
+ }
return result
}
@@ -130,215 +126,173 @@
assertion.checkAssertion(this)
null
} catch (error: Throwable) {
- status = RunStatus.ASSERTION_FAILED
+ this.artifacts.status = RunStatus.ASSERTION_FAILED
FlickerAssertionErrorBuilder()
.fromError(error)
.atTag(assertion.tag)
- .withTrace(this.mTraceFile)
+ .withTrace(this.artifacts)
.build()
}
}
- /**
- * Parse a [trace] file into a anything asynchronously
- *
- * The parsed result is available in [promise]
- */
- class AsyncParser<SubjectType>(
- val trace: Path,
- parser: ((Path) -> SubjectType?)?
- ) {
- val promise: Deferred<SubjectType?>? = parser?.run { SCOPE.async { parser(trace) } }
+ private val taggedStateBuilders: MutableMap<String, MutableList<StateDumpFileNames>> =
+ mutableMapOf()
+ private var wmTraceFileName: String? = null
+ private var layersTraceFileName: String? = null
+ private var transactionsTraceFileName: String? = null
+ private var transitionsTraceFileName: String? = null
+
+ data class StateDumpFileNames(
+ val wmDumpFileName: String,
+ val layersDumpFileName: String,
+ )
+
+ @VisibleForTesting
+ var eventLog: List<FocusEvent>? = null
+
+ fun setStatus(status: RunStatus) {
+ this.artifacts.status = status
}
- class Builder {
- private var wmTraceData: AsyncParser<WindowManagerTraceSubject>? = null
- private var layersTraceData: AsyncParser<LayersTraceSubject>? = null
- private var transitionsTraceData: AsyncParser<TransitionsTrace>? = null
- private var transactionsTraceData: AsyncParser<TransactionsTrace>? = null
- var screenRecording: Path? = null
+ fun setWmTrace(traceFile: File) {
+ wmTraceFileName = traceFile.name
+ this.artifacts.addFile(traceFile)
+ }
- /**
- * List of focus events, if collected
- */
- var eventLog: List<FocusEvent>? = null
+ fun setLayersTrace(traceFile: File) {
+ layersTraceFileName = traceFile.name
+ this.artifacts.addFile(traceFile)
+ }
- /**
- * Parses a [WindowManagerTraceSubject]
- *
- * @param traceFile of the trace file to parse
- * @param parser lambda to parse the trace into a [WindowManagerTraceSubject]
- */
- fun setWmTrace(traceFile: Path, parser: (Path) -> WindowManagerTraceSubject?) {
- wmTraceData = AsyncParser(traceFile, parser)
+ fun setScreenRecording(screenRecording: File) {
+ this.artifacts.addFile(screenRecording)
+ }
+
+ fun setTransactionsTrace(traceFile: File) {
+ transactionsTraceFileName = traceFile.name
+ this.artifacts.addFile(traceFile)
+ }
+
+ fun setTransitionsTrace(traceFile: File) {
+ transitionsTraceFileName = traceFile.name
+ this.artifacts.addFile(traceFile)
+ }
+
+ fun addTaggedState(
+ tag: String,
+ wmDumpFile: File,
+ layersDumpFile: File
+ ) {
+ if (taggedStateBuilders[tag] == null) {
+ taggedStateBuilders[tag] = mutableListOf()
}
+ // Append random string to support multiple dumps with the same tag
+ val randomString = (1..10)
+ .map { i -> kotlin.random.Random.nextInt(0, CHAR_POOL.size) }
+ .map(CHAR_POOL::get)
+ .joinToString("")
+ val wmDumpArchiveName = wmDumpFile.name + randomString
+ val layersDumpArchiveName = layersDumpFile.name + randomString
+ taggedStateBuilders[tag]!!
+ .add(StateDumpFileNames(wmDumpArchiveName, layersDumpArchiveName))
+ this.artifacts.addFile(wmDumpFile, wmDumpArchiveName)
+ this.artifacts.addFile(layersDumpFile, layersDumpArchiveName)
+ }
- /**
- * Parses a [LayersTraceSubject]
- *
- * @param traceFile of the trace file to parse
- * @param parser lambda to parse the trace into a [LayersTraceSubject]
- */
- fun setLayersTrace(traceFile: Path, parser: (Path) -> LayersTraceSubject?) {
- layersTraceData = AsyncParser(traceFile, parser)
- }
+ fun setResultsFromMonitor(resultSetter: IResultSetter) {
+ resultSetter.setResult(this)
+ }
- /**
- * Parses a [TransitionsTrace]
- *
- * @param traceFile of the trace file to parse
- * @param parser lambda to parse the trace file into a [TransitionsTrace]
- */
- fun setTransitionsTrace(traceFile: Path, parser: (Path) -> TransitionsTrace?) {
- transitionsTraceData = AsyncParser(traceFile, parser)
- }
+ internal fun buildWmTrace(): WindowManagerTrace? {
+ val wmTraceFileName = this.wmTraceFileName ?: return null
+ val traceData = this.artifacts.getFileBytes(wmTraceFileName)
+ return WindowManagerTraceParser.parseFromTrace(traceData)
+ }
- /**
- * Parses a [TransactionsTrace]
- *
- * @param traceFile of the trace file to parse
- * @param parser lambda to parse the trace file into a [TransactionsTrace]
- */
- fun setTransactionsTrace(traceFile: Path, parser: (Path) -> TransactionsTrace?) {
- transactionsTraceData = AsyncParser(traceFile, parser)
- }
+ internal fun buildLayersTrace(): LayersTrace? {
+ val wmTraceFileName = this.layersTraceFileName ?: return null
+ val traceData = this.artifacts.getFileBytes(wmTraceFileName)
+ return LayersTraceParser.parseFromTrace(traceData)
+ }
- private fun buildResult(
- assertionTag: String,
- wmSubject: FlickerSubject?,
- layersSubject: FlickerSubject?,
- status: RunStatus,
- traceFile: Path? = null,
- eventLogSubject: EventLogSubject? = null,
- transitionsTrace: TransitionsTrace? = null,
- transactionsTrace: TransactionsTrace? = null
- ): FlickerRunResult {
- val result = FlickerRunResult(
- traceFile,
- assertionTag,
- wmSubject,
- layersSubject,
- eventLogSubject,
- transitionsTrace,
- transactionsTrace
- )
- result.status = status
- return result
- }
+ private fun buildTransactionsTrace(): TransactionsTrace? {
+ val transactionsTrace = this.transactionsTraceFileName ?: return null
+ val traceData = this.artifacts.getFileBytes(transactionsTrace)
+ return TransactionsTraceParser.parseFromTrace(traceData)
+ }
- /**
- * Builds a new [FlickerRunResult] for a trace
- *
- * @param assertionTag Tag to associate with the result
- * @param wmTrace WindowManager trace
- * @param layersTrace Layers trace
- */
- fun buildStateResult(
- assertionTag: String,
- wmTrace: WindowManagerTrace?,
- layersTrace: LayersTrace?,
- wmTraceFile: Path?,
- layersTraceFile: Path?,
- testName: String,
- iteration: Int,
- status: RunStatus
- ): FlickerRunResult {
- val wmSubject = wmTrace?.let { WindowManagerTraceSubject.assertThat(it).first() }
- val layersSubject = layersTrace?.let { LayersTraceSubject.assertThat(it).first() }
+ internal fun buildTransitionsTrace(): TransitionsTrace? {
+ val transactionsTrace = buildTransactionsTrace()
+ val transitionsTrace = this.transitionsTraceFileName ?: return null
+ val traceData = this.artifacts.getFileBytes(transitionsTrace)
+ return TransitionsTraceParser.parseFromTrace(traceData, transactionsTrace!!)
+ }
- val traceFiles = mutableListOf<File>()
- wmTraceFile?.let { traceFiles.add(it.toFile()) }
- layersTraceFile?.let { traceFiles.add(it.toFile()) }
- val traceFile = compress(traceFiles, "${assertionTag}_${testName}_$iteration.zip")
+ private fun buildTaggedStates(): Map<String, List<StateDump>> {
+ val taggedStates = mutableMapOf<String, List<StateDump>>()
+ for ((tag, states) in taggedStateBuilders.entries) {
+ val taggedStatesList = mutableListOf<StateDump>()
+ taggedStates[tag] = taggedStatesList
+ for (state in states) {
+ val wmDumpData = this.artifacts.getFileBytes(state.wmDumpFileName)
+ val layersDumpData = this.artifacts.getFileBytes(state.layersDumpFileName)
+ val deviceState = DeviceDumpParser.fromDump(wmDumpData, layersDumpData)
- return buildResult(
- assertionTag, wmSubject, layersSubject, status,
- traceFile = traceFile
- )
- }
-
- @VisibleForTesting
- fun buildEventLogResult(status: RunStatus): FlickerRunResult {
- val events = eventLog ?: emptyList()
- return buildResult(
- AssertionTag.ALL,
- wmSubject = null,
- layersSubject = null,
- eventLogSubject = EventLogSubject.assertThat(events),
- status = status
- )
- }
-
- @VisibleForTesting
- fun buildTraceResults(
- testName: String,
- iteration: Int,
- status: RunStatus
- ): RunResults = runBlocking {
- val wmSubject = wmTraceData?.promise?.await()
- val layersSubject = layersTraceData?.promise?.await()
- val transitionsTrace = transitionsTraceData?.promise?.await()
- val transactionsTrace = transactionsTraceData?.promise?.await()
-
- val traceFile = compress(testName, iteration)
- val traceResult = buildResult(
- AssertionTag.ALL, wmSubject, layersSubject, transitionsTrace = transitionsTrace,
- transactionsTrace = transactionsTrace, traceFile = traceFile, status = status
- )
- val initialStateResult = buildResult(
- AssertionTag.START, wmSubject?.first(), layersSubject?.first(), status = status
- )
- initialStateResult.mTraceFile = traceResult.mTraceFile
-
- val finalStateResult = buildResult(
- AssertionTag.END, wmSubject?.last(), layersSubject?.last(), status = status
- )
- finalStateResult.mTraceFile = traceResult.mTraceFile
-
- RunResults(initialStateResult, finalStateResult, traceResult)
- }
-
- private fun compress(testName: String, iteration: Int): Path? {
- val traceFiles = mutableListOf<File>()
- wmTraceData?.trace?.let { traceFiles.add(it.toFile()) }
- layersTraceData?.trace?.let { traceFiles.add(it.toFile()) }
- screenRecording?.let { traceFiles.add(it.toFile()) }
-
- return compress(traceFiles, "${testName}_$iteration.zip")
- }
-
- private fun compress(traceFiles: List<File>, archiveName: String): Path? {
- val files = traceFiles.filter { it.exists() }
- if (files.isEmpty()) {
- return null
+ val wmStateSubject = deviceState.wmState
+ ?.asTrace()?.let { WindowManagerTraceSubject.assertThat(it).first() }
+ val layersStateSubject = deviceState.layerState
+ ?.asTrace()?.let { LayersTraceSubject.assertThat(it).first() }
+ taggedStatesList.add(StateDump(wmStateSubject, layersStateSubject))
}
-
- val firstFile = files.first()
- val compressedFile = firstFile.resolveSibling(archiveName)
- ZipUtil.createZip(traceFiles, compressedFile)
- traceFiles.forEach {
- it.delete()
- }
-
- return compressedFile.toPath()
}
- fun buildAll(testName: String, iteration: Int, status: RunStatus): RunResults {
- val results = buildTraceResults(testName, iteration, status)
- if (eventLog != null) {
- results.eventLogResult = buildEventLogResult(status = status)
- }
+ require(taggedStates[AssertionTag.START] == null) { "START tag is reserved" }
+ taggedStates[AssertionTag.START] = listOf(buildStartState())
+ require(taggedStates[AssertionTag.END] == null) { "END tag is reserved" }
+ taggedStates[AssertionTag.END] = listOf(buildEndState())
- return results
- }
+ return taggedStates
+ }
- fun setResultFrom(resultSetter: IResultSetter) {
- resultSetter.setResult(this)
- }
+ private fun buildStartState(): StateDump {
+ return StateDump(buildWmTraceSubject()?.first(), buildLayersTraceSubject()?.first())
+ }
+
+ private fun buildEndState(): StateDump {
+ return StateDump(buildWmTraceSubject()?.last(), buildLayersTraceSubject()?.last())
+ }
+
+ private fun buildEventLog(): EventLogSubject? {
+ val eventLog = eventLog ?: return null
+ return EventLogSubject.assertThat(eventLog)
+ }
+
+ private fun buildWmTraceSubject(): WindowManagerTraceSubject? {
+ val wmTrace = buildWmTrace()
+ return if (wmTrace != null)
+ WindowManagerTraceSubject.assertThat(wmTrace) else null
+ }
+
+ private fun buildLayersTraceSubject(): LayersTraceSubject? {
+ val layersTrace = buildLayersTrace()
+ return if (layersTrace != null)
+ LayersTraceSubject.assertThat(layersTrace) else null
+ }
+
+ internal fun lock() {
+ this.artifacts.lock()
+ }
+
+ fun clearFromMemory() {
+ wmTraceSubject = null
+ layersTraceSubject = null
+ eventLogSubject = null
+ transitionsTrace = null
+ taggedStates = null
}
interface IResultSetter {
- fun setResult(builder: Builder)
+ fun setResult(result: FlickerRunResult)
}
companion object {
@@ -384,4 +338,9 @@
}
}
}
+
+ data class StateDump(
+ val wmState: WindowManagerStateSubject?,
+ val layersState: LayerTraceEntrySubject?
+ )
}
diff --git a/libraries/flicker/src/com/android/server/wm/flicker/RunResultArtifacts.kt b/libraries/flicker/src/com/android/server/wm/flicker/RunResultArtifacts.kt
new file mode 100644
index 0000000..a68ace3
--- /dev/null
+++ b/libraries/flicker/src/com/android/server/wm/flicker/RunResultArtifacts.kt
@@ -0,0 +1,155 @@
+/*
+ * Copyright (C) 2022 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.android.server.wm.flicker
+
+import android.util.Log
+import com.android.server.wm.flicker.FlickerRunResult.Companion.RunStatus
+import java.io.BufferedInputStream
+import java.io.BufferedOutputStream
+import java.io.ByteArrayOutputStream
+import java.io.File
+import java.io.FileInputStream
+import java.io.FileOutputStream
+import java.io.IOException
+import java.nio.file.Path
+import java.util.zip.ZipEntry
+import java.util.zip.ZipInputStream
+import java.util.zip.ZipOutputStream
+
+class RunResultArtifacts(_filePath: Path) {
+
+ init {
+ _filePath.parent.toFile().mkdirs()
+ _filePath.toFile().createNewFile()
+ }
+
+ var path = _filePath
+ private set
+
+ private var archiveLocked = false
+
+ private val zipOutputStream by lazy { openZipOutputStream() }
+ private fun openZipOutputStream(): ZipOutputStream {
+ return ZipOutputStream(BufferedOutputStream(
+ FileOutputStream(path.toFile()), BUFFER_SIZE))
+ }
+
+ internal val traceName = path.fileName ?: "UNNAMED_TRACE"
+
+ var status: RunStatus = RunStatus.UNDEFINED
+ internal set(value) {
+ if (field != value) {
+ require(value != RunStatus.UNDEFINED) {
+ "Can't set status to UNDEFINED after being defined"
+ }
+ require(!field.isFailure) {
+ "Status of run already set to a failed status $field " +
+ "and can't be changed to $value."
+ }
+ field = value
+ syncFileWithStatus()
+ }
+ }
+
+ private fun syncFileWithStatus() {
+ // Since we don't expect this to run in a multi-threaded context this is fine
+ val localTraceFile = path
+ try {
+ val newFileName = "${status.prefix}_$traceName"
+ val dst = localTraceFile.resolveSibling(newFileName)
+ Utils.renameFile(localTraceFile, dst)
+ path = dst
+ } catch (e: IOException) {
+ Log.e(FLICKER_TAG, "Unable to update file status $this", e)
+ }
+ }
+
+ internal fun addFile(file: File, nameInArchive: String = file.name) {
+ require(!archiveLocked) { "Archive is locked. Can't add to it." }
+ val fi = FileInputStream(file)
+ val inputStream = BufferedInputStream(fi, BUFFER_SIZE)
+ inputStream.use {
+ val entry = ZipEntry(nameInArchive)
+ zipOutputStream.putNextEntry(entry)
+ val data = ByteArray(BUFFER_SIZE)
+ var count: Int = it.read(data, 0, BUFFER_SIZE)
+ while (count != -1) {
+ zipOutputStream.write(data, 0, count)
+ count = it.read(data, 0, BUFFER_SIZE)
+ }
+ }
+ zipOutputStream.closeEntry()
+ file.delete()
+ }
+
+ internal fun lock() {
+ if (!archiveLocked) {
+ zipOutputStream.close()
+ archiveLocked = true
+ }
+ }
+
+ internal fun getFileBytes(fileName: String): ByteArray {
+ require (archiveLocked) { "Can't get files from archive before it is closed" }
+
+ val tmpBuffer = ByteArray(BUFFER_SIZE)
+ val zipInputStream: ZipInputStream
+ try {
+ zipInputStream =
+ ZipInputStream(
+ BufferedInputStream(FileInputStream(path.toFile()), BUFFER_SIZE))
+ } catch (e: Throwable) {
+ return ByteArray(0)
+ }
+ val outByteArray = ByteArrayOutputStream()
+ var foundFile = false
+
+ try {
+ var zipEntry: ZipEntry? = zipInputStream.nextEntry
+ while (zipEntry != null) {
+ if (zipEntry.name == fileName) {
+ val outputStream = BufferedOutputStream(outByteArray, BUFFER_SIZE)
+ try {
+ var size = zipInputStream.read(tmpBuffer, 0, BUFFER_SIZE)
+ while (size > 0) {
+ outputStream.write(tmpBuffer, 0, size)
+ size = zipInputStream.read(tmpBuffer, 0, BUFFER_SIZE)
+ }
+ zipInputStream.closeEntry()
+ } finally {
+ outputStream.flush()
+ outputStream.close()
+ }
+ foundFile = true
+ break
+ }
+ zipEntry = zipInputStream.nextEntry
+ }
+ } finally {
+ zipInputStream.closeEntry()
+ zipInputStream.close()
+ }
+
+ require(foundFile) { "$fileName not found in archive..." }
+
+ return outByteArray.toByteArray()
+ }
+
+ companion object {
+ private const val BUFFER_SIZE = 2048
+ }
+}
diff --git a/libraries/flicker/src/com/android/server/wm/flicker/TraceFile.kt b/libraries/flicker/src/com/android/server/wm/flicker/TraceFile.kt
deleted file mode 100644
index 3b21e4a..0000000
--- a/libraries/flicker/src/com/android/server/wm/flicker/TraceFile.kt
+++ /dev/null
@@ -1,58 +0,0 @@
-/*
- * Copyright (C) 2022 The Android Open Source Project
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package com.android.server.wm.flicker
-
-import android.util.Log
-import com.android.server.wm.flicker.FlickerRunResult.Companion.RunStatus
-import java.io.IOException
-import java.nio.file.Path
-
-class TraceFile(_traceFile: Path) {
-
- var traceFile = _traceFile
- private set
-
- internal val traceName = traceFile.fileName ?: "UNNAMED_TRACE"
-
- var status: RunStatus = RunStatus.UNDEFINED
- internal set(value) {
- if (field != value) {
- require(value != RunStatus.UNDEFINED) {
- "Can't set status to UNDEFINED after being defined"
- }
- require(!field.isFailure) {
- "Status of run already set to a failed status $field " +
- "and can't be changed to $value."
- }
- field = value
- syncFileWithStatus()
- }
- }
-
- private fun syncFileWithStatus() {
- // Since we don't expect this to run in a multi-threaded context this is fine
- val localTraceFile = traceFile
- try {
- val newFileName = "${status.prefix}_$traceName"
- val dst = localTraceFile.resolveSibling(newFileName)
- Utils.renameFile(localTraceFile, dst)
- traceFile = dst
- } catch (e: IOException) {
- Log.e(FLICKER_TAG, "Unable to update file status $this", e)
- }
- }
-}
diff --git a/libraries/flicker/src/com/android/server/wm/flicker/TransitionRunner.kt b/libraries/flicker/src/com/android/server/wm/flicker/TransitionRunner.kt
index 4eef598..6ffa4af 100644
--- a/libraries/flicker/src/com/android/server/wm/flicker/TransitionRunner.kt
+++ b/libraries/flicker/src/com/android/server/wm/flicker/TransitionRunner.kt
@@ -17,15 +17,11 @@
package com.android.server.wm.flicker
import android.util.Log
-import com.android.server.wm.flicker.FlickerRunResult.Companion.RunResults
import com.android.server.wm.flicker.FlickerRunResult.Companion.RunStatus
import com.android.server.wm.flicker.monitor.IFileGeneratingMonitor
import com.android.server.wm.flicker.monitor.ITransitionMonitor
-import com.android.server.wm.flicker.traces.layers.LayersTraceSubject
-import com.android.server.wm.flicker.traces.windowmanager.WindowManagerTraceSubject
import com.android.server.wm.traces.common.ConditionList
import com.android.server.wm.traces.common.WindowManagerConditionsFactory
-import com.android.server.wm.traces.parser.DeviceDumpParser
import com.android.server.wm.traces.parser.getCurrentState
import java.io.IOException
import java.nio.file.Files
@@ -50,10 +46,12 @@
/**
* Iteration identifier during test run
*/
- internal var iteration = 0
+ internal var iteration = -1
private set
private val tags = mutableSetOf<String>()
- private var tagsResults = mutableListOf<FlickerRunResult>()
+
+ // Iteration to resultBuilder
+ private var results = mutableMapOf<Int, FlickerRunResult>()
/**
* Executes the setup, transitions and teardown defined in [flicker]
@@ -61,7 +59,7 @@
* @param flicker test specification
* @throws IllegalArgumentException If the transitions are empty or repetitions is set to 0
*/
- open fun execute(flicker: Flicker): FlickerResult {
+ open fun execute(flicker: Flicker, useCacheIfAvailable: Boolean = true): FlickerResult {
check(flicker)
return run(flicker)
}
@@ -83,7 +81,7 @@
open fun cleanUp() {
tags.clear()
- tagsResults.clear()
+ results.clear()
}
/**
@@ -92,13 +90,12 @@
* @param flicker test specification
*/
internal open fun run(flicker: Flicker): FlickerResult {
- val runs = mutableListOf<FlickerRunResult>()
val executionErrors = mutableListOf<ExecutionError>()
- safeExecution(flicker, runs, executionErrors) {
+ safeExecution(flicker, executionErrors) {
runTestSetup(flicker)
-
for (x in 0 until flicker.repetitions) {
iteration = x
+ results[iteration] = FlickerRunResult(flicker.testName, iteration)
val description = Description.createSuiteDescription(flicker.testName)
if (flicker.faasEnabled) {
flicker.faas.setCriticalUserJourneyName(flicker.testName)
@@ -107,7 +104,7 @@
runTransitionSetup(flicker)
runTransition(flicker)
runTransitionTeardown(flicker)
- processRunTraces(flicker, runs, RunStatus.ASSERTION_SUCCESS)
+ processRunTraces(flicker, RunStatus.ASSERTION_SUCCESS)
if (flicker.faasEnabled) {
flicker.faas.testFinished(description)
if (flicker.faas.executionErrors.isNotEmpty()) {
@@ -119,29 +116,32 @@
runTestTeardown(flicker)
}
- runs.addAll(tagsResults)
- val result = FlickerResult(runs.toList(), tags.toSet(), executionErrors)
+ val result = FlickerResult(
+ results.values.toList(), // toList ensures we clone the list before cleanUp
+ tags.toSet(),
+ executionErrors
+ )
cleanUp()
return result
}
private fun safeExecution(
flicker: Flicker,
- runs: MutableList<FlickerRunResult>,
executionErrors: MutableList<ExecutionError>,
execution: () -> Unit
) {
try {
execution()
} catch (e: TestSetupFailure) {
- // If we failure on the test setup we can't run any of the transitions
+ // If we fail on the test setup we can't run any of the transitions
executionErrors.add(e)
} catch (e: TransitionSetupFailure) {
// If we fail on the transition run setup then we don't want to run any further
// transitions nor save any results for this run. We simply want to run the test
// teardown.
executionErrors.add(e)
- safeExecution(flicker, runs, executionErrors) {
+ getCurrentRunResult().setStatus(RunStatus.RUN_FAILED)
+ safeExecution(flicker, executionErrors) {
runTestTeardown(flicker)
}
} catch (e: TransitionExecutionFailure) {
@@ -150,8 +150,8 @@
// want to run the test teardown
executionErrors.add(e)
flicker.traceMonitors.forEach { it.tryStop() }
- safeExecution(flicker, runs, executionErrors) {
- processRunTraces(flicker, runs, RunStatus.RUN_FAILED)
+ safeExecution(flicker, executionErrors) {
+ processRunTraces(flicker, RunStatus.RUN_FAILED)
runTestTeardown(flicker)
}
} catch (e: TransitionTeardownFailure) {
@@ -160,15 +160,15 @@
// But, we do want to run the test teardown.
executionErrors.add(e)
flicker.traceMonitors.forEach { it.tryStop() }
- safeExecution(flicker, runs, executionErrors) {
- processRunTraces(flicker, runs, RunStatus.RUN_FAILED)
+ safeExecution(flicker, executionErrors) {
+ processRunTraces(flicker, RunStatus.RUN_FAILED)
runTestTeardown(flicker)
}
} catch (e: TraceProcessingFailure) {
// If we fail to process the run traces we still want to run the teardowns and report
// the execution error.
executionErrors.add(e)
- safeExecution(flicker, runs, executionErrors) {
+ safeExecution(flicker, executionErrors) {
runTransitionTeardown(flicker)
runTestTeardown(flicker)
}
@@ -176,32 +176,32 @@
// If we fail in the execution of the test teardown there is nothing else to do apart
// from reporting the execution error.
executionErrors.add(e)
- for (run in runs) {
- run.setRunFailed()
- }
+ getCurrentRunResult().setStatus(RunStatus.RUN_FAILED)
}
}
/**
* Parses the traces collected by the monitors to generate FlickerRunResults containing the
* parsed trace and information about the status of the run.
- * The run results are added to the runs list which is then used to run Flicker assertions on.
+ * The run results are added to the resultBuilders list which is then used to run Flicker
+ * assertions on.
*/
@Throws(TraceProcessingFailure::class)
private fun processRunTraces(
flicker: Flicker,
- runs: MutableList<FlickerRunResult>,
status: RunStatus
) {
try {
- val runResults = buildRunResults(flicker, iteration, status)
- runs.addAll(runResults.toList())
+ val result = getCurrentRunResult()
+ result.setStatus(status)
+ setMonitorResults(flicker, result)
+ result.lock()
if (flicker.faasEnabled && !status.isFailure) {
// Don't run FaaS on failed transitions
- val wmTrace = (runResults.traceResult.wmSubject as WindowManagerTraceSubject).trace
- val layersTrace = (runResults.traceResult.layersSubject as LayersTraceSubject).trace
- val transitionsTrace = runResults.traceResult.transitionsTrace
+ val wmTrace = result.buildWmTrace()
+ val layersTrace = result.buildLayersTrace()
+ val transitionsTrace = result.buildTransitionsTrace()
flicker.faasTracesCollector.wmTrace = wmTrace
flicker.faasTracesCollector.layersTrace = layersTrace
@@ -220,13 +220,6 @@
}
throw TraceProcessingFailure(e)
}
-
- // Update the status of all the tags created in this iteration and add them to runs
- for (result in tagsResults) {
- result.status = status
- runs.add(result)
- }
- tagsResults.clear()
}
@Throws(TestSetupFailure::class)
@@ -282,17 +275,16 @@
}
}
- private fun buildRunResults(
+ private fun setMonitorResults(
flicker: Flicker,
- iteration: Int,
- status: RunStatus
- ): RunResults {
- val resultBuilder = FlickerRunResult.Builder()
+ result: FlickerRunResult,
+ ): FlickerRunResult {
+
flicker.traceMonitors.forEach {
- resultBuilder.setResultFrom(it)
+ result.setResultsFromMonitor(it)
}
- return resultBuilder.buildAll(flicker.testName, iteration, status)
+ return result
}
private fun ITransitionMonitor.tryStop() {
@@ -327,36 +319,31 @@
tags.add(tag)
val deviceStateBytes = getCurrentState(flicker.instrumentation.uiAutomation)
- val deviceState = DeviceDumpParser.fromDump(deviceStateBytes.first, deviceStateBytes.second)
try {
- val wmTraceFile = flicker.outputDir.resolve(
- getTaggedFilePath(flicker, tag, "wm_trace")
+ val wmDumpFile = flicker.outputDir.resolve(
+ getTaggedFilePath(flicker, tag, "wm_dump")
)
- Files.write(wmTraceFile, deviceStateBytes.first)
+ Files.write(wmDumpFile, deviceStateBytes.first)
- val layersTraceFile = flicker.outputDir.resolve(
- getTaggedFilePath(flicker, tag, "layers_trace")
+ val layersDumpFile = flicker.outputDir.resolve(
+ getTaggedFilePath(flicker, tag, "layers_dump")
)
- Files.write(layersTraceFile, deviceStateBytes.second)
+ Files.write(layersDumpFile, deviceStateBytes.second)
- val builder = FlickerRunResult.Builder()
- val result = builder.buildStateResult(
+ getCurrentRunResult().addTaggedState(
tag,
- deviceState.wmState?.asTrace(),
- deviceState.layerState?.asTrace(),
- wmTraceFile,
- layersTraceFile,
- flicker.testName,
- iteration,
- // Undefined until it is updated in processRunTraces
- RunStatus.UNDEFINED
+ wmDumpFile.toFile(),
+ layersDumpFile.toFile(),
)
- tagsResults.add(result)
} catch (e: IOException) {
throw RuntimeException("Unable to create trace file: ${e.message}", e)
}
}
+ private fun getCurrentRunResult(): FlickerRunResult {
+ return results[iteration]!!
+ }
+
companion object {
/**
* Conditions that determine when the UI is in a stable stable and no windows or layers are
@@ -369,7 +356,7 @@
)
)
- open class ExecutionError(val inner: Throwable) : Throwable(inner) {
+ open class ExecutionError(private val inner: Throwable) : Throwable(inner) {
init {
super.setStackTrace(inner.stackTrace)
}
diff --git a/libraries/flicker/src/com/android/server/wm/flicker/TransitionRunnerWithRules.kt b/libraries/flicker/src/com/android/server/wm/flicker/TransitionRunnerWithRules.kt
index 1d27c18..c92a268 100644
--- a/libraries/flicker/src/com/android/server/wm/flicker/TransitionRunnerWithRules.kt
+++ b/libraries/flicker/src/com/android/server/wm/flicker/TransitionRunnerWithRules.kt
@@ -47,7 +47,7 @@
override fun cleanUp() {
super.cleanUp()
- result = null
+ result?.clearFromMemory()
}
/**
diff --git a/libraries/flicker/src/com/android/server/wm/flicker/assertions/AssertionData.kt b/libraries/flicker/src/com/android/server/wm/flicker/assertions/AssertionData.kt
index cbb4f11..1673c55 100644
--- a/libraries/flicker/src/com/android/server/wm/flicker/assertions/AssertionData.kt
+++ b/libraries/flicker/src/com/android/server/wm/flicker/assertions/AssertionData.kt
@@ -43,7 +43,10 @@
* @param run Run to be asserted
*/
fun checkAssertion(run: FlickerRunResult) {
- val subjects = run.getSubjects().firstOrNull { expectedSubjectClass.isInstance(it) }
- subjects?.run { assertion(this) }
+ val subjects = run.getSubjects(tag).filter { expectedSubjectClass.isInstance(it) }
+ if (subjects.isEmpty()) {
+ return
+ }
+ subjects.forEach { it.run { assertion(this) } }
}
}
diff --git a/libraries/flicker/src/com/android/server/wm/flicker/assertions/FlickerAssertionError.kt b/libraries/flicker/src/com/android/server/wm/flicker/assertions/FlickerAssertionError.kt
index 1dceb87..cf4200f 100644
--- a/libraries/flicker/src/com/android/server/wm/flicker/assertions/FlickerAssertionError.kt
+++ b/libraries/flicker/src/com/android/server/wm/flicker/assertions/FlickerAssertionError.kt
@@ -16,11 +16,11 @@
package com.android.server.wm.flicker.assertions
-import com.android.server.wm.flicker.TraceFile
+import com.android.server.wm.flicker.RunResultArtifacts
import kotlin.AssertionError
class FlickerAssertionError(
message: String,
cause: Throwable?,
- val traceFile: TraceFile?
+ val traceFile: RunResultArtifacts?
) : AssertionError(message, cause)
diff --git a/libraries/flicker/src/com/android/server/wm/flicker/assertions/FlickerAssertionErrorBuilder.kt b/libraries/flicker/src/com/android/server/wm/flicker/assertions/FlickerAssertionErrorBuilder.kt
index 2768706..b046636 100644
--- a/libraries/flicker/src/com/android/server/wm/flicker/assertions/FlickerAssertionErrorBuilder.kt
+++ b/libraries/flicker/src/com/android/server/wm/flicker/assertions/FlickerAssertionErrorBuilder.kt
@@ -16,7 +16,7 @@
package com.android.server.wm.flicker.assertions
-import com.android.server.wm.flicker.TraceFile
+import com.android.server.wm.flicker.RunResultArtifacts
import com.android.server.wm.flicker.dsl.AssertionTag
import com.android.server.wm.flicker.traces.FlickerSubjectException
import com.google.common.truth.Fact
@@ -25,14 +25,14 @@
class FlickerAssertionErrorBuilder {
private var error: Throwable? = null
- private var traceFile: TraceFile? = null
+ private var traceFile: RunResultArtifacts? = null
private var tag = ""
fun fromError(error: Throwable): FlickerAssertionErrorBuilder = apply {
this.error = error
}
- fun withTrace(traceFile: TraceFile?): FlickerAssertionErrorBuilder = apply {
+ fun withTrace(traceFile: RunResultArtifacts?): FlickerAssertionErrorBuilder = apply {
this.traceFile = traceFile
}
@@ -73,7 +73,7 @@
}
private val traceFileMessage get() = buildString {
- traceFile?.traceFile?.let {
+ traceFile?.path?.let {
append("\t")
append(it)
}
diff --git a/libraries/flicker/src/com/android/server/wm/flicker/dsl/FlickerBuilder.kt b/libraries/flicker/src/com/android/server/wm/flicker/dsl/FlickerBuilder.kt
index 784abaf..8ada4e2 100644
--- a/libraries/flicker/src/com/android/server/wm/flicker/dsl/FlickerBuilder.kt
+++ b/libraries/flicker/src/com/android/server/wm/flicker/dsl/FlickerBuilder.kt
@@ -290,6 +290,10 @@
traceMonitors.add(TransactionsTraceMonitor(outputDir))
}
+ require(testName.isNotEmpty()) {
+ "Test name must be provided by calling .withTestName {} on builder"
+ }
+
return Flicker(
instrumentation,
device,
diff --git a/libraries/flicker/src/com/android/server/wm/flicker/helpers/ClearableLazyDelegate.kt b/libraries/flicker/src/com/android/server/wm/flicker/helpers/ClearableLazyDelegate.kt
new file mode 100644
index 0000000..c385ee7
--- /dev/null
+++ b/libraries/flicker/src/com/android/server/wm/flicker/helpers/ClearableLazyDelegate.kt
@@ -0,0 +1,43 @@
+/*
+ * Copyright (C) 2022 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.android.server.wm.flicker.helpers
+
+import kotlin.reflect.KProperty
+
+fun <T> clearableLazy(initializer: () -> T) = ClearableLazyDelegate(initializer)
+
+/**
+ * NOTE: This implementation is not thread safe
+ */
+class ClearableLazyDelegate<T>(private val initializer: () -> T) {
+ private var uninitialized = true
+ private var value: T? = null
+
+ operator fun getValue(thisRef: Any?, property: KProperty<*>): T {
+ if (uninitialized) {
+ value = initializer()
+ uninitialized = false
+ }
+ return value!!
+ }
+
+ operator fun setValue(thisRef: Any?, property: KProperty<*>, value: T) {
+ require(value === null) { "Can only set to null to clear memory" }
+ this.value = null
+ uninitialized = true
+ }
+}
diff --git a/libraries/flicker/src/com/android/server/wm/flicker/monitor/EventLogMonitor.kt b/libraries/flicker/src/com/android/server/wm/flicker/monitor/EventLogMonitor.kt
index ac639be..387821e 100644
--- a/libraries/flicker/src/com/android/server/wm/flicker/monitor/EventLogMonitor.kt
+++ b/libraries/flicker/src/com/android/server/wm/flicker/monitor/EventLogMonitor.kt
@@ -71,8 +71,8 @@
_logs = getEventLogs(EVENT_LOG_INPUT_FOCUS_TAG)
}
- override fun setResult(builder: FlickerRunResult.Builder) {
- builder.eventLog = buildProcessedEventLogs()
+ override fun setResult(result: FlickerRunResult) {
+ result.eventLog = buildProcessedEventLogs()
}
private fun buildProcessedEventLogs(): List<FocusEvent> {
diff --git a/libraries/flicker/src/com/android/server/wm/flicker/monitor/LayersTraceMonitor.kt b/libraries/flicker/src/com/android/server/wm/flicker/monitor/LayersTraceMonitor.kt
index 4d88494..b40009a 100644
--- a/libraries/flicker/src/com/android/server/wm/flicker/monitor/LayersTraceMonitor.kt
+++ b/libraries/flicker/src/com/android/server/wm/flicker/monitor/LayersTraceMonitor.kt
@@ -16,15 +16,11 @@
package com.android.server.wm.flicker.monitor
-import android.util.Log
import android.view.WindowManagerGlobal
-import com.android.server.wm.flicker.FLICKER_TAG
import com.android.server.wm.flicker.FlickerRunResult
import com.android.server.wm.flicker.getDefaultFlickerOutputDir
import com.android.server.wm.flicker.traces.layers.LayersTraceSubject
import com.android.server.wm.traces.common.layers.LayersTrace
-import com.android.server.wm.traces.parser.layers.LayersTraceParser
-import java.nio.file.Files
import java.nio.file.Path
/**
@@ -52,13 +48,8 @@
override val isEnabled: Boolean
get() = windowManager.isLayerTracing
- override fun setResult(builder: FlickerRunResult.Builder) {
- builder.setLayersTrace(outputFile) {
- Log.v(FLICKER_TAG, "Parsing Layers trace")
- val traceData = Files.readAllBytes(outputFile)
- val layersTrace = LayersTraceParser.parseFromTrace(traceData)
- LayersTraceSubject.assertThat(layersTrace)
- }
+ override fun setResult(result: FlickerRunResult) {
+ result.setLayersTrace(outputFile.toFile())
}
companion object {
diff --git a/libraries/flicker/src/com/android/server/wm/flicker/monitor/ScreenRecorder.kt b/libraries/flicker/src/com/android/server/wm/flicker/monitor/ScreenRecorder.kt
index 12ff42c..289b6b8 100644
--- a/libraries/flicker/src/com/android/server/wm/flicker/monitor/ScreenRecorder.kt
+++ b/libraries/flicker/src/com/android/server/wm/flicker/monitor/ScreenRecorder.kt
@@ -133,8 +133,8 @@
override val isEnabled: Boolean
get() = mediaRecorder != null
- override fun setResult(builder: FlickerRunResult.Builder) {
- builder.screenRecording = outputFile
+ override fun setResult(result: FlickerRunResult) {
+ result.setScreenRecording(outputFile.toFile())
}
override fun toString(): String {
diff --git a/libraries/flicker/src/com/android/server/wm/flicker/monitor/TransactionsTraceMonitor.kt b/libraries/flicker/src/com/android/server/wm/flicker/monitor/TransactionsTraceMonitor.kt
index 250638b..008a621 100644
--- a/libraries/flicker/src/com/android/server/wm/flicker/monitor/TransactionsTraceMonitor.kt
+++ b/libraries/flicker/src/com/android/server/wm/flicker/monitor/TransactionsTraceMonitor.kt
@@ -16,15 +16,11 @@
package com.android.server.wm.flicker.monitor
-import android.util.Log
import android.view.WindowManagerGlobal
-import com.android.server.wm.flicker.FLICKER_TAG
import com.android.server.wm.flicker.FlickerRunResult
import com.android.server.wm.flicker.getDefaultFlickerOutputDir
import com.android.server.wm.flicker.traces.layers.LayersTraceSubject
import com.android.server.wm.traces.common.layers.LayersTrace
-import com.android.server.wm.traces.parser.transaction.TransactionsTraceParser
-import java.nio.file.Files
import java.nio.file.Path
/**
@@ -50,11 +46,7 @@
override val isEnabled: Boolean
get() = true
- override fun setResult(builder: FlickerRunResult.Builder) {
- builder.setTransactionsTrace(outputFile) {
- Log.v(FLICKER_TAG, "Parsing Transactions trace")
- val traceData = Files.readAllBytes(outputFile)
- TransactionsTraceParser.parseFromTrace(traceData)
- }
+ override fun setResult(result: FlickerRunResult) {
+ result.setTransactionsTrace(outputFile.toFile())
}
}
diff --git a/libraries/flicker/src/com/android/server/wm/flicker/monitor/TransitionMonitor.kt b/libraries/flicker/src/com/android/server/wm/flicker/monitor/TransitionMonitor.kt
index bc3589c..0229e41 100644
--- a/libraries/flicker/src/com/android/server/wm/flicker/monitor/TransitionMonitor.kt
+++ b/libraries/flicker/src/com/android/server/wm/flicker/monitor/TransitionMonitor.kt
@@ -16,7 +16,6 @@
package com.android.server.wm.flicker.monitor
-import com.android.server.wm.flicker.FlickerRunResult
import java.nio.file.Files
import java.nio.file.Path
@@ -42,9 +41,6 @@
this.stop()
}
- val builder = FlickerRunResult.Builder()
- builder.setResultFrom(this)
-
return outputFile.let {
Files.readAllBytes(it).also { _ -> Files.delete(it) }
} ?: error("Unable to acquire trace")
diff --git a/libraries/flicker/src/com/android/server/wm/flicker/monitor/TransitionsTraceMonitor.kt b/libraries/flicker/src/com/android/server/wm/flicker/monitor/TransitionsTraceMonitor.kt
index 027f309..d0468fd 100644
--- a/libraries/flicker/src/com/android/server/wm/flicker/monitor/TransitionsTraceMonitor.kt
+++ b/libraries/flicker/src/com/android/server/wm/flicker/monitor/TransitionsTraceMonitor.kt
@@ -16,15 +16,11 @@
package com.android.server.wm.flicker.monitor
-import android.util.Log
import android.view.WindowManagerGlobal
-import com.android.server.wm.flicker.FLICKER_TAG
import com.android.server.wm.flicker.FlickerRunResult
import com.android.server.wm.flicker.getDefaultFlickerOutputDir
import com.android.server.wm.flicker.traces.layers.LayersTraceSubject
import com.android.server.wm.traces.common.layers.LayersTrace
-import com.android.server.wm.traces.parser.transition.TransitionsTraceParser
-import java.nio.file.Files
import java.nio.file.Path
/**
@@ -50,11 +46,7 @@
override val isEnabled: Boolean
get() = windowManager.isTransitionTraceEnabled
- override fun setResult(builder: FlickerRunResult.Builder) {
- builder.setTransitionsTrace(outputFile) {
- Log.v(FLICKER_TAG, "Parsing Transition trace")
- val traceData = Files.readAllBytes(outputFile)
- TransitionsTraceParser.parseFromTrace(traceData)
- }
+ override fun setResult(result: FlickerRunResult) {
+ result.setTransitionsTrace(outputFile.toFile())
}
}
diff --git a/libraries/flicker/src/com/android/server/wm/flicker/monitor/WindowManagerTraceMonitor.kt b/libraries/flicker/src/com/android/server/wm/flicker/monitor/WindowManagerTraceMonitor.kt
index e6aba82..51742b6 100644
--- a/libraries/flicker/src/com/android/server/wm/flicker/monitor/WindowManagerTraceMonitor.kt
+++ b/libraries/flicker/src/com/android/server/wm/flicker/monitor/WindowManagerTraceMonitor.kt
@@ -16,15 +16,11 @@
package com.android.server.wm.flicker.monitor
-import android.util.Log
import android.view.WindowManagerGlobal
-import com.android.server.wm.flicker.FLICKER_TAG
import com.android.server.wm.flicker.FlickerRunResult
import com.android.server.wm.flicker.getDefaultFlickerOutputDir
import com.android.server.wm.flicker.traces.windowmanager.WindowManagerTraceSubject
import com.android.server.wm.traces.common.windowmanager.WindowManagerTrace
-import com.android.server.wm.traces.parser.windowmanager.WindowManagerTraceParser
-import java.nio.file.Files
import java.nio.file.Path
/**
@@ -48,12 +44,7 @@
override val isEnabled: Boolean
get() = windowManager.isWindowTraceEnabled
- override fun setResult(builder: FlickerRunResult.Builder) {
- builder.setWmTrace(outputFile) {
- Log.v(FLICKER_TAG, "Parsing WM trace")
- val traceData = Files.readAllBytes(outputFile)
- val wmTrace = WindowManagerTraceParser.parseFromTrace(traceData)
- WindowManagerTraceSubject.assertThat(wmTrace)
- }
+ override fun setResult(result: FlickerRunResult) {
+ result.setWmTrace(outputFile.toFile())
}
}
diff --git a/libraries/flicker/src/com/android/server/wm/flicker/service/FlickerServiceTracesCollector.kt b/libraries/flicker/src/com/android/server/wm/flicker/service/FlickerServiceTracesCollector.kt
index 1b2269d..094ea98 100644
--- a/libraries/flicker/src/com/android/server/wm/flicker/service/FlickerServiceTracesCollector.kt
+++ b/libraries/flicker/src/com/android/server/wm/flicker/service/FlickerServiceTracesCollector.kt
@@ -68,9 +68,6 @@
FlickerService.getFassFilePath(outputDir, "transition_trace"),
FlickerService.getFassFilePath(outputDir, "transactions_trace")
)
- transactionsTrace = getTransactionsTraceFromFile(
- FlickerService.getFassFilePath(outputDir, "transactions_trace")
- )
}
override fun getCollectedTraces(): Traces {
@@ -78,9 +75,7 @@
val layersTrace = layersTrace ?: error("Make sure tracing was stopped before calling this")
val transitionsTrace = transitionsTrace
?: error("Make sure tracing was stopped before calling this")
- val transactionsTrace = transactionsTrace
- ?: error("Make sure tracing was stopped before calling this")
- return Traces(wmTrace, layersTrace, transitionsTrace, transactionsTrace)
+ return Traces(wmTrace, layersTrace, transitionsTrace)
}
private fun reset() {
diff --git a/libraries/flicker/src/com/android/server/wm/flicker/service/FlickerTestCase.kt b/libraries/flicker/src/com/android/server/wm/flicker/service/FlickerTestCase.kt
index 9615dac..ca9c502 100644
--- a/libraries/flicker/src/com/android/server/wm/flicker/service/FlickerTestCase.kt
+++ b/libraries/flicker/src/com/android/server/wm/flicker/service/FlickerTestCase.kt
@@ -18,12 +18,14 @@
import com.android.server.wm.flicker.service.assertors.AssertionResult
import junit.framework.Assert
-import org.junit.Test
class FlickerTestCase(val results: List<AssertionResult>) {
- @Test
- fun runTest(param: Any) {
+ // Used by the FlickerBlockJUnit4ClassRunner to identify the test method within this class
+ annotation class InjectedTest
+
+ @InjectedTest
+ fun injectedTest(param: Any) {
if (containsFailures) {
Assert.fail(assertionMessage)
}
diff --git a/libraries/flicker/src/com/android/server/wm/traces/common/transactions/Transaction.kt b/libraries/flicker/src/com/android/server/wm/traces/common/transactions/Transaction.kt
index 2cbee36..c39b865 100644
--- a/libraries/flicker/src/com/android/server/wm/traces/common/transactions/Transaction.kt
+++ b/libraries/flicker/src/com/android/server/wm/traces/common/transactions/Transaction.kt
@@ -16,10 +16,15 @@
package com.android.server.wm.traces.common.transactions
-class Transaction(
+data class Transaction(
val pid: Int,
val uid: Int,
val vSyncId: Long,
val postTime: Long,
val id: Long,
-)
+) {
+ override fun toString(): String {
+ return "Transaction#${hashCode()}" +
+ "(pid=$pid, uid=$uid, vSyncId=$vSyncId, postTime=$postTime, id=$id)"
+ }
+}
diff --git a/libraries/flicker/test/src/com/android/server/wm/flicker/EventLogSubjectTest.kt b/libraries/flicker/test/src/com/android/server/wm/flicker/EventLogSubjectTest.kt
index b4d12f8..ec7e984 100644
--- a/libraries/flicker/test/src/com/android/server/wm/flicker/EventLogSubjectTest.kt
+++ b/libraries/flicker/test/src/com/android/server/wm/flicker/EventLogSubjectTest.kt
@@ -16,7 +16,6 @@
package com.android.server.wm.flicker
-import com.android.server.wm.flicker.FlickerRunResult.Companion.RunStatus.RUN_SUCCESS
import com.android.server.wm.flicker.traces.eventlog.EventLogSubject
import com.android.server.wm.flicker.traces.eventlog.FocusEvent
import org.junit.Test
@@ -28,13 +27,13 @@
class EventLogSubjectTest {
@Test
fun canDetectFocusChanges() {
- val builder = FlickerRunResult.Builder()
- builder.eventLog =
+ val runResult = FlickerRunResult("testName", 0 /*iteration*/)
+ runResult.eventLog =
listOf(FocusEvent(0, "WinB", FocusEvent.Focus.GAINED, "test"),
FocusEvent(0, "test WinA window", FocusEvent.Focus.LOST, "test"),
FocusEvent(0, "WinB", FocusEvent.Focus.LOST, "test"),
FocusEvent(0, "test WinC", FocusEvent.Focus.GAINED, "test"))
- val result = builder.buildEventLogResult(RUN_SUCCESS).eventLogSubject
+ val result = runResult.eventLogSubject
requireNotNull(result) { "Event log subject was not built" }
result.focusChanges("WinA", "WinB", "WinC")
.forAllEntries()
@@ -47,9 +46,10 @@
@Test
fun canDetectFocusDoesNotChange() {
- val builder = FlickerRunResult.Builder()
- val result = builder.buildEventLogResult(RUN_SUCCESS).eventLogSubject
- require(result != null) { "Event log subject was not built" }
+ val runResult = FlickerRunResult("testName", 0 /*iteration*/)
+ runResult.eventLog = emptyList()
+ val result = runResult.eventLogSubject
+ requireNotNull(result) { "Event log subject was not built" }
result.focusDoesNotChange().forAllEntries()
}
}
diff --git a/libraries/flicker/test/src/com/android/server/wm/flicker/FlickerDSLTest.kt b/libraries/flicker/test/src/com/android/server/wm/flicker/FlickerDSLTest.kt
index 4859718..f4e4d8b 100644
--- a/libraries/flicker/test/src/com/android/server/wm/flicker/FlickerDSLTest.kt
+++ b/libraries/flicker/test/src/com/android/server/wm/flicker/FlickerDSLTest.kt
@@ -38,13 +38,15 @@
import com.android.server.wm.flicker.traces.windowmanager.WindowManagerStateSubject
import com.android.server.wm.flicker.traces.windowmanager.WindowManagerTraceSubject
import com.google.common.truth.Truth
+import java.lang.RuntimeException
+import kotlin.reflect.KClass
import org.junit.Assert
import org.junit.Before
import org.junit.FixMethodOrder
import org.junit.Test
import org.junit.runners.MethodSorters
-import java.lang.RuntimeException
-import kotlin.reflect.KClass
+
+private const val TEST_NAME = "FlickerDSLTest"
/**
* Contains [Flicker] and [FlickerBuilder] tests.
@@ -99,7 +101,9 @@
@Test
fun checkBuiltLayersEndAssertion() {
- val assertion = FlickerTestParameter.buildLayersEndAssertion { executed = true }
+ val assertion = FlickerTestParameter.buildLayersEndAssertion {
+ executed = true
+ }
validateAssertion(assertion, LayerTraceEntrySubject::class, AssertionTag.END)
runAndAssertExecuted(assertion)
}
@@ -139,12 +143,12 @@
this.device.pressHome()
}
}
- }
+ }.withTestName { TEST_NAME }
val flicker = builder.build().execute()
flicker.checkAssertion(assertion)
- Truth.assertWithMessage("Should have asserted $TAG 2x")
+ Truth.assertWithMessage("Should have asserted $TAG twice")
.that(count)
.isEqualTo(2)
}
@@ -156,8 +160,10 @@
transitions {
this.createTag("inv lid")
}
- }
- builder.build().execute()
+ }.withTestName { TEST_NAME }
+ val flicker = builder.build()
+ flicker.execute()
+ flicker.result!!.checkForExecutionErrors()
Assert.fail("Should not have allowed invalid tag name")
} catch (e: Throwable) {
Truth.assertWithMessage("Did not validate tag name")
@@ -173,7 +179,7 @@
this.createTag(TAG)
device.pressHome()
}
- }
+ }.withTestName { TEST_NAME }
val flicker = builder.build()
val passAssertion = FlickerTestParameter.buildWMTagAssertion(TAG) {
this.isNotEmpty()
@@ -189,7 +195,7 @@
@Test
fun detectEmptyResults() {
try {
- FlickerBuilder(instrumentation).build().execute()
+ FlickerBuilder(instrumentation).withTestName { TEST_NAME }.build().execute()
Assert.fail("Should not have allowed empty transition")
} catch (e: Throwable) {
Truth.assertWithMessage("Flicker did not warn of empty transitions")
@@ -201,12 +207,13 @@
@Test
fun detectCrashedTransition() {
val exceptionMessage = "Crashed transition"
- val builder = FlickerBuilder(instrumentation)
+ val builder = FlickerBuilder(instrumentation).withTestName { TEST_NAME }
builder.transitions { error("Crashed transition") }
val flicker = builder.build()
try {
flicker.execute()
- Assert.fail("Should have raised an exception with message $exceptionMessage")
+ flicker.result!!.checkForExecutionErrors()
+ Assert.fail("Should have raised an exception with message :: $exceptionMessage")
} catch (e: Throwable) {
Truth.assertWithMessage("Incorrect exception type")
.that(e)
@@ -219,7 +226,7 @@
@Test
fun exceptionContainsDebugInfo() {
- val builder = FlickerBuilder(instrumentation)
+ val builder = FlickerBuilder(instrumentation).withTestName { TEST_NAME }
builder.transitions { device.pressHome() }
val flicker = builder.build()
flicker.execute()
@@ -239,7 +246,7 @@
@Test
fun canDetectTestSetupExecutionError() {
- val builder = FlickerBuilder(instrumentation)
+ val builder = FlickerBuilder(instrumentation).withTestName { TEST_NAME }
builder.transitions(SIMPLE_TRANSITION)
builder.setup {
test {
@@ -252,7 +259,7 @@
@Test
fun canDetectTransitionSetupExecutionError() {
- val builder = FlickerBuilder(instrumentation)
+ val builder = FlickerBuilder(instrumentation).withTestName { TEST_NAME }
builder.transitions(SIMPLE_TRANSITION)
builder.setup {
eachRun {
@@ -265,7 +272,7 @@
@Test
fun canDetectTransitionExecutionError() {
- val builder = FlickerBuilder(instrumentation)
+ val builder = FlickerBuilder(instrumentation).withTestName { TEST_NAME }
builder.transitions {
throw RuntimeException("Failed to execute transition")
}
@@ -275,7 +282,7 @@
@Test
fun canDetectTransitionTeardownExecutionError() {
- val builder = FlickerBuilder(instrumentation)
+ val builder = FlickerBuilder(instrumentation).withTestName { TEST_NAME }
builder.transitions(SIMPLE_TRANSITION)
builder.teardown {
eachRun {
@@ -288,7 +295,7 @@
@Test
fun canDetectTestTeardownExecutionError() {
- val builder = FlickerBuilder(instrumentation)
+ val builder = FlickerBuilder(instrumentation).withTestName { TEST_NAME }
builder.transitions(SIMPLE_TRANSITION)
builder.teardown {
test {
@@ -302,7 +309,7 @@
@Test
fun runsAssertionsOnSuccessfulTransitionsEvenIfSomeFailToExecute() {
val repetitions = 3
- val builder = FlickerBuilder(instrumentation)
+ val builder = FlickerBuilder(instrumentation).withTestName { TEST_NAME }
failOnLastTransitionRun(builder, repetitions)
var assertionExecutionCounter = 0
val assertions = listOf(
@@ -327,7 +334,7 @@
@Test
fun canHandleAndTrackMultipleExecutionErrors() {
val repetitions = 2
- val builder = FlickerBuilder(instrumentation)
+ val builder = FlickerBuilder(instrumentation).withTestName { TEST_NAME }
failOnLastTransitionRun(builder, repetitions)
builder.teardown {
test {
@@ -362,7 +369,7 @@
@Test
fun savesTracesOfFailedTransitionExecution() {
- val builder = FlickerBuilder(instrumentation)
+ val builder = FlickerBuilder(instrumentation).withTestName { TEST_NAME }
builder.transitions {
throw RuntimeException("Failed to execute transition")
}
@@ -374,7 +381,8 @@
} catch (e: TransitionExecutionFailure) {
// A TransitionExecutionFailure is expected
}
- assertArchiveContainsAllTraces(runStatus = FlickerRunResult.Companion.RunStatus.RUN_FAILED)
+ assertArchiveContainsAllTraces(runStatus = FlickerRunResult.Companion.RunStatus.RUN_FAILED,
+ testName = TEST_NAME)
}
@Test
@@ -382,6 +390,7 @@
val runner = TransitionRunner()
val repetitions = 5
val flicker = FlickerBuilder(instrumentation)
+ .withTestName { TEST_NAME }
.apply {
transitions {}
}
@@ -392,7 +401,8 @@
for (iteration in 0 until repetitions) {
assertArchiveContainsAllTraces(
runStatus = ASSERTION_SUCCESS,
- iteration = iteration
+ iteration = iteration,
+ testName = TEST_NAME
)
}
}
@@ -432,6 +442,7 @@
private fun checkTracesAreSavedWithAssertionFailure(assertion: AssertionData) {
val runner = TransitionRunner()
val flicker = FlickerBuilder(instrumentation)
+ .withTestName { TEST_NAME }
.apply {
transitions {}
}
@@ -439,12 +450,12 @@
runAndAssertFlickerFailsWithException(flicker, FlickerAssertionError::class.java,
listOf(assertion))
- assertArchiveContainsAllTraces(runStatus = ASSERTION_FAILED)
+ assertArchiveContainsAllTraces(runStatus = ASSERTION_FAILED, testName = TEST_NAME)
}
private fun runAndAssertExecuted(assertion: AssertionData) {
executed = false
- val builder = FlickerBuilder(instrumentation)
+ val builder = FlickerBuilder(instrumentation).withTestName { TEST_NAME }
builder.transitions(SIMPLE_TRANSITION)
val flicker = builder.build()
runFlicker(flicker, assertion)
@@ -468,6 +479,8 @@
for (assertion in assertions) {
flicker.checkAssertion(assertion)
}
+ // Execution errors are reported in the FlickerBlockJUnit4ClassRunner after each test
+ flicker.result!!.checkForExecutionErrors()
flicker.clear()
}
diff --git a/libraries/flicker/test/src/com/android/server/wm/flicker/TransitionRunnerTest.kt b/libraries/flicker/test/src/com/android/server/wm/flicker/TransitionRunnerTest.kt
index b819591..8b3ace7 100644
--- a/libraries/flicker/test/src/com/android/server/wm/flicker/TransitionRunnerTest.kt
+++ b/libraries/flicker/test/src/com/android/server/wm/flicker/TransitionRunnerTest.kt
@@ -22,6 +22,7 @@
import com.android.server.wm.flicker.FlickerRunResult.Companion.RunStatus
import com.android.server.wm.flicker.dsl.FlickerBuilder
import com.google.common.truth.Truth
+import java.lang.RuntimeException
import org.junit.After
import org.junit.Before
import org.junit.FixMethodOrder
@@ -29,7 +30,8 @@
import org.junit.runner.RunWith
import org.junit.runners.MethodSorters
import org.mockito.junit.MockitoJUnitRunner
-import java.lang.RuntimeException
+
+private const val TEST_NAME = "TransitionRunnerTest"
/**
* Contains [TransitionRunner] tests.
@@ -59,7 +61,10 @@
fun canRunTransition() {
val runner = TransitionRunner()
var executed = false
+ val iterations = 1
val flicker = FlickerBuilder(instrumentation)
+ .withTestName { TEST_NAME }
+ .repeat { iterations }
.apply {
transitions {
executed = true
@@ -70,13 +75,14 @@
runner.cleanUp()
Truth.assertThat(executed).isTrue()
Truth.assertThat(result.executionErrors).isEmpty()
- Truth.assertThat(result.successfulRuns).hasSize(4)
+ Truth.assertThat(result.successfulRuns).hasSize(iterations)
}
@Test
fun storesTransitionExecutionErrors() {
val runner = TransitionRunner()
val flicker = FlickerBuilder(instrumentation)
+ .withTestName { TEST_NAME }
.apply {
transitions {
throw RuntimeException("Failed to execute transition")
@@ -94,6 +100,7 @@
val runner = TransitionRunner()
val flicker = FlickerBuilder(instrumentation)
+ .withTestName { TEST_NAME }
.apply {
transitions {
transitionRunCounter++
@@ -108,7 +115,7 @@
Truth.assertThat(result.executionErrors).isNotEmpty()
// One for each monitor for each repetition expect the last one
// for which the transition failed to execute
- val expectedResultCount = flicker.traceMonitors.size * (repetitions - 1)
+ val expectedResultCount = repetitions - 1
Truth.assertThat(result.successfulRuns.size).isEqualTo(expectedResultCount)
}
@@ -116,6 +123,7 @@
fun storesSuccessExecutionStatusInRunResult() {
val runner = TransitionRunner()
val flicker = FlickerBuilder(instrumentation)
+ .withTestName { TEST_NAME }
.apply {
transitions {}
}.repeat { 3 }.build(runner)
@@ -129,6 +137,7 @@
fun storesFailedExecutionStatusInRunResult() {
val runner = TransitionRunner()
val flicker = FlickerBuilder(instrumentation)
+ .withTestName { TEST_NAME }
.apply {
transitions {
throw RuntimeException("Failed to execute transition")
@@ -144,6 +153,7 @@
fun savesTraceOnTransitionExecutionErrors() {
val runner = TransitionRunner()
val flicker = FlickerBuilder(instrumentation)
+ .withTestName { TEST_NAME }
.apply {
transitions {
throw Throwable()
@@ -152,13 +162,14 @@
.build(runner)
runner.execute(flicker)
- assertArchiveContainsAllTraces(runStatus = RunStatus.RUN_FAILED)
+ assertArchiveContainsAllTraces(runStatus = RunStatus.RUN_FAILED, testName = TEST_NAME)
}
@Test
fun savesTraceOnRunCleanupErrors() {
val runner = TransitionRunner()
val flicker = FlickerBuilder(instrumentation)
+ .withTestName { TEST_NAME }
.apply {
transitions {}
teardown {
@@ -170,13 +181,14 @@
.build(runner)
runner.execute(flicker)
- assertArchiveContainsAllTraces(runStatus = RunStatus.RUN_FAILED)
+ assertArchiveContainsAllTraces(runStatus = RunStatus.RUN_FAILED, testName = TEST_NAME)
}
@Test
fun savesTraceOnTestCleanupErrors() {
val runner = TransitionRunner()
val flicker = FlickerBuilder(instrumentation)
+ .withTestName { TEST_NAME }
.apply {
transitions {}
teardown {
@@ -188,6 +200,6 @@
.build(runner)
runner.execute(flicker)
- assertArchiveContainsAllTraces(runStatus = RunStatus.RUN_FAILED)
+ assertArchiveContainsAllTraces(runStatus = RunStatus.RUN_FAILED, testName = TEST_NAME)
}
}
diff --git a/libraries/flicker/test/src/com/android/server/wm/flicker/Utils.kt b/libraries/flicker/test/src/com/android/server/wm/flicker/Utils.kt
index 245093b..4ce10a6 100644
--- a/libraries/flicker/test/src/com/android/server/wm/flicker/Utils.kt
+++ b/libraries/flicker/test/src/com/android/server/wm/flicker/Utils.kt
@@ -117,7 +117,7 @@
fun assertArchiveContainsAllTraces(
runStatus: FlickerRunResult.Companion.RunStatus = ASSERTION_SUCCESS,
- testName: String = "",
+ testName: String,
iteration: Int = 0
) {
val archiveFileName = "${runStatus.prefix}_${testName}_$iteration.zip"
diff --git a/libraries/flicker/test/src/com/android/server/wm/flicker/monitor/EventLogMonitorTest.kt b/libraries/flicker/test/src/com/android/server/wm/flicker/monitor/EventLogMonitorTest.kt
index 65f64b0..e224625 100644
--- a/libraries/flicker/test/src/com/android/server/wm/flicker/monitor/EventLogMonitorTest.kt
+++ b/libraries/flicker/test/src/com/android/server/wm/flicker/monitor/EventLogMonitorTest.kt
@@ -42,21 +42,21 @@
"com.android.phone.settings.fdn.FdnSetting (server)",
"reason=test")
- val resultBuilder = FlickerRunResult.Builder()
- resultBuilder.setResultFrom(monitor)
+ val result = FlickerRunResult("testName", 0 /*iteration*/)
+ result.setResultsFromMonitor(monitor)
- assertEquals(2, resultBuilder.eventLog?.size)
+ assertEquals(2, result.eventLog?.size)
assertEquals(
"4749f88 com.android.phone/com.android.phone.settings.fdn.FdnSetting (server)",
- resultBuilder.eventLog?.get(0)?.window)
- assertEquals(FocusEvent.Focus.LOST, resultBuilder.eventLog?.get(0)?.focus)
+ result.eventLog?.get(0)?.window)
+ assertEquals(FocusEvent.Focus.LOST, result.eventLog?.get(0)?.focus)
assertEquals(
"7c01447 com.android.phone/com.android.phone.settings.fdn.FdnSetting (server)",
- resultBuilder.eventLog?.get(1)?.window)
- assertEquals(FocusEvent.Focus.GAINED, resultBuilder.eventLog?.get(1)?.focus)
- assertTrue(resultBuilder.eventLog?.get(0)?.timestamp ?: 0
- <= resultBuilder.eventLog?.get(1)?.timestamp ?: 0)
- assertEquals(resultBuilder.eventLog?.get(0)?.reason, "test")
+ result.eventLog?.get(1)?.window)
+ assertEquals(FocusEvent.Focus.GAINED, result.eventLog?.get(1)?.focus)
+ assertTrue(result.eventLog?.get(0)?.timestamp ?: 0
+ <= result.eventLog?.get(1)?.timestamp ?: 0)
+ assertEquals(result.eventLog?.get(0)?.reason, "test")
}
@Test
@@ -84,21 +84,21 @@
"reason=test")
monitor.stop()
- val resultBuilder = FlickerRunResult.Builder()
- resultBuilder.setResultFrom(monitor)
+ val result = FlickerRunResult("testName", 0 /*iteration*/)
+ result.setResultsFromMonitor(monitor)
- assertEquals(2, resultBuilder.eventLog?.size)
+ assertEquals(2, result.eventLog?.size)
assertEquals("479f88 " +
"com.android.phone/" +
"com.android.phone.settings.fdn.FdnSetting (server)",
- resultBuilder.eventLog?.get(0)?.window)
- assertEquals(FocusEvent.Focus.LOST, resultBuilder.eventLog?.get(0)?.focus)
+ result.eventLog?.get(0)?.window)
+ assertEquals(FocusEvent.Focus.LOST, result.eventLog?.get(0)?.focus)
assertEquals("7c01447 com.android.phone/" +
"com.android.phone.settings.fdn.FdnSetting (server)",
- resultBuilder.eventLog?.get(1)?.window)
- assertEquals(FocusEvent.Focus.GAINED, resultBuilder.eventLog?.get(1)?.focus)
- assertTrue(resultBuilder.eventLog?.get(0)?.timestamp ?: 0
- <= resultBuilder.eventLog?.get(1)?.timestamp ?: 0)
+ result.eventLog?.get(1)?.window)
+ assertEquals(FocusEvent.Focus.GAINED, result.eventLog?.get(1)?.focus)
+ assertTrue(result.eventLog?.get(0)?.timestamp ?: 0
+ <= result.eventLog?.get(1)?.timestamp ?: 0)
}
@Test
@@ -119,21 +119,21 @@
"reason=test")
monitor.stop()
- val resultBuilder = FlickerRunResult.Builder()
- resultBuilder.setResultFrom(monitor)
+ val result = FlickerRunResult("testName", 0 /*iteration*/)
+ result.setResultsFromMonitor(monitor)
- assertEquals(2, resultBuilder.eventLog?.size)
+ assertEquals(2, result.eventLog?.size)
assertEquals(
"4749f88 com.android.phone/com.android.phone.settings.fdn.FdnSetting (server)",
- resultBuilder.eventLog?.get(0)?.window)
- assertEquals(FocusEvent.Focus.LOST, resultBuilder.eventLog?.get(0)?.focus)
+ result.eventLog?.get(0)?.window)
+ assertEquals(FocusEvent.Focus.LOST, result.eventLog?.get(0)?.focus)
assertEquals(
"7c01447 com.android.phone/com.android.phone.settings.fdn.FdnSetting (server)",
- resultBuilder.eventLog?.get(1)?.window)
- assertEquals(FocusEvent.Focus.GAINED, resultBuilder.eventLog?.get(1)?.focus)
- assertTrue(resultBuilder.eventLog?.get(0)?.timestamp ?: 0
- <= resultBuilder.eventLog?.get(1)?.timestamp ?: 0)
- assertEquals(resultBuilder.eventLog?.get(0)?.reason, "test")
+ result.eventLog?.get(1)?.window)
+ assertEquals(FocusEvent.Focus.GAINED, result.eventLog?.get(1)?.focus)
+ assertTrue(result.eventLog?.get(0)?.timestamp ?: 0
+ <= result.eventLog?.get(1)?.timestamp ?: 0)
+ assertEquals(result.eventLog?.get(0)?.reason, "test")
}
private companion object {
