Add androidx.benchmark.iterations argument for controlling measurement iters
Test: ./gradlew benchmark:benchmark-common:cC -P android.testInstrumentationRunnerArguments.androidx.benchmark.iterations=100 -P android.testInstrumentationRunnerArguments.class=androidx.benchmark.BenchmarkStateTest
Fixes: 194137879
Note: this doesn't modify warmup, which is still an arbitrary number
of iterations. The initial intent is to support profiling use cases
(which should use androidx.benchmark.profiling.*), or experimenting
locally when providing feedback on automatic iteration determination.
Change-Id: I1682ccff011f430da564cedf9113fb628b8d87b4
diff --git a/benchmark/benchmark-common/src/androidTest/java/androidx/benchmark/ArgumentInjectingApplication.kt b/benchmark/benchmark-common/src/androidTest/java/androidx/benchmark/ArgumentInjectingApplication.kt
index 6f2d759..3a06fab 100644
--- a/benchmark/benchmark-common/src/androidTest/java/androidx/benchmark/ArgumentInjectingApplication.kt
+++ b/benchmark/benchmark-common/src/androidTest/java/androidx/benchmark/ArgumentInjectingApplication.kt
@@ -39,6 +39,9 @@
super.onCreate()
argumentSource = Bundle().apply {
+ // allow cli args to pass through
+ putAll(InstrumentationRegistry.getArguments())
+
// Since these benchmark correctness tests run as part of the regular
// (non-performance-test) suite, they will have debuggable=true, won't be clock-locked,
// can run with low-battery or on an emulator, and code coverage enabled.
diff --git a/benchmark/benchmark-common/src/androidTest/java/androidx/benchmark/BenchmarkStateTest.kt b/benchmark/benchmark-common/src/androidTest/java/androidx/benchmark/BenchmarkStateTest.kt
index 0deb870..f05047f 100644
--- a/benchmark/benchmark-common/src/androidTest/java/androidx/benchmark/BenchmarkStateTest.kt
+++ b/benchmark/benchmark-common/src/androidTest/java/androidx/benchmark/BenchmarkStateTest.kt
@@ -17,7 +17,6 @@
package androidx.benchmark
import android.Manifest
-import android.util.Log
import androidx.benchmark.BenchmarkState.Companion.ExperimentalExternalReport
import androidx.test.ext.junit.runners.AndroidJUnit4
import androidx.test.filters.FlakyTest
@@ -29,6 +28,7 @@
import org.junit.Assert.assertNotNull
import org.junit.Assert.assertTrue
import org.junit.Assert.fail
+import org.junit.Assume.assumeTrue
import org.junit.Rule
import org.junit.Test
import org.junit.runner.RunWith
@@ -166,6 +166,10 @@
val expectedCount = report.warmupIterations + report.repeatIterations * expectedRepeatCount
assertEquals(expectedCount, total)
+ if (Arguments.iterations != null) {
+ assertEquals(Arguments.iterations, report.repeatIterations)
+ }
+
// verify we're not in warmup mode
assertTrue(report.warmupIterations > 0)
assertTrue(report.repeatIterations > 1)
@@ -180,16 +184,14 @@
@Test
public fun iterationCheck_withAllocations() {
- if (CpuInfo.locked ||
- IsolationActivity.sustainedPerformanceModeInUse ||
- Errors.isEmulator
- ) {
- // In any of these conditions, it's known that throttling won't happen, so it's safe
- // to check for allocation count, by setting checkingForThermalThrottling = false
- iterationCheck(checkingForThermalThrottling = false)
- } else {
- Log.d(BenchmarkState.TAG, "Warning - bypassing iterationCheck_withAllocations")
- }
+ // In any of these conditions, it's known that throttling won't happen, so it's safe
+ // to check for allocation count, by setting checkingForThermalThrottling = false
+ assumeTrue(
+ CpuInfo.locked ||
+ IsolationActivity.sustainedPerformanceModeInUse ||
+ Errors.isEmulator
+ )
+ iterationCheck(checkingForThermalThrottling = false)
}
@Test
diff --git a/benchmark/benchmark-common/src/main/java/androidx/benchmark/Arguments.kt b/benchmark/benchmark-common/src/main/java/androidx/benchmark/Arguments.kt
index c14a978..04d0004 100644
--- a/benchmark/benchmark-common/src/main/java/androidx/benchmark/Arguments.kt
+++ b/benchmark/benchmark-common/src/main/java/androidx/benchmark/Arguments.kt
@@ -42,6 +42,7 @@
internal val outputEnable: Boolean
internal val startupMode: Boolean
internal val dryRunMode: Boolean
+ internal val iterations: Int?
internal val profiler: Profiler?
internal val profilerSampleFrequency: Int
internal val profilerSampleDurationSeconds: Long
@@ -81,6 +82,9 @@
outputEnable = !dryRunMode &&
(arguments.getBenchmarkArgument("output.enable")?.toBoolean() ?: true)
+ iterations =
+ arguments.getBenchmarkArgument("iterations")?.toInt()
+
// Transform comma-delimited list into set of suppressed errors
// E.g. "DEBUGGABLE, UNLOCKED" -> setOf("DEBUGGABLE", "UNLOCKED")
suppressedErrors = arguments.getBenchmarkArgument("suppressErrors", "")
diff --git a/benchmark/benchmark-common/src/main/java/androidx/benchmark/BenchmarkState.kt b/benchmark/benchmark-common/src/main/java/androidx/benchmark/BenchmarkState.kt
index b7d6f56..518fbc7 100644
--- a/benchmark/benchmark-common/src/main/java/androidx/benchmark/BenchmarkState.kt
+++ b/benchmark/benchmark-common/src/main/java/androidx/benchmark/BenchmarkState.kt
@@ -440,11 +440,9 @@
}
private fun computeMaxIterations(): Int {
- var idealIterations =
- (REPEAT_DURATION_TARGET_NS / warmupManager.estimatedIterationTimeNs).toInt()
- idealIterations = idealIterations.coerceIn(MIN_TEST_ITERATIONS, MAX_TEST_ITERATIONS)
- OVERRIDE_ITERATIONS?.let { idealIterations = OVERRIDE_ITERATIONS }
- return idealIterations
+ return OVERRIDE_ITERATIONS
+ ?: (REPEAT_DURATION_TARGET_NS / warmupManager.estimatedIterationTimeNs).toInt()
+ .coerceIn(MIN_TEST_ITERATIONS, MAX_TEST_ITERATIONS)
}
private fun throwIfPaused() = check(!paused) {
@@ -547,11 +545,13 @@
internal const val REPEAT_COUNT_ALLOCATION = 5
- private val OVERRIDE_ITERATIONS = if (
+ private val OVERRIDE_ITERATIONS = when {
Arguments.dryRunMode ||
- Arguments.startupMode ||
- Arguments.profiler?.requiresSingleMeasurementIteration == true
- ) 1 else null
+ Arguments.startupMode ||
+ Arguments.profiler?.requiresSingleMeasurementIteration == true -> 1
+ Arguments.iterations != null -> Arguments.iterations
+ else -> null
+ }
internal val REPEAT_DURATION_TARGET_NS = when (Arguments.profiler?.requiresExtraRuntime) {
// longer measurements while profiling to ensure we have enough data
diff --git a/benchmark/benchmark-junit4/src/androidTest/java/androidx/benchmark/junit4/ArgumentInjectingApplication.kt b/benchmark/benchmark-junit4/src/androidTest/java/androidx/benchmark/junit4/ArgumentInjectingApplication.kt
index c8085ea..202b117 100644
--- a/benchmark/benchmark-junit4/src/androidTest/java/androidx/benchmark/junit4/ArgumentInjectingApplication.kt
+++ b/benchmark/benchmark-junit4/src/androidTest/java/androidx/benchmark/junit4/ArgumentInjectingApplication.kt
@@ -19,6 +19,7 @@
import android.app.Application
import android.os.Bundle
import androidx.benchmark.argumentSource
+import androidx.test.platform.app.InstrumentationRegistry
/**
* Hack to enable overriding benchmark arguments (since we can't easily do this in CI, per apk)
@@ -39,7 +40,8 @@
super.onCreate()
argumentSource = Bundle().apply {
- putString("androidx.benchmark.output.enable", "true")
+ // allow cli args to pass through
+ putAll(InstrumentationRegistry.getArguments())
// Since these benchmark correctness tests run as part of the regular
// (non-performance-test) suite, they will have debuggable=true, won't be clock-locked,
