Merge Android 24Q2 Release (ab/11526283) to aosp-main-future
Bug: 337098550
Merged-In: I4fade3c5f344cd7942273606986a3b43882e5441
Change-Id: I0743edca5f0b34aadb7eb58bfa3deca27d6b1159
diff --git a/report/src/com/android/catbox/result/JsonResultReporter.java b/report/src/com/android/catbox/result/JsonResultReporter.java
index 9363f9d..035cc6a 100644
--- a/report/src/com/android/catbox/result/JsonResultReporter.java
+++ b/report/src/com/android/catbox/result/JsonResultReporter.java
@@ -79,6 +79,12 @@
private Map<String, String> mReportTestNameMap = new HashMap<String, String>();
@Option(
+ name = "exclude-metric-from-test-name",
+ description = "Exclude metric from test name in report.")
+ private Map<String, String> mExcludeMetricTestNameMap = new HashMap<String, String>();
+
+
+ @Option(
name = "report-all-metrics",
description = "Report all the generated metrics. Default to 'true'.")
private boolean mReportAllMetrics = true;
@@ -209,7 +215,7 @@
writeAllMetrics(reportLog, metrics);
} else {
// Write metrics for given keys to the report
- writeMetricsForGivenKeys(reportLog, metrics);
+ writeMetricsForGivenKeys(reportLog, metrics, streamName);
}
// Submit Report Log
@@ -237,7 +243,7 @@
/** Write given set of metrics to JSON Report */
private void writeMetricsForGivenKeys(
- MetricsReportLog reportLog, Map<String, String> metrics) {
+ MetricsReportLog reportLog, Map<String, String> metrics, String streamName) {
CLog.logAndDisplay(LogLevel.INFO, "Writing given set of metrics to JSON report.");
if (mReportMetricKeyMap == null || mReportMetricKeyMap.isEmpty()) {
CLog.logAndDisplay(
@@ -249,6 +255,10 @@
CLog.logAndDisplay(LogLevel.WARN, String.format("%s metric key is missing.", key));
continue;
}
+ if(mExcludeMetricTestNameMap != null && mExcludeMetricTestNameMap.get(key) !=null && mExcludeMetricTestNameMap.get(key).equals(streamName)){
+ CLog.logAndDisplay(LogLevel.WARN, String.format("Excluding %s metric key from %s test.", key, streamName));
+ continue;
+ }
try {
double value = Double.parseDouble(metrics.get(key));
reportLog.addValue(
diff --git a/tools/catbox-common/res/config/catbox-performance-app-start-up-dialer-base.xml b/tools/catbox-common/res/config/catbox-performance-app-start-up-dialer-base.xml
new file mode 100644
index 0000000..d95daaa
--- /dev/null
+++ b/tools/catbox-common/res/config/catbox-performance-app-start-up-dialer-base.xml
@@ -0,0 +1,29 @@
+<?xml version="1.0" encoding="utf-8"?>
+<!-- Copyright (C) 2023 Google Inc.
+
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<configuration description="Automotive Dialer Proc Load and Start-up options">
+ <!-- Proc Load Options -->
+ <option name="compatibility:module-arg" value="AndroidAutomotiveDialScenarioTests:instrumentation-arg:procload-collector:per_run:=true" />
+ <option name="compatibility:module-arg" value="AndroidAutomotiveDialScenarioTests:instrumentation-arg:procload-collector:proc-loadavg-interval:=20000" />
+ <option name="compatibility:module-arg" value="AndroidAutomotiveDialScenarioTests:instrumentation-arg:procload-collector:proc-loadavg-threshold:=4.0" />
+ <option name="compatibility:module-arg" value="AndroidAutomotiveDialScenarioTests:instrumentation-arg:procload-collector:proc-loadavg-timeout:=900000" />
+
+ <!-- App Start Up Options -->
+ <option name="compatibility:module-arg" value="AndroidAutomotiveDialScenarioTests:instrumentation-arg:listener:=android.device.collectors.ProcLoadListener,android.device.collectors.AppStartupListener" />
+ <option name="compatibility:module-arg" value="AndroidAutomotiveDialScenarioTests:instrumentation-arg:iterations:=20" />
+ <option name="compatibility:module-arg" value="AndroidAutomotiveDialScenarioTests:instrumentation-arg:favor-shell-commands:=true" />
+ <option name="compatibility:module-arg" value="AndroidAutomotiveDialScenarioTests:instrumentation-arg:rename-iterations:=true" />
+ <option name="compatibility:module-arg" value="AndroidAutomotiveDialScenarioTests:instrumentation-arg:iteration-separator:=$" />
+</configuration>
diff --git a/tools/catbox-common/res/config/catbox-performance-app-start-up-settings-base.xml b/tools/catbox-common/res/config/catbox-performance-app-start-up-settings-base.xml
new file mode 100644
index 0000000..f38444a
--- /dev/null
+++ b/tools/catbox-common/res/config/catbox-performance-app-start-up-settings-base.xml
@@ -0,0 +1,29 @@
+<?xml version="1.0" encoding="utf-8"?>
+<!-- Copyright (C) 2023 Google Inc.
+
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<configuration description="Automotive Settings Proc Load and Start-up options">
+ <!-- Proc Load Options -->
+ <option name="compatibility:module-arg" value="AndroidAutomotiveSettingsScenarioTests:instrumentation-arg:procload-collector:per_run:=true" />
+ <option name="compatibility:module-arg" value="AndroidAutomotiveSettingsScenarioTests:instrumentation-arg:procload-collector:proc-loadavg-interval:=20000" />
+ <option name="compatibility:module-arg" value="AndroidAutomotiveSettingsScenarioTests:instrumentation-arg:procload-collector:proc-loadavg-threshold:=4.0" />
+ <option name="compatibility:module-arg" value="AndroidAutomotiveSettingsScenarioTests:instrumentation-arg:procload-collector:proc-loadavg-timeout:=900000" />
+
+ <!-- App Start Up Options -->
+ <option name="compatibility:module-arg" value="AndroidAutomotiveSettingsScenarioTests:instrumentation-arg:listener:=android.device.collectors.ProcLoadListener,android.device.collectors.AppStartupListener" />
+ <option name="compatibility:module-arg" value="AndroidAutomotiveSettingsScenarioTests:instrumentation-arg:iterations:=20" />
+ <option name="compatibility:module-arg" value="AndroidAutomotiveSettingsScenarioTests:instrumentation-arg:favor-shell-commands:=true" />
+ <option name="compatibility:module-arg" value="AndroidAutomotiveSettingsScenarioTests:instrumentation-arg:rename-iterations:=true" />
+ <option name="compatibility:module-arg" value="AndroidAutomotiveSettingsScenarioTests:instrumentation-arg:iteration-separator:=$" />
+</configuration>
diff --git a/tools/catbox-common/res/config/catbox-performance-base.xml b/tools/catbox-common/res/config/catbox-performance-base.xml
index 1b1bec5..01a0d97 100644
--- a/tools/catbox-common/res/config/catbox-performance-base.xml
+++ b/tools/catbox-common/res/config/catbox-performance-base.xml
@@ -45,6 +45,8 @@
</target_preparer>
<!-- Enable perfetto host side metric collector -->
- <include name="perfetto-puller"/>
-
+ <metrics_collector class="com.android.tradefed.device.metric.PerfettoPullerMetricCollector">
+ <option name="collect-on-run-ended-only" value="false" />
+ <option name="pull-pattern-keys" value="perfetto_file_path" />
+ </metrics_collector>
</configuration>
diff --git a/tools/catbox-common/res/config/catbox-performance-postprocessors.xml b/tools/catbox-common/res/config/catbox-performance-postprocessors.xml
index edbaddf..0fcebad 100644
--- a/tools/catbox-common/res/config/catbox-performance-postprocessors.xml
+++ b/tools/catbox-common/res/config/catbox-performance-postprocessors.xml
@@ -20,4 +20,7 @@
<option name="test-iteration-separator" value="$" />
<option name="enable-per-test-log" value="false" />
</metric_post_processor>
+ <metric_post_processor class="com.android.tradefed.postprocessor.PerfettoGenericPostProcessor">
+ <option name="perfetto-proto-file-prefix" value="metric_perfetto" />
+ </metric_post_processor>
</configuration>
diff --git a/tools/catbox-common/res/config/perfetto-puller.xml b/tools/catbox-common/res/config/perfetto-puller.xml
deleted file mode 100644
index a29cf72..0000000
--- a/tools/catbox-common/res/config/perfetto-puller.xml
+++ /dev/null
@@ -1,20 +0,0 @@
-<?xml version="1.0" encoding="utf-8"?>
-<!--
- ~ Copyright (C) 2023 The Android Open Source Project
- ~
- ~ Licensed under the Apache License, Version 2.0 (the "License");
- ~ you may not use this file except in compliance with the License.
- ~ You may obtain a copy of the License at
- ~
- ~ http://www.apache.org/licenses/LICENSE-2.0
- ~
- ~ Unless required by applicable law or agreed to in writing, software
- ~ distributed under the License is distributed on an "AS IS" BASIS,
- ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- ~ See the License for the specific language governing permissions and
- ~ limitations under the License.
- -->
-<configuration description="Pull the files from the device and add it to the test logs.">
- <!-- Pulls the perfetto files from the device and collects the metrics from the trace files -->
- <metrics_collector class="com.android.tradefed.device.metric.PerfettoPullerMetricCollector" />
-</configuration>
\ No newline at end of file
diff --git a/tools/catbox-tradefed/Android.bp b/tools/catbox-tradefed/Android.bp
index c94cc9f..6c92af0 100644
--- a/tools/catbox-tradefed/Android.bp
+++ b/tools/catbox-tradefed/Android.bp
@@ -23,12 +23,11 @@
"cts-tradefed-harness",
"catbox-common",
"catbox-preparer-lib",
- "catbox-report-lib"
+ "catbox-report-lib",
],
required: ["compatibility-host-util"],
}
-
tradefed_binary_host {
name: "catbox-tradefed",
java_resource_dirs: ["res"],
@@ -40,10 +39,10 @@
"cts-tradefed-harness",
"catbox-common",
"catbox-preparer-lib",
- "catbox-test-runner-lib"
+ "catbox-test-runner-lib",
],
required: [
"compatibility-host-util",
- "catbox-report-lib"
+ "catbox-report-lib",
],
}
diff --git a/tools/catbox-tradefed/res/config/catbox-performance-cold-app-start-up-settings.xml b/tools/catbox-tradefed/res/config/catbox-performance-cold-app-start-up-settings.xml
index 21cb686..df05367 100644
--- a/tools/catbox-tradefed/res/config/catbox-performance-cold-app-start-up-settings.xml
+++ b/tools/catbox-tradefed/res/config/catbox-performance-cold-app-start-up-settings.xml
@@ -49,4 +49,4 @@
<option name="report-test-name-mapping" key="android.platform.test.scenario.settings.OpenAppMicrobenchmark#testOpen" value="settings_app_start_up" />
<option name="report-all-metrics" value="false" />
<option name="report-metric-key-mapping" key="cold_startup_com.android.car.settings-median" value="cold_start_up" />
-</configuration>
+</configuration>
\ No newline at end of file
diff --git a/tools/catbox-tradefed/res/config/catbox-performance-successive-boottime.xml b/tools/catbox-tradefed/res/config/catbox-performance-successive-boottime.xml
index 863b7e1..24be4be 100644
--- a/tools/catbox-tradefed/res/config/catbox-performance-successive-boottime.xml
+++ b/tools/catbox-tradefed/res/config/catbox-performance-successive-boottime.xml
@@ -15,50 +15,70 @@
-->
<configuration>
- <!-- Extend existing boottime config-->
- <include name="google/test/boottime-successive"/>
-
<include name="catbox-performance-base" />
+ <target_preparer class="com.android.tradefed.targetprep.PushFilePreparer" />
+ <!-- Needed to set perfetto trace property before fastboot commands -->
+ <target_preparer class="com.android.tradefed.targetprep.RunCommandTargetPreparer" />
+ <target_preparer class="com.android.tradefed.targetprep.FastbootCommandPreparer" />
+ <!-- Needed multiple run command target preparer for running commands before/after install. -->
+ <target_preparer class="com.android.tradefed.targetprep.RunCommandTargetPreparer" />
+ <target_preparer class="com.android.tradefed.targetprep.TestAppInstallSetup" />
+ <target_preparer class="com.android.tradefed.targetprep.RunCommandTargetPreparer" />
+ <target_preparer class="com.android.tradefed.targetprep.InstrumentationPreparer" >
+ <option name="disable" value="true" />
+ </target_preparer>
+
+ <test class="com.android.tradefed.testtype.HostTest" >
+ <option name="class" value="android.boottime.BootTimeTest" />
+ <option name="set-option" value="boot-delay:20000" />
+ <option name="set-option"
+ value="successive-boot-prepare-cmd:am start -n com.google.android.car.setupwizard/.ExitActivity"/>
+ <option name="set-option"
+ value="successive-boot-prepare-cmd:setprop persist.aae.suw.disable_short_notice true"/>
+ <option name="set-option"
+ value="successive-boot-prepare-cmd:cmd car_service enable-feature car_evs_service"/>
+ </test>
+
+ <metrics_collector class="com.android.tradefed.device.metric.AtraceCollector"/>
+ <metrics_collector class="com.android.tradefed.device.metric.PerfettoPullerMetricCollector">
+ <option name="collect-on-run-ended-only" value="false" />
+ </metrics_collector>
+
+ <metric_post_processor class="com.android.tradefed.postprocessor.PerfettoGenericPostProcessor" />
+ <metric_post_processor class="com.android.tradefed.postprocessor.MetricFilePostProcessor">
+ <option name="aggregate-similar-tests" value="true" />
+ </metric_post_processor>
+ <metric_post_processor class="android.boottime.postprocessor.LogcatPostProcessor">
+ <option name="file-regex" value=".*Successive_reboots_logcat.*"/>
+ <!-- For custom boot time metrics -->
+ <option name="boot-time-pattern" key="boot_to_ble_on"
+ value="BluetoothAdapterService: updateAdapterState\(\) \- Broadcasting state BLE_ON"/>
+ <option name="boot-time-pattern" key="boot_to_carlauncher_fully_drawn"
+ value="Fully drawn com.android.car.carlauncher/.CarLauncher"/>
+ <option name="boot-time-pattern" key="boot_to_first_user_unlocked"
+ value="CarServiceHelper: Time to unlock 1st user"/>
+ <option name="boot-time-pattern" key="boot_to_maps_displayed"
+ value="Displayed com.google.android.apps.maps"/>
+ <option name="boot-time-pattern" key="boot_to_settings_fallbackhome_displayed"
+ value="Displayed com.android.car.settings/.FallbackHome"/>
+ <option name="boot-time-pattern" key="boot_to_user0_unlocked"
+ value="SystemServerTiming: SystemUserUnlock took to complete"/>
+ <option name="boot-time-pattern" key="boot_to_user10_unlocked"
+ value="SystemServerTiming: User10Unlock took to complete"/>
+ <option name="boot-time-pattern" key="boot_to_gmscore_started_experimental"
+ value="ActivityManager: Start proc \d+:com.google.android.gms/u0"/>
+ <option name="boot-time-pattern" key="boot_to_car_evs_service_started"
+ value="CAR.InitTiming: CarEvsService took to complete"/>s
+ </metric_post_processor>
+ <metric_post_processor class="android.boottime.postprocessor.DmesgPostProcessor">
+ <option name="file-regex" value=".*Successive_reboots_dmesg.*"/>
+ </metric_post_processor>
+
<!-- Artificially disabling tests in CompatibilityTestSuite -->
<!-- See com.android.tradefed.testtype.suite.CompatibilityTestSuite:loadTests() -->
<option name="compatibility:reverse-exclude-filters" value="true" />
- <option name="boot-delay" value="20000"/>
- <option name="boot-count" value="20"/>
- <option name="dmesg-info" value="true"/>
- <option name="granular-boot-info" value="true"/>
- <option name="online-timeout" value="120000"/>
- <option name="successive-boot" value="true"/>
-
- <!-- Prepare commands -->
- <option name="successive-boot-prepare-cmd"
- value="am start -n com.google.android.car.setupwizard/.ExitActivity"/>
- <option name="successive-boot-prepare-cmd"
- value="setprop persist.aae.suw.disable_short_notice true"/>
- <option name="successive-boot-prepare-cmd"
- value="cmd car_service enable-feature car_evs_service"/>
-
- <!-- For custom boot time metrics -->
- <option name="boot-time-pattern" key="boot_to_ble_on"
- value="BluetoothAdapterService: updateAdapterState\(\) \- Broadcasting state BLE_ON"/>
- <option name="boot-time-pattern" key="boot_to_carlauncher_fully_drawn"
- value="Fully drawn com.android.car.carlauncher/.CarLauncher"/>
- <option name="boot-time-pattern" key="boot_to_first_user_unlocked"
- value="CarServiceHelper: Time to unlock 1st user"/>
- <option name="boot-time-pattern" key="boot_to_maps_displayed"
- value="Displayed com.google.android.apps.maps"/>
- <option name="boot-time-pattern" key="boot_to_settings_fallbackhome_displayed"
- value="Displayed com.android.car.settings/.FallbackHome"/>
- <option name="boot-time-pattern" key="boot_to_user0_unlocked"
- value="SystemServerTiming: SystemUserUnlock took to complete"/>
- <option name="boot-time-pattern" key="boot_to_user10_unlocked"
- value="SystemServerTiming: User10Unlock took to complete"/>
- <option name="boot-time-pattern" key="boot_to_gmscore_started_experimental"
- value="ActivityManager: Start proc \d+:com.google.android.gms/u0"/>
- <option name="boot-time-pattern" key="boot_to_car_evs_service_started"
- value="CAR.InitTiming: CarEvsService took to complete"/>
-
<option name="report-log-name" value="CatboxPerformanceTests" />
<option name="report-test-name-mapping" key="BootTimeTest.BootTimeTest#SuccessiveBootTest" value="boot_time_test" />
<option name="report-all-metrics" value="false" />
