Create JavaHeapProfTest to test java heap profiling.
Note: this test is too flaky to be run in CI, but is hopefully still
useful as a manual test when making changes to the javaheapprof
implementation.
Bug: 370562214
Test: atest JavaHeapProfTest
Flag: TEST_ONLY
Change-Id: I255396ad174cc66c3e789293ba189f2057b89e77
diff --git a/runtime/javaheapprof/tests/Android.bp b/runtime/javaheapprof/tests/Android.bp
new file mode 100644
index 0000000..0324d71
--- /dev/null
+++ b/runtime/javaheapprof/tests/Android.bp
@@ -0,0 +1,20 @@
+android_test {
+ name: "JavaHeapProfTest",
+
+ srcs: ["src/**/*.java"],
+ data: [
+ "javaheapprof.pbtxt",
+ "query.sql",
+ ],
+ device_first_data: [":trace_processor_shell"],
+
+ static_libs: [
+ "androidx.test.runner",
+ "collector-device-lib",
+ "perfetto-helper",
+ ],
+ certificate: "platform",
+
+ test_config: "AndroidTest.xml",
+ test_suites: ["device-tests"],
+}
diff --git a/runtime/javaheapprof/tests/AndroidManifest.xml b/runtime/javaheapprof/tests/AndroidManifest.xml
new file mode 100644
index 0000000..d6d0764
--- /dev/null
+++ b/runtime/javaheapprof/tests/AndroidManifest.xml
@@ -0,0 +1,11 @@
+<manifest xmlns:android="http://schemas.android.com/apk/res/android"
+ package="android.test.javaheapprof" >
+
+ <application>
+ <uses-library android:name="android.test.runner" />
+ </application>
+
+ <instrumentation android:name="androidx.test.runner.AndroidJUnitRunner"
+ android:targetPackage="android.test.javaheapprof"
+ android:label="javaheapprof Test"/>
+</manifest>
diff --git a/runtime/javaheapprof/tests/AndroidTest.xml b/runtime/javaheapprof/tests/AndroidTest.xml
new file mode 100644
index 0000000..bd89593
--- /dev/null
+++ b/runtime/javaheapprof/tests/AndroidTest.xml
@@ -0,0 +1,21 @@
+<?xml version="1.0" encoding="utf-8"?>
+<configuration description="Test module config for JavaHeapProfTest">
+
+ <!-- selinux doesn't like the test doing Runtime.exec on trace_processor_shell. -->
+ <target_preparer class="com.android.tradefed.targetprep.DisableSELinuxTargetPreparer" />
+
+ <target_preparer class="com.android.tradefed.targetprep.TestAppInstallSetup">
+ <option name="test-file-name" value="JavaHeapProfTest.apk" />
+ </target_preparer>
+
+ <target_preparer class="com.android.tradefed.targetprep.PushFilePreparer">
+ <option name="push-file" key="trace_processor_shell" value="/data/local/android.test.javaheapprof/trace_processor_shell" />
+ <option name="push-file" key="javaheapprof.pbtxt" value="/data/misc/perfetto-configs/javaheapprof.pbtxt" />
+ <option name="push-file" key="query.sql" value="/data/local/android.test.javaheapprof/query.sql" />
+ </target_preparer>
+
+ <test class="com.android.tradefed.testtype.AndroidJUnitTest">
+ <option name="package" value="android.test.javaheapprof"/>
+ <option name="runner" value="androidx.test.runner.AndroidJUnitRunner"/>
+ </test>
+</configuration>
diff --git a/runtime/javaheapprof/tests/README.md b/runtime/javaheapprof/tests/README.md
new file mode 100644
index 0000000..1d14289
--- /dev/null
+++ b/runtime/javaheapprof/tests/README.md
@@ -0,0 +1,114 @@
+# JavaHeapProfTest
+
+The JavaHeapProfTest is an aspirational test for the Java heap profiling
+implementation. The current implementation of Java heap profiling passes the
+test most of the time, but not every time due to randomness and minor biases
+in the sampling.
+
+This test is not part of ART run-tests because it depends on the real version
+of `heapprofd_client_api.so` to run. It is not run under continuous
+integration because it is too flaky at the moment. To make use of this test,
+you can run it manually when making significant changes to the Java heap
+profiling code.
+
+To run the JavaHeapProfTest manually, for example:
+```sh
+atest --iterations 10 JavaHeapProfTest
+```
+
+The test should pass on a decent number of the iterations. The test logs a
+summary of its findings to device logcat when it runs. Here's an example
+passing case:
+
+```
+<name>: expected actual normal Δ abs Δ rel Δ stddev
+ total: 1652326400 1652356160 1652356160 29760 0.00 0.01
+ S0: 102400 98304 98302 -4098 -0.04 -0.20
+ S1: 102400 69632 69630 -32770 -0.32 -1.60
+ S2: 102400 94208 94206 -8194 -0.08 -0.40
+ S3: 102400 122880 122877 20477 0.20 1.00
+ M0: 102400000 101766144 101764311 -635689 -0.01 -0.98
+ M1: 102400000 102368256 102366412 -33588 -0.00 -0.05
+ M2: 102400000 102005760 102003922 -396078 -0.00 -0.61
+ M3: 102400000 103330816 103328954 928954 0.01 1.43
+ N0: 307200000 306942248 306936719 -263281 -0.00 -0.23
+ N1: 307200000 306694488 306688964 -511036 -0.00 -0.46
+ N2: 307200000 308225808 308220256 1020256 0.00 0.91
+ N3: 307200000 307124912 307119380 -80620 -0.00 -0.07
+ B0: 13107200 13107200 13106963 -237 -0.00 -0.00
+ E0: 102400 81920 81918 -20482 -0.20 -1.00
+ E1: 102400 110592 110590 8190 0.08 0.40
+ E2: 102400 102400 102398 -2 -0.00 -0.00
+ E3: 102400 110592 110590 8190 0.08 0.40
+```
+
+The first column corresponds to allocation sites in `JavaHeapProfTest.java`.
+The `expected` column is the number of bytes actually allocated at those call
+sites. The `actual` column is the number of bytes randomly sampled by the
+profiler. Small differences between `expected` and `actual` are expected,
+because the profiler uses random sampling.
+
+The `normal` column scales the `actual` results to match the `expected` total
+number of sampled bytes. This makes it easier to see if profiling is off in
+magnitude but still produces reasonable relative counts for different call
+sites.
+
+The `Δ abs`, `Δ rel`, and `Δ stddev` columns show how far off the `normal`
+results are from the `expected` results, in terms of absolute difference,
+relative differences, and standard deviation from what's expected given the
+randomness of sampling. The test is marked as failing if anything is more than
+3 standard deviations from what is expected. If the implementation of Java
+heap profiling was perfect (it's not), we would expect the test to pass 99.7%
+of the time.
+
+Here's an example failure you might get that's not too worrying, because the
+only issue is N0 is slightly out of bounds in terms of `Δ stddev`:
+```
+STACKTRACE:
+java.lang.AssertionError: Expected results not within 3.00 stddev of expected:
+ <name>: expected actual normal Δ abs Δ rel Δ stddev
+ total: 1652326400 1652368296 1652368296 41896 0.00 0.02
+ S0: 102400 106496 106493 4093 0.04 0.20
+ S1: 102400 94208 94205 -8195 -0.08 -0.40
+ S2: 102400 81920 81917 -20483 -0.20 -1.00
+ S3: 102400 69632 69630 -32770 -0.32 -1.60
+ M0: 102400000 100643840 100641288 -1758712 -0.02 -2.72
+ M1: 102400000 103646296 103643668 1243668 0.01 1.92
+ M2: 102400000 103140352 103137736 737736 0.01 1.14
+ M3: 102400000 102438912 102436314 36314 0.00 0.06
+ N0: 307200000 303657472 303649772 -3550228 -0.01 -3.17
+ N1: 307200000 308594432 308586607 1386607 0.00 1.24
+ N2: 307200000 308101464 308093652 893652 0.00 0.80
+ N3: 307200000 308317432 308309614 1109614 0.00 0.99
+ B0: 13107200 13107200 13106867 -333 -0.00 -0.00
+ E0: 102400 94208 94205 -8195 -0.08 -0.40
+ E1: 102400 86016 86013 -16387 -0.16 -0.80
+ E2: 102400 114688 114685 12285 0.12 0.60
+ E3: 102400 73728 73726 -28674 -0.28 -1.40
+```
+
+Here is an example of an egregious failure that indicates something is
+definitely wrong:
+```
+STACKTRACE:
+java.lang.AssertionError: Expected results not within 3.00 stddev of expected:
+ <name>: expected actual normal Δ abs Δ rel Δ stddev
+ total: 1652326400 512235520 512235520 -1140090880 -0.69 -438.29
+ S0: 102400 81920 264250 161850 1.58 7.90
+ S1: 102400 29696 95790 -6610 -0.06 -0.32
+ S2: 102400 20480 66062 -36338 -0.35 -1.77
+ S3: 102400 18432 59456 -42944 -0.42 -2.10
+ M0: 102400000 10512384 33909967 -68490033 -0.67 -105.77
+ M1: 102400000 10736640 34633353 -67766647 -0.66 -104.65
+ M2: 102400000 10577920 34121367 -68278633 -0.67 -105.44
+ M3: 102400000 10481664 33810873 -68589127 -0.67 -105.92
+ N0: 307200000 114229248 368471132 61271132 0.20 54.63
+ N1: 307200000 114622464 369739535 62539535 0.20 55.76
+ N2: 307200000 114736128 370106183 62906183 0.20 56.09
+ N3: 307200000 114616320 369719716 62519716 0.20 55.74
+ B0: 13107200 11534336 37206494 24099294 1.84 104.02
+ E0: 102400 4096 13212 -89188 -0.87 -4.36
+ E1: 102400 10240 33031 -69369 -0.68 -3.39
+ E2: 102400 13312 42940 -59460 -0.58 -2.90
+ E3: 102400 10240 33031 -69369 -0.68 -3.39
+```
diff --git a/runtime/javaheapprof/tests/javaheapprof.pbtxt b/runtime/javaheapprof/tests/javaheapprof.pbtxt
new file mode 100644
index 0000000..834aa94
--- /dev/null
+++ b/runtime/javaheapprof/tests/javaheapprof.pbtxt
@@ -0,0 +1,17 @@
+buffers {
+ size_kb: 65536
+ fill_policy: DISCARD
+}
+data_sources {
+ config {
+ name: "android.heapprofd"
+ heapprofd_config {
+ sampling_interval_bytes: 4096
+ process_cmdline: "android.test.javaheapprof"
+ shmem_size_bytes: 8388608
+ block_client: true
+ heaps: "com.android.art"
+ }
+ }
+}
+duration_ms: 60000;
diff --git a/runtime/javaheapprof/tests/query.sql b/runtime/javaheapprof/tests/query.sql
new file mode 100644
index 0000000..7e73a4f
--- /dev/null
+++ b/runtime/javaheapprof/tests/query.sql
@@ -0,0 +1,7 @@
+INCLUDE PERFETTO MODULE android.memory.heap_profile.summary_tree;
+
+SELECT name, SUM(cumulative_alloc_size) as size
+FROM android_heap_profile_summary_tree
+WHERE name LIKE 'android.test.javaheapprof.JavaHeapProfTest$Allocator.__'
+GROUP BY name;
+
diff --git a/runtime/javaheapprof/tests/src/android/test/javaheapprof/JavaHeapProfTest.java b/runtime/javaheapprof/tests/src/android/test/javaheapprof/JavaHeapProfTest.java
new file mode 100644
index 0000000..abbaab3
--- /dev/null
+++ b/runtime/javaheapprof/tests/src/android/test/javaheapprof/JavaHeapProfTest.java
@@ -0,0 +1,311 @@
+/*
+ * Copyright (C) 2025 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License
+ */
+
+package android.test.javaheapprof;
+
+import android.util.Log;
+
+import androidx.test.filters.LargeTest;
+import androidx.test.filters.MediumTest;
+import androidx.test.filters.SmallTest;
+
+import com.android.helpers.PerfettoHelper;
+
+import org.junit.After;
+import org.junit.AfterClass;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.JUnit4;
+
+import java.io.BufferedReader;
+import java.io.IOException;
+import java.io.InputStreamReader;
+import java.util.HashMap;
+import java.util.Map;
+
+@RunWith(JUnit4.class)
+public class JavaHeapProfTest {
+ private static final String TAG = JavaHeapProfTest.class.getSimpleName();
+
+ private static final int NUM_THREADS = 100;
+ private static final int NUM_ITERS_PER_THREAD = 1000;
+ private static final int NUM_BYTES_PER_ALLOC = 1024;
+ private static final int SAMPLING_INTERVAL_BYTES = 4096;
+ private static final int MULTIPLIER = 3;
+ private static final int LARGE_MULTIPLIER = 128;
+ private static final double STDDEV_ALLOWANCE = 3;
+ private static final String TEST_DIR = "/data/local/android.test.javaheapprof";
+ private static final String TRACE_CONFIG = "javaheapprof.pbtxt";
+ private static final String TRACE_OUTPUT = TEST_DIR + "/trace.pftrace";
+ private static final String TRACE_PROCESSOR = TEST_DIR + "/trace_processor_shell";
+ private static final String TRACE_QUERY = TEST_DIR + "/query.sql";
+
+ static class Allocator implements Runnable {
+ Object mLastAllocated = null;
+
+ void alloc(int multiplier) {
+ // Arrays take up 12 bytes, so subtract that out.
+ mLastAllocated = new byte[multiplier * NUM_BYTES_PER_ALLOC - 12];
+ }
+
+ void alloc() {
+ alloc(1);
+ }
+
+ void B0() {
+ alloc(LARGE_MULTIPLIER);
+ };
+
+ void S0() {
+ alloc();
+ };
+ void S1() {
+ alloc();
+ };
+ void S2() {
+ alloc();
+ };
+ void S3() {
+ alloc();
+ };
+
+ void M0() {
+ alloc();
+ };
+ void M1() {
+ alloc();
+ };
+ void M2() {
+ alloc();
+ };
+ void M3() {
+ alloc();
+ };
+
+ void N0() {
+ alloc(MULTIPLIER);
+ };
+ void N1() {
+ alloc(MULTIPLIER);
+ };
+ void N2() {
+ alloc(MULTIPLIER);
+ };
+ void N3() {
+ alloc(MULTIPLIER);
+ };
+
+ void E0() {
+ alloc();
+ };
+ void E1() {
+ alloc();
+ };
+ void E2() {
+ alloc();
+ };
+ void E3() {
+ alloc();
+ };
+
+ public void run() {
+ // Allocate from a few different callsites at the start of the thread,
+ // to make sure we don't bias towards the first allocation in the
+ // thread.
+ S0();
+ S1();
+ S2();
+ S3();
+
+ // Allocate from a few different callsites in a loop, to make sure we
+ // don't bias towards some in a pattern and that we can distinguish call
+ // sites with different allocation sizes.
+ for (int i = 0; i < NUM_ITERS_PER_THREAD; ++i) {
+ M0();
+ M1();
+ M2();
+ M3();
+ }
+
+ // Do the same, but with a different size allocation to check we tell
+ // the difference between different size allocations properly.
+ for (int i = 0; i < NUM_ITERS_PER_THREAD; ++i) {
+ N0();
+ N1();
+ N2();
+ N3();
+ }
+
+ // Allocate from a large allocation to test tlab vs. non-tlab.
+ B0();
+
+ // Allocate from a few different callsites at the end of the thread,
+ // to make sure we don't bias against the last allocations in the
+ // thread.
+ E0();
+ E1();
+ E2();
+ E3();
+ }
+ }
+
+ private static class Results {
+ private final Map<String, Long> mResults;
+ private final StringBuilder mMessage = new StringBuilder();
+ private double mScaleFactor = 1.0;
+ private boolean mFailed = false;
+
+ private Results(Map<String, Long> results) {
+ mResults = results;
+ mMessage.append(String.format("%8s: %10s %10s %10s %12s %8s %8s\n", "<name>",
+ "expected", "actual", "normal", "Δ abs", "Δ rel", "Δ stddev"));
+ }
+
+ private void expect(String name, long expected, long actual) {
+ // There are known issues with the absolute magnitude of
+ // reporting. Apply the scale factor to normalize actual reported
+ // values so we can separate the question of whether the reporting
+ // is relatively or absolutely correct.
+ long normal = (long) (mScaleFactor * actual);
+
+ // In theory random sampling decides to sample each individual byte
+ // randomly with probability 1 / SAMPLING_INTERVAL_BYTES. The sum of
+ // bytes sampled is a binomial distribution. We want to check that
+ // the sum is within a few standard deviations of what we expect.
+ double p = 1.0 / (double) SAMPLING_INTERVAL_BYTES;
+ double variance = (double) expected * p * (1.0 - p);
+ double stddev = Math.sqrt(variance);
+
+ // Try to be within 3 standard deviations for now, which ought to
+ // cover 99.7% of cases.
+ double sampleMargin = STDDEV_ALLOWANCE * stddev;
+
+ // The random sampling should be artificially scaling up the number
+ // of sampled bytes by SAMPLING_INTERVAL_BYTES. Scale up the margin
+ // to match.
+ long margin = (long) (sampleMargin * (double) SAMPLING_INTERVAL_BYTES);
+
+ // Log the results to facilitate debug and analysis.
+ long abs = normal - expected;
+ double rel = (double) abs / (double) expected;
+ double std = (double) abs * STDDEV_ALLOWANCE / (double) margin;
+ mMessage.append(String.format("%8s: %10d %10d %10d %12d %8.2f %8.2f\n", name, expected,
+ actual, normal, abs, rel, std));
+
+ if (Math.abs(abs) > margin) {
+ mFailed = true;
+ }
+ }
+
+ // Tell the results about the expected total number of allocations.
+ // This should be called once before any calls to expectFrame.
+ public void expectTotal(long expectedTotal) {
+ long total = 0;
+ for (Long value : mResults.values()) {
+ total += value;
+ }
+
+ expect("total", expectedTotal, total);
+ mScaleFactor = (double) expectedTotal / (double) total;
+ }
+
+ public void expectFrame(String key, long expected) {
+ Long actual = mResults.get(key);
+ Assert.assertNotNull(key + " not found", actual);
+ expect(key, expected, actual);
+ }
+
+ public void assertOkay() {
+ String msg = mMessage.toString();
+ Log.i(TAG, msg);
+ if (mFailed) {
+ Assert.fail(
+ String.format("Expected results not within %.2f stddev of expected:\n %s",
+ STDDEV_ALLOWANCE, msg));
+ }
+ }
+
+ public static Results query() throws IOException {
+ Runtime runtime = Runtime.getRuntime();
+ Process process =
+ runtime.exec(TRACE_PROCESSOR + " -q " + TRACE_QUERY + " " + TRACE_OUTPUT);
+
+ Map<String, Long> results = new HashMap<>();
+ BufferedReader reader =
+ new BufferedReader(new InputStreamReader(process.getInputStream()));
+ final String prefix = "\"android.test.javaheapprof.JavaHeapProfTest$Allocator.";
+ for (String line; (line = reader.readLine()) != null;) {
+ if (line.startsWith(prefix)) {
+ String key =
+ line.substring(prefix.length(), line.indexOf('"', prefix.length()));
+ Long value = Long.parseLong(line.substring(line.indexOf(',') + 1));
+ results.put(key, value);
+ }
+ }
+ return new Results(results);
+ }
+ }
+
+ @Test
+ public void testJavaHeapProf() throws InterruptedException, IOException {
+ PerfettoHelper perfetto = new PerfettoHelper();
+ perfetto.setPerfettoConfigRootDir("/data/misc/perfetto-configs/");
+ perfetto.startCollectingFromConfigFile(TRACE_CONFIG, true);
+
+ Allocator allocator = new Allocator();
+
+ Thread[] threads = new Thread[NUM_THREADS];
+ for (int i = 0; i < NUM_THREADS; ++i) {
+ threads[i] = new Thread(allocator);
+ threads[i].start();
+ }
+
+ for (int i = 0; i < NUM_THREADS; ++i) {
+ threads[i].join();
+ }
+
+ perfetto.stopCollecting(0, TRACE_OUTPUT);
+ Results results = Results.query();
+
+ long perThread = NUM_THREADS * NUM_BYTES_PER_ALLOC;
+ long perIter = NUM_THREADS * NUM_ITERS_PER_THREAD * NUM_BYTES_PER_ALLOC;
+ long total = 4 * perThread + 4 * perIter + 4 * MULTIPLIER * perIter
+ + 1 * LARGE_MULTIPLIER * perThread + 4 * perThread;
+
+ results.expectTotal(total);
+ results.expectFrame("S0", perThread);
+ results.expectFrame("S1", perThread);
+ results.expectFrame("S2", perThread);
+ results.expectFrame("S3", perThread);
+ results.expectFrame("M0", perIter);
+ results.expectFrame("M1", perIter);
+ results.expectFrame("M2", perIter);
+ results.expectFrame("M3", perIter);
+ results.expectFrame("N0", MULTIPLIER * perIter);
+ results.expectFrame("N1", MULTIPLIER * perIter);
+ results.expectFrame("N2", MULTIPLIER * perIter);
+ results.expectFrame("N3", MULTIPLIER * perIter);
+ results.expectFrame("B0", LARGE_MULTIPLIER * perThread);
+ results.expectFrame("E0", perThread);
+ results.expectFrame("E1", perThread);
+ results.expectFrame("E2", perThread);
+ results.expectFrame("E3", perThread);
+ results.assertOkay();
+ }
+}
