Merge "Add simpleperf collector for collecting simpleperf samples." into rvc-dev am: 739b3caa6d
Original change: https://googleplex-android-review.googlesource.com/c/platform/platform_testing/+/12211742
Change-Id: Ie720081c39afc2d46f4ef7a81b7f81f363431060
diff --git a/libraries/collectors-helper/simpleperf/Android.bp b/libraries/collectors-helper/simpleperf/Android.bp
new file mode 100644
index 0000000..98d66b0
--- /dev/null
+++ b/libraries/collectors-helper/simpleperf/Android.bp
@@ -0,0 +1,29 @@
+// Copyright (C) 2020 The Android Open Source Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Used for collecting simpleperf samples.
+java_library {
+ name: "simpleperf-helper",
+ defaults: ["tradefed_errorprone_defaults"],
+
+ srcs: ["src/**/*.java"],
+
+ static_libs: [
+ "androidx.test.runner",
+ "ub-uiautomator",
+ ],
+
+ sdk_version: "current",
+}
+
diff --git a/libraries/collectors-helper/simpleperf/src/com/android/helpers/SimpleperfHelper.java b/libraries/collectors-helper/simpleperf/src/com/android/helpers/SimpleperfHelper.java
new file mode 100644
index 0000000..2e08f65
--- /dev/null
+++ b/libraries/collectors-helper/simpleperf/src/com/android/helpers/SimpleperfHelper.java
@@ -0,0 +1,204 @@
+/*
+ * Copyright (C) 2020 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.android.helpers;
+
+import android.os.SystemClock;
+import android.support.test.uiautomator.UiDevice;
+import android.util.Log;
+import androidx.test.InstrumentationRegistry;
+
+import java.io.IOException;
+import java.io.File;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+
+/**
+ * SimpleperfHelper is used to start and stop simpleperf sample collection and move the output
+ * sample file to the destination folder.
+ */
+public class SimpleperfHelper {
+
+ private static final String LOG_TAG = SimpleperfHelper.class.getSimpleName();
+ private static final String SIMPLEPERF_TMP_FILE_PATH = "/data/local/tmp/perf.data";
+
+ private static final String SIMPLEPERF_START_CMD =
+ "simpleperf record -o %s -g --post-unwind=yes -f 500 -a --exclude-perf";
+ private static final String SIMPLEPERF_STOP_CMD = "pkill -INT simpleperf";
+ private static final String SIMPLEPERF_PROC_ID_CMD = "pidof simpleperf";
+ private static final String REMOVE_CMD = "rm %s";
+ private static final String MOVE_CMD = "mv %s %s";
+
+ private static final int SIMPLEPERF_STOP_WAIT_COUNT = 12;
+ private static final long SIMPLEPERF_STOP_WAIT_TIME = 5000;
+
+ private UiDevice mUiDevice;
+
+ public boolean startCollecting() {
+ mUiDevice = UiDevice.getInstance(InstrumentationRegistry.getInstrumentation());
+ try {
+ // Cleanup any running simpleperf sessions.
+ Log.i(LOG_TAG, "Cleanup simpleperf before starting.");
+ if (isSimpleperfRunning()) {
+ Log.i(LOG_TAG, "Simpleperf is already running. Stopping simpleperf.");
+ if (!stopSimpleperf()) {
+ return false;
+ }
+ }
+
+ Log.i(LOG_TAG, String.format("Starting simpleperf"));
+ new Thread() {
+ @Override
+ public void run() {
+ UiDevice uiDevice =
+ UiDevice.getInstance(InstrumentationRegistry.getInstrumentation());
+ try {
+ uiDevice.executeShellCommand(
+ String.format(SIMPLEPERF_START_CMD, SIMPLEPERF_TMP_FILE_PATH));
+ } catch (IOException e) {
+ Log.e(LOG_TAG, "Failed to start simpleperf.");
+ }
+ }
+ }.start();
+
+ if (!isSimpleperfRunning()) {
+ Log.e(LOG_TAG, "Simpleperf sampling failed to start.");
+ return false;
+ }
+ } catch (IOException e) {
+ Log.e(LOG_TAG, "Unable to start simpleperf sampling due to :" + e.getMessage());
+ return false;
+ }
+ Log.i(LOG_TAG, "Simpleperf sampling started successfully.");
+ return true;
+ }
+
+ /**
+ * Stop the simpleperf sample collection under /data/local/tmp/perf.data and copy the output to
+ * the destination file.
+ *
+ * @param destinationFile file to copy the simpleperf sample file to.
+ * @return true if the trace collection is successful otherwise false.
+ */
+ public boolean stopCollecting(String destinationFile) {
+ Log.i(LOG_TAG, "Stopping simpleperf.");
+ try {
+ if (stopSimpleperf()) {
+ if (!copyFileOutput(destinationFile)) {
+ return false;
+ }
+ } else {
+ Log.e(LOG_TAG, "Simpleperf failed to stop");
+ return false;
+ }
+ } catch (IOException e) {
+ Log.e(LOG_TAG, "Unable to stop the simpleperf samping due to " + e.getMessage());
+ return false;
+ }
+ return true;
+ }
+
+ /**
+ * Utility method for sending the signal to stop simpleperf.
+ *
+ * @return true if simpleperf is successfully stopped.
+ */
+ public boolean stopSimpleperf() throws IOException {
+ if (!isSimpleperfRunning()) {
+ Log.e(LOG_TAG, "Simpleperf stop called, but simpleperf is not running.");
+ return false;
+ }
+
+ String stopOutput = mUiDevice.executeShellCommand(SIMPLEPERF_STOP_CMD);
+ Log.i(LOG_TAG, String.format("Simpleperf stop command ran"));
+ int waitCount = 0;
+ while (isSimpleperfRunning()) {
+ if (waitCount < SIMPLEPERF_STOP_WAIT_COUNT) {
+ SystemClock.sleep(SIMPLEPERF_STOP_WAIT_TIME);
+ waitCount++;
+ continue;
+ }
+ return false;
+ }
+ Log.e(LOG_TAG, "Simpleperf stopped successfully.");
+ return true;
+ }
+
+ /**
+ * Check if there is a simpleperf instance running.
+ *
+ * @return true if there is a running simpleperf instance, otherwise false.
+ */
+ private boolean isSimpleperfRunning() {
+ try {
+ String simpleperfProcId = mUiDevice.executeShellCommand(SIMPLEPERF_PROC_ID_CMD);
+ Log.i(LOG_TAG, String.format("Simpleperf process id - %s", simpleperfProcId));
+ if (simpleperfProcId.isEmpty()) {
+ return false;
+ }
+ } catch (IOException e) {
+ Log.e(LOG_TAG, "Unable to check simpleperf status: " + e.getMessage());
+ return false;
+ }
+ return true;
+ }
+
+ /**
+ * Copy the temporary simpleperf output file to the given destinationFile.
+ *
+ * @param destinationFile file to copy simpleperf output into.
+ * @return true if the simpleperf file copied successfully, otherwise false.
+ */
+ private boolean copyFileOutput(String destinationFile) {
+ Path path = Paths.get(destinationFile);
+ String destDirectory = path.getParent().toString();
+ // Check if directory already exists
+ File directory = new File(destDirectory);
+ if (!directory.exists()) {
+ boolean success = directory.mkdirs();
+ if (!success) {
+ Log.e(
+ LOG_TAG,
+ String.format(
+ "Result output directory %s not created successfully.",
+ destDirectory));
+ return false;
+ }
+ }
+
+ // Copy the collected trace from /data/local/tmp to the destinationFile.
+ try {
+ String moveResult =
+ mUiDevice.executeShellCommand(
+ String.format(MOVE_CMD, SIMPLEPERF_TMP_FILE_PATH, destinationFile));
+ if (!moveResult.isEmpty()) {
+ Log.e(
+ LOG_TAG,
+ String.format(
+ "Unable to move simpleperf output file from %s to %s due to %s",
+ SIMPLEPERF_TMP_FILE_PATH, destinationFile, moveResult));
+ return false;
+ }
+ } catch (IOException e) {
+ Log.e(
+ LOG_TAG,
+ "Unable to move the simpleperf sample file to destination file."
+ + e.getMessage());
+ return false;
+ }
+ return true;
+ }
+}
diff --git a/libraries/collectors-helper/simpleperf/test/Android.bp b/libraries/collectors-helper/simpleperf/test/Android.bp
new file mode 100644
index 0000000..860dbac
--- /dev/null
+++ b/libraries/collectors-helper/simpleperf/test/Android.bp
@@ -0,0 +1,30 @@
+// Copyright (C) 2020 The Android Open Source Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+java_library {
+ name: "simpleperf-helper-test",
+ defaults: ["tradefed_errorprone_defaults"],
+
+ srcs: ["src/**/*.java"],
+
+ static_libs: [
+ "androidx.test.runner",
+ "android-support-test",
+ "simpleperf-helper",
+ "junit",
+
+ ],
+
+ sdk_version: "current",
+}
diff --git a/libraries/collectors-helper/simpleperf/test/src/com/android/helpers/tests/SimpleperfHelperTest.java b/libraries/collectors-helper/simpleperf/test/src/com/android/helpers/tests/SimpleperfHelperTest.java
new file mode 100644
index 0000000..7c2fdd7
--- /dev/null
+++ b/libraries/collectors-helper/simpleperf/test/src/com/android/helpers/tests/SimpleperfHelperTest.java
@@ -0,0 +1,95 @@
+/*
+ * Copyright (C) 2020 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.android.helpers.tests;
+
+import android.support.test.uiautomator.UiDevice;
+import androidx.test.InstrumentationRegistry;
+import androidx.test.runner.AndroidJUnit4;
+
+import com.android.helpers.SimpleperfHelper;
+
+import java.io.IOException;
+
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+
+/**
+ * Android Unit tests for {@link SimpleperfHelper}.
+ *
+ * <p>atest CollectorsHelperTest:com.android.helpers.tests.SimpleperfHelperTest
+ */
+@RunWith(AndroidJUnit4.class)
+public class SimpleperfHelperTest {
+
+ private static final String REMOVE_CMD = "rm %s";
+ private static final String FILE_SIZE_IN_BYTES = "wc -c %s";
+
+ private SimpleperfHelper simpleperfHelper;
+
+ @Before
+ public void setUp() {
+ simpleperfHelper = new SimpleperfHelper();
+ }
+
+ @After
+ public void teardown() throws IOException {
+ simpleperfHelper.stopCollecting("data/local/tmp/perf.data");
+ UiDevice uiDevice = UiDevice.getInstance(InstrumentationRegistry.getInstrumentation());
+ uiDevice.executeShellCommand(String.format(REMOVE_CMD, "/data/local/tmp/perf.data"));
+ }
+
+ /** Test simpleperf collection starts collecting properly. */
+ @Test
+ public void testSimpleperfStartSuccess() throws Exception {
+ assertTrue(simpleperfHelper.startCollecting());
+ }
+
+ /** Test if the path name is prefixed with /. */
+ @Test
+ public void testSimpleperfValidOutputPath() throws Exception {
+ assertTrue(simpleperfHelper.startCollecting());
+ assertTrue(simpleperfHelper.stopCollecting("data/local/tmp/perf.data"));
+ }
+
+ /** Test the invalid output path. */
+ @Test
+ public void testSimpleperfInvalidOutputPath() throws Exception {
+ assertTrue(simpleperfHelper.startCollecting());
+ // Don't have permission to create new folder under /data
+ assertFalse(simpleperfHelper.stopCollecting("/data/dummy/xyz/perf.data"));
+ }
+
+ /** Test simpleperf collection returns true and output file size greater than zero */
+ @Test
+ public void testSimpleperfSuccess() throws Exception {
+ assertTrue(simpleperfHelper.startCollecting());
+ Thread.sleep(1000);
+ assertTrue(simpleperfHelper.stopCollecting("/data/local/tmp/perf.data"));
+ Thread.sleep(1000);
+ UiDevice uiDevice = UiDevice.getInstance(InstrumentationRegistry.getInstrumentation());
+ String[] fileStats =
+ uiDevice.executeShellCommand(
+ String.format(FILE_SIZE_IN_BYTES, "/data/local/tmp/perf.data"))
+ .split(" ");
+ int fileSize = Integer.parseInt(fileStats[0].trim());
+ assertTrue(fileSize > 0);
+ }
+}
diff --git a/libraries/device-collectors/src/main/Android.bp b/libraries/device-collectors/src/main/Android.bp
index 9ae62aa..5da043a 100644
--- a/libraries/device-collectors/src/main/Android.bp
+++ b/libraries/device-collectors/src/main/Android.bp
@@ -25,6 +25,7 @@
"memory-helper",
"perfetto-helper",
"power-helper",
+ "simpleperf-helper",
"ub-uiautomator",
"system-metric-helper",
],
diff --git a/libraries/device-collectors/src/main/java/android/device/collectors/SimpleperfListener.java b/libraries/device-collectors/src/main/java/android/device/collectors/SimpleperfListener.java
new file mode 100644
index 0000000..61fc87a
--- /dev/null
+++ b/libraries/device-collectors/src/main/java/android/device/collectors/SimpleperfListener.java
@@ -0,0 +1,207 @@
+/*
+ * Copyright (C) 2020 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package android.device.collectors;
+
+import android.device.collectors.annotations.OptionClass;
+import android.os.Bundle;
+import android.util.Log;
+import androidx.annotation.VisibleForTesting;
+import com.android.helpers.SimpleperfHelper;
+
+import java.io.IOException;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.UUID;
+import org.junit.runner.Description;
+import org.junit.runner.Result;
+import org.junit.runner.notification.Failure;
+
+/**
+ * A {@link SimpleperfListener} that captures simpleperf samples for a test run or per test method
+ * run and saves the results under
+ * <root_folder>/<test_display_name>/SimpleperfListener/<test_display_name>-<invocation_count>.perf
+ */
+@OptionClass(alias = "simpleperf-collector")
+public class SimpleperfListener extends BaseMetricListener {
+
+ // Default output folder to store the simpleperf sample files.
+ private static final String DEFAULT_OUTPUT_ROOT = "/sdcard/test_results";
+ // Destination directory to save the trace results.
+ private static final String TEST_OUTPUT_ROOT = "test_output_root";
+ // Simpleperf file path key.
+ private static final String SIMPLEPERF_FILE_PATH = "simpleperf_file_path";
+ // Argument determining whether we collect for the entire run, or per test.
+ public static final String COLLECT_PER_RUN = "per_run";
+ public static final String SIMPLEPERF_PREFIX = "simpleperf_";
+ // Skip failure metrics collection if set to true.
+ public static final String SKIP_TEST_FAILURE_METRICS = "skip_test_failure_metrics";
+
+ // Simpleperf samples collected during the test will be saved under this root folder.
+ private String mTestOutputRoot;
+ // Store the method name and invocation count to create a unique filename for each trace.
+ private Map<String, Integer> mTestIdInvocationCount = new HashMap<>();
+ private boolean mSimpleperfStartSuccess = false;
+ private boolean mIsCollectPerRun;
+ private boolean mIsTestFailed = false;
+ private boolean mSkipTestFailureMetrics;
+
+ private SimpleperfHelper mSimpleperfHelper = new SimpleperfHelper();
+
+ public SimpleperfListener() {
+ super();
+ }
+
+ /**
+ * Constructor to simulate receiving the instrumentation arguments. Shoud not be used except for
+ * testing.
+ */
+ @VisibleForTesting
+ SimpleperfListener(Bundle args, SimpleperfHelper helper, Map invocationMap) {
+ super(args);
+ mSimpleperfHelper = helper;
+ mTestIdInvocationCount = invocationMap;
+ }
+
+ @Override
+ public void onTestRunStart(DataRecord runData, Description description) {
+ Bundle args = getArgsBundle();
+
+ // Whether to collect for the entire run, or per test.
+ mIsCollectPerRun = Boolean.parseBoolean(args.getString(COLLECT_PER_RUN));
+
+ // Destination folder in the device to save all simpleperf sample files.
+ // Defaulted to /sdcard/test_results if test_output_root is not passed.
+ mTestOutputRoot = args.getString(TEST_OUTPUT_ROOT, DEFAULT_OUTPUT_ROOT);
+
+ // By default this flag is set to false to collect metrics on test failure.
+ mSkipTestFailureMetrics = "true".equals(args.getString(SKIP_TEST_FAILURE_METRICS));
+
+ if (!mIsCollectPerRun) {
+ return;
+ }
+
+ Log.i(getTag(), "Starting simpleperf before test run started.");
+ startSimpleperf();
+ }
+
+ @Override
+ public void onTestStart(DataRecord testData, Description description) {
+ mIsTestFailed = false;
+ if (mIsCollectPerRun) {
+ return;
+ }
+
+ mTestIdInvocationCount.compute(
+ getTestFileName(description), (key, value) -> (value == null) ? 1 : value + 1);
+ Log.i(getTag(), "Starting simpleperf before test started.");
+ startSimpleperf();
+ }
+
+ @Override
+ public void onTestFail(DataRecord testData, Description description, Failure failure) {
+ mIsTestFailed = true;
+ }
+
+ @Override
+ public void onTestEnd(DataRecord testData, Description description) {
+ if (mIsCollectPerRun) {
+ return;
+ }
+
+ if (!mSimpleperfStartSuccess) {
+ Log.i(
+ getTag(),
+ "Skipping simpleperf stop attempt onTestEnd because simpleperf did not start"
+ + "successfully");
+ return;
+ }
+
+ if (mSkipTestFailureMetrics && mIsTestFailed) {
+ Log.i(getTag(), "Skipping metric collection due to test failure");
+ // Stop the existing simpleperf session.
+ try {
+ if (!mSimpleperfHelper.stopSimpleperf()) {
+ Log.e(getTag(), "Failed to stop the simpleperf process.");
+ }
+ } catch (IOException e) {
+ Log.e(getTag(), "Failed to stop simpleperf", e);
+ }
+ } else {
+ Log.i(getTag(), "Stopping simpleperf after test ended.");
+ // Construct test output directory in the below format
+ // <root_folder>/<test_name>/SimpleperfListener/<test_name>-<count>.data
+ Path path =
+ Paths.get(
+ mTestOutputRoot,
+ getTestFileName(description),
+ this.getClass().getSimpleName(),
+ String.format(
+ "%s%s-%d.data",
+ SIMPLEPERF_PREFIX,
+ getTestFileName(description),
+ mTestIdInvocationCount.get(getTestFileName(description))));
+ stopSimpleperf(path, testData);
+ }
+ }
+
+ @Override
+ public void onTestRunEnd(DataRecord runData, Result result) {
+ if (!mIsCollectPerRun) {
+ return;
+ }
+
+ if (!mSimpleperfStartSuccess) {
+ Log.i(getTag(), "Skipping simpleperf stop attempt as simpleperf failed to start.");
+ return;
+ }
+
+ Log.i(getTag(), "Stopping simpleperf after test run ended");
+ Path path =
+ Paths.get(
+ mTestOutputRoot,
+ this.getClass().getSimpleName(),
+ String.format(
+ "%s%d.data", SIMPLEPERF_PREFIX, UUID.randomUUID().hashCode()));
+ stopSimpleperf(path, runData);
+ }
+
+ /** Start simpleperf sampling. */
+ public void startSimpleperf() {
+ mSimpleperfStartSuccess = mSimpleperfHelper.startCollecting();
+ if (!mSimpleperfStartSuccess) {
+ Log.e(getTag(), "Simpleperf did not start successfully.");
+ }
+ }
+
+ /** Stop simpleperf sampling and dump the collected file into the given path. */
+ private void stopSimpleperf(Path path, DataRecord record) {
+ if (!mSimpleperfHelper.stopCollecting(path.toString())) {
+ Log.e(getTag(), "Failed to collect the simpleperf output.");
+ } else {
+ record.addStringMetric(SIMPLEPERF_FILE_PATH, path.toString());
+ }
+ }
+
+ /**
+ * Returns the packagename.classname_methodname which has no special characters and is used to
+ * create file names.
+ */
+ public static String getTestFileName(Description description) {
+ return String.format("%s_%s", description.getClassName(), description.getMethodName());
+ }
+}
diff --git a/libraries/device-collectors/src/test/java/android/device/collectors/SimpleperfListenerTest.java b/libraries/device-collectors/src/test/java/android/device/collectors/SimpleperfListenerTest.java
new file mode 100644
index 0000000..7473f71
--- /dev/null
+++ b/libraries/device-collectors/src/test/java/android/device/collectors/SimpleperfListenerTest.java
@@ -0,0 +1,251 @@
+/*
+ * Copyright (C) 2020 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package android.device.collectors;
+
+import static org.junit.Assert.assertEquals;
+import static org.mockito.ArgumentMatchers.anyString;
+import static org.mockito.Mockito.doReturn;
+import static org.mockito.Mockito.spy;
+import static org.mockito.Mockito.times;
+import static org.mockito.Mockito.verify;
+
+import android.app.Instrumentation;
+import android.os.Bundle;
+import androidx.test.runner.AndroidJUnit4;
+import com.android.helpers.SimpleperfHelper;
+import java.util.HashMap;
+import java.util.Map;
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.runner.Description;
+import org.junit.runner.notification.Failure;
+import org.junit.runner.Result;
+import org.junit.runner.RunWith;
+import org.mockito.Mock;
+import org.mockito.MockitoAnnotations;
+import org.mockito.Spy;
+
+/**
+ * Android Unit tests for {@link SimpleperfListener}.
+ *
+ * <p>To run: atest CollectorDeviceLibTest:android.device.collectors.SimpleperfListenerTest
+ */
+@RunWith(AndroidJUnit4.class)
+public class SimpleperfListenerTest {
+
+ // A {@code Description} to pass when faking a test run start call.
+ private static final Description FAKE_DESCRIPTION = Description.createSuiteDescription("run");
+
+ private static final Description FAKE_TEST_DESCRIPTION =
+ Description.createTestDescription("class", "method");
+
+ private Description mRunDesc;
+ private Description mTest1Desc;
+ private Description mTest2Desc;
+ private SimpleperfListener mListener;
+ @Mock private Instrumentation mInstrumentation;
+ private Map<String, Integer> mInvocationCount;
+ private DataRecord mDataRecord;
+
+ @Spy private SimpleperfHelper mSimpleperfHelper;
+
+ @Before
+ public void setUp() {
+ MockitoAnnotations.initMocks(this);
+ mRunDesc = Description.createSuiteDescription("run");
+ mTest1Desc = Description.createTestDescription("run", "test1");
+ mTest2Desc = Description.createTestDescription("run", "test2");
+ }
+
+ private SimpleperfListener initListener(Bundle b) {
+ mInvocationCount = new HashMap<>();
+
+ SimpleperfListener listener =
+ spy(new SimpleperfListener(b, mSimpleperfHelper, mInvocationCount));
+
+ mDataRecord = listener.createDataRecord();
+ listener.setInstrumentation(mInstrumentation);
+ return listener;
+ }
+
+ /*
+ * Verify simpleperf start and stop collection methods called exactly once for single test.
+ */
+ @Test
+ public void testSimpleperfPerTestSuccessFlow() throws Exception {
+ Bundle b = new Bundle();
+ mListener = initListener(b);
+ doReturn(true).when(mSimpleperfHelper).startCollecting();
+ doReturn(true).when(mSimpleperfHelper).stopCollecting(anyString());
+ // Test run start behavior
+ mListener.testRunStarted(mRunDesc);
+
+ // Test test start behavior
+ mListener.testStarted(mTest1Desc);
+ verify(mSimpleperfHelper, times(1)).startCollecting();
+ mListener.onTestEnd(mDataRecord, mTest1Desc);
+ verify(mSimpleperfHelper, times(1)).stopCollecting(anyString());
+ }
+
+ /*
+ * Verify stop collecting called exactly once when the test failed and the
+ * skip test failure mmetrics is enabled.
+ */
+ @Test
+ public void testSimpleperfPerTestFailureFlowDefault() throws Exception {
+ Bundle b = new Bundle();
+ b.putString(SimpleperfListener.SKIP_TEST_FAILURE_METRICS, "false");
+ mListener = initListener(b);
+
+ doReturn(true).when(mSimpleperfHelper).startCollecting();
+ doReturn(true).when(mSimpleperfHelper).stopCollecting(anyString());
+ // Test run start behavior
+ mListener.testRunStarted(mRunDesc);
+
+ // Test test start behavior
+ mListener.testStarted(mTest1Desc);
+ verify(mSimpleperfHelper, times(1)).startCollecting();
+
+ // Test fail behaviour
+ Failure failureDesc = new Failure(FAKE_TEST_DESCRIPTION, new Exception());
+ mListener.onTestFail(mDataRecord, mTest1Desc, failureDesc);
+ mListener.onTestEnd(mDataRecord, mTest1Desc);
+ verify(mSimpleperfHelper, times(1)).stopCollecting(anyString());
+ }
+
+ /*
+ * Verify stop simpleperf called exactly once when the test failed and the
+ * skip test failure metrics is enabled.
+ */
+ @Test
+ public void testSimpleperfPerTestFailureFlowWithSkipMmetrics() throws Exception {
+ Bundle b = new Bundle();
+ b.putString(SimpleperfListener.SKIP_TEST_FAILURE_METRICS, "true");
+ mListener = initListener(b);
+
+ doReturn(true).when(mSimpleperfHelper).startCollecting();
+ doReturn(true).when(mSimpleperfHelper).stopSimpleperf();
+ // Test run start behavior
+ mListener.testRunStarted(mRunDesc);
+
+ // Test test start behavior
+ mListener.testStarted(mTest1Desc);
+ verify(mSimpleperfHelper, times(1)).startCollecting();
+
+ // Test fail behaviour
+ Failure failureDesc = new Failure(FAKE_TEST_DESCRIPTION, new Exception());
+ mListener.onTestFail(mDataRecord, mTest1Desc, failureDesc);
+ mListener.onTestEnd(mDataRecord, mTest1Desc);
+ verify(mSimpleperfHelper, times(1)).stopSimpleperf();
+ }
+
+ /*
+ * Verify simpleperf start and stop collection methods called exactly once for test run.
+ * and not during each test method.
+ */
+ @Test
+ public void testSimpleperfPerRunSuccessFlow() throws Exception {
+ Bundle b = new Bundle();
+ b.putString(SimpleperfListener.COLLECT_PER_RUN, "true");
+ mListener = initListener(b);
+ doReturn(true).when(mSimpleperfHelper).startCollecting();
+ doReturn(true).when(mSimpleperfHelper).stopCollecting(anyString());
+
+ // Test run start behavior
+ mListener.onTestRunStart(mListener.createDataRecord(), FAKE_DESCRIPTION);
+ verify(mSimpleperfHelper, times(1)).startCollecting();
+ mListener.testStarted(mTest1Desc);
+ verify(mSimpleperfHelper, times(1)).startCollecting();
+ mListener.onTestEnd(mDataRecord, mTest1Desc);
+ verify(mSimpleperfHelper, times(0)).stopCollecting(anyString());
+ mListener.onTestRunEnd(mListener.createDataRecord(), new Result());
+ verify(mSimpleperfHelper, times(1)).stopCollecting(anyString());
+ }
+
+ /*
+ * Verify stop is not called if Simpleperf start did not succeed.
+ */
+ @Test
+ public void testSimpleperfPerRunFailureFlow() throws Exception {
+ Bundle b = new Bundle();
+ b.putString(SimpleperfListener.COLLECT_PER_RUN, "true");
+ mListener = initListener(b);
+ doReturn(false).when(mSimpleperfHelper).startCollecting();
+
+ // Test run start behavior
+ mListener.onTestRunStart(mListener.createDataRecord(), FAKE_DESCRIPTION);
+ verify(mSimpleperfHelper, times(1)).startCollecting();
+ mListener.onTestRunEnd(mListener.createDataRecord(), new Result());
+ verify(mSimpleperfHelper, times(0)).stopCollecting(anyString());
+ }
+
+ /*
+ * Verify simpleperf stop is not invoked if start did not succeed.
+ */
+ @Test
+ public void testSimpleperfStartFailureFlow() throws Exception {
+ Bundle b = new Bundle();
+ mListener = initListener(b);
+ doReturn(false).when(mSimpleperfHelper).startCollecting();
+
+ // Test run start behavior
+ mListener.testRunStarted(mRunDesc);
+
+ // Test test start behavior
+ mListener.testStarted(mTest1Desc);
+ verify(mSimpleperfHelper, times(1)).startCollecting();
+ mListener.onTestEnd(mDataRecord, mTest1Desc);
+ verify(mSimpleperfHelper, times(0)).stopCollecting(anyString());
+ }
+
+ /*
+ * Verify test method invocation count is updated successfully based on the number of times the
+ * test method is invoked.
+ */
+ @Test
+ public void testSimpleperfInvocationCount() throws Exception {
+ Bundle b = new Bundle();
+ mListener = initListener(b);
+ doReturn(true).when(mSimpleperfHelper).startCollecting();
+ doReturn(true).when(mSimpleperfHelper).stopCollecting(anyString());
+
+ // Test run start behavior
+ mListener.testRunStarted(mRunDesc);
+
+ // Test1 invocation 1 start behavior
+ mListener.testStarted(mTest1Desc);
+ verify(mSimpleperfHelper, times(1)).startCollecting();
+ mListener.onTestEnd(mDataRecord, mTest1Desc);
+ verify(mSimpleperfHelper, times(1)).stopCollecting(anyString());
+
+ // Test1 invocation 2 start behaviour
+ mListener.testStarted(mTest1Desc);
+ verify(mSimpleperfHelper, times(2)).startCollecting();
+ mListener.onTestEnd(mDataRecord, mTest1Desc);
+ verify(mSimpleperfHelper, times(2)).stopCollecting(anyString());
+
+ // Test2 invocation 1 start behaviour
+ mListener.testStarted(mTest2Desc);
+ verify(mSimpleperfHelper, times(3)).startCollecting();
+ mDataRecord = mListener.createDataRecord();
+ mListener.onTestEnd(mDataRecord, mTest2Desc);
+ verify(mSimpleperfHelper, times(3)).stopCollecting(anyString());
+
+ // Check if the test count is incremented properly.
+ assertEquals(2, (int) mInvocationCount.get(mListener.getTestFileName(mTest1Desc)));
+ assertEquals(1, (int) mInvocationCount.get(mListener.getTestFileName(mTest2Desc)));
+ }
+}
