Merge "Delete performance related classes that are unused"
diff --git a/src/com/android/performance/tests/FioBenchmarkTest.java b/src/com/android/performance/tests/FioBenchmarkTest.java
deleted file mode 100644
index 83b119f..0000000
--- a/src/com/android/performance/tests/FioBenchmarkTest.java
+++ /dev/null
@@ -1,1211 +0,0 @@
-/*
- * Copyright (C) 2011 The Android Open Source Project
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package com.android.performance.tests;
-
-import com.android.ddmlib.IDevice;
-import com.android.ddmlib.MultiLineReceiver;
-import com.android.ddmlib.NullOutputReceiver;
-import com.android.tradefed.config.Option;
-import com.android.tradefed.device.DeviceNotAvailableException;
-import com.android.tradefed.device.ITestDevice;
-import com.android.tradefed.log.LogUtil.CLog;
-import com.android.tradefed.result.FileInputStreamSource;
-import com.android.tradefed.result.ITestInvocationListener;
-import com.android.tradefed.result.InputStreamSource;
-import com.android.tradefed.result.LogDataType;
-import com.android.tradefed.testtype.IDeviceTest;
-import com.android.tradefed.testtype.IRemoteTest;
-import com.android.tradefed.util.FileUtil;
-import com.android.tradefed.util.StreamUtil;
-import com.android.tradefed.util.proto.TfMetricProtoUtil;
-
-import junit.framework.TestCase;
-
-import org.junit.Assert;
-
-import java.io.File;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.LinkedList;
-import java.util.List;
-import java.util.Map;
-import java.util.Map.Entry;
-import java.util.Set;
-import java.util.concurrent.TimeUnit;
-
-/**
- * Runs the FIO benchmarks.
- *
- * <p>This test pushes the FIO executable to the device and runs several benchmarks. Running each
- * benchmark consists of creating a config file, creating one or more data files, clearing the disk
- * cache and then running FIO. The test runs a variety of different configurations including a
- * simple benchmark with a single thread, a storage benchmark with 4 threads, a media server
- * emulator, and a media scanner emulator.
- */
-public class FioBenchmarkTest implements IDeviceTest, IRemoteTest {
- // TODO: Refactor this to only pick out fields we care about.
- private static final String[] FIO_V0_RESULT_FIELDS = {
- "jobname",
- "groupid",
- "error",
- // Read stats
- "read-kb-io",
- "read-bandwidth",
- "read-runtime",
- "read-slat-min",
- "read-slat-max",
- "read-slat-mean",
- "read-slat-stddev",
- "read-clat-min",
- "read-clat-max",
- "read-clat-mean",
- "read-clat-stddev",
- "read-bandwidth-min",
- "read-bandwidth-max",
- "read-bandwidth-percent",
- "read-bandwidth-mean",
- "read-bandwidth-stddev",
- // Write stats
- "write-kb-io",
- "write-bandwidth",
- "write-runtime",
- "write-slat-min",
- "write-slat-max",
- "write-slat-mean",
- "write-slat-stddev",
- "write-clat-min",
- "write-clat-max",
- "write-clat-mean",
- "write-clat-stddev",
- "write-bandwidth-min",
- "write-bandwidth-max",
- "write-bandwidth-percent",
- "write-bandwidth-mean",
- "write-bandwidth-stddev",
- // CPU stats
- "cpu-user",
- "cpu-system",
- "cpu-context-switches",
- "cpu-major-page-faults",
- "cpu-minor-page-faults",
- // IO depth stats
- "io-depth-1",
- "io-depth-2",
- "io-depth-4",
- "io-depth-8",
- "io-depth-16",
- "io-depth-32",
- "io-depth-64",
- // IO lat stats
- "io-lat-2-ms",
- "io-lat-4-ms",
- "io-lat-10-ms",
- "io-lat-20-ms",
- "io-lat-50-ms",
- "io-lat-100-ms",
- "io-lat-250-ms",
- "io-lat-500-ms",
- "io-lat-750-ms",
- "io-lat-1000-ms",
- "io-lat-2000-ms"
- };
- private static final String[] FIO_V3_RESULT_FIELDS = {
- "terse-version",
- "fio-version",
- "jobname",
- "groupid",
- "error",
- // Read stats
- "read-kb-io",
- "read-bandwidth",
- "read-iops",
- "read-runtime",
- "read-slat-min",
- "read-slat-max",
- "read-slat-mean",
- "read-slat-stddev",
- "read-clat-min",
- "read-clat-max",
- "read-clat-mean",
- "read-clat-stddev",
- null,
- null,
- null,
- null,
- null,
- null,
- null,
- null,
- null,
- null,
- null,
- null,
- null,
- null,
- null,
- null,
- null,
- null,
- null,
- null,
- "read-lat-min",
- "read-lat-max",
- "read-lat-mean",
- "read-lat-stddev",
- "read-bandwidth-min",
- "read-bandwidth-max",
- "read-bandwidth-percent",
- "read-bandwidth-mean",
- "read-bandwidth-stddev",
- // Write stats
- "write-kb-io",
- "write-bandwidth",
- "write-iops",
- "write-runtime",
- "write-slat-min",
- "write-slat-max",
- "write-slat-mean",
- "write-slat-stddev",
- "write-clat-min",
- "write-clat-max",
- "write-clat-mean",
- "write-clat-stddev",
- null,
- null,
- null,
- null,
- null,
- null,
- null,
- null,
- null,
- null,
- null,
- null,
- null,
- null,
- null,
- null,
- null,
- null,
- null,
- null,
- "write-lat-min",
- "write-lat-max",
- "write-lat-mean",
- "write-lat-stddev",
- "write-bandwidth-min",
- "write-bandwidth-max",
- "write-bandwidth-percent",
- "write-bandwidth-mean",
- "write-bandwidth-stddev",
- // CPU stats
- "cpu-user",
- "cpu-system",
- "cpu-context-switches",
- "cpu-major-page-faults",
- "cpu-minor-page-faults",
- // IO depth stats
- "io-depth-1",
- "io-depth-2",
- "io-depth-4",
- "io-depth-8",
- "io-depth-16",
- "io-depth-32",
- "io-depth-64",
- // IO lat stats
- "io-lat-2-us",
- "io-lat-4-us",
- "io-lat-10-us",
- "io-lat-20-us",
- "io-lat-50-us",
- "io-lat-100-us",
- "io-lat-250-us",
- "io-lat-500-us",
- "io-lat-750-us",
- "io-lat-1000-us",
- "io-lat-2-ms",
- "io-lat-4-ms",
- "io-lat-10-ms",
- "io-lat-20-ms",
- "io-lat-50-ms",
- "io-lat-100-ms",
- "io-lat-250-ms",
- "io-lat-500-ms",
- "io-lat-750-ms",
- "io-lat-1000-ms",
- "io-lat-2000-ms",
- "io-lat-greater"
- };
-
- private List<TestInfo> mTestCases = null;
-
- /**
- * Holds info about a job. The job translates into a job in the FIO config file. Contains the
- * job name and a map from keys to values.
- */
- private static class JobInfo {
- public String mJobName = null;
- public Map<String, String> mParameters = new HashMap<>();
-
- /**
- * Gets the job as a string.
- *
- * @return a string of the job formatted for the config file.
- */
- public String createJob() {
- if (mJobName == null) {
- return "";
- }
- StringBuilder sb = new StringBuilder();
- sb.append(String.format("[%s]\n", mJobName));
- for (Entry<String, String> parameter : mParameters.entrySet()) {
- if (parameter.getValue() == null) {
- sb.append(String.format("%s\n", parameter.getKey()));
- } else {
- sb.append(String.format("%s=%s\n", parameter.getKey(), parameter.getValue()));
- }
- }
- return sb.toString();
- }
- }
-
- /**
- * Holds info about a file used in the benchmark. Because of limitations in FIO on Android, the
- * file needs to be created before the tests are run. Contains the file name and size in kB.
- */
- private static class TestFileInfo {
- public String mFileName = null;
- public int mSize = -1;
- }
-
- /** Holds info about the perf metric that are cared about for a given job. */
- private static class PerfMetricInfo {
- public enum ResultType {
- STRING,
- INT,
- FLOAT,
- PERCENT;
-
- String value(String input) {
- switch (this) {
- case STRING:
- case INT:
- case FLOAT:
- return input;
- case PERCENT:
- if (input.length() < 2 || !input.endsWith("%")) {
- return null;
- }
- try {
- return String.format(
- "%f",
- Double.parseDouble(input.substring(0, input.length() - 1))
- / 100);
- } catch (NumberFormatException e) {
- return null;
- }
- default:
- return null;
- }
- }
- }
-
- public String mJobName = null;
- public String mFieldName = null;
- public String mPostKey = null;
- public ResultType mType = ResultType.STRING;
- }
-
- /**
- * Holds the info associated with a test.
- *
- * <p>Contains the test name, key, a list of {@link JobInfo}, a set of {@link TestFileInfo}, and
- * a set of {@link PerfMetricInfo}.
- */
- private static class TestInfo {
- public String mTestName = null;
- public String mKey = null;
- public List<JobInfo> mJobs = new LinkedList<>();
- public Set<TestFileInfo> mTestFiles = new HashSet<>();
- public Set<PerfMetricInfo> mPerfMetrics = new HashSet<>();
-
- /**
- * Gets the config file.
- *
- * @return a string containing the contents of the config file needed to run the benchmark.
- */
- private String createConfig() {
- StringBuilder sb = new StringBuilder();
- for (JobInfo job : mJobs) {
- sb.append(String.format("%s\n", job.createJob()));
- }
- return sb.toString();
- }
- }
-
- /**
- * Parses the output of the FIO and allows the values to be looked up by job name and property.
- */
- private static class FioParser extends MultiLineReceiver {
- public Map<String, Map<String, String>> mResults = new HashMap<>();
-
- /**
- * Gets the result for a job and property, or null if the job or the property do not exist.
- *
- * @param job the name of the job.
- * @param property the name of the property. See {@code FIO_RESULT_FIELDS}.
- * @return the fio results for the job and property or null if it does not exist.
- */
- public String getResult(String job, String property) {
- if (!mResults.containsKey(job)) {
- return null;
- }
- return mResults.get(job).get(property);
- }
-
- /** {@inheritDoc} */
- @Override
- public void processNewLines(String[] lines) {
- for (String line : lines) {
- CLog.d(line);
- String[] fields = line.split(";");
- if (fields.length < FIO_V0_RESULT_FIELDS.length) {
- continue;
- }
- if (fields.length < FIO_V3_RESULT_FIELDS.length) {
- Map<String, String> r = new HashMap<>();
- for (int i = 0; i < FIO_V0_RESULT_FIELDS.length; i++) {
- r.put(FIO_V0_RESULT_FIELDS[i], fields[i]);
- }
- mResults.put(fields[0], r); // Job name is index 0
- } else if ("3".equals(fields[0])) {
- Map<String, String> r = new HashMap<>();
- for (int i = 0; i < FIO_V3_RESULT_FIELDS.length; i++) {
- r.put(FIO_V3_RESULT_FIELDS[i], fields[i]);
- }
- mResults.put(fields[2], r); // Job name is index 2
- } else {
- Assert.fail("Unknown fio terse output version");
- }
- }
- }
-
- /** {@inheritDoc} */
- @Override
- public boolean isCancelled() {
- return false;
- }
- }
-
- ITestDevice mTestDevice = null;
-
- private String mFioDir = null;
- private String mFioBin = null;
- private String mFioConfig = null;
-
- @Option(
- name = "fio-location",
- description =
- "The path to the precompiled FIO executable. If "
- + "unset, try to use fio from the system image.")
- private String mFioLocation = null;
-
- @Option(name = "tmp-dir", description = "The directory used for interal benchmarks.")
- private String mTmpDir = "/data/tmp/fio";
-
- @Option(name = "internal-test-dir", description = "The directory used for interal benchmarks.")
- private String mInternalTestDir = "/data/fio/data";
-
- @Option(name = "media-test-dir", description = "The directory used for media benchmarks.")
- private String mMediaTestDir = "${EXTERNAL_STORAGE}/fio";
-
- @Option(name = "external-test-dir", description = "The directory used for external benchmarks.")
- private String mExternalTestDir = "${EXTERNAL_STORAGE}/fio";
-
- @Option(name = "collect-yaffs-logs", description = "Collect yaffs logs before and after tests")
- private Boolean mCollectYaffsLogs = false;
-
- @Option(name = "run-simple-internal-test", description = "Run the simple internal benchmark.")
- private Boolean mRunSimpleInternalTest = true;
-
- @Option(
- name = "simple-internal-file-size",
- description = "The file size of the simple internal benchmark in MB.")
- private int mSimpleInternalFileSize = 256;
-
- @Option(name = "run-simple-external-test", description = "Run the simple external benchmark.")
- private Boolean mRunSimpleExternalTest = false;
-
- @Option(
- name = "simple-external-file-size",
- description = "The file size of the simple external benchmark in MB.")
- private int mSimpleExternalFileSize = 256;
-
- @Option(name = "run-storage-internal-test", description = "Run the storage internal benchmark.")
- private Boolean mRunStorageInternalTest = true;
-
- @Option(
- name = "storage-internal-file-size",
- description = "The file size of the storage internal benchmark in MB.")
- private int mStorageInternalFileSize = 256;
-
- @Option(
- name = "storage-internal-job-count",
- description = "The number of jobs for the storage internal benchmark.")
- private int mStorageInternalJobCount = 4;
-
- @Option(name = "run-storage-external-test", description = "Run the storage external benchmark.")
- private Boolean mRunStorageExternalTest = false;
-
- @Option(
- name = "storage-external-file-size",
- description = "The file size of the storage external benchmark in MB.")
- private int mStorageExternalFileSize = 256;
-
- @Option(
- name = "storage-external-job-count",
- description = "The number of jobs for the storage external benchmark.")
- private int mStorageExternalJobCount = 4;
-
- @Option(name = "run-media-server-test", description = "Run the media server benchmark.")
- private Boolean mRunMediaServerTest = false;
-
- @Option(
- name = "media-server-duration",
- description = "The duration of the media server benchmark in secs.")
- private long mMediaServerDuration = 30;
-
- @Option(
- name = "media-server-media-file-size",
- description = "The media file size of the media server benchmark in MB.")
- private int mMediaServerMediaFileSize = 256;
-
- @Option(
- name = "media-server-worker-file-size",
- description = "The worker file size of the media server benchmark in MB.")
- private int mMediaServerWorkerFileSize = 256;
-
- @Option(
- name = "media-server-worker-job-count",
- description = "The number of worker jobs for the media server benchmark.")
- private int mMediaServerWorkerJobCount = 4;
-
- @Option(name = "run-media-scanner-test", description = "Run the media scanner benchmark.")
- private Boolean mRunMediaScannerTest = false;
-
- @Option(
- name = "media-scanner-media-file-size",
- description = "The media file size of the media scanner benchmark in kB.")
- private int mMediaScannerMediaFileSize = 8;
-
- @Option(
- name = "media-scanner-media-file-count",
- description = "The number of media files to scan.")
- private int mMediaScannerMediaFileCount = 256;
-
- @Option(
- name = "media-scanner-worker-file-size",
- description = "The worker file size of the media scanner benchmark in MB.")
- private int mMediaScannerWorkerFileSize = 256;
-
- @Option(
- name = "media-scanner-worker-job-count",
- description = "The number of worker jobs for the media server benchmark.")
- private int mMediaScannerWorkerJobCount = 4;
-
- @Option(
- name = "key-suffix",
- description = "The suffix to add to the reporting key in order to override the default")
- private String mKeySuffix = null;
-
- /** Sets up all the benchmarks. */
- private void setupTests() {
- if (mTestCases != null) {
- // assume already set up
- return;
- }
-
- mTestCases = new LinkedList<>();
-
- if (mRunSimpleInternalTest) {
- addSimpleTest("read", "sync", true);
- addSimpleTest("write", "sync", true);
- addSimpleTest("randread", "sync", true);
- addSimpleTest("randwrite", "sync", true);
- addSimpleTest("randread", "mmap", true);
- addSimpleTest("randwrite", "mmap", true);
- }
-
- if (mRunSimpleExternalTest) {
- addSimpleTest("read", "sync", false);
- addSimpleTest("write", "sync", false);
- addSimpleTest("randread", "sync", false);
- addSimpleTest("randwrite", "sync", false);
- addSimpleTest("randread", "mmap", false);
- addSimpleTest("randwrite", "mmap", false);
- }
-
- if (mRunStorageInternalTest) {
- addStorageTest("read", "sync", true);
- addStorageTest("write", "sync", true);
- addStorageTest("randread", "sync", true);
- addStorageTest("randwrite", "sync", true);
- addStorageTest("randread", "mmap", true);
- addStorageTest("randwrite", "mmap", true);
- }
-
- if (mRunStorageExternalTest) {
- addStorageTest("read", "sync", false);
- addStorageTest("write", "sync", false);
- addStorageTest("randread", "sync", false);
- addStorageTest("randwrite", "sync", false);
- addStorageTest("randread", "mmap", false);
- addStorageTest("randwrite", "mmap", false);
- }
-
- if (mRunMediaServerTest) {
- addMediaServerTest("read");
- addMediaServerTest("write");
- }
-
- if (mRunMediaScannerTest) {
- addMediaScannerTest();
- }
- }
-
- /**
- * Sets up the simple FIO benchmark.
- *
- * <p>The test consists of a single process reading or writing to a file.
- *
- * @param rw the type of IO pattern. One of {@code read}, {@code write}, {@code randread}, or
- * {@code randwrite}.
- * @param ioengine defines how the job issues I/O. Such as {@code sync}, {@code vsync}, {@code
- * mmap}, or {@code cpuio} and others.
- * @param internal whether the test should be run on the internal (/data) partition or the
- * external partition.
- */
- private void addSimpleTest(String rw, String ioengine, boolean internal) {
- String fileName = "testfile";
- String jobName = "job";
-
- String directory;
- int fileSize;
-
- TestInfo t = new TestInfo();
- if (internal) {
- t.mTestName = String.format("SimpleBenchmark-int-%s-%s", ioengine, rw);
- t.mKey = "fio_simple_int_benchmark";
- directory = mInternalTestDir;
- fileSize = mSimpleInternalFileSize;
- } else {
- t.mTestName = String.format("SimpleBenchmark-ext-%s-%s", ioengine, rw);
- t.mKey = "fio_simple_ext_benchmark";
- directory = mExternalTestDir;
- fileSize = mSimpleExternalFileSize;
- }
-
- TestFileInfo f = new TestFileInfo();
- f.mFileName = new File(directory, fileName).getAbsolutePath();
- f.mSize = fileSize * 1024; // fileSize is in MB but we want it in kB.
- t.mTestFiles.add(f);
-
- JobInfo j = new JobInfo();
- j.mJobName = jobName;
- j.mParameters.put("directory", directory);
- j.mParameters.put("filename", fileName);
- j.mParameters.put("fsync", "1024");
- j.mParameters.put("ioengine", ioengine);
- j.mParameters.put("rw", rw);
- j.mParameters.put("size", String.format("%dM", fileSize));
- t.mJobs.add(j);
-
- PerfMetricInfo m = new PerfMetricInfo();
- m.mJobName = jobName;
- if ("sync".equals(ioengine)) {
- m.mPostKey = String.format("%s_bandwidth", rw);
- } else {
- m.mPostKey = String.format("%s_%s_bandwidth", rw, ioengine);
- }
- m.mType = PerfMetricInfo.ResultType.FLOAT;
- if (rw.endsWith("read")) {
- m.mFieldName = "read-bandwidth-mean";
- } else if (rw.endsWith("write")) {
- m.mFieldName = "write-bandwidth-mean";
- }
- t.mPerfMetrics.add(m);
-
- mTestCases.add(t);
- }
-
- /**
- * Sets up the storage FIO benchmark.
- *
- * <p>The test consists of several processes reading or writing to a file.
- *
- * @param rw the type of IO pattern. One of {@code read}, {@code write}, {@code randread}, or
- * {@code randwrite}.
- * @param ioengine defines how the job issues I/O. Such as {@code sync}, {@code vsync}, {@code
- * mmap}, or {@code cpuio} and others.
- * @param internal whether the test should be run on the internal (/data) partition or the
- * external partition.
- */
- private void addStorageTest(String rw, String ioengine, boolean internal) {
- String fileName = "testfile";
- String jobName = "workers";
-
- String directory;
- int fileSize;
- int jobCount;
-
- TestInfo t = new TestInfo();
- if (internal) {
- t.mTestName = String.format("StorageBenchmark-int-%s-%s", ioengine, rw);
- t.mKey = "fio_storage_int_benchmark";
- directory = mInternalTestDir;
- fileSize = mStorageInternalFileSize;
- jobCount = mStorageInternalJobCount;
- } else {
- t.mTestName = String.format("StorageBenchmark-ext-%s-%s", ioengine, rw);
- t.mKey = "fio_storage_ext_benchmark";
- directory = mExternalTestDir;
- fileSize = mStorageExternalFileSize;
- jobCount = mStorageExternalJobCount;
- }
-
- TestFileInfo f = new TestFileInfo();
- f.mFileName = new File(directory, fileName).getAbsolutePath();
- f.mSize = fileSize * 1024; // fileSize is in MB but we want it in kB.
- t.mTestFiles.add(f);
-
- JobInfo j = new JobInfo();
- j.mJobName = jobName;
- j.mParameters.put("directory", directory);
- j.mParameters.put("filename", fileName);
- j.mParameters.put("fsync", "1024");
- j.mParameters.put("group_reporting", null);
- j.mParameters.put("ioengine", ioengine);
- j.mParameters.put("new_group", null);
- j.mParameters.put("numjobs", String.format("%d", jobCount));
- j.mParameters.put("rw", rw);
- j.mParameters.put("size", String.format("%dM", fileSize));
- t.mJobs.add(j);
-
- PerfMetricInfo m = new PerfMetricInfo();
- m.mJobName = jobName;
- if ("sync".equals(ioengine)) {
- m.mPostKey = String.format("%s_bandwidth", rw);
- } else {
- m.mPostKey = String.format("%s_%s_bandwidth", rw, ioengine);
- }
- m.mType = PerfMetricInfo.ResultType.FLOAT;
- if (rw.endsWith("read")) {
- m.mFieldName = "read-bandwidth-mean";
- } else if (rw.endsWith("write")) {
- m.mFieldName = "write-bandwidth-mean";
- }
- t.mPerfMetrics.add(m);
-
- m = new PerfMetricInfo();
- m.mJobName = jobName;
- if ("sync".equals(ioengine)) {
- m.mPostKey = String.format("%s_latency", rw);
- } else {
- m.mPostKey = String.format("%s_%s_latency", rw, ioengine);
- }
- m.mType = PerfMetricInfo.ResultType.FLOAT;
- if (rw.endsWith("read")) {
- m.mFieldName = "read-clat-mean";
- } else if (rw.endsWith("write")) {
- m.mFieldName = "write-clat-mean";
- }
- t.mPerfMetrics.add(m);
-
- mTestCases.add(t);
- }
-
- /**
- * Sets up the media server benchmark.
- *
- * <p>The test consists of a single process at a higher priority reading or writing to a file
- * while several other worker processes read and write to a different file.
- *
- * @param rw the type of IO pattern. One of {@code read}, {@code write}
- */
- private void addMediaServerTest(String rw) {
- String mediaJob = "media-server";
- String mediaFile = "mediafile";
- String workerJob = "workers";
- String workerFile = "workerfile";
-
- TestInfo t = new TestInfo();
- t.mTestName = String.format("MediaServerBenchmark-%s", rw);
- t.mKey = "fio_media_server_benchmark";
-
- TestFileInfo f = new TestFileInfo();
- f.mFileName = new File(mMediaTestDir, mediaFile).getAbsolutePath();
- f.mSize = mMediaServerMediaFileSize * 1024; // File size is in MB but we want it in kB.
- t.mTestFiles.add(f);
-
- f = new TestFileInfo();
- f.mFileName = new File(mMediaTestDir, workerFile).getAbsolutePath();
- f.mSize = mMediaServerWorkerFileSize * 1024; // File size is in MB but we want it in kB.
- t.mTestFiles.add(f);
-
- JobInfo j = new JobInfo();
- j.mJobName = "global";
- j.mParameters.put("directory", mMediaTestDir);
- j.mParameters.put("fsync", "1024");
- j.mParameters.put("ioengine", "sync");
- j.mParameters.put("runtime", String.format("%d", mMediaServerDuration));
- j.mParameters.put("time_based", null);
- t.mJobs.add(j);
-
- j = new JobInfo();
- j.mJobName = mediaJob;
- j.mParameters.put("filename", mediaFile);
- j.mParameters.put("iodepth", "32");
- j.mParameters.put("nice", "-16");
- j.mParameters.put("rate", "6m");
- j.mParameters.put("rw", rw);
- j.mParameters.put("size", String.format("%dM", mMediaServerMediaFileSize));
- t.mJobs.add(j);
-
- j = new JobInfo();
- j.mJobName = workerJob;
- j.mParameters.put("filename", workerFile);
- j.mParameters.put("group_reporting", null);
- j.mParameters.put("new_group", null);
- j.mParameters.put("nice", "0");
- j.mParameters.put("numjobs", String.format("%d", mMediaServerWorkerJobCount));
- j.mParameters.put("rw", "randrw");
- j.mParameters.put("size", String.format("%dM", mMediaServerWorkerFileSize));
- t.mJobs.add(j);
-
- PerfMetricInfo m = new PerfMetricInfo();
- m.mJobName = mediaJob;
- m.mPostKey = String.format("%s_media_bandwidth", rw);
- m.mType = PerfMetricInfo.ResultType.FLOAT;
- if (rw.endsWith("read")) {
- m.mFieldName = "read-bandwidth-mean";
- } else if (rw.endsWith("write")) {
- m.mFieldName = "write-bandwidth-mean";
- }
- t.mPerfMetrics.add(m);
-
- m = new PerfMetricInfo();
- m.mJobName = mediaJob;
- m.mPostKey = String.format("%s_media_latency", rw);
- m.mType = PerfMetricInfo.ResultType.FLOAT;
- if (rw.endsWith("read")) {
- m.mFieldName = "read-clat-mean";
- } else if (rw.endsWith("write")) {
- m.mFieldName = "write-clat-mean";
- }
- t.mPerfMetrics.add(m);
-
- m = new PerfMetricInfo();
- m.mJobName = workerJob;
- m.mPostKey = String.format("%s_workers_read_bandwidth", rw);
- m.mFieldName = "read-bandwidth-mean";
- m.mType = PerfMetricInfo.ResultType.FLOAT;
- t.mPerfMetrics.add(m);
-
- m = new PerfMetricInfo();
- m.mJobName = workerJob;
- m.mPostKey = String.format("%s_workers_write_bandwidth", rw);
- m.mFieldName = "write-bandwidth-mean";
- m.mType = PerfMetricInfo.ResultType.FLOAT;
- t.mPerfMetrics.add(m);
-
- mTestCases.add(t);
- }
-
- /**
- * Sets up the media scanner benchmark.
- *
- * <p>The test consists of a single process reading several small files while several other
- * worker processes read and write to a different file.
- */
- private void addMediaScannerTest() {
- String mediaJob = "media-server";
- String mediaFile = "mediafile.%d";
- String workerJob = "workers";
- String workerFile = "workerfile";
-
- TestInfo t = new TestInfo();
- t.mTestName = "MediaScannerBenchmark";
- t.mKey = "fio_media_scanner_benchmark";
-
- TestFileInfo f;
- for (int i = 0; i < mMediaScannerMediaFileCount; i++) {
- f = new TestFileInfo();
- f.mFileName = new File(mMediaTestDir, String.format(mediaFile, i)).getAbsolutePath();
- f.mSize = mMediaScannerMediaFileSize; // File size is already in kB so do nothing.
- t.mTestFiles.add(f);
- }
-
- f = new TestFileInfo();
- f.mFileName = new File(mMediaTestDir, workerFile).getAbsolutePath();
- f.mSize = mMediaScannerWorkerFileSize * 1024; // File size is in MB but we want it in kB.
- t.mTestFiles.add(f);
-
- JobInfo j = new JobInfo();
- j.mJobName = "global";
- j.mParameters.put("directory", mMediaTestDir);
- j.mParameters.put("fsync", "1024");
- j.mParameters.put("ioengine", "sync");
- t.mJobs.add(j);
-
- j = new JobInfo();
- j.mJobName = mediaJob;
- StringBuilder fileNames = new StringBuilder();
- fileNames.append(String.format(mediaFile, 0));
- for (int i = 1; i < mMediaScannerMediaFileCount; i++) {
- fileNames.append(String.format(":%s", String.format(mediaFile, i)));
- }
- j.mParameters.put("filename", fileNames.toString());
- j.mParameters.put("exitall", null);
- j.mParameters.put("openfiles", "4");
- j.mParameters.put("rw", "read");
- t.mJobs.add(j);
-
- j = new JobInfo();
- j.mJobName = workerJob;
- j.mParameters.put("filename", workerFile);
- j.mParameters.put("group_reporting", null);
- j.mParameters.put("new_group", null);
- j.mParameters.put("numjobs", String.format("%d", mMediaScannerWorkerJobCount));
- j.mParameters.put("rw", "randrw");
- j.mParameters.put("size", String.format("%dM", mMediaScannerWorkerFileSize));
- t.mJobs.add(j);
-
- PerfMetricInfo m = new PerfMetricInfo();
- m.mJobName = mediaJob;
- m.mPostKey = "media_bandwidth";
- m.mFieldName = "read-bandwidth-mean";
- m.mType = PerfMetricInfo.ResultType.FLOAT;
- t.mPerfMetrics.add(m);
-
- m = new PerfMetricInfo();
- m.mJobName = mediaJob;
- m.mPostKey = "media_latency";
- m.mFieldName = "read-clat-mean";
- m.mType = PerfMetricInfo.ResultType.FLOAT;
- t.mPerfMetrics.add(m);
-
- m = new PerfMetricInfo();
- m.mJobName = workerJob;
- m.mPostKey = "workers_read_bandwidth";
- m.mFieldName = "read-bandwidth-mean";
- m.mType = PerfMetricInfo.ResultType.FLOAT;
- t.mPerfMetrics.add(m);
-
- m = new PerfMetricInfo();
- m.mJobName = workerJob;
- m.mPostKey = "workers_write_bandwidth";
- m.mFieldName = "write-bandwidth-mean";
- m.mType = PerfMetricInfo.ResultType.FLOAT;
- t.mPerfMetrics.add(m);
-
- mTestCases.add(t);
- }
-
- /**
- * Creates the directories needed to run FIO, pushes the FIO executable to the device, and stops
- * the runtime.
- *
- * @throws DeviceNotAvailableException if the device is not available.
- */
- private void setupDevice() throws DeviceNotAvailableException {
- mTestDevice.executeShellCommand("stop");
- mTestDevice.executeShellCommand(String.format("mkdir -p %s", mFioDir));
- mTestDevice.executeShellCommand(String.format("mkdir -p %s", mTmpDir));
- mTestDevice.executeShellCommand(String.format("mkdir -p %s", mInternalTestDir));
- mTestDevice.executeShellCommand(String.format("mkdir -p %s", mMediaTestDir));
- if (mExternalTestDir != null) {
- mTestDevice.executeShellCommand(String.format("mkdir -p %s", mExternalTestDir));
- }
- if (mFioLocation != null) {
- mTestDevice.pushFile(new File(mFioLocation), mFioBin);
- mTestDevice.executeShellCommand(String.format("chmod 755 %s", mFioBin));
- }
- }
-
- /**
- * Reverses the actions of {@link #setDevice(ITestDevice)}.
- *
- * @throws DeviceNotAvailableException If the device is not available.
- */
- private void cleanupDevice() throws DeviceNotAvailableException {
- if (mExternalTestDir != null) {
- mTestDevice.executeShellCommand(String.format("rm -r %s", mExternalTestDir));
- }
- mTestDevice.executeShellCommand(String.format("rm -r %s", mMediaTestDir));
- mTestDevice.executeShellCommand(String.format("rm -r %s", mInternalTestDir));
- mTestDevice.executeShellCommand(String.format("rm -r %s", mTmpDir));
- mTestDevice.executeShellCommand(String.format("rm -r %s", mFioDir));
- mTestDevice.executeShellCommand("start");
- mTestDevice.waitForDeviceAvailable();
- }
-
- /**
- * Runs a single test, including creating the test files, clearing the cache, collecting before
- * and after files, running the benchmark, and reporting the results.
- *
- * @param test the benchmark.
- * @param listener the ITestInvocationListener
- * @throws DeviceNotAvailableException if the device is not available.
- */
- private void runTest(TestInfo test, ITestInvocationListener listener)
- throws DeviceNotAvailableException {
- CLog.i("Running %s benchmark", test.mTestName);
- mTestDevice.executeShellCommand(String.format("rm -r %s/*", mTmpDir));
- mTestDevice.executeShellCommand(String.format("rm -r %s/*", mInternalTestDir));
- mTestDevice.executeShellCommand(String.format("rm -r %s/*", mMediaTestDir));
- if (mExternalTestDir != null) {
- mTestDevice.executeShellCommand(String.format("rm -r %s/*", mExternalTestDir));
- }
-
- for (TestFileInfo file : test.mTestFiles) {
- CLog.v("Creating file: %s, size: %dkB", file.mFileName, file.mSize);
- String cmd =
- String.format(
- "dd if=/dev/urandom of=%s bs=1024 count=%d",
- file.mFileName, file.mSize);
- int timeout = file.mSize * 2 * 1000; // Timeout is 2 seconds per kB.
- mTestDevice.executeShellCommand(
- cmd, new NullOutputReceiver(), timeout, TimeUnit.MILLISECONDS, 2);
- }
-
- CLog.i("Creating config");
- CLog.d("Config file:\n%s", test.createConfig());
- mTestDevice.pushString(test.createConfig(), mFioConfig);
-
- CLog.i("Dropping cache");
- mTestDevice.executeShellCommand("echo 3 > /proc/sys/vm/drop_caches");
-
- collectLogs(test, listener, "before");
-
- CLog.i("Running test");
- FioParser output = new FioParser();
- // Run FIO with a timeout of 1 hour.
- mTestDevice.executeShellCommand(
- String.format("%s --minimal %s", mFioBin, mFioConfig),
- output,
- 60 * 60 * 1000,
- TimeUnit.MILLISECONDS,
- 2);
-
- collectLogs(test, listener, "after");
-
- // Report metrics
- Map<String, String> metrics = new HashMap<>();
- String key = mKeySuffix == null ? test.mKey : test.mKey + mKeySuffix;
-
- listener.testRunStarted(key, 0);
- for (PerfMetricInfo m : test.mPerfMetrics) {
- if (!output.mResults.containsKey(m.mJobName)) {
- CLog.w("Job name %s was not found in the results", m.mJobName);
- continue;
- }
-
- String value = output.getResult(m.mJobName, m.mFieldName);
- if (value != null) {
- metrics.put(m.mPostKey, m.mType.value(value));
- } else {
- CLog.w("%s was not in results for the job %s", m.mFieldName, m.mJobName);
- }
- }
-
- CLog.d("About to report metrics to %s: %s", key, metrics);
- listener.testRunEnded(0, TfMetricProtoUtil.upgradeConvert(metrics));
- }
-
- private void collectLogs(TestInfo testInfo, ITestInvocationListener listener, String descriptor)
- throws DeviceNotAvailableException {
- if (mCollectYaffsLogs && mTestDevice.doesFileExist("/proc/yaffs")) {
- logFile(
- "/proc/yaffs",
- String.format("%s-yaffs-%s", testInfo.mTestName, descriptor),
- mTestDevice,
- listener);
- }
- }
-
- private void logFile(
- String remoteFileName,
- String localFileName,
- ITestDevice testDevice,
- ITestInvocationListener listener)
- throws DeviceNotAvailableException {
- File outputFile = null;
- InputStreamSource outputSource = null;
- try {
- outputFile = testDevice.pullFile(remoteFileName);
- if (outputFile != null) {
- CLog.d("Sending %d byte file %s to logosphere!", outputFile.length(), outputFile);
- outputSource = new FileInputStreamSource(outputFile);
- listener.testLog(localFileName, LogDataType.TEXT, outputSource);
- }
- } finally {
- FileUtil.deleteFile(outputFile);
- StreamUtil.cancel(outputSource);
- }
- }
-
- /** {@inheritDoc} */
- @Override
- public void run(ITestInvocationListener listener) throws DeviceNotAvailableException {
- Assert.assertNotNull(mTestDevice);
-
- mFioDir = new File(mTestDevice.getMountPoint(IDevice.MNT_DATA), "fio").getAbsolutePath();
- if (mFioLocation != null) {
- mFioBin = new File(mFioDir, "fio").getAbsolutePath();
- } else {
- mFioBin = "fio";
- }
- mFioConfig = new File(mFioDir, "config.fio").getAbsolutePath();
-
- setupTests();
- setupDevice();
-
- for (TestInfo test : mTestCases) {
- runTest(test, listener);
- }
-
- cleanupDevice();
- }
-
- /** {@inheritDoc} */
- @Override
- public void setDevice(ITestDevice device) {
- mTestDevice = device;
- }
-
- /** {@inheritDoc} */
- @Override
- public ITestDevice getDevice() {
- return mTestDevice;
- }
-
- /** A meta-test to ensure that the bits of FioBenchmarkTest are working properly. */
- public static class MetaTest extends TestCase {
-
- /** Test that {@link JobInfo#createJob()} properly formats a job. */
- public void testCreateJob() {
- JobInfo j = new JobInfo();
- assertEquals("", j.createJob());
- j.mJobName = "job";
- assertEquals("[job]\n", j.createJob());
- j.mParameters.put("param1", null);
- j.mParameters.put("param2", "value");
- String[] lines = j.createJob().split("\n");
- assertEquals(3, lines.length);
- assertEquals("[job]", lines[0]);
- Set<String> params = new HashSet<>(2);
- params.add(lines[1]);
- params.add(lines[2]);
- assertTrue(params.contains("param1"));
- assertTrue(params.contains("param2=value"));
- }
-
- /** Test that {@link TestInfo#createConfig()} properly formats a config. */
- public void testCreateConfig() {
- TestInfo t = new TestInfo();
- JobInfo j = new JobInfo();
- j.mJobName = "job1";
- j.mParameters.put("param1", "value1");
- t.mJobs.add(j);
-
- j = new JobInfo();
- j.mJobName = "job2";
- j.mParameters.put("param2", "value2");
- t.mJobs.add(j);
-
- j = new JobInfo();
- j.mJobName = "job3";
- j.mParameters.put("param3", "value3");
- t.mJobs.add(j);
-
- assertEquals(
- "[job1]\nparam1=value1\n\n"
- + "[job2]\nparam2=value2\n\n"
- + "[job3]\nparam3=value3\n\n",
- t.createConfig());
- }
-
- /**
- * Test that output lines are parsed correctly by the FioParser, invalid lines are ignored,
- * and that the various fields are accessible with {@link FioParser#getResult(String,
- * String)}.
- */
- public void testFioParser() {
- String[] lines = new String[4];
- // We build the lines up as follows (assuming FIO_RESULTS_FIELDS.length == 58):
- // 0;3;6;...;171
- // 1;4;7;...;172
- // 2;5;8;...;173
- for (int i = 0; i < 3; i++) {
- StringBuilder sb = new StringBuilder();
- sb.append(i);
- for (int j = 1; j < FIO_V0_RESULT_FIELDS.length; j++) {
- sb.append(";");
- sb.append(j * 3 + i);
- }
- lines[i] = sb.toString();
- }
- // A line may have an optional description on the end which we don't care about, make
- // sure it still parses.
- lines[2] = lines[2] += ";description";
- // Make sure an invalid output line does not parse.
- lines[3] = "invalid";
-
- FioParser p = new FioParser();
- p.processNewLines(lines);
-
- for (int i = 0; i < 3; i++) {
- for (int j = 0; j < FIO_V0_RESULT_FIELDS.length; j++) {
- assertEquals(
- String.format("job=%d, field=%s", i, FIO_V0_RESULT_FIELDS[j]),
- String.format("%d", j * 3 + i),
- p.getResult(String.format("%d", i), FIO_V0_RESULT_FIELDS[j]));
- }
- }
- assertNull(p.getResult("missing", "jobname"));
- assertNull(p.getResult("invalid", "jobname"));
- assertNull(p.getResult("0", "missing"));
- }
-
- /**
- * Test that {@link PerfMetricInfo.ResultType#value(String)} correctly transforms strings
- * based on the result type.
- */
- public void testResultTypeValue() {
- assertEquals("test", PerfMetricInfo.ResultType.STRING.value("test"));
- assertEquals("1", PerfMetricInfo.ResultType.INT.value("1"));
- assertEquals("3.14159", PerfMetricInfo.ResultType.FLOAT.value("3.14159"));
- assertEquals(
- String.format("%f", 0.34567),
- PerfMetricInfo.ResultType.PERCENT.value("34.567%"));
- assertNull(PerfMetricInfo.ResultType.PERCENT.value(""));
- assertNull(PerfMetricInfo.ResultType.PERCENT.value("34.567"));
- assertNull(PerfMetricInfo.ResultType.PERCENT.value("test%"));
- }
- }
-}
diff --git a/src/com/android/performance/tests/GLBenchmarkTest.java b/src/com/android/performance/tests/GLBenchmarkTest.java
deleted file mode 100644
index a501de1..0000000
--- a/src/com/android/performance/tests/GLBenchmarkTest.java
+++ /dev/null
@@ -1,290 +0,0 @@
-/*
- * Copyright (C) 2013 The Android Open Source Project
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package com.android.performance.tests;
-
-import com.android.tradefed.config.Option;
-import com.android.tradefed.config.Option.Importance;
-import com.android.tradefed.device.DeviceNotAvailableException;
-import com.android.tradefed.device.ITestDevice;
-import com.android.tradefed.log.LogUtil.CLog;
-import com.android.tradefed.result.ITestInvocationListener;
-import com.android.tradefed.result.TestDescription;
-import com.android.tradefed.testtype.IDeviceTest;
-import com.android.tradefed.testtype.IRemoteTest;
-import com.android.tradefed.util.RunUtil;
-import com.android.tradefed.util.proto.TfMetricProtoUtil;
-
-import org.w3c.dom.Document;
-import org.w3c.dom.Element;
-import org.w3c.dom.Node;
-import org.w3c.dom.NodeList;
-import org.xml.sax.SAXException;
-
-import java.io.File;
-import java.io.IOException;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.Map;
-
-import javax.xml.parsers.DocumentBuilder;
-import javax.xml.parsers.DocumentBuilderFactory;
-import javax.xml.parsers.ParserConfigurationException;
-
-/**
- * A harness that launches GLBenchmark and reports result. Requires GLBenchmark custom XML. Assumes
- * GLBenchmark app is already installed on device.
- */
-public class GLBenchmarkTest implements IDeviceTest, IRemoteTest {
-
- private static final String RUN_KEY = "glbenchmark";
- private static final long TIMEOUT_MS = 30 * 60 * 1000;
- private static final long POLLING_INTERVAL_MS = 5 * 1000;
- private static final Map<String, String> METRICS_KEY_MAP = createMetricsKeyMap();
-
- private ITestDevice mDevice;
-
- @Option(
- name = "custom-xml-path",
- description = "local path for GLBenchmark custom.xml",
- importance = Importance.ALWAYS)
- private File mGlbenchmarkCustomXmlLocal = new File("/tmp/glbenchmark_custom.xml");
-
- @Option(name = "gl-package-name", description = "GLBenchmark package name")
- private String mGlbenchmarkPackageName = "com.glbenchmark.glbenchmark25";
-
- @Option(name = "gl-version", description = "GLBenchmark version (e.g. 2.5.1_b306a5)")
- private String mGlbenchmarkVersion = "2.5.1_b306a5";
-
- private String mGlbenchmarkCacheDir =
- "${EXTERNAL_STORAGE}/Android/data/" + mGlbenchmarkPackageName + "/cache/";
- private String mGlbenchmarkCustomXmlPath = mGlbenchmarkCacheDir + "custom.xml";
- private String mGlbenchmarkResultXmlPath =
- mGlbenchmarkCacheDir + "last_results_" + mGlbenchmarkVersion + ".xml";
- private String mGlbenchmarkExcelResultXmlPath = mGlbenchmarkCacheDir + "results_%s_0.xml";
- private String mGlbenchmarkAllResultXmlPath = mGlbenchmarkCacheDir + "results*.xml";
-
- private static Map<String, String> createMetricsKeyMap() {
- Map<String, String> result = new HashMap<>();
- result.put("Fill rate - C24Z16", "fill-rate");
- result.put("Fill rate - C24Z16 Offscreen", "fill-rate-offscreen");
- result.put("Triangle throughput: Textured - C24Z16", "triangle-c24z16");
- result.put("Triangle throughput: Textured - C24Z16 Offscreen", "triangle-c24z16-offscreen");
- result.put(
- "Triangle throughput: Textured - C24Z16 Vertex lit", "triangle-c24z16-vertex-lit");
- result.put(
- "Triangle throughput: Textured - C24Z16 Offscreen Vertex lit",
- "triangle-c24z16-offscreen-vertex-lit");
- result.put(
- "Triangle throughput: Textured - C24Z16 Fragment lit",
- "triangle-c24z16-fragment-lit");
- result.put(
- "Triangle throughput: Textured - C24Z16 Offscreen Fragment lit",
- "triangle-c24z16-offscreen-fragment-lit");
- result.put("GLBenchmark 2.5 Egypt HD - C24Z16", "egypt-hd-c24z16");
- result.put("GLBenchmark 2.5 Egypt HD - C24Z16 Offscreen", "egypt-hd-c24z16-offscreen");
- result.put("GLBenchmark 2.5 Egypt HD PVRTC4 - C24Z16", "egypt-hd-pvrtc4-c24z16");
- result.put(
- "GLBenchmark 2.5 Egypt HD PVRTC4 - C24Z16 Offscreen",
- "egypt-hd-pvrtc4-c24z16-offscreen");
- result.put("GLBenchmark 2.5 Egypt HD - C24Z24MS4", "egypt-hd-c24z24ms4");
- result.put(
- "GLBenchmark 2.5 Egypt HD - C24Z16 Fixed timestep",
- "egypt-hd-c24z16-fixed-timestep");
- result.put(
- "GLBenchmark 2.5 Egypt HD - C24Z16 Fixed timestep Offscreen",
- "egypt-hd-c24z16-fixed-timestep-offscreen");
- result.put("GLBenchmark 2.1 Egypt Classic - C16Z16", "egypt-classic-c16z16");
- result.put(
- "GLBenchmark 2.1 Egypt Classic - C16Z16 Offscreen",
- "egypt-classic-c16z16-offscreen");
- return Collections.unmodifiableMap(result);
- }
-
- /** {@inheritDoc} */
- @Override
- public void setDevice(ITestDevice device) {
- mDevice = device;
- }
-
- /** {@inheritDoc} */
- @Override
- public ITestDevice getDevice() {
- return mDevice;
- }
-
- /** {@inheritDoc} */
- @Override
- public void run(ITestInvocationListener listener) throws DeviceNotAvailableException {
- TestDescription testId = new TestDescription(getClass().getCanonicalName(), RUN_KEY);
- ITestDevice device = getDevice();
-
- // delete old result
- device.executeShellCommand(String.format("rm %s", mGlbenchmarkResultXmlPath));
- device.executeShellCommand(String.format("rm %s", mGlbenchmarkAllResultXmlPath));
-
- // push glbenchmark custom xml to device
- device.pushFile(mGlbenchmarkCustomXmlLocal, mGlbenchmarkCustomXmlPath);
-
- listener.testRunStarted(RUN_KEY, 0);
- listener.testStarted(testId);
-
- long testStartTime = System.currentTimeMillis();
- boolean isRunningBenchmark;
-
- boolean isTimedOut = false;
- boolean isResultGenerated = false;
- Map<String, String> metrics = new HashMap<>();
- String errMsg = null;
-
- String deviceModel = device.executeShellCommand("getprop ro.product.model");
- String resultExcelXmlPath =
- String.format(
- mGlbenchmarkExcelResultXmlPath,
- deviceModel.trim().replaceAll("[ -]", "_").toLowerCase());
- CLog.i("Result excel xml path:" + resultExcelXmlPath);
-
- // start glbenchmark and wait for test to complete
- isTimedOut = false;
- long benchmarkStartTime = System.currentTimeMillis();
-
- device.executeShellCommand(
- "am start -a android.intent.action.MAIN "
- + "-n com.glbenchmark.glbenchmark25/com.glbenchmark.activities.MainActivity "
- + "--ez custom true");
- isRunningBenchmark = true;
- while (isRunningBenchmark && !isResultGenerated && !isTimedOut) {
- RunUtil.getDefault().sleep(POLLING_INTERVAL_MS);
- isTimedOut = (System.currentTimeMillis() - benchmarkStartTime >= TIMEOUT_MS);
- isResultGenerated = device.doesFileExist(resultExcelXmlPath);
- isRunningBenchmark = device.executeShellCommand("ps").contains("glbenchmark");
- }
-
- if (isTimedOut) {
- errMsg = "GLBenchmark timed out.";
- } else {
- // pull result from device
- File benchmarkReport = device.pullFile(mGlbenchmarkResultXmlPath);
- if (benchmarkReport != null) {
- // parse result
- CLog.i("== GLBenchmark result ==");
- Map<String, String> benchmarkResult = parseResultXml(benchmarkReport);
- if (benchmarkResult == null) {
- errMsg = "Failed to parse GLBenchmark result XML.";
- } else {
- metrics = benchmarkResult;
- }
- // delete results from device and host
- device.executeShellCommand(String.format("rm %s", mGlbenchmarkResultXmlPath));
- device.executeShellCommand(String.format("rm %s", resultExcelXmlPath));
- benchmarkReport.delete();
- } else {
- errMsg = "GLBenchmark report not found.";
- }
- }
- if (errMsg != null) {
- CLog.e(errMsg);
- listener.testFailed(testId, errMsg);
- listener.testEnded(testId, TfMetricProtoUtil.upgradeConvert(metrics));
- listener.testRunFailed(errMsg);
- } else {
- long durationMs = System.currentTimeMillis() - testStartTime;
- listener.testEnded(testId, TfMetricProtoUtil.upgradeConvert(metrics));
- listener.testRunEnded(durationMs, TfMetricProtoUtil.upgradeConvert(metrics));
- }
- }
-
- /**
- * Parse GLBenchmark result XML.
- *
- * @param resultXml GLBenchmark result XML {@link File}
- * @return a {@link HashMap} that contains metrics key and result
- */
- private Map<String, String> parseResultXml(File resultXml) {
- Map<String, String> benchmarkResult = new HashMap<>();
- DocumentBuilderFactory dbFactory = DocumentBuilderFactory.newInstance();
- Document doc = null;
- try {
- DocumentBuilder dBuilder = dbFactory.newDocumentBuilder();
- doc = dBuilder.parse(resultXml);
- } catch (ParserConfigurationException e) {
- return null;
- } catch (IOException e) {
- return null;
- } catch (SAXException e) {
- return null;
- } catch (IllegalArgumentException e) {
- return null;
- }
- doc.getDocumentElement().normalize();
-
- NodeList nodes = doc.getElementsByTagName("test_result");
- for (int i = 0; i < nodes.getLength(); i++) {
- Node node = nodes.item(i);
- if (node.getNodeType() == Node.ELEMENT_NODE) {
- Element testResult = (Element) node;
- String testTitle = getData(testResult, "title");
- String testType = getData(testResult, "type");
- String fps = getData(testResult, "fps");
- String score = getData(testResult, "score");
- String testName = String.format("%s - %s", testTitle, testType);
- if (METRICS_KEY_MAP.containsKey(testName)) {
- if (testName.contains("Fill") || testName.contains("Triangle")) {
- // Use Mtexels/sec or MTriangles/sec as unit
- score = String.valueOf((long) (Double.parseDouble(score) / 1.0E6));
- }
- CLog.i(String.format("%s: %s (fps=%s)", testName, score, fps));
- String testKey = METRICS_KEY_MAP.get(testName);
- if (score != null && !score.trim().equals("0")) {
- benchmarkResult.put(testKey, score);
- if (fps != null && !fps.trim().equals("0.0")) {
- try {
- float fpsValue = Float.parseFloat(fps.replace("fps", ""));
- benchmarkResult.put(testKey + "-fps", String.valueOf(fpsValue));
- } catch (NumberFormatException e) {
- CLog.i(String.format("Got %s for fps value. Ignored.", fps));
- }
- }
- }
- }
- }
- }
- return benchmarkResult;
- }
-
- /**
- * Get value in the first matching tag under the element
- *
- * @param element the parent {@link Element} of the tag
- * @param tag {@link String} of the tag name
- * @return a {@link String} that contains the value in the tag; returns null if not found.
- */
- private String getData(Element element, String tag) {
- NodeList tagNodes = element.getElementsByTagName(tag);
- if (tagNodes.getLength() > 0) {
- Node tagNode = tagNodes.item(0);
- if (tagNode.getNodeType() == Node.ELEMENT_NODE) {
- Node node = tagNode.getChildNodes().item(0);
- if (node != null) {
- return node.getNodeValue();
- } else {
- return null;
- }
- }
- }
- return null;
- }
-}
diff --git a/src/com/android/performance/tests/GeekbenchTest.java b/src/com/android/performance/tests/GeekbenchTest.java
deleted file mode 100644
index 0e69ab6..0000000
--- a/src/com/android/performance/tests/GeekbenchTest.java
+++ /dev/null
@@ -1,202 +0,0 @@
-/*
- * Copyright (C) 2013 The Android Open Source Project
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package com.android.performance.tests;
-
-import com.android.ddmlib.CollectingOutputReceiver;
-import com.android.tradefed.device.DeviceNotAvailableException;
-import com.android.tradefed.device.ITestDevice;
-import com.android.tradefed.log.LogUtil.CLog;
-import com.android.tradefed.result.ITestInvocationListener;
-import com.android.tradefed.result.TestDescription;
-import com.android.tradefed.testtype.IDeviceTest;
-import com.android.tradefed.testtype.IRemoteTest;
-import com.android.tradefed.util.proto.TfMetricProtoUtil;
-
-import org.json.JSONArray;
-import org.json.JSONException;
-import org.json.JSONObject;
-import org.json.JSONTokener;
-import org.junit.Assert;
-
-import java.io.BufferedReader;
-import java.io.File;
-import java.io.FileReader;
-import java.io.IOException;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.Map;
-import java.util.concurrent.TimeUnit;
-
-/**
- * A harness that launches Geekbench and reports result. Requires Geekbench binary and plar file in
- * device temporary directory.
- */
-public class GeekbenchTest implements IDeviceTest, IRemoteTest {
-
- private static final String RUN_KEY = "geekbench3";
- private static final int MAX_ATTEMPTS = 3;
- private static final int TIMEOUT_MS = 10 * 60 * 1000;
-
- private static final String DEVICE_TEMPORARY_DIR_PATH = "/data/local/tmp/";
- private static final String GEEKBENCH_BINARY_FILENAME = "geekbench_armeabi-v7a_32";
- private static final String GEEKBENCH_BINARY_DEVICE_PATH =
- DEVICE_TEMPORARY_DIR_PATH + GEEKBENCH_BINARY_FILENAME;
- private static final String GEEKBENCH_PLAR_DEVICE_PATH =
- DEVICE_TEMPORARY_DIR_PATH + "geekbench.plar";
- private static final String GEEKBENCH_RESULT_DEVICE_PATH =
- DEVICE_TEMPORARY_DIR_PATH + "result.txt";
-
- private static final String OVERALL_SCORE_NAME = "Overall Score";
- private static final Map<String, String> METRICS_KEY_MAP = createMetricsKeyMap();
-
- private ITestDevice mDevice;
-
- private static Map<String, String> createMetricsKeyMap() {
- Map<String, String> result = new HashMap<>();
- result.put(OVERALL_SCORE_NAME, "overall");
- result.put("Integer", "integer");
- result.put("Floating Point", "floating-point");
- result.put("Memory", "memory");
- return Collections.unmodifiableMap(result);
- }
-
- /** {@inheritDoc} */
- @Override
- public void setDevice(ITestDevice device) {
- mDevice = device;
- }
-
- /** {@inheritDoc} */
- @Override
- public ITestDevice getDevice() {
- return mDevice;
- }
-
- /** {@inheritDoc} */
- @Override
- public void run(ITestInvocationListener listener) throws DeviceNotAvailableException {
- TestDescription testId = new TestDescription(getClass().getCanonicalName(), RUN_KEY);
- ITestDevice device = getDevice();
-
- // delete old results
- device.executeShellCommand(String.format("rm %s", GEEKBENCH_RESULT_DEVICE_PATH));
-
- Assert.assertTrue(
- String.format(
- "Geekbench binary not found on device: %s", GEEKBENCH_BINARY_DEVICE_PATH),
- device.doesFileExist(GEEKBENCH_BINARY_DEVICE_PATH));
- Assert.assertTrue(
- String.format(
- "Geekbench binary not found on device: %s", GEEKBENCH_PLAR_DEVICE_PATH),
- device.doesFileExist(GEEKBENCH_PLAR_DEVICE_PATH));
-
- listener.testRunStarted(RUN_KEY, 0);
- listener.testStarted(testId);
-
- long testStartTime = System.currentTimeMillis();
- Map<String, String> metrics = new HashMap<>();
- String errMsg = null;
-
- // start geekbench and wait for test to complete
- CollectingOutputReceiver receiver = new CollectingOutputReceiver();
- device.executeShellCommand(
- String.format(
- "%s --no-upload --export-json %s",
- GEEKBENCH_BINARY_DEVICE_PATH, GEEKBENCH_RESULT_DEVICE_PATH),
- receiver,
- TIMEOUT_MS,
- TimeUnit.MILLISECONDS,
- MAX_ATTEMPTS);
- CLog.v(receiver.getOutput());
-
- // pull result from device
- File benchmarkReport = device.pullFile(GEEKBENCH_RESULT_DEVICE_PATH);
- if (benchmarkReport != null) {
- // parse result
- CLog.i("== Geekbench 3 result ==");
- Map<String, String> benchmarkResult = parseResultJSON(benchmarkReport);
- if (benchmarkResult == null) {
- errMsg = "Failed to parse Geekbench result JSON.";
- } else {
- metrics = benchmarkResult;
- }
-
- // delete results from device and host
- benchmarkReport.delete();
- device.executeShellCommand("rm " + GEEKBENCH_RESULT_DEVICE_PATH);
- } else {
- errMsg = "Geekbench report not found.";
- }
-
- if (errMsg != null) {
- CLog.e(errMsg);
- listener.testFailed(testId, errMsg);
- listener.testEnded(testId, TfMetricProtoUtil.upgradeConvert(metrics));
- listener.testRunFailed(errMsg);
- } else {
- long durationMs = System.currentTimeMillis() - testStartTime;
- listener.testEnded(testId, TfMetricProtoUtil.upgradeConvert(metrics));
- listener.testRunEnded(durationMs, TfMetricProtoUtil.upgradeConvert(metrics));
- }
- }
-
- private Map<String, String> parseResultJSON(File resultJson) {
- Map<String, String> benchmarkResult = new HashMap<>();
- try {
- BufferedReader bufferedReader = new BufferedReader(new FileReader(resultJson));
- String line = bufferedReader.readLine();
- bufferedReader.close();
- if (line != null) {
- JSONTokener tokener = new JSONTokener(line);
- JSONObject root = new JSONObject(tokener);
- String overallScore = root.getString("score");
- benchmarkResult.put(METRICS_KEY_MAP.get(OVERALL_SCORE_NAME), overallScore);
- CLog.i(String.format("Overall: %s", overallScore));
- String overallScoreMulticore = root.getString("multicore_score");
- benchmarkResult.put(
- METRICS_KEY_MAP.get(OVERALL_SCORE_NAME) + "-multi", overallScoreMulticore);
- CLog.i(String.format("Overall-multicore: %s", overallScore));
- JSONArray arr = root.getJSONArray("sections");
- for (int i = 0; i < arr.length(); i++) {
- JSONObject section = arr.getJSONObject(i);
- String sectionName = section.getString("name");
- String sectionScore = section.getString("score");
- String sectionScoreMulticore = section.getString("multicore_score");
- if (METRICS_KEY_MAP.containsKey(sectionName)) {
- CLog.i(String.format("%s: %s", sectionName, sectionScore));
- CLog.i(
- String.format(
- "%s-multicore: %s", sectionName, sectionScoreMulticore));
- benchmarkResult.put(METRICS_KEY_MAP.get(sectionName), sectionScore);
- CLog.i(
- String.format(
- "%s-multicore: %s", sectionName, sectionScoreMulticore));
- benchmarkResult.put(
- METRICS_KEY_MAP.get(sectionName) + "-multi", sectionScoreMulticore);
- }
- }
- }
- } catch (IOException e) {
- CLog.e("Geekbench3 result file not found.");
- return null;
- } catch (JSONException e) {
- CLog.e("Error parsing Geekbench3 JSON result.");
- return null;
- }
- return benchmarkResult;
- }
-}
diff --git a/src/com/android/performance/tests/StartupMetricsTest.java b/src/com/android/performance/tests/StartupMetricsTest.java
deleted file mode 100644
index 9bfc1cb..0000000
--- a/src/com/android/performance/tests/StartupMetricsTest.java
+++ /dev/null
@@ -1,274 +0,0 @@
-/*
- * Copyright (C) 2011 The Android Open Source Project
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package com.android.performance.tests;
-
-import com.android.loganalysis.item.BugreportItem;
-import com.android.loganalysis.item.MemInfoItem;
-import com.android.loganalysis.item.ProcrankItem;
-import com.android.loganalysis.parser.BugreportParser;
-import com.android.tradefed.config.Option;
-import com.android.tradefed.device.DeviceNotAvailableException;
-import com.android.tradefed.device.ITestDevice;
-import com.android.tradefed.device.ITestDevice.RecoveryMode;
-import com.android.tradefed.log.LogUtil.CLog;
-import com.android.tradefed.result.ITestInvocationListener;
-import com.android.tradefed.result.InputStreamSource;
-import com.android.tradefed.result.LogDataType;
-import com.android.tradefed.testtype.IDeviceTest;
-import com.android.tradefed.testtype.IRemoteTest;
-import com.android.tradefed.util.RunUtil;
-import com.android.tradefed.util.proto.TfMetricProtoUtil;
-
-import org.junit.Assert;
-
-import java.io.BufferedReader;
-import java.io.IOException;
-import java.io.InputStreamReader;
-import java.util.HashMap;
-import java.util.Map;
-
-/** Tests to gather device metrics from during and immediately after boot */
-public class StartupMetricsTest implements IDeviceTest, IRemoteTest {
- public static final String BUGREPORT_LOG_NAME = "bugreport_startup.txt";
-
- @Option(name = "boot-time-ms", description = "Timeout in ms to wait for device to boot.")
- private long mBootTimeMs = 20 * 60 * 1000;
-
- @Option(
- name = "boot-poll-time-ms",
- description = "Delay in ms between polls for device to boot.")
- private long mBootPoolTimeMs = 500;
-
- @Option(
- name = "post-boot-delay-ms",
- description = "Delay in ms after boot complete before taking the bugreport.")
- private long mPostBootDelayMs = 60000;
-
- @Option(
- name = "skip-memory-stats",
- description = "Report boot time only, without memory stats.")
- private boolean mSkipMemoryStats = false;
-
- ITestDevice mTestDevice = null;
-
- @Override
- public void run(ITestInvocationListener listener) throws DeviceNotAvailableException {
- Assert.assertNotNull(mTestDevice);
- executeRebootTest(listener);
- if (!mSkipMemoryStats) {
- fetchBugReportMetrics(listener);
- }
- }
-
- /**
- * Check how long the device takes to come online and become available after a reboot.
- *
- * @param listener the {@link ITestInvocationListener} of test results
- */
- void executeRebootTest(ITestInvocationListener listener) throws DeviceNotAvailableException {
-
- /*Not using /proc/uptime to get the initial boot time
- because it includes delays running 'permission utils',
- and the CPU throttling waiter*/
- Map<String, String> runMetrics = new HashMap<>();
- // Initial reboot
- mTestDevice.rebootIntoBootloader();
- mTestDevice.setUseFastbootErase(false);
- mTestDevice.fastbootWipePartition("userdata");
- mTestDevice.executeFastbootCommand("reboot");
- mTestDevice.waitForDeviceOnline(10 * 60 * 1000);
- long initOnlineTime = System.currentTimeMillis();
- Assert.assertTrue(waitForBootComplete(mTestDevice, mBootTimeMs, mBootPoolTimeMs));
- long initAvailableTime = System.currentTimeMillis();
- double initUnavailDuration = initAvailableTime - initOnlineTime;
- runMetrics.put("init-boot", Double.toString(initUnavailDuration / 1000.0));
-
- mTestDevice.setRecoveryMode(RecoveryMode.NONE);
- CLog.d("Reboot test start.");
- mTestDevice.nonBlockingReboot();
- long startTime = System.currentTimeMillis();
- mTestDevice.waitForDeviceOnline();
- long onlineTime = System.currentTimeMillis();
- Assert.assertTrue(waitForBootComplete(mTestDevice, mBootTimeMs, mBootPoolTimeMs));
- long availableTime = System.currentTimeMillis();
-
- double offlineDuration = onlineTime - startTime;
- double unavailDuration = availableTime - startTime;
- CLog.d(
- "Reboot: %f millis until online, %f until available",
- offlineDuration, unavailDuration);
- runMetrics.put("online", Double.toString(offlineDuration / 1000.0));
- runMetrics.put("bootcomplete", Double.toString(unavailDuration / 1000.0));
-
- reportMetrics(listener, "boottime", runMetrics);
- }
-
- /**
- * Fetch proc rank metrics from the bugreport after reboot.
- *
- * @param listener the {@link ITestInvocationListener} of test results
- * @throws DeviceNotAvailableException
- */
- void fetchBugReportMetrics(ITestInvocationListener listener)
- throws DeviceNotAvailableException {
- // Make sure the device is available and settled, before getting bugreport.
- mTestDevice.waitForDeviceAvailable();
- RunUtil.getDefault().sleep(mPostBootDelayMs);
- BugreportParser parser = new BugreportParser();
- BugreportItem bugreport = null;
- // Retrieve bugreport
- try (InputStreamSource bugSource = mTestDevice.getBugreport()) {
- listener.testLog(BUGREPORT_LOG_NAME, LogDataType.BUGREPORT, bugSource);
- bugreport =
- parser.parse(
- new BufferedReader(
- new InputStreamReader(bugSource.createInputStream())));
- } catch (IOException e) {
- Assert.fail(
- String.format(
- "Failed to fetch and parse bugreport for device %s: %s",
- mTestDevice.getSerialNumber(), e));
- }
-
- if (bugreport != null) {
- // Process meminfo information and post it to the dashboard
- MemInfoItem item = bugreport.getMemInfo();
- if (item != null) {
- Map<String, String> memInfoMap = convertMap(item);
- reportMetrics(listener, "startup-meminfo", memInfoMap);
- }
-
- // Process procrank information and post it to the dashboard
- if (bugreport.getProcrank() != null) {
- parseProcRankMap(listener, bugreport.getProcrank());
- }
- }
- }
-
- /**
- * Helper method to convert a {@link MemInfoItem} to Map<String, String>.
- *
- * @param input the {@link Map} to convert from
- * @return output the converted {@link Map}
- */
- Map<String, String> convertMap(MemInfoItem item) {
- Map<String, String> output = new HashMap<>();
- for (String key : item.keySet()) {
- output.put(key, item.get(key).toString());
- }
- return output;
- }
-
- /**
- * Report run metrics by creating an empty test run to stick them in
- *
- * @param listener the {@link ITestInvocationListener} of test results
- * @param runName the test name
- * @param metrics the {@link Map} that contains metrics for the given test
- */
- void reportMetrics(
- ITestInvocationListener listener, String runName, Map<String, String> metrics) {
- // Create an empty testRun to report the parsed runMetrics
- CLog.d("About to report metrics: %s", metrics);
- listener.testRunStarted(runName, 0);
- listener.testRunEnded(0, TfMetricProtoUtil.upgradeConvert(metrics));
- }
-
- /**
- * Aggregates the procrank data by the pss, rss, and uss values.
- *
- * @param listener the {@link ITestInvocationListener} of test results
- * @param procrank the {@link Map} parsed from brillopad for the procrank section
- */
- void parseProcRankMap(ITestInvocationListener listener, ProcrankItem procrank) {
- // final maps for pss, rss, and uss.
- Map<String, String> pssOutput = new HashMap<>();
- Map<String, String> rssOutput = new HashMap<>();
- Map<String, String> ussOutput = new HashMap<>();
- // total number of processes.
- Integer numProcess = 0;
- // aggregate pss, rss, uss across all processes.
- Integer pssTotal = 0;
- Integer rssTotal = 0;
- Integer ussTotal = 0;
-
- for (Integer pid : procrank.getPids()) {
- numProcess++;
- Integer pss = procrank.getPss(pid);
- Integer rss = procrank.getRss(pid);
- Integer uss = procrank.getUss(pid);
- if (pss != null) {
- pssTotal += pss;
- pssOutput.put(procrank.getProcessName(pid), pss.toString());
- }
- if (rss != null) {
- rssTotal += rss;
- rssOutput.put(procrank.getProcessName(pid), rss.toString());
- }
- if (uss != null) {
- ussTotal += pss;
- ussOutput.put(procrank.getProcessName(pid), uss.toString());
- }
- }
- // Add aggregation data.
- pssOutput.put("count", numProcess.toString());
- pssOutput.put("total", pssTotal.toString());
- rssOutput.put("count", numProcess.toString());
- rssOutput.put("total", rssTotal.toString());
- ussOutput.put("count", numProcess.toString());
- ussOutput.put("total", ussTotal.toString());
-
- // Report metrics to dashboard
- reportMetrics(listener, "startup-procrank-pss", pssOutput);
- reportMetrics(listener, "startup-procrank-rss", rssOutput);
- reportMetrics(listener, "startup-procrank-uss", ussOutput);
- }
-
- /**
- * Blocks until the device's boot complete flag is set.
- *
- * @param device the {@link ITestDevice}
- * @param timeOut time in msecs to wait for the flag to be set
- * @param pollDelay time in msecs between checks
- * @return true if device's boot complete flag is set within the timeout
- * @throws DeviceNotAvailableException
- */
- private boolean waitForBootComplete(ITestDevice device, long timeOut, long pollDelay)
- throws DeviceNotAvailableException {
- long startTime = System.currentTimeMillis();
- while ((System.currentTimeMillis() - startTime) < timeOut) {
- String output = device.executeShellCommand("getprop dev.bootcomplete");
- output = output.replace('#', ' ').trim();
- if (output.equals("1")) {
- return true;
- }
- RunUtil.getDefault().sleep(pollDelay);
- }
- CLog.w("Device %s did not boot after %d ms", device.getSerialNumber(), timeOut);
- return false;
- }
-
- @Override
- public void setDevice(ITestDevice device) {
- mTestDevice = device;
- }
-
- @Override
- public ITestDevice getDevice() {
- return mTestDevice;
- }
-}
diff --git a/src/com/android/performance/tests/VellamoBenchmark.java b/src/com/android/performance/tests/VellamoBenchmark.java
deleted file mode 100644
index 7c4f4d2..0000000
--- a/src/com/android/performance/tests/VellamoBenchmark.java
+++ /dev/null
@@ -1,158 +0,0 @@
-/*
- * Copyright (C) 2013 The Android Open Source Project
- *
- * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except
- * in compliance with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software distributed under the License
- * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
- * or implied. See the License for the specific language governing permissions and limitations under
- * the License.
- */
-
-package com.android.performance.tests;
-
-import com.android.tradefed.device.DeviceNotAvailableException;
-import com.android.tradefed.device.ITestDevice;
-import com.android.tradefed.log.LogUtil.CLog;
-import com.android.tradefed.metrics.proto.MetricMeasurement.Metric;
-import com.android.tradefed.result.ITestInvocationListener;
-import com.android.tradefed.result.InputStreamSource;
-import com.android.tradefed.result.TestDescription;
-import com.android.tradefed.testtype.IDeviceTest;
-import com.android.tradefed.testtype.IRemoteTest;
-import com.android.tradefed.util.RunUtil;
-import com.android.tradefed.util.proto.TfMetricProtoUtil;
-
-import java.io.BufferedReader;
-import java.io.IOException;
-import java.io.InputStreamReader;
-import java.util.HashMap;
-import java.util.Map;
-
-/** A harness that launches VellamoBenchmark and reports result. Requires VellamoBenchmark apk. */
-public class VellamoBenchmark implements IDeviceTest, IRemoteTest {
-
- private static final String LOGTAG = "VAUTOMATIC";
- private static final String RUN_KEY = "vellamobenchmark-3202";
- private static final String PACKAGE_NAME = "com.quicinc.vellamo";
- private static final long TIMEOUT_MS = 30 * 60 * 1000;
- private static final long POLLING_INTERVAL_MS = 10 * 1000;
- private static final int INDEX_NAME = 0;
- private static final int INDEX_CODE = 4;
- private static final int INDEX_SCORE = 5;
-
- private ITestDevice mDevice;
-
- /** {@inheritDoc} */
- @Override
- public void setDevice(ITestDevice device) {
- mDevice = device;
- }
-
- /** {@inheritDoc} */
- @Override
- public ITestDevice getDevice() {
- return mDevice;
- }
-
- /** {@inheritDoc} */
- @Override
- public void run(ITestInvocationListener listener) throws DeviceNotAvailableException {
- TestDescription testId = new TestDescription(getClass().getCanonicalName(), RUN_KEY);
- ITestDevice device = getDevice();
- listener.testRunStarted(RUN_KEY, 0);
- listener.testStarted(testId);
-
- long testStartTime = System.currentTimeMillis();
- Map<String, String> metrics = new HashMap<>();
- String errMsg = null;
-
- boolean isTimedOut = false;
- boolean isResultGenerated = false;
- boolean hasScore = false;
- double sumScore = 0;
- int errorCode = 0;
- device.clearErrorDialogs();
- isTimedOut = false;
-
- long benchmarkStartTime = System.currentTimeMillis();
- // start the vellamo benchmark app and run all the tests
- // the documentation and binary for the Vellamo 3.2.2 for Automation APK
- // can be found here:
- // https://b.corp.google.com/issue?id=23107318
- CLog.i("Starting Vellamo Benchmark");
- device.executeShellCommand(
- "am start -a com.quicinc.vellamo.AUTOMATIC"
- + " -e w com.quicinc.skunkworks.wvb" // use System WebView
- + " -n com.quicinc.vellamo/.main.MainActivity");
- String line;
- while (!isResultGenerated && !isTimedOut) {
- RunUtil.getDefault().sleep(POLLING_INTERVAL_MS);
- isTimedOut = (System.currentTimeMillis() - benchmarkStartTime >= TIMEOUT_MS);
-
- // get the logcat and parse
- try (InputStreamSource logcatSource = device.getLogcat();
- BufferedReader logcat =
- new BufferedReader(
- new InputStreamReader(logcatSource.createInputStream()))) {
- while ((line = logcat.readLine()) != null) {
- // filter only output from the Vellamo process
- if (!line.contains(LOGTAG)) {
- continue;
- }
- line = line.substring(line.indexOf(LOGTAG) + LOGTAG.length());
- // we need to see if the score is generated since there are some
- // cases the result with </automatic> tag is generated but no score is included
- if (line.contains("</automatic>")) {
- if (hasScore) {
- isResultGenerated = true;
- break;
- }
- }
- // get the score out
- if (line.contains(" b: ")) {
- hasScore = true;
- String[] results = line.split(" b: ")[1].split(",");
- errorCode = Integer.parseInt(results[INDEX_CODE]);
- if (errorCode != 0) {
- CLog.w(
- "Non-zero error code: %d from becnhmark '%s'",
- errorCode, results[INDEX_NAME]);
- } else {
- sumScore += Double.parseDouble(results[INDEX_SCORE]);
- }
- metrics.put(results[INDEX_NAME], results[INDEX_SCORE]);
- CLog.i("%s :: %s", results[INDEX_NAME], results[INDEX_SCORE]);
- }
- }
- } catch (IOException e) {
- CLog.e(e);
- }
-
- if (null == device.getProcessByName(PACKAGE_NAME)) {
- break;
- }
- }
-
- if (isTimedOut) {
- errMsg = "Vellamo Benchmark timed out.";
- } else {
- CLog.i("Done running Vellamo Benchmark");
- }
- if (!hasScore) {
- errMsg = "Test ended but no scores can be found.";
- }
- if (errMsg != null) {
- CLog.e(errMsg);
- listener.testFailed(testId, errMsg);
- }
- long durationMs = System.currentTimeMillis() - testStartTime;
- metrics.put("total", Double.toString(sumScore));
- CLog.i("total :: %f", sumScore);
- listener.testEnded(testId, new HashMap<String, Metric>());
- listener.testRunEnded(durationMs, TfMetricProtoUtil.upgradeConvert(metrics));
- }
-}
