src/com/android/nn/benchmark/app/BenchmarkTestBase.java - platform/test/mlts/benchmark - Git at Google

 /*
  * Copyright (C) 2018 The Android Open Source Project
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
  * You may obtain a copy of the License at
  *
  *      http://www.apache.org/licenses/LICENSE-2.0
  *
  * Unless required by applicable law or agreed to in writing, software
  * distributed under the License is distributed on an "AS IS" BASIS,
  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */

 package com.android.nn.benchmark.app;

 import android.app.Activity;
 import android.content.BroadcastReceiver;
 import android.content.Context;
 import android.content.Intent;
 import android.content.IntentFilter;
 import android.os.BatteryManager;
 import android.os.Trace;
 import android.test.ActivityInstrumentationTestCase2;
 import android.util.Log;

 import androidx.test.InstrumentationRegistry;

 import com.android.nn.benchmark.core.BenchmarkException;
 import com.android.nn.benchmark.core.BenchmarkResult;
 import com.android.nn.benchmark.core.TestModels;
 import com.android.nn.benchmark.core.TestModels.TestModelEntry;

 import org.junit.After;
 import org.junit.Before;
 import org.junit.runner.RunWith;
 import org.junit.runners.Parameterized;
 import org.junit.runners.Parameterized.Parameters;

 import java.io.IOException;
 import java.util.List;
 import java.util.concurrent.CountDownLatch;

 /**
  * Benchmark test-case super-class.
  *
  * Helper code for managing NNAPI/NNAPI-less benchamarks.
  */
 @RunWith(Parameterized.class)
 public class BenchmarkTestBase extends ActivityInstrumentationTestCase2<NNBenchmark> {

     // Only run 1 iteration now to fit the MediumTest time requirement.
     // One iteration means running the tests continuous for 1s.
     private NNBenchmark mActivity;
     protected final TestModelEntry mModel;

     // The default 0.3s warmup and 1.0s runtime give reasonably repeatable results (run-to-run
     // variability of ~20%) when run under performance settings (fixed CPU cores enabled and at
     // fixed frequency). The continuous build is not allowed to take much more than 1s so we
     // can't change the defaults for @MediumTest.
     protected static final float WARMUP_SHORT_SECONDS = 0.3f;
     protected static final float RUNTIME_SHORT_SECONDS = 1.f;

     // For running like a normal user-initiated app, the variability for 0.3s/1.0s is easily 3x.
     // With 2s/10s it's 20-50%. This @LargeTest allows running with these timings.
     protected static final float WARMUP_REPEATABLE_SECONDS = 2.f;
     protected static final float RUNTIME_REPEATABLE_SECONDS = 10.f;

     // For running a complete dataset, the run should complete under 5 minutes.
     protected static final float COMPLETE_SET_TIMEOUT_SECOND = 300.f;

     // For running compilation benchmarks.
     protected static final float COMPILATION_WARMUP_SECONDS = 2.f;
     protected static final float COMPILATION_RUNTIME_SECONDS = 10.f;
     protected static final int COMPILATION_MAX_ITERATIONS = 100;

     public BenchmarkTestBase(TestModelEntry model) {
         super(NNBenchmark.class);
         mModel = model;
     }

     protected void setUseNNApi(boolean useNNApi) {
         mActivity.setUseNNApi(useNNApi);
     }

     protected void setCompleteInputSet(boolean completeInputSet) {
         mActivity.setCompleteInputSet(completeInputSet);
     }

     protected void enableCompilationCachingBenchmarks() {
         mActivity.enableCompilationCachingBenchmarks(COMPILATION_WARMUP_SECONDS,
                 COMPILATION_RUNTIME_SECONDS, COMPILATION_MAX_ITERATIONS);
     }

     // Initialize the parameter for ImageProcessingActivityJB.
     protected void prepareTest() {
         injectInstrumentation(InstrumentationRegistry.getInstrumentation());
         mActivity = getActivity();
         mActivity.prepareInstrumentationTest();
         setUseNNApi(true);
     }

     public void waitUntilCharged() {
         BenchmarkTestBase.waitUntilCharged(mActivity, -1);
     }

     public static void waitUntilCharged(Context context, int minChargeLevel) {
         Log.v(NNBenchmark.TAG, "Waiting for the device to charge");

         final CountDownLatch chargedLatch = new CountDownLatch(1);
         BroadcastReceiver receiver = new BroadcastReceiver() {
             @Override
             public void onReceive(Context context, Intent intent) {
                 int level = intent.getIntExtra(BatteryManager.EXTRA_LEVEL, -1);
                 int scale = intent.getIntExtra(BatteryManager.EXTRA_SCALE, -1);
                 int percentage = level * 100 / scale;
                 if (minChargeLevel > 0 && minChargeLevel < 100) {
                     if (percentage >= minChargeLevel) {
                         Log.v(NNBenchmark.TAG,
                                 String.format(
                                         "Battery level: %d%% is greater than requested %d%%. "
                                                 + "Considering the device ready.",
                                         percentage, minChargeLevel));

                         chargedLatch.countDown();
                         return;
                     }
                 }

                 Log.v(NNBenchmark.TAG, String.format("Battery level: %d%%", percentage));

                 int status = intent.getIntExtra(BatteryManager.EXTRA_STATUS, -1);
                 if (status == BatteryManager.BATTERY_STATUS_FULL) {
                     chargedLatch.countDown();
                 } else if (status != BatteryManager.BATTERY_STATUS_CHARGING) {
                     Log.e(NNBenchmark.TAG,
                             String.format("Device is not charging, status is %d", status));
                 }
             }
         };

         context.registerReceiver(receiver, new IntentFilter(Intent.ACTION_BATTERY_CHANGED));
         try {
             chargedLatch.await();
         } catch (InterruptedException ignored) {
             Thread.currentThread().interrupt();
         }

         context.unregisterReceiver(receiver);
     }

     @Override
     @Before
     public void setUp() throws Exception {
         super.setUp();
         prepareTest();
         setActivityInitialTouchMode(false);
     }

     @Override
     @After
     public void tearDown() throws Exception {
         super.tearDown();
     }

     interface Joinable extends Runnable {
         // Syncrhonises the caller with the completion of the current action
         void join();
     }

     class TestAction implements Joinable {

         private final TestModelEntry mTestModel;
         private final float mMaxWarmupTimeSeconds;
         private final float mMaxRunTimeSeconds;
         private final CountDownLatch actionComplete;

         BenchmarkResult mResult;
         Throwable mException;

         public TestAction(TestModelEntry testName, float maxWarmupTimeSeconds,
                 float maxRunTimeSeconds) {
             mTestModel = testName;
             mMaxWarmupTimeSeconds = maxWarmupTimeSeconds;
             mMaxRunTimeSeconds = maxRunTimeSeconds;
             actionComplete = new CountDownLatch(1);
         }

         public void run() {
             Log.v(NNBenchmark.TAG, String.format(
                     "Starting benchmark for test '%s' running for max %f seconds",
                     mTestModel.mTestName,
                     mMaxRunTimeSeconds));
             try {
                 mResult = mActivity.runSynchronously(
                         mTestModel, mMaxWarmupTimeSeconds, mMaxRunTimeSeconds);
             } catch (BenchmarkException | IOException e) {
                 mException = e;
                 Log.e(NNBenchmark.TAG,
                         String.format("Error running Benchmark for test '%s'", mTestModel), e);
             } catch (Throwable e) {
                 mException = e;
                 Log.e(NNBenchmark.TAG,
                         String.format("Failure running Benchmark for test '%s'!!", mTestModel), e);
                 throw e;
             } finally {
                 actionComplete.countDown();
             }
         }

         public BenchmarkResult getBenchmark() {
             if (mException != null) {
                 throw new Error("run failed", mException);
             }
             return mResult;
         }

         @Override
         public void join() {
             try {
                 actionComplete.await();
             } catch (InterruptedException e) {
                 Thread.currentThread().interrupt();
                 Log.v(NNBenchmark.TAG, "Interrupted while waiting for action running", e);
             }
         }
     }

     // Set the benchmark thread to run on ui thread
     // Synchronized the thread such that the test will wait for the benchmark thread to finish
     public void runOnUiThread(Joinable action) {
         mActivity.runOnUiThread(action);
         action.join();
     }

     public void runTest(TestAction ta, String testName) {
         float sum = 0;
         // For NNAPI systrace usage documentation, see
         // frameworks/ml/nn/common/include/Tracing.h.
         final String traceName = "[NN_LA_PO]" + testName;
         try {
             Trace.beginSection(traceName);
             runOnUiThread(ta);
         } finally {
             Trace.endSection();
         }
         BenchmarkResult bmValue = ta.getBenchmark();

         // post result to INSTRUMENTATION_STATUS
         getInstrumentation().sendStatus(Activity.RESULT_OK, bmValue.toBundle(testName));
     }

     @Parameters(name = "{0}")
     public static List<TestModelEntry> modelsList() {
         return TestModels.modelsList();
     }
 }
	/*
	* Copyright (C) 2018 The Android Open Source Project
	*
	* Licensed under the Apache License, Version 2.0 (the "License");
	* you may not use this file except in compliance with the License.
	* You may obtain a copy of the License at
	*
	* http://www.apache.org/licenses/LICENSE-2.0
	*
	* Unless required by applicable law or agreed to in writing, software
	* distributed under the License is distributed on an "AS IS" BASIS,
	* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
	* See the License for the specific language governing permissions and
	* limitations under the License.
	*/

	package com.android.nn.benchmark.app;

	import android.app.Activity;
	import android.content.BroadcastReceiver;
	import android.content.Context;
	import android.content.Intent;
	import android.content.IntentFilter;
	import android.os.BatteryManager;
	import android.os.Trace;
	import android.test.ActivityInstrumentationTestCase2;
	import android.util.Log;

	import androidx.test.InstrumentationRegistry;

	import com.android.nn.benchmark.core.BenchmarkException;
	import com.android.nn.benchmark.core.BenchmarkResult;
	import com.android.nn.benchmark.core.TestModels;
	import com.android.nn.benchmark.core.TestModels.TestModelEntry;

	import org.junit.After;
	import org.junit.Before;
	import org.junit.runner.RunWith;
	import org.junit.runners.Parameterized;
	import org.junit.runners.Parameterized.Parameters;

	import java.io.IOException;
	import java.util.List;
	import java.util.concurrent.CountDownLatch;

	/**
	* Benchmark test-case super-class.
	*
	* Helper code for managing NNAPI/NNAPI-less benchamarks.
	*/
	@RunWith(Parameterized.class)
	public class BenchmarkTestBase extends ActivityInstrumentationTestCase2<NNBenchmark> {

	// Only run 1 iteration now to fit the MediumTest time requirement.
	// One iteration means running the tests continuous for 1s.
	private NNBenchmark mActivity;
	protected final TestModelEntry mModel;

	// The default 0.3s warmup and 1.0s runtime give reasonably repeatable results (run-to-run
	// variability of ~20%) when run under performance settings (fixed CPU cores enabled and at
	// fixed frequency). The continuous build is not allowed to take much more than 1s so we
	// can't change the defaults for @MediumTest.
	protected static final float WARMUP_SHORT_SECONDS = 0.3f;
	protected static final float RUNTIME_SHORT_SECONDS = 1.f;

	// For running like a normal user-initiated app, the variability for 0.3s/1.0s is easily 3x.
	// With 2s/10s it's 20-50%. This @LargeTest allows running with these timings.
	protected static final float WARMUP_REPEATABLE_SECONDS = 2.f;
	protected static final float RUNTIME_REPEATABLE_SECONDS = 10.f;

	// For running a complete dataset, the run should complete under 5 minutes.
	protected static final float COMPLETE_SET_TIMEOUT_SECOND = 300.f;

	// For running compilation benchmarks.
	protected static final float COMPILATION_WARMUP_SECONDS = 2.f;
	protected static final float COMPILATION_RUNTIME_SECONDS = 10.f;
	protected static final int COMPILATION_MAX_ITERATIONS = 100;

	public BenchmarkTestBase(TestModelEntry model) {
	super(NNBenchmark.class);
	mModel = model;
	}

	protected void setUseNNApi(boolean useNNApi) {
	mActivity.setUseNNApi(useNNApi);
	}

	protected void setCompleteInputSet(boolean completeInputSet) {
	mActivity.setCompleteInputSet(completeInputSet);
	}

	protected void enableCompilationCachingBenchmarks() {
	mActivity.enableCompilationCachingBenchmarks(COMPILATION_WARMUP_SECONDS,
	COMPILATION_RUNTIME_SECONDS, COMPILATION_MAX_ITERATIONS);
	}

	// Initialize the parameter for ImageProcessingActivityJB.
	protected void prepareTest() {
	injectInstrumentation(InstrumentationRegistry.getInstrumentation());
	mActivity = getActivity();
	mActivity.prepareInstrumentationTest();
	setUseNNApi(true);
	}

	public void waitUntilCharged() {
	BenchmarkTestBase.waitUntilCharged(mActivity, -1);
	}

	public static void waitUntilCharged(Context context, int minChargeLevel) {
	Log.v(NNBenchmark.TAG, "Waiting for the device to charge");

	final CountDownLatch chargedLatch = new CountDownLatch(1);
	BroadcastReceiver receiver = new BroadcastReceiver() {
	@Override
	public void onReceive(Context context, Intent intent) {
	int level = intent.getIntExtra(BatteryManager.EXTRA_LEVEL, -1);
	int scale = intent.getIntExtra(BatteryManager.EXTRA_SCALE, -1);
	int percentage = level * 100 / scale;
	if (minChargeLevel > 0 && minChargeLevel < 100) {
	if (percentage >= minChargeLevel) {
	Log.v(NNBenchmark.TAG,
	String.format(
	"Battery level: %d%% is greater than requested %d%%. "
	+ "Considering the device ready.",
	percentage, minChargeLevel));

	chargedLatch.countDown();
	return;
	}
	}

	Log.v(NNBenchmark.TAG, String.format("Battery level: %d%%", percentage));

	int status = intent.getIntExtra(BatteryManager.EXTRA_STATUS, -1);
	if (status == BatteryManager.BATTERY_STATUS_FULL) {
	chargedLatch.countDown();
	} else if (status != BatteryManager.BATTERY_STATUS_CHARGING) {
	Log.e(NNBenchmark.TAG,
	String.format("Device is not charging, status is %d", status));
	}
	}
	};

	context.registerReceiver(receiver, new IntentFilter(Intent.ACTION_BATTERY_CHANGED));
	try {
	chargedLatch.await();
	} catch (InterruptedException ignored) {
	Thread.currentThread().interrupt();
	}

	context.unregisterReceiver(receiver);
	}

	@Override
	@Before
	public void setUp() throws Exception {
	super.setUp();
	prepareTest();
	setActivityInitialTouchMode(false);
	}

	@Override
	@After
	public void tearDown() throws Exception {
	super.tearDown();
	}

	interface Joinable extends Runnable {
	// Syncrhonises the caller with the completion of the current action
	void join();
	}

	class TestAction implements Joinable {

	private final TestModelEntry mTestModel;
	private final float mMaxWarmupTimeSeconds;
	private final float mMaxRunTimeSeconds;
	private final CountDownLatch actionComplete;

	BenchmarkResult mResult;
	Throwable mException;

	public TestAction(TestModelEntry testName, float maxWarmupTimeSeconds,
	float maxRunTimeSeconds) {
	mTestModel = testName;
	mMaxWarmupTimeSeconds = maxWarmupTimeSeconds;
	mMaxRunTimeSeconds = maxRunTimeSeconds;
	actionComplete = new CountDownLatch(1);
	}

	public void run() {
	Log.v(NNBenchmark.TAG, String.format(
	"Starting benchmark for test '%s' running for max %f seconds",
	mTestModel.mTestName,
	mMaxRunTimeSeconds));
	try {
	mResult = mActivity.runSynchronously(
	mTestModel, mMaxWarmupTimeSeconds, mMaxRunTimeSeconds);
	} catch (BenchmarkException \| IOException e) {
	mException = e;
	Log.e(NNBenchmark.TAG,
	String.format("Error running Benchmark for test '%s'", mTestModel), e);
	} catch (Throwable e) {
	mException = e;
	Log.e(NNBenchmark.TAG,
	String.format("Failure running Benchmark for test '%s'!!", mTestModel), e);
	throw e;
	} finally {
	actionComplete.countDown();
	}
	}

	public BenchmarkResult getBenchmark() {
	if (mException != null) {
	throw new Error("run failed", mException);
	}
	return mResult;
	}

	@Override
	public void join() {
	try {
	actionComplete.await();
	} catch (InterruptedException e) {
	Thread.currentThread().interrupt();
	Log.v(NNBenchmark.TAG, "Interrupted while waiting for action running", e);
	}
	}
	}

	// Set the benchmark thread to run on ui thread
	// Synchronized the thread such that the test will wait for the benchmark thread to finish
	public void runOnUiThread(Joinable action) {
	mActivity.runOnUiThread(action);
	action.join();
	}

	public void runTest(TestAction ta, String testName) {
	float sum = 0;
	// For NNAPI systrace usage documentation, see
	// frameworks/ml/nn/common/include/Tracing.h.
	final String traceName = "[NN_LA_PO]" + testName;
	try {
	Trace.beginSection(traceName);
	runOnUiThread(ta);
	} finally {
	Trace.endSection();
	}
	BenchmarkResult bmValue = ta.getBenchmark();

	// post result to INSTRUMENTATION_STATUS
	getInstrumentation().sendStatus(Activity.RESULT_OK, bmValue.toBundle(testName));
	}

	@Parameters(name = "{0}")
	public static List<TestModelEntry> modelsList() {
	return TestModels.modelsList();
	}
	}