| /* |
| * Copyright (C) 2012 The Android Open Source Project |
| * |
| * Licensed under the Apache License, Version 2.0 (the "License"); |
| * you may not use this file except in compliance with the License. |
| * You may obtain a copy of the License at |
| * |
| * http://www.apache.org/licenses/LICENSE-2.0 |
| * |
| * Unless required by applicable law or agreed to in writing, software |
| * distributed under the License is distributed on an "AS IS" BASIS, |
| * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| * See the License for the specific language governing permissions and |
| * limitations under the License. |
| */ |
| |
| package com.android.test.hwuicompare; |
| |
| import java.io.File; |
| import java.io.FileInputStream; |
| import java.io.FileOutputStream; |
| import java.io.IOException; |
| import java.util.ArrayList; |
| import java.util.Comparator; |
| import java.util.HashMap; |
| import java.util.TreeSet; |
| |
| import org.json.JSONException; |
| import org.json.JSONObject; |
| |
| import android.os.Bundle; |
| import android.os.Environment; |
| import android.os.Trace; |
| import android.util.Log; |
| import android.widget.ImageView; |
| import android.widget.Toast; |
| |
| public class AutomaticActivity extends CompareActivity { |
| private static final String LOG_TAG = "AutomaticActivity"; |
| private static final float ERROR_DISPLAY_THRESHOLD = 0.01f; |
| protected static final boolean DRAW_BITMAPS = false; |
| |
| /** |
| * Threshold of error change required to consider a test regressed/improved |
| */ |
| private static final float ERROR_CHANGE_THRESHOLD = 0.001f; |
| |
| private static final float[] ERROR_CUTOFFS = { |
| 0, 0.005f, 0.01f, 0.02f, 0.05f, 0.1f, 0.25f, 0.5f, 1f, 2f |
| }; |
| |
| private final float[] mErrorRates = new float[ERROR_CUTOFFS.length]; |
| private float mTotalTests = 0; |
| private float mTotalError = 0; |
| private int mTestsRegressed = 0; |
| private int mTestsImproved = 0; |
| |
| private ImageView mSoftwareImageView = null; |
| private ImageView mHardwareImageView = null; |
| |
| |
| public abstract static class FinalCallback { |
| abstract void report(String name, float value); |
| void complete() {}; |
| } |
| |
| private final ArrayList<FinalCallback> mFinalCallbacks = new ArrayList<FinalCallback>(); |
| |
| Runnable mRunnable = new Runnable() { |
| @Override |
| public void run() { |
| loadBitmaps(); |
| if (mSoftwareBitmap == null || mHardwareBitmap == null) { |
| Log.e(LOG_TAG, "bitmap is null"); |
| return; |
| } |
| |
| if (DRAW_BITMAPS) { |
| mSoftwareImageView.setImageBitmap(mSoftwareBitmap); |
| mHardwareImageView.setImageBitmap(mHardwareBitmap); |
| } |
| |
| Trace.traceBegin(Trace.TRACE_TAG_ALWAYS, "calculateError"); |
| float error = mErrorCalculator.calcErrorRS(mSoftwareBitmap, mHardwareBitmap); |
| Trace.traceEnd(Trace.TRACE_TAG_ALWAYS); |
| |
| final String[] modifierNames = DisplayModifier.getLastAppliedModifications(); |
| handleError(modifierNames, error); |
| |
| if (DisplayModifier.step()) { |
| finishTest(); |
| } else { |
| mHardwareView.invalidate(); |
| if (DRAW_BITMAPS) { |
| mSoftwareImageView.invalidate(); |
| mHardwareImageView.invalidate(); |
| } |
| } |
| mHandler.removeCallbacks(mRunnable); |
| } |
| }; |
| |
| @Override |
| protected void onPause() { |
| super.onPause(); |
| mHandler.removeCallbacks(mRunnable); |
| }; |
| |
| @Override |
| protected void onCreate(Bundle savedInstanceState) { |
| super.onCreate(savedInstanceState); |
| setContentView(R.layout.automatic_layout); |
| |
| mSoftwareImageView = (ImageView) findViewById(R.id.software_image_view); |
| mHardwareImageView = (ImageView) findViewById(R.id.hardware_image_view); |
| |
| onCreateCommon(mRunnable); |
| beginTest(); |
| } |
| |
| private static class TestResult { |
| TestResult(String label, float error) { |
| mLabel = label; |
| mTotalError = error; |
| mCount = 1; |
| } |
| public void addInto(float error) { |
| mTotalError += error; |
| mCount++; |
| } |
| public float getAverage() { |
| return mTotalError / mCount; |
| } |
| final String mLabel; |
| float mTotalError; |
| int mCount; |
| } |
| |
| JSONObject mOutputJson = null; |
| JSONObject mInputJson = null; |
| final HashMap<String, TestResult> mModifierResults = new HashMap<String, TestResult>(); |
| final HashMap<String, TestResult> mIndividualResults = new HashMap<String, TestResult>(); |
| final HashMap<String, TestResult> mModifierDiffResults = new HashMap<String, TestResult>(); |
| final HashMap<String, TestResult> mIndividualDiffResults = new HashMap<String, TestResult>(); |
| private void beginTest() { |
| mFinalCallbacks.add(new FinalCallback() { |
| @Override |
| void report(String name, float value) { |
| Log.d(LOG_TAG, name + " " + value); |
| }; |
| }); |
| |
| File inputFile = new File(Environment.getExternalStorageDirectory(), |
| "CanvasCompareInput.json"); |
| if (inputFile.exists() && inputFile.canRead() && inputFile.length() > 0) { |
| try { |
| FileInputStream inputStream = new FileInputStream(inputFile); |
| Log.d(LOG_TAG, "Parsing input file..."); |
| StringBuffer content = new StringBuffer((int)inputFile.length()); |
| byte[] buffer = new byte[1024]; |
| while (inputStream.read(buffer) != -1) { |
| content.append(new String(buffer)); |
| } |
| mInputJson = new JSONObject(content.toString()); |
| inputStream.close(); |
| Log.d(LOG_TAG, "Parsed input file with " + mInputJson.length() + " entries"); |
| } catch (JSONException e) { |
| Log.e(LOG_TAG, "error parsing input json", e); |
| } catch (IOException e) { |
| Log.e(LOG_TAG, "error reading input json from sd", e); |
| } |
| } |
| |
| mOutputJson = new JSONObject(); |
| } |
| |
| private static void logTestResultHash(String label, HashMap<String, TestResult> map) { |
| Log.d(LOG_TAG, "---------------"); |
| Log.d(LOG_TAG, label + ":"); |
| Log.d(LOG_TAG, "---------------"); |
| TreeSet<TestResult> set = new TreeSet<TestResult>(new Comparator<TestResult>() { |
| @Override |
| public int compare(TestResult lhs, TestResult rhs) { |
| if (lhs == rhs) return 0; // don't need to worry about complex equality |
| |
| int cmp = Float.compare(lhs.getAverage(), rhs.getAverage()); |
| if (cmp != 0) { |
| return cmp; |
| } |
| return lhs.mLabel.compareTo(rhs.mLabel); |
| } |
| }); |
| |
| for (TestResult t : map.values()) { |
| set.add(t); |
| } |
| |
| for (TestResult t : set.descendingSet()) { |
| if (Math.abs(t.getAverage()) > ERROR_DISPLAY_THRESHOLD) { |
| Log.d(LOG_TAG, String.format("%2.4f : %s", t.getAverage(), t.mLabel)); |
| } |
| } |
| Log.d(LOG_TAG, ""); |
| } |
| |
| private void finishTest() { |
| for (FinalCallback c : mFinalCallbacks) { |
| c.report("averageError", (mTotalError / mTotalTests)); |
| for (int i = 1; i < ERROR_CUTOFFS.length; i++) { |
| c.report(String.format("tests with error over %1.3f", ERROR_CUTOFFS[i]), |
| mErrorRates[i]); |
| } |
| if (mInputJson != null) { |
| c.report("tests regressed", mTestsRegressed); |
| c.report("tests improved", mTestsImproved); |
| } |
| c.complete(); |
| } |
| |
| try { |
| if (mOutputJson != null) { |
| String outputString = mOutputJson.toString(4); |
| File outputFile = new File(Environment.getExternalStorageDirectory(), |
| "CanvasCompareOutput.json"); |
| FileOutputStream outputStream = new FileOutputStream(outputFile); |
| outputStream.write(outputString.getBytes()); |
| outputStream.close(); |
| Log.d(LOG_TAG, "Saved output file with " + mOutputJson.length() + " entries"); |
| } |
| } catch (JSONException e) { |
| Log.e(LOG_TAG, "error during JSON stringify", e); |
| } catch (IOException e) { |
| Log.e(LOG_TAG, "error storing JSON output on sd", e); |
| } |
| |
| logTestResultHash("Modifier change vs previous", mModifierDiffResults); |
| logTestResultHash("Invidual test change vs previous", mIndividualDiffResults); |
| logTestResultHash("Modifier average test results", mModifierResults); |
| logTestResultHash("Individual test results", mIndividualResults); |
| |
| Toast.makeText(getApplicationContext(), "done!", Toast.LENGTH_SHORT).show(); |
| finish(); |
| } |
| |
| /** |
| * Inserts the error value into all TestResult objects, associated with each of its modifiers |
| */ |
| private static void addForAllModifiers(String fullName, float error, String[] modifierNames, |
| HashMap<String, TestResult> modifierResults) { |
| for (String modifierName : modifierNames) { |
| TestResult r = modifierResults.get(fullName); |
| if (r == null) { |
| modifierResults.put(modifierName, new TestResult(modifierName, error)); |
| } else { |
| r.addInto(error); |
| } |
| } |
| } |
| |
| private void handleError(final String[] modifierNames, final float error) { |
| String fullName = ""; |
| for (String s : modifierNames) { |
| fullName = fullName.concat("." + s); |
| } |
| fullName = fullName.substring(1); |
| |
| float deltaError = 0; |
| if (mInputJson != null) { |
| try { |
| deltaError = error - (float)mInputJson.getDouble(fullName); |
| } catch (JSONException e) { |
| Log.w(LOG_TAG, "Warning: unable to read from input json", e); |
| } |
| if (deltaError > ERROR_CHANGE_THRESHOLD) mTestsRegressed++; |
| if (deltaError < -ERROR_CHANGE_THRESHOLD) mTestsImproved++; |
| mIndividualDiffResults.put(fullName, new TestResult(fullName, deltaError)); |
| addForAllModifiers(fullName, deltaError, modifierNames, mModifierDiffResults); |
| } |
| |
| mIndividualResults.put(fullName, new TestResult(fullName, error)); |
| addForAllModifiers(fullName, error, modifierNames, mModifierResults); |
| |
| try { |
| if (mOutputJson != null) { |
| mOutputJson.put(fullName, error); |
| } |
| } catch (JSONException e) { |
| Log.e(LOG_TAG, "exception during JSON recording", e); |
| mOutputJson = null; |
| } |
| |
| for (int i = 0; i < ERROR_CUTOFFS.length; i++) { |
| if (error <= ERROR_CUTOFFS[i]) break; |
| mErrorRates[i]++; |
| } |
| mTotalError += error; |
| mTotalTests++; |
| } |
| |
| @Override |
| protected boolean forceRecreateBitmaps() { |
| // disable, unless needed for drawing into imageviews |
| return DRAW_BITMAPS; |
| } |
| |
| // FOR TESTING |
| public void setFinalCallback(FinalCallback c) { |
| mFinalCallbacks.add(c); |
| } |
| } |