| /* |
| * Copyright 2013 The Android Open Source Project |
| * |
| * Licensed under the Apache License, Version 2.0 (the "License"); |
| * you may not use this file except in compliance with the License. |
| * You may obtain a copy of the License at |
| * |
| * http://www.apache.org/licenses/LICENSE-2.0 |
| * |
| * Unless required by applicable law or agreed to in writing, software |
| * distributed under the License is distributed on an "AS IS" BASIS, |
| * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| * See the License for the specific language governing permissions and |
| * limitations under the License. |
| */ |
| // Takes sharpness score, rates the image good if above 10, bad otherwise |
| |
| package androidx.media.filterfw.samples.simplecamera; |
| |
| import android.graphics.Bitmap; |
| import android.os.AsyncTask; |
| import android.util.Log; |
| import android.widget.ImageView; |
| import androidx.media.filterfw.Filter; |
| import androidx.media.filterfw.FrameImage2D; |
| import androidx.media.filterfw.FrameType; |
| import androidx.media.filterfw.FrameValue; |
| import androidx.media.filterfw.MffContext; |
| import androidx.media.filterfw.OutputPort; |
| import androidx.media.filterfw.Signature; |
| |
| public class ImageGoodnessFilter extends Filter { |
| |
| private static final String TAG = "ImageGoodnessFilter"; |
| private static boolean mLogVerbose = Log.isLoggable(TAG, Log.VERBOSE); |
| |
| private final static String GREAT = "Great Picture!"; |
| private final static String GOOD = "Good Picture!"; |
| private final static String OK = "Ok Picture"; |
| private final static String BAD = "Bad Picture"; |
| private final static String AWFUL = "Awful Picture"; |
| private final static float SMALL_SCORE_INC = 0.25f; |
| private final static float BIG_SCORE_INC = 0.5f; |
| private final static float LOW_VARIANCE = 0.1f; |
| private final static float MEDIUM_VARIANCE = 10; |
| private final static float HIGH_VARIANCE = 100; |
| private float sharpnessMean = 0; |
| private float sharpnessVar = 0; |
| private float underExposureMean = 0; |
| private float underExposureVar = 0; |
| private float overExposureMean = 0; |
| private float overExposureVar = 0; |
| private float contrastMean = 0; |
| private float contrastVar = 0; |
| private float colorfulnessMean = 0; |
| private float colorfulnessVar = 0; |
| private float brightnessMean = 0; |
| private float brightnessVar = 0; |
| |
| private float motionMean = 0; |
| private float scoreMean = 0; |
| private static final float DECAY = 0.03f; |
| /** |
| * @param context |
| * @param name |
| */ |
| public ImageGoodnessFilter(MffContext context, String name) { |
| super(context, name); |
| } |
| |
| @Override |
| public Signature getSignature() { |
| FrameType floatT = FrameType.single(float.class); |
| FrameType imageIn = FrameType.image2D(FrameType.ELEMENT_RGBA8888, FrameType.READ_GPU); |
| |
| return new Signature() |
| .addInputPort("sharpness", Signature.PORT_REQUIRED, floatT) |
| .addInputPort("overExposure", Signature.PORT_REQUIRED, floatT) |
| .addInputPort("underExposure", Signature.PORT_REQUIRED, floatT) |
| .addInputPort("colorfulness", Signature.PORT_REQUIRED, floatT) |
| .addInputPort("contrastRating", Signature.PORT_REQUIRED, floatT) |
| .addInputPort("motionValues", Signature.PORT_REQUIRED, FrameType.array(float.class)) |
| .addInputPort("brightness", Signature.PORT_REQUIRED, floatT) |
| .addInputPort("capturing", Signature.PORT_REQUIRED, FrameType.single(boolean.class)) |
| .addInputPort("image", Signature.PORT_REQUIRED, imageIn) |
| .addOutputPort("goodOrBadPic", Signature.PORT_REQUIRED, |
| FrameType.single(String.class)) |
| .addOutputPort("score", Signature.PORT_OPTIONAL, floatT) |
| .disallowOtherPorts(); |
| } |
| |
| /** |
| * @see androidx.media.filterfw.Filter#onProcess() |
| */ |
| @Override |
| protected void onProcess() { |
| FrameValue sharpnessFrameValue = |
| getConnectedInputPort("sharpness").pullFrame().asFrameValue(); |
| float sharpness = ((Float)sharpnessFrameValue.getValue()).floatValue(); |
| |
| FrameValue overExposureFrameValue = |
| getConnectedInputPort("overExposure").pullFrame().asFrameValue(); |
| float overExposure = ((Float)overExposureFrameValue.getValue()).floatValue(); |
| |
| FrameValue underExposureFrameValue = |
| getConnectedInputPort("underExposure").pullFrame().asFrameValue(); |
| float underExposure = ((Float)underExposureFrameValue.getValue()).floatValue(); |
| |
| FrameValue colorfulnessFrameValue = |
| getConnectedInputPort("colorfulness").pullFrame().asFrameValue(); |
| float colorfulness = ((Float)colorfulnessFrameValue.getValue()).floatValue(); |
| |
| FrameValue contrastRatingFrameValue = |
| getConnectedInputPort("contrastRating").pullFrame().asFrameValue(); |
| float contrastRating = ((Float)contrastRatingFrameValue.getValue()).floatValue(); |
| |
| FrameValue brightnessFrameValue = |
| getConnectedInputPort("brightness").pullFrame().asFrameValue(); |
| float brightness = ((Float)brightnessFrameValue.getValue()).floatValue(); |
| |
| FrameValue motionValuesFrameValue = |
| getConnectedInputPort("motionValues").pullFrame().asFrameValue(); |
| float[] motionValues = (float[]) motionValuesFrameValue.getValue(); |
| |
| |
| float vectorAccel = (float) Math.sqrt(Math.pow(motionValues[0], 2) + |
| Math.pow(motionValues[1], 2) + Math.pow(motionValues[2], 2)); |
| String outStr; |
| |
| FrameValue capturingFrameValue = |
| getConnectedInputPort("capturing").pullFrame().asFrameValue(); |
| boolean capturing = (Boolean) capturingFrameValue.getValue(); |
| |
| FrameImage2D inputImage = getConnectedInputPort("image").pullFrame().asFrameImage2D(); |
| |
| |
| // TODO: get rid of magic numbers |
| float score = 0.0f; |
| score = computePictureScore(vectorAccel, sharpness, underExposure, overExposure, |
| contrastRating, colorfulness, brightness); |
| if (scoreMean == 0) scoreMean = score; |
| else scoreMean = scoreMean * (1 - DECAY) + score * DECAY; |
| |
| if (motionMean == 0) motionMean = vectorAccel; |
| else motionMean = motionMean * (1 - DECAY) + vectorAccel * DECAY; |
| |
| float classifierScore = classifierComputeScore(vectorAccel, sharpness, underExposure, |
| colorfulness, contrastRating, score); |
| |
| // Log.v(TAG, "ClassifierScore:: " + classifierScore); |
| final float GREAT_SCORE = 3.5f; |
| final float GOOD_SCORE = 2.5f; |
| final float OK_SCORE = 1.5f; |
| final float BAD_SCORE = 0.5f; |
| |
| if (score >= GREAT_SCORE) { |
| outStr = GREAT; |
| } else if (score >= GOOD_SCORE) { |
| outStr = GOOD; |
| } else if (score >= OK_SCORE) { |
| outStr = OK; |
| } else if (score >= BAD_SCORE) { |
| outStr = BAD; |
| } else { |
| outStr = AWFUL; |
| } |
| |
| if(capturing) { |
| if (outStr.equals(GREAT)) { |
| // take a picture |
| Bitmap bitmap = inputImage.toBitmap(); |
| |
| new AsyncOperation().execute(bitmap); |
| final float RESET_FEATURES = 0.01f; |
| sharpnessMean = RESET_FEATURES; |
| underExposureMean = RESET_FEATURES; |
| overExposureMean = RESET_FEATURES; |
| contrastMean = RESET_FEATURES; |
| colorfulnessMean = RESET_FEATURES; |
| brightnessMean = RESET_FEATURES; |
| } |
| } |
| |
| OutputPort outPort = getConnectedOutputPort("goodOrBadPic"); |
| FrameValue stringFrame = outPort.fetchAvailableFrame(null).asFrameValue(); |
| stringFrame.setValue(outStr); |
| outPort.pushFrame(stringFrame); |
| |
| OutputPort scoreOutPort = getConnectedOutputPort("score"); |
| FrameValue scoreFrame = scoreOutPort.fetchAvailableFrame(null).asFrameValue(); |
| scoreFrame.setValue(score); |
| scoreOutPort.pushFrame(scoreFrame); |
| |
| } |
| |
| private class AsyncOperation extends AsyncTask<Bitmap, Void, String> { |
| private Bitmap b; |
| protected void onPostExecute(String result) { |
| ImageView view = SmartCamera.getImageView(); |
| view.setImageBitmap(b); |
| } |
| |
| @Override |
| protected String doInBackground(Bitmap... params) { |
| // TODO Auto-generated method stub |
| b = params[0]; |
| return null; |
| } |
| |
| } |
| // Returns a number between -1 and 1 |
| private float classifierComputeScore(float vectorAccel, float sharpness, float underExposure, |
| float colorfulness, float contrast, float score) { |
| float result = (-0.0223f * sharpness + -0.0563f * underExposure + 0.0137f * colorfulness |
| + 0.3102f * contrast + 0.0314f * vectorAccel + -0.0094f * score + 0.0227f * |
| sharpnessMean + 0.0459f * underExposureMean + -0.3934f * contrastMean + |
| -0.0697f * motionMean + 0.0091f * scoreMean + -0.0152f); |
| return result; |
| } |
| |
| // Returns a number between -1 and 4 representing the score for this picture |
| private float computePictureScore(float vector_accel, float sharpness, |
| float underExposure, float overExposure, float contrastRating, float colorfulness, |
| float brightness) { |
| final float ACCELERATION_THRESHOLD_VERY_STEADY = 0.1f; |
| final float ACCELERATION_THRESHOLD_STEADY = 0.3f; |
| final float ACCELERATION_THRESHOLD_MOTION = 2f; |
| |
| float score = 0.0f; |
| if (vector_accel > ACCELERATION_THRESHOLD_MOTION) { |
| score -= (BIG_SCORE_INC + BIG_SCORE_INC); // set score to -1, bad pic |
| } else if (vector_accel > ACCELERATION_THRESHOLD_STEADY) { |
| score -= BIG_SCORE_INC; |
| score = subComputeScore(sharpness, underExposure, overExposure, contrastRating, |
| colorfulness, brightness, score); |
| } else if (vector_accel < ACCELERATION_THRESHOLD_VERY_STEADY) { |
| score += BIG_SCORE_INC; |
| score = subComputeScore(sharpness, underExposure, overExposure, contrastRating, |
| colorfulness, brightness, score); |
| } else { |
| score = subComputeScore(sharpness, underExposure, overExposure, contrastRating, |
| colorfulness, brightness, score); |
| } |
| return score; |
| } |
| |
| // Changes the score by at most +/- 3.5 |
| private float subComputeScore(float sharpness, float underExposure, float overExposure, |
| float contrastRating, float colorfulness, float brightness, float score) { |
| // The score methods return values -0.5 to 0.5 |
| final float SHARPNESS_WEIGHT = 2; |
| score += SHARPNESS_WEIGHT * sharpnessScore(sharpness); |
| score += underExposureScore(underExposure); |
| score += overExposureScore(overExposure); |
| score += contrastScore(contrastRating); |
| score += colorfulnessScore(colorfulness); |
| score += brightnessScore(brightness); |
| return score; |
| } |
| |
| private float sharpnessScore(float sharpness) { |
| if (sharpnessMean == 0) { |
| sharpnessMean = sharpness; |
| sharpnessVar = 0; |
| return 0; |
| } else { |
| sharpnessMean = sharpnessMean * (1 - DECAY) + sharpness * DECAY; |
| sharpnessVar = sharpnessVar * (1 - DECAY) + (sharpness - sharpnessMean) * |
| (sharpness - sharpnessMean) * DECAY; |
| if (sharpnessVar < LOW_VARIANCE) { |
| return BIG_SCORE_INC; |
| } else if (sharpness < sharpnessMean && sharpnessVar > MEDIUM_VARIANCE) { |
| return -BIG_SCORE_INC; |
| } else if (sharpness < sharpnessMean) { |
| return -SMALL_SCORE_INC; |
| } else if (sharpness > sharpnessMean && sharpnessVar > HIGH_VARIANCE) { |
| return 0; |
| } else if (sharpness > sharpnessMean && sharpnessVar > MEDIUM_VARIANCE) { |
| return SMALL_SCORE_INC; |
| } else { |
| return BIG_SCORE_INC; // low variance, sharpness above the mean |
| } |
| } |
| } |
| |
| private float underExposureScore(float underExposure) { |
| if (underExposureMean == 0) { |
| underExposureMean = underExposure; |
| underExposureVar = 0; |
| return 0; |
| } else { |
| underExposureMean = underExposureMean * (1 - DECAY) + underExposure * DECAY; |
| underExposureVar = underExposureVar * (1 - DECAY) + (underExposure - underExposureMean) |
| * (underExposure - underExposureMean) * DECAY; |
| if (underExposureVar < LOW_VARIANCE) { |
| return BIG_SCORE_INC; |
| } else if (underExposure > underExposureMean && underExposureVar > MEDIUM_VARIANCE) { |
| return -BIG_SCORE_INC; |
| } else if (underExposure > underExposureMean) { |
| return -SMALL_SCORE_INC; |
| } else if (underExposure < underExposureMean && underExposureVar > HIGH_VARIANCE) { |
| return 0; |
| } else if (underExposure < underExposureMean && underExposureVar > MEDIUM_VARIANCE) { |
| return SMALL_SCORE_INC; |
| } else { |
| return BIG_SCORE_INC; // low variance, underExposure below the mean |
| } |
| } |
| } |
| |
| private float overExposureScore(float overExposure) { |
| if (overExposureMean == 0) { |
| overExposureMean = overExposure; |
| overExposureVar = 0; |
| return 0; |
| } else { |
| overExposureMean = overExposureMean * (1 - DECAY) + overExposure * DECAY; |
| overExposureVar = overExposureVar * (1 - DECAY) + (overExposure - overExposureMean) * |
| (overExposure - overExposureMean) * DECAY; |
| if (overExposureVar < LOW_VARIANCE) { |
| return BIG_SCORE_INC; |
| } else if (overExposure > overExposureMean && overExposureVar > MEDIUM_VARIANCE) { |
| return -BIG_SCORE_INC; |
| } else if (overExposure > overExposureMean) { |
| return -SMALL_SCORE_INC; |
| } else if (overExposure < overExposureMean && overExposureVar > HIGH_VARIANCE) { |
| return 0; |
| } else if (overExposure < overExposureMean && overExposureVar > MEDIUM_VARIANCE) { |
| return SMALL_SCORE_INC; |
| } else { |
| return BIG_SCORE_INC; // low variance, overExposure below the mean |
| } |
| } |
| } |
| |
| private float contrastScore(float contrast) { |
| if (contrastMean == 0) { |
| contrastMean = contrast; |
| contrastVar = 0; |
| return 0; |
| } else { |
| contrastMean = contrastMean * (1 - DECAY) + contrast * DECAY; |
| contrastVar = contrastVar * (1 - DECAY) + (contrast - contrastMean) * |
| (contrast - contrastMean) * DECAY; |
| if (contrastVar < LOW_VARIANCE) { |
| return BIG_SCORE_INC; |
| } else if (contrast < contrastMean && contrastVar > MEDIUM_VARIANCE) { |
| return -BIG_SCORE_INC; |
| } else if (contrast < contrastMean) { |
| return -SMALL_SCORE_INC; |
| } else if (contrast > contrastMean && contrastVar > 100) { |
| return 0; |
| } else if (contrast > contrastMean && contrastVar > MEDIUM_VARIANCE) { |
| return SMALL_SCORE_INC; |
| } else { |
| return BIG_SCORE_INC; // low variance, contrast above the mean |
| } |
| } |
| } |
| |
| private float colorfulnessScore(float colorfulness) { |
| if (colorfulnessMean == 0) { |
| colorfulnessMean = colorfulness; |
| colorfulnessVar = 0; |
| return 0; |
| } else { |
| colorfulnessMean = colorfulnessMean * (1 - DECAY) + colorfulness * DECAY; |
| colorfulnessVar = colorfulnessVar * (1 - DECAY) + (colorfulness - colorfulnessMean) * |
| (colorfulness - colorfulnessMean) * DECAY; |
| if (colorfulnessVar < LOW_VARIANCE) { |
| return BIG_SCORE_INC; |
| } else if (colorfulness < colorfulnessMean && colorfulnessVar > MEDIUM_VARIANCE) { |
| return -BIG_SCORE_INC; |
| } else if (colorfulness < colorfulnessMean) { |
| return -SMALL_SCORE_INC; |
| } else if (colorfulness > colorfulnessMean && colorfulnessVar > 100) { |
| return 0; |
| } else if (colorfulness > colorfulnessMean && colorfulnessVar > MEDIUM_VARIANCE) { |
| return SMALL_SCORE_INC; |
| } else { |
| return BIG_SCORE_INC; // low variance, colorfulness above the mean |
| } |
| } |
| } |
| |
| private float brightnessScore(float brightness) { |
| if (brightnessMean == 0) { |
| brightnessMean = brightness; |
| brightnessVar = 0; |
| return 0; |
| } else { |
| brightnessMean = brightnessMean * (1 - DECAY) + brightness * DECAY; |
| brightnessVar = brightnessVar * (1 - DECAY) + (brightness - brightnessMean) * |
| (brightness - brightnessMean) * DECAY; |
| if (brightnessVar < LOW_VARIANCE) { |
| return BIG_SCORE_INC; |
| } else if (brightness < brightnessMean && brightnessVar > MEDIUM_VARIANCE) { |
| return -BIG_SCORE_INC; |
| } else if (brightness < brightnessMean) { |
| return -SMALL_SCORE_INC; |
| } else if (brightness > brightnessMean && brightnessVar > 100) { |
| return 0; |
| } else if (brightness > brightnessMean && brightnessVar > MEDIUM_VARIANCE) { |
| return SMALL_SCORE_INC; |
| } else { |
| return BIG_SCORE_INC; // low variance, brightness above the mean |
| } |
| } |
| } |
| } |