| /* |
| * Copyright (C) 2011 The Android Open Source Project |
| * |
| * Licensed under the Apache License, Version 2.0 (the "License"); |
| * you may not use this file except in compliance with the License. |
| * You may obtain a copy of the License at |
| * |
| * http://www.apache.org/licenses/LICENSE-2.0 |
| * |
| * Unless required by applicable law or agreed to in writing, software |
| * distributed under the License is distributed on an "AS IS" BASIS, |
| * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| * See the License for the specific language governing permissions and |
| * limitations under the License. |
| */ |
| |
| package android.filterpacks.videoproc; |
| |
| import android.filterfw.core.Filter; |
| import android.filterfw.core.FilterContext; |
| import android.filterfw.core.GenerateFieldPort; |
| import android.filterfw.core.GenerateFinalPort; |
| import android.filterfw.core.Frame; |
| import android.filterfw.core.GLFrame; |
| import android.filterfw.core.FrameFormat; |
| import android.filterfw.core.MutableFrameFormat; |
| import android.filterfw.core.ShaderProgram; |
| import android.filterfw.format.ImageFormat; |
| import android.opengl.GLES20; |
| import android.os.SystemClock; |
| import android.os.SystemProperties; |
| import android.util.Log; |
| |
| import java.lang.Math; |
| import java.util.Arrays; |
| import java.nio.ByteBuffer; |
| |
| /** |
| * @hide |
| */ |
| public class BackDropperFilter extends Filter { |
| /** User-visible parameters */ |
| |
| private final int BACKGROUND_STRETCH = 0; |
| private final int BACKGROUND_FIT = 1; |
| private final int BACKGROUND_FILL_CROP = 2; |
| |
| @GenerateFieldPort(name = "backgroundFitMode", hasDefault = true) |
| private int mBackgroundFitMode = BACKGROUND_FILL_CROP; |
| @GenerateFieldPort(name = "learningDuration", hasDefault = true) |
| private int mLearningDuration = DEFAULT_LEARNING_DURATION; |
| @GenerateFieldPort(name = "learningVerifyDuration", hasDefault = true) |
| private int mLearningVerifyDuration = DEFAULT_LEARNING_VERIFY_DURATION; |
| @GenerateFieldPort(name = "acceptStddev", hasDefault = true) |
| private float mAcceptStddev = DEFAULT_ACCEPT_STDDEV; |
| @GenerateFieldPort(name = "hierLrgScale", hasDefault = true) |
| private float mHierarchyLrgScale = DEFAULT_HIER_LRG_SCALE; |
| @GenerateFieldPort(name = "hierMidScale", hasDefault = true) |
| private float mHierarchyMidScale = DEFAULT_HIER_MID_SCALE; |
| @GenerateFieldPort(name = "hierSmlScale", hasDefault = true) |
| private float mHierarchySmlScale = DEFAULT_HIER_SML_SCALE; |
| |
| // Dimensions of foreground / background mask. Optimum value should take into account only |
| // image contents, NOT dimensions of input video stream. |
| @GenerateFieldPort(name = "maskWidthExp", hasDefault = true) |
| private int mMaskWidthExp = DEFAULT_MASK_WIDTH_EXPONENT; |
| @GenerateFieldPort(name = "maskHeightExp", hasDefault = true) |
| private int mMaskHeightExp = DEFAULT_MASK_HEIGHT_EXPONENT; |
| |
| // Levels at which to compute foreground / background decision. Think of them as are deltas |
| // SUBTRACTED from maskWidthExp and maskHeightExp. |
| @GenerateFieldPort(name = "hierLrgExp", hasDefault = true) |
| private int mHierarchyLrgExp = DEFAULT_HIER_LRG_EXPONENT; |
| @GenerateFieldPort(name = "hierMidExp", hasDefault = true) |
| private int mHierarchyMidExp = DEFAULT_HIER_MID_EXPONENT; |
| @GenerateFieldPort(name = "hierSmlExp", hasDefault = true) |
| private int mHierarchySmlExp = DEFAULT_HIER_SML_EXPONENT; |
| |
| @GenerateFieldPort(name = "lumScale", hasDefault = true) |
| private float mLumScale = DEFAULT_Y_SCALE_FACTOR; |
| @GenerateFieldPort(name = "chromaScale", hasDefault = true) |
| private float mChromaScale = DEFAULT_UV_SCALE_FACTOR; |
| @GenerateFieldPort(name = "maskBg", hasDefault = true) |
| private float mMaskBg = DEFAULT_MASK_BLEND_BG; |
| @GenerateFieldPort(name = "maskFg", hasDefault = true) |
| private float mMaskFg = DEFAULT_MASK_BLEND_FG; |
| @GenerateFieldPort(name = "exposureChange", hasDefault = true) |
| private float mExposureChange = DEFAULT_EXPOSURE_CHANGE; |
| @GenerateFieldPort(name = "whitebalanceredChange", hasDefault = true) |
| private float mWhiteBalanceRedChange = DEFAULT_WHITE_BALANCE_RED_CHANGE; |
| @GenerateFieldPort(name = "whitebalanceblueChange", hasDefault = true) |
| private float mWhiteBalanceBlueChange = DEFAULT_WHITE_BALANCE_BLUE_CHANGE; |
| @GenerateFieldPort(name = "autowbToggle", hasDefault = true) |
| private int mAutoWBToggle = DEFAULT_WHITE_BALANCE_TOGGLE; |
| |
| // TODO: These are not updatable: |
| @GenerateFieldPort(name = "learningAdaptRate", hasDefault = true) |
| private float mAdaptRateLearning = DEFAULT_LEARNING_ADAPT_RATE; |
| @GenerateFieldPort(name = "adaptRateBg", hasDefault = true) |
| private float mAdaptRateBg = DEFAULT_ADAPT_RATE_BG; |
| @GenerateFieldPort(name = "adaptRateFg", hasDefault = true) |
| private float mAdaptRateFg = DEFAULT_ADAPT_RATE_FG; |
| @GenerateFieldPort(name = "maskVerifyRate", hasDefault = true) |
| private float mVerifyRate = DEFAULT_MASK_VERIFY_RATE; |
| @GenerateFieldPort(name = "learningDoneListener", hasDefault = true) |
| private LearningDoneListener mLearningDoneListener = null; |
| |
| @GenerateFieldPort(name = "useTheForce", hasDefault = true) |
| private boolean mUseTheForce = false; |
| |
| @GenerateFinalPort(name = "provideDebugOutputs", hasDefault = true) |
| private boolean mProvideDebugOutputs = false; |
| |
| // Whether to mirror the background or not. For ex, the Camera app |
| // would mirror the preview for the front camera |
| @GenerateFieldPort(name = "mirrorBg", hasDefault = true) |
| private boolean mMirrorBg = false; |
| |
| // The orientation of the display. This will change the flipping |
| // coordinates, if we were to mirror the background |
| @GenerateFieldPort(name = "orientation", hasDefault = true) |
| private int mOrientation = 0; |
| |
| /** Default algorithm parameter values, for non-shader use */ |
| |
| // Frame count for learning bg model |
| private static final int DEFAULT_LEARNING_DURATION = 40; |
| // Frame count for learning verification |
| private static final int DEFAULT_LEARNING_VERIFY_DURATION = 10; |
| // Maximum distance (in standard deviations) for considering a pixel as background |
| private static final float DEFAULT_ACCEPT_STDDEV = 0.85f; |
| // Variance threshold scale factor for large scale of hierarchy |
| private static final float DEFAULT_HIER_LRG_SCALE = 0.7f; |
| // Variance threshold scale factor for medium scale of hierarchy |
| private static final float DEFAULT_HIER_MID_SCALE = 0.6f; |
| // Variance threshold scale factor for small scale of hierarchy |
| private static final float DEFAULT_HIER_SML_SCALE = 0.5f; |
| // Width of foreground / background mask. |
| private static final int DEFAULT_MASK_WIDTH_EXPONENT = 8; |
| // Height of foreground / background mask. |
| private static final int DEFAULT_MASK_HEIGHT_EXPONENT = 8; |
| // Area over which to average for large scale (length in pixels = 2^HIERARCHY_*_EXPONENT) |
| private static final int DEFAULT_HIER_LRG_EXPONENT = 3; |
| // Area over which to average for medium scale |
| private static final int DEFAULT_HIER_MID_EXPONENT = 2; |
| // Area over which to average for small scale |
| private static final int DEFAULT_HIER_SML_EXPONENT = 0; |
| // Scale factor for luminance channel in distance calculations (larger = more significant) |
| private static final float DEFAULT_Y_SCALE_FACTOR = 0.40f; |
| // Scale factor for chroma channels in distance calculations |
| private static final float DEFAULT_UV_SCALE_FACTOR = 1.35f; |
| // Mask value to start blending away from background |
| private static final float DEFAULT_MASK_BLEND_BG = 0.65f; |
| // Mask value to start blending away from foreground |
| private static final float DEFAULT_MASK_BLEND_FG = 0.95f; |
| // Exposure stop number to change the brightness of foreground |
| private static final float DEFAULT_EXPOSURE_CHANGE = 1.0f; |
| // White balance change in Red channel for foreground |
| private static final float DEFAULT_WHITE_BALANCE_RED_CHANGE = 0.0f; |
| // White balance change in Blue channel for foreground |
| private static final float DEFAULT_WHITE_BALANCE_BLUE_CHANGE = 0.0f; |
| // Variable to control automatic white balance effect |
| // 0.f -> Auto WB is off; 1.f-> Auto WB is on |
| private static final int DEFAULT_WHITE_BALANCE_TOGGLE = 0; |
| |
| // Default rate at which to learn bg model during learning period |
| private static final float DEFAULT_LEARNING_ADAPT_RATE = 0.2f; |
| // Default rate at which to learn bg model from new background pixels |
| private static final float DEFAULT_ADAPT_RATE_BG = 0.0f; |
| // Default rate at which to learn bg model from new foreground pixels |
| private static final float DEFAULT_ADAPT_RATE_FG = 0.0f; |
| // Default rate at which to verify whether background is stable |
| private static final float DEFAULT_MASK_VERIFY_RATE = 0.25f; |
| // Default rate at which to verify whether background is stable |
| private static final int DEFAULT_LEARNING_DONE_THRESHOLD = 20; |
| |
| // Default 3x3 matrix, column major, for fitting background 1:1 |
| private static final float[] DEFAULT_BG_FIT_TRANSFORM = new float[] { |
| 1.0f, 0.0f, 0.0f, |
| 0.0f, 1.0f, 0.0f, |
| 0.0f, 0.0f, 1.0f |
| }; |
| |
| /** Default algorithm parameter values, for shader use */ |
| |
| // Area over which to blur binary mask values (length in pixels = 2^MASK_SMOOTH_EXPONENT) |
| private static final String MASK_SMOOTH_EXPONENT = "2.0"; |
| // Scale value for mapping variance distance to fit nicely to 0-1, 8-bit |
| private static final String DISTANCE_STORAGE_SCALE = "0.6"; |
| // Scale value for mapping variance to fit nicely to 0-1, 8-bit |
| private static final String VARIANCE_STORAGE_SCALE = "5.0"; |
| // Default scale of auto white balance parameters |
| private static final String DEFAULT_AUTO_WB_SCALE = "0.25"; |
| // Minimum variance (0-255 scale) |
| private static final String MIN_VARIANCE = "3.0"; |
| // Column-major array for 4x4 matrix converting RGB to YCbCr, JPEG definition (no pedestal) |
| private static final String RGB_TO_YUV_MATRIX = "0.299, -0.168736, 0.5, 0.000, " + |
| "0.587, -0.331264, -0.418688, 0.000, " + |
| "0.114, 0.5, -0.081312, 0.000, " + |
| "0.000, 0.5, 0.5, 1.000 "; |
| /** Stream names */ |
| |
| private static final String[] mInputNames = {"video", |
| "background"}; |
| |
| private static final String[] mOutputNames = {"video"}; |
| |
| private static final String[] mDebugOutputNames = {"debug1", |
| "debug2"}; |
| |
| /** Other private variables */ |
| |
| private FrameFormat mOutputFormat; |
| private MutableFrameFormat mMemoryFormat; |
| private MutableFrameFormat mMaskFormat; |
| private MutableFrameFormat mAverageFormat; |
| |
| private final boolean mLogVerbose; |
| private static final String TAG = "BackDropperFilter"; |
| |
| /** Shader source code */ |
| |
| // Shared uniforms and utility functions |
| private static String mSharedUtilShader = |
| "precision mediump float;\n" + |
| "uniform float fg_adapt_rate;\n" + |
| "uniform float bg_adapt_rate;\n" + |
| "const mat4 coeff_yuv = mat4(" + RGB_TO_YUV_MATRIX + ");\n" + |
| "const float dist_scale = " + DISTANCE_STORAGE_SCALE + ";\n" + |
| "const float inv_dist_scale = 1. / dist_scale;\n" + |
| "const float var_scale=" + VARIANCE_STORAGE_SCALE + ";\n" + |
| "const float inv_var_scale = 1. / var_scale;\n" + |
| "const float min_variance = inv_var_scale *" + MIN_VARIANCE + "/ 256.;\n" + |
| "const float auto_wb_scale = " + DEFAULT_AUTO_WB_SCALE + ";\n" + |
| "\n" + |
| // Variance distance in luminance between current pixel and background model |
| "float gauss_dist_y(float y, float mean, float variance) {\n" + |
| " float dist = (y - mean) * (y - mean) / variance;\n" + |
| " return dist;\n" + |
| "}\n" + |
| // Sum of variance distances in chroma between current pixel and background |
| // model |
| "float gauss_dist_uv(vec2 uv, vec2 mean, vec2 variance) {\n" + |
| " vec2 dist = (uv - mean) * (uv - mean) / variance;\n" + |
| " return dist.r + dist.g;\n" + |
| "}\n" + |
| // Select learning rate for pixel based on smoothed decision mask alpha |
| "float local_adapt_rate(float alpha) {\n" + |
| " return mix(bg_adapt_rate, fg_adapt_rate, alpha);\n" + |
| "}\n" + |
| "\n"; |
| |
| // Distance calculation shader. Calculates a distance metric between the foreground and the |
| // current background model, in both luminance and in chroma (yuv space). Distance is |
| // measured in variances from the mean background value. For chroma, the distance is the sum |
| // of the two individual color channel distances. The distances are output on the b and alpha |
| // channels, r and g are for debug information. |
| // Inputs: |
| // tex_sampler_0: Mip-map for foreground (live) video frame. |
| // tex_sampler_1: Background mean mask. |
| // tex_sampler_2: Background variance mask. |
| // subsample_level: Level on foreground frame's mip-map. |
| private static final String mBgDistanceShader = |
| "uniform sampler2D tex_sampler_0;\n" + |
| "uniform sampler2D tex_sampler_1;\n" + |
| "uniform sampler2D tex_sampler_2;\n" + |
| "uniform float subsample_level;\n" + |
| "varying vec2 v_texcoord;\n" + |
| "void main() {\n" + |
| " vec4 fg_rgb = texture2D(tex_sampler_0, v_texcoord, subsample_level);\n" + |
| " vec4 fg = coeff_yuv * vec4(fg_rgb.rgb, 1.);\n" + |
| " vec4 mean = texture2D(tex_sampler_1, v_texcoord);\n" + |
| " vec4 variance = inv_var_scale * texture2D(tex_sampler_2, v_texcoord);\n" + |
| "\n" + |
| " float dist_y = gauss_dist_y(fg.r, mean.r, variance.r);\n" + |
| " float dist_uv = gauss_dist_uv(fg.gb, mean.gb, variance.gb);\n" + |
| " gl_FragColor = vec4(0.5*fg.rg, dist_scale*dist_y, dist_scale*dist_uv);\n" + |
| "}\n"; |
| |
| // Foreground/background mask decision shader. Decides whether a frame is in the foreground or |
| // the background using a hierarchical threshold on the distance. Binary foreground/background |
| // mask is placed in the alpha channel. The RGB channels contain debug information. |
| private static final String mBgMaskShader = |
| "uniform sampler2D tex_sampler_0;\n" + |
| "uniform float accept_variance;\n" + |
| "uniform vec2 yuv_weights;\n" + |
| "uniform float scale_lrg;\n" + |
| "uniform float scale_mid;\n" + |
| "uniform float scale_sml;\n" + |
| "uniform float exp_lrg;\n" + |
| "uniform float exp_mid;\n" + |
| "uniform float exp_sml;\n" + |
| "varying vec2 v_texcoord;\n" + |
| // Decide whether pixel is foreground or background based on Y and UV |
| // distance and maximum acceptable variance. |
| // yuv_weights.x is smaller than yuv_weights.y to discount the influence of shadow |
| "bool is_fg(vec2 dist_yc, float accept_variance) {\n" + |
| " return ( dot(yuv_weights, dist_yc) >= accept_variance );\n" + |
| "}\n" + |
| "void main() {\n" + |
| " vec4 dist_lrg_sc = texture2D(tex_sampler_0, v_texcoord, exp_lrg);\n" + |
| " vec4 dist_mid_sc = texture2D(tex_sampler_0, v_texcoord, exp_mid);\n" + |
| " vec4 dist_sml_sc = texture2D(tex_sampler_0, v_texcoord, exp_sml);\n" + |
| " vec2 dist_lrg = inv_dist_scale * dist_lrg_sc.ba;\n" + |
| " vec2 dist_mid = inv_dist_scale * dist_mid_sc.ba;\n" + |
| " vec2 dist_sml = inv_dist_scale * dist_sml_sc.ba;\n" + |
| " vec2 norm_dist = 0.75 * dist_sml / accept_variance;\n" + // For debug viz |
| " bool is_fg_lrg = is_fg(dist_lrg, accept_variance * scale_lrg);\n" + |
| " bool is_fg_mid = is_fg_lrg || is_fg(dist_mid, accept_variance * scale_mid);\n" + |
| " float is_fg_sml =\n" + |
| " float(is_fg_mid || is_fg(dist_sml, accept_variance * scale_sml));\n" + |
| " float alpha = 0.5 * is_fg_sml + 0.3 * float(is_fg_mid) + 0.2 * float(is_fg_lrg);\n" + |
| " gl_FragColor = vec4(alpha, norm_dist, is_fg_sml);\n" + |
| "}\n"; |
| |
| // Automatic White Balance parameter decision shader |
| // Use the Gray World assumption that in a white balance corrected image, the average of R, G, B |
| // channel will be a common gray value. |
| // To match the white balance of foreground and background, the average of R, G, B channel of |
| // two videos should match. |
| // Inputs: |
| // tex_sampler_0: Mip-map for foreground (live) video frame. |
| // tex_sampler_1: Mip-map for background (playback) video frame. |
| // pyramid_depth: Depth of input frames' mip-maps. |
| private static final String mAutomaticWhiteBalance = |
| "uniform sampler2D tex_sampler_0;\n" + |
| "uniform sampler2D tex_sampler_1;\n" + |
| "uniform float pyramid_depth;\n" + |
| "uniform bool autowb_toggle;\n" + |
| "varying vec2 v_texcoord;\n" + |
| "void main() {\n" + |
| " vec4 mean_video = texture2D(tex_sampler_0, v_texcoord, pyramid_depth);\n"+ |
| " vec4 mean_bg = texture2D(tex_sampler_1, v_texcoord, pyramid_depth);\n" + |
| // If Auto WB is toggled off, the return texture will be a unicolor texture of value 1 |
| // If Auto WB is toggled on, the return texture will be a unicolor texture with |
| // adjustment parameters for R and B channels stored in the corresponding channel |
| " float green_normalizer = mean_video.g / mean_bg.g;\n"+ |
| " vec4 adjusted_value = vec4(mean_bg.r / mean_video.r * green_normalizer, 1., \n" + |
| " mean_bg.b / mean_video.b * green_normalizer, 1.) * auto_wb_scale; \n" + |
| " gl_FragColor = autowb_toggle ? adjusted_value : vec4(auto_wb_scale);\n" + |
| "}\n"; |
| |
| |
| // Background subtraction shader. Uses a mipmap of the binary mask map to blend smoothly between |
| // foreground and background |
| // Inputs: |
| // tex_sampler_0: Foreground (live) video frame. |
| // tex_sampler_1: Background (playback) video frame. |
| // tex_sampler_2: Foreground/background mask. |
| // tex_sampler_3: Auto white-balance factors. |
| private static final String mBgSubtractShader = |
| "uniform mat3 bg_fit_transform;\n" + |
| "uniform float mask_blend_bg;\n" + |
| "uniform float mask_blend_fg;\n" + |
| "uniform float exposure_change;\n" + |
| "uniform float whitebalancered_change;\n" + |
| "uniform float whitebalanceblue_change;\n" + |
| "uniform sampler2D tex_sampler_0;\n" + |
| "uniform sampler2D tex_sampler_1;\n" + |
| "uniform sampler2D tex_sampler_2;\n" + |
| "uniform sampler2D tex_sampler_3;\n" + |
| "varying vec2 v_texcoord;\n" + |
| "void main() {\n" + |
| " vec2 bg_texcoord = (bg_fit_transform * vec3(v_texcoord, 1.)).xy;\n" + |
| " vec4 bg_rgb = texture2D(tex_sampler_1, bg_texcoord);\n" + |
| // The foreground texture is modified by multiplying both manual and auto white balance changes in R and B |
| // channel and multiplying exposure change in all R, G, B channels. |
| " vec4 wb_auto_scale = texture2D(tex_sampler_3, v_texcoord) * exposure_change / auto_wb_scale;\n" + |
| " vec4 wb_manual_scale = vec4(1. + whitebalancered_change, 1., 1. + whitebalanceblue_change, 1.);\n" + |
| " vec4 fg_rgb = texture2D(tex_sampler_0, v_texcoord);\n" + |
| " vec4 fg_adjusted = fg_rgb * wb_manual_scale * wb_auto_scale;\n"+ |
| " vec4 mask = texture2D(tex_sampler_2, v_texcoord, \n" + |
| " " + MASK_SMOOTH_EXPONENT + ");\n" + |
| " float alpha = smoothstep(mask_blend_bg, mask_blend_fg, mask.a);\n" + |
| " gl_FragColor = mix(bg_rgb, fg_adjusted, alpha);\n"; |
| |
| // May the Force... Makes the foreground object translucent blue, with a bright |
| // blue-white outline |
| private static final String mBgSubtractForceShader = |
| " vec4 ghost_rgb = (fg_adjusted * 0.7 + vec4(0.3,0.3,0.4,0.))*0.65 + \n" + |
| " 0.35*bg_rgb;\n" + |
| " float glow_start = 0.75 * mask_blend_bg; \n"+ |
| " float glow_max = mask_blend_bg; \n"+ |
| " gl_FragColor = mask.a < glow_start ? bg_rgb : \n" + |
| " mask.a < glow_max ? mix(bg_rgb, vec4(0.9,0.9,1.0,1.0), \n" + |
| " (mask.a - glow_start) / (glow_max - glow_start) ) : \n" + |
| " mask.a < mask_blend_fg ? mix(vec4(0.9,0.9,1.0,1.0), ghost_rgb, \n" + |
| " (mask.a - glow_max) / (mask_blend_fg - glow_max) ) : \n" + |
| " ghost_rgb;\n" + |
| "}\n"; |
| |
| // Background model mean update shader. Skews the current model mean toward the most recent pixel |
| // value for a pixel, weighted by the learning rate and by whether the pixel is classified as |
| // foreground or background. |
| // Inputs: |
| // tex_sampler_0: Mip-map for foreground (live) video frame. |
| // tex_sampler_1: Background mean mask. |
| // tex_sampler_2: Foreground/background mask. |
| // subsample_level: Level on foreground frame's mip-map. |
| private static final String mUpdateBgModelMeanShader = |
| "uniform sampler2D tex_sampler_0;\n" + |
| "uniform sampler2D tex_sampler_1;\n" + |
| "uniform sampler2D tex_sampler_2;\n" + |
| "uniform float subsample_level;\n" + |
| "varying vec2 v_texcoord;\n" + |
| "void main() {\n" + |
| " vec4 fg_rgb = texture2D(tex_sampler_0, v_texcoord, subsample_level);\n" + |
| " vec4 fg = coeff_yuv * vec4(fg_rgb.rgb, 1.);\n" + |
| " vec4 mean = texture2D(tex_sampler_1, v_texcoord);\n" + |
| " vec4 mask = texture2D(tex_sampler_2, v_texcoord, \n" + |
| " " + MASK_SMOOTH_EXPONENT + ");\n" + |
| "\n" + |
| " float alpha = local_adapt_rate(mask.a);\n" + |
| " vec4 new_mean = mix(mean, fg, alpha);\n" + |
| " gl_FragColor = new_mean;\n" + |
| "}\n"; |
| |
| // Background model variance update shader. Skews the current model variance toward the most |
| // recent variance for the pixel, weighted by the learning rate and by whether the pixel is |
| // classified as foreground or background. |
| // Inputs: |
| // tex_sampler_0: Mip-map for foreground (live) video frame. |
| // tex_sampler_1: Background mean mask. |
| // tex_sampler_2: Background variance mask. |
| // tex_sampler_3: Foreground/background mask. |
| // subsample_level: Level on foreground frame's mip-map. |
| // TODO: to improve efficiency, use single mark for mean + variance, then merge this into |
| // mUpdateBgModelMeanShader. |
| private static final String mUpdateBgModelVarianceShader = |
| "uniform sampler2D tex_sampler_0;\n" + |
| "uniform sampler2D tex_sampler_1;\n" + |
| "uniform sampler2D tex_sampler_2;\n" + |
| "uniform sampler2D tex_sampler_3;\n" + |
| "uniform float subsample_level;\n" + |
| "varying vec2 v_texcoord;\n" + |
| "void main() {\n" + |
| " vec4 fg_rgb = texture2D(tex_sampler_0, v_texcoord, subsample_level);\n" + |
| " vec4 fg = coeff_yuv * vec4(fg_rgb.rgb, 1.);\n" + |
| " vec4 mean = texture2D(tex_sampler_1, v_texcoord);\n" + |
| " vec4 variance = inv_var_scale * texture2D(tex_sampler_2, v_texcoord);\n" + |
| " vec4 mask = texture2D(tex_sampler_3, v_texcoord, \n" + |
| " " + MASK_SMOOTH_EXPONENT + ");\n" + |
| "\n" + |
| " float alpha = local_adapt_rate(mask.a);\n" + |
| " vec4 cur_variance = (fg-mean)*(fg-mean);\n" + |
| " vec4 new_variance = mix(variance, cur_variance, alpha);\n" + |
| " new_variance = max(new_variance, vec4(min_variance));\n" + |
| " gl_FragColor = var_scale * new_variance;\n" + |
| "}\n"; |
| |
| // Background verification shader. Skews the current background verification mask towards the |
| // most recent frame, weighted by the learning rate. |
| private static final String mMaskVerifyShader = |
| "uniform sampler2D tex_sampler_0;\n" + |
| "uniform sampler2D tex_sampler_1;\n" + |
| "uniform float verify_rate;\n" + |
| "varying vec2 v_texcoord;\n" + |
| "void main() {\n" + |
| " vec4 lastmask = texture2D(tex_sampler_0, v_texcoord);\n" + |
| " vec4 mask = texture2D(tex_sampler_1, v_texcoord);\n" + |
| " float newmask = mix(lastmask.a, mask.a, verify_rate);\n" + |
| " gl_FragColor = vec4(0., 0., 0., newmask);\n" + |
| "}\n"; |
| |
| /** Shader program objects */ |
| |
| private ShaderProgram mBgDistProgram; |
| private ShaderProgram mBgMaskProgram; |
| private ShaderProgram mBgSubtractProgram; |
| private ShaderProgram mBgUpdateMeanProgram; |
| private ShaderProgram mBgUpdateVarianceProgram; |
| private ShaderProgram mCopyOutProgram; |
| private ShaderProgram mAutomaticWhiteBalanceProgram; |
| private ShaderProgram mMaskVerifyProgram; |
| private ShaderProgram copyShaderProgram; |
| |
| /** Background model storage */ |
| |
| private boolean mPingPong; |
| private GLFrame mBgMean[]; |
| private GLFrame mBgVariance[]; |
| private GLFrame mMaskVerify[]; |
| private GLFrame mDistance; |
| private GLFrame mAutoWB; |
| private GLFrame mMask; |
| private GLFrame mVideoInput; |
| private GLFrame mBgInput; |
| private GLFrame mMaskAverage; |
| |
| /** Overall filter state */ |
| |
| private boolean isOpen; |
| private int mFrameCount; |
| private boolean mStartLearning; |
| private boolean mBackgroundFitModeChanged; |
| private float mRelativeAspect; |
| private int mPyramidDepth; |
| private int mSubsampleLevel; |
| |
| /** Learning listener object */ |
| |
| public interface LearningDoneListener { |
| public void onLearningDone(BackDropperFilter filter); |
| } |
| |
| /** Public Filter methods */ |
| |
| public BackDropperFilter(String name) { |
| super(name); |
| |
| mLogVerbose = Log.isLoggable(TAG, Log.VERBOSE); |
| |
| String adjStr = SystemProperties.get("ro.media.effect.bgdropper.adj"); |
| if (adjStr.length() > 0) { |
| try { |
| mAcceptStddev += Float.parseFloat(adjStr); |
| if (mLogVerbose) { |
| Log.v(TAG, "Adjusting accept threshold by " + adjStr + |
| ", now " + mAcceptStddev); |
| } |
| } catch (NumberFormatException e) { |
| Log.e(TAG, |
| "Badly formatted property ro.media.effect.bgdropper.adj: " + adjStr); |
| } |
| } |
| } |
| |
| @Override |
| public void setupPorts() { |
| // Inputs. |
| // TODO: Target should be GPU, but relaxed for now. |
| FrameFormat imageFormat = ImageFormat.create(ImageFormat.COLORSPACE_RGBA, |
| FrameFormat.TARGET_UNSPECIFIED); |
| for (String inputName : mInputNames) { |
| addMaskedInputPort(inputName, imageFormat); |
| } |
| // Normal outputs |
| for (String outputName : mOutputNames) { |
| addOutputBasedOnInput(outputName, "video"); |
| } |
| |
| // Debug outputs |
| if (mProvideDebugOutputs) { |
| for (String outputName : mDebugOutputNames) { |
| addOutputBasedOnInput(outputName, "video"); |
| } |
| } |
| } |
| |
| @Override |
| public FrameFormat getOutputFormat(String portName, FrameFormat inputFormat) { |
| // Create memory format based on video input. |
| MutableFrameFormat format = inputFormat.mutableCopy(); |
| // Is this a debug output port? If so, leave dimensions unspecified. |
| if (!Arrays.asList(mOutputNames).contains(portName)) { |
| format.setDimensions(FrameFormat.SIZE_UNSPECIFIED, FrameFormat.SIZE_UNSPECIFIED); |
| } |
| return format; |
| } |
| |
| private boolean createMemoryFormat(FrameFormat inputFormat) { |
| // We can't resize because that would require re-learning. |
| if (mMemoryFormat != null) { |
| return false; |
| } |
| |
| if (inputFormat.getWidth() == FrameFormat.SIZE_UNSPECIFIED || |
| inputFormat.getHeight() == FrameFormat.SIZE_UNSPECIFIED) { |
| throw new RuntimeException("Attempting to process input frame with unknown size"); |
| } |
| |
| mMaskFormat = inputFormat.mutableCopy(); |
| int maskWidth = (int)Math.pow(2, mMaskWidthExp); |
| int maskHeight = (int)Math.pow(2, mMaskHeightExp); |
| mMaskFormat.setDimensions(maskWidth, maskHeight); |
| |
| mPyramidDepth = Math.max(mMaskWidthExp, mMaskHeightExp); |
| mMemoryFormat = mMaskFormat.mutableCopy(); |
| int widthExp = Math.max(mMaskWidthExp, pyramidLevel(inputFormat.getWidth())); |
| int heightExp = Math.max(mMaskHeightExp, pyramidLevel(inputFormat.getHeight())); |
| mPyramidDepth = Math.max(widthExp, heightExp); |
| int memWidth = Math.max(maskWidth, (int)Math.pow(2, widthExp)); |
| int memHeight = Math.max(maskHeight, (int)Math.pow(2, heightExp)); |
| mMemoryFormat.setDimensions(memWidth, memHeight); |
| mSubsampleLevel = mPyramidDepth - Math.max(mMaskWidthExp, mMaskHeightExp); |
| |
| if (mLogVerbose) { |
| Log.v(TAG, "Mask frames size " + maskWidth + " x " + maskHeight); |
| Log.v(TAG, "Pyramid levels " + widthExp + " x " + heightExp); |
| Log.v(TAG, "Memory frames size " + memWidth + " x " + memHeight); |
| } |
| |
| mAverageFormat = inputFormat.mutableCopy(); |
| mAverageFormat.setDimensions(1,1); |
| return true; |
| } |
| |
| public void prepare(FilterContext context){ |
| if (mLogVerbose) Log.v(TAG, "Preparing BackDropperFilter!"); |
| |
| mBgMean = new GLFrame[2]; |
| mBgVariance = new GLFrame[2]; |
| mMaskVerify = new GLFrame[2]; |
| copyShaderProgram = ShaderProgram.createIdentity(context); |
| } |
| |
| private void allocateFrames(FrameFormat inputFormat, FilterContext context) { |
| if (!createMemoryFormat(inputFormat)) { |
| return; // All set. |
| } |
| if (mLogVerbose) Log.v(TAG, "Allocating BackDropperFilter frames"); |
| |
| // Create initial background model values |
| int numBytes = mMaskFormat.getSize(); |
| byte[] initialBgMean = new byte[numBytes]; |
| byte[] initialBgVariance = new byte[numBytes]; |
| byte[] initialMaskVerify = new byte[numBytes]; |
| for (int i = 0; i < numBytes; i++) { |
| initialBgMean[i] = (byte)128; |
| initialBgVariance[i] = (byte)10; |
| initialMaskVerify[i] = (byte)0; |
| } |
| |
| // Get frames to store background model in |
| for (int i = 0; i < 2; i++) { |
| mBgMean[i] = (GLFrame)context.getFrameManager().newFrame(mMaskFormat); |
| mBgMean[i].setData(initialBgMean, 0, numBytes); |
| |
| mBgVariance[i] = (GLFrame)context.getFrameManager().newFrame(mMaskFormat); |
| mBgVariance[i].setData(initialBgVariance, 0, numBytes); |
| |
| mMaskVerify[i] = (GLFrame)context.getFrameManager().newFrame(mMaskFormat); |
| mMaskVerify[i].setData(initialMaskVerify, 0, numBytes); |
| } |
| |
| // Get frames to store other textures in |
| if (mLogVerbose) Log.v(TAG, "Done allocating texture for Mean and Variance objects!"); |
| |
| mDistance = (GLFrame)context.getFrameManager().newFrame(mMaskFormat); |
| mMask = (GLFrame)context.getFrameManager().newFrame(mMaskFormat); |
| mAutoWB = (GLFrame)context.getFrameManager().newFrame(mAverageFormat); |
| mVideoInput = (GLFrame)context.getFrameManager().newFrame(mMemoryFormat); |
| mBgInput = (GLFrame)context.getFrameManager().newFrame(mMemoryFormat); |
| mMaskAverage = (GLFrame)context.getFrameManager().newFrame(mAverageFormat); |
| |
| // Create shader programs |
| mBgDistProgram = new ShaderProgram(context, mSharedUtilShader + mBgDistanceShader); |
| mBgDistProgram.setHostValue("subsample_level", (float)mSubsampleLevel); |
| |
| mBgMaskProgram = new ShaderProgram(context, mSharedUtilShader + mBgMaskShader); |
| mBgMaskProgram.setHostValue("accept_variance", mAcceptStddev * mAcceptStddev); |
| float[] yuvWeights = { mLumScale, mChromaScale }; |
| mBgMaskProgram.setHostValue("yuv_weights", yuvWeights ); |
| mBgMaskProgram.setHostValue("scale_lrg", mHierarchyLrgScale); |
| mBgMaskProgram.setHostValue("scale_mid", mHierarchyMidScale); |
| mBgMaskProgram.setHostValue("scale_sml", mHierarchySmlScale); |
| mBgMaskProgram.setHostValue("exp_lrg", (float)(mSubsampleLevel + mHierarchyLrgExp)); |
| mBgMaskProgram.setHostValue("exp_mid", (float)(mSubsampleLevel + mHierarchyMidExp)); |
| mBgMaskProgram.setHostValue("exp_sml", (float)(mSubsampleLevel + mHierarchySmlExp)); |
| |
| if (mUseTheForce) { |
| mBgSubtractProgram = new ShaderProgram(context, mSharedUtilShader + mBgSubtractShader + mBgSubtractForceShader); |
| } else { |
| mBgSubtractProgram = new ShaderProgram(context, mSharedUtilShader + mBgSubtractShader + "}\n"); |
| } |
| mBgSubtractProgram.setHostValue("bg_fit_transform", DEFAULT_BG_FIT_TRANSFORM); |
| mBgSubtractProgram.setHostValue("mask_blend_bg", mMaskBg); |
| mBgSubtractProgram.setHostValue("mask_blend_fg", mMaskFg); |
| mBgSubtractProgram.setHostValue("exposure_change", mExposureChange); |
| mBgSubtractProgram.setHostValue("whitebalanceblue_change", mWhiteBalanceBlueChange); |
| mBgSubtractProgram.setHostValue("whitebalancered_change", mWhiteBalanceRedChange); |
| |
| |
| mBgUpdateMeanProgram = new ShaderProgram(context, mSharedUtilShader + mUpdateBgModelMeanShader); |
| mBgUpdateMeanProgram.setHostValue("subsample_level", (float)mSubsampleLevel); |
| |
| mBgUpdateVarianceProgram = new ShaderProgram(context, mSharedUtilShader + mUpdateBgModelVarianceShader); |
| mBgUpdateVarianceProgram.setHostValue("subsample_level", (float)mSubsampleLevel); |
| |
| mCopyOutProgram = ShaderProgram.createIdentity(context); |
| |
| mAutomaticWhiteBalanceProgram = new ShaderProgram(context, mSharedUtilShader + mAutomaticWhiteBalance); |
| mAutomaticWhiteBalanceProgram.setHostValue("pyramid_depth", (float)mPyramidDepth); |
| mAutomaticWhiteBalanceProgram.setHostValue("autowb_toggle", mAutoWBToggle); |
| |
| mMaskVerifyProgram = new ShaderProgram(context, mSharedUtilShader + mMaskVerifyShader); |
| mMaskVerifyProgram.setHostValue("verify_rate", mVerifyRate); |
| |
| if (mLogVerbose) Log.v(TAG, "Shader width set to " + mMemoryFormat.getWidth()); |
| |
| mRelativeAspect = 1.f; |
| |
| mFrameCount = 0; |
| mStartLearning = true; |
| } |
| |
| public void process(FilterContext context) { |
| // Grab inputs and ready intermediate frames and outputs. |
| Frame video = pullInput("video"); |
| Frame background = pullInput("background"); |
| allocateFrames(video.getFormat(), context); |
| |
| // Update learning rate after initial learning period |
| if (mStartLearning) { |
| if (mLogVerbose) Log.v(TAG, "Starting learning"); |
| mBgUpdateMeanProgram.setHostValue("bg_adapt_rate", mAdaptRateLearning); |
| mBgUpdateMeanProgram.setHostValue("fg_adapt_rate", mAdaptRateLearning); |
| mBgUpdateVarianceProgram.setHostValue("bg_adapt_rate", mAdaptRateLearning); |
| mBgUpdateVarianceProgram.setHostValue("fg_adapt_rate", mAdaptRateLearning); |
| mFrameCount = 0; |
| } |
| |
| // Select correct pingpong buffers |
| int inputIndex = mPingPong ? 0 : 1; |
| int outputIndex = mPingPong ? 1 : 0; |
| mPingPong = !mPingPong; |
| |
| // Check relative aspect ratios |
| updateBgScaling(video, background, mBackgroundFitModeChanged); |
| mBackgroundFitModeChanged = false; |
| |
| // Make copies for input frames to GLFrames |
| |
| copyShaderProgram.process(video, mVideoInput); |
| copyShaderProgram.process(background, mBgInput); |
| |
| mVideoInput.generateMipMap(); |
| mVideoInput.setTextureParameter(GLES20.GL_TEXTURE_MIN_FILTER, |
| GLES20.GL_LINEAR_MIPMAP_NEAREST); |
| |
| mBgInput.generateMipMap(); |
| mBgInput.setTextureParameter(GLES20.GL_TEXTURE_MIN_FILTER, |
| GLES20.GL_LINEAR_MIPMAP_NEAREST); |
| |
| if (mStartLearning) { |
| copyShaderProgram.process(mVideoInput, mBgMean[inputIndex]); |
| mStartLearning = false; |
| } |
| |
| // Process shaders |
| Frame[] distInputs = { mVideoInput, mBgMean[inputIndex], mBgVariance[inputIndex] }; |
| mBgDistProgram.process(distInputs, mDistance); |
| mDistance.generateMipMap(); |
| mDistance.setTextureParameter(GLES20.GL_TEXTURE_MIN_FILTER, |
| GLES20.GL_LINEAR_MIPMAP_NEAREST); |
| |
| mBgMaskProgram.process(mDistance, mMask); |
| mMask.generateMipMap(); |
| mMask.setTextureParameter(GLES20.GL_TEXTURE_MIN_FILTER, |
| GLES20.GL_LINEAR_MIPMAP_NEAREST); |
| |
| Frame[] autoWBInputs = { mVideoInput, mBgInput }; |
| mAutomaticWhiteBalanceProgram.process(autoWBInputs, mAutoWB); |
| |
| if (mFrameCount <= mLearningDuration) { |
| // During learning |
| pushOutput("video", video); |
| |
| if (mFrameCount == mLearningDuration - mLearningVerifyDuration) { |
| copyShaderProgram.process(mMask, mMaskVerify[outputIndex]); |
| |
| mBgUpdateMeanProgram.setHostValue("bg_adapt_rate", mAdaptRateBg); |
| mBgUpdateMeanProgram.setHostValue("fg_adapt_rate", mAdaptRateFg); |
| mBgUpdateVarianceProgram.setHostValue("bg_adapt_rate", mAdaptRateBg); |
| mBgUpdateVarianceProgram.setHostValue("fg_adapt_rate", mAdaptRateFg); |
| |
| |
| } else if (mFrameCount > mLearningDuration - mLearningVerifyDuration) { |
| // In the learning verification stage, compute background masks and a weighted average |
| // with weights grow exponentially with time |
| Frame[] maskVerifyInputs = {mMaskVerify[inputIndex], mMask}; |
| mMaskVerifyProgram.process(maskVerifyInputs, mMaskVerify[outputIndex]); |
| mMaskVerify[outputIndex].generateMipMap(); |
| mMaskVerify[outputIndex].setTextureParameter(GLES20.GL_TEXTURE_MIN_FILTER, |
| GLES20.GL_LINEAR_MIPMAP_NEAREST); |
| } |
| |
| if (mFrameCount == mLearningDuration) { |
| // In the last verification frame, verify if the verification mask is almost blank |
| // If not, restart learning |
| copyShaderProgram.process(mMaskVerify[outputIndex], mMaskAverage); |
| ByteBuffer mMaskAverageByteBuffer = mMaskAverage.getData(); |
| byte[] mask_average = mMaskAverageByteBuffer.array(); |
| int bi = (int)(mask_average[3] & 0xFF); |
| |
| if (mLogVerbose) { |
| Log.v(TAG, |
| String.format("Mask_average is %d, threshold is %d", |
| bi, DEFAULT_LEARNING_DONE_THRESHOLD)); |
| } |
| |
| if (bi >= DEFAULT_LEARNING_DONE_THRESHOLD) { |
| mStartLearning = true; // Restart learning |
| } else { |
| if (mLogVerbose) Log.v(TAG, "Learning done"); |
| if (mLearningDoneListener != null) { |
| mLearningDoneListener.onLearningDone(this); |
| } |
| } |
| } |
| } else { |
| Frame output = context.getFrameManager().newFrame(video.getFormat()); |
| Frame[] subtractInputs = { video, background, mMask, mAutoWB }; |
| mBgSubtractProgram.process(subtractInputs, output); |
| pushOutput("video", output); |
| output.release(); |
| } |
| |
| // Compute mean and variance of the background |
| if (mFrameCount < mLearningDuration - mLearningVerifyDuration || |
| mAdaptRateBg > 0.0 || mAdaptRateFg > 0.0) { |
| Frame[] meanUpdateInputs = { mVideoInput, mBgMean[inputIndex], mMask }; |
| mBgUpdateMeanProgram.process(meanUpdateInputs, mBgMean[outputIndex]); |
| mBgMean[outputIndex].generateMipMap(); |
| mBgMean[outputIndex].setTextureParameter(GLES20.GL_TEXTURE_MIN_FILTER, |
| GLES20.GL_LINEAR_MIPMAP_NEAREST); |
| |
| Frame[] varianceUpdateInputs = { |
| mVideoInput, mBgMean[inputIndex], mBgVariance[inputIndex], mMask |
| }; |
| mBgUpdateVarianceProgram.process(varianceUpdateInputs, mBgVariance[outputIndex]); |
| mBgVariance[outputIndex].generateMipMap(); |
| mBgVariance[outputIndex].setTextureParameter(GLES20.GL_TEXTURE_MIN_FILTER, |
| GLES20.GL_LINEAR_MIPMAP_NEAREST); |
| } |
| |
| // Provide debug output to two smaller viewers |
| if (mProvideDebugOutputs) { |
| Frame dbg1 = context.getFrameManager().newFrame(video.getFormat()); |
| mCopyOutProgram.process(video, dbg1); |
| pushOutput("debug1", dbg1); |
| dbg1.release(); |
| |
| Frame dbg2 = context.getFrameManager().newFrame(mMemoryFormat); |
| mCopyOutProgram.process(mMask, dbg2); |
| pushOutput("debug2", dbg2); |
| dbg2.release(); |
| } |
| |
| mFrameCount++; |
| |
| if (mLogVerbose) { |
| if (mFrameCount % 30 == 0) { |
| if (startTime == -1) { |
| context.getGLEnvironment().activate(); |
| GLES20.glFinish(); |
| startTime = SystemClock.elapsedRealtime(); |
| } else { |
| context.getGLEnvironment().activate(); |
| GLES20.glFinish(); |
| long endTime = SystemClock.elapsedRealtime(); |
| Log.v(TAG, "Avg. frame duration: " + String.format("%.2f",(endTime-startTime)/30.) + |
| " ms. Avg. fps: " + String.format("%.2f", 1000./((endTime-startTime)/30.)) ); |
| startTime = endTime; |
| } |
| } |
| } |
| } |
| |
| private long startTime = -1; |
| |
| public void close(FilterContext context) { |
| if (mMemoryFormat == null) { |
| return; |
| } |
| |
| if (mLogVerbose) Log.v(TAG, "Filter Closing!"); |
| for (int i = 0; i < 2; i++) { |
| mBgMean[i].release(); |
| mBgVariance[i].release(); |
| mMaskVerify[i].release(); |
| } |
| mDistance.release(); |
| mMask.release(); |
| mAutoWB.release(); |
| mVideoInput.release(); |
| mBgInput.release(); |
| mMaskAverage.release(); |
| |
| mMemoryFormat = null; |
| } |
| |
| // Relearn background model |
| synchronized public void relearn() { |
| // Let the processing thread know about learning restart |
| mStartLearning = true; |
| } |
| |
| @Override |
| public void fieldPortValueUpdated(String name, FilterContext context) { |
| // TODO: Many of these can be made ProgramPorts! |
| if (name.equals("backgroundFitMode")) { |
| mBackgroundFitModeChanged = true; |
| } else if (name.equals("acceptStddev")) { |
| mBgMaskProgram.setHostValue("accept_variance", mAcceptStddev * mAcceptStddev); |
| } else if (name.equals("hierLrgScale")) { |
| mBgMaskProgram.setHostValue("scale_lrg", mHierarchyLrgScale); |
| } else if (name.equals("hierMidScale")) { |
| mBgMaskProgram.setHostValue("scale_mid", mHierarchyMidScale); |
| } else if (name.equals("hierSmlScale")) { |
| mBgMaskProgram.setHostValue("scale_sml", mHierarchySmlScale); |
| } else if (name.equals("hierLrgExp")) { |
| mBgMaskProgram.setHostValue("exp_lrg", (float)(mSubsampleLevel + mHierarchyLrgExp)); |
| } else if (name.equals("hierMidExp")) { |
| mBgMaskProgram.setHostValue("exp_mid", (float)(mSubsampleLevel + mHierarchyMidExp)); |
| } else if (name.equals("hierSmlExp")) { |
| mBgMaskProgram.setHostValue("exp_sml", (float)(mSubsampleLevel + mHierarchySmlExp)); |
| } else if (name.equals("lumScale") || name.equals("chromaScale")) { |
| float[] yuvWeights = { mLumScale, mChromaScale }; |
| mBgMaskProgram.setHostValue("yuv_weights", yuvWeights ); |
| } else if (name.equals("maskBg")) { |
| mBgSubtractProgram.setHostValue("mask_blend_bg", mMaskBg); |
| } else if (name.equals("maskFg")) { |
| mBgSubtractProgram.setHostValue("mask_blend_fg", mMaskFg); |
| } else if (name.equals("exposureChange")) { |
| mBgSubtractProgram.setHostValue("exposure_change", mExposureChange); |
| } else if (name.equals("whitebalanceredChange")) { |
| mBgSubtractProgram.setHostValue("whitebalancered_change", mWhiteBalanceRedChange); |
| } else if (name.equals("whitebalanceblueChange")) { |
| mBgSubtractProgram.setHostValue("whitebalanceblue_change", mWhiteBalanceBlueChange); |
| } else if (name.equals("autowbToggle")){ |
| mAutomaticWhiteBalanceProgram.setHostValue("autowb_toggle", mAutoWBToggle); |
| } |
| } |
| |
| private void updateBgScaling(Frame video, Frame background, boolean fitModeChanged) { |
| float foregroundAspect = (float)video.getFormat().getWidth() / video.getFormat().getHeight(); |
| float backgroundAspect = (float)background.getFormat().getWidth() / background.getFormat().getHeight(); |
| float currentRelativeAspect = foregroundAspect/backgroundAspect; |
| if (currentRelativeAspect != mRelativeAspect || fitModeChanged) { |
| mRelativeAspect = currentRelativeAspect; |
| float xMin = 0.f, xWidth = 1.f, yMin = 0.f, yWidth = 1.f; |
| switch (mBackgroundFitMode) { |
| case BACKGROUND_STRETCH: |
| // Just map 1:1 |
| break; |
| case BACKGROUND_FIT: |
| if (mRelativeAspect > 1.0f) { |
| // Foreground is wider than background, scale down |
| // background in X |
| xMin = 0.5f - 0.5f * mRelativeAspect; |
| xWidth = 1.f * mRelativeAspect; |
| } else { |
| // Foreground is taller than background, scale down |
| // background in Y |
| yMin = 0.5f - 0.5f / mRelativeAspect; |
| yWidth = 1 / mRelativeAspect; |
| } |
| break; |
| case BACKGROUND_FILL_CROP: |
| if (mRelativeAspect > 1.0f) { |
| // Foreground is wider than background, crop |
| // background in Y |
| yMin = 0.5f - 0.5f / mRelativeAspect; |
| yWidth = 1.f / mRelativeAspect; |
| } else { |
| // Foreground is taller than background, crop |
| // background in X |
| xMin = 0.5f - 0.5f * mRelativeAspect; |
| xWidth = mRelativeAspect; |
| } |
| break; |
| } |
| // If mirroring is required (for ex. the camera mirrors the preview |
| // in the front camera) |
| // TODO: Backdropper does not attempt to apply any other transformation |
| // than just flipping. However, in the current state, it's "x-axis" is always aligned |
| // with the Camera's width. Hence, we need to define the mirroring based on the camera |
| // orientation. In the future, a cleaner design would be to cast away all the rotation |
| // in a separate place. |
| if (mMirrorBg) { |
| if (mLogVerbose) Log.v(TAG, "Mirroring the background!"); |
| // Mirroring in portrait |
| if (mOrientation == 0 || mOrientation == 180) { |
| xWidth = -xWidth; |
| xMin = 1.0f - xMin; |
| } else { |
| // Mirroring in landscape |
| yWidth = -yWidth; |
| yMin = 1.0f - yMin; |
| } |
| } |
| if (mLogVerbose) Log.v(TAG, "bgTransform: xMin, yMin, xWidth, yWidth : " + |
| xMin + ", " + yMin + ", " + xWidth + ", " + yWidth + |
| ", mRelAspRatio = " + mRelativeAspect); |
| // The following matrix is the transpose of the actual matrix |
| float[] bgTransform = {xWidth, 0.f, 0.f, |
| 0.f, yWidth, 0.f, |
| xMin, yMin, 1.f}; |
| mBgSubtractProgram.setHostValue("bg_fit_transform", bgTransform); |
| } |
| } |
| |
| private int pyramidLevel(int size) { |
| return (int)Math.floor(Math.log10(size) / Math.log10(2)) - 1; |
| } |
| |
| } |
