android/filterpacks/videosrc/SurfaceTextureSource.java - platform/prebuilts/fullsdk/sources/android-29 - Git at Google

 /*
  * Copyright (C) 2011 The Android Open Source Project
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
  * You may obtain a copy of the License at
  *
  *      http://www.apache.org/licenses/LICENSE-2.0
  *
  * Unless required by applicable law or agreed to in writing, software
  * distributed under the License is distributed on an "AS IS" BASIS,
  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */

 package android.filterpacks.videosrc;

 import android.filterfw.core.Filter;
 import android.filterfw.core.FilterContext;
 import android.filterfw.core.Frame;
 import android.filterfw.core.FrameFormat;
 import android.filterfw.core.GenerateFieldPort;
 import android.filterfw.core.GenerateFinalPort;
 import android.filterfw.core.GLFrame;
 import android.filterfw.core.MutableFrameFormat;
 import android.filterfw.core.ShaderProgram;
 import android.filterfw.format.ImageFormat;
 import android.graphics.SurfaceTexture;
 import android.os.ConditionVariable;
 import android.opengl.Matrix;

 import android.util.Log;

 /** <p>A filter that converts textures from a SurfaceTexture object into frames for
  * processing in the filter framework.</p>
  *
  * <p>To use, connect up the sourceListener callback, and then when executing
  * the graph, use the SurfaceTexture object passed to the callback to feed
  * frames into the filter graph. For example, pass the SurfaceTexture into
  * {@link
  * android.hardware.Camera.setPreviewTexture(android.graphics.SurfaceTexture)}.
  * This filter is intended for applications that need for flexibility than the
  * CameraSource and MediaSource provide. Note that the application needs to
  * provide width and height information for the SurfaceTextureSource, which it
  * should obtain from wherever the SurfaceTexture data is coming from to avoid
  * unnecessary resampling.</p>
  *
  * @hide
  */
 public class SurfaceTextureSource extends Filter {

     /** User-visible parameters */

     /** The callback interface for the sourceListener parameter */
     public interface SurfaceTextureSourceListener {
         public void onSurfaceTextureSourceReady(SurfaceTexture source);
     }
     /** A callback to send the internal SurfaceTexture object to, once it is
      * created. This callback will be called when the the filter graph is
      * preparing to execute, but before any processing has actually taken
      * place. The SurfaceTexture object passed to this callback is the only way
      * to feed this filter. When the filter graph is shutting down, this
      * callback will be called again with null as the source.
      *
      * This callback may be called from an arbitrary thread, so it should not
      * assume it is running in the UI thread in particular.
      */
     @GenerateFinalPort(name = "sourceListener")
     private SurfaceTextureSourceListener mSourceListener;

     /** The width of the output image frame. If the texture width for the
      * SurfaceTexture source is known, use it here to minimize resampling. */
     @GenerateFieldPort(name = "width")
     private int mWidth;

     /** The height of the output image frame. If the texture height for the
      * SurfaceTexture source is known, use it here to minimize resampling. */
     @GenerateFieldPort(name = "height")
     private int mHeight;

     /** Whether the filter will always wait for a new frame from its
      * SurfaceTexture, or whether it will output an old frame again if a new
      * frame isn't available. The filter will always wait for the first frame,
      * to avoid outputting a blank frame. Defaults to true.
      */
     @GenerateFieldPort(name = "waitForNewFrame", hasDefault = true)
     private boolean mWaitForNewFrame = true;

     /** Maximum timeout before signaling error when waiting for a new frame. Set
      * this to zero to disable the timeout and wait indefinitely. In milliseconds.
      */
     @GenerateFieldPort(name = "waitTimeout", hasDefault = true)
     private int mWaitTimeout = 1000;

     /** Whether a timeout is an exception-causing failure, or just causes the
      * filter to close.
      */
     @GenerateFieldPort(name = "closeOnTimeout", hasDefault = true)
     private boolean mCloseOnTimeout = false;

     // Variables for input->output conversion
     private GLFrame mMediaFrame;
     private ShaderProgram mFrameExtractor;
     private SurfaceTexture mSurfaceTexture;
     private MutableFrameFormat mOutputFormat;
     private ConditionVariable mNewFrameAvailable;
     private boolean mFirstFrame;

     private float[] mFrameTransform;
     private float[] mMappedCoords;
     // These default source coordinates perform the necessary flip
     // for converting from MFF/Bitmap origin to OpenGL origin.
     private static final float[] mSourceCoords = { 0, 1, 0, 1,
                                                    1, 1, 0, 1,
                                                    0, 0, 0, 1,
                                                    1, 0, 0, 1 };
     // Shader for output
     private final String mRenderShader =
             "#extension GL_OES_EGL_image_external : require\n" +
             "precision mediump float;\n" +
             "uniform samplerExternalOES tex_sampler_0;\n" +
             "varying vec2 v_texcoord;\n" +
             "void main() {\n" +
             "  gl_FragColor = texture2D(tex_sampler_0, v_texcoord);\n" +
             "}\n";

     // Variables for logging

     private static final String TAG = "SurfaceTextureSource";
     private static final boolean mLogVerbose = Log.isLoggable(TAG, Log.VERBOSE);

     public SurfaceTextureSource(String name) {
         super(name);
         mNewFrameAvailable = new ConditionVariable();
         mFrameTransform = new float[16];
         mMappedCoords = new float[16];
     }

     @Override
     public void setupPorts() {
         // Add input port
         addOutputPort("video", ImageFormat.create(ImageFormat.COLORSPACE_RGBA,
                                                   FrameFormat.TARGET_GPU));
     }

     private void createFormats() {
         mOutputFormat = ImageFormat.create(mWidth, mHeight,
                                            ImageFormat.COLORSPACE_RGBA,
                                            FrameFormat.TARGET_GPU);
     }

     @Override
     protected void prepare(FilterContext context) {
         if (mLogVerbose) Log.v(TAG, "Preparing SurfaceTextureSource");

         createFormats();

         // Prepare input
         mMediaFrame = (GLFrame)context.getFrameManager().newBoundFrame(mOutputFormat,
                                                                        GLFrame.EXTERNAL_TEXTURE,
                                                                        0);

         // Prepare output
         mFrameExtractor = new ShaderProgram(context, mRenderShader);
     }

     @Override
     public void open(FilterContext context) {
         if (mLogVerbose) Log.v(TAG, "Opening SurfaceTextureSource");
         // Create SurfaceTexture anew each time - it can use substantial memory.
         mSurfaceTexture = new SurfaceTexture(mMediaFrame.getTextureId());
         // Connect SurfaceTexture to callback
         mSurfaceTexture.setOnFrameAvailableListener(onFrameAvailableListener);
         // Connect SurfaceTexture to source
         mSourceListener.onSurfaceTextureSourceReady(mSurfaceTexture);
         mFirstFrame = true;
     }

     @Override
     public void process(FilterContext context) {
         if (mLogVerbose) Log.v(TAG, "Processing new frame");

         // First, get new frame if available
         if (mWaitForNewFrame || mFirstFrame) {
             boolean gotNewFrame;
             if (mWaitTimeout != 0) {
                 gotNewFrame = mNewFrameAvailable.block(mWaitTimeout);
                 if (!gotNewFrame) {
                     if (!mCloseOnTimeout) {
                         throw new RuntimeException("Timeout waiting for new frame");
                     } else {
                         if (mLogVerbose) Log.v(TAG, "Timeout waiting for a new frame. Closing.");
                         closeOutputPort("video");
                         return;
                     }
                 }
             } else {
                 mNewFrameAvailable.block();
             }
             mNewFrameAvailable.close();
             mFirstFrame = false;
         }

         mSurfaceTexture.updateTexImage();

         mSurfaceTexture.getTransformMatrix(mFrameTransform);
         Matrix.multiplyMM(mMappedCoords, 0,
                           mFrameTransform, 0,
                           mSourceCoords, 0);
         mFrameExtractor.setSourceRegion(mMappedCoords[0], mMappedCoords[1],
                                         mMappedCoords[4], mMappedCoords[5],
                                         mMappedCoords[8], mMappedCoords[9],
                                         mMappedCoords[12], mMappedCoords[13]);
         // Next, render to output
         Frame output = context.getFrameManager().newFrame(mOutputFormat);
         mFrameExtractor.process(mMediaFrame, output);

         output.setTimestamp(mSurfaceTexture.getTimestamp());

         pushOutput("video", output);
         output.release();
     }

     @Override
     public void close(FilterContext context) {
         if (mLogVerbose) Log.v(TAG, "SurfaceTextureSource closed");
         mSourceListener.onSurfaceTextureSourceReady(null);
         mSurfaceTexture.release();
         mSurfaceTexture = null;
     }

     @Override
     public void tearDown(FilterContext context) {
         if (mMediaFrame != null) {
             mMediaFrame.release();
         }
     }

     @Override
     public void fieldPortValueUpdated(String name, FilterContext context) {
         if (name.equals("width") || name.equals("height") ) {
             mOutputFormat.setDimensions(mWidth, mHeight);
         }
     }

     private SurfaceTexture.OnFrameAvailableListener onFrameAvailableListener =
             new SurfaceTexture.OnFrameAvailableListener() {
         public void onFrameAvailable(SurfaceTexture surfaceTexture) {
             if (mLogVerbose) Log.v(TAG, "New frame from SurfaceTexture");
             mNewFrameAvailable.open();
         }
     };
 }
	/*
	* Copyright (C) 2011 The Android Open Source Project
	*
	* Licensed under the Apache License, Version 2.0 (the "License");
	* you may not use this file except in compliance with the License.
	* You may obtain a copy of the License at
	*
	* http://www.apache.org/licenses/LICENSE-2.0
	*
	* Unless required by applicable law or agreed to in writing, software
	* distributed under the License is distributed on an "AS IS" BASIS,
	* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
	* See the License for the specific language governing permissions and
	* limitations under the License.
	*/

	package android.filterpacks.videosrc;

	import android.filterfw.core.Filter;
	import android.filterfw.core.FilterContext;
	import android.filterfw.core.Frame;
	import android.filterfw.core.FrameFormat;
	import android.filterfw.core.GenerateFieldPort;
	import android.filterfw.core.GenerateFinalPort;
	import android.filterfw.core.GLFrame;
	import android.filterfw.core.MutableFrameFormat;
	import android.filterfw.core.ShaderProgram;
	import android.filterfw.format.ImageFormat;
	import android.graphics.SurfaceTexture;
	import android.os.ConditionVariable;
	import android.opengl.Matrix;

	import android.util.Log;

	/** <p>A filter that converts textures from a SurfaceTexture object into frames for
	* processing in the filter framework.</p>
	*
	* <p>To use, connect up the sourceListener callback, and then when executing
	* the graph, use the SurfaceTexture object passed to the callback to feed
	* frames into the filter graph. For example, pass the SurfaceTexture into
	* {@link
	* android.hardware.Camera.setPreviewTexture(android.graphics.SurfaceTexture)}.
	* This filter is intended for applications that need for flexibility than the
	* CameraSource and MediaSource provide. Note that the application needs to
	* provide width and height information for the SurfaceTextureSource, which it
	* should obtain from wherever the SurfaceTexture data is coming from to avoid
	* unnecessary resampling.</p>
	*
	* @hide
	*/
	public class SurfaceTextureSource extends Filter {

	/** User-visible parameters */

	/** The callback interface for the sourceListener parameter */
	public interface SurfaceTextureSourceListener {
	public void onSurfaceTextureSourceReady(SurfaceTexture source);
	}
	/** A callback to send the internal SurfaceTexture object to, once it is
	* created. This callback will be called when the the filter graph is
	* preparing to execute, but before any processing has actually taken
	* place. The SurfaceTexture object passed to this callback is the only way
	* to feed this filter. When the filter graph is shutting down, this
	* callback will be called again with null as the source.
	*
	* This callback may be called from an arbitrary thread, so it should not
	* assume it is running in the UI thread in particular.
	*/
	@GenerateFinalPort(name = "sourceListener")
	private SurfaceTextureSourceListener mSourceListener;

	/** The width of the output image frame. If the texture width for the
	* SurfaceTexture source is known, use it here to minimize resampling. */
	@GenerateFieldPort(name = "width")
	private int mWidth;

	/** The height of the output image frame. If the texture height for the
	* SurfaceTexture source is known, use it here to minimize resampling. */
	@GenerateFieldPort(name = "height")
	private int mHeight;

	/** Whether the filter will always wait for a new frame from its
	* SurfaceTexture, or whether it will output an old frame again if a new
	* frame isn't available. The filter will always wait for the first frame,
	* to avoid outputting a blank frame. Defaults to true.
	*/
	@GenerateFieldPort(name = "waitForNewFrame", hasDefault = true)
	private boolean mWaitForNewFrame = true;

	/** Maximum timeout before signaling error when waiting for a new frame. Set
	* this to zero to disable the timeout and wait indefinitely. In milliseconds.
	*/
	@GenerateFieldPort(name = "waitTimeout", hasDefault = true)
	private int mWaitTimeout = 1000;

	/** Whether a timeout is an exception-causing failure, or just causes the
	* filter to close.
	*/
	@GenerateFieldPort(name = "closeOnTimeout", hasDefault = true)
	private boolean mCloseOnTimeout = false;

	// Variables for input->output conversion
	private GLFrame mMediaFrame;
	private ShaderProgram mFrameExtractor;
	private SurfaceTexture mSurfaceTexture;
	private MutableFrameFormat mOutputFormat;
	private ConditionVariable mNewFrameAvailable;
	private boolean mFirstFrame;

	private float[] mFrameTransform;
	private float[] mMappedCoords;
	// These default source coordinates perform the necessary flip
	// for converting from MFF/Bitmap origin to OpenGL origin.
	private static final float[] mSourceCoords = { 0, 1, 0, 1,
	1, 1, 0, 1,
	0, 0, 0, 1,
	1, 0, 0, 1 };
	// Shader for output
	private final String mRenderShader =
	"#extension GL_OES_EGL_image_external : require\n" +
	"precision mediump float;\n" +
	"uniform samplerExternalOES tex_sampler_0;\n" +
	"varying vec2 v_texcoord;\n" +
	"void main() {\n" +
	" gl_FragColor = texture2D(tex_sampler_0, v_texcoord);\n" +
	"}\n";

	// Variables for logging

	private static final String TAG = "SurfaceTextureSource";
	private static final boolean mLogVerbose = Log.isLoggable(TAG, Log.VERBOSE);

	public SurfaceTextureSource(String name) {
	super(name);
	mNewFrameAvailable = new ConditionVariable();
	mFrameTransform = new float[16];
	mMappedCoords = new float[16];
	}

	@Override
	public void setupPorts() {
	// Add input port
	addOutputPort("video", ImageFormat.create(ImageFormat.COLORSPACE_RGBA,
	FrameFormat.TARGET_GPU));
	}

	private void createFormats() {
	mOutputFormat = ImageFormat.create(mWidth, mHeight,
	ImageFormat.COLORSPACE_RGBA,
	FrameFormat.TARGET_GPU);
	}

	@Override
	protected void prepare(FilterContext context) {
	if (mLogVerbose) Log.v(TAG, "Preparing SurfaceTextureSource");

	createFormats();

	// Prepare input
	mMediaFrame = (GLFrame)context.getFrameManager().newBoundFrame(mOutputFormat,
	GLFrame.EXTERNAL_TEXTURE,
	0);

	// Prepare output
	mFrameExtractor = new ShaderProgram(context, mRenderShader);
	}

	@Override
	public void open(FilterContext context) {
	if (mLogVerbose) Log.v(TAG, "Opening SurfaceTextureSource");
	// Create SurfaceTexture anew each time - it can use substantial memory.
	mSurfaceTexture = new SurfaceTexture(mMediaFrame.getTextureId());
	// Connect SurfaceTexture to callback
	mSurfaceTexture.setOnFrameAvailableListener(onFrameAvailableListener);
	// Connect SurfaceTexture to source
	mSourceListener.onSurfaceTextureSourceReady(mSurfaceTexture);
	mFirstFrame = true;
	}

	@Override
	public void process(FilterContext context) {
	if (mLogVerbose) Log.v(TAG, "Processing new frame");

	// First, get new frame if available
	if (mWaitForNewFrame \|\| mFirstFrame) {
	boolean gotNewFrame;
	if (mWaitTimeout != 0) {
	gotNewFrame = mNewFrameAvailable.block(mWaitTimeout);
	if (!gotNewFrame) {
	if (!mCloseOnTimeout) {
	throw new RuntimeException("Timeout waiting for new frame");
	} else {
	if (mLogVerbose) Log.v(TAG, "Timeout waiting for a new frame. Closing.");
	closeOutputPort("video");
	return;
	}
	}
	} else {
	mNewFrameAvailable.block();
	}
	mNewFrameAvailable.close();
	mFirstFrame = false;
	}

	mSurfaceTexture.updateTexImage();

	mSurfaceTexture.getTransformMatrix(mFrameTransform);
	Matrix.multiplyMM(mMappedCoords, 0,
	mFrameTransform, 0,
	mSourceCoords, 0);
	mFrameExtractor.setSourceRegion(mMappedCoords[0], mMappedCoords[1],
	mMappedCoords[4], mMappedCoords[5],
	mMappedCoords[8], mMappedCoords[9],
	mMappedCoords[12], mMappedCoords[13]);
	// Next, render to output
	Frame output = context.getFrameManager().newFrame(mOutputFormat);
	mFrameExtractor.process(mMediaFrame, output);

	output.setTimestamp(mSurfaceTexture.getTimestamp());

	pushOutput("video", output);
	output.release();
	}

	@Override
	public void close(FilterContext context) {
	if (mLogVerbose) Log.v(TAG, "SurfaceTextureSource closed");
	mSourceListener.onSurfaceTextureSourceReady(null);
	mSurfaceTexture.release();
	mSurfaceTexture = null;
	}

	@Override
	public void tearDown(FilterContext context) {
	if (mMediaFrame != null) {
	mMediaFrame.release();
	}
	}

	@Override
	public void fieldPortValueUpdated(String name, FilterContext context) {
	if (name.equals("width") \|\| name.equals("height") ) {
	mOutputFormat.setDimensions(mWidth, mHeight);
	}
	}

	private SurfaceTexture.OnFrameAvailableListener onFrameAvailableListener =
	new SurfaceTexture.OnFrameAvailableListener() {
	public void onFrameAvailable(SurfaceTexture surfaceTexture) {
	if (mLogVerbose) Log.v(TAG, "New frame from SurfaceTexture");
	mNewFrameAvailable.open();
	}
	};
	}