graphics/java/android/graphics/SurfaceTexture.java

/*
 * Copyright (C) 2010 The Android Open Source Project
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *      http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package android.graphics;

import android.annotation.FloatRange;
import android.annotation.Nullable;
import android.annotation.SuppressLint;
import android.compat.annotation.UnsupportedAppUsage;
import android.hardware.DataSpace.NamedDataSpace;
import android.os.Build;
import android.os.Handler;
import android.os.Looper;
import android.os.Message;
import android.os.Trace;
import android.view.Surface;
import android.view.TextureView;
import android.view.flags.Flags;

import java.lang.ref.WeakReference;

/**
 * Captures frames from an image stream as an OpenGL ES texture.
 *
 * <p>The image stream may come from either camera preview or video decode. A
 * {@link android.view.Surface} created from a SurfaceTexture can be used as an output
 * destination for the {@link android.hardware.camera2}, {@link android.media.MediaCodec},
 * {@link android.media.MediaPlayer}, and {@link android.renderscript.Allocation} APIs.
 * When {@link #updateTexImage} is called, the contents of the texture object specified
 * when the SurfaceTexture was created are updated to contain the most recent image from the image
 * stream.  This may cause some frames of the stream to be skipped.
 *
 * <p>A SurfaceTexture may also be used in place of a SurfaceHolder when specifying the output
 * destination of the older {@link android.hardware.Camera} API. Doing so will cause all the
 * frames from the image stream to be sent to the SurfaceTexture object rather than to the device's
 * display.
 *
 * <p>A typical pattern is to use SurfaceTexture to render frames to a {@link TextureView}; however,
 * a TextureView is not <i>required</i> for using the texture object. The texture object may be used
 * as part of an OpenGL ES shader.
 *
 * <p>When sampling from the texture one should first transform the texture coordinates using the
 * matrix queried via {@link #getTransformMatrix(float[])}.  The transform matrix may change each
 * time {@link #updateTexImage} is called, so it should be re-queried each time the texture image
 * is updated.
 * This matrix transforms traditional 2D OpenGL ES texture coordinate column vectors of the form (s,
 * t, 0, 1) where s and t are on the inclusive interval [0, 1] to the proper sampling location in
 * the streamed texture.  This transform compensates for any properties of the image stream source
 * that cause it to appear different from a traditional OpenGL ES texture.  For example, sampling
 * from the bottom left corner of the image can be accomplished by transforming the column vector
 * (0, 0, 0, 1) using the queried matrix, while sampling from the top right corner of the image can
 * be done by transforming (1, 1, 0, 1).
 *
 * <p>The texture object uses the GL_TEXTURE_EXTERNAL_OES texture target, which is defined by the
 * <a href="http://www.khronos.org/registry/gles/extensions/OES/OES_EGL_image_external.txt">
 * GL_OES_EGL_image_external</a> OpenGL ES extension.  This limits how the texture may be used.
 * Each time the texture is bound it must be bound to the GL_TEXTURE_EXTERNAL_OES target rather than
 * the GL_TEXTURE_2D target.  Additionally, any OpenGL ES 2.0 shader that samples from the texture
 * must declare its use of this extension using, for example, an "#extension
 * GL_OES_EGL_image_external : require" directive.  Such shaders must also access the texture using
 * the samplerExternalOES GLSL sampler type.
 *
 * <p>SurfaceTexture objects may be created on any thread.  {@link #updateTexImage} may only be
 * called on the thread with the OpenGL ES context that contains the texture object.  The
 * frame-available callback is called on an arbitrary thread, so unless special care is taken {@link
 * #updateTexImage} should not be called directly from the callback.
 */
@android.ravenwood.annotation.RavenwoodKeepWholeClass
public class SurfaceTexture {
    private final Looper mCreatorLooper;
    @UnsupportedAppUsage(maxTargetSdk = Build.VERSION_CODES.R, trackingBug = 170729553)
    private Handler mOnFrameAvailableHandler;
    private Handler mOnSetFrameRateHandler;

    /**
     * These fields are used by native code, do not access or modify.
     */
    @UnsupportedAppUsage(trackingBug = 176388660)
    private long mSurfaceTexture;
    @UnsupportedAppUsage(trackingBug = 176388660)
    private long mProducer;
    @UnsupportedAppUsage(maxTargetSdk = Build.VERSION_CODES.R, trackingBug = 170729553)
    private long mFrameAvailableListener;

    private boolean mIsSingleBuffered;

    /**
     * Callback interface for being notified that a new stream frame is available.
     */
    public interface OnFrameAvailableListener {
        void onFrameAvailable(SurfaceTexture surfaceTexture);
    }

    /**
     * Callback interface for being notified that a producer set a frame rate
     * @hide
     */
    public interface OnSetFrameRateListener {
        /**
         * Called when the producer sets a frame rate
         * @hide
         */
        void onSetFrameRate(SurfaceTexture surfaceTexture,
                            @FloatRange(from = 0.0) float frameRate,
                                 @Surface.FrameRateCompatibility int compatibility,
                                 @Surface.ChangeFrameRateStrategy int changeFrameRateStrategy);
    }

    /**
     * Exception thrown when a SurfaceTexture couldn't be created or resized.
     *
     * @deprecated No longer thrown. {@link android.view.Surface.OutOfResourcesException}
     * is used instead.
     */
    @SuppressWarnings("serial")
    @Deprecated
    public static class OutOfResourcesException extends Exception {
        public OutOfResourcesException() {
        }
        public OutOfResourcesException(String name) {
            super(name);
        }
    }

    /**
     * Construct a new SurfaceTexture to stream images to a given OpenGL texture.
     *
     * @param texName the OpenGL texture object name (e.g. generated via glGenTextures)
     *
     * @throws android.view.Surface.OutOfResourcesException If the SurfaceTexture cannot be created.
     */
    public SurfaceTexture(int texName) {
        this(texName, false);
    }

    /**
     * Construct a new SurfaceTexture to stream images to a given OpenGL texture.
     * <p>
     * In single buffered mode the application is responsible for serializing access to the image
     * content buffer. Each time the image content is to be updated, the
     * {@link #releaseTexImage()} method must be called before the image content producer takes
     * ownership of the buffer. For example, when producing image content with the NDK
     * ANativeWindow_lock and ANativeWindow_unlockAndPost functions, {@link #releaseTexImage()}
     * must be called before each ANativeWindow_lock, or that call will fail. When producing
     * image content with OpenGL ES, {@link #releaseTexImage()} must be called before the first
     * OpenGL ES function call each frame.
     *
     * @param texName the OpenGL texture object name (e.g. generated via glGenTextures)
     * @param singleBufferMode whether the SurfaceTexture will be in single buffered mode.
     *
     * @throws android.view.Surface.OutOfResourcesException If the SurfaceTexture cannot be created.
     */
    public SurfaceTexture(int texName, boolean singleBufferMode) {
        mCreatorLooper = Looper.myLooper();
        mIsSingleBuffered = singleBufferMode;
        nativeInit(false, texName, singleBufferMode, new WeakReference<SurfaceTexture>(this));
    }

    /**
     * Construct a new SurfaceTexture to stream images to a given OpenGL texture.
     * <p>
     * In single buffered mode the application is responsible for serializing access to the image
     * content buffer. Each time the image content is to be updated, the
     * {@link #releaseTexImage()} method must be called before the image content producer takes
     * ownership of the buffer. For example, when producing image content with the NDK
     * ANativeWindow_lock and ANativeWindow_unlockAndPost functions, {@link #releaseTexImage()}
     * must be called before each ANativeWindow_lock, or that call will fail. When producing
     * image content with OpenGL ES, {@link #releaseTexImage()} must be called before the first
     * OpenGL ES function call each frame.
     * <p>
     * Unlike {@link #SurfaceTexture(int, boolean)}, which takes an OpenGL texture object name,
     * this constructor creates the SurfaceTexture in detached mode. A texture name must be passed
     * in using {@link #attachToGLContext} before calling {@link #releaseTexImage()} and producing
     * image content using OpenGL ES.
     *
     * @param singleBufferMode whether the SurfaceTexture will be in single buffered mode.
     *
     * @throws android.view.Surface.OutOfResourcesException If the SurfaceTexture cannot be created.
     */
    public SurfaceTexture(boolean singleBufferMode) {
        mCreatorLooper = Looper.myLooper();
        mIsSingleBuffered = singleBufferMode;
        nativeInit(true, 0, singleBufferMode, new WeakReference<SurfaceTexture>(this));
    }

    /**
     * Register a callback to be invoked when a new image frame becomes available to the
     * SurfaceTexture.
     * <p>
     * The callback may be called on an arbitrary thread, so it is not
     * safe to call {@link #updateTexImage} without first binding the OpenGL ES context to the
     * thread invoking the callback.
     * </p>
     *
     * @param listener The listener to use, or null to remove the listener.
     */
    public void setOnFrameAvailableListener(@Nullable OnFrameAvailableListener listener) {
        setOnFrameAvailableListener(listener, null);
    }

    /**
     * Register a callback to be invoked when a new image frame becomes available to the
     * SurfaceTexture.
     * <p>
     * If a handler is specified, the callback will be invoked on that handler's thread.
     * If no handler is specified, then the callback may be called on an arbitrary thread,
     * so it is not safe to call {@link #updateTexImage} without first binding the OpenGL ES
     * context to the thread invoking the callback.
     * </p>
     *
     * @param listener The listener to use, or null to remove the listener.
     * @param handler The handler on which the listener should be invoked, or null
     * to use an arbitrary thread.
     */
    public void setOnFrameAvailableListener(@Nullable final OnFrameAvailableListener listener,
            @Nullable Handler handler) {
        if (listener != null) {
            // Although we claim the thread is arbitrary, earlier implementation would
            // prefer to send the callback on the creating looper or the main looper
            // so we preserve this behavior here.
            Looper looper = handler != null ? handler.getLooper() :
                    mCreatorLooper != null ? mCreatorLooper : Looper.getMainLooper();
            mOnFrameAvailableHandler = new Handler(looper, null, true /*async*/) {
                @Override
                public void handleMessage(Message msg) {
                    listener.onFrameAvailable(SurfaceTexture.this);
                }
            };
        } else {
            mOnFrameAvailableHandler = null;
        }
    }

    private static class SetFrameRateArgs {
        SetFrameRateArgs(@FloatRange(from = 0.0) float frameRate,
                                @Surface.FrameRateCompatibility int compatibility,
                   @Surface.ChangeFrameRateStrategy int changeFrameRateStrategy) {
            this.mFrameRate = frameRate;
            this.mCompatibility = compatibility;
            this.mChangeFrameRateStrategy = changeFrameRateStrategy;
        }
        final float mFrameRate;
        final int mCompatibility;
        final int mChangeFrameRateStrategy;
    }

    /**
     * Register a callback to be invoked when the producer sets a frame rate using
     * Surface.setFrameRate.
     * @hide
     */
    public void setOnSetFrameRateListener(@Nullable final OnSetFrameRateListener listener,
                                            @Nullable Handler handler) {
        if (listener != null) {
            Looper looper = handler != null ? handler.getLooper() :
                    mCreatorLooper != null ? mCreatorLooper : Looper.getMainLooper();
            mOnSetFrameRateHandler = new Handler(looper, null, true /*async*/) {
                @Override
                public void handleMessage(Message msg) {
                    Trace.traceBegin(Trace.TRACE_TAG_VIEW, "onSetFrameRateHandler");
                    try {
                        SetFrameRateArgs args = (SetFrameRateArgs) msg.obj;
                        listener.onSetFrameRate(SurfaceTexture.this,
                                args.mFrameRate, args.mCompatibility,
                                args.mChangeFrameRateStrategy);
                    } finally {
                        Trace.traceEnd(Trace.TRACE_TAG_VIEW);
                    }
                }
            };
        } else {
            mOnSetFrameRateHandler = null;
        }
    }

    /**
     * Set the default size of the image buffers.  The image producer may override the buffer size,
     * in which case the producer-set buffer size will be used, not the default size set by this
     * method.  Both video and camera based image producers do override the size.  This method may
     * be used to set the image size when producing images with {@link android.graphics.Canvas} (via
     * {@link android.view.Surface#lockCanvas}), or OpenGL ES (via an EGLSurface).
     * <p>
     * The new default buffer size will take effect the next time the image producer requests a
     * buffer to fill.  For {@link android.graphics.Canvas} this will be the next time {@link
     * android.view.Surface#lockCanvas} is called.  For OpenGL ES, the EGLSurface should be
     * destroyed (via eglDestroySurface), made not-current (via eglMakeCurrent), and then recreated
     * (via {@code eglCreateWindowSurface}) to ensure that the new default size has taken effect.
     * <p>
     * The width and height parameters must be no greater than the minimum of
     * {@code GL_MAX_VIEWPORT_DIMS} and {@code GL_MAX_TEXTURE_SIZE} (see
     * {@link javax.microedition.khronos.opengles.GL10#glGetIntegerv glGetIntegerv}).
     * An error due to invalid dimensions might not be reported until
     * updateTexImage() is called.
     */
    public void setDefaultBufferSize(int width, int height) {
        nativeSetDefaultBufferSize(width, height);
    }

    /**
     * Update the texture image to the most recent frame from the image stream.  This may only be
     * called while the OpenGL ES context that owns the texture is current on the calling thread.
     * It will implicitly bind its texture to the {@code GL_TEXTURE_EXTERNAL_OES} texture target.
     */
    public void updateTexImage() {
        nativeUpdateTexImage();
    }

    /**
     * Releases the texture content. This is needed in single buffered mode to allow the image
     * content producer to take ownership of the image buffer.
     * <p>
     * For more information see {@link #SurfaceTexture(int, boolean)}.
     */
    public void releaseTexImage() {
        nativeReleaseTexImage();
    }

    /**
     * Detach the SurfaceTexture from the OpenGL ES context that owns the OpenGL ES texture object.
     * This call must be made with the OpenGL ES context current on the calling thread.  The OpenGL
     * ES texture object will be deleted as a result of this call.  After calling this method all
     * calls to {@link #updateTexImage} will throw an {@link java.lang.IllegalStateException} until
     * a successful call to {@link #attachToGLContext} is made.
     * <p>
     * This can be used to access the SurfaceTexture image contents from multiple OpenGL ES
     * contexts.  Note, however, that the image contents are only accessible from one OpenGL ES
     * context at a time.
     */
    public void detachFromGLContext() {
        int err = nativeDetachFromGLContext();
        if (err != 0) {
            throw new RuntimeException("Error during detachFromGLContext (see logcat for details)");
        }
    }

    /**
     * Attach the SurfaceTexture to the OpenGL ES context that is current on the calling thread.  A
     * new OpenGL ES texture object is created and populated with the SurfaceTexture image frame
     * that was current at the time of the last call to {@link #detachFromGLContext}.  This new
     * texture is bound to the {@code GL_TEXTURE_EXTERNAL_OES} texture target.
     * <p>
     * This can be used to access the SurfaceTexture image contents from multiple OpenGL ES
     * contexts.  Note, however, that the image contents are only accessible from one OpenGL ES
     * context at a time.
     *
     * @param texName The name of the OpenGL ES texture that will be created.  This texture name
     * must be unused in the OpenGL ES context that is current on the calling thread.
     */
    public void attachToGLContext(int texName) {
        int err = nativeAttachToGLContext(texName);
        if (err != 0) {
            throw new RuntimeException("Error during attachToGLContext (see logcat for details)");
        }
    }

    /**
     * Retrieve the 4x4 texture coordinate transform matrix associated with the texture image set by
     * the most recent call to {@link #updateTexImage}.
     * <p>
     * This transform matrix maps 2D homogeneous texture coordinates of the form (s, t, 0, 1) with s
     * and t in the inclusive range [0, 1] to the texture coordinate that should be used to sample
     * that location from the texture.  Sampling the texture outside of the range of this transform
     * is undefined.
     * <p>
     * The matrix is stored in column-major order so that it may be passed directly to OpenGL ES via
     * the {@code glLoadMatrixf} or {@code glUniformMatrix4fv} functions.
     * <p>
     * If the underlying buffer has a crop associated with it, the transformation will also include
     * a slight scale to cut off a 1-texel border around the edge of the crop. This ensures that
     * when the texture is bilinear sampled that no texels outside of the buffer's valid region
     * are accessed by the GPU, avoiding any sampling artifacts when scaling.
     *
     * @param mtx the array into which the 4x4 matrix will be stored.  The array must have exactly
     *     16 elements.
     */
    public void getTransformMatrix(float[] mtx) {
        // Note we intentionally don't check mtx for null, so this will result in a
        // NullPointerException. But it's safe because it happens before the call to native.
        if (mtx.length != 16) {
            throw new IllegalArgumentException();
        }
        nativeGetTransformMatrix(mtx);
    }

    /**
     * Retrieve the timestamp associated with the texture image set by the most recent call to
     * {@link #updateTexImage}.
     *
     * <p>This timestamp is in nanoseconds, and is normally monotonically increasing. The timestamp
     * should be unaffected by time-of-day adjustments. The specific meaning and zero point of the
     * timestamp depends on the source providing images to the SurfaceTexture. Unless otherwise
     * specified by the image source, timestamps cannot generally be compared across SurfaceTexture
     * instances, or across multiple program invocations. It is mostly useful for determining time
     * offsets between subsequent frames.</p>
     *
     * <p>For camera sources, timestamps should be strictly monotonic. Timestamps from MediaPlayer
     * sources may be reset when the playback position is set. For EGL and Vulkan producers, the
     * timestamp is the desired present time set with the {@code EGL_ANDROID_presentation_time} or
     * {@code VK_GOOGLE_display_timing} extensions.</p>
     */

    public long getTimestamp() {
        return nativeGetTimestamp();
    }

    /**
     * Retrieve the dataspace associated with the texture image
     * set by the most recent call to {@link #updateTexImage}.
     */
    @SuppressLint("MethodNameUnits")
    public @NamedDataSpace int getDataSpace() {
        return nativeGetDataSpace();
    }

    /**
     * {@code release()} frees all the buffers and puts the SurfaceTexture into the
     * 'abandoned' state. Once put in this state the SurfaceTexture can never
     * leave it. When in the 'abandoned' state, all methods of the
     * {@code IGraphicBufferProducer} interface will fail with the {@code NO_INIT}
     * error.
     * <p>
     * Note that while calling this method causes all the buffers to be freed
     * from the perspective of the SurfaceTexture, if there are additional
     * references on the buffers (e.g. if a buffer is referenced by a client or
     * by OpenGL ES as a texture) then those buffer will remain allocated.
     * <p>
     * Always call this method when you are done with SurfaceTexture. Failing
     * to do so may delay resource deallocation for a significant amount of
     * time.
     *
     * @see #isReleased()
     */
    public void release() {
        nativeRelease();
    }

    /**
     * Returns {@code true} if the SurfaceTexture was released.
     *
     * @see #release()
     */
    public boolean isReleased() {
        return nativeIsReleased();
    }

    @Override
    protected void finalize() throws Throwable {
        try {
            nativeFinalize();
        } finally {
            super.finalize();
        }
    }

    /**
     * This method is invoked from native code only.
     */
    @SuppressWarnings({"UnusedDeclaration"})
    @UnsupportedAppUsage(maxTargetSdk = Build.VERSION_CODES.R, trackingBug = 170729553)
    private static void postEventFromNative(WeakReference<SurfaceTexture> weakSelf) {
        SurfaceTexture st = weakSelf.get();
        if (st != null) {
            Handler handler = st.mOnFrameAvailableHandler;
            if (handler != null) {
                handler.sendEmptyMessage(0);
            }
        }
    }

    /**
     * This method is invoked from native code only.
     * @hide
     */
    @SuppressWarnings({"UnusedDeclaration"})
    private static void postOnSetFrameRateEventFromNative(WeakReference<SurfaceTexture> weakSelf,
            @FloatRange(from = 0.0) float frameRate,
            @Surface.FrameRateCompatibility int compatibility,
            @Surface.ChangeFrameRateStrategy int changeFrameRateStrategy) {
        Trace.traceBegin(Trace.TRACE_TAG_VIEW, "postOnSetFrameRateEventFromNative");
        try {
            if (Flags.toolkitSetFrameRateReadOnly()) {
                SurfaceTexture st = weakSelf.get();
                if (st != null) {
                    Handler handler = st.mOnSetFrameRateHandler;
                    if (handler != null) {
                        Message msg = new Message();
                        msg.obj = new SetFrameRateArgs(frameRate, compatibility,
                                changeFrameRateStrategy);
                        handler.sendMessage(msg);
                    }
                }
            }
        } finally {
            Trace.traceEnd(Trace.TRACE_TAG_VIEW);
        }

    }

    /**
     * Returns {@code true} if the SurfaceTexture is single-buffered.
     * @hide
     */
    public boolean isSingleBuffered() {
        return mIsSingleBuffered;
    }

    private native void nativeInit(boolean isDetached, int texName,
            boolean singleBufferMode, WeakReference<SurfaceTexture> weakSelf)
            throws Surface.OutOfResourcesException;
    private native void nativeFinalize();
    private native void nativeGetTransformMatrix(float[] mtx);
    private native long nativeGetTimestamp();
    private native int nativeGetDataSpace();
    private native void nativeSetDefaultBufferSize(int width, int height);
    private native void nativeUpdateTexImage();
    private native void nativeReleaseTexImage();
    @UnsupportedAppUsage
    private native int nativeDetachFromGLContext();
    private native int nativeAttachToGLContext(int texName);
    private native void nativeRelease();
    private native boolean nativeIsReleased();
}