Android VideoRendererGui: Refactor GLES rendering
This CL should not change any visible behaviour. It does the following:
* Extract GLES rendering into separate class GlRectDrawer. This class is also needed for future video encode with OES texture input.
* Clean up current ScalingType -> display size calculation and introduce new SCALE_ASPECT_BALANCED (b/21735609) and remove unused SCALE_FILL.
* Replace current mirror/rotation index juggling with android.opengl.Matrix operations instead.
Review URL: https://codereview.webrtc.org/1191243005
Cr-Commit-Position: refs/heads/master@{#9496}
diff --git a/talk/app/webrtc/java/android/org/webrtc/GlRectDrawer.java b/talk/app/webrtc/java/android/org/webrtc/GlRectDrawer.java
new file mode 100644
index 0000000..abd0ddf
--- /dev/null
+++ b/talk/app/webrtc/java/android/org/webrtc/GlRectDrawer.java
@@ -0,0 +1,205 @@
+/*
+ * libjingle
+ * Copyright 2015 Google Inc.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ * 1. Redistributions of source code must retain the above copyright notice,
+ * this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright notice,
+ * this list of conditions and the following disclaimer in the documentation
+ * and/or other materials provided with the distribution.
+ * 3. The name of the author may not be used to endorse or promote products
+ * derived from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR IMPLIED
+ * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
+ * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO
+ * EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+ * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
+ * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
+ * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
+ * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
+ * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+package org.webrtc;
+
+import android.opengl.GLES11Ext;
+import android.opengl.GLES20;
+
+import org.webrtc.GlShader;
+import org.webrtc.GlUtil;
+
+import java.nio.ByteBuffer;
+import java.nio.FloatBuffer;
+import java.util.Arrays;
+
+/**
+ * Helper class to draw a quad that covers the entire viewport. Rotation, mirror, and cropping is
+ * specified using a 4x4 texture coordinate transform matrix. The frame input can either be an OES
+ * texture or YUV textures in I420 format. The GL state must be preserved between draw calls, this
+ * is intentional to maximize performance. The function release() must be called manually to free
+ * the resources held by this object.
+ */
+public class GlRectDrawer {
+ // Simple vertex shader, used for both YUV and OES.
+ private static final String VERTEX_SHADER_STRING =
+ "varying vec2 interp_tc;\n"
+ + "attribute vec4 in_pos;\n"
+ + "attribute vec4 in_tc;\n"
+ + "\n"
+ + "uniform mat4 texMatrix;\n"
+ + "\n"
+ + "void main() {\n"
+ + " gl_Position = in_pos;\n"
+ + " interp_tc = (texMatrix * in_tc).xy;\n"
+ + "}\n";
+
+ private static final String YUV_FRAGMENT_SHADER_STRING =
+ "precision mediump float;\n"
+ + "varying vec2 interp_tc;\n"
+ + "\n"
+ + "uniform sampler2D y_tex;\n"
+ + "uniform sampler2D u_tex;\n"
+ + "uniform sampler2D v_tex;\n"
+ + "\n"
+ + "void main() {\n"
+ // CSC according to http://www.fourcc.org/fccyvrgb.php
+ + " float y = texture2D(y_tex, interp_tc).r;\n"
+ + " float u = texture2D(u_tex, interp_tc).r - 0.5;\n"
+ + " float v = texture2D(v_tex, interp_tc).r - 0.5;\n"
+ + " gl_FragColor = vec4(y + 1.403 * v, "
+ + " y - 0.344 * u - 0.714 * v, "
+ + " y + 1.77 * u, 1);\n"
+ + "}\n";
+
+ private static final String OES_FRAGMENT_SHADER_STRING =
+ "#extension GL_OES_EGL_image_external : require\n"
+ + "precision mediump float;\n"
+ + "varying vec2 interp_tc;\n"
+ + "\n"
+ + "uniform samplerExternalOES oes_tex;\n"
+ + "\n"
+ + "void main() {\n"
+ + " gl_FragColor = texture2D(oes_tex, interp_tc);\n"
+ + "}\n";
+
+ private static final FloatBuffer FULL_RECTANGLE_BUF =
+ GlUtil.createFloatBuffer(new float[] {
+ -1.0f, -1.0f, // Bottom left.
+ 1.0f, -1.0f, // Bottom right.
+ -1.0f, 1.0f, // Top left.
+ 1.0f, 1.0f, // Top right.
+ });
+
+ private static final FloatBuffer FULL_RECTANGLE_TEX_BUF =
+ GlUtil.createFloatBuffer(new float[] {
+ 0.0f, 0.0f, // Bottom left.
+ 1.0f, 0.0f, // Bottom right.
+ 0.0f, 1.0f, // Top left.
+ 1.0f, 1.0f // Top right.
+ });
+
+ private GlShader oesShader;
+ private GlShader yuvShader;
+ private GlShader currentShader;
+ private float[] currentTexMatrix;
+ private int texMatrixLocation;
+
+ private void initGeometry(GlShader shader) {
+ shader.setVertexAttribArray("in_pos", 2, FULL_RECTANGLE_BUF);
+ shader.setVertexAttribArray("in_tc", 2, FULL_RECTANGLE_TEX_BUF);
+ }
+
+ /**
+ * Draw an OES texture frame with specified texture transformation matrix. Required resources are
+ * allocated at the first call to this function.
+ */
+ public void drawOes(int oesTextureId, float[] texMatrix) {
+ // Lazy allocation.
+ if (oesShader == null) {
+ oesShader = new GlShader(VERTEX_SHADER_STRING, OES_FRAGMENT_SHADER_STRING);
+ oesShader.useProgram();
+ initGeometry(oesShader);
+ }
+
+ // Set GLES state to OES.
+ if (currentShader != oesShader) {
+ currentShader = oesShader;
+ oesShader.useProgram();
+ GLES20.glActiveTexture(GLES20.GL_TEXTURE0);
+ currentTexMatrix = null;
+ texMatrixLocation = oesShader.getUniformLocation("texMatrix");
+ }
+
+ // updateTexImage() may be called from another thread in another EGL context, so we need to
+ // bind/unbind the texture in each draw call so that GLES understads it's a new texture.
+ GLES20.glBindTexture(GLES11Ext.GL_TEXTURE_EXTERNAL_OES, oesTextureId);
+ drawRectangle(texMatrix);
+ GLES20.glBindTexture(GLES11Ext.GL_TEXTURE_EXTERNAL_OES, 0);
+ }
+
+ /**
+ * Draw a YUV frame with specified texture transformation matrix. Required resources are
+ * allocated at the first call to this function.
+ */
+ public void drawYuv(int width, int height, int[] yuvTextures, float[] texMatrix) {
+ // Lazy allocation.
+ if (yuvShader == null) {
+ yuvShader = new GlShader(VERTEX_SHADER_STRING, YUV_FRAGMENT_SHADER_STRING);
+ yuvShader.useProgram();
+
+ // Set texture samplers.
+ GLES20.glUniform1i(yuvShader.getUniformLocation("y_tex"), 0);
+ GLES20.glUniform1i(yuvShader.getUniformLocation("u_tex"), 1);
+ GLES20.glUniform1i(yuvShader.getUniformLocation("v_tex"), 2);
+ GlUtil.checkNoGLES2Error("y/u/v_tex glGetUniformLocation");
+
+ initGeometry(yuvShader);
+ }
+
+ // Set GLES state to YUV.
+ if (currentShader != yuvShader) {
+ currentShader = yuvShader;
+ yuvShader.useProgram();
+ currentTexMatrix = null;
+ texMatrixLocation = yuvShader.getUniformLocation("texMatrix");
+ }
+
+ // Bind the textures.
+ for (int i = 0; i < 3; ++i) {
+ GLES20.glActiveTexture(GLES20.GL_TEXTURE0 + i);
+ GLES20.glBindTexture(GLES20.GL_TEXTURE_2D, yuvTextures[i]);
+ }
+
+ drawRectangle(texMatrix);
+ }
+
+ private void drawRectangle(float[] texMatrix) {
+ // Try avoid uploading the texture if possible.
+ if (!Arrays.equals(currentTexMatrix, texMatrix)) {
+ currentTexMatrix = texMatrix.clone();
+ // Copy the texture transformation matrix over.
+ GLES20.glUniformMatrix4fv(texMatrixLocation, 1, false, texMatrix, 0);
+ }
+ // Draw quad.
+ GLES20.glDrawArrays(GLES20.GL_TRIANGLE_STRIP, 0, 4);
+ }
+
+ /**
+ * Release all GLES resources. This needs to be done manually, otherwise the resources are leaked.
+ */
+ public void release() {
+ if (oesShader != null) {
+ oesShader.release();
+ oesShader = null;
+ }
+ if (yuvShader != null) {
+ yuvShader.release();
+ yuvShader = null;
+ }
+ }
+}
diff --git a/talk/app/webrtc/java/android/org/webrtc/GlShader.java b/talk/app/webrtc/java/android/org/webrtc/GlShader.java
index 712af2b..3014aab 100644
--- a/talk/app/webrtc/java/android/org/webrtc/GlShader.java
+++ b/talk/app/webrtc/java/android/org/webrtc/GlShader.java
@@ -30,6 +30,8 @@
import android.opengl.GLES20;
import android.util.Log;
+import java.nio.FloatBuffer;
+
// Helper class for handling OpenGL shaders and shader programs.
public class GlShader {
private static final String TAG = "GlShader";
@@ -88,6 +90,20 @@
return location;
}
+ /**
+ * Enable and upload a vertex array for attribute |label|. The vertex data is specified in
+ * |buffer| with |dimension| number of components per vertex.
+ */
+ public void setVertexAttribArray(String label, int dimension, FloatBuffer buffer) {
+ if (program == -1) {
+ throw new RuntimeException("The program has been released");
+ }
+ int location = getAttribLocation(label);
+ GLES20.glEnableVertexAttribArray(location);
+ GLES20.glVertexAttribPointer(location, dimension, GLES20.GL_FLOAT, false, 0, buffer);
+ GlUtil.checkNoGLES2Error("setVertexAttribArray");
+ }
+
public int getUniformLocation(String label) {
if (program == -1) {
throw new RuntimeException("The program has been released");
diff --git a/talk/app/webrtc/java/android/org/webrtc/VideoRendererGui.java b/talk/app/webrtc/java/android/org/webrtc/VideoRendererGui.java
index 7e85b10..0c910f1 100644
--- a/talk/app/webrtc/java/android/org/webrtc/VideoRendererGui.java
+++ b/talk/app/webrtc/java/android/org/webrtc/VideoRendererGui.java
@@ -27,9 +27,6 @@
package org.webrtc;
-import java.nio.ByteBuffer;
-import java.nio.ByteOrder;
-import java.nio.FloatBuffer;
import java.util.ArrayList;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.LinkedBlockingQueue;
@@ -38,12 +35,14 @@
import javax.microedition.khronos.opengles.GL10;
import android.annotation.SuppressLint;
+import android.graphics.Point;
+import android.graphics.Rect;
import android.graphics.SurfaceTexture;
import android.opengl.EGL14;
import android.opengl.EGLContext;
-import android.opengl.GLES11Ext;
import android.opengl.GLES20;
import android.opengl.GLSurfaceView;
+import android.opengl.Matrix;
import android.util.Log;
import org.webrtc.VideoRenderer.I420Frame;
@@ -70,65 +69,27 @@
private int screenHeight;
// List of yuv renderers.
private ArrayList<YuvImageRenderer> yuvImageRenderers;
- private GlShader yuvShader;
- private GlShader oesShader;
+ private GlRectDrawer drawer;
+ // The minimum fraction of the frame content that will be shown for |SCALE_ASPECT_BALANCED|.
+ // This limits excessive cropping when adjusting display size.
+ private static float BALANCED_VISIBLE_FRACTION = 0.56f;
// Types of video scaling:
// SCALE_ASPECT_FIT - video frame is scaled to fit the size of the view by
// maintaining the aspect ratio (black borders may be displayed).
// SCALE_ASPECT_FILL - video frame is scaled to fill the size of the view by
// maintaining the aspect ratio. Some portion of the video frame may be
// clipped.
- // SCALE_FILL - video frame is scaled to to fill the size of the view. Video
- // aspect ratio is changed if necessary.
+ // SCALE_ASPECT_BALANCED - Compromise between FIT and FILL. Video frame will fill as much as
+ // possible of the view while maintaining aspect ratio, under the constraint that at least
+ // |BALANCED_VISIBLE_FRACTION| of the frame content will be shown.
public static enum ScalingType
- { SCALE_ASPECT_FIT, SCALE_ASPECT_FILL, SCALE_FILL };
+ { SCALE_ASPECT_FIT, SCALE_ASPECT_FILL, SCALE_ASPECT_BALANCED }
private static final int EGL14_SDK_VERSION =
android.os.Build.VERSION_CODES.JELLY_BEAN_MR1;
// Current SDK version.
private static final int CURRENT_SDK_VERSION =
android.os.Build.VERSION.SDK_INT;
- private final String VERTEX_SHADER_STRING =
- "varying vec2 interp_tc;\n" +
- "attribute vec4 in_pos;\n" +
- "attribute vec2 in_tc;\n" +
- "\n" +
- "void main() {\n" +
- " gl_Position = in_pos;\n" +
- " interp_tc = in_tc;\n" +
- "}\n";
-
- private final String YUV_FRAGMENT_SHADER_STRING =
- "precision mediump float;\n" +
- "varying vec2 interp_tc;\n" +
- "\n" +
- "uniform sampler2D y_tex;\n" +
- "uniform sampler2D u_tex;\n" +
- "uniform sampler2D v_tex;\n" +
- "\n" +
- "void main() {\n" +
- // CSC according to http://www.fourcc.org/fccyvrgb.php
- " float y = texture2D(y_tex, interp_tc).r;\n" +
- " float u = texture2D(u_tex, interp_tc).r - 0.5;\n" +
- " float v = texture2D(v_tex, interp_tc).r - 0.5;\n" +
- " gl_FragColor = vec4(y + 1.403 * v, " +
- " y - 0.344 * u - 0.714 * v, " +
- " y + 1.77 * u, 1);\n" +
- "}\n";
-
-
- private static final String OES_FRAGMENT_SHADER_STRING =
- "#extension GL_OES_EGL_image_external : require\n" +
- "precision mediump float;\n" +
- "varying vec2 interp_tc;\n" +
- "\n" +
- "uniform samplerExternalOES oes_tex;\n" +
- "\n" +
- "void main() {\n" +
- " gl_FragColor = texture2D(oes_tex, interp_tc);\n" +
- "}\n";
-
-
private VideoRendererGui(GLSurfaceView surface) {
this.surface = surface;
// Create an OpenGL ES 2.0 context.
@@ -148,8 +109,6 @@
private static class YuvImageRenderer implements VideoRenderer.Callbacks {
private GLSurfaceView surface;
private int id;
- private GlShader yuvShader;
- private GlShader oesShader;
private int[] yuvTextures = { -1, -1, -1 };
private int oesTexture = -1;
@@ -182,14 +141,13 @@
// Time in ns spent in renderFrame() function - including copying frame
// data to rendering planes.
private long copyTimeNs;
- // Texture vertices.
- private float texLeft;
- private float texRight;
- private float texTop;
- private float texBottom;
- private FloatBuffer textureVertices;
- // Texture UV coordinates.
- private FloatBuffer textureCoords;
+ // The allowed view area in percentage of screen size.
+ private final Rect layoutInPercentage;
+ // The actual view area in pixels. It is a centered subrectangle of the rectangle defined by
+ // |layoutInPercentage|.
+ private final Rect displayLayout = new Rect();
+ // Cached texture transformation matrix, calculated from current layout parameters.
+ private final float[] texMatrix = new float[16];
// Flag if texture vertices or coordinates update is needed.
private boolean updateTextureProperties;
// Texture properties update lock.
@@ -205,23 +163,6 @@
// it rendered up right.
private int rotationDegree;
- // Mapping array from original UV mapping to the rotated mapping. The number
- // is the position where the original UV coordination should be mapped
- // to. (0,1) is the top left coord. (2,3) is the bottom left. (4,5) is the
- // top right. (6,7) is the bottom right.
- private static int rotation_matrix[][] =
- // 0 1 2 3 4 5 6 7 // arrays indices
- { {4, 5, 0, 1, 6, 7, 2, 3}, // 90 degree (clockwise)
- {6, 7, 4, 5, 2, 3, 0, 1}, // 180 degree (clockwise)
- {2, 3, 6, 7, 0, 1, 4, 5} }; // 270 degree (clockwise)
-
- private static int mirror_matrix[][] =
- // 0 1 2 3 4 5 6 7 // arrays indices
- { {4, 1, 6, 3, 0, 5, 2, 7}, // 0 degree mirror - u swap
- {0, 5, 2, 7, 4, 1, 6, 3}, // 90 degree mirror - v swap
- {4, 1, 6, 3, 0, 5, 2, 7}, // 180 degree mirror - u swap
- {0, 5, 2, 7, 4, 1, 6, 3} }; // 270 degree mirror - v swap
-
private YuvImageRenderer(
GLSurfaceView surface, int id,
int x, int y, int width, int height,
@@ -232,40 +173,20 @@
this.scalingType = scalingType;
this.mirror = mirror;
frameToRenderQueue = new LinkedBlockingQueue<I420Frame>(1);
- // Create texture vertices.
- texLeft = (x - 50) / 50.0f;
- texTop = (50 - y) / 50.0f;
- texRight = Math.min(1.0f, (x + width - 50) / 50.0f);
- texBottom = Math.max(-1.0f, (50 - y - height) / 50.0f);
- float textureVeticesFloat[] = new float[] {
- texLeft, texTop,
- texLeft, texBottom,
- texRight, texTop,
- texRight, texBottom
- };
- textureVertices = GlUtil.createFloatBuffer(textureVeticesFloat);
- // Create texture UV coordinates.
- float textureCoordinatesFloat[] = new float[] {
- 0, 0, 0, 1, 1, 0, 1, 1
- };
- textureCoords = GlUtil.createFloatBuffer(textureCoordinatesFloat);
+ layoutInPercentage = new Rect(x, y, Math.min(100, x + width), Math.min(100, y + height));
updateTextureProperties = false;
rotationDegree = 0;
}
- private void createTextures(GlShader yuvShader, GlShader oesShader) {
+ private void createTextures() {
Log.d(TAG, " YuvImageRenderer.createTextures " + id + " on GL thread:" +
Thread.currentThread().getId());
- this.yuvShader = yuvShader;
- this.oesShader = oesShader;
// Generate 3 texture ids for Y/U/V and place them into |yuvTextures|.
GLES20.glGenTextures(3, yuvTextures, 0);
for (int i = 0; i < 3; i++) {
GLES20.glActiveTexture(GLES20.GL_TEXTURE0 + i);
GLES20.glBindTexture(GLES20.GL_TEXTURE_2D, yuvTextures[i]);
- GLES20.glTexImage2D(GLES20.GL_TEXTURE_2D, 0, GLES20.GL_LUMINANCE,
- 128, 128, 0, GLES20.GL_LUMINANCE, GLES20.GL_UNSIGNED_BYTE, null);
GLES20.glTexParameterf(GLES20.GL_TEXTURE_2D,
GLES20.GL_TEXTURE_MIN_FILTER, GLES20.GL_LINEAR);
GLES20.glTexParameterf(GLES20.GL_TEXTURE_2D,
@@ -278,144 +199,99 @@
GlUtil.checkNoGLES2Error("y/u/v glGenTextures");
}
+ private static float convertScalingTypeToVisibleFraction(ScalingType scalingType) {
+ switch (scalingType) {
+ case SCALE_ASPECT_FIT:
+ return 1.0f;
+ case SCALE_ASPECT_FILL:
+ return 0.0f;
+ case SCALE_ASPECT_BALANCED:
+ return BALANCED_VISIBLE_FRACTION;
+ default:
+ throw new IllegalArgumentException();
+ }
+ }
+
+ private static Point getDisplaySize(float minVisibleFraction, float videoAspectRatio,
+ int maxDisplayWidth, int maxDisplayHeight) {
+ // If there is no constraint on the amount of cropping, fill the allowed display area.
+ if (minVisibleFraction == 0) {
+ return new Point(maxDisplayWidth, maxDisplayHeight);
+ }
+ // Each dimension is constrained on max display size and how much we are allowed to crop.
+ final int width = Math.min(maxDisplayWidth,
+ (int) (maxDisplayHeight / minVisibleFraction * videoAspectRatio));
+ final int height = Math.min(maxDisplayHeight,
+ (int) (maxDisplayWidth / minVisibleFraction / videoAspectRatio));
+ return new Point(width, height);
+ }
+
private void checkAdjustTextureCoords() {
synchronized(updateTextureLock) {
- if (!updateTextureProperties || scalingType == ScalingType.SCALE_FILL) {
+ if (!updateTextureProperties) {
return;
}
- // Re - calculate texture vertices to preserve video aspect ratio.
- float texRight = this.texRight;
- float texLeft = this.texLeft;
- float texTop = this.texTop;
- float texBottom = this.texBottom;
- float texOffsetU = 0;
- float texOffsetV = 0;
- float displayWidth = (texRight - texLeft) * screenWidth / 2;
- float displayHeight = (texTop - texBottom) * screenHeight / 2;
- Log.d(TAG, "ID: " + id + ". AdjustTextureCoords. Display: " + displayWidth +
- " x " + displayHeight + ". Video: " + videoWidth +
- " x " + videoHeight + ". Rotation: " + rotationDegree + ". Mirror: " + mirror);
- if (displayWidth > 1 && displayHeight > 1 &&
- videoWidth > 1 && videoHeight > 1) {
- float displayAspectRatio = displayWidth / displayHeight;
- // videoAspectRatio should be the one after rotation applied.
- float videoAspectRatio = 0;
- if (rotationDegree == 90 || rotationDegree == 270) {
- videoAspectRatio = (float)videoHeight / videoWidth;
- } else {
- videoAspectRatio = (float)videoWidth / videoHeight;
- }
- if (scalingType == ScalingType.SCALE_ASPECT_FIT) {
- // Need to re-adjust vertices width or height to match video AR.
- if (displayAspectRatio > videoAspectRatio) {
- float deltaX = (displayWidth - videoAspectRatio * displayHeight) /
- instance.screenWidth;
- texRight -= deltaX;
- texLeft += deltaX;
- } else {
- float deltaY = (displayHeight - displayWidth / videoAspectRatio) /
- instance.screenHeight;
- texTop -= deltaY;
- texBottom += deltaY;
- }
- }
- if (scalingType == ScalingType.SCALE_ASPECT_FILL) {
- // Need to re-adjust UV coordinates to match display AR.
- boolean adjustU = true;
- float ratio = 0;
- if (displayAspectRatio > videoAspectRatio) {
- ratio = (1.0f - videoAspectRatio / displayAspectRatio) /
- 2.0f;
- adjustU = (rotationDegree == 90 || rotationDegree == 270);
- } else {
- ratio = (1.0f - displayAspectRatio / videoAspectRatio) /
- 2.0f;
- adjustU = (rotationDegree == 0 || rotationDegree == 180);
- }
- if (adjustU) {
- texOffsetU = ratio;
- } else {
- texOffsetV = ratio;
- }
- }
- Log.d(TAG, " Texture vertices: (" + texLeft + "," + texBottom +
- ") - (" + texRight + "," + texTop + ")");
- float textureVeticesFloat[] = new float[] {
- texLeft, texTop,
- texLeft, texBottom,
- texRight, texTop,
- texRight, texBottom
- };
- textureVertices = GlUtil.createFloatBuffer(textureVeticesFloat);
-
- float uLeft = texOffsetU;
- float uRight = 1.0f - texOffsetU;
- float vTop = texOffsetV;
- float vBottom = 1.0f - texOffsetV;
- Log.d(TAG, " Texture UV: (" + uLeft + "," + vTop +
- ") - (" + uRight + "," + vBottom + ")");
- float textureCoordinatesFloat[] = new float[] {
- uLeft, vTop, // top left
- uLeft, vBottom, // bottom left
- uRight, vTop, // top right
- uRight, vBottom // bottom right
- };
-
- // Rotation needs to be done before mirroring.
- textureCoordinatesFloat = applyRotation(textureCoordinatesFloat,
- rotationDegree);
- textureCoordinatesFloat = applyMirror(textureCoordinatesFloat,
- mirror);
- textureCoords = GlUtil.createFloatBuffer(textureCoordinatesFloat);
+ // Initialize to maximum allowed area. Round to integer coordinates inwards the layout
+ // bounding box (ceil left/top and floor right/bottom) to not break constraints.
+ displayLayout.set(
+ (screenWidth * layoutInPercentage.left + 99) / 100,
+ (screenHeight * layoutInPercentage.top + 99) / 100,
+ (screenWidth * layoutInPercentage.right) / 100,
+ (screenHeight * layoutInPercentage.bottom) / 100);
+ Log.d(TAG, "ID: " + id + ". AdjustTextureCoords. Allowed display size: "
+ + displayLayout.width() + " x " + displayLayout.height() + ". Video: " + videoWidth
+ + " x " + videoHeight + ". Rotation: " + rotationDegree + ". Mirror: " + mirror);
+ final float videoAspectRatio = (rotationDegree % 180 == 0)
+ ? (float) videoWidth / videoHeight
+ : (float) videoHeight / videoWidth;
+ // Adjust display size based on |scalingType|.
+ final float minVisibleFraction = convertScalingTypeToVisibleFraction(scalingType);
+ final Point displaySize = getDisplaySize(minVisibleFraction, videoAspectRatio,
+ displayLayout.width(), displayLayout.height());
+ displayLayout.inset((displayLayout.width() - displaySize.x) / 2,
+ (displayLayout.height() - displaySize.y) / 2);
+ Log.d(TAG, " Adjusted display size: " + displayLayout.width() + " x "
+ + displayLayout.height());
+ // The matrix stack is using post-multiplication, which means that matrix operations:
+ // A; B; C; will end up as A * B * C. When you apply this to a vertex, it will result in:
+ // v' = A * B * C * v, i.e. the last matrix operation is the first thing that affects the
+ // vertex. This is the opposite of what you might expect.
+ Matrix.setIdentityM(texMatrix, 0);
+ // Move coordinates back to [0,1]x[0,1].
+ Matrix.translateM(texMatrix, 0, 0.5f, 0.5f, 0.0f);
+ // Rotate frame clockwise in the XY-plane (around the Z-axis).
+ Matrix.rotateM(texMatrix, 0, -rotationDegree, 0, 0, 1);
+ // Scale one dimension until video and display size have same aspect ratio.
+ final float displayAspectRatio = (float) displayLayout.width() / displayLayout.height();
+ if (displayAspectRatio > videoAspectRatio) {
+ Matrix.scaleM(texMatrix, 0, 1, videoAspectRatio / displayAspectRatio, 1);
+ } else {
+ Matrix.scaleM(texMatrix, 0, displayAspectRatio / videoAspectRatio, 1, 1);
}
+ // TODO(magjed): We currently ignore the texture transform matrix from the SurfaceTexture.
+ // It contains a vertical flip that is hardcoded here instead.
+ Matrix.scaleM(texMatrix, 0, 1, -1, 1);
+ // Apply optional horizontal flip.
+ if (mirror) {
+ Matrix.scaleM(texMatrix, 0, -1, 1, 1);
+ }
+ // Center coordinates around origin.
+ Matrix.translateM(texMatrix, 0, -0.5f, -0.5f, 0.0f);
updateTextureProperties = false;
Log.d(TAG, " AdjustTextureCoords done");
}
}
-
- private float[] applyMirror(float textureCoordinatesFloat[],
- boolean mirror) {
- if (!mirror) {
- return textureCoordinatesFloat;
- }
-
- int index = rotationDegree / 90;
- return applyMatrixOperation(textureCoordinatesFloat,
- mirror_matrix[index]);
- }
-
- private float[] applyRotation(float textureCoordinatesFloat[],
- int rotationDegree) {
- if (rotationDegree == 0) {
- return textureCoordinatesFloat;
- }
-
- int index = rotationDegree / 90 - 1;
- return applyMatrixOperation(textureCoordinatesFloat,
- rotation_matrix[index]);
- }
-
- private float[] applyMatrixOperation(float textureCoordinatesFloat[],
- int matrix_operation[]) {
- float textureCoordinatesModifiedFloat[] =
- new float[textureCoordinatesFloat.length];
-
- for(int i = 0; i < textureCoordinatesFloat.length; i++) {
- textureCoordinatesModifiedFloat[matrix_operation[i]] =
- textureCoordinatesFloat[i];
- }
- return textureCoordinatesModifiedFloat;
- }
-
- private void draw() {
+ private void draw(GlRectDrawer drawer) {
if (!seenFrame) {
// No frame received yet - nothing to render.
return;
}
long now = System.nanoTime();
- GlShader currentShader;
+ // OpenGL defaults to lower left origin.
+ GLES20.glViewport(displayLayout.left, screenHeight - displayLayout.bottom,
+ displayLayout.width(), displayLayout.height());
I420Frame frameFromQueue;
synchronized (frameToRenderQueue) {
@@ -428,33 +304,22 @@
startTimeNs = now;
}
- if (rendererType == RendererType.RENDERER_YUV) {
- // YUV textures rendering.
- yuvShader.useProgram();
- currentShader = yuvShader;
-
- for (int i = 0; i < 3; ++i) {
- GLES20.glActiveTexture(GLES20.GL_TEXTURE0 + i);
- GLES20.glBindTexture(GLES20.GL_TEXTURE_2D, yuvTextures[i]);
- if (frameFromQueue != null) {
- int w = (i == 0) ?
- frameFromQueue.width : frameFromQueue.width / 2;
- int h = (i == 0) ?
- frameFromQueue.height : frameFromQueue.height / 2;
+ if (frameFromQueue != null) {
+ if (frameFromQueue.yuvFrame) {
+ // YUV textures rendering. Upload YUV data as textures.
+ for (int i = 0; i < 3; ++i) {
+ GLES20.glActiveTexture(GLES20.GL_TEXTURE0 + i);
+ GLES20.glBindTexture(GLES20.GL_TEXTURE_2D, yuvTextures[i]);
+ int w = (i == 0) ? frameFromQueue.width : frameFromQueue.width / 2;
+ int h = (i == 0) ? frameFromQueue.height : frameFromQueue.height / 2;
GLES20.glTexImage2D(GLES20.GL_TEXTURE_2D, 0, GLES20.GL_LUMINANCE,
w, h, 0, GLES20.GL_LUMINANCE, GLES20.GL_UNSIGNED_BYTE,
frameFromQueue.yuvPlanes[i]);
}
- }
- GLES20.glUniform1i(yuvShader.getUniformLocation("y_tex"), 0);
- GLES20.glUniform1i(yuvShader.getUniformLocation("u_tex"), 1);
- GLES20.glUniform1i(yuvShader.getUniformLocation("v_tex"), 2);
- } else {
- // External texture rendering.
- oesShader.useProgram();
- currentShader = oesShader;
-
- if (frameFromQueue != null) {
+ } else {
+ // External texture rendering. Copy texture id and update texture image to latest.
+ // TODO(magjed): We should not make an unmanaged copy of texture id. Also, this is not
+ // the best place to call updateTexImage.
oesTexture = frameFromQueue.textureId;
if (frameFromQueue.textureObject instanceof SurfaceTexture) {
SurfaceTexture surfaceTexture =
@@ -462,31 +327,16 @@
surfaceTexture.updateTexImage();
}
}
- GLES20.glActiveTexture(GLES20.GL_TEXTURE0);
- GLES20.glBindTexture(GLES11Ext.GL_TEXTURE_EXTERNAL_OES, oesTexture);
- }
- if (frameFromQueue != null) {
frameToRenderQueue.poll();
}
}
- int posLocation = currentShader.getAttribLocation("in_pos");
- GLES20.glEnableVertexAttribArray(posLocation);
- GLES20.glVertexAttribPointer(
- posLocation, 2, GLES20.GL_FLOAT, false, 0, textureVertices);
-
- int texLocation = currentShader.getAttribLocation("in_tc");
- GLES20.glEnableVertexAttribArray(texLocation);
- GLES20.glVertexAttribPointer(
- texLocation, 2, GLES20.GL_FLOAT, false, 0, textureCoords);
-
- GLES20.glDrawArrays(GLES20.GL_TRIANGLE_STRIP, 0, 4);
-
- GLES20.glDisableVertexAttribArray(posLocation);
- GLES20.glDisableVertexAttribArray(texLocation);
-
- GlUtil.checkNoGLES2Error("draw done");
+ if (rendererType == RendererType.RENDERER_YUV) {
+ drawer.drawYuv(videoWidth, videoHeight, yuvTextures, texMatrix);
+ } else {
+ drawer.drawOes(oesTexture, texMatrix);
+ }
if (frameFromQueue != null) {
framesRendered++;
@@ -526,23 +376,17 @@
public void setPosition(int x, int y, int width, int height,
ScalingType scalingType, boolean mirror) {
- float texLeft = (x - 50) / 50.0f;
- float texTop = (50 - y) / 50.0f;
- float texRight = Math.min(1.0f, (x + width - 50) / 50.0f);
- float texBottom = Math.max(-1.0f, (50 - y - height) / 50.0f);
+ final Rect layoutInPercentage =
+ new Rect(x, y, Math.min(100, x + width), Math.min(100, y + height));
synchronized(updateTextureLock) {
- if (texLeft == this.texLeft && texTop == this.texTop && texRight == this.texRight &&
- texBottom == this.texBottom && scalingType == this.scalingType &&
- mirror == this.mirror) {
+ if (layoutInPercentage.equals(this.layoutInPercentage) && scalingType == this.scalingType
+ && mirror == this.mirror) {
return;
}
Log.d(TAG, "ID: " + id + ". YuvImageRenderer.setPosition: (" + x + ", " + y +
") " + width + " x " + height + ". Scaling: " + scalingType +
". Mirror: " + mirror);
- this.texLeft = texLeft;
- this.texTop = texTop;
- this.texRight = texRight;
- this.texBottom = texBottom;
+ this.layoutInPercentage.set(layoutInPercentage);
this.scalingType = scalingType;
this.mirror = mirror;
updateTextureProperties = true;
@@ -694,8 +538,7 @@
final CountDownLatch countDownLatch = new CountDownLatch(1);
instance.surface.queueEvent(new Runnable() {
public void run() {
- yuvImageRenderer.createTextures(
- instance.yuvShader, instance.oesShader);
+ yuvImageRenderer.createTextures();
yuvImageRenderer.setScreenSize(
instance.screenWidth, instance.screenHeight);
countDownLatch.countDown();
@@ -754,14 +597,13 @@
Log.d(TAG, "VideoRendererGui EGL Context: " + eglContext);
}
- // Create YUV and OES shaders.
- yuvShader = new GlShader(VERTEX_SHADER_STRING, YUV_FRAGMENT_SHADER_STRING);
- oesShader = new GlShader(VERTEX_SHADER_STRING, OES_FRAGMENT_SHADER_STRING);
+ // Create drawer for YUV/OES frames.
+ drawer = new GlRectDrawer();
synchronized (yuvImageRenderers) {
// Create textures for all images.
for (YuvImageRenderer yuvImageRenderer : yuvImageRenderers) {
- yuvImageRenderer.createTextures(yuvShader, oesShader);
+ yuvImageRenderer.createTextures();
}
onSurfaceCreatedCalled = true;
}
@@ -780,7 +622,6 @@
width + " x " + height + " ");
screenWidth = width;
screenHeight = height;
- GLES20.glViewport(0, 0, width, height);
synchronized (yuvImageRenderers) {
for (YuvImageRenderer yuvImageRenderer : yuvImageRenderers) {
yuvImageRenderer.setScreenSize(screenWidth, screenHeight);
@@ -790,10 +631,11 @@
@Override
public void onDrawFrame(GL10 unused) {
+ GLES20.glViewport(0, 0, screenWidth, screenHeight);
GLES20.glClear(GLES20.GL_COLOR_BUFFER_BIT);
synchronized (yuvImageRenderers) {
for (YuvImageRenderer yuvImageRenderer : yuvImageRenderers) {
- yuvImageRenderer.draw();
+ yuvImageRenderer.draw(drawer);
}
}
}
diff --git a/talk/libjingle.gyp b/talk/libjingle.gyp
index abeeb55..ed925e3 100755
--- a/talk/libjingle.gyp
+++ b/talk/libjingle.gyp
@@ -141,6 +141,7 @@
# and include it here.
'android_java_files': [
'app/webrtc/java/android/org/webrtc/EglBase.java',
+ 'app/webrtc/java/android/org/webrtc/GlRectDrawer.java',
'app/webrtc/java/android/org/webrtc/GlShader.java',
'app/webrtc/java/android/org/webrtc/GlUtil.java',
'app/webrtc/java/android/org/webrtc/VideoRendererGui.java',
