Implement Rotation in Android Renderer.
Make use of rotation information from the frame and rotate it accordingly when we render the frame.
BUG=4145
R=glaznev@webrtc.org
Committed: https://code.google.com/p/webrtc/source/detail?r=8770
Review URL: https://webrtc-codereview.appspot.com/50369004
Cr-Commit-Position: refs/heads/master@{#8781}
git-svn-id: http://webrtc.googlecode.com/svn/trunk@8781 4adac7df-926f-26a2-2b94-8c16560cd09d
diff --git a/talk/app/webrtc/androidtests/src/org/webrtc/VideoCapturerAndroidTest.java b/talk/app/webrtc/androidtests/src/org/webrtc/VideoCapturerAndroidTest.java
index 688ce96..24294ab 100644
--- a/talk/app/webrtc/androidtests/src/org/webrtc/VideoCapturerAndroidTest.java
+++ b/talk/app/webrtc/androidtests/src/org/webrtc/VideoCapturerAndroidTest.java
@@ -52,6 +52,12 @@
}
}
+ // TODO(guoweis): Remove this once chrome code base is updated.
+ @Override
+ public boolean canApplyRotation() {
+ return false;
+ }
+
public int WaitForNextFrameToRender() throws InterruptedException {
synchronized (frameLock) {
frameLock.wait();
diff --git a/talk/app/webrtc/java/android/org/webrtc/VideoRendererGui.java b/talk/app/webrtc/java/android/org/webrtc/VideoRendererGui.java
index ef0fbc6..4ad8ccc 100644
--- a/talk/app/webrtc/java/android/org/webrtc/VideoRendererGui.java
+++ b/talk/app/webrtc/java/android/org/webrtc/VideoRendererGui.java
@@ -37,6 +37,7 @@
import javax.microedition.khronos.egl.EGLConfig;
import javax.microedition.khronos.opengles.GL10;
+import android.annotation.SuppressLint;
import android.graphics.SurfaceTexture;
import android.opengl.EGL14;
import android.opengl.EGLContext;
@@ -264,6 +265,22 @@
private int videoWidth;
private int videoHeight;
+ // This is the degree that the frame should be rotated clockwisely to have
+ // it rendered up right.
+ private int rotationDegree;
+
+ // Mapping array from original UV mapping to the rotated mapping. The number
+ // is the position where the original UV coordination should be mapped
+ // to. (0,1) is the left top coord. (2,3) is the top bottom. (4,5) is the
+ // right top. (6,7) is the right bottom. Note that this is the coordination
+ // that got rotated. For example, using the original left bottom (2,3) as
+ // the top left (0,1) means 90 degree clockwise rotation.
+ private static int rotation_matrix[][] =
+ { {0, 1, 2, 3, 4, 5, 6, 7}, // 0 degree
+ {2, 3, 6, 7, 0, 1, 4, 5}, // 90 degree (clockwise)
+ {6, 7, 4, 5, 2, 3, 0, 1}, // 180 degree (clockwise)
+ {4, 5, 0, 1, 6, 7, 2, 3} };// 270 degree (clockwise)
+
private YuvImageRenderer(
GLSurfaceView surface, int id,
int x, int y, int width, int height,
@@ -292,6 +309,7 @@
};
textureCoords = directNativeFloatBuffer(textureCoordinatesFloat);
updateTextureProperties = false;
+ rotationDegree = 0;
}
private void createTextures(int yuvProgram, int oesProgram) {
@@ -340,7 +358,13 @@
if (displayWidth > 1 && displayHeight > 1 &&
videoWidth > 1 && videoHeight > 1) {
float displayAspectRatio = displayWidth / displayHeight;
- float videoAspectRatio = (float)videoWidth / videoHeight;
+ // videoAspectRatio should be the one after rotation applied.
+ float videoAspectRatio = 0;
+ if (rotationDegree == 90 || rotationDegree == 270) {
+ videoAspectRatio = (float)videoHeight / videoWidth;
+ } else {
+ videoAspectRatio = (float)videoWidth / videoHeight;
+ }
if (scalingType == ScalingType.SCALE_ASPECT_FIT) {
// Need to re-adjust vertices width or height to match video AR.
if (displayAspectRatio > videoAspectRatio) {
@@ -389,7 +413,21 @@
uRight, texOffsetV, // right top
uRight, 1.0f - texOffsetV // right bottom
};
- textureCoords = directNativeFloatBuffer(textureCoordinatesFloat);
+
+ float textureCoordinatesRotatedFloat[];
+ if (rotationDegree == 0) {
+ textureCoordinatesRotatedFloat = textureCoordinatesFloat;
+ } else {
+ textureCoordinatesRotatedFloat =
+ new float[textureCoordinatesFloat.length];
+ int index = rotationDegree / 90;
+ for(int i = 0; i < textureCoordinatesFloat.length; i++) {
+ textureCoordinatesRotatedFloat[rotation_matrix[index][i]] =
+ textureCoordinatesFloat[i];
+ }
+ }
+ textureCoords =
+ directNativeFloatBuffer(textureCoordinatesRotatedFloat);
}
updateTextureProperties = false;
}
@@ -528,16 +566,19 @@
}
}
- private void setSize(final int width, final int height) {
- if (width == videoWidth && height == videoHeight) {
- return;
+ private void setSize(final int width, final int height,
+ final int rotation) {
+ if (width == videoWidth && height == videoHeight &&
+ rotation == rotationDegree) {
+ return;
}
Log.d(TAG, "ID: " + id + ". YuvImageRenderer.setSize: " +
- width + " x " + height);
+ width + " x " + height + " rotation " + rotation);
videoWidth = width;
videoHeight = height;
+ rotationDegree = rotation;
int[] strides = { width, width / 2, width / 2 };
// Frame re-allocation need to be synchronized with copying
// frame to textures in draw() function to avoid re-allocating
@@ -546,15 +587,17 @@
// Clear rendering queue.
frameToRenderQueue.poll();
// Re-allocate / allocate the frame.
- yuvFrameToRender = new I420Frame(width, height, strides, null);
- textureFrameToRender = new I420Frame(width, height, null, -1);
+ yuvFrameToRender = new I420Frame(width, height, rotationDegree,
+ strides, null);
+ textureFrameToRender = new I420Frame(width, height, rotationDegree,
+ null, -1);
updateTextureProperties = true;
}
}
@Override
public synchronized void renderFrame(I420Frame frame) {
- setSize(frame.width, frame.height);
+ setSize(frame.width, frame.height, frame.rotationDegree);
long now = System.nanoTime();
framesReceived++;
// Skip rendering of this frame if setSize() was not called.
@@ -602,6 +645,11 @@
surface.requestRender();
}
+ // TODO(guoweis): Remove this once chrome code base is updated.
+ @Override
+ public boolean canApplyRotation() {
+ return true;
+ }
}
/** Passes GLSurfaceView to video renderer. */
@@ -712,6 +760,7 @@
}
}
+ @SuppressLint("NewApi")
@Override
public void onSurfaceCreated(GL10 unused, EGLConfig config) {
Log.d(TAG, "VideoRendererGui.onSurfaceCreated");
diff --git a/talk/app/webrtc/java/jni/peerconnection_jni.cc b/talk/app/webrtc/java/jni/peerconnection_jni.cc
index 6403a9e..7b4228b 100644
--- a/talk/app/webrtc/java/jni/peerconnection_jni.cc
+++ b/talk/app/webrtc/java/jni/peerconnection_jni.cc
@@ -709,6 +709,9 @@
renderer_->RenderFrame(frame);
}
+ // TODO(guoweis): Remove this once chrome code base is updated.
+ bool CanApplyRotation() override { return true; }
+
private:
explicit VideoRendererWrapper(cricket::VideoRenderer* renderer)
: renderer_(renderer), width_(0), height_(0) {}
@@ -725,14 +728,19 @@
j_render_frame_id_(GetMethodID(
jni, GetObjectClass(jni, j_callbacks), "renderFrame",
"(Lorg/webrtc/VideoRenderer$I420Frame;)V")),
+ j_can_apply_rotation_id_(GetMethodID(
+ jni, GetObjectClass(jni, j_callbacks),
+ "canApplyRotation", "()Z")),
j_frame_class_(jni,
FindClass(jni, "org/webrtc/VideoRenderer$I420Frame")),
j_i420_frame_ctor_id_(GetMethodID(
- jni, *j_frame_class_, "<init>", "(II[I[Ljava/nio/ByteBuffer;)V")),
+ jni, *j_frame_class_, "<init>", "(III[I[Ljava/nio/ByteBuffer;)V")),
j_texture_frame_ctor_id_(GetMethodID(
jni, *j_frame_class_, "<init>",
- "(IILjava/lang/Object;I)V")),
- j_byte_buffer_class_(jni, FindClass(jni, "java/nio/ByteBuffer")) {
+ "(IIILjava/lang/Object;I)V")),
+ j_byte_buffer_class_(jni, FindClass(jni, "java/nio/ByteBuffer")),
+ can_apply_rotation_set_(false),
+ can_apply_rotation_(false) {
CHECK_EXCEPTION(jni);
}
@@ -741,10 +749,12 @@
void RenderFrame(const cricket::VideoFrame* video_frame) override {
ScopedLocalRefFrame local_ref_frame(jni());
- // TODO(guoweis): Remove once the java implementation supports rotation.
- const cricket::VideoFrame* frame =
- video_frame->GetCopyWithRotationApplied();
+ // Calling CanApplyRotation here to ensure can_apply_rotation_ is set.
+ CanApplyRotation();
+ const cricket::VideoFrame* frame =
+ can_apply_rotation_ ? video_frame
+ : video_frame->GetCopyWithRotationApplied();
if (frame->GetNativeHandle() != NULL) {
jobject j_frame = CricketToJavaTextureFrame(frame);
jni()->CallVoidMethod(*j_callbacks_, j_render_frame_id_, j_frame);
@@ -756,6 +766,21 @@
}
}
+ // TODO(guoweis): Report that rotation is supported as RenderFrame calls
+ // GetCopyWithRotationApplied.
+ virtual bool CanApplyRotation() override {
+ if (can_apply_rotation_set_) {
+ return can_apply_rotation_;
+ }
+ ScopedLocalRefFrame local_ref_frame(jni());
+ jboolean ret =
+ jni()->CallBooleanMethod(*j_callbacks_, j_can_apply_rotation_id_);
+ CHECK_EXCEPTION(jni());
+ can_apply_rotation_ = ret;
+ can_apply_rotation_set_ = true;
+ return ret;
+ }
+
private:
// Return a VideoRenderer.I420Frame referring to the data in |frame|.
jobject CricketToJavaI420Frame(const cricket::VideoFrame* frame) {
@@ -778,7 +803,9 @@
jni()->SetObjectArrayElement(planes, 2, v_buffer);
return jni()->NewObject(
*j_frame_class_, j_i420_frame_ctor_id_,
- frame->GetWidth(), frame->GetHeight(), strides, planes);
+ frame->GetWidth(), frame->GetHeight(),
+ static_cast<int>(frame->GetVideoRotation()),
+ strides, planes);
}
// Return a VideoRenderer.I420Frame referring texture object in |frame|.
@@ -789,7 +816,9 @@
int texture_id = handle->GetTextureId();
return jni()->NewObject(
*j_frame_class_, j_texture_frame_ctor_id_,
- frame->GetWidth(), frame->GetHeight(), texture_object, texture_id);
+ frame->GetWidth(), frame->GetHeight(),
+ static_cast<int>(frame->GetVideoRotation()),
+ texture_object, texture_id);
}
JNIEnv* jni() {
@@ -798,10 +827,13 @@
ScopedGlobalRef<jobject> j_callbacks_;
jmethodID j_render_frame_id_;
+ jmethodID j_can_apply_rotation_id_;
ScopedGlobalRef<jclass> j_frame_class_;
jmethodID j_i420_frame_ctor_id_;
jmethodID j_texture_frame_ctor_id_;
ScopedGlobalRef<jclass> j_byte_buffer_class_;
+ bool can_apply_rotation_set_;
+ bool can_apply_rotation_;
};
diff --git a/talk/app/webrtc/java/src/org/webrtc/VideoRenderer.java b/talk/app/webrtc/java/src/org/webrtc/VideoRenderer.java
index 45fafa5..29b3743 100644
--- a/talk/app/webrtc/java/src/org/webrtc/VideoRenderer.java
+++ b/talk/app/webrtc/java/src/org/webrtc/VideoRenderer.java
@@ -47,13 +47,18 @@
public Object textureObject;
public int textureId;
+ // rotationDegree is the degree that the frame must be rotated clockwisely
+ // to be rendered correctly.
+ public int rotationDegree;
+
/**
* Construct a frame of the given dimensions with the specified planar
* data. If |yuvPlanes| is null, new planes of the appropriate sizes are
* allocated.
*/
public I420Frame(
- int width, int height, int[] yuvStrides, ByteBuffer[] yuvPlanes) {
+ int width, int height, int rotationDegree,
+ int[] yuvStrides, ByteBuffer[] yuvPlanes) {
this.width = width;
this.height = height;
this.yuvStrides = yuvStrides;
@@ -65,13 +70,15 @@
}
this.yuvPlanes = yuvPlanes;
this.yuvFrame = true;
+ this.rotationDegree = rotationDegree;
}
/**
* Construct a texture frame of the given dimensions with data in SurfaceTexture
*/
public I420Frame(
- int width, int height, Object textureObject, int textureId) {
+ int width, int height, int rotationDegree,
+ Object textureObject, int textureId) {
this.width = width;
this.height = height;
this.yuvStrides = null;
@@ -79,6 +86,7 @@
this.textureObject = textureObject;
this.textureId = textureId;
this.yuvFrame = false;
+ this.rotationDegree = rotationDegree;
}
/**
@@ -98,10 +106,12 @@
source.yuvStrides[1], yuvPlanes[1], yuvStrides[1]);
nativeCopyPlane(source.yuvPlanes[2], width / 2, height / 2,
source.yuvStrides[2], yuvPlanes[2], yuvStrides[2]);
+ rotationDegree = source.rotationDegree;
return this;
} else if (!source.yuvFrame && !yuvFrame) {
textureObject = source.textureObject;
textureId = source.textureId;
+ rotationDegree = source.rotationDegree;
return this;
} else {
throw new RuntimeException("Mismatched frame types! Source: " +
@@ -109,7 +119,7 @@
}
}
- public I420Frame copyFrom(byte[] yuvData) {
+ public I420Frame copyFrom(byte[] yuvData, int rotationDegree) {
if (yuvData.length < width * height * 3 / 2) {
throw new RuntimeException("Wrong arrays size: " + yuvData.length);
}
@@ -128,6 +138,7 @@
yuvPlanes[i].position(0);
yuvPlanes[i].limit(yuvPlanes[i].capacity());
}
+ this.rotationDegree = rotationDegree;
return this;
}
@@ -147,6 +158,8 @@
// |frame| might have pending rotation and implementation of Callbacks
// should handle that by applying rotation during rendering.
public void renderFrame(I420Frame frame);
+ // TODO(guoweis): Remove this once chrome code base is updated.
+ public boolean canApplyRotation();
}
// |this| either wraps a native (GUI) renderer or a client-supplied Callbacks
diff --git a/talk/app/webrtc/java/testcommon/src/org/webrtc/PeerConnectionTest.java b/talk/app/webrtc/java/testcommon/src/org/webrtc/PeerConnectionTest.java
index 81100b2..224225c 100644
--- a/talk/app/webrtc/java/testcommon/src/org/webrtc/PeerConnectionTest.java
+++ b/talk/app/webrtc/java/testcommon/src/org/webrtc/PeerConnectionTest.java
@@ -138,6 +138,12 @@
--expectedFramesDelivered;
}
+ // TODO(guoweis): Remove this once chrome code base is updated.
+ @Override
+ public boolean canApplyRotation() {
+ return false;
+ }
+
public synchronized void expectSignalingChange(SignalingState newState) {
expectedSignalingChanges.add(newState);
}
@@ -438,6 +444,12 @@
public void renderFrame(VideoRenderer.I420Frame frame) {
++numFramesDelivered;
}
+
+ // TODO(guoweis): Remove this once chrome code base is updated.
+ @Override
+ public boolean canApplyRotation() {
+ return false;
+ }
}
private static VideoRenderer createVideoRenderer(
diff --git a/talk/examples/androidtests/src/org/appspot/apprtc/test/PeerConnectionClientTest.java b/talk/examples/androidtests/src/org/appspot/apprtc/test/PeerConnectionClientTest.java
index 55e8bdf..f7dadba 100644
--- a/talk/examples/androidtests/src/org/appspot/apprtc/test/PeerConnectionClientTest.java
+++ b/talk/examples/androidtests/src/org/appspot/apprtc/test/PeerConnectionClientTest.java
@@ -101,6 +101,12 @@
doneRendering = new CountDownLatch(expectedFrames);
}
+ // TODO(guoweis): Remove this once chrome code base is updated.
+ @Override
+ public boolean canApplyRotation() {
+ return false;
+ }
+
@Override
public synchronized void renderFrame(VideoRenderer.I420Frame frame) {
if (!renderFrameCalled) {
