Scale pointer coordinates to screen size from the webRTC client
The webRTC client has a video tag with a fixed size of 360x720 and
sent the pointer events to the server in that scale. The server then
scaled those cordinates to a hardcoded size of 720x1440 and sent that
to the guest. No further scaling was performed so other screen sizes
were not properly supported. Even if the final events were to be
scaled to the screen size this wouldn't work well the screen's aspect
ration was other than 1:2 since the video stream in the client won't
necessary occupy the entire height or width of the element capturing
the events.
With this change the webRTC client scales the event coordinates to the
resolution of the video stream, which matches the cf screen size. By
doing this in the client it can take into consideration the fact that
the video won't scale evenly when it has different aspect ratios.
Bug: 141887532
Test: locally
Change-Id: I80ba8496bde3aceb0563876e24bee268daa4a977
diff --git a/host/frontend/gcastv2/webrtc/MyWebSocketHandler.cpp b/host/frontend/gcastv2/webrtc/MyWebSocketHandler.cpp
index 3e38dfb..c0a7fb8 100644
--- a/host/frontend/gcastv2/webrtc/MyWebSocketHandler.cpp
+++ b/host/frontend/gcastv2/webrtc/MyWebSocketHandler.cpp
@@ -248,8 +248,8 @@
sp<ABuffer> accessUnit = new ABuffer(3 * sizeof(int32_t));
int32_t *data = reinterpret_cast<int32_t *>(accessUnit->data());
data[0] = down;
- data[1] = (x * 720) / 360;
- data[2] = (y * 1440) / 720;
+ data[1] = x;
+ data[2] = y;
mTouchSink->onAccessUnit(accessUnit);
} else if (type == "inject-multi-touch") {
@@ -276,8 +276,8 @@
int32_t *data = reinterpret_cast<int32_t *>(accessUnit->data());
data[0] = id;
data[1] = (initialDown != 0);
- data[2] = (x * 720) / 360;
- data[3] = (y * 1440) / 720;
+ data[2] = x;
+ data[3] = y;
data[4] = slot;
mTouchSink->onAccessUnit(accessUnit);
diff --git a/host/frontend/gcastv2/webrtc/assets/js/receive.js b/host/frontend/gcastv2/webrtc/assets/js/receive.js
index a86ba93..25b1c71 100644
--- a/host/frontend/gcastv2/webrtc/assets/js/receive.js
+++ b/host/frontend/gcastv2/webrtc/assets/js/receive.js
@@ -429,8 +429,35 @@
}
function sendMouseUpdate(down, e) {
- const x = e.pageX - 7;
- const y = e.pageY - 46;
+ var x = e.offsetX;
+ var y = e.offsetY;
+
+ const videoWidth = videoElement.videoWidth;
+ const videoHeight = videoElement.videoHeight;
+ const elementWidth = videoElement.width;
+ const elementHeight = videoElement.height;
+
+ // vh*ew > eh*vw? then scale h instead of w
+ const scaleHeight = videoHeight * elementWidth > videoWidth * elementHeight;
+ var elementScaling = 0, videoScaling = 0;
+ if (scaleHeight) {
+ elementScaling = elementHeight;
+ videoScaling = videoHeight;
+ } else {
+ elementScaling = elementWidth;
+ videoScaling = videoWidth;
+ }
+
+ // Substract the offset produced by the difference in aspect ratio if any.
+ if (scaleHeight) {
+ x -= (elementWidth - elementScaling * videoWidth / videoScaling) / 2;
+ } else {
+ y -= (elementHeight - elementScaling * videoHeight / videoScaling) / 2;
+ }
+
+ // Convert to coordinates relative to the video
+ x = videoScaling * x / elementScaling;
+ y = videoScaling * y / elementScaling;
ws.send('{\r\n'
+ '"type": "set-mouse-position",\r\n'
