Merge changes from topic "Cuttlefish-sensor-injection" into main
* changes:
Implement Cuttlefish motion sensors UI
Inject Accelerometer and magnetometer values.
diff --git a/build/Android.bp b/build/Android.bp
index d517ac4..0f0a660 100644
--- a/build/Android.bp
+++ b/build/Android.bp
@@ -164,6 +164,7 @@
"webrtc_index.css",
"webrtc_controls.css",
"webrtc_trusted.pem",
+ "webrtc_sensors.js",
]
cvd_host_model_simulator_files = [
diff --git a/guest/commands/sensor_injection/main.cpp b/guest/commands/sensor_injection/main.cpp
index fe91387..ccef057 100644
--- a/guest/commands/sensor_injection/main.cpp
+++ b/guest/commands/sensor_injection/main.cpp
@@ -20,9 +20,9 @@
#include <android-base/chrono_utils.h>
#include <android-base/logging.h>
#include <android/binder_manager.h>
+#include <android-base/parsedouble.h>
#include <android-base/parseint.h>
#include <utils/SystemClock.h>
-
#include <aidl/android/hardware/sensors/BnSensors.h>
using aidl::android::hardware::sensors::Event;
@@ -106,6 +106,66 @@
endSensorInjection(sensors);
}
+// Inject accelerometer event based on rotation in device position.
+void InjectAccelerometer(double x, double y, double z) {
+ auto sensors = startSensorInjection();
+ int handle = getSensorHandle(SensorType::ACCELEROMETER, sensors);
+ Event event;
+ event.sensorHandle = handle;
+ event.sensorType = SensorType::ACCELEROMETER;
+
+ Event::EventPayload::Vec3 vec3;
+ vec3.x = x;
+ vec3.y = y;
+ vec3.z = z;
+ vec3.status = SensorStatus::ACCURACY_HIGH;
+ event.payload.set<Event::EventPayload::Tag::vec3>(vec3);
+ event.timestamp = android::elapsedRealtimeNano();
+ auto result = sensors->injectSensorData(event);
+ CHECK(result.isOk()) << "Unable to inject ISensors accelerometer event: "
+ << result.getDescription();
+}
+
+// Inject Magnetometer event based on rotation in device position.
+void InjectMagnetometer(double x, double y, double z) {
+ auto sensors = startSensorInjection();
+ int handle = getSensorHandle(SensorType::MAGNETIC_FIELD, sensors);
+ Event event;
+ event.sensorHandle = handle;
+ event.sensorType = SensorType::MAGNETIC_FIELD;
+
+ Event::EventPayload::Vec3 vec3;
+ vec3.x = x;
+ vec3.y = y;
+ vec3.z = z;
+ vec3.status = SensorStatus::ACCURACY_HIGH;
+ event.payload.set<Event::EventPayload::Tag::vec3>(vec3);
+ event.timestamp = android::elapsedRealtimeNano();
+ auto result = sensors->injectSensorData(event);
+ CHECK(result.isOk()) << "Unable to inject ISensors magnetometer event: "
+ << result.getDescription();
+}
+
+// Inject Gyroscope event based on rotation in device position.
+void InjectGyroscope(double x, double y, double z){
+ auto sensors = startSensorInjection();
+ int handle = getSensorHandle(SensorType::GYROSCOPE, sensors);
+ Event event;
+ event.sensorHandle = handle;
+ event.sensorType = SensorType::GYROSCOPE;
+
+ Event::EventPayload::Vec3 vec3;
+ vec3.x = x;
+ vec3.y = y;
+ vec3.z = z;
+ vec3.status = SensorStatus::ACCURACY_HIGH;
+ event.payload.set<Event::EventPayload::Tag::vec3>(vec3);
+ event.timestamp = android::elapsedRealtimeNano();
+ auto result = sensors->injectSensorData(event);
+ CHECK(result.isOk()) << "Unable to inject ISensors gyroscope event: "
+ << result.getDescription();
+}
+
// Inject a single HINGE_ANGLE event at the given angle.
void InjectHingeAngle(int angle) {
auto sensors = startSensorInjection();
@@ -126,11 +186,11 @@
}
int main(int argc, char** argv) {
- ::android::base::InitLogging(argv, android::base::LogdLogger(android::base::SYSTEM));
-
- CHECK(argc == 3)
- << "Expected command line args 'rotate <angle>' or 'hinge_angle <value>'";
-
+ ::android::base::InitLogging(
+ argv, android::base::LogdLogger(android::base::SYSTEM));
+ CHECK(argc == 3 || argc == 11)
+ << "Expected command line args 'rotate <angle>', 'hinge_angle <value>', or 'motion " <<
+ "<acc_x> <acc_y> <acc_z> <mgn_x> <mgn_y> <mgn_z> <gyro_x> <gyro_y> <gyro_z>'";
if (!strcmp(argv[1], "rotate")) {
int rotationDeg;
CHECK(android::base::ParseInt(argv[2], &rotationDeg))
@@ -142,6 +202,20 @@
<< "Hinge angle must be an integer";
CHECK(angle >= 0 && angle <= 360) << "Bad hinge_angle value: " << argv[2];
InjectHingeAngle(angle);
+ } else if (!strcmp(argv[1], "motion")) {
+ double acc_x, acc_y, acc_z, mgn_x, mgn_y, mgn_z, gyro_x, gyro_y, gyro_z;
+ CHECK(android::base::ParseDouble(argv[2], &acc_x)) << "Accelerometer x value must be a double";
+ CHECK(android::base::ParseDouble(argv[3], &acc_y)) << "Accelerometer x value must be a double";
+ CHECK(android::base::ParseDouble(argv[4], &acc_z)) << "Accelerometer x value must be a double";
+ CHECK(android::base::ParseDouble(argv[5], &mgn_x)) << "Magnetometer x value must be a double";
+ CHECK(android::base::ParseDouble(argv[6], &mgn_y)) << "Magnetometer y value must be a double";
+ CHECK(android::base::ParseDouble(argv[7], &mgn_z)) << "Magnetometer z value must be a double";
+ CHECK(android::base::ParseDouble(argv[8], &gyro_x)) << "Gyroscope x value must be a double";
+ CHECK(android::base::ParseDouble(argv[9], &gyro_y)) << "Gyroscope y value must be a double";
+ CHECK(android::base::ParseDouble(argv[10], &gyro_z)) << "Gyroscope z value must be a double";
+ InjectAccelerometer(acc_x, acc_y, acc_z);
+ InjectMagnetometer(mgn_x, mgn_y, mgn_z);
+ InjectGyroscope(gyro_x, gyro_y, gyro_z);
} else {
LOG(FATAL) << "Unknown arg: " << argv[1];
}
diff --git a/host/frontend/webrtc/html_client/Android.bp b/host/frontend/webrtc/html_client/Android.bp
index 2476f15..a7f6c7b 100644
--- a/host/frontend/webrtc/html_client/Android.bp
+++ b/host/frontend/webrtc/html_client/Android.bp
@@ -80,3 +80,9 @@
sub_dir: "webrtc/assets/js",
}
+prebuilt_usr_share_host {
+ name: "webrtc_sensors.js",
+ src: "js/sensors.js",
+ filename: "sensors.js",
+ sub_dir: "webrtc/assets/js",
+}
diff --git a/host/frontend/webrtc/html_client/client.html b/host/frontend/webrtc/html_client/client.html
index 2995a5a5..e51049c 100644
--- a/host/frontend/webrtc/html_client/client.html
+++ b/host/frontend/webrtc/html_client/client.html
@@ -51,6 +51,7 @@
<button id='mic_btn' title='Microphone' disabled='true' class='material-icons'>mic</button>
<button id='location-modal-button' title='location console' class='material-icons'>location_on</button>
<button id='device-details-button' title='Device Details' class='material-icons'>info</button>
+ <button id='rotation-modal-button' title='Rotation sensors' class='material-icons'>more_vert</button>
</div>
<div id='control-panel-custom-buttons' class='control-panel-column'></div>
<!-- tabindex="-1" allows this element to capture keyboard events -->
@@ -199,11 +200,51 @@
</div>
</div>
-
+ <div id='rotation-modal' class='modal'>
+ <div id='rotation-modal-header' class='modal-header'>
+ <h2>Rotation sensors</h2>
+ <button id='rotation-modal-close' title='Close' class='material-icons modal-close'>close</button>
+ </div>
+ <hr>
+ <h3>Rotate the device</h3>
+ <span id='rotation-bar'>
+ <div class='roation-slider'>
+ X
+ <input class='rotation-slider-range' type='range' value='0' min='-180' max='180' step='0.1'>
+ <span class='rotation-slider-value'>0</span>
+ </div>
+ <br>
+ <div class='rotation-slider'>
+ Y
+ <input class='rotation-slider-range' type='range' value='0' min='-180' max='180' step='0.1'>
+ <span class='rotation-slider-value'>0</span>
+ </div>
+ <br>
+ <div class='rotation-slider'>
+ Z
+ <input class='rotation-slider-range' type='range' value='0' min='-180' max='180' step='0.1'>
+ <span class='rotation-slider-value'>0</span>
+ </div>
+ <br>
+ </span>
+ <div class='sensors'>
+ <div id='accelerometer'>
+ Accelerometer:
+ <span id='accelerometer-value'>0.00 9.81 0.00</span>
+ </div>
+ <div id='magnetometer'>Magnetometer:
+ <span id='magnetometer-value'>0 5.9 -48.4</span>
+ </div>
+ <div id='gyroscope'>Gyroscope:
+ <span id='gyroscope-value'>0.00 0.00 0.00</span>
+ </div>
+ </div>
+ </div>
<script src="js/adb.js"></script>
<script src="js/location.js"></script>
<script src="js/rootcanal.js"></script>
<script src="js/cf_webrtc.js" type="module"></script>
+ <script src="js/sensors.js"></script>
<script src="js/controls.js"></script>
<script src="js/app.js"></script>
<template id="display-template">
diff --git a/host/frontend/webrtc/html_client/js/app.js b/host/frontend/webrtc/html_client/js/app.js
index 03f2d13..f1e8360 100644
--- a/host/frontend/webrtc/html_client/js/app.js
+++ b/host/frontend/webrtc/html_client/js/app.js
@@ -143,6 +143,10 @@
#deviceCount = 0;
#micActive = false;
#adbConnected = false;
+ #motion = {
+ orientation: [0, 0, 0],
+ time: window.performance.now(),
+ };
constructor(deviceConnection, parentController) {
this.#deviceConnection = deviceConnection;
@@ -211,6 +215,9 @@
'device-details-button', 'device-details-modal',
'device-details-close');
createModalButton(
+ 'rotation-modal-button', 'rotation-modal',
+ 'rotation-modal-close');
+ createModalButton(
'bluetooth-modal-button', 'bluetooth-prompt', 'bluetooth-prompt-close');
createModalButton(
'bluetooth-prompt-wizard', 'bluetooth-wizard', 'bluetooth-wizard-close',
@@ -243,7 +250,7 @@
createModalButton(
'location-set-cancel', 'location-prompt-modal', 'location-set-modal-close',
'location-set-modal');
-
+ positionModal('rotation-modal-button', 'rotation-modal');
positionModal('device-details-button', 'bluetooth-modal');
positionModal('device-details-button', 'bluetooth-prompt');
positionModal('device-details-button', 'bluetooth-wizard');
@@ -273,6 +280,8 @@
createButtonListener('location-set-confirm', null, this.#deviceConnection,
evt => this.#onSendLocation(this.#deviceConnection, evt));
+ createSliderListener('rotation-slider', () => this.#onMotionChanged());
+
if (this.#deviceConnection.description.custom_control_panel_buttons.length >
0) {
document.getElementById('control-panel-custom-buttons').style.display =
@@ -421,6 +430,35 @@
let location_msg = longitude + "," +latitude + "," + altitude;
deviceConnection.sendLocationMessage(location_msg);
}
+
+ // Inject sensors' events on each change.
+ #onMotionChanged() {
+ const acc_val = document.getElementById('accelerometer-value');
+ const mgn_val = document.getElementById('magnetometer-value');
+ const gyro_val = document.getElementById('gyroscope-value');
+ let values = document.getElementsByClassName('rotation-slider-value');
+ let current_time = window.performance.now();
+ let xyz = [];
+ for (var i = 0; i < values.length; i++) {
+ xyz[i] = values[i].innerHTML;
+ }
+
+ // Calculate sensor values.
+ let acc_xyz = calculateAcceleration(xyz);
+ let mgn_xyz = calculateMagnetometer(xyz);
+ let time_dif = (current_time - this.#motion.time) * 1e-3;
+ let gyro_xyz = calculateGyroscope(this.#motion.orientation, xyz, time_dif);
+ this.#motion.time = current_time;
+ this.#motion.orientation = xyz;
+ // Inject sensors with new values.
+ adbShell(`/vendor/bin/cuttlefish_sensor_injection motion ${acc_xyz[0]} ${acc_xyz[1]} ${acc_xyz[2]} ${mgn_xyz[0]} ${mgn_xyz[1]} ${mgn_xyz[2]} ${gyro_xyz[0]} ${gyro_xyz[1]} ${gyro_xyz[2]}`);
+
+ // Display new sensor values after injection.
+ acc_val.textContent = `${acc_xyz[0]} ${acc_xyz[1]} ${acc_xyz[2]}`;
+ mgn_val.textContent = `${mgn_xyz[0]} ${mgn_xyz[1]} ${mgn_xyz[2]}`;
+ gyro_val.textContent = `${gyro_xyz[0]} ${gyro_xyz[1]} ${gyro_xyz[2]}`;
+ }
+
#onImportLocationsFile(deviceConnection, evt) {
function onLoad_send_kml_data(xml) {
diff --git a/host/frontend/webrtc/html_client/js/controls.js b/host/frontend/webrtc/html_client/js/controls.js
index eb21a34..09d7ad5 100644
--- a/host/frontend/webrtc/html_client/js/controls.js
+++ b/host/frontend/webrtc/html_client/js/controls.js
@@ -71,6 +71,28 @@
}
}
+// Bind the update of slider value to slider input,
+// and trigger a function to be called on input change and slider stop.
+function createSliderListener(slider_class, listener) {
+ const sliders = document.getElementsByClassName(slider_class + '-range');
+ const values = document.getElementsByClassName(slider_class + '-value');
+
+ for (let i = 0; i < sliders.length; i++) {
+ let slider = sliders[i];
+ let value = values[i];
+ // Trigger value update when the slider value changes while sliding.
+ slider.addEventListener('input', () => {
+ value.textContent = slider.value;
+ listener();
+ });
+ // Trigger value update when the slider stops sliding.
+ slider.addEventListener('change', () => {
+ listener();
+ });
+
+ }
+}
+
function createInputListener(input_id, func, listener) {
input = document.getElementById(input_id);
if (func != null) {
diff --git a/host/frontend/webrtc/html_client/js/sensors.js b/host/frontend/webrtc/html_client/js/sensors.js
new file mode 100644
index 0000000..4672e07
--- /dev/null
+++ b/host/frontend/webrtc/html_client/js/sensors.js
@@ -0,0 +1,177 @@
+/*
+ * Copyright (C) 2023 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+'use strict';
+
+const g = 9.80665; // meter per second^2
+const gravityVec = [0, g, 0];
+const magnetic_field = [0, 5.9, -48.4];
+
+function toRadians(x) {
+ return x * Math.PI / 180;
+}
+
+function determinantOfMatrix(M) {
+ // Only compute results for square matrices.
+ if (M.length != M[0].length) {
+ return 0;
+ }
+ if (M.length == 2) {
+ return M[0][0] * M[1][1] - M[1][0] * M[0][1];
+ }
+ let result = 0.0;
+ for (let i = 0; i < M.length; i++) {
+ let factor = M[0][i] * (i % 2? -1 : 1);
+ let subM = getSubmatrix(M, 0, i);
+ result += factor * determinantOfMatrix(subM);
+ }
+ return result;
+}
+
+// Get submatrix that excludes row i, column j.
+function getSubmatrix(M, i, j) {
+ let subM = [];
+ for (let k = 0; k < M.length; k++) {
+ if (k == i) {
+ continue;
+ }
+ let tmp = [];
+ for (let l = 0; l < M.length; l++) {
+ if (l == j) {
+ continue;
+ }
+ tmp.push(M[k][l]);
+ }
+ subM.push(tmp);
+ }
+ return subM;
+}
+
+function invertMatrix(M) {
+ // M ^ -1 = adj(M) / det(M)
+ // adj(M) = transpose(cofactor(M))
+ // Cij = (-1) ^ (i+j) det (Mij)
+ let det = determinantOfMatrix(M);
+ // If matrix is not invertible, return an empty matrix.
+ if (det == 0) {
+ return [[]];
+ }
+ let invM = [];
+ for (let i = 0; i < M.length; i++) {
+ let tmp = [];
+ for (let j = 0; j < M.length; j++) {
+ tmp.push(determinantOfMatrix(getSubmatrix(M, i,j)) * Math.pow(-1, i + j) / det);
+ }
+ invM.push(tmp);
+ }
+ invM = transposeMatrix(invM);
+ return invM;
+}
+
+function transposeMatrix(M) {
+ let transposedM = [];
+ for (let j = 0; j < M.at(0).length; j++) {
+ let tmp = [];
+ for (let i = 0; i < M.length; i++) {
+ tmp.push(M[i][j]);
+ }
+ transposedM.push(tmp);
+ }
+ return transposedM;
+}
+
+function matrixDotProduct(MA, MB) {
+ // If given matrices are not valid for multiplication,
+ // return an empty matrix.
+ if (MA[0].length != MB.length) {
+ return [[]];
+ }
+
+ let vec = [];
+ for (let r = 0; r < MA.length; r++) {
+ let tmp = [];
+ for (let c = 0; c < MB[0].length; c++) {
+ let dot = 0.0;
+ for (let i = 0; i < MA[0].length; i++) {
+ dot += MA[r][i] * MB[i][c];
+ }
+ tmp.push(dot);
+ }
+ vec.push(tmp);
+ }
+ return vec;
+}
+
+// Calculate the rotation matrix of the pitch, yaw, and roll angles.
+function getRotationMatrix(xR, yR, zR) {
+ xR = toRadians(-xR);
+ yR = toRadians(-yR);
+ zR = toRadians(-zR);
+ let rz = [[Math.cos(zR), -Math.sin(zR), 0],
+ [Math.sin(zR), Math.cos(zR), 0],
+ [0, 0, 1]];
+ let ry = [[Math.cos(yR), 0, Math.sin(yR)],
+ [0, 1, 0],
+ [-Math.sin(yR), 0, Math.cos(yR)]];
+ let rx = [[1, 0, 0],
+ [0, Math.cos(xR), -Math.sin(xR)],
+ [0, Math.sin(xR), Math.cos(xR)]];
+ let vec = matrixDotProduct(ry, rx);
+ vec = matrixDotProduct(rz, vec);
+ return vec;
+}
+
+// Calculate new Accelerometer values of the new rotation degrees.
+function calculateAcceleration(rotation) {
+ let rotationM = getRotationMatrix(rotation[0], rotation[1], rotation[2]);
+ let acc = transposeMatrix(matrixDotProduct(rotationM, transposeMatrix([gravityVec])))[0];
+ return acc.map((x) => x.toFixed(3));
+}
+
+// Calculate new Magnetometer values of the new rotation degrees.
+function calculateMagnetometer(rotation) {
+ let rotationM = getRotationMatrix(rotation[0], rotation[1], rotation[2]);
+ let mgn = transposeMatrix(matrixDotProduct(rotationM, transposeMatrix([magnetic_field])))[0];
+ return mgn.map((x) => x.toFixed(3));
+}
+
+// Convert rotation matrix to angular velocity numerator.
+function getAngularRotation(m) {
+ let trace = 0;
+ for (let i = 0; i < m.length; i++) {
+ trace += m[i][i];
+ }
+ let angle = Math.acos((trace - 1) / 2.0);
+ if (angle == 0) {
+ return [0, 0, 0];
+ }
+ let factor = 1.0 / (2 * Math.sin(angle));
+ let axis = [m[2][1] - m[1][2],
+ m[0][2] - m[2][0],
+ m[1][0] - m[0][1]];
+ // Get angular velocity numerator
+ return axis.map((x) => x * factor * angle);
+}
+
+// Calculate new Gyroscope values relative to the new rotation degrees.
+function calculateGyroscope(rotationA, rotationB, time_dif) {
+ let priorRotationM = getRotationMatrix(rotationA[0], rotationA[1], rotationA[2]);
+ let currentRotationM = getRotationMatrix(rotationB[0], rotationB[1], rotationB[2]);
+ let transitionMatrix = matrixDotProduct(priorRotationM, invertMatrix(currentRotationM));
+
+ const gyro = getAngularRotation(transitionMatrix);
+ return gyro.map((x) => (x / time_dif).toFixed(3));
+}
diff --git a/host/frontend/webrtc/html_client/style.css b/host/frontend/webrtc/html_client/style.css
index a9ee60a..0e00271 100644
--- a/host/frontend/webrtc/html_client/style.css
+++ b/host/frontend/webrtc/html_client/style.css
@@ -267,7 +267,12 @@
.location-button {
text-align: center;
}
-
+.sensors{
+ position: sticky;
+ right: 0;
+ top: 0;
+ text-align: right;
+}
.control-panel-column {
width: 50px;
/* Items inside this use a column Flexbox.*/
