[automerger skipped] Fix Segv on unknown address error flagged by fuzzer test. am: d44311374e -s ours am: a371fa8751 -s ours am: 5e34d3792a -s ours am: 7a8d8fe3ad -s ours am: d6dae88a95 -s ours am: 8158125455 -s ours am: 2b7ed94e76 -s ours
am skip reason: Merged-In I255cd68b7641e96ac47ab81479b9b46b78c15580 with SHA-1 9b2183ae5d is already in history
Original change: https://googleplex-android-review.googlesource.com/c/platform/frameworks/av/+/23914014
Change-Id: I17b3b813307383342d6f5f60fe975a65f08bb454
Signed-off-by: Automerger Merge Worker <android-build-automerger-merge-worker@system.gserviceaccount.com>
diff --git a/apex/manifest.json b/apex/manifest.json
index 4b75b04..1a26bff 100644
--- a/apex/manifest.json
+++ b/apex/manifest.json
@@ -1,6 +1,6 @@
{
"name": "com.android.media",
- "version": 339990000,
+ "version": 330400000,
"requireNativeLibs": [
"libandroid.so",
"libbinder_ndk.so",
diff --git a/apex/manifest_codec.json b/apex/manifest_codec.json
index fbcbb69..9426e59 100644
--- a/apex/manifest_codec.json
+++ b/apex/manifest_codec.json
@@ -1,6 +1,6 @@
{
"name": "com.android.media.swcodec",
- "version": 339990000,
+ "version": 330400000,
"requireNativeLibs": [
":sphal"
]
diff --git a/camera/Android.bp b/camera/Android.bp
index e44202b..3e28e4f 100644
--- a/camera/Android.bp
+++ b/camera/Android.bp
@@ -93,6 +93,7 @@
"libgui",
"libcamera_metadata",
"libnativewindow",
+ "lib-platform-compat-native-api",
],
include_dirs: [
diff --git a/camera/Camera.cpp b/camera/Camera.cpp
index 604dbb8..2244682 100644
--- a/camera/Camera.cpp
+++ b/camera/Camera.cpp
@@ -71,10 +71,11 @@
}
sp<Camera> Camera::connect(int cameraId, const String16& clientPackageName,
- int clientUid, int clientPid, int targetSdkVersion)
+ int clientUid, int clientPid, int targetSdkVersion, bool overrideToPortrait,
+ bool forceSlowJpegMode)
{
return CameraBaseT::connect(cameraId, clientPackageName, clientUid,
- clientPid, targetSdkVersion);
+ clientPid, targetSdkVersion, overrideToPortrait, forceSlowJpegMode);
}
status_t Camera::reconnect()
diff --git a/camera/CameraBase.cpp b/camera/CameraBase.cpp
index 24c9108..9ae4607 100644
--- a/camera/CameraBase.cpp
+++ b/camera/CameraBase.cpp
@@ -23,6 +23,7 @@
#include <cutils/properties.h>
#include <android/hardware/ICameraService.h>
+#include <com/android/internal/compat/IPlatformCompatNative.h>
#include <binder/IPCThreadState.h>
#include <binder/IServiceManager.h>
@@ -161,7 +162,8 @@
template <typename TCam, typename TCamTraits>
sp<TCam> CameraBase<TCam, TCamTraits>::connect(int cameraId,
const String16& clientPackageName,
- int clientUid, int clientPid, int targetSdkVersion)
+ int clientUid, int clientPid, int targetSdkVersion,
+ bool overrideToPortrait, bool forceSlowJpegMode)
{
ALOGV("%s: connect", __FUNCTION__);
sp<TCam> c = new TCam(cameraId);
@@ -171,8 +173,11 @@
binder::Status ret;
if (cs != nullptr) {
TCamConnectService fnConnectService = TCamTraits::fnConnectService;
+ ALOGI("Connect camera (legacy API) - overrideToPortrait %d, forceSlowJpegMode %d",
+ overrideToPortrait, forceSlowJpegMode);
ret = (cs.get()->*fnConnectService)(cl, cameraId, clientPackageName, clientUid,
- clientPid, targetSdkVersion, /*out*/ &c->mCamera);
+ clientPid, targetSdkVersion, overrideToPortrait, forceSlowJpegMode,
+ /*out*/ &c->mCamera);
}
if (ret.isOk() && c->mCamera != nullptr) {
IInterface::asBinder(c->mCamera)->linkToDeath(c);
@@ -273,10 +278,11 @@
// this can be in BaseCamera but it should be an instance method
template <typename TCam, typename TCamTraits>
status_t CameraBase<TCam, TCamTraits>::getCameraInfo(int cameraId,
+ bool overrideToPortrait,
struct hardware::CameraInfo* cameraInfo) {
const sp<::android::hardware::ICameraService> cs = getCameraService();
if (cs == 0) return UNKNOWN_ERROR;
- binder::Status res = cs->getCameraInfo(cameraId, cameraInfo);
+ binder::Status res = cs->getCameraInfo(cameraId, overrideToPortrait, cameraInfo);
return res.isOk() ? OK : res.serviceSpecificErrorCode();
}
diff --git a/camera/aidl/android/hardware/ICameraService.aidl b/camera/aidl/android/hardware/ICameraService.aidl
index 1e748c7..9f32595 100644
--- a/camera/aidl/android/hardware/ICameraService.aidl
+++ b/camera/aidl/android/hardware/ICameraService.aidl
@@ -67,7 +67,7 @@
/**
* Fetch basic camera information for a camera device
*/
- CameraInfo getCameraInfo(int cameraId);
+ CameraInfo getCameraInfo(int cameraId, boolean overrideToPortrait);
/**
* Default UID/PID values for non-privileged callers of
@@ -83,7 +83,9 @@
int cameraId,
String opPackageName,
int clientUid, int clientPid,
- int targetSdkVersion);
+ int targetSdkVersion,
+ boolean overrideToPortrait,
+ boolean forceSlowJpegMode);
/**
* Open a camera device through the new camera API
@@ -94,7 +96,8 @@
String opPackageName,
@nullable String featureId,
int clientUid, int oomScoreOffset,
- int targetSdkVersion);
+ int targetSdkVersion,
+ boolean overrideToPortrait);
/**
* Add listener for changes to camera device and flashlight state.
@@ -135,7 +138,8 @@
* Read the static camera metadata for a camera device.
* Only supported for device HAL versions >= 3.2
*/
- CameraMetadataNative getCameraCharacteristics(String cameraId, int targetSdkVersion);
+ CameraMetadataNative getCameraCharacteristics(String cameraId, int targetSdkVersion,
+ boolean overrideToPortrait);
/**
* Read in the vendor tag descriptors from the camera module HAL.
diff --git a/camera/aidl/android/hardware/ICameraServiceProxy.aidl b/camera/aidl/android/hardware/ICameraServiceProxy.aidl
index 88783fb..fefea13 100644
--- a/camera/aidl/android/hardware/ICameraServiceProxy.aidl
+++ b/camera/aidl/android/hardware/ICameraServiceProxy.aidl
@@ -48,5 +48,5 @@
/**
* Checks if the camera has been disabled via device policy.
*/
- boolean isCameraDisabled();
+ boolean isCameraDisabled(int userId);
}
diff --git a/camera/include/camera/Camera.h b/camera/include/camera/Camera.h
index 58ccd69..21b57af 100644
--- a/camera/include/camera/Camera.h
+++ b/camera/include/camera/Camera.h
@@ -58,7 +58,7 @@
typedef ::android::hardware::ICameraClient TCamCallbacks;
typedef ::android::binder::Status(::android::hardware::ICameraService::*TCamConnectService)
(const sp<::android::hardware::ICameraClient>&,
- int, const String16&, int, int, int,
+ int, const String16&, int, int, int, bool, bool,
/*out*/
sp<::android::hardware::ICamera>*);
static TCamConnectService fnConnectService;
@@ -81,7 +81,8 @@
static sp<Camera> create(const sp<::android::hardware::ICamera>& camera);
static sp<Camera> connect(int cameraId,
const String16& clientPackageName,
- int clientUid, int clientPid, int targetSdkVersion);
+ int clientUid, int clientPid, int targetSdkVersion,
+ bool overrideToPortrait, bool forceSlowJpegMode);
virtual ~Camera();
diff --git a/camera/include/camera/CameraBase.h b/camera/include/camera/CameraBase.h
index 8e53968..b20dc1b 100644
--- a/camera/include/camera/CameraBase.h
+++ b/camera/include/camera/CameraBase.h
@@ -119,7 +119,8 @@
static sp<TCam> connect(int cameraId,
const String16& clientPackageName,
- int clientUid, int clientPid, int targetSdkVersion);
+ int clientUid, int clientPid, int targetSdkVersion,
+ bool overrideToPortrait, bool forceSlowJpegMode);
virtual void disconnect();
void setListener(const sp<TCamListener>& listener);
@@ -127,6 +128,7 @@
static int getNumberOfCameras();
static status_t getCameraInfo(int cameraId,
+ bool overrideToPortrait,
/*out*/
struct hardware::CameraInfo* cameraInfo);
diff --git a/camera/ndk/impl/ACameraManager.cpp b/camera/ndk/impl/ACameraManager.cpp
index 5892f1a..02047ae 100644
--- a/camera/ndk/impl/ACameraManager.cpp
+++ b/camera/ndk/impl/ACameraManager.cpp
@@ -692,10 +692,11 @@
ALOGE("%s: Cannot reach camera service!", __FUNCTION__);
return ACAMERA_ERROR_CAMERA_DISCONNECTED;
}
+
CameraMetadata rawMetadata;
int targetSdkVersion = android_get_application_target_sdk_version();
binder::Status serviceRet = cs->getCameraCharacteristics(String16(cameraIdStr),
- targetSdkVersion, &rawMetadata);
+ targetSdkVersion, /*overrideToPortrait*/false, &rawMetadata);
if (!serviceRet.isOk()) {
switch(serviceRet.serviceSpecificErrorCode()) {
case hardware::ICameraService::ERROR_DISCONNECTED:
@@ -747,7 +748,7 @@
binder::Status serviceRet = cs->connectDevice(
callbacks, String16(cameraId), String16(""), {},
hardware::ICameraService::USE_CALLING_UID, /*oomScoreOffset*/0,
- targetSdkVersion, /*out*/&deviceRemote);
+ targetSdkVersion, /*overrideToPortrait*/false, /*out*/&deviceRemote);
if (!serviceRet.isOk()) {
ALOGE("%s: connect camera device failed: %s", __FUNCTION__, serviceRet.toString8().string());
diff --git a/camera/tests/CameraBinderTests.cpp b/camera/tests/CameraBinderTests.cpp
index 17ea512..1af5637 100644
--- a/camera/tests/CameraBinderTests.cpp
+++ b/camera/tests/CameraBinderTests.cpp
@@ -370,7 +370,7 @@
// Check metadata binder call
CameraMetadata metadata;
res = service->getCameraCharacteristics(cameraId,
- /*targetSdkVersion*/__ANDROID_API_FUTURE__, &metadata);
+ /*targetSdkVersion*/__ANDROID_API_FUTURE__, /*overrideToPortrait*/false, &metadata);
EXPECT_TRUE(res.isOk()) << res;
EXPECT_FALSE(metadata.isEmpty());
@@ -386,7 +386,8 @@
sp<hardware::camera2::ICameraDeviceUser> device;
res = service->connectDevice(callbacks, cameraId, String16("meeeeeeeee!"),
{}, hardware::ICameraService::USE_CALLING_UID, /*oomScoreOffset*/ 0,
- /*targetSdkVersion*/__ANDROID_API_FUTURE__, /*out*/&device);
+ /*targetSdkVersion*/__ANDROID_API_FUTURE__,
+ /*overrideToPortrait*/false, /*out*/&device);
EXPECT_TRUE(res.isOk()) << res;
ASSERT_NE(nullptr, device.get());
device->disconnect();
@@ -429,7 +430,8 @@
SCOPED_TRACE("openNewDevice");
binder::Status res = service->connectDevice(callbacks, deviceId, String16("meeeeeeeee!"),
{}, hardware::ICameraService::USE_CALLING_UID, /*oomScoreOffset*/ 0,
- /*targetSdkVersion*/__ANDROID_API_FUTURE__, /*out*/&device);
+ /*targetSdkVersion*/__ANDROID_API_FUTURE__,
+ /*overrideToPortrait*/false, /*out*/&device);
EXPECT_TRUE(res.isOk()) << res;
}
auto p = std::make_pair(callbacks, device);
diff --git a/camera/tests/CameraCharacteristicsPermission.cpp b/camera/tests/CameraCharacteristicsPermission.cpp
index 76dc38c..f2fa48c 100644
--- a/camera/tests/CameraCharacteristicsPermission.cpp
+++ b/camera/tests/CameraCharacteristicsPermission.cpp
@@ -74,7 +74,8 @@
CameraMetadata metadata;
std::vector<int32_t> tagsNeedingPermission;
rc = mCameraService->getCameraCharacteristics(cameraIdStr,
- /*targetSdkVersion*/__ANDROID_API_FUTURE__, &metadata);
+ /*targetSdkVersion*/__ANDROID_API_FUTURE__,
+ /*overrideToPortrait*/false, &metadata);
ASSERT_TRUE(rc.isOk());
EXPECT_FALSE(metadata.isEmpty());
EXPECT_EQ(metadata.removePermissionEntries(CAMERA_METADATA_INVALID_VENDOR_ID,
diff --git a/camera/tests/CameraZSLTests.cpp b/camera/tests/CameraZSLTests.cpp
index efd9dae..6423709 100644
--- a/camera/tests/CameraZSLTests.cpp
+++ b/camera/tests/CameraZSLTests.cpp
@@ -182,7 +182,8 @@
CameraMetadata metadata;
rc = mCameraService->getCameraCharacteristics(cameraIdStr,
- /*targetSdkVersion*/__ANDROID_API_FUTURE__, &metadata);
+ /*targetSdkVersion*/__ANDROID_API_FUTURE__, /*overrideToPortrait*/false,
+ &metadata);
if (!rc.isOk()) {
// The test is relevant only for cameras with Hal 3.x
// support.
@@ -209,7 +210,8 @@
rc = mCameraService->connect(this, cameraId,
String16("ZSLTest"), hardware::ICameraService::USE_CALLING_UID,
hardware::ICameraService::USE_CALLING_PID,
- /*targetSdkVersion*/__ANDROID_API_FUTURE__, &cameraDevice);
+ /*targetSdkVersion*/__ANDROID_API_FUTURE__,
+ /*overrideToPortrait*/false, /*forceSlowJpegMode*/false, &cameraDevice);
EXPECT_TRUE(rc.isOk());
CameraParameters params(cameraDevice->getParameters());
diff --git a/cmds/screenrecord/screenrecord.cpp b/cmds/screenrecord/screenrecord.cpp
index 2e0b678..d866c18 100644
--- a/cmds/screenrecord/screenrecord.cpp
+++ b/cmds/screenrecord/screenrecord.cpp
@@ -13,6 +13,9 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
+#include <algorithm>
+#include <string_view>
+#include <type_traits>
#include <assert.h>
#include <ctype.h>
@@ -100,7 +103,6 @@
static const uint32_t kFallbackHeight = 720;
static const char* kMimeTypeAvc = "video/avc";
static const char* kMimeTypeApplicationOctetstream = "application/octet-stream";
-static const char* kWinscopeMagicString = "#VV1NSC0PET1ME!#";
// Command-line parameters.
static bool gVerbose = false; // chatty on stdout
@@ -354,14 +356,15 @@
}
/*
- * Writes an unsigned integer byte-by-byte in little endian order regardless
+ * Writes an unsigned/signed integer byte-by-byte in little endian order regardless
* of the platform endianness.
*/
-template <typename UINT>
-static void writeValueLE(UINT value, uint8_t* buffer) {
- for (int i = 0; i < sizeof(UINT); ++i) {
- buffer[i] = static_cast<uint8_t>(value);
- value >>= 8;
+template <typename T>
+static void writeValueLE(T value, uint8_t* buffer) {
+ std::remove_const_t<T> temp = value;
+ for (int i = 0; i < sizeof(T); ++i) {
+ buffer[i] = static_cast<std::uint8_t>(temp & 0xff);
+ temp >>= 8;
}
}
@@ -377,16 +380,18 @@
* - for every frame its presentation time relative to the elapsed realtime clock in microseconds
* (as little endian uint64).
*/
-static status_t writeWinscopeMetadata(const Vector<int64_t>& timestamps,
+static status_t writeWinscopeMetadataLegacy(const Vector<int64_t>& timestamps,
const ssize_t metaTrackIdx, AMediaMuxer *muxer) {
- ALOGV("Writing metadata");
+ static constexpr auto kWinscopeMagicStringLegacy = "#VV1NSC0PET1ME!#";
+
+ ALOGV("Writing winscope metadata legacy");
int64_t systemTimeToElapsedTimeOffsetMicros = (android::elapsedRealtimeNano()
- systemTime(SYSTEM_TIME_MONOTONIC)) / 1000;
sp<ABuffer> buffer = new ABuffer(timestamps.size() * sizeof(int64_t)
- + sizeof(uint32_t) + strlen(kWinscopeMagicString));
+ + sizeof(uint32_t) + strlen(kWinscopeMagicStringLegacy));
uint8_t* pos = buffer->data();
- strcpy(reinterpret_cast<char*>(pos), kWinscopeMagicString);
- pos += strlen(kWinscopeMagicString);
+ strcpy(reinterpret_cast<char*>(pos), kWinscopeMagicStringLegacy);
+ pos += strlen(kWinscopeMagicStringLegacy);
writeValueLE<uint32_t>(timestamps.size(), pos);
pos += sizeof(uint32_t);
for (size_t idx = 0; idx < timestamps.size(); ++idx) {
@@ -395,10 +400,79 @@
pos += sizeof(uint64_t);
}
AMediaCodecBufferInfo bufferInfo = {
- 0,
+ 0 /* offset */,
static_cast<int32_t>(buffer->size()),
- timestamps[0],
- 0
+ timestamps[0] /* presentationTimeUs */,
+ 0 /* flags */
+ };
+ return AMediaMuxer_writeSampleData(muxer, metaTrackIdx, buffer->data(), &bufferInfo);
+}
+
+/*
+ * Saves metadata needed by Winscope to synchronize the screen recording playback with other traces.
+ *
+ * The metadata (version 2) is written as a binary array with the following format:
+ * - winscope magic string (#VV1NSC0PET1ME2#, 16B).
+ * - the metadata version number (4B little endian).
+ * - Realtime-to-elapsed time offset in nanoseconds (8B little endian).
+ * - the recorded frames count (8B little endian)
+ * - for each recorded frame:
+ * - System time in elapsed clock timebase in nanoseconds (8B little endian).
+ *
+ *
+ * Metadata version 2 changes
+ *
+ * Use elapsed time for compatibility with other UI traces (most of them):
+ * - Realtime-to-elapsed time offset (instead of realtime-to-monotonic)
+ * - Frame timestamps in elapsed clock timebase (instead of monotonic)
+ */
+static status_t writeWinscopeMetadata(const Vector<std::int64_t>& timestampsMonotonicUs,
+ const ssize_t metaTrackIdx, AMediaMuxer *muxer) {
+ ALOGV("Writing winscope metadata");
+
+ static constexpr auto kWinscopeMagicString = std::string_view {"#VV1NSC0PET1ME2#"};
+ static constexpr std::uint32_t metadataVersion = 2;
+
+ const auto elapsedTimeNs = android::elapsedRealtimeNano();
+ const std::int64_t elapsedToMonotonicTimeOffsetNs =
+ elapsedTimeNs - systemTime(SYSTEM_TIME_MONOTONIC);
+ const std::int64_t realToElapsedTimeOffsetNs =
+ systemTime(SYSTEM_TIME_REALTIME) - elapsedTimeNs;
+ const std::uint32_t framesCount = static_cast<std::uint32_t>(timestampsMonotonicUs.size());
+
+ sp<ABuffer> buffer = new ABuffer(
+ kWinscopeMagicString.size() +
+ sizeof(decltype(metadataVersion)) +
+ sizeof(decltype(realToElapsedTimeOffsetNs)) +
+ sizeof(decltype(framesCount)) +
+ framesCount * sizeof(std::uint64_t)
+ );
+ std::uint8_t* pos = buffer->data();
+
+ std::copy(kWinscopeMagicString.cbegin(), kWinscopeMagicString.cend(), pos);
+ pos += kWinscopeMagicString.size();
+
+ writeValueLE(metadataVersion, pos);
+ pos += sizeof(decltype(metadataVersion));
+
+ writeValueLE(realToElapsedTimeOffsetNs, pos);
+ pos += sizeof(decltype(realToElapsedTimeOffsetNs));
+
+ writeValueLE(framesCount, pos);
+ pos += sizeof(decltype(framesCount));
+
+ for (const auto timestampMonotonicUs : timestampsMonotonicUs) {
+ const auto timestampElapsedNs =
+ elapsedToMonotonicTimeOffsetNs + timestampMonotonicUs * 1000;
+ writeValueLE<std::uint64_t>(timestampElapsedNs, pos);
+ pos += sizeof(std::uint64_t);
+ }
+
+ AMediaCodecBufferInfo bufferInfo = {
+ 0 /* offset */,
+ static_cast<std::int32_t>(buffer->size()),
+ timestampsMonotonicUs[0] /* presentationTimeUs */,
+ 0 /* flags */
};
return AMediaMuxer_writeSampleData(muxer, metaTrackIdx, buffer->data(), &bufferInfo);
}
@@ -418,11 +492,12 @@
static int kTimeout = 250000; // be responsive on signal
status_t err;
ssize_t trackIdx = -1;
+ ssize_t metaLegacyTrackIdx = -1;
ssize_t metaTrackIdx = -1;
uint32_t debugNumFrames = 0;
int64_t startWhenNsec = systemTime(CLOCK_MONOTONIC);
int64_t endWhenNsec = startWhenNsec + seconds_to_nanoseconds(gTimeLimitSec);
- Vector<int64_t> timestamps;
+ Vector<int64_t> timestampsMonotonicUs;
bool firstFrame = true;
assert((rawFp == NULL && muxer != NULL) || (rawFp != NULL && muxer == NULL));
@@ -520,9 +595,9 @@
sp<ABuffer> buffer = new ABuffer(
buffers[bufIndex]->data(), buffers[bufIndex]->size());
AMediaCodecBufferInfo bufferInfo = {
- 0,
+ 0 /* offset */,
static_cast<int32_t>(buffer->size()),
- ptsUsec,
+ ptsUsec /* presentationTimeUs */,
flags
};
err = AMediaMuxer_writeSampleData(muxer, trackIdx, buffer->data(), &bufferInfo);
@@ -532,7 +607,7 @@
return err;
}
if (gOutputFormat == FORMAT_MP4) {
- timestamps.add(ptsUsec);
+ timestampsMonotonicUs.add(ptsUsec);
}
}
debugNumFrames++;
@@ -565,6 +640,7 @@
if (gOutputFormat == FORMAT_MP4) {
AMediaFormat *metaFormat = AMediaFormat_new();
AMediaFormat_setString(metaFormat, AMEDIAFORMAT_KEY_MIME, kMimeTypeApplicationOctetstream);
+ metaLegacyTrackIdx = AMediaMuxer_addTrack(muxer, metaFormat);
metaTrackIdx = AMediaMuxer_addTrack(muxer, metaFormat);
AMediaFormat_delete(metaFormat);
}
@@ -604,10 +680,16 @@
systemTime(CLOCK_MONOTONIC) - startWhenNsec));
fflush(stdout);
}
- if (metaTrackIdx >= 0 && !timestamps.isEmpty()) {
- err = writeWinscopeMetadata(timestamps, metaTrackIdx, muxer);
+ if (metaLegacyTrackIdx >= 0 && metaTrackIdx >= 0 && !timestampsMonotonicUs.isEmpty()) {
+ err = writeWinscopeMetadataLegacy(timestampsMonotonicUs, metaLegacyTrackIdx, muxer);
if (err != NO_ERROR) {
- fprintf(stderr, "Failed writing metadata to muxer (err=%d)\n", err);
+ fprintf(stderr, "Failed writing legacy winscope metadata to muxer (err=%d)\n", err);
+ return err;
+ }
+
+ err = writeWinscopeMetadata(timestampsMonotonicUs, metaTrackIdx, muxer);
+ if (err != NO_ERROR) {
+ fprintf(stderr, "Failed writing winscope metadata to muxer (err=%d)\n", err);
return err;
}
}
diff --git a/drm/libmediadrm/DrmHalHidl.cpp b/drm/libmediadrm/DrmHalHidl.cpp
index c38dbef..ea99483 100644
--- a/drm/libmediadrm/DrmHalHidl.cpp
+++ b/drm/libmediadrm/DrmHalHidl.cpp
@@ -513,10 +513,14 @@
if (mimeType == "") {
// isCryptoSchemeSupported(uuid)
*isSupported = true;
- } else {
- // isCryptoSchemeSupported(uuid, mimeType)
- *isSupported = factory->isContentTypeSupported(mimeType.string());
+ return OK;
}
+ // isCryptoSchemeSupported(uuid, mimeType)
+ auto hResult = factory->isContentTypeSupported(mimeType.string());
+ if (!hResult.isOk()) {
+ return DEAD_OBJECT;
+ }
+ *isSupported = hResult;
return OK;
} else if (mimeType == "") {
return BAD_VALUE;
@@ -526,8 +530,12 @@
if (factoryV1_2 == NULL) {
return ERROR_UNSUPPORTED;
} else {
- *isSupported = factoryV1_2->isCryptoSchemeSupported_1_2(uuid, mimeType.string(),
+ auto hResult = factoryV1_2->isCryptoSchemeSupported_1_2(uuid, mimeType.string(),
toHidlSecurityLevel(level));
+ if (!hResult.isOk()) {
+ return DEAD_OBJECT;
+ }
+ *isSupported = hResult;
return OK;
}
}
@@ -537,7 +545,8 @@
Mutex::Autolock autoLock(mLock);
*isSupported = false;
for (ssize_t i = mFactories.size() - 1; i >= 0; i--) {
- if (mFactories[i]->isCryptoSchemeSupported(uuid)) {
+ auto hResult = mFactories[i]->isCryptoSchemeSupported(uuid);
+ if (hResult.isOk() && hResult) {
return matchMimeTypeAndSecurityLevel(mFactories[i], uuid, mimeType, level, isSupported);
}
}
diff --git a/drm/mediadrm/plugins/clearkey/aidl/CryptoPlugin.cpp b/drm/mediadrm/plugins/clearkey/aidl/CryptoPlugin.cpp
index 201cf02..afc9b6a 100644
--- a/drm/mediadrm/plugins/clearkey/aidl/CryptoPlugin.cpp
+++ b/drm/mediadrm/plugins/clearkey/aidl/CryptoPlugin.cpp
@@ -144,6 +144,11 @@
clearDataLengths.push_back(ss.numBytesOfClearData);
encryptedDataLengths.push_back(ss.numBytesOfEncryptedData);
}
+ if (in_args.keyId.size() != kBlockSize || in_args.iv.size() != kBlockSize) {
+ android_errorWriteLog(0x534e4554, "244569759");
+ detailedError = "invalid decrypt parameter size";
+ return toNdkScopedAStatus(Status::ERROR_DRM_CANNOT_HANDLE, detailedError);
+ }
auto res =
mSession->decrypt(in_args.keyId.data(), in_args.iv.data(),
srcPtr, static_cast<uint8_t*>(destPtr),
diff --git a/drm/mediadrm/plugins/clearkey/hidl/CryptoPlugin.cpp b/drm/mediadrm/plugins/clearkey/hidl/CryptoPlugin.cpp
index 3a675f6..7bc320d 100644
--- a/drm/mediadrm/plugins/clearkey/hidl/CryptoPlugin.cpp
+++ b/drm/mediadrm/plugins/clearkey/hidl/CryptoPlugin.cpp
@@ -206,6 +206,11 @@
return Void();
} else if (mode == Mode::AES_CTR) {
size_t bytesDecrypted;
+ if (keyId.size() != kBlockSize || iv.size() != kBlockSize) {
+ android_errorWriteLog(0x534e4554, "244569759");
+ _hidl_cb(Status_V1_2::ERROR_DRM_CANNOT_HANDLE, 0, "invalid decrypt parameter size");
+ return Void();
+ }
Status_V1_2 res = mSession->decrypt(keyId.data(), iv.data(), srcPtr,
static_cast<uint8_t*>(destPtr), toVector(subSamples), &bytesDecrypted);
if (res == Status_V1_2::OK) {
diff --git a/include/private/media/VideoFrame.h b/include/private/media/VideoFrame.h
index d4025e5..78ea2a1 100644
--- a/include/private/media/VideoFrame.h
+++ b/include/private/media/VideoFrame.h
@@ -42,9 +42,15 @@
mWidth(width), mHeight(height),
mDisplayWidth(displayWidth), mDisplayHeight(displayHeight),
mTileWidth(tileWidth), mTileHeight(tileHeight), mDurationUs(0),
- mRotationAngle(angle), mBytesPerPixel(bpp), mRowBytes(bpp * width),
- mSize(hasData ? (bpp * width * height) : 0),
- mIccSize(iccSize), mBitDepth(bitDepth) {
+ mRotationAngle(angle), mBytesPerPixel(bpp), mIccSize(iccSize),
+ mBitDepth(bitDepth) {
+ uint32_t multVal;
+ mRowBytes = __builtin_mul_overflow(bpp, width, &multVal) ? 0 : multVal;
+ mSize = __builtin_mul_overflow(multVal, height, &multVal) ? 0 : multVal;
+ if (hasData && (mRowBytes == 0 || mSize == 0)) {
+ ALOGE("Frame rowBytes/ size overflow %dx%d bpp %d", width, height, bpp);
+ android_errorWriteLog(0x534e4554, "233006499");
+ }
}
void init(const VideoFrame& copy, const void* iccData, size_t iccSize) {
diff --git a/media/TEST_MAPPING b/media/TEST_MAPPING
index a4c03ba..048301f 100644
--- a/media/TEST_MAPPING
+++ b/media/TEST_MAPPING
@@ -23,8 +23,51 @@
{
"path": "frameworks/av/drm/mediadrm/plugins"
}
+ ],
+
+ "platinum-postsubmit": [
+ // runs regularly, independent of changes in this tree.
+ // signals if changes elsewhere break media functionality
+ // @FlakyTest: in staged-postsubmit, but not postsubmit
+ {
+ "name": "CtsMediaCodecTestCases",
+ "options": [
+ {
+ "include-filter": "android.media.codec.cts.EncodeDecodeTest"
+ }
+ ]
+ },
+ {
+ "name": "CtsMediaCodecTestCases",
+ "options": [
+ {
+ "include-filter": "android.media.codec.cts.DecodeEditEncodeTest"
+ },
+ {
+ "exclude-annotation": "androidx.test.filters.FlakyTest"
+ }
+ ]
+ }
+ ],
+
+ "staged-platinum-postsubmit": [
+ // runs every four hours
+ {
+ "name": "CtsMediaCodecTestCases",
+ "options": [
+ {
+ "include-filter": "android.media.codec.cts.EncodeDecodeTest"
+ }
+ ]
+ },
+ {
+ "name": "CtsMediaCodecTestCases",
+ "options": [
+ {
+ "include-filter": "android.media.codec.cts.DecodeEditEncodeTest"
+ }
+ ]
+ }
]
- // TODO (b/229286407) Add EncodeDecodeTest and DecodeEditEncodeTest to
- // platinum-postsubmit once issues in cuttlefish are fixed
}
diff --git a/media/codec2/hidl/1.0/utils/types.cpp b/media/codec2/hidl/1.0/utils/types.cpp
index 5c24bd7..319ba62 100644
--- a/media/codec2/hidl/1.0/utils/types.cpp
+++ b/media/codec2/hidl/1.0/utils/types.cpp
@@ -1447,6 +1447,10 @@
bool objcpy(C2BaseBlock* d, const BaseBlock& s) {
switch (s.getDiscriminator()) {
case BaseBlock::hidl_discriminator::nativeBlock: {
+ if (s.nativeBlock() == nullptr) {
+ LOG(ERROR) << "Null BaseBlock::nativeBlock handle";
+ return false;
+ }
native_handle_t* sHandle =
native_handle_clone(s.nativeBlock());
if (sHandle == nullptr) {
@@ -1609,6 +1613,7 @@
// assuming blob is const here
size_t size = blob.size();
size_t ix = 0;
+ size_t old_ix = 0;
const uint8_t *data = blob.data();
C2Param *p = nullptr;
@@ -1616,8 +1621,13 @@
p = C2ParamUtils::ParseFirst(data + ix, size - ix);
if (p) {
params->emplace_back(p);
+ old_ix = ix;
ix += p->size();
ix = align(ix, PARAMS_ALIGNMENT);
+ if (ix <= old_ix || ix > size) {
+ android_errorWriteLog(0x534e4554, "238083570");
+ break;
+ }
}
} while (p);
diff --git a/media/codec2/sfplugin/CCodec.cpp b/media/codec2/sfplugin/CCodec.cpp
index 296d7ed..feb0876 100644
--- a/media/codec2/sfplugin/CCodec.cpp
+++ b/media/codec2/sfplugin/CCodec.cpp
@@ -877,6 +877,16 @@
if (msg->findInt32(KEY_PUSH_BLANK_BUFFERS_ON_STOP, &pushBlankBuffersOnStop)) {
config->mPushBlankBuffersOnStop = pushBlankBuffersOnStop == 1;
}
+ // secure compoment or protected content default with
+ // "push-blank-buffers-on-shutdown" flag
+ if (!config->mPushBlankBuffersOnStop) {
+ int32_t usageProtected;
+ if (comp->getName().find(".secure") != std::string::npos) {
+ config->mPushBlankBuffersOnStop = true;
+ } else if (msg->findInt32("protected", &usageProtected) && usageProtected) {
+ config->mPushBlankBuffersOnStop = true;
+ }
+ }
}
}
setSurface(surface);
@@ -1869,18 +1879,14 @@
}
state->set(STOPPING);
}
- {
- Mutexed<std::unique_ptr<Config>>::Locked configLocked(mConfig);
- const std::unique_ptr<Config> &config = *configLocked;
- if (config->mPushBlankBuffersOnStop) {
- mChannel->pushBlankBufferToOutputSurface();
- }
- }
mChannel->reset();
- (new AMessage(kWhatStop, this))->post();
+ bool pushBlankBuffer = mConfig.lock().get()->mPushBlankBuffersOnStop;
+ sp<AMessage> stopMessage(new AMessage(kWhatStop, this));
+ stopMessage->setInt32("pushBlankBuffer", pushBlankBuffer);
+ stopMessage->post();
}
-void CCodec::stop() {
+void CCodec::stop(bool pushBlankBuffer) {
std::shared_ptr<Codec2Client::Component> comp;
{
Mutexed<State>::Locked state(mState);
@@ -1899,6 +1905,7 @@
comp = state->comp;
}
status_t err = comp->stop();
+ mChannel->stopUseOutputSurface(pushBlankBuffer);
if (err != C2_OK) {
// TODO: convert err into status_t
mCallback->onError(UNKNOWN_ERROR, ACTION_CODE_FATAL);
@@ -1963,21 +1970,16 @@
config->mInputSurfaceDataspace = HAL_DATASPACE_UNKNOWN;
}
}
- {
- Mutexed<std::unique_ptr<Config>>::Locked configLocked(mConfig);
- const std::unique_ptr<Config> &config = *configLocked;
- if (config->mPushBlankBuffersOnStop) {
- mChannel->pushBlankBufferToOutputSurface();
- }
- }
mChannel->reset();
+ bool pushBlankBuffer = mConfig.lock().get()->mPushBlankBuffersOnStop;
// thiz holds strong ref to this while the thread is running.
sp<CCodec> thiz(this);
- std::thread([thiz, sendCallback] { thiz->release(sendCallback); }).detach();
+ std::thread([thiz, sendCallback, pushBlankBuffer]
+ { thiz->release(sendCallback, pushBlankBuffer); }).detach();
}
-void CCodec::release(bool sendCallback) {
+void CCodec::release(bool sendCallback, bool pushBlankBuffer) {
std::shared_ptr<Codec2Client::Component> comp;
{
Mutexed<State>::Locked state(mState);
@@ -1992,6 +1994,7 @@
comp = state->comp;
}
comp->release();
+ mChannel->stopUseOutputSurface(pushBlankBuffer);
{
Mutexed<State>::Locked state(mState);
@@ -2005,6 +2008,7 @@
}
status_t CCodec::setSurface(const sp<Surface> &surface) {
+ bool pushBlankBuffer = false;
{
Mutexed<std::unique_ptr<Config>>::Locked configLocked(mConfig);
const std::unique_ptr<Config> &config = *configLocked;
@@ -2030,8 +2034,9 @@
return err;
}
}
+ pushBlankBuffer = config->mPushBlankBuffersOnStop;
}
- return mChannel->setSurface(surface);
+ return mChannel->setSurface(surface, pushBlankBuffer);
}
void CCodec::signalFlush() {
@@ -2139,7 +2144,8 @@
std::map<size_t, sp<MediaCodecBuffer>> clientInputBuffers;
status_t err = mChannel->prepareInitialInputBuffers(&clientInputBuffers);
- if (err != OK) {
+ // FIXME(b/237656746)
+ if (err != OK && err != NO_MEMORY) {
ALOGE("Resume request for Input Buffers failed");
mCallback->onError(err, ACTION_CODE_FATAL);
return;
@@ -2329,7 +2335,11 @@
case kWhatStop: {
// C2Component::stop() should return within 500ms.
setDeadline(now, 1500ms, "stop");
- stop();
+ int32_t pushBlankBuffer;
+ if (!msg->findInt32("pushBlankBuffer", &pushBlankBuffer)) {
+ pushBlankBuffer = 0;
+ }
+ stop(static_cast<bool>(pushBlankBuffer));
break;
}
case kWhatFlush: {
diff --git a/media/codec2/sfplugin/CCodecBufferChannel.cpp b/media/codec2/sfplugin/CCodecBufferChannel.cpp
index 5ecb130..c7ab360 100644
--- a/media/codec2/sfplugin/CCodecBufferChannel.cpp
+++ b/media/codec2/sfplugin/CCodecBufferChannel.cpp
@@ -960,9 +960,9 @@
hdrDynamicInfo->m.data + hdrDynamicInfo->flexCount());
}
qbi.setHdrMetadata(hdr);
-
- SetHdrMetadataToGralloc4Handle(hdrStaticInfo, hdrDynamicInfo, block.handle());
}
+ SetMetadataToGralloc4Handle(dataSpace, hdrStaticInfo, hdrDynamicInfo, block.handle());
+
// we don't have dirty regions
qbi.setSurfaceDamage(Region::INVALID_REGION);
android::IGraphicBufferProducer::QueueBufferOutput qbo;
@@ -1578,6 +1578,25 @@
mFirstValidFrameIndex = mFrameIndex.load(std::memory_order_relaxed);
}
+void CCodecBufferChannel::stopUseOutputSurface(bool pushBlankBuffer) {
+ sp<Surface> surface = mOutputSurface.lock()->surface;
+ if (surface) {
+ C2BlockPool::local_id_t outputPoolId;
+ {
+ Mutexed<BlockPools>::Locked pools(mBlockPools);
+ outputPoolId = pools->outputPoolId;
+ }
+ if (mComponent) mComponent->stopUsingOutputSurface(outputPoolId);
+
+ if (pushBlankBuffer) {
+ sp<ANativeWindow> anw = static_cast<ANativeWindow *>(surface.get());
+ if (anw) {
+ pushBlankBuffersToNativeWindow(anw.get());
+ }
+ }
+ }
+}
+
void CCodecBufferChannel::reset() {
stop();
if (mInputSurface != nullptr) {
@@ -1593,14 +1612,6 @@
Mutexed<Output>::Locked output(mOutput);
output->buffers.reset();
}
- if (mOutputSurface.lock()->surface) {
- C2BlockPool::local_id_t outputPoolId;
- {
- Mutexed<BlockPools>::Locked pools(mBlockPools);
- outputPoolId = pools->outputPoolId;
- }
- mComponent->stopUsingOutputSurface(outputPoolId);
- }
}
void CCodecBufferChannel::release() {
@@ -2094,14 +2105,20 @@
}
}
-status_t CCodecBufferChannel::setSurface(const sp<Surface> &newSurface) {
+status_t CCodecBufferChannel::setSurface(const sp<Surface> &newSurface, bool pushBlankBuffer) {
static std::atomic_uint32_t surfaceGeneration{0};
uint32_t generation = (getpid() << 10) |
((surfaceGeneration.fetch_add(1, std::memory_order_relaxed) + 1)
& ((1 << 10) - 1));
sp<IGraphicBufferProducer> producer;
- int maxDequeueCount = mOutputSurface.lock()->maxDequeueBuffers;
+ int maxDequeueCount;
+ sp<Surface> oldSurface;
+ {
+ Mutexed<OutputSurface>::Locked outputSurface(mOutputSurface);
+ maxDequeueCount = outputSurface->maxDequeueBuffers;
+ oldSurface = outputSurface->surface;
+ }
if (newSurface) {
newSurface->setScalingMode(NATIVE_WINDOW_SCALING_MODE_SCALE_TO_WINDOW);
newSurface->setDequeueTimeout(kDequeueTimeoutNs);
@@ -2138,6 +2155,15 @@
output->generation = generation;
}
+ if (oldSurface && pushBlankBuffer) {
+ // When ReleaseSurface was set from MediaCodec,
+ // pushing a blank buffer at the end might be necessary.
+ sp<ANativeWindow> anw = static_cast<ANativeWindow *>(oldSurface.get());
+ if (anw) {
+ pushBlankBuffersToNativeWindow(anw.get());
+ }
+ }
+
return OK;
}
@@ -2227,13 +2253,4 @@
}
}
-status_t CCodecBufferChannel::pushBlankBufferToOutputSurface() {
- Mutexed<OutputSurface>::Locked output(mOutputSurface);
- sp<ANativeWindow> nativeWindow = static_cast<ANativeWindow *>(output->surface.get());
- if (nativeWindow == nullptr) {
- return INVALID_OPERATION;
- }
- return pushBlankBuffersToNativeWindow(nativeWindow.get());
-}
-
} // namespace android
diff --git a/media/codec2/sfplugin/CCodecBufferChannel.h b/media/codec2/sfplugin/CCodecBufferChannel.h
index f29a225..a52d4dc 100644
--- a/media/codec2/sfplugin/CCodecBufferChannel.h
+++ b/media/codec2/sfplugin/CCodecBufferChannel.h
@@ -102,7 +102,7 @@
/**
* Set output graphic surface for rendering.
*/
- status_t setSurface(const sp<Surface> &surface);
+ status_t setSurface(const sp<Surface> &surface, bool pushBlankBuffer);
/**
* Set GraphicBufferSource object from which the component extracts input
@@ -149,6 +149,14 @@
std::map<size_t, sp<MediaCodecBuffer>> &&clientInputBuffers);
/**
+ * Stop using buffers of the current output surface for other Codec
+ * instances to use the surface safely.
+ *
+ * \param pushBlankBuffer[in] push a blank buffer at the end if true
+ */
+ void stopUseOutputSurface(bool pushBlankBuffer);
+
+ /**
* Stop queueing buffers to the component. This object should never queue
* buffers after this call, until start() is called.
*/
@@ -195,11 +203,6 @@
void setMetaMode(MetaMode mode);
- /**
- * Push a blank buffer to the configured native output surface.
- */
- status_t pushBlankBufferToOutputSurface();
-
private:
class QueueGuard;
diff --git a/media/codec2/sfplugin/Codec2Buffer.cpp b/media/codec2/sfplugin/Codec2Buffer.cpp
index cde4c72..55e0c45 100644
--- a/media/codec2/sfplugin/Codec2Buffer.cpp
+++ b/media/codec2/sfplugin/Codec2Buffer.cpp
@@ -998,9 +998,11 @@
}
using ::aidl::android::hardware::graphics::common::Cta861_3;
+using ::aidl::android::hardware::graphics::common::Dataspace;
using ::aidl::android::hardware::graphics::common::Smpte2086;
using ::android::gralloc4::MetadataType_Cta861_3;
+using ::android::gralloc4::MetadataType_Dataspace;
using ::android::gralloc4::MetadataType_Smpte2086;
using ::android::gralloc4::MetadataType_Smpte2094_40;
@@ -1156,7 +1158,8 @@
return err;
}
-c2_status_t SetHdrMetadataToGralloc4Handle(
+c2_status_t SetMetadataToGralloc4Handle(
+ android_dataspace_t dataSpace,
const std::shared_ptr<const C2StreamHdrStaticMetadataInfo::output> &staticInfo,
const std::shared_ptr<const C2StreamHdrDynamicMetadataInfo::output> &dynamicInfo,
const C2Handle *const handle) {
@@ -1167,6 +1170,17 @@
// Gralloc4 not supported; nothing to do
return err;
}
+ {
+ hidl_vec<uint8_t> metadata;
+ if (gralloc4::encodeDataspace(static_cast<Dataspace>(dataSpace), &metadata) == OK) {
+ Return<Error4> ret = mapper->set(buffer.get(), MetadataType_Dataspace, metadata);
+ if (!ret.isOk()) {
+ err = C2_REFUSED;
+ } else if (ret != Error4::NONE) {
+ err = C2_CORRUPTED;
+ }
+ }
+ }
if (staticInfo && *staticInfo) {
ALOGV("Setting static HDR info as gralloc4 metadata");
std::optional<Smpte2086> smpte2086 = Smpte2086{
diff --git a/media/codec2/sfplugin/Codec2Buffer.h b/media/codec2/sfplugin/Codec2Buffer.h
index b02b042..b73acab 100644
--- a/media/codec2/sfplugin/Codec2Buffer.h
+++ b/media/codec2/sfplugin/Codec2Buffer.h
@@ -410,14 +410,16 @@
std::shared_ptr<C2StreamHdrDynamicMetadataInfo::input> *dynamicInfo);
/**
- * Set HDR metadata to Gralloc4 handle.
+ * Set metadata to Gralloc4 handle.
*
+ * \param[in] dataSpace Dataspace to set.
* \param[in] staticInfo HDR static info to set. Ignored if null or invalid.
* \param[in] dynamicInfo HDR dynamic info to set. Ignored if null or invalid.
* \param[out] handle handle of the allocation.
* \return C2_OK if successful
*/
-c2_status_t SetHdrMetadataToGralloc4Handle(
+c2_status_t SetMetadataToGralloc4Handle(
+ const android_dataspace_t dataSpace,
const std::shared_ptr<const C2StreamHdrStaticMetadataInfo::output> &staticInfo,
const std::shared_ptr<const C2StreamHdrDynamicMetadataInfo::output> &dynamicInfo,
const C2Handle *const handle);
diff --git a/media/codec2/sfplugin/include/media/stagefright/CCodec.h b/media/codec2/sfplugin/include/media/stagefright/CCodec.h
index ec18128..13713bc 100644
--- a/media/codec2/sfplugin/include/media/stagefright/CCodec.h
+++ b/media/codec2/sfplugin/include/media/stagefright/CCodec.h
@@ -109,9 +109,9 @@
void allocate(const sp<MediaCodecInfo> &codecInfo);
void configure(const sp<AMessage> &msg);
void start();
- void stop();
+ void stop(bool pushBlankBuffer);
void flush();
- void release(bool sendCallback);
+ void release(bool sendCallback, bool pushBlankBuffer);
/**
* Creates an input surface for the current device configuration compatible with CCodec.
diff --git a/media/codec2/vndk/include/C2SurfaceSyncObj.h b/media/codec2/vndk/include/C2SurfaceSyncObj.h
index ac87fe4..d858f27 100644
--- a/media/codec2/vndk/include/C2SurfaceSyncObj.h
+++ b/media/codec2/vndk/include/C2SurfaceSyncObj.h
@@ -72,12 +72,13 @@
/**
* Notify a buffer is queued. Return whether the upcoming dequeue operation
* is not blocked. if it's blocked and waitId is non-null, waitId is returned
- * to be used for waiting.
+ * to be used for waiting. Notify(wake-up) waitors only when 'notify' is
+ * true.
*
* \retval false dequeue operation is blocked now.
* \retval true dequeue operation is possible.
*/
- bool notifyQueuedLocked(uint32_t *waitId = nullptr);
+ bool notifyQueuedLocked(uint32_t *waitId = nullptr, bool notify = true);
/**
* Notify a buffer is dequeued.
diff --git a/media/codec2/vndk/platform/C2BqBuffer.cpp b/media/codec2/vndk/platform/C2BqBuffer.cpp
index e67e42f..f2cd585 100644
--- a/media/codec2/vndk/platform/C2BqBuffer.cpp
+++ b/media/codec2/vndk/platform/C2BqBuffer.cpp
@@ -432,12 +432,16 @@
if (fence) {
static constexpr int kFenceWaitTimeMs = 10;
+ if (bufferNeedsReallocation) {
+ mBuffers[slot].clear();
+ }
+
status_t status = fence->wait(kFenceWaitTimeMs);
if (status == -ETIME) {
// fence is not signalled yet.
if (syncVar) {
- syncVar->lock();
(void)mProducer->cancelBuffer(slot, hFenceWrapper.getHandle()).isOk();
+ syncVar->lock();
dequeueable = syncVar->notifyQueuedLocked(&waitId);
syncVar->unlock();
if (c2Fence) {
@@ -452,8 +456,8 @@
if (status != android::NO_ERROR) {
ALOGD("buffer fence wait error %d", status);
if (syncVar) {
- syncVar->lock();
(void)mProducer->cancelBuffer(slot, hFenceWrapper.getHandle()).isOk();
+ syncVar->lock();
syncVar->notifyQueuedLocked();
syncVar->unlock();
if (c2Fence) {
@@ -502,8 +506,8 @@
} else if (status != android::NO_ERROR) {
slotBuffer.clear();
if (syncVar) {
- syncVar->lock();
(void)mProducer->cancelBuffer(slot, hFenceWrapper.getHandle()).isOk();
+ syncVar->lock();
syncVar->notifyQueuedLocked();
syncVar->unlock();
if (c2Fence) {
@@ -550,8 +554,8 @@
// Block was not created. call requestBuffer# again next time.
slotBuffer.clear();
if (syncVar) {
- syncVar->lock();
(void)mProducer->cancelBuffer(slot, hFenceWrapper.getHandle()).isOk();
+ syncVar->lock();
syncVar->notifyQueuedLocked();
syncVar->unlock();
if (c2Fence) {
@@ -813,11 +817,10 @@
if (mGeneration == mCurrentGeneration && mBqId == mCurrentBqId && !mOwner.expired()) {
C2SyncVariables *syncVar = mSyncMem ? mSyncMem->mem() : nullptr;
if (syncVar) {
+ mIgbp->cancelBuffer(mBqSlot, hidl_handle{}).isOk();
syncVar->lock();
- if (syncVar->getSyncStatusLocked() == C2SyncVariables::STATUS_ACTIVE) {
- mIgbp->cancelBuffer(mBqSlot, hidl_handle{}).isOk();
- syncVar->notifyQueuedLocked();
- }
+ syncVar->notifyQueuedLocked(nullptr,
+ syncVar->getSyncStatusLocked() == C2SyncVariables::STATUS_ACTIVE);
syncVar->unlock();
} else {
mIgbp->cancelBuffer(mBqSlot, hidl_handle{}).isOk();
@@ -826,11 +829,10 @@
} else if (!mOwner.expired()) {
C2SyncVariables *syncVar = mSyncMem ? mSyncMem->mem() : nullptr;
if (syncVar) {
+ mIgbp->cancelBuffer(mBqSlot, hidl_handle{}).isOk();
syncVar->lock();
- if (syncVar->getSyncStatusLocked() != C2SyncVariables::STATUS_SWITCHING) {
- mIgbp->cancelBuffer(mBqSlot, hidl_handle{}).isOk();
- syncVar->notifyQueuedLocked();
- }
+ syncVar->notifyQueuedLocked(nullptr,
+ syncVar->getSyncStatusLocked() != C2SyncVariables::STATUS_SWITCHING);
syncVar->unlock();
} else {
mIgbp->cancelBuffer(mBqSlot, hidl_handle{}).isOk();
diff --git a/media/codec2/vndk/platform/C2SurfaceSyncObj.cpp b/media/codec2/vndk/platform/C2SurfaceSyncObj.cpp
index 6be4d09..bf4ca32 100644
--- a/media/codec2/vndk/platform/C2SurfaceSyncObj.cpp
+++ b/media/codec2/vndk/platform/C2SurfaceSyncObj.cpp
@@ -182,12 +182,14 @@
return true;
}
-bool C2SyncVariables::notifyQueuedLocked(uint32_t *waitId) {
+bool C2SyncVariables::notifyQueuedLocked(uint32_t *waitId, bool notify) {
// Note. thundering herds may occur. Edge trigged signalling.
// But one waiter will guarantee to dequeue. others may wait again.
// Minimize futex syscall(trap) for the main use case(one waiter case).
if (mMaxDequeueCount == mCurDequeueCount--) {
- broadcast();
+ if (notify) {
+ broadcast();
+ }
return true;
}
diff --git a/media/extractors/mp4/MPEG4Extractor.cpp b/media/extractors/mp4/MPEG4Extractor.cpp
index 7f97ddc..fb9dfb4 100644
--- a/media/extractors/mp4/MPEG4Extractor.cpp
+++ b/media/extractors/mp4/MPEG4Extractor.cpp
@@ -5897,12 +5897,18 @@
return -EINVAL;
}
- int32_t dataOffsetDelta;
- if (!mDataSource->getUInt32(offset, (uint32_t*)&dataOffsetDelta)) {
+ uint32_t dataOffsetDelta;
+ if (!mDataSource->getUInt32(offset, &dataOffsetDelta)) {
return ERROR_MALFORMED;
}
- dataOffset = mTrackFragmentHeaderInfo.mBaseDataOffset + dataOffsetDelta;
+ if (__builtin_add_overflow(
+ mTrackFragmentHeaderInfo.mBaseDataOffset, dataOffsetDelta, &dataOffset)) {
+ ALOGW("b/232242894 mBaseDataOffset(%" PRIu64 ") + dataOffsetDelta(%u) overflows uint64",
+ mTrackFragmentHeaderInfo.mBaseDataOffset, dataOffsetDelta);
+ android_errorWriteLog(0x534e4554, "232242894");
+ return ERROR_MALFORMED;
+ }
offset += 4;
size -= 4;
@@ -6036,7 +6042,12 @@
return NO_MEMORY;
}
- dataOffset += sampleSize;
+ if (__builtin_add_overflow(dataOffset, sampleSize, &dataOffset)) {
+ ALOGW("b/232242894 dataOffset(%" PRIu64 ") + sampleSize(%u) overflows uint64",
+ dataOffset, sampleSize);
+ android_errorWriteLog(0x534e4554, "232242894");
+ return ERROR_MALFORMED;
+ }
}
mTrackFragmentHeaderInfo.mDataOffset = dataOffset;
diff --git a/media/libaudioclient/AidlConversion.cpp b/media/libaudioclient/AidlConversion.cpp
index 11724e0..4133bd0 100644
--- a/media/libaudioclient/AidlConversion.cpp
+++ b/media/libaudioclient/AidlConversion.cpp
@@ -3388,4 +3388,24 @@
enumToMask_index<int32_t, media::AudioDirectMode>);
}
+ConversionResult<audio_latency_mode_t>
+aidl2legacy_LatencyMode_audio_latency_mode_t(media::LatencyMode aidl) {
+ switch (aidl) {
+ case media::LatencyMode::FREE:
+ return AUDIO_LATENCY_MODE_FREE;
+ case media::LatencyMode::LOW:
+ return AUDIO_LATENCY_MODE_LOW;
+ }
+ return unexpected(BAD_VALUE);
+}
+ConversionResult<media::LatencyMode>
+legacy2aidl_audio_latency_mode_t_LatencyMode(audio_latency_mode_t legacy) {
+ switch (legacy) {
+ case AUDIO_LATENCY_MODE_FREE:
+ return media::LatencyMode::FREE;
+ case AUDIO_LATENCY_MODE_LOW:
+ return media::LatencyMode::LOW;
+ }
+ return unexpected(BAD_VALUE);
+}
} // namespace android
diff --git a/media/libaudioclient/Android.bp b/media/libaudioclient/Android.bp
index 69a9c68..dccef0e 100644
--- a/media/libaudioclient/Android.bp
+++ b/media/libaudioclient/Android.bp
@@ -342,6 +342,7 @@
"aidl/android/media/AudioUniqueIdUse.aidl",
"aidl/android/media/AudioVibratorInfo.aidl",
"aidl/android/media/EffectDescriptor.aidl",
+ "aidl/android/media/LatencyMode.aidl",
"aidl/android/media/TrackSecondaryOutputInfo.aidl",
],
imports: [
diff --git a/media/libaudioclient/AudioEffect.cpp b/media/libaudioclient/AudioEffect.cpp
index 7b273ec..dc21c52 100644
--- a/media/libaudioclient/AudioEffect.cpp
+++ b/media/libaudioclient/AudioEffect.cpp
@@ -540,7 +540,6 @@
if (cb != nullptr) {
cb->onError(mStatus);
}
- mIEffect.clear();
}
// -------------------------------------------------------------------------
diff --git a/media/libaudioclient/AudioSystem.cpp b/media/libaudioclient/AudioSystem.cpp
index de8c298..bacca3a 100644
--- a/media/libaudioclient/AudioSystem.cpp
+++ b/media/libaudioclient/AudioSystem.cpp
@@ -670,6 +670,30 @@
return Status::ok();
}
+Status AudioSystem::AudioFlingerClient::onSupportedLatencyModesChanged(
+ int output, const std::vector<media::LatencyMode>& latencyModes) {
+ audio_io_handle_t outputLegacy = VALUE_OR_RETURN_BINDER_STATUS(
+ aidl2legacy_int32_t_audio_io_handle_t(output));
+ std::vector<audio_latency_mode_t> modesLegacy = VALUE_OR_RETURN_BINDER_STATUS(
+ convertContainer<std::vector<audio_latency_mode_t>>(
+ latencyModes, aidl2legacy_LatencyMode_audio_latency_mode_t));
+
+ std::vector<sp<SupportedLatencyModesCallback>> callbacks;
+ {
+ Mutex::Autolock _l(mLock);
+ for (auto callback : mSupportedLatencyModesCallbacks) {
+ if (auto ref = callback.promote(); ref != nullptr) {
+ callbacks.push_back(ref);
+ }
+ }
+ }
+ for (const auto& callback : callbacks) {
+ callback->onSupportedLatencyModesChanged(outputLegacy, modesLegacy);
+ }
+
+ return Status::ok();
+}
+
status_t AudioSystem::AudioFlingerClient::getInputBufferSize(
uint32_t sampleRate, audio_format_t format,
audio_channel_mask_t channelMask, size_t* buffSize) {
@@ -749,6 +773,31 @@
return NO_ERROR;
}
+status_t AudioSystem::AudioFlingerClient::addSupportedLatencyModesCallback(
+ const sp<SupportedLatencyModesCallback>& callback) {
+ Mutex::Autolock _l(mLock);
+ if (std::find(mSupportedLatencyModesCallbacks.begin(),
+ mSupportedLatencyModesCallbacks.end(),
+ callback) != mSupportedLatencyModesCallbacks.end()) {
+ return INVALID_OPERATION;
+ }
+ mSupportedLatencyModesCallbacks.push_back(callback);
+ return NO_ERROR;
+}
+
+status_t AudioSystem::AudioFlingerClient::removeSupportedLatencyModesCallback(
+ const sp<SupportedLatencyModesCallback>& callback) {
+ Mutex::Autolock _l(mLock);
+ auto it = std::find(mSupportedLatencyModesCallbacks.begin(),
+ mSupportedLatencyModesCallbacks.end(),
+ callback);
+ if (it == mSupportedLatencyModesCallbacks.end()) {
+ return INVALID_OPERATION;
+ }
+ mSupportedLatencyModesCallbacks.erase(it);
+ return NO_ERROR;
+}
+
/* static */ uintptr_t AudioSystem::addErrorCallback(audio_error_callback cb) {
Mutex::Autolock _l(gLockErrorCallbacks);
gAudioErrorCallbacks.insert(cb);
@@ -1662,6 +1711,24 @@
return afc->removeAudioDeviceCallback(callback, audioIo, portId);
}
+status_t AudioSystem::addSupportedLatencyModesCallback(
+ const sp<SupportedLatencyModesCallback>& callback) {
+ const sp<AudioFlingerClient> afc = getAudioFlingerClient();
+ if (afc == 0) {
+ return NO_INIT;
+ }
+ return afc->addSupportedLatencyModesCallback(callback);
+}
+
+status_t AudioSystem::removeSupportedLatencyModesCallback(
+ const sp<SupportedLatencyModesCallback>& callback) {
+ const sp<AudioFlingerClient> afc = getAudioFlingerClient();
+ if (afc == 0) {
+ return NO_INIT;
+ }
+ return afc->removeSupportedLatencyModesCallback(callback);
+}
+
audio_port_handle_t AudioSystem::getDeviceIdForIo(audio_io_handle_t audioIo) {
const sp<IAudioFlinger>& af = AudioSystem::get_audio_flinger();
if (af == 0) return PERMISSION_DENIED;
@@ -2338,6 +2405,24 @@
return NO_ERROR;
}
+status_t AudioSystem::setRequestedLatencyMode(
+ audio_io_handle_t output, audio_latency_mode_t mode) {
+ const sp<IAudioFlinger>& af = AudioSystem::get_audio_flinger();
+ if (af == nullptr) {
+ return PERMISSION_DENIED;
+ }
+ return af->setRequestedLatencyMode(output, mode);
+}
+
+status_t AudioSystem::getSupportedLatencyModes(audio_io_handle_t output,
+ std::vector<audio_latency_mode_t>* modes) {
+ const sp<IAudioFlinger>& af = AudioSystem::get_audio_flinger();
+ if (af == nullptr) {
+ return PERMISSION_DENIED;
+ }
+ return af->getSupportedLatencyModes(output, modes);
+}
+
class CaptureStateListenerImpl : public media::BnCaptureStateListener,
public IBinder::DeathRecipient {
public:
diff --git a/media/libaudioclient/AudioTrack.cpp b/media/libaudioclient/AudioTrack.cpp
index 6ab8339..b899951 100644
--- a/media/libaudioclient/AudioTrack.cpp
+++ b/media/libaudioclient/AudioTrack.cpp
@@ -1789,13 +1789,21 @@
status_t AudioTrack::setOutputDevice(audio_port_handle_t deviceId) {
AutoMutex lock(mLock);
- ALOGV("%s(%d): deviceId=%d mSelectedDeviceId=%d",
- __func__, mPortId, deviceId, mSelectedDeviceId);
+ ALOGV("%s(%d): deviceId=%d mSelectedDeviceId=%d mRoutedDeviceId %d",
+ __func__, mPortId, deviceId, mSelectedDeviceId, mRoutedDeviceId);
if (mSelectedDeviceId != deviceId) {
mSelectedDeviceId = deviceId;
- if (mStatus == NO_ERROR) {
- android_atomic_or(CBLK_INVALID, &mCblk->mFlags);
- mProxy->interrupt();
+ if (mStatus == NO_ERROR && mSelectedDeviceId != mRoutedDeviceId) {
+ if (isPlaying_l()) {
+ android_atomic_or(CBLK_INVALID, &mCblk->mFlags);
+ mProxy->interrupt();
+ } else {
+ // if the track is idle, try to restore now and
+ // defer to next start if not possible
+ if (restoreTrack_l("setOutputDevice") != OK) {
+ android_atomic_or(CBLK_INVALID, &mCblk->mFlags);
+ }
+ }
}
}
return NO_ERROR;
diff --git a/media/libaudioclient/IAudioFlinger.cpp b/media/libaudioclient/IAudioFlinger.cpp
index 292d92f..f290453 100644
--- a/media/libaudioclient/IAudioFlinger.cpp
+++ b/media/libaudioclient/IAudioFlinger.cpp
@@ -22,10 +22,10 @@
#include <stdint.h>
#include <sys/types.h>
-
+#include "IAudioFlinger.h"
#include <binder/IPCThreadState.h>
#include <binder/Parcel.h>
-#include "IAudioFlinger.h"
+#include <system/thread_defs.h>
namespace android {
@@ -810,10 +810,39 @@
return statusTFromBinderStatus(mDelegate->setDeviceConnectedState(aidlPort, connected));
}
+status_t AudioFlingerClientAdapter::setRequestedLatencyMode(
+ audio_io_handle_t output, audio_latency_mode_t mode) {
+ int32_t outputAidl = VALUE_OR_RETURN_STATUS(legacy2aidl_audio_io_handle_t_int32_t(output));
+ media::LatencyMode modeAidl = VALUE_OR_RETURN_STATUS(
+ legacy2aidl_audio_latency_mode_t_LatencyMode(mode));
+ return statusTFromBinderStatus(mDelegate->setRequestedLatencyMode(outputAidl, modeAidl));
+}
+
+status_t AudioFlingerClientAdapter::getSupportedLatencyModes(
+ audio_io_handle_t output, std::vector<audio_latency_mode_t>* modes) {
+ if (modes == nullptr) {
+ return BAD_VALUE;
+ }
+
+ int32_t outputAidl = VALUE_OR_RETURN_STATUS(legacy2aidl_audio_io_handle_t_int32_t(output));
+ std::vector<media::LatencyMode> modesAidl;
+
+ RETURN_STATUS_IF_ERROR(statusTFromBinderStatus(
+ mDelegate->getSupportedLatencyModes(outputAidl, &modesAidl)));
+
+ *modes = VALUE_OR_RETURN_STATUS(
+ convertContainer<std::vector<audio_latency_mode_t>>(modesAidl,
+ aidl2legacy_LatencyMode_audio_latency_mode_t));
+
+ return NO_ERROR;
+}
+
////////////////////////////////////////////////////////////////////////////////////////////////////
// AudioFlingerServerAdapter
AudioFlingerServerAdapter::AudioFlingerServerAdapter(
- const sp<AudioFlingerServerAdapter::Delegate>& delegate) : mDelegate(delegate) {}
+ const sp<AudioFlingerServerAdapter::Delegate>& delegate) : mDelegate(delegate) {
+ setMinSchedulerPolicy(SCHED_NORMAL, ANDROID_PRIORITY_AUDIO);
+}
status_t AudioFlingerServerAdapter::onTransact(uint32_t code,
const Parcel& data,
@@ -1304,4 +1333,28 @@
return Status::fromStatusT(mDelegate->setDeviceConnectedState(&portLegacy, connected));
}
+Status AudioFlingerServerAdapter::setRequestedLatencyMode(
+ int32_t output, media::LatencyMode modeAidl) {
+ audio_io_handle_t outputLegacy = VALUE_OR_RETURN_BINDER(
+ aidl2legacy_int32_t_audio_io_handle_t(output));
+ audio_latency_mode_t modeLegacy = VALUE_OR_RETURN_BINDER(
+ aidl2legacy_LatencyMode_audio_latency_mode_t(modeAidl));
+ return Status::fromStatusT(mDelegate->setRequestedLatencyMode(
+ outputLegacy, modeLegacy));
+}
+
+Status AudioFlingerServerAdapter::getSupportedLatencyModes(
+ int output, std::vector<media::LatencyMode>* _aidl_return) {
+ audio_io_handle_t outputLegacy = VALUE_OR_RETURN_BINDER(
+ aidl2legacy_int32_t_audio_io_handle_t(output));
+ std::vector<audio_latency_mode_t> modesLegacy;
+
+ RETURN_BINDER_IF_ERROR(mDelegate->getSupportedLatencyModes(outputLegacy, &modesLegacy));
+
+ *_aidl_return = VALUE_OR_RETURN_BINDER(
+ convertContainer<std::vector<media::LatencyMode>>(
+ modesLegacy, legacy2aidl_audio_latency_mode_t_LatencyMode));
+ return Status::ok();
+}
+
} // namespace android
diff --git a/media/libaudioclient/aidl/android/media/IAudioFlingerClient.aidl b/media/libaudioclient/aidl/android/media/IAudioFlingerClient.aidl
index 421c31c..a2bb024 100644
--- a/media/libaudioclient/aidl/android/media/IAudioFlingerClient.aidl
+++ b/media/libaudioclient/aidl/android/media/IAudioFlingerClient.aidl
@@ -18,6 +18,7 @@
import android.media.AudioIoConfigEvent;
import android.media.AudioIoDescriptor;
+import android.media.LatencyMode;
/**
* A callback interface for AudioFlinger.
@@ -27,4 +28,10 @@
interface IAudioFlingerClient {
oneway void ioConfigChanged(AudioIoConfigEvent event,
in AudioIoDescriptor ioDesc);
+ /**
+ * Called when the latency modes supported on a given output stream change.
+ * output is the I/O handle of the output stream for which the change is signalled.
+ * latencyModes is the new list of supported latency modes (See LatencyMode.aidl).
+ */
+ oneway void onSupportedLatencyModesChanged(int output, in LatencyMode[] latencyModes);
}
diff --git a/media/libaudioclient/aidl/android/media/IAudioFlingerService.aidl b/media/libaudioclient/aidl/android/media/IAudioFlingerService.aidl
index 10da028..9b8a843 100644
--- a/media/libaudioclient/aidl/android/media/IAudioFlingerService.aidl
+++ b/media/libaudioclient/aidl/android/media/IAudioFlingerService.aidl
@@ -35,6 +35,7 @@
import android.media.IAudioFlingerClient;
import android.media.IAudioRecord;
import android.media.IAudioTrack;
+import android.media.LatencyMode;
import android.media.MicrophoneInfoData;
import android.media.RenderPosition;
import android.media.TrackSecondaryOutputInfo;
@@ -228,6 +229,23 @@
void setDeviceConnectedState(in AudioPort devicePort, boolean connected);
+ /**
+ * Requests a given latency mode (See LatencyMode.aidl) on an output stream.
+ * This can be used when some use case on a given mixer/stream can only be enabled
+ * if a specific latency mode is selected on the audio path below the HAL.
+ * For instance spatial audio with head tracking.
+ * output is the I/O handle of the output stream for which the request is made.
+ * latencyMode is the requested latency mode.
+ */
+ void setRequestedLatencyMode(int output, LatencyMode latencyMode);
+
+ /**
+ * Queries the list of latency modes (See LatencyMode.aidl) supported by an output stream.
+ * output is the I/O handle of the output stream to which the query applies.
+ * returns the list of supported latency modes.
+ */
+ LatencyMode[] getSupportedLatencyModes(int output);
+
// When adding a new method, please review and update
// IAudioFlinger.h AudioFlingerServerAdapter::Delegate::TransactionCode
// AudioFlinger.cpp AudioFlinger::onTransactWrapper()
diff --git a/media/libaudioclient/aidl/android/media/ISpatializer.aidl b/media/libaudioclient/aidl/android/media/ISpatializer.aidl
index a61ad58..250c450 100644
--- a/media/libaudioclient/aidl/android/media/ISpatializer.aidl
+++ b/media/libaudioclient/aidl/android/media/ISpatializer.aidl
@@ -96,17 +96,33 @@
/**
* Sets the display orientation.
+ *
+ * This is the rotation of the displayed content relative to its natural orientation.
+ *
* Orientation is expressed in the angle of rotation from the physical "up" side of the screen
* to the logical "up" side of the content displayed the screen. Counterclockwise angles, as
* viewed while facing the screen are positive.
+ *
+ * Note: DisplayManager currently only returns this in increments of 90 degrees,
+ * so the values will be 0, PI/2, PI, 3PI/2.
*/
void setDisplayOrientation(float physicalToLogicalAngle);
/**
* Sets the hinge angle for foldable devices.
+ *
+ * Per the hinge angle sensor, this returns a value from 0 to 2PI.
+ * The value of 0 is considered closed, and PI is considered flat open.
*/
void setHingeAngle(float hingeAngle);
+ /**
+ * Sets whether a foldable is considered "folded" or not.
+ *
+ * The fold state may affect which physical screen is active for display.
+ */
+ void setFoldState(boolean folded);
+
/** Reports the list of supported spatialization modess (see SpatializationMode.aidl).
* The list should never be empty if an ISpatializer interface was successfully
* retrieved with IAudioPolicyService.getSpatializer().
diff --git a/media/libaudioclient/aidl/android/media/LatencyMode.aidl b/media/libaudioclient/aidl/android/media/LatencyMode.aidl
new file mode 100644
index 0000000..0b2a72b
--- /dev/null
+++ b/media/libaudioclient/aidl/android/media/LatencyMode.aidl
@@ -0,0 +1,29 @@
+/*
+ * Copyright 2021 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package android.media;
+
+/**
+ * The latency mode currently used by the spatializer mixer.
+ * {@hide}
+ */
+@Backing(type="byte")
+enum LatencyMode {
+ /** No specific constraint on the latency */
+ FREE = 0,
+ /** A relatively low latency compatible with head tracking operation (e.g less than 100ms) */
+ LOW = 1,
+}
diff --git a/media/libaudioclient/include/media/AidlConversion.h b/media/libaudioclient/include/media/AidlConversion.h
index e769303..1e66164 100644
--- a/media/libaudioclient/include/media/AidlConversion.h
+++ b/media/libaudioclient/include/media/AidlConversion.h
@@ -35,6 +35,7 @@
#include <android/media/AudioTimestampInternal.h>
#include <android/media/AudioUniqueIdUse.h>
#include <android/media/EffectDescriptor.h>
+#include <android/media/LatencyMode.h>
#include <android/media/TrackSecondaryOutputInfo.h>
#include <android/media/audio/common/AudioChannelLayout.h>
#include <android/media/audio/common/AudioConfig.h>
@@ -454,5 +455,9 @@
ConversionResult<audio_direct_mode_t> aidl2legacy_int32_t_audio_direct_mode_t_mask(int32_t aidl);
ConversionResult<int32_t> legacy2aidl_audio_direct_mode_t_int32_t_mask(audio_direct_mode_t legacy);
+ConversionResult<audio_latency_mode_t>
+aidl2legacy_LatencyMode_audio_latency_mode_t(media::LatencyMode aidl);
+ConversionResult<media::LatencyMode>
+legacy2aidl_audio_latency_mode_t_LatencyMode(audio_latency_mode_t legacy);
} // namespace android
diff --git a/media/libaudioclient/include/media/AudioPolicy.h b/media/libaudioclient/include/media/AudioPolicy.h
index 08b3da1..fbb1100 100644
--- a/media/libaudioclient/include/media/AudioPolicy.h
+++ b/media/libaudioclient/include/media/AudioPolicy.h
@@ -137,6 +137,11 @@
== MIX_ROUTE_FLAG_LOOP_BACK_AND_RENDER;
}
+static inline bool is_mix_loopback(uint32_t routeFlags) {
+ return (routeFlags & MIX_ROUTE_FLAG_LOOP_BACK)
+ == MIX_ROUTE_FLAG_LOOP_BACK;
+}
+
}; // namespace android
#endif // ANDROID_AUDIO_POLICY_H
diff --git a/media/libaudioclient/include/media/AudioSystem.h b/media/libaudioclient/include/media/AudioSystem.h
index 360b83d..9411f46 100644
--- a/media/libaudioclient/include/media/AudioSystem.h
+++ b/media/libaudioclient/include/media/AudioSystem.h
@@ -569,6 +569,12 @@
static status_t getDirectProfilesForAttributes(const audio_attributes_t* attr,
std::vector<audio_profile>* audioProfiles);
+ static status_t setRequestedLatencyMode(
+ audio_io_handle_t output, audio_latency_mode_t mode);
+
+ static status_t getSupportedLatencyModes(audio_io_handle_t output,
+ std::vector<audio_latency_mode_t>* modes);
+
// A listener for capture state changes.
class CaptureStateListener : public virtual RefBase {
public:
@@ -644,6 +650,22 @@
audio_io_handle_t audioIo,
audio_port_handle_t portId);
+ class SupportedLatencyModesCallback : public virtual RefBase
+ {
+ public:
+
+ SupportedLatencyModesCallback() = default;
+ virtual ~SupportedLatencyModesCallback() = default;
+
+ virtual void onSupportedLatencyModesChanged(
+ audio_io_handle_t output, const std::vector<audio_latency_mode_t>& modes) = 0;
+ };
+
+ static status_t addSupportedLatencyModesCallback(
+ const sp<SupportedLatencyModesCallback>& callback);
+ static status_t removeSupportedLatencyModesCallback(
+ const sp<SupportedLatencyModesCallback>& callback);
+
static audio_port_handle_t getDeviceIdForIo(audio_io_handle_t audioIo);
static status_t setVibratorInfos(const std::vector<media::AudioVibratorInfo>& vibratorInfos);
@@ -682,6 +704,9 @@
media::AudioIoConfigEvent event,
const media::AudioIoDescriptor& ioDesc) override;
+ binder::Status onSupportedLatencyModesChanged(
+ int output, const std::vector<media::LatencyMode>& latencyModes) override;
+
status_t addAudioDeviceCallback(const wp<AudioDeviceCallback>& callback,
audio_io_handle_t audioIo,
audio_port_handle_t portId);
@@ -689,6 +714,11 @@
audio_io_handle_t audioIo,
audio_port_handle_t portId);
+ status_t addSupportedLatencyModesCallback(
+ const sp<SupportedLatencyModesCallback>& callback);
+ status_t removeSupportedLatencyModesCallback(
+ const sp<SupportedLatencyModesCallback>& callback);
+
audio_port_handle_t getDeviceIdForIo(audio_io_handle_t audioIo);
private:
@@ -697,6 +727,10 @@
std::map<audio_io_handle_t, std::map<audio_port_handle_t, wp<AudioDeviceCallback>>>
mAudioDeviceCallbacks;
+
+ std::vector<wp<SupportedLatencyModesCallback>>
+ mSupportedLatencyModesCallbacks GUARDED_BY(mLock);
+
// cached values for recording getInputBufferSize() queries
size_t mInBuffSize; // zero indicates cache is invalid
uint32_t mInSamplingRate;
diff --git a/media/libaudioclient/include/media/AudioTrack.h b/media/libaudioclient/include/media/AudioTrack.h
index 9f540e6..ee81ef8 100644
--- a/media/libaudioclient/include/media/AudioTrack.h
+++ b/media/libaudioclient/include/media/AudioTrack.h
@@ -1176,6 +1176,9 @@
bool isPlaying() {
AutoMutex lock(mLock);
+ return isPlaying_l();
+ }
+ bool isPlaying_l() {
return mState == STATE_ACTIVE || mState == STATE_STOPPING;
}
diff --git a/media/libaudioclient/include/media/IAudioFlinger.h b/media/libaudioclient/include/media/IAudioFlinger.h
index 3c3715d..c891ae6 100644
--- a/media/libaudioclient/include/media/IAudioFlinger.h
+++ b/media/libaudioclient/include/media/IAudioFlinger.h
@@ -360,6 +360,13 @@
virtual int32_t getAAudioHardwareBurstMinUsec() = 0;
virtual status_t setDeviceConnectedState(const struct audio_port_v7 *port, bool connected) = 0;
+
+ virtual status_t setRequestedLatencyMode(
+ audio_io_handle_t output, audio_latency_mode_t mode) = 0;
+
+ virtual status_t getSupportedLatencyModes(audio_io_handle_t output,
+ std::vector<audio_latency_mode_t>* modes) = 0;
+
};
/**
@@ -462,6 +469,10 @@
int32_t getAAudioMixerBurstCount() override;
int32_t getAAudioHardwareBurstMinUsec() override;
status_t setDeviceConnectedState(const struct audio_port_v7 *port, bool connected) override;
+ status_t setRequestedLatencyMode(audio_io_handle_t output,
+ audio_latency_mode_t mode) override;
+ status_t getSupportedLatencyModes(
+ audio_io_handle_t output, std::vector<audio_latency_mode_t>* modes) override;
private:
const sp<media::IAudioFlingerService> mDelegate;
@@ -551,6 +562,8 @@
GET_AAUDIO_MIXER_BURST_COUNT = media::BnAudioFlingerService::TRANSACTION_getAAudioMixerBurstCount,
GET_AAUDIO_HARDWARE_BURST_MIN_USEC = media::BnAudioFlingerService::TRANSACTION_getAAudioHardwareBurstMinUsec,
SET_DEVICE_CONNECTED_STATE = media::BnAudioFlingerService::TRANSACTION_setDeviceConnectedState,
+ SET_REQUESTED_LATENCY_MODE = media::BnAudioFlingerService::TRANSACTION_setRequestedLatencyMode,
+ GET_SUPPORTED_LATENCY_MODES = media::BnAudioFlingerService::TRANSACTION_getSupportedLatencyModes,
};
protected:
@@ -672,7 +685,9 @@
Status getAAudioMixerBurstCount(int32_t* _aidl_return) override;
Status getAAudioHardwareBurstMinUsec(int32_t* _aidl_return) override;
Status setDeviceConnectedState(const media::AudioPort& port, bool connected) override;
-
+ Status setRequestedLatencyMode(int output, media::LatencyMode mode) override;
+ Status getSupportedLatencyModes(int output,
+ std::vector<media::LatencyMode>* _aidl_return) override;
private:
const sp<AudioFlingerServerAdapter::Delegate> mDelegate;
};
diff --git a/media/libaudiohal/impl/EffectHalHidl.cpp b/media/libaudiohal/impl/EffectHalHidl.cpp
index fdfe225..3956a6c 100644
--- a/media/libaudiohal/impl/EffectHalHidl.cpp
+++ b/media/libaudiohal/impl/EffectHalHidl.cpp
@@ -17,11 +17,16 @@
#define LOG_TAG "EffectHalHidl"
//#define LOG_NDEBUG 0
+#include <android/hidl/manager/1.0/IServiceManager.h>
+#include <android-base/stringprintf.h>
#include <common/all-versions/VersionUtils.h>
#include <cutils/native_handle.h>
+#include <cutils/properties.h>
#include <hwbinder/IPCThreadState.h>
#include <media/EffectsFactoryApi.h>
+#include <mediautils/SchedulingPolicyService.h>
#include <mediautils/TimeCheck.h>
+#include <system/audio_effects/effect_spatializer.h>
#include <utils/Log.h>
#include <util/EffectUtils.h>
@@ -50,6 +55,18 @@
effect_descriptor_t halDescriptor{};
if (EffectHalHidl::getDescriptor(&halDescriptor) == NO_ERROR) {
mIsInput = (halDescriptor.flags & EFFECT_FLAG_TYPE_PRE_PROC) == EFFECT_FLAG_TYPE_PRE_PROC;
+ const bool isSpatializer =
+ memcmp(&halDescriptor.type, FX_IID_SPATIALIZER, sizeof(effect_uuid_t)) == 0;
+ if (isSpatializer) {
+ constexpr int32_t kRTPriorityMin = 1;
+ constexpr int32_t kRTPriorityMax = 3;
+ const int32_t priorityBoost = property_get_int32("audio.spatializer.priority", 1);
+ if (priorityBoost >= kRTPriorityMin && priorityBoost <= kRTPriorityMax) {
+ ALOGD("%s: audio.spatializer.priority %d on effect %lld",
+ __func__, priorityBoost, (long long)effectId);
+ mHalThreadPriority = priorityBoost;
+ }
+ }
}
}
@@ -93,13 +110,13 @@
}
status_t EffectHalHidl::process() {
- TIME_CHECK();
+ // TIME_CHECK(); // TODO(b/238654698) reenable only when optimized.
return processImpl(static_cast<uint32_t>(MessageQueueFlagBits::REQUEST_PROCESS));
}
status_t EffectHalHidl::processReverse() {
- TIME_CHECK();
+ // TIME_CHECK(); // TODO(b/238654698) reenable only when optimized.
return processImpl(static_cast<uint32_t>(MessageQueueFlagBits::REQUEST_PROCESS_REVERSE));
}
@@ -127,6 +144,8 @@
ALOGE_IF(!mEfGroup, "Event flag creation for effects failed");
return NO_INIT;
}
+
+ (void)checkHalThreadPriority();
mStatusMQ = std::move(tempStatusMQ);
return OK;
}
@@ -317,5 +336,67 @@
return result;
}
+status_t EffectHalHidl::getHalPid(pid_t *pid) const {
+ using ::android::hidl::base::V1_0::DebugInfo;
+ using ::android::hidl::manager::V1_0::IServiceManager;
+ DebugInfo debugInfo;
+ const auto ret = mEffect->getDebugInfo([&] (const auto &info) {
+ debugInfo = info;
+ });
+ if (!ret.isOk()) {
+ ALOGW("%s: cannot get effect debug info", __func__);
+ return INVALID_OPERATION;
+ }
+ if (debugInfo.pid != (int)IServiceManager::PidConstant::NO_PID) {
+ *pid = debugInfo.pid;
+ return NO_ERROR;
+ }
+ ALOGW("%s: effect debug info does not contain pid", __func__);
+ return NAME_NOT_FOUND;
+}
+
+status_t EffectHalHidl::getHalWorkerTid(pid_t *tid) {
+ int32_t reply = -1;
+ uint32_t replySize = sizeof(reply);
+ const status_t status =
+ command('gtid', 0 /* cmdSize */, nullptr /* pCmdData */, &replySize, &reply);
+ if (status == OK) {
+ *tid = (pid_t)reply;
+ } else {
+ ALOGW("%s: failed with status:%d", __func__, status);
+ }
+ return status;
+}
+
+bool EffectHalHidl::requestHalThreadPriority(pid_t threadPid, pid_t threadId) {
+ if (mHalThreadPriority == kRTPriorityDisabled) {
+ return true;
+ }
+ const int err = requestPriority(
+ threadPid, threadId,
+ mHalThreadPriority, false /*isForApp*/, true /*asynchronous*/);
+ ALOGW_IF(err, "%s: failed to set RT priority %d for pid %d tid %d; error %d",
+ __func__, mHalThreadPriority, threadPid, threadId, err);
+ // Audio will still work, but may be more susceptible to glitches.
+ return err == 0;
+}
+
+status_t EffectHalHidl::checkHalThreadPriority() {
+ if (mHalThreadPriority == kRTPriorityDisabled) return OK;
+ if (mHalThreadPriority < kRTPriorityMin
+ || mHalThreadPriority > kRTPriorityMax) return BAD_VALUE;
+
+ pid_t halPid, halWorkerTid;
+ const status_t status = getHalPid(&halPid) ?: getHalWorkerTid(&halWorkerTid);
+ const bool success = status == OK && requestHalThreadPriority(halPid, halWorkerTid);
+ ALOGD("%s: effectId %lld RT priority(%d) request %s%s",
+ __func__, (long long)mEffectId, mHalThreadPriority,
+ success ? "succeeded" : "failed",
+ status == OK
+ ? base::StringPrintf(" for pid:%d tid:%d", halPid, halWorkerTid).c_str()
+ : " (pid / tid cannot be read)");
+ return success ? OK : status != OK ? status : INVALID_OPERATION /* request failed */;
+}
+
} // namespace effect
} // namespace android
diff --git a/media/libaudiohal/impl/EffectHalHidl.h b/media/libaudiohal/impl/EffectHalHidl.h
index e139768..94dcd7e 100644
--- a/media/libaudiohal/impl/EffectHalHidl.h
+++ b/media/libaudiohal/impl/EffectHalHidl.h
@@ -78,6 +78,11 @@
std::unique_ptr<StatusMQ> mStatusMQ;
EventFlag* mEfGroup;
bool mIsInput = false;
+ static constexpr int32_t kRTPriorityMin = 1;
+ static constexpr int32_t kRTPriorityMax = 3;
+ static constexpr int kRTPriorityDisabled = 0;
+ // Typical RealTime mHalThreadPriority ranges from 1 (low) to 3 (high).
+ int mHalThreadPriority = kRTPriorityDisabled;
// Can not be constructed directly by clients.
EffectHalHidl(const sp<IEffect>& effect, uint64_t effectId);
@@ -93,6 +98,10 @@
uint32_t cmdCode, uint32_t cmdSize, void *pCmdData,
uint32_t *replySize, void *pReplyData);
status_t setProcessBuffers();
+ status_t getHalPid(pid_t *pid) const;
+ status_t getHalWorkerTid(pid_t *tid);
+ bool requestHalThreadPriority(pid_t threadPid, pid_t threadId);
+ status_t checkHalThreadPriority();
};
} // namespace effect
diff --git a/media/libaudiohal/impl/EffectsFactoryHalHidl.cpp b/media/libaudiohal/impl/EffectsFactoryHalHidl.cpp
index d7217fc..3470380 100644
--- a/media/libaudiohal/impl/EffectsFactoryHalHidl.cpp
+++ b/media/libaudiohal/impl/EffectsFactoryHalHidl.cpp
@@ -17,6 +17,9 @@
#define LOG_TAG "EffectsFactoryHalHidl"
//#define LOG_NDEBUG 0
+#include <optional>
+#include <tuple>
+
#include <cutils/native_handle.h>
#include <UuidUtils.h>
@@ -38,51 +41,71 @@
using namespace ::android::hardware::audio::common::CPP_VERSION;
using namespace ::android::hardware::audio::effect::CPP_VERSION;
+class EffectDescriptorCache {
+ public:
+ using QueryResult = std::tuple<Return<void>, Result, hidl_vec<EffectDescriptor>>;
+ QueryResult queryAllDescriptors(IEffectsFactory* effectsFactory);
+ private:
+ std::mutex mLock;
+ std::optional<hidl_vec<EffectDescriptor>> mLastDescriptors; // GUARDED_BY(mLock)
+};
+
+EffectDescriptorCache::QueryResult EffectDescriptorCache::queryAllDescriptors(
+ IEffectsFactory* effectsFactory) {
+ {
+ std::lock_guard l(mLock);
+ if (mLastDescriptors.has_value()) {
+ return {::android::hardware::Void(), Result::OK, mLastDescriptors.value()};
+ }
+ }
+ Result retval = Result::NOT_INITIALIZED;
+ hidl_vec<EffectDescriptor> descriptors;
+ Return<void> ret = effectsFactory->getAllDescriptors(
+ [&](Result r, const hidl_vec<EffectDescriptor>& result) {
+ retval = r;
+ if (retval == Result::OK) {
+ descriptors = result;
+ }
+ });
+ if (ret.isOk() && retval == Result::OK) {
+ std::lock_guard l(mLock);
+ mLastDescriptors = descriptors;
+ }
+ return {std::move(ret), retval, std::move(descriptors)};
+}
+
EffectsFactoryHalHidl::EffectsFactoryHalHidl(sp<IEffectsFactory> effectsFactory)
- : EffectConversionHelperHidl("EffectsFactory") {
+ : EffectConversionHelperHidl("EffectsFactory"), mCache(new EffectDescriptorCache) {
ALOG_ASSERT(effectsFactory != nullptr, "Provided IEffectsFactory service is NULL");
mEffectsFactory = effectsFactory;
}
-status_t EffectsFactoryHalHidl::queryAllDescriptors() {
- if (mEffectsFactory == 0) return NO_INIT;
- Result retval = Result::NOT_INITIALIZED;
- Return<void> ret = mEffectsFactory->getAllDescriptors(
- [&](Result r, const hidl_vec<EffectDescriptor>& result) {
- retval = r;
- if (retval == Result::OK) {
- mLastDescriptors = result;
- }
- });
- if (ret.isOk()) {
- return retval == Result::OK ? OK : NO_INIT;
- }
- mLastDescriptors.resize(0);
- return processReturn(__FUNCTION__, ret);
-}
-
status_t EffectsFactoryHalHidl::queryNumberEffects(uint32_t *pNumEffects) {
- status_t queryResult = queryAllDescriptors();
- if (queryResult == OK) {
- *pNumEffects = mLastDescriptors.size();
+ if (mEffectsFactory == 0) return NO_INIT;
+ auto [ret, retval, descriptors] = mCache->queryAllDescriptors(mEffectsFactory.get());
+ if (ret.isOk() && retval == Result::OK) {
+ *pNumEffects = descriptors.size();
+ return OK;
+ } else if (ret.isOk()) {
+ return NO_INIT;
}
- return queryResult;
+ return processReturn(__FUNCTION__, ret);
}
status_t EffectsFactoryHalHidl::getDescriptor(
uint32_t index, effect_descriptor_t *pDescriptor) {
- // TODO: We need somehow to track the changes on the server side
- // or figure out how to convert everybody to query all the descriptors at once.
if (pDescriptor == nullptr) {
return BAD_VALUE;
}
- if (mLastDescriptors.size() == 0) {
- status_t queryResult = queryAllDescriptors();
- if (queryResult != OK) return queryResult;
+ if (mEffectsFactory == 0) return NO_INIT;
+ auto [ret, retval, descriptors] = mCache->queryAllDescriptors(mEffectsFactory.get());
+ if (ret.isOk() && retval == Result::OK) {
+ if (index >= descriptors.size()) return NAME_NOT_FOUND;
+ EffectUtils::effectDescriptorToHal(descriptors[index], pDescriptor);
+ } else if (ret.isOk()) {
+ return NO_INIT;
}
- if (index >= mLastDescriptors.size()) return NAME_NOT_FOUND;
- EffectUtils::effectDescriptorToHal(mLastDescriptors[index], pDescriptor);
- return OK;
+ return processReturn(__FUNCTION__, ret);
}
status_t EffectsFactoryHalHidl::getDescriptor(
@@ -114,21 +137,15 @@
if (pEffectType == nullptr || descriptors == nullptr) {
return BAD_VALUE;
}
+ if (mEffectsFactory == 0) return NO_INIT;
- uint32_t numEffects = 0;
- status_t status = queryNumberEffects(&numEffects);
- if (status != NO_ERROR) {
- ALOGW("%s error %d from FactoryHal queryNumberEffects", __func__, status);
- return status;
+ auto [ret, retval, hidlDescs] = mCache->queryAllDescriptors(mEffectsFactory.get());
+ if (!ret.isOk() || retval != Result::OK) {
+ return processReturn(__FUNCTION__, ret, retval);
}
-
- for (uint32_t i = 0; i < numEffects; i++) {
+ for (const auto& hidlDesc : hidlDescs) {
effect_descriptor_t descriptor;
- status = getDescriptor(i, &descriptor);
- if (status != NO_ERROR) {
- ALOGW("%s error %d from FactoryHal getDescriptor", __func__, status);
- continue;
- }
+ EffectUtils::effectDescriptorToHal(hidlDesc, &descriptor);
if (memcmp(&descriptor.type, pEffectType, sizeof(effect_uuid_t)) == 0) {
descriptors->push_back(descriptor);
}
diff --git a/media/libaudiohal/impl/EffectsFactoryHalHidl.h b/media/libaudiohal/impl/EffectsFactoryHalHidl.h
index e1882e1..7ccddc6 100644
--- a/media/libaudiohal/impl/EffectsFactoryHalHidl.h
+++ b/media/libaudiohal/impl/EffectsFactoryHalHidl.h
@@ -17,6 +17,8 @@
#ifndef ANDROID_HARDWARE_EFFECTS_FACTORY_HAL_HIDL_H
#define ANDROID_HARDWARE_EFFECTS_FACTORY_HAL_HIDL_H
+#include <memory>
+
#include PATH(android/hardware/audio/effect/FILE_VERSION/IEffectsFactory.h)
#include <media/audiohal/EffectsFactoryHalInterface.h>
@@ -28,6 +30,8 @@
using ::android::hardware::hidl_vec;
using namespace ::android::hardware::audio::effect::CPP_VERSION;
+class EffectDescriptorCache;
+
class EffectsFactoryHalHidl : public EffectsFactoryHalInterface, public EffectConversionHelperHidl
{
public:
@@ -63,9 +67,7 @@
private:
sp<IEffectsFactory> mEffectsFactory;
- hidl_vec<EffectDescriptor> mLastDescriptors;
-
- status_t queryAllDescriptors();
+ std::unique_ptr<EffectDescriptorCache> mCache;
};
} // namespace effect
diff --git a/media/libaudiohal/impl/StreamHalHidl.cpp b/media/libaudiohal/impl/StreamHalHidl.cpp
index 021ec51..76f9a60 100644
--- a/media/libaudiohal/impl/StreamHalHidl.cpp
+++ b/media/libaudiohal/impl/StreamHalHidl.cpp
@@ -441,7 +441,7 @@
#endif
status_t StreamOutHalHidl::write(const void *buffer, size_t bytes, size_t *written) {
- TIME_CHECK();
+ // TIME_CHECK(); // TODO(b/238654698) reenable only when optimized.
if (mStream == 0) return NO_INIT;
*written = 0;
@@ -587,7 +587,7 @@
}
status_t StreamOutHalHidl::getRenderPosition(uint32_t *dspFrames) {
- TIME_CHECK();
+ // TIME_CHECK(); // TODO(b/238654698) reenable only when optimized.
if (mStream == 0) return NO_INIT;
Result retval;
Return<void> ret = mStream->getRenderPosition(
@@ -668,7 +668,7 @@
}
status_t StreamOutHalHidl::getPresentationPosition(uint64_t *frames, struct timespec *timestamp) {
- TIME_CHECK();
+ // TIME_CHECK(); // TODO(b/238654698) reenable only when optimized.
if (mStream == 0) return NO_INIT;
if (mWriterClient == gettid() && mCommandMQ) {
return callWriterThread(
@@ -1012,7 +1012,7 @@
}
status_t StreamInHalHidl::read(void *buffer, size_t bytes, size_t *read) {
- TIME_CHECK();
+ // TIME_CHECK(); // TODO(b/238654698) reenable only when optimized.
if (mStream == 0) return NO_INIT;
*read = 0;
@@ -1146,7 +1146,7 @@
}
status_t StreamInHalHidl::getCapturePosition(int64_t *frames, int64_t *time) {
- TIME_CHECK();
+ // TIME_CHECK(); // TODO(b/238654698) reenable only when optimized.
if (mStream == 0) return NO_INIT;
if (mReaderClient == gettid() && mCommandMQ) {
ReadParameters params;
diff --git a/media/libheadtracking/Android.bp b/media/libheadtracking/Android.bp
index 9d63f9b..9955862 100644
--- a/media/libheadtracking/Android.bp
+++ b/media/libheadtracking/Android.bp
@@ -16,11 +16,17 @@
"Pose.cpp",
"PoseBias.cpp",
"PoseDriftCompensator.cpp",
+ "PosePredictor.cpp",
"PoseRateLimiter.cpp",
"QuaternionUtil.cpp",
"ScreenHeadFusion.cpp",
"StillnessDetector.cpp",
"Twist.cpp",
+ "VectorRecorder.cpp",
+ ],
+ shared_libs: [
+ "libaudioutils",
+ "libbase",
],
export_include_dirs: [
"include",
@@ -31,6 +37,15 @@
export_header_lib_headers: [
"libeigen",
],
+ cflags: [
+ "-Wthread-safety",
+ ],
+ product_variables: {
+ debuggable: {
+ // enable experiments only in userdebug and eng builds
+ cflags: ["-DENABLE_VERIFICATION"],
+ },
+ },
}
cc_library {
@@ -39,6 +54,7 @@
"SensorPoseProvider.cpp",
],
shared_libs: [
+ "libbase",
"libheadtracking",
"liblog",
"libsensor",
@@ -71,6 +87,7 @@
"Pose-test.cpp",
"PoseBias-test.cpp",
"PoseDriftCompensator-test.cpp",
+ "PosePredictor.cpp",
"PoseRateLimiter-test.cpp",
"QuaternionUtil-test.cpp",
"ScreenHeadFusion-test.cpp",
@@ -78,6 +95,8 @@
"Twist-test.cpp",
],
shared_libs: [
+ "libaudioutils",
+ "libbase", // StringAppendF
"libheadtracking",
],
}
diff --git a/media/libheadtracking/HeadTrackingProcessor-test.cpp b/media/libheadtracking/HeadTrackingProcessor-test.cpp
index 299192f..5190f52 100644
--- a/media/libheadtracking/HeadTrackingProcessor-test.cpp
+++ b/media/libheadtracking/HeadTrackingProcessor-test.cpp
@@ -15,10 +15,10 @@
*/
#include "media/HeadTrackingProcessor.h"
+#include "media/QuaternionUtil.h"
#include <gtest/gtest.h>
-#include "QuaternionUtil.h"
#include "TestUtil.h"
namespace android {
@@ -82,6 +82,8 @@
std::unique_ptr<HeadTrackingProcessor> processor = createHeadTrackingProcessor(
Options{.predictionDuration = 2.f}, HeadTrackingMode::WORLD_RELATIVE);
+ processor->setPosePredictorType(PosePredictorType::TWIST);
+
// Establish a baseline for the drift compensators.
processor->setWorldToHeadPose(0, Pose3f(), Twist3f());
processor->setWorldToScreenPose(0, Pose3f());
diff --git a/media/libheadtracking/HeadTrackingProcessor.cpp b/media/libheadtracking/HeadTrackingProcessor.cpp
index 71fae8a..54d08d2 100644
--- a/media/libheadtracking/HeadTrackingProcessor.cpp
+++ b/media/libheadtracking/HeadTrackingProcessor.cpp
@@ -13,12 +13,16 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
+#include <inttypes.h>
+#include <android-base/stringprintf.h>
+#include <audio_utils/SimpleLog.h>
#include "media/HeadTrackingProcessor.h"
+#include "media/QuaternionUtil.h"
#include "ModeSelector.h"
#include "PoseBias.h"
-#include "QuaternionUtil.h"
+#include "PosePredictor.h"
#include "ScreenHeadFusion.h"
#include "StillnessDetector.h"
@@ -26,6 +30,7 @@
namespace media {
namespace {
+using android::base::StringAppendF;
using Eigen::Quaternionf;
using Eigen::Vector3f;
@@ -55,8 +60,8 @@
void setWorldToHeadPose(int64_t timestamp, const Pose3f& worldToHead,
const Twist3f& headTwist) override {
- Pose3f predictedWorldToHead =
- worldToHead * integrate(headTwist, mOptions.predictionDuration);
+ const Pose3f predictedWorldToHead = mPosePredictor.predict(
+ timestamp, worldToHead, headTwist, mOptions.predictionDuration);
mHeadPoseBias.setInput(predictedWorldToHead);
mHeadStillnessDetector.setInput(timestamp, predictedWorldToHead);
mWorldToHeadTimestamp = timestamp;
@@ -84,14 +89,16 @@
}
void calculate(int64_t timestamp) override {
- // Handle the screen first, since it might trigger a recentering of the head.
+ bool screenStable = true;
+
+ // Handle the screen first, since it might: trigger a recentering of the head.
if (mWorldToScreenTimestamp.has_value()) {
const Pose3f worldToLogicalScreen = mScreenPoseBias.getOutput();
- bool screenStable = mScreenStillnessDetector.calculate(timestamp);
+ screenStable = mScreenStillnessDetector.calculate(timestamp);
mModeSelector.setScreenStable(mWorldToScreenTimestamp.value(), screenStable);
// Whenever the screen is unstable, recenter the head pose.
if (!screenStable) {
- recenter(true, false);
+ recenter(true, false, "calculate: screen movement");
}
mScreenHeadFusion.setWorldToScreenPose(mWorldToScreenTimestamp.value(),
worldToLogicalScreen);
@@ -101,8 +108,9 @@
if (mWorldToHeadTimestamp.has_value()) {
Pose3f worldToHead = mHeadPoseBias.getOutput();
// Auto-recenter.
- if (mHeadStillnessDetector.calculate(timestamp)) {
- recenter(true, false);
+ bool headStable = mHeadStillnessDetector.calculate(timestamp);
+ if (headStable || !screenStable) {
+ recenter(true, false, "calculate: head movement");
worldToHead = mHeadPoseBias.getOutput();
}
@@ -132,14 +140,16 @@
HeadTrackingMode getActualMode() const override { return mModeSelector.getActualMode(); }
- void recenter(bool recenterHead, bool recenterScreen) override {
+ void recenter(bool recenterHead, bool recenterScreen, std::string source) override {
if (recenterHead) {
mHeadPoseBias.recenter();
mHeadStillnessDetector.reset();
+ mLocalLog.log("recenter Head from %s", source.c_str());
}
if (recenterScreen) {
mScreenPoseBias.recenter();
mScreenStillnessDetector.reset();
+ mLocalLog.log("recenter Screen from %s", source.c_str());
}
// If a sensor being recentered is included in the current mode, apply rate limiting to
@@ -152,6 +162,41 @@
}
}
+ void setPosePredictorType(PosePredictorType type) override {
+ mPosePredictor.setPosePredictorType(type);
+ }
+
+ std::string toString_l(unsigned level) const override {
+ std::string prefixSpace(level, ' ');
+ std::string ss = prefixSpace + "HeadTrackingProcessor:\n";
+ StringAppendF(&ss, "%s maxTranslationalVelocity: %f meter/second\n", prefixSpace.c_str(),
+ mOptions.maxTranslationalVelocity);
+ StringAppendF(&ss, "%s maxRotationalVelocity: %f rad/second\n", prefixSpace.c_str(),
+ mOptions.maxRotationalVelocity);
+ StringAppendF(&ss, "%s freshnessTimeout: %0.4f ms\n", prefixSpace.c_str(),
+ media::nsToFloatMs(mOptions.freshnessTimeout));
+ StringAppendF(&ss, "%s predictionDuration: %0.4f ms\n", prefixSpace.c_str(),
+ media::nsToFloatMs(mOptions.predictionDuration));
+ StringAppendF(&ss, "%s autoRecenterWindowDuration: %0.4f ms\n", prefixSpace.c_str(),
+ media::nsToFloatMs(mOptions.autoRecenterWindowDuration));
+ StringAppendF(&ss, "%s autoRecenterTranslationalThreshold: %f meter\n", prefixSpace.c_str(),
+ mOptions.autoRecenterTranslationalThreshold);
+ StringAppendF(&ss, "%s autoRecenterRotationalThreshold: %f radians\n", prefixSpace.c_str(),
+ mOptions.autoRecenterRotationalThreshold);
+ StringAppendF(&ss, "%s screenStillnessWindowDuration: %0.4f ms\n", prefixSpace.c_str(),
+ media::nsToFloatMs(mOptions.screenStillnessWindowDuration));
+ StringAppendF(&ss, "%s screenStillnessTranslationalThreshold: %f meter\n",
+ prefixSpace.c_str(), mOptions.screenStillnessTranslationalThreshold);
+ StringAppendF(&ss, "%s screenStillnessRotationalThreshold: %f radians\n",
+ prefixSpace.c_str(), mOptions.screenStillnessRotationalThreshold);
+ ss += mModeSelector.toString(level + 1);
+ ss += mRateLimiter.toString(level + 1);
+ ss += mPosePredictor.toString(level + 1);
+ ss.append(prefixSpace + "ReCenterHistory:\n");
+ ss += mLocalLog.dumpToString((prefixSpace + " ").c_str(), mMaxLocalLogLine);
+ return ss;
+ }
+
private:
const Options mOptions;
float mPhysicalToLogicalAngle = 0;
@@ -168,6 +213,9 @@
ScreenHeadFusion mScreenHeadFusion;
ModeSelector mModeSelector;
PoseRateLimiter mRateLimiter;
+ PosePredictor mPosePredictor;
+ static constexpr std::size_t mMaxLocalLogLine = 10;
+ SimpleLog mLocalLog{mMaxLocalLogLine};
};
} // namespace
@@ -177,5 +225,38 @@
return std::make_unique<HeadTrackingProcessorImpl>(options, initialMode);
}
+std::string toString(HeadTrackingMode mode) {
+ switch (mode) {
+ case HeadTrackingMode::STATIC:
+ return "STATIC";
+ case HeadTrackingMode::WORLD_RELATIVE:
+ return "WORLD_RELATIVE";
+ case HeadTrackingMode::SCREEN_RELATIVE:
+ return "SCREEN_RELATIVE";
+ }
+ return "EnumNotImplemented";
+};
+
+std::string toString(PosePredictorType posePredictorType) {
+ switch (posePredictorType) {
+ case PosePredictorType::AUTO: return "AUTO";
+ case PosePredictorType::LAST: return "LAST";
+ case PosePredictorType::TWIST: return "TWIST";
+ case PosePredictorType::LEAST_SQUARES: return "LEAST_SQUARES";
+ }
+ return "UNKNOWN" + std::to_string((int)posePredictorType);
+}
+
+bool isValidPosePredictorType(PosePredictorType posePredictorType) {
+ switch (posePredictorType) {
+ case PosePredictorType::AUTO:
+ case PosePredictorType::LAST:
+ case PosePredictorType::TWIST:
+ case PosePredictorType::LEAST_SQUARES:
+ return true;
+ }
+ return false;
+}
+
} // namespace media
} // namespace android
diff --git a/media/libheadtracking/ModeSelector-test.cpp b/media/libheadtracking/ModeSelector-test.cpp
index a136e6b..6925908 100644
--- a/media/libheadtracking/ModeSelector-test.cpp
+++ b/media/libheadtracking/ModeSelector-test.cpp
@@ -18,7 +18,7 @@
#include <gtest/gtest.h>
-#include "QuaternionUtil.h"
+#include "media/QuaternionUtil.h"
#include "TestUtil.h"
namespace android {
diff --git a/media/libheadtracking/ModeSelector.cpp b/media/libheadtracking/ModeSelector.cpp
index cb3a27f..7ee21b3 100644
--- a/media/libheadtracking/ModeSelector.cpp
+++ b/media/libheadtracking/ModeSelector.cpp
@@ -13,11 +13,13 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
+#include <android-base/stringprintf.h>
#include "ModeSelector.h"
namespace android {
namespace media {
+using android::base::StringAppendF;
ModeSelector::ModeSelector(const Options& options, HeadTrackingMode initialMode)
: mOptions(options), mDesiredMode(initialMode), mActualMode(initialMode) {}
@@ -47,12 +49,15 @@
}
void ModeSelector::calculateActualMode(int64_t timestamp) {
- bool isValidScreenToHead = mScreenToHead.has_value() &&
- timestamp - mScreenToHeadTimestamp < mOptions.freshnessTimeout;
- bool isValidWorldToHead = mWorldToHead.has_value() &&
- timestamp - mWorldToHeadTimestamp < mOptions.freshnessTimeout;
- bool isValidScreenStable = mScreenStable.has_value() &&
- timestamp - mScreenStableTimestamp < mOptions.freshnessTimeout;
+ int64_t screenToHeadGap = timestamp - mScreenToHeadTimestamp;
+ int64_t worldToHeadGap = timestamp - mWorldToHeadTimestamp;
+ int64_t screenStableGap = timestamp - mScreenStableTimestamp;
+ bool isValidScreenToHead =
+ mScreenToHead.has_value() && screenToHeadGap < mOptions.freshnessTimeout;
+ bool isValidWorldToHead =
+ mWorldToHead.has_value() && worldToHeadGap < mOptions.freshnessTimeout;
+ bool isValidScreenStable =
+ mScreenStable.has_value() && screenStableGap < mOptions.freshnessTimeout;
HeadTrackingMode mode = mDesiredMode;
@@ -70,7 +75,17 @@
}
}
- mActualMode = mode;
+ if (mode != mActualMode) {
+ mLocalLog.log(
+ "HT mode change from %s to %s, this ts %0.4f ms, lastTs+gap [ScreenToHead %0.4f + "
+ "%0.4f, WorldToHead %0.4f + %0.4f, ScreenStable %0.4f + %0.4f] ms",
+ media::toString(mActualMode).c_str(), media::toString(mode).c_str(),
+ media::nsToFloatMs(timestamp), media::nsToFloatMs(mScreenToHeadTimestamp),
+ media::nsToFloatMs(screenToHeadGap), media::nsToFloatMs(mWorldToHeadTimestamp),
+ media::nsToFloatMs(worldToHeadGap), media::nsToFloatMs(mScreenStableTimestamp),
+ media::nsToFloatMs(screenStableGap));
+ mActualMode = mode;
+ }
}
void ModeSelector::calculate(int64_t timestamp) {
@@ -99,5 +114,17 @@
return mActualMode;
}
+std::string ModeSelector::toString(unsigned level) const {
+ std::string prefixSpace(level, ' ');
+ std::string ss(prefixSpace);
+ ss.append("ModeSelector: ScreenToStage ")
+ .append(mScreenToStage.toString())
+ .append("\n")
+ .append(prefixSpace)
+ .append("Mode change history:\n")
+ .append(mLocalLog.dumpToString((prefixSpace + " ").c_str(), sMaxLocalLogLine));
+ return ss;
+}
+
} // namespace media
} // namespace android
diff --git a/media/libheadtracking/ModeSelector.h b/media/libheadtracking/ModeSelector.h
index e537040..2475a5b 100644
--- a/media/libheadtracking/ModeSelector.h
+++ b/media/libheadtracking/ModeSelector.h
@@ -16,6 +16,7 @@
#pragma once
#include <optional>
+#include <audio_utils/SimpleLog.h>
#include "media/HeadTrackingMode.h"
#include "media/Pose.h"
@@ -114,6 +115,8 @@
*/
HeadTrackingMode getActualMode() const;
+ std::string toString(unsigned level) const;
+
private:
const Options mOptions;
@@ -129,6 +132,9 @@
HeadTrackingMode mActualMode;
Pose3f mHeadToStage;
+ static constexpr std::size_t sMaxLocalLogLine = 10;
+ SimpleLog mLocalLog{sMaxLocalLogLine};
+
void calculateActualMode(int64_t timestamp);
};
diff --git a/media/libheadtracking/Pose-test.cpp b/media/libheadtracking/Pose-test.cpp
index a9e18ce..29dba29 100644
--- a/media/libheadtracking/Pose-test.cpp
+++ b/media/libheadtracking/Pose-test.cpp
@@ -18,7 +18,7 @@
#include <gtest/gtest.h>
-#include "QuaternionUtil.h"
+#include "media/QuaternionUtil.h"
#include "TestUtil.h"
using android::media::Pose3f;
diff --git a/media/libheadtracking/Pose.cpp b/media/libheadtracking/Pose.cpp
index ae39512..e03725b 100644
--- a/media/libheadtracking/Pose.cpp
+++ b/media/libheadtracking/Pose.cpp
@@ -13,14 +13,16 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
+#include <android-base/stringprintf.h>
#include "media/Pose.h"
+#include "media/QuaternionUtil.h"
#include "media/Twist.h"
-#include "QuaternionUtil.h"
namespace android {
namespace media {
+using android::base::StringAppendF;
using Eigen::Vector3f;
std::optional<Pose3f> Pose3f::fromVector(const std::vector<float>& vec) {
@@ -35,6 +37,19 @@
return {mTranslation[0], mTranslation[1], mTranslation[2], rot[0], rot[1], rot[2]};
}
+std::string Pose3f::toString() const {
+ const auto& vec = this->toVector();
+ std::string ss = "[";
+ for (auto f = vec.begin(); f != vec.end(); ++f) {
+ if (f != vec.begin()) {
+ ss.append(", ");
+ }
+ StringAppendF(&ss, "%0.2f", *f);
+ }
+ ss.append("]");
+ return ss;
+}
+
std::tuple<Pose3f, bool> moveWithRateLimit(const Pose3f& from, const Pose3f& to, float t,
float maxTranslationalVelocity,
float maxRotationalVelocity) {
diff --git a/media/libheadtracking/PoseBias-test.cpp b/media/libheadtracking/PoseBias-test.cpp
index 9f42a2c..659dda0 100644
--- a/media/libheadtracking/PoseBias-test.cpp
+++ b/media/libheadtracking/PoseBias-test.cpp
@@ -17,7 +17,8 @@
#include <gtest/gtest.h>
#include "PoseBias.h"
-#include "QuaternionUtil.h"
+
+#include "media/QuaternionUtil.h"
#include "TestUtil.h"
namespace android {
diff --git a/media/libheadtracking/PoseDriftCompensator-test.cpp b/media/libheadtracking/PoseDriftCompensator-test.cpp
index df0a05f..521e3eb 100644
--- a/media/libheadtracking/PoseDriftCompensator-test.cpp
+++ b/media/libheadtracking/PoseDriftCompensator-test.cpp
@@ -18,7 +18,8 @@
#include <cmath>
#include "PoseDriftCompensator.h"
-#include "QuaternionUtil.h"
+
+#include "media/QuaternionUtil.h"
#include "TestUtil.h"
namespace android {
diff --git a/media/libheadtracking/PoseDriftCompensator.cpp b/media/libheadtracking/PoseDriftCompensator.cpp
index 0e90cad..2775790 100644
--- a/media/libheadtracking/PoseDriftCompensator.cpp
+++ b/media/libheadtracking/PoseDriftCompensator.cpp
@@ -18,7 +18,7 @@
#include <cmath>
-#include "QuaternionUtil.h"
+#include "media/QuaternionUtil.h"
namespace android {
namespace media {
diff --git a/media/libheadtracking/PosePredictor.cpp b/media/libheadtracking/PosePredictor.cpp
new file mode 100644
index 0000000..5209d54
--- /dev/null
+++ b/media/libheadtracking/PosePredictor.cpp
@@ -0,0 +1,246 @@
+/*
+ * Copyright (C) 2023 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "PosePredictor.h"
+
+namespace android::media {
+
+namespace {
+#ifdef ENABLE_VERIFICATION
+constexpr bool kEnableVerification = true;
+constexpr std::array<int, 3> kLookAheadMs{ 50, 100, 200 };
+#else
+constexpr bool kEnableVerification = false;
+constexpr std::array<int, 0> kLookAheadMs{};
+#endif
+
+} // namespace
+
+void LeastSquaresPredictor::add(int64_t atNs, const Pose3f& pose, const Twist3f& twist)
+{
+ (void)twist;
+ mLastAtNs = atNs;
+ mLastPose = pose;
+ const auto q = pose.rotation();
+ const double datNs = static_cast<double>(atNs);
+ mRw.add({datNs, q.w()});
+ mRx.add({datNs, q.x()});
+ mRy.add({datNs, q.y()});
+ mRz.add({datNs, q.z()});
+}
+
+Pose3f LeastSquaresPredictor::predict(int64_t atNs) const
+{
+ if (mRw.getN() < kMinimumSamplesForPrediction) return mLastPose;
+
+ /*
+ * Using parametric form, we have q(t) = { w(t), x(t), y(t), z(t) }.
+ * We compute the least squares prediction of w, x, y, z.
+ */
+ const double dLookahead = static_cast<double>(atNs);
+ Eigen::Quaternionf lsq(
+ mRw.getYFromX(dLookahead),
+ mRx.getYFromX(dLookahead),
+ mRy.getYFromX(dLookahead),
+ mRz.getYFromX(dLookahead));
+
+ /*
+ * We cheat here, since the result lsq is the least squares prediction
+ * in H (arbitrary quaternion), not the least squares prediction in
+ * SO(3) (unit quaternion).
+ *
+ * In other words, the result for lsq is most likely not a unit quaternion.
+ * To solve this, we normalize, thereby selecting the closest unit quaternion
+ * in SO(3) to the prediction in H.
+ */
+ lsq.normalize();
+ return Pose3f(lsq);
+}
+
+void LeastSquaresPredictor::reset() {
+ mLastAtNs = {};
+ mLastPose = {};
+ mRw.reset();
+ mRx.reset();
+ mRy.reset();
+ mRz.reset();
+}
+
+std::string LeastSquaresPredictor::toString(size_t index) const {
+ std::string s(index, ' ');
+ s.append("LeastSquaresPredictor using alpha: ")
+ .append(std::to_string(mAlpha))
+ .append(" last pose: ")
+ .append(mLastPose.toString())
+ .append("\n");
+ return s;
+}
+
+// Formatting
+static inline std::vector<size_t> createDelimiterIdx(size_t predictors, size_t lookaheads) {
+ if (lookaheads == 0) return {};
+ --lookaheads;
+ std::vector<size_t> delimiterIdx(lookaheads);
+ for (size_t i = 0; i < lookaheads; ++i) {
+ delimiterIdx[i] = (i + 1) * predictors;
+ }
+ return delimiterIdx;
+}
+
+PosePredictor::PosePredictor()
+ : mPredictors{
+ // First predictors must match switch in getCurrentPredictor()
+ std::make_shared<LastPredictor>(),
+ std::make_shared<TwistPredictor>(),
+ std::make_shared<LeastSquaresPredictor>(),
+ // After this, can place additional predictors here for comparison such as
+ // std::make_shared<LeastSquaresPredictor>(0.25),
+ }
+ , mLookaheadMs(kLookAheadMs.begin(), kLookAheadMs.end())
+ , mVerifiers(std::size(mLookaheadMs) * std::size(mPredictors))
+ , mDelimiterIdx(createDelimiterIdx(std::size(mPredictors), std::size(mLookaheadMs)))
+ , mPredictionRecorder(
+ std::size(mVerifiers) /* vectorSize */, std::chrono::seconds(1), 10 /* maxLogLine */,
+ mDelimiterIdx)
+ , mPredictionDurableRecorder(
+ std::size(mVerifiers) /* vectorSize */, std::chrono::minutes(1), 10 /* maxLogLine */,
+ mDelimiterIdx)
+ {
+}
+
+Pose3f PosePredictor::predict(
+ int64_t timestampNs, const Pose3f& pose, const Twist3f& twist, float predictionDurationNs)
+{
+ if (timestampNs - mLastTimestampNs > kMaximumSampleIntervalBeforeResetNs) {
+ for (const auto& predictor : mPredictors) {
+ predictor->reset();
+ }
+ ++mResets;
+ }
+ mLastTimestampNs = timestampNs;
+
+ auto selectedPredictor = getCurrentPredictor();
+ if constexpr (kEnableVerification) {
+ // Update all Predictors
+ for (const auto& predictor : mPredictors) {
+ predictor->add(timestampNs, pose, twist);
+ }
+
+ // Update Verifiers and calculate errors
+ std::vector<float> error(std::size(mVerifiers));
+ for (size_t i = 0; i < mLookaheadMs.size(); ++i) {
+ constexpr float RADIAN_TO_DEGREES = 180 / M_PI;
+ const int64_t atNs =
+ timestampNs + mLookaheadMs[i] * PosePredictorVerifier::kMillisToNanos;
+
+ for (size_t j = 0; j < mPredictors.size(); ++j) {
+ const size_t idx = i * std::size(mPredictors) + j;
+ mVerifiers[idx].verifyActualPose(timestampNs, pose);
+ mVerifiers[idx].addPredictedPose(atNs, mPredictors[j]->predict(atNs));
+ error[idx] = RADIAN_TO_DEGREES * mVerifiers[idx].lastError();
+ }
+ }
+ // Record errors
+ mPredictionRecorder.record(error);
+ mPredictionDurableRecorder.record(error);
+ } else /* constexpr */ {
+ selectedPredictor->add(timestampNs, pose, twist);
+ }
+
+ // Deliver prediction
+ const int64_t predictionTimeNs = timestampNs + (int64_t)predictionDurationNs;
+ return selectedPredictor->predict(predictionTimeNs);
+}
+
+void PosePredictor::setPosePredictorType(PosePredictorType type) {
+ if (!isValidPosePredictorType(type)) return;
+ if (type == mSetType) return;
+ mSetType = type;
+ if (type == android::media::PosePredictorType::AUTO) {
+ type = android::media::PosePredictorType::LEAST_SQUARES;
+ }
+ if (type != mCurrentType) {
+ mCurrentType = type;
+ if constexpr (!kEnableVerification) {
+ // Verification keeps all predictors up-to-date.
+ // If we don't enable verification, we must reset the current predictor.
+ getCurrentPredictor()->reset();
+ }
+ }
+}
+
+std::string PosePredictor::toString(size_t index) const {
+ std::string prefixSpace(index, ' ');
+ std::string ss(prefixSpace);
+ ss.append("PosePredictor:\n")
+ .append(prefixSpace)
+ .append(" Current Prediction Type: ")
+ .append(android::media::toString(mCurrentType))
+ .append("\n")
+ .append(prefixSpace)
+ .append(" Resets: ")
+ .append(std::to_string(mResets))
+ .append("\n")
+ .append(getCurrentPredictor()->toString(index + 1));
+ if constexpr (kEnableVerification) {
+ // dump verification
+ ss.append(prefixSpace)
+ .append(" Prediction abs error (L1) degrees [ type (");
+ for (size_t i = 0; i < mPredictors.size(); ++i) {
+ if (i > 0) ss.append(" , ");
+ ss.append(mPredictors[i]->name());
+ }
+ ss.append(" ) x ( ");
+ for (size_t i = 0; i < mLookaheadMs.size(); ++i) {
+ if (i > 0) ss.append(" : ");
+ ss.append(std::to_string(mLookaheadMs[i]));
+ }
+ std::vector<float> cumulativeAverageErrors(std::size(mVerifiers));
+ for (size_t i = 0; i < cumulativeAverageErrors.size(); ++i) {
+ cumulativeAverageErrors[i] = mVerifiers[i].cumulativeAverageError();
+ }
+ ss.append(" ) ms ]\n")
+ .append(prefixSpace)
+ .append(" Cumulative Average Error:\n")
+ .append(prefixSpace)
+ .append(" ")
+ .append(VectorRecorder::toString(cumulativeAverageErrors, mDelimiterIdx, "%.3g"))
+ .append("\n")
+ .append(prefixSpace)
+ .append(" PerMinuteHistory:\n")
+ .append(mPredictionDurableRecorder.toString(index + 3))
+ .append(prefixSpace)
+ .append(" PerSecondHistory:\n")
+ .append(mPredictionRecorder.toString(index + 3));
+ }
+ return ss;
+}
+
+std::shared_ptr<PredictorBase> PosePredictor::getCurrentPredictor() const {
+ // we don't use a map here, we look up directly
+ switch (mCurrentType) {
+ default:
+ case android::media::PosePredictorType::LAST:
+ return mPredictors[0];
+ case android::media::PosePredictorType::TWIST:
+ return mPredictors[1];
+ case android::media::PosePredictorType::AUTO: // shouldn't occur here.
+ case android::media::PosePredictorType::LEAST_SQUARES:
+ return mPredictors[2];
+ }
+}
+
+} // namespace android::media
diff --git a/media/libheadtracking/PosePredictor.h b/media/libheadtracking/PosePredictor.h
new file mode 100644
index 0000000..53211e3
--- /dev/null
+++ b/media/libheadtracking/PosePredictor.h
@@ -0,0 +1,215 @@
+/*
+ * Copyright (C) 2023 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#pragma once
+
+#include "PosePredictorVerifier.h"
+#include <memory>
+#include <audio_utils/Statistics.h>
+#include <media/PosePredictorType.h>
+#include <media/Twist.h>
+#include <media/VectorRecorder.h>
+
+namespace android::media {
+
+// Interface for generic pose predictors
+class PredictorBase {
+public:
+ virtual ~PredictorBase() = default;
+ virtual void add(int64_t atNs, const Pose3f& pose, const Twist3f& twist) = 0;
+ virtual Pose3f predict(int64_t atNs) const = 0;
+ virtual void reset() = 0;
+ virtual std::string name() const = 0;
+ virtual std::string toString(size_t index) const = 0;
+};
+
+/**
+ * LastPredictor uses the last sample Pose for prediction
+ *
+ * This class is not thread-safe.
+ */
+class LastPredictor : public PredictorBase {
+public:
+ void add(int64_t atNs, const Pose3f& pose, const Twist3f& twist) override {
+ (void)atNs;
+ (void)twist;
+ mLastPose = pose;
+ }
+
+ Pose3f predict(int64_t atNs) const override {
+ (void)atNs;
+ return mLastPose;
+ }
+
+ void reset() override {
+ mLastPose = {};
+ }
+
+ std::string name() const override {
+ return "LAST";
+ }
+
+ std::string toString(size_t index) const override {
+ std::string s(index, ' ');
+ s.append("LastPredictor using last pose: ")
+ .append(mLastPose.toString())
+ .append("\n");
+ return s;
+ }
+
+private:
+ Pose3f mLastPose;
+};
+
+/**
+ * TwistPredictor uses the last sample Twist and Pose for prediction
+ *
+ * This class is not thread-safe.
+ */
+class TwistPredictor : public PredictorBase {
+public:
+ void add(int64_t atNs, const Pose3f& pose, const Twist3f& twist) override {
+ mLastAtNs = atNs;
+ mLastPose = pose;
+ mLastTwist = twist;
+ }
+
+ Pose3f predict(int64_t atNs) const override {
+ return mLastPose * integrate(mLastTwist, atNs - mLastAtNs);
+ }
+
+ void reset() override {
+ mLastAtNs = {};
+ mLastPose = {};
+ mLastTwist = {};
+ }
+
+ std::string name() const override {
+ return "TWIST";
+ }
+
+ std::string toString(size_t index) const override {
+ std::string s(index, ' ');
+ s.append("TwistPredictor using last pose: ")
+ .append(mLastPose.toString())
+ .append(" last twist: ")
+ .append(mLastTwist.toString())
+ .append("\n");
+ return s;
+ }
+
+private:
+ int64_t mLastAtNs{};
+ Pose3f mLastPose;
+ Twist3f mLastTwist;
+};
+
+
+/**
+ * LeastSquaresPredictor uses the Pose history for prediction.
+ *
+ * A exponential weighted least squares is used.
+ *
+ * This class is not thread-safe.
+ */
+class LeastSquaresPredictor : public PredictorBase {
+public:
+ // alpha is the exponential decay.
+ LeastSquaresPredictor(double alpha = kDefaultAlphaEstimator)
+ : mAlpha(alpha)
+ , mRw(alpha)
+ , mRx(alpha)
+ , mRy(alpha)
+ , mRz(alpha)
+ {}
+
+ void add(int64_t atNs, const Pose3f& pose, const Twist3f& twist) override;
+ Pose3f predict(int64_t atNs) const override;
+ void reset() override;
+ std::string name() const override {
+ return "LEAST_SQUARES(" + std::to_string(mAlpha) + ")";
+ }
+ std::string toString(size_t index) const override;
+
+private:
+ const double mAlpha;
+ int64_t mLastAtNs{};
+ Pose3f mLastPose;
+ static constexpr double kDefaultAlphaEstimator = 0.2;
+ static constexpr size_t kMinimumSamplesForPrediction = 4;
+ audio_utils::LinearLeastSquaresFit<double> mRw;
+ audio_utils::LinearLeastSquaresFit<double> mRx;
+ audio_utils::LinearLeastSquaresFit<double> mRy;
+ audio_utils::LinearLeastSquaresFit<double> mRz;
+};
+
+/*
+ * PosePredictor predicts the pose given sensor input at a time in the future.
+ *
+ * This class is not thread safe.
+ */
+class PosePredictor {
+public:
+ PosePredictor();
+
+ Pose3f predict(int64_t timestampNs, const Pose3f& pose, const Twist3f& twist,
+ float predictionDurationNs);
+
+ void setPosePredictorType(PosePredictorType type);
+
+ // convert predictions to a printable string
+ std::string toString(size_t index) const;
+
+private:
+ static constexpr int64_t kMaximumSampleIntervalBeforeResetNs =
+ 300'000'000;
+
+ // Predictors
+ const std::vector<std::shared_ptr<PredictorBase>> mPredictors;
+
+ // Verifiers, create one for an array of future lookaheads for comparison.
+ const std::vector<int> mLookaheadMs;
+
+ std::vector<PosePredictorVerifier> mVerifiers;
+
+ const std::vector<size_t> mDelimiterIdx;
+
+ // Recorders
+ media::VectorRecorder mPredictionRecorder{
+ std::size(mVerifiers) /* vectorSize */, std::chrono::seconds(1), 10 /* maxLogLine */,
+ mDelimiterIdx};
+ media::VectorRecorder mPredictionDurableRecorder{
+ std::size(mVerifiers) /* vectorSize */, std::chrono::minutes(1), 10 /* maxLogLine */,
+ mDelimiterIdx};
+
+ // Status
+
+ // SetType is the externally set predictor type. It may include AUTO.
+ PosePredictorType mSetType = PosePredictorType::LEAST_SQUARES;
+
+ // CurrentType is the actual predictor type used by this class.
+ // It does not include AUTO because that metatype means the class
+ // chooses the best predictor type based on sensor statistics.
+ PosePredictorType mCurrentType = PosePredictorType::LEAST_SQUARES;
+
+ int64_t mResets{};
+ int64_t mLastTimestampNs{};
+
+ // Returns current predictor
+ std::shared_ptr<PredictorBase> getCurrentPredictor() const;
+};
+
+} // namespace android::media
diff --git a/media/libheadtracking/PosePredictorVerifier.h b/media/libheadtracking/PosePredictorVerifier.h
new file mode 100644
index 0000000..6b4a357
--- /dev/null
+++ b/media/libheadtracking/PosePredictorVerifier.h
@@ -0,0 +1,75 @@
+/*
+ * Copyright (C) 2023 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#pragma once
+
+#include <string>
+
+#include <audio_utils/Statistics.h>
+#include <media/Pose.h>
+
+namespace android::media {
+
+/**
+ * PosePredictorVerifier is used to validate predictions
+ *
+ * This class is not thread-safe
+ */
+class PosePredictorVerifier {
+public:
+ std::string toString() const {
+ return mErrorStats.toString();
+ }
+
+ static constexpr int64_t kMillisToNanos = 1000000;
+
+ void verifyActualPose(int64_t timestampNs, const Pose3f& pose) {
+ for (auto it = mPredictions.begin(); it != mPredictions.end();) {
+ if (it->first < timestampNs) {
+ it = mPredictions.erase(it);
+ } else {
+ int64_t dt = it->first - timestampNs;
+ if (std::abs(dt) < 10 * kMillisToNanos) {
+ const float angle = pose.rotation().angularDistance(it->second.rotation());
+ const float error = std::abs(angle); // L1 (absolute difference) here.
+ mLastError = error;
+ mErrorStats.add(error);
+ }
+ break;
+ }
+ }
+ }
+
+ void addPredictedPose(int64_t atNs, const Pose3f& pose) {
+ mPredictions.emplace_back(atNs, pose);
+ }
+
+ float lastError() const {
+ return mLastError;
+ }
+
+ float cumulativeAverageError() const {
+ return mErrorStats.getMean();
+ }
+
+private:
+ static constexpr double kCumulativeErrorAlpha = 0.999;
+ std::deque<std::pair<int64_t, Pose3f>> mPredictions;
+ float mLastError{};
+ android::audio_utils::Statistics<double> mErrorStats{kCumulativeErrorAlpha};
+};
+
+} // namespace androd::media
diff --git a/media/libheadtracking/PoseRateLimiter-test.cpp b/media/libheadtracking/PoseRateLimiter-test.cpp
index f306183..ded874a 100644
--- a/media/libheadtracking/PoseRateLimiter-test.cpp
+++ b/media/libheadtracking/PoseRateLimiter-test.cpp
@@ -17,7 +17,8 @@
#include <gtest/gtest.h>
#include "PoseRateLimiter.h"
-#include "QuaternionUtil.h"
+
+#include "media/QuaternionUtil.h"
#include "TestUtil.h"
namespace android {
diff --git a/media/libheadtracking/PoseRateLimiter.cpp b/media/libheadtracking/PoseRateLimiter.cpp
index 380e22b..060bb4b 100644
--- a/media/libheadtracking/PoseRateLimiter.cpp
+++ b/media/libheadtracking/PoseRateLimiter.cpp
@@ -13,11 +13,13 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
+#include <android-base/stringprintf.h>
#include "PoseRateLimiter.h"
namespace android {
namespace media {
+using android::base::StringAppendF;
PoseRateLimiter::PoseRateLimiter(const Options& options) : mOptions(options), mLimiting(false) {}
@@ -48,5 +50,15 @@
return pose;
}
+std::string PoseRateLimiter::toString(unsigned level) const {
+ std::string ss(level, ' ');
+ if (mLimiting) {
+ StringAppendF(&ss, "PoseRateLimiter: enabled with target: %s\n",
+ mTargetPose.has_value() ? mTargetPose.value().toString().c_str() : "NULL");
+ } else {
+ StringAppendF(&ss, "PoseRateLimiter: disabled\n");
+ }
+ return ss;
+}
} // namespace media
} // namespace android
diff --git a/media/libheadtracking/PoseRateLimiter.h b/media/libheadtracking/PoseRateLimiter.h
index aa2fe80..c673a33 100644
--- a/media/libheadtracking/PoseRateLimiter.h
+++ b/media/libheadtracking/PoseRateLimiter.h
@@ -77,6 +77,8 @@
Pose3f calculatePose(int64_t timestamp);
+ std::string toString(unsigned level) const;
+
private:
struct Point {
Pose3f pose;
diff --git a/media/libheadtracking/QuaternionUtil-test.cpp b/media/libheadtracking/QuaternionUtil-test.cpp
index e79e54a..cfeca00 100644
--- a/media/libheadtracking/QuaternionUtil-test.cpp
+++ b/media/libheadtracking/QuaternionUtil-test.cpp
@@ -16,7 +16,7 @@
#include <gtest/gtest.h>
-#include "QuaternionUtil.h"
+#include "media/QuaternionUtil.h"
#include "TestUtil.h"
using Eigen::Quaternionf;
@@ -51,6 +51,92 @@
EXPECT_EQ(vec, quaternionToRotationVector(rotationVectorToQuaternion(vec)));
}
+// Float precision necessitates this precision (1e-4f fails)
+constexpr float NEAR = 1e-3f;
+
+TEST(QuaternionUtil, quaternionToAngles_basic) {
+ float pitch, roll, yaw;
+
+ // angles as reported.
+ // choose 11 angles between -M_PI / 2 to M_PI / 2
+ for (int step = -5; step <= 5; ++step) {
+ const float angle = M_PI * step * 0.1f;
+
+ quaternionToAngles(rotationVectorToQuaternion({angle, 0.f, 0.f}), &pitch, &roll, &yaw);
+ EXPECT_NEAR(angle, pitch, NEAR);
+ EXPECT_NEAR(0.f, roll, NEAR);
+ EXPECT_NEAR(0.f, yaw, NEAR);
+
+ quaternionToAngles(rotationVectorToQuaternion({0.f, angle, 0.f}), &pitch, &roll, &yaw);
+ EXPECT_NEAR(0.f, pitch, NEAR);
+ EXPECT_NEAR(angle, roll, NEAR);
+ EXPECT_NEAR(0.f, yaw, NEAR);
+
+ quaternionToAngles(rotationVectorToQuaternion({0.f, 0.f, angle}), &pitch, &roll, &yaw);
+ EXPECT_NEAR(0.f, pitch, NEAR);
+ EXPECT_NEAR(0.f, roll, NEAR);
+ EXPECT_NEAR(angle, yaw, NEAR);
+ }
+
+ // Generates a debug string
+ const std::string s = quaternionToAngles<true /* DEBUG */>(
+ rotationVectorToQuaternion({M_PI, 0.f, 0.f}), &pitch, &roll, &yaw);
+ ASSERT_FALSE(s.empty());
+}
+
+TEST(QuaternionUtil, quaternionToAngles_zaxis) {
+ float pitch, roll, yaw;
+
+ for (int rot_step = -10; rot_step <= 10; ++rot_step) {
+ const float rot_angle = M_PI * rot_step * 0.1f;
+ // pitch independent of world Z rotation
+
+ // We don't test the boundaries of pitch +-M_PI/2 as roll can become
+ // degenerate and atan(0, 0) may report 0, PI, or -PI.
+ for (int step = -4; step <= 4; ++step) {
+ const float angle = M_PI * step * 0.1f;
+ auto q = rotationVectorToQuaternion({angle, 0.f, 0.f});
+ auto world_z = rotationVectorToQuaternion({0.f, 0.f, rot_angle});
+
+ // Sequential active rotations (on world frame) compose as R_2 * R_1.
+ quaternionToAngles(world_z * q, &pitch, &roll, &yaw);
+
+ EXPECT_NEAR(angle, pitch, NEAR);
+ EXPECT_NEAR(0.f, roll, NEAR);
+ }
+
+ // roll independent of world Z rotation
+ for (int step = -5; step <= 5; ++step) {
+ const float angle = M_PI * step * 0.1f;
+ auto q = rotationVectorToQuaternion({0.f, angle, 0.f});
+ auto world_z = rotationVectorToQuaternion({0.f, 0.f, rot_angle});
+
+ // Sequential active rotations (on world frame) compose as R_2 * R_1.
+ quaternionToAngles(world_z * q, &pitch, &roll, &yaw);
+
+ EXPECT_NEAR(0.f, pitch, NEAR);
+ EXPECT_NEAR(angle, roll, NEAR);
+
+ // Convert extrinsic (world-based) active rotations to a sequence of
+ // intrinsic rotations (each rotation based off of previous rotation
+ // frame).
+ //
+ // R_1 * R_intrinsic = R_extrinsic * R_1
+ // implies
+ // R_intrinsic = (R_1)^-1 R_extrinsic R_1
+ //
+ auto world_z_intrinsic = rotationVectorToQuaternion(
+ q.inverse() * Vector3f(0.f, 0.f, rot_angle));
+
+ // Sequential intrinsic rotations compose as R_1 * R_2.
+ quaternionToAngles(q * world_z_intrinsic, &pitch, &roll, &yaw);
+
+ EXPECT_NEAR(0.f, pitch, NEAR);
+ EXPECT_NEAR(angle, roll, NEAR);
+ }
+ }
+}
+
} // namespace
} // namespace media
} // namespace android
diff --git a/media/libheadtracking/QuaternionUtil.cpp b/media/libheadtracking/QuaternionUtil.cpp
index 5d090de..e245c80 100644
--- a/media/libheadtracking/QuaternionUtil.cpp
+++ b/media/libheadtracking/QuaternionUtil.cpp
@@ -14,7 +14,7 @@
* limitations under the License.
*/
-#include "QuaternionUtil.h"
+#include "media/QuaternionUtil.h"
#include <cassert>
diff --git a/media/libheadtracking/QuaternionUtil.h b/media/libheadtracking/QuaternionUtil.h
deleted file mode 100644
index f7a2ca9..0000000
--- a/media/libheadtracking/QuaternionUtil.h
+++ /dev/null
@@ -1,56 +0,0 @@
-/*
- * Copyright (C) 2021 The Android Open Source Project
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-#pragma once
-
-#include <Eigen/Geometry>
-
-namespace android {
-namespace media {
-
-/**
- * Converts a rotation vector to an equivalent quaternion.
- * The rotation vector is given as a 3-vector whose direction represents the rotation axis and its
- * magnitude the rotation angle (in radians) around that axis.
- */
-Eigen::Quaternionf rotationVectorToQuaternion(const Eigen::Vector3f& rotationVector);
-
-/**
- * Converts a quaternion to an equivalent rotation vector.
- * The rotation vector is given as a 3-vector whose direction represents the rotation axis and its
- * magnitude the rotation angle (in radians) around that axis.
- */
-Eigen::Vector3f quaternionToRotationVector(const Eigen::Quaternionf& quaternion);
-
-/**
- * Returns a quaternion representing a rotation around the X-axis with the given amount (in
- * radians).
- */
-Eigen::Quaternionf rotateX(float angle);
-
-/**
- * Returns a quaternion representing a rotation around the Y-axis with the given amount (in
- * radians).
- */
-Eigen::Quaternionf rotateY(float angle);
-
-/**
- * Returns a quaternion representing a rotation around the Z-axis with the given amount (in
- * radians).
- */
-Eigen::Quaternionf rotateZ(float angle);
-
-} // namespace media
-} // namespace android
diff --git a/media/libheadtracking/SensorPoseProvider.cpp b/media/libheadtracking/SensorPoseProvider.cpp
index 8ebaf6e..8a29027 100644
--- a/media/libheadtracking/SensorPoseProvider.cpp
+++ b/media/libheadtracking/SensorPoseProvider.cpp
@@ -18,26 +18,31 @@
#define LOG_TAG "SensorPoseProvider"
-#include <inttypes.h>
-
+#include <algorithm>
#include <future>
+#include <inttypes.h>
+#include <limits>
#include <map>
#include <thread>
+#include <android-base/stringprintf.h>
#include <android-base/thread_annotations.h>
#include <log/log_main.h>
#include <sensor/SensorEventQueue.h>
#include <sensor/SensorManager.h>
#include <utils/Looper.h>
-#include "QuaternionUtil.h"
+#include "media/QuaternionUtil.h"
namespace android {
namespace media {
namespace {
+using android::base::StringAppendF;
+
// Identifier to use for our event queue on the loop.
// The number 19 is arbitrary, only useful if using multiple objects on the same looper.
+// Note: Instead of a fixed number, the SensorEventQueue's fd could be used instead.
constexpr int kIdent = 19;
static inline Looper* ALooper_to_Looper(ALooper* alooper) {
@@ -56,7 +61,8 @@
EventQueueGuard(const sp<SensorEventQueue>& queue, Looper* looper) : mQueue(queue) {
mQueue->looper = Looper_to_ALooper(looper);
mQueue->requestAdditionalInfo = false;
- looper->addFd(mQueue->getFd(), kIdent, ALOOPER_EVENT_INPUT, nullptr, nullptr);
+ looper->addFd(mQueue->getFd(), kIdent, ALOOPER_EVENT_INPUT,
+ nullptr /* callback */, nullptr /* data */);
}
~EventQueueGuard() {
@@ -71,7 +77,7 @@
[[nodiscard]] SensorEventQueue* get() const { return mQueue.get(); }
private:
- sp<SensorEventQueue> mQueue;
+ const sp<SensorEventQueue> mQueue;
};
/**
@@ -91,10 +97,7 @@
}
}
- SensorEnableGuard(const SensorEnableGuard&) = delete;
- SensorEnableGuard& operator=(const SensorEnableGuard&) = delete;
-
- // Enable moving.
+ // Enable move and delete default copy-ctor/copy-assignment.
SensorEnableGuard(SensorEnableGuard&& other) : mQueue(other.mQueue), mSensor(other.mSensor) {
other.mSensor = SensorPoseProvider::INVALID_HANDLE;
}
@@ -118,6 +121,7 @@
~SensorPoseProviderImpl() override {
// Disable all active sensors.
mEnabledSensors.clear();
+ mQuit = true;
mLooper->wake();
mThread.join();
}
@@ -126,33 +130,73 @@
// Figure out the sensor's data format.
DataFormat format = getSensorFormat(sensor);
if (format == DataFormat::kUnknown) {
- ALOGE("Unknown format for sensor %" PRId32, sensor);
+ ALOGE("%s: Unknown format for sensor %" PRId32, __func__, sensor);
return false;
}
{
std::lock_guard lock(mMutex);
- mEnabledSensorsExtra.emplace(sensor, SensorExtra{ .format = format });
+ mEnabledSensorsExtra.emplace(
+ sensor,
+ SensorExtra{.format = format,
+ .samplingPeriod = static_cast<int32_t>(samplingPeriod.count())});
}
// Enable the sensor.
if (mQueue->enableSensor(sensor, samplingPeriod.count(), 0, 0)) {
- ALOGE("Failed to enable sensor");
+ ALOGE("%s: Failed to enable sensor %" PRId32, __func__, sensor);
std::lock_guard lock(mMutex);
mEnabledSensorsExtra.erase(sensor);
return false;
}
- mEnabledSensors.emplace(sensor, SensorEnableGuard(mQueue.get(), sensor));
+ mEnabledSensors.emplace(sensor, SensorEnableGuard(mQueue, sensor));
+ ALOGD("%s: Sensor %" PRId32 " started", __func__, sensor);
return true;
}
void stopSensor(int handle) override {
+ ALOGD("%s: Sensor %" PRId32 " stopped", __func__, handle);
mEnabledSensors.erase(handle);
std::lock_guard lock(mMutex);
mEnabledSensorsExtra.erase(handle);
}
+ std::string toString(unsigned level) override {
+ std::string prefixSpace(level, ' ');
+ std::string ss = prefixSpace + "SensorPoseProvider:\n";
+ bool needUnlock = false;
+
+ prefixSpace += " ";
+ auto now = std::chrono::steady_clock::now();
+ if (!mMutex.try_lock_until(now + media::kSpatializerDumpSysTimeOutInSecond)) {
+ ss.append(prefixSpace).append("try_lock failed, dumpsys below maybe INACCURATE!\n");
+ } else {
+ needUnlock = true;
+ }
+
+ // Enabled sensor information
+ StringAppendF(&ss, "%sSensors total number %zu:\n", prefixSpace.c_str(),
+ mEnabledSensorsExtra.size());
+ for (auto sensor : mEnabledSensorsExtra) {
+ StringAppendF(&ss,
+ "%s[Handle: 0x%08x, Format %s Period (set %d max %0.4f min %0.4f) ms",
+ prefixSpace.c_str(), sensor.first, toString(sensor.second.format).c_str(),
+ sensor.second.samplingPeriod, media::nsToFloatMs(sensor.second.maxPeriod),
+ media::nsToFloatMs(sensor.second.minPeriod));
+ if (sensor.second.discontinuityCount.has_value()) {
+ StringAppendF(&ss, ", DiscontinuityCount: %d",
+ sensor.second.discontinuityCount.value());
+ }
+ ss += "]\n";
+ }
+
+ if (needUnlock) {
+ mMutex.unlock();
+ }
+ return ss;
+ }
+
private:
enum DataFormat {
kUnknown,
@@ -167,18 +211,22 @@
};
struct SensorExtra {
- DataFormat format;
+ DataFormat format = DataFormat::kUnknown;
+ int32_t samplingPeriod = 0;
+ int64_t latestTimestamp = 0;
+ int64_t maxPeriod = 0;
+ int64_t minPeriod = std::numeric_limits<int64_t>::max();
std::optional<int32_t> discontinuityCount;
};
+ bool mQuit = false;
sp<Looper> mLooper;
Listener* const mListener;
SensorManager* const mSensorManager;
- std::thread mThread;
- std::mutex mMutex;
+ std::timed_mutex mMutex;
+ sp<SensorEventQueue> mQueue;
std::map<int32_t, SensorEnableGuard> mEnabledSensors;
std::map<int32_t, SensorExtra> mEnabledSensorsExtra GUARDED_BY(mMutex);
- sp<SensorEventQueue> mQueue;
// We must do some of the initialization operations on the worker thread, because the API relies
// on the thread-local looper. In addition, as a matter of convenience, we store some of the
@@ -187,18 +235,25 @@
// the worker thread and that thread would notify, via the promise below whenever initialization
// is finished, and whether it was successful.
std::promise<bool> mInitPromise;
+ std::thread mThread;
SensorPoseProviderImpl(const char* packageName, Listener* listener)
: mListener(listener),
- mSensorManager(&SensorManager::getInstanceForPackage(String16(packageName))),
- mThread([this] { threadFunc(); }) {}
-
+ mSensorManager(&SensorManager::getInstanceForPackage(String16(packageName))) {
+ mThread = std::thread([this] { threadFunc(); });
+ }
void initFinished(bool success) { mInitPromise.set_value(success); }
bool waitInitFinished() { return mInitPromise.get_future().get(); }
void threadFunc() {
- // Obtain looper.
+ // Name our std::thread to help identification. As is, canCallJava == false.
+ androidSetThreadName("SensorPoseProvider-looper");
+
+ // Run at the highest non-realtime priority.
+ androidSetThreadPriority(gettid(), PRIORITY_URGENT_AUDIO);
+
+ // The looper is started on the created std::thread.
mLooper = Looper::prepare(ALOOPER_PREPARE_ALLOW_NON_CALLBACKS);
// Create event queue.
@@ -214,20 +269,28 @@
initFinished(true);
- while (true) {
- int ret = mLooper->pollOnce(-1 /* no timeout */, nullptr, nullptr, nullptr);
+ while (!mQuit) {
+ const int ret = mLooper->pollOnce(-1 /* no timeout */, nullptr /* outFd */,
+ nullptr /* outEvents */, nullptr /* outData */);
switch (ret) {
case ALOOPER_POLL_WAKE:
- // Normal way to exit.
- return;
+ // Continue to see if mQuit flag is set.
+ // This can be spurious (due to bugreport being taken).
+ continue;
case kIdent:
// Possible events on our queue.
break;
default:
- ALOGE("Unexpected status out of Looper::pollOnce: %d", ret);
+ // Besides WAKE and kIdent, there should be no timeouts, callbacks,
+ // ALOOPER_POLL_ERROR, or other events.
+ // Exit now to avoid high frequency log spam on error,
+ // e.g. if the fd becomes invalid (b/31093485).
+ ALOGE("%s: Unexpected status out of Looper::pollOnce: %d", __func__, ret);
+ mQuit = true;
+ continue;
}
// Process an event.
@@ -239,7 +302,8 @@
ssize_t size = mQueue->filterEvents(&event, actual);
if (size < 0 || size > 1) {
- ALOGE("Unexpected return value from SensorEventQueue::filterEvents: %zd", size);
+ ALOGE("%s: Unexpected return value from SensorEventQueue::filterEvents: %zd",
+ __func__, size);
break;
}
if (size == 0) {
@@ -249,6 +313,7 @@
handleEvent(event);
}
+ ALOGD("%s: Exiting sensor event loop", __func__);
}
void handleEvent(const ASensorEvent& event) {
@@ -261,6 +326,7 @@
return;
}
value = parseEvent(event, iter->second.format, &iter->second.discontinuityCount);
+ updateEventTimestamp(event, iter->second);
}
mListener->onPose(event.timestamp, event.sensor, value.pose, value.twist,
value.isNewReference);
@@ -316,6 +382,15 @@
return std::nullopt;
}
+ void updateEventTimestamp(const ASensorEvent& event, SensorExtra& extra) {
+ if (extra.latestTimestamp != 0) {
+ int64_t gap = event.timestamp - extra.latestTimestamp;
+ extra.maxPeriod = std::max(gap, extra.maxPeriod);
+ extra.minPeriod = std::min(gap, extra.minPeriod);
+ }
+ extra.latestTimestamp = event.timestamp;
+ }
+
static PoseEvent parseEvent(const ASensorEvent& event, DataFormat format,
std::optional<int32_t>* discontinutyCount) {
switch (format) {
@@ -345,6 +420,19 @@
LOG_ALWAYS_FATAL("Unexpected sensor type: %d", static_cast<int>(format));
}
}
+
+ const static std::string toString(DataFormat format) {
+ switch (format) {
+ case DataFormat::kUnknown:
+ return "kUnknown";
+ case DataFormat::kQuaternion:
+ return "kQuaternion";
+ case DataFormat::kRotationVectorsAndDiscontinuityCount:
+ return "kRotationVectorsAndDiscontinuityCount";
+ default:
+ return "NotImplemented";
+ }
+ }
};
} // namespace
diff --git a/media/libheadtracking/StillnessDetector-test.cpp b/media/libheadtracking/StillnessDetector-test.cpp
index b6cd479..56e7b4e 100644
--- a/media/libheadtracking/StillnessDetector-test.cpp
+++ b/media/libheadtracking/StillnessDetector-test.cpp
@@ -16,8 +16,9 @@
#include <gtest/gtest.h>
-#include "QuaternionUtil.h"
#include "StillnessDetector.h"
+
+#include "media/QuaternionUtil.h"
#include "TestUtil.h"
namespace android {
diff --git a/media/libheadtracking/StillnessDetector.cpp b/media/libheadtracking/StillnessDetector.cpp
index be7c893..5232084 100644
--- a/media/libheadtracking/StillnessDetector.cpp
+++ b/media/libheadtracking/StillnessDetector.cpp
@@ -26,6 +26,9 @@
mFifo.clear();
mWindowFull = false;
mSuppressionDeadline.reset();
+ // A "true" state indicates stillness is detected (default = true)
+ mCurrentState = true;
+ mPreviousState = true;
}
void StillnessDetector::setInput(int64_t timestamp, const Pose3f& input) {
@@ -33,7 +36,15 @@
discardOld(timestamp);
}
+bool StillnessDetector::getPreviousState() const {
+ return mPreviousState;
+}
+
bool StillnessDetector::calculate(int64_t timestamp) {
+ // Move the current stillness state to the previous state.
+ // This allows us to detect transitions into and out of stillness.
+ mPreviousState = mCurrentState;
+
discardOld(timestamp);
// Check whether all the poses in the queue are in the proximity of the new one. We want to do
@@ -60,15 +71,17 @@
// If the window has not been full, return the default value.
if (!mWindowFull) {
- return mOptions.defaultValue;
+ mCurrentState = mOptions.defaultValue;
}
-
// Force "in motion" while the suppression deadline is active.
- if (mSuppressionDeadline.has_value()) {
- return false;
+ else if (mSuppressionDeadline.has_value()) {
+ mCurrentState = false;
+ }
+ else {
+ mCurrentState = !moved;
}
- return !moved;
+ return mCurrentState;
}
void StillnessDetector::discardOld(int64_t timestamp) {
diff --git a/media/libheadtracking/StillnessDetector.h b/media/libheadtracking/StillnessDetector.h
index ee4b2d8..abd7cc4 100644
--- a/media/libheadtracking/StillnessDetector.h
+++ b/media/libheadtracking/StillnessDetector.h
@@ -80,7 +80,8 @@
void setInput(int64_t timestamp, const Pose3f& input);
/** Calculate whether the stream is still at the given timestamp. */
bool calculate(int64_t timestamp);
-
+ /** Return the stillness state from the previous call to calculate() */
+ bool getPreviousState() const;
private:
struct TimestampedPose {
int64_t timestamp;
@@ -92,6 +93,8 @@
const float mCosHalfRotationalThreshold;
std::deque<TimestampedPose> mFifo;
bool mWindowFull = false;
+ bool mCurrentState = true;
+ bool mPreviousState = true;
// As soon as motion is detected, this will be set for the time of detection + window duration,
// and during this time we will always consider outselves in motion without checking. This is
// used for hyteresis purposes, since because of the approximate method we use for determining
diff --git a/media/libheadtracking/Twist-test.cpp b/media/libheadtracking/Twist-test.cpp
index 7984e1e..9fbf81f 100644
--- a/media/libheadtracking/Twist-test.cpp
+++ b/media/libheadtracking/Twist-test.cpp
@@ -16,9 +16,7 @@
#include "media/Twist.h"
-#include <gtest/gtest.h>
-
-#include "QuaternionUtil.h"
+#include "media/QuaternionUtil.h"
#include "TestUtil.h"
using Eigen::Quaternionf;
diff --git a/media/libheadtracking/Twist.cpp b/media/libheadtracking/Twist.cpp
index 664c4d5..fdec694 100644
--- a/media/libheadtracking/Twist.cpp
+++ b/media/libheadtracking/Twist.cpp
@@ -15,8 +15,8 @@
*/
#include "media/Twist.h"
-
-#include "QuaternionUtil.h"
+#include <android-base/stringprintf.h>
+#include "media/QuaternionUtil.h"
namespace android {
namespace media {
@@ -39,5 +39,11 @@
return os;
}
+std::string Twist3f::toString() const {
+ return base::StringPrintf("[%0.2f, %0.2f, %0.2f, %0.2f, %0.2f, %0.2f]",
+ mTranslationalVelocity[0], mTranslationalVelocity[1], mTranslationalVelocity[2],
+ mRotationalVelocity[0], mRotationalVelocity[1], mRotationalVelocity[2]);
+}
+
} // namespace media
} // namespace android
diff --git a/media/libheadtracking/VectorRecorder.cpp b/media/libheadtracking/VectorRecorder.cpp
new file mode 100644
index 0000000..5c87d05
--- /dev/null
+++ b/media/libheadtracking/VectorRecorder.cpp
@@ -0,0 +1,88 @@
+/*
+ * Copyright (C) 2023 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "media/VectorRecorder.h"
+
+namespace android::media {
+
+// Convert data to string with level indentation.
+// No need for a lock as the SimpleLog is thread-safe.
+std::string VectorRecorder::toString(size_t indent) const {
+ return mRecordLog.dumpToString(std::string(indent, ' ').c_str(), mMaxLocalLogLine);
+}
+
+// Record into local log when it is time.
+void VectorRecorder::record(const std::vector<float>& record) {
+ if (record.size() != mVectorSize) return;
+
+ // Protect against concurrent calls to record().
+ std::lock_guard lg(mLock);
+
+ // if it is time, record average data and reset.
+ if (shouldRecordLog_l()) {
+ sumToAverage_l();
+ mRecordLog.log(
+ "mean: %s, min: %s, max %s, calculated %zu samples in %0.4f second(s)",
+ toString(mSum, mDelimiterIdx, mFormatString.c_str()).c_str(),
+ toString(mMin, mDelimiterIdx, mFormatString.c_str()).c_str(),
+ toString(mMax, mDelimiterIdx, mFormatString.c_str()).c_str(),
+ mNumberOfSamples,
+ mNumberOfSecondsSinceFirstSample.count());
+ resetRecord_l();
+ }
+
+ // update stream average.
+ if (mNumberOfSamples++ == 0) {
+ mFirstSampleTimestamp = std::chrono::steady_clock::now();
+ for (size_t i = 0; i < mVectorSize; ++i) {
+ const float value = record[i];
+ mSum[i] += value;
+ mMax[i] = value;
+ mMin[i] = value;
+ }
+ } else {
+ for (size_t i = 0; i < mVectorSize; ++i) {
+ const float value = record[i];
+ mSum[i] += value;
+ mMax[i] = std::max(mMax[i], value);
+ mMin[i] = std::min(mMin[i], value);
+ }
+ }
+}
+
+bool VectorRecorder::shouldRecordLog_l() {
+ mNumberOfSecondsSinceFirstSample = std::chrono::duration_cast<std::chrono::seconds>(
+ std::chrono::steady_clock::now() - mFirstSampleTimestamp);
+ return mNumberOfSecondsSinceFirstSample >= mRecordThreshold;
+}
+
+void VectorRecorder::resetRecord_l() {
+ mSum.assign(mVectorSize, 0);
+ mMax.assign(mVectorSize, 0);
+ mMin.assign(mVectorSize, 0);
+ mNumberOfSamples = 0;
+ mNumberOfSecondsSinceFirstSample = std::chrono::seconds(0);
+}
+
+void VectorRecorder::sumToAverage_l() {
+ if (mNumberOfSamples == 0) return;
+ const float reciprocal = 1.f / mNumberOfSamples;
+ for (auto& p : mSum) {
+ p *= reciprocal;
+ }
+}
+
+} // namespace android::media
diff --git a/media/libheadtracking/include/media/HeadTrackingMode.h b/media/libheadtracking/include/media/HeadTrackingMode.h
index 38496e8..92d1165 100644
--- a/media/libheadtracking/include/media/HeadTrackingMode.h
+++ b/media/libheadtracking/include/media/HeadTrackingMode.h
@@ -15,6 +15,8 @@
*/
#pragma once
+#include <string>
+
namespace android {
namespace media {
@@ -30,5 +32,7 @@
SCREEN_RELATIVE,
};
+std::string toString(HeadTrackingMode mode);
+
} // namespace media
} // namespace android
diff --git a/media/libheadtracking/include/media/HeadTrackingProcessor.h b/media/libheadtracking/include/media/HeadTrackingProcessor.h
index 1744be3..d2b78f2 100644
--- a/media/libheadtracking/include/media/HeadTrackingProcessor.h
+++ b/media/libheadtracking/include/media/HeadTrackingProcessor.h
@@ -19,6 +19,7 @@
#include "HeadTrackingMode.h"
#include "Pose.h"
+#include "PosePredictorType.h"
#include "Twist.h"
namespace android {
@@ -95,9 +96,19 @@
/**
* This causes the current poses for both the head and/or screen to be considered "center".
*/
- virtual void recenter(bool recenterHead = true, bool recenterScreen = true) = 0;
-};
+ virtual void recenter(
+ bool recenterHead = true, bool recenterScreen = true, std::string source = "") = 0;
+ /**
+ * Set the predictor type.
+ */
+ virtual void setPosePredictorType(PosePredictorType type) = 0;
+
+ /**
+ * Dump HeadTrackingProcessor parameters under caller lock.
+ */
+ virtual std::string toString_l(unsigned level) const = 0;
+};
/**
* Creates an instance featuring a default implementation of the HeadTrackingProcessor interface.
*/
diff --git a/media/libheadtracking/include/media/Pose.h b/media/libheadtracking/include/media/Pose.h
index e660bb9..50294ed 100644
--- a/media/libheadtracking/include/media/Pose.h
+++ b/media/libheadtracking/include/media/Pose.h
@@ -16,6 +16,7 @@
#pragma once
#include <optional>
+#include <string>
#include <vector>
#include <Eigen/Geometry>
@@ -63,6 +64,9 @@
*/
std::vector<float> toVector() const;
+ // Convert instance to a string representation.
+ std::string toString() const;
+
Pose3f& operator=(const Pose3f& other) {
mTranslation = other.mTranslation;
mRotation = other.mRotation;
@@ -128,5 +132,10 @@
float maxTranslationalVelocity,
float maxRotationalVelocity);
+template <typename T>
+static float nsToFloatMs(T ns) {
+ return ns * 1e-6f;
+}
+
} // namespace media
} // namespace android
diff --git a/media/libheadtracking/include/media/PosePredictorType.h b/media/libheadtracking/include/media/PosePredictorType.h
new file mode 100644
index 0000000..aa76d5d
--- /dev/null
+++ b/media/libheadtracking/include/media/PosePredictorType.h
@@ -0,0 +1,39 @@
+/*
+ * Copyright (C) 2023 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#pragma once
+
+#include <string>
+
+namespace android::media {
+
+enum class PosePredictorType {
+ /** Use best predictor determined from sensor input */
+ AUTO,
+
+ /** Use last pose for future prediction */
+ LAST,
+
+ /** Use twist angular velocity for future prediction */
+ TWIST,
+
+ /** Use weighted least squares history of prior poses (ignoring twist) */
+ LEAST_SQUARES,
+};
+
+std::string toString(PosePredictorType posePredictorType);
+bool isValidPosePredictorType(PosePredictorType posePredictorType);
+
+} // namespace android::media
diff --git a/media/libheadtracking/include/media/QuaternionUtil.h b/media/libheadtracking/include/media/QuaternionUtil.h
new file mode 100644
index 0000000..a711d17
--- /dev/null
+++ b/media/libheadtracking/include/media/QuaternionUtil.h
@@ -0,0 +1,297 @@
+/*
+ * Copyright (C) 2021 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#pragma once
+
+#include <android-base/stringprintf.h>
+#include <Eigen/Geometry>
+#include <media/Pose.h>
+
+namespace android {
+namespace media {
+
+/**
+ * Converts a rotation vector to an equivalent quaternion.
+ * The rotation vector is given as a 3-vector whose direction represents the rotation axis and its
+ * magnitude the rotation angle (in radians) around that axis.
+ */
+Eigen::Quaternionf rotationVectorToQuaternion(const Eigen::Vector3f& rotationVector);
+
+/**
+ * Converts a quaternion to an equivalent rotation vector.
+ * The rotation vector is given as a 3-vector whose direction represents the rotation axis and its
+ * magnitude the rotation angle (in radians) around that axis.
+ */
+Eigen::Vector3f quaternionToRotationVector(const Eigen::Quaternionf& quaternion);
+
+/**
+ * Returns a quaternion representing a rotation around the X-axis with the given amount (in
+ * radians).
+ */
+Eigen::Quaternionf rotateX(float angle);
+
+/**
+ * Returns a quaternion representing a rotation around the Y-axis with the given amount (in
+ * radians).
+ */
+Eigen::Quaternionf rotateY(float angle);
+
+/**
+ * Returns a quaternion representing a rotation around the Z-axis with the given amount (in
+ * radians).
+ */
+Eigen::Quaternionf rotateZ(float angle);
+
+/**
+ * Compute separate roll, pitch, and yaw angles from a quaternion
+ *
+ * The roll, pitch, and yaw follow standard 3DOF virtual reality definitions
+ * with angles increasing counter-clockwise by the right hand rule.
+ *
+ * https://en.wikipedia.org/wiki/Six_degrees_of_freedom
+ *
+ * The roll, pitch, and yaw angles are calculated separately from the device frame
+ * rotation from the world frame. This is not to be confused with the
+ * intrinsic Euler xyz roll, pitch, yaw 'nautical' angles.
+ *
+ * The input quarternion is the active rotation that transforms the
+ * World/Stage frame to the Head/Screen frame.
+ *
+ * The input quaternion may come from two principal sensors: DEVICE and HEADSET
+ * and are interpreted as below.
+ *
+ * DEVICE SENSOR
+ *
+ * Android sensor stack assumes device coordinates along the x/y axis.
+ *
+ * https://developer.android.com/reference/android/hardware/SensorEvent#sensor.type_rotation_vector:
+ *
+ * Looking down from the clouds. Android Device coordinate system (not used)
+ * DEVICE --> X (Y goes through top speaker towards the observer)
+ * | Z
+ * V
+ * USER
+ *
+ * Internally within this library, we transform the device sensor coordinate
+ * system by rotating the coordinate system around the X axis by -M_PI/2.
+ * This aligns the device coordinate system to match that of the
+ * Head Tracking sensor (see below), should the user be facing the device in
+ * natural (phone == portrait, tablet == ?) orientation.
+ *
+ * Looking down from the clouds. Spatializer device frame.
+ * Y
+ * ^
+ * |
+ * DEVICE --> X (Z goes through top of the DEVICE towards the observer)
+ *
+ * USER
+ *
+ * The reference world frame is the device in vertical
+ * natural (phone == portrait) orientation with the top pointing straight
+ * up from the ground and the front-to-back direction facing north.
+ * The world frame is presumed locally fixed by magnetic and gravitational reference.
+ *
+ * HEADSET SENSOR
+ * https://developer.android.com/reference/android/hardware/SensorEvent#sensor.type_head_tracker:
+ *
+ * Looking down from the clouds. Headset frame.
+ * Y
+ * ^
+ * |
+ * USER ---> X
+ * (Z goes through the top of the USER head towards the observer)
+ *
+ * The Z axis goes from the neck to the top of the head, the X axis goes
+ * from the left ear to the right ear, the Y axis goes from the back of the
+ * head through the nose.
+ *
+ * Typically for a headset sensor, the X and Y axes have some arbitrary fixed
+ * reference.
+ *
+ * ROLL
+ * Roll is the counter-clockwise L/R motion around the Y axis (hence ZX plane).
+ * The right hand convention means the plane is ZX not XZ.
+ * This can be considered the azimuth angle in spherical coordinates
+ * with Pitch being the elevation angle.
+ *
+ * Roll has a range of -M_PI to M_PI radians.
+ *
+ * Rolling a device changes between portrait and landscape
+ * modes, and for L/R speakers will limit the amount of crosstalk cancellation.
+ * Roll increases as the device (if vertical like a coin) rolls from left to right.
+ *
+ * By this definition, Roll is less accurate when the device is flat
+ * on a table rather than standing on edge.
+ * When perfectly flat on the table, roll may report as 0, M_PI, or -M_PI
+ * due ambiguity / degeneracy of atan(0, 0) in this case (the device Y axis aligns with
+ * the world Z axis), but exactly flat rarely occurs.
+ *
+ * Roll for a headset is the angle the head is inclined to the right side
+ * (like sleeping).
+ *
+ * PITCH
+ * Pitch is the Surface normal Y deviation (along the Z axis away from the earth).
+ * This can be considered the elevation angle in spherical coordinates using
+ * Roll as the azimuth angle.
+ *
+ * Pitch for a device determines whether the device is "upright" or lying
+ * flat on the table (i.e. surface normal). Pitch is 0 when upright, decreases
+ * as the device top moves away from the user to -M_PI/2 when lying down face up.
+ * Pitch increases from 0 to M_PI/2 when the device tilts towards the user, and is
+ * M_PI/2 degrees when face down.
+ *
+ * Pitch for a headset is the user tilting the head/chin up or down,
+ * like nodding.
+ *
+ * Pitch has a range of -M_PI/2, M_PI/2 radians.
+ *
+ * YAW
+ * Yaw is the rotational component along the earth's XY tangential plane,
+ * where the Z axis points radially away from the earth.
+ *
+ * Yaw has a range of -M_PI to M_PI radians. If used for azimuth angle in
+ * spherical coordinates, the elevation angle may be derived from the Z axis.
+ *
+ * A positive increase means the phone is rotating from right to left
+ * when considered flat on the table.
+ * (headset: the user is rotating their head to look left).
+ * If left speaker or right earbud is pointing straight up or down,
+ * this value is imprecise and Pitch or Roll is a more useful measure.
+ *
+ * Yaw for a device is like spinning a vertical device along the axis of
+ * gravity, like spinning a coin. Yaw increases as the coin / device
+ * spins from right to left, rotating around the Z axis.
+ *
+ * Yaw for a headset is the user turning the head to look left or right
+ * like shaking the head for no. Yaw is the primary angle for a binaural
+ * head tracking device.
+ *
+ * @param q input active rotation Eigen quaternion.
+ * @param pitch output set to pitch if not nullptr
+ * @param roll output set to roll if not nullptr
+ * @param yaw output set to yaw if not nullptr
+ * @return (DEBUG==true) a debug string with intermediate transformation matrix
+ * interpreted as the unit basis vectors.
+ */
+
+// DEBUG returns a debug string for analysis.
+// We save unneeded rotation matrix computation by keeping the DEBUG option constexpr.
+template <bool DEBUG = false>
+auto quaternionToAngles(const Eigen::Quaternionf& q, float *pitch, float *roll, float *yaw) {
+ /*
+ * The quaternion here is the active rotation that transforms from the world frame
+ * to the device frame: the observer remains in the world frame,
+ * and the device (frame) moves.
+ *
+ * We use this to map device coordinates to world coordinates.
+ *
+ * Device: We transform the device right speaker (X == 1), top speaker (Z == 1),
+ * and surface inwards normal (Y == 1) positions to the world frame.
+ *
+ * Headset: We transform the headset right bud (X == 1), top (Z == 1) and
+ * nose normal (Y == 1) positions to the world frame.
+ *
+ * This is the same as the world frame coordinates of the
+ * unit device vector in the X dimension (ux),
+ * unit device vector in the Y dimension (uy),
+ * unit device vector in the Z dimension (uz).
+ *
+ * Rather than doing the rotation on unit vectors individually,
+ * one can simply use the columns of the rotation matrix of
+ * the world-to-body quaternion, so the computation is exceptionally fast.
+ *
+ * Furthermore, Eigen inlines the "toRotationMatrix" method
+ * and we rely on unused expression removal for efficiency
+ * and any elements not used should not be computed.
+ *
+ * Side note: For applying a rotation to several points,
+ * it is more computationally efficient to extract and
+ * use the rotation matrix form than the quaternion.
+ * So use of the rotation matrix is good for many reasons.
+ */
+ const auto rotation = q.toRotationMatrix();
+
+ /*
+ * World location of unit vector right speaker assuming the phone is situated
+ * natural (phone == portrait) mode.
+ * (headset: right bud).
+ *
+ * auto ux = q.rotation() * Eigen::Vector3f{1.f, 0.f, 0.f};
+ * = rotation.col(0);
+ */
+ [[maybe_unused]] const auto ux_0 = rotation.coeff(0, 0);
+ [[maybe_unused]] const auto ux_1 = rotation.coeff(1, 0);
+ [[maybe_unused]] const auto ux_2 = rotation.coeff(2, 0);
+
+ [[maybe_unused]] std::string coordinates;
+ if constexpr (DEBUG) {
+ base::StringAppendF(&coordinates, "ux: %f %f %f", ux_0, ux_1, ux_2);
+ }
+
+ /*
+ * World location of screen-inwards normal assuming the phone is situated
+ * in natural (phone == portrait) mode.
+ * (headset: user nose).
+ *
+ * auto uy = q.rotation() * Eigen::Vector3f{0.f, 1.f, 0.f};
+ * = rotation.col(1);
+ */
+ [[maybe_unused]] const auto uy_0 = rotation.coeff(0, 1);
+ [[maybe_unused]] const auto uy_1 = rotation.coeff(1, 1);
+ [[maybe_unused]] const auto uy_2 = rotation.coeff(2, 1);
+ if constexpr (DEBUG) {
+ base::StringAppendF(&coordinates, "uy: %f %f %f", uy_0, uy_1, uy_2);
+ }
+
+ /*
+ * World location of unit vector top speaker.
+ * (headset: top of head).
+ * auto uz = q.rotation() * Eigen::Vector3f{0.f, 0.f, 1.f};
+ * = rotation.col(2);
+ */
+ [[maybe_unused]] const auto uz_0 = rotation.coeff(0, 2);
+ [[maybe_unused]] const auto uz_1 = rotation.coeff(1, 2);
+ [[maybe_unused]] const auto uz_2 = rotation.coeff(2, 2);
+ if constexpr (DEBUG) {
+ base::StringAppendF(&coordinates, "uz: %f %f %f", uz_0, uz_1, uz_2);
+ }
+
+ // pitch computed from nose world Z coordinate;
+ // hence independent of rotation around world Z.
+ if (pitch != nullptr) {
+ *pitch = asin(std::clamp(uy_2, -1.f, 1.f));
+ }
+
+ // roll computed from head/right world Z coordinate;
+ // hence independent of rotation around world Z.
+ if (roll != nullptr) {
+ // atan2 takes care of implicit scale normalization of Z, X.
+ *roll = -atan2(ux_2, uz_2);
+ }
+
+ // yaw computed from right ear angle projected onto world XY plane
+ // where world Z == 0. This is the rotation around world Z.
+ if (yaw != nullptr) {
+ // atan2 takes care of implicit scale normalization of X, Y.
+ *yaw = atan2(ux_1, ux_0);
+ }
+
+ if constexpr (DEBUG) {
+ return coordinates;
+ }
+}
+
+} // namespace media
+} // namespace android
diff --git a/media/libheadtracking/include/media/SensorPoseProvider.h b/media/libheadtracking/include/media/SensorPoseProvider.h
index 0f42074..4609e0c 100644
--- a/media/libheadtracking/include/media/SensorPoseProvider.h
+++ b/media/libheadtracking/include/media/SensorPoseProvider.h
@@ -28,6 +28,9 @@
namespace android {
namespace media {
+// Timeout for Spatializer dumpsys trylock, don't block for more than 3 seconds.
+constexpr auto kSpatializerDumpSysTimeOutInSecond = std::chrono::seconds(3);
+
/**
* A utility providing streaming of pose data from motion sensors provided by the Sensor Framework.
*
@@ -100,6 +103,11 @@
* discover properties of the sensor.
*/
virtual std::optional<const Sensor> getSensorByHandle(int32_t handle) = 0;
+
+ /**
+ * Dump SensorPoseProvider parameters and history data.
+ */
+ virtual std::string toString(unsigned level) = 0;
};
} // namespace media
diff --git a/media/libheadtracking/include/media/Twist.h b/media/libheadtracking/include/media/Twist.h
index 291cea3..51b83d8 100644
--- a/media/libheadtracking/include/media/Twist.h
+++ b/media/libheadtracking/include/media/Twist.h
@@ -66,6 +66,9 @@
return Twist3f(mTranslationalVelocity / s, mRotationalVelocity / s);
}
+ // Convert instance to a string representation.
+ std::string toString() const;
+
private:
Eigen::Vector3f mTranslationalVelocity;
Eigen::Vector3f mRotationalVelocity;
diff --git a/media/libheadtracking/include/media/VectorRecorder.h b/media/libheadtracking/include/media/VectorRecorder.h
new file mode 100644
index 0000000..4103a7d
--- /dev/null
+++ b/media/libheadtracking/include/media/VectorRecorder.h
@@ -0,0 +1,151 @@
+/*
+ * Copyright (C) 2023 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#pragma once
+
+#include <android-base/stringprintf.h>
+#include <android-base/thread_annotations.h>
+#include <audio_utils/SimpleLog.h>
+#include <chrono>
+#include <math.h>
+#include <mutex>
+#include <vector>
+
+namespace android::media {
+
+/**
+ * VectorRecorder records a vector of floats computing the average, max, and min
+ * over given time periods.
+ *
+ * The class is thread-safe.
+ */
+class VectorRecorder {
+ public:
+ /**
+ * @param vectorSize is the size of the vector input.
+ * If the input does not match this size, it is ignored.
+ * @param threshold is the time interval we bucket for averaging.
+ * @param maxLogLine is the number of lines we log. At this
+ * threshold, the oldest line will expire when the new line comes in.
+ * @param delimiterIdx is an optional array of delimiter indices that
+ * replace the ',' with a ':'. For example if delimiterIdx = { 3 } then
+ * the above example would format as [0.00, 0.00, 0.00 : -1.29, -0.50, 15.27].
+ * @param formatString is the sprintf format string for the double converted data
+ * to use.
+ */
+ VectorRecorder(
+ size_t vectorSize, std::chrono::duration<double> threshold, int maxLogLine,
+ std::vector<size_t> delimiterIdx = {},
+ const std::string_view formatString = {})
+ : mVectorSize(vectorSize)
+ , mDelimiterIdx(std::move(delimiterIdx))
+ , mFormatString(formatString)
+ , mRecordLog(maxLogLine)
+ , mRecordThreshold(threshold)
+ {
+ resetRecord_l(); // OK to call - we're in the constructor.
+ }
+
+ /** Convert recorded vector data to string with level indentation */
+ std::string toString(size_t indent) const;
+
+ /**
+ * @brief Record a vector of floats.
+ *
+ * @param record a vector of floats.
+ */
+ void record(const std::vector<float>& record);
+
+ /**
+ * Format vector to a string, [0.00, 0.00, 0.00, -1.29, -0.50, 15.27].
+ *
+ * @param delimiterIdx is an optional array of delimiter indices that
+ * replace the ',' with a ':'. For example if delimiterIdx = { 3 } then
+ * the above example would format as [0.00, 0.00, 0.00 : -1.29, -0.50, 15.27].
+ * @param formatString is the sprintf format string for the double converted data
+ * to use.
+ */
+ template <typename T>
+ static std::string toString(const std::vector<T>& record,
+ const std::vector<size_t>& delimiterIdx = {},
+ const char * const formatString = nullptr) {
+ if (record.size() == 0) {
+ return "[]";
+ }
+
+ std::string ss = "[";
+ auto nextDelimiter = delimiterIdx.begin();
+ for (size_t i = 0; i < record.size(); ++i) {
+ if (i > 0) {
+ if (nextDelimiter != delimiterIdx.end()
+ && *nextDelimiter <= i) {
+ ss.append(" : ");
+ ++nextDelimiter;
+ } else {
+ ss.append(", ");
+ }
+ }
+ if (formatString != nullptr && *formatString) {
+ base::StringAppendF(&ss, formatString, static_cast<double>(record[i]));
+ } else {
+ base::StringAppendF(&ss, "%5.2lf", static_cast<double>(record[i]));
+ }
+ }
+ ss.append("]");
+ return ss;
+ }
+
+ private:
+ static constexpr int mMaxLocalLogLine = 10;
+
+ const size_t mVectorSize;
+ const std::vector<size_t> mDelimiterIdx;
+ const std::string mFormatString;
+
+ // Local log for historical vector data.
+ // Locked internally, so does not need mutex below.
+ SimpleLog mRecordLog{mMaxLocalLogLine};
+
+ std::mutex mLock;
+
+ // Time threshold to record vectors in the local log.
+ // Vector data will be recorded into log at least every mRecordThreshold.
+ std::chrono::duration<double> mRecordThreshold GUARDED_BY(mLock);
+
+ // Number of seconds since first sample in mSum.
+ std::chrono::duration<double> mNumberOfSecondsSinceFirstSample GUARDED_BY(mLock);
+
+ // Timestamp of first sample recorded in mSum.
+ std::chrono::time_point<std::chrono::steady_clock> mFirstSampleTimestamp GUARDED_BY(mLock);
+
+ // Number of samples in mSum.
+ size_t mNumberOfSamples GUARDED_BY(mLock) = 0;
+
+ std::vector<double> mSum GUARDED_BY(mLock);
+ std::vector<float> mMax GUARDED_BY(mLock);
+ std::vector<float> mMin GUARDED_BY(mLock);
+
+ // Computes mNumberOfSecondsSinceFirstSample, returns true if time to record.
+ bool shouldRecordLog_l() REQUIRES(mLock);
+
+ // Resets the running mNumberOfSamples, mSum, mMax, mMin.
+ void resetRecord_l() REQUIRES(mLock);
+
+ // Convert mSum to an average.
+ void sumToAverage_l() REQUIRES(mLock);
+}; // VectorRecorder
+
+} // namespace android::media
diff --git a/media/libmediametrics/include/MediaMetricsConstants.h b/media/libmediametrics/include/MediaMetricsConstants.h
index 1c30510..27f987d 100644
--- a/media/libmediametrics/include/MediaMetricsConstants.h
+++ b/media/libmediametrics/include/MediaMetricsConstants.h
@@ -57,6 +57,10 @@
// The Audio Spatializer key appends the spatializerId (currently 0)
#define AMEDIAMETRICS_KEY_PREFIX_AUDIO_SPATIALIZER AMEDIAMETRICS_KEY_PREFIX_AUDIO "spatializer."
+// The Audio Spatializer device key appends the device type.
+#define AMEDIAMETRICS_KEY_PREFIX_AUDIO_SPATIALIZER_DEVICE \
+ AMEDIAMETRICS_KEY_PREFIX_AUDIO_SPATIALIZER "device."
+
// The AudioStream key appends the "streamId" to the prefix.
#define AMEDIAMETRICS_KEY_PREFIX_AUDIO_STREAM AMEDIAMETRICS_KEY_PREFIX_AUDIO "stream."
@@ -83,6 +87,7 @@
#define AMEDIAMETRICS_PROP_PREFIX_EFFECTIVE "effective."
#define AMEDIAMETRICS_PROP_PREFIX_HAL "hal."
#define AMEDIAMETRICS_PROP_PREFIX_HAPTIC "haptic."
+#define AMEDIAMETRICS_PROP_PREFIX_LAST "last."
#define AMEDIAMETRICS_PROP_PREFIX_SERVER "server."
// Properties within mediametrics are string constants denoted by
@@ -107,6 +112,7 @@
// of suppressed in the Time Machine.
#define AMEDIAMETRICS_PROP_SUFFIX_CHAR_DUPLICATES_ALLOWED '#'
+#define AMEDIAMETRICS_PROP_ADDRESS "address" // string, for example MAC address
#define AMEDIAMETRICS_PROP_ALLOWUID "_allowUid" // int32_t, allow client uid to post
#define AMEDIAMETRICS_PROP_AUDIOMODE "audioMode" // string (audio.flinger)
#define AMEDIAMETRICS_PROP_AUXEFFECTID "auxEffectId" // int32 (AudioTrack)
@@ -116,6 +122,8 @@
#define AMEDIAMETRICS_PROP_CALLERNAME "callerName" // string, eg. "aaudio"
#define AMEDIAMETRICS_PROP_CHANNELCOUNT "channelCount" // int32
#define AMEDIAMETRICS_PROP_CHANNELMASK "channelMask" // int32
+#define AMEDIAMETRICS_PROP_CHANNELMASKS "channelMasks" // string with channelMask values
+ // separated by |.
#define AMEDIAMETRICS_PROP_CONTENTTYPE "contentType" // string attributes (AudioTrack)
#define AMEDIAMETRICS_PROP_CUMULATIVETIMENS "cumulativeTimeNs" // int64_t playback/record time
// since start
@@ -132,6 +140,7 @@
#define AMEDIAMETRICS_PROP_DIRECTION "direction" // string AAudio input or output
#define AMEDIAMETRICS_PROP_DURATIONNS "durationNs" // int64 duration time span
+#define AMEDIAMETRICS_PROP_ENABLED "enabled" // string true/false.
#define AMEDIAMETRICS_PROP_ENCODING "encoding" // string value of format
#define AMEDIAMETRICS_PROP_EVENT "event#" // string value (often func name)
@@ -141,6 +150,8 @@
#define AMEDIAMETRICS_PROP_FLAGS "flags"
#define AMEDIAMETRICS_PROP_FRAMECOUNT "frameCount" // int32
+#define AMEDIAMETRICS_PROP_HASHEADTRACKER "hasHeadTracker" // string true/false
+#define AMEDIAMETRICS_PROP_HEADTRACKERENABLED "headTrackerEnabled" // string true/false
#define AMEDIAMETRICS_PROP_HEADTRACKINGMODES "headTrackingModes" // string |, like modes.
#define AMEDIAMETRICS_PROP_INPUTDEVICES "inputDevices" // string value
#define AMEDIAMETRICS_PROP_INTERNALTRACKID "internalTrackId" // int32
diff --git a/media/libmediaplayerservice/MediaPlayerService.cpp b/media/libmediaplayerservice/MediaPlayerService.cpp
index 8e19d02..e654bce 100644
--- a/media/libmediaplayerservice/MediaPlayerService.cpp
+++ b/media/libmediaplayerservice/MediaPlayerService.cpp
@@ -1836,7 +1836,6 @@
} else {
mAttributes = NULL;
}
-
setMinBufferCount();
}
@@ -2683,7 +2682,7 @@
// This is a benign busy-wait, with the next data request generated 10 ms or more later;
// nevertheless for power reasons, we don't want to see too many of these.
- ALOGV_IF(actualSize == 0 && buffer->size > 0, "callbackwrapper: empty buffer returned");
+ ALOGV_IF(actualSize == 0 && buffer.size() > 0, "callbackwrapper: empty buffer returned");
unlock();
return actualSize;
}
diff --git a/media/libmediaplayerservice/StagefrightRecorder.cpp b/media/libmediaplayerservice/StagefrightRecorder.cpp
index ea1fdf4..6497b58 100644
--- a/media/libmediaplayerservice/StagefrightRecorder.cpp
+++ b/media/libmediaplayerservice/StagefrightRecorder.cpp
@@ -129,6 +129,7 @@
mRTPCVOExtMap(-1),
mRTPCVODegrees(0),
mRTPSockDscp(0),
+ mRTPSockOptEcn(0),
mRTPSockNetwork(0),
mLastSeqNo(0),
mStarted(false),
@@ -910,6 +911,13 @@
return OK;
}
+status_t StagefrightRecorder::setParamRtpEcn(int32_t ecn) {
+ ALOGV("setParamRtpEcn: %d", ecn);
+
+ mRTPSockOptEcn = ecn;
+ return OK;
+}
+
status_t StagefrightRecorder::requestIDRFrame() {
status_t ret = BAD_VALUE;
if (mVideoEncoderSource != NULL) {
@@ -1091,6 +1099,11 @@
if (safe_strtoi32(value.string(), &dscp)) {
return setParamRtpDscp(dscp);
}
+ } else if (key == "rtp-param-set-socket-ecn") {
+ int32_t targetEcn;
+ if (safe_strtoi32(value.string(), &targetEcn)) {
+ return setParamRtpEcn(targetEcn);
+ }
} else if (key == "rtp-param-set-socket-network") {
int64_t networkHandle;
if (safe_strtoi64(value.string(), &networkHandle)) {
@@ -1272,6 +1285,9 @@
if (mRTPSockDscp > 0) {
meta->setInt32(kKeyRtpDscp, mRTPSockDscp);
}
+ if (mRTPSockOptEcn > 0) {
+ meta->setInt32(kKeyRtpEcn, mRTPSockOptEcn);
+ }
status = mWriter->start(meta.get());
break;
@@ -1966,6 +1982,10 @@
format->setString("mime", MEDIA_MIMETYPE_VIDEO_HEVC);
break;
+ case VIDEO_ENCODER_DOLBY_VISION:
+ format->setString("mime", MEDIA_MIMETYPE_VIDEO_DOLBY_VISION);
+ break;
+
default:
CHECK(!"Should not be here, unsupported video encoding.");
break;
diff --git a/media/libmediaplayerservice/StagefrightRecorder.h b/media/libmediaplayerservice/StagefrightRecorder.h
index d7785da..0801101 100644
--- a/media/libmediaplayerservice/StagefrightRecorder.h
+++ b/media/libmediaplayerservice/StagefrightRecorder.h
@@ -153,6 +153,7 @@
int32_t mRTPCVOExtMap;
int32_t mRTPCVODegrees;
int32_t mRTPSockDscp;
+ int32_t mRTPSockOptEcn;
int64_t mRTPSockNetwork;
uint32_t mLastSeqNo;
@@ -247,6 +248,7 @@
status_t setRTPCVOExtMap(int32_t extmap);
status_t setRTPCVODegrees(int32_t cvoDegrees);
status_t setParamRtpDscp(int32_t dscp);
+ status_t setParamRtpEcn(int32_t ecn);
status_t setSocketNetwork(int64_t networkHandle);
status_t requestIDRFrame();
void clipVideoBitRate();
diff --git a/media/libmediaplayerservice/nuplayer/AWakeLock.cpp b/media/libmediaplayerservice/nuplayer/AWakeLock.cpp
index c3bd207..25a8ae4 100644
--- a/media/libmediaplayerservice/nuplayer/AWakeLock.cpp
+++ b/media/libmediaplayerservice/nuplayer/AWakeLock.cpp
@@ -59,11 +59,10 @@
if (mPowerManager != NULL) {
sp<IBinder> binder = new BBinder();
int64_t token = IPCThreadState::self()->clearCallingIdentity();
- binder::Status status = mPowerManager->acquireWakeLock(
+ binder::Status status = mPowerManager->acquireWakeLockAsync(
binder, POWERMANAGER_PARTIAL_WAKE_LOCK,
String16("AWakeLock"), String16("media"),
- {} /* workSource */, {} /* historyTag */, -1 /* displayId */,
- nullptr /* callback */);
+ {} /* workSource */, {} /* historyTag */);
IPCThreadState::self()->restoreCallingIdentity(token);
if (status.isOk()) {
mWakeLockToken = binder;
diff --git a/media/libmediaplayerservice/nuplayer/NuPlayerDrm.cpp b/media/libmediaplayerservice/nuplayer/NuPlayerDrm.cpp
index 6788b56..5e29b3f 100644
--- a/media/libmediaplayerservice/nuplayer/NuPlayerDrm.cpp
+++ b/media/libmediaplayerservice/nuplayer/NuPlayerDrm.cpp
@@ -191,8 +191,8 @@
uint8_t key[kBlockSize],
uint8_t iv[kBlockSize],
CryptoPlugin::Mode mode,
- size_t *clearbytes,
- size_t *encryptedbytes)
+ uint32_t *clearbytes,
+ uint32_t *encryptedbytes)
{
// size needed to store all the crypto data
size_t cryptosize;
@@ -236,7 +236,7 @@
if (!meta.findData(kKeyEncryptedSizes, &type, &crypteddata, &cryptedsize)) {
return NULL;
}
- size_t numSubSamples = cryptedsize / sizeof(size_t);
+ size_t numSubSamples = cryptedsize / sizeof(uint32_t);
if (numSubSamples <= 0) {
ALOGE("getSampleCryptoInfo INVALID numSubSamples: %zu", numSubSamples);
@@ -285,8 +285,8 @@
(uint8_t*) key,
(uint8_t*) iv,
(CryptoPlugin::Mode)mode,
- (size_t*) cleardata,
- (size_t*) crypteddata);
+ (uint32_t*) cleardata,
+ (uint32_t*) crypteddata);
}
} // namespace android
diff --git a/media/libmediaplayerservice/nuplayer/NuPlayerRenderer.cpp b/media/libmediaplayerservice/nuplayer/NuPlayerRenderer.cpp
index 45fac76..0382df3 100644
--- a/media/libmediaplayerservice/nuplayer/NuPlayerRenderer.cpp
+++ b/media/libmediaplayerservice/nuplayer/NuPlayerRenderer.cpp
@@ -1947,22 +1947,12 @@
CHECK(format->findInt32("channel-count", &numChannels));
// channel mask info as read from the audio format
- int32_t channelMaskFromFormat;
+ int32_t mediaFormatChannelMask;
// channel mask to use for native playback
audio_channel_mask_t channelMask;
- if (format->findInt32("channel-mask", &channelMaskFromFormat)) {
+ if (format->findInt32("channel-mask", &mediaFormatChannelMask)) {
// KEY_CHANNEL_MASK follows the android.media.AudioFormat java mask
- // which is left-bitshifted by 2 relative to the native mask
- if ((channelMaskFromFormat & 0b11) != 0) {
- // received an unexpected mask (supposed to follow AudioFormat constants
- // for output masks with the 2 least-significant bits at 0), but
- // it may come from an extractor that uses native masks: keeping
- // the mask as given is ok as it contains at least mono or stereo
- // and potentially the haptic channels
- channelMask = static_cast<audio_channel_mask_t>(channelMaskFromFormat);
- } else {
- channelMask = static_cast<audio_channel_mask_t>(channelMaskFromFormat >> 2);
- }
+ channelMask = audio_channel_mask_from_media_format_mask(mediaFormatChannelMask);
} else {
// no mask found: the mask will be derived from the channel count
channelMask = CHANNEL_MASK_USE_CHANNEL_ORDER;
diff --git a/media/libmediaplayerservice/nuplayer/RTPSource.cpp b/media/libmediaplayerservice/nuplayer/RTPSource.cpp
index 6a17972..fd03150 100644
--- a/media/libmediaplayerservice/nuplayer/RTPSource.cpp
+++ b/media/libmediaplayerservice/nuplayer/RTPSource.cpp
@@ -115,7 +115,7 @@
int sockRtp, sockRtcp;
ARTPConnection::MakeRTPSocketPair(&sockRtp, &sockRtcp, info->mLocalIp, info->mRemoteIp,
- info->mLocalPort, info->mRemotePort, info->mSocketNetwork);
+ info->mLocalPort, info->mRemotePort, info->mSocketNetwork, info->mRtpSockOptEcn);
sp<AMessage> notify = new AMessage('accu', this);
@@ -125,6 +125,8 @@
mRTPConn->addStream(sockRtp, sockRtcp, desc, i + 1, notify, false);
mRTPConn->setSelfID(info->mSelfID);
mRTPConn->setStaticJitterTimeMs(info->mJbTimeMs);
+ mRTPConn->setRtpSockOptEcn(info->mRtpSockOptEcn);
+ mRTPConn->setIsIPv6(info->mLocalIp);
unsigned long PT;
AString formatDesc, formatParams;
@@ -719,6 +721,8 @@
} else if (key == "rtp-param-set-socket-network") {
int64_t networkHandle = atoll(value);
setSocketNetwork(networkHandle);
+ } else if (key == "rtp-param-set-socket-ecn") {
+ info->mRtpSockOptEcn = atoi(value);
} else if (key == "rtp-param-jitter-buffer-time") {
// clamping min at 40, max at 3000
info->mJbTimeMs = std::min(std::max(40, atoi(value)), 3000);
diff --git a/media/libmediaplayerservice/nuplayer/include/nuplayer/NuPlayerDrm.h b/media/libmediaplayerservice/nuplayer/include/nuplayer/NuPlayerDrm.h
index 4360656..232638c 100644
--- a/media/libmediaplayerservice/nuplayer/include/nuplayer/NuPlayerDrm.h
+++ b/media/libmediaplayerservice/nuplayer/include/nuplayer/NuPlayerDrm.h
@@ -106,8 +106,8 @@
uint8_t key[kBlockSize],
uint8_t iv[kBlockSize],
CryptoPlugin::Mode mode,
- size_t *clearbytes,
- size_t *encryptedbytes);
+ uint32_t *clearbytes,
+ uint32_t *encryptedbytes);
static CryptoInfo *getSampleCryptoInfo(MetaDataBase &meta);
diff --git a/media/libmediaplayerservice/nuplayer/include/nuplayer/RTPSource.h b/media/libmediaplayerservice/nuplayer/include/nuplayer/RTPSource.h
index 7d9bb8f..b2afe86 100644
--- a/media/libmediaplayerservice/nuplayer/include/nuplayer/RTPSource.h
+++ b/media/libmediaplayerservice/nuplayer/include/nuplayer/RTPSource.h
@@ -121,6 +121,8 @@
uint32_t mSelfID;
/* extmap:<value> for CVO will be set to here */
int32_t mCVOExtMap;
+ /* To check ECN is supported or not */
+ int32_t mRtpSockOptEcn;
/* a copy of TrackInfo in RTSPSource */
sp<AnotherPacketSource> mSource;
diff --git a/media/libstagefright/CameraSource.cpp b/media/libstagefright/CameraSource.cpp
index 9607425..967c316 100644
--- a/media/libstagefright/CameraSource.cpp
+++ b/media/libstagefright/CameraSource.cpp
@@ -150,7 +150,8 @@
if (camera == 0) {
mCamera = Camera::connect(cameraId, clientName, clientUid, clientPid,
- /*targetSdkVersion*/__ANDROID_API_FUTURE__);
+ /*targetSdkVersion*/__ANDROID_API_FUTURE__, /*overrideToPortrait*/false,
+ /*forceSlowJpegMode*/false);
if (mCamera == 0) return -EBUSY;
mCameraFlags &= ~FLAGS_HOT_CAMERA;
} else {
diff --git a/media/libstagefright/HevcUtils.cpp b/media/libstagefright/HevcUtils.cpp
index 5f9c20e..60df162 100644
--- a/media/libstagefright/HevcUtils.cpp
+++ b/media/libstagefright/HevcUtils.cpp
@@ -102,10 +102,11 @@
static bool findParam(uint32_t key, T *param,
KeyedVector<uint32_t, uint64_t> ¶ms) {
CHECK(param);
- if (params.indexOfKey(key) < 0) {
+ ssize_t index = params.indexOfKey(key);
+ if (index < 0) {
return false;
}
- *param = (T) params[key];
+ *param = (T) params[index];
return true;
}
diff --git a/media/libstagefright/MediaCodec.cpp b/media/libstagefright/MediaCodec.cpp
index d71faa7..3044c20 100644
--- a/media/libstagefright/MediaCodec.cpp
+++ b/media/libstagefright/MediaCodec.cpp
@@ -354,8 +354,11 @@
}
//static
-Mutex MediaCodec::ResourceManagerServiceProxy::sLockCookies;
-std::set<void*> MediaCodec::ResourceManagerServiceProxy::sCookies;
+// these are no_destroy to keep them from being destroyed at process exit
+// where some thread calls exit() while other threads are still running.
+// see b/194783918
+[[clang::no_destroy]] Mutex MediaCodec::ResourceManagerServiceProxy::sLockCookies;
+[[clang::no_destroy]] std::set<void*> MediaCodec::ResourceManagerServiceProxy::sCookies;
//static
void MediaCodec::ResourceManagerServiceProxy::addCookie(void* cookie) {
@@ -807,9 +810,7 @@
mWidth(0),
mHeight(0),
mRotationDegrees(0),
- mConfigColorTransfer(-1),
- mHDRStaticInfo(false),
- mHDR10PlusInfo(false),
+ mHdrInfoFlags(0),
mDequeueInputTimeoutGeneration(0),
mDequeueInputReplyID(0),
mDequeueOutputTimeoutGeneration(0),
@@ -968,29 +969,73 @@
mIndexOfFirstFrameWhenLowLatencyOn);
}
- mediametrics_setInt32(mMetricsHandle, kCodecHDRStaticInfo, mHDRStaticInfo ? 1 : 0);
- mediametrics_setInt32(mMetricsHandle, kCodecHDR10PlusInfo, mHDR10PlusInfo ? 1 : 0);
#if 0
// enable for short term, only while debugging
updateEphemeralMediametrics(mMetricsHandle);
#endif
}
-void MediaCodec::updateHDRFormatMetric() {
+void MediaCodec::updateHdrMetrics(bool isConfig) {
+ if ((mDomain != DOMAIN_VIDEO && mDomain != DOMAIN_IMAGE) || mMetricsHandle == 0) {
+ return;
+ }
+
+ int32_t colorStandard = -1;
+ if (mOutputFormat->findInt32(KEY_COLOR_STANDARD, &colorStandard)) {
+ mediametrics_setInt32(mMetricsHandle,
+ isConfig ? kCodecConfigColorStandard : kCodecParsedColorStandard, colorStandard);
+ }
+ int32_t colorRange = -1;
+ if (mOutputFormat->findInt32(KEY_COLOR_RANGE, &colorRange)) {
+ mediametrics_setInt32(mMetricsHandle,
+ isConfig ? kCodecConfigColorRange : kCodecParsedColorRange, colorRange);
+ }
+ int32_t colorTransfer = -1;
+ if (mOutputFormat->findInt32(KEY_COLOR_TRANSFER, &colorTransfer)) {
+ mediametrics_setInt32(mMetricsHandle,
+ isConfig ? kCodecConfigColorTransfer : kCodecParsedColorTransfer, colorTransfer);
+ }
+ HDRStaticInfo info;
+ if (ColorUtils::getHDRStaticInfoFromFormat(mOutputFormat, &info)
+ && ColorUtils::isHDRStaticInfoValid(&info)) {
+ mHdrInfoFlags |= kFlagHasHdrStaticInfo;
+ }
+ mediametrics_setInt32(mMetricsHandle, kCodecHDRStaticInfo,
+ (mHdrInfoFlags & kFlagHasHdrStaticInfo) ? 1 : 0);
+ sp<ABuffer> hdr10PlusInfo;
+ if (mOutputFormat->findBuffer("hdr10-plus-info", &hdr10PlusInfo)
+ && hdr10PlusInfo != nullptr && hdr10PlusInfo->size() > 0) {
+ mHdrInfoFlags |= kFlagHasHdr10PlusInfo;
+ }
+ mediametrics_setInt32(mMetricsHandle, kCodecHDR10PlusInfo,
+ (mHdrInfoFlags & kFlagHasHdr10PlusInfo) ? 1 : 0);
+
+ // hdr format
+ sp<AMessage> codedFormat = (mFlags & kFlagIsEncoder) ? mOutputFormat : mInputFormat;
+
+ AString mime;
int32_t profile = -1;
- AString mediaType;
- if (mOutputFormat->findInt32(KEY_PROFILE, &profile)
- && mOutputFormat->findString("mime", &mediaType)) {
- hdr_format hdrFormat = getHDRFormat(profile, mConfigColorTransfer, mediaType);
+
+ if (codedFormat->findString("mime", &mime)
+ && codedFormat->findInt32(KEY_PROFILE, &profile)
+ && colorTransfer != -1) {
+ hdr_format hdrFormat = getHdrFormat(mime, profile, colorTransfer);
mediametrics_setInt32(mMetricsHandle, kCodecHDRFormat, static_cast<int>(hdrFormat));
}
}
-hdr_format MediaCodec::getHDRFormat(const int32_t profile, const int32_t transfer,
- const AString &mediaType) {
- switch (transfer) {
+hdr_format MediaCodec::getHdrFormat(const AString &mime, const int32_t profile,
+ const int32_t colorTransfer) {
+ return (mFlags & kFlagIsEncoder)
+ ? getHdrFormatForEncoder(mime, profile, colorTransfer)
+ : getHdrFormatForDecoder(mime, profile, colorTransfer);
+}
+
+hdr_format MediaCodec::getHdrFormatForEncoder(const AString &mime, const int32_t profile,
+ const int32_t colorTransfer) {
+ switch (colorTransfer) {
case COLOR_TRANSFER_ST2084:
- if (mediaType.equalsIgnoreCase(MEDIA_MIMETYPE_VIDEO_VP9)) {
+ if (mime.equalsIgnoreCase(MEDIA_MIMETYPE_VIDEO_VP9)) {
switch (profile) {
case VP9Profile2HDR:
return HDR_FORMAT_HDR10;
@@ -999,7 +1044,7 @@
default:
return HDR_FORMAT_NONE;
}
- } else if (mediaType.equalsIgnoreCase(MEDIA_MIMETYPE_VIDEO_AV1)) {
+ } else if (mime.equalsIgnoreCase(MEDIA_MIMETYPE_VIDEO_AV1)) {
switch (profile) {
case AV1ProfileMain10HDR10:
return HDR_FORMAT_HDR10;
@@ -1008,7 +1053,7 @@
default:
return HDR_FORMAT_NONE;
}
- } else if (mediaType.equalsIgnoreCase(MEDIA_MIMETYPE_VIDEO_HEVC)) {
+ } else if (mime.equalsIgnoreCase(MEDIA_MIMETYPE_VIDEO_HEVC)) {
switch (profile) {
case HEVCProfileMain10HDR10:
return HDR_FORMAT_HDR10;
@@ -1021,7 +1066,7 @@
return HDR_FORMAT_NONE;
}
case COLOR_TRANSFER_HLG:
- if (!mediaType.equalsIgnoreCase(MEDIA_MIMETYPE_VIDEO_DOLBY_VISION)) {
+ if (!mime.equalsIgnoreCase(MEDIA_MIMETYPE_VIDEO_DOLBY_VISION)) {
return HDR_FORMAT_HLG;
} else {
// TODO: DOLBY format
@@ -1032,6 +1077,47 @@
}
}
+hdr_format MediaCodec::getHdrFormatForDecoder(const AString &mime, const int32_t profile,
+ const int32_t colorTransfer) {
+ switch (colorTransfer) {
+ case COLOR_TRANSFER_ST2084:
+ if (!(mHdrInfoFlags & kFlagHasHdrStaticInfo) || !profileSupport10Bits(mime, profile)) {
+ return HDR_FORMAT_NONE;
+ }
+ return mHdrInfoFlags & kFlagHasHdr10PlusInfo ? HDR_FORMAT_HDR10PLUS : HDR_FORMAT_HDR10;
+ case COLOR_TRANSFER_HLG:
+ if (!mime.equalsIgnoreCase(MEDIA_MIMETYPE_VIDEO_DOLBY_VISION)) {
+ return HDR_FORMAT_HLG;
+ }
+ // TODO: DOLBY format
+ }
+ return HDR_FORMAT_NONE;
+}
+
+bool MediaCodec::profileSupport10Bits(const AString &mime, const int32_t profile) {
+ if (mime.equalsIgnoreCase(MEDIA_MIMETYPE_VIDEO_AV1)) {
+ return true;
+ } else if (mime.equalsIgnoreCase(MEDIA_MIMETYPE_VIDEO_VP9)) {
+ switch (profile) {
+ case VP9Profile2:
+ case VP9Profile3:
+ case VP9Profile2HDR:
+ case VP9Profile3HDR:
+ case VP9Profile2HDR10Plus:
+ case VP9Profile3HDR10Plus:
+ return true;
+ }
+ } else if (mime.equalsIgnoreCase(MEDIA_MIMETYPE_VIDEO_HEVC)) {
+ switch (profile) {
+ case HEVCProfileMain10:
+ case HEVCProfileMain10HDR10:
+ case HEVCProfileMain10HDR10Plus:
+ return true;
+ }
+ }
+ return false;
+}
+
// called to update info being passed back via getMetrics(), which is a
// unique copy for that call, no concurrent access worries.
@@ -1081,6 +1167,7 @@
// ensure mutex while we do our own work
Mutex::Autolock _lock(mMetricsLock);
+ mHdrInfoFlags = 0;
if (mMetricsHandle != 0) {
if (mediametrics_count(mMetricsHandle) > 0) {
mediametrics_selfRecord(mMetricsHandle);
@@ -1735,24 +1822,6 @@
mediametrics_setInt32(nextMetricsHandle, kCodecPriority, priority);
}
}
- int32_t colorStandard = -1;
- if (format->findInt32(KEY_COLOR_STANDARD, &colorStandard)) {
- mediametrics_setInt32(mMetricsHandle, kCodecConfigColorStandard, colorStandard);
- }
- int32_t colorRange = -1;
- if (format->findInt32(KEY_COLOR_RANGE, &colorRange)) {
- mediametrics_setInt32(mMetricsHandle, kCodecConfigColorRange, colorRange);
- }
- int32_t colorTransfer = -1;
- if (format->findInt32(KEY_COLOR_TRANSFER, &colorTransfer)) {
- mConfigColorTransfer = colorTransfer;
- mediametrics_setInt32(mMetricsHandle, kCodecConfigColorTransfer, colorTransfer);
- }
- HDRStaticInfo info;
- if (ColorUtils::getHDRStaticInfoFromFormat(format, &info)
- && ColorUtils::isHDRStaticInfoValid(&info)) {
- mHDRStaticInfo = true;
- }
}
// Prevent possible integer overflow in downstream code.
@@ -3439,8 +3508,6 @@
CHECK(msg->findMessage("input-format", &mInputFormat));
CHECK(msg->findMessage("output-format", &mOutputFormat));
- updateHDRFormatMetric();
-
// limit to confirming the opt-in behavior to minimize any behavioral change
if (mSurface != nullptr && !mAllowFrameDroppingBySurface) {
// signal frame dropping mode in the input format as this may also be
@@ -3483,6 +3550,7 @@
if (interestingFormat->findInt32("level", &level)) {
mediametrics_setInt32(mMetricsHandle, kCodecLevel, level);
}
+ updateHdrMetrics(true /* isConfig */);
// bitrate and bitrate mode, encoder only
if (mFlags & kFlagIsEncoder) {
// encoder specific values
@@ -3522,7 +3590,6 @@
mComponentName.c_str(),
mInputFormat->debugString(4).c_str(),
mOutputFormat->debugString(4).c_str());
- updateHDRFormatMetric();
CHECK(obj != NULL);
response->setObject("input-surface", obj);
mHaveInputSurface = true;
@@ -3547,7 +3614,6 @@
if (!msg->findInt32("err", &err)) {
CHECK(msg->findMessage("input-format", &mInputFormat));
CHECK(msg->findMessage("output-format", &mOutputFormat));
- updateHDRFormatMetric();
mHaveInputSurface = true;
} else {
response->setInt32("err", err);
@@ -3591,6 +3657,17 @@
}
setState(STARTED);
postPendingRepliesAndDeferredMessages("kWhatStartCompleted");
+
+ // Now that the codec has started, configure, by default, the peek behavior to
+ // be undefined for backwards compatibility with older releases. Later, if an
+ // app explicitly enables or disables peek, the parameter will be turned off and
+ // the legacy undefined behavior is disallowed.
+ // See updateTunnelPeek called in onSetParameters for more details.
+ if (mTunneled && mTunnelPeekState == TunnelPeekState::kLegacyMode) {
+ sp<AMessage> params = new AMessage;
+ params->setInt32("android._tunnel-peek-set-legacy", 1);
+ mCodec->signalSetParameters(params);
+ }
break;
}
@@ -4049,14 +4126,6 @@
mTunneled = false;
}
- // If mTunnelPeekState is still in kLegacyMode at this point,
- // configure the codec in legacy mode
- if (mTunneled && (mTunnelPeekState == TunnelPeekState::kLegacyMode)) {
- sp<AMessage> params = new AMessage;
- params->setInt32("android._tunnel-peek-set-legacy", 1);
- onSetParameters(params);
- }
-
int32_t background = 0;
if (format->findInt32("android._background-mode", &background) && background) {
androidSetThreadPriority(gettid(), ANDROID_PRIORITY_BACKGROUND);
@@ -4757,7 +4826,6 @@
buffer->meta()->setObject("changedKeys", changedKeys);
}
mOutputFormat = format;
- updateHDRFormatMetric();
mapFormat(mComponentName, format, nullptr, true);
ALOGV("[%s] output format changed to: %s",
mComponentName.c_str(), mOutputFormat->debugString(4).c_str());
@@ -4783,9 +4851,6 @@
HDRStaticInfo info;
if (ColorUtils::getHDRStaticInfoFromFormat(mOutputFormat, &info)) {
setNativeWindowHdrMetadata(mSurface.get(), &info);
- if (ColorUtils::isHDRStaticInfoValid(&info)) {
- mHDRStaticInfo = true;
- }
}
}
@@ -4794,7 +4859,6 @@
&& hdr10PlusInfo != nullptr && hdr10PlusInfo->size() > 0) {
native_window_set_buffers_hdr10_plus_metadata(mSurface.get(),
hdr10PlusInfo->size(), hdr10PlusInfo->data());
- mHDR10PlusInfo = true;
}
if (mime.startsWithIgnoreCase("video/")) {
@@ -4840,21 +4904,8 @@
}
}
- if (mMetricsHandle != 0) {
- int32_t colorStandard = -1;
- if (format->findInt32(KEY_COLOR_STANDARD, &colorStandard)) {
- mediametrics_setInt32(mMetricsHandle, kCodecParsedColorStandard, colorStandard);
- }
- int32_t colorRange = -1;
- if (format->findInt32( KEY_COLOR_RANGE, &colorRange)) {
- mediametrics_setInt32(mMetricsHandle, kCodecParsedColorRange, colorRange);
- }
- int32_t colorTransfer = -1;
- if (format->findInt32(KEY_COLOR_TRANSFER, &colorTransfer)) {
- mediametrics_setInt32(mMetricsHandle, kCodecParsedColorTransfer, colorTransfer);
- }
- }
-}
+ updateHdrMetrics(false /* isConfig */);
+ }
void MediaCodec::extractCSD(const sp<AMessage> &format) {
mCSD.clear();
diff --git a/media/libstagefright/SurfaceUtils.cpp b/media/libstagefright/SurfaceUtils.cpp
index 1f569ef..f526e05 100644
--- a/media/libstagefright/SurfaceUtils.cpp
+++ b/media/libstagefright/SurfaceUtils.cpp
@@ -222,6 +222,11 @@
static_cast<Surface*>(nativeWindow)->getIGraphicBufferProducer()->allowAllocation(true);
+ // In nonblocking mode(timetout = 0), native_window_dequeue_buffer_and_wait()
+ // can fail with timeout. Changing to blocking mode will ensure that dequeue
+ // does not timeout.
+ static_cast<Surface*>(nativeWindow)->getIGraphicBufferProducer()->setDequeueTimeout(-1);
+
err = nativeWindow->query(nativeWindow,
NATIVE_WINDOW_MIN_UNDEQUEUED_BUFFERS, &minUndequeuedBufs);
if (err != NO_ERROR) {
diff --git a/media/libstagefright/colorconversion/ColorConverter.cpp b/media/libstagefright/colorconversion/ColorConverter.cpp
index 4b4f65f..4df79ef 100644
--- a/media/libstagefright/colorconversion/ColorConverter.cpp
+++ b/media/libstagefright/colorconversion/ColorConverter.cpp
@@ -340,6 +340,14 @@
return mCropBottom - mCropTop + 1;
}
+bool ColorConverter::BitmapParams::isValid() const {
+ if (!((mStride & 1) == 0 // stride must be even
+ && mStride >= mBpp * cropWidth())) {
+ return false;
+ }
+ return true;
+}
+
status_t ColorConverter::convert(
const void *srcBits,
size_t srcWidth, size_t srcHeight, size_t srcStride,
@@ -359,9 +367,11 @@
dstWidth, dstHeight, dstStride,
dstCropLeft, dstCropTop, dstCropRight, dstCropBottom, mDstFormat);
- if (!((src.mCropLeft & 1) == 0
- && src.cropWidth() == dst.cropWidth()
- && src.cropHeight() == dst.cropHeight())) {
+ if (!(src.isValid()
+ && dst.isValid()
+ && (src.mCropLeft & 1) == 0
+ && src.cropWidth() == dst.cropWidth()
+ && src.cropHeight() == dst.cropHeight())) {
return ERROR_UNSUPPORTED;
}
@@ -463,6 +473,7 @@
}
}
+// Interleaved YUV 422 CbYCrY to RGB565
status_t ColorConverter::convertCbYCrY(
const BitmapParams &src, const BitmapParams &dst) {
// XXX Untested
@@ -485,10 +496,10 @@
+ dst.mCropTop * dst.mWidth + dst.mCropLeft;
const uint8_t *src_ptr = (const uint8_t *)src.mBits
- + (src.mCropTop * dst.mWidth + src.mCropLeft) * 2;
+ + (src.mCropTop * src.mWidth + src.mCropLeft) * 2;
for (size_t y = 0; y < src.cropHeight(); ++y) {
- for (size_t x = 0; x < src.cropWidth(); x += 2) {
+ for (size_t x = 0; x < src.cropWidth() - 1; x += 2) {
signed y1 = (signed)src_ptr[2 * x + 1] - _c16;
signed y2 = (signed)src_ptr[2 * x + 3] - _c16;
signed u = (signed)src_ptr[2 * x] - 128;
@@ -658,13 +669,15 @@
*u = ((uint8_t*)src_u)[x / 2] - 128;
*v = ((uint8_t*)src_v)[x / 2] - 128;
};
+ // this format stores 10 bits content with 16 bits
+ // converting it to 8 bits src
case OMX_COLOR_FormatYUV420Planar16:
return [](void *src_y, void *src_u, void *src_v, size_t x,
signed *y1, signed *y2, signed *u, signed *v) {
- *y1 = (signed)(((uint16_t*)src_y)[x] >> 2);
- *y2 = (signed)(((uint16_t*)src_y)[x + 1] >> 2);
- *u = (signed)(((uint16_t*)src_u)[x / 2] >> 2) - 128;
- *v = (signed)(((uint16_t*)src_v)[x / 2] >> 2) - 128;
+ *y1 = (uint8_t)(((uint16_t*)src_y)[x] >> 2);
+ *y2 = (uint8_t)(((uint16_t*)src_y)[x + 1] >> 2);
+ *u = (uint8_t)(((uint16_t*)src_u)[x / 2] >> 2) - 128;
+ *v = (uint8_t)(((uint16_t*)src_v)[x / 2] >> 2) - 128;
};
default:
TRESPASS();
@@ -1122,46 +1135,25 @@
status_t ColorConverter::convertQCOMYUV420SemiPlanar(
const BitmapParams &src, const BitmapParams &dst) {
- const uint8_t *src_y =
- (const uint8_t *)src.mBits + src.mCropTop * src.mWidth + src.mCropLeft;
-
- const uint8_t *src_u =
- (const uint8_t *)src_y + src.mWidth * src.mHeight
- + src.mCropTop * src.mWidth + src.mCropLeft;
-
/* QCOMYUV420SemiPlanar is NV21, while MediaCodec uses NV12 */
return convertYUV420SemiPlanarBase(
- src, dst, src_y, src_u, src.mWidth /* row_inc */, true /* isNV21 */);
+ src, dst, src.mWidth /* row_inc */, true /* isNV21 */);
}
status_t ColorConverter::convertTIYUV420PackedSemiPlanar(
const BitmapParams &src, const BitmapParams &dst) {
- const uint8_t *src_y =
- (const uint8_t *)src.mBits + src.mCropTop * src.mWidth + src.mCropLeft;
-
- const uint8_t *src_u =
- (const uint8_t *)src_y + src.mWidth * (src.mHeight - src.mCropTop / 2);
-
return convertYUV420SemiPlanarBase(
- src, dst, src_y, src_u, src.mWidth /* row_inc */);
+ src, dst, src.mWidth /* row_inc */);
}
status_t ColorConverter::convertYUV420SemiPlanar(
const BitmapParams &src, const BitmapParams &dst) {
- const uint8_t *src_y =
- (const uint8_t *)src.mBits + src.mCropTop * src.mStride + src.mCropLeft;
-
- const uint8_t *src_u =
- (const uint8_t *)src.mBits + src.mHeight * src.mStride +
- (src.mCropTop / 2) * src.mStride + src.mCropLeft;
-
return convertYUV420SemiPlanarBase(
- src, dst, src_y, src_u, src.mStride /* row_inc */);
+ src, dst, src.mStride /* row_inc */);
}
-status_t ColorConverter::convertYUV420SemiPlanarBase(
- const BitmapParams &src, const BitmapParams &dst,
- const uint8_t *src_y, const uint8_t *src_u, size_t row_inc, bool isNV21) {
+status_t ColorConverter::convertYUV420SemiPlanarBase(const BitmapParams &src,
+ const BitmapParams &dst, size_t row_inc, bool isNV21) {
const struct Coeffs *matrix = getMatrix();
if (!matrix) {
return ERROR_UNSUPPORTED;
@@ -1179,6 +1171,12 @@
uint16_t *dst_ptr = (uint16_t *)((uint8_t *)
dst.mBits + dst.mCropTop * dst.mStride + dst.mCropLeft * dst.mBpp);
+ const uint8_t *src_y =
+ (const uint8_t *)src.mBits + src.mCropTop * row_inc + src.mCropLeft;
+
+ const uint8_t *src_u = (const uint8_t *)src.mBits + src.mHeight * row_inc +
+ (src.mCropTop / 2) * row_inc + src.mCropLeft;
+
for (size_t y = 0; y < src.cropHeight(); ++y) {
for (size_t x = 0; x < src.cropWidth(); x += 2) {
signed y1 = (signed)src_y[x] - _c16;
diff --git a/media/libstagefright/include/media/stagefright/ColorConverter.h b/media/libstagefright/include/media/stagefright/ColorConverter.h
index 7a05f00..da3267e 100644
--- a/media/libstagefright/include/media/stagefright/ColorConverter.h
+++ b/media/libstagefright/include/media/stagefright/ColorConverter.h
@@ -74,6 +74,8 @@
size_t cropWidth() const;
size_t cropHeight() const;
+ bool isValid() const;
+
void *mBits;
OMX_COLOR_FORMATTYPE mColorFormat;
size_t mWidth, mHeight;
@@ -121,7 +123,7 @@
status_t convertYUV420SemiPlanarBase(
const BitmapParams &src, const BitmapParams &dst,
- const uint8_t *src_y, const uint8_t *src_u, size_t row_inc, bool isNV21 = false);
+ size_t row_inc, bool isNV21 = false);
status_t convertTIYUV420PackedSemiPlanar(
const BitmapParams &src, const BitmapParams &dst);
diff --git a/media/libstagefright/include/media/stagefright/MediaCodec.h b/media/libstagefright/include/media/stagefright/MediaCodec.h
index bc6765c..6644e8e 100644
--- a/media/libstagefright/include/media/stagefright/MediaCodec.h
+++ b/media/libstagefright/include/media/stagefright/MediaCodec.h
@@ -456,12 +456,19 @@
int32_t mRotationDegrees;
int32_t mAllowFrameDroppingBySurface;
- int32_t mConfigColorTransfer;
- bool mHDRStaticInfo;
- bool mHDR10PlusInfo;
- void updateHDRFormatMetric();
- hdr_format getHDRFormat(const int32_t profile, const int32_t transfer,
- const AString &mediaType);
+ enum {
+ kFlagHasHdrStaticInfo = 1,
+ kFlagHasHdr10PlusInfo = 2,
+ };
+ uint32_t mHdrInfoFlags;
+ void updateHdrMetrics(bool isConfig);
+ hdr_format getHdrFormat(const AString &mime, const int32_t profile,
+ const int32_t colorTransfer);
+ hdr_format getHdrFormatForEncoder(const AString &mime, const int32_t profile,
+ const int32_t colorTransfer);
+ hdr_format getHdrFormatForDecoder(const AString &mime, const int32_t profile,
+ const int32_t colorTransfer);
+ bool profileSupport10Bits(const AString &mime, const int32_t profile);
// initial create parameters
AString mInitName;
diff --git a/media/libstagefright/include/media/stagefright/MetaDataBase.h b/media/libstagefright/include/media/stagefright/MetaDataBase.h
index 88c1f3f..31faafb 100644
--- a/media/libstagefright/include/media/stagefright/MetaDataBase.h
+++ b/media/libstagefright/include/media/stagefright/MetaDataBase.h
@@ -267,6 +267,7 @@
kKeyRtpExtMap = 'extm', // int32_t, rtp extension ID for cvo on RTP protocol.
kKeyRtpCvoDegrees = 'cvod', // int32_t, rtp cvo degrees as per 3GPP 26.114.
kKeyRtpDscp = 'dscp', // int32_t, DSCP(Differentiated services codepoint) of RFC 2474.
+ kKeyRtpEcn = 'sEcn', // int32_t, ECN (Explicit Congestion Notification) of RFC 3168
kKeySocketNetwork = 'sNet', // int64_t, socket will be bound to network handle.
// Slow-motion markers
diff --git a/media/libstagefright/rtsp/AAVCAssembler.cpp b/media/libstagefright/rtsp/AAVCAssembler.cpp
index 88f7be7..100c0cd 100644
--- a/media/libstagefright/rtsp/AAVCAssembler.cpp
+++ b/media/libstagefright/rtsp/AAVCAssembler.cpp
@@ -46,7 +46,6 @@
mFirstIFrameProvided(false),
mLastCvo(-1),
mLastIFrameProvidedAtMs(0),
- mLastRtpTimeJitterDataUs(0),
mWidth(0),
mHeight(0) {
}
@@ -123,20 +122,11 @@
}
sp<ABuffer> buffer = *queue->begin();
+ uint32_t seqNum = (uint32_t)buffer->int32Data();
buffer->meta()->setObject("source", source);
- /**
- * RFC3550 calculates the interarrival jitter time for 'ALL packets'.
- * But that is not useful as an ingredient of buffering time.
- * Instead, we calculates the time only for all 'NAL units'.
- */
int64_t rtpTime = findRTPTime(firstRTPTime, buffer);
int64_t nowTimeUs = ALooper::GetNowUs();
- if (rtpTime != mLastRtpTimeJitterDataUs) {
- source->putBaseJitterData(rtpTime, nowTimeUs);
- mLastRtpTimeJitterDataUs = rtpTime;
- }
- source->putInterArrivalJitterData(rtpTime, nowTimeUs);
const int64_t startTimeMs = source->mSysAnchorTime / 1000;
const int64_t nowTimeMs = nowTimeUs / 1000;
@@ -168,7 +158,7 @@
const int32_t dynamicJbTimeMs = std::min(dynamicJitterTimeMs, 150);
const int64_t dynamicJbTimeRtp = MsToRtp(dynamicJbTimeMs, clockRate);
/* Fundamental jitter time */
- const int32_t jitterTimeMs = baseJbTimeMs;
+ const int32_t jitterTimeMs = baseJbTimeMs + dynamicJbTimeMs;
const int64_t jitterTimeRtp = MsToRtp(jitterTimeMs, clockRate);
// Till (T), this assembler waits unconditionally to collect current NAL unit
@@ -177,7 +167,7 @@
bool isExpired = (diffTimeRtp >= 0); // It's expired if T is passed away
// From (T), this assembler tries to complete the NAL till (T + try)
- int32_t tryJbTimeMs = baseJitterTimeMs / 2 + dynamicJbTimeMs;
+ int32_t tryJbTimeMs = dynamicJbTimeMs;
int64_t tryJbTimeRtp = MsToRtp(tryJbTimeMs, clockRate);
bool isFirstLineBroken = (diffTimeRtp > tryJbTimeRtp);
@@ -208,10 +198,10 @@
String8 info;
info.appendFormat("RTP diff from exp =%lld \t MS diff from stamp = %lld\t\t"
"Seq# %d \t ExpSeq# %d \t"
- "JitterMs %d + (%d + %d * %.3f)",
+ "JitterMs [%d + (~%d~)] + %d * %.3f",
(long long)diffTimeRtp, (long long)totalDiffTimeMs,
- buffer->int32Data(), mNextExpectedSeqNo,
- jitterTimeMs, tryJbTimeMs, dynamicJbTimeMs, JITTER_MULTIPLE);
+ seqNum, mNextExpectedSeqNo,
+ baseJbTimeMs, dynamicJbTimeMs, tryJbTimeMs, JITTER_MULTIPLE);
if (isSecondLineBroken) {
ALOGE("%s", info.string());
printNowTimeMs(startTimeMs, nowTimeMs, playedTimeMs);
@@ -223,6 +213,9 @@
}
if (mNextExpectedSeqNoValid) {
+ if (mNextExpectedSeqNo > seqNum) {
+ ALOGE("Reversed exp seq# %d \t current head %d", mNextExpectedSeqNo, seqNum);
+ }
mNextExpectedSeqNo = pickStartSeq(queue, firstRTPTime, playedTimeRtp, jitterTimeRtp);
int32_t cntRemove = deleteUnitUnderSeq(queue, mNextExpectedSeqNo);
@@ -241,10 +234,10 @@
if (!mNextExpectedSeqNoValid) {
mNextExpectedSeqNoValid = true;
- mNextExpectedSeqNo = (uint32_t)buffer->int32Data();
- } else if ((uint32_t)buffer->int32Data() != mNextExpectedSeqNo) {
- ALOGV("Not the sequence number I expected");
-
+ mNextExpectedSeqNo = seqNum;
+ } else if (seqNum != mNextExpectedSeqNo) {
+ ALOGV("Not the sequence number(%d) I expected. Actual seq# is %d",
+ mNextExpectedSeqNo, seqNum);
return WRONG_SEQUENCE_NUMBER;
}
diff --git a/media/libstagefright/rtsp/AHEVCAssembler.cpp b/media/libstagefright/rtsp/AHEVCAssembler.cpp
index 72dd981..7b5c24a 100644
--- a/media/libstagefright/rtsp/AHEVCAssembler.cpp
+++ b/media/libstagefright/rtsp/AHEVCAssembler.cpp
@@ -53,7 +53,6 @@
mFirstIFrameProvided(false),
mLastCvo(-1),
mLastIFrameProvidedAtMs(0),
- mLastRtpTimeJitterDataUs(0),
mWidth(0),
mHeight(0) {
@@ -133,20 +132,11 @@
}
sp<ABuffer> buffer = *queue->begin();
+ uint32_t seqNum = (uint32_t)buffer->int32Data();
buffer->meta()->setObject("source", source);
- /**
- * RFC3550 calculates the interarrival jitter time for 'ALL packets'.
- * But that is not useful as an ingredient of buffering time.
- * Instead, we calculates the time only for all 'NAL units'.
- */
int64_t rtpTime = findRTPTime(firstRTPTime, buffer);
int64_t nowTimeUs = ALooper::GetNowUs();
- if (rtpTime != mLastRtpTimeJitterDataUs) {
- source->putBaseJitterData(rtpTime, nowTimeUs);
- mLastRtpTimeJitterDataUs = rtpTime;
- }
- source->putInterArrivalJitterData(rtpTime, nowTimeUs);
const int64_t startTimeMs = source->mSysAnchorTime / 1000;
const int64_t nowTimeMs = nowTimeUs / 1000;
@@ -178,7 +168,7 @@
const int32_t dynamicJbTimeMs = std::min(dynamicJitterTimeMs, 150);
const int64_t dynamicJbTimeRtp = MsToRtp(dynamicJbTimeMs, clockRate);
/* Fundamental jitter time */
- const int32_t jitterTimeMs = baseJbTimeMs;
+ const int32_t jitterTimeMs = baseJbTimeMs + dynamicJbTimeMs;
const int64_t jitterTimeRtp = MsToRtp(jitterTimeMs, clockRate);
// Till (T), this assembler waits unconditionally to collect current NAL unit
@@ -187,7 +177,7 @@
bool isExpired = (diffTimeRtp >= 0); // It's expired if T is passed away
// From (T), this assembler tries to complete the NAL till (T + try)
- int32_t tryJbTimeMs = baseJitterTimeMs / 2 + dynamicJbTimeMs;
+ int32_t tryJbTimeMs = dynamicJbTimeMs;
int64_t tryJbTimeRtp = MsToRtp(tryJbTimeMs, clockRate);
bool isFirstLineBroken = (diffTimeRtp > tryJbTimeRtp);
@@ -218,10 +208,10 @@
String8 info;
info.appendFormat("RTP diff from exp =%lld \t MS diff from stamp = %lld\t\t"
"Seq# %d \t ExpSeq# %d \t"
- "JitterMs %d + (%d + %d * %.3f)",
+ "JitterMs [%d + (~%d~)] + %d * %.3f",
(long long)diffTimeRtp, (long long)totalDiffTimeMs,
- buffer->int32Data(), mNextExpectedSeqNo,
- jitterTimeMs, tryJbTimeMs, dynamicJbTimeMs, JITTER_MULTIPLE);
+ seqNum, mNextExpectedSeqNo,
+ baseJbTimeMs, dynamicJbTimeMs, tryJbTimeMs, JITTER_MULTIPLE);
if (isSecondLineBroken) {
ALOGE("%s", info.string());
printNowTimeMs(startTimeMs, nowTimeMs, playedTimeMs);
@@ -251,10 +241,10 @@
if (!mNextExpectedSeqNoValid) {
mNextExpectedSeqNoValid = true;
- mNextExpectedSeqNo = (uint32_t)buffer->int32Data();
- } else if ((uint32_t)buffer->int32Data() != mNextExpectedSeqNo) {
- ALOGV("Not the sequence number I expected");
-
+ mNextExpectedSeqNo = seqNum;
+ } else if (seqNum != mNextExpectedSeqNo) {
+ ALOGV("Not the sequence number(%d) I expected. Actual seq# is %d",
+ mNextExpectedSeqNo, seqNum);
return WRONG_SEQUENCE_NUMBER;
}
diff --git a/media/libstagefright/rtsp/ARTPConnection.cpp b/media/libstagefright/rtsp/ARTPConnection.cpp
index a61f48f..165c336 100644
--- a/media/libstagefright/rtsp/ARTPConnection.cpp
+++ b/media/libstagefright/rtsp/ARTPConnection.cpp
@@ -16,6 +16,12 @@
//#define LOG_NDEBUG 0
#define LOG_TAG "ARTPConnection"
+#define INET_ECN_NOT_ECT 0x00 /* ECN was not enabled */
+#define INET_ECN_ECT_1 0x01 /* ECN capable packet */
+#define INET_ECN_ECT_0 0x02 /* ECN capable packet */
+#define INET_ECN_CE 0x03 /* ECN congestion */
+#define INET_ECN_MASK 0x03 /* Mask of ECN bits */
+
#include <utils/Log.h>
#include <media/stagefright/rtsp/ARTPAssembler.h>
@@ -56,6 +62,7 @@
// static
const int64_t ARTPConnection::kSelectTimeoutUs = 1000LL;
+const int64_t ARTPConnection::kMinOneSecondNotifyDelayUs = 100000ll;
struct ARTPConnection::StreamInfo {
bool isIPv6;
@@ -84,7 +91,10 @@
mPollEventPending(false),
mLastReceiverReportTimeUs(-1),
mLastBitrateReportTimeUs(-1),
+ mLastCongestionNotifyTimeUs(-1),
mTargetBitrate(-1),
+ mRtpSockOptEcn(0),
+ mIsIPv6(false),
mStaticJitterTimeMs(kStaticJitterTimeMs) {
}
@@ -175,7 +185,7 @@
// static
void ARTPConnection::MakeRTPSocketPair(
int *rtpSocket, int *rtcpSocket, const char *localIp, const char *remoteIp,
- unsigned localPort, unsigned remotePort, int64_t socketNetwork) {
+ unsigned localPort, unsigned remotePort, int64_t socketNetwork, int32_t sockOptEcn) {
bool isIPv6 = false;
if (strchr(localIp, ':') != NULL)
isIPv6 = true;
@@ -204,6 +214,24 @@
}
}
+ if (sockOptEcn != 0) {
+ int sockOptForTOS = 1;
+ if (setsockopt(*rtpSocket, isIPv6 ? IPPROTO_IPV6 : IPPROTO_IP,
+ isIPv6 ? IPV6_RECVTCLASS : IP_RECVTOS,
+ (int *)&sockOptForTOS, sizeof(sockOptForTOS)) < 0) {
+ ALOGE("failed to set recv sockopt TOS on rtpsock(%d). err=%s", *rtpSocket,
+ strerror(errno));
+ } else {
+ ALOGD("successfully set recv sockopt TOS on rtpsock(%d)", *rtpSocket);
+ int result = setsockopt(*rtcpSocket, isIPv6 ? IPPROTO_IPV6 : IPPROTO_IP,
+ isIPv6 ? IPV6_RECVTCLASS : IP_RECVTOS,
+ (int *)&sockOptForTOS, sizeof(sockOptForTOS));
+ if (result >= 0) {
+ ALOGD("successfully set recv sockopt TOS on rtcpsock(%d).", *rtcpSocket);
+ }
+ }
+ }
+
bumpSocketBufferSize(*rtcpSocket);
struct sockaddr *addr;
@@ -593,32 +621,25 @@
sp<ABuffer> buffer = new ABuffer(65536);
- struct sockaddr *pRemoteRTCPAddr;
- int sizeSockSt;
- if (s->isIPv6) {
- pRemoteRTCPAddr = (struct sockaddr *)&s->mRemoteRTCPAddr6;
- sizeSockSt = sizeof(struct sockaddr_in6);
- } else {
- pRemoteRTCPAddr = (struct sockaddr *)&s->mRemoteRTCPAddr;
- sizeSockSt = sizeof(struct sockaddr_in);
- }
- socklen_t remoteAddrLen =
- (!receiveRTP && s->mNumRTCPPacketsReceived == 0)
- ? sizeSockSt : 0;
+ struct msghdr sMsg = {};
+ struct iovec sIov[1] = {};
- if (mFlags & kViLTEConnection) {
- remoteAddrLen = 0;
- }
+ sIov[0].iov_base = (char *) buffer->data();
+ sIov[0].iov_len = buffer->capacity();
+
+ sMsg.msg_iov = sIov;
+ sMsg.msg_iovlen = 1;
+
+ int cMsgSize = sizeof(struct cmsghdr) + sizeof(uint8_t);
+ char buf[CMSG_SPACE(cMsgSize)];
+ sMsg.msg_control = buf;
+ sMsg.msg_controllen = sizeof(buf);
+ sMsg.msg_flags = 0;
ssize_t nbytes;
do {
- nbytes = recvfrom(
- receiveRTP ? s->mRTPSocket : s->mRTCPSocket,
- buffer->data(),
- buffer->capacity(),
- 0,
- remoteAddrLen > 0 ? pRemoteRTCPAddr : NULL,
- remoteAddrLen > 0 ? &remoteAddrLen : NULL);
+ // Used recvmsg to get the TOS header of incoming packet
+ nbytes = recvmsg(receiveRTP ? s->mRTPSocket : s->mRTCPSocket, &sMsg, 0);
mCumulativeBytes += nbytes;
} while (nbytes < 0 && errno == EINTR);
@@ -633,6 +654,10 @@
}
}
+ if (nbytes > 0) {
+ handleIpHeadersIfReceived(s, sMsg);
+ }
+
buffer->setRange(0, nbytes);
// ALOGI("received %d bytes.", buffer->size());
@@ -647,13 +672,68 @@
return err;
}
+/* This function will check if TOS is present or not in received IP packet.
+ * After that if it is present then it will notify about congestion to upper
+ * layer if CE bit is set in TOS header.
+ **/
+void ARTPConnection::handleIpHeadersIfReceived(StreamInfo *s, struct msghdr sMsg) {
+ struct cmsghdr *cMsg;
+ cMsg = CMSG_FIRSTHDR(&sMsg);
+
+ if (cMsg == NULL) {
+ ALOGV("cmsg is null");
+ }
+
+ for (; cMsg != NULL; cMsg = CMSG_NXTHDR(&sMsg, cMsg)) {
+ bool isTOSHeader = ((cMsg->cmsg_level == (mIsIPv6 ? IPPROTO_IPV6 : IPPROTO_IP))
+ && (cMsg->cmsg_type == (mIsIPv6 ? IPV6_TCLASS : IP_TOS))
+ && (cMsg->cmsg_len));
+ if (isTOSHeader) {
+ uint8_t receivedTOS;
+ receivedTOS = *((uint8_t *) CMSG_DATA(cMsg));
+ // checking CE bit is set
+ bool isCEBitMarked = ((receivedTOS & INET_ECN_MASK) == INET_ECN_CE);
+
+ ALOGV("receivedTos(value -> %d)", receivedTOS);
+
+ if (isCEBitMarked) {
+ ALOGD("receivedTos(value -> %d), is ECN CE marked = %d",
+ receivedTOS, isCEBitMarked);
+ notifyCongestionToUpperLayerIfNeeded(s);
+ }
+ break;
+ }
+ }
+}
+
+/* this function will be use to notify congestion in video call to upper layer */
+void ARTPConnection::notifyCongestionToUpperLayerIfNeeded(StreamInfo *s) {
+ int64_t nowUs = ALooper::GetNowUs();
+
+ if (mLastCongestionNotifyTimeUs <= 0) {
+ mLastCongestionNotifyTimeUs = nowUs;
+ }
+
+ bool isNeedToUpdate = (mLastCongestionNotifyTimeUs + kMinOneSecondNotifyDelayUs <= nowUs);
+ ALOGD("ECN info set by upper layer=%d, isNeedToUpdate=%d", mRtpSockOptEcn, isNeedToUpdate);
+
+ if ((mRtpSockOptEcn != 0) && (isNeedToUpdate)) {
+ sp<AMessage> notify = s->mNotifyMsg->dup();
+ notify->setInt32("rtcp-event", 1);
+ notify->setInt32("payload-type", ARTPSource::RTP_QUALITY_CD);
+ notify->post();
+ mLastCongestionNotifyTimeUs = nowUs;
+ ALOGD("Congestion detected in n/w, Notify upper layer");
+ }
+}
+
ssize_t ARTPConnection::send(const StreamInfo *info, const sp<ABuffer> buffer) {
struct sockaddr* pRemoteRTCPAddr;
int sizeSockSt;
/* It seems this isIPv6 variable is useless.
* We should remove it to prevent confusion */
- if (info->isIPv6) {
+ if (mIsIPv6) {
pRemoteRTCPAddr = (struct sockaddr *)&info->mRemoteRTCPAddr6;
sizeSockSt = sizeof(struct sockaddr_in6);
} else {
@@ -1215,12 +1295,20 @@
mTargetBitrate = targetBitrate;
}
+void ARTPConnection::setRtpSockOptEcn(int32_t sockOptEcn) {
+ mRtpSockOptEcn = sockOptEcn;
+}
+
+void ARTPConnection::setIsIPv6(const char *localIp) {
+ mIsIPv6 = (strchr(localIp, ':') != nullptr);
+}
+
void ARTPConnection::checkRxBitrate(int64_t nowUs) {
if (mLastBitrateReportTimeUs <= 0) {
mCumulativeBytes = 0;
mLastBitrateReportTimeUs = nowUs;
}
- else if (mLastEarlyNotifyTimeUs + 100000ll <= nowUs) {
+ else if (mLastEarlyNotifyTimeUs + kMinOneSecondNotifyDelayUs <= nowUs) {
int32_t timeDiff = (nowUs - mLastBitrateReportTimeUs) / 1000000ll;
int32_t bitrate = mCumulativeBytes * 8 / timeDiff;
mLastEarlyNotifyTimeUs = nowUs;
diff --git a/media/libstagefright/rtsp/ARTPSource.cpp b/media/libstagefright/rtsp/ARTPSource.cpp
index 717d8af..c5b0a1e 100644
--- a/media/libstagefright/rtsp/ARTPSource.cpp
+++ b/media/libstagefright/rtsp/ARTPSource.cpp
@@ -264,12 +264,12 @@
bool ARTPSource::queuePacket(const sp<ABuffer> &buffer) {
int64_t nowUs = ALooper::GetNowUs();
+ int64_t rtpTime = 0;
uint32_t seqNum = (uint32_t)buffer->int32Data();
- int32_t ssrc = 0, rtpTime = 0;
+ int32_t ssrc = 0;
buffer->meta()->findInt32("ssrc", &ssrc);
CHECK(buffer->meta()->findInt32("rtp-time", (int32_t *)&rtpTime));
- mLatestRtpTime = rtpTime;
if (mNumBuffersReceived++ == 0 && mFirstSysTime == 0) {
mFirstSysTime = nowUs;
@@ -277,7 +277,7 @@
mLastSysAnchorTimeUpdatedUs = nowUs;
mHighestSeqNumber = seqNum;
mBaseSeqNumber = seqNum;
- mFirstRtpTime = rtpTime;
+ mFirstRtpTime = (uint32_t)rtpTime;
mFirstSsrc = ssrc;
ALOGD("first-rtp arrived: first-rtp-time=%u, sys-time=%lld, seq-num=%u, ssrc=%d",
mFirstRtpTime, (long long)mFirstSysTime, mHighestSeqNumber, mFirstSsrc);
@@ -352,6 +352,18 @@
mQueue.insert(it, buffer);
+ /**
+ * RFC3550 calculates the interarrival jitter time for 'ALL packets'.
+ * We calculate anothor jitter only for all 'Head NAL units'
+ */
+ ALOGV("<======== Insert %d", seqNum);
+ rtpTime = mAssembler->findRTPTime(mFirstRtpTime, buffer);
+ if (rtpTime != mLatestRtpTime) {
+ mJitterCalc->putBaseData(rtpTime, nowUs);
+ }
+ mJitterCalc->putInterArrivalData(rtpTime, nowUs);
+ mLatestRtpTime = rtpTime;
+
return true;
}
@@ -680,14 +692,6 @@
mStaticJbTimeMs = jbTimeMs;
}
-void ARTPSource::putBaseJitterData(uint32_t timeStamp, int64_t arrivalTime) {
- mJitterCalc->putBaseData(timeStamp, arrivalTime);
-}
-
-void ARTPSource::putInterArrivalJitterData(uint32_t timeStamp, int64_t arrivalTime) {
- mJitterCalc->putInterArrivalData(timeStamp, arrivalTime);
-}
-
void ARTPSource::setJbTimer(const sp<AMessage> timer) {
mJbTimer = timer;
}
diff --git a/media/libstagefright/rtsp/ARTPWriter.cpp b/media/libstagefright/rtsp/ARTPWriter.cpp
index 8990f0c..41f2d67 100644
--- a/media/libstagefright/rtsp/ARTPWriter.cpp
+++ b/media/libstagefright/rtsp/ARTPWriter.cpp
@@ -255,9 +255,34 @@
if (params->findInt32(kKeyRtpCvoDegrees, &rtpCVODegrees))
mRTPCVODegrees = rtpCVODegrees;
+ bool needToSetSockOpt = false;
int32_t dscp = 0;
- if (params->findInt32(kKeyRtpDscp, &dscp))
- updateSocketDscp(dscp);
+ if (params->findInt32(kKeyRtpDscp, &dscp)) {
+ mRtpLayer3Dscp = dscp << 2;
+ needToSetSockOpt = true;
+ }
+
+ int32_t ecn = 0;
+ if (params->findInt32(kKeyRtpEcn, &ecn)) {
+ /*
+ * @ecn, possible value for ECN.
+ * +-----+-----+
+ * | ECN FIELD |
+ * +-----+-----+
+ * ECT CE [Obsolete] RFC 2481 names for the ECN bits.
+ * 0 0 Not-ECT
+ * 0 1 ECT (ECN-Capable Transport) (1)
+ * 1 0 ECT (ECN-Capable Transport) (0)
+ * 1 1 CE (Congestion Experienced)
+ *
+ */
+ mRtpSockOptEcn = ecn;
+ needToSetSockOpt = true;
+ }
+
+ if (needToSetSockOpt) {
+ updateSocketOpt();
+ }
int64_t sockNetwork = 0;
if (params->findInt64(kKeySocketNetwork, &sockNetwork))
@@ -1438,18 +1463,29 @@
mPayloadType = payloadType;
}
-void ARTPWriter::updateSocketDscp(int32_t dscp) {
- mRtpLayer3Dscp = dscp << 2;
+/*
+ * This function will set socket option in IP header
+ */
+void ARTPWriter::updateSocketOpt() {
+ /*
+ * 0 1 2 3 4 5 6 7
+ * +-----+-----+-----+-----+-----+-----+-----+-----+
+ * | DS FIELD, DSCP | ECN FIELD |
+ * +-----+-----+-----+-----+-----+-----+-----+-----+
+ */
+ int sockOpt = mRtpLayer3Dscp ^ mRtpSockOptEcn;
+ ALOGD("Update socket opt with sockopt=%d, mRtpLayer3Dscp=%d, mRtpSockOptEcn=%d",
+ sockOpt, mRtpLayer3Dscp, mRtpSockOptEcn);
- /* mRtpLayer3Dscp will be mapped to WMM(Wifi) as per operator's requirement */
- if (setsockopt(mRTPSocket, IPPROTO_IP, IP_TOS,
- (int *)&mRtpLayer3Dscp, sizeof(mRtpLayer3Dscp)) < 0) {
- ALOGE("failed to set dscp on rtpsock. err=%s", strerror(errno));
+ /* sockOpt will be used to set socket option in IP header */
+ if (setsockopt(mRTPSocket, mIsIPv6 ? IPPROTO_IPV6 : IPPROTO_IP, mIsIPv6 ? IPV6_TCLASS : IP_TOS,
+ (int *)&sockOpt, sizeof(sockOpt)) < 0) {
+ ALOGE("failed to set sockopt on rtpsock. err=%s", strerror(errno));
} else {
- ALOGD("successfully set dscp on rtpsock. opt=%d", mRtpLayer3Dscp);
- setsockopt(mRTCPSocket, IPPROTO_IP, IP_TOS,
- (int *)&mRtpLayer3Dscp, sizeof(mRtpLayer3Dscp));
- ALOGD("successfully set dscp on rtcpsock. opt=%d", mRtpLayer3Dscp);
+ ALOGD("successfully set sockopt. opt=%d", sockOpt);
+ setsockopt(mRTCPSocket, mIsIPv6 ? IPPROTO_IPV6 : IPPROTO_IP, mIsIPv6 ? IPV6_TCLASS : IP_TOS,
+ (int *)&sockOpt, sizeof(sockOpt));
+ ALOGD("successfully set sockopt rtcpsock. opt=%d", sockOpt);
}
}
diff --git a/media/libstagefright/rtsp/include/media/stagefright/rtsp/AAVCAssembler.h b/media/libstagefright/rtsp/include/media/stagefright/rtsp/AAVCAssembler.h
index 2f8b8ba..70ce388 100644
--- a/media/libstagefright/rtsp/include/media/stagefright/rtsp/AAVCAssembler.h
+++ b/media/libstagefright/rtsp/include/media/stagefright/rtsp/AAVCAssembler.h
@@ -50,7 +50,6 @@
bool mFirstIFrameProvided;
int32_t mLastCvo;
uint64_t mLastIFrameProvidedAtMs;
- int64_t mLastRtpTimeJitterDataUs;
int32_t mWidth;
int32_t mHeight;
List<sp<ABuffer> > mNALUnits;
diff --git a/media/libstagefright/rtsp/include/media/stagefright/rtsp/AHEVCAssembler.h b/media/libstagefright/rtsp/include/media/stagefright/rtsp/AHEVCAssembler.h
index 9575d8cb..ed3f1ae 100644
--- a/media/libstagefright/rtsp/include/media/stagefright/rtsp/AHEVCAssembler.h
+++ b/media/libstagefright/rtsp/include/media/stagefright/rtsp/AHEVCAssembler.h
@@ -51,7 +51,6 @@
bool mFirstIFrameProvided;
int32_t mLastCvo;
uint64_t mLastIFrameProvidedAtMs;
- int64_t mLastRtpTimeJitterDataUs;
int32_t mWidth;
int32_t mHeight;
List<sp<ABuffer> > mNALUnits;
diff --git a/media/libstagefright/rtsp/include/media/stagefright/rtsp/ARTPAssembler.h b/media/libstagefright/rtsp/include/media/stagefright/rtsp/ARTPAssembler.h
index 39161b6..8f87642 100644
--- a/media/libstagefright/rtsp/include/media/stagefright/rtsp/ARTPAssembler.h
+++ b/media/libstagefright/rtsp/include/media/stagefright/rtsp/ARTPAssembler.h
@@ -44,6 +44,13 @@
virtual void onByeReceived() = 0;
virtual bool initCheck() { return true; }
+ // Utility functions
+ inline int64_t findRTPTime(const uint32_t& firstRTPTime, const sp<ABuffer>& buffer);
+ inline int64_t MsToRtp(int64_t ms, int64_t clockRate);
+ inline int64_t RtpToMs(int64_t rtp, int64_t clockRate);
+ inline void printNowTimeMs(int64_t start, int64_t now, int64_t play);
+ inline void printRTPTime(int64_t rtp, int64_t play, int64_t exp, bool isExp);
+
protected:
virtual AssemblyStatus assembleMore(const sp<ARTPSource> &source) = 0;
virtual void packetLost() = 0;
@@ -64,13 +71,6 @@
bool mShowQueue;
int32_t mShowQueueCnt;
- // Utility functions
- inline int64_t findRTPTime(const uint32_t& firstRTPTime, const sp<ABuffer>& buffer);
- inline int64_t MsToRtp(int64_t ms, int64_t clockRate);
- inline int64_t RtpToMs(int64_t rtp, int64_t clockRate);
- inline void printNowTimeMs(int64_t start, int64_t now, int64_t play);
- inline void printRTPTime(int64_t rtp, int64_t play, int64_t exp, bool isExp);
-
private:
int64_t mFirstFailureTimeUs;
diff --git a/media/libstagefright/rtsp/include/media/stagefright/rtsp/ARTPConnection.h b/media/libstagefright/rtsp/include/media/stagefright/rtsp/ARTPConnection.h
index 73d2866..250de71 100644
--- a/media/libstagefright/rtsp/include/media/stagefright/rtsp/ARTPConnection.h
+++ b/media/libstagefright/rtsp/include/media/stagefright/rtsp/ARTPConnection.h
@@ -20,6 +20,7 @@
#include <media/stagefright/foundation/AHandler.h>
#include <utils/List.h>
+#include <sys/socket.h>
namespace android {
@@ -48,6 +49,8 @@
void setSelfID(const uint32_t selfID);
void setStaticJitterTimeMs(const uint32_t jbTimeMs);
void setTargetBitrate(int32_t targetBitrate);
+ void setRtpSockOptEcn(int32_t sockOptEcn);
+ void setIsIPv6(const char *localIp);
// Creates a pair of UDP datagram sockets bound to adjacent ports
// (the rtpSocket is bound to an even port, the rtcpSocket to the
@@ -60,7 +63,8 @@
static void MakeRTPSocketPair(
int *rtpSocket, int *rtcpSocket,
const char *localIp, const char *remoteIp,
- unsigned localPort, unsigned remotePort, int64_t socketNetwork = 0);
+ unsigned localPort, unsigned remotePort, int64_t socketNetwork = 0,
+ int32_t sockOptEcn = 0);
protected:
virtual ~ARTPConnection();
@@ -77,6 +81,7 @@
};
static const int64_t kSelectTimeoutUs;
+ static const int64_t kMinOneSecondNotifyDelayUs;
uint32_t mFlags;
@@ -87,9 +92,12 @@
int64_t mLastReceiverReportTimeUs;
int64_t mLastBitrateReportTimeUs;
int64_t mLastEarlyNotifyTimeUs;
+ int64_t mLastCongestionNotifyTimeUs;
int32_t mSelfID;
int32_t mTargetBitrate;
+ int32_t mRtpSockOptEcn;
+ bool mIsIPv6;
uint32_t mStaticJitterTimeMs;
@@ -103,6 +111,8 @@
void onInjectPacket(const sp<AMessage> &msg);
void onSendReceiverReports();
void checkRxBitrate(int64_t nowUs);
+ void notifyCongestionToUpperLayerIfNeeded(StreamInfo *s);
+ void handleIpHeadersIfReceived(StreamInfo *s, struct msghdr sMsg);
status_t receive(StreamInfo *info, bool receiveRTP);
ssize_t send(const StreamInfo *info, const sp<ABuffer> buffer);
diff --git a/media/libstagefright/rtsp/include/media/stagefright/rtsp/ARTPSource.h b/media/libstagefright/rtsp/include/media/stagefright/rtsp/ARTPSource.h
index e9b4942..7d1faf2 100644
--- a/media/libstagefright/rtsp/include/media/stagefright/rtsp/ARTPSource.h
+++ b/media/libstagefright/rtsp/include/media/stagefright/rtsp/ARTPSource.h
@@ -50,6 +50,7 @@
RTCP_FIRST_PACKET = 101,
RTP_QUALITY = 102,
RTP_QUALITY_EMC = 103,
+ RTP_QUALITY_CD = 104,
RTCP_SR = 200,
RTCP_RR = 201,
RTCP_TSFB = 205,
@@ -81,8 +82,6 @@
int32_t getBaseJitterTimeMs();
int32_t getInterArrivalJitterTimeMs();
void setStaticJitterTimeMs(const uint32_t jbTimeMs);
- void putBaseJitterData(uint32_t timeStamp, int64_t arrivalTime);
- void putInterArrivalJitterData(uint32_t timeStamp, int64_t arrivalTime);
void setJbTimer(const sp<AMessage> timer);
void setJbAlarmTime(int64_t nowTimeUs, int64_t alarmAfterUs);
diff --git a/media/libstagefright/rtsp/include/media/stagefright/rtsp/ARTPWriter.h b/media/libstagefright/rtsp/include/media/stagefright/rtsp/ARTPWriter.h
index 2982cf6..ecd29d0 100644
--- a/media/libstagefright/rtsp/include/media/stagefright/rtsp/ARTPWriter.h
+++ b/media/libstagefright/rtsp/include/media/stagefright/rtsp/ARTPWriter.h
@@ -50,7 +50,7 @@
virtual status_t pause();
void updateCVODegrees(int32_t cvoDegrees);
void updatePayloadType(int32_t payloadType);
- void updateSocketDscp(int32_t dscp);
+ void updateSocketOpt();
void updateSocketNetwork(int64_t socketNetwork);
uint32_t getSequenceNum();
virtual uint64_t getAccumulativeBytes() override;
@@ -98,6 +98,7 @@
struct sockaddr_in6 mRTPAddr6;
struct sockaddr_in6 mRTCPAddr6;
int32_t mRtpLayer3Dscp;
+ int32_t mRtpSockOptEcn;
net_handle_t mRTPSockNetwork;
AString mProfileLevel;
diff --git a/media/ndk/NdkImage.cpp b/media/ndk/NdkImage.cpp
index 12a0d53..c46a692 100644
--- a/media/ndk/NdkImage.cpp
+++ b/media/ndk/NdkImage.cpp
@@ -247,6 +247,13 @@
case HAL_PIXEL_FORMAT_YCrCb_420_SP:
*pixelStride = (planeIdx == 0) ? 1 : 2;
return AMEDIA_OK;
+ case HAL_PIXEL_FORMAT_YCBCR_P010:
+ if (mLockedBuffer->dataCb && mLockedBuffer->dataCr) {
+ *pixelStride = (planeIdx == 0) ? 2 : mLockedBuffer->chromaStep;
+ } else {
+ *pixelStride = (planeIdx == 0) ? 2 : 4;
+ }
+ return AMEDIA_OK;
case HAL_PIXEL_FORMAT_Y8:
*pixelStride = 1;
return AMEDIA_OK;
@@ -316,6 +323,13 @@
*rowStride = (planeIdx == 0) ? mLockedBuffer->stride
: ALIGN(mLockedBuffer->stride / 2, 16);
return AMEDIA_OK;
+ case HAL_PIXEL_FORMAT_YCBCR_P010:
+ if (mLockedBuffer->dataCb && mLockedBuffer->dataCr) {
+ *rowStride = (planeIdx == 0) ? mLockedBuffer->stride : mLockedBuffer->chromaStride;
+ } else {
+ *rowStride = mLockedBuffer->stride * 2;
+ }
+ return AMEDIA_OK;
case HAL_PIXEL_FORMAT_RAW10:
case HAL_PIXEL_FORMAT_RAW12:
// RAW10 and RAW12 are used for 10-bit and 12-bit raw data, they are single plane
@@ -473,6 +487,47 @@
: (planeIdx == 1) ? cb : cr;
dataSize = (planeIdx == 0) ? ySize : cSize;
break;
+ case HAL_PIXEL_FORMAT_YCBCR_P010:
+ if (mLockedBuffer->height % 2 != 0) {
+ ALOGE("YCBCR_P010: height (%d) should be a multiple of 2", mLockedBuffer->height);
+ return AMEDIA_ERROR_UNKNOWN;
+ }
+
+ if (mLockedBuffer->width <= 0) {
+ ALOGE("YCBCR_P010: width (%d) should be a > 0", mLockedBuffer->width);
+ return AMEDIA_ERROR_UNKNOWN;
+ }
+
+ if (mLockedBuffer->height <= 0) {
+ ALOGE("YCBCR_P010: height (%d) should be a > 0", mLockedBuffer->height);
+ return AMEDIA_ERROR_UNKNOWN;
+ }
+
+ if (mLockedBuffer->dataCb && mLockedBuffer->dataCr) {
+ pData = (planeIdx == 0) ? mLockedBuffer->data :
+ (planeIdx == 1) ? mLockedBuffer->dataCb : mLockedBuffer->dataCr;
+ // only map until last pixel
+ if (planeIdx == 0) {
+ cStride = mLockedBuffer->stride;
+ dataSize = cStride * (mLockedBuffer->height - 1) + mLockedBuffer->width * 2;
+ } else {
+ bytesPerPixel = mLockedBuffer->chromaStep;
+ cStride = mLockedBuffer->chromaStride;
+ dataSize = cStride * (mLockedBuffer->height / 2 - 1) +
+ bytesPerPixel * (mLockedBuffer->width / 2);
+ }
+ break;
+ }
+
+ cStride = mLockedBuffer->stride * 2;
+ ySize = cStride * mLockedBuffer->height;
+ cSize = ySize / 2;
+ cb = mLockedBuffer->data + ySize;
+ cr = cb + 2;
+
+ pData = (planeIdx == 0) ? mLockedBuffer->data : (planeIdx == 1) ? cb : cr;
+ dataSize = (planeIdx == 0) ? ySize : cSize;
+ break;
case HAL_PIXEL_FORMAT_Y8:
// Single plane, 8bpp.
diff --git a/media/ndk/NdkImageReader.cpp b/media/ndk/NdkImageReader.cpp
index 1067e24..9270499 100644
--- a/media/ndk/NdkImageReader.cpp
+++ b/media/ndk/NdkImageReader.cpp
@@ -73,6 +73,7 @@
case AIMAGE_FORMAT_HEIC:
case AIMAGE_FORMAT_DEPTH_JPEG:
case AIMAGE_FORMAT_RAW_DEPTH10:
+ case HAL_PIXEL_FORMAT_YCBCR_P010:
return true;
case AIMAGE_FORMAT_PRIVATE:
// For private format, cpu usage is prohibited.
@@ -86,6 +87,7 @@
AImageReader::getNumPlanesForFormat(int32_t format) {
switch (format) {
case AIMAGE_FORMAT_YUV_420_888:
+ case HAL_PIXEL_FORMAT_YCBCR_P010:
return 3;
case AIMAGE_FORMAT_RGBA_8888:
case AIMAGE_FORMAT_RGBX_8888:
diff --git a/media/ndk/NdkMediaCodec.cpp b/media/ndk/NdkMediaCodec.cpp
index 38e422d..ed31c02 100644
--- a/media/ndk/NdkMediaCodec.cpp
+++ b/media/ndk/NdkMediaCodec.cpp
@@ -14,6 +14,7 @@
* limitations under the License.
*/
+#include <charconv>
#include <inttypes.h>
#include <mutex>
#include <set>
@@ -324,29 +325,61 @@
}
AMessage::Type type;
- int64_t mediaTimeUs, systemNano;
- size_t index = 0;
+ size_t n = data->countEntries();
- // TODO. This code has dependency with MediaCodec::CreateFramesRenderedMessage.
- for (size_t ix = 0; ix < data->countEntries(); ix++) {
- AString name = data->getEntryNameAt(ix, &type);
- if (name.startsWith(AStringPrintf("%zu-media-time-us", index).c_str())) {
- AMessage::ItemData data = msg->getEntryAt(index);
- data.find(&mediaTimeUs);
- } else if (name.startsWith(AStringPrintf("%zu-system-nano", index).c_str())) {
- AMessage::ItemData data = msg->getEntryAt(index);
- data.find(&systemNano);
+ thread_local std::vector<std::optional<int64_t>> mediaTimesInUs;
+ thread_local std::vector<std::optional<int64_t>> systemTimesInNs;
+ mediaTimesInUs.resize(n);
+ systemTimesInNs.resize(n);
+ std::fill_n(mediaTimesInUs.begin(), n, std::nullopt);
+ std::fill_n(systemTimesInNs.begin(), n, std::nullopt);
+ for (size_t i = 0; i < n; i++) {
+ AString name = data->getEntryNameAt(i, &type);
+ if (name.endsWith("-media-time-us")) {
+ int64_t mediaTimeUs;
+ AMessage::ItemData itemData = data->getEntryAt(i);
+ itemData.find(&mediaTimeUs);
- Mutex::Autolock _l(mCodec->mFrameRenderedCallbackLock);
- if (mCodec->mFrameRenderedCallback != NULL) {
+ int index = -1;
+ std::from_chars_result result = std::from_chars(
+ name.c_str(), name.c_str() + name.find("-"), index);
+ if (result.ec == std::errc() && 0 <= index && index < n) {
+ mediaTimesInUs[index] = mediaTimeUs;
+ } else {
+ std::error_code ec = std::make_error_code(result.ec);
+ ALOGE("Unexpected media time index: #%d with value %lldus (err=%d %s)",
+ index, (long long)mediaTimeUs, ec.value(), ec.message().c_str());
+ }
+ } else if (name.endsWith("-system-nano")) {
+ int64_t systemNano;
+ AMessage::ItemData itemData = data->getEntryAt(i);
+ itemData.find(&systemNano);
+
+ int index = -1;
+ std::from_chars_result result = std::from_chars(
+ name.c_str(), name.c_str() + name.find("-"), index);
+ if (result.ec == std::errc() && 0 <= index && index < n) {
+ systemTimesInNs[index] = systemNano;
+ } else {
+ std::error_code ec = std::make_error_code(result.ec);
+ ALOGE("Unexpected system time index: #%d with value %lldns (err=%d %s)",
+ index, (long long)systemNano, ec.value(), ec.message().c_str());
+ }
+ }
+ }
+
+ Mutex::Autolock _l(mCodec->mFrameRenderedCallbackLock);
+ if (mCodec->mFrameRenderedCallback != NULL) {
+ for (size_t i = 0; i < n; ++i) {
+ if (mediaTimesInUs[i] && systemTimesInNs[i]) {
mCodec->mFrameRenderedCallback(
mCodec,
mCodec->mFrameRenderedCallbackUserData,
- mediaTimeUs,
- systemNano);
+ mediaTimesInUs[i].value(),
+ systemTimesInNs[i].value());
+ } else {
+ break;
}
-
- index++;
}
}
break;
@@ -529,22 +562,31 @@
AMediaCodecOnAsyncNotifyCallback callback,
void *userdata) {
- Mutex::Autolock _l(mData->mAsyncCallbackLock);
-
- if (mData->mAsyncNotify == NULL) {
- mData->mAsyncNotify = new AMessage(kWhatAsyncNotify, mData->mHandler);
+ {
+ Mutex::Autolock _l(mData->mAsyncCallbackLock);
+ if (mData->mAsyncNotify == NULL) {
+ mData->mAsyncNotify = new AMessage(kWhatAsyncNotify, mData->mHandler);
+ }
+ // we set this ahead so that we can be ready
+ // to receive callbacks as soon as the next call is a
+ // success.
+ mData->mAsyncCallback = callback;
+ mData->mAsyncCallbackUserData = userdata;
}
// always call, codec may have been reset/re-configured since last call.
status_t err = mData->mCodec->setCallback(mData->mAsyncNotify);
if (err != OK) {
+ {
+ //The setup gone wrong. clean up the pointers.
+ Mutex::Autolock _l(mData->mAsyncCallbackLock);
+ mData->mAsyncCallback = {};
+ mData->mAsyncCallbackUserData = nullptr;
+ }
ALOGE("setAsyncNotifyCallback: err(%d), failed to set async callback", err);
return translate_error(err);
}
- mData->mAsyncCallback = callback;
- mData->mAsyncCallbackUserData = userdata;
-
return AMEDIA_OK;
}
diff --git a/media/ndk/NdkMediaDrm.cpp b/media/ndk/NdkMediaDrm.cpp
index 59c1103..f4674de 100644
--- a/media/ndk/NdkMediaDrm.cpp
+++ b/media/ndk/NdkMediaDrm.cpp
@@ -183,7 +183,7 @@
AMediaDrmSessionId asid = {sessionId.data(), sessionId.size()};
int32_t dataSize = data.size();
const uint8_t *dataPtr = data.data();
- if (dataSize > 0) {
+ if (dataSize >= 0) {
(*mEventListener)(mObj, &asid, ndkEventType, 0, dataPtr, dataSize);
} else {
ALOGE("invalid event data size=%d", dataSize);
diff --git a/media/utils/MethodStatistics.cpp b/media/utils/MethodStatistics.cpp
index b179b20..086757b 100644
--- a/media/utils/MethodStatistics.cpp
+++ b/media/utils/MethodStatistics.cpp
@@ -22,7 +22,8 @@
std::shared_ptr<std::vector<std::string>>
getStatisticsClassesForModule(std::string_view moduleName) {
- static const std::map<std::string, std::shared_ptr<std::vector<std::string>>> m {
+ static const std::map<std::string, std::shared_ptr<std::vector<std::string>>,
+ std::less<> /* transparent comparator */> m {
{
METHOD_STATISTICS_MODULE_NAME_AUDIO_HIDL,
std::shared_ptr<std::vector<std::string>>(
@@ -34,13 +35,14 @@
})
},
};
- auto it = m.find({moduleName.begin(), moduleName.end()});
+ auto it = m.find(moduleName);
if (it == m.end()) return {};
return it->second;
}
static void addClassesToMap(const std::shared_ptr<std::vector<std::string>> &classNames,
- std::map<std::string, std::shared_ptr<MethodStatistics<std::string>>> &map) {
+ std::map<std::string, std::shared_ptr<MethodStatistics<std::string>>,
+ std::less<> /* transparent comparator */> &map) {
if (classNames) {
for (const auto& className : *classNames) {
map.emplace(className, std::make_shared<MethodStatistics<std::string>>());
@@ -51,17 +53,18 @@
// singleton statistics for DeviceHalHidl StreamOutHalHidl StreamInHalHidl
std::shared_ptr<MethodStatistics<std::string>>
getStatisticsForClass(std::string_view className) {
- static const std::map<std::string, std::shared_ptr<MethodStatistics<std::string>>> m =
+ static const std::map<std::string, std::shared_ptr<MethodStatistics<std::string>>,
+ std::less<> /* transparent comparator */> m =
// copy elided initialization of map m.
[](){
- std::map<std::string, std::shared_ptr<MethodStatistics<std::string>>> m;
+ std::map<std::string, std::shared_ptr<MethodStatistics<std::string>>, std::less<>> m;
addClassesToMap(
getStatisticsClassesForModule(METHOD_STATISTICS_MODULE_NAME_AUDIO_HIDL),
m);
return m;
}();
- auto it = m.find({className.begin(), className.end()});
+ auto it = m.find(className);
if (it == m.end()) return {};
return it->second;
}
diff --git a/media/utils/SchedulingPolicyService.cpp b/media/utils/SchedulingPolicyService.cpp
index ad38862..6e515ff 100644
--- a/media/utils/SchedulingPolicyService.cpp
+++ b/media/utils/SchedulingPolicyService.cpp
@@ -18,6 +18,7 @@
//#define LOG_NDEBUG 0
#include <binder/IServiceManager.h>
+#include <cutils/properties.h>
#include <utils/Mutex.h>
#include "ISchedulingPolicyService.h"
#include "mediautils/SchedulingPolicyService.h"
@@ -86,4 +87,25 @@
return ret;
}
+int requestSpatializerPriority(pid_t pid, pid_t tid) {
+ if (pid == -1 || tid == -1) return BAD_VALUE;
+
+ // update priority to RT if specified.
+ constexpr int32_t kRTPriorityMin = 1;
+ constexpr int32_t kRTPriorityMax = 3;
+ const int32_t priorityBoost =
+ property_get_int32("audio.spatializer.priority", kRTPriorityMin);
+ if (priorityBoost >= kRTPriorityMin && priorityBoost <= kRTPriorityMax) {
+ const status_t status = requestPriority(
+ pid, tid, priorityBoost, false /* isForApp */, true /*asynchronous*/);
+ if (status != OK) {
+ ALOGW("%s: Cannot request spatializer priority boost %d, status:%d",
+ __func__, priorityBoost, status);
+ return status < 0 ? status : UNKNOWN_ERROR;
+ }
+ return priorityBoost;
+ }
+ return 0; // no boost requested
+}
+
} // namespace android
diff --git a/media/utils/TimeCheck.cpp b/media/utils/TimeCheck.cpp
index 0848eb3..6823f4f 100644
--- a/media/utils/TimeCheck.cpp
+++ b/media/utils/TimeCheck.cpp
@@ -21,6 +21,7 @@
#include <android-base/logging.h>
#include <audio_utils/clock.h>
#include <mediautils/EventLog.h>
+#include <mediautils/FixedString.h>
#include <mediautils/MethodStatistics.h>
#include <mediautils/TimeCheck.h>
#include <utils/Log.h>
@@ -138,22 +139,24 @@
return getTimeCheckThread().toString();
}
-TimeCheck::TimeCheck(std::string tag, OnTimerFunc&& onTimer, uint32_t timeoutMs,
- bool crashOnTimeout)
- : mTimeCheckHandler(new TimeCheckHandler{
- std::move(tag), std::move(onTimer), crashOnTimeout,
- std::chrono::system_clock::now(), gettid()})
- , mTimerHandle(timeoutMs == 0
+TimeCheck::TimeCheck(std::string_view tag, OnTimerFunc&& onTimer, Duration requestedTimeoutDuration,
+ Duration secondChanceDuration, bool crashOnTimeout)
+ : mTimeCheckHandler{ std::make_shared<TimeCheckHandler>(
+ tag, std::move(onTimer), crashOnTimeout, requestedTimeoutDuration,
+ secondChanceDuration, std::chrono::system_clock::now(), gettid()) }
+ , mTimerHandle(requestedTimeoutDuration.count() == 0
+ /* for TimeCheck we don't consider a non-zero secondChanceDuration here */
? getTimeCheckThread().trackTask(mTimeCheckHandler->tag)
: getTimeCheckThread().scheduleTask(
mTimeCheckHandler->tag,
// Pass in all the arguments by value to this task for safety.
// The thread could call the callback before the constructor is finished.
// The destructor is not blocked on callback.
- [ timeCheckHandler = mTimeCheckHandler ] {
- timeCheckHandler->onTimeout();
+ [ timeCheckHandler = mTimeCheckHandler ](TimerThread::Handle timerHandle) {
+ timeCheckHandler->onTimeout(timerHandle);
},
- std::chrono::milliseconds(timeoutMs))) {}
+ requestedTimeoutDuration,
+ secondChanceDuration)) {}
TimeCheck::~TimeCheck() {
if (mTimeCheckHandler) {
@@ -161,23 +164,77 @@
}
}
+/* static */
+std::string TimeCheck::analyzeTimeouts(
+ float requestedTimeoutMs, float elapsedSteadyMs, float elapsedSystemMs) {
+ // Track any OS clock issues with suspend.
+ // It is possible that the elapsedSystemMs is much greater than elapsedSteadyMs if
+ // a suspend occurs; however, we always expect the timeout ms should always be slightly
+ // less than the elapsed steady ms regardless of whether a suspend occurs or not.
+
+ std::string s("Timeout ms ");
+ s.append(std::to_string(requestedTimeoutMs))
+ .append(" elapsed steady ms ").append(std::to_string(elapsedSteadyMs))
+ .append(" elapsed system ms ").append(std::to_string(elapsedSystemMs));
+
+ // Is there something unusual?
+ static constexpr float TOLERANCE_CONTEXT_SWITCH_MS = 200.f;
+
+ if (requestedTimeoutMs > elapsedSteadyMs || requestedTimeoutMs > elapsedSystemMs) {
+ s.append("\nError: early expiration - "
+ "requestedTimeoutMs should be less than elapsed time");
+ }
+
+ if (elapsedSteadyMs > elapsedSystemMs + TOLERANCE_CONTEXT_SWITCH_MS) {
+ s.append("\nWarning: steady time should not advance faster than system time");
+ }
+
+ // This has been found in suspend stress testing.
+ if (elapsedSteadyMs > requestedTimeoutMs + TOLERANCE_CONTEXT_SWITCH_MS) {
+ s.append("\nWarning: steady time significantly exceeds timeout "
+ "- possible thread stall or aborted suspend");
+ }
+
+ // This has been found in suspend stress testing.
+ if (elapsedSystemMs > requestedTimeoutMs + TOLERANCE_CONTEXT_SWITCH_MS) {
+ s.append("\nInformation: system time significantly exceeds timeout "
+ "- possible suspend");
+ }
+ return s;
+}
+
+// To avoid any potential race conditions, the timer handle
+// (expiration = clock steady start + timeout) is passed into the callback.
void TimeCheck::TimeCheckHandler::onCancel(TimerThread::Handle timerHandle) const
{
if (TimeCheck::getTimeCheckThread().cancelTask(timerHandle) && onTimer) {
- const std::chrono::system_clock::time_point endTime = std::chrono::system_clock::now();
- onTimer(false /* timeout */,
- std::chrono::duration_cast<std::chrono::duration<float, std::milli>>(
- endTime - startTime).count());
+ const std::chrono::steady_clock::time_point endSteadyTime =
+ std::chrono::steady_clock::now();
+ const float elapsedSteadyMs = std::chrono::duration_cast<FloatMs>(
+ endSteadyTime - timerHandle + timeoutDuration).count();
+ // send the elapsed steady time for statistics.
+ onTimer(false /* timeout */, elapsedSteadyMs);
}
}
-void TimeCheck::TimeCheckHandler::onTimeout() const
+// To avoid any potential race conditions, the timer handle
+// (expiration = clock steady start + timeout) is passed into the callback.
+void TimeCheck::TimeCheckHandler::onTimeout(TimerThread::Handle timerHandle) const
{
- const std::chrono::system_clock::time_point endTime = std::chrono::system_clock::now();
+ const std::chrono::steady_clock::time_point endSteadyTime = std::chrono::steady_clock::now();
+ const std::chrono::system_clock::time_point endSystemTime = std::chrono::system_clock::now();
+ // timerHandle incorporates the timeout
+ const float elapsedSteadyMs = std::chrono::duration_cast<FloatMs>(
+ endSteadyTime - (timerHandle - timeoutDuration)).count();
+ const float elapsedSystemMs = std::chrono::duration_cast<FloatMs>(
+ endSystemTime - startSystemTime).count();
+ const float requestedTimeoutMs = std::chrono::duration_cast<FloatMs>(
+ timeoutDuration).count();
+ const float secondChanceMs = std::chrono::duration_cast<FloatMs>(
+ secondChanceDuration).count();
+
if (onTimer) {
- onTimer(true /* timeout */,
- std::chrono::duration_cast<std::chrono::duration<float, std::milli>>(
- endTime - startTime).count());
+ onTimer(true /* timeout */, elapsedSteadyMs);
}
if (!crashOnTimeout) return;
@@ -207,8 +264,10 @@
// Create abort message string - caution: this can be very large.
const std::string abortMessage = std::string("TimeCheck timeout for ")
.append(tag)
- .append(" scheduled ").append(formatTime(startTime))
+ .append(" scheduled ").append(formatTime(startSystemTime))
.append(" on thread ").append(std::to_string(tid)).append("\n")
+ .append(analyzeTimeouts(requestedTimeoutMs + secondChanceMs,
+ elapsedSteadyMs, elapsedSystemMs)).append("\n")
.append(halPids).append("\n")
.append(summary);
@@ -231,16 +290,16 @@
mediautils::getStatisticsForClass(className);
if (!statistics) return {}; // empty TimeCheck.
return mediautils::TimeCheck(
- std::string(className).append("::").append(methodName),
- [ clazz = std::string(className), method = std::string(methodName),
+ FixedString62(className).append("::").append(methodName),
+ [ safeMethodName = FixedString30(methodName),
stats = std::move(statistics) ]
(bool timeout, float elapsedMs) {
if (timeout) {
; // ignored, there is no timeout value.
} else {
- stats->event(method, elapsedMs);
+ stats->event(safeMethodName.asStringView(), elapsedMs);
}
- }, 0 /* timeoutMs */);
+ }, {} /* timeoutDuration */, {} /* secondChanceDuration */, false /* crashOnTimeout */);
}
} // namespace android::mediautils
diff --git a/media/utils/TimerThread-test.cpp b/media/utils/TimerThread-test.cpp
index 93cd64c..9452c07 100644
--- a/media/utils/TimerThread-test.cpp
+++ b/media/utils/TimerThread-test.cpp
@@ -33,10 +33,28 @@
return std::count(s.begin(), s.end(), c);
}
-TEST(TimerThread, Basic) {
+
+// Split msec time between timeout and second chance time
+// This tests expiration times weighted between timeout and the second chance time.
+#define DISTRIBUTE_TIMEOUT_SECONDCHANCE_MS_FRAC(msec, frac) \
+ std::chrono::milliseconds(int((msec) * (frac)) + 1), \
+ std::chrono::milliseconds(int((msec) * (1.f - (frac))))
+
+// The TimerThreadTest is parameterized on a fraction between 0.f and 1.f which
+// is how the total timeout time is split between the first timeout and the second chance time.
+//
+class TimerThreadTest : public ::testing::TestWithParam<float> {
+protected:
+
+static void testBasic() {
+ const auto frac = GetParam();
+
std::atomic<bool> taskRan = false;
TimerThread thread;
- thread.scheduleTask("Basic", [&taskRan] { taskRan = true; }, 100ms);
+ TimerThread::Handle handle =
+ thread.scheduleTask("Basic", [&taskRan](TimerThread::Handle handle __unused) {
+ taskRan = true; }, DISTRIBUTE_TIMEOUT_SECONDCHANCE_MS_FRAC(100, frac));
+ ASSERT_TRUE(TimerThread::isTimeoutHandle(handle));
std::this_thread::sleep_for(100ms - kJitter);
ASSERT_FALSE(taskRan);
std::this_thread::sleep_for(2 * kJitter);
@@ -44,11 +62,15 @@
ASSERT_EQ(1, countChars(thread.retiredToString(), REQUEST_START));
}
-TEST(TimerThread, Cancel) {
+static void testCancel() {
+ const auto frac = GetParam();
+
std::atomic<bool> taskRan = false;
TimerThread thread;
TimerThread::Handle handle =
- thread.scheduleTask("Cancel", [&taskRan] { taskRan = true; }, 100ms);
+ thread.scheduleTask("Cancel", [&taskRan](TimerThread::Handle handle __unused) {
+ taskRan = true; }, DISTRIBUTE_TIMEOUT_SECONDCHANCE_MS_FRAC(100, frac));
+ ASSERT_TRUE(TimerThread::isTimeoutHandle(handle));
std::this_thread::sleep_for(100ms - kJitter);
ASSERT_FALSE(taskRan);
ASSERT_TRUE(thread.cancelTask(handle));
@@ -57,88 +79,87 @@
ASSERT_EQ(1, countChars(thread.retiredToString(), REQUEST_START));
}
-TEST(TimerThread, CancelAfterRun) {
+static void testCancelAfterRun() {
+ const auto frac = GetParam();
+
std::atomic<bool> taskRan = false;
TimerThread thread;
TimerThread::Handle handle =
- thread.scheduleTask("CancelAfterRun", [&taskRan] { taskRan = true; }, 100ms);
+ thread.scheduleTask("CancelAfterRun",
+ [&taskRan](TimerThread::Handle handle __unused) {
+ taskRan = true; },
+ DISTRIBUTE_TIMEOUT_SECONDCHANCE_MS_FRAC(100, frac));
+ ASSERT_TRUE(TimerThread::isTimeoutHandle(handle));
std::this_thread::sleep_for(100ms + kJitter);
ASSERT_TRUE(taskRan);
ASSERT_FALSE(thread.cancelTask(handle));
ASSERT_EQ(1, countChars(thread.retiredToString(), REQUEST_START));
}
-TEST(TimerThread, MultipleTasks) {
+static void testMultipleTasks() {
+ const auto frac = GetParam();
+
std::array<std::atomic<bool>, 6> taskRan{};
TimerThread thread;
auto startTime = std::chrono::steady_clock::now();
- thread.scheduleTask("0", [&taskRan] { taskRan[0] = true; }, 300ms);
- thread.scheduleTask("1", [&taskRan] { taskRan[1] = true; }, 100ms);
- thread.scheduleTask("2", [&taskRan] { taskRan[2] = true; }, 200ms);
- thread.scheduleTask("3", [&taskRan] { taskRan[3] = true; }, 400ms);
- auto handle4 = thread.scheduleTask("4", [&taskRan] { taskRan[4] = true; }, 200ms);
- thread.scheduleTask("5", [&taskRan] { taskRan[5] = true; }, 200ms);
+ thread.scheduleTask("0", [&taskRan](TimerThread::Handle handle __unused) {
+ taskRan[0] = true; }, DISTRIBUTE_TIMEOUT_SECONDCHANCE_MS_FRAC(300, frac));
+ thread.scheduleTask("1", [&taskRan](TimerThread::Handle handle __unused) {
+ taskRan[1] = true; }, DISTRIBUTE_TIMEOUT_SECONDCHANCE_MS_FRAC(100, frac));
+ thread.scheduleTask("2", [&taskRan](TimerThread::Handle handle __unused) {
+ taskRan[2] = true; }, DISTRIBUTE_TIMEOUT_SECONDCHANCE_MS_FRAC(200, frac));
+ thread.scheduleTask("3", [&taskRan](TimerThread::Handle handle __unused) {
+ taskRan[3] = true; }, DISTRIBUTE_TIMEOUT_SECONDCHANCE_MS_FRAC(400, frac));
+ auto handle4 = thread.scheduleTask("4", [&taskRan](TimerThread::Handle handle __unused) {
+ taskRan[4] = true; }, DISTRIBUTE_TIMEOUT_SECONDCHANCE_MS_FRAC(200, frac));
+ thread.scheduleTask("5", [&taskRan](TimerThread::Handle handle __unused) {
+ taskRan[5] = true; }, DISTRIBUTE_TIMEOUT_SECONDCHANCE_MS_FRAC(200, frac));
// 6 tasks pending
ASSERT_EQ(6, countChars(thread.pendingToString(), REQUEST_START));
// 0 tasks completed
ASSERT_EQ(0, countChars(thread.retiredToString(), REQUEST_START));
+ // None of the tasks are expected to have finished at the start.
+ std::array<std::atomic<bool>, 6> expected{};
+
// Task 1 should trigger around 100ms.
std::this_thread::sleep_until(startTime + 100ms - kJitter);
- ASSERT_FALSE(taskRan[0]);
- ASSERT_FALSE(taskRan[1]);
- ASSERT_FALSE(taskRan[2]);
- ASSERT_FALSE(taskRan[3]);
- ASSERT_FALSE(taskRan[4]);
- ASSERT_FALSE(taskRan[5]);
+
+ ASSERT_EQ(expected, taskRan);
+
std::this_thread::sleep_until(startTime + 100ms + kJitter);
- ASSERT_FALSE(taskRan[0]);
- ASSERT_TRUE(taskRan[1]);
- ASSERT_FALSE(taskRan[2]);
- ASSERT_FALSE(taskRan[3]);
- ASSERT_FALSE(taskRan[4]);
- ASSERT_FALSE(taskRan[5]);
+
+ expected[1] = true;
+ ASSERT_EQ(expected, taskRan);
// Cancel task 4 before it gets a chance to run.
thread.cancelTask(handle4);
// Tasks 2 and 5 should trigger around 200ms.
std::this_thread::sleep_until(startTime + 200ms - kJitter);
- ASSERT_FALSE(taskRan[0]);
- ASSERT_TRUE(taskRan[1]);
- ASSERT_FALSE(taskRan[2]);
- ASSERT_FALSE(taskRan[3]);
- ASSERT_FALSE(taskRan[4]);
- ASSERT_FALSE(taskRan[5]);
+
+ ASSERT_EQ(expected, taskRan);
+
std::this_thread::sleep_until(startTime + 200ms + kJitter);
- ASSERT_FALSE(taskRan[0]);
- ASSERT_TRUE(taskRan[1]);
- ASSERT_TRUE(taskRan[2]);
- ASSERT_FALSE(taskRan[3]);
- ASSERT_FALSE(taskRan[4]);
- ASSERT_TRUE(taskRan[5]);
+
+ expected[2] = true;
+ expected[5] = true;
+ ASSERT_EQ(expected, taskRan);
// Task 0 should trigger around 300ms.
std::this_thread::sleep_until(startTime + 300ms - kJitter);
- ASSERT_FALSE(taskRan[0]);
- ASSERT_TRUE(taskRan[1]);
- ASSERT_TRUE(taskRan[2]);
- ASSERT_FALSE(taskRan[3]);
- ASSERT_FALSE(taskRan[4]);
- ASSERT_TRUE(taskRan[5]);
+
+ ASSERT_EQ(expected, taskRan);
std::this_thread::sleep_until(startTime + 300ms + kJitter);
- ASSERT_TRUE(taskRan[0]);
- ASSERT_TRUE(taskRan[1]);
- ASSERT_TRUE(taskRan[2]);
- ASSERT_FALSE(taskRan[3]);
- ASSERT_FALSE(taskRan[4]);
- ASSERT_TRUE(taskRan[5]);
+
+ expected[0] = true;
+ ASSERT_EQ(expected, taskRan);
// 1 task pending
ASSERT_EQ(1, countChars(thread.pendingToString(), REQUEST_START));
@@ -147,23 +168,16 @@
// Task 3 should trigger around 400ms.
std::this_thread::sleep_until(startTime + 400ms - kJitter);
- ASSERT_TRUE(taskRan[0]);
- ASSERT_TRUE(taskRan[1]);
- ASSERT_TRUE(taskRan[2]);
- ASSERT_FALSE(taskRan[3]);
- ASSERT_FALSE(taskRan[4]);
- ASSERT_TRUE(taskRan[5]);
+
+ ASSERT_EQ(expected, taskRan);
// 4 tasks ran and 1 cancelled
ASSERT_EQ(4 + 1, countChars(thread.retiredToString(), REQUEST_START));
std::this_thread::sleep_until(startTime + 400ms + kJitter);
- ASSERT_TRUE(taskRan[0]);
- ASSERT_TRUE(taskRan[1]);
- ASSERT_TRUE(taskRan[2]);
- ASSERT_TRUE(taskRan[3]);
- ASSERT_FALSE(taskRan[4]);
- ASSERT_TRUE(taskRan[5]);
+
+ expected[3] = true;
+ ASSERT_EQ(expected, taskRan);
// 0 tasks pending
ASSERT_EQ(0, countChars(thread.pendingToString(), REQUEST_START));
@@ -171,6 +185,30 @@
ASSERT_EQ(5 + 1, countChars(thread.retiredToString(), REQUEST_START));
}
+}; // class TimerThreadTest
+
+TEST_P(TimerThreadTest, Basic) {
+ testBasic();
+}
+
+TEST_P(TimerThreadTest, Cancel) {
+ testCancel();
+}
+
+TEST_P(TimerThreadTest, CancelAfterRun) {
+ testCancelAfterRun();
+}
+
+TEST_P(TimerThreadTest, MultipleTasks) {
+ testMultipleTasks();
+}
+
+INSTANTIATE_TEST_CASE_P(
+ TimerThread,
+ TimerThreadTest,
+ ::testing::Values(0.f, 0.5f, 1.f)
+ );
+
TEST(TimerThread, TrackedTasks) {
TimerThread thread;
@@ -178,6 +216,10 @@
auto handle1 = thread.trackTask("1");
auto handle2 = thread.trackTask("2");
+ ASSERT_TRUE(TimerThread::isNoTimeoutHandle(handle0));
+ ASSERT_TRUE(TimerThread::isNoTimeoutHandle(handle1));
+ ASSERT_TRUE(TimerThread::isNoTimeoutHandle(handle2));
+
// 3 tasks pending
ASSERT_EQ(3, countChars(thread.pendingToString(), REQUEST_START));
// 0 tasks retired
@@ -201,6 +243,7 @@
// Add another tracked task.
auto handle3 = thread.trackTask("3");
+ ASSERT_TRUE(TimerThread::isNoTimeoutHandle(handle3));
// 2 tasks pending
ASSERT_EQ(2, countChars(thread.pendingToString(), REQUEST_START));
diff --git a/media/utils/TimerThread.cpp b/media/utils/TimerThread.cpp
index 6de6b13..5e58a3d 100644
--- a/media/utils/TimerThread.cpp
+++ b/media/utils/TimerThread.cpp
@@ -23,30 +23,35 @@
#include <mediautils/MediaUtilsDelayed.h>
#include <mediautils/TimerThread.h>
+#include <utils/Log.h>
#include <utils/ThreadDefs.h>
+using namespace std::chrono_literals;
+
namespace android::mediautils {
extern std::string formatTime(std::chrono::system_clock::time_point t);
extern std::string_view timeSuffix(std::string_view time1, std::string_view time2);
TimerThread::Handle TimerThread::scheduleTask(
- std::string tag, std::function<void()>&& func, std::chrono::milliseconds timeout) {
+ std::string_view tag, TimerCallback&& func,
+ Duration timeoutDuration, Duration secondChanceDuration) {
const auto now = std::chrono::system_clock::now();
- std::shared_ptr<const Request> request{
- new Request{ now, now + timeout, gettid(), std::move(tag) }};
- return mMonitorThread.add(std::move(request), std::move(func), timeout);
+ auto request = std::make_shared<const Request>(now, now +
+ std::chrono::duration_cast<std::chrono::system_clock::duration>(timeoutDuration),
+ secondChanceDuration, gettid(), tag);
+ return mMonitorThread.add(std::move(request), std::move(func), timeoutDuration);
}
-TimerThread::Handle TimerThread::trackTask(std::string tag) {
+TimerThread::Handle TimerThread::trackTask(std::string_view tag) {
const auto now = std::chrono::system_clock::now();
- std::shared_ptr<const Request> request{
- new Request{ now, now, gettid(), std::move(tag) }};
+ auto request = std::make_shared<const Request>(now, now,
+ Duration{} /* secondChanceDuration */, gettid(), tag);
return mNoTimeoutMap.add(std::move(request));
}
bool TimerThread::cancelTask(Handle handle) {
- std::shared_ptr<const Request> request = mNoTimeoutMap.isValidHandle(handle) ?
+ std::shared_ptr<const Request> request = isNoTimeoutHandle(handle) ?
mNoTimeoutMap.remove(handle) : mMonitorThread.remove(handle);
if (!request) return false;
mRetiredQueue.add(std::move(request));
@@ -84,6 +89,8 @@
return std::string("now ")
.append(formatTime(std::chrono::system_clock::now()))
+ .append("\nsecondChanceCount ")
+ .append(std::to_string(mMonitorThread.getSecondChanceCount()))
.append(analysisSummary)
.append("\ntimeout [ ")
.append(requestsToString(timeoutRequests))
@@ -106,10 +113,10 @@
//
/* static */
bool TimerThread::isRequestFromHal(const std::shared_ptr<const Request>& request) {
- const size_t hidlPos = request->tag.find("Hidl");
+ const size_t hidlPos = request->tag.asStringView().find("Hidl");
if (hidlPos == std::string::npos) return false;
// should be a separator afterwards Hidl which indicates the string was in the class.
- const size_t separatorPos = request->tag.find("::", hidlPos);
+ const size_t separatorPos = request->tag.asStringView().find("::", hidlPos);
return separatorPos != std::string::npos;
}
@@ -127,12 +134,12 @@
std::vector<std::shared_ptr<const Request>> pendingExact;
std::vector<std::shared_ptr<const Request>> pendingPossible;
- // We look at pending requests that were scheduled no later than kDuration
+ // We look at pending requests that were scheduled no later than kPendingDuration
// after the timeout request. This prevents false matches with calls
// that naturally block for a short period of time
// such as HAL write() and read().
//
- auto constexpr kDuration = std::chrono::milliseconds(1000);
+ constexpr Duration kPendingDuration = 1000ms;
for (const auto& pending : pendingRequests) {
// If the pending tid is the same as timeout tid, problem identified.
if (pending->tid == timeout->tid) {
@@ -141,7 +148,7 @@
}
// if the pending tid is scheduled within time limit
- if (pending->scheduled - timeout->scheduled < kDuration) {
+ if (pending->scheduled - timeout->scheduled < kPendingDuration) {
pendingPossible.emplace_back(pending);
}
}
@@ -241,15 +248,11 @@
}
}
-bool TimerThread::NoTimeoutMap::isValidHandle(Handle handle) const {
- return handle > getIndexedHandle(mNoTimeoutRequests);
-}
-
TimerThread::Handle TimerThread::NoTimeoutMap::add(std::shared_ptr<const Request> request) {
std::lock_guard lg(mNTMutex);
// A unique handle is obtained by mNoTimeoutRequests.fetch_add(1),
// This need not be under a lock, but we do so anyhow.
- const Handle handle = getIndexedHandle(mNoTimeoutRequests++);
+ const Handle handle = getUniqueHandle_l();
mMap[handle] = request;
return handle;
}
@@ -271,16 +274,6 @@
}
}
-TimerThread::Handle TimerThread::MonitorThread::getUniqueHandle_l(
- std::chrono::milliseconds timeout) {
- // To avoid key collisions, advance by 1 tick until the key is unique.
- auto deadline = std::chrono::steady_clock::now() + timeout;
- for (; mMonitorRequests.find(deadline) != mMonitorRequests.end();
- deadline += std::chrono::steady_clock::duration(1))
- ;
- return deadline;
-}
-
TimerThread::MonitorThread::MonitorThread(RequestQueue& timeoutQueue)
: mTimeoutQueue(timeoutQueue)
, mThread([this] { threadFunc(); }) {
@@ -300,24 +293,78 @@
void TimerThread::MonitorThread::threadFunc() {
std::unique_lock _l(mMutex);
while (!mShouldExit) {
+ Handle nextDeadline = INVALID_HANDLE;
+ Handle now = INVALID_HANDLE;
if (!mMonitorRequests.empty()) {
- Handle nextDeadline = mMonitorRequests.begin()->first;
- if (nextDeadline < std::chrono::steady_clock::now()) {
+ nextDeadline = mMonitorRequests.begin()->first;
+ now = std::chrono::steady_clock::now();
+ if (nextDeadline < now) {
+ auto node = mMonitorRequests.extract(mMonitorRequests.begin());
// Deadline has expired, handle the request.
+ auto secondChanceDuration = node.mapped().first->secondChanceDuration;
+ if (secondChanceDuration.count() != 0) {
+ // We now apply the second chance duration to find the clock
+ // monotonic second deadline. The unique key is then the
+ // pair<second_deadline, first_deadline>.
+ //
+ // The second chance prevents a false timeout should there be
+ // any clock monotonic advancement during suspend.
+ auto newHandle = now + secondChanceDuration;
+ ALOGD("%s: TimeCheck second chance applied for %s",
+ __func__, node.mapped().first->tag.c_str()); // should be rare event.
+ mSecondChanceRequests.emplace_hint(mSecondChanceRequests.end(),
+ std::make_pair(newHandle, nextDeadline),
+ std::move(node.mapped()));
+ // increment second chance counter.
+ mSecondChanceCount.fetch_add(1 /* arg */, std::memory_order_relaxed);
+ } else {
+ {
+ _l.unlock();
+ // We add Request to retired queue early so that it can be dumped out.
+ mTimeoutQueue.add(std::move(node.mapped().first));
+ node.mapped().second(nextDeadline);
+ // Caution: we don't hold lock when we call TimerCallback,
+ // but this is the timeout case! We will crash soon,
+ // maybe before returning.
+ // anything left over is released here outside lock.
+ }
+ // reacquire the lock - if something was added, we loop immediately to check.
+ _l.lock();
+ }
+ // always process expiring monitor requests first.
+ continue;
+ }
+ }
+ // now process any second chance requests.
+ if (!mSecondChanceRequests.empty()) {
+ Handle secondDeadline = mSecondChanceRequests.begin()->first.first;
+ if (now == INVALID_HANDLE) now = std::chrono::steady_clock::now();
+ if (secondDeadline < now) {
+ auto node = mSecondChanceRequests.extract(mSecondChanceRequests.begin());
{
- auto node = mMonitorRequests.extract(mMonitorRequests.begin());
_l.unlock();
// We add Request to retired queue early so that it can be dumped out.
mTimeoutQueue.add(std::move(node.mapped().first));
- node.mapped().second(); // Caution: we don't hold lock here - but do we care?
- // this is the timeout case! We will crash soon,
- // maybe before returning.
- // anything left over is released here outside lock.
+ const Handle originalHandle = node.key().second;
+ node.mapped().second(originalHandle);
+ // Caution: we don't hold lock when we call TimerCallback.
+ // This is benign issue - we permit concurrent operations
+ // while in the callback to the MonitorQueue.
+ //
+ // Anything left over is released here outside lock.
}
// reacquire the lock - if something was added, we loop immediately to check.
_l.lock();
continue;
}
+ // update the deadline.
+ if (nextDeadline == INVALID_HANDLE) {
+ nextDeadline = secondDeadline;
+ } else {
+ nextDeadline = std::min(nextDeadline, secondDeadline);
+ }
+ }
+ if (nextDeadline != INVALID_HANDLE) {
mCond.wait_until(_l, nextDeadline);
} else {
mCond.wait(_l);
@@ -326,26 +373,39 @@
}
TimerThread::Handle TimerThread::MonitorThread::add(
- std::shared_ptr<const Request> request, std::function<void()>&& func,
- std::chrono::milliseconds timeout) {
+ std::shared_ptr<const Request> request, TimerCallback&& func, Duration timeout) {
std::lock_guard _l(mMutex);
const Handle handle = getUniqueHandle_l(timeout);
- mMonitorRequests.emplace(handle, std::make_pair(std::move(request), std::move(func)));
+ mMonitorRequests.emplace_hint(mMonitorRequests.end(),
+ handle, std::make_pair(std::move(request), std::move(func)));
mCond.notify_all();
return handle;
}
std::shared_ptr<const TimerThread::Request> TimerThread::MonitorThread::remove(Handle handle) {
+ std::pair<std::shared_ptr<const Request>, TimerCallback> data;
std::unique_lock ul(mMutex);
- const auto it = mMonitorRequests.find(handle);
- if (it == mMonitorRequests.end()) {
- return {};
+ if (const auto it = mMonitorRequests.find(handle);
+ it != mMonitorRequests.end()) {
+ data = std::move(it->second);
+ mMonitorRequests.erase(it);
+ ul.unlock(); // manually release lock here so func (data.second)
+ // is released outside of lock.
+ return data.first; // request
}
- std::shared_ptr<const TimerThread::Request> request = std::move(it->second.first);
- std::function<void()> func = std::move(it->second.second);
- mMonitorRequests.erase(it);
- ul.unlock(); // manually release lock here so func is released outside of lock.
- return request;
+
+ // this check is O(N), but since the second chance requests are ordered
+ // in terms of earliest expiration time, we would expect better than average results.
+ for (auto it = mSecondChanceRequests.begin(); it != mSecondChanceRequests.end(); ++it) {
+ if (it->first.second == handle) {
+ data = std::move(it->second);
+ mSecondChanceRequests.erase(it);
+ ul.unlock(); // manually release lock here so func (data.second)
+ // is released outside of lock.
+ return data.first; // request
+ }
+ }
+ return {};
}
void TimerThread::MonitorThread::copyRequests(
@@ -354,6 +414,13 @@
for (const auto &[deadline, monitorpair] : mMonitorRequests) {
requests.emplace_back(monitorpair.first);
}
+ // we combine the second map with the first map - this is
+ // everything that is pending on the monitor thread.
+ // The second map will be older than the first map so this
+ // is in order.
+ for (const auto &[deadline, monitorpair] : mSecondChanceRequests) {
+ requests.emplace_back(monitorpair.first);
+ }
}
} // namespace android::mediautils
diff --git a/media/utils/fuzzers/TimeCheckFuzz.cpp b/media/utils/fuzzers/TimeCheckFuzz.cpp
index 7966469..65b2885 100644
--- a/media/utils/fuzzers/TimeCheckFuzz.cpp
+++ b/media/utils/fuzzers/TimeCheckFuzz.cpp
@@ -48,7 +48,9 @@
std::string name = data_provider.ConsumeRandomLengthString(kMaxStringLen);
// 3. The constructor, which is fuzzed here:
- android::mediautils::TimeCheck timeCheck(name.c_str(), {} /* onTimer */, timeoutMs);
+ android::mediautils::TimeCheck timeCheck(name.c_str(), {} /* onTimer */,
+ std::chrono::milliseconds(timeoutMs),
+ {} /* secondChanceDuration */, true /* crashOnTimeout */);
// We will leave some buffer to avoid sleeping too long
uint8_t sleep_amount_ms = data_provider.ConsumeIntegralInRange<uint8_t>(0, timeoutMs / 2);
diff --git a/media/utils/include/mediautils/FixedString.h b/media/utils/include/mediautils/FixedString.h
new file mode 100644
index 0000000..047aa82
--- /dev/null
+++ b/media/utils/include/mediautils/FixedString.h
@@ -0,0 +1,282 @@
+/*
+ * Copyright (C) 2022 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#pragma once
+
+#include <algorithm>
+#include <string>
+#include <string_view>
+
+namespace android::mediautils {
+
+/*
+ * FixedString is a stack allocatable string buffer that supports
+ * simple appending of other strings and string_views.
+ *
+ * It is designed for no-malloc operation when std::string
+ * small buffer optimization is insufficient.
+ *
+ * To keep code small, use asStringView() for operations on this.
+ *
+ * Notes:
+ * 1) Appending beyond the internal buffer size results in truncation.
+ *
+ * Alternatives:
+ * 1) If you want a sharable copy-on-write string implementation,
+ * consider using the legacy android::String8().
+ * 2) Using std::string with a fixed stack allocator may suit your needs,
+ * but exception avoidance is tricky.
+ * 3) Using C++20 ranges https://en.cppreference.com/w/cpp/ranges if you don't
+ * need backing store. Be careful about allocation with ranges.
+ *
+ * Good small sizes are multiples of 16 minus 2, e.g. 14, 30, 46, 62.
+ *
+ * Implementation Notes:
+ * 1) No iterators or [] for FixedString - please convert to std::string_view.
+ * 2) For small N (e.g. less than 62), consider a change to always zero fill and
+ * potentially prevent always zero terminating (if one only does append).
+ *
+ * Possible arguments to create/append:
+ * 1) A FixedString.
+ * 2) A single char.
+ * 3) A char * pointer.
+ * 4) A std::string.
+ * 5) A std::string_view (or something convertible to it).
+ *
+ * Example:
+ *
+ * FixedString s1(std::string("a")); // ctor
+ * s1 << "bc" << 'd' << '\n'; // streaming append
+ * s1 += "hello"; // += append
+ * ASSERT_EQ(s1, "abcd\nhello");
+ */
+template <uint32_t N>
+struct FixedString
+{
+ // Find the best size type.
+ using strsize_t = std::conditional_t<(N > 255), uint32_t, uint8_t>;
+
+ // constructors
+ FixedString() { // override default
+ buffer_[0] = '\0';
+ }
+
+ FixedString(const FixedString& other) { // override default.
+ copyFrom<N>(other);
+ }
+
+ // The following constructor is not explicit to allow
+ // FixedString<8> s = "abcd";
+ template <typename ...Types>
+ FixedString(Types&&... args) {
+ append(std::forward<Types>(args)...);
+ }
+
+ // copy assign (copyFrom checks for equality and returns *this).
+ FixedString& operator=(const FixedString& other) { // override default.
+ return copyFrom<N>(other);
+ }
+
+ template <typename ...Types>
+ FixedString& operator=(Types&&... args) {
+ size_ = 0;
+ return append(std::forward<Types>(args)...);
+ }
+
+ // operator equals
+ bool operator==(const char *s) const {
+ return strncmp(c_str(), s, capacity() + 1) == 0;
+ }
+
+ bool operator==(std::string_view s) const {
+ return size() == s.size() && memcmp(data(), s.data(), size()) == 0;
+ }
+
+ // operator not-equals
+ template <typename T>
+ bool operator!=(const T& other) const {
+ return !operator==(other);
+ }
+
+ // operator +=
+ template <typename ...Types>
+ FixedString& operator+=(Types&&... args) {
+ return append(std::forward<Types>(args)...);
+ }
+
+ // conversion to std::string_view.
+ operator std::string_view() const {
+ return asStringView();
+ }
+
+ // basic observers
+ size_t buffer_offset() const { return offsetof(std::decay_t<decltype(*this)>, buffer_); }
+ static constexpr uint32_t capacity() { return N; }
+ uint32_t size() const { return size_; }
+ uint32_t remaining() const { return size_ >= N ? 0 : N - size_; }
+ bool empty() const { return size_ == 0; }
+ bool full() const { return size_ == N; } // when full, possible truncation risk.
+ char * data() { return buffer_; }
+ const char * data() const { return buffer_; }
+ const char * c_str() const { return buffer_; }
+
+ inline std::string_view asStringView() const {
+ return { buffer_, static_cast<size_t>(size_) };
+ }
+ inline std::string asString() const {
+ return { buffer_, static_cast<size_t>(size_) };
+ }
+
+ void clear() { size_ = 0; buffer_[0] = 0; }
+
+ // Implementation of append - using templates
+ // to guarantee precedence in the presence of ambiguity.
+ //
+ // Consider C++20 template overloading through constraints and concepts.
+ template <typename T>
+ FixedString& append(const T& t) {
+ using decayT = std::decay_t<T>;
+ if constexpr (is_specialization_v<decayT, FixedString>) {
+ // A FixedString<U>
+ if (size_ == 0) {
+ // optimization to erase everything.
+ return copyFrom(t);
+ } else {
+ return appendStringView({t.data(), t.size()});
+ }
+ } else if constexpr(std::is_same_v<decayT, char>) {
+ if (size_ < N) {
+ buffer_[size_++] = t;
+ buffer_[size_] = '\0';
+ }
+ return *this;
+ } else if constexpr(std::is_same_v<decayT, char *>) {
+ // Some char* ptr.
+ return appendString(t);
+ } else if constexpr (std::is_convertible_v<decayT, std::string_view>) {
+ // std::string_view, std::string, or some other convertible type.
+ return appendStringView(t);
+ } else /* constexpr */ {
+ static_assert(dependent_false_v<T>, "non-matching append type");
+ }
+ }
+
+ FixedString& appendStringView(std::string_view s) {
+ uint32_t total = std::min(static_cast<size_t>(N - size_), s.size());
+ memcpy(buffer_ + size_, s.data(), total);
+ size_ += total;
+ buffer_[size_] = '\0';
+ return *this;
+ }
+
+ FixedString& appendString(const char *s) {
+ // strncpy zero pads to the end,
+ // strlcpy returns total expected length,
+ // we don't have strncpy_s in Bionic,
+ // so we write our own here.
+ while (size_ < N && *s != '\0') {
+ buffer_[size_++] = *s++;
+ }
+ buffer_[size_] = '\0';
+ return *this;
+ }
+
+ // Copy initialize the struct.
+ // Note: We are POD but customize the copying for acceleration
+ // of moving small strings embedded in a large buffers.
+ template <uint32_t U>
+ FixedString& copyFrom(const FixedString<U>& other) {
+ if ((void*)this != (void*)&other) { // not a self-assignment
+ if (other.size() == 0) {
+ size_ = 0;
+ buffer_[0] = '\0';
+ return *this;
+ }
+ constexpr size_t kSizeToCopyWhole = 64;
+ if constexpr (N == U &&
+ sizeof(*this) == sizeof(other) &&
+ sizeof(*this) <= kSizeToCopyWhole) {
+ // As we have the same str size type, we can just
+ // memcpy with fixed size, which can be easily optimized.
+ memcpy(static_cast<void*>(this), static_cast<const void*>(&other), sizeof(*this));
+ return *this;
+ }
+ if constexpr (std::is_same_v<strsize_t, typename FixedString<U>::strsize_t>) {
+ constexpr size_t kAlign = 8; // align to a multiple of 8.
+ static_assert((kAlign & (kAlign - 1)) == 0); // power of 2.
+ // we check any alignment issues.
+ if (buffer_offset() == other.buffer_offset() && other.size() <= capacity()) {
+ // improve on standard POD copying by reducing size.
+ const size_t mincpy = buffer_offset() + other.size() + 1 /* nul */;
+ const size_t maxcpy = std::min(sizeof(*this), sizeof(other));
+ const size_t cpysize = std::min(mincpy + kAlign - 1 & ~(kAlign - 1), maxcpy);
+ memcpy(static_cast<void*>(this), static_cast<const void*>(&other), cpysize);
+ return *this;
+ }
+ }
+ size_ = std::min(other.size(), capacity());
+ memcpy(buffer_, other.data(), size_);
+ buffer_[size_] = '\0'; // zero terminate.
+ }
+ return *this;
+ }
+
+private:
+ // Template helper methods
+
+ template <typename Test, template <uint32_t> class Ref>
+ struct is_specialization : std::false_type {};
+
+ template <template <uint32_t> class Ref, uint32_t UU>
+ struct is_specialization<Ref<UU>, Ref>: std::true_type {};
+
+ template <typename Test, template <uint32_t> class Ref>
+ static inline constexpr bool is_specialization_v = is_specialization<Test, Ref>::value;
+
+ // For static assert(false) we need a template version to avoid early failure.
+ template <typename T>
+ static inline constexpr bool dependent_false_v = false;
+
+ // POD variables
+ strsize_t size_ = 0;
+ char buffer_[N + 1 /* allow zero termination */];
+};
+
+// Stream operator syntactic sugar.
+// Example:
+// s << 'b' << "c" << "d" << '\n';
+template <uint32_t N, typename ...Types>
+FixedString<N>& operator<<(FixedString<N>& fs, Types&&... args) {
+ return fs.append(std::forward<Types>(args)...);
+}
+
+// We do not use a default size for fixed string as changing
+// the default size would lead to different behavior - we want the
+// size to be explicitly known.
+
+// FixedString62 of 62 chars fits in one typical cache line.
+using FixedString62 = FixedString<62>;
+
+// Slightly smaller
+using FixedString30 = FixedString<30>;
+
+// Since we have added copy and assignment optimizations,
+// we are no longer trivially assignable and copyable.
+// But we check standard layout here to prevent inclusion of unacceptable members or virtuals.
+static_assert(std::is_standard_layout_v<FixedString62>);
+static_assert(std::is_standard_layout_v<FixedString30>);
+
+} // namespace android::mediautils
diff --git a/media/utils/include/mediautils/MethodStatistics.h b/media/utils/include/mediautils/MethodStatistics.h
index 700fbaa..6d7e990 100644
--- a/media/utils/include/mediautils/MethodStatistics.h
+++ b/media/utils/include/mediautils/MethodStatistics.h
@@ -55,15 +55,23 @@
/**
* Adds a method event, typically execution time in ms.
*/
- void event(Code code, FloatType executeMs) {
+ template <typename C>
+ void event(C&& code, FloatType executeMs) {
std::lock_guard lg(mLock);
- mStatisticsMap[code].add(executeMs);
+ auto it = mStatisticsMap.lower_bound(code);
+ if (it != mStatisticsMap.end() && it->first == code) {
+ it->second.add(executeMs);
+ } else {
+ // StatsType ctor takes an optional array of data for initialization.
+ FloatType dataArray[1] = { executeMs };
+ mStatisticsMap.emplace_hint(it, std::forward<C>(code), dataArray);
+ }
}
/**
* Returns the name for the method code.
*/
- std::string getMethodForCode(Code code) const {
+ std::string getMethodForCode(const Code& code) const {
auto it = mMethodMap.find(code);
return it == mMethodMap.end() ? std::to_string((int)code) : it->second;
}
@@ -71,7 +79,7 @@
/**
* Returns the number of times the method was invoked by event().
*/
- size_t getMethodCount(Code code) const {
+ size_t getMethodCount(const Code& code) const {
std::lock_guard lg(mLock);
auto it = mStatisticsMap.find(code);
return it == mStatisticsMap.end() ? 0 : it->second.getN();
@@ -80,7 +88,7 @@
/**
* Returns the statistics object for the method.
*/
- StatsType getStatistics(Code code) const {
+ StatsType getStatistics(const Code& code) const {
std::lock_guard lg(mLock);
auto it = mStatisticsMap.find(code);
return it == mStatisticsMap.end() ? StatsType{} : it->second;
@@ -107,9 +115,10 @@
}
private:
- const std::map<Code, std::string> mMethodMap;
+ // Note: we use a transparent comparator std::less<> for heterogeneous key lookup.
+ const std::map<Code, std::string, std::less<>> mMethodMap;
mutable std::mutex mLock;
- std::map<Code, StatsType> mStatisticsMap GUARDED_BY(mLock);
+ std::map<Code, StatsType, std::less<>> mStatisticsMap GUARDED_BY(mLock);
};
// Managed Statistics support.
diff --git a/media/utils/include/mediautils/SchedulingPolicyService.h b/media/utils/include/mediautils/SchedulingPolicyService.h
index 546cec5..af1fcd2 100644
--- a/media/utils/include/mediautils/SchedulingPolicyService.h
+++ b/media/utils/include/mediautils/SchedulingPolicyService.h
@@ -23,7 +23,7 @@
class IBinder;
// Request elevated priority for thread tid, whose thread group leader must be pid.
-// The priority parameter is currently restricted to either 1 or 2.
+// The priority parameter is currently restricted from 1 to 3.
// The asynchronous parameter should be 'true' to return immediately,
// after the request is enqueued but not necessarily executed.
// The default value 'false' means to return after request has been enqueued and executed.
@@ -37,6 +37,12 @@
// 'client' is ignored in this case.
int requestCpusetBoost(bool enable, const sp<IBinder> &client);
+// Audio: Request Spatializer RT priority for thread tid, whose thread group leader must be pid.
+// returns positive value if successful, the RT priority used
+// zero, if no RT priority selected
+// negative status code if RT priority unable to be set.
+int requestSpatializerPriority(pid_t pid, pid_t tid);
+
} // namespace android
#endif // _ANDROID_SCHEDULING_POLICY_SERVICE_H
diff --git a/media/utils/include/mediautils/TimeCheck.h b/media/utils/include/mediautils/TimeCheck.h
index ef03aef..bdb5337 100644
--- a/media/utils/include/mediautils/TimeCheck.h
+++ b/media/utils/include/mediautils/TimeCheck.h
@@ -16,6 +16,7 @@
#pragma once
+#include <chrono>
#include <vector>
#include <mediautils/TimerThread.h>
@@ -27,10 +28,33 @@
class TimeCheck {
public:
+
+ // Duration for TimeCheck is based on steady_clock, typically nanoseconds.
+ using Duration = std::chrono::steady_clock::duration;
+
+ // Duration for printing is in milliseconds, using float for additional precision.
+ using FloatMs = std::chrono::duration<float, std::milli>;
+
+ // OnTimerFunc is the callback function with 2 parameters.
+ // bool timeout (which is true when the TimeCheck object
+ // times out, false when the TimeCheck object is
+ // destroyed or leaves scope before the timer expires.)
+ // float elapsedMs (the elapsed time to this event).
using OnTimerFunc = std::function<void(bool /* timeout */, float /* elapsedMs */ )>;
// The default timeout is chosen to be less than system server watchdog timeout
- static constexpr uint32_t kDefaultTimeOutMs = 5000;
+ // Note: kDefaultTimeOutMs should be no less than 2 seconds, otherwise spurious timeouts
+ // may occur with system suspend.
+ static constexpr TimeCheck::Duration kDefaultTimeoutDuration = std::chrono::milliseconds(3000);
+
+ // Due to suspend abort not incrementing the monotonic clock,
+ // we allow another second chance timeout after the first timeout expires.
+ //
+ // The total timeout is therefore kDefaultTimeoutDuration + kDefaultSecondChanceDuration,
+ // and the result is more stable when the monotonic clock increments during suspend.
+ //
+ static constexpr TimeCheck::Duration kDefaultSecondChanceDuration =
+ std::chrono::milliseconds(2000);
/**
* TimeCheck is a RAII object which will notify a callback
@@ -44,24 +68,24 @@
* the deallocation.
*
* \param tag string associated with the TimeCheck object.
- * \param onTimer callback function with 2 parameters
- * bool timeout (which is true when the TimeCheck object
- * times out, false when the TimeCheck object is
- * destroyed or leaves scope before the timer expires.)
- * float elapsedMs (the elapsed time to this event).
+ * \param onTimer callback function with 2 parameters (described above in OnTimerFunc).
* The callback when timeout is true will be called on a different thread.
* This will cancel the callback on the destructor but is not guaranteed
* to block for callback completion if it is already in progress
* (for maximum concurrency and reduced deadlock potential), so use proper
* lifetime analysis (e.g. shared or weak pointers).
- * \param timeoutMs timeout in milliseconds.
+ * \param requestedTimeoutDuration timeout in milliseconds.
* A zero timeout means no timeout is set -
* the callback is called only when
* the TimeCheck object is destroyed or leaves scope.
+ * \param secondChanceDuration additional milliseconds to wait if the first timeout expires.
+ * This is used to prevent false timeouts if the steady (monotonic)
+ * clock advances on aborted suspend.
* \param crashOnTimeout true if the object issues an abort on timeout.
*/
- explicit TimeCheck(std::string tag, OnTimerFunc&& onTimer = {},
- uint32_t timeoutMs = kDefaultTimeOutMs, bool crashOnTimeout = true);
+ explicit TimeCheck(std::string_view tag, OnTimerFunc&& onTimer,
+ Duration requestedTimeoutDuration, Duration secondChanceDuration,
+ bool crashOnTimeout);
TimeCheck() = default;
// Remove copy constructors as there should only be one call to the destructor.
@@ -79,16 +103,36 @@
// The usage here is const safe.
class TimeCheckHandler {
public:
- const std::string tag;
+ template <typename S, typename F>
+ TimeCheckHandler(S&& _tag, F&& _onTimer, bool _crashOnTimeout,
+ Duration _timeoutDuration, Duration _secondChanceDuration,
+ std::chrono::system_clock::time_point _startSystemTime,
+ pid_t _tid)
+ : tag(std::forward<S>(_tag))
+ , onTimer(std::forward<F>(_onTimer))
+ , crashOnTimeout(_crashOnTimeout)
+ , timeoutDuration(_timeoutDuration)
+ , secondChanceDuration(_secondChanceDuration)
+ , startSystemTime(_startSystemTime)
+ , tid(_tid)
+ {}
+ const FixedString62 tag;
const OnTimerFunc onTimer;
const bool crashOnTimeout;
- const std::chrono::system_clock::time_point startTime;
+ const Duration timeoutDuration;
+ const Duration secondChanceDuration;
+ const std::chrono::system_clock::time_point startSystemTime;
const pid_t tid;
void onCancel(TimerThread::Handle handle) const;
- void onTimeout() const;
+ void onTimeout(TimerThread::Handle handle) const;
};
+ // Returns a string that represents the timeout vs elapsed time,
+ // and diagnostics if there are any potential issues.
+ static std::string analyzeTimeouts(
+ float timeoutMs, float elapsedSteadyMs, float elapsedSystemMs);
+
static TimerThread& getTimeCheckThread();
static void accessAudioHalPids(std::vector<pid_t>* pids, bool update);
diff --git a/media/utils/include/mediautils/TimerThread.h b/media/utils/include/mediautils/TimerThread.h
index ffee602..c76fa7d 100644
--- a/media/utils/include/mediautils/TimerThread.h
+++ b/media/utils/include/mediautils/TimerThread.h
@@ -27,6 +27,8 @@
#include <android-base/thread_annotations.h>
+#include <mediautils/FixedString.h>
+
namespace android::mediautils {
/**
@@ -34,19 +36,93 @@
*/
class TimerThread {
public:
- // A Handle is a time_point that serves as a unique key. It is ordered.
+ // A Handle is a time_point that serves as a unique key to access a queued
+ // request to the TimerThread.
using Handle = std::chrono::steady_clock::time_point;
+ // Duration is based on steady_clock (typically nanoseconds)
+ // vs the system_clock duration (typically microseconds).
+ using Duration = std::chrono::steady_clock::duration;
+
static inline constexpr Handle INVALID_HANDLE =
std::chrono::steady_clock::time_point::min();
+ // Handle implementation details:
+ // A Handle represents the timer expiration time based on std::chrono::steady_clock
+ // (clock monotonic). This Handle is computed as now() + timeout.
+ //
+ // The lsb of the Handle time_point is adjusted to indicate whether there is
+ // a timeout action (1) or not (0).
+ //
+
+ template <size_t COUNT>
+ static constexpr bool is_power_of_2_v = COUNT > 0 && (COUNT & (COUNT - 1)) == 0;
+
+ template <size_t COUNT>
+ static constexpr size_t mask_from_count_v = COUNT - 1;
+
+ static constexpr size_t HANDLE_TYPES = 2;
+ // HANDLE_TYPES must be a power of 2.
+ static_assert(is_power_of_2_v<HANDLE_TYPES>);
+
+ // The handle types
+ enum class HANDLE_TYPE : size_t {
+ NO_TIMEOUT = 0,
+ TIMEOUT = 1,
+ };
+
+ static constexpr size_t HANDLE_TYPE_MASK = mask_from_count_v<HANDLE_TYPES>;
+
+ template <typename T>
+ static constexpr auto enum_as_value(T x) {
+ return static_cast<std::underlying_type_t<T>>(x);
+ }
+
+ static inline bool isNoTimeoutHandle(Handle handle) {
+ return (handle.time_since_epoch().count() & HANDLE_TYPE_MASK) ==
+ enum_as_value(HANDLE_TYPE::NO_TIMEOUT);
+ }
+
+ static inline bool isTimeoutHandle(Handle handle) {
+ return (handle.time_since_epoch().count() & HANDLE_TYPE_MASK) ==
+ enum_as_value(HANDLE_TYPE::TIMEOUT);
+ }
+
+ // Returns a unique Handle that doesn't exist in the container.
+ template <size_t MAX_TYPED_HANDLES, size_t HANDLE_TYPE_AS_VALUE, typename C, typename T>
+ static Handle getUniqueHandleForHandleType_l(C container, T timeout) {
+ static_assert(MAX_TYPED_HANDLES > 0 && HANDLE_TYPE_AS_VALUE < MAX_TYPED_HANDLES
+ && is_power_of_2_v<MAX_TYPED_HANDLES>,
+ " handles must be power of two");
+
+ // Our initial handle is the deadline as computed from steady_clock.
+ auto deadline = std::chrono::steady_clock::now() + timeout;
+
+ // We adjust the lsbs by the minimum increment to have the correct
+ // HANDLE_TYPE in the least significant bits.
+ auto remainder = deadline.time_since_epoch().count() & HANDLE_TYPE_MASK;
+ size_t offset = HANDLE_TYPE_AS_VALUE > remainder ? HANDLE_TYPE_AS_VALUE - remainder :
+ MAX_TYPED_HANDLES + HANDLE_TYPE_AS_VALUE - remainder;
+ deadline += std::chrono::steady_clock::duration(offset);
+
+ // To avoid key collisions, advance the handle by MAX_TYPED_HANDLES (the modulus factor)
+ // until the key is unique.
+ while (container.find(deadline) != container.end()) {
+ deadline += std::chrono::steady_clock::duration(MAX_TYPED_HANDLES);
+ }
+ return deadline;
+ }
+
+ // TimerCallback invoked on timeout or cancel.
+ using TimerCallback = std::function<void(Handle)>;
+
/**
* Schedules a task to be executed in the future (`timeout` duration from now).
*
* \param tag string associated with the task. This need not be unique,
* as the Handle returned is used for cancelling.
* \param func callback function that is invoked at the timeout.
- * \param timeout timeout duration which is converted to milliseconds with at
+ * \param timeoutDuration timeout duration which is converted to milliseconds with at
* least 45 integer bits.
* A timeout of 0 (or negative) means the timer never expires
* so func() is never called. These tasks are stored internally
@@ -54,7 +130,8 @@
* \returns a handle that can be used for cancellation.
*/
Handle scheduleTask(
- std::string tag, std::function<void()>&& func, std::chrono::milliseconds timeout);
+ std::string_view tag, TimerCallback&& func,
+ Duration timeoutDuration, Duration secondChanceDuration);
/**
* Tracks a task that shows up on toString() until cancelled.
@@ -62,7 +139,7 @@
* \param tag string associated with the task.
* \returns a handle that can be used for cancellation.
*/
- Handle trackTask(std::string tag);
+ Handle trackTask(std::string_view tag);
/**
* Cancels a task previously scheduled with scheduleTask()
@@ -128,14 +205,30 @@
private:
// To minimize movement of data, we pass around shared_ptrs to Requests.
// These are allocated and deallocated outside of the lock.
+ // TODO(b/243839867) consider options to merge Request with the
+ // TimeCheck::TimeCheckHandler struct.
struct Request {
+ Request(std::chrono::system_clock::time_point _scheduled,
+ std::chrono::system_clock::time_point _deadline,
+ Duration _secondChanceDuration,
+ pid_t _tid,
+ std::string_view _tag)
+ : scheduled(_scheduled)
+ , deadline(_deadline)
+ , secondChanceDuration(_secondChanceDuration)
+ , tid(_tid)
+ , tag(_tag)
+ {}
+
const std::chrono::system_clock::time_point scheduled;
- const std::chrono::system_clock::time_point deadline; // deadline := scheduled + timeout
+ const std::chrono::system_clock::time_point deadline; // deadline := scheduled
+ // + timeoutDuration
+ // + secondChanceDuration
// if deadline == scheduled, no
// timeout, task not executed.
+ Duration secondChanceDuration;
const pid_t tid;
- const std::string tag;
-
+ const FixedString62 tag;
std::string toString() const;
};
@@ -160,15 +253,17 @@
mRequestQueue GUARDED_BY(mRQMutex);
};
- // A storage map of tasks without timeouts. There is no std::function<void()>
+ // A storage map of tasks without timeouts. There is no TimerCallback
// required, it just tracks the tasks with the tag, scheduled time and the tid.
// These tasks show up on a pendingToString() until manually cancelled.
class NoTimeoutMap {
- // This a counter of the requests that have no timeout (timeout == 0).
- std::atomic<size_t> mNoTimeoutRequests{};
-
mutable std::mutex mNTMutex;
std::map<Handle, std::shared_ptr<const Request>> mMap GUARDED_BY(mNTMutex);
+ Handle getUniqueHandle_l() REQUIRES(mNTMutex) {
+ return getUniqueHandleForHandleType_l<
+ HANDLE_TYPES, enum_as_value(HANDLE_TYPE::NO_TIMEOUT)>(
+ mMap, Duration{} /* timeout */);
+ }
public:
bool isValidHandle(Handle handle) const; // lock free
@@ -182,14 +277,26 @@
// call on timeout.
// This class is thread-safe.
class MonitorThread {
+ std::atomic<size_t> mSecondChanceCount{};
mutable std::mutex mMutex;
- mutable std::condition_variable mCond;
+ mutable std::condition_variable mCond GUARDED_BY(mMutex);
// Ordered map of requests based on time of deadline.
//
- std::map<Handle, std::pair<std::shared_ptr<const Request>, std::function<void()>>>
+ std::map<Handle, std::pair<std::shared_ptr<const Request>, TimerCallback>>
mMonitorRequests GUARDED_BY(mMutex);
+ // Due to monotonic/steady clock inaccuracies during suspend,
+ // we allow an additional second chance waiting time to prevent
+ // false removal.
+
+ // This mSecondChanceRequests queue is almost always empty.
+ // Using a pair with the original handle allows lookup and keeps
+ // the Key unique.
+ std::map<std::pair<Handle /* new */, Handle /* original */>,
+ std::pair<std::shared_ptr<const Request>, TimerCallback>>
+ mSecondChanceRequests GUARDED_BY(mMutex);
+
RequestQueue& mTimeoutQueue; // locked internally, added to when request times out.
// Worker thread variables
@@ -200,16 +307,23 @@
std::thread mThread;
void threadFunc();
- Handle getUniqueHandle_l(std::chrono::milliseconds timeout) REQUIRES(mMutex);
+ Handle getUniqueHandle_l(Duration timeout) REQUIRES(mMutex) {
+ return getUniqueHandleForHandleType_l<
+ HANDLE_TYPES, enum_as_value(HANDLE_TYPE::TIMEOUT)>(
+ mMonitorRequests, timeout);
+ }
public:
MonitorThread(RequestQueue &timeoutQueue);
~MonitorThread();
- Handle add(std::shared_ptr<const Request> request, std::function<void()>&& func,
- std::chrono::milliseconds timeout);
+ Handle add(std::shared_ptr<const Request> request, TimerCallback&& func,
+ Duration timeout);
std::shared_ptr<const Request> remove(Handle handle);
void copyRequests(std::vector<std::shared_ptr<const Request>>& requests) const;
+ size_t getSecondChanceCount() const {
+ return mSecondChanceCount.load(std::memory_order_relaxed);
+ }
};
// Analysis contains info deduced by analysisTimeout().
@@ -244,16 +358,6 @@
std::vector<std::shared_ptr<const Request>> getPendingRequests() const;
- // A no-timeout request is represented by a handles at the end of steady_clock time,
- // counting down by the number of no timeout requests previously requested.
- // We manage them on the NoTimeoutMap, but conceptually they could be scheduled
- // on the MonitorThread because those time handles won't expire in
- // the lifetime of the device.
- static inline Handle getIndexedHandle(size_t index) {
- return std::chrono::time_point<std::chrono::steady_clock>::max() -
- std::chrono::time_point<std::chrono::steady_clock>::duration(index);
- }
-
static constexpr size_t kRetiredQueueMax = 16;
RequestQueue mRetiredQueue{kRetiredQueueMax}; // locked internally
diff --git a/media/utils/tests/Android.bp b/media/utils/tests/Android.bp
index 1024018..232cc4e 100644
--- a/media/utils/tests/Android.bp
+++ b/media/utils/tests/Android.bp
@@ -124,6 +124,27 @@
}
cc_test {
+ name: "mediautils_fixedstring_tests",
+
+ cflags: [
+ "-Wall",
+ "-Werror",
+ "-Wextra",
+ ],
+
+ shared_libs: [
+ "libaudioutils",
+ "liblog",
+ "libmediautils",
+ "libutils",
+ ],
+
+ srcs: [
+ "mediautils_fixedstring_tests.cpp",
+ ],
+}
+
+cc_test {
name: "mediautils_scopedstatistics_tests",
cflags: [
diff --git a/media/utils/tests/mediautils_fixedstring_tests.cpp b/media/utils/tests/mediautils_fixedstring_tests.cpp
new file mode 100644
index 0000000..7ab9a9f
--- /dev/null
+++ b/media/utils/tests/mediautils_fixedstring_tests.cpp
@@ -0,0 +1,151 @@
+/*
+ * Copyright (C) 2022 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#define LOG_TAG "mediautils_fixedstring_tests"
+
+#include <mediautils/FixedString.h>
+
+#include <gtest/gtest.h>
+#include <utils/Log.h>
+
+using namespace android::mediautils;
+
+TEST(mediautils_fixedstring_tests, ctor) {
+ FixedString<8> s0("abcde");
+
+ ASSERT_FALSE(s0.empty());
+ ASSERT_EQ(8U, s0.capacity());
+
+ ASSERT_EQ(5U, s0.size());
+ ASSERT_EQ(3U, s0.remaining());
+ ASSERT_EQ(0, strcmp(s0.c_str(), "abcde"));
+
+ ASSERT_EQ(0, strcmp(s0.data(), "abcde"));
+
+ // overflow
+ FixedString<8> s1("abcdefghijk");
+ ASSERT_EQ(8U, s1.size());
+ ASSERT_TRUE(s1.full());
+ ASSERT_EQ(0U, s1.remaining());
+ ASSERT_EQ(0, strcmp(s1.c_str(), "abcdefgh"));
+
+ // overflow
+ FixedString<8> s2(std::string("abcdefghijk"));
+ ASSERT_TRUE(s2.full());
+
+ ASSERT_EQ(8U, s2.size());
+ ASSERT_EQ(0, strcmp(s2.c_str(), "abcdefgh"));
+
+ // complex
+ ASSERT_EQ(s1, s2);
+ ASSERT_EQ(FixedString<12>().append(s1), s2);
+ ASSERT_NE(s1, "bcd");
+
+ // string and stringview
+ ASSERT_EQ(s1.asString(), s1.asStringView());
+
+ FixedString30 s3;
+ s3 = std::string("abcd");
+ ASSERT_EQ(s3, "abcd");
+
+ s3.clear();
+ ASSERT_EQ(s3, "");
+ ASSERT_NE(s3, "abcd");
+ ASSERT_EQ(0U, s3.size());
+}
+
+TEST(mediautils_fixedstring_tests, append) {
+ FixedString<8> s0;
+ ASSERT_EQ(0U, s0.size());
+ ASSERT_EQ(0, strcmp(s0.c_str(), ""));
+ ASSERT_TRUE(s0.empty());
+ ASSERT_FALSE(s0.full());
+
+ s0.append("abc");
+ ASSERT_EQ(3U, s0.size());
+ ASSERT_EQ(0, strcmp(s0.c_str(), "abc"));
+
+ s0.append(std::string("d"));
+ ASSERT_EQ(4U, s0.size());
+ ASSERT_EQ(0, strcmp(s0.c_str(), "abcd"));
+
+ // overflow
+ s0.append("efghijk");
+ ASSERT_EQ(8U, s0.size());
+ ASSERT_EQ(0, strcmp(s0.c_str(), "abcdefgh"));
+ ASSERT_TRUE(s0.full());
+
+ // concatenated
+ ASSERT_EQ(FixedString62("abcd"),
+ FixedString<8>("ab").append("c").append(FixedString<8>("d")));
+ ASSERT_EQ(FixedString<12>("a").append(FixedString<12>("b")), "ab");
+}
+
+TEST(mediautils_fixedstring_tests, plus_equals) {
+ FixedString<8> s0;
+ ASSERT_EQ(0U, s0.size());
+ ASSERT_EQ(0, strcmp(s0.c_str(), ""));
+
+ s0 += "abc";
+ s0 += "def";
+ ASSERT_EQ(s0, "abcdef");
+}
+
+TEST(mediautils_fixedstring_tests, stream_operator) {
+ FixedString<8> s0('a');
+
+ s0 << 'b' << "c" << "d" << '\n';
+ ASSERT_EQ(s0, "abcd\n");
+}
+
+TEST(mediautils_fixedstring_tests, examples) {
+ FixedString30 s1(std::string("a"));
+ s1 << "bc" << 'd' << '\n';
+ s1 += "hello";
+
+ ASSERT_EQ(s1, "abcd\nhello");
+
+ FixedString30 s2;
+ for (const auto &c : s1.asStringView()) {
+ s2.append(c);
+ };
+ ASSERT_EQ(s1, s2);
+
+ FixedString30 s3(std::move(s1));
+}
+
+// Ensure type alias works fine as well.
+using FixedString1024 = FixedString<1024>;
+
+TEST(mediautils_fixedstring_tests, copy) {
+ FixedString1024 s0("abc");
+ FixedString62 s1(s0);
+
+ ASSERT_EQ(3U, s1.size());
+ ASSERT_EQ(0, strcmp(s1.c_str(), "abc"));
+ ASSERT_EQ(s0, s1);
+
+ FixedString<1024> s2(s1);
+ ASSERT_EQ(3U, s2.size());
+ ASSERT_EQ(0, strcmp(s2.c_str(), "abc"));
+ ASSERT_EQ(s2, "abc");
+ ASSERT_NE(s2, "def");
+ ASSERT_EQ(s2, std::string("abc"));
+ ASSERT_NE(s2, std::string("def"));
+ ASSERT_EQ(s1, s2);
+ ASSERT_EQ(s0, s2);
+ ASSERT_EQ(s2, FixedString62(FixedString1024("abc")));
+}
diff --git a/media/utils/tests/timecheck_tests.cpp b/media/utils/tests/timecheck_tests.cpp
index 6ebf44d..8236174 100644
--- a/media/utils/tests/timecheck_tests.cpp
+++ b/media/utils/tests/timecheck_tests.cpp
@@ -39,7 +39,7 @@
timeoutRegistered = timeout;
elapsedMsRegistered = elapsedMs;
event = true;
- }, 1000 /* msec */, false /* crash */);
+ }, 1000ms /* timeoutDuration */, {} /* secondChanceDuration */, false /* crash */);
}
ASSERT_TRUE(event);
ASSERT_FALSE(timeoutRegistered);
@@ -58,7 +58,7 @@
timeoutRegistered = timeout;
elapsedMsRegistered = elapsedMs;
event = true; // store-release, must be last.
- }, 1 /* msec */, false /* crash */);
+ }, 1ms /* timeoutDuration */, {} /* secondChanceDuration */, false /* crash */);
std::this_thread::sleep_for(100ms);
}
ASSERT_TRUE(event); // load-acquire, must be first.
@@ -69,4 +69,4 @@
// Note: We do not test TimeCheck crash because TimeCheck is multithreaded and the
// EXPECT_EXIT() signal catching is imperfect due to the gtest fork.
-} // namespace
\ No newline at end of file
+} // namespace
diff --git a/services/audioflinger/AudioFlinger.cpp b/services/audioflinger/AudioFlinger.cpp
index 23a3a36..d2363d8 100644
--- a/services/audioflinger/AudioFlinger.cpp
+++ b/services/audioflinger/AudioFlinger.cpp
@@ -226,6 +226,9 @@
BINDER_METHOD_ENTRY(getAAudioMixerBurstCount) \
BINDER_METHOD_ENTRY(getAAudioHardwareBurstMinUsec) \
BINDER_METHOD_ENTRY(setDeviceConnectedState) \
+BINDER_METHOD_ENTRY(setRequestedLatencyMode) \
+BINDER_METHOD_ENTRY(getSupportedLatencyModes) \
+
// singleton for Binder Method Statistics for IAudioFlinger
static auto& getIAudioFlingerStatistics() {
@@ -1611,6 +1614,32 @@
return NO_ERROR;
}
+status_t AudioFlinger::setRequestedLatencyMode(
+ audio_io_handle_t output, audio_latency_mode_t mode) {
+ if (output == AUDIO_IO_HANDLE_NONE) {
+ return BAD_VALUE;
+ }
+ AutoMutex lock(mLock);
+ PlaybackThread *thread = checkPlaybackThread_l(output);
+ if (thread == nullptr) {
+ return BAD_VALUE;
+ }
+ return thread->setRequestedLatencyMode(mode);
+}
+
+status_t AudioFlinger::getSupportedLatencyModes(audio_io_handle_t output,
+ std::vector<audio_latency_mode_t>* modes) {
+ if (output == AUDIO_IO_HANDLE_NONE) {
+ return BAD_VALUE;
+ }
+ AutoMutex lock(mLock);
+ PlaybackThread *thread = checkPlaybackThread_l(output);
+ if (thread == nullptr) {
+ return BAD_VALUE;
+ }
+ return thread->getSupportedLatencyModes(modes);
+}
+
status_t AudioFlinger::setStreamMute(audio_stream_type_t stream, bool muted)
{
// check calling permissions
@@ -2110,6 +2139,21 @@
}
}
+void AudioFlinger::onSupportedLatencyModesChanged(
+ audio_io_handle_t output, const std::vector<audio_latency_mode_t>& modes) {
+ int32_t outputAidl = VALUE_OR_FATAL(legacy2aidl_audio_io_handle_t_int32_t(output));
+ std::vector<media::LatencyMode> modesAidl = VALUE_OR_FATAL(
+ convertContainer<std::vector<media::LatencyMode>>(modes,
+ legacy2aidl_audio_latency_mode_t_LatencyMode));
+
+ Mutex::Autolock _l(mClientLock);
+ size_t size = mNotificationClients.size();
+ for (size_t i = 0; i < size; i++) {
+ mNotificationClients.valueAt(i)->audioFlingerClient()
+ ->onSupportedLatencyModesChanged(outputAidl, modesAidl);
+ }
+}
+
// removeClient_l() must be called with AudioFlinger::mClientLock held
void AudioFlinger::removeClient_l(pid_t pid)
{
@@ -3689,6 +3733,12 @@
using namespace std::chrono_literals;
auto inChannelMask = audio_channel_mask_out_to_in(track->channelMask());
+ if (inChannelMask == AUDIO_CHANNEL_INVALID) {
+ // The downstream PatchTrack has the proper output channel mask,
+ // so if there is no input channel mask equivalent, we can just
+ // use an index mask here to create the PatchRecord.
+ inChannelMask = audio_channel_mask_out_to_in_index_mask(track->channelMask());
+ }
sp patchRecord = new RecordThread::PatchRecord(nullptr /* thread */,
track->sampleRate(),
inChannelMask,
@@ -4506,6 +4556,8 @@
case TransactionCode::SET_RECORD_SILENCED:
case TransactionCode::AUDIO_POLICY_READY:
case TransactionCode::SET_DEVICE_CONNECTED_STATE:
+ case TransactionCode::SET_REQUESTED_LATENCY_MODE:
+ case TransactionCode::GET_SUPPORTED_LATENCY_MODES:
ALOGW("%s: transaction %d received from PID %d",
__func__, code, IPCThreadState::self()->getCallingPid());
// return status only for non void methods
@@ -4586,7 +4638,9 @@
} else {
getIAudioFlingerStatistics().event(code, elapsedMs);
}
- });
+ }, mediautils::TimeCheck::kDefaultTimeoutDuration,
+ mediautils::TimeCheck::kDefaultSecondChanceDuration,
+ true /* crashOnTimeout */);
// Make sure we connect to Audio Policy Service before calling into AudioFlinger:
// - AudioFlinger can call into Audio Policy Service with its global mutex held
diff --git a/services/audioflinger/AudioFlinger.h b/services/audioflinger/AudioFlinger.h
index 8e4383c..fc4c807 100644
--- a/services/audioflinger/AudioFlinger.h
+++ b/services/audioflinger/AudioFlinger.h
@@ -293,6 +293,12 @@
virtual status_t setDeviceConnectedState(const struct audio_port_v7 *port, bool connected);
+ virtual status_t setRequestedLatencyMode(
+ audio_io_handle_t output, audio_latency_mode_t mode);
+
+ virtual status_t getSupportedLatencyModes(audio_io_handle_t output,
+ std::vector<audio_latency_mode_t>* modes);
+
status_t onTransactWrapper(TransactionCode code, const Parcel& data, uint32_t flags,
const std::function<status_t()>& delegate) override;
@@ -764,6 +770,8 @@
void ioConfigChanged(audio_io_config_event_t event,
const sp<AudioIoDescriptor>& ioDesc,
pid_t pid = 0);
+ void onSupportedLatencyModesChanged(
+ audio_io_handle_t output, const std::vector<audio_latency_mode_t>& modes);
// Allocate an audio_unique_id_t.
// Specific types are audio_io_handle_t, audio_session_t, effect ID (int),
diff --git a/services/audioflinger/Effects.cpp b/services/audioflinger/Effects.cpp
index e6d7cf7..9ebff0b 100644
--- a/services/audioflinger/Effects.cpp
+++ b/services/audioflinger/Effects.cpp
@@ -1730,6 +1730,7 @@
mNotifyFramesProcessed(notifyFramesProcessed)
{
ALOGV("constructor %p client %p", this, client.get());
+ setMinSchedulerPolicy(SCHED_NORMAL, ANDROID_PRIORITY_AUDIO);
if (client == 0) {
return;
@@ -1790,7 +1791,7 @@
} else {
getIEffectStatistics().event(code, elapsedMs);
}
- }, 0 /* timeoutMs */);
+ }, {} /* timeoutDuration */, {} /* secondChanceDuration */, false /* crashOnTimeout */);
return BnEffect::onTransact(code, data, reply, flags);
}
diff --git a/services/audioflinger/PatchPanel.cpp b/services/audioflinger/PatchPanel.cpp
index 45dd258..b54b41f 100644
--- a/services/audioflinger/PatchPanel.cpp
+++ b/services/audioflinger/PatchPanel.cpp
@@ -313,12 +313,19 @@
patch->sources[0].config_mask & AUDIO_PORT_CONFIG_FLAGS ?
patch->sources[0].flags.input : AUDIO_INPUT_FLAG_NONE;
audio_io_handle_t input = AUDIO_IO_HANDLE_NONE;
+ audio_source_t source = AUDIO_SOURCE_MIC;
+ // For telephony patches, propagate voice communication use case to record side
+ if (patch->num_sources == 2
+ && patch->sources[1].ext.mix.usecase.stream
+ == AUDIO_STREAM_VOICE_CALL) {
+ source = AUDIO_SOURCE_VOICE_COMMUNICATION;
+ }
sp<ThreadBase> thread = mAudioFlinger.openInput_l(srcModule,
&input,
&config,
device,
address,
- AUDIO_SOURCE_MIC,
+ source,
flags,
outputDevice,
outputDeviceAddress);
@@ -516,9 +523,14 @@
audio_output_flags_t outputFlags = mAudioPatch.sinks[0].config_mask & AUDIO_PORT_CONFIG_FLAGS ?
mAudioPatch.sinks[0].flags.output : AUDIO_OUTPUT_FLAG_NONE;
audio_stream_type_t streamType = AUDIO_STREAM_PATCH;
+ audio_source_t source = AUDIO_SOURCE_DEFAULT;
if (mAudioPatch.num_sources == 2 && mAudioPatch.sources[1].type == AUDIO_PORT_TYPE_MIX) {
// "reuse one existing output mix" case
streamType = mAudioPatch.sources[1].ext.mix.usecase.stream;
+ // For telephony patches, propagate voice communication use case to record side
+ if (streamType == AUDIO_STREAM_VOICE_CALL) {
+ source = AUDIO_SOURCE_VOICE_COMMUNICATION;
+ }
}
if (mPlayback.thread()->hasFastMixer()) {
// Create a fast track if the playback thread has fast mixer to get better performance.
@@ -546,7 +558,8 @@
inChannelMask,
format,
frameCount,
- inputFlags);
+ inputFlags,
+ source);
} else {
// use a pseudo LCM between input and output framecount
int playbackShift = __builtin_ctz(playbackFrameCount);
@@ -566,7 +579,9 @@
frameCount,
nullptr,
(size_t)0 /* bufferSize */,
- inputFlags);
+ inputFlags,
+ {} /* timeout */,
+ source);
}
status = mRecord.checkTrack(tempRecordTrack.get());
if (status != NO_ERROR) {
diff --git a/services/audioflinger/RecordTracks.h b/services/audioflinger/RecordTracks.h
index e8552c4..daec57e 100644
--- a/services/audioflinger/RecordTracks.h
+++ b/services/audioflinger/RecordTracks.h
@@ -87,6 +87,10 @@
&& (flags & AUDIO_INPUT_FLAG_HW_AV_SYNC) == 0;
}
+ using SinkMetadatas = std::vector<record_track_metadata_v7_t>;
+ using MetadataInserter = std::back_insert_iterator<SinkMetadatas>;
+ virtual void copyMetadataTo(MetadataInserter& backInserter) const;
+
private:
friend class AudioFlinger; // for mState
@@ -134,7 +138,8 @@
void *buffer,
size_t bufferSize,
audio_input_flags_t flags,
- const Timeout& timeout = {});
+ const Timeout& timeout = {},
+ audio_source_t source = AUDIO_SOURCE_DEFAULT);
virtual ~PatchRecord();
virtual Source* getSource() { return nullptr; }
@@ -166,7 +171,8 @@
audio_channel_mask_t channelMask,
audio_format_t format,
size_t frameCount,
- audio_input_flags_t flags);
+ audio_input_flags_t flags,
+ audio_source_t source = AUDIO_SOURCE_DEFAULT);
Source* getSource() override { return static_cast<Source*>(this); }
diff --git a/services/audioflinger/ThreadMetrics.h b/services/audioflinger/ThreadMetrics.h
index 6526655..5493b3c 100644
--- a/services/audioflinger/ThreadMetrics.h
+++ b/services/audioflinger/ThreadMetrics.h
@@ -148,7 +148,14 @@
item.set(AMEDIAMETRICS_PROP_CUMULATIVETIMENS, mCumulativeTimeNs)
.set(AMEDIAMETRICS_PROP_DEVICETIMENS, mDeviceTimeNs)
.set(AMEDIAMETRICS_PROP_EVENT, eventName)
- .set(AMEDIAMETRICS_PROP_INTERVALCOUNT, (int32_t)mIntervalCount);
+ .set(AMEDIAMETRICS_PROP_INTERVALCOUNT, (int32_t)mIntervalCount)
+ // we set "last" device to indicate the device the group was
+ // associated with (because a createPatch which is logged in ThreadMetrics
+ // could have changed the device).
+ .set(mIsOut
+ ? AMEDIAMETRICS_PROP_PREFIX_LAST AMEDIAMETRICS_PROP_OUTPUTDEVICES
+ : AMEDIAMETRICS_PROP_PREFIX_LAST AMEDIAMETRICS_PROP_INPUTDEVICES,
+ mDevices.c_str());
if (mDeviceLatencyMs.getN() > 0) {
item.set(AMEDIAMETRICS_PROP_DEVICELATENCYMS, mDeviceLatencyMs.getMean());
}
diff --git a/services/audioflinger/Threads.cpp b/services/audioflinger/Threads.cpp
index b4accab..73ee691 100644
--- a/services/audioflinger/Threads.cpp
+++ b/services/audioflinger/Threads.cpp
@@ -178,6 +178,11 @@
// Direct output thread minimum sleep time in idle or active(underrun) state
static const nsecs_t kDirectMinSleepTimeUs = 10000;
+// Minimum amount of time between checking to see if the timestamp is advancing
+// for underrun detection. If we check too frequently, we may not detect a
+// timestamp update and will falsely detect underrun.
+static const nsecs_t kMinimumTimeBetweenTimestampChecksNs = 150 /* ms */ * 1000;
+
// The universal constant for ubiquitous 20ms value. The value of 20ms seems to provide a good
// balance between power consumption and latency, and allows threads to be scheduled reliably
// by the CFS scheduler.
@@ -264,6 +269,23 @@
return ss.str();
}
+static std::string toString(audio_latency_mode_t mode) {
+ // We convert to the AIDL type to print (eventually the legacy type will be removed).
+ const auto result = legacy2aidl_audio_latency_mode_t_LatencyMode(mode);
+ return result.has_value() ? media::toString(*result) : "UNKNOWN";
+}
+
+// Could be made a template, but other toString overloads for std::vector are confused.
+static std::string toString(const std::vector<audio_latency_mode_t>& elements) {
+ std::string s("{ ");
+ for (const auto& e : elements) {
+ s.append(toString(e));
+ s.append(" ");
+ }
+ s.append("}");
+ return s;
+}
+
static pthread_once_t sFastTrackMultiplierOnce = PTHREAD_ONCE_INIT;
static void sFastTrackMultiplierInit()
@@ -741,6 +763,12 @@
sendConfigEvent_l(configEvent);
}
+void AudioFlinger::ThreadBase::sendHalLatencyModesChangedEvent_l()
+{
+ sp<ConfigEvent> configEvent = sp<HalLatencyModesChangedEvent>::make();
+ sendConfigEvent_l(configEvent);
+}
+
// post condition: mConfigEvents.isEmpty()
void AudioFlinger::ThreadBase::processConfigEvents_l()
{
@@ -808,6 +836,10 @@
setCheckOutputStageEffects();
} break;
+ case CFG_EVENT_HAL_LATENCY_MODES_CHANGED: {
+ onHalLatencyModesChanged_l();
+ } break;
+
default:
ALOG_ASSERT(false, "processConfigEvents_l() unknown event type %d", event->mType);
break;
@@ -2030,7 +2062,8 @@
mFastTrackAvailMask(((1 << FastMixerState::sMaxFastTracks) - 1) & ~1),
mHwSupportsPause(false), mHwPaused(false), mFlushPending(false),
mLeftVolFloat(-1.0), mRightVolFloat(-1.0),
- mDownStreamPatch{}
+ mDownStreamPatch{},
+ mIsTimestampAdvancing(kMinimumTimeBetweenTimestampChecksNs)
{
snprintf(mThreadName, kThreadNameLength, "AudioOut_%X", id);
mNBLogWriter = audioFlinger->newWriter_l(kLogSize, mThreadName);
@@ -3136,10 +3169,7 @@
auto backInserter = std::back_inserter(metadata.tracks);
for (const sp<Track> &track : mActiveTracks) {
// No track is invalid as this is called after prepareTrack_l in the same critical section
- // Do not forward metadata for PatchTrack with unspecified stream type
- if (track->streamType() != AUDIO_STREAM_PATCH) {
- track->copyMetadataTo(backInserter);
- }
+ track->copyMetadataTo(backInserter);
}
sendMetadataToBackend_l(metadata);
}
@@ -3922,6 +3952,8 @@
// stop(), pause(), etc.), but the threadLoop is entitled to call audio
// data / buffer methods on tracks from activeTracks without the ThreadBase lock.
activeTracks.insert(activeTracks.end(), mActiveTracks.begin(), mActiveTracks.end());
+
+ setHalLatencyMode_l();
} // mLock scope ends
if (mBytesRemaining == 0) {
@@ -4087,10 +4119,19 @@
mEffectBufferFormat,
mNormalFrameCount * mHapticChannelCount);
}
-
- memcpy_by_audio_format(mSinkBuffer, mFormat, effectBuffer, mEffectBufferFormat,
- mNormalFrameCount * (mChannelCount + mHapticChannelCount));
-
+ const size_t framesToCopy = mNormalFrameCount * (mChannelCount + mHapticChannelCount);
+ if (mFormat == AUDIO_FORMAT_PCM_FLOAT &&
+ mEffectBufferFormat == AUDIO_FORMAT_PCM_FLOAT) {
+ // Clamp PCM float values more than this distance from 0 to insulate
+ // a HAL which doesn't handle NaN correctly.
+ static constexpr float HAL_FLOAT_SAMPLE_LIMIT = 2.0f;
+ memcpy_to_float_from_float_with_clamping(static_cast<float*>(mSinkBuffer),
+ static_cast<const float*>(effectBuffer),
+ framesToCopy, HAL_FLOAT_SAMPLE_LIMIT /* absMax */);
+ } else {
+ memcpy_by_audio_format(mSinkBuffer, mFormat,
+ effectBuffer, mEffectBufferFormat, framesToCopy);
+ }
// The sample data is partially interleaved when haptic channels exist,
// we need to adjust channels here.
if (mHapticChannelCount > 0) {
@@ -5021,6 +5062,7 @@
mCallbackThread->setDraining(mDrainSequence);
}
mHwPaused = false;
+ setHalLatencyMode_l();
}
void AudioFlinger::PlaybackThread::onAddNewTrack_l()
@@ -5894,18 +5936,35 @@
return trackCount;
}
-bool AudioFlinger::PlaybackThread::checkRunningTimestamp()
+bool AudioFlinger::PlaybackThread::IsTimestampAdvancing::check(AudioStreamOut * output)
{
+ // Check the timestamp to see if it's advancing once every 150ms. If we check too frequently, we
+ // could falsely detect that the frame position has stalled due to underrun because we haven't
+ // given the Audio HAL enough time to update.
+ const nsecs_t nowNs = systemTime();
+ if (nowNs - mPreviousNs < mMinimumTimeBetweenChecksNs) {
+ return mLatchedValue;
+ }
+ mPreviousNs = nowNs;
+ mLatchedValue = false;
+ // Determine if the presentation position is still advancing.
uint64_t position = 0;
struct timespec unused;
- const status_t ret = mOutput->getPresentationPosition(&position, &unused);
+ const status_t ret = output->getPresentationPosition(&position, &unused);
if (ret == NO_ERROR) {
- if (position != mLastCheckedTimestampPosition) {
- mLastCheckedTimestampPosition = position;
- return true;
+ if (position != mPreviousPosition) {
+ mPreviousPosition = position;
+ mLatchedValue = true;
}
}
- return false;
+ return mLatchedValue;
+}
+
+void AudioFlinger::PlaybackThread::IsTimestampAdvancing::clear()
+{
+ mLatchedValue = true;
+ mPreviousPosition = 0;
+ mPreviousNs = 0;
}
// isTrackAllowed_l() must be called with ThreadBase::mLock held
@@ -6340,9 +6399,9 @@
// No buffers for this track. Give it a few chances to
// fill a buffer, then remove it from active list.
// Only consider last track started for mixer state control
+ bool isTimestampAdvancing = mIsTimestampAdvancing.check(mOutput);
if (--(track->mRetryCount) <= 0) {
- const bool running = checkRunningTimestamp();
- if (running) { // still running, give us more time.
+ if (isTimestampAdvancing) { // HAL is still playing audio, give us more time.
track->mRetryCount = kMaxTrackRetriesOffload;
} else {
ALOGV("BUFFER TIMEOUT: remove track(%d) from active list", trackId);
@@ -6923,9 +6982,9 @@
} else {
// No buffers for this track. Give it a few chances to
// fill a buffer, then remove it from active list.
+ bool isTimestampAdvancing = mIsTimestampAdvancing.check(mOutput);
if (--(track->mRetryCount) <= 0) {
- const bool running = checkRunningTimestamp();
- if (running) { // still running, give us more time.
+ if (isTimestampAdvancing) { // HAL is still playing audio, give us more time.
track->mRetryCount = kMaxTrackRetriesOffload;
} else {
ALOGV("OffloadThread: BUFFER TIMEOUT: remove track(%d) from active list",
@@ -7264,6 +7323,110 @@
{
}
+void AudioFlinger::SpatializerThread::onFirstRef() {
+ PlaybackThread::onFirstRef();
+
+ Mutex::Autolock _l(mLock);
+ status_t status = mOutput->stream->setLatencyModeCallback(this);
+ if (status != INVALID_OPERATION) {
+ updateHalSupportedLatencyModes_l();
+ }
+
+ const pid_t tid = getTid();
+ if (tid == -1) {
+ // Unusual: PlaybackThread::onFirstRef() should set the threadLoop running.
+ ALOGW("%s: Cannot update Spatializer mixer thread priority, not running", __func__);
+ } else {
+ const int priorityBoost = requestSpatializerPriority(getpid(), tid);
+ if (priorityBoost > 0) {
+ stream()->setHalThreadPriority(priorityBoost);
+ }
+ }
+}
+
+status_t AudioFlinger::SpatializerThread::createAudioPatch_l(const struct audio_patch *patch,
+ audio_patch_handle_t *handle)
+{
+ status_t status = MixerThread::createAudioPatch_l(patch, handle);
+ updateHalSupportedLatencyModes_l();
+ return status;
+}
+
+void AudioFlinger::SpatializerThread::updateHalSupportedLatencyModes_l() {
+ std::vector<audio_latency_mode_t> latencyModes;
+ const status_t status = mOutput->stream->getRecommendedLatencyModes(&latencyModes);
+ if (status != NO_ERROR) {
+ latencyModes.clear();
+ }
+ if (latencyModes != mSupportedLatencyModes) {
+ ALOGD("%s: thread(%d) status %d supported latency modes: %s",
+ __func__, mId, status, toString(latencyModes).c_str());
+ mSupportedLatencyModes.swap(latencyModes);
+ sendHalLatencyModesChangedEvent_l();
+ }
+}
+
+void AudioFlinger::SpatializerThread::onHalLatencyModesChanged_l() {
+ mAudioFlinger->onSupportedLatencyModesChanged(mId, mSupportedLatencyModes);
+}
+
+void AudioFlinger::SpatializerThread::setHalLatencyMode_l() {
+ // if mSupportedLatencyModes is empty, the HAL stream does not support
+ // latency mode control and we can exit.
+ if (mSupportedLatencyModes.empty()) {
+ return;
+ }
+ audio_latency_mode_t latencyMode = AUDIO_LATENCY_MODE_FREE;
+ if (mSupportedLatencyModes.size() == 1) {
+ // If the HAL only support one latency mode currently, confirm the choice
+ latencyMode = mSupportedLatencyModes[0];
+ } else if (mSupportedLatencyModes.size() > 1) {
+ // Request low latency if:
+ // - The low latency mode is requested by the spatializer controller
+ // (mRequestedLatencyMode = AUDIO_LATENCY_MODE_LOW)
+ // AND
+ // - At least one active track is spatialized
+ bool hasSpatializedActiveTrack = false;
+ for (const auto& track : mActiveTracks) {
+ if (track->isSpatialized()) {
+ hasSpatializedActiveTrack = true;
+ break;
+ }
+ }
+ if (hasSpatializedActiveTrack && mRequestedLatencyMode == AUDIO_LATENCY_MODE_LOW) {
+ latencyMode = AUDIO_LATENCY_MODE_LOW;
+ }
+ }
+
+ if (latencyMode != mSetLatencyMode) {
+ status_t status = mOutput->stream->setLatencyMode(latencyMode);
+ ALOGD("%s: thread(%d) setLatencyMode(%s) returned %d",
+ __func__, mId, toString(latencyMode).c_str(), status);
+ if (status == NO_ERROR) {
+ mSetLatencyMode = latencyMode;
+ }
+ }
+}
+
+status_t AudioFlinger::SpatializerThread::setRequestedLatencyMode(audio_latency_mode_t mode) {
+ if (mode != AUDIO_LATENCY_MODE_LOW && mode != AUDIO_LATENCY_MODE_FREE) {
+ return BAD_VALUE;
+ }
+ Mutex::Autolock _l(mLock);
+ mRequestedLatencyMode = mode;
+ return NO_ERROR;
+}
+
+status_t AudioFlinger::SpatializerThread::getSupportedLatencyModes(
+ std::vector<audio_latency_mode_t>* modes) {
+ if (modes == nullptr) {
+ return BAD_VALUE;
+ }
+ Mutex::Autolock _l(mLock);
+ *modes = mSupportedLatencyModes;
+ return NO_ERROR;
+}
+
void AudioFlinger::SpatializerThread::checkOutputStageEffects()
{
bool hasVirtualizer = false;
@@ -7316,6 +7479,16 @@
}
}
+void AudioFlinger::SpatializerThread::onRecommendedLatencyModeChanged(
+ std::vector<audio_latency_mode_t> modes) {
+ Mutex::Autolock _l(mLock);
+ if (modes != mSupportedLatencyModes) {
+ ALOGD("%s: thread(%d) supported latency modes: %s",
+ __func__, mId, toString(modes).c_str());
+ mSupportedLatencyModes.swap(modes);
+ sendHalLatencyModesChangedEvent_l();
+ }
+}
// ----------------------------------------------------------------------------
// Record
@@ -8614,21 +8787,9 @@
return; // nothing to do
}
StreamInHalInterface::SinkMetadata metadata;
+ auto backInserter = std::back_inserter(metadata.tracks);
for (const sp<RecordTrack> &track : mActiveTracks) {
- // Do not forward PatchRecord metadata to audio HAL
- if (track->isPatchTrack()) {
- continue;
- }
- // No track is invalid as this is called after prepareTrack_l in the same critical section
- record_track_metadata_v7_t trackMetadata;
- trackMetadata.base = {
- .source = track->attributes().source,
- .gain = 1, // capture tracks do not have volumes
- };
- trackMetadata.channel_mask = track->channelMask(),
- strncpy(trackMetadata.tags, track->attributes().tags, AUDIO_ATTRIBUTES_TAGS_MAX_SIZE);
-
- metadata.tracks.push_back(trackMetadata);
+ track->copyMetadataTo(backInserter);
}
mInput->stream->updateSinkMetadata(metadata);
}
@@ -10084,19 +10245,22 @@
void AudioFlinger::MmapThread::checkInvalidTracks_l()
{
+ sp<MmapStreamCallback> callback;
for (const sp<MmapTrack> &track : mActiveTracks) {
if (track->isInvalid()) {
- sp<MmapStreamCallback> callback = mCallback.promote();
- if (callback != 0) {
- mLock.unlock();
- callback->onTearDown(track->portId());
- mLock.lock();
- } else if (mNoCallbackWarningCount < kMaxNoCallbackWarnings) {
- ALOGW("Could not notify MMAP stream tear down: no onTearDown callback!");
+ callback = mCallback.promote();
+ if (callback == nullptr && mNoCallbackWarningCount < kMaxNoCallbackWarnings) {
+ ALOGW("Could not notify MMAP stream tear down: no onRoutingChanged callback!");
mNoCallbackWarningCount++;
}
+ break;
}
}
+ if (callback != 0) {
+ mLock.unlock();
+ callback->onRoutingChanged(AUDIO_PORT_HANDLE_NONE);
+ mLock.lock();
+ }
}
void AudioFlinger::MmapThread::dumpInternals_l(int fd, const Vector<String16>& args __unused)
diff --git a/services/audioflinger/Threads.h b/services/audioflinger/Threads.h
index 074ae8f..ad5617d 100644
--- a/services/audioflinger/Threads.h
+++ b/services/audioflinger/Threads.h
@@ -55,7 +55,8 @@
CFG_EVENT_RELEASE_AUDIO_PATCH,
CFG_EVENT_UPDATE_OUT_DEVICE,
CFG_EVENT_RESIZE_BUFFER,
- CFG_EVENT_CHECK_OUTPUT_STAGE_EFFECTS
+ CFG_EVENT_CHECK_OUTPUT_STAGE_EFFECTS,
+ CFG_EVENT_HAL_LATENCY_MODES_CHANGED,
};
class ConfigEventData: public RefBase {
@@ -282,6 +283,15 @@
virtual ~CheckOutputStageEffectsEvent() {}
};
+ class HalLatencyModesChangedEvent : public ConfigEvent {
+ public:
+ HalLatencyModesChangedEvent() :
+ ConfigEvent(CFG_EVENT_HAL_LATENCY_MODES_CHANGED) {
+ }
+
+ virtual ~HalLatencyModesChangedEvent() {}
+ };
+
class PMDeathRecipient : public IBinder::DeathRecipient {
public:
@@ -353,6 +363,7 @@
void sendResizeBufferConfigEvent_l(int32_t maxSharedAudioHistoryMs);
void sendCheckOutputStageEffectsEvent();
void sendCheckOutputStageEffectsEvent_l();
+ void sendHalLatencyModesChangedEvent_l();
void processConfigEvents_l();
virtual void setCheckOutputStageEffects() {}
@@ -364,7 +375,7 @@
virtual void toAudioPortConfig(struct audio_port_config *config) = 0;
virtual void resizeInputBuffer_l(int32_t maxSharedAudioHistoryMs);
-
+ virtual void onHalLatencyModesChanged_l() {}
// see note at declaration of mStandby, mOutDevice and mInDevice
@@ -921,6 +932,8 @@
}
virtual void checkOutputStageEffects() {}
+ virtual void setHalLatencyMode_l() {}
+
void dumpInternals_l(int fd, const Vector<String16>& args) override;
void dumpTracks_l(int fd, const Vector<String16>& args) override;
@@ -1064,6 +1077,15 @@
bool hasMixer() const {
return mType == MIXER || mType == DUPLICATING || mType == SPATIALIZER;
}
+
+ virtual status_t setRequestedLatencyMode(
+ audio_latency_mode_t mode __unused) { return INVALID_OPERATION; }
+
+ virtual status_t getSupportedLatencyModes(
+ std::vector<audio_latency_mode_t>* modes __unused) {
+ return INVALID_OPERATION;
+ }
+
protected:
// updated by readOutputParameters_l()
size_t mNormalFrameCount; // normal mixer and effects
@@ -1150,7 +1172,7 @@
volatile int32_t mSuspended;
int64_t mBytesWritten;
- int64_t mFramesWritten; // not reset on standby
+ std::atomic<int64_t> mFramesWritten; // not reset on standby
int64_t mLastFramesWritten = -1; // track changes in timestamp
// server frames written.
int64_t mSuspendedFrames; // not reset on standby
@@ -1364,6 +1386,7 @@
virtual bool hasFastMixer() const = 0;
virtual FastTrackUnderruns getFastTrackUnderruns(size_t fastIndex __unused) const
{ FastTrackUnderruns dummy; return dummy; }
+ const std::atomic<int64_t>& framesWritten() const { return mFramesWritten; }
protected:
// accessed by both binder threads and within threadLoop(), lock on mutex needed
@@ -1381,13 +1404,38 @@
std::atomic_bool mCheckOutputStageEffects{};
- // A differential check on the timestamps to see if there is a change in the
- // timestamp frame position between the last call to checkRunningTimestamp.
- uint64_t mLastCheckedTimestampPosition = ~0LL;
- bool checkRunningTimestamp();
+ // Provides periodic checking for timestamp advancement for underrun detection.
+ class IsTimestampAdvancing {
+ public:
+ // The timestamp will not be checked any faster than the specified time.
+ IsTimestampAdvancing(nsecs_t minimumTimeBetweenChecksNs)
+ : mMinimumTimeBetweenChecksNs(minimumTimeBetweenChecksNs)
+ {
+ clear();
+ }
+ // Check if the presentation position has advanced in the last periodic time.
+ bool check(AudioStreamOut * output);
+ // Clear the internal state when the playback state changes for the output
+ // stream.
+ void clear();
+ private:
+ // The minimum time between timestamp checks.
+ const nsecs_t mMinimumTimeBetweenChecksNs;
+ // Add differential check on the timestamps to see if there is a change in the
+ // timestamp frame position between the last call to check.
+ uint64_t mPreviousPosition;
+ // The time at which the last check occurred, to ensure we don't check too
+ // frequently, giving the Audio HAL enough time to update its timestamps.
+ nsecs_t mPreviousNs;
+ // The valued is latched so we don't check timestamps too frequently.
+ bool mLatchedValue;
+ };
+ IsTimestampAdvancing mIsTimestampAdvancing;
- virtual void flushHw_l() { mLastCheckedTimestampPosition = ~0LL; }
+ virtual void flushHw_l() {
+ mIsTimestampAdvancing.clear();
+ }
};
class MixerThread : public PlaybackThread {
@@ -1682,7 +1730,8 @@
}
};
-class SpatializerThread : public MixerThread {
+class SpatializerThread : public MixerThread,
+ public StreamOutHalInterfaceLatencyModeCallback {
public:
SpatializerThread(const sp<AudioFlinger>& audioFlinger,
AudioStreamOut* output,
@@ -1693,10 +1742,35 @@
bool hasFastMixer() const override { return false; }
+ status_t createAudioPatch_l(const struct audio_patch *patch,
+ audio_patch_handle_t *handle) override;
+
+ // RefBase
+ virtual void onFirstRef();
+
+ // StreamOutHalInterfaceLatencyModeCallback
+ void onRecommendedLatencyModeChanged(std::vector<audio_latency_mode_t> modes) override;
+
+ status_t setRequestedLatencyMode(audio_latency_mode_t mode) override;
+ status_t getSupportedLatencyModes(std::vector<audio_latency_mode_t>* modes) override;
+
protected:
void checkOutputStageEffects() override;
+ void onHalLatencyModesChanged_l() override;
+ void setHalLatencyMode_l() override;
private:
+ void updateHalSupportedLatencyModes_l();
+
+ // Support low latency mode by default as unless explicitly indicated by the audio HAL
+ // we assume the audio path is compatible with the head tracking latency requirements
+ std::vector<audio_latency_mode_t> mSupportedLatencyModes = {AUDIO_LATENCY_MODE_LOW};
+ // default to invalid value to force first update to the audio HAL
+ audio_latency_mode_t mSetLatencyMode =
+ (audio_latency_mode_t)AUDIO_LATENCY_MODE_INVALID;
+ // Do not request a specific mode by default
+ audio_latency_mode_t mRequestedLatencyMode = AUDIO_LATENCY_MODE_FREE;
+
sp<EffectHandle> mFinalDownMixer;
};
diff --git a/services/audioflinger/TrackBase.h b/services/audioflinger/TrackBase.h
index 2677ab3..20bfbb0 100644
--- a/services/audioflinger/TrackBase.h
+++ b/services/audioflinger/TrackBase.h
@@ -397,6 +397,8 @@
int64_t mLogStartFrames = 0; // Timestamp frames at start()
double mLogLatencyMs = 0.; // Track the last log latency
+ bool mLogForceVolumeUpdate = true; // force volume update to TrackMetrics.
+
TrackMetrics mTrackMetrics;
bool mServerLatencySupported = false;
diff --git a/services/audioflinger/TrackMetrics.h b/services/audioflinger/TrackMetrics.h
index 30d69ab..6fc70d6 100644
--- a/services/audioflinger/TrackMetrics.h
+++ b/services/audioflinger/TrackMetrics.h
@@ -64,7 +64,6 @@
AMEDIAMETRICS_PROP_EVENT_VALUE_BEGINAUDIOINTERVALGROUP, devices.c_str());
}
++mIntervalCount;
- mIntervalStartTimeNs = systemTime();
}
void logConstructor(pid_t creatorPid, uid_t creatorUid, int32_t internalTrackId,
@@ -90,11 +89,9 @@
// Called when we are removed from the Thread.
void logEndInterval() {
std::lock_guard l(mLock);
- if (mIntervalStartTimeNs != 0) {
- const int64_t elapsedTimeNs = systemTime() - mIntervalStartTimeNs;
- mIntervalStartTimeNs = 0;
- mCumulativeTimeNs += elapsedTimeNs;
- mDeviceTimeNs += elapsedTimeNs;
+ if (mLastVolumeChangeTimeNs != 0) {
+ logVolume_l(mVolume); // flush out the last volume.
+ mLastVolumeChangeTimeNs = 0;
}
}
@@ -133,20 +130,8 @@
// may be called multiple times during an interval
void logVolume(float volume) {
- const int64_t timeNs = systemTime();
std::lock_guard l(mLock);
- if (mStartVolumeTimeNs == 0) {
- mDeviceVolume = mVolume = volume;
- mLastVolumeChangeTimeNs = mStartVolumeTimeNs = timeNs;
- updateMinMaxVolume(0, mVolume);
- return;
- }
- const int64_t durationNs = timeNs - mLastVolumeChangeTimeNs;
- updateMinMaxVolume(durationNs, mVolume);
- mDeviceVolume = (mDeviceVolume * (mLastVolumeChangeTimeNs - mStartVolumeTimeNs) +
- mVolume * durationNs) / (timeNs - mStartVolumeTimeNs);
- mVolume = volume;
- mLastVolumeChangeTimeNs = timeNs;
+ logVolume_l(volume);
}
// Use absolute numbers returned by AudioTrackShared.
@@ -158,6 +143,7 @@
}
private:
+
// no lock required - all arguments and constants.
void deliverDeviceMetrics(const char *eventName, const char *devices) const {
mediametrics::LogItem(mMetricsId)
@@ -167,6 +153,23 @@
.record();
}
+ void logVolume_l(float volume) REQUIRES(mLock) {
+ const int64_t timeNs = systemTime();
+ const int64_t durationNs = mLastVolumeChangeTimeNs == 0
+ ? 0 : timeNs - mLastVolumeChangeTimeNs;
+ if (durationNs > 0) {
+ // See West's algorithm for weighted averages
+ // https://en.wikipedia.org/wiki/Algorithms_for_calculating_variance
+ mDeviceVolume += (mVolume - mDeviceVolume) * durationNs
+ / (durationNs + mDeviceTimeNs);
+ mDeviceTimeNs += durationNs;
+ mCumulativeTimeNs += durationNs;
+ }
+ updateMinMaxVolume(durationNs, mVolume); // always update.
+ mVolume = volume;
+ mLastVolumeChangeTimeNs = timeNs;
+ }
+
void deliverCumulativeMetrics(const char *eventName) const REQUIRES(mLock) {
if (mIntervalCount > 0) {
mediametrics::LogItem item(mMetricsId);
@@ -199,14 +202,12 @@
// mDevices is not reset by resetIntervalGroupMetrics.
mIntervalCount = 0;
- mIntervalStartTimeNs = 0;
// mCumulativeTimeNs is not reset by resetIntervalGroupMetrics.
mDeviceTimeNs = 0;
mVolume = 0.f;
mDeviceVolume = 0.f;
- mStartVolumeTimeNs = 0;
- mLastVolumeChangeTimeNs = 0;
+ mLastVolumeChangeTimeNs = 0; // last time volume logged, cleared on endInterval
mMinVolume = AMEDIAMETRICS_INITIAL_MIN_VOLUME;
mMaxVolume = AMEDIAMETRICS_INITIAL_MAX_VOLUME;
mMinVolumeDurationNs = 0;
@@ -230,14 +231,12 @@
// Number of intervals and playing time
int32_t mIntervalCount GUARDED_BY(mLock) = 0;
- int64_t mIntervalStartTimeNs GUARDED_BY(mLock) = 0;
- int64_t mCumulativeTimeNs GUARDED_BY(mLock) = 0;
- int64_t mDeviceTimeNs GUARDED_BY(mLock) = 0;
+ int64_t mCumulativeTimeNs GUARDED_BY(mLock) = 0; // total time.
+ int64_t mDeviceTimeNs GUARDED_BY(mLock) = 0; // time on device.
// Average volume
- double mVolume GUARDED_BY(mLock) = 0.f;
- double mDeviceVolume GUARDED_BY(mLock) = 0.f;
- int64_t mStartVolumeTimeNs GUARDED_BY(mLock) = 0;
+ double mVolume GUARDED_BY(mLock) = 0.f; // last set volume.
+ double mDeviceVolume GUARDED_BY(mLock) = 0.f; // running average volume.
int64_t mLastVolumeChangeTimeNs GUARDED_BY(mLock) = 0;
// Min/Max volume
diff --git a/services/audioflinger/Tracks.cpp b/services/audioflinger/Tracks.cpp
index 83a8bb0..10f9f73 100644
--- a/services/audioflinger/Tracks.cpp
+++ b/services/audioflinger/Tracks.cpp
@@ -334,6 +334,7 @@
: BnAudioTrack(),
mTrack(track)
{
+ setMinSchedulerPolicy(SCHED_NORMAL, ANDROID_PRIORITY_AUDIO);
}
AudioFlinger::TrackHandle::~TrackHandle() {
@@ -1091,12 +1092,22 @@
__func__, mId, (int)mThreadIoHandle);
}
- // states to reset position info for non-offloaded/direct tracks
- if (!isOffloaded() && !isDirect()
+ PlaybackThread *playbackThread = (PlaybackThread *)thread.get();
+
+ // states to reset position info for pcm tracks
+ if (audio_is_linear_pcm(mFormat)
&& (state == IDLE || state == STOPPED || state == FLUSHED)) {
mFrameMap.reset();
+
+ if (!isFastTrack() && (isDirect() || isOffloaded())) {
+ // Start point of track -> sink frame map. If the HAL returns a
+ // frame position smaller than the first written frame in
+ // updateTrackFrameInfo, the timestamp can be interpolated
+ // instead of using a larger value.
+ mFrameMap.push(mAudioTrackServerProxy->framesReleased(),
+ playbackThread->framesWritten());
+ }
}
- PlaybackThread *playbackThread = (PlaybackThread *)thread.get();
if (isFastTrack()) {
// refresh fast track underruns on start because that field is never cleared
// by the fast mixer; furthermore, the same track can be recycled, i.e. start
@@ -1120,6 +1131,7 @@
.mPosition[ExtendedTimestamp::LOCATION_KERNEL];
mLogLatencyMs = 0.;
}
+ mLogForceVolumeUpdate = true; // at least one volume logged for metrics when starting.
if (status == NO_ERROR || status == ALREADY_EXISTS) {
// for streaming tracks, remove the buffer read stop limit.
@@ -1391,12 +1403,21 @@
if (mFinalVolume != volume) { // Compare to an epsilon if too many meaningless updates
mFinalVolume = volume;
setMetadataHasChanged();
- mTrackMetrics.logVolume(volume);
+ mLogForceVolumeUpdate = true;
+ }
+ if (mLogForceVolumeUpdate) {
+ mLogForceVolumeUpdate = false;
+ mTrackMetrics.logVolume(mFinalVolume);
}
}
void AudioFlinger::PlaybackThread::Track::copyMetadataTo(MetadataInserter& backInserter) const
{
+ // Do not forward metadata for PatchTrack with unspecified stream type
+ if (mStreamType == AUDIO_STREAM_PATCH) {
+ return;
+ }
+
playback_track_metadata_v7_t metadata;
metadata.base = {
.usage = mAttr.usage,
@@ -1457,7 +1478,7 @@
}
}
- metadata.channel_mask = mChannelMask,
+ metadata.channel_mask = mChannelMask;
strncpy(metadata.tags, mAttr.tags, AUDIO_ATTRIBUTES_TAGS_MAX_SIZE);
*backInserter++ = metadata;
}
@@ -2001,7 +2022,6 @@
{
Buffer *pInBuffer;
Buffer inBuffer;
- bool outputBufferFull = false;
inBuffer.frameCount = frames;
inBuffer.raw = data;
@@ -2031,7 +2051,6 @@
ALOGV("%s(%d): thread %d no more output buffers; status %d",
__func__, mId,
(int)mThreadIoHandle, status);
- outputBufferFull = true;
break;
}
uint32_t waitTimeMs = (uint32_t)ns2ms(systemTime() - startTime);
@@ -2311,6 +2330,7 @@
: BnAudioRecord(),
mRecordTrack(recordTrack)
{
+ setMinSchedulerPolicy(SCHED_NORMAL, ANDROID_PRIORITY_AUDIO);
}
AudioFlinger::RecordHandle::~RecordHandle() {
@@ -2727,6 +2747,25 @@
}
}
+void AudioFlinger::RecordThread::RecordTrack::copyMetadataTo(MetadataInserter& backInserter) const
+{
+
+ // Do not forward PatchRecord metadata with unspecified audio source
+ if (mAttr.source == AUDIO_SOURCE_DEFAULT) {
+ return;
+ }
+
+ // No track is invalid as this is called after prepareTrack_l in the same critical section
+ record_track_metadata_v7_t metadata;
+ metadata.base = {
+ .source = mAttr.source,
+ .gain = 1, // capture tracks do not have volumes
+ };
+ metadata.channel_mask = mChannelMask;
+ strncpy(metadata.tags, mAttr.tags, AUDIO_ATTRIBUTES_TAGS_MAX_SIZE);
+
+ *backInserter++ = metadata;
+}
// ----------------------------------------------------------------------------
#undef LOG_TAG
@@ -2740,9 +2779,10 @@
void *buffer,
size_t bufferSize,
audio_input_flags_t flags,
- const Timeout& timeout)
+ const Timeout& timeout,
+ audio_source_t source)
: RecordTrack(recordThread, NULL,
- audio_attributes_t{} /* currently unused for patch track */,
+ audio_attributes_t{ .source = source } ,
sampleRate, format, channelMask, frameCount,
buffer, bufferSize, AUDIO_SESSION_NONE, getpid(),
audioServerAttributionSource(getpid()), flags, TYPE_PATCH),
@@ -2853,9 +2893,10 @@
audio_channel_mask_t channelMask,
audio_format_t format,
size_t frameCount,
- audio_input_flags_t flags)
+ audio_input_flags_t flags,
+ audio_source_t source)
: PatchRecord(recordThread, sampleRate, channelMask, format, frameCount,
- nullptr /*buffer*/, 0 /*bufferSize*/, flags),
+ nullptr /*buffer*/, 0 /*bufferSize*/, flags, {} /* timeout */, source),
mPatchRecordAudioBufferProvider(*this),
mSinkBuffer(allocAligned(32, mFrameCount * mFrameSize)),
mStubBuffer(allocAligned(32, mFrameCount * mFrameSize))
diff --git a/services/audiopolicy/common/managerdefinitions/include/AudioOutputDescriptor.h b/services/audiopolicy/common/managerdefinitions/include/AudioOutputDescriptor.h
index 75fa595..0e4bc6c 100644
--- a/services/audiopolicy/common/managerdefinitions/include/AudioOutputDescriptor.h
+++ b/services/audiopolicy/common/managerdefinitions/include/AudioOutputDescriptor.h
@@ -301,6 +301,10 @@
return mActiveClients;
}
+ // Returns 0 if not all active clients have the same exclusive preferred device
+ // or the number of active clients with the same exclusive preferred device
+ size_t sameExclusivePreferredDevicesCount() const;
+
bool useHwGain() const
{
return !devices().isEmpty() ? devices().itemAt(0)->hasGainController() : false;
diff --git a/services/audiopolicy/common/managerdefinitions/include/AudioPolicyMix.h b/services/audiopolicy/common/managerdefinitions/include/AudioPolicyMix.h
index c2a20c6..bb1699e 100644
--- a/services/audiopolicy/common/managerdefinitions/include/AudioPolicyMix.h
+++ b/services/audiopolicy/common/managerdefinitions/include/AudioPolicyMix.h
@@ -70,8 +70,9 @@
* @return OK if the request is valid
* otherwise if the request is not supported
*/
- status_t getOutputForAttr(const audio_attributes_t& attributes, uid_t uid,
- audio_output_flags_t flags,
+ status_t getOutputForAttr(const audio_attributes_t& attributes,
+ const audio_config_base_t& config,
+ uid_t uid, audio_output_flags_t flags,
sp<AudioPolicyMix> &primaryMix,
std::vector<sp<AudioPolicyMix>> *secondaryMixes);
@@ -126,6 +127,7 @@
enum class MixMatchStatus { MATCH, NO_MATCH, INVALID_MIX };
MixMatchStatus mixMatch(const AudioMix* mix, size_t mixIndex,
const audio_attributes_t& attributes,
+ const audio_config_base_t& config,
uid_t uid);
};
diff --git a/services/audiopolicy/common/managerdefinitions/include/ClientDescriptor.h b/services/audiopolicy/common/managerdefinitions/include/ClientDescriptor.h
index 0431619..7119b85 100644
--- a/services/audiopolicy/common/managerdefinitions/include/ClientDescriptor.h
+++ b/services/audiopolicy/common/managerdefinitions/include/ClientDescriptor.h
@@ -239,12 +239,13 @@
}
void setUseSwBridge() { mUseSwBridge = true; }
bool useSwBridge() const { return mUseSwBridge; }
+ bool canCloseOutput() const { return mCloseOutput; }
bool isConnected() const { return mPatchHandle != AUDIO_PATCH_HANDLE_NONE; }
audio_patch_handle_t getPatchHandle() const { return mPatchHandle; }
sp<DeviceDescriptor> srcDevice() const { return mSrcDevice; }
sp<DeviceDescriptor> sinkDevice() const { return mSinkDevice; }
wp<SwAudioOutputDescriptor> swOutput() const { return mSwOutput; }
- void setSwOutput(const sp<SwAudioOutputDescriptor>& swOutput);
+ void setSwOutput(const sp<SwAudioOutputDescriptor>& swOutput, bool closeOutput = false);
wp<HwAudioOutputDescriptor> hwOutput() const { return mHwOutput; }
void setHwOutput(const sp<HwAudioOutputDescriptor>& hwOutput);
@@ -258,6 +259,15 @@
wp<SwAudioOutputDescriptor> mSwOutput;
wp<HwAudioOutputDescriptor> mHwOutput;
bool mUseSwBridge = false;
+ /**
+ * For either HW bridge associated to a SwOutput for activity / volume or SwBridge for also
+ * sample rendering / activity & volume, an existing playback thread may be reused (e.g.
+ * not already opened at APM startup or Direct Output).
+ * If reusing an already opened output, when this output is not used anymore, the AudioFlinger
+ * patch must be updated to refine the output device(s) information and ensure the right
+ * behavior of AudioDeviceCallback.
+ */
+ bool mCloseOutput = false;
};
/**
diff --git a/services/audiopolicy/common/managerdefinitions/src/AudioOutputDescriptor.cpp b/services/audiopolicy/common/managerdefinitions/src/AudioOutputDescriptor.cpp
index 8eefe775..41fbebc 100644
--- a/services/audiopolicy/common/managerdefinitions/src/AudioOutputDescriptor.cpp
+++ b/services/audiopolicy/common/managerdefinitions/src/AudioOutputDescriptor.cpp
@@ -237,6 +237,27 @@
return clients;
}
+size_t AudioOutputDescriptor::sameExclusivePreferredDevicesCount() const
+{
+ audio_port_handle_t deviceId = AUDIO_PORT_HANDLE_NONE;
+ size_t count = 0;
+ for (const auto &client : getClientIterable()) {
+ if (client->active()) {
+ if (!(client->hasPreferredDevice() &&
+ client->isPreferredDeviceForExclusiveUse())) {
+ return 0;
+ }
+ if (deviceId == AUDIO_PORT_HANDLE_NONE) {
+ deviceId = client->preferredDeviceId();
+ } else if (deviceId != client->preferredDeviceId()) {
+ return 0;
+ }
+ count++;
+ }
+ }
+ return count;
+}
+
bool AudioOutputDescriptor::isAnyActive(VolumeSource volumeSourceToIgnore) const
{
return std::find_if(begin(mActiveClients), end(mActiveClients),
diff --git a/services/audiopolicy/common/managerdefinitions/src/AudioPolicyMix.cpp b/services/audiopolicy/common/managerdefinitions/src/AudioPolicyMix.cpp
index 546f56b..e142bef 100644
--- a/services/audiopolicy/common/managerdefinitions/src/AudioPolicyMix.cpp
+++ b/services/audiopolicy/common/managerdefinitions/src/AudioPolicyMix.cpp
@@ -151,7 +151,7 @@
}
status_t AudioPolicyMixCollection::getOutputForAttr(
- const audio_attributes_t& attributes, uid_t uid,
+ const audio_attributes_t& attributes, const audio_config_base_t& config, uid_t uid,
audio_output_flags_t flags,
sp<AudioPolicyMix> &primaryMix,
std::vector<sp<AudioPolicyMix>> *secondaryMixes)
@@ -177,7 +177,7 @@
continue; // Primary output already found
}
- switch (mixMatch(policyMix.get(), i, attributes, uid)) {
+ switch (mixMatch(policyMix.get(), i, attributes, config, uid)) {
case MixMatchStatus::INVALID_MIX:
// The mix has contradictory rules, ignore it
// TODO: reject invalid mix at registration
@@ -202,7 +202,8 @@
}
AudioPolicyMixCollection::MixMatchStatus AudioPolicyMixCollection::mixMatch(
- const AudioMix* mix, size_t mixIndex, const audio_attributes_t& attributes, uid_t uid) {
+ const AudioMix* mix, size_t mixIndex, const audio_attributes_t& attributes,
+ const audio_config_base_t& config, uid_t uid) {
if (mix->mMixType == MIX_TYPE_PLAYERS) {
// Loopback render mixes are created from a public API and thus restricted
@@ -229,6 +230,15 @@
}
}
+ // Permit match only if requested format and mix format are PCM and can be format
+ // adapted by the mixer, or are the same (compressed) format.
+ if (!is_mix_loopback(mix->mRouteFlags) &&
+ !((audio_is_linear_pcm(config.format) && audio_is_linear_pcm(mix->mFormat.format)) ||
+ (config.format == mix->mFormat.format)) &&
+ config.format != AUDIO_CONFIG_BASE_INITIALIZER.format) {
+ return MixMatchStatus::NO_MATCH;
+ }
+
int userId = (int) multiuser_get_user_id(uid);
// TODO if adding more player rules (currently only 2), make rule handling "generic"
diff --git a/services/audiopolicy/common/managerdefinitions/src/ClientDescriptor.cpp b/services/audiopolicy/common/managerdefinitions/src/ClientDescriptor.cpp
index 713b0ac..8b6866e 100644
--- a/services/audiopolicy/common/managerdefinitions/src/ClientDescriptor.cpp
+++ b/services/audiopolicy/common/managerdefinitions/src/ClientDescriptor.cpp
@@ -105,9 +105,11 @@
{
}
-void SourceClientDescriptor::setSwOutput(const sp<SwAudioOutputDescriptor>& swOutput)
+void SourceClientDescriptor::setSwOutput(
+ const sp<SwAudioOutputDescriptor>& swOutput, bool closeOutput)
{
mSwOutput = swOutput;
+ mCloseOutput = closeOutput;
}
void SourceClientDescriptor::setHwOutput(const sp<HwAudioOutputDescriptor>& hwOutput)
diff --git a/services/audiopolicy/config/bluetooth_with_le_audio_policy_configuration_7_0.xml b/services/audiopolicy/config/bluetooth_with_le_audio_policy_configuration_7_0.xml
index e7908eb..c453dea 100644
--- a/services/audiopolicy/config/bluetooth_with_le_audio_policy_configuration_7_0.xml
+++ b/services/audiopolicy/config/bluetooth_with_le_audio_policy_configuration_7_0.xml
@@ -12,6 +12,7 @@
</mixPort>
<!-- Le Audio Audio Ports -->
<mixPort name="le audio output" role="source" />
+ <mixPort name="le audio broadcast output" role="source" />
<mixPort name="le audio input" role="sink">
<profile name="" format="AUDIO_FORMAT_PCM_16_BIT"
samplingRates="8000 16000 24000 32000 44100 48000"
@@ -47,6 +48,7 @@
<devicePort tagName="BLE Headset Out" type="AUDIO_DEVICE_OUT_BLE_HEADSET" role="sink"/>
<devicePort tagName="BLE Speaker Out" type="AUDIO_DEVICE_OUT_BLE_SPEAKER" role="sink"/>
<devicePort tagName="BLE Headset In" type="AUDIO_DEVICE_IN_BLE_HEADSET" role="source"/>
+ <devicePort tagName="BLE Broadcast Out" type="AUDIO_DEVICE_OUT_BLE_BROADCAST" role="sink"/>
</devicePorts>
<routes>
<route type="mix" sink="BT A2DP Out"
@@ -63,5 +65,7 @@
sources="BLE Headset In"/>
<route type="mix" sink="BLE Speaker Out"
sources="le audio output"/>
+ <route type="mix" sink="BLE Broadcast Out"
+ sources="le audio broadcast output"/>
</routes>
</module>
diff --git a/services/audiopolicy/engine/common/include/VolumeGroup.h b/services/audiopolicy/engine/common/include/VolumeGroup.h
index 5378f64..f40ab1c 100644
--- a/services/audiopolicy/engine/common/include/VolumeGroup.h
+++ b/services/audiopolicy/engine/common/include/VolumeGroup.h
@@ -39,7 +39,7 @@
VolumeCurves *getVolumeCurves() { return &mGroupVolumeCurves; }
void addSupportedAttributes(const audio_attributes_t &attr);
- AttributesVector getSupportedAttributes() const { return mGroupVolumeCurves.getAttributes(); }
+ AttributesVector getSupportedAttributes() const;
void addSupportedStream(audio_stream_type_t stream);
StreamTypeVector getStreamTypes() const { return mGroupVolumeCurves.getStreamTypes(); }
diff --git a/services/audiopolicy/engine/common/src/VolumeGroup.cpp b/services/audiopolicy/engine/common/src/VolumeGroup.cpp
index e189807..f5ffbba 100644
--- a/services/audiopolicy/engine/common/src/VolumeGroup.cpp
+++ b/services/audiopolicy/engine/common/src/VolumeGroup.cpp
@@ -37,6 +37,17 @@
{
}
+// Used for introspection, e.g. JAVA
+AttributesVector VolumeGroup::getSupportedAttributes() const
+{
+ AttributesVector supportedAttributes = {};
+ for (auto &aa : mGroupVolumeCurves.getAttributes()) {
+ aa.source = AUDIO_SOURCE_INVALID;
+ supportedAttributes.push_back(aa);
+ }
+ return supportedAttributes;
+}
+
void VolumeGroup::dump(String8 *dst, int spaces) const
{
dst->appendFormat("\n%*s-%s (id: %d)\n", spaces, "", mName.c_str(), mId);
diff --git a/services/audiopolicy/enginedefault/src/Engine.cpp b/services/audiopolicy/enginedefault/src/Engine.cpp
index d4d514d..52b9826 100644
--- a/services/audiopolicy/enginedefault/src/Engine.cpp
+++ b/services/audiopolicy/enginedefault/src/Engine.cpp
@@ -195,6 +195,12 @@
AUDIO_DEVICE_OUT_BLUETOOTH_A2DP, AUDIO_DEVICE_OUT_BLUETOOTH_A2DP_HEADPHONES,
AUDIO_DEVICE_OUT_BLUETOOTH_A2DP_SPEAKER, }));
}
+ // If connected to a dock, never use the device speaker for calls
+ if (!availableOutputDevices.getDevicesFromTypes({AUDIO_DEVICE_OUT_DGTL_DOCK_HEADSET})
+ .isEmpty()) {
+ availableOutputDevices.remove(
+ availableOutputDevices.getDevicesFromTypes({AUDIO_DEVICE_OUT_SPEAKER}));
+ }
} break;
case STRATEGY_ACCESSIBILITY: {
// do not route accessibility prompts to a digital output currently configured with a
@@ -267,10 +273,15 @@
break;
case STRATEGY_PHONE: {
- devices = availableOutputDevices.getDevicesFromType(AUDIO_DEVICE_OUT_HEARING_AID);
- if (!devices.isEmpty()) break;
+ // TODO(b/243670205): remove this logic that gives preference to last removable devices
+ // once a UX decision has been made
devices = availableOutputDevices.getFirstDevicesFromTypes(
- getLastRemovableMediaDevices(GROUP_NONE, {AUDIO_DEVICE_OUT_BLE_HEADSET}));
+ getLastRemovableMediaDevices(GROUP_NONE, {
+ // excluding HEARING_AID and BLE_HEADSET because Dialer uses
+ // setCommunicationDevice to select them explicitly
+ AUDIO_DEVICE_OUT_HEARING_AID,
+ AUDIO_DEVICE_OUT_BLE_HEADSET
+ }));
if (!devices.isEmpty()) break;
devices = availableOutputDevices.getFirstDevicesFromTypes({
AUDIO_DEVICE_OUT_DGTL_DOCK_HEADSET, AUDIO_DEVICE_OUT_EARPIECE});
@@ -285,7 +296,9 @@
if ((strategy == STRATEGY_SONIFICATION) ||
(getForceUse(AUDIO_POLICY_FORCE_FOR_SYSTEM) == AUDIO_POLICY_FORCE_SYSTEM_ENFORCED)) {
- devices = availableOutputDevices.getDevicesFromType(AUDIO_DEVICE_OUT_SPEAKER);
+ // favor dock over speaker when available
+ devices = availableOutputDevices.getFirstDevicesFromTypes({
+ AUDIO_DEVICE_OUT_DGTL_DOCK_HEADSET, AUDIO_DEVICE_OUT_SPEAKER});
}
// if SCO headset is connected and we are told to use it, play ringtone over
diff --git a/services/audiopolicy/managerdefault/AudioPolicyManager.cpp b/services/audiopolicy/managerdefault/AudioPolicyManager.cpp
index 744609f..f625fdb 100644
--- a/services/audiopolicy/managerdefault/AudioPolicyManager.cpp
+++ b/services/audiopolicy/managerdefault/AudioPolicyManager.cpp
@@ -788,7 +788,8 @@
ALOGV("%s between source %s and sink %s", __func__,
srcDevice->toString().c_str(), sinkDevice->toString().c_str());
auto callTxSourceClientPortId = PolicyAudioPort::getNextUniqueId();
- const audio_attributes_t aa = { .source = AUDIO_SOURCE_VOICE_COMMUNICATION };
+ const auto aa = mEngine->getAttributesForStreamType(AUDIO_STREAM_VOICE_CALL);
+
struct audio_port_config source = {};
srcDevice->toAudioPortConfig(&source);
mCallTxSourceClient = new InternalSourceClientDescriptor(
@@ -1054,12 +1055,12 @@
if (curProfile->getFlags() != AUDIO_OUTPUT_FLAG_SPATIALIZER) {
continue;
}
- // reject profiles not corresponding to a device currently available
- DeviceVector supportedDevices = curProfile->getSupportedDevices();
- if (!mAvailableOutputDevices.containsAtLeastOne(supportedDevices)) {
- continue;
- }
if (!devices.empty()) {
+ // reject profiles not corresponding to a device currently available
+ DeviceVector supportedDevices = curProfile->getSupportedDevices();
+ if (!mAvailableOutputDevices.containsAtLeastOne(supportedDevices)) {
+ continue;
+ }
if (supportedDevices.getDevicesFromDeviceTypeAddrVec(devices).size()
!= devices.size()) {
continue;
@@ -1161,7 +1162,12 @@
// otherwise, fallback to the dynamic policies, if none match, query the engine.
// Secondary outputs are always found by dynamic policies as the engine do not support them
sp<AudioPolicyMix> primaryMix;
- status = mPolicyMixes.getOutputForAttr(*resultAttr, uid, *flags, primaryMix, secondaryMixes);
+ const audio_config_base_t clientConfig = {.sample_rate = config->sample_rate,
+ .channel_mask = config->channel_mask,
+ .format = config->format,
+ };
+ status = mPolicyMixes.getOutputForAttr(*resultAttr, clientConfig, uid, *flags, primaryMix,
+ secondaryMixes);
if (status != OK) {
return status;
}
@@ -1170,10 +1176,9 @@
bool usePrimaryOutputFromPolicyMixes = requestedDevice == nullptr && primaryMix != nullptr;
// FIXME: in case of RENDER policy, the output capabilities should be checked
- if ((usePrimaryOutputFromPolicyMixes
- || (secondaryMixes != nullptr && !secondaryMixes->empty()))
- && !audio_is_linear_pcm(config->format)) {
- ALOGD("%s: rejecting request as dynamic audio policy only support pcm", __func__);
+ if ((secondaryMixes != nullptr && !secondaryMixes->empty())
+ && !audio_is_linear_pcm(config->format)) {
+ ALOGD("%s: rejecting request as secondary mixes only support pcm", __func__);
return BAD_VALUE;
}
if (usePrimaryOutputFromPolicyMixes) {
@@ -1182,19 +1187,27 @@
primaryMix->mDeviceAddress,
AUDIO_FORMAT_DEFAULT);
sp<SwAudioOutputDescriptor> policyDesc = primaryMix->getOutput();
- if (deviceDesc != nullptr
- && (policyDesc == nullptr || (policyDesc->mFlags & AUDIO_OUTPUT_FLAG_DIRECT))) {
+ bool tryDirectForFlags = policyDesc == nullptr ||
+ (policyDesc->mFlags & AUDIO_OUTPUT_FLAG_DIRECT);
+ // if a direct output can be opened to deliver the track's multi-channel content to the
+ // output rather than being downmixed by the primary output, then use this direct
+ // output by by-passing the primary mix if possible, otherwise fall-through to primary
+ // mix.
+ bool tryDirectForChannelMask = policyDesc != nullptr
+ && (audio_channel_count_from_out_mask(policyDesc->getConfig().channel_mask) <
+ audio_channel_count_from_out_mask(config->channel_mask));
+ if (deviceDesc != nullptr && (tryDirectForFlags || tryDirectForChannelMask)) {
audio_io_handle_t newOutput;
status = openDirectOutput(
*stream, session, config,
(audio_output_flags_t)(*flags | AUDIO_OUTPUT_FLAG_DIRECT),
DeviceVector(deviceDesc), &newOutput);
- if (status != NO_ERROR) {
- policyDesc = nullptr;
- } else {
+ if (status == NO_ERROR) {
policyDesc = mOutputs.valueFor(newOutput);
primaryMix->setOutput(policyDesc);
- }
+ } else if (tryDirectForFlags) {
+ policyDesc = nullptr;
+ } // otherwise use primary if available.
}
if (policyDesc != nullptr) {
policyDesc->mPolicyMix = primaryMix;
@@ -1868,7 +1881,8 @@
// (see b/200293124, the incorrect selection of AUDIO_OUTPUT_FLAG_VOIP_RX).
// 3: the output supporting the exact channel mask
// 4: the output with a higher channel count than requested
- // 5: the output with a higher sampling rate than requested
+ // 5: the output with the highest sampling rate if the requested sample rate is
+ // greater than default sampling rate
// 6: the output with the highest number of requested performance flags
// 7: the output with the bit depth the closest to the requested one
// 8: the primary output
@@ -1928,8 +1942,7 @@
}
// sampling rate match
- if (samplingRate > SAMPLE_RATE_HZ_DEFAULT &&
- samplingRate <= outputDesc->getSamplingRate()) {
+ if (samplingRate > SAMPLE_RATE_HZ_DEFAULT) {
currentMatchCriteria[4] = outputDesc->getSamplingRate();
}
@@ -1989,6 +2002,10 @@
outputDesc->stop();
return status;
}
+ if (client->hasPreferredDevice()) {
+ // playback activity with preferred device impacts routing occurred, inform upper layers
+ mpClientInterface->onRoutingUpdated();
+ }
if (delayMs != 0) {
usleep(delayMs * 1000);
}
@@ -2071,8 +2088,7 @@
outputDesc->setClientActive(client, true);
if (client->hasPreferredDevice(true)) {
- if (outputDesc->clientsList(true /*activeOnly*/).size() == 1 &&
- client->isPreferredDeviceForExclusiveUse()) {
+ if (outputDesc->sameExclusivePreferredDevicesCount() > 0) {
// Preferred device may be exclusive, use only if no other active clients on this output
devices = DeviceVector(
mAvailableOutputDevices.getDeviceFromId(client->preferredDeviceId()));
@@ -2234,6 +2250,11 @@
}
sp<TrackClientDescriptor> client = outputDesc->getClient(portId);
+ if (client->hasPreferredDevice(true)) {
+ // playback activity with preferred device impacts routing occurred, inform upper layers
+ mpClientInterface->onRoutingUpdated();
+ }
+
ALOGV("stopOutput() output %d, stream %d, session %d",
outputDesc->mIoHandle, client->stream(), client->session());
@@ -2271,7 +2292,8 @@
}
}
bool forceDeviceUpdate = false;
- if (client->hasPreferredDevice(true)) {
+ if (client->hasPreferredDevice(true) &&
+ outputDesc->sameExclusivePreferredDevicesCount() < 2) {
checkStrategyRoute(client->strategy(), AUDIO_IO_HANDLE_NONE);
forceDeviceUpdate = true;
}
@@ -2996,6 +3018,10 @@
status_t status = NO_ERROR;
IVolumeCurves &curves = getVolumeCurves(attributes);
VolumeSource vs = toVolumeSource(group);
+ // AUDIO_STREAM_BLUETOOTH_SCO is only used for volume control so we remap
+ // to AUDIO_STREAM_VOICE_CALL to match with relevant playback activity
+ VolumeSource activityVs = (vs == toVolumeSource(AUDIO_STREAM_BLUETOOTH_SCO, false)) ?
+ toVolumeSource(AUDIO_STREAM_VOICE_CALL, false) : vs;
product_strategy_t strategy = mEngine->getProductStrategyForAttributes(attributes);
status = setVolumeCurveIndex(index, device, curves);
@@ -3034,7 +3060,8 @@
if (curDevices.erase(AUDIO_DEVICE_OUT_SPEAKER_SAFE)) {
curDevices.insert(AUDIO_DEVICE_OUT_SPEAKER);
}
- if (!(desc->isActive(vs) || isInCall())) {
+
+ if (!(desc->isActive(activityVs) || isInCallOrScreening())) {
continue;
}
if (device != AUDIO_DEVICE_OUT_DEFAULT_FOR_VOLUME &&
@@ -3068,7 +3095,7 @@
bool isPreempted = false;
bool isHigherPriority = productStrategy < strategy;
for (const auto &client : activeClients) {
- if (isHigherPriority && (client->volumeSource() != vs)) {
+ if (isHigherPriority && (client->volumeSource() != activityVs)) {
ALOGV("%s: Strategy=%d (\nrequester:\n"
" group %d, volumeGroup=%d attributes=%s)\n"
" higher priority source active:\n"
@@ -3081,7 +3108,7 @@
break;
}
// However, continue for loop to ensure no higher prio clients running on output
- if (client->volumeSource() == vs) {
+ if (client->volumeSource() == activityVs) {
applyVolume = true;
}
}
@@ -4081,6 +4108,9 @@
status_t AudioPolicyManager::getDirectProfilesForAttributes(const audio_attributes_t* attr,
AudioProfileVector& audioProfilesVector) {
+ if (mEffects.isNonOffloadableEffectEnabled()) {
+ return OK;
+ }
AudioDeviceTypeAddrVector devices;
status_t status = getDevicesForAttributes(*attr, &devices, false /* forVolume */);
if (status != OK) {
@@ -4519,7 +4549,7 @@
// In case of Hw bridge, it is a Work Around. The mixPort used is the one declared
// in config XML to reach the sink so that is can be declared as available.
audio_io_handle_t output = AUDIO_IO_HANDLE_NONE;
- sp<SwAudioOutputDescriptor> outputDesc = nullptr;
+ sp<SwAudioOutputDescriptor> outputDesc;
if (!sourceDesc->isInternal()) {
// take care of dynamic routing for SwOutput selection,
audio_attributes_t attributes = sourceDesc->attributes();
@@ -4548,7 +4578,8 @@
ALOGE("%s output is duplicated", __func__);
return INVALID_OPERATION;
}
- sourceDesc->setSwOutput(outputDesc);
+ bool closeOutput = outputDesc->mDirectOpenCount != 0;
+ sourceDesc->setSwOutput(outputDesc, closeOutput);
} else {
// Same for "raw patches" aka created from createAudioPatch API
SortedVector<audio_io_handle_t> outputs =
@@ -4567,7 +4598,7 @@
__func__, sinkDevice->toString().c_str());
return INVALID_OPERATION;
}
- sourceDesc->setSwOutput(outputDesc);
+ sourceDesc->setSwOutput(outputDesc, /* closeOutput= */ false);
}
// create a software bridge in PatchPanel if:
// - source and sink devices are on different HW modules OR
@@ -4589,7 +4620,8 @@
audio_port_config srcMixPortConfig = {};
outputDesc->toAudioPortConfig(&srcMixPortConfig, nullptr);
// for volume control, we may need a valid stream
- srcMixPortConfig.ext.mix.usecase.stream = !sourceDesc->isInternal() ?
+ srcMixPortConfig.ext.mix.usecase.stream =
+ (!sourceDesc->isInternal() || isCallTxAudioSource(sourceDesc)) ?
mEngine->getStreamTypeForAttributes(sourceDesc->attributes()) :
AUDIO_STREAM_PATCH;
patchBuilder.addSource(srcMixPortConfig);
@@ -4696,17 +4728,29 @@
// releaseOutput has already called closeOutput in case of direct output
return NO_ERROR;
}
- if (!outputDesc->isActive() && !sourceDesc->useSwBridge()) {
- resetOutputDevice(outputDesc);
- } else {
- // Reuse patch handle if still valid / do not force rerouting if still routed
- patchHandle = outputDesc->getPatchHandle();
- setOutputDevices(outputDesc,
- getNewOutputDevices(outputDesc, true /*fromCache*/),
- patchHandle == AUDIO_PATCH_HANDLE_NONE, /*force*/
- 0,
- patchHandle == AUDIO_PATCH_HANDLE_NONE ? nullptr : &patchHandle);
- }
+ patchHandle = outputDesc->getPatchHandle();
+ // When a Sw bridge is released, the mixer used by this bridge will release its
+ // patch at AudioFlinger side. Hence, the mixer audio patch must be recreated
+ // Reuse patch handle to force audio flinger removing initial mixer patch removal
+ // updating hal patch handle (prevent leaks).
+ // While using a HwBridge, force reconsidering device only if not reusing an existing
+ // output and no more activity on output (will force to close).
+ bool force = sourceDesc->useSwBridge() ||
+ (sourceDesc->canCloseOutput() && !outputDesc->isActive());
+ // APM pattern is to have always outputs opened / patch realized for reachable devices.
+ // Update device may result to NONE (empty), coupled with force, it releases the patch.
+ // Reconsider device only for cases:
+ // 1 / Active Output
+ // 2 / Inactive Output previously hosting HwBridge
+ // 3 / Inactive Output previously hosting SwBridge that can be closed.
+ bool updateDevice = outputDesc->isActive() || !sourceDesc->useSwBridge() ||
+ sourceDesc->canCloseOutput();
+ setOutputDevices(outputDesc,
+ updateDevice ? getNewOutputDevices(outputDesc, true /*fromCache*/) :
+ outputDesc->devices(),
+ force,
+ 0,
+ patchHandle == AUDIO_PATCH_HANDLE_NONE ? nullptr : &patchHandle);
} else {
return BAD_VALUE;
}
@@ -5353,10 +5397,6 @@
}
}
- // The caller can have the devices criteria ignored by passing and empty vector, and
- // getSpatializerOutputProfile() will ignore the devices when looking for a match.
- // Otherwise an output profile supporting a spatializer effect that can be routed
- // to the specified devices must exist.
sp<IOProfile> profile =
getSpatializerOutputProfile(config, devices);
if (profile == nullptr) {
@@ -6317,10 +6357,10 @@
SortedVector<audio_io_handle_t> dstOutputs = getOutputsForDevices(newDevices, mOutputs);
uint32_t maxLatency = 0;
- bool invalidate = false;
+ std::vector<sp<SwAudioOutputDescriptor>> invalidatedOutputs;
// take into account dynamic audio policies related changes: if a client is now associated
// to a different policy mix than at creation time, invalidate corresponding stream
- for (size_t i = 0; i < mPreviousOutputs.size() && !invalidate; i++) {
+ for (size_t i = 0; i < mPreviousOutputs.size(); i++) {
const sp<SwAudioOutputDescriptor>& desc = mPreviousOutputs.valueAt(i);
if (desc->isDuplicated()) {
continue;
@@ -6330,22 +6370,21 @@
continue;
}
sp<AudioPolicyMix> primaryMix;
- status_t status = mPolicyMixes.getOutputForAttr(client->attributes(), client->uid(),
- client->flags(), primaryMix, nullptr);
+ status_t status = mPolicyMixes.getOutputForAttr(client->attributes(), client->config(),
+ client->uid(), client->flags(), primaryMix, nullptr);
if (status != OK) {
continue;
}
if (client->getPrimaryMix() != primaryMix || client->hasLostPrimaryMix()) {
- invalidate = true;
- if (desc->isStrategyActive(psId)) {
+ if (desc->isStrategyActive(psId) && maxLatency < desc->latency()) {
maxLatency = desc->latency();
}
- break;
+ invalidatedOutputs.push_back(desc);
}
}
}
- if (srcOutputs != dstOutputs || invalidate) {
+ if (srcOutputs != dstOutputs || !invalidatedOutputs.empty()) {
// get maximum latency of all source outputs to determine the minimum mute time guaranteeing
// audio from invalidated tracks will be rendered when unmuting
for (audio_io_handle_t srcOut : srcOutputs) {
@@ -6356,8 +6395,7 @@
maxLatency = desc->latency();
}
- if (invalidate) continue;
-
+ bool invalidate = false;
for (auto client : desc->clientsList(false /*activeOnly*/)) {
if (desc->isDuplicated() || !desc->mProfile->isDirectOutput()) {
// a client on a non direct outputs has necessarily a linear PCM format
@@ -6385,21 +6423,14 @@
}
}
}
- }
-
- ALOGV_IF(!(srcOutputs.isEmpty() || dstOutputs.isEmpty()),
- "%s: strategy %d, moving from output %s to output %s", __func__, psId,
- std::to_string(srcOutputs[0]).c_str(),
- std::to_string(dstOutputs[0]).c_str());
- // mute strategy while moving tracks from one output to another
- for (audio_io_handle_t srcOut : srcOutputs) {
- sp<SwAudioOutputDescriptor> desc = mPreviousOutputs.valueFor(srcOut);
- if (desc == nullptr) continue;
-
- if (desc->isStrategyActive(psId)) {
- setStrategyMute(psId, true, desc);
- setStrategyMute(psId, false, desc, maxLatency * LATENCY_MUTE_FACTOR,
- newDevices.types());
+ // mute strategy while moving tracks from one output to another
+ if (invalidate) {
+ invalidatedOutputs.push_back(desc);
+ if (desc->isStrategyActive(psId)) {
+ setStrategyMute(psId, true, desc);
+ setStrategyMute(psId, false, desc, maxLatency * LATENCY_MUTE_FACTOR,
+ newDevices.types());
+ }
}
sp<SourceClientDescriptor> source = getSourceForAttributesOnOutput(srcOut, attr);
if (source != nullptr && !isCallRxAudioSource(source) && !source->isInternal()) {
@@ -6407,19 +6438,21 @@
}
}
+ ALOGV_IF(!(srcOutputs.isEmpty() || dstOutputs.isEmpty()),
+ "%s: strategy %d, moving from output %s to output %s", __func__, psId,
+ std::to_string(srcOutputs[0]).c_str(),
+ std::to_string(dstOutputs[0]).c_str());
+
// Move effects associated to this stream from previous output to new output
if (followsSameRouting(attr, attributes_initializer(AUDIO_USAGE_MEDIA))) {
selectOutputForMusicEffects();
}
// Move tracks associated to this stream (and linked) from previous output to new output
- if (invalidate) {
+ if (!invalidatedOutputs.empty()) {
for (auto stream : mEngine->getStreamTypesForProductStrategy(psId)) {
mpClientInterface->invalidateStream(stream);
}
- for (audio_io_handle_t srcOut : srcOutputs) {
- sp<SwAudioOutputDescriptor> desc = mPreviousOutputs.valueFor(srcOut);
- if (desc == nullptr) continue;
-
+ for (sp<SwAudioOutputDescriptor> desc : invalidatedOutputs) {
desc->setTracksInvalidatedStatusByStrategy(psId);
}
}
@@ -6443,8 +6476,8 @@
for (const sp<TrackClientDescriptor>& client : outputDescriptor->getClientIterable()) {
sp<AudioPolicyMix> primaryMix;
std::vector<sp<AudioPolicyMix>> secondaryMixes;
- status_t status = mPolicyMixes.getOutputForAttr(client->attributes(), client->uid(),
- client->flags(), primaryMix, &secondaryMixes);
+ status_t status = mPolicyMixes.getOutputForAttr(client->attributes(), client->config(),
+ client->uid(), client->flags(), primaryMix, &secondaryMixes);
std::vector<sp<SwAudioOutputDescriptor>> secondaryDescs;
for (auto &secondaryMix : secondaryMixes) {
sp<SwAudioOutputDescriptor> outputDesc = secondaryMix->getOutput();
@@ -6702,8 +6735,8 @@
// check dynamic policies but only for primary descriptors (secondary not used for audible
// audio routing, only used for duplication for playback capture)
sp<AudioPolicyMix> policyMix;
- status_t status = mPolicyMixes.getOutputForAttr(attr, 0 /*uid unknown here*/,
- AUDIO_OUTPUT_FLAG_NONE, policyMix, nullptr /* secondaryMixes */);
+ status_t status = mPolicyMixes.getOutputForAttr(attr, AUDIO_CONFIG_BASE_INITIALIZER,
+ 0 /*uid unknown here*/, AUDIO_OUTPUT_FLAG_NONE, policyMix, nullptr);
if (status != OK) {
return status;
}
@@ -7480,14 +7513,18 @@
return is_state_in_call(state);
}
-bool AudioPolicyManager::isCallAudioAccessible()
-{
+bool AudioPolicyManager::isCallAudioAccessible() const {
audio_mode_t mode = mEngine->getPhoneState();
return (mode == AUDIO_MODE_IN_CALL)
|| (mode == AUDIO_MODE_CALL_SCREEN)
|| (mode == AUDIO_MODE_CALL_REDIRECT);
}
+bool AudioPolicyManager::isInCallOrScreening() const {
+ audio_mode_t mode = mEngine->getPhoneState();
+ return isStateInCall(mode) || mode == AUDIO_MODE_CALL_SCREEN;
+}
+
void AudioPolicyManager::cleanUpForDevice(const sp<DeviceDescriptor>& deviceDesc)
{
for (ssize_t i = (ssize_t)mAudioSources.size() - 1; i >= 0; i--) {
diff --git a/services/audiopolicy/managerdefault/AudioPolicyManager.h b/services/audiopolicy/managerdefault/AudioPolicyManager.h
index db0ee15..a69e088 100644
--- a/services/audiopolicy/managerdefault/AudioPolicyManager.h
+++ b/services/audiopolicy/managerdefault/AudioPolicyManager.h
@@ -602,7 +602,9 @@
// true if given state represents a device in a telephony or VoIP call
virtual bool isStateInCall(int state) const;
// true if playback to call TX or capture from call RX is possible
- bool isCallAudioAccessible();
+ bool isCallAudioAccessible() const;
+ // true if device is in a telephony or VoIP call or call screening is active
+ bool isInCallOrScreening() const;
// when a device is connected, checks if an open output can be routed
// to this device. If none is open, tries to open one of the available outputs.
@@ -639,6 +641,10 @@
return mCallRxSourceClient != nullptr && source == mCallRxSourceClient;
}
+ bool isCallTxAudioSource(const sp<SourceClientDescriptor> &source) {
+ return mCallTxSourceClient != nullptr && source == mCallTxSourceClient;
+ }
+
void connectTelephonyRxAudioSource();
void disconnectTelephonyAudioSource(sp<SourceClientDescriptor> &clientDesc);
@@ -1099,6 +1105,18 @@
const audio_config_t *config,
const AudioDeviceTypeAddrVector &devices) const;
+
+ /**
+ * @brief Gets an IOProfile for a spatializer output with the best match with
+ * provided arguments.
+ * The caller can have the devices criteria ignored by passing and empty vector, and
+ * getSpatializerOutputProfile() will ignore the devices when looking for a match.
+ * Otherwise an output profile supporting a spatializer effect that can be routed
+ * to the specified devices must exist.
+ * @param config audio configuration describing the audio format, channels, sample rate...
+ * @param devices the sink audio device selected for playback
+ * @return an IOProfile that canbe used to open a spatializer output.
+ */
sp<IOProfile> getSpatializerOutputProfile(const audio_config_t *config,
const AudioDeviceTypeAddrVector &devices) const;
diff --git a/services/audiopolicy/service/AudioPolicyInterfaceImpl.cpp b/services/audiopolicy/service/AudioPolicyInterfaceImpl.cpp
index 5dd9b8c..140cabb 100644
--- a/services/audiopolicy/service/AudioPolicyInterfaceImpl.cpp
+++ b/services/audiopolicy/service/AudioPolicyInterfaceImpl.cpp
@@ -301,7 +301,8 @@
audio_stream_type_t stream = VALUE_OR_RETURN_BINDER_STATUS(
aidl2legacy_AudioStreamType_audio_stream_type_t(streamAidl));
- if (uint32_t(stream) >= AUDIO_STREAM_PUBLIC_CNT) {
+ if (uint32_t(stream) >= AUDIO_STREAM_PUBLIC_CNT
+ && stream != AUDIO_STREAM_ASSISTANT && stream != AUDIO_STREAM_CALL_ASSISTANT) {
*_aidl_return = VALUE_OR_RETURN_BINDER_STATUS(
legacy2aidl_audio_io_handle_t_int32_t(AUDIO_IO_HANDLE_NONE));
return Status::ok();
diff --git a/services/audiopolicy/service/AudioPolicyService.cpp b/services/audiopolicy/service/AudioPolicyService.cpp
index e7d945f..21fa82e 100644
--- a/services/audiopolicy/service/AudioPolicyService.cpp
+++ b/services/audiopolicy/service/AudioPolicyService.cpp
@@ -202,6 +202,7 @@
mCaptureStateNotifier(false),
mCreateAudioPolicyManager(createAudioPolicyManager),
mDestroyAudioPolicyManager(destroyAudioPolicyManager) {
+ setMinSchedulerPolicy(SCHED_NORMAL, ANDROID_PRIORITY_AUDIO);
}
void AudioPolicyService::loadAudioPolicyManager()
@@ -272,6 +273,11 @@
if (hasSpatializer) {
mSpatializer = Spatializer::create(this);
}
+ if (mSpatializer == nullptr) {
+ // No spatializer created, signal the reason: NO_INIT a failure, OK means intended.
+ const status_t createStatus = hasSpatializer ? NO_INIT : OK;
+ Spatializer::sendEmptyCreateSpatializerMetricWithStatus(createStatus);
+ }
}
AudioSystem::audioPolicyReady();
}
@@ -516,9 +522,8 @@
void AudioPolicyService::doOnCheckSpatializer()
{
- Mutex::Autolock _l(mLock);
-
- ALOGI("%s mSpatializer %p level %d", __func__, mSpatializer.get(), (int)mSpatializer->getLevel());
+ ALOGV("%s mSpatializer %p level %d",
+ __func__, mSpatializer.get(), (int)mSpatializer->getLevel());
if (mSpatializer != nullptr) {
// Note: mSpatializer != nullptr => mAudioPolicyManager != nullptr
@@ -527,6 +532,8 @@
audio_io_handle_t newOutput;
const audio_attributes_t attr = attributes_initializer(AUDIO_USAGE_MEDIA);
audio_config_base_t config = mSpatializer->getAudioInConfig();
+
+ Mutex::Autolock _l(mLock);
status_t status =
mAudioPolicyManager->getSpatializerOutput(&config, &attr, &newOutput);
ALOGV("%s currentOutput %d newOutput %d channel_mask %#x",
@@ -538,21 +545,19 @@
mLock.unlock();
// It is OK to call detachOutput() is none is already attached.
mSpatializer->detachOutput();
- if (status != NO_ERROR || newOutput == AUDIO_IO_HANDLE_NONE) {
- mLock.lock();
- return;
+ if (status == NO_ERROR && newOutput != AUDIO_IO_HANDLE_NONE) {
+ status = mSpatializer->attachOutput(newOutput, numActiveTracks);
}
- status = mSpatializer->attachOutput(newOutput, numActiveTracks);
mLock.lock();
if (status != NO_ERROR) {
mAudioPolicyManager->releaseSpatializerOutput(newOutput);
}
} else if (mSpatializer->getLevel() == media::SpatializationLevel::NONE
&& mSpatializer->getOutput() != AUDIO_IO_HANDLE_NONE) {
- mLock.unlock();
audio_io_handle_t output = mSpatializer->detachOutput();
- mLock.lock();
+
if (output != AUDIO_IO_HANDLE_NONE) {
+ Mutex::Autolock _l(mLock);
mAudioPolicyManager->releaseSpatializerOutput(output);
}
}
@@ -581,19 +586,16 @@
void AudioPolicyService::doOnUpdateActiveSpatializerTracks()
{
- sp<Spatializer> spatializer;
+ if (mSpatializer == nullptr) {
+ return;
+ }
+ audio_io_handle_t output = mSpatializer->getOutput();
size_t activeClients;
{
Mutex::Autolock _l(mLock);
- if (mSpatializer == nullptr) {
- return;
- }
- spatializer = mSpatializer;
- activeClients = countActiveClientsOnOutput_l(mSpatializer->getOutput());
+ activeClients = countActiveClientsOnOutput_l(output);
}
- if (spatializer != nullptr) {
- spatializer->updateActiveTracks(activeClients);
- }
+ mSpatializer->updateActiveTracks(activeClients);
}
status_t AudioPolicyService::clientCreateAudioPatch(const struct audio_patch *patch,
@@ -1215,6 +1217,14 @@
dumpReleaseLock(mLock, locked);
+ if (mSpatializer != nullptr) {
+ std::string dumpString = mSpatializer->toString(1 /* level */);
+ write(fd, dumpString.c_str(), dumpString.size());
+ } else {
+ String8 spatializerPtr = String8::format("Spatializer no supportted on this device\n");
+ write(fd, spatializerPtr.c_str(), spatializerPtr.size());
+ }
+
{
std::string timeCheckStats = getIAudioPolicyServiceStatistics().dump();
dprintf(fd, "\nIAudioPolicyService binder call profile\n");
@@ -1338,7 +1348,9 @@
} else {
getIAudioPolicyServiceStatistics().event(code, elapsedMs);
}
- });
+ }, mediautils::TimeCheck::kDefaultTimeoutDuration,
+ mediautils::TimeCheck::kDefaultSecondChanceDuration,
+ true /* crashOnTimeout */);
switch (code) {
case SHELL_COMMAND_TRANSACTION: {
diff --git a/services/audiopolicy/service/AudioPolicyService.h b/services/audiopolicy/service/AudioPolicyService.h
index 3a08cf8..a87d871 100644
--- a/services/audiopolicy/service/AudioPolicyService.h
+++ b/services/audiopolicy/service/AudioPolicyService.h
@@ -1060,6 +1060,7 @@
CaptureStateNotifier mCaptureStateNotifier;
+ // created in onFirstRef() and never cleared: does not need to be guarded by mLock
sp<Spatializer> mSpatializer;
void *mLibraryHandle = nullptr;
diff --git a/services/audiopolicy/service/Spatializer.cpp b/services/audiopolicy/service/Spatializer.cpp
index c199a76..5db82f7 100644
--- a/services/audiopolicy/service/Spatializer.cpp
+++ b/services/audiopolicy/service/Spatializer.cpp
@@ -15,11 +15,13 @@
** limitations under the License.
*/
-
+#include <string>
#define LOG_TAG "Spatializer"
//#define LOG_NDEBUG 0
#include <utils/Log.h>
+#include <algorithm>
+#include <inttypes.h>
#include <limits.h>
#include <stdint.h>
#include <sys/types.h>
@@ -32,7 +34,9 @@
#include <media/stagefright/foundation/AHandler.h>
#include <media/stagefright/foundation/AMessage.h>
#include <media/MediaMetricsItem.h>
+#include <media/QuaternionUtil.h>
#include <media/ShmemCompat.h>
+#include <mediautils/SchedulingPolicyService.h>
#include <mediautils/ServiceUtilities.h>
#include <utils/Thread.h>
@@ -58,11 +62,13 @@
if (!_tmp.ok()) return aidl_utils::binderStatusFromStatusT(_tmp.error()); \
std::move(_tmp.value()); })
-audio_channel_mask_t getMaxChannelMask(std::vector<audio_channel_mask_t> masks) {
+static audio_channel_mask_t getMaxChannelMask(
+ const std::vector<audio_channel_mask_t>& masks, size_t channelLimit = SIZE_MAX) {
uint32_t maxCount = 0;
audio_channel_mask_t maxMask = AUDIO_CHANNEL_NONE;
for (auto mask : masks) {
const size_t count = audio_channel_count_from_out_mask(mask);
+ if (count > channelLimit) continue; // ignore masks greater than channelLimit
if (count > maxCount) {
maxMask = mask;
maxCount = count;
@@ -71,6 +77,34 @@
return maxMask;
}
+static std::vector<float> recordFromTranslationRotationVector(
+ const std::vector<float>& trVector) {
+ auto headToStageOpt = Pose3f::fromVector(trVector);
+ if (!headToStageOpt) return {};
+
+ const auto stageToHead = headToStageOpt.value().inverse();
+ const auto stageToHeadTranslation = stageToHead.translation();
+ constexpr float RAD_TO_DEGREE = 180.f / M_PI;
+ std::vector<float> record{
+ stageToHeadTranslation[0], stageToHeadTranslation[1], stageToHeadTranslation[2],
+ 0.f, 0.f, 0.f};
+ media::quaternionToAngles(stageToHead.rotation(), &record[3], &record[4], &record[5]);
+ record[3] *= RAD_TO_DEGREE;
+ record[4] *= RAD_TO_DEGREE;
+ record[5] *= RAD_TO_DEGREE;
+ return record;
+}
+
+template<typename T>
+static constexpr const T& safe_clamp(const T& value, const T& low, const T& high) {
+ if constexpr (std::is_floating_point_v<T>) {
+ return value != value /* constexpr isnan */
+ ? low : std::clamp(value, low, high);
+ } else /* constexpr */ {
+ return std::clamp(value, low, high);
+ }
+}
+
// ---------------------------------------------------------------------------
class Spatializer::EngineCallbackHandler : public AHandler {
@@ -84,6 +118,7 @@
kWhatOnFramesProcessed, // AudioEffect::EVENT_FRAMES_PROCESSED
kWhatOnHeadToStagePose, // SpatializerPoseController::Listener::onHeadToStagePose
kWhatOnActualModeChange, // SpatializerPoseController::Listener::onActualModeChange
+ kWhatOnLatencyModesChanged, // Spatializer::onSupportedLatencyModesChanged
};
static constexpr const char *kNumFramesKey = "numFrames";
static constexpr const char *kModeKey = "mode";
@@ -93,15 +128,35 @@
static constexpr const char *kRotation0Key = "rotation0";
static constexpr const char *kRotation1Key = "rotation1";
static constexpr const char *kRotation2Key = "rotation2";
+ static constexpr const char *kLatencyModesKey = "latencyModes";
+
+ class LatencyModes : public RefBase {
+ public:
+ LatencyModes(audio_io_handle_t output,
+ const std::vector<audio_latency_mode_t>& latencyModes)
+ : mOutput(output), mLatencyModes(latencyModes) {}
+ ~LatencyModes() = default;
+
+ audio_io_handle_t mOutput;
+ std::vector<audio_latency_mode_t> mLatencyModes;
+ };
void onMessageReceived(const sp<AMessage> &msg) override {
+ // No ALooper method to get the tid so update
+ // Spatializer priority on the first message received.
+ std::call_once(mPrioritySetFlag, [](){
+ const pid_t pid = getpid();
+ const pid_t tid = gettid();
+ (void)requestSpatializerPriority(pid, tid);
+ });
+
+ sp<Spatializer> spatializer = mSpatializer.promote();
+ if (spatializer == nullptr) {
+ ALOGW("%s: Cannot promote spatializer", __func__);
+ return;
+ }
switch (msg->what()) {
case kWhatOnFramesProcessed: {
- sp<Spatializer> spatializer = mSpatializer.promote();
- if (spatializer == nullptr) {
- ALOGW("%s: Cannot promote spatializer", __func__);
- return;
- }
int numFrames;
if (!msg->findInt32(kNumFramesKey, &numFrames)) {
ALOGE("%s: Cannot find num frames!", __func__);
@@ -112,11 +167,6 @@
}
} break;
case kWhatOnHeadToStagePose: {
- sp<Spatializer> spatializer = mSpatializer.promote();
- if (spatializer == nullptr) {
- ALOGW("%s: Cannot promote spatializer", __func__);
- return;
- }
std::vector<float> headToStage(sHeadPoseKeys.size());
for (size_t i = 0 ; i < sHeadPoseKeys.size(); i++) {
if (!msg->findFloat(sHeadPoseKeys[i], &headToStage[i])) {
@@ -127,24 +177,32 @@
spatializer->onHeadToStagePoseMsg(headToStage);
} break;
case kWhatOnActualModeChange: {
- sp<Spatializer> spatializer = mSpatializer.promote();
- if (spatializer == nullptr) {
- ALOGW("%s: Cannot promote spatializer", __func__);
- return;
- }
int mode;
- if (!msg->findInt32(EngineCallbackHandler::kModeKey, &mode)) {
+ if (!msg->findInt32(kModeKey, &mode)) {
ALOGE("%s: Cannot find actualMode!", __func__);
return;
}
spatializer->onActualModeChangeMsg(static_cast<HeadTrackingMode>(mode));
} break;
+
+ case kWhatOnLatencyModesChanged: {
+ sp<RefBase> object;
+ if (!msg->findObject(kLatencyModesKey, &object)) {
+ ALOGE("%s: Cannot find latency modes!", __func__);
+ return;
+ }
+ sp<LatencyModes> latencyModes = static_cast<LatencyModes*>(object.get());
+ spatializer->onSupportedLatencyModesChangedMsg(
+ latencyModes->mOutput, std::move(latencyModes->mLatencyModes));
+ } break;
+
default:
LOG_ALWAYS_FATAL("Invalid callback message %d", msg->what());
}
}
private:
wp<Spatializer> mSpatializer;
+ std::once_flag mPrioritySetFlag;
};
const std::vector<const char *> Spatializer::sHeadPoseKeys = {
@@ -176,17 +234,22 @@
ALOG_ASSERT(!descriptors.empty(),
"%s getDescriptors() returned no error but empty list", __func__);
- //TODO: get supported spatialization modes from FX engine or descriptor
-
+ // TODO: get supported spatialization modes from FX engine or descriptor
sp<EffectHalInterface> effect;
status = effectsFactoryHal->createEffect(&descriptors[0].uuid, AUDIO_SESSION_OUTPUT_STAGE,
AUDIO_IO_HANDLE_NONE, AUDIO_PORT_HANDLE_NONE, &effect);
- ALOGI("%s FX create status %d effect %p", __func__, status, effect.get());
+ ALOGI("%s FX create status %d effect ID %" PRId64, __func__, status,
+ effect ? effect->effectId() : 0);
if (status == NO_ERROR && effect != nullptr) {
spatializer = new Spatializer(descriptors[0], callback);
if (spatializer->loadEngineConfiguration(effect) != NO_ERROR) {
spatializer.clear();
+ ALOGW("%s loadEngine error: %d effect Id %" PRId64,
+ __func__, status, effect ? effect->effectId() : 0);
+ } else {
+ spatializer->mLocalLog.log("%s with effect Id %" PRId64, __func__,
+ effect ? effect->effectId() : 0);
}
}
@@ -197,6 +260,7 @@
: mEngineDescriptor(engineDescriptor),
mPolicyCallback(callback) {
ALOGV("%s", __func__);
+ setMinSchedulerPolicy(SCHED_NORMAL, ANDROID_PRIORITY_AUDIO);
}
void Spatializer::onFirstRef() {
@@ -205,7 +269,7 @@
mLooper->start(
/*runOnCallingThread*/false,
/*canCallJava*/ false,
- PRIORITY_AUDIO);
+ PRIORITY_URGENT_AUDIO);
mHandler = new EngineCallbackHandler(this);
mLooper->registerHandler(mHandler);
@@ -221,6 +285,16 @@
mHandler.clear();
}
+static std::string channelMaskVectorToString(
+ const std::vector<audio_channel_mask_t>& masks) {
+ std::stringstream ss;
+ for (const auto &mask : masks) {
+ if (ss.tellp() != 0) ss << "|";
+ ss << mask;
+ }
+ return ss.str();
+}
+
status_t Spatializer::loadEngineConfiguration(sp<EffectHalInterface> effect) {
ALOGV("%s", __func__);
@@ -268,6 +342,7 @@
ALOGW("%s: cannot get SPATIALIZER_PARAM_SUPPORTED_SPATIALIZATION_MODES", __func__);
return status;
}
+
for (const auto spatializationMode : spatializationModes) {
if (!aidl_utils::isValidEnum(spatializationMode)) {
ALOGW("%s: ignoring spatializationMode:%d", __func__, (int)spatializationMode);
@@ -309,7 +384,7 @@
}
mediametrics::LogItem(mMetricsId)
.set(AMEDIAMETRICS_PROP_EVENT, AMEDIAMETRICS_PROP_EVENT_VALUE_CREATE)
- .set(AMEDIAMETRICS_PROP_CHANNELMASK, (int32_t)getMaxChannelMask(mChannelMasks))
+ .set(AMEDIAMETRICS_PROP_CHANNELMASKS, channelMaskVectorToString(mChannelMasks))
.set(AMEDIAMETRICS_PROP_LEVELS, aidl_utils::enumsToString(mLevels))
.set(AMEDIAMETRICS_PROP_MODES, aidl_utils::enumsToString(mSpatializationModes))
.set(AMEDIAMETRICS_PROP_HEADTRACKINGMODES, aidl_utils::enumsToString(mHeadTrackingModes))
@@ -318,12 +393,24 @@
return NO_ERROR;
}
+/* static */
+void Spatializer::sendEmptyCreateSpatializerMetricWithStatus(status_t status) {
+ mediametrics::LogItem(kDefaultMetricsId)
+ .set(AMEDIAMETRICS_PROP_EVENT, AMEDIAMETRICS_PROP_EVENT_VALUE_CREATE)
+ .set(AMEDIAMETRICS_PROP_CHANNELMASKS, "")
+ .set(AMEDIAMETRICS_PROP_LEVELS, "")
+ .set(AMEDIAMETRICS_PROP_MODES, "")
+ .set(AMEDIAMETRICS_PROP_HEADTRACKINGMODES, "")
+ .set(AMEDIAMETRICS_PROP_STATUS, (int32_t)status)
+ .record();
+}
+
/** Gets the channel mask, sampling rate and format set for the spatializer input. */
audio_config_base_t Spatializer::getAudioInConfig() const {
std::lock_guard lock(mLock);
audio_config_base_t config = AUDIO_CONFIG_BASE_INITIALIZER;
// For now use highest supported channel count
- config.channel_mask = getMaxChannelMask(mChannelMasks);
+ config.channel_mask = getMaxChannelMask(mChannelMasks, FCC_LIMIT);
return config;
}
@@ -334,6 +421,17 @@
return BAD_VALUE;
}
+ if (mSpatializerCallback != nullptr) {
+ if (IInterface::asBinder(callback) == IInterface::asBinder(mSpatializerCallback)) {
+ ALOGW("%s: Registering callback %p again",
+ __func__, mSpatializerCallback.get());
+ return NO_ERROR;
+ }
+ ALOGE("%s: Already one client registered with callback %p",
+ __func__, mSpatializerCallback.get());
+ return INVALID_OPERATION;
+ }
+
sp<IBinder> binder = IInterface::asBinder(callback);
status_t status = binder->linkToDeath(this);
if (status == NO_ERROR) {
@@ -366,7 +464,8 @@
}
Status Spatializer::setLevel(SpatializationLevel level) {
- ALOGV("%s level %d", __func__, (int)level);
+ ALOGV("%s level %s", __func__, media::toString(level).c_str());
+ mLocalLog.log("%s with %s", __func__, media::toString(level).c_str());
if (level != SpatializationLevel::NONE
&& std::find(mLevels.begin(), mLevels.end(), level) == mLevels.end()) {
return binderStatusFromStatusT(BAD_VALUE);
@@ -426,11 +525,12 @@
}
Status Spatializer::setDesiredHeadTrackingMode(SpatializerHeadTrackingMode mode) {
- ALOGV("%s mode %d", __func__, (int)mode);
+ ALOGV("%s mode %s", __func__, media::toString(mode).c_str());
if (!mSupportsHeadTracking) {
return binderStatusFromStatusT(INVALID_OPERATION);
}
+ mLocalLog.log("%s with %s", __func__, media::toString(mode).c_str());
std::lock_guard lock(mLock);
switch (mode) {
case SpatializerHeadTrackingMode::OTHER:
@@ -485,6 +585,8 @@
}
std::lock_guard lock(mLock);
if (mPoseController != nullptr) {
+ mLocalLog.log("%s with screenToStage %s", __func__,
+ media::VectorRecorder::toString<float>(screenToStage).c_str());
mPoseController->setScreenToStagePose(maybePose.value());
}
return Status::ok();
@@ -520,6 +622,7 @@
}
std::lock_guard lock(mLock);
if (mHeadSensor != sensorHandle) {
+ mLocalLog.log("%s with 0x%08x", __func__, sensorHandle);
mHeadSensor = sensorHandle;
checkPoseController_l();
checkSensorsState_l();
@@ -534,6 +637,7 @@
}
std::lock_guard lock(mLock);
if (mScreenSensor != sensorHandle) {
+ mLocalLog.log("%s with 0x%08x", __func__, sensorHandle);
mScreenSensor = sensorHandle;
// TODO: consider a new method setHeadAndScreenSensor()
// because we generally set both at the same time.
@@ -545,26 +649,48 @@
Status Spatializer::setDisplayOrientation(float physicalToLogicalAngle) {
ALOGV("%s physicalToLogicalAngle %f", __func__, physicalToLogicalAngle);
- if (!mSupportsHeadTracking) {
- return binderStatusFromStatusT(INVALID_OPERATION);
- }
+ mLocalLog.log("%s with %f", __func__, physicalToLogicalAngle);
+ const float angle = safe_clamp(physicalToLogicalAngle, 0.f, (float)(2. * M_PI));
+ // It is possible due to numerical inaccuracies to exceed the boundaries of 0 to 2 * M_PI.
+ ALOGI_IF(angle != physicalToLogicalAngle,
+ "%s: clamping %f to %f", __func__, physicalToLogicalAngle, angle);
std::lock_guard lock(mLock);
- mDisplayOrientation = physicalToLogicalAngle;
+ mDisplayOrientation = angle;
if (mPoseController != nullptr) {
- mPoseController->setDisplayOrientation(mDisplayOrientation);
+ // This turns on the rate-limiter.
+ mPoseController->setDisplayOrientation(angle);
}
if (mEngine != nullptr) {
setEffectParameter_l(
- SPATIALIZER_PARAM_DISPLAY_ORIENTATION, std::vector<float>{physicalToLogicalAngle});
+ SPATIALIZER_PARAM_DISPLAY_ORIENTATION, std::vector<float>{angle});
}
return Status::ok();
}
Status Spatializer::setHingeAngle(float hingeAngle) {
- std::lock_guard lock(mLock);
ALOGV("%s hingeAngle %f", __func__, hingeAngle);
+ mLocalLog.log("%s with %f", __func__, hingeAngle);
+ const float angle = safe_clamp(hingeAngle, 0.f, (float)(2. * M_PI));
+ // It is possible due to numerical inaccuracies to exceed the boundaries of 0 to 2 * M_PI.
+ ALOGI_IF(angle != hingeAngle,
+ "%s: clamping %f to %f", __func__, hingeAngle, angle);
+ std::lock_guard lock(mLock);
+ mHingeAngle = angle;
if (mEngine != nullptr) {
- setEffectParameter_l(SPATIALIZER_PARAM_HINGE_ANGLE, std::vector<float>{hingeAngle});
+ setEffectParameter_l(SPATIALIZER_PARAM_HINGE_ANGLE, std::vector<float>{angle});
+ }
+ return Status::ok();
+}
+
+Status Spatializer::setFoldState(bool folded) {
+ ALOGV("%s foldState %d", __func__, (int)folded);
+ mLocalLog.log("%s with %d", __func__, (int)folded);
+ std::lock_guard lock(mLock);
+ mFoldedState = folded;
+ if (mEngine != nullptr) {
+ // we don't suppress multiple calls with the same folded state - that's
+ // done at the caller.
+ setEffectParameter_l(SPATIALIZER_PARAM_FOLD_STATE, std::vector<uint8_t>{mFoldedState});
}
return Status::ok();
}
@@ -642,6 +768,17 @@
msg->post();
}
+void Spatializer::resetEngineHeadPose_l() {
+ ALOGV("%s mEngine %p", __func__, mEngine.get());
+ if (mEngine == nullptr) {
+ return;
+ }
+ const std::vector<float> headToStage(6, 0.0);
+ setEffectParameter_l(SPATIALIZER_PARAM_HEAD_TO_STAGE, headToStage);
+ setEffectParameter_l(SPATIALIZER_PARAM_HEADTRACKING_MODE,
+ std::vector<SpatializerHeadTrackingMode>{SpatializerHeadTrackingMode::DISABLED});
+}
+
void Spatializer::onHeadToStagePoseMsg(const std::vector<float>& headToStage) {
ALOGV("%s", __func__);
sp<media::ISpatializerHeadTrackingCallback> callback;
@@ -650,6 +787,9 @@
callback = mHeadTrackingCallback;
if (mEngine != nullptr) {
setEffectParameter_l(SPATIALIZER_PARAM_HEAD_TO_STAGE, headToStage);
+ const auto record = recordFromTranslationRotationVector(headToStage);
+ mPoseRecorder.record(record);
+ mPoseDurableRecorder.record(record);
}
}
@@ -659,9 +799,9 @@
}
void Spatializer::onActualModeChange(HeadTrackingMode mode) {
- ALOGV("%s(%d)", __func__, (int)mode);
- sp<AMessage> msg =
- new AMessage(EngineCallbackHandler::kWhatOnActualModeChange, mHandler);
+ std::string modeStr = media::toString(mode);
+ ALOGV("%s(%s)", __func__, modeStr.c_str());
+ sp<AMessage> msg = new AMessage(EngineCallbackHandler::kWhatOnActualModeChange, mHandler);
msg->setInt32(EngineCallbackHandler::kModeKey, static_cast<int>(mode));
msg->post();
}
@@ -691,10 +831,15 @@
}
mActualHeadTrackingMode = spatializerMode;
if (mEngine != nullptr) {
- setEffectParameter_l(SPATIALIZER_PARAM_HEADTRACKING_MODE,
- std::vector<SpatializerHeadTrackingMode>{spatializerMode});
+ if (spatializerMode == SpatializerHeadTrackingMode::DISABLED) {
+ resetEngineHeadPose_l();
+ } else {
+ setEffectParameter_l(SPATIALIZER_PARAM_HEADTRACKING_MODE,
+ std::vector<SpatializerHeadTrackingMode>{spatializerMode});
+ }
}
callback = mHeadTrackingCallback;
+ mLocalLog.log("%s: updating mode to %s", __func__, media::toString(mode).c_str());
}
if (callback != nullptr) {
callback->onHeadTrackingModeChanged(spatializerMode);
@@ -708,13 +853,17 @@
{
std::lock_guard lock(mLock);
ALOGV("%s output %d mOutput %d", __func__, (int)output, (int)mOutput);
+ mLocalLog.log("%s with output %d tracks %zu (mOutput %d)", __func__, (int)output,
+ numActiveTracks, (int)mOutput);
if (mOutput != AUDIO_IO_HANDLE_NONE) {
LOG_ALWAYS_FATAL_IF(mEngine == nullptr, "%s output set without FX engine", __func__);
// remove FX instance
mEngine->setEnabled(false);
mEngine.clear();
mPoseController.reset();
+ AudioSystem::removeSupportedLatencyModesCallback(this);
}
+
// create FX instance on output
AttributionSourceState attributionSource = AttributionSourceState();
mEngine = new AudioEffect(attributionSource);
@@ -730,6 +879,13 @@
outputChanged = mOutput != output;
mOutput = output;
mNumActiveTracks = numActiveTracks;
+ AudioSystem::addSupportedLatencyModesCallback(this);
+
+ std::vector<audio_latency_mode_t> latencyModes;
+ status = AudioSystem::getSupportedLatencyModes(mOutput, &latencyModes);
+ if (status == OK) {
+ mSupportedLatencyModes = latencyModes;
+ }
checkEngineState_l();
if (mSupportsHeadTracking) {
@@ -737,6 +893,14 @@
checkSensorsState_l();
}
callback = mSpatializerCallback;
+
+ // Restore common effect state.
+ setEffectParameter_l(SPATIALIZER_PARAM_DISPLAY_ORIENTATION,
+ std::vector<float>{mDisplayOrientation});
+ setEffectParameter_l(SPATIALIZER_PARAM_FOLD_STATE,
+ std::vector<uint8_t>{mFoldedState});
+ setEffectParameter_l(SPATIALIZER_PARAM_HINGE_ANGLE,
+ std::vector<float>{mHingeAngle});
}
if (outputChanged && callback != nullptr) {
@@ -752,6 +916,7 @@
{
std::lock_guard lock(mLock);
+ mLocalLog.log("%s with output %d tracks %zu", __func__, (int)mOutput, mNumActiveTracks);
ALOGV("%s mOutput %d", __func__, (int)mOutput);
if (mOutput == AUDIO_IO_HANDLE_NONE) {
return output;
@@ -759,6 +924,7 @@
// remove FX instance
mEngine->setEnabled(false);
mEngine.clear();
+ AudioSystem::removeSupportedLatencyModesCallback(this);
output = mOutput;
mOutput = AUDIO_IO_HANDLE_NONE;
mPoseController.reset();
@@ -771,9 +937,31 @@
return output;
}
+void Spatializer::onSupportedLatencyModesChanged(
+ audio_io_handle_t output, const std::vector<audio_latency_mode_t>& modes) {
+ ALOGV("%s output %d num modes %zu", __func__, (int)output, modes.size());
+ sp<AMessage> msg =
+ new AMessage(EngineCallbackHandler::kWhatOnLatencyModesChanged, mHandler);
+ msg->setObject(EngineCallbackHandler::kLatencyModesKey,
+ sp<EngineCallbackHandler::LatencyModes>::make(output, modes));
+ msg->post();
+}
+
+void Spatializer::onSupportedLatencyModesChangedMsg(
+ audio_io_handle_t output, std::vector<audio_latency_mode_t>&& modes) {
+ std::lock_guard lock(mLock);
+ ALOGV("%s output %d mOutput %d num modes %zu",
+ __func__, (int)output, (int)mOutput, modes.size());
+ if (output == mOutput) {
+ mSupportedLatencyModes = std::move(modes);
+ checkSensorsState_l();
+ }
+}
+
void Spatializer::updateActiveTracks(size_t numActiveTracks) {
std::lock_guard lock(mLock);
if (mNumActiveTracks != numActiveTracks) {
+ mLocalLog.log("%s from %zu to %zu", __func__, mNumActiveTracks, numActiveTracks);
mNumActiveTracks = numActiveTracks;
checkEngineState_l();
checkSensorsState_l();
@@ -781,17 +969,40 @@
}
void Spatializer::checkSensorsState_l() {
- if (mSupportsHeadTracking && mPoseController != nullptr) {
- if (mNumActiveTracks > 0 && mLevel != SpatializationLevel::NONE
- && mDesiredHeadTrackingMode != HeadTrackingMode::STATIC
- && mHeadSensor != SpatializerPoseController::INVALID_SENSOR) {
- mPoseController->setHeadSensor(mHeadSensor);
- mPoseController->setScreenSensor(mScreenSensor);
+ audio_latency_mode_t requestedLatencyMode = AUDIO_LATENCY_MODE_FREE;
+ const bool supportsSetLatencyMode = !mSupportedLatencyModes.empty();
+ const bool supportsLowLatencyMode = supportsSetLatencyMode && std::find(
+ mSupportedLatencyModes.begin(), mSupportedLatencyModes.end(),
+ AUDIO_LATENCY_MODE_LOW) != mSupportedLatencyModes.end();
+ if (mSupportsHeadTracking) {
+ if (mPoseController != nullptr) {
+ // TODO(b/253297301, b/255433067) reenable low latency condition check
+ // for Head Tracking after Bluetooth HAL supports it correctly.
+ if (mNumActiveTracks > 0 && mLevel != SpatializationLevel::NONE
+ && mDesiredHeadTrackingMode != HeadTrackingMode::STATIC
+ && mHeadSensor != SpatializerPoseController::INVALID_SENSOR) {
+ if (mEngine != nullptr) {
+ setEffectParameter_l(SPATIALIZER_PARAM_HEADTRACKING_MODE,
+ std::vector<SpatializerHeadTrackingMode>{mActualHeadTrackingMode});
+ }
+ mPoseController->setHeadSensor(mHeadSensor);
+ mPoseController->setScreenSensor(mScreenSensor);
+ if (supportsLowLatencyMode) requestedLatencyMode = AUDIO_LATENCY_MODE_LOW;
+ } else {
+ mPoseController->setHeadSensor(SpatializerPoseController::INVALID_SENSOR);
+ mPoseController->setScreenSensor(SpatializerPoseController::INVALID_SENSOR);
+ resetEngineHeadPose_l();
+ }
} else {
- mPoseController->setHeadSensor(SpatializerPoseController::INVALID_SENSOR);
- mPoseController->setScreenSensor(SpatializerPoseController::INVALID_SENSOR);
+ resetEngineHeadPose_l();
}
}
+ if (mOutput != AUDIO_IO_HANDLE_NONE && supportsSetLatencyMode) {
+ const status_t status =
+ AudioSystem::setRequestedLatencyMode(mOutput, requestedLatencyMode);
+ ALOGD("%s: setRequestedLatencyMode for output thread(%d) to %s returned %d",
+ __func__, mOutput, toString(requestedLatencyMode).c_str(), status);
+ }
}
void Spatializer::checkEngineState_l() {
@@ -800,8 +1011,6 @@
mEngine->setEnabled(true);
setEffectParameter_l(SPATIALIZER_PARAM_LEVEL,
std::vector<SpatializationLevel>{mLevel});
- setEffectParameter_l(SPATIALIZER_PARAM_HEADTRACKING_MODE,
- std::vector<SpatializerHeadTrackingMode>{mActualHeadTrackingMode});
} else {
setEffectParameter_l(SPATIALIZER_PARAM_LEVEL,
std::vector<SpatializationLevel>{SpatializationLevel::NONE});
@@ -823,6 +1032,7 @@
mPoseController->setDisplayOrientation(mDisplayOrientation);
} else if (!isControllerNeeded && mPoseController != nullptr) {
mPoseController.reset();
+ resetEngineHeadPose_l();
}
if (mPoseController != nullptr) {
mPoseController->setDesiredMode(mDesiredHeadTrackingMode);
@@ -861,4 +1071,80 @@
msg->post();
}
+std::string Spatializer::toString(unsigned level) const {
+ std::string prefixSpace(level, ' ');
+ std::string ss = prefixSpace + "Spatializer:\n";
+ bool needUnlock = false;
+
+ prefixSpace += ' ';
+ if (!mLock.try_lock()) {
+ // dumpsys even try_lock failed, information dump can be useful although may not accurate
+ ss.append(prefixSpace).append("try_lock failed, dumpsys below maybe INACCURATE!\n");
+ } else {
+ needUnlock = true;
+ }
+
+ // Spatializer class information.
+ // 1. Capabilities (mLevels, mHeadTrackingModes, mSpatializationModes, mChannelMasks, etc)
+ ss.append(prefixSpace).append("Supported levels: [");
+ for (auto& level : mLevels) {
+ base::StringAppendF(&ss, " %s", media::toString(level).c_str());
+ }
+ base::StringAppendF(&ss, "], mLevel: %s", media::toString(mLevel).c_str());
+
+ base::StringAppendF(&ss, "\n%smHeadTrackingModes: [", prefixSpace.c_str());
+ for (auto& mode : mHeadTrackingModes) {
+ base::StringAppendF(&ss, " %s", media::toString(mode).c_str());
+ }
+ base::StringAppendF(&ss, "], Desired: %s, Actual %s\n",
+ media::toString(mDesiredHeadTrackingMode).c_str(),
+ media::toString(mActualHeadTrackingMode).c_str());
+
+ base::StringAppendF(&ss, "%smSpatializationModes: [", prefixSpace.c_str());
+ for (auto& mode : mSpatializationModes) {
+ base::StringAppendF(&ss, " %s", media::toString(mode).c_str());
+ }
+ ss += "]\n";
+
+ base::StringAppendF(&ss, "%smChannelMasks: ", prefixSpace.c_str());
+ for (auto& mask : mChannelMasks) {
+ base::StringAppendF(&ss, "%s", audio_channel_out_mask_to_string(mask));
+ }
+ base::StringAppendF(&ss, "\n%smSupportsHeadTracking: %s\n", prefixSpace.c_str(),
+ mSupportsHeadTracking ? "true" : "false");
+ // 2. Settings (Output, tracks)
+ base::StringAppendF(&ss, "%smNumActiveTracks: %zu\n", prefixSpace.c_str(), mNumActiveTracks);
+ base::StringAppendF(&ss, "%sOutputStreamHandle: %d\n", prefixSpace.c_str(), (int)mOutput);
+
+ // 3. Sensors, Effect information.
+ base::StringAppendF(&ss, "%sHeadSensorHandle: 0x%08x\n", prefixSpace.c_str(), mHeadSensor);
+ base::StringAppendF(&ss, "%sScreenSensorHandle: 0x%08x\n", prefixSpace.c_str(), mScreenSensor);
+ base::StringAppendF(&ss, "%sEffectHandle: %p\n", prefixSpace.c_str(), mEngine.get());
+ base::StringAppendF(&ss, "%sDisplayOrientation: %f\n", prefixSpace.c_str(),
+ mDisplayOrientation);
+
+ ss.append(prefixSpace + "CommandLog:\n");
+ ss += mLocalLog.dumpToString((prefixSpace + " ").c_str(), mMaxLocalLogLine);
+
+ // PostController dump.
+ if (mPoseController != nullptr) {
+ ss.append(mPoseController->toString(level + 1))
+ .append(prefixSpace)
+ .append("Pose (active stage-to-head) [tx, ty, tz : pitch, roll, yaw]:\n")
+ .append(prefixSpace)
+ .append(" PerMinuteHistory:\n")
+ .append(mPoseDurableRecorder.toString(level + 3))
+ .append(prefixSpace)
+ .append(" PerSecondHistory:\n")
+ .append(mPoseRecorder.toString(level + 3));
+ } else {
+ ss.append(prefixSpace).append("SpatializerPoseController not exist\n");
+ }
+
+ if (needUnlock) {
+ mLock.unlock();
+ }
+ return ss;
+}
+
} // namespace android
diff --git a/services/audiopolicy/service/Spatializer.h b/services/audiopolicy/service/Spatializer.h
index 29f4b08..b101ae6 100644
--- a/services/audiopolicy/service/Spatializer.h
+++ b/services/audiopolicy/service/Spatializer.h
@@ -17,15 +17,20 @@
#ifndef ANDROID_MEDIA_SPATIALIZER_H
#define ANDROID_MEDIA_SPATIALIZER_H
+#include <android-base/stringprintf.h>
#include <android/media/BnEffect.h>
#include <android/media/BnSpatializer.h>
#include <android/media/SpatializationLevel.h>
#include <android/media/SpatializationMode.h>
#include <android/media/SpatializerHeadTrackingMode.h>
+#include <audio_utils/SimpleLog.h>
+#include <math.h>
+#include <media/AudioEffect.h>
+#include <media/VectorRecorder.h>
#include <media/audiohal/EffectHalInterface.h>
#include <media/stagefright/foundation/ALooper.h>
-#include <media/AudioEffect.h>
#include <system/audio_effects/effect_spatializer.h>
+#include <string>
#include "SpatializerPoseController.h"
@@ -85,7 +90,8 @@
*/
class Spatializer : public media::BnSpatializer,
public IBinder::DeathRecipient,
- private SpatializerPoseController::Listener {
+ private SpatializerPoseController::Listener,
+ public virtual AudioSystem::SupportedLatencyModesCallback {
public:
static sp<Spatializer> create(SpatializerPolicyCallback *callback);
@@ -112,6 +118,7 @@
binder::Status setScreenSensor(int sensorHandle) override;
binder::Status setDisplayOrientation(float physicalToLogicalAngle) override;
binder::Status setHingeAngle(float hingeAngle) override;
+ binder::Status setFoldState(bool folded) override;
binder::Status getSupportedModes(std::vector<media::SpatializationMode>* modes) override;
binder::Status registerHeadTrackingCallback(
const sp<media::ISpatializerHeadTrackingCallback>& callback) override;
@@ -122,6 +129,10 @@
/** IBinder::DeathRecipient. Listen to the death of the INativeSpatializerCallback. */
virtual void binderDied(const wp<IBinder>& who);
+ /** SupportedLatencyModesCallback */
+ void onSupportedLatencyModesChanged(
+ audio_io_handle_t output, const std::vector<audio_latency_mode_t>& modes) override;
+
/** Registers a INativeSpatializerCallback when a client is attached to this Spatializer
* by audio policy service.
*/
@@ -150,6 +161,22 @@
void calculateHeadPose();
+ /** Convert fields in Spatializer and sub-modules to a string. Disable thread-safety-analysis
+ * here because we want to dump mutex guarded members even try_lock failed to provide as much
+ * information as possible for debugging purpose. */
+ std::string toString(unsigned level) const NO_THREAD_SAFETY_ANALYSIS;
+
+ static std::string toString(audio_latency_mode_t mode) {
+ // We convert to the AIDL type to print (eventually the legacy type will be removed).
+ const auto result = legacy2aidl_audio_latency_mode_t_LatencyMode(mode);
+ return result.has_value() ? media::toString(*result) : "unknown_latency_mode";
+ }
+
+ // If the Spatializer is not created, we send the status for metrics purposes.
+ // OK: Spatializer not expected to be created.
+ // NO_INIT: Spatializer creation failed.
+ static void sendEmptyCreateSpatializerMetricWithStatus(status_t status);
+
private:
Spatializer(effect_descriptor_t engineDescriptor,
SpatializerPolicyCallback *callback);
@@ -162,6 +189,8 @@
void onHeadToStagePoseMsg(const std::vector<float>& headToStage);
void onActualModeChangeMsg(media::HeadTrackingMode mode);
+ void onSupportedLatencyModesChangedMsg(
+ audio_io_handle_t output, std::vector<audio_latency_mode_t>&& modes);
static constexpr int kMaxEffectParamValues = 10;
/**
@@ -295,13 +324,21 @@
*/
void checkEngineState_l() REQUIRES(mLock);
+ /**
+ * Reset head tracking mode and recenter pose in engine: Called when the head tracking
+ * is disabled.
+ */
+ void resetEngineHeadPose_l() REQUIRES(mLock);
+
/** Effect engine descriptor */
const effect_descriptor_t mEngineDescriptor;
/** Callback interface to parent audio policy service */
SpatializerPolicyCallback* const mPolicyCallback;
/** Currently there is only one version of the spatializer running */
- const std::string mMetricsId = AMEDIAMETRICS_KEY_PREFIX_AUDIO_SPATIALIZER "0";
+ static constexpr const char* kDefaultMetricsId =
+ AMEDIAMETRICS_KEY_PREFIX_AUDIO_SPATIALIZER "0";
+ const std::string mMetricsId = kDefaultMetricsId;
/** Mutex protecting internal state */
mutable std::mutex mLock;
@@ -338,8 +375,13 @@
int32_t mScreenSensor GUARDED_BY(mLock) = SpatializerPoseController::INVALID_SENSOR;
/** Last display orientation received */
- static constexpr float kDisplayOrientationInvalid = 1000;
- float mDisplayOrientation GUARDED_BY(mLock) = kDisplayOrientationInvalid;
+ float mDisplayOrientation GUARDED_BY(mLock) = 0.f; // aligned to natural up orientation.
+
+ /** Last folded state */
+ bool mFoldedState GUARDED_BY(mLock) = false; // foldable: true means folded.
+
+ /** Last hinge angle */
+ float mHingeAngle GUARDED_BY(mLock) = 0.f; // foldable: 0.f is closed, M_PI flat open.
std::vector<media::SpatializationLevel> mLevels;
std::vector<media::SpatializerHeadTrackingMode> mHeadTrackingModes;
@@ -354,10 +396,25 @@
sp<EngineCallbackHandler> mHandler;
size_t mNumActiveTracks GUARDED_BY(mLock) = 0;
+ std::vector<audio_latency_mode_t> mSupportedLatencyModes GUARDED_BY(mLock);
- static const std::vector<const char *> sHeadPoseKeys;
-};
+ static const std::vector<const char*> sHeadPoseKeys;
+ // Local log for command messages.
+ static constexpr int mMaxLocalLogLine = 10;
+ SimpleLog mLocalLog{mMaxLocalLogLine};
+
+ /**
+ * @brief Calculate and record sensor data.
+ * Dump to local log with max/average pose angle every mPoseRecordThreshold.
+ */
+ // Record one log line per second (up to mMaxLocalLogLine) to capture most recent sensor data.
+ media::VectorRecorder mPoseRecorder GUARDED_BY(mLock) {
+ 6 /* vectorSize */, std::chrono::seconds(1), mMaxLocalLogLine, { 3 } /* delimiterIdx */};
+ // Record one log line per minute (up to mMaxLocalLogLine) to capture durable sensor data.
+ media::VectorRecorder mPoseDurableRecorder GUARDED_BY(mLock) {
+ 6 /* vectorSize */, std::chrono::minutes(1), mMaxLocalLogLine, { 3 } /* delimiterIdx */};
+}; // Spatializer
}; // namespace android
diff --git a/services/audiopolicy/service/SpatializerPoseController.cpp b/services/audiopolicy/service/SpatializerPoseController.cpp
index 304d44a..874bde4 100644
--- a/services/audiopolicy/service/SpatializerPoseController.cpp
+++ b/services/audiopolicy/service/SpatializerPoseController.cpp
@@ -13,12 +13,19 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
+
#include "SpatializerPoseController.h"
+#include <android-base/stringprintf.h>
+#include <chrono>
+#include <cstdint>
+#include <string>
#define LOG_TAG "SpatializerPoseController"
//#define LOG_NDEBUG 0
+#include <cutils/properties.h>
#include <sensor/Sensor.h>
#include <media/MediaMetricsItem.h>
+#include <media/QuaternionUtil.h>
#include <utils/Log.h>
#include <utils/SystemClock.h>
@@ -39,16 +46,24 @@
constexpr float kMaxTranslationalVelocity = 2;
// This is how fast, in rad/s, we allow rotation angle to shift during rate-limiting.
-constexpr float kMaxRotationalVelocity = 8;
+constexpr float kMaxRotationalVelocity = 0.8f;
-// This is how far into the future we predict the head pose, using linear extrapolation based on
-// twist (velocity). It should be set to a value that matches the characteristic durations of moving
-// one's head. The higher we set this, the more latency we are able to reduce, but setting this too
-// high will result in high prediction errors whenever the head accelerates (changes velocity).
-constexpr auto kPredictionDuration = 50ms;
+// This is how far into the future we predict the head pose.
+// The prediction duration should be based on the actual latency from
+// head-tracker to audio output, though setting the prediction duration too
+// high may result in higher prediction errors when the head accelerates or
+// decelerates (changes velocity).
+//
+// The head tracking predictor will do a best effort to achieve the requested
+// prediction duration. If the duration is too far in the future based on
+// current sensor variance, the predictor may internally restrict duration to what
+// is achievable with reasonable confidence as the "best prediction".
+constexpr auto kPredictionDuration = 120ms;
// After not getting a pose sample for this long, we would treat the measurement as stale.
-constexpr auto kFreshnessTimeout = 50ms;
+// The max connection interval is 50ms, and HT sensor event interval can differ depending on the
+// sampling rate, scheduling, sensor eventQ FIFO etc. 120 (2 * 50 + 20) ms seems reasonable for now.
+constexpr auto kFreshnessTimeout = 120ms;
// Auto-recenter kicks in after the head has been still for this long.
constexpr auto kAutoRecenterWindowDuration = 6s;
@@ -61,13 +76,13 @@
// Screen is considered to be unstable (not still) if it has moved significantly within the last
// time window of this duration.
-constexpr auto kScreenStillnessWindowDuration = 3s;
+constexpr auto kScreenStillnessWindowDuration = 750ms;
// Screen is considered to have moved significantly if translated by this much (in meter, approx).
constexpr float kScreenStillnessTranslationThreshold = 0.1f;
// Screen is considered to have moved significantly if rotated by this much (in radians, approx).
-constexpr float kScreenStillnessRotationThreshold = 7.0f / 180 * M_PI;
+constexpr float kScreenStillnessRotationThreshold = 15.0f / 180 * M_PI;
// Time units for system clock ticks. This is what the Sensor Framework timestamps represent and
// what we use for pose filtering.
@@ -91,7 +106,15 @@
.maxTranslationalVelocity = kMaxTranslationalVelocity / kTicksPerSecond,
.maxRotationalVelocity = kMaxRotationalVelocity / kTicksPerSecond,
.freshnessTimeout = Ticks(kFreshnessTimeout).count(),
- .predictionDuration = Ticks(kPredictionDuration).count(),
+ .predictionDuration = []() -> float {
+ const int duration_ms =
+ property_get_int32("audio.spatializer.prediction_duration_ms", -1);
+ if (duration_ms >= 0) {
+ return duration_ms * 1'000'000LL;
+ } else {
+ return Ticks(kPredictionDuration).count();
+ }
+ }(),
.autoRecenterWindowDuration = Ticks(kAutoRecenterWindowDuration).count(),
.autoRecenterTranslationalThreshold = kAutoRecenterTranslationThreshold,
.autoRecenterRotationalThreshold = kAutoRecenterRotationThreshold,
@@ -100,7 +123,10 @@
.screenStillnessRotationalThreshold = kScreenStillnessRotationThreshold,
})),
mPoseProvider(SensorPoseProvider::create("headtracker", this)),
- mThread([this, maxUpdatePeriod] {
+ mThread([this, maxUpdatePeriod] { // It's important that mThread is initialized after
+ // everything else because it runs a member
+ // function that may use any member
+ // of this class.
while (true) {
Pose3f headToStage;
std::optional<HeadTrackingMode> modeIfChanged;
@@ -136,7 +162,14 @@
mShouldCalculate = false;
}
}
- }) {}
+ }) {
+ const media::PosePredictorType posePredictorType =
+ (media::PosePredictorType)
+ property_get_int32("audio.spatializer.pose_predictor_type", -1);
+ if (isValidPosePredictorType(posePredictorType)) {
+ mProcessor->setPosePredictorType(posePredictorType);
+ }
+ }
SpatializerPoseController::~SpatializerPoseController() {
{
@@ -183,7 +216,7 @@
mHeadSensor = INVALID_SENSOR;
}
- mProcessor->recenter(true /* recenterHead */, false /* recenterScreen */);
+ mProcessor->recenter(true /* recenterHead */, false /* recenterScreen */, __func__);
}
void SpatializerPoseController::setScreenSensor(int32_t sensor) {
@@ -220,7 +253,7 @@
mScreenSensor = INVALID_SENSOR;
}
- mProcessor->recenter(false /* recenterHead */, true /* recenterScreen */);
+ mProcessor->recenter(false /* recenterHead */, true /* recenterScreen */, __func__);
}
void SpatializerPoseController::setDesiredMode(HeadTrackingMode mode) {
@@ -267,25 +300,132 @@
void SpatializerPoseController::recenter() {
std::lock_guard lock(mMutex);
- mProcessor->recenter();
+ mProcessor->recenter(true /* recenterHead */, true /* recenterScreen */, __func__);
}
void SpatializerPoseController::onPose(int64_t timestamp, int32_t sensor, const Pose3f& pose,
const std::optional<Twist3f>& twist, bool isNewReference) {
std::lock_guard lock(mMutex);
+ constexpr float NANOS_TO_MILLIS = 1e-6;
+ constexpr float RAD_TO_DEGREE = 180.f / M_PI;
+
+ const float delayMs = (elapsedRealtimeNano() - timestamp) * NANOS_TO_MILLIS; // CLOCK_BOOTTIME
+
if (sensor == mHeadSensor) {
+ std::vector<float> pryprydt(8); // pitch, roll, yaw, d_pitch, d_roll, d_yaw,
+ // discontinuity, timestamp_delay
+ media::quaternionToAngles(pose.rotation(), &pryprydt[0], &pryprydt[1], &pryprydt[2]);
+ if (twist) {
+ const auto rotationalVelocity = twist->rotationalVelocity();
+ // The rotational velocity is an intrinsic transform (i.e. based on the head
+ // coordinate system, not the world coordinate system). It is a 3 element vector:
+ // axis (d theta / dt).
+ //
+ // We leave rotational velocity relative to the head coordinate system,
+ // as the initial head tracking sensor's world frame is arbitrary.
+ media::quaternionToAngles(media::rotationVectorToQuaternion(rotationalVelocity),
+ &pryprydt[3], &pryprydt[4], &pryprydt[5]);
+ }
+ pryprydt[6] = isNewReference;
+ pryprydt[7] = delayMs;
+ for (size_t i = 0; i < 6; ++i) {
+ // pitch, roll, yaw in degrees, referenced in degrees on the world frame.
+ // d_pitch, d_roll, d_yaw rotational velocity in degrees/s, based on the world frame.
+ pryprydt[i] *= RAD_TO_DEGREE;
+ }
+ mHeadSensorRecorder.record(pryprydt);
+ mHeadSensorDurableRecorder.record(pryprydt);
+
mProcessor->setWorldToHeadPose(timestamp, pose,
twist.value_or(Twist3f()) / kTicksPerSecond);
if (isNewReference) {
- mProcessor->recenter(true, false);
+ mProcessor->recenter(true, false, __func__);
}
}
if (sensor == mScreenSensor) {
+ std::vector<float> pryt{ 0.f, 0.f, 0.f, delayMs}; // pitch, roll, yaw, timestamp_delay
+ media::quaternionToAngles(pose.rotation(), &pryt[0], &pryt[1], &pryt[2]);
+ for (size_t i = 0; i < 3; ++i) {
+ pryt[i] *= RAD_TO_DEGREE;
+ }
+ mScreenSensorRecorder.record(pryt);
+ mScreenSensorDurableRecorder.record(pryt);
+
mProcessor->setWorldToScreenPose(timestamp, pose);
if (isNewReference) {
- mProcessor->recenter(false, true);
+ mProcessor->recenter(false, true, __func__);
}
}
}
+std::string SpatializerPoseController::toString(unsigned level) const {
+ std::string prefixSpace(level, ' ');
+ std::string ss = prefixSpace + "SpatializerPoseController:\n";
+ bool needUnlock = false;
+
+ prefixSpace += ' ';
+ auto now = std::chrono::steady_clock::now();
+ if (!mMutex.try_lock_until(now + media::kSpatializerDumpSysTimeOutInSecond)) {
+ ss.append(prefixSpace).append("try_lock failed, dumpsys maybe INACCURATE!\n");
+ } else {
+ needUnlock = true;
+ }
+
+ ss += prefixSpace;
+ if (mHeadSensor == INVALID_SENSOR) {
+ ss += "HeadSensor: INVALID\n";
+ } else {
+ base::StringAppendF(&ss, "HeadSensor: 0x%08x "
+ "(active world-to-head : head-relative velocity) "
+ "[ pitch, roll, yaw : d_pitch, d_roll, d_yaw : disc : delay ] "
+ "(degrees, degrees/s, bool, ms)\n", mHeadSensor);
+ ss.append(prefixSpace)
+ .append(" PerMinuteHistory:\n")
+ .append(mHeadSensorDurableRecorder.toString(level + 3))
+ .append(prefixSpace)
+ .append(" PerSecondHistory:\n")
+ .append(mHeadSensorRecorder.toString(level + 3));
+ }
+
+ ss += prefixSpace;
+ if (mScreenSensor == INVALID_SENSOR) {
+ ss += "ScreenSensor: INVALID\n";
+ } else {
+ base::StringAppendF(&ss, "ScreenSensor: 0x%08x (active world-to-screen) "
+ "[ pitch, roll, yaw : delay ] "
+ "(degrees, ms)\n", mScreenSensor);
+ ss.append(prefixSpace)
+ .append(" PerMinuteHistory:\n")
+ .append(mScreenSensorDurableRecorder.toString(level + 3))
+ .append(prefixSpace)
+ .append(" PerSecondHistory:\n")
+ .append(mScreenSensorRecorder.toString(level + 3));
+ }
+
+ ss += prefixSpace;
+ if (mActualMode.has_value()) {
+ base::StringAppendF(&ss, "ActualMode: %s\n", media::toString(mActualMode.value()).c_str());
+ } else {
+ ss += "ActualMode NOTEXIST\n";
+ }
+
+ if (mProcessor) {
+ ss += mProcessor->toString_l(level + 1);
+ } else {
+ ss.append(prefixSpace.c_str()).append("HeadTrackingProcessor not exist\n");
+ }
+
+ if (mPoseProvider) {
+ ss += mPoseProvider->toString(level + 1);
+ } else {
+ ss.append(prefixSpace.c_str()).append("SensorPoseProvider not exist\n");
+ }
+
+ if (needUnlock) {
+ mMutex.unlock();
+ }
+ // TODO: 233092747 add history sensor info with SimpleLog.
+ return ss;
+}
+
} // namespace android
diff --git a/services/audiopolicy/service/SpatializerPoseController.h b/services/audiopolicy/service/SpatializerPoseController.h
index 2c6d79a..9d78188 100644
--- a/services/audiopolicy/service/SpatializerPoseController.h
+++ b/services/audiopolicy/service/SpatializerPoseController.h
@@ -24,6 +24,7 @@
#include <media/HeadTrackingProcessor.h>
#include <media/SensorPoseProvider.h>
+#include <media/VectorRecorder.h>
namespace android {
@@ -113,8 +114,11 @@
*/
void waitUntilCalculated();
+ // convert fields to a printable string
+ std::string toString(unsigned level) const;
+
private:
- mutable std::mutex mMutex;
+ mutable std::timed_mutex mMutex;
Listener* const mListener;
const std::chrono::microseconds mSensorPeriod;
// Order matters for the following two members to ensure correct destruction.
@@ -123,12 +127,29 @@
int32_t mHeadSensor = media::SensorPoseProvider::INVALID_HANDLE;
int32_t mScreenSensor = media::SensorPoseProvider::INVALID_HANDLE;
std::optional<media::HeadTrackingMode> mActualMode;
- std::thread mThread;
- std::condition_variable mCondVar;
+ std::condition_variable_any mCondVar;
bool mShouldCalculate = true;
bool mShouldExit = false;
bool mCalculated = false;
+ media::VectorRecorder mHeadSensorRecorder{
+ 8 /* vectorSize */, std::chrono::seconds(1), 10 /* maxLogLine */,
+ { 3, 6, 7 } /* delimiterIdx */};
+ media::VectorRecorder mHeadSensorDurableRecorder{
+ 8 /* vectorSize */, std::chrono::minutes(1), 10 /* maxLogLine */,
+ { 3, 6, 7 } /* delimiterIdx */};
+
+ media::VectorRecorder mScreenSensorRecorder{
+ 4 /* vectorSize */, std::chrono::seconds(1), 10 /* maxLogLine */,
+ { 3 } /* delimiterIdx */};
+ media::VectorRecorder mScreenSensorDurableRecorder{
+ 4 /* vectorSize */, std::chrono::minutes(1), 10 /* maxLogLine */,
+ { 3 } /* delimiterIdx */};
+
+ // It's important that mThread is the last variable in this class
+ // since we starts mThread in initializer list
+ std::thread mThread;
+
void onPose(int64_t timestamp, int32_t sensor, const media::Pose3f& pose,
const std::optional<media::Twist3f>& twist, bool isNewReference) override;
diff --git a/services/audiopolicy/tests/audiopolicymanager_tests.cpp b/services/audiopolicy/tests/audiopolicymanager_tests.cpp
index 5429176..43b1a2a 100644
--- a/services/audiopolicy/tests/audiopolicymanager_tests.cpp
+++ b/services/audiopolicy/tests/audiopolicymanager_tests.cpp
@@ -148,6 +148,8 @@
std::unique_ptr<AudioPolicyManagerTestClient> mClient;
std::unique_ptr<AudioPolicyTestManager> mManager;
+
+ const uint32_t k48000SamplingRate = 48000;
};
void AudioPolicyManagerTest::SetUp() {
@@ -414,11 +416,11 @@
AudioPolicyConfig& config = mManager->getConfig();
mMsdOutputDevice = new DeviceDescriptor(AUDIO_DEVICE_OUT_BUS);
sp<AudioProfile> pcmOutputProfile = new AudioProfile(
- AUDIO_FORMAT_PCM_16_BIT, AUDIO_CHANNEL_OUT_STEREO, 48000);
+ AUDIO_FORMAT_PCM_16_BIT, AUDIO_CHANNEL_OUT_STEREO, k48000SamplingRate);
sp<AudioProfile> ac3OutputProfile = new AudioProfile(
- AUDIO_FORMAT_AC3, AUDIO_CHANNEL_OUT_5POINT1, 48000);
+ AUDIO_FORMAT_AC3, AUDIO_CHANNEL_OUT_5POINT1, k48000SamplingRate);
sp<AudioProfile> iec958OutputProfile = new AudioProfile(
- AUDIO_FORMAT_IEC60958, AUDIO_CHANNEL_OUT_STEREO, 48000);
+ AUDIO_FORMAT_IEC60958, AUDIO_CHANNEL_OUT_STEREO, k48000SamplingRate);
mMsdOutputDevice->addAudioProfile(pcmOutputProfile);
mMsdOutputDevice->addAudioProfile(ac3OutputProfile);
mMsdOutputDevice->addAudioProfile(iec958OutputProfile);
@@ -473,7 +475,7 @@
// Add a profile with another encoding to the default device to test routing
// of streams that are not supported by MSD.
sp<AudioProfile> dtsOutputProfile = new AudioProfile(
- AUDIO_FORMAT_DTS, AUDIO_CHANNEL_OUT_5POINT1, 48000);
+ AUDIO_FORMAT_DTS, AUDIO_CHANNEL_OUT_5POINT1, k48000SamplingRate);
config.getDefaultOutputDevice()->addAudioProfile(dtsOutputProfile);
sp<OutputProfile> primaryEncodedOutputProfile = new OutputProfile("encoded");
primaryEncodedOutputProfile->addAudioProfile(dtsOutputProfile);
@@ -491,7 +493,7 @@
// Add HDMI input device with IEC60958 profile for HDMI in -> MSD patching.
mHdmiInputDevice = new DeviceDescriptor(AUDIO_DEVICE_IN_HDMI);
sp<AudioProfile> iec958InputProfile = new AudioProfile(
- AUDIO_FORMAT_IEC60958, AUDIO_CHANNEL_IN_STEREO, 48000);
+ AUDIO_FORMAT_IEC60958, AUDIO_CHANNEL_IN_STEREO, k48000SamplingRate);
mHdmiInputDevice->addAudioProfile(iec958InputProfile);
config.addDevice(mHdmiInputDevice);
sp<InputProfile> hdmiInputProfile = new InputProfile("hdmi input");
@@ -556,8 +558,8 @@
TEST_P(AudioPolicyManagerTestMsd, GetOutputForAttrEncodedRoutesToMsd) {
const PatchCountCheck patchCount = snapshotPatchCount();
audio_port_handle_t selectedDeviceId = AUDIO_PORT_HANDLE_NONE;
- getOutputForAttr(&selectedDeviceId,
- AUDIO_FORMAT_AC3, AUDIO_CHANNEL_OUT_5POINT1, 48000, AUDIO_OUTPUT_FLAG_DIRECT);
+ getOutputForAttr(&selectedDeviceId, AUDIO_FORMAT_AC3, AUDIO_CHANNEL_OUT_5POINT1,
+ k48000SamplingRate, AUDIO_OUTPUT_FLAG_DIRECT);
ASSERT_EQ(selectedDeviceId, mDefaultOutputDevice->getId());
ASSERT_EQ(mExpectedAudioPatchCount, patchCount.deltaFromSnapshot());
}
@@ -566,7 +568,7 @@
const PatchCountCheck patchCount = snapshotPatchCount();
audio_port_handle_t selectedDeviceId = AUDIO_PORT_HANDLE_NONE;
getOutputForAttr(&selectedDeviceId,
- AUDIO_FORMAT_PCM_16_BIT, AUDIO_CHANNEL_OUT_STEREO, 48000);
+ AUDIO_FORMAT_PCM_16_BIT, AUDIO_CHANNEL_OUT_STEREO, k48000SamplingRate);
ASSERT_EQ(selectedDeviceId, mDefaultOutputDevice->getId());
ASSERT_EQ(mExpectedAudioPatchCount, patchCount.deltaFromSnapshot());
}
@@ -574,13 +576,13 @@
TEST_P(AudioPolicyManagerTestMsd, GetOutputForAttrEncodedPlusPcmRoutesToMsd) {
const PatchCountCheck patchCount = snapshotPatchCount();
audio_port_handle_t selectedDeviceId = AUDIO_PORT_HANDLE_NONE;
- getOutputForAttr(&selectedDeviceId,
- AUDIO_FORMAT_AC3, AUDIO_CHANNEL_OUT_5POINT1, 48000, AUDIO_OUTPUT_FLAG_DIRECT);
+ getOutputForAttr(&selectedDeviceId, AUDIO_FORMAT_AC3, AUDIO_CHANNEL_OUT_5POINT1,
+ k48000SamplingRate, AUDIO_OUTPUT_FLAG_DIRECT);
ASSERT_EQ(selectedDeviceId, mDefaultOutputDevice->getId());
ASSERT_EQ(mExpectedAudioPatchCount, patchCount.deltaFromSnapshot());
selectedDeviceId = AUDIO_PORT_HANDLE_NONE;
getOutputForAttr(&selectedDeviceId,
- AUDIO_FORMAT_PCM_16_BIT, AUDIO_CHANNEL_OUT_STEREO, 48000);
+ AUDIO_FORMAT_PCM_16_BIT, AUDIO_CHANNEL_OUT_STEREO, k48000SamplingRate);
ASSERT_EQ(selectedDeviceId, mDefaultOutputDevice->getId());
ASSERT_EQ(mExpectedAudioPatchCount, patchCount.deltaFromSnapshot());
}
@@ -588,8 +590,8 @@
TEST_P(AudioPolicyManagerTestMsd, GetOutputForAttrUnsupportedFormatBypassesMsd) {
const PatchCountCheck patchCount = snapshotPatchCount();
audio_port_handle_t selectedDeviceId = AUDIO_PORT_HANDLE_NONE;
- getOutputForAttr(&selectedDeviceId,
- AUDIO_FORMAT_DTS, AUDIO_CHANNEL_OUT_5POINT1, 48000, AUDIO_OUTPUT_FLAG_DIRECT);
+ getOutputForAttr(&selectedDeviceId, AUDIO_FORMAT_DTS, AUDIO_CHANNEL_OUT_5POINT1,
+ k48000SamplingRate, AUDIO_OUTPUT_FLAG_DIRECT);
ASSERT_NE(selectedDeviceId, mMsdOutputDevice->getId());
ASSERT_EQ(0, patchCount.deltaFromSnapshot());
}
@@ -600,9 +602,8 @@
const PatchCountCheck patchCount = snapshotPatchCount();
audio_port_handle_t selectedDeviceId = AUDIO_PORT_HANDLE_NONE;
audio_port_handle_t portId;
- getOutputForAttr(&selectedDeviceId,
- AUDIO_FORMAT_AC3, AUDIO_CHANNEL_OUT_5POINT1, 48000, AUDIO_OUTPUT_FLAG_DIRECT,
- nullptr /*output*/, &portId);
+ getOutputForAttr(&selectedDeviceId, AUDIO_FORMAT_AC3, AUDIO_CHANNEL_OUT_5POINT1,
+ k48000SamplingRate, AUDIO_OUTPUT_FLAG_DIRECT, nullptr /*output*/, &portId);
ASSERT_EQ(selectedDeviceId, mDefaultOutputDevice->getId());
ASSERT_EQ(mExpectedAudioPatchCount, patchCount.deltaFromSnapshot());
mManager->releaseOutput(portId);
@@ -612,9 +613,8 @@
const PatchCountCheck patchCount = snapshotPatchCount();
audio_port_handle_t selectedDeviceId = AUDIO_PORT_HANDLE_NONE;
audio_port_handle_t portId;
- getOutputForAttr(&selectedDeviceId,
- AUDIO_FORMAT_DTS, AUDIO_CHANNEL_OUT_5POINT1, 48000, AUDIO_OUTPUT_FLAG_DIRECT,
- nullptr /*output*/, &portId);
+ getOutputForAttr(&selectedDeviceId, AUDIO_FORMAT_DTS, AUDIO_CHANNEL_OUT_5POINT1,
+ k48000SamplingRate, AUDIO_OUTPUT_FLAG_DIRECT, nullptr /*output*/, &portId);
ASSERT_NE(selectedDeviceId, mMsdOutputDevice->getId());
ASSERT_EQ(-static_cast<int>(mExpectedAudioPatchCount), patchCount.deltaFromSnapshot());
mManager->releaseOutput(portId);
@@ -623,8 +623,8 @@
{
const PatchCountCheck patchCount = snapshotPatchCount();
audio_port_handle_t selectedDeviceId = AUDIO_PORT_HANDLE_NONE;
- getOutputForAttr(&selectedDeviceId,
- AUDIO_FORMAT_AC3, AUDIO_CHANNEL_OUT_5POINT1, 48000, AUDIO_OUTPUT_FLAG_DIRECT);
+ getOutputForAttr(&selectedDeviceId, AUDIO_FORMAT_AC3, AUDIO_CHANNEL_OUT_5POINT1,
+ k48000SamplingRate, AUDIO_OUTPUT_FLAG_DIRECT);
ASSERT_EQ(selectedDeviceId, mDefaultOutputDevice->getId());
ASSERT_EQ(0, patchCount.deltaFromSnapshot());
}
@@ -653,8 +653,8 @@
ASSERT_EQ(AUDIO_FORMAT_IEC60958, patch->mPatch.sinks[0].format);
ASSERT_EQ(AUDIO_CHANNEL_IN_STEREO, patch->mPatch.sources[0].channel_mask);
ASSERT_EQ(AUDIO_CHANNEL_OUT_STEREO, patch->mPatch.sinks[0].channel_mask);
- ASSERT_EQ(48000, patch->mPatch.sources[0].sample_rate);
- ASSERT_EQ(48000, patch->mPatch.sinks[0].sample_rate);
+ ASSERT_EQ(k48000SamplingRate, patch->mPatch.sources[0].sample_rate);
+ ASSERT_EQ(k48000SamplingRate, patch->mPatch.sinks[0].sample_rate);
ASSERT_EQ(1, patchCount.deltaFromSnapshot());
}
@@ -996,7 +996,7 @@
clearPolicyMix();
audioConfig.channel_mask = AUDIO_CHANNEL_OUT_STEREO;
audioConfig.format = AUDIO_FORMAT_PCM_16_BIT;
- audioConfig.sample_rate = 48000;
+ audioConfig.sample_rate = k48000SamplingRate;
ret = addPolicyMix(MIX_TYPE_PLAYERS, MIX_ROUTE_FLAG_LOOP_BACK,
AUDIO_DEVICE_OUT_REMOTE_SUBMIX, mMixAddress, audioConfig,
std::vector<PolicyMixTuple>());
@@ -1035,7 +1035,7 @@
audioConfig.channel_mask = AUDIO_CHANNEL_OUT_STEREO;
audioConfig.format = AUDIO_FORMAT_PCM_16_BIT;
- audioConfig.sample_rate = 48000;
+ audioConfig.sample_rate = k48000SamplingRate;
ret = addPolicyMix(MIX_TYPE_PLAYERS, MIX_ROUTE_FLAG_LOOP_BACK,
AUDIO_DEVICE_OUT_REMOTE_SUBMIX, mMixAddress, audioConfig,
std::vector<PolicyMixTuple>());
@@ -1273,7 +1273,7 @@
audio_config_t audioConfig = AUDIO_CONFIG_INITIALIZER;
audioConfig.channel_mask = AUDIO_CHANNEL_OUT_STEREO;
audioConfig.format = AUDIO_FORMAT_PCM_16_BIT;
- audioConfig.sample_rate = 48000;
+ audioConfig.sample_rate = k48000SamplingRate;
ret = addPolicyMix(MIX_TYPE_PLAYERS, MIX_ROUTE_FLAG_LOOP_BACK,
AUDIO_DEVICE_OUT_REMOTE_SUBMIX, "", audioConfig, std::vector<PolicyMixTuple>());
ASSERT_EQ(INVALID_OPERATION, ret);
@@ -1307,7 +1307,7 @@
audio_config_t audioConfig = AUDIO_CONFIG_INITIALIZER;
audioConfig.channel_mask = AUDIO_CHANNEL_OUT_STEREO;
audioConfig.format = AUDIO_FORMAT_PCM_16_BIT;
- audioConfig.sample_rate = 48000;
+ audioConfig.sample_rate = k48000SamplingRate;
status_t ret = addPolicyMix(MIX_TYPE_PLAYERS, MIX_ROUTE_FLAG_LOOP_BACK,
AUDIO_DEVICE_OUT_REMOTE_SUBMIX, mMixAddress, audioConfig, mUsageRules);
ASSERT_EQ(NO_ERROR, ret);
@@ -1323,7 +1323,7 @@
std::string tags = "addr=" + mMixAddress;
strncpy(attr.tags, tags.c_str(), AUDIO_ATTRIBUTES_TAGS_MAX_SIZE - 1);
getInputForAttr(attr, mTracker->getRiid(), &selectedDeviceId, AUDIO_FORMAT_PCM_16_BIT,
- AUDIO_CHANNEL_IN_STEREO, 48000 /*sampleRate*/, AUDIO_INPUT_FLAG_NONE, &mPortId);
+ AUDIO_CHANNEL_IN_STEREO, k48000SamplingRate, AUDIO_INPUT_FLAG_NONE, &mPortId);
ASSERT_EQ(NO_ERROR, mManager->startInput(mPortId));
ASSERT_EQ(extractionPort.id, selectedDeviceId);
@@ -1350,8 +1350,8 @@
audio_port_handle_t playbackRoutedPortId = AUDIO_PORT_HANDLE_NONE;
getOutputForAttr(&playbackRoutedPortId, AUDIO_FORMAT_PCM_16_BIT, AUDIO_CHANNEL_OUT_STEREO,
- 48000 /*sampleRate*/, AUDIO_OUTPUT_FLAG_NONE,
- nullptr /*output*/, nullptr /*portId*/, attr);
+ k48000SamplingRate, AUDIO_OUTPUT_FLAG_NONE, nullptr /*output*/, nullptr /*portId*/,
+ attr);
if (std::find_if(begin(mUsageRules), end(mUsageRules), [&usage](const auto &usageRule) {
return (std::get<0>(usageRule) == usage) &&
(std::get<2>(usageRule) == RULE_MATCH_ATTRIBUTE_USAGE);}) != end(mUsageRules) ||
@@ -1493,7 +1493,7 @@
audio_config_t audioConfig = AUDIO_CONFIG_INITIALIZER;
audioConfig.channel_mask = AUDIO_CHANNEL_IN_STEREO;
audioConfig.format = AUDIO_FORMAT_PCM_16_BIT;
- audioConfig.sample_rate = 48000;
+ audioConfig.sample_rate = k48000SamplingRate;
status_t ret = addPolicyMix(MIX_TYPE_RECORDERS, MIX_ROUTE_FLAG_LOOP_BACK,
AUDIO_DEVICE_IN_REMOTE_SUBMIX, mMixAddress, audioConfig, mSourceRules);
ASSERT_EQ(NO_ERROR, ret);
@@ -1509,7 +1509,7 @@
std::string tags = std::string("addr=") + mMixAddress;
strncpy(attr.tags, tags.c_str(), AUDIO_ATTRIBUTES_TAGS_MAX_SIZE - 1);
getOutputForAttr(&selectedDeviceId, AUDIO_FORMAT_PCM_16_BIT, AUDIO_CHANNEL_OUT_STEREO,
- 48000 /*sampleRate*/, AUDIO_OUTPUT_FLAG_NONE, nullptr /*output*/, &mPortId, attr);
+ k48000SamplingRate, AUDIO_OUTPUT_FLAG_NONE, nullptr /*output*/, &mPortId, attr);
ASSERT_EQ(NO_ERROR, mManager->startOutput(mPortId));
ASSERT_EQ(injectionPort.id, getDeviceIdFromPatch(mClient->getLastAddedPatch()));
@@ -1537,7 +1537,7 @@
audio_port_handle_t captureRoutedPortId = AUDIO_PORT_HANDLE_NONE;
audio_port_handle_t portId = AUDIO_PORT_HANDLE_NONE;
getInputForAttr(attr, mTracker->getRiid(), &captureRoutedPortId, AUDIO_FORMAT_PCM_16_BIT,
- AUDIO_CHANNEL_IN_STEREO, 48000 /*sampleRate*/, AUDIO_INPUT_FLAG_NONE, &portId);
+ AUDIO_CHANNEL_IN_STEREO, k48000SamplingRate, AUDIO_INPUT_FLAG_NONE, &portId);
if (std::find_if(begin(mSourceRules), end(mSourceRules), [&source](const auto &sourceRule) {
return (std::get<1>(sourceRule) == source) &&
(std::get<2>(sourceRule) == RULE_MATCH_ATTRIBUTE_CAPTURE_PRESET);})
@@ -1687,11 +1687,11 @@
// Try start input or output according to the device type
if (audio_is_output_devices(type)) {
getOutputForAttr(&routedPortId, AUDIO_FORMAT_PCM_16_BIT, AUDIO_CHANNEL_OUT_STEREO,
- 48000 /*sampleRate*/, AUDIO_OUTPUT_FLAG_NONE);
+ k48000SamplingRate, AUDIO_OUTPUT_FLAG_NONE);
} else if (audio_is_input_device(type)) {
RecordingActivityTracker tracker;
getInputForAttr({}, tracker.getRiid(), &routedPortId, AUDIO_FORMAT_PCM_16_BIT,
- AUDIO_CHANNEL_IN_STEREO, 48000 /*sampleRate*/, AUDIO_INPUT_FLAG_NONE);
+ AUDIO_CHANNEL_IN_STEREO, k48000SamplingRate, AUDIO_INPUT_FLAG_NONE);
}
ASSERT_EQ(devicePort.id, routedPortId);
@@ -1715,6 +1715,57 @@
)
);
+class AudioPolicyManagerCarTest : public AudioPolicyManagerTestDynamicPolicy {
+protected:
+ std::string getConfigFile() override { return sCarConfig; }
+
+ static const std::string sCarConfig;
+};
+
+const std::string AudioPolicyManagerCarTest::sCarConfig =
+ AudioPolicyManagerCarTest::sExecutableDir + "test_car_ap_atmos_offload_configuration.xml";
+
+TEST_F(AudioPolicyManagerCarTest, InitSuccess) {
+ // SetUp must finish with no assertions.
+}
+
+TEST_F(AudioPolicyManagerCarTest, Dump) {
+ dumpToLog();
+}
+
+TEST_F(AudioPolicyManagerCarTest, GetOutputForAttrAtmosOutputAfterRegisteringPolicyMix) {
+ status_t ret;
+ audio_config_t audioConfig = AUDIO_CONFIG_INITIALIZER;
+ const std::string kTestBusMediaOutput = "bus0_media_out";
+ ret = addPolicyMix(MIX_TYPE_PLAYERS, MIX_ROUTE_FLAG_RENDER,
+ AUDIO_DEVICE_OUT_BUS, kTestBusMediaOutput, audioConfig, std::vector<PolicyMixTuple>());
+ ASSERT_EQ(NO_ERROR, ret);
+
+ audio_port_handle_t selectedDeviceId = AUDIO_PORT_HANDLE_NONE;
+ audio_io_handle_t output;
+ audio_port_handle_t portId;
+ getOutputForAttr(&selectedDeviceId, AUDIO_FORMAT_E_AC3_JOC, AUDIO_CHANNEL_OUT_5POINT1,
+ k48000SamplingRate, AUDIO_OUTPUT_FLAG_DIRECT, &output, &portId);
+ ASSERT_NE(AUDIO_PORT_HANDLE_NONE, selectedDeviceId);
+ sp<SwAudioOutputDescriptor> outDesc = mManager->getOutputs().valueFor(output);
+ ASSERT_NE(nullptr, outDesc.get());
+ ASSERT_EQ(AUDIO_FORMAT_E_AC3_JOC, outDesc->getFormat());
+ ASSERT_EQ(AUDIO_CHANNEL_OUT_5POINT1, outDesc->getChannelMask());
+ ASSERT_EQ(k48000SamplingRate, outDesc->getSamplingRate());
+
+ selectedDeviceId = AUDIO_PORT_HANDLE_NONE;
+ output = AUDIO_IO_HANDLE_NONE;
+ portId = AUDIO_PORT_HANDLE_NONE;
+ getOutputForAttr(&selectedDeviceId, AUDIO_FORMAT_PCM_16_BIT, AUDIO_CHANNEL_OUT_7POINT1POINT4,
+ k48000SamplingRate, AUDIO_OUTPUT_FLAG_DIRECT, &output, &portId);
+ ASSERT_NE(AUDIO_PORT_HANDLE_NONE, selectedDeviceId);
+ outDesc = mManager->getOutputs().valueFor(output);
+ ASSERT_NE(nullptr, outDesc.get());
+ ASSERT_EQ(AUDIO_FORMAT_PCM_16_BIT, outDesc->getFormat());
+ ASSERT_EQ(AUDIO_CHANNEL_OUT_7POINT1POINT4, outDesc->getChannelMask());
+ ASSERT_EQ(k48000SamplingRate, outDesc->getSamplingRate());
+}
+
class AudioPolicyManagerTVTest : public AudioPolicyManagerTestWithConfigurationFile {
protected:
std::string getConfigFile() override { return sTvConfig; }
@@ -1735,8 +1786,8 @@
audio_port_handle_t selectedDeviceId = AUDIO_PORT_HANDLE_NONE;
audio_io_handle_t output;
audio_port_handle_t portId;
- getOutputForAttr(&selectedDeviceId, AUDIO_FORMAT_PCM_16_BIT, AUDIO_CHANNEL_OUT_STEREO, 48000,
- flags, &output, &portId);
+ getOutputForAttr(&selectedDeviceId, AUDIO_FORMAT_PCM_16_BIT, AUDIO_CHANNEL_OUT_STEREO,
+ k48000SamplingRate, flags, &output, &portId);
sp<SwAudioOutputDescriptor> outDesc = mManager->getOutputs().valueFor(output);
ASSERT_NE(nullptr, outDesc.get());
audio_port_v7 port = {};
diff --git a/services/audiopolicy/tests/resources/Android.bp b/services/audiopolicy/tests/resources/Android.bp
index ff4d568..5e71210 100644
--- a/services/audiopolicy/tests/resources/Android.bp
+++ b/services/audiopolicy/tests/resources/Android.bp
@@ -12,6 +12,7 @@
srcs: [
"test_audio_policy_configuration.xml",
"test_audio_policy_primary_only_configuration.xml",
+ "test_car_ap_atmos_offload_configuration.xml",
"test_invalid_audio_policy_configuration.xml",
"test_tv_apm_configuration.xml",
"test_settop_box_surround_configuration.xml",
diff --git a/services/audiopolicy/tests/resources/test_car_ap_atmos_offload_configuration.xml b/services/audiopolicy/tests/resources/test_car_ap_atmos_offload_configuration.xml
new file mode 100644
index 0000000..d131ed8
--- /dev/null
+++ b/services/audiopolicy/tests/resources/test_car_ap_atmos_offload_configuration.xml
@@ -0,0 +1,308 @@
+<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
+<!-- Copyright (C) 2018 The Android Open Source Project
+
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+<audioPolicyConfiguration version="1.0" xmlns:xi="http://www.w3.org/2001/XInclude">
+ <globalConfiguration speaker_drc_enabled="true"/>
+
+ <modules>
+ <!-- Primary Audio HAL -->
+ <module name="primary" halVersion="3.0">
+ <attachedDevices>
+ <!-- One bus per context -->
+ <item>bus0_media_out</item>
+ <item>bus1_navigation_out</item>
+ <item>bus2_voice_command_out</item>
+ <item>bus3_call_ring_out</item>
+ <item>bus4_call_out</item>
+ <item>bus5_alarm_out</item>
+ <item>bus6_notification_out</item>
+ <item>bus7_system_sound_out</item>
+ <!-- names with _audio_zone_# are used for defined an emulator rear seat audio zone
+ where each number # is the zone id number -->
+ <item>bus100_audio_zone_1</item>
+ <item>bus200_audio_zone_2</item>
+ <item>Built-In Mic</item>
+ <item>Built-In Back Mic</item>
+ <item>Echo-Reference Mic</item>
+ <item>FM Tuner</item>
+ <item>Tone Generator 0</item>
+ <item>Tone Generator 1</item>
+ </attachedDevices>
+ <defaultOutputDevice>bus0_media_out</defaultOutputDevice>
+ <mixPorts>
+ <mixPort name="mixport_bus0_media_out" role="source"
+ flags="AUDIO_OUTPUT_FLAG_PRIMARY">
+ <profile name="" format="AUDIO_FORMAT_PCM_16_BIT"
+ samplingRates="48000"
+ channelMasks="AUDIO_CHANNEL_OUT_STEREO"/>
+ </mixPort>
+ <mixPort name="mixport_bus0_media_out_atmos" role="source"
+ flags="AUDIO_OUTPUT_FLAG_DIRECT">
+ <profile name="" format="AUDIO_FORMAT_E_AC3_JOC"
+ samplingRates="48000"
+ channelMasks="AUDIO_CHANNEL_OUT_STEREO,AUDIO_CHANNEL_OUT_5POINT1"/>
+ </mixPort>
+ <mixPort name="mixport_bus0_media_out_atmos_pcm" role="source"
+ flags="AUDIO_OUTPUT_FLAG_DIRECT">
+ <profile name="" format="AUDIO_FORMAT_PCM_16_BIT"
+ samplingRates="48000"
+ channelMasks="AUDIO_CHANNEL_OUT_7POINT1POINT4"/>
+ </mixPort>
+ <mixPort name="mixport_bus1_navigation_out" role="source">
+ <profile name="" format="AUDIO_FORMAT_PCM_16_BIT"
+ samplingRates="48000"
+ channelMasks="AUDIO_CHANNEL_OUT_STEREO"/>
+ </mixPort>
+ <mixPort name="mixport_bus2_voice_command_out" role="source">
+ <profile name="" format="AUDIO_FORMAT_PCM_16_BIT"
+ samplingRates="48000"
+ channelMasks="AUDIO_CHANNEL_OUT_STEREO"/>
+ </mixPort>
+ <mixPort name="mixport_bus3_call_ring_out" role="source">
+ <profile name="" format="AUDIO_FORMAT_PCM_16_BIT"
+ samplingRates="48000"
+ channelMasks="AUDIO_CHANNEL_OUT_STEREO"/>
+ </mixPort>
+ <mixPort name="mixport_bus4_call_out" role="source">
+ <profile name="" format="AUDIO_FORMAT_PCM_16_BIT"
+ samplingRates="48000"
+ channelMasks="AUDIO_CHANNEL_OUT_STEREO"/>
+ </mixPort>
+ <mixPort name="mixport_bus5_alarm_out" role="source">
+ <profile name="" format="AUDIO_FORMAT_PCM_16_BIT"
+ samplingRates="48000"
+ channelMasks="AUDIO_CHANNEL_OUT_STEREO"/>
+ </mixPort>
+ <mixPort name="mixport_bus6_notification_out" role="source">
+ <profile name="" format="AUDIO_FORMAT_PCM_16_BIT"
+ samplingRates="48000"
+ channelMasks="AUDIO_CHANNEL_OUT_STEREO"/>
+ </mixPort>
+ <mixPort name="mixport_bus7_system_sound_out" role="source">
+ <profile name="" format="AUDIO_FORMAT_PCM_16_BIT"
+ samplingRates="48000"
+ channelMasks="AUDIO_CHANNEL_OUT_STEREO"/>
+ </mixPort>
+ <mixPort name="mixport_bus100_audio_zone_1" role="source">
+ <profile name="" format="AUDIO_FORMAT_PCM_16_BIT"
+ samplingRates="48000"
+ channelMasks="AUDIO_CHANNEL_OUT_STEREO"/>
+ </mixPort>
+ <mixPort name="mixport_bus200_audio_zone_2" role="source">
+ <profile name="" format="AUDIO_FORMAT_PCM_16_BIT"
+ samplingRates="48000"
+ channelMasks="AUDIO_CHANNEL_OUT_STEREO"/>
+ </mixPort>
+ <mixPort name="primary input" role="sink">
+ <profile name="" format="AUDIO_FORMAT_PCM_16_BIT"
+ samplingRates="8000,11025,12000,16000,22050,24000,32000,44100,48000"
+ channelMasks="AUDIO_CHANNEL_IN_MONO,AUDIO_CHANNEL_IN_STEREO,AUDIO_CHANNEL_IN_FRONT_BACK"/>
+ </mixPort>
+ <mixPort name="mixport_tuner0" role="sink">
+ <profile name="" format="AUDIO_FORMAT_PCM_16_BIT"
+ samplingRates="48000"
+ channelMasks="AUDIO_CHANNEL_IN_STEREO"/>
+ </mixPort>
+ <mixPort name="mixport_input_bus_tone_zone_0" role="sink">
+ <profile name="" format="AUDIO_FORMAT_PCM_16_BIT"
+ samplingRates="48000"
+ channelMasks="AUDIO_CHANNEL_IN_STEREO"/>
+ </mixPort>
+ <mixPort name="mixport_input_bus_tone_zone_1" role="sink">
+ <profile name="" format="AUDIO_FORMAT_PCM_16_BIT"
+ samplingRates="48000"
+ channelMasks="AUDIO_CHANNEL_IN_STEREO"/>
+ </mixPort>
+ </mixPorts>
+ <devicePorts>
+ <devicePort tagName="bus0_media_out" role="sink" type="AUDIO_DEVICE_OUT_BUS"
+ address="bus0_media_out">
+ <profile name="" format="AUDIO_FORMAT_PCM_16_BIT"
+ samplingRates="48000" channelMasks="AUDIO_CHANNEL_OUT_STEREO"/>
+ <profile name="" format="AUDIO_FORMAT_E_AC3_JOC"
+ samplingRates="48000"
+ channelMasks="AUDIO_CHANNEL_OUT_STEREO,AUDIO_CHANNEL_OUT_5POINT1"/>
+ <profile name="" format="AUDIO_FORMAT_PCM_16_BIT"
+ samplingRates="48000" channelMasks="AUDIO_CHANNEL_OUT_7POINT1POINT4"/>
+ <gains>
+ <gain name="" mode="AUDIO_GAIN_MODE_JOINT"
+ minValueMB="-3200" maxValueMB="600" defaultValueMB="0"
+ stepValueMB="100"/>
+ </gains>
+ </devicePort>
+ <devicePort tagName="bus1_navigation_out" role="sink" type="AUDIO_DEVICE_OUT_BUS"
+ address="bus1_navigation_out">
+ <profile name="" format="AUDIO_FORMAT_PCM_16_BIT"
+ samplingRates="48000" channelMasks="AUDIO_CHANNEL_OUT_STEREO"/>
+ <gains>
+ <gain name="" mode="AUDIO_GAIN_MODE_JOINT"
+ minValueMB="-3200" maxValueMB="600" defaultValueMB="0"
+ stepValueMB="100"/>
+ </gains>
+ </devicePort>
+ <devicePort tagName="bus2_voice_command_out" role="sink" type="AUDIO_DEVICE_OUT_BUS"
+ address="bus2_voice_command_out">
+ <profile name="" format="AUDIO_FORMAT_PCM_16_BIT"
+ samplingRates="48000" channelMasks="AUDIO_CHANNEL_OUT_STEREO"/>
+ <gains>
+ <gain name="" mode="AUDIO_GAIN_MODE_JOINT"
+ minValueMB="-3200" maxValueMB="600" defaultValueMB="0"
+ stepValueMB="100"/>
+ </gains>
+ </devicePort>
+ <devicePort tagName="bus3_call_ring_out" role="sink" type="AUDIO_DEVICE_OUT_BUS"
+ address="bus3_call_ring_out">
+ <profile name="" format="AUDIO_FORMAT_PCM_16_BIT"
+ samplingRates="48000" channelMasks="AUDIO_CHANNEL_OUT_STEREO"/>
+ <gains>
+ <gain name="" mode="AUDIO_GAIN_MODE_JOINT"
+ minValueMB="-3200" maxValueMB="600" defaultValueMB="0"
+ stepValueMB="100"/>
+ </gains>
+ </devicePort>
+ <devicePort tagName="bus4_call_out" role="sink" type="AUDIO_DEVICE_OUT_BUS"
+ address="bus4_call_out">
+ <profile name="" format="AUDIO_FORMAT_PCM_16_BIT"
+ samplingRates="48000" channelMasks="AUDIO_CHANNEL_OUT_STEREO"/>
+ <gains>
+ <gain name="" mode="AUDIO_GAIN_MODE_JOINT"
+ minValueMB="-3200" maxValueMB="600" defaultValueMB="0"
+ stepValueMB="100"/>
+ </gains>
+ </devicePort>
+ <devicePort tagName="bus5_alarm_out" role="sink" type="AUDIO_DEVICE_OUT_BUS"
+ address="bus5_alarm_out">
+ <profile name="" format="AUDIO_FORMAT_PCM_16_BIT"
+ samplingRates="48000" channelMasks="AUDIO_CHANNEL_OUT_STEREO"/>
+ <gains>
+ <gain name="" mode="AUDIO_GAIN_MODE_JOINT"
+ minValueMB="-3200" maxValueMB="600" defaultValueMB="0"
+ stepValueMB="100"/>
+ </gains>
+ </devicePort>
+ <devicePort tagName="bus6_notification_out" role="sink" type="AUDIO_DEVICE_OUT_BUS"
+ address="bus6_notification_out">
+ <profile name="" format="AUDIO_FORMAT_PCM_16_BIT"
+ samplingRates="48000" channelMasks="AUDIO_CHANNEL_OUT_STEREO"/>
+ <gains>
+ <gain name="" mode="AUDIO_GAIN_MODE_JOINT"
+ minValueMB="-3200" maxValueMB="600" defaultValueMB="0"
+ stepValueMB="100"/>
+ </gains>
+ </devicePort>
+ <devicePort tagName="bus7_system_sound_out" role="sink" type="AUDIO_DEVICE_OUT_BUS"
+ address="bus7_system_sound_out">
+ <profile name="" format="AUDIO_FORMAT_PCM_16_BIT"
+ samplingRates="48000" channelMasks="AUDIO_CHANNEL_OUT_STEREO"/>
+ <gains>
+ <gain name="" mode="AUDIO_GAIN_MODE_JOINT"
+ minValueMB="-3200" maxValueMB="600" defaultValueMB="0"
+ stepValueMB="100"/>
+ </gains>
+ </devicePort>
+ <devicePort tagName="bus100_audio_zone_1" role="sink" type="AUDIO_DEVICE_OUT_BUS"
+ address="bus100_audio_zone_1">
+ <profile name="" format="AUDIO_FORMAT_PCM_16_BIT"
+ samplingRates="48000" channelMasks="AUDIO_CHANNEL_OUT_STEREO"/>
+ <gains>
+ <gain name="" mode="AUDIO_GAIN_MODE_JOINT"
+ minValueMB="-3200" maxValueMB="600" defaultValueMB="0"
+ stepValueMB="100"/>
+ </gains>
+ </devicePort>
+ <devicePort tagName="bus200_audio_zone_2" role="sink" type="AUDIO_DEVICE_OUT_BUS"
+ address="bus200_audio_zone_2">
+ <profile name="" format="AUDIO_FORMAT_PCM_16_BIT"
+ samplingRates="48000" channelMasks="AUDIO_CHANNEL_OUT_STEREO"/>
+ <gains>
+ <gain name="" mode="AUDIO_GAIN_MODE_JOINT"
+ minValueMB="-3200" maxValueMB="600" defaultValueMB="0"
+ stepValueMB="100"/>
+ </gains>
+ </devicePort>
+ <devicePort tagName="Built-In Mic" type="AUDIO_DEVICE_IN_BUILTIN_MIC" role="source">
+ <profile name="" format="AUDIO_FORMAT_PCM_16_BIT"
+ samplingRates="8000,11025,12000,16000,22050,24000,32000,44100,48000"
+ channelMasks="AUDIO_CHANNEL_IN_MONO,AUDIO_CHANNEL_IN_STEREO,AUDIO_CHANNEL_IN_FRONT_BACK"/>
+ </devicePort>
+ <devicePort tagName="Built-In Back Mic" type="AUDIO_DEVICE_IN_BACK_MIC"
+ role="source">
+ <profile name="" format="AUDIO_FORMAT_PCM_16_BIT"
+ samplingRates="8000,11025,12000,16000,22050,24000,32000,44100,48000"
+ channelMasks="AUDIO_CHANNEL_IN_MONO,AUDIO_CHANNEL_IN_STEREO,AUDIO_CHANNEL_IN_FRONT_BACK"/>
+ </devicePort>
+ <devicePort tagName="Echo-Reference Mic" type="AUDIO_DEVICE_IN_ECHO_REFERENCE"
+ role="source">
+ <profile name="" format="AUDIO_FORMAT_PCM_16_BIT"
+ samplingRates="8000,11025,12000,16000,22050,24000,32000,44100,48000"
+ channelMasks="AUDIO_CHANNEL_IN_MONO,AUDIO_CHANNEL_IN_STEREO,AUDIO_CHANNEL_IN_FRONT_BACK"/>
+ </devicePort>
+ <devicePort tagName="FM Tuner" type="AUDIO_DEVICE_IN_FM_TUNER" role="source"
+ address="tuner0">
+ <profile name="" format="AUDIO_FORMAT_PCM_16_BIT"
+ samplingRates="48000" channelMasks="AUDIO_CHANNEL_IN_STEREO"/>
+ <gains>
+ <gain name="" mode="AUDIO_GAIN_MODE_JOINT"
+ minValueMB="-3200" maxValueMB="600" defaultValueMB="0"
+ stepValueMB="100"/>
+ </gains>
+ </devicePort>
+ <devicePort tagName="Tone Generator 0" type="AUDIO_DEVICE_IN_BUS" role="source"
+ address="input_bus_tone_zone_0">
+ <profile name="" format="AUDIO_FORMAT_PCM_16_BIT"
+ samplingRates="48000" channelMasks="AUDIO_CHANNEL_IN_STEREO"/>
+ <gains>
+ <gain name="" mode="AUDIO_GAIN_MODE_JOINT"
+ minValueMB="-3200" maxValueMB="600" defaultValueMB="0"
+ stepValueMB="100"/>
+ </gains>
+ </devicePort>
+ <devicePort tagName="Tone Generator 1" type="AUDIO_DEVICE_IN_BUS" role="source"
+ address="input_bus_tone_zone_1">
+ <profile name="" format="AUDIO_FORMAT_PCM_16_BIT"
+ samplingRates="48000" channelMasks="AUDIO_CHANNEL_IN_STEREO"/>
+ <gains>
+ <gain name="" mode="AUDIO_GAIN_MODE_JOINT"
+ minValueMB="-3200" maxValueMB="600" defaultValueMB="0"
+ stepValueMB="100"/>
+ </gains>
+ </devicePort>
+ </devicePorts>
+ <!-- route declaration, i.e. list all available sources for a given sink -->
+ <routes>
+ <route type="mix" sink="bus0_media_out"
+ sources="mixport_bus0_media_out,mixport_bus0_media_out_atmos,mixport_bus0_media_out_atmos_pcm"/>
+ <route type="mix" sink="bus1_navigation_out" sources="mixport_bus1_navigation_out"/>
+ <route type="mix" sink="bus2_voice_command_out"
+ sources="mixport_bus2_voice_command_out"/>
+ <route type="mix" sink="bus3_call_ring_out" sources="mixport_bus3_call_ring_out"/>
+ <route type="mix" sink="bus4_call_out" sources="mixport_bus4_call_out"/>
+ <route type="mix" sink="bus5_alarm_out" sources="mixport_bus5_alarm_out"/>
+ <route type="mix" sink="bus6_notification_out"
+ sources="mixport_bus6_notification_out"/>
+ <route type="mix" sink="bus7_system_sound_out"
+ sources="mixport_bus7_system_sound_out"/>
+ <route type="mix" sink="bus100_audio_zone_1" sources="mixport_bus100_audio_zone_1"/>
+ <route type="mix" sink="bus200_audio_zone_2" sources="mixport_bus200_audio_zone_2"/>
+ <route type="mix" sink="primary input"
+ sources="Built-In Mic,Built-In Back Mic,Echo-Reference Mic"/>
+ <route type="mix" sink="mixport_tuner0" sources="FM Tuner"/>
+ <route type="mix" sink="mixport_input_bus_tone_zone_0" sources="Tone Generator 0"/>
+ <route type="mix" sink="mixport_input_bus_tone_zone_1" sources="Tone Generator 1"/>
+ </routes>
+ </module>
+ </modules>
+</audioPolicyConfiguration>
diff --git a/services/camera/libcameraservice/CameraService.cpp b/services/camera/libcameraservice/CameraService.cpp
index 80410ab..a6cd5cd 100644
--- a/services/camera/libcameraservice/CameraService.cpp
+++ b/services/camera/libcameraservice/CameraService.cpp
@@ -203,6 +203,7 @@
status_t res;
std::vector<std::string> deviceIds;
+ std::unordered_map<std::string, std::set<std::string>> unavailPhysicalIds;
{
Mutex::Autolock l(mServiceLock);
@@ -233,7 +234,7 @@
ALOGE("Failed to enumerate flash units: %s (%d)", strerror(-res), res);
}
- deviceIds = mCameraProviderManager->getCameraDeviceIds();
+ deviceIds = mCameraProviderManager->getCameraDeviceIds(&unavailPhysicalIds);
}
@@ -242,6 +243,12 @@
if (getCameraState(id8) == nullptr) {
onDeviceStatusChanged(id8, CameraDeviceStatus::PRESENT);
}
+ if (unavailPhysicalIds.count(cameraId) > 0) {
+ for (const auto& physicalId : unavailPhysicalIds[cameraId]) {
+ String8 physicalId8 = String8(physicalId.c_str());
+ onDeviceStatusChanged(id8, physicalId8, CameraDeviceStatus::NOT_PRESENT);
+ }
+ }
}
// Derive primary rear/front cameras, and filter their charactierstics.
@@ -335,7 +342,9 @@
int facing = -1;
int orientation = 0;
String8 cameraId8(cameraId.c_str());
- getDeviceVersion(cameraId8, /*out*/&facing, /*out*/&orientation);
+ int portraitRotation;
+ getDeviceVersion(cameraId8, /*overrideToPortrait*/false, /*out*/&portraitRotation,
+ /*out*/&facing, /*out*/&orientation);
if (facing == -1) {
ALOGE("%s: Unable to get camera device \"%s\" facing", __FUNCTION__, cameraId.c_str());
return;
@@ -495,7 +504,7 @@
if (state == nullptr) {
ALOGE("%s: Physical camera id %s status change on a non-present ID %s",
- __FUNCTION__, id.string(), physicalId.string());
+ __FUNCTION__, physicalId.string(), id.string());
return;
}
@@ -675,7 +684,7 @@
return Status::ok();
}
-Status CameraService::getCameraInfo(int cameraId,
+Status CameraService::getCameraInfo(int cameraId, bool overrideToPortrait,
CameraInfo* cameraInfo) {
ATRACE_CALL();
Mutex::Autolock l(mServiceLock);
@@ -703,8 +712,9 @@
}
Status ret = Status::ok();
+ int portraitRotation;
status_t err = mCameraProviderManager->getCameraInfo(
- cameraIdStr.c_str(), cameraInfo);
+ cameraIdStr.c_str(), overrideToPortrait, &portraitRotation, cameraInfo);
if (err != OK) {
ret = STATUS_ERROR_FMT(ERROR_INVALID_OPERATION,
"Error retrieving camera info from device %d: %s (%d)", cameraId,
@@ -739,7 +749,7 @@
}
Status CameraService::getCameraCharacteristics(const String16& cameraId,
- int targetSdkVersion, CameraMetadata* cameraInfo) {
+ int targetSdkVersion, bool overrideToPortrait, CameraMetadata* cameraInfo) {
ATRACE_CALL();
if (!cameraInfo) {
ALOGE("%s: cameraInfo is NULL", __FUNCTION__);
@@ -766,7 +776,7 @@
SessionConfigurationUtils::targetPerfClassPrimaryCamera(mPerfClassPrimaryCameraIds,
cameraIdStr, targetSdkVersion);
status_t res = mCameraProviderManager->getCameraCharacteristics(
- cameraIdStr, overrideForPerfClass, cameraInfo);
+ cameraIdStr, overrideForPerfClass, cameraInfo, overrideToPortrait);
if (res != OK) {
if (res == NAME_NOT_FOUND) {
return STATUS_ERROR_FMT(ERROR_ILLEGAL_ARGUMENT, "Unable to retrieve camera "
@@ -887,8 +897,8 @@
BasicClient::BasicClient::sCameraService = nullptr;
}
-std::pair<int, IPCTransport> CameraService::getDeviceVersion(const String8& cameraId, int* facing,
- int* orientation) {
+std::pair<int, IPCTransport> CameraService::getDeviceVersion(const String8& cameraId,
+ bool overrideToPortrait, int* portraitRotation, int* facing, int* orientation) {
ATRACE_CALL();
int deviceVersion = 0;
@@ -907,7 +917,8 @@
hardware::CameraInfo info;
if (facing) {
- res = mCameraProviderManager->getCameraInfo(cameraId.string(), &info);
+ res = mCameraProviderManager->getCameraInfo(cameraId.string(), overrideToPortrait,
+ portraitRotation, &info);
if (res != OK) {
return std::make_pair(-1, IPCTransport::INVALID);
}
@@ -942,7 +953,8 @@
const std::optional<String16>& featureId, const String8& cameraId,
int api1CameraId, int facing, int sensorOrientation, int clientPid, uid_t clientUid,
int servicePid, std::pair<int, IPCTransport> deviceVersionAndTransport,
- apiLevel effectiveApiLevel, bool overrideForPerfClass, /*out*/sp<BasicClient>* client) {
+ apiLevel effectiveApiLevel, bool overrideForPerfClass, bool overrideToPortrait,
+ bool forceSlowJpegMode, /*out*/sp<BasicClient>* client) {
// For HIDL devices
if (deviceVersionAndTransport.second == IPCTransport::HIDL) {
// Create CameraClient based on device version reported by the HAL.
@@ -975,13 +987,16 @@
sp<ICameraClient> tmp = static_cast<ICameraClient*>(cameraCb.get());
*client = new Camera2Client(cameraService, tmp, packageName, featureId,
cameraId, api1CameraId, facing, sensorOrientation, clientPid, clientUid,
- servicePid, overrideForPerfClass);
+ servicePid, overrideForPerfClass, overrideToPortrait, forceSlowJpegMode);
+ ALOGI("%s: Camera1 API (legacy), override to portrait %d, forceSlowJpegMode %d",
+ __FUNCTION__, overrideToPortrait, forceSlowJpegMode);
} else { // Camera2 API route
sp<hardware::camera2::ICameraDeviceCallbacks> tmp =
static_cast<hardware::camera2::ICameraDeviceCallbacks*>(cameraCb.get());
*client = new CameraDeviceClient(cameraService, tmp, packageName,
systemNativeClient, featureId, cameraId, facing, sensorOrientation,
- clientPid, clientUid, servicePid, overrideForPerfClass);
+ clientPid, clientUid, servicePid, overrideForPerfClass, overrideToPortrait);
+ ALOGI("%s: Camera2 API, override to portrait %d", __FUNCTION__, overrideToPortrait);
}
return Status::ok();
}
@@ -1071,7 +1086,8 @@
sp<ICameraClient>{nullptr}, id, cameraId,
internalPackageName, /*systemNativeClient*/ false, {}, uid, USE_CALLING_PID,
API_1, /*shimUpdateOnly*/ true, /*oomScoreOffset*/ 0,
- /*targetSdkVersion*/ __ANDROID_API_FUTURE__, /*out*/ tmp)
+ /*targetSdkVersion*/ __ANDROID_API_FUTURE__, /*overrideToPortrait*/ true,
+ /*forceSlowJpegMode*/false, /*out*/ tmp)
).isOk()) {
ALOGE("%s: Error initializing shim metadata: %s", __FUNCTION__, ret.toString8().string());
}
@@ -1587,6 +1603,8 @@
int clientUid,
int clientPid,
int targetSdkVersion,
+ bool overrideToPortrait,
+ bool forceSlowJpegMode,
/*out*/
sp<ICamera>* device) {
@@ -1597,7 +1615,8 @@
sp<Client> client = nullptr;
ret = connectHelper<ICameraClient,Client>(cameraClient, id, api1CameraId,
clientPackageName,/*systemNativeClient*/ false, {}, clientUid, clientPid, API_1,
- /*shimUpdateOnly*/ false, /*oomScoreOffset*/ 0, targetSdkVersion, /*out*/client);
+ /*shimUpdateOnly*/ false, /*oomScoreOffset*/ 0, targetSdkVersion,
+ overrideToPortrait, forceSlowJpegMode, /*out*/client);
if(!ret.isOk()) {
logRejected(id, CameraThreadState::getCallingPid(), String8(clientPackageName),
@@ -1672,6 +1691,7 @@
const String16& clientPackageName,
const std::optional<String16>& clientFeatureId,
int clientUid, int oomScoreOffset, int targetSdkVersion,
+ bool overrideToPortrait,
/*out*/
sp<hardware::camera2::ICameraDeviceUser>* device) {
@@ -1698,7 +1718,13 @@
return STATUS_ERROR(ERROR_ILLEGAL_ARGUMENT, msg.string());
}
- if (CameraServiceProxyWrapper::isCameraDisabled()) {
+ userid_t clientUserId = multiuser_get_user_id(clientUid);
+ int callingUid = CameraThreadState::getCallingUid();
+ if (clientUid == USE_CALLING_UID) {
+ clientUserId = multiuser_get_user_id(callingUid);
+ }
+
+ if (CameraServiceProxyWrapper::isCameraDisabled(clientUserId)) {
String8 msg =
String8::format("Camera disabled by device policy");
ALOGE("%s: %s", __FUNCTION__, msg.string());
@@ -1719,7 +1745,8 @@
ret = connectHelper<hardware::camera2::ICameraDeviceCallbacks,CameraDeviceClient>(cameraCb, id,
/*api1CameraId*/-1, clientPackageNameAdj, systemNativeClient,clientFeatureId,
clientUid, USE_CALLING_PID, API_2, /*shimUpdateOnly*/ false, oomScoreOffset,
- targetSdkVersion, /*out*/client);
+ targetSdkVersion, overrideToPortrait, /*forceSlowJpegMode*/false,
+ /*out*/client);
if(!ret.isOk()) {
logRejected(id, callingPid, String8(clientPackageNameAdj), ret.toString8());
@@ -1746,20 +1773,69 @@
return ret;
}
+String16 CameraService::getPackageNameFromUid(int clientUid) {
+ String16 packageName("");
+
+ sp<IServiceManager> sm = defaultServiceManager();
+ sp<IBinder> binder = sm->getService(String16(kPermissionServiceName));
+ if (binder == 0) {
+ ALOGE("Cannot get permission service");
+ // Return empty package name and the further interaction
+ // with camera will likely fail
+ return packageName;
+ }
+
+ sp<IPermissionController> permCtrl = interface_cast<IPermissionController>(binder);
+ Vector<String16> packages;
+
+ permCtrl->getPackagesForUid(clientUid, packages);
+
+ if (packages.isEmpty()) {
+ ALOGE("No packages for calling UID %d", clientUid);
+ // Return empty package name and the further interaction
+ // with camera will likely fail
+ return packageName;
+ }
+
+ // Arbitrarily pick the first name in the list
+ packageName = packages[0];
+
+ return packageName;
+}
+
template<class CALLBACK, class CLIENT>
Status CameraService::connectHelper(const sp<CALLBACK>& cameraCb, const String8& cameraId,
- int api1CameraId, const String16& clientPackageName, bool systemNativeClient,
+ int api1CameraId, const String16& clientPackageNameMaybe, bool systemNativeClient,
const std::optional<String16>& clientFeatureId, int clientUid, int clientPid,
apiLevel effectiveApiLevel, bool shimUpdateOnly, int oomScoreOffset, int targetSdkVersion,
+ bool overrideToPortrait, bool forceSlowJpegMode,
/*out*/sp<CLIENT>& device) {
binder::Status ret = binder::Status::ok();
+ bool isNonSystemNdk = false;
+ String16 clientPackageName;
+ if (clientPackageNameMaybe.size() <= 0) {
+ // NDK calls don't come with package names, but we need one for various cases.
+ // Generally, there's a 1:1 mapping between UID and package name, but shared UIDs
+ // do exist. For all authentication cases, all packages under the same UID get the
+ // same permissions, so picking any associated package name is sufficient. For some
+ // other cases, this may give inaccurate names for clients in logs.
+ isNonSystemNdk = true;
+ int packageUid = (clientUid == USE_CALLING_UID) ?
+ CameraThreadState::getCallingUid() : clientUid;
+ clientPackageName = getPackageNameFromUid(packageUid);
+ } else {
+ clientPackageName = clientPackageNameMaybe;
+ }
+
String8 clientName8(clientPackageName);
int originalClientPid = 0;
+ int packagePid = (clientPid == USE_CALLING_PID) ?
+ CameraThreadState::getCallingPid() : clientPid;
ALOGI("CameraService::connect call (PID %d \"%s\", camera ID %s) and "
- "Camera API version %d", clientPid, clientName8.string(), cameraId.string(),
+ "Camera API version %d", packagePid, clientName8.string(), cameraId.string(),
static_cast<int>(effectiveApiLevel));
nsecs_t openTimeNs = systemTime();
@@ -1767,7 +1843,7 @@
sp<CLIENT> client = nullptr;
int facing = -1;
int orientation = 0;
- bool isNonSystemNdk = (clientPackageName.size() == 0);
+
{
// Acquire mServiceLock and prevent other clients from connecting
std::unique_ptr<AutoConditionLock> lock =
@@ -1832,8 +1908,10 @@
// give flashlight a chance to close devices if necessary.
mFlashlight->prepareDeviceOpen(cameraId);
+ int portraitRotation;
auto deviceVersionAndTransport =
- getDeviceVersion(cameraId, /*out*/&facing, /*out*/&orientation);
+ getDeviceVersion(cameraId, overrideToPortrait, /*out*/&portraitRotation,
+ /*out*/&facing, /*out*/&orientation);
if (facing == -1) {
ALOGE("%s: Unable to get camera device \"%s\" facing", __FUNCTION__, cameraId.string());
return STATUS_ERROR_FMT(ERROR_INVALID_OPERATION,
@@ -1847,6 +1925,7 @@
clientFeatureId, cameraId, api1CameraId, facing, orientation,
clientPid, clientUid, getpid(),
deviceVersionAndTransport, effectiveApiLevel, overrideForPerfClass,
+ overrideToPortrait, forceSlowJpegMode,
/*out*/&tmp)).isOk()) {
return ret;
}
@@ -1901,11 +1980,31 @@
}
}
+ // Enable/disable camera service watchdog
+ client->setCameraServiceWatchdog(mCameraServiceWatchdogEnabled);
+
// Set rotate-and-crop override behavior
if (mOverrideRotateAndCropMode != ANDROID_SCALER_ROTATE_AND_CROP_AUTO) {
client->setRotateAndCropOverride(mOverrideRotateAndCropMode);
+ } else if (overrideToPortrait && portraitRotation != 0) {
+ uint8_t rotateAndCropMode = ANDROID_SCALER_ROTATE_AND_CROP_AUTO;
+ switch (portraitRotation) {
+ case 90:
+ rotateAndCropMode = ANDROID_SCALER_ROTATE_AND_CROP_90;
+ break;
+ case 180:
+ rotateAndCropMode = ANDROID_SCALER_ROTATE_AND_CROP_180;
+ break;
+ case 270:
+ rotateAndCropMode = ANDROID_SCALER_ROTATE_AND_CROP_270;
+ break;
+ default:
+ ALOGE("Unexpected portrait rotation: %d", portraitRotation);
+ break;
+ }
+ client->setRotateAndCropOverride(rotateAndCropMode);
} else {
- client->setRotateAndCropOverride(
+ client->setRotateAndCropOverride(
CameraServiceProxyWrapper::getRotateAndCropOverride(
clientPackageName, facing, multiuser_get_user_id(clientUid)));
}
@@ -1949,6 +2048,7 @@
}
client->setImageDumpMask(mImageDumpMask);
+ client->setStreamUseCaseOverrides(mStreamUseCaseOverrides);
} // lock is destroyed, allow further connect calls
// Important: release the mutex here so the client can call back into the service from its
@@ -2409,6 +2509,11 @@
ATRACE_CALL();
+ {
+ Mutex::Autolock lock(mServiceLock);
+ mDeviceState = newState;
+ }
+
mCameraProviderManager->notifyDeviceStateChange(newState);
return Status::ok();
@@ -2442,12 +2547,12 @@
for (auto& current : clients) {
if (current != nullptr) {
const auto basicClient = current->getValue();
- if (basicClient.get() != nullptr) {
- basicClient->setRotateAndCropOverride(
- CameraServiceProxyWrapper::getRotateAndCropOverride(
- basicClient->getPackageName(),
- basicClient->getCameraFacing(),
- multiuser_get_user_id(basicClient->getClientUid())));
+ if (basicClient.get() != nullptr && !basicClient->getOverrideToPortrait()) {
+ basicClient->setRotateAndCropOverride(
+ CameraServiceProxyWrapper::getRotateAndCropOverride(
+ basicClient->getPackageName(),
+ basicClient->getCameraFacing(),
+ multiuser_get_user_id(basicClient->getClientUid())));
}
}
}
@@ -2719,7 +2824,8 @@
return STATUS_ERROR(ERROR_ILLEGAL_ARGUMENT, msg.string());
}
- auto deviceVersionAndTransport = getDeviceVersion(id);
+ int portraitRotation;
+ auto deviceVersionAndTransport = getDeviceVersion(id, false, &portraitRotation);
if (deviceVersionAndTransport.first == -1) {
String8 msg = String8::format("Unknown camera ID %s", id.string());
ALOGE("%s: %s", __FUNCTION__, msg.string());
@@ -3204,13 +3310,13 @@
const String8& cameraIdStr,
int api1CameraId, int cameraFacing, int sensorOrientation,
int clientPid, uid_t clientUid,
- int servicePid) :
+ int servicePid, bool overrideToPortrait) :
CameraService::BasicClient(cameraService,
IInterface::asBinder(cameraClient),
clientPackageName, systemNativeClient, clientFeatureId,
cameraIdStr, cameraFacing, sensorOrientation,
clientPid, clientUid,
- servicePid),
+ servicePid, overrideToPortrait),
mCameraId(api1CameraId)
{
int callingPid = CameraThreadState::getCallingPid();
@@ -3240,7 +3346,7 @@
const String16& clientPackageName, bool nativeClient,
const std::optional<String16>& clientFeatureId, const String8& cameraIdStr,
int cameraFacing, int sensorOrientation, int clientPid, uid_t clientUid,
- int servicePid):
+ int servicePid, bool overrideToPortrait):
mDestructionStarted(false),
mCameraIdStr(cameraIdStr), mCameraFacing(cameraFacing), mOrientation(sensorOrientation),
mClientPackageName(clientPackageName), mSystemNativeClient(nativeClient),
@@ -3248,6 +3354,7 @@
mClientPid(clientPid), mClientUid(clientUid),
mServicePid(servicePid),
mDisconnected(false), mUidIsTrusted(false),
+ mOverrideToPortrait(overrideToPortrait),
mAudioRestriction(hardware::camera2::ICameraDeviceUser::AUDIO_RESTRICTION_NONE),
mRemoteBinder(remoteCallback),
mOpsActive(false),
@@ -3257,37 +3364,6 @@
sCameraService = cameraService;
}
- // In some cases the calling code has no access to the package it runs under.
- // For example, NDK camera API.
- // In this case we will get the packages for the calling UID and pick the first one
- // for attributing the app op. This will work correctly for runtime permissions
- // as for legacy apps we will toggle the app op for all packages in the UID.
- // The caveat is that the operation may be attributed to the wrong package and
- // stats based on app ops may be slightly off.
- if (mClientPackageName.size() <= 0) {
- sp<IServiceManager> sm = defaultServiceManager();
- sp<IBinder> binder = sm->getService(String16(kPermissionServiceName));
- if (binder == 0) {
- ALOGE("Cannot get permission service");
- // Leave mClientPackageName unchanged (empty) and the further interaction
- // with camera will fail in BasicClient::startCameraOps
- return;
- }
-
- sp<IPermissionController> permCtrl = interface_cast<IPermissionController>(binder);
- Vector<String16> packages;
-
- permCtrl->getPackagesForUid(mClientUid, packages);
-
- if (packages.isEmpty()) {
- ALOGE("No packages for calling UID");
- // Leave mClientPackageName unchanged (empty) and the further interaction
- // with camera will fail in BasicClient::startCameraOps
- return;
- }
- mClientPackageName = packages[0];
- }
-
// There are 2 scenarios in which a client won't have AppOps operations
// (both scenarios : native clients)
// 1) It's an system native client*, the package name will be empty
@@ -4349,6 +4425,13 @@
String8 activeClientString = mActiveClientManager.toString();
dprintf(fd, "Active Camera Clients:\n%s", activeClientString.string());
dprintf(fd, "Allowed user IDs: %s\n", toString(mAllowedUsers).string());
+ if (mStreamUseCaseOverrides.size() > 0) {
+ dprintf(fd, "Active stream use case overrides:");
+ for (int64_t useCaseOverride : mStreamUseCaseOverrides) {
+ dprintf(fd, " %" PRId64, useCaseOverride);
+ }
+ dprintf(fd, "\n");
+ }
dumpEventLog(fd);
@@ -4840,8 +4923,14 @@
return handleGetImageDumpMask(out);
} else if (args.size() >= 2 && args[0] == String16("set-camera-mute")) {
return handleSetCameraMute(args);
+ } else if (args.size() >= 2 && args[0] == String16("set-stream-use-case-override")) {
+ return handleSetStreamUseCaseOverrides(args);
+ } else if (args.size() >= 1 && args[0] == String16("clear-stream-use-case-override")) {
+ return handleClearStreamUseCaseOverrides();
} else if (args.size() >= 2 && args[0] == String16("watch")) {
return handleWatchCommand(args, in, out);
+ } else if (args.size() >= 2 && args[0] == String16("set-watchdog")) {
+ return handleSetCameraServiceWatchdog(args);
} else if (args.size() == 1 && args[0] == String16("help")) {
printHelp(out);
return OK;
@@ -4935,6 +5024,28 @@
return OK;
}
+status_t CameraService::handleSetCameraServiceWatchdog(const Vector<String16>& args) {
+ int enableWatchdog = atoi(String8(args[1]));
+
+ if (enableWatchdog < 0 || enableWatchdog > 1) return BAD_VALUE;
+
+ Mutex::Autolock lock(mServiceLock);
+
+ mCameraServiceWatchdogEnabled = enableWatchdog;
+
+ const auto clients = mActiveClientManager.getAll();
+ for (auto& current : clients) {
+ if (current != nullptr) {
+ const auto basicClient = current->getValue();
+ if (basicClient.get() != nullptr) {
+ basicClient->setCameraServiceWatchdog(enableWatchdog);
+ }
+ }
+ }
+
+ return OK;
+}
+
status_t CameraService::handleGetRotateAndCrop(int out) {
Mutex::Autolock lock(mServiceLock);
@@ -4988,6 +5099,43 @@
return OK;
}
+status_t CameraService::handleSetStreamUseCaseOverrides(const Vector<String16>& args) {
+ std::vector<int64_t> useCasesOverride;
+ for (size_t i = 1; i < args.size(); i++) {
+ int64_t useCase = ANDROID_SCALER_AVAILABLE_STREAM_USE_CASES_DEFAULT;
+ String8 arg8 = String8(args[i]);
+ if (arg8 == "DEFAULT") {
+ useCase = ANDROID_SCALER_AVAILABLE_STREAM_USE_CASES_DEFAULT;
+ } else if (arg8 == "PREVIEW") {
+ useCase = ANDROID_SCALER_AVAILABLE_STREAM_USE_CASES_PREVIEW;
+ } else if (arg8 == "STILL_CAPTURE") {
+ useCase = ANDROID_SCALER_AVAILABLE_STREAM_USE_CASES_STILL_CAPTURE;
+ } else if (arg8 == "VIDEO_RECORD") {
+ useCase = ANDROID_SCALER_AVAILABLE_STREAM_USE_CASES_VIDEO_RECORD;
+ } else if (arg8 == "PREVIEW_VIDEO_STILL") {
+ useCase = ANDROID_SCALER_AVAILABLE_STREAM_USE_CASES_PREVIEW_VIDEO_STILL;
+ } else if (arg8 == "VIDEO_CALL") {
+ useCase = ANDROID_SCALER_AVAILABLE_STREAM_USE_CASES_VIDEO_CALL;
+ } else {
+ ALOGE("%s: Invalid stream use case %s", __FUNCTION__, String8(args[i]).c_str());
+ return BAD_VALUE;
+ }
+ useCasesOverride.push_back(useCase);
+ }
+
+ Mutex::Autolock lock(mServiceLock);
+ mStreamUseCaseOverrides = std::move(useCasesOverride);
+
+ return OK;
+}
+
+status_t CameraService::handleClearStreamUseCaseOverrides() {
+ Mutex::Autolock lock(mServiceLock);
+ mStreamUseCaseOverrides.clear();
+
+ return OK;
+}
+
status_t CameraService::handleWatchCommand(const Vector<String16>& args, int inFd, int outFd) {
if (args.size() >= 3 && args[1] == String16("start")) {
return startWatchingTags(args, outFd);
@@ -5342,6 +5490,15 @@
" Valid values 0=OFF, 1=ON for JPEG\n"
" get-image-dump-mask returns the current image-dump-mask value\n"
" set-camera-mute <0/1> enable or disable camera muting\n"
+ " set-stream-use-case-override <usecase1> <usecase2> ... override stream use cases\n"
+ " Use cases applied in descending resolutions. So usecase1 is assigned to the\n"
+ " largest resolution, usecase2 is assigned to the 2nd largest resolution, and so\n"
+ " on. In case the number of usecases is smaller than the number of streams, the\n"
+ " last use case is assigned to all the remaining streams. In case of multiple\n"
+ " streams with the same resolution, the tie-breaker is (JPEG, RAW, YUV, and PRIV)\n"
+ " Valid values are (case sensitive): DEFAULT, PREVIEW, STILL_CAPTURE, VIDEO_RECORD,\n"
+ " PREVIEW_VIDEO_STILL, VIDEO_CALL\n"
+ " clear-stream-use-case-override clear the stream use case override\n"
" watch <start|stop|dump|print|clear> manages tag monitoring in connected clients\n"
" help print this message\n");
}
diff --git a/services/camera/libcameraservice/CameraService.h b/services/camera/libcameraservice/CameraService.h
index d96ea00..70293f4 100644
--- a/services/camera/libcameraservice/CameraService.h
+++ b/services/camera/libcameraservice/CameraService.h
@@ -127,10 +127,10 @@
// ICameraService
virtual binder::Status getNumberOfCameras(int32_t type, int32_t* numCameras);
- virtual binder::Status getCameraInfo(int cameraId,
- hardware::CameraInfo* cameraInfo);
+ virtual binder::Status getCameraInfo(int cameraId, bool overrideToPortrait,
+ hardware::CameraInfo* cameraInfo) override;
virtual binder::Status getCameraCharacteristics(const String16& cameraId,
- int targetSdkVersion, CameraMetadata* cameraInfo);
+ int targetSdkVersion, bool overrideToPortrait, CameraMetadata* cameraInfo) override;
virtual binder::Status getCameraVendorTagDescriptor(
/*out*/
hardware::camera2::params::VendorTagDescriptor* desc);
@@ -141,13 +141,14 @@
virtual binder::Status connect(const sp<hardware::ICameraClient>& cameraClient,
int32_t cameraId, const String16& clientPackageName,
int32_t clientUid, int clientPid, int targetSdkVersion,
+ bool overrideToPortrait, bool forceSlowJpegMode,
/*out*/
- sp<hardware::ICamera>* device);
+ sp<hardware::ICamera>* device) override;
virtual binder::Status connectDevice(
const sp<hardware::camera2::ICameraDeviceCallbacks>& cameraCb, const String16& cameraId,
const String16& clientPackageName, const std::optional<String16>& clientFeatureId,
- int32_t clientUid, int scoreOffset, int targetSdkVersion,
+ int32_t clientUid, int scoreOffset, int targetSdkVersion, bool overrideToPortrait,
/*out*/
sp<hardware::camera2::ICameraDeviceUser>* device);
@@ -243,8 +244,9 @@
/////////////////////////////////////////////////////////////////////
// CameraDeviceFactory functionality
- std::pair<int, IPCTransport> getDeviceVersion(const String8& cameraId, int* facing = nullptr,
- int* orientation = nullptr);
+ std::pair<int, IPCTransport> getDeviceVersion(const String8& cameraId,
+ bool overrideToPortrait, int* portraitRotation,
+ int* facing = nullptr, int* orientation = nullptr);
/////////////////////////////////////////////////////////////////////
// Methods to be used in CameraService class tests only
@@ -282,6 +284,10 @@
return mRemoteBinder;
}
+ bool getOverrideToPortrait() const {
+ return mOverrideToPortrait;
+ }
+
// Disallows dumping over binder interface
virtual status_t dump(int fd, const Vector<String16>& args);
// Internal dump method to be called by CameraService
@@ -339,6 +345,16 @@
// Set/reset camera mute
virtual status_t setCameraMute(bool enabled) = 0;
+ // Set Camera service watchdog
+ virtual status_t setCameraServiceWatchdog(bool enabled) = 0;
+
+ // Set stream use case overrides
+ virtual void setStreamUseCaseOverrides(
+ const std::vector<int64_t>& useCaseOverrides) = 0;
+
+ // Clear stream use case overrides
+ virtual void clearStreamUseCaseOverrides() = 0;
+
// The injection camera session to replace the internal camera
// session.
virtual status_t injectCamera(const String8& injectedCamId,
@@ -358,7 +374,8 @@
int sensorOrientation,
int clientPid,
uid_t clientUid,
- int servicePid);
+ int servicePid,
+ bool overrideToPortrait);
virtual ~BasicClient();
@@ -381,6 +398,7 @@
const pid_t mServicePid;
bool mDisconnected;
bool mUidIsTrusted;
+ bool mOverrideToPortrait;
mutable Mutex mAudioRestrictionLock;
int32_t mAudioRestriction;
@@ -470,7 +488,8 @@
int sensorOrientation,
int clientPid,
uid_t clientUid,
- int servicePid);
+ int servicePid,
+ bool overrideToPortrait);
~Client();
// return our camera client
@@ -490,6 +509,7 @@
virtual bool canCastToApiClient(apiLevel level) const;
void setImageDumpMask(int /*mask*/) { }
+ void setStreamUseCaseOverrides(const std::vector<int64_t>& /*usecaseOverrides*/) { }
protected:
// Initialized in constructor
@@ -825,12 +845,22 @@
// sorted in alpha-numeric order.
void filterAPI1SystemCameraLocked(const std::vector<std::string> &normalDeviceIds);
+ // In some cases the calling code has no access to the package it runs under.
+ // For example, NDK camera API.
+ // In this case we will get the packages for the calling UID and pick the first one
+ // for attributing the app op. This will work correctly for runtime permissions
+ // as for legacy apps we will toggle the app op for all packages in the UID.
+ // The caveat is that the operation may be attributed to the wrong package and
+ // stats based on app ops may be slightly off.
+ String16 getPackageNameFromUid(int clientUid);
+
// Single implementation shared between the various connect calls
template<class CALLBACK, class CLIENT>
binder::Status connectHelper(const sp<CALLBACK>& cameraCb, const String8& cameraId,
- int api1CameraId, const String16& clientPackageName, bool systemNativeClient,
+ int api1CameraId, const String16& clientPackageNameMaybe, bool systemNativeClient,
const std::optional<String16>& clientFeatureId, int clientUid, int clientPid,
apiLevel effectiveApiLevel, bool shimUpdateOnly, int scoreOffset, int targetSdkVersion,
+ bool overrideToPortrait, bool forceSlowJpegMode,
/*out*/sp<CLIENT>& device);
// Lock guarding camera service state
@@ -1195,9 +1225,18 @@
// Set the camera mute state
status_t handleSetCameraMute(const Vector<String16>& args);
+ // Set the stream use case overrides
+ status_t handleSetStreamUseCaseOverrides(const Vector<String16>& args);
+
+ // Clear the stream use case overrides
+ status_t handleClearStreamUseCaseOverrides();
+
// Handle 'watch' command as passed through 'cmd'
status_t handleWatchCommand(const Vector<String16> &args, int inFd, int outFd);
+ // Set the camera service watchdog
+ status_t handleSetCameraServiceWatchdog(const Vector<String16>& args);
+
// Enable tag monitoring of the given tags in provided clients
status_t startWatchingTags(const Vector<String16> &args, int outFd);
@@ -1243,7 +1282,8 @@
const String8& cameraId, int api1CameraId, int facing, int sensorOrientation,
int clientPid, uid_t clientUid, int servicePid,
std::pair<int, IPCTransport> deviceVersionAndIPCTransport, apiLevel effectiveApiLevel,
- bool overrideForPerfClass, /*out*/sp<BasicClient>* client);
+ bool overrideForPerfClass, bool overrideToPortrait, bool forceSlowJpegMode,
+ /*out*/sp<BasicClient>* client);
status_t checkCameraAccess(const String16& opPackageName);
@@ -1284,6 +1324,12 @@
// Current camera mute mode
bool mOverrideCameraMuteMode = false;
+ // Camera Service watchdog flag
+ bool mCameraServiceWatchdogEnabled = true;
+
+ // Current stream use case overrides
+ std::vector<int64_t> mStreamUseCaseOverrides;
+
/**
* A listener class that implements the IBinder::DeathRecipient interface
* for use to call back the error state injected by the external camera, and
@@ -1337,6 +1383,9 @@
// Guard mInjectionInternalCamId and mInjectionInitPending.
Mutex mInjectionParametersLock;
+ // Track the folded/unfoled device state. 0 == UNFOLDED, 4 == FOLDED
+ int64_t mDeviceState;
+
void updateTorchUidMapLocked(const String16& cameraId, int uid);
};
diff --git a/services/camera/libcameraservice/CameraServiceWatchdog.cpp b/services/camera/libcameraservice/CameraServiceWatchdog.cpp
index a169667..e101dd3 100644
--- a/services/camera/libcameraservice/CameraServiceWatchdog.cpp
+++ b/services/camera/libcameraservice/CameraServiceWatchdog.cpp
@@ -66,6 +66,17 @@
}
}
+void CameraServiceWatchdog::setEnabled(bool enable)
+{
+ AutoMutex _l(mEnabledLock);
+
+ if (enable) {
+ mEnabled = true;
+ } else {
+ mEnabled = false;
+ }
+}
+
void CameraServiceWatchdog::stop(uint32_t tid)
{
AutoMutex _l(mWatchdogLock);
diff --git a/services/camera/libcameraservice/CameraServiceWatchdog.h b/services/camera/libcameraservice/CameraServiceWatchdog.h
index 29ddab1..e35d69e 100644
--- a/services/camera/libcameraservice/CameraServiceWatchdog.h
+++ b/services/camera/libcameraservice/CameraServiceWatchdog.h
@@ -21,10 +21,12 @@
* expected duration has exceeded.
* Notes on multi-threaded behaviors:
* - The threadloop is blocked/paused when there are no calls being
- * monitored.
+ * monitored (when the TID cycle to counter map is empty).
* - The start and stop functions handle simultaneous call monitoring
* and single call monitoring differently. See function documentation for
* more details.
+ * To disable/enable:
+ * - adb shell cmd media.camera set-cameraservice-watchdog [0/1]
*/
#pragma once
#include <chrono>
@@ -49,15 +51,19 @@
public:
explicit CameraServiceWatchdog() : mPause(true), mMaxCycles(kMaxCycles),
- mCycleLengthMs(kCycleLengthMs) {};
+ mCycleLengthMs(kCycleLengthMs), mEnabled(true) {};
- explicit CameraServiceWatchdog (size_t maxCycles, uint32_t cycleLengthMs) :
- mPause(true), mMaxCycles(maxCycles), mCycleLengthMs(cycleLengthMs) {};
+ explicit CameraServiceWatchdog (size_t maxCycles, uint32_t cycleLengthMs, bool enabled) :
+ mPause(true), mMaxCycles(maxCycles), mCycleLengthMs(cycleLengthMs), mEnabled(enabled)
+ {};
virtual ~CameraServiceWatchdog() {};
virtual void requestExit();
+ /** Enables/disables the watchdog */
+ void setEnabled(bool enable);
+
/** Used to wrap monitored calls in start and stop functions using custom timer values */
template<typename T>
auto watchThread(T func, uint32_t tid, uint32_t cycles, uint32_t cycleLength) {
@@ -66,9 +72,20 @@
if (cycles != mMaxCycles || cycleLength != mCycleLengthMs) {
// Create another instance of the watchdog to prevent disruption
// of timer for current monitored calls
+
+ // Lock for mEnabled
+ mEnabledLock.lock();
sp<CameraServiceWatchdog> tempWatchdog =
- new CameraServiceWatchdog(cycles, cycleLength);
- tempWatchdog->run("CameraServiceWatchdog");
+ new CameraServiceWatchdog(cycles, cycleLength, mEnabled);
+ mEnabledLock.unlock();
+
+ status_t status = tempWatchdog->run("CameraServiceWatchdog");
+ if (status != OK) {
+ ALOGE("Unable to watch thread: %s (%d)", strerror(-status), status);
+ res = watchThread(func, tid);
+ return res;
+ }
+
res = tempWatchdog->watchThread(func, tid);
tempWatchdog->requestExit();
tempWatchdog.clear();
@@ -84,10 +101,16 @@
/** Used to wrap monitored calls in start and stop functions using class timer values */
template<typename T>
auto watchThread(T func, uint32_t tid) {
+ decltype(func()) res;
+ AutoMutex _l(mEnabledLock);
- start(tid);
- auto res = func();
- stop(tid);
+ if (mEnabled) {
+ start(tid);
+ res = func();
+ stop(tid);
+ } else {
+ res = func();
+ }
return res;
}
@@ -108,11 +131,13 @@
virtual bool threadLoop();
- Mutex mWatchdogLock; // Lock for condition variable
- Condition mWatchdogCondition; // Condition variable for stop/start
- bool mPause; // True if thread is currently paused
- uint32_t mMaxCycles; // Max cycles
- uint32_t mCycleLengthMs; // Length of time elapsed per cycle
+ Mutex mWatchdogLock; // Lock for condition variable
+ Mutex mEnabledLock; // Lock for enabled status
+ Condition mWatchdogCondition; // Condition variable for stop/start
+ bool mPause; // True if tid map is empty
+ uint32_t mMaxCycles; // Max cycles
+ uint32_t mCycleLengthMs; // Length of time elapsed per cycle
+ bool mEnabled; // True if watchdog is enabled
std::unordered_map<uint32_t, uint32_t> tidToCycleCounterMap; // Thread Id to cycle counter map
};
diff --git a/services/camera/libcameraservice/api1/Camera2Client.cpp b/services/camera/libcameraservice/api1/Camera2Client.cpp
index 68dc7f8..ac07376 100644
--- a/services/camera/libcameraservice/api1/Camera2Client.cpp
+++ b/services/camera/libcameraservice/api1/Camera2Client.cpp
@@ -61,11 +61,14 @@
int clientPid,
uid_t clientUid,
int servicePid,
- bool overrideForPerfClass):
+ bool overrideForPerfClass,
+ bool overrideToPortrait,
+ bool forceSlowJpegMode):
Camera2ClientBase(cameraService, cameraClient, clientPackageName,
false/*systemNativeClient - since no ndk for api1*/, clientFeatureId,
cameraDeviceId, api1CameraId, cameraFacing, sensorOrientation, clientPid,
- clientUid, servicePid, overrideForPerfClass, /*legacyClient*/ true),
+ clientUid, servicePid, overrideForPerfClass, overrideToPortrait,
+ /*legacyClient*/ true),
mParameters(api1CameraId, cameraFacing)
{
ATRACE_CALL();
@@ -76,6 +79,9 @@
SharedParameters::Lock l(mParameters);
l.mParameters.state = Parameters::DISCONNECTED;
+ if (forceSlowJpegMode) {
+ l.mParameters.isSlowJpegModeForced = true;
+ }
}
status_t Camera2Client::initialize(sp<CameraProviderManager> manager, const String8& monitorTags) {
@@ -2316,6 +2322,10 @@
return INVALID_OPERATION;
}
+status_t Camera2Client::setCameraServiceWatchdog(bool enabled) {
+ return mDevice->setCameraServiceWatchdog(enabled);
+}
+
status_t Camera2Client::setRotateAndCropOverride(uint8_t rotateAndCrop) {
if (rotateAndCrop > ANDROID_SCALER_ROTATE_AND_CROP_AUTO) return BAD_VALUE;
@@ -2341,6 +2351,15 @@
return mDevice->setCameraMute(enabled);
}
+void Camera2Client::setStreamUseCaseOverrides(
+ const std::vector<int64_t>& useCaseOverrides) {
+ mDevice->setStreamUseCaseOverrides(useCaseOverrides);
+}
+
+void Camera2Client::clearStreamUseCaseOverrides() {
+ mDevice->clearStreamUseCaseOverrides();
+}
+
status_t Camera2Client::waitUntilCurrentRequestIdLocked() {
int32_t activeRequestId = mStreamingProcessor->getActiveRequestId();
if (activeRequestId != 0) {
diff --git a/services/camera/libcameraservice/api1/Camera2Client.h b/services/camera/libcameraservice/api1/Camera2Client.h
index c8dfc46..bbad6d8 100644
--- a/services/camera/libcameraservice/api1/Camera2Client.h
+++ b/services/camera/libcameraservice/api1/Camera2Client.h
@@ -90,6 +90,12 @@
virtual bool supportsCameraMute();
virtual status_t setCameraMute(bool enabled);
+ virtual status_t setCameraServiceWatchdog(bool enabled);
+
+ virtual void setStreamUseCaseOverrides(
+ const std::vector<int64_t>& useCaseOverrides);
+ virtual void clearStreamUseCaseOverrides();
+
/**
* Interface used by CameraService
*/
@@ -105,7 +111,9 @@
int clientPid,
uid_t clientUid,
int servicePid,
- bool overrideForPerfClass);
+ bool overrideForPerfClass,
+ bool overrideToPortrait,
+ bool forceSlowJpegMode);
virtual ~Camera2Client();
diff --git a/services/camera/libcameraservice/api1/client2/FrameProcessor.cpp b/services/camera/libcameraservice/api1/client2/FrameProcessor.cpp
index 2daacd1..74423e5 100644
--- a/services/camera/libcameraservice/api1/client2/FrameProcessor.cpp
+++ b/services/camera/libcameraservice/api1/client2/FrameProcessor.cpp
@@ -59,6 +59,8 @@
m3aState.aeState = ANDROID_CONTROL_AE_STATE_INACTIVE;
m3aState.afState = ANDROID_CONTROL_AF_STATE_INACTIVE;
m3aState.awbState = ANDROID_CONTROL_AWB_STATE_INACTIVE;
+
+ mLastFocalLength = l.mParameters.params.getFloat(CameraParameters::KEY_FOCAL_LENGTH);
}
}
@@ -92,9 +94,32 @@
client->notifyRequestId(mCurrentRequestId);
}
+ processLensState(frame.mMetadata, client);
+
return FrameProcessorBase::processSingleFrame(frame, device);
}
+void FrameProcessor::processLensState(const CameraMetadata &frame,
+ const sp<Camera2Client> &client) {
+ ATRACE_CALL();
+ camera_metadata_ro_entry_t entry;
+
+ entry = frame.find(ANDROID_LENS_FOCAL_LENGTH);
+ if (entry.count == 0) {
+ return;
+ }
+
+ if (fabs(entry.data.f[0] - mLastFocalLength) > 0.001f) {
+ SharedParameters::Lock l(client->getParameters());
+ l.mParameters.params.setFloat(
+ CameraParameters::KEY_FOCAL_LENGTH,
+ entry.data.f[0]);
+ l.mParameters.paramsFlattened = l.mParameters.params.flatten();
+
+ mLastFocalLength = entry.data.f[0];
+ }
+}
+
status_t FrameProcessor::processFaceDetect(const CameraMetadata &frame,
const sp<Camera2Client> &client) {
status_t res = BAD_VALUE;
diff --git a/services/camera/libcameraservice/api1/client2/FrameProcessor.h b/services/camera/libcameraservice/api1/client2/FrameProcessor.h
index bb985f6..6c8d221 100644
--- a/services/camera/libcameraservice/api1/client2/FrameProcessor.h
+++ b/services/camera/libcameraservice/api1/client2/FrameProcessor.h
@@ -57,6 +57,9 @@
virtual bool processSingleFrame(CaptureResult &frame,
const sp<FrameProducer> &device);
+ void processLensState(const CameraMetadata &frame,
+ const sp<Camera2Client> &client);
+
status_t processFaceDetect(const CameraMetadata &frame,
const sp<Camera2Client> &client);
@@ -110,6 +113,9 @@
// Emit FaceDetection event to java if faces changed
void callbackFaceDetection(const sp<Camera2Client>& client,
const camera_frame_metadata &metadata);
+
+ // Track most recent focal length sent by the camera device
+ float mLastFocalLength;
};
diff --git a/services/camera/libcameraservice/api1/client2/Parameters.cpp b/services/camera/libcameraservice/api1/client2/Parameters.cpp
index 123cd75..50f1a7a 100644
--- a/services/camera/libcameraservice/api1/client2/Parameters.cpp
+++ b/services/camera/libcameraservice/api1/client2/Parameters.cpp
@@ -984,9 +984,8 @@
Size maxJpegSize = getMaxSize(getAvailableJpegSizes());
int64_t minFrameDurationNs = getJpegStreamMinFrameDurationNs(maxJpegSize);
- slowJpegMode = false;
- if (minFrameDurationNs > kSlowJpegModeThreshold) {
- slowJpegMode = true;
+ slowJpegMode = isSlowJpegModeForced || minFrameDurationNs > kSlowJpegModeThreshold;
+ if (slowJpegMode) {
// Slow jpeg devices does not support video snapshot without
// slowing down preview.
// TODO: support video size video snapshot only?
@@ -2083,7 +2082,7 @@
paramsFlattened = newParams.flatten();
params = newParams;
- slowJpegMode = false;
+ slowJpegMode = isSlowJpegModeForced;
Size pictureSize = { pictureWidth, pictureHeight };
bool zslFrameRateSupported = false;
int64_t jpegMinFrameDurationNs = getJpegStreamMinFrameDurationNs(pictureSize);
diff --git a/services/camera/libcameraservice/api1/client2/Parameters.h b/services/camera/libcameraservice/api1/client2/Parameters.h
index cbe62a7..afad024 100644
--- a/services/camera/libcameraservice/api1/client2/Parameters.h
+++ b/services/camera/libcameraservice/api1/client2/Parameters.h
@@ -182,6 +182,8 @@
bool isDeviceZslSupported;
// Whether the device supports geometric distortion correction
bool isDistortionCorrectionSupported;
+ // Whether slowJpegMode is forced regardless of jpeg stream FPS
+ bool isSlowJpegModeForced;
// Overall camera state
enum State {
diff --git a/services/camera/libcameraservice/api2/CameraDeviceClient.cpp b/services/camera/libcameraservice/api2/CameraDeviceClient.cpp
index 5e91501..1f86414 100644
--- a/services/camera/libcameraservice/api2/CameraDeviceClient.cpp
+++ b/services/camera/libcameraservice/api2/CameraDeviceClient.cpp
@@ -66,7 +66,8 @@
int sensorOrientation,
int clientPid,
uid_t clientUid,
- int servicePid) :
+ int servicePid,
+ bool overrideToPortrait) :
BasicClient(cameraService,
IInterface::asBinder(remoteCallback),
clientPackageName,
@@ -77,7 +78,8 @@
sensorOrientation,
clientPid,
clientUid,
- servicePid),
+ servicePid,
+ overrideToPortrait),
mRemoteCallback(remoteCallback) {
// We don't need it for API2 clients, but Camera2ClientBase requires it.
(void) api1CameraId;
@@ -96,10 +98,11 @@
int clientPid,
uid_t clientUid,
int servicePid,
- bool overrideForPerfClass) :
+ bool overrideForPerfClass,
+ bool overrideToPortrait) :
Camera2ClientBase(cameraService, remoteCallback, clientPackageName, systemNativeClient,
clientFeatureId, cameraId, /*API1 camera ID*/ -1, cameraFacing, sensorOrientation,
- clientPid, clientUid, servicePid, overrideForPerfClass),
+ clientPid, clientUid, servicePid, overrideForPerfClass, overrideToPortrait),
mInputStream(),
mStreamingRequestId(REQUEST_ID_NONE),
mRequestIdCounter(0),
@@ -1730,6 +1733,10 @@
return binder::Status::ok();
}
+status_t CameraDeviceClient::setCameraServiceWatchdog(bool enabled) {
+ return mDevice->setCameraServiceWatchdog(enabled);
+}
+
status_t CameraDeviceClient::setRotateAndCropOverride(uint8_t rotateAndCrop) {
if (rotateAndCrop > ANDROID_SCALER_ROTATE_AND_CROP_AUTO) return BAD_VALUE;
@@ -1745,6 +1752,15 @@
return mDevice->setCameraMute(enabled);
}
+void CameraDeviceClient::setStreamUseCaseOverrides(
+ const std::vector<int64_t>& useCaseOverrides) {
+ mDevice->setStreamUseCaseOverrides(useCaseOverrides);
+}
+
+void CameraDeviceClient::clearStreamUseCaseOverrides() {
+ mDevice->clearStreamUseCaseOverrides();
+}
+
binder::Status CameraDeviceClient::switchToOffline(
const sp<hardware::camera2::ICameraDeviceCallbacks>& cameraCb,
const std::vector<int>& offlineOutputIds,
diff --git a/services/camera/libcameraservice/api2/CameraDeviceClient.h b/services/camera/libcameraservice/api2/CameraDeviceClient.h
index c5aad6b..c95bb4a 100644
--- a/services/camera/libcameraservice/api2/CameraDeviceClient.h
+++ b/services/camera/libcameraservice/api2/CameraDeviceClient.h
@@ -58,7 +58,8 @@
int sensorOrientation,
int clientPid,
uid_t clientUid,
- int servicePid);
+ int servicePid,
+ bool overrideToPortrait);
sp<hardware::camera2::ICameraDeviceCallbacks> mRemoteCallback;
};
@@ -187,7 +188,8 @@
int clientPid,
uid_t clientUid,
int servicePid,
- bool overrideForPerfClass);
+ bool overrideForPerfClass,
+ bool overrideToPortrait);
virtual ~CameraDeviceClient();
virtual status_t initialize(sp<CameraProviderManager> manager,
@@ -206,6 +208,11 @@
virtual status_t stopWatchingTags(int out);
virtual status_t dumpWatchedEventsToVector(std::vector<std::string> &out);
+ virtual status_t setCameraServiceWatchdog(bool enabled);
+
+ virtual void setStreamUseCaseOverrides(const std::vector<int64_t>& useCaseOverrides);
+ virtual void clearStreamUseCaseOverrides() override;
+
/**
* Device listener interface
*/
diff --git a/services/camera/libcameraservice/api2/CameraOfflineSessionClient.cpp b/services/camera/libcameraservice/api2/CameraOfflineSessionClient.cpp
index 9303fd2..2810667 100644
--- a/services/camera/libcameraservice/api2/CameraOfflineSessionClient.cpp
+++ b/services/camera/libcameraservice/api2/CameraOfflineSessionClient.cpp
@@ -66,6 +66,10 @@
return OK;
}
+status_t CameraOfflineSessionClient::setCameraServiceWatchdog(bool) {
+ return OK;
+}
+
status_t CameraOfflineSessionClient::setRotateAndCropOverride(uint8_t /*rotateAndCrop*/) {
// Since we're not submitting more capture requests, changes to rotateAndCrop override
// make no difference.
@@ -81,6 +85,12 @@
return INVALID_OPERATION;
}
+void CameraOfflineSessionClient::setStreamUseCaseOverrides(
+ const std::vector<int64_t>& /*useCaseOverrides*/) {
+}
+
+void CameraOfflineSessionClient::clearStreamUseCaseOverrides() {
+}
status_t CameraOfflineSessionClient::dump(int fd, const Vector<String16>& args) {
return BasicClient::dump(fd, args);
diff --git a/services/camera/libcameraservice/api2/CameraOfflineSessionClient.h b/services/camera/libcameraservice/api2/CameraOfflineSessionClient.h
index f2c42d8..23e1f3d 100644
--- a/services/camera/libcameraservice/api2/CameraOfflineSessionClient.h
+++ b/services/camera/libcameraservice/api2/CameraOfflineSessionClient.h
@@ -56,7 +56,8 @@
IInterface::asBinder(remoteCallback),
// (v)ndk doesn't have offline session support
clientPackageName, /*overridePackageName*/false, clientFeatureId,
- cameraIdStr, cameraFacing, sensorOrientation, clientPid, clientUid, servicePid),
+ cameraIdStr, cameraFacing, sensorOrientation, clientPid, clientUid, servicePid,
+ /*overrideToPortrait*/false),
mRemoteCallback(remoteCallback), mOfflineSession(session),
mCompositeStreamMap(offlineCompositeStreamMap) {}
@@ -84,6 +85,13 @@
bool supportsCameraMute() override;
status_t setCameraMute(bool enabled) override;
+ status_t setCameraServiceWatchdog(bool enabled) override;
+
+ void setStreamUseCaseOverrides(
+ const std::vector<int64_t>& useCaseOverrides) override;
+
+ void clearStreamUseCaseOverrides() override;
+
// permissions management
status_t startCameraOps() override;
status_t finishCameraOps() override;
diff --git a/services/camera/libcameraservice/api2/HeicCompositeStream.cpp b/services/camera/libcameraservice/api2/HeicCompositeStream.cpp
index 54cc27a..237ce5e 100644
--- a/services/camera/libcameraservice/api2/HeicCompositeStream.cpp
+++ b/services/camera/libcameraservice/api2/HeicCompositeStream.cpp
@@ -1161,11 +1161,13 @@
inputFrame.fileFd = -1;
// Fill in HEIC header
- uint8_t *header = static_cast<uint8_t*>(dstBuffer) + mMaxHeicBufferSize - sizeof(CameraBlob);
- CameraBlob *blobHeader = (CameraBlob *)header;
// Must be in sync with CAMERA3_HEIC_BLOB_ID in android_media_Utils.cpp
- blobHeader->blobId = static_cast<CameraBlobId>(0x00FE);
- blobHeader->blobSizeBytes = fSize;
+ uint8_t *header = static_cast<uint8_t*>(dstBuffer) + mMaxHeicBufferSize - sizeof(CameraBlob);
+ CameraBlob blobHeader = {
+ .blobId = static_cast<CameraBlobId>(0x00FE),
+ .blobSizeBytes = static_cast<int32_t>(fSize)
+ };
+ memcpy(header, &blobHeader, sizeof(CameraBlob));
res = native_window_set_buffers_timestamp(mOutputSurface.get(), inputFrame.timestamp);
if (res != OK) {
diff --git a/services/camera/libcameraservice/common/Camera2ClientBase.cpp b/services/camera/libcameraservice/common/Camera2ClientBase.cpp
index a3d24ff..bf6be64 100644
--- a/services/camera/libcameraservice/common/Camera2ClientBase.cpp
+++ b/services/camera/libcameraservice/common/Camera2ClientBase.cpp
@@ -41,8 +41,6 @@
namespace android {
-const static size_t kDisconnectTimeoutMs = 2500;
-
using namespace camera2;
// Interface used by CameraService
@@ -62,10 +60,11 @@
uid_t clientUid,
int servicePid,
bool overrideForPerfClass,
+ bool overrideToPortrait,
bool legacyClient):
TClientBase(cameraService, remoteCallback, clientPackageName, systemNativeClient,
clientFeatureId, cameraId, api1CameraId, cameraFacing, sensorOrientation, clientPid,
- clientUid, servicePid),
+ clientUid, servicePid, overrideToPortrait),
mSharedCameraCallbacks(remoteCallback),
mDeviceActive(false), mApi1CameraId(api1CameraId)
{
@@ -119,12 +118,12 @@
case IPCTransport::HIDL:
mDevice =
new HidlCamera3Device(TClientBase::mCameraIdStr, mOverrideForPerfClass,
- mLegacyClient);
+ TClientBase::mOverrideToPortrait, mLegacyClient);
break;
case IPCTransport::AIDL:
mDevice =
new AidlCamera3Device(TClientBase::mCameraIdStr, mOverrideForPerfClass,
- mLegacyClient);
+ TClientBase::mOverrideToPortrait, mLegacyClient);
break;
default:
ALOGE("%s Invalid transport for camera id %s", __FUNCTION__,
@@ -252,10 +251,16 @@
template <typename TClientBase>
binder::Status Camera2ClientBase<TClientBase>::disconnect() {
- if (mCameraServiceWatchdog != nullptr) {
+ if (mCameraServiceWatchdog != nullptr && mDevice != nullptr) {
+ // Timer for the disconnect call should be greater than getExpectedInFlightDuration
+ // since this duration is used to error handle methods in the disconnect sequence
+ // thus allowing existing error handling methods to execute first
+ uint64_t maxExpectedDuration =
+ ns2ms(mDevice->getExpectedInFlightDuration() + kBufferTimeDisconnectNs);
+
// Initialization from hal succeeded, time disconnect.
return mCameraServiceWatchdog->WATCH_CUSTOM_TIMER(disconnectImpl(),
- kDisconnectTimeoutMs / kCycleLengthMs, kCycleLengthMs);
+ maxExpectedDuration / kCycleLengthMs, kCycleLengthMs);
}
return disconnectImpl();
}
@@ -338,6 +343,29 @@
}
template <typename TClientBase>
+void Camera2ClientBase<TClientBase>::notifyPhysicalCameraChange(const std::string &physicalId) {
+ // We're only interested in this notification if overrideToPortrait is turned on.
+ if (!TClientBase::mOverrideToPortrait) {
+ return;
+ }
+
+ String8 physicalId8(physicalId.c_str());
+ auto physicalCameraMetadata = mDevice->infoPhysical(physicalId8);
+ auto orientationEntry = physicalCameraMetadata.find(ANDROID_SENSOR_ORIENTATION);
+
+ if (orientationEntry.count == 1) {
+ int orientation = orientationEntry.data.i32[0];
+ int rotateAndCropMode = ANDROID_SCALER_ROTATE_AND_CROP_NONE;
+
+ if (orientation == 0 || orientation == 180) {
+ rotateAndCropMode = ANDROID_SCALER_ROTATE_AND_CROP_90;
+ }
+
+ static_cast<TClientBase *>(this)->setRotateAndCropOverride(rotateAndCropMode);
+ }
+}
+
+template <typename TClientBase>
status_t Camera2ClientBase<TClientBase>::notifyActive(float maxPreviewFps) {
if (!mDeviceActive) {
status_t res = TClientBase::startCameraStreamingOps();
diff --git a/services/camera/libcameraservice/common/Camera2ClientBase.h b/services/camera/libcameraservice/common/Camera2ClientBase.h
index 3af781b..705fe69 100644
--- a/services/camera/libcameraservice/common/Camera2ClientBase.h
+++ b/services/camera/libcameraservice/common/Camera2ClientBase.h
@@ -59,6 +59,7 @@
uid_t clientUid,
int servicePid,
bool overrideForPerfClass,
+ bool overrideToPortrait,
bool legacyClient = false);
virtual ~Camera2ClientBase();
@@ -74,6 +75,7 @@
virtual void notifyError(int32_t errorCode,
const CaptureResultExtras& resultExtras);
+ virtual void notifyPhysicalCameraChange(const std::string &physicalId) override;
// Returns errors on app ops permission failures
virtual status_t notifyActive(float maxPreviewFps);
virtual void notifyIdle(int64_t /*requestCount*/, int64_t /*resultErrorCount*/,
@@ -132,6 +134,9 @@
protected:
+ // Used for watchdog timeout to monitor disconnect
+ static const nsecs_t kBufferTimeDisconnectNs = 3000000000; // 3 sec.
+
// The PID provided in the constructor call
pid_t mInitialClientPid;
bool mOverrideForPerfClass = false;
diff --git a/services/camera/libcameraservice/common/CameraDeviceBase.h b/services/camera/libcameraservice/common/CameraDeviceBase.h
index 7e2f93c..8f7b16d 100644
--- a/services/camera/libcameraservice/common/CameraDeviceBase.h
+++ b/services/camera/libcameraservice/common/CameraDeviceBase.h
@@ -449,6 +449,11 @@
virtual status_t setCameraMute(bool enabled) = 0;
/**
+ * Enable/disable camera service watchdog
+ */
+ virtual status_t setCameraServiceWatchdog(bool enabled) = 0;
+
+ /**
* Get the status tracker of the camera device
*/
virtual wp<camera3::StatusTracker> getStatusTracker() = 0;
@@ -459,6 +464,15 @@
void setImageDumpMask(int mask) { mImageDumpMask = mask; }
/**
+ * Set stream use case overrides
+ */
+ void setStreamUseCaseOverrides(const std::vector<int64_t>& useCaseOverrides) {
+ mStreamUseCaseOverrides = useCaseOverrides;
+ }
+
+ void clearStreamUseCaseOverrides() {}
+
+ /**
* The injection camera session to replace the internal camera
* session.
*/
@@ -472,6 +486,7 @@
protected:
bool mImageDumpMask = 0;
+ std::vector<int64_t> mStreamUseCaseOverrides;
};
}; // namespace android
diff --git a/services/camera/libcameraservice/common/CameraOfflineSessionBase.h b/services/camera/libcameraservice/common/CameraOfflineSessionBase.h
index f39b92a..63abcf0 100644
--- a/services/camera/libcameraservice/common/CameraOfflineSessionBase.h
+++ b/services/camera/libcameraservice/common/CameraOfflineSessionBase.h
@@ -40,6 +40,10 @@
// Required for API 1 and 2
virtual void notifyError(int32_t errorCode,
const CaptureResultExtras &resultExtras) = 0;
+
+ // Optional for API 1 and 2
+ virtual void notifyPhysicalCameraChange(const std::string &/*physicalId*/) {}
+
// May return an error since it checks appops
virtual status_t notifyActive(float maxPreviewFps) = 0;
virtual void notifyIdle(int64_t requestCount, int64_t resultError, bool deviceError,
diff --git a/services/camera/libcameraservice/common/CameraProviderManager.cpp b/services/camera/libcameraservice/common/CameraProviderManager.cpp
index 6ef16b3..43f92a9 100644
--- a/services/camera/libcameraservice/common/CameraProviderManager.cpp
+++ b/services/camera/libcameraservice/common/CameraProviderManager.cpp
@@ -197,12 +197,17 @@
return std::make_pair(systemCameraCount, publicCameraCount);
}
-std::vector<std::string> CameraProviderManager::getCameraDeviceIds() const {
+std::vector<std::string> CameraProviderManager::getCameraDeviceIds(std::unordered_map<
+ std::string, std::set<std::string>>* unavailablePhysicalIds) const {
std::lock_guard<std::mutex> lock(mInterfaceMutex);
std::vector<std::string> deviceIds;
for (auto& provider : mProviders) {
for (auto& id : provider->mUniqueCameraIds) {
deviceIds.push_back(id);
+ if (unavailablePhysicalIds != nullptr &&
+ provider->mUnavailablePhysicalCameras.count(id) > 0) {
+ (*unavailablePhysicalIds)[id] = provider->mUnavailablePhysicalCameras.at(id);
+ }
}
}
return deviceIds;
@@ -318,13 +323,13 @@
}
status_t CameraProviderManager::getCameraInfo(const std::string &id,
- hardware::CameraInfo* info) const {
+ bool overrideToPortrait, int *portraitRotation, hardware::CameraInfo* info) const {
std::lock_guard<std::mutex> lock(mInterfaceMutex);
auto deviceInfo = findDeviceInfoLocked(id);
if (deviceInfo == nullptr) return NAME_NOT_FOUND;
- return deviceInfo->getCameraInfo(info);
+ return deviceInfo->getCameraInfo(overrideToPortrait, portraitRotation, info);
}
status_t CameraProviderManager::isSessionConfigurationSupported(const std::string& id,
@@ -356,9 +361,11 @@
}
status_t CameraProviderManager::getCameraCharacteristics(const std::string &id,
- bool overrideForPerfClass, CameraMetadata* characteristics) const {
+ bool overrideForPerfClass, CameraMetadata* characteristics,
+ bool overrideToPortrait) const {
std::lock_guard<std::mutex> lock(mInterfaceMutex);
- return getCameraCharacteristicsLocked(id, overrideForPerfClass, characteristics);
+ return getCameraCharacteristicsLocked(id, overrideForPerfClass, characteristics,
+ overrideToPortrait);
}
status_t CameraProviderManager::getHighestSupportedVersion(const std::string &id,
@@ -839,9 +846,6 @@
void CameraProviderManager::ProviderInfo::initializeProviderInfoCommon(
const std::vector<std::string> &devices) {
-
- sp<StatusListener> listener = mManager->getStatusListener();
-
for (auto& device : devices) {
std::string id;
status_t res = addDevice(device, CameraDeviceStatus::PRESENT, &id);
@@ -856,38 +860,22 @@
mProviderName.c_str(), mDevices.size());
// Process cached status callbacks
- std::unique_ptr<std::vector<CameraStatusInfoT>> cachedStatus =
- std::make_unique<std::vector<CameraStatusInfoT>>();
{
std::lock_guard<std::mutex> lock(mInitLock);
for (auto& statusInfo : mCachedStatus) {
std::string id, physicalId;
- status_t res = OK;
if (statusInfo.isPhysicalCameraStatus) {
- res = physicalCameraDeviceStatusChangeLocked(&id, &physicalId,
+ physicalCameraDeviceStatusChangeLocked(&id, &physicalId,
statusInfo.cameraId, statusInfo.physicalCameraId, statusInfo.status);
} else {
- res = cameraDeviceStatusChangeLocked(&id, statusInfo.cameraId, statusInfo.status);
- }
- if (res == OK) {
- cachedStatus->emplace_back(statusInfo.isPhysicalCameraStatus,
- id.c_str(), physicalId.c_str(), statusInfo.status);
+ cameraDeviceStatusChangeLocked(&id, statusInfo.cameraId, statusInfo.status);
}
}
mCachedStatus.clear();
mInitialized = true;
}
-
- // The cached status change callbacks cannot be fired directly from this
- // function, due to same-thread deadlock trying to acquire mInterfaceMutex
- // twice.
- if (listener != nullptr) {
- mInitialStatusCallbackFuture = std::async(std::launch::async,
- &CameraProviderManager::ProviderInfo::notifyInitialStatusChange, this,
- listener, std::move(cachedStatus));
- }
}
CameraProviderManager::ProviderInfo::DeviceInfo* CameraProviderManager::findDeviceInfoLocked(
@@ -1957,6 +1945,7 @@
for (auto it = mDevices.begin(); it != mDevices.end(); it++) {
if ((*it)->mId == id) {
mUniqueCameraIds.erase(id);
+ mUnavailablePhysicalCameras.erase(id);
if ((*it)->isAPI1Compatible()) {
mUniqueAPI1CompatibleCameraIds.erase(std::remove(
mUniqueAPI1CompatibleCameraIds.begin(),
@@ -2027,7 +2016,9 @@
dprintf(fd, " Has a flash unit: %s\n",
device->hasFlashUnit() ? "true" : "false");
hardware::CameraInfo info;
- status_t res = device->getCameraInfo(&info);
+ int portraitRotation;
+ status_t res = device->getCameraInfo(/*overrideToPortrait*/false, &portraitRotation,
+ &info);
if (res != OK) {
dprintf(fd, " <Error reading camera info: %s (%d)>\n",
strerror(-res), res);
@@ -2037,7 +2028,8 @@
dprintf(fd, " Orientation: %d\n", info.orientation);
}
CameraMetadata info2;
- res = device->getCameraCharacteristics(true /*overrideForPerfClass*/, &info2);
+ res = device->getCameraCharacteristics(true /*overrideForPerfClass*/, &info2,
+ /*overrideToPortrait*/false);
if (res == INVALID_OPERATION) {
dprintf(fd, " API2 not directly supported\n");
} else if (res != OK) {
@@ -2224,6 +2216,15 @@
return BAD_VALUE;
}
+ if (mUnavailablePhysicalCameras.count(cameraId) == 0) {
+ mUnavailablePhysicalCameras.emplace(cameraId, std::set<std::string>{});
+ }
+ if (newStatus != CameraDeviceStatus::PRESENT) {
+ mUnavailablePhysicalCameras[cameraId].insert(physicalCameraDeviceName);
+ } else {
+ mUnavailablePhysicalCameras[cameraId].erase(physicalCameraDeviceName);
+ }
+
*id = cameraId;
*physicalId = physicalCameraDeviceName.c_str();
return OK;
@@ -2282,20 +2283,6 @@
}
}
-void CameraProviderManager::ProviderInfo::notifyInitialStatusChange(
- sp<StatusListener> listener,
- std::unique_ptr<std::vector<CameraStatusInfoT>> cachedStatus) {
- for (auto& statusInfo : *cachedStatus) {
- if (statusInfo.isPhysicalCameraStatus) {
- listener->onDeviceStatusChanged(String8(statusInfo.cameraId.c_str()),
- String8(statusInfo.physicalCameraId.c_str()), statusInfo.status);
- } else {
- listener->onDeviceStatusChanged(
- String8(statusInfo.cameraId.c_str()), statusInfo.status);
- }
- }
-}
-
CameraProviderManager::ProviderInfo::DeviceInfo3::DeviceInfo3(const std::string& name,
const metadata_vendor_id_t tagId, const std::string &id,
uint16_t minorVersion,
@@ -2314,6 +2301,7 @@
}
status_t CameraProviderManager::ProviderInfo::DeviceInfo3::getCameraInfo(
+ bool overrideToPortrait, int *portraitRotation,
hardware::CameraInfo *info) const {
if (info == nullptr) return BAD_VALUE;
@@ -2344,6 +2332,17 @@
return NAME_NOT_FOUND;
}
+ if (overrideToPortrait && (info->orientation == 0 || info->orientation == 180)) {
+ *portraitRotation = 90;
+ if (info->facing == hardware::CAMERA_FACING_FRONT) {
+ info->orientation = (360 + info->orientation - 90) % 360;
+ } else {
+ info->orientation = (360 + info->orientation + 90) % 360;
+ }
+ } else {
+ *portraitRotation = 0;
+ }
+
return OK;
}
bool CameraProviderManager::ProviderInfo::DeviceInfo3::isAPI1Compatible() const {
@@ -2369,7 +2368,7 @@
}
status_t CameraProviderManager::ProviderInfo::DeviceInfo3::getCameraCharacteristics(
- bool overrideForPerfClass, CameraMetadata *characteristics) const {
+ bool overrideForPerfClass, CameraMetadata *characteristics, bool overrideToPortrait) {
if (characteristics == nullptr) return BAD_VALUE;
if (!overrideForPerfClass && mCameraCharNoPCOverride != nullptr) {
@@ -2378,6 +2377,37 @@
*characteristics = mCameraCharacteristics;
}
+ if (overrideToPortrait) {
+ const auto &lensFacingEntry = characteristics->find(ANDROID_LENS_FACING);
+ const auto &sensorOrientationEntry = characteristics->find(ANDROID_SENSOR_ORIENTATION);
+ uint8_t lensFacing = lensFacingEntry.data.u8[0];
+ if (lensFacingEntry.count > 0 && sensorOrientationEntry.count > 0) {
+ int32_t sensorOrientation = sensorOrientationEntry.data.i32[0];
+ int32_t newSensorOrientation = sensorOrientation;
+
+ if (sensorOrientation == 0 || sensorOrientation == 180) {
+ if (lensFacing == ANDROID_LENS_FACING_FRONT) {
+ newSensorOrientation = (360 + sensorOrientation - 90) % 360;
+ } else if (lensFacing == ANDROID_LENS_FACING_BACK) {
+ newSensorOrientation = (360 + sensorOrientation + 90) % 360;
+ }
+ }
+
+ if (newSensorOrientation != sensorOrientation) {
+ ALOGV("%s: Update ANDROID_SENSOR_ORIENTATION for lens facing %d "
+ "from %d to %d", __FUNCTION__, lensFacing, sensorOrientation,
+ newSensorOrientation);
+ characteristics->update(ANDROID_SENSOR_ORIENTATION, &newSensorOrientation, 1);
+ }
+ }
+
+ if (characteristics->exists(ANDROID_INFO_DEVICE_STATE_ORIENTATIONS)) {
+ ALOGV("%s: Erasing ANDROID_INFO_DEVICE_STATE_ORIENTATIONS for lens facing %d",
+ __FUNCTION__, lensFacing);
+ characteristics->erase(ANDROID_INFO_DEVICE_STATE_ORIENTATIONS);
+ }
+ }
+
return OK;
}
@@ -2409,8 +2439,8 @@
for (size_t i = 0; i < streamConfigs.count; i += 4) {
if ((streamConfigs.data.i32[i] == HAL_PIXEL_FORMAT_BLOB) && (streamConfigs.data.i32[i+3] ==
ANDROID_SCALER_AVAILABLE_STREAM_CONFIGURATIONS_OUTPUT)) {
- if (streamConfigs.data.i32[i+1] < thresholdW ||
- streamConfigs.data.i32[i+2] < thresholdH) {
+ if (streamConfigs.data.i32[i+1] * streamConfigs.data.i32[i+2] <
+ thresholdW * thresholdH) {
continue;
} else {
largeJpegCount ++;
@@ -2430,8 +2460,8 @@
mCameraCharacteristics.find(ANDROID_SCALER_AVAILABLE_MIN_FRAME_DURATIONS);
for (size_t i = 0; i < minDurations.count; i += 4) {
if (minDurations.data.i64[i] == HAL_PIXEL_FORMAT_BLOB) {
- if (minDurations.data.i64[i+1] < thresholdW ||
- minDurations.data.i64[i+2] < thresholdH) {
+ if ((int32_t)minDurations.data.i64[i+1] * (int32_t)minDurations.data.i64[i+2] <
+ thresholdW * thresholdH) {
continue;
} else {
largeJpegCount++;
@@ -2451,8 +2481,8 @@
mCameraCharacteristics.find(ANDROID_SCALER_AVAILABLE_STALL_DURATIONS);
for (size_t i = 0; i < stallDurations.count; i += 4) {
if (stallDurations.data.i64[i] == HAL_PIXEL_FORMAT_BLOB) {
- if (stallDurations.data.i64[i+1] < thresholdW ||
- stallDurations.data.i64[i+2] < thresholdH) {
+ if ((int32_t)stallDurations.data.i64[i+1] * (int32_t)stallDurations.data.i64[i+2] <
+ thresholdW * thresholdH) {
continue;
} else {
largeJpegCount++;
@@ -2645,9 +2675,6 @@
}
CameraProviderManager::ProviderInfo::~ProviderInfo() {
- if (mInitialStatusCallbackFuture.valid()) {
- mInitialStatusCallbackFuture.wait();
- }
// Destruction of ProviderInfo is only supposed to happen when the respective
// CameraProvider interface dies, so do not unregister callbacks.
}
@@ -2710,10 +2737,12 @@
}
status_t CameraProviderManager::getCameraCharacteristicsLocked(const std::string &id,
- bool overrideForPerfClass, CameraMetadata* characteristics) const {
+ bool overrideForPerfClass, CameraMetadata* characteristics,
+ bool overrideToPortrait) const {
auto deviceInfo = findDeviceInfoLocked(id);
if (deviceInfo != nullptr) {
- return deviceInfo->getCameraCharacteristics(overrideForPerfClass, characteristics);
+ return deviceInfo->getCameraCharacteristics(overrideForPerfClass, characteristics,
+ overrideToPortrait);
}
// Find hidden physical camera characteristics
@@ -2748,7 +2777,9 @@
combo.push_back(deviceId);
hardware::CameraInfo info;
- status_t res = deviceInfo->getCameraInfo(&info);
+ int portraitRotation;
+ status_t res = deviceInfo->getCameraInfo(/*overrideToPortrait*/false, &portraitRotation,
+ &info);
if (res != OK) {
ALOGE("%s: Error reading camera info: %s (%d)", __FUNCTION__, strerror(-res), res);
continue;
diff --git a/services/camera/libcameraservice/common/CameraProviderManager.h b/services/camera/libcameraservice/common/CameraProviderManager.h
index d049aff..e8d9a37 100644
--- a/services/camera/libcameraservice/common/CameraProviderManager.h
+++ b/services/camera/libcameraservice/common/CameraProviderManager.h
@@ -23,7 +23,6 @@
#include <set>
#include <string>
#include <mutex>
-#include <future>
#include <camera/camera2/ConcurrentCamera.h>
#include <camera/CameraParameters2.h>
@@ -220,7 +219,14 @@
*/
std::pair<int, int> getCameraCount() const;
- std::vector<std::string> getCameraDeviceIds() const;
+ /**
+ * Upon the function return, if unavailablePhysicalIds is not nullptr, it
+ * will contain all of the unavailable physical camera Ids represented in
+ * the form of:
+ * {[logicalCamera, {physicalCamera1, physicalCamera2, ...}], ...}.
+ */
+ std::vector<std::string> getCameraDeviceIds(std::unordered_map<
+ std::string, std::set<std::string>>* unavailablePhysicalIds = nullptr) const;
/**
* Retrieve the number of API1 compatible cameras; these are internal and
@@ -251,14 +257,15 @@
* Return the old camera API camera info
*/
status_t getCameraInfo(const std::string &id,
- hardware::CameraInfo* info) const;
+ bool overrideToPortrait, int *portraitRotation, hardware::CameraInfo* info) const;
/**
* Return API2 camera characteristics - returns NAME_NOT_FOUND if a device ID does
* not have a v3 or newer HAL version.
*/
status_t getCameraCharacteristics(const std::string &id,
- bool overrideForPerfClass, CameraMetadata* characteristics) const;
+ bool overrideForPerfClass, CameraMetadata* characteristics,
+ bool overrideToPortrait) const;
status_t isConcurrentSessionConfigurationSupported(
const std::vector<hardware::camera2::utils::CameraIdAndSessionConfiguration>
@@ -560,13 +567,16 @@
virtual status_t setTorchMode(bool enabled) = 0;
virtual status_t turnOnTorchWithStrengthLevel(int32_t torchStrength) = 0;
virtual status_t getTorchStrengthLevel(int32_t *torchStrength) = 0;
- virtual status_t getCameraInfo(hardware::CameraInfo *info) const = 0;
+ virtual status_t getCameraInfo(bool overrideToPortrait,
+ int *portraitRotation,
+ hardware::CameraInfo *info) const = 0;
virtual bool isAPI1Compatible() const = 0;
virtual status_t dumpState(int fd) = 0;
virtual status_t getCameraCharacteristics(bool overrideForPerfClass,
- CameraMetadata *characteristics) const {
+ CameraMetadata *characteristics, bool overrideToPortrait) {
(void) overrideForPerfClass;
(void) characteristics;
+ (void) overrideToPortrait;
return INVALID_OPERATION;
}
virtual status_t getPhysicalCameraCharacteristics(const std::string& physicalCameraId,
@@ -607,6 +617,7 @@
};
std::vector<std::unique_ptr<DeviceInfo>> mDevices;
std::unordered_set<std::string> mUniqueCameraIds;
+ std::unordered_map<std::string, std::set<std::string>> mUnavailablePhysicalCameras;
int mUniqueDeviceCount;
std::vector<std::string> mUniqueAPI1CompatibleCameraIds;
// The initial public camera IDs published by the camera provider.
@@ -622,12 +633,15 @@
virtual status_t setTorchMode(bool enabled) = 0;
virtual status_t turnOnTorchWithStrengthLevel(int32_t torchStrength) = 0;
virtual status_t getTorchStrengthLevel(int32_t *torchStrength) = 0;
- virtual status_t getCameraInfo(hardware::CameraInfo *info) const override;
+ virtual status_t getCameraInfo(bool overrideToPortrait,
+ int *portraitRotation,
+ hardware::CameraInfo *info) const override;
virtual bool isAPI1Compatible() const override;
virtual status_t dumpState(int fd) = 0;
virtual status_t getCameraCharacteristics(
bool overrideForPerfClass,
- CameraMetadata *characteristics) const override;
+ CameraMetadata *characteristics,
+ bool overrideToPortrait) override;
virtual status_t getPhysicalCameraCharacteristics(const std::string& physicalCameraId,
CameraMetadata *characteristics) const override;
virtual status_t isSessionConfigurationSupported(
@@ -715,8 +729,6 @@
std::vector<CameraStatusInfoT> mCachedStatus;
// End of scope for mInitLock
- std::future<void> mInitialStatusCallbackFuture;
-
std::unique_ptr<ProviderInfo::DeviceInfo>
virtual initializeDeviceInfo(
const std::string &name, const metadata_vendor_id_t tagId,
@@ -724,9 +736,6 @@
virtual status_t reCacheConcurrentStreamingCameraIdsLocked() = 0;
- void notifyInitialStatusChange(sp<StatusListener> listener,
- std::unique_ptr<std::vector<CameraStatusInfoT>> cachedStatus);
-
std::vector<std::unordered_set<std::string>> mConcurrentCameraIdCombinations;
// Parse provider instance name for type and id
@@ -830,7 +839,7 @@
const hardware::camera::common::V1_0::TorchModeStatus&);
status_t getCameraCharacteristicsLocked(const std::string &id, bool overrideForPerfClass,
- CameraMetadata* characteristics) const;
+ CameraMetadata* characteristics, bool overrideToPortrait) const;
void filterLogicalCameraIdsLocked(std::vector<std::string>& deviceIds) const;
status_t getSystemCameraKindLocked(const std::string& id, SystemCameraKind *kind) const;
diff --git a/services/camera/libcameraservice/common/aidl/AidlProviderInfo.cpp b/services/camera/libcameraservice/common/aidl/AidlProviderInfo.cpp
index ef68f28..67c4841 100644
--- a/services/camera/libcameraservice/common/aidl/AidlProviderInfo.cpp
+++ b/services/camera/libcameraservice/common/aidl/AidlProviderInfo.cpp
@@ -759,7 +759,8 @@
bool overrideForPerfClass =
SessionConfigurationUtils::targetPerfClassPrimaryCamera(
perfClassPrimaryCameraIds, cameraId, targetSdkVersion);
- res = mManager->getCameraCharacteristicsLocked(cameraId, overrideForPerfClass, &deviceInfo);
+ res = mManager->getCameraCharacteristicsLocked(cameraId, overrideForPerfClass, &deviceInfo,
+ /*overrideToPortrait*/false);
if (res != OK) {
return res;
}
@@ -767,7 +768,8 @@
[this](const String8 &id, bool overrideForPerfClass) {
CameraMetadata physicalDeviceInfo;
mManager->getCameraCharacteristicsLocked(id.string(), overrideForPerfClass,
- &physicalDeviceInfo);
+ &physicalDeviceInfo,
+ /*overrideToPortrait*/false);
return physicalDeviceInfo;
};
std::vector<std::string> physicalCameraIds;
diff --git a/services/camera/libcameraservice/common/hidl/HidlProviderInfo.cpp b/services/camera/libcameraservice/common/hidl/HidlProviderInfo.cpp
index d60565f..630090b 100644
--- a/services/camera/libcameraservice/common/hidl/HidlProviderInfo.cpp
+++ b/services/camera/libcameraservice/common/hidl/HidlProviderInfo.cpp
@@ -919,7 +919,8 @@
bool overrideForPerfClass =
SessionConfigurationUtils::targetPerfClassPrimaryCamera(
perfClassPrimaryCameraIds, cameraId, targetSdkVersion);
- res = mManager->getCameraCharacteristicsLocked(cameraId, overrideForPerfClass, &deviceInfo);
+ res = mManager->getCameraCharacteristicsLocked(cameraId, overrideForPerfClass, &deviceInfo,
+ /*overrideToPortrait*/false);
if (res != OK) {
return res;
}
@@ -927,7 +928,7 @@
[this](const String8 &id, bool overrideForPerfClass) {
CameraMetadata physicalDeviceInfo;
mManager->getCameraCharacteristicsLocked(id.string(), overrideForPerfClass,
- &physicalDeviceInfo);
+ &physicalDeviceInfo, /*overrideToPortrait*/false);
return physicalDeviceInfo;
};
std::vector<std::string> physicalCameraIds;
diff --git a/services/camera/libcameraservice/device3/Camera3Device.cpp b/services/camera/libcameraservice/device3/Camera3Device.cpp
index 7c2f34f..e55d724 100644
--- a/services/camera/libcameraservice/device3/Camera3Device.cpp
+++ b/services/camera/libcameraservice/device3/Camera3Device.cpp
@@ -73,7 +73,8 @@
namespace android {
-Camera3Device::Camera3Device(const String8 &id, bool overrideForPerfClass, bool legacyClient):
+Camera3Device::Camera3Device(const String8 &id, bool overrideForPerfClass, bool overrideToPortrait,
+ bool legacyClient):
mId(id),
mLegacyClient(legacyClient),
mOperatingMode(NO_MODE),
@@ -94,7 +95,11 @@
mVendorTagId(CAMERA_METADATA_INVALID_VENDOR_ID),
mLastTemplateId(-1),
mNeedFixupMonochromeTags(false),
- mOverrideForPerfClass(overrideForPerfClass)
+ mOverrideForPerfClass(overrideForPerfClass),
+ mOverrideToPortrait(overrideToPortrait),
+ mRotateAndCropOverride(ANDROID_SCALER_ROTATE_AND_CROP_NONE),
+ mComposerOutput(false),
+ mActivePhysicalId("")
{
ATRACE_CALL();
ALOGV("%s: Created device for camera %s", __FUNCTION__, mId.string());
@@ -113,10 +118,6 @@
status_t Camera3Device::initializeCommonLocked() {
- /** Start watchdog thread */
- mCameraServiceWatchdog = new CameraServiceWatchdog();
- mCameraServiceWatchdog->run("CameraServiceWatchdog");
-
/** Start up status tracker thread */
mStatusTracker = new StatusTracker(this);
status_t res = mStatusTracker->run(String8::format("C3Dev-%s-Status", mId.string()).string());
@@ -170,7 +171,7 @@
/** Start up request queue thread */
mRequestThread = createNewRequestThread(
this, mStatusTracker, mInterface, sessionParamKeys,
- mUseHalBufManager, mSupportCameraMute);
+ mUseHalBufManager, mSupportCameraMute, mOverrideToPortrait);
res = mRequestThread->run(String8::format("C3Dev-%s-ReqQueue", mId.string()).string());
if (res != OK) {
SET_ERR_L("Unable to start request queue thread: %s (%d)",
@@ -230,6 +231,15 @@
// Hidl/AidlCamera3DeviceInjectionMethods
mInjectionMethods = createCamera3DeviceInjectionMethods(this);
+ /** Start watchdog thread */
+ mCameraServiceWatchdog = new CameraServiceWatchdog();
+ res = mCameraServiceWatchdog->run("CameraServiceWatchdog");
+ if (res != OK) {
+ SET_ERR_L("Unable to start camera service watchdog thread: %s (%d)",
+ strerror(-res), res);
+ return res;
+ }
+
return OK;
}
@@ -1345,12 +1355,34 @@
set_camera_metadata_vendor_id(meta, mVendorTagId);
filteredParams.unlock(meta);
if (availableSessionKeys.count > 0) {
+ bool rotateAndCropSessionKey = false;
for (size_t i = 0; i < availableSessionKeys.count; i++) {
camera_metadata_ro_entry entry = params.find(
availableSessionKeys.data.i32[i]);
if (entry.count > 0) {
filteredParams.update(entry);
}
+ if (ANDROID_SCALER_ROTATE_AND_CROP == availableSessionKeys.data.i32[i]) {
+ rotateAndCropSessionKey = true;
+ }
+ }
+
+ if (rotateAndCropSessionKey) {
+ sp<CaptureRequest> request = new CaptureRequest();
+ PhysicalCameraSettings settingsList;
+ settingsList.metadata = filteredParams;
+ request->mSettingsList.push_back(settingsList);
+
+ auto rotateAndCropEntry = filteredParams.find(ANDROID_SCALER_ROTATE_AND_CROP);
+ if (rotateAndCropEntry.count > 0 &&
+ rotateAndCropEntry.data.u8[0] == ANDROID_SCALER_ROTATE_AND_CROP_AUTO) {
+ request->mRotateAndCropAuto = true;
+ } else {
+ request->mRotateAndCropAuto = false;
+ }
+
+ overrideAutoRotateAndCrop(request, mOverrideToPortrait, mRotateAndCropOverride);
+ filteredParams = request->mSettingsList.begin()->metadata;
}
}
@@ -1737,7 +1769,7 @@
}
// Calculate expected duration for flush with additional buffer time in ms for watchdog
- uint64_t maxExpectedDuration = (getExpectedInFlightDuration() + kBaseGetBufferWait) / 1e6;
+ uint64_t maxExpectedDuration = ns2ms(getExpectedInFlightDuration() + kBaseGetBufferWait);
status_t res = mCameraServiceWatchdog->WATCH_CUSTOM_TIMER(mRequestThread->flush(),
maxExpectedDuration / kCycleLengthMs, kCycleLengthMs);
@@ -2330,6 +2362,9 @@
tryRemoveFakeStreamLocked();
}
+ // Override stream use case based on "adb shell command"
+ overrideStreamUseCaseLocked();
+
// Start configuring the streams
ALOGV("%s: Camera %s: Starting stream configuration", __FUNCTION__, mId.string());
@@ -2359,7 +2394,7 @@
}
mGroupIdPhysicalCameraMap.clear();
- bool composerSurfacePresent = false;
+ mComposerOutput = false;
for (size_t i = 0; i < mOutputStreams.size(); i++) {
// Don't configure bidi streams twice, nor add them twice to the list
@@ -2402,7 +2437,7 @@
}
if (outputStream->usage & GraphicBuffer::USAGE_HW_COMPOSER) {
- composerSurfacePresent = true;
+ mComposerOutput = true;
}
}
@@ -2471,7 +2506,7 @@
}
}
- mRequestThread->setComposerSurface(composerSurfacePresent);
+ mRequestThread->setComposerSurface(mComposerOutput);
// Request thread needs to know to avoid using repeat-last-settings protocol
// across configure_streams() calls
@@ -2674,7 +2709,7 @@
status_t Camera3Device::registerInFlight(uint32_t frameNumber,
int32_t numBuffers, CaptureResultExtras resultExtras, bool hasInput,
bool hasAppCallback, nsecs_t minExpectedDuration, nsecs_t maxExpectedDuration,
- const std::set<std::set<String8>>& physicalCameraIds,
+ bool isFixedFps, const std::set<std::set<String8>>& physicalCameraIds,
bool isStillCapture, bool isZslCapture, bool rotateAndCropAuto,
const std::set<std::string>& cameraIdsWithZoom,
const SurfaceMap& outputSurfaces, nsecs_t requestTimeNs) {
@@ -2683,7 +2718,7 @@
ssize_t res;
res = mInFlightMap.add(frameNumber, InFlightRequest(numBuffers, resultExtras, hasInput,
- hasAppCallback, minExpectedDuration, maxExpectedDuration, physicalCameraIds,
+ hasAppCallback, minExpectedDuration, maxExpectedDuration, isFixedFps, physicalCameraIds,
isStillCapture, isZslCapture, rotateAndCropAuto, cameraIdsWithZoom, requestTimeNs,
outputSurfaces));
if (res < 0) return res;
@@ -2874,7 +2909,8 @@
sp<StatusTracker> statusTracker,
sp<HalInterface> interface, const Vector<int32_t>& sessionParamKeys,
bool useHalBufManager,
- bool supportCameraMute) :
+ bool supportCameraMute,
+ bool overrideToPortrait) :
Thread(/*canCallJava*/false),
mParent(parent),
mStatusTracker(statusTracker),
@@ -2903,7 +2939,8 @@
mSessionParamKeys(sessionParamKeys),
mLatestSessionParams(sessionParamKeys.size()),
mUseHalBufManager(useHalBufManager),
- mSupportCameraMute(supportCameraMute){
+ mSupportCameraMute(supportCameraMute),
+ mOverrideToPortrait(overrideToPortrait) {
mStatusId = statusTracker->addComponent("RequestThread");
}
@@ -3243,16 +3280,18 @@
return true;
}
-std::pair<nsecs_t, nsecs_t> Camera3Device::RequestThread::calculateExpectedDurationRange(
- const camera_metadata_t *request) {
- std::pair<nsecs_t, nsecs_t> expectedRange(
+Camera3Device::RequestThread::ExpectedDurationInfo
+ Camera3Device::RequestThread::calculateExpectedDurationRange(
+ const camera_metadata_t *request) {
+ ExpectedDurationInfo expectedDurationInfo = {
InFlightRequest::kDefaultMinExpectedDuration,
- InFlightRequest::kDefaultMaxExpectedDuration);
+ InFlightRequest::kDefaultMaxExpectedDuration,
+ /*isFixedFps*/false};
camera_metadata_ro_entry_t e = camera_metadata_ro_entry_t();
find_camera_metadata_ro_entry(request,
ANDROID_CONTROL_AE_MODE,
&e);
- if (e.count == 0) return expectedRange;
+ if (e.count == 0) return expectedDurationInfo;
switch (e.data.u8[0]) {
case ANDROID_CONTROL_AE_MODE_OFF:
@@ -3260,29 +3299,32 @@
ANDROID_SENSOR_EXPOSURE_TIME,
&e);
if (e.count > 0) {
- expectedRange.first = e.data.i64[0];
- expectedRange.second = expectedRange.first;
+ expectedDurationInfo.minDuration = e.data.i64[0];
+ expectedDurationInfo.maxDuration = expectedDurationInfo.minDuration;
}
find_camera_metadata_ro_entry(request,
ANDROID_SENSOR_FRAME_DURATION,
&e);
if (e.count > 0) {
- expectedRange.first = std::max(e.data.i64[0], expectedRange.first);
- expectedRange.second = expectedRange.first;
+ expectedDurationInfo.minDuration =
+ std::max(e.data.i64[0], expectedDurationInfo.minDuration);
+ expectedDurationInfo.maxDuration = expectedDurationInfo.minDuration;
}
+ expectedDurationInfo.isFixedFps = false;
break;
default:
find_camera_metadata_ro_entry(request,
ANDROID_CONTROL_AE_TARGET_FPS_RANGE,
&e);
if (e.count > 1) {
- expectedRange.first = 1e9 / e.data.i32[1];
- expectedRange.second = 1e9 / e.data.i32[0];
+ expectedDurationInfo.minDuration = 1e9 / e.data.i32[1];
+ expectedDurationInfo.maxDuration = 1e9 / e.data.i32[0];
}
+ expectedDurationInfo.isFixedFps = (e.data.i32[1] == e.data.i32[0]);
break;
}
- return expectedRange;
+ return expectedDurationInfo;
}
bool Camera3Device::RequestThread::skipHFRTargetFPSUpdate(int32_t tag,
@@ -3421,6 +3463,16 @@
latestRequestId = NAME_NOT_FOUND;
}
+ for (size_t i = 0; i < mNextRequests.size(); i++) {
+ auto& nextRequest = mNextRequests.editItemAt(i);
+ sp<CaptureRequest> captureRequest = nextRequest.captureRequest;
+ // Do not override rotate&crop for stream configurations that include
+ // SurfaceViews(HW_COMPOSER) output, unless mOverrideToPortrait is set.
+ // The display rotation there will be compensated by NATIVE_WINDOW_TRANSFORM_INVERSE_DISPLAY
+ captureRequest->mRotateAndCropChanged = (mComposerOutput && !mOverrideToPortrait) ? false :
+ overrideAutoRotateAndCrop(captureRequest);
+ }
+
// 'mNextRequests' will at this point contain either a set of HFR batched requests
// or a single request from streaming or burst. In either case the first element
// should contain the latest camera settings that we need to check for any session
@@ -3566,18 +3618,13 @@
bool triggersMixedIn = (triggerCount > 0 || mPrevTriggers > 0);
mPrevTriggers = triggerCount;
- // Do not override rotate&crop for stream configurations that include
- // SurfaceViews(HW_COMPOSER) output. The display rotation there will be
- // compensated by NATIVE_WINDOW_TRANSFORM_INVERSE_DISPLAY
- bool rotateAndCropChanged = mComposerOutput ? false :
- overrideAutoRotateAndCrop(captureRequest);
bool testPatternChanged = overrideTestPattern(captureRequest);
// If the request is the same as last, or we had triggers now or last time or
// changing overrides this time
bool newRequest =
(mPrevRequest != captureRequest || triggersMixedIn ||
- rotateAndCropChanged || testPatternChanged) &&
+ captureRequest->mRotateAndCropChanged || testPatternChanged) &&
// Request settings are all the same within one batch, so only treat the first
// request in a batch as new
!(batchedRequest && i > 0);
@@ -3897,13 +3944,14 @@
isZslCapture = true;
}
}
- auto expectedDurationRange = calculateExpectedDurationRange(settings);
+ auto expectedDurationInfo = calculateExpectedDurationRange(settings);
res = parent->registerInFlight(halRequest->frame_number,
totalNumBuffers, captureRequest->mResultExtras,
/*hasInput*/halRequest->input_buffer != NULL,
hasCallback,
- /*min*/expectedDurationRange.first,
- /*max*/expectedDurationRange.second,
+ expectedDurationInfo.minDuration,
+ expectedDurationInfo.maxDuration,
+ expectedDurationInfo.isFixedFps,
requestedPhysicalCameras, isStillCapture, isZslCapture,
captureRequest->mRotateAndCropAuto, mPrevCameraIdsWithZoom,
(mUseHalBufManager) ? uniqueSurfaceIdMap :
@@ -4039,9 +4087,6 @@
camera_metadata_enum_android_scaler_rotate_and_crop_t rotateAndCropValue) {
ATRACE_CALL();
Mutex::Autolock l(mTriggerMutex);
- if (rotateAndCropValue == ANDROID_SCALER_ROTATE_AND_CROP_AUTO) {
- return BAD_VALUE;
- }
mRotateAndCropOverride = rotateAndCropValue;
return OK;
}
@@ -4094,6 +4139,30 @@
}
}
+status_t Camera3Device::setCameraServiceWatchdog(bool enabled) {
+ Mutex::Autolock il(mInterfaceLock);
+ Mutex::Autolock l(mLock);
+
+ if (mCameraServiceWatchdog != NULL) {
+ mCameraServiceWatchdog->setEnabled(enabled);
+ }
+
+ return OK;
+}
+
+void Camera3Device::setStreamUseCaseOverrides(
+ const std::vector<int64_t>& useCaseOverrides) {
+ Mutex::Autolock il(mInterfaceLock);
+ Mutex::Autolock l(mLock);
+ mStreamUseCaseOverrides = useCaseOverrides;
+}
+
+void Camera3Device::clearStreamUseCaseOverrides() {
+ Mutex::Autolock il(mInterfaceLock);
+ Mutex::Autolock l(mLock);
+ mStreamUseCaseOverrides.clear();
+}
+
void Camera3Device::RequestThread::cleanUpFailedRequests(bool sendRequestError) {
if (mNextRequests.empty()) {
return;
@@ -4594,29 +4663,44 @@
return OK;
}
-bool Camera3Device::RequestThread::overrideAutoRotateAndCrop(
- const sp<CaptureRequest> &request) {
+bool Camera3Device::RequestThread::overrideAutoRotateAndCrop(const sp<CaptureRequest> &request) {
+ ATRACE_CALL();
+ Mutex::Autolock l(mTriggerMutex);
+ return Camera3Device::overrideAutoRotateAndCrop(request, this->mOverrideToPortrait,
+ this->mRotateAndCropOverride);
+}
+
+bool Camera3Device::overrideAutoRotateAndCrop(const sp<CaptureRequest> &request,
+ bool overrideToPortrait,
+ camera_metadata_enum_android_scaler_rotate_and_crop_t rotateAndCropOverride) {
ATRACE_CALL();
+ if (overrideToPortrait) {
+ uint8_t rotateAndCrop_u8 = rotateAndCropOverride;
+ CameraMetadata &metadata = request->mSettingsList.begin()->metadata;
+ metadata.update(ANDROID_SCALER_ROTATE_AND_CROP,
+ &rotateAndCrop_u8, 1);
+ return true;
+ }
+
if (request->mRotateAndCropAuto) {
- Mutex::Autolock l(mTriggerMutex);
CameraMetadata &metadata = request->mSettingsList.begin()->metadata;
auto rotateAndCropEntry = metadata.find(ANDROID_SCALER_ROTATE_AND_CROP);
if (rotateAndCropEntry.count > 0) {
- if (rotateAndCropEntry.data.u8[0] == mRotateAndCropOverride) {
+ if (rotateAndCropEntry.data.u8[0] == rotateAndCropOverride) {
return false;
} else {
- rotateAndCropEntry.data.u8[0] = mRotateAndCropOverride;
+ rotateAndCropEntry.data.u8[0] = rotateAndCropOverride;
return true;
}
} else {
- uint8_t rotateAndCrop_u8 = mRotateAndCropOverride;
- metadata.update(ANDROID_SCALER_ROTATE_AND_CROP,
- &rotateAndCrop_u8, 1);
+ uint8_t rotateAndCrop_u8 = rotateAndCropOverride;
+ metadata.update(ANDROID_SCALER_ROTATE_AND_CROP, &rotateAndCrop_u8, 1);
return true;
}
}
+
return false;
}
@@ -5102,6 +5186,10 @@
if (mRequestThread == nullptr) {
return INVALID_OPERATION;
}
+ if (rotateAndCropValue == ANDROID_SCALER_ROTATE_AND_CROP_AUTO) {
+ return BAD_VALUE;
+ }
+ mRotateAndCropOverride = rotateAndCropValue;
return mRequestThread->setRotateAndCropAutoBehavior(rotateAndCropValue);
}
@@ -5198,4 +5286,55 @@
return mInjectionMethods->stopInjection();
}
+void Camera3Device::overrideStreamUseCaseLocked() {
+ if (mStreamUseCaseOverrides.size() == 0) {
+ return;
+ }
+
+ // Start from an array of indexes in mStreamUseCaseOverrides, and sort them
+ // based first on size, and second on formats of [JPEG, RAW, YUV, PRIV].
+ std::vector<int> outputStreamsIndices(mOutputStreams.size());
+ for (size_t i = 0; i < outputStreamsIndices.size(); i++) {
+ outputStreamsIndices[i] = i;
+ }
+
+ std::sort(outputStreamsIndices.begin(), outputStreamsIndices.end(),
+ [&](int a, int b) -> bool {
+
+ auto formatScore = [](int format) {
+ switch (format) {
+ case HAL_PIXEL_FORMAT_BLOB:
+ return 4;
+ case HAL_PIXEL_FORMAT_RAW16:
+ case HAL_PIXEL_FORMAT_RAW10:
+ case HAL_PIXEL_FORMAT_RAW12:
+ return 3;
+ case HAL_PIXEL_FORMAT_YCBCR_420_888:
+ return 2;
+ case HAL_PIXEL_FORMAT_IMPLEMENTATION_DEFINED:
+ return 1;
+ default:
+ return 0;
+ }
+ };
+
+ int sizeA = mOutputStreams[a]->getWidth() * mOutputStreams[a]->getHeight();
+ int sizeB = mOutputStreams[a]->getWidth() * mOutputStreams[a]->getHeight();
+ int formatAScore = formatScore(mOutputStreams[a]->getFormat());
+ int formatBScore = formatScore(mOutputStreams[b]->getFormat());
+ if (sizeA > sizeB ||
+ (sizeA == sizeB && formatAScore >= formatBScore)) {
+ return true;
+ } else {
+ return false;
+ }
+ });
+
+ size_t overlapSize = std::min(mStreamUseCaseOverrides.size(), mOutputStreams.size());
+ for (size_t i = 0; i < mOutputStreams.size(); i++) {
+ mOutputStreams[outputStreamsIndices[i]]->setStreamUseCase(
+ mStreamUseCaseOverrides[std::min(i, overlapSize-1)]);
+ }
+}
+
}; // namespace android
diff --git a/services/camera/libcameraservice/device3/Camera3Device.h b/services/camera/libcameraservice/device3/Camera3Device.h
index f927b4d..e6073a9 100644
--- a/services/camera/libcameraservice/device3/Camera3Device.h
+++ b/services/camera/libcameraservice/device3/Camera3Device.h
@@ -82,7 +82,8 @@
friend class AidlCamera3Device;
public:
- explicit Camera3Device(const String8& id, bool overrideForPerfClass, bool legacyClient = false);
+ explicit Camera3Device(const String8& id, bool overrideForPerfClass, bool overrideToPortrait,
+ bool legacyClient = false);
virtual ~Camera3Device();
// Delete and optionally close native handles and clear the input vector afterward
@@ -281,6 +282,18 @@
*/
status_t setCameraMute(bool enabled);
+ /**
+ * Enables/disables camera service watchdog
+ */
+ status_t setCameraServiceWatchdog(bool enabled);
+
+ // Set stream use case overrides
+ void setStreamUseCaseOverrides(
+ const std::vector<int64_t>& useCaseOverrides);
+
+ // Clear stream use case overrides
+ void clearStreamUseCaseOverrides();
+
// Get the status trackeer for the camera device
wp<camera3::StatusTracker> getStatusTracker() { return mStatusTracker; }
@@ -568,6 +581,9 @@
// overriding of ROTATE_AND_CROP value and adjustment of coordinates
// in several other controls in both the request and the result
bool mRotateAndCropAuto;
+ // Indicates that the ROTATE_AND_CROP value within 'mSettingsList' was modified
+ // irrespective of the original value.
+ bool mRotateAndCropChanged = false;
// Whether this capture request has its zoom ratio set to 1.0x before
// the framework overrides it for camera HAL consumption.
@@ -754,6 +770,11 @@
*/
static nsecs_t getMonoToBoottimeOffset();
+ // Override rotate_and_crop control if needed
+ static bool overrideAutoRotateAndCrop(const sp<CaptureRequest> &request /*out*/,
+ bool overrideToPortrait,
+ camera_metadata_enum_android_scaler_rotate_and_crop_t rotateAndCropOverride);
+
struct RequestTrigger {
// Metadata tag number, e.g. android.control.aePrecaptureTrigger
uint32_t metadataTag;
@@ -783,7 +804,8 @@
sp<HalInterface> interface,
const Vector<int32_t>& sessionParamKeys,
bool useHalBufManager,
- bool supportCameraMute);
+ bool supportCameraMute,
+ bool overrideToPortrait);
~RequestThread();
void setNotificationListener(wp<NotificationListener> listener);
@@ -903,7 +925,7 @@
status_t addFakeTriggerIds(const sp<CaptureRequest> &request);
// Override rotate_and_crop control if needed; returns true if the current value was changed
- bool overrideAutoRotateAndCrop(const sp<CaptureRequest> &request);
+ bool overrideAutoRotateAndCrop(const sp<CaptureRequest> &request /*out*/);
// Override test_pattern control if needed for camera mute; returns true
// if the current value was changed
@@ -962,8 +984,13 @@
// send request in mNextRequests to HAL in a batch. Return true = sucssess
bool sendRequestsBatch();
- // Calculate the expected (minimum, maximum) duration range for a request
- std::pair<nsecs_t, nsecs_t> calculateExpectedDurationRange(
+ // Calculate the expected (minimum, maximum, isFixedFps) duration info for a request
+ struct ExpectedDurationInfo {
+ nsecs_t minDuration;
+ nsecs_t maxDuration;
+ bool isFixedFps;
+ };
+ ExpectedDurationInfo calculateExpectedDurationRange(
const camera_metadata_t *request);
// Check and update latest session parameters based on the current request settings.
@@ -1055,6 +1082,7 @@
const bool mUseHalBufManager;
const bool mSupportCameraMute;
+ const bool mOverrideToPortrait;
};
virtual sp<RequestThread> createNewRequestThread(wp<Camera3Device> /*parent*/,
@@ -1062,7 +1090,8 @@
sp<HalInterface> /*interface*/,
const Vector<int32_t>& /*sessionParamKeys*/,
bool /*useHalBufManager*/,
- bool /*supportCameraMute*/) = 0;
+ bool /*supportCameraMute*/,
+ bool /*overrideToPortrait*/) = 0;
sp<RequestThread> mRequestThread;
@@ -1082,7 +1111,7 @@
status_t registerInFlight(uint32_t frameNumber,
int32_t numBuffers, CaptureResultExtras resultExtras, bool hasInput,
bool callback, nsecs_t minExpectedDuration, nsecs_t maxExpectedDuration,
- const std::set<std::set<String8>>& physicalCameraIds,
+ bool isFixedFps, const std::set<std::set<String8>>& physicalCameraIds,
bool isStillCapture, bool isZslCapture, bool rotateAndCropAuto,
const std::set<std::string>& cameraIdsWithZoom, const SurfaceMap& outputSurfaces,
nsecs_t requestTimeNs);
@@ -1332,8 +1361,20 @@
// performance class.
bool mOverrideForPerfClass;
+ // Whether the camera framework overrides the device characteristics for
+ // app compatibility reasons.
+ bool mOverrideToPortrait;
+ camera_metadata_enum_android_scaler_rotate_and_crop_t mRotateAndCropOverride;
+ bool mComposerOutput;
+
+ // Current active physical id of the logical multi-camera, if any
+ std::string mActivePhysicalId;
+
// The current minimum expected frame duration based on AE_TARGET_FPS_RANGE
nsecs_t mMinExpectedDuration = 0;
+ // Whether the camera device runs at fixed frame rate based on AE_MODE and
+ // AE_TARGET_FPS_RANGE
+ bool mIsFixedFps = false;
// Injection camera related methods.
class Camera3DeviceInjectionMethods : public virtual RefBase {
@@ -1417,6 +1458,8 @@
sp<Camera3DeviceInjectionMethods> mInjectionMethods;
+ void overrideStreamUseCaseLocked();
+
}; // class Camera3Device
}; // namespace android
diff --git a/services/camera/libcameraservice/device3/Camera3FakeStream.h b/services/camera/libcameraservice/device3/Camera3FakeStream.h
index 8cecabd..1e9f478 100644
--- a/services/camera/libcameraservice/device3/Camera3FakeStream.h
+++ b/services/camera/libcameraservice/device3/Camera3FakeStream.h
@@ -100,7 +100,9 @@
virtual status_t setBatchSize(size_t batchSize) override;
- virtual void onMinDurationChanged(nsecs_t /*duration*/) {}
+ virtual void onMinDurationChanged(nsecs_t /*duration*/, bool /*fixedFps*/) {}
+
+ virtual void setStreamUseCase(int64_t /*streamUseCase*/) {}
protected:
/**
diff --git a/services/camera/libcameraservice/device3/Camera3IOStreamBase.cpp b/services/camera/libcameraservice/device3/Camera3IOStreamBase.cpp
index add1483..f594f84 100644
--- a/services/camera/libcameraservice/device3/Camera3IOStreamBase.cpp
+++ b/services/camera/libcameraservice/device3/Camera3IOStreamBase.cpp
@@ -41,8 +41,10 @@
physicalCameraId, sensorPixelModesUsed, setId, isMultiResolution,
dynamicRangeProfile, streamUseCase, deviceTimeBaseIsRealtime, timestampBase),
mTotalBufferCount(0),
+ mMaxCachedBufferCount(0),
mHandoutTotalBufferCount(0),
mHandoutOutputBufferCount(0),
+ mCachedOutputBufferCount(0),
mFrameCount(0),
mLastTimestamp(0) {
@@ -95,8 +97,8 @@
lines.appendFormat(" Timestamp base: %d\n", getTimestampBase());
lines.appendFormat(" Frames produced: %d, last timestamp: %" PRId64 " ns\n",
mFrameCount, mLastTimestamp);
- lines.appendFormat(" Total buffers: %zu, currently dequeued: %zu\n",
- mTotalBufferCount, mHandoutTotalBufferCount);
+ lines.appendFormat(" Total buffers: %zu, currently dequeued: %zu, currently cached: %zu\n",
+ mTotalBufferCount, mHandoutTotalBufferCount, mCachedOutputBufferCount);
write(fd, lines.string(), lines.size());
Camera3Stream::dump(fd, args);
@@ -135,6 +137,14 @@
return (mHandoutTotalBufferCount - mHandoutOutputBufferCount);
}
+size_t Camera3IOStreamBase::getCachedOutputBufferCountLocked() const {
+ return mCachedOutputBufferCount;
+}
+
+size_t Camera3IOStreamBase::getMaxCachedOutputBuffersLocked() const {
+ return mMaxCachedBufferCount;
+}
+
status_t Camera3IOStreamBase::disconnectLocked() {
switch (mState) {
case STATE_IN_RECONFIG:
diff --git a/services/camera/libcameraservice/device3/Camera3IOStreamBase.h b/services/camera/libcameraservice/device3/Camera3IOStreamBase.h
index f389d53..ca1f238 100644
--- a/services/camera/libcameraservice/device3/Camera3IOStreamBase.h
+++ b/services/camera/libcameraservice/device3/Camera3IOStreamBase.h
@@ -56,11 +56,18 @@
int getMaxTotalBuffers() const { return mTotalBufferCount; }
protected:
size_t mTotalBufferCount;
+ // The maximum number of cached buffers allowed for this stream
+ size_t mMaxCachedBufferCount;
+
// sum of input and output buffers that are currently acquired by HAL
size_t mHandoutTotalBufferCount;
// number of output buffers that are currently acquired by HAL. This will be
// Redundant when camera3 streams are no longer bidirectional streams.
size_t mHandoutOutputBufferCount;
+ // number of cached output buffers that are currently queued in the camera
+ // server but not yet queued to the buffer queue.
+ size_t mCachedOutputBufferCount;
+
uint32_t mFrameCount;
// Last received output buffer's timestamp
nsecs_t mLastTimestamp;
@@ -97,6 +104,9 @@
virtual size_t getHandoutInputBufferCountLocked();
+ virtual size_t getCachedOutputBufferCountLocked() const;
+ virtual size_t getMaxCachedOutputBuffersLocked() const;
+
virtual status_t getEndpointUsage(uint64_t *usage) const = 0;
status_t getBufferPreconditionCheckLocked() const;
diff --git a/services/camera/libcameraservice/device3/Camera3OfflineSession.h b/services/camera/libcameraservice/device3/Camera3OfflineSession.h
index a799719..5ee6ca5 100644
--- a/services/camera/libcameraservice/device3/Camera3OfflineSession.h
+++ b/services/camera/libcameraservice/device3/Camera3OfflineSession.h
@@ -248,6 +248,9 @@
// The current minimum expected frame duration based on AE_TARGET_FPS_RANGE
nsecs_t mMinExpectedDuration = 0;
+ // Whether the camera device runs at fixed frame rate based on AE_MODE and
+ // AE_TARGET_FPS_RANGE
+ bool mIsFixedFps = false;
// SetErrorInterface
void setErrorState(const char *fmt, ...) override;
diff --git a/services/camera/libcameraservice/device3/Camera3OutputStream.cpp b/services/camera/libcameraservice/device3/Camera3OutputStream.cpp
index 69163a5..ef12b64 100644
--- a/services/camera/libcameraservice/device3/Camera3OutputStream.cpp
+++ b/services/camera/libcameraservice/device3/Camera3OutputStream.cpp
@@ -419,6 +419,7 @@
mLock.unlock();
ANativeWindowBuffer *anwBuffer = container_of(buffer.buffer, ANativeWindowBuffer, handle);
+ bool bufferDeferred = false;
/**
* Return buffer back to ANativeWindow
*/
@@ -478,6 +479,7 @@
__FUNCTION__, mId, strerror(-res), res);
return res;
}
+ bufferDeferred = true;
} else {
nsecs_t presentTime = mSyncToDisplay ?
syncTimestampToDisplayLocked(captureTime) : captureTime;
@@ -501,6 +503,10 @@
}
mLock.lock();
+ if (bufferDeferred) {
+ mCachedOutputBufferCount++;
+ }
+
// Once a valid buffer has been returned to the queue, can no longer
// dequeue all buffers for preallocation.
if (buffer.status != CAMERA_BUFFER_STATUS_ERROR) {
@@ -696,10 +702,15 @@
!isVideoStream());
if (forceChoreographer || defaultToChoreographer) {
mSyncToDisplay = true;
+ // For choreographer synced stream, extra buffers aren't kept by
+ // camera service. So no need to update mMaxCachedBufferCount.
mTotalBufferCount += kDisplaySyncExtraBuffer;
} else if (defaultToSpacer) {
mPreviewFrameSpacer = new PreviewFrameSpacer(this, mConsumer);
- mTotalBufferCount ++;
+ // For preview frame spacer, the extra buffer is kept by camera
+ // service. So update mMaxCachedBufferCount.
+ mMaxCachedBufferCount = 1;
+ mTotalBufferCount += mMaxCachedBufferCount;
res = mPreviewFrameSpacer->run(String8::format("PreviewSpacer-%d", mId).string());
if (res != OK) {
ALOGE("%s: Unable to start preview spacer", __FUNCTION__);
@@ -968,6 +979,14 @@
return true;
}
+void Camera3OutputStream::onCachedBufferQueued() {
+ Mutex::Autolock l(mLock);
+ mCachedOutputBufferCount--;
+ // Signal whoever is waiting for the buffer to be returned to the buffer
+ // queue.
+ mOutputBufferReturnedSignal.signal();
+}
+
status_t Camera3OutputStream::disconnectLocked() {
status_t res;
@@ -1367,9 +1386,15 @@
return OK;
}
-void Camera3OutputStream::onMinDurationChanged(nsecs_t duration) {
+void Camera3OutputStream::onMinDurationChanged(nsecs_t duration, bool fixedFps) {
Mutex::Autolock l(mLock);
mMinExpectedDuration = duration;
+ mFixedFps = fixedFps;
+}
+
+void Camera3OutputStream::setStreamUseCase(int64_t streamUseCase) {
+ Mutex::Autolock l(mLock);
+ camera_stream::use_case = streamUseCase;
}
void Camera3OutputStream::returnPrefetchedBuffersLocked() {
@@ -1390,29 +1415,39 @@
}
nsecs_t Camera3OutputStream::syncTimestampToDisplayLocked(nsecs_t t) {
+ nsecs_t currentTime = systemTime();
+ if (!mFixedFps) {
+ mLastCaptureTime = t;
+ mLastPresentTime = currentTime;
+ return t;
+ }
+
ParcelableVsyncEventData parcelableVsyncEventData;
auto res = mDisplayEventReceiver.getLatestVsyncEventData(&parcelableVsyncEventData);
if (res != OK) {
ALOGE("%s: Stream %d: Error getting latest vsync event data: %s (%d)",
__FUNCTION__, mId, strerror(-res), res);
mLastCaptureTime = t;
- mLastPresentTime = t;
+ mLastPresentTime = currentTime;
return t;
}
const VsyncEventData& vsyncEventData = parcelableVsyncEventData.vsync;
- nsecs_t currentTime = systemTime();
+ nsecs_t minPresentT = mLastPresentTime + vsyncEventData.frameInterval / 2;
- // Reset capture to present time offset if:
- // - More than 1 second between frames.
- // - The frame duration deviates from multiples of vsync frame intervals.
+ // Find the best presentation time without worrying about previous frame's
+ // presentation time if capture interval is more than kSpacingResetIntervalNs.
+ //
+ // When frame interval is more than 50 ms apart (3 vsyncs for 60hz refresh rate),
+ // there is little risk in starting over and finding the earliest vsync to latch onto.
+ // - Update captureToPresentTime offset to be used for later frames.
+ // - Example use cases:
+ // - when frame rate drops down to below 20 fps, or
+ // - A new streaming session starts (stopPreview followed by
+ // startPreview)
+ //
nsecs_t captureInterval = t - mLastCaptureTime;
- float captureToVsyncIntervalRatio = 1.0f * captureInterval / vsyncEventData.frameInterval;
- float ratioDeviation = std::fabs(
- captureToVsyncIntervalRatio - std::roundf(captureToVsyncIntervalRatio));
- if (captureInterval > kSpacingResetIntervalNs ||
- ratioDeviation >= kMaxIntervalRatioDeviation) {
- nsecs_t minPresentT = mLastPresentTime + vsyncEventData.frameInterval / 2;
+ if (captureInterval > kSpacingResetIntervalNs) {
for (size_t i = 0; i < VsyncEventData::kFrameTimelinesLength; i++) {
const auto& timeline = vsyncEventData.frameTimelines[i];
if (timeline.deadlineTimestamp >= currentTime &&
@@ -1434,21 +1469,54 @@
nsecs_t idealPresentT = t + mCaptureToPresentOffset;
nsecs_t expectedPresentT = mLastPresentTime;
nsecs_t minDiff = INT64_MAX;
- // Derive minimum intervals between presentation times based on minimal
+
+ // In fixed FPS case, when frame durations are close to multiples of display refresh
+ // rate, derive minimum intervals between presentation times based on minimal
// expected duration. The minimum number of Vsyncs is:
// - 0 if minFrameDuration in (0, 1.5] * vSyncInterval,
// - 1 if minFrameDuration in (1.5, 2.5] * vSyncInterval,
// - and so on.
+ //
+ // This spaces out the displaying of the frames so that the frame
+ // presentations are roughly in sync with frame captures.
int minVsyncs = (mMinExpectedDuration - vsyncEventData.frameInterval / 2) /
vsyncEventData.frameInterval;
if (minVsyncs < 0) minVsyncs = 0;
nsecs_t minInterval = minVsyncs * vsyncEventData.frameInterval;
+
+ // In fixed FPS case, if the frame duration deviates from multiples of
+ // display refresh rate, find the closest Vsync without requiring a minimum
+ // number of Vsync.
+ //
+ // Example: (24fps camera, 60hz refresh):
+ // capture readout: | t1 | t1 | .. | t1 | .. | t1 | .. | t1 |
+ // display VSYNC: | t2 | t2 | ... | t2 | ... | t2 | ... | t2 |
+ // | : 1 frame
+ // t1 : 41.67ms
+ // t2 : 16.67ms
+ // t1/t2 = 2.5
+ //
+ // 24fps is a commonly used video frame rate. Because the capture
+ // interval is 2.5 times of display refresh interval, the minVsyncs
+ // calculation will directly fall at the boundary condition. In this case,
+ // we should fall back to the basic logic of finding closest vsync
+ // timestamp without worrying about minVsyncs.
+ float captureToVsyncIntervalRatio = 1.0f * mMinExpectedDuration / vsyncEventData.frameInterval;
+ float ratioDeviation = std::fabs(
+ captureToVsyncIntervalRatio - std::roundf(captureToVsyncIntervalRatio));
+ bool captureDeviateFromVsync = ratioDeviation >= kMaxIntervalRatioDeviation;
+ bool cameraDisplayInSync = (mFixedFps && !captureDeviateFromVsync);
+
// Find best timestamp in the vsync timelines:
- // - Only use at most 3 timelines to avoid long latency
- // - closest to the ideal present time,
+ // - Only use at most kMaxTimelines timelines to avoid long latency
+ // - closest to the ideal presentation time,
// - deadline timestamp is greater than the current time, and
- // - the candidate present time is at least minInterval in the future
- // compared to last present time.
+ // - For fixed FPS, if the capture interval doesn't deviate too much from refresh interval,
+ // the candidate presentation time is at least minInterval in the future compared to last
+ // presentation time.
+ // - For variable FPS, or if the capture interval deviates from refresh
+ // interval for more than 5%, find a presentation time closest to the
+ // (lastPresentationTime + captureToPresentOffset) instead.
int maxTimelines = std::min(kMaxTimelines, (int)VsyncEventData::kFrameTimelinesLength);
float biasForShortDelay = 1.0f;
for (int i = 0; i < maxTimelines; i ++) {
@@ -1461,12 +1529,51 @@
}
if (std::abs(vsyncTime.expectedPresentationTime - idealPresentT) < minDiff &&
vsyncTime.deadlineTimestamp >= currentTime &&
- vsyncTime.expectedPresentationTime >
- mLastPresentTime + minInterval + biasForShortDelay * kTimelineThresholdNs) {
+ ((!cameraDisplayInSync && vsyncTime.expectedPresentationTime > minPresentT) ||
+ (cameraDisplayInSync && vsyncTime.expectedPresentationTime >
+ mLastPresentTime + minInterval +
+ static_cast<nsecs_t>(biasForShortDelay * kTimelineThresholdNs)))) {
expectedPresentT = vsyncTime.expectedPresentationTime;
minDiff = std::abs(vsyncTime.expectedPresentationTime - idealPresentT);
}
}
+
+ if (expectedPresentT == mLastPresentTime && expectedPresentT <
+ vsyncEventData.frameTimelines[maxTimelines-1].expectedPresentationTime) {
+ // Couldn't find a reasonable presentation time. Using last frame's
+ // presentation time would cause a frame drop. The best option now
+ // is to use the next VSync as long as the last presentation time
+ // doesn't already has the maximum latency, in which case dropping the
+ // buffer is more desired than increasing latency.
+ //
+ // Example: (60fps camera, 59.9hz refresh):
+ // capture readout: | t1 | t1 | .. | t1 | .. | t1 | .. | t1 |
+ // \ \ \ \ \ \ \ \ \
+ // queue to BQ: | | | | | | | | |
+ // \ \ \ \ \ \ \ \ \
+ // display VSYNC: | t2 | t2 | ... | t2 | ... | t2 | ... | t2 |
+ //
+ // |: 1 frame
+ // t1 : 16.67ms
+ // t2 : 16.69ms
+ //
+ // It takes 833 frames for capture readout count and display VSYNC count to be off
+ // by 1.
+ // - At frames [0, 832], presentationTime is set to timeline[0]
+ // - At frames [833, 833*2-1], presentationTime is set to timeline[1]
+ // - At frames [833*2, 833*3-1] presentationTime is set to timeline[2]
+ // - At frame 833*3, no presentation time is found because we only
+ // search for timeline[0..2].
+ // - Drop one buffer is better than further extend the presentation
+ // time.
+ //
+ // However, if frame 833*2 arrives 16.67ms early (right after frame
+ // 833*2-1), no presentation time can be found because
+ // getLatestVsyncEventData is called early. In that case, it's better to
+ // set presentation time by offseting last presentation time.
+ expectedPresentT += vsyncEventData.frameInterval;
+ }
+
mLastCaptureTime = t;
mLastPresentTime = expectedPresentT;
diff --git a/services/camera/libcameraservice/device3/Camera3OutputStream.h b/services/camera/libcameraservice/device3/Camera3OutputStream.h
index 3587af4..a719d6b 100644
--- a/services/camera/libcameraservice/device3/Camera3OutputStream.h
+++ b/services/camera/libcameraservice/device3/Camera3OutputStream.h
@@ -247,9 +247,15 @@
virtual status_t setBatchSize(size_t batchSize = 1) override;
/**
- * Notify the stream on change of min frame durations.
+ * Notify the stream on change of min frame durations or variable/fixed
+ * frame rate.
*/
- virtual void onMinDurationChanged(nsecs_t duration) override;
+ virtual void onMinDurationChanged(nsecs_t duration, bool fixedFps) override;
+
+ /**
+ * Modify stream use case
+ */
+ virtual void setStreamUseCase(int64_t streamUseCase) override;
/**
* Apply ZSL related consumer usage quirk.
@@ -258,6 +264,7 @@
void setImageDumpMask(int mask) { mImageDumpMask = mask; }
bool shouldLogError(status_t res);
+ void onCachedBufferQueued();
protected:
Camera3OutputStream(int id, camera_stream_type_t type,
@@ -419,6 +426,7 @@
// Re-space frames by overriding timestamp to align with display Vsync.
// Default is on for SurfaceView bound streams.
+ bool mFixedFps = false;
nsecs_t mMinExpectedDuration = 0;
bool mSyncToDisplay = false;
DisplayEventReceiver mDisplayEventReceiver;
@@ -429,7 +437,7 @@
static constexpr nsecs_t kSpacingResetIntervalNs = 50000000LL; // 50 millisecond
static constexpr nsecs_t kTimelineThresholdNs = 1000000LL; // 1 millisecond
static constexpr float kMaxIntervalRatioDeviation = 0.05f;
- static constexpr int kMaxTimelines = 3;
+ static constexpr int kMaxTimelines = 2;
nsecs_t syncTimestampToDisplayLocked(nsecs_t t);
// Re-space frames by delaying queueBuffer so that frame delivery has
diff --git a/services/camera/libcameraservice/device3/Camera3OutputStreamInterface.h b/services/camera/libcameraservice/device3/Camera3OutputStreamInterface.h
index a6d4b96..4baa7e8 100644
--- a/services/camera/libcameraservice/device3/Camera3OutputStreamInterface.h
+++ b/services/camera/libcameraservice/device3/Camera3OutputStreamInterface.h
@@ -110,12 +110,18 @@
virtual status_t setBatchSize(size_t batchSize = 1) = 0;
/**
- * Notify the output stream that the minimum frame duration has changed.
+ * Notify the output stream that the minimum frame duration has changed, or
+ * frame rate has switched between variable and fixed.
*
* The minimum frame duration is calculated based on the upper bound of
* AE_TARGET_FPS_RANGE in the capture request.
*/
- virtual void onMinDurationChanged(nsecs_t duration) = 0;
+ virtual void onMinDurationChanged(nsecs_t duration, bool fixedFps) = 0;
+
+ /**
+ * Modify the stream use case for this output.
+ */
+ virtual void setStreamUseCase(int64_t streamUseCase) = 0;
};
// Helper class to organize a synchronized mapping of stream IDs to stream instances
diff --git a/services/camera/libcameraservice/device3/Camera3OutputUtils.cpp b/services/camera/libcameraservice/device3/Camera3OutputUtils.cpp
index f4e3fad..5021f29 100644
--- a/services/camera/libcameraservice/device3/Camera3OutputUtils.cpp
+++ b/services/camera/libcameraservice/device3/Camera3OutputUtils.cpp
@@ -521,27 +521,50 @@
if (result->partial_result != 0)
request.resultExtras.partialResultCount = result->partial_result;
- if ((result->result != nullptr) && !states.legacyClient) {
+ if (result->result != nullptr) {
camera_metadata_ro_entry entry;
auto ret = find_camera_metadata_ro_entry(result->result,
ANDROID_LOGICAL_MULTI_CAMERA_ACTIVE_PHYSICAL_ID, &entry);
if ((ret == OK) && (entry.count > 0)) {
std::string physicalId(reinterpret_cast<const char *>(entry.data.u8));
- auto deviceInfo = states.physicalDeviceInfoMap.find(physicalId);
- if (deviceInfo != states.physicalDeviceInfoMap.end()) {
- auto orientation = deviceInfo->second.find(ANDROID_SENSOR_ORIENTATION);
- if (orientation.count > 0) {
- ret = CameraUtils::getRotationTransform(deviceInfo->second,
- OutputConfiguration::MIRROR_MODE_AUTO, &request.transform);
- if (ret != OK) {
- ALOGE("%s: Failed to calculate current stream transformation: %s (%d)",
- __FUNCTION__, strerror(-ret), ret);
+ if (!states.activePhysicalId.empty() && physicalId != states.activePhysicalId) {
+ states.listener->notifyPhysicalCameraChange(physicalId);
+ }
+ states.activePhysicalId = physicalId;
+
+ if (!states.legacyClient && !states.overrideToPortrait) {
+ auto deviceInfo = states.physicalDeviceInfoMap.find(physicalId);
+ if (deviceInfo != states.physicalDeviceInfoMap.end()) {
+ auto orientation = deviceInfo->second.find(ANDROID_SENSOR_ORIENTATION);
+ if (orientation.count > 0) {
+ int32_t transform;
+ ret = CameraUtils::getRotationTransform(deviceInfo->second,
+ OutputConfiguration::MIRROR_MODE_AUTO, &transform);
+ if (ret == OK) {
+ // It is possible for camera providers to return the capture
+ // results after the processed frames. In such scenario, we will
+ // not be able to set the output transformation before the frames
+ // return back to the consumer for the current capture request
+ // but we could still try and configure it for any future requests
+ // that are still in flight. The assumption is that the physical
+ // device id remains the same for the duration of the pending queue.
+ for (size_t i = 0; i < states.inflightMap.size(); i++) {
+ auto &r = states.inflightMap.editValueAt(i);
+ if (r.requestTimeNs >= request.requestTimeNs) {
+ r.transform = transform;
+ }
+ }
+ } else {
+ ALOGE("%s: Failed to calculate current stream transformation: %s "
+ "(%d)", __FUNCTION__, strerror(-ret), ret);
+ }
+ } else {
+ ALOGE("%s: Physical device orientation absent!", __FUNCTION__);
}
} else {
- ALOGE("%s: Physical device orientation absent!", __FUNCTION__);
+ ALOGE("%s: Physical device not found in device info map found!",
+ __FUNCTION__);
}
- } else {
- ALOGE("%s: Physical device not found in device info map found!", __FUNCTION__);
}
}
}
@@ -858,12 +881,14 @@
r.resultExtras.hasReadoutTimestamp = true;
r.resultExtras.readoutTimestamp = msg.readout_timestamp;
}
- if (r.minExpectedDuration != states.minFrameDuration) {
+ if (r.minExpectedDuration != states.minFrameDuration ||
+ r.isFixedFps != states.isFixedFps) {
for (size_t i = 0; i < states.outputStreams.size(); i++) {
auto outputStream = states.outputStreams[i];
- outputStream->onMinDurationChanged(r.minExpectedDuration);
+ outputStream->onMinDurationChanged(r.minExpectedDuration, r.isFixedFps);
}
states.minFrameDuration = r.minExpectedDuration;
+ states.isFixedFps = r.isFixedFps;
}
if (r.hasCallback) {
ALOGVV("Camera %s: %s: Shutter fired for frame %d (id %d) at %" PRId64,
diff --git a/services/camera/libcameraservice/device3/Camera3OutputUtils.h b/services/camera/libcameraservice/device3/Camera3OutputUtils.h
index d6107c2..d5328c5 100644
--- a/services/camera/libcameraservice/device3/Camera3OutputUtils.h
+++ b/services/camera/libcameraservice/device3/Camera3OutputUtils.h
@@ -106,6 +106,9 @@
BufferRecordsInterface& bufferRecordsIntf;
bool legacyClient;
nsecs_t& minFrameDuration;
+ bool& isFixedFps;
+ bool overrideToPortrait;
+ std::string &activePhysicalId;
};
void processCaptureResult(CaptureOutputStates& states, const camera_capture_result *result);
diff --git a/services/camera/libcameraservice/device3/Camera3Stream.cpp b/services/camera/libcameraservice/device3/Camera3Stream.cpp
index 7ad6649..88be9ff 100644
--- a/services/camera/libcameraservice/device3/Camera3Stream.cpp
+++ b/services/camera/libcameraservice/device3/Camera3Stream.cpp
@@ -665,11 +665,19 @@
}
}
- // Wait for new buffer returned back if we are running into the limit.
+ // Wait for new buffer returned back if we are running into the limit. There
+ // are 2 limits:
+ // 1. The number of HAL buffers is greater than max_buffers
+ // 2. The number of HAL buffers + cached buffers is greater than max_buffers
+ // + maxCachedBuffers
size_t numOutstandingBuffers = getHandoutOutputBufferCountLocked();
- if (numOutstandingBuffers == camera_stream::max_buffers) {
- ALOGV("%s: Already dequeued max output buffers (%d), wait for next returned one.",
- __FUNCTION__, camera_stream::max_buffers);
+ size_t numCachedBuffers = getCachedOutputBufferCountLocked();
+ size_t maxNumCachedBuffers = getMaxCachedOutputBuffersLocked();
+ while (numOutstandingBuffers == camera_stream::max_buffers ||
+ numOutstandingBuffers + numCachedBuffers ==
+ camera_stream::max_buffers + maxNumCachedBuffers) {
+ ALOGV("%s: Already dequeued max output buffers (%d(+%zu)), wait for next returned one.",
+ __FUNCTION__, camera_stream::max_buffers, maxNumCachedBuffers);
nsecs_t waitStart = systemTime(SYSTEM_TIME_MONOTONIC);
if (waitBufferTimeout < kWaitForBufferDuration) {
waitBufferTimeout = kWaitForBufferDuration;
@@ -687,12 +695,16 @@
}
size_t updatedNumOutstandingBuffers = getHandoutOutputBufferCountLocked();
- if (updatedNumOutstandingBuffers >= numOutstandingBuffers) {
- ALOGE("%s: outsanding buffer count goes from %zu to %zu, "
+ size_t updatedNumCachedBuffers = getCachedOutputBufferCountLocked();
+ if (updatedNumOutstandingBuffers >= numOutstandingBuffers &&
+ updatedNumCachedBuffers == numCachedBuffers) {
+ ALOGE("%s: outstanding buffer count goes from %zu to %zu, "
"getBuffer(s) call must not run in parallel!", __FUNCTION__,
numOutstandingBuffers, updatedNumOutstandingBuffers);
return INVALID_OPERATION;
}
+ numOutstandingBuffers = updatedNumOutstandingBuffers;
+ numCachedBuffers = updatedNumCachedBuffers;
}
res = getBufferLocked(buffer, surface_ids);
@@ -1057,11 +1069,20 @@
}
size_t numOutstandingBuffers = getHandoutOutputBufferCountLocked();
- // Wait for new buffer returned back if we are running into the limit.
- while (numOutstandingBuffers + numBuffersRequested > camera_stream::max_buffers) {
- ALOGV("%s: Already dequeued %zu output buffers and requesting %zu (max is %d), waiting.",
- __FUNCTION__, numOutstandingBuffers, numBuffersRequested,
- camera_stream::max_buffers);
+ size_t numCachedBuffers = getCachedOutputBufferCountLocked();
+ size_t maxNumCachedBuffers = getMaxCachedOutputBuffersLocked();
+ // Wait for new buffer returned back if we are running into the limit. There
+ // are 2 limits:
+ // 1. The number of HAL buffers is greater than max_buffers
+ // 2. The number of HAL buffers + cached buffers is greater than max_buffers
+ // + maxCachedBuffers
+ while (numOutstandingBuffers + numBuffersRequested > camera_stream::max_buffers ||
+ numOutstandingBuffers + numCachedBuffers + numBuffersRequested >
+ camera_stream::max_buffers + maxNumCachedBuffers) {
+ ALOGV("%s: Already dequeued %zu(+%zu) output buffers and requesting %zu "
+ "(max is %d(+%zu)), waiting.", __FUNCTION__, numOutstandingBuffers,
+ numCachedBuffers, numBuffersRequested, camera_stream::max_buffers,
+ maxNumCachedBuffers);
nsecs_t waitStart = systemTime(SYSTEM_TIME_MONOTONIC);
if (waitBufferTimeout < kWaitForBufferDuration) {
waitBufferTimeout = kWaitForBufferDuration;
@@ -1078,13 +1099,16 @@
return res;
}
size_t updatedNumOutstandingBuffers = getHandoutOutputBufferCountLocked();
- if (updatedNumOutstandingBuffers >= numOutstandingBuffers) {
- ALOGE("%s: outsanding buffer count goes from %zu to %zu, "
+ size_t updatedNumCachedBuffers = getCachedOutputBufferCountLocked();
+ if (updatedNumOutstandingBuffers >= numOutstandingBuffers &&
+ updatedNumCachedBuffers == numCachedBuffers) {
+ ALOGE("%s: outstanding buffer count goes from %zu to %zu, "
"getBuffer(s) call must not run in parallel!", __FUNCTION__,
numOutstandingBuffers, updatedNumOutstandingBuffers);
return INVALID_OPERATION;
}
numOutstandingBuffers = updatedNumOutstandingBuffers;
+ numCachedBuffers = updatedNumCachedBuffers;
}
res = getBuffersLocked(buffers);
diff --git a/services/camera/libcameraservice/device3/Camera3Stream.h b/services/camera/libcameraservice/device3/Camera3Stream.h
index d429e6c..214618a 100644
--- a/services/camera/libcameraservice/device3/Camera3Stream.h
+++ b/services/camera/libcameraservice/device3/Camera3Stream.h
@@ -558,6 +558,10 @@
// Get handout input buffer count.
virtual size_t getHandoutInputBufferCountLocked() = 0;
+ // Get cached output buffer count.
+ virtual size_t getCachedOutputBufferCountLocked() const = 0;
+ virtual size_t getMaxCachedOutputBuffersLocked() const = 0;
+
// Get the usage flags for the other endpoint, or return
// INVALID_OPERATION if they cannot be obtained.
virtual status_t getEndpointUsage(uint64_t *usage) const = 0;
@@ -576,6 +580,8 @@
uint64_t mUsage;
+ Condition mOutputBufferReturnedSignal;
+
private:
// Previously configured stream properties (post HAL override)
uint64_t mOldUsage;
@@ -583,7 +589,6 @@
int mOldFormat;
android_dataspace mOldDataSpace;
- Condition mOutputBufferReturnedSignal;
Condition mInputBufferReturnedSignal;
static const nsecs_t kWaitForBufferDuration = 3000000000LL; // 3000 ms
diff --git a/services/camera/libcameraservice/device3/InFlightRequest.h b/services/camera/libcameraservice/device3/InFlightRequest.h
index fa00495..444445b 100644
--- a/services/camera/libcameraservice/device3/InFlightRequest.h
+++ b/services/camera/libcameraservice/device3/InFlightRequest.h
@@ -152,6 +152,9 @@
// For auto-exposure modes, equal to 1/(lower end of target FPS range)
nsecs_t maxExpectedDuration;
+ // Whether the FPS range is fixed, aka, minFps == maxFps
+ bool isFixedFps;
+
// Whether the result metadata for this request is to be skipped. The
// result metadata should be skipped in the case of
// REQUEST/RESULT error.
@@ -205,6 +208,7 @@
hasCallback(true),
minExpectedDuration(kDefaultMinExpectedDuration),
maxExpectedDuration(kDefaultMaxExpectedDuration),
+ isFixedFps(false),
skipResultMetadata(false),
errorBufStrategy(ERROR_BUF_CACHE),
stillCapture(false),
@@ -215,7 +219,7 @@
}
InFlightRequest(int numBuffers, CaptureResultExtras extras, bool hasInput,
- bool hasAppCallback, nsecs_t minDuration, nsecs_t maxDuration,
+ bool hasAppCallback, nsecs_t minDuration, nsecs_t maxDuration, bool fixedFps,
const std::set<std::set<String8>>& physicalCameraIdSet, bool isStillCapture,
bool isZslCapture, bool rotateAndCropAuto, const std::set<std::string>& idsWithZoom,
nsecs_t requestNs, const SurfaceMap& outSurfaces = SurfaceMap{}) :
@@ -229,6 +233,7 @@
hasCallback(hasAppCallback),
minExpectedDuration(minDuration),
maxExpectedDuration(maxDuration),
+ isFixedFps(fixedFps),
skipResultMetadata(false),
errorBufStrategy(ERROR_BUF_CACHE),
physicalCameraIds(physicalCameraIdSet),
diff --git a/services/camera/libcameraservice/device3/PreviewFrameSpacer.cpp b/services/camera/libcameraservice/device3/PreviewFrameSpacer.cpp
index 0439501..83caa00 100644
--- a/services/camera/libcameraservice/device3/PreviewFrameSpacer.cpp
+++ b/services/camera/libcameraservice/device3/PreviewFrameSpacer.cpp
@@ -68,8 +68,10 @@
return true;
}
- // Cache the frame to match readout time interval, for up to 33ms
- nsecs_t expectedQueueTime = mLastCameraPresentTime + readoutInterval;
+ // Cache the frame to match readout time interval, for up to kMaxFrameWaitTime
+ // Because the code between here and queueBuffer() takes time to execute, make sure the
+ // presentationInterval is slightly shorter than readoutInterval.
+ nsecs_t expectedQueueTime = mLastCameraPresentTime + readoutInterval - kFrameAdjustThreshold;
nsecs_t frameWaitTime = std::min(kMaxFrameWaitTime, expectedQueueTime - currentTime);
if (frameWaitTime > 0 && mPendingBuffers.size() < 2) {
mBufferCond.waitRelative(mLock, frameWaitTime);
@@ -78,9 +80,9 @@
}
currentTime = systemTime();
}
- ALOGV("%s: readoutInterval %" PRId64 ", queueInterval %" PRId64 ", waited for %" PRId64
+ ALOGV("%s: readoutInterval %" PRId64 ", waited for %" PRId64
", timestamp %" PRId64, __FUNCTION__, readoutInterval,
- currentTime - mLastCameraPresentTime, frameWaitTime, buffer.timestamp);
+ mPendingBuffers.size() < 2 ? frameWaitTime : 0, buffer.timestamp);
mPendingBuffers.pop();
queueBufferToClientLocked(buffer, currentTime);
return true;
@@ -122,6 +124,7 @@
}
}
+ parent->onCachedBufferQueued();
mLastCameraPresentTime = currentTime;
mLastCameraReadoutTime = bufferHolder.readoutTimestamp;
}
diff --git a/services/camera/libcameraservice/device3/PreviewFrameSpacer.h b/services/camera/libcameraservice/device3/PreviewFrameSpacer.h
index e165768..f46de3d 100644
--- a/services/camera/libcameraservice/device3/PreviewFrameSpacer.h
+++ b/services/camera/libcameraservice/device3/PreviewFrameSpacer.h
@@ -85,7 +85,8 @@
nsecs_t mLastCameraPresentTime = 0;
static constexpr nsecs_t kWaitDuration = 5000000LL; // 50ms
static constexpr nsecs_t kFrameIntervalThreshold = 80000000LL; // 80ms
- static constexpr nsecs_t kMaxFrameWaitTime = 33333333LL; // 33ms
+ static constexpr nsecs_t kMaxFrameWaitTime = 10000000LL; // 10ms
+ static constexpr nsecs_t kFrameAdjustThreshold = 2000000LL; // 2ms
};
}; //namespace camera3
diff --git a/services/camera/libcameraservice/device3/RotateAndCropMapper.cpp b/services/camera/libcameraservice/device3/RotateAndCropMapper.cpp
index a02e5f6..9cdd365 100644
--- a/services/camera/libcameraservice/device3/RotateAndCropMapper.cpp
+++ b/services/camera/libcameraservice/device3/RotateAndCropMapper.cpp
@@ -142,13 +142,13 @@
ch : // pillarbox or 1:1, full height
cw / mRotateAspect; // letterbox, not full height
switch (rotateMode) {
- case ANDROID_SCALER_ROTATE_AND_CROP_90:
+ case ANDROID_SCALER_ROTATE_AND_CROP_270:
transformMat[1] = -rw / ch; // +y -> -x
transformMat[2] = rh / cw; // +x -> +y
xShift = (cw + rw) / 2; // left edge of crop to right edge of rotated
yShift = (ch - rh) / 2; // top edge of crop to top edge of rotated
break;
- case ANDROID_SCALER_ROTATE_AND_CROP_270:
+ case ANDROID_SCALER_ROTATE_AND_CROP_90:
transformMat[1] = rw / ch; // +y -> +x
transformMat[2] = -rh / cw; // +x -> -y
xShift = (cw - rw) / 2; // left edge of crop to left edge of rotated
@@ -271,13 +271,13 @@
rx = cx + (cw - rw) / 2;
ry = cy + (ch - rh) / 2;
switch (rotateMode) {
- case ANDROID_SCALER_ROTATE_AND_CROP_90:
+ case ANDROID_SCALER_ROTATE_AND_CROP_270:
transformMat[1] = ch / rw; // +y -> +x
transformMat[2] = -cw / rh; // +x -> -y
xShift = -(cw - rw) / 2; // left edge of rotated to left edge of cropped
yShift = ry - cy + ch; // top edge of rotated to bottom edge of cropped
break;
- case ANDROID_SCALER_ROTATE_AND_CROP_270:
+ case ANDROID_SCALER_ROTATE_AND_CROP_90:
transformMat[1] = -ch / rw; // +y -> -x
transformMat[2] = cw / rh; // +x -> +y
xShift = (cw + rw) / 2; // left edge of rotated to left edge of cropped
diff --git a/services/camera/libcameraservice/device3/aidl/AidlCamera3Device.cpp b/services/camera/libcameraservice/device3/aidl/AidlCamera3Device.cpp
index f05520f..1bc9ef2 100644
--- a/services/camera/libcameraservice/device3/aidl/AidlCamera3Device.cpp
+++ b/services/camera/libcameraservice/device3/aidl/AidlCamera3Device.cpp
@@ -51,6 +51,7 @@
#include <aidl/android/hardware/camera/device/ICameraInjectionSession.h>
#include <aidlcommonsupport/NativeHandle.h>
+#include <android/binder_ibinder_platform.h>
#include <android/hardware/camera2/ICameraDeviceUser.h>
#include "utils/CameraTraces.h"
@@ -162,8 +163,9 @@
}
AidlCamera3Device::AidlCamera3Device(const String8& id, bool overrideForPerfClass,
- bool legacyClient) : Camera3Device(id, overrideForPerfClass, legacyClient) {
- mCallbacks = ndk::SharedRefBase::make<AidlCameraDeviceCallbacks>(this);
+ bool overrideToPortrait, bool legacyClient) :
+ Camera3Device(id, overrideForPerfClass, overrideToPortrait, legacyClient) {
+ mCallbacks = ndk::SharedRefBase::make<AidlCameraDeviceCallbacks>(this);
}
status_t AidlCamera3Device::initialize(sp<CameraProviderManager> manager,
@@ -192,7 +194,8 @@
SET_ERR("Session iface returned is null");
return INVALID_OPERATION;
}
- res = manager->getCameraCharacteristics(mId.string(), mOverrideForPerfClass, &mDeviceInfo);
+ res = manager->getCameraCharacteristics(mId.string(), mOverrideForPerfClass, &mDeviceInfo,
+ mOverrideToPortrait);
if (res != OK) {
SET_ERR_L("Could not retrieve camera characteristics: %s (%d)", strerror(-res), res);
session->close();
@@ -206,7 +209,8 @@
for (auto& physicalId : physicalCameraIds) {
// Do not override characteristics for physical cameras
res = manager->getCameraCharacteristics(
- physicalId, /*overrideForPerfClass*/false, &mPhysicalDeviceInfoMap[physicalId]);
+ physicalId, /*overrideForPerfClass*/false, &mPhysicalDeviceInfoMap[physicalId],
+ mOverrideToPortrait);
if (res != OK) {
SET_ERR_L("Could not retrieve camera %s characteristics: %s (%d)",
physicalId.c_str(), strerror(-res), res);
@@ -371,7 +375,8 @@
mNumPartialResults, mVendorTagId, mDeviceInfo, mPhysicalDeviceInfoMap,
mDistortionMappers, mZoomRatioMappers, mRotateAndCropMappers,
mTagMonitor, mInputStream, mOutputStreams, mSessionStatsBuilder, listener, *this,
- *this, *(mInterface), mLegacyClient, mMinExpectedDuration}, mResultMetadataQueue
+ *this, *(mInterface), mLegacyClient, mMinExpectedDuration, mIsFixedFps,
+ mOverrideToPortrait, mActivePhysicalId}, mResultMetadataQueue
};
for (const auto& result : results) {
@@ -412,7 +417,8 @@
mNumPartialResults, mVendorTagId, mDeviceInfo, mPhysicalDeviceInfoMap,
mDistortionMappers, mZoomRatioMappers, mRotateAndCropMappers,
mTagMonitor, mInputStream, mOutputStreams, mSessionStatsBuilder, listener, *this,
- *this, *(mInterface), mLegacyClient, mMinExpectedDuration}, mResultMetadataQueue
+ *this, *(mInterface), mLegacyClient, mMinExpectedDuration, mIsFixedFps,
+ mOverrideToPortrait, mActivePhysicalId}, mResultMetadataQueue
};
for (const auto& msg : msgs) {
camera3::notify(states, msg);
@@ -669,6 +675,12 @@
return p->returnStreamBuffers(buffers);
}
+::ndk::SpAIBinder AidlCamera3Device::AidlCameraDeviceCallbacks::createBinder() {
+ auto binder = BnCameraDeviceCallback::createBinder();
+ AIBinder_setInheritRt(binder.get(), /*inheritRt*/ true);
+ return binder;
+}
+
::ndk::ScopedAStatus AidlCamera3Device::returnStreamBuffers(
const std::vector<camera::device::StreamBuffer>& buffers) {
ReturnBufferStates states {
@@ -1399,9 +1411,10 @@
sp<HalInterface> interface,
const Vector<int32_t>& sessionParamKeys,
bool useHalBufManager,
- bool supportCameraMute) :
+ bool supportCameraMute,
+ bool overrideToPortrait) :
RequestThread(parent, statusTracker, interface, sessionParamKeys, useHalBufManager,
- supportCameraMute) {}
+ supportCameraMute, overrideToPortrait) {}
status_t AidlCamera3Device::AidlRequestThread::switchToOffline(
const std::vector<int32_t>& streamsToKeep,
@@ -1570,9 +1583,10 @@
sp<Camera3Device::HalInterface> interface,
const Vector<int32_t>& sessionParamKeys,
bool useHalBufManager,
- bool supportCameraMute) {
+ bool supportCameraMute,
+ bool overrideToPortrait) {
return new AidlRequestThread(parent, statusTracker, interface, sessionParamKeys,
- useHalBufManager, supportCameraMute);
+ useHalBufManager, supportCameraMute, overrideToPortrait);
};
sp<Camera3Device::Camera3DeviceInjectionMethods>
diff --git a/services/camera/libcameraservice/device3/aidl/AidlCamera3Device.h b/services/camera/libcameraservice/device3/aidl/AidlCamera3Device.h
index d20a7eb..630985f 100644
--- a/services/camera/libcameraservice/device3/aidl/AidlCamera3Device.h
+++ b/services/camera/libcameraservice/device3/aidl/AidlCamera3Device.h
@@ -39,7 +39,7 @@
using AidlRequestMetadataQueue = AidlMessageQueue<int8_t, SynchronizedReadWrite>;
class AidlCameraDeviceCallbacks;
friend class AidlCameraDeviceCallbacks;
- explicit AidlCamera3Device(const String8& id, bool overrideForPerfClass,
+ explicit AidlCamera3Device(const String8& id, bool overrideForPerfClass, bool overrideToPortrait,
bool legacyClient = false);
virtual ~AidlCamera3Device() { }
@@ -174,7 +174,8 @@
sp<HalInterface> interface,
const Vector<int32_t>& sessionParamKeys,
bool useHalBufManager,
- bool supportCameraMute);
+ bool supportCameraMute,
+ bool overrideToPortrait);
status_t switchToOffline(
const std::vector<int32_t>& streamsToKeep,
@@ -242,6 +243,10 @@
::ndk::ScopedAStatus returnStreamBuffers(
const std::vector<
aidl::android::hardware::camera::device::StreamBuffer>& buffers) override;
+
+ protected:
+ ::ndk::SpAIBinder createBinder() override;
+
private:
wp<AidlCamera3Device> mParent = nullptr;
};
@@ -255,7 +260,8 @@
sp<HalInterface> interface,
const Vector<int32_t>& sessionParamKeys,
bool useHalBufManager,
- bool supportCameraMute) override;
+ bool supportCameraMute,
+ bool overrideToPortrait) override;
virtual sp<Camera3DeviceInjectionMethods>
createCamera3DeviceInjectionMethods(wp<Camera3Device>) override;
diff --git a/services/camera/libcameraservice/device3/aidl/AidlCamera3OfflineSession.cpp b/services/camera/libcameraservice/device3/aidl/AidlCamera3OfflineSession.cpp
index 336719d..4b1fb1d 100644
--- a/services/camera/libcameraservice/device3/aidl/AidlCamera3OfflineSession.cpp
+++ b/services/camera/libcameraservice/device3/aidl/AidlCamera3OfflineSession.cpp
@@ -30,6 +30,7 @@
#include <utils/Trace.h>
#include <android/hardware/camera2/ICameraDeviceCallbacks.h>
+#include <android/binder_ibinder_platform.h>
#include "device3/aidl/AidlCamera3OfflineSession.h"
#include "device3/Camera3OutputStream.h"
@@ -110,6 +111,7 @@
listener = mListener.promote();
}
+ std::string activePhysicalId(""); // Unused
AidlCaptureOutputStates states {
{mId,
mOfflineReqsLock, mLastCompletedRegularFrameNumber,
@@ -123,7 +125,8 @@
mNumPartialResults, mVendorTagId, mDeviceInfo, mPhysicalDeviceInfoMap,
mDistortionMappers, mZoomRatioMappers, mRotateAndCropMappers,
mTagMonitor, mInputStream, mOutputStreams, mSessionStatsBuilder, listener, *this,
- *this, mBufferRecords, /*legacyClient*/ false, mMinExpectedDuration}, mResultMetadataQueue
+ *this, mBufferRecords, /*legacyClient*/ false, mMinExpectedDuration, mIsFixedFps,
+ /*overrideToPortrait*/false, activePhysicalId}, mResultMetadataQueue
};
std::lock_guard<std::mutex> lock(mProcessCaptureResultLock);
@@ -155,6 +158,7 @@
listener = mListener.promote();
}
+ std::string activePhysicalId(""); // Unused
AidlCaptureOutputStates states {
{mId,
mOfflineReqsLock, mLastCompletedRegularFrameNumber,
@@ -168,7 +172,8 @@
mNumPartialResults, mVendorTagId, mDeviceInfo, mPhysicalDeviceInfoMap,
mDistortionMappers, mZoomRatioMappers, mRotateAndCropMappers,
mTagMonitor, mInputStream, mOutputStreams, mSessionStatsBuilder, listener, *this,
- *this, mBufferRecords, /*legacyClient*/ false, mMinExpectedDuration}, mResultMetadataQueue
+ *this, mBufferRecords, /*legacyClient*/ false, mMinExpectedDuration, mIsFixedFps,
+ /*overrideToPortrait*/false, activePhysicalId}, mResultMetadataQueue
};
for (const auto& msg : msgs) {
camera3::notify(states, msg);
@@ -218,6 +223,12 @@
return p->returnStreamBuffers(buffers);
}
+::ndk::SpAIBinder AidlCamera3OfflineSession::AidlCameraDeviceCallbacks::createBinder() {
+ auto binder = BnCameraDeviceCallback::createBinder();
+ AIBinder_setInheritRt(binder.get(), /*inheritRt*/ true);
+ return binder;
+}
+
::ndk::ScopedAStatus AidlCamera3OfflineSession::returnStreamBuffers(
const std::vector<camera::device::StreamBuffer>& buffers) {
{
diff --git a/services/camera/libcameraservice/device3/aidl/AidlCamera3OfflineSession.h b/services/camera/libcameraservice/device3/aidl/AidlCamera3OfflineSession.h
index 33de2c5..d107af6 100644
--- a/services/camera/libcameraservice/device3/aidl/AidlCamera3OfflineSession.h
+++ b/services/camera/libcameraservice/device3/aidl/AidlCamera3OfflineSession.h
@@ -97,6 +97,10 @@
::ndk::ScopedAStatus returnStreamBuffers(
const std::vector<
aidl::android::hardware::camera::device::StreamBuffer>& buffers) override;
+ protected:
+
+ ::ndk::SpAIBinder createBinder() override;
+
private:
wp<AidlCamera3OfflineSession> mParent = nullptr;
};
diff --git a/services/camera/libcameraservice/device3/hidl/HidlCamera3Device.cpp b/services/camera/libcameraservice/device3/hidl/HidlCamera3Device.cpp
index 4bb426c..c675c63 100644
--- a/services/camera/libcameraservice/device3/hidl/HidlCamera3Device.cpp
+++ b/services/camera/libcameraservice/device3/hidl/HidlCamera3Device.cpp
@@ -162,7 +162,8 @@
return res;
}
- res = manager->getCameraCharacteristics(mId.string(), mOverrideForPerfClass, &mDeviceInfo);
+ res = manager->getCameraCharacteristics(mId.string(), mOverrideForPerfClass, &mDeviceInfo,
+ /*overrideToPortrait*/false);
if (res != OK) {
SET_ERR_L("Could not retrieve camera characteristics: %s (%d)", strerror(-res), res);
session->close();
@@ -176,7 +177,8 @@
for (auto& physicalId : physicalCameraIds) {
// Do not override characteristics for physical cameras
res = manager->getCameraCharacteristics(
- physicalId, /*overrideForPerfClass*/false, &mPhysicalDeviceInfoMap[physicalId]);
+ physicalId, /*overrideForPerfClass*/false, &mPhysicalDeviceInfoMap[physicalId],
+ /*overrideToPortrait*/false);
if (res != OK) {
SET_ERR_L("Could not retrieve camera %s characteristics: %s (%d)",
physicalId.c_str(), strerror(-res), res);
@@ -363,7 +365,8 @@
mNumPartialResults, mVendorTagId, mDeviceInfo, mPhysicalDeviceInfoMap,
mDistortionMappers, mZoomRatioMappers, mRotateAndCropMappers,
mTagMonitor, mInputStream, mOutputStreams, mSessionStatsBuilder, listener, *this, *this,
- *mInterface, mLegacyClient, mMinExpectedDuration}, mResultMetadataQueue
+ *mInterface, mLegacyClient, mMinExpectedDuration, mIsFixedFps, mOverrideToPortrait,
+ mActivePhysicalId}, mResultMetadataQueue
};
//HidlCaptureOutputStates hidlStates {
@@ -425,7 +428,8 @@
mNumPartialResults, mVendorTagId, mDeviceInfo, mPhysicalDeviceInfoMap,
mDistortionMappers, mZoomRatioMappers, mRotateAndCropMappers,
mTagMonitor, mInputStream, mOutputStreams, mSessionStatsBuilder, listener, *this, *this,
- *mInterface, mLegacyClient, mMinExpectedDuration}, mResultMetadataQueue
+ *mInterface, mLegacyClient, mMinExpectedDuration, mIsFixedFps, mOverrideToPortrait,
+ mActivePhysicalId}, mResultMetadataQueue
};
for (const auto& result : results) {
@@ -472,7 +476,8 @@
mNumPartialResults, mVendorTagId, mDeviceInfo, mPhysicalDeviceInfoMap,
mDistortionMappers, mZoomRatioMappers, mRotateAndCropMappers,
mTagMonitor, mInputStream, mOutputStreams, mSessionStatsBuilder, listener, *this, *this,
- *mInterface, mLegacyClient, mMinExpectedDuration}, mResultMetadataQueue
+ *mInterface, mLegacyClient, mMinExpectedDuration, mIsFixedFps, mOverrideToPortrait,
+ mActivePhysicalId}, mResultMetadataQueue
};
for (const auto& msg : msgs) {
camera3::notify(states, msg);
@@ -698,9 +703,10 @@
sp<Camera3Device::HalInterface> interface,
const Vector<int32_t>& sessionParamKeys,
bool useHalBufManager,
- bool supportCameraMute) {
+ bool supportCameraMute,
+ bool overrideToPortrait) {
return new HidlRequestThread(parent, statusTracker, interface, sessionParamKeys,
- useHalBufManager, supportCameraMute);
+ useHalBufManager, supportCameraMute, overrideToPortrait);
};
sp<Camera3Device::Camera3DeviceInjectionMethods>
@@ -1693,9 +1699,10 @@
sp<HalInterface> interface,
const Vector<int32_t>& sessionParamKeys,
bool useHalBufManager,
- bool supportCameraMute) :
+ bool supportCameraMute,
+ bool overrideToPortrait) :
RequestThread(parent, statusTracker, interface, sessionParamKeys, useHalBufManager,
- supportCameraMute) {}
+ supportCameraMute, overrideToPortrait) {}
status_t HidlCamera3Device::HidlRequestThread::switchToOffline(
const std::vector<int32_t>& streamsToKeep,
diff --git a/services/camera/libcameraservice/device3/hidl/HidlCamera3Device.h b/services/camera/libcameraservice/device3/hidl/HidlCamera3Device.h
index 2e98fe0..2bd4660 100644
--- a/services/camera/libcameraservice/device3/hidl/HidlCamera3Device.h
+++ b/services/camera/libcameraservice/device3/hidl/HidlCamera3Device.h
@@ -31,8 +31,9 @@
public Camera3Device {
public:
- explicit HidlCamera3Device(const String8& id, bool overrideForPerfClass,
- bool legacyClient = false) : Camera3Device(id, overrideForPerfClass, legacyClient) { }
+ explicit HidlCamera3Device(const String8& id, bool overrideForPerfClass, bool overrideToPortrait,
+ bool legacyClient = false) : Camera3Device(id, overrideForPerfClass, overrideToPortrait,
+ legacyClient) { }
virtual ~HidlCamera3Device() {}
@@ -175,7 +176,8 @@
sp<HalInterface> interface,
const Vector<int32_t>& sessionParamKeys,
bool useHalBufManager,
- bool supportCameraMute);
+ bool supportCameraMute,
+ bool overrideToPortrait);
status_t switchToOffline(
const std::vector<int32_t>& streamsToKeep,
@@ -222,7 +224,8 @@
sp<HalInterface> interface,
const Vector<int32_t>& sessionParamKeys,
bool useHalBufManager,
- bool supportCameraMute) override;
+ bool supportCameraMute,
+ bool overrideToPortrait) override;
virtual sp<Camera3DeviceInjectionMethods>
createCamera3DeviceInjectionMethods(wp<Camera3Device>) override;
diff --git a/services/camera/libcameraservice/device3/hidl/HidlCamera3OfflineSession.cpp b/services/camera/libcameraservice/device3/hidl/HidlCamera3OfflineSession.cpp
index 5c97f0e..0a6a6f7 100644
--- a/services/camera/libcameraservice/device3/hidl/HidlCamera3OfflineSession.cpp
+++ b/services/camera/libcameraservice/device3/hidl/HidlCamera3OfflineSession.cpp
@@ -92,6 +92,7 @@
listener = mListener.promote();
}
+ std::string activePhysicalId("");
HidlCaptureOutputStates states {
{mId,
mOfflineReqsLock, mLastCompletedRegularFrameNumber,
@@ -105,7 +106,8 @@
mNumPartialResults, mVendorTagId, mDeviceInfo, mPhysicalDeviceInfoMap,
mDistortionMappers, mZoomRatioMappers, mRotateAndCropMappers,
mTagMonitor, mInputStream, mOutputStreams, mSessionStatsBuilder, listener, *this, *this,
- mBufferRecords, /*legacyClient*/ false, mMinExpectedDuration}, mResultMetadataQueue
+ mBufferRecords, /*legacyClient*/ false, mMinExpectedDuration, mIsFixedFps,
+ /*overrideToPortrait*/false, activePhysicalId}, mResultMetadataQueue
};
std::lock_guard<std::mutex> lock(mProcessCaptureResultLock);
@@ -132,6 +134,7 @@
hardware::hidl_vec<hardware::camera::device::V3_4::PhysicalCameraMetadata> noPhysMetadata;
+ std::string activePhysicalId("");
HidlCaptureOutputStates states {
{mId,
mOfflineReqsLock, mLastCompletedRegularFrameNumber,
@@ -145,7 +148,8 @@
mNumPartialResults, mVendorTagId, mDeviceInfo, mPhysicalDeviceInfoMap,
mDistortionMappers, mZoomRatioMappers, mRotateAndCropMappers,
mTagMonitor, mInputStream, mOutputStreams, mSessionStatsBuilder, listener, *this, *this,
- mBufferRecords, /*legacyClient*/ false, mMinExpectedDuration}, mResultMetadataQueue
+ mBufferRecords, /*legacyClient*/ false, mMinExpectedDuration, mIsFixedFps,
+ /*overrideToPortrait*/false, activePhysicalId}, mResultMetadataQueue
};
std::lock_guard<std::mutex> lock(mProcessCaptureResultLock);
@@ -167,6 +171,7 @@
listener = mListener.promote();
}
+ std::string activePhysicalId("");
HidlCaptureOutputStates states {
{mId,
mOfflineReqsLock, mLastCompletedRegularFrameNumber,
@@ -180,7 +185,8 @@
mNumPartialResults, mVendorTagId, mDeviceInfo, mPhysicalDeviceInfoMap,
mDistortionMappers, mZoomRatioMappers, mRotateAndCropMappers,
mTagMonitor, mInputStream, mOutputStreams, mSessionStatsBuilder, listener, *this, *this,
- mBufferRecords, /*legacyClient*/ false, mMinExpectedDuration}, mResultMetadataQueue
+ mBufferRecords, /*legacyClient*/ false, mMinExpectedDuration, mIsFixedFps,
+ /*overrideToPortrait*/false, activePhysicalId}, mResultMetadataQueue
};
for (const auto& msg : msgs) {
camera3::notify(states, msg);
diff --git a/services/camera/libcameraservice/hidl/HidlCameraService.cpp b/services/camera/libcameraservice/hidl/HidlCameraService.cpp
index 65a0300..fc063ab 100644
--- a/services/camera/libcameraservice/hidl/HidlCameraService.cpp
+++ b/services/camera/libcameraservice/hidl/HidlCameraService.cpp
@@ -65,7 +65,8 @@
HStatus status = HStatus::NO_ERROR;
binder::Status serviceRet =
mAidlICameraService->getCameraCharacteristics(String16(cameraId.c_str()),
- /*targetSdkVersion*/__ANDROID_API_FUTURE__, &cameraMetadata);
+ /*targetSdkVersion*/__ANDROID_API_FUTURE__, /*overrideToPortrait*/false,
+ &cameraMetadata);
HCameraMetadata hidlMetadata;
if (!serviceRet.isOk()) {
switch(serviceRet.serviceSpecificErrorCode()) {
@@ -116,7 +117,8 @@
binder::Status serviceRet = mAidlICameraService->connectDevice(
callbacks, String16(cameraId.c_str()), String16(""), {},
hardware::ICameraService::USE_CALLING_UID, 0/*oomScoreOffset*/,
- /*targetSdkVersion*/__ANDROID_API_FUTURE__, /*out*/&deviceRemote);
+ /*targetSdkVersion*/__ANDROID_API_FUTURE__, /*overrideToPortrait*/false,
+ /*out*/&deviceRemote);
HStatus status = HStatus::NO_ERROR;
if (!serviceRet.isOk()) {
ALOGE("%s: Unable to connect to camera device", __FUNCTION__);
diff --git a/services/camera/libcameraservice/hidl/VndkVersionMetadataTags.h b/services/camera/libcameraservice/hidl/VndkVersionMetadataTags.h
index d3377f4..ae4d5dd 100644
--- a/services/camera/libcameraservice/hidl/VndkVersionMetadataTags.h
+++ b/services/camera/libcameraservice/hidl/VndkVersionMetadataTags.h
@@ -31,47 +31,48 @@
std::map<int, std::vector<camera_metadata_tag>> static_api_level_to_keys{
{30, {
ANDROID_CONTROL_AVAILABLE_EXTENDED_SCENE_MODE_MAX_SIZES,
+ ANDROID_CONTROL_AVAILABLE_EXTENDED_SCENE_MODE_ZOOM_RATIO_RANGES,
ANDROID_CONTROL_ZOOM_RATIO_RANGE,
ANDROID_SCALER_AVAILABLE_ROTATE_AND_CROP_MODES,
- ANDROID_CONTROL_AVAILABLE_EXTENDED_SCENE_MODE_ZOOM_RATIO_RANGES,
} },
{31, {
- ANDROID_SENSOR_INFO_ACTIVE_ARRAY_SIZE_MAXIMUM_RESOLUTION,
- ANDROID_SCALER_AVAILABLE_STREAM_CONFIGURATIONS_MAXIMUM_RESOLUTION,
- ANDROID_SENSOR_INFO_PIXEL_ARRAY_SIZE_MAXIMUM_RESOLUTION,
- ANDROID_SCALER_AVAILABLE_MIN_FRAME_DURATIONS_MAXIMUM_RESOLUTION,
- ANDROID_SCALER_AVAILABLE_INPUT_OUTPUT_FORMATS_MAP_MAXIMUM_RESOLUTION,
- ANDROID_SCALER_AVAILABLE_STALL_DURATIONS_MAXIMUM_RESOLUTION,
- ANDROID_HEIC_AVAILABLE_HEIC_STALL_DURATIONS_MAXIMUM_RESOLUTION,
- ANDROID_DEPTH_AVAILABLE_DYNAMIC_DEPTH_STALL_DURATIONS_MAXIMUM_RESOLUTION,
ANDROID_CONTROL_AVAILABLE_HIGH_SPEED_VIDEO_CONFIGURATIONS_MAXIMUM_RESOLUTION,
- ANDROID_DEPTH_AVAILABLE_DEPTH_STALL_DURATIONS_MAXIMUM_RESOLUTION,
- ANDROID_LENS_INTRINSIC_CALIBRATION_MAXIMUM_RESOLUTION,
- ANDROID_DEPTH_AVAILABLE_DYNAMIC_DEPTH_STREAM_CONFIGURATIONS_MAXIMUM_RESOLUTION,
- ANDROID_HEIC_AVAILABLE_HEIC_STREAM_CONFIGURATIONS_MAXIMUM_RESOLUTION,
- ANDROID_SCALER_PHYSICAL_CAMERA_MULTI_RESOLUTION_STREAM_CONFIGURATIONS,
- ANDROID_DEPTH_AVAILABLE_DEPTH_STREAM_CONFIGURATIONS_MAXIMUM_RESOLUTION,
- ANDROID_SCALER_MULTI_RESOLUTION_STREAM_SUPPORTED,
- ANDROID_DEPTH_AVAILABLE_DYNAMIC_DEPTH_MIN_FRAME_DURATIONS_MAXIMUM_RESOLUTION,
- ANDROID_SENSOR_INFO_PRE_CORRECTION_ACTIVE_ARRAY_SIZE_MAXIMUM_RESOLUTION,
ANDROID_DEPTH_AVAILABLE_DEPTH_MIN_FRAME_DURATIONS_MAXIMUM_RESOLUTION,
- ANDROID_LENS_DISTORTION_MAXIMUM_RESOLUTION,
- ANDROID_SCALER_DEFAULT_SECURE_IMAGE_SIZE,
+ ANDROID_DEPTH_AVAILABLE_DEPTH_STALL_DURATIONS_MAXIMUM_RESOLUTION,
+ ANDROID_DEPTH_AVAILABLE_DEPTH_STREAM_CONFIGURATIONS_MAXIMUM_RESOLUTION,
+ ANDROID_DEPTH_AVAILABLE_DYNAMIC_DEPTH_MIN_FRAME_DURATIONS_MAXIMUM_RESOLUTION,
+ ANDROID_DEPTH_AVAILABLE_DYNAMIC_DEPTH_STALL_DURATIONS_MAXIMUM_RESOLUTION,
+ ANDROID_DEPTH_AVAILABLE_DYNAMIC_DEPTH_STREAM_CONFIGURATIONS_MAXIMUM_RESOLUTION,
ANDROID_HEIC_AVAILABLE_HEIC_MIN_FRAME_DURATIONS_MAXIMUM_RESOLUTION,
- ANDROID_SENSOR_OPAQUE_RAW_SIZE_MAXIMUM_RESOLUTION,
+ ANDROID_HEIC_AVAILABLE_HEIC_STALL_DURATIONS_MAXIMUM_RESOLUTION,
+ ANDROID_HEIC_AVAILABLE_HEIC_STREAM_CONFIGURATIONS_MAXIMUM_RESOLUTION,
+ ANDROID_LENS_DISTORTION_MAXIMUM_RESOLUTION,
+ ANDROID_LENS_INTRINSIC_CALIBRATION_MAXIMUM_RESOLUTION,
+ ANDROID_SCALER_AVAILABLE_INPUT_OUTPUT_FORMATS_MAP_MAXIMUM_RESOLUTION,
+ ANDROID_SCALER_AVAILABLE_MIN_FRAME_DURATIONS_MAXIMUM_RESOLUTION,
+ ANDROID_SCALER_AVAILABLE_STALL_DURATIONS_MAXIMUM_RESOLUTION,
+ ANDROID_SCALER_AVAILABLE_STREAM_CONFIGURATIONS_MAXIMUM_RESOLUTION,
+ ANDROID_SCALER_DEFAULT_SECURE_IMAGE_SIZE,
+ ANDROID_SCALER_MULTI_RESOLUTION_STREAM_SUPPORTED,
+ ANDROID_SCALER_PHYSICAL_CAMERA_MULTI_RESOLUTION_STREAM_CONFIGURATIONS,
+ ANDROID_SENSOR_INFO_ACTIVE_ARRAY_SIZE_MAXIMUM_RESOLUTION,
ANDROID_SENSOR_INFO_BINNING_FACTOR,
+ ANDROID_SENSOR_INFO_PIXEL_ARRAY_SIZE_MAXIMUM_RESOLUTION,
+ ANDROID_SENSOR_INFO_PRE_CORRECTION_ACTIVE_ARRAY_SIZE_MAXIMUM_RESOLUTION,
+ ANDROID_SENSOR_OPAQUE_RAW_SIZE_MAXIMUM_RESOLUTION,
} },
{32, {
ANDROID_INFO_DEVICE_STATE_ORIENTATIONS,
} },
{33, {
- ANDROID_FLASH_INFO_STRENGTH_DEFAULT_LEVEL,
ANDROID_AUTOMOTIVE_LENS_FACING,
ANDROID_AUTOMOTIVE_LOCATION,
+ ANDROID_FLASH_INFO_STRENGTH_DEFAULT_LEVEL,
+ ANDROID_FLASH_INFO_STRENGTH_MAXIMUM_LEVEL,
+ ANDROID_REQUEST_AVAILABLE_DYNAMIC_RANGE_PROFILES_MAP,
ANDROID_REQUEST_RECOMMENDED_TEN_BIT_DYNAMIC_RANGE_PROFILE,
ANDROID_SCALER_AVAILABLE_STREAM_USE_CASES,
- ANDROID_REQUEST_AVAILABLE_DYNAMIC_RANGE_PROFILES_MAP,
- ANDROID_FLASH_INFO_STRENGTH_MAXIMUM_LEVEL,
+ ANDROID_SENSOR_READOUT_TIMESTAMP,
} },
};
@@ -81,9 +82,9 @@
*/
std::map<int, std::vector<camera_metadata_tag>> dynamic_api_level_to_keys{
{30, {
+ ANDROID_CONTROL_EXTENDED_SCENE_MODE,
ANDROID_CONTROL_ZOOM_RATIO,
ANDROID_SCALER_ROTATE_AND_CROP,
- ANDROID_CONTROL_EXTENDED_SCENE_MODE,
} },
{31, {
ANDROID_SENSOR_PIXEL_MODE,
diff --git a/services/camera/libcameraservice/libcameraservice_fuzzer/camera_service_fuzzer.cpp b/services/camera/libcameraservice/libcameraservice_fuzzer/camera_service_fuzzer.cpp
index 97d7bf4..120d43d 100644
--- a/services/camera/libcameraservice/libcameraservice_fuzzer/camera_service_fuzzer.cpp
+++ b/services/camera/libcameraservice/libcameraservice_fuzzer/camera_service_fuzzer.cpp
@@ -229,11 +229,11 @@
mCameraService->getCameraVendorTagCache(&cache);
CameraInfo cameraInfo;
- mCameraService->getCameraInfo(cameraId, &cameraInfo);
+ mCameraService->getCameraInfo(cameraId, /*overrideToPortrait*/false, &cameraInfo);
CameraMetadata metadata;
mCameraService->getCameraCharacteristics(cameraIdStr,
- /*targetSdkVersion*/__ANDROID_API_FUTURE__, &metadata);
+ /*targetSdkVersion*/__ANDROID_API_FUTURE__, /*overrideToPortrait*/false, &metadata);
}
void CameraFuzzer::invokeCameraSound() {
@@ -320,7 +320,9 @@
rc = mCameraService->connect(this, cameraId, String16(),
android::CameraService::USE_CALLING_UID, android::CameraService::USE_CALLING_PID,
- /*targetSdkVersion*/__ANDROID_API_FUTURE__, &cameraDevice);
+ /*targetSdkVersion*/__ANDROID_API_FUTURE__, /*overrideToPortrait*/true,
+ /*forceSlowJpegMode*/false,
+ &cameraDevice);
if (!rc.isOk()) {
// camera not connected
return;
@@ -534,7 +536,8 @@
sp<hardware::camera2::ICameraDeviceUser> device;
mCameraService->connectDevice(callbacks, String16(s.cameraId), String16(), {},
android::CameraService::USE_CALLING_UID, 0/*oomScoreDiff*/,
- /*targetSdkVersion*/__ANDROID_API_FUTURE__, &device);
+ /*targetSdkVersion*/__ANDROID_API_FUTURE__, /*overrideToPortrait*/true,
+ &device);
if (device == nullptr) {
continue;
}
diff --git a/services/camera/libcameraservice/tests/CameraProviderManagerTest.cpp b/services/camera/libcameraservice/tests/CameraProviderManagerTest.cpp
index e9f6979..2f55def 100644
--- a/services/camera/libcameraservice/tests/CameraProviderManagerTest.cpp
+++ b/services/camera/libcameraservice/tests/CameraProviderManagerTest.cpp
@@ -102,23 +102,57 @@
sp<device::V3_2::ICameraDevice> mDeviceInterface;
hardware::hidl_vec<common::V1_0::VendorTagSection> mVendorTagSections;
+ // Whether to call a physical camera unavailable callback upon setCallback
+ bool mHasPhysicalCameraUnavailableCallback;
+ hardware::hidl_string mLogicalCameraId;
+ hardware::hidl_string mUnavailablePhysicalCameraId;
+
TestICameraProvider(const std::vector<hardware::hidl_string> &devices,
const hardware::hidl_vec<common::V1_0::VendorTagSection> &vendorSection) :
mDeviceNames(devices),
mDeviceInterface(new TestDeviceInterface(devices)),
- mVendorTagSections (vendorSection) {}
+ mVendorTagSections (vendorSection),
+ mHasPhysicalCameraUnavailableCallback(false) {}
TestICameraProvider(const std::vector<hardware::hidl_string> &devices,
const hardware::hidl_vec<common::V1_0::VendorTagSection> &vendorSection,
android::hardware::hidl_vec<uint8_t> chars) :
mDeviceNames(devices),
mDeviceInterface(new TestDeviceInterface(devices, chars)),
- mVendorTagSections (vendorSection) {}
+ mVendorTagSections (vendorSection),
+ mHasPhysicalCameraUnavailableCallback(false) {}
+
+ TestICameraProvider(const std::vector<hardware::hidl_string> &devices,
+ const hardware::hidl_vec<common::V1_0::VendorTagSection> &vendorSection,
+ android::hardware::hidl_vec<uint8_t> chars,
+ const hardware::hidl_string& logicalCameraId,
+ const hardware::hidl_string& unavailablePhysicalCameraId) :
+ mDeviceNames(devices),
+ mDeviceInterface(new TestDeviceInterface(devices, chars)),
+ mVendorTagSections (vendorSection),
+ mHasPhysicalCameraUnavailableCallback(true),
+ mLogicalCameraId(logicalCameraId),
+ mUnavailablePhysicalCameraId(unavailablePhysicalCameraId) {}
virtual hardware::Return<Status> setCallback(
const sp<provider::V2_4::ICameraProviderCallback>& callbacks) override {
mCalledCounter[SET_CALLBACK]++;
mCallbacks = callbacks;
+ if (mHasPhysicalCameraUnavailableCallback) {
+ auto cast26 = provider::V2_6::ICameraProviderCallback::castFrom(callbacks);
+ if (!cast26.isOk()) {
+ ADD_FAILURE() << "Failed to cast ICameraProviderCallback to V2_6";
+ } else {
+ sp<provider::V2_6::ICameraProviderCallback> callback26 = cast26;
+ if (callback26 == nullptr) {
+ ADD_FAILURE() << "V2_6::ICameraProviderCallback is null after conversion";
+ } else {
+ callback26->physicalCameraDeviceStatusChange(mLogicalCameraId,
+ mUnavailablePhysicalCameraId,
+ android::hardware::camera::common::V1_0::CameraDeviceStatus::NOT_PRESENT);
+ }
+ }
+ }
return hardware::Return<Status>(Status::OK);
}
@@ -266,12 +300,16 @@
};
struct TestStatusListener : public CameraProviderManager::StatusListener {
+ int mPhysicalCameraStatusChangeCount = 0;
+
~TestStatusListener() {}
void onDeviceStatusChanged(const String8 &,
CameraDeviceStatus) override {}
void onDeviceStatusChanged(const String8 &, const String8 &,
- CameraDeviceStatus) override {}
+ CameraDeviceStatus) override {
+ mPhysicalCameraStatusChangeCount++;
+ }
void onTorchStatusChanged(const String8 &,
TorchModeStatus) override {}
void onTorchStatusChanged(const String8 &,
@@ -634,3 +672,46 @@
ASSERT_EQ(deviceCount, deviceNames.size()) <<
"Unexpected amount of camera devices";
}
+
+// Test that CameraProviderManager does not trigger
+// onDeviceStatusChanged(NOT_PRESENT) for physical camera before initialize()
+// returns.
+TEST(CameraProviderManagerTest, PhysicalCameraAvailabilityCallbackRaceTest) {
+ std::vector<hardware::hidl_string> deviceNames;
+ deviceNames.push_back("device@3.2/test/0");
+ hardware::hidl_vec<common::V1_0::VendorTagSection> vendorSection;
+
+ sp<CameraProviderManager> providerManager = new CameraProviderManager();
+ sp<TestStatusListener> statusListener = new TestStatusListener();
+ TestInteractionProxy serviceProxy;
+
+ android::hardware::hidl_vec<uint8_t> chars;
+ CameraMetadata meta;
+ int32_t charKeys[] = { ANDROID_REQUEST_AVAILABLE_CAPABILITIES };
+ meta.update(ANDROID_REQUEST_AVAILABLE_CHARACTERISTICS_KEYS, charKeys,
+ sizeof(charKeys) / sizeof(charKeys[0]));
+ uint8_t capabilities[] = { ANDROID_REQUEST_AVAILABLE_CAPABILITIES_LOGICAL_MULTI_CAMERA };
+ meta.update(ANDROID_REQUEST_AVAILABLE_CAPABILITIES, capabilities,
+ sizeof(capabilities)/sizeof(capabilities[0]));
+ uint8_t physicalCameraIds[] = { '2', '\0', '3', '\0' };
+ meta.update(ANDROID_LOGICAL_MULTI_CAMERA_PHYSICAL_IDS, physicalCameraIds,
+ sizeof(physicalCameraIds)/sizeof(physicalCameraIds[0]));
+ camera_metadata_t* metaBuffer = const_cast<camera_metadata_t*>(meta.getAndLock());
+ chars.setToExternal(reinterpret_cast<uint8_t*>(metaBuffer),
+ get_camera_metadata_size(metaBuffer));
+
+ sp<TestICameraProvider> provider = new TestICameraProvider(deviceNames,
+ vendorSection, chars, "device@3.2/test/0", "2");
+ serviceProxy.setProvider(provider);
+
+ status_t res = providerManager->initialize(statusListener, &serviceProxy);
+ ASSERT_EQ(res, OK) << "Unable to initialize provider manager";
+
+ ASSERT_EQ(statusListener->mPhysicalCameraStatusChangeCount, 0)
+ << "Unexpected physical camera status change callback upon provider init.";
+
+ std::unordered_map<std::string, std::set<std::string>> unavailablePhysicalIds;
+ auto cameraIds = providerManager->getCameraDeviceIds(&unavailablePhysicalIds);
+ ASSERT_TRUE(unavailablePhysicalIds.count("0") > 0 && unavailablePhysicalIds["0"].count("2") > 0)
+ << "Unavailable physical camera Ids not set properly.";
+}
diff --git a/services/camera/libcameraservice/tests/RotateAndCropMapperTest.cpp b/services/camera/libcameraservice/tests/RotateAndCropMapperTest.cpp
index 3c187cd..9f86526 100644
--- a/services/camera/libcameraservice/tests/RotateAndCropMapperTest.cpp
+++ b/services/camera/libcameraservice/tests/RotateAndCropMapperTest.cpp
@@ -195,6 +195,7 @@
// Round-trip results can't be exact since we've gone from a large int range -> small int range
// and back, leading to quantization. For 4/3 aspect ratio, no more than +-1 error expected
+
e = result.find(ANDROID_CONTROL_AE_REGIONS);
EXPECT_EQUAL_WITHIN_N(full_region, e.data.i32, 1, "Round-tripped AE region isn't right");
@@ -209,11 +210,11 @@
EXPECT_EQUAL_WITHIN_N(full_face, e.data.i32, 1, "App-side face rectangle isn't right");
auto full_landmarks = std::vector<int32_t> {
- full_crop[0], full_crop[1] + full_crop[3],
full_crop[0] + full_crop[2], full_crop[1],
- full_crop[0] + full_crop[2]/4, full_crop[1] + 3*full_crop[3]/4,
+ full_crop[0], full_crop[1] + full_crop[3],
+ full_crop[0] + 3*full_crop[2]/4, full_crop[1] + full_crop[3]/4,
full_crop[0] + full_crop[2]/2, full_crop[1] + full_crop[3]/2,
- full_crop[0] + 3*full_crop[2]/4, full_crop[1] + full_crop[3]/4
+ full_crop[0] + full_crop[2]/4, full_crop[1] + 3*full_crop[3]/4
};
e = result.find(ANDROID_STATISTICS_FACE_LANDMARKS);
EXPECT_EQUAL_WITHIN_N(full_landmarks, e.data.i32, 1, "App-side face landmarks aren't right");
@@ -286,7 +287,6 @@
// Round-trip results can't be exact since we've gone from a large int range -> small int range
// and back, leading to quantization. For 4/3 aspect ratio, no more than +-1 error expected
-
e = result.find(ANDROID_CONTROL_AE_REGIONS);
EXPECT_EQUAL_WITHIN_N(full_region, e.data.i32, 1, "Round-tripped AE region isn't right");
@@ -301,11 +301,11 @@
EXPECT_EQUAL_WITHIN_N(full_face, e.data.i32, 1, "App-side face rectangle isn't right");
auto full_landmarks = std::vector<int32_t> {
- full_crop[0] + full_crop[2], full_crop[1],
full_crop[0], full_crop[1] + full_crop[3],
- full_crop[0] + 3*full_crop[2]/4, full_crop[1] + full_crop[3]/4,
+ full_crop[0] + full_crop[2], full_crop[1],
+ full_crop[0] + full_crop[2]/4, full_crop[1] + 3*full_crop[3]/4,
full_crop[0] + full_crop[2]/2, full_crop[1] + full_crop[3]/2,
- full_crop[0] + full_crop[2]/4, full_crop[1] + 3*full_crop[3]/4
+ full_crop[0] + 3*full_crop[2]/4, full_crop[1] + full_crop[3]/4
};
e = result.find(ANDROID_STATISTICS_FACE_LANDMARKS);
EXPECT_EQUAL_WITHIN_N(full_landmarks, e.data.i32, 1, "App-side face landmarks aren't right");
diff --git a/services/camera/libcameraservice/utils/CameraServiceProxyWrapper.cpp b/services/camera/libcameraservice/utils/CameraServiceProxyWrapper.cpp
index 69175cc..dae5eea 100644
--- a/services/camera/libcameraservice/utils/CameraServiceProxyWrapper.cpp
+++ b/services/camera/libcameraservice/utils/CameraServiceProxyWrapper.cpp
@@ -262,11 +262,11 @@
sessionStats->onClose(latencyMs);
}
-bool CameraServiceProxyWrapper::isCameraDisabled() {
+bool CameraServiceProxyWrapper::isCameraDisabled(int userId) {
sp<ICameraServiceProxy> proxyBinder = getCameraServiceProxy();
if (proxyBinder == nullptr) return true;
bool ret = false;
- auto status = proxyBinder->isCameraDisabled(&ret);
+ auto status = proxyBinder->isCameraDisabled(userId, &ret);
if (!status.isOk()) {
ALOGE("%s: Failed during camera disabled query: %s", __FUNCTION__,
status.exceptionMessage().c_str());
diff --git a/services/camera/libcameraservice/utils/CameraServiceProxyWrapper.h b/services/camera/libcameraservice/utils/CameraServiceProxyWrapper.h
index e34a8f0..eb818d1 100644
--- a/services/camera/libcameraservice/utils/CameraServiceProxyWrapper.h
+++ b/services/camera/libcameraservice/utils/CameraServiceProxyWrapper.h
@@ -97,7 +97,7 @@
static int getRotateAndCropOverride(String16 packageName, int lensFacing, int userId);
// Detect if the camera is disabled by device policy.
- static bool isCameraDisabled();
+ static bool isCameraDisabled(int userId);
};
} // android
diff --git a/services/mediametrics/AudioAnalytics.cpp b/services/mediametrics/AudioAnalytics.cpp
index 99e3691..b03e418 100644
--- a/services/mediametrics/AudioAnalytics.cpp
+++ b/services/mediametrics/AudioAnalytics.cpp
@@ -240,6 +240,35 @@
"sharing_requested",
};
+static constexpr const char * HeadTrackerDeviceEnabledFields[] {
+ "mediametrics_headtrackerdeviceenabled_reported",
+ "type",
+ "event",
+ "enabled",
+};
+
+static constexpr const char * HeadTrackerDeviceSupportedFields[] {
+ "mediametrics_headtrackerdevicesupported_reported",
+ "type",
+ "event",
+ "supported",
+};
+
+static constexpr const char * SpatializerCapabilitiesFields[] {
+ "mediametrics_spatializer_reported",
+ "head_tracking_modes",
+ "spatializer_levels",
+ "spatializer_modes",
+ "channel_masks",
+};
+
+static constexpr const char * SpatializerDeviceEnabledFields[] {
+ "mediametrics_spatializerdeviceenabled_reported",
+ "type",
+ "event",
+ "enabled",
+};
+
/**
* printFields is a helper method that prints the fields and corresponding values
* in a human readable style.
@@ -459,6 +488,15 @@
[this](const std::shared_ptr<const android::mediametrics::Item> &item){
mAudioPowerUsage.checkCreatePatch(item);
}));
+
+ // Handle Spatializer - these keys are prefixed by "audio.spatializer."
+ mActions.addAction(
+ AMEDIAMETRICS_KEY_PREFIX_AUDIO_SPATIALIZER "*." AMEDIAMETRICS_PROP_EVENT,
+ std::monostate{}, /* match any event */
+ std::make_shared<AnalyticsActions::Function>(
+ [this](const std::shared_ptr<const android::mediametrics::Item> &item){
+ mSpatializer.onEvent(item);
+ }));
}
AudioAnalytics::~AudioAnalytics()
@@ -750,15 +788,9 @@
int32_t frameCount = 0;
mAudioAnalytics.mAnalyticsState->timeMachine().get(
key, AMEDIAMETRICS_PROP_FRAMECOUNT, &frameCount);
- std::string inputDevicePairs;
- mAudioAnalytics.mAnalyticsState->timeMachine().get(
- key, AMEDIAMETRICS_PROP_INPUTDEVICES, &inputDevicePairs);
int32_t intervalCount = 0;
mAudioAnalytics.mAnalyticsState->timeMachine().get(
key, AMEDIAMETRICS_PROP_INTERVALCOUNT, &intervalCount);
- std::string outputDevicePairs;
- mAudioAnalytics.mAnalyticsState->timeMachine().get(
- key, AMEDIAMETRICS_PROP_OUTPUTDEVICES, &outputDevicePairs);
int32_t sampleRate = 0;
mAudioAnalytics.mAnalyticsState->timeMachine().get(
key, AMEDIAMETRICS_PROP_SAMPLERATE, &sampleRate);
@@ -766,53 +798,16 @@
mAudioAnalytics.mAnalyticsState->timeMachine().get(
key, AMEDIAMETRICS_PROP_FLAGS, &flags);
- // We may have several devices.
- // Accumulate the bit flags for input and output devices.
- std::stringstream oss;
- long_enum_type_t outputDeviceBits{};
- { // compute outputDevices
- const auto devaddrvec = stringutils::getDeviceAddressPairs(outputDevicePairs);
- for (const auto& [device, addr] : devaddrvec) {
- if (oss.tellp() > 0) oss << "|"; // delimit devices with '|'.
- oss << device;
- outputDeviceBits += types::lookup<types::OUTPUT_DEVICE, long_enum_type_t>(device);
- }
- }
- const std::string outputDevices = oss.str();
-
- std::stringstream iss;
- long_enum_type_t inputDeviceBits{};
- { // compute inputDevices
- const auto devaddrvec = stringutils::getDeviceAddressPairs(inputDevicePairs);
- for (const auto& [device, addr] : devaddrvec) {
- if (iss.tellp() > 0) iss << "|"; // delimit devices with '|'.
- iss << device;
- inputDeviceBits += types::lookup<types::INPUT_DEVICE, long_enum_type_t>(device);
- }
- }
- const std::string inputDevices = iss.str();
-
- // Get connected device name if from bluetooth.
- bool isBluetooth = false;
-
- std::string inputDeviceNames; // not filled currently.
- std::string outputDeviceNames;
- if (outputDevices.find("AUDIO_DEVICE_OUT_BLUETOOTH") != std::string::npos) {
- isBluetooth = true;
- outputDeviceNames = SUPPRESSED;
-#if 0 // TODO(b/161554630) sanitize name
- mAudioAnalytics.mAnalyticsState->timeMachine().get(
- "audio.device.bt_a2dp", AMEDIAMETRICS_PROP_NAME, &outputDeviceNames);
- // Remove | if present
- stringutils::replace(outputDeviceNames, "|", '?');
- if (outputDeviceNames.size() > STATSD_DEVICE_NAME_MAX_LENGTH) {
- outputDeviceNames.resize(STATSD_DEVICE_NAME_MAX_LENGTH); // truncate
- }
-#endif
- }
-
switch (itemType) {
case RECORD: {
+ std::string inputDevicePairs;
+ mAudioAnalytics.mAnalyticsState->timeMachine().get(
+ key, AMEDIAMETRICS_PROP_INPUTDEVICES, &inputDevicePairs);
+
+ const auto [ inputDeviceStatsd, inputDevices ] =
+ stringutils::parseInputDevicePairs(inputDevicePairs);
+ const std::string inputDeviceNames; // not filled currently.
+
std::string callerName;
const bool clientCalled = mAudioAnalytics.mAnalyticsState->timeMachine().get(
key, AMEDIAMETRICS_PROP_CALLERNAME, &callerName) == OK;
@@ -848,7 +843,7 @@
LOG(LOG_LEVEL) << "key:" << key
<< " id:" << id
- << " inputDevices:" << inputDevices << "(" << inputDeviceBits
+ << " inputDevices:" << inputDevices << "(" << inputDeviceStatsd
<< ") inputDeviceNames:" << inputDeviceNames
<< " deviceTimeNs:" << deviceTimeNs
<< " encoding:" << encoding << "(" << encodingForStats
@@ -866,7 +861,7 @@
&& mAudioAnalytics.mDeliverStatistics) {
const auto [ result, str ] = sendToStatsd(AudioRecordDeviceUsageFields,
CONDITION(android::util::MEDIAMETRICS_AUDIORECORDDEVICEUSAGE_REPORTED)
- , ENUM_EXTRACT(inputDeviceBits)
+ , ENUM_EXTRACT(inputDeviceStatsd)
, inputDeviceNames.c_str()
, deviceTimeNs
, ENUM_EXTRACT(encodingForStats)
@@ -895,18 +890,35 @@
key, AMEDIAMETRICS_PROP_UNDERRUN, &underrun);
const bool isInput = types::isInputThreadType(type);
+
+ // get device information
+ std::string devicePairs;
+ std::string deviceStatsd;
+ std::string devices;
+ std::string deviceNames;
+ if (isInput) {
+ // Note we get the "last" device which is the one associated with group.
+ item->get(AMEDIAMETRICS_PROP_PREFIX_LAST AMEDIAMETRICS_PROP_INPUTDEVICES,
+ &devicePairs);
+ std::tie(deviceStatsd, devices) = stringutils::parseInputDevicePairs(devicePairs);
+ } else {
+ // Note we get the "last" device which is the one associated with group.
+ item->get(AMEDIAMETRICS_PROP_PREFIX_LAST AMEDIAMETRICS_PROP_OUTPUTDEVICES,
+ &devicePairs);
+ std::tie(deviceStatsd, devices) = stringutils::parseOutputDevicePairs(devicePairs);
+ deviceNames = mAudioAnalytics.getDeviceNamesFromOutputDevices(devices);
+ }
+
const auto encodingForStats = types::lookup<types::ENCODING, short_enum_type_t>(encoding);
const auto flagsForStats =
(isInput ? types::lookup<types::INPUT_FLAG, short_enum_type_t>(flags)
: types::lookup<types::OUTPUT_FLAG, short_enum_type_t>(flags));
const auto typeForStats = types::lookup<types::THREAD_TYPE, short_enum_type_t>(type);
- LOG(LOG_LEVEL) << "key:" << key
+ LOG(LOG_LEVEL) << "key:" << key
<< " id:" << id
- << " inputDevices:" << inputDevices << "(" << inputDeviceBits
- << ") outputDevices:" << outputDevices << "(" << outputDeviceBits
- << ") inputDeviceNames:" << inputDeviceNames
- << " outputDeviceNames:" << outputDeviceNames
+ << " devices:" << devices << "(" << deviceStatsd
+ << ") deviceNames:" << deviceNames
<< " deviceTimeNs:" << deviceTimeNs
<< " encoding:" << encoding << "(" << encodingForStats
<< ") frameCount:" << frameCount
@@ -919,8 +931,8 @@
if (mAudioAnalytics.mDeliverStatistics) {
const auto [ result, str ] = sendToStatsd(AudioThreadDeviceUsageFields,
CONDITION(android::util::MEDIAMETRICS_AUDIOTHREADDEVICEUSAGE_REPORTED)
- , isInput ? ENUM_EXTRACT(inputDeviceBits) : ENUM_EXTRACT(outputDeviceBits)
- , isInput ? inputDeviceNames.c_str() : outputDeviceNames.c_str()
+ , ENUM_EXTRACT(deviceStatsd)
+ , deviceNames.c_str()
, deviceTimeNs
, ENUM_EXTRACT(encodingForStats)
, frameCount
@@ -936,6 +948,15 @@
}
} break;
case TRACK: {
+ std::string outputDevicePairs;
+ mAudioAnalytics.mAnalyticsState->timeMachine().get(
+ key, AMEDIAMETRICS_PROP_OUTPUTDEVICES, &outputDevicePairs);
+
+ const auto [ outputDeviceStatsd, outputDevices ] =
+ stringutils::parseOutputDevicePairs(outputDevicePairs);
+ const std::string outputDeviceNames =
+ mAudioAnalytics.getDeviceNamesFromOutputDevices(outputDevices);
+
std::string callerName;
const bool clientCalled = mAudioAnalytics.mAnalyticsState->timeMachine().get(
key, AMEDIAMETRICS_PROP_CALLERNAME, &callerName) == OK;
@@ -1003,7 +1024,7 @@
LOG(LOG_LEVEL) << "key:" << key
<< " id:" << id
- << " outputDevices:" << outputDevices << "(" << outputDeviceBits
+ << " outputDevices:" << outputDevices << "(" << outputDeviceStatsd
<< ") outputDeviceNames:" << outputDeviceNames
<< " deviceTimeNs:" << deviceTimeNs
<< " encoding:" << encoding << "(" << encodingForStats
@@ -1030,7 +1051,7 @@
&& mAudioAnalytics.mDeliverStatistics) {
const auto [ result, str ] = sendToStatsd(AudioTrackDeviceUsageFields,
CONDITION(android::util::MEDIAMETRICS_AUDIOTRACKDEVICEUSAGE_REPORTED)
- , ENUM_EXTRACT(outputDeviceBits)
+ , ENUM_EXTRACT(outputDeviceStatsd)
, outputDeviceNames.c_str()
, deviceTimeNs
, ENUM_EXTRACT(encodingForStats)
@@ -1057,11 +1078,6 @@
}
} break;
}
-
- // Report this as needed.
- if (isBluetooth) {
- // report this for Bluetooth
- }
}
// DeviceConnection helper class.
@@ -1525,5 +1541,191 @@
return { s, n };
}
+// Classifies the setting event for statsd (use generated statsd enums.proto constants).
+static int32_t classifySettingEvent(bool isSetAlready, bool withinBoot) {
+ if (isSetAlready) {
+ return util::MEDIAMETRICS_SPATIALIZER_DEVICE_ENABLED_REPORTED__EVENT__SPATIALIZER_SETTING_EVENT_NORMAL;
+ }
+ if (withinBoot) {
+ return util::MEDIAMETRICS_SPATIALIZER_DEVICE_ENABLED_REPORTED__EVENT__SPATIALIZER_SETTING_EVENT_BOOT;
+ }
+ return util::MEDIAMETRICS_SPATIALIZER_DEVICE_ENABLED_REPORTED__EVENT__SPATIALIZER_SETTING_EVENT_FIRST;
+}
+
+void AudioAnalytics::Spatializer::onEvent(
+ const std::shared_ptr<const android::mediametrics::Item> &item)
+{
+ const auto key = item->getKey();
+
+ if (!startsWith(key, AMEDIAMETRICS_KEY_PREFIX_AUDIO_SPATIALIZER)) return;
+
+ const std::string suffix =
+ key.substr(std::size(AMEDIAMETRICS_KEY_PREFIX_AUDIO_SPATIALIZER) - 1);
+
+ std::string eventStr; // optional - find the actual event string.
+ (void)item->get(AMEDIAMETRICS_PROP_EVENT, &eventStr);
+
+ const size_t delim = suffix.find('.'); // note could use split.
+ if (delim == suffix.npos) {
+ // on create with suffix == "0" for the first spatializer effect.
+
+ std::string headTrackingModes;
+ (void)item->get(AMEDIAMETRICS_PROP_HEADTRACKINGMODES, &headTrackingModes);
+
+ std::string levels;
+ (void)item->get(AMEDIAMETRICS_PROP_LEVELS, &levels);
+
+ std::string modes;
+ (void)item->get(AMEDIAMETRICS_PROP_MODES, &modes);
+
+ std::string channelMasks;
+ (void)item->get(AMEDIAMETRICS_PROP_CHANNELMASKS, &channelMasks);
+
+ LOG(LOG_LEVEL) << "key:" << key
+ << " headTrackingModes:" << headTrackingModes
+ << " levels:" << levels
+ << " modes:" << modes
+ << " channelMasks:" << channelMasks
+ ;
+
+ const std::vector<int32_t> headTrackingModesVector =
+ types::vectorFromMap(headTrackingModes, types::getHeadTrackingModeMap());
+ const std::vector<int32_t> levelsVector =
+ types::vectorFromMap(levels, types::getSpatializerLevelMap());
+ const std::vector<int32_t> modesVector =
+ types::vectorFromMap(modes, types::getSpatializerModeMap());
+ const std::vector<int64_t> channelMasksVector =
+ types::channelMaskVectorFromString(channelMasks);
+
+ const auto [ result, str ] = sendToStatsd(SpatializerCapabilitiesFields,
+ CONDITION(android::util::MEDIAMETRICS_SPATIALIZERCAPABILITIES_REPORTED)
+ , headTrackingModesVector
+ , levelsVector
+ , modesVector
+ , channelMasksVector
+ );
+
+ mAudioAnalytics.mStatsdLog->log(
+ android::util::MEDIAMETRICS_SPATIALIZERCAPABILITIES_REPORTED, str);
+
+ std::lock_guard lg(mLock);
+ if (mFirstCreateTimeNs == 0) {
+ // Only update the create time once to prevent audioserver restart
+ // from looking like a boot.
+ mFirstCreateTimeNs = item->getTimestamp();
+ }
+ mSimpleLog.log("%s suffix: %s item: %s",
+ __func__, suffix.c_str(), item->toString().c_str());
+ } else {
+ std::string subtype = suffix.substr(0, delim);
+ if (subtype != "device") return; // not a device.
+
+ const std::string deviceType = suffix.substr(std::size("device.") - 1);
+
+ const int32_t deviceTypeStatsd =
+ types::lookup<types::AUDIO_DEVICE_INFO_TYPE, int32_t>(deviceType);
+
+ std::string address;
+ (void)item->get(AMEDIAMETRICS_PROP_ADDRESS, &address);
+ std::string enabled;
+ (void)item->get(AMEDIAMETRICS_PROP_ENABLED, &enabled);
+ std::string hasHeadTracker;
+ (void)item->get(AMEDIAMETRICS_PROP_HASHEADTRACKER, &hasHeadTracker);
+ std::string headTrackerEnabled;
+ (void)item->get(AMEDIAMETRICS_PROP_HEADTRACKERENABLED, &headTrackerEnabled);
+
+ std::lock_guard lg(mLock);
+
+ // Validate from our cached state
+
+ // Our deviceKey takes the device type and appends the address if any.
+ // This distinguishes different wireless devices for the purposes of tracking.
+ std::string deviceKey(deviceType);
+ deviceKey.append("_").append(address);
+ DeviceState& deviceState = mDeviceStateMap[deviceKey];
+
+ // check whether the settings event is within a certain time of spatializer creation.
+ const bool withinBoot =
+ item->getTimestamp() - mFirstCreateTimeNs < kBootDurationThreshold;
+
+ if (!enabled.empty()) {
+ if (enabled != deviceState.enabled) {
+ const int32_t settingEventStatsd =
+ classifySettingEvent(!deviceState.enabled.empty(), withinBoot);
+ deviceState.enabled = enabled;
+ const bool enabledStatsd = enabled == "true";
+ const auto [ result, str ] = sendToStatsd(SpatializerDeviceEnabledFields,
+ CONDITION(android::util::MEDIAMETRICS_SPATIALIZERDEVICEENABLED_REPORTED)
+ , deviceTypeStatsd
+ , settingEventStatsd
+ , enabledStatsd
+ );
+ mAudioAnalytics.mStatsdLog->log(
+ android::util::MEDIAMETRICS_SPATIALIZERDEVICEENABLED_REPORTED, str);
+ }
+ }
+ if (!hasHeadTracker.empty()) {
+ if (hasHeadTracker != deviceState.hasHeadTracker) {
+ const int32_t settingEventStatsd =
+ classifySettingEvent(!deviceState.hasHeadTracker.empty(), withinBoot);
+ deviceState.hasHeadTracker = hasHeadTracker;
+ const bool supportedStatsd = hasHeadTracker == "true";
+ const auto [ result, str ] = sendToStatsd(HeadTrackerDeviceSupportedFields,
+ CONDITION(android::util::MEDIAMETRICS_HEADTRACKERDEVICESUPPORTED_REPORTED)
+ , deviceTypeStatsd
+ , settingEventStatsd
+ , supportedStatsd
+ );
+ mAudioAnalytics.mStatsdLog->log(
+ android::util::MEDIAMETRICS_HEADTRACKERDEVICESUPPORTED_REPORTED, str);
+ }
+ }
+ if (!headTrackerEnabled.empty()) {
+ if (headTrackerEnabled != deviceState.headTrackerEnabled) {
+ const int32_t settingEventStatsd =
+ classifySettingEvent(!deviceState.headTrackerEnabled.empty(), withinBoot);
+ deviceState.headTrackerEnabled = headTrackerEnabled;
+ const bool enabledStatsd = headTrackerEnabled == "true";
+ const auto [ result, str ] = sendToStatsd(HeadTrackerDeviceEnabledFields,
+ CONDITION(android::util::MEDIAMETRICS_HEADTRACKERDEVICEENABLED_REPORTED)
+ , deviceTypeStatsd
+ , settingEventStatsd
+ , enabledStatsd
+ );
+ mAudioAnalytics.mStatsdLog->log(
+ android::util::MEDIAMETRICS_HEADTRACKERDEVICEENABLED_REPORTED, str);
+ }
+ }
+ mSimpleLog.log("%s deviceKey: %s item: %s",
+ __func__, deviceKey.c_str(), item->toString().c_str());
+ }
+}
+
+std::pair<std::string, int32_t> AudioAnalytics::Spatializer::dump(
+ int32_t lines, const char *prefix) const
+{
+ std::lock_guard lg(mLock);
+ std::string s = mSimpleLog.dumpToString(prefix == nullptr ? "" : prefix, lines);
+ size_t n = std::count(s.begin(), s.end(), '\n');
+ return { s, n };
+}
+
+// This method currently suppresses the name.
+std::string AudioAnalytics::getDeviceNamesFromOutputDevices(std::string_view devices) const {
+ std::string deviceNames;
+ if (stringutils::hasBluetoothOutputDevice(devices)) {
+ deviceNames = SUPPRESSED;
+#if 0 // TODO(b/161554630) sanitize name
+ mAudioAnalytics.mAnalyticsState->timeMachine().get(
+ "audio.device.bt_a2dp", AMEDIAMETRICS_PROP_NAME, &deviceNames);
+ // Remove | if present
+ stringutils::replace(deviceNames, "|", '?');
+ if (deviceNames.size() > STATSD_DEVICE_NAME_MAX_LENGTH) {
+ deviceNames.resize(STATSD_DEVICE_NAME_MAX_LENGTH); // truncate
+ }
+#endif
+ }
+ return deviceNames;
+}
} // namespace android::mediametrics
diff --git a/services/mediametrics/AudioTypes.cpp b/services/mediametrics/AudioTypes.cpp
index 6e24a58..d2b4aab 100644
--- a/services/mediametrics/AudioTypes.cpp
+++ b/services/mediametrics/AudioTypes.cpp
@@ -135,6 +135,94 @@
return map;
}
+// A map for the Java AudioDeviceInfo types to internal (native) output devices.
+const std::unordered_map<std::string, int32_t>& getAudioDeviceOutCompactMap() {
+ // DO NOT MODIFY VALUES (OK to add new ones).
+ static std::unordered_map<std::string, int32_t> map{
+ // should "unknown" go to AUDIO_DEVICE_NONE?
+ {"earpiece", AUDIO_DEVICE_OUT_EARPIECE},
+ {"speaker", AUDIO_DEVICE_OUT_SPEAKER},
+ {"headset", AUDIO_DEVICE_OUT_WIRED_HEADSET},
+ {"headphone", AUDIO_DEVICE_OUT_WIRED_HEADPHONE},
+ {"bt_sco", AUDIO_DEVICE_OUT_BLUETOOTH_SCO},
+ {"bt_sco_hs", AUDIO_DEVICE_OUT_BLUETOOTH_SCO_HEADSET},
+ {"bt_sco_carkit", AUDIO_DEVICE_OUT_BLUETOOTH_SCO_CARKIT},
+ {"bt_a2dp", AUDIO_DEVICE_OUT_BLUETOOTH_A2DP},
+ {"bt_a2dp_hp", AUDIO_DEVICE_OUT_BLUETOOTH_A2DP_HEADPHONES},
+ {"bt_a2dp_spk", AUDIO_DEVICE_OUT_BLUETOOTH_A2DP_SPEAKER},
+ {"aux_digital", AUDIO_DEVICE_OUT_AUX_DIGITAL},
+ {"hdmi", AUDIO_DEVICE_OUT_HDMI},
+ {"analog_dock", AUDIO_DEVICE_OUT_ANLG_DOCK_HEADSET},
+ {"digital_dock", AUDIO_DEVICE_OUT_DGTL_DOCK_HEADSET},
+ {"usb_accessory", AUDIO_DEVICE_OUT_USB_ACCESSORY},
+ {"usb_device", AUDIO_DEVICE_OUT_USB_DEVICE},
+ {"remote_submix", AUDIO_DEVICE_OUT_REMOTE_SUBMIX},
+ {"telephony_tx", AUDIO_DEVICE_OUT_TELEPHONY_TX},
+ {"line", AUDIO_DEVICE_OUT_LINE},
+ {"hdmi_arc", AUDIO_DEVICE_OUT_HDMI_ARC},
+ {"hdmi_earc", AUDIO_DEVICE_OUT_HDMI_EARC},
+ {"spdif", AUDIO_DEVICE_OUT_SPDIF},
+ {"fm_transmitter", AUDIO_DEVICE_OUT_FM},
+ {"aux_line", AUDIO_DEVICE_OUT_AUX_LINE},
+ {"speaker_safe", AUDIO_DEVICE_OUT_SPEAKER_SAFE},
+ {"ip", AUDIO_DEVICE_OUT_IP},
+ {"bus", AUDIO_DEVICE_OUT_BUS},
+ {"proxy", AUDIO_DEVICE_OUT_PROXY},
+ {"usb_headset", AUDIO_DEVICE_OUT_USB_HEADSET},
+ {"hearing_aid_out", AUDIO_DEVICE_OUT_HEARING_AID},
+ {"echo_canceller", AUDIO_DEVICE_OUT_ECHO_CANCELLER},
+ // default does not exist
+ {"ble_headset", AUDIO_DEVICE_OUT_BLE_HEADSET},
+ {"ble_speaker", AUDIO_DEVICE_OUT_BLE_SPEAKER},
+ {"ble_broadcast", AUDIO_DEVICE_OUT_BLE_BROADCAST},
+ };
+ return map;
+}
+
+// A map for the Java AudioDeviceInfo types.
+// This uses generated statsd enums.proto constants.
+const std::unordered_map<std::string, int32_t>& getAudioDeviceInfoTypeMap() {
+ // DO NOT MODIFY VALUES (OK to add new ones).
+ static std::unordered_map<std::string, int32_t> map{
+ {"unknown", util::MEDIAMETRICS_SPATIALIZER_DEVICE_ENABLED_REPORTED__TYPE__AUDIO_DEVICE_INFO_TYPE_UNKNOWN},
+ {"earpiece", util::MEDIAMETRICS_SPATIALIZER_DEVICE_ENABLED_REPORTED__TYPE__AUDIO_DEVICE_INFO_TYPE_BUILTIN_EARPIECE},
+ {"speaker", util::MEDIAMETRICS_SPATIALIZER_DEVICE_ENABLED_REPORTED__TYPE__AUDIO_DEVICE_INFO_TYPE_BUILTIN_SPEAKER},
+ {"headset", util::MEDIAMETRICS_SPATIALIZER_DEVICE_ENABLED_REPORTED__TYPE__AUDIO_DEVICE_INFO_TYPE_WIRED_HEADSET},
+ {"headphone", util::MEDIAMETRICS_SPATIALIZER_DEVICE_ENABLED_REPORTED__TYPE__AUDIO_DEVICE_INFO_TYPE_WIRED_HEADPHONES}, // sic
+ {"bt_sco", util::MEDIAMETRICS_SPATIALIZER_DEVICE_ENABLED_REPORTED__TYPE__AUDIO_DEVICE_INFO_TYPE_BLUETOOTH_SCO},
+ {"bt_sco_hs", util::MEDIAMETRICS_SPATIALIZER_DEVICE_ENABLED_REPORTED__TYPE__AUDIO_DEVICE_INFO_TYPE_BLUETOOTH_SCO},
+ {"bt_sco_carkit", util::MEDIAMETRICS_SPATIALIZER_DEVICE_ENABLED_REPORTED__TYPE__AUDIO_DEVICE_INFO_TYPE_BLUETOOTH_SCO},
+ {"bt_a2dp", util::MEDIAMETRICS_SPATIALIZER_DEVICE_ENABLED_REPORTED__TYPE__AUDIO_DEVICE_INFO_TYPE_BLUETOOTH_A2DP},
+ {"bt_a2dp_hp", util::MEDIAMETRICS_SPATIALIZER_DEVICE_ENABLED_REPORTED__TYPE__AUDIO_DEVICE_INFO_TYPE_BLUETOOTH_A2DP},
+ {"bt_a2dp_spk", util::MEDIAMETRICS_SPATIALIZER_DEVICE_ENABLED_REPORTED__TYPE__AUDIO_DEVICE_INFO_TYPE_BLUETOOTH_A2DP},
+ {"aux_digital", util::MEDIAMETRICS_SPATIALIZER_DEVICE_ENABLED_REPORTED__TYPE__AUDIO_DEVICE_INFO_TYPE_HDMI},
+ {"hdmi", util::MEDIAMETRICS_SPATIALIZER_DEVICE_ENABLED_REPORTED__TYPE__AUDIO_DEVICE_INFO_TYPE_HDMI},
+ {"analog_dock", util::MEDIAMETRICS_SPATIALIZER_DEVICE_ENABLED_REPORTED__TYPE__AUDIO_DEVICE_INFO_TYPE_DOCK},
+ {"digital_dock", util::MEDIAMETRICS_SPATIALIZER_DEVICE_ENABLED_REPORTED__TYPE__AUDIO_DEVICE_INFO_TYPE_DOCK},
+ {"usb_accessory", util::MEDIAMETRICS_SPATIALIZER_DEVICE_ENABLED_REPORTED__TYPE__AUDIO_DEVICE_INFO_TYPE_USB_ACCESSORY},
+ {"usb_device", util::MEDIAMETRICS_SPATIALIZER_DEVICE_ENABLED_REPORTED__TYPE__AUDIO_DEVICE_INFO_TYPE_USB_DEVICE},
+ {"usb_headset", util::MEDIAMETRICS_SPATIALIZER_DEVICE_ENABLED_REPORTED__TYPE__AUDIO_DEVICE_INFO_TYPE_USB_HEADSET},
+ {"remote_submix", util::MEDIAMETRICS_SPATIALIZER_DEVICE_ENABLED_REPORTED__TYPE__AUDIO_DEVICE_INFO_TYPE_REMOTE_SUBMIX},
+ {"telephony_tx", util::MEDIAMETRICS_SPATIALIZER_DEVICE_ENABLED_REPORTED__TYPE__AUDIO_DEVICE_INFO_TYPE_TELEPHONY},
+ {"line", util::MEDIAMETRICS_SPATIALIZER_DEVICE_ENABLED_REPORTED__TYPE__AUDIO_DEVICE_INFO_TYPE_LINE_ANALOG},
+ {"hdmi_arc", util::MEDIAMETRICS_SPATIALIZER_DEVICE_ENABLED_REPORTED__TYPE__AUDIO_DEVICE_INFO_TYPE_HDMI_ARC},
+ {"hdmi_earc", util::MEDIAMETRICS_SPATIALIZER_DEVICE_ENABLED_REPORTED__TYPE__AUDIO_DEVICE_INFO_TYPE_HDMI_EARC},
+ {"spdif", util::MEDIAMETRICS_SPATIALIZER_DEVICE_ENABLED_REPORTED__TYPE__AUDIO_DEVICE_INFO_TYPE_LINE_DIGITAL},
+ {"fm_transmitter", util::MEDIAMETRICS_SPATIALIZER_DEVICE_ENABLED_REPORTED__TYPE__AUDIO_DEVICE_INFO_TYPE_FM},
+ {"aux_line", util::MEDIAMETRICS_SPATIALIZER_DEVICE_ENABLED_REPORTED__TYPE__AUDIO_DEVICE_INFO_TYPE_AUX_LINE},
+ {"speaker_safe", util::MEDIAMETRICS_SPATIALIZER_DEVICE_ENABLED_REPORTED__TYPE__AUDIO_DEVICE_INFO_TYPE_BUILTIN_SPEAKER_SAFE},
+ {"ip", util::MEDIAMETRICS_SPATIALIZER_DEVICE_ENABLED_REPORTED__TYPE__AUDIO_DEVICE_INFO_TYPE_IP},
+ {"bus", util::MEDIAMETRICS_SPATIALIZER_DEVICE_ENABLED_REPORTED__TYPE__AUDIO_DEVICE_INFO_TYPE_BUS},
+ {"proxy", util::MEDIAMETRICS_SPATIALIZER_DEVICE_ENABLED_REPORTED__TYPE__AUDIO_DEVICE_INFO_TYPE_UNKNOWN /* AUDIO_DEVICE_INFO_TYPE_PROXY */},
+ {"hearing_aid_out", util::MEDIAMETRICS_SPATIALIZER_DEVICE_ENABLED_REPORTED__TYPE__AUDIO_DEVICE_INFO_TYPE_HEARING_AID},
+ {"echo_canceller", util::MEDIAMETRICS_SPATIALIZER_DEVICE_ENABLED_REPORTED__TYPE__AUDIO_DEVICE_INFO_TYPE_ECHO_REFERENCE}, // sic
+ {"ble_headset", util::MEDIAMETRICS_SPATIALIZER_DEVICE_ENABLED_REPORTED__TYPE__AUDIO_DEVICE_INFO_TYPE_BLE_HEADSET},
+ {"ble_speaker", util::MEDIAMETRICS_SPATIALIZER_DEVICE_ENABLED_REPORTED__TYPE__AUDIO_DEVICE_INFO_TYPE_BLE_SPEAKER},
+ {"ble_broadcast", util::MEDIAMETRICS_SPATIALIZER_DEVICE_ENABLED_REPORTED__TYPE__AUDIO_DEVICE_INFO_TYPE_BLE_BROADCAST},
+ };
+ return map;
+}
+
const std::unordered_map<std::string, int32_t>& getAudioThreadTypeMap() {
// DO NOT MODIFY VALUES (OK to add new ones).
// This may be found in frameworks/av/services/audioflinger/Threads.h
@@ -197,6 +285,41 @@
return map;
}
+const std::unordered_map<std::string, int32_t>& getHeadTrackingModeMap() {
+ // DO NOT MODIFY VALUES(OK to add new ones).
+ // frameworks/base/media/java/android/media/Spatializer.java
+ // frameworks/av/media/libaudioclient/aidl/android/media/SpatializerHeadTrackingMode.aidl
+ static std::unordered_map<std::string, int32_t> map {
+ {"OTHER", 0},
+ {"DISABLED", -1},
+ {"RELATIVE_WORLD", 1},
+ {"RELATIVE_SCREEN", 2},
+ };
+ return map;
+}
+
+const std::unordered_map<std::string, int32_t>& getSpatializerLevelMap() {
+ // DO NOT MODIFY VALUES(OK to add new ones).
+ // frameworks/base/media/java/android/media/Spatializer.java
+ // frameworks/av/media/libaudioclient/aidl/android/media/SpatializerHeadTrackingMode.aidl
+ static std::unordered_map<std::string, int32_t> map {
+ {"NONE", 0},
+ {"SPATIALIZER_MULTICHANNEL", 1},
+ {"SPATIALIZER_MCHAN_BED_PLUS_OBJECTS", 2},
+ };
+ return map;
+}
+
+const std::unordered_map<std::string, int32_t>& getSpatializerModeMap() {
+ // DO NOT MODIFY VALUES(OK to add new ones).
+ // frameworks/av/media/libaudioclient/aidl/android/media/SpatializationMode.aidl
+ static std::unordered_map<std::string, int32_t> map {
+ {"SPATIALIZER_BINAURAL", 0},
+ {"SPATIALIZER_TRANSAURAL", 1},
+ };
+ return map;
+}
+
const std::unordered_map<std::string, int32_t>& getStatusMap() {
// DO NOT MODIFY VALUES(OK to add new ones).
static std::unordered_map<std::string, int32_t> map {
@@ -286,6 +409,35 @@
return value;
}
+std::vector<int32_t> vectorFromMap(
+ const std::string &str, const std::unordered_map<std::string, int32_t>& map)
+{
+ std::vector<int32_t> v;
+
+ if (str.empty()) return v;
+
+ const auto result = stringutils::split(str, "|");
+ for (const auto &s : result) {
+ auto it = map.find(s);
+ if (it == map.end()) continue;
+ v.push_back(it->second);
+ }
+ return v;
+}
+
+std::vector<int64_t> channelMaskVectorFromString(const std::string &s)
+{
+ std::vector<int64_t> v;
+
+ const auto result = stringutils::split(s, "|");
+ for (const auto &mask : result) {
+ // 0 if undetected or if actually 0.
+ int64_t int64Mask = strtoll(mask.c_str(), nullptr, 0);
+ v.push_back(int64Mask);
+ }
+ return v;
+}
+
template <>
int32_t lookup<CONTENT_TYPE>(const std::string &contentType)
{
@@ -441,6 +593,17 @@
}
template <>
+int32_t lookup<AUDIO_DEVICE_INFO_TYPE>(const std::string& audioDeviceInfoType)
+{
+ auto& map = getAudioDeviceInfoTypeMap();
+ auto it = map.find(audioDeviceInfoType);
+ if (it == map.end()) {
+ return 0;
+ }
+ return it->second;
+}
+
+template <>
int32_t lookup<CALLER_NAME>(const std::string &callerName)
{
auto& map = getAudioCallerNameMap();
@@ -463,6 +626,39 @@
}
template <>
+int32_t lookup<HEAD_TRACKING_MODE>(const std::string& headTrackingMode)
+{
+ auto& map = getHeadTrackingModeMap();
+ auto it = map.find(headTrackingMode);
+ if (it == map.end()) {
+ return 0;
+ }
+ return it->second;
+}
+
+template <>
+int32_t lookup<SPATIALIZER_LEVEL>(const std::string& spatializerLevel)
+{
+ auto& map = getSpatializerLevelMap();
+ auto it = map.find(spatializerLevel);
+ if (it == map.end()) {
+ return 0;
+ }
+ return it->second;
+}
+
+template <>
+int32_t lookup<SPATIALIZER_MODE>(const std::string& spatializerMode)
+{
+ auto& map = getSpatializerModeMap();
+ auto it = map.find(spatializerMode);
+ if (it == map.end()) {
+ return 0;
+ }
+ return it->second;
+}
+
+template <>
int32_t lookup<STATUS>(const std::string &status)
{
auto& map = getStatusMap();
diff --git a/services/mediametrics/MediaMetricsService.cpp b/services/mediametrics/MediaMetricsService.cpp
index ff16b9e..8a7e8a9 100644
--- a/services/mediametrics/MediaMetricsService.cpp
+++ b/services/mediametrics/MediaMetricsService.cpp
@@ -336,6 +336,15 @@
result << "-- some lines may be truncated --\n";
}
+ const int32_t spatializerLinesToDump = all ? INT32_MAX : 15;
+ result << "\nSpatializer Message Log:";
+ const auto [ spatializerDumpString, spatializerLines ] =
+ mAudioAnalytics.dumpSpatializer(spatializerLinesToDump);
+ result << "\n" << spatializerDumpString;
+ if (spatializerLines == spatializerLinesToDump) {
+ result << "-- some lines may be truncated --\n";
+ }
+
result << "\nLogSessionId:\n"
<< mediametrics::ValidateId::get()->dump();
diff --git a/services/mediametrics/StringUtils.cpp b/services/mediametrics/StringUtils.cpp
index 50525bc..d1c7a18 100644
--- a/services/mediametrics/StringUtils.cpp
+++ b/services/mediametrics/StringUtils.cpp
@@ -20,6 +20,8 @@
#include "StringUtils.h"
+#include "AudioTypes.h"
+
namespace android::mediametrics::stringutils {
std::string tokenizer(std::string::const_iterator& it,
@@ -99,4 +101,30 @@
return replaced;
}
+template <types::AudioEnumCategory CATEGORY>
+std::pair<std::string /* external statsd */, std::string /* internal */>
+parseDevicePairs(const std::string& devicePairs) {
+ std::pair<std::string, std::string> result{};
+ const auto devaddrvec = stringutils::getDeviceAddressPairs(devicePairs);
+ for (const auto& [device, addr] : devaddrvec) { // addr ignored for now.
+ if (!result.second.empty()) {
+ result.second.append("|"); // delimit devices with '|'.
+ result.first.append("|");
+ }
+ result.second.append(device);
+ result.first.append(types::lookup<CATEGORY, std::string>(device));
+ }
+ return result;
+}
+
+std::pair<std::string /* external statsd */, std::string /* internal */>
+parseOutputDevicePairs(const std::string& devicePairs) {
+ return parseDevicePairs<types::OUTPUT_DEVICE>(devicePairs);
+}
+
+std::pair<std::string /* external statsd */, std::string /* internal */>
+parseInputDevicePairs(const std::string& devicePairs) {
+ return parseDevicePairs<types::INPUT_DEVICE>(devicePairs);
+}
+
} // namespace android::mediametrics::stringutils
diff --git a/services/mediametrics/include/mediametricsservice/AudioAnalytics.h b/services/mediametrics/include/mediametricsservice/AudioAnalytics.h
index 5ee8c30..82e928e 100644
--- a/services/mediametrics/include/mediametricsservice/AudioAnalytics.h
+++ b/services/mediametrics/include/mediametricsservice/AudioAnalytics.h
@@ -92,6 +92,15 @@
return mHealth.dump(lines);
}
+ /**
+ * Returns a pair consisting of the dump string and the number of lines in the string.
+ *
+ * Spatializer dump.
+ */
+ std::pair<std::string, int32_t> dumpSpatializer(int32_t lines = INT32_MAX) const {
+ return mSpatializer.dump(lines);
+ }
+
void clear() {
// underlying state is locked.
mPreviousAnalyticsState->clear();
@@ -152,6 +161,13 @@
*/
std::string getThreadFromTrack(const std::string& track) const;
+ /**
+ * return the device name, if present.
+ *
+ * This is currently enabled only for Bluetooth output devices.
+ */
+ std::string getDeviceNamesFromOutputDevices(std::string_view devices) const;
+
const bool mDeliverStatistics;
// Actions is individually locked
@@ -317,6 +333,36 @@
SimpleLog mSimpleLog GUARDED_BY(mLock) {64};
} mHealth{*this};
+ // Spatializer is a nested class that tracks related messages.
+ class Spatializer {
+ public:
+ explicit Spatializer(AudioAnalytics &audioAnalytics)
+ : mAudioAnalytics(audioAnalytics) {}
+
+ // an item that starts with "audio.spatializer"
+ void onEvent(const std::shared_ptr<const android::mediametrics::Item> &item);
+
+ std::pair<std::string, int32_t> dump(
+ int32_t lines = INT32_MAX, const char *prefix = nullptr) const;
+
+ private:
+
+ // Current device state as strings:
+ // "" means unknown, "true" or "false".
+ struct DeviceState {
+ std::string enabled;
+ std::string hasHeadTracker;
+ std::string headTrackerEnabled;
+ };
+
+ AudioAnalytics& mAudioAnalytics;
+ static constexpr int64_t kBootDurationThreshold = 120 /* seconds */ * 1e9;
+ mutable std::mutex mLock;
+ int64_t mFirstCreateTimeNs GUARDED_BY(mLock) = 0;
+ std::map<std::string, DeviceState> mDeviceStateMap GUARDED_BY(mLock);
+ SimpleLog mSimpleLog GUARDED_BY(mLock) {64};
+ } mSpatializer{*this};
+
AudioPowerUsage mAudioPowerUsage;
};
diff --git a/services/mediametrics/include/mediametricsservice/AudioTypes.h b/services/mediametrics/include/mediametricsservice/AudioTypes.h
index 5dbff9b..b5fe28b 100644
--- a/services/mediametrics/include/mediametricsservice/AudioTypes.h
+++ b/services/mediametrics/include/mediametricsservice/AudioTypes.h
@@ -29,6 +29,14 @@
const std::unordered_map<std::string, int64_t>& getAudioDeviceOutMap();
const std::unordered_map<std::string, int32_t>& getAudioThreadTypeMap();
const std::unordered_map<std::string, int32_t>& getAudioTrackTraitsMap();
+const std::unordered_map<std::string, int32_t>& getHeadTrackingModeMap();
+const std::unordered_map<std::string, int32_t>& getSpatializerLevelMap();
+const std::unordered_map<std::string, int32_t>& getSpatializerModeMap();
+
+std::vector<int32_t> vectorFromMap(
+ const std::string &str, const std::unordered_map<std::string, int32_t>& map);
+
+std::vector<int64_t> channelMaskVectorFromString(const std::string &s);
// Enumeration for the device connection results.
enum DeviceConnectionResult : int32_t {
@@ -47,14 +55,18 @@
AAUDIO_DIRECTION,
AAUDIO_PERFORMANCE_MODE,
AAUDIO_SHARING_MODE,
+ AUDIO_DEVICE_INFO_TYPE,
CALLER_NAME,
CONTENT_TYPE,
ENCODING,
+ HEAD_TRACKING_MODE,
INPUT_DEVICE, // int64_t
INPUT_FLAG,
OUTPUT_DEVICE, // int64_t
OUTPUT_FLAG,
SOURCE_TYPE,
+ SPATIALIZER_LEVEL,
+ SPATIALIZER_MODE,
STATUS,
STREAM_TYPE,
THREAD_TYPE,
diff --git a/services/mediametrics/include/mediametricsservice/MediaMetricsService.h b/services/mediametrics/include/mediametricsservice/MediaMetricsService.h
index 8d0b1cf..3ec5ac7 100644
--- a/services/mediametrics/include/mediametricsservice/MediaMetricsService.h
+++ b/services/mediametrics/include/mediametricsservice/MediaMetricsService.h
@@ -125,7 +125,7 @@
std::atomic<int64_t> mItemsSubmitted{}; // accessed outside of lock.
// mStatsdLog is locked internally (thread-safe) and shows the last atoms logged
- static constexpr size_t STATSD_LOG_LINES_MAX = 30; // recent log lines to keep
+ static constexpr size_t STATSD_LOG_LINES_MAX = 48; // recent log lines to keep
static constexpr size_t STATSD_LOG_LINES_DUMP = 4; // normal amount of lines to dump
const std::shared_ptr<mediametrics::StatsdLog> mStatsdLog{
std::make_shared<mediametrics::StatsdLog>(STATSD_LOG_LINES_MAX)};
diff --git a/services/mediametrics/include/mediametricsservice/StringUtils.h b/services/mediametrics/include/mediametricsservice/StringUtils.h
index a56f5b8..78c25ff 100644
--- a/services/mediametrics/include/mediametricsservice/StringUtils.h
+++ b/services/mediametrics/include/mediametricsservice/StringUtils.h
@@ -23,6 +23,19 @@
namespace android::mediametrics::stringutils {
+// Define a way of printing a vector - this
+// is used for proto repeated arguments.
+template <typename T>
+inline std::ostream & operator<< (std::ostream& s,
+ std::vector<T> const& v) {
+ s << "{ ";
+ for (const auto& e : v) {
+ s << e << " ";
+ }
+ s << "}";
+ return s;
+}
+
/**
* fieldPrint is a helper method that logs to a stringstream a sequence of
* field names (in a fixed size array) together with a variable number of arg parameters.
@@ -204,4 +217,14 @@
return { key, "" };
}
+std::pair<std::string /* external statsd */, std::string /* internal */>
+parseOutputDevicePairs(const std::string& outputDevicePairs);
+
+std::pair<std::string /* external statsd */, std::string /* internal */>
+parseInputDevicePairs(const std::string& inputDevicePairs);
+
+inline bool hasBluetoothOutputDevice(std::string_view devices) {
+ return devices.find("AUDIO_DEVICE_OUT_BLUETOOTH") != std::string::npos;
+}
+
} // namespace android::mediametrics::stringutils
diff --git a/services/oboeservice/AAudioServiceEndpointMMAP.cpp b/services/oboeservice/AAudioServiceEndpointMMAP.cpp
index 3f18b95..ea817ab 100644
--- a/services/oboeservice/AAudioServiceEndpointMMAP.cpp
+++ b/services/oboeservice/AAudioServiceEndpointMMAP.cpp
@@ -71,6 +71,8 @@
aaudio_result_t AAudioServiceEndpointMMAP::open(const aaudio::AAudioStreamRequest &request) {
aaudio_result_t result = AAUDIO_OK;
copyFrom(request.getConstantConfiguration());
+ mRequestedDeviceId = getDeviceId();
+
mMmapClient.attributionSource = request.getAttributionSource();
// TODO b/182392769: use attribution source util
mMmapClient.attributionSource.uid = VALUE_OR_FATAL(
@@ -115,7 +117,7 @@
const audio_attributes_t attributes = getAudioAttributesFrom(this);
- mRequestedDeviceId = deviceId = getDeviceId();
+ deviceId = mRequestedDeviceId;
// Fill in config
config.format = audioFormat;
@@ -151,6 +153,10 @@
audio_session_t sessionId = AAudioConvert_aaudioToAndroidSessionId(requestedSessionId);
// Open HAL stream. Set mMmapStream
+ ALOGD("%s trying to open MMAP stream with format=%#x, "
+ "sample_rate=%u, channel_mask=%#x, device=%d",
+ __func__, config.format, config.sample_rate,
+ config.channel_mask, deviceId);
status_t status = MmapStreamInterface::openMmapStream(streamDirection,
&attributes,
&config,
@@ -221,6 +227,9 @@
error:
close();
+ // restore original requests
+ setDeviceId(mRequestedDeviceId);
+ setSessionId(requestedSessionId);
return result;
}
diff --git a/services/oboeservice/AAudioServiceStreamBase.cpp b/services/oboeservice/AAudioServiceStreamBase.cpp
index 9f48f80..f4ee84f 100644
--- a/services/oboeservice/AAudioServiceStreamBase.cpp
+++ b/services/oboeservice/AAudioServiceStreamBase.cpp
@@ -435,7 +435,15 @@
}
}
if (isIdle_l() && AudioClock::getNanoseconds() >= standbyTime) {
- standby_l();
+ aaudio_result_t result = standby_l();
+ if (result != AAUDIO_OK) {
+ // If standby failed because of the function is not implemented, there is no
+ // need to retry. Otherwise, retry standby later.
+ ALOGW("Failed to enter standby, error=%d", result);
+ standbyTime = result == AAUDIO_ERROR_UNIMPLEMENTED
+ ? std::numeric_limits<int64_t>::max()
+ : AudioClock::getNanoseconds() + IDLE_TIMEOUT_NANOS;
+ }
}
if (command != nullptr) {
diff --git a/services/oboeservice/AAudioServiceStreamBase.h b/services/oboeservice/AAudioServiceStreamBase.h
index b2ba725..b5f8b90 100644
--- a/services/oboeservice/AAudioServiceStreamBase.h
+++ b/services/oboeservice/AAudioServiceStreamBase.h
@@ -320,7 +320,7 @@
}
virtual aaudio_result_t standby_l() REQUIRES(mLock) {
- return AAUDIO_ERROR_UNAVAILABLE;
+ return AAUDIO_ERROR_UNIMPLEMENTED;
}
class ExitStandbyParam : public AAudioCommandParam {
public:
