[automerger skipped] Merge "[DO NOT MERGE] Fix heap buffer overflow in clearkey CryptoPlugin::decrypt" into pi-dev am: d651cdcfaf -s ours am: a67cf12a10 Change-Id: Icf5ce88a4b654bf3b6886e70069b5725fb946510
diff --git a/apex/ld.config.txt b/apex/ld.config.txt index a5937fd..af8ec06 100644 --- a/apex/ld.config.txt +++ b/apex/ld.config.txt
@@ -37,9 +37,11 @@ namespace.platform.isolated = true -namespace.platform.search.paths = /system/${LIB} +namespace.platform.search.paths = /system/${LIB} +namespace.platform.search.paths += /apex/com.android.runtime/${LIB} namespace.platform.asan.search.paths = /data/asan/system/${LIB} namespace.platform.asan.search.paths += /system/${LIB} +namespace.platform.asan.search.paths += /apex/com.android.runtime/${LIB} # /system/lib/libc.so, etc are symlinks to /apex/com.android.lib/lib/bionic/libc.so, etc. # Add /apex/... pat to the permitted paths because linker uses realpath(3)
diff --git a/apex/manifest.json b/apex/manifest.json index 3011ee8..b11187d 100644 --- a/apex/manifest.json +++ b/apex/manifest.json
@@ -1,4 +1,4 @@ { "name": "com.android.media", - "version": 290000000 + "version": 299900000 }
diff --git a/apex/manifest_codec.json b/apex/manifest_codec.json index 83a5178..09c436d 100644 --- a/apex/manifest_codec.json +++ b/apex/manifest_codec.json
@@ -1,4 +1,4 @@ { "name": "com.android.media.swcodec", - "version": 290000000 + "version": 299900000 }
diff --git a/camera/cameraserver/main_cameraserver.cpp b/camera/cameraserver/main_cameraserver.cpp index 53b3d84..cef8ef5 100644 --- a/camera/cameraserver/main_cameraserver.cpp +++ b/camera/cameraserver/main_cameraserver.cpp
@@ -34,6 +34,7 @@ sp<IServiceManager> sm = defaultServiceManager(); ALOGI("ServiceManager: %p", sm.get()); CameraService::instantiate(); + ALOGI("ServiceManager: %p done instantiate", sm.get()); ProcessState::self()->startThreadPool(); IPCThreadState::self()->joinThreadPool(); }
diff --git a/camera/ndk/impl/ACameraCaptureSession.cpp b/camera/ndk/impl/ACameraCaptureSession.cpp index d6f1412..68db233 100644 --- a/camera/ndk/impl/ACameraCaptureSession.cpp +++ b/camera/ndk/impl/ACameraCaptureSession.cpp
@@ -33,7 +33,9 @@ dev->unlockDevice(); } // Fire onClosed callback - (*mUserSessionCallback.onClosed)(mUserSessionCallback.context, this); + if (mUserSessionCallback.onClosed != nullptr) { + (*mUserSessionCallback.onClosed)(mUserSessionCallback.context, this); + } ALOGV("~ACameraCaptureSession: %p is deleted", this); }
diff --git a/camera/ndk/impl/ACameraDevice.cpp b/camera/ndk/impl/ACameraDevice.cpp index d24cb81..46a8dae 100644 --- a/camera/ndk/impl/ACameraDevice.cpp +++ b/camera/ndk/impl/ACameraDevice.cpp
@@ -29,7 +29,7 @@ #include "ACameraCaptureSession.inc" ACameraDevice::~ACameraDevice() { - mDevice->stopLooper(); + mDevice->stopLooperAndDisconnect(); } namespace android { @@ -112,19 +112,7 @@ } } -// Device close implementaiton -CameraDevice::~CameraDevice() { - sp<ACameraCaptureSession> session = mCurrentSession.promote(); - { - Mutex::Autolock _l(mDeviceLock); - if (!isClosed()) { - disconnectLocked(session); - } - LOG_ALWAYS_FATAL_IF(mCbLooper != nullptr, - "CameraDevice looper should've been stopped before ~CameraDevice"); - mCurrentSession = nullptr; - } -} +CameraDevice::~CameraDevice() { } void CameraDevice::postSessionMsgAndCleanup(sp<AMessage>& msg) { @@ -892,8 +880,14 @@ return; } -void CameraDevice::stopLooper() { +void CameraDevice::stopLooperAndDisconnect() { Mutex::Autolock _l(mDeviceLock); + sp<ACameraCaptureSession> session = mCurrentSession.promote(); + if (!isClosed()) { + disconnectLocked(session); + } + mCurrentSession = nullptr; + if (mCbLooper != nullptr) { mCbLooper->unregisterHandler(mHandler->id()); mCbLooper->stop();
diff --git a/camera/ndk/impl/ACameraDevice.h b/camera/ndk/impl/ACameraDevice.h index 7a35bf0..6c2ceb3 100644 --- a/camera/ndk/impl/ACameraDevice.h +++ b/camera/ndk/impl/ACameraDevice.h
@@ -40,6 +40,7 @@ #include <camera/NdkCameraManager.h> #include <camera/NdkCameraCaptureSession.h> + #include "ACameraMetadata.h" namespace android { @@ -110,7 +111,7 @@ inline ACameraDevice* getWrapper() const { return mWrapper; }; // Stop the looper thread and unregister the handler - void stopLooper(); + void stopLooperAndDisconnect(); private: friend ACameraCaptureSession;
diff --git a/camera/ndk/ndk_vendor/impl/ACameraDevice.cpp b/camera/ndk/ndk_vendor/impl/ACameraDevice.cpp index 35c8355..e511a3f 100644 --- a/camera/ndk/ndk_vendor/impl/ACameraDevice.cpp +++ b/camera/ndk/ndk_vendor/impl/ACameraDevice.cpp
@@ -45,7 +45,7 @@ using namespace android; ACameraDevice::~ACameraDevice() { - mDevice->stopLooper(); + mDevice->stopLooperAndDisconnect(); } namespace android { @@ -125,19 +125,7 @@ } } -// Device close implementaiton -CameraDevice::~CameraDevice() { - sp<ACameraCaptureSession> session = mCurrentSession.promote(); - { - Mutex::Autolock _l(mDeviceLock); - if (!isClosed()) { - disconnectLocked(session); - } - mCurrentSession = nullptr; - LOG_ALWAYS_FATAL_IF(mCbLooper != nullptr, - "CameraDevice looper should've been stopped before ~CameraDevice"); - } -} +CameraDevice::~CameraDevice() { } void CameraDevice::postSessionMsgAndCleanup(sp<AMessage>& msg) { @@ -1388,6 +1376,7 @@ // before cbh goes out of scope and causing we call the session // destructor while holding device lock cbh.mSession.clear(); + postSessionMsgAndCleanup(msg); } @@ -1400,8 +1389,13 @@ } } -void CameraDevice::stopLooper() { +void CameraDevice::stopLooperAndDisconnect() { Mutex::Autolock _l(mDeviceLock); + sp<ACameraCaptureSession> session = mCurrentSession.promote(); + if (!isClosed()) { + disconnectLocked(session); + } + mCurrentSession = nullptr; if (mCbLooper != nullptr) { mCbLooper->unregisterHandler(mHandler->id()); mCbLooper->stop();
diff --git a/camera/ndk/ndk_vendor/impl/ACameraDevice.h b/camera/ndk/ndk_vendor/impl/ACameraDevice.h index 3328a85..7fc699e 100644 --- a/camera/ndk/ndk_vendor/impl/ACameraDevice.h +++ b/camera/ndk/ndk_vendor/impl/ACameraDevice.h
@@ -36,6 +36,7 @@ #include <camera/NdkCameraManager.h> #include <camera/NdkCameraCaptureSession.h> + #include "ACameraMetadata.h" #include "utils.h" @@ -134,10 +135,11 @@ inline ACameraDevice* getWrapper() const { return mWrapper; }; // Stop the looper thread and unregister the handler - void stopLooper(); + void stopLooperAndDisconnect(); private: friend ACameraCaptureSession; + friend ACameraDevice; camera_status_t checkCameraClosedOrErrorLocked() const; @@ -387,7 +389,6 @@ mDevice(new android::acam::CameraDevice(id, cb, std::move(chars), this)) {} ~ACameraDevice(); - /******************* * NDK public APIs * *******************/
diff --git a/camera/ndk/ndk_vendor/tests/AImageReaderVendorTest.cpp b/camera/ndk/ndk_vendor/tests/AImageReaderVendorTest.cpp index 37de30a..938b5f5 100644 --- a/camera/ndk/ndk_vendor/tests/AImageReaderVendorTest.cpp +++ b/camera/ndk/ndk_vendor/tests/AImageReaderVendorTest.cpp
@@ -24,6 +24,7 @@ #include <algorithm> #include <mutex> #include <string> +#include <variant> #include <vector> #include <stdio.h> #include <stdio.h> @@ -49,6 +50,7 @@ static constexpr int kTestImageFormat = AIMAGE_FORMAT_YUV_420_888; using android::hardware::camera::common::V1_0::helper::VendorTagDescriptorCache; +using ConfiguredWindows = std::set<native_handle_t *>; class CameraHelper { public: @@ -60,9 +62,12 @@ const char* physicalCameraId; native_handle_t* anw; }; - int initCamera(native_handle_t* imgReaderAnw, + + // Retaining the error code in case the caller needs to analyze it. + std::variant<int, ConfiguredWindows> initCamera(native_handle_t* imgReaderAnw, const std::vector<PhysicalImgReaderInfo>& physicalImgReaders, bool usePhysicalSettings) { + ConfiguredWindows configuredWindows; if (imgReaderAnw == nullptr) { ALOGE("Cannot initialize camera before image reader get initialized."); return -1; @@ -78,7 +83,7 @@ ret = ACameraManager_openCamera(mCameraManager, mCameraId, &mDeviceCb, &mDevice); if (ret != AMEDIA_OK || mDevice == nullptr) { ALOGE("Failed to open camera, ret=%d, mDevice=%p.", ret, mDevice); - return -1; + return ret; } // Create capture session @@ -97,8 +102,9 @@ ALOGE("ACaptureSessionOutputContainer_add failed, ret=%d", ret); return ret; } - + configuredWindows.insert(mImgReaderAnw); std::vector<const char*> idPointerList; + std::set<const native_handle_t*> physicalStreamMap; for (auto& physicalStream : physicalImgReaders) { ACaptureSessionOutput* sessionOutput = nullptr; ret = ACaptureSessionPhysicalOutput_create(physicalStream.anw, @@ -112,21 +118,25 @@ ALOGE("ACaptureSessionOutputContainer_add failed, ret=%d", ret); return ret; } - mExtraOutputs.push_back(sessionOutput); + ret = ACameraDevice_isSessionConfigurationSupported(mDevice, mOutputs); + if (ret != ACAMERA_OK && ret != ACAMERA_ERROR_UNSUPPORTED_OPERATION) { + ALOGW("ACameraDevice_isSessionConfigurationSupported failed, ret=%d camera id %s", + ret, mCameraId); + ACaptureSessionOutputContainer_remove(mOutputs, sessionOutput); + ACaptureSessionOutput_free(sessionOutput); + continue; + } + configuredWindows.insert(physicalStream.anw); // Assume that at most one physical stream per physical camera. mPhysicalCameraIds.push_back(physicalStream.physicalCameraId); idPointerList.push_back(physicalStream.physicalCameraId); + physicalStreamMap.insert(physicalStream.anw); + mSessionPhysicalOutputs.push_back(sessionOutput); } ACameraIdList cameraIdList; cameraIdList.numCameras = idPointerList.size(); cameraIdList.cameraIds = idPointerList.data(); - ret = ACameraDevice_isSessionConfigurationSupported(mDevice, mOutputs); - if (ret != ACAMERA_OK && ret != ACAMERA_ERROR_UNSUPPORTED_OPERATION) { - ALOGE("ACameraDevice_isSessionConfigurationSupported failed, ret=%d", ret); - return ret; - } - ret = ACameraDevice_createCaptureSession(mDevice, mOutputs, &mSessionCb, &mSession); if (ret != AMEDIA_OK) { ALOGE("ACameraDevice_createCaptureSession failed, ret=%d", ret); @@ -157,6 +167,10 @@ } for (auto& physicalStream : physicalImgReaders) { + if (physicalStreamMap.find(physicalStream.anw) == physicalStreamMap.end()) { + ALOGI("Skipping physicalStream anw=%p", physicalStream.anw); + continue; + } ACameraOutputTarget* outputTarget = nullptr; ret = ACameraOutputTarget_create(physicalStream.anw, &outputTarget); if (ret != AMEDIA_OK) { @@ -168,11 +182,11 @@ ALOGE("ACaptureRequest_addTarget failed, ret=%d", ret); return ret; } - mReqExtraOutputs.push_back(outputTarget); + mReqPhysicalOutputs.push_back(outputTarget); } mIsCameraReady = true; - return 0; + return configuredWindows; } @@ -184,10 +198,10 @@ ACameraOutputTarget_free(mReqImgReaderOutput); mReqImgReaderOutput = nullptr; } - for (auto& outputTarget : mReqExtraOutputs) { + for (auto& outputTarget : mReqPhysicalOutputs) { ACameraOutputTarget_free(outputTarget); } - mReqExtraOutputs.clear(); + mReqPhysicalOutputs.clear(); if (mStillRequest) { ACaptureRequest_free(mStillRequest); mStillRequest = nullptr; @@ -201,10 +215,10 @@ ACaptureSessionOutput_free(mImgReaderOutput); mImgReaderOutput = nullptr; } - for (auto& extraOutput : mExtraOutputs) { + for (auto& extraOutput : mSessionPhysicalOutputs) { ACaptureSessionOutput_free(extraOutput); } - mExtraOutputs.clear(); + mSessionPhysicalOutputs.clear(); if (mOutputs) { ACaptureSessionOutputContainer_free(mOutputs); mOutputs = nullptr; @@ -239,21 +253,9 @@ return true; } - static void onDeviceDisconnected(void* /*obj*/, ACameraDevice* /*device*/) {} - - static void onDeviceError(void* /*obj*/, ACameraDevice* /*device*/, int /*errorCode*/) {} - - static void onSessionClosed(void* /*obj*/, ACameraCaptureSession* /*session*/) {} - - static void onSessionReady(void* /*obj*/, ACameraCaptureSession* /*session*/) {} - - static void onSessionActive(void* /*obj*/, ACameraCaptureSession* /*session*/) {} - private: - ACameraDevice_StateCallbacks mDeviceCb{this, onDeviceDisconnected, - onDeviceError}; - ACameraCaptureSession_stateCallbacks mSessionCb{ - this, onSessionClosed, onSessionReady, onSessionActive}; + ACameraDevice_StateCallbacks mDeviceCb{this, nullptr, nullptr}; + ACameraCaptureSession_stateCallbacks mSessionCb{ this, nullptr, nullptr, nullptr}; native_handle_t* mImgReaderAnw = nullptr; // not owned by us. @@ -262,13 +264,13 @@ // Capture session ACaptureSessionOutputContainer* mOutputs = nullptr; ACaptureSessionOutput* mImgReaderOutput = nullptr; - std::vector<ACaptureSessionOutput*> mExtraOutputs; + std::vector<ACaptureSessionOutput*> mSessionPhysicalOutputs; ACameraCaptureSession* mSession = nullptr; // Capture request ACaptureRequest* mStillRequest = nullptr; ACameraOutputTarget* mReqImgReaderOutput = nullptr; - std::vector<ACameraOutputTarget*> mReqExtraOutputs; + std::vector<ACameraOutputTarget*> mReqPhysicalOutputs; bool mIsCameraReady = false; const char* mCameraId; @@ -581,9 +583,11 @@ } CameraHelper cameraHelper(id, mCameraManager); - ret = cameraHelper.initCamera(testCase.getNativeWindow(), - {}/*physicalImageReaders*/, false/*usePhysicalSettings*/); - if (ret < 0) { + std::variant<int, ConfiguredWindows> retInit = + cameraHelper.initCamera(testCase.getNativeWindow(), {}/*physicalImageReaders*/, + false/*usePhysicalSettings*/); + int *retp = std::get_if<int>(&retInit); + if (retp) { ALOGE("Unable to initialize camera helper"); return false; } @@ -751,10 +755,15 @@ physicalImgReaderInfo.push_back({physicalCameraIds[0], testCases[1]->getNativeWindow()}); physicalImgReaderInfo.push_back({physicalCameraIds[1], testCases[2]->getNativeWindow()}); - int ret = cameraHelper.initCamera(testCases[0]->getNativeWindow(), - physicalImgReaderInfo, usePhysicalSettings); - ASSERT_EQ(ret, 0); - + std::variant<int, ConfiguredWindows> retInit = + cameraHelper.initCamera(testCases[0]->getNativeWindow(), physicalImgReaderInfo, + usePhysicalSettings); + int *retp = std::get_if<int>(&retInit); + ASSERT_EQ(retp, nullptr); + ConfiguredWindows *configuredWindowsp = std::get_if<ConfiguredWindows>(&retInit); + ASSERT_NE(configuredWindowsp, nullptr); + ASSERT_LE(configuredWindowsp->size(), testCases.size()); + int ret = 0; if (!cameraHelper.isCameraReady()) { ALOGW("Camera is not ready after successful initialization. It's either due to camera " "on board lacks BACKWARDS_COMPATIBLE capability or the device does not have " @@ -776,9 +785,15 @@ break; } } - ASSERT_EQ(testCases[0]->getAcquiredImageCount(), pictureCount); - ASSERT_EQ(testCases[1]->getAcquiredImageCount(), pictureCount); - ASSERT_EQ(testCases[2]->getAcquiredImageCount(), pictureCount); + for(auto &testCase : testCases) { + auto it = configuredWindowsp->find(testCase->getNativeWindow()); + if (it == configuredWindowsp->end()) { + continue; + } + ALOGI("Testing window %p", testCase->getNativeWindow()); + ASSERT_EQ(testCase->getAcquiredImageCount(), pictureCount); + } + ASSERT_TRUE(cameraHelper.checkCallbacks(pictureCount)); ACameraMetadata_free(staticMetadata);
diff --git a/drm/mediadrm/plugins/clearkey/hidl/DrmPlugin.cpp b/drm/mediadrm/plugins/clearkey/hidl/DrmPlugin.cpp index 7cb5a38..aab475e 100644 --- a/drm/mediadrm/plugins/clearkey/hidl/DrmPlugin.cpp +++ b/drm/mediadrm/plugins/clearkey/hidl/DrmPlugin.cpp
@@ -797,13 +797,23 @@ } Return<Status> DrmPlugin::releaseSecureStops(const SecureStopRelease& ssRelease) { - if (ssRelease.opaqueData.size() == 0) { + // OpaqueData starts with 4 byte decimal integer string + const size_t kFourBytesOffset = 4; + if (ssRelease.opaqueData.size() < kFourBytesOffset) { + ALOGE("Invalid secureStopRelease length"); return Status::BAD_VALUE; } Status status = Status::OK; std::vector<uint8_t> input = toVector(ssRelease.opaqueData); + if (input.size() < kSecureStopIdSize + kFourBytesOffset) { + // The minimum size of SecureStopRelease has to contain + // a 4 bytes count and one secureStop id + ALOGE("Total size of secureStops is too short"); + return Status::BAD_VALUE; + } + // The format of opaqueData is shared between the server // and the drm service. The clearkey implementation consists of: // count - number of secure stops @@ -819,20 +829,29 @@ // Avoid divide by 0 below. if (count == 0) { + ALOGE("Invalid 0 secureStop count"); return Status::BAD_VALUE; } - size_t secureStopSize = (input.size() - countBufferSize) / count; - uint8_t buffer[secureStopSize]; - size_t offset = countBufferSize; // skip the count + // Computes the fixed length secureStop size + size_t secureStopSize = (input.size() - kFourBytesOffset) / count; + if (secureStopSize < kSecureStopIdSize) { + // A valid secureStop contains the id plus data + ALOGE("Invalid secureStop size"); + return Status::BAD_VALUE; + } + uint8_t* buffer = new uint8_t[secureStopSize]; + size_t offset = kFourBytesOffset; // skip the count for (size_t i = 0; i < count; ++i, offset += secureStopSize) { memcpy(buffer, input.data() + offset, secureStopSize); - std::vector<uint8_t> id(buffer, buffer + kSecureStopIdSize); + // A secureStop contains id+data, we only use the id for removal + std::vector<uint8_t> id(buffer, buffer + kSecureStopIdSize); status = removeSecureStop(toHidlVec(id)); if (Status::OK != status) break; } + delete[] buffer; return status; }
diff --git a/media/bufferpool/1.0/AccessorImpl.cpp b/media/bufferpool/1.0/AccessorImpl.cpp index fa17f15..a5366f6 100644 --- a/media/bufferpool/1.0/AccessorImpl.cpp +++ b/media/bufferpool/1.0/AccessorImpl.cpp
@@ -151,6 +151,7 @@ newConnection->initialize(accessor, id); *connection = newConnection; *pConnectionId = id; + mBufferPool.mConnectionIds.insert(id); ++sSeqId; } } @@ -305,7 +306,12 @@ found->second->mSenderValidated = true; return true; } - // TODO: verify there is target connection Id + if (mConnectionIds.find(message.targetConnectionId) == mConnectionIds.end()) { + // N.B: it could be fake or receive connection already closed. + ALOGD("bufferpool %p receiver connection %lld is no longer valid", + this, (long long)message.targetConnectionId); + return false; + } mStats.onBufferSent(); mTransactions.insert(std::make_pair( message.transactionId, @@ -450,6 +456,7 @@ } } } + mConnectionIds.erase(connectionId); return true; }
diff --git a/media/bufferpool/1.0/AccessorImpl.h b/media/bufferpool/1.0/AccessorImpl.h index c04dbf3..84cb685 100644 --- a/media/bufferpool/1.0/AccessorImpl.h +++ b/media/bufferpool/1.0/AccessorImpl.h
@@ -94,6 +94,7 @@ std::map<BufferId, std::unique_ptr<InternalBuffer>> mBuffers; std::set<BufferId> mFreeBuffers; + std::set<ConnectionId> mConnectionIds; /// Buffer pool statistics which tracks allocation and transfer statistics. struct Stats {
diff --git a/media/bufferpool/2.0/AccessorImpl.cpp b/media/bufferpool/2.0/AccessorImpl.cpp index 94cf006..0d591d7 100644 --- a/media/bufferpool/2.0/AccessorImpl.cpp +++ b/media/bufferpool/2.0/AccessorImpl.cpp
@@ -37,7 +37,7 @@ static constexpr int64_t kLogDurationUs = 5000000; // 5 secs static constexpr size_t kMinAllocBytesForEviction = 1024*1024*15; - static constexpr size_t kMinBufferCountForEviction = 40; + static constexpr size_t kMinBufferCountForEviction = 25; } // Buffer structure in bufferpool process @@ -163,6 +163,7 @@ *connection = newConnection; *pConnectionId = id; *pMsgId = mBufferPool.mInvalidation.mInvalidationId; + mBufferPool.mConnectionIds.insert(id); mBufferPool.mInvalidationChannel.getDesc(invDescPtr); mBufferPool.mInvalidation.onConnect(id, observer); ++sSeqId; @@ -474,7 +475,12 @@ found->second->mSenderValidated = true; return true; } - // TODO: verify there is target connection Id + if (mConnectionIds.find(message.targetConnectionId) == mConnectionIds.end()) { + // N.B: it could be fake or receive connection already closed. + ALOGD("bufferpool2 %p receiver connection %lld is no longer valid", + this, (long long)message.targetConnectionId); + return false; + } mStats.onBufferSent(); mTransactions.insert(std::make_pair( message.transactionId, @@ -644,6 +650,7 @@ } } } + mConnectionIds.erase(connectionId); return true; } @@ -711,8 +718,8 @@ mStats.mTotalFetches, mStats.mTotalTransfers); } for (auto freeIt = mFreeBuffers.begin(); freeIt != mFreeBuffers.end();) { - if (!clearCache && mStats.mSizeCached < kMinAllocBytesForEviction - && mBuffers.size() < kMinBufferCountForEviction) { + if (!clearCache && (mStats.mSizeCached < kMinAllocBytesForEviction + || mBuffers.size() < kMinBufferCountForEviction)) { break; } auto it = mBuffers.find(*freeIt); @@ -774,11 +781,19 @@ std::mutex &mutex, std::condition_variable &cv, bool &ready) { + constexpr uint32_t NUM_SPIN_TO_INCREASE_SLEEP = 1024; + constexpr uint32_t NUM_SPIN_TO_LOG = 1024*8; + constexpr useconds_t MAX_SLEEP_US = 10000; + uint32_t numSpin = 0; + useconds_t sleepUs = 1; + while(true) { std::map<uint32_t, const std::weak_ptr<Accessor::Impl>> copied; { std::unique_lock<std::mutex> lock(mutex); if (!ready) { + numSpin = 0; + sleepUs = 1; cv.wait(lock); } copied.insert(accessors.begin(), accessors.end()); @@ -800,9 +815,20 @@ if (accessors.size() == 0) { ready = false; } else { - // prevent draining cpu. + // TODO Use an efficient way to wait over FMQ. + // N.B. Since there is not a efficient way to wait over FMQ, + // polling over the FMQ is the current way to prevent draining + // CPU. lock.unlock(); - std::this_thread::yield(); + ++numSpin; + if (numSpin % NUM_SPIN_TO_INCREASE_SLEEP == 0 && + sleepUs < MAX_SLEEP_US) { + sleepUs *= 10; + } + if (numSpin % NUM_SPIN_TO_LOG == 0) { + ALOGW("invalidator thread spinning"); + } + ::usleep(sleepUs); } } }
diff --git a/media/bufferpool/2.0/AccessorImpl.h b/media/bufferpool/2.0/AccessorImpl.h index eea72b9..807e0f1 100644 --- a/media/bufferpool/2.0/AccessorImpl.h +++ b/media/bufferpool/2.0/AccessorImpl.h
@@ -111,6 +111,7 @@ std::map<BufferId, std::unique_ptr<InternalBuffer>> mBuffers; std::set<BufferId> mFreeBuffers; + std::set<ConnectionId> mConnectionIds; struct Invalidation { static std::atomic<std::uint32_t> sInvSeqId;
diff --git a/media/bufferpool/2.0/Android.bp b/media/bufferpool/2.0/Android.bp index c71ac17..e8a69c9 100644 --- a/media/bufferpool/2.0/Android.bp +++ b/media/bufferpool/2.0/Android.bp
@@ -1,9 +1,5 @@ -cc_library { - name: "libstagefright_bufferpool@2.0", - vendor_available: true, - vndk: { - enabled: true, - }, +cc_defaults { + name: "libstagefright_bufferpool@2.0-default", srcs: [ "Accessor.cpp", "AccessorImpl.cpp", @@ -31,3 +27,23 @@ "android.hardware.media.bufferpool@2.0", ], } + +cc_library { + name: "libstagefright_bufferpool@2.0.1", + defaults: ["libstagefright_bufferpool@2.0-default"], + vendor_available: true, + cflags: [ + "-DBUFFERPOOL_CLONE_HANDLES", + ], +} + +// Deprecated. Do not use. Use libstagefright_bufferpool@2.0.1 instead. +cc_library { + name: "libstagefright_bufferpool@2.0", + defaults: ["libstagefright_bufferpool@2.0-default"], + vendor_available: true, + vndk: { + enabled: true, + }, +} +
diff --git a/media/bufferpool/2.0/ClientManager.cpp b/media/bufferpool/2.0/ClientManager.cpp index c31d313..87ee4e8 100644 --- a/media/bufferpool/2.0/ClientManager.cpp +++ b/media/bufferpool/2.0/ClientManager.cpp
@@ -351,7 +351,21 @@ } client = it->second; } +#ifdef BUFFERPOOL_CLONE_HANDLES + native_handle_t *origHandle; + ResultStatus res = client->allocate(params, &origHandle, buffer); + if (res != ResultStatus::OK) { + return res; + } + *handle = native_handle_clone(origHandle); + if (handle == NULL) { + buffer->reset(); + return ResultStatus::NO_MEMORY; + } + return ResultStatus::OK; +#else return client->allocate(params, handle, buffer); +#endif } ResultStatus ClientManager::Impl::receive( @@ -367,7 +381,22 @@ } client = it->second; } +#ifdef BUFFERPOOL_CLONE_HANDLES + native_handle_t *origHandle; + ResultStatus res = client->receive( + transactionId, bufferId, timestampUs, &origHandle, buffer); + if (res != ResultStatus::OK) { + return res; + } + *handle = native_handle_clone(origHandle); + if (handle == NULL) { + buffer->reset(); + return ResultStatus::NO_MEMORY; + } + return ResultStatus::OK; +#else return client->receive(transactionId, bufferId, timestampUs, handle, buffer); +#endif } ResultStatus ClientManager::Impl::postSend(
diff --git a/media/bufferpool/2.0/include/bufferpool/ClientManager.h b/media/bufferpool/2.0/include/bufferpool/ClientManager.h index 953c304..24b61f4 100644 --- a/media/bufferpool/2.0/include/bufferpool/ClientManager.h +++ b/media/bufferpool/2.0/include/bufferpool/ClientManager.h
@@ -104,7 +104,9 @@ ResultStatus flush(ConnectionId connectionId); /** - * Allocates a buffer from the specified connection. + * Allocates a buffer from the specified connection. The output parameter + * handle is cloned from the internal handle. So it is safe to use directly, + * and it should be deleted and destroyed after use. * * @param connectionId The id of the connection. * @param params The allocation parameters. @@ -123,7 +125,9 @@ std::shared_ptr<BufferPoolData> *buffer); /** - * Receives a buffer for the transaction. + * Receives a buffer for the transaction. The output parameter handle is + * cloned from the internal handle. So it is safe to use directly, and it + * should be deleted and destoyed after use. * * @param connectionId The id of the receiving connection. * @param transactionId The id for the transaction.
diff --git a/media/codec2/components/aac/C2SoftAacEnc.cpp b/media/codec2/components/aac/C2SoftAacEnc.cpp index 8e3852c..be52a1d 100644 --- a/media/codec2/components/aac/C2SoftAacEnc.cpp +++ b/media/codec2/components/aac/C2SoftAacEnc.cpp
@@ -157,9 +157,10 @@ mSentCodecSpecificData(false), mInputTimeSet(false), mInputSize(0), - mInputTimeUs(0), + mNextFrameTimestampUs(0), mSignalledError(false), - mOutIndex(0u) { + mOutIndex(0u), + mRemainderLen(0u) { } C2SoftAacEnc::~C2SoftAacEnc() { @@ -183,8 +184,9 @@ mSentCodecSpecificData = false; mInputTimeSet = false; mInputSize = 0u; - mInputTimeUs = 0; + mNextFrameTimestampUs = 0; mSignalledError = false; + mRemainderLen = 0; return C2_OK; } @@ -201,7 +203,7 @@ mSentCodecSpecificData = false; mInputTimeSet = false; mInputSize = 0u; - mInputTimeUs = 0; + mNextFrameTimestampUs = 0; return C2_OK; } @@ -365,21 +367,25 @@ capacity = view.capacity(); } if (!mInputTimeSet && capacity > 0) { - mInputTimeUs = work->input.ordinal.timestamp; + mNextFrameTimestampUs = work->input.ordinal.timestamp; mInputTimeSet = true; } - size_t numFrames = (capacity + mInputSize + (eos ? mNumBytesPerInputFrame - 1 : 0)) - / mNumBytesPerInputFrame; - ALOGV("capacity = %zu; mInputSize = %zu; numFrames = %zu mNumBytesPerInputFrame = %u", - capacity, mInputSize, numFrames, mNumBytesPerInputFrame); + size_t numFrames = + (mRemainderLen + capacity + mInputSize + (eos ? mNumBytesPerInputFrame - 1 : 0)) + / mNumBytesPerInputFrame; + ALOGV("capacity = %zu; mInputSize = %zu; numFrames = %zu " + "mNumBytesPerInputFrame = %u inputTS = %lld remaining = %zu", + capacity, mInputSize, numFrames, + mNumBytesPerInputFrame, work->input.ordinal.timestamp.peekll(), + mRemainderLen); std::shared_ptr<C2LinearBlock> block; - std::shared_ptr<C2Buffer> buffer; std::unique_ptr<C2WriteView> wView; uint8_t *outPtr = temp; size_t outAvailable = 0u; uint64_t inputIndex = work->input.ordinal.frameIndex.peeku(); + size_t bytesPerSample = channelCount * sizeof(int16_t); AACENC_InArgs inargs; AACENC_OutArgs outargs; @@ -442,9 +448,31 @@ const std::shared_ptr<C2Buffer> mBuffer; }; - C2WorkOrdinalStruct outOrdinal = work->input.ordinal; + struct OutputBuffer { + std::shared_ptr<C2Buffer> buffer; + c2_cntr64_t timestampUs; + }; + std::list<OutputBuffer> outputBuffers; - while (encoderErr == AACENC_OK && inargs.numInSamples > 0) { + if (mRemainderLen > 0) { + size_t offset = 0; + for (; mRemainderLen < bytesPerSample && offset < capacity; ++offset) { + mRemainder[mRemainderLen++] = data[offset]; + } + data += offset; + capacity -= offset; + if (mRemainderLen == bytesPerSample) { + inBuffer[0] = mRemainder; + inBufferSize[0] = bytesPerSample; + inargs.numInSamples = channelCount; + mRemainderLen = 0; + ALOGV("Processing remainder"); + } else { + // We have exhausted the input already + inargs.numInSamples = 0; + } + } + while (encoderErr == AACENC_OK && inargs.numInSamples >= channelCount) { if (numFrames && !block) { C2MemoryUsage usage = { C2MemoryUsage::CPU_READ, C2MemoryUsage::CPU_WRITE }; // TODO: error handling, proper usage, etc. @@ -473,43 +501,40 @@ &outargs); if (encoderErr == AACENC_OK) { - if (buffer) { - outOrdinal.frameIndex = mOutIndex++; - outOrdinal.timestamp = mInputTimeUs; - cloneAndSend( - inputIndex, - work, - FillWork(C2FrameData::FLAG_INCOMPLETE, outOrdinal, buffer)); - buffer.reset(); - } - if (outargs.numOutBytes > 0) { mInputSize = 0; int consumed = (capacity / sizeof(int16_t)) - inargs.numInSamples + outargs.numInSamples; - mInputTimeUs = work->input.ordinal.timestamp + c2_cntr64_t currentFrameTimestampUs = mNextFrameTimestampUs; + mNextFrameTimestampUs = work->input.ordinal.timestamp + (consumed * 1000000ll / channelCount / sampleRate); - buffer = createLinearBuffer(block, 0, outargs.numOutBytes); -#if defined(LOG_NDEBUG) && !LOG_NDEBUG + std::shared_ptr<C2Buffer> buffer = createLinearBuffer(block, 0, outargs.numOutBytes); +#if 0 hexdump(outPtr, std::min(outargs.numOutBytes, 256)); #endif outPtr = temp; outAvailable = 0; block.reset(); + + outputBuffers.push_back({buffer, currentFrameTimestampUs}); } else { mInputSize += outargs.numInSamples * sizeof(int16_t); } - if (outargs.numInSamples > 0) { + if (inBuffer[0] == mRemainder) { + inBuffer[0] = const_cast<uint8_t *>(data); + inBufferSize[0] = capacity; + inargs.numInSamples = capacity / sizeof(int16_t); + } else if (outargs.numInSamples > 0) { inBuffer[0] = (int16_t *)inBuffer[0] + outargs.numInSamples; inBufferSize[0] -= outargs.numInSamples * sizeof(int16_t); inargs.numInSamples -= outargs.numInSamples; } } - ALOGV("encoderErr = %d mInputSize = %zu inargs.numInSamples = %d, mInputTimeUs = %lld", - encoderErr, mInputSize, inargs.numInSamples, mInputTimeUs.peekll()); + ALOGV("encoderErr = %d mInputSize = %zu " + "inargs.numInSamples = %d, mNextFrameTimestampUs = %lld", + encoderErr, mInputSize, inargs.numInSamples, mNextFrameTimestampUs.peekll()); } - if (eos && inBufferSize[0] > 0) { if (numFrames && !block) { C2MemoryUsage usage = { C2MemoryUsage::CPU_READ, C2MemoryUsage::CPU_WRITE }; @@ -540,12 +565,37 @@ &outBufDesc, &inargs, &outargs); + inBufferSize[0] = 0; } - outOrdinal.frameIndex = mOutIndex++; - outOrdinal.timestamp = mInputTimeUs; + if (inBufferSize[0] > 0) { + for (size_t i = 0; i < inBufferSize[0]; ++i) { + mRemainder[i] = static_cast<uint8_t *>(inBuffer[0])[i]; + } + mRemainderLen = inBufferSize[0]; + } + + while (outputBuffers.size() > 1) { + const OutputBuffer& front = outputBuffers.front(); + C2WorkOrdinalStruct ordinal = work->input.ordinal; + ordinal.frameIndex = mOutIndex++; + ordinal.timestamp = front.timestampUs; + cloneAndSend( + inputIndex, + work, + FillWork(C2FrameData::FLAG_INCOMPLETE, ordinal, front.buffer)); + outputBuffers.pop_front(); + } + std::shared_ptr<C2Buffer> buffer; + C2WorkOrdinalStruct ordinal = work->input.ordinal; + ordinal.frameIndex = mOutIndex++; + if (!outputBuffers.empty()) { + ordinal.timestamp = outputBuffers.front().timestampUs; + buffer = outputBuffers.front().buffer; + } + // Mark the end of frame FillWork((C2FrameData::flags_t)(eos ? C2FrameData::FLAG_END_OF_STREAM : 0), - outOrdinal, buffer)(work); + ordinal, buffer)(work); } c2_status_t C2SoftAacEnc::drain( @@ -569,7 +619,7 @@ mSentCodecSpecificData = false; mInputTimeSet = false; mInputSize = 0u; - mInputTimeUs = 0; + mNextFrameTimestampUs = 0; // TODO: we don't have any pending work at this time to drain. return C2_OK;
diff --git a/media/codec2/components/aac/C2SoftAacEnc.h b/media/codec2/components/aac/C2SoftAacEnc.h index a38be19..6ecfbdd 100644 --- a/media/codec2/components/aac/C2SoftAacEnc.h +++ b/media/codec2/components/aac/C2SoftAacEnc.h
@@ -56,11 +56,15 @@ bool mSentCodecSpecificData; bool mInputTimeSet; size_t mInputSize; - c2_cntr64_t mInputTimeUs; + c2_cntr64_t mNextFrameTimestampUs; bool mSignalledError; std::atomic_uint64_t mOutIndex; + // We support max 6 channels + uint8_t mRemainder[6 * sizeof(int16_t)]; + size_t mRemainderLen; + status_t initEncoder(); status_t setAudioParams();
diff --git a/media/codec2/components/aom/Android.bp b/media/codec2/components/aom/Android.bp index 0fabf5c..61dbd4c 100644 --- a/media/codec2/components/aom/Android.bp +++ b/media/codec2/components/aom/Android.bp
@@ -1,10 +1,16 @@ cc_library_shared { - name: "libcodec2_soft_av1dec", + name: "libcodec2_soft_av1dec_aom", defaults: [ "libcodec2_soft-defaults", "libcodec2_soft_sanitize_all-defaults", ], + // coordinated with frameworks/av/media/codec2/components/gav1/Android.bp + // so only 1 of them has the official c2.android.av1.decoder name + cflags: [ + "-DCODECNAME=\"c2.android.av1-aom.decoder\"", + ], + srcs: ["C2SoftAomDec.cpp"], static_libs: ["libaom"],
diff --git a/media/codec2/components/aom/C2SoftAomDec.cpp b/media/codec2/components/aom/C2SoftAomDec.cpp index 769895c..36137e6 100644 --- a/media/codec2/components/aom/C2SoftAomDec.cpp +++ b/media/codec2/components/aom/C2SoftAomDec.cpp
@@ -29,7 +29,8 @@ namespace android { -constexpr char COMPONENT_NAME[] = "c2.android.av1.decoder"; +// codecname set and passed in as a compile flag from Android.bp +constexpr char COMPONENT_NAME[] = CODECNAME; class C2SoftAomDec::IntfImpl : public SimpleInterface<void>::BaseParams { public: @@ -340,6 +341,7 @@ aom_codec_flags_t flags; memset(&flags, 0, sizeof(aom_codec_flags_t)); + ALOGV("Using libaom AV1 software decoder."); aom_codec_err_t err; if ((err = aom_codec_dec_init(mCodecCtx, aom_codec_av1_dx(), &cfg, 0))) { ALOGE("av1 decoder failed to initialize. (%d)", err);
diff --git a/media/codec2/components/avc/C2SoftAvcDec.cpp b/media/codec2/components/avc/C2SoftAvcDec.cpp index 3f015b4..fa98178 100644 --- a/media/codec2/components/avc/C2SoftAvcDec.cpp +++ b/media/codec2/components/avc/C2SoftAvcDec.cpp
@@ -33,7 +33,8 @@ namespace { constexpr char COMPONENT_NAME[] = "c2.android.avc.decoder"; - +constexpr uint32_t kDefaultOutputDelay = 8; +constexpr uint32_t kMaxOutputDelay = 16; } // namespace class C2SoftAvcDec::IntfImpl : public SimpleInterface<void>::BaseParams { @@ -54,7 +55,9 @@ // TODO: Proper support for reorder depth. addParameter( DefineParam(mActualOutputDelay, C2_PARAMKEY_OUTPUT_DELAY) - .withConstValue(new C2PortActualDelayTuning::output(8u)) + .withDefault(new C2PortActualDelayTuning::output(kDefaultOutputDelay)) + .withFields({C2F(mActualOutputDelay, value).inRange(0, kMaxOutputDelay)}) + .withSetter(Setter<decltype(*mActualOutputDelay)>::StrictValueWithNoDeps) .build()); // TODO: output latency and reordering @@ -196,7 +199,6 @@ 0u, HAL_PIXEL_FORMAT_YCBCR_420_888)) .build()); } - static C2R SizeSetter(bool mayBlock, const C2P<C2StreamPictureSizeInfo::output> &oldMe, C2P<C2StreamPictureSizeInfo::output> &me) { (void)mayBlock; @@ -333,6 +335,7 @@ mDecHandle(nullptr), mOutBufferFlush(nullptr), mIvColorFormat(IV_YUV_420P), + mOutputDelay(kDefaultOutputDelay), mWidth(320), mHeight(240), mHeaderDecoded(false), @@ -882,6 +885,26 @@ work->result = C2_CORRUPTED; return; } + if (s_decode_op.i4_reorder_depth >= 0 && mOutputDelay != s_decode_op.i4_reorder_depth) { + mOutputDelay = s_decode_op.i4_reorder_depth; + ALOGV("New Output delay %d ", mOutputDelay); + + C2PortActualDelayTuning::output outputDelay(mOutputDelay); + std::vector<std::unique_ptr<C2SettingResult>> failures; + c2_status_t err = + mIntf->config({&outputDelay}, C2_MAY_BLOCK, &failures); + if (err == OK) { + work->worklets.front()->output.configUpdate.push_back( + C2Param::Copy(outputDelay)); + } else { + ALOGE("Cannot set output delay"); + mSignalledError = true; + work->workletsProcessed = 1u; + work->result = C2_CORRUPTED; + return; + } + continue; + } if (0 < s_decode_op.u4_pic_wd && 0 < s_decode_op.u4_pic_ht) { if (mHeaderDecoded == false) { mHeaderDecoded = true;
diff --git a/media/codec2/components/avc/C2SoftAvcDec.h b/media/codec2/components/avc/C2SoftAvcDec.h index 72ee583..4414a26 100644 --- a/media/codec2/components/avc/C2SoftAvcDec.h +++ b/media/codec2/components/avc/C2SoftAvcDec.h
@@ -157,7 +157,7 @@ size_t mNumCores; IV_COLOR_FORMAT_T mIvColorFormat; - + uint32_t mOutputDelay; uint32_t mWidth; uint32_t mHeight; uint32_t mStride;
diff --git a/media/codec2/components/g711/C2SoftG711Dec.cpp b/media/codec2/components/g711/C2SoftG711Dec.cpp index 43b843a..4ff0793 100644 --- a/media/codec2/components/g711/C2SoftG711Dec.cpp +++ b/media/codec2/components/g711/C2SoftG711Dec.cpp
@@ -74,7 +74,7 @@ addParameter( DefineParam(mChannelCount, C2_PARAMKEY_CHANNEL_COUNT) .withDefault(new C2StreamChannelCountInfo::output(0u, 1)) - .withFields({C2F(mChannelCount, value).equalTo(1)}) + .withFields({C2F(mChannelCount, value).inRange(1, 6)}) .withSetter(Setter<decltype(*mChannelCount)>::StrictValueWithNoDeps) .build());
diff --git a/media/codec2/components/gav1/Android.bp b/media/codec2/components/gav1/Android.bp new file mode 100644 index 0000000..5c4abb7 --- /dev/null +++ b/media/codec2/components/gav1/Android.bp
@@ -0,0 +1,20 @@ +cc_library_shared { + name: "libcodec2_soft_av1dec_gav1", + defaults: [ + "libcodec2_soft-defaults", + "libcodec2_soft_sanitize_all-defaults", + ], + + // coordinated with frameworks/av/media/codec2/components/aom/Android.bp + // so only 1 of them has the official c2.android.av1.decoder name + cflags: [ + "-DCODECNAME=\"c2.android.av1.decoder\"", + ], + + srcs: ["C2SoftGav1Dec.cpp"], + static_libs: ["libgav1"], + + include_dirs: [ + "external/libgav1/libgav1/", + ], +}
diff --git a/media/codec2/components/gav1/C2SoftGav1Dec.cpp b/media/codec2/components/gav1/C2SoftGav1Dec.cpp new file mode 100644 index 0000000..ec5f549 --- /dev/null +++ b/media/codec2/components/gav1/C2SoftGav1Dec.cpp
@@ -0,0 +1,792 @@ +/* + * Copyright (C) 2019 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +//#define LOG_NDEBUG 0 +#define LOG_TAG "C2SoftGav1Dec" +#include "C2SoftGav1Dec.h" + +#include <C2Debug.h> +#include <C2PlatformSupport.h> +#include <SimpleC2Interface.h> +#include <log/log.h> +#include <media/stagefright/foundation/AUtils.h> +#include <media/stagefright/foundation/MediaDefs.h> + +namespace android { + +// codecname set and passed in as a compile flag from Android.bp +constexpr char COMPONENT_NAME[] = CODECNAME; + +class C2SoftGav1Dec::IntfImpl : public SimpleInterface<void>::BaseParams { + public: + explicit IntfImpl(const std::shared_ptr<C2ReflectorHelper> &helper) + : SimpleInterface<void>::BaseParams( + helper, COMPONENT_NAME, C2Component::KIND_DECODER, + C2Component::DOMAIN_VIDEO, MEDIA_MIMETYPE_VIDEO_AV1) { + noPrivateBuffers(); // TODO: account for our buffers here. + noInputReferences(); + noOutputReferences(); + noInputLatency(); + noTimeStretch(); + + addParameter(DefineParam(mAttrib, C2_PARAMKEY_COMPONENT_ATTRIBUTES) + .withConstValue(new C2ComponentAttributesSetting( + C2Component::ATTRIB_IS_TEMPORAL)) + .build()); + + addParameter( + DefineParam(mSize, C2_PARAMKEY_PICTURE_SIZE) + .withDefault(new C2StreamPictureSizeInfo::output(0u, 320, 240)) + .withFields({ + C2F(mSize, width).inRange(2, 2048, 2), + C2F(mSize, height).inRange(2, 2048, 2), + }) + .withSetter(SizeSetter) + .build()); + + addParameter(DefineParam(mProfileLevel, C2_PARAMKEY_PROFILE_LEVEL) + .withDefault(new C2StreamProfileLevelInfo::input( + 0u, C2Config::PROFILE_AV1_0, C2Config::LEVEL_AV1_2_1)) + .withFields({C2F(mProfileLevel, profile) + .oneOf({C2Config::PROFILE_AV1_0, + C2Config::PROFILE_AV1_1}), + C2F(mProfileLevel, level) + .oneOf({ + C2Config::LEVEL_AV1_2, + C2Config::LEVEL_AV1_2_1, + C2Config::LEVEL_AV1_2_2, + C2Config::LEVEL_AV1_3, + C2Config::LEVEL_AV1_3_1, + C2Config::LEVEL_AV1_3_2, + })}) + .withSetter(ProfileLevelSetter, mSize) + .build()); + + mHdr10PlusInfoInput = C2StreamHdr10PlusInfo::input::AllocShared(0); + addParameter( + DefineParam(mHdr10PlusInfoInput, C2_PARAMKEY_INPUT_HDR10_PLUS_INFO) + .withDefault(mHdr10PlusInfoInput) + .withFields({ + C2F(mHdr10PlusInfoInput, m.value).any(), + }) + .withSetter(Hdr10PlusInfoInputSetter) + .build()); + + mHdr10PlusInfoOutput = C2StreamHdr10PlusInfo::output::AllocShared(0); + addParameter( + DefineParam(mHdr10PlusInfoOutput, C2_PARAMKEY_OUTPUT_HDR10_PLUS_INFO) + .withDefault(mHdr10PlusInfoOutput) + .withFields({ + C2F(mHdr10PlusInfoOutput, m.value).any(), + }) + .withSetter(Hdr10PlusInfoOutputSetter) + .build()); + + addParameter( + DefineParam(mMaxSize, C2_PARAMKEY_MAX_PICTURE_SIZE) + .withDefault(new C2StreamMaxPictureSizeTuning::output(0u, 320, 240)) + .withFields({ + C2F(mSize, width).inRange(2, 2048, 2), + C2F(mSize, height).inRange(2, 2048, 2), + }) + .withSetter(MaxPictureSizeSetter, mSize) + .build()); + + addParameter(DefineParam(mMaxInputSize, C2_PARAMKEY_INPUT_MAX_BUFFER_SIZE) + .withDefault(new C2StreamMaxBufferSizeInfo::input( + 0u, 320 * 240 * 3 / 4)) + .withFields({ + C2F(mMaxInputSize, value).any(), + }) + .calculatedAs(MaxInputSizeSetter, mMaxSize) + .build()); + + C2ChromaOffsetStruct locations[1] = {C2ChromaOffsetStruct::ITU_YUV_420_0()}; + std::shared_ptr<C2StreamColorInfo::output> defaultColorInfo = + C2StreamColorInfo::output::AllocShared(1u, 0u, 8u /* bitDepth */, + C2Color::YUV_420); + memcpy(defaultColorInfo->m.locations, locations, sizeof(locations)); + + defaultColorInfo = C2StreamColorInfo::output::AllocShared( + {C2ChromaOffsetStruct::ITU_YUV_420_0()}, 0u, 8u /* bitDepth */, + C2Color::YUV_420); + helper->addStructDescriptors<C2ChromaOffsetStruct>(); + + addParameter(DefineParam(mColorInfo, C2_PARAMKEY_CODED_COLOR_INFO) + .withConstValue(defaultColorInfo) + .build()); + + addParameter( + DefineParam(mDefaultColorAspects, C2_PARAMKEY_DEFAULT_COLOR_ASPECTS) + .withDefault(new C2StreamColorAspectsTuning::output( + 0u, C2Color::RANGE_UNSPECIFIED, C2Color::PRIMARIES_UNSPECIFIED, + C2Color::TRANSFER_UNSPECIFIED, C2Color::MATRIX_UNSPECIFIED)) + .withFields( + {C2F(mDefaultColorAspects, range) + .inRange(C2Color::RANGE_UNSPECIFIED, C2Color::RANGE_OTHER), + C2F(mDefaultColorAspects, primaries) + .inRange(C2Color::PRIMARIES_UNSPECIFIED, + C2Color::PRIMARIES_OTHER), + C2F(mDefaultColorAspects, transfer) + .inRange(C2Color::TRANSFER_UNSPECIFIED, + C2Color::TRANSFER_OTHER), + C2F(mDefaultColorAspects, matrix) + .inRange(C2Color::MATRIX_UNSPECIFIED, + C2Color::MATRIX_OTHER)}) + .withSetter(DefaultColorAspectsSetter) + .build()); + + // TODO: support more formats? + addParameter(DefineParam(mPixelFormat, C2_PARAMKEY_PIXEL_FORMAT) + .withConstValue(new C2StreamPixelFormatInfo::output( + 0u, HAL_PIXEL_FORMAT_YCBCR_420_888)) + .build()); + } + + static C2R SizeSetter(bool mayBlock, + const C2P<C2StreamPictureSizeInfo::output> &oldMe, + C2P<C2StreamPictureSizeInfo::output> &me) { + (void)mayBlock; + C2R res = C2R::Ok(); + if (!me.F(me.v.width).supportsAtAll(me.v.width)) { + res = res.plus(C2SettingResultBuilder::BadValue(me.F(me.v.width))); + me.set().width = oldMe.v.width; + } + if (!me.F(me.v.height).supportsAtAll(me.v.height)) { + res = res.plus(C2SettingResultBuilder::BadValue(me.F(me.v.height))); + me.set().height = oldMe.v.height; + } + return res; + } + + static C2R MaxPictureSizeSetter( + bool mayBlock, C2P<C2StreamMaxPictureSizeTuning::output> &me, + const C2P<C2StreamPictureSizeInfo::output> &size) { + (void)mayBlock; + // TODO: get max width/height from the size's field helpers vs. + // hardcoding + me.set().width = c2_min(c2_max(me.v.width, size.v.width), 4096u); + me.set().height = c2_min(c2_max(me.v.height, size.v.height), 4096u); + return C2R::Ok(); + } + + static C2R MaxInputSizeSetter( + bool mayBlock, C2P<C2StreamMaxBufferSizeInfo::input> &me, + const C2P<C2StreamMaxPictureSizeTuning::output> &maxSize) { + (void)mayBlock; + // assume compression ratio of 2 + me.set().value = + (((maxSize.v.width + 63) / 64) * ((maxSize.v.height + 63) / 64) * 3072); + return C2R::Ok(); + } + + static C2R DefaultColorAspectsSetter( + bool mayBlock, C2P<C2StreamColorAspectsTuning::output> &me) { + (void)mayBlock; + if (me.v.range > C2Color::RANGE_OTHER) { + me.set().range = C2Color::RANGE_OTHER; + } + if (me.v.primaries > C2Color::PRIMARIES_OTHER) { + me.set().primaries = C2Color::PRIMARIES_OTHER; + } + if (me.v.transfer > C2Color::TRANSFER_OTHER) { + me.set().transfer = C2Color::TRANSFER_OTHER; + } + if (me.v.matrix > C2Color::MATRIX_OTHER) { + me.set().matrix = C2Color::MATRIX_OTHER; + } + return C2R::Ok(); + } + + static C2R ProfileLevelSetter( + bool mayBlock, C2P<C2StreamProfileLevelInfo::input> &me, + const C2P<C2StreamPictureSizeInfo::output> &size) { + (void)mayBlock; + (void)size; + (void)me; // TODO: validate + return C2R::Ok(); + } + + std::shared_ptr<C2StreamColorAspectsTuning::output> + getDefaultColorAspects_l() { + return mDefaultColorAspects; + } + + static C2R Hdr10PlusInfoInputSetter(bool mayBlock, + C2P<C2StreamHdr10PlusInfo::input> &me) { + (void)mayBlock; + (void)me; // TODO: validate + return C2R::Ok(); + } + + static C2R Hdr10PlusInfoOutputSetter(bool mayBlock, + C2P<C2StreamHdr10PlusInfo::output> &me) { + (void)mayBlock; + (void)me; // TODO: validate + return C2R::Ok(); + } + + private: + std::shared_ptr<C2StreamProfileLevelInfo::input> mProfileLevel; + std::shared_ptr<C2StreamPictureSizeInfo::output> mSize; + std::shared_ptr<C2StreamMaxPictureSizeTuning::output> mMaxSize; + std::shared_ptr<C2StreamMaxBufferSizeInfo::input> mMaxInputSize; + std::shared_ptr<C2StreamColorInfo::output> mColorInfo; + std::shared_ptr<C2StreamPixelFormatInfo::output> mPixelFormat; + std::shared_ptr<C2StreamColorAspectsTuning::output> mDefaultColorAspects; + std::shared_ptr<C2StreamHdr10PlusInfo::input> mHdr10PlusInfoInput; + std::shared_ptr<C2StreamHdr10PlusInfo::output> mHdr10PlusInfoOutput; +}; + +C2SoftGav1Dec::C2SoftGav1Dec(const char *name, c2_node_id_t id, + const std::shared_ptr<IntfImpl> &intfImpl) + : SimpleC2Component( + std::make_shared<SimpleInterface<IntfImpl>>(name, id, intfImpl)), + mIntf(intfImpl), + mCodecCtx(nullptr) { + gettimeofday(&mTimeStart, nullptr); + gettimeofday(&mTimeEnd, nullptr); +} + +C2SoftGav1Dec::~C2SoftGav1Dec() { onRelease(); } + +c2_status_t C2SoftGav1Dec::onInit() { + return initDecoder() ? C2_OK : C2_CORRUPTED; +} + +c2_status_t C2SoftGav1Dec::onStop() { + mSignalledError = false; + mSignalledOutputEos = false; + return C2_OK; +} + +void C2SoftGav1Dec::onReset() { + (void)onStop(); + c2_status_t err = onFlush_sm(); + if (err != C2_OK) { + ALOGW("Failed to flush the av1 decoder. Trying to hard reset."); + destroyDecoder(); + if (!initDecoder()) { + ALOGE("Hard reset failed."); + } + } +} + +void C2SoftGav1Dec::onRelease() { destroyDecoder(); } + +c2_status_t C2SoftGav1Dec::onFlush_sm() { + Libgav1StatusCode status = + mCodecCtx->EnqueueFrame(/*data=*/nullptr, /*size=*/0, + /*user_private_data=*/0); + if (status != kLibgav1StatusOk) { + ALOGE("Failed to flush av1 decoder. status: %d.", status); + return C2_CORRUPTED; + } + + // Dequeue frame (if any) that was enqueued previously. + const libgav1::DecoderBuffer *buffer; + status = mCodecCtx->DequeueFrame(&buffer); + if (status != kLibgav1StatusOk) { + ALOGE("Failed to dequeue frame after flushing the av1 decoder. status: %d", + status); + return C2_CORRUPTED; + } + + mSignalledError = false; + mSignalledOutputEos = false; + + return C2_OK; +} + +static int GetCPUCoreCount() { + int cpuCoreCount = 1; +#if defined(_SC_NPROCESSORS_ONLN) + cpuCoreCount = sysconf(_SC_NPROCESSORS_ONLN); +#else + // _SC_NPROC_ONLN must be defined... + cpuCoreCount = sysconf(_SC_NPROC_ONLN); +#endif + CHECK(cpuCoreCount >= 1); + ALOGV("Number of CPU cores: %d", cpuCoreCount); + return cpuCoreCount; +} + +bool C2SoftGav1Dec::initDecoder() { + mSignalledError = false; + mSignalledOutputEos = false; + mCodecCtx.reset(new libgav1::Decoder()); + + if (mCodecCtx == nullptr) { + ALOGE("mCodecCtx is null"); + return false; + } + + libgav1::DecoderSettings settings = {}; + settings.threads = GetCPUCoreCount(); + + ALOGV("Using libgav1 AV1 software decoder."); + Libgav1StatusCode status = mCodecCtx->Init(&settings); + if (status != kLibgav1StatusOk) { + ALOGE("av1 decoder failed to initialize. status: %d.", status); + return false; + } + + return true; +} + +void C2SoftGav1Dec::destroyDecoder() { mCodecCtx = nullptr; } + +void fillEmptyWork(const std::unique_ptr<C2Work> &work) { + uint32_t flags = 0; + if (work->input.flags & C2FrameData::FLAG_END_OF_STREAM) { + flags |= C2FrameData::FLAG_END_OF_STREAM; + ALOGV("signalling eos"); + } + work->worklets.front()->output.flags = (C2FrameData::flags_t)flags; + work->worklets.front()->output.buffers.clear(); + work->worklets.front()->output.ordinal = work->input.ordinal; + work->workletsProcessed = 1u; +} + +void C2SoftGav1Dec::finishWork(uint64_t index, + const std::unique_ptr<C2Work> &work, + const std::shared_ptr<C2GraphicBlock> &block) { + std::shared_ptr<C2Buffer> buffer = + createGraphicBuffer(block, C2Rect(mWidth, mHeight)); + auto fillWork = [buffer, index](const std::unique_ptr<C2Work> &work) { + uint32_t flags = 0; + if ((work->input.flags & C2FrameData::FLAG_END_OF_STREAM) && + (c2_cntr64_t(index) == work->input.ordinal.frameIndex)) { + flags |= C2FrameData::FLAG_END_OF_STREAM; + ALOGV("signalling eos"); + } + work->worklets.front()->output.flags = (C2FrameData::flags_t)flags; + work->worklets.front()->output.buffers.clear(); + work->worklets.front()->output.buffers.push_back(buffer); + work->worklets.front()->output.ordinal = work->input.ordinal; + work->workletsProcessed = 1u; + }; + if (work && c2_cntr64_t(index) == work->input.ordinal.frameIndex) { + fillWork(work); + } else { + finish(index, fillWork); + } +} + +void C2SoftGav1Dec::process(const std::unique_ptr<C2Work> &work, + const std::shared_ptr<C2BlockPool> &pool) { + work->result = C2_OK; + work->workletsProcessed = 0u; + work->worklets.front()->output.configUpdate.clear(); + work->worklets.front()->output.flags = work->input.flags; + if (mSignalledError || mSignalledOutputEos) { + work->result = C2_BAD_VALUE; + return; + } + + size_t inOffset = 0u; + size_t inSize = 0u; + C2ReadView rView = mDummyReadView; + if (!work->input.buffers.empty()) { + rView = work->input.buffers[0]->data().linearBlocks().front().map().get(); + inSize = rView.capacity(); + if (inSize && rView.error()) { + ALOGE("read view map failed %d", rView.error()); + work->result = C2_CORRUPTED; + return; + } + } + + bool codecConfig = + ((work->input.flags & C2FrameData::FLAG_CODEC_CONFIG) != 0); + bool eos = ((work->input.flags & C2FrameData::FLAG_END_OF_STREAM) != 0); + + ALOGV("in buffer attr. size %zu timestamp %d frameindex %d, flags %x", inSize, + (int)work->input.ordinal.timestamp.peeku(), + (int)work->input.ordinal.frameIndex.peeku(), work->input.flags); + + if (codecConfig) { + fillEmptyWork(work); + return; + } + + int64_t frameIndex = work->input.ordinal.frameIndex.peekll(); + if (inSize) { + uint8_t *bitstream = const_cast<uint8_t *>(rView.data() + inOffset); + int32_t decodeTime = 0; + int32_t delay = 0; + + GETTIME(&mTimeStart, nullptr); + TIME_DIFF(mTimeEnd, mTimeStart, delay); + + const Libgav1StatusCode status = + mCodecCtx->EnqueueFrame(bitstream, inSize, frameIndex); + + GETTIME(&mTimeEnd, nullptr); + TIME_DIFF(mTimeStart, mTimeEnd, decodeTime); + ALOGV("decodeTime=%4d delay=%4d\n", decodeTime, delay); + + if (status != kLibgav1StatusOk) { + ALOGE("av1 decoder failed to decode frame. status: %d.", status); + work->result = C2_CORRUPTED; + work->workletsProcessed = 1u; + mSignalledError = true; + return; + } + + } else { + const Libgav1StatusCode status = + mCodecCtx->EnqueueFrame(/*data=*/nullptr, /*size=*/0, + /*user_private_data=*/0); + if (status != kLibgav1StatusOk) { + ALOGE("Failed to flush av1 decoder. status: %d.", status); + work->result = C2_CORRUPTED; + work->workletsProcessed = 1u; + mSignalledError = true; + return; + } + } + + (void)outputBuffer(pool, work); + + if (eos) { + drainInternal(DRAIN_COMPONENT_WITH_EOS, pool, work); + mSignalledOutputEos = true; + } else if (!inSize) { + fillEmptyWork(work); + } +} + +static void copyOutputBufferToYV12Frame(uint8_t *dst, const uint8_t *srcY, + const uint8_t *srcU, + const uint8_t *srcV, size_t srcYStride, + size_t srcUStride, size_t srcVStride, + uint32_t width, uint32_t height) { + const size_t dstYStride = align(width, 16); + const size_t dstUVStride = align(dstYStride / 2, 16); + uint8_t *const dstStart = dst; + + for (size_t i = 0; i < height; ++i) { + memcpy(dst, srcY, width); + srcY += srcYStride; + dst += dstYStride; + } + + dst = dstStart + dstYStride * height; + for (size_t i = 0; i < height / 2; ++i) { + memcpy(dst, srcV, width / 2); + srcV += srcVStride; + dst += dstUVStride; + } + + dst = dstStart + (dstYStride * height) + (dstUVStride * height / 2); + for (size_t i = 0; i < height / 2; ++i) { + memcpy(dst, srcU, width / 2); + srcU += srcUStride; + dst += dstUVStride; + } +} + +static void convertYUV420Planar16ToY410(uint32_t *dst, const uint16_t *srcY, + const uint16_t *srcU, + const uint16_t *srcV, size_t srcYStride, + size_t srcUStride, size_t srcVStride, + size_t dstStride, size_t width, + size_t height) { + // Converting two lines at a time, slightly faster + for (size_t y = 0; y < height; y += 2) { + uint32_t *dstTop = (uint32_t *)dst; + uint32_t *dstBot = (uint32_t *)(dst + dstStride); + uint16_t *ySrcTop = (uint16_t *)srcY; + uint16_t *ySrcBot = (uint16_t *)(srcY + srcYStride); + uint16_t *uSrc = (uint16_t *)srcU; + uint16_t *vSrc = (uint16_t *)srcV; + + uint32_t u01, v01, y01, y23, y45, y67, uv0, uv1; + size_t x = 0; + for (; x < width - 3; x += 4) { + u01 = *((uint32_t *)uSrc); + uSrc += 2; + v01 = *((uint32_t *)vSrc); + vSrc += 2; + + y01 = *((uint32_t *)ySrcTop); + ySrcTop += 2; + y23 = *((uint32_t *)ySrcTop); + ySrcTop += 2; + y45 = *((uint32_t *)ySrcBot); + ySrcBot += 2; + y67 = *((uint32_t *)ySrcBot); + ySrcBot += 2; + + uv0 = (u01 & 0x3FF) | ((v01 & 0x3FF) << 20); + uv1 = (u01 >> 16) | ((v01 >> 16) << 20); + + *dstTop++ = 3 << 30 | ((y01 & 0x3FF) << 10) | uv0; + *dstTop++ = 3 << 30 | ((y01 >> 16) << 10) | uv0; + *dstTop++ = 3 << 30 | ((y23 & 0x3FF) << 10) | uv1; + *dstTop++ = 3 << 30 | ((y23 >> 16) << 10) | uv1; + + *dstBot++ = 3 << 30 | ((y45 & 0x3FF) << 10) | uv0; + *dstBot++ = 3 << 30 | ((y45 >> 16) << 10) | uv0; + *dstBot++ = 3 << 30 | ((y67 & 0x3FF) << 10) | uv1; + *dstBot++ = 3 << 30 | ((y67 >> 16) << 10) | uv1; + } + + // There should be at most 2 more pixels to process. Note that we don't + // need to consider odd case as the buffer is always aligned to even. + if (x < width) { + u01 = *uSrc; + v01 = *vSrc; + y01 = *((uint32_t *)ySrcTop); + y45 = *((uint32_t *)ySrcBot); + uv0 = (u01 & 0x3FF) | ((v01 & 0x3FF) << 20); + *dstTop++ = ((y01 & 0x3FF) << 10) | uv0; + *dstTop++ = ((y01 >> 16) << 10) | uv0; + *dstBot++ = ((y45 & 0x3FF) << 10) | uv0; + *dstBot++ = ((y45 >> 16) << 10) | uv0; + } + + srcY += srcYStride * 2; + srcU += srcUStride; + srcV += srcVStride; + dst += dstStride * 2; + } +} + +static void convertYUV420Planar16ToYUV420Planar( + uint8_t *dst, const uint16_t *srcY, const uint16_t *srcU, + const uint16_t *srcV, size_t srcYStride, size_t srcUStride, + size_t srcVStride, size_t dstStride, size_t width, size_t height) { + uint8_t *dstY = (uint8_t *)dst; + size_t dstYSize = dstStride * height; + size_t dstUVStride = align(dstStride / 2, 16); + size_t dstUVSize = dstUVStride * height / 2; + uint8_t *dstV = dstY + dstYSize; + uint8_t *dstU = dstV + dstUVSize; + + for (size_t y = 0; y < height; ++y) { + for (size_t x = 0; x < width; ++x) { + dstY[x] = (uint8_t)(srcY[x] >> 2); + } + + srcY += srcYStride; + dstY += dstStride; + } + + for (size_t y = 0; y < (height + 1) / 2; ++y) { + for (size_t x = 0; x < (width + 1) / 2; ++x) { + dstU[x] = (uint8_t)(srcU[x] >> 2); + dstV[x] = (uint8_t)(srcV[x] >> 2); + } + + srcU += srcUStride; + srcV += srcVStride; + dstU += dstUVStride; + dstV += dstUVStride; + } +} + +bool C2SoftGav1Dec::outputBuffer(const std::shared_ptr<C2BlockPool> &pool, + const std::unique_ptr<C2Work> &work) { + if (!(work && pool)) return false; + + const libgav1::DecoderBuffer *buffer; + const Libgav1StatusCode status = mCodecCtx->DequeueFrame(&buffer); + + if (status != kLibgav1StatusOk) { + ALOGE("av1 decoder DequeueFrame failed. status: %d.", status); + return false; + } + + // |buffer| can be NULL if status was equal to kLibgav1StatusOk. This is not + // an error. This could mean one of two things: + // - The EnqueueFrame() call was either a flush (called with nullptr). + // - The enqueued frame did not have any displayable frames. + if (!buffer) { + return false; + } + + const int width = buffer->displayed_width[0]; + const int height = buffer->displayed_height[0]; + if (width != mWidth || height != mHeight) { + mWidth = width; + mHeight = height; + + C2StreamPictureSizeInfo::output size(0u, mWidth, mHeight); + std::vector<std::unique_ptr<C2SettingResult>> failures; + c2_status_t err = mIntf->config({&size}, C2_MAY_BLOCK, &failures); + if (err == C2_OK) { + work->worklets.front()->output.configUpdate.push_back( + C2Param::Copy(size)); + } else { + ALOGE("Config update size failed"); + mSignalledError = true; + work->result = C2_CORRUPTED; + work->workletsProcessed = 1u; + return false; + } + } + + // TODO(vigneshv): Add support for monochrome videos since AV1 supports it. + CHECK(buffer->image_format == libgav1::kImageFormatYuv420); + + std::shared_ptr<C2GraphicBlock> block; + uint32_t format = HAL_PIXEL_FORMAT_YV12; + if (buffer->bitdepth == 10) { + IntfImpl::Lock lock = mIntf->lock(); + std::shared_ptr<C2StreamColorAspectsTuning::output> defaultColorAspects = + mIntf->getDefaultColorAspects_l(); + + if (defaultColorAspects->primaries == C2Color::PRIMARIES_BT2020 && + defaultColorAspects->matrix == C2Color::MATRIX_BT2020 && + defaultColorAspects->transfer == C2Color::TRANSFER_ST2084) { + format = HAL_PIXEL_FORMAT_RGBA_1010102; + } + } + C2MemoryUsage usage = {C2MemoryUsage::CPU_READ, C2MemoryUsage::CPU_WRITE}; + + c2_status_t err = pool->fetchGraphicBlock(align(mWidth, 16), mHeight, format, + usage, &block); + + if (err != C2_OK) { + ALOGE("fetchGraphicBlock for Output failed with status %d", err); + work->result = err; + return false; + } + + C2GraphicView wView = block->map().get(); + + if (wView.error()) { + ALOGE("graphic view map failed %d", wView.error()); + work->result = C2_CORRUPTED; + return false; + } + + ALOGV("provided (%dx%d) required (%dx%d), out frameindex %d", block->width(), + block->height(), mWidth, mHeight, (int)buffer->user_private_data); + + uint8_t *dst = const_cast<uint8_t *>(wView.data()[C2PlanarLayout::PLANE_Y]); + size_t srcYStride = buffer->stride[0]; + size_t srcUStride = buffer->stride[1]; + size_t srcVStride = buffer->stride[2]; + + if (buffer->bitdepth == 10) { + const uint16_t *srcY = (const uint16_t *)buffer->plane[0]; + const uint16_t *srcU = (const uint16_t *)buffer->plane[1]; + const uint16_t *srcV = (const uint16_t *)buffer->plane[2]; + + if (format == HAL_PIXEL_FORMAT_RGBA_1010102) { + convertYUV420Planar16ToY410( + (uint32_t *)dst, srcY, srcU, srcV, srcYStride / 2, srcUStride / 2, + srcVStride / 2, align(mWidth, 16), mWidth, mHeight); + } else { + convertYUV420Planar16ToYUV420Planar(dst, srcY, srcU, srcV, srcYStride / 2, + srcUStride / 2, srcVStride / 2, + align(mWidth, 16), mWidth, mHeight); + } + } else { + const uint8_t *srcY = (const uint8_t *)buffer->plane[0]; + const uint8_t *srcU = (const uint8_t *)buffer->plane[1]; + const uint8_t *srcV = (const uint8_t *)buffer->plane[2]; + copyOutputBufferToYV12Frame(dst, srcY, srcU, srcV, srcYStride, srcUStride, + srcVStride, mWidth, mHeight); + } + finishWork(buffer->user_private_data, work, std::move(block)); + block = nullptr; + return true; +} + +c2_status_t C2SoftGav1Dec::drainInternal( + uint32_t drainMode, const std::shared_ptr<C2BlockPool> &pool, + const std::unique_ptr<C2Work> &work) { + if (drainMode == NO_DRAIN) { + ALOGW("drain with NO_DRAIN: no-op"); + return C2_OK; + } + if (drainMode == DRAIN_CHAIN) { + ALOGW("DRAIN_CHAIN not supported"); + return C2_OMITTED; + } + + Libgav1StatusCode status = + mCodecCtx->EnqueueFrame(/*data=*/nullptr, /*size=*/0, + /*user_private_data=*/0); + if (status != kLibgav1StatusOk) { + ALOGE("Failed to flush av1 decoder. status: %d.", status); + return C2_CORRUPTED; + } + + while (outputBuffer(pool, work)) { + } + + if (drainMode == DRAIN_COMPONENT_WITH_EOS && work && + work->workletsProcessed == 0u) { + fillEmptyWork(work); + } + + return C2_OK; +} + +c2_status_t C2SoftGav1Dec::drain(uint32_t drainMode, + const std::shared_ptr<C2BlockPool> &pool) { + return drainInternal(drainMode, pool, nullptr); +} + +class C2SoftGav1Factory : public C2ComponentFactory { + public: + C2SoftGav1Factory() + : mHelper(std::static_pointer_cast<C2ReflectorHelper>( + GetCodec2PlatformComponentStore()->getParamReflector())) {} + + virtual c2_status_t createComponent( + c2_node_id_t id, std::shared_ptr<C2Component> *const component, + std::function<void(C2Component *)> deleter) override { + *component = std::shared_ptr<C2Component>( + new C2SoftGav1Dec(COMPONENT_NAME, id, + std::make_shared<C2SoftGav1Dec::IntfImpl>(mHelper)), + deleter); + return C2_OK; + } + + virtual c2_status_t createInterface( + c2_node_id_t id, std::shared_ptr<C2ComponentInterface> *const interface, + std::function<void(C2ComponentInterface *)> deleter) override { + *interface = std::shared_ptr<C2ComponentInterface>( + new SimpleInterface<C2SoftGav1Dec::IntfImpl>( + COMPONENT_NAME, id, + std::make_shared<C2SoftGav1Dec::IntfImpl>(mHelper)), + deleter); + return C2_OK; + } + + virtual ~C2SoftGav1Factory() override = default; + + private: + std::shared_ptr<C2ReflectorHelper> mHelper; +}; + +} // namespace android + +extern "C" ::C2ComponentFactory *CreateCodec2Factory() { + ALOGV("in %s", __func__); + return new ::android::C2SoftGav1Factory(); +} + +extern "C" void DestroyCodec2Factory(::C2ComponentFactory *factory) { + ALOGV("in %s", __func__); + delete factory; +}
diff --git a/media/codec2/components/gav1/C2SoftGav1Dec.h b/media/codec2/components/gav1/C2SoftGav1Dec.h new file mode 100644 index 0000000..a7c08bb --- /dev/null +++ b/media/codec2/components/gav1/C2SoftGav1Dec.h
@@ -0,0 +1,77 @@ +/* + * Copyright (C) 2019 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef ANDROID_C2_SOFT_GAV1_DEC_H_ +#define ANDROID_C2_SOFT_GAV1_DEC_H_ + +#include <SimpleC2Component.h> +#include "libgav1/src/decoder.h" +#include "libgav1/src/decoder_settings.h" + +#define GETTIME(a, b) gettimeofday(a, b); +#define TIME_DIFF(start, end, diff) \ + diff = (((end).tv_sec - (start).tv_sec) * 1000000) + \ + ((end).tv_usec - (start).tv_usec); + +namespace android { + +struct C2SoftGav1Dec : public SimpleC2Component { + class IntfImpl; + + C2SoftGav1Dec(const char* name, c2_node_id_t id, + const std::shared_ptr<IntfImpl>& intfImpl); + ~C2SoftGav1Dec(); + + // Begin SimpleC2Component overrides. + c2_status_t onInit() override; + c2_status_t onStop() override; + void onReset() override; + void onRelease() override; + c2_status_t onFlush_sm() override; + void process(const std::unique_ptr<C2Work>& work, + const std::shared_ptr<C2BlockPool>& pool) override; + c2_status_t drain(uint32_t drainMode, + const std::shared_ptr<C2BlockPool>& pool) override; + // End SimpleC2Component overrides. + + private: + std::shared_ptr<IntfImpl> mIntf; + std::unique_ptr<libgav1::Decoder> mCodecCtx; + + uint32_t mWidth; + uint32_t mHeight; + bool mSignalledOutputEos; + bool mSignalledError; + + struct timeval mTimeStart; // Time at the start of decode() + struct timeval mTimeEnd; // Time at the end of decode() + + bool initDecoder(); + void destroyDecoder(); + void finishWork(uint64_t index, const std::unique_ptr<C2Work>& work, + const std::shared_ptr<C2GraphicBlock>& block); + bool outputBuffer(const std::shared_ptr<C2BlockPool>& pool, + const std::unique_ptr<C2Work>& work); + c2_status_t drainInternal(uint32_t drainMode, + const std::shared_ptr<C2BlockPool>& pool, + const std::unique_ptr<C2Work>& work); + + C2_DO_NOT_COPY(C2SoftGav1Dec); +}; + +} // namespace android + +#endif // ANDROID_C2_SOFT_GAV1_DEC_H_
diff --git a/media/codec2/components/hevc/C2SoftHevcDec.cpp b/media/codec2/components/hevc/C2SoftHevcDec.cpp index 7232572..df677c2 100644 --- a/media/codec2/components/hevc/C2SoftHevcDec.cpp +++ b/media/codec2/components/hevc/C2SoftHevcDec.cpp
@@ -33,7 +33,8 @@ namespace { constexpr char COMPONENT_NAME[] = "c2.android.hevc.decoder"; - +constexpr uint32_t kDefaultOutputDelay = 8; +constexpr uint32_t kMaxOutputDelay = 16; } // namespace class C2SoftHevcDec::IntfImpl : public SimpleInterface<void>::BaseParams { @@ -54,7 +55,9 @@ // TODO: Proper support for reorder depth. addParameter( DefineParam(mActualOutputDelay, C2_PARAMKEY_OUTPUT_DELAY) - .withConstValue(new C2PortActualDelayTuning::output(8u)) + .withDefault(new C2PortActualDelayTuning::output(kDefaultOutputDelay)) + .withFields({C2F(mActualOutputDelay, value).inRange(0, kMaxOutputDelay)}) + .withSetter(Setter<decltype(*mActualOutputDelay)>::StrictValueWithNoDeps) .build()); addParameter( @@ -327,6 +330,7 @@ mDecHandle(nullptr), mOutBufferFlush(nullptr), mIvColorformat(IV_YUV_420P), + mOutputDelay(kDefaultOutputDelay), mWidth(320), mHeight(240), mHeaderDecoded(false), @@ -877,6 +881,26 @@ work->result = C2_CORRUPTED; return; } + if (s_decode_op.i4_reorder_depth >= 0 && mOutputDelay != s_decode_op.i4_reorder_depth) { + mOutputDelay = s_decode_op.i4_reorder_depth; + ALOGV("New Output delay %d ", mOutputDelay); + + C2PortActualDelayTuning::output outputDelay(mOutputDelay); + std::vector<std::unique_ptr<C2SettingResult>> failures; + c2_status_t err = + mIntf->config({&outputDelay}, C2_MAY_BLOCK, &failures); + if (err == OK) { + work->worklets.front()->output.configUpdate.push_back( + C2Param::Copy(outputDelay)); + } else { + ALOGE("Cannot set output delay"); + mSignalledError = true; + work->workletsProcessed = 1u; + work->result = C2_CORRUPTED; + return; + } + continue; + } if (0 < s_decode_op.u4_pic_wd && 0 < s_decode_op.u4_pic_ht) { if (mHeaderDecoded == false) { mHeaderDecoded = true;
diff --git a/media/codec2/components/hevc/C2SoftHevcDec.h b/media/codec2/components/hevc/C2SoftHevcDec.h index b7664e6..ce63a6c 100644 --- a/media/codec2/components/hevc/C2SoftHevcDec.h +++ b/media/codec2/components/hevc/C2SoftHevcDec.h
@@ -115,7 +115,7 @@ size_t mNumCores; IV_COLOR_FORMAT_T mIvColorformat; - + uint32_t mOutputDelay; uint32_t mWidth; uint32_t mHeight; uint32_t mStride;
diff --git a/media/codec2/components/mpeg4_h263/C2SoftMpeg4Enc.cpp b/media/codec2/components/mpeg4_h263/C2SoftMpeg4Enc.cpp index 36053f6..54c8c47 100644 --- a/media/codec2/components/mpeg4_h263/C2SoftMpeg4Enc.cpp +++ b/media/codec2/components/mpeg4_h263/C2SoftMpeg4Enc.cpp
@@ -517,9 +517,11 @@ if (layout.planes[layout.PLANE_Y].colInc == 1 && layout.planes[layout.PLANE_U].colInc == 1 && layout.planes[layout.PLANE_V].colInc == 1 + && yStride == align(width, 16) && uStride == vStride && yStride == 2 * vStride) { - // I420 compatible - planes are already set up above + // I420 compatible with yStride being equal to aligned width + // planes are already set up above break; }
diff --git a/media/codec2/components/vpx/C2SoftVpxDec.cpp b/media/codec2/components/vpx/C2SoftVpxDec.cpp index a52ca15..a759e8f 100644 --- a/media/codec2/components/vpx/C2SoftVpxDec.cpp +++ b/media/codec2/components/vpx/C2SoftVpxDec.cpp
@@ -593,12 +593,10 @@ } } - int64_t frameIndex = work->input.ordinal.frameIndex.peekll(); - if (inSize) { uint8_t *bitstream = const_cast<uint8_t *>(rView.data() + inOffset); vpx_codec_err_t err = vpx_codec_decode( - mCodecCtx, bitstream, inSize, &frameIndex, 0); + mCodecCtx, bitstream, inSize, &work->input.ordinal.frameIndex, 0); if (err != VPX_CODEC_OK) { ALOGE("on2 decoder failed to decode frame. err: %d", err); mSignalledError = true; @@ -608,7 +606,20 @@ } } - (void)outputBuffer(pool, work); + status_t err = outputBuffer(pool, work); + if (err == NOT_ENOUGH_DATA) { + if (inSize > 0) { + ALOGV("Maybe non-display frame at %lld.", + work->input.ordinal.frameIndex.peekll()); + // send the work back with empty buffer. + inSize = 0; + } + } else if (err != OK) { + ALOGD("Error while getting the output frame out"); + // work->result would be already filled; do fillEmptyWork() below to + // send the work back. + inSize = 0; + } if (eos) { drainInternal(DRAIN_COMPONENT_WITH_EOS, pool, work); @@ -742,16 +753,16 @@ } return; } -bool C2SoftVpxDec::outputBuffer( +status_t C2SoftVpxDec::outputBuffer( const std::shared_ptr<C2BlockPool> &pool, const std::unique_ptr<C2Work> &work) { - if (!(work && pool)) return false; + if (!(work && pool)) return BAD_VALUE; vpx_codec_iter_t iter = nullptr; vpx_image_t *img = vpx_codec_get_frame(mCodecCtx, &iter); - if (!img) return false; + if (!img) return NOT_ENOUGH_DATA; if (img->d_w != mWidth || img->d_h != mHeight) { mWidth = img->d_w; @@ -768,7 +779,7 @@ mSignalledError = true; work->workletsProcessed = 1u; work->result = C2_CORRUPTED; - return false; + return UNKNOWN_ERROR; } } @@ -791,18 +802,19 @@ if (err != C2_OK) { ALOGE("fetchGraphicBlock for Output failed with status %d", err); work->result = err; - return false; + return UNKNOWN_ERROR; } C2GraphicView wView = block->map().get(); if (wView.error()) { ALOGE("graphic view map failed %d", wView.error()); work->result = C2_CORRUPTED; - return false; + return UNKNOWN_ERROR; } - ALOGV("provided (%dx%d) required (%dx%d), out frameindex %d", - block->width(), block->height(), mWidth, mHeight, (int)*(int64_t *)img->user_priv); + ALOGV("provided (%dx%d) required (%dx%d), out frameindex %lld", + block->width(), block->height(), mWidth, mHeight, + ((c2_cntr64_t *)img->user_priv)->peekll()); uint8_t *dst = const_cast<uint8_t *>(wView.data()[C2PlanarLayout::PLANE_Y]); size_t srcYStride = img->stride[VPX_PLANE_Y]; @@ -858,8 +870,8 @@ dstYStride, dstUVStride, mWidth, mHeight); } - finishWork(*(int64_t *)img->user_priv, work, std::move(block)); - return true; + finishWork(((c2_cntr64_t *)img->user_priv)->peekull(), work, std::move(block)); + return OK; } c2_status_t C2SoftVpxDec::drainInternal( @@ -875,7 +887,7 @@ return C2_OMITTED; } - while ((outputBuffer(pool, work))) { + while (outputBuffer(pool, work) == OK) { } if (drainMode == DRAIN_COMPONENT_WITH_EOS &&
diff --git a/media/codec2/components/vpx/C2SoftVpxDec.h b/media/codec2/components/vpx/C2SoftVpxDec.h index e51bcee..2065165 100644 --- a/media/codec2/components/vpx/C2SoftVpxDec.h +++ b/media/codec2/components/vpx/C2SoftVpxDec.h
@@ -85,7 +85,7 @@ status_t destroyDecoder(); void finishWork(uint64_t index, const std::unique_ptr<C2Work> &work, const std::shared_ptr<C2GraphicBlock> &block); - bool outputBuffer( + status_t outputBuffer( const std::shared_ptr<C2BlockPool> &pool, const std::unique_ptr<C2Work> &work); c2_status_t drainInternal(
diff --git a/media/codec2/hidl/1.0/utils/Android.bp b/media/codec2/hidl/1.0/utils/Android.bp index 63fe36b..f1f1536 100644 --- a/media/codec2/hidl/1.0/utils/Android.bp +++ b/media/codec2/hidl/1.0/utils/Android.bp
@@ -24,7 +24,7 @@ "libgui", "libhidlbase", "liblog", - "libstagefright_bufferpool@2.0", + "libstagefright_bufferpool@2.0.1", "libui", "libutils", ], @@ -37,7 +37,7 @@ "android.hardware.media.c2@1.0", "libcodec2", "libgui", - "libstagefright_bufferpool@2.0", + "libstagefright_bufferpool@2.0.1", "libui", ], } @@ -83,7 +83,7 @@ "libhidltransport", "libhwbinder", "liblog", - "libstagefright_bufferpool@2.0", + "libstagefright_bufferpool@2.0.1", "libstagefright_bufferqueue_helper", "libui", "libutils", @@ -98,7 +98,7 @@ "libcodec2", "libcodec2_vndk", "libhidlbase", - "libstagefright_bufferpool@2.0", + "libstagefright_bufferpool@2.0.1", "libui", ], }
diff --git a/media/codec2/hidl/1.0/utils/types.cpp b/media/codec2/hidl/1.0/utils/types.cpp index 07dbf67..04fa59c 100644 --- a/media/codec2/hidl/1.0/utils/types.cpp +++ b/media/codec2/hidl/1.0/utils/types.cpp
@@ -1434,6 +1434,11 @@ d->type = C2BaseBlock::GRAPHIC; return true; } + if (cHandle) { + // Though we got cloned handle, creating block failed. + native_handle_close(cHandle); + native_handle_delete(cHandle); + } LOG(ERROR) << "Unknown handle type in BaseBlock::pooledBlock."; return false;
diff --git a/media/codec2/hidl/client/Android.bp b/media/codec2/hidl/client/Android.bp index 6038a40..e184223 100644 --- a/media/codec2/hidl/client/Android.bp +++ b/media/codec2/hidl/client/Android.bp
@@ -19,7 +19,7 @@ "libhidlbase", "libhidltransport", "liblog", - "libstagefright_bufferpool@2.0", + "libstagefright_bufferpool@2.0.1", "libui", "libutils", ],
diff --git a/media/codec2/hidl/client/client.cpp b/media/codec2/hidl/client/client.cpp index 5ed54f1..c747190 100644 --- a/media/codec2/hidl/client/client.cpp +++ b/media/codec2/hidl/client/client.cpp
@@ -125,6 +125,9 @@ if (!mClient) { mClient = Codec2Client::_CreateFromIndex(mIndex); } + CHECK(mClient) << "Failed to create Codec2Client to service \"" + << GetServiceNames()[mIndex] << "\". (Index = " + << mIndex << ")."; return mClient; } @@ -832,6 +835,7 @@ c2_status_t Codec2Client::ForAllServices( const std::string &key, + size_t numberOfAttempts, std::function<c2_status_t(const std::shared_ptr<Codec2Client>&)> predicate) { c2_status_t status = C2_NO_INIT; // no IComponentStores present @@ -860,33 +864,45 @@ for (size_t index : indices) { Cache& cache = Cache::List()[index]; - std::shared_ptr<Codec2Client> client{cache.getClient()}; - if (client) { + for (size_t tries = numberOfAttempts; tries > 0; --tries) { + std::shared_ptr<Codec2Client> client{cache.getClient()}; status = predicate(client); if (status == C2_OK) { std::scoped_lock lock{key2IndexMutex}; key2Index[key] = index; // update last known client index return C2_OK; + } else if (status == C2_TRANSACTION_FAILED) { + LOG(WARNING) << "\"" << key << "\" failed for service \"" + << client->getName() + << "\" due to transaction failure. " + << "(Service may have crashed.)" + << (tries > 1 ? " Retrying..." : ""); + cache.invalidate(); + continue; } - } - if (wasMapped) { - LOG(INFO) << "Could not find \"" << key << "\"" - " in the last instance. Retrying..."; - wasMapped = false; - cache.invalidate(); + if (wasMapped) { + LOG(INFO) << "\"" << key << "\" became invalid in service \"" + << client->getName() << "\". Retrying..."; + wasMapped = false; + } + break; } } - return status; // return the last status from a valid client + return status; // return the last status from a valid client } std::shared_ptr<Codec2Client::Component> Codec2Client::CreateComponentByName( const char* componentName, const std::shared_ptr<Listener>& listener, - std::shared_ptr<Codec2Client>* owner) { + std::shared_ptr<Codec2Client>* owner, + size_t numberOfAttempts) { + std::string key{"create:"}; + key.append(componentName); std::shared_ptr<Component> component; c2_status_t status = ForAllServices( - componentName, + key, + numberOfAttempts, [owner, &component, componentName, &listener]( const std::shared_ptr<Codec2Client> &client) -> c2_status_t { @@ -907,8 +923,9 @@ return status; }); if (status != C2_OK) { - LOG(DEBUG) << "Could not create component \"" << componentName << "\". " - "Status = " << status << "."; + LOG(DEBUG) << "Failed to create component \"" << componentName + << "\" from all known services. " + "Last returned status = " << status << "."; } return component; } @@ -916,10 +933,14 @@ std::shared_ptr<Codec2Client::Interface> Codec2Client::CreateInterfaceByName( const char* interfaceName, - std::shared_ptr<Codec2Client>* owner) { + std::shared_ptr<Codec2Client>* owner, + size_t numberOfAttempts) { + std::string key{"create:"}; + key.append(interfaceName); std::shared_ptr<Interface> interface; c2_status_t status = ForAllServices( - interfaceName, + key, + numberOfAttempts, [owner, &interface, interfaceName]( const std::shared_ptr<Codec2Client> &client) -> c2_status_t { @@ -939,8 +960,9 @@ return status; }); if (status != C2_OK) { - LOG(DEBUG) << "Could not create interface \"" << interfaceName << "\". " - "Status = " << status << "."; + LOG(DEBUG) << "Failed to create interface \"" << interfaceName + << "\" from all known services. " + "Last returned status = " << status << "."; } return interface; }
diff --git a/media/codec2/hidl/client/include/codec2/hidl/client.h b/media/codec2/hidl/client/include/codec2/hidl/client.h index b8a7fb5..c37407f 100644 --- a/media/codec2/hidl/client/include/codec2/hidl/client.h +++ b/media/codec2/hidl/client/include/codec2/hidl/client.h
@@ -179,17 +179,21 @@ static std::vector<std::shared_ptr<Codec2Client>> CreateFromAllServices(); // Try to create a component with a given name from all known - // IComponentStore services. + // IComponentStore services. numberOfAttempts determines the number of times + // to retry the HIDL call if the transaction fails. static std::shared_ptr<Component> CreateComponentByName( char const* componentName, std::shared_ptr<Listener> const& listener, - std::shared_ptr<Codec2Client>* owner = nullptr); + std::shared_ptr<Codec2Client>* owner = nullptr, + size_t numberOfAttempts = 10); // Try to create a component interface with a given name from all known - // IComponentStore services. + // IComponentStore services. numberOfAttempts determines the number of times + // to retry the HIDL call if the transaction fails. static std::shared_ptr<Interface> CreateInterfaceByName( char const* interfaceName, - std::shared_ptr<Codec2Client>* owner = nullptr); + std::shared_ptr<Codec2Client>* owner = nullptr, + size_t numberOfAttempts = 10); // List traits from all known IComponentStore services. static std::vector<C2Component::Traits> const& ListComponents(); @@ -204,11 +208,25 @@ protected: sp<Base> mBase; - // Finds the first store where the predicate returns OK, and returns the last - // predicate result. Uses key to remember the last store found, and if cached, - // it tries that store before trying all stores (one retry). + // Finds the first store where the predicate returns C2_OK and returns the + // last predicate result. The predicate will be tried on all stores. The + // function will return C2_OK the first time the predicate returns C2_OK, + // or it will return the value from the last time that predicate is tried. + // (The latter case corresponds to a failure on every store.) The order of + // the stores to try is the same as the return value of GetServiceNames(). + // + // key is used to remember the last store with which the predicate last + // succeeded. If the last successful store is cached, it will be tried + // first before all the stores are tried. Note that the last successful + // store will be tried twice---first before all the stores, and another time + // with all the stores. + // + // If an attempt to evaluate the predicate results in a transaction failure, + // repeated attempts will be made until the predicate returns without a + // transaction failure or numberOfAttempts attempts have been made. static c2_status_t ForAllServices( const std::string& key, + size_t numberOfAttempts, std::function<c2_status_t(std::shared_ptr<Codec2Client> const&)> predicate);
diff --git a/media/codec2/sfplugin/CCodec.cpp b/media/codec2/sfplugin/CCodec.cpp index 9d1cc60..4a31953 100644 --- a/media/codec2/sfplugin/CCodec.cpp +++ b/media/codec2/sfplugin/CCodec.cpp
@@ -375,7 +375,11 @@ // consumer usage is queried earlier. - ALOGD("ISConfig%s", status.str().c_str()); + if (status.str().empty()) { + ALOGD("ISConfig not changed"); + } else { + ALOGD("ISConfig%s", status.str().c_str()); + } return err; } @@ -810,9 +814,17 @@ } { - double value; - if (msg->findDouble("time-lapse-fps", &value)) { - config->mISConfig->mCaptureFps = value; + bool captureFpsFound = false; + double timeLapseFps; + float captureRate; + if (msg->findDouble("time-lapse-fps", &timeLapseFps)) { + config->mISConfig->mCaptureFps = timeLapseFps; + captureFpsFound = true; + } else if (msg->findAsFloat(KEY_CAPTURE_RATE, &captureRate)) { + config->mISConfig->mCaptureFps = captureRate; + captureFpsFound = true; + } + if (captureFpsFound) { (void)msg->findAsFloat(KEY_FRAME_RATE, &config->mISConfig->mCodedFps); } }
diff --git a/media/codec2/sfplugin/CCodecBufferChannel.cpp b/media/codec2/sfplugin/CCodecBufferChannel.cpp index 8308292..2efb987 100644 --- a/media/codec2/sfplugin/CCodecBufferChannel.cpp +++ b/media/codec2/sfplugin/CCodecBufferChannel.cpp
@@ -948,7 +948,11 @@ uint32_t outputGeneration; { Mutexed<OutputSurface>::Locked output(mOutputSurface); - output->maxDequeueBuffers = numOutputSlots + reorderDepth.value + kRenderingDepth; + output->maxDequeueBuffers = numOutputSlots + + reorderDepth.value + kRenderingDepth; + if (!secure) { + output->maxDequeueBuffers += numInputSlots; + } outputSurface = output->surface ? output->surface->getIGraphicBufferProducer() : nullptr; if (outputSurface) { @@ -1328,13 +1332,18 @@ case C2PortReorderBufferDepthTuning::CORE_INDEX: { C2PortReorderBufferDepthTuning::output reorderDepth; if (reorderDepth.updateFrom(*param)) { + bool secure = mComponent->getName().find(".secure") != std::string::npos; mReorderStash.lock()->setDepth(reorderDepth.value); ALOGV("[%s] onWorkDone: updated reorder depth to %u", mName, reorderDepth.value); size_t numOutputSlots = mOutput.lock()->numSlots; + size_t numInputSlots = mInput.lock()->numSlots; Mutexed<OutputSurface>::Locked output(mOutputSurface); - output->maxDequeueBuffers = - numOutputSlots + reorderDepth.value + kRenderingDepth; + output->maxDequeueBuffers = numOutputSlots + + reorderDepth.value + kRenderingDepth; + if (!secure) { + output->maxDequeueBuffers += numInputSlots; + } if (output->surface) { output->surface->setMaxDequeuedBufferCount(output->maxDequeueBuffers); } @@ -1378,10 +1387,12 @@ if (outputDelay.updateFrom(*param)) { ALOGV("[%s] onWorkDone: updating output delay %u", mName, outputDelay.value); + bool secure = mComponent->getName().find(".secure") != std::string::npos; (void)mPipelineWatcher.lock()->outputDelay(outputDelay.value); bool outputBuffersChanged = false; size_t numOutputSlots = 0; + size_t numInputSlots = mInput.lock()->numSlots; { Mutexed<Output>::Locked output(mOutput); output->outputDelay = outputDelay.value; @@ -1407,6 +1418,9 @@ uint32_t depth = mReorderStash.lock()->depth(); Mutexed<OutputSurface>::Locked output(mOutputSurface); output->maxDequeueBuffers = numOutputSlots + depth + kRenderingDepth; + if (!secure) { + output->maxDequeueBuffers += numInputSlots; + } if (output->surface) { output->surface->setMaxDequeuedBufferCount(output->maxDequeueBuffers); }
diff --git a/media/codec2/sfplugin/CCodecConfig.cpp b/media/codec2/sfplugin/CCodecConfig.cpp index 1cfdc19..5adcd94 100644 --- a/media/codec2/sfplugin/CCodecConfig.cpp +++ b/media/codec2/sfplugin/CCodecConfig.cpp
@@ -235,7 +235,10 @@ const std::vector<ConfigMapper> &getConfigMappersForSdkKey(std::string key) const { auto it = mConfigMappers.find(key); if (it == mConfigMappers.end()) { - ALOGD("no c2 equivalents for %s", key.c_str()); + if (mComplained.count(key) == 0) { + ALOGD("no c2 equivalents for %s", key.c_str()); + mComplained.insert(key); + } return NO_MAPPERS; } ALOGV("found %zu eqs for %s", it->second.size(), key.c_str()); @@ -304,6 +307,7 @@ private: std::map<SdkKey, std::vector<ConfigMapper>> mConfigMappers; + mutable std::set<std::string> mComplained; }; const std::vector<ConfigMapper> StandardParams::NO_MAPPERS; @@ -508,7 +512,8 @@ .limitTo(D::ENCODER & D::VIDEO)); // convert to timestamp base add(ConfigMapper(KEY_I_FRAME_INTERVAL, C2_PARAMKEY_SYNC_FRAME_INTERVAL, "value") - .withMappers([](C2Value v) -> C2Value { + .limitTo(D::VIDEO & D::ENCODER & D::CONFIG) + .withMapper([](C2Value v) -> C2Value { // convert from i32 to float int32_t i32Value; float fpValue; @@ -518,12 +523,6 @@ return int64_t(c2_min(1000000 * fpValue + 0.5, (double)INT64_MAX)); } return C2Value(); - }, [](C2Value v) -> C2Value { - int64_t i64; - if (v.get(&i64)) { - return float(i64) / 1000000; - } - return C2Value(); })); // remove when codecs switch to proper coding.gop (add support for calculating gop) deprecated(ConfigMapper("i-frame-period", "coding.gop", "intra-period") @@ -1033,7 +1032,25 @@ } ReflectedParamUpdater::Dict reflected = mParamUpdater->getParams(paramPointers); - ALOGD("c2 config is %s", reflected.debugString().c_str()); + std::string config = reflected.debugString(); + std::set<std::string> configLines; + std::string diff; + for (size_t start = 0; start != std::string::npos; ) { + size_t end = config.find('\n', start); + size_t count = (end == std::string::npos) + ? std::string::npos + : end - start + 1; + std::string line = config.substr(start, count); + configLines.insert(line); + if (mLastConfig.count(line) == 0) { + diff.append(line); + } + start = (end == std::string::npos) ? std::string::npos : end + 1; + } + if (!diff.empty()) { + ALOGD("c2 config diff is %s", diff.c_str()); + } + mLastConfig.swap(configLines); bool changed = false; if (domain & mInputDomain) {
diff --git a/media/codec2/sfplugin/CCodecConfig.h b/media/codec2/sfplugin/CCodecConfig.h index 3bafe3f..a61c8b7 100644 --- a/media/codec2/sfplugin/CCodecConfig.h +++ b/media/codec2/sfplugin/CCodecConfig.h
@@ -134,6 +134,8 @@ /// For now support a validation function. std::map<C2Param::Index, LocalParamValidator> mLocalParams; + std::set<std::string> mLastConfig; + CCodecConfig(); /// initializes the members required to manage the format: descriptors, reflector,
diff --git a/media/codec2/sfplugin/PipelineWatcher.cpp b/media/codec2/sfplugin/PipelineWatcher.cpp index 74d14e8..0ee9056 100644 --- a/media/codec2/sfplugin/PipelineWatcher.cpp +++ b/media/codec2/sfplugin/PipelineWatcher.cpp
@@ -146,7 +146,7 @@ std::chrono::duration_cast<std::chrono::milliseconds>(elapsed).count()); durations.push_back(elapsed); } - std::nth_element(durations.begin(), durations.end(), durations.begin() + n, + std::nth_element(durations.begin(), durations.begin() + n, durations.end(), std::greater<Clock::duration>()); return durations[n]; }
diff --git a/media/codec2/vndk/Android.bp b/media/codec2/vndk/Android.bp index b6ddfab..52cc7ad 100644 --- a/media/codec2/vndk/Android.bp +++ b/media/codec2/vndk/Android.bp
@@ -66,7 +66,7 @@ "liblog", "libnativewindow", "libstagefright_foundation", - "libstagefright_bufferpool@2.0", + "libstagefright_bufferpool@2.0.1", "libui", "libutils", ],
diff --git a/media/codec2/vndk/C2Buffer.cpp b/media/codec2/vndk/C2Buffer.cpp index 710b536..2d99b53 100644 --- a/media/codec2/vndk/C2Buffer.cpp +++ b/media/codec2/vndk/C2Buffer.cpp
@@ -413,17 +413,14 @@ std::shared_ptr<C2LinearAllocation> alloc; if (C2AllocatorIon::isValid(cHandle)) { - native_handle_t *handle = native_handle_clone(cHandle); - if (handle) { - c2_status_t err = sAllocator->priorLinearAllocation(handle, &alloc); - const std::shared_ptr<C2PooledBlockPoolData> poolData = - std::make_shared<C2PooledBlockPoolData>(data); - if (err == C2_OK && poolData) { - // TODO: config params? - std::shared_ptr<C2LinearBlock> block = - _C2BlockFactory::CreateLinearBlock(alloc, poolData); - return block; - } + c2_status_t err = sAllocator->priorLinearAllocation(cHandle, &alloc); + const std::shared_ptr<C2PooledBlockPoolData> poolData = + std::make_shared<C2PooledBlockPoolData>(data); + if (err == C2_OK && poolData) { + // TODO: config params? + std::shared_ptr<C2LinearBlock> block = + _C2BlockFactory::CreateLinearBlock(alloc, poolData); + return block; } } return nullptr; @@ -674,17 +671,14 @@ ResultStatus status = mBufferPoolManager->allocate( mConnectionId, params, &cHandle, &bufferPoolData); if (status == ResultStatus::OK) { - native_handle_t *handle = native_handle_clone(cHandle); - if (handle) { - std::shared_ptr<C2LinearAllocation> alloc; - std::shared_ptr<C2PooledBlockPoolData> poolData = - std::make_shared<C2PooledBlockPoolData>(bufferPoolData); - c2_status_t err = mAllocator->priorLinearAllocation(handle, &alloc); - if (err == C2_OK && poolData && alloc) { - *block = _C2BlockFactory::CreateLinearBlock(alloc, poolData, 0, capacity); - if (*block) { - return C2_OK; - } + std::shared_ptr<C2LinearAllocation> alloc; + std::shared_ptr<C2PooledBlockPoolData> poolData = + std::make_shared<C2PooledBlockPoolData>(bufferPoolData); + c2_status_t err = mAllocator->priorLinearAllocation(cHandle, &alloc); + if (err == C2_OK && poolData && alloc) { + *block = _C2BlockFactory::CreateLinearBlock(alloc, poolData, 0, capacity); + if (*block) { + return C2_OK; } } return C2_NO_MEMORY; @@ -710,19 +704,16 @@ ResultStatus status = mBufferPoolManager->allocate( mConnectionId, params, &cHandle, &bufferPoolData); if (status == ResultStatus::OK) { - native_handle_t *handle = native_handle_clone(cHandle); - if (handle) { - std::shared_ptr<C2GraphicAllocation> alloc; - std::shared_ptr<C2PooledBlockPoolData> poolData = - std::make_shared<C2PooledBlockPoolData>(bufferPoolData); - c2_status_t err = mAllocator->priorGraphicAllocation( - handle, &alloc); - if (err == C2_OK && poolData && alloc) { - *block = _C2BlockFactory::CreateGraphicBlock( - alloc, poolData, C2Rect(width, height)); - if (*block) { - return C2_OK; - } + std::shared_ptr<C2GraphicAllocation> alloc; + std::shared_ptr<C2PooledBlockPoolData> poolData = + std::make_shared<C2PooledBlockPoolData>(bufferPoolData); + c2_status_t err = mAllocator->priorGraphicAllocation( + cHandle, &alloc); + if (err == C2_OK && poolData && alloc) { + *block = _C2BlockFactory::CreateGraphicBlock( + alloc, poolData, C2Rect(width, height)); + if (*block) { + return C2_OK; } } return C2_NO_MEMORY; @@ -1117,17 +1108,14 @@ std::shared_ptr<C2GraphicAllocation> alloc; if (C2AllocatorGralloc::isValid(cHandle)) { - native_handle_t *handle = native_handle_clone(cHandle); - if (handle) { - c2_status_t err = sAllocator->priorGraphicAllocation(handle, &alloc); - const std::shared_ptr<C2PooledBlockPoolData> poolData = - std::make_shared<C2PooledBlockPoolData>(data); - if (err == C2_OK && poolData) { - // TODO: config setup? - std::shared_ptr<C2GraphicBlock> block = - _C2BlockFactory::CreateGraphicBlock(alloc, poolData); - return block; - } + c2_status_t err = sAllocator->priorGraphicAllocation(cHandle, &alloc); + const std::shared_ptr<C2PooledBlockPoolData> poolData = + std::make_shared<C2PooledBlockPoolData>(data); + if (err == C2_OK && poolData) { + // TODO: config setup? + std::shared_ptr<C2GraphicBlock> block = + _C2BlockFactory::CreateGraphicBlock(alloc, poolData); + return block; } } return nullptr;
diff --git a/media/codec2/vndk/C2Store.cpp b/media/codec2/vndk/C2Store.cpp index f8afa7c..5b2bd7b 100644 --- a/media/codec2/vndk/C2Store.cpp +++ b/media/codec2/vndk/C2Store.cpp
@@ -848,7 +848,8 @@ emplace("libcodec2_soft_amrnbenc.so"); emplace("libcodec2_soft_amrwbdec.so"); emplace("libcodec2_soft_amrwbenc.so"); - emplace("libcodec2_soft_av1dec.so"); + //emplace("libcodec2_soft_av1dec_aom.so"); // deprecated for the gav1 implementation + emplace("libcodec2_soft_av1dec_gav1.so"); emplace("libcodec2_soft_avcdec.so"); emplace("libcodec2_soft_avcenc.so"); emplace("libcodec2_soft_flacdec.so");
diff --git a/media/libaaudio/src/client/AudioStreamInternalCapture.cpp b/media/libaaudio/src/client/AudioStreamInternalCapture.cpp index a6cc45b..366cc87 100644 --- a/media/libaaudio/src/client/AudioStreamInternalCapture.cpp +++ b/media/libaaudio/src/client/AudioStreamInternalCapture.cpp
@@ -89,7 +89,11 @@ if (mAudioEndpoint.isFreeRunning()) { //ALOGD("AudioStreamInternalCapture::processDataNow() - update remote counter"); // Update data queue based on the timing model. - int64_t estimatedRemoteCounter = mClockModel.convertTimeToPosition(currentNanoTime); + // Jitter in the DSP can cause late writes to the FIFO. + // This might be caused by resampling. + // We want to read the FIFO after the latest possible time + // that the DSP could have written the data. + int64_t estimatedRemoteCounter = mClockModel.convertLatestTimeToPosition(currentNanoTime); // TODO refactor, maybe use setRemoteCounter() mAudioEndpoint.setDataWriteCounter(estimatedRemoteCounter); } @@ -139,7 +143,7 @@ // the writeCounter might have just advanced in the background, // causing us to sleep until a later burst. int64_t nextPosition = mAudioEndpoint.getDataReadCounter() + mFramesPerBurst; - wakeTime = mClockModel.convertPositionToTime(nextPosition); + wakeTime = mClockModel.convertPositionToLatestTime(nextPosition); } break; default:
diff --git a/media/libaaudio/src/client/IsochronousClockModel.cpp b/media/libaaudio/src/client/IsochronousClockModel.cpp index 747d0e1..9abdf53 100644 --- a/media/libaaudio/src/client/IsochronousClockModel.cpp +++ b/media/libaaudio/src/client/IsochronousClockModel.cpp
@@ -19,12 +19,11 @@ #include <log/log.h> #include <stdint.h> +#include <algorithm> #include "utility/AudioClock.h" #include "IsochronousClockModel.h" -#define MIN_LATENESS_NANOS (10 * AAUDIO_NANOS_PER_MICROSECOND) - using namespace aaudio; IsochronousClockModel::IsochronousClockModel() @@ -32,7 +31,7 @@ , mMarkerNanoTime(0) , mSampleRate(48000) , mFramesPerBurst(64) - , mMaxLatenessInNanos(0) + , mMaxMeasuredLatenessNanos(0) , mState(STATE_STOPPED) { } @@ -41,8 +40,7 @@ } void IsochronousClockModel::setPositionAndTime(int64_t framePosition, int64_t nanoTime) { - ALOGV("setPositionAndTime(%lld, %lld)", - (long long) framePosition, (long long) nanoTime); + ALOGV("setPositionAndTime, %lld, %lld", (long long) framePosition, (long long) nanoTime); mMarkerFramePosition = framePosition; mMarkerNanoTime = nanoTime; } @@ -54,7 +52,9 @@ } void IsochronousClockModel::stop(int64_t nanoTime) { - ALOGV("stop(nanos = %lld)\n", (long long) nanoTime); + ALOGD("stop(nanos = %lld) max lateness = %d micros\n", + (long long) nanoTime, + (int) (mMaxMeasuredLatenessNanos / 1000)); setPositionAndTime(convertTimeToPosition(nanoTime), nanoTime); // TODO should we set position? mState = STATE_STOPPED; @@ -69,9 +69,10 @@ } void IsochronousClockModel::processTimestamp(int64_t framePosition, int64_t nanoTime) { -// ALOGD("processTimestamp() - framePosition = %lld at nanoTime %llu", -// (long long)framePosition, -// (long long)nanoTime); + mTimestampCount++; +// Log position and time in CSV format so we can import it easily into spreadsheets. + //ALOGD("%s() CSV, %d, %lld, %lld", __func__, + //mTimestampCount, (long long)framePosition, (long long)nanoTime); int64_t framesDelta = framePosition - mMarkerFramePosition; int64_t nanosDelta = nanoTime - mMarkerNanoTime; if (nanosDelta < 1000) { @@ -108,17 +109,56 @@ case STATE_RUNNING: if (nanosDelta < expectedNanosDelta) { // Earlier than expected timestamp. - // This data is probably more accurate so use it. - // or we may be drifting due to a slow HW clock. -// ALOGD("processTimestamp() - STATE_RUNNING - %d < %d micros - EARLY", -// (int) (nanosDelta / 1000), (int)(expectedNanosDelta / 1000)); + // This data is probably more accurate, so use it. + // Or we may be drifting due to a fast HW clock. + //int microsDelta = (int) (nanosDelta / 1000); + //int expectedMicrosDelta = (int) (expectedNanosDelta / 1000); + //ALOGD("%s() - STATE_RUNNING - #%d, %4d micros EARLY", + //__func__, mTimestampCount, expectedMicrosDelta - microsDelta); + setPositionAndTime(framePosition, nanoTime); - } else if (nanosDelta > (expectedNanosDelta + mMaxLatenessInNanos)) { - // Later than expected timestamp. -// ALOGD("processTimestamp() - STATE_RUNNING - %d > %d + %d micros - LATE", -// (int) (nanosDelta / 1000), (int)(expectedNanosDelta / 1000), -// (int) (mMaxLatenessInNanos / 1000)); - setPositionAndTime(framePosition - mFramesPerBurst, nanoTime - mMaxLatenessInNanos); + } else if (nanosDelta > (expectedNanosDelta + (2 * mBurstPeriodNanos))) { + // In this case we do not update mMaxMeasuredLatenessNanos because it + // would force it too high. + // mMaxMeasuredLatenessNanos should range from 1 to 2 * mBurstPeriodNanos + //int32_t measuredLatenessNanos = (int32_t)(nanosDelta - expectedNanosDelta); + //ALOGD("%s() - STATE_RUNNING - #%d, lateness %d - max %d = %4d micros VERY LATE", + //__func__, + //mTimestampCount, + //measuredLatenessNanos / 1000, + //mMaxMeasuredLatenessNanos / 1000, + //(measuredLatenessNanos - mMaxMeasuredLatenessNanos) / 1000 + //); + + // This typically happens when we are modelling a service instead of a DSP. + setPositionAndTime(framePosition, nanoTime - (2 * mBurstPeriodNanos)); + } else if (nanosDelta > (expectedNanosDelta + mMaxMeasuredLatenessNanos)) { + //int32_t previousLatenessNanos = mMaxMeasuredLatenessNanos; + mMaxMeasuredLatenessNanos = (int32_t)(nanosDelta - expectedNanosDelta); + + //ALOGD("%s() - STATE_RUNNING - #%d, newmax %d - oldmax %d = %4d micros LATE", + //__func__, + //mTimestampCount, + //mMaxMeasuredLatenessNanos / 1000, + //previousLatenessNanos / 1000, + //(mMaxMeasuredLatenessNanos - previousLatenessNanos) / 1000 + //); + + // When we are late, it may be because of preemption in the kernel, + // or timing jitter caused by resampling in the DSP, + // or we may be drifting due to a slow HW clock. + // We add slight drift value just in case there is actual long term drift + // forward caused by a slower clock. + // If the clock is faster than the model will get pushed earlier + // by the code in the preceding branch. + // The two opposing forces should allow the model to track the real clock + // over a long time. + int64_t driftingTime = mMarkerNanoTime + expectedNanosDelta + kDriftNanos; + setPositionAndTime(framePosition, driftingTime); + //ALOGD("%s() - #%d, max lateness = %d micros", + //__func__, + //mTimestampCount, + //(int) (mMaxMeasuredLatenessNanos / 1000)); } break; default: @@ -138,9 +178,12 @@ update(); } +// Update expected lateness based on sampleRate and framesPerBurst void IsochronousClockModel::update() { - int64_t nanosLate = convertDeltaPositionToTime(mFramesPerBurst); // uses mSampleRate - mMaxLatenessInNanos = (nanosLate > MIN_LATENESS_NANOS) ? nanosLate : MIN_LATENESS_NANOS; + mBurstPeriodNanos = convertDeltaPositionToTime(mFramesPerBurst); // uses mSampleRate + // Timestamps may be late by up to a burst because we are randomly sampling the time period + // after the DSP position is actually updated. + mMaxMeasuredLatenessNanos = mBurstPeriodNanos; } int64_t IsochronousClockModel::convertDeltaPositionToTime(int64_t framesDelta) const { @@ -183,11 +226,25 @@ return position; } +int32_t IsochronousClockModel::getLateTimeOffsetNanos() const { + // This will never be < 0 because mMaxLatenessNanos starts at + // mBurstPeriodNanos and only gets bigger. + return (mMaxMeasuredLatenessNanos - mBurstPeriodNanos) + kExtraLatenessNanos; +} + +int64_t IsochronousClockModel::convertPositionToLatestTime(int64_t framePosition) const { + return convertPositionToTime(framePosition) + getLateTimeOffsetNanos(); +} + +int64_t IsochronousClockModel::convertLatestTimeToPosition(int64_t nanoTime) const { + return convertTimeToPosition(nanoTime - getLateTimeOffsetNanos()); +} + void IsochronousClockModel::dump() const { ALOGD("mMarkerFramePosition = %lld", (long long) mMarkerFramePosition); ALOGD("mMarkerNanoTime = %lld", (long long) mMarkerNanoTime); ALOGD("mSampleRate = %6d", mSampleRate); ALOGD("mFramesPerBurst = %6d", mFramesPerBurst); - ALOGD("mMaxLatenessInNanos = %6d", mMaxLatenessInNanos); + ALOGD("mMaxMeasuredLatenessNanos = %6d", mMaxMeasuredLatenessNanos); ALOGD("mState = %6d", mState); }
diff --git a/media/libaaudio/src/client/IsochronousClockModel.h b/media/libaaudio/src/client/IsochronousClockModel.h index 46ca48e..582bf4e 100644 --- a/media/libaaudio/src/client/IsochronousClockModel.h +++ b/media/libaaudio/src/client/IsochronousClockModel.h
@@ -18,6 +18,7 @@ #define ANDROID_AAUDIO_ISOCHRONOUS_CLOCK_MODEL_H #include <stdint.h> +#include "utility/AudioClock.h" namespace aaudio { @@ -79,6 +80,15 @@ int64_t convertPositionToTime(int64_t framePosition) const; /** + * Calculate the latest estimated time that the stream will be at that position. + * The more jittery the clock is then the later this will be. + * + * @param framePosition + * @return time in nanoseconds + */ + int64_t convertPositionToLatestTime(int64_t framePosition) const; + + /** * Calculate an estimated position where the stream will be at the specified time. * * @param nanoTime time of interest @@ -87,6 +97,18 @@ int64_t convertTimeToPosition(int64_t nanoTime) const; /** + * Calculate the corresponding estimated position based on the specified time being + * the latest possible time. + * + * For the same nanoTime, this may return an earlier position than + * convertTimeToPosition(). + * + * @param nanoTime + * @return position in frames + */ + int64_t convertLatestTimeToPosition(int64_t nanoTime) const; + + /** * @param framesDelta difference in frames * @return duration in nanoseconds */ @@ -101,6 +123,9 @@ void dump() const; private: + + int32_t getLateTimeOffsetNanos() const; + enum clock_model_state_t { STATE_STOPPED, STATE_STARTING, @@ -108,13 +133,23 @@ STATE_RUNNING }; + // Amount of time to drift forward when we get a late timestamp. + // This value was calculated to allow tracking of a clock with 50 ppm error. + static constexpr int32_t kDriftNanos = 10 * 1000; + // TODO review value of kExtraLatenessNanos + static constexpr int32_t kExtraLatenessNanos = 100 * 1000; + int64_t mMarkerFramePosition; int64_t mMarkerNanoTime; int32_t mSampleRate; int32_t mFramesPerBurst; - int32_t mMaxLatenessInNanos; + int32_t mBurstPeriodNanos; + // Includes mBurstPeriodNanos because we sample randomly over time. + int32_t mMaxMeasuredLatenessNanos; clock_model_state_t mState; + int32_t mTimestampCount = 0; + void update(); };
diff --git a/media/libaaudio/src/core/AudioStream.cpp b/media/libaaudio/src/core/AudioStream.cpp index 9b77223..f5c97d8 100644 --- a/media/libaaudio/src/core/AudioStream.cpp +++ b/media/libaaudio/src/core/AudioStream.cpp
@@ -473,7 +473,7 @@ void AudioStream::MyPlayerBase::registerWithAudioManager() { if (!mRegistered) { - init(android::PLAYER_TYPE_AAUDIO, AUDIO_USAGE_MEDIA); + init(android::PLAYER_TYPE_AAUDIO, AAudioConvert_usageToInternal(mParent->getUsage())); mRegistered = true; } }
diff --git a/media/libaudioclient/IAudioFlinger.cpp b/media/libaudioclient/IAudioFlinger.cpp index dd95e34..efa0512 100644 --- a/media/libaudioclient/IAudioFlinger.cpp +++ b/media/libaudioclient/IAudioFlinger.cpp
@@ -1339,10 +1339,14 @@ } case GET_EFFECT_DESCRIPTOR: { CHECK_INTERFACE(IAudioFlinger, data, reply); - effect_uuid_t uuid; - data.read(&uuid, sizeof(effect_uuid_t)); - effect_uuid_t type; - data.read(&type, sizeof(effect_uuid_t)); + effect_uuid_t uuid = {}; + if (data.read(&uuid, sizeof(effect_uuid_t)) != NO_ERROR) { + android_errorWriteLog(0x534e4554, "139417189"); + } + effect_uuid_t type = {}; + if (data.read(&type, sizeof(effect_uuid_t)) != NO_ERROR) { + android_errorWriteLog(0x534e4554, "139417189"); + } uint32_t preferredTypeFlag = data.readUint32(); effect_descriptor_t desc = {}; status_t status = getEffectDescriptor(&uuid, &type, preferredTypeFlag, &desc);
diff --git a/media/libeffects/lvm/wrapper/Bundle/EffectBundle.cpp b/media/libeffects/lvm/wrapper/Bundle/EffectBundle.cpp index 3fbbc09..10dda19 100644 --- a/media/libeffects/lvm/wrapper/Bundle/EffectBundle.cpp +++ b/media/libeffects/lvm/wrapper/Bundle/EffectBundle.cpp
@@ -302,6 +302,8 @@ for (int i = 0; i < FIVEBAND_NUMBANDS; i++) { pContext->pBundledContext->bandGaindB[i] = EQNB_5BandSoftPresets[i]; } + pContext->pBundledContext->effectProcessCalled = 0; + pContext->pBundledContext->effectInDrain = 0; ALOGV("\tEffectCreate - Calling LvmBundle_init"); ret = LvmBundle_init(pContext); @@ -394,6 +396,8 @@ // Clear the instantiated flag for the effect // protect agains the case where an effect is un-instantiated without being disabled + + int &effectInDrain = pContext->pBundledContext->effectInDrain; if(pContext->EffectType == LVM_BASS_BOOST) { ALOGV("\tEffectRelease LVM_BASS_BOOST Clearing global intstantiated flag"); pSessionContext->bBassInstantiated = LVM_FALSE; @@ -418,12 +422,16 @@ } else if(pContext->EffectType == LVM_VOLUME) { ALOGV("\tEffectRelease LVM_VOLUME Clearing global intstantiated flag"); pSessionContext->bVolumeInstantiated = LVM_FALSE; - if (pContext->pBundledContext->bVolumeEnabled == LVM_TRUE){ + // There is no samplesToExitCount for volume so we also use the drain flag to check + // if we should decrement the effects enabled. + if (pContext->pBundledContext->bVolumeEnabled == LVM_TRUE + || (effectInDrain & 1 << LVM_VOLUME) != 0) { pContext->pBundledContext->NumberEffectsEnabled--; } } else { ALOGV("\tLVM_ERROR : EffectRelease : Unsupported effect\n\n\n\n\n\n\n"); } + effectInDrain &= ~(1 << pContext->EffectType); // no need to drain if released // Disable effect, in this case ignore errors (return codes) // if an effect has already been disabled @@ -3124,8 +3132,9 @@ int Effect_setEnabled(EffectContext *pContext, bool enabled) { - ALOGV("\tEffect_setEnabled() type %d, enabled %d", pContext->EffectType, enabled); - + ALOGV("%s effectType %d, enabled %d, currently enabled %d", __func__, + pContext->EffectType, enabled, pContext->pBundledContext->NumberEffectsEnabled); + int &effectInDrain = pContext->pBundledContext->effectInDrain; if (enabled) { // Bass boost or Virtualizer can be temporarily disabled if playing over device speaker due // to their nature. @@ -3139,6 +3148,7 @@ if(pContext->pBundledContext->SamplesToExitCountBb <= 0){ pContext->pBundledContext->NumberEffectsEnabled++; } + effectInDrain &= ~(1 << LVM_BASS_BOOST); pContext->pBundledContext->SamplesToExitCountBb = (LVM_INT32)(pContext->pBundledContext->SamplesPerSecond*0.1); pContext->pBundledContext->bBassEnabled = LVM_TRUE; @@ -3152,6 +3162,7 @@ if(pContext->pBundledContext->SamplesToExitCountEq <= 0){ pContext->pBundledContext->NumberEffectsEnabled++; } + effectInDrain &= ~(1 << LVM_EQUALIZER); pContext->pBundledContext->SamplesToExitCountEq = (LVM_INT32)(pContext->pBundledContext->SamplesPerSecond*0.1); pContext->pBundledContext->bEqualizerEnabled = LVM_TRUE; @@ -3164,6 +3175,7 @@ if(pContext->pBundledContext->SamplesToExitCountVirt <= 0){ pContext->pBundledContext->NumberEffectsEnabled++; } + effectInDrain &= ~(1 << LVM_VIRTUALIZER); pContext->pBundledContext->SamplesToExitCountVirt = (LVM_INT32)(pContext->pBundledContext->SamplesPerSecond*0.1); pContext->pBundledContext->bVirtualizerEnabled = LVM_TRUE; @@ -3174,7 +3186,10 @@ ALOGV("\tEffect_setEnabled() LVM_VOLUME is already enabled"); return -EINVAL; } - pContext->pBundledContext->NumberEffectsEnabled++; + if ((effectInDrain & 1 << LVM_VOLUME) == 0) { + pContext->pBundledContext->NumberEffectsEnabled++; + } + effectInDrain &= ~(1 << LVM_VOLUME); pContext->pBundledContext->bVolumeEnabled = LVM_TRUE; break; default: @@ -3192,6 +3207,7 @@ return -EINVAL; } pContext->pBundledContext->bBassEnabled = LVM_FALSE; + effectInDrain |= 1 << LVM_BASS_BOOST; break; case LVM_EQUALIZER: if (pContext->pBundledContext->bEqualizerEnabled == LVM_FALSE) { @@ -3199,6 +3215,7 @@ return -EINVAL; } pContext->pBundledContext->bEqualizerEnabled = LVM_FALSE; + effectInDrain |= 1 << LVM_EQUALIZER; break; case LVM_VIRTUALIZER: if (pContext->pBundledContext->bVirtualizerEnabled == LVM_FALSE) { @@ -3206,6 +3223,7 @@ return -EINVAL; } pContext->pBundledContext->bVirtualizerEnabled = LVM_FALSE; + effectInDrain |= 1 << LVM_VIRTUALIZER; break; case LVM_VOLUME: if (pContext->pBundledContext->bVolumeEnabled == LVM_FALSE) { @@ -3213,6 +3231,7 @@ return -EINVAL; } pContext->pBundledContext->bVolumeEnabled = LVM_FALSE; + effectInDrain |= 1 << LVM_VOLUME; break; default: ALOGV("\tEffect_setEnabled() invalid effect type"); @@ -3283,6 +3302,38 @@ ALOGV("\tLVM_ERROR : Effect_process() ERROR NULL INPUT POINTER OR FRAME COUNT IS WRONG"); return -EINVAL; } + + int &effectProcessCalled = pContext->pBundledContext->effectProcessCalled; + int &effectInDrain = pContext->pBundledContext->effectInDrain; + if ((effectProcessCalled & 1 << pContext->EffectType) != 0) { + ALOGW("Effect %d already called", pContext->EffectType); + const int undrainedEffects = effectInDrain & ~effectProcessCalled; + if ((undrainedEffects & 1 << LVM_BASS_BOOST) != 0) { + ALOGW("Draining BASS_BOOST"); + pContext->pBundledContext->SamplesToExitCountBb = 0; + --pContext->pBundledContext->NumberEffectsEnabled; + effectInDrain &= ~(1 << LVM_BASS_BOOST); + } + if ((undrainedEffects & 1 << LVM_EQUALIZER) != 0) { + ALOGW("Draining EQUALIZER"); + pContext->pBundledContext->SamplesToExitCountEq = 0; + --pContext->pBundledContext->NumberEffectsEnabled; + effectInDrain &= ~(1 << LVM_EQUALIZER); + } + if ((undrainedEffects & 1 << LVM_VIRTUALIZER) != 0) { + ALOGW("Draining VIRTUALIZER"); + pContext->pBundledContext->SamplesToExitCountVirt = 0; + --pContext->pBundledContext->NumberEffectsEnabled; + effectInDrain &= ~(1 << LVM_VIRTUALIZER); + } + if ((undrainedEffects & 1 << LVM_VOLUME) != 0) { + ALOGW("Draining VOLUME"); + --pContext->pBundledContext->NumberEffectsEnabled; + effectInDrain &= ~(1 << LVM_VOLUME); + } + } + effectProcessCalled |= 1 << pContext->EffectType; + if ((pContext->pBundledContext->bBassEnabled == LVM_FALSE)&& (pContext->EffectType == LVM_BASS_BOOST)){ //ALOGV("\tEffect_process() LVM_BASS_BOOST Effect is not enabled"); @@ -3291,9 +3342,12 @@ //ALOGV("\tEffect_process: Waiting to turn off BASS_BOOST, %d samples left", // pContext->pBundledContext->SamplesToExitCountBb); } - if(pContext->pBundledContext->SamplesToExitCountBb <= 0) { + if (pContext->pBundledContext->SamplesToExitCountBb <= 0) { status = -ENODATA; - pContext->pBundledContext->NumberEffectsEnabled--; + if ((effectInDrain & 1 << LVM_BASS_BOOST) != 0) { + pContext->pBundledContext->NumberEffectsEnabled--; + effectInDrain &= ~(1 << LVM_BASS_BOOST); + } ALOGV("\tEffect_process() this is the last frame for LVM_BASS_BOOST"); } } @@ -3301,7 +3355,10 @@ (pContext->EffectType == LVM_VOLUME)){ //ALOGV("\tEffect_process() LVM_VOLUME Effect is not enabled"); status = -ENODATA; - pContext->pBundledContext->NumberEffectsEnabled--; + if ((effectInDrain & 1 << LVM_VOLUME) != 0) { + pContext->pBundledContext->NumberEffectsEnabled--; + effectInDrain &= ~(1 << LVM_VOLUME); + } } if ((pContext->pBundledContext->bEqualizerEnabled == LVM_FALSE)&& (pContext->EffectType == LVM_EQUALIZER)){ @@ -3311,9 +3368,12 @@ //ALOGV("\tEffect_process: Waiting to turn off EQUALIZER, %d samples left", // pContext->pBundledContext->SamplesToExitCountEq); } - if(pContext->pBundledContext->SamplesToExitCountEq <= 0) { + if (pContext->pBundledContext->SamplesToExitCountEq <= 0) { status = -ENODATA; - pContext->pBundledContext->NumberEffectsEnabled--; + if ((effectInDrain & 1 << LVM_EQUALIZER) != 0) { + pContext->pBundledContext->NumberEffectsEnabled--; + effectInDrain &= ~(1 << LVM_EQUALIZER); + } ALOGV("\tEffect_process() this is the last frame for LVM_EQUALIZER"); } } @@ -3326,9 +3386,12 @@ //ALOGV("\tEffect_process: Waiting for to turn off VIRTUALIZER, %d samples left", // pContext->pBundledContext->SamplesToExitCountVirt); } - if(pContext->pBundledContext->SamplesToExitCountVirt <= 0) { + if (pContext->pBundledContext->SamplesToExitCountVirt <= 0) { status = -ENODATA; - pContext->pBundledContext->NumberEffectsEnabled--; + if ((effectInDrain & 1 << LVM_VIRTUALIZER) != 0) { + pContext->pBundledContext->NumberEffectsEnabled--; + effectInDrain &= ~(1 << LVM_VIRTUALIZER); + } ALOGV("\tEffect_process() this is the last frame for LVM_VIRTUALIZER"); } } @@ -3337,8 +3400,18 @@ pContext->pBundledContext->NumberEffectsCalled++; } - if(pContext->pBundledContext->NumberEffectsCalled == - pContext->pBundledContext->NumberEffectsEnabled){ + if (pContext->pBundledContext->NumberEffectsCalled >= + pContext->pBundledContext->NumberEffectsEnabled) { + + // We expect the # effects called to be equal to # effects enabled in sequence (including + // draining effects). Warn if this is not the case due to inconsistent calls. + ALOGW_IF(pContext->pBundledContext->NumberEffectsCalled > + pContext->pBundledContext->NumberEffectsEnabled, + "%s Number of effects called %d is greater than number of effects enabled %d", + __func__, pContext->pBundledContext->NumberEffectsCalled, + pContext->pBundledContext->NumberEffectsEnabled); + effectProcessCalled = 0; // reset our consistency check. + //ALOGV("\tEffect_process Calling process with %d effects enabled, %d called: Effect %d", //pContext->pBundledContext->NumberEffectsEnabled, //pContext->pBundledContext->NumberEffectsCalled, pContext->EffectType);
diff --git a/media/libeffects/lvm/wrapper/Bundle/EffectBundle.h b/media/libeffects/lvm/wrapper/Bundle/EffectBundle.h index 6af4554..e4aacd0 100644 --- a/media/libeffects/lvm/wrapper/Bundle/EffectBundle.h +++ b/media/libeffects/lvm/wrapper/Bundle/EffectBundle.h
@@ -110,6 +110,14 @@ #ifdef SUPPORT_MC LVM_INT32 ChMask; #endif + + /* Bitmask whether drain is in progress due to disabling the effect. + The corresponding bit to an effect is set by 1 << lvm_effect_en. */ + int effectInDrain; + + /* Bitmask whether process() was called for a particular effect. + The corresponding bit to an effect is set by 1 << lvm_effect_en. */ + int effectProcessCalled; }; /* SessionContext : One session */
diff --git a/media/libmedia/IResourceManagerService.cpp b/media/libmedia/IResourceManagerService.cpp index 9724fc1..f8a0a14 100644 --- a/media/libmedia/IResourceManagerService.cpp +++ b/media/libmedia/IResourceManagerService.cpp
@@ -32,6 +32,7 @@ CONFIG = IBinder::FIRST_CALL_TRANSACTION, ADD_RESOURCE, REMOVE_RESOURCE, + REMOVE_CLIENT, RECLAIM_RESOURCE, }; @@ -72,12 +73,14 @@ virtual void addResource( int pid, + int uid, int64_t clientId, const sp<IResourceManagerClient> client, const Vector<MediaResource> &resources) { Parcel data, reply; data.writeInterfaceToken(IResourceManagerService::getInterfaceDescriptor()); data.writeInt32(pid); + data.writeInt32(uid); data.writeInt64(clientId); data.writeStrongBinder(IInterface::asBinder(client)); writeToParcel(&data, resources); @@ -85,13 +88,23 @@ remote()->transact(ADD_RESOURCE, data, &reply); } - virtual void removeResource(int pid, int64_t clientId) { + virtual void removeResource(int pid, int64_t clientId, const Vector<MediaResource> &resources) { + Parcel data, reply; + data.writeInterfaceToken(IResourceManagerService::getInterfaceDescriptor()); + data.writeInt32(pid); + data.writeInt64(clientId); + writeToParcel(&data, resources); + + remote()->transact(REMOVE_RESOURCE, data, &reply); + } + + virtual void removeClient(int pid, int64_t clientId) { Parcel data, reply; data.writeInterfaceToken(IResourceManagerService::getInterfaceDescriptor()); data.writeInt32(pid); data.writeInt64(clientId); - remote()->transact(REMOVE_RESOURCE, data, &reply); + remote()->transact(REMOVE_CLIENT, data, &reply); } virtual bool reclaimResource(int callingPid, const Vector<MediaResource> &resources) { @@ -129,6 +142,7 @@ case ADD_RESOURCE: { CHECK_INTERFACE(IResourceManagerService, data, reply); int pid = data.readInt32(); + int uid = data.readInt32(); int64_t clientId = data.readInt64(); sp<IResourceManagerClient> client( interface_cast<IResourceManagerClient>(data.readStrongBinder())); @@ -137,7 +151,7 @@ } Vector<MediaResource> resources; readFromParcel(data, &resources); - addResource(pid, clientId, client, resources); + addResource(pid, uid, clientId, client, resources); return NO_ERROR; } break; @@ -145,7 +159,17 @@ CHECK_INTERFACE(IResourceManagerService, data, reply); int pid = data.readInt32(); int64_t clientId = data.readInt64(); - removeResource(pid, clientId); + Vector<MediaResource> resources; + readFromParcel(data, &resources); + removeResource(pid, clientId, resources); + return NO_ERROR; + } break; + + case REMOVE_CLIENT: { + CHECK_INTERFACE(IResourceManagerService, data, reply); + int pid = data.readInt32(); + int64_t clientId = data.readInt64(); + removeClient(pid, clientId); return NO_ERROR; } break;
diff --git a/media/libmedia/Visualizer.cpp b/media/libmedia/Visualizer.cpp index cb8d375..2bf0802 100644 --- a/media/libmedia/Visualizer.cpp +++ b/media/libmedia/Visualizer.cpp
@@ -77,10 +77,13 @@ if (t != 0) { if (enabled) { if (t->exitPending()) { + mCaptureLock.unlock(); if (t->requestExitAndWait() == WOULD_BLOCK) { + mCaptureLock.lock(); ALOGE("Visualizer::enable() called from thread"); return INVALID_OPERATION; } + mCaptureLock.lock(); } } t->mLock.lock();
diff --git a/media/libmedia/include/media/IResourceManagerService.h b/media/libmedia/include/media/IResourceManagerService.h index 1e4f6de..8992f8b 100644 --- a/media/libmedia/include/media/IResourceManagerService.h +++ b/media/libmedia/include/media/IResourceManagerService.h
@@ -39,11 +39,15 @@ virtual void addResource( int pid, + int uid, int64_t clientId, const sp<IResourceManagerClient> client, const Vector<MediaResource> &resources) = 0; - virtual void removeResource(int pid, int64_t clientId) = 0; + virtual void removeResource(int pid, int64_t clientId, + const Vector<MediaResource> &resources) = 0; + + virtual void removeClient(int pid, int64_t clientId) = 0; virtual bool reclaimResource( int callingPid,
diff --git a/media/libmedia/include/media/MediaResource.h b/media/libmedia/include/media/MediaResource.h index e1fdb9b..10a07bb 100644 --- a/media/libmedia/include/media/MediaResource.h +++ b/media/libmedia/include/media/MediaResource.h
@@ -31,12 +31,13 @@ kNonSecureCodec, kGraphicMemory, kCpuBoost, + kBattery, }; enum SubType { kUnspecifiedSubType = 0, kAudioCodec, - kVideoCodec + kVideoCodec, }; MediaResource(); @@ -62,6 +63,8 @@ case MediaResource::kSecureCodec: return "secure-codec"; case MediaResource::kNonSecureCodec: return "non-secure-codec"; case MediaResource::kGraphicMemory: return "graphic-memory"; + case MediaResource::kCpuBoost: return "cpu-boost"; + case MediaResource::kBattery: return "battery"; default: return def; } }
diff --git a/media/libmediaplayerservice/nuplayer/StreamingSource.cpp b/media/libmediaplayerservice/nuplayer/StreamingSource.cpp index afdcd37..f21d2b3 100644 --- a/media/libmediaplayerservice/nuplayer/StreamingSource.cpp +++ b/media/libmediaplayerservice/nuplayer/StreamingSource.cpp
@@ -130,29 +130,32 @@ } else if (n < 0) { break; } else { - if (buffer[0] == 0x00) { + if (buffer[0] == 0x00) { // OK to access buffer[0] since n must be > 0 here // XXX legacy if (extra == NULL) { extra = new AMessage; } - uint8_t type = buffer[1]; + uint8_t type = 0; + if (n > 1) { + type = buffer[1]; - if (type & 2) { - int64_t mediaTimeUs; - memcpy(&mediaTimeUs, &buffer[2], sizeof(mediaTimeUs)); + if ((type & 2) && (n >= 2 + sizeof(int64_t))) { + int64_t mediaTimeUs; + memcpy(&mediaTimeUs, &buffer[2], sizeof(mediaTimeUs)); - extra->setInt64(kATSParserKeyMediaTimeUs, mediaTimeUs); + extra->setInt64(kATSParserKeyMediaTimeUs, mediaTimeUs); + } } mTSParser->signalDiscontinuity( ((type & 1) == 0) - ? ATSParser::DISCONTINUITY_TIME - : ATSParser::DISCONTINUITY_FORMATCHANGE, + ? ATSParser::DISCONTINUITY_TIME + : ATSParser::DISCONTINUITY_FORMATCHANGE, extra); } else { - status_t err = mTSParser->feedTSPacket(buffer, sizeof(buffer)); + status_t err = mTSParser->feedTSPacket(buffer, n); if (err != OK) { ALOGE("TS Parser returned error %d", err);
diff --git a/media/libstagefright/ACodec.cpp b/media/libstagefright/ACodec.cpp index 3d67c91..44f246d 100644 --- a/media/libstagefright/ACodec.cpp +++ b/media/libstagefright/ACodec.cpp
@@ -1826,6 +1826,23 @@ mRepeatFrameDelayUs = -1LL; } + if (!msg->findDouble("time-lapse-fps", &mCaptureFps)) { + float captureRate; + if (msg->findAsFloat(KEY_CAPTURE_RATE, &captureRate)) { + mCaptureFps = captureRate; + } else { + mCaptureFps = -1.0; + } + } + + if (!msg->findInt32( + KEY_CREATE_INPUT_SURFACE_SUSPENDED, + (int32_t*)&mCreateInputBuffersSuspended)) { + mCreateInputBuffersSuspended = false; + } + } + + if (encoder && (mIsVideo || mIsImage)) { // only allow 32-bit value, since we pass it as U32 to OMX. if (!msg->findInt64(KEY_MAX_PTS_GAP_TO_ENCODER, &mMaxPtsGapUs)) { mMaxPtsGapUs = 0LL; @@ -1842,16 +1859,6 @@ if (mMaxPtsGapUs < 0LL) { mMaxFps = -1; } - - if (!msg->findDouble("time-lapse-fps", &mCaptureFps)) { - mCaptureFps = -1.0; - } - - if (!msg->findInt32( - KEY_CREATE_INPUT_SURFACE_SUSPENDED, - (int32_t*)&mCreateInputBuffersSuspended)) { - mCreateInputBuffersSuspended = false; - } } // NOTE: we only use native window for video decoders @@ -4505,22 +4512,38 @@ status_t ACodec::configureImageGrid( const sp<AMessage> &msg, sp<AMessage> &outputFormat) { int32_t tileWidth, tileHeight, gridRows, gridCols; - if (!msg->findInt32("tile-width", &tileWidth) || - !msg->findInt32("tile-height", &tileHeight) || - !msg->findInt32("grid-rows", &gridRows) || - !msg->findInt32("grid-cols", &gridCols)) { + OMX_BOOL useGrid = OMX_FALSE; + if (msg->findInt32("tile-width", &tileWidth) && + msg->findInt32("tile-height", &tileHeight) && + msg->findInt32("grid-rows", &gridRows) && + msg->findInt32("grid-cols", &gridCols)) { + useGrid = OMX_TRUE; + } else { + // when bEnabled is false, the tile info is not used, + // but clear out these too. + tileWidth = tileHeight = gridRows = gridCols = 0; + } + + if (!mIsImage && !useGrid) { return OK; } OMX_VIDEO_PARAM_ANDROID_IMAGEGRIDTYPE gridType; InitOMXParams(&gridType); gridType.nPortIndex = kPortIndexOutput; - gridType.bEnabled = OMX_TRUE; + gridType.bEnabled = useGrid; gridType.nTileWidth = tileWidth; gridType.nTileHeight = tileHeight; gridType.nGridRows = gridRows; gridType.nGridCols = gridCols; + ALOGV("sending image grid info to component: bEnabled %d, tile %dx%d, grid %dx%d", + gridType.bEnabled, + gridType.nTileWidth, + gridType.nTileHeight, + gridType.nGridRows, + gridType.nGridCols); + status_t err = mOMXNode->setParameter( (OMX_INDEXTYPE)OMX_IndexParamVideoAndroidImageGrid, &gridType, sizeof(gridType)); @@ -4541,6 +4564,13 @@ (OMX_INDEXTYPE)OMX_IndexParamVideoAndroidImageGrid, &gridType, sizeof(gridType)); + ALOGV("received image grid info from component: bEnabled %d, tile %dx%d, grid %dx%d", + gridType.bEnabled, + gridType.nTileWidth, + gridType.nTileHeight, + gridType.nGridRows, + gridType.nGridCols); + if (err == OK && gridType.bEnabled) { outputFormat->setInt32("tile-width", gridType.nTileWidth); outputFormat->setInt32("tile-height", gridType.nTileHeight); @@ -6871,7 +6901,7 @@ } } - if (mCodec->mMaxPtsGapUs != 0LL) { + if (mCodec->mIsVideo && mCodec->mMaxPtsGapUs != 0LL) { OMX_PARAM_U32TYPE maxPtsGapParams; InitOMXParams(&maxPtsGapParams); maxPtsGapParams.nPortIndex = kPortIndexInput;
diff --git a/media/libstagefright/MPEG4Writer.cpp b/media/libstagefright/MPEG4Writer.cpp index 2f13dc9..f130c9b 100644 --- a/media/libstagefright/MPEG4Writer.cpp +++ b/media/libstagefright/MPEG4Writer.cpp
@@ -1635,8 +1635,13 @@ return BAD_VALUE; } + // Increase moovExtraSize once only irrespective of how many times + // setCaptureRate is called. + bool containsCaptureFps = mMetaKeys->contains(kMetaKey_CaptureFps); mMetaKeys->setFloat(kMetaKey_CaptureFps, captureFps); - mMoovExtraSize += sizeof(kMetaKey_CaptureFps) + 4 + 32; + if (!containsCaptureFps) { + mMoovExtraSize += sizeof(kMetaKey_CaptureFps) + 4 + 32; + } return OK; }
diff --git a/media/libstagefright/MediaCodec.cpp b/media/libstagefright/MediaCodec.cpp index f579e9d..eceb84e 100644 --- a/media/libstagefright/MediaCodec.cpp +++ b/media/libstagefright/MediaCodec.cpp
@@ -48,6 +48,7 @@ #include <media/stagefright/foundation/avc_utils.h> #include <media/stagefright/foundation/hexdump.h> #include <media/stagefright/ACodec.h> +#include <media/stagefright/BatteryChecker.h> #include <media/stagefright/BufferProducerWrapper.h> #include <media/stagefright/MediaCodec.h> #include <media/stagefright/MediaCodecList.h> @@ -57,7 +58,6 @@ #include <media/stagefright/OMXClient.h> #include <media/stagefright/PersistentSurface.h> #include <media/stagefright/SurfaceUtils.h> -#include <mediautils/BatteryNotifier.h> #include <private/android_filesystem_config.h> #include <utils/Singleton.h> @@ -166,8 +166,9 @@ DISALLOW_EVIL_CONSTRUCTORS(ResourceManagerClient); }; -MediaCodec::ResourceManagerServiceProxy::ResourceManagerServiceProxy(pid_t pid) - : mPid(pid) { +MediaCodec::ResourceManagerServiceProxy::ResourceManagerServiceProxy( + pid_t pid, uid_t uid) + : mPid(pid), mUid(uid) { if (mPid == MediaCodec::kNoPid) { mPid = IPCThreadState::self()->getCallingPid(); } @@ -204,15 +205,25 @@ if (mService == NULL) { return; } - mService->addResource(mPid, clientId, client, resources); + mService->addResource(mPid, mUid, clientId, client, resources); } -void MediaCodec::ResourceManagerServiceProxy::removeResource(int64_t clientId) { +void MediaCodec::ResourceManagerServiceProxy::removeResource( + int64_t clientId, + const Vector<MediaResource> &resources) { Mutex::Autolock _l(mLock); if (mService == NULL) { return; } - mService->removeResource(mPid, clientId); + mService->removeResource(mPid, clientId, resources); +} + +void MediaCodec::ResourceManagerServiceProxy::removeClient(int64_t clientId) { + Mutex::Autolock _l(mLock); + if (mService == NULL) { + return; + } + mService->removeClient(mPid, clientId); } bool MediaCodec::ResourceManagerServiceProxy::reclaimResource( @@ -517,9 +528,6 @@ mStickyError(OK), mSoftRenderer(NULL), mAnalyticsItem(NULL), - mResourceManagerClient(new ResourceManagerClient(this)), - mResourceManagerService(new ResourceManagerServiceProxy(pid)), - mBatteryStatNotified(false), mIsVideo(false), mVideoWidth(0), mVideoHeight(0), @@ -537,13 +545,15 @@ } else { mUid = uid; } + mResourceManagerClient = new ResourceManagerClient(this); + mResourceManagerService = new ResourceManagerServiceProxy(pid, mUid); initAnalyticsItem(); } MediaCodec::~MediaCodec() { CHECK_EQ(mState, UNINITIALIZED); - mResourceManagerService->removeResource(getId(mResourceManagerClient)); + mResourceManagerService->removeClient(getId(mResourceManagerClient)); flushAnalyticsItem(); } @@ -742,6 +752,12 @@ return; } + if (mBatteryChecker != nullptr) { + mBatteryChecker->onCodecActivity([this] () { + addResource(MediaResource::kBattery, MediaResource::kVideoCodec, 1); + }); + } + const int64_t nowNs = systemTime(SYSTEM_TIME_MONOTONIC); BufferFlightTiming_t startdata = { presentationUs, nowNs }; @@ -776,6 +792,12 @@ return; } + if (mBatteryChecker != nullptr) { + mBatteryChecker->onCodecActivity([this] () { + addResource(MediaResource::kBattery, MediaResource::kVideoCodec, 1); + }); + } + BufferFlightTiming_t startdata; bool valid = false; while (mBuffersInFlight.size() > 0) { @@ -964,6 +986,10 @@ mAnalyticsItem->setCString(kCodecMode, mIsVideo ? kCodecModeVideo : kCodecModeAudio); } + if (mIsVideo) { + mBatteryChecker = new BatteryChecker(new AMessage(kWhatCheckBatteryStats, this)); + } + status_t err; Vector<MediaResource> resources; MediaResource::Type type = @@ -1221,6 +1247,13 @@ getId(mResourceManagerClient), mResourceManagerClient, resources); } +void MediaCodec::removeResource( + MediaResource::Type type, MediaResource::SubType subtype, uint64_t value) { + Vector<MediaResource> resources; + resources.push_back(MediaResource(type, subtype, value)); + mResourceManagerService->removeResource(getId(mResourceManagerClient), resources); +} + status_t MediaCodec::start() { sp<AMessage> msg = new AMessage(kWhatStart, this); @@ -1682,6 +1715,59 @@ } } +BatteryChecker::BatteryChecker(const sp<AMessage> &msg, int64_t timeoutUs) + : mTimeoutUs(timeoutUs) + , mLastActivityTimeUs(-1ll) + , mBatteryStatNotified(false) + , mBatteryCheckerGeneration(0) + , mIsExecuting(false) + , mBatteryCheckerMsg(msg) {} + +void BatteryChecker::onCodecActivity(std::function<void()> batteryOnCb) { + if (!isExecuting()) { + // ignore if not executing + return; + } + if (!mBatteryStatNotified) { + batteryOnCb(); + mBatteryStatNotified = true; + sp<AMessage> msg = mBatteryCheckerMsg->dup(); + msg->setInt32("generation", mBatteryCheckerGeneration); + + // post checker and clear last activity time + msg->post(mTimeoutUs); + mLastActivityTimeUs = -1ll; + } else { + // update last activity time + mLastActivityTimeUs = ALooper::GetNowUs(); + } +} + +void BatteryChecker::onCheckBatteryTimer( + const sp<AMessage> &msg, std::function<void()> batteryOffCb) { + // ignore if this checker already expired because the client resource was removed + int32_t generation; + if (!msg->findInt32("generation", &generation) + || generation != mBatteryCheckerGeneration) { + return; + } + + if (mLastActivityTimeUs < 0ll) { + // timed out inactive, do not repost checker + batteryOffCb(); + mBatteryStatNotified = false; + } else { + // repost checker and clear last activity time + msg->post(mTimeoutUs + mLastActivityTimeUs - ALooper::GetNowUs()); + mLastActivityTimeUs = -1ll; + } +} + +void BatteryChecker::onClientRemoved() { + mBatteryStatNotified = false; + mBatteryCheckerGeneration++; +} + //////////////////////////////////////////////////////////////////////////////// void MediaCodec::cancelPendingDequeueOperations() { @@ -2318,7 +2404,12 @@ mFlags &= ~kFlagIsComponentAllocated; - mResourceManagerService->removeResource(getId(mResourceManagerClient)); + // off since we're removing all resources including the battery on + if (mBatteryChecker != nullptr) { + mBatteryChecker->onClientRemoved(); + } + + mResourceManagerService->removeClient(getId(mResourceManagerClient)); (new AMessage)->postReply(mReplyID); break; @@ -3029,6 +3120,16 @@ break; } + case kWhatCheckBatteryStats: + { + if (mBatteryChecker != nullptr) { + mBatteryChecker->onCheckBatteryTimer(msg, [this] () { + removeResource(MediaResource::kBattery, MediaResource::kVideoCodec, 1); + }); + } + break; + } + default: TRESPASS(); } @@ -3125,9 +3226,11 @@ mState = newState; - cancelPendingDequeueOperations(); + if (mBatteryChecker != nullptr) { + mBatteryChecker->setExecuting(isExecuting()); + } - updateBatteryStat(); + cancelPendingDequeueOperations(); } void MediaCodec::returnBuffersToCodec(bool isReclaim) { @@ -3631,20 +3734,6 @@ return OK; } -void MediaCodec::updateBatteryStat() { - if (!mIsVideo) { - return; - } - - if (mState == CONFIGURED && !mBatteryStatNotified) { - BatteryNotifier::getInstance().noteStartVideo(mUid); - mBatteryStatNotified = true; - } else if (mState == UNINITIALIZED && mBatteryStatNotified) { - BatteryNotifier::getInstance().noteStopVideo(mUid); - mBatteryStatNotified = false; - } -} - std::string MediaCodec::stateString(State state) { const char *rval = NULL; char rawbuffer[16]; // room for "%d"
diff --git a/media/libstagefright/MediaMuxer.cpp b/media/libstagefright/MediaMuxer.cpp index 9ba2add..7ebdb1a 100644 --- a/media/libstagefright/MediaMuxer.cpp +++ b/media/libstagefright/MediaMuxer.cpp
@@ -96,10 +96,18 @@ sp<MediaAdapter> newTrack = new MediaAdapter(trackMeta); status_t result = mWriter->addSource(newTrack); - if (result == OK) { - return mTrackList.add(newTrack); + if (result != OK) { + return -1; } - return -1; + float captureFps = -1.0; + if (format->findAsFloat("time-lapse-fps", &captureFps)) { + ALOGV("addTrack() time-lapse-fps: %f", captureFps); + result = mWriter->setCaptureRate(captureFps); + if (result != OK) { + ALOGW("addTrack() setCaptureRate failed :%d", result); + } + } + return mTrackList.add(newTrack); } status_t MediaMuxer::setOrientationHint(int degrees) {
diff --git a/media/libstagefright/TEST_MAPPING b/media/libstagefright/TEST_MAPPING index 96818eb..c1b270c 100644 --- a/media/libstagefright/TEST_MAPPING +++ b/media/libstagefright/TEST_MAPPING
@@ -7,6 +7,9 @@ "include-annotation": "android.platform.test.annotations.RequiresDevice" } ] + }, + { + "name": "BatteryChecker_test" } ] }
diff --git a/media/libstagefright/codecs/vorbis/dec/SoftVorbis.cpp b/media/libstagefright/codecs/vorbis/dec/SoftVorbis.cpp index 76a400c..1293a74 100644 --- a/media/libstagefright/codecs/vorbis/dec/SoftVorbis.cpp +++ b/media/libstagefright/codecs/vorbis/dec/SoftVorbis.cpp
@@ -290,7 +290,7 @@ } bool SoftVorbis::isConfigured() const { - return mInputBufferCount >= 2; + return (mState != NULL && mVi != NULL); } static void makeBitReader(
diff --git a/media/libstagefright/include/media/stagefright/BatteryChecker.h b/media/libstagefright/include/media/stagefright/BatteryChecker.h new file mode 100644 index 0000000..2ec4ac0 --- /dev/null +++ b/media/libstagefright/include/media/stagefright/BatteryChecker.h
@@ -0,0 +1,47 @@ +/* + * Copyright (C) 2019 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef BATTERY_CHECKER_H_ +#define BATTERY_CHECKER_H_ + +#include <media/stagefright/foundation/AMessage.h> + +namespace android { + +struct BatteryChecker : public RefBase { + BatteryChecker(const sp<AMessage> &msg, int64_t timeout = 3000000ll); + + void setExecuting(bool executing) { mIsExecuting = executing; } + void onCodecActivity(std::function<void()> batteryOnCb); + void onCheckBatteryTimer(const sp<AMessage>& msg, std::function<void()> batteryOffCb); + void onClientRemoved(); + +private: + const int64_t mTimeoutUs; + int64_t mLastActivityTimeUs; + bool mBatteryStatNotified; + int32_t mBatteryCheckerGeneration; + bool mIsExecuting; + sp<AMessage> mBatteryCheckerMsg; + + bool isExecuting() { return mIsExecuting; } + + DISALLOW_EVIL_CONSTRUCTORS(BatteryChecker); +}; + +} // namespace android + +#endif // BATTERY_CHECKER_H_
diff --git a/media/libstagefright/include/media/stagefright/MediaCodec.h b/media/libstagefright/include/media/stagefright/MediaCodec.h index 89cca63..cd30347 100644 --- a/media/libstagefright/include/media/stagefright/MediaCodec.h +++ b/media/libstagefright/include/media/stagefright/MediaCodec.h
@@ -36,6 +36,7 @@ struct AMessage; struct AReplyToken; struct AString; +struct BatteryChecker; class BufferChannelBase; struct CodecBase; class IBatteryStats; @@ -257,6 +258,7 @@ kWhatSetCallback = 'setC', kWhatSetNotification = 'setN', kWhatDrmReleaseCrypto = 'rDrm', + kWhatCheckBatteryStats = 'chkB', }; enum { @@ -283,7 +285,7 @@ }; struct ResourceManagerServiceProxy : public IBinder::DeathRecipient { - ResourceManagerServiceProxy(pid_t pid); + ResourceManagerServiceProxy(pid_t pid, uid_t uid); ~ResourceManagerServiceProxy(); void init(); @@ -296,7 +298,11 @@ const sp<IResourceManagerClient> &client, const Vector<MediaResource> &resources); - void removeResource(int64_t clientId); + void removeResource( + int64_t clientId, + const Vector<MediaResource> &resources); + + void removeClient(int64_t clientId); bool reclaimResource(const Vector<MediaResource> &resources); @@ -304,6 +310,7 @@ Mutex mLock; sp<IResourceManagerService> mService; pid_t mPid; + uid_t mUid; }; State mState; @@ -335,7 +342,6 @@ sp<IResourceManagerClient> mResourceManagerClient; sp<ResourceManagerServiceProxy> mResourceManagerService; - bool mBatteryStatNotified; bool mIsVideo; int32_t mVideoWidth; int32_t mVideoHeight; @@ -425,11 +431,11 @@ status_t onSetParameters(const sp<AMessage> ¶ms); status_t amendOutputFormatWithCodecSpecificData(const sp<MediaCodecBuffer> &buffer); - void updateBatteryStat(); bool isExecuting() const; uint64_t getGraphicBufferSize(); void addResource(MediaResource::Type type, MediaResource::SubType subtype, uint64_t value); + void removeResource(MediaResource::Type type, MediaResource::SubType subtype, uint64_t value); void requestCpuBoostIfNeeded(); bool hasPendingBuffer(int portIndex); @@ -458,6 +464,8 @@ Mutex mLatencyLock; int64_t mLatencyUnknown; // buffers for which we couldn't calculate latency + sp<BatteryChecker> mBatteryChecker; + void statsBufferSent(int64_t presentationUs); void statsBufferReceived(int64_t presentationUs);
diff --git a/media/libstagefright/include/media/stagefright/MediaWriter.h b/media/libstagefright/include/media/stagefright/MediaWriter.h index 2c12a87..972ae1d 100644 --- a/media/libstagefright/include/media/stagefright/MediaWriter.h +++ b/media/libstagefright/include/media/stagefright/MediaWriter.h
@@ -35,6 +35,10 @@ virtual status_t start(MetaData *params = NULL) = 0; virtual status_t stop() = 0; virtual status_t pause() = 0; + virtual status_t setCaptureRate(float /* captureFps */) { + ALOGW("setCaptureRate unsupported"); + return ERROR_UNSUPPORTED; + } virtual void setMaxFileSize(int64_t bytes) { mMaxFileSizeLimitBytes = bytes; } virtual void setMaxFileDuration(int64_t durationUs) { mMaxFileDurationLimitUs = durationUs; }
diff --git a/media/libstagefright/tests/Android.bp b/media/libstagefright/tests/Android.bp index be10fdc..a7f94c1 100644 --- a/media/libstagefright/tests/Android.bp +++ b/media/libstagefright/tests/Android.bp
@@ -27,3 +27,21 @@ "-Wall", ], } + +cc_test { + name: "BatteryChecker_test", + srcs: ["BatteryChecker_test.cpp"], + test_suites: ["device-tests"], + + shared_libs: [ + "libstagefright", + "libstagefright_foundation", + "libutils", + "liblog", + ], + + cflags: [ + "-Werror", + "-Wall", + ], +} \ No newline at end of file
diff --git a/media/libstagefright/tests/BatteryChecker_test.cpp b/media/libstagefright/tests/BatteryChecker_test.cpp new file mode 100644 index 0000000..0c5ee9b --- /dev/null +++ b/media/libstagefright/tests/BatteryChecker_test.cpp
@@ -0,0 +1,242 @@ +/* + * Copyright 2019 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// #define LOG_NDEBUG 0 +#define LOG_TAG "BatteryChecker_test" +#include <utils/Log.h> + +#include <gtest/gtest.h> + +#include <media/stagefright/BatteryChecker.h> +#include <media/stagefright/foundation/ADebug.h> +#include <media/stagefright/foundation/AHandler.h> + +#include <vector> + +namespace android { + +static const int kBatteryTimeoutUs = 1000000ll; // 1 seconds +static const int kTestMarginUs = 50000ll; // 50ms +static const int kWaitStatusChangeUs = kBatteryTimeoutUs + kTestMarginUs; +static const int kSparseFrameIntervalUs = kBatteryTimeoutUs - kTestMarginUs; + +class BatteryCheckerTestHandler : public AHandler { + enum EventType { + // Events simulating MediaCodec + kWhatStart = 0, // codec entering executing state + kWhatStop, // codec exiting executing state + kWhatActivity, // codec queue input or dequeue output + kWhatReleased, // codec released + kWhatCheckpoint, // test checkpoing with expected values on On/Off + + // Message for battery checker monitor (not for testing through runTest()) + kWhatBatteryChecker, + }; + + struct Operation { + int32_t event; + int64_t delay = 0; + uint32_t repeatCount = 0; + int32_t expectedOnCounter = 0; + int32_t expectedOffCounter = 0; + }; + + std::vector<Operation> mOps; + sp<BatteryChecker> mBatteryChecker; + int32_t mOnCounter; + int32_t mOffCounter; + Condition mDone; + Mutex mLock; + + BatteryCheckerTestHandler() : mOnCounter(0), mOffCounter(0) {} + + void runTest(const std::vector<Operation> &ops, int64_t timeoutUs) { + mOps = ops; + + mBatteryChecker = new BatteryChecker( + new AMessage(kWhatBatteryChecker, this), kBatteryTimeoutUs); + + (new AMessage(ops[0].event, this))->post(); + + // wait for done + AutoMutex lock(mLock); + EXPECT_NE(TIMED_OUT, mDone.waitRelative(mLock, timeoutUs * 1000ll)); + } + + virtual void onMessageReceived(const sp<AMessage> &msg); + + friend class BatteryCheckerTest; +}; + +class BatteryCheckerTest : public ::testing::Test { +public: + BatteryCheckerTest() + : mLooper(new ALooper) + , mHandler(new BatteryCheckerTestHandler()) { + mLooper->setName("BatterCheckerLooper"); + mLooper->start(false, false, ANDROID_PRIORITY_AUDIO); + mLooper->registerHandler(mHandler); + } + +protected: + using EventType = BatteryCheckerTestHandler::EventType; + using Operation = BatteryCheckerTestHandler::Operation; + + virtual ~BatteryCheckerTest() { + mLooper->stop(); + mLooper->unregisterHandler(mHandler->id()); + } + + void runTest(const std::vector<Operation> &ops, int64_t timeoutUs) { + mHandler->runTest(ops, timeoutUs); + } + + sp<ALooper> mLooper; + sp<BatteryCheckerTestHandler> mHandler; +}; + +void BatteryCheckerTestHandler::onMessageReceived(const sp<AMessage> &msg) { + switch(msg->what()) { + case kWhatStart: + mBatteryChecker->setExecuting(true); + break; + case kWhatStop: + mBatteryChecker->setExecuting(false); + break; + case kWhatActivity: + mBatteryChecker->onCodecActivity([this] () { mOnCounter++; }); + break; + case kWhatReleased: + mBatteryChecker->onClientRemoved(); + break; + case kWhatBatteryChecker: + mBatteryChecker->onCheckBatteryTimer(msg, [this] () { mOffCounter++; }); + break; + case kWhatCheckpoint: + // verify ON/OFF state and total events + EXPECT_EQ(mOnCounter, mOps[0].expectedOnCounter); + EXPECT_EQ(mOffCounter, mOps[0].expectedOffCounter); + break; + default: + TRESPASS(); + } + if (msg->what() != kWhatBatteryChecker) { + EXPECT_EQ(msg->what(), mOps[0].event); + // post next message + if (!mOps[0].repeatCount) { + mOps.erase(mOps.begin()); + } else { + mOps[0].repeatCount--; + } + int64_t duration = mOps[0].delay; + if (!mOps.empty()) { + (new AMessage(mOps[0].event, this))->post(duration); + } else { + AutoMutex lock(mLock); + mDone.signal(); + } + } +} + +TEST_F(BatteryCheckerTest, testNormalOperations) { + runTest({ + {EventType::kWhatStart, 0ll}, + {EventType::kWhatCheckpoint, 0ll, 0, 0, 0}, + {EventType::kWhatActivity, 33333ll}, + {EventType::kWhatCheckpoint, 0ll, 0, 1, 0}, // ON + {EventType::kWhatActivity, 33333ll, 2*kWaitStatusChangeUs/33333ll}, + {EventType::kWhatCheckpoint, 0ll, 0, 1, 0}, + {EventType::kWhatCheckpoint, kWaitStatusChangeUs, 0, 1, 1}, // OFF + }, 10000000ll); +} + +TEST_F(BatteryCheckerTest, testPauseResume) { + runTest({ + {EventType::kWhatStart, 0ll}, + {EventType::kWhatCheckpoint, 0ll, 0, 0, 0}, + {EventType::kWhatActivity, 33333ll}, + {EventType::kWhatCheckpoint, 0ll, 0, 1, 0}, // ON + {EventType::kWhatCheckpoint, kWaitStatusChangeUs, 0, 1, 1}, // OFF + {EventType::kWhatActivity, 33333ll}, + {EventType::kWhatCheckpoint, 0ll, 0, 2, 1}, // ON + {EventType::kWhatCheckpoint, kWaitStatusChangeUs, 0, 2, 2}, // OFF + }, 10000000ll); +} + +TEST_F(BatteryCheckerTest, testClientRemovedAndRestart) { + runTest({ + {EventType::kWhatStart, 0ll}, + {EventType::kWhatActivity, 33333ll, kWaitStatusChangeUs/33333ll}, + {EventType::kWhatCheckpoint, 0ll, 0, 1, 0}, + + // stop executing state itself shouldn't trigger any calls + {EventType::kWhatStop, 0ll}, + {EventType::kWhatCheckpoint, 0ll, 0, 1, 0}, + + // release shouldn't trigger any calls either, + // client resource will be removed entirely + {EventType::kWhatReleased, 0ll}, + {EventType::kWhatCheckpoint, 0ll, 0, 1, 0}, + {EventType::kWhatCheckpoint, kWaitStatusChangeUs, 0, 1, 0}, + + // start pushing buffers again, On should be received without any Off + {EventType::kWhatStart, 0ll}, + {EventType::kWhatActivity, 33333ll}, + {EventType::kWhatCheckpoint, 0ll, 0, 2, 0}, + + // double check that only new checker msg triggers OFF, + // left-over checker msg from stale generate discarded + {EventType::kWhatCheckpoint, kWaitStatusChangeUs, 0, 2, 1}, + }, 10000000ll); +} + +TEST_F(BatteryCheckerTest, testActivityWhileNotExecuting) { + runTest({ + // activity before start shouldn't trigger + {EventType::kWhatActivity, 0ll}, + {EventType::kWhatCheckpoint, 0ll, 0, 0, 0}, + + {EventType::kWhatStart, 0ll}, + {EventType::kWhatCheckpoint, 0ll, 0, 0, 0}, + + // activity after start before stop should trigger + {EventType::kWhatActivity, 33333ll}, + {EventType::kWhatCheckpoint, 0ll, 0, 1, 0}, + + // stop executing state itself shouldn't trigger any calls + {EventType::kWhatStop, 0ll}, + {EventType::kWhatCheckpoint, 0ll, 0, 1, 0}, + + // keep pushing another 3 seconds after stop, expected to OFF + {EventType::kWhatActivity, 33333ll, kWaitStatusChangeUs/33333ll}, + {EventType::kWhatCheckpoint, 0ll, 0, 1, 1}, + }, 10000000ll); +} + +TEST_F(BatteryCheckerTest, testSparseActivity) { + runTest({ + {EventType::kWhatStart, 0ll}, + {EventType::kWhatCheckpoint, 0ll, 0, 0, 0}, + + // activity arrives sparsely with interval only slightly small than timeout + // should only trigger 1 ON + {EventType::kWhatActivity, kSparseFrameIntervalUs, 2}, + {EventType::kWhatCheckpoint, 0ll, 0, 1, 0}, + {EventType::kWhatCheckpoint, kSparseFrameIntervalUs, 0, 1, 0}, + {EventType::kWhatCheckpoint, kTestMarginUs, 0, 1, 1}, // OFF + }, 10000000ll); +} +} // namespace android
diff --git a/media/libstagefright/timedtext/TextDescriptions2.cpp b/media/libstagefright/timedtext/TextDescriptions2.cpp index fd42d3a..f48eacc 100644 --- a/media/libstagefright/timedtext/TextDescriptions2.cpp +++ b/media/libstagefright/timedtext/TextDescriptions2.cpp
@@ -145,7 +145,7 @@ tmpData += 8; size_t remaining = size - 8; - if (chunkSize <= 8 || size < chunkSize) { + if (size < chunkSize) { return OK; } switch(chunkType) {
diff --git a/media/ndk/NdkImage.cpp b/media/ndk/NdkImage.cpp index 1883f63..1145b7b 100644 --- a/media/ndk/NdkImage.cpp +++ b/media/ndk/NdkImage.cpp
@@ -35,6 +35,7 @@ int64_t timestamp, int32_t width, int32_t height, int32_t numPlanes) : mReader(reader), mFormat(format), mUsage(usage), mBuffer(buffer), mLockedBuffer(nullptr), mTimestamp(timestamp), mWidth(width), mHeight(height), mNumPlanes(numPlanes) { + LOG_FATAL_IF(reader == nullptr, "AImageReader shouldn't be null while creating AImage"); } AImage::~AImage() { @@ -57,14 +58,9 @@ if (mIsClosed) { return; } - sp<AImageReader> reader = mReader.promote(); - if (reader != nullptr) { - reader->releaseImageLocked(this, releaseFenceFd); - } else if (mBuffer != nullptr) { - LOG_ALWAYS_FATAL("%s: parent AImageReader closed without releasing image %p", - __FUNCTION__, this); + if (!mReader->mIsClosed) { + mReader->releaseImageLocked(this, releaseFenceFd); } - // Should have been set to nullptr in releaseImageLocked // Set to nullptr here for extra safety only mBuffer = nullptr; @@ -83,22 +79,12 @@ void AImage::lockReader() const { - sp<AImageReader> reader = mReader.promote(); - if (reader == nullptr) { - // Reader has been closed - return; - } - reader->mLock.lock(); + mReader->mLock.lock(); } void AImage::unlockReader() const { - sp<AImageReader> reader = mReader.promote(); - if (reader == nullptr) { - // Reader has been closed - return; - } - reader->mLock.unlock(); + mReader->mLock.unlock(); } media_status_t
diff --git a/media/ndk/NdkImagePriv.h b/media/ndk/NdkImagePriv.h index e0f16da..0e8cbcb 100644 --- a/media/ndk/NdkImagePriv.h +++ b/media/ndk/NdkImagePriv.h
@@ -72,7 +72,7 @@ uint32_t getJpegSize() const; // When reader is close, AImage will only accept close API call - wp<AImageReader> mReader; + const sp<AImageReader> mReader; const int32_t mFormat; const uint64_t mUsage; // AHARDWAREBUFFER_USAGE_* flags. BufferItem* mBuffer;
diff --git a/media/ndk/NdkImageReader.cpp b/media/ndk/NdkImageReader.cpp index baa4fc7..c0ceb3d 100644 --- a/media/ndk/NdkImageReader.cpp +++ b/media/ndk/NdkImageReader.cpp
@@ -113,12 +113,12 @@ void AImageReader::FrameListener::onFrameAvailable(const BufferItem& /*item*/) { - Mutex::Autolock _l(mLock); sp<AImageReader> reader = mReader.promote(); if (reader == nullptr) { ALOGW("A frame is available after AImageReader closed!"); return; // reader has been closed } + Mutex::Autolock _l(mLock); if (mListener.onImageAvailable == nullptr) { return; // No callback registered } @@ -143,12 +143,12 @@ void AImageReader::BufferRemovedListener::onBufferFreed(const wp<GraphicBuffer>& graphicBuffer) { - Mutex::Autolock _l(mLock); sp<AImageReader> reader = mReader.promote(); if (reader == nullptr) { ALOGW("A frame is available after AImageReader closed!"); return; // reader has been closed } + Mutex::Autolock _l(mLock); if (mListener.onBufferRemoved == nullptr) { return; // No callback registered } @@ -272,6 +272,11 @@ mFrameListener(new FrameListener(this)), mBufferRemovedListener(new BufferRemovedListener(this)) {} +AImageReader::~AImageReader() { + Mutex::Autolock _l(mLock); + LOG_FATAL_IF("AImageReader not closed before destruction", mIsClosed != true); +} + media_status_t AImageReader::init() { PublicFormat publicFormat = static_cast<PublicFormat>(mFormat); @@ -347,8 +352,12 @@ return AMEDIA_OK; } -AImageReader::~AImageReader() { +void AImageReader::close() { Mutex::Autolock _l(mLock); + if (mIsClosed) { + return; + } + mIsClosed = true; AImageReader_ImageListener nullListener = {nullptr, nullptr}; setImageListenerLocked(&nullListener); @@ -741,6 +750,7 @@ void AImageReader_delete(AImageReader* reader) { ALOGV("%s", __FUNCTION__); if (reader != nullptr) { + reader->close(); reader->decStrong((void*) AImageReader_delete); } return;
diff --git a/media/ndk/NdkImageReaderPriv.h b/media/ndk/NdkImageReaderPriv.h index e328cb1..0779a71 100644 --- a/media/ndk/NdkImageReaderPriv.h +++ b/media/ndk/NdkImageReaderPriv.h
@@ -76,6 +76,7 @@ int32_t getHeight() const { return mHeight; }; int32_t getFormat() const { return mFormat; }; int32_t getMaxImages() const { return mMaxImages; }; + void close(); private: @@ -134,7 +135,7 @@ private: AImageReader_ImageListener mListener = {nullptr, nullptr}; - wp<AImageReader> mReader; + const wp<AImageReader> mReader; Mutex mLock; }; sp<FrameListener> mFrameListener; @@ -149,7 +150,7 @@ private: AImageReader_BufferRemovedListener mListener = {nullptr, nullptr}; - wp<AImageReader> mReader; + const wp<AImageReader> mReader; Mutex mLock; }; sp<BufferRemovedListener> mBufferRemovedListener; @@ -165,6 +166,7 @@ native_handle_t* mWindowHandle = nullptr; List<AImage*> mAcquiredImages; + bool mIsClosed = false; Mutex mLock; };
diff --git a/media/ndk/NdkMediaCodec.cpp b/media/ndk/NdkMediaCodec.cpp index c23f19b..e041533 100644 --- a/media/ndk/NdkMediaCodec.cpp +++ b/media/ndk/NdkMediaCodec.cpp
@@ -221,7 +221,13 @@ break; } - AMediaFormat *aMediaFormat = AMediaFormat_fromMsg(&format); + // Here format is MediaCodec's internal copy of output format. + // Make a copy since the client might modify it. + sp<AMessage> copy; + if (format != nullptr) { + copy = format->dup(); + } + AMediaFormat *aMediaFormat = AMediaFormat_fromMsg(©); Mutex::Autolock _l(mCodec->mAsyncCallbackLock); if (mCodec->mAsyncCallbackUserData != NULL
diff --git a/media/ndk/NdkMediaDrm.cpp b/media/ndk/NdkMediaDrm.cpp index 85dbffe..cd5a23a 100644 --- a/media/ndk/NdkMediaDrm.cpp +++ b/media/ndk/NdkMediaDrm.cpp
@@ -89,7 +89,7 @@ }; void DrmListener::notify(DrmPlugin::EventType eventType, int extra, const Parcel *obj) { - if (!mEventListener && !mExpirationUpdateListener && !mKeysChangeListener) { + if (!mEventListener || !mExpirationUpdateListener || !mKeysChangeListener) { ALOGE("No listeners are specified"); return; }
diff --git a/media/utils/Android.bp b/media/utils/Android.bp index 3adb40f..e2cd4e3 100644 --- a/media/utils/Android.bp +++ b/media/utils/Android.bp
@@ -27,6 +27,7 @@ ], shared_libs: [ "libbinder", + "libcutils", "liblog", "libutils", "libmemunreachable",
diff --git a/media/utils/ServiceUtilities.cpp b/media/utils/ServiceUtilities.cpp index b824212..bc8fff6 100644 --- a/media/utils/ServiceUtilities.cpp +++ b/media/utils/ServiceUtilities.cpp
@@ -63,7 +63,10 @@ uid_t uid, bool start) { // Okay to not track in app ops as audio server is us and if // device is rooted security model is considered compromised. - if (isAudioServerOrRootUid(uid)) return true; + // system_server loses its RECORD_AUDIO permission when a secondary + // user is active, but it is a core system service so let it through. + // TODO(b/141210120): UserManager.DISALLOW_RECORD_AUDIO should not affect system user 0 + if (isAudioServerOrSystemServerOrRootUid(uid)) return true; // We specify a pid and uid here as mediaserver (aka MediaRecorder or StageFrightRecorder) // may open a record track on behalf of a client. Note that pid may be a tid.
diff --git a/media/utils/include/mediautils/ServiceUtilities.h b/media/utils/include/mediautils/ServiceUtilities.h index 2a6e609..e1089d5 100644 --- a/media/utils/include/mediautils/ServiceUtilities.h +++ b/media/utils/include/mediautils/ServiceUtilities.h
@@ -58,6 +58,12 @@ return multiuser_get_app_id(uid) == AID_SYSTEM || uid == AID_AUDIOSERVER; } +// used for calls that should come from system_server or audio_server and +// include AID_ROOT for command-line tests. +static inline bool isAudioServerOrSystemServerOrRootUid(uid_t uid) { + return multiuser_get_app_id(uid) == AID_SYSTEM || uid == AID_AUDIOSERVER || uid == AID_ROOT; +} + // Mediaserver may forward the client PID and UID as part of a binder interface call; // otherwise the calling UID must be equal to the client UID. static inline bool isAudioServerOrMediaServerUid(uid_t uid) {
diff --git a/services/audioflinger/AudioFlinger.cpp b/services/audioflinger/AudioFlinger.cpp index d38190d..8bbdc69 100644 --- a/services/audioflinger/AudioFlinger.cpp +++ b/services/audioflinger/AudioFlinger.cpp
@@ -810,7 +810,33 @@ continue; } - size_t frameCount = std::lcm(thread->frameCount(), secondaryThread->frameCount()); + size_t sourceFrameCount = thread->frameCount() * output.sampleRate + / thread->sampleRate(); + size_t sinkFrameCount = secondaryThread->frameCount() * output.sampleRate + / secondaryThread->sampleRate(); + // If the secondary output has just been opened, the first secondaryThread write + // will not block as it will fill the empty startup buffer of the HAL, + // so a second sink buffer needs to be ready for the immediate next blocking write. + // Additionally, have a margin of one main thread buffer as the scheduling jitter + // can reorder the writes (eg if thread A&B have the same write intervale, + // the scheduler could schedule AB...BA) + size_t frameCountToBeReady = 2 * sinkFrameCount + sourceFrameCount; + // Total secondary output buffer must be at least as the read frames plus + // the margin of a few buffers on both sides in case the + // threads scheduling has some jitter. + // That value should not impact latency as the secondary track is started before + // its buffer is full, see frameCountToBeReady. + size_t frameCount = frameCountToBeReady + 2 * (sourceFrameCount + sinkFrameCount); + // The frameCount should also not be smaller than the secondary thread min frame + // count + size_t minFrameCount = AudioSystem::calculateMinFrameCount( + [&] { Mutex::Autolock _l(secondaryThread->mLock); + return secondaryThread->latency_l(); }(), + secondaryThread->mNormalFrameCount, + secondaryThread->mSampleRate, + output.sampleRate, + input.speed); + frameCount = std::max(frameCount, minFrameCount); using namespace std::chrono_literals; auto inChannelMask = audio_channel_mask_out_to_in(input.config.channel_mask); @@ -843,7 +869,8 @@ patchRecord->buffer(), patchRecord->bufferSize(), outputFlags, - 0ns /* timeout */); + 0ns /* timeout */, + frameCountToBeReady); status = patchTrack->initCheck(); if (status != NO_ERROR) { ALOGE("Secondary output patchTrack init failed: %d", status);
diff --git a/services/audioflinger/Effects.cpp b/services/audioflinger/Effects.cpp index 3c4fbba..13152d0 100644 --- a/services/audioflinger/Effects.cpp +++ b/services/audioflinger/Effects.cpp
@@ -24,6 +24,7 @@ #include "Configuration.h" #include <utils/Log.h> #include <system/audio_effects/effect_aec.h> +#include <system/audio_effects/effect_dynamicsprocessing.h> #include <system/audio_effects/effect_ns.h> #include <system/audio_effects/effect_visualizer.h> #include <audio_utils/channels.h> @@ -2569,7 +2570,8 @@ if ((mSessionId == AUDIO_SESSION_OUTPUT_MIX) && (((desc.flags & EFFECT_FLAG_TYPE_MASK) == EFFECT_FLAG_TYPE_AUXILIARY) || (memcmp(&desc.type, SL_IID_VISUALIZATION, sizeof(effect_uuid_t)) == 0) || - (memcmp(&desc.type, SL_IID_VOLUME, sizeof(effect_uuid_t)) == 0))) { + (memcmp(&desc.type, SL_IID_VOLUME, sizeof(effect_uuid_t)) == 0) || + (memcmp(&desc.type, SL_IID_DYNAMICSPROCESSING, sizeof(effect_uuid_t)) == 0))) { return false; } return true;
diff --git a/services/audioflinger/PlaybackTracks.h b/services/audioflinger/PlaybackTracks.h index a093893..17adba5 100644 --- a/services/audioflinger/PlaybackTracks.h +++ b/services/audioflinger/PlaybackTracks.h
@@ -74,7 +74,10 @@ uid_t uid, audio_output_flags_t flags, track_type type, - audio_port_handle_t portId = AUDIO_PORT_HANDLE_NONE); + audio_port_handle_t portId = AUDIO_PORT_HANDLE_NONE, + /** default behaviour is to start when there are as many frames + * ready as possible (aka. Buffer is full). */ + size_t frameCountToBeReady = SIZE_MAX); virtual ~Track(); virtual status_t initCheck() const; @@ -263,6 +266,8 @@ }; sp<AudioVibrationController> mAudioVibrationController; sp<os::ExternalVibration> mExternalVibration; + /** How many frames should be in the buffer before the track is considered ready */ + const size_t mFrameCountToBeReady; private: void interceptBuffer(const AudioBufferProvider::Buffer& buffer); @@ -384,7 +389,11 @@ void *buffer, size_t bufferSize, audio_output_flags_t flags, - const Timeout& timeout = {}); + const Timeout& timeout = {}, + size_t frameCountToBeReady = 1 /** Default behaviour is to start + * as soon as possible to have + * the lowest possible latency + * even if it might glitch. */); virtual ~PatchTrack(); virtual status_t start(AudioSystem::sync_event_t event = @@ -402,5 +411,4 @@ private: void restartIfDisabled(); - }; // end of PatchTrack
diff --git a/services/audioflinger/Threads.cpp b/services/audioflinger/Threads.cpp index 68c73bf..bd9bf7b 100644 --- a/services/audioflinger/Threads.cpp +++ b/services/audioflinger/Threads.cpp
@@ -3956,6 +3956,32 @@ return INVALID_OPERATION; } +// For dedicated VoIP outputs, let the HAL apply the stream volume. Track volume is +// still applied by the mixer. +// All tracks attached to a mixer with flag VOIP_RX are tied to the same +// stream type STREAM_VOICE_CALL so this will only change the HAL volume once even +// if more than one track are active +status_t AudioFlinger::PlaybackThread::handleVoipVolume_l(float *volume) +{ + status_t result = NO_ERROR; + if ((mOutput->flags & AUDIO_OUTPUT_FLAG_VOIP_RX) != 0) { + if (*volume != mLeftVolFloat) { + result = mOutput->stream->setVolume(*volume, *volume); + ALOGE_IF(result != OK, + "Error when setting output stream volume: %d", result); + if (result == NO_ERROR) { + mLeftVolFloat = *volume; + } + } + // if stream volume was successfully sent to the HAL, mLeftVolFloat == v here and we + // remove stream volume contribution from software volume. + if (mLeftVolFloat == *volume) { + *volume = 1.0f; + } + } + return result; +} + status_t AudioFlinger::MixerThread::createAudioPatch_l(const struct audio_patch *patch, audio_patch_handle_t *handle) { @@ -4758,22 +4784,25 @@ // no acknowledgement required for newly active tracks } sp<AudioTrackServerProxy> proxy = track->mAudioTrackServerProxy; + float volume; + if (track->isPlaybackRestricted() || mStreamTypes[track->streamType()].mute) { + volume = 0.f; + } else { + volume = masterVolume * mStreamTypes[track->streamType()].volume; + } + + handleVoipVolume_l(&volume); + // cache the combined master volume and stream type volume for fast mixer; this // lacks any synchronization or barrier so VolumeProvider may read a stale value const float vh = track->getVolumeHandler()->getVolume( - proxy->framesReleased()).first; - float volume; - if (track->isPlaybackRestricted()) { - volume = 0.f; - } else { - volume = masterVolume - * mStreamTypes[track->streamType()].volume - * vh; - } + proxy->framesReleased()).first; + volume *= vh; track->mCachedVolume = volume; gain_minifloat_packed_t vlr = proxy->getVolumeLR(); float vlf = volume * float_from_gain(gain_minifloat_unpack_left(vlr)); float vrf = volume * float_from_gain(gain_minifloat_unpack_right(vlr)); + track->setFinalVolume((vlf + vrf) / 2.f); ++fastTracks; } else { @@ -4916,20 +4945,22 @@ uint32_t vl, vr; // in U8.24 integer format float vlf, vrf, vaf; // in [0.0, 1.0] float format // read original volumes with volume control - float typeVolume = mStreamTypes[track->streamType()].volume; - float v = masterVolume * typeVolume; + float v = masterVolume * mStreamTypes[track->streamType()].volume; // Always fetch volumeshaper volume to ensure state is updated. const sp<AudioTrackServerProxy> proxy = track->mAudioTrackServerProxy; const float vh = track->getVolumeHandler()->getVolume( track->mAudioTrackServerProxy->framesReleased()).first; - if (track->isPausing() || mStreamTypes[track->streamType()].mute - || track->isPlaybackRestricted()) { + if (mStreamTypes[track->streamType()].mute || track->isPlaybackRestricted()) { + v = 0; + } + + handleVoipVolume_l(&v); + + if (track->isPausing()) { vl = vr = 0; vlf = vrf = vaf = 0.; - if (track->isPausing()) { - track->setPaused(); - } + track->setPaused(); } else { gain_minifloat_packed_t vlr = proxy->getVolumeLR(); vlf = float_from_gain(gain_minifloat_unpack_left(vlr)); @@ -4981,25 +5012,6 @@ track->mHasVolumeController = false; } - // For dedicated VoIP outputs, let the HAL apply the stream volume. Track volume is - // still applied by the mixer. - if ((mOutput->flags & AUDIO_OUTPUT_FLAG_VOIP_RX) != 0) { - v = mStreamTypes[track->streamType()].mute ? 0.0f : v; - if (v != mLeftVolFloat) { - status_t result = mOutput->stream->setVolume(v, v); - ALOGE_IF(result != OK, "Error when setting output stream volume: %d", result); - if (result == OK) { - mLeftVolFloat = v; - } - } - // if stream volume was successfully sent to the HAL, mLeftVolFloat == v here and we - // remove stream volume contribution from software volume. - if (v != 0.0f && mLeftVolFloat == v) { - vlf = min(1.0f, vlf / v); - vrf = min(1.0f, vrf / v); - vaf = min(1.0f, vaf / v); - } - } // XXX: these things DON'T need to be done each time mAudioMixer->setBufferProvider(trackId, track); mAudioMixer->enable(trackId);
diff --git a/services/audioflinger/Threads.h b/services/audioflinger/Threads.h index c6a8201..87bebf3 100644 --- a/services/audioflinger/Threads.h +++ b/services/audioflinger/Threads.h
@@ -747,6 +747,7 @@ // is safe to do so. That will drop the final ref count and destroy the tracks. virtual mixer_state prepareTracks_l(Vector< sp<Track> > *tracksToRemove) = 0; void removeTracks_l(const Vector< sp<Track> >& tracksToRemove); + status_t handleVoipVolume_l(float *volume); // StreamOutHalInterfaceCallback implementation virtual void onWriteReady();
diff --git a/services/audioflinger/Tracks.cpp b/services/audioflinger/Tracks.cpp index 2a5a713..51e57b5 100644 --- a/services/audioflinger/Tracks.cpp +++ b/services/audioflinger/Tracks.cpp
@@ -511,7 +511,8 @@ uid_t uid, audio_output_flags_t flags, track_type type, - audio_port_handle_t portId) + audio_port_handle_t portId, + size_t frameCountToBeReady) : TrackBase(thread, client, attr, sampleRate, format, channelMask, frameCount, (sharedBuffer != 0) ? sharedBuffer->pointer() : buffer, (sharedBuffer != 0) ? sharedBuffer->size() : bufferSize, @@ -530,6 +531,7 @@ mVolumeHandler(new media::VolumeHandler(sampleRate)), mOpPlayAudioMonitor(OpPlayAudioMonitor::createIfNeeded(uid, attr, id(), streamType)), // mSinkTimestamp + mFrameCountToBeReady(frameCountToBeReady), mFastIndex(-1), mCachedVolume(1.0), /* The track might not play immediately after being active, similarly as if its volume was 0. @@ -837,7 +839,7 @@ auto spent = ceil<std::chrono::microseconds>(std::chrono::steady_clock::now() - start); using namespace std::chrono_literals; // Average is ~20us per track, this should virtually never be logged (Logging takes >200us) - ALOGD_IF(spent > 200us, "%s: took %lldus to intercept %zu tracks", __func__, + ALOGD_IF(spent > 500us, "%s: took %lldus to intercept %zu tracks", __func__, spent.count(), mTeePatches.size()); } @@ -910,8 +912,12 @@ return true; } - if (framesReady() >= mServerProxy->getBufferSizeInFrames() || - (mCblk->mFlags & CBLK_FORCEREADY)) { + size_t bufferSizeInFrames = mServerProxy->getBufferSizeInFrames(); + size_t framesToBeReady = std::min(mFrameCountToBeReady, bufferSizeInFrames); + + if (framesReady() >= framesToBeReady || (mCblk->mFlags & CBLK_FORCEREADY)) { + ALOGV("%s(%d): consider track ready with %zu/%zu, target was %zu)", + __func__, mId, framesReady(), bufferSizeInFrames, framesToBeReady); mFillingUpStatus = FS_FILLED; android_atomic_and(~CBLK_FORCEREADY, &mCblk->mFlags); return true; @@ -1413,6 +1419,7 @@ void AudioFlinger::PlaybackThread::Track::disable() { + // TODO(b/142394888): the filling status should also be reset to filling signalClientFlag(CBLK_DISABLED); } @@ -1790,12 +1797,14 @@ void *buffer, size_t bufferSize, audio_output_flags_t flags, - const Timeout& timeout) + const Timeout& timeout, + size_t frameCountToBeReady) : Track(playbackThread, NULL, streamType, audio_attributes_t{} /* currently unused for patch track */, sampleRate, format, channelMask, frameCount, buffer, bufferSize, nullptr /* sharedBuffer */, - AUDIO_SESSION_NONE, getpid(), AID_AUDIOSERVER, flags, TYPE_PATCH), + AUDIO_SESSION_NONE, getpid(), AID_AUDIOSERVER, flags, TYPE_PATCH, + AUDIO_PORT_HANDLE_NONE, frameCountToBeReady), PatchTrackBase(new ClientProxy(mCblk, mBuffer, frameCount, mFrameSize, true, true), *playbackThread, timeout) { @@ -1869,7 +1878,6 @@ { mProxy->releaseBuffer(buffer); restartIfDisabled(); - android_atomic_or(CBLK_FORCEREADY, &mCblk->mFlags); } void AudioFlinger::PlaybackThread::PatchTrack::restartIfDisabled()
diff --git a/services/audiopolicy/common/managerdefinitions/src/HwModule.cpp b/services/audiopolicy/common/managerdefinitions/src/HwModule.cpp index 96a8337..1f9b725 100644 --- a/services/audiopolicy/common/managerdefinitions/src/HwModule.cpp +++ b/services/audiopolicy/common/managerdefinitions/src/HwModule.cpp
@@ -333,9 +333,10 @@ if (encodedFormat != AUDIO_FORMAT_DEFAULT) { moduleDevice->setEncodedFormat(encodedFormat); } - moduleDevice->setAddress(devAddress); if (allowToCreate) { moduleDevice->attach(hwModule); + moduleDevice->setAddress(devAddress); + moduleDevice->setName(String8(name)); } return moduleDevice; }
diff --git a/services/audiopolicy/managerdefault/AudioPolicyManager.cpp b/services/audiopolicy/managerdefault/AudioPolicyManager.cpp index 1d4dacd..c048de3 100644 --- a/services/audiopolicy/managerdefault/AudioPolicyManager.cpp +++ b/services/audiopolicy/managerdefault/AudioPolicyManager.cpp
@@ -5690,8 +5690,9 @@ const auto ringVolumeSrc = toVolumeSource(AUDIO_STREAM_RING); const auto musicVolumeSrc = toVolumeSource(AUDIO_STREAM_MUSIC); const auto alarmVolumeSrc = toVolumeSource(AUDIO_STREAM_ALARM); + const auto a11yVolumeSrc = toVolumeSource(AUDIO_STREAM_ACCESSIBILITY); - if (volumeSource == toVolumeSource(AUDIO_STREAM_ACCESSIBILITY) + if (volumeSource == a11yVolumeSrc && (AUDIO_MODE_RINGTONE == mEngine->getPhoneState()) && mOutputs.isActive(ringVolumeSrc, 0)) { auto &ringCurves = getVolumeCurves(AUDIO_STREAM_RING); @@ -5708,7 +5709,7 @@ volumeSource == toVolumeSource(AUDIO_STREAM_NOTIFICATION) || volumeSource == toVolumeSource(AUDIO_STREAM_ENFORCED_AUDIBLE) || volumeSource == toVolumeSource(AUDIO_STREAM_DTMF) || - volumeSource == toVolumeSource(AUDIO_STREAM_ACCESSIBILITY))) { + volumeSource == a11yVolumeSrc)) { auto &voiceCurves = getVolumeCurves(callVolumeSrc); int voiceVolumeIndex = voiceCurves.getVolumeIndex(device); const float maxVoiceVolDb = @@ -5720,7 +5721,9 @@ // VOICE_CALL stream has minVolumeIndex > 0 : Users cannot set the volume of voice calls to // 0. We don't want to cap volume when the system has programmatically muted the voice call // stream. See setVolumeCurveIndex() for more information. - bool exemptFromCapping = (volumeSource == ringVolumeSrc) && (voiceVolumeIndex == 0); + bool exemptFromCapping = + ((volumeSource == ringVolumeSrc) || (volumeSource == a11yVolumeSrc)) + && (voiceVolumeIndex == 0); ALOGV_IF(exemptFromCapping, "%s volume source %d at vol=%f not capped", __func__, volumeSource, volumeDb); if ((volumeDb > maxVoiceVolDb) && !exemptFromCapping) {
diff --git a/services/audiopolicy/service/AudioPolicyService.cpp b/services/audiopolicy/service/AudioPolicyService.cpp index 77f7997..a6cda20 100644 --- a/services/audiopolicy/service/AudioPolicyService.cpp +++ b/services/audiopolicy/service/AudioPolicyService.cpp
@@ -410,12 +410,17 @@ // Another client in the same UID has already been allowed to capture // OR The client is the assistant // AND an accessibility service is on TOP or a RTT call is active -// AND the source is VOICE_RECOGNITION or HOTWORD -// OR uses VOICE_RECOGNITION AND is on TOP -// OR uses HOTWORD +// AND the source is VOICE_RECOGNITION or HOTWORD +// OR uses VOICE_RECOGNITION AND is on TOP +// OR uses HOTWORD // AND there is no active privacy sensitive capture or call // OR client has CAPTURE_AUDIO_OUTPUT privileged permission // OR The client is an accessibility service +// AND Is on TOP +// AND the source is VOICE_RECOGNITION or HOTWORD +// OR The assistant is not on TOP +// AND there is no active privacy sensitive capture or call +// OR client has CAPTURE_AUDIO_OUTPUT privileged permission // AND is on TOP // AND the source is VOICE_RECOGNITION or HOTWORD // OR the client source is virtual (remote submix, call audio TX or RX...) @@ -423,7 +428,7 @@ // AND The assistant is not on TOP // AND is on TOP or latest started // AND there is no active privacy sensitive capture or call -// OR client has CAPTURE_AUDIO_OUTPUT privileged permission +// OR client has CAPTURE_AUDIO_OUTPUT privileged permission sp<AudioRecordClient> topActive; sp<AudioRecordClient> latestActive; @@ -459,16 +464,24 @@ continue; } - if (appState == APP_STATE_TOP) { + bool isAssistant = mUidPolicy->isAssistantUid(current->uid); + bool isAccessibility = mUidPolicy->isA11yUid(current->uid); + if (appState == APP_STATE_TOP && !isAccessibility) { if (current->startTimeNs > topStartNs) { topActive = current; topStartNs = current->startTimeNs; } - if (mUidPolicy->isAssistantUid(current->uid)) { + if (isAssistant) { isAssistantOnTop = true; } } - if (current->startTimeNs > latestStartNs) { + // Assistant capturing for HOTWORD or Accessibility services not considered + // for latest active to avoid masking regular clients started before + if (current->startTimeNs > latestStartNs + && !((current->attributes.source == AUDIO_SOURCE_HOTWORD + || isA11yOnTop || rttCallActive) + && isAssistant) + && !isAccessibility) { latestActive = current; latestStartNs = current->startTimeNs; } @@ -541,10 +554,20 @@ } else if (mUidPolicy->isA11yUid(current->uid)) { // For accessibility service allow capture if: // Is on TOP - // AND the source is VOICE_RECOGNITION or HOTWORD - if (isA11yOnTop && - (source == AUDIO_SOURCE_VOICE_RECOGNITION || source == AUDIO_SOURCE_HOTWORD)) { - allowCapture = true; + // AND the source is VOICE_RECOGNITION or HOTWORD + // Or + // The assistant is not on TOP + // AND there is no active privacy sensitive capture or call + // OR client has CAPTURE_AUDIO_OUTPUT privileged permission + if (isA11yOnTop) { + if (source == AUDIO_SOURCE_VOICE_RECOGNITION || source == AUDIO_SOURCE_HOTWORD) { + allowCapture = true; + } + } else { + if (!isAssistantOnTop + && (!(isSensitiveActive || isInCall) || current->canCaptureOutput)) { + allowCapture = true; + } } } setAppState_l(current->uid, @@ -703,11 +726,11 @@ if (in == BAD_TYPE || out == BAD_TYPE || err == BAD_TYPE) { return BAD_VALUE; } - if (args.size() == 3 && args[0] == String16("set-uid-state")) { + if (args.size() >= 3 && args[0] == String16("set-uid-state")) { return handleSetUidState(args, err); - } else if (args.size() == 2 && args[0] == String16("reset-uid-state")) { + } else if (args.size() >= 2 && args[0] == String16("reset-uid-state")) { return handleResetUidState(args, err); - } else if (args.size() == 2 && args[0] == String16("get-uid-state")) { + } else if (args.size() >= 2 && args[0] == String16("get-uid-state")) { return handleGetUidState(args, out, err); } else if (args.size() == 1 && args[0] == String16("help")) { printHelp(out); @@ -717,14 +740,32 @@ return BAD_VALUE; } -status_t AudioPolicyService::handleSetUidState(Vector<String16>& args, int err) { - PermissionController pc; - int uid = pc.getPackageUid(args[1], 0); - if (uid <= 0) { - ALOGE("Unknown package: '%s'", String8(args[1]).string()); - dprintf(err, "Unknown package: '%s'\n", String8(args[1]).string()); +static status_t getUidForPackage(String16 packageName, int userId, /*inout*/uid_t& uid, int err) { + if (userId < 0) { + ALOGE("Invalid user: %d", userId); + dprintf(err, "Invalid user: %d\n", userId); return BAD_VALUE; } + + PermissionController pc; + uid = pc.getPackageUid(packageName, 0); + if (uid <= 0) { + ALOGE("Unknown package: '%s'", String8(packageName).string()); + dprintf(err, "Unknown package: '%s'\n", String8(packageName).string()); + return BAD_VALUE; + } + + uid = multiuser_get_uid(userId, uid); + return NO_ERROR; +} + +status_t AudioPolicyService::handleSetUidState(Vector<String16>& args, int err) { + // Valid arg.size() is 3 or 5, args.size() is 5 with --user option. + if (!(args.size() == 3 || args.size() == 5)) { + printHelp(err); + return BAD_VALUE; + } + bool active = false; if (args[2] == String16("active")) { active = true; @@ -732,30 +773,59 @@ ALOGE("Expected active or idle but got: '%s'", String8(args[2]).string()); return BAD_VALUE; } + + int userId = 0; + if (args.size() >= 5 && args[3] == String16("--user")) { + userId = atoi(String8(args[4])); + } + + uid_t uid; + if (getUidForPackage(args[1], userId, uid, err) == BAD_VALUE) { + return BAD_VALUE; + } + mUidPolicy->addOverrideUid(uid, active); return NO_ERROR; } status_t AudioPolicyService::handleResetUidState(Vector<String16>& args, int err) { - PermissionController pc; - int uid = pc.getPackageUid(args[1], 0); - if (uid < 0) { - ALOGE("Unknown package: '%s'", String8(args[1]).string()); - dprintf(err, "Unknown package: '%s'\n", String8(args[1]).string()); + // Valid arg.size() is 2 or 4, args.size() is 4 with --user option. + if (!(args.size() == 2 || args.size() == 4)) { + printHelp(err); return BAD_VALUE; } + + int userId = 0; + if (args.size() >= 4 && args[2] == String16("--user")) { + userId = atoi(String8(args[3])); + } + + uid_t uid; + if (getUidForPackage(args[1], userId, uid, err) == BAD_VALUE) { + return BAD_VALUE; + } + mUidPolicy->removeOverrideUid(uid); return NO_ERROR; } status_t AudioPolicyService::handleGetUidState(Vector<String16>& args, int out, int err) { - PermissionController pc; - int uid = pc.getPackageUid(args[1], 0); - if (uid < 0) { - ALOGE("Unknown package: '%s'", String8(args[1]).string()); - dprintf(err, "Unknown package: '%s'\n", String8(args[1]).string()); + // Valid arg.size() is 2 or 4, args.size() is 4 with --user option. + if (!(args.size() == 2 || args.size() == 4)) { + printHelp(err); return BAD_VALUE; } + + int userId = 0; + if (args.size() >= 4 && args[2] == String16("--user")) { + userId = atoi(String8(args[3])); + } + + uid_t uid; + if (getUidForPackage(args[1], userId, uid, err) == BAD_VALUE) { + return BAD_VALUE; + } + if (mUidPolicy->isUidActive(uid)) { return dprintf(out, "active\n"); } else { @@ -765,9 +835,9 @@ status_t AudioPolicyService::printHelp(int out) { return dprintf(out, "Audio policy service commands:\n" - " get-uid-state <PACKAGE> gets the uid state\n" - " set-uid-state <PACKAGE> <active|idle> overrides the uid state\n" - " reset-uid-state <PACKAGE> clears the uid state override\n" + " get-uid-state <PACKAGE> [--user USER_ID] gets the uid state\n" + " set-uid-state <PACKAGE> <active|idle> [--user USER_ID] overrides the uid state\n" + " reset-uid-state <PACKAGE> [--user USER_ID] clears the uid state override\n" " help print this message\n"); }
diff --git a/services/camera/libcameraservice/CameraService.cpp b/services/camera/libcameraservice/CameraService.cpp index 3e62102..b20c9a4 100644 --- a/services/camera/libcameraservice/CameraService.cpp +++ b/services/camera/libcameraservice/CameraService.cpp
@@ -153,8 +153,6 @@ mInitialized = true; } - CameraService::pingCameraServiceProxy(); - mUidPolicy = new UidPolicy(this); mUidPolicy->registerSelf(); mSensorPrivacyPolicy = new SensorPrivacyPolicy(this); @@ -164,6 +162,11 @@ ALOGE("%s: Failed to register default android.frameworks.cameraservice.service@1.0", __FUNCTION__); } + + // This needs to be last call in this function, so that it's as close to + // ServiceManager::addService() as possible. + CameraService::pingCameraServiceProxy(); + ALOGI("CameraService pinged cameraservice proxy"); } status_t CameraService::enumerateProviders() { @@ -253,6 +256,15 @@ enumerateProviders(); } +bool CameraService::isPublicallyHiddenSecureCamera(const String8& cameraId) { + auto state = getCameraState(cameraId); + if (state != nullptr) { + return state->isPublicallyHiddenSecureCamera(); + } + // Hidden physical camera ids won't have CameraState + return mCameraProviderManager->isPublicallyHiddenSecureCamera(cameraId.c_str()); +} + void CameraService::updateCameraNumAndIds() { Mutex::Autolock l(mServiceLock); mNumberOfCameras = mCameraProviderManager->getCameraCount(); @@ -268,6 +280,8 @@ ALOGE("Failed to query device resource cost: %s (%d)", strerror(-res), res); return; } + bool isPublicallyHiddenSecureCamera = + mCameraProviderManager->isPublicallyHiddenSecureCamera(id.string()); std::set<String8> conflicting; for (size_t i = 0; i < cost.conflictingDevices.size(); i++) { conflicting.emplace(String8(cost.conflictingDevices[i].c_str())); @@ -276,7 +290,8 @@ { Mutex::Autolock lock(mCameraStatesLock); mCameraStates.emplace(id, std::make_shared<CameraState>(id, cost.resourceCost, - conflicting)); + conflicting, + isPublicallyHiddenSecureCamera)); } if (mFlashlight->hasFlashUnit(id)) { @@ -514,8 +529,16 @@ "Camera subsystem is not available");; } - Status ret{}; + if (shouldRejectHiddenCameraConnection(String8(cameraId))) { + ALOGW("Attempting to retrieve characteristics for system-only camera id %s, rejected", + String8(cameraId).string()); + return STATUS_ERROR_FMT(ERROR_DISCONNECTED, + "No camera device with ID \"%s\" currently available", + String8(cameraId).string()); + } + + Status ret{}; status_t res = mCameraProviderManager->getCameraCharacteristics( String8(cameraId).string(), cameraInfo); if (res != OK) { @@ -1149,6 +1172,8 @@ clientPid, states[states.size() - 1]); + resource_policy::ClientPriority clientPriority = clientDescriptor->getPriority(); + // Find clients that would be evicted auto evicted = mActiveClientManager.wouldEvict(clientDescriptor); @@ -1166,8 +1191,7 @@ String8 msg = String8::format("%s : DENIED connect device %s client for package %s " "(PID %d, score %d state %d) due to eviction policy", curTime.string(), cameraId.string(), packageName.string(), clientPid, - priorityScores[priorityScores.size() - 1], - states[states.size() - 1]); + clientPriority.getScore(), clientPriority.getState()); for (auto& i : incompatibleClients) { msg.appendFormat("\n - Blocked by existing device %s client for package %s" @@ -1212,9 +1236,8 @@ i->getKey().string(), String8{clientSp->getPackageName()}.string(), i->getOwnerId(), i->getPriority().getScore(), i->getPriority().getState(), cameraId.string(), - packageName.string(), clientPid, - priorityScores[priorityScores.size() - 1], - states[states.size() - 1])); + packageName.string(), clientPid, clientPriority.getScore(), + clientPriority.getState())); // Notify the client of disconnection clientSp->notifyError(hardware::camera2::ICameraDeviceCallbacks::ERROR_CAMERA_DISCONNECTED, @@ -1330,7 +1353,7 @@ // publically hidden, we should reject the connection. if (!hardware::IPCThreadState::self()->isServingCall() && CameraThreadState::getCallingPid() != getpid() && - mCameraProviderManager->isPublicallyHiddenSecureCamera(cameraId.c_str())) { + isPublicallyHiddenSecureCamera(cameraId)) { return true; } return false; @@ -1348,14 +1371,19 @@ Status ret = Status::ok(); String8 id = String8(cameraId); sp<CameraDeviceClient> client = nullptr; - + String16 clientPackageNameAdj = clientPackageName; + if (hardware::IPCThreadState::self()->isServingCall()) { + std::string vendorClient = + StringPrintf("vendor.client.pid<%d>", CameraThreadState::getCallingPid()); + clientPackageNameAdj = String16(vendorClient.c_str()); + } ret = connectHelper<hardware::camera2::ICameraDeviceCallbacks,CameraDeviceClient>(cameraCb, id, /*api1CameraId*/-1, - CAMERA_HAL_API_VERSION_UNSPECIFIED, clientPackageName, + CAMERA_HAL_API_VERSION_UNSPECIFIED, clientPackageNameAdj, clientUid, USE_CALLING_PID, API_2, /*shimUpdateOnly*/ false, /*out*/client); if(!ret.isOk()) { - logRejected(id, CameraThreadState::getCallingPid(), String8(clientPackageName), + logRejected(id, CameraThreadState::getCallingPid(), String8(clientPackageNameAdj), ret.toString8()); return ret; } @@ -1799,16 +1827,25 @@ { Mutex::Autolock lock(mCameraStatesLock); for (auto& i : mCameraStates) { - if (!isVendorListener && - mCameraProviderManager->isPublicallyHiddenSecureCamera(i.first.c_str())) { - ALOGV("Cannot add public listener for hidden system-only %s for pid %d", - i.first.c_str(), CameraThreadState::getCallingPid()); - continue; - } cameraStatuses->emplace_back(i.first, mapToInterface(i.second->getStatus())); } } + // Remove the camera statuses that should be hidden from the client, we do + // this after collecting the states in order to avoid holding + // mCameraStatesLock and mInterfaceLock (held in + // isPublicallyHiddenSecureCamera()) at the same time. + cameraStatuses->erase(std::remove_if(cameraStatuses->begin(), cameraStatuses->end(), + [this, &isVendorListener](const hardware::CameraStatus& s) { + bool ret = !isVendorListener && isPublicallyHiddenSecureCamera(s.cameraId); + if (ret) { + ALOGV("Cannot add public listener for hidden system-only %s for pid %d", + s.cameraId.c_str(), CameraThreadState::getCallingPid()); + } + return ret; + }), + cameraStatuses->end()); + /* * Immediately signal current torch status to this listener only * This may be a subset of all the devices, so don't include it in the response directly @@ -2368,11 +2405,7 @@ } mClientPackageName = packages[0]; } - if (hardware::IPCThreadState::self()->isServingCall()) { - std::string vendorClient = - StringPrintf("vendor.client.pid<%d>", CameraThreadState::getCallingPid()); - mClientPackageName = String16(vendorClient.c_str()); - } else { + if (!hardware::IPCThreadState::self()->isServingCall()) { mAppOpsManager = std::make_unique<AppOpsManager>(); } } @@ -2872,8 +2905,9 @@ // ---------------------------------------------------------------------------- CameraService::CameraState::CameraState(const String8& id, int cost, - const std::set<String8>& conflicting) : mId(id), - mStatus(StatusInternal::NOT_PRESENT), mCost(cost), mConflicting(conflicting) {} + const std::set<String8>& conflicting, bool isHidden) : mId(id), + mStatus(StatusInternal::NOT_PRESENT), mCost(cost), mConflicting(conflicting), + mIsPublicallyHiddenSecureCamera(isHidden) {} CameraService::CameraState::~CameraState() {} @@ -2902,6 +2936,10 @@ return mId; } +bool CameraService::CameraState::isPublicallyHiddenSecureCamera() const { + return mIsPublicallyHiddenSecureCamera; +} + // ---------------------------------------------------------------------------- // ClientEventListener // ---------------------------------------------------------------------------- @@ -3237,10 +3275,10 @@ cameraId.string()); return; } - + bool isHidden = isPublicallyHiddenSecureCamera(cameraId); // Update the status for this camera state, then send the onStatusChangedCallbacks to each // of the listeners with both the mStatusStatus and mStatusListenerLock held - state->updateStatus(status, cameraId, rejectSourceStates, [this] + state->updateStatus(status, cameraId, rejectSourceStates, [this,&isHidden] (const String8& cameraId, StatusInternal status) { if (status != StatusInternal::ENUMERATING) { @@ -3262,8 +3300,7 @@ Mutex::Autolock lock(mStatusListenerLock); for (auto& listener : mListenerList) { - if (!listener.first && - mCameraProviderManager->isPublicallyHiddenSecureCamera(cameraId.c_str())) { + if (!listener.first && isHidden) { ALOGV("Skipping camera discovery callback for system-only camera %s", cameraId.c_str()); continue;
diff --git a/services/camera/libcameraservice/CameraService.h b/services/camera/libcameraservice/CameraService.h index 065157d..22842a1 100644 --- a/services/camera/libcameraservice/CameraService.h +++ b/services/camera/libcameraservice/CameraService.h
@@ -470,7 +470,8 @@ * Make a new CameraState and set the ID, cost, and conflicting devices using the values * returned in the HAL's camera_info struct for each device. */ - CameraState(const String8& id, int cost, const std::set<String8>& conflicting); + CameraState(const String8& id, int cost, const std::set<String8>& conflicting, + bool isHidden); virtual ~CameraState(); /** @@ -522,6 +523,11 @@ */ String8 getId() const; + /** + * Return if the camera device is a publically hidden secure camera + */ + bool isPublicallyHiddenSecureCamera() const; + private: const String8 mId; StatusInternal mStatus; // protected by mStatusLock @@ -529,6 +535,7 @@ std::set<String8> mConflicting; mutable Mutex mStatusLock; CameraParameters mShimParams; + const bool mIsPublicallyHiddenSecureCamera; }; // class CameraState // Observer for UID lifecycle enforcing that UIDs in idle @@ -633,7 +640,9 @@ // Should an operation attempt on a cameraId be rejected, if the camera id is // advertised as a publically hidden secure camera, by the camera HAL ? - bool shouldRejectHiddenCameraConnection(const String8 & cameraId); + bool shouldRejectHiddenCameraConnection(const String8& cameraId); + + bool isPublicallyHiddenSecureCamera(const String8& cameraId); // Single implementation shared between the various connect calls template<class CALLBACK, class CLIENT>
diff --git a/services/camera/libcameraservice/api2/CameraDeviceClient.cpp b/services/camera/libcameraservice/api2/CameraDeviceClient.cpp index c7a4f2b..3587db3 100644 --- a/services/camera/libcameraservice/api2/CameraDeviceClient.cpp +++ b/services/camera/libcameraservice/api2/CameraDeviceClient.cpp
@@ -2004,6 +2004,15 @@ } } + for (size_t i = 0; i < mCompositeStreamMap.size(); i++) { + auto ret = mCompositeStreamMap.valueAt(i)->deleteInternalStreams(); + if (ret != OK) { + ALOGE("%s: Failed removing composite stream %s (%d)", __FUNCTION__, + strerror(-ret), ret); + } + } + mCompositeStreamMap.clear(); + Camera2ClientBase::detachDevice(); }
diff --git a/services/camera/libcameraservice/api2/DepthCompositeStream.cpp b/services/camera/libcameraservice/api2/DepthCompositeStream.cpp index 8ebaa2b..0b91016 100644 --- a/services/camera/libcameraservice/api2/DepthCompositeStream.cpp +++ b/services/camera/libcameraservice/api2/DepthCompositeStream.cpp
@@ -247,7 +247,7 @@ return ret; } -status_t DepthCompositeStream::processInputFrame(const InputFrame &inputFrame) { +status_t DepthCompositeStream::processInputFrame(nsecs_t ts, const InputFrame &inputFrame) { status_t res; sp<ANativeWindow> outputANW = mOutputSurface; ANativeWindowBuffer *anb; @@ -370,6 +370,13 @@ return NO_MEMORY; } + res = native_window_set_buffers_timestamp(mOutputSurface.get(), ts); + if (res != OK) { + ALOGE("%s: Stream %d: Error setting timestamp: %s (%d)", __FUNCTION__, + getStreamId(), strerror(-res), res); + return res; + } + ALOGV("%s: Final jpeg size: %zu", __func__, finalJpegSize); uint8_t* header = static_cast<uint8_t *> (dstBuffer) + (gb->getWidth() - sizeof(struct camera3_jpeg_blob)); @@ -459,7 +466,7 @@ } } - auto res = processInputFrame(mPendingInputFrames[currentTs]); + auto res = processInputFrame(currentTs, mPendingInputFrames[currentTs]); Mutex::Autolock l(mMutex); if (res != OK) { ALOGE("%s: Failed processing frame with timestamp: %" PRIu64 ": %s (%d)", __FUNCTION__,
diff --git a/services/camera/libcameraservice/api2/DepthCompositeStream.h b/services/camera/libcameraservice/api2/DepthCompositeStream.h index 975c59b..28a7826 100644 --- a/services/camera/libcameraservice/api2/DepthCompositeStream.h +++ b/services/camera/libcameraservice/api2/DepthCompositeStream.h
@@ -97,7 +97,7 @@ size_t maxJpegSize, uint8_t jpegQuality, std::vector<std::unique_ptr<Item>>* items /*out*/); std::unique_ptr<ImagingModel> getImagingModel(); - status_t processInputFrame(const InputFrame &inputFrame); + status_t processInputFrame(nsecs_t ts, const InputFrame &inputFrame); // Buffer/Results handling void compilePendingInputLocked();
diff --git a/services/camera/libcameraservice/api2/HeicCompositeStream.cpp b/services/camera/libcameraservice/api2/HeicCompositeStream.cpp index 5a87134..9f15be0 100644 --- a/services/camera/libcameraservice/api2/HeicCompositeStream.cpp +++ b/services/camera/libcameraservice/api2/HeicCompositeStream.cpp
@@ -31,7 +31,6 @@ #include <media/ICrypto.h> #include <media/MediaCodecBuffer.h> #include <media/stagefright/foundation/ABuffer.h> -#include <media/stagefright/foundation/AMessage.h> #include <media/stagefright/foundation/MediaDefs.h> #include <media/stagefright/MediaCodecConstants.h> @@ -61,12 +60,13 @@ mUseGrid(false), mAppSegmentStreamId(-1), mAppSegmentSurfaceId(-1), - mAppSegmentBufferAcquired(false), mMainImageStreamId(-1), mMainImageSurfaceId(-1), mYuvBufferAcquired(false), mProducerListener(new ProducerListener()), - mOutputBufferCounter(0), + mDequeuedOutputBufferCnt(0), + mLockedAppSegmentBufferCnt(0), + mCodecOutputCounter(0), mGridTimestampUs(0) { } @@ -132,7 +132,7 @@ sp<IGraphicBufferProducer> producer; sp<IGraphicBufferConsumer> consumer; BufferQueue::createBufferQueue(&producer, &consumer); - mAppSegmentConsumer = new CpuConsumer(consumer, 1); + mAppSegmentConsumer = new CpuConsumer(consumer, kMaxAcquiredAppSegment); mAppSegmentConsumer->setFrameAvailableListener(this); mAppSegmentConsumer->setName(String8("Camera3-HeicComposite-AppSegmentStream")); mAppSegmentSurface = new Surface(producer); @@ -231,6 +231,8 @@ if (bufferInfo.mError) return; mCodecOutputBufferTimestamps.push(bufferInfo.mTimestamp); + ALOGV("%s: [%" PRId64 "]: Adding codecOutputBufferTimestamp (%zu timestamps in total)", + __FUNCTION__, bufferInfo.mTimestamp, mCodecOutputBufferTimestamps.size()); } // We need to get the settings early to handle the case where the codec output @@ -361,6 +363,8 @@ mCodecOutputBuffers.push_back(outputBufferInfo); mInputReadyCondition.signal(); } else { + ALOGV("%s: Releasing output buffer: size %d flags: 0x%x ", __FUNCTION__, + outputBufferInfo.size, outputBufferInfo.flags); mCodec->releaseOutputBuffer(outputBufferInfo.index); } } else { @@ -414,8 +418,10 @@ mNumOutputTiles = 1; } - ALOGV("%s: mNumOutputTiles is %zu", __FUNCTION__, mNumOutputTiles); mFormat = newFormat; + + ALOGV("%s: mNumOutputTiles is %zu", __FUNCTION__, mNumOutputTiles); + mInputReadyCondition.signal(); } void HeicCompositeStream::onHeicCodecError() { @@ -459,9 +465,8 @@ // Cannot use SourceSurface buffer count since it could be codec's 512*512 tile // buffer count. - int maxProducerBuffers = 1; if ((res = native_window_set_buffer_count( - anwConsumer, maxProducerBuffers + maxConsumerBuffers)) != OK) { + anwConsumer, kMaxOutputSurfaceProducerCount + maxConsumerBuffers)) != OK) { ALOGE("%s: Unable to set buffer count for stream %d", __FUNCTION__, mMainImageStreamId); return res; } @@ -505,6 +510,8 @@ } if (mSettingsByFrameNumber.find(resultExtras.frameNumber) != mSettingsByFrameNumber.end()) { + ALOGV("%s: [%" PRId64 "]: frameNumber %" PRId64, __FUNCTION__, + timestamp, resultExtras.frameNumber); mFrameNumberMap.emplace(resultExtras.frameNumber, timestamp); mSettingsByTimestamp[timestamp] = mSettingsByFrameNumber[resultExtras.frameNumber]; mSettingsByFrameNumber.erase(resultExtras.frameNumber); @@ -520,12 +527,12 @@ mSettingsByTimestamp.erase(it); } - while (!mInputAppSegmentBuffers.empty() && !mAppSegmentBufferAcquired) { + while (!mInputAppSegmentBuffers.empty()) { CpuConsumer::LockedBuffer imgBuffer; auto it = mInputAppSegmentBuffers.begin(); auto res = mAppSegmentConsumer->lockNextBuffer(&imgBuffer); if (res == NOT_ENOUGH_DATA) { - // Canot not lock any more buffers. + // Can not lock any more buffers. break; } else if ((res != OK) || (*it != imgBuffer.timestamp)) { if (res != OK) { @@ -535,6 +542,7 @@ ALOGE("%s: Expecting JPEG_APP_SEGMENTS buffer with time stamp: %" PRId64 " received buffer with time stamp: %" PRId64, __FUNCTION__, *it, imgBuffer.timestamp); + mAppSegmentConsumer->unlockBuffer(imgBuffer); } mPendingInputFrames[*it].error = true; mInputAppSegmentBuffers.erase(it); @@ -546,7 +554,7 @@ mAppSegmentConsumer->unlockBuffer(imgBuffer); } else { mPendingInputFrames[imgBuffer.timestamp].appSegmentBuffer = imgBuffer; - mAppSegmentBufferAcquired = true; + mLockedAppSegmentBufferCnt++; } mInputAppSegmentBuffers.erase(it); } @@ -556,7 +564,7 @@ auto it = mInputYuvBuffers.begin(); auto res = mMainImageConsumer->lockNextBuffer(&imgBuffer); if (res == NOT_ENOUGH_DATA) { - // Canot not lock any more buffers. + // Can not lock any more buffers. break; } else if (res != OK) { ALOGE("%s: Error locking YUV_888 image buffer: %s (%d)", __FUNCTION__, @@ -589,17 +597,20 @@ // to look up timestamp. int64_t bufferTime = -1; if (mCodecOutputBufferTimestamps.empty()) { - ALOGE("%s: Failed to find buffer timestamp for codec output buffer!", __FUNCTION__); + ALOGV("%s: Failed to find buffer timestamp for codec output buffer!", __FUNCTION__); + break; } else { // Direct mapping between camera timestamp (in ns) and codec timestamp (in us). bufferTime = mCodecOutputBufferTimestamps.front(); - mOutputBufferCounter++; - if (mOutputBufferCounter == mNumOutputTiles) { + mCodecOutputCounter++; + if (mCodecOutputCounter == mNumOutputTiles) { mCodecOutputBufferTimestamps.pop(); - mOutputBufferCounter = 0; + mCodecOutputCounter = 0; } mPendingInputFrames[bufferTime].codecOutputBuffers.push_back(*it); + ALOGV("%s: [%" PRId64 "]: Pushing codecOutputBuffers (time %" PRId64 " us)", + __FUNCTION__, bufferTime, it->timeUs); } mCodecOutputBuffers.erase(it); } @@ -607,6 +618,7 @@ while (!mFrameNumberMap.empty()) { auto it = mFrameNumberMap.begin(); mPendingInputFrames[it->second].frameNumber = it->first; + ALOGV("%s: [%" PRId64 "]: frameNumber is %" PRId64, __FUNCTION__, it->second, it->first); mFrameNumberMap.erase(it); } @@ -675,16 +687,29 @@ } bool newInputAvailable = false; - for (const auto& it : mPendingInputFrames) { + for (auto& it : mPendingInputFrames) { + // New input is considered to be available only if: + // 1. input buffers are ready, or + // 2. App segment and muxer is created, or + // 3. A codec output tile is ready, and an output buffer is available. + // This makes sure that muxer gets created only when an output tile is + // generated, because right now we only handle 1 HEIC output buffer at a + // time (max dequeued buffer count is 1). bool appSegmentReady = (it.second.appSegmentBuffer.data != nullptr) && - !it.second.appSegmentWritten && it.second.result != nullptr; + !it.second.appSegmentWritten && it.second.result != nullptr && + it.second.muxer != nullptr; bool codecOutputReady = !it.second.codecOutputBuffers.empty(); bool codecInputReady = (it.second.yuvBuffer.data != nullptr) && (!it.second.codecInputBuffers.empty()); + bool hasOutputBuffer = it.second.muxer != nullptr || + (mDequeuedOutputBufferCnt < kMaxOutputSurfaceProducerCount); if ((!it.second.error) && (it.first < *currentTs) && - (appSegmentReady || codecOutputReady || codecInputReady)) { + (appSegmentReady || (codecOutputReady && hasOutputBuffer) || codecInputReady)) { *currentTs = it.first; + if (it.second.format == nullptr && mFormat != nullptr) { + it.second.format = mFormat->dup(); + } newInputAvailable = true; break; } @@ -716,15 +741,17 @@ status_t res = OK; bool appSegmentReady = inputFrame.appSegmentBuffer.data != nullptr && - !inputFrame.appSegmentWritten && inputFrame.result != nullptr; + !inputFrame.appSegmentWritten && inputFrame.result != nullptr && + inputFrame.muxer != nullptr; bool codecOutputReady = inputFrame.codecOutputBuffers.size() > 0; bool codecInputReady = inputFrame.yuvBuffer.data != nullptr && - !inputFrame.codecInputBuffers.empty(); + !inputFrame.codecInputBuffers.empty(); + bool hasOutputBuffer = inputFrame.muxer != nullptr || + (mDequeuedOutputBufferCnt < kMaxOutputSurfaceProducerCount); - if (!appSegmentReady && !codecOutputReady && !codecInputReady) { - ALOGW("%s: No valid appSegmentBuffer/codec input/outputBuffer available!", __FUNCTION__); - return OK; - } + ALOGV("%s: [%" PRId64 "]: appSegmentReady %d, codecOutputReady %d, codecInputReady %d," + " dequeuedOutputBuffer %d", __FUNCTION__, timestamp, appSegmentReady, + codecOutputReady, codecInputReady, mDequeuedOutputBufferCnt); // Handle inputs for Hevc tiling if (codecInputReady) { @@ -736,7 +763,13 @@ } } - // Initialize and start muxer if not yet done so + if (!(codecOutputReady && hasOutputBuffer) && !appSegmentReady) { + return OK; + } + + // Initialize and start muxer if not yet done so. In this case, + // codecOutputReady must be true. Otherwise, appSegmentReady is guaranteed + // to be false, and the function must have returned early. if (inputFrame.muxer == nullptr) { res = startMuxerForInputFrame(timestamp, inputFrame); if (res != OK) { @@ -747,7 +780,7 @@ } // Write JPEG APP segments data to the muxer. - if (appSegmentReady && inputFrame.muxer != nullptr) { + if (appSegmentReady) { res = processAppSegment(timestamp, inputFrame); if (res != OK) { ALOGE("%s: Failed to process JPEG APP segments: %s (%d)", __FUNCTION__, @@ -766,12 +799,18 @@ } } - if (inputFrame.appSegmentWritten && inputFrame.pendingOutputTiles == 0) { - res = processCompletedInputFrame(timestamp, inputFrame); - if (res != OK) { - ALOGE("%s: Failed to process completed input frame: %s (%d)", __FUNCTION__, - strerror(-res), res); - return res; + if (inputFrame.pendingOutputTiles == 0) { + if (inputFrame.appSegmentWritten) { + res = processCompletedInputFrame(timestamp, inputFrame); + if (res != OK) { + ALOGE("%s: Failed to process completed input frame: %s (%d)", __FUNCTION__, + strerror(-res), res); + return res; + } + } else if (mLockedAppSegmentBufferCnt == kMaxAcquiredAppSegment) { + ALOGE("%s: Out-of-order app segment buffers reaches limit %u", __FUNCTION__, + kMaxAcquiredAppSegment); + return INVALID_OPERATION; } } @@ -780,11 +819,6 @@ status_t HeicCompositeStream::startMuxerForInputFrame(nsecs_t timestamp, InputFrame &inputFrame) { sp<ANativeWindow> outputANW = mOutputSurface; - if (inputFrame.codecOutputBuffers.size() == 0) { - // No single codec output buffer has been generated. Continue to - // wait. - return OK; - } auto res = outputANW->dequeueBuffer(mOutputSurface.get(), &inputFrame.anb, &inputFrame.fenceFd); if (res != OK) { @@ -792,6 +826,7 @@ res); return res; } + mDequeuedOutputBufferCnt++; // Combine current thread id, stream id and timestamp to uniquely identify image. std::ostringstream tempOutputFile; @@ -828,7 +863,7 @@ } } - ssize_t trackId = inputFrame.muxer->addTrack(mFormat); + ssize_t trackId = inputFrame.muxer->addTrack(inputFrame.format); if (trackId < 0) { ALOGE("%s: Failed to addTrack to the muxer: %zd", __FUNCTION__, trackId); return NO_INIT; @@ -844,6 +879,8 @@ return res; } + ALOGV("%s: [%" PRId64 "]: Muxer started for inputFrame", __FUNCTION__, + timestamp); return OK; } @@ -852,9 +889,6 @@ auto appSegmentSize = findAppSegmentsSize(inputFrame.appSegmentBuffer.data, inputFrame.appSegmentBuffer.width * inputFrame.appSegmentBuffer.height, &app1Size); - ALOGV("%s: appSegmentSize is %zu, width %d, height %d, app1Size %zu", __FUNCTION__, - appSegmentSize, inputFrame.appSegmentBuffer.width, - inputFrame.appSegmentBuffer.height, app1Size); if (appSegmentSize == 0) { ALOGE("%s: Failed to find JPEG APP segment size", __FUNCTION__); return NO_INIT; @@ -910,7 +944,16 @@ __FUNCTION__, strerror(-res), res); return res; } + + ALOGV("%s: [%" PRId64 "]: appSegmentSize is %zu, width %d, height %d, app1Size %zu", + __FUNCTION__, timestamp, appSegmentSize, inputFrame.appSegmentBuffer.width, + inputFrame.appSegmentBuffer.height, app1Size); + inputFrame.appSegmentWritten = true; + // Release the buffer now so any pending input app segments can be processed + mAppSegmentConsumer->unlockBuffer(inputFrame.appSegmentBuffer); + inputFrame.appSegmentBuffer.data = nullptr; + mLockedAppSegmentBufferCnt--; return OK; } @@ -934,8 +977,9 @@ mOutputWidth - tileX * mGridWidth : mGridWidth; size_t height = (tileY == static_cast<size_t>(mGridRows) - 1) ? mOutputHeight - tileY * mGridHeight : mGridHeight; - ALOGV("%s: inputBuffer tileIndex [%zu, %zu], top %zu, left %zu, width %zu, height %zu", - __FUNCTION__, tileX, tileY, top, left, width, height); + ALOGV("%s: inputBuffer tileIndex [%zu, %zu], top %zu, left %zu, width %zu, height %zu," + " timeUs %" PRId64, __FUNCTION__, tileX, tileY, top, left, width, height, + inputBuffer.timeUs); res = copyOneYuvTile(buffer, inputFrame.yuvBuffer, top, left, width, height); if (res != OK) { @@ -990,6 +1034,9 @@ } inputFrame.codecOutputBuffers.erase(inputFrame.codecOutputBuffers.begin()); + + ALOGV("%s: [%" PRId64 "]: Output buffer index %d", + __FUNCTION__, timestamp, it->index); return OK; } @@ -1046,7 +1093,9 @@ return res; } inputFrame.anb = nullptr; + mDequeuedOutputBufferCnt--; + ALOGV("%s: [%" PRId64 "]", __FUNCTION__, timestamp); ATRACE_ASYNC_END("HEIC capture", inputFrame.frameNumber); return OK; } @@ -1060,7 +1109,6 @@ if (inputFrame->appSegmentBuffer.data != nullptr) { mAppSegmentConsumer->unlockBuffer(inputFrame->appSegmentBuffer); inputFrame->appSegmentBuffer.data = nullptr; - mAppSegmentBufferAcquired = false; } while (!inputFrame->codecOutputBuffers.empty()) { @@ -1098,11 +1146,13 @@ } } -void HeicCompositeStream::releaseInputFramesLocked(int64_t currentTs) { +void HeicCompositeStream::releaseInputFramesLocked() { auto it = mPendingInputFrames.begin(); while (it != mPendingInputFrames.end()) { - if (it->first <= currentTs) { - releaseInputFrameLocked(&it->second); + auto& inputFrame = it->second; + if (inputFrame.error || + (inputFrame.appSegmentWritten && inputFrame.pendingOutputTiles == 0)) { + releaseInputFrameLocked(&inputFrame); it = mPendingInputFrames.erase(it); } else { it++; @@ -1506,7 +1556,7 @@ // In case we landed in error state, return any pending buffers and // halt all further processing. compilePendingInputLocked(); - releaseInputFramesLocked(currentTs); + releaseInputFramesLocked(); return false; } @@ -1548,11 +1598,7 @@ mPendingInputFrames[currentTs].error = true; } - if (mPendingInputFrames[currentTs].error || - (mPendingInputFrames[currentTs].appSegmentWritten && - mPendingInputFrames[currentTs].pendingOutputTiles == 0)) { - releaseInputFramesLocked(currentTs); - } + releaseInputFramesLocked(); return true; } @@ -1671,8 +1717,13 @@ ALOGE("CB_OUTPUT_FORMAT_CHANGED: format is expected."); break; } - - parent->onHeicFormatChanged(format); + // Here format is MediaCodec's internal copy of output format. + // Make a copy since onHeicFormatChanged() might modify it. + sp<AMessage> formatCopy; + if (format != nullptr) { + formatCopy = format->dup(); + } + parent->onHeicFormatChanged(formatCopy); break; }
diff --git a/services/camera/libcameraservice/api2/HeicCompositeStream.h b/services/camera/libcameraservice/api2/HeicCompositeStream.h index 260c68e..04e7b83 100644 --- a/services/camera/libcameraservice/api2/HeicCompositeStream.h +++ b/services/camera/libcameraservice/api2/HeicCompositeStream.h
@@ -25,6 +25,7 @@ #include <media/hardware/VideoAPI.h> #include <media/MediaCodecBuffer.h> #include <media/stagefright/foundation/ALooper.h> +#include <media/stagefright/foundation/AMessage.h> #include <media/stagefright/MediaCodec.h> #include <media/stagefright/MediaMuxer.h> @@ -157,6 +158,7 @@ bool errorNotified; int64_t frameNumber; + sp<AMessage> format; sp<MediaMuxer> muxer; int fenceFd; int fileFd; @@ -187,7 +189,7 @@ status_t processCompletedInputFrame(nsecs_t timestamp, InputFrame &inputFrame); void releaseInputFrameLocked(InputFrame *inputFrame /*out*/); - void releaseInputFramesLocked(int64_t currentTs); + void releaseInputFramesLocked(); size_t findAppSegmentsSize(const uint8_t* appSegmentBuffer, size_t maxSize, size_t* app1SegmentSize); @@ -205,11 +207,13 @@ static_cast<android_dataspace>(HAL_DATASPACE_JPEG_APP_SEGMENTS); static const android_dataspace kHeifDataSpace = static_cast<android_dataspace>(HAL_DATASPACE_HEIF); + // Use the limit of pipeline depth in the API sepc as maximum number of acquired + // app segment buffers. + static const uint32_t kMaxAcquiredAppSegment = 8; int mAppSegmentStreamId, mAppSegmentSurfaceId; sp<CpuConsumer> mAppSegmentConsumer; sp<Surface> mAppSegmentSurface; - bool mAppSegmentBufferAcquired; size_t mAppSegmentMaxSize; CameraMetadata mStaticInfo; @@ -218,9 +222,10 @@ sp<CpuConsumer> mMainImageConsumer; // Only applicable for HEVC codec. bool mYuvBufferAcquired; // Only applicable to HEVC codec + static const int32_t kMaxOutputSurfaceProducerCount = 1; sp<Surface> mOutputSurface; sp<ProducerListener> mProducerListener; - + int32_t mDequeuedOutputBufferCnt; // Map from frame number to JPEG setting of orientation+quality std::map<int64_t, std::pair<int32_t, int32_t>> mSettingsByFrameNumber; @@ -229,11 +234,12 @@ // Keep all incoming APP segment Blob buffer pending further processing. std::vector<int64_t> mInputAppSegmentBuffers; + int32_t mLockedAppSegmentBufferCnt; // Keep all incoming HEIC blob buffer pending further processing. std::vector<CodecOutputBufferInfo> mCodecOutputBuffers; std::queue<int64_t> mCodecOutputBufferTimestamps; - size_t mOutputBufferCounter; + size_t mCodecOutputCounter; // Keep all incoming Yuv buffer pending tiling and encoding (for HEVC YUV tiling only) std::vector<int64_t> mInputYuvBuffers;
diff --git a/services/camera/libcameraservice/common/CameraProviderManager.cpp b/services/camera/libcameraservice/common/CameraProviderManager.cpp index 09638d0..fdb5657 100644 --- a/services/camera/libcameraservice/common/CameraProviderManager.cpp +++ b/services/camera/libcameraservice/common/CameraProviderManager.cpp
@@ -29,6 +29,7 @@ #include <future> #include <inttypes.h> #include <hardware/camera_common.h> +#include <android/hidl/manager/1.2/IServiceManager.h> #include <hidl/ServiceManagement.h> #include <functional> #include <camera_metadata_hidden.h> @@ -47,10 +48,6 @@ using std::literals::chrono_literals::operator""s; namespace { -// Hardcoded name for the passthrough HAL implementation, since it can't be discovered via the -// service manager -const std::string kLegacyProviderName("legacy/0"); -const std::string kExternalProviderName("external/0"); const bool kEnableLazyHal(property_get_bool("ro.camera.enableLazyHal", false)); } // anonymous namespace @@ -62,6 +59,19 @@ CameraProviderManager::~CameraProviderManager() { } +hardware::hidl_vec<hardware::hidl_string> +CameraProviderManager::HardwareServiceInteractionProxy::listServices() { + hardware::hidl_vec<hardware::hidl_string> ret; + auto manager = hardware::defaultServiceManager1_2(); + if (manager != nullptr) { + manager->listManifestByInterface(provider::V2_4::ICameraProvider::descriptor, + [&ret](const hardware::hidl_vec<hardware::hidl_string> ®istered) { + ret = registered; + }); + } + return ret; +} + status_t CameraProviderManager::initialize(wp<CameraProviderManager::StatusListener> listener, ServiceInteractionProxy* proxy) { std::lock_guard<std::mutex> lock(mInterfaceMutex); @@ -84,9 +94,10 @@ return INVALID_OPERATION; } - // See if there's a passthrough HAL, but let's not complain if there's not - addProviderLocked(kLegacyProviderName, /*expected*/ false); - addProviderLocked(kExternalProviderName, /*expected*/ false); + + for (const auto& instance : mServiceProxy->listServices()) { + this->addProviderLocked(instance); + } IPCThreadState::self()->flushCommands(); @@ -97,7 +108,13 @@ std::lock_guard<std::mutex> lock(mInterfaceMutex); int count = 0; for (auto& provider : mProviders) { - count += provider->mUniqueCameraIds.size(); + for (auto& id : provider->mUniqueCameraIds) { + // Hidden secure camera ids are not to be exposed to camera1 api. + if (isPublicallyHiddenSecureCameraLocked(id)) { + continue; + } + count++; + } } return count; } @@ -123,7 +140,11 @@ // for each camera facing, only take the first id advertised by HAL in // all [logical, physical1, physical2, ...] id combos, and filter out the rest. filterLogicalCameraIdsLocked(providerDeviceIds); - + // Hidden secure camera ids are not to be exposed to camera1 api. + providerDeviceIds.erase(std::remove_if(providerDeviceIds.begin(), providerDeviceIds.end(), + [this](const std::string& s) { + return this->isPublicallyHiddenSecureCameraLocked(s);}), + providerDeviceIds.end()); deviceIds.insert(deviceIds.end(), providerDeviceIds.begin(), providerDeviceIds.end()); } @@ -1035,23 +1056,42 @@ return deviceInfo->mIsLogicalCamera; } -bool CameraProviderManager::isPublicallyHiddenSecureCamera(const std::string& id) { +bool CameraProviderManager::isPublicallyHiddenSecureCamera(const std::string& id) const { std::lock_guard<std::mutex> lock(mInterfaceMutex); - - auto deviceInfo = findDeviceInfoLocked(id); - if (deviceInfo == nullptr) { - return false; - } - return deviceInfo->mIsPublicallyHiddenSecureCamera; + return isPublicallyHiddenSecureCameraLocked(id); } -bool CameraProviderManager::isHiddenPhysicalCamera(const std::string& cameraId) { +bool CameraProviderManager::isPublicallyHiddenSecureCameraLocked(const std::string& id) const { + auto deviceInfo = findDeviceInfoLocked(id); + if (deviceInfo != nullptr) { + return deviceInfo->mIsPublicallyHiddenSecureCamera; + } + // If this is a hidden physical camera, we should return what kind of + // camera the enclosing logical camera is. + auto isHiddenAndParent = isHiddenPhysicalCameraInternal(id); + if (isHiddenAndParent.first) { + LOG_ALWAYS_FATAL_IF(id == isHiddenAndParent.second->mId, + "%s: hidden physical camera id %s and enclosing logical camera id %s are the same", + __FUNCTION__, id.c_str(), isHiddenAndParent.second->mId.c_str()); + return isPublicallyHiddenSecureCameraLocked(isHiddenAndParent.second->mId); + } + // Invalid camera id + return true; +} + +bool CameraProviderManager::isHiddenPhysicalCamera(const std::string& cameraId) const { + return isHiddenPhysicalCameraInternal(cameraId).first; +} + +std::pair<bool, CameraProviderManager::ProviderInfo::DeviceInfo *> +CameraProviderManager::isHiddenPhysicalCameraInternal(const std::string& cameraId) const { + auto falseRet = std::make_pair(false, nullptr); for (auto& provider : mProviders) { for (auto& deviceInfo : provider->mDevices) { if (deviceInfo->mId == cameraId) { // cameraId is found in public camera IDs advertised by the // provider. - return false; + return falseRet; } } } @@ -1063,7 +1103,7 @@ if (res != OK) { ALOGE("%s: Failed to getCameraCharacteristics for id %s", __FUNCTION__, deviceInfo->mId.c_str()); - return false; + return falseRet; } std::vector<std::string> physicalIds; @@ -1075,19 +1115,19 @@ if (deviceVersion < CAMERA_DEVICE_API_VERSION_3_5) { ALOGE("%s: Wrong deviceVersion %x for hiddenPhysicalCameraId %s", __FUNCTION__, deviceVersion, cameraId.c_str()); - return false; + return falseRet; } else { - return true; + return std::make_pair(true, deviceInfo.get()); } } } } } - return false; + return falseRet; } -status_t CameraProviderManager::addProviderLocked(const std::string& newProvider, bool expected) { +status_t CameraProviderManager::addProviderLocked(const std::string& newProvider) { for (const auto& providerInfo : mProviders) { if (providerInfo->mProviderName == newProvider) { ALOGW("%s: Camera provider HAL with name '%s' already registered", __FUNCTION__, @@ -1100,13 +1140,9 @@ interface = mServiceProxy->getService(newProvider); if (interface == nullptr) { - if (expected) { - ALOGE("%s: Camera provider HAL '%s' is not actually available", __FUNCTION__, - newProvider.c_str()); - return BAD_VALUE; - } else { - return OK; - } + ALOGE("%s: Camera provider HAL '%s' is not actually available", __FUNCTION__, + newProvider.c_str()); + return BAD_VALUE; } sp<ProviderInfo> providerInfo = new ProviderInfo(newProvider, this); @@ -2058,6 +2094,13 @@ return OK; } bool CameraProviderManager::ProviderInfo::DeviceInfo3::isAPI1Compatible() const { + // Do not advertise NIR cameras to API1 camera app. + camera_metadata_ro_entry cfa = mCameraCharacteristics.find( + ANDROID_SENSOR_INFO_COLOR_FILTER_ARRANGEMENT); + if (cfa.count == 1 && cfa.data.u8[0] == ANDROID_SENSOR_INFO_COLOR_FILTER_ARRANGEMENT_NIR) { + return false; + } + bool isBackwardCompatible = false; camera_metadata_ro_entry_t caps = mCameraCharacteristics.find( ANDROID_REQUEST_AVAILABLE_CAPABILITIES);
diff --git a/services/camera/libcameraservice/common/CameraProviderManager.h b/services/camera/libcameraservice/common/CameraProviderManager.h index a42fb4d..3a4655c 100644 --- a/services/camera/libcameraservice/common/CameraProviderManager.h +++ b/services/camera/libcameraservice/common/CameraProviderManager.h
@@ -78,6 +78,7 @@ ¬ification) = 0; virtual sp<hardware::camera::provider::V2_4::ICameraProvider> getService( const std::string &serviceName) = 0; + virtual hardware::hidl_vec<hardware::hidl_string> listServices() = 0; virtual ~ServiceInteractionProxy() {} }; @@ -95,6 +96,8 @@ const std::string &serviceName) override { return hardware::camera::provider::V2_4::ICameraProvider::getService(serviceName); } + + virtual hardware::hidl_vec<hardware::hidl_string> listServices() override; }; /** @@ -269,8 +272,8 @@ */ bool isLogicalCamera(const std::string& id, std::vector<std::string>* physicalCameraIds); - bool isPublicallyHiddenSecureCamera(const std::string& id); - bool isHiddenPhysicalCamera(const std::string& cameraId); + bool isPublicallyHiddenSecureCamera(const std::string& id) const; + bool isHiddenPhysicalCamera(const std::string& cameraId) const; static const float kDepthARTolerance; private: @@ -567,7 +570,7 @@ hardware::hidl_version minVersion = hardware::hidl_version{0,0}, hardware::hidl_version maxVersion = hardware::hidl_version{1000,0}) const; - status_t addProviderLocked(const std::string& newProvider, bool expected = true); + status_t addProviderLocked(const std::string& newProvider); status_t removeProvider(const std::string& provider); sp<StatusListener> getStatusListener() const; @@ -591,7 +594,15 @@ status_t getCameraCharacteristicsLocked(const std::string &id, CameraMetadata* characteristics) const; + + bool isPublicallyHiddenSecureCameraLocked(const std::string& id) const; + void filterLogicalCameraIdsLocked(std::vector<std::string>& deviceIds) const; + + bool isPublicallyHiddenSecureCameraLocked(const std::string& id); + + std::pair<bool, CameraProviderManager::ProviderInfo::DeviceInfo *> + isHiddenPhysicalCameraInternal(const std::string& cameraId) const; }; } // namespace android
diff --git a/services/camera/libcameraservice/common/DepthPhotoProcessor.cpp b/services/camera/libcameraservice/common/DepthPhotoProcessor.cpp index 3c90de0..94541d8 100644 --- a/services/camera/libcameraservice/common/DepthPhotoProcessor.cpp +++ b/services/camera/libcameraservice/common/DepthPhotoProcessor.cpp
@@ -419,7 +419,7 @@ std::vector<std::unique_ptr<Item>> items; std::vector<std::unique_ptr<Camera>> cameraList; - auto image = Image::FromDataForPrimaryImage("android/mainimage", &items); + auto image = Image::FromDataForPrimaryImage("image/jpeg", &items); std::unique_ptr<CameraParams> cameraParams(new CameraParams(std::move(image))); if (cameraParams == nullptr) { ALOGE("%s: Failed to initialize camera parameters", __FUNCTION__);
diff --git a/services/camera/libcameraservice/device3/Camera3Device.cpp b/services/camera/libcameraservice/device3/Camera3Device.cpp index a8e80fa..4227a3b 100644 --- a/services/camera/libcameraservice/device3/Camera3Device.cpp +++ b/services/camera/libcameraservice/device3/Camera3Device.cpp
@@ -29,6 +29,9 @@ #define CLOGE(fmt, ...) ALOGE("Camera %s: %s: " fmt, mId.string(), __FUNCTION__, \ ##__VA_ARGS__) +#define CLOGW(fmt, ...) ALOGW("Camera %s: %s: " fmt, mId.string(), __FUNCTION__, \ + ##__VA_ARGS__) + // Convenience macros for transitioning to the error state #define SET_ERR(fmt, ...) setErrorState( \ "%s: " fmt, __FUNCTION__, \ @@ -3267,14 +3270,19 @@ ALOGVV("%s: removed frame %d from InFlightMap", __FUNCTION__, frameNumber); } - // Sanity check - if we have too many in-flight frames, something has - // likely gone wrong - if (!mIsConstrainedHighSpeedConfiguration && mInFlightMap.size() > kInFlightWarnLimit) { - CLOGE("In-flight list too large: %zu", mInFlightMap.size()); - } else if (mIsConstrainedHighSpeedConfiguration && mInFlightMap.size() > - kInFlightWarnLimitHighSpeed) { - CLOGE("In-flight list too large for high speed configuration: %zu", - mInFlightMap.size()); + // Sanity check - if we have too many in-flight frames with long total inflight duration, + // something has likely gone wrong. This might still be legit only if application send in + // a long burst of long exposure requests. + if (mExpectedInflightDuration > kMinWarnInflightDuration) { + if (!mIsConstrainedHighSpeedConfiguration && mInFlightMap.size() > kInFlightWarnLimit) { + CLOGW("In-flight list too large: %zu, total inflight duration %" PRIu64, + mInFlightMap.size(), mExpectedInflightDuration); + } else if (mIsConstrainedHighSpeedConfiguration && mInFlightMap.size() > + kInFlightWarnLimitHighSpeed) { + CLOGW("In-flight list too large for high speed configuration: %zu," + "total inflight duration %" PRIu64, + mInFlightMap.size(), mExpectedInflightDuration); + } } }
diff --git a/services/camera/libcameraservice/device3/Camera3Device.h b/services/camera/libcameraservice/device3/Camera3Device.h index 6e8ac84..cae34ce 100644 --- a/services/camera/libcameraservice/device3/Camera3Device.h +++ b/services/camera/libcameraservice/device3/Camera3Device.h
@@ -227,6 +227,7 @@ static const size_t kDumpLockAttempts = 10; static const size_t kDumpSleepDuration = 100000; // 0.10 sec static const nsecs_t kActiveTimeout = 500000000; // 500 ms + static const nsecs_t kMinWarnInflightDuration = 5000000000; // 5 s static const size_t kInFlightWarnLimit = 30; static const size_t kInFlightWarnLimitHighSpeed = 256; // batch size 32 * pipe depth 8 static const nsecs_t kDefaultExpectedDuration = 100000000; // 100 ms
diff --git a/services/camera/libcameraservice/device3/Camera3OutputStream.cpp b/services/camera/libcameraservice/device3/Camera3OutputStream.cpp index acb8b3c..e1d35e8 100644 --- a/services/camera/libcameraservice/device3/Camera3OutputStream.cpp +++ b/services/camera/libcameraservice/device3/Camera3OutputStream.cpp
@@ -54,9 +54,8 @@ mState = STATE_ERROR; } - if (setId > CAMERA3_STREAM_SET_ID_INVALID) { - mBufferReleasedListener = new BufferReleasedListener(this); - } + bool needsReleaseNotify = setId > CAMERA3_STREAM_SET_ID_INVALID; + mBufferProducerListener = new BufferProducerListener(this, needsReleaseNotify); } Camera3OutputStream::Camera3OutputStream(int id, @@ -87,9 +86,8 @@ mState = STATE_ERROR; } - if (setId > CAMERA3_STREAM_SET_ID_INVALID) { - mBufferReleasedListener = new BufferReleasedListener(this); - } + bool needsReleaseNotify = setId > CAMERA3_STREAM_SET_ID_INVALID; + mBufferProducerListener = new BufferProducerListener(this, needsReleaseNotify); } Camera3OutputStream::Camera3OutputStream(int id, @@ -124,10 +122,8 @@ } mConsumerName = String8("Deferred"); - if (setId > CAMERA3_STREAM_SET_ID_INVALID) { - mBufferReleasedListener = new BufferReleasedListener(this); - } - + bool needsReleaseNotify = setId > CAMERA3_STREAM_SET_ID_INVALID; + mBufferProducerListener = new BufferProducerListener(this, needsReleaseNotify); } Camera3OutputStream::Camera3OutputStream(int id, camera3_stream_type_t type, @@ -151,9 +147,8 @@ mDropBuffers(false), mDequeueBufferLatency(kDequeueLatencyBinSize) { - if (setId > CAMERA3_STREAM_SET_ID_INVALID) { - mBufferReleasedListener = new BufferReleasedListener(this); - } + bool needsReleaseNotify = setId > CAMERA3_STREAM_SET_ID_INVALID; + mBufferProducerListener = new BufferProducerListener(this, needsReleaseNotify); // Subclasses expected to initialize mConsumer themselves } @@ -261,7 +256,7 @@ notifyBufferReleased(anwBuffer); if (mUseBufferManager) { // Return this buffer back to buffer manager. - mBufferReleasedListener->onBufferReleased(); + mBufferProducerListener->onBufferReleased(); } } else { if (mTraceFirstBuffer && (stream_type == CAMERA3_STREAM_OUTPUT)) { @@ -387,8 +382,8 @@ // Configure consumer-side ANativeWindow interface. The listener may be used // to notify buffer manager (if it is used) of the returned buffers. res = mConsumer->connect(NATIVE_WINDOW_API_CAMERA, - /*listener*/mBufferReleasedListener, - /*reportBufferRemoval*/true); + /*reportBufferRemoval*/true, + /*listener*/mBufferProducerListener); if (res != OK) { ALOGE("%s: Unable to connect to native window for stream %d", __FUNCTION__, mId); @@ -790,7 +785,7 @@ return INVALID_OPERATION; } -void Camera3OutputStream::BufferReleasedListener::onBufferReleased() { +void Camera3OutputStream::BufferProducerListener::onBufferReleased() { sp<Camera3OutputStream> stream = mParent.promote(); if (stream == nullptr) { ALOGV("%s: Parent camera3 output stream was destroyed", __FUNCTION__); @@ -823,6 +818,25 @@ } } +void Camera3OutputStream::BufferProducerListener::onBuffersDiscarded( + const std::vector<sp<GraphicBuffer>>& buffers) { + sp<Camera3OutputStream> stream = mParent.promote(); + if (stream == nullptr) { + ALOGV("%s: Parent camera3 output stream was destroyed", __FUNCTION__); + return; + } + + if (buffers.size() > 0) { + Mutex::Autolock l(stream->mLock); + stream->onBuffersRemovedLocked(buffers); + if (stream->mUseBufferManager) { + stream->mBufferManager->onBuffersRemoved(stream->getId(), + stream->getStreamSetId(), buffers.size()); + } + ALOGV("Stream %d: %zu Buffers discarded.", stream->getId(), buffers.size()); + } +} + void Camera3OutputStream::onBuffersRemovedLocked( const std::vector<sp<GraphicBuffer>>& removedBuffers) { sp<Camera3StreamBufferFreedListener> callback = mBufferFreedListener.promote();
diff --git a/services/camera/libcameraservice/device3/Camera3OutputStream.h b/services/camera/libcameraservice/device3/Camera3OutputStream.h index 729c655..b4e49f9 100644 --- a/services/camera/libcameraservice/device3/Camera3OutputStream.h +++ b/services/camera/libcameraservice/device3/Camera3OutputStream.h
@@ -146,18 +146,22 @@ */ virtual status_t setConsumers(const std::vector<sp<Surface>>& consumers); - class BufferReleasedListener : public BnProducerListener { + class BufferProducerListener : public SurfaceListener { public: - BufferReleasedListener(wp<Camera3OutputStream> parent) : mParent(parent) {} + BufferProducerListener(wp<Camera3OutputStream> parent, bool needsReleaseNotify) + : mParent(parent), mNeedsReleaseNotify(needsReleaseNotify) {} - /** - * Implementation of IProducerListener, used to notify this stream that the consumer - * has returned a buffer and it is ready to return to Camera3BufferManager for reuse. - */ - virtual void onBufferReleased(); + /** + * Implementation of IProducerListener, used to notify this stream that the consumer + * has returned a buffer and it is ready to return to Camera3BufferManager for reuse. + */ + virtual void onBufferReleased(); + virtual bool needsReleaseNotify() { return mNeedsReleaseNotify; } + virtual void onBuffersDiscarded(const std::vector<sp<GraphicBuffer>>& buffers); private: - wp<Camera3OutputStream> mParent; + wp<Camera3OutputStream> mParent; + bool mNeedsReleaseNotify; }; virtual status_t detachBuffer(sp<GraphicBuffer>* buffer, int* fenceFd); @@ -262,10 +266,10 @@ sp<Camera3BufferManager> mBufferManager; /** - * Buffer released listener, used to notify the buffer manager that a buffer is released - * from consumer side. + * Buffer producer listener, used to handle notification when a buffer is released + * from consumer side, or a set of buffers are discarded by the consumer. */ - sp<BufferReleasedListener> mBufferReleasedListener; + sp<BufferProducerListener> mBufferProducerListener; /** * Flag indicating if the buffer manager is used to allocate the stream buffers
diff --git a/services/camera/libcameraservice/device3/Camera3StreamSplitter.cpp b/services/camera/libcameraservice/device3/Camera3StreamSplitter.cpp index 84c2ec7..3089181 100644 --- a/services/camera/libcameraservice/device3/Camera3StreamSplitter.cpp +++ b/services/camera/libcameraservice/device3/Camera3StreamSplitter.cpp
@@ -139,7 +139,9 @@ mOutputSlots.clear(); mConsumerBufferCount.clear(); - mConsumer->consumerDisconnect(); + if (mConsumer.get() != nullptr) { + mConsumer->consumerDisconnect(); + } if (mBuffers.size() > 0) { SP_LOGW("%zu buffers still being tracked", mBuffers.size());
diff --git a/services/camera/libcameraservice/tests/CameraProviderManagerTest.cpp b/services/camera/libcameraservice/tests/CameraProviderManagerTest.cpp index f47e5a5..78d737d 100644 --- a/services/camera/libcameraservice/tests/CameraProviderManagerTest.cpp +++ b/services/camera/libcameraservice/tests/CameraProviderManagerTest.cpp
@@ -205,6 +205,11 @@ return mTestCameraProvider; } + virtual hardware::hidl_vec<hardware::hidl_string> listServices() override { + hardware::hidl_vec<hardware::hidl_string> ret = {"test/0"}; + return ret; + } + }; struct TestStatusListener : public CameraProviderManager::StatusListener { @@ -231,37 +236,24 @@ vendorSection); serviceProxy.setProvider(provider); + int numProviders = static_cast<int>(serviceProxy.listServices().size()); + res = providerManager->initialize(statusListener, &serviceProxy); ASSERT_EQ(res, OK) << "Unable to initialize provider manager"; // Check that both "legacy" and "external" providers (really the same object) are called // once for all the init methods - EXPECT_EQ(provider->mCalledCounter[TestICameraProvider::SET_CALLBACK], 2) << + EXPECT_EQ(provider->mCalledCounter[TestICameraProvider::SET_CALLBACK], numProviders) << "Only one call to setCallback per provider expected during init"; - EXPECT_EQ(provider->mCalledCounter[TestICameraProvider::GET_VENDOR_TAGS], 2) << + EXPECT_EQ(provider->mCalledCounter[TestICameraProvider::GET_VENDOR_TAGS], numProviders) << "Only one call to getVendorTags per provider expected during init"; - EXPECT_EQ(provider->mCalledCounter[TestICameraProvider::IS_SET_TORCH_MODE_SUPPORTED], 2) << + EXPECT_EQ(provider->mCalledCounter[TestICameraProvider::IS_SET_TORCH_MODE_SUPPORTED], + numProviders) << "Only one call to isSetTorchModeSupported per provider expected during init"; - EXPECT_EQ(provider->mCalledCounter[TestICameraProvider::GET_CAMERA_ID_LIST], 2) << + EXPECT_EQ(provider->mCalledCounter[TestICameraProvider::GET_CAMERA_ID_LIST], numProviders) << "Only one call to getCameraIdList per provider expected during init"; - EXPECT_EQ(provider->mCalledCounter[TestICameraProvider::NOTIFY_DEVICE_STATE], 2) << + EXPECT_EQ(provider->mCalledCounter[TestICameraProvider::NOTIFY_DEVICE_STATE], numProviders) << "Only one call to notifyDeviceState per provider expected during init"; - std::string legacyInstanceName = "legacy/0"; - std::string externalInstanceName = "external/0"; - bool gotLegacy = false; - bool gotExternal = false; - EXPECT_EQ(2u, serviceProxy.mLastRequestedServiceNames.size()) << - "Only two service queries expected to be seen by hardware service manager"; - - for (auto& serviceName : serviceProxy.mLastRequestedServiceNames) { - if (serviceName == legacyInstanceName) gotLegacy = true; - if (serviceName == externalInstanceName) gotExternal = true; - } - ASSERT_TRUE(gotLegacy) << - "Legacy instance not requested from service manager"; - ASSERT_TRUE(gotExternal) << - "External instance not requested from service manager"; - hardware::hidl_string testProviderFqInterfaceName = "android.hardware.camera.provider@2.4::ICameraProvider"; hardware::hidl_string testProviderInstanceName = "test/0";
diff --git a/services/mediacodec/main_codecservice.cpp b/services/mediacodec/main_codecservice.cpp index f668c33..6a82b1b 100644 --- a/services/mediacodec/main_codecservice.cpp +++ b/services/mediacodec/main_codecservice.cpp
@@ -21,6 +21,7 @@ #include "minijail.h" #include <binder/ProcessState.h> +#include <cutils/properties.h> #include <hidl/HidlTransportSupport.h> #include <media/stagefright/omx/1.0/Omx.h> #include <media/stagefright/omx/1.0/OmxStore.h> @@ -57,7 +58,8 @@ } else { LOG(INFO) << "IOmx HAL service created."; } - sp<IOmxStore> omxStore = new implementation::OmxStore(omx); + sp<IOmxStore> omxStore = new implementation::OmxStore( + property_get_int64("vendor.media.omx", 1) ? omx : nullptr); if (omxStore == nullptr) { LOG(ERROR) << "Cannot create IOmxStore HAL service."; } else if (omxStore->registerAsService() != OK) {
diff --git a/services/mediacodec/registrant/Android.bp b/services/mediacodec/registrant/Android.bp index 17c2e02..fa5bc4a 100644 --- a/services/mediacodec/registrant/Android.bp +++ b/services/mediacodec/registrant/Android.bp
@@ -42,7 +42,8 @@ "libcodec2_soft_opusenc", "libcodec2_soft_vp8dec", "libcodec2_soft_vp9dec", - "libcodec2_soft_av1dec", + // "libcodec2_soft_av1dec_aom", // replaced by the gav1 implementation + "libcodec2_soft_av1dec_gav1", "libcodec2_soft_vp8enc", "libcodec2_soft_vp9enc", "libcodec2_soft_rawdec",
diff --git a/services/mediaresourcemanager/ResourceManagerService.cpp b/services/mediaresourcemanager/ResourceManagerService.cpp index 28bfd3f..bdcd5e4 100644 --- a/services/mediaresourcemanager/ResourceManagerService.cpp +++ b/services/mediaresourcemanager/ResourceManagerService.cpp
@@ -21,8 +21,11 @@ #include <binder/IMediaResourceMonitor.h> #include <binder/IServiceManager.h> +#include <cutils/sched_policy.h> #include <dirent.h> #include <media/stagefright/ProcessInfo.h> +#include <mediautils/BatteryNotifier.h> +#include <mediautils/SchedulingPolicyService.h> #include <string.h> #include <sys/types.h> #include <sys/stat.h> @@ -31,8 +34,7 @@ #include "ResourceManagerService.h" #include "ServiceLog.h" -#include "mediautils/SchedulingPolicyService.h" -#include <cutils/sched_policy.h> + namespace android { namespace { @@ -69,9 +71,9 @@ return itemsStr; } -static bool hasResourceType(MediaResource::Type type, const Vector<MediaResource>& resources) { - for (size_t i = 0; i < resources.size(); ++i) { - if (resources[i].mType == type) { +static bool hasResourceType(MediaResource::Type type, const ResourceList& resources) { + for (auto it = resources.begin(); it != resources.end(); it++) { + if (it->second.mType == type) { return true; } } @@ -101,20 +103,22 @@ } static ResourceInfo& getResourceInfoForEdit( + uid_t uid, int64_t clientId, const sp<IResourceManagerClient>& client, ResourceInfos& infos) { - for (size_t i = 0; i < infos.size(); ++i) { - if (infos[i].clientId == clientId) { - return infos.editItemAt(i); - } + ssize_t index = infos.indexOfKey(clientId); + + if (index < 0) { + ResourceInfo info; + info.uid = uid; + info.clientId = clientId; + info.client = client; + + index = infos.add(clientId, info); } - ResourceInfo info; - info.clientId = clientId; - info.client = client; - info.cpuBoost = false; - infos.push_back(info); - return infos.editItemAt(infos.size() - 1); + + return infos.editValueAt(index); } static void notifyResourceGranted(int pid, const Vector<MediaResource> &resources) { @@ -181,10 +185,10 @@ snprintf(buffer, SIZE, " Name: %s\n", infos[j].client->getName().string()); result.append(buffer); - Vector<MediaResource> resources = infos[j].resources; + const ResourceList &resources = infos[j].resources; result.append(" Resources:\n"); - for (size_t k = 0; k < resources.size(); ++k) { - snprintf(buffer, SIZE, " %s\n", resources[k].toString().string()); + for (auto it = resources.begin(); it != resources.end(); it++) { + snprintf(buffer, SIZE, " %s\n", it->second.toString().string()); result.append(buffer); } } @@ -196,15 +200,45 @@ return OK; } -ResourceManagerService::ResourceManagerService() - : ResourceManagerService(new ProcessInfo()) {} +struct SystemCallbackImpl : + public ResourceManagerService::SystemCallbackInterface { + SystemCallbackImpl() {} -ResourceManagerService::ResourceManagerService(sp<ProcessInfoInterface> processInfo) + virtual void noteStartVideo(int uid) override { + BatteryNotifier::getInstance().noteStartVideo(uid); + } + virtual void noteStopVideo(int uid) override { + BatteryNotifier::getInstance().noteStopVideo(uid); + } + virtual void noteResetVideo() override { + BatteryNotifier::getInstance().noteResetVideo(); + } + virtual bool requestCpusetBoost( + bool enable, const sp<IInterface> &client) override { + return android::requestCpusetBoost(enable, client); + } + +protected: + virtual ~SystemCallbackImpl() {} + +private: + DISALLOW_EVIL_CONSTRUCTORS(SystemCallbackImpl); +}; + +ResourceManagerService::ResourceManagerService() + : ResourceManagerService(new ProcessInfo(), new SystemCallbackImpl()) {} + +ResourceManagerService::ResourceManagerService( + const sp<ProcessInfoInterface> &processInfo, + const sp<SystemCallbackInterface> &systemResource) : mProcessInfo(processInfo), + mSystemCB(systemResource), mServiceLog(new ServiceLog()), mSupportsMultipleSecureCodecs(true), mSupportsSecureWithNonSecureCodec(true), - mCpuBoostCount(0) {} + mCpuBoostCount(0) { + mSystemCB->noteResetVideo(); +} ResourceManagerService::~ResourceManagerService() {} @@ -224,8 +258,41 @@ } } +void ResourceManagerService::onFirstAdded( + const MediaResource& resource, const ResourceInfo& clientInfo) { + // first time added + if (resource.mType == MediaResource::kCpuBoost + && resource.mSubType == MediaResource::kUnspecifiedSubType) { + // Request it on every new instance of kCpuBoost, as the media.codec + // could have died, if we only do it the first time subsequent instances + // never gets the boost. + if (mSystemCB->requestCpusetBoost(true, this) != OK) { + ALOGW("couldn't request cpuset boost"); + } + mCpuBoostCount++; + } else if (resource.mType == MediaResource::kBattery + && resource.mSubType == MediaResource::kVideoCodec) { + mSystemCB->noteStartVideo(clientInfo.uid); + } +} + +void ResourceManagerService::onLastRemoved( + const MediaResource& resource, const ResourceInfo& clientInfo) { + if (resource.mType == MediaResource::kCpuBoost + && resource.mSubType == MediaResource::kUnspecifiedSubType + && mCpuBoostCount > 0) { + if (--mCpuBoostCount == 0) { + mSystemCB->requestCpusetBoost(false, this); + } + } else if (resource.mType == MediaResource::kBattery + && resource.mSubType == MediaResource::kVideoCodec) { + mSystemCB->noteStopVideo(clientInfo.uid); + } +} + void ResourceManagerService::addResource( int pid, + int uid, int64_t clientId, const sp<IResourceManagerClient> client, const Vector<MediaResource> &resources) { @@ -239,20 +306,15 @@ return; } ResourceInfos& infos = getResourceInfosForEdit(pid, mMap); - ResourceInfo& info = getResourceInfoForEdit(clientId, client, infos); - // TODO: do the merge instead of append. - info.resources.appendVector(resources); + ResourceInfo& info = getResourceInfoForEdit(uid, clientId, client, infos); for (size_t i = 0; i < resources.size(); ++i) { - if (resources[i].mType == MediaResource::kCpuBoost && !info.cpuBoost) { - info.cpuBoost = true; - // Request it on every new instance of kCpuBoost, as the media.codec - // could have died, if we only do it the first time subsequent instances - // never gets the boost. - if (requestCpusetBoost(true, this) != OK) { - ALOGW("couldn't request cpuset boost"); - } - mCpuBoostCount++; + const auto resType = std::make_pair(resources[i].mType, resources[i].mSubType); + if (info.resources.find(resType) == info.resources.end()) { + onFirstAdded(resources[i], info); + info.resources[resType] = resources[i]; + } else { + info.resources[resType].mValue += resources[i].mValue; } } if (info.deathNotifier == nullptr) { @@ -262,7 +324,48 @@ notifyResourceGranted(pid, resources); } -void ResourceManagerService::removeResource(int pid, int64_t clientId) { +void ResourceManagerService::removeResource(int pid, int64_t clientId, + const Vector<MediaResource> &resources) { + String8 log = String8::format("removeResource(pid %d, clientId %lld, resources %s)", + pid, (long long) clientId, getString(resources).string()); + mServiceLog->add(log); + + Mutex::Autolock lock(mLock); + if (!mProcessInfo->isValidPid(pid)) { + ALOGE("Rejected removeResource call with invalid pid."); + return; + } + ssize_t index = mMap.indexOfKey(pid); + if (index < 0) { + ALOGV("removeResource: didn't find pid %d for clientId %lld", pid, (long long) clientId); + return; + } + ResourceInfos &infos = mMap.editValueAt(index); + + index = infos.indexOfKey(clientId); + if (index < 0) { + ALOGV("removeResource: didn't find clientId %lld", (long long) clientId); + return; + } + + ResourceInfo &info = infos.editValueAt(index); + + for (size_t i = 0; i < resources.size(); ++i) { + const auto resType = std::make_pair(resources[i].mType, resources[i].mSubType); + // ignore if we don't have it + if (info.resources.find(resType) != info.resources.end()) { + MediaResource &resource = info.resources[resType]; + if (resource.mValue > resources[i].mValue) { + resource.mValue -= resources[i].mValue; + } else { + onLastRemoved(resources[i], info); + info.resources.erase(resType); + } + } + } +} + +void ResourceManagerService::removeClient(int pid, int64_t clientId) { removeResource(pid, clientId, true); } @@ -282,24 +385,22 @@ ALOGV("removeResource: didn't find pid %d for clientId %lld", pid, (long long) clientId); return; } - bool found = false; ResourceInfos &infos = mMap.editValueAt(index); - for (size_t j = 0; j < infos.size(); ++j) { - if (infos[j].clientId == clientId) { - if (infos[j].cpuBoost && mCpuBoostCount > 0) { - if (--mCpuBoostCount == 0) { - requestCpusetBoost(false, this); - } - } - IInterface::asBinder(infos[j].client)->unlinkToDeath(infos[j].deathNotifier); - j = infos.removeAt(j); - found = true; - break; - } + + index = infos.indexOfKey(clientId); + if (index < 0) { + ALOGV("removeResource: didn't find clientId %lld", (long long) clientId); + return; } - if (!found) { - ALOGV("didn't find client"); + + const ResourceInfo &info = infos[index]; + for (auto it = info.resources.begin(); it != info.resources.end(); it++) { + onLastRemoved(it->second, info); } + + IInterface::asBinder(info.client)->unlinkToDeath(info.deathNotifier); + + infos.removeItemsAt(index); } void ResourceManagerService::getClientForResource_l( @@ -410,7 +511,7 @@ ResourceInfos &infos = mMap.editValueAt(i); for (size_t j = 0; j < infos.size();) { if (infos[j].client == failedClient) { - j = infos.removeAt(j); + j = infos.removeItemsAt(j); found = true; } else { ++j; @@ -538,11 +639,12 @@ uint64_t largestValue = 0; const ResourceInfos &infos = mMap.valueAt(index); for (size_t i = 0; i < infos.size(); ++i) { - Vector<MediaResource> resources = infos[i].resources; - for (size_t j = 0; j < resources.size(); ++j) { - if (resources[j].mType == type) { - if (resources[j].mValue > largestValue) { - largestValue = resources[j].mValue; + const ResourceList &resources = infos[i].resources; + for (auto it = resources.begin(); it != resources.end(); it++) { + const MediaResource &resource = it->second; + if (resource.mType == type) { + if (resource.mValue > largestValue) { + largestValue = resource.mValue; clientTemp = infos[i].client; } }
diff --git a/services/mediaresourcemanager/ResourceManagerService.h b/services/mediaresourcemanager/ResourceManagerService.h index 82d2a0b..f086dc3 100644 --- a/services/mediaresourcemanager/ResourceManagerService.h +++ b/services/mediaresourcemanager/ResourceManagerService.h
@@ -33,15 +33,17 @@ class ServiceLog; struct ProcessInfoInterface; +typedef std::map<std::pair<MediaResource::Type, MediaResource::SubType>, MediaResource> ResourceList; struct ResourceInfo { int64_t clientId; + uid_t uid; sp<IResourceManagerClient> client; sp<IBinder::DeathRecipient> deathNotifier; - Vector<MediaResource> resources; - bool cpuBoost; + ResourceList resources; }; -typedef Vector<ResourceInfo> ResourceInfos; +// TODO: convert these to std::map +typedef KeyedVector<int64_t, ResourceInfo> ResourceInfos; typedef KeyedVector<int, ResourceInfos> PidResourceInfosMap; class ResourceManagerService @@ -49,23 +51,37 @@ public BnResourceManagerService { public: + struct SystemCallbackInterface : public RefBase { + virtual void noteStartVideo(int uid) = 0; + virtual void noteStopVideo(int uid) = 0; + virtual void noteResetVideo() = 0; + virtual bool requestCpusetBoost( + bool enable, const sp<IInterface> &client) = 0; + }; + static char const *getServiceName() { return "media.resource_manager"; } virtual status_t dump(int fd, const Vector<String16>& args); ResourceManagerService(); - explicit ResourceManagerService(sp<ProcessInfoInterface> processInfo); + explicit ResourceManagerService( + const sp<ProcessInfoInterface> &processInfo, + const sp<SystemCallbackInterface> &systemResource); // IResourceManagerService interface virtual void config(const Vector<MediaResourcePolicy> &policies); virtual void addResource( int pid, + int uid, int64_t clientId, const sp<IResourceManagerClient> client, const Vector<MediaResource> &resources); - virtual void removeResource(int pid, int64_t clientId); + virtual void removeResource(int pid, int64_t clientId, + const Vector<MediaResource> &resources); + + virtual void removeClient(int pid, int64_t clientId); // Tries to reclaim resource from processes with lower priority than the calling process // according to the requested resources. @@ -107,8 +123,12 @@ void getClientForResource_l( int callingPid, const MediaResource *res, Vector<sp<IResourceManagerClient>> *clients); + void onFirstAdded(const MediaResource& res, const ResourceInfo& clientInfo); + void onLastRemoved(const MediaResource& res, const ResourceInfo& clientInfo); + mutable Mutex mLock; sp<ProcessInfoInterface> mProcessInfo; + sp<SystemCallbackInterface> mSystemCB; sp<ServiceLog> mServiceLog; PidResourceInfosMap mMap; bool mSupportsMultipleSecureCodecs;
diff --git a/services/mediaresourcemanager/TEST_MAPPING b/services/mediaresourcemanager/TEST_MAPPING new file mode 100644 index 0000000..418b159 --- /dev/null +++ b/services/mediaresourcemanager/TEST_MAPPING
@@ -0,0 +1,10 @@ +{ + "presubmit": [ + { + "name": "ResourceManagerService_test" + }, + { + "name": "ServiceLog_test" + } + ] +}
diff --git a/services/mediaresourcemanager/test/Android.bp b/services/mediaresourcemanager/test/Android.bp index 70e8833..543c87c 100644 --- a/services/mediaresourcemanager/test/Android.bp +++ b/services/mediaresourcemanager/test/Android.bp
@@ -2,6 +2,7 @@ cc_test { name: "ResourceManagerService_test", srcs: ["ResourceManagerService_test.cpp"], + test_suites: ["device-tests"], shared_libs: [ "libbinder", "liblog", @@ -23,6 +24,7 @@ cc_test { name: "ServiceLog_test", srcs: ["ServiceLog_test.cpp"], + test_suites: ["device-tests"], shared_libs: [ "liblog", "libmedia",
diff --git a/services/mediaresourcemanager/test/ResourceManagerService_test.cpp b/services/mediaresourcemanager/test/ResourceManagerService_test.cpp index ed0b0ef..ae97ec8 100644 --- a/services/mediaresourcemanager/test/ResourceManagerService_test.cpp +++ b/services/mediaresourcemanager/test/ResourceManagerService_test.cpp
@@ -52,13 +52,69 @@ DISALLOW_EVIL_CONSTRUCTORS(TestProcessInfo); }; +struct TestSystemCallback : + public ResourceManagerService::SystemCallbackInterface { + TestSystemCallback() : + mLastEvent({EventType::INVALID, 0}), mEventCount(0) {} + + enum EventType { + INVALID = -1, + VIDEO_ON = 0, + VIDEO_OFF = 1, + VIDEO_RESET = 2, + CPUSET_ENABLE = 3, + CPUSET_DISABLE = 4, + }; + + struct EventEntry { + EventType type; + int arg; + }; + + virtual void noteStartVideo(int uid) override { + mLastEvent = {EventType::VIDEO_ON, uid}; + mEventCount++; + } + + virtual void noteStopVideo(int uid) override { + mLastEvent = {EventType::VIDEO_OFF, uid}; + mEventCount++; + } + + virtual void noteResetVideo() override { + mLastEvent = {EventType::VIDEO_RESET, 0}; + mEventCount++; + } + + virtual bool requestCpusetBoost( + bool enable, const sp<IInterface> &/*client*/) override { + mLastEvent = {enable ? EventType::CPUSET_ENABLE : EventType::CPUSET_DISABLE, 0}; + mEventCount++; + return true; + } + + size_t eventCount() { return mEventCount; } + EventType lastEventType() { return mLastEvent.type; } + EventEntry lastEvent() { return mLastEvent; } + +protected: + virtual ~TestSystemCallback() {} + +private: + EventEntry mLastEvent; + size_t mEventCount; + + DISALLOW_EVIL_CONSTRUCTORS(TestSystemCallback); +}; + + struct TestClient : public BnResourceManagerClient { TestClient(int pid, sp<ResourceManagerService> service) : mReclaimed(false), mPid(pid), mService(service) {} virtual bool reclaimResource() { sp<IResourceManagerClient> client(this); - mService->removeResource(mPid, (int64_t) client.get()); + mService->removeClient(mPid, (int64_t) client.get()); mReclaimed = true; return true; } @@ -86,16 +142,26 @@ }; static const int kTestPid1 = 30; +static const int kTestUid1 = 1010; + static const int kTestPid2 = 20; +static const int kTestUid2 = 1011; static const int kLowPriorityPid = 40; static const int kMidPriorityPid = 25; static const int kHighPriorityPid = 10; +using EventType = TestSystemCallback::EventType; +using EventEntry = TestSystemCallback::EventEntry; +bool operator== (const EventEntry& lhs, const EventEntry& rhs) { + return lhs.type == rhs.type && lhs.arg == rhs.arg; +} + class ResourceManagerServiceTest : public ::testing::Test { public: ResourceManagerServiceTest() - : mService(new ResourceManagerService(new TestProcessInfo)), + : mSystemCB(new TestSystemCallback()), + mService(new ResourceManagerService(new TestProcessInfo, mSystemCB)), mTestClient1(new TestClient(kTestPid1, mService)), mTestClient2(new TestClient(kTestPid2, mService)), mTestClient3(new TestClient(kTestPid2, mService)) { @@ -103,20 +169,21 @@ protected: static bool isEqualResources(const Vector<MediaResource> &resources1, - const Vector<MediaResource> &resources2) { - if (resources1.size() != resources2.size()) { - return false; - } + const ResourceList &resources2) { + // convert resource1 to ResourceList + ResourceList r1; for (size_t i = 0; i < resources1.size(); ++i) { - if (resources1[i] != resources2[i]) { - return false; - } + const auto resType = std::make_pair(resources1[i].mType, resources1[i].mSubType); + r1[resType] = resources1[i]; } - return true; + return r1 == resources2; } - static void expectEqResourceInfo(const ResourceInfo &info, sp<IResourceManagerClient> client, + static void expectEqResourceInfo(const ResourceInfo &info, + int uid, + sp<IResourceManagerClient> client, const Vector<MediaResource> &resources) { + EXPECT_EQ(uid, info.uid); EXPECT_EQ(client, info.client); EXPECT_TRUE(isEqualResources(resources, info.resources)); } @@ -153,24 +220,24 @@ // kTestPid1 mTestClient1 Vector<MediaResource> resources1; resources1.push_back(MediaResource(MediaResource::kSecureCodec, 1)); - mService->addResource(kTestPid1, getId(mTestClient1), mTestClient1, resources1); + mService->addResource(kTestPid1, kTestUid1, getId(mTestClient1), mTestClient1, resources1); resources1.push_back(MediaResource(MediaResource::kGraphicMemory, 200)); Vector<MediaResource> resources11; resources11.push_back(MediaResource(MediaResource::kGraphicMemory, 200)); - mService->addResource(kTestPid1, getId(mTestClient1), mTestClient1, resources11); + mService->addResource(kTestPid1, kTestUid1, getId(mTestClient1), mTestClient1, resources11); // kTestPid2 mTestClient2 Vector<MediaResource> resources2; resources2.push_back(MediaResource(MediaResource::kNonSecureCodec, 1)); resources2.push_back(MediaResource(MediaResource::kGraphicMemory, 300)); - mService->addResource(kTestPid2, getId(mTestClient2), mTestClient2, resources2); + mService->addResource(kTestPid2, kTestUid2, getId(mTestClient2), mTestClient2, resources2); // kTestPid2 mTestClient3 Vector<MediaResource> resources3; - mService->addResource(kTestPid2, getId(mTestClient3), mTestClient3, resources3); + mService->addResource(kTestPid2, kTestUid2, getId(mTestClient3), mTestClient3, resources3); resources3.push_back(MediaResource(MediaResource::kSecureCodec, 1)); resources3.push_back(MediaResource(MediaResource::kGraphicMemory, 100)); - mService->addResource(kTestPid2, getId(mTestClient3), mTestClient3, resources3); + mService->addResource(kTestPid2, kTestUid2, getId(mTestClient3), mTestClient3, resources3); const PidResourceInfosMap &map = mService->mMap; EXPECT_EQ(2u, map.size()); @@ -178,14 +245,14 @@ ASSERT_GE(index1, 0); const ResourceInfos &infos1 = map[index1]; EXPECT_EQ(1u, infos1.size()); - expectEqResourceInfo(infos1[0], mTestClient1, resources1); + expectEqResourceInfo(infos1.valueFor(getId(mTestClient1)), kTestUid1, mTestClient1, resources1); ssize_t index2 = map.indexOfKey(kTestPid2); ASSERT_GE(index2, 0); const ResourceInfos &infos2 = map[index2]; EXPECT_EQ(2u, infos2.size()); - expectEqResourceInfo(infos2[0], mTestClient2, resources2); - expectEqResourceInfo(infos2[1], mTestClient3, resources3); + expectEqResourceInfo(infos2.valueFor(getId(mTestClient2)), kTestUid2, mTestClient2, resources2); + expectEqResourceInfo(infos2.valueFor(getId(mTestClient3)), kTestUid2, mTestClient3, resources3); } void testConfig() { @@ -219,10 +286,84 @@ EXPECT_TRUE(mService->mSupportsSecureWithNonSecureCodec); } + void testCombineResource() { + // kTestPid1 mTestClient1 + Vector<MediaResource> resources1; + resources1.push_back(MediaResource(MediaResource::kSecureCodec, 1)); + mService->addResource(kTestPid1, kTestUid1, getId(mTestClient1), mTestClient1, resources1); + + Vector<MediaResource> resources11; + resources11.push_back(MediaResource(MediaResource::kGraphicMemory, 200)); + mService->addResource(kTestPid1, kTestUid1, getId(mTestClient1), mTestClient1, resources11); + + const PidResourceInfosMap &map = mService->mMap; + EXPECT_EQ(1u, map.size()); + ssize_t index1 = map.indexOfKey(kTestPid1); + ASSERT_GE(index1, 0); + const ResourceInfos &infos1 = map[index1]; + EXPECT_EQ(1u, infos1.size()); + + // test adding existing types to combine values + resources1.push_back(MediaResource(MediaResource::kGraphicMemory, 100)); + mService->addResource(kTestPid1, kTestUid1, getId(mTestClient1), mTestClient1, resources1); + + Vector<MediaResource> expected; + expected.push_back(MediaResource(MediaResource::kSecureCodec, 2)); + expected.push_back(MediaResource(MediaResource::kGraphicMemory, 300)); + expectEqResourceInfo(infos1.valueFor(getId(mTestClient1)), kTestUid1, mTestClient1, expected); + + // test adding new types (including types that differs only in subType) + resources11.push_back(MediaResource(MediaResource::kNonSecureCodec, 1)); + resources11.push_back(MediaResource(MediaResource::kSecureCodec, MediaResource::kVideoCodec, 1)); + mService->addResource(kTestPid1, kTestUid1, getId(mTestClient1), mTestClient1, resources11); + + expected.clear(); + expected.push_back(MediaResource(MediaResource::kSecureCodec, 2)); + expected.push_back(MediaResource(MediaResource::kNonSecureCodec, 1)); + expected.push_back(MediaResource(MediaResource::kSecureCodec, MediaResource::kVideoCodec, 1)); + expected.push_back(MediaResource(MediaResource::kGraphicMemory, 500)); + expectEqResourceInfo(infos1.valueFor(getId(mTestClient1)), kTestUid1, mTestClient1, expected); + } + void testRemoveResource() { + // kTestPid1 mTestClient1 + Vector<MediaResource> resources1; + resources1.push_back(MediaResource(MediaResource::kSecureCodec, 1)); + mService->addResource(kTestPid1, kTestUid1, getId(mTestClient1), mTestClient1, resources1); + + Vector<MediaResource> resources11; + resources11.push_back(MediaResource(MediaResource::kGraphicMemory, 200)); + mService->addResource(kTestPid1, kTestUid1, getId(mTestClient1), mTestClient1, resources11); + + const PidResourceInfosMap &map = mService->mMap; + EXPECT_EQ(1u, map.size()); + ssize_t index1 = map.indexOfKey(kTestPid1); + ASSERT_GE(index1, 0); + const ResourceInfos &infos1 = map[index1]; + EXPECT_EQ(1u, infos1.size()); + + // test partial removal + resources11.editItemAt(0).mValue = 100; + mService->removeResource(kTestPid1, getId(mTestClient1), resources11); + + Vector<MediaResource> expected; + expected.push_back(MediaResource(MediaResource::kSecureCodec, 1)); + expected.push_back(MediaResource(MediaResource::kGraphicMemory, 100)); + expectEqResourceInfo(infos1.valueFor(getId(mTestClient1)), kTestUid1, mTestClient1, expected); + + // test complete removal with overshoot value + resources11.editItemAt(0).mValue = 1000; + mService->removeResource(kTestPid1, getId(mTestClient1), resources11); + + expected.clear(); + expected.push_back(MediaResource(MediaResource::kSecureCodec, 1)); + expectEqResourceInfo(infos1.valueFor(getId(mTestClient1)), kTestUid1, mTestClient1, expected); + } + + void testRemoveClient() { addResource(); - mService->removeResource(kTestPid2, getId(mTestClient2)); + mService->removeClient(kTestPid2, getId(mTestClient2)); const PidResourceInfosMap &map = mService->mMap; EXPECT_EQ(2u, map.size()); @@ -231,6 +372,7 @@ EXPECT_EQ(1u, infos1.size()); EXPECT_EQ(1u, infos2.size()); // mTestClient2 has been removed. + // (OK to use infos2[0] as there is only 1 entry) EXPECT_EQ(mTestClient3, infos2[0].client); } @@ -246,6 +388,7 @@ EXPECT_TRUE(mService->getAllClients_l(kHighPriorityPid, type, &clients)); EXPECT_EQ(2u, clients.size()); + // (OK to require ordering in clients[], as the pid map is sorted) EXPECT_EQ(mTestClient3, clients[0]); EXPECT_EQ(mTestClient1, clients[1]); } @@ -438,7 +581,7 @@ verifyClients(true /* c1 */, false /* c2 */, false /* c3 */); // clean up client 3 which still left - mService->removeResource(kTestPid2, getId(mTestClient3)); + mService->removeClient(kTestPid2, getId(mTestClient3)); } } @@ -498,6 +641,84 @@ EXPECT_TRUE(mService->isCallingPriorityHigher_l(99, 100)); } + void testBatteryStats() { + // reset should always be called when ResourceManagerService is created (restarted) + EXPECT_EQ(1u, mSystemCB->eventCount()); + EXPECT_EQ(EventType::VIDEO_RESET, mSystemCB->lastEventType()); + + // new client request should cause VIDEO_ON + Vector<MediaResource> resources1; + resources1.push_back(MediaResource(MediaResource::kBattery, MediaResource::kVideoCodec, 1)); + mService->addResource(kTestPid1, kTestUid1, getId(mTestClient1), mTestClient1, resources1); + EXPECT_EQ(2u, mSystemCB->eventCount()); + EXPECT_EQ(EventEntry({EventType::VIDEO_ON, kTestUid1}), mSystemCB->lastEvent()); + + // each client should only cause 1 VIDEO_ON + mService->addResource(kTestPid1, kTestUid1, getId(mTestClient1), mTestClient1, resources1); + EXPECT_EQ(2u, mSystemCB->eventCount()); + + // new client request should cause VIDEO_ON + Vector<MediaResource> resources2; + resources2.push_back(MediaResource(MediaResource::kBattery, MediaResource::kVideoCodec, 2)); + mService->addResource(kTestPid2, kTestUid2, getId(mTestClient2), mTestClient2, resources2); + EXPECT_EQ(3u, mSystemCB->eventCount()); + EXPECT_EQ(EventEntry({EventType::VIDEO_ON, kTestUid2}), mSystemCB->lastEvent()); + + // partially remove mTestClient1's request, shouldn't be any VIDEO_OFF + mService->removeResource(kTestPid1, getId(mTestClient1), resources1); + EXPECT_EQ(3u, mSystemCB->eventCount()); + + // remove mTestClient1's request, should be VIDEO_OFF for kTestUid1 + // (use resource2 to test removing more instances than previously requested) + mService->removeResource(kTestPid1, getId(mTestClient1), resources2); + EXPECT_EQ(4u, mSystemCB->eventCount()); + EXPECT_EQ(EventEntry({EventType::VIDEO_OFF, kTestUid1}), mSystemCB->lastEvent()); + + // remove mTestClient2, should be VIDEO_OFF for kTestUid2 + mService->removeClient(kTestPid2, getId(mTestClient2)); + EXPECT_EQ(5u, mSystemCB->eventCount()); + EXPECT_EQ(EventEntry({EventType::VIDEO_OFF, kTestUid2}), mSystemCB->lastEvent()); + } + + void testCpusetBoost() { + // reset should always be called when ResourceManagerService is created (restarted) + EXPECT_EQ(1u, mSystemCB->eventCount()); + EXPECT_EQ(EventType::VIDEO_RESET, mSystemCB->lastEventType()); + + // new client request should cause CPUSET_ENABLE + Vector<MediaResource> resources1; + resources1.push_back(MediaResource(MediaResource::kCpuBoost, 1)); + mService->addResource(kTestPid1, kTestUid1, getId(mTestClient1), mTestClient1, resources1); + EXPECT_EQ(2u, mSystemCB->eventCount()); + EXPECT_EQ(EventType::CPUSET_ENABLE, mSystemCB->lastEventType()); + + // each client should only cause 1 CPUSET_ENABLE + mService->addResource(kTestPid1, kTestUid1, getId(mTestClient1), mTestClient1, resources1); + EXPECT_EQ(2u, mSystemCB->eventCount()); + + // new client request should cause CPUSET_ENABLE + Vector<MediaResource> resources2; + resources2.push_back(MediaResource(MediaResource::kCpuBoost, 2)); + mService->addResource(kTestPid2, kTestUid2, getId(mTestClient2), mTestClient2, resources2); + EXPECT_EQ(3u, mSystemCB->eventCount()); + EXPECT_EQ(EventType::CPUSET_ENABLE, mSystemCB->lastEventType()); + + // remove mTestClient2 should not cause CPUSET_DISABLE, mTestClient1 still active + mService->removeClient(kTestPid2, getId(mTestClient2)); + EXPECT_EQ(3u, mSystemCB->eventCount()); + + // remove 1 cpuboost from mTestClient1, should not be CPUSET_DISABLE (still 1 left) + mService->removeResource(kTestPid1, getId(mTestClient1), resources1); + EXPECT_EQ(3u, mSystemCB->eventCount()); + + // remove 2 cpuboost from mTestClient1, should be CPUSET_DISABLE + // (use resource2 to test removing more than previously requested) + mService->removeResource(kTestPid1, getId(mTestClient1), resources2); + EXPECT_EQ(4u, mSystemCB->eventCount()); + EXPECT_EQ(EventType::CPUSET_DISABLE, mSystemCB->lastEventType()); + } + + sp<TestSystemCallback> mSystemCB; sp<ResourceManagerService> mService; sp<IResourceManagerClient> mTestClient1; sp<IResourceManagerClient> mTestClient2; @@ -512,10 +733,18 @@ addResource(); } +TEST_F(ResourceManagerServiceTest, combineResource) { + testCombineResource(); +} + TEST_F(ResourceManagerServiceTest, removeResource) { testRemoveResource(); } +TEST_F(ResourceManagerServiceTest, removeClient) { + testRemoveClient(); +} + TEST_F(ResourceManagerServiceTest, reclaimResource) { testReclaimResourceSecure(); testReclaimResourceNonSecure(); @@ -541,4 +770,12 @@ testIsCallingPriorityHigher(); } +TEST_F(ResourceManagerServiceTest, testBatteryStats) { + testBatteryStats(); +} + +TEST_F(ResourceManagerServiceTest, testCpusetBoost) { + testCpusetBoost(); +} + } // namespace android