| /* |
| * Copyright (C) 2012 The Android Open Source Project |
| * |
| * Licensed under the Apache License, Version 2.0 (the "License"); |
| * you may not use this file except in compliance with the License. |
| * You may obtain a copy of the License at |
| * |
| * http://www.apache.org/licenses/LICENSE-2.0 |
| * |
| * Unless required by applicable law or agreed to in writing, software |
| * distributed under the License is distributed on an "AS IS" BASIS, |
| * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| * See the License for the specific language governing permissions and |
| * limitations under the License. |
| */ |
| |
| #define LOG_TAG "Camera2-FrameProcessor" |
| #define ATRACE_TAG ATRACE_TAG_CAMERA |
| //#define LOG_NDEBUG 0 |
| |
| #include <utils/Log.h> |
| #include <utils/Trace.h> |
| |
| #include "common/CameraDeviceBase.h" |
| #include "api1/Camera2Client.h" |
| #include "api1/client2/FrameProcessor.h" |
| |
| namespace android { |
| namespace camera2 { |
| |
| FrameProcessor::FrameProcessor(wp<CameraDeviceBase> device, |
| sp<Camera2Client> client) : |
| FrameProcessorBase(device), |
| mClient(client), |
| mLastFrameNumberOfFaces(0), |
| mLast3AFrameNumber(-1) { |
| |
| sp<CameraDeviceBase> d = device.promote(); |
| mSynthesize3ANotify = !(d->willNotify3A()); |
| |
| { |
| SharedParameters::Lock l(client->getParameters()); |
| |
| if (client->getCameraDeviceVersion() >= CAMERA_DEVICE_API_VERSION_3_2) { |
| mUsePartialResult = (mNumPartialResults > 1); |
| } else { |
| mUsePartialResult = l.mParameters.quirks.partialResults; |
| } |
| |
| // Initialize starting 3A state |
| m3aState.afTriggerId = l.mParameters.afTriggerCounter; |
| m3aState.aeTriggerId = l.mParameters.precaptureTriggerCounter; |
| // Check if lens is fixed-focus |
| if (l.mParameters.focusMode == Parameters::FOCUS_MODE_FIXED) { |
| m3aState.afMode = ANDROID_CONTROL_AF_MODE_OFF; |
| } |
| } |
| } |
| |
| FrameProcessor::~FrameProcessor() { |
| } |
| |
| bool FrameProcessor::processSingleFrame(CaptureResult &frame, |
| const sp<CameraDeviceBase> &device) { |
| |
| sp<Camera2Client> client = mClient.promote(); |
| if (!client.get()) { |
| return false; |
| } |
| |
| bool isPartialResult = false; |
| if (mUsePartialResult) { |
| if (client->getCameraDeviceVersion() >= CAMERA_DEVICE_API_VERSION_3_2) { |
| isPartialResult = frame.mResultExtras.partialResultCount < mNumPartialResults; |
| } else { |
| camera_metadata_entry_t entry; |
| entry = frame.mMetadata.find(ANDROID_QUIRKS_PARTIAL_RESULT); |
| if (entry.count > 0 && |
| entry.data.u8[0] == ANDROID_QUIRKS_PARTIAL_RESULT_PARTIAL) { |
| isPartialResult = true; |
| } |
| } |
| } |
| |
| if (!isPartialResult && processFaceDetect(frame.mMetadata, client) != OK) { |
| return false; |
| } |
| |
| if (mSynthesize3ANotify) { |
| process3aState(frame, client); |
| } |
| |
| return FrameProcessorBase::processSingleFrame(frame, device); |
| } |
| |
| status_t FrameProcessor::processFaceDetect(const CameraMetadata &frame, |
| const sp<Camera2Client> &client) { |
| status_t res = BAD_VALUE; |
| ATRACE_CALL(); |
| camera_metadata_ro_entry_t entry; |
| bool enableFaceDetect; |
| |
| { |
| SharedParameters::Lock l(client->getParameters()); |
| enableFaceDetect = l.mParameters.enableFaceDetect; |
| } |
| entry = frame.find(ANDROID_STATISTICS_FACE_DETECT_MODE); |
| |
| // TODO: This should be an error once implementations are compliant |
| if (entry.count == 0) { |
| return OK; |
| } |
| |
| uint8_t faceDetectMode = entry.data.u8[0]; |
| |
| camera_frame_metadata metadata; |
| Vector<camera_face_t> faces; |
| metadata.number_of_faces = 0; |
| |
| if (enableFaceDetect && |
| faceDetectMode != ANDROID_STATISTICS_FACE_DETECT_MODE_OFF) { |
| |
| SharedParameters::Lock l(client->getParameters()); |
| entry = frame.find(ANDROID_STATISTICS_FACE_RECTANGLES); |
| if (entry.count == 0) { |
| // No faces this frame |
| /* warning: locks SharedCameraCallbacks */ |
| callbackFaceDetection(client, metadata); |
| return OK; |
| } |
| metadata.number_of_faces = entry.count / 4; |
| if (metadata.number_of_faces > |
| l.mParameters.fastInfo.maxFaces) { |
| ALOGE("%s: Camera %d: More faces than expected! (Got %d, max %d)", |
| __FUNCTION__, client->getCameraId(), |
| metadata.number_of_faces, l.mParameters.fastInfo.maxFaces); |
| return res; |
| } |
| const int32_t *faceRects = entry.data.i32; |
| |
| entry = frame.find(ANDROID_STATISTICS_FACE_SCORES); |
| if (entry.count == 0) { |
| ALOGE("%s: Camera %d: Unable to read face scores", |
| __FUNCTION__, client->getCameraId()); |
| return res; |
| } |
| const uint8_t *faceScores = entry.data.u8; |
| |
| const int32_t *faceLandmarks = NULL; |
| const int32_t *faceIds = NULL; |
| |
| if (faceDetectMode == ANDROID_STATISTICS_FACE_DETECT_MODE_FULL) { |
| entry = frame.find(ANDROID_STATISTICS_FACE_LANDMARKS); |
| if (entry.count == 0) { |
| ALOGE("%s: Camera %d: Unable to read face landmarks", |
| __FUNCTION__, client->getCameraId()); |
| return res; |
| } |
| faceLandmarks = entry.data.i32; |
| |
| entry = frame.find(ANDROID_STATISTICS_FACE_IDS); |
| |
| if (entry.count == 0) { |
| ALOGE("%s: Camera %d: Unable to read face IDs", |
| __FUNCTION__, client->getCameraId()); |
| return res; |
| } |
| faceIds = entry.data.i32; |
| } |
| |
| entry = frame.find(ANDROID_SCALER_CROP_REGION); |
| if (entry.count < 4) { |
| ALOGE("%s: Camera %d: Unable to read crop region (count = %d)", |
| __FUNCTION__, client->getCameraId(), entry.count); |
| return res; |
| } |
| |
| Parameters::CropRegion scalerCrop = { |
| static_cast<float>(entry.data.i32[0]), |
| static_cast<float>(entry.data.i32[1]), |
| static_cast<float>(entry.data.i32[2]), |
| static_cast<float>(entry.data.i32[3])}; |
| |
| faces.setCapacity(metadata.number_of_faces); |
| |
| size_t maxFaces = metadata.number_of_faces; |
| for (size_t i = 0; i < maxFaces; i++) { |
| if (faceScores[i] == 0) { |
| metadata.number_of_faces--; |
| continue; |
| } |
| if (faceScores[i] > 100) { |
| ALOGW("%s: Face index %zu with out of range score %d", |
| __FUNCTION__, i, faceScores[i]); |
| } |
| |
| camera_face_t face; |
| |
| face.rect[0] = l.mParameters.arrayXToNormalizedWithCrop( |
| faceRects[i*4 + 0], scalerCrop); |
| face.rect[1] = l.mParameters.arrayYToNormalizedWithCrop( |
| faceRects[i*4 + 1], scalerCrop); |
| face.rect[2] = l.mParameters.arrayXToNormalizedWithCrop( |
| faceRects[i*4 + 2], scalerCrop); |
| face.rect[3] = l.mParameters.arrayYToNormalizedWithCrop( |
| faceRects[i*4 + 3], scalerCrop); |
| |
| face.score = faceScores[i]; |
| if (faceDetectMode == ANDROID_STATISTICS_FACE_DETECT_MODE_FULL) { |
| face.id = faceIds[i]; |
| face.left_eye[0] = l.mParameters.arrayXToNormalizedWithCrop( |
| faceLandmarks[i*6 + 0], scalerCrop); |
| face.left_eye[1] = l.mParameters.arrayYToNormalizedWithCrop( |
| faceLandmarks[i*6 + 1], scalerCrop); |
| face.right_eye[0] = l.mParameters.arrayXToNormalizedWithCrop( |
| faceLandmarks[i*6 + 2], scalerCrop); |
| face.right_eye[1] = l.mParameters.arrayYToNormalizedWithCrop( |
| faceLandmarks[i*6 + 3], scalerCrop); |
| face.mouth[0] = l.mParameters.arrayXToNormalizedWithCrop( |
| faceLandmarks[i*6 + 4], scalerCrop); |
| face.mouth[1] = l.mParameters.arrayYToNormalizedWithCrop( |
| faceLandmarks[i*6 + 5], scalerCrop); |
| } else { |
| face.id = 0; |
| face.left_eye[0] = face.left_eye[1] = -2000; |
| face.right_eye[0] = face.right_eye[1] = -2000; |
| face.mouth[0] = face.mouth[1] = -2000; |
| } |
| faces.push_back(face); |
| } |
| |
| metadata.faces = faces.editArray(); |
| } |
| |
| /* warning: locks SharedCameraCallbacks */ |
| callbackFaceDetection(client, metadata); |
| |
| return OK; |
| } |
| |
| status_t FrameProcessor::process3aState(const CaptureResult &frame, |
| const sp<Camera2Client> &client) { |
| |
| ATRACE_CALL(); |
| const CameraMetadata &metadata = frame.mMetadata; |
| camera_metadata_ro_entry_t entry; |
| int cameraId = client->getCameraId(); |
| |
| entry = metadata.find(ANDROID_REQUEST_FRAME_COUNT); |
| int32_t frameNumber = entry.data.i32[0]; |
| |
| // Don't send 3A notifications for the same frame number twice |
| if (frameNumber <= mLast3AFrameNumber) { |
| ALOGV("%s: Already sent 3A for frame number %d, skipping", |
| __FUNCTION__, frameNumber); |
| return OK; |
| } |
| |
| mLast3AFrameNumber = frameNumber; |
| |
| // Get 3A states from result metadata |
| bool gotAllStates = true; |
| |
| AlgState new3aState; |
| |
| // TODO: Also use AE mode, AE trigger ID |
| |
| gotAllStates &= get3aResult<uint8_t>(metadata, ANDROID_CONTROL_AF_MODE, |
| &new3aState.afMode, frameNumber, cameraId); |
| |
| gotAllStates &= get3aResult<uint8_t>(metadata, ANDROID_CONTROL_AWB_MODE, |
| &new3aState.awbMode, frameNumber, cameraId); |
| |
| gotAllStates &= get3aResult<uint8_t>(metadata, ANDROID_CONTROL_AE_STATE, |
| &new3aState.aeState, frameNumber, cameraId); |
| |
| gotAllStates &= get3aResult<uint8_t>(metadata, ANDROID_CONTROL_AF_STATE, |
| &new3aState.afState, frameNumber, cameraId); |
| |
| gotAllStates &= get3aResult<uint8_t>(metadata, ANDROID_CONTROL_AWB_STATE, |
| &new3aState.awbState, frameNumber, cameraId); |
| |
| if (client->getCameraDeviceVersion() >= CAMERA_DEVICE_API_VERSION_3_2) { |
| new3aState.afTriggerId = frame.mResultExtras.afTriggerId; |
| new3aState.aeTriggerId = frame.mResultExtras.precaptureTriggerId; |
| } else { |
| gotAllStates &= get3aResult<int32_t>(metadata, ANDROID_CONTROL_AF_TRIGGER_ID, |
| &new3aState.afTriggerId, frameNumber, cameraId); |
| |
| gotAllStates &= get3aResult<int32_t>(metadata, ANDROID_CONTROL_AE_PRECAPTURE_ID, |
| &new3aState.aeTriggerId, frameNumber, cameraId); |
| } |
| |
| if (!gotAllStates) return BAD_VALUE; |
| |
| if (new3aState.aeState != m3aState.aeState) { |
| ALOGV("%s: Camera %d: AE state %d->%d", |
| __FUNCTION__, cameraId, |
| m3aState.aeState, new3aState.aeState); |
| client->notifyAutoExposure(new3aState.aeState, new3aState.aeTriggerId); |
| } |
| |
| if (new3aState.afState != m3aState.afState || |
| new3aState.afMode != m3aState.afMode || |
| new3aState.afTriggerId != m3aState.afTriggerId) { |
| ALOGV("%s: Camera %d: AF state %d->%d. AF mode %d->%d. Trigger %d->%d", |
| __FUNCTION__, cameraId, |
| m3aState.afState, new3aState.afState, |
| m3aState.afMode, new3aState.afMode, |
| m3aState.afTriggerId, new3aState.afTriggerId); |
| client->notifyAutoFocus(new3aState.afState, new3aState.afTriggerId); |
| } |
| if (new3aState.awbState != m3aState.awbState || |
| new3aState.awbMode != m3aState.awbMode) { |
| ALOGV("%s: Camera %d: AWB state %d->%d. AWB mode %d->%d", |
| __FUNCTION__, cameraId, |
| m3aState.awbState, new3aState.awbState, |
| m3aState.awbMode, new3aState.awbMode); |
| client->notifyAutoWhitebalance(new3aState.awbState, |
| new3aState.aeTriggerId); |
| } |
| |
| m3aState = new3aState; |
| |
| return OK; |
| } |
| |
| template<typename Src, typename T> |
| bool FrameProcessor::get3aResult(const CameraMetadata& result, int32_t tag, |
| T* value, int32_t frameNumber, int cameraId) { |
| camera_metadata_ro_entry_t entry; |
| if (value == NULL) { |
| ALOGE("%s: Camera %d: Value to write to is NULL", |
| __FUNCTION__, cameraId); |
| return false; |
| } |
| |
| entry = result.find(tag); |
| if (entry.count == 0) { |
| ALOGE("%s: Camera %d: No %s provided by HAL for frame %d!", |
| __FUNCTION__, cameraId, |
| get_camera_metadata_tag_name(tag), frameNumber); |
| return false; |
| } else { |
| switch(sizeof(Src)){ |
| case sizeof(uint8_t): |
| *value = static_cast<T>(entry.data.u8[0]); |
| break; |
| case sizeof(int32_t): |
| *value = static_cast<T>(entry.data.i32[0]); |
| break; |
| default: |
| ALOGE("%s: Camera %d: Unsupported source", |
| __FUNCTION__, cameraId); |
| return false; |
| } |
| } |
| return true; |
| } |
| |
| |
| void FrameProcessor::callbackFaceDetection(sp<Camera2Client> client, |
| const camera_frame_metadata &metadata) { |
| |
| camera_frame_metadata *metadata_ptr = |
| const_cast<camera_frame_metadata*>(&metadata); |
| |
| /** |
| * Filter out repeated 0-face callbacks, |
| * but not when the last frame was >0 |
| */ |
| if (metadata.number_of_faces != 0 || |
| mLastFrameNumberOfFaces != metadata.number_of_faces) { |
| |
| Camera2Client::SharedCameraCallbacks::Lock |
| l(client->mSharedCameraCallbacks); |
| if (l.mRemoteCallback != NULL) { |
| l.mRemoteCallback->dataCallback(CAMERA_MSG_PREVIEW_METADATA, |
| NULL, |
| metadata_ptr); |
| } |
| } |
| |
| mLastFrameNumberOfFaces = metadata.number_of_faces; |
| } |
| |
| }; // namespace camera2 |
| }; // namespace android |
