services/camera/libcameraservice/api1/client2/ZslProcessor.cpp - platform/frameworks/av - Git at Google

 /*
  * Copyright (C) 2013 The Android Open Source Project
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
  * You may obtain a copy of the License at
  *
  *      http://www.apache.org/licenses/LICENSE-2.0
  *
  * Unless required by applicable law or agreed to in writing, software
  * distributed under the License is distributed on an "AS IS" BASIS,
  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */

 #define LOG_TAG "Camera2-ZslProcessor"
 #define ATRACE_TAG ATRACE_TAG_CAMERA
 //#define LOG_NDEBUG 0
 //#define LOG_NNDEBUG 0

 #ifdef LOG_NNDEBUG
 #define ALOGVV(...) ALOGV(__VA_ARGS__)
 #else
 #define ALOGVV(...) if (0) ALOGV(__VA_ARGS__)
 #endif

 #include <inttypes.h>

 #include <utils/Log.h>
 #include <utils/Trace.h>
 #include <gui/Surface.h>

 #include "common/CameraDeviceBase.h"
 #include "api1/Camera2Client.h"
 #include "api1/client2/CaptureSequencer.h"
 #include "api1/client2/ZslProcessor.h"
 #include "device3/Camera3Device.h"

 namespace android {
 namespace camera2 {

 ZslProcessor::ZslProcessor(
     sp<Camera2Client> client,
     wp<CaptureSequencer> sequencer):
         Thread(false),
         mLatestClearedBufferTimestamp(0),
         mState(RUNNING),
         mClient(client),
         mSequencer(sequencer),
         mId(client->getCameraId()),
         mZslStreamId(NO_STREAM),
         mFrameListHead(0),
         mHasFocuser(false) {
     // Initialize buffer queue and frame list based on pipeline max depth.
     size_t pipelineMaxDepth = kDefaultMaxPipelineDepth;
     if (client != 0) {
         sp<Camera3Device> device =
         static_cast<Camera3Device*>(client->getCameraDevice().get());
         if (device != 0) {
             camera_metadata_ro_entry_t entry =
                 device->info().find(ANDROID_REQUEST_PIPELINE_MAX_DEPTH);
             if (entry.count == 1) {
                 pipelineMaxDepth = entry.data.u8[0];
             } else {
                 ALOGW("%s: Unable to find the android.request.pipelineMaxDepth,"
                         " use default pipeline max depth %d", __FUNCTION__,
                         kDefaultMaxPipelineDepth);
             }

             entry = device->info().find(ANDROID_LENS_INFO_MINIMUM_FOCUS_DISTANCE);
             if (entry.count > 0 && entry.data.f[0] != 0.) {
                 mHasFocuser = true;
             }
         }
     }

     ALOGV("%s: Initialize buffer queue and frame list depth based on max pipeline depth (%zu)",
           __FUNCTION__, pipelineMaxDepth);
     // Need to keep buffer queue longer than metadata queue because sometimes buffer arrives
     // earlier than metadata which causes the buffer corresponding to oldest metadata being
     // removed.
     mFrameListDepth = pipelineMaxDepth;
     mBufferQueueDepth = mFrameListDepth + 1;


     mZslQueue.insertAt(0, mBufferQueueDepth);
     mFrameList.insertAt(0, mFrameListDepth);
     sp<CaptureSequencer> captureSequencer = mSequencer.promote();
     if (captureSequencer != 0) captureSequencer->setZslProcessor(this);
 }

 ZslProcessor::~ZslProcessor() {
     ALOGV("%s: Exit", __FUNCTION__);
     deleteStream();
 }

 void ZslProcessor::onResultAvailable(const CaptureResult &result) {
     ATRACE_CALL();
     ALOGV("%s:", __FUNCTION__);
     Mutex::Autolock l(mInputMutex);
     camera_metadata_ro_entry_t entry;
     entry = result.mMetadata.find(ANDROID_SENSOR_TIMESTAMP);
     nsecs_t timestamp = entry.data.i64[0];
     if (entry.count == 0) {
         ALOGE("%s: metadata doesn't have timestamp, skip this result", __FUNCTION__);
         return;
     }

     entry = result.mMetadata.find(ANDROID_REQUEST_FRAME_COUNT);
     if (entry.count == 0) {
         ALOGE("%s: metadata doesn't have frame number, skip this result", __FUNCTION__);
         return;
     }
     int32_t frameNumber = entry.data.i32[0];

     ALOGVV("Got preview metadata for frame %d with timestamp %" PRId64, frameNumber, timestamp);

     if (mState != RUNNING) return;

     // Corresponding buffer has been cleared. No need to push into mFrameList
     if (timestamp <= mLatestClearedBufferTimestamp) return;

     mFrameList.editItemAt(mFrameListHead) = result.mMetadata;
     mFrameListHead = (mFrameListHead + 1) % mFrameListDepth;
 }

 status_t ZslProcessor::updateStream(const Parameters &params) {
     ATRACE_CALL();
     ALOGV("%s: Configuring ZSL streams", __FUNCTION__);
     status_t res;

     Mutex::Autolock l(mInputMutex);

     sp<Camera2Client> client = mClient.promote();
     if (client == 0) {
         ALOGE("%s: Camera %d: Client does not exist", __FUNCTION__, mId);
         return INVALID_OPERATION;
     }
     sp<Camera3Device> device =
         static_cast<Camera3Device*>(client->getCameraDevice().get());
     if (device == 0) {
         ALOGE("%s: Camera %d: Device does not exist", __FUNCTION__, mId);
         return INVALID_OPERATION;
     }

     if (mZslStreamId != NO_STREAM) {
         // Check if stream parameters have to change
         uint32_t currentWidth, currentHeight;
         res = device->getStreamInfo(mZslStreamId,
                 &currentWidth, &currentHeight, 0, 0);
         if (res != OK) {
             ALOGE("%s: Camera %d: Error querying capture output stream info: "
                     "%s (%d)", __FUNCTION__,
                     client->getCameraId(), strerror(-res), res);
             return res;
         }
         if (currentWidth != (uint32_t)params.fastInfo.arrayWidth ||
                 currentHeight != (uint32_t)params.fastInfo.arrayHeight) {
             ALOGV("%s: Camera %d: Deleting stream %d since the buffer "
                   "dimensions changed",
                 __FUNCTION__, client->getCameraId(), mZslStreamId);
             res = device->deleteStream(mZslStreamId);
             if (res == -EBUSY) {
                 ALOGV("%s: Camera %d: Device is busy, call updateStream again "
                       " after it becomes idle", __FUNCTION__, mId);
                 return res;
             } else if(res != OK) {
                 ALOGE("%s: Camera %d: Unable to delete old output stream "
                         "for ZSL: %s (%d)", __FUNCTION__,
                         client->getCameraId(), strerror(-res), res);
                 return res;
             }
             mZslStreamId = NO_STREAM;
         }
     }

     if (mZslStreamId == NO_STREAM) {
         // Create stream for HAL production
         // TODO: Sort out better way to select resolution for ZSL

         // Note that format specified internally in Camera3ZslStream
         res = device->createZslStream(
                 params.fastInfo.arrayWidth, params.fastInfo.arrayHeight,
                 mBufferQueueDepth,
                 &mZslStreamId,
                 &mZslStream);
         if (res != OK) {
             ALOGE("%s: Camera %d: Can't create ZSL stream: "
                     "%s (%d)", __FUNCTION__, client->getCameraId(),
                     strerror(-res), res);
             return res;
         }

         // Only add the camera3 buffer listener when the stream is created.
         mZslStream->addBufferListener(this);
     }

     client->registerFrameListener(Camera2Client::kPreviewRequestIdStart,
             Camera2Client::kPreviewRequestIdEnd,
             this,
             /*sendPartials*/false);

     return OK;
 }

 status_t ZslProcessor::deleteStream() {
     ATRACE_CALL();
     status_t res;

     Mutex::Autolock l(mInputMutex);

     if (mZslStreamId != NO_STREAM) {
         sp<Camera2Client> client = mClient.promote();
         if (client == 0) {
             ALOGE("%s: Camera %d: Client does not exist", __FUNCTION__, mId);
             return INVALID_OPERATION;
         }

         sp<Camera3Device> device =
             reinterpret_cast<Camera3Device*>(client->getCameraDevice().get());
         if (device == 0) {
             ALOGE("%s: Camera %d: Device does not exist", __FUNCTION__, mId);
             return INVALID_OPERATION;
         }

         res = device->deleteStream(mZslStreamId);
         if (res != OK) {
             ALOGE("%s: Camera %d: Cannot delete ZSL output stream %d: "
                     "%s (%d)", __FUNCTION__, client->getCameraId(),
                     mZslStreamId, strerror(-res), res);
             return res;
         }

         mZslStreamId = NO_STREAM;
     }
     return OK;
 }

 int ZslProcessor::getStreamId() const {
     Mutex::Autolock l(mInputMutex);
     return mZslStreamId;
 }

 status_t ZslProcessor::updateRequestWithDefaultStillRequest(CameraMetadata &request) const {
     sp<Camera2Client> client = mClient.promote();
     if (client == 0) {
         ALOGE("%s: Camera %d: Client does not exist", __FUNCTION__, mId);
         return INVALID_OPERATION;
     }
     sp<Camera3Device> device =
         static_cast<Camera3Device*>(client->getCameraDevice().get());
     if (device == 0) {
         ALOGE("%s: Camera %d: Device does not exist", __FUNCTION__, mId);
         return INVALID_OPERATION;
     }

     CameraMetadata stillTemplate;
     device->createDefaultRequest(CAMERA3_TEMPLATE_STILL_CAPTURE, &stillTemplate);

     // Find some of the post-processing tags, and assign the value from template to the request.
     // Only check the aberration mode and noise reduction mode for now, as they are very important
     // for image quality.
     uint32_t postProcessingTags[] = {
             ANDROID_NOISE_REDUCTION_MODE,
             ANDROID_COLOR_CORRECTION_ABERRATION_MODE,
             ANDROID_COLOR_CORRECTION_MODE,
             ANDROID_TONEMAP_MODE,
             ANDROID_SHADING_MODE,
             ANDROID_HOT_PIXEL_MODE,
             ANDROID_EDGE_MODE
     };

     camera_metadata_entry_t entry;
     for (size_t i = 0; i < sizeof(postProcessingTags) / sizeof(uint32_t); i++) {
         entry = stillTemplate.find(postProcessingTags[i]);
         if (entry.count > 0) {
             request.update(postProcessingTags[i], entry.data.u8, 1);
         }
     }

     return OK;
 }

 status_t ZslProcessor::pushToReprocess(int32_t requestId) {
     ALOGV("%s: Send in reprocess request with id %d",
             __FUNCTION__, requestId);
     Mutex::Autolock l(mInputMutex);
     status_t res;
     sp<Camera2Client> client = mClient.promote();

     if (client == 0) {
         ALOGE("%s: Camera %d: Client does not exist", __FUNCTION__, mId);
         return INVALID_OPERATION;
     }

     IF_ALOGV() {
         dumpZslQueue(-1);
     }

     size_t metadataIdx;
     nsecs_t candidateTimestamp = getCandidateTimestampLocked(&metadataIdx);

     if (candidateTimestamp == -1) {
         ALOGE("%s: Could not find good candidate for ZSL reprocessing",
               __FUNCTION__);
         return NOT_ENOUGH_DATA;
     }

     res = mZslStream->enqueueInputBufferByTimestamp(candidateTimestamp,
                                                     /*actualTimestamp*/NULL);

     if (res == mZslStream->NO_BUFFER_AVAILABLE) {
         ALOGV("%s: No ZSL buffers yet", __FUNCTION__);
         return NOT_ENOUGH_DATA;
     } else if (res != OK) {
         ALOGE("%s: Unable to push buffer for reprocessing: %s (%d)",
                 __FUNCTION__, strerror(-res), res);
         return res;
     }

     {
         CameraMetadata request = mFrameList[metadataIdx];

         // Verify that the frame is reasonable for reprocessing

         camera_metadata_entry_t entry;
         entry = request.find(ANDROID_CONTROL_AE_STATE);
         if (entry.count == 0) {
             ALOGE("%s: ZSL queue frame has no AE state field!",
                     __FUNCTION__);
             return BAD_VALUE;
         }
         if (entry.data.u8[0] != ANDROID_CONTROL_AE_STATE_CONVERGED &&
                 entry.data.u8[0] != ANDROID_CONTROL_AE_STATE_LOCKED) {
             ALOGV("%s: ZSL queue frame AE state is %d, need full capture",
                     __FUNCTION__, entry.data.u8[0]);
             return NOT_ENOUGH_DATA;
         }

         uint8_t requestType = ANDROID_REQUEST_TYPE_REPROCESS;
         res = request.update(ANDROID_REQUEST_TYPE,
                 &requestType, 1);
         if (res != OK) {
             ALOGE("%s: Unable to update request type",
                   __FUNCTION__);
             return INVALID_OPERATION;
         }

         int32_t inputStreams[1] =
                 { mZslStreamId };
         res = request.update(ANDROID_REQUEST_INPUT_STREAMS,
                 inputStreams, 1);
         if (res != OK) {
             ALOGE("%s: Unable to update request input streams",
                   __FUNCTION__);
             return INVALID_OPERATION;
         }

         uint8_t captureIntent =
                 static_cast<uint8_t>(ANDROID_CONTROL_CAPTURE_INTENT_STILL_CAPTURE);
         res = request.update(ANDROID_CONTROL_CAPTURE_INTENT,
                 &captureIntent, 1);
         if (res != OK ) {
             ALOGE("%s: Unable to update request capture intent",
                   __FUNCTION__);
             return INVALID_OPERATION;
         }

         // TODO: Shouldn't we also update the latest preview frame?
         int32_t outputStreams[1] =
                 { client->getCaptureStreamId() };
         res = request.update(ANDROID_REQUEST_OUTPUT_STREAMS,
                 outputStreams, 1);
         if (res != OK) {
             ALOGE("%s: Unable to update request output streams",
                   __FUNCTION__);
             return INVALID_OPERATION;
         }

         res = request.update(ANDROID_REQUEST_ID,
                 &requestId, 1);
         if (res != OK ) {
             ALOGE("%s: Unable to update frame to a reprocess request",
                   __FUNCTION__);
             return INVALID_OPERATION;
         }

         res = client->stopStream();
         if (res != OK) {
             ALOGE("%s: Camera %d: Unable to stop preview for ZSL capture: "
                 "%s (%d)",
                 __FUNCTION__, client->getCameraId(), strerror(-res), res);
             return INVALID_OPERATION;
         }

         // Update JPEG settings
         {
             SharedParameters::Lock l(client->getParameters());
             res = l.mParameters.updateRequestJpeg(&request);
             if (res != OK) {
                 ALOGE("%s: Camera %d: Unable to update JPEG entries of ZSL "
                         "capture request: %s (%d)", __FUNCTION__,
                         client->getCameraId(),
                         strerror(-res), res);
                 return res;
             }
         }

         // Update post-processing settings
         res = updateRequestWithDefaultStillRequest(request);
         if (res != OK) {
             ALOGW("%s: Unable to update post-processing tags, the reprocessed image quality "
                     "may be compromised", __FUNCTION__);
         }

         mLatestCapturedRequest = request;
         res = client->getCameraDevice()->capture(request);
         if (res != OK ) {
             ALOGE("%s: Unable to send ZSL reprocess request to capture: %s"
                   " (%d)", __FUNCTION__, strerror(-res), res);
             return res;
         }

         mState = LOCKED;
     }

     return OK;
 }

 status_t ZslProcessor::clearZslQueue() {
     Mutex::Autolock l(mInputMutex);
     // If in middle of capture, can't clear out queue
     if (mState == LOCKED) return OK;

     return clearZslQueueLocked();
 }

 status_t ZslProcessor::clearZslQueueLocked() {
     if (mZslStream != 0) {
         // clear result metadata list first.
         clearZslResultQueueLocked();
         return mZslStream->clearInputRingBuffer(&mLatestClearedBufferTimestamp);
     }
     return OK;
 }

 void ZslProcessor::clearZslResultQueueLocked() {
     mFrameList.clear();
     mFrameListHead = 0;
     mFrameList.insertAt(0, mFrameListDepth);
 }

 void ZslProcessor::dump(int fd, const Vector<String16>& /*args*/) const {
     Mutex::Autolock l(mInputMutex);
     if (!mLatestCapturedRequest.isEmpty()) {
         String8 result("    Latest ZSL capture request:\n");
         write(fd, result.string(), result.size());
         mLatestCapturedRequest.dump(fd, 2, 6);
     } else {
         String8 result("    Latest ZSL capture request: none yet\n");
         write(fd, result.string(), result.size());
     }
     dumpZslQueue(fd);
 }

 bool ZslProcessor::threadLoop() {
     // TODO: remove dependency on thread. For now, shut thread down right
     // away.
     return false;
 }

 void ZslProcessor::dumpZslQueue(int fd) const {
     String8 header("ZSL queue contents:");
     String8 indent("    ");
     ALOGV("%s", header.string());
     if (fd != -1) {
         header = indent + header + "\n";
         write(fd, header.string(), header.size());
     }
     for (size_t i = 0; i < mZslQueue.size(); i++) {
         const ZslPair &queueEntry = mZslQueue[i];
         nsecs_t bufferTimestamp = queueEntry.buffer.mTimestamp;
         camera_metadata_ro_entry_t entry;
         nsecs_t frameTimestamp = 0;
         int frameAeState = -1;
         if (!queueEntry.frame.isEmpty()) {
             entry = queueEntry.frame.find(ANDROID_SENSOR_TIMESTAMP);
             if (entry.count > 0) frameTimestamp = entry.data.i64[0];
             entry = queueEntry.frame.find(ANDROID_CONTROL_AE_STATE);
             if (entry.count > 0) frameAeState = entry.data.u8[0];
         }
         String8 result =
                 String8::format("   %zu: b: %" PRId64 "\tf: %" PRId64 ", AE state: %d", i,
                         bufferTimestamp, frameTimestamp, frameAeState);
         ALOGV("%s", result.string());
         if (fd != -1) {
             result = indent + result + "\n";
             write(fd, result.string(), result.size());
         }

     }
 }

 bool ZslProcessor::isFixedFocusMode(uint8_t afMode) const {
     switch (afMode) {
         case ANDROID_CONTROL_AF_MODE_AUTO:
         case ANDROID_CONTROL_AF_MODE_CONTINUOUS_VIDEO:
         case ANDROID_CONTROL_AF_MODE_CONTINUOUS_PICTURE:
         case ANDROID_CONTROL_AF_MODE_MACRO:
             return false;
             break;
         case ANDROID_CONTROL_AF_MODE_OFF:
         case ANDROID_CONTROL_AF_MODE_EDOF:
             return true;
         default:
             ALOGE("%s: unknown focus mode %d", __FUNCTION__, afMode);
             return false;
     }
 }

 nsecs_t ZslProcessor::getCandidateTimestampLocked(size_t* metadataIdx) const {
     /**
      * Find the smallest timestamp we know about so far
      * - ensure that aeState is either converged or locked
      */

     size_t idx = 0;
     nsecs_t minTimestamp = -1;

     size_t emptyCount = mFrameList.size();

     for (size_t j = 0; j < mFrameList.size(); j++) {
         const CameraMetadata &frame = mFrameList[j];
         if (!frame.isEmpty()) {

             emptyCount--;

             camera_metadata_ro_entry_t entry;
             entry = frame.find(ANDROID_SENSOR_TIMESTAMP);
             if (entry.count == 0) {
                 ALOGE("%s: Can't find timestamp in frame!",
                         __FUNCTION__);
                 continue;
             }
             nsecs_t frameTimestamp = entry.data.i64[0];
             if (minTimestamp > frameTimestamp || minTimestamp == -1) {

                 entry = frame.find(ANDROID_CONTROL_AE_STATE);

                 if (entry.count == 0) {
                     /**
                      * This is most likely a HAL bug. The aeState field is
                      * mandatory, so it should always be in a metadata packet.
                      */
                     ALOGW("%s: ZSL queue frame has no AE state field!",
                             __FUNCTION__);
                     continue;
                 }
                 if (entry.data.u8[0] != ANDROID_CONTROL_AE_STATE_CONVERGED &&
                         entry.data.u8[0] != ANDROID_CONTROL_AE_STATE_LOCKED) {
                     ALOGVV("%s: ZSL queue frame AE state is %d, need "
                            "full capture",  __FUNCTION__, entry.data.u8[0]);
                     continue;
                 }

                 entry = frame.find(ANDROID_CONTROL_AF_MODE);
                 if (entry.count == 0) {
                     ALOGW("%s: ZSL queue frame has no AF mode field!",
                             __FUNCTION__);
                     continue;
                 }
                 uint8_t afMode = entry.data.u8[0];
                 if (afMode == ANDROID_CONTROL_AF_MODE_OFF) {
                     // Skip all the ZSL buffer for manual AF mode, as we don't really
                     // know the af state.
                     continue;
                 }

                 // Check AF state if device has focuser and focus mode isn't fixed
                 if (mHasFocuser && !isFixedFocusMode(afMode)) {
                     // Make sure the candidate frame has good focus.
                     entry = frame.find(ANDROID_CONTROL_AF_STATE);
                     if (entry.count == 0) {
                         ALOGW("%s: ZSL queue frame has no AF state field!",
                                 __FUNCTION__);
                         continue;
                     }
                     uint8_t afState = entry.data.u8[0];
                     if (afState != ANDROID_CONTROL_AF_STATE_PASSIVE_FOCUSED &&
                             afState != ANDROID_CONTROL_AF_STATE_FOCUSED_LOCKED &&
                             afState != ANDROID_CONTROL_AF_STATE_NOT_FOCUSED_LOCKED) {
                         ALOGVV("%s: ZSL queue frame AF state is %d is not good for capture, skip it",
                                 __FUNCTION__, afState);
                         continue;
                     }
                 }

                 minTimestamp = frameTimestamp;
                 idx = j;
             }

             ALOGVV("%s: Saw timestamp %" PRId64, __FUNCTION__, frameTimestamp);
         }
     }

     if (emptyCount == mFrameList.size()) {
         /**
          * This could be mildly bad and means our ZSL was triggered before
          * there were any frames yet received by the camera framework.
          *
          * This is a fairly corner case which can happen under:
          * + a user presses the shutter button real fast when the camera starts
          *     (startPreview followed immediately by takePicture).
          * + burst capture case (hitting shutter button as fast possible)
          *
          * If this happens in steady case (preview running for a while, call
          *     a single takePicture) then this might be a fwk bug.
          */
         ALOGW("%s: ZSL queue has no metadata frames", __FUNCTION__);
     }

     ALOGV("%s: Candidate timestamp %" PRId64 " (idx %zu), empty frames: %zu",
           __FUNCTION__, minTimestamp, idx, emptyCount);

     if (metadataIdx) {
         *metadataIdx = idx;
     }

     return minTimestamp;
 }

 void ZslProcessor::onBufferAcquired(const BufferInfo& /*bufferInfo*/) {
     // Intentionally left empty
     // Although theoretically we could use this to get better dump info
 }

 void ZslProcessor::onBufferReleased(const BufferInfo& bufferInfo) {

     // ignore output buffers
     if (bufferInfo.mOutput) {
         return;
     }

     // Lock mutex only once we know this is an input buffer returned to avoid
     // potential deadlock
     Mutex::Autolock l(mInputMutex);
     // TODO: Verify that the buffer is in our queue by looking at timestamp
     // theoretically unnecessary unless we change the following assumptions:
     // -- only 1 buffer reprocessed at a time (which is the case now)

     // Erase entire ZSL queue since we've now completed the capture and preview
     // is stopped.
     //
     // We need to guarantee that if we do two back-to-back captures,
     // the second won't use a buffer that's older/the same as the first, which
     // is theoretically possible if we don't clear out the queue and the
     // selection criteria is something like 'newest'. Clearing out the result
     // metadata queue on a completed capture ensures we'll only use new timestamp.
     // Calling clearZslQueueLocked is a guaranteed deadlock because this callback
     // holds the Camera3Stream internal lock (mLock), and clearZslQueueLocked requires
     // to hold the same lock.
     // TODO: need figure out a way to clear the Zsl buffer queue properly. Right now
     // it is safe not to do so, as back to back ZSL capture requires stop and start
     // preview, which will flush ZSL queue automatically.
     ALOGV("%s: Memory optimization, clearing ZSL queue",
           __FUNCTION__);
     clearZslResultQueueLocked();

     // Required so we accept more ZSL requests
     mState = RUNNING;
 }

 }; // namespace camera2
 }; // namespace android
	/*
	* Copyright (C) 2013 The Android Open Source Project
	*
	* Licensed under the Apache License, Version 2.0 (the "License");
	* you may not use this file except in compliance with the License.
	* You may obtain a copy of the License at
	*
	* http://www.apache.org/licenses/LICENSE-2.0
	*
	* Unless required by applicable law or agreed to in writing, software
	* distributed under the License is distributed on an "AS IS" BASIS,
	* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
	* See the License for the specific language governing permissions and
	* limitations under the License.
	*/

	#define LOG_TAG "Camera2-ZslProcessor"
	#define ATRACE_TAG ATRACE_TAG_CAMERA
	//#define LOG_NDEBUG 0
	//#define LOG_NNDEBUG 0

	#ifdef LOG_NNDEBUG
	#define ALOGVV(...) ALOGV(__VA_ARGS__)
	#else
	#define ALOGVV(...) if (0) ALOGV(__VA_ARGS__)
	#endif

	#include <inttypes.h>

	#include <utils/Log.h>
	#include <utils/Trace.h>
	#include <gui/Surface.h>

	#include "common/CameraDeviceBase.h"
	#include "api1/Camera2Client.h"
	#include "api1/client2/CaptureSequencer.h"
	#include "api1/client2/ZslProcessor.h"
	#include "device3/Camera3Device.h"

	namespace android {
	namespace camera2 {

	ZslProcessor::ZslProcessor(
	sp<Camera2Client> client,
	wp<CaptureSequencer> sequencer):
	Thread(false),
	mLatestClearedBufferTimestamp(0),
	mState(RUNNING),
	mClient(client),
	mSequencer(sequencer),
	mId(client->getCameraId()),
	mZslStreamId(NO_STREAM),
	mFrameListHead(0),
	mHasFocuser(false) {
	// Initialize buffer queue and frame list based on pipeline max depth.
	size_t pipelineMaxDepth = kDefaultMaxPipelineDepth;
	if (client != 0) {
	sp<Camera3Device> device =
	static_cast<Camera3Device*>(client->getCameraDevice().get());
	if (device != 0) {
	camera_metadata_ro_entry_t entry =
	device->info().find(ANDROID_REQUEST_PIPELINE_MAX_DEPTH);
	if (entry.count == 1) {
	pipelineMaxDepth = entry.data.u8[0];
	} else {
	ALOGW("%s: Unable to find the android.request.pipelineMaxDepth,"
	" use default pipeline max depth %d", __FUNCTION__,
	kDefaultMaxPipelineDepth);
	}

	entry = device->info().find(ANDROID_LENS_INFO_MINIMUM_FOCUS_DISTANCE);
	if (entry.count > 0 && entry.data.f[0] != 0.) {
	mHasFocuser = true;
	}
	}
	}

	ALOGV("%s: Initialize buffer queue and frame list depth based on max pipeline depth (%zu)",
	__FUNCTION__, pipelineMaxDepth);
	// Need to keep buffer queue longer than metadata queue because sometimes buffer arrives
	// earlier than metadata which causes the buffer corresponding to oldest metadata being
	// removed.
	mFrameListDepth = pipelineMaxDepth;
	mBufferQueueDepth = mFrameListDepth + 1;


	mZslQueue.insertAt(0, mBufferQueueDepth);
	mFrameList.insertAt(0, mFrameListDepth);
	sp<CaptureSequencer> captureSequencer = mSequencer.promote();
	if (captureSequencer != 0) captureSequencer->setZslProcessor(this);
	}

	ZslProcessor::~ZslProcessor() {
	ALOGV("%s: Exit", __FUNCTION__);
	deleteStream();
	}

	void ZslProcessor::onResultAvailable(const CaptureResult &result) {
	ATRACE_CALL();
	ALOGV("%s:", __FUNCTION__);
	Mutex::Autolock l(mInputMutex);
	camera_metadata_ro_entry_t entry;
	entry = result.mMetadata.find(ANDROID_SENSOR_TIMESTAMP);
	nsecs_t timestamp = entry.data.i64[0];
	if (entry.count == 0) {
	ALOGE("%s: metadata doesn't have timestamp, skip this result", __FUNCTION__);
	return;
	}

	entry = result.mMetadata.find(ANDROID_REQUEST_FRAME_COUNT);
	if (entry.count == 0) {
	ALOGE("%s: metadata doesn't have frame number, skip this result", __FUNCTION__);
	return;
	}
	int32_t frameNumber = entry.data.i32[0];

	ALOGVV("Got preview metadata for frame %d with timestamp %" PRId64, frameNumber, timestamp);

	if (mState != RUNNING) return;

	// Corresponding buffer has been cleared. No need to push into mFrameList
	if (timestamp <= mLatestClearedBufferTimestamp) return;

	mFrameList.editItemAt(mFrameListHead) = result.mMetadata;
	mFrameListHead = (mFrameListHead + 1) % mFrameListDepth;
	}

	status_t ZslProcessor::updateStream(const Parameters &params) {
	ATRACE_CALL();
	ALOGV("%s: Configuring ZSL streams", __FUNCTION__);
	status_t res;

	Mutex::Autolock l(mInputMutex);

	sp<Camera2Client> client = mClient.promote();
	if (client == 0) {
	ALOGE("%s: Camera %d: Client does not exist", __FUNCTION__, mId);
	return INVALID_OPERATION;
	}
	sp<Camera3Device> device =
	static_cast<Camera3Device*>(client->getCameraDevice().get());
	if (device == 0) {
	ALOGE("%s: Camera %d: Device does not exist", __FUNCTION__, mId);
	return INVALID_OPERATION;
	}

	if (mZslStreamId != NO_STREAM) {
	// Check if stream parameters have to change
	uint32_t currentWidth, currentHeight;
	res = device->getStreamInfo(mZslStreamId,
	&currentWidth, &currentHeight, 0, 0);
	if (res != OK) {
	ALOGE("%s: Camera %d: Error querying capture output stream info: "
	"%s (%d)", __FUNCTION__,
	client->getCameraId(), strerror(-res), res);
	return res;
	}
	if (currentWidth != (uint32_t)params.fastInfo.arrayWidth \|\|
	currentHeight != (uint32_t)params.fastInfo.arrayHeight) {
	ALOGV("%s: Camera %d: Deleting stream %d since the buffer "
	"dimensions changed",
	__FUNCTION__, client->getCameraId(), mZslStreamId);
	res = device->deleteStream(mZslStreamId);
	if (res == -EBUSY) {
	ALOGV("%s: Camera %d: Device is busy, call updateStream again "
	" after it becomes idle", __FUNCTION__, mId);
	return res;
	} else if(res != OK) {
	ALOGE("%s: Camera %d: Unable to delete old output stream "
	"for ZSL: %s (%d)", __FUNCTION__,
	client->getCameraId(), strerror(-res), res);
	return res;
	}
	mZslStreamId = NO_STREAM;
	}
	}

	if (mZslStreamId == NO_STREAM) {
	// Create stream for HAL production
	// TODO: Sort out better way to select resolution for ZSL

	// Note that format specified internally in Camera3ZslStream
	res = device->createZslStream(
	params.fastInfo.arrayWidth, params.fastInfo.arrayHeight,
	mBufferQueueDepth,
	&mZslStreamId,
	&mZslStream);
	if (res != OK) {
	ALOGE("%s: Camera %d: Can't create ZSL stream: "
	"%s (%d)", __FUNCTION__, client->getCameraId(),
	strerror(-res), res);
	return res;
	}

	// Only add the camera3 buffer listener when the stream is created.
	mZslStream->addBufferListener(this);
	}

	client->registerFrameListener(Camera2Client::kPreviewRequestIdStart,
	Camera2Client::kPreviewRequestIdEnd,
	this,
	/sendPartials/false);

	return OK;
	}

	status_t ZslProcessor::deleteStream() {
	ATRACE_CALL();
	status_t res;

	Mutex::Autolock l(mInputMutex);

	if (mZslStreamId != NO_STREAM) {
	sp<Camera2Client> client = mClient.promote();
	if (client == 0) {
	ALOGE("%s: Camera %d: Client does not exist", __FUNCTION__, mId);
	return INVALID_OPERATION;
	}

	sp<Camera3Device> device =
	reinterpret_cast<Camera3Device*>(client->getCameraDevice().get());
	if (device == 0) {
	ALOGE("%s: Camera %d: Device does not exist", __FUNCTION__, mId);
	return INVALID_OPERATION;
	}

	res = device->deleteStream(mZslStreamId);
	if (res != OK) {
	ALOGE("%s: Camera %d: Cannot delete ZSL output stream %d: "
	"%s (%d)", __FUNCTION__, client->getCameraId(),
	mZslStreamId, strerror(-res), res);
	return res;
	}

	mZslStreamId = NO_STREAM;
	}
	return OK;
	}

	int ZslProcessor::getStreamId() const {
	Mutex::Autolock l(mInputMutex);
	return mZslStreamId;
	}

	status_t ZslProcessor::updateRequestWithDefaultStillRequest(CameraMetadata &request) const {
	sp<Camera2Client> client = mClient.promote();
	if (client == 0) {
	ALOGE("%s: Camera %d: Client does not exist", __FUNCTION__, mId);
	return INVALID_OPERATION;
	}
	sp<Camera3Device> device =
	static_cast<Camera3Device*>(client->getCameraDevice().get());
	if (device == 0) {
	ALOGE("%s: Camera %d: Device does not exist", __FUNCTION__, mId);
	return INVALID_OPERATION;
	}

	CameraMetadata stillTemplate;
	device->createDefaultRequest(CAMERA3_TEMPLATE_STILL_CAPTURE, &stillTemplate);

	// Find some of the post-processing tags, and assign the value from template to the request.
	// Only check the aberration mode and noise reduction mode for now, as they are very important
	// for image quality.
	uint32_t postProcessingTags[] = {
	ANDROID_NOISE_REDUCTION_MODE,
	ANDROID_COLOR_CORRECTION_ABERRATION_MODE,
	ANDROID_COLOR_CORRECTION_MODE,
	ANDROID_TONEMAP_MODE,
	ANDROID_SHADING_MODE,
	ANDROID_HOT_PIXEL_MODE,
	ANDROID_EDGE_MODE
	};

	camera_metadata_entry_t entry;
	for (size_t i = 0; i < sizeof(postProcessingTags) / sizeof(uint32_t); i++) {
	entry = stillTemplate.find(postProcessingTags[i]);
	if (entry.count > 0) {
	request.update(postProcessingTags[i], entry.data.u8, 1);
	}
	}

	return OK;
	}

	status_t ZslProcessor::pushToReprocess(int32_t requestId) {
	ALOGV("%s: Send in reprocess request with id %d",
	__FUNCTION__, requestId);
	Mutex::Autolock l(mInputMutex);
	status_t res;
	sp<Camera2Client> client = mClient.promote();

	if (client == 0) {
	ALOGE("%s: Camera %d: Client does not exist", __FUNCTION__, mId);
	return INVALID_OPERATION;
	}

	IF_ALOGV() {
	dumpZslQueue(-1);
	}

	size_t metadataIdx;
	nsecs_t candidateTimestamp = getCandidateTimestampLocked(&metadataIdx);

	if (candidateTimestamp == -1) {
	ALOGE("%s: Could not find good candidate for ZSL reprocessing",
	__FUNCTION__);
	return NOT_ENOUGH_DATA;
	}

	res = mZslStream->enqueueInputBufferByTimestamp(candidateTimestamp,
	/actualTimestamp/NULL);

	if (res == mZslStream->NO_BUFFER_AVAILABLE) {
	ALOGV("%s: No ZSL buffers yet", __FUNCTION__);
	return NOT_ENOUGH_DATA;
	} else if (res != OK) {
	ALOGE("%s: Unable to push buffer for reprocessing: %s (%d)",
	__FUNCTION__, strerror(-res), res);
	return res;
	}

	{
	CameraMetadata request = mFrameList[metadataIdx];

	// Verify that the frame is reasonable for reprocessing

	camera_metadata_entry_t entry;
	entry = request.find(ANDROID_CONTROL_AE_STATE);
	if (entry.count == 0) {
	ALOGE("%s: ZSL queue frame has no AE state field!",
	__FUNCTION__);
	return BAD_VALUE;
	}
	if (entry.data.u8[0] != ANDROID_CONTROL_AE_STATE_CONVERGED &&
	entry.data.u8[0] != ANDROID_CONTROL_AE_STATE_LOCKED) {
	ALOGV("%s: ZSL queue frame AE state is %d, need full capture",
	__FUNCTION__, entry.data.u8[0]);
	return NOT_ENOUGH_DATA;
	}

	uint8_t requestType = ANDROID_REQUEST_TYPE_REPROCESS;
	res = request.update(ANDROID_REQUEST_TYPE,
	&requestType, 1);
	if (res != OK) {
	ALOGE("%s: Unable to update request type",
	__FUNCTION__);
	return INVALID_OPERATION;
	}

	int32_t inputStreams[1] =
	{ mZslStreamId };
	res = request.update(ANDROID_REQUEST_INPUT_STREAMS,
	inputStreams, 1);
	if (res != OK) {
	ALOGE("%s: Unable to update request input streams",
	__FUNCTION__);
	return INVALID_OPERATION;
	}

	uint8_t captureIntent =
	static_cast<uint8_t>(ANDROID_CONTROL_CAPTURE_INTENT_STILL_CAPTURE);
	res = request.update(ANDROID_CONTROL_CAPTURE_INTENT,
	&captureIntent, 1);
	if (res != OK ) {
	ALOGE("%s: Unable to update request capture intent",
	__FUNCTION__);
	return INVALID_OPERATION;
	}

	// TODO: Shouldn't we also update the latest preview frame?
	int32_t outputStreams[1] =
	{ client->getCaptureStreamId() };
	res = request.update(ANDROID_REQUEST_OUTPUT_STREAMS,
	outputStreams, 1);
	if (res != OK) {
	ALOGE("%s: Unable to update request output streams",
	__FUNCTION__);
	return INVALID_OPERATION;
	}

	res = request.update(ANDROID_REQUEST_ID,
	&requestId, 1);
	if (res != OK ) {
	ALOGE("%s: Unable to update frame to a reprocess request",
	__FUNCTION__);
	return INVALID_OPERATION;
	}

	res = client->stopStream();
	if (res != OK) {
	ALOGE("%s: Camera %d: Unable to stop preview for ZSL capture: "
	"%s (%d)",
	__FUNCTION__, client->getCameraId(), strerror(-res), res);
	return INVALID_OPERATION;
	}

	// Update JPEG settings
	{
	SharedParameters::Lock l(client->getParameters());
	res = l.mParameters.updateRequestJpeg(&request);
	if (res != OK) {
	ALOGE("%s: Camera %d: Unable to update JPEG entries of ZSL "
	"capture request: %s (%d)", __FUNCTION__,
	client->getCameraId(),
	strerror(-res), res);
	return res;
	}
	}

	// Update post-processing settings
	res = updateRequestWithDefaultStillRequest(request);
	if (res != OK) {
	ALOGW("%s: Unable to update post-processing tags, the reprocessed image quality "
	"may be compromised", __FUNCTION__);
	}

	mLatestCapturedRequest = request;
	res = client->getCameraDevice()->capture(request);
	if (res != OK ) {
	ALOGE("%s: Unable to send ZSL reprocess request to capture: %s"
	" (%d)", __FUNCTION__, strerror(-res), res);
	return res;
	}

	mState = LOCKED;
	}

	return OK;
	}

	status_t ZslProcessor::clearZslQueue() {
	Mutex::Autolock l(mInputMutex);
	// If in middle of capture, can't clear out queue
	if (mState == LOCKED) return OK;

	return clearZslQueueLocked();
	}

	status_t ZslProcessor::clearZslQueueLocked() {
	if (mZslStream != 0) {
	// clear result metadata list first.
	clearZslResultQueueLocked();
	return mZslStream->clearInputRingBuffer(&mLatestClearedBufferTimestamp);
	}
	return OK;
	}

	void ZslProcessor::clearZslResultQueueLocked() {
	mFrameList.clear();
	mFrameListHead = 0;
	mFrameList.insertAt(0, mFrameListDepth);
	}

	void ZslProcessor::dump(int fd, const Vector<String16>& /args/) const {
	Mutex::Autolock l(mInputMutex);
	if (!mLatestCapturedRequest.isEmpty()) {
	String8 result(" Latest ZSL capture request:\n");
	write(fd, result.string(), result.size());
	mLatestCapturedRequest.dump(fd, 2, 6);
	} else {
	String8 result(" Latest ZSL capture request: none yet\n");
	write(fd, result.string(), result.size());
	}
	dumpZslQueue(fd);
	}

	bool ZslProcessor::threadLoop() {
	// TODO: remove dependency on thread. For now, shut thread down right
	// away.
	return false;
	}

	void ZslProcessor::dumpZslQueue(int fd) const {
	String8 header("ZSL queue contents:");
	String8 indent(" ");
	ALOGV("%s", header.string());
	if (fd != -1) {
	header = indent + header + "\n";
	write(fd, header.string(), header.size());
	}
	for (size_t i = 0; i < mZslQueue.size(); i++) {
	const ZslPair &queueEntry = mZslQueue[i];
	nsecs_t bufferTimestamp = queueEntry.buffer.mTimestamp;
	camera_metadata_ro_entry_t entry;
	nsecs_t frameTimestamp = 0;
	int frameAeState = -1;
	if (!queueEntry.frame.isEmpty()) {
	entry = queueEntry.frame.find(ANDROID_SENSOR_TIMESTAMP);
	if (entry.count > 0) frameTimestamp = entry.data.i64[0];
	entry = queueEntry.frame.find(ANDROID_CONTROL_AE_STATE);
	if (entry.count > 0) frameAeState = entry.data.u8[0];
	}
	String8 result =
	String8::format(" %zu: b: %" PRId64 "\tf: %" PRId64 ", AE state: %d", i,
	bufferTimestamp, frameTimestamp, frameAeState);
	ALOGV("%s", result.string());
	if (fd != -1) {
	result = indent + result + "\n";
	write(fd, result.string(), result.size());
	}

	}
	}

	bool ZslProcessor::isFixedFocusMode(uint8_t afMode) const {
	switch (afMode) {
	case ANDROID_CONTROL_AF_MODE_AUTO:
	case ANDROID_CONTROL_AF_MODE_CONTINUOUS_VIDEO:
	case ANDROID_CONTROL_AF_MODE_CONTINUOUS_PICTURE:
	case ANDROID_CONTROL_AF_MODE_MACRO:
	return false;
	break;
	case ANDROID_CONTROL_AF_MODE_OFF:
	case ANDROID_CONTROL_AF_MODE_EDOF:
	return true;
	default:
	ALOGE("%s: unknown focus mode %d", __FUNCTION__, afMode);
	return false;
	}
	}

	nsecs_t ZslProcessor::getCandidateTimestampLocked(size_t* metadataIdx) const {
	/**
	* Find the smallest timestamp we know about so far
	* - ensure that aeState is either converged or locked
	*/

	size_t idx = 0;
	nsecs_t minTimestamp = -1;

	size_t emptyCount = mFrameList.size();

	for (size_t j = 0; j < mFrameList.size(); j++) {
	const CameraMetadata &frame = mFrameList[j];
	if (!frame.isEmpty()) {

	emptyCount--;

	camera_metadata_ro_entry_t entry;
	entry = frame.find(ANDROID_SENSOR_TIMESTAMP);
	if (entry.count == 0) {
	ALOGE("%s: Can't find timestamp in frame!",
	__FUNCTION__);
	continue;
	}
	nsecs_t frameTimestamp = entry.data.i64[0];
	if (minTimestamp > frameTimestamp \|\| minTimestamp == -1) {

	entry = frame.find(ANDROID_CONTROL_AE_STATE);

	if (entry.count == 0) {
	/**
	* This is most likely a HAL bug. The aeState field is
	* mandatory, so it should always be in a metadata packet.
	*/
	ALOGW("%s: ZSL queue frame has no AE state field!",
	__FUNCTION__);
	continue;
	}
	if (entry.data.u8[0] != ANDROID_CONTROL_AE_STATE_CONVERGED &&
	entry.data.u8[0] != ANDROID_CONTROL_AE_STATE_LOCKED) {
	ALOGVV("%s: ZSL queue frame AE state is %d, need "
	"full capture", __FUNCTION__, entry.data.u8[0]);
	continue;
	}

	entry = frame.find(ANDROID_CONTROL_AF_MODE);
	if (entry.count == 0) {
	ALOGW("%s: ZSL queue frame has no AF mode field!",
	__FUNCTION__);
	continue;
	}
	uint8_t afMode = entry.data.u8[0];
	if (afMode == ANDROID_CONTROL_AF_MODE_OFF) {
	// Skip all the ZSL buffer for manual AF mode, as we don't really
	// know the af state.
	continue;
	}

	// Check AF state if device has focuser and focus mode isn't fixed
	if (mHasFocuser && !isFixedFocusMode(afMode)) {
	// Make sure the candidate frame has good focus.
	entry = frame.find(ANDROID_CONTROL_AF_STATE);
	if (entry.count == 0) {
	ALOGW("%s: ZSL queue frame has no AF state field!",
	__FUNCTION__);
	continue;
	}
	uint8_t afState = entry.data.u8[0];
	if (afState != ANDROID_CONTROL_AF_STATE_PASSIVE_FOCUSED &&
	afState != ANDROID_CONTROL_AF_STATE_FOCUSED_LOCKED &&
	afState != ANDROID_CONTROL_AF_STATE_NOT_FOCUSED_LOCKED) {
	ALOGVV("%s: ZSL queue frame AF state is %d is not good for capture, skip it",
	__FUNCTION__, afState);
	continue;
	}
	}

	minTimestamp = frameTimestamp;
	idx = j;
	}

	ALOGVV("%s: Saw timestamp %" PRId64, __FUNCTION__, frameTimestamp);
	}
	}

	if (emptyCount == mFrameList.size()) {
	/**
	* This could be mildly bad and means our ZSL was triggered before
	* there were any frames yet received by the camera framework.
	*
	* This is a fairly corner case which can happen under:
	* + a user presses the shutter button real fast when the camera starts
	* (startPreview followed immediately by takePicture).
	* + burst capture case (hitting shutter button as fast possible)
	*
	* If this happens in steady case (preview running for a while, call
	* a single takePicture) then this might be a fwk bug.
	*/
	ALOGW("%s: ZSL queue has no metadata frames", __FUNCTION__);
	}

	ALOGV("%s: Candidate timestamp %" PRId64 " (idx %zu), empty frames: %zu",
	__FUNCTION__, minTimestamp, idx, emptyCount);

	if (metadataIdx) {
	*metadataIdx = idx;
	}

	return minTimestamp;
	}

	void ZslProcessor::onBufferAcquired(const BufferInfo& /bufferInfo/) {
	// Intentionally left empty
	// Although theoretically we could use this to get better dump info
	}

	void ZslProcessor::onBufferReleased(const BufferInfo& bufferInfo) {

	// ignore output buffers
	if (bufferInfo.mOutput) {
	return;
	}

	// Lock mutex only once we know this is an input buffer returned to avoid
	// potential deadlock
	Mutex::Autolock l(mInputMutex);
	// TODO: Verify that the buffer is in our queue by looking at timestamp
	// theoretically unnecessary unless we change the following assumptions:
	// -- only 1 buffer reprocessed at a time (which is the case now)

	// Erase entire ZSL queue since we've now completed the capture and preview
	// is stopped.
	//
	// We need to guarantee that if we do two back-to-back captures,
	// the second won't use a buffer that's older/the same as the first, which
	// is theoretically possible if we don't clear out the queue and the
	// selection criteria is something like 'newest'. Clearing out the result
	// metadata queue on a completed capture ensures we'll only use new timestamp.
	// Calling clearZslQueueLocked is a guaranteed deadlock because this callback
	// holds the Camera3Stream internal lock (mLock), and clearZslQueueLocked requires
	// to hold the same lock.
	// TODO: need figure out a way to clear the Zsl buffer queue properly. Right now
	// it is safe not to do so, as back to back ZSL capture requires stop and start
	// preview, which will flush ZSL queue automatically.
	ALOGV("%s: Memory optimization, clearing ZSL queue",
	__FUNCTION__);
	clearZslResultQueueLocked();

	// Required so we accept more ZSL requests
	mState = RUNNING;
	}

	}; // namespace camera2
	}; // namespace android