blob: 65791a9cb3677348806f381c0ccaa1adca44d6bb [file] [log] [blame]
/*
* Copyright (C) 2013-2018 The Android Open Source Project
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#ifndef ANDROID_SERVERS_CAMERA3_OUTPUT_STREAM_H
#define ANDROID_SERVERS_CAMERA3_OUTPUT_STREAM_H
#include <mutex>
#include <utils/RefBase.h>
#include <gui/IProducerListener.h>
#include <gui/Surface.h>
#include <gui/DisplayEventReceiver.h>
#include "utils/IPCTransport.h"
#include "utils/LatencyHistogram.h"
#include "Camera3Stream.h"
#include "Camera3IOStreamBase.h"
#include "Camera3OutputStreamInterface.h"
#include "Camera3BufferManager.h"
#include "PreviewFrameSpacer.h"
namespace android {
namespace camera3 {
class Camera3BufferManager;
/**
* Stream info structure that holds the necessary stream info for buffer manager to use for
* buffer allocation and management.
*/
struct StreamInfo {
int streamId;
int streamSetId;
uint32_t width;
uint32_t height;
uint32_t format;
android_dataspace dataSpace;
uint64_t combinedUsage;
size_t totalBufferCount;
bool isConfigured;
bool isMultiRes;
explicit StreamInfo(int id = CAMERA3_STREAM_ID_INVALID,
int setId = CAMERA3_STREAM_SET_ID_INVALID,
uint32_t w = 0,
uint32_t h = 0,
uint32_t fmt = 0,
android_dataspace ds = HAL_DATASPACE_UNKNOWN,
uint64_t usage = 0,
size_t bufferCount = 0,
bool configured = false,
bool multiRes = false) :
streamId(id),
streamSetId(setId),
width(w),
height(h),
format(fmt),
dataSpace(ds),
combinedUsage(usage),
totalBufferCount(bufferCount),
isConfigured(configured),
isMultiRes(multiRes) {}
};
/**
* A class for managing a single stream of output data from the camera device.
*/
class Camera3OutputStream :
public Camera3IOStreamBase,
public Camera3OutputStreamInterface {
public:
/**
* Set up a stream for formats that have 2 dimensions, such as RAW and YUV.
* A valid stream set id needs to be set to support buffer sharing between multiple
* streams.
*/
Camera3OutputStream(int id, sp<Surface> consumer,
uint32_t width, uint32_t height, int format,
android_dataspace dataSpace, camera_stream_rotation_t rotation,
nsecs_t timestampOffset, const std::string& physicalCameraId,
const std::unordered_set<int32_t> &sensorPixelModesUsed, IPCTransport transport,
int setId = CAMERA3_STREAM_SET_ID_INVALID, bool isMultiResolution = false,
int64_t dynamicProfile = ANDROID_REQUEST_AVAILABLE_DYNAMIC_RANGE_PROFILES_MAP_STANDARD,
int64_t streamUseCase = ANDROID_SCALER_AVAILABLE_STREAM_USE_CASES_DEFAULT,
bool deviceTimeBaseIsRealtime = false,
int timestampBase = OutputConfiguration::TIMESTAMP_BASE_DEFAULT,
int mirrorMode = OutputConfiguration::MIRROR_MODE_AUTO,
int32_t colorSpace = ANDROID_REQUEST_AVAILABLE_COLOR_SPACE_PROFILES_MAP_UNSPECIFIED,
bool useReadoutTimestamp = false);
/**
* Set up a stream for formats that have a variable buffer size for the same
* dimensions, such as compressed JPEG.
* A valid stream set id needs to be set to support buffer sharing between multiple
* streams.
*/
Camera3OutputStream(int id, sp<Surface> consumer,
uint32_t width, uint32_t height, size_t maxSize, int format,
android_dataspace dataSpace, camera_stream_rotation_t rotation,
nsecs_t timestampOffset, const std::string& physicalCameraId,
const std::unordered_set<int32_t> &sensorPixelModesUsed, IPCTransport transport,
int setId = CAMERA3_STREAM_SET_ID_INVALID, bool isMultiResolution = false,
int64_t dynamicProfile = ANDROID_REQUEST_AVAILABLE_DYNAMIC_RANGE_PROFILES_MAP_STANDARD,
int64_t streamUseCase = ANDROID_SCALER_AVAILABLE_STREAM_USE_CASES_DEFAULT,
bool deviceTimeBaseIsRealtime = false,
int timestampBase = OutputConfiguration::TIMESTAMP_BASE_DEFAULT,
int mirrorMode = OutputConfiguration::MIRROR_MODE_AUTO,
int32_t colorSpace = ANDROID_REQUEST_AVAILABLE_COLOR_SPACE_PROFILES_MAP_UNSPECIFIED,
bool useReadoutTimestamp = false);
/**
* Set up a stream with deferred consumer for formats that have 2 dimensions, such as
* RAW and YUV. The consumer must be set before using this stream for output. A valid
* stream set id needs to be set to support buffer sharing between multiple streams.
*/
Camera3OutputStream(int id, uint32_t width, uint32_t height, int format,
uint64_t consumerUsage, android_dataspace dataSpace,
camera_stream_rotation_t rotation, nsecs_t timestampOffset,
const std::string& physicalCameraId,
const std::unordered_set<int32_t> &sensorPixelModesUsed, IPCTransport transport,
int setId = CAMERA3_STREAM_SET_ID_INVALID, bool isMultiResolution = false,
int64_t dynamicProfile = ANDROID_REQUEST_AVAILABLE_DYNAMIC_RANGE_PROFILES_MAP_STANDARD,
int64_t streamUseCase = ANDROID_SCALER_AVAILABLE_STREAM_USE_CASES_DEFAULT,
bool deviceTimeBaseIsRealtime = false,
int timestampBase = OutputConfiguration::TIMESTAMP_BASE_DEFAULT,
int mirrorMode = OutputConfiguration::MIRROR_MODE_AUTO,
int32_t colorSpace = ANDROID_REQUEST_AVAILABLE_COLOR_SPACE_PROFILES_MAP_UNSPECIFIED,
bool useReadoutTimestamp = false);
virtual ~Camera3OutputStream();
/**
* Camera3Stream interface
*/
virtual void dump(int fd, const Vector<String16> &args) const;
/**
* Set the transform on the output stream; one of the
* HAL_TRANSFORM_* / NATIVE_WINDOW_TRANSFORM_* constants.
*/
status_t setTransform(int transform, bool mayChangeMirror);
/**
* Return if this output stream is for video encoding.
*/
bool isVideoStream() const;
/**
* Return if this output stream is consumed by hardware composer.
*/
bool isConsumedByHWComposer() const;
/**
* Return if this output stream is consumed by hardware texture.
*/
bool isConsumedByHWTexture() const;
/**
* Return if this output stream is consumed by CPU.
*/
bool isConsumedByCPU() const;
/**
* Return if the consumer configuration of this stream is deferred.
*/
virtual bool isConsumerConfigurationDeferred(size_t surface_id) const;
/**
* Set the consumer surfaces to the output stream.
*/
virtual status_t setConsumers(const std::vector<sp<Surface>>& consumers);
class BufferProducerListener : public SurfaceListener {
public:
BufferProducerListener(wp<Camera3OutputStream> parent, bool needsReleaseNotify)
: mParent(parent), mNeedsReleaseNotify(needsReleaseNotify) {}
/**
* Implementation of IProducerListener, used to notify this stream that the consumer
* has returned a buffer and it is ready to return to Camera3BufferManager for reuse.
*/
virtual void onBufferReleased();
virtual bool needsReleaseNotify() { return mNeedsReleaseNotify; }
virtual void onBuffersDiscarded(const std::vector<sp<GraphicBuffer>>& buffers);
private:
wp<Camera3OutputStream> mParent;
bool mNeedsReleaseNotify;
};
virtual status_t detachBuffer(sp<GraphicBuffer>* buffer, int* fenceFd);
/**
* Notify that the buffer is being released to the buffer queue instead of
* being queued to the consumer.
*/
virtual status_t notifyBufferReleased(ANativeWindowBuffer *anwBuffer);
/**
* Drop buffers if dropping is true. If dropping is false, do not drop buffers.
*/
virtual status_t dropBuffers(bool dropping) override;
/**
* Query the physical camera id for the output stream.
*/
virtual const std::string& getPhysicalCameraId() const override;
/**
* Set the graphic buffer manager to get/return the stream buffers.
*
* It is only legal to call this method when stream is in STATE_CONSTRUCTED state.
*/
status_t setBufferManager(sp<Camera3BufferManager> bufferManager);
/**
* Query the ouput surface id.
*/
virtual ssize_t getSurfaceId(const sp<Surface> &/*surface*/) { return 0; }
virtual status_t getUniqueSurfaceIds(const std::vector<size_t>&,
/*out*/std::vector<size_t>*) { return INVALID_OPERATION; };
/**
* Update the stream output surfaces.
*/
virtual status_t updateStream(const std::vector<sp<Surface>> &outputSurfaces,
const std::vector<OutputStreamInfo> &outputInfo,
const std::vector<size_t> &removedSurfaceIds,
KeyedVector<sp<Surface>, size_t> *outputMap/*out*/);
/**
* Set the batch size for buffer operations. The output stream will request
* buffers from buffer queue on a batch basis. Currently only video streams
* are allowed to set the batch size. Also if the stream is managed by
* buffer manager (Surface group in Java API) then batching is also not
* supported. Changing batch size on the fly while there is already batched
* buffers in the stream is also not supported.
* If the batch size is larger than the max dequeue count set
* by the camera HAL, the batch size will be set to the max dequeue count
* instead.
*/
virtual status_t setBatchSize(size_t batchSize = 1) override;
/**
* Notify the stream on change of min frame durations or variable/fixed
* frame rate.
*/
virtual void onMinDurationChanged(nsecs_t duration, bool fixedFps) override;
/**
* Modify stream use case
*/
virtual void setStreamUseCase(int64_t streamUseCase) override;
/**
* Apply ZSL related consumer usage quirk.
*/
static void applyZSLUsageQuirk(int format, uint64_t *consumerUsage /*inout*/);
void setImageDumpMask(int mask) { mImageDumpMask = mask; }
bool shouldLogError(status_t res);
void onCachedBufferQueued();
protected:
Camera3OutputStream(int id, camera_stream_type_t type,
uint32_t width, uint32_t height, int format,
android_dataspace dataSpace, camera_stream_rotation_t rotation,
const std::string& physicalCameraId,
const std::unordered_set<int32_t> &sensorPixelModesUsed, IPCTransport transport,
uint64_t consumerUsage = 0, nsecs_t timestampOffset = 0,
int setId = CAMERA3_STREAM_SET_ID_INVALID, bool isMultiResolution = false,
int64_t dynamicProfile = ANDROID_REQUEST_AVAILABLE_DYNAMIC_RANGE_PROFILES_MAP_STANDARD,
int64_t streamUseCase = ANDROID_SCALER_AVAILABLE_STREAM_USE_CASES_DEFAULT,
bool deviceTimeBaseIsRealtime = false,
int timestampBase = OutputConfiguration::TIMESTAMP_BASE_DEFAULT,
int mirrorMode = OutputConfiguration::MIRROR_MODE_AUTO,
int32_t colorSpace = ANDROID_REQUEST_AVAILABLE_COLOR_SPACE_PROFILES_MAP_UNSPECIFIED,
bool useReadoutTimestamp = false);
/**
* Note that we release the lock briefly in this function
*/
virtual status_t returnBufferCheckedLocked(
const camera_stream_buffer &buffer,
nsecs_t timestamp,
nsecs_t readoutTimestamp,
bool output,
int32_t transform,
const std::vector<size_t>& surface_ids,
/*out*/
sp<Fence> *releaseFenceOut);
virtual status_t disconnectLocked();
status_t fixUpHidlJpegBlobHeader(ANativeWindowBuffer* anwBuffer, int fence);
status_t getEndpointUsageForSurface(uint64_t *usage,
const sp<Surface>& surface) const;
status_t configureConsumerQueueLocked(bool allowPreviewRespace);
// Consumer as the output of camera HAL
sp<Surface> mConsumer;
uint64_t getPresetConsumerUsage() const { return mConsumerUsage; }
static const nsecs_t kDequeueBufferTimeout = 1000000000; // 1 sec
status_t getBufferLockedCommon(ANativeWindowBuffer** anb, int* fenceFd);
private:
int mTransform;
virtual status_t setTransformLocked(int transform);
bool mTraceFirstBuffer;
// Name of Surface consumer
std::string mConsumerName;
/**
* GraphicBuffer manager this stream is registered to. Used to replace the buffer
* allocation/deallocation role of BufferQueue.
*/
sp<Camera3BufferManager> mBufferManager;
/**
* Buffer producer listener, used to handle notification when a buffer is released
* from consumer side, or a set of buffers are discarded by the consumer.
*/
sp<BufferProducerListener> mBufferProducerListener;
/**
* Flag indicating if the buffer manager is used to allocate the stream buffers
*/
bool mUseBufferManager;
/**
* Offset used to override camera HAL produced timestamps
*
* The offset is first initialized to bootTime - monotonicTime in
* constructor, and may later be updated based on the client's timestampBase
* setting.
*/
nsecs_t mTimestampOffset;
/**
* If camera readout time is used rather than the start-of-exposure time.
*/
bool mUseReadoutTime;
/**
* Consumer end point usage flag set by the constructor for the deferred
* consumer case.
*/
uint64_t mConsumerUsage;
// Whether to drop valid buffers.
bool mDropBuffers;
// The batch size for buffer operation
std::atomic_size_t mBatchSize = 1;
// Protecting batch states below, must be acquired after mLock
std::mutex mBatchLock;
// Prefetched buffers (ready to be handed to client)
std::vector<Surface::BatchBuffer> mBatchedBuffers;
// ---- End of mBatchLock protected scope ----
const int mMirrorMode;
/**
* Internal Camera3Stream interface
*/
virtual status_t getBufferLocked(camera_stream_buffer *buffer,
const std::vector<size_t>& surface_ids);
virtual status_t getBuffersLocked(/*out*/std::vector<OutstandingBuffer>* buffers) override;
virtual status_t returnBufferLocked(
const camera_stream_buffer &buffer,
nsecs_t timestamp, nsecs_t readoutTimestamp,
int32_t transform, const std::vector<size_t>& surface_ids);
virtual status_t queueBufferToConsumer(sp<ANativeWindow>& consumer,
ANativeWindowBuffer* buffer, int anwReleaseFence,
const std::vector<size_t>& surface_ids);
virtual status_t configureQueueLocked();
virtual status_t getEndpointUsage(uint64_t *usage) const;
/**
* Private methods
*/
void onBuffersRemovedLocked(const std::vector<sp<GraphicBuffer>>&);
status_t detachBufferLocked(sp<GraphicBuffer>* buffer, int* fenceFd);
// Call this after each dequeueBuffer/attachBuffer/detachNextBuffer call to get update on
// removed buffers. Set notifyBufferManager to false when the call is initiated by buffer
// manager so buffer manager doesn't need to be notified.
void checkRemovedBuffersLocked(bool notifyBufferManager = true);
// Check return status of IGBP calls and set abandoned state accordingly
void checkRetAndSetAbandonedLocked(status_t res);
// If the status indicates abandonded stream, only log when state hasn't been updated to
// STATE_ABANDONED
static bool shouldLogError(status_t res, StreamState state);
// Dump images to disk before returning to consumer
void dumpImageToDisk(nsecs_t timestamp, ANativeWindowBuffer* anwBuffer, int fence);
void returnPrefetchedBuffersLocked();
static const int32_t kDequeueLatencyBinSize = 5; // in ms
CameraLatencyHistogram mDequeueBufferLatency;
IPCTransport mIPCTransport = IPCTransport::INVALID;
int mImageDumpMask = 0;
// Re-space frames by overriding timestamp to align with display Vsync.
// Default is on for SurfaceView bound streams.
bool mFixedFps = false;
nsecs_t mMinExpectedDuration = 0;
bool mSyncToDisplay = false;
DisplayEventReceiver mDisplayEventReceiver;
nsecs_t mLastCaptureTime = 0;
nsecs_t mLastPresentTime = 0;
nsecs_t mCaptureToPresentOffset = 0;
static constexpr size_t kDisplaySyncExtraBuffer = 2;
static constexpr nsecs_t kSpacingResetIntervalNs = 50000000LL; // 50 millisecond
static constexpr nsecs_t kTimelineThresholdNs = 1000000LL; // 1 millisecond
static constexpr float kMaxIntervalRatioDeviation = 0.05f;
static constexpr int kMaxTimelines = 2;
nsecs_t syncTimestampToDisplayLocked(nsecs_t t, sp<Fence> releaseFence);
// In case of fence being used
sp<Fence> mReferenceFrameFence;
nsecs_t mReferenceCaptureTime = 0;
nsecs_t mReferenceArrivalTime = 0;
nsecs_t mFenceSignalOffset = 0;
VsyncEventData mRefVsyncData;
// Re-space frames by delaying queueBuffer so that frame delivery has
// the same cadence as capture. Default is on for SurfaceTexture bound
// streams.
sp<PreviewFrameSpacer> mPreviewFrameSpacer;
}; // class Camera3OutputStream
} // namespace camera3
} // namespace android
#endif