Added VP9FrameBufferPool, a memory pool that is shared between libvpx and webrtc. Using the VP9 codec, the libvpx decoder will obtain its buffers from our memory pool. This lets us reuse the same buffers for our I420VideoFrames and not have to copy a frame for every decode (from libvpx buffers to webrtc/I420VideoFrame buffers).
(This is similar to chromium's MemoryPool in vpx_video_decoder.cc.)
BUG=1128
R=kjellander@webrtc.org, magjed@webrtc.org, stefan@webrtc.org
Review URL: https://webrtc-codereview.appspot.com/48149004
Cr-Commit-Position: refs/heads/master@{#9141}
diff --git a/DEPS b/DEPS
index 7b3d029..1e56c87 100644
--- a/DEPS
+++ b/DEPS
@@ -42,6 +42,7 @@
'+third_party',
'+unicode',
'+webrtc',
+ '+vpx',
]
# checkdeps.py shouldn't check include paths for files in these dirs:
diff --git a/webrtc/modules/video_coding/BUILD.gn b/webrtc/modules/video_coding/BUILD.gn
index bf04adf..405a173 100644
--- a/webrtc/modules/video_coding/BUILD.gn
+++ b/webrtc/modules/video_coding/BUILD.gn
@@ -182,6 +182,8 @@
if (rtc_build_vp9) {
sources = [
"codecs/vp9/include/vp9.h",
+ "codecs/vp9/vp9_frame_buffer_pool.cc",
+ "codecs/vp9/vp9_frame_buffer_pool.h",
"codecs/vp9/vp9_impl.cc",
"codecs/vp9/vp9_impl.h",
]
diff --git a/webrtc/modules/video_coding/codecs/vp9/vp9.gyp b/webrtc/modules/video_coding/codecs/vp9/vp9.gyp
index 795db62..ac7e67a 100644
--- a/webrtc/modules/video_coding/codecs/vp9/vp9.gyp
+++ b/webrtc/modules/video_coding/codecs/vp9/vp9.gyp
@@ -28,6 +28,8 @@
['build_vp9==1', {
'sources': [
'include/vp9.h',
+ 'vp9_frame_buffer_pool.cc',
+ 'vp9_frame_buffer_pool.h',
'vp9_impl.cc',
'vp9_impl.h',
],
diff --git a/webrtc/modules/video_coding/codecs/vp9/vp9_frame_buffer_pool.cc b/webrtc/modules/video_coding/codecs/vp9/vp9_frame_buffer_pool.cc
new file mode 100644
index 0000000..6e16bc1
--- /dev/null
+++ b/webrtc/modules/video_coding/codecs/vp9/vp9_frame_buffer_pool.cc
@@ -0,0 +1,136 @@
+/*
+ * Copyright (c) 2015 The WebRTC project authors. All Rights Reserved.
+ *
+ * Use of this source code is governed by a BSD-style license
+ * that can be found in the LICENSE file in the root of the source
+ * tree. An additional intellectual property rights grant can be found
+ * in the file PATENTS. All contributing project authors may
+ * be found in the AUTHORS file in the root of the source tree.
+ *
+ */
+
+#include "webrtc/modules/video_coding/codecs/vp9/vp9_frame_buffer_pool.h"
+
+#include "vpx/vpx_codec.h"
+#include "vpx/vpx_decoder.h"
+#include "vpx/vpx_frame_buffer.h"
+
+#include "webrtc/base/checks.h"
+#include "webrtc/system_wrappers/interface/logging.h"
+
+namespace webrtc {
+
+uint8_t* Vp9FrameBufferPool::Vp9FrameBuffer::GetData() {
+ return data_.data<uint8_t>();
+}
+
+size_t Vp9FrameBufferPool::Vp9FrameBuffer::GetDataSize() const {
+ return data_.size();
+}
+
+void Vp9FrameBufferPool::Vp9FrameBuffer::SetSize(size_t size) {
+ data_.SetSize(size);
+}
+
+bool Vp9FrameBufferPool::InitializeVpxUsePool(
+ vpx_codec_ctx* vpx_codec_context) {
+ DCHECK(vpx_codec_context);
+ // Tell libvpx to use this pool.
+ if (vpx_codec_set_frame_buffer_functions(
+ // In which context to use these callback functions.
+ vpx_codec_context,
+ // Called by libvpx when it needs another frame buffer.
+ &Vp9FrameBufferPool::VpxGetFrameBuffer,
+ // Called by libvpx when it no longer uses a frame buffer.
+ &Vp9FrameBufferPool::VpxReleaseFrameBuffer,
+ // |this| will be passed as |user_priv| to VpxGetFrameBuffer.
+ this)) {
+ // Failed to configure libvpx to use Vp9FrameBufferPool.
+ return false;
+ }
+ return true;
+}
+
+rtc::scoped_refptr<Vp9FrameBufferPool::Vp9FrameBuffer>
+Vp9FrameBufferPool::GetFrameBuffer(size_t min_size) {
+ DCHECK_GT(min_size, 0u);
+ rtc::scoped_refptr<Vp9FrameBuffer> available_buffer = nullptr;
+ {
+ rtc::CritScope cs(&buffers_lock_);
+ // Do we have a buffer we can recycle?
+ for (const auto& buffer : allocated_buffers_) {
+ if (buffer->HasOneRef()) {
+ available_buffer = buffer;
+ break;
+ }
+ }
+ // Otherwise create one.
+ if (available_buffer == nullptr) {
+ available_buffer = new rtc::RefCountedObject<Vp9FrameBuffer>();
+ allocated_buffers_.push_back(available_buffer);
+ if (allocated_buffers_.size() > max_num_buffers_) {
+ LOG(LS_WARNING)
+ << allocated_buffers_.size() << " Vp9FrameBuffers have been "
+ << "allocated by a Vp9FrameBufferPool (exceeding what is "
+ << "considered reasonable, " << max_num_buffers_ << ").";
+ RTC_NOTREACHED();
+ }
+ }
+ }
+
+ available_buffer->SetSize(min_size);
+ return available_buffer;
+}
+
+int Vp9FrameBufferPool::GetNumBuffersInUse() const {
+ int num_buffers_in_use = 0;
+ rtc::CritScope cs(&buffers_lock_);
+ for (const auto& buffer : allocated_buffers_) {
+ if (!buffer->HasOneRef())
+ ++num_buffers_in_use;
+ }
+ return num_buffers_in_use;
+}
+
+void Vp9FrameBufferPool::ClearPool() {
+ rtc::CritScope cs(&buffers_lock_);
+ allocated_buffers_.clear();
+}
+
+// static
+int32 Vp9FrameBufferPool::VpxGetFrameBuffer(void* user_priv,
+ size_t min_size,
+ vpx_codec_frame_buffer* fb) {
+ DCHECK(user_priv);
+ DCHECK(fb);
+ Vp9FrameBufferPool* pool = static_cast<Vp9FrameBufferPool*>(user_priv);
+
+ rtc::scoped_refptr<Vp9FrameBuffer> buffer = pool->GetFrameBuffer(min_size);
+ fb->data = buffer->GetData();
+ fb->size = buffer->GetDataSize();
+ // Store Vp9FrameBuffer* in |priv| for use in VpxReleaseFrameBuffer.
+ // This also makes vpx_codec_get_frame return images with their |fb_priv| set
+ // to |buffer| which is important for external reference counting.
+ // Release from refptr so that the buffer's |ref_count_| remains 1 when
+ // |buffer| goes out of scope.
+ fb->priv = static_cast<void*>(buffer.release());
+ return 0;
+}
+
+// static
+int32 Vp9FrameBufferPool::VpxReleaseFrameBuffer(void* user_priv,
+ vpx_codec_frame_buffer* fb) {
+ DCHECK(user_priv);
+ DCHECK(fb);
+ Vp9FrameBuffer* buffer = static_cast<Vp9FrameBuffer*>(fb->priv);
+ if (buffer != nullptr) {
+ buffer->Release();
+ // When libvpx fails to decode and you continue to try to decode (and fail)
+ // libvpx can for some reason try to release the same buffer multiple times.
+ // Setting |priv| to null protects against trying to Release multiple times.
+ fb->priv = nullptr;
+ }
+ return 0;
+}
+
+} // namespace webrtc
diff --git a/webrtc/modules/video_coding/codecs/vp9/vp9_frame_buffer_pool.h b/webrtc/modules/video_coding/codecs/vp9/vp9_frame_buffer_pool.h
new file mode 100644
index 0000000..68ebbee
--- /dev/null
+++ b/webrtc/modules/video_coding/codecs/vp9/vp9_frame_buffer_pool.h
@@ -0,0 +1,117 @@
+/*
+ * Copyright (c) 2015 The WebRTC project authors. All Rights Reserved.
+ *
+ * Use of this source code is governed by a BSD-style license
+ * that can be found in the LICENSE file in the root of the source
+ * tree. An additional intellectual property rights grant can be found
+ * in the file PATENTS. All contributing project authors may
+ * be found in the AUTHORS file in the root of the source tree.
+ *
+ */
+
+#ifndef WEBRTC_MODULES_VIDEO_CODING_CODECS_VP9_FRAME_BUFFER_POOL_H_
+#define WEBRTC_MODULES_VIDEO_CODING_CODECS_VP9_FRAME_BUFFER_POOL_H_
+
+#include <vector>
+
+#include "webrtc/base/basictypes.h"
+#include "webrtc/base/buffer.h"
+#include "webrtc/base/criticalsection.h"
+#include "webrtc/base/refcount.h"
+#include "webrtc/base/scoped_ref_ptr.h"
+
+struct vpx_codec_ctx;
+struct vpx_codec_frame_buffer;
+
+namespace webrtc {
+
+// This memory pool is used to serve buffers to libvpx for decoding purposes in
+// VP9, which is set up in InitializeVPXUsePool. After the initialization any
+// time libvpx wants to decode a frame it will use buffers provided and released
+// through VpxGetFrameBuffer and VpxReleaseFrameBuffer.
+// The benefit of owning the pool that libvpx relies on for decoding is that the
+// decoded frames returned by libvpx (from vpx_codec_get_frame) use parts of our
+// buffers for the decoded image data. By retaining ownership of this buffer
+// using scoped_refptr, the image buffer can be reused by I420VideoFrames and no
+// frame copy has to occur during decoding and frame delivery.
+//
+// Pseudo example usage case:
+// Vp9FrameBufferPool pool;
+// pool.InitializeVpxUsePool(decoder_ctx);
+// ...
+//
+// // During decoding, libvpx will get and release buffers from the pool.
+// vpx_codec_decode(decoder_ctx, ...);
+//
+// vpx_image_t* img = vpx_codec_get_frame(decoder_ctx, &iter);
+// // Important to use scoped_refptr to protect it against being recycled by
+// // the pool.
+// scoped_refptr<Vp9FrameBuffer> img_buffer = (Vp9FrameBuffer*)img->fb_priv;
+// ...
+//
+// // Destroying the codec will make libvpx release any buffers it was using.
+// vpx_codec_destroy(decoder_ctx);
+class Vp9FrameBufferPool {
+ public:
+ class Vp9FrameBuffer : public rtc::RefCountInterface {
+ public:
+ uint8_t* GetData();
+ size_t GetDataSize() const;
+ void SetSize(size_t size);
+
+ virtual bool HasOneRef() const = 0;
+
+ private:
+ // Data as an easily resizable buffer.
+ rtc::Buffer data_;
+ };
+
+ // Configures libvpx to, in the specified context, use this memory pool for
+ // buffers used to decompress frames. This is only supported for VP9.
+ bool InitializeVpxUsePool(vpx_codec_ctx* vpx_codec_context);
+
+ // Gets a frame buffer of at least |min_size|, recycling an available one or
+ // creating a new one. When no longer referenced from the outside the buffer
+ // becomes recyclable.
+ rtc::scoped_refptr<Vp9FrameBuffer> GetFrameBuffer(size_t min_size);
+ // Gets the number of buffers currently in use (not ready to be recycled).
+ int GetNumBuffersInUse() const;
+ // Releases allocated buffers, deleting available buffers. Buffers in use are
+ // not deleted until they are no longer referenced.
+ void ClearPool();
+
+ // InitializeVpxUsePool configures libvpx to call this function when it needs
+ // a new frame buffer. Parameters:
+ // |user_priv| Private data passed to libvpx, InitializeVpxUsePool sets it up
+ // to be a pointer to the pool.
+ // |min_size| Minimum size needed by libvpx (to decompress a frame).
+ // |fb| Pointer to the libvpx frame buffer object, this is updated to
+ // use the pool's buffer.
+ // Returns 0 on success. Returns < 0 on failure.
+ static int32 VpxGetFrameBuffer(void* user_priv,
+ size_t min_size,
+ vpx_codec_frame_buffer* fb);
+
+ // InitializeVpxUsePool configures libvpx to call this function when it has
+ // finished using one of the pool's frame buffer. Parameters:
+ // |user_priv| Private data passed to libvpx, InitializeVpxUsePool sets it up
+ // to be a pointer to the pool.
+ // |fb| Pointer to the libvpx frame buffer object, its |priv| will be
+ // a pointer to one of the pool's Vp9FrameBuffer.
+ static int32 VpxReleaseFrameBuffer(void* user_priv,
+ vpx_codec_frame_buffer* fb);
+
+ private:
+ // Protects |allocated_buffers_|.
+ mutable rtc::CriticalSection buffers_lock_;
+ // All buffers, in use or ready to be recycled.
+ std::vector<rtc::scoped_refptr<Vp9FrameBuffer>> allocated_buffers_
+ GUARDED_BY(buffers_lock_);
+ // If more buffers than this are allocated we print warnings, and crash if
+ // in debug mode.
+ static const size_t max_num_buffers_ = 10;
+};
+
+} // namespace webrtc
+
+#endif // WEBRTC_MODULES_VIDEO_CODING_CODECS_VP9_FRAME_BUFFER_POOL_H_
diff --git a/webrtc/modules/video_coding/codecs/vp9/vp9_impl.cc b/webrtc/modules/video_coding/codecs/vp9/vp9_impl.cc
index 310e53a..02f91ec 100644
--- a/webrtc/modules/video_coding/codecs/vp9/vp9_impl.cc
+++ b/webrtc/modules/video_coding/codecs/vp9/vp9_impl.cc
@@ -21,13 +21,25 @@
#include "vpx/vp8cx.h"
#include "vpx/vp8dx.h"
+#include "webrtc/base/bind.h"
#include "webrtc/base/checks.h"
#include "webrtc/common.h"
#include "webrtc/common_video/libyuv/include/webrtc_libyuv.h"
#include "webrtc/modules/interface/module_common_types.h"
+#include "webrtc/system_wrappers/interface/logging.h"
#include "webrtc/system_wrappers/interface/tick_util.h"
#include "webrtc/system_wrappers/interface/trace_event.h"
+namespace {
+
+// VP9DecoderImpl::ReturnFrame helper function used with WrappedI420Buffer.
+static void WrappedI420BufferNoLongerUsedCb(
+ webrtc::Vp9FrameBufferPool::Vp9FrameBuffer* img_buffer) {
+ img_buffer->Release();
+}
+
+} // anonymous namespace
+
namespace webrtc {
VP9Encoder* VP9Encoder::Create() {
@@ -388,6 +400,14 @@
VP9DecoderImpl::~VP9DecoderImpl() {
inited_ = true; // in order to do the actual release
Release();
+ int num_buffers_in_use = frame_buffer_pool_.GetNumBuffersInUse();
+ if (num_buffers_in_use > 0) {
+ // The frame buffers are reference counted and frames are exposed after
+ // decoding. There may be valid usage cases where previous frames are still
+ // referenced after ~VP9DecoderImpl that is not a leak.
+ LOG(LS_INFO) << num_buffers_in_use << " Vp9FrameBuffers are still "
+ << "referenced during ~VP9DecoderImpl.";
+ }
}
int VP9DecoderImpl::Reset() {
@@ -421,6 +441,11 @@
// Save VideoCodec instance for later; mainly for duplicating the decoder.
codec_ = *inst;
}
+
+ if (!frame_buffer_pool_.InitializeVpxUsePool(decoder_)) {
+ return WEBRTC_VIDEO_CODEC_MEMORY;
+ }
+
inited_ = true;
// Always start with a complete key frame.
key_frame_required_ = true;
@@ -455,6 +480,8 @@
if (input_image._length == 0) {
buffer = NULL; // Triggers full frame concealment.
}
+ // During decode libvpx may get and release buffers from |frame_buffer_pool_|.
+ // In practice libvpx keeps a few (~3-4) buffers alive at a time.
if (vpx_codec_decode(decoder_,
buffer,
static_cast<unsigned int>(input_image._length),
@@ -462,6 +489,9 @@
VPX_DL_REALTIME)) {
return WEBRTC_VIDEO_CODEC_ERROR;
}
+ // |img->fb_priv| contains the image data, a reference counted Vp9FrameBuffer.
+ // It may be released by libvpx during future vpx_codec_decode or
+ // vpx_codec_destroy calls.
img = vpx_codec_get_frame(decoder_, &iter);
int ret = ReturnFrame(img, input_image._timeStamp);
if (ret != 0) {
@@ -475,15 +505,32 @@
// Decoder OK and NULL image => No show frame.
return WEBRTC_VIDEO_CODEC_NO_OUTPUT;
}
- decoded_image_.CreateFrame(img->planes[VPX_PLANE_Y],
- img->planes[VPX_PLANE_U],
- img->planes[VPX_PLANE_V],
- img->d_w, img->d_h,
- img->stride[VPX_PLANE_Y],
- img->stride[VPX_PLANE_U],
- img->stride[VPX_PLANE_V]);
- decoded_image_.set_timestamp(timestamp);
- int ret = decode_complete_callback_->Decoded(decoded_image_);
+
+ // This buffer contains all of |img|'s image data, a reference counted
+ // Vp9FrameBuffer. Performing AddRef/Release ensures it is not released and
+ // recycled during use (libvpx is done with the buffers after a few
+ // vpx_codec_decode calls or vpx_codec_destroy).
+ Vp9FrameBufferPool::Vp9FrameBuffer* img_buffer =
+ static_cast<Vp9FrameBufferPool::Vp9FrameBuffer*>(img->fb_priv);
+ img_buffer->AddRef();
+ // The buffer can be used directly by the I420VideoFrame (without copy) by
+ // using a WrappedI420Buffer.
+ rtc::scoped_refptr<WrappedI420Buffer> img_wrapped_buffer(
+ new rtc::RefCountedObject<webrtc::WrappedI420Buffer>(
+ img->d_w, img->d_h,
+ img->d_w, img->d_h,
+ img->planes[VPX_PLANE_Y], img->stride[VPX_PLANE_Y],
+ img->planes[VPX_PLANE_U], img->stride[VPX_PLANE_U],
+ img->planes[VPX_PLANE_V], img->stride[VPX_PLANE_V],
+ // WrappedI420Buffer's mechanism for allowing the release of its frame
+ // buffer is through a callback function. This is where we should
+ // release |img_buffer|.
+ rtc::Bind(&WrappedI420BufferNoLongerUsedCb, img_buffer)));
+
+ I420VideoFrame decoded_image;
+ decoded_image.set_video_frame_buffer(img_wrapped_buffer);
+ decoded_image.set_timestamp(timestamp);
+ int ret = decode_complete_callback_->Decoded(decoded_image);
if (ret != 0)
return ret;
return WEBRTC_VIDEO_CODEC_OK;
@@ -497,12 +544,18 @@
int VP9DecoderImpl::Release() {
if (decoder_ != NULL) {
+ // When a codec is destroyed libvpx will release any buffers of
+ // |frame_buffer_pool_| it is currently using.
if (vpx_codec_destroy(decoder_)) {
return WEBRTC_VIDEO_CODEC_MEMORY;
}
delete decoder_;
decoder_ = NULL;
}
+ // Releases buffers from the pool. Any buffers not in use are deleted. Buffers
+ // still referenced externally are deleted once fully released, not returning
+ // to the pool.
+ frame_buffer_pool_.ClearPool();
inited_ = false;
return WEBRTC_VIDEO_CODEC_OK;
}
diff --git a/webrtc/modules/video_coding/codecs/vp9/vp9_impl.h b/webrtc/modules/video_coding/codecs/vp9/vp9_impl.h
index 28b9ecc..bd7c7f5 100644
--- a/webrtc/modules/video_coding/codecs/vp9/vp9_impl.h
+++ b/webrtc/modules/video_coding/codecs/vp9/vp9_impl.h
@@ -13,6 +13,7 @@
#define WEBRTC_MODULES_VIDEO_CODING_CODECS_VP9_IMPL_H_
#include "webrtc/modules/video_coding/codecs/vp9/include/vp9.h"
+#include "webrtc/modules/video_coding/codecs/vp9/vp9_frame_buffer_pool.h"
#include "vpx/vpx_decoder.h"
#include "vpx/vpx_encoder.h"
@@ -101,7 +102,8 @@
private:
int ReturnFrame(const vpx_image_t* img, uint32_t timeStamp);
- I420VideoFrame decoded_image_;
+ // Memory pool used to share buffers between libvpx and webrtc.
+ Vp9FrameBufferPool frame_buffer_pool_;
DecodedImageCallback* decode_complete_callback_;
bool inited_;
vpx_codec_ctx_t* decoder_;
