Add HW fallback option to software encoding.
Permits falling back to software encoding for unsupported resolutions.
BUG=chromium:475116, chromium:487934
R=mflodman@webrtc.org, stefan@webrtc.org
Review URL: https://webrtc-codereview.appspot.com/46279004
Cr-Commit-Position: refs/heads/master@{#9227}
diff --git a/talk/media/webrtc/webrtcvideoengine2.cc b/talk/media/webrtc/webrtcvideoengine2.cc
index a7fc6bd..7d8cc02 100644
--- a/talk/media/webrtc/webrtcvideoengine2.cc
+++ b/talk/media/webrtc/webrtcvideoengine2.cc
@@ -1634,6 +1634,21 @@
codec_settings(codec_settings) {
}
+WebRtcVideoChannel2::WebRtcVideoSendStream::AllocatedEncoder::AllocatedEncoder(
+ webrtc::VideoEncoder* encoder,
+ webrtc::VideoCodecType type,
+ bool external)
+ : encoder(encoder),
+ external_encoder(nullptr),
+ type(type),
+ external(external) {
+ if (external) {
+ external_encoder = encoder;
+ this->encoder =
+ new webrtc::VideoEncoderSoftwareFallbackWrapper(type, encoder);
+ }
+}
+
WebRtcVideoChannel2::WebRtcVideoSendStream::WebRtcVideoSendStream(
webrtc::Call* call,
WebRtcVideoEncoderFactory* external_encoder_factory,
@@ -1885,10 +1900,9 @@
void WebRtcVideoChannel2::WebRtcVideoSendStream::DestroyVideoEncoder(
AllocatedEncoder* encoder) {
if (encoder->external) {
- external_encoder_factory_->DestroyVideoEncoder(encoder->encoder);
- } else {
- delete encoder->encoder;
+ external_encoder_factory_->DestroyVideoEncoder(encoder->external_encoder);
}
+ delete encoder->encoder;
}
void WebRtcVideoChannel2::WebRtcVideoSendStream::SetCodecAndOptions(
diff --git a/talk/media/webrtc/webrtcvideoengine2.h b/talk/media/webrtc/webrtcvideoengine2.h
index 8d24f29..5897309 100644
--- a/talk/media/webrtc/webrtcvideoengine2.h
+++ b/talk/media/webrtc/webrtcvideoengine2.h
@@ -321,9 +321,9 @@
struct AllocatedEncoder {
AllocatedEncoder(webrtc::VideoEncoder* encoder,
webrtc::VideoCodecType type,
- bool external)
- : encoder(encoder), type(type), external(external) {}
+ bool external);
webrtc::VideoEncoder* encoder;
+ webrtc::VideoEncoder* external_encoder;
webrtc::VideoCodecType type;
bool external;
};
diff --git a/webrtc/video/BUILD.gn b/webrtc/video/BUILD.gn
index 02c56a2..0504dbe 100644
--- a/webrtc/video/BUILD.gn
+++ b/webrtc/video/BUILD.gn
@@ -22,6 +22,7 @@
"transport_adapter.cc",
"transport_adapter.h",
"video_decoder.cc",
+ "video_encoder.cc",
"video_receive_stream.cc",
"video_receive_stream.h",
"video_send_stream.cc",
diff --git a/webrtc/video/call.cc b/webrtc/video/call.cc
index f19c3b4..37f9f2d 100644
--- a/webrtc/video/call.cc
+++ b/webrtc/video/call.cc
@@ -36,16 +36,6 @@
#include "webrtc/video/video_send_stream.h"
namespace webrtc {
-VideoEncoder* VideoEncoder::Create(VideoEncoder::EncoderType codec_type) {
- switch (codec_type) {
- case kVp8:
- return VP8Encoder::Create();
- case kVp9:
- return VP9Encoder::Create();
- }
- RTC_NOTREACHED();
- return nullptr;
-}
const int Call::Config::kDefaultStartBitrateBps = 300000;
diff --git a/webrtc/video/video_encoder.cc b/webrtc/video/video_encoder.cc
new file mode 100644
index 0000000..6931856
--- /dev/null
+++ b/webrtc/video/video_encoder.cc
@@ -0,0 +1,125 @@
+/*
+ * Copyright (c) 2015 The WebRTC project authors. All Rights Reserved.
+ *
+ * Use of this source code is governed by a BSD-style license
+ * that can be found in the LICENSE file in the root of the source
+ * tree. An additional intellectual property rights grant can be found
+ * in the file PATENTS. All contributing project authors may
+ * be found in the AUTHORS file in the root of the source tree.
+ */
+
+#include "webrtc/video_encoder.h"
+
+#include "webrtc/base/checks.h"
+#include "webrtc/modules/video_coding/codecs/vp8/include/vp8.h"
+#include "webrtc/modules/video_coding/codecs/vp9/include/vp9.h"
+#include "webrtc/system_wrappers/interface/logging.h"
+
+namespace webrtc {
+VideoEncoder* VideoEncoder::Create(VideoEncoder::EncoderType codec_type) {
+ switch (codec_type) {
+ case kVp8:
+ return VP8Encoder::Create();
+ case kVp9:
+ return VP9Encoder::Create();
+ case kUnsupportedCodec:
+ RTC_NOTREACHED();
+ return nullptr;
+ }
+ RTC_NOTREACHED();
+ return nullptr;
+}
+
+VideoEncoder::EncoderType CodecToEncoderType(VideoCodecType codec_type) {
+ switch (codec_type) {
+ case kVideoCodecVP8:
+ return VideoEncoder::kVp8;
+ case kVideoCodecVP9:
+ return VideoEncoder::kVp9;
+ default:
+ return VideoEncoder::kUnsupportedCodec;
+ }
+}
+
+VideoEncoderSoftwareFallbackWrapper::VideoEncoderSoftwareFallbackWrapper(
+ VideoCodecType codec_type,
+ webrtc::VideoEncoder* encoder)
+ : encoder_type_(CodecToEncoderType(codec_type)),
+ encoder_(encoder),
+ callback_(nullptr) {
+}
+
+int32_t VideoEncoderSoftwareFallbackWrapper::InitEncode(
+ const VideoCodec* codec_settings,
+ int32_t number_of_cores,
+ size_t max_payload_size) {
+ int32_t ret =
+ encoder_->InitEncode(codec_settings, number_of_cores, max_payload_size);
+ if (ret == WEBRTC_VIDEO_CODEC_OK || encoder_type_ == kUnsupportedCodec) {
+ fallback_encoder_.reset();
+ if (callback_)
+ encoder_->RegisterEncodeCompleteCallback(callback_);
+ return ret;
+ }
+ // Try to instantiate software codec.
+ fallback_encoder_.reset(VideoEncoder::Create(encoder_type_));
+ if (fallback_encoder_->InitEncode(codec_settings, number_of_cores,
+ max_payload_size) ==
+ WEBRTC_VIDEO_CODEC_OK) {
+ if (callback_)
+ fallback_encoder_->RegisterEncodeCompleteCallback(callback_);
+ return WEBRTC_VIDEO_CODEC_OK;
+ }
+ // Software encoder failed, reset and use original return code.
+ fallback_encoder_.reset();
+ return ret;
+}
+
+int32_t VideoEncoderSoftwareFallbackWrapper::RegisterEncodeCompleteCallback(
+ EncodedImageCallback* callback) {
+ callback_ = callback;
+ int32_t ret = encoder_->RegisterEncodeCompleteCallback(callback);
+ if (fallback_encoder_)
+ return fallback_encoder_->RegisterEncodeCompleteCallback(callback);
+ return ret;
+}
+
+int32_t VideoEncoderSoftwareFallbackWrapper::Release() {
+ if (fallback_encoder_)
+ return fallback_encoder_->Release();
+ return encoder_->Release();
+}
+
+int32_t VideoEncoderSoftwareFallbackWrapper::Encode(
+ const I420VideoFrame& frame,
+ const CodecSpecificInfo* codec_specific_info,
+ const std::vector<VideoFrameType>* frame_types) {
+ if (fallback_encoder_)
+ return fallback_encoder_->Encode(frame, codec_specific_info, frame_types);
+ return encoder_->Encode(frame, codec_specific_info, frame_types);
+}
+
+int32_t VideoEncoderSoftwareFallbackWrapper::SetChannelParameters(
+ uint32_t packet_loss,
+ int64_t rtt) {
+ int32_t ret = encoder_->SetChannelParameters(packet_loss, rtt);
+ if (fallback_encoder_)
+ return fallback_encoder_->SetChannelParameters(packet_loss, rtt);
+ return ret;
+}
+
+int32_t VideoEncoderSoftwareFallbackWrapper::SetRates(uint32_t bitrate,
+ uint32_t framerate) {
+ int32_t ret = encoder_->SetRates(bitrate, framerate);
+ if (fallback_encoder_)
+ return fallback_encoder_->SetRates(bitrate, framerate);
+ return ret;
+}
+
+void VideoEncoderSoftwareFallbackWrapper::OnDroppedFrame() {
+ if (fallback_encoder_)
+ return fallback_encoder_->OnDroppedFrame();
+ return encoder_->OnDroppedFrame();
+}
+
+} // namespace webrtc
diff --git a/webrtc/video/video_encoder_unittest.cc b/webrtc/video/video_encoder_unittest.cc
new file mode 100644
index 0000000..acfc4de
--- /dev/null
+++ b/webrtc/video/video_encoder_unittest.cc
@@ -0,0 +1,200 @@
+/*
+ * Copyright (c) 2015 The WebRTC project authors. All Rights Reserved.
+ *
+ * Use of this source code is governed by a BSD-style license
+ * that can be found in the LICENSE file in the root of the source
+ * tree. An additional intellectual property rights grant can be found
+ * in the file PATENTS. All contributing project authors may
+ * be found in the AUTHORS file in the root of the source tree.
+ */
+
+#include "webrtc/video_encoder.h"
+
+#include "testing/gtest/include/gtest/gtest.h"
+#include "webrtc/modules/video_coding/codecs/interface/video_error_codes.h"
+
+namespace webrtc {
+
+const size_t kMaxPayloadSize = 800;
+
+class VideoEncoderSoftwareFallbackWrapperTest : public ::testing::Test {
+ protected:
+ VideoEncoderSoftwareFallbackWrapperTest()
+ : fallback_wrapper_(kVideoCodecVP8, &fake_encoder_) {}
+
+ class CountingFakeEncoder : public VideoEncoder {
+ public:
+ int32_t InitEncode(const VideoCodec* codec_settings,
+ int32_t number_of_cores,
+ size_t max_payload_size) override {
+ ++init_encode_count_;
+ return init_encode_return_code_;
+ }
+ int32_t Encode(const I420VideoFrame& frame,
+ const CodecSpecificInfo* codec_specific_info,
+ const std::vector<VideoFrameType>* frame_types) override {
+ ++encode_count_;
+ return WEBRTC_VIDEO_CODEC_OK;
+ }
+
+ int32_t RegisterEncodeCompleteCallback(
+ EncodedImageCallback* callback) override {
+ encode_complete_callback_ = callback;
+ return WEBRTC_VIDEO_CODEC_OK;
+ }
+
+ int32_t Release() override {
+ ++release_count_;
+ return WEBRTC_VIDEO_CODEC_OK;
+ }
+
+ int32_t SetChannelParameters(uint32_t packet_loss, int64_t rtt) override {
+ ++set_channel_parameters_count_;
+ return WEBRTC_VIDEO_CODEC_OK;
+ }
+
+ int32_t SetRates(uint32_t bitrate, uint32_t framerate) override {
+ ++set_rates_count_;
+ return WEBRTC_VIDEO_CODEC_OK;
+ }
+
+ void OnDroppedFrame() override { ++on_dropped_frame_count_; }
+
+ int init_encode_count_ = 0;
+ int32_t init_encode_return_code_ = WEBRTC_VIDEO_CODEC_OK;
+ int encode_count_ = 0;
+ EncodedImageCallback* encode_complete_callback_ = nullptr;
+ int release_count_ = 0;
+ int set_channel_parameters_count_ = 0;
+ int set_rates_count_ = 0;
+ int on_dropped_frame_count_ = 0;
+ };
+
+ class FakeEncodedImageCallback : public EncodedImageCallback {
+ public:
+ int32_t Encoded(const EncodedImage& encoded_image,
+ const CodecSpecificInfo* codec_specific_info,
+ const RTPFragmentationHeader* fragmentation) override {
+ return ++callback_count_;
+ }
+ int callback_count_ = 0;
+ };
+
+ void UtilizeFallbackEncoder();
+
+ FakeEncodedImageCallback callback_;
+ CountingFakeEncoder fake_encoder_;
+ VideoEncoderSoftwareFallbackWrapper fallback_wrapper_;
+ VideoCodec codec_ = {};
+ I420VideoFrame frame_;
+};
+
+void VideoEncoderSoftwareFallbackWrapperTest::UtilizeFallbackEncoder() {
+ static const int kWidth = 320;
+ static const int kHeight = 240;
+ fallback_wrapper_.RegisterEncodeCompleteCallback(&callback_);
+ EXPECT_EQ(&callback_, fake_encoder_.encode_complete_callback_);
+
+ // Register with failing fake encoder. Should succeed with VP8 fallback.
+ codec_.codecType = kVideoCodecVP8;
+ codec_.maxFramerate = 30;
+ codec_.width = kWidth;
+ codec_.height = kHeight;
+ fake_encoder_.init_encode_return_code_ = WEBRTC_VIDEO_CODEC_ERROR;
+ EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK,
+ fallback_wrapper_.InitEncode(&codec_, 2, kMaxPayloadSize));
+ EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK, fallback_wrapper_.SetRates(300, 30));
+
+ frame_.CreateEmptyFrame(kWidth, kHeight, kWidth, (kWidth + 1) / 2,
+ (kWidth + 1) / 2);
+ memset(frame_.buffer(webrtc::kYPlane), 16,
+ frame_.allocated_size(webrtc::kYPlane));
+ memset(frame_.buffer(webrtc::kUPlane), 128,
+ frame_.allocated_size(webrtc::kUPlane));
+ memset(frame_.buffer(webrtc::kVPlane), 128,
+ frame_.allocated_size(webrtc::kVPlane));
+
+ std::vector<VideoFrameType> types(1, kKeyFrame);
+ EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK,
+ fallback_wrapper_.Encode(frame_, nullptr, &types));
+ EXPECT_EQ(0, fake_encoder_.encode_count_);
+ EXPECT_GT(callback_.callback_count_, 0);
+}
+
+TEST_F(VideoEncoderSoftwareFallbackWrapperTest, InitializesEncoder) {
+ VideoCodec codec = {};
+ fallback_wrapper_.InitEncode(&codec, 2, kMaxPayloadSize);
+ EXPECT_EQ(1, fake_encoder_.init_encode_count_);
+}
+
+TEST_F(VideoEncoderSoftwareFallbackWrapperTest, CanUtilizeFallbackEncoder) {
+ UtilizeFallbackEncoder();
+ EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK, fallback_wrapper_.Release());
+}
+
+TEST_F(VideoEncoderSoftwareFallbackWrapperTest,
+ InternalEncoderNotReleasedDuringFallback) {
+ UtilizeFallbackEncoder();
+ EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK, fallback_wrapper_.Release());
+ EXPECT_EQ(0, fake_encoder_.release_count_);
+}
+
+TEST_F(VideoEncoderSoftwareFallbackWrapperTest,
+ InternalEncoderNotEncodingDuringFallback) {
+ UtilizeFallbackEncoder();
+ EXPECT_EQ(0, fake_encoder_.encode_count_);
+
+ EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK, fallback_wrapper_.Release());
+}
+
+TEST_F(VideoEncoderSoftwareFallbackWrapperTest,
+ CanRegisterCallbackWhileUsingFallbackEncoder) {
+ UtilizeFallbackEncoder();
+ // Registering an encode-complete callback should still work when fallback
+ // encoder is being used.
+ FakeEncodedImageCallback callback2;
+ fallback_wrapper_.RegisterEncodeCompleteCallback(&callback2);
+ EXPECT_EQ(&callback2, fake_encoder_.encode_complete_callback_);
+
+ // Encoding a frame using the fallback should arrive at the new callback.
+ std::vector<VideoFrameType> types(1, kKeyFrame);
+ frame_.set_timestamp(frame_.timestamp() + 1000);
+ EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK,
+ fallback_wrapper_.Encode(frame_, nullptr, &types));
+
+ EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK, fallback_wrapper_.Release());
+}
+
+TEST_F(VideoEncoderSoftwareFallbackWrapperTest,
+ SetChannelParametersForwardedDuringFallback) {
+ UtilizeFallbackEncoder();
+ EXPECT_EQ(0, fake_encoder_.set_channel_parameters_count_);
+ fallback_wrapper_.SetChannelParameters(1, 1);
+ EXPECT_EQ(1, fake_encoder_.set_channel_parameters_count_);
+ EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK, fallback_wrapper_.Release());
+}
+
+TEST_F(VideoEncoderSoftwareFallbackWrapperTest,
+ SetRatesForwardedDuringFallback) {
+ UtilizeFallbackEncoder();
+ EXPECT_EQ(1, fake_encoder_.set_rates_count_);
+ fallback_wrapper_.SetRates(1, 1);
+ EXPECT_EQ(2, fake_encoder_.set_rates_count_);
+ EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK, fallback_wrapper_.Release());
+}
+
+TEST_F(VideoEncoderSoftwareFallbackWrapperTest,
+ OnDroppedFrameForwardedWithoutFallback) {
+ fallback_wrapper_.OnDroppedFrame();
+ EXPECT_EQ(1, fake_encoder_.on_dropped_frame_count_);
+}
+
+TEST_F(VideoEncoderSoftwareFallbackWrapperTest,
+ OnDroppedFrameNotForwardedDuringFallback) {
+ UtilizeFallbackEncoder();
+ fallback_wrapper_.OnDroppedFrame();
+ EXPECT_EQ(0, fake_encoder_.on_dropped_frame_count_);
+ EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK, fallback_wrapper_.Release());
+}
+
+} // namespace webrtc
diff --git a/webrtc/video/webrtc_video.gypi b/webrtc/video/webrtc_video.gypi
index 37047c6..545f284 100644
--- a/webrtc/video/webrtc_video.gypi
+++ b/webrtc/video/webrtc_video.gypi
@@ -23,6 +23,7 @@
'video/transport_adapter.cc',
'video/transport_adapter.h',
'video/video_decoder.cc',
+ 'video/video_encoder.cc',
'video/video_receive_stream.cc',
'video/video_receive_stream.h',
'video/video_send_stream.cc',
diff --git a/webrtc/video_encoder.h b/webrtc/video_encoder.h
index ce2569b..832f40f 100644
--- a/webrtc/video_encoder.h
+++ b/webrtc/video_encoder.h
@@ -39,6 +39,7 @@
enum EncoderType {
kVp8,
kVp9,
+ kUnsupportedCodec,
};
static VideoEncoder* Create(EncoderType codec_type);
@@ -125,5 +126,36 @@
virtual void OnDroppedFrame() {};
};
+// Class used to wrap external VideoEncoders to provide a fallback option on
+// software encoding when a hardware encoder fails to encode a stream due to
+// hardware restrictions, such as max resolution.
+class VideoEncoderSoftwareFallbackWrapper : public VideoEncoder {
+ public:
+ VideoEncoderSoftwareFallbackWrapper(VideoCodecType codec_type,
+ webrtc::VideoEncoder* encoder);
+
+ int32_t InitEncode(const VideoCodec* codec_settings,
+ int32_t number_of_cores,
+ size_t max_payload_size) override;
+
+ int32_t RegisterEncodeCompleteCallback(
+ EncodedImageCallback* callback) override;
+
+ int32_t Release() override;
+ int32_t Encode(const I420VideoFrame& frame,
+ const CodecSpecificInfo* codec_specific_info,
+ const std::vector<VideoFrameType>* frame_types) override;
+ int32_t SetChannelParameters(uint32_t packet_loss, int64_t rtt) override;
+
+ int32_t SetRates(uint32_t bitrate, uint32_t framerate) override;
+ void OnDroppedFrame() override;
+
+ private:
+ const EncoderType encoder_type_;
+ webrtc::VideoEncoder* const encoder_;
+
+ rtc::scoped_ptr<webrtc::VideoEncoder> fallback_encoder_;
+ EncodedImageCallback* callback_;
+};
} // namespace webrtc
#endif // WEBRTC_VIDEO_ENCODER_H_
diff --git a/webrtc/webrtc_tests.gypi b/webrtc/webrtc_tests.gypi
index a47129a..6a4dec9 100644
--- a/webrtc/webrtc_tests.gypi
+++ b/webrtc/webrtc_tests.gypi
@@ -151,6 +151,7 @@
'video/end_to_end_tests.cc',
'video/send_statistics_proxy_unittest.cc',
'video/video_decoder_unittest.cc',
+ 'video/video_encoder_unittest.cc',
'video/video_send_stream_tests.cc',
],
'dependencies': [
