In screenshare mode, suppress VP8 bitrate overshoot and increase quality
This change includes several improvements:
* VP8 configured with new rate control
* Detection of frame dropping, with qp bump for next frame
* Increased target and TL0 bitrates
* Reworked rate control (TL allocation) in screenshare_layers
A note on performance: PSNR and SSIM is expected to get slightly worse with this cl. Frame drops and delays should however improve.
BUG=4171
R=pbos@webrtc.org, stefan@webrtc.org
Review URL: https://codereview.webrtc.org/1193513006.
Cr-Commit-Position: refs/heads/master@{#9495}
diff --git a/talk/media/webrtc/simulcast.cc b/talk/media/webrtc/simulcast.cc
index 6a822b8..759cfef 100755
--- a/talk/media/webrtc/simulcast.cc
+++ b/talk/media/webrtc/simulcast.cc
@@ -411,7 +411,7 @@
static const int kScreenshareMinBitrateKbps = 50;
static const int kScreenshareMaxBitrateKbps = 6000;
-static const int kScreenshareDefaultTl0BitrateKbps = 100;
+static const int kScreenshareDefaultTl0BitrateKbps = 200;
static const int kScreenshareDefaultTl1BitrateKbps = 1000;
static const char* kScreencastLayerFieldTrialName =
diff --git a/talk/media/webrtc/webrtcvideoengine2_unittest.cc b/talk/media/webrtc/webrtcvideoengine2_unittest.cc
index d83e8c4..230ca3b 100644
--- a/talk/media/webrtc/webrtcvideoengine2_unittest.cc
+++ b/talk/media/webrtc/webrtcvideoengine2_unittest.cc
@@ -1438,7 +1438,8 @@
TEST_F(WebRtcVideoChannel2Test,
ConferenceModeScreencastConfiguresTemporalLayer) {
- static const int kConferenceScreencastTemporalBitrateBps = 100000;
+ static const int kConferenceScreencastTemporalBitrateBps =
+ ScreenshareLayerConfig::GetDefault().tl0_bitrate_kbps * 1000;
VideoOptions options;
options.conference_mode.Set(true);
channel_->SetOptions(options);
diff --git a/webrtc/modules/video_coding/codecs/interface/video_error_codes.h b/webrtc/modules/video_coding/codecs/interface/video_error_codes.h
index 349a39a..28e5a32 100644
--- a/webrtc/modules/video_coding/codecs/interface/video_error_codes.h
+++ b/webrtc/modules/video_coding/codecs/interface/video_error_codes.h
@@ -27,5 +27,6 @@
#define WEBRTC_VIDEO_CODEC_UNINITIALIZED -7
#define WEBRTC_VIDEO_CODEC_ERR_REQUEST_SLI -12
#define WEBRTC_VIDEO_CODEC_FALLBACK_SOFTWARE -13
+#define WEBRTC_VIDEO_CODEC_TARGET_BITRATE_OVERSHOOT -14
#endif // WEBRTC_MODULES_VIDEO_CODING_CODECS_INTERFACE_VIDEO_ERROR_CODES_H
diff --git a/webrtc/modules/video_coding/codecs/vp8/default_temporal_layers.h b/webrtc/modules/video_coding/codecs/vp8/default_temporal_layers.h
index 61f281f..19846ba 100644
--- a/webrtc/modules/video_coding/codecs/vp8/default_temporal_layers.h
+++ b/webrtc/modules/video_coding/codecs/vp8/default_temporal_layers.h
@@ -24,20 +24,22 @@
// Returns the recommended VP8 encode flags needed. May refresh the decoder
// and/or update the reference buffers.
- virtual int EncodeFlags(uint32_t timestamp);
+ int EncodeFlags(uint32_t timestamp) override;
- virtual bool ConfigureBitrates(int bitrate_kbit,
- int max_bitrate_kbit,
- int framerate,
- vpx_codec_enc_cfg_t* cfg);
+ bool ConfigureBitrates(int bitrate_kbit,
+ int max_bitrate_kbit,
+ int framerate,
+ vpx_codec_enc_cfg_t* cfg) override;
- virtual void PopulateCodecSpecific(bool base_layer_sync,
- CodecSpecificInfoVP8* vp8_info,
- uint32_t timestamp);
+ void PopulateCodecSpecific(bool base_layer_sync,
+ CodecSpecificInfoVP8* vp8_info,
+ uint32_t timestamp) override;
- virtual void FrameEncoded(unsigned int size, uint32_t timestamp) {}
+ void FrameEncoded(unsigned int size, uint32_t timestamp, int qp) override {}
- virtual int CurrentLayerId() const;
+ bool UpdateConfiguration(vpx_codec_enc_cfg_t* cfg) override { return false; }
+
+ int CurrentLayerId() const override;
private:
enum TemporalReferences {
diff --git a/webrtc/modules/video_coding/codecs/vp8/realtime_temporal_layers.cc b/webrtc/modules/video_coding/codecs/vp8/realtime_temporal_layers.cc
index f16c756..15b5af9 100644
--- a/webrtc/modules/video_coding/codecs/vp8/realtime_temporal_layers.cc
+++ b/webrtc/modules/video_coding/codecs/vp8/realtime_temporal_layers.cc
@@ -239,7 +239,9 @@
}
}
- void FrameEncoded(unsigned int size, uint32_t timestamp) {}
+ void FrameEncoded(unsigned int size, uint32_t timestamp, int qp) override {}
+
+ bool UpdateConfiguration(vpx_codec_enc_cfg_t* cfg) override { return false; }
private:
int temporal_layers_;
diff --git a/webrtc/modules/video_coding/codecs/vp8/screenshare_layers.cc b/webrtc/modules/video_coding/codecs/vp8/screenshare_layers.cc
index 63ef227..ecaf3dd 100644
--- a/webrtc/modules/video_coding/codecs/vp8/screenshare_layers.cc
+++ b/webrtc/modules/video_coding/codecs/vp8/screenshare_layers.cc
@@ -11,32 +11,52 @@
#include <stdlib.h>
+#include "webrtc/base/checks.h"
#include "vpx/vpx_encoder.h"
#include "vpx/vp8cx.h"
#include "webrtc/modules/video_coding/codecs/interface/video_codec_interface.h"
namespace webrtc {
-enum { kOneSecond90Khz = 90000 };
+static const int kOneSecond90Khz = 90000;
+static const int kMinTimeBetweenSyncs = kOneSecond90Khz * 5;
+static const int kMaxTimeBetweenSyncs = kOneSecond90Khz * 10;
+static const int kQpDeltaThresholdForSync = 8;
const double ScreenshareLayers::kMaxTL0FpsReduction = 2.5;
const double ScreenshareLayers::kAcceptableTargetOvershoot = 2.0;
+// Since this is TL0 we only allow updating and predicting from the LAST
+// reference frame.
+const int ScreenshareLayers::kTl0Flags =
+ VP8_EFLAG_NO_UPD_GF | VP8_EFLAG_NO_UPD_ARF | VP8_EFLAG_NO_REF_GF |
+ VP8_EFLAG_NO_REF_ARF;
+
+// Allow predicting from both TL0 and TL1.
+const int ScreenshareLayers::kTl1Flags =
+ VP8_EFLAG_NO_REF_ARF | VP8_EFLAG_NO_UPD_ARF | VP8_EFLAG_NO_UPD_LAST;
+
+// Allow predicting from only TL0 to allow participants to switch to the high
+// bitrate stream. This means predicting only from the LAST reference frame, but
+// only updating GF to not corrupt TL0.
+const int ScreenshareLayers::kTl1SyncFlags =
+ VP8_EFLAG_NO_REF_ARF | VP8_EFLAG_NO_REF_GF | VP8_EFLAG_NO_UPD_ARF |
+ VP8_EFLAG_NO_UPD_LAST;
+
ScreenshareLayers::ScreenshareLayers(int num_temporal_layers,
- uint8_t initial_tl0_pic_idx,
- FrameDropper* tl0_frame_dropper,
- FrameDropper* tl1_frame_dropper)
- : tl0_frame_dropper_(tl0_frame_dropper),
- tl1_frame_dropper_(tl1_frame_dropper),
- number_of_temporal_layers_(num_temporal_layers),
+ uint8_t initial_tl0_pic_idx)
+ : number_of_temporal_layers_(num_temporal_layers),
last_base_layer_sync_(false),
tl0_pic_idx_(initial_tl0_pic_idx),
- active_layer_(0),
- framerate_(5),
- last_sync_timestamp_(-1) {
+ active_layer_(-1),
+ last_timestamp_(-1),
+ last_sync_timestamp_(-1),
+ min_qp_(-1),
+ max_qp_(-1),
+ max_debt_bytes_(0),
+ frame_rate_(-1) {
assert(num_temporal_layers > 0);
assert(num_temporal_layers <= 2);
- assert(tl0_frame_dropper && tl1_frame_dropper);
}
int ScreenshareLayers::CurrentLayerId() const {
@@ -49,84 +69,125 @@
// No flags needed for 1 layer screenshare.
return 0;
}
- CalculateFramerate(timestamp);
+
+ int64_t unwrapped_timestamp = time_wrap_handler_.Unwrap(timestamp);
int flags = 0;
- // Note that ARF on purpose isn't used in this scheme since it is allocated
- // for the last key frame to make key frame caching possible.
- if (tl0_frame_dropper_->DropFrame()) {
- // Must drop TL0, encode TL1 instead.
- if (tl1_frame_dropper_->DropFrame()) {
- // Must drop both TL0 and TL1.
- flags = -1;
- } else {
- active_layer_ = 1;
- if (TimeToSync(timestamp)) {
- last_sync_timestamp_ = timestamp;
- // Allow predicting from only TL0 to allow participants to switch to the
- // high bitrate stream. This means predicting only from the LAST
- // reference frame, but only updating GF to not corrupt TL0.
- flags = VP8_EFLAG_NO_REF_ARF;
- flags |= VP8_EFLAG_NO_REF_GF;
- flags |= VP8_EFLAG_NO_UPD_ARF;
- flags |= VP8_EFLAG_NO_UPD_LAST;
+
+ if (active_layer_ == -1 ||
+ layers_[active_layer_].state != TemporalLayer::State::kDropped) {
+ if (layers_[0].debt_bytes_ > max_debt_bytes_) {
+ // Must drop TL0, encode TL1 instead.
+ if (layers_[1].debt_bytes_ > max_debt_bytes_) {
+ // Must drop both TL0 and TL1.
+ active_layer_ = -1;
} else {
- // Allow predicting from both TL0 and TL1.
- flags = VP8_EFLAG_NO_REF_ARF;
- flags |= VP8_EFLAG_NO_UPD_ARF;
- flags |= VP8_EFLAG_NO_UPD_LAST;
+ active_layer_ = 1;
}
+ } else {
+ active_layer_ = 0;
}
- } else {
- active_layer_ = 0;
- // Since this is TL0 we only allow updating and predicting from the LAST
- // reference frame.
- flags = VP8_EFLAG_NO_UPD_GF;
- flags |= VP8_EFLAG_NO_UPD_ARF;
- flags |= VP8_EFLAG_NO_REF_GF;
- flags |= VP8_EFLAG_NO_REF_ARF;
}
+
+ switch (active_layer_) {
+ case 0:
+ flags = kTl0Flags;
+ break;
+ case 1:
+ if (TimeToSync(unwrapped_timestamp)) {
+ last_sync_timestamp_ = unwrapped_timestamp;
+ flags = kTl1SyncFlags;
+ } else {
+ flags = kTl1Flags;
+ }
+ break;
+ case -1:
+ flags = -1;
+ break;
+ default:
+ flags = -1;
+ RTC_NOTREACHED();
+ }
+
// Make sure both frame droppers leak out bits.
- tl0_frame_dropper_->Leak(framerate_);
- tl1_frame_dropper_->Leak(framerate_);
+ int64_t ts_diff;
+ if (last_timestamp_ == -1) {
+ ts_diff = kOneSecond90Khz / (frame_rate_ <= 0 ? 5 : frame_rate_);
+ } else {
+ ts_diff = unwrapped_timestamp - last_timestamp_;
+ }
+
+ layers_[0].UpdateDebt(ts_diff / 90);
+ layers_[1].UpdateDebt(ts_diff / 90);
+ last_timestamp_ = timestamp;
return flags;
}
-bool ScreenshareLayers::ConfigureBitrates(int bitrate_kbit,
- int max_bitrate_kbit,
+bool ScreenshareLayers::ConfigureBitrates(int bitrate_kbps,
+ int max_bitrate_kbps,
int framerate,
vpx_codec_enc_cfg_t* cfg) {
- if (framerate > 0)
- framerate_ = framerate;
+ layers_[0].target_rate_kbps_ = bitrate_kbps;
+ layers_[1].target_rate_kbps_ = max_bitrate_kbps;
- tl0_frame_dropper_->SetRates(bitrate_kbit, framerate_);
- tl1_frame_dropper_->SetRates(max_bitrate_kbit, framerate_);
+ int target_bitrate_kbps = bitrate_kbps;
if (cfg != nullptr) {
// Calculate a codec target bitrate. This may be higher than TL0, gaining
// quality at the expense of frame rate at TL0. Constraints:
// - TL0 frame rate should not be less than framerate / kMaxTL0FpsReduction.
// - Target rate * kAcceptableTargetOvershoot should not exceed TL1 rate.
- double target_bitrate =
- std::min(bitrate_kbit * kMaxTL0FpsReduction,
- max_bitrate_kbit / kAcceptableTargetOvershoot);
- cfg->rc_target_bitrate =
- std::max(static_cast<unsigned int>(bitrate_kbit),
- static_cast<unsigned int>(target_bitrate + 0.5));
+ target_bitrate_kbps =
+ std::min(bitrate_kbps * kMaxTL0FpsReduction,
+ max_bitrate_kbps / kAcceptableTargetOvershoot);
+
+ // Don't reconfigure qp limits during quality boost frames.
+ if (layers_[active_layer_].state != TemporalLayer::State::kQualityBoost) {
+ min_qp_ = cfg->rc_min_quantizer;
+ max_qp_ = cfg->rc_max_quantizer;
+ // After a dropped frame, a frame with max qp will be encoded and the
+ // quality will then ramp up from there. To boost the speed of recovery,
+ // encode the next frame with lower max qp. TL0 is the most important to
+ // improve since the errors in this layer will propagate to TL1.
+ // Currently, reduce max qp by 20% for TL0 and 15% for TL1.
+ layers_[0].enhanced_max_qp = min_qp_ + (((max_qp_ - min_qp_) * 80) / 100);
+ layers_[1].enhanced_max_qp = min_qp_ + (((max_qp_ - min_qp_) * 85) / 100);
+ }
+
+ cfg->rc_target_bitrate = std::max(bitrate_kbps, target_bitrate_kbps);
}
+ int avg_frame_size = (target_bitrate_kbps * 1000) / (8 * framerate);
+ max_debt_bytes_ = 4 * avg_frame_size;
+
return true;
}
-void ScreenshareLayers::FrameEncoded(unsigned int size, uint32_t timestamp) {
- if (active_layer_ == 0) {
- tl0_frame_dropper_->Fill(size, true);
+void ScreenshareLayers::FrameEncoded(unsigned int size,
+ uint32_t timestamp,
+ int qp) {
+ if (size == 0) {
+ layers_[active_layer_].state = TemporalLayer::State::kDropped;
+ return;
}
- tl1_frame_dropper_->Fill(size, true);
+ if (layers_[active_layer_].state == TemporalLayer::State::kDropped) {
+ layers_[active_layer_].state = TemporalLayer::State::kQualityBoost;
+ }
+
+ if (qp != -1)
+ layers_[active_layer_].last_qp = qp;
+
+ if (active_layer_ == 0) {
+ layers_[0].debt_bytes_ += size;
+ layers_[1].debt_bytes_ += size;
+ } else if (active_layer_ == 1) {
+ layers_[1].debt_bytes_ += size;
+ }
}
void ScreenshareLayers::PopulateCodecSpecific(bool base_layer_sync,
CodecSpecificInfoVP8 *vp8_info,
uint32_t timestamp) {
+ int64_t unwrapped_timestamp = time_wrap_handler_.Unwrap(timestamp);
if (number_of_temporal_layers_ == 1) {
vp8_info->temporalIdx = kNoTemporalIdx;
vp8_info->layerSync = false;
@@ -135,13 +196,14 @@
vp8_info->temporalIdx = active_layer_;
if (base_layer_sync) {
vp8_info->temporalIdx = 0;
- last_sync_timestamp_ = timestamp;
+ last_sync_timestamp_ = unwrapped_timestamp;
} else if (last_base_layer_sync_ && vp8_info->temporalIdx != 0) {
// Regardless of pattern the frame after a base layer sync will always
// be a layer sync.
- last_sync_timestamp_ = timestamp;
+ last_sync_timestamp_ = unwrapped_timestamp;
}
- vp8_info->layerSync = (last_sync_timestamp_ == timestamp);
+ vp8_info->layerSync = last_sync_timestamp_ != -1 &&
+ last_sync_timestamp_ == unwrapped_timestamp;
if (vp8_info->temporalIdx == 0) {
tl0_pic_idx_++;
}
@@ -150,27 +212,65 @@
}
}
-bool ScreenshareLayers::TimeToSync(uint32_t timestamp) const {
- const uint32_t timestamp_diff = timestamp - last_sync_timestamp_;
- return last_sync_timestamp_ < 0 || timestamp_diff > kOneSecond90Khz;
+bool ScreenshareLayers::TimeToSync(int64_t timestamp) const {
+ if (active_layer_ != 1) {
+ RTC_NOTREACHED();
+ return false;
+ }
+ DCHECK_NE(-1, layers_[0].last_qp);
+ if (layers_[1].last_qp == -1) {
+ // First frame in TL1 should only depend on TL0 since there are no
+ // previous frames in TL1.
+ return true;
+ }
+
+ DCHECK_NE(-1, last_sync_timestamp_);
+ int64_t timestamp_diff = timestamp - last_sync_timestamp_;
+ if (timestamp_diff > kMaxTimeBetweenSyncs) {
+ // After a certain time, force a sync frame.
+ return true;
+ } else if (timestamp_diff < kMinTimeBetweenSyncs) {
+ // If too soon from previous sync frame, don't issue a new one.
+ return false;
+ }
+ // Issue a sync frame if difference in quality between TL0 and TL1 isn't too
+ // large.
+ if (layers_[0].last_qp - layers_[1].last_qp < kQpDeltaThresholdForSync)
+ return true;
+ return false;
}
-void ScreenshareLayers::CalculateFramerate(uint32_t timestamp) {
- timestamp_list_.push_front(timestamp);
- // Remove timestamps older than 1 second from the list.
- uint32_t timestamp_diff = timestamp - timestamp_list_.back();
- while (timestamp_diff > kOneSecond90Khz) {
- timestamp_list_.pop_back();
- timestamp_diff = timestamp - timestamp_list_.back();
+bool ScreenshareLayers::UpdateConfiguration(vpx_codec_enc_cfg_t* cfg) {
+ if (max_qp_ == -1)
+ return false;
+
+ // If layer is in the quality boost state (following a dropped frame), update
+ // the configuration with the adjusted (lower) qp and set the state back to
+ // normal.
+ unsigned int adjusted_max_qp;
+ if (layers_[active_layer_].state == TemporalLayer::State::kQualityBoost &&
+ layers_[active_layer_].enhanced_max_qp != -1) {
+ adjusted_max_qp = layers_[active_layer_].enhanced_max_qp;
+ layers_[active_layer_].state = TemporalLayer::State::kNormal;
+ } else {
+ if (max_qp_ == -1)
+ return false;
+ adjusted_max_qp = max_qp_; // Set the normal max qp.
}
- // If we have encoded frames within the last second, that number of frames
- // is a reasonable first estimate of the framerate.
- framerate_ = timestamp_list_.size();
- if (timestamp_diff > 0) {
- // Estimate the framerate by dividing the number of timestamp diffs with
- // the sum of the timestamp diffs (with rounding).
- framerate_ = (kOneSecond90Khz * (timestamp_list_.size() - 1) +
- timestamp_diff / 2) / timestamp_diff;
+
+ if (adjusted_max_qp == cfg->rc_max_quantizer)
+ return false;
+
+ cfg->rc_max_quantizer = adjusted_max_qp;
+ return true;
+}
+
+void ScreenshareLayers::TemporalLayer::UpdateDebt(int64_t delta_ms) {
+ uint32_t debt_reduction_bytes = target_rate_kbps_ * delta_ms / 8;
+ if (debt_reduction_bytes >= debt_bytes_) {
+ debt_bytes_ = 0;
+ } else {
+ debt_bytes_ -= debt_reduction_bytes;
}
}
diff --git a/webrtc/modules/video_coding/codecs/vp8/screenshare_layers.h b/webrtc/modules/video_coding/codecs/vp8/screenshare_layers.h
index 0bc571e..90a8b1b 100644
--- a/webrtc/modules/video_coding/codecs/vp8/screenshare_layers.h
+++ b/webrtc/modules/video_coding/codecs/vp8/screenshare_layers.h
@@ -13,6 +13,7 @@
#include "vpx/vpx_encoder.h"
+#include "webrtc/base/timeutils.h"
#include "webrtc/modules/video_coding/codecs/vp8/temporal_layers.h"
#include "webrtc/modules/video_coding/utility/include/frame_dropper.h"
#include "webrtc/typedefs.h"
@@ -25,43 +26,73 @@
public:
static const double kMaxTL0FpsReduction;
static const double kAcceptableTargetOvershoot;
+ static const int kTl0Flags;
+ static const int kTl1Flags;
+ static const int kTl1SyncFlags;
- ScreenshareLayers(int num_temporal_layers,
- uint8_t initial_tl0_pic_idx,
- FrameDropper* tl0_frame_dropper,
- FrameDropper* tl1_frame_dropper);
+ ScreenshareLayers(int num_temporal_layers, uint8_t initial_tl0_pic_idx);
virtual ~ScreenshareLayers() {}
// Returns the recommended VP8 encode flags needed. May refresh the decoder
// and/or update the reference buffers.
- virtual int EncodeFlags(uint32_t timestamp);
+ int EncodeFlags(uint32_t timestamp) override;
- virtual bool ConfigureBitrates(int bitrate_kbit,
- int max_bitrate_kbit,
- int framerate,
- vpx_codec_enc_cfg_t* cfg);
+ bool ConfigureBitrates(int bitrate_kbps,
+ int max_bitrate_kbps,
+ int framerate,
+ vpx_codec_enc_cfg_t* cfg) override;
- virtual void PopulateCodecSpecific(bool base_layer_sync,
- CodecSpecificInfoVP8 *vp8_info,
- uint32_t timestamp);
+ void PopulateCodecSpecific(bool base_layer_sync,
+ CodecSpecificInfoVP8* vp8_info,
+ uint32_t timestamp) override;
- virtual void FrameEncoded(unsigned int size, uint32_t timestamp);
+ void FrameEncoded(unsigned int size, uint32_t timestamp, int qp) override;
- virtual int CurrentLayerId() const;
+ int CurrentLayerId() const override;
+
+ // Allows the layers adapter to update the encoder configuration prior to a
+ // frame being encoded. Return true if the configuration should be updated
+ // and false if now change is needed.
+ bool UpdateConfiguration(vpx_codec_enc_cfg_t* cfg) override;
private:
- void CalculateFramerate(uint32_t timestamp);
- bool TimeToSync(uint32_t timestamp) const;
+ bool TimeToSync(int64_t timestamp) const;
- FrameDropper* tl0_frame_dropper_;
- FrameDropper* tl1_frame_dropper_;
int number_of_temporal_layers_;
bool last_base_layer_sync_;
uint8_t tl0_pic_idx_;
int active_layer_;
- std::list<uint32_t> timestamp_list_;
- int framerate_;
+ int64_t last_timestamp_;
int64_t last_sync_timestamp_;
+ rtc::TimestampWrapAroundHandler time_wrap_handler_;
+ int min_qp_;
+ int max_qp_;
+ uint32_t max_debt_bytes_;
+ int frame_rate_;
+
+ static const int kMaxNumTemporalLayers = 2;
+ struct TemporalLayer {
+ TemporalLayer()
+ : state(State::kNormal),
+ enhanced_max_qp(-1),
+ last_qp(-1),
+ debt_bytes_(0),
+ target_rate_kbps_(0) {}
+
+ enum class State {
+ kNormal,
+ kDropped,
+ kReencoded,
+ kQualityBoost,
+ } state;
+
+ int enhanced_max_qp;
+ int last_qp;
+ uint32_t debt_bytes_;
+ uint32_t target_rate_kbps_;
+
+ void UpdateDebt(int64_t delta_ms);
+ } layers_[kMaxNumTemporalLayers];
};
} // namespace webrtc
diff --git a/webrtc/modules/video_coding/codecs/vp8/screenshare_layers_unittest.cc b/webrtc/modules/video_coding/codecs/vp8/screenshare_layers_unittest.cc
index e12f9ce..198be2a 100644
--- a/webrtc/modules/video_coding/codecs/vp8/screenshare_layers_unittest.cc
+++ b/webrtc/modules/video_coding/codecs/vp8/screenshare_layers_unittest.cc
@@ -22,62 +22,19 @@
namespace webrtc {
-enum { kTimestampDelta5Fps = 90000 / 5 }; // 5 frames per second at 90 kHz.
-enum { kTimestampDelta30Fps = 90000 / 30 }; // 30 frames per second at 90 kHz.
-enum { kFrameSize = 2500 };
-
-const int kFlagsTL0 = VP8_EFLAG_NO_UPD_GF | VP8_EFLAG_NO_UPD_ARF |
- VP8_EFLAG_NO_REF_GF | VP8_EFLAG_NO_REF_ARF;
-const int kFlagsTL1 = VP8_EFLAG_NO_REF_ARF | VP8_EFLAG_NO_UPD_ARF |
- VP8_EFLAG_NO_UPD_LAST;
-const int kFlagsTL1Sync = VP8_EFLAG_NO_REF_ARF | VP8_EFLAG_NO_REF_GF |
- VP8_EFLAG_NO_UPD_ARF | VP8_EFLAG_NO_UPD_LAST;
-
-class ScreenshareLayersFT : public ScreenshareLayers {
- public:
- ScreenshareLayersFT(int num_temporal_layers,
- uint8_t initial_tl0_pic_idx,
- FrameDropper* tl0_frame_dropper,
- FrameDropper* tl1_frame_dropper)
- : ScreenshareLayers(num_temporal_layers,
- initial_tl0_pic_idx,
- tl0_frame_dropper,
- tl1_frame_dropper) {}
- virtual ~ScreenshareLayersFT() {}
-};
+// 5 frames per second at 90 kHz.
+const uint32_t kTimestampDelta5Fps = 90000 / 5;
+const int kDefaultQp = 54;
+const int kDefaultTl0BitrateKbps = 200;
+const int kDefaultTl1BitrateKbps = 2000;
+const int kFrameRate = 5;
+const int kSyncPeriodSeconds = 5;
+const int kMaxSyncPeriodSeconds = 10;
class ScreenshareLayerTest : public ::testing::Test {
protected:
- void SetEncodeExpectations(bool drop_tl0, bool drop_tl1, int framerate) {
- EXPECT_CALL(tl0_frame_dropper_, DropFrame())
- .Times(1)
- .WillRepeatedly(Return(drop_tl0));
- if (drop_tl0) {
- EXPECT_CALL(tl1_frame_dropper_, DropFrame())
- .Times(1)
- .WillRepeatedly(Return(drop_tl1));
- }
- EXPECT_CALL(tl0_frame_dropper_, Leak(framerate))
- .Times(1);
- EXPECT_CALL(tl1_frame_dropper_, Leak(framerate))
- .Times(1);
- if (drop_tl0) {
- EXPECT_CALL(tl0_frame_dropper_, Fill(_, _))
- .Times(0);
- if (drop_tl1) {
- EXPECT_CALL(tl1_frame_dropper_, Fill(_, _))
- .Times(0);
- } else {
- EXPECT_CALL(tl1_frame_dropper_, Fill(kFrameSize, true))
- .Times(1);
- }
- } else {
- EXPECT_CALL(tl0_frame_dropper_, Fill(kFrameSize, true))
- .Times(1);
- EXPECT_CALL(tl1_frame_dropper_, Fill(kFrameSize, true))
- .Times(1);
- }
- }
+ ScreenshareLayerTest() : min_qp_(2), max_qp_(kDefaultQp), frame_size_(-1) {}
+ virtual ~ScreenshareLayerTest() {}
void EncodeFrame(uint32_t timestamp,
bool base_sync,
@@ -85,39 +42,83 @@
int* flags) {
*flags = layers_->EncodeFlags(timestamp);
layers_->PopulateCodecSpecific(base_sync, vp8_info, timestamp);
- layers_->FrameEncoded(kFrameSize, timestamp);
+ ASSERT_NE(-1, frame_size_);
+ layers_->FrameEncoded(frame_size_, timestamp, kDefaultQp);
}
- NiceMock<MockFrameDropper> tl0_frame_dropper_;
- NiceMock<MockFrameDropper> tl1_frame_dropper_;
- rtc::scoped_ptr<ScreenshareLayersFT> layers_;
+ void ConfigureBitrates() {
+ vpx_codec_enc_cfg_t vpx_cfg;
+ memset(&vpx_cfg, 0, sizeof(vpx_codec_enc_cfg_t));
+ vpx_cfg.rc_min_quantizer = min_qp_;
+ vpx_cfg.rc_max_quantizer = max_qp_;
+ EXPECT_TRUE(layers_->ConfigureBitrates(
+ kDefaultTl0BitrateKbps, kDefaultTl1BitrateKbps, kFrameRate, &vpx_cfg));
+ frame_size_ = ((vpx_cfg.rc_target_bitrate * 1000) / 8) / kFrameRate;
+ }
+
+ void WithQpLimits(int min_qp, int max_qp) {
+ min_qp_ = min_qp;
+ max_qp_ = max_qp;
+ }
+
+ int RunGracePeriod() {
+ int flags = 0;
+ uint32_t timestamp = 0;
+ CodecSpecificInfoVP8 vp8_info;
+ bool got_tl0 = false;
+ bool got_tl1 = false;
+ for (int i = 0; i < 10; ++i) {
+ EncodeFrame(timestamp, false, &vp8_info, &flags);
+ timestamp += kTimestampDelta5Fps;
+ if (vp8_info.temporalIdx == 0) {
+ got_tl0 = true;
+ } else {
+ got_tl1 = true;
+ }
+ if (got_tl0 && got_tl1)
+ return timestamp;
+ }
+ ADD_FAILURE() << "Frames from both layers not received in time.";
+ return 0;
+ }
+
+ int SkipUntilTl(int layer, int timestamp) {
+ CodecSpecificInfoVP8 vp8_info;
+ for (int i = 0; i < 5; ++i) {
+ layers_->EncodeFlags(timestamp);
+ timestamp += kTimestampDelta5Fps;
+ layers_->PopulateCodecSpecific(false, &vp8_info, timestamp);
+ if (vp8_info.temporalIdx != layer) {
+ layers_->FrameEncoded(frame_size_, timestamp, kDefaultQp);
+ } else {
+ return timestamp;
+ }
+ }
+ ADD_FAILURE() << "Did not get a frame of TL" << layer << " in time.";
+ return 0;
+ }
+
+ int min_qp_;
+ int max_qp_;
+ int frame_size_;
+ rtc::scoped_ptr<ScreenshareLayers> layers_;
};
TEST_F(ScreenshareLayerTest, 1Layer) {
- layers_.reset(
- new ScreenshareLayersFT(1, 0, &tl0_frame_dropper_, &tl1_frame_dropper_));
- EXPECT_TRUE(layers_->ConfigureBitrates(100, 1000, 5, NULL));
+ layers_.reset(new ScreenshareLayers(1, 0));
+ ConfigureBitrates();
int flags = 0;
uint32_t timestamp = 0;
CodecSpecificInfoVP8 vp8_info;
// One layer screenshare should not use the frame dropper as all frames will
// belong to the base layer.
- EXPECT_CALL(tl0_frame_dropper_, DropFrame())
- .Times(0);
- EXPECT_CALL(tl1_frame_dropper_, DropFrame())
- .Times(0);
flags = layers_->EncodeFlags(timestamp);
EXPECT_EQ(0, flags);
layers_->PopulateCodecSpecific(false, &vp8_info, timestamp);
EXPECT_EQ(static_cast<uint8_t>(kNoTemporalIdx), vp8_info.temporalIdx);
EXPECT_FALSE(vp8_info.layerSync);
EXPECT_EQ(kNoTl0PicIdx, vp8_info.tl0PicIdx);
- layers_->FrameEncoded(kFrameSize, timestamp);
-
- EXPECT_CALL(tl0_frame_dropper_, DropFrame())
- .Times(0);
- EXPECT_CALL(tl1_frame_dropper_, DropFrame())
- .Times(0);
+ layers_->FrameEncoded(frame_size_, timestamp, kDefaultQp);
flags = layers_->EncodeFlags(timestamp);
EXPECT_EQ(0, flags);
timestamp += kTimestampDelta5Fps;
@@ -125,139 +126,241 @@
EXPECT_EQ(static_cast<uint8_t>(kNoTemporalIdx), vp8_info.temporalIdx);
EXPECT_FALSE(vp8_info.layerSync);
EXPECT_EQ(kNoTl0PicIdx, vp8_info.tl0PicIdx);
- layers_->FrameEncoded(kFrameSize, timestamp);
+ layers_->FrameEncoded(frame_size_, timestamp, kDefaultQp);
}
TEST_F(ScreenshareLayerTest, 2Layer) {
- layers_.reset(
- new ScreenshareLayersFT(2, 0, &tl0_frame_dropper_, &tl1_frame_dropper_));
- EXPECT_TRUE(layers_->ConfigureBitrates(100, 1000, 5, NULL));
+ layers_.reset(new ScreenshareLayers(2, 0));
+ ConfigureBitrates();
int flags = 0;
uint32_t timestamp = 0;
uint8_t expected_tl0_idx = 0;
CodecSpecificInfoVP8 vp8_info;
- SetEncodeExpectations(false, false, 1);
EncodeFrame(timestamp, false, &vp8_info, &flags);
- EXPECT_EQ(kFlagsTL0, flags);
+ EXPECT_EQ(ScreenshareLayers::kTl0Flags, flags);
EXPECT_EQ(0, vp8_info.temporalIdx);
EXPECT_FALSE(vp8_info.layerSync);
++expected_tl0_idx;
EXPECT_EQ(expected_tl0_idx, vp8_info.tl0PicIdx);
- EXPECT_CALL(tl1_frame_dropper_, SetRates(1000, 1))
- .Times(1);
- EXPECT_TRUE(layers_->ConfigureBitrates(100, 1000, -1, NULL));
- // Insert 5 frames at 30 fps. All should belong to TL0.
+ // Insert 5 frames, cover grace period. All should be in TL0.
for (int i = 0; i < 5; ++i) {
- timestamp += kTimestampDelta30Fps;
- // First iteration has a framerate based on a single frame, thus 1.
- SetEncodeExpectations(false, false, 30);
+ timestamp += kTimestampDelta5Fps;
EncodeFrame(timestamp, false, &vp8_info, &flags);
EXPECT_EQ(0, vp8_info.temporalIdx);
EXPECT_FALSE(vp8_info.layerSync);
++expected_tl0_idx;
EXPECT_EQ(expected_tl0_idx, vp8_info.tl0PicIdx);
}
- // Drop two frames from TL0, thus being coded in TL1.
- timestamp += kTimestampDelta30Fps;
- SetEncodeExpectations(true, false, 30);
+
+ // First frame in TL0.
+ timestamp += kTimestampDelta5Fps;
EncodeFrame(timestamp, false, &vp8_info, &flags);
- EXPECT_EQ(kFlagsTL1Sync, flags);
+ EXPECT_EQ(ScreenshareLayers::kTl0Flags, flags);
+ EXPECT_EQ(0, vp8_info.temporalIdx);
+ EXPECT_FALSE(vp8_info.layerSync);
+ ++expected_tl0_idx;
+ EXPECT_EQ(expected_tl0_idx, vp8_info.tl0PicIdx);
+
+ // Drop two frames from TL0, thus being coded in TL1.
+ timestamp += kTimestampDelta5Fps;
+ EncodeFrame(timestamp, false, &vp8_info, &flags);
+ // First frame is sync frame.
+ EXPECT_EQ(ScreenshareLayers::kTl1SyncFlags, flags);
EXPECT_EQ(1, vp8_info.temporalIdx);
EXPECT_TRUE(vp8_info.layerSync);
EXPECT_EQ(expected_tl0_idx, vp8_info.tl0PicIdx);
- timestamp += kTimestampDelta30Fps;
- SetEncodeExpectations(true, false, 30);
+ timestamp += kTimestampDelta5Fps;
EncodeFrame(timestamp, false, &vp8_info, &flags);
- EXPECT_EQ(kFlagsTL1, flags);
+ EXPECT_EQ(ScreenshareLayers::kTl1Flags, flags);
EXPECT_EQ(1, vp8_info.temporalIdx);
EXPECT_FALSE(vp8_info.layerSync);
EXPECT_EQ(expected_tl0_idx, vp8_info.tl0PicIdx);
}
TEST_F(ScreenshareLayerTest, 2LayersPeriodicSync) {
- layers_.reset(
- new ScreenshareLayersFT(2, 0, &tl0_frame_dropper_, &tl1_frame_dropper_));
- EXPECT_TRUE(layers_->ConfigureBitrates(100, 1000, 5, NULL));
+ layers_.reset(new ScreenshareLayers(2, 0));
+ ConfigureBitrates();
int flags = 0;
uint32_t timestamp = 0;
CodecSpecificInfoVP8 vp8_info;
- const int kNumFrames = 10;
- const bool kDrops[kNumFrames] = {false, true, true, true, true,
- true, true, true, true, true};
- const int kExpectedFramerates[kNumFrames] = {1, 5, 5, 5, 5, 5, 5, 5, 5, 5};
- const bool kExpectedSyncs[kNumFrames] = {false, true, false, false, false,
- false, false, true, false, false};
- const int kExpectedTemporalIdx[kNumFrames] = {0, 1, 1, 1, 1, 1, 1, 1, 1, 1};
+ std::vector<int> sync_times;
+
+ const int kNumFrames = kSyncPeriodSeconds * kFrameRate * 2 - 1;
for (int i = 0; i < kNumFrames; ++i) {
timestamp += kTimestampDelta5Fps;
- SetEncodeExpectations(kDrops[i], false, kExpectedFramerates[i]);
EncodeFrame(timestamp, false, &vp8_info, &flags);
- EXPECT_EQ(kExpectedTemporalIdx[i], vp8_info.temporalIdx);
- EXPECT_EQ(kExpectedSyncs[i], vp8_info.layerSync) << "Iteration: " << i;
- EXPECT_EQ(1, vp8_info.tl0PicIdx);
+ if (vp8_info.temporalIdx == 1 && vp8_info.layerSync) {
+ sync_times.push_back(timestamp);
+ }
}
+
+ ASSERT_EQ(2u, sync_times.size());
+ EXPECT_GE(sync_times[1] - sync_times[0], 90000 * kSyncPeriodSeconds);
+}
+
+TEST_F(ScreenshareLayerTest, 2LayersSyncAfterTimeout) {
+ layers_.reset(new ScreenshareLayers(2, 0));
+ ConfigureBitrates();
+ uint32_t timestamp = 0;
+ CodecSpecificInfoVP8 vp8_info;
+ std::vector<int> sync_times;
+
+ const int kNumFrames = kMaxSyncPeriodSeconds * kFrameRate * 2 - 1;
+ for (int i = 0; i < kNumFrames; ++i) {
+ timestamp += kTimestampDelta5Fps;
+ layers_->EncodeFlags(timestamp);
+ layers_->PopulateCodecSpecific(false, &vp8_info, timestamp);
+
+ // Simulate TL1 being at least 8 qp steps better.
+ if (vp8_info.temporalIdx == 0) {
+ layers_->FrameEncoded(frame_size_, timestamp, kDefaultQp);
+ } else {
+ layers_->FrameEncoded(frame_size_, timestamp, kDefaultQp - 8);
+ }
+
+ if (vp8_info.temporalIdx == 1 && vp8_info.layerSync)
+ sync_times.push_back(timestamp);
+ }
+
+ ASSERT_EQ(2u, sync_times.size());
+ EXPECT_GE(sync_times[1] - sync_times[0], 90000 * kMaxSyncPeriodSeconds);
+}
+
+TEST_F(ScreenshareLayerTest, 2LayersSyncAfterSimilarQP) {
+ layers_.reset(new ScreenshareLayers(2, 0));
+ ConfigureBitrates();
+ uint32_t timestamp = 0;
+ CodecSpecificInfoVP8 vp8_info;
+ std::vector<int> sync_times;
+
+ const int kNumFrames = (kSyncPeriodSeconds +
+ ((kMaxSyncPeriodSeconds - kSyncPeriodSeconds) / 2)) *
+ kFrameRate;
+ for (int i = 0; i < kNumFrames; ++i) {
+ timestamp += kTimestampDelta5Fps;
+ layers_->EncodeFlags(timestamp);
+ layers_->PopulateCodecSpecific(false, &vp8_info, timestamp);
+
+ // Simulate TL1 being at least 8 qp steps better.
+ if (vp8_info.temporalIdx == 0) {
+ layers_->FrameEncoded(frame_size_, timestamp, kDefaultQp);
+ } else {
+ layers_->FrameEncoded(frame_size_, timestamp, kDefaultQp - 8);
+ }
+
+ if (vp8_info.temporalIdx == 1 && vp8_info.layerSync)
+ sync_times.push_back(timestamp);
+ }
+
+ ASSERT_EQ(1u, sync_times.size());
+
+ bool bumped_tl0_quality = false;
+ for (int i = 0; i < 3; ++i) {
+ timestamp += kTimestampDelta5Fps;
+ int flags = layers_->EncodeFlags(timestamp);
+ layers_->PopulateCodecSpecific(false, &vp8_info, timestamp);
+
+ if (vp8_info.temporalIdx == 0) {
+ // Bump TL0 to same quality as TL1.
+ layers_->FrameEncoded(frame_size_, timestamp, kDefaultQp - 8);
+ bumped_tl0_quality = true;
+ } else {
+ layers_->FrameEncoded(frame_size_, timestamp, kDefaultQp - 8);
+ if (bumped_tl0_quality) {
+ EXPECT_TRUE(vp8_info.layerSync);
+ EXPECT_EQ(ScreenshareLayers::kTl1SyncFlags, flags);
+ return;
+ }
+ }
+ }
+ ADD_FAILURE() << "No TL1 frame arrived within time limit.";
}
TEST_F(ScreenshareLayerTest, 2LayersToggling) {
- layers_.reset(
- new ScreenshareLayersFT(2, 0, &tl0_frame_dropper_, &tl1_frame_dropper_));
- EXPECT_TRUE(layers_->ConfigureBitrates(100, 1000, 5, NULL));
+ layers_.reset(new ScreenshareLayers(2, 0));
+ ConfigureBitrates();
+ int flags = 0;
+ CodecSpecificInfoVP8 vp8_info;
+ uint32_t timestamp = RunGracePeriod();
+
+ // Insert 50 frames. 2/5 should be TL0.
+ int tl0_frames = 0;
+ int tl1_frames = 0;
+ for (int i = 0; i < 50; ++i) {
+ timestamp += kTimestampDelta5Fps;
+ EncodeFrame(timestamp, false, &vp8_info, &flags);
+ switch (vp8_info.temporalIdx) {
+ case 0:
+ ++tl0_frames;
+ break;
+ case 1:
+ ++tl1_frames;
+ break;
+ default:
+ abort();
+ }
+ }
+ EXPECT_EQ(20, tl0_frames);
+ EXPECT_EQ(30, tl1_frames);
+}
+
+TEST_F(ScreenshareLayerTest, AllFitsLayer0) {
+ layers_.reset(new ScreenshareLayers(2, 0));
+ ConfigureBitrates();
+ frame_size_ = ((kDefaultTl0BitrateKbps * 1000) / 8) / kFrameRate;
+
int flags = 0;
uint32_t timestamp = 0;
CodecSpecificInfoVP8 vp8_info;
- const int kNumFrames = 10;
- const bool kDrops[kNumFrames] = {false, true, false, true, false,
- true, false, true, false, true};
- const int kExpectedFramerates[kNumFrames] = {1, 5, 5, 5, 5, 5, 5, 5, 5, 5};
- const bool kExpectedSyncs[kNumFrames] = {false, true, false, false, false,
- false, false, true, false, false};
- const int kExpectedTemporalIdx[kNumFrames] = {0, 1, 0, 1, 0, 1, 0, 1, 0, 1};
- const int kExpectedTl0Idx[kNumFrames] = {1, 1, 2, 2, 3, 3, 4, 4, 5, 5};
- for (int i = 0; i < kNumFrames; ++i) {
- timestamp += kTimestampDelta5Fps;
- SetEncodeExpectations(kDrops[i], false, kExpectedFramerates[i]);
+ // Insert 50 frames, small enough that all fits in TL0.
+ for (int i = 0; i < 50; ++i) {
EncodeFrame(timestamp, false, &vp8_info, &flags);
- EXPECT_EQ(kExpectedTemporalIdx[i], vp8_info.temporalIdx);
- EXPECT_EQ(kExpectedSyncs[i], vp8_info.layerSync) << "Iteration: " << i;
- EXPECT_EQ(kExpectedTl0Idx[i], vp8_info.tl0PicIdx);
+ timestamp += kTimestampDelta5Fps;
+ EXPECT_EQ(ScreenshareLayers::kTl0Flags, flags);
+ EXPECT_EQ(0, vp8_info.temporalIdx);
}
}
-TEST_F(ScreenshareLayerTest, 2LayersBothDrops) {
- layers_.reset(
- new ScreenshareLayersFT(2, 0, &tl0_frame_dropper_, &tl1_frame_dropper_));
- EXPECT_TRUE(layers_->ConfigureBitrates(100, 1000, 5, NULL));
+TEST_F(ScreenshareLayerTest, TooHighBitrate) {
+ layers_.reset(new ScreenshareLayers(2, 0));
+ ConfigureBitrates();
+ frame_size_ = 2 * ((kDefaultTl1BitrateKbps * 1000) / 8) / kFrameRate;
int flags = 0;
- uint32_t timestamp = 0;
- uint8_t expected_tl0_idx = 0;
CodecSpecificInfoVP8 vp8_info;
- SetEncodeExpectations(false, false, 1);
- EncodeFrame(timestamp, false, &vp8_info, &flags);
- EXPECT_EQ(kFlagsTL0, flags);
- EXPECT_EQ(0, vp8_info.temporalIdx);
- EXPECT_FALSE(vp8_info.layerSync);
- ++expected_tl0_idx;
- EXPECT_EQ(expected_tl0_idx, vp8_info.tl0PicIdx);
+ uint32_t timestamp = RunGracePeriod();
- timestamp += kTimestampDelta5Fps;
- SetEncodeExpectations(true, false, 5);
- EncodeFrame(timestamp, false, &vp8_info, &flags);
- EXPECT_EQ(kFlagsTL1Sync, flags);
- EXPECT_EQ(1, vp8_info.temporalIdx);
- EXPECT_TRUE(vp8_info.layerSync);
- EXPECT_EQ(expected_tl0_idx, vp8_info.tl0PicIdx);
+ // Insert 100 frames. Half should be dropped.
+ int tl0_frames = 0;
+ int tl1_frames = 0;
+ int dropped_frames = 0;
+ for (int i = 0; i < 100; ++i) {
+ timestamp += kTimestampDelta5Fps;
+ EncodeFrame(timestamp, false, &vp8_info, &flags);
+ if (flags == -1) {
+ ++dropped_frames;
+ } else {
+ switch (vp8_info.temporalIdx) {
+ case 0:
+ ++tl0_frames;
+ break;
+ case 1:
+ ++tl1_frames;
+ break;
+ default:
+ abort();
+ }
+ }
+ }
- timestamp += kTimestampDelta5Fps;
- SetEncodeExpectations(true, true, 5);
- flags = layers_->EncodeFlags(timestamp);
- EXPECT_EQ(-1, flags);
+ EXPECT_EQ(5, tl0_frames);
+ EXPECT_EQ(45, tl1_frames);
+ EXPECT_EQ(50, dropped_frames);
}
TEST_F(ScreenshareLayerTest, TargetBitrateCappedByTL0) {
- layers_.reset(
- new ScreenshareLayersFT(2, 0, &tl0_frame_dropper_, &tl1_frame_dropper_));
+ layers_.reset(new ScreenshareLayers(2, 0));
vpx_codec_enc_cfg_t cfg;
layers_->ConfigureBitrates(100, 1000, 5, &cfg);
@@ -268,8 +371,7 @@
}
TEST_F(ScreenshareLayerTest, TargetBitrateCappedByTL1) {
- layers_.reset(
- new ScreenshareLayersFT(2, 0, &tl0_frame_dropper_, &tl1_frame_dropper_));
+ layers_.reset(new ScreenshareLayers(2, 0));
vpx_codec_enc_cfg_t cfg;
layers_->ConfigureBitrates(100, 450, 5, &cfg);
@@ -279,12 +381,64 @@
}
TEST_F(ScreenshareLayerTest, TargetBitrateBelowTL0) {
- layers_.reset(
- new ScreenshareLayersFT(2, 0, &tl0_frame_dropper_, &tl1_frame_dropper_));
+ layers_.reset(new ScreenshareLayers(2, 0));
vpx_codec_enc_cfg_t cfg;
layers_->ConfigureBitrates(100, 100, 5, &cfg);
EXPECT_EQ(100U, cfg.rc_target_bitrate);
}
+TEST_F(ScreenshareLayerTest, EncoderDrop) {
+ layers_.reset(new ScreenshareLayers(2, 0));
+ ConfigureBitrates();
+ CodecSpecificInfoVP8 vp8_info;
+ vpx_codec_enc_cfg_t cfg;
+ cfg.rc_max_quantizer = kDefaultQp;
+
+ uint32_t timestamp = RunGracePeriod();
+ timestamp = SkipUntilTl(0, timestamp);
+
+ // Size 0 indicates dropped frame.
+ layers_->FrameEncoded(0, timestamp, kDefaultQp);
+ timestamp += kTimestampDelta5Fps;
+ EXPECT_FALSE(layers_->UpdateConfiguration(&cfg));
+ EXPECT_EQ(ScreenshareLayers::kTl0Flags, layers_->EncodeFlags(timestamp));
+ layers_->PopulateCodecSpecific(false, &vp8_info, timestamp);
+ layers_->FrameEncoded(frame_size_, timestamp, kDefaultQp);
+
+ timestamp = SkipUntilTl(0, timestamp);
+ EXPECT_TRUE(layers_->UpdateConfiguration(&cfg));
+ EXPECT_LT(cfg.rc_max_quantizer, static_cast<unsigned int>(kDefaultQp));
+ layers_->FrameEncoded(frame_size_, timestamp, kDefaultQp);
+
+ layers_->EncodeFlags(timestamp);
+ timestamp += kTimestampDelta5Fps;
+ EXPECT_TRUE(layers_->UpdateConfiguration(&cfg));
+ layers_->PopulateCodecSpecific(false, &vp8_info, timestamp);
+ EXPECT_EQ(cfg.rc_max_quantizer, static_cast<unsigned int>(kDefaultQp));
+ layers_->FrameEncoded(frame_size_, timestamp, kDefaultQp);
+
+ // Next drop in TL1.
+
+ timestamp = SkipUntilTl(1, timestamp);
+ layers_->FrameEncoded(0, timestamp, kDefaultQp);
+ timestamp += kTimestampDelta5Fps;
+ EXPECT_FALSE(layers_->UpdateConfiguration(&cfg));
+ EXPECT_EQ(ScreenshareLayers::kTl1Flags, layers_->EncodeFlags(timestamp));
+ layers_->PopulateCodecSpecific(false, &vp8_info, timestamp);
+ layers_->FrameEncoded(frame_size_, timestamp, kDefaultQp);
+
+ timestamp = SkipUntilTl(1, timestamp);
+ EXPECT_TRUE(layers_->UpdateConfiguration(&cfg));
+ EXPECT_LT(cfg.rc_max_quantizer, static_cast<unsigned int>(kDefaultQp));
+ layers_->FrameEncoded(frame_size_, timestamp, kDefaultQp);
+
+ layers_->EncodeFlags(timestamp);
+ timestamp += kTimestampDelta5Fps;
+ EXPECT_TRUE(layers_->UpdateConfiguration(&cfg));
+ layers_->PopulateCodecSpecific(false, &vp8_info, timestamp);
+ EXPECT_EQ(cfg.rc_max_quantizer, static_cast<unsigned int>(kDefaultQp));
+ layers_->FrameEncoded(frame_size_, timestamp, kDefaultQp);
+}
+
} // namespace webrtc
diff --git a/webrtc/modules/video_coding/codecs/vp8/simulcast_encoder_adapter.cc b/webrtc/modules/video_coding/codecs/vp8/simulcast_encoder_adapter.cc
index 280296c..ee7fd85 100644
--- a/webrtc/modules/video_coding/codecs/vp8/simulcast_encoder_adapter.cc
+++ b/webrtc/modules/video_coding/codecs/vp8/simulcast_encoder_adapter.cc
@@ -102,10 +102,7 @@
virtual webrtc::TemporalLayers* Create(int num_temporal_layers,
uint8_t initial_tl0_pic_idx) const {
- return new webrtc::ScreenshareLayers(num_temporal_layers,
- rand(),
- &tl0_frame_dropper_,
- &tl1_frame_dropper_);
+ return new webrtc::ScreenshareLayers(num_temporal_layers, rand());
}
mutable webrtc::FrameDropper tl0_frame_dropper_;
diff --git a/webrtc/modules/video_coding/codecs/vp8/simulcast_unittest.h b/webrtc/modules/video_coding/codecs/vp8/simulcast_unittest.h
index fffe4ab..2e436a9 100644
--- a/webrtc/modules/video_coding/codecs/vp8/simulcast_unittest.h
+++ b/webrtc/modules/video_coding/codecs/vp8/simulcast_unittest.h
@@ -196,12 +196,16 @@
layers_->PopulateCodecSpecific(base_layer_sync, vp8_info, timestamp);
}
- void FrameEncoded(unsigned int size, uint32_t timestamp) override {
- layers_->FrameEncoded(size, timestamp);
+ void FrameEncoded(unsigned int size, uint32_t timestamp, int qp) override {
+ layers_->FrameEncoded(size, timestamp, qp);
}
int CurrentLayerId() const override { return layers_->CurrentLayerId(); }
+ bool UpdateConfiguration(vpx_codec_enc_cfg_t* cfg) override {
+ return false;
+ }
+
int configured_bitrate_;
TemporalLayers* layers_;
};
diff --git a/webrtc/modules/video_coding/codecs/vp8/temporal_layers.h b/webrtc/modules/video_coding/codecs/vp8/temporal_layers.h
index b3b73a6..7607210 100644
--- a/webrtc/modules/video_coding/codecs/vp8/temporal_layers.h
+++ b/webrtc/modules/video_coding/codecs/vp8/temporal_layers.h
@@ -47,9 +47,11 @@
CodecSpecificInfoVP8* vp8_info,
uint32_t timestamp) = 0;
- virtual void FrameEncoded(unsigned int size, uint32_t timestamp) = 0;
+ virtual void FrameEncoded(unsigned int size, uint32_t timestamp, int qp) = 0;
virtual int CurrentLayerId() const = 0;
+
+ virtual bool UpdateConfiguration(vpx_codec_enc_cfg_t* cfg) = 0;
};
// Factory for a temporal layers strategy that adaptively changes the number of
diff --git a/webrtc/modules/video_coding/codecs/vp8/vp8_impl.cc b/webrtc/modules/video_coding/codecs/vp8/vp8_impl.cc
index 4370424..05e0799 100644
--- a/webrtc/modules/video_coding/codecs/vp8/vp8_impl.cc
+++ b/webrtc/modules/video_coding/codecs/vp8/vp8_impl.cc
@@ -278,7 +278,6 @@
int tl0_bitrate = std::min(codec_.targetBitrate, target_bitrate);
max_bitrate = std::min(codec_.maxBitrate, target_bitrate);
target_bitrate = tl0_bitrate;
- framerate = -1;
}
configurations_[i].rc_target_bitrate = target_bitrate;
temporal_layers_[stream_idx]->ConfigureBitrates(target_bitrate,
@@ -312,10 +311,8 @@
if (num_streams == 1) {
if (codec.mode == kScreensharing) {
// Special mode when screensharing on a single stream.
- temporal_layers_.push_back(new ScreenshareLayers(num_temporal_layers,
- rand(),
- &tl0_frame_dropper_,
- &tl1_frame_dropper_));
+ temporal_layers_.push_back(
+ new ScreenshareLayers(num_temporal_layers, rand()));
} else {
temporal_layers_.push_back(
tl_factory.Create(num_temporal_layers, rand()));
@@ -670,8 +667,10 @@
static_cast<vp8e_token_partitions>(token_partitions_));
vpx_codec_control(&(encoders_[i]), VP8E_SET_MAX_INTRA_BITRATE_PCT,
rc_max_intra_target_);
+ // VP8E_SET_SCREEN_CONTENT_MODE 2 = screen content with more aggressive
+ // rate control (drop frames on large target bitrate overshoot)
vpx_codec_control(&(encoders_[i]), VP8E_SET_SCREEN_CONTENT_MODE,
- codec_.mode == kScreensharing);
+ codec_.mode == kScreensharing ? 2 : 0);
}
inited_ = true;
return WEBRTC_VIDEO_CODEC_OK;
@@ -698,15 +697,12 @@
const std::vector<VideoFrameType>* frame_types) {
TRACE_EVENT1("webrtc", "VP8::Encode", "timestamp", frame.timestamp());
- if (!inited_) {
+ if (!inited_)
return WEBRTC_VIDEO_CODEC_UNINITIALIZED;
- }
- if (frame.IsZeroSize()) {
+ if (frame.IsZeroSize())
return WEBRTC_VIDEO_CODEC_ERR_PARAMETER;
- }
- if (encoded_complete_callback_ == NULL) {
+ if (encoded_complete_callback_ == NULL)
return WEBRTC_VIDEO_CODEC_UNINITIALIZED;
- }
// Only apply scaling to improve for single-layer streams. The scaling metrics
// use frame drops as a signal and is only applicable when we drop frames.
@@ -851,6 +847,16 @@
// whereas |encoder_| is from highest to lowest resolution.
size_t stream_idx = encoders_.size() - 1;
for (size_t i = 0; i < encoders_.size(); ++i, --stream_idx) {
+ // Allow the layers adapter to temporarily modify the configuration. This
+ // change isn't stored in configurations_ so change will be discarded at
+ // the next update.
+ vpx_codec_enc_cfg_t temp_config;
+ memcpy(&temp_config, &configurations_[i], sizeof(vpx_codec_enc_cfg_t));
+ if (temporal_layers_[stream_idx]->UpdateConfiguration(&temp_config)) {
+ if (vpx_codec_enc_config_set(&encoders_[i], &temp_config))
+ return WEBRTC_VIDEO_CODEC_ERROR;
+ }
+
vpx_codec_control(&encoders_[i], VP8E_SET_FRAME_FLAGS, flags[stream_idx]);
vpx_codec_control(&encoders_[i],
VP8E_SET_TEMPORAL_LAYER_ID,
@@ -873,9 +879,8 @@
vpx_codec_control(&(encoders_[0]), VP8E_SET_MAX_INTRA_BITRATE_PCT,
rc_max_intra_target_);
}
- if (error) {
+ if (error)
return WEBRTC_VIDEO_CODEC_ERROR;
- }
timestamp_ += duration;
return GetEncodedPartitions(input_image, only_predict_from_key_frame);
}
@@ -933,6 +938,7 @@
int VP8EncoderImpl::GetEncodedPartitions(const VideoFrame& input_image,
bool only_predicting_from_key_frame) {
int stream_idx = static_cast<int>(encoders_.size()) - 1;
+ int result = WEBRTC_VIDEO_CODEC_OK;
for (size_t encoder_idx = 0; encoder_idx < encoders_.size();
++encoder_idx, --stream_idx) {
vpx_codec_iter_t iter = NULL;
@@ -981,9 +987,12 @@
encoded_images_[encoder_idx]._timeStamp = input_image.timestamp();
encoded_images_[encoder_idx].capture_time_ms_ =
input_image.render_time_ms();
+
+ int qp = -1;
+ vpx_codec_control(&encoders_[encoder_idx], VP8E_GET_LAST_QUANTIZER_64, &qp);
temporal_layers_[stream_idx]->FrameEncoded(
encoded_images_[encoder_idx]._length,
- encoded_images_[encoder_idx]._timeStamp);
+ encoded_images_[encoder_idx]._timeStamp, qp);
if (send_stream_[stream_idx]) {
if (encoded_images_[encoder_idx]._length > 0) {
TRACE_COUNTER_ID1("webrtc", "EncodedFrameSize", encoder_idx,
@@ -994,6 +1003,8 @@
codec_.simulcastStream[stream_idx].width;
encoded_complete_callback_->Encoded(encoded_images_[encoder_idx],
&codec_specific, &frag_info);
+ } else if (codec_.mode == kScreensharing) {
+ result = WEBRTC_VIDEO_CODEC_TARGET_BITRATE_OVERSHOOT;
}
} else {
// Required in case padding is applied to dropped frames.
@@ -1017,7 +1028,7 @@
quality_scaler_.ReportDroppedFrame();
}
}
- return WEBRTC_VIDEO_CODEC_OK;
+ return result;
}
int VP8EncoderImpl::SetChannelParameters(uint32_t packetLoss, int64_t rtt) {
diff --git a/webrtc/modules/video_coding/main/source/generic_encoder.cc b/webrtc/modules/video_coding/main/source/generic_encoder.cc
index 1091501..f2cdd59 100644
--- a/webrtc/modules/video_coding/main/source/generic_encoder.cc
+++ b/webrtc/modules/video_coding/main/source/generic_encoder.cc
@@ -60,7 +60,8 @@
bit_rate_(0),
frame_rate_(0),
internal_source_(internalSource),
- rotation_(kVideoRotation_0) {
+ rotation_(kVideoRotation_0),
+ is_screenshare_(false) {
}
VCMGenericEncoder::~VCMGenericEncoder()
@@ -90,6 +91,7 @@
frame_rate_ = settings->maxFramerate;
}
+ is_screenshare_ = settings->mode == VideoCodecMode::kScreensharing;
if (encoder_->InitEncode(settings, numberOfCores, maxPayloadSize) != 0) {
LOG(LS_ERROR) << "Failed to initialize the encoder associated with "
"payload name: " << settings->plName;
@@ -114,7 +116,15 @@
vcm_encoded_frame_callback_->SetRotation(rotation_);
}
- return encoder_->Encode(inputFrame, codecSpecificInfo, &video_frame_types);
+ int32_t result =
+ encoder_->Encode(inputFrame, codecSpecificInfo, &video_frame_types);
+ if (is_screenshare_ &&
+ result == WEBRTC_VIDEO_CODEC_TARGET_BITRATE_OVERSHOOT) {
+ // Target bitrate exceeded, encoder state has been reset - try again.
+ return encoder_->Encode(inputFrame, codecSpecificInfo, &video_frame_types);
+ }
+
+ return result;
}
int32_t
diff --git a/webrtc/modules/video_coding/main/source/generic_encoder.h b/webrtc/modules/video_coding/main/source/generic_encoder.h
index a722772..862c06b 100644
--- a/webrtc/modules/video_coding/main/source/generic_encoder.h
+++ b/webrtc/modules/video_coding/main/source/generic_encoder.h
@@ -149,6 +149,7 @@
const bool internal_source_;
mutable rtc::CriticalSection rates_lock_;
VideoRotation rotation_;
+ bool is_screenshare_;
}; // end of VCMGenericEncoder class
} // namespace webrtc
diff --git a/webrtc/video/full_stack.cc b/webrtc/video/full_stack.cc
index 63341b1..f0c5d79 100644
--- a/webrtc/video/full_stack.cc
+++ b/webrtc/video/full_stack.cc
@@ -649,8 +649,8 @@
{"screenshare_slides", 1850, 1110, 5},
true,
50000,
- 100000,
- 1000000,
+ 200000,
+ 2000000,
0.0,
0.0,
kFullStackTestDurationSecs};
diff --git a/webrtc/video/screenshare_loopback.cc b/webrtc/video/screenshare_loopback.cc
index c0f96d4..f2133a2 100644
--- a/webrtc/video/screenshare_loopback.cc
+++ b/webrtc/video/screenshare_loopback.cc
@@ -45,12 +45,12 @@
return static_cast<size_t>(FLAGS_min_bitrate);
}
-DEFINE_int32(tl0_bitrate, 100, "Temporal layer 0 target bitrate.");
+DEFINE_int32(tl0_bitrate, 200, "Temporal layer 0 target bitrate.");
size_t StartBitrate() {
return static_cast<size_t>(FLAGS_tl0_bitrate);
}
-DEFINE_int32(tl1_bitrate, 1000, "Temporal layer 1 target bitrate.");
+DEFINE_int32(tl1_bitrate, 2000, "Temporal layer 1 target bitrate.");
size_t MaxBitrate() {
return static_cast<size_t>(FLAGS_tl1_bitrate);
}
