Allow for framerate reduction for HW encoder.
R=pbos@webrtc.org, stefan@webrtc.org
TBR=glaznev@google.com
Review URL: https://webrtc-codereview.appspot.com/51159004 .
Cr-Commit-Position: refs/heads/master@{#9573}
diff --git a/talk/app/webrtc/java/jni/androidmediaencoder_jni.cc b/talk/app/webrtc/java/jni/androidmediaencoder_jni.cc
index 4db089c..25ffe91 100644
--- a/talk/app/webrtc/java/jni/androidmediaencoder_jni.cc
+++ b/talk/app/webrtc/java/jni/androidmediaencoder_jni.cc
@@ -100,6 +100,8 @@
void OnDroppedFrame() override;
+ int GetTargetFramerate() override;
+
private:
// CHECK-fail if not running on |codec_thread_|.
void CheckOnCodecThread();
@@ -199,6 +201,7 @@
scoped_ptr<webrtc::QualityScaler> quality_scaler_;
// Dynamic resolution change, off by default.
bool scale_;
+ int updated_framerate_;
};
MediaCodecVideoEncoder::~MediaCodecVideoEncoder() {
@@ -294,11 +297,12 @@
ALOGD("InitEncode request");
scale_ = false;
- if (codecType_ == kVideoCodecVP8) {
- quality_scaler_->Init(kMaxQP / kLowQpThresholdDenominator);
+ if (scale_ && codecType_ == kVideoCodecVP8) {
+ quality_scaler_->Init(kMaxQP / kLowQpThresholdDenominator, true);
quality_scaler_->SetMinResolution(kMinWidth, kMinHeight);
quality_scaler_->ReportFramerate(codec_settings->maxFramerate);
}
+ updated_framerate_ = codec_settings->maxFramerate;
return codec_thread_->Invoke<int32_t>(
Bind(&MediaCodecVideoEncoder::InitEncodeOnCodecThread,
this,
@@ -337,8 +341,11 @@
int32_t MediaCodecVideoEncoder::SetRates(uint32_t new_bit_rate,
uint32_t frame_rate) {
- if (codecType_ == kVideoCodecVP8)
+ if (scale_ && codecType_ == kVideoCodecVP8) {
quality_scaler_->ReportFramerate(frame_rate);
+ } else {
+ updated_framerate_ = frame_rate;
+ }
return codec_thread_->Invoke<int32_t>(
Bind(&MediaCodecVideoEncoder::SetRatesOnCodecThread,
this,
@@ -493,9 +500,13 @@
}
CHECK(frame_types->size() == 1) << "Unexpected stream count";
- const VideoFrame& input_frame = (scale_ && codecType_ == kVideoCodecVP8)
- ? quality_scaler_->GetScaledFrame(frame)
- : frame;
+ // Check framerate before spatial resolution change.
+ if (scale_ && codecType_ == kVideoCodecVP8) {
+ quality_scaler_->OnEncodeFrame(frame);
+ updated_framerate_ = quality_scaler_->GetTargetFramerate();
+ }
+ const VideoFrame& input_frame = (scale_ && codecType_ == kVideoCodecVP8) ?
+ quality_scaler_->GetScaledFrame(frame) : frame;
if (input_frame.width() != width_ || input_frame.height() != height_) {
ALOGD("Frame resolution change from %d x %d to %d x %d",
@@ -506,8 +517,6 @@
return WEBRTC_VIDEO_CODEC_OK;
}
- bool key_frame = frame_types->front() != webrtc::kDeltaFrame;
-
// Check if we accumulated too many frames in encoder input buffers
// or the encoder latency exceeds 70 ms and drop frame if so.
if (frames_in_queue_ > 0 && last_input_timestamp_ms_ >= 0) {
@@ -566,6 +575,7 @@
render_times_ms_.push_back(input_frame.render_time_ms());
frame_rtc_times_ms_.push_back(GetCurrentTimeMs());
+ bool key_frame = frame_types->front() != webrtc::kDeltaFrame;
bool encode_status = jni->CallBooleanMethod(*j_media_codec_video_encoder_,
j_encode_method_,
key_frame,
@@ -712,7 +722,7 @@
last_input_timestamp_ms_ - last_output_timestamp_ms_,
frame_encoding_time_ms);
- if (payload_size && codecType_ == kVideoCodecVP8)
+ if (payload_size && scale_ && codecType_ == kVideoCodecVP8)
quality_scaler_->ReportQP(webrtc::vp8::GetQP(payload));
// Calculate and print encoding statistics - every 3 seconds.
@@ -860,10 +870,14 @@
}
void MediaCodecVideoEncoder::OnDroppedFrame() {
- if (codecType_ == kVideoCodecVP8)
+ if (scale_ && codecType_ == kVideoCodecVP8)
quality_scaler_->ReportDroppedFrame();
}
+int MediaCodecVideoEncoder::GetTargetFramerate() {
+ return updated_framerate_;
+}
+
MediaCodecVideoEncoderFactory::MediaCodecVideoEncoderFactory() {
JNIEnv* jni = AttachCurrentThreadIfNeeded();
ScopedLocalRefFrame local_ref_frame(jni);
diff --git a/webrtc/modules/video_coding/codecs/vp8/simulcast_encoder_adapter.cc b/webrtc/modules/video_coding/codecs/vp8/simulcast_encoder_adapter.cc
index ee7fd85..7c4164c 100644
--- a/webrtc/modules/video_coding/codecs/vp8/simulcast_encoder_adapter.cc
+++ b/webrtc/modules/video_coding/codecs/vp8/simulcast_encoder_adapter.cc
@@ -504,4 +504,8 @@
streaminfos_[0].encoder->OnDroppedFrame();
}
+int SimulcastEncoderAdapter::GetTargetFramerate() {
+ return streaminfos_[0].encoder->GetTargetFramerate();
+}
+
} // namespace webrtc
diff --git a/webrtc/modules/video_coding/codecs/vp8/simulcast_encoder_adapter.h b/webrtc/modules/video_coding/codecs/vp8/simulcast_encoder_adapter.h
index cd78234..bca1e00 100644
--- a/webrtc/modules/video_coding/codecs/vp8/simulcast_encoder_adapter.h
+++ b/webrtc/modules/video_coding/codecs/vp8/simulcast_encoder_adapter.h
@@ -57,6 +57,8 @@
void OnDroppedFrame() override;
+ int GetTargetFramerate() override;
+
private:
struct StreamInfo {
StreamInfo()
diff --git a/webrtc/modules/video_coding/codecs/vp8/vp8_impl.cc b/webrtc/modules/video_coding/codecs/vp8/vp8_impl.cc
index 05e0799..31b92ea 100644
--- a/webrtc/modules/video_coding/codecs/vp8/vp8_impl.cc
+++ b/webrtc/modules/video_coding/codecs/vp8/vp8_impl.cc
@@ -580,7 +580,8 @@
}
rps_.Init();
- quality_scaler_.Init(codec_.qpMax / QualityScaler::kDefaultLowQpDenominator);
+ quality_scaler_.Init(codec_.qpMax / QualityScaler::kDefaultLowQpDenominator,
+ false);
quality_scaler_.ReportFramerate(codec_.maxFramerate);
return InitAndSetControlSettings();
@@ -709,6 +710,8 @@
const bool use_quality_scaler = encoders_.size() == 1 &&
configurations_[0].rc_dropframe_thresh > 0 &&
codec_.codecSpecific.VP8.automaticResizeOn;
+ if (use_quality_scaler)
+ quality_scaler_.OnEncodeFrame(frame);
const VideoFrame& input_image =
use_quality_scaler ? quality_scaler_.GetScaledFrame(frame) : frame;
diff --git a/webrtc/modules/video_coding/main/interface/video_coding_defines.h b/webrtc/modules/video_coding/main/interface/video_coding_defines.h
index 259e75e..d40023c 100644
--- a/webrtc/modules/video_coding/main/interface/video_coding_defines.h
+++ b/webrtc/modules/video_coding/main/interface/video_coding_defines.h
@@ -182,6 +182,8 @@
const uint32_t width,
const uint32_t height) = 0;
+ virtual void SetTargetFramerate(int frame_rate) = 0;
+
protected:
virtual ~VCMQMSettingsCallback() {
}
diff --git a/webrtc/modules/video_coding/main/source/generic_encoder.cc b/webrtc/modules/video_coding/main/source/generic_encoder.cc
index f2cdd59..63e3976 100644
--- a/webrtc/modules/video_coding/main/source/generic_encoder.cc
+++ b/webrtc/modules/video_coding/main/source/generic_encoder.cc
@@ -214,6 +214,10 @@
return encoder_->SupportsNativeHandle();
}
+int VCMGenericEncoder::GetTargetFramerate() {
+ return encoder_->GetTargetFramerate();
+}
+
/***************************
* Callback Implementation
***************************/
diff --git a/webrtc/modules/video_coding/main/source/generic_encoder.h b/webrtc/modules/video_coding/main/source/generic_encoder.h
index 862c06b..25235b6 100644
--- a/webrtc/modules/video_coding/main/source/generic_encoder.h
+++ b/webrtc/modules/video_coding/main/source/generic_encoder.h
@@ -140,6 +140,8 @@
bool SupportsNativeHandle() const;
+ int GetTargetFramerate();
+
private:
VideoEncoder* const encoder_;
VideoEncoderRateObserver* const rate_observer_;
diff --git a/webrtc/modules/video_coding/main/source/video_sender.cc b/webrtc/modules/video_coding/main/source/video_sender.cc
index fb04b0e..93d8208 100644
--- a/webrtc/modules/video_coding/main/source/video_sender.cc
+++ b/webrtc/modules/video_coding/main/source/video_sender.cc
@@ -363,6 +363,8 @@
for (size_t i = 0; i < _nextFrameTypes.size(); ++i) {
_nextFrameTypes[i] = kVideoFrameDelta; // Default frame type.
}
+ if (qm_settings_callback_)
+ qm_settings_callback_->SetTargetFramerate(_encoder->GetTargetFramerate());
return VCM_OK;
}
diff --git a/webrtc/modules/video_coding/utility/include/quality_scaler.h b/webrtc/modules/video_coding/utility/include/quality_scaler.h
index 1d6c917..8535171 100644
--- a/webrtc/modules/video_coding/utility/include/quality_scaler.h
+++ b/webrtc/modules/video_coding/utility/include/quality_scaler.h
@@ -25,14 +25,16 @@
};
QualityScaler();
- void Init(int low_qp_threshold);
+ void Init(int low_qp_threshold, bool use_framerate_reduction);
void SetMinResolution(int min_width, int min_height);
void ReportFramerate(int framerate);
void ReportQP(int qp);
void ReportDroppedFrame();
void Reset(int framerate, int bitrate, int width, int height);
- Resolution GetScaledResolution(const VideoFrame& frame);
+ void OnEncodeFrame(const VideoFrame& frame);
+ Resolution GetScaledResolution() const;
const VideoFrame& GetScaledFrame(const VideoFrame& frame);
+ int GetTargetFramerate() const;
private:
void AdjustScale(bool up);
@@ -42,11 +44,16 @@
VideoFrame scaled_frame_;
size_t num_samples_;
+ int framerate_;
+ int target_framerate_;
int low_qp_threshold_;
MovingAverage<int> framedrop_percent_;
MovingAverage<int> average_qp_;
+ Resolution res_;
int downscale_shift_;
+ int framerate_down_;
+ bool use_framerate_reduction_;
int min_width_;
int min_height_;
};
diff --git a/webrtc/modules/video_coding/utility/quality_scaler.cc b/webrtc/modules/video_coding/utility/quality_scaler.cc
index 7a2a9c0..f7d4651 100644
--- a/webrtc/modules/video_coding/utility/quality_scaler.cc
+++ b/webrtc/modules/video_coding/utility/quality_scaler.cc
@@ -24,13 +24,16 @@
: num_samples_(0),
low_qp_threshold_(-1),
downscale_shift_(0),
+ framerate_down_(false),
min_width_(kDefaultMinDownscaleDimension),
min_height_(kDefaultMinDownscaleDimension) {
}
-void QualityScaler::Init(int low_qp_threshold) {
+void QualityScaler::Init(int low_qp_threshold, bool use_framerate_reduction) {
ClearSamples();
low_qp_threshold_ = low_qp_threshold;
+ use_framerate_reduction_ = use_framerate_reduction;
+ target_framerate_ = -1;
}
void QualityScaler::SetMinResolution(int min_width, int min_height) {
@@ -42,6 +45,7 @@
void QualityScaler::ReportFramerate(int framerate) {
num_samples_ = static_cast<size_t>(
kMeasureSeconds * (framerate < kMinFps ? kMinFps : framerate));
+ framerate_ = framerate;
}
void QualityScaler::ReportQP(int qp) {
@@ -53,41 +57,65 @@
framedrop_percent_.AddSample(100);
}
-QualityScaler::Resolution QualityScaler::GetScaledResolution(
- const VideoFrame& frame) {
+void QualityScaler::OnEncodeFrame(const VideoFrame& frame) {
// Should be set through InitEncode -> Should be set by now.
assert(low_qp_threshold_ >= 0);
assert(num_samples_ > 0);
-
- Resolution res;
- res.width = frame.width();
- res.height = frame.height();
+ res_.width = frame.width();
+ res_.height = frame.height();
// Update scale factor.
int avg_drop = 0;
int avg_qp = 0;
+
+ // When encoder consistently overshoots, framerate reduction and spatial
+ // resizing will be triggered to get a smoother video.
if (framedrop_percent_.GetAverage(num_samples_, &avg_drop) &&
avg_drop >= kFramedropPercentThreshold) {
- AdjustScale(false);
+ // Reducing frame rate before spatial resolution change.
+ // Reduce frame rate only when it is above a certain number.
+ // Only one reduction is allowed for now.
+ // TODO(jackychen): Allow more than one framerate reduction.
+ if (use_framerate_reduction_ && !framerate_down_ && framerate_ >= 20) {
+ target_framerate_ = framerate_ / 2;
+ framerate_down_ = true;
+ // If frame rate has been updated, clear the buffer. We don't want
+ // spatial resolution to change right after frame rate change.
+ ClearSamples();
+ } else {
+ AdjustScale(false);
+ }
} else if (average_qp_.GetAverage(num_samples_, &avg_qp) &&
avg_qp <= low_qp_threshold_) {
- AdjustScale(true);
+ if (use_framerate_reduction_ && framerate_down_) {
+ target_framerate_ = -1;
+ framerate_down_ = false;
+ ClearSamples();
+ } else {
+ AdjustScale(true);
+ }
}
assert(downscale_shift_ >= 0);
for (int shift = downscale_shift_;
- shift > 0 && (res.width >> 1 >= min_width_) &&
- (res.height >> 1 >= min_height_);
+ shift > 0 && (res_.width / 2 >= min_width_) &&
+ (res_.height / 2 >= min_height_);
--shift) {
- res.width >>= 1;
- res.height >>= 1;
+ res_.width /= 2;
+ res_.height /= 2;
}
+}
- return res;
+QualityScaler::Resolution QualityScaler::GetScaledResolution() const {
+ return res_;
+}
+
+int QualityScaler::GetTargetFramerate() const {
+ return target_framerate_;
}
const VideoFrame& QualityScaler::GetScaledFrame(const VideoFrame& frame) {
- Resolution res = GetScaledResolution(frame);
+ Resolution res = GetScaledResolution();
if (res.width == frame.width())
return frame;
diff --git a/webrtc/modules/video_coding/utility/quality_scaler_unittest.cc b/webrtc/modules/video_coding/utility/quality_scaler_unittest.cc
index c09dffb..d1710b6 100644
--- a/webrtc/modules/video_coding/utility/quality_scaler_unittest.cc
+++ b/webrtc/modules/video_coding/utility/quality_scaler_unittest.cc
@@ -25,18 +25,28 @@
} // namespace
class QualityScalerTest : public ::testing::Test {
+ public:
+ // Temporal and spatial resolution.
+ struct Resolution {
+ int framerate;
+ int width;
+ int height;
+ };
protected:
enum ScaleDirection { kScaleDown, kScaleUp };
+ enum BadQualityMetric { kDropFrame, kReportLowQP };
QualityScalerTest() {
input_frame_.CreateEmptyFrame(
kWidth, kHeight, kWidth, kHalfWidth, kHalfWidth);
- qs_.Init(kMaxQp / QualityScaler::kDefaultLowQpDenominator);
+ qs_.Init(kMaxQp / QualityScaler::kDefaultLowQpDenominator, false);
qs_.ReportFramerate(kFramerate);
+ qs_.OnEncodeFrame(input_frame_);
}
bool TriggerScale(ScaleDirection scale_direction) {
- int initial_width = qs_.GetScaledResolution(input_frame_).width;
+ qs_.OnEncodeFrame(input_frame_);
+ int initial_width = qs_.GetScaledResolution().width;
for (int i = 0; i < kFramerate * kNumSeconds; ++i) {
switch (scale_direction) {
case kScaleUp:
@@ -46,8 +56,8 @@
qs_.ReportDroppedFrame();
break;
}
-
- if (qs_.GetScaledResolution(input_frame_).width != initial_width)
+ qs_.OnEncodeFrame(input_frame_);
+ if (qs_.GetScaledResolution().width != initial_width)
return true;
}
@@ -60,7 +70,8 @@
}
void ExpectScaleUsingReportedResolution() {
- QualityScaler::Resolution res = qs_.GetScaledResolution(input_frame_);
+ qs_.OnEncodeFrame(input_frame_);
+ QualityScaler::Resolution res = qs_.GetScaledResolution();
const VideoFrame& scaled_frame = qs_.GetScaledFrame(input_frame_);
EXPECT_EQ(res.width, scaled_frame.width());
EXPECT_EQ(res.height, scaled_frame.height());
@@ -70,6 +81,14 @@
void DoesNotDownscaleFrameDimensions(int width, int height);
+ Resolution TriggerResolutionChange(BadQualityMetric dropframe_lowqp,
+ int num_second,
+ int initial_framerate);
+
+ void VerifyQualityAdaptation(int initial_framerate, int seconds,
+ bool expect_spatial_resize,
+ bool expect_framerate_reduction);
+
void DownscaleEndsAt(int input_width,
int input_height,
int end_width,
@@ -84,7 +103,8 @@
}
TEST_F(QualityScalerTest, ReportsOriginalResolutionInitially) {
- QualityScaler::Resolution res = qs_.GetScaledResolution(input_frame_);
+ qs_.OnEncodeFrame(input_frame_);
+ QualityScaler::Resolution res = qs_.GetScaledResolution();
EXPECT_EQ(input_frame_.width(), res.width);
EXPECT_EQ(input_frame_.height(), res.height);
}
@@ -92,7 +112,7 @@
TEST_F(QualityScalerTest, DownscalesAfterContinuousFramedrop) {
EXPECT_TRUE(TriggerScale(kScaleDown)) << "No downscale within " << kNumSeconds
<< " seconds.";
- QualityScaler::Resolution res = qs_.GetScaledResolution(input_frame_);
+ QualityScaler::Resolution res = qs_.GetScaledResolution();
EXPECT_LT(res.width, input_frame_.width());
EXPECT_LT(res.height, input_frame_.height());
}
@@ -102,7 +122,8 @@
qs_.ReportQP(kNormalQp);
qs_.ReportDroppedFrame();
qs_.ReportDroppedFrame();
- if (qs_.GetScaledResolution(input_frame_).width < input_frame_.width())
+ qs_.OnEncodeFrame(input_frame_);
+ if (qs_.GetScaledResolution().width < input_frame_.width())
return;
}
@@ -112,7 +133,8 @@
TEST_F(QualityScalerTest, DoesNotDownscaleOnNormalQp) {
for (int i = 0; i < kFramerate * kNumSeconds; ++i) {
qs_.ReportQP(kNormalQp);
- ASSERT_EQ(input_frame_.width(), qs_.GetScaledResolution(input_frame_).width)
+ qs_.OnEncodeFrame(input_frame_);
+ ASSERT_EQ(input_frame_.width(), qs_.GetScaledResolution().width)
<< "Unexpected scale on half framedrop.";
}
}
@@ -120,11 +142,13 @@
TEST_F(QualityScalerTest, DoesNotDownscaleAfterHalfFramedrop) {
for (int i = 0; i < kFramerate * kNumSeconds / 2; ++i) {
qs_.ReportQP(kNormalQp);
- ASSERT_EQ(input_frame_.width(), qs_.GetScaledResolution(input_frame_).width)
+ qs_.OnEncodeFrame(input_frame_);
+ ASSERT_EQ(input_frame_.width(), qs_.GetScaledResolution().width)
<< "Unexpected scale on half framedrop.";
qs_.ReportDroppedFrame();
- ASSERT_EQ(input_frame_.width(), qs_.GetScaledResolution(input_frame_).width)
+ qs_.OnEncodeFrame(input_frame_);
+ ASSERT_EQ(input_frame_.width(), qs_.GetScaledResolution().width)
<< "Unexpected scale on half framedrop.";
}
}
@@ -139,7 +163,8 @@
while (min_dimension >= 2 * QualityScaler::kDefaultMinDownscaleDimension) {
EXPECT_TRUE(TriggerScale(kScaleDown)) << "No downscale within "
<< kNumSeconds << " seconds.";
- QualityScaler::Resolution res = qs_.GetScaledResolution(input_frame_);
+ qs_.OnEncodeFrame(input_frame_);
+ QualityScaler::Resolution res = qs_.GetScaledResolution();
min_dimension = res.width < res.height ? res.width : res.height;
++current_shift;
ASSERT_EQ(input_frame_.width() >> current_shift, res.width);
@@ -151,7 +176,8 @@
while (min_dimension < initial_min_dimension) {
EXPECT_TRUE(TriggerScale(kScaleUp)) << "No upscale within " << kNumSeconds
<< " seconds.";
- QualityScaler::Resolution res = qs_.GetScaledResolution(input_frame_);
+ qs_.OnEncodeFrame(input_frame_);
+ QualityScaler::Resolution res = qs_.GetScaledResolution();
min_dimension = res.width < res.height ? res.width : res.height;
--current_shift;
ASSERT_EQ(input_frame_.width() >> current_shift, res.width);
@@ -186,7 +212,8 @@
for (int i = 0; i < kFramerate * kNumSeconds; ++i) {
qs_.ReportDroppedFrame();
- ASSERT_EQ(input_frame_.width(), qs_.GetScaledResolution(input_frame_).width)
+ qs_.OnEncodeFrame(input_frame_);
+ ASSERT_EQ(input_frame_.width(), qs_.GetScaledResolution().width)
<< "Unexpected scale of minimal-size frame.";
}
}
@@ -203,6 +230,94 @@
DoesNotDownscaleFrameDimensions(1, 1);
}
+QualityScalerTest::Resolution QualityScalerTest::TriggerResolutionChange(
+ BadQualityMetric dropframe_lowqp, int num_second, int initial_framerate) {
+ QualityScalerTest::Resolution res;
+ res.framerate = initial_framerate;
+ qs_.OnEncodeFrame(input_frame_);
+ res.width = qs_.GetScaledResolution().width;
+ res.height = qs_.GetScaledResolution().height;
+ for (int i = 0; i < kFramerate * num_second; ++i) {
+ switch (dropframe_lowqp) {
+ case kReportLowQP:
+ qs_.ReportQP(kLowQp);
+ break;
+ case kDropFrame:
+ qs_.ReportDroppedFrame();
+ break;
+ }
+ qs_.OnEncodeFrame(input_frame_);
+ // Simulate the case when SetRates is called right after reducing
+ // framerate.
+ qs_.ReportFramerate(initial_framerate);
+ res.framerate = qs_.GetTargetFramerate();
+ if (res.framerate != -1)
+ qs_.ReportFramerate(res.framerate);
+ res.width = qs_.GetScaledResolution().width;
+ res.height = qs_.GetScaledResolution().height;
+ }
+ return res;
+}
+
+void QualityScalerTest::VerifyQualityAdaptation(
+ int initial_framerate, int seconds, bool expect_spatial_resize,
+ bool expect_framerate_reduction) {
+ qs_.Init(kMaxQp / QualityScaler::kDefaultLowQpDenominator, true);
+ qs_.OnEncodeFrame(input_frame_);
+ int init_width = qs_.GetScaledResolution().width;
+ int init_height = qs_.GetScaledResolution().height;
+
+ // Test reducing framerate by dropping frame continuously.
+ QualityScalerTest::Resolution res = TriggerResolutionChange(
+ kDropFrame, seconds, initial_framerate);
+
+ if (expect_framerate_reduction) {
+ EXPECT_LT(res.framerate, initial_framerate);
+ } else {
+ // No framerate reduction, video decimator should be disabled.
+ EXPECT_EQ(-1, res.framerate);
+ }
+
+ if (expect_spatial_resize) {
+ EXPECT_LT(res.width, init_width);
+ EXPECT_LT(res.height, init_height);
+ } else {
+ EXPECT_EQ(init_width, res.width);
+ EXPECT_EQ(init_height, res.height);
+ }
+
+ // The "seconds * 1.5" is to ensure spatial resolution to recover.
+ // For example, in 10 seconds test, framerate reduction happens in the first
+ // 5 seconds from 30fps to 15fps and causes the buffer size to be half of the
+ // original one. Then it will take only 75 samples to downscale (twice in 150
+ // samples). So to recover the resolution changes, we need more than 10
+ // seconds (i.e, seconds * 1.5). This is because the framerate increases
+ // before spatial size recovers, so it will take 150 samples to recover
+ // spatial size (300 for twice).
+ res = TriggerResolutionChange(kReportLowQP, seconds * 1.5, initial_framerate);
+ EXPECT_EQ(-1, res.framerate);
+ EXPECT_EQ(init_width, res.width);
+ EXPECT_EQ(init_height, res.height);
+}
+
+// In 5 seconds test, only framerate adjusting should happen.
+TEST_F(QualityScalerTest, ChangeFramerateOnly) {
+ VerifyQualityAdaptation(kFramerate, 5, false, true);
+}
+
+// In 10 seconds test, framerate adjusting and scaling are both
+// triggered, it shows that scaling would happen after framerate
+// adjusting.
+TEST_F(QualityScalerTest, ChangeFramerateAndSpatialSize) {
+ VerifyQualityAdaptation(kFramerate, 10, true, true);
+}
+
+// When starting from a low framerate, only spatial size will be changed.
+TEST_F(QualityScalerTest, ChangeSpatialSizeOnly) {
+ qs_.ReportFramerate(kFramerate >> 1);
+ VerifyQualityAdaptation(kFramerate >> 1, 10, true, false);
+}
+
TEST_F(QualityScalerTest, DoesNotDownscaleBelow2xDefaultMinDimensionsWidth) {
DoesNotDownscaleFrameDimensions(
2 * QualityScaler::kDefaultMinDownscaleDimension - 1, 1000);
@@ -227,7 +342,7 @@
// Drop all frames to force-trigger downscaling.
while (true) {
TriggerScale(kScaleDown);
- QualityScaler::Resolution res = qs_.GetScaledResolution(input_frame_);
+ QualityScaler::Resolution res = qs_.GetScaledResolution();
if (last_width == res.width) {
EXPECT_EQ(last_height, res.height);
EXPECT_EQ(end_width, res.width);
diff --git a/webrtc/modules/video_processing/main/interface/video_processing.h b/webrtc/modules/video_processing/main/interface/video_processing.h
index 0cab9fc..08b9db7 100644
--- a/webrtc/modules/video_processing/main/interface/video_processing.h
+++ b/webrtc/modules/video_processing/main/interface/video_processing.h
@@ -214,6 +214,8 @@
uint32_t height,
uint32_t frame_rate) = 0;
+ virtual void SetTargetFramerate(int frame_rate) {}
+
/**
Get decimated(target) frame rate
*/
diff --git a/webrtc/modules/video_processing/main/source/frame_preprocessor.cc b/webrtc/modules/video_processing/main/source/frame_preprocessor.cc
index cf4bc81..a9d77c2 100644
--- a/webrtc/modules/video_processing/main/source/frame_preprocessor.cc
+++ b/webrtc/modules/video_processing/main/source/frame_preprocessor.cc
@@ -38,7 +38,6 @@
frame_cnt_ = 0;
}
-
void VPMFramePreprocessor::EnableTemporalDecimation(bool enable) {
vd_->EnableTemporalDecimation(enable);
}
@@ -62,12 +61,19 @@
if (ret_val < 0) return ret_val;
- ret_val = vd_->SetTargetFramerate(frame_rate);
- if (ret_val < 0) return ret_val;
-
+ vd_->SetTargetFramerate(frame_rate);
return VPM_OK;
}
+void VPMFramePreprocessor::SetTargetFramerate(int frame_rate) {
+ if (frame_rate == -1) {
+ vd_->EnableTemporalDecimation(false);
+ } else {
+ vd_->EnableTemporalDecimation(true);
+ vd_->SetTargetFramerate(frame_rate);
+ }
+}
+
void VPMFramePreprocessor::UpdateIncomingframe_rate() {
vd_->UpdateIncomingframe_rate();
}
diff --git a/webrtc/modules/video_processing/main/source/frame_preprocessor.h b/webrtc/modules/video_processing/main/source/frame_preprocessor.h
index 81e92ed..895e457 100644
--- a/webrtc/modules/video_processing/main/source/frame_preprocessor.h
+++ b/webrtc/modules/video_processing/main/source/frame_preprocessor.h
@@ -41,6 +41,9 @@
int32_t SetTargetResolution(uint32_t width, uint32_t height,
uint32_t frame_rate);
+ // Set target frame rate.
+ void SetTargetFramerate(int frame_rate);
+
// Update incoming frame rate/dimension.
void UpdateIncomingframe_rate();
diff --git a/webrtc/modules/video_processing/main/source/video_decimator.cc b/webrtc/modules/video_processing/main/source/video_decimator.cc
index bf05bd7..449c3bd 100644
--- a/webrtc/modules/video_processing/main/source/video_decimator.cc
+++ b/webrtc/modules/video_processing/main/source/video_decimator.cc
@@ -8,6 +8,7 @@
* be found in the AUTHORS file in the root of the source tree.
*/
+#include "webrtc/base/checks.h"
#include "webrtc/modules/video_processing/main/interface/video_processing.h"
#include "webrtc/modules/video_processing/main/source/video_decimator.h"
#include "webrtc/system_wrappers/interface/tick_util.h"
@@ -36,11 +37,9 @@
enable_temporal_decimation_ = enable;
}
-int32_t VPMVideoDecimator::SetTargetFramerate(uint32_t frame_rate) {
- if (frame_rate == 0) return VPM_PARAMETER_ERROR;
-
+void VPMVideoDecimator::SetTargetFramerate(int frame_rate) {
+ DCHECK(frame_rate);
target_frame_rate_ = frame_rate;
- return VPM_OK;
}
bool VPMVideoDecimator::DropFrame() {
diff --git a/webrtc/modules/video_processing/main/source/video_decimator.h b/webrtc/modules/video_processing/main/source/video_decimator.h
index fca74ae..3d4573c 100644
--- a/webrtc/modules/video_processing/main/source/video_decimator.h
+++ b/webrtc/modules/video_processing/main/source/video_decimator.h
@@ -25,7 +25,7 @@
void EnableTemporalDecimation(bool enable);
- int32_t SetTargetFramerate(uint32_t frame_rate);
+ void SetTargetFramerate(int frame_rate);
bool DropFrame();
diff --git a/webrtc/modules/video_processing/main/source/video_processing_impl.cc b/webrtc/modules/video_processing/main/source/video_processing_impl.cc
index afb8290..80419e8 100644
--- a/webrtc/modules/video_processing/main/source/video_processing_impl.cc
+++ b/webrtc/modules/video_processing/main/source/video_processing_impl.cc
@@ -148,6 +148,11 @@
return frame_pre_processor_.SetTargetResolution(width, height, frame_rate);
}
+void VideoProcessingModuleImpl::SetTargetFramerate(int frame_rate) {
+ CriticalSectionScoped cs(&mutex_);
+ frame_pre_processor_.SetTargetFramerate(frame_rate);
+}
+
uint32_t VideoProcessingModuleImpl::Decimatedframe_rate() {
CriticalSectionScoped cs(&mutex_);
return frame_pre_processor_.Decimatedframe_rate();
diff --git a/webrtc/modules/video_processing/main/source/video_processing_impl.h b/webrtc/modules/video_processing/main/source/video_processing_impl.h
index b78c019..437d1d6 100644
--- a/webrtc/modules/video_processing/main/source/video_processing_impl.h
+++ b/webrtc/modules/video_processing/main/source/video_processing_impl.h
@@ -48,6 +48,8 @@
uint32_t height,
uint32_t frame_rate) override;
+ void SetTargetFramerate(int frame_rate) override;
+
// Get decimated values: frame rate/dimension
uint32_t Decimatedframe_rate() override;
uint32_t DecimatedWidth() const override;
diff --git a/webrtc/video/video_encoder.cc b/webrtc/video/video_encoder.cc
index fd213f8..8847a10 100644
--- a/webrtc/video/video_encoder.cc
+++ b/webrtc/video/video_encoder.cc
@@ -134,4 +134,10 @@
return encoder_->SupportsNativeHandle();
}
+int VideoEncoderSoftwareFallbackWrapper::GetTargetFramerate() {
+ if (fallback_encoder_)
+ return fallback_encoder_->GetTargetFramerate();
+ return encoder_->GetTargetFramerate();
+}
+
} // namespace webrtc
diff --git a/webrtc/video_encoder.h b/webrtc/video_encoder.h
index 776b22b..a512c98 100644
--- a/webrtc/video_encoder.h
+++ b/webrtc/video_encoder.h
@@ -125,6 +125,7 @@
return -1;
}
virtual void OnDroppedFrame() {}
+ virtual int GetTargetFramerate() { return -1; }
virtual bool SupportsNativeHandle() const { return false; }
};
@@ -151,6 +152,7 @@
int32_t SetRates(uint32_t bitrate, uint32_t framerate) override;
void OnDroppedFrame() override;
+ int GetTargetFramerate() override;
bool SupportsNativeHandle() const override;
private:
diff --git a/webrtc/video_engine/vie_encoder.cc b/webrtc/video_engine/vie_encoder.cc
index 7ed207f..0eea5f5 100644
--- a/webrtc/video_engine/vie_encoder.cc
+++ b/webrtc/video_engine/vie_encoder.cc
@@ -78,6 +78,9 @@
const uint32_t width,
const uint32_t height);
+ // Update target frame rate.
+ void SetTargetFramerate(int frame_rate);
+
private:
VideoProcessingModule* vpm_;
};
@@ -868,4 +871,8 @@
return vpm_->SetTargetResolution(width, height, frame_rate);
}
+void QMVideoSettingsCallback::SetTargetFramerate(int frame_rate) {
+ vpm_->SetTargetFramerate(frame_rate);
+}
+
} // namespace webrtc
