Add DecodedImageCallback::Decoded() function with custom decode time value.
On Android, we would like to use MediaCodec output buffers to hold decoded frames until they can be rendered to a texture. There can only be one texture buffer used at the same time and therefore the calculated decode time in VCMTiming will be wrong since that calculation will also include the time where the decoder waited for the upper layers (that depend on network jitter and actual render time) to release the frame.
This new method will be used in
https://codereview.webrtc.org/1422963003/
BUG=webrtc:4993
R=stefan@webrtc.org
TBR=mflodman@webrtc.org
Review URL: https://codereview.webrtc.org/1414693006 .
Cr-Commit-Position: refs/heads/master@{#10576}
diff --git a/webrtc/modules/video_coding/codecs/interface/mock/mock_video_codec_interface.h b/webrtc/modules/video_coding/codecs/interface/mock/mock_video_codec_interface.h
index 6c926d4..b3af479 100644
--- a/webrtc/modules/video_coding/codecs/interface/mock/mock_video_codec_interface.h
+++ b/webrtc/modules/video_coding/codecs/interface/mock/mock_video_codec_interface.h
@@ -48,6 +48,8 @@
class MockDecodedImageCallback : public DecodedImageCallback {
public:
MOCK_METHOD1(Decoded, int32_t(VideoFrame& decodedImage));
+ MOCK_METHOD2(Decoded, int32_t(VideoFrame& decodedImage,
+ int64_t decode_time_ms));
MOCK_METHOD1(ReceivedDecodedReferenceFrame,
int32_t(const uint64_t pictureId));
MOCK_METHOD1(ReceivedDecodedFrame,
diff --git a/webrtc/modules/video_coding/codecs/test/videoprocessor.h b/webrtc/modules/video_coding/codecs/test/videoprocessor.h
index 0b094ae..582d903 100644
--- a/webrtc/modules/video_coding/codecs/test/videoprocessor.h
+++ b/webrtc/modules/video_coding/codecs/test/videoprocessor.h
@@ -13,6 +13,7 @@
#include <string>
+#include "webrtc/base/checks.h"
#include "webrtc/common_video/libyuv/include/scaler.h"
#include "webrtc/common_video/libyuv/include/webrtc_libyuv.h"
#include "webrtc/modules/video_coding/codecs/interface/video_codec_interface.h"
@@ -248,6 +249,11 @@
: video_processor_(vp) {
}
int32_t Decoded(webrtc::VideoFrame& image) override;
+ int32_t Decoded(
+ webrtc::VideoFrame& image, int64_t decode_time_ms) override {
+ RTC_NOTREACHED();
+ return -1;
+ }
private:
VideoProcessorImpl* video_processor_;
diff --git a/webrtc/modules/video_coding/codecs/vp8/simulcast_unittest.h b/webrtc/modules/video_coding/codecs/vp8/simulcast_unittest.h
index e4fc986..2577bad 100644
--- a/webrtc/modules/video_coding/codecs/vp8/simulcast_unittest.h
+++ b/webrtc/modules/video_coding/codecs/vp8/simulcast_unittest.h
@@ -14,6 +14,7 @@
#include <algorithm>
#include <vector>
+#include "webrtc/base/checks.h"
#include "webrtc/base/scoped_ptr.h"
#include "webrtc/common.h"
#include "webrtc/common_video/libyuv/include/webrtc_libyuv.h"
@@ -123,7 +124,7 @@
Vp8TestDecodedImageCallback()
: decoded_frames_(0) {
}
- virtual int32_t Decoded(VideoFrame& decoded_image) {
+ int32_t Decoded(VideoFrame& decoded_image) override {
for (int i = 0; i < decoded_image.width(); ++i) {
EXPECT_NEAR(kColorY, decoded_image.buffer(kYPlane)[i], 1);
}
@@ -136,6 +137,10 @@
decoded_frames_++;
return 0;
}
+ int32_t Decoded(VideoFrame& decoded_image, int64_t decode_time_ms) override {
+ RTC_NOTREACHED();
+ return -1;
+ }
int DecodedFrames() {
return decoded_frames_;
}
diff --git a/webrtc/modules/video_coding/codecs/vp8/test/vp8_impl_unittest.cc b/webrtc/modules/video_coding/codecs/vp8/test/vp8_impl_unittest.cc
index 5ec674f..7650a25 100644
--- a/webrtc/modules/video_coding/codecs/vp8/test/vp8_impl_unittest.cc
+++ b/webrtc/modules/video_coding/codecs/vp8/test/vp8_impl_unittest.cc
@@ -11,6 +11,7 @@
#include <stdio.h>
#include "testing/gtest/include/gtest/gtest.h"
+#include "webrtc/base/checks.h"
#include "webrtc/base/scoped_ptr.h"
#include "webrtc/common_video/libyuv/include/webrtc_libyuv.h"
#include "webrtc/modules/video_coding/codecs/vp8/include/vp8.h"
@@ -78,7 +79,11 @@
public:
explicit Vp8UnitTestDecodeCompleteCallback(VideoFrame* frame)
: decoded_frame_(frame), decode_complete(false) {}
- int Decoded(webrtc::VideoFrame& frame);
+ int32_t Decoded(VideoFrame& frame) override;
+ int32_t Decoded(VideoFrame& frame, int64_t decode_time_ms) override {
+ RTC_NOTREACHED();
+ return -1;
+ }
bool DecodeComplete();
private:
diff --git a/webrtc/modules/video_coding/codecs/vp8/vp8_sequence_coder.cc b/webrtc/modules/video_coding/codecs/vp8/vp8_sequence_coder.cc
index 5843d83..ea7db5a 100644
--- a/webrtc/modules/video_coding/codecs/vp8/vp8_sequence_coder.cc
+++ b/webrtc/modules/video_coding/codecs/vp8/vp8_sequence_coder.cc
@@ -9,6 +9,7 @@
*/
#include "testing/gtest/include/gtest/gtest.h"
+#include "webrtc/base/checks.h"
#include "webrtc/base/scoped_ptr.h"
#include "webrtc/common_video/interface/video_image.h"
#include "webrtc/common_video/libyuv/include/webrtc_libyuv.h"
@@ -68,7 +69,11 @@
public:
explicit Vp8SequenceCoderDecodeCallback(FILE* decoded_file)
: decoded_file_(decoded_file) {}
- int Decoded(webrtc::VideoFrame& frame);
+ int32_t Decoded(webrtc::VideoFrame& frame) override;
+ int32_t Decoded(webrtc::VideoFrame& frame, int64_t decode_time_ms) override {
+ RTC_NOTREACHED();
+ return -1;;
+ }
bool DecodeComplete();
private:
diff --git a/webrtc/modules/video_coding/main/source/codec_timer.cc b/webrtc/modules/video_coding/main/source/codec_timer.cc
index a462258..57985c7 100644
--- a/webrtc/modules/video_coding/main/source/codec_timer.cc
+++ b/webrtc/modules/video_coding/main/source/codec_timer.cc
@@ -28,13 +28,6 @@
Reset();
}
-int32_t VCMCodecTimer::StopTimer(int64_t startTimeMs, int64_t nowMs)
-{
- const int32_t timeDiff = static_cast<int32_t>(nowMs - startTimeMs);
- MaxFilter(timeDiff, nowMs);
- return timeDiff;
-}
-
void VCMCodecTimer::Reset()
{
_filteredMax = 0;
diff --git a/webrtc/modules/video_coding/main/source/codec_timer.h b/webrtc/modules/video_coding/main/source/codec_timer.h
index cb7e813..a7abeb8 100644
--- a/webrtc/modules/video_coding/main/source/codec_timer.h
+++ b/webrtc/modules/video_coding/main/source/codec_timer.h
@@ -35,8 +35,8 @@
public:
VCMCodecTimer();
- // Updates and returns the max filtered decode time.
- int32_t StopTimer(int64_t startTimeMs, int64_t nowMs);
+ // Updates the max filtered decode time.
+ void MaxFilter(int32_t newDecodeTimeMs, int64_t nowMs);
// Empty the list of timers.
void Reset();
@@ -46,7 +46,6 @@
private:
void UpdateMaxHistory(int32_t decodeTime, int64_t now);
- void MaxFilter(int32_t newTime, int64_t nowMs);
void ProcessHistory(int64_t nowMs);
int32_t _filteredMax;
diff --git a/webrtc/modules/video_coding/main/source/generic_decoder.cc b/webrtc/modules/video_coding/main/source/generic_decoder.cc
index 8b2d397..4f4a09b 100644
--- a/webrtc/modules/video_coding/main/source/generic_decoder.cc
+++ b/webrtc/modules/video_coding/main/source/generic_decoder.cc
@@ -47,6 +47,11 @@
}
int32_t VCMDecodedFrameCallback::Decoded(VideoFrame& decodedImage) {
+ return Decoded(decodedImage, -1);
+}
+
+int32_t VCMDecodedFrameCallback::Decoded(VideoFrame& decodedImage,
+ int64_t decode_time_ms) {
// TODO(holmer): We should improve this so that we can handle multiple
// callbacks from one call to Decode().
VCMFrameInformation* frameInfo;
@@ -63,10 +68,15 @@
return WEBRTC_VIDEO_CODEC_OK;
}
+ const int64_t now_ms = _clock->TimeInMilliseconds();
+ if (decode_time_ms < 0) {
+ decode_time_ms =
+ static_cast<int32_t>(now_ms - frameInfo->decodeStartTimeMs);
+ }
_timing.StopDecodeTimer(
decodedImage.timestamp(),
- frameInfo->decodeStartTimeMs,
- _clock->TimeInMilliseconds(),
+ decode_time_ms,
+ now_ms,
frameInfo->renderTimeMs);
if (callback != NULL)
diff --git a/webrtc/modules/video_coding/main/source/generic_decoder.h b/webrtc/modules/video_coding/main/source/generic_decoder.h
index c1298bb..7788425 100644
--- a/webrtc/modules/video_coding/main/source/generic_decoder.h
+++ b/webrtc/modules/video_coding/main/source/generic_decoder.h
@@ -41,6 +41,7 @@
VCMReceiveCallback* UserReceiveCallback();
virtual int32_t Decoded(VideoFrame& decodedImage);
+ virtual int32_t Decoded(VideoFrame& decodedImage, int64_t decode_time_ms);
virtual int32_t ReceivedDecodedReferenceFrame(const uint64_t pictureId);
virtual int32_t ReceivedDecodedFrame(const uint64_t pictureId);
diff --git a/webrtc/modules/video_coding/main/source/timing.cc b/webrtc/modules/video_coding/main/source/timing.cc
index 8d59135..d8ab96d 100644
--- a/webrtc/modules/video_coding/main/source/timing.cc
+++ b/webrtc/modules/video_coding/main/source/timing.cc
@@ -165,13 +165,13 @@
}
int32_t VCMTiming::StopDecodeTimer(uint32_t time_stamp,
- int64_t start_time_ms,
+ int32_t decode_time_ms,
int64_t now_ms,
int64_t render_time_ms) {
CriticalSectionScoped cs(crit_sect_);
- int32_t time_diff_ms = codec_timer_.StopTimer(start_time_ms, now_ms);
- assert(time_diff_ms >= 0);
- last_decode_ms_ = time_diff_ms;
+ codec_timer_.MaxFilter(decode_time_ms, now_ms);
+ assert(decode_time_ms >= 0);
+ last_decode_ms_ = decode_time_ms;
// Update stats.
++num_decoded_frames_;
diff --git a/webrtc/modules/video_coding/main/source/timing.h b/webrtc/modules/video_coding/main/source/timing.h
index d3b8fa6..41dec8c 100644
--- a/webrtc/modules/video_coding/main/source/timing.h
+++ b/webrtc/modules/video_coding/main/source/timing.h
@@ -58,7 +58,7 @@
// Stops the decoder timer, should be called when the decoder returns a frame
// or when the decoded frame callback is called.
int32_t StopDecodeTimer(uint32_t time_stamp,
- int64_t start_time_ms,
+ int32_t decode_time_ms,
int64_t now_ms,
int64_t render_time_ms);
diff --git a/webrtc/modules/video_coding/main/source/timing_unittest.cc b/webrtc/modules/video_coding/main/source/timing_unittest.cc
index 694a600..8ab2aed 100644
--- a/webrtc/modules/video_coding/main/source/timing_unittest.cc
+++ b/webrtc/modules/video_coding/main/source/timing_unittest.cc
@@ -85,8 +85,10 @@
for (int i = 0; i < 10; i++) {
int64_t startTimeMs = clock.TimeInMilliseconds();
clock.AdvanceTimeMilliseconds(10);
- timing.StopDecodeTimer(timeStamp, startTimeMs,
- clock.TimeInMilliseconds(), timing.RenderTimeMs(
+ timing.StopDecodeTimer(timeStamp,
+ clock.TimeInMilliseconds() - startTimeMs,
+ clock.TimeInMilliseconds(),
+ timing.RenderTimeMs(
timeStamp, clock.TimeInMilliseconds()));
timeStamp += 90000 / 25;
clock.AdvanceTimeMilliseconds(1000 / 25 - 10);
diff --git a/webrtc/video/video_decoder_unittest.cc b/webrtc/video/video_decoder_unittest.cc
index be09b19..fd1605d 100644
--- a/webrtc/video/video_decoder_unittest.cc
+++ b/webrtc/video/video_decoder_unittest.cc
@@ -11,6 +11,7 @@
#include "webrtc/video_decoder.h"
#include "testing/gtest/include/gtest/gtest.h"
+#include "webrtc/base/checks.h"
#include "webrtc/modules/video_coding/codecs/interface/video_error_codes.h"
namespace webrtc {
@@ -148,6 +149,11 @@
ForwardsRegisterDecodeCompleteCallback) {
class FakeDecodedImageCallback : public DecodedImageCallback {
int32_t Decoded(VideoFrame& decodedImage) override { return 0; }
+ int32_t Decoded(
+ webrtc::VideoFrame& decodedImage, int64_t decode_time_ms) override {
+ RTC_NOTREACHED();
+ return -1;
+ }
} callback, callback2;
VideoCodec codec = {};
diff --git a/webrtc/video_decoder.h b/webrtc/video_decoder.h
index 2822677..3d04104 100644
--- a/webrtc/video_decoder.h
+++ b/webrtc/video_decoder.h
@@ -29,6 +29,16 @@
virtual ~DecodedImageCallback() {}
virtual int32_t Decoded(VideoFrame& decodedImage) = 0;
+ // Provides an alternative interface that allows the decoder to specify the
+ // decode time excluding waiting time for any previous pending frame to
+ // return. This is necessary for breaking positive feedback in the delay
+ // estimation when the decoder has a single output buffer.
+ // TODO(perkj): Remove default implementation when chromium has been updated.
+ virtual int32_t Decoded(VideoFrame& decodedImage, int64_t decode_time_ms) {
+ // The default implementation ignores custom decode time value.
+ return Decoded(decodedImage);
+ }
+
virtual int32_t ReceivedDecodedReferenceFrame(const uint64_t pictureId) {
return -1;
}
