blob: 03ec0ea9565519fb4fbce52c552fa6b33246537e [file] [log] [blame]
// Copyright 2013 The Chromium Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
#include "media/cast/video_receiver/video_receiver.h"
#include <algorithm>
#include "base/bind.h"
#include "base/logging.h"
#include "base/message_loop/message_loop.h"
#include "media/cast/cast_defines.h"
#include "media/cast/framer/framer.h"
#include "media/cast/video_receiver/video_decoder.h"
namespace media {
namespace cast {
const int64 kMinSchedulingDelayMs = 1;
static const int64 kMinTimeBetweenOffsetUpdatesMs = 2000;
static const int kTimeOffsetFilter = 8;
static const int64_t kMinProcessIntervalMs = 5;
// Local implementation of RtpData (defined in rtp_rtcp_defines.h).
// Used to pass payload data into the video receiver.
class LocalRtpVideoData : public RtpData {
public:
explicit LocalRtpVideoData(base::TickClock* clock,
VideoReceiver* video_receiver)
: clock_(clock),
video_receiver_(video_receiver),
time_updated_(false),
incoming_rtp_timestamp_(0) {
}
virtual ~LocalRtpVideoData() {}
virtual void OnReceivedPayloadData(const uint8* payload_data,
size_t payload_size,
const RtpCastHeader* rtp_header) OVERRIDE {
base::TimeTicks now = clock_->NowTicks();
if (time_incoming_packet_.is_null() || now - time_incoming_packet_ >
base::TimeDelta::FromMilliseconds(kMinTimeBetweenOffsetUpdatesMs)) {
incoming_rtp_timestamp_ = rtp_header->webrtc.header.timestamp;
time_incoming_packet_ = now;
time_updated_ = true;
}
video_receiver_->IncomingRtpPacket(payload_data, payload_size, *rtp_header);
}
bool GetPacketTimeInformation(base::TimeTicks* time_incoming_packet,
uint32* incoming_rtp_timestamp) {
*time_incoming_packet = time_incoming_packet_;
*incoming_rtp_timestamp = incoming_rtp_timestamp_;
bool time_updated = time_updated_;
time_updated_ = false;
return time_updated;
}
private:
base::TickClock* clock_; // Not owned by this class.
VideoReceiver* video_receiver_;
bool time_updated_;
base::TimeTicks time_incoming_packet_;
uint32 incoming_rtp_timestamp_;
};
// Local implementation of RtpPayloadFeedback (defined in rtp_defines.h)
// Used to convey cast-specific feedback from receiver to sender.
// Callback triggered by the Framer (cast message builder).
class LocalRtpVideoFeedback : public RtpPayloadFeedback {
public:
explicit LocalRtpVideoFeedback(VideoReceiver* video_receiver)
: video_receiver_(video_receiver) {
}
virtual void CastFeedback(const RtcpCastMessage& cast_message) OVERRIDE {
video_receiver_->CastFeedback(cast_message);
}
private:
VideoReceiver* video_receiver_;
};
// Local implementation of RtpReceiverStatistics (defined by rtcp.h).
// Used to pass statistics data from the RTP module to the RTCP module.
class LocalRtpReceiverStatistics : public RtpReceiverStatistics {
public:
explicit LocalRtpReceiverStatistics(RtpReceiver* rtp_receiver)
: rtp_receiver_(rtp_receiver) {
}
virtual void GetStatistics(uint8* fraction_lost,
uint32* cumulative_lost, // 24 bits valid.
uint32* extended_high_sequence_number,
uint32* jitter) OVERRIDE {
rtp_receiver_->GetStatistics(fraction_lost,
cumulative_lost,
extended_high_sequence_number,
jitter);
}
private:
RtpReceiver* rtp_receiver_;
};
VideoReceiver::VideoReceiver(scoped_refptr<CastEnvironment> cast_environment,
const VideoReceiverConfig& video_config,
PacedPacketSender* const packet_sender)
: cast_environment_(cast_environment),
codec_(video_config.codec),
incoming_ssrc_(video_config.incoming_ssrc),
target_delay_delta_(
base::TimeDelta::FromMilliseconds(video_config.rtp_max_delay_ms)),
frame_delay_(base::TimeDelta::FromMilliseconds(
1000 / video_config.max_frame_rate)),
incoming_payload_callback_(
new LocalRtpVideoData(cast_environment_->Clock(), this)),
incoming_payload_feedback_(new LocalRtpVideoFeedback(this)),
rtp_receiver_(cast_environment_->Clock(), NULL, &video_config,
incoming_payload_callback_.get()),
rtp_video_receiver_statistics_(
new LocalRtpReceiverStatistics(&rtp_receiver_)),
weak_factory_(this) {
int max_unacked_frames = video_config.rtp_max_delay_ms *
video_config.max_frame_rate / 1000;
DCHECK(max_unacked_frames) << "Invalid argument";
framer_.reset(new Framer(cast_environment->Clock(),
incoming_payload_feedback_.get(),
video_config.incoming_ssrc,
video_config.decoder_faster_than_max_frame_rate,
max_unacked_frames));
if (!video_config.use_external_decoder) {
video_decoder_.reset(new VideoDecoder(video_config));
}
rtcp_.reset(
new Rtcp(cast_environment_->Clock(),
NULL,
packet_sender,
NULL,
rtp_video_receiver_statistics_.get(),
video_config.rtcp_mode,
base::TimeDelta::FromMilliseconds(video_config.rtcp_interval),
false,
video_config.feedback_ssrc,
video_config.rtcp_c_name));
rtcp_->SetRemoteSSRC(video_config.incoming_ssrc);
ScheduleNextRtcpReport();
ScheduleNextCastMessage();
}
VideoReceiver::~VideoReceiver() {}
void VideoReceiver::GetRawVideoFrame(
const VideoFrameDecodedCallback& callback) {
GetEncodedVideoFrame(base::Bind(&VideoReceiver::DecodeVideoFrame,
weak_factory_.GetWeakPtr(),
callback));
}
// Called when we have a frame to decode.
void VideoReceiver::DecodeVideoFrame(
const VideoFrameDecodedCallback& callback,
scoped_ptr<EncodedVideoFrame> encoded_frame,
const base::TimeTicks& render_time) {
// Hand the ownership of the encoded frame to the decode thread.
cast_environment_->PostTask(CastEnvironment::VIDEO_DECODER, FROM_HERE,
base::Bind(&VideoReceiver::DecodeVideoFrameThread,
weak_factory_.GetWeakPtr(), base::Passed(&encoded_frame),
render_time, callback));
}
// Utility function to run the decoder on a designated decoding thread.
void VideoReceiver::DecodeVideoFrameThread(
scoped_ptr<EncodedVideoFrame> encoded_frame,
const base::TimeTicks render_time,
const VideoFrameDecodedCallback& frame_decoded_callback) {
DCHECK(cast_environment_->CurrentlyOn(CastEnvironment::VIDEO_DECODER));
DCHECK(video_decoder_);
// TODO(mikhal): Allow the application to allocate this memory.
scoped_ptr<I420VideoFrame> video_frame(new I420VideoFrame());
bool success = video_decoder_->DecodeVideoFrame(encoded_frame.get(),
render_time, video_frame.get());
if (success) {
VLOG(1) << "Decoded frame " << static_cast<int>(encoded_frame->frame_id);
// Frame decoded - return frame to the user via callback.
cast_environment_->PostTask(CastEnvironment::MAIN, FROM_HERE,
base::Bind(frame_decoded_callback,
base::Passed(&video_frame), render_time));
} else {
// This will happen if we decide to decode but not show a frame.
cast_environment_->PostTask(CastEnvironment::MAIN, FROM_HERE,
base::Bind(&VideoReceiver::GetRawVideoFrame,
weak_factory_.GetWeakPtr(), frame_decoded_callback));
}
}
// Called from the main cast thread.
void VideoReceiver::GetEncodedVideoFrame(
const VideoFrameEncodedCallback& callback) {
scoped_ptr<EncodedVideoFrame> encoded_frame(new EncodedVideoFrame());
uint32 rtp_timestamp = 0;
bool next_frame = false;
if (!framer_->GetEncodedVideoFrame(encoded_frame.get(), &rtp_timestamp,
&next_frame)) {
// We have no video frames. Wait for new packet(s).
queued_encoded_callbacks_.push_back(callback);
return;
}
base::TimeTicks render_time;
if (PullEncodedVideoFrame(rtp_timestamp, next_frame, &encoded_frame,
&render_time)) {
cast_environment_->PostTask(CastEnvironment::MAIN, FROM_HERE,
base::Bind(callback, base::Passed(&encoded_frame), render_time));
} else {
// We have a video frame; however we are missing packets and we have time
// to wait for new packet(s).
queued_encoded_callbacks_.push_back(callback);
}
}
// Should we pull the encoded video frame from the framer? decided by if this is
// the next frame or we are running out of time and have to pull the following
// frame.
// If the frame it too old to be rendered we set the don't show flag in the
// video bitstream where possible.
bool VideoReceiver::PullEncodedVideoFrame(uint32 rtp_timestamp,
bool next_frame, scoped_ptr<EncodedVideoFrame>* encoded_frame,
base::TimeTicks* render_time) {
base::TimeTicks now = cast_environment_->Clock()->NowTicks();
*render_time = GetRenderTime(now, rtp_timestamp);
// Minimum time before a frame is due to be rendered before we pull it for
// decode.
base::TimeDelta min_wait_delta = frame_delay_;
base::TimeDelta time_until_render = *render_time - now;
if (!next_frame && (time_until_render > min_wait_delta)) {
// Example:
// We have decoded frame 1 and we have received the complete frame 3, but
// not frame 2. If we still have time before frame 3 should be rendered we
// will wait for 2 to arrive, however if 2 never show up this timer will hit
// and we will pull out frame 3 for decoding and rendering.
base::TimeDelta time_until_release = time_until_render - min_wait_delta;
cast_environment_->PostDelayedTask(CastEnvironment::MAIN, FROM_HERE,
base::Bind(&VideoReceiver::PlayoutTimeout, weak_factory_.GetWeakPtr()),
time_until_release);
VLOG(0) << "Wait before releasing frame "
<< static_cast<int>((*encoded_frame)->frame_id)
<< " time " << time_until_release.InMilliseconds();
return false;
}
base::TimeDelta dont_show_timeout_delta =
base::TimeDelta::FromMilliseconds(-kDontShowTimeoutMs);
if (codec_ == kVp8 && time_until_render < dont_show_timeout_delta) {
(*encoded_frame)->data[0] &= 0xef;
VLOG(0) << "Don't show frame "
<< static_cast<int>((*encoded_frame)->frame_id)
<< " time_until_render:" << time_until_render.InMilliseconds();
} else {
VLOG(1) << "Show frame "
<< static_cast<int>((*encoded_frame)->frame_id)
<< " time_until_render:" << time_until_render.InMilliseconds();
}
// We have a copy of the frame, release this one.
framer_->ReleaseFrame((*encoded_frame)->frame_id);
(*encoded_frame)->codec = codec_;
return true;
}
void VideoReceiver::PlayoutTimeout() {
if (queued_encoded_callbacks_.empty()) return;
uint32 rtp_timestamp = 0;
bool next_frame = false;
scoped_ptr<EncodedVideoFrame> encoded_frame(new EncodedVideoFrame());
if (!framer_->GetEncodedVideoFrame(encoded_frame.get(), &rtp_timestamp,
&next_frame)) {
// We have no video frames. Wait for new packet(s).
// Since the application can post multiple VideoFrameEncodedCallback and
// we only check the next frame to play out we might have multiple timeout
// events firing after each other; however this should be a rare event.
VLOG(1) << "Failed to retrieved a complete frame at this point in time";
return;
}
VLOG(1) << "PlayoutTimeout retrieved frame "
<< static_cast<int>(encoded_frame->frame_id);
base::TimeTicks render_time;
if (PullEncodedVideoFrame(rtp_timestamp, next_frame, &encoded_frame,
&render_time)) {
if (!queued_encoded_callbacks_.empty()) {
VideoFrameEncodedCallback callback = queued_encoded_callbacks_.front();
queued_encoded_callbacks_.pop_front();
cast_environment_->PostTask(CastEnvironment::MAIN, FROM_HERE,
base::Bind(callback, base::Passed(&encoded_frame), render_time));
}
} else {
// We have a video frame; however we are missing packets and we have time
// to wait for new packet(s).
}
}
base::TimeTicks VideoReceiver::GetRenderTime(base::TimeTicks now,
uint32 rtp_timestamp) {
// Senders time in ms when this frame was captured.
// Note: the senders clock and our local clock might not be synced.
base::TimeTicks rtp_timestamp_in_ticks;
base::TimeTicks time_incoming_packet;
uint32 incoming_rtp_timestamp;
if (time_offset_.InMilliseconds() == 0) {
incoming_payload_callback_->GetPacketTimeInformation(
&time_incoming_packet, &incoming_rtp_timestamp);
if (!rtcp_->RtpTimestampInSenderTime(kVideoFrequency,
incoming_rtp_timestamp,
&rtp_timestamp_in_ticks)) {
// We have not received any RTCP to sync the stream play it out as soon as
// possible.
return now;
}
time_offset_ = time_incoming_packet - rtp_timestamp_in_ticks;
} else if (incoming_payload_callback_->GetPacketTimeInformation(
&time_incoming_packet, &incoming_rtp_timestamp)) {
if (rtcp_->RtpTimestampInSenderTime(kVideoFrequency,
incoming_rtp_timestamp,
&rtp_timestamp_in_ticks)) {
// Time to update the time_offset.
base::TimeDelta time_offset =
time_incoming_packet - rtp_timestamp_in_ticks;
time_offset_ = ((kTimeOffsetFilter - 1) * time_offset_ + time_offset)
/ kTimeOffsetFilter;
}
}
if (!rtcp_->RtpTimestampInSenderTime(kVideoFrequency,
rtp_timestamp,
&rtp_timestamp_in_ticks)) {
// This can fail if we have not received any RTCP packets in a long time.
return now;
}
return (rtp_timestamp_in_ticks + time_offset_ + target_delay_delta_);
}
void VideoReceiver::IncomingPacket(const uint8* packet, size_t length,
const base::Closure callback) {
if (Rtcp::IsRtcpPacket(packet, length)) {
rtcp_->IncomingRtcpPacket(packet, length);
} else {
rtp_receiver_.ReceivedPacket(packet, length);
}
cast_environment_->PostTask(CastEnvironment::MAIN, FROM_HERE, callback);
}
void VideoReceiver::IncomingRtpPacket(const uint8* payload_data,
size_t payload_size,
const RtpCastHeader& rtp_header) {
bool complete = framer_->InsertPacket(payload_data, payload_size, rtp_header);
if (!complete) return; // Video frame not complete; wait for more packets.
if (queued_encoded_callbacks_.empty()) return; // No pending callback.
VideoFrameEncodedCallback callback = queued_encoded_callbacks_.front();
queued_encoded_callbacks_.pop_front();
cast_environment_->PostTask(CastEnvironment::MAIN, FROM_HERE,
base::Bind(&VideoReceiver::GetEncodedVideoFrame,
weak_factory_.GetWeakPtr(), callback));
}
// Send a cast feedback message. Actual message created in the framer (cast
// message builder).
void VideoReceiver::CastFeedback(const RtcpCastMessage& cast_message) {
rtcp_->SendRtcpCast(cast_message);
time_last_sent_cast_message_= cast_environment_->Clock()->NowTicks();
}
// Send a key frame request to the sender.
void VideoReceiver::RequestKeyFrame() {
rtcp_->SendRtcpPli(incoming_ssrc_);
}
// Cast messages should be sent within a maximum interval. Schedule a call
// if not triggered elsewhere, e.g. by the cast message_builder.
void VideoReceiver::ScheduleNextCastMessage() {
base::TimeTicks send_time;
framer_->TimeToSendNextCastMessage(&send_time);
base::TimeDelta time_to_send = send_time -
cast_environment_->Clock()->NowTicks();
time_to_send = std::max(time_to_send,
base::TimeDelta::FromMilliseconds(kMinSchedulingDelayMs));
cast_environment_->PostDelayedTask(CastEnvironment::MAIN, FROM_HERE,
base::Bind(&VideoReceiver::SendNextCastMessage,
weak_factory_.GetWeakPtr()), time_to_send);
}
void VideoReceiver::SendNextCastMessage() {
framer_->SendCastMessage(); // Will only send a message if it is time.
ScheduleNextCastMessage();
}
// Schedule the next RTCP report to be sent back to the sender.
void VideoReceiver::ScheduleNextRtcpReport() {
base::TimeDelta time_to_next = rtcp_->TimeToSendNextRtcpReport() -
cast_environment_->Clock()->NowTicks();
time_to_next = std::max(time_to_next,
base::TimeDelta::FromMilliseconds(kMinSchedulingDelayMs));
cast_environment_->PostDelayedTask(CastEnvironment::MAIN, FROM_HERE,
base::Bind(&VideoReceiver::SendNextRtcpReport,
weak_factory_.GetWeakPtr()), time_to_next);
}
void VideoReceiver::SendNextRtcpReport() {
rtcp_->SendRtcpReport(incoming_ssrc_);
ScheduleNextRtcpReport();
}
} // namespace cast
} // namespace media