media/cast/video_sender/video_sender.cc

// Copyright 2013 The Chromium Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.

#include "media/cast/video_sender/video_sender.h"

#include <list>

#include "base/bind.h"
#include "base/logging.h"
#include "base/message_loop/message_loop.h"
#include "crypto/encryptor.h"
#include "crypto/symmetric_key.h"
#include "media/cast/cast_defines.h"
#include "media/cast/net/pacing/paced_sender.h"
#include "media/cast/video_sender/video_encoder.h"

namespace media {
namespace cast {

const int64 kMinSchedulingDelayMs = 1;

class LocalRtcpVideoSenderFeedback : public RtcpSenderFeedback {
 public:
  explicit LocalRtcpVideoSenderFeedback(VideoSender* video_sender)
      : video_sender_(video_sender) {
  }

  virtual void OnReceivedCastFeedback(
      const RtcpCastMessage& cast_feedback) OVERRIDE {
    video_sender_->OnReceivedCastFeedback(cast_feedback);
  }

 private:
  VideoSender* video_sender_;
};

class LocalRtpVideoSenderStatistics : public RtpSenderStatistics {
 public:
  explicit LocalRtpVideoSenderStatistics(RtpSender* rtp_sender)
     : rtp_sender_(rtp_sender) {
  }

  virtual void GetStatistics(const base::TimeTicks& now,
                             RtcpSenderInfo* sender_info) OVERRIDE {
    rtp_sender_->RtpStatistics(now, sender_info);
  }

 private:
  RtpSender* rtp_sender_;
};

VideoSender::VideoSender(
    scoped_refptr<CastEnvironment> cast_environment,
    const VideoSenderConfig& video_config,
    VideoEncoderController* const video_encoder_controller,
    PacedPacketSender* const paced_packet_sender)
    : rtp_max_delay_(
          base::TimeDelta::FromMilliseconds(video_config.rtp_max_delay_ms)),
      max_frame_rate_(video_config.max_frame_rate),
      cast_environment_(cast_environment),
      rtcp_feedback_(new LocalRtcpVideoSenderFeedback(this)),
      rtp_sender_(new RtpSender(cast_environment, NULL, &video_config,
                                paced_packet_sender)),
      last_acked_frame_id_(-1),
      last_sent_frame_id_(-1),
      duplicate_ack_(0),
      last_skip_count_(0),
      congestion_control_(cast_environment->Clock(),
                          video_config.congestion_control_back_off,
                          video_config.max_bitrate,
                          video_config.min_bitrate,
                          video_config.start_bitrate),
      initialized_(false),
      weak_factory_(this) {
  max_unacked_frames_ = static_cast<uint8>(video_config.rtp_max_delay_ms *
      video_config.max_frame_rate / 1000) + 1;
  VLOG(1) << "max_unacked_frames " << static_cast<int>(max_unacked_frames_);
  DCHECK_GT(max_unacked_frames_, 0) << "Invalid argument";

  rtp_video_sender_statistics_.reset(
      new LocalRtpVideoSenderStatistics(rtp_sender_.get()));

  if (video_config.use_external_encoder) {
    DCHECK(video_encoder_controller) << "Invalid argument";
    video_encoder_controller_ = video_encoder_controller;
  } else {
    video_encoder_.reset(new VideoEncoder(cast_environment, video_config,
        max_unacked_frames_));
    video_encoder_controller_ = video_encoder_.get();
  }

  if (video_config.aes_iv_mask.size() == kAesKeySize &&
      video_config.aes_key.size() == kAesKeySize) {
    iv_mask_ = video_config.aes_iv_mask;
    crypto::SymmetricKey* key = crypto::SymmetricKey::Import(
        crypto::SymmetricKey::AES, video_config.aes_key);
    encryptor_.reset(new crypto::Encryptor());
    encryptor_->Init(key, crypto::Encryptor::CTR, std::string());
  } else if (video_config.aes_iv_mask.size() != 0 ||
             video_config.aes_key.size() != 0) {
    DCHECK(false) << "Invalid crypto configuration";
  }

  rtcp_.reset(new Rtcp(
      cast_environment_,
      rtcp_feedback_.get(),
      paced_packet_sender,
      rtp_video_sender_statistics_.get(),
      NULL,
      video_config.rtcp_mode,
      base::TimeDelta::FromMilliseconds(video_config.rtcp_interval),
      video_config.sender_ssrc,
      video_config.incoming_feedback_ssrc,
      video_config.rtcp_c_name));
}

VideoSender::~VideoSender() {}

void VideoSender::InitializeTimers() {
  DCHECK(cast_environment_->CurrentlyOn(CastEnvironment::MAIN));
  if (!initialized_) {
    initialized_ = true;
    ScheduleNextRtcpReport();
    ScheduleNextResendCheck();
    ScheduleNextSkippedFramesCheck();
  }
}

void VideoSender::InsertRawVideoFrame(
    const scoped_refptr<media::VideoFrame>& video_frame,
    const base::TimeTicks& capture_time) {
  DCHECK(cast_environment_->CurrentlyOn(CastEnvironment::MAIN));
  DCHECK(video_encoder_.get()) << "Invalid state";
  cast_environment_->Logging()->InsertFrameEvent(kVideoFrameReceived,
      GetVideoRtpTimestamp(capture_time), kFrameIdUnknown);

  if (!video_encoder_->EncodeVideoFrame(video_frame, capture_time,
      base::Bind(&VideoSender::SendEncodedVideoFrameMainThread,
          weak_factory_.GetWeakPtr()))) {
  }
}

void VideoSender::InsertCodedVideoFrame(const EncodedVideoFrame* encoded_frame,
                                        const base::TimeTicks& capture_time,
                                        const base::Closure callback) {
  DCHECK(!video_encoder_.get()) << "Invalid state";
  DCHECK(encoded_frame) << "Invalid argument";
  DCHECK(cast_environment_->CurrentlyOn(CastEnvironment::MAIN));

  SendEncodedVideoFrame(encoded_frame, capture_time);
  cast_environment_->PostTask(CastEnvironment::MAIN, FROM_HERE, callback);
}

void VideoSender::SendEncodedVideoFrameMainThread(
    scoped_ptr<EncodedVideoFrame> video_frame,
    const base::TimeTicks& capture_time) {
  SendEncodedVideoFrame(video_frame.get(), capture_time);
}

bool VideoSender::EncryptVideoFrame(const EncodedVideoFrame& video_frame,
                                    EncodedVideoFrame* encrypted_frame) {
  DCHECK(encryptor_) << "Invalid state";

  if (!encryptor_->SetCounter(GetAesNonce(video_frame.frame_id, iv_mask_))) {
    NOTREACHED() << "Failed to set counter";
    return false;
  }

  if (!encryptor_->Encrypt(video_frame.data, &encrypted_frame->data)) {
    NOTREACHED() << "Encrypt error";
    return false;
  }
  encrypted_frame->codec = video_frame.codec;
  encrypted_frame->key_frame = video_frame.key_frame;
  encrypted_frame->frame_id = video_frame.frame_id;
  encrypted_frame->last_referenced_frame_id =
      video_frame.last_referenced_frame_id;
  return true;
}

void VideoSender::SendEncodedVideoFrame(const EncodedVideoFrame* encoded_frame,
                                        const base::TimeTicks& capture_time) {
  DCHECK(cast_environment_->CurrentlyOn(CastEnvironment::MAIN));
  last_send_time_ = cast_environment_->Clock()->NowTicks();

  if (encryptor_) {
    EncodedVideoFrame encrypted_video_frame;

    if (!EncryptVideoFrame(*encoded_frame, &encrypted_video_frame)) {
      // Logging already done.
      return;
    }
    rtp_sender_->IncomingEncodedVideoFrame(&encrypted_video_frame,
                                           capture_time);
  } else {
    rtp_sender_->IncomingEncodedVideoFrame(encoded_frame, capture_time);
  }
  if (encoded_frame->key_frame) {
    VLOG(1) << "Send encoded key frame; frame_id:"
            << static_cast<int>(encoded_frame->frame_id);
  }
  last_sent_frame_id_ = static_cast<int>(encoded_frame->frame_id);
  UpdateFramesInFlight();
  InitializeTimers();
}

void VideoSender::IncomingRtcpPacket(const uint8* packet, size_t length,
                                     const base::Closure callback) {
  DCHECK(cast_environment_->CurrentlyOn(CastEnvironment::MAIN));
  rtcp_->IncomingRtcpPacket(packet, length);
  cast_environment_->PostTask(CastEnvironment::MAIN, FROM_HERE, callback);
}

void VideoSender::ScheduleNextRtcpReport() {
  DCHECK(cast_environment_->CurrentlyOn(CastEnvironment::MAIN));
  base::TimeDelta time_to_next = rtcp_->TimeToSendNextRtcpReport() -
      cast_environment_->Clock()->NowTicks();

  time_to_next = std::max(time_to_next,
      base::TimeDelta::FromMilliseconds(kMinSchedulingDelayMs));

  cast_environment_->PostDelayedTask(CastEnvironment::MAIN, FROM_HERE,
      base::Bind(&VideoSender::SendRtcpReport, weak_factory_.GetWeakPtr()),
                 time_to_next);
}

void VideoSender::SendRtcpReport() {
  DCHECK(cast_environment_->CurrentlyOn(CastEnvironment::MAIN));

  RtcpSenderLogMessage sender_log_message;
  const FrameRawMap& frame_raw_map =
      cast_environment_->Logging()->GetFrameRawData();

  FrameRawMap::const_iterator it = frame_raw_map.begin();
  while (it != frame_raw_map.end()) {
    RtcpSenderFrameLogMessage frame_message;
    frame_message.rtp_timestamp = it->first;
    frame_message.frame_status = kRtcpSenderFrameStatusUnknown;
    if (it->second.type.empty()) {
      ++it;
      continue;
    }
    CastLoggingEvent last_event = it->second.type.back();
    switch (last_event) {
      case kVideoFrameCaptured:
        frame_message.frame_status = kRtcpSenderFrameStatusDroppedByFlowControl;
        break;
      case kVideoFrameSentToEncoder:
        frame_message.frame_status = kRtcpSenderFrameStatusDroppedByEncoder;
        break;
      case kVideoFrameEncoded:
        frame_message.frame_status = kRtcpSenderFrameStatusSentToNetwork;
        break;
      default:
        ++it;
        continue;
    }
    ++it;
    if (it == frame_raw_map.end()) {
      // Last message on our map; only send if it is kVideoFrameEncoded.
      if (last_event != kVideoFrameEncoded) {
        // For other events we will wait for it to finish and report the result
        // in the next report.
        break;
      }
    }
    sender_log_message.push_back(frame_message);
  }
  rtcp_->SendRtcpFromRtpSender(&sender_log_message);
  if (!sender_log_message.empty()) {
    VLOG(1) << "Failed to send all log messages";
  }

  // TODO(pwestin): When we start pulling out the logging by other means we need
  // to synchronize this.
  cast_environment_->Logging()->Reset();
  ScheduleNextRtcpReport();
}

void VideoSender::ScheduleNextResendCheck() {
  DCHECK(cast_environment_->CurrentlyOn(CastEnvironment::MAIN));
  base::TimeDelta time_to_next;
  if (last_send_time_.is_null()) {
    time_to_next = rtp_max_delay_;
  } else {
    time_to_next = last_send_time_ - cast_environment_->Clock()->NowTicks() +
        rtp_max_delay_;
  }
  time_to_next = std::max(time_to_next,
      base::TimeDelta::FromMilliseconds(kMinSchedulingDelayMs));

  cast_environment_->PostDelayedTask(CastEnvironment::MAIN, FROM_HERE,
      base::Bind(&VideoSender::ResendCheck, weak_factory_.GetWeakPtr()),
                 time_to_next);
}

void VideoSender::ResendCheck() {
  DCHECK(cast_environment_->CurrentlyOn(CastEnvironment::MAIN));
  if (!last_send_time_.is_null() && last_sent_frame_id_ != -1) {
    base::TimeDelta time_since_last_send =
       cast_environment_->Clock()->NowTicks() - last_send_time_;
    if (time_since_last_send > rtp_max_delay_) {
      if (last_acked_frame_id_ == -1) {
        // We have not received any ack, send a key frame.
        video_encoder_controller_->GenerateKeyFrame();
        last_acked_frame_id_ = -1;
        last_sent_frame_id_ = -1;
        UpdateFramesInFlight();
      } else {
        DCHECK_LE(0, last_acked_frame_id_);

        uint32 frame_id = static_cast<uint32>(last_acked_frame_id_ + 1);
        VLOG(1) << "ACK timeout resend frame:" << static_cast<int>(frame_id);
        ResendFrame(frame_id);
      }
    }
  }
  ScheduleNextResendCheck();
}

void VideoSender::ScheduleNextSkippedFramesCheck() {
  DCHECK(cast_environment_->CurrentlyOn(CastEnvironment::MAIN));
  base::TimeDelta time_to_next;
  if (last_checked_skip_count_time_.is_null()) {
    time_to_next =
        base::TimeDelta::FromMilliseconds(kSkippedFramesCheckPeriodkMs);
  } else {
    time_to_next = last_checked_skip_count_time_ -
        cast_environment_->Clock()->NowTicks() +
        base::TimeDelta::FromMilliseconds(kSkippedFramesCheckPeriodkMs);
  }
  time_to_next = std::max(time_to_next,
      base::TimeDelta::FromMilliseconds(kMinSchedulingDelayMs));

  cast_environment_->PostDelayedTask(CastEnvironment::MAIN, FROM_HERE,
      base::Bind(&VideoSender::SkippedFramesCheck, weak_factory_.GetWeakPtr()),
                 time_to_next);
}

void VideoSender::SkippedFramesCheck() {
  DCHECK(cast_environment_->CurrentlyOn(CastEnvironment::MAIN));
  int skip_count = video_encoder_controller_->NumberOfSkippedFrames();
  if (skip_count - last_skip_count_ >
      kSkippedFramesThreshold * max_frame_rate_) {
      // TODO(pwestin): Propagate this up to the application.
  }
  last_skip_count_ = skip_count;
  last_checked_skip_count_time_ = cast_environment_->Clock()->NowTicks();
  ScheduleNextSkippedFramesCheck();
}

void VideoSender::OnReceivedCastFeedback(const RtcpCastMessage& cast_feedback) {
  DCHECK(cast_environment_->CurrentlyOn(CastEnvironment::MAIN));
  base::TimeDelta rtt;
  base::TimeDelta avg_rtt;
  base::TimeDelta min_rtt;
  base::TimeDelta max_rtt;

  if (rtcp_->Rtt(&rtt, &avg_rtt, &min_rtt, &max_rtt)) {
    cast_environment_->Logging()->InsertGenericEvent(kRttMs,
        rtt.InMilliseconds());
    // Don't use a RTT lower than our average.
    rtt = std::max(rtt, avg_rtt);
  } else {
    // We have no measured value use default.
    rtt = base::TimeDelta::FromMilliseconds(kStartRttMs);
  }
  if (cast_feedback.missing_frames_and_packets_.empty()) {
    // No lost packets.
    int resend_frame = -1;
    if (last_sent_frame_id_ == -1) return;

    video_encoder_controller_->LatestFrameIdToReference(
        cast_feedback.ack_frame_id_);

    if (static_cast<uint32>(last_acked_frame_id_ + 1) ==
        cast_feedback.ack_frame_id_) {
      uint32 new_bitrate = 0;
      if (congestion_control_.OnAck(rtt, &new_bitrate)) {
        video_encoder_controller_->SetBitRate(new_bitrate);
      }
    }
    if (static_cast<uint32>(last_acked_frame_id_) == cast_feedback.ack_frame_id_
        // We only count duplicate ACKs when we have sent newer frames.
        && IsNewerFrameId(last_sent_frame_id_, last_acked_frame_id_)) {
      duplicate_ack_++;
    } else {
      duplicate_ack_ = 0;
    }
    if (duplicate_ack_ >= 2 && duplicate_ack_ % 3 == 2) {
      // Resend last ACK + 1 frame.
      resend_frame = static_cast<uint32>(last_acked_frame_id_ + 1);
    }
    if (resend_frame != -1) {
      DCHECK_LE(0, resend_frame);
      VLOG(1) << "Received duplicate ACK for frame:"
              << static_cast<int>(resend_frame);
      ResendFrame(static_cast<uint32>(resend_frame));
    }
  } else {
    rtp_sender_->ResendPackets(cast_feedback.missing_frames_and_packets_);
    last_send_time_ = cast_environment_->Clock()->NowTicks();

    uint32 new_bitrate = 0;
    if (congestion_control_.OnNack(rtt, &new_bitrate)) {
      video_encoder_controller_->SetBitRate(new_bitrate);
    }
  }
  ReceivedAck(cast_feedback.ack_frame_id_);
}

void VideoSender::ReceivedAck(uint32 acked_frame_id) {
  DCHECK(cast_environment_->CurrentlyOn(CastEnvironment::MAIN));
  last_acked_frame_id_ = static_cast<int>(acked_frame_id);
  cast_environment_->Logging()->InsertGenericEvent(kAckReceived,
                                                   acked_frame_id);
  VLOG(1) << "ReceivedAck:" << static_cast<int>(acked_frame_id);
  last_acked_frame_id_ = acked_frame_id;
  UpdateFramesInFlight();
}

void VideoSender::UpdateFramesInFlight() {
  DCHECK(cast_environment_->CurrentlyOn(CastEnvironment::MAIN));
  if (last_sent_frame_id_ != -1) {
    DCHECK_LE(0, last_sent_frame_id_);
    uint32 frames_in_flight;
    if (last_acked_frame_id_ != -1) {
      DCHECK_LE(0, last_acked_frame_id_);
      frames_in_flight = static_cast<uint32>(last_sent_frame_id_) -
                         static_cast<uint32>(last_acked_frame_id_);
    } else {
      frames_in_flight = static_cast<uint32>(last_sent_frame_id_) + 1;
    }
    VLOG(1) << "Frames in flight; last sent: " << last_sent_frame_id_
            << " last acked:" << last_acked_frame_id_;
    if (frames_in_flight >= max_unacked_frames_) {
      video_encoder_controller_->SkipNextFrame(true);
      return;
    }
  }
  video_encoder_controller_->SkipNextFrame(false);
}

void VideoSender::ResendFrame(uint32 resend_frame_id) {
  DCHECK(cast_environment_->CurrentlyOn(CastEnvironment::MAIN));
  MissingFramesAndPacketsMap missing_frames_and_packets;
  PacketIdSet missing;
  missing_frames_and_packets.insert(std::make_pair(resend_frame_id, missing));
  rtp_sender_->ResendPackets(missing_frames_and_packets);
  last_send_time_ = cast_environment_->Clock()->NowTicks();
}

}  // namespace cast
}  // namespace media