webrtc/modules/video_coding/main/source/video_receiver.cc

/*
 *  Copyright (c) 2013 The WebRTC project authors. All Rights Reserved.
 *
 *  Use of this source code is governed by a BSD-style license
 *  that can be found in the LICENSE file in the root of the source
 *  tree. An additional intellectual property rights grant can be found
 *  in the file PATENTS.  All contributing project authors may
 *  be found in the AUTHORS file in the root of the source tree.
 */

#include "webrtc/common_types.h"
#include "webrtc/common_video/libyuv/include/webrtc_libyuv.h"
#include "webrtc/modules/video_coding/codecs/interface/video_codec_interface.h"
#include "webrtc/modules/video_coding/main/source/encoded_frame.h"
#include "webrtc/modules/video_coding/main/source/jitter_buffer.h"
#include "webrtc/modules/video_coding/main/source/packet.h"
#include "webrtc/modules/video_coding/main/source/video_coding_impl.h"
#include "webrtc/system_wrappers/interface/clock.h"
#include "webrtc/system_wrappers/interface/logging.h"
#include "webrtc/system_wrappers/interface/trace_event.h"

// #define DEBUG_DECODER_BIT_STREAM

namespace webrtc {
namespace vcm {

VideoReceiver::VideoReceiver(Clock* clock, EventFactory* event_factory)
    : clock_(clock),
      process_crit_sect_(CriticalSectionWrapper::CreateCriticalSection()),
      _receiveCritSect(CriticalSectionWrapper::CreateCriticalSection()),
      _receiverInited(false),
      _timing(clock_),
      _receiver(&_timing, clock_, event_factory, true),
      _decodedFrameCallback(_timing, clock_),
      _frameTypeCallback(NULL),
      _receiveStatsCallback(NULL),
      _decoderTimingCallback(NULL),
      _packetRequestCallback(NULL),
      render_buffer_callback_(NULL),
      _decoder(NULL),
#ifdef DEBUG_DECODER_BIT_STREAM
      _bitStreamBeforeDecoder(NULL),
#endif
      _frameFromFile(),
      _keyRequestMode(kKeyOnError),
      _scheduleKeyRequest(false),
      max_nack_list_size_(0),
      pre_decode_image_callback_(NULL),
      _codecDataBase(),
      _receiveStatsTimer(1000, clock_),
      _retransmissionTimer(10, clock_),
      _keyRequestTimer(500, clock_) {
  assert(clock_);
#ifdef DEBUG_DECODER_BIT_STREAM
  _bitStreamBeforeDecoder = fopen("decoderBitStream.bit", "wb");
#endif
}

VideoReceiver::~VideoReceiver() {
  delete _receiveCritSect;
#ifdef DEBUG_DECODER_BIT_STREAM
  fclose(_bitStreamBeforeDecoder);
#endif
}

int32_t VideoReceiver::Process() {
  int32_t returnValue = VCM_OK;

  // Receive-side statistics
  if (_receiveStatsTimer.TimeUntilProcess() == 0) {
    _receiveStatsTimer.Processed();
    CriticalSectionScoped cs(process_crit_sect_.get());
    if (_receiveStatsCallback != NULL) {
      uint32_t bitRate;
      uint32_t frameRate;
      _receiver.ReceiveStatistics(&bitRate, &frameRate);
      _receiveStatsCallback->OnReceiveStatisticsUpdate(bitRate, frameRate);
    }

    if (_decoderTimingCallback != NULL) {
      int decode_ms;
      int max_decode_ms;
      int current_delay_ms;
      int target_delay_ms;
      int jitter_buffer_ms;
      int min_playout_delay_ms;
      int render_delay_ms;
      _timing.GetTimings(&decode_ms,
                         &max_decode_ms,
                         &current_delay_ms,
                         &target_delay_ms,
                         &jitter_buffer_ms,
                         &min_playout_delay_ms,
                         &render_delay_ms);
      _decoderTimingCallback->OnDecoderTiming(decode_ms,
                                              max_decode_ms,
                                              current_delay_ms,
                                              target_delay_ms,
                                              jitter_buffer_ms,
                                              min_playout_delay_ms,
                                              render_delay_ms);
    }

    // Size of render buffer.
    if (render_buffer_callback_) {
      int buffer_size_ms = _receiver.RenderBufferSizeMs();
      render_buffer_callback_->RenderBufferSizeMs(buffer_size_ms);
    }
  }

  // Key frame requests
  if (_keyRequestTimer.TimeUntilProcess() == 0) {
    _keyRequestTimer.Processed();
    bool request_key_frame = false;
    {
      CriticalSectionScoped cs(process_crit_sect_.get());
      request_key_frame = _scheduleKeyRequest && _frameTypeCallback != NULL;
    }
    if (request_key_frame) {
      const int32_t ret = RequestKeyFrame();
      if (ret != VCM_OK && returnValue == VCM_OK) {
        returnValue = ret;
      }
    }
  }

  // Packet retransmission requests
  // TODO(holmer): Add API for changing Process interval and make sure it's
  // disabled when NACK is off.
  if (_retransmissionTimer.TimeUntilProcess() == 0) {
    _retransmissionTimer.Processed();
    bool callback_registered = false;
    uint16_t length;
    {
      CriticalSectionScoped cs(process_crit_sect_.get());
      length = max_nack_list_size_;
      callback_registered = _packetRequestCallback != NULL;
    }
    if (callback_registered && length > 0) {
      std::vector<uint16_t> nackList(length);
      const int32_t ret = NackList(&nackList[0], &length);
      if (ret != VCM_OK && returnValue == VCM_OK) {
        returnValue = ret;
      }
      if (ret == VCM_OK && length > 0) {
        CriticalSectionScoped cs(process_crit_sect_.get());
        if (_packetRequestCallback != NULL) {
          _packetRequestCallback->ResendPackets(&nackList[0], length);
        }
      }
    }
  }

  return returnValue;
}

int64_t VideoReceiver::TimeUntilNextProcess() {
  int64_t timeUntilNextProcess = _receiveStatsTimer.TimeUntilProcess();
  if (_receiver.NackMode() != kNoNack) {
    // We need a Process call more often if we are relying on
    // retransmissions
    timeUntilNextProcess =
        VCM_MIN(timeUntilNextProcess, _retransmissionTimer.TimeUntilProcess());
  }
  timeUntilNextProcess =
      VCM_MIN(timeUntilNextProcess, _keyRequestTimer.TimeUntilProcess());

  return timeUntilNextProcess;
}

int32_t VideoReceiver::SetReceiveChannelParameters(uint32_t rtt) {
  CriticalSectionScoped receiveCs(_receiveCritSect);
  _receiver.UpdateRtt(rtt);
  return 0;
}

// Enable or disable a video protection method.
// Note: This API should be deprecated, as it does not offer a distinction
// between the protection method and decoding with or without errors. If such a
// behavior is desired, use the following API: SetReceiverRobustnessMode.
int32_t VideoReceiver::SetVideoProtection(VCMVideoProtection videoProtection,
                                          bool enable) {
  // By default, do not decode with errors.
  _receiver.SetDecodeErrorMode(kNoErrors);
  switch (videoProtection) {
    case kProtectionNack:
    case kProtectionNackReceiver: {
      CriticalSectionScoped cs(_receiveCritSect);
      if (enable) {
        // Enable NACK and always wait for retransmits.
        _receiver.SetNackMode(kNack, -1, -1);
      } else {
        _receiver.SetNackMode(kNoNack, -1, -1);
      }
      break;
    }

    case kProtectionKeyOnLoss: {
      CriticalSectionScoped cs(_receiveCritSect);
      if (enable) {
        _keyRequestMode = kKeyOnLoss;
        _receiver.SetDecodeErrorMode(kWithErrors);
      } else if (_keyRequestMode == kKeyOnLoss) {
        _keyRequestMode = kKeyOnError;  // default mode
      } else {
        return VCM_PARAMETER_ERROR;
      }
      break;
    }

    case kProtectionKeyOnKeyLoss: {
      CriticalSectionScoped cs(_receiveCritSect);
      if (enable) {
        _keyRequestMode = kKeyOnKeyLoss;
      } else if (_keyRequestMode == kKeyOnKeyLoss) {
        _keyRequestMode = kKeyOnError;  // default mode
      } else {
        return VCM_PARAMETER_ERROR;
      }
      break;
    }

    case kProtectionNackFEC: {
      CriticalSectionScoped cs(_receiveCritSect);
      if (enable) {
        // Enable hybrid NACK/FEC. Always wait for retransmissions
        // and don't add extra delay when RTT is above
        // kLowRttNackMs.
        _receiver.SetNackMode(kNack, media_optimization::kLowRttNackMs, -1);
        _receiver.SetDecodeErrorMode(kNoErrors);
        _receiver.SetDecodeErrorMode(kNoErrors);
      } else {
        _receiver.SetNackMode(kNoNack, -1, -1);
      }
      break;
    }
    case kProtectionNackSender:
    case kProtectionFEC:
    case kProtectionPeriodicKeyFrames:
      // Ignore encoder modes.
      return VCM_OK;
  }
  return VCM_OK;
}

// Initialize receiver, resets codec database etc
int32_t VideoReceiver::InitializeReceiver() {
  int32_t ret = _receiver.Initialize();
  if (ret < 0) {
    return ret;
  }

  {
    CriticalSectionScoped receive_cs(_receiveCritSect);
    _codecDataBase.ResetReceiver();
    _timing.Reset();
    _receiverInited = true;
  }

  {
    CriticalSectionScoped process_cs(process_crit_sect_.get());
    _decoder = NULL;
    _decodedFrameCallback.SetUserReceiveCallback(NULL);
    _frameTypeCallback = NULL;
    _receiveStatsCallback = NULL;
    _decoderTimingCallback = NULL;
    _packetRequestCallback = NULL;
    _keyRequestMode = kKeyOnError;
    _scheduleKeyRequest = false;
  }

  return VCM_OK;
}

// Register a receive callback. Will be called whenever there is a new frame
// ready for rendering.
int32_t VideoReceiver::RegisterReceiveCallback(
    VCMReceiveCallback* receiveCallback) {
  CriticalSectionScoped cs(_receiveCritSect);
  _decodedFrameCallback.SetUserReceiveCallback(receiveCallback);
  return VCM_OK;
}

int32_t VideoReceiver::RegisterReceiveStatisticsCallback(
    VCMReceiveStatisticsCallback* receiveStats) {
  CriticalSectionScoped cs(process_crit_sect_.get());
  _receiveStatsCallback = receiveStats;
  return VCM_OK;
}

int32_t VideoReceiver::RegisterDecoderTimingCallback(
    VCMDecoderTimingCallback* decoderTiming) {
  CriticalSectionScoped cs(process_crit_sect_.get());
  _decoderTimingCallback = decoderTiming;
  return VCM_OK;
}

// Register an externally defined decoder/render object.
// Can be a decoder only or a decoder coupled with a renderer.
int32_t VideoReceiver::RegisterExternalDecoder(VideoDecoder* externalDecoder,
                                               uint8_t payloadType,
                                               bool internalRenderTiming) {
  CriticalSectionScoped cs(_receiveCritSect);
  if (externalDecoder == NULL) {
    // Make sure the VCM updates the decoder next time it decodes.
    _decoder = NULL;
    return _codecDataBase.DeregisterExternalDecoder(payloadType) ? 0 : -1;
  }
  return _codecDataBase.RegisterExternalDecoder(
             externalDecoder, payloadType, internalRenderTiming)
             ? 0
             : -1;
}

// Register a frame type request callback.
int32_t VideoReceiver::RegisterFrameTypeCallback(
    VCMFrameTypeCallback* frameTypeCallback) {
  CriticalSectionScoped cs(process_crit_sect_.get());
  _frameTypeCallback = frameTypeCallback;
  return VCM_OK;
}

int32_t VideoReceiver::RegisterPacketRequestCallback(
    VCMPacketRequestCallback* callback) {
  CriticalSectionScoped cs(process_crit_sect_.get());
  _packetRequestCallback = callback;
  return VCM_OK;
}

int VideoReceiver::RegisterRenderBufferSizeCallback(
    VCMRenderBufferSizeCallback* callback) {
  CriticalSectionScoped cs(process_crit_sect_.get());
  render_buffer_callback_ = callback;
  return VCM_OK;
}

// Decode next frame, blocking.
// Should be called as often as possible to get the most out of the decoder.
int32_t VideoReceiver::Decode(uint16_t maxWaitTimeMs) {
  int64_t nextRenderTimeMs;
  bool supports_render_scheduling;
  {
    CriticalSectionScoped cs(_receiveCritSect);
    if (!_receiverInited) {
      return VCM_UNINITIALIZED;
    }
    if (!_codecDataBase.DecoderRegistered()) {
      return VCM_NO_CODEC_REGISTERED;
    }
    supports_render_scheduling = _codecDataBase.SupportsRenderScheduling();
  }

  VCMEncodedFrame* frame = _receiver.FrameForDecoding(
      maxWaitTimeMs, nextRenderTimeMs, supports_render_scheduling);

  if (frame == NULL) {
    return VCM_FRAME_NOT_READY;
  } else {
    CriticalSectionScoped cs(_receiveCritSect);

    // If this frame was too late, we should adjust the delay accordingly
    _timing.UpdateCurrentDelay(frame->RenderTimeMs(),
                               clock_->TimeInMilliseconds());

    if (pre_decode_image_callback_) {
      EncodedImage encoded_image(frame->EncodedImage());
      pre_decode_image_callback_->Encoded(encoded_image);
    }

#ifdef DEBUG_DECODER_BIT_STREAM
    if (_bitStreamBeforeDecoder != NULL) {
      // Write bit stream to file for debugging purposes
      if (fwrite(
              frame->Buffer(), 1, frame->Length(), _bitStreamBeforeDecoder) !=
          frame->Length()) {
        return -1;
      }
    }
#endif
    const int32_t ret = Decode(*frame);
    _receiver.ReleaseFrame(frame);
    frame = NULL;
    if (ret != VCM_OK) {
      return ret;
    }
  }
  return VCM_OK;
}

int32_t VideoReceiver::RequestSliceLossIndication(
    const uint64_t pictureID) const {
  TRACE_EVENT1("webrtc", "RequestSLI", "picture_id", pictureID);
  CriticalSectionScoped cs(process_crit_sect_.get());
  if (_frameTypeCallback != NULL) {
    const int32_t ret =
        _frameTypeCallback->SliceLossIndicationRequest(pictureID);
    if (ret < 0) {
      return ret;
    }
  } else {
    return VCM_MISSING_CALLBACK;
  }
  return VCM_OK;
}

int32_t VideoReceiver::RequestKeyFrame() {
  TRACE_EVENT0("webrtc", "RequestKeyFrame");
  CriticalSectionScoped process_cs(process_crit_sect_.get());
  if (_frameTypeCallback != NULL) {
    const int32_t ret = _frameTypeCallback->RequestKeyFrame();
    if (ret < 0) {
      return ret;
    }
    _scheduleKeyRequest = false;
  } else {
    return VCM_MISSING_CALLBACK;
  }
  return VCM_OK;
}

// Must be called from inside the receive side critical section.
int32_t VideoReceiver::Decode(const VCMEncodedFrame& frame) {
  TRACE_EVENT_ASYNC_STEP1("webrtc",
                          "Video",
                          frame.TimeStamp(),
                          "Decode",
                          "type",
                          frame.FrameType());
  // Change decoder if payload type has changed
  const bool renderTimingBefore = _codecDataBase.SupportsRenderScheduling();
  _decoder =
      _codecDataBase.GetDecoder(frame.PayloadType(), &_decodedFrameCallback);
  if (renderTimingBefore != _codecDataBase.SupportsRenderScheduling()) {
    // Make sure we reset the decode time estimate since it will
    // be zero for codecs without render timing.
    _timing.ResetDecodeTime();
  }
  if (_decoder == NULL) {
    return VCM_NO_CODEC_REGISTERED;
  }
  // Decode a frame
  int32_t ret = _decoder->Decode(frame, clock_->TimeInMilliseconds());

  // Check for failed decoding, run frame type request callback if needed.
  bool request_key_frame = false;
  if (ret < 0) {
    if (ret == VCM_ERROR_REQUEST_SLI) {
      return RequestSliceLossIndication(
          _decodedFrameCallback.LastReceivedPictureID() + 1);
    } else {
      request_key_frame = true;
    }
  } else if (ret == VCM_REQUEST_SLI) {
    ret = RequestSliceLossIndication(
        _decodedFrameCallback.LastReceivedPictureID() + 1);
  }
  if (!frame.Complete() || frame.MissingFrame()) {
    switch (_keyRequestMode) {
      case kKeyOnKeyLoss: {
        if (frame.FrameType() == kVideoFrameKey) {
          request_key_frame = true;
          ret = VCM_OK;
        }
        break;
      }
      case kKeyOnLoss: {
        request_key_frame = true;
        ret = VCM_OK;
      }
      default:
        break;
    }
  }
  if (request_key_frame) {
    CriticalSectionScoped cs(process_crit_sect_.get());
    _scheduleKeyRequest = true;
  }
  TRACE_EVENT_ASYNC_END0("webrtc", "Video", frame.TimeStamp());
  return ret;
}

// Reset the decoder state
int32_t VideoReceiver::ResetDecoder() {
  bool reset_key_request = false;
  {
    CriticalSectionScoped cs(_receiveCritSect);
    if (_decoder != NULL) {
      _receiver.Initialize();
      _timing.Reset();
      reset_key_request = true;
      _decoder->Reset();
    }
  }
  if (reset_key_request) {
    CriticalSectionScoped cs(process_crit_sect_.get());
    _scheduleKeyRequest = false;
  }
  return VCM_OK;
}

// Register possible receive codecs, can be called multiple times
int32_t VideoReceiver::RegisterReceiveCodec(const VideoCodec* receiveCodec,
                                            int32_t numberOfCores,
                                            bool requireKeyFrame) {
  CriticalSectionScoped cs(_receiveCritSect);
  if (receiveCodec == NULL) {
    return VCM_PARAMETER_ERROR;
  }
  if (!_codecDataBase.RegisterReceiveCodec(
          receiveCodec, numberOfCores, requireKeyFrame)) {
    return -1;
  }
  return 0;
}

// Get current received codec
int32_t VideoReceiver::ReceiveCodec(VideoCodec* currentReceiveCodec) const {
  CriticalSectionScoped cs(_receiveCritSect);
  if (currentReceiveCodec == NULL) {
    return VCM_PARAMETER_ERROR;
  }
  return _codecDataBase.ReceiveCodec(currentReceiveCodec) ? 0 : -1;
}

// Get current received codec
VideoCodecType VideoReceiver::ReceiveCodec() const {
  CriticalSectionScoped cs(_receiveCritSect);
  return _codecDataBase.ReceiveCodec();
}

// Incoming packet from network parsed and ready for decode, non blocking.
int32_t VideoReceiver::IncomingPacket(const uint8_t* incomingPayload,
                                      size_t payloadLength,
                                      const WebRtcRTPHeader& rtpInfo) {
  if (rtpInfo.frameType == kVideoFrameKey) {
    TRACE_EVENT1("webrtc",
                 "VCM::PacketKeyFrame",
                 "seqnum",
                 rtpInfo.header.sequenceNumber);
  }
  if (incomingPayload == NULL) {
    // The jitter buffer doesn't handle non-zero payload lengths for packets
    // without payload.
    // TODO(holmer): We should fix this in the jitter buffer.
    payloadLength = 0;
  }
  const VCMPacket packet(incomingPayload, payloadLength, rtpInfo);
  int32_t ret = _receiver.InsertPacket(packet, rtpInfo.type.Video.width,
                                       rtpInfo.type.Video.height);
  // TODO(holmer): Investigate if this somehow should use the key frame
  // request scheduling to throttle the requests.
  if (ret == VCM_FLUSH_INDICATOR) {
    RequestKeyFrame();
    ResetDecoder();
  } else if (ret < 0) {
    return ret;
  }
  return VCM_OK;
}

// Minimum playout delay (used for lip-sync). This is the minimum delay required
// to sync with audio. Not included in  VideoCodingModule::Delay()
// Defaults to 0 ms.
int32_t VideoReceiver::SetMinimumPlayoutDelay(uint32_t minPlayoutDelayMs) {
  _timing.set_min_playout_delay(minPlayoutDelayMs);
  return VCM_OK;
}

// The estimated delay caused by rendering, defaults to
// kDefaultRenderDelayMs = 10 ms
int32_t VideoReceiver::SetRenderDelay(uint32_t timeMS) {
  _timing.set_render_delay(timeMS);
  return VCM_OK;
}

// Current video delay
int32_t VideoReceiver::Delay() const { return _timing.TargetVideoDelay(); }

// Nack list
int32_t VideoReceiver::NackList(uint16_t* nackList, uint16_t* size) {
  VCMNackStatus nackStatus = kNackOk;
  uint16_t nack_list_length = 0;
  // Collect sequence numbers from the default receiver
  // if in normal nack mode.
  if (_receiver.NackMode() != kNoNack) {
    nackStatus = _receiver.NackList(nackList, *size, &nack_list_length);
  }
  *size = nack_list_length;
  if (nackStatus == kNackKeyFrameRequest) {
      return RequestKeyFrame();
  }
  return VCM_OK;
}

uint32_t VideoReceiver::DiscardedPackets() const {
  return _receiver.DiscardedPackets();
}

int VideoReceiver::SetReceiverRobustnessMode(
    ReceiverRobustness robustnessMode,
    VCMDecodeErrorMode decode_error_mode) {
  CriticalSectionScoped cs(_receiveCritSect);
  switch (robustnessMode) {
    case VideoCodingModule::kNone:
      _receiver.SetNackMode(kNoNack, -1, -1);
      if (decode_error_mode == kNoErrors) {
        _keyRequestMode = kKeyOnLoss;
      } else {
        _keyRequestMode = kKeyOnError;
      }
      break;
    case VideoCodingModule::kHardNack:
      // Always wait for retransmissions (except when decoding with errors).
      _receiver.SetNackMode(kNack, -1, -1);
      _keyRequestMode = kKeyOnError;  // TODO(hlundin): On long NACK list?
      break;
    case VideoCodingModule::kSoftNack:
#if 1
      assert(false);  // TODO(hlundin): Not completed.
      return VCM_NOT_IMPLEMENTED;
#else
      // Enable hybrid NACK/FEC. Always wait for retransmissions and don't add
      // extra delay when RTT is above kLowRttNackMs.
      _receiver.SetNackMode(kNack, media_optimization::kLowRttNackMs, -1);
      _keyRequestMode = kKeyOnError;
      break;
#endif
    case VideoCodingModule::kReferenceSelection:
#if 1
      assert(false);  // TODO(hlundin): Not completed.
      return VCM_NOT_IMPLEMENTED;
#else
      if (decode_error_mode == kNoErrors) {
        return VCM_PARAMETER_ERROR;
      }
      _receiver.SetNackMode(kNoNack, -1, -1);
      break;
#endif
  }
  _receiver.SetDecodeErrorMode(decode_error_mode);
  return VCM_OK;
}

void VideoReceiver::SetDecodeErrorMode(VCMDecodeErrorMode decode_error_mode) {
  CriticalSectionScoped cs(_receiveCritSect);
  _receiver.SetDecodeErrorMode(decode_error_mode);
}

void VideoReceiver::SetNackSettings(size_t max_nack_list_size,
                                    int max_packet_age_to_nack,
                                    int max_incomplete_time_ms) {
  if (max_nack_list_size != 0) {
    CriticalSectionScoped process_cs(process_crit_sect_.get());
    max_nack_list_size_ = max_nack_list_size;
  }
  _receiver.SetNackSettings(
      max_nack_list_size, max_packet_age_to_nack, max_incomplete_time_ms);
}

int VideoReceiver::SetMinReceiverDelay(int desired_delay_ms) {
  return _receiver.SetMinReceiverDelay(desired_delay_ms);
}

void VideoReceiver::RegisterPreDecodeImageCallback(
    EncodedImageCallback* observer) {
  CriticalSectionScoped cs(_receiveCritSect);
  pre_decode_image_callback_ = observer;
}
void VideoReceiver::RegisterFrameCountObserver(
    FrameCountObserver* frame_count_observer) {
  _receiver.RegisterFrameCountObserver(frame_count_observer);
}

}  // namespace vcm
}  // namespace webrtc