blob: b41b2a82934f943d1e5ab9bb08d2cdc524c5fbcb [file] [log] [blame]
// Copyright 2020 The Chromium Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
#include "cast/standalone_sender/streaming_opus_encoder.h"
#include <opus/opus.h>
#include <algorithm>
#include <chrono>
#include "util/chrono_helpers.h"
namespace openscreen {
namespace cast {
using openscreen::operator<<; // To pretty-print chrono values.
namespace {
// The bitrate at which virtually all stereo audio can be encoded and decoded
// without human-perceivable artifacts. Source:
// https://wiki.hydrogenaud.io/index.php?title=Opus#Bitrate_performance
constexpr opus_int32 kTransparentBitrate = 160000;
// The maximum number of Cast audio frames the encoder may fall behind by before
// skipping-ahead the RTP timestamps to compensate.
constexpr int kMaxCastFramesBeforeSkip = 3;
} // namespace
StreamingOpusEncoder::StreamingOpusEncoder(int num_channels,
int cast_frames_per_second,
Sender* sender)
: num_channels_(num_channels),
sender_(sender),
samples_per_cast_frame_(sample_rate() / cast_frames_per_second),
approximate_cast_frame_duration_(
Clock::to_duration(std::chrono::seconds(1)) / cast_frames_per_second),
encoder_storage_(new uint8_t[opus_encoder_get_size(num_channels_)]),
input_(new float[num_channels_ * samples_per_cast_frame_]),
output_(new uint8_t[kOpusMaxPayloadSize]) {
OSP_CHECK_GT(cast_frames_per_second, 0);
OSP_DCHECK(sender_);
OSP_CHECK_GT(samples_per_cast_frame_, 0);
OSP_CHECK_EQ(sample_rate() % cast_frames_per_second, 0);
OSP_CHECK(approximate_cast_frame_duration_ > Clock::duration::zero());
frame_.dependency = EncodedFrame::KEY_FRAME;
const auto init_result = opus_encoder_init(
encoder(), sample_rate(), num_channels_, OPUS_APPLICATION_AUDIO);
OSP_CHECK_EQ(init_result, OPUS_OK);
UseStandardQuality();
}
StreamingOpusEncoder::~StreamingOpusEncoder() = default;
int StreamingOpusEncoder::GetBitrate() const {
opus_int32 bitrate;
const auto ctl_result =
opus_encoder_ctl(encoder(), OPUS_GET_BITRATE(&bitrate));
OSP_CHECK_EQ(ctl_result, OPUS_OK);
return bitrate;
}
void StreamingOpusEncoder::UseStandardQuality() {
const auto ctl_result =
opus_encoder_ctl(encoder(), OPUS_SET_BITRATE(OPUS_AUTO));
OSP_CHECK_EQ(ctl_result, OPUS_OK);
UpdateCodecDelay();
}
void StreamingOpusEncoder::UseHighQuality() {
// kTransparentBitrate assumes stereo audio. Scale it by the actual number of
// channels.
const opus_int32 bitrate = kTransparentBitrate * num_channels_ / 2;
const auto ctl_result =
opus_encoder_ctl(encoder(), OPUS_SET_BITRATE(bitrate));
OSP_CHECK_EQ(ctl_result, OPUS_OK);
UpdateCodecDelay();
}
void StreamingOpusEncoder::EncodeAndSend(const float* interleaved_samples,
int num_samples,
Clock::time_point reference_time) {
OSP_DCHECK(interleaved_samples);
OSP_DCHECK_GT(num_samples, 0);
ResolveTimestampsAndMaybeSkip(reference_time);
while (num_samples > 0) {
const int samples_copied =
FillInputBuffer(interleaved_samples, num_samples);
num_samples -= samples_copied;
interleaved_samples += num_channels_ * samples_copied;
if (num_samples_queued_ < samples_per_cast_frame_) {
return; // Not enough yet for a full Cast audio frame.
}
const opus_int32 packet_size_or_error =
opus_encode_float(encoder(), input_.get(), num_samples_queued_,
output_.get(), kOpusMaxPayloadSize);
num_samples_queued_ = 0;
if (packet_size_or_error < 0) {
OSP_LOG_FATAL << "AUDIO[" << sender_->ssrc()
<< "] Error code from opus_encode_float(): "
<< packet_size_or_error;
return;
}
frame_.frame_id = sender_->GetNextFrameId();
frame_.referenced_frame_id = frame_.frame_id;
// Note: It's possible for Opus to encode a zero byte packet. Send a Cast
// audio frame anyway, to represent the passage of silence and to send other
// stream metadata.
frame_.data = absl::Span<uint8_t>(output_.get(), packet_size_or_error);
last_sent_frame_reference_time_ = frame_.reference_time;
switch (sender_->EnqueueFrame(frame_)) {
case Sender::OK:
break;
case Sender::PAYLOAD_TOO_LARGE:
OSP_NOTREACHED(); // The Opus packet cannot possibly be too large.
break;
case Sender::REACHED_ID_SPAN_LIMIT:
OSP_LOG_WARN << "AUDIO[" << sender_->ssrc()
<< "] Dropping: FrameId span limit reached.";
break;
case Sender::MAX_DURATION_IN_FLIGHT:
OSP_LOG_INFO << "AUDIO[" << sender_->ssrc()
<< "] Dropping: In-flight duration would be too high.";
break;
}
frame_.rtp_timestamp += RtpTimeDelta::FromTicks(samples_per_cast_frame_);
frame_.reference_time += approximate_cast_frame_duration_;
}
}
void StreamingOpusEncoder::UpdateCodecDelay() {
opus_int32 lookahead = 0;
const auto ctl_result =
opus_encoder_ctl(encoder(), OPUS_GET_LOOKAHEAD(&lookahead));
OSP_CHECK_EQ(ctl_result, OPUS_OK);
codec_delay_ = RtpTimeDelta::FromTicks(lookahead).ToDuration<Clock::duration>(
sample_rate());
}
void StreamingOpusEncoder::ResolveTimestampsAndMaybeSkip(
Clock::time_point reference_time) {
// Back-track the reference time to account for the audio delay introduced by
// the codec.
reference_time -= codec_delay_;
// Special case: Nothing special for the first frame's timestamps.
if (start_time_ == Clock::time_point::min()) {
frame_.rtp_timestamp = RtpTimeTicks();
frame_.reference_time = start_time_ = reference_time;
last_sent_frame_reference_time_ =
reference_time - approximate_cast_frame_duration_;
return;
}
const RtpTimeTicks current_position =
frame_.rtp_timestamp + RtpTimeDelta::FromTicks(num_samples_queued_);
const RtpTimeTicks reference_position = RtpTimeTicks::FromTimeSinceOrigin(
reference_time - start_time_, sample_rate());
const RtpTimeDelta rtp_advancement = reference_position - current_position;
const RtpTimeDelta skip_threshold =
RtpTimeDelta::FromTicks(samples_per_cast_frame_) *
kMaxCastFramesBeforeSkip;
if (rtp_advancement > skip_threshold) {
OSP_LOG_WARN << "Detected audio gap "
<< rtp_advancement.ToDuration<microseconds>(sample_rate())
<< ", skipping ahead...";
num_samples_queued_ = 0;
frame_.rtp_timestamp = reference_position;
}
// Further back-track the reference time to account for the already-queued
// samples.
reference_time -= RtpTimeDelta::FromTicks(num_samples_queued_)
.ToDuration<Clock::duration>(sample_rate());
// Frame reference times must be monotonically increasing. A little noise in
// the negative direction is okay to cap-off. Log a warning if there's a
// bigger problem (at the source).
const Clock::time_point lower_bound =
last_sent_frame_reference_time_ +
RtpTimeDelta::FromTicks(1).ToDuration<Clock::duration>(sample_rate());
if (reference_time < lower_bound) {
const Clock::duration backwards_amount =
last_sent_frame_reference_time_ - reference_time;
OSP_LOG_IF(WARN, backwards_amount >= approximate_cast_frame_duration_)
<< "Reference time went *backwards* too much (" << backwards_amount
<< " in wrong direction). A/V sync may suffer at the Receiver!";
reference_time = lower_bound;
}
frame_.reference_time = reference_time;
}
int StreamingOpusEncoder::FillInputBuffer(const float* interleaved_samples,
int num_samples) {
const int samples_needed = samples_per_cast_frame_ - num_samples_queued_;
const int samples_to_copy = std::min(num_samples, samples_needed);
std::copy(interleaved_samples,
interleaved_samples + num_channels_ * samples_to_copy,
input_.get() + num_channels_ * num_samples_queued_);
num_samples_queued_ += samples_to_copy;
return samples_to_copy;
}
// static
constexpr int StreamingOpusEncoder::kDefaultCastAudioFramesPerSecond;
// static
constexpr int StreamingOpusEncoder::kOpusMaxPayloadSize;
} // namespace cast
} // namespace openscreen