| // Copyright 2020 The Chromium Authors. All rights reserved. |
| // Use of this source code is governed by a BSD-style license that can be |
| // found in the LICENSE file. |
| |
| #include "cast/standalone_sender/streaming_opus_encoder.h" |
| |
| #include <opus/opus.h> |
| |
| #include <algorithm> |
| #include <chrono> |
| |
| #include "util/chrono_helpers.h" |
| |
| namespace openscreen { |
| namespace cast { |
| |
| using openscreen::operator<<; // To pretty-print chrono values. |
| |
| namespace { |
| |
| // The bitrate at which virtually all stereo audio can be encoded and decoded |
| // without human-perceivable artifacts. Source: |
| // https://wiki.hydrogenaud.io/index.php?title=Opus#Bitrate_performance |
| constexpr opus_int32 kTransparentBitrate = 160000; |
| |
| // The maximum number of Cast audio frames the encoder may fall behind by before |
| // skipping-ahead the RTP timestamps to compensate. |
| constexpr int kMaxCastFramesBeforeSkip = 3; |
| |
| } // namespace |
| |
| StreamingOpusEncoder::StreamingOpusEncoder(int num_channels, |
| int cast_frames_per_second, |
| Sender* sender) |
| : num_channels_(num_channels), |
| sender_(sender), |
| samples_per_cast_frame_(sample_rate() / cast_frames_per_second), |
| approximate_cast_frame_duration_(Clock::to_duration(seconds(1)) / |
| cast_frames_per_second), |
| encoder_storage_(new uint8_t[opus_encoder_get_size(num_channels_)]), |
| input_(new float[num_channels_ * samples_per_cast_frame_]), |
| output_(new uint8_t[kOpusMaxPayloadSize]) { |
| OSP_CHECK_GT(cast_frames_per_second, 0); |
| OSP_DCHECK(sender_); |
| OSP_CHECK_GT(samples_per_cast_frame_, 0); |
| OSP_CHECK_EQ(sample_rate() % cast_frames_per_second, 0); |
| OSP_CHECK(approximate_cast_frame_duration_ > Clock::duration::zero()); |
| |
| frame_.dependency = EncodedFrame::KEY_FRAME; |
| |
| const auto init_result = opus_encoder_init( |
| encoder(), sample_rate(), num_channels_, OPUS_APPLICATION_AUDIO); |
| OSP_CHECK_EQ(init_result, OPUS_OK); |
| |
| UseStandardQuality(); |
| } |
| |
| StreamingOpusEncoder::~StreamingOpusEncoder() = default; |
| |
| int StreamingOpusEncoder::GetBitrate() const { |
| opus_int32 bitrate; |
| const auto ctl_result = |
| opus_encoder_ctl(encoder(), OPUS_GET_BITRATE(&bitrate)); |
| OSP_CHECK_EQ(ctl_result, OPUS_OK); |
| return bitrate; |
| } |
| |
| void StreamingOpusEncoder::UseStandardQuality() { |
| const auto ctl_result = |
| opus_encoder_ctl(encoder(), OPUS_SET_BITRATE(OPUS_AUTO)); |
| OSP_CHECK_EQ(ctl_result, OPUS_OK); |
| UpdateCodecDelay(); |
| } |
| |
| void StreamingOpusEncoder::UseHighQuality() { |
| // kTransparentBitrate assumes stereo audio. Scale it by the actual number of |
| // channels. |
| const opus_int32 bitrate = kTransparentBitrate * num_channels_ / 2; |
| const auto ctl_result = |
| opus_encoder_ctl(encoder(), OPUS_SET_BITRATE(bitrate)); |
| OSP_CHECK_EQ(ctl_result, OPUS_OK); |
| UpdateCodecDelay(); |
| } |
| |
| void StreamingOpusEncoder::EncodeAndSend(const float* interleaved_samples, |
| int num_samples, |
| Clock::time_point reference_time) { |
| OSP_DCHECK(interleaved_samples); |
| OSP_DCHECK_GT(num_samples, 0); |
| |
| ResolveTimestampsAndMaybeSkip(reference_time); |
| |
| while (num_samples > 0) { |
| const int samples_copied = |
| FillInputBuffer(interleaved_samples, num_samples); |
| num_samples -= samples_copied; |
| interleaved_samples += num_channels_ * samples_copied; |
| |
| if (num_samples_queued_ < samples_per_cast_frame_) { |
| return; // Not enough yet for a full Cast audio frame. |
| } |
| |
| const opus_int32 packet_size_or_error = |
| opus_encode_float(encoder(), input_.get(), num_samples_queued_, |
| output_.get(), kOpusMaxPayloadSize); |
| num_samples_queued_ = 0; |
| if (packet_size_or_error < 0) { |
| OSP_LOG_FATAL << "AUDIO[" << sender_->ssrc() |
| << "] Error code from opus_encode_float(): " |
| << packet_size_or_error; |
| return; |
| } |
| |
| frame_.frame_id = sender_->GetNextFrameId(); |
| frame_.referenced_frame_id = frame_.frame_id; |
| // Note: It's possible for Opus to encode a zero byte packet. Send a Cast |
| // audio frame anyway, to represent the passage of silence and to send other |
| // stream metadata. |
| frame_.data = absl::Span<uint8_t>(output_.get(), packet_size_or_error); |
| last_sent_frame_reference_time_ = frame_.reference_time; |
| switch (sender_->EnqueueFrame(frame_)) { |
| case Sender::OK: |
| break; |
| case Sender::PAYLOAD_TOO_LARGE: |
| OSP_NOTREACHED(); // The Opus packet cannot possibly be too large. |
| break; |
| case Sender::REACHED_ID_SPAN_LIMIT: |
| OSP_LOG_WARN << "AUDIO[" << sender_->ssrc() |
| << "] Dropping: FrameId span limit reached."; |
| break; |
| case Sender::MAX_DURATION_IN_FLIGHT: |
| OSP_LOG_INFO << "AUDIO[" << sender_->ssrc() |
| << "] Dropping: In-flight duration would be too high."; |
| break; |
| } |
| |
| frame_.rtp_timestamp += RtpTimeDelta::FromTicks(samples_per_cast_frame_); |
| frame_.reference_time += approximate_cast_frame_duration_; |
| } |
| } |
| |
| void StreamingOpusEncoder::UpdateCodecDelay() { |
| opus_int32 lookahead = 0; |
| const auto ctl_result = |
| opus_encoder_ctl(encoder(), OPUS_GET_LOOKAHEAD(&lookahead)); |
| OSP_CHECK_EQ(ctl_result, OPUS_OK); |
| codec_delay_ = RtpTimeDelta::FromTicks(lookahead).ToDuration<Clock::duration>( |
| sample_rate()); |
| } |
| |
| void StreamingOpusEncoder::ResolveTimestampsAndMaybeSkip( |
| Clock::time_point reference_time) { |
| // Back-track the reference time to account for the audio delay introduced by |
| // the codec. |
| reference_time -= codec_delay_; |
| |
| // Special case: Nothing special for the first frame's timestamps. |
| if (start_time_ == Clock::time_point::min()) { |
| frame_.rtp_timestamp = RtpTimeTicks(); |
| frame_.reference_time = start_time_ = reference_time; |
| last_sent_frame_reference_time_ = |
| reference_time - approximate_cast_frame_duration_; |
| return; |
| } |
| |
| const RtpTimeTicks current_position = |
| frame_.rtp_timestamp + RtpTimeDelta::FromTicks(num_samples_queued_); |
| const RtpTimeTicks reference_position = RtpTimeTicks::FromTimeSinceOrigin( |
| reference_time - start_time_, sample_rate()); |
| const RtpTimeDelta rtp_advancement = reference_position - current_position; |
| const RtpTimeDelta skip_threshold = |
| RtpTimeDelta::FromTicks(samples_per_cast_frame_) * |
| kMaxCastFramesBeforeSkip; |
| if (rtp_advancement > skip_threshold) { |
| OSP_LOG_WARN << "Detected audio gap " |
| << rtp_advancement.ToDuration<microseconds>(sample_rate()) |
| << ", skipping ahead..."; |
| num_samples_queued_ = 0; |
| frame_.rtp_timestamp = reference_position; |
| } |
| |
| // Further back-track the reference time to account for the already-queued |
| // samples. |
| reference_time -= RtpTimeDelta::FromTicks(num_samples_queued_) |
| .ToDuration<Clock::duration>(sample_rate()); |
| |
| // Frame reference times must be monotonically increasing. A little noise in |
| // the negative direction is okay to cap-off. Log a warning if there's a |
| // bigger problem (at the source). |
| const Clock::time_point lower_bound = |
| last_sent_frame_reference_time_ + |
| RtpTimeDelta::FromTicks(1).ToDuration<Clock::duration>(sample_rate()); |
| if (reference_time < lower_bound) { |
| const Clock::duration backwards_amount = |
| last_sent_frame_reference_time_ - reference_time; |
| OSP_LOG_IF(WARN, backwards_amount >= approximate_cast_frame_duration_) |
| << "Reference time went *backwards* too much (" << backwards_amount |
| << " in wrong direction). A/V sync may suffer at the Receiver!"; |
| reference_time = lower_bound; |
| } |
| |
| frame_.reference_time = reference_time; |
| } |
| |
| int StreamingOpusEncoder::FillInputBuffer(const float* interleaved_samples, |
| int num_samples) { |
| const int samples_needed = samples_per_cast_frame_ - num_samples_queued_; |
| const int samples_to_copy = std::min(num_samples, samples_needed); |
| std::copy(interleaved_samples, |
| interleaved_samples + num_channels_ * samples_to_copy, |
| input_.get() + num_channels_ * num_samples_queued_); |
| num_samples_queued_ += samples_to_copy; |
| return samples_to_copy; |
| } |
| |
| // static |
| constexpr int StreamingOpusEncoder::kDefaultCastAudioFramesPerSecond; |
| // static |
| constexpr int StreamingOpusEncoder::kOpusMaxPayloadSize; |
| |
| } // namespace cast |
| } // namespace openscreen |