Stream opus instead of raw pcm16
Bug: 120082384
Test: stream youtube on pie
Change-Id: I7b166722b75b9ee95d4ab77d9eb0d4710fd46391
diff --git a/host/frontend/stream_audio/Android.bp b/host/frontend/stream_audio/Android.bp
index ba9388e..ab5ff8e 100644
--- a/host/frontend/stream_audio/Android.bp
+++ b/host/frontend/stream_audio/Android.bp
@@ -23,12 +23,14 @@
"libbase",
"libcuttlefish_utils",
"libcuttlefish_fs",
+ "libopus",
"vsoc_lib",
],
static_libs: [
"libcuttlefish_host_config",
"libjsoncpp",
"libgflags",
+ "libopuscpp",
],
cpp_std: "c++17",
defaults: ["cuttlefish_host_only"],
diff --git a/host/frontend/stream_audio/main.cpp b/host/frontend/stream_audio/main.cpp
index 53621f2..f63e315 100644
--- a/host/frontend/stream_audio/main.cpp
+++ b/host/frontend/stream_audio/main.cpp
@@ -15,20 +15,20 @@
* limitations under the License.
*/
-// TODO(b/120082384) I plan on changing this to use something better than pcm
-// this is just to get something minimal running.
-//
// For each client that connects initially a header is sent with the following,
// in this order, all as uint16_t in network-byte-order:
-// sample_width, number of channels, frame rate
+// number of channels, frame rate
//
// Following, audio packets are sent as a uint32_t length (network byte order)
+// indicating the number of bytes
+// followed by the (opus) frame_size as a uint32_t
// followed by <length> bytes.
#include "common/libs/tcp_socket/tcp_socket.h"
#include "common/vsoc/lib/audio_data_region_view.h"
#include "common/vsoc/lib/circqueue_impl.h"
#include "common/vsoc/lib/vsoc_audio_message.h"
+#include "host/frontend/stream_audio/opuscpp/opus_wrapper.h"
#include "host/libs/config/cuttlefish_config.h"
#include <android-base/logging.h>
@@ -60,11 +60,25 @@
buffer_cv_.wait(guard);
}
- const size_t num_channels = header_.frame_size / sizeof(int16_t);
- return cvd::CreateMessage(
- static_cast<std::uint16_t>(header_.frame_size / num_channels),
- static_cast<std::uint16_t>(num_channels),
- static_cast<std::uint16_t>(header_.frame_rate));
+ const size_t num_channels = header_.frame_size / sizeof(opus_int16);
+ return cvd::CreateMessage(static_cast<std::uint16_t>(num_channels),
+ static_cast<std::uint16_t>(header_.frame_rate));
+ }
+
+ std::uint32_t frame_rate() const {
+ std::unique_lock guard(buffer_lock_);
+ while (!audio_buffer_) {
+ buffer_cv_.wait(guard);
+ }
+ return header_.frame_rate;
+ }
+
+ std::uint32_t num_channels() const {
+ std::unique_lock guard(buffer_lock_);
+ while (!audio_buffer_) {
+ buffer_cv_.wait(guard);
+ }
+ return header_.frame_size / sizeof(opus_int16);
}
// Returns the frame id and audio frame
@@ -188,18 +202,35 @@
void HandleClient(AudioStreamer* audio_streamer,
cvd::ClientSocket client_socket) {
+ auto num_channels = audio_streamer->num_channels();
+ opus::Encoder enc(audio_streamer->frame_rate(),
+ audio_streamer->num_channels(), OPUS_APPLICATION_AUDIO);
+ CHECK(enc.valid()) << "Could not construct Encoder. Maybe bad frame_rate ("
+ << audio_streamer->frame_rate() <<") or num_channels ("
+ << audio_streamer->num_channels() << ")?";
+
auto header = audio_streamer->MakeAudioDescriptionHeader();
client_socket.SendNoSignal(header);
std::int64_t previous_frame_num = 0;
while (!client_socket.closed()) {
+ CHECK(enc.valid()) << "encoder in invalid state";
auto [frame_num, audio_data] =
audio_streamer->audio_buffer(previous_frame_num);
- auto length_message =
- cvd::CreateMessage(static_cast<std::uint32_t>(audio_data->size()));
- client_socket.SendNoSignal(length_message);
- client_socket.SendNoSignal(*audio_data);
previous_frame_num = frame_num;
+
+ std::vector<opus_int16> pcm(audio_data->size() / sizeof(opus_int16));
+ std::memcpy(pcm.data(), audio_data->data(), audio_data->size());
+ // in opus terms "frame_size" is the number of unencoded samples per frame
+ const std::uint32_t frame_size = pcm.size() / num_channels;
+ auto encoded = enc.Encode(pcm, frame_size);
+ for (auto&& p : encoded) {
+ auto length_message =
+ cvd::CreateMessage(static_cast<std::uint32_t>(p.size()));
+ client_socket.SendNoSignal(length_message);
+ client_socket.SendNoSignal(cvd::CreateMessage(frame_size));
+ client_socket.SendNoSignal(p);
+ }
}
}
diff --git a/host/frontend/stream_audio/opuscpp/Android.bp b/host/frontend/stream_audio/opuscpp/Android.bp
new file mode 100644
index 0000000..e0b97ac
--- /dev/null
+++ b/host/frontend/stream_audio/opuscpp/Android.bp
@@ -0,0 +1,27 @@
+//
+// Copyright (C) 2019 The Android Open Source Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+cc_library_host_static {
+ name: "libopuscpp",
+ srcs: [
+ "opus_wrapper.cc",
+ ],
+ shared_libs: [
+ "libbase",
+ "libopus",
+ ],
+ cpp_std: "c++17",
+ defaults: ["cuttlefish_host_only"],
+}
diff --git a/host/frontend/stream_audio/opuscpp/opus_wrapper.cc b/host/frontend/stream_audio/opuscpp/opus_wrapper.cc
new file mode 100644
index 0000000..538bee8
--- /dev/null
+++ b/host/frontend/stream_audio/opuscpp/opus_wrapper.cc
@@ -0,0 +1,177 @@
+// Copyright 2018 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// https://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// https://github.com/google/opuscpp
+
+#include <iterator>
+#include <memory>
+#include <string>
+#include <vector>
+
+#include <android-base/logging.h>
+#include "host/frontend/stream_audio/opuscpp/opus_wrapper.h"
+
+std::string opus::ErrorToString(int error) {
+ switch (error) {
+ case OPUS_OK:
+ return "OK";
+ case OPUS_BAD_ARG:
+ return "One or more invalid/out of range arguments.";
+ case OPUS_BUFFER_TOO_SMALL:
+ return "The mode struct passed is invalid.";
+ case OPUS_INTERNAL_ERROR:
+ return "An internal error was detected.";
+ case OPUS_INVALID_PACKET:
+ return "The compressed data passed is corrupted.";
+ case OPUS_UNIMPLEMENTED:
+ return "Invalid/unsupported request number.";
+ case OPUS_INVALID_STATE:
+ return "An encoder or decoder structure is invalid or already freed.";
+ default:
+ return "Unknown error code: " + std::to_string(error);
+ }
+}
+
+void opus::internal::OpusDestroyer::operator()(OpusEncoder* encoder) const
+ noexcept {
+ opus_encoder_destroy(encoder);
+}
+
+void opus::internal::OpusDestroyer::operator()(OpusDecoder* decoder) const
+ noexcept {
+ opus_decoder_destroy(decoder);
+}
+
+opus::Encoder::Encoder(opus_int32 sample_rate, int num_channels,
+ int application, int expected_loss_percent)
+ : num_channels_{num_channels} {
+ int error{};
+ encoder_.reset(
+ opus_encoder_create(sample_rate, num_channels, application, &error));
+ valid_ = error == OPUS_OK;
+ if (!valid()) {
+ LOG(INFO) << "Could not construct encoder. Error: " << ErrorToString(error);
+ return;
+ }
+ if (expected_loss_percent > 0) {
+ LOG(INFO) << "Enabling FEC in the encoder.";
+ Ctl(OPUS_SET_INBAND_FEC(1));
+ Ctl(OPUS_SET_PACKET_LOSS_PERC(expected_loss_percent));
+ }
+}
+
+bool opus::Encoder::ResetState() {
+ valid_ = Ctl(OPUS_RESET_STATE) == OPUS_OK;
+ return valid_;
+}
+
+bool opus::Encoder::SetBitrate(int bitrate) {
+ valid_ = Ctl(OPUS_SET_BITRATE(bitrate)) == OPUS_OK;
+ return valid_;
+}
+
+bool opus::Encoder::SetVariableBitrate(int vbr) {
+ valid_ = Ctl(OPUS_SET_VBR(vbr)) == OPUS_OK;
+ return valid_;
+}
+
+bool opus::Encoder::SetComplexity(int complexity) {
+ valid_ = Ctl(OPUS_SET_COMPLEXITY(complexity)) == OPUS_OK;
+ return valid_;
+}
+
+int opus::Encoder::GetLookahead() {
+ opus_int32 skip{};
+ valid_ = Ctl(OPUS_GET_LOOKAHEAD(&skip)) == OPUS_OK;
+ return skip;
+}
+
+std::vector<std::vector<unsigned char>> opus::Encoder::Encode(
+ const std::vector<opus_int16>& pcm, int frame_size) {
+ constexpr auto sample_size = sizeof(pcm[0]);
+ const auto frame_length = frame_size * num_channels_ * sample_size;
+ auto data_length = pcm.size() * sample_size;
+ if (data_length % frame_length != 0u) {
+ LOG(WARNING) << "PCM samples contain an incomplete frame. Ignoring the "
+ "incomplete frame.";
+ data_length -= (data_length % frame_length);
+ }
+
+ std::vector<std::vector<unsigned char>> encoded;
+ for (std::size_t i{}; i < data_length; i += frame_length) {
+ encoded.push_back(EncodeFrame(pcm.begin() + (i / sample_size), frame_size));
+ }
+ return encoded;
+}
+
+std::vector<unsigned char> opus::Encoder::EncodeFrame(
+ const std::vector<opus_int16>::const_iterator& frame_start,
+ int frame_size) {
+ const auto frame_length = (frame_size * num_channels_ * sizeof(*frame_start));
+ std::vector<unsigned char> encoded(frame_length);
+ auto num_bytes = opus_encode(encoder_.get(), &*frame_start, frame_size,
+ encoded.data(), encoded.size());
+ if (num_bytes < 0) {
+ LOG(ERROR) << "Encode error: " << opus::ErrorToString(num_bytes);
+ return {};
+ }
+ encoded.resize(num_bytes);
+ return encoded;
+}
+
+opus::Decoder::Decoder(opus_uint32 sample_rate, int num_channels)
+ : num_channels_(num_channels) {
+ int error{};
+ decoder_.reset(opus_decoder_create(sample_rate, num_channels, &error));
+ valid_ = error == OPUS_OK;
+}
+
+std::vector<opus_int16> opus::Decoder::Decode(
+ const std::vector<std::vector<unsigned char>>& packets, int frame_size,
+ bool decode_fec) {
+ std::vector<opus_int16> decoded;
+ for (const auto& enc : packets) {
+ auto just_decoded = Decode(enc, frame_size, decode_fec);
+ decoded.insert(std::end(decoded), std::begin(just_decoded),
+ std::end(just_decoded));
+ }
+ return decoded;
+}
+
+std::vector<opus_int16> opus::Decoder::Decode(
+ const std::vector<unsigned char>& packet, int frame_size, bool decode_fec) {
+ const auto frame_length = (frame_size * num_channels_ * sizeof(opus_int16));
+ std::vector<opus_int16> decoded(frame_length);
+ auto num_samples = opus_decode(decoder_.get(), packet.data(), packet.size(),
+ decoded.data(), frame_size, decode_fec);
+ if (num_samples < 0) {
+ LOG(ERROR) << "Decode error: " << opus::ErrorToString(num_samples);
+ return {};
+ }
+ decoded.resize(num_samples * num_channels_);
+ return decoded;
+}
+
+std::vector<opus_int16> opus::Decoder::DecodeDummy(int frame_size) {
+ const auto frame_length = (frame_size * num_channels_ * sizeof(opus_int16));
+ std::vector<opus_int16> decoded(frame_length);
+ auto num_samples =
+ opus_decode(decoder_.get(), nullptr, 0, decoded.data(), frame_size, true);
+ if (num_samples < 0) {
+ LOG(ERROR) << "Decode error: " << opus::ErrorToString(num_samples);
+ return {};
+ }
+ decoded.resize(num_samples * num_channels_);
+ return decoded;
+}
diff --git a/host/frontend/stream_audio/opuscpp/opus_wrapper.h b/host/frontend/stream_audio/opuscpp/opus_wrapper.h
new file mode 100644
index 0000000..07e932e
--- /dev/null
+++ b/host/frontend/stream_audio/opuscpp/opus_wrapper.h
@@ -0,0 +1,133 @@
+// Copyright 2018 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// https://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// https://github.com/google/opuscpp
+
+#ifndef OPUSCPP_OPUS_WRAPPER_H_
+#define OPUSCPP_OPUS_WRAPPER_H_
+
+#include <memory>
+#include <string>
+#include <vector>
+
+#include "opus.h"
+
+namespace opus {
+
+std::string ErrorToString(int error);
+
+namespace internal {
+// Deleter for OpusEncoders and OpusDecoders
+struct OpusDestroyer {
+ void operator()(OpusEncoder* encoder) const noexcept;
+ void operator()(OpusDecoder* decoder) const noexcept;
+};
+template <typename T>
+using opus_uptr = std::unique_ptr<T, OpusDestroyer>;
+} // namespace internal
+
+class Encoder {
+ public:
+ // see documentation at:
+ // https://mf4.xiph.org/jenkins/view/opus/job/opus/ws/doc/html/group__opus__encoder.html#gaa89264fd93c9da70362a0c9b96b9ca88
+ // Fs corresponds to sample_rate
+ //
+ // If expected_loss_percent is positive, FEC will be enabled
+ Encoder(opus_int32 sample_rate, int num_channels, int application,
+ int expected_loss_percent = 0);
+
+ // Resets internal state of encoder. This should be called between encoding
+ // different streams so that back-to-back decoding and one-at-a-time decoding
+ // give the same result. Returns true on success.
+ bool ResetState();
+
+ // Sets the desired bitrate. Rates from 500 to 512000 are meaningful as well
+ // as the special values OPUS_AUTO and OPUS_BITRATE_MAX. If this method
+ // is not called, the default value of OPUS_AUTO is used.
+ // Returns true on success.
+ bool SetBitrate(int bitrate);
+
+ // Enables or disables variable bitrate in the encoder. By default, variable
+ // bitrate is enabled. Returns true on success.
+ bool SetVariableBitrate(int vbr);
+
+ // Sets the computational complexity of the encoder, in the range of 0 to 10,
+ // inclusive, with 10 being the highest complexity. Returns true on success.
+ bool SetComplexity(int complexity);
+
+ // Gets the total samples of delay added by the entire codec. This value
+ // is the minimum amount of 'preskip' that has to be specified in an
+ // ogg-stream that encapsulates the encoded audio.
+ int GetLookahead();
+
+ // Takes audio data and encodes it. Returns a sequence of encoded packets.
+ // pcm.size() must be divisible by frame_size * (number of channels);
+ // pcm must not contain any incomplete packets.
+ // see documentation for pcm and frame_size at:
+ // https://mf4.xiph.org/jenkins/view/opus/job/opus/ws/doc/html/group__opus__encoder.html#gad2d6bf6a9ffb6674879d7605ed073e25
+ std::vector<std::vector<unsigned char>> Encode(
+ const std::vector<opus_int16>& pcm, int frame_size);
+
+ int valid() const { return valid_; }
+
+ private:
+ std::vector<unsigned char> EncodeFrame(
+ const std::vector<opus_int16>::const_iterator& frame_start,
+ int frame_size);
+
+ template <typename... Ts>
+ int Ctl(int request, Ts... args) const {
+ return opus_encoder_ctl(encoder_.get(), request, args...);
+ }
+
+ int num_channels_{};
+ bool valid_{};
+ internal::opus_uptr<OpusEncoder> encoder_;
+};
+
+class Decoder {
+ public:
+ // see documentation at:
+ // https://mf4.xiph.org/jenkins/view/opus/job/opus/ws/doc/html/group__opus__decoder.html#ga753f6fe0b699c81cfd47d70c8e15a0bd
+ // Fs corresponds to sample_rate
+ Decoder(opus_uint32 sample_rate, int num_channels);
+
+ // Takes a sequence of encoded packets and decodes them. Returns the decoded
+ // audio.
+ // see documentation at:
+ // https://mf4.xiph.org/jenkins/view/opus/job/opus/ws/doc/html/group__opus__decoder.html#ga7d1111f64c36027ddcb81799df9b3fc9
+ std::vector<opus_int16> Decode(
+ const std::vector<std::vector<unsigned char>>& packets, int frame_size,
+ bool decode_fec);
+
+ int valid() const { return valid_; }
+
+ // Takes an encoded packet and decodes it. Returns the decoded audio
+ // see documentation at:
+ // https://mf4.xiph.org/jenkins/view/opus/job/opus/ws/doc/html/group__opus__decoder.html#ga7d1111f64c36027ddcb81799df9b3fc9
+ std::vector<opus_int16> Decode(const std::vector<unsigned char>& packet,
+ int frame_size, bool decode_fec);
+
+ // Generates a dummy frame by passing nullptr to the underlying opus decode.
+ std::vector<opus_int16> DecodeDummy(int frame_size);
+
+ private:
+ int num_channels_{};
+ bool valid_{};
+ internal::opus_uptr<OpusDecoder> decoder_;
+};
+
+} // namespace opus
+
+#endif
