Send Guest and Host event FDs over to QEmu
Change-Id: I20132c5babaed27a086c8599c97619562b5cac07
(cherry picked from commit 1bf152026ecd1d8565e2ef9b32090f7825f3ea97)
diff --git a/ivserver/hald_client.cc b/ivserver/hald_client.cc
index 2b535b8..d7eb8e2 100644
--- a/ivserver/hald_client.cc
+++ b/ivserver/hald_client.cc
@@ -99,7 +99,7 @@
};
avd::InbandMessageHeader hdr{nullptr, 0, &vec, 1, 0};
avd::SharedFD fds[3] = {guest_to_host_efd, host_to_guest_efd,
- shared_mem.shared_mem_fd()};
+ shared_mem.SharedMemFD()};
rval = client_socket_->SendMsgAndFDs<3>(hdr, MSG_NOSIGNAL, fds);
if (rval == -1) {
LOG(ERROR) << "failed to send Host FD: " << client_socket_->StrError();
diff --git a/ivserver/hald_client_test.cc b/ivserver/hald_client_test.cc
index bde3dc5..02b33cc 100644
--- a/ivserver/hald_client_test.cc
+++ b/ivserver/hald_client_test.cc
@@ -11,11 +11,11 @@
#include "host/ivserver/hald_client.h"
#include "host/ivserver/vsocsharedmem_mock.h"
-using ::testing::_;
using ::testing::DoAll;
using ::testing::Return;
using ::testing::ReturnRef;
using ::testing::SetArgPointee;
+using ::testing::_;
namespace ivserver {
namespace test {
@@ -24,14 +24,16 @@
public:
void SetUp() override {
std::string socket_location;
- ASSERT_TRUE(GetTempLocation(&socket_location)) << "Could not create temp file";
+ ASSERT_TRUE(GetTempLocation(&socket_location))
+ << "Could not create temp file";
LOG(INFO) << "Temp file location: " << socket_location;
hald_server_socket_ = avd::SharedFD::SocketLocalServer(
socket_location.c_str(), false, SOCK_STREAM, 0666);
- test_socket_ =
- avd::SharedFD::SocketLocalClient(socket_location.c_str(), false, SOCK_STREAM);
- hald_socket_ = avd::SharedFD::Accept(*hald_server_socket_, nullptr, nullptr);
+ test_socket_ = avd::SharedFD::SocketLocalClient(socket_location.c_str(),
+ false, SOCK_STREAM);
+ hald_socket_ =
+ avd::SharedFD::Accept(*hald_server_socket_, nullptr, nullptr);
EXPECT_TRUE(hald_server_socket_->IsOpen());
EXPECT_TRUE(test_socket_->IsOpen());
@@ -71,24 +73,25 @@
};
TEST_F(HaldClientTest, HandshakeTerminatedByHald) {
- std::thread thread([this]() {
- auto client(HaldClient::New(vsoc_, hald_socket_));
- EXPECT_FALSE(client)
- << "Handshake should not complete when client terminates early.";
- });
+ std::thread thread(
+ [this]() {
+ auto client(HaldClient::New(vsoc_, hald_socket_));
+ EXPECT_FALSE(client)
+ << "Handshake should not complete when client terminates early.";
+ });
test_socket_->Close();
thread.join();
}
TEST_F(HaldClientTest, HandshakeTerminatedByInvalidRegionSize) {
- uint16_t sizes[] = { 0, VSoCSharedMemory::kMaxRegionNameLength + 1, 0xffff };
+ uint16_t sizes[] = {0, VSoCSharedMemory::kMaxRegionNameLength + 1, 0xffff};
for (uint16_t size : sizes) {
std::thread thread([this, size]() {
- auto client(HaldClient::New(vsoc_,hald_socket_));
- EXPECT_FALSE(client) << "Handle should not be created when size is "
- << size;
+ auto client(HaldClient::New(vsoc_, hald_socket_));
+ EXPECT_FALSE(client) << "Handle should not be created when size is "
+ << size;
});
int32_t proto_version;
@@ -104,28 +107,28 @@
TEST_F(HaldClientTest, FullSaneHandshake) {
std::string temp;
ASSERT_TRUE(GetTempLocation(&temp));
- avd::SharedFD host_fd(avd::SharedFD::Open(temp.c_str(), O_CREAT | O_RDWR, 0666));
+ avd::SharedFD host_fd(
+ avd::SharedFD::Open(temp.c_str(), O_CREAT | O_RDWR, 0666));
EXPECT_TRUE(host_fd->IsOpen());
ASSERT_TRUE(GetTempLocation(&temp));
- avd::SharedFD guest_fd(avd::SharedFD::Open(temp.c_str(), O_CREAT | O_RDWR, 0666));
+ avd::SharedFD guest_fd(
+ avd::SharedFD::Open(temp.c_str(), O_CREAT | O_RDWR, 0666));
EXPECT_TRUE(guest_fd->IsOpen());
ASSERT_TRUE(GetTempLocation(&temp));
- avd::SharedFD shmem_fd(avd::SharedFD::Open(temp.c_str(), O_CREAT | O_RDWR, 0666));
+ avd::SharedFD shmem_fd(
+ avd::SharedFD::Open(temp.c_str(), O_CREAT | O_RDWR, 0666));
EXPECT_TRUE(shmem_fd->IsOpen());
const std::string test_location("testing");
EXPECT_CALL(vsoc_, GetEventFdPairForRegion(test_location, _, _))
- .WillOnce(DoAll(
- SetArgPointee<1>(host_fd),
- SetArgPointee<2>(guest_fd),
- Return(true)));
- EXPECT_CALL(vsoc_, shared_mem_fd())
- .WillOnce(ReturnRef(shmem_fd));
+ .WillOnce(DoAll(SetArgPointee<1>(host_fd), SetArgPointee<2>(guest_fd),
+ Return(true)));
+ EXPECT_CALL(vsoc_, SharedMemFD()).WillOnce(ReturnRef(shmem_fd));
std::thread thread([this]() {
- auto client(HaldClient::New(vsoc_,hald_socket_));
+ auto client(HaldClient::New(vsoc_, hald_socket_));
EXPECT_TRUE(client);
});
@@ -135,7 +138,8 @@
test_socket_->Recv(&proto_version, sizeof(proto_version), MSG_NOSIGNAL));
uint16_t size = test_location.size();
- EXPECT_EQ(sizeof(size), test_socket_->Send(&size, sizeof(size), MSG_NOSIGNAL));
+ EXPECT_EQ(sizeof(size),
+ test_socket_->Send(&size, sizeof(size), MSG_NOSIGNAL));
EXPECT_EQ(size, test_socket_->Send(test_location.data(), size, MSG_NOSIGNAL));
// TODO(ender): delete this once no longer necessary. Currently, absence of
@@ -155,6 +159,5 @@
thread.join();
}
-
} // namespace test
} // namespace ivserver
diff --git a/ivserver/ivserver.cc b/ivserver/ivserver.cc
index 92cd7cc..a3e3a30 100644
--- a/ivserver/ivserver.cc
+++ b/ivserver/ivserver.cc
@@ -53,15 +53,10 @@
}
void IVServer::HandleNewQemuConnection() {
- std::unique_ptr<QemuClient> res =
- QemuClient::New(vsoc_shmem_->shared_mem_fd(),
- avd::SharedFD::Accept(*qemu_channel_, nullptr, nullptr));
+ std::unique_ptr<QemuClient> res = QemuClient::New(
+ *vsoc_shmem_, avd::SharedFD::Accept(*qemu_channel_, nullptr, nullptr));
- if (res) {
- // TODO(romitd): how to recover if some clients failed? should we retry?
- // Why are we doing this?
- vsoc_shmem_->BroadcastQemuSocket(res->client_socket());
- } else {
+ if (!res) {
LOG(WARNING) << "Could not accept new QEmu client.";
}
}
diff --git a/ivserver/qemu_client.cc b/ivserver/qemu_client.cc
index a18f8af..83164ec 100644
--- a/ivserver/qemu_client.cc
+++ b/ivserver/qemu_client.cc
@@ -3,14 +3,9 @@
#include <glog/logging.h>
namespace ivserver {
-namespace {
-// QEMU expects version 0 of the QEMU <--> ivserver protocol.
-const uint64_t kQemuIvshMemProtocolVersion = 0;
-const uint64_t kQemuVMId = 1;
-} // anonymous namespace
-std::unique_ptr<QemuClient> QemuClient::New(const avd::SharedFD& socket,
- avd::SharedFD shmemfd) {
+std::unique_ptr<QemuClient> QemuClient::New(const VSoCSharedMemory& shmem,
+ const avd::SharedFD& socket) {
std::unique_ptr<QemuClient> res;
if (!socket->IsOpen()) {
LOG(WARNING) << "Invalid socket passed to QemuClient: "
@@ -19,8 +14,8 @@
}
res.reset(new QemuClient(std::move(socket)));
- if (!res->PerformHandshake(shmemfd)) {
- LOG(ERROR) << "QEmu handshake failed. Dropping connection.";
+ if (!res->PerformHandshake(shmem)) {
+ LOG(ERROR) << "Qemu handshake failed. Dropping connection.";
res.reset();
}
@@ -31,37 +26,82 @@
// Once the QemuClient object is constructed, invoking the following
// method will perform the actual handshake with a QEMU instance.
-bool QemuClient::PerformHandshake(const avd::SharedFD& shmem_fd) {
- int rval =
- client_socket_->Send(&kQemuIvshMemProtocolVersion,
- sizeof(kQemuIvshMemProtocolVersion), MSG_NOSIGNAL);
- if (rval != sizeof(kQemuIvshMemProtocolVersion)) {
+bool QemuClient::PerformHandshake(const VSoCSharedMemory& shmem) {
+ // 1. The protocol version number, currently zero. The client should
+ // close the connection on receipt of versions it can't handle.
+ int64_t msg = QemuConstants::kIvshMemProtocolVersion;
+ int rval = client_socket_->Send(&msg, sizeof(msg), MSG_NOSIGNAL);
+ if (rval != sizeof(msg)) {
LOG(ERROR) << "Failed to send protocol version: "
<< client_socket_->StrError();
return false;
}
- rval = client_socket_->Send(&kQemuVMId, sizeof(kQemuVMId), MSG_NOSIGNAL);
- if (rval != sizeof(kQemuVMId)) {
+ // 2. The client's ID. This is unique among all clients of this server.
+ // IDs must be between 0 and 65535, because the Doorbell register
+ // provides only 16 bits for them.
+ msg = QemuConstants::kGuestID;
+ rval = client_socket_->Send(&msg, sizeof(msg), MSG_NOSIGNAL);
+ if (rval != sizeof(msg)) {
LOG(ERROR) << "Failed to send VM Id: " << client_socket_->StrError();
return false;
}
- // Send FD to remote process over unix domain socket using control message.
- uint64_t control_data = ~0;
- struct iovec vec {
- &control_data, sizeof(control_data)
- };
- avd::InbandMessageHeader hdr{nullptr, 0, &vec, 1, 0};
- avd::SharedFD fds[] = {shmem_fd};
- rval = client_socket_->SendMsgAndFDs(hdr, 0, fds);
- if (rval == -1) {
- LOG(ERROR) << "failed to send shared_mem_fd: "
+ // 3. The number -1, accompanied by the file descriptor for the shared
+ // memory.
+ if (!SendSocketInfo(QemuFDMsg::kSharedMem, shmem.SharedMemFD())) {
+ LOG(ERROR) << "Failed to send Shared Memory socket: "
<< client_socket_->StrError();
return false;
}
+ // 4. Connect notifications for existing other clients, if any. This is
+ // a peer ID (number between 0 and 65535 other than the client's ID),
+ // repeated N times. Each repetition is accompanied by one file
+ // descriptor. These are for interrupting the peer with that ID using
+ // vector 0,..,N-1, in order. If the client is configured for fewer
+ // vectors, it closes the extra file descriptors. If it is configured
+ // for more, the extra vectors remain unconnected.
+ for (const auto region_pair : shmem.Regions()) {
+ if (!SendSocketInfo(QemuFDMsg::kHostSideHald, region_pair.second.host_fd)) {
+ LOG(ERROR) << "Failed to send Host Side FD for region "
+ << region_pair.first << ": " << client_socket_->StrError();
+ return false;
+ }
+ }
+
+ // 5. Interrupt setup. This is the client's own ID, repeated N times.
+ // Each repetition is accompanied by one file descriptor. These are
+ // for receiving interrupts from peers using vector 0,..,N-1, in
+ // order. If the client is configured for fewer vectors, it closes
+ // the extra file descriptors. If it is configured for more, the
+ // extra vectors remain unconnected.
+ for (const auto region_pair : shmem.Regions()) {
+ if (!SendSocketInfo(QemuFDMsg::kGuestSideHal,
+ region_pair.second.guest_fd)) {
+ LOG(ERROR) << "Failed to send Guest Side FD for region "
+ << region_pair.first << ": " << client_socket_->StrError();
+ return false;
+ }
+ }
+
return true;
}
-} // namespace ivserver
+bool QemuClient::SendSocketInfo(QemuFDMsg message,
+ const avd::SharedFD& socket) {
+ struct iovec vec {
+ &message, sizeof(message)
+ };
+ avd::InbandMessageHeader hdr{nullptr, 0, &vec, 1, 0};
+ avd::SharedFD fds[] = {socket};
+ int rval = client_socket_->SendMsgAndFDs(hdr, 0, fds);
+ if (rval == -1) {
+ LOG(ERROR) << "failed to send shared_mem_fd: "
+ << client_socket_->StrError();
+ return false;
+ }
+ return true;
+}
+
+} // namespace ivserver
\ No newline at end of file
diff --git a/ivserver/qemu_client.h b/ivserver/qemu_client.h
index f8c4c59..d0f80c1 100644
--- a/ivserver/qemu_client.h
+++ b/ivserver/qemu_client.h
@@ -14,12 +14,37 @@
// https://github.com/qemu/qemu/blob/stable-2.8/docs/specs/ivshmem-spec.txt
class QemuClient final {
public:
- static std::unique_ptr<QemuClient> New(const avd::SharedFD &connection,
- avd::SharedFD shmemfd);
+ static std::unique_ptr<QemuClient> New(const VSoCSharedMemory &shmem,
+ const avd::SharedFD &connection);
avd::SharedFD client_socket() const { return client_socket_; }
private:
+ enum QemuConstants : int64_t {
+ kIvshMemProtocolVersion = 0,
+ // HostID is in fact a Peer ID and can take multiple values, depending on
+ // how many subsystems we would like Guest to talk to.
+ kHostBaseID = 0,
+ // GuestID is a unique form of Peer ID (see above), that identifies newly
+ // created quest in IvSharedMem world.
+ kGuestID = 1024
+ };
+
+ static_assert(QemuConstants::kHostBaseID < QemuConstants::kGuestID,
+ "Guest and host should have different IDs");
+ // Type of QEmu FD messages.
+ // QEmu uses these messages to identify purpose of socket it is
+ // receiving.
+ enum class QemuFDMsg : int64_t {
+ // Represents SharedMemory FD.
+ kSharedMem = -1,
+ // Represents primary (and currently only) FD that is owned and managed by
+ // Host side.
+ kHostSideHald = QemuConstants::kHostBaseID,
+ // Represents FDs that are owned by Guest.
+ kGuestSideHal = QemuConstants::kGuestID,
+ };
+
avd::SharedFD client_socket_;
// Initialize new instance of QemuClient.
@@ -27,7 +52,10 @@
// Once the QemuClient object is constructed, invoking the following
// method will perform the actual handshake with a QEMU instance.
- bool PerformHandshake(const avd::SharedFD &shmem_fd);
+ bool PerformHandshake(const VSoCSharedMemory &shmem_fd);
+
+ // Send socket data to Qemu.
+ bool SendSocketInfo(QemuFDMsg message, const avd::SharedFD &socket);
QemuClient(const QemuClient &) = delete;
QemuClient &operator=(const QemuClient &) = delete;
diff --git a/ivserver/vsocsharedmem.cc b/ivserver/vsocsharedmem.cc
index 120c248..dce7f2a 100644
--- a/ivserver/vsocsharedmem.cc
+++ b/ivserver/vsocsharedmem.cc
@@ -30,9 +30,9 @@
avd::SharedFD *guest_to_host,
avd::SharedFD *host_to_guest) const override;
- const avd::SharedFD &shared_mem_fd() const override;
+ const avd::SharedFD &SharedMemFD() const override;
- void BroadcastQemuSocket(const avd::SharedFD &qemu_socket) const override;
+ const std::map<std::string, Region> &Regions() const override;
private:
void CreateLayout();
@@ -40,10 +40,10 @@
const uint32_t size_;
const Json::Value &json_root_;
avd::SharedFD shared_mem_fd_;
- std::map<std::string, std::pair<avd::SharedFD, avd::SharedFD>> eventfd_data_;
+ std::map<std::string, Region> eventfd_data_;
VSoCSharedMemoryImpl(const VSoCSharedMemoryImpl &) = delete;
- VSoCSharedMemoryImpl& operator=(const VSoCSharedMemoryImpl& other) = delete;
+ VSoCSharedMemoryImpl &operator=(const VSoCSharedMemoryImpl &other) = delete;
};
VSoCSharedMemoryImpl::VSoCSharedMemoryImpl(const uint32_t size_mib,
@@ -63,10 +63,15 @@
CreateLayout();
}
-const avd::SharedFD &VSoCSharedMemoryImpl::shared_mem_fd() const {
+const avd::SharedFD &VSoCSharedMemoryImpl::SharedMemFD() const {
return shared_mem_fd_;
}
+const std::map<std::string, VSoCSharedMemory::Region>
+ &VSoCSharedMemoryImpl::Regions() const {
+ return eventfd_data_;
+}
+
void VSoCSharedMemoryImpl::CreateLayout() {
uint32_t offset = 0;
void *mmap_addr =
@@ -147,19 +152,17 @@
// Create one pair of eventfds for this region. Note that the guest to host
// eventfd is non-blocking, whereas the host to guest eventfd is blocking.
// This is in anticipation of blocking semantics for the host side locks.
- avd::SharedFD host_efd(avd::SharedFD::Event());
+ avd::SharedFD host_efd(avd::SharedFD::Event(0, EFD_NONBLOCK));
LOG_IF(FATAL, !host_efd->IsOpen())
<< "Failed to create host eventfd for " << device_name << ": "
<< host_efd->StrError();
- avd::SharedFD guest_efd(avd::SharedFD::Event());
+ avd::SharedFD guest_efd(avd::SharedFD::Event(0, EFD_NONBLOCK));
LOG_IF(FATAL, !guest_efd->IsOpen())
<< "Failed to create guest eventfd for " << device_name << ": "
<< guest_efd->StrError();
- eventfd_data_.emplace(
- region["device_name"].asString(),
- std::pair<avd::SharedFD, avd::SharedFD>{host_efd, guest_efd});
+ eventfd_data_.emplace(device_name, Region{host_efd, guest_efd});
}
munmap(mmap_addr, size_);
@@ -171,41 +174,17 @@
auto it = eventfd_data_.find(region_name);
if (it == eventfd_data_.end()) return false;
- *guest_to_host = it->second.first;
- *host_to_guest = it->second.second;
+ *guest_to_host = it->second.host_fd;
+ *host_to_guest = it->second.guest_fd;
return true;
}
-void VSoCSharedMemoryImpl::BroadcastQemuSocket(const avd::SharedFD &qemu_fd) const {
- uint64_t control_data = 0;
- struct iovec vec {
- &control_data, sizeof(control_data)
- };
- avd::InbandMessageHeader hdr{nullptr, 0, &vec, sizeof(vec), 0};
-
- avd::SharedFD fds[] = {qemu_fd};
- // TODO(ghartman, romitd): how should we recover from these?
- for (const auto it : eventfd_data_) {
- int result;
- result = it.second.first->SendMsgAndFDs(hdr, 0, fds);
- if (result == -1) {
- LOG(ERROR) << "failed to send QEmu FD to " << it.first
- << " Host: " << it.second.first->StrError();
- }
-
- result = it.second.second->SendMsgAndFDs(hdr, 0, fds);
- if (result == -1) {
- LOG(ERROR) << "failed to send QEmu FD to " << it.first
- << " Guest: " << it.second.second->StrError();
- }
- }
-}
-
} // anonymous namespace
std::unique_ptr<VSoCSharedMemory> VSoCSharedMemory::New(
- const uint32_t size_mb, const std::string& name, const Json::Value& root) {
- return std::unique_ptr<VSoCSharedMemory>(new VSoCSharedMemoryImpl(size_mb, name, root));
+ const uint32_t size_mb, const std::string &name, const Json::Value &root) {
+ return std::unique_ptr<VSoCSharedMemory>(
+ new VSoCSharedMemoryImpl(size_mb, name, root));
}
} // namespace ivserver
diff --git a/ivserver/vsocsharedmem.h b/ivserver/vsocsharedmem.h
index 8f6f051..0669ba3 100644
--- a/ivserver/vsocsharedmem.h
+++ b/ivserver/vsocsharedmem.h
@@ -2,20 +2,25 @@
#include <inttypes.h>
#include <json/json.h>
-#include <unistd.h>
#include <map>
#include <memory>
#include <string>
#include "common/libs/fs/shared_fd.h"
+#include "uapi/vsoc_shm.h"
namespace ivserver {
class VSoCSharedMemory {
public:
+ // Region describes a VSoCSharedMem region.
+ struct Region {
+ avd::SharedFD host_fd;
+ avd::SharedFD guest_fd;
+ };
+
// Max name length of a memory region.
- // TODO(ender): set this value from trusted source.
- static constexpr int32_t kMaxRegionNameLength = 16;
+ static constexpr int32_t kMaxRegionNameLength = sizeof(vsoc_device_name);
VSoCSharedMemory() = default;
virtual ~VSoCSharedMemory() = default;
@@ -25,16 +30,15 @@
const Json::Value &json_root);
virtual bool GetEventFdPairForRegion(const std::string ®ion_name,
- avd::SharedFD *guest_to_host,
- avd::SharedFD *host_to_guest) const = 0;
+ avd::SharedFD *guest_to_host,
+ avd::SharedFD *host_to_guest) const = 0;
- virtual const avd::SharedFD &shared_mem_fd() const = 0;
-
- virtual void BroadcastQemuSocket(const avd::SharedFD &qemu_socket) const = 0;
+ virtual const avd::SharedFD &SharedMemFD() const = 0;
+ virtual const std::map<std::string, Region> &Regions() const = 0;
private:
VSoCSharedMemory(const VSoCSharedMemory &) = delete;
- VSoCSharedMemory& operator=(const VSoCSharedMemory& other) = delete;
+ VSoCSharedMemory &operator=(const VSoCSharedMemory &other) = delete;
};
} // namespace ivserver
diff --git a/ivserver/vsocsharedmem_mock.h b/ivserver/vsocsharedmem_mock.h
index 58ec680..8f96250 100644
--- a/ivserver/vsocsharedmem_mock.h
+++ b/ivserver/vsocsharedmem_mock.h
@@ -8,9 +8,11 @@
namespace test {
class VSoCSharedMemoryMock : public VSoCSharedMemory {
public:
- MOCK_CONST_METHOD3(GetEventFdPairForRegion, bool(const std::string&, avd::SharedFD*, avd::SharedFD*));
- MOCK_CONST_METHOD0(shared_mem_fd, const avd::SharedFD&());
- MOCK_CONST_METHOD1(BroadcastQemuSocket, void(const avd::SharedFD&));
+ MOCK_CONST_METHOD3(GetEventFdPairForRegion,
+ bool(const std::string&, avd::SharedFD*, avd::SharedFD*));
+ MOCK_CONST_METHOD0(SharedMemFD, const avd::SharedFD&());
+ MOCK_CONST_METHOD0(Regions,
+ const std::map<std::string, VSoCSharedMemory::Region>&());
};
-}
-}
\ No newline at end of file
+} // namespace test
+} // namespace ivserver
\ No newline at end of file
