host/libs/vm_manager/crosvm_manager.cpp

/*
 * Copyright (C) 2019 The Android Open Source Project
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *      http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

#include "host/libs/vm_manager/crosvm_manager.h"

#include <android-base/file.h>
#include <android-base/logging.h>
#include <android-base/strings.h>
#include <sys/stat.h>
#include <sys/types.h>
#include <vulkan/vulkan.h>

#include <cassert>
#include <string>
#include <vector>

#include "common/libs/utils/environment.h"
#include "common/libs/utils/files.h"
#include "common/libs/utils/network.h"
#include "common/libs/utils/subprocess.h"
#include "host/libs/config/cuttlefish_config.h"
#include "host/libs/config/known_paths.h"
#include "host/libs/vm_manager/crosvm_builder.h"
#include "host/libs/vm_manager/qemu_manager.h"

namespace cuttlefish {
namespace vm_manager {

namespace {

std::string GetControlSocketPath(
    const CuttlefishConfig::InstanceSpecific& instance,
    const std::string& socket_name) {
  return instance.PerInstanceInternalPath(socket_name.c_str());
}

}  // namespace

bool CrosvmManager::IsSupported() {
#ifdef __ANDROID__
  return true;
#else
  return HostSupportsQemuCli();
#endif
}

std::vector<std::string> CrosvmManager::ConfigureGpuMode(
    const std::string& gpu_mode) {
  // Override the default HAL search paths in all cases. We do this because
  // the HAL search path allows for fallbacks, and fallbacks in conjunction
  // with properities lead to non-deterministic behavior while loading the
  // HALs.
  if (gpu_mode == kGpuModeGuestSwiftshader) {
    return {
        "androidboot.cpuvulkan.version=" + std::to_string(VK_API_VERSION_1_2),
        "androidboot.hardware.gralloc=minigbm",
        "androidboot.hardware.hwcomposer=ranchu",
        "androidboot.hardware.egl=angle",
        "androidboot.hardware.vulkan=pastel",
    };
  }

  if (gpu_mode == kGpuModeDrmVirgl) {
    return {
      "androidboot.cpuvulkan.version=0",
      "androidboot.hardware.gralloc=minigbm",
      "androidboot.hardware.hwcomposer=drm_minigbm",
      "androidboot.hardware.egl=mesa",
    };
  }
  if (gpu_mode == kGpuModeGfxStream) {
    return {
        "androidboot.cpuvulkan.version=0",
        "androidboot.hardware.gralloc=minigbm",
        "androidboot.hardware.hwcomposer=ranchu",
        "androidboot.hardware.egl=emulation",
        "androidboot.hardware.vulkan=ranchu",
        "androidboot.hardware.gltransport=virtio-gpu-asg",
    };
  }
  return {};
}

std::string CrosvmManager::ConfigureBootDevices(int num_disks) {
  // TODO There is no way to control this assignment with crosvm (yet)
  if (HostArch() == Arch::X86_64) {
    // crosvm has an additional PCI device for an ISA bridge
    return ConfigureMultipleBootDevices("pci0000:00/0000:00:", 1, num_disks);
  } else {
    // On ARM64 crosvm, block devices are on their own bridge, so we don't
    // need to calculate it, and the path is always the same
    return "androidboot.boot_devices=10000.pci";
  }
}

constexpr auto crosvm_socket = "crosvm_control.sock";

std::vector<Command> CrosvmManager::StartCommands(
    const CuttlefishConfig& config) {
  auto instance = config.ForDefaultInstance();
  CrosvmBuilder crosvm_cmd;
  crosvm_cmd.SetBinary(config.crosvm_binary());
  crosvm_cmd.AddControlSocket(GetControlSocketPath(instance, crosvm_socket));

  if (!config.smt()) {
    crosvm_cmd.Cmd().AddParameter("--no-smt");
  }

  if (config.vhost_net()) {
    crosvm_cmd.Cmd().AddParameter("--vhost-net");
  }

#ifdef ENFORCE_MAC80211_HWSIM
  if (!config.vhost_user_mac80211_hwsim().empty()) {
    crosvm_cmd.Cmd().AddParameter("--vhost-user-mac80211-hwsim=",
                                  config.vhost_user_mac80211_hwsim());
  }
#endif

  if (config.protected_vm()) {
    crosvm_cmd.Cmd().AddParameter("--protected-vm");
  }

  if (config.gdb_port() > 0) {
    CHECK(config.cpus() == 1) << "CPUs must be 1 for crosvm gdb mode";
    crosvm_cmd.Cmd().AddParameter("--gdb=", config.gdb_port());
  }

  auto gpu_capture_enabled = !config.gpu_capture_binary().empty();
  auto gpu_mode = config.gpu_mode();
  if (gpu_mode == kGpuModeGuestSwiftshader) {
    crosvm_cmd.Cmd().AddParameter("--gpu=2D");
  } else if (gpu_mode == kGpuModeDrmVirgl || gpu_mode == kGpuModeGfxStream) {
    crosvm_cmd.Cmd().AddParameter(
        gpu_mode == kGpuModeGfxStream ? "--gpu=gfxstream," : "--gpu=",
        "egl=true,surfaceless=true,glx=false,gles=true");
  }

  for (const auto& display_config : config.display_configs()) {
    crosvm_cmd.Cmd().AddParameter(
        "--gpu-display=", "width=", display_config.width, ",",
        "height=", display_config.height);
  }

  crosvm_cmd.Cmd().AddParameter("--wayland-sock=",
                                instance.frames_socket_path());

  // crosvm_cmd.Cmd().AddParameter("--null-audio");
  crosvm_cmd.Cmd().AddParameter("--mem=", config.memory_mb());
  crosvm_cmd.Cmd().AddParameter("--cpus=", config.cpus());

  auto disk_num = instance.virtual_disk_paths().size();
  CHECK_GE(VmManager::kMaxDisks, disk_num)
      << "Provided too many disks (" << disk_num << "), maximum "
      << VmManager::kMaxDisks << "supported";
  for (const auto& disk : instance.virtual_disk_paths()) {
    crosvm_cmd.Cmd().AddParameter(
        config.protected_vm() ? "--disk=" : "--rwdisk=", disk);
  }

  if (config.enable_vnc_server() || config.enable_webrtc()) {
    auto touch_type_parameter =
        config.enable_webrtc() ? "--multi-touch=" : "--single-touch=";

    auto display_configs = config.display_configs();
    CHECK_GE(display_configs.size(), 1);

    for (int i = 0; i < display_configs.size(); ++i) {
      auto display_config = display_configs[i];

      crosvm_cmd.Cmd().AddParameter(
          touch_type_parameter, instance.touch_socket_path(i), ":",
          display_config.width, ":", display_config.height);
    }
    crosvm_cmd.Cmd().AddParameter("--keyboard=",
                                  instance.keyboard_socket_path());
  }
  if (config.enable_webrtc()) {
    crosvm_cmd.Cmd().AddParameter("--switches=",
                                  instance.switches_socket_path());
  }

  SharedFD wifi_tap;
  // GPU capture can only support named files and not file descriptors due to
  // having to pass arguments to crosvm via a wrapper script.
  if (!gpu_capture_enabled) {
    crosvm_cmd.AddTap(instance.mobile_tap_name());
    crosvm_cmd.AddTap(instance.ethernet_tap_name());

    // TODO(b/199103204): remove this as well when
    // PRODUCT_ENFORCE_MAC80211_HWSIM is removed
#ifndef ENFORCE_MAC80211_HWSIM
    wifi_tap = crosvm_cmd.AddTap(instance.wifi_tap_name());
#endif
  }

  if (FileExists(instance.access_kregistry_path())) {
    crosvm_cmd.Cmd().AddParameter("--rw-pmem-device=",
                                  instance.access_kregistry_path());
  }

  if (FileExists(instance.pstore_path())) {
    crosvm_cmd.Cmd().AddParameter("--pstore=path=", instance.pstore_path(),
                                  ",size=", FileSize(instance.pstore_path()));
  }

  if (config.enable_sandbox()) {
    const bool seccomp_exists = DirectoryExists(config.seccomp_policy_dir());
    const std::string& var_empty_dir = kCrosvmVarEmptyDir;
    const bool var_empty_available = DirectoryExists(var_empty_dir);
    if (!var_empty_available || !seccomp_exists) {
      LOG(FATAL) << var_empty_dir << " is not an existing, empty directory."
                 << "seccomp-policy-dir, " << config.seccomp_policy_dir()
                 << " does not exist " << std::endl;
      return {};
    }
    crosvm_cmd.Cmd().AddParameter("--seccomp-policy-dir=",
                                  config.seccomp_policy_dir());
  } else {
    crosvm_cmd.Cmd().AddParameter("--disable-sandbox");
  }

  if (instance.vsock_guest_cid() >= 2) {
    crosvm_cmd.Cmd().AddParameter("--cid=", instance.vsock_guest_cid());
  }

  // Use a virtio-console instance for the main kernel console. All
  // messages will switch from earlycon to virtio-console after the driver
  // is loaded, and crosvm will append to the kernel log automatically
  crosvm_cmd.AddHvcConsoleReadOnly(instance.kernel_log_pipe_name());

  if (config.console()) {
    // stdin is the only currently supported way to write data to a serial port in
    // crosvm. A file (named pipe) is used here instead of stdout to ensure only
    // the serial port output is received by the console forwarder as crosvm may
    // print other messages to stdout.
    if (config.kgdb() || config.use_bootloader()) {
      crosvm_cmd.AddSerialConsoleReadWrite(instance.console_out_pipe_name(),
                                           instance.console_in_pipe_name());
      // In kgdb mode, we have the interactive console on ttyS0 (both Android's
      // console and kdb), so we can disable the virtio-console port usually
      // allocated to Android's serial console, and redirect it to a sink. This
      // ensures that that the PCI device assignments (and thus sepolicy) don't
      // have to change
      crosvm_cmd.AddHvcSink();
    } else {
      crosvm_cmd.AddSerialSink();
      crosvm_cmd.AddHvcReadWrite(instance.console_out_pipe_name(),
                                 instance.console_in_pipe_name());
    }
  } else {
    // Use an 8250 UART (ISA or platform device) for earlycon, as the
    // virtio-console driver may not be available for early messages
    // In kgdb mode, earlycon is an interactive console, and so early
    // dmesg will go there instead of the kernel.log
    if (config.kgdb() || config.use_bootloader()) {
      crosvm_cmd.AddSerialConsoleReadOnly(instance.kernel_log_pipe_name());
    }

    // as above, create a fake virtio-console 'sink' port when the serial
    // console is disabled, so the PCI device ID assignments don't move
    // around
    crosvm_cmd.AddHvcSink();
  }

  auto crosvm_logs_path = instance.PerInstanceInternalPath("crosvm.fifo");
  auto crosvm_logs = SharedFD::Fifo(crosvm_logs_path, 0666);
  if (!crosvm_logs->IsOpen()) {
    LOG(FATAL) << "Failed to create log fifo for crosvm's stdout/stderr: "
               << crosvm_logs->StrError();
    return {};
  }

  Command crosvm_log_tee_cmd(HostBinaryPath("log_tee"));
  crosvm_log_tee_cmd.AddParameter("--process_name=crosvm");
  crosvm_log_tee_cmd.AddParameter("--log_fd_in=", crosvm_logs);

  // Serial port for logcat, redirected to a pipe
  crosvm_cmd.AddHvcReadOnly(instance.logcat_pipe_name());

  crosvm_cmd.AddHvcReadWrite(
      instance.PerInstanceInternalPath("keymaster_fifo_vm.out"),
      instance.PerInstanceInternalPath("keymaster_fifo_vm.in"));
  crosvm_cmd.AddHvcReadWrite(
      instance.PerInstanceInternalPath("gatekeeper_fifo_vm.out"),
      instance.PerInstanceInternalPath("gatekeeper_fifo_vm.in"));

  if (config.enable_host_bluetooth()) {
    crosvm_cmd.AddHvcReadWrite(
        instance.PerInstanceInternalPath("bt_fifo_vm.out"),
        instance.PerInstanceInternalPath("bt_fifo_vm.in"));
  } else {
    crosvm_cmd.AddHvcSink();
  }
  if (config.enable_gnss_grpc_proxy()) {
    crosvm_cmd.AddHvcReadWrite(
        instance.PerInstanceInternalPath("gnsshvc_fifo_vm.out"),
        instance.PerInstanceInternalPath("gnsshvc_fifo_vm.in"));
  } else {
    crosvm_cmd.AddHvcSink();
  }

  for (auto i = 0; i < VmManager::kMaxDisks - disk_num; i++) {
    crosvm_cmd.AddHvcSink();
  }
  CHECK(crosvm_cmd.HvcNum() + disk_num ==
        VmManager::kMaxDisks + VmManager::kDefaultNumHvcs)
      << "HVC count (" << crosvm_cmd.HvcNum() << ") + disk count (" << disk_num
      << ") is not the expected total of "
      << VmManager::kMaxDisks + VmManager::kDefaultNumHvcs << " devices";

  if (config.enable_audio()) {
    crosvm_cmd.Cmd().AddParameter(
        "--sound=", config.ForDefaultInstance().audio_server_path());
  }

  // TODO(b/162071003): virtiofs crashes without sandboxing, this should be fixed
  if (config.enable_sandbox()) {
    // Set up directory shared with virtiofs
    crosvm_cmd.Cmd().AddParameter(
        "--shared-dir=", instance.PerInstancePath(kSharedDirName),
        ":shared:type=fs");
  }

  // This needs to be the last parameter
  crosvm_cmd.Cmd().AddParameter("--bios=", config.bootloader());

  // TODO(b/199103204): remove this as well when PRODUCT_ENFORCE_MAC80211_HWSIM
  // is removed
  // Only run the leases workaround if we are not using the new network
  // bridge architecture - in that case, we have a wider DHCP address
  // space and stale leases should be much less of an issue
  if (!FileExists("/var/run/cuttlefish-dnsmasq-cvd-wbr.leases") &&
      wifi_tap->IsOpen()) {
    // TODO(schuffelen): QEMU also needs this and this is not the best place for
    // this code. Find a better place to put it.
    auto lease_file =
        ForCurrentInstance("/var/run/cuttlefish-dnsmasq-cvd-wbr-") + ".leases";

    std::uint8_t dhcp_server_ip[] = {
        192, 168, 96, (std::uint8_t)(ForCurrentInstance(1) * 4 - 3)};
    if (!ReleaseDhcpLeases(lease_file, wifi_tap, dhcp_server_ip)) {
      LOG(ERROR) << "Failed to release wifi DHCP leases. Connecting to the wifi "
                 << "network may not work.";
    }
  }

  std::vector<Command> ret;

  if (gpu_capture_enabled) {
    const std::string gpu_capture_basename =
        cpp_basename(config.gpu_capture_binary());

    auto gpu_capture_logs_path =
        instance.PerInstanceInternalPath("gpu_capture.fifo");
    auto gpu_capture_logs = SharedFD::Fifo(gpu_capture_logs_path, 0666);
    if (!gpu_capture_logs->IsOpen()) {
      LOG(FATAL)
          << "Failed to create log fifo for gpu capture's stdout/stderr: "
          << gpu_capture_logs->StrError();
      return {};
    }

    Command gpu_capture_log_tee_cmd(HostBinaryPath("log_tee"));
    gpu_capture_log_tee_cmd.AddParameter("--process_name=",
                                         gpu_capture_basename);
    gpu_capture_log_tee_cmd.AddParameter("--log_fd_in=", gpu_capture_logs);

    Command gpu_capture_command(config.gpu_capture_binary());
    if (gpu_capture_basename == "ngfx") {
      // Crosvm depends on command line arguments being passed as multiple
      // arguments but ngfx only allows a single `--args`. To work around this,
      // create a wrapper script that launches crosvm with all of the arguments
      // and pass this wrapper script to ngfx.
      const std::string crosvm_wrapper_path =
          instance.PerInstanceInternalPath("crosvm_wrapper.sh");
      const std::string crosvm_wrapper_content =
          crosvm_cmd.Cmd().AsBashScript(crosvm_logs_path);

      CHECK(android::base::WriteStringToFile(crosvm_wrapper_content,
                                             crosvm_wrapper_path));
      CHECK(MakeFileExecutable(crosvm_wrapper_path));

      gpu_capture_command.AddParameter("--exe=", crosvm_wrapper_path);
      gpu_capture_command.AddParameter("--launch-detached");
      gpu_capture_command.AddParameter("--verbose");
      gpu_capture_command.AddParameter("--activity=Frame Debugger");
    } else {
      // TODO(natsu): renderdoc
      LOG(FATAL) << "Unhandled GPU capture binary: "
                 << config.gpu_capture_binary();
    }

    gpu_capture_command.RedirectStdIO(Subprocess::StdIOChannel::kStdOut,
                                      gpu_capture_logs);
    gpu_capture_command.RedirectStdIO(Subprocess::StdIOChannel::kStdErr,
                                      gpu_capture_logs);

    ret.push_back(std::move(gpu_capture_log_tee_cmd));
    ret.push_back(std::move(gpu_capture_command));
  } else {
    crosvm_cmd.Cmd().RedirectStdIO(Subprocess::StdIOChannel::kStdOut,
                                   crosvm_logs);
    crosvm_cmd.Cmd().RedirectStdIO(Subprocess::StdIOChannel::kStdErr,
                                   crosvm_logs);

    ret.push_back(std::move(crosvm_cmd.Cmd()));
  }

  ret.push_back(std::move(crosvm_log_tee_cmd));
  return ret;
}

} // namespace vm_manager
} // namespace cuttlefish