Use libvirt to start cuttlefish.
This change employs libvirt to set up virtual machine monitor for us.
Libvirt is a library for configuring and managing VM instances.
Change-Id: Ib42b0642f07c5309243e75e790d85e539d801fa4
(cherry picked from commit 0413c7e535e78472e187b83917ca4ba747b266fa)
diff --git a/README.md b/README.md
index 10f4cf1..0c2c55a 100644
--- a/README.md
+++ b/README.md
@@ -1 +1,89 @@
-# Host-side binaries for Android Virtual Device.
+# Host-side binaries for Android Virtual Device
+
+## Launcher package
+
+### Requirements
+
+* Cuttlefish requires the following packages to be installed on your system:
+ * binaries
+ * python3
+ * libvirt-bin
+ * libvirt-dev
+ * qemu-2.8 or newer
+ * python packages (to be installed with pip):
+ * argparse
+ * eventfd
+ * glog
+ * linuxfd
+ * libvirt
+ * posix_ipc
+ * pylint (development purposes only)
+
+* Users running cuttlefish must be a member of a relevant group enabling them to
+ use `virsh` tool, eg. `libvirtd`.
+ * Group is created automatically when installing `libvirt-bin` package.
+ * Users may need to log out after their membership has been updated; optionally
+ you can use `newgrp` to switch currently active group to `libvirtd`.
+ * Once configured, users should be able to execute
+
+ ```sh
+ $ virsh -c qemu:///system net-list --all
+ Name State Autostart Persistent
+ ----------------------------------------------------------
+ [...]
+ ```
+
+ * You will need to update your configuration `/etc/libvirt/qemu.conf` to disable
+ dynamic permission management for image files. Uncomment and modify relevant
+ config line:
+
+ ```sh
+ dynamic_ownership = 1
+ user = "libvirt-qemu"
+ group = "kvm"
+ # Apparmor would stop us from creating files in /tmp.
+ # TODO(ender): find out a better way to manage these permissions.
+ security_driver = "none"
+ ```
+
+ and restart `libvirt-bin` service:
+
+ ```sh
+ sudo service libvirt-bin restart
+ ```
+
+* Make sure to start the `abr0` android bridge using
+ `experimental/etc/avd/android-metadata-server.sh` script.
+
+ TODO(ender): remove this and make this script part of init.
+
+### I'm seeing `permission denied` errors
+
+libvirt is not executing virtual machines on behalf of the calling user.
+Instead, it calls its own privileged process to configure VM on user's behalf.
+If you're seeing `permission denied` errors chances are that the QEmu does
+not have access to relevant files _OR folders_.
+
+To work with this problem, it's best to copy (not _link_!) all files QEmu would
+need to a separate folder (placed eg. under `/tmp` or `/run`), and give that
+folder proper permissions.
+
+```sh
+➜ ls -l /run/cf
+total 1569216
+drwxr-x--- 2 libvirt-qemu eng 180 Jun 28 14:27 .
+drwxr-xr-x 45 root root 2080 Jun 28 14:27 ..
+-rwxr-x--- 1 root root 2147483648 Jun 28 14:27 cache.img
+-rwxr-x--- 1 root root 10737418240 Jun 28 14:27 data.img
+-rwxr-x--- 1 root root 825340 Jun 28 14:27 gce_ramdisk.img
+-rwxr-x--- 1 root root 6065728 Jun 28 14:27 kernel
+-rwxr-x--- 1 root root 2083099 Jun 28 14:27 ramdisk.img
+-rwxr-x--- 1 root root 3221225472 Jun 28 14:27 system.img
+```
+
+**Note**: the `/run/cf` folder's owner is `libvirt-qemu:eng`. This allows QEmu
+to access images - and me to poke in the folder.
+
+Now don't worry about the `root` ownership. Libvirt manages permissions dynamically.
+You may want to give yourself write permissions to these files during development,
+though.
diff --git a/launcher/BUILD b/launcher/BUILD
index f5c0fd2..d57c02f 100644
--- a/launcher/BUILD
+++ b/launcher/BUILD
@@ -5,8 +5,9 @@
srcs = [
"channel.py",
"clientconnection.py",
- "errors.py",
+ "guest_definition.py",
"ivserver.py",
+ "libvirt_client.py",
"vmconnection.py",
"vsocsharedmem.py",
],
diff --git a/launcher/channel.py b/launcher/channel.py
index cf91da3..ae624d9 100644
--- a/launcher/channel.py
+++ b/launcher/channel.py
@@ -1,61 +1,70 @@
'''
- Related to the UNIX Domain Socket.
+ Related to the UNIX Domain Socket.
'''
import fcntl
+import os
import socket
+import stat
import struct
+import glog
# From include/uapi/asm-generic/ioctls.h
# #define FIONBIO 0x5421
_FIONBIO = 0x5421
def start_listener(path):
- uds = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
- uds.bind(path)
- uds.listen(5)
- return uds
+ uds = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
+ uds.bind(path)
+ uds.listen(5)
+ os.chmod(path,
+ stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR |
+ stat.S_IRGRP | stat.S_IWGRP | stat.S_IXGRP |
+ stat.S_IROTH | stat.S_IWOTH | stat.S_IXOTH)
+ glog.info('Socket %s ready.' % path)
+ return uds
def connect_to_channel(path):
- uds = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
- uds.connect(path)
- return uds
+ uds = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
+ uds.connect(path)
+ return uds
def handle_new_connection(uds, nonblocking=True):
- sock, addr = uds.accept()
+ sock, _ = uds.accept()
- if nonblocking:
- # Set the new socket to non-blocking mode.
- try:
- fcntl.ioctl(sock.fileno(), _FIONBIO, struct.pack('L', 1))
- except OSError as e:
- print('Failed in changing the socket to Non-blocking mode ' + e)
- return sock
+ if nonblocking:
+ # Set the new socket to non-blocking mode.
+ try:
+ fcntl.ioctl(sock.fileno(), _FIONBIO, struct.pack('L', 1))
+ except OSError as exc:
+ glog.exception('Exception caught while trying to make client nonblocking: ', exc)
+ return sock
-#
-# Send 8 bytes.
-#
+
def send_msg_8(sock, data):
- sock.sendmsg([bytes(struct.pack('q', data))])
+ """Send 8 bytes of data over the socket.
+ """
+ sock.sendmsg([bytes(struct.pack('q', data))])
+
def send_ctrl_msg(sock, fd, quad_data):
- sock.sendmsg([bytes(struct.pack('q', quad_data))],
- [(socket.SOL_SOCKET, socket.SCM_RIGHTS,
- bytes(struct.pack('i', fd)))])
+ sock.sendmsg([bytes(struct.pack('q', quad_data))],
+ [(socket.SOL_SOCKET, socket.SCM_RIGHTS,
+ bytes(struct.pack('i', fd)))])
def send_msg_utf8(sock, data):
- sock.sendall(bytes(data, encoding='ascii'))
+ sock.sendall(bytes(data, encoding='ascii'))
def recv_msg(sock, expected_length):
- chunks = []
- received = 0
- while received < expected_length:
- part = sock.recv(expected_length - received)
- received += len(part)
- chunks.append(part.decode(encoding='ascii'))
- data = ''.join(chunks)
- return data
+ chunks = []
+ received = 0
+ while received < expected_length:
+ part = sock.recv(expected_length - received)
+ received += len(part)
+ chunks.append(part.decode(encoding='ascii'))
+ data = ''.join(chunks)
+ return data
diff --git a/launcher/errors.py b/launcher/errors.py
deleted file mode 100644
index 09d0d34..0000000
--- a/launcher/errors.py
+++ /dev/null
@@ -1,7 +0,0 @@
-
-class VersionException(BaseException):
- def __init__(self):
- pass
-
- def __str__(self):
- return "Python 3 needed to run this program"
diff --git a/launcher/guest_definition.py b/launcher/guest_definition.py
new file mode 100644
index 0000000..9784a74
--- /dev/null
+++ b/launcher/guest_definition.py
@@ -0,0 +1,406 @@
+"""Guest definition.
+"""
+# pylint: disable=too-many-instance-attributes,no-self-use
+
+import os
+from xml.etree import ElementTree as ET
+import glog
+
+class GuestDefinition(object):
+ """Guest resource requirements definition.
+
+ Args:
+ lvc LibVirtClient instance.
+ """
+ def __init__(self, lvc):
+ self._cmdline = None
+ self._initrd = None
+ self._instance_name = None
+ self._instance_id = None
+ self._kernel = None
+ self._memory_mb = None
+ self._net_mobile_bridge = None
+ self._iv_vectors = None
+ self._iv_socket_path = None
+ self._iv_socket_path = None
+ self._part_cache = None
+ self._part_data = None
+ self._part_ramdisk = None
+ self._part_system = None
+ self._vcpus = None
+ self._vmm_path = None
+ # Initialize the following fields. These should not be configured by user.
+ self._num_net_interfaces = 0
+ self._num_ttys_interfaces = 0
+ self._num_virtio_channels = 0
+
+ self._lvc = lvc
+ self.set_instance_id(1)
+ self.set_num_vcpus(1)
+ self.set_memory_mb(512)
+ self.set_kernel('vmlinuz')
+ self.set_initrd('initrd.img')
+ self.set_net_mobile_bridge(None)
+
+
+ def set_instance_id(self, inst_id):
+ """Set instance ID of this guest.
+
+ Args:
+ inst_id Numerical instance ID, starting at 1.
+ """
+ if inst_id < 1 or inst_id > 255:
+ glog.error('Ignoring invalid instance id requested: %d.', inst_id)
+ return
+ self._instance_id = inst_id
+ self._instance_name = None
+
+
+ def get_instance_name(self):
+ """Return name of this instance.
+ """
+ if self._instance_name is None:
+ self._instance_name = self._lvc.build_instance_name(self._instance_id)
+ return self._instance_name
+
+
+ def set_num_vcpus(self, cpus):
+ """Set number of virtual CPUs for this guest.
+
+ Number of VCPUs will be checked for sanity and local maximums.
+ """
+ max_cpus = self._lvc.get_max_vcpus()
+ if cpus < 0 or cpus > max_cpus:
+ glog.error('Ignoring invalid number of vcpus requested (%d); ' +
+ 'max is %d.', cpus, max_cpus)
+ return
+ self._vcpus = cpus
+
+
+ def set_memory_mb(self, memory_mb):
+ """Set memory size allocated for the quest in MB.
+
+ Args:
+ memory_mb Total amount of memory allocated for the guest in MB.
+ """
+ if memory_mb < 0:
+ glog.error('Ignoring invalid amount of memory requested (%d).', memory_mb)
+ return
+ self._memory_mb = memory_mb
+
+
+ def set_kernel(self, kernel):
+ """Specify kernel path.
+
+ Args:
+ kernel Path to vmlinuz file.
+ """
+ if not kernel:
+ glog.error('Kernel path must be specified.')
+ return
+ self._kernel = kernel
+
+
+ def set_initrd(self, initrd):
+ """Specify initrd path.
+
+ Args:
+ initrd Path to initrd.img file.
+ """
+ if not initrd:
+ glog.error('Initial ramdisk path must be specified.')
+ return
+ self._initrd = initrd
+
+
+ def set_cmdline(self, cmdline):
+ """Specify kernel command line arguments.
+
+ Args:
+ cmdline Additional kernel command line arguments.
+ """
+ self._cmdline = cmdline
+
+
+ def set_cf_ramdisk_path(self, path):
+ """Specify cuttlefish ramdisk path.
+
+ Args:
+ path Cuttlefish built 'ramdisk.img' path.
+ """
+ if path is not None and not os.path.exists(path):
+ glog.warning("Cuttlefish ramdisk.img not found at %s", path)
+ self._part_ramdisk = path
+
+
+ def set_cf_system_path(self, path):
+ """Specify cuttlefish system path.
+
+ Args:
+ path Cuttlefish built 'system.img' path.
+ """
+ if path is not None and not os.path.exists(path):
+ glog.warning("Cuttlefish system.img not found at %s", path)
+ self._part_system = path
+
+
+ def set_cf_data_path(self, path):
+ """Specify cuttlefish data path.
+
+ Args:
+ path Cuttlefish built 'data.img' path.
+ """
+ if path is not None and not os.path.exists(path):
+ glog.warning("Cuttlefish data.img not found at %s", path)
+ self._part_data = path
+
+
+ def set_cf_cache_path(self, path):
+ """Specify cuttlefish cache path.
+
+ Args:
+ path Cuttlefish built 'cache.img' path.
+ """
+ if path is not None and not os.path.exists(path):
+ glog.warning("Cuttlefish cache.img not found at %s", path)
+ self._part_cache = path
+
+
+ def set_net_mobile_bridge(self, bridge):
+ """Specify mobile network bridge name.
+
+ Args:
+ bridge Name of the mobile network bridge.
+ """
+ if bridge is not None:
+ # TODO(ender): check if bridge exists.
+ pass
+ self._net_mobile_bridge = bridge
+
+
+ def set_vmm_path(self, path):
+ """Specify path to virtual machine monitor that will be running our guest.
+
+ Args:
+ path Path to virtual machine monitor, eg. /usr/bin/qemu.
+ """
+ self._vmm_path = path
+
+
+ def set_ivshmem_vectors(self, num):
+ """Specify number of IV Shared Memory vectors.
+
+ Args:
+ num Number of vectors (non-negative).
+ """
+ if num < 0:
+ glog.error('Invalid number of iv shared memory vectors: %d', num)
+ return
+ self._iv_vectors = num
+
+
+ def set_ivshmem_socket_path(self, path):
+ """Specify path to unix socket managed by IV Shared Memory daemon.
+
+ Args:
+ path Path to unix domain socket.
+ """
+ self._iv_socket_path = path
+
+
+ def _configure_vm(self, tree):
+ """Create basic guest details.
+
+ Args:
+ tree Top level 'domain' element of the XML tree.
+ """
+ ET.SubElement(tree, 'name').text = self.get_instance_name()
+ ET.SubElement(tree, 'on_poweroff').text = 'destroy'
+ ET.SubElement(tree, 'on_reboot').text = 'restart'
+ # TODO(ender): should this be restart?
+ ET.SubElement(tree, 'on_crash').text = 'destroy'
+ ET.SubElement(tree, 'vcpu').text = str(self._vcpus)
+ ET.SubElement(tree, 'memory').text = str(self._memory_mb << 10)
+
+
+ def _configure_kernel(self, tree):
+ """Configure boot parameters for guest.
+
+ Args:
+ tree Top level 'domain' element of the XML tree.
+ """
+ node = ET.SubElement(tree, 'os')
+ desc = ET.SubElement(node, 'type')
+ desc.set('arch', 'x86_64')
+ desc.set('machine', 'pc')
+ desc.text = 'hvm'
+
+ ET.SubElement(node, 'kernel').text = self._kernel
+ ET.SubElement(node, 'initrd').text = self._initrd
+ if self._cmdline is not None:
+ ET.SubElement(node, 'cmdline').text = self._cmdline
+
+
+ def _build_device_serial_port(self):
+ """Configure serial ports for guest.
+
+ More useful information can be found here:
+ https://libvirt.org/formatdomain.html#elementCharSerial
+ """
+ index = self._num_ttys_interfaces
+ self._num_ttys_interfaces += 1
+ path = '/tmp/%s-ttyS%d.log' % (self.get_instance_name(), index)
+ tty = ET.Element('serial')
+ tty.set('type', 'file')
+ src = ET.SubElement(tty, 'source')
+ src.set('path', path)
+ src.set('append', 'no')
+ ET.SubElement(tty, 'target').set('port', str(index))
+ glog.info('Serial port %d will send data to %s', index, path)
+ return tty
+
+
+ def _build_device_virtio_channel(self):
+ """Build fast paravirtualized virtio channel.
+
+ More useful information can be found here:
+ https://libvirt.org/formatdomain.html#elementCharSerial
+ """
+ index = self._num_virtio_channels
+ self._num_virtio_channels += 1
+ path = '/tmp/%s-vport0p%d.log' % (self.get_instance_name(), index)
+ vio = ET.Element('channel')
+ vio.set('type', 'file')
+ src = ET.SubElement(vio, 'source')
+ src.set('path', path)
+ src.set('append', 'no')
+ tgt = ET.SubElement(vio, 'target')
+ tgt.set('type', 'virtio')
+ tgt.set('name', 'vport0p%d' % index)
+ adr = ET.SubElement(vio, 'address')
+ adr.set('type', 'virtio-serial')
+ adr.set('controller', '0')
+ adr.set('bus', '0')
+ adr.set('port', str(index))
+ glog.info('Virtio channel %d will send data to %s', index, path)
+ return vio
+
+
+ def _build_device_disk_node(self, path, name, target_dev):
+ """Create disk node for guest.
+
+ More useful information can be found here:
+ https://libvirt.org/formatdomain.html#elementsDisks
+
+ Args:
+ path Path to file containing partition or disk image.
+ name Purpose of partition or disk image.
+ target_dev Target device.
+ """
+ bus = 'ide'
+ if target_dev.startswith('sd'):
+ bus = 'sata'
+ elif target_dev.startswith('vd'):
+ bus = 'virtio'
+
+ if path is None:
+ glog.fatal('No file specified for %s; (%s) %s is not available.',
+ name, bus, target_dev)
+ return None
+
+ disk = ET.Element('disk')
+ disk.set('type', 'file')
+ # disk.set('snapshot', 'external')
+ drvr = ET.SubElement(disk, 'driver')
+ drvr.set('name', 'qemu')
+ drvr.set('type', 'raw')
+ drvr.set('io', 'threads')
+ trgt = ET.SubElement(disk, 'target')
+ trgt.set('dev', target_dev)
+ trgt.set('bus', bus)
+ srce = ET.SubElement(disk, 'source')
+ srce.set('file', path)
+ return disk
+
+
+ def _build_mac_address(self, index):
+ """Create mac address from local instance number.
+ """
+ return '00:41:56:44:%02X:%02X' % (self._instance_id, index + 1)
+
+
+ def _build_device_net_node(self, local_node, bridge):
+ """Create virtual ethernet for guest.
+
+ More useful information can be found here:
+ https://libvirt.org/formatdomain.html#elementsNICSVirtual
+ https://wiki.libvirt.org/page/Virtio
+
+ Args:
+ local_node Name of the local interface.
+ bridge Name of the corresponding bridge.
+ """
+ index = self._num_net_interfaces
+ self._num_net_interfaces += 1
+ net = ET.Element('interface')
+ net.set('type', 'bridge')
+ ET.SubElement(net, 'source').set('bridge', bridge)
+ ET.SubElement(net, 'mac').set('address', self._build_mac_address(index))
+ ET.SubElement(net, 'model').set('type', 'e1000')
+ ET.SubElement(net, 'target').set('dev', '%s%d' % (local_node, self._instance_id))
+ return net
+
+ def _configure_devices(self, tree):
+ """Configure guest devices.
+
+ Args:
+ tree Top level 'domain' element of the XML tree.
+ """
+ dev = ET.SubElement(tree, 'devices')
+ if self._vmm_path:
+ ET.SubElement(dev, 'emulator').text = self._vmm_path
+ dev.append(self._build_device_serial_port())
+ dev.append(self._build_device_virtio_channel())
+ dev.append(self._build_device_disk_node(self._part_ramdisk, 'ramdisk', 'vda'))
+ dev.append(self._build_device_disk_node(self._part_system, 'system', 'vdb'))
+ dev.append(self._build_device_disk_node(self._part_data, 'data', 'vdc'))
+ dev.append(self._build_device_disk_node(self._part_cache, 'cache', 'vdd'))
+ dev.append(self._build_device_net_node('amobile', self._net_mobile_bridge))
+
+
+ def _configure_ivshmem(self, tree):
+ """Configure InterVM Shared Memory region.
+
+ Args:
+ tree Top level 'domain' element of the XML tree.
+ """
+ if self._iv_vectors:
+ cmd = ET.SubElement(tree, 'qemu:commandline')
+ ET.SubElement(cmd, 'qemu:arg').set('value', '-chardev')
+ ET.SubElement(cmd, 'qemu:arg').set(
+ 'value', 'socket,path=%s,id=ivsocket' % (self._iv_socket_path))
+ ET.SubElement(cmd, 'qemu:arg').set('value', '-device')
+ ET.SubElement(cmd, 'qemu:arg').set(
+ 'value', 'ivshmem-doorbell,chardev=ivsocket,vectors=%d' % (self._iv_vectors)
+ )
+
+
+ def to_xml(self):
+ """Build XML document describing guest properties.
+
+ The created document will be used directly by libvirt to create corresponding
+ virtual machine.
+
+ Returns:
+ string containing an XML document describing a VM.
+ """
+ tree = ET.Element('domain')
+ tree.set('type', 'kvm')
+ tree.set('xmlns:qemu', 'http://libvirt.org/schemas/domain/qemu/1.0')
+
+ self._configure_vm(tree)
+ self._configure_kernel(tree)
+ self._configure_devices(tree)
+ self._configure_ivshmem(tree)
+
+ return ET.tostring(tree).decode('utf-8')
diff --git a/launcher/ivserver.py b/launcher/ivserver.py
index 486d55f..a7a645d 100644
--- a/launcher/ivserver.py
+++ b/launcher/ivserver.py
@@ -1,227 +1,205 @@
'''
- ivshmem server main
+ ivshmem server main
'''
+# pylint: disable=too-many-instance-attributes,relative-beyond-top-level
+
import argparse
import json
-import linuxfd
import os
import select
-import subprocess
+import signal
import sys
+import threading
+import glog
+from .libvirt_client import LibVirtClient
+from .guest_definition import GuestDefinition
from . import clientconnection
from . import channel
-from . import errors
from . import vmconnection
from . import vsocsharedmem
-#
-# eventfd for synchronizing ivshmemserver initialization and QEMU launch.
-# Also used to pass the vector count to QEMU.
-#
-efd = {
- 'efd' : None
-}
+class IVServer(object):
+ def __init__(self, region_name, region_size,
+ vm_socket_path, client_socket_path, layout_json):
+ self.layout_json = layout_json
-class IVServer():
- def __init__(self, args, layout_json):
- self.args = args
- self.layout_json = layout_json
+ # Create the SharedMemory. Linux zeroes out the contents initially.
+ self.shmobject = vsocsharedmem.VSOCSharedMemory(region_size, region_name)
- # Create the SharedMemory. Linux zeroes out the contents initially.
- self.shmobject = vsocsharedmem.VSOCSharedMemory(self.args.size,
- self.args.name)
+ # Populate the shared memory with the data from layout description.
+ self.shmobject.create_layout(self.layout_json)
- # Populate the shared memory with the data from layout description.
- self.shmobject.create_layout(self.layout_json)
+ # get the number of vectors. This will be passed to qemu.
+ self.num_vectors = self.get_vector_count()
- # get the number of vectors. This will be passed to qemu.
- self.num_vectors = self.get_vector_count()
+ # Establish the listener socket for QEMU.
+ self.vm_listener_socket_path = vm_socket_path
+ self.vm_listener_socket = channel.start_listener(vm_socket_path)
- # Establish the listener socket for QEMU.
- self.vm_listener_socket = channel.start_listener(self.args.path)
+ # Establish the listener socket for Clients.
+ self.client_listener_socket = channel.start_listener(client_socket_path)
- # Establish the listener socket for Clients.
- self.client_listener_socket = channel.start_listener(self.args.client)
+ self.vm_connection = None
+ self.client_connection = None
- self.vm_connection = None
- self.client_connection = None
+ # _control_channel and _thread_channel are two ends of the same, control pipe.
+ # _thread_channel will be used by the serving thread until pipe is closed.
+ (self._control_channel, self._thread_channel) = os.pipe()
+ self._thread = threading.Thread(target=self._serve_in_background)
+
+ def get_vector_count(self):
+ # TODO: Parse from json instead of picking it up from shmobject.
+ return self.shmobject.num_vectors
+
+ def get_socket_path(self):
+ return self.vm_listener_socket_path
+
+ def serve(self):
+ """Begin serving IVShMem data to QEmu.
+ """
+ self._thread.start()
+
+ def stop(self):
+ """Stop serving data to QEmu and join the serving thread.
+ """
+ os.close(self._control_channel)
+ self._thread.join()
+
+ def _serve_in_background(self):
+ readfdlist = [
+ self.vm_listener_socket,
+ self.client_listener_socket,
+ self._thread_channel
+ ]
+
+ while True:
+ readable, _, _ = select.select(readfdlist, [], [])
+
+ for client in readable:
+ if client == self.vm_listener_socket:
+ self.handle_new_vm_connection(client)
+ elif client == self.client_listener_socket:
+ self.handle_new_client_connection(client)
+ elif client == self._thread_channel:
+ # For now we do not expect any communication over pipe.
+ # Since this bit of code is going away, we'll just assume
+ # that the parent wants the thread to exit.
+ return
- # TODO: Parse from json instead of picking it up from shmobject.
- def get_vector_count(self):
- return self.shmobject.num_vectors
+ def handle_new_client_connection(self, listenersocket):
+ client_socket = channel.handle_new_connection(listenersocket,
+ nonblocking=False)
+ print(client_socket)
+ self.client_connection = \
+ clientconnection.ClientConnection(client_socket,
+ self.shmobject.posix_shm.fd,
+ self.layout_json,
+ 0)
+ self.client_connection.handshake()
- def serve(self):
- readfdlist = [self.vm_listener_socket, self.client_listener_socket]
- writefdlist = []
- exceptionfdlist = [self.vm_listener_socket, self.client_listener_socket]
- #
- # We are almost ready.
- # There still may be a race between the following select and QEMU
- # execution.
- #
- efd['efd'].write(self.num_vectors)
- while True:
- readable, writeable, exceptions = \
- select.select(readfdlist, writefdlist, exceptionfdlist)
+ def handle_new_vm_connection(self, listenersocket):
+ vm_socket = channel.handle_new_connection(listenersocket)
+ print(vm_socket)
+ self.vm_connection = vmconnection.VMConnection(self.layout_json,
+ self.shmobject.posix_shm,
+ vm_socket,
+ self.num_vectors,
+ hostid=0,
+ vmid=1)
+ self.vm_connection.handshake()
- if exceptions:
- print(exceptions)
- return
-
- for listenersocket in readable:
- if listenersocket == self.vm_listener_socket:
- self.handle_new_vm_connection(listenersocket)
- elif listenersocket == self.client_listener_socket:
- self.handle_new_client_connection(listenersocket)
-
-
- def handle_new_client_connection(self, listenersocket):
- client_socket = channel.handle_new_connection(listenersocket,
- nonblocking=False)
- print(client_socket)
- self.client_connection = \
- clientconnection.ClientConnection(client_socket,
- self.shmobject.posix_shm.fd,
- self.layout_json,
- 0)
- self.client_connection.handshake()
-
-
- def handle_new_vm_connection(self, listenersocket):
- vm_socket = channel.handle_new_connection(listenersocket)
- print(vm_socket)
- self.vm_connection = vmconnection.VMConnection(self.layout_json,
- self.shmobject.posix_shm,
- vm_socket,
- self.num_vectors,
- hostid=0,
- vmid=1)
- self.vm_connection.handshake()
def setup_arg_parser():
- def unsigned_integer(size):
- size = int(size)
- if size < 1:
- raise argparse.ArgumentTypeError('should be >= 1 but we have %r' % size)
- return size
- parser = argparse.ArgumentParser()
- parser.add_argument('-c', '--cpu', type=unsigned_integer, default=2,
- help='Number of cpus to use in the guest')
- parser.add_argument('-C', '--client', type=str,
- default='/tmp/ivshmem_socket_client')
- parser.add_argument('-i', '--image_dir', type=str, required=True,
- help='Path to the directory of image files for the guest')
- parser.add_argument('-s', '--script_dir', type=str, required=True,
- help='Path to a directory of scripts')
- parser.add_argument('-I', '--instance_number', type=unsigned_integer,
- default=1,
- help='Instance number for this device')
- parser.add_argument('-L', '--layoutfile', type=str, required=True)
- parser.add_argument('-M', '--memory', type=unsigned_integer, default=2048,
- help='Size of the non-shared guest RAM in MiB')
- parser.add_argument('-N', '--name', type=str, default='ivshmem',
- help='Name of the POSIX shared memory segment')
- parser.add_argument('-P', '--path', type=str, default='/tmp/ivshmem_socket',
- help='Path to UNIX Domain Socket, default=/tmp/ivshmem_socket')
- parser.add_argument('-S', '--size', type=unsigned_integer, default=4,
- help='Size of shared memory region in MiB, default=4MiB')
- return parser
-
-
-def make_telnet_chardev(args, tcp_base, name):
- return 'socket,nowait,server,host=127.0.0.1,port=%d,ipv4,nodelay,id=%s' % (
- tcp_base + args.instance_number, name)
-
-
-def make_network_netdev(name, args):
- return ','.join((
- 'type=tap',
- 'id=%s' % name,
- 'ifname=android%d' % args.instance_number,
- 'script=%s/android-ifup' % args.script_dir,
- 'downscript=%s/android-ifdown' % args.script_dir))
-
-
-def make_network_device(name, instance_number):
- mac_addr = '00:41:56:44:%02X:%02X' % (
- instance_number / 10, instance_number % 10)
- return 'e1000,netdev=%s,mac=%s' % (name, mac_addr)
+ def unsigned_integer(size):
+ size = int(size)
+ if size < 1:
+ raise argparse.ArgumentTypeError(
+ 'should be >= 1 but we have %r' % size)
+ return size
+ parser = argparse.ArgumentParser()
+ parser.add_argument('-c', '--cpu', type=unsigned_integer, default=2,
+ help='Number of cpus to use in the guest')
+ parser.add_argument('-C', '--client', type=str,
+ default='/tmp/ivshmem_socket_client')
+ parser.add_argument('-i', '--image_dir', type=str, required=True,
+ help='Path to the directory of image files for the guest')
+ parser.add_argument('-s', '--script_dir', type=str, required=True,
+ help='Path to a directory of scripts')
+ parser.add_argument('-I', '--instance_number', type=unsigned_integer,
+ default=1,
+ help='Instance number for this device')
+ parser.add_argument('-L', '--layoutfile', type=str, required=True)
+ parser.add_argument('-M', '--memory', type=unsigned_integer, default=2048,
+ help='Size of the non-shared guest RAM in MiB')
+ parser.add_argument('-N', '--name', type=str, default='ivshmem',
+ help='Name of the POSIX shared memory segment')
+ parser.add_argument('-P', '--path', type=str, default='/tmp/ivshmem_socket',
+ help='Path to UNIX Domain Socket, default=/tmp/ivshmem_socket')
+ parser.add_argument('-S', '--size', type=unsigned_integer, default=4,
+ help='Size of shared memory region in MiB, default=4MiB')
+ return parser
def check_version():
- if sys.version_info.major != 3:
- raise errors.VersionException
+ if sys.version_info.major != 3:
+ glog.fatal('This program requires python3 to work.')
+ sys.exit(1)
def main():
- check_version()
- parser = setup_arg_parser()
- args = parser.parse_args()
- layout_json = json.loads(open(args.layoutfile).read())
- efd['efd'] = linuxfd.eventfd(initval=0,
- semaphore=False,
- nonBlocking=False,
- closeOnExec=True)
- pid = os.fork()
+ glog.setLevel(glog.INFO)
+ try:
+ check_version()
+ parser = setup_arg_parser()
+ args = parser.parse_args()
- if pid:
- ivshmem_server = IVServer(args, layout_json)
- ivshmem_server.serve()
- else:
- #
- # wait for server to complete initialization
- # the initializing process will also write the
- # number of vectors as a part of signaling us.
- #
- num_vectors = efd['efd'].read()
- qemu_args = []
- qemu_args.append(layout_json['guest']['vmm_path'])
- qemu_args += ('-smp', '%d' % args.cpu)
- qemu_args += ('-m', '%d' % args.memory)
- qemu_args.append('-enable-kvm')
- qemu_args.append('-nographic')
- # Serial port setup
- qemu_args += (
- '-chardev', make_telnet_chardev(args, 10000, 'serial_kernel'),
- '-device', 'isa-serial,chardev=serial_kernel')
- qemu_args += (
- '-device', 'virtio-serial',
- '-chardev', make_telnet_chardev(args, 10100, 'serial_logcat'),
- '-device', 'virtserialport,chardev=serial_logcat')
- # network setup
- qemu_args += (
- '-netdev', make_network_netdev('net0', args),
- '-device', make_network_device('net0', args.instance_number)
- )
- # Configure image files
- launchable = True
- # Use %% to protect the path. The index will be set outside the loop,
- # the path will be set inside.
- DRIVE_VALUE = 'file=%%s,index=%d,format=raw,if=virtio,media=disk'
- for flag, template, path in (
- ('-kernel', '%s', 'kernel'),
- ('-initrd', '%s', 'gce_ramdisk.img'),
- ('-drive', DRIVE_VALUE % 0, 'ramdisk.img'),
- ('-drive', DRIVE_VALUE % 1, 'system.img'),
- ('-drive', DRIVE_VALUE % 2, 'data-%d.img' % args.instance_number),
- ('-drive', DRIVE_VALUE % 3, 'cache-%d.img' % args.instance_number)):
- full_path = os.path.join(args.image_dir, path)
- if not os.path.isfile(full_path):
- print('Missing required image file %s' % full_path)
- launchable = False
- qemu_args += (flag, template % full_path)
- # TODO(romitd): Should path and id be configured per-instance?
- qemu_args += ('-chardev', 'socket,path=%s,id=ivsocket' % (args.path))
- qemu_args += (
- '-device', ('ivshmem-doorbell,chardev=ivsocket,vectors=%d' %
- num_vectors))
- qemu_args += ('-append', ' '.join(layout_json['guest']['kernel_command_line']))
- if not launchable:
- print('Refusing to launch due to errors')
- sys.exit(2)
- subprocess.Popen(qemu_args)
+ lvc = LibVirtClient()
+
+ layout_json = json.loads(open(args.layoutfile).read())
+ ivshmem_server = IVServer(args.name, args.size, args.path, args.client, layout_json)
+
+ if 'movbe' not in lvc.get_cpu_features():
+ glog.warning('host CPU may not support movbe instruction')
+
+ guest = GuestDefinition(lvc)
+
+ guest.set_num_vcpus(args.cpu)
+ guest.set_memory_mb(args.memory)
+ guest.set_instance_id(args.instance_number)
+ guest.set_cmdline(' '.join(layout_json['guest']['kernel_command_line']))
+ guest.set_kernel(os.path.join(args.image_dir, 'kernel'))
+ guest.set_initrd(os.path.join(args.image_dir, 'gce_ramdisk.img'))
+
+ guest.set_cf_ramdisk_path(os.path.join(args.image_dir, 'ramdisk.img'))
+ guest.set_cf_system_path(os.path.join(args.image_dir, 'system.img'))
+ guest.set_cf_data_path(os.path.join(args.image_dir, 'data.img'))
+ guest.set_cf_cache_path(os.path.join(args.image_dir, 'cache.img'))
+ guest.set_net_mobile_bridge('abr0')
+
+ guest.set_ivshmem_vectors(ivshmem_server.get_vector_count())
+ guest.set_ivshmem_socket_path(ivshmem_server.get_socket_path())
+
+ guest.set_vmm_path(layout_json['guest']['vmm_path'])
+
+ # Accept and process IVShMem connections from QEmu.
+ ivshmem_server.serve()
+
+ glog.info('Creating virtual instance...')
+ dom = lvc.create_instance(guest.to_xml())
+ glog.info('VM ready.')
+ dom.resume()
+ try:
+ signal.pause()
+ except KeyboardInterrupt:
+ glog.info('Stopping IVShMem server')
+ dom.destroy()
+ ivshmem_server.stop()
+
+
+ except Exception as exception:
+ glog.exception('Could not start VM: %s', exception)
if __name__ == '__main__':
- main()
+ main()
diff --git a/launcher/launcher.sh b/launcher/launcher.sh
index f1bcaf4..4219337 100755
--- a/launcher/launcher.sh
+++ b/launcher/launcher.sh
@@ -1,9 +1,9 @@
#!/bin/bash
-rm -f /tmp/ivshmem_socket /tmp/ivshmem_socket_client
+trap 'rm -f /tmp/ivshmem_socket /tmp/ivshmem_socket_client' INT KILL EXIT
DIR=$(dirname $(realpath $0))
WS=$(bazel info workspace)
BIN=$(bazel info bazel-bin)
DIR=${DIR##${WS}}
bazel build /${DIR}:launcher
-sudo ${BIN}/${DIR}/launcher "$@"
+${BIN}/${DIR}/launcher "$@"
diff --git a/launcher/libvirt_client.py b/launcher/libvirt_client.py
new file mode 100644
index 0000000..864f95d
--- /dev/null
+++ b/launcher/libvirt_client.py
@@ -0,0 +1,114 @@
+"""libvirt interface.
+
+Primary purpose of this class is to aid communication between libvirt and other classes.
+libvirt's preferred method of delivery of larger object is, sadly, xml (rather than objects).
+"""
+
+# pylint: disable=no-self-use
+
+from xml.etree import ElementTree
+import glog
+import libvirt
+
+class LibVirtClient(object):
+ """Client of the libvirt library.
+ """
+ def __init__(self):
+ # Open channel to QEmu instance running locally.
+ self.lvch = libvirt.open('qemu:///system')
+ if self.lvch is None:
+ raise Exception('Could not open libvirt channel. Did you install libvirt package?')
+
+ self.capabilities = None
+
+ # Parse host capabilities. Confirm our CPU is capable of executing movbe instruction
+ # which allows further compatibility with atom cpus.
+ self.host_capabilities = ElementTree.fromstring(self.lvch.getCapabilities())
+ glog.info('Starting cuttlefish on %s', self.get_hostname())
+ glog.info('Max number of virtual CPUs: %d', self.get_max_vcpus())
+ glog.info('Supported virtualization type: %s', self.get_virtualization_type())
+
+
+ def get_hostname(self):
+ """Return name of the host that will run guest images.
+
+ Returns:
+ hostname as string.
+ """
+ return self.lvch.getHostname()
+
+
+ def get_max_vcpus(self):
+ """Query max number of VCPUs that can be used by virtual instance.
+
+ Returns:
+ number of VCPUs that can be used by virtual instance.
+ """
+ return self.lvch.getMaxVcpus(None)
+
+
+ def get_virtualization_type(self):
+ """Query virtualization type supported by host.
+
+ Returns:
+ string describing supported virtualization type.
+ """
+ return self.lvch.getType()
+
+
+ def get_instance(self, name):
+ """Get libvirt instance matching supplied name.
+
+ Args:
+ name Name of the virtual instance.
+ Returns:
+ libvirt instance or None, if no instance by that name was found.
+ """
+ return self.lvch.lookupByName(name)
+
+
+ def create_instance(self, description):
+ """Create new instance based on the XML description.
+
+ Args:
+ description XML string describing instance.
+ Returns:
+ libvirt domain representing started domain. Domain will be automatically
+ destroyed when this handle is orphaned.
+ """
+ return self.lvch.createXML(description,
+ libvirt.VIR_DOMAIN_START_AUTODESTROY)
+
+
+ def get_cpu_features(self):
+ """Get host capabilities from libvirt.
+
+ Returns:
+ set of CPU features reported by the host.
+ """
+ caps = self.capabilities or set()
+ try:
+ if self.capabilities is None:
+ features = self.host_capabilities.findall('./host/cpu/feature')
+ if features is None:
+ glog.warning('no \'host.cpu.feature\' nodes reported by libvirt.')
+ return caps
+
+ for feature in features:
+ caps.add(feature.get('name'))
+ return caps
+
+ finally:
+ # Make sure to update self.capabilities with empty set if anything goes wrong.
+ self.capabilities = caps
+
+
+ def build_instance_name(self, instance_number: int):
+ """Convert instance number to an instance id (or domain).
+
+ Args:
+ instance_number Number of Cuttlefish instance.
+ Returns:
+ string representing instance (domain) name.
+ """
+ return 'android_cuttlefish_{}'.format(instance_number)
