| #!/usr/bin/env python |
| # |
| # Copyright 2018 - The Android Open Source Project |
| # |
| # Licensed under the Apache License, Version 2.0 (the "License"); |
| # you may not use this file except in compliance with the License. |
| # You may obtain a copy of the License at |
| # |
| # http://www.apache.org/licenses/LICENSE-2.0 |
| # |
| # Unless required by applicable law or agreed to in writing, software |
| # distributed under the License is distributed on an "AS IS" BASIS, |
| # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| # See the License for the specific language governing permissions and |
| # limitations under the License. |
| """Common operations between managing GCE and Cuttlefish devices. |
| |
| This module provides the common operations between managing GCE (device_driver) |
| and Cuttlefish (create_cuttlefish_action) devices. Should not be called |
| directly. |
| """ |
| |
| from __future__ import print_function |
| import getpass |
| import logging |
| import os |
| import subprocess |
| |
| from acloud import errors |
| from acloud.public import avd |
| from acloud.public import report |
| from acloud.internal import constants |
| from acloud.internal.lib import utils |
| |
| logger = logging.getLogger(__name__) |
| |
| def CreateSshKeyPairIfNecessary(cfg): |
| """Create ssh key pair if necessary. |
| |
| Args: |
| cfg: An Acloudconfig instance. |
| |
| Raises: |
| error.DriverError: If it falls into an unexpected condition. |
| """ |
| if not cfg.ssh_public_key_path: |
| logger.warning( |
| "ssh_public_key_path is not specified in acloud config. " |
| "Project-wide public key will " |
| "be used when creating AVD instances. " |
| "Please ensure you have the correct private half of " |
| "a project-wide public key if you want to ssh into the " |
| "instances after creation.") |
| elif cfg.ssh_public_key_path and not cfg.ssh_private_key_path: |
| logger.warning( |
| "Only ssh_public_key_path is specified in acloud config, " |
| "but ssh_private_key_path is missing. " |
| "Please ensure you have the correct private half " |
| "if you want to ssh into the instances after creation.") |
| elif cfg.ssh_public_key_path and cfg.ssh_private_key_path: |
| utils.CreateSshKeyPairIfNotExist(cfg.ssh_private_key_path, |
| cfg.ssh_public_key_path) |
| else: |
| # Should never reach here. |
| raise errors.DriverError( |
| "Unexpected error in CreateSshKeyPairIfNecessary") |
| |
| |
| class DevicePool(object): |
| """A class that manages a pool of virtual devices. |
| |
| Attributes: |
| devices: A list of devices in the pool. |
| """ |
| |
| def __init__(self, device_factory, devices=None): |
| """Constructs a new DevicePool. |
| |
| Args: |
| device_factory: A device factory capable of producing a goldfish or |
| cuttlefish device. The device factory must expose an attribute with |
| the credentials that can be used to retrieve information from the |
| constructed device. |
| devices: List of devices managed by this pool. |
| """ |
| self._devices = devices or [] |
| self._device_factory = device_factory |
| self._compute_client = device_factory.GetComputeClient() |
| |
| def _CollectAdbLogcats(self, output_dir): |
| """Collect Adb logcats. |
| |
| Args: |
| output_dir: String, the output file directory to store adb logcats. |
| |
| Returns: |
| The file information dictionary with file path and file name. |
| """ |
| file_dict = {} |
| for device in self._devices: |
| if not device.adb_port: |
| # If device adb tunnel is not established, do not do adb logcat |
| continue |
| file_name = "%s_adb_logcat.log" % device.instance_name |
| full_file_path = os.path.join(output_dir, file_name) |
| logger.info("Get adb %s:%s logcat for instance %s", |
| constants.LOCALHOST, device.adb_port, |
| device.instance_name) |
| try: |
| subprocess.check_call( |
| ["adb -s %s:%s logcat -b all -d > %s" % ( |
| constants.LOCALHOST, device.adb_port, full_file_path)], |
| shell=True) |
| file_dict[full_file_path] = file_name |
| except subprocess.CalledProcessError: |
| logging.error("Failed to get adb logcat for %s for instance %s", |
| device.serial_number, device.instance_name) |
| return file_dict |
| |
| def CreateDevices(self, num): |
| """Creates |num| devices for given build_target and build_id. |
| |
| Args: |
| num: Number of devices to create. |
| """ |
| # Create host instances for cuttlefish/goldfish device. |
| # Currently one instance supports only 1 device. |
| for _ in range(num): |
| instance = self._device_factory.CreateInstance() |
| ip = self._compute_client.GetInstanceIP(instance) |
| self.devices.append( |
| avd.AndroidVirtualDevice(ip=ip, instance_name=instance)) |
| |
| @utils.TimeExecute(function_description="Waiting for AVD(s) to boot up", |
| result_evaluator=utils.BootEvaluator) |
| def WaitForBoot(self): |
| """Waits for all devices to boot up. |
| |
| Returns: |
| A dictionary that contains all the failures. |
| The key is the name of the instance that fails to boot, |
| and the value is an errors.DeviceBootError object. |
| """ |
| failures = {} |
| for device in self._devices: |
| try: |
| self._compute_client.WaitForBoot(device.instance_name) |
| except errors.DeviceBootError as e: |
| failures[device.instance_name] = e |
| return failures |
| |
| def PullLogs(self, source_files, output_dir, user=None, ssh_rsa_path=None): |
| """Tar logs from GCE instance into output_dir. |
| |
| Args: |
| source_files: List of file names to be pulled. |
| output_dir: String. The output file dirtory |
| user: String, the ssh username to access GCE |
| ssh_rsa_path: String, the ssh rsa key path to access GCE |
| |
| Returns: |
| The file dictionary with file_path and file_name |
| """ |
| |
| file_dict = {} |
| for device in self._devices: |
| if isinstance(source_files, basestring): |
| source_files = [source_files] |
| for source_file in source_files: |
| file_name = "%s_%s" % (device.instance_name, |
| os.path.basename(source_file)) |
| dst_file = os.path.join(output_dir, file_name) |
| logger.info("Pull %s for instance %s with user %s to %s", |
| source_file, device.instance_name, user, dst_file) |
| try: |
| utils.ScpPullFile(source_file, dst_file, device.ip, |
| user_name=user, rsa_key_file=ssh_rsa_path) |
| file_dict[dst_file] = file_name |
| except errors.DeviceConnectionError as e: |
| logger.warning("Failed to pull %s from instance %s: %s", |
| source_file, device.instance_name, e) |
| return file_dict |
| |
| def CollectSerialPortLogs(self, output_file, |
| port=constants.DEFAULT_SERIAL_PORT): |
| """Tar the instance serial logs into specified output_file. |
| |
| Args: |
| output_file: String, the output tar file path |
| port: The serial port number to be collected |
| """ |
| # For emulator, the serial log is the virtual host serial log. |
| # For GCE AVD device, the serial log is the AVD device serial log. |
| with utils.TempDir() as tempdir: |
| src_dict = {} |
| for device in self._devices: |
| logger.info("Store instance %s serial port %s output to %s", |
| device.instance_name, port, output_file) |
| serial_log = self._compute_client.GetSerialPortOutput( |
| instance=device.instance_name, port=port) |
| file_name = "%s_serial_%s.log" % (device.instance_name, port) |
| file_path = os.path.join(tempdir, file_name) |
| src_dict[file_path] = file_name |
| with open(file_path, "w") as f: |
| f.write(serial_log.encode("utf-8")) |
| utils.MakeTarFile(src_dict, output_file) |
| |
| def CollectLogcats(self, output_file, ssh_user, ssh_rsa_path): |
| """Tar the instances' logcat and other logs into specified output_file. |
| |
| Args: |
| output_file: String, the output tar file path |
| ssh_user: The ssh user name |
| ssh_rsa_path: The ssh rsa key path |
| """ |
| with utils.TempDir() as tempdir: |
| file_dict = {} |
| if getattr(self._device_factory, "LOG_FILES", None): |
| file_dict = self.PullLogs( |
| self._device_factory.LOG_FILES, tempdir, user=ssh_user, |
| ssh_rsa_path=ssh_rsa_path) |
| # If the device is auto-connected, get adb logcat |
| for file_path, file_name in self._CollectAdbLogcats( |
| tempdir).items(): |
| file_dict[file_path] = file_name |
| utils.MakeTarFile(file_dict, output_file) |
| |
| @property |
| def devices(self): |
| """Returns a list of devices in the pool. |
| |
| Returns: |
| A list of devices in the pool. |
| """ |
| return self._devices |
| |
| # TODO: Delete unused-argument when b/119614469 is resolved. |
| # pylint: disable=unused-argument |
| # pylint: disable=too-many-locals |
| def CreateDevices(command, cfg, device_factory, num, |
| report_internal_ip=False, autoconnect=False, |
| vnc_port=None, adb_port=None, |
| serial_log_file=None, logcat_file=None): |
| """Create a set of devices using the given factory. |
| |
| Main jobs in create devices. |
| 1. Create GCE instance: Launch instance in GCP(Google Cloud Platform). |
| 2. Starting up AVD: Wait device boot up. |
| |
| Args: |
| command: The name of the command, used for reporting. |
| cfg: An AcloudConfig instance. |
| device_factory: A factory capable of producing a single device. |
| num: The number of devices to create. |
| report_internal_ip: Boolean to report the internal ip instead of |
| external ip. |
| vnc_port: (int) The VNC port to use for a remote instance. |
| adb_port: (int) The ADB port to use for a remote instance. |
| serial_log_file: String, the file path to tar the serial logs. |
| logcat_file: String, the file path to tar the logcats. |
| autoconnect: Boolean, whether to auto connect to device. |
| |
| Raises: |
| errors: Create instance fail. |
| |
| Returns: |
| A Report instance. |
| """ |
| reporter = report.Report(command=command) |
| try: |
| CreateSshKeyPairIfNecessary(cfg) |
| device_pool = DevicePool(device_factory) |
| device_pool.CreateDevices(num) |
| failures = device_pool.WaitForBoot() |
| if failures: |
| reporter.SetStatus(report.Status.BOOT_FAIL) |
| else: |
| reporter.SetStatus(report.Status.SUCCESS) |
| |
| # Collect logs |
| if serial_log_file: |
| device_pool.CollectSerialPortLogs( |
| serial_log_file, port=constants.DEFAULT_SERIAL_PORT) |
| # TODO(b/119614469): Refactor CollectLogcats into a utils lib and |
| # turn it on inside the reporting loop. |
| # if logcat_file: |
| # device_pool.CollectLogcats(logcat_file, ssh_user, ssh_rsa_path) |
| |
| # Write result to report. |
| for device in device_pool.devices: |
| ip = (device.ip.internal if report_internal_ip |
| else device.ip.external) |
| device_dict = { |
| "ip": ip, |
| "instance_name": device.instance_name |
| } |
| if autoconnect: |
| if (not vnc_port) or (not adb_port): |
| logger.error("vnc_port and adb_port must be specified to" |
| " use autoconnect") |
| forwarded_ports = utils.AutoConnect(ip, |
| cfg.ssh_private_key_path, |
| vnc_port, |
| adb_port, |
| getpass.getuser()) |
| device_dict[constants.VNC_PORT] = forwarded_ports.vnc_port |
| device_dict[constants.ADB_PORT] = forwarded_ports.adb_port |
| if device.instance_name in failures: |
| reporter.AddData(key="devices_failing_boot", value=device_dict) |
| reporter.AddError(str(failures[device.instance_name])) |
| else: |
| reporter.AddData(key="devices", value=device_dict) |
| except errors.DriverError as e: |
| reporter.AddError(str(e)) |
| reporter.SetStatus(report.Status.FAIL) |
| return reporter |
