| # Copyright 2019 - The Android Open Source Project |
| # |
| # Licensed under the Apache License, Version 2.0 (the "License"); |
| # you may not use this file except in compliance with the License. |
| # You may obtain a copy of the License at |
| # |
| # http://www.apache.org/licenses/LICENSE-2.0 |
| # |
| # Unless required by applicable law or agreed to in writing, software |
| # distributed under the License is distributed on an "AS IS" BASIS, |
| # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| # See the License for the specific language governing permissions and |
| # limitations under the License. |
| |
| """RemoteInstanceDeviceFactory provides basic interface to create a cuttlefish |
| device factory.""" |
| |
| import glob |
| import logging |
| import os |
| import shutil |
| import subprocess |
| import tempfile |
| |
| from acloud import errors |
| from acloud.internal import constants |
| from acloud.internal.lib import utils |
| from acloud.internal.lib import ssh |
| from acloud.public.actions import gce_device_factory |
| |
| |
| logger = logging.getLogger(__name__) |
| _ALL_FILES = "*" |
| # bootloader and kernel are files required to launch AVD. |
| _BOOTLOADER = "bootloader" |
| _KERNEL = "kernel" |
| _ARTIFACT_FILES = ["*.img", _BOOTLOADER, _KERNEL] |
| _HOME_FOLDER = os.path.expanduser("~") |
| |
| |
| class RemoteInstanceDeviceFactory(gce_device_factory.GCEDeviceFactory): |
| """A class that can produce a cuttlefish device. |
| |
| Attributes: |
| avd_spec: AVDSpec object that tells us what we're going to create. |
| cfg: An AcloudConfig instance. |
| local_image_artifact: A string, path to local image. |
| cvd_host_package_artifact: A string, path to cvd host package. |
| report_internal_ip: Boolean, True for the internal ip is used when |
| connecting from another GCE instance. |
| credentials: An oauth2client.OAuth2Credentials instance. |
| compute_client: An object of cvd_compute_client.CvdComputeClient. |
| ssh: An Ssh object. |
| """ |
| def __init__(self, avd_spec, local_image_artifact=None, |
| cvd_host_package_artifact=None): |
| super().__init__(avd_spec, local_image_artifact) |
| self._cvd_host_package_artifact = cvd_host_package_artifact |
| |
| # pylint: disable=broad-except |
| def CreateInstance(self): |
| """Create a single configured cuttlefish device. |
| |
| GCE: |
| 1. Create gcp instance. |
| 2. Upload local built artifacts to remote instance or fetch build on |
| remote instance. |
| 3. Launch CVD. |
| |
| Remote host: |
| 1. Init remote host. |
| 2. Download the artifacts to local and upload the artifacts to host |
| 3. Launch CVD. |
| |
| Returns: |
| A string, representing instance name. |
| """ |
| if self._avd_spec.instance_type == constants.INSTANCE_TYPE_HOST: |
| instance = self._InitRemotehost() |
| self._ProcessRemoteHostArtifacts() |
| self._LaunchCvd(instance=instance, |
| decompress_kernel=None, |
| boot_timeout_secs=self._avd_spec.boot_timeout_secs) |
| else: |
| instance = self._CreateGceInstance() |
| # If instance is failed, no need to go next step. |
| if instance in self.GetFailures(): |
| return instance |
| try: |
| self._ProcessArtifacts(self._avd_spec.image_source) |
| self._LaunchCvd(instance=instance, |
| boot_timeout_secs=self._avd_spec.boot_timeout_secs) |
| except Exception as e: |
| self._SetFailures(instance, e) |
| |
| return instance |
| |
| def _InitRemotehost(self): |
| """Initialize remote host. |
| |
| Determine the remote host instance name, and activate ssh. It need to |
| get the IP address in the common_operation. So need to pass the IP and |
| ssh to compute_client. |
| |
| build_target: The format is like "aosp_cf_x86_phone". We only get info |
| from the user build image file name. If the file name is |
| not custom format (no "-"), we will use $TARGET_PRODUCT |
| from environment variable as build_target. |
| |
| Returns: |
| A string, representing instance name. |
| """ |
| image_name = os.path.basename( |
| self._local_image_artifact) if self._local_image_artifact else "" |
| build_target = (os.environ.get(constants.ENV_BUILD_TARGET) if "-" not |
| in image_name else image_name.split("-")[0]) |
| build_id = self._USER_BUILD |
| if self._avd_spec.image_source == constants.IMAGE_SRC_REMOTE: |
| build_id = self._avd_spec.remote_image[constants.BUILD_ID] |
| |
| instance = "%s-%s-%s-%s" % (constants.INSTANCE_TYPE_HOST, |
| self._avd_spec.remote_host, |
| build_id, build_target) |
| ip = ssh.IP(ip=self._avd_spec.remote_host) |
| self._ssh = ssh.Ssh( |
| ip=ip, |
| user=self._avd_spec.host_user, |
| ssh_private_key_path=(self._avd_spec.host_ssh_private_key_path or |
| self._cfg.ssh_private_key_path), |
| extra_args_ssh_tunnel=self._cfg.extra_args_ssh_tunnel, |
| report_internal_ip=self._report_internal_ip) |
| self._compute_client.InitRemoteHost( |
| self._ssh, ip, self._avd_spec.host_user) |
| return instance |
| |
| @utils.TimeExecute(function_description="Downloading Android Build artifact") |
| def _DownloadArtifacts(self, extract_path): |
| """Download the CF image artifacts and process them. |
| |
| - Download images from the Android Build system. |
| - Download cvd host package from the Android Build system. |
| |
| Args: |
| extract_path: String, a path include extracted files. |
| |
| Raises: |
| errors.GetRemoteImageError: Fails to download rom images. |
| """ |
| cfg = self._avd_spec.cfg |
| build_id = self._avd_spec.remote_image[constants.BUILD_ID] |
| build_branch = self._avd_spec.remote_image[constants.BUILD_BRANCH] |
| build_target = self._avd_spec.remote_image[constants.BUILD_TARGET] |
| |
| # Download images with fetch_cvd |
| fetch_cvd = os.path.join(extract_path, constants.FETCH_CVD) |
| self._compute_client.build_api.DownloadFetchcvd(fetch_cvd, |
| cfg.fetch_cvd_version) |
| fetch_cvd_build_args = self._compute_client.build_api.GetFetchBuildArgs( |
| build_id, build_branch, build_target, |
| self._avd_spec.system_build_info.get(constants.BUILD_ID), |
| self._avd_spec.system_build_info.get(constants.BUILD_BRANCH), |
| self._avd_spec.system_build_info.get(constants.BUILD_TARGET), |
| self._avd_spec.kernel_build_info.get(constants.BUILD_ID), |
| self._avd_spec.kernel_build_info.get(constants.BUILD_BRANCH), |
| self._avd_spec.kernel_build_info.get(constants.BUILD_TARGET), |
| self._avd_spec.bootloader_build_info.get(constants.BUILD_ID), |
| self._avd_spec.bootloader_build_info.get(constants.BUILD_BRANCH), |
| self._avd_spec.bootloader_build_info.get(constants.BUILD_TARGET)) |
| creds_cache_file = os.path.join(_HOME_FOLDER, cfg.creds_cache_file) |
| fetch_cvd_cert_arg = self._compute_client.build_api.GetFetchCertArg( |
| creds_cache_file) |
| fetch_cvd_args = [fetch_cvd, "-directory=%s" % extract_path, |
| fetch_cvd_cert_arg] |
| fetch_cvd_args.extend(fetch_cvd_build_args) |
| logger.debug("Download images command: %s", fetch_cvd_args) |
| try: |
| subprocess.check_call(fetch_cvd_args) |
| except subprocess.CalledProcessError as e: |
| raise errors.GetRemoteImageError("Fails to download images: %s" % e) |
| |
| def _ProcessRemoteHostArtifacts(self): |
| """Process remote host artifacts. |
| |
| - If images source is local, tool will upload images from local site to |
| remote host. |
| - If images source is remote, tool will download images from android |
| build to local and unzip it then upload to remote host, because there |
| is no permission to fetch build rom on the remote host. |
| """ |
| self._compute_client.SetStage(constants.STAGE_ARTIFACT) |
| if self._avd_spec.image_source == constants.IMAGE_SRC_LOCAL: |
| self._UploadLocalImageArtifacts( |
| self._local_image_artifact, self._cvd_host_package_artifact, |
| self._avd_spec.local_image_dir) |
| else: |
| try: |
| artifacts_path = tempfile.mkdtemp() |
| logger.debug("Extracted path of artifacts: %s", artifacts_path) |
| self._DownloadArtifacts(artifacts_path) |
| self._UploadRemoteImageArtifacts(artifacts_path) |
| finally: |
| shutil.rmtree(artifacts_path) |
| |
| def _ProcessArtifacts(self, image_source): |
| """Process artifacts. |
| |
| - If images source is local, tool will upload images from local site to |
| remote instance. |
| - If images source is remote, tool will download images from android |
| build to remote instance. Before download images, we have to update |
| fetch_cvd to remote instance. |
| |
| Args: |
| image_source: String, the type of image source is remote or local. |
| """ |
| if image_source == constants.IMAGE_SRC_LOCAL: |
| self._UploadLocalImageArtifacts(self._local_image_artifact, |
| self._cvd_host_package_artifact, |
| self._avd_spec.local_image_dir) |
| elif image_source == constants.IMAGE_SRC_REMOTE: |
| self._compute_client.UpdateFetchCvd() |
| self._FetchBuild(self._avd_spec) |
| |
| def _FetchBuild(self, avd_spec): |
| """Download CF artifacts from android build. |
| |
| Args: |
| avd_spec: AVDSpec object that tells us what we're going to create. |
| """ |
| self._compute_client.FetchBuild( |
| avd_spec.remote_image[constants.BUILD_ID], |
| avd_spec.remote_image[constants.BUILD_BRANCH], |
| avd_spec.remote_image[constants.BUILD_TARGET], |
| avd_spec.system_build_info[constants.BUILD_ID], |
| avd_spec.system_build_info[constants.BUILD_BRANCH], |
| avd_spec.system_build_info[constants.BUILD_TARGET], |
| avd_spec.kernel_build_info[constants.BUILD_ID], |
| avd_spec.kernel_build_info[constants.BUILD_BRANCH], |
| avd_spec.kernel_build_info[constants.BUILD_TARGET], |
| avd_spec.bootloader_build_info[constants.BUILD_ID], |
| avd_spec.bootloader_build_info[constants.BUILD_BRANCH], |
| avd_spec.bootloader_build_info[constants.BUILD_TARGET]) |
| |
| @utils.TimeExecute(function_description="Processing and uploading local images") |
| def _UploadLocalImageArtifacts(self, |
| local_image_zip, |
| cvd_host_package_artifact, |
| images_dir): |
| """Upload local images and avd local host package to instance. |
| |
| There are two ways to upload local images. |
| 1. Using local image zip, it would be decompressed by install_zip.sh. |
| 2. Using local image directory, this directory contains all images. |
| Images are compressed/decompressed by lzop during upload process. |
| |
| Args: |
| local_image_zip: String, path to zip of local images which |
| build from 'm dist'. |
| cvd_host_package_artifact: String, path to cvd host package. |
| images_dir: String, directory of local images which build |
| from 'm'. |
| """ |
| if local_image_zip: |
| remote_cmd = ("/usr/bin/install_zip.sh . < %s" % local_image_zip) |
| logger.debug("remote_cmd:\n %s", remote_cmd) |
| self._ssh.Run(remote_cmd) |
| else: |
| # Compress image files for faster upload. |
| try: |
| images_path = os.path.join(images_dir, "required_images") |
| with open(images_path, "r") as images: |
| artifact_files = images.read().splitlines() |
| except IOError: |
| # Older builds may not have a required_images file. In this case |
| # we fall back to *.img. |
| artifact_files = [] |
| for file_name in _ARTIFACT_FILES: |
| artifact_files.extend( |
| os.path.basename(image) for image in glob.glob( |
| os.path.join(images_dir, file_name))) |
| cmd = ("tar -cf - --lzop -S -C {images_dir} {artifact_files} | " |
| "{ssh_cmd} -- tar -xf - --lzop -S".format( |
| images_dir=images_dir, |
| artifact_files=" ".join(artifact_files), |
| ssh_cmd=self._ssh.GetBaseCmd(constants.SSH_BIN))) |
| logger.debug("cmd:\n %s", cmd) |
| ssh.ShellCmdWithRetry(cmd) |
| |
| # host_package |
| remote_cmd = ("tar -x -z -f - < %s" % cvd_host_package_artifact) |
| logger.debug("remote_cmd:\n %s", remote_cmd) |
| self._ssh.Run(remote_cmd) |
| |
| @utils.TimeExecute(function_description="Uploading remote image artifacts") |
| def _UploadRemoteImageArtifacts(self, images_dir): |
| """Upload remote image artifacts to instance. |
| |
| Args: |
| images_dir: String, directory of local artifacts downloaded by fetch_cvd. |
| """ |
| artifact_files = [ |
| os.path.basename(image) |
| for image in glob.glob(os.path.join(images_dir, _ALL_FILES)) |
| ] |
| # TODO(b/182259589): Refactor upload image command into a function. |
| cmd = ("tar -cf - --lzop -S -C {images_dir} {artifact_files} | " |
| "{ssh_cmd} -- tar -xf - --lzop -S".format( |
| images_dir=images_dir, |
| artifact_files=" ".join(artifact_files), |
| ssh_cmd=self._ssh.GetBaseCmd(constants.SSH_BIN))) |
| logger.debug("cmd:\n %s", cmd) |
| ssh.ShellCmdWithRetry(cmd) |
| |
| def _LaunchCvd(self, instance, decompress_kernel=None, |
| boot_timeout_secs=None): |
| """Launch CVD. |
| |
| Args: |
| instance: String, instance name. |
| boot_timeout_secs: Integer, the maximum time to wait for the |
| command to respond. |
| """ |
| # TODO(b/140076771) Support kernel image for local image mode. |
| self._compute_client.LaunchCvd( |
| instance, |
| self._avd_spec, |
| self._cfg.extra_data_disk_size_gb, |
| decompress_kernel, |
| boot_timeout_secs) |
| |
| def GetBuildInfoDict(self): |
| """Get build info dictionary. |
| |
| Returns: |
| A build info dictionary. None for local image case. |
| """ |
| if self._avd_spec.image_source == constants.IMAGE_SRC_LOCAL: |
| return None |
| build_info_dict = { |
| key: val for key, val in self._avd_spec.remote_image.items() if val} |
| |
| # kernel_target have default value "kernel". If user provide kernel_build_id |
| # or kernel_branch, then start to process kernel image. |
| if (self._avd_spec.kernel_build_info[constants.BUILD_ID] |
| or self._avd_spec.kernel_build_info[constants.BUILD_BRANCH]): |
| build_info_dict.update( |
| {"kernel_%s" % key: val |
| for key, val in self._avd_spec.kernel_build_info.items() if val} |
| ) |
| build_info_dict.update( |
| {"system_%s" % key: val |
| for key, val in self._avd_spec.system_build_info.items() if val} |
| ) |
| build_info_dict.update( |
| {"bootloader_%s" % key: val |
| for key, val in self._avd_spec.bootloader_build_info.items() if val} |
| ) |
| return build_info_dict |