Acloud create: upload local images to instance with lzop.
Bug: 129376163
Test: acloud create --local-image
acloud create --local-image image_zip_path
Change-Id: Ie09ddf286b16194dd48d3b2f25c7b8fb93e5f085
diff --git a/create/avd_spec_test.py b/create/avd_spec_test.py
index 48b8038..551eaa7 100644
--- a/create/avd_spec_test.py
+++ b/create/avd_spec_test.py
@@ -21,7 +21,6 @@
from acloud import errors
from acloud.create import avd_spec
-from acloud.create import create_common
from acloud.internal import constants
from acloud.internal.lib import driver_test_lib
from acloud.internal.lib import utils
@@ -43,8 +42,6 @@
# pylint: disable=protected-access
def testProcessLocalImageArgs(self):
"""Test process args.local_image."""
- self.Patch(create_common, "ZipCFImageFiles",
- return_value="/path/cf_x86_phone-img-eng.user.zip")
self.Patch(glob, "glob", return_value=["fake.img"])
expected_image_artifact = "/path/cf_x86_phone-img-eng.user.zip"
expected_image_dir = "/path-to-image-dir"
diff --git a/create/create_common.py b/create/create_common.py
index 3480228..565a6d9 100644
--- a/create/create_common.py
+++ b/create/create_common.py
@@ -17,20 +17,7 @@
from __future__ import print_function
-import glob
-import logging
-import os
-import tempfile
-import time
-import zipfile
-
from acloud import errors
-from acloud.internal import constants
-from acloud.internal.lib import utils
-
-logger = logging.getLogger(__name__)
-
-_ACLOUD_IMAGE_ZIP_POSTFIX = "-local-img-%s.zip"
def ParseHWPropertyArgs(dict_str, item_separator=",", key_value_separator=":"):
@@ -66,42 +53,3 @@
hw_dict[key.strip()] = value.strip()
return hw_dict
-
-
-@utils.TimeExecute(function_description="Compressing images")
-def ZipCFImageFiles(basedir):
- """Zip images from basedir.
-
- TODO(b/129376163):Use lzop for fast sparse image upload when host image
- support it.
-
- Args:
- basedir: String of local images path.
-
- Return:
- Strings of zipped image path.
- """
- tmp_folder = os.path.join(tempfile.gettempdir(),
- constants.TEMP_ARTIFACTS_FOLDER)
- if not os.path.exists(tmp_folder):
- os.makedirs(tmp_folder)
- archive_name = "%s-local-%d.zip" % (os.environ.get(constants.ENV_BUILD_TARGET),
- int(time.time()))
- archive_file = os.path.join(tmp_folder, archive_name)
- if os.path.exists(archive_file):
- raise errors.ZipImageError("This file shouldn't exist, please delete: %s"
- % archive_file)
-
- zip_file = zipfile.ZipFile(archive_file, 'w', zipfile.ZIP_DEFLATED,
- allowZip64=True)
- required_files = ([os.path.join(basedir, "android-info.txt")] +
- glob.glob(os.path.join(basedir, "*.img")))
- logger.debug("archiving images: %s", required_files)
-
- for f in required_files:
- # Pass arcname arg to remove the directory structure.
- zip_file.write(f, arcname=os.path.basename(f))
-
- zip_file.close()
- logger.debug("zip images done:%s", archive_file)
- return archive_file
diff --git a/create/create_common_test.py b/create/create_common_test.py
index 2af3177..497dea0 100644
--- a/create/create_common_test.py
+++ b/create/create_common_test.py
@@ -13,19 +13,13 @@
# limitations under the License.
"""Tests for create_common."""
-import os
-import tempfile
-import time
import unittest
-import zipfile
from acloud import errors
from acloud.create import create_common
-from acloud.internal import constants
from acloud.internal.lib import driver_test_lib
-
class FakeZipFile(object):
"""Fake implementation of ZipFile()"""
@@ -65,26 +59,6 @@
result_dict = create_common.ParseHWPropertyArgs(args_str)
self.assertTrue(expected_dict == result_dict)
- def testZipCFImageFiles(self):
- """Test ZipCFImageFiles."""
- # Should raise error if zip file already exists
- fake_image_path = "/fake_image_dir/"
- self.Patch(os.path, "exists", return_value=True)
- self.Patch(os, "makedirs")
- self.assertRaises(errors.ZipImageError,
- create_common.ZipCFImageFiles,
- fake_image_path)
-
- # Test should get archive name by timestamp if zip file does not exist.
- self.Patch(zipfile, "ZipFile", return_value=FakeZipFile())
- self.Patch(os.path, "exists", return_value=False)
- self.Patch(os.environ, "get", return_value="fake_build_target")
- self.Patch(time, "time", return_value=12345)
- self.Patch(tempfile, "gettempdir", return_value="/fake_temp")
- self.assertEqual(create_common.ZipCFImageFiles(fake_image_path),
- "/fake_temp/%s/fake_build_target-local-12345.zip" %
- constants.TEMP_ARTIFACTS_FOLDER)
-
if __name__ == "__main__":
unittest.main()
diff --git a/create/local_image_remote_instance.py b/create/local_image_remote_instance.py
index e507fb4..7e428ef 100644
--- a/create/local_image_remote_instance.py
+++ b/create/local_image_remote_instance.py
@@ -21,13 +21,13 @@
from distutils.spawn import find_executable
import getpass
+import glob
import logging
import os
import subprocess
from acloud import errors
from acloud.create import base_avd_create
-from acloud.create import create_common
from acloud.internal import constants
from acloud.internal.lib import auth
from acloud.internal.lib import cvd_compute_client
@@ -92,7 +92,8 @@
self._SetAVDenv(_CVD_USER)
self._UploadArtifacts(_CVD_USER,
self._local_image_artifact,
- self._cvd_host_package_artifact)
+ self._cvd_host_package_artifact,
+ self._avd_spec.local_image_dir)
self._LaunchCvd(_CVD_USER, self._avd_spec.hw_property)
return instance
@@ -129,15 +130,16 @@
Override method from parent class.
build_target: The format is like "aosp_cf_x86_phone". We only get info
from the user build image file name. If the file name is
- not custom format (no "-"), We will use the original
- flavor as our build_target.
+ not custom format (no "-"), we will use $TARGET_PRODUCT
+ from environment variable as build_target.
Returns:
A string, representing instance name.
"""
- image_name = os.path.basename(self._local_image_artifact)
- build_target = self._avd_spec.flavor if "-" not in image_name else image_name.split(
- "-")[0]
+ image_name = os.path.basename(
+ self._local_image_artifact) if self._local_image_artifact else ""
+ build_target = (os.environ.get(constants.ENV_BUILD_TARGET) if "-" not
+ in image_name else image_name.split("-")[0])
instance = self._compute_client.GenerateInstanceName(
build_target=build_target, build_id=_USER_BUILD)
# Create an instance from Stable Host Image
@@ -171,23 +173,45 @@
logger.debug("remote_cmd:\n %s", remote_cmd)
self._ShellCmdWithRetry(self._ssh_cmd + remote_cmd)
- @utils.TimeExecute(function_description="Uploading local image")
+ @utils.TimeExecute(function_description="Processing and uploading local images")
def _UploadArtifacts(self,
cvd_user,
- local_image_artifact,
- cvd_host_package_artifact):
- """Upload local image and avd local host package to instance.
+ local_image_zip,
+ cvd_host_package_artifact,
+ images_dir):
+ """Upload local images and avd local host package to instance.
+
+ There are two ways to upload local images.
+ 1. Using local image zip, it would be decompressed by install_zip.sh.
+ 2. Using local image directory, this directory contains all images.
+ Images are compressed/decompressed by lzop during upload process.
Args:
- cvd_user: A string, user upload the artifacts to instance.
- local_image_artifact: A string, path to local image.
- cvd_host_package_artifact: A string, path to cvd host package.
+ cvd_user: String, user upload the artifacts to instance.
+ local_image_zip: String, path to zip of local images which
+ build from 'm dist'.
+ cvd_host_package_artifact: String, path to cvd host package.
+ images_dir: String, directory of local images which build
+ from 'm'.
"""
- # TODO(b/129376163) Use lzop for fast sparse image upload
- remote_cmd = ("\"sudo su -c '/usr/bin/install_zip.sh .' - '%s'\" < %s" %
- (cvd_user, local_image_artifact))
- logger.debug("remote_cmd:\n %s", remote_cmd)
- self._ShellCmdWithRetry(self._ssh_cmd + remote_cmd)
+ # TODO(b/133461252) Deprecate acloud create with local image zip.
+ # Upload local image zip file
+ if local_image_zip:
+ remote_cmd = ("\"sudo su -c '/usr/bin/install_zip.sh .' - '%s'\" < %s"
+ % (cvd_user, local_image_zip))
+ logger.debug("remote_cmd:\n %s", remote_cmd)
+ self._ShellCmdWithRetry(self._ssh_cmd + remote_cmd)
+ else:
+ # Compress image files for faster upload.
+ artifact_files = [os.path.basename(image) for image in glob.glob(
+ os.path.join(images_dir, "*.img"))]
+ cmd = ("tar -cf - --lzop -S -C {images_dir} {artifact_files} | "
+ "{ssh_cmd} -- tar -xf - --lzop -S".format(
+ images_dir=images_dir,
+ artifact_files=" ".join(artifact_files),
+ ssh_cmd=self._ssh_cmd))
+ logger.debug("cmd:\n %s", cmd)
+ self._ShellCmdWithRetry(cmd)
# host_package
remote_cmd = ("\"sudo su -c 'tar -x -z -f -' - '%s'\" < %s" %
@@ -272,16 +296,9 @@
no_prompts: Boolean, True to skip all prompts.
"""
self.cvd_host_package_artifact = self.VerifyHostPackageArtifactsExist()
-
- if avd_spec.local_image_artifact:
- local_image_artifact = avd_spec.local_image_artifact
- else:
- local_image_artifact = create_common.ZipCFImageFiles(
- avd_spec.local_image_dir)
-
device_factory = RemoteInstanceDeviceFactory(
avd_spec,
- local_image_artifact,
+ avd_spec.local_image_artifact,
self.cvd_host_package_artifact)
report = common_operations.CreateDevices(
"create_cf", avd_spec.cfg, device_factory, avd_spec.num,
diff --git a/create/local_image_remote_instance_test.py b/create/local_image_remote_instance_test.py
index 1509d9f..ce898b2 100644
--- a/create/local_image_remote_instance_test.py
+++ b/create/local_image_remote_instance_test.py
@@ -30,7 +30,6 @@
from acloud import errors
from acloud.create import avd_spec
-from acloud.create import create_common
from acloud.create import local_image_remote_instance
from acloud.internal import constants
from acloud.internal.lib import auth
@@ -103,13 +102,12 @@
self.assertEqual(factory._ShellCmdWithRetry("fake cmd"), True)
# pylint: disable=protected-access
+ @mock.patch.dict(os.environ, {constants.ENV_BUILD_TARGET:'fake-target'})
def testCreateGceInstanceName(self):
"""test create gce instance."""
self.Patch(utils, "GetBuildEnvironmentVariable",
return_value="test_environ")
self.Patch(glob, "glob", return_vale=["fake.img"])
- self.Patch(create_common, "ZipCFImageFiles",
- return_value="/fake/aosp_cf_x86_phone-img-eng.username.zip")
# Mock uuid
args = mock.MagicMock()
args.config_file = ""
@@ -130,12 +128,22 @@
fake_host_package_name)
self.assertEqual(factory._CreateGceInstance(), "ins-1234-userbuild-aosp-cf-x86-phone")
+ # Can't get target name from zip file name.
fake_image_name = "/fake/aosp_cf_x86_phone.username.zip"
factory = local_image_remote_instance.RemoteInstanceDeviceFactory(
fake_avd_spec,
fake_image_name,
fake_host_package_name)
- self.assertEqual(factory._CreateGceInstance(), "ins-1234-userbuild-phone")
+ self.assertEqual(factory._CreateGceInstance(), "ins-1234-userbuild-fake-target")
+
+ # No image zip path, it uses local build images.
+ fake_image_name = ""
+ factory = local_image_remote_instance.RemoteInstanceDeviceFactory(
+ fake_avd_spec,
+ fake_image_name,
+ fake_host_package_name)
+ self.assertEqual(factory._CreateGceInstance(), "ins-1234-userbuild-fake-target")
+
if __name__ == "__main__":
unittest.main()
