Use docker for all crosvm build modes
-- Move GCE and ARM-board build to containers
-- Use debian-10 containers
-- Known limitations: --docker_arch=aarch64 builds do not work on GCE;
--docker_arch=x86_64 builds do not work on ARM
To build using GCE:
./device/google/cuttlefish_vmm/rebuild.sh --gce
To build using a previously-set-up GCE instance:
./device/google/cuttlefish_vmm/rebuild.sh --gce --reuse
To build locally:
./device/google/cuttlefish_vmm/rebuild.sh --docker
To build locally for aarch64:
./device/google/cuttlefish_vmm/rebuild.sh --docker \
--docker_arch aarch64
To reuse a previously-set-up local docker image:
./device/google/cuttlefish_vmm/rebuild.sh --docker --reuse
To reuse a previously-set-up local docker image for aarch64:
./device/google/cuttlefish_vmm/rebuild.sh --docker --reuse \
--docker_arch aarch64
To build using an ARM board reachable at cf@192.168.0.6:
./device/google/cuttlefish_vmm/rebuild.sh \
--arm --arm_instance 192.168.0.6 --arm_user cf \
--docker_arch aarch64
To build using the same previously-set-up ARM board:
./device/google/cuttlefish_vmm/rebuild.sh \
--arm --arm_instance 192.168.0.6 --arm_user cf \
--docker_arch aarch64 \
--reuse
Bug: 148642775 Clean up the crosvm build
Test: built with GCE, ARM board, and locally
Change-Id: I75c4109be5ee72544580cc6a4374d1fdcc47d07f
Signed-off-by: Iliyan Malchev <malchev@google.com>
diff --git a/.dockerignore b/.dockerignore
index f33aa59..178260b 100644
--- a/.dockerignore
+++ b/.dockerignore
@@ -5,3 +5,4 @@
!rebuild-internal.sh
!x86_64-linux-gnu/manifest.xml
!aarch64-linux-gnu/manifest.xml
+!custom.xml
diff --git a/Dockerfile b/Dockerfile
index 86c6827..24142d6 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -34,6 +34,7 @@
COPY --chown=$USER x86_64-linux-gnu/manifest.xml /static/x86_64-linux-gnu/manifest.xml
COPY --chown=$USER aarch64-linux-gnu/manifest.xml /static/aarch64-linux-gnu/manifest.xml
+COPY --chown=$USER custom.xml /static/custom.xml
COPY --chown=$USER rebuild-internal.sh /static/rebuild-internal.sh
RUN TOOLS_DIR=/static/tools /static/rebuild-internal.sh install_packages
diff --git a/rebuild.sh b/rebuild.sh
index aa0edf3..59f6404 100755
--- a/rebuild.sh
+++ b/rebuild.sh
@@ -27,7 +27,7 @@
DEFINE_string docker_container "docker_vmm" "Name of docker container to create"
DEFINE_string docker_source "" "Path to sources checked out using manifest"
DEFINE_string docker_working "" "Path to working directory"
-DEFINE_string docker_output "${ANDROID_BUILD_TOP}/device/google/cuttlefish_vmm/${FLAGS_docker_arch}-linux-gnu" "Output directory"
+DEFINE_string docker_output "" "Output directory (when --docker is specified)"
DEFINE_string docker_user "${USER}" "Docker-container user"
DEFINE_string docker_uid "${UID}" "Docker-container user ID"
@@ -43,11 +43,7 @@
# Common options
-# The /./ pattern in the path defition below is interpreted by rsync; do not
-# remove it!
-DEFINE_string manifest \
- "${ANDROID_BUILD_TOP}/device/google/cuttlefish_vmm/./$(uname -m)-linux-gnu/manifest.xml" \
- "manifest to use for the build"
+DEFINE_string manifest "" "Path to custom manifest to use for the build"
DEFINE_boolean reuse false "Set to true to reuse a previously-set-up instance."
DEFINE_boolean reuse_resync false "Reuse a previously-set-up instance, but clean and re-sync the sources. Overrides --reuse if both are specified."
@@ -70,6 +66,14 @@
echo Options --docker_container must not be empty 1>&2
fail=1
fi
+ if [[ -z "${FLAGS_docker_user}" ]]; then
+ echo Options --docker_user must not be empty 1>&2
+ fail=1
+ fi
+ if [[ -z "${FLAGS_docker_uid}" ]]; then
+ echo Options --docker_uid must not be empty 1>&2
+ fail=1
+ fi
# Volume mapping are specified only when a container is created. With
# --reuse, an already-created persistent container is reused, which implies
# that we cannot change the volume maps. For non-persistent containers, we
@@ -78,11 +82,15 @@
# that we passed when we ran the non-persistent continer the first time.
if [[ ${_reuse} -eq 1 && ${FLAGS_docker_persistent} -eq ${FLAGS_TRUE} ]]; then
if [ -n "${FLAGS_docker_source}" ]; then
- echo Option --docker_source may not be specified with --reuse 1>&2
+ echo Option --docker_source may not be specified with --reuse and --docker_persistent 1>&2
fail=1
fi
if [ -n "${FLAGS_docker_working}" ]; then
- echo Option --docker_working may not be specified with --reuse 1>&2
+ echo Option --docker_working may not be specified with --reuse and --docker_persistent 1>&2
+ fail=1
+ fi
+ if [ -n "${FLAGS_docker_output}" ]; then
+ echo Option --docker_output may not be specified with --reuse and --docker_persistent 1>&2
fail=1
fi
fi
@@ -100,30 +108,34 @@
fail=1
;;
esac
- if [[ -z "${FLAGS_docker_user}" ]]; then
- echo Options --docker_user must not be empty 1>&2
- fail=1
- fi
- if [[ -z "${FLAGS_docker_uid}" ]]; then
- echo Options --docker_uid must not be empty 1>&2
- fail=1
- fi
if [[ "${fail}" -ne 0 ]]; then
exit "${fail}"
fi
local -i _persistent=0
- if [[ ${FLAGS_persistent} -eq ${FLAGS_TRUE} ]]; then
+ if [[ ${FLAGS_docker_persistent} -eq ${FLAGS_TRUE} ]]; then
_persistent=1
fi
+
local -i _build_image=0
if [[ ${FLAGS_docker_build_image} -eq ${FLAGS_TRUE} ]]; then
_build_image=1
fi
- local _docker_output="${FLAGS_docker_output}"
- if [[ "${_docker_output}" == "${ANDROID_BUILD_TOP}/device/google/cuttlefish_vmm/$(uname -m)-linux-gnu" && \
- "$(uname -m)" != ${FLAGS_docker_arch} ]]; then
+
+ local _docker_output=""
+ if [ -z "${FLAGS_docker_output}" ]; then
_docker_output="${ANDROID_BUILD_TOP}/device/google/cuttlefish_vmm/${FLAGS_docker_arch}-linux-gnu"
+ else
+ _docker_output="${FLAGS_docker_output}"
fi
+
+ local _temp="$(mktemp -d)"
+ rsync -avR "${relative_source_files[@]/#/${DIR}/./}" "${_temp}"
+ if [ -n "${custom_manifest}" ]; then
+ cp "${custom_manifest}" "${_temp}"/custom.xml
+ else
+ touch "${_temp}"/custom.xml
+ fi
+
${DIR}/rebuild-docker.sh "${FLAGS_docker_image}" \
"${FLAGS_docker_container}" \
"${FLAGS_docker_arch}" \
@@ -135,13 +147,16 @@
"x${_docker_output}" \
"${_reuse}" \
"${_build_image}" \
- "${DIR}/Dockerfile" \
- "${DIR}" \
+ "${_temp}/Dockerfile" \
+ "${_temp}" \
"${#docker_flags[@]}" "${docker_flags[@]}" \
"${#_prepare_source[@]}" "${_prepare_source[@]}"
+
+ rm -rf "${_temp}"
}
function build_on_gce() {
+ check_common_docker_options
if [[ -z "${FLAGS_gce_instance}" ]]; then
echo Must specify instance 1>&2
fail=1
@@ -161,18 +176,73 @@
if [ ${_reuse} -eq 0 ]; then
delete_instances=("${FLAGS_gce_instance}")
gcloud compute instances delete -q \
+ "${delete_instances[@]}" \
+ "${project_zone_flags[@]}" || \
+ echo Instance does not exist
+ gcloud compute images delete -q \
+ "${delete_instances[@]/%/-image}" \
+ --project "${FLAGS_gce_project}" || \
+ echo Image does not exist
+ gcloud compute disks delete -q \
+ "${delete_instances[@]/%/-disk}" \
+ "${project_zone_flags[@]}" || \
+ echo Disk does not exist
+
+ gcloud compute disks create \
+ "${delete_instances[@]/%/-disk}" \
"${project_zone_flags[@]}" \
- "${delete_instances[@]}" || \
- echo Not running
- gcloud compute instances create \
- "${project_zone_flags[@]}" \
- --boot-disk-size=200GB \
- --machine-type=n1-standard-4 \
- --image-family="${FLAGS_gce_source_image_family}" \
--image-project="${FLAGS_gce_source_image_project}" \
- "${FLAGS_gce_instance}"
- wait_for_instance "${FLAGS_gce_instance}"
+ --image-family="${FLAGS_gce_source_image_family}"
+ gcloud compute images create \
+ "${delete_instances[@]/%/-image}" \
+ --source-disk "${delete_instances[@]/%/-disk}" \
+ --project "${FLAGS_gce_project}" --source-disk-zone "${FLAGS_gce_zone}" \
+ --licenses "https://www.googleapis.com/compute/v1/projects/vm-options/global/licenses/enable-vmx"
+ gcloud compute instances create \
+ "${delete_instances[@]}" \
+ "${project_zone_flags[@]}" \
+ --image "${delete_instances[@]/%/-image}" \
+ --boot-disk-size=200GB \
+ --machine-type=n1-standard-8 \
+ --min-cpu-platform "Intel Skylake"
+
+ wait_for_instance "${FLAGS_gce_instance}" "${project_zone_flags[@]}"
+
+ # install docker
+ gcloud beta compute ssh "${SSH_FLAGS[@]}" \
+ "${project_zone_flags[@]}" \
+ "${FLAGS_gce_user}@${FLAGS_gce_instance}" -- \
+ 'curl -fsSL https://get.docker.com | /bin/bash'
+ gcloud beta compute ssh "${SSH_FLAGS[@]}" \
+ "${project_zone_flags[@]}" \
+ "${FLAGS_gce_user}@${FLAGS_gce_instance}" -- \
+ sudo usermod -aG docker "${FLAGS_gce_user}"
+
+ # beta for the --internal-ip flag that may be passed via SSH_FLAGS
+
+ gcloud beta compute ssh "${SSH_FLAGS[@]}" \
+ "${project_zone_flags[@]}" \
+ "${FLAGS_gce_user}@${FLAGS_gce_instance}" -- \
+ mkdir -p '$PWD/docker $PWD/docker/source $PWD/docker/working $PWD/docker/output'
+
+ tar czv -C "${DIR}" -f - "${relative_source_files[@]}" | \
+ gcloud beta compute ssh "${SSH_FLAGS[@]}" \
+ "${project_zone_flags[@]}" \
+ "${FLAGS_gce_user}@${FLAGS_gce_instance}" -- \
+ 'tar xzv -C ~/docker -f -'
+ if [ -n "${custom_manifest}" ]; then
+ gcloud beta compute scp "${SSH_FLAGS[@]}" \
+ "${project_zone_flags[@]}" \
+ "${custom_manifest}" \
+ "${FLAGS_gce_user}@${FLAGS_gce_instance}:~/docker/custom.xml"
+ else
+ gcloud beta compute ssh "${SSH_FLAGS[@]}" \
+ "${project_zone_flags[@]}" \
+ "${FLAGS_gce_user}@${FLAGS_gce_instance}" -- \
+ "touch ~/docker/custom.xml"
+ fi
fi
+
local _status=$(gcloud compute instances list \
--project="${FLAGS_gce_project}" \
--zones="${FLAGS_gce_zone}" \
@@ -182,32 +252,40 @@
echo "Instance ${FLAGS_gce_instance} is not running."
exit 1;
fi
- # beta for the --internal-ip flag that may be passed via SSH_FLAGS
- gcloud beta compute scp "${SSH_FLAGS[@]}" \
- "${project_zone_flags[@]}" \
- "${source_files[@]}" \
- "${FLAGS_gce_user}@${FLAGS_gce_instance}:"
- if [ ${_reuse} -eq 0 ]; then
- gcloud compute ssh "${SSH_FLAGS[@]}" \
- "${project_zone_flags[@]}" \
- "${FLAGS_gce_user}@${FLAGS_gce_instance}" -- \
- ./rebuild-internal.sh install_packages
+
+ local -i _persistent=0
+ if [[ ${FLAGS_docker_persistent} -eq ${FLAGS_TRUE} ]]; then
+ _persistent=1
fi
- if [ ${_reuse} -eq 0 ]; then
- gcloud compute ssh "${SSH_FLAGS[@]}" \
- "${project_zone_flags[@]}" \
- "${FLAGS_gce_user}@${FLAGS_gce_instance}" -- \
- ./rebuild-internal.sh "${gce_flags[@]}" ${_prepare_source[@]} '$(uname -m)_build'
- else
- gcloud compute ssh "${SSH_FLAGS[@]}" \
- "${project_zone_flags[@]}" \
- "${FLAGS_gce_user}@${FLAGS_gce_instance}" -- \
- ./rebuild-internal.sh "${gce_flags[@]}" ${_prepare_source[@]} '$(uname -m)_retry'
+ local -i _build_image=0
+ if [[ ${FLAGS_docker_build_image} -eq ${FLAGS_TRUE} ]]; then
+ _build_image=1
fi
- gcloud beta compute scp --recurse "${SSH_FLAGS[@]}" \
- "${project_zone_flags[@]}" \
- "${FLAGS_gce_user}@${FLAGS_gce_instance}":x86_64-linux-gnu \
- "${ANDROID_BUILD_TOP}/device/google/cuttlefish_vmm"
+ gcloud beta compute ssh "${SSH_FLAGS[@]}" \
+ "${project_zone_flags[@]}" \
+ "${FLAGS_gce_user}@${FLAGS_gce_instance}" -- \
+ ./docker/rebuild-docker.sh "${FLAGS_docker_image}" \
+ "${FLAGS_docker_container}" \
+ "${FLAGS_docker_arch}" \
+ '${USER}' \
+ '${UID}' \
+ "${_persistent}" \
+ 'x$PWD/docker/source' \
+ 'x$PWD/docker/working' \
+ 'x$PWD/docker/output' \
+ "${_reuse}" \
+ "${_build_image}" \
+ '~/docker/Dockerfile' \
+ '~/docker/' \
+ "${#docker_flags[@]}" "${docker_flags[@]}" \
+ "${#_prepare_source[@]}" "${_prepare_source[@]}"
+
+ gcloud beta compute ssh "${SSH_FLAGS[@]}" \
+ "${project_zone_flags[@]}" \
+ "${FLAGS_gce_user}@${FLAGS_gce_instance}" --command \
+ 'tar czv -C $PWD/docker/output -f - $(find $PWD/docker/output -printf "%P\n")' | \
+ tar xzv -C ${DIR}/${FLAGS_docker_arch}-linux-gnu -f -
+
gcloud compute disks describe \
"${project_zone_flags[@]}" "${FLAGS_gce_instance}" | \
grep ^sourceImage: > "${DIR}"/x86_64-linux-gnu/builder_image.txt
@@ -235,11 +313,18 @@
rm -rf '$PWD/docker'
fi
rsync -avR -e ssh \
- "${source_files[@]}" \
+ "${relative_source_files[@]/#/${DIR}/./}" \
"${FLAGS_arm_user}@${FLAGS_arm_instance}:~/docker/"
+ if [ -n "${custom_manifest}" ]; then
+ scp "${custom_manifest}" "${FLAGS_arm_user}@${FLAGS_arm_instance}":~/docker/custom.xml
+ else
+ ssh -t "${FLAGS_arm_user}@${FLAGS_arm_instance}" -- \
+ "touch ~/docker/custom.xml"
+ fi
+
local -i _persistent=0
- if [[ ${FLAGS_persistent} -eq ${FLAGS_TRUE} ]]; then
+ if [[ ${FLAGS_docker_persistent} -eq ${FLAGS_TRUE} ]]; then
_persistent=1
fi
local -i _build_image=0
@@ -273,18 +358,14 @@
set -o errexit
set -x
fail=0
- # The /./ patterns in the path defitions below are interpreted by rsync; do not
- # remove them!
- source_files=("${DIR}"/./rebuild-docker.sh \
- "${DIR}"/./rebuild-internal.sh \
- "${DIR}"/./Dockerfile \
- "${DIR}"/./x86_64-linux-gnu/manifest.xml \
- "${DIR}"/./aarch64-linux-gnu/manifest.xml \
- "${DIR}"/./.dockerignore)
+ relative_source_files=(rebuild-docker.sh \
+ rebuild-internal.sh \
+ Dockerfile \
+ x86_64-linux-gnu/manifest.xml \
+ aarch64-linux-gnu/manifest.xml \
+ .dockerignore)
# These must match the definitions in the Dockerfile
docker_flags=("-eSOURCE_DIR=/source" "-eWORKING_DIR=/working" "-eOUTPUT_DIR=/output" "-eTOOLS_DIR=/static/tools")
- gce_flags=()
- arm_flags=()
if [[ $(( $((${FLAGS_gce}==${FLAGS_TRUE})) + $((${FLAGS_arm}==${FLAGS_TRUE})) + $((${FLAGS_docker}==${FLAGS_TRUE})) )) > 1 ]]; then
echo You may specify only one of --gce, --docker, or --arm 1>&2
@@ -296,16 +377,23 @@
echo custom manifest not found: ${FLAGS_manifest} 1>&1
exit 2
fi
- source_files+=("${FLAGS_manifest}")
- docker_flags+=("-eCUSTOM_MANIFEST=/static/${FLAGS_docker_arch}-linux-gnu/$(basename ${FLAGS_manifest})")
- gce_flags+=("CUSTOM_MANIFEST=/home/${FLAGS_gce_user}/$(basename "${FLAGS_manifest}")")
- arm_flags+=("CUSTOM_MANIFEST=/home/${FLAGS_arm_user}/$(basename "${FLAGS_manifest}")")
+ custom_manifest="${FLAGS_manifest}"
+ docker_flags+=("-eCUSTOM_MANIFEST=/static/custom.xml")
+ else
+ custom_manifest="${DIR}/${FLAGS_docker_arch}-linux-gnu/manifest.xml"
+ docker_flags+=("-eCUSTOM_MANIFEST=/static/${FLAGS_docker_arch}-linux-gnu/manifest.xml")
fi
local -a _prepare_source=(setup_env fetch_source);
local -i _reuse=0
if [[ ${FLAGS_reuse} -eq ${FLAGS_TRUE} ]]; then
# neither install packages, nor sync sources; skip to building them
_prepare_source=(setup_env)
+ # unless you're setting up a non-persistent container and --docker_source is
+ # the empty string; in this case, --reuse implies --reuse_resync
+ if [[ "${FLAGS_docker_persistent}" -eq ${FLAGS_FALSE} && \
+ -z "${FLAGS_docker_source}" ]]; then
+ _prepare_source+=(resync_source)
+ fi
_reuse=1
fi
if [[ ${FLAGS_reuse_resync} -eq ${FLAGS_TRUE} ]]; then
