blob: ee79c867c46fda7726c87560908cbb60b9127ebb [file] [log] [blame]
#!/bin/bash
#
# Copyright (C) 2018 The Android Open Source Project
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
set -e
set -u
SCRIPT_DIR=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd -P)
usage() {
echo -n "usage: $0 [-h] [-s bullseye|bullseye-cuttlefish|bullseye-rockpi|bullseye-server] "
echo -n "[-a i386|amd64|armhf|arm64] -k /path/to/kernel "
echo -n "-i /path/to/initramfs.gz [-d /path/to/dtb:subdir] "
echo "[-m http://mirror/debian] [-n rootfs|disk] [-r initrd] [-e] [-g]"
exit 1
}
mirror=http://ftp.debian.org/debian
embed_kernel_initrd_dtb=0
install_grub=0
suite=bullseye
arch=amd64
dtb_subdir=
initramfs=
kernel=
ramdisk=
disk=
dtb=
while getopts ":hs:a:m:n:r:k:i:d:eg" opt; do
case "${opt}" in
h)
usage
;;
s)
if [[ "${OPTARG%-*}" != "bullseye" ]]; then
echo "Invalid suite: ${OPTARG}" >&2
usage
fi
suite="${OPTARG}"
;;
a)
arch="${OPTARG}"
;;
m)
mirror="${OPTARG}"
;;
n)
disk="${OPTARG}"
;;
r)
ramdisk="${OPTARG}"
;;
k)
kernel="${OPTARG}"
;;
i)
initramfs="${OPTARG}"
;;
d)
dtb="${OPTARG%:*}"
if [ "${OPTARG#*:}" != "${dtb}" ]; then
dtb_subdir="${OPTARG#*:}/"
fi
;;
e)
embed_kernel_initrd_dtb=1
;;
g)
install_grub=1
;;
\?)
echo "Invalid option: ${OPTARG}" >&2
usage
;;
:)
echo "Invalid option: ${OPTARG} requires an argument" >&2
usage
;;
esac
done
# Disable Debian's "persistent" network device renaming
cmdline="net.ifnames=0 rw 8250.nr_uarts=2 PATH=/usr/sbin:/bin:/usr/bin"
cmdline="${cmdline} embed_kernel_initrd_dtb=${embed_kernel_initrd_dtb}"
cmdline="${cmdline} install_grub=${install_grub}"
case "${arch}" in
i386)
cmdline="${cmdline} console=ttyS0 exitcode=/dev/ttyS1"
machine="pc-i440fx-2.8,accel=kvm"
qemu="qemu-system-i386"
partguid="8303"
cpu="max"
;;
amd64)
cmdline="${cmdline} console=ttyS0 exitcode=/dev/ttyS1"
machine="pc-i440fx-2.8,accel=kvm"
qemu="qemu-system-x86_64"
partguid="8304"
cpu="max"
;;
armhf)
cmdline="${cmdline} console=ttyAMA0 exitcode=/dev/ttyS0"
machine="virt,gic-version=2"
qemu="qemu-system-arm"
partguid="8307"
cpu="cortex-a15"
;;
arm64)
cmdline="${cmdline} console=ttyAMA0 exitcode=/dev/ttyS0"
machine="virt,gic-version=2"
qemu="qemu-system-aarch64"
partguid="8305"
cpu="cortex-a53" # "max" is too slow
;;
*)
echo "Invalid arch: ${OPTARG}" >&2
usage
;;
esac
if [[ -z "${disk}" ]]; then
if [[ "${install_grub}" = "1" ]]; then
base_image_name=disk
else
base_image_name=rootfs
fi
disk="${base_image_name}.${arch}.${suite}.$(date +%Y%m%d)"
fi
disk=$(realpath "${disk}")
if [[ -z "${ramdisk}" ]]; then
ramdisk="initrd.${arch}.${suite}.$(date +%Y%m%d)"
fi
ramdisk=$(realpath "${ramdisk}")
if [[ -z "${kernel}" ]]; then
echo "$0: Path to kernel image must be specified (with '-k')"
usage
elif [[ ! -e "${kernel}" ]]; then
echo "$0: Kernel image not found at '${kernel}'"
exit 2
fi
if [[ -z "${initramfs}" ]]; then
echo "Path to initial ramdisk image must be specified (with '-i')"
usage
elif [[ ! -e "${initramfs}" ]]; then
echo "Initial ramdisk image not found at '${initramfs}'"
exit 3
fi
# Sometimes it isn't obvious when the script fails
failure() {
echo "Filesystem generation process failed." >&2
rm -f "${disk}" "${ramdisk}"
}
trap failure ERR
# Import the package list for this release
packages=$(cpp "${SCRIPT_DIR}/rootfs/${suite}.list" | grep -v "^#" | xargs | tr -s ' ' ',')
# For the debootstrap intermediates
tmpdir=$(mktemp -d)
tmpdir_remove() {
echo "Removing temporary files.." >&2
sudo rm -rf "${tmpdir}"
}
trap tmpdir_remove EXIT
workdir="${tmpdir}/_"
mkdir "${workdir}"
chmod 0755 "${workdir}"
sudo chown root:root "${workdir}"
# Run the debootstrap first
cd "${workdir}"
retries=5
while ! sudo debootstrap --arch="${arch}" --variant=minbase --include="${packages}" \
--foreign "${suite%-*}" . "${mirror}"; do
retries=$((${retries} - 1))
if [ ${retries} -le 0 ]; then
failure
exit 1
fi
echo "debootstrap failed - trying again - ${retries} retries left"
done
# Copy some bootstrapping scripts into the rootfs
sudo cp -a "${SCRIPT_DIR}"/rootfs/*.sh root/
sudo cp -a "${SCRIPT_DIR}"/rootfs/net_test.sh sbin/net_test.sh
sudo chown root:root sbin/net_test.sh
# Extract the ramdisk to bootstrap with to /
lz4 -lcd "${initramfs}" | sudo cpio -idum lib/modules/*
# Create /host, for the pivot_root and 9p mount use cases
sudo mkdir host
# debootstrap workaround: Run debootstrap in docker sometimes causes the
# /proc being a symlink in first stage. We need to fix the symlink to an empty
# directory.
if [ -L "${workdir}/proc" ]; then
echo "/proc in debootstrap 1st stage is a symlink. Fixed!"
sudo rm -f "${workdir}/proc"
sudo mkdir "${workdir}/proc"
fi
# Leave the workdir, to build the filesystem
cd -
# For the initial ramdisk, and later for the final rootfs
mount=$(mktemp -d)
mount_remove() {
rmdir "${mount}"
tmpdir_remove
}
trap mount_remove EXIT
# The initial ramdisk filesystem must be <=512M, or QEMU's -initrd
# option won't touch it
initrd=$(mktemp)
initrd_remove() {
rm -f "${initrd}"
mount_remove
}
trap initrd_remove EXIT
truncate -s 512M "${initrd}"
/sbin/mke2fs -F -t ext4 -L ROOT "${initrd}"
# Mount the new filesystem locally
sudo mount -o loop -t ext4 "${initrd}" "${mount}"
image_unmount() {
sudo umount "${mount}"
initrd_remove
}
trap image_unmount EXIT
# Copy the patched debootstrap results into the new filesystem
sudo cp -a "${workdir}"/* "${mount}"
sudo rm -rf "${workdir}"
# Unmount the initial ramdisk
sudo umount "${mount}"
trap initrd_remove EXIT
if [[ "${install_grub}" = 1 ]]; then
part_num=0
# $1 partition size
# $2 gpt partition type
# $3 partition name
# $4 bypass alignment checks (use on <1MB partitions only)
# $5 partition attribute bit to set
sgdisk() {
part_num=$((part_num+1))
[[ -n "${4:-}" ]] && prefix="-a1" || prefix=
[[ -n "${5:-}" ]] && suffix="-A:${part_num}:set:$5" || suffix=
/sbin/sgdisk ${prefix} \
"-n:${part_num}:$1" "-t:${part_num}:$2" "-c:${part_num}:$3" \
${suffix} "${disk}" >/dev/null 2>&1
}
# If there's a bootloader, we need to make space for the GPT header, GPT
# footer and EFI system partition (legacy boot is not supported)
# Keep this simple - modern gdisk reserves 1MB for the GPT header and
# assumes all partitions are 1MB aligned
truncate -s "$((1 + 128 + 10 * 1024 + 1))M" "${disk}"
/sbin/sgdisk --zap-all "${disk}" >/dev/null 2>&1
# On RockPi devices, steal a bit of space at the start of the disk for
# some special bootloader partitions. Some of these have to start/end
# at specific offsets as well
if [[ "${suite#*-}" = "rockpi" ]]; then
# See https://opensource.rock-chips.com/wiki_Boot_option
# Keep in sync with rootfs/*-rockpi.sh
sgdisk "64:8127" "8301" "idbloader" "true"
sgdisk "8128:+64" "8301" "uboot_env" "true"
sgdisk "8M:+4M" "8301" "uboot"
sgdisk "12M:+4M" "8301" "trust"
sgdisk "16M:+1M" "8301" "misc"
sgdisk "17M:+128M" "ef00" "esp" "" "0"
sgdisk "145M:0" "8305" "rootfs" "" "2"
system_partition="6"
rootfs_partition="7"
else
sgdisk "0:+128M" "ef00" "esp" "" "0"
sgdisk "0:0" "${partguid}" "rootfs" "" "2"
system_partition="1"
rootfs_partition="2"
fi
# Create an empty EFI system partition; it will be initialized later
system_partition_start=$(partx -g -o START -s -n "${system_partition}" "${disk}" | xargs)
system_partition_end=$(partx -g -o END -s -n "${system_partition}" "${disk}" | xargs)
system_partition_num_sectors=$((${system_partition_end} - ${system_partition_start} + 1))
system_partition_num_vfat_blocks=$((${system_partition_num_sectors} / 2))
/sbin/mkfs.vfat -n SYSTEM -F 16 --offset=${system_partition_start} "${disk}" ${system_partition_num_vfat_blocks} >/dev/null
# Copy the rootfs to just after the EFI system partition
rootfs_partition_start=$(partx -g -o START -s -n "${rootfs_partition}" "${disk}" | xargs)
rootfs_partition_end=$(partx -g -o END -s -n "${rootfs_partition}" "${disk}" | xargs)
rootfs_partition_num_sectors=$((${rootfs_partition_end} - ${rootfs_partition_start} + 1))
rootfs_partition_offset=$((${rootfs_partition_start} * 512))
rootfs_partition_size=$((${rootfs_partition_num_sectors} * 512))
dd if="${initrd}" of="${disk}" bs=512 seek="${rootfs_partition_start}" conv=fsync,notrunc 2>/dev/null
/sbin/e2fsck -p -f "${disk}"?offset=${rootfs_partition_offset} || true
disksize=$(stat -c %s "${disk}")
/sbin/resize2fs "${disk}"?offset=${rootfs_partition_offset} ${rootfs_partition_num_sectors}s
truncate -s "${disksize}" "${disk}"
/sbin/sgdisk -e "${disk}"
/sbin/e2fsck -p -f "${disk}"?offset=${rootfs_partition_offset} || true
/sbin/e2fsck -fy "${disk}"?offset=${rootfs_partition_offset} || true
else
# If there's no bootloader, the initrd is the disk image
cp -a "${initrd}" "${disk}"
truncate -s 10G "${disk}"
/sbin/e2fsck -p -f "${disk}" || true
/sbin/resize2fs "${disk}"
system_partition=
rootfs_partition="raw"
fi
# Create another fake block device for initrd.img writeout
raw_initrd=$(mktemp)
raw_initrd_remove() {
rm -f "${raw_initrd}"
initrd_remove
}
trap raw_initrd_remove EXIT
truncate -s 64M "${raw_initrd}"
# Get number of cores for qemu. Restrict the maximum value to 8.
qemucpucores=$(nproc)
if [[ ${qemucpucores} -gt 8 ]]; then
qemucpucores=8
fi
# Complete the bootstrap process using QEMU and the specified kernel
${qemu} -machine "${machine}" -cpu "${cpu}" -m 2048 >&2 \
-kernel "${kernel}" -initrd "${initrd}" -no-user-config -nodefaults \
-no-reboot -display none -nographic -serial stdio -parallel none \
-smp "${qemucpucores}",sockets="${qemucpucores}",cores=1,threads=1 \
-object rng-random,id=objrng0,filename=/dev/urandom \
-device virtio-rng-pci-non-transitional,rng=objrng0,id=rng0,max-bytes=1024,period=2000 \
-drive file="${disk}",format=raw,if=none,aio=threads,id=drive-virtio-disk0 \
-device virtio-blk-pci-non-transitional,scsi=off,drive=drive-virtio-disk0 \
-drive file="${raw_initrd}",format=raw,if=none,aio=threads,id=drive-virtio-disk1 \
-device virtio-blk-pci-non-transitional,scsi=off,drive=drive-virtio-disk1 \
-chardev file,id=exitcode,path=exitcode \
-device pci-serial,chardev=exitcode \
-append "root=/dev/ram0 ramdisk_size=524288 init=/root/stage1.sh ${cmdline}"
[[ -s exitcode ]] && exitcode=$(cat exitcode | tr -d '\r') || exitcode=2
rm -f exitcode
if [ "${exitcode}" != "0" ]; then
echo "Second stage debootstrap failed (err=${exitcode})"
exit "${exitcode}"
fi
# Fix up any issues from the unclean shutdown
if [[ ${rootfs_partition} = "raw" ]]; then
sudo e2fsck -p -f "${disk}" || true
else
rootfs_partition_start=$(partx -g -o START -s -n "${rootfs_partition}" "${disk}" | xargs)
rootfs_partition_end=$(partx -g -o END -s -n "${rootfs_partition}" "${disk}" | xargs)
rootfs_partition_num_sectors=$((${rootfs_partition_end} - ${rootfs_partition_start} + 1))
rootfs_partition_offset=$((${rootfs_partition_start} * 512))
rootfs_partition_tempfile2=$(mktemp)
dd if="${disk}" of="${rootfs_partition_tempfile2}" bs=512 skip=${rootfs_partition_start} count=${rootfs_partition_num_sectors}
e2fsck -p -f "${rootfs_partition_tempfile2}" || true
dd if="${rootfs_partition_tempfile2}" of="${disk}" bs=512 seek=${rootfs_partition_start} count=${rootfs_partition_num_sectors} conv=fsync,notrunc
rm -f "${rootfs_partition_tempfile2}"
e2fsck -fy "${disk}"?offset=${rootfs_partition_offset} || true
fi
if [[ -n "${system_partition}" ]]; then
system_partition_start=$(partx -g -o START -s -n "${system_partition}" "${disk}" | xargs)
system_partition_end=$(partx -g -o END -s -n "${system_partition}" "${disk}" | xargs)
system_partition_num_sectors=$((${system_partition_end} - ${system_partition_start} + 1))
system_partition_offset=$((${system_partition_start} * 512))
system_partition_size=$((${system_partition_num_sectors} * 512))
system_partition_tempfile=$(mktemp)
dd if="${disk}" of="${system_partition_tempfile}" bs=512 skip=${system_partition_start} count=${system_partition_num_sectors}
/sbin/fsck.vfat -a "${system_partition_tempfile}" || true
dd if="${system_partition_tempfile}" of="${disk}" bs=512 seek=${system_partition_start} count=${system_partition_num_sectors} conv=fsync,notrunc
rm -f "${system_partition_tempfile}"
fi
# New workdir for the initrd extraction
workdir="${tmpdir}/initrd"
mkdir "${workdir}"
chmod 0755 "${workdir}"
sudo chown root:root "${workdir}"
# Change into workdir to repack initramfs
cd "${workdir}"
# Process the initrd to remove kernel-specific metadata
kernel_version=$(basename $(lz4 -lcd "${raw_initrd}" | sudo cpio -idumv 2>&1 | grep usr/lib/modules/ - | head -n1))
lz4 -lcd "${raw_initrd}" | sudo cpio -idumv
sudo rm -rf usr/lib/modules
sudo mkdir -p usr/lib/modules
# Debian symlinks /usr/lib to /lib, but we'd prefer the other way around
# so that it more closely matches what happens in Android initramfs images.
# This enables 'cat ramdiskA.img ramdiskB.img >ramdiskC.img' to "just work".
sudo rm -f lib
sudo mv usr/lib lib
sudo ln -s /lib usr/lib
# Repack the ramdisk to the final output
find * | sudo cpio -H newc -o --quiet | lz4 -lc9 >"${ramdisk}"
# Pack another ramdisk with the combined artifacts, for boot testing
cat "${ramdisk}" "${initramfs}" >"${initrd}"
# Leave workdir to boot-test combined initrd
cd -
rootfs_partition_tempfile=$(mktemp)
# Mount the new filesystem locally
if [[ ${rootfs_partition} = "raw" ]]; then
sudo mount -o loop -t ext4 "${disk}" "${mount}"
else
rootfs_partition_start=$(partx -g -o START -s -n "${rootfs_partition}" "${disk}" | xargs)
rootfs_partition_offset=$((${rootfs_partition_start} * 512))
rootfs_partition_end=$(partx -g -o END -s -n "${rootfs_partition}" "${disk}" | xargs)
rootfs_partition_num_sectors=$((${rootfs_partition_end} - ${rootfs_partition_start} + 1))
dd if="${disk}" of="${rootfs_partition_tempfile}" bs=512 skip=${rootfs_partition_start} count=${rootfs_partition_num_sectors}
fi
image_unmount2() {
sudo umount "${mount}"
raw_initrd_remove
}
if [[ ${rootfs_partition} = "raw" ]]; then
trap image_unmount2 EXIT
fi
# Embed the kernel and dtb images now, if requested
if [[ ${rootfs_partition} = "raw" ]]; then
if [[ "${embed_kernel_initrd_dtb}" = "1" ]]; then
if [ -n "${dtb}" ]; then
sudo mkdir -p "${mount}/boot/dtb/${dtb_subdir}"
sudo cp -a "${dtb}" "${mount}/boot/dtb/${dtb_subdir}"
sudo chown -R root:root "${mount}/boot/dtb/${dtb_subdir}"
fi
sudo cp -a "${kernel}" "${mount}/boot/vmlinuz-${kernel_version}"
sudo chown root:root "${mount}/boot/vmlinuz-${kernel_version}"
fi
else
if [[ "${embed_kernel_initrd_dtb}" = "1" ]]; then
if [ -n "${dtb}" ]; then
e2mkdir -G 0 -O 0 "${rootfs_partition_tempfile}":"/boot/dtb/${dtb_subdir}"
e2cp -G 0 -O 0 "${dtb}" "${rootfs_partition_tempfile}":"/boot/dtb/${dtb_subdir}"
fi
e2cp -G 0 -O 0 "${kernel}" "${rootfs_partition_tempfile}":"/boot/vmlinuz-${kernel_version}"
fi
fi
# Unmount the initial ramdisk
if [[ ${rootfs_partition} = "raw" ]]; then
sudo umount "${mount}"
else
rootfs_partition_start=$(partx -g -o START -s -n "${rootfs_partition}" "${disk}" | xargs)
rootfs_partition_end=$(partx -g -o END -s -n "${rootfs_partition}" "${disk}" | xargs)
rootfs_partition_num_sectors=$((${rootfs_partition_end} - ${rootfs_partition_start} + 1))
dd if="${rootfs_partition_tempfile}" of="${disk}" bs=512 seek=${rootfs_partition_start} count=${rootfs_partition_num_sectors} conv=fsync,notrunc
fi
rm -f "${rootfs_partition_tempfile}"
trap raw_initrd_remove EXIT
# Boot test the new system and run stage 3
${qemu} -machine "${machine}" -cpu "${cpu}" -m 2048 >&2 \
-kernel "${kernel}" -initrd "${initrd}" -no-user-config -nodefaults \
-no-reboot -display none -nographic -serial stdio -parallel none \
-smp "${qemucpucores}",sockets="${qemucpucores}",cores=1,threads=1 \
-object rng-random,id=objrng0,filename=/dev/urandom \
-device virtio-rng-pci-non-transitional,rng=objrng0,id=rng0,max-bytes=1024,period=2000 \
-drive file="${disk}",format=raw,if=none,aio=threads,id=drive-virtio-disk0 \
-device virtio-blk-pci-non-transitional,scsi=off,drive=drive-virtio-disk0 \
-chardev file,id=exitcode,path=exitcode \
-device pci-serial,chardev=exitcode \
-netdev user,id=usernet0,ipv6=off \
-device virtio-net-pci-non-transitional,netdev=usernet0,id=net0 \
-append "root=LABEL=ROOT init=/root/${suite}.sh ${cmdline}"
[[ -s exitcode ]] && exitcode=$(cat exitcode | tr -d '\r') || exitcode=2
rm -f exitcode
if [ "${exitcode}" != "0" ]; then
echo "Root filesystem finalization failed (err=${exitcode})"
exit "${exitcode}"
fi
# Fix up any issues from the unclean shutdown
if [[ ${rootfs_partition} = "raw" ]]; then
sudo e2fsck -p -f "${disk}" || true
else
rootfs_partition_start=$(partx -g -o START -s -n "${rootfs_partition}" "${disk}" | xargs)
rootfs_partition_end=$(partx -g -o END -s -n "${rootfs_partition}" "${disk}" | xargs)
rootfs_partition_num_sectors=$((${rootfs_partition_end} - ${rootfs_partition_start} + 1))
rootfs_partition_offset=$((${rootfs_partition_start} * 512))
rootfs_partition_tempfile2=$(mktemp)
dd if="${disk}" of="${rootfs_partition_tempfile2}" bs=512 skip=${rootfs_partition_start} count=${rootfs_partition_num_sectors}
e2fsck -p -f "${rootfs_partition_tempfile2}" || true
dd if="${rootfs_partition_tempfile2}" of="${disk}" bs=512 seek=${rootfs_partition_start} count=${rootfs_partition_num_sectors} conv=fsync,notrunc
rm -f "${rootfs_partition_tempfile2}"
e2fsck -fy "${disk}"?offset=${rootfs_partition_offset} || true
fi
if [[ -n "${system_partition}" ]]; then
system_partition_start=$(partx -g -o START -s -n "${system_partition}" "${disk}" | xargs)
system_partition_end=$(partx -g -o END -s -n "${system_partition}" "${disk}" | xargs)
system_partition_num_sectors=$((${system_partition_end} - ${system_partition_start} + 1))
system_partition_offset=$((${system_partition_start} * 512))
system_partition_size=$((${system_partition_num_sectors} * 512))
system_partition_tempfile=$(mktemp)
dd if="${disk}" of="${system_partition_tempfile}" bs=512 skip=${system_partition_start} count=${system_partition_num_sectors}
/sbin/fsck.vfat -a "${system_partition_tempfile}" || true
dd if="${system_partition_tempfile}" of="${disk}" bs=512 seek=${system_partition_start} count=${system_partition_num_sectors} conv=fsync,notrunc
rm -f "${system_partition_tempfile}"
fi
# Mount the final disk image locally
if [[ ${rootfs_partition} = "raw" ]]; then
sudo mount -o loop -t ext4 "${disk}" "${mount}"
else
rootfs_partition_start=$(partx -g -o START -s -n "${rootfs_partition}" "${disk}" | xargs)
rootfs_partition_offset=$((${rootfs_partition_start} * 512))
sudo mount -o loop,offset=${rootfs_partition_offset} -t ext4 "${disk}" "${mount}"
fi
image_unmount3() {
sudo umount "${mount}"
raw_initrd_remove
}
trap image_unmount3 EXIT
# Fill the rest of the space with zeroes, to optimize compression
sudo dd if=/dev/zero of="${mount}/sparse" bs=1M 2>/dev/null || true
sudo rm -f "${mount}/sparse"
echo "Debian ${suite} for ${arch} filesystem generated at '${disk}'."
echo "Initial ramdisk generated at '${ramdisk}'."