diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index b1eb6d61eeb..718082e8e65 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -218,6 +218,9 @@ jobs: formats=$(echo "$formats" | tr ' ' '\n' | sed 's/equinix_metal/packet/g') for format in ${formats}; do + if [ "${format}" = qemu ] || [ "${format}" = qemu_uefi_secure ]; then + continue + fi echo " ################### VENDOR '${format}' ################### " ./run_sdk_container -n "${container_name}" \ ./image_to_vm.sh --format "${format}" --board="${arch}-usr" \ @@ -225,6 +228,8 @@ jobs: --image_compression_formats=bz2 done + # Zip doesn't handle symlinks well, remove them + rm -f artifacts/${arch}-usr/latest/flatcar_production_{qemu,qemu_uefi_secure}_image.img* # upload-artifacts cannot handle artifact uploads from sym-linked directories (no, really) # so we move things around. mkdir -p artifacts/images diff --git a/build_library/release_util.sh b/build_library/release_util.sh index 21d5aa58057..1dc9148a2ee 100644 --- a/build_library/release_util.sh +++ b/build_library/release_util.sh @@ -67,9 +67,21 @@ compress_file() { ;; esac - ${IMAGE_ZIPPER} -f "${filepath}" 2>&1 >/dev/null || die "failed to compress ${filepath}" + # Check if symlink in which case we set up a "compressed" symlink + local compressed_name="${filepath}.${compression_format}" + if [ -L "${filepath}" ]; then + # We could also test if the target exists and otherwise do the compression + # but we might then end up with two different compressed artifacts + local link_target + link_target=$(readlink -f "${filepath}") + local target_basename + target_basename=$(basename "${link_target}") + ln -fs "${target_basename}.${compression_format}" "${compressed_name}" + else + ${IMAGE_ZIPPER} -f "${filepath}" 2>&1 >/dev/null || die "failed to compress ${filepath}" + fi - echo -n "${filepath}.${compression_format}" + echo -n "${compressed_name}" } compress_disk_images() { @@ -85,7 +97,7 @@ compress_disk_images() { # Files that did not match the filter for disk images. local -n local_extra_files="$3" - info "Compressing images" + info "Compressing ${#local_files_to_evaluate[@]} images" # We want to compress images, but we also want to remove the uncompressed files # from the list of uploadable files. for filename in "${local_files_to_evaluate[@]}"; do @@ -98,6 +110,9 @@ compress_disk_images() { # An associative array we set an element on whenever we process a format. # This way we don't process the same format twice. A unique for array elements. + # (But first we need to unset the previous loop or we can only compress a single + # file per list of files). + unset processed_format declare -A processed_format for format in "${FORMATS[@]}";do if [ -z "${processed_format[${format}]}" ]; then @@ -113,7 +128,10 @@ compress_disk_images() { [ "${filename##*/}" != "flatcar_production_image.bin" ] && [ "${filename##*/}" != "flatcar_production_update.bin" ] && ! echo "${FORMATS[@]}" | grep -q "none"; then + info "Removing ${filename}" rm "${filename}" + else + info "Keeping ${filename}" fi else local_extra_files+=( "${filename}" ) diff --git a/build_library/vm_image_util.sh b/build_library/vm_image_util.sh index be58017907c..b747b4aad9b 100644 --- a/build_library/vm_image_util.sh +++ b/build_library/vm_image_util.sh @@ -22,9 +22,7 @@ VALID_IMG_TYPES=( packet parallels pxe - qemu qemu_uefi - qemu_uefi_secure rackspace rackspace_onmetal rackspace_vhd @@ -125,13 +123,6 @@ IMG_DEFAULT_MEM=2048 IMG_DEFAULT_CPUS=2 ## qemu -IMG_qemu_DISK_FORMAT=qcow2 -IMG_qemu_DISK_LAYOUT=vm -IMG_qemu_CONF_FORMAT=qemu -IMG_qemu_OEM_USE=qemu -IMG_qemu_OEM_PACKAGE=common-oem-files -IMG_qemu_OEM_SYSEXT=oem-qemu - IMG_qemu_uefi_DISK_FORMAT=qcow2 IMG_qemu_uefi_DISK_LAYOUT=vm IMG_qemu_uefi_CONF_FORMAT=qemu_uefi @@ -139,13 +130,6 @@ IMG_qemu_uefi_OEM_USE=qemu IMG_qemu_uefi_OEM_PACKAGE=common-oem-files IMG_qemu_uefi_OEM_SYSEXT=oem-qemu -IMG_qemu_uefi_secure_DISK_FORMAT=qcow2 -IMG_qemu_uefi_secure_DISK_LAYOUT=vm -IMG_qemu_uefi_secure_CONF_FORMAT=qemu_uefi_secure -IMG_qemu_uefi_secure_OEM_USE=qemu -IMG_qemu_uefi_secure_OEM_PACKAGE=common-oem-files -IMG_qemu_uefi_secure_OEM_SYSEXT=oem-qemu - ## xen IMG_xen_CONF_FORMAT=xl @@ -320,7 +304,7 @@ get_default_vm_type() { local board="$1" case "$board" in amd64-usr) - echo "qemu" + echo "qemu_uefi" ;; arm64-usr) echo "qemu_uefi" @@ -595,6 +579,18 @@ write_vm_disk() { info "Writing $disk_format image $(basename "${VM_DST_IMG}")" _write_${disk_format}_disk "${VM_TMP_IMG}" "${VM_DST_IMG}" + # We now only support building qemu_uefi and set up symlinks + # for the qemu and qemu_uefi_secure images + if [ "${VM_IMG_TYPE}" = qemu_uefi ]; then + local qemu="${VM_DST_IMG/qemu_uefi/qemu}" + local qemu_uefi_secure="${VM_DST_IMG/qemu_uefi/qemu_uefi_secure}" + local target_basename + target_basename=$(basename "${VM_DST_IMG}") + ln -fs "${target_basename}" "${qemu}" + ln -fs "${target_basename}" "${qemu_uefi_secure}" + VM_GENERATED_FILES+=( "${qemu}" "${qemu_uefi_secure}" ) + fi + # Add disk image to final file list if it isn't going to be bundled if [[ -z "$(_get_vm_opt BUNDLE_FORMAT)" ]]; then VM_GENERATED_FILES+=( "${VM_DST_IMG}" ) @@ -680,7 +676,7 @@ _write_cpio_disk() { local grub_name="$(_dst_name "_grub.efi")" _write_cpio_common $@ # Pull the kernel and loader out of the filesystem - cp "${base_dir}"/boot/flatcar/vmlinuz-a "${dst_dir}/${vmlinuz_name}" + ln -fs flatcar_production_image.vmlinuz "${dst_dir}/${vmlinuz_name}" local grub_arch case $BOARD in @@ -796,6 +792,13 @@ _write_qemu_uefi_conf() { sed -e "s%^VM_PFLASH_RO=.*%VM_PFLASH_RO='${flash_ro}'%" \ -e "s%^VM_PFLASH_RW=.*%VM_PFLASH_RW='${flash_rw}'%" -i "${script}" VM_GENERATED_FILES+=( "$(_dst_dir)/${flash_ro}" "$(_dst_dir)/${flash_rw}" ) + + # We now only support building qemu_uefi and generate the + # other artifacts from here + if [ "${VM_IMG_TYPE}" = qemu_uefi ]; then + VM_IMG_TYPE=qemu _write_qemu_conf + VM_IMG_TYPE=qemu_uefi_secure _write_qemu_uefi_secure_conf + fi } _write_qemu_uefi_secure_conf() { diff --git a/changelog/changes/2024-04-08-unify-qemu-images.md b/changelog/changes/2024-04-08-unify-qemu-images.md new file mode 100644 index 00000000000..4b6b72d3777 --- /dev/null +++ b/changelog/changes/2024-04-08-unify-qemu-images.md @@ -0,0 +1 @@ +- SDK: Unified qemu image formats, so that the `qemu_uefi` build target provides the regular `qemu` and the `qemu_uefi_secure` artifacts diff --git a/ci-automation/vms.sh b/ci-automation/vms.sh index e35bc577a30..7d4d9d31980 100644 --- a/ci-automation/vms.sh +++ b/ci-automation/vms.sh @@ -125,6 +125,9 @@ function _vm_build_impl() { mv "${images_in}" "${CONTAINER_IMAGE_ROOT}/${arch}-usr/latest-input" for format in ${formats}; do + if [ "${format}" = qemu ] || [ "${format}" = qemu_uefi_secure ]; then + continue + fi echo " ################### VENDOR '${format}' ################### " COMPRESSION_FORMAT="bz2" if [[ "${format}" =~ ^(openstack_mini|digitalocean)$ ]];then