Set up symlinks for same image artifacts to remove qemu/qemu_uefi_secure

The qemu and qemu_uefi_secure images have the same contents as the
qemu_uefi image which wastes space on the release server. A similar
case is the PXE vmlinuz which is the same as the regular one, too.

Set up symlinks for same images, and also detect this when compressing
to set up symlinks there as well. To reduce complexity, the qemu and
qemu_uefi_secure images are not supported anymore and the Jenkins or
GitHub CI will skip over them if specified. Users that build their own
images need to adapt, though.

.github/workflows/ci.yaml

@@ -218,13 +218,23 @@ jobs:
           formats=$(echo "$formats" | tr ' ' '\n' | sed 's/equinix_metal/packet/g')
 
           for format in ${formats}; do
+              if [ "${format}" = qemu ] || [ "${format}" = qemu_uefi_secure ]; then
+                  continue
+              fi
               echo " ###################  VENDOR '${format}' ################### "
               ./run_sdk_container -n "${container_name}" \
                   ./image_to_vm.sh --format "${format}" --board="${arch}-usr" \
                       --from "${CI_CONTAINER_ARTIFACT_ROOT}/${arch}-usr/latest" \
                       --image_compression_formats=bz2
           done
 
+          # Zip doesn't handle symlinks well, remove them
+          rm -f artifacts/${arch}-usr/latest/flatcar_production_{qemu,qemu_uefi_secure}_image.img*
+          # or create an explicit copy:
+          if [ -e artifacts/${arch}-usr/latest/flatcar_production_pxe.vmlinuz ]; then
+            rm -f artifacts/${arch}-usr/latest/flatcar_production_pxe.vmlinuz
+            cp artifacts/${arch}-usr/latest/flatcar_production_{image,pxe}.vmlinuz
+          fi
           # upload-artifacts cannot handle artifact uploads from sym-linked directories (no, really)
           #  so we move things around.
           mkdir -p artifacts/images

build_library/release_util.sh

@@ -67,9 +67,21 @@ compress_file() {
         ;;
     esac
 
-    ${IMAGE_ZIPPER} -f "${filepath}" 2>&1 >/dev/null || die "failed to compress ${filepath}"
+    # Check if symlink in which case we set up a "compressed" symlink
+    local compressed_name="${filepath}.${compression_format}"
+    if [ -L "${filepath}" ]; then
+        # We could also test if the target exists and otherwise do the compression
+        # but we might then end up with two different compressed artifacts
+        local link_target
+        link_target=$(readlink -f "${filepath}")
+        local target_basename
+        target_basename=$(basename "${link_target}")
+        ln -fs "${target_basename}.${compression_format}" "${compressed_name}"
+    else
+        ${IMAGE_ZIPPER} -f "${filepath}" 2>&1 >/dev/null || die "failed to compress ${filepath}"
+    fi
 
-    echo -n "${filepath}.${compression_format}"
+    echo -n "${compressed_name}"
 }
 
 compress_disk_images() {
@@ -85,7 +97,7 @@ compress_disk_images() {
     # Files that did not match the filter for disk images.
     local -n local_extra_files="$3"
 
-    info "Compressing images"
+    info "Compressing ${#local_files_to_evaluate[@]} images"
     # We want to compress images, but we also want to remove the uncompressed files
     # from the list of uploadable files.
     for filename in "${local_files_to_evaluate[@]}"; do
@@ -98,6 +110,9 @@ compress_disk_images() {
 
             # An associative array we set an element on whenever we process a format.
             # This way we don't process the same format twice. A unique for array elements.
+            # (But first we need to unset the previous loop or we can only compress a single
+            # file per list of files).
+            unset processed_format
             declare -A processed_format
             for format in "${FORMATS[@]}";do
                 if [ -z "${processed_format[${format}]}" ]; then
@@ -113,7 +128,10 @@ compress_disk_images() {
                [ "${filename##*/}" != "flatcar_production_image.bin" ] &&
                [ "${filename##*/}" != "flatcar_production_update.bin" ] &&
                ! echo "${FORMATS[@]}" | grep -q "none"; then
+                info "Removing ${filename}"
                 rm "${filename}"
+            else
+                info "Keeping ${filename}"
             fi
         else
             local_extra_files+=( "${filename}" )            

build_library/vm_image_util.sh

@@ -23,9 +23,7 @@ VALID_IMG_TYPES=(
     packet
     parallels
     pxe
-    qemu
     qemu_uefi
-    qemu_uefi_secure
     rackspace
     rackspace_onmetal
     rackspace_vhd
@@ -126,27 +124,13 @@ IMG_DEFAULT_MEM=2048
 IMG_DEFAULT_CPUS=2
 
 ## qemu
-IMG_qemu_DISK_FORMAT=qcow2
-IMG_qemu_DISK_LAYOUT=vm
-IMG_qemu_CONF_FORMAT=qemu
-IMG_qemu_OEM_USE=qemu
-IMG_qemu_OEM_PACKAGE=common-oem-files
-IMG_qemu_OEM_SYSEXT=oem-qemu
-
 IMG_qemu_uefi_DISK_FORMAT=qcow2
 IMG_qemu_uefi_DISK_LAYOUT=vm
 IMG_qemu_uefi_CONF_FORMAT=qemu_uefi
 IMG_qemu_uefi_OEM_USE=qemu
 IMG_qemu_uefi_OEM_PACKAGE=common-oem-files
 IMG_qemu_uefi_OEM_SYSEXT=oem-qemu
 
-IMG_qemu_uefi_secure_DISK_FORMAT=qcow2
-IMG_qemu_uefi_secure_DISK_LAYOUT=vm
-IMG_qemu_uefi_secure_CONF_FORMAT=qemu_uefi_secure
-IMG_qemu_uefi_secure_OEM_USE=qemu
-IMG_qemu_uefi_secure_OEM_PACKAGE=common-oem-files
-IMG_qemu_uefi_secure_OEM_SYSEXT=oem-qemu
-
 ## xen
 IMG_xen_CONF_FORMAT=xl
 
@@ -326,7 +310,7 @@ get_default_vm_type() {
     local board="$1"
     case "$board" in
     amd64-usr)
-        echo "qemu"
+        echo "qemu_uefi"
         ;;
     arm64-usr)
         echo "qemu_uefi"
@@ -603,6 +587,18 @@ write_vm_disk() {
     info "Writing $disk_format image $(basename "${VM_DST_IMG}")"
     _write_${disk_format}_disk "${VM_TMP_IMG}" "${VM_DST_IMG}"
 
+    # We now only support building qemu_uefi and set up symlinks
+    # for the qemu and qemu_uefi_secure images
+    if [ "${VM_IMG_TYPE}" = qemu_uefi ]; then
+        local qemu="${VM_DST_IMG/qemu_uefi/qemu}"
+        local qemu_uefi_secure="${VM_DST_IMG/qemu_uefi/qemu_uefi_secure}"
+        local target_basename
+        target_basename=$(basename "${VM_DST_IMG}")
+        ln -fs "${target_basename}" "${qemu}"
+        ln -fs "${target_basename}" "${qemu_uefi_secure}"
+        VM_GENERATED_FILES+=( "${qemu}" "${qemu_uefi_secure}" )
+    fi
+
     # Add disk image to final file list if it isn't going to be bundled
     if [[ -z "$(_get_vm_opt BUNDLE_FORMAT)" ]]; then
         VM_GENERATED_FILES+=( "${VM_DST_IMG}" )
@@ -693,7 +689,7 @@ _write_cpio_disk() {
     local grub_name="$(_dst_name "_grub.efi")"
     _write_cpio_common $@
     # Pull the kernel and loader out of the filesystem
-    cp "${base_dir}"/boot/flatcar/vmlinuz-a "${dst_dir}/${vmlinuz_name}"
+    ln -fs flatcar_production_image.vmlinuz "${dst_dir}/${vmlinuz_name}"
 
     local grub_arch
     case $BOARD in
@@ -809,6 +805,13 @@ _write_qemu_uefi_conf() {
     sed -e "s%^VM_PFLASH_RO=.*%VM_PFLASH_RO='${flash_ro}'%" \
         -e "s%^VM_PFLASH_RW=.*%VM_PFLASH_RW='${flash_rw}'%" -i "${script}"
     VM_GENERATED_FILES+=( "$(_dst_dir)/${flash_ro}" "$(_dst_dir)/${flash_rw}" )
+
+    # We now only support building qemu_uefi and generate the
+    # other artifacts from here
+    if [ "${VM_IMG_TYPE}" = qemu_uefi ]; then
+      VM_IMG_TYPE=qemu _write_qemu_conf
+      VM_IMG_TYPE=qemu_uefi_secure _write_qemu_uefi_secure_conf
+    fi
 }
 
 _write_qemu_uefi_secure_conf() {

changelog/changes/2024-04-08-unify-qemu-images.md

@@ -0,0 +1 @@
+- SDK: Unified qemu image formats, so that the `qemu_uefi` build target provides the regular `qemu` and the `qemu_uefi_secure` artifacts ([scripts#1847](https://github.com/flatcar/scripts/pull/1847))

ci-automation/vms.sh

@@ -125,6 +125,9 @@ function _vm_build_impl() {
             mv "${images_in}" "${CONTAINER_IMAGE_ROOT}/${arch}-usr/latest-input"
 
     for format in ${formats}; do
+        if [ "${format}" = qemu ] || [ "${format}" = qemu_uefi_secure ]; then
+           continue
+        fi
         echo " ###################  VENDOR '${format}' ################### "
         COMPRESSION_FORMAT="bz2"
         if [[ "${format}" =~ ^(openstack_mini|digitalocean)$ ]];then