Use fedoraproject published EC2 images

Previously a very complex, manual, and failure-prone `import_images`
stage was required to bring raw images into EC2.  Primarily this was
necessary because beta images aren't published on EC2 by the
fedoraproject.  However, since the original implementation, CI
operations against rawhide have largely supplanted the need to support
testing against the beta images.  This means the 'import_images' stage
can be completely dropped, and the 'base_images' stage can simply source
images (including `rawhide` if necessary) published by the Fedora
project.

Signed-off-by: Chris Evich <cevich@redhat.com>

.cirrus.star

@@ -6,7 +6,6 @@ load("cirrus", "fs")
 def main():
   return {
     "env": {
-      "IMG_SFX": fs.read("IMG_SFX").strip(),
-      "IMPORT_IMG_SFX": fs.read("IMPORT_IMG_SFX").strip()
+      "IMG_SFX": fs.read("IMG_SFX").strip()
     },
   }

IMG_SFX

@@ -1 +1 @@
-20240620t153000z-f40f39d13
+20240701t155130z-f40f39d13

Makefile

@@ -20,15 +20,14 @@ if_ci_else = $(if $(findstring true,$(CI)),$(1),$(2))
 
 export CENTOS_STREAM_RELEASE = 9
 
+# Warning: Beta Fedora releases are not supported.  Verifiy EC2 AMI availability
+# here: https://fedoraproject.org/cloud/download
 export FEDORA_RELEASE = 40
 export PRIOR_FEDORA_RELEASE = 39
 
 # This should always be one-greater than $FEDORA_RELEASE (assuming it's actually the latest)
 export RAWHIDE_RELEASE = 41
 
-# See import_images/README.md
-export FEDORA_IMPORT_IMG_SFX = $(_IMPORT_IMG_SFX)
-
 # Automation assumes the actual release number (after SID upgrade)
 # is always one-greater than the latest DEBIAN_BASE_FAMILY (GCE image).
 export DEBIAN_RELEASE = 13
@@ -104,7 +103,6 @@ override _HLPFMT = "%-20s %s\n"
 
 # Suffix value for any images built from this make execution
 _IMG_SFX ?= $(file <IMG_SFX)
-_IMPORT_IMG_SFX ?= $(file <IMPORT_IMG_SFX)
 
 # Env. vars needed by packer
 export CHECKPOINT_DISABLE = 1  # Disable hashicorp phone-home
@@ -152,11 +150,6 @@ timebomb-check:
 	        false; \
 	    fi
 
-.PHONY: IMPORT_IMG_SFX
-IMPORT_IMG_SFX:  ## Generate a new date-based import-image suffix, store in the file IMPORT_IMG_SFX
-	$(file >$@,$(shell date --utc +%Y%m%dt%H%M%Sz)-f$(FEDORA_RELEASE)f$(PRIOR_FEDORA_RELEASE)d$(subst .,,$(DEBIAN_RELEASE)))
-	@echo "$(file <IMPORT_IMG_SFX)"
-
 .PHONY: ci_debug
 ci_debug: $(_TEMPDIR)/ci_debug.tar ## Build and enter container for local development/debugging of container-based Cirrus-CI tasks
 	/usr/bin/podman run -it --rm \
@@ -268,113 +261,13 @@ image_builder_debug: $(_TEMPDIR)/image_builder_debug.tar ## Build and enter cont
 		-e PACKER_INSTALL_DIR=/usr/local/bin \
 		-e PACKER_VERSION=$(call err_if_empty,PACKER_VERSION) \
 		-e IMG_SFX=$(call err_if_empty,_IMG_SFX) \
-		-e IMPORT_IMG_SFX=$(call err_if_empty,_IMPORT_IMG_SFX) \
 		-e GAC_FILEPATH=$(GAC_FILEPATH) \
 		-e AWS_SHARED_CREDENTIALS_FILE=$(AWS_SHARED_CREDENTIALS_FILE) \
 		docker-archive:$<
 
 $(_TEMPDIR)/image_builder_debug.tar: $(_TEMPDIR) $(wildcard image_builder/*)
 	$(call podman_build,$@,image_builder_debug,image_builder)
 
-# Avoid re-downloading unnecessarily
-# Ref: https://www.gnu.org/software/make/manual/html_node/Special-Targets.html#Special-Targets
-.PRECIOUS: $(_TEMPDIR)/fedora-aws-$(_IMPORT_IMG_SFX).$(IMPORT_FORMAT)
-$(_TEMPDIR)/fedora-aws-$(_IMPORT_IMG_SFX).$(IMPORT_FORMAT): $(_TEMPDIR)
-	bash import_images/handle_image.sh \
-		$@ \
-		$(call err_if_empty,FEDORA_IMAGE_URL) \
-		$(call err_if_empty,FEDORA_CSUM_URL)
-
-$(_TEMPDIR)/fedora-aws-arm64-$(_IMPORT_IMG_SFX).$(IMPORT_FORMAT): $(_TEMPDIR)
-	bash import_images/handle_image.sh \
-		$@ \
-		$(call err_if_empty,FEDORA_ARM64_IMAGE_URL) \
-		$(call err_if_empty,FEDORA_ARM64_CSUM_URL)
-
-$(_TEMPDIR)/%.md5: $(_TEMPDIR)/%.$(IMPORT_FORMAT)
-	openssl md5 -binary $< | base64 > $@.tmp
-	mv $@.tmp $@
-
-# MD5 metadata value checked by AWS after upload + 5 retries.
-# Cache disabled to avoid sync. issues w/ vmimport service if
-# image re-uploaded.
-# TODO: Use sha256 from ..._CSUM_URL file instead of recalculating
-# https://docs.aws.amazon.com/AmazonS3/latest/userguide/checking-object-integrity.html
-# Avoid re-uploading unnecessarily
-.SECONDARY: $(_TEMPDIR)/%.uploaded
-$(_TEMPDIR)/%.uploaded: $(_TEMPDIR)/%.$(IMPORT_FORMAT) $(_TEMPDIR)/%.md5
-	-$(AWS) s3 rm --quiet s3://packer-image-import/%.$(IMPORT_FORMAT)
-	$(AWS) s3api put-object \
-		--content-md5 "$(file < $(_TEMPDIR)/$*.md5)" \
-		--content-encoding binary/octet-stream \
-		--cache-control no-cache \
-		--bucket packer-image-import \
-		--key $*.$(IMPORT_FORMAT) \
-		--body $(_TEMPDIR)/$*.$(IMPORT_FORMAT) > $@.tmp
-	mv $@.tmp $@
-
-# For whatever reason, the 'Format' value must be all upper-case.
-# Avoid creating unnecessary/duplicate import tasks
-.SECONDARY: $(_TEMPDIR)/%.import_task_id
-$(_TEMPDIR)/%.import_task_id: $(_TEMPDIR)/%.uploaded
-	$(AWS) ec2 import-snapshot \
-		--disk-container Format=$(shell tr '[:lower:]' '[:upper:]'<<<"$(IMPORT_FORMAT)"),UserBucket="{S3Bucket=packer-image-import,S3Key=$*.$(IMPORT_FORMAT)}" > $@.tmp.json
-	@cat $@.tmp.json
-	jq -r -e .ImportTaskId $@.tmp.json > $@.tmp
-	mv $@.tmp $@
-
-# Avoid importing multiple snapshots for the same image
-.PRECIOUS: $(_TEMPDIR)/%.snapshot_id
-$(_TEMPDIR)/%.snapshot_id: $(_TEMPDIR)/%.import_task_id
-	bash import_images/wait_import_task.sh "$<" > $@.tmp
-	mv $@.tmp $@
-
-define _register_sed
-	sed -r \
-		-e 's/@@@NAME@@@/$(1)/' \
-		-e 's/@@@IMPORT_IMG_SFX@@@/$(_IMPORT_IMG_SFX)/' \
-		-e 's/@@@ARCH@@@/$(2)/' \
-		-e 's/@@@SNAPSHOT_ID@@@/$(3)/' \
-		import_images/register.json.in \
-	> $(4)
-endef
-
-$(_TEMPDIR)/fedora-aws-$(_IMPORT_IMG_SFX).register.json: $(_TEMPDIR)/fedora-aws-$(_IMPORT_IMG_SFX).snapshot_id import_images/register.json.in
-	$(call _register_sed,fedora-aws,x86_64,$(file <$<),$@)
-
-$(_TEMPDIR)/fedora-aws-arm64-$(_IMPORT_IMG_SFX).register.json: $(_TEMPDIR)/fedora-aws-arm64-$(_IMPORT_IMG_SFX).snapshot_id import_images/register.json.in
-	$(call _register_sed,fedora-aws-arm64,arm64,$(file <$<),$@)
-
-# Avoid multiple registrations for the same image
-.PRECIOUS: $(_TEMPDIR)/%.ami.id
-$(_TEMPDIR)/%.ami.id: $(_TEMPDIR)/%.register.json
-	$(AWS) ec2 register-image --cli-input-json "$$(<$<)" > $@.tmp.json
-	cat $@.tmp.json
-	jq -r -e .ImageId $@.tmp.json > $@.tmp
-	mv $@.tmp $@
-
-$(_TEMPDIR)/%.ami.name: $(_TEMPDIR)/%.register.json
-	jq -r -e .Name $< > $@.tmp
-	mv $@.tmp $@
-
-$(_TEMPDIR)/%.ami.json: $(_TEMPDIR)/%.ami.id $(_TEMPDIR)/%.ami.name
-	$(AWS) ec2 create-tags \
-		--resources "$$(<$(_TEMPDIR)/$*.ami.id)" \
-		--tags \
-			Key=Name,Value=$$(<$(_TEMPDIR)/$*.ami.name) \
-			Key=automation,Value=false
-	$(AWS) --output table ec2 describe-images --image-ids "$$(<$(_TEMPDIR)/$*.ami.id)" \
-		| tee $@
-
-.PHONY: import_images
-import_images: $(_TEMPDIR)/fedora-aws-$(_IMPORT_IMG_SFX).ami.json $(_TEMPDIR)/fedora-aws-arm64-$(_IMPORT_IMG_SFX).ami.json import_images/manifest.json.in  ## Import generic Fedora cloud images into AWS EC2.
-	sed -r \
-		-e 's/@@@IMG_SFX@@@/$(_IMPORT_IMG_SFX)/' \
-		-e 's/@@@CIRRUS_TASK_ID@@@/$(CIRRUS_TASK_ID)/' \
-		import_images/manifest.json.in \
-	> import_images/manifest.json
-	@echo "Image import(s) successful!"
-
 .PHONY: base_images
 # This needs to run in a virt/nested-virt capable environment
 base_images: base_images/manifest.json ## Create, prepare, and import base-level images into GCE.

README.md

@@ -52,7 +52,7 @@ However, all steps are listed below for completeness.
 For more information on the overall process of importing custom GCE VM
 Images, please [refer to the documentation](https://cloud.google.com/compute/docs/import/import-existing-image).  For references to the latest pre-build AWS
 EC2 Fedora AMI's see [the
-upstream cloud page](https://alt.fedoraproject.org/cloud/).
+upstream cloud page](https://fedoraproject.org/cloud/download).
 For more information on the primary tool (*packer*) used for this process,
 please [see it's documentation page](https://www.packer.io/docs).
 

base_images/cloud.yml

@@ -26,8 +26,6 @@ variables:  # Empty value means it must be passed in on command-line
     PRIOR_FEDORA_IMAGE_URL: "{{env `PRIOR_FEDORA_IMAGE_URL`}}"
     PRIOR_FEDORA_CSUM_URL: "{{env `PRIOR_FEDORA_CSUM_URL`}}"
 
-    FEDORA_IMPORT_IMG_SFX: "{{env `FEDORA_IMPORT_IMG_SFX`}}"
-
     DEBIAN_RELEASE: "{{env `DEBIAN_RELEASE`}}"
     DEBIAN_BASE_FAMILY: "{{env `DEBIAN_BASE_FAMILY`}}"
 
@@ -109,20 +107,18 @@ builders:
     - &fedora-aws
       name: 'fedora-aws'
       type: 'amazon-ebs'
-      source_ami_filter:  # Will fail if >1 or no AMI found
+      source_ami_filter:
+        # Many of these search filter values (like account ID and name) aren't publicized
+        # anywhere.  They were found by examining AWS EC2 AMIs published/referenced from
+        # the AWS sections on https://fedoraproject.org/cloud/download
         owners:
-            # Docs are wrong, specifying the Account ID required to make AMIs private.
-            # The Account ID is hard-coded here out of expediency, since passing in
-            # more packer args from the command-line (in Makefile) is non-trivial.
-            - &accountid '449134212816'
-        # It's necessary to 'search' for the base-image by these criteria.  If
-        # more than one image is found, Packer will fail the build (and display
-        # the conflicting AMI IDs).
+            - &fedora_accountid 125523088429
+        most_recent: true  # Required b/c >1 search result likely to be returned
         filters: &ami_filters
             architecture: 'x86_64'
             image-type: 'machine'
-            is-public: 'false'
-            name: '{{build_name}}-i{{user `FEDORA_IMPORT_IMG_SFX`}}'
+            is-public: 'true'
+            name: 'Fedora-Cloud-Base*-{{user `FEDORA_RELEASE`}}-*us-east-1*'
             root-device-type: 'ebs'
             state: 'available'
             virtualization-type: 'hvm'
@@ -146,7 +142,6 @@ builders:
           volume_type: 'gp2'
           delete_on_termination: true
       # These are critical and used by security-polciy to enforce instance launch limits.
-
       tags: &awstags
         <<: *imgcpylabels
         # EC2 expects "Name" to be capitalized
@@ -160,7 +155,7 @@ builders:
       # This is necessary for security - The CI service accounts are not permitted
       # to use AMI's from any other account, including public ones.
       ami_users:
-        - *accountid
+        - &accountid '449134212816'
       ssh_username: 'fedora'
       ssh_clear_authorized_keys: true
       # N/B: Required Packer >= 1.8.0
@@ -171,7 +166,8 @@ builders:
       name: 'fedora-aws-arm64'
       source_ami_filter:
         owners:
-            - *accountid
+            - *fedora_accountid
+        most_recent: true  # Required b/c >1 search result likely to be returned
         filters:
             <<: *ami_filters
             architecture: 'arm64'

imgobsolete/entrypoint.sh

@@ -11,7 +11,7 @@ set -eo pipefail
 # shellcheck source=imgts/lib_entrypoint.sh
 source /usr/local/bin/lib_entrypoint.sh
 
-req_env_vars GCPJSON GCPNAME GCPPROJECT AWSINI IMG_SFX IMPORT_IMG_SFX
+req_env_vars GCPJSON GCPNAME GCPPROJECT AWSINI IMG_SFX
 
 gcloud_init
 
@@ -159,10 +159,10 @@ for (( i=nr_amis ; i ; i-- )); do
         continue
     fi
 
-    # Any image matching the currently in-use IMG_SFX or IMPORT_IMG_SFX
+    # Any image matching the currently in-use IMG_SFX
     # must always be preserved.  Values are defined in cirrus.yml
     # shellcheck disable=SC2154
-    if [[ "$name" =~ $IMG_SFX ]] || [[ "$name" =~ $IMPORT_IMG_SFX ]]; then
+    if [[ "$name" =~ $IMG_SFX ]]; then
         msg "Retaining current (latest) image $name | $tags"
         continue
     fi

imgprune/entrypoint.sh

@@ -11,7 +11,7 @@ set -e
 # shellcheck source=imgts/lib_entrypoint.sh
 source /usr/local/bin/lib_entrypoint.sh
 
-req_env_vars GCPJSON GCPNAME GCPPROJECT AWSINI IMG_SFX IMPORT_IMG_SFX
+req_env_vars GCPJSON GCPNAME GCPPROJECT AWSINI IMG_SFX
 
 gcloud_init
 
@@ -48,7 +48,7 @@ $GCLOUD compute images list --show-deprecated \
         # Any image matching the currently in-use IMG_SFX must always be preserved.
         # Values are defined in cirrus.yml
         # shellcheck disable=SC2154
-        if [[ "$name" =~ $IMG_SFX ]] || [[ "$name" =~ $IMPORT_IMG_SFX ]]; then
+        if [[ "$name" =~ $IMG_SFX ]]; then
             msg "    Skipping current (latest) image $name"
             continue
         fi
@@ -91,9 +91,9 @@ for (( i=nr_amis ; i ; i-- )); do
         warn 0 "    EC2 AMI ID '$ami_id' is missing a 'Name' tag"
     fi
 
-    # Any image matching the currently in-use IMG_SFX or IMPORT_IMG_SFX
+    # Any image matching the currently in-use IMG_SFX
     # must always be preserved.
-    if [[ "$name" =~ $IMG_SFX ]] || [[ "$name" =~ $IMPORT_IMG_SFX ]]; then
+    if [[ "$name" =~ $IMG_SFX ]]; then
         warn 0 "    Retaining current (latest) image $name id $ami_id"
         $AWS ec2 disable-image-deprecation --image-id "$ami_id" > /dev/null
         continue