feat: implement out-of-band descriptor calculation (#558)

oci/private/descriptor.sh

@@ -0,0 +1,32 @@
+#!/usr/bin/env bash
+set -o pipefail -o errexit -o nounset
+
+# This shell script creates a partial `descriptor` for the given archive to be later used by image.sh.
+# Partial in this context means that not all properties are in the final state, it must be processed 
+# again by the image.sh wrapper to create a final descriptor. That said all expensive work, such as 
+# `diffid` and `digest` calculation is already done here, there is not much left to do in image.sh 
+# other than figuring out `mediaType` field by looking the base image media type and use an appropriate
+# `mediaType` for the new layer. 
+# 
+# See: https://github.com/opencontainers/image-spec/blob/main/descriptor.md
+
+# shellcheck disable=SC2153
+PATH="$HPATH:$PATH"
+archive="$1"
+output="$2"
+
+digest="$(regctl digest <"$archive")"
+diffid="$digest"
+size=$(coreutils wc -c "$archive" | coreutils cut -f1 -d' ')
+compression=
+
+if [[ $(coreutils od -An -t x1 --read-bytes 2 "$archive") == " 1f 8b" ]]; then
+    compression="gzip"
+    diffid=$(zstd --decompress --format=gzip <"$archive" | regctl digest)
+elif zstd -t <"$archive" 2>/dev/null; then
+    compression="zstd"
+    diffid=$(zstd --decompress --format=zstd <"$archive" | regctl digest)
+fi
+
+jq -n --arg compression "$compression" --arg diffid "$diffid" --arg digest "$digest" --argjson size "$size" \
+    '{digest: $digest, diffid: $diffid, compression: $compression, size: $size }' >"$output"

oci/private/image.bzl

@@ -87,6 +87,7 @@ If `group/gid` is not specified, the default group and supplementary groups of t
     "labels": attr.label(doc = "A file containing a dictionary of labels. Each line should be in the form `name=value`.", allow_single_file = True),
     "annotations": attr.label(doc = "A file containing a dictionary of annotations. Each line should be in the form `name=value`.", allow_single_file = True),
     "_image_sh": attr.label(default = "image.sh", allow_single_file = True),
+    "_descriptor_sh": attr.label(default = "descriptor.sh", executable = True, cfg = "exec", allow_single_file = True),
     "_windows_constraint": attr.label(default = "@platforms//os:windows"),
 }
 
@@ -96,6 +97,35 @@ def _platform_str(os, arch, variant = None):
         parts["variant"] = variant
     return json.encode(parts)
 
+def _calculate_descriptor(ctx, idx, layer, zstd, jq, coreutils, regctl):
+    descriptor = ctx.actions.declare_file("%s.%s.descriptor.json" % (ctx.label.name, idx))
+    args = ctx.actions.args()
+    args.add(layer)
+    args.add(descriptor)
+    ctx.actions.run(
+        executable = util.maybe_wrap_launcher_for_windows(ctx, ctx.executable._descriptor_sh),
+        inputs = [layer],
+        outputs = [descriptor],
+        arguments = [args],
+        env = {
+            "HPATH": ":".join([
+                zstd.zstdinfo.binary.dirname,
+                jq.jqinfo.bin.dirname,
+                coreutils.coreutils_info.bin.dirname,
+                regctl.regctl_info.binary.dirname,
+            ]),
+        },
+        tools = [
+            jq.jqinfo.bin,
+            zstd.zstdinfo.binary,
+            coreutils.coreutils_info.bin,
+            regctl.regctl_info.binary,
+        ],
+        mnemonic = "OCIDescriptor",
+        progress_message = "OCI Descriptor %{input}",
+    )
+    return descriptor
+
 def _oci_image_impl(ctx):
     if not ctx.attr.base and (not ctx.attr.os or not ctx.attr.architecture):
         fail("'os' and 'architecture' are mandatory when 'base' is unspecified.")
@@ -119,12 +149,12 @@ def _oci_image_impl(ctx):
             "{{regctl_path}}": regctl.regctl_info.binary.dirname,
             "{{jq_path}}": jq.jqinfo.bin.dirname,
             "{{coreutils_path}}": coreutils.coreutils_info.bin.dirname,
-            "{{zstd_path}}": zstd.zstdinfo.binary.dirname,
             "{{output}}": output.path,
         },
     )
 
     inputs = [builder] + ctx.files.tars
+
     args = ctx.actions.args()
 
     if ctx.attr.base:
@@ -136,9 +166,12 @@ def _oci_image_impl(ctx):
         args.add(_platform_str(ctx.attr.os, ctx.attr.architecture, ctx.attr.variant), format = "--scratch=%s")
 
     # add layers
-    for layer in ctx.attr.tars:
+    for (i, layer) in enumerate(ctx.files.tars):
+        descriptor = _calculate_descriptor(ctx, i, layer, zstd, jq, coreutils, regctl)
+        inputs.append(descriptor)
+
         # tars are already added as input above.
-        args.add_all(layer[DefaultInfo].files, format_each = "--layer=%s")
+        args.add_joined([layer, descriptor], join_with = "=", format_joined = "--layer=%s")
 
     if ctx.attr.entrypoint:
         args.add(ctx.file.entrypoint.path, format = "--entrypoint=%s")
@@ -191,7 +224,6 @@ def _oci_image_impl(ctx):
             regctl.regctl_info.binary,
             jq.jqinfo.bin,
             coreutils.coreutils_info.bin,
-            zstd.zstdinfo.binary,
         ],
         mnemonic = "OCIImage",
         progress_message = "OCI Image %{label}",

oci/private/image.sh

@@ -5,7 +5,6 @@ set -o pipefail -o errexit -o nounset
 PATH="{{jq_path}}:$PATH"
 PATH="{{regctl_path}}:$PATH"
 PATH="{{coreutils_path}}:$PATH"
-PATH="{{zstd_path}}:$PATH"
 
 # Constants
 readonly OUTPUT="{{output}}"
@@ -18,15 +17,15 @@ readonly ENV_EXPAND_FILTER='[$raw | match("\\${?([a-zA-Z0-9_]+)}?"; "gm")] | red
 
 function base_from_scratch() {
   local platform="$1"
-   # Create a new manifest
+  # Create a new manifest
   jq -n '{
     schemaVersion: 2, 
     mediaType: "application/vnd.oci.image.manifest.v1+json", 
     config: { mediaType: "application/vnd.oci.image.config.v1+json", size: 0 },
     layers: []
   }' | update_manifest
   # Create the image config when there is annotations
-  jq -n --argjson platform "$platform" '{config:{}, rootfs:{type: "layers", diff_ids:[]}} + $platform' | update_config > /dev/null
+  jq -n --argjson platform "$platform" '{config:{}, rootfs:{type: "layers", diff_ids:[]}} + $platform' | update_config >/dev/null
 }
 
 function base_from() {
@@ -63,53 +62,41 @@ function update_manifest() {
 }
 
 function add_layer() {
-  local path=
+  local path="$1"
+  local desc=
   local media_type=
-  local compression=
-  local digest=
-  local diffid=
-  local size=
-
-  path="$1"
-  digest=$(regctl digest <"$path")
-  diffid="$digest"
-  size=$(wc -c "$path" | awk '{print $1}')
-
-  if [[ $(coreutils od -An -t x1 --read-bytes 2 "$path") == " 1f 8b" ]]; then
-    compression="gzip"
-    diffid=$(zstd --decompress --format=gzip <"$path" | regctl digest)
-  elif zstd -t "$path" 2>/dev/null; then
-    compression="zstd"
-    diffid=$(zstd --decompress --format=zstd <"$path" | regctl digest)
-  fi
+  local comp_ext=
 
-  # If the base image uses docker media types, then add new layer with oci-spec 
+  desc="$(cat "$2")"
+
+  # If the base image uses docker media types, then add new layer with oci-spec
   # interchangable media type.
-  if [[ $(get_manifest | jq -r '.mediaType') == "application/vnd.docker."* ]]; then 
+  if [[ $(get_manifest | jq -r '.mediaType') == "application/vnd.docker."* ]]; then
     media_type="application/vnd.docker.image.rootfs.diff.tar"
-    if [[ -n "$compression" ]]; then
-      media_type="$media_type.$compression"
-    fi
-  # otherwise, use oci-spec media types.
-  else 
+    comp_ext="."
+  else
+    # otherwise, use oci-spec media types.
     media_type="application/vnd.oci.image.layer.v1.tar"
-    if [[ -n "$compression" ]]; then
-      media_type="$media_type+$compression"
-    fi 
+    comp_ext="+"
   fi
 
-  # echo "$media_type $diffid $digest"
+  desc="$(jq --arg comp_ext "${comp_ext}" '.compression |= (if . != "" then "\($comp_ext)\(.)" end)' <<< "$desc")"
 
-  new_config_digest=$(get_config | jq --arg diffid "$diffid" '.rootfs.diff_ids += [$diffid]' | update_config)
+  new_config_digest=$(
+    get_config | jq --argjson desc "$desc" '.rootfs.diff_ids += [$desc.diffid]' | update_config
+  )
 
   get_manifest |
-    jq '.config.digest = $config_digest | .layers += [{size: $size, digest: $layer_digest, mediaType: $media_type}]' \
-      --arg config_digest "$new_config_digest" \
-      --arg layer_digest "$digest" \
-      --arg media_type "$media_type" \
-      --argjson size "$size" | update_manifest
+    jq '.config.digest = $config_digest |
+        .layers += [{size: $desc.size, digest: $desc.digest, mediaType: "\($media_type)\($desc.compression)"}]' \
+      --arg config_digest "${new_config_digest}" \
+      --argjson desc "${desc}" \
+      --arg media_type "${media_type}" | update_manifest
+
+  local digest_path= 
+  digest_path="$(jq -r '.digest | sub(":"; "/")' <<< "$desc")"
 
-  regctl blob put "$REF" <"$path" >/dev/null
+  coreutils cat "$path" > "$OUTPUT/blobs/$digest_path"
 }
 
 CONFIG="{}"
@@ -122,7 +109,10 @@ for ARG in "$@"; do
   --from=*)
     base_from "${ARG#--from=}"
     ;;
-  --layer=*) add_layer "${ARG#--layer=}" ;;
+  --layer=*)
+    IFS='=' read -r layer descriptor <<<"${ARG#--layer=}"
+    add_layer "${layer}" "$descriptor"
+    ;;
   --env=*)
     # Get environment from existing config
     env=$(get_config | jq '(.config.Env // []) | map(. | split("=") | {"key": .[0], "value": .[1:] | join("=")})')
@@ -171,4 +161,4 @@ done
 
 get_config | jq --argjson config "$CONFIG" '. *= $config' | update_config >/dev/null
 ## TODO: container structure is broken
-(JSON="$(cat "$OUTPUT/index.json")" && jq "del(.manifests[].annotations)" > "$OUTPUT/index.json" <<< "$JSON" )
+(JSON="$(cat "$OUTPUT/index.json")" && jq "del(.manifests[].annotations)" >"$OUTPUT/index.json" <<<"$JSON")