Skip to content

Commit

Permalink
add configurable annotation
Browse files Browse the repository at this point in the history
  • Loading branch information
Tim Seagren committed Apr 10, 2023
1 parent c376201 commit 4540a16
Show file tree
Hide file tree
Showing 4 changed files with 123 additions and 89 deletions.
2 changes: 1 addition & 1 deletion VERSION
View file
Original file line number Diff line number Diff line change
@@ -1 +1 @@
v1.2.3
v1.3.0
15 changes: 15 additions & 0 deletions images.yaml.example
View file
Original file line number Diff line number Diff line change
@@ -1,16 +1,31 @@
# destination configures settings for the destination registry
destination:
# The hostname or IP of the destination registry
registry: 192.168.106.2:5000
# Enable/disable TLS on destination
secure: true
# Settings dictating how images are collected
collection:
# image_key specifies an alternate location to check for image names in Pod definitions
# This is useful when mutating webhooks are in place to rewrite image names
image_name_annotation_key: "my.webhook.annotation/[a-zA-Z0-9]+"
# Cosign signature validation settings
cosign_verifiers:
# Registry for verifier
- registry: registry1.dso.mil
# Subpath in registry for verifier
repo: ironbank/*
# Cosign key used to verify signatures
key: /app/ib-cosign.pub
- registry: registry.example.com
repo: foo/bar/baz
key: /app/example-cosign.pub
# Image names to unconditionally exclude from final "images" key
exclude:
- name: docker.io/library/alpine:3
# Images to be synced
images:
- name: docker.io/library/busybox:latest
# Image names to unconditionally include in final "images" key
include:
- name: docker.io/library/nginx:1.23.3
10 changes: 8 additions & 2 deletions imagesync.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -63,9 +63,15 @@ def main():
if args.command == "tidy":
log.info("Collecting images from cluster...")
used_images = []
collector = collect.Collector(
config.collection["image_name_annotation_key"]
if "image_name_annotation_key" in config.collection
else "",
args.version,
)

try:
used_images.extend(collect.cluster_images())
used_images.extend(collector.cluster_images())
except HTTPError as e:
log.error(f"Error retrieving images from cluster: {e.code} {e.reason}")
sys.exit(1)
Expand All @@ -77,7 +83,7 @@ def main():
sys.exit(1)

try:
used_images.extend(collect.bigbang_images(args.version))
used_images.extend(collector.bigbang_images())
except HTTPError as e:
log.error(f"Error retreving images.txt from BigBang: {e.code} {e.reason}")
sys.exit(1)
Expand Down
185 changes: 99 additions & 86 deletions modules/collect.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@
import re
import requests

from dataclasses import dataclass
from .utils.image import Image
from modules.utils import logger

Expand All @@ -16,91 +17,103 @@
)


def bigbang_images(version) -> list[str]:
r = requests.get(f"{BIGBANG_IMAGES_URL}/{version}/images.txt")
r.raise_for_status()

return [Image(image) for image in r.content.decode().splitlines()]


def cluster_images() -> set[Image]:
resources = ["pods", "jobs", "cronjobs"]
images = []
for resource in resources:
cmd = [
"kubectl",
"get",
f"{resource}",
"-A",
"-o",
"json",
]

result = subprocess.run(cmd, check=True, capture_output=True, text=True)
resource_json = json.loads(result.stdout)

for item in resource_json["items"]:
annotation = False
if resource == "pods":
resource_images = _get_images_from_imageswap_annotations(
item["metadata"]
)
if resource_images:
annotation = True
for image in resource_images:
images += [image] if image not in images else []
# If the original images we not added to the image list from annotations, retrieve images from the spec as usual
if not annotation:
resource_images = _get_images_from_nested_spec(item["spec"])
@dataclass(frozen=True, slots=True)
class Collector:
image_name_annotation_key: str
bigbang_version: str = ""

def bigbang_images(self) -> list[str]:
r = requests.get(f"{BIGBANG_IMAGES_URL}/{self.bigbang_version}/images.txt")
r.raise_for_status()

return [Image(image) for image in r.content.decode().splitlines()]

def cluster_images(self) -> set[Image]:
resources = ["pods", "jobs", "cronjobs"]
images = []
for resource in resources:
cmd = [
"kubectl",
"get",
f"{resource}",
"-A",
"-o",
"json",
]

result = subprocess.run(cmd, check=True, capture_output=True, text=True)
resource_json = json.loads(result.stdout)

for item in resource_json["items"]:
annotation = False
if resource == "pods":
resource_images = (
(
self._get_images_from_mutating_webhook_annotations(
item["metadata"]
)
)
if self.image_name_annotation_key
else []
)
if resource_images:
annotation = True
for image in resource_images:
images += [image] if image not in images else []
elif resource == "jobs":
resource_images = _get_images_from_nested_spec(
item["spec"]["template"]["spec"]
)
for image in resource_images:
images += [image] if image not in images else []
elif resource == "cronjobs":
resource_images = _get_images_from_nested_spec(
item["spec"]["jobTemplate"]["spec"]["template"]["spec"]
)
for image in resource_images:
images += [image] if image not in images else []
else:
log.error("Unsupported resource. Cannot be parsed")
return images


# parse images out of item
def _get_images_from_nested_spec(item_nested_spec) -> list[Image]:
"""
Call with:
for item in resource['items']:
image_set = _get_images_from_nested(item['spec']['blah'])
"""
images = []
for container in item_nested_spec["containers"]:
images.append(Image(container["image"]))
for container in item_nested_spec.get("initContainers", []):
images.append(Image(container["image"]))
return images


def _get_images_from_imageswap_annotations(item_metadata) -> list[Image]:
"""
Call with:
for item in resource_json['items']:
if resource == 'pods':
images += _get_images_from_imageswap_annoations(item['metadata'])
"""
# TODO: Update this annotation once imageswap is updated to allow for arbitrary annotation key
imageswap_re = re.compile(r"imageswap.ironbank.dso.mil/[0-9]+")

if "annotations" not in item_metadata:
return []

return [
Image(v)
for k, v in item_metadata["annotations"].items()
if imageswap_re.match(k)
]
# If the original images we not added to the image list from annotations, retrieve images from the spec as usual
if not annotation:
resource_images = self._get_images_from_nested_spec(
item["spec"]
)
for image in resource_images:
images += [image] if image not in images else []
elif resource == "jobs":
resource_images = self._get_images_from_nested_spec(
item["spec"]["template"]["spec"]
)
for image in resource_images:
images += [image] if image not in images else []
elif resource == "cronjobs":
resource_images = self._get_images_from_nested_spec(
item["spec"]["jobTemplate"]["spec"]["template"]["spec"]
)
for image in resource_images:
images += [image] if image not in images else []
else:
log.error("Unsupported resource. Cannot be parsed")
return images

# parse images out of item
def _get_images_from_nested_spec(self, item_nested_spec) -> list[Image]:
"""
Call with:
for item in resource['items']:
image_set = _get_images_from_nested(item['spec']['blah'])
"""
images = []
for container in item_nested_spec["containers"]:
images.append(Image(container["image"]))
for container in item_nested_spec.get("initContainers", []):
images.append(Image(container["image"]))
return images

def _get_images_from_mutating_webhook_annotations(
self, item_metadata
) -> list[Image]:
"""
Call with:
for item in resource_json['items']:
if resource == 'pods':
images += _get_images_from_mutating_webhook_annoations(item['metadata'])
"""
log.debug("Finding images using annotation %s", self.image_name_annotation_key)
mutating_webhook_re = re.compile(self.image_name_annotation_key)

if "annotations" not in item_metadata:
return []

return [
Image(v)
for k, v in item_metadata["annotations"].items()
if mutating_webhook_re.match(k)
]

0 comments on commit 4540a16

Please sign in to comment.