Merge pull request #1 from swade1987/age

fix: adding missing files

bin/check-for-unencrypted-secrets.sh

@@ -0,0 +1,39 @@
+#!/bin/bash
+
+function main {
+
+  clear
+
+  printf "Checking for unencrypted secrets in the 'secrets' directory ...\n\n"
+
+  FILES=$(find secrets -type f -name "*.yaml")
+  UNENCRYPTED_FILE_COUNT=0
+
+  # Loop around each file increase the "UNENCRYPTED_FILE_COUNT" variable if the file is not encrypted.
+  for file in $FILES; do
+
+    if [ "${file}" = "templates/platform-system-gcr-registry-creds.yaml" ]; then
+      continue
+    fi
+
+    # if file does not contain the word "sops" its not encrypted, therefore up the failure count.
+    IS_ENCRYPTED=$(yq eval '.sops | has("version")' "${file}")
+
+    if [ "${IS_ENCRYPTED}" != "true" ]; then
+      printf "FAIL -- %s\n" "$file"
+      ((UNENCRYPTED_FILE_COUNT=UNENCRYPTED_FILE_COUNT+1))
+    else
+      printf "OK -- %s\n" "$file"
+    fi
+
+  done
+
+  if [ "${UNENCRYPTED_FILE_COUNT}" -ne 0 ]; then
+    printf "\nFAILED -- %s file(s) were NOT encrypted.\n" "$UNENCRYPTED_FILE_COUNT"
+    exit 1
+  fi
+
+  exit 0
+}
+
+main

bin/decrypt-secrets.sh

@@ -0,0 +1,81 @@
+#!/bin/bash
+
+function get_clusters {
+    for dir in secrets/* ; do
+        echo -n "${dir##*/} "
+    done
+}
+
+function decrypt_secrets {
+
+    for secret in $(ag Secret "$secrets_dir" -l); do
+
+      IS_ENCRYPTED=$(yq eval '.sops | has("version")' "${secret}")
+
+      if [ "${IS_ENCRYPTED}" != "true" ]; then
+        printf "SKIPPED - %s already decrypted.\n" "${secret}"
+      else
+        sops -d -i "$secret"
+        printf "DECRYPTED - %s successfully decrypted.\n" "${secret}"
+      fi
+
+    done
+}
+
+# ======================================================================================================================
+
+if [[ "$1" == "" ]]; then
+    echo "Please provide an environment you want to seal (e.g. bh)"
+    exit
+fi
+
+if ! which sops > /dev/null 2>&1; then
+  echo 'Please install sops. On MacOS and Linux you can use "make initialise".'
+  exit
+fi
+
+if ! which yq > /dev/null 2>&1; then
+  echo 'Please install yq. On MacOS and Linux you can use "make initialise".'
+  exit
+fi
+
+if ! which ag > /dev/null 2>&1; then
+  echo 'Please install the_silver_searcher. On MacOS and Linux you can use "make initialise".'
+  exit
+fi
+
+# Make sure you have the latest version of sops installed.
+SOPS_VERSION_CHECK=$(sops --version | grep -c "3.6.1")
+if [ "${SOPS_VERSION_CHECK}" -ne 1 ]; then
+  echo 'Please install sops v3.6.1'
+  exit 1
+fi
+
+# Set a variable to specify the environment we are working on.
+clusters=$1
+
+if [[ ${clusters} == "all" ]]; then
+    clusters=$(get_clusters)
+    printf "\nclusters contains: %s \n\n" "${clusters}"
+fi
+
+for cluster in ${clusters}; do
+
+    secrets_dir="secrets/${cluster}"
+
+    # Handle the case when the directory does not exist
+    if [[ ! -e ${secrets_dir} ]]; then
+      printf "The directory %s does not exist, skipping ... \n" "${secrets_dir}"
+      continue
+    fi
+
+    # Handle the case when the directory is empty.
+    if [[ $(ag Secret "$secrets_dir" -l  | wc -l ) -eq 0 ]]; then
+       printf "The directory %s does not contain any secrets, skipping ... \n" "${secrets_dir}"
+       continue
+    fi
+
+    decrypt_secrets
+done
+
+printf "\nAll clusters completed"

bin/encrypt-secrets.sh

@@ -0,0 +1,115 @@
+#!/bin/bash
+function validation {
+    validate_secret_name "$1" && validate_image_pull_secret "$1"
+}
+
+function validate_secret_name {
+    FILE=$1
+
+    NAME=$(yq e --unwrapScalar=false '.metadata.name' "$FILE")
+    NAMESPACE=$(yq e --unwrapScalar=false '.metadata.namespace' "$FILE")
+
+    if [[ ! $(basename "$FILE") == "${NAMESPACE}-${NAME}.yaml" ]]; then
+        printf "WARN - Skipping secret %s; file name does not match <namespace>-<secret name>.yaml\n" "${secret}"
+        return 1
+    else
+        return 0
+    fi
+}
+
+function validate_image_pull_secret {
+    FILE=$1
+
+    if [[ $(basename "$FILE") == *"image-pull-secret.yaml" ]]; then
+        TYPE=$(yq e --unwrapScalar=false '.type' "$FILE")
+        if [[ "${TYPE}" != "kubernetes.io/dockerconfigjson" ]]; then
+            printf "WARN - Skipping secret %s; image pull secrets must be of type kubernetes.io/dockerconfigjson\n" "${secret}"
+            return 1
+        fi
+    fi
+    return 0
+}
+
+function get_clusters {
+    for dir in secrets/* ; do
+        echo -n "${dir##*/} "
+    done
+}
+
+function encrypt_secrets {
+
+    for secret in $(ag Secret "$secrets_dir" -l); do
+
+        if validation "${secret}"; then
+
+          # Gracefully handle when secret has already been encrypted.
+          IS_ENCRYPTED=$(yq eval '.sops | has("version")' "${secret}")
+
+          if [ "${IS_ENCRYPTED}" != "true" ]; then
+             # Use sops to encrypt the secret  with the same name.
+            sops -e -i "$secret"
+            printf "ENCRYPTED - %s successfully encrypted.\n" "${secret}"
+          else
+            printf "SKIPPED - %s already encrypted.\n" "${secret}"
+          fi
+        fi
+    done
+}
+
+# ======================================================================================================================
+
+if [[ "$1" == "" ]]; then
+    echo "Please provide an environment you want to seal (e.g. bh)"
+    exit
+fi
+
+if ! which sops > /dev/null 2>&1; then
+  echo 'Please install sops. On MacOS and Linux you can use "make initialise".'
+  exit
+fi
+
+if ! which yq > /dev/null 2>&1; then
+  echo 'Please install yq. On MacOS and Linux you can use "make initialise".'
+  exit
+fi
+
+if ! which ag > /dev/null 2>&1; then
+  echo 'Please install the_silver_searcher. On MacOS and Linux you can use "make initialise".'
+  exit
+fi
+
+# Make sure you have the latest version of sops installed.
+SOPS_VERSION_CHECK=$(sops --version | grep -c "3.6.1")
+if [ "${SOPS_VERSION_CHECK}" -ne 1 ]; then
+  echo 'Please install sops v3.6.1'
+  exit 1
+fi
+
+# Set a variable to specify the environment we are working on.
+clusters=$1
+
+if [[ ${clusters} == "all" ]]; then
+    clusters=$(get_clusters)
+    printf "\nclusters contains: %s \n\n" "${clusters}"
+fi
+
+for cluster in ${clusters}; do
+
+    secrets_dir="secrets/${cluster}"
+
+    # Handle the case when the directory does not exist
+    if [[ ! -e ${secrets_dir} ]]; then
+      printf "The directory %s does not exist, skipping ... \n" "${secrets_dir}"
+      continue
+    fi
+
+    # Handle the case when the directory is empty.
+    if [[ $(ag Secret "$secrets_dir" -l  | wc -l ) -eq 0 ]]; then
+       printf "The directory %s does not contain any secrets, skipping ... \n" "${secrets_dir}"
+       continue
+    fi
+
+    encrypt_secrets
+done
+
+printf "\nAll clusters completed"

bin/pre-commit-check-for-unencrypted-secrets.sh

@@ -0,0 +1,47 @@
+#!/bin/bash
+set -e
+
+function check_dependencies() {
+    if ! command -v yq &> /dev/null; then
+        echo "❌ Error: yq is not installed. Please install yq first."
+        exit 1
+    fi
+}
+
+function main {
+    # Find all yaml files in secrets directory
+    FILES=$(find secrets -type f -name "*.yaml" 2>/dev/null | sort -u)
+
+    if [ -z "$FILES" ]; then
+        echo "⚠️  Warning: No YAML files found in secrets directory"
+        exit 0
+    fi
+
+    unencrypted_count=0
+    UNENCRYPTED_FILES=""
+
+    while IFS= read -r file; do
+        IS_ENCRYPTED=$(yq eval '.sops | has("version")' "${file}")
+        if [ "${IS_ENCRYPTED}" = "false" ]; then
+            UNENCRYPTED_FILES+="   - $file"$'\n'
+            ((unencrypted_count++))
+        fi
+    done <<< "$FILES"
+
+    if [ $unencrypted_count -gt 0 ]; then
+        echo "❌ Error: Found $unencrypted_count unencrypted secret file(s):"
+        echo
+        echo -n "$UNENCRYPTED_FILES"
+        echo "Please encrypt these files using SOPS before committing."
+        exit 1
+    fi
+
+    echo "✅ All secret files are properly encrypted!"
+    exit 0
+}
+
+# Check for required dependencies
+check_dependencies
+
+# Run main function
+main

docs/adding-new-cluster.md

@@ -0,0 +1,21 @@
+# Adding a new environment
+
+The following page describes the steps required to add a new cluster to this repository.
+
+## 1. Adding a new directory within our .sops.yaml file
+
+The majority of the configuration is within the [`.sops.yaml`](../.sops.yaml) file.
+
+For you to add a new cluster it is recommended you copy a `path_regex` block that is already existing.
+
+The following changes are required within the [`.sops.yaml`](../.sops.yaml) file.
+
+### Change location
+
+Change the location to look within to be `secrets/<new directory>`.
+
+The directory can should follow the naming convention `<region>-<team>-<environment>` e.g. `us-west-2-platform-engineering-prd`.
+
+### Configure the correct IAM role arn
+
+The `arn` under `kms` needs to be the arn for the specific KMS key used for encryption/decryption.

docs/deployment.md

@@ -0,0 +1,79 @@
+# Deployment mechanism
+
+As you are probably aware we use the GitOps controller [Flux](https://github.com/fluxcd/flux2) to sync workloads into our clusters. This repository is no different in this regard.
+
+Flux has in-built support for [Mozilla](https://github.com/getsops/sops) for more information see [here](https://toolkit.fluxcd.io/guides/mozilla-sops/).
+
+## Technical overview
+
+Each cluster has been configured with a KMS key specifically for SOPs encryption. For more information on KMS, see [here](https://aws.amazon.com/kms/).
+
+Additionally, each cluster has an IAM role `flux-secrets` which has the ability to encrypt and decrypt using this key.
+
+### Flux configuration
+
+When configuring our Flux instance we specify an annotation on the pod to allow it to assume the role `flux-secrets`.
+
+```
+apiVersion: kustomize.config.k8s.io/v1
+kind: Kustomization
+resources:
+  - gotk-components.yaml
+  - gotk-sync.yaml
+patches:
+  # Patch kustomize-controller deployment with annotation to allow assume role to import secrets
+  - patch: |
+      apiVersion: apps/v1
+      kind: Deployment
+      metadata:
+        name: kustomize-controller
+        namespace: flux-system
+      spec:
+        template:
+          metadata:
+            annotations:
+              iam.amazonaws.com/role: flux-secrets-assume-us-west-2
+    target:
+      kind: Deployment
+      name: kustomize-controller
+      namespace: flux-system
+      apiVersion: kustomize.config.k8s.io/v1
+```
+
+Finally, we have to configure flux to be aware this repository leverages SOPs (see below)
+
+```
+apiVersion: kustomize.toolkit.fluxcd.io/v1
+kind: Kustomization
+metadata:
+  name: k8s-secrets
+  namespace: flux-repos
+spec:
+  interval: 10m0s
+  sourceRef:
+    kind: GitRepository
+    name: k8s-secrets
+  prune: true
+  decryption:
+    provider: sops
+```
+
+### Flux reconciliation
+
+When the flux instance reconciles the repository it looks at the directory its secrets are stored within (e.g. `secrets/platform-engineering-sbx`).
+
+It then looks in the [`.sops.yaml`](../.sops.yaml) file for a path that matches the directory it's reconciling (see below)
+
+```
+  - path_regex: secrets/us-west-2-platform-engineering-sbx
+    encrypted_regex: "^(data|stringData)$"
+    shamir_threshold: 1
+    key_groups:
+      - kms:
+          - arn: arn:aws:kms:us-west-2:<redacted>:key/<redacted>
+            role: arn:aws:iam::<redacted>:role/flux-secrets
+```
+
+It then uses the KMS key arn (listed above as `arn`) to decrypt the encrypted portion of the secrets within that directory.
+
+You will notice the `role` key in the block above needs to be the ARN of the role that our flux instance assumes.