From c15755fd0fd015d608b4257328c580bca50b5d1c Mon Sep 17 00:00:00 2001 From: Otavio Fernandes Date: Mon, 24 Jun 2024 11:01:40 +0200 Subject: [PATCH 1/2] feat: Kubernetes Resource Amend Script --- .../rhtap-infrastructure/scripts/oc-amend.sh | 1 + scripts/oc-amend.sh | 75 +++++++++++++++++++ 2 files changed, 76 insertions(+) create mode 120000 charts/rhtap-infrastructure/scripts/oc-amend.sh create mode 100755 scripts/oc-amend.sh diff --git a/charts/rhtap-infrastructure/scripts/oc-amend.sh b/charts/rhtap-infrastructure/scripts/oc-amend.sh new file mode 120000 index 00000000..95b56177 --- /dev/null +++ b/charts/rhtap-infrastructure/scripts/oc-amend.sh @@ -0,0 +1 @@ +../../../scripts/oc-amend.sh \ No newline at end of file diff --git a/scripts/oc-amend.sh b/scripts/oc-amend.sh new file mode 100755 index 00000000..75782e37 --- /dev/null +++ b/scripts/oc-amend.sh @@ -0,0 +1,75 @@ +#!/usr/bin/env bash + +shopt -s inherit_errexit +set -Eeu -o pipefail + +# Subcommand to be use in combination with "oc" (or "kubectl"). +declare -r SUBCOMMAND="${1:-}" +# Remove the first argument from the list. +shift +# Storing the remaining arguments in an array, they represent the final arguments +# to be informed to the "oc" command and subcommand. +declare -r ARGS=("${@}") + +# Kubernetes kind to be used in the "oc" command. +declare -r KIND="${KIND:-}" +# Kubernetes namespace. +declare -r NAMESPACE="${NAMESPACE:-}" +# Kubernetes resource name. +declare -r RESOURCE_NAME="${RESOURCE_NAME:-}" + +fail() { + echo "# [ERROR] ${*}" >&2 + exit 1 +} + +warn() { + echo "# [WARN] ${*}" >&2 +} + +# Asserting that the required variables are set. +assert_variables() { + [[ -z "${KIND}" ]] && + fail "KIND is not set!" + [[ -z "${NAMESPACE}" ]] && + warn "NAMESPACE is not set!" + [[ -z "${RESOURCE_NAME}" ]] && + fail "RESOURCE_NAME is not set!" +} + +# Amends a Kubernetes resource using the "oc" command. +oc_amend() { + echo "# Trying to '${SUBCOMMAND}' the resource '${KIND}/${RESOURCE_NAME}' " \ + "with '${ARGS[*]}'..." + set -x + oc "${SUBCOMMAND}" "${KIND}" "${RESOURCE_NAME}" \ + --namespace="${NAMESPACE}" \ + "${ARGS[@]}" + local ret_code=${?} + set +x + return ${ret_code} +} + +# Retries the "oc_amend" function up to 30 times. +retry_oc_amend() { + echo "# Kubernetes namespace: '${NAMESPACE}'" + for i in {1..30}; do + oc_amend && + return 0 + + wait=$((i * 5)) + echo "### [${i}/30] Waiting for ${wait} seconds before retrying..." + sleep ${wait} + done + return 1 +} + +# +# Main +# + +if retry_oc_amend; then + echo "# [INFO] Successfully amended Kubernetes resource." +else + fail "Failed to amend Kubernetes resource!" +fi From aa2d1ae7c54e69823fa6bf3180b75cea3f147576 Mon Sep 17 00:00:00 2001 From: Otavio Fernandes Date: Mon, 24 Jun 2024 11:02:23 +0200 Subject: [PATCH 2/2] feat: OpenShift Pipelines Configuration Moving the Tekton Chains configuration to the infrastructure chart, replacing the pre-deploy hook with a post-deploy job that will create the Tekton Chains resources. Improved the Helm chart testing to wait for OpenShift Pipelines to be ready, and using official Chainguard images to run `cosign.` --- .../hooks/pre-deploy.sh | 61 -------------- .../openshift-gitops/job-post-deploy.yaml | 2 +- charts/rhtap-infrastructure/Chart.yaml | 2 +- .../rhtap-infrastructure/templates/NOTES.txt | 44 +++++++--- .../templates/_copy-scripts.tpl | 25 ++++++ .../job-tekton-chains.yaml | 52 ++++++++++++ .../job-tekton-config.yaml | 82 +++++++++++++++++++ .../openshift-pipelines/service-account.yaml | 74 +++++++++++++++++ .../templates/tests/test.yaml | 45 ++++++---- charts/rhtap-infrastructure/values.yaml | 31 ++++++- charts/values.yaml.tpl | 8 ++ 11 files changed, 333 insertions(+), 93 deletions(-) delete mode 100755 charts/rhtap-backing-services/hooks/pre-deploy.sh create mode 100644 charts/rhtap-infrastructure/templates/_copy-scripts.tpl create mode 100644 charts/rhtap-infrastructure/templates/openshift-pipelines/job-tekton-chains.yaml create mode 100644 charts/rhtap-infrastructure/templates/openshift-pipelines/job-tekton-config.yaml create mode 100644 charts/rhtap-infrastructure/templates/openshift-pipelines/service-account.yaml diff --git a/charts/rhtap-backing-services/hooks/pre-deploy.sh b/charts/rhtap-backing-services/hooks/pre-deploy.sh deleted file mode 100755 index ed609745..00000000 --- a/charts/rhtap-backing-services/hooks/pre-deploy.sh +++ /dev/null @@ -1,61 +0,0 @@ -#!/usr/bin/env bash - -shopt -s inherit_errexit -set -Eeu -o pipefail - -make_helm_managed() { - KIND="$1" - NAMESPACE="$2" - NAME="$3" - - echo -n "." - kubectl annotate "$KIND" "$NAME" meta.helm.sh/release-name=rhtap-backing-services >/dev/null - echo -n "." - kubectl annotate "$KIND" "$NAME" meta.helm.sh/release-namespace="$NAMESPACE" >/dev/null - echo -n "." - kubectl label "$KIND" "$NAME" app.kubernetes.io/managed-by=Helm >/dev/null -} - -openshift_pipelines() { - ### Workaround - ### OpenShift Pipelines cannot be configured before the operator is deployed, - ### as it is creating the TektonConfig resource, which default content cannot - ### be modified at install time. - # Allow the TektonConfig resource to be managed by Helm - echo -n "* Configuring OpenShift Pipelines: " - make_helm_managed "TektonConfig" "rhtap" "config" - echo "OK" - - ### Workaround - ### OpenShift Pipelines cannot be configured to generate the signing secret - ### at deployment time. - echo -n "* Configuring Chains secret: " - NAMESPACE="openshift-pipelines" - SECRET="signing-secrets" - if [ "$(kubectl get secret -n "$NAMESPACE" "$SECRET" -o jsonpath='{.data}' --ignore-not-found --allow-missing-template-keys)" == "" ]; then - # Delete the empty secret/signing-secrets - echo -n "." - kubectl delete secrets -n "$NAMESPACE" "$SECRET" --ignore-not-found=true >/dev/null - - # To run without user input, generate a password - echo -n "." - COSIGN_PASSWORD=$( openssl rand -base64 30 ) - export COSIGN_PASSWORD - - # Generate the key pair secret directly in the cluster. - # The secret will be created as immutable. - echo -n "." - cosign generate-key-pair "k8s://$NAMESPACE/$SECRET" >/dev/null 2>&1 - rm cosign.pub - fi - echo "OK" -} - -# -# Main -# -main() { - openshift_pipelines -} - -main diff --git a/charts/rhtap-backing-services/templates/openshift-gitops/job-post-deploy.yaml b/charts/rhtap-backing-services/templates/openshift-gitops/job-post-deploy.yaml index 76482e67..96ec2342 100644 --- a/charts/rhtap-backing-services/templates/openshift-gitops/job-post-deploy.yaml +++ b/charts/rhtap-backing-services/templates/openshift-gitops/job-post-deploy.yaml @@ -1,6 +1,6 @@ {{- if .Values.backingServices.argoCD.enabled }} {{- $argoCD := .Values.backingServices.argoCD }} - {{- $name := printf "%s-post-deploy" $argoCD.name }} + {{- $name := printf "%s-post-deploy-%d" $argoCD.name .Release.Revision }} {{- $argoCDEnvFile := "/rhtap/argocd/env" -}} # # Generates the ArgoCD API token and stores it on a Kubernetes secret. The steps diff --git a/charts/rhtap-infrastructure/Chart.yaml b/charts/rhtap-infrastructure/Chart.yaml index 981efd9f..79ac7b0d 100644 --- a/charts/rhtap-infrastructure/Chart.yaml +++ b/charts/rhtap-infrastructure/Chart.yaml @@ -3,4 +3,4 @@ apiVersion: v2 name: rhtap-infrastructure description: RHTAP Infrastructure type: application -version: 0.0.1 +version: 0.0.2 diff --git a/charts/rhtap-infrastructure/templates/NOTES.txt b/charts/rhtap-infrastructure/templates/NOTES.txt index efe122a8..d2c6b969 100644 --- a/charts/rhtap-infrastructure/templates/NOTES.txt +++ b/charts/rhtap-infrastructure/templates/NOTES.txt @@ -1,17 +1,17 @@ MinIO Tenants: {{- range $k, $v := include "infrastructure.minIOTentants.enabled" . | fromYaml }} - - Name: {{ $k }} - Namespace: {{ $v.namespace }} - Root Credentials: {{ $v.rootSecretName }} (Secret) + - Name: {{ $k }} + Namespace: {{ $v.namespace }} + Root Credentials: {{ $v.rootSecretName }} (Secret) Storage User Credentials: {{ $v.storageUserSecretName }} (Secret) {{- end }} -Kafkas: +Kafka Brokers: {{- range $k, $v := include "infrastructure.kafkas.enabled" . | fromYaml }} - - Name: {{ $k }} - Namespace: {{ $v.namespace }} - Username: {{ $v.username }} - Credentials: {{ $v.username }} (Secret) + - Name: {{ $k }} + Namespace: {{ $v.namespace }} + Username: {{ $v.username }} + Credentials: {{ $v.username }} (Secret) Topics: {{- range $t := $v.topics }} - {{ $t.name }} @@ -20,7 +20,29 @@ Kafkas: PostgreSQL Clusters: {{- range $k, $v := include "infrastructure.postgresClusters.enabled" . | fromYaml }} - - Namespace: {{ $v.namespace }} - Name: {{ $k }} - Version: {{ $v.postgresVersion }} + - Namespace: {{ $v.namespace }} + Name: {{ $k }} + Version: {{ $v.postgresVersion }} +{{- end }} + +{{- if .Values.infrastructure.openShiftPipelines.enabled }} + {{- $osp := .Values.infrastructure.openShiftPipelines -}} +OpenShift Pipelines: + {{- if or + $osp.patchClusterTektonConfig.annotations + $osp.patchClusterTektonConfig.labels + }} + Tekton Config Patch: + - Name: {{ $osp.patchClusterTektonConfig.resourceName }} + {{- with $osp.patchClusterTektonConfig.annotations }} + Annotations: + {{ . | toYaml | indent 8 }} + {{- end }} + {{- with $osp.patchClusterTektonConfig.labels }} + Labels: + {{ . | toYaml | indent 8 }} + {{- end }} + {{- end }} + Tekton Chains: + - Signing Keys Secret: {{ $osp.tektonChains.signingSecretName }} {{- end }} diff --git a/charts/rhtap-infrastructure/templates/_copy-scripts.tpl b/charts/rhtap-infrastructure/templates/_copy-scripts.tpl new file mode 100644 index 00000000..2775e69c --- /dev/null +++ b/charts/rhtap-infrastructure/templates/_copy-scripts.tpl @@ -0,0 +1,25 @@ +{{/* + + POD container spec to copy scripts. + +*/}} +{{- define "infrastructure.copyScripts" -}} +- name: copy-scripts + image: registry.access.redhat.com/ubi8/ubi-minimal:latest + workingDir: /scripts + command: + - /bin/bash + - -c + - | + set -x -e + {{- range $path, $content := .Files.Glob "scripts/*.sh" -}} + {{- $script := trimPrefix "scripts/" $path }} + printf '%s' "{{ $content | toString | b64enc }}" | base64 -d >{{ $script }} + chmod +x {{ $script }} + {{- end }} + volumeMounts: + - name: scripts + mountPath: /scripts + securityContext: + allowPrivilegeEscalation: false +{{- end -}} diff --git a/charts/rhtap-infrastructure/templates/openshift-pipelines/job-tekton-chains.yaml b/charts/rhtap-infrastructure/templates/openshift-pipelines/job-tekton-chains.yaml new file mode 100644 index 00000000..95a5fba3 --- /dev/null +++ b/charts/rhtap-infrastructure/templates/openshift-pipelines/job-tekton-chains.yaml @@ -0,0 +1,52 @@ +{{- if .Values.infrastructure.openShiftPipelines.enabled }} + {{- $osp := .Values.infrastructure.openShiftPipelines -}} + {{- $signingSecretName := required + ".infrastructure.openShiftPipelines.tektonChains.signingSecretName" + $osp.tektonChains.signingSecretName + -}} + {{- $secretObj := ( + lookup "v1" "Secret" $osp.namespace $signingSecretName + ) | default dict + -}} + {{- $secretData := (get $secretObj "data") | default dict -}} + {{- $cosignKey := (get $secretData "cosign.key") | default "" -}} + {{- if eq $cosignKey "" }} +--- +apiVersion: batch/v1 +kind: Job +metadata: + annotations: + helm.sh/hook: post-install,post-upgrade + helm.sh/hook-delete-policy: hook-succeeded + helm.sh/hook-weight: "2" + labels: + {{- include "rhtap-infrastructure.labels" . | nindent 4 }} + namespace: {{ $osp.namespace }} + name: {{ printf "cosign-%s-%d" $osp.name .Release.Revision }} +spec: + template: + spec: + serviceAccountName: {{ printf "patch-%s" $osp.name }} + restartPolicy: Never + containers: + - name: tekton-chains-cosign + image: ghcr.io/sigstore/cosign/cosign:latest + env: + - name: COSIGN_PASSWORD + value: {{ randAlphaNum 32 }} + workingDir: /workspace + command: + - cosign + args: + - generate-key-pair + - {{ printf "k8s://%s/%s" $osp.namespace $signingSecretName }} + volumeMounts: + - name: workspace + mountPath: /workspace + securityContext: + allowPrivilegeEscalation: false + volumes: + - name: workspace + emppyDir: {} + {{- end }} +{{- end }} diff --git a/charts/rhtap-infrastructure/templates/openshift-pipelines/job-tekton-config.yaml b/charts/rhtap-infrastructure/templates/openshift-pipelines/job-tekton-config.yaml new file mode 100644 index 00000000..2590fc05 --- /dev/null +++ b/charts/rhtap-infrastructure/templates/openshift-pipelines/job-tekton-config.yaml @@ -0,0 +1,82 @@ +{{- if .Values.infrastructure.openShiftPipelines.enabled }} + {{- $osp := .Values.infrastructure.openShiftPipelines -}} + {{- if or + $osp.patchClusterTektonConfig.annotations + $osp.patchClusterTektonConfig.labels + }} +--- +apiVersion: batch/v1 +kind: Job +metadata: + annotations: + helm.sh/hook: post-install,post-upgrade + helm.sh/hook-delete-policy: hook-succeeded + helm.sh/hook-weight: "1" + labels: + {{- include "rhtap-infrastructure.labels" . | nindent 4 }} + namespace: {{ $osp.namespace }} + name: {{ printf "patch-tekton-%s-%d" $osp.name .Release.Revision }} +spec: + template: + spec: + serviceAccountName: {{ printf "patch-%s" $osp.name }} + restartPolicy: Never + containers: + # + # Copying the scripts that will be used on the subsequent containers, the + # scripts are shared via the "/scripts" volume. + # + {{- include "infrastructure.copyScripts" . | nindent 8 }} + {{- if $osp.patchClusterTektonConfig.annotations }} + # + # Patch the Tekton Config with the provided annotations. + # + - name: patch-annotations + image: quay.io/codeready-toolchain/oc-client-base:latest + env: + - name: KIND + value: tektonconfig + - name: RESOURCE_NAME + value: {{ $osp.patchClusterTektonConfig.resourceName }} + command: + - /scripts/oc-amend.sh + args: + - annotate + {{- range $k, $v := $osp.patchClusterTektonConfig.annotations }} + - {{ printf "%s=%s" $k $v | quote }} + {{- end }} + volumeMounts: + - name: scripts + mountPath: /scripts + securityContext: + allowPrivilegeEscalation: false + {{- end }} + {{- if $osp.patchClusterTektonConfig.labels }} + # + # Patch the Tekton Config with the provided labels. + # + - name: patch-labels + image: quay.io/codeready-toolchain/oc-client-base:latest + env: + - name: KIND + value: tektonconfig + - name: RESOURCE_NAME + value: {{ $osp.patchClusterTektonConfig.resourceName }} + command: + - /scripts/oc-amend.sh + args: + - label + {{- range $k, $v := $osp.patchClusterTektonConfig.labels }} + - {{ printf "%s=%s" $k $v | quote }} + {{- end }} + volumeMounts: + - name: scripts + mountPath: /scripts + securityContext: + allowPrivilegeEscalation: false + {{- end }} + volumes: + - name: scripts + emppyDir: {} + {{- end }} +{{- end }} diff --git a/charts/rhtap-infrastructure/templates/openshift-pipelines/service-account.yaml b/charts/rhtap-infrastructure/templates/openshift-pipelines/service-account.yaml new file mode 100644 index 00000000..ab6e3a97 --- /dev/null +++ b/charts/rhtap-infrastructure/templates/openshift-pipelines/service-account.yaml @@ -0,0 +1,74 @@ +{{- if .Values.infrastructure.openShiftPipelines.enabled }} + {{- $osp := .Values.infrastructure.openShiftPipelines -}} + {{- $name := printf "patch-%s" $osp.name }} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ $name }} + namespace: {{ $osp.namespace }} + +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + namespace: {{ $osp.namespace }} + name: {{ $name }} +rules: + - apiGroups: + - "" + resources: + - secrets + verbs: + - get + - list + - create + +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + namespace: {{ $osp.namespace }} + name: {{ $name }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: {{ $name }} +subjects: + - kind: ServiceAccount + namespace: {{ $osp.namespace }} + name: {{ $name }} + +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ $name }} +rules: + - apiGroups: + - operator.tekton.dev + resources: + - tektonconfigs + verbs: + - get + - list + - watch + - create + - update + - patch + - delete + +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ $name }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ $name }} +subjects: + - kind: ServiceAccount + namespace: {{ $osp.namespace }} + name: {{ $name }} +{{- end }} diff --git a/charts/rhtap-infrastructure/templates/tests/test.yaml b/charts/rhtap-infrastructure/templates/tests/test.yaml index 2c395935..ce6ebdfc 100644 --- a/charts/rhtap-infrastructure/templates/tests/test.yaml +++ b/charts/rhtap-infrastructure/templates/tests/test.yaml @@ -13,24 +13,11 @@ spec: restartPolicy: Never serviceAccountName: {{ .Release.Name }} containers: - - name: copy-scripts - image: registry.access.redhat.com/ubi8/ubi-minimal:latest - workingDir: /scripts - command: - - /bin/bash - - -c - - | - set -x -e -{{- range $path, $content := .Files.Glob "scripts/*.sh" -}} - {{- $name := trimPrefix "scripts/" $path }} - printf '%s' "{{ $content | toString | b64enc }}" | base64 -d >{{ $name }} - chmod +x {{ $name }} -{{- end }} - volumeMounts: - - name: scripts - mountPath: /scripts - securityContext: - allowPrivilegeEscalation: false + # + # Copying the scripts that will be used on the subsequent containers, the + # scripts are shared via the "/scripts" volume. + # +{{- include "infrastructure.copyScripts" . | nindent 4 }} {{- range $k, $v := include "infrastructure.kafkas.enabled" . | fromYaml }} - name: {{ printf "%s-kafka-topics-%s" $name $k }} image: quay.io/codeready-toolchain/oc-client-base:latest @@ -89,6 +76,28 @@ spec: mountPath: /scripts securityContext: allowPrivilegeEscalation: false +{{- end }} +{{- if .Values.infrastructure.openShiftPipelines.enabled }} + # + # Tests the OpenShift Pipelines rollout status. + # + {{- $osp := .Values.infrastructure.openShiftPipelines }} + - name: {{ printf "openshift-pipelines-%s" $osp.name }} + image: quay.io/codeready-toolchain/oc-client-base:latest + env: + - name: NAMESPACE + value: {{ $osp.namespace }} + - name: RESOURCE_TYPE + value: "deployment" + command: + - /scripts/test-rollout-status.sh + args: + - {{ printf "app.kubernetes.io/part-of=%s" $osp.name | quote }} + volumeMounts: + - name: scripts + mountPath: /scripts + securityContext: + allowPrivilegeEscalation: false {{- end }} volumes: - name: scripts diff --git a/charts/rhtap-infrastructure/values.yaml b/charts/rhtap-infrastructure/values.yaml index e2783b34..54e46e8c 100644 --- a/charts/rhtap-infrastructure/values.yaml +++ b/charts/rhtap-infrastructure/values.yaml @@ -1,5 +1,8 @@ --- infrastructure: + # + # Kafka Brokers + # kafkas: tpa: enabled: false @@ -15,7 +18,7 @@ infrastructure: version: 3.6.0 config: default.replication.factor: 1 - inter.broker.protocol.version: '3.6' + inter.broker.protocol.version: "3.6" min.insync.replicas: 1 offsets.topic.replication.factor: 1 transaction.state.log.min.isr: 1 @@ -110,6 +113,9 @@ infrastructure: resources: requests: storage: 500Mi + # + # PostgreSQL Clusters + # postgresClusters: keycloak: enabled: false @@ -165,3 +171,26 @@ infrastructure: pg_hba: - host all all 0.0.0.0/0 scram-sha-256 - host all all ::1/128 scram-sha-256 + # + # OpenShift Pipelines + # + openShiftPipelines: + # Toggles the settings related to OpenShift Pipelines components. + enabled: false + # The primary OpenShift Pipelines instance name. + name: tekton-pipeline + # The namespace where the OpenShift Pipelines components is installed. + namespace: openshift-pipelines + # Patches the global TektonConfig resource. + patchClusterTektonConfig: + # Resource name to patch. + resourceName: config + # Annotations to add to the resource. + annotations: {} + # Labels to add to the resource. + labels: {} + # Tekton Chains settings. + tektonChains: + # Secret name where the signing key is stored, the password randomly + # generated and stored on the same secret. + signingSecretName: signing-secrets diff --git a/charts/values.yaml.tpl b/charts/values.yaml.tpl index f2e4a3d9..e9ffad86 100644 --- a/charts/values.yaml.tpl +++ b/charts/values.yaml.tpl @@ -104,6 +104,14 @@ infrastructure: guac: enabled: {{ $tpa.Enabled }} namespace: {{ $tpa.Namespace }} + openShiftPipelines: + enabled: {{ $rhdh.Enabled }} + patchClusterTektonConfig: + annotations: + meta.helm.sh/release-name: rhtap-backing-services + meta.helm.sh/release-namespace: {{ $argoCDNamespace }} + labels: + app.kubernetes.io/managed-by: Helm # # rhtap-backing-services