From fa70b291ab468dae44e999d00ac334859c2224b1 Mon Sep 17 00:00:00 2001 From: Christina Harlow Date: Mon, 25 Oct 2021 10:22:41 -0400 Subject: [PATCH 1/3] scrub readme of mozilla-it refs --- README.md | 23 +++-------------------- 1 file changed, 3 insertions(+), 20 deletions(-) diff --git a/README.md b/README.md index ce3ed09..ec52649 100644 --- a/README.md +++ b/README.md @@ -1,27 +1,10 @@ # helm-charts -[![Build Status](https://travis-ci.com/mozilla-it/helm-charts.svg?branch=main)](https://travis-ci.com/mozilla-it/helm-charts) - -This repository contains a collection of Helm charts curated by Mozilla's Service Engineering team - -## Adding more charts - -## Compiling and publishing charts - -## Referencing images in ECR -Charts referencing Docker Images stored in an ECR repo under `itsre-apps` subaccount can't be pulled without credentials. -Travis is configured to get Pull credentials for repositories inside that account. -In order to instruct the testing suite to -use those credentials you have to create a folder named `ci` inside your chart, a file `test-values.yaml` inside it with the next content: -``` -imagePullSecrets: - - name: ecr-registry -``` -Check [here](https://github.com/mozilla-it/helm-charts/pull/39/commits/1a0fbfed5810a6d6875ca0172adac5065ee03b74#diff-245000fef8fab28267cb8040d6a3d7f6) for an example. +This repository contains a collection of Helm charts used in the Common Voice project. ## Installing Helm Charts from this repository This repository is serving Helm Charts using the webserver provided by Github pages. In order to install Helm charts in your cluster -add this repository to your helm repository list running `helm repo add mozilla-helm-charts https://mozilla-it.github.io/helm-charts/` +add this repository to your helm repository list running `helm repo add cv-helm-charts https://common-voice.github.io/helm-charts/` ## Debug a Chart deployment This section describes how to verify that your chart is installed correctly as well as what to do if it is not. @@ -53,7 +36,7 @@ The same information found out above can be found in the logs of the 2 component Running `kubectl logs -l=app=flux -n fluxcd` you will get the logs of Flux where we can make sure that it correctly detected the change in the Chart and will try to apply it. ``` kubectl logs -l=app=flux -n fluxcd -ts=2020-04-17T09:24:41.796293402Z caller=loop.go:133 component=sync-loop event=refreshed url=ssh://git@github.com/mozilla-it/voice-infra branch=main HEAD=26626653ecca80f3d43f6e42aa3376af15755622 +ts=2020-04-17T09:24:41.796293402Z caller=loop.go:133 component=sync-loop event=refreshed url=ssh://git@github.com/common-voice/voice-infra branch=main HEAD=26626653ecca80f3d43f6e42aa3376af15755622 ``` Nothing wrong here, moving on and checking helm-operator logs From 189df05d8ffc1abe236520506959d178ee5542b4 Mon Sep 17 00:00:00 2001 From: Christina Harlow Date: Mon, 25 Oct 2021 10:49:01 -0400 Subject: [PATCH 2/3] move helm release from old travis sh to GHA --- .github/workflows/release.yml | 33 +++++++++++++++++++++++++++++++++ ci/release.sh | 25 ------------------------- 2 files changed, 33 insertions(+), 25 deletions(-) create mode 100644 .github/workflows/release.yml delete mode 100755 ci/release.sh diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 0000000..232a130 --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,33 @@ +name: Release Charts + +on: + push: + branches: + - main + +jobs: + release: + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v2 + with: + fetch-depth: 0 + + - name: Configure Git + run: | + git config user.name "$GITHUB_ACTOR" + git config user.email "$GITHUB_ACTOR@users.noreply.github.com" + + - name: Install Helm + uses: azure/setup-helm@v1 + with: + version: v3.4.0 + + - name: Run chart-releaser + uses: helm/chart-releaser-action@v1.2.1 + with: + charts_dir: charts + charts_repo_url: https://common-voice.github.io/helm-charts/ + env: + CR_TOKEN: "${{ secrets.GITHUB_TOKEN }}" diff --git a/ci/release.sh b/ci/release.sh deleted file mode 100755 index 2526bdf..0000000 --- a/ci/release.sh +++ /dev/null @@ -1,25 +0,0 @@ -#!/bin/bash - -set -o errexit -set -o nounset -set -o pipefail - -readonly REPO_ROOT=$(git rev-parse --show-toplevel) - -readonly CHARTS_DIR="${REPO_ROOT}/charts" -readonly RELEASES_DIR="${REPO_ROOT}/releases" - -mkdir -p "${RELEASES_DIR}" - -cd "${CHARTS_DIR}" -for i in $(find . -maxdepth 1 -mindepth 1 -type d) ; do - helm lint "${i}" - helm dependency update "${i}" - helm package "${i}" -d "${RELEASES_DIR}" -done -cd .. - -git checkout -f gh-pages -mv ./*.tgz "${RELEASES_DIR}" -cd "${RELEASES_DIR}" -helm repo index --url https://mozilla-it.github.io/helm-charts/ --merge ../index.yaml . From 2b7f3231487199e774f95c440eaa6249969e7234 Mon Sep 17 00:00:00 2001 From: Christina Harlow Date: Mon, 25 Oct 2021 15:17:03 -0400 Subject: [PATCH 3/3] move CI to GHA --- {ci => .github}/ct.yaml | 4 +- .github/workflows/ci.yml | 44 +++++++++++ .github/workflows/release.yml | 13 ++- charts/mozilla-common-voice/Chart.yaml | 15 ++-- charts/sentence-collector/Chart.yaml | 13 +-- ci/deps/careers/install.sh | 19 ----- ci/deps/ctms/install.sh | 19 ----- ci/deps/discourse/install.sh | 31 -------- ci/deps/install.sh | 9 --- ci/e2e.sh | 105 ------------------------- ci/functions.sh | 27 ------- ci/kind.yaml | 5 -- ci/lint.sh | 22 ------ ci/setup.sh | 23 ------ 14 files changed, 71 insertions(+), 278 deletions(-) rename {ci => .github}/ct.yaml (82%) create mode 100644 .github/workflows/ci.yml delete mode 100755 ci/deps/careers/install.sh delete mode 100755 ci/deps/ctms/install.sh delete mode 100644 ci/deps/discourse/install.sh delete mode 100755 ci/deps/install.sh delete mode 100755 ci/e2e.sh delete mode 100644 ci/functions.sh delete mode 100644 ci/kind.yaml delete mode 100755 ci/lint.sh delete mode 100755 ci/setup.sh diff --git a/ci/ct.yaml b/.github/ct.yaml similarity index 82% rename from ci/ct.yaml rename to .github/ct.yaml index bc4fe9c..7afed54 100644 --- a/ci/ct.yaml +++ b/.github/ct.yaml @@ -2,8 +2,8 @@ remote: origin helm-extra-args: --timeout 600s check-version-increment: true validate-yaml: true -validate-chart-schema: false -validate-maintainers: false +validate-chart-schema: true +validate-maintainers: true target-branch: main chart-dirs: diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..1075f8c --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,44 @@ +name: Lint & Test Charts + +on: [ push, pull_request_target ] + +jobs: + lint-test: + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v2 + with: + fetch-depth: 0 + + - name: Set up Helm + uses: azure/setup-helm@v1 + with: + version: v3.4.0 + + # helm/chart-testing-action requires python version 3.7 + # see: https://github.com/helm/chart-testing-action/issues/65 + - uses: actions/setup-python@v2 + with: + python-version: 3.7 + + - name: Set up chart-testing + uses: helm/chart-testing-action@b0d4458c71155b54fcf33e11dd465dc923550009 + + - name: Run chart-testing (list-changed) + id: list-changed + run: | + changed=$(ct list-changed --config .github/ct.yaml) + if [[ -n "$changed" ]]; then + echo "::set-output name=changed::true" + fi + + - name: Run chart-testing (lint) + run: ct lint --config .github/ct.yaml + + - name: Create kind cluster + uses: helm/kind-action@v1.2.0 + if: steps.list-changed.outputs.changed == 'true' + + - name: Run chart-testing (install) + run: ct install --config .github/ct.yaml diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 232a130..d1e5988 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -1,6 +1,6 @@ name: Release Charts -on: +on: push: branches: - main @@ -16,14 +16,21 @@ jobs: - name: Configure Git run: | - git config user.name "$GITHUB_ACTOR" - git config user.email "$GITHUB_ACTOR@users.noreply.github.com" + git config user.name "voice-automation" + git config user.email "voice-automation@users.noreply.github.com" - name: Install Helm uses: azure/setup-helm@v1 with: version: v3.4.0 + # https://github.com/helm/chart-releaser/issues/114 + - name: Add Helm Repos for Dependencies + run: | + helm repo add elastic https://helm.elastic.co + helm repo add jetstack https://charts.jetstack.io + helm repo add bitnami https://charts.bitnami.com/bitnami + - name: Run chart-releaser uses: helm/chart-releaser-action@v1.2.1 with: diff --git a/charts/mozilla-common-voice/Chart.yaml b/charts/mozilla-common-voice/Chart.yaml index df05a9b..a611fca 100644 --- a/charts/mozilla-common-voice/Chart.yaml +++ b/charts/mozilla-common-voice/Chart.yaml @@ -1,8 +1,8 @@ apiVersion: v2 name: mozilla-common-voice icon: https://voice.mozilla.org/dist/f01895c77138837851f87c2e40acbd58.svg -version: 0.4.5 -description: Deploy Mozilla Common Voice web application +version: 1.0.2 +description: Deploy Common Voice web application type: application keywords: - Mozilla @@ -11,16 +11,17 @@ keywords: - deepspeech home: https://voice.mozilla.org sources: - - https://github.com/mozilla/voice-web + - https://github.com/common-voice/common-voice maintainers: - - name: Web SRE Team - email: it-sre@mozilla.com -dependencies: + - name: jzinner + email: jenc@mozillafoundation.org + - name: JZ + email: jenny@mozillafoundation.org +dependencies: # used only in CI testing - name: mysql version: 8.4.2 condition: mysql.enabled repository: https://charts.bitnami.com/bitnami - - name: redis version: 12.7.4 condition: redis.enabled diff --git a/charts/sentence-collector/Chart.yaml b/charts/sentence-collector/Chart.yaml index efa932a..6981934 100644 --- a/charts/sentence-collector/Chart.yaml +++ b/charts/sentence-collector/Chart.yaml @@ -1,8 +1,8 @@ apiVersion: v2 name: sentence-collector -description: A Helm chart for Common Voice's Sentence Collector +description: Deploy Common Voice's Sentence Collector type: application -version: 0.1.2 +version: 1.0.2 keywords: - Mozilla - common-voice @@ -12,10 +12,11 @@ home: https://commonvoice.mozilla.org/sentence-collector sources: - https://github.com/Common-Voice/sentence-collector maintainers: - - name: Web SRE Team - email: it-sre@mozilla.com - -dependencies: + - name: jzinner + email: jenc@mozillafoundation.org + - name: JZ + email: jenny@mozillafoundation.org +dependencies: # used only in CI testing - name: mysql version: 8.4.2 condition: mysql.enabled diff --git a/ci/deps/careers/install.sh b/ci/deps/careers/install.sh deleted file mode 100755 index 3c9a939..0000000 --- a/ci/deps/careers/install.sh +++ /dev/null @@ -1,19 +0,0 @@ -#!/usr/bin/env bash - -set -e - -helm repo add bitnami https://charts.bitnami.com/bitnami -helm repo update - -readonly NAMESPACE="chart-ci-e2e" - - - -helm upgrade --install \ - --version 10.4.2 \ - --namespace $NAMESPACE \ - --set postgresqlPassword=defaultpassword \ - --set postgresqlUsername=careers \ - --set postgresqlDatabase=careers \ - --set fullnameOverride=careers-psql \ - careers-psql bitnami/postgresql diff --git a/ci/deps/ctms/install.sh b/ci/deps/ctms/install.sh deleted file mode 100755 index b9e0fbf..0000000 --- a/ci/deps/ctms/install.sh +++ /dev/null @@ -1,19 +0,0 @@ -#!/usr/bin/env bash - -set -e - -helm repo add bitnami https://charts.bitnami.com/bitnami -helm repo update - -readonly NAMESPACE="chart-ci-e2e" - - - -helm upgrade --install \ - --version 10.4.2 \ - --namespace $NAMESPACE \ - --set postgresqlPassword=defaultpassword \ - --set postgresqlUsername=ctms \ - --set postgresqlDatabase=ctms \ - --set fullnameOverride=postgres \ - postgresql bitnami/postgresql diff --git a/ci/deps/discourse/install.sh b/ci/deps/discourse/install.sh deleted file mode 100644 index 91d5b83..0000000 --- a/ci/deps/discourse/install.sh +++ /dev/null @@ -1,31 +0,0 @@ -#!/usr/bin/env bash - -set -e - -helm repo add bitnami https://charts.bitnami.com/bitnami -helm repo update - -readonly NAMESPACE="chart-ci-e2e" - - - -helm upgrade --install \ - --version 10.8.0 \ - --namespace $NAMESPACE \ - --set fullnameOverride=discourse-psql \ - --set persistence.enabled=false \ - --set postgresqlDatabase=discourse \ - --set postgresqlPassword=discourse-psql-password \ - --set postgresqlUsername=postgres \ - postgresql-discourse bitnami/postgresql - - - -helm upgrade --install \ - --version 14.8.6 \ - --namespace $NAMESPACE \ - --set auth.enabled=true \ - --set auth.password=discourse-redis-password \ - --set fullnameOverride=discourse-redis \ - --set persistence.enabled=false \ - redis-discourse bitnami/redis diff --git a/ci/deps/install.sh b/ci/deps/install.sh deleted file mode 100755 index adede62..0000000 --- a/ci/deps/install.sh +++ /dev/null @@ -1,9 +0,0 @@ -#!/usr/bin/env bash - -set -e - -for d in ci/deps/*/; do - if ls "$d"/*.sh 1> /dev/null 2>&1; then - for f in "$d"/*.sh; do bash "$f" -H ; done - fi -done diff --git a/ci/e2e.sh b/ci/e2e.sh deleted file mode 100755 index 062b934..0000000 --- a/ci/e2e.sh +++ /dev/null @@ -1,105 +0,0 @@ -#!/bin/bash - -readonly REPO_ROOT=$(git rev-parse --show-toplevel) -readonly FUNCTIONS="${REPO_ROOT}/ci/functions.sh" -readonly CLUSTER_NAME="${CLUSTER_NAME:-kind}" -readonly CONFIG_FILE="${REPO_ROOT}/ci/kind.yaml" -readonly NAMESPACE="chart-ci-e2e" - -# shellcheck source=ci/functions.sh -[ -f "${FUNCTIONS}" ] && source "${FUNCTIONS}" || exit 1 - -create_kind_cluster() { - echo 'Installing kind ..' - - kind create cluster \ - --name "${CLUSTER_NAME}" \ - --config "${CONFIG_FILE}" \ - --wait 60s -} - -configure_kind_cluster() { - configure_kube - - kubectl cluster-info - echo - - kubectl get nodes - echo - - if [[ $(kubectl get namespaces | grep -w $NAMESPACE) ]] ; then - # If the NS exists, delete it and recreate it to get a clean state - kubectl delete namespace $NAMESPACE - kubectl create namespace $NAMESPACE - else - kubectl create namespace $NAMESPACE - fi - echo - - echo 'Cluster ready..' - echo -} - -configure_kube() { - - kind get kubeconfig > kind.config - - echo 'Copying kubeconfig to container' - docker_exec sh -c 'mkdir -p /root/.kube' - chmod 600 kind.config - docker cp kind.config ct:/root/.kube/config - echo -} - -install_deps () { - echo 'Installing some dependencies for this pipeline...' - - docker_exec ./ci/deps/install.sh -} - -install_charts () { - echo 'Installing charts...' - docker_exec ct install --namespace $NAMESPACE - echo -} - -check_cluster_exists () { - # 0 if cluster already exists - kind get clusters | grep -q -w "$CLUSTER_NAME" - echo $? -} - -create_ecr_secret () { - # Gets a valid token to pull from ECR and creates a secret with it - ACCOUNT=$(aws sts get-caller-identity --output text --query Account) - REGION=us-west-2 - SECRET_NAME=ecr-registry - EMAIL=itse@mozilla.com - - # Fetch token (which will expire in 12 hours) - TOKEN=$(aws ecr --region=$REGION get-authorization-token --output text --query authorizationData[].authorizationToken | base64 -d | cut -d: -f2) - - # Create or replace secret - kubectl delete secret -n $NAMESPACE --ignore-not-found "$SECRET_NAME" - kubectl create secret -n $NAMESPACE docker-registry "$SECRET_NAME" \ - --docker-server="https://${ACCOUNT}.dkr.ecr.${REGION}.amazonaws.com" \ - --docker-username=AWS \ - --docker-password="${TOKEN}" \ - --docker-email="${EMAIL}" -} - -main() { - run_ct_container - trap cleanup EXIT - - if [[ $(check_cluster_exists) != 0 ]] ; then - create_kind_cluster - fi - - configure_kind_cluster - create_ecr_secret - install_deps - install_charts -} - -main diff --git a/ci/functions.sh b/ci/functions.sh deleted file mode 100644 index 649a051..0000000 --- a/ci/functions.sh +++ /dev/null @@ -1,27 +0,0 @@ - -set -o errexit -set -o nounset -set -o pipefail - -readonly CT_VERSION=v3.3.1 - -run_ct_container() { - echo 'Running ct container...' - docker run --rm --interactive --detach --network host --name ct \ - --volume "$(pwd)/ci/ct.yaml:/etc/ct/ct.yaml" \ - --volume "$(pwd):/workdir" \ - --workdir /workdir \ - "quay.io/helmpack/chart-testing:${CT_VERSION}" \ - cat - echo -} - -cleanup() { - echo 'Removing ct container...' - docker kill ct > /dev/null 2>&1 - echo 'Done!' -} - -docker_exec() { - docker exec --interactive ct "$@" -} diff --git a/ci/kind.yaml b/ci/kind.yaml deleted file mode 100644 index cebee6d..0000000 --- a/ci/kind.yaml +++ /dev/null @@ -1,5 +0,0 @@ -kind: Cluster -apiVersion: kind.x-k8s.io/v1alpha4 -nodes: - - role: control-plane - - role: worker diff --git a/ci/lint.sh b/ci/lint.sh deleted file mode 100755 index 4e04869..0000000 --- a/ci/lint.sh +++ /dev/null @@ -1,22 +0,0 @@ -#!/bin/bash - -set -o errexit -set -o nounset -set -o pipefail - -readonly REPO_ROOT=$(git rev-parse --show-toplevel) -readonly FUNCTIONS="${REPO_ROOT}/ci/functions.sh" - -# shellcheck source=ci/functions.sh -[ -f "${FUNCTIONS}" ] && source "${FUNCTIONS}" || exit 1 - -main() { - - run_ct_container - trap cleanup EXIT - - echo "Linting chart..." - docker_exec ct lint -} - -main diff --git a/ci/setup.sh b/ci/setup.sh deleted file mode 100755 index 829c3ca..0000000 --- a/ci/setup.sh +++ /dev/null @@ -1,23 +0,0 @@ -#!/bin/bash - -# setups test with required packages - -set -o errexit -set -o nounset -set -o pipefail - -KIND_VERSION=v0.8.0 -HELM_VERSION=v3.3.3 - -echo "=== Install kubectl" -sudo curl -L https://storage.googleapis.com/kubernetes-release/release/$(curl -s https://storage.googleapis.com/kubernetes-release/release/stable.txt)/bin/linux/amd64/kubectl -o /usr/local/bin/kubectl && sudo chmod +x /usr/local/bin/kubectl - -echo "=== Install helm" -curl -L "https://get.helm.sh/helm-${HELM_VERSION}-linux-amd64.tar.gz" | tar xz && sudo mv linux-amd64/helm /bin/helm && sudo rm -rf linux-amd64 - -echo "=== Install kind ${KIND_VERSION}" -curl -sSLo kind "https://github.com/kubernetes-sigs/kind/releases/download/${KIND_VERSION}/kind-linux-amd64" && chmod +x kind && sudo mv kind /usr/local/bin/kind - -echo "=== Install awscli" -pip install --user awscli -