From 40ea78cf2224b38aea8605057759af6a40a6b5b9 Mon Sep 17 00:00:00 2001 From: Zach Wolfenbarger Date: Tue, 30 Jan 2024 16:41:37 -0600 Subject: [PATCH 1/4] Process hotmail complaints via GHA --- .github/workflows/build_image.yaml | 16 +++++++++++++++ kubernetes/cron_script.yml | 33 ++++++++++++++++++++++++++++++ kubernetes/manual_script.yml | 30 +++++++++++++++++++++++++++ process.py | 3 ++- 4 files changed, 81 insertions(+), 1 deletion(-) create mode 100644 .github/workflows/build_image.yaml create mode 100644 kubernetes/cron_script.yml create mode 100644 kubernetes/manual_script.yml diff --git a/.github/workflows/build_image.yaml b/.github/workflows/build_image.yaml new file mode 100644 index 0000000..beba0dc --- /dev/null +++ b/.github/workflows/build_image.yaml @@ -0,0 +1,16 @@ +name: Build & Push Image + +on: + push: + branches: + - master + workflow_dispatch: + +jobs: + build_and_push_image: + name: Build and Push Image + uses: zooniverse/ci-cd/.github/workflows/build_and_push_image.yaml@main + with: + repo_name: hotmail-complaint-processor + commit_id: ${{ github.sha }} + latest: true \ No newline at end of file diff --git a/kubernetes/cron_script.yml b/kubernetes/cron_script.yml new file mode 100644 index 0000000..6009d1c --- /dev/null +++ b/kubernetes/cron_script.yml @@ -0,0 +1,33 @@ +apiVersion: batch/v1 +kind: CronJob +metadata: + name: hotmail-complaint-processor-cronjob +spec: + schedule: "0 * * * *" + jobTemplate: + spec: + template: + metadata: + name: hotmail-complaint-processor-cronjob + spec: + containers: + - name: hotmail-complaint-processor-cronjob + image: ghcr.io/zooniverse/hotmail-complaint-processor + volumeMounts: + - name: hotmail-complaint-config + mountPath: "/run/secrets/config.yml" + subPath: "config.yml" + readOnly: true + - name: hotmail-aws-creds + mountPath: "/root/.aws/credentials" + subPath: "credentials" + readOnly: true + volumes: + - name: hotmail-complaint-config + secret: + secretName: hotmail-complaint-config + - name: hotmail-aws-creds + secret: + secretName: hotmail-aws-creds + restartPolicy: Never + backoffLimit: 2 \ No newline at end of file diff --git a/kubernetes/manual_script.yml b/kubernetes/manual_script.yml new file mode 100644 index 0000000..95da1e4 --- /dev/null +++ b/kubernetes/manual_script.yml @@ -0,0 +1,30 @@ +apiVersion: batch/v1 +kind: Job +metadata: + generateName: hotmail-complaint-processor-job- +spec: + template: + metadata: + name: hotmail-complaint-processor-job + spec: + containers: + - name: hotmail-complaint-processor-job + image: zooniverse/hotmail-complaint-processor + volumeMounts: + - name: hotmail-complaint-config + mountPath: "/run/secrets/config.yml" + subPath: "config.yml" + readOnly: true + - name: hotmail-aws-creds + mountPath: "/root/.aws/credentials" + subPath: "credentials" + readOnly: true + volumes: + - name: hotmail-complaint-config + secret: + secretName: hotmail-complaint-config + - name: hotmail-aws-creds + secret: + secretName: hotmail-aws-creds + restartPolicy: Never + backoffLimit: 2 diff --git a/process.py b/process.py index e1633ac..e621533 100644 --- a/process.py +++ b/process.py @@ -11,7 +11,8 @@ with open('/run/secrets/config.yml', 'r') as f: CONFIG = yaml.load(f) -s3 = boto3.resource('s3') +session = boto3.Session(profile_name='default') +s3 = session.resource('s3') addresses_to_unsubscribe = set() processed_s3_keys = [] From 4fb5b2250bd56d7518b70f73a5ebc12b68b5d3e4 Mon Sep 17 00:00:00 2001 From: Zach Wolfenbarger Date: Tue, 30 Jan 2024 22:20:20 -0600 Subject: [PATCH 2/4] Deploy workflows --- .github/workflows/build_image.yaml | 16 ----------- .github/workflows/cron_deploy.yaml | 43 ++++++++++++++++++++++++++++++ .github/workflows/manual_run.yaml | 31 +++++++++++++++++++++ 3 files changed, 74 insertions(+), 16 deletions(-) delete mode 100644 .github/workflows/build_image.yaml create mode 100644 .github/workflows/cron_deploy.yaml create mode 100644 .github/workflows/manual_run.yaml diff --git a/.github/workflows/build_image.yaml b/.github/workflows/build_image.yaml deleted file mode 100644 index beba0dc..0000000 --- a/.github/workflows/build_image.yaml +++ /dev/null @@ -1,16 +0,0 @@ -name: Build & Push Image - -on: - push: - branches: - - master - workflow_dispatch: - -jobs: - build_and_push_image: - name: Build and Push Image - uses: zooniverse/ci-cd/.github/workflows/build_and_push_image.yaml@main - with: - repo_name: hotmail-complaint-processor - commit_id: ${{ github.sha }} - latest: true \ No newline at end of file diff --git a/.github/workflows/cron_deploy.yaml b/.github/workflows/cron_deploy.yaml new file mode 100644 index 0000000..1cb0326 --- /dev/null +++ b/.github/workflows/cron_deploy.yaml @@ -0,0 +1,43 @@ +name: Deploy Hotmail Processing Cron Task + +on: + push: + branches: + - master + workflow_dispatch: + +jobs: + build_and_push_image: + name: Build and Push Image + uses: zooniverse/ci-cd/.github/workflows/build_and_push_image.yaml@main + with: + repo_name: hotmail-complaint-processor + commit_id: ${{ github.sha }} + latest: true + + deploy_cron: + runs-on: ubuntu-latest + needs: build_and_push_image + steps: + - name: Checkout + uses: actions/checkout@v3.5.2 + + - name: Login to GitHub Container Registry + uses: docker/login-action@v2 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - uses: azure/login@v1 + with: + creds: ${{ secrets.AZURE_AKS }} + + - name: Set the target AKS cluster + uses: Azure/aks-set-context@v3 + with: + cluster-name: microservices + resource-group: kubernetes + + - name: Modify & apply template + run: kubectl create -f kubernetes/cron_script.yml diff --git a/.github/workflows/manual_run.yaml b/.github/workflows/manual_run.yaml new file mode 100644 index 0000000..ad25c63 --- /dev/null +++ b/.github/workflows/manual_run.yaml @@ -0,0 +1,31 @@ +name: Manually Run Hotmail Complaint Processing + +on: + workflow_dispatch: + +jobs: + manual_sync: + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v3.5.2 + + - name: Login to GitHub Container Registry + uses: docker/login-action@v2 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - uses: azure/login@v1 + with: + creds: ${{ secrets.AZURE_AKS }} + + - name: Set the target AKS cluster + uses: Azure/aks-set-context@v3 + with: + cluster-name: microservices + resource-group: kubernetes + + - name: Modify & apply template + run: kubectl create -f kubernetes/manual_script.yml From b023e7fdc6e6a4192b8ed9a087f43903497d4611 Mon Sep 17 00:00:00 2001 From: Zach Wolfenbarger Date: Tue, 30 Jan 2024 22:23:35 -0600 Subject: [PATCH 3/4] Use GHCR --- kubernetes/manual_script.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kubernetes/manual_script.yml b/kubernetes/manual_script.yml index 95da1e4..7380cf4 100644 --- a/kubernetes/manual_script.yml +++ b/kubernetes/manual_script.yml @@ -9,7 +9,7 @@ spec: spec: containers: - name: hotmail-complaint-processor-job - image: zooniverse/hotmail-complaint-processor + image: ghcr.io/zooniverse/hotmail-complaint-processor volumeMounts: - name: hotmail-complaint-config mountPath: "/run/secrets/config.yml" From 968ea54d9898705063e279f237e10fe0b0aabb25 Mon Sep 17 00:00:00 2001 From: Zach Wolfenbarger Date: Tue, 30 Jan 2024 22:36:16 -0600 Subject: [PATCH 4/4] newline --- kubernetes/cron_script.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kubernetes/cron_script.yml b/kubernetes/cron_script.yml index 6009d1c..aa0b134 100644 --- a/kubernetes/cron_script.yml +++ b/kubernetes/cron_script.yml @@ -30,4 +30,4 @@ spec: secret: secretName: hotmail-aws-creds restartPolicy: Never - backoffLimit: 2 \ No newline at end of file + backoffLimit: 2