feat: AS-314 Docker Builds And Deploys (#2)

.github/workflows/build-and-publish.yml

@@ -4,47 +4,71 @@ on:
   push:
     branches:
       - "main"
+    paths:
+      - 'voxelbot/**'
+      - 'ansible/**'
+      - 'dockerfile'
+      - 'docker-compose.yaml'
+      - '.github/workflows/build-and-publish.yml'
+
+concurrency:
+  group: "${{ github.ref_name }}-build-and-deploy"
 
 jobs:
   build:
     runs-on: ubuntu-latest
     permissions:
       contents: 'write'
       id-token: 'write'
-    env:
-      GCP_LOCATION: ''
-      GCP_PROJECT: ''
-      GCP_DOCKER_REPOSITORY: ''
-      GCP_HELM_REGISTRY: ''
-      GCP_SERVICE_ACCOUNT: ''
-      VERSION: ${{ github.sha }}
+
     steps:
       - uses: actions/checkout@v4
+
       - name: Authenticate to Google Cloud
         uses: google-github-actions/auth@v2
         with:
-          project_id: ${{ env.GCP_PROJECT }}
-          service_account: ${{ env.GCP_SERVICE_ACCOUNT }}
+          project_id: ${{ secrets.GCP_PROJECT }}
+          service_account: ${{ secrets.GCP_SERVICE_ACCOUNT }}
           workload_identity_provider: ${{ secrets.ORG_GOOGLE_WORKLOAD_IDP }}
+
       - name: Set Up Cloud SDK
         uses: google-github-actions/setup-gcloud@v2
+        with:
+          install_components: 'beta'
+
       - name: Docker login
         run: |
           gcloud auth print-access-token | docker login \
             -u oauth2accesstoken \
-            --password-stdin "https://${{ env.GCP_LOCATION }}-docker.pkg.dev"
-      - name: Helm login
-        run: |
-          gcloud auth print-access-token | \
-            helm registry login -u oauth2accesstoken \
-            --password-stdin "https://${{ env.GCP_LOCATION }}-docker.pkg.dev"
+            --password-stdin "https://${{ secrets.GCP_LOCATION }}-docker.pkg.dev"
+
       - name: Set up Docker Buildx
         uses: docker/setup-buildx-action@v3
+
       - name: Build and push
         uses: docker/build-push-action@v6
         with:
           push: true
           platforms: linux/amd64,linux/arm64
-          file: ./external/VoxelBot/dockerfile
-          context: ./external/VoxelBot
-          tags: ${{ env.GCP_LOCATION }}-docker.pkg.dev/${{ env.GCP_PROJECT }}/${{ env.GCP_DOCKER_REPOSITORY }}/voxel51-discordbot:${{ env.VERSION }}
+          file: dockerfile
+          context: .
+          tags: ${{ secrets.GCP_LOCATION }}-docker.pkg.dev/${{ secrets.GCP_PROJECT }}/${{ secrets.GCP_DOCKER_REPOSITORY }}/voxel51-discordbot:${{ github.sha }}
+          cache-from: type=gha
+          cache-to: type=gha,node=max
+
+      - name: Deploy via ansible
+        shell: bash
+        env:
+          DOCKER_REGISTRY: "${{ secrets.GCP_LOCATION }}-docker.pkg.dev/${{ secrets.GCP_PROJECT }}/${{ secrets.GCP_DOCKER_REPOSITORY }}/"
+          GCP_SM_KEY: "${{ secrets.GCP_SM_KEY }}"
+          TAG: ${{ github.sha }}
+          GCP_COMPUTE_SERVER_NAME: "${{ secrets.GCP_COMPUTE_SERVER_NAME }}"
+          GCP_LOCATION: ${{ secrets.GCP_LOCATION }}
+          GCP_PROJECT: ${{ secrets.GCP_PROJECT }}
+        run: |
+          pushd ansible
+          yq -i ".projects |= [\"$GCP_PROJECT\"]" ./inventory/gcp.yml
+          yq -i ".zones |= [\"$GCP_LOCATION\"]" ./inventory/gcp.yml
+          sudo pipx inject ansible-core -r requirements.txt
+          ansible-playbook site.yml
+          popd

ansible/README.md

@@ -0,0 +1,43 @@
+# Ansible Automation For Docker Compose Systems
+
+In order to provide a GitOps
+flow for docker compose, we wrote a suite of ansible tooling
+to be triggered via GitHub actions.
+
+This automation includes a few task sets:
+
+1. A task set to log in to GCP and internal docker registries
+1. A task set to deploy our docker compose stacks. This includes:
+      1. Ensuring there is a `.env` file either in google secrets manager
+         or on disk for the stack to use.
+      1. Ensuring the ansible user is part of the `docker` linux group.
+      1. Bringing up the docker compose stack
+
+## Variables
+
+Host variables are documented and defaulted in [this](group_vars/all.yaml) file.
+
+## Running
+
+You can run locally via the `ansible-playbook` command.
+You can set your hosts via the command line via the `GCP_COMPUTE_SERVER_NAME`
+environment variable.
+
+A list of environment variables:
+
+* `DOCKER_REGISTRY` - The registry to pull images from
+* `GCP_COMPUTE_SERVER_NAME` - The ansbile host or group to deploy to
+* `GCP_LOCATION` - The GCP location of the registry
+* `GCP_SM_KEY` - The GCP secret with .env file contents
+* `TAG` - The image tag to deploy
+
+An example:
+
+```shell
+export DOCKER_REGISTRY="us.gcr.io/.../..."
+export GCP_COMPUTE_SERVER_NAME=some-server-name
+export GCP_LOCATION=some-gcp-location
+export GCP_SM_KEY="some-key-name"
+export TAG="abc123"
+ansible-playbook main.yml
+```

ansible/ansible.cfg

@@ -0,0 +1,16 @@
+[inventory]
+enable_plugins = gcp_compute
+
+[defaults]
+inventory = inventory/gcp.yml
+
+[ssh_connection]
+# Enabling pipelining reduces the number of SSH operations required
+# to execute a module on the remote server.
+# This can result in a significant performance improvement
+# when enabled.
+pipelining = True
+ssh_executable = scripts/gcp-ssh-wrapper.sh
+ssh_args = None
+scp_if_ssh = True
+scp_executable = scripts/gcp-scp-wrapper.sh

ansible/group_vars/all.yaml

@@ -0,0 +1,8 @@
+---
+ansible_ssh_args: --tunnel-through-iap --zone={{ zone }} --project={{ project }} --no-user-output-enabled --quiet
+ansible_scp_extra_args: --tunnel-through-iap --zone={{ zone }} --quiet
+docker_dir: /deploy/voxel51-discordbot
+compose_async_timeout: 30
+gcp_sm_key: "{{ lookup('env', 'GCP_SM_KEY') }}"
+docker_registry: "{{ lookup('env', 'DOCKER_REGISTRY') }}"
+tag: "{{ lookup('env', 'TAG') }}"

ansible/inventory/gcp.yml

@@ -0,0 +1,8 @@
+plugin: google.cloud.gcp_compute
+projects: []
+zones: []
+filters:
+  - status = RUNNING
+auth_kind: application
+hostnames:
+  - name

ansible/requirements.txt

@@ -0,0 +1,2 @@
+google-auth==2.35.0
+requests==2.31.0

ansible/scripts/gcp-scp-wrapper.sh

@@ -0,0 +1,24 @@
+#!/bin/bash
+# This is a wrapper script allowing to use GCP's IAP option to connect
+# to our servers.
+
+# Ansible passes a large number of SSH parameters along with the hostname as the
+# second to last argument and the command as the last. We will pop the last two
+# arguments off of the list and then pass all of the other SSH flags through
+# without modification:
+host="${@: -2: 1}"
+cmd="${@: -1: 1}"
+
+# Unfortunately ansible has hardcoded scp options, so we need to filter these out
+# It's an ugly hack, but for now we'll only accept the options starting with '--'
+declare -a opts
+for scp_arg in "${@: 1: $# -3}" ; do
+        if [[ "${scp_arg}" == --* ]] ; then
+                opts+="${scp_arg} "
+        fi
+done
+
+# Remove [] around our host, as gcloud scp doesn't understand this syntax
+cmd=`echo "${cmd}" | tr -d []`
+
+exec gcloud beta compute scp $opts "${host}" "${cmd}"

ansible/scripts/gcp-ssh-wrapper.sh

@@ -0,0 +1,21 @@
+#!/bin/bash
+# This is a wrapper script allowing to use GCP's IAP SSH option to connect
+# to our servers.
+
+# Ansible passes a large number of SSH parameters along with the hostname as the
+# second to last argument and the command as the last. We will pop the last two
+# arguments off of the list and then pass all of the other SSH flags through
+# without modification:
+host="${@: -2: 1}"
+cmd="${@: -1: 1}"
+
+# Unfortunately ansible has hardcoded ssh options, so we need to filter these out
+# It's an ugly hack, but for now we'll only accept the options starting with '--'
+declare -a opts
+for ssh_arg in "${@: 1: $# -3}" ; do
+        if [[ "${ssh_arg}" == --* ]] ; then
+                opts+="${ssh_arg} "
+        fi
+done
+
+exec gcloud beta compute ssh $opts "${host}" -- -C "${cmd}"

ansible/site.yml

@@ -0,0 +1,14 @@
+---
+- name: Deploy Voxel51 Discord Bot
+  hosts: "{{ lookup('env', 'GCP_COMPUTE_SERVER_NAME') }}"
+  gather_facts: true
+
+  tasks:
+    - name: Sync User Permissions
+      include_tasks: tasks/users.yml
+
+    - name: Ensure paths and dot envs
+      include_tasks: tasks/fs.yml
+
+    - name: Deploy stack
+      include_tasks: tasks/docker.yml

ansible/tasks/docker.yml

@@ -0,0 +1,33 @@
+---
+- name: Docker login
+  shell: |
+    gcloud auth configure-docker \
+      "{{ lookup('env', 'GCP_LOCATION') }}-docker.pkg.dev" \
+      --quiet
+
+- name: Pull docker images
+  community.docker.docker_compose_v2_pull:
+    project_src: "{{ docker_dir }}"
+  environment:
+    DOCKER_REGISTRY: "{{ docker_registry }}"
+    TAG: "{{ tag }}"
+
+- name: Create and start services
+  community.docker.docker_compose_v2:
+    project_src: "{{ docker_dir }}"
+    build: "never"
+  environment:
+    DOCKER_REGISTRY: "{{ docker_registry }}"
+    TAG: "{{ tag }}"
+  register: compose_out
+
+# Assert that states == running here
+- name: Verify all services are running
+  ansible.builtin.assert:
+    that:
+      - item.State == 'running'
+    msg: "{{ item.Name }} failed to start properly"
+    quiet: true
+  with_items: "{{ compose_out.containers }}"
+  loop_control:
+    label: "{{ item.Name }}"

ansible/tasks/fs.yml

@@ -0,0 +1,38 @@
+---
+
+- name: Create directories
+  become: true
+  ansible.builtin.file:
+    path: "{{ item }}"
+    state: directory
+    mode: '0770'
+    owner: "{{ ansible_user_id }}"
+    group: fiftyone
+    recurse: true
+  loop:
+    - /deploy
+    - "{{ docker_dir }}"
+
+- name: Read from gcp
+  ansible.builtin.shell: |
+    gcloud secrets versions access latest \
+      --secret="{{ gcp_sm_key }}" \
+      --project="{{ project }}"
+  register: _env
+
+- name: Save to .env file
+  ansible.builtin.copy:
+    content: "{{ _env.stdout }}"
+    dest: "{{ docker_dir }}/.env"
+    owner: "{{ ansible_user_id }}"
+    group: fiftyone
+    mode: '0660'
+  no_log: true
+
+- name: Move compose file over
+  ansible.builtin.copy:
+    src: "{{ playbook_dir }}/../docker-compose.yaml"
+    dest: "{{ docker_dir }}/docker-compose.yaml"
+    group: fiftyone
+    mode: '0660'
+  no_log: true

ansible/tasks/users.yml

@@ -0,0 +1,19 @@
+---
+- name: Add groups to the system
+  become: true
+  ansible.builtin.group:
+    name: "{{ item }}"
+    state: present
+  loop:
+    - docker
+    - fiftyone
+
+- name: Add user to group
+  become: true
+  ansible.builtin.user:
+    name: "{{ ansible_user_id }}"
+    groups: "{{ item }}"
+    append: true
+  loop:
+    - docker
+    - fiftyone

docker-compose.yaml

@@ -0,0 +1,16 @@
+---
+
+services:
+  discordbot:
+    image: "${DOCKER_REGISTRY:-}voxel51-discordbot:${TAG:-latest}"
+    build:
+      context: .
+      dockerfile: dockerfile
+    env_file:
+      - .env
+    restart: unless-stopped
+    volumes:
+      - discordbot:/app/persist/:rw
+
+volumes:
+  discordbot: {}