Prometheus grafana & add cirun.io

.cirun.yml

@@ -0,0 +1,15 @@
+# Self-Hosted Github Action Runners on DigitalOcean via Cirun.io
+# Reference: https://docs.cirun.io/reference/yaml.html
+runners:
+  - name: run-k8s-tests
+    # Cloud Provider: DigitalOcean
+    cloud: digitalocean
+    # Cheapest VM on DigitalOcean
+    instance_type: s-4vcpu-8gb
+    # Ubuntu-20.4  image"
+    machine_image: docker-20-04
+    region: nyc1
+    # Path of the relevant workflow file
+    workflow: .github/workflows/kubernetes_test.yaml
+    # Number of runners to provision on every trigger on Actions job
+    count: 1

.github/workflows/kubernetes_test.yaml

@@ -0,0 +1,150 @@
+name: "Kubernetes Tests"
+
+on:
+  pull_request: # Workflow only runs for PR against main anyway
+  push:
+    branches:
+      - '**'
+    tags:
+      - 'v*'
+    paths-ignore:
+      - "docs/**"
+      - "*.md"
+jobs:
+  test-kubernetes:
+    name: "Kubernetes Tests"
+    runs-on: self-hosted
+    defaults:
+      run:
+        shell: bash -l {0}
+    steps:
+      - name: 'QHUB_GH_BRANCH set for PR'
+        if: ${{ github.event_name == 'pull_request' }}
+        run: |
+          echo "QHUB_GH_BRANCH=${GITHUB_HEAD_REF}" >> $GITHUB_ENV
+          echo "GITHUB_BASE_REF: ${GITHUB_BASE_REF}"
+          echo "GITHUB_HEAD_REF: ${GITHUB_HEAD_REF}"
+          echo "GITHUB_REF: ${GITHUB_REF}"
+      - name: 'QHUB_GH_BRANCH set for a branch (not a tag)'
+        if: ${{ github.event_name == 'push' && startsWith(github.ref, 'refs/heads/') }}
+        # e.g. QHUB_GH_BRANCH="main"
+        run: |
+          echo "QHUB_GH_BRANCH=${GITHUB_REF:11}" >> $GITHUB_ENV
+
+      - name: 'Checkout Infrastructure'
+        uses: actions/checkout@main
+      - name: Set up Python
+        uses: conda-incubator/setup-miniconda@v2
+        env:
+          CONDA: /home/runnerx/miniconda3
+        with:
+          python-version: 3.8
+          miniconda-version: "latest"
+      - name: Install QHub
+        run: |
+          conda install -c anaconda pip
+          pip install .[dev]
+
+      - name: Download and Install Minikube and Kubectl
+        run: |
+          mkdir -p bin
+          pushd bin
+          curl -L https://github.com/kubernetes/minikube/releases/download/v1.22.0/minikube-linux-amd64 -o minikube
+          chmod +x minikube
+
+          curl -LO https://storage.googleapis.com/kubernetes-release/release/v1.19.0/bin/linux/amd64/kubectl
+          chmod +x kubectl
+
+          echo "$PWD" >> $GITHUB_PATH
+          popd
+      - name: Start Minikube
+        run: |
+          minikube start --kubernetes-version=1.19.4 --driver=docker --cpus 2 --memory 6500 --wait=all
+      - name: Versions
+        run: |
+          minikube version
+          kubectl version
+      - name: Add nfs client to kubernetes docker node
+        run: |
+          minikube ssh "sudo apt update; sudo apt install nfs-common -y"
+      - name: Get routing table for docker pods
+        run: |
+          ip route
+      - name: Configure LoadBalancer IPs
+        run: |
+          python tests/scripts/minikube-loadbalancer-ip.py
+      - name: Enable Minikube metallb
+        run: |
+          minikube addons enable metallb
+      - name: Basic kubectl checks before deployment
+        run: |
+          kubectl get all,cm,secret,ing -A
+      - name: Initialize QHub Cloud
+        run: |
+          mkdir -p local-deployment
+          cd local-deployment
+          qhub init local --project=thisisatest  --domain github-actions.qhub.dev --auth-provider=password
+
+          # Need smaller profiles on Minikube
+          sed -i -E 's/(cpu_guarantee):\s+[0-9\.]+/\1: 0.25/g' "qhub-config.yaml"
+          sed -i -E 's/(mem_guarantee):\s+[A-Za-z0-9\.]+/\1: 0.25G/g' "qhub-config.yaml"
+
+          cat qhub-config.yaml
+      - name: Deploy QHub Cloud
+        run: |
+          cd local-deployment
+          qhub deploy --config qhub-config.yaml --disable-prompt
+      - name: Basic kubectl checks after deployment
+        run: |
+          kubectl get all,cm,secret,ing -A
+      - name: Check github-actions.qhub.dev resolves
+        run: |
+          nslookup github-actions.qhub.dev
+      - name: Curl jupyterhub login page
+        run: |
+          curl -k https://github-actions.qhub.dev/hub/home -i
+
+      ### CYPRESS TESTS
+      - name: Setup Node
+        uses: actions/setup-node@v2
+        with:
+          node-version: '14'
+      - name: npm version
+        run: |
+          npm --version
+      - name: Install Cypress dependencies
+        run: |
+          sudo apt-get -y update
+          sudo apt-get install -y libgtk2.0-0 libgtk-3-0 libgbm-dev libnotify-dev libgconf-2-4 libnss3 libxss1 libasound2 libxtst6 xauth xvfb
+
+      - name: Read example-user password
+        run: python -c "import tempfile, os; print('CYPRESS_EXAMPLE_USER_PASSWORD='+open(os.path.join(tempfile.gettempdir(), 'QHUB_DEFAULT_PASSWORD')).read())" >> $GITHUB_ENV
+
+      - name: Get qhub-config.yaml full path
+        run: echo "QHUB_CONFIG_PATH=`realpath ./local-deployment/qhub-config.yaml`" >> $GITHUB_ENV
+
+      - name: Cypress run
+        uses: cypress-io/github-action@v2
+        env:
+          CYPRESS_BASE_URL: https://github-actions.qhub.dev/
+        with:
+          working-directory: tests_e2e
+
+      - name: Save Cypress screenshots and videos
+        if: always()
+        uses: actions/upload-artifact@v2
+        with:
+          name: e2e-cypress
+          path: |
+            ./tests_e2e/cypress/screenshots/
+            ./tests_e2e/cypress/videos/
+
+      ### CLEANUP AFTER CYPRESS
+
+      - name: Cleanup qhub deployment
+        run: |
+          cd local-deployment
+          qhub destroy --config qhub-config.yaml
+      - name: Basic kubectl checks after cleanup
+        run: |
+          kubectl get all,cm,secret,ing -A

.github/workflows/test.yaml

@@ -89,123 +89,3 @@ jobs:
         with:
           name: "qhub-${{ matrix.provider }}-artifact"
           path: "qhub-${{ matrix.provider }}-deployment"
-
-  test-kubernetes:
-    name: "Kubernetes Tests"
-    runs-on: ubuntu-latest
-
-    steps:
-      - name: 'QHUB_GH_BRANCH set for PR'
-        if: ${{ github.event_name == 'pull_request' }}
-        run: |
-          echo "QHUB_GH_BRANCH=${GITHUB_HEAD_REF}" >> $GITHUB_ENV
-          echo "GITHUB_BASE_REF: ${GITHUB_BASE_REF}"
-          echo "GITHUB_HEAD_REF: ${GITHUB_HEAD_REF}"
-          echo "GITHUB_REF: ${GITHUB_REF}"
-      - name: 'QHUB_GH_BRANCH set for a branch (not a tag)'
-        if: ${{ github.event_name == 'push' && startsWith(github.ref, 'refs/heads/') }}
-        # e.g. QHUB_GH_BRANCH="main"
-        run: |
-          echo "QHUB_GH_BRANCH=${GITHUB_REF:11}" >> $GITHUB_ENV
-
-      - name: 'Checkout Infrastructure'
-        uses: actions/checkout@main
-      - name: Set up Python
-        uses: actions/setup-python@v1
-        with:
-          python-version: 3.8
-      - name: Install QHub
-        run: |
-          pip install .[dev]
-      - name: Download and Install Minikube and Kubectl
-        run: |
-          mkdir -p bin
-          pushd bin
-          curl -L https://github.com/kubernetes/minikube/releases/download/v1.22.0/minikube-linux-amd64 -o minikube
-          chmod +x minikube
-
-          curl -LO https://storage.googleapis.com/kubernetes-release/release/v1.19.0/bin/linux/amd64/kubectl
-          chmod +x kubectl
-
-          echo "$PWD" >> $GITHUB_PATH
-          popd
-      - name: Start Minikube
-        run: |
-          minikube start --kubernetes-version=1.19.4 --driver=docker --cpus 2 --memory 4096 --wait=all
-      - name: Versions
-        run: |
-          minikube version
-          kubectl version
-      - name: Add nfs client to kubernetes docker node
-        run: |
-          minikube ssh "sudo apt update; sudo apt install nfs-common -y"
-      - name: Get routing table for docker pods
-        run: |
-          ip route
-      - name: Configure LoadBalancer IPs
-        run: |
-          python tests/scripts/minikube-loadbalancer-ip.py
-      - name: Enable Minikube metallb
-        run: |
-          minikube addons enable metallb
-      - name: Basic kubectl checks before deployment
-        run: |
-          kubectl get all,cm,secret,ing -A
-      - name: Initialize QHub Cloud
-        run: |
-          mkdir -p local-deployment
-          cd local-deployment
-          qhub init local --project=thisisatest  --domain github-actions.qhub.dev --auth-provider=password
-
-          # Need smaller profiles on Minikube
-          sed -i -E 's/(cpu_guarantee):\s+[0-9\.]+/\1: 0.25/g' "qhub-config.yaml"
-          sed -i -E 's/(mem_guarantee):\s+[A-Za-z0-9\.]+/\1: 0.25G/g' "qhub-config.yaml"
-
-          cat qhub-config.yaml
-      - name: Deploy QHub Cloud
-        run: |
-          cd local-deployment
-          qhub deploy --config qhub-config.yaml --disable-prompt
-      - name: Basic kubectl checks after deployment
-        run: |
-          kubectl get all,cm,secret,ing -A
-      - name: Check github-actions.qhub.dev resolves
-        run: |
-          nslookup github-actions.qhub.dev
-      - name: Curl jupyterhub login page
-        run: |
-          curl -k https://github-actions.qhub.dev/hub/home -i
-
-      ### CYPRESS TESTS
-
-      - name: Read example-user password
-        run: python -c "import tempfile, os; print('CYPRESS_EXAMPLE_USER_PASSWORD='+open(os.path.join(tempfile.gettempdir(), 'QHUB_DEFAULT_PASSWORD')).read())" >> $GITHUB_ENV
-
-      - name: Get qhub-config.yaml full path
-        run: echo "QHUB_CONFIG_PATH=`realpath ./local-deployment/qhub-config.yaml`" >> $GITHUB_ENV
-
-      - name: Cypress run
-        uses: cypress-io/github-action@v2
-        env:
-          CYPRESS_BASE_URL: https://github-actions.qhub.dev/
-        with:
-          working-directory: tests_e2e
-
-      - name: Save Cypress screenshots and videos
-        if: always()
-        uses: actions/upload-artifact@v2
-        with:
-          name: e2e-cypress
-          path: |
-            ./tests_e2e/cypress/screenshots/
-            ./tests_e2e/cypress/videos/
-
-      ### CLEANUP AFTER CYPRESS
-
-      - name: Cleanup qhub deployment
-        run: |
-          cd local-deployment
-          qhub destroy --config qhub-config.yaml
-      - name: Basic kubectl checks after cleanup
-        run: |
-          kubectl get all,cm,secret,ing -A

RELEASE.md

@@ -12,6 +12,8 @@
 ## Upcoming Release
 
 ### Feature changes and enhancements
+
+ - Added basic cluster monitoring capability via Grafana/Prometheus integration
 
 ### Bug fixes
 

docs/index.md

@@ -63,6 +63,7 @@ source/admin_guide/upgrade.md
 source/admin_guide/gpu.md
 source/admin_guide/preemptible-spot-instances.md
 source/admin_guide/system_maintenance.md
+source/admin_guide/monitoring.md
 source/admin_guide/clearml.md
 source/admin_guide/prefect.md
 source/admin_guide/faq.md

docs/source/admin_guide/monitoring.md

@@ -0,0 +1,16 @@
+# Monitoring
+
+Cluster monitoring via Grafana/Prometheus comes built in with QHub, and is enabled by default.
+
+## Accessing the Grafana Dashboards
+
+The monitoring dashboards can be accessed via Grafana at: `your-qhub-domain.com/monitoring`.  The initial login credentials are username: `admin` and password: `prom-operator`, but users should change the admin password immediately after the first log in. 
+
+## Disabling Cluster Monitoring
+
+1. To disable cluster monitoring on QHub deployments, simply disable the feature flag within your `qhub-config.yaml` file. For example:
+
+```yaml
+monitoring:
+  enabled: false
+```

docs/source/introduction.md

@@ -43,6 +43,8 @@ with.
 
 + [**prefect**](https://www.prefect.io/) workflow management
 + [**clearml**](https://clear.ml/) machine learning platform
++ [**prometheus**](https://prometheus.io/) cluster monitoring
++ [**grafana**](https://grafana.com/) cluster monitoring visualizations
 
 # Why use QHub?
 

qhub/initialize.py

@@ -57,6 +57,9 @@
             "h2_color": "#652e8e",
         }
     },
+    "monitoring": {
+        "enabled": True,
+    },
     "cdsdashboards": {
         "enabled": True,
         "cds_hide_user_named_servers": True,

qhub/schema.py

@@ -56,6 +56,13 @@ class CICD(Base):
     after_script: typing.Optional[typing.List[str]]
 
 
+# ============== Monitoring =============
+
+
+class Monitoring(Base):
+    enabled: bool
+
+
 # ============== ClearML =============
 
 
@@ -362,6 +369,7 @@ class Main(Base):
     theme: Theme
     profiles: Profiles
     environments: typing.Dict[str, CondaEnvironment]
+    monitoring: typing.Optional[Monitoring]
     clearml: typing.Optional[ClearML]
 
 

qhub/template/cookiecutter.json

@@ -13,6 +13,9 @@
     "certificate": {
       "type": "self-signed"
     },
+    "monitoring": {
+        "enabled": null
+    },
     "clearml": {
         "enabled": null
     },

qhub/template/{{ cookiecutter.repo_directory }}/infrastructure/kubernetes.tf

@@ -238,6 +238,19 @@ module "prefect" {
 }
 {% endif -%}
 
+{% if cookiecutter.monitoring.enabled -%}
+module "monitoring" {
+  source       = "./modules/kubernetes/services/monitoring"
+  namespace    = var.environment
+  external-url = var.endpoint
+  tls          = module.qhub.tls
+  depends_on = [
+    module.qhub
+  ]
+}
+{% endif -%}
+
+
 {% if cookiecutter.clearml.enabled -%}
 module "clearml" {
   source       = "./modules/kubernetes/services/clearml"