feat(app): use dataconf and telepresence (#1186)

SwissDataScienceCenter · Aug 2, 2022 · 08e6937 · 08e6937
1 parent 1ba7291
commit 08e6937
Show file tree

Hide file tree

Showing 44 changed files with 958 additions and 715 deletions.
diff --git a/.dockerignore b/.dockerignore
@@ -6,3 +6,4 @@ travis-deploy.sh
 github_deploy_key.enc
 run-telepresence.sh
 secrets.tar.enc
+.config.hocon
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -10,7 +10,7 @@ jobs:
     - name: Set up Python
       uses: actions/setup-python@v1
       with:
-        python-version: '3.7'
+        python-version: '3.8'
     - name: Install dependencies
       run: |
         python -m pip install --upgrade pip poetry
@@ -38,7 +38,7 @@ jobs:
     - name: Set up Python
       uses: actions/setup-python@v1
       with:
-        python-version: '3.7'
+        python-version: '3.8'
     - name: Install dependencies
       run: |
         python -m pip install --upgrade pip poetry
@@ -56,7 +56,7 @@ jobs:
       - uses: actions/checkout@v2
       - uses: actions/setup-python@v1
         with:
-          python-version: 3.7
+          python-version: '3.8'
       - name: Install helm
         uses: azure/setup-helm@v1
       - name: Test chart
@@ -104,7 +104,7 @@ jobs:
     - uses: azure/setup-helm@v1
     - uses: actions/setup-python@v2
       with:
-        python-version: 3.7
+        python-version: 3.8
     - name: Push latest image
       env:
         DOCKER_USERNAME: ${{ secrets.RENKU_DOCKER_USERNAME }}

diff --git a/.github/workflows/integration-tests.yml b/.github/workflows/integration-tests.yml
@@ -83,7 +83,7 @@ jobs:
         EOF
     - uses: actions/setup-python@v2
       with:
-        python-version: 3.7
+        python-version: '3.8'
     - name: Setup chartpress
       run: |
         pip install --user poetry

diff --git a/.gitignore b/.gitignore
@@ -274,14 +274,14 @@ tags
 [._]*.un~
 
 ### VisualStudioCode ###
-.vscode
-!.vscode/settings.json
-!.vscode/tasks.json
+.vscode/*
 !.vscode/launch.json
-!.vscode/extensions.json
+!.vscode/tasks.json
 
 ### VisualStudioCode Patch ###
 # Ignore all local history of files
 .history
 
+.config.hocon
+
 # End of https://www.gitignore.io/api/vim,linux,emacs,macos,python,sublimetext,visualstudiocode
diff --git a/.vscode/launch.json b/.vscode/launch.json
@@ -0,0 +1,37 @@
+{
+    // Use IntelliSense to learn about possible attributes.
+    // Hover to view descriptions of existing attributes.
+    // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387
+    "version": "0.2.0",
+    "configurations": [
+        {
+            "name": "Python: Flask",
+            "type": "python",
+            "request": "launch",
+            "module": "flask",
+            "env": {
+                "FLASK_APP": "renku_notebooks/wsgi.py",
+                "FLASK_ENV": "development",
+                "CONFIG_FILE": ".config.hocon"
+            },
+            "args": [
+                "run",
+                "--no-debugger",
+                "-h",
+                "localhost",
+                "-p",
+                "8000"
+            ],
+            "jinja": false,
+            "justMyCode": true,
+        },
+        {
+            "name": "Python: Current File",
+            "type": "python",
+            "request": "launch",
+            "program": "${file}",
+            "console": "integratedTerminal",
+            "justMyCode": true,
+        },
+    ]
+}
diff --git a/Dockerfile b/Dockerfile
@@ -1,4 +1,4 @@
-FROM python:3.7-slim
+FROM python:3.8-slim
 
 LABEL maintainer="info@datascience.ch"
 

diff --git a/Dockerfile.tests b/Dockerfile.tests
@@ -1,4 +1,4 @@
-FROM python:3.7-slim
+FROM python:3.8-slim
 
 LABEL maintainer="info@datascience.ch"
 

diff --git a/README.md b/README.md
@@ -104,6 +104,23 @@ every time they launch a session.
 To build the images and render the chart locally, use [chartpress]. Install it
 with `pip` or use `poetry install`.
 
+## Development flow
+
+You can run the notebook service locally in a few easy steps:
+- install poetry
+- run `poetry install`
+- create a copy of `example.config.hocon` in the root of the repository called `.config.hocon` and fill in the required values
+- if using VS code simply use the `Flask` configuration from `.vscode/launch.json`
+- if not using VS code execute `FLASK_APP=renku_notebooks/wsgi.py FLASK_ENV=development CONFIG_FILE=.config.hocon poetry run flask run --no-debugger -h localhost -p 8000`
+
+In addition to the above steps if you have a running Renku deployment you can use [telepresence]
+(https://www.telepresence.io/docs/latest/install/) to route traffic from a deployment to your development 
+environment. After you have setup telepresence you can simply run the `run-telepresence.sh` script.
+This script will try to find a Renku Helm deployment in your current K8s context and active namespace.
+Then it will redirect all traffik for the notebooks service from the deployment to your local machine at
+port `8000`. Combininig telepresence with the steps above can be used to quickly test a notebook
+service in a full Renku deployment.
+
   [CI]: https://github.com/SwissDataScienceCenter/renku-notebooks/workflows/CI/badge.svg
   [1]: https://github.com/SwissDataScienceCenter/renku-notebooks/actions?query=branch%3Amaster+workflow%3ACI
   [Conventional Commits]: https://img.shields.io/badge/Conventional%20Commits-1.0.0-yellow.svg?style=flat-square

diff --git a/example.config.hocon b/example.config.hocon
@@ -0,0 +1,30 @@
+# For available options refer to renku_notebooks.config.dynamic
+sessions {
+    ingress {
+        host = dev.renku.ch
+        tls_secret = dev-renku
+    }
+    oidc {
+        allow_unverified_email = True
+        client_secret = XXXXXXXXX
+        token_url = https://dev.renku.ch/auth/realms/Renku/protocol/openid-connect/token
+        auth_url = https://dev.renku.ch/auth/realms/Renku/protocol/openid-connect/auth
+    }
+    default_image = renku/singleuser:X.X.X
+    git_proxy {
+        image = renku/git-https-proxy:X.X.X
+    }
+    git_clone {
+        image = renku/git-clone:X.X.X
+    }
+    git_rpc_server {
+        image = renku/git-rpc-server:X.X.X
+    }
+}
+git {
+    url = https://dev.renku.ch/gitlab
+    registry = registry.dev.renku.ch
+}
+k8s {
+    namespace = namespace_where_notebooks_run
+}
diff --git a/helm-chart/renku-notebooks/templates/statefulset.yaml b/helm-chart/renku-notebooks/templates/statefulset.yaml
@@ -34,113 +34,113 @@ spec:
             {{- toYaml .Values.securityContext | nindent 12 }}
           env:
             {{ if .Values.userSessionPersistentVolumes.enabled }}
-            - name: NOTEBOOKS_SESSION_PVS_ENABLED
+            - name: NB_SESSIONS__STORAGE__PVS_ENABLED
               value: {{ .Values.userSessionPersistentVolumes.enabled | quote }}
-            - name: NOTEBOOKS_SESSION_PVS_STORAGE_CLASS
+            - name: NB_SESSIONS__STORAGE__PVS_STORAGE_CLASS
               value: {{ .Values.userSessionPersistentVolumes.storageClass | quote}}
             {{ end }}
-            - name: USE_EMPTY_DIR_SIZE_LIMIT
+            - name: NB_SESSIONS__STORAGE__USE_EMPTY_DIR_SIZE_LIMIT
               value: {{ .Values.userSessionPersistentVolumes.useEmptyDirSizeLimit | quote }}
-            - name: NOTEBOOKS_DEFAULT_IMAGE
+            - name: NB_SESSIONS__DEFAULT_IMAGE
               value: "{{ .Values.defaultSessionImage }}"
-            - name: NOTEBOOKS_SERVER_OPTIONS_DEFAULTS_PATH
+            - name: NB_SERVER_OPTIONS__DEFAULTS_PATH
               value: /etc/renku-notebooks/server_options/server_defaults.json
-            - name: NOTEBOOKS_SERVER_OPTIONS_UI_PATH
+            - name: NB_SERVER_OPTIONS__UI_CHOICES_PATH
               value: /etc/renku-notebooks/server_options/server_options.json
             {{ if eq .Values.debug true }}
             - name: FLASK_DEBUG
               value: "1"
             {{ end }}
-            - name: OIDC_CLIENT_ID
+            - name: NB_SESSIONS__OIDC__CLIENT_ID
               value: {{ .Values.oidc.clientId }}
-            - name: OIDC_CLIENT_SECRET
+            - name: NB_SESSIONS__OIDC__CLIENT_SECRET
               value: {{ .Values.oidc.clientSecret }}
-            - name: OIDC_AUTH_URL
+            - name: NB_SESSIONS__OIDC__AUTH_URL
               value: {{ .Values.oidc.authUrl }}
-            - name: OIDC_TOKEN_URL
+            - name: NB_SESSIONS__OIDC__TOKEN_URL
               value: {{ .Values.oidc.tokenUrl }}
-            - name: OIDC_ALLOW_UNVERIFIED_EMAIL
+            - name: NB_SESSIONS__OIDC__ALLOW_UNVERIFIED_EMAIL
               value: {{ .Values.oidc.allowUnverifiedEmail | quote }}
-            - name: SESSION_HOST
+            - name: NB_SESSIONS__INGRESS__HOST
               value: {{ .Values.sessionIngress.host }}
-            - name: SESSION_TLS_SECRET
+            - name: NB_SESSIONS__INGRESS__TLS_SECRET
               value: {{ .Values.sessionIngress.tlsSecret }}
-            - name: SESSION_INGRESS_ANNOTATIONS
+            - name: NB_SESSIONS__INGRESS__ANNOTATIONS
               value: |
                 {{- .Values.sessionIngress.annotations | toYaml | nindent 16 }}
-            - name: GITLAB_URL
+            - name: NB_GIT__URL
             {{ if .Values.gitlab.url }}
               value: {{ .Values.gitlab.url }}
             {{ else }}
               value: {{ template "notebooks.http" . }}://{{ .Values.global.renku.domain}}{{ .Values.global.gitlab.urlPrefix }}
             {{ end }}
-            - name: IMAGE_REGISTRY
+            - name: NB_GIT__REGISTRY
               value: {{ required "An image registry must be specified." .Values.gitlab.registry.host }}
-            - name: GIT_RPC_SERVER_IMAGE
+            - name: NB_SESSIONS__GIT_RPC_SERVER__IMAGE
               value: "{{ .Values.gitRpcServer.image.name }}:{{ .Values.gitRpcServer.image.tag }}"
-            - name: GIT_HTTPS_PROXY_IMAGE
+            - name: NB_SESSIONS__GIT_PROXY__IMAGE
               value: "{{ .Values.gitHttpsProxy.image.name }}:{{ .Values.gitHttpsProxy.image.tag }}"
-            - name: GIT_CLONE_IMAGE
+            - name: NB_SESSIONS__GIT_CLONE__IMAGE
               value: "{{ .Values.gitClone.image.name }}:{{ .Values.gitClone.image.tag }}"
-            - name: ANONYMOUS_SESSIONS_ENABLED
+            - name: NB_ANONYMOUS_SESSIONS_ENABLED
               value: {{ .Values.global.anonymousSessions.enabled | quote }}
-            - name: CULLING_REGISTERED_IDLE_SESSIONS_THRESHOLD_SECONDS
+            - name: NB_SESSIONS__CULLING__REGISTERED__IDLE_SECONDS
               value: {{ .Values.culling.idleThresholdSeconds.registered | quote }}
-            - name: CULLING_ANONYMOUS_IDLE_SESSIONS_THRESHOLD_SECONDS
+            - name: NB_SESSIONS__CULLING__ANONYMOUS__IDLE_SECONDS
               value: {{ .Values.culling.idleThresholdSeconds.anonymous | quote }}
-            - name: CULLING_REGISTERED_MAX_AGE_THRESHOLD_SECONDS
+            - name: NB_SESSIONS__CULLING__REGISTERED__MAX_AGE_SECONDS
               value: {{ .Values.culling.maxAgeSecondsThreshold.registered | quote }}
-            - name: CULLING_ANONYMOUS_MAX_AGE_THRESHOLD_SECONDS
+            - name: NB_SESSIONS__CULLING__ANONYMOUS__MAX_AGE_SECONDS
               value: {{ .Values.culling.maxAgeSecondsThreshold.anonymous | quote }}
-            - name: CRD_GROUP
+            - name: NB_AMALTHEA__GROUP
               value: {{ .Values.amalthea.crdApiGroup }}
-            - name: CRD_VERSION
+            - name: NB_AMALTHEA__VERSION
               value: {{ .Values.amalthea.crdApiVersion }}
-            - name: CRD_PLURAL
+            - name: NB_AMALTHEA__PLURAL
               value: {{ .Values.amalthea.crdNames.plural }}
-            - name: SENTRY_ENABLED
+            - name: NB_SENTRY__ENABLED
               value: {{ .Values.sentry.enabled | quote }}
-            - name: SENTRY_DSN
+            - name: NB_SENTRY__DSN
               value: {{ .Values.sentry.dsn | quote }}
-            - name: SENTRY_ENV
+            - name: NB_SENTRY__ENV
               value: {{ .Values.sentry.environment | quote }}
-            - name: SENTRY_SAMPLE_RATE
+            - name: NB_SENTRY__SAMPLE_RATE
               value: {{ .Values.sentry.sampleRate | quote }}
             - name: SENTRY_RELEASE
               value: {{ .Chart.Version | quote }}
-            - name: GIT_CLONE_SENTRY_ENABLED
+            - name: NB_SESSIONS__GIT_CLONE__SENTRY__ENABLED
               value: {{ .Values.sessionSentry.gitClone.enabled | quote }}
-            - name: GIT_CLONE_SENTRY_DSN
+            - name: NB_SESSIONS__GIT_CLONE__SENTRY__DSN
               value: {{ .Values.sessionSentry.gitClone.dsn | quote }}
-            - name: GIT_CLONE_SENTRY_ENV
+            - name: NB_SESSIONS__GIT_CLONE__SENTRY__ENV
               value: {{ .Values.sessionSentry.gitClone.environment | quote }}
-            - name: GIT_CLONE_SENTRY_SAMPLE_RATE
+            - name: NB_SESSIONS__GIT_CLONE__SENTRY__SAMPLE_RATE
               value: {{ .Values.sessionSentry.gitClone.sampleRate | quote }}
-            - name: SIDECAR_SENTRY_ENABLED
+            - name: NB_SESSIONS__GIT_RPC_SERVER__SENTRY__ENABLED
               value: {{ .Values.sessionSentry.sidecar.enabled | quote }}
-            - name: SIDECAR_SENTRY_DSN
+            - name: NB_SESSIONS__GIT_RPC_SERVER__SENTRY__DSN
               value: {{ .Values.sessionSentry.sidecar.dsn | quote }}
-            - name: SIDECAR_SENTRY_ENV
+            - name: NB_SESSIONS__GIT_RPC_SERVER__SENTRY__ENV
               value: {{ .Values.sessionSentry.sidecar.environment | quote }}
-            - name: SIDECAR_SENTRY_SAMPLE_RATE
+            - name: NB_SESSIONS__GIT_RPC_SERVER__SENTRY__SAMPLE_RATE
               value: {{ .Values.sessionSentry.sidecar.sampleRate | quote }}
-            - name: CERTIFICATES_IMAGE
+            - name: NB_SESSIONS__CA_CERTS__IMAGE
               value: "{{ .Values.global.certificates.image.repository }}:{{ .Values.global.certificates.image.tag }}"
-            - name: CUSTOM_CA_CERTS_SECRETS
+            - name: NB_SESSIONS__CA_CERTS__SECRETS
               value: |
                 {{- .Values.global.certificates.customCAs | toYaml | nindent 16 }}
             {{- with .Values.sessionNodeSelector }}
-            - name: SESSION_NODE_SELECTOR
+            - name: NB_SESSIONS__NODE_SELECTOR
               value: |
                 {{- toYaml . | nindent 16 }}
             {{- end }}
             {{- with .Values.sessionAffinity }}
-            - name: SESSION_AFFINITY
+            - name: NB_SESSIONS__AFFINITY
               value: |
                 {{- toYaml . | nindent 16 }}
             {{- end }}
             {{- with .Values.sessionTolerations }}
-            - name: SESSION_TOLERATIONS
+            - name: NB_SESSIONS__TOLERATIONS
               value: |
                 {{- toYaml . | nindent 16 }}
             {{- end }}
@@ -150,16 +150,20 @@ spec:
                   apiVersion: v1
                   fieldPath: metadata.namespace
             {{- include "certificates.env.python" . | nindent 12 }}
-            - name: ENFORCE_CPU_LIMITS
+            - name: NB_SESSIONS__ENFORCE_CPU_LIMITS
               value: {{ .Values.enforceCPULimits | quote }}
-            - name: S3_MOUNTS_ENABLED
+            - name: NB_S3_MOUNTS_ENABLED
               value: {{ .Values.cloudstorage.s3.enabled | quote }}
-            - name: SESSION_TERMINATION_GRACE_PERIOD_SECONDS
+            - name: NB_SESSIONS__TERMINATION_GRACE_PERIOD_SECONDS
               value: {{ .Values.sessionAutosave.terminationGracePeriodSeconds | quote }}
-            - name: AUTOSAVE_MINIMUM_LFS_FILE_SIZE_BYTES
+            - name: NB_SESSIONS__AUTOSAVE_MINIMUM_LFS_FILE_SIZE_BYTES
               value: {{ .Values.sessionAutosave.minimumLFSFileSizeBytes | quote }}
-            - name: NOTEBOOKS_SERVICE_VERSION
+            - name: NB_VERSION
               value: {{ .Values.image.tag | quote }}
+            - name: NB_K8S__NAMESPACE
+              valueFrom:
+                fieldRef:
+                  fieldPath: metadata.namespace
           ports:
             - name: http
               containerPort: 8000
@@ -179,7 +183,7 @@ spec:
               port: http
             periodSeconds: 30
           resources:
-{{ toYaml .Values.resources | indent 12 }}
+            {{- toYaml .Values.resources | nindent 12 }}
       initContainers:
         {{- include "certificates.initContainer" . | nindent 8 }}
         - name: k8s-resource-schema-migrations
@@ -214,7 +218,7 @@ spec:
       serviceAccountName: {{ if .Values.rbac.create }}"{{ template "notebooks.fullname" . }}"{{ else }}"{{ .Values.rbac.serviceAccountName }}"{{ end }}
     {{- with .Values.nodeSelector }}
       nodeSelector:
-{{ toYaml . | indent 8 }}
+        {{ toYaml . | indent 8 }}
     {{- end }}
       affinity:
         podAntiAffinity:
@@ -228,13 +232,13 @@ spec:
                       values:
                         - {{ template "notebooks.name" . }}
                 topologyKey: "kubernetes.io/hostname"
-    {{- with .Values.affinity }}
-{{ toYaml . | indent 8 }}
-    {{- end }}
-    {{- with .Values.tolerations }}
+        {{- with .Values.affinity }}
+        {{ toYaml . | indent 8 }}
+        {{- end }}
+      {{- with .Values.tolerations }}
       tolerations:
-{{ toYaml . | indent 8 }}
-    {{- end }}
+        {{ toYaml . | indent 8 }}
+      {{- end }}
       {{- if .Values.image.pullSecrets }}
       imagePullSecrets:
       {{- range .Values.image.pullSecrets }}