Merge pull request #563 from lsst-sqre/tickets/DM-33560

Rely on the event loop: use `kubernetes_asyncio` instead of `kubernetes` and dedicated threads
jupyterhub · Feb 22, 2022 · 4743a22 · 4743a22
2 parents a669816 + dc18487
commit 4743a22
Show file tree

Hide file tree

Showing 15 changed files with 581 additions and 510 deletions.
diff --git a/.github/workflows/publish.yaml b/.github/workflows/publish.yaml
@@ -25,7 +25,7 @@ jobs:
       - uses: actions/checkout@v2
       - uses: actions/setup-python@v2
         with:
-          python-version: 3.8
+          python-version: "3.9"
 
       - name: install build package
         run: |

diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml
@@ -38,41 +38,24 @@ jobs:
         # gain meaning on how job steps use them.
         #
         # k3s-channel:        https://update.k3s.io/v1-release/channels
-        # k8s-python-client:  https://github.com/kubernetes-client/python#compatibility
+        # kubernetes_asyncio:  https://github.com/tomplus/kubernetes_asyncio/tags
         #
         include:
-          # Tests with various k8s-python-client versions
-          - python: 3.7
-            k3s: v1.16
-            k8s-python-client: 10.* # supports k8s 1.14
-          - python: 3.8
-            k3s: v1.16
-            k8s-python-client: 11.* # supports k8s 1.15
-          - python: 3.9
-            k3s: v1.16
-            k8s-python-client: 12.* # supports k8s 1.16
-          - python: 3.9
+          # Tests with oldest supported Python, k8s, and k8s client
+          - python: "3.7"
             k3s: v1.17
-            k8s-python-client: 17.* # supports k8s 1.17
+            test_dependencies: kubernetes_asyncio==19.*
 
-          # Test with pre-releases of k8s-python-client
-          - python: 3.9
-            k3s: v1.18
-            k8s-python-client: pre
-
-          # Test with various recent k8s versions
-          - python: 3.8
+          # Test with modern python and k8s versions
+          - python: "3.9"
             k3s: stable
-            k8s-python-client: 17.*
-          - python: 3.8
+          - python: "3.9"
             k3s: latest
-            k8s-python-client: 17.*
 
-          # Test with JupyterHub in main branch
-          - python: 3.8
-            k3s: v1.16
-            k8s-python-client: 17.*
-            jupyterhub: main
+          # Test with latest python and JupyterHub in main branch
+          - python: "3.10"
+            k3s: latest
+            test_dependencies: git+https://github.com/jupyterhub/jupyterhub
 
     steps:
       - uses: actions/checkout@v2
@@ -81,31 +64,11 @@ jobs:
           python-version: "${{ matrix.python }}"
 
       - name: Install package and test dependencies
-        env:
-          K8S_PYTHON_CLIENT_VERSION: "${{ matrix.k8s-python-client }}"
-          JUPYTERHUB_VERSION: ${{ matrix.jupyterhub }}
         run: |
-          if [[ "$K8S_PYTHON_CLIENT_VERSION" == "pre" ]]; then
-              PRE="--pre"
-              PINS=""
-          else
-              PRE=""
-              PINS="kubernetes==${K8S_PYTHON_CLIENT_VERSION}"
-          fi
-          if [ "$JUPYTERHUB_VERSION" == "main" ]; then
-              PINS="$PINS git+https://github.com/jupyterhub/jupyterhub"
-          fi
-
-          pip install --upgrade setuptools pip
-          pip install -e ".[test]" ${PRE} ${PINS}
+          pip install --upgrade pip
+          pip install -e ".[test]" ${{ matrix.test_dependencies }}
           pip freeze
 
-      # flake8 runs a very quick code analysis without running any of the code
-      # it analyses
-      - name: Run flake8
-        run: |
-          flake8 kubespawner
-
       # Starts a k8s cluster with NetworkPolicy enforcement and installs both
       # kubectl and helm. We won't need network policy enforcement or helm
       # though.
@@ -141,7 +104,7 @@ jobs:
       - uses: actions/checkout@v2
       - uses: actions/setup-python@v2
         with:
-          python-version: 3.8
+          python-version: "3.9"
 
       - name: install build package
         run: |

diff --git a/kubespawner/clients.py b/kubespawner/clients.py
@@ -1,28 +1,35 @@
-"""Shared clients for kubernetes
+"""Configures and instantiates REST API clients of various kinds to
+communicate with a Kubernetes api-server, but only one instance per kind is
+instantiated.
 
-avoids creating multiple kubernetes client objects,
-each of which spawns an unused max-size thread pool
+The instances of these REST API clients are also patched to avoid the creation
+of unused threads.
 """
 import weakref
 from unittest.mock import Mock
 
-import kubernetes.client
-from kubernetes.client import api_client
+import kubernetes_asyncio.client
+from kubernetes_asyncio.client import api_client
+from kubernetes_asyncio.client import Configuration
 
-# FIXME: remove when instantiating a kubernetes client
-# doesn't create N-CPUs threads unconditionally.
-# monkeypatch threadpool in kubernetes api_client
-# to avoid instantiating ThreadPools.
-# This is known to work for kubernetes-4.0
-# and may need updating with later kubernetes clients
+# FIXME: Remove this workaround when instantiating a k8s client doesn't
+#        automatically create a ThreadPool with 1 thread that we won't use
+#        anyhow. To know if that has happened, reading
+#        https://github.com/jupyterhub/kubespawner/issues/567 may be helpful.
+#
+#        The workaround is to monkeypatch ThreadPool in the kubernetes
+#        api_client to avoid ThreadPools. This is known to work with both
+#        `kubernetes` and `kubernetes_asyncio`.
+#
 _dummy_pool = Mock()
 api_client.ThreadPool = lambda *args, **kwargs: _dummy_pool
 
 _client_cache = {}
 
 
 def shared_client(ClientType, *args, **kwargs):
-    """Return a single shared kubernetes client instance
+    """Return a shared kubernetes client instance
+    based on the provided arguments.
 
     A weak reference to the instance is cached,
     so that concurrent calls to shared_client
@@ -38,11 +45,38 @@ def shared_client(ClientType, *args, **kwargs):
         client = _client_cache[cache_key]()
 
     if client is None:
-        # Kubernetes client configuration is handled globally
-        # in kubernetes.py and is already called in spawner.py
-        # or proxy.py prior to a shared_client being instantiated
-        Client = getattr(kubernetes.client, ClientType)
+        # Kubernetes client configuration is handled globally and should already
+        # be configured from spawner.py or proxy.py via the load_config function
+        # prior to a shared_client being instantiated.
+        Client = getattr(kubernetes_asyncio.client, ClientType)
         client = Client(*args, **kwargs)
         # cache weakref so that clients can be garbage collected
         _client_cache[cache_key] = weakref.ref(client)
+
     return client
+
+
+async def load_config(caller):
+    """
+    Loads global configuration for the Python client we use to communicate with
+    a Kubernetes API server, and optionally tweaks that configuration based on
+    specific settings on the passed caller object.
+
+    This needs to be called before creating a kubernetes client, so practically
+    before the shared_client function is called. The caller must have both the
+    k8s_api_ssl_ca_cert and k8s_api_host attributes. KubeSpawner and
+    KubeIngressProxy both have these attributes.
+    """
+    try:
+        kubernetes_asyncio.config.load_incluster_config()
+    except kubernetes_asyncio.config.ConfigException:
+        await kubernetes_asyncio.config.load_kube_config()
+
+    if caller.k8s_api_ssl_ca_cert:
+        global_conf = Configuration.get_default_copy()
+        global_conf.ssl_ca_cert = caller.k8s_api_ssl_ca_cert
+        Configuration.set_default(global_conf)
+    if caller.k8s_api_host:
+        global_conf = Configuration.get_default_copy()
+        global_conf.host = caller.k8s_api_host
+        Configuration.set_default(global_conf)
diff --git a/kubespawner/objects.py b/kubespawner/objects.py
@@ -9,50 +9,49 @@
 import re
 from urllib.parse import urlparse
 
-from kubernetes.client.models import V1Affinity
-from kubernetes.client.models import V1Container
-from kubernetes.client.models import V1ContainerPort
-from kubernetes.client.models import V1EndpointAddress
-from kubernetes.client.models import V1Endpoints
-from kubernetes.client.models import V1EndpointSubset
-from kubernetes.client.models import V1EnvVar
-from kubernetes.client.models import V1LabelSelector
-from kubernetes.client.models import V1Lifecycle
-from kubernetes.client.models import V1LocalObjectReference
-from kubernetes.client.models import V1Namespace
-from kubernetes.client.models import V1NodeAffinity
-from kubernetes.client.models import V1NodeSelector
-from kubernetes.client.models import V1NodeSelectorRequirement
-from kubernetes.client.models import V1NodeSelectorTerm
-from kubernetes.client.models import V1ObjectMeta
-from kubernetes.client.models import V1OwnerReference
-from kubernetes.client.models import V1PersistentVolumeClaim
-from kubernetes.client.models import V1PersistentVolumeClaimSpec
-from kubernetes.client.models import V1Pod
-from kubernetes.client.models import V1PodAffinity
-from kubernetes.client.models import V1PodAffinityTerm
-from kubernetes.client.models import V1PodAntiAffinity
-from kubernetes.client.models import V1PodSecurityContext
-from kubernetes.client.models import V1PodSpec
-from kubernetes.client.models import V1PreferredSchedulingTerm
-from kubernetes.client.models import V1ResourceRequirements
-from kubernetes.client.models import V1Secret
-from kubernetes.client.models import V1SecurityContext
-from kubernetes.client.models import V1Service
-from kubernetes.client.models import V1ServicePort
-from kubernetes.client.models import V1ServiceSpec
-from kubernetes.client.models import V1Toleration
-from kubernetes.client.models import V1Volume
-from kubernetes.client.models import V1VolumeMount
-from kubernetes.client.models import V1WeightedPodAffinityTerm
+from kubernetes_asyncio.client.models import V1Affinity
+from kubernetes_asyncio.client.models import V1Container
+from kubernetes_asyncio.client.models import V1ContainerPort
+from kubernetes_asyncio.client.models import V1EndpointAddress
+from kubernetes_asyncio.client.models import V1Endpoints
+from kubernetes_asyncio.client.models import V1EndpointSubset
+from kubernetes_asyncio.client.models import V1EnvVar
+from kubernetes_asyncio.client.models import V1LabelSelector
+from kubernetes_asyncio.client.models import V1Lifecycle
+from kubernetes_asyncio.client.models import V1LocalObjectReference
+from kubernetes_asyncio.client.models import V1Namespace
+from kubernetes_asyncio.client.models import V1NodeAffinity
+from kubernetes_asyncio.client.models import V1NodeSelector
+from kubernetes_asyncio.client.models import V1NodeSelectorRequirement
+from kubernetes_asyncio.client.models import V1NodeSelectorTerm
+from kubernetes_asyncio.client.models import V1ObjectMeta
+from kubernetes_asyncio.client.models import V1OwnerReference
+from kubernetes_asyncio.client.models import V1PersistentVolumeClaim
+from kubernetes_asyncio.client.models import V1PersistentVolumeClaimSpec
+from kubernetes_asyncio.client.models import V1Pod
+from kubernetes_asyncio.client.models import V1PodAffinity
+from kubernetes_asyncio.client.models import V1PodAffinityTerm
+from kubernetes_asyncio.client.models import V1PodAntiAffinity
+from kubernetes_asyncio.client.models import V1PodSecurityContext
+from kubernetes_asyncio.client.models import V1PodSpec
+from kubernetes_asyncio.client.models import V1PreferredSchedulingTerm
+from kubernetes_asyncio.client.models import V1ResourceRequirements
+from kubernetes_asyncio.client.models import V1Secret
+from kubernetes_asyncio.client.models import V1SecurityContext
+from kubernetes_asyncio.client.models import V1Service
+from kubernetes_asyncio.client.models import V1ServicePort
+from kubernetes_asyncio.client.models import V1ServiceSpec
+from kubernetes_asyncio.client.models import V1Toleration
+from kubernetes_asyncio.client.models import V1Volume
+from kubernetes_asyncio.client.models import V1VolumeMount
+from kubernetes_asyncio.client.models import V1WeightedPodAffinityTerm
 
 try:
-    from kubernetes.client.models import CoreV1EndpointPort
+    from kubernetes_asyncio.client.models import CoreV1EndpointPort
 except ImportError:
-    from kubernetes.client.models import V1EndpointPort as CoreV1EndpointPort
+    from kubernetes_asyncio.client.models import V1EndpointPort as CoreV1EndpointPort
 
-from kubespawner.utils import get_k8s_model
-from kubespawner.utils import update_k8s_model
+from kubespawner.utils import get_k8s_model, update_k8s_model
 
 
 def make_pod(
@@ -734,23 +733,33 @@ def make_ingress(name, routespec, target, labels, data):
     # to keep compatibility with older K8S versions
 
     try:
-        from kubernetes.client.models import (
-            ExtensionsV1beta1Ingress,
-            ExtensionsV1beta1IngressSpec,
-            ExtensionsV1beta1IngressRule,
-            ExtensionsV1beta1HTTPIngressRuleValue,
+        from kubernetes_asyncio.client.models import (
             ExtensionsV1beta1HTTPIngressPath,
+            ExtensionsV1beta1HTTPIngressRuleValue,
+            ExtensionsV1beta1Ingress,
             ExtensionsV1beta1IngressBackend,
+            ExtensionsV1beta1IngressRule,
+            ExtensionsV1beta1IngressSpec,
         )
     except ImportError:
-        from kubernetes.client.models import (
-            V1beta1Ingress as ExtensionsV1beta1Ingress,
-            V1beta1IngressSpec as ExtensionsV1beta1IngressSpec,
-            V1beta1IngressRule as ExtensionsV1beta1IngressRule,
-            V1beta1HTTPIngressRuleValue as ExtensionsV1beta1HTTPIngressRuleValue,
+        from kubernetes_asyncio.client.models import (
             V1beta1HTTPIngressPath as ExtensionsV1beta1HTTPIngressPath,
+        )
+        from kubernetes_asyncio.client.models import (
+            V1beta1HTTPIngressRuleValue as ExtensionsV1beta1HTTPIngressRuleValue,
+        )
+        from kubernetes_asyncio.client.models import (
+            V1beta1Ingress as ExtensionsV1beta1Ingress,
+        )
+        from kubernetes_asyncio.client.models import (
             V1beta1IngressBackend as ExtensionsV1beta1IngressBackend,
         )
+        from kubernetes_asyncio.client.models import (
+            V1beta1IngressRule as ExtensionsV1beta1IngressRule,
+        )
+        from kubernetes_asyncio.client.models import (
+            V1beta1IngressSpec as ExtensionsV1beta1IngressSpec,
+        )
 
     meta = V1ObjectMeta(
         name=name,