add readiness-probe check to k8sV1 backend

terascope · Aug 21, 2024 · 93a87a2 · 93a87a2
1 parent 6330ea1
commit 93a87a2
Show file tree

Hide file tree

Showing 2 changed files with 25 additions and 2 deletions.
diff --git a/packages/teraslice/src/lib/cluster/services/cluster/backends/kubernetes/index.ts b/packages/teraslice/src/lib/cluster/services/cluster/backends/kubernetes/index.ts
@@ -132,6 +132,7 @@ export class KubernetesClusterBackend {
 
         const pod = await this.k8s.waitForSelectedPod(
             `${controllerLabel}=${controllerUid}`,
+            'pod-status',
             undefined,
             this.context.sysconfig.teraslice.slicer_timeout
         );
@@ -162,6 +163,14 @@ export class KubernetesClusterBackend {
         // @ts-expect-error
         execution.k8sUid = jobs.items[0].metadata.uid;
 
+        /// Wait for ex readiness probe to return 'Ready'
+        await this.k8s.waitForSelectedPod(
+            selector,
+            'readiness-probe',
+            undefined,
+            this.context.sysconfig.teraslice.slicer_timeout
+        );
+
         const kr = new K8sResource(
             'deployments',
             'worker',

diff --git a/packages/teraslice/src/lib/cluster/services/cluster/backends/kubernetes/k8s.ts b/packages/teraslice/src/lib/cluster/services/cluster/backends/kubernetes/k8s.ts
@@ -87,14 +87,15 @@ export class K8s {
      * NOTE: If your selector will return multiple pods, this method probably
      * won't work for you.
      * @param {String} selector kubernetes selector, like 'controller-uid=XXX'
+     * @param {String} statusType type of status to check, either pod status or readiness probe
      * @param {String} ns       namespace to search, this will override the default
      * @param {Number} timeout  time, in ms, to wait for pod to start
      * @return {Object}         pod
      *
      * TODO: Should this use the cluster state that gets polled periodically,
      * rather than making it's own k8s API calls
      */
-    async waitForSelectedPod(selector: string, ns?: string, timeout = 10000) {
+    async waitForSelectedPod(selector: string, statusType: string, ns?: string, timeout = 10000) {
         const namespace = ns || this.defaultNamespace;
         let now = Date.now();
         const end = now + timeout;
@@ -112,7 +113,20 @@ export class K8s {
             }
 
             if (typeof pod !== 'undefined' && pod) {
-                if (get(pod, 'status.phase') === 'Running') return pod;
+                if (statusType === 'readiness-probe') {
+                    if (pod.status?.conditions) {
+                        for (const condition of pod.status.conditions) {
+                            if (
+                                condition.type === 'ContainersReady'
+                                && condition.status === 'True'
+                            ) {
+                                return pod;
+                            }
+                        }
+                    }
+                } else if (statusType === 'pod-status') {
+                    if (get(pod, 'status.phase') === 'Running') return pod;
+                }
             }
             if (now > end) throw new Error(`Timeout waiting for pod matching: ${selector}`);
             this.logger.debug(`waiting for pod matching: ${selector}`);