From 9cdaca42b846354f990566249bfc0f6536f48345 Mon Sep 17 00:00:00 2001 From: busma13 Date: Wed, 30 Oct 2024 13:59:06 -0700 Subject: [PATCH 01/30] rename resource folders to not be plural --- .../backends/kubernetesV2/{deployments => deployment}/worker.hbs | 0 .../backends/kubernetesV2/{jobs => job}/execution_controller.hbs | 0 .../kubernetesV2/{services => service}/execution_controller.hbs | 0 3 files changed, 0 insertions(+), 0 deletions(-) rename packages/teraslice/src/lib/cluster/services/cluster/backends/kubernetesV2/{deployments => deployment}/worker.hbs (100%) rename packages/teraslice/src/lib/cluster/services/cluster/backends/kubernetesV2/{jobs => job}/execution_controller.hbs (100%) rename packages/teraslice/src/lib/cluster/services/cluster/backends/kubernetesV2/{services => service}/execution_controller.hbs (100%) diff --git a/packages/teraslice/src/lib/cluster/services/cluster/backends/kubernetesV2/deployments/worker.hbs b/packages/teraslice/src/lib/cluster/services/cluster/backends/kubernetesV2/deployment/worker.hbs similarity index 100% rename from packages/teraslice/src/lib/cluster/services/cluster/backends/kubernetesV2/deployments/worker.hbs rename to packages/teraslice/src/lib/cluster/services/cluster/backends/kubernetesV2/deployment/worker.hbs diff --git a/packages/teraslice/src/lib/cluster/services/cluster/backends/kubernetesV2/jobs/execution_controller.hbs b/packages/teraslice/src/lib/cluster/services/cluster/backends/kubernetesV2/job/execution_controller.hbs similarity index 100% rename from packages/teraslice/src/lib/cluster/services/cluster/backends/kubernetesV2/jobs/execution_controller.hbs rename to packages/teraslice/src/lib/cluster/services/cluster/backends/kubernetesV2/job/execution_controller.hbs diff --git a/packages/teraslice/src/lib/cluster/services/cluster/backends/kubernetesV2/services/execution_controller.hbs b/packages/teraslice/src/lib/cluster/services/cluster/backends/kubernetesV2/service/execution_controller.hbs similarity index 100% rename from packages/teraslice/src/lib/cluster/services/cluster/backends/kubernetesV2/services/execution_controller.hbs rename to packages/teraslice/src/lib/cluster/services/cluster/backends/kubernetesV2/service/execution_controller.hbs From 8810387f4b9d928668cd2939df6938e2e995ee24 Mon Sep 17 00:00:00 2001 From: busma13 Date: Wed, 30 Oct 2024 13:59:30 -0700 Subject: [PATCH 02/30] add types to k8s v2 --- .../cluster/backends/kubernetesV2/index.ts | 71 ++-- .../backends/kubernetesV2/interfaces.ts | 89 +++++ .../cluster/backends/kubernetesV2/k8s.ts | 334 +++++++++--------- .../backends/kubernetesV2/k8sResource.ts | 254 +++++++------ .../cluster/backends/kubernetesV2/k8sState.ts | 82 +++-- .../cluster/backends/kubernetesV2/utils.ts | 55 ++- 6 files changed, 515 insertions(+), 370 deletions(-) create mode 100644 packages/teraslice/src/lib/cluster/services/cluster/backends/kubernetesV2/interfaces.ts diff --git a/packages/teraslice/src/lib/cluster/services/cluster/backends/kubernetesV2/index.ts b/packages/teraslice/src/lib/cluster/services/cluster/backends/kubernetesV2/index.ts index d779308303b..39e8f3860aa 100644 --- a/packages/teraslice/src/lib/cluster/services/cluster/backends/kubernetesV2/index.ts +++ b/packages/teraslice/src/lib/cluster/services/cluster/backends/kubernetesV2/index.ts @@ -10,6 +10,7 @@ import { gen } from './k8sState.js'; import { K8s } from './k8s.js'; import { getRetryConfig } from './utils.js'; import { StopExecutionOptions } from '../../../interfaces.js'; +import { ResourceType } from './interfaces.js'; /* Execution Life Cycle for _status @@ -42,8 +43,7 @@ export class KubernetesClusterBackendV2 { this.logger, null, kubernetesNamespace, - // @ts-expect-error - context.sysconfig.teraslice.kubernetes_api_poll_delay, + context.sysconfig.teraslice.kubernetes_api_poll_delay || 1000, context.sysconfig.teraslice.shutdown_timeout ); @@ -65,18 +65,19 @@ export class KubernetesClusterBackendV2 { * app.kubernetes.io/name=teraslice * app.kubernetes.io/instance=${clusterNameLabel} * @constructor - * @return {Promise} [description] + * @return {Promise} void */ private async _getClusterState() { - return this.k8s.list(`app.kubernetes.io/name=teraslice,app.kubernetes.io/instance=${this.clusterNameLabel}`, 'pods') - .then((k8sPods) => gen(k8sPods, this.clusterState)) - .catch((err) => { - // TODO: We might need to do more here. I think it's OK to just - // log though. This only gets used to show slicer info through - // the API. We wouldn't want to disrupt the cluster master - // for rare failures to reach the k8s API. - logError(this.logger, err, 'Error listing teraslice pods in k8s'); - }); + try { + const k8sPods: K8sClient.V1PodList = await this.k8s.list(`app.kubernetes.io/name=teraslice,app.kubernetes.io/instance=${this.clusterNameLabel}`, 'pod'); + gen(k8sPods, this.clusterState, this.logger); + } catch (err) { + // TODO: We might need to do more here. I think it's OK to just + // log though. This only gets used to show slicer info through + // the API. We wouldn't want to disrupt the cluster master + // for rare failures to reach the k8s API. + logError(this.logger, err, 'Error listing teraslice pods in k8s'); + } } /** @@ -107,35 +108,41 @@ export class KubernetesClusterBackendV2 { execution.slicer_port = 45680; const exJobResource = new K8sResource( - 'jobs', + 'job', 'execution_controller', this.context.sysconfig.teraslice, execution, this.logger ); + + if (!(exJobResource.resource instanceof K8sClient.V1Job)) { + throw new Error(`exJobResource.resource must be of type k8s.V1Job`); + } + const exJob = exJobResource.resource; this.logger.debug(exJob, 'execution allocating slicer'); - const jobResult = await this.k8s.post(exJob, 'job') as K8sClient.V1Job; - - // I need to add these here to create the ex service resource - // @ts-expect-error - execution.k8sName = jobResult.metadata.name; - // @ts-expect-error - execution.k8sUid = jobResult.metadata.uid; + const jobResult = await this.k8s.post(exJob, 'job'); const exServiceResource = new K8sResource( - 'services', + 'service', 'execution_controller', this.context.sysconfig.teraslice, execution, - this.logger + this.logger, + // Needed to create the deployment and service resource ownerReferences + jobResult.metadata?.name, + jobResult.metadata?.uid ); + if (!(exServiceResource.resource instanceof K8sClient.V1Service)) { + throw new Error(`exJobResource.resource must be of type k8s.V1Service`); + } + const exService = exServiceResource.resource; - const serviceResult = await this.k8s.post(exService, 'service') as K8sClient.V1Service; + const serviceResult = await this.k8s.post(exService, 'service'); this.logger.debug(jobResult, 'k8s slicer job submitted'); @@ -183,12 +190,8 @@ export class KubernetesClusterBackendV2 { // instead. const selector = `app.kubernetes.io/component=execution_controller,teraslice.terascope.io/jobId=${execution.job_id}`; const jobs = await pRetry( - () => this.k8s.nonEmptyList(selector, 'jobs'), getRetryConfig() + () => this.k8s.nonEmptyJobList(selector), getRetryConfig() ); - // @ts-expect-error - execution.k8sName = jobs.items[0].metadata.name; - // @ts-expect-error - execution.k8sUid = jobs.items[0].metadata.uid; /// Wait for ex readiness probe to return 'Ready' await this.k8s.waitForSelectedPod( @@ -199,13 +202,18 @@ export class KubernetesClusterBackendV2 { ); const kr = new K8sResource( - 'deployments', + 'deployment', 'worker', this.context.sysconfig.teraslice, execution, - this.logger + this.logger, + jobs.items[0].metadata?.name, + jobs.items[0].metadata?.uid ); + if (!(kr.resource instanceof K8sClient.V1Deployment)) { + throw new Error(`exJobResource.resource must be of type k8s.V1Deployment`); + } const workerDeployment = kr.resource; this.logger.debug(`workerDeployment:\n\n${JSON.stringify(workerDeployment, null, 2)}`); @@ -265,7 +273,8 @@ export class KubernetesClusterBackendV2 { */ async listResourcesForJobId(jobId: string) { const resources = []; - const resourceTypes = ['pods', 'deployments', 'services', 'jobs', 'replicasets']; + const resourceTypes: ResourceType[] = ['pod', 'deployment', 'service', 'job', 'replicaset']; + for (const type of resourceTypes) { const list = await this.k8s.list(`teraslice.terascope.io/jobId=${jobId}`, type); if (list.items.length > 0) { diff --git a/packages/teraslice/src/lib/cluster/services/cluster/backends/kubernetesV2/interfaces.ts b/packages/teraslice/src/lib/cluster/services/cluster/backends/kubernetesV2/interfaces.ts new file mode 100644 index 00000000000..f46a1d2c208 --- /dev/null +++ b/packages/teraslice/src/lib/cluster/services/cluster/backends/kubernetesV2/interfaces.ts @@ -0,0 +1,89 @@ +import { IncomingMessage } from 'node:http'; +import * as k8s from '@kubernetes/client-node'; + +export interface K8sConfig { + clusterName: string; + clusterNameLabel: string; + configMapName: string; + dockerImage: string; + execution: string; + exId: string; + exName: string; + exUid: string; + jobId: string; + jobNameLabel: string; + name: string; + namespace: string; + nodeType: NodeType; + replicas: number; + shutdownTimeout: number; +} + +export type ResourceType = 'deployment' | 'job' | 'pod' | 'replicaset' | 'service'; + +export type ResourceList = k8s.V1DeploymentList | k8s.V1JobList + | k8s.V1PodList | k8s.V1ReplicaSetList | k8s.V1ServiceList; + +export type Resource = k8s.V1Deployment | k8s.V1Job | k8s.V1Pod | k8s.V1ReplicaSet | k8s.V1Service; + +export interface ResourceListApiResponse { + response: IncomingMessage; + body: ResourceList; +} + +export interface ResourceApiResponse { + response: IncomingMessage; + body: Resource; +} + +export interface PatchApiResponse { + response: IncomingMessage; + body: k8s.V1Deployment; +} + +export type DeleteResponseBody = k8s.V1Pod | k8s.V1Status | k8s.V1Service; + +export interface DeleteApiResponse { + response: IncomingMessage; + body: DeleteResponseBody; +} + +export type DeleteParams = [ + string, + string, + string | undefined, + string | undefined, + number | undefined, + boolean | undefined, + string | undefined, + k8s.V1DeleteOptions | undefined +]; + +export type NodeType = 'worker' | 'execution_controller'; + +export type ScaleOp = `set` | `add` | `remove`; + +// export type PostDeploymentArgs = { +// manifest: k8s.V1Deployment; +// manifestType: 'deployment'; +// }; + +// export type PostJobArgs = { +// manifest: k8s.V1Job; +// manifestType: 'job'; +// }; +// export type PostPodArgs = { +// manifest: k8s.V1Pod; +// manifestType: 'pod'; +// }; +// export type PostReplicaSetArgs = { +// manifest: k8s.V1ReplicaSet; +// manifestType: 'replicaset'; +// }; +// export type PostServiceArgs = { +// manifest: k8s.V1Service; +// manifestType: 'service'; +// }; + +// export type PostArgs = PostDeploymentArgs | PostJobArgs +// | PostPodArgs | PostReplicaSetArgs | PostServiceArgs; diff --git a/packages/teraslice/src/lib/cluster/services/cluster/backends/kubernetesV2/k8s.ts b/packages/teraslice/src/lib/cluster/services/cluster/backends/kubernetesV2/k8s.ts index 7fad705fc43..f8311b658f1 100644 --- a/packages/teraslice/src/lib/cluster/services/cluster/backends/kubernetesV2/k8s.ts +++ b/packages/teraslice/src/lib/cluster/services/cluster/backends/kubernetesV2/k8s.ts @@ -3,8 +3,13 @@ import { pDelay, pRetry, Logger } from '@terascope/utils'; import * as k8s from '@kubernetes/client-node'; -import { IncomingMessage } from 'node:http'; -import { getRetryConfig } from './utils.js'; +import { getRetryConfig, isDeployment, isJob, isPod, isReplicaSet, isService } from './utils.js'; +import { + DeleteApiResponse, DeleteParams, ResourceListApiResponse, + ResourceType, PatchApiResponse, Resource, + ResourceList, ResourceApiResponse, DeleteResponseBody, + NodeType, ScaleOp +} from './interfaces.js'; interface KubeConfigOptions { clusters: k8s.Cluster[]; @@ -82,7 +87,7 @@ export class K8s { * @param {String} selector kubernetes selector, like 'controller-uid=XXX' * @param {String} ns namespace to search, this will override the default * @param {Number} timeout time, in ms, to wait for pod to start - * @return {Object} pod + * @return {k8s.V1Pod} pod object * * TODO: Should this use the cluster state that gets polled periodically, * rather than making it's own k8s API calls @@ -132,7 +137,7 @@ export class K8s { * @param {String} selector kubernetes selector, like 'controller-uid=XXX' * @param {String} ns namespace to search, this will override the default * @param {Number} timeout time, in ms, to wait for pod to start - * @return {Array} Array of pod objects + * @return {k8s.V1Pod[]} Array of pod objects * * TODO: Should this use the cluster state that gets polled periodically, * rather than making it's own k8s API calls? @@ -147,12 +152,11 @@ export class K8s { .listNamespacedPod(namespace, undefined, undefined, undefined, undefined, selector), getRetryConfig()); - const podList: k8s.V1Pod[] | undefined = get(result, 'body.items'); + const podList: k8s.V1Pod[] = get(result, 'body.items'); - if (podList && Array.isArray(podList)) { - if (podList.length === number) return podList; - } - const msg = `Waiting: pods matching ${selector} is ${podList.length}/${number}`; // FIXME: this could be undefined -> typeError + if (podList.length === number) return podList; + + const msg = `Waiting: pods matching ${selector} is ${podList.length}/${number}`; if (now > end) throw new Error(`Timeout ${msg}`); this.logger.debug(msg); @@ -163,19 +167,25 @@ export class K8s { /** * returns list of k8s objects matching provided selector - * @param {String} selector kubernetes selector, like 'app=teraslice' - * @param {String} objType Type of k8s object to get, valid options: - * 'pods', 'deployment', 'services', 'jobs', 'replicasets' - * @param {String} ns namespace to search, this will override the default - * @return {Object} body of k8s get response. + * @param {String} selector kubernetes selector, like 'app=teraslice' + * @param {ResourceType} objType Type of k8s object to get, valid options: + * 'deployments', 'jobs', 'pods', 'replicasets', 'services' + * @param {String} ns namespace to search, this will override the default + * @return {k8s.V1PodList + * | k8s.V1DeploymentList + * | k8s.V1ServiceList + * | k8s.V1ReplicaSetList + * | k8s.V1JobList} list of k8s objects. */ - async list(selector: string, objType: string, ns?: string) { + async list(selector: string, objType: 'deployment', ns?: string): Promise; + async list(selector: string, objType: 'job', ns?: string): Promise; + async list(selector: string, objType: 'pod', ns?: string): Promise; + async list(selector: string, objType: 'replicaset', ns?: string): Promise; + async list(selector: string, objType: 'service', ns?: string): Promise; + async list(selector: string, objType: ResourceType, ns?: string): Promise; + async list(selector: string, objType: ResourceType, ns?: string): Promise { const namespace = ns || this.defaultNamespace; - let responseObj: { - response: IncomingMessage; - body: k8s.V1PodList | k8s.V1DeploymentList - | k8s.V1ServiceList | k8s.V1JobList | k8s.V1ReplicaSetList; - }; + let responseObj: ResourceListApiResponse; const params: [ string, @@ -193,63 +203,37 @@ export class K8s { selector ]; + const listFunctions: { [resource: string]: () => Promise } = { + deployment: () => this.k8sAppsV1Api.listNamespacedDeployment(...params), + job: () => this.k8sBatchV1Api.listNamespacedJob(...params), + pod: () => this.k8sCoreV1Api.listNamespacedPod(...params), + replicaset: () => this.k8sAppsV1Api.listNamespacedReplicaSet(...params), + service: () => this.k8sCoreV1Api.listNamespacedService(...params) + }; + try { - if (objType === 'pods') { - responseObj = await pRetry( - () => this.k8sCoreV1Api.listNamespacedPod(...params), - getRetryConfig() - ); - } else if (objType === 'deployments') { - responseObj = await pRetry( - () => this.k8sAppsV1Api.listNamespacedDeployment(...params), - getRetryConfig() - ); - } else if (objType === 'services') { - responseObj = await pRetry( - () => this.k8sCoreV1Api.listNamespacedService(...params), - getRetryConfig() - ); - } else if (objType === 'jobs') { - responseObj = await pRetry( - () => this.k8sBatchV1Api.listNamespacedJob(...params), - getRetryConfig() - ); - } else if (objType === 'replicasets') { - responseObj = await pRetry( - () => this.k8sAppsV1Api.listNamespacedReplicaSet(...params), - getRetryConfig() - ); - } else { - const error = new Error(`Wrong objType provided to get: ${objType}`); - this.logger.error(error); - return Promise.reject(error); - } + responseObj = await pRetry( + listFunctions[objType], + getRetryConfig() + ); + return responseObj.body; } catch (e) { const err = new Error(`Request k8s.list of ${objType} with selector ${selector} failed: ${e}`); this.logger.error(err); return Promise.reject(err); } - - if (responseObj.response.statusCode && responseObj.response.statusCode >= 400) { - const err = new TSError(`Problem when trying to k8s.list ${objType}`); - this.logger.error(err); - err.code = responseObj.response.statusCode.toString(); - return Promise.reject(err); - } - - return responseObj.body; } - async nonEmptyList(selector: string, objType: string) { - const jobs = await this.list(selector, objType); + async nonEmptyJobList(selector: string) { + const jobs = await this.list(selector, 'job'); if (jobs.items.length === 1) { return jobs; } else if (jobs.items.length === 0) { - const msg = `Teraslice ${objType} matching the following selector was not found: ${selector} (retriable)`; + const msg = `Teraslice job matching the following selector was not found: ${selector} (retriable)`; this.logger.warn(msg); throw new TSError(msg, { retryable: true }); } else { - throw new TSError(`Unexpected number of Teraslice ${objType}s matching the following selector: ${selector}`, { + throw new TSError(`Unexpected number of Teraslice jobs matching the following selector: ${selector}`, { retryable: true }); } @@ -257,43 +241,61 @@ export class K8s { /** * posts manifest to k8s - * @param {Object} manifest service manifest - * @param {String} manifestType 'service', 'deployment', 'job' - * @return {Object} body of k8s API response object + * @param {Resource} manifest resource manifest + * @param {ResourceType} manifestType 'deployment', 'job', 'pod', 'replicaset', 'service' + * @return {k8s.V1Deployment + * | k8s.V1Job + * | k8s.V1Pod + * | k8s.V1ReplicaSet + * | k8s.V1Service} body of k8s API response object */ - async post(manifest: Record, manifestType: string) { - let responseObj: { - response: IncomingMessage; - body: k8s.V1Service | k8s.V1Deployment | k8s.V1Job; - }; + async post(manifest: k8s.V1Deployment, manifestType: 'deployment'): Promise; + async post(manifest: k8s.V1Job, manifestType: 'job'): Promise; + async post(manifest: k8s.V1Pod, manifestType: 'pod'): Promise; + async post(manifest: k8s.V1ReplicaSet, manifestType: 'replicaset'): Promise; + async post(manifest: k8s.V1Service, manifestType: 'service'): Promise; + async post(manifest: Resource, manifestType: ResourceType): Promise { + let responseObj: ResourceApiResponse; + + // const postFunctions = { + // deployment: async (man: k8s.V1Deployment) => await this.k8sAppsV1Api + // .createNamespacedDeployment(this.defaultNamespace, man), + // job: async (man: k8s.V1Job) => await this.k8sBatchV1Api + // .createNamespacedJob(this.defaultNamespace, man), + // pod: async (man: k8s.V1Pod) => await this.k8sCoreV1Api + // .createNamespacedPod(this.defaultNamespace, man), + // replicaset: async (man: k8s.V1ReplicaSet) => await this.k8sAppsV1Api + // .createNamespacedReplicaSet(this.defaultNamespace, man), + // service: async (man: k8s.V1Service) => await this.k8sCoreV1Api + // .createNamespacedService(this.defaultNamespace, man) + // }; try { - if (manifestType === 'service') { - responseObj = await this.k8sCoreV1Api - .createNamespacedService(this.defaultNamespace, manifest); - } else if (manifestType === 'deployment') { + if (isDeployment(manifest)) { responseObj = await this.k8sAppsV1Api .createNamespacedDeployment(this.defaultNamespace, manifest); - } else if (manifestType === 'job') { + } else if (isJob(manifest)) { responseObj = await this.k8sBatchV1Api .createNamespacedJob(this.defaultNamespace, manifest); + } else if (isPod(manifest)) { + responseObj = await this.k8sCoreV1Api + .createNamespacedPod(this.defaultNamespace, manifest); + } else if (isReplicaSet(manifest)) { + responseObj = await this.k8sAppsV1Api + .createNamespacedReplicaSet(this.defaultNamespace, manifest); + } else if (isService(manifest)) { + responseObj = await this.k8sCoreV1Api + .createNamespacedService(this.defaultNamespace, manifest); } else { const error = new Error(`Invalid manifestType: ${manifestType}`); return Promise.reject(error); } + // responseObj = await postFunctions[manifestType](manifest); + return responseObj.body; } catch (e) { const err = new Error(`Request k8s.post of ${manifestType} with body ${JSON.stringify(manifest)} failed: ${e}`); return Promise.reject(err); } - - if (responseObj.response.statusCode && responseObj.response.statusCode >= 400) { - const err = new TSError(`Problem when trying to k8s.post ${manifestType} with body ${JSON.stringify(manifest)}`); - this.logger.error(err); - err.code = responseObj.response.statusCode.toString(); - return Promise.reject(err); - } - - return responseObj.body; } /** @@ -306,11 +308,7 @@ export class K8s { // the low level k8s api method, I expect to eventually change the interface // on this to require `objType` to support patching other things async patch(record: Record, name: string) { - let responseObj: { - response: IncomingMessage; - body: k8s.V1Service | k8s.V1Deployment | k8s.V1Job; - }; - + let responseObj: PatchApiResponse; try { const options = { headers: { 'Content-type': k8s.PatchUtils.PATCH_FORMAT_JSON_PATCH } }; responseObj = await pRetry(() => this.k8sAppsV1Api @@ -325,40 +323,43 @@ export class K8s { undefined, options, ), getRetryConfig()); + return responseObj.body; } catch (e) { - const err = new Error(`Request k8s.patch with ${name} failed with: ${e}`); + const err = new Error(`Request k8s.patch with name: ${name} failed with: ${e}`); this.logger.error(err); return Promise.reject(err); } - - if (responseObj.response.statusCode && responseObj.response.statusCode >= 400) { - const err = new TSError(`Unexpected response code (${responseObj.response.statusCode}), when patching ${name} with body ${JSON.stringify(record)}`); - this.logger.error(err); - err.code = responseObj.response.statusCode.toString(); - return Promise.reject(err); - } - - return responseObj.body; } /** * Deletes k8s object of specified objType - * @param {String} name Name of the resource to delete - * @param {String} objType Type of k8s object to get, valid options: - * 'deployments', 'services', 'jobs', 'pods' - * @param {Boolean} force Forcefully delete resource by setting gracePeriodSeconds to 1 + * @param {String} name Name of the resource to delete + * @param {ResourceType} objType Type of k8s object to get, valid options: + * 'deployments', 'services', 'jobs', 'pods', 'replicasets' + * @param {Boolean} force Forcefully delete resource by setting gracePeriodSeconds to 1 * to be forcefully stopped. * @return {Object} k8s delete response body. */ - async delete(name: string, objType: string, force?: boolean) { + async delete( + name: string, objType: 'pod', force?: boolean + ): Promise; + async delete( + name: string, objType: 'deployment' | 'job' | 'replicaset', force?: boolean + ): Promise; + async delete( + name: string, objType: 'service', force?: boolean + ): Promise; + async delete( + name: string, objType: ResourceType, force?: boolean + ): Promise; + async delete( + name: string, objType: ResourceType, force?: boolean + ): Promise { if (name === undefined || name.trim() === '') { throw new Error(`Name of resource to delete must be specified. Received: "${name}".`); } - let responseObj: { - response: IncomingMessage; - body: k8s.V1Status | k8s.V1Pod | k8s.V1Service; - }; + let responseObj: DeleteApiResponse; // To get a Job to remove the associated pods you have to // include a body like the one below with the delete request. @@ -373,16 +374,7 @@ export class K8s { deleteOptions.gracePeriodSeconds = 1; } - const params: [ - string, - string, - string | undefined, - string | undefined, - number | undefined, - boolean | undefined, - string | undefined, - k8s.V1DeleteOptions | undefined - ] = [ + const params: DeleteParams = [ name, this.defaultNamespace, undefined, @@ -393,10 +385,15 @@ export class K8s { deleteOptions ]; - const deleteWithErrorHandling = async (deleteFn: () => Promise<{ - response: IncomingMessage; - body: k8s.V1Status | k8s.V1Pod | k8s.V1Service; - }>) => { + const deleteFunctions: { [resource: string]: () => Promise } = { + deployment: () => this.k8sAppsV1Api.deleteNamespacedDeployment(...params), + job: () => this.k8sBatchV1Api.deleteNamespacedJob(...params), + pod: () => this.k8sCoreV1Api.deleteNamespacedPod(...params), + replicaset: () => this.k8sAppsV1Api.deleteNamespacedReplicaSet(...params), + service: () => this.k8sCoreV1Api.deleteNamespacedService(...params) + }; + + const deleteWithErrorHandling = async (deleteFn: () => Promise) => { try { const res = await deleteFn(); return res; @@ -407,44 +404,22 @@ export class K8s { this.logger.info(`No ${objType} with name ${name} found while attempting to delete.`); return e; } - - if (e.statusCode >= 400) { - const err = new TSError(`Unexpected response code (${e.statusCode}), when deleting name: ${name}`); - this.logger.error(err); - err.code = e.statusCode.toString(); - return Promise.reject(err); - } } throw e; } }; try { - if (objType === 'services') { - responseObj = await pRetry(() => deleteWithErrorHandling(() => this.k8sCoreV1Api - .deleteNamespacedService(...params)), getRetryConfig()); - } else if (objType === 'deployments') { - responseObj = await pRetry(() => deleteWithErrorHandling(() => this.k8sAppsV1Api - .deleteNamespacedDeployment(...params)), getRetryConfig()); - } else if (objType === 'jobs') { - responseObj = await pRetry(() => deleteWithErrorHandling(() => this.k8sBatchV1Api - .deleteNamespacedJob(...params)), getRetryConfig()); - } else if (objType === 'pods') { - responseObj = await pRetry(() => deleteWithErrorHandling(() => this.k8sCoreV1Api - .deleteNamespacedPod(...params)), getRetryConfig()); - } else if (objType === 'replicasets') { - responseObj = await pRetry(() => deleteWithErrorHandling(() => this.k8sAppsV1Api - .deleteNamespacedReplicaSet(...params)), getRetryConfig()); - } else { - throw new Error(`Invalid objType: ${objType}`); - } + responseObj = await pRetry( + () => deleteWithErrorHandling(deleteFunctions[objType]), + getRetryConfig() + ); + return responseObj.body; } catch (e) { const err = new Error(`Request k8s.delete with name: ${name} failed with: ${e}`); this.logger.error(err); return Promise.reject(err); } - - return responseObj.body; } /** @@ -462,31 +437,43 @@ export class K8s { if (force) { // Order matters. If we delete a parent resource before its children it // will be marked for background deletion and then can't be force deleted. - await this._deleteObjByExId(exId, 'worker', 'pods', force); - await this._deleteObjByExId(exId, 'worker', 'replicasets', force); - await this._deleteObjByExId(exId, 'worker', 'deployments', force); - await this._deleteObjByExId(exId, 'execution_controller', 'pods', force); - await this._deleteObjByExId(exId, 'execution_controller', 'services', force); + await this._deleteObjByExId(exId, 'worker', 'pod', force); + await this._deleteObjByExId(exId, 'worker', 'replicaset', force); + await this._deleteObjByExId(exId, 'worker', 'deployment', force); + await this._deleteObjByExId(exId, 'execution_controller', 'pod', force); + await this._deleteObjByExId(exId, 'execution_controller', 'service', force); } - await this._deleteObjByExId(exId, 'execution_controller', 'jobs', force); + await this._deleteObjByExId(exId, 'execution_controller', 'job', force); } /** * Finds the k8s objects by nodeType and exId and then deletes them * @param {String} exId Execution ID - * @param {String} nodeType valid Teraslice k8s node type: + * @param {NodeType} nodeType valid Teraslice k8s node type: * 'worker', 'execution_controller' - * @param {String} objType valid object type: `services`, `deployments`, - * `jobs`, `pods`, `replicasets` + * @param {ResourceType} objType valid object type: `service`, `deployment`, + * `job`, `pod`, `replicaset` * @param {Boolean} force Forcefully stop all resources * @return {Promise} */ - async _deleteObjByExId(exId: string, nodeType: string, objType: string, force?: boolean) { + async _deleteObjByExId( + exId: string, nodeType: NodeType, objType: 'pod', force?: boolean + ): Promise; + + async _deleteObjByExId( + exId: string, nodeType: NodeType, objType: 'job' | 'replicaset' | 'deployment', force?: boolean + ): Promise; + + async _deleteObjByExId( + exId: string, nodeType: NodeType, objType: 'service', force?: boolean + ): Promise; + + async _deleteObjByExId( + exId: string, nodeType: NodeType, objType: ResourceType, force?: boolean + ): Promise { let objList: K8sObjectList; - const deleteResponses: Array< - k8s.V1Pod[] | k8s.V1Pod | k8s.V1Service | k8s.V1Status | k8s.V1ReplicaSet - > = []; + const deleteResponses: Array = []; try { objList = await this.list(`app.kubernetes.io/component=${nodeType},teraslice.terascope.io/exId=${exId}`, objType); @@ -536,19 +523,28 @@ export class K8s { * of workers. * @param {String} exId exId of execution to scale * @param {number} numWorkers number of workers to scale by - * @param {String} op Scale operation: `set`, `add`, `remove` + * @param {ScaleOp} op Scale operation: `set`, `add`, `remove` * @return {Object} Body of patch response. */ - async scaleExecution(exId: string, numWorkers: number, op: string) { - let newScale; + async scaleExecution(exId: string, numWorkers: number, op: ScaleOp): Promise { + let newScale: number; + const selector = `app.kubernetes.io/component=worker,teraslice.terascope.io/exId=${exId}`; this.logger.info(`Scaling exId: ${exId}, op: ${op}, numWorkers: ${numWorkers}`); - const listResponse = await this.list(`app.kubernetes.io/component=worker,teraslice.terascope.io/exId=${exId}`, 'deployments') as k8s.V1DeploymentList; + const listResponse = await this.list(selector, 'deployment'); this.logger.debug(`k8s worker query listResponse: ${JSON.stringify(listResponse)}`); // the selector provided to list above should always result in a single // deployment in the response. - // TODO: test for more than 1 and error + if (listResponse.items.length === 0) { + const msg = `Teraslice deployment matching the following selector was not found: ${selector} (retriable)`; + this.logger.warn(msg); + throw new TSError(msg, { retryable: true }); + } else if (listResponse.items.length > 1) { + throw new TSError(`Unexpected number of Teraslice deployments matching the following selector: ${selector}`, { + retryable: true + }); + } const workerDeployment = listResponse.items[0]; if (workerDeployment.spec?.replicas === undefined) { throw new Error('replicas is undefined in worker deployment spec'); @@ -579,7 +575,7 @@ export class K8s { throw new Error('name is undefined in worker deployment metadata'); } const patchResponseBody = await this - .patch(scalePatch, workerDeployment.metadata.name) as k8s.V1Deployment; + .patch(scalePatch, workerDeployment.metadata.name); this.logger.debug(`k8s.scaleExecution patchResponseBody: ${JSON.stringify(patchResponseBody)}`); return patchResponseBody; } diff --git a/packages/teraslice/src/lib/cluster/services/cluster/backends/kubernetesV2/k8sResource.ts b/packages/teraslice/src/lib/cluster/services/cluster/backends/kubernetesV2/k8sResource.ts index 22812e5f8f8..78e1be6dbe5 100644 --- a/packages/teraslice/src/lib/cluster/services/cluster/backends/kubernetesV2/k8sResource.ts +++ b/packages/teraslice/src/lib/cluster/services/cluster/backends/kubernetesV2/k8sResource.ts @@ -1,60 +1,46 @@ -import fs from 'node:fs'; -import path from 'node:path'; -// @ts-expect-error no types found -import barbe from 'barbe'; import _ from 'lodash'; +import * as k8s from '@kubernetes/client-node'; import { isNumber, Logger } from '@terascope/utils'; import type { TerasliceConfig, ExecutionConfig } from '@terascope/job-components'; import { safeEncode } from '../../../../../utils/encoding_utils.js'; -import { setMaxOldSpaceViaEnv } from './utils.js'; - -const resourcePath = path.join(process.cwd(), './packages/teraslice/src/lib/cluster/services/cluster/backends/kubernetesV2/'); - -interface K8sConfig { - clusterName: string; - clusterNameLabel: string; - configMapName: string; - dockerImage: string; - execution: string; - exId: string; - exName: string; - exUid: string; - jobId: string; - jobNameLabel: string; - name: string; - namespace: string; - nodeType: string; - replicas: number; - shutdownTimeout: number; -} +import { makeTemplate, setMaxOldSpaceViaEnv } from './utils.js'; +import { K8sConfig, NodeType } from './interfaces.js'; export class K8sResource { execution: ExecutionConfig; jobLabelPrefix: string; jobPropertyLabelPrefix: string; logger: Logger; - nodeType: string; + nodeType: NodeType; nameInfix: string; terasliceConfig: TerasliceConfig; - templateGenerator: (record: Record) => any; + templateGenerator: (config: K8sConfig) => k8s.V1Deployment | k8s.V1Job | k8s.V1Service; templateConfig: K8sConfig; - resource: any; + resource: k8s.V1Deployment | k8s.V1Job | k8s.V1Service; + exName?: string; + exUid?: string; /** * K8sResource allows the generation of k8s resources based on templates. * After creating the object, the k8s resource is accessible on the objects * .resource property. * - * @param {String} resourceType - jobs/services/deployments - * @param {String} resourceName - worker/execution_controller + * @param {'deployment' | 'job' | 'service'} resourceType - job/service/deployment + * @param {NodeType} resourceName - worker/execution_controller * @param {Object} terasliceConfig - teraslice cluster config from context * @param {Object} execution - teraslice execution + * @param {Logger} logger - teraslice logger + * @param {String} exName(optional) - name from execution resource (deployment and service only) + * @param {String} exUid(optional) - uid from execution resource (deployment and service only) */ constructor( - resourceType: string, - resourceName: string, + resourceType: 'deployment' | 'job' | 'service', + resourceName: NodeType, terasliceConfig: TerasliceConfig, execution: ExecutionConfig, - logger: Logger + logger: Logger, + exName?: string, + exUid?: string + ) { this.execution = execution; this.jobLabelPrefix = 'job.teraslice.terascope.io'; @@ -62,6 +48,8 @@ export class K8sResource { this.logger = logger; this.nodeType = resourceName; this.terasliceConfig = terasliceConfig; + this.exName = exName || undefined; + this.exUid = exUid || undefined; if (resourceName === 'worker') { this.nameInfix = 'wkr'; @@ -71,33 +59,33 @@ export class K8sResource { throw new Error(`Unsupported resourceName: ${resourceName}`); } - this.templateGenerator = this._makeTemplate(resourceType, resourceName); + this.templateGenerator = makeTemplate(resourceType, resourceName); this.templateConfig = this._makeConfig(); this.resource = this.templateGenerator(this.templateConfig); - if (resourceType !== 'services') { - this._setJobLabels(); + if (!(this.resource instanceof k8s.V1Service)) { + this._setJobLabels(this.resource); // Apply job `targets` setting as k8s nodeAffinity // We assume that multiple targets require both to match ... // NOTE: If you specify multiple `matchExpressions` associated with // `nodeSelectorTerms`, then the pod can be scheduled onto a node // only if *all* `matchExpressions` can be satisfied. - this._setTargets(); - this._setResources(); - this._setVolumes(); + this._setTargets(this.resource); + this._setResources(this.resource); + this._setVolumes(this.resource); if (process.env.MOUNT_LOCAL_TERASLICE !== undefined) { - this._mountLocalTeraslice(); + this._mountLocalTeraslice(this.resource); } this._setEnvVariables(); - this._setAssetsVolume(); - this._setImagePullSecret(); - this._setEphemeralStorage(); - this._setExternalPorts(); - this._setPriorityClassName(); + this._setAssetsVolume(this.resource); + this._setImagePullSecret(this.resource); + this._setEphemeralStorage(this.resource); + this._setExternalPorts(this.resource); + this._setPriorityClassName(this.resource); if (resourceName === 'worker') { - this._setWorkerAntiAffinity(); + this._setWorkerAntiAffinity(this.resource); } // Execution controller targets are required nodeAffinities, if @@ -105,11 +93,11 @@ export class K8sResource { // will have to be satisfied for the job to be scheduled. This also // adds tolerations for any specified targets if (resourceName === 'execution_controller') { - this._setExecutionControllerTargets(); + this._setExecutionControllerTargets(this.resource); } if (this.terasliceConfig.kubernetes_overrides_enabled) { - this._mergePodSpecOverlay(); + this._mergePodSpecOverlay(this.resource); } } } @@ -117,15 +105,17 @@ export class K8sResource { _setEnvVariables() { } - _mountLocalTeraslice(): void { + _mountLocalTeraslice(resource: k8s.V1Job | k8s.V1Deployment): void { const devMounts = JSON.parse(process.env.MOUNT_LOCAL_TERASLICE as string); - this.resource.spec.template.spec.containers[0].volumeMounts.push(...devMounts.volumeMounts); - this.resource.spec.template.spec.volumes.push(...devMounts.volumes); + resource.spec?.template?.spec?.containers[0]?.volumeMounts?.push(...devMounts.volumeMounts); + resource.spec?.template?.spec?.volumes?.push(...devMounts.volumes); - this.resource.spec.template.spec.containers[0].args = [ - 'node', - 'service.js' - ]; + if (resource.spec?.template?.spec?.containers[0]) { + resource.spec.template.spec.containers[0].args = [ + 'node', + 'service.js' + ]; + } } _makeConfig(): K8sConfig { @@ -160,10 +150,8 @@ export class K8sResource { dockerImage, execution: safeEncode(this.execution), exId: this.execution.ex_id, - // @ts-expect-error TODO: not sure where these come from - exName: this.execution.k8sName, - // @ts-expect-error TODO: not sure where these come from - exUid: this.execution.k8sUid, + exName: this.exName, + exUid: this.exUid, jobId: this.execution.job_id, jobNameLabel, name, @@ -176,26 +164,15 @@ export class K8sResource { return config; } - _makeTemplate(folder: string, fileName: string) { - const filePath = path.join(resourcePath, folder, `${fileName}.hbs`); - const templateData = fs.readFileSync(filePath, 'utf-8'); - const templateKeys = ['{{', '}}']; - - return (config: Record) => { - const templated = barbe(templateData, templateKeys, config); - return JSON.parse(templated); - }; - } - - _setWorkerAntiAffinity() { + _setWorkerAntiAffinity(resource: k8s.V1Job | k8s.V1Deployment) { if (this.terasliceConfig.kubernetes_worker_antiaffinity) { const targetKey = 'spec.template.spec.affinity.podAntiAffinity.preferredDuringSchedulingIgnoredDuringExecution'; if (!_.has(this.resource, targetKey)) { _.set(this.resource, targetKey, []); } - this.resource.spec.template.spec.affinity - .podAntiAffinity.preferredDuringSchedulingIgnoredDuringExecution.push( + resource?.spec?.template?.spec?.affinity?.podAntiAffinity + ?.preferredDuringSchedulingIgnoredDuringExecution?.push( { weight: 1, podAffinityTerm: { @@ -233,37 +210,37 @@ export class K8sResource { * test for an example. If the syntax for this were to change, we should * also consider changing `execution.targets`, which is a change on the job. */ - _setExecutionControllerTargets() { + _setExecutionControllerTargets(resource: k8s.V1Job | k8s.V1Deployment) { if (this.terasliceConfig.execution_controller_targets) { _.forEach(this.terasliceConfig.execution_controller_targets, (target) => { - this._setTargetRequired(target); - this._setTargetAccepted(target); + this._setTargetRequired(target, resource); + this._setTargetAccepted(target, resource); }); } } - _setEphemeralStorage() { + _setEphemeralStorage(resource: k8s.V1Job | k8s.V1Deployment) { if (this.execution.ephemeral_storage) { - this.resource.spec.template.spec.containers[0].volumeMounts.push({ + resource.spec?.template.spec?.containers[0]?.volumeMounts?.push({ name: 'ephemeral-volume', mountPath: '/ephemeral0' }); - this.resource.spec.template.spec.volumes.push({ + resource.spec?.template.spec?.volumes?.push({ name: 'ephemeral-volume', emptyDir: {} }); } } - _setExternalPorts() { + _setExternalPorts(resource: k8s.V1Job | k8s.V1Deployment) { if (this.execution.external_ports) { _.forEach(this.execution.external_ports, (portValue) => { if (isNumber(portValue)) { - this.resource.spec.template.spec.containers[0].ports - .push({ containerPort: portValue }); + resource.spec?.template.spec?.containers[0].ports + ?.push({ containerPort: portValue }); } else { - this.resource.spec.template.spec.containers[0].ports - .push( + resource.spec?.template.spec?.containers[0].ports + ?.push( { name: portValue.name, containerPort: portValue.port @@ -274,68 +251,81 @@ export class K8sResource { } } - _setImagePullSecret() { - if (this.terasliceConfig.kubernetes_image_pull_secret) { - this.resource.spec.template.spec.imagePullSecrets = [ - { name: this.terasliceConfig.kubernetes_image_pull_secret } - ]; + _setImagePullSecret(resource: k8s.V1Job | k8s.V1Deployment) { + if (this.terasliceConfig.kubernetes_image_pull_secret && resource.spec?.template.spec) { + if (resource.spec.template.spec.imagePullSecrets) { + resource.spec.template.spec.imagePullSecrets.push( + { name: this.terasliceConfig.kubernetes_image_pull_secret } + ); + } else { + resource.spec.template.spec.imagePullSecrets = [ + { name: this.terasliceConfig.kubernetes_image_pull_secret } + ]; + } } } - _setPriorityClassName() { + _setPriorityClassName(resource: k8s.V1Job | k8s.V1Deployment) { if (this.terasliceConfig.kubernetes_priority_class_name) { const className = this.terasliceConfig.kubernetes_priority_class_name; if (this.nodeType === 'execution_controller') { - this.resource.spec.template.spec.priorityClassName = className; - if (this.execution.stateful) { - this.resource.spec.template.metadata.labels[`${this.jobPropertyLabelPrefix}/stateful`] = 'true'; + if (resource.spec?.template.spec) { + resource.spec.template.spec.priorityClassName = className; + } + if (this.execution.stateful && resource.spec?.template.metadata?.labels) { + resource.spec.template.metadata.labels[`${this.jobPropertyLabelPrefix}/stateful`] = 'true'; } } if (this.nodeType === 'worker' && this.execution.stateful) { - this.resource.spec.template.spec.priorityClassName = className; - this.resource.spec.template.metadata.labels[`${this.jobPropertyLabelPrefix}/stateful`] = 'true'; + if (resource.spec?.template.spec) { + resource.spec.template.spec.priorityClassName = className; + } + if (resource.spec?.template.metadata?.labels) { + resource.spec.template.metadata.labels[`${this.jobPropertyLabelPrefix}/stateful`] = 'true'; + } } } } - _setAssetsVolume() { - if (this.terasliceConfig.assets_directory && this.terasliceConfig.assets_volume) { - this.resource.spec.template.spec.volumes.push({ + _setAssetsVolume(resource: k8s.V1Job | k8s.V1Deployment) { + if (this.terasliceConfig.assets_volume + && this.terasliceConfig.assets_directory + && typeof this.terasliceConfig.assets_directory === 'string' + ) { + resource.spec?.template.spec?.volumes?.push({ name: this.terasliceConfig.assets_volume, persistentVolumeClaim: { claimName: this.terasliceConfig.assets_volume } }); - this.resource.spec.template.spec.containers[0].volumeMounts.push({ + resource.spec?.template.spec?.containers[0].volumeMounts?.push({ name: this.terasliceConfig.assets_volume, mountPath: this.terasliceConfig.assets_directory }); } } - _setJobLabels() { + _setJobLabels(resource: k8s.V1Job | k8s.V1Deployment) { if (this.execution.labels != null) { Object.entries(this.execution.labels).forEach(([k, v]) => { const key = `${this.jobLabelPrefix}/${_.replace(k, /[^a-zA-Z0-9\-._]/g, '-').substring(0, 63)}`; const value = _.replace(v, /[^a-zA-Z0-9\-._]/g, '-').substring(0, 63); - this.resource.metadata.labels[key] = value; - if (this.resource.kind !== 'Service') { - // Services don't have templates, so if it's a service, - // don't add this - this.resource.spec.template.metadata.labels[key] = value; + if (resource.metadata?.labels && resource.spec?.template.metadata?.labels) { + resource.metadata.labels[key] = value; + resource.spec.template.metadata.labels[key] = value; } }); } } - _setVolumes() { + _setVolumes(resource: k8s.V1Job | k8s.V1Deployment) { if (this.execution.volumes != null) { _.forEach(this.execution.volumes, (volume) => { - this.resource.spec.template.spec.volumes.push({ + resource.spec?.template.spec?.volumes?.push({ name: volume.name, persistentVolumeClaim: { claimName: volume.name } }); - this.resource.spec.template.spec.containers[0].volumeMounts.push({ + resource.spec?.template.spec?.containers[0].volumeMounts?.push({ name: volume.name, mountPath: volume.path }); @@ -343,12 +333,15 @@ export class K8sResource { } } - _setResources() { + _setResources(resource: k8s.V1Job | k8s.V1Deployment) { let cpu; let memory; let maxMemory; - const container = this.resource.spec.template.spec.containers[0]; + const container = resource.spec?.template.spec?.containers[0]; + if (container === undefined) { + throw new Error('Resource container undefined while setting resources.'); + } // use teraslice config as defaults and execution config will override it const envVars = Object.assign({}, this.terasliceConfig.env_vars, this.execution.env_vars); @@ -398,30 +391,33 @@ export class K8sResource { // NOTE: This sucks, this manages the memory env var but it ALSO is // responsible for doing the config and execution env var merge, which // should NOT be in this function + if (container.env === undefined) { + throw new Error('Resource container undefined while setting resources.'); + } setMaxOldSpaceViaEnv(container.env, envVars, maxMemory as number); } - _setTargets() { + _setTargets(resource: k8s.V1Job | k8s.V1Deployment) { if (_.has(this.execution, 'targets') && (!_.isEmpty(this.execution.targets))) { _.forEach(this.execution.targets, (target: any) => { // `required` is the default if no `constraint` is provided for // backwards compatibility and as the most likely case if (target.constraint === 'required' || !_.has(target, 'constraint')) { - this._setTargetRequired(target); + this._setTargetRequired(target, resource); } if (target.constraint === 'preferred') { - this._setTargetPreferred(target); + this._setTargetPreferred(target, resource); } if (target.constraint === 'accepted') { - this._setTargetAccepted(target); + this._setTargetAccepted(target, resource); } }); } } - _setTargetRequired(target: any) { + _setTargetRequired(target: any, resource: k8s.V1Job | k8s.V1Deployment) { const targetKey = 'spec.template.spec.affinity.nodeAffinity.requiredDuringSchedulingIgnoredDuringExecution'; if (!_.has(this.resource, targetKey)) { const nodeSelectorObj = { @@ -430,23 +426,23 @@ export class K8sResource { _.set(this.resource, targetKey, nodeSelectorObj); } - this.resource.spec.template.spec.affinity.nodeAffinity - .requiredDuringSchedulingIgnoredDuringExecution - .nodeSelectorTerms[0].matchExpressions.push({ + resource.spec?.template.spec?.affinity?.nodeAffinity + ?.requiredDuringSchedulingIgnoredDuringExecution + ?.nodeSelectorTerms[0].matchExpressions?.push({ key: target.key, operator: 'In', values: [target.value] }); } - _setTargetPreferred(target: any) { + _setTargetPreferred(target: any, resource: k8s.V1Job | k8s.V1Deployment) { const targetKey = 'spec.template.spec.affinity.nodeAffinity.preferredDuringSchedulingIgnoredDuringExecution'; if (!_.has(this.resource, targetKey)) { _.set(this.resource, targetKey, []); } - this.resource.spec.template.spec.affinity.nodeAffinity - .preferredDuringSchedulingIgnoredDuringExecution.push({ + resource.spec?.template.spec?.affinity?.nodeAffinity + ?.preferredDuringSchedulingIgnoredDuringExecution?.push({ weight: 1, preference: { matchExpressions: [{ @@ -458,13 +454,13 @@ export class K8sResource { }); } - _setTargetAccepted(target: any) { + _setTargetAccepted(target: any, resource: k8s.V1Job | k8s.V1Deployment) { const targetKey = 'spec.template.spec.tolerations'; if (!_.has(this.resource, targetKey)) { _.set(this.resource, targetKey, []); } - this.resource.spec.template.spec.tolerations.push({ + resource.spec?.template.spec?.tolerations?.push({ key: target.key, operator: 'Equal', value: target.value, @@ -486,10 +482,12 @@ export class K8sResource { * * Job setting: `pod_spec_override` */ - _mergePodSpecOverlay() { - this.resource.spec.template.spec = _.merge( - this.resource.spec.template.spec, - this.execution.pod_spec_override - ); + _mergePodSpecOverlay(resource: k8s.V1Job | k8s.V1Deployment) { + if (resource.spec?.template.spec) { + resource.spec.template.spec = _.merge( + resource.spec?.template.spec, + this.execution.pod_spec_override + ); + } } } diff --git a/packages/teraslice/src/lib/cluster/services/cluster/backends/kubernetesV2/k8sState.ts b/packages/teraslice/src/lib/cluster/services/cluster/backends/kubernetesV2/k8sState.ts index e64e90e5ecb..37cfbf3f052 100644 --- a/packages/teraslice/src/lib/cluster/services/cluster/backends/kubernetesV2/k8sState.ts +++ b/packages/teraslice/src/lib/cluster/services/cluster/backends/kubernetesV2/k8sState.ts @@ -1,4 +1,7 @@ import _ from 'lodash'; +import * as k8s from '@kubernetes/client-node'; +import { Logger } from '@terascope/types'; +import { logError } from '@terascope/utils'; /** * Given the k8s Pods API output generates the appropriate Teraslice cluster @@ -7,8 +10,9 @@ import _ from 'lodash'; * @param {Object} k8sPods k8s pods API object (k8s v1.10+) * @param {Object} clusterState Teraslice Cluster State * @param {String} clusterNameLabel k8s label containing clusterName + * @param {Logger} logger Teraslice logger */ -export function gen(k8sPods: any, clusterState: Record) { +export function gen(k8sPods: k8s.V1PodList, clusterState: Record, logger: Logger) { // Make sure we clean up the old const hostIPs = _.uniq(_.map(k8sPods.items, 'status.hostIP')); const oldHostIps = _.difference(_.keys(clusterState), hostIPs); @@ -24,41 +28,51 @@ export function gen(k8sPods: any, clusterState: Record) { }); // add a worker for each pod - k8sPods.items.forEach((pod: any) => { - if (!_.has(clusterState, pod.status.hostIP)) { - // If the node isn't in clusterState, add it - clusterState[pod.status.hostIP] = { - node_id: pod.status.hostIP, - hostname: pod.status.hostIP, - pid: 'N/A', - node_version: 'N/A', - teraslice_version: 'N/A', - total: 'N/A', - state: 'connected', - available: 'N/A', - active: [] - }; - } + k8sPods.items.forEach((pod) => { + if (pod.status?.hostIP) { + if (!_.has(clusterState, pod.status.hostIP)) { + // If the node isn't in clusterState, add it + clusterState[pod.status.hostIP] = { + node_id: pod.status.hostIP, + hostname: pod.status.hostIP, + pid: 'N/A', + node_version: 'N/A', + teraslice_version: 'N/A', + total: 'N/A', + state: 'connected', + available: 'N/A', + active: [] + }; + } - const worker = { - assets: [], - assignment: pod.metadata.labels['app.kubernetes.io/component'], - ex_id: pod.metadata.labels['teraslice.terascope.io/exId'], - // WARNING: This makes the assumption that the first container - // in the pod is the teraslice container. Currently it is the - // only container, so this assumption is safe for now. - image: pod.spec.containers[0].image, - job_id: pod.metadata.labels['teraslice.terascope.io/jobId'], - pod_name: pod.metadata.name, - pod_ip: pod.status.podIP, - worker_id: pod.metadata.name, - }; + if (pod.metadata?.labels && pod.spec) { + const worker = { + assets: [], + assignment: pod.metadata.labels['app.kubernetes.io/component'], + ex_id: pod.metadata.labels['teraslice.terascope.io/exId'], + // WARNING: This makes the assumption that the first container + // in the pod is the teraslice container. Currently it is the + // only container, so this assumption is safe for now. + image: pod.spec.containers[0].image, + job_id: pod.metadata.labels['teraslice.terascope.io/jobId'], + pod_name: pod.metadata.name, + pod_ip: pod.status.podIP, + worker_id: pod.metadata.name, + }; - // k8s pods can have status.phase = `Pending`, `Running`, `Succeeded`, - // `Failed`, `Unknown`. We will only add `Running` pods to the - // Teraslice cluster state. - if (pod.status.phase === 'Running') { - clusterState[pod.status.hostIP].active.push(worker); + // k8s pods can have status.phase = `Pending`, `Running`, `Succeeded`, + // `Failed`, `Unknown`. We will only add `Running` pods to the + // Teraslice cluster state. + if (pod.status.phase === 'Running') { + clusterState[pod.status.hostIP].active.push(worker); + } + } else { + // TODO: We might need to do more here. I think it's OK to just + // log though. This only gets used to show slicer info through + // the API. We wouldn't want to disrupt the cluster master + // for rare failures to reach the k8s API. + logError(logger, 'K8s pod missing a required field necessary to update cluster state'); + } } }); } diff --git a/packages/teraslice/src/lib/cluster/services/cluster/backends/kubernetesV2/utils.ts b/packages/teraslice/src/lib/cluster/services/cluster/backends/kubernetesV2/utils.ts index 21a6bce99cd..c49b8279142 100644 --- a/packages/teraslice/src/lib/cluster/services/cluster/backends/kubernetesV2/utils.ts +++ b/packages/teraslice/src/lib/cluster/services/cluster/backends/kubernetesV2/utils.ts @@ -3,17 +3,42 @@ import path from 'node:path'; // @ts-expect-error import barbe from 'barbe'; import { isTest } from '@terascope/utils'; +import * as k8s from '@kubernetes/client-node'; +import { K8sConfig, NodeType, Resource } from './interfaces.js'; const MAX_RETRIES = isTest ? 2 : 3; const RETRY_DELAY = isTest ? 50 : 1000; // time in ms const resourcePath = path.join(process.cwd(), './packages/teraslice/src/lib/cluster/services/cluster/backends/kubernetesV2/'); -export function makeTemplate(folder: string, fileName: string) { +export function makeTemplate( + folder: 'deployment', + fileName: NodeType +): (config: K8sConfig) => k8s.V1Deployment; +export function makeTemplate( + folder: 'job', + fileName: NodeType +): (config: K8sConfig) => k8s.V1Job; +export function makeTemplate( + folder: 'service', + fileName: NodeType +): (config: K8sConfig) => k8s.V1Service; +export function makeTemplate( + folder: 'deployment' | 'job' | 'service', + fileName: NodeType +): (config: K8sConfig) => k8s.V1Deployment | k8s.V1Job | k8s.V1Service; +export function makeTemplate( + folder: 'deployment' | 'job' | 'service', + fileName: NodeType +): (config: K8sConfig) => k8s.V1Deployment | k8s.V1Job | k8s.V1Service { + const availableTemplates = ['deployment', 'job', 'service']; + if (!availableTemplates.includes(folder)) { + throw new Error(`Unsupported template folder: ${folder}. Available template folders are: deployment, job, service.`); + } const filePath = path.join(resourcePath, folder, `${fileName}.hbs`); const templateData = fs.readFileSync(filePath, 'utf-8'); const templateKeys = ['{{', '}}']; - return (config: any) => { + return (config: K8sConfig) => { const templated = barbe(templateData, templateKeys, config); return JSON.parse(templated); }; @@ -24,13 +49,8 @@ export function getMaxOldSpace(memory: number) { return Math.round(0.9 * (memory / 1024 / 1024)); } -interface EnvObject { - name: string; - value: any; -} - export function setMaxOldSpaceViaEnv( - envArr: EnvObject[], + envArr: k8s.V1EnvVar[], jobEnv: Record, memory: number ) { @@ -57,3 +77,22 @@ export function getRetryConfig() { delay: RETRY_DELAY }; } + +export function isDeployment(manifest: Resource): manifest is k8s.V1Deployment { + return manifest.kind === 'Deployment'; +} + +export function isJob(manifest: Resource): manifest is k8s.V1Job { + return manifest.kind === 'Job'; +} + +export function isPod(manifest: Resource): manifest is k8s.V1Pod { + return manifest.kind === 'Pod'; +} +export function isReplicaSet(manifest: Resource): manifest is k8s.V1ReplicaSet { + return manifest.kind === 'ReplicaSet'; +} + +export function isService(manifest: Resource): manifest is k8s.V1Service { + return manifest.kind === 'Service'; +} From af9d6380fcc22e0ce7ef389f77b53eb4e9f5a787 Mon Sep 17 00:00:00 2001 From: busma13 Date: Wed, 30 Oct 2024 13:59:47 -0700 Subject: [PATCH 03/30] add types to tests for k8s v2 --- .../backends/kubernetes/v2/k8s-v2-spec.ts | 100 ++-- .../kubernetes/v2/k8sResource-v2-spec.ts | 485 ++++++++++++------ .../v2/k8sState-multicluster-v2-spec.ts | 14 +- .../kubernetes/v2/k8sState-v2-spec.ts | 18 +- .../backends/kubernetes/v2/utils-v2-spec.ts | 44 +- 5 files changed, 436 insertions(+), 225 deletions(-) diff --git a/packages/teraslice/test/lib/cluster/services/cluster/backends/kubernetes/v2/k8s-v2-spec.ts b/packages/teraslice/test/lib/cluster/services/cluster/backends/kubernetes/v2/k8s-v2-spec.ts index 966ad875354..74061cd6150 100644 --- a/packages/teraslice/test/lib/cluster/services/cluster/backends/kubernetes/v2/k8s-v2-spec.ts +++ b/packages/teraslice/test/lib/cluster/services/cluster/backends/kubernetes/v2/k8s-v2-spec.ts @@ -5,6 +5,7 @@ import nock from 'nock'; import { debugLogger } from '@terascope/job-components'; import { K8s } from '../../../../../../../../src/lib/cluster/services/cluster/backends/kubernetesV2/k8s.js'; +import { V1Service } from '@kubernetes/client-node'; const logger = debugLogger('k8s-v2-spec'); @@ -70,7 +71,7 @@ describe('k8s', () => { .query({ labelSelector: 'app=teraslice' }) .reply(200, { kind: 'PodList' }); - const pods = await k8s.list('app=teraslice', 'pods'); + const pods = await k8s.list('app=teraslice', 'pod'); expect(pods.kind).toEqual('PodList'); }); @@ -80,7 +81,7 @@ describe('k8s', () => { .query({ labelSelector: 'app=teraslice' }) .reply(200, { kind: 'ServiceList' }); - const pods = await k8s.list('app=teraslice', 'services'); + const pods = await k8s.list('app=teraslice', 'service'); expect(pods.kind).toEqual('ServiceList'); }); @@ -90,7 +91,7 @@ describe('k8s', () => { .query({ labelSelector: 'app=teraslice' }) .reply(200, { kind: 'DeploymentList' }); - const deployments = await k8s.list('app=teraslice', 'deployments'); + const deployments = await k8s.list('app=teraslice', 'deployment'); expect(deployments.kind).toEqual('DeploymentList'); }); @@ -100,7 +101,7 @@ describe('k8s', () => { .query({ labelSelector: 'app=teraslice' }) .reply(200, { kind: 'JobList' }); - const jobs = await k8s.list('app=teraslice', 'jobs'); + const jobs = await k8s.list('app=teraslice', 'job'); expect(jobs.kind).toEqual('JobList'); }); @@ -110,7 +111,7 @@ describe('k8s', () => { .query({ labelSelector: 'app=teraslice' }) .reply(200, { kind: 'ReplicaSetList' }); - const jobs = await k8s.list('app=teraslice', 'replicasets'); + const jobs = await k8s.list('app=teraslice', 'replicaset'); expect(jobs.kind).toEqual('ReplicaSetList'); }); }); @@ -127,7 +128,7 @@ describe('k8s', () => { }] }); - const jobs = await k8s.nonEmptyList('app=teraslice', 'jobs'); + const jobs = await k8s.nonEmptyJobList('app=teraslice'); expect(jobs.items[0]).toEqual({ apiVersion: '1.0.0', kind: 'job', @@ -143,21 +144,12 @@ describe('k8s', () => { .query({ labelSelector: 'app=teraslice' }) .reply(200, { items: [] }); - await expect(k8s.nonEmptyList('app=teraslice', 'jobs')) - .rejects.toThrow('Teraslice jobs matching the following selector was not found: app=teraslice (retriable)'); + await expect(k8s.nonEmptyJobList('app=teraslice')) + .rejects.toThrow('Teraslice job matching the following selector was not found: app=teraslice (retriable)'); }); }); describe('->post', () => { - it('can post a service', async () => { - nock(_url, { encodedQueryParams: true }) - .post('/api/v1/namespaces/default/services') - .reply(201, { kind: 'Service' }); - - const response = await k8s.post({ kind: 'Service' }, 'service'); - expect(response.kind).toEqual('Service'); - }); - it('can post a deployment', async () => { nock(_url, { encodedQueryParams: true }) .post('/apis/apps/v1/namespaces/default/deployments') @@ -175,29 +167,69 @@ describe('k8s', () => { const response = await k8s.post({ kind: 'Job' }, 'job'); expect(response.kind).toEqual('Job'); }); + + it('can post a pod', async () => { + nock(_url, { encodedQueryParams: true }) + .post('/api/v1/namespaces/default/pods') + .reply(201, { kind: 'Pod' }); + + const response = await k8s.post({ kind: 'Pod' }, 'pod'); + expect(response.kind).toEqual('Pod'); + }); + + it('can post a replicaSet', async () => { + nock(_url, { encodedQueryParams: true }) + .post('/apis/apps/v1/namespaces/default/replicasets') + .reply(201, { kind: 'ReplicaSet' }); + + const response = await k8s.post({ kind: 'ReplicaSet' }, 'replicaset'); + expect(response.kind).toEqual('ReplicaSet'); + }); + + it('can post a service', async () => { + nock(_url, { encodedQueryParams: true }) + .post('/api/v1/namespaces/default/services') + .reply(201, { kind: 'Service' }); + + const manifest: V1Service = { kind: 'Service' }; + + const response = await k8s.post(manifest, 'service'); + expect(response.kind).toEqual('Service'); + }); }); describe('->patch', () => { - beforeEach(() => { + it('can patch a deployment by name', async () => { nock(_url, { encodedQueryParams: true }) .patch('/apis/apps/v1/namespaces/default/deployments/test1') .reply(204, { }); - }); - it('can patch a deployment by name', async () => { const response = await k8s.patch({ name: 'testName' }, 'test1'); expect(response).toEqual({}); }); + + it('will throw on a reponse code >= 400', async () => { + nock(_url) + .patch('/apis/apps/v1/namespaces/default/deployments/bad-response') + .replyWithError({ statusCode: 400 }) + .patch('/apis/apps/v1/namespaces/default/deployments/bad-response') + .replyWithError({ statusCode: 400 }) + .patch('/apis/apps/v1/namespaces/default/deployments/bad-response') + .replyWithError({ statusCode: 400 }); + + await expect(k8s.patch({ name: 'bad-response' }, 'bad-response')) + .rejects.toThrow('Request k8s.patch with name: bad-response failed with: TSError: {"statusCode":400}'); + }); }); describe('->delete', () => { it('will throw if name is undefined', async () => { - await expect(k8s.delete(undefined as unknown as string, 'deployments')) + await expect(k8s.delete(undefined as unknown as string, 'deployment')) .rejects.toThrow('Name of resource to delete must be specified. Received: "undefined".'); }); it('will throw if name is an empty string', async () => { - await expect(k8s.delete('', 'deployments')) + await expect(k8s.delete('', 'deployment')) .rejects.toThrow('Name of resource to delete must be specified. Received: "".'); }); @@ -206,7 +238,7 @@ describe('k8s', () => { .delete('/apis/apps/v1/namespaces/default/deployments/test1') .reply(200, {}); - const response = await k8s.delete('test1', 'deployments'); + const response = await k8s.delete('test1', 'deployment'); expect(response).toEqual({}); }); @@ -215,7 +247,7 @@ describe('k8s', () => { .delete('/api/v1/namespaces/default/services/test1') .reply(200, {}); - const response = await k8s.delete('test1', 'services'); + const response = await k8s.delete('test1', 'service'); expect(response).toEqual({}); }); @@ -224,7 +256,7 @@ describe('k8s', () => { .delete('/apis/batch/v1/namespaces/default/jobs/test1') .reply(200, {}); - const response = await k8s.delete('test1', 'jobs'); + const response = await k8s.delete('test1', 'job'); expect(response).toEqual({}); }); @@ -233,7 +265,7 @@ describe('k8s', () => { .delete('/api/v1/namespaces/default/pods/test1') .reply(200, {}); - const response = await k8s.delete('test1', 'pods'); + const response = await k8s.delete('test1', 'pod'); expect(response).toEqual({}); }); @@ -242,7 +274,7 @@ describe('k8s', () => { .delete('/apis/apps/v1/namespaces/default/replicasets/test1') .reply(200, {}); - const response = await k8s.delete('test1', 'replicasets'); + const response = await k8s.delete('test1', 'replicaset'); expect(response).toEqual({}); }); @@ -255,8 +287,8 @@ describe('k8s', () => { .delete('/api/v1/namespaces/default/pods/bad-response') .replyWithError({ statusCode: 400 }); - await expect(k8s.delete('bad-response', 'pods')) - .rejects.toThrow('Request k8s.delete with name: bad-response failed with: TSError: Unexpected response code (400), when deleting name: bad-response'); + await expect(k8s.delete('bad-response', 'pod')) + .rejects.toThrow('Request k8s.delete with name: bad-response failed with: TSError: {"statusCode":400}'); }); it('will succeed on a 404 response code', async () => { @@ -277,7 +309,7 @@ describe('k8s', () => { .delete('/api/v1/namespaces/default/pods/non-existent') .replyWithError(notFoundResponse); - const response = await k8s.delete('non-existent', 'pods'); + const response = await k8s.delete('non-existent', 'pod'); expect(response).toEqual(notFoundResponse.body); }); }); @@ -328,8 +360,8 @@ describe('k8s', () => { items: [jobNoName] }); - await expect(k8s._deleteObjByExId('no-name', 'execution_controller', 'jobs')) - .rejects.toThrow('Cannot delete jobs for ExId: no-name by name because it has no name'); + await expect(k8s._deleteObjByExId('no-name', 'execution_controller', 'job')) + .rejects.toThrow('Cannot delete job for ExId: no-name by name because it has no name'); }); it('can delete a single object', async () => { @@ -345,7 +377,7 @@ describe('k8s', () => { .delete('/apis/batch/v1/namespaces/default/jobs/testJob1') .reply(200, status); - const response = await k8s._deleteObjByExId('testJob1', 'execution_controller', 'jobs'); + const response = await k8s._deleteObjByExId('testJob1', 'execution_controller', 'job'); expect(response).toEqual([expect.objectContaining(status)]); }); @@ -365,7 +397,7 @@ describe('k8s', () => { .delete('/api/v1/namespaces/default/pods/testPod2') .reply(200, testPod2); - const response = await k8s._deleteObjByExId('testPods', 'worker', 'pods'); + const response = await k8s._deleteObjByExId('testPods', 'worker', 'pod'); expect(response).toEqual([ expect.objectContaining(testPod1), expect.objectContaining(testPod2) diff --git a/packages/teraslice/test/lib/cluster/services/cluster/backends/kubernetes/v2/k8sResource-v2-spec.ts b/packages/teraslice/test/lib/cluster/services/cluster/backends/kubernetes/v2/k8sResource-v2-spec.ts index e235e182b99..b4bf015f26d 100644 --- a/packages/teraslice/test/lib/cluster/services/cluster/backends/kubernetes/v2/k8sResource-v2-spec.ts +++ b/packages/teraslice/test/lib/cluster/services/cluster/backends/kubernetes/v2/k8sResource-v2-spec.ts @@ -1,5 +1,6 @@ import _ from 'lodash'; import yaml from 'js-yaml'; +import * as k8s from '@kubernetes/client-node'; import { debugLogger } from '@terascope/utils'; import { K8sResource } from '../../../../../../../../src/lib/cluster/services/cluster/backends/kubernetesV2/k8sResource.js'; @@ -37,27 +38,37 @@ describe('k8sResource', () => { describe('worker deployment', () => { it('has valid resource object.', () => { - const kr = new K8sResource('deployments', 'worker', terasliceConfig, execution, logger); + const kr = new K8sResource('deployment', 'worker', terasliceConfig, execution, logger); + const resource = kr.resource as k8s.V1Deployment; - expect(kr.resource.kind).toBe('Deployment'); - expect(kr.resource.spec.replicas).toBe(2); - expect(kr.resource.metadata.name).toBe('ts-wkr-example-data-generator-job-7ba9afb0-417a'); + expect(resource.kind).toBe('Deployment'); + expect(resource.spec?.replicas).toBe(2); + expect(resource.metadata?.name).toBe('ts-wkr-example-data-generator-job-7ba9afb0-417a'); // The following properties should be absent in the default case // Note: This tests that both affinity and podAntiAffinity are absent - expect(kr.resource.spec.template.spec).not.toHaveProperty('affinity'); - expect(kr.resource.spec.template.spec).not.toHaveProperty('imagePullSecrets'); - expect(kr.resource.spec.template.spec).not.toHaveProperty('priorityClassName'); + expect(resource.spec?.template.spec).not.toHaveProperty('affinity'); + expect(resource.spec?.template.spec).not.toHaveProperty('imagePullSecrets'); + expect(resource.spec?.template.spec).not.toHaveProperty('priorityClassName'); + let configVolume: k8s.V1Volume | undefined = undefined; + if (resource.spec?.template.spec?.volumes) { + configVolume = resource.spec.template.spec.volumes[0]; + } + + let configVolumeMount: k8s.V1VolumeMount | undefined = undefined; + if (resource.spec?.template.spec?.containers[0].volumeMounts) { + configVolumeMount = resource.spec.template.spec.containers[0].volumeMounts[0]; + } // Configmaps should be mounted on all workers - expect(kr.resource.spec.template.spec.volumes[0]).toEqual(yaml.load(` + expect(configVolume).toEqual(yaml.load(` name: config configMap: name: ts-dev1-worker items: - key: teraslice.yaml path: teraslice.yaml`)); - expect(kr.resource.spec.template.spec.containers[0].volumeMounts[0]) + expect(configVolumeMount) .toEqual(yaml.load(` mountPath: /app/config name: config`)); @@ -65,9 +76,15 @@ describe('k8sResource', () => { it('has valid resource object when terasliceConfig has kubernetes_image_pull_secret.', () => { terasliceConfig.kubernetes_image_pull_secret = 'teraslice-image-pull-secret'; - const kr = new K8sResource('deployments', 'worker', terasliceConfig, execution, logger); + const kr = new K8sResource('deployment', 'worker', terasliceConfig, execution, logger); + const resource = kr.resource as k8s.V1Deployment; + + let firstSecret: k8s.V1LocalObjectReference | undefined = undefined; + if (resource.spec?.template.spec?.imagePullSecrets) { + firstSecret = resource.spec.template.spec.imagePullSecrets[0]; + } - expect(kr.resource.spec.template.spec.imagePullSecrets[0]).toEqual( + expect(firstSecret).toEqual( yaml.load(` name: teraslice-image-pull-secret`) ); @@ -75,10 +92,11 @@ describe('k8sResource', () => { it('has podAntiAffinity when terasliceConfig has kubernetes_worker_antiaffinity true.', () => { terasliceConfig.kubernetes_worker_antiaffinity = true; - const kr = new K8sResource('deployments', 'worker', terasliceConfig, execution, logger); + const kr = new K8sResource('deployment', 'worker', terasliceConfig, execution, logger); + const resource = kr.resource as k8s.V1Deployment; // console.log(yaml.dump(kr.resource.spec.template.spec.affinity)); - expect(kr.resource.spec.template.spec.affinity).toEqual( + expect(resource.spec?.template.spec?.affinity).toEqual( yaml.load(` podAntiAffinity: preferredDuringSchedulingIgnoredDuringExecution: @@ -102,32 +120,53 @@ describe('k8sResource', () => { terasliceConfig.assets_directory = '/assets'; terasliceConfig.assets_volume = 'asset-volume'; - const kr = new K8sResource('deployments', 'worker', terasliceConfig, execution, logger); + const kr = new K8sResource('deployment', 'worker', terasliceConfig, execution, logger); + const resource = kr.resource as k8s.V1Deployment; + + let configVolume: k8s.V1Volume | undefined = undefined; + if (resource.spec?.template.spec?.volumes) { + configVolume = resource.spec.template.spec.volumes[0]; + } + + let assetVolume: k8s.V1Volume | undefined = undefined; + if (resource.spec?.template.spec?.volumes) { + assetVolume = resource.spec.template.spec.volumes[1]; + } + + let configVolumeMount: k8s.V1VolumeMount | undefined = undefined; + if (resource.spec?.template.spec?.containers[0].volumeMounts) { + configVolumeMount = resource.spec.template.spec.containers[0].volumeMounts[0]; + } + + let assetVolumeMount: k8s.V1VolumeMount | undefined = undefined; + if (resource.spec?.template.spec?.containers[0].volumeMounts) { + assetVolumeMount = resource.spec.template.spec.containers[0].volumeMounts[1]; + } - expect(kr.resource.spec.replicas).toBe(2); - expect(kr.resource.metadata.name).toBe('ts-wkr-example-data-generator-job-7ba9afb0-417a'); + expect(resource.spec?.replicas).toBe(2); + expect(resource.metadata?.name).toBe('ts-wkr-example-data-generator-job-7ba9afb0-417a'); // The following properties should be absent in the default case - expect(kr.resource.spec.template.spec).not.toHaveProperty('affinity'); - expect(kr.resource.spec.template.spec).not.toHaveProperty('imagePullSecrets'); - expect(kr.resource.spec.template.spec.volumes[0]).toEqual(yaml.load(` + expect(resource.spec?.template.spec).not.toHaveProperty('affinity'); + expect(resource.spec?.template.spec).not.toHaveProperty('imagePullSecrets'); + expect(configVolume).toEqual(yaml.load(` name: config configMap: name: ts-dev1-worker items: - key: teraslice.yaml path: teraslice.yaml`)); - expect(kr.resource.spec.template.spec.containers[0].volumeMounts[0]) + expect(configVolumeMount) .toEqual(yaml.load(` mountPath: /app/config name: config`)); // Now check for the assets volume - expect(kr.resource.spec.template.spec.volumes[1]).toEqual(yaml.load(` + expect(assetVolume).toEqual(yaml.load(` name: asset-volume persistentVolumeClaim: claimName: asset-volume`)); - expect(kr.resource.spec.template.spec.containers[0].volumeMounts[1]) + expect(assetVolumeMount) .toEqual(yaml.load(` name: asset-volume mountPath: /assets`)); @@ -138,28 +177,45 @@ describe('k8sResource', () => { { name: 'teraslice-data1', path: '/data' } ]; - const kr = new K8sResource('deployments', 'worker', terasliceConfig, execution, logger); + const kr = new K8sResource('deployment', 'worker', terasliceConfig, execution, logger); + const resource = kr.resource as k8s.V1Deployment; + + const configVolume1 = resource.spec?.template.spec?.volumes + ? resource.spec.template.spec.volumes[0] + : undefined; + + const configVolume2 = resource.spec?.template.spec?.volumes + ? resource.spec.template.spec.volumes[1] + : undefined; + + const configVolumeMount1 = resource.spec?.template.spec?.containers[0].volumeMounts + ? resource.spec.template.spec.containers[0].volumeMounts[0] + : undefined; + + const configVolumeMount2 = resource.spec?.template.spec?.containers[0].volumeMounts + ? resource.spec.template.spec.containers[0].volumeMounts[1] + : undefined; // First check the configMap volumes, which should be present on all // deployments - expect(kr.resource.spec.template.spec.volumes[0]).toEqual(yaml.load(` + expect(configVolume1).toEqual(yaml.load(` name: config configMap: name: ts-dev1-worker items: - key: teraslice.yaml path: teraslice.yaml`)); - expect(kr.resource.spec.template.spec.containers[0].volumeMounts[0]) + expect(configVolumeMount1) .toEqual(yaml.load(` mountPath: /app/config name: config`)); // Now check for the volume added via config - expect(kr.resource.spec.template.spec.volumes[1]).toEqual(yaml.load(` + expect(configVolume2).toEqual(yaml.load(` name: teraslice-data1 persistentVolumeClaim: claimName: teraslice-data1`)); - expect(kr.resource.spec.template.spec.containers[0].volumeMounts[1]) + expect(configVolumeMount2) .toEqual(yaml.load(` name: teraslice-data1 mountPath: /data`)); @@ -171,36 +227,57 @@ describe('k8sResource', () => { { name: 'tmp', path: '/tmp' } ]; - const kr = new K8sResource('deployments', 'worker', terasliceConfig, execution, logger); + const kr = new K8sResource('deployment', 'worker', terasliceConfig, execution, logger); + const resource = kr.resource as k8s.V1Deployment; + + const configVolume2 = resource.spec?.template.spec?.volumes + ? resource.spec.template.spec.volumes[1] + : undefined; + + const configVolume3 = resource.spec?.template.spec?.volumes + ? resource.spec.template.spec.volumes[2] + : undefined; + + const configVolumeMount2 = resource.spec?.template.spec?.containers[0].volumeMounts + ? resource.spec.template.spec.containers[0].volumeMounts[1] + : undefined; + + const configVolumeMount3 = resource.spec?.template.spec?.containers[0].volumeMounts + ? resource.spec.template.spec.containers[0].volumeMounts[2] + : undefined; // Now check for the volumes added via job - expect(kr.resource.spec.template.spec.volumes[1]).toEqual(yaml.load(` + expect(configVolume2).toEqual(yaml.load(` name: teraslice-data1 persistentVolumeClaim: claimName: teraslice-data1`)); - expect(kr.resource.spec.template.spec.containers[0].volumeMounts[1]) + expect(configVolumeMount2) .toEqual(yaml.load(` name: teraslice-data1 mountPath: /data`)); - expect(kr.resource.spec.template.spec.volumes[2]).toEqual(yaml.load(` + expect(configVolume3).toEqual(yaml.load(` name: tmp persistentVolumeClaim: claimName: tmp`)); - expect(kr.resource.spec.template.spec.containers[0].volumeMounts[2]) + expect(configVolumeMount3) .toEqual(yaml.load(` name: tmp mountPath: /tmp`)); }); it('does not have memory/cpu limits/requests when not set in config or execution', () => { - const kr = new K8sResource('deployments', 'worker', terasliceConfig, execution, logger); + const kr = new K8sResource('deployment', 'worker', terasliceConfig, execution, logger); + const resource = kr.resource as k8s.V1Deployment; - expect(kr.resource.metadata.labels['teraslice.terascope.io/exId']) + const exId = resource.metadata?.labels ? resource.metadata.labels['teraslice.terascope.io/exId'] : undefined; + expect(exId) .toEqual('e76a0278-d9bc-4d78-bf14-431bcd97528c'); - expect(kr.resource.spec.template.spec.containers[0].resources).not.toBeDefined(); - const envArray = kr.resource.spec.template.spec.containers[0].env; + const resources = resource.spec?.template.spec?.containers[0].resources; + expect(resources).not.toBeDefined(); + + const envArray = resource.spec?.template.spec?.containers[0].env; expect(envArray).not.toContain('NODE_OPTIONS'); }); @@ -208,11 +285,15 @@ describe('k8sResource', () => { terasliceConfig.cpu = 1; terasliceConfig.memory = 2147483648; - const kr = new K8sResource('deployments', 'worker', terasliceConfig, execution, logger); + const kr = new K8sResource('deployment', 'worker', terasliceConfig, execution, logger); + const resource = kr.resource as k8s.V1Deployment; - expect(kr.resource.metadata.labels['teraslice.terascope.io/exId']) + const exId = resource.metadata?.labels ? resource.metadata.labels['teraslice.terascope.io/exId'] : undefined; + expect(exId) .toEqual('e76a0278-d9bc-4d78-bf14-431bcd97528c'); - expect(kr.resource.spec.template.spec.containers[0].resources).toEqual(yaml.load(` + + const resources = resource.spec?.template.spec?.containers[0].resources; + expect(resources).toEqual(yaml.load(` requests: memory: 2147483648 cpu: 1 @@ -220,21 +301,26 @@ describe('k8sResource', () => { memory: 2147483648 cpu: 1`)); - const envArray = kr.resource.spec.template.spec.containers[0].env; - expect(_.find(envArray, { name: 'NODE_OPTIONS' }).value) + const envArray = resource.spec?.template.spec?.containers[0].env; + const nodeOptionsEnv = _.find(envArray, { name: 'NODE_OPTIONS' }); + expect(nodeOptionsEnv?.value) .toEqual('--max-old-space-size=1843'); }); it('has the ability to set custom env', () => { - const kr = new K8sResource('deployments', 'worker', terasliceConfig, execution, logger); + const kr = new K8sResource('deployment', 'worker', terasliceConfig, execution, logger); + const resource = kr.resource as k8s.V1Deployment; // NOTE: the env var merge happens in _setResources(), which is // somewhat out of place. - const envArray = kr.resource.spec.template.spec.containers[0].env; - expect(_.find(envArray, { name: 'FOO' }).value) + const envArray = resource.spec?.template.spec?.containers[0].env; + + const fooEnv = _.find(envArray, { name: 'FOO' }); + expect(fooEnv?.value) .toEqual('baz'); - expect(_.find(envArray, { name: 'EXAMPLE' }).value) + const exampleEnv = _.find(envArray, { name: 'EXAMPLE' }); + expect(exampleEnv?.value) .toEqual('test'); }); @@ -244,11 +330,13 @@ describe('k8sResource', () => { terasliceConfig.cpu = 1; terasliceConfig.memory = 2147483648; - const kr = new K8sResource('deployments', 'worker', terasliceConfig, execution, logger); + const kr = new K8sResource('deployment', 'worker', terasliceConfig, execution, logger); + const resource = kr.resource as k8s.V1Deployment; - expect(kr.resource.metadata.labels['teraslice.terascope.io/exId']) + const exId = resource.metadata?.labels ? resource.metadata.labels['teraslice.terascope.io/exId'] : undefined; + expect(exId) .toEqual('e76a0278-d9bc-4d78-bf14-431bcd97528c'); - expect(kr.resource.spec.template.spec.containers[0].resources).toEqual(yaml.load(` + expect(resource.spec?.template.spec?.containers[0].resources).toEqual(yaml.load(` requests: memory: 1073741824 cpu: 2 @@ -256,8 +344,9 @@ describe('k8sResource', () => { memory: 1073741824 cpu: 2`)); - const envArray = kr.resource.spec.template.spec.containers[0].env; - expect(_.find(envArray, { name: 'NODE_OPTIONS' }).value) + const envArray = resource.spec?.template.spec?.containers[0].env; + const nodeOptionsEnv = _.find(envArray, { name: 'NODE_OPTIONS' }); + expect(nodeOptionsEnv?.value) .toEqual('--max-old-space-size=922'); }); @@ -266,11 +355,13 @@ describe('k8sResource', () => { terasliceConfig.cpu = 1; terasliceConfig.memory = 2147483648; - const kr = new K8sResource('deployments', 'worker', terasliceConfig, execution, logger); + const kr = new K8sResource('deployment', 'worker', terasliceConfig, execution, logger); + const resource = kr.resource as k8s.V1Deployment; - expect(kr.resource.metadata.labels['teraslice.terascope.io/exId']) + const exId = resource.metadata?.labels ? resource.metadata.labels['teraslice.terascope.io/exId'] : undefined; + expect(exId) .toEqual('e76a0278-d9bc-4d78-bf14-431bcd97528c'); - expect(kr.resource.spec.template.spec.containers[0].resources).toEqual(yaml.load(` + expect(resource.spec?.template.spec?.containers[0].resources).toEqual(yaml.load(` requests: memory: 2147483648 cpu: 2 @@ -278,8 +369,9 @@ describe('k8sResource', () => { memory: 2147483648 cpu: 2`)); - const envArray = kr.resource.spec.template.spec.containers[0].env; - expect(_.find(envArray, { name: 'NODE_OPTIONS' }).value) + const envArray = resource.spec?.template.spec?.containers[0].env; + const nodeOptionsEnv = _.find(envArray, { name: 'NODE_OPTIONS' }); + expect(nodeOptionsEnv?.value) .toEqual('--max-old-space-size=1843'); }); @@ -287,11 +379,13 @@ describe('k8sResource', () => { execution.cpu = 1; execution.memory = 2147483648; - const kr = new K8sResource('deployments', 'worker', terasliceConfig, execution, logger); + const kr = new K8sResource('deployment', 'worker', terasliceConfig, execution, logger); + const resource = kr.resource as k8s.V1Deployment; - expect(kr.resource.metadata.labels['teraslice.terascope.io/exId']) + const exId = resource.metadata?.labels ? resource.metadata.labels['teraslice.terascope.io/exId'] : undefined; + expect(exId) .toEqual('e76a0278-d9bc-4d78-bf14-431bcd97528c'); - expect(kr.resource.spec.template.spec.containers[0].resources).toEqual(yaml.load(` + expect(resource.spec?.template.spec?.containers[0].resources).toEqual(yaml.load(` requests: memory: 2147483648 cpu: 1 @@ -299,8 +393,9 @@ describe('k8sResource', () => { memory: 2147483648 cpu: 1`)); - const envArray = kr.resource.spec.template.spec.containers[0].env; - expect(_.find(envArray, { name: 'NODE_OPTIONS' }).value) + const envArray = resource.spec?.template.spec?.containers[0].env; + const nodeOptionsEnv = _.find(envArray, { name: 'NODE_OPTIONS' }); + expect(nodeOptionsEnv?.value) .toEqual('--max-old-space-size=1843'); }); @@ -310,11 +405,13 @@ describe('k8sResource', () => { execution.resources_requests_memory = 2147483648; execution.resources_limits_memory = 3147483648; - const kr = new K8sResource('deployments', 'worker', terasliceConfig, execution, logger); + const kr = new K8sResource('deployment', 'worker', terasliceConfig, execution, logger); + const resource = kr.resource as k8s.V1Deployment; - expect(kr.resource.metadata.labels['teraslice.terascope.io/exId']) + const exId = resource.metadata?.labels ? resource.metadata.labels['teraslice.terascope.io/exId'] : undefined; + expect(exId) .toEqual('e76a0278-d9bc-4d78-bf14-431bcd97528c'); - expect(kr.resource.spec.template.spec.containers[0].resources).toEqual(yaml.load(` + expect(resource.spec?.template.spec?.containers[0].resources).toEqual(yaml.load(` requests: memory: 2147483648 cpu: 1 @@ -322,33 +419,37 @@ describe('k8sResource', () => { memory: 3147483648 cpu: 2`)); - const envArray = kr.resource.spec.template.spec.containers[0].env; - expect(_.find(envArray, { name: 'NODE_OPTIONS' }).value) + const envArray = resource.spec?.template.spec?.containers[0].env; + const nodeOptionsEnv = _.find(envArray, { name: 'NODE_OPTIONS' }); + expect(nodeOptionsEnv?.value) .toEqual('--max-old-space-size=2702'); }); it('has memory limits and requests when set on execution', () => { execution.memory = 2147483648; - const kr = new K8sResource('deployments', 'worker', terasliceConfig, execution, logger); + const kr = new K8sResource('deployment', 'worker', terasliceConfig, execution, logger); + const resource = kr.resource as k8s.V1Deployment; - expect(kr.resource.spec.template.spec.containers[0].resources).toEqual(yaml.load(` + expect(resource.spec?.template.spec?.containers[0].resources).toEqual(yaml.load(` requests: memory: 2147483648 limits: memory: 2147483648`)); - const envArray = kr.resource.spec.template.spec.containers[0].env; - expect(_.find(envArray, { name: 'NODE_OPTIONS' }).value) + const envArray = resource.spec?.template.spec?.containers[0].env; + const nodeOptionsEnv = _.find(envArray, { name: 'NODE_OPTIONS' }); + expect(nodeOptionsEnv?.value) .toEqual('--max-old-space-size=1843'); }); it('has cpu limits and requests when set on execution', () => { execution.cpu = 1; - const kr = new K8sResource('deployments', 'worker', terasliceConfig, execution, logger); + const kr = new K8sResource('deployment', 'worker', terasliceConfig, execution, logger); + const resource = kr.resource as k8s.V1Deployment; - expect(kr.resource.spec.template.spec.containers[0].resources).toEqual(yaml.load(` + expect(resource.spec?.template.spec?.containers[0].resources).toEqual(yaml.load(` requests: cpu: 1 limits: @@ -358,9 +459,10 @@ describe('k8sResource', () => { it('has scratch volume when ephemeral_storage is set true on execution', () => { execution.ephemeral_storage = true; - const kr = new K8sResource('deployments', 'worker', terasliceConfig, execution, logger); + const kr = new K8sResource('deployment', 'worker', terasliceConfig, execution, logger); + const resource = kr.resource as k8s.V1Deployment; - expect(kr.resource.spec.template.spec.containers[0].volumeMounts) + expect(resource.spec?.template.spec?.containers[0].volumeMounts) .toEqual( [ { mountPath: '/app/config', name: 'config' }, @@ -372,9 +474,10 @@ describe('k8sResource', () => { it('does not have scratch volume when ephemeral_storage is set false on execution', () => { execution.ephemeral_storage = false; - const kr = new K8sResource('deployments', 'worker', terasliceConfig, execution, logger); + const kr = new K8sResource('deployment', 'worker', terasliceConfig, execution, logger); + const resource = kr.resource as k8s.V1Deployment; - expect(kr.resource.spec.template.spec.containers[0].volumeMounts) + expect(resource.spec?.template.spec?.containers[0].volumeMounts) .toEqual([{ mountPath: '/app/config', name: 'config' }]); }); }); @@ -382,19 +485,21 @@ describe('k8sResource', () => { describe('worker deployments with targets', () => { it('does not have affinity or toleration properties when targets equals [].', () => { execution.targets = []; - const kr = new K8sResource('deployments', 'worker', terasliceConfig, execution, logger); + const kr = new K8sResource('deployment', 'worker', terasliceConfig, execution, logger); + const resource = kr.resource as k8s.V1Deployment; - expect(kr.resource.spec.template.spec).not.toHaveProperty('affinity'); - expect(kr.resource.spec.template.spec).not.toHaveProperty('toleration'); + expect(resource.spec?.template.spec).not.toHaveProperty('affinity'); + expect(resource.spec?.template.spec).not.toHaveProperty('toleration'); }); it('has valid resource object with affinity when execution has one target without constraint', () => { execution.targets = [ { key: 'zone', value: 'west' } ]; - const kr = new K8sResource('deployments', 'worker', terasliceConfig, execution, logger); + const kr = new K8sResource('deployment', 'worker', terasliceConfig, execution, logger); + const resource = kr.resource as k8s.V1Deployment; - expect(kr.resource.spec.template.spec.affinity).toEqual(yaml.load(` + expect(resource.spec?.template.spec?.affinity).toEqual(yaml.load(` nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: @@ -410,9 +515,10 @@ describe('k8sResource', () => { { key: 'zone', value: 'west' } ]; terasliceConfig.kubernetes_worker_antiaffinity = true; - const kr = new K8sResource('deployments', 'worker', terasliceConfig, execution, logger); + const kr = new K8sResource('deployment', 'worker', terasliceConfig, execution, logger); + const resource = kr.resource as k8s.V1Deployment; - expect(kr.resource.spec.template.spec.affinity).toEqual(yaml.load(` + expect(resource.spec?.template.spec?.affinity).toEqual(yaml.load(` nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: @@ -442,9 +548,10 @@ describe('k8sResource', () => { execution.targets = [ { key: 'zone', value: 'west', constraint: 'required' } ]; - const kr = new K8sResource('deployments', 'worker', terasliceConfig, execution, logger); + const kr = new K8sResource('deployment', 'worker', terasliceConfig, execution, logger); + const resource = kr.resource as k8s.V1Deployment; - expect(kr.resource.spec.template.spec.affinity).toEqual(yaml.load(` + expect(resource.spec?.template.spec?.affinity).toEqual(yaml.load(` nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: @@ -460,9 +567,10 @@ describe('k8sResource', () => { { key: 'zone', value: 'west', constraint: 'required' }, { key: 'region', value: '42', constraint: 'required' } ]; - const kr = new K8sResource('deployments', 'worker', terasliceConfig, execution, logger); + const kr = new K8sResource('deployment', 'worker', terasliceConfig, execution, logger); + const resource = kr.resource as k8s.V1Deployment; - expect(kr.resource.spec.template.spec.affinity).toEqual(yaml.load(` + expect(resource.spec?.template.spec?.affinity).toEqual(yaml.load(` nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: @@ -481,9 +589,10 @@ describe('k8sResource', () => { execution.targets = [ { key: 'zone', value: 'west', constraint: 'preferred' } ]; - const kr = new K8sResource('deployments', 'worker', terasliceConfig, execution, logger); + const kr = new K8sResource('deployment', 'worker', terasliceConfig, execution, logger); + const resource = kr.resource as k8s.V1Deployment; - expect(kr.resource.spec.template.spec.affinity).toEqual(yaml.load(` + expect(resource.spec?.template.spec?.affinity).toEqual(yaml.load(` nodeAffinity: preferredDuringSchedulingIgnoredDuringExecution: - weight: 1 @@ -500,9 +609,10 @@ describe('k8sResource', () => { { key: 'zone', value: 'west', constraint: 'preferred' }, { key: 'region', value: 'texas', constraint: 'preferred' } ]; - const kr = new K8sResource('deployments', 'worker', terasliceConfig, execution, logger); + const kr = new K8sResource('deployment', 'worker', terasliceConfig, execution, logger); + const resource = kr.resource as k8s.V1Deployment; - expect(kr.resource.spec.template.spec.affinity).toEqual(yaml.load(` + expect(resource.spec?.template.spec?.affinity).toEqual(yaml.load(` nodeAffinity: preferredDuringSchedulingIgnoredDuringExecution: - weight: 1 @@ -525,10 +635,11 @@ describe('k8sResource', () => { execution.targets = [ { key: 'zone', value: 'west', constraint: 'accepted' } ]; - const kr = new K8sResource('deployments', 'worker', terasliceConfig, execution, logger); + const kr = new K8sResource('deployment', 'worker', terasliceConfig, execution, logger); + const resource = kr.resource as k8s.V1Deployment; // console.log(yaml.dump(kr.resource.spec.template.spec.tolerations)); - expect(kr.resource.spec.template.spec.tolerations).toEqual(yaml.load(` + expect(resource.spec?.template.spec?.tolerations).toEqual(yaml.load(` - key: zone operator: Equal value: west @@ -540,10 +651,11 @@ describe('k8sResource', () => { { key: 'zone', value: 'west', constraint: 'accepted' }, { key: 'region', value: 'texas', constraint: 'accepted' } ]; - const kr = new K8sResource('deployments', 'worker', terasliceConfig, execution, logger); + const kr = new K8sResource('deployment', 'worker', terasliceConfig, execution, logger); + const resource = kr.resource as k8s.V1Deployment; // console.log(yaml.dump(kr.resource.spec.template.spec.tolerations)); - expect(kr.resource.spec.template.spec.tolerations).toEqual(yaml.load(` + expect(resource.spec?.template.spec?.tolerations).toEqual(yaml.load(` - key: zone operator: Equal value: west @@ -559,10 +671,11 @@ describe('k8sResource', () => { { key: 'zone', value: 'west', constraint: 'required' }, { key: 'region', value: 'texas', constraint: 'preferred' } ]; - const kr = new K8sResource('deployments', 'worker', terasliceConfig, execution, logger); + const kr = new K8sResource('deployment', 'worker', terasliceConfig, execution, logger); + const resource = kr.resource as k8s.V1Deployment; // console.log(yaml.dump(kr.resource.spec.template.spec.tolerations)); - expect(kr.resource.spec.template.spec.affinity).toEqual(yaml.load(` + expect(resource.spec?.template.spec?.affinity).toEqual(yaml.load(` nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: @@ -590,9 +703,10 @@ describe('k8sResource', () => { { key: 'zone', value: 'west', constraint: 'accepted' }, { key: 'region', value: 'texas', constraint: 'accepted' } ]; - const kr = new K8sResource('deployments', 'worker', terasliceConfig, execution, logger); + const kr = new K8sResource('deployment', 'worker', terasliceConfig, execution, logger); + const resource = kr.resource as k8s.V1Deployment; - expect(kr.resource.spec.template.spec.affinity).toEqual(yaml.load(` + expect(resource.spec?.template.spec?.affinity).toEqual(yaml.load(` nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: @@ -620,7 +734,7 @@ describe('k8sResource', () => { operator: In values: - texas`)); - expect(kr.resource.spec.template.spec.tolerations).toEqual(yaml.load(` + expect(resource.spec?.template.spec?.tolerations).toEqual(yaml.load(` - key: zone operator: Equal value: west @@ -639,11 +753,14 @@ describe('k8sResource', () => { key2: 'value2' }; - const kr = new K8sResource('deployments', 'worker', terasliceConfig, execution, logger); + const kr = new K8sResource('deployment', 'worker', terasliceConfig, execution, logger); + const resource = kr.resource as k8s.V1Deployment; // console.log(yaml.dump(kr.resource)); - expect(kr.resource.metadata.labels['job.teraslice.terascope.io/key1']).toEqual('value1'); - expect(kr.resource.metadata.labels['job.teraslice.terascope.io/key2']).toEqual('value2'); + const key1Value = resource.metadata?.labels ? resource.metadata.labels['job.teraslice.terascope.io/key1'] : undefined; + const key2Value = resource.metadata?.labels ? resource.metadata.labels['job.teraslice.terascope.io/key2'] : undefined; + expect(key1Value).toEqual('value1'); + expect(key2Value).toEqual('value2'); }); it('generates valid k8s resources with keys containing forbidden characters', () => { @@ -653,21 +770,33 @@ describe('k8sResource', () => { abcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghij1234: 'value3', }; - const kr = new K8sResource('deployments', 'worker', terasliceConfig, execution, logger); + const kr = new K8sResource('deployment', 'worker', terasliceConfig, execution, logger); + const resource = kr.resource as k8s.V1Deployment; + // console.log(yaml.dump(kr.resource)); - expect(kr.resource.metadata.labels['job.teraslice.terascope.io/key-1']).toEqual('value1'); - expect(kr.resource.metadata.labels['job.teraslice.terascope.io/key2-']).toEqual('value2'); - expect(kr.resource.metadata.labels['job.teraslice.terascope.io/abcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghij123']).toEqual('value3'); + const forbidden1Value = resource.metadata?.labels + ? resource.metadata.labels['job.teraslice.terascope.io/key-1'] + : undefined; + const forbidden2Value = resource.metadata?.labels + ? resource.metadata.labels['job.teraslice.terascope.io/key2-'] + : undefined; + const forbidden3Value = resource.metadata?.labels + ? resource.metadata.labels['job.teraslice.terascope.io/abcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghij123'] + : undefined; + expect(forbidden1Value).toEqual('value1'); + expect(forbidden2Value).toEqual('value2'); + expect(forbidden3Value).toEqual('value3'); }); }); describe('teraslice job with one valid external_ports set', () => { it('generates k8s worker deployment with containerPort on container', () => { execution.external_ports = [9090]; - const kr = new K8sResource('deployments', 'worker', terasliceConfig, execution, logger); + const kr = new K8sResource('deployment', 'worker', terasliceConfig, execution, logger); + const resource = kr.resource as k8s.V1Deployment; // console.log(yaml.dump(kr.resource.spec.template.spec.containers[0].ports)); - expect(kr.resource.spec.template.spec.containers[0].ports) + expect(resource.spec?.template.spec?.containers[0].ports) .toEqual([ { containerPort: 45680 }, { containerPort: 9090 } @@ -676,10 +805,11 @@ describe('k8sResource', () => { it('generates k8s execution controller job with containerPort on container', () => { execution.external_ports = [9090]; - const kr = new K8sResource('jobs', 'execution_controller', terasliceConfig, execution, logger); + const kr = new K8sResource('job', 'execution_controller', terasliceConfig, execution, logger); + const resource = kr.resource as k8s.V1Job; // console.log(yaml.dump(kr.resource.spec.template.spec.containers[0].ports)); - expect(kr.resource.spec.template.spec.containers[0].ports) + expect(resource.spec?.template.spec?.containers[0].ports) .toEqual([ { containerPort: 45680 }, { containerPort: 9090 } @@ -693,10 +823,11 @@ describe('k8sResource', () => { 9090, { name: 'metrics', port: 9091 } ]; - const kr = new K8sResource('deployments', 'worker', terasliceConfig, execution, logger); + const kr = new K8sResource('deployment', 'worker', terasliceConfig, execution, logger); + const resource = kr.resource as k8s.V1Deployment; // console.log(yaml.dump(kr.resource.spec.template.spec.containers[0].ports)); - expect(kr.resource.spec.template.spec.containers[0].ports) + expect(resource.spec?.template.spec?.containers[0].ports) .toEqual([ { containerPort: 45680 }, { containerPort: 9090 }, @@ -706,10 +837,11 @@ describe('k8sResource', () => { it('generates k8s execution controller job with containerPort on container', () => { execution.external_ports = [9090, 9091]; - const kr = new K8sResource('jobs', 'execution_controller', terasliceConfig, execution, logger); + const kr = new K8sResource('job', 'execution_controller', terasliceConfig, execution, logger); + const resource = kr.resource as k8s.V1Job; // console.log(yaml.dump(kr.resource.spec.template.spec.containers[0].ports)); - expect(kr.resource.spec.template.spec.containers[0].ports) + expect(resource.spec?.template.spec?.containers[0].ports) .toEqual([ { containerPort: 45680 }, { containerPort: 9090 }, @@ -720,25 +852,37 @@ describe('k8sResource', () => { describe('execution_controller job', () => { it('has valid resource object.', () => { - const kr = new K8sResource('jobs', 'execution_controller', terasliceConfig, execution, logger); + const kr = new K8sResource('job', 'execution_controller', terasliceConfig, execution, logger); + const resource = kr.resource as k8s.V1Job; - expect(kr.resource.kind).toBe('Job'); - expect(kr.resource.metadata.name).toBe('ts-exc-example-data-generator-job-7ba9afb0-417a'); + expect(resource.kind).toBe('Job'); + expect(resource.metadata?.name).toBe('ts-exc-example-data-generator-job-7ba9afb0-417a'); // The following properties should be absent in the default case - expect(kr.resource.spec.template.spec).not.toHaveProperty('affinity'); - expect(kr.resource.spec.template.spec).not.toHaveProperty('imagePullSecrets'); - expect(kr.resource.spec.template.spec).not.toHaveProperty('priorityClassName'); + expect(resource.spec?.template.spec).not.toHaveProperty('affinity'); + expect(resource.spec?.template.spec).not.toHaveProperty('imagePullSecrets'); + expect(resource.spec?.template.spec).not.toHaveProperty('priorityClassName'); // Configmaps should be mounted on all workers - expect(kr.resource.spec.template.spec.volumes[0]).toEqual(yaml.load(` + + let configVolume: k8s.V1Volume | undefined = undefined; + if (resource.spec?.template.spec?.volumes) { + configVolume = resource.spec.template.spec.volumes[0]; + } + + let configVolumeMount: k8s.V1VolumeMount | undefined = undefined; + if (resource.spec?.template.spec?.containers[0].volumeMounts) { + configVolumeMount = resource.spec.template.spec.containers[0].volumeMounts[0]; + } + + expect(configVolume).toEqual(yaml.load(` name: config configMap: name: ts-dev1-worker items: - key: teraslice.yaml path: teraslice.yaml`)); - expect(kr.resource.spec.template.spec.containers[0].volumeMounts[0]) + expect(configVolumeMount) .toEqual(yaml.load(` mountPath: /app/config name: config`)); @@ -748,9 +892,10 @@ describe('k8sResource', () => { terasliceConfig.cpu_execution_controller = 1; terasliceConfig.memory_execution_controller = 2147483648; - const kr = new K8sResource('jobs', 'execution_controller', terasliceConfig, execution, logger); + const kr = new K8sResource('job', 'execution_controller', terasliceConfig, execution, logger); + const resource = kr.resource as k8s.V1Job; - expect(kr.resource.spec.template.spec.containers[0].resources).toEqual(yaml.load(` + expect(resource.spec?.template.spec?.containers[0].resources).toEqual(yaml.load(` requests: memory: 2147483648 cpu: 1 @@ -758,8 +903,9 @@ describe('k8sResource', () => { memory: 2147483648 cpu: 1`)); - const envArray = kr.resource.spec.template.spec.containers[0].env; - expect(_.find(envArray, { name: 'NODE_OPTIONS' }).value) + const envArray = resource.spec?.template.spec?.containers[0].env; + const nodeOptionsEnv = _.find(envArray, { name: 'NODE_OPTIONS' }); + expect(nodeOptionsEnv?.value) .toEqual('--max-old-space-size=1843'); }); @@ -769,9 +915,10 @@ describe('k8sResource', () => { terasliceConfig.cpu_execution_controller = 1; terasliceConfig.memory_execution_controller = 2147483648; - const kr = new K8sResource('jobs', 'execution_controller', terasliceConfig, execution, logger); + const kr = new K8sResource('job', 'execution_controller', terasliceConfig, execution, logger); + const resource = kr.resource as k8s.V1Job; - expect(kr.resource.spec.template.spec.containers[0].resources).toEqual(yaml.load(` + expect(resource.spec?.template.spec?.containers[0].resources).toEqual(yaml.load(` requests: memory: 1073741824 cpu: 2 @@ -779,8 +926,9 @@ describe('k8sResource', () => { memory: 1073741824 cpu: 2`)); - const envArray = kr.resource.spec.template.spec.containers[0].env; - expect(_.find(envArray, { name: 'NODE_OPTIONS' }).value) + const envArray = resource.spec?.template.spec?.containers[0].env; + const nodeOptionsEnv = _.find(envArray, { name: 'NODE_OPTIONS' }); + expect(nodeOptionsEnv?.value) .toEqual('--max-old-space-size=922'); }); }); @@ -796,9 +944,10 @@ describe('k8sResource', () => { ['teraslice-JOB-name', 'ts-wkr-teraslice-job-name-7ba9afb0-417a'] ])('when Job Name is %s the k8s worker name is: %s', (jobName, k8sName) => { execution.name = jobName; - const kr = new K8sResource('deployments', 'worker', terasliceConfig, execution, logger); + const kr = new K8sResource('deployment', 'worker', terasliceConfig, execution, logger); + const resource = kr.resource as k8s.V1Deployment; - expect(kr.resource.metadata.name).toBe(k8sName); + expect(resource.metadata?.name).toBe(k8sName); }); }); @@ -809,14 +958,15 @@ describe('k8sResource', () => { { key: 'key2', value: 'value2' } ]; - const kr = new K8sResource('jobs', 'execution_controller', terasliceConfig, execution, logger); + const kr = new K8sResource('job', 'execution_controller', terasliceConfig, execution, logger); + const resource = kr.resource as k8s.V1Deployment; - expect(kr.resource.kind).toBe('Job'); - expect(kr.resource.metadata.name).toBe('ts-exc-example-data-generator-job-7ba9afb0-417a'); + expect(resource.kind).toBe('Job'); + expect(resource.metadata?.name).toBe('ts-exc-example-data-generator-job-7ba9afb0-417a'); - expect(kr.resource.spec.template.spec).toHaveProperty('affinity'); - expect(kr.resource.spec.template.spec).toHaveProperty('tolerations'); - expect(kr.resource.spec.template.spec.affinity).toEqual(yaml.load(` + expect(resource.spec?.template.spec).toHaveProperty('affinity'); + expect(resource.spec?.template.spec).toHaveProperty('tolerations'); + expect(resource.spec?.template.spec?.affinity).toEqual(yaml.load(` nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: @@ -829,7 +979,7 @@ describe('k8sResource', () => { operator: In values: - value2`)); - expect(kr.resource.spec.template.spec.tolerations).toEqual(yaml.load(` + expect(resource.spec?.template.spec?.tolerations).toEqual(yaml.load(` - key: key1 operator: Equal value: value1 @@ -852,16 +1002,17 @@ describe('k8sResource', () => { { key: 'region', value: 'texas', constraint: 'accepted' } ]; - const kr = new K8sResource('jobs', 'execution_controller', terasliceConfig, execution, logger); + const kr = new K8sResource('job', 'execution_controller', terasliceConfig, execution, logger); + const resource = kr.resource as k8s.V1Deployment; - expect(kr.resource.kind).toBe('Job'); - expect(kr.resource.metadata.name).toBe('ts-exc-example-data-generator-job-7ba9afb0-417a'); + expect(resource.kind).toBe('Job'); + expect(resource.metadata?.name).toBe('ts-exc-example-data-generator-job-7ba9afb0-417a'); - expect(kr.resource.spec.template.spec).toHaveProperty('affinity'); - expect(kr.resource.spec.template.spec).toHaveProperty('tolerations'); + expect(resource.spec?.template.spec).toHaveProperty('affinity'); + expect(resource.spec?.template.spec).toHaveProperty('tolerations'); // console.log(yaml.dump(kr.resource.spec.template.spec.affinity)); - expect(kr.resource.spec.template.spec.affinity).toEqual(yaml.load(` + expect(resource.spec?.template.spec?.affinity).toEqual(yaml.load(` nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: @@ -876,7 +1027,7 @@ describe('k8sResource', () => { - value1`)); // console.log(yaml.dump(kr.resource.spec.template.spec.tolerations)); - expect(kr.resource.spec.template.spec.tolerations).toEqual(yaml.load(` + expect(resource.spec?.template.spec?.tolerations).toEqual(yaml.load(` - key: region operator: Equal value: texas @@ -893,23 +1044,31 @@ describe('k8sResource', () => { execution.stateful = true; terasliceConfig.kubernetes_priority_class_name = 'testPriorityClass'; - const krWorker = new K8sResource('deployments', 'worker', terasliceConfig, execution, logger); + const krWorker = new K8sResource('deployment', 'worker', terasliceConfig, execution, logger); + const wkrResource = krWorker.resource as k8s.V1Deployment; - expect(krWorker.resource.kind).toBe('Deployment'); - expect(krWorker.resource.spec.template.spec).toHaveProperty('priorityClassName'); - expect(krWorker.resource.spec.template.spec.priorityClassName).toEqual('testPriorityClass'); + expect(wkrResource.kind).toBe('Deployment'); + expect(wkrResource.spec?.template.spec).toHaveProperty('priorityClassName'); + expect(wkrResource.spec?.template.spec?.priorityClassName).toEqual('testPriorityClass'); - expect(krWorker.resource.spec.template.metadata.labels['job-property.teraslice.terascope.io/stateful']).toEqual('true'); + const wkrStatefulLabel = wkrResource.spec?.template.metadata?.labels + ? wkrResource.spec.template.metadata.labels['job-property.teraslice.terascope.io/stateful'] + : undefined; + expect(wkrStatefulLabel).toEqual('true'); - const krExporter = new K8sResource('jobs', 'execution_controller', terasliceConfig, execution, logger); + const krExporter = new K8sResource('job', 'execution_controller', terasliceConfig, execution, logger); + const exporterResource = krExporter.resource as k8s.V1Job; - expect(krExporter.resource.kind).toBe('Job'); - expect(krExporter.resource.metadata.name).toBe('ts-exc-example-data-generator-job-7ba9afb0-417a'); + expect(exporterResource.kind).toBe('Job'); + expect(exporterResource.metadata?.name).toBe('ts-exc-example-data-generator-job-7ba9afb0-417a'); - expect(krExporter.resource.spec.template.spec).toHaveProperty('priorityClassName'); - expect(krExporter.resource.spec.template.spec.priorityClassName).toEqual('testPriorityClass'); + expect(exporterResource.spec?.template.spec).toHaveProperty('priorityClassName'); + expect(exporterResource.spec?.template.spec?.priorityClassName).toEqual('testPriorityClass'); - expect(krExporter.resource.spec.template.metadata.labels['job-property.teraslice.terascope.io/stateful']).toEqual('true'); + const exporterStatefulLabel = exporterResource.spec?.template.metadata?.labels + ? exporterResource.spec.template.metadata.labels['job-property.teraslice.terascope.io/stateful'] + : undefined; + expect(exporterStatefulLabel).toEqual('true'); }); }); @@ -918,8 +1077,10 @@ describe('k8sResource', () => { execution.pod_spec_override = { initContainers: [] }; terasliceConfig.kubernetes_overrides_enabled = false; - const krWorker = new K8sResource('deployments', 'worker', terasliceConfig, execution, logger); - expect(krWorker.resource.spec.template.spec).not.toHaveProperty('initContainers'); + const krWorker = new K8sResource('deployment', 'worker', terasliceConfig, execution, logger); + const resource = krWorker.resource as k8s.V1Deployment; + + expect(resource.spec?.template.spec).not.toHaveProperty('initContainers'); }); }); @@ -928,8 +1089,10 @@ describe('k8sResource', () => { execution.pod_spec_override = { initContainers: [] }; terasliceConfig.kubernetes_overrides_enabled = true; - const krWorker = new K8sResource('deployments', 'worker', terasliceConfig, execution, logger); - expect(krWorker.resource.spec.template.spec).toHaveProperty('initContainers'); + const krWorker = new K8sResource('deployment', 'worker', terasliceConfig, execution, logger); + const resource = krWorker.resource as k8s.V1Deployment; + + expect(resource.spec?.template.spec).toHaveProperty('initContainers'); }); }); }); diff --git a/packages/teraslice/test/lib/cluster/services/cluster/backends/kubernetes/v2/k8sState-multicluster-v2-spec.ts b/packages/teraslice/test/lib/cluster/services/cluster/backends/kubernetes/v2/k8sState-multicluster-v2-spec.ts index 5fcaf9f66f2..b7fd9787a7f 100644 --- a/packages/teraslice/test/lib/cluster/services/cluster/backends/kubernetes/v2/k8sState-multicluster-v2-spec.ts +++ b/packages/teraslice/test/lib/cluster/services/cluster/backends/kubernetes/v2/k8sState-multicluster-v2-spec.ts @@ -1,13 +1,17 @@ import _ from 'lodash'; +import { debugLogger } from '@terascope/utils'; +import * as k8s from '@kubernetes/client-node'; import _podsJobRunning from '../files/job-running-v1-k8s-pods-multicluster.json'; import { gen } from '../../../../../../../../src/lib/cluster/services/cluster/backends/kubernetesV2/k8sState.js'; describe('k8sState with pods from multiple clusters', () => { + const logger = debugLogger('k8sResource'); + it('should generate cluster state correctly on first call', () => { - const podsJobRunning = _.cloneDeep(_podsJobRunning); + const podsJobRunning: k8s.V1PodList = _.cloneDeep(_podsJobRunning as any); const clusterState = {}; - gen(podsJobRunning, clusterState); + gen(podsJobRunning, clusterState, logger); // console.log(`clusterState\n\n${JSON.stringify(clusterState, null, 2)}`); // console.log(JSON.stringify(podsJobRunning, null, 2)); @@ -38,11 +42,11 @@ describe('k8sState with pods from multiple clusters', () => { }); it('should generate cluster state correctly on second call', () => { - const podsJobRunning = _.cloneDeep(_podsJobRunning); + const podsJobRunning = _.cloneDeep(_podsJobRunning as any); const clusterState = {}; - gen(podsJobRunning, clusterState); - gen(podsJobRunning, clusterState); + gen(podsJobRunning, clusterState, logger); + gen(podsJobRunning, clusterState, logger); expect(clusterState['192.168.99.100'].state).toEqual('connected'); expect(clusterState['192.168.99.100'].active.length).toEqual(3); diff --git a/packages/teraslice/test/lib/cluster/services/cluster/backends/kubernetes/v2/k8sState-v2-spec.ts b/packages/teraslice/test/lib/cluster/services/cluster/backends/kubernetes/v2/k8sState-v2-spec.ts index 811b1ac4459..89db984329e 100644 --- a/packages/teraslice/test/lib/cluster/services/cluster/backends/kubernetes/v2/k8sState-v2-spec.ts +++ b/packages/teraslice/test/lib/cluster/services/cluster/backends/kubernetes/v2/k8sState-v2-spec.ts @@ -1,13 +1,17 @@ import _ from 'lodash'; +import { debugLogger } from '@terascope/utils'; +import * as k8s from '@kubernetes/client-node'; import _podsJobRunning from '../files/job-running-v1-k8s-pods.json'; import { gen } from '../../../../../../../../src/lib/cluster/services/cluster/backends/kubernetesV2/k8sState.js'; describe('k8sState', () => { + const logger = debugLogger('k8sResource'); + it('should generate cluster state correctly on first call', () => { - const podsJobRunning = _.cloneDeep(_podsJobRunning); + const podsJobRunning: k8s.V1PodList = _.cloneDeep(_podsJobRunning as any); const clusterState = {}; - gen(podsJobRunning, clusterState); + gen(podsJobRunning, clusterState, logger); // console.log(`clusterState\n\n${JSON.stringify(clusterState, null, 2)}`); expect(clusterState['192.168.99.100'].state).toEqual('connected'); @@ -37,11 +41,11 @@ describe('k8sState', () => { }); it('should generate cluster state correctly on second call', () => { - const podsJobRunning = _.cloneDeep(_podsJobRunning); + const podsJobRunning = _.cloneDeep(_podsJobRunning as any); const clusterState = {}; - gen(podsJobRunning, clusterState); - gen(podsJobRunning, clusterState); + gen(podsJobRunning, clusterState, logger); + gen(podsJobRunning, clusterState, logger); expect(clusterState['192.168.99.100'].state).toEqual('connected'); expect(clusterState['192.168.99.100'].active.length).toEqual(3); @@ -70,14 +74,14 @@ describe('k8sState', () => { }); it('should remove old host ips', () => { - const podsJobRunning = _.cloneDeep(_podsJobRunning); + const podsJobRunning = _.cloneDeep(_podsJobRunning as any); const clusterState = {}; clusterState['2.2.2.2'] = { state: 'idk', active: [] }; - gen(podsJobRunning, clusterState); + gen(podsJobRunning, clusterState, logger); expect(clusterState['192.168.99.100'].active.length).toEqual(3); expect(clusterState['2.2.2.2']).toBeUndefined(); }); diff --git a/packages/teraslice/test/lib/cluster/services/cluster/backends/kubernetes/v2/utils-v2-spec.ts b/packages/teraslice/test/lib/cluster/services/cluster/backends/kubernetes/v2/utils-v2-spec.ts index fdfb5be6d49..f3b60b1cb6d 100644 --- a/packages/teraslice/test/lib/cluster/services/cluster/backends/kubernetes/v2/utils-v2-spec.ts +++ b/packages/teraslice/test/lib/cluster/services/cluster/backends/kubernetes/v2/utils-v2-spec.ts @@ -1,3 +1,4 @@ +import { K8sConfig } from '../../../../../../../../src/lib/cluster/services/cluster/backends/kubernetesV2/interfaces.js'; import { makeTemplate, getMaxOldSpace } from '../../../../../../../../src/lib/cluster/services/cluster/backends/kubernetesV2/utils.js'; @@ -6,8 +7,13 @@ import { safeEncode } from '../../../../../../../../src/lib/utils/encoding_utils describe('K8s Utils', () => { describe('->makeTemplate', () => { it('should be able to support the execution_controller job', () => { - const exJobTemplate = makeTemplate('jobs', 'execution_controller'); - const config = { + const exJobTemplate = makeTemplate('job', 'execution_controller'); + const config: K8sConfig = { + clusterName: 'teracluster', + configMapName: 'teracluster-worker', + exName: 'ts-exc-example-job-74ab9324-9f67', + exUid: '0512dae5-8ca2-437d-b744-fdc50695fd91', + replicas: 1, name: 'example', jobNameLabel: 'example-job', clusterNameLabel: 'example-cluster', @@ -35,13 +41,13 @@ describe('K8s Utils', () => { namespace: config.namespace }); - expect(exJob.spec.template.metadata.labels).toEqual(exJob.metadata.labels); + expect(exJob.spec?.template.metadata?.labels).toEqual(exJob.metadata?.labels); - const templateSpec = exJob.spec.template.spec; + const templateSpec = exJob.spec?.template.spec; - expect(templateSpec.containers[0].image).toEqual(config.dockerImage); - expect(templateSpec.containers[0].name).toEqual(config.name); - expect(templateSpec.containers[0].env).toEqual([ + expect(templateSpec?.containers[0].image).toEqual(config.dockerImage); + expect(templateSpec?.containers[0].name).toEqual(config.name); + expect(templateSpec?.containers[0].env).toEqual([ { name: 'NODE_TYPE', value: config.nodeType @@ -59,12 +65,14 @@ describe('K8s Utils', () => { } } ]); - expect(templateSpec.terminationGracePeriodSeconds).toEqual(config.shutdownTimeout); + expect(templateSpec?.terminationGracePeriodSeconds).toEqual(config.shutdownTimeout); }); it('should be able to support the worker deployment', () => { - const workerDeploymentTemplate = makeTemplate('deployments', 'worker'); - const config = { + const workerDeploymentTemplate = makeTemplate('deployment', 'worker'); + const config: K8sConfig = { + clusterName: 'teracluster', + configMapName: 'teracluster-worker', name: 'example', jobNameLabel: 'example-job', clusterNameLabel: 'example-cluster', @@ -105,16 +113,16 @@ describe('K8s Utils', () => { ], }); - expect(workerDeployment.spec.replicas).toEqual(config.replicas); + expect(workerDeployment.spec?.replicas).toEqual(config.replicas); - const { labels } = workerDeployment.spec.template.metadata; - expect(labels).toEqual(workerDeployment.metadata.labels); + const labels = workerDeployment.spec?.template.metadata?.labels; + expect(labels).toEqual(workerDeployment.metadata?.labels); - const templateSpec = workerDeployment.spec.template.spec; + const templateSpec = workerDeployment.spec?.template.spec; - expect(templateSpec.containers[0].image).toEqual(config.dockerImage); - expect(templateSpec.containers[0].name).toEqual(config.name); - expect(templateSpec.containers[0].env).toEqual([ + expect(templateSpec?.containers[0].image).toEqual(config.dockerImage); + expect(templateSpec?.containers[0].name).toEqual(config.name); + expect(templateSpec?.containers[0].env).toEqual([ { name: 'NODE_TYPE', value: config.nodeType @@ -132,7 +140,7 @@ describe('K8s Utils', () => { } } ]); - expect(templateSpec.terminationGracePeriodSeconds).toEqual(config.shutdownTimeout); + expect(templateSpec?.terminationGracePeriodSeconds).toEqual(config.shutdownTimeout); }); }); From 41a1dae5f1cac1a3899794f0cac8526ec1f6c62d Mon Sep 17 00:00:00 2001 From: busma13 Date: Wed, 30 Oct 2024 14:02:02 -0700 Subject: [PATCH 04/30] remove commented out code, fix import formatting --- .../backends/kubernetesV2/interfaces.ts | 25 ------------------- .../cluster/backends/kubernetesV2/k8s.ts | 5 +++- 2 files changed, 4 insertions(+), 26 deletions(-) diff --git a/packages/teraslice/src/lib/cluster/services/cluster/backends/kubernetesV2/interfaces.ts b/packages/teraslice/src/lib/cluster/services/cluster/backends/kubernetesV2/interfaces.ts index f46a1d2c208..b39c65516a8 100644 --- a/packages/teraslice/src/lib/cluster/services/cluster/backends/kubernetesV2/interfaces.ts +++ b/packages/teraslice/src/lib/cluster/services/cluster/backends/kubernetesV2/interfaces.ts @@ -62,28 +62,3 @@ export type DeleteParams = [ export type NodeType = 'worker' | 'execution_controller'; export type ScaleOp = `set` | `add` | `remove`; - -// export type PostDeploymentArgs = { -// manifest: k8s.V1Deployment; -// manifestType: 'deployment'; -// }; - -// export type PostJobArgs = { -// manifest: k8s.V1Job; -// manifestType: 'job'; -// }; -// export type PostPodArgs = { -// manifest: k8s.V1Pod; -// manifestType: 'pod'; -// }; -// export type PostReplicaSetArgs = { -// manifest: k8s.V1ReplicaSet; -// manifestType: 'replicaset'; -// }; -// export type PostServiceArgs = { -// manifest: k8s.V1Service; -// manifestType: 'service'; -// }; - -// export type PostArgs = PostDeploymentArgs | PostJobArgs -// | PostPodArgs | PostReplicaSetArgs | PostServiceArgs; diff --git a/packages/teraslice/src/lib/cluster/services/cluster/backends/kubernetesV2/k8s.ts b/packages/teraslice/src/lib/cluster/services/cluster/backends/kubernetesV2/k8s.ts index f8311b658f1..553216a7401 100644 --- a/packages/teraslice/src/lib/cluster/services/cluster/backends/kubernetesV2/k8s.ts +++ b/packages/teraslice/src/lib/cluster/services/cluster/backends/kubernetesV2/k8s.ts @@ -3,7 +3,10 @@ import { pDelay, pRetry, Logger } from '@terascope/utils'; import * as k8s from '@kubernetes/client-node'; -import { getRetryConfig, isDeployment, isJob, isPod, isReplicaSet, isService } from './utils.js'; +import { + getRetryConfig, isDeployment, isJob, + isPod, isReplicaSet, isService +} from './utils.js'; import { DeleteApiResponse, DeleteParams, ResourceListApiResponse, ResourceType, PatchApiResponse, Resource, From 7eb3e10e701d6f0fe6f9e2c4e34384831a9f83fd Mon Sep 17 00:00:00 2001 From: busma13 Date: Wed, 30 Oct 2024 14:18:32 -0700 Subject: [PATCH 05/30] revert _getClusterState to previous syntax --- .../cluster/backends/kubernetesV2/index.ts | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/packages/teraslice/src/lib/cluster/services/cluster/backends/kubernetesV2/index.ts b/packages/teraslice/src/lib/cluster/services/cluster/backends/kubernetesV2/index.ts index 39e8f3860aa..48985172689 100644 --- a/packages/teraslice/src/lib/cluster/services/cluster/backends/kubernetesV2/index.ts +++ b/packages/teraslice/src/lib/cluster/services/cluster/backends/kubernetesV2/index.ts @@ -68,16 +68,15 @@ export class KubernetesClusterBackendV2 { * @return {Promise} void */ private async _getClusterState() { - try { - const k8sPods: K8sClient.V1PodList = await this.k8s.list(`app.kubernetes.io/name=teraslice,app.kubernetes.io/instance=${this.clusterNameLabel}`, 'pod'); - gen(k8sPods, this.clusterState, this.logger); - } catch (err) { - // TODO: We might need to do more here. I think it's OK to just - // log though. This only gets used to show slicer info through - // the API. We wouldn't want to disrupt the cluster master - // for rare failures to reach the k8s API. - logError(this.logger, err, 'Error listing teraslice pods in k8s'); - } + return this.k8s.list(`app.kubernetes.io/name=teraslice,app.kubernetes.io/instance=${this.clusterNameLabel}`, 'pod') + .then((k8sPods) => gen(k8sPods, this.clusterState, this.logger)) + .catch((err) => { + // TODO: We might need to do more here. I think it's OK to just + // log though. This only gets used to show slicer info through + // the API. We wouldn't want to disrupt the cluster master + // for rare failures to reach the k8s API. + logError(this.logger, err, 'Error listing teraslice pods in k8s'); + }); } /** From c6d50e268d9286689932fe5156f26561b8811d88 Mon Sep 17 00:00:00 2001 From: busma13 Date: Wed, 30 Oct 2024 15:07:51 -0700 Subject: [PATCH 06/30] replace instanceof checks --- .../services/cluster/backends/kubernetesV2/index.ts | 9 ++++----- .../cluster/backends/kubernetesV2/k8sResource.ts | 4 ++-- 2 files changed, 6 insertions(+), 7 deletions(-) diff --git a/packages/teraslice/src/lib/cluster/services/cluster/backends/kubernetesV2/index.ts b/packages/teraslice/src/lib/cluster/services/cluster/backends/kubernetesV2/index.ts index 48985172689..78c4703d176 100644 --- a/packages/teraslice/src/lib/cluster/services/cluster/backends/kubernetesV2/index.ts +++ b/packages/teraslice/src/lib/cluster/services/cluster/backends/kubernetesV2/index.ts @@ -3,12 +3,11 @@ import { cloneDeep, pRetry, Logger } from '@terascope/utils'; import type { Context, ExecutionConfig } from '@terascope/job-components'; -import * as K8sClient from '@kubernetes/client-node'; import { makeLogger } from '../../../../../workers/helpers/terafoundation.js'; import { K8sResource } from './k8sResource.js'; import { gen } from './k8sState.js'; import { K8s } from './k8s.js'; -import { getRetryConfig } from './utils.js'; +import { getRetryConfig, isDeployment, isJob, isService } from './utils.js'; import { StopExecutionOptions } from '../../../interfaces.js'; import { ResourceType } from './interfaces.js'; @@ -114,7 +113,7 @@ export class KubernetesClusterBackendV2 { this.logger ); - if (!(exJobResource.resource instanceof K8sClient.V1Job)) { + if (!(isJob(exJobResource.resource))) { throw new Error(`exJobResource.resource must be of type k8s.V1Job`); } @@ -135,7 +134,7 @@ export class KubernetesClusterBackendV2 { jobResult.metadata?.uid ); - if (!(exServiceResource.resource instanceof K8sClient.V1Service)) { + if (!(isService(exServiceResource.resource))) { throw new Error(`exJobResource.resource must be of type k8s.V1Service`); } @@ -210,7 +209,7 @@ export class KubernetesClusterBackendV2 { jobs.items[0].metadata?.uid ); - if (!(kr.resource instanceof K8sClient.V1Deployment)) { + if (!(isDeployment(kr.resource))) { throw new Error(`exJobResource.resource must be of type k8s.V1Deployment`); } const workerDeployment = kr.resource; diff --git a/packages/teraslice/src/lib/cluster/services/cluster/backends/kubernetesV2/k8sResource.ts b/packages/teraslice/src/lib/cluster/services/cluster/backends/kubernetesV2/k8sResource.ts index 78e1be6dbe5..8b78b2a4a35 100644 --- a/packages/teraslice/src/lib/cluster/services/cluster/backends/kubernetesV2/k8sResource.ts +++ b/packages/teraslice/src/lib/cluster/services/cluster/backends/kubernetesV2/k8sResource.ts @@ -3,7 +3,7 @@ import * as k8s from '@kubernetes/client-node'; import { isNumber, Logger } from '@terascope/utils'; import type { TerasliceConfig, ExecutionConfig } from '@terascope/job-components'; import { safeEncode } from '../../../../../utils/encoding_utils.js'; -import { makeTemplate, setMaxOldSpaceViaEnv } from './utils.js'; +import { isService, makeTemplate, setMaxOldSpaceViaEnv } from './utils.js'; import { K8sConfig, NodeType } from './interfaces.js'; export class K8sResource { @@ -63,7 +63,7 @@ export class K8sResource { this.templateConfig = this._makeConfig(); this.resource = this.templateGenerator(this.templateConfig); - if (!(this.resource instanceof k8s.V1Service)) { + if (!(isService(this.resource))) { this._setJobLabels(this.resource); // Apply job `targets` setting as k8s nodeAffinity From f38613d3e32486947622cfb66272272b7f70d23f Mon Sep 17 00:00:00 2001 From: busma13 Date: Wed, 30 Oct 2024 16:26:39 -0700 Subject: [PATCH 07/30] refactor post and list functions --- .../cluster/backends/kubernetesV2/index.ts | 6 +- .../backends/kubernetesV2/interfaces.ts | 20 ++++ .../cluster/backends/kubernetesV2/k8s.ts | 109 ++++++++---------- .../backends/kubernetes/v2/k8s-v2-spec.ts | 13 +-- 4 files changed, 75 insertions(+), 73 deletions(-) diff --git a/packages/teraslice/src/lib/cluster/services/cluster/backends/kubernetesV2/index.ts b/packages/teraslice/src/lib/cluster/services/cluster/backends/kubernetesV2/index.ts index 78c4703d176..7e9b348572a 100644 --- a/packages/teraslice/src/lib/cluster/services/cluster/backends/kubernetesV2/index.ts +++ b/packages/teraslice/src/lib/cluster/services/cluster/backends/kubernetesV2/index.ts @@ -121,7 +121,7 @@ export class KubernetesClusterBackendV2 { this.logger.debug(exJob, 'execution allocating slicer'); - const jobResult = await this.k8s.post(exJob, 'job'); + const jobResult = await this.k8s.post(exJob); const exServiceResource = new K8sResource( 'service', @@ -140,7 +140,7 @@ export class KubernetesClusterBackendV2 { const exService = exServiceResource.resource; - const serviceResult = await this.k8s.post(exService, 'service'); + const serviceResult = await this.k8s.post(exService); this.logger.debug(jobResult, 'k8s slicer job submitted'); @@ -216,7 +216,7 @@ export class KubernetesClusterBackendV2 { this.logger.debug(`workerDeployment:\n\n${JSON.stringify(workerDeployment, null, 2)}`); - return this.k8s.post(workerDeployment, 'deployment') + return this.k8s.post(workerDeployment) .then((result) => this.logger.debug(`k8s worker deployment submitted: ${JSON.stringify(result)}`)) .catch((err) => { const error = new TSError(err, { diff --git a/packages/teraslice/src/lib/cluster/services/cluster/backends/kubernetesV2/interfaces.ts b/packages/teraslice/src/lib/cluster/services/cluster/backends/kubernetesV2/interfaces.ts index b39c65516a8..6fa0451502d 100644 --- a/packages/teraslice/src/lib/cluster/services/cluster/backends/kubernetesV2/interfaces.ts +++ b/packages/teraslice/src/lib/cluster/services/cluster/backends/kubernetesV2/interfaces.ts @@ -1,6 +1,17 @@ import { IncomingMessage } from 'node:http'; import * as k8s from '@kubernetes/client-node'; +export interface KubeConfigOptions { + clusters: k8s.Cluster[]; + contexts: k8s.Context[]; + currentContext: k8s.Context['name']; + users: k8s.User[]; +} + +export type K8sObjectList = + k8s.V1DeploymentList | k8s.V1ServiceList + | k8s.V1JobList | k8s.V1PodList | k8s.V1ReplicaSetList; + export interface K8sConfig { clusterName: string; clusterNameLabel: string; @@ -48,6 +59,15 @@ export interface DeleteApiResponse { body: DeleteResponseBody; } +export type ListParams = [ + string, + string | undefined, + boolean | undefined, + string | undefined, + string | undefined, + string +]; + export type DeleteParams = [ string, string, diff --git a/packages/teraslice/src/lib/cluster/services/cluster/backends/kubernetesV2/k8s.ts b/packages/teraslice/src/lib/cluster/services/cluster/backends/kubernetesV2/k8s.ts index 553216a7401..8c263ff195c 100644 --- a/packages/teraslice/src/lib/cluster/services/cluster/backends/kubernetesV2/k8s.ts +++ b/packages/teraslice/src/lib/cluster/services/cluster/backends/kubernetesV2/k8s.ts @@ -8,23 +8,13 @@ import { isPod, isReplicaSet, isService } from './utils.js'; import { - DeleteApiResponse, DeleteParams, ResourceListApiResponse, - ResourceType, PatchApiResponse, Resource, - ResourceList, ResourceApiResponse, DeleteResponseBody, - NodeType, ScaleOp + DeleteApiResponse, DeleteParams, DeleteResponseBody, + K8sObjectList, KubeConfigOptions, ListParams, + NodeType, PatchApiResponse, Resource, + ResourceApiResponse, ResourceList, ResourceListApiResponse, + ResourceType, ScaleOp } from './interfaces.js'; -interface KubeConfigOptions { - clusters: k8s.Cluster[]; - contexts: k8s.Context[]; - currentContext: k8s.Context['name']; - users: k8s.User[]; -} - -type K8sObjectList = - k8s.V1DeploymentList | k8s.V1ServiceList - | k8s.V1JobList | k8s.V1PodList | k8s.V1ReplicaSetList; - export class K8s { logger: Logger; apiPollDelay: number; @@ -190,14 +180,7 @@ export class K8s { const namespace = ns || this.defaultNamespace; let responseObj: ResourceListApiResponse; - const params: [ - string, - string | undefined, - boolean | undefined, - string | undefined, - string | undefined, - string - ] = [ + const params: ListParams = [ namespace, undefined, undefined, @@ -206,19 +189,38 @@ export class K8s { selector ]; - const listFunctions: { [resource: string]: () => Promise } = { - deployment: () => this.k8sAppsV1Api.listNamespacedDeployment(...params), - job: () => this.k8sBatchV1Api.listNamespacedJob(...params), - pod: () => this.k8sCoreV1Api.listNamespacedPod(...params), - replicaset: () => this.k8sAppsV1Api.listNamespacedReplicaSet(...params), - service: () => this.k8sCoreV1Api.listNamespacedService(...params) - }; - try { - responseObj = await pRetry( - listFunctions[objType], - getRetryConfig() - ); + if (objType === 'deployment') { + responseObj = await pRetry( + () => this.k8sAppsV1Api.listNamespacedDeployment(...params), + getRetryConfig() + ); + } else if (objType === 'job') { + responseObj = await pRetry( + () => this.k8sBatchV1Api.listNamespacedJob(...params), + getRetryConfig() + ); + } else if (objType === 'pod') { + responseObj = await pRetry( + () => this.k8sCoreV1Api.listNamespacedPod(...params), + getRetryConfig() + ); + } else if (objType === 'replicaset') { + responseObj = await pRetry( + () => this.k8sAppsV1Api.listNamespacedReplicaSet(...params), + getRetryConfig() + ); + } else if (objType === 'service') { + responseObj = await pRetry( + () => this.k8sCoreV1Api.listNamespacedService(...params), + getRetryConfig() + ); + } else { + const error = new Error(`Invalid objType provided to get: ${objType}`); + this.logger.error(error); + return Promise.reject(error); + } + return responseObj.body; } catch (e) { const err = new Error(`Request k8s.list of ${objType} with selector ${selector} failed: ${e}`); @@ -246,33 +248,16 @@ export class K8s { * posts manifest to k8s * @param {Resource} manifest resource manifest * @param {ResourceType} manifestType 'deployment', 'job', 'pod', 'replicaset', 'service' - * @return {k8s.V1Deployment - * | k8s.V1Job - * | k8s.V1Pod - * | k8s.V1ReplicaSet - * | k8s.V1Service} body of k8s API response object + * @return {Resource} body of k8s API response object */ - async post(manifest: k8s.V1Deployment, manifestType: 'deployment'): Promise; - async post(manifest: k8s.V1Job, manifestType: 'job'): Promise; - async post(manifest: k8s.V1Pod, manifestType: 'pod'): Promise; - async post(manifest: k8s.V1ReplicaSet, manifestType: 'replicaset'): Promise; - async post(manifest: k8s.V1Service, manifestType: 'service'): Promise; - async post(manifest: Resource, manifestType: ResourceType): Promise { + async post(manifest: k8s.V1Deployment): Promise; + async post(manifest: k8s.V1Job): Promise; + async post(manifest: k8s.V1Pod): Promise; + async post(manifest: k8s.V1ReplicaSet): Promise; + async post(manifest: k8s.V1Service): Promise; + async post(manifest: Resource): Promise { let responseObj: ResourceApiResponse; - // const postFunctions = { - // deployment: async (man: k8s.V1Deployment) => await this.k8sAppsV1Api - // .createNamespacedDeployment(this.defaultNamespace, man), - // job: async (man: k8s.V1Job) => await this.k8sBatchV1Api - // .createNamespacedJob(this.defaultNamespace, man), - // pod: async (man: k8s.V1Pod) => await this.k8sCoreV1Api - // .createNamespacedPod(this.defaultNamespace, man), - // replicaset: async (man: k8s.V1ReplicaSet) => await this.k8sAppsV1Api - // .createNamespacedReplicaSet(this.defaultNamespace, man), - // service: async (man: k8s.V1Service) => await this.k8sCoreV1Api - // .createNamespacedService(this.defaultNamespace, man) - // }; - try { if (isDeployment(manifest)) { responseObj = await this.k8sAppsV1Api @@ -290,13 +275,13 @@ export class K8s { responseObj = await this.k8sCoreV1Api .createNamespacedService(this.defaultNamespace, manifest); } else { - const error = new Error(`Invalid manifestType: ${manifestType}`); + const error = new Error('Invalid manifest type'); return Promise.reject(error); } - // responseObj = await postFunctions[manifestType](manifest); + return responseObj.body; } catch (e) { - const err = new Error(`Request k8s.post of ${manifestType} with body ${JSON.stringify(manifest)} failed: ${e}`); + const err = new Error(`Request k8s.post of ${manifest.kind} with body ${JSON.stringify(manifest)} failed: ${e}`); return Promise.reject(err); } } diff --git a/packages/teraslice/test/lib/cluster/services/cluster/backends/kubernetes/v2/k8s-v2-spec.ts b/packages/teraslice/test/lib/cluster/services/cluster/backends/kubernetes/v2/k8s-v2-spec.ts index 74061cd6150..27329681dfd 100644 --- a/packages/teraslice/test/lib/cluster/services/cluster/backends/kubernetes/v2/k8s-v2-spec.ts +++ b/packages/teraslice/test/lib/cluster/services/cluster/backends/kubernetes/v2/k8s-v2-spec.ts @@ -5,7 +5,6 @@ import nock from 'nock'; import { debugLogger } from '@terascope/job-components'; import { K8s } from '../../../../../../../../src/lib/cluster/services/cluster/backends/kubernetesV2/k8s.js'; -import { V1Service } from '@kubernetes/client-node'; const logger = debugLogger('k8s-v2-spec'); @@ -155,7 +154,7 @@ describe('k8s', () => { .post('/apis/apps/v1/namespaces/default/deployments') .reply(201, { kind: 'Deployment' }); - const response = await k8s.post({ kind: 'Deployment' }, 'deployment'); + const response = await k8s.post({ kind: 'Deployment' }); expect(response.kind).toEqual('Deployment'); }); @@ -164,7 +163,7 @@ describe('k8s', () => { .post('/apis/batch/v1/namespaces/default/jobs') .reply(201, { kind: 'Job' }); - const response = await k8s.post({ kind: 'Job' }, 'job'); + const response = await k8s.post({ kind: 'Job' }); expect(response.kind).toEqual('Job'); }); @@ -173,7 +172,7 @@ describe('k8s', () => { .post('/api/v1/namespaces/default/pods') .reply(201, { kind: 'Pod' }); - const response = await k8s.post({ kind: 'Pod' }, 'pod'); + const response = await k8s.post({ kind: 'Pod' }); expect(response.kind).toEqual('Pod'); }); @@ -182,7 +181,7 @@ describe('k8s', () => { .post('/apis/apps/v1/namespaces/default/replicasets') .reply(201, { kind: 'ReplicaSet' }); - const response = await k8s.post({ kind: 'ReplicaSet' }, 'replicaset'); + const response = await k8s.post({ kind: 'ReplicaSet' }); expect(response.kind).toEqual('ReplicaSet'); }); @@ -191,9 +190,7 @@ describe('k8s', () => { .post('/api/v1/namespaces/default/services') .reply(201, { kind: 'Service' }); - const manifest: V1Service = { kind: 'Service' }; - - const response = await k8s.post(manifest, 'service'); + const response = await k8s.post({ kind: 'Service' }); expect(response.kind).toEqual('Service'); }); }); From 033d56d7be03d7d29a60372134da01c212b732f0 Mon Sep 17 00:00:00 2001 From: busma13 Date: Wed, 30 Oct 2024 16:30:32 -0700 Subject: [PATCH 08/30] remove unneeded check --- .../cluster/services/cluster/backends/kubernetesV2/utils.ts | 4 ---- 1 file changed, 4 deletions(-) diff --git a/packages/teraslice/src/lib/cluster/services/cluster/backends/kubernetesV2/utils.ts b/packages/teraslice/src/lib/cluster/services/cluster/backends/kubernetesV2/utils.ts index c49b8279142..123332ab616 100644 --- a/packages/teraslice/src/lib/cluster/services/cluster/backends/kubernetesV2/utils.ts +++ b/packages/teraslice/src/lib/cluster/services/cluster/backends/kubernetesV2/utils.ts @@ -30,10 +30,6 @@ export function makeTemplate( folder: 'deployment' | 'job' | 'service', fileName: NodeType ): (config: K8sConfig) => k8s.V1Deployment | k8s.V1Job | k8s.V1Service { - const availableTemplates = ['deployment', 'job', 'service']; - if (!availableTemplates.includes(folder)) { - throw new Error(`Unsupported template folder: ${folder}. Available template folders are: deployment, job, service.`); - } const filePath = path.join(resourcePath, folder, `${fileName}.hbs`); const templateData = fs.readFileSync(filePath, 'utf-8'); const templateKeys = ['{{', '}}']; From 4aea696496a4e844efbe6874e24d44ae6e800cdb Mon Sep 17 00:00:00 2001 From: busma13 Date: Thu, 31 Oct 2024 08:41:50 -0700 Subject: [PATCH 09/30] handle undefined k8sConfig fields --- .../backends/kubernetesV2/interfaces.ts | 6 +- .../backends/kubernetesV2/k8sResource.ts | 4 +- .../cluster/backends/kubernetesV2/utils.ts | 8 + .../backends/kubernetes/v2/utils-v2-spec.ts | 438 +++++++++++++----- 4 files changed, 327 insertions(+), 129 deletions(-) diff --git a/packages/teraslice/src/lib/cluster/services/cluster/backends/kubernetesV2/interfaces.ts b/packages/teraslice/src/lib/cluster/services/cluster/backends/kubernetesV2/interfaces.ts index 6fa0451502d..4770f3c23dc 100644 --- a/packages/teraslice/src/lib/cluster/services/cluster/backends/kubernetesV2/interfaces.ts +++ b/packages/teraslice/src/lib/cluster/services/cluster/backends/kubernetesV2/interfaces.ts @@ -16,11 +16,11 @@ export interface K8sConfig { clusterName: string; clusterNameLabel: string; configMapName: string; - dockerImage: string; + dockerImage: string | undefined; execution: string; exId: string; - exName: string; - exUid: string; + exName: string | undefined; + exUid: string | undefined; jobId: string; jobNameLabel: string; name: string; diff --git a/packages/teraslice/src/lib/cluster/services/cluster/backends/kubernetesV2/k8sResource.ts b/packages/teraslice/src/lib/cluster/services/cluster/backends/kubernetesV2/k8sResource.ts index 8b78b2a4a35..e925dced80b 100644 --- a/packages/teraslice/src/lib/cluster/services/cluster/backends/kubernetesV2/k8sResource.ts +++ b/packages/teraslice/src/lib/cluster/services/cluster/backends/kubernetesV2/k8sResource.ts @@ -141,7 +141,7 @@ export class K8sResource { const shutdownTimeoutMs = _.get(this.terasliceConfig, 'shutdown_timeout', 60000); const shutdownTimeoutSeconds = Math.round(shutdownTimeoutMs / 1000); - const config = { + const config: K8sConfig = { // assetsDirectory: _.get(this.terasliceConfig, 'assets_directory', ''), // assetsVolume: _.get(this.terasliceConfig, 'assets_volume', ''), clusterName, @@ -159,7 +159,7 @@ export class K8sResource { nodeType: this.nodeType, replicas: this.execution.workers, shutdownTimeout: shutdownTimeoutSeconds - } as K8sConfig; + }; return config; } diff --git a/packages/teraslice/src/lib/cluster/services/cluster/backends/kubernetesV2/utils.ts b/packages/teraslice/src/lib/cluster/services/cluster/backends/kubernetesV2/utils.ts index 123332ab616..4661785929c 100644 --- a/packages/teraslice/src/lib/cluster/services/cluster/backends/kubernetesV2/utils.ts +++ b/packages/teraslice/src/lib/cluster/services/cluster/backends/kubernetesV2/utils.ts @@ -35,6 +35,14 @@ export function makeTemplate( const templateKeys = ['{{', '}}']; return (config: K8sConfig) => { + if (folder !== 'job' && (config.exName === undefined || config.exUid === undefined)) { + throw new Error(`K8s config requires ${config.exName === undefined ? 'exName' : 'exUid'} to create a ${folder} template`); + } + + if (folder !== 'service' && config.dockerImage === undefined) { + throw new Error(`K8s config requires a dockerImage to create a ${folder} template`); + } + const templated = barbe(templateData, templateKeys, config); return JSON.parse(templated); }; diff --git a/packages/teraslice/test/lib/cluster/services/cluster/backends/kubernetes/v2/utils-v2-spec.ts b/packages/teraslice/test/lib/cluster/services/cluster/backends/kubernetes/v2/utils-v2-spec.ts index f3b60b1cb6d..e3df5aa6f12 100644 --- a/packages/teraslice/test/lib/cluster/services/cluster/backends/kubernetes/v2/utils-v2-spec.ts +++ b/packages/teraslice/test/lib/cluster/services/cluster/backends/kubernetes/v2/utils-v2-spec.ts @@ -6,141 +6,331 @@ import { safeEncode } from '../../../../../../../../src/lib/utils/encoding_utils describe('K8s Utils', () => { describe('->makeTemplate', () => { - it('should be able to support the execution_controller job', () => { - const exJobTemplate = makeTemplate('job', 'execution_controller'); - const config: K8sConfig = { - clusterName: 'teracluster', - configMapName: 'teracluster-worker', - exName: 'ts-exc-example-job-74ab9324-9f67', - exUid: '0512dae5-8ca2-437d-b744-fdc50695fd91', - replicas: 1, - name: 'example', - jobNameLabel: 'example-job', - clusterNameLabel: 'example-cluster', - exId: 'some-ex-id', - jobId: 'some-job-id', - nodeType: 'execution_controller', - namespace: 'some-namespace', - dockerImage: 'some/docker-image', - execution: safeEncode({ example: 'hello' }), - shutdownTimeout: 12345 - }; - const exJob = exJobTemplate(config); - - expect(exJob.kind).toEqual('Job'); - expect(exJob.metadata).toEqual({ - labels: { - 'app.kubernetes.io/name': 'teraslice', - 'app.kubernetes.io/component': config.nodeType, - 'teraslice.terascope.io/exId': config.exId, - 'teraslice.terascope.io/jobId': config.jobId, - 'teraslice.terascope.io/jobName': config.jobNameLabel, - 'app.kubernetes.io/instance': config.clusterNameLabel - }, - name: config.name, - namespace: config.namespace - }); + describe('execution_controller job', () => { + it('should be able to support the execution_controller job', () => { + const exJobTemplate = makeTemplate('job', 'execution_controller'); + const config: K8sConfig = { + clusterName: 'teracluster', + configMapName: 'teracluster-worker', + exName: 'ts-exc-example-job-74ab9324-9f67', + exUid: '0512dae5-8ca2-437d-b744-fdc50695fd91', + replicas: 1, + name: 'example', + jobNameLabel: 'example-job', + clusterNameLabel: 'example-cluster', + exId: 'some-ex-id', + jobId: 'some-job-id', + nodeType: 'execution_controller', + namespace: 'some-namespace', + dockerImage: 'some/docker-image', + execution: safeEncode({ example: 'hello' }), + shutdownTimeout: 12345 + }; + const exJob = exJobTemplate(config); + expect(exJob.kind).toEqual('Job'); + expect(exJob.metadata).toEqual({ + labels: { + 'app.kubernetes.io/name': 'teraslice', + 'app.kubernetes.io/component': config.nodeType, + 'teraslice.terascope.io/exId': config.exId, + 'teraslice.terascope.io/jobId': config.jobId, + 'teraslice.terascope.io/jobName': config.jobNameLabel, + 'app.kubernetes.io/instance': config.clusterNameLabel + }, + name: config.name, + namespace: config.namespace + }); + + expect(exJob.spec?.template.metadata?.labels).toEqual(exJob.metadata?.labels); - expect(exJob.spec?.template.metadata?.labels).toEqual(exJob.metadata?.labels); - - const templateSpec = exJob.spec?.template.spec; - - expect(templateSpec?.containers[0].image).toEqual(config.dockerImage); - expect(templateSpec?.containers[0].name).toEqual(config.name); - expect(templateSpec?.containers[0].env).toEqual([ - { - name: 'NODE_TYPE', - value: config.nodeType - }, - { - name: 'EX', - value: config.execution - }, - { - name: 'POD_IP', - valueFrom: { - fieldRef: { - fieldPath: 'status.podIP' + const templateSpec = exJob.spec?.template.spec; + + expect(templateSpec?.containers[0].image).toEqual(config.dockerImage); + expect(templateSpec?.containers[0].name).toEqual(config.name); + expect(templateSpec?.containers[0].env).toEqual([ + { + name: 'NODE_TYPE', + value: config.nodeType + }, + { + name: 'EX', + value: config.execution + }, + { + name: 'POD_IP', + valueFrom: { + fieldRef: { + fieldPath: 'status.podIP' + } } } - } - ]); - expect(templateSpec?.terminationGracePeriodSeconds).toEqual(config.shutdownTimeout); + ]); + expect(templateSpec?.terminationGracePeriodSeconds).toEqual(config.shutdownTimeout); + }); + + it('should throw error if docker image undefined on config for job', () => { + const exJobTemplate = makeTemplate('job', 'execution_controller'); + const config: K8sConfig = { + clusterName: 'teracluster', + configMapName: 'teracluster-worker', + exName: 'ts-exc-example-job-74ab9324-9f67', + exUid: '0512dae5-8ca2-437d-b744-fdc50695fd91', + replicas: 1, + name: 'example', + jobNameLabel: 'example-job', + clusterNameLabel: 'example-cluster', + exId: 'some-ex-id', + jobId: 'some-job-id', + nodeType: 'execution_controller', + namespace: 'some-namespace', + dockerImage: undefined, + execution: safeEncode({ example: 'hello' }), + shutdownTimeout: 12345 + }; + expect(() => exJobTemplate(config)) + .toThrow('K8s config requires a dockerImage to create a job template'); + }); }); - it('should be able to support the worker deployment', () => { + describe('worker deployment', () => { const workerDeploymentTemplate = makeTemplate('deployment', 'worker'); - const config: K8sConfig = { - clusterName: 'teracluster', - configMapName: 'teracluster-worker', - name: 'example', - jobNameLabel: 'example-job', - clusterNameLabel: 'example-cluster', - exId: 'some-ex-id', - exName: 'example-job-abcd', - exUid: 'UID1', - jobId: 'some-job-id', - nodeType: 'worker', - namespace: 'some-namespace', - dockerImage: 'some/docker-image', - execution: safeEncode({ example: 'hello' }), - replicas: 1, - shutdownTimeout: 12345 - }; - const workerDeployment = workerDeploymentTemplate(config); - - expect(workerDeployment.kind).toEqual('Deployment'); - expect(workerDeployment.metadata).toEqual({ - labels: { - 'app.kubernetes.io/name': 'teraslice', - 'app.kubernetes.io/component': config.nodeType, - 'teraslice.terascope.io/exId': config.exId, - 'teraslice.terascope.io/jobId': config.jobId, - 'teraslice.terascope.io/jobName': config.jobNameLabel, - 'app.kubernetes.io/instance': config.clusterNameLabel - }, - name: config.name, - namespace: config.namespace, - ownerReferences: [ + it('should be able to support the worker deployment', () => { + const config: K8sConfig = { + clusterName: 'teracluster', + configMapName: 'teracluster-worker', + name: 'example', + jobNameLabel: 'example-job', + clusterNameLabel: 'example-cluster', + exId: 'some-ex-id', + exName: 'example-job-abcd', + exUid: 'UID1', + jobId: 'some-job-id', + nodeType: 'worker', + namespace: 'some-namespace', + dockerImage: 'some/docker-image', + execution: safeEncode({ example: 'hello' }), + replicas: 1, + shutdownTimeout: 12345 + }; + const workerDeployment = workerDeploymentTemplate(config); + + expect(workerDeployment.kind).toEqual('Deployment'); + expect(workerDeployment.metadata).toEqual({ + labels: { + 'app.kubernetes.io/name': 'teraslice', + 'app.kubernetes.io/component': config.nodeType, + 'teraslice.terascope.io/exId': config.exId, + 'teraslice.terascope.io/jobId': config.jobId, + 'teraslice.terascope.io/jobName': config.jobNameLabel, + 'app.kubernetes.io/instance': config.clusterNameLabel + }, + name: config.name, + namespace: config.namespace, + ownerReferences: [ + { + apiVersion: 'batch/v1', + blockOwnerDeletion: false, + controller: false, + kind: 'Job', + name: 'example-job-abcd', + uid: 'UID1', + }, + ], + }); + + expect(workerDeployment.spec?.replicas).toEqual(config.replicas); + + const labels = workerDeployment.spec?.template.metadata?.labels; + expect(labels).toEqual(workerDeployment.metadata?.labels); + + const templateSpec = workerDeployment.spec?.template.spec; + + expect(templateSpec?.containers[0].image).toEqual(config.dockerImage); + expect(templateSpec?.containers[0].name).toEqual(config.name); + expect(templateSpec?.containers[0].env).toEqual([ { - apiVersion: 'batch/v1', - blockOwnerDeletion: false, - controller: false, - kind: 'Job', - name: 'example-job-abcd', - uid: 'UID1', + name: 'NODE_TYPE', + value: config.nodeType }, - ], + { + name: 'EX', + value: config.execution + }, + { + name: 'POD_IP', + valueFrom: { + fieldRef: { + fieldPath: 'status.podIP' + } + } + } + ]); + expect(templateSpec?.terminationGracePeriodSeconds).toEqual(config.shutdownTimeout); }); - expect(workerDeployment.spec?.replicas).toEqual(config.replicas); - - const labels = workerDeployment.spec?.template.metadata?.labels; - expect(labels).toEqual(workerDeployment.metadata?.labels); - - const templateSpec = workerDeployment.spec?.template.spec; - - expect(templateSpec?.containers[0].image).toEqual(config.dockerImage); - expect(templateSpec?.containers[0].name).toEqual(config.name); - expect(templateSpec?.containers[0].env).toEqual([ - { - name: 'NODE_TYPE', - value: config.nodeType - }, - { - name: 'EX', - value: config.execution - }, - { - name: 'POD_IP', - valueFrom: { - fieldRef: { - fieldPath: 'status.podIP' - } + it('should throw error if docker image undefined on config for deployment', () => { + const config: K8sConfig = { + clusterName: 'teracluster', + configMapName: 'teracluster-worker', + name: 'example', + jobNameLabel: 'example-job', + clusterNameLabel: 'example-cluster', + exId: 'some-ex-id', + exName: 'example-job-abcd', + exUid: 'UID1', + jobId: 'some-job-id', + nodeType: 'worker', + namespace: 'some-namespace', + dockerImage: undefined, + execution: safeEncode({ example: 'hello' }), + replicas: 1, + shutdownTimeout: 12345 + }; + expect(() => workerDeploymentTemplate(config)) + .toThrow('K8s config requires a dockerImage to create a deployment template'); + }); + + it('should throw error if exName undefined on config for deployment', () => { + const config: K8sConfig = { + clusterName: 'teracluster', + configMapName: 'teracluster-worker', + name: 'example', + jobNameLabel: 'example-job', + clusterNameLabel: 'example-cluster', + exId: 'some-ex-id', + exName: undefined, + exUid: 'UID1', + jobId: 'some-job-id', + nodeType: 'worker', + namespace: 'some-namespace', + dockerImage: 'some/docker-image', + execution: safeEncode({ example: 'hello' }), + replicas: 1, + shutdownTimeout: 12345 + }; + expect(() => workerDeploymentTemplate(config)).toThrow('K8s config requires exName to create a deployment template'); + }); + + it('should throw error if exUid undefined on config for deployment', () => { + const config: K8sConfig = { + clusterName: 'teracluster', + configMapName: 'teracluster-worker', + name: 'example', + jobNameLabel: 'example-job', + clusterNameLabel: 'example-cluster', + exId: 'some-ex-id', + exName: 'example-job-abcd', + exUid: undefined, + jobId: 'some-job-id', + nodeType: 'worker', + namespace: 'some-namespace', + dockerImage: 'some/docker-image', + execution: safeEncode({ example: 'hello' }), + replicas: 1, + shutdownTimeout: 12345 + }; + expect(() => workerDeploymentTemplate(config)).toThrow('K8s config requires exUid to create a deployment template'); + }); + }); + + describe('execution_controller service', () => { + const exServiceTemplate = makeTemplate('service', 'execution_controller'); + it('should be able to support the execution_controller service', () => { + const config: K8sConfig = { + clusterName: 'teracluster', + configMapName: 'teracluster-exService', + name: 'example', + jobNameLabel: 'example-job', + clusterNameLabel: 'example-cluster', + exId: 'some-ex-id', + exName: 'example-job-abcd', + exUid: 'UID1', + jobId: 'some-job-id', + nodeType: 'execution_controller', + namespace: 'some-namespace', + dockerImage: 'some/docker-image', + execution: safeEncode({ example: 'hello' }), + replicas: 1, + shutdownTimeout: 12345 + }; + const exService = exServiceTemplate(config); + + expect(exService.kind).toEqual('Service'); + expect(exService.metadata).toEqual({ + labels: { + 'app.kubernetes.io/name': 'teraslice', + 'app.kubernetes.io/component': config.nodeType, + 'teraslice.terascope.io/exId': config.exId, + 'teraslice.terascope.io/jobId': config.jobId + }, + name: `svc-${config.name}`, + namespace: config.namespace, + ownerReferences: [ + { + apiVersion: 'batch/v1', + blockOwnerDeletion: false, + controller: false, + kind: 'Job', + name: 'example-job-abcd', + uid: 'UID1', + }, + ], + }); + + expect(exService.spec?.selector).toEqual({ + 'app.kubernetes.io/component': 'execution_controller', + 'teraslice.terascope.io/exId': config.exId + }); + + expect(exService.spec?.ports).toEqual([ + { + port: 45680, + targetPort: 45680 } - } - ]); - expect(templateSpec?.terminationGracePeriodSeconds).toEqual(config.shutdownTimeout); + ]); + }); + + it('should throw error if exName undefined on config for service', () => { + const config: K8sConfig = { + clusterName: 'teracluster', + configMapName: 'teracluster-exService', + name: 'example', + jobNameLabel: 'example-job', + clusterNameLabel: 'example-cluster', + exId: 'some-ex-id', + exName: undefined, + exUid: 'UID1', + jobId: 'some-job-id', + nodeType: 'execution_controller', + namespace: 'some-namespace', + dockerImage: 'some/docker-image', + execution: safeEncode({ example: 'hello' }), + replicas: 1, + shutdownTimeout: 12345 + }; + expect(() => exServiceTemplate(config)).toThrow('K8s config requires exName to create a service template'); + }); + + it('should throw error if exUid undefined on config for service', () => { + const config: K8sConfig = { + clusterName: 'teracluster', + configMapName: 'teracluster-exService', + name: 'example', + jobNameLabel: 'example-job', + clusterNameLabel: 'example-cluster', + exId: 'some-ex-id', + exName: 'example-job-abcd', + exUid: undefined, + jobId: 'some-job-id', + nodeType: 'execution_controller', + namespace: 'some-namespace', + dockerImage: 'some/docker-image', + execution: safeEncode({ example: 'hello' }), + replicas: 1, + shutdownTimeout: 12345 + }; + expect(() => exServiceTemplate(config)).toThrow('K8s config requires exUid to create a service template'); + }); }); }); From 5e45e0e340cae352535aa6fbe933723ffc126631 Mon Sep 17 00:00:00 2001 From: busma13 Date: Thu, 31 Oct 2024 09:36:12 -0700 Subject: [PATCH 10/30] pass required parameters to tests --- .../cluster/backends/kubernetesV2/k8s.ts | 2 +- .../backends/kubernetesV2/k8sResource.ts | 6 +- .../kubernetes/v2/k8sResource-v2-spec.ts | 72 +++++++++---------- 3 files changed, 40 insertions(+), 40 deletions(-) diff --git a/packages/teraslice/src/lib/cluster/services/cluster/backends/kubernetesV2/k8s.ts b/packages/teraslice/src/lib/cluster/services/cluster/backends/kubernetesV2/k8s.ts index 8c263ff195c..66bfd67879a 100644 --- a/packages/teraslice/src/lib/cluster/services/cluster/backends/kubernetesV2/k8s.ts +++ b/packages/teraslice/src/lib/cluster/services/cluster/backends/kubernetesV2/k8s.ts @@ -162,7 +162,7 @@ export class K8s { * returns list of k8s objects matching provided selector * @param {String} selector kubernetes selector, like 'app=teraslice' * @param {ResourceType} objType Type of k8s object to get, valid options: - * 'deployments', 'jobs', 'pods', 'replicasets', 'services' + * 'deployment', 'job', 'pod', 'replicaset', 'service' * @param {String} ns namespace to search, this will override the default * @return {k8s.V1PodList * | k8s.V1DeploymentList diff --git a/packages/teraslice/src/lib/cluster/services/cluster/backends/kubernetesV2/k8sResource.ts b/packages/teraslice/src/lib/cluster/services/cluster/backends/kubernetesV2/k8sResource.ts index e925dced80b..362301f7fa8 100644 --- a/packages/teraslice/src/lib/cluster/services/cluster/backends/kubernetesV2/k8sResource.ts +++ b/packages/teraslice/src/lib/cluster/services/cluster/backends/kubernetesV2/k8sResource.ts @@ -1,7 +1,7 @@ import _ from 'lodash'; import * as k8s from '@kubernetes/client-node'; import { isNumber, Logger } from '@terascope/utils'; -import type { TerasliceConfig, ExecutionConfig } from '@terascope/job-components'; +import type { Config, ExecutionConfig } from '@terascope/types'; import { safeEncode } from '../../../../../utils/encoding_utils.js'; import { isService, makeTemplate, setMaxOldSpaceViaEnv } from './utils.js'; import { K8sConfig, NodeType } from './interfaces.js'; @@ -13,7 +13,7 @@ export class K8sResource { logger: Logger; nodeType: NodeType; nameInfix: string; - terasliceConfig: TerasliceConfig; + terasliceConfig: Config; templateGenerator: (config: K8sConfig) => k8s.V1Deployment | k8s.V1Job | k8s.V1Service; templateConfig: K8sConfig; resource: k8s.V1Deployment | k8s.V1Job | k8s.V1Service; @@ -35,7 +35,7 @@ export class K8sResource { constructor( resourceType: 'deployment' | 'job' | 'service', resourceName: NodeType, - terasliceConfig: TerasliceConfig, + terasliceConfig: Config, execution: ExecutionConfig, logger: Logger, exName?: string, diff --git a/packages/teraslice/test/lib/cluster/services/cluster/backends/kubernetes/v2/k8sResource-v2-spec.ts b/packages/teraslice/test/lib/cluster/services/cluster/backends/kubernetes/v2/k8sResource-v2-spec.ts index b4bf015f26d..bb0adaa4da5 100644 --- a/packages/teraslice/test/lib/cluster/services/cluster/backends/kubernetes/v2/k8sResource-v2-spec.ts +++ b/packages/teraslice/test/lib/cluster/services/cluster/backends/kubernetes/v2/k8sResource-v2-spec.ts @@ -38,7 +38,7 @@ describe('k8sResource', () => { describe('worker deployment', () => { it('has valid resource object.', () => { - const kr = new K8sResource('deployment', 'worker', terasliceConfig, execution, logger); + const kr = new K8sResource('deployment', 'worker', terasliceConfig, execution, logger, 'example-job-abcd', 'UID1'); const resource = kr.resource as k8s.V1Deployment; expect(resource.kind).toBe('Deployment'); @@ -76,7 +76,7 @@ describe('k8sResource', () => { it('has valid resource object when terasliceConfig has kubernetes_image_pull_secret.', () => { terasliceConfig.kubernetes_image_pull_secret = 'teraslice-image-pull-secret'; - const kr = new K8sResource('deployment', 'worker', terasliceConfig, execution, logger); + const kr = new K8sResource('deployment', 'worker', terasliceConfig, execution, logger, 'example-job-abcd', 'UID1'); const resource = kr.resource as k8s.V1Deployment; let firstSecret: k8s.V1LocalObjectReference | undefined = undefined; @@ -92,7 +92,7 @@ describe('k8sResource', () => { it('has podAntiAffinity when terasliceConfig has kubernetes_worker_antiaffinity true.', () => { terasliceConfig.kubernetes_worker_antiaffinity = true; - const kr = new K8sResource('deployment', 'worker', terasliceConfig, execution, logger); + const kr = new K8sResource('deployment', 'worker', terasliceConfig, execution, logger, 'example-job-abcd', 'UID1'); const resource = kr.resource as k8s.V1Deployment; // console.log(yaml.dump(kr.resource.spec.template.spec.affinity)); @@ -120,7 +120,7 @@ describe('k8sResource', () => { terasliceConfig.assets_directory = '/assets'; terasliceConfig.assets_volume = 'asset-volume'; - const kr = new K8sResource('deployment', 'worker', terasliceConfig, execution, logger); + const kr = new K8sResource('deployment', 'worker', terasliceConfig, execution, logger, 'example-job-abcd', 'UID1'); const resource = kr.resource as k8s.V1Deployment; let configVolume: k8s.V1Volume | undefined = undefined; @@ -177,7 +177,7 @@ describe('k8sResource', () => { { name: 'teraslice-data1', path: '/data' } ]; - const kr = new K8sResource('deployment', 'worker', terasliceConfig, execution, logger); + const kr = new K8sResource('deployment', 'worker', terasliceConfig, execution, logger, 'example-job-abcd', 'UID1'); const resource = kr.resource as k8s.V1Deployment; const configVolume1 = resource.spec?.template.spec?.volumes @@ -227,7 +227,7 @@ describe('k8sResource', () => { { name: 'tmp', path: '/tmp' } ]; - const kr = new K8sResource('deployment', 'worker', terasliceConfig, execution, logger); + const kr = new K8sResource('deployment', 'worker', terasliceConfig, execution, logger, 'example-job-abcd', 'UID1'); const resource = kr.resource as k8s.V1Deployment; const configVolume2 = resource.spec?.template.spec?.volumes @@ -267,7 +267,7 @@ describe('k8sResource', () => { }); it('does not have memory/cpu limits/requests when not set in config or execution', () => { - const kr = new K8sResource('deployment', 'worker', terasliceConfig, execution, logger); + const kr = new K8sResource('deployment', 'worker', terasliceConfig, execution, logger, 'example-job-abcd', 'UID1'); const resource = kr.resource as k8s.V1Deployment; const exId = resource.metadata?.labels ? resource.metadata.labels['teraslice.terascope.io/exId'] : undefined; @@ -285,7 +285,7 @@ describe('k8sResource', () => { terasliceConfig.cpu = 1; terasliceConfig.memory = 2147483648; - const kr = new K8sResource('deployment', 'worker', terasliceConfig, execution, logger); + const kr = new K8sResource('deployment', 'worker', terasliceConfig, execution, logger, 'example-job-abcd', 'UID1'); const resource = kr.resource as k8s.V1Deployment; const exId = resource.metadata?.labels ? resource.metadata.labels['teraslice.terascope.io/exId'] : undefined; @@ -308,7 +308,7 @@ describe('k8sResource', () => { }); it('has the ability to set custom env', () => { - const kr = new K8sResource('deployment', 'worker', terasliceConfig, execution, logger); + const kr = new K8sResource('deployment', 'worker', terasliceConfig, execution, logger, 'example-job-abcd', 'UID1'); const resource = kr.resource as k8s.V1Deployment; // NOTE: the env var merge happens in _setResources(), which is @@ -330,7 +330,7 @@ describe('k8sResource', () => { terasliceConfig.cpu = 1; terasliceConfig.memory = 2147483648; - const kr = new K8sResource('deployment', 'worker', terasliceConfig, execution, logger); + const kr = new K8sResource('deployment', 'worker', terasliceConfig, execution, logger, 'example-job-abcd', 'UID1'); const resource = kr.resource as k8s.V1Deployment; const exId = resource.metadata?.labels ? resource.metadata.labels['teraslice.terascope.io/exId'] : undefined; @@ -355,7 +355,7 @@ describe('k8sResource', () => { terasliceConfig.cpu = 1; terasliceConfig.memory = 2147483648; - const kr = new K8sResource('deployment', 'worker', terasliceConfig, execution, logger); + const kr = new K8sResource('deployment', 'worker', terasliceConfig, execution, logger, 'example-job-abcd', 'UID1'); const resource = kr.resource as k8s.V1Deployment; const exId = resource.metadata?.labels ? resource.metadata.labels['teraslice.terascope.io/exId'] : undefined; @@ -379,7 +379,7 @@ describe('k8sResource', () => { execution.cpu = 1; execution.memory = 2147483648; - const kr = new K8sResource('deployment', 'worker', terasliceConfig, execution, logger); + const kr = new K8sResource('deployment', 'worker', terasliceConfig, execution, logger, 'example-job-abcd', 'UID1'); const resource = kr.resource as k8s.V1Deployment; const exId = resource.metadata?.labels ? resource.metadata.labels['teraslice.terascope.io/exId'] : undefined; @@ -405,7 +405,7 @@ describe('k8sResource', () => { execution.resources_requests_memory = 2147483648; execution.resources_limits_memory = 3147483648; - const kr = new K8sResource('deployment', 'worker', terasliceConfig, execution, logger); + const kr = new K8sResource('deployment', 'worker', terasliceConfig, execution, logger, 'example-job-abcd', 'UID1'); const resource = kr.resource as k8s.V1Deployment; const exId = resource.metadata?.labels ? resource.metadata.labels['teraslice.terascope.io/exId'] : undefined; @@ -428,7 +428,7 @@ describe('k8sResource', () => { it('has memory limits and requests when set on execution', () => { execution.memory = 2147483648; - const kr = new K8sResource('deployment', 'worker', terasliceConfig, execution, logger); + const kr = new K8sResource('deployment', 'worker', terasliceConfig, execution, logger, 'example-job-abcd', 'UID1'); const resource = kr.resource as k8s.V1Deployment; expect(resource.spec?.template.spec?.containers[0].resources).toEqual(yaml.load(` @@ -446,7 +446,7 @@ describe('k8sResource', () => { it('has cpu limits and requests when set on execution', () => { execution.cpu = 1; - const kr = new K8sResource('deployment', 'worker', terasliceConfig, execution, logger); + const kr = new K8sResource('deployment', 'worker', terasliceConfig, execution, logger, 'example-job-abcd', 'UID1'); const resource = kr.resource as k8s.V1Deployment; expect(resource.spec?.template.spec?.containers[0].resources).toEqual(yaml.load(` @@ -459,7 +459,7 @@ describe('k8sResource', () => { it('has scratch volume when ephemeral_storage is set true on execution', () => { execution.ephemeral_storage = true; - const kr = new K8sResource('deployment', 'worker', terasliceConfig, execution, logger); + const kr = new K8sResource('deployment', 'worker', terasliceConfig, execution, logger, 'example-job-abcd', 'UID1'); const resource = kr.resource as k8s.V1Deployment; expect(resource.spec?.template.spec?.containers[0].volumeMounts) @@ -474,7 +474,7 @@ describe('k8sResource', () => { it('does not have scratch volume when ephemeral_storage is set false on execution', () => { execution.ephemeral_storage = false; - const kr = new K8sResource('deployment', 'worker', terasliceConfig, execution, logger); + const kr = new K8sResource('deployment', 'worker', terasliceConfig, execution, logger, 'example-job-abcd', 'UID1'); const resource = kr.resource as k8s.V1Deployment; expect(resource.spec?.template.spec?.containers[0].volumeMounts) @@ -485,7 +485,7 @@ describe('k8sResource', () => { describe('worker deployments with targets', () => { it('does not have affinity or toleration properties when targets equals [].', () => { execution.targets = []; - const kr = new K8sResource('deployment', 'worker', terasliceConfig, execution, logger); + const kr = new K8sResource('deployment', 'worker', terasliceConfig, execution, logger, 'example-job-abcd', 'UID1'); const resource = kr.resource as k8s.V1Deployment; expect(resource.spec?.template.spec).not.toHaveProperty('affinity'); @@ -496,7 +496,7 @@ describe('k8sResource', () => { execution.targets = [ { key: 'zone', value: 'west' } ]; - const kr = new K8sResource('deployment', 'worker', terasliceConfig, execution, logger); + const kr = new K8sResource('deployment', 'worker', terasliceConfig, execution, logger, 'example-job-abcd', 'UID1'); const resource = kr.resource as k8s.V1Deployment; expect(resource.spec?.template.spec?.affinity).toEqual(yaml.load(` @@ -515,7 +515,7 @@ describe('k8sResource', () => { { key: 'zone', value: 'west' } ]; terasliceConfig.kubernetes_worker_antiaffinity = true; - const kr = new K8sResource('deployment', 'worker', terasliceConfig, execution, logger); + const kr = new K8sResource('deployment', 'worker', terasliceConfig, execution, logger, 'example-job-abcd', 'UID1'); const resource = kr.resource as k8s.V1Deployment; expect(resource.spec?.template.spec?.affinity).toEqual(yaml.load(` @@ -548,7 +548,7 @@ describe('k8sResource', () => { execution.targets = [ { key: 'zone', value: 'west', constraint: 'required' } ]; - const kr = new K8sResource('deployment', 'worker', terasliceConfig, execution, logger); + const kr = new K8sResource('deployment', 'worker', terasliceConfig, execution, logger, 'example-job-abcd', 'UID1'); const resource = kr.resource as k8s.V1Deployment; expect(resource.spec?.template.spec?.affinity).toEqual(yaml.load(` @@ -567,7 +567,7 @@ describe('k8sResource', () => { { key: 'zone', value: 'west', constraint: 'required' }, { key: 'region', value: '42', constraint: 'required' } ]; - const kr = new K8sResource('deployment', 'worker', terasliceConfig, execution, logger); + const kr = new K8sResource('deployment', 'worker', terasliceConfig, execution, logger, 'example-job-abcd', 'UID1'); const resource = kr.resource as k8s.V1Deployment; expect(resource.spec?.template.spec?.affinity).toEqual(yaml.load(` @@ -589,7 +589,7 @@ describe('k8sResource', () => { execution.targets = [ { key: 'zone', value: 'west', constraint: 'preferred' } ]; - const kr = new K8sResource('deployment', 'worker', terasliceConfig, execution, logger); + const kr = new K8sResource('deployment', 'worker', terasliceConfig, execution, logger, 'example-job-abcd', 'UID1'); const resource = kr.resource as k8s.V1Deployment; expect(resource.spec?.template.spec?.affinity).toEqual(yaml.load(` @@ -609,7 +609,7 @@ describe('k8sResource', () => { { key: 'zone', value: 'west', constraint: 'preferred' }, { key: 'region', value: 'texas', constraint: 'preferred' } ]; - const kr = new K8sResource('deployment', 'worker', terasliceConfig, execution, logger); + const kr = new K8sResource('deployment', 'worker', terasliceConfig, execution, logger, 'example-job-abcd', 'UID1'); const resource = kr.resource as k8s.V1Deployment; expect(resource.spec?.template.spec?.affinity).toEqual(yaml.load(` @@ -635,7 +635,7 @@ describe('k8sResource', () => { execution.targets = [ { key: 'zone', value: 'west', constraint: 'accepted' } ]; - const kr = new K8sResource('deployment', 'worker', terasliceConfig, execution, logger); + const kr = new K8sResource('deployment', 'worker', terasliceConfig, execution, logger, 'example-job-abcd', 'UID1'); const resource = kr.resource as k8s.V1Deployment; // console.log(yaml.dump(kr.resource.spec.template.spec.tolerations)); @@ -651,7 +651,7 @@ describe('k8sResource', () => { { key: 'zone', value: 'west', constraint: 'accepted' }, { key: 'region', value: 'texas', constraint: 'accepted' } ]; - const kr = new K8sResource('deployment', 'worker', terasliceConfig, execution, logger); + const kr = new K8sResource('deployment', 'worker', terasliceConfig, execution, logger, 'example-job-abcd', 'UID1'); const resource = kr.resource as k8s.V1Deployment; // console.log(yaml.dump(kr.resource.spec.template.spec.tolerations)); @@ -671,7 +671,7 @@ describe('k8sResource', () => { { key: 'zone', value: 'west', constraint: 'required' }, { key: 'region', value: 'texas', constraint: 'preferred' } ]; - const kr = new K8sResource('deployment', 'worker', terasliceConfig, execution, logger); + const kr = new K8sResource('deployment', 'worker', terasliceConfig, execution, logger, 'example-job-abcd', 'UID1'); const resource = kr.resource as k8s.V1Deployment; // console.log(yaml.dump(kr.resource.spec.template.spec.tolerations)); @@ -703,7 +703,7 @@ describe('k8sResource', () => { { key: 'zone', value: 'west', constraint: 'accepted' }, { key: 'region', value: 'texas', constraint: 'accepted' } ]; - const kr = new K8sResource('deployment', 'worker', terasliceConfig, execution, logger); + const kr = new K8sResource('deployment', 'worker', terasliceConfig, execution, logger, 'example-job-abcd', 'UID1'); const resource = kr.resource as k8s.V1Deployment; expect(resource.spec?.template.spec?.affinity).toEqual(yaml.load(` @@ -753,7 +753,7 @@ describe('k8sResource', () => { key2: 'value2' }; - const kr = new K8sResource('deployment', 'worker', terasliceConfig, execution, logger); + const kr = new K8sResource('deployment', 'worker', terasliceConfig, execution, logger, 'example-job-abcd', 'UID1'); const resource = kr.resource as k8s.V1Deployment; // console.log(yaml.dump(kr.resource)); @@ -770,7 +770,7 @@ describe('k8sResource', () => { abcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghij1234: 'value3', }; - const kr = new K8sResource('deployment', 'worker', terasliceConfig, execution, logger); + const kr = new K8sResource('deployment', 'worker', terasliceConfig, execution, logger, 'example-job-abcd', 'UID1'); const resource = kr.resource as k8s.V1Deployment; // console.log(yaml.dump(kr.resource)); @@ -792,7 +792,7 @@ describe('k8sResource', () => { describe('teraslice job with one valid external_ports set', () => { it('generates k8s worker deployment with containerPort on container', () => { execution.external_ports = [9090]; - const kr = new K8sResource('deployment', 'worker', terasliceConfig, execution, logger); + const kr = new K8sResource('deployment', 'worker', terasliceConfig, execution, logger, 'example-job-abcd', 'UID1'); const resource = kr.resource as k8s.V1Deployment; // console.log(yaml.dump(kr.resource.spec.template.spec.containers[0].ports)); @@ -823,7 +823,7 @@ describe('k8sResource', () => { 9090, { name: 'metrics', port: 9091 } ]; - const kr = new K8sResource('deployment', 'worker', terasliceConfig, execution, logger); + const kr = new K8sResource('deployment', 'worker', terasliceConfig, execution, logger, 'example-job-abcd', 'UID1'); const resource = kr.resource as k8s.V1Deployment; // console.log(yaml.dump(kr.resource.spec.template.spec.containers[0].ports)); @@ -944,7 +944,7 @@ describe('k8sResource', () => { ['teraslice-JOB-name', 'ts-wkr-teraslice-job-name-7ba9afb0-417a'] ])('when Job Name is %s the k8s worker name is: %s', (jobName, k8sName) => { execution.name = jobName; - const kr = new K8sResource('deployment', 'worker', terasliceConfig, execution, logger); + const kr = new K8sResource('deployment', 'worker', terasliceConfig, execution, logger, 'example-job-abcd', 'UID1'); const resource = kr.resource as k8s.V1Deployment; expect(resource.metadata?.name).toBe(k8sName); @@ -1044,7 +1044,7 @@ describe('k8sResource', () => { execution.stateful = true; terasliceConfig.kubernetes_priority_class_name = 'testPriorityClass'; - const krWorker = new K8sResource('deployment', 'worker', terasliceConfig, execution, logger); + const krWorker = new K8sResource('deployment', 'worker', terasliceConfig, execution, logger, 'example-job-abcd', 'UID1'); const wkrResource = krWorker.resource as k8s.V1Deployment; expect(wkrResource.kind).toBe('Deployment'); @@ -1077,7 +1077,7 @@ describe('k8sResource', () => { execution.pod_spec_override = { initContainers: [] }; terasliceConfig.kubernetes_overrides_enabled = false; - const krWorker = new K8sResource('deployment', 'worker', terasliceConfig, execution, logger); + const krWorker = new K8sResource('deployment', 'worker', terasliceConfig, execution, logger, 'example-job-abcd', 'UID1'); const resource = krWorker.resource as k8s.V1Deployment; expect(resource.spec?.template.spec).not.toHaveProperty('initContainers'); @@ -1089,7 +1089,7 @@ describe('k8sResource', () => { execution.pod_spec_override = { initContainers: [] }; terasliceConfig.kubernetes_overrides_enabled = true; - const krWorker = new K8sResource('deployment', 'worker', terasliceConfig, execution, logger); + const krWorker = new K8sResource('deployment', 'worker', terasliceConfig, execution, logger, 'example-job-abcd', 'UID1'); const resource = krWorker.resource as k8s.V1Deployment; expect(resource.spec?.template.spec).toHaveProperty('initContainers'); From c5fa86c24bcc78eecde52735349eeea8ea6f0201 Mon Sep 17 00:00:00 2001 From: busma13 Date: Thu, 31 Oct 2024 11:28:06 -0700 Subject: [PATCH 11/30] refactor delete fn --- .../cluster/backends/kubernetesV2/k8s.ts | 33 +++++++++++-------- 1 file changed, 20 insertions(+), 13 deletions(-) diff --git a/packages/teraslice/src/lib/cluster/services/cluster/backends/kubernetesV2/k8s.ts b/packages/teraslice/src/lib/cluster/services/cluster/backends/kubernetesV2/k8s.ts index 66bfd67879a..d565551f47b 100644 --- a/packages/teraslice/src/lib/cluster/services/cluster/backends/kubernetesV2/k8s.ts +++ b/packages/teraslice/src/lib/cluster/services/cluster/backends/kubernetesV2/k8s.ts @@ -323,7 +323,7 @@ export class K8s { * Deletes k8s object of specified objType * @param {String} name Name of the resource to delete * @param {ResourceType} objType Type of k8s object to get, valid options: - * 'deployments', 'services', 'jobs', 'pods', 'replicasets' + * 'deployment', 'service', 'job', 'pod', 'replicaset' * @param {Boolean} force Forcefully delete resource by setting gracePeriodSeconds to 1 * to be forcefully stopped. * @return {Object} k8s delete response body. @@ -373,14 +373,6 @@ export class K8s { deleteOptions ]; - const deleteFunctions: { [resource: string]: () => Promise } = { - deployment: () => this.k8sAppsV1Api.deleteNamespacedDeployment(...params), - job: () => this.k8sBatchV1Api.deleteNamespacedJob(...params), - pod: () => this.k8sCoreV1Api.deleteNamespacedPod(...params), - replicaset: () => this.k8sAppsV1Api.deleteNamespacedReplicaSet(...params), - service: () => this.k8sCoreV1Api.deleteNamespacedService(...params) - }; - const deleteWithErrorHandling = async (deleteFn: () => Promise) => { try { const res = await deleteFn(); @@ -398,10 +390,25 @@ export class K8s { }; try { - responseObj = await pRetry( - () => deleteWithErrorHandling(deleteFunctions[objType]), - getRetryConfig() - ); + if (objType === 'service') { + responseObj = await pRetry(() => deleteWithErrorHandling(() => this.k8sCoreV1Api + .deleteNamespacedService(...params)), getRetryConfig()); + } else if (objType === 'deployment') { + responseObj = await pRetry(() => deleteWithErrorHandling(() => this.k8sAppsV1Api + .deleteNamespacedDeployment(...params)), getRetryConfig()); + } else if (objType === 'job') { + responseObj = await pRetry(() => deleteWithErrorHandling(() => this.k8sBatchV1Api + .deleteNamespacedJob(...params)), getRetryConfig()); + } else if (objType === 'pod') { + responseObj = await pRetry(() => deleteWithErrorHandling(() => this.k8sCoreV1Api + .deleteNamespacedPod(...params)), getRetryConfig()); + } else if (objType === 'replicaset') { + responseObj = await pRetry(() => deleteWithErrorHandling(() => this.k8sAppsV1Api + .deleteNamespacedReplicaSet(...params)), getRetryConfig()); + } else { + throw new Error(`Invalid objType: ${objType}`); + } + return responseObj.body; } catch (e) { const err = new Error(`Request k8s.delete with name: ${name} failed with: ${e}`); From 374333428fee66a5cfa8f3f53f5f50e9e37f8f09 Mon Sep 17 00:00:00 2001 From: busma13 Date: Thu, 31 Oct 2024 14:59:43 -0700 Subject: [PATCH 12/30] remove retryable --- .../cluster/services/cluster/backends/kubernetesV2/k8s.ts | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/packages/teraslice/src/lib/cluster/services/cluster/backends/kubernetesV2/k8s.ts b/packages/teraslice/src/lib/cluster/services/cluster/backends/kubernetesV2/k8s.ts index d565551f47b..8bc6b385007 100644 --- a/packages/teraslice/src/lib/cluster/services/cluster/backends/kubernetesV2/k8s.ts +++ b/packages/teraslice/src/lib/cluster/services/cluster/backends/kubernetesV2/k8s.ts @@ -532,13 +532,11 @@ export class K8s { // the selector provided to list above should always result in a single // deployment in the response. if (listResponse.items.length === 0) { - const msg = `Teraslice deployment matching the following selector was not found: ${selector} (retriable)`; + const msg = `Teraslice deployment matching the following selector was not found: ${selector}`; this.logger.warn(msg); - throw new TSError(msg, { retryable: true }); + throw new TSError(msg); } else if (listResponse.items.length > 1) { - throw new TSError(`Unexpected number of Teraslice deployments matching the following selector: ${selector}`, { - retryable: true - }); + throw new TSError(`Unexpected number of Teraslice deployments matching the following selector: ${selector}`); } const workerDeployment = listResponse.items[0]; if (workerDeployment.spec?.replicas === undefined) { From dc4b90d14e1446ca5a9c35b2e21f5bd5e46eeb9b Mon Sep 17 00:00:00 2001 From: busma13 Date: Fri, 1 Nov 2024 09:30:53 -0700 Subject: [PATCH 13/30] make resourceTypes plural --- .../{deployment => deployments}/worker.hbs | 0 .../cluster/backends/kubernetesV2/index.ts | 10 +-- .../backends/kubernetesV2/interfaces.ts | 2 +- .../{job => jobs}/execution_controller.hbs | 0 .../cluster/backends/kubernetesV2/k8s.ts | 58 ++++++------ .../backends/kubernetesV2/k8sResource.ts | 2 +- .../execution_controller.hbs | 0 .../cluster/backends/kubernetesV2/utils.ts | 14 +-- .../backends/kubernetes/v2/k8s-v2-spec.ts | 34 +++---- .../kubernetes/v2/k8sResource-v2-spec.ts | 88 +++++++++---------- .../backends/kubernetes/v2/utils-v2-spec.ts | 8 +- 11 files changed, 108 insertions(+), 108 deletions(-) rename packages/teraslice/src/lib/cluster/services/cluster/backends/kubernetesV2/{deployment => deployments}/worker.hbs (100%) rename packages/teraslice/src/lib/cluster/services/cluster/backends/kubernetesV2/{job => jobs}/execution_controller.hbs (100%) rename packages/teraslice/src/lib/cluster/services/cluster/backends/kubernetesV2/{service => services}/execution_controller.hbs (100%) diff --git a/packages/teraslice/src/lib/cluster/services/cluster/backends/kubernetesV2/deployment/worker.hbs b/packages/teraslice/src/lib/cluster/services/cluster/backends/kubernetesV2/deployments/worker.hbs similarity index 100% rename from packages/teraslice/src/lib/cluster/services/cluster/backends/kubernetesV2/deployment/worker.hbs rename to packages/teraslice/src/lib/cluster/services/cluster/backends/kubernetesV2/deployments/worker.hbs diff --git a/packages/teraslice/src/lib/cluster/services/cluster/backends/kubernetesV2/index.ts b/packages/teraslice/src/lib/cluster/services/cluster/backends/kubernetesV2/index.ts index 7e9b348572a..80391224f0b 100644 --- a/packages/teraslice/src/lib/cluster/services/cluster/backends/kubernetesV2/index.ts +++ b/packages/teraslice/src/lib/cluster/services/cluster/backends/kubernetesV2/index.ts @@ -67,7 +67,7 @@ export class KubernetesClusterBackendV2 { * @return {Promise} void */ private async _getClusterState() { - return this.k8s.list(`app.kubernetes.io/name=teraslice,app.kubernetes.io/instance=${this.clusterNameLabel}`, 'pod') + return this.k8s.list(`app.kubernetes.io/name=teraslice,app.kubernetes.io/instance=${this.clusterNameLabel}`, 'pods') .then((k8sPods) => gen(k8sPods, this.clusterState, this.logger)) .catch((err) => { // TODO: We might need to do more here. I think it's OK to just @@ -106,7 +106,7 @@ export class KubernetesClusterBackendV2 { execution.slicer_port = 45680; const exJobResource = new K8sResource( - 'job', + 'jobs', 'execution_controller', this.context.sysconfig.teraslice, execution, @@ -124,7 +124,7 @@ export class KubernetesClusterBackendV2 { const jobResult = await this.k8s.post(exJob); const exServiceResource = new K8sResource( - 'service', + 'services', 'execution_controller', this.context.sysconfig.teraslice, execution, @@ -200,7 +200,7 @@ export class KubernetesClusterBackendV2 { ); const kr = new K8sResource( - 'deployment', + 'deployments', 'worker', this.context.sysconfig.teraslice, execution, @@ -271,7 +271,7 @@ export class KubernetesClusterBackendV2 { */ async listResourcesForJobId(jobId: string) { const resources = []; - const resourceTypes: ResourceType[] = ['pod', 'deployment', 'service', 'job', 'replicaset']; + const resourceTypes: ResourceType[] = ['pods', 'deployments', 'services', 'jobs', 'replicasets']; for (const type of resourceTypes) { const list = await this.k8s.list(`teraslice.terascope.io/jobId=${jobId}`, type); diff --git a/packages/teraslice/src/lib/cluster/services/cluster/backends/kubernetesV2/interfaces.ts b/packages/teraslice/src/lib/cluster/services/cluster/backends/kubernetesV2/interfaces.ts index 4770f3c23dc..3ecd3dab0bd 100644 --- a/packages/teraslice/src/lib/cluster/services/cluster/backends/kubernetesV2/interfaces.ts +++ b/packages/teraslice/src/lib/cluster/services/cluster/backends/kubernetesV2/interfaces.ts @@ -30,7 +30,7 @@ export interface K8sConfig { shutdownTimeout: number; } -export type ResourceType = 'deployment' | 'job' | 'pod' | 'replicaset' | 'service'; +export type ResourceType = 'deployments' | 'jobs' | 'pods' | 'replicasets' | 'services'; export type ResourceList = k8s.V1DeploymentList | k8s.V1JobList | k8s.V1PodList | k8s.V1ReplicaSetList | k8s.V1ServiceList; diff --git a/packages/teraslice/src/lib/cluster/services/cluster/backends/kubernetesV2/job/execution_controller.hbs b/packages/teraslice/src/lib/cluster/services/cluster/backends/kubernetesV2/jobs/execution_controller.hbs similarity index 100% rename from packages/teraslice/src/lib/cluster/services/cluster/backends/kubernetesV2/job/execution_controller.hbs rename to packages/teraslice/src/lib/cluster/services/cluster/backends/kubernetesV2/jobs/execution_controller.hbs diff --git a/packages/teraslice/src/lib/cluster/services/cluster/backends/kubernetesV2/k8s.ts b/packages/teraslice/src/lib/cluster/services/cluster/backends/kubernetesV2/k8s.ts index 8bc6b385007..dadeae806af 100644 --- a/packages/teraslice/src/lib/cluster/services/cluster/backends/kubernetesV2/k8s.ts +++ b/packages/teraslice/src/lib/cluster/services/cluster/backends/kubernetesV2/k8s.ts @@ -170,11 +170,11 @@ export class K8s { * | k8s.V1ReplicaSetList * | k8s.V1JobList} list of k8s objects. */ - async list(selector: string, objType: 'deployment', ns?: string): Promise; - async list(selector: string, objType: 'job', ns?: string): Promise; - async list(selector: string, objType: 'pod', ns?: string): Promise; - async list(selector: string, objType: 'replicaset', ns?: string): Promise; - async list(selector: string, objType: 'service', ns?: string): Promise; + async list(selector: string, objType: 'deployments', ns?: string): Promise; + async list(selector: string, objType: 'jobs', ns?: string): Promise; + async list(selector: string, objType: 'pods', ns?: string): Promise; + async list(selector: string, objType: 'replicasets', ns?: string): Promise; + async list(selector: string, objType: 'services', ns?: string): Promise; async list(selector: string, objType: ResourceType, ns?: string): Promise; async list(selector: string, objType: ResourceType, ns?: string): Promise { const namespace = ns || this.defaultNamespace; @@ -190,27 +190,27 @@ export class K8s { ]; try { - if (objType === 'deployment') { + if (objType === 'deployments') { responseObj = await pRetry( () => this.k8sAppsV1Api.listNamespacedDeployment(...params), getRetryConfig() ); - } else if (objType === 'job') { + } else if (objType === 'jobs') { responseObj = await pRetry( () => this.k8sBatchV1Api.listNamespacedJob(...params), getRetryConfig() ); - } else if (objType === 'pod') { + } else if (objType === 'pods') { responseObj = await pRetry( () => this.k8sCoreV1Api.listNamespacedPod(...params), getRetryConfig() ); - } else if (objType === 'replicaset') { + } else if (objType === 'replicasets') { responseObj = await pRetry( () => this.k8sAppsV1Api.listNamespacedReplicaSet(...params), getRetryConfig() ); - } else if (objType === 'service') { + } else if (objType === 'services') { responseObj = await pRetry( () => this.k8sCoreV1Api.listNamespacedService(...params), getRetryConfig() @@ -230,7 +230,7 @@ export class K8s { } async nonEmptyJobList(selector: string) { - const jobs = await this.list(selector, 'job'); + const jobs = await this.list(selector, 'jobs'); if (jobs.items.length === 1) { return jobs; } else if (jobs.items.length === 0) { @@ -329,13 +329,13 @@ export class K8s { * @return {Object} k8s delete response body. */ async delete( - name: string, objType: 'pod', force?: boolean + name: string, objType: 'pods', force?: boolean ): Promise; async delete( - name: string, objType: 'deployment' | 'job' | 'replicaset', force?: boolean + name: string, objType: 'deployments' | 'jobs' | 'replicasets', force?: boolean ): Promise; async delete( - name: string, objType: 'service', force?: boolean + name: string, objType: 'services', force?: boolean ): Promise; async delete( name: string, objType: ResourceType, force?: boolean @@ -390,19 +390,19 @@ export class K8s { }; try { - if (objType === 'service') { + if (objType === 'services') { responseObj = await pRetry(() => deleteWithErrorHandling(() => this.k8sCoreV1Api .deleteNamespacedService(...params)), getRetryConfig()); - } else if (objType === 'deployment') { + } else if (objType === 'deployments') { responseObj = await pRetry(() => deleteWithErrorHandling(() => this.k8sAppsV1Api .deleteNamespacedDeployment(...params)), getRetryConfig()); - } else if (objType === 'job') { + } else if (objType === 'jobs') { responseObj = await pRetry(() => deleteWithErrorHandling(() => this.k8sBatchV1Api .deleteNamespacedJob(...params)), getRetryConfig()); - } else if (objType === 'pod') { + } else if (objType === 'pods') { responseObj = await pRetry(() => deleteWithErrorHandling(() => this.k8sCoreV1Api .deleteNamespacedPod(...params)), getRetryConfig()); - } else if (objType === 'replicaset') { + } else if (objType === 'replicasets') { responseObj = await pRetry(() => deleteWithErrorHandling(() => this.k8sAppsV1Api .deleteNamespacedReplicaSet(...params)), getRetryConfig()); } else { @@ -432,14 +432,14 @@ export class K8s { if (force) { // Order matters. If we delete a parent resource before its children it // will be marked for background deletion and then can't be force deleted. - await this._deleteObjByExId(exId, 'worker', 'pod', force); - await this._deleteObjByExId(exId, 'worker', 'replicaset', force); - await this._deleteObjByExId(exId, 'worker', 'deployment', force); - await this._deleteObjByExId(exId, 'execution_controller', 'pod', force); - await this._deleteObjByExId(exId, 'execution_controller', 'service', force); + await this._deleteObjByExId(exId, 'worker', 'pods', force); + await this._deleteObjByExId(exId, 'worker', 'replicasets', force); + await this._deleteObjByExId(exId, 'worker', 'deployments', force); + await this._deleteObjByExId(exId, 'execution_controller', 'pods', force); + await this._deleteObjByExId(exId, 'execution_controller', 'services', force); } - await this._deleteObjByExId(exId, 'execution_controller', 'job', force); + await this._deleteObjByExId(exId, 'execution_controller', 'jobs', force); } /** @@ -453,15 +453,15 @@ export class K8s { * @return {Promise} */ async _deleteObjByExId( - exId: string, nodeType: NodeType, objType: 'pod', force?: boolean + exId: string, nodeType: NodeType, objType: 'pods', force?: boolean ): Promise; async _deleteObjByExId( - exId: string, nodeType: NodeType, objType: 'job' | 'replicaset' | 'deployment', force?: boolean + exId: string, nodeType: NodeType, objType: 'jobs' | 'replicasets' | 'deployments', force?: boolean ): Promise; async _deleteObjByExId( - exId: string, nodeType: NodeType, objType: 'service', force?: boolean + exId: string, nodeType: NodeType, objType: 'services', force?: boolean ): Promise; async _deleteObjByExId( @@ -526,7 +526,7 @@ export class K8s { const selector = `app.kubernetes.io/component=worker,teraslice.terascope.io/exId=${exId}`; this.logger.info(`Scaling exId: ${exId}, op: ${op}, numWorkers: ${numWorkers}`); - const listResponse = await this.list(selector, 'deployment'); + const listResponse = await this.list(selector, 'deployments'); this.logger.debug(`k8s worker query listResponse: ${JSON.stringify(listResponse)}`); // the selector provided to list above should always result in a single diff --git a/packages/teraslice/src/lib/cluster/services/cluster/backends/kubernetesV2/k8sResource.ts b/packages/teraslice/src/lib/cluster/services/cluster/backends/kubernetesV2/k8sResource.ts index 362301f7fa8..641629a0de7 100644 --- a/packages/teraslice/src/lib/cluster/services/cluster/backends/kubernetesV2/k8sResource.ts +++ b/packages/teraslice/src/lib/cluster/services/cluster/backends/kubernetesV2/k8sResource.ts @@ -33,7 +33,7 @@ export class K8sResource { * @param {String} exUid(optional) - uid from execution resource (deployment and service only) */ constructor( - resourceType: 'deployment' | 'job' | 'service', + resourceType: 'deployments' | 'jobs' | 'services', resourceName: NodeType, terasliceConfig: Config, execution: ExecutionConfig, diff --git a/packages/teraslice/src/lib/cluster/services/cluster/backends/kubernetesV2/service/execution_controller.hbs b/packages/teraslice/src/lib/cluster/services/cluster/backends/kubernetesV2/services/execution_controller.hbs similarity index 100% rename from packages/teraslice/src/lib/cluster/services/cluster/backends/kubernetesV2/service/execution_controller.hbs rename to packages/teraslice/src/lib/cluster/services/cluster/backends/kubernetesV2/services/execution_controller.hbs diff --git a/packages/teraslice/src/lib/cluster/services/cluster/backends/kubernetesV2/utils.ts b/packages/teraslice/src/lib/cluster/services/cluster/backends/kubernetesV2/utils.ts index 4661785929c..6d96a3bde83 100644 --- a/packages/teraslice/src/lib/cluster/services/cluster/backends/kubernetesV2/utils.ts +++ b/packages/teraslice/src/lib/cluster/services/cluster/backends/kubernetesV2/utils.ts @@ -11,23 +11,23 @@ const RETRY_DELAY = isTest ? 50 : 1000; // time in ms const resourcePath = path.join(process.cwd(), './packages/teraslice/src/lib/cluster/services/cluster/backends/kubernetesV2/'); export function makeTemplate( - folder: 'deployment', + folder: 'deployments', fileName: NodeType ): (config: K8sConfig) => k8s.V1Deployment; export function makeTemplate( - folder: 'job', + folder: 'jobs', fileName: NodeType ): (config: K8sConfig) => k8s.V1Job; export function makeTemplate( - folder: 'service', + folder: 'services', fileName: NodeType ): (config: K8sConfig) => k8s.V1Service; export function makeTemplate( - folder: 'deployment' | 'job' | 'service', + folder: 'deployments' | 'jobs' | 'services', fileName: NodeType ): (config: K8sConfig) => k8s.V1Deployment | k8s.V1Job | k8s.V1Service; export function makeTemplate( - folder: 'deployment' | 'job' | 'service', + folder: 'deployments' | 'jobs' | 'services', fileName: NodeType ): (config: K8sConfig) => k8s.V1Deployment | k8s.V1Job | k8s.V1Service { const filePath = path.join(resourcePath, folder, `${fileName}.hbs`); @@ -35,11 +35,11 @@ export function makeTemplate( const templateKeys = ['{{', '}}']; return (config: K8sConfig) => { - if (folder !== 'job' && (config.exName === undefined || config.exUid === undefined)) { + if (folder !== 'jobs' && (config.exName === undefined || config.exUid === undefined)) { throw new Error(`K8s config requires ${config.exName === undefined ? 'exName' : 'exUid'} to create a ${folder} template`); } - if (folder !== 'service' && config.dockerImage === undefined) { + if (folder !== 'services' && config.dockerImage === undefined) { throw new Error(`K8s config requires a dockerImage to create a ${folder} template`); } diff --git a/packages/teraslice/test/lib/cluster/services/cluster/backends/kubernetes/v2/k8s-v2-spec.ts b/packages/teraslice/test/lib/cluster/services/cluster/backends/kubernetes/v2/k8s-v2-spec.ts index 27329681dfd..b827883a2c0 100644 --- a/packages/teraslice/test/lib/cluster/services/cluster/backends/kubernetes/v2/k8s-v2-spec.ts +++ b/packages/teraslice/test/lib/cluster/services/cluster/backends/kubernetes/v2/k8s-v2-spec.ts @@ -70,7 +70,7 @@ describe('k8s', () => { .query({ labelSelector: 'app=teraslice' }) .reply(200, { kind: 'PodList' }); - const pods = await k8s.list('app=teraslice', 'pod'); + const pods = await k8s.list('app=teraslice', 'pods'); expect(pods.kind).toEqual('PodList'); }); @@ -80,7 +80,7 @@ describe('k8s', () => { .query({ labelSelector: 'app=teraslice' }) .reply(200, { kind: 'ServiceList' }); - const pods = await k8s.list('app=teraslice', 'service'); + const pods = await k8s.list('app=teraslice', 'services'); expect(pods.kind).toEqual('ServiceList'); }); @@ -90,7 +90,7 @@ describe('k8s', () => { .query({ labelSelector: 'app=teraslice' }) .reply(200, { kind: 'DeploymentList' }); - const deployments = await k8s.list('app=teraslice', 'deployment'); + const deployments = await k8s.list('app=teraslice', 'deployments'); expect(deployments.kind).toEqual('DeploymentList'); }); @@ -100,7 +100,7 @@ describe('k8s', () => { .query({ labelSelector: 'app=teraslice' }) .reply(200, { kind: 'JobList' }); - const jobs = await k8s.list('app=teraslice', 'job'); + const jobs = await k8s.list('app=teraslice', 'jobs'); expect(jobs.kind).toEqual('JobList'); }); @@ -110,7 +110,7 @@ describe('k8s', () => { .query({ labelSelector: 'app=teraslice' }) .reply(200, { kind: 'ReplicaSetList' }); - const jobs = await k8s.list('app=teraslice', 'replicaset'); + const jobs = await k8s.list('app=teraslice', 'replicasets'); expect(jobs.kind).toEqual('ReplicaSetList'); }); }); @@ -221,12 +221,12 @@ describe('k8s', () => { describe('->delete', () => { it('will throw if name is undefined', async () => { - await expect(k8s.delete(undefined as unknown as string, 'deployment')) + await expect(k8s.delete(undefined as unknown as string, 'deployments')) .rejects.toThrow('Name of resource to delete must be specified. Received: "undefined".'); }); it('will throw if name is an empty string', async () => { - await expect(k8s.delete('', 'deployment')) + await expect(k8s.delete('', 'deployments')) .rejects.toThrow('Name of resource to delete must be specified. Received: "".'); }); @@ -235,7 +235,7 @@ describe('k8s', () => { .delete('/apis/apps/v1/namespaces/default/deployments/test1') .reply(200, {}); - const response = await k8s.delete('test1', 'deployment'); + const response = await k8s.delete('test1', 'deployments'); expect(response).toEqual({}); }); @@ -244,7 +244,7 @@ describe('k8s', () => { .delete('/api/v1/namespaces/default/services/test1') .reply(200, {}); - const response = await k8s.delete('test1', 'service'); + const response = await k8s.delete('test1', 'services'); expect(response).toEqual({}); }); @@ -253,7 +253,7 @@ describe('k8s', () => { .delete('/apis/batch/v1/namespaces/default/jobs/test1') .reply(200, {}); - const response = await k8s.delete('test1', 'job'); + const response = await k8s.delete('test1', 'jobs'); expect(response).toEqual({}); }); @@ -262,7 +262,7 @@ describe('k8s', () => { .delete('/api/v1/namespaces/default/pods/test1') .reply(200, {}); - const response = await k8s.delete('test1', 'pod'); + const response = await k8s.delete('test1', 'pods'); expect(response).toEqual({}); }); @@ -271,7 +271,7 @@ describe('k8s', () => { .delete('/apis/apps/v1/namespaces/default/replicasets/test1') .reply(200, {}); - const response = await k8s.delete('test1', 'replicaset'); + const response = await k8s.delete('test1', 'replicasets'); expect(response).toEqual({}); }); @@ -284,7 +284,7 @@ describe('k8s', () => { .delete('/api/v1/namespaces/default/pods/bad-response') .replyWithError({ statusCode: 400 }); - await expect(k8s.delete('bad-response', 'pod')) + await expect(k8s.delete('bad-response', 'pods')) .rejects.toThrow('Request k8s.delete with name: bad-response failed with: TSError: {"statusCode":400}'); }); @@ -306,7 +306,7 @@ describe('k8s', () => { .delete('/api/v1/namespaces/default/pods/non-existent') .replyWithError(notFoundResponse); - const response = await k8s.delete('non-existent', 'pod'); + const response = await k8s.delete('non-existent', 'pods'); expect(response).toEqual(notFoundResponse.body); }); }); @@ -357,7 +357,7 @@ describe('k8s', () => { items: [jobNoName] }); - await expect(k8s._deleteObjByExId('no-name', 'execution_controller', 'job')) + await expect(k8s._deleteObjByExId('no-name', 'execution_controller', 'jobs')) .rejects.toThrow('Cannot delete job for ExId: no-name by name because it has no name'); }); @@ -374,7 +374,7 @@ describe('k8s', () => { .delete('/apis/batch/v1/namespaces/default/jobs/testJob1') .reply(200, status); - const response = await k8s._deleteObjByExId('testJob1', 'execution_controller', 'job'); + const response = await k8s._deleteObjByExId('testJob1', 'execution_controller', 'jobs'); expect(response).toEqual([expect.objectContaining(status)]); }); @@ -394,7 +394,7 @@ describe('k8s', () => { .delete('/api/v1/namespaces/default/pods/testPod2') .reply(200, testPod2); - const response = await k8s._deleteObjByExId('testPods', 'worker', 'pod'); + const response = await k8s._deleteObjByExId('testPods', 'worker', 'pods'); expect(response).toEqual([ expect.objectContaining(testPod1), expect.objectContaining(testPod2) diff --git a/packages/teraslice/test/lib/cluster/services/cluster/backends/kubernetes/v2/k8sResource-v2-spec.ts b/packages/teraslice/test/lib/cluster/services/cluster/backends/kubernetes/v2/k8sResource-v2-spec.ts index bb0adaa4da5..54bf26360cb 100644 --- a/packages/teraslice/test/lib/cluster/services/cluster/backends/kubernetes/v2/k8sResource-v2-spec.ts +++ b/packages/teraslice/test/lib/cluster/services/cluster/backends/kubernetes/v2/k8sResource-v2-spec.ts @@ -38,7 +38,7 @@ describe('k8sResource', () => { describe('worker deployment', () => { it('has valid resource object.', () => { - const kr = new K8sResource('deployment', 'worker', terasliceConfig, execution, logger, 'example-job-abcd', 'UID1'); + const kr = new K8sResource('deployments', 'worker', terasliceConfig, execution, logger, 'example-job-abcd', 'UID1'); const resource = kr.resource as k8s.V1Deployment; expect(resource.kind).toBe('Deployment'); @@ -76,7 +76,7 @@ describe('k8sResource', () => { it('has valid resource object when terasliceConfig has kubernetes_image_pull_secret.', () => { terasliceConfig.kubernetes_image_pull_secret = 'teraslice-image-pull-secret'; - const kr = new K8sResource('deployment', 'worker', terasliceConfig, execution, logger, 'example-job-abcd', 'UID1'); + const kr = new K8sResource('deployments', 'worker', terasliceConfig, execution, logger, 'example-job-abcd', 'UID1'); const resource = kr.resource as k8s.V1Deployment; let firstSecret: k8s.V1LocalObjectReference | undefined = undefined; @@ -92,7 +92,7 @@ describe('k8sResource', () => { it('has podAntiAffinity when terasliceConfig has kubernetes_worker_antiaffinity true.', () => { terasliceConfig.kubernetes_worker_antiaffinity = true; - const kr = new K8sResource('deployment', 'worker', terasliceConfig, execution, logger, 'example-job-abcd', 'UID1'); + const kr = new K8sResource('deployments', 'worker', terasliceConfig, execution, logger, 'example-job-abcd', 'UID1'); const resource = kr.resource as k8s.V1Deployment; // console.log(yaml.dump(kr.resource.spec.template.spec.affinity)); @@ -120,7 +120,7 @@ describe('k8sResource', () => { terasliceConfig.assets_directory = '/assets'; terasliceConfig.assets_volume = 'asset-volume'; - const kr = new K8sResource('deployment', 'worker', terasliceConfig, execution, logger, 'example-job-abcd', 'UID1'); + const kr = new K8sResource('deployments', 'worker', terasliceConfig, execution, logger, 'example-job-abcd', 'UID1'); const resource = kr.resource as k8s.V1Deployment; let configVolume: k8s.V1Volume | undefined = undefined; @@ -177,7 +177,7 @@ describe('k8sResource', () => { { name: 'teraslice-data1', path: '/data' } ]; - const kr = new K8sResource('deployment', 'worker', terasliceConfig, execution, logger, 'example-job-abcd', 'UID1'); + const kr = new K8sResource('deployments', 'worker', terasliceConfig, execution, logger, 'example-job-abcd', 'UID1'); const resource = kr.resource as k8s.V1Deployment; const configVolume1 = resource.spec?.template.spec?.volumes @@ -227,7 +227,7 @@ describe('k8sResource', () => { { name: 'tmp', path: '/tmp' } ]; - const kr = new K8sResource('deployment', 'worker', terasliceConfig, execution, logger, 'example-job-abcd', 'UID1'); + const kr = new K8sResource('deployments', 'worker', terasliceConfig, execution, logger, 'example-job-abcd', 'UID1'); const resource = kr.resource as k8s.V1Deployment; const configVolume2 = resource.spec?.template.spec?.volumes @@ -267,7 +267,7 @@ describe('k8sResource', () => { }); it('does not have memory/cpu limits/requests when not set in config or execution', () => { - const kr = new K8sResource('deployment', 'worker', terasliceConfig, execution, logger, 'example-job-abcd', 'UID1'); + const kr = new K8sResource('deployments', 'worker', terasliceConfig, execution, logger, 'example-job-abcd', 'UID1'); const resource = kr.resource as k8s.V1Deployment; const exId = resource.metadata?.labels ? resource.metadata.labels['teraslice.terascope.io/exId'] : undefined; @@ -285,7 +285,7 @@ describe('k8sResource', () => { terasliceConfig.cpu = 1; terasliceConfig.memory = 2147483648; - const kr = new K8sResource('deployment', 'worker', terasliceConfig, execution, logger, 'example-job-abcd', 'UID1'); + const kr = new K8sResource('deployments', 'worker', terasliceConfig, execution, logger, 'example-job-abcd', 'UID1'); const resource = kr.resource as k8s.V1Deployment; const exId = resource.metadata?.labels ? resource.metadata.labels['teraslice.terascope.io/exId'] : undefined; @@ -308,7 +308,7 @@ describe('k8sResource', () => { }); it('has the ability to set custom env', () => { - const kr = new K8sResource('deployment', 'worker', terasliceConfig, execution, logger, 'example-job-abcd', 'UID1'); + const kr = new K8sResource('deployments', 'worker', terasliceConfig, execution, logger, 'example-job-abcd', 'UID1'); const resource = kr.resource as k8s.V1Deployment; // NOTE: the env var merge happens in _setResources(), which is @@ -330,7 +330,7 @@ describe('k8sResource', () => { terasliceConfig.cpu = 1; terasliceConfig.memory = 2147483648; - const kr = new K8sResource('deployment', 'worker', terasliceConfig, execution, logger, 'example-job-abcd', 'UID1'); + const kr = new K8sResource('deployments', 'worker', terasliceConfig, execution, logger, 'example-job-abcd', 'UID1'); const resource = kr.resource as k8s.V1Deployment; const exId = resource.metadata?.labels ? resource.metadata.labels['teraslice.terascope.io/exId'] : undefined; @@ -355,7 +355,7 @@ describe('k8sResource', () => { terasliceConfig.cpu = 1; terasliceConfig.memory = 2147483648; - const kr = new K8sResource('deployment', 'worker', terasliceConfig, execution, logger, 'example-job-abcd', 'UID1'); + const kr = new K8sResource('deployments', 'worker', terasliceConfig, execution, logger, 'example-job-abcd', 'UID1'); const resource = kr.resource as k8s.V1Deployment; const exId = resource.metadata?.labels ? resource.metadata.labels['teraslice.terascope.io/exId'] : undefined; @@ -379,7 +379,7 @@ describe('k8sResource', () => { execution.cpu = 1; execution.memory = 2147483648; - const kr = new K8sResource('deployment', 'worker', terasliceConfig, execution, logger, 'example-job-abcd', 'UID1'); + const kr = new K8sResource('deployments', 'worker', terasliceConfig, execution, logger, 'example-job-abcd', 'UID1'); const resource = kr.resource as k8s.V1Deployment; const exId = resource.metadata?.labels ? resource.metadata.labels['teraslice.terascope.io/exId'] : undefined; @@ -405,7 +405,7 @@ describe('k8sResource', () => { execution.resources_requests_memory = 2147483648; execution.resources_limits_memory = 3147483648; - const kr = new K8sResource('deployment', 'worker', terasliceConfig, execution, logger, 'example-job-abcd', 'UID1'); + const kr = new K8sResource('deployments', 'worker', terasliceConfig, execution, logger, 'example-job-abcd', 'UID1'); const resource = kr.resource as k8s.V1Deployment; const exId = resource.metadata?.labels ? resource.metadata.labels['teraslice.terascope.io/exId'] : undefined; @@ -428,7 +428,7 @@ describe('k8sResource', () => { it('has memory limits and requests when set on execution', () => { execution.memory = 2147483648; - const kr = new K8sResource('deployment', 'worker', terasliceConfig, execution, logger, 'example-job-abcd', 'UID1'); + const kr = new K8sResource('deployments', 'worker', terasliceConfig, execution, logger, 'example-job-abcd', 'UID1'); const resource = kr.resource as k8s.V1Deployment; expect(resource.spec?.template.spec?.containers[0].resources).toEqual(yaml.load(` @@ -446,7 +446,7 @@ describe('k8sResource', () => { it('has cpu limits and requests when set on execution', () => { execution.cpu = 1; - const kr = new K8sResource('deployment', 'worker', terasliceConfig, execution, logger, 'example-job-abcd', 'UID1'); + const kr = new K8sResource('deployments', 'worker', terasliceConfig, execution, logger, 'example-job-abcd', 'UID1'); const resource = kr.resource as k8s.V1Deployment; expect(resource.spec?.template.spec?.containers[0].resources).toEqual(yaml.load(` @@ -459,7 +459,7 @@ describe('k8sResource', () => { it('has scratch volume when ephemeral_storage is set true on execution', () => { execution.ephemeral_storage = true; - const kr = new K8sResource('deployment', 'worker', terasliceConfig, execution, logger, 'example-job-abcd', 'UID1'); + const kr = new K8sResource('deployments', 'worker', terasliceConfig, execution, logger, 'example-job-abcd', 'UID1'); const resource = kr.resource as k8s.V1Deployment; expect(resource.spec?.template.spec?.containers[0].volumeMounts) @@ -474,7 +474,7 @@ describe('k8sResource', () => { it('does not have scratch volume when ephemeral_storage is set false on execution', () => { execution.ephemeral_storage = false; - const kr = new K8sResource('deployment', 'worker', terasliceConfig, execution, logger, 'example-job-abcd', 'UID1'); + const kr = new K8sResource('deployments', 'worker', terasliceConfig, execution, logger, 'example-job-abcd', 'UID1'); const resource = kr.resource as k8s.V1Deployment; expect(resource.spec?.template.spec?.containers[0].volumeMounts) @@ -485,7 +485,7 @@ describe('k8sResource', () => { describe('worker deployments with targets', () => { it('does not have affinity or toleration properties when targets equals [].', () => { execution.targets = []; - const kr = new K8sResource('deployment', 'worker', terasliceConfig, execution, logger, 'example-job-abcd', 'UID1'); + const kr = new K8sResource('deployments', 'worker', terasliceConfig, execution, logger, 'example-job-abcd', 'UID1'); const resource = kr.resource as k8s.V1Deployment; expect(resource.spec?.template.spec).not.toHaveProperty('affinity'); @@ -496,7 +496,7 @@ describe('k8sResource', () => { execution.targets = [ { key: 'zone', value: 'west' } ]; - const kr = new K8sResource('deployment', 'worker', terasliceConfig, execution, logger, 'example-job-abcd', 'UID1'); + const kr = new K8sResource('deployments', 'worker', terasliceConfig, execution, logger, 'example-job-abcd', 'UID1'); const resource = kr.resource as k8s.V1Deployment; expect(resource.spec?.template.spec?.affinity).toEqual(yaml.load(` @@ -515,7 +515,7 @@ describe('k8sResource', () => { { key: 'zone', value: 'west' } ]; terasliceConfig.kubernetes_worker_antiaffinity = true; - const kr = new K8sResource('deployment', 'worker', terasliceConfig, execution, logger, 'example-job-abcd', 'UID1'); + const kr = new K8sResource('deployments', 'worker', terasliceConfig, execution, logger, 'example-job-abcd', 'UID1'); const resource = kr.resource as k8s.V1Deployment; expect(resource.spec?.template.spec?.affinity).toEqual(yaml.load(` @@ -548,7 +548,7 @@ describe('k8sResource', () => { execution.targets = [ { key: 'zone', value: 'west', constraint: 'required' } ]; - const kr = new K8sResource('deployment', 'worker', terasliceConfig, execution, logger, 'example-job-abcd', 'UID1'); + const kr = new K8sResource('deployments', 'worker', terasliceConfig, execution, logger, 'example-job-abcd', 'UID1'); const resource = kr.resource as k8s.V1Deployment; expect(resource.spec?.template.spec?.affinity).toEqual(yaml.load(` @@ -567,7 +567,7 @@ describe('k8sResource', () => { { key: 'zone', value: 'west', constraint: 'required' }, { key: 'region', value: '42', constraint: 'required' } ]; - const kr = new K8sResource('deployment', 'worker', terasliceConfig, execution, logger, 'example-job-abcd', 'UID1'); + const kr = new K8sResource('deployments', 'worker', terasliceConfig, execution, logger, 'example-job-abcd', 'UID1'); const resource = kr.resource as k8s.V1Deployment; expect(resource.spec?.template.spec?.affinity).toEqual(yaml.load(` @@ -589,7 +589,7 @@ describe('k8sResource', () => { execution.targets = [ { key: 'zone', value: 'west', constraint: 'preferred' } ]; - const kr = new K8sResource('deployment', 'worker', terasliceConfig, execution, logger, 'example-job-abcd', 'UID1'); + const kr = new K8sResource('deployments', 'worker', terasliceConfig, execution, logger, 'example-job-abcd', 'UID1'); const resource = kr.resource as k8s.V1Deployment; expect(resource.spec?.template.spec?.affinity).toEqual(yaml.load(` @@ -609,7 +609,7 @@ describe('k8sResource', () => { { key: 'zone', value: 'west', constraint: 'preferred' }, { key: 'region', value: 'texas', constraint: 'preferred' } ]; - const kr = new K8sResource('deployment', 'worker', terasliceConfig, execution, logger, 'example-job-abcd', 'UID1'); + const kr = new K8sResource('deployments', 'worker', terasliceConfig, execution, logger, 'example-job-abcd', 'UID1'); const resource = kr.resource as k8s.V1Deployment; expect(resource.spec?.template.spec?.affinity).toEqual(yaml.load(` @@ -635,7 +635,7 @@ describe('k8sResource', () => { execution.targets = [ { key: 'zone', value: 'west', constraint: 'accepted' } ]; - const kr = new K8sResource('deployment', 'worker', terasliceConfig, execution, logger, 'example-job-abcd', 'UID1'); + const kr = new K8sResource('deployments', 'worker', terasliceConfig, execution, logger, 'example-job-abcd', 'UID1'); const resource = kr.resource as k8s.V1Deployment; // console.log(yaml.dump(kr.resource.spec.template.spec.tolerations)); @@ -651,7 +651,7 @@ describe('k8sResource', () => { { key: 'zone', value: 'west', constraint: 'accepted' }, { key: 'region', value: 'texas', constraint: 'accepted' } ]; - const kr = new K8sResource('deployment', 'worker', terasliceConfig, execution, logger, 'example-job-abcd', 'UID1'); + const kr = new K8sResource('deployments', 'worker', terasliceConfig, execution, logger, 'example-job-abcd', 'UID1'); const resource = kr.resource as k8s.V1Deployment; // console.log(yaml.dump(kr.resource.spec.template.spec.tolerations)); @@ -671,7 +671,7 @@ describe('k8sResource', () => { { key: 'zone', value: 'west', constraint: 'required' }, { key: 'region', value: 'texas', constraint: 'preferred' } ]; - const kr = new K8sResource('deployment', 'worker', terasliceConfig, execution, logger, 'example-job-abcd', 'UID1'); + const kr = new K8sResource('deployments', 'worker', terasliceConfig, execution, logger, 'example-job-abcd', 'UID1'); const resource = kr.resource as k8s.V1Deployment; // console.log(yaml.dump(kr.resource.spec.template.spec.tolerations)); @@ -703,7 +703,7 @@ describe('k8sResource', () => { { key: 'zone', value: 'west', constraint: 'accepted' }, { key: 'region', value: 'texas', constraint: 'accepted' } ]; - const kr = new K8sResource('deployment', 'worker', terasliceConfig, execution, logger, 'example-job-abcd', 'UID1'); + const kr = new K8sResource('deployments', 'worker', terasliceConfig, execution, logger, 'example-job-abcd', 'UID1'); const resource = kr.resource as k8s.V1Deployment; expect(resource.spec?.template.spec?.affinity).toEqual(yaml.load(` @@ -753,7 +753,7 @@ describe('k8sResource', () => { key2: 'value2' }; - const kr = new K8sResource('deployment', 'worker', terasliceConfig, execution, logger, 'example-job-abcd', 'UID1'); + const kr = new K8sResource('deployments', 'worker', terasliceConfig, execution, logger, 'example-job-abcd', 'UID1'); const resource = kr.resource as k8s.V1Deployment; // console.log(yaml.dump(kr.resource)); @@ -770,7 +770,7 @@ describe('k8sResource', () => { abcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghij1234: 'value3', }; - const kr = new K8sResource('deployment', 'worker', terasliceConfig, execution, logger, 'example-job-abcd', 'UID1'); + const kr = new K8sResource('deployments', 'worker', terasliceConfig, execution, logger, 'example-job-abcd', 'UID1'); const resource = kr.resource as k8s.V1Deployment; // console.log(yaml.dump(kr.resource)); @@ -792,7 +792,7 @@ describe('k8sResource', () => { describe('teraslice job with one valid external_ports set', () => { it('generates k8s worker deployment with containerPort on container', () => { execution.external_ports = [9090]; - const kr = new K8sResource('deployment', 'worker', terasliceConfig, execution, logger, 'example-job-abcd', 'UID1'); + const kr = new K8sResource('deployments', 'worker', terasliceConfig, execution, logger, 'example-job-abcd', 'UID1'); const resource = kr.resource as k8s.V1Deployment; // console.log(yaml.dump(kr.resource.spec.template.spec.containers[0].ports)); @@ -805,7 +805,7 @@ describe('k8sResource', () => { it('generates k8s execution controller job with containerPort on container', () => { execution.external_ports = [9090]; - const kr = new K8sResource('job', 'execution_controller', terasliceConfig, execution, logger); + const kr = new K8sResource('jobs', 'execution_controller', terasliceConfig, execution, logger); const resource = kr.resource as k8s.V1Job; // console.log(yaml.dump(kr.resource.spec.template.spec.containers[0].ports)); @@ -823,7 +823,7 @@ describe('k8sResource', () => { 9090, { name: 'metrics', port: 9091 } ]; - const kr = new K8sResource('deployment', 'worker', terasliceConfig, execution, logger, 'example-job-abcd', 'UID1'); + const kr = new K8sResource('deployments', 'worker', terasliceConfig, execution, logger, 'example-job-abcd', 'UID1'); const resource = kr.resource as k8s.V1Deployment; // console.log(yaml.dump(kr.resource.spec.template.spec.containers[0].ports)); @@ -837,7 +837,7 @@ describe('k8sResource', () => { it('generates k8s execution controller job with containerPort on container', () => { execution.external_ports = [9090, 9091]; - const kr = new K8sResource('job', 'execution_controller', terasliceConfig, execution, logger); + const kr = new K8sResource('jobs', 'execution_controller', terasliceConfig, execution, logger); const resource = kr.resource as k8s.V1Job; // console.log(yaml.dump(kr.resource.spec.template.spec.containers[0].ports)); @@ -852,7 +852,7 @@ describe('k8sResource', () => { describe('execution_controller job', () => { it('has valid resource object.', () => { - const kr = new K8sResource('job', 'execution_controller', terasliceConfig, execution, logger); + const kr = new K8sResource('jobs', 'execution_controller', terasliceConfig, execution, logger); const resource = kr.resource as k8s.V1Job; expect(resource.kind).toBe('Job'); @@ -892,7 +892,7 @@ describe('k8sResource', () => { terasliceConfig.cpu_execution_controller = 1; terasliceConfig.memory_execution_controller = 2147483648; - const kr = new K8sResource('job', 'execution_controller', terasliceConfig, execution, logger); + const kr = new K8sResource('jobs', 'execution_controller', terasliceConfig, execution, logger); const resource = kr.resource as k8s.V1Job; expect(resource.spec?.template.spec?.containers[0].resources).toEqual(yaml.load(` @@ -915,7 +915,7 @@ describe('k8sResource', () => { terasliceConfig.cpu_execution_controller = 1; terasliceConfig.memory_execution_controller = 2147483648; - const kr = new K8sResource('job', 'execution_controller', terasliceConfig, execution, logger); + const kr = new K8sResource('jobs', 'execution_controller', terasliceConfig, execution, logger); const resource = kr.resource as k8s.V1Job; expect(resource.spec?.template.spec?.containers[0].resources).toEqual(yaml.load(` @@ -944,7 +944,7 @@ describe('k8sResource', () => { ['teraslice-JOB-name', 'ts-wkr-teraslice-job-name-7ba9afb0-417a'] ])('when Job Name is %s the k8s worker name is: %s', (jobName, k8sName) => { execution.name = jobName; - const kr = new K8sResource('deployment', 'worker', terasliceConfig, execution, logger, 'example-job-abcd', 'UID1'); + const kr = new K8sResource('deployments', 'worker', terasliceConfig, execution, logger, 'example-job-abcd', 'UID1'); const resource = kr.resource as k8s.V1Deployment; expect(resource.metadata?.name).toBe(k8sName); @@ -958,7 +958,7 @@ describe('k8sResource', () => { { key: 'key2', value: 'value2' } ]; - const kr = new K8sResource('job', 'execution_controller', terasliceConfig, execution, logger); + const kr = new K8sResource('jobs', 'execution_controller', terasliceConfig, execution, logger); const resource = kr.resource as k8s.V1Deployment; expect(resource.kind).toBe('Job'); @@ -1002,7 +1002,7 @@ describe('k8sResource', () => { { key: 'region', value: 'texas', constraint: 'accepted' } ]; - const kr = new K8sResource('job', 'execution_controller', terasliceConfig, execution, logger); + const kr = new K8sResource('jobs', 'execution_controller', terasliceConfig, execution, logger); const resource = kr.resource as k8s.V1Deployment; expect(resource.kind).toBe('Job'); @@ -1044,7 +1044,7 @@ describe('k8sResource', () => { execution.stateful = true; terasliceConfig.kubernetes_priority_class_name = 'testPriorityClass'; - const krWorker = new K8sResource('deployment', 'worker', terasliceConfig, execution, logger, 'example-job-abcd', 'UID1'); + const krWorker = new K8sResource('deployments', 'worker', terasliceConfig, execution, logger, 'example-job-abcd', 'UID1'); const wkrResource = krWorker.resource as k8s.V1Deployment; expect(wkrResource.kind).toBe('Deployment'); @@ -1056,7 +1056,7 @@ describe('k8sResource', () => { : undefined; expect(wkrStatefulLabel).toEqual('true'); - const krExporter = new K8sResource('job', 'execution_controller', terasliceConfig, execution, logger); + const krExporter = new K8sResource('jobs', 'execution_controller', terasliceConfig, execution, logger); const exporterResource = krExporter.resource as k8s.V1Job; expect(exporterResource.kind).toBe('Job'); @@ -1077,7 +1077,7 @@ describe('k8sResource', () => { execution.pod_spec_override = { initContainers: [] }; terasliceConfig.kubernetes_overrides_enabled = false; - const krWorker = new K8sResource('deployment', 'worker', terasliceConfig, execution, logger, 'example-job-abcd', 'UID1'); + const krWorker = new K8sResource('deployments', 'worker', terasliceConfig, execution, logger, 'example-job-abcd', 'UID1'); const resource = krWorker.resource as k8s.V1Deployment; expect(resource.spec?.template.spec).not.toHaveProperty('initContainers'); @@ -1089,7 +1089,7 @@ describe('k8sResource', () => { execution.pod_spec_override = { initContainers: [] }; terasliceConfig.kubernetes_overrides_enabled = true; - const krWorker = new K8sResource('deployment', 'worker', terasliceConfig, execution, logger, 'example-job-abcd', 'UID1'); + const krWorker = new K8sResource('deployments', 'worker', terasliceConfig, execution, logger, 'example-job-abcd', 'UID1'); const resource = krWorker.resource as k8s.V1Deployment; expect(resource.spec?.template.spec).toHaveProperty('initContainers'); diff --git a/packages/teraslice/test/lib/cluster/services/cluster/backends/kubernetes/v2/utils-v2-spec.ts b/packages/teraslice/test/lib/cluster/services/cluster/backends/kubernetes/v2/utils-v2-spec.ts index e3df5aa6f12..2390bf6bded 100644 --- a/packages/teraslice/test/lib/cluster/services/cluster/backends/kubernetes/v2/utils-v2-spec.ts +++ b/packages/teraslice/test/lib/cluster/services/cluster/backends/kubernetes/v2/utils-v2-spec.ts @@ -8,7 +8,7 @@ describe('K8s Utils', () => { describe('->makeTemplate', () => { describe('execution_controller job', () => { it('should be able to support the execution_controller job', () => { - const exJobTemplate = makeTemplate('job', 'execution_controller'); + const exJobTemplate = makeTemplate('jobs', 'execution_controller'); const config: K8sConfig = { clusterName: 'teracluster', configMapName: 'teracluster-worker', @@ -69,7 +69,7 @@ describe('K8s Utils', () => { }); it('should throw error if docker image undefined on config for job', () => { - const exJobTemplate = makeTemplate('job', 'execution_controller'); + const exJobTemplate = makeTemplate('jobs', 'execution_controller'); const config: K8sConfig = { clusterName: 'teracluster', configMapName: 'teracluster-worker', @@ -93,7 +93,7 @@ describe('K8s Utils', () => { }); describe('worker deployment', () => { - const workerDeploymentTemplate = makeTemplate('deployment', 'worker'); + const workerDeploymentTemplate = makeTemplate('deployments', 'worker'); it('should be able to support the worker deployment', () => { const config: K8sConfig = { clusterName: 'teracluster', @@ -234,7 +234,7 @@ describe('K8s Utils', () => { }); describe('execution_controller service', () => { - const exServiceTemplate = makeTemplate('service', 'execution_controller'); + const exServiceTemplate = makeTemplate('services', 'execution_controller'); it('should be able to support the execution_controller service', () => { const config: K8sConfig = { clusterName: 'teracluster', From 993d60532ed9e52d668f4f82d0802dde4ef60bda Mon Sep 17 00:00:00 2001 From: busma13 Date: Fri, 1 Nov 2024 16:01:43 -0700 Subject: [PATCH 14/30] extend K8sResource class --- .../cluster/backends/kubernetesV2/index.ts | 61 +-- .../backends/kubernetesV2/interfaces.ts | 4 +- .../kubernetesV2/k8sDeploymentResource.ts | 72 +++ .../backends/kubernetesV2/k8sJobResource.ts | 66 +++ .../backends/kubernetesV2/k8sResource.ts | 89 +--- .../kubernetesV2/k8sServiceResource.ts | 47 ++ .../cluster/backends/kubernetesV2/utils.ts | 28 +- .../kubernetes/v2/k8sResource-v2-spec.ts | 417 +++++++++--------- .../backends/kubernetes/v2/utils-v2-spec.ts | 7 +- 9 files changed, 441 insertions(+), 350 deletions(-) create mode 100644 packages/teraslice/src/lib/cluster/services/cluster/backends/kubernetesV2/k8sDeploymentResource.ts create mode 100644 packages/teraslice/src/lib/cluster/services/cluster/backends/kubernetesV2/k8sJobResource.ts create mode 100644 packages/teraslice/src/lib/cluster/services/cluster/backends/kubernetesV2/k8sServiceResource.ts diff --git a/packages/teraslice/src/lib/cluster/services/cluster/backends/kubernetesV2/index.ts b/packages/teraslice/src/lib/cluster/services/cluster/backends/kubernetesV2/index.ts index 80391224f0b..c9fcbd600ac 100644 --- a/packages/teraslice/src/lib/cluster/services/cluster/backends/kubernetesV2/index.ts +++ b/packages/teraslice/src/lib/cluster/services/cluster/backends/kubernetesV2/index.ts @@ -4,12 +4,14 @@ import { } from '@terascope/utils'; import type { Context, ExecutionConfig } from '@terascope/job-components'; import { makeLogger } from '../../../../../workers/helpers/terafoundation.js'; -import { K8sResource } from './k8sResource.js'; import { gen } from './k8sState.js'; import { K8s } from './k8s.js'; -import { getRetryConfig, isDeployment, isJob, isService } from './utils.js'; +import { getRetryConfig } from './utils.js'; import { StopExecutionOptions } from '../../../interfaces.js'; import { ResourceType } from './interfaces.js'; +import { K8sJobResource } from './k8sJobResource.js'; +import { K8sServiceResource } from './k8sServiceResource.js'; +import { K8sDeploymentResource } from './k8sDeploymentResource.js'; /* Execution Life Cycle for _status @@ -29,6 +31,10 @@ export class KubernetesClusterBackendV2 { readonly clusterNameLabel: string; constructor(context: Context, clusterMasterServer: any) { + if (!context.sysconfig.teraslice.kubernetes_api_poll_delay) { + throw new Error('Kubernetes clustering requires kubernetes_api_poll_delay to be defined.'); + } + const kubernetesNamespace = get(context, 'sysconfig.teraslice.kubernetes_namespace', 'default'); const clusterName = get(context, 'sysconfig.teraslice.name'); @@ -42,7 +48,7 @@ export class KubernetesClusterBackendV2 { this.logger, null, kubernetesNamespace, - context.sysconfig.teraslice.kubernetes_api_poll_delay || 1000, + context.sysconfig.teraslice.kubernetes_api_poll_delay, context.sysconfig.teraslice.shutdown_timeout ); @@ -105,39 +111,38 @@ export class KubernetesClusterBackendV2 { execution.slicer_port = 45680; - const exJobResource = new K8sResource( - 'jobs', - 'execution_controller', + const exJobResource = new K8sJobResource( this.context.sysconfig.teraslice, execution, this.logger ); - if (!(isJob(exJobResource.resource))) { - throw new Error(`exJobResource.resource must be of type k8s.V1Job`); - } - const exJob = exJobResource.resource; this.logger.debug(exJob, 'execution allocating slicer'); const jobResult = await this.k8s.post(exJob); - const exServiceResource = new K8sResource( - 'services', - 'execution_controller', + // fixme + if (!jobResult.metadata) { + throw new Error('Required field metadata missing from jobResult'); + } + if (!jobResult.metadata.name) { + throw new Error('Required field name missing from jobResult.metadata'); + } + if (!jobResult.metadata.uid) { + throw new Error('Required field uid missing from jobResult.metadata'); + } + + const exServiceResource = new K8sServiceResource( this.context.sysconfig.teraslice, execution, this.logger, // Needed to create the deployment and service resource ownerReferences - jobResult.metadata?.name, - jobResult.metadata?.uid + jobResult.metadata.name, + jobResult.metadata.uid ); - if (!(isService(exServiceResource.resource))) { - throw new Error(`exJobResource.resource must be of type k8s.V1Service`); - } - const exService = exServiceResource.resource; const serviceResult = await this.k8s.post(exService); @@ -199,9 +204,18 @@ export class KubernetesClusterBackendV2 { this.context.sysconfig.teraslice.slicer_timeout ); - const kr = new K8sResource( - 'deployments', - 'worker', + // fixme + if (!jobs.items[0].metadata) { + throw new Error('Required field metadata missing from jobResult'); + } + if (!jobs.items[0].metadata.name) { + throw new Error('Required field name missing from jobResult.metadata'); + } + if (!jobs.items[0].metadata.uid) { + throw new Error('Required field uid missing from jobResult.metadata'); + } + + const kr = new K8sDeploymentResource( this.context.sysconfig.teraslice, execution, this.logger, @@ -209,9 +223,6 @@ export class KubernetesClusterBackendV2 { jobs.items[0].metadata?.uid ); - if (!(isDeployment(kr.resource))) { - throw new Error(`exJobResource.resource must be of type k8s.V1Deployment`); - } const workerDeployment = kr.resource; this.logger.debug(`workerDeployment:\n\n${JSON.stringify(workerDeployment, null, 2)}`); diff --git a/packages/teraslice/src/lib/cluster/services/cluster/backends/kubernetesV2/interfaces.ts b/packages/teraslice/src/lib/cluster/services/cluster/backends/kubernetesV2/interfaces.ts index 3ecd3dab0bd..fe18b8d9d2f 100644 --- a/packages/teraslice/src/lib/cluster/services/cluster/backends/kubernetesV2/interfaces.ts +++ b/packages/teraslice/src/lib/cluster/services/cluster/backends/kubernetesV2/interfaces.ts @@ -19,8 +19,8 @@ export interface K8sConfig { dockerImage: string | undefined; execution: string; exId: string; - exName: string | undefined; - exUid: string | undefined; + exName?: string; + exUid?: string; jobId: string; jobNameLabel: string; name: string; diff --git a/packages/teraslice/src/lib/cluster/services/cluster/backends/kubernetesV2/k8sDeploymentResource.ts b/packages/teraslice/src/lib/cluster/services/cluster/backends/kubernetesV2/k8sDeploymentResource.ts new file mode 100644 index 00000000000..1ab8ec98fe4 --- /dev/null +++ b/packages/teraslice/src/lib/cluster/services/cluster/backends/kubernetesV2/k8sDeploymentResource.ts @@ -0,0 +1,72 @@ +import _ from 'lodash'; +import * as k8s from '@kubernetes/client-node'; +import { Logger } from '@terascope/utils'; +import type { Config, ExecutionConfig } from '@terascope/types'; +import { makeTemplate } from './utils.js'; +import { K8sConfig, NodeType } from './interfaces.js'; +import { K8sResource } from './k8sResource.js'; + +export class K8sDeploymentResource extends K8sResource { + nodeType: NodeType = 'worker'; + nameInfix = 'wkr'; + templateGenerator: (config: K8sConfig) => k8s.V1Deployment; + templateConfig; + resource; + exName: string; + exUid: string; + + /** + * K8sDeploymentResource allows the generation of a k8s deployment based on a template. + * After creating the object, the k8s deployment is accessible on the objects + * .resource property. + * + * @param {Object} terasliceConfig - teraslice cluster config from context + * @param {Object} execution - teraslice execution + * @param {Logger} logger - teraslice logger + * @param {String} exName - name from execution resource + * @param {String} exUid - uid from execution resource + */ + constructor( + terasliceConfig: Config, + execution: ExecutionConfig, + logger: Logger, + exName: string, + exUid: string + ) { + super(terasliceConfig, execution, logger); + this.execution = execution; + this.logger = logger; + this.terasliceConfig = terasliceConfig; + this.exName = exName; + this.exUid = exUid; + this.templateGenerator = makeTemplate('deployments', this.nodeType); + this.templateConfig = this._makeConfig(this.nameInfix, exName, exUid); + this.resource = this.templateGenerator(this.templateConfig); + + this._setJobLabels(this.resource); + + // Apply job `targets` setting as k8s nodeAffinity + // We assume that multiple targets require both to match ... + // NOTE: If you specify multiple `matchExpressions` associated with + // `nodeSelectorTerms`, then the pod can be scheduled onto a node + // only if *all* `matchExpressions` can be satisfied. + this._setTargets(this.resource); + this._setResources(this.resource); + this._setVolumes(this.resource); + if (process.env.MOUNT_LOCAL_TERASLICE !== undefined) { + this._mountLocalTeraslice(this.resource); + } + this._setEnvVariables(); + this._setAssetsVolume(this.resource); + this._setImagePullSecret(this.resource); + this._setEphemeralStorage(this.resource); + this._setExternalPorts(this.resource); + this._setPriorityClassName(this.resource); + this._setWorkerAntiAffinity(this.resource); + + // override must happen last + if (this.terasliceConfig.kubernetes_overrides_enabled) { + this._mergePodSpecOverlay(this.resource); + } + } +} diff --git a/packages/teraslice/src/lib/cluster/services/cluster/backends/kubernetesV2/k8sJobResource.ts b/packages/teraslice/src/lib/cluster/services/cluster/backends/kubernetesV2/k8sJobResource.ts new file mode 100644 index 00000000000..647f12168d5 --- /dev/null +++ b/packages/teraslice/src/lib/cluster/services/cluster/backends/kubernetesV2/k8sJobResource.ts @@ -0,0 +1,66 @@ +import _ from 'lodash'; +import * as k8s from '@kubernetes/client-node'; +import { Logger } from '@terascope/utils'; +import type { Config, ExecutionConfig } from '@terascope/types'; +import { makeTemplate } from './utils.js'; +import { K8sConfig, NodeType } from './interfaces.js'; +import { K8sResource } from './k8sResource.js'; + +export class K8sJobResource extends K8sResource { + nodeType: NodeType = 'execution_controller'; + nameInfix = 'exc'; + templateGenerator: (config: K8sConfig) => k8s.V1Job; + templateConfig; + resource; + + /** + * K8sJobResource allows the generation of a k8s job based on a template. + * After creating the object, the k8s job is accessible on the objects + * .resource property. + * + * @param {Object} terasliceConfig - teraslice cluster config from context + * @param {Object} execution - teraslice execution + * @param {Logger} logger - teraslice logger + */ + constructor( + terasliceConfig: Config, + execution: ExecutionConfig, + logger: Logger + ) { + super(terasliceConfig, execution, logger); + this.templateGenerator = makeTemplate('jobs', this.nodeType); + this.templateConfig = this._makeConfig(this.nameInfix); + this.resource = this.templateGenerator(this.templateConfig); + + this._setJobLabels(this.resource); + + // Apply job `targets` setting as k8s nodeAffinity + // We assume that multiple targets require both to match ... + // NOTE: If you specify multiple `matchExpressions` associated with + // `nodeSelectorTerms`, then the pod can be scheduled onto a node + // only if *all* `matchExpressions` can be satisfied. + this._setTargets(this.resource); + this._setResources(this.resource); + this._setVolumes(this.resource); + if (process.env.MOUNT_LOCAL_TERASLICE !== undefined) { + this._mountLocalTeraslice(this.resource); + } + this._setEnvVariables(); + this._setAssetsVolume(this.resource); + this._setImagePullSecret(this.resource); + this._setEphemeralStorage(this.resource); + this._setExternalPorts(this.resource); + this._setPriorityClassName(this.resource); + + // Execution controller targets are required nodeAffinities, if + // required job targets are also supplied, then *all* of the matches + // will have to be satisfied for the job to be scheduled. This also + // adds tolerations for any specified targets + this._setExecutionControllerTargets(this.resource); + + // override must happen last + if (this.terasliceConfig.kubernetes_overrides_enabled) { + this._mergePodSpecOverlay(this.resource); + } + } +} diff --git a/packages/teraslice/src/lib/cluster/services/cluster/backends/kubernetesV2/k8sResource.ts b/packages/teraslice/src/lib/cluster/services/cluster/backends/kubernetesV2/k8sResource.ts index 641629a0de7..322fd1353f3 100644 --- a/packages/teraslice/src/lib/cluster/services/cluster/backends/kubernetesV2/k8sResource.ts +++ b/packages/teraslice/src/lib/cluster/services/cluster/backends/kubernetesV2/k8sResource.ts @@ -3,103 +3,40 @@ import * as k8s from '@kubernetes/client-node'; import { isNumber, Logger } from '@terascope/utils'; import type { Config, ExecutionConfig } from '@terascope/types'; import { safeEncode } from '../../../../../utils/encoding_utils.js'; -import { isService, makeTemplate, setMaxOldSpaceViaEnv } from './utils.js'; +import { setMaxOldSpaceViaEnv } from './utils.js'; import { K8sConfig, NodeType } from './interfaces.js'; -export class K8sResource { +export abstract class K8sResource { execution: ExecutionConfig; jobLabelPrefix: string; jobPropertyLabelPrefix: string; logger: Logger; - nodeType: NodeType; - nameInfix: string; terasliceConfig: Config; - templateGenerator: (config: K8sConfig) => k8s.V1Deployment | k8s.V1Job | k8s.V1Service; - templateConfig: K8sConfig; - resource: k8s.V1Deployment | k8s.V1Job | k8s.V1Service; - exName?: string; - exUid?: string; + abstract nodeType: NodeType; + abstract nameInfix: string; + abstract templateGenerator: (config: K8sConfig) => T; + abstract templateConfig: K8sConfig; + abstract resource: T; + /** * K8sResource allows the generation of k8s resources based on templates. * After creating the object, the k8s resource is accessible on the objects * .resource property. * - * @param {'deployment' | 'job' | 'service'} resourceType - job/service/deployment - * @param {NodeType} resourceName - worker/execution_controller * @param {Object} terasliceConfig - teraslice cluster config from context * @param {Object} execution - teraslice execution * @param {Logger} logger - teraslice logger - * @param {String} exName(optional) - name from execution resource (deployment and service only) - * @param {String} exUid(optional) - uid from execution resource (deployment and service only) */ constructor( - resourceType: 'deployments' | 'jobs' | 'services', - resourceName: NodeType, terasliceConfig: Config, execution: ExecutionConfig, - logger: Logger, - exName?: string, - exUid?: string - + logger: Logger ) { this.execution = execution; this.jobLabelPrefix = 'job.teraslice.terascope.io'; this.jobPropertyLabelPrefix = 'job-property.teraslice.terascope.io'; this.logger = logger; - this.nodeType = resourceName; this.terasliceConfig = terasliceConfig; - this.exName = exName || undefined; - this.exUid = exUid || undefined; - - if (resourceName === 'worker') { - this.nameInfix = 'wkr'; - } else if (resourceName === 'execution_controller') { - this.nameInfix = 'exc'; - } else { - throw new Error(`Unsupported resourceName: ${resourceName}`); - } - - this.templateGenerator = makeTemplate(resourceType, resourceName); - this.templateConfig = this._makeConfig(); - this.resource = this.templateGenerator(this.templateConfig); - - if (!(isService(this.resource))) { - this._setJobLabels(this.resource); - - // Apply job `targets` setting as k8s nodeAffinity - // We assume that multiple targets require both to match ... - // NOTE: If you specify multiple `matchExpressions` associated with - // `nodeSelectorTerms`, then the pod can be scheduled onto a node - // only if *all* `matchExpressions` can be satisfied. - this._setTargets(this.resource); - this._setResources(this.resource); - this._setVolumes(this.resource); - if (process.env.MOUNT_LOCAL_TERASLICE !== undefined) { - this._mountLocalTeraslice(this.resource); - } - this._setEnvVariables(); - this._setAssetsVolume(this.resource); - this._setImagePullSecret(this.resource); - this._setEphemeralStorage(this.resource); - this._setExternalPorts(this.resource); - this._setPriorityClassName(this.resource); - - if (resourceName === 'worker') { - this._setWorkerAntiAffinity(this.resource); - } - - // Execution controller targets are required nodeAffinities, if - // required job targets are also supplied, then *all* of the matches - // will have to be satisfied for the job to be scheduled. This also - // adds tolerations for any specified targets - if (resourceName === 'execution_controller') { - this._setExecutionControllerTargets(this.resource); - } - - if (this.terasliceConfig.kubernetes_overrides_enabled) { - this._mergePodSpecOverlay(this.resource); - } - } } _setEnvVariables() { @@ -118,7 +55,7 @@ export class K8sResource { } } - _makeConfig(): K8sConfig { + _makeConfig(nameInfix: string, exName?: string, exUid?: string): K8sConfig { const clusterName = _.get(this.terasliceConfig, 'name'); const clusterNameLabel = clusterName.replace(/[^a-zA-Z0-9_\-.]/g, '_').substring(0, 63); const configMapName = _.get( @@ -137,7 +74,7 @@ export class K8sResource { .replace(/^[^a-z]/, 'a') .replace(/[^a-z0-9]$/, '0') .substring(0, 63); - const name = `ts-${this.nameInfix}-${jobNameLabel.substring(0, 35)}-${this.execution.job_id.substring(0, 13)}`; + const name = `ts-${nameInfix}-${jobNameLabel.substring(0, 35)}-${this.execution.job_id.substring(0, 13)}`; const shutdownTimeoutMs = _.get(this.terasliceConfig, 'shutdown_timeout', 60000); const shutdownTimeoutSeconds = Math.round(shutdownTimeoutMs / 1000); @@ -150,8 +87,8 @@ export class K8sResource { dockerImage, execution: safeEncode(this.execution), exId: this.execution.ex_id, - exName: this.exName, - exUid: this.exUid, + exName: exName, + exUid: exUid, jobId: this.execution.job_id, jobNameLabel, name, diff --git a/packages/teraslice/src/lib/cluster/services/cluster/backends/kubernetesV2/k8sServiceResource.ts b/packages/teraslice/src/lib/cluster/services/cluster/backends/kubernetesV2/k8sServiceResource.ts new file mode 100644 index 00000000000..f311b0bb25c --- /dev/null +++ b/packages/teraslice/src/lib/cluster/services/cluster/backends/kubernetesV2/k8sServiceResource.ts @@ -0,0 +1,47 @@ +import _ from 'lodash'; +import * as k8s from '@kubernetes/client-node'; +import { Logger } from '@terascope/utils'; +import type { Config, ExecutionConfig } from '@terascope/types'; +import { makeTemplate } from './utils.js'; +import { K8sConfig, NodeType } from './interfaces.js'; +import { K8sResource } from './k8sResource.js'; + +export class K8sServiceResource extends K8sResource { + nodeType: NodeType = 'execution_controller'; + nameInfix = 'exc'; + templateGenerator: (config: K8sConfig) => k8s.V1Service; + templateConfig; + resource; + exName: string; + exUid: string; + + /** + * K8sServiceResource allows the generation of a k8s service based on a template. + * After creating the object, the k8s service is accessible on the objects + * .resource property. + * + * @param {Object} terasliceConfig - teraslice cluster config from context + * @param {Object} execution - teraslice execution + * @param {Logger} logger - teraslice logger + * @param {String} exName - name from execution resource + * @param {String} exUid - uid from execution resource + */ + constructor( + terasliceConfig: Config, + execution: ExecutionConfig, + logger: Logger, + exName: string, + exUid: string + + ) { + super(terasliceConfig, execution, logger); + this.execution = execution; + this.logger = logger; + this.terasliceConfig = terasliceConfig; + this.exName = exName; + this.exUid = exUid; + this.templateGenerator = makeTemplate('services', this.nodeType); + this.templateConfig = this._makeConfig(this.nameInfix, exName, exUid); + this.resource = this.templateGenerator(this.templateConfig); + } +} diff --git a/packages/teraslice/src/lib/cluster/services/cluster/backends/kubernetesV2/utils.ts b/packages/teraslice/src/lib/cluster/services/cluster/backends/kubernetesV2/utils.ts index 6d96a3bde83..99e3243a65e 100644 --- a/packages/teraslice/src/lib/cluster/services/cluster/backends/kubernetesV2/utils.ts +++ b/packages/teraslice/src/lib/cluster/services/cluster/backends/kubernetesV2/utils.ts @@ -10,39 +10,15 @@ const MAX_RETRIES = isTest ? 2 : 3; const RETRY_DELAY = isTest ? 50 : 1000; // time in ms const resourcePath = path.join(process.cwd(), './packages/teraslice/src/lib/cluster/services/cluster/backends/kubernetesV2/'); -export function makeTemplate( - folder: 'deployments', - fileName: NodeType -): (config: K8sConfig) => k8s.V1Deployment; -export function makeTemplate( - folder: 'jobs', - fileName: NodeType -): (config: K8sConfig) => k8s.V1Job; -export function makeTemplate( - folder: 'services', - fileName: NodeType -): (config: K8sConfig) => k8s.V1Service; -export function makeTemplate( +export function makeTemplate( folder: 'deployments' | 'jobs' | 'services', fileName: NodeType -): (config: K8sConfig) => k8s.V1Deployment | k8s.V1Job | k8s.V1Service; -export function makeTemplate( - folder: 'deployments' | 'jobs' | 'services', - fileName: NodeType -): (config: K8sConfig) => k8s.V1Deployment | k8s.V1Job | k8s.V1Service { +): (config: K8sConfig) => T { const filePath = path.join(resourcePath, folder, `${fileName}.hbs`); const templateData = fs.readFileSync(filePath, 'utf-8'); const templateKeys = ['{{', '}}']; return (config: K8sConfig) => { - if (folder !== 'jobs' && (config.exName === undefined || config.exUid === undefined)) { - throw new Error(`K8s config requires ${config.exName === undefined ? 'exName' : 'exUid'} to create a ${folder} template`); - } - - if (folder !== 'services' && config.dockerImage === undefined) { - throw new Error(`K8s config requires a dockerImage to create a ${folder} template`); - } - const templated = barbe(templateData, templateKeys, config); return JSON.parse(templated); }; diff --git a/packages/teraslice/test/lib/cluster/services/cluster/backends/kubernetes/v2/k8sResource-v2-spec.ts b/packages/teraslice/test/lib/cluster/services/cluster/backends/kubernetes/v2/k8sResource-v2-spec.ts index 54bf26360cb..aa11be4af01 100644 --- a/packages/teraslice/test/lib/cluster/services/cluster/backends/kubernetes/v2/k8sResource-v2-spec.ts +++ b/packages/teraslice/test/lib/cluster/services/cluster/backends/kubernetes/v2/k8sResource-v2-spec.ts @@ -2,7 +2,9 @@ import _ from 'lodash'; import yaml from 'js-yaml'; import * as k8s from '@kubernetes/client-node'; import { debugLogger } from '@terascope/utils'; -import { K8sResource } from '../../../../../../../../src/lib/cluster/services/cluster/backends/kubernetesV2/k8sResource.js'; +import { K8sDeploymentResource } from '../../../../../../../../src/lib/cluster/services/cluster/backends/kubernetesV2/k8sDeploymentResource.js'; +import { K8sJobResource } from '../../../../../../../../src/lib/cluster/services/cluster/backends/kubernetesV2/k8sJobResource.js'; +import { K8sServiceResource } from '../../../../../../../../src/lib/cluster/services/cluster/backends/kubernetesV2/k8sServiceResource.js'; describe('k8sResource', () => { const logger = debugLogger('k8sResource'); @@ -38,27 +40,26 @@ describe('k8sResource', () => { describe('worker deployment', () => { it('has valid resource object.', () => { - const kr = new K8sResource('deployments', 'worker', terasliceConfig, execution, logger, 'example-job-abcd', 'UID1'); - const resource = kr.resource as k8s.V1Deployment; + const kr = new K8sDeploymentResource(terasliceConfig, execution, logger, 'example-job-abcd', 'UID1'); - expect(resource.kind).toBe('Deployment'); - expect(resource.spec?.replicas).toBe(2); - expect(resource.metadata?.name).toBe('ts-wkr-example-data-generator-job-7ba9afb0-417a'); + expect(kr.resource.kind).toBe('Deployment'); + expect(kr.resource.spec?.replicas).toBe(2); + expect(kr.resource.metadata?.name).toBe('ts-wkr-example-data-generator-job-7ba9afb0-417a'); // The following properties should be absent in the default case // Note: This tests that both affinity and podAntiAffinity are absent - expect(resource.spec?.template.spec).not.toHaveProperty('affinity'); - expect(resource.spec?.template.spec).not.toHaveProperty('imagePullSecrets'); - expect(resource.spec?.template.spec).not.toHaveProperty('priorityClassName'); + expect(kr.resource.spec?.template.spec).not.toHaveProperty('affinity'); + expect(kr.resource.spec?.template.spec).not.toHaveProperty('imagePullSecrets'); + expect(kr.resource.spec?.template.spec).not.toHaveProperty('priorityClassName'); let configVolume: k8s.V1Volume | undefined = undefined; - if (resource.spec?.template.spec?.volumes) { - configVolume = resource.spec.template.spec.volumes[0]; + if (kr.resource.spec?.template.spec?.volumes) { + configVolume = kr.resource.spec.template.spec.volumes[0]; } let configVolumeMount: k8s.V1VolumeMount | undefined = undefined; - if (resource.spec?.template.spec?.containers[0].volumeMounts) { - configVolumeMount = resource.spec.template.spec.containers[0].volumeMounts[0]; + if (kr.resource.spec?.template.spec?.containers[0].volumeMounts) { + configVolumeMount = kr.resource.spec.template.spec.containers[0].volumeMounts[0]; } // Configmaps should be mounted on all workers expect(configVolume).toEqual(yaml.load(` @@ -76,12 +77,11 @@ describe('k8sResource', () => { it('has valid resource object when terasliceConfig has kubernetes_image_pull_secret.', () => { terasliceConfig.kubernetes_image_pull_secret = 'teraslice-image-pull-secret'; - const kr = new K8sResource('deployments', 'worker', terasliceConfig, execution, logger, 'example-job-abcd', 'UID1'); - const resource = kr.resource as k8s.V1Deployment; + const kr = new K8sDeploymentResource(terasliceConfig, execution, logger, 'example-job-abcd', 'UID1'); let firstSecret: k8s.V1LocalObjectReference | undefined = undefined; - if (resource.spec?.template.spec?.imagePullSecrets) { - firstSecret = resource.spec.template.spec.imagePullSecrets[0]; + if (kr.resource.spec?.template.spec?.imagePullSecrets) { + firstSecret = kr.resource.spec.template.spec.imagePullSecrets[0]; } expect(firstSecret).toEqual( @@ -92,11 +92,10 @@ describe('k8sResource', () => { it('has podAntiAffinity when terasliceConfig has kubernetes_worker_antiaffinity true.', () => { terasliceConfig.kubernetes_worker_antiaffinity = true; - const kr = new K8sResource('deployments', 'worker', terasliceConfig, execution, logger, 'example-job-abcd', 'UID1'); - const resource = kr.resource as k8s.V1Deployment; + const kr = new K8sDeploymentResource(terasliceConfig, execution, logger, 'example-job-abcd', 'UID1'); // console.log(yaml.dump(kr.resource.spec.template.spec.affinity)); - expect(resource.spec?.template.spec?.affinity).toEqual( + expect(kr.resource.spec?.template.spec?.affinity).toEqual( yaml.load(` podAntiAffinity: preferredDuringSchedulingIgnoredDuringExecution: @@ -120,35 +119,34 @@ describe('k8sResource', () => { terasliceConfig.assets_directory = '/assets'; terasliceConfig.assets_volume = 'asset-volume'; - const kr = new K8sResource('deployments', 'worker', terasliceConfig, execution, logger, 'example-job-abcd', 'UID1'); - const resource = kr.resource as k8s.V1Deployment; + const kr = new K8sDeploymentResource(terasliceConfig, execution, logger, 'example-job-abcd', 'UID1'); let configVolume: k8s.V1Volume | undefined = undefined; - if (resource.spec?.template.spec?.volumes) { - configVolume = resource.spec.template.spec.volumes[0]; + if (kr.resource.spec?.template.spec?.volumes) { + configVolume = kr.resource.spec.template.spec.volumes[0]; } let assetVolume: k8s.V1Volume | undefined = undefined; - if (resource.spec?.template.spec?.volumes) { - assetVolume = resource.spec.template.spec.volumes[1]; + if (kr.resource.spec?.template.spec?.volumes) { + assetVolume = kr.resource.spec.template.spec.volumes[1]; } let configVolumeMount: k8s.V1VolumeMount | undefined = undefined; - if (resource.spec?.template.spec?.containers[0].volumeMounts) { - configVolumeMount = resource.spec.template.spec.containers[0].volumeMounts[0]; + if (kr.resource.spec?.template.spec?.containers[0].volumeMounts) { + configVolumeMount = kr.resource.spec.template.spec.containers[0].volumeMounts[0]; } let assetVolumeMount: k8s.V1VolumeMount | undefined = undefined; - if (resource.spec?.template.spec?.containers[0].volumeMounts) { - assetVolumeMount = resource.spec.template.spec.containers[0].volumeMounts[1]; + if (kr.resource.spec?.template.spec?.containers[0].volumeMounts) { + assetVolumeMount = kr.resource.spec.template.spec.containers[0].volumeMounts[1]; } - expect(resource.spec?.replicas).toBe(2); - expect(resource.metadata?.name).toBe('ts-wkr-example-data-generator-job-7ba9afb0-417a'); + expect(kr.resource.spec?.replicas).toBe(2); + expect(kr.resource.metadata?.name).toBe('ts-wkr-example-data-generator-job-7ba9afb0-417a'); // The following properties should be absent in the default case - expect(resource.spec?.template.spec).not.toHaveProperty('affinity'); - expect(resource.spec?.template.spec).not.toHaveProperty('imagePullSecrets'); + expect(kr.resource.spec?.template.spec).not.toHaveProperty('affinity'); + expect(kr.resource.spec?.template.spec).not.toHaveProperty('imagePullSecrets'); expect(configVolume).toEqual(yaml.load(` name: config configMap: @@ -177,23 +175,22 @@ describe('k8sResource', () => { { name: 'teraslice-data1', path: '/data' } ]; - const kr = new K8sResource('deployments', 'worker', terasliceConfig, execution, logger, 'example-job-abcd', 'UID1'); - const resource = kr.resource as k8s.V1Deployment; + const kr = new K8sDeploymentResource(terasliceConfig, execution, logger, 'example-job-abcd', 'UID1'); - const configVolume1 = resource.spec?.template.spec?.volumes - ? resource.spec.template.spec.volumes[0] + const configVolume1 = kr.resource.spec?.template.spec?.volumes + ? kr.resource.spec.template.spec.volumes[0] : undefined; - const configVolume2 = resource.spec?.template.spec?.volumes - ? resource.spec.template.spec.volumes[1] + const configVolume2 = kr.resource.spec?.template.spec?.volumes + ? kr.resource.spec.template.spec.volumes[1] : undefined; - const configVolumeMount1 = resource.spec?.template.spec?.containers[0].volumeMounts - ? resource.spec.template.spec.containers[0].volumeMounts[0] + const configVolumeMount1 = kr.resource.spec?.template.spec?.containers[0].volumeMounts + ? kr.resource.spec.template.spec.containers[0].volumeMounts[0] : undefined; - const configVolumeMount2 = resource.spec?.template.spec?.containers[0].volumeMounts - ? resource.spec.template.spec.containers[0].volumeMounts[1] + const configVolumeMount2 = kr.resource.spec?.template.spec?.containers[0].volumeMounts + ? kr.resource.spec.template.spec.containers[0].volumeMounts[1] : undefined; // First check the configMap volumes, which should be present on all @@ -227,23 +224,22 @@ describe('k8sResource', () => { { name: 'tmp', path: '/tmp' } ]; - const kr = new K8sResource('deployments', 'worker', terasliceConfig, execution, logger, 'example-job-abcd', 'UID1'); - const resource = kr.resource as k8s.V1Deployment; + const kr = new K8sDeploymentResource(terasliceConfig, execution, logger, 'example-job-abcd', 'UID1'); - const configVolume2 = resource.spec?.template.spec?.volumes - ? resource.spec.template.spec.volumes[1] + const configVolume2 = kr.resource.spec?.template.spec?.volumes + ? kr.resource.spec.template.spec.volumes[1] : undefined; - const configVolume3 = resource.spec?.template.spec?.volumes - ? resource.spec.template.spec.volumes[2] + const configVolume3 = kr.resource.spec?.template.spec?.volumes + ? kr.resource.spec.template.spec.volumes[2] : undefined; - const configVolumeMount2 = resource.spec?.template.spec?.containers[0].volumeMounts - ? resource.spec.template.spec.containers[0].volumeMounts[1] + const configVolumeMount2 = kr.resource.spec?.template.spec?.containers[0].volumeMounts + ? kr.resource.spec.template.spec.containers[0].volumeMounts[1] : undefined; - const configVolumeMount3 = resource.spec?.template.spec?.containers[0].volumeMounts - ? resource.spec.template.spec.containers[0].volumeMounts[2] + const configVolumeMount3 = kr.resource.spec?.template.spec?.containers[0].volumeMounts + ? kr.resource.spec.template.spec.containers[0].volumeMounts[2] : undefined; // Now check for the volumes added via job @@ -267,17 +263,16 @@ describe('k8sResource', () => { }); it('does not have memory/cpu limits/requests when not set in config or execution', () => { - const kr = new K8sResource('deployments', 'worker', terasliceConfig, execution, logger, 'example-job-abcd', 'UID1'); - const resource = kr.resource as k8s.V1Deployment; + const kr = new K8sDeploymentResource(terasliceConfig, execution, logger, 'example-job-abcd', 'UID1'); - const exId = resource.metadata?.labels ? resource.metadata.labels['teraslice.terascope.io/exId'] : undefined; + const exId = kr.resource.metadata?.labels ? kr.resource.metadata.labels['teraslice.terascope.io/exId'] : undefined; expect(exId) .toEqual('e76a0278-d9bc-4d78-bf14-431bcd97528c'); - const resources = resource.spec?.template.spec?.containers[0].resources; + const resources = kr.resource.spec?.template.spec?.containers[0].resources; expect(resources).not.toBeDefined(); - const envArray = resource.spec?.template.spec?.containers[0].env; + const envArray = kr.resource.spec?.template.spec?.containers[0].env; expect(envArray).not.toContain('NODE_OPTIONS'); }); @@ -285,14 +280,13 @@ describe('k8sResource', () => { terasliceConfig.cpu = 1; terasliceConfig.memory = 2147483648; - const kr = new K8sResource('deployments', 'worker', terasliceConfig, execution, logger, 'example-job-abcd', 'UID1'); - const resource = kr.resource as k8s.V1Deployment; + const kr = new K8sDeploymentResource(terasliceConfig, execution, logger, 'example-job-abcd', 'UID1'); - const exId = resource.metadata?.labels ? resource.metadata.labels['teraslice.terascope.io/exId'] : undefined; + const exId = kr.resource.metadata?.labels ? kr.resource.metadata.labels['teraslice.terascope.io/exId'] : undefined; expect(exId) .toEqual('e76a0278-d9bc-4d78-bf14-431bcd97528c'); - const resources = resource.spec?.template.spec?.containers[0].resources; + const resources = kr.resource.spec?.template.spec?.containers[0].resources; expect(resources).toEqual(yaml.load(` requests: memory: 2147483648 @@ -301,19 +295,18 @@ describe('k8sResource', () => { memory: 2147483648 cpu: 1`)); - const envArray = resource.spec?.template.spec?.containers[0].env; + const envArray = kr.resource.spec?.template.spec?.containers[0].env; const nodeOptionsEnv = _.find(envArray, { name: 'NODE_OPTIONS' }); expect(nodeOptionsEnv?.value) .toEqual('--max-old-space-size=1843'); }); it('has the ability to set custom env', () => { - const kr = new K8sResource('deployments', 'worker', terasliceConfig, execution, logger, 'example-job-abcd', 'UID1'); - const resource = kr.resource as k8s.V1Deployment; + const kr = new K8sDeploymentResource(terasliceConfig, execution, logger, 'example-job-abcd', 'UID1'); // NOTE: the env var merge happens in _setResources(), which is // somewhat out of place. - const envArray = resource.spec?.template.spec?.containers[0].env; + const envArray = kr.resource.spec?.template.spec?.containers[0].env; const fooEnv = _.find(envArray, { name: 'FOO' }); expect(fooEnv?.value) @@ -330,13 +323,12 @@ describe('k8sResource', () => { terasliceConfig.cpu = 1; terasliceConfig.memory = 2147483648; - const kr = new K8sResource('deployments', 'worker', terasliceConfig, execution, logger, 'example-job-abcd', 'UID1'); - const resource = kr.resource as k8s.V1Deployment; + const kr = new K8sDeploymentResource(terasliceConfig, execution, logger, 'example-job-abcd', 'UID1'); - const exId = resource.metadata?.labels ? resource.metadata.labels['teraslice.terascope.io/exId'] : undefined; + const exId = kr.resource.metadata?.labels ? kr.resource.metadata.labels['teraslice.terascope.io/exId'] : undefined; expect(exId) .toEqual('e76a0278-d9bc-4d78-bf14-431bcd97528c'); - expect(resource.spec?.template.spec?.containers[0].resources).toEqual(yaml.load(` + expect(kr.resource.spec?.template.spec?.containers[0].resources).toEqual(yaml.load(` requests: memory: 1073741824 cpu: 2 @@ -344,7 +336,7 @@ describe('k8sResource', () => { memory: 1073741824 cpu: 2`)); - const envArray = resource.spec?.template.spec?.containers[0].env; + const envArray = kr.resource.spec?.template.spec?.containers[0].env; const nodeOptionsEnv = _.find(envArray, { name: 'NODE_OPTIONS' }); expect(nodeOptionsEnv?.value) .toEqual('--max-old-space-size=922'); @@ -355,13 +347,12 @@ describe('k8sResource', () => { terasliceConfig.cpu = 1; terasliceConfig.memory = 2147483648; - const kr = new K8sResource('deployments', 'worker', terasliceConfig, execution, logger, 'example-job-abcd', 'UID1'); - const resource = kr.resource as k8s.V1Deployment; + const kr = new K8sDeploymentResource(terasliceConfig, execution, logger, 'example-job-abcd', 'UID1'); - const exId = resource.metadata?.labels ? resource.metadata.labels['teraslice.terascope.io/exId'] : undefined; + const exId = kr.resource.metadata?.labels ? kr.resource.metadata.labels['teraslice.terascope.io/exId'] : undefined; expect(exId) .toEqual('e76a0278-d9bc-4d78-bf14-431bcd97528c'); - expect(resource.spec?.template.spec?.containers[0].resources).toEqual(yaml.load(` + expect(kr.resource.spec?.template.spec?.containers[0].resources).toEqual(yaml.load(` requests: memory: 2147483648 cpu: 2 @@ -369,7 +360,7 @@ describe('k8sResource', () => { memory: 2147483648 cpu: 2`)); - const envArray = resource.spec?.template.spec?.containers[0].env; + const envArray = kr.resource.spec?.template.spec?.containers[0].env; const nodeOptionsEnv = _.find(envArray, { name: 'NODE_OPTIONS' }); expect(nodeOptionsEnv?.value) .toEqual('--max-old-space-size=1843'); @@ -379,13 +370,12 @@ describe('k8sResource', () => { execution.cpu = 1; execution.memory = 2147483648; - const kr = new K8sResource('deployments', 'worker', terasliceConfig, execution, logger, 'example-job-abcd', 'UID1'); - const resource = kr.resource as k8s.V1Deployment; + const kr = new K8sDeploymentResource(terasliceConfig, execution, logger, 'example-job-abcd', 'UID1'); - const exId = resource.metadata?.labels ? resource.metadata.labels['teraslice.terascope.io/exId'] : undefined; + const exId = kr.resource.metadata?.labels ? kr.resource.metadata.labels['teraslice.terascope.io/exId'] : undefined; expect(exId) .toEqual('e76a0278-d9bc-4d78-bf14-431bcd97528c'); - expect(resource.spec?.template.spec?.containers[0].resources).toEqual(yaml.load(` + expect(kr.resource.spec?.template.spec?.containers[0].resources).toEqual(yaml.load(` requests: memory: 2147483648 cpu: 1 @@ -393,7 +383,7 @@ describe('k8sResource', () => { memory: 2147483648 cpu: 1`)); - const envArray = resource.spec?.template.spec?.containers[0].env; + const envArray = kr.resource.spec?.template.spec?.containers[0].env; const nodeOptionsEnv = _.find(envArray, { name: 'NODE_OPTIONS' }); expect(nodeOptionsEnv?.value) .toEqual('--max-old-space-size=1843'); @@ -405,13 +395,12 @@ describe('k8sResource', () => { execution.resources_requests_memory = 2147483648; execution.resources_limits_memory = 3147483648; - const kr = new K8sResource('deployments', 'worker', terasliceConfig, execution, logger, 'example-job-abcd', 'UID1'); - const resource = kr.resource as k8s.V1Deployment; + const kr = new K8sDeploymentResource(terasliceConfig, execution, logger, 'example-job-abcd', 'UID1'); - const exId = resource.metadata?.labels ? resource.metadata.labels['teraslice.terascope.io/exId'] : undefined; + const exId = kr.resource.metadata?.labels ? kr.resource.metadata.labels['teraslice.terascope.io/exId'] : undefined; expect(exId) .toEqual('e76a0278-d9bc-4d78-bf14-431bcd97528c'); - expect(resource.spec?.template.spec?.containers[0].resources).toEqual(yaml.load(` + expect(kr.resource.spec?.template.spec?.containers[0].resources).toEqual(yaml.load(` requests: memory: 2147483648 cpu: 1 @@ -419,7 +408,7 @@ describe('k8sResource', () => { memory: 3147483648 cpu: 2`)); - const envArray = resource.spec?.template.spec?.containers[0].env; + const envArray = kr.resource.spec?.template.spec?.containers[0].env; const nodeOptionsEnv = _.find(envArray, { name: 'NODE_OPTIONS' }); expect(nodeOptionsEnv?.value) .toEqual('--max-old-space-size=2702'); @@ -428,16 +417,15 @@ describe('k8sResource', () => { it('has memory limits and requests when set on execution', () => { execution.memory = 2147483648; - const kr = new K8sResource('deployments', 'worker', terasliceConfig, execution, logger, 'example-job-abcd', 'UID1'); - const resource = kr.resource as k8s.V1Deployment; + const kr = new K8sDeploymentResource(terasliceConfig, execution, logger, 'example-job-abcd', 'UID1'); - expect(resource.spec?.template.spec?.containers[0].resources).toEqual(yaml.load(` + expect(kr.resource.spec?.template.spec?.containers[0].resources).toEqual(yaml.load(` requests: memory: 2147483648 limits: memory: 2147483648`)); - const envArray = resource.spec?.template.spec?.containers[0].env; + const envArray = kr.resource.spec?.template.spec?.containers[0].env; const nodeOptionsEnv = _.find(envArray, { name: 'NODE_OPTIONS' }); expect(nodeOptionsEnv?.value) .toEqual('--max-old-space-size=1843'); @@ -446,10 +434,9 @@ describe('k8sResource', () => { it('has cpu limits and requests when set on execution', () => { execution.cpu = 1; - const kr = new K8sResource('deployments', 'worker', terasliceConfig, execution, logger, 'example-job-abcd', 'UID1'); - const resource = kr.resource as k8s.V1Deployment; + const kr = new K8sDeploymentResource(terasliceConfig, execution, logger, 'example-job-abcd', 'UID1'); - expect(resource.spec?.template.spec?.containers[0].resources).toEqual(yaml.load(` + expect(kr.resource.spec?.template.spec?.containers[0].resources).toEqual(yaml.load(` requests: cpu: 1 limits: @@ -459,10 +446,9 @@ describe('k8sResource', () => { it('has scratch volume when ephemeral_storage is set true on execution', () => { execution.ephemeral_storage = true; - const kr = new K8sResource('deployments', 'worker', terasliceConfig, execution, logger, 'example-job-abcd', 'UID1'); - const resource = kr.resource as k8s.V1Deployment; + const kr = new K8sDeploymentResource(terasliceConfig, execution, logger, 'example-job-abcd', 'UID1'); - expect(resource.spec?.template.spec?.containers[0].volumeMounts) + expect(kr.resource.spec?.template.spec?.containers[0].volumeMounts) .toEqual( [ { mountPath: '/app/config', name: 'config' }, @@ -474,10 +460,9 @@ describe('k8sResource', () => { it('does not have scratch volume when ephemeral_storage is set false on execution', () => { execution.ephemeral_storage = false; - const kr = new K8sResource('deployments', 'worker', terasliceConfig, execution, logger, 'example-job-abcd', 'UID1'); - const resource = kr.resource as k8s.V1Deployment; + const kr = new K8sDeploymentResource(terasliceConfig, execution, logger, 'example-job-abcd', 'UID1'); - expect(resource.spec?.template.spec?.containers[0].volumeMounts) + expect(kr.resource.spec?.template.spec?.containers[0].volumeMounts) .toEqual([{ mountPath: '/app/config', name: 'config' }]); }); }); @@ -485,21 +470,19 @@ describe('k8sResource', () => { describe('worker deployments with targets', () => { it('does not have affinity or toleration properties when targets equals [].', () => { execution.targets = []; - const kr = new K8sResource('deployments', 'worker', terasliceConfig, execution, logger, 'example-job-abcd', 'UID1'); - const resource = kr.resource as k8s.V1Deployment; + const kr = new K8sDeploymentResource(terasliceConfig, execution, logger, 'example-job-abcd', 'UID1'); - expect(resource.spec?.template.spec).not.toHaveProperty('affinity'); - expect(resource.spec?.template.spec).not.toHaveProperty('toleration'); + expect(kr.resource.spec?.template.spec).not.toHaveProperty('affinity'); + expect(kr.resource.spec?.template.spec).not.toHaveProperty('toleration'); }); it('has valid resource object with affinity when execution has one target without constraint', () => { execution.targets = [ { key: 'zone', value: 'west' } ]; - const kr = new K8sResource('deployments', 'worker', terasliceConfig, execution, logger, 'example-job-abcd', 'UID1'); - const resource = kr.resource as k8s.V1Deployment; + const kr = new K8sDeploymentResource(terasliceConfig, execution, logger, 'example-job-abcd', 'UID1'); - expect(resource.spec?.template.spec?.affinity).toEqual(yaml.load(` + expect(kr.resource.spec?.template.spec?.affinity).toEqual(yaml.load(` nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: @@ -515,10 +498,9 @@ describe('k8sResource', () => { { key: 'zone', value: 'west' } ]; terasliceConfig.kubernetes_worker_antiaffinity = true; - const kr = new K8sResource('deployments', 'worker', terasliceConfig, execution, logger, 'example-job-abcd', 'UID1'); - const resource = kr.resource as k8s.V1Deployment; + const kr = new K8sDeploymentResource(terasliceConfig, execution, logger, 'example-job-abcd', 'UID1'); - expect(resource.spec?.template.spec?.affinity).toEqual(yaml.load(` + expect(kr.resource.spec?.template.spec?.affinity).toEqual(yaml.load(` nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: @@ -548,10 +530,9 @@ describe('k8sResource', () => { execution.targets = [ { key: 'zone', value: 'west', constraint: 'required' } ]; - const kr = new K8sResource('deployments', 'worker', terasliceConfig, execution, logger, 'example-job-abcd', 'UID1'); - const resource = kr.resource as k8s.V1Deployment; + const kr = new K8sDeploymentResource(terasliceConfig, execution, logger, 'example-job-abcd', 'UID1'); - expect(resource.spec?.template.spec?.affinity).toEqual(yaml.load(` + expect(kr.resource.spec?.template.spec?.affinity).toEqual(yaml.load(` nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: @@ -567,10 +548,9 @@ describe('k8sResource', () => { { key: 'zone', value: 'west', constraint: 'required' }, { key: 'region', value: '42', constraint: 'required' } ]; - const kr = new K8sResource('deployments', 'worker', terasliceConfig, execution, logger, 'example-job-abcd', 'UID1'); - const resource = kr.resource as k8s.V1Deployment; + const kr = new K8sDeploymentResource(terasliceConfig, execution, logger, 'example-job-abcd', 'UID1'); - expect(resource.spec?.template.spec?.affinity).toEqual(yaml.load(` + expect(kr.resource.spec?.template.spec?.affinity).toEqual(yaml.load(` nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: @@ -589,10 +569,9 @@ describe('k8sResource', () => { execution.targets = [ { key: 'zone', value: 'west', constraint: 'preferred' } ]; - const kr = new K8sResource('deployments', 'worker', terasliceConfig, execution, logger, 'example-job-abcd', 'UID1'); - const resource = kr.resource as k8s.V1Deployment; + const kr = new K8sDeploymentResource(terasliceConfig, execution, logger, 'example-job-abcd', 'UID1'); - expect(resource.spec?.template.spec?.affinity).toEqual(yaml.load(` + expect(kr.resource.spec?.template.spec?.affinity).toEqual(yaml.load(` nodeAffinity: preferredDuringSchedulingIgnoredDuringExecution: - weight: 1 @@ -609,10 +588,9 @@ describe('k8sResource', () => { { key: 'zone', value: 'west', constraint: 'preferred' }, { key: 'region', value: 'texas', constraint: 'preferred' } ]; - const kr = new K8sResource('deployments', 'worker', terasliceConfig, execution, logger, 'example-job-abcd', 'UID1'); - const resource = kr.resource as k8s.V1Deployment; + const kr = new K8sDeploymentResource(terasliceConfig, execution, logger, 'example-job-abcd', 'UID1'); - expect(resource.spec?.template.spec?.affinity).toEqual(yaml.load(` + expect(kr.resource.spec?.template.spec?.affinity).toEqual(yaml.load(` nodeAffinity: preferredDuringSchedulingIgnoredDuringExecution: - weight: 1 @@ -635,11 +613,10 @@ describe('k8sResource', () => { execution.targets = [ { key: 'zone', value: 'west', constraint: 'accepted' } ]; - const kr = new K8sResource('deployments', 'worker', terasliceConfig, execution, logger, 'example-job-abcd', 'UID1'); - const resource = kr.resource as k8s.V1Deployment; + const kr = new K8sDeploymentResource(terasliceConfig, execution, logger, 'example-job-abcd', 'UID1'); // console.log(yaml.dump(kr.resource.spec.template.spec.tolerations)); - expect(resource.spec?.template.spec?.tolerations).toEqual(yaml.load(` + expect(kr.resource.spec?.template.spec?.tolerations).toEqual(yaml.load(` - key: zone operator: Equal value: west @@ -651,11 +628,10 @@ describe('k8sResource', () => { { key: 'zone', value: 'west', constraint: 'accepted' }, { key: 'region', value: 'texas', constraint: 'accepted' } ]; - const kr = new K8sResource('deployments', 'worker', terasliceConfig, execution, logger, 'example-job-abcd', 'UID1'); - const resource = kr.resource as k8s.V1Deployment; + const kr = new K8sDeploymentResource(terasliceConfig, execution, logger, 'example-job-abcd', 'UID1'); // console.log(yaml.dump(kr.resource.spec.template.spec.tolerations)); - expect(resource.spec?.template.spec?.tolerations).toEqual(yaml.load(` + expect(kr.resource.spec?.template.spec?.tolerations).toEqual(yaml.load(` - key: zone operator: Equal value: west @@ -671,11 +647,10 @@ describe('k8sResource', () => { { key: 'zone', value: 'west', constraint: 'required' }, { key: 'region', value: 'texas', constraint: 'preferred' } ]; - const kr = new K8sResource('deployments', 'worker', terasliceConfig, execution, logger, 'example-job-abcd', 'UID1'); - const resource = kr.resource as k8s.V1Deployment; + const kr = new K8sDeploymentResource(terasliceConfig, execution, logger, 'example-job-abcd', 'UID1'); // console.log(yaml.dump(kr.resource.spec.template.spec.tolerations)); - expect(resource.spec?.template.spec?.affinity).toEqual(yaml.load(` + expect(kr.resource.spec?.template.spec?.affinity).toEqual(yaml.load(` nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: @@ -703,10 +678,9 @@ describe('k8sResource', () => { { key: 'zone', value: 'west', constraint: 'accepted' }, { key: 'region', value: 'texas', constraint: 'accepted' } ]; - const kr = new K8sResource('deployments', 'worker', terasliceConfig, execution, logger, 'example-job-abcd', 'UID1'); - const resource = kr.resource as k8s.V1Deployment; + const kr = new K8sDeploymentResource(terasliceConfig, execution, logger, 'example-job-abcd', 'UID1'); - expect(resource.spec?.template.spec?.affinity).toEqual(yaml.load(` + expect(kr.resource.spec?.template.spec?.affinity).toEqual(yaml.load(` nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: @@ -734,7 +708,7 @@ describe('k8sResource', () => { operator: In values: - texas`)); - expect(resource.spec?.template.spec?.tolerations).toEqual(yaml.load(` + expect(kr.resource.spec?.template.spec?.tolerations).toEqual(yaml.load(` - key: zone operator: Equal value: west @@ -753,12 +727,11 @@ describe('k8sResource', () => { key2: 'value2' }; - const kr = new K8sResource('deployments', 'worker', terasliceConfig, execution, logger, 'example-job-abcd', 'UID1'); - const resource = kr.resource as k8s.V1Deployment; + const kr = new K8sDeploymentResource(terasliceConfig, execution, logger, 'example-job-abcd', 'UID1'); // console.log(yaml.dump(kr.resource)); - const key1Value = resource.metadata?.labels ? resource.metadata.labels['job.teraslice.terascope.io/key1'] : undefined; - const key2Value = resource.metadata?.labels ? resource.metadata.labels['job.teraslice.terascope.io/key2'] : undefined; + const key1Value = kr.resource.metadata?.labels ? kr.resource.metadata.labels['job.teraslice.terascope.io/key1'] : undefined; + const key2Value = kr.resource.metadata?.labels ? kr.resource.metadata.labels['job.teraslice.terascope.io/key2'] : undefined; expect(key1Value).toEqual('value1'); expect(key2Value).toEqual('value2'); }); @@ -770,18 +743,17 @@ describe('k8sResource', () => { abcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghij1234: 'value3', }; - const kr = new K8sResource('deployments', 'worker', terasliceConfig, execution, logger, 'example-job-abcd', 'UID1'); - const resource = kr.resource as k8s.V1Deployment; + const kr = new K8sDeploymentResource(terasliceConfig, execution, logger, 'example-job-abcd', 'UID1'); // console.log(yaml.dump(kr.resource)); - const forbidden1Value = resource.metadata?.labels - ? resource.metadata.labels['job.teraslice.terascope.io/key-1'] + const forbidden1Value = kr.resource.metadata?.labels + ? kr.resource.metadata.labels['job.teraslice.terascope.io/key-1'] : undefined; - const forbidden2Value = resource.metadata?.labels - ? resource.metadata.labels['job.teraslice.terascope.io/key2-'] + const forbidden2Value = kr.resource.metadata?.labels + ? kr.resource.metadata.labels['job.teraslice.terascope.io/key2-'] : undefined; - const forbidden3Value = resource.metadata?.labels - ? resource.metadata.labels['job.teraslice.terascope.io/abcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghij123'] + const forbidden3Value = kr.resource.metadata?.labels + ? kr.resource.metadata.labels['job.teraslice.terascope.io/abcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghij123'] : undefined; expect(forbidden1Value).toEqual('value1'); expect(forbidden2Value).toEqual('value2'); @@ -792,11 +764,10 @@ describe('k8sResource', () => { describe('teraslice job with one valid external_ports set', () => { it('generates k8s worker deployment with containerPort on container', () => { execution.external_ports = [9090]; - const kr = new K8sResource('deployments', 'worker', terasliceConfig, execution, logger, 'example-job-abcd', 'UID1'); - const resource = kr.resource as k8s.V1Deployment; + const kr = new K8sDeploymentResource(terasliceConfig, execution, logger, 'example-job-abcd', 'UID1'); // console.log(yaml.dump(kr.resource.spec.template.spec.containers[0].ports)); - expect(resource.spec?.template.spec?.containers[0].ports) + expect(kr.resource.spec?.template.spec?.containers[0].ports) .toEqual([ { containerPort: 45680 }, { containerPort: 9090 } @@ -805,11 +776,10 @@ describe('k8sResource', () => { it('generates k8s execution controller job with containerPort on container', () => { execution.external_ports = [9090]; - const kr = new K8sResource('jobs', 'execution_controller', terasliceConfig, execution, logger); - const resource = kr.resource as k8s.V1Job; + const kr = new K8sJobResource(terasliceConfig, execution, logger); // console.log(yaml.dump(kr.resource.spec.template.spec.containers[0].ports)); - expect(resource.spec?.template.spec?.containers[0].ports) + expect(kr.resource.spec?.template.spec?.containers[0].ports) .toEqual([ { containerPort: 45680 }, { containerPort: 9090 } @@ -823,11 +793,10 @@ describe('k8sResource', () => { 9090, { name: 'metrics', port: 9091 } ]; - const kr = new K8sResource('deployments', 'worker', terasliceConfig, execution, logger, 'example-job-abcd', 'UID1'); - const resource = kr.resource as k8s.V1Deployment; + const kr = new K8sDeploymentResource(terasliceConfig, execution, logger, 'example-job-abcd', 'UID1'); // console.log(yaml.dump(kr.resource.spec.template.spec.containers[0].ports)); - expect(resource.spec?.template.spec?.containers[0].ports) + expect(kr.resource.spec?.template.spec?.containers[0].ports) .toEqual([ { containerPort: 45680 }, { containerPort: 9090 }, @@ -837,11 +806,10 @@ describe('k8sResource', () => { it('generates k8s execution controller job with containerPort on container', () => { execution.external_ports = [9090, 9091]; - const kr = new K8sResource('jobs', 'execution_controller', terasliceConfig, execution, logger); - const resource = kr.resource as k8s.V1Job; + const kr = new K8sJobResource(terasliceConfig, execution, logger); // console.log(yaml.dump(kr.resource.spec.template.spec.containers[0].ports)); - expect(resource.spec?.template.spec?.containers[0].ports) + expect(kr.resource.spec?.template.spec?.containers[0].ports) .toEqual([ { containerPort: 45680 }, { containerPort: 9090 }, @@ -852,27 +820,26 @@ describe('k8sResource', () => { describe('execution_controller job', () => { it('has valid resource object.', () => { - const kr = new K8sResource('jobs', 'execution_controller', terasliceConfig, execution, logger); - const resource = kr.resource as k8s.V1Job; + const kr = new K8sJobResource(terasliceConfig, execution, logger); - expect(resource.kind).toBe('Job'); - expect(resource.metadata?.name).toBe('ts-exc-example-data-generator-job-7ba9afb0-417a'); + expect(kr.resource.kind).toBe('Job'); + expect(kr.resource.metadata?.name).toBe('ts-exc-example-data-generator-job-7ba9afb0-417a'); // The following properties should be absent in the default case - expect(resource.spec?.template.spec).not.toHaveProperty('affinity'); - expect(resource.spec?.template.spec).not.toHaveProperty('imagePullSecrets'); - expect(resource.spec?.template.spec).not.toHaveProperty('priorityClassName'); + expect(kr.resource.spec?.template.spec).not.toHaveProperty('affinity'); + expect(kr.resource.spec?.template.spec).not.toHaveProperty('imagePullSecrets'); + expect(kr.resource.spec?.template.spec).not.toHaveProperty('priorityClassName'); // Configmaps should be mounted on all workers let configVolume: k8s.V1Volume | undefined = undefined; - if (resource.spec?.template.spec?.volumes) { - configVolume = resource.spec.template.spec.volumes[0]; + if (kr.resource.spec?.template.spec?.volumes) { + configVolume = kr.resource.spec.template.spec.volumes[0]; } let configVolumeMount: k8s.V1VolumeMount | undefined = undefined; - if (resource.spec?.template.spec?.containers[0].volumeMounts) { - configVolumeMount = resource.spec.template.spec.containers[0].volumeMounts[0]; + if (kr.resource.spec?.template.spec?.containers[0].volumeMounts) { + configVolumeMount = kr.resource.spec.template.spec.containers[0].volumeMounts[0]; } expect(configVolume).toEqual(yaml.load(` @@ -892,10 +859,9 @@ describe('k8sResource', () => { terasliceConfig.cpu_execution_controller = 1; terasliceConfig.memory_execution_controller = 2147483648; - const kr = new K8sResource('jobs', 'execution_controller', terasliceConfig, execution, logger); - const resource = kr.resource as k8s.V1Job; + const kr = new K8sJobResource(terasliceConfig, execution, logger); - expect(resource.spec?.template.spec?.containers[0].resources).toEqual(yaml.load(` + expect(kr.resource.spec?.template.spec?.containers[0].resources).toEqual(yaml.load(` requests: memory: 2147483648 cpu: 1 @@ -903,7 +869,7 @@ describe('k8sResource', () => { memory: 2147483648 cpu: 1`)); - const envArray = resource.spec?.template.spec?.containers[0].env; + const envArray = kr.resource.spec?.template.spec?.containers[0].env; const nodeOptionsEnv = _.find(envArray, { name: 'NODE_OPTIONS' }); expect(nodeOptionsEnv?.value) .toEqual('--max-old-space-size=1843'); @@ -915,10 +881,9 @@ describe('k8sResource', () => { terasliceConfig.cpu_execution_controller = 1; terasliceConfig.memory_execution_controller = 2147483648; - const kr = new K8sResource('jobs', 'execution_controller', terasliceConfig, execution, logger); - const resource = kr.resource as k8s.V1Job; + const kr = new K8sJobResource(terasliceConfig, execution, logger); - expect(resource.spec?.template.spec?.containers[0].resources).toEqual(yaml.load(` + expect(kr.resource.spec?.template.spec?.containers[0].resources).toEqual(yaml.load(` requests: memory: 1073741824 cpu: 2 @@ -926,7 +891,7 @@ describe('k8sResource', () => { memory: 1073741824 cpu: 2`)); - const envArray = resource.spec?.template.spec?.containers[0].env; + const envArray = kr.resource.spec?.template.spec?.containers[0].env; const nodeOptionsEnv = _.find(envArray, { name: 'NODE_OPTIONS' }); expect(nodeOptionsEnv?.value) .toEqual('--max-old-space-size=922'); @@ -944,10 +909,9 @@ describe('k8sResource', () => { ['teraslice-JOB-name', 'ts-wkr-teraslice-job-name-7ba9afb0-417a'] ])('when Job Name is %s the k8s worker name is: %s', (jobName, k8sName) => { execution.name = jobName; - const kr = new K8sResource('deployments', 'worker', terasliceConfig, execution, logger, 'example-job-abcd', 'UID1'); - const resource = kr.resource as k8s.V1Deployment; + const kr = new K8sDeploymentResource(terasliceConfig, execution, logger, 'example-job-abcd', 'UID1'); - expect(resource.metadata?.name).toBe(k8sName); + expect(kr.resource.metadata?.name).toBe(k8sName); }); }); @@ -958,15 +922,14 @@ describe('k8sResource', () => { { key: 'key2', value: 'value2' } ]; - const kr = new K8sResource('jobs', 'execution_controller', terasliceConfig, execution, logger); - const resource = kr.resource as k8s.V1Deployment; + const kr = new K8sJobResource(terasliceConfig, execution, logger); - expect(resource.kind).toBe('Job'); - expect(resource.metadata?.name).toBe('ts-exc-example-data-generator-job-7ba9afb0-417a'); + expect(kr.resource.kind).toBe('Job'); + expect(kr.resource.metadata?.name).toBe('ts-exc-example-data-generator-job-7ba9afb0-417a'); - expect(resource.spec?.template.spec).toHaveProperty('affinity'); - expect(resource.spec?.template.spec).toHaveProperty('tolerations'); - expect(resource.spec?.template.spec?.affinity).toEqual(yaml.load(` + expect(kr.resource.spec?.template.spec).toHaveProperty('affinity'); + expect(kr.resource.spec?.template.spec).toHaveProperty('tolerations'); + expect(kr.resource.spec?.template.spec?.affinity).toEqual(yaml.load(` nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: @@ -979,7 +942,7 @@ describe('k8sResource', () => { operator: In values: - value2`)); - expect(resource.spec?.template.spec?.tolerations).toEqual(yaml.load(` + expect(kr.resource.spec?.template.spec?.tolerations).toEqual(yaml.load(` - key: key1 operator: Equal value: value1 @@ -1002,17 +965,16 @@ describe('k8sResource', () => { { key: 'region', value: 'texas', constraint: 'accepted' } ]; - const kr = new K8sResource('jobs', 'execution_controller', terasliceConfig, execution, logger); - const resource = kr.resource as k8s.V1Deployment; + const kr = new K8sJobResource(terasliceConfig, execution, logger); - expect(resource.kind).toBe('Job'); - expect(resource.metadata?.name).toBe('ts-exc-example-data-generator-job-7ba9afb0-417a'); + expect(kr.resource.kind).toBe('Job'); + expect(kr.resource.metadata?.name).toBe('ts-exc-example-data-generator-job-7ba9afb0-417a'); - expect(resource.spec?.template.spec).toHaveProperty('affinity'); - expect(resource.spec?.template.spec).toHaveProperty('tolerations'); + expect(kr.resource.spec?.template.spec).toHaveProperty('affinity'); + expect(kr.resource.spec?.template.spec).toHaveProperty('tolerations'); // console.log(yaml.dump(kr.resource.spec.template.spec.affinity)); - expect(resource.spec?.template.spec?.affinity).toEqual(yaml.load(` + expect(kr.resource.spec?.template.spec?.affinity).toEqual(yaml.load(` nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: @@ -1027,7 +989,7 @@ describe('k8sResource', () => { - value1`)); // console.log(yaml.dump(kr.resource.spec.template.spec.tolerations)); - expect(resource.spec?.template.spec?.tolerations).toEqual(yaml.load(` + expect(kr.resource.spec?.template.spec?.tolerations).toEqual(yaml.load(` - key: region operator: Equal value: texas @@ -1044,29 +1006,27 @@ describe('k8sResource', () => { execution.stateful = true; terasliceConfig.kubernetes_priority_class_name = 'testPriorityClass'; - const krWorker = new K8sResource('deployments', 'worker', terasliceConfig, execution, logger, 'example-job-abcd', 'UID1'); - const wkrResource = krWorker.resource as k8s.V1Deployment; + const krWorker = new K8sDeploymentResource(terasliceConfig, execution, logger, 'example-job-abcd', 'UID1'); - expect(wkrResource.kind).toBe('Deployment'); - expect(wkrResource.spec?.template.spec).toHaveProperty('priorityClassName'); - expect(wkrResource.spec?.template.spec?.priorityClassName).toEqual('testPriorityClass'); + expect(krWorker.resource.kind).toBe('Deployment'); + expect(krWorker.resource.spec?.template.spec).toHaveProperty('priorityClassName'); + expect(krWorker.resource.spec?.template.spec?.priorityClassName).toEqual('testPriorityClass'); - const wkrStatefulLabel = wkrResource.spec?.template.metadata?.labels - ? wkrResource.spec.template.metadata.labels['job-property.teraslice.terascope.io/stateful'] + const wkrStatefulLabel = krWorker.resource.spec?.template.metadata?.labels + ? krWorker.resource.spec.template.metadata.labels['job-property.teraslice.terascope.io/stateful'] : undefined; expect(wkrStatefulLabel).toEqual('true'); - const krExporter = new K8sResource('jobs', 'execution_controller', terasliceConfig, execution, logger); - const exporterResource = krExporter.resource as k8s.V1Job; + const krExporter = new K8sJobResource(terasliceConfig, execution, logger); - expect(exporterResource.kind).toBe('Job'); - expect(exporterResource.metadata?.name).toBe('ts-exc-example-data-generator-job-7ba9afb0-417a'); + expect(krExporter.resource.kind).toBe('Job'); + expect(krExporter.resource.metadata?.name).toBe('ts-exc-example-data-generator-job-7ba9afb0-417a'); - expect(exporterResource.spec?.template.spec).toHaveProperty('priorityClassName'); - expect(exporterResource.spec?.template.spec?.priorityClassName).toEqual('testPriorityClass'); + expect(krExporter.resource.spec?.template.spec).toHaveProperty('priorityClassName'); + expect(krExporter.resource.spec?.template.spec?.priorityClassName).toEqual('testPriorityClass'); - const exporterStatefulLabel = exporterResource.spec?.template.metadata?.labels - ? exporterResource.spec.template.metadata.labels['job-property.teraslice.terascope.io/stateful'] + const exporterStatefulLabel = krExporter.resource.spec?.template.metadata?.labels + ? krExporter.resource.spec.template.metadata.labels['job-property.teraslice.terascope.io/stateful'] : undefined; expect(exporterStatefulLabel).toEqual('true'); }); @@ -1077,10 +1037,9 @@ describe('k8sResource', () => { execution.pod_spec_override = { initContainers: [] }; terasliceConfig.kubernetes_overrides_enabled = false; - const krWorker = new K8sResource('deployments', 'worker', terasliceConfig, execution, logger, 'example-job-abcd', 'UID1'); - const resource = krWorker.resource as k8s.V1Deployment; + const krWorker = new K8sDeploymentResource(terasliceConfig, execution, logger, 'example-job-abcd', 'UID1'); - expect(resource.spec?.template.spec).not.toHaveProperty('initContainers'); + expect(krWorker.resource.spec?.template.spec).not.toHaveProperty('initContainers'); }); }); @@ -1089,10 +1048,32 @@ describe('k8sResource', () => { execution.pod_spec_override = { initContainers: [] }; terasliceConfig.kubernetes_overrides_enabled = true; - const krWorker = new K8sResource('deployments', 'worker', terasliceConfig, execution, logger, 'example-job-abcd', 'UID1'); - const resource = krWorker.resource as k8s.V1Deployment; + const krWorker = new K8sDeploymentResource(terasliceConfig, execution, logger, 'example-job-abcd', 'UID1'); - expect(resource.spec?.template.spec).toHaveProperty('initContainers'); + expect(krWorker.resource.spec?.template.spec).toHaveProperty('initContainers'); + }); + }); + + describe('execution_controller service', () => { + it('has valid resource object.', () => { + const kr = new K8sServiceResource(terasliceConfig, execution, logger, 'example-job-abcd', 'UID1'); + + expect(kr.resource.kind).toBe('Service'); + expect(kr.resource.metadata?.name).toBe('svc-ts-exc-example-data-generator-job-7ba9afb0-417a'); + + let serviceSpec: k8s.V1ServiceSpec | undefined = undefined; + if (kr.resource.spec) { + serviceSpec = kr.resource.spec; + } + + expect(serviceSpec).toEqual(yaml.load(` + selector: + app.kubernetes.io/component: "execution_controller" + teraslice.terascope.io/exId: "e76a0278-d9bc-4d78-bf14-431bcd97528c" + ports: + - port: 45680 + targetPort: 45680 + `)); }); }); }); diff --git a/packages/teraslice/test/lib/cluster/services/cluster/backends/kubernetes/v2/utils-v2-spec.ts b/packages/teraslice/test/lib/cluster/services/cluster/backends/kubernetes/v2/utils-v2-spec.ts index 2390bf6bded..a7de37fa3b3 100644 --- a/packages/teraslice/test/lib/cluster/services/cluster/backends/kubernetes/v2/utils-v2-spec.ts +++ b/packages/teraslice/test/lib/cluster/services/cluster/backends/kubernetes/v2/utils-v2-spec.ts @@ -1,3 +1,4 @@ +import * as k8s from '@kubernetes/client-node'; import { K8sConfig } from '../../../../../../../../src/lib/cluster/services/cluster/backends/kubernetesV2/interfaces.js'; import { makeTemplate, getMaxOldSpace @@ -8,7 +9,7 @@ describe('K8s Utils', () => { describe('->makeTemplate', () => { describe('execution_controller job', () => { it('should be able to support the execution_controller job', () => { - const exJobTemplate = makeTemplate('jobs', 'execution_controller'); + const exJobTemplate = makeTemplate('jobs', 'execution_controller'); const config: K8sConfig = { clusterName: 'teracluster', configMapName: 'teracluster-worker', @@ -93,7 +94,7 @@ describe('K8s Utils', () => { }); describe('worker deployment', () => { - const workerDeploymentTemplate = makeTemplate('deployments', 'worker'); + const workerDeploymentTemplate = makeTemplate('deployments', 'worker'); it('should be able to support the worker deployment', () => { const config: K8sConfig = { clusterName: 'teracluster', @@ -234,7 +235,7 @@ describe('K8s Utils', () => { }); describe('execution_controller service', () => { - const exServiceTemplate = makeTemplate('services', 'execution_controller'); + const exServiceTemplate = makeTemplate('services', 'execution_controller'); it('should be able to support the execution_controller service', () => { const config: K8sConfig = { clusterName: 'teracluster', From c46ad38731dfb44462cb45fdaeb716238e1f9ce9 Mon Sep 17 00:00:00 2001 From: busma13 Date: Fri, 1 Nov 2024 16:13:49 -0700 Subject: [PATCH 15/30] fix jsdoc comments --- .../services/cluster/backends/kubernetesV2/k8s.ts | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/packages/teraslice/src/lib/cluster/services/cluster/backends/kubernetesV2/k8s.ts b/packages/teraslice/src/lib/cluster/services/cluster/backends/kubernetesV2/k8s.ts index dadeae806af..87ee14d652b 100644 --- a/packages/teraslice/src/lib/cluster/services/cluster/backends/kubernetesV2/k8s.ts +++ b/packages/teraslice/src/lib/cluster/services/cluster/backends/kubernetesV2/k8s.ts @@ -162,7 +162,7 @@ export class K8s { * returns list of k8s objects matching provided selector * @param {String} selector kubernetes selector, like 'app=teraslice' * @param {ResourceType} objType Type of k8s object to get, valid options: - * 'deployment', 'job', 'pod', 'replicaset', 'service' + * 'deployments', 'jobs', 'pods', 'replicasets', 'services' * @param {String} ns namespace to search, this will override the default * @return {k8s.V1PodList * | k8s.V1DeploymentList @@ -247,7 +247,6 @@ export class K8s { /** * posts manifest to k8s * @param {Resource} manifest resource manifest - * @param {ResourceType} manifestType 'deployment', 'job', 'pod', 'replicaset', 'service' * @return {Resource} body of k8s API response object */ async post(manifest: k8s.V1Deployment): Promise; @@ -323,7 +322,7 @@ export class K8s { * Deletes k8s object of specified objType * @param {String} name Name of the resource to delete * @param {ResourceType} objType Type of k8s object to get, valid options: - * 'deployment', 'service', 'job', 'pod', 'replicaset' + * 'deployments', 'services', 'jobs', 'pods', 'replicasetss' * @param {Boolean} force Forcefully delete resource by setting gracePeriodSeconds to 1 * to be forcefully stopped. * @return {Object} k8s delete response body. @@ -447,8 +446,8 @@ export class K8s { * @param {String} exId Execution ID * @param {NodeType} nodeType valid Teraslice k8s node type: * 'worker', 'execution_controller' - * @param {ResourceType} objType valid object type: `service`, `deployment`, - * `job`, `pod`, `replicaset` + * @param {ResourceType} objType valid object type: `services`, `deployments`, + * `jobs`, `pods`, `replicasets` * @param {Boolean} force Forcefully stop all resources * @return {Promise} */ From 827aab6dcde0dc5a22e1e3a07a216a89cd288573 Mon Sep 17 00:00:00 2001 From: busma13 Date: Fri, 1 Nov 2024 16:14:01 -0700 Subject: [PATCH 16/30] removed unused imports --- .../cluster/backends/kubernetesV2/k8sDeploymentResource.ts | 1 - .../services/cluster/backends/kubernetesV2/k8sJobResource.ts | 1 - .../services/cluster/backends/kubernetesV2/k8sServiceResource.ts | 1 - 3 files changed, 3 deletions(-) diff --git a/packages/teraslice/src/lib/cluster/services/cluster/backends/kubernetesV2/k8sDeploymentResource.ts b/packages/teraslice/src/lib/cluster/services/cluster/backends/kubernetesV2/k8sDeploymentResource.ts index 1ab8ec98fe4..460eb83236c 100644 --- a/packages/teraslice/src/lib/cluster/services/cluster/backends/kubernetesV2/k8sDeploymentResource.ts +++ b/packages/teraslice/src/lib/cluster/services/cluster/backends/kubernetesV2/k8sDeploymentResource.ts @@ -1,4 +1,3 @@ -import _ from 'lodash'; import * as k8s from '@kubernetes/client-node'; import { Logger } from '@terascope/utils'; import type { Config, ExecutionConfig } from '@terascope/types'; diff --git a/packages/teraslice/src/lib/cluster/services/cluster/backends/kubernetesV2/k8sJobResource.ts b/packages/teraslice/src/lib/cluster/services/cluster/backends/kubernetesV2/k8sJobResource.ts index 647f12168d5..cf1514f440a 100644 --- a/packages/teraslice/src/lib/cluster/services/cluster/backends/kubernetesV2/k8sJobResource.ts +++ b/packages/teraslice/src/lib/cluster/services/cluster/backends/kubernetesV2/k8sJobResource.ts @@ -1,4 +1,3 @@ -import _ from 'lodash'; import * as k8s from '@kubernetes/client-node'; import { Logger } from '@terascope/utils'; import type { Config, ExecutionConfig } from '@terascope/types'; diff --git a/packages/teraslice/src/lib/cluster/services/cluster/backends/kubernetesV2/k8sServiceResource.ts b/packages/teraslice/src/lib/cluster/services/cluster/backends/kubernetesV2/k8sServiceResource.ts index f311b0bb25c..92aaa4f1582 100644 --- a/packages/teraslice/src/lib/cluster/services/cluster/backends/kubernetesV2/k8sServiceResource.ts +++ b/packages/teraslice/src/lib/cluster/services/cluster/backends/kubernetesV2/k8sServiceResource.ts @@ -1,4 +1,3 @@ -import _ from 'lodash'; import * as k8s from '@kubernetes/client-node'; import { Logger } from '@terascope/utils'; import type { Config, ExecutionConfig } from '@terascope/types'; From 383b68e2521e6b99f0980fdedeafede789de10da Mon Sep 17 00:00:00 2001 From: busma13 Date: Fri, 1 Nov 2024 16:35:01 -0700 Subject: [PATCH 17/30] missed a resourceType rename in test --- .../services/cluster/backends/kubernetes/v2/k8s-v2-spec.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/teraslice/test/lib/cluster/services/cluster/backends/kubernetes/v2/k8s-v2-spec.ts b/packages/teraslice/test/lib/cluster/services/cluster/backends/kubernetes/v2/k8s-v2-spec.ts index b827883a2c0..0453230c801 100644 --- a/packages/teraslice/test/lib/cluster/services/cluster/backends/kubernetes/v2/k8s-v2-spec.ts +++ b/packages/teraslice/test/lib/cluster/services/cluster/backends/kubernetes/v2/k8s-v2-spec.ts @@ -358,7 +358,7 @@ describe('k8s', () => { }); await expect(k8s._deleteObjByExId('no-name', 'execution_controller', 'jobs')) - .rejects.toThrow('Cannot delete job for ExId: no-name by name because it has no name'); + .rejects.toThrow('Cannot delete jobs for ExId: no-name by name because it has no name'); }); it('can delete a single object', async () => { From 7cfc9c1872ebcb4ae12ed26f5086419a56858efe Mon Sep 17 00:00:00 2001 From: busma13 Date: Fri, 1 Nov 2024 16:35:34 -0700 Subject: [PATCH 18/30] accidentally removed undefined checks in makeTemplate --- .../services/cluster/backends/kubernetesV2/utils.ts | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/packages/teraslice/src/lib/cluster/services/cluster/backends/kubernetesV2/utils.ts b/packages/teraslice/src/lib/cluster/services/cluster/backends/kubernetesV2/utils.ts index 99e3243a65e..36342cdfbc0 100644 --- a/packages/teraslice/src/lib/cluster/services/cluster/backends/kubernetesV2/utils.ts +++ b/packages/teraslice/src/lib/cluster/services/cluster/backends/kubernetesV2/utils.ts @@ -19,6 +19,14 @@ export function makeTemplate { + if (folder !== 'jobs' && (config.exName === undefined || config.exUid === undefined)) { + throw new Error(`K8s config requires ${config.exName === undefined ? 'exName' : 'exUid'} to create a ${folder} template`); + } + + if (folder !== 'services' && config.dockerImage === undefined) { + throw new Error(`K8s config requires a dockerImage to create a ${folder} template`); + } + const templated = barbe(templateData, templateKeys, config); return JSON.parse(templated); }; From fe4f2b03df5b3be78ef570311ee35d1eac44fdd1 Mon Sep 17 00:00:00 2001 From: busma13 Date: Fri, 1 Nov 2024 17:04:32 -0700 Subject: [PATCH 19/30] fix tests --- .../cluster/backends/kubernetes/v2/utils-v2-spec.ts | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/packages/teraslice/test/lib/cluster/services/cluster/backends/kubernetes/v2/utils-v2-spec.ts b/packages/teraslice/test/lib/cluster/services/cluster/backends/kubernetes/v2/utils-v2-spec.ts index a7de37fa3b3..5324d97833c 100644 --- a/packages/teraslice/test/lib/cluster/services/cluster/backends/kubernetes/v2/utils-v2-spec.ts +++ b/packages/teraslice/test/lib/cluster/services/cluster/backends/kubernetes/v2/utils-v2-spec.ts @@ -89,7 +89,7 @@ describe('K8s Utils', () => { shutdownTimeout: 12345 }; expect(() => exJobTemplate(config)) - .toThrow('K8s config requires a dockerImage to create a job template'); + .toThrow('K8s config requires a dockerImage to create a jobs template'); }); }); @@ -188,7 +188,7 @@ describe('K8s Utils', () => { shutdownTimeout: 12345 }; expect(() => workerDeploymentTemplate(config)) - .toThrow('K8s config requires a dockerImage to create a deployment template'); + .toThrow('K8s config requires a dockerImage to create a deployments template'); }); it('should throw error if exName undefined on config for deployment', () => { @@ -209,7 +209,7 @@ describe('K8s Utils', () => { replicas: 1, shutdownTimeout: 12345 }; - expect(() => workerDeploymentTemplate(config)).toThrow('K8s config requires exName to create a deployment template'); + expect(() => workerDeploymentTemplate(config)).toThrow('K8s config requires exName to create a deployments template'); }); it('should throw error if exUid undefined on config for deployment', () => { @@ -230,7 +230,7 @@ describe('K8s Utils', () => { replicas: 1, shutdownTimeout: 12345 }; - expect(() => workerDeploymentTemplate(config)).toThrow('K8s config requires exUid to create a deployment template'); + expect(() => workerDeploymentTemplate(config)).toThrow('K8s config requires exUid to create a deployments template'); }); }); @@ -309,7 +309,7 @@ describe('K8s Utils', () => { replicas: 1, shutdownTimeout: 12345 }; - expect(() => exServiceTemplate(config)).toThrow('K8s config requires exName to create a service template'); + expect(() => exServiceTemplate(config)).toThrow('K8s config requires exName to create a services template'); }); it('should throw error if exUid undefined on config for service', () => { @@ -330,7 +330,7 @@ describe('K8s Utils', () => { replicas: 1, shutdownTimeout: 12345 }; - expect(() => exServiceTemplate(config)).toThrow('K8s config requires exUid to create a service template'); + expect(() => exServiceTemplate(config)).toThrow('K8s config requires exUid to create a services template'); }); }); }); From 6cadcd5291f0c5bbe2126b9df71141770862554d Mon Sep 17 00:00:00 2001 From: busma13 Date: Wed, 6 Nov 2024 09:14:05 -0700 Subject: [PATCH 20/30] extend k8s resources to TS resources that have required fields --- .../cluster/backends/kubernetesV2/index.ts | 38 +- .../backends/kubernetesV2/interfaces.ts | 128 ++++++- .../cluster/backends/kubernetesV2/k8s.ts | 98 +++--- .../kubernetesV2/k8sDeploymentResource.ts | 7 +- .../backends/kubernetesV2/k8sJobResource.ts | 7 +- .../backends/kubernetesV2/k8sResource.ts | 92 ++--- .../kubernetesV2/k8sServiceResource.ts | 7 +- .../cluster/backends/kubernetesV2/k8sState.ts | 82 ++--- .../cluster/backends/kubernetesV2/utils.ts | 170 ++++++++- .../backends/kubernetes/v2/k8s-v2-spec.ts | 235 ++++++++----- .../kubernetes/v2/k8sResource-v2-spec.ts | 328 +++++++----------- .../v2/k8sState-multicluster-v2-spec.ts | 15 +- .../kubernetes/v2/k8sState-v2-spec.ts | 19 +- .../backends/kubernetes/v2/utils-v2-spec.ts | 11 +- 14 files changed, 706 insertions(+), 531 deletions(-) diff --git a/packages/teraslice/src/lib/cluster/services/cluster/backends/kubernetesV2/index.ts b/packages/teraslice/src/lib/cluster/services/cluster/backends/kubernetesV2/index.ts index c9fcbd600ac..2ca7b06a0ea 100644 --- a/packages/teraslice/src/lib/cluster/services/cluster/backends/kubernetesV2/index.ts +++ b/packages/teraslice/src/lib/cluster/services/cluster/backends/kubernetesV2/index.ts @@ -74,7 +74,7 @@ export class KubernetesClusterBackendV2 { */ private async _getClusterState() { return this.k8s.list(`app.kubernetes.io/name=teraslice,app.kubernetes.io/instance=${this.clusterNameLabel}`, 'pods') - .then((k8sPods) => gen(k8sPods, this.clusterState, this.logger)) + .then((tsPodList) => gen(tsPodList, this.clusterState)) .catch((err) => { // TODO: We might need to do more here. I think it's OK to just // log though. This only gets used to show slicer info through @@ -123,17 +123,6 @@ export class KubernetesClusterBackendV2 { const jobResult = await this.k8s.post(exJob); - // fixme - if (!jobResult.metadata) { - throw new Error('Required field metadata missing from jobResult'); - } - if (!jobResult.metadata.name) { - throw new Error('Required field name missing from jobResult.metadata'); - } - if (!jobResult.metadata.uid) { - throw new Error('Required field uid missing from jobResult.metadata'); - } - const exServiceResource = new K8sServiceResource( this.context.sysconfig.teraslice, execution, @@ -150,7 +139,7 @@ export class KubernetesClusterBackendV2 { this.logger.debug(jobResult, 'k8s slicer job submitted'); let controllerLabel: string; - if (jobResult.spec?.selector?.matchLabels?.['controller-uid'] !== undefined) { + if (jobResult.spec.selector.matchLabels['controller-uid'] !== undefined) { /// If running on kubernetes < v1.27.0 controllerLabel = 'controller-uid'; } else { @@ -158,7 +147,7 @@ export class KubernetesClusterBackendV2 { controllerLabel = 'batch.kubernetes.io/controller-uid'; } - const controllerUid = jobResult.spec?.selector?.matchLabels?.[controllerLabel]; + const controllerUid = jobResult.spec.selector.matchLabels[controllerLabel]; const pod = await this.k8s.waitForSelectedPod( `${controllerLabel}=${controllerUid}`, @@ -171,7 +160,7 @@ export class KubernetesClusterBackendV2 { const error = new Error('pod.status.podIP must be defined'); return Promise.reject(error); } - const exServiceName = serviceResult.metadata?.name; + const exServiceName = serviceResult.metadata.name; const exServiceHostName = `${exServiceName}.${this.k8s.defaultNamespace}`; this.logger.debug(`Slicer is using host name: ${exServiceHostName}`); @@ -204,23 +193,12 @@ export class KubernetesClusterBackendV2 { this.context.sysconfig.teraslice.slicer_timeout ); - // fixme - if (!jobs.items[0].metadata) { - throw new Error('Required field metadata missing from jobResult'); - } - if (!jobs.items[0].metadata.name) { - throw new Error('Required field name missing from jobResult.metadata'); - } - if (!jobs.items[0].metadata.uid) { - throw new Error('Required field uid missing from jobResult.metadata'); - } - const kr = new K8sDeploymentResource( this.context.sysconfig.teraslice, execution, this.logger, - jobs.items[0].metadata?.name, - jobs.items[0].metadata?.uid + jobs.items[0].metadata.name, + jobs.items[0].metadata.uid ); const workerDeployment = kr.resource; @@ -277,8 +255,8 @@ export class KubernetesClusterBackendV2 { /** * Returns a list of all k8s resources associated with a job ID * @param {string} jobId The job ID of the job to list associated resources - * @returns {Array} + * @returns {Array} */ async listResourcesForJobId(jobId: string) { const resources = []; diff --git a/packages/teraslice/src/lib/cluster/services/cluster/backends/kubernetesV2/interfaces.ts b/packages/teraslice/src/lib/cluster/services/cluster/backends/kubernetesV2/interfaces.ts index fe18b8d9d2f..b8b28cd5e01 100644 --- a/packages/teraslice/src/lib/cluster/services/cluster/backends/kubernetesV2/interfaces.ts +++ b/packages/teraslice/src/lib/cluster/services/cluster/backends/kubernetesV2/interfaces.ts @@ -8,10 +8,6 @@ export interface KubeConfigOptions { users: k8s.User[]; } -export type K8sObjectList = - k8s.V1DeploymentList | k8s.V1ServiceList - | k8s.V1JobList | k8s.V1PodList | k8s.V1ReplicaSetList; - export interface K8sConfig { clusterName: string; clusterNameLabel: string; @@ -32,19 +28,135 @@ export interface K8sConfig { export type ResourceType = 'deployments' | 'jobs' | 'pods' | 'replicasets' | 'services'; -export type ResourceList = k8s.V1DeploymentList | k8s.V1JobList +export type K8sResource = k8s.V1Deployment | k8s.V1Job + | k8s.V1Pod | k8s.V1ReplicaSet | k8s.V1Service; + +export type TSResource = TSDeployment | TSJob | TSPod | TSReplicaSet | TSService; + +export interface TSDeployment extends k8s.V1Deployment { + kind: NonNullable; + metadata: NonNullable & { + labels: { + [key: string]: string; + }; + name: string; + }; + spec: NonNullable & { + replicas: NonNullable; + template: NonNullable & { + metadata: NonNullable & { + labels: { + [key: string]: string; + }; + }; + spec: NonNullable & { + containers: k8s.V1Container[] & { + ports: NonNullable; + volumeMounts: [k8s.V1VolumeMount, ...k8s.V1VolumeMount[]]; + }[]; + volumes: NonNullable; + }; + }; + }; +} + +export interface TSJob extends k8s.V1Job { + kind: NonNullable; + metadata: NonNullable & { + labels: { + [key: string]: string; + }; + name: string; + uid: string; + }; + spec: NonNullable & { + template: k8s.V1PodTemplateSpec & { + metadata: NonNullable & { + labels: { + [key: string]: string; + }; + }; + spec: NonNullable & { + containers: k8s.V1Container[] & { + ports: NonNullable; + volumeMounts: [k8s.V1VolumeMount, ...k8s.V1VolumeMount[]]; + }[]; + volumes: NonNullable; + }; + }; + selector: NonNullable & { + matchLabels: { + [key: string]: string; + }; + }; + }; +} + +export interface TSPod extends k8s.V1Pod { + kind: NonNullable; + metadata: NonNullable & { + labels: { + [key: string]: string; + }; + name: string; + }; + spec: NonNullable; + status: NonNullable & { + hostIP: 'string'; + }; +} + +export interface TSReplicaSet extends k8s.V1ReplicaSet { + kind: NonNullable; + metadata: NonNullable & { + name: string; + }; + status: NonNullable; +} + +export interface TSService extends k8s.V1Service { + kind: NonNullable; + metadata: NonNullable & { + name: string; + }; + spec: NonNullable & { + selector: { + [key: string]: string; + }; + ports: NonNullable; + }; +} + +export type K8sResourceList = k8s.V1DeploymentList | k8s.V1JobList | k8s.V1PodList | k8s.V1ReplicaSetList | k8s.V1ServiceList; -export type Resource = k8s.V1Deployment | k8s.V1Job | k8s.V1Pod | k8s.V1ReplicaSet | k8s.V1Service; +export type TSResourceList = TSDeploymentList | TSJobList + | TSPodList | TSReplicaSetList | TSServiceList; + +export interface TSDeploymentList extends k8s.V1DeploymentList { + items: TSDeployment[]; +} +export interface TSJobList extends k8s.V1JobList { + items: TSJob[]; +} +export interface TSPodList extends k8s.V1PodList { + items: TSPod[]; +} +export interface TSReplicaSetList extends k8s.V1ReplicaSetList { + items: TSReplicaSet[]; +} +export interface TSServiceList extends k8s.V1ServiceList { + items: TSService[]; +} export interface ResourceListApiResponse { response: IncomingMessage; - body: ResourceList; + body: K8sResourceList; } export interface ResourceApiResponse { response: IncomingMessage; - body: Resource; + body: K8sResource; } export interface PatchApiResponse { diff --git a/packages/teraslice/src/lib/cluster/services/cluster/backends/kubernetesV2/k8s.ts b/packages/teraslice/src/lib/cluster/services/cluster/backends/kubernetesV2/k8s.ts index 87ee14d652b..6378e8796f7 100644 --- a/packages/teraslice/src/lib/cluster/services/cluster/backends/kubernetesV2/k8s.ts +++ b/packages/teraslice/src/lib/cluster/services/cluster/backends/kubernetesV2/k8s.ts @@ -4,15 +4,18 @@ import { } from '@terascope/utils'; import * as k8s from '@kubernetes/client-node'; import { - getRetryConfig, isDeployment, isJob, - isPod, isReplicaSet, isService + convertToTSResource, convertToTSResourceList, getRetryConfig, isDeployment, + isJob, isPod, isReplicaSet, isService, isTSPod } from './utils.js'; import { DeleteApiResponse, DeleteParams, DeleteResponseBody, - K8sObjectList, KubeConfigOptions, ListParams, - NodeType, PatchApiResponse, Resource, - ResourceApiResponse, ResourceList, ResourceListApiResponse, - ResourceType, ScaleOp + KubeConfigOptions, K8sResource, ListParams, + NodeType, PatchApiResponse, ResourceApiResponse, + ResourceListApiResponse, ResourceType, ScaleOp, + TSDeployment, TSDeploymentList, TSJob, + TSJobList, TSPod, TSPodList, + TSReplicaSet, TSReplicaSetList, TSResource, + TSResourceList, TSService, TSServiceList } from './interfaces.js'; export class K8s { @@ -94,15 +97,12 @@ export class K8s { const result = await pRetry(() => this.k8sCoreV1Api .listNamespacedPod(namespace, undefined, undefined, undefined, undefined, selector), getRetryConfig()); - let pod: k8s.V1Pod | undefined; - if (typeof result !== 'undefined' && result) { - // NOTE: This assumes the first pod returned. - pod = get(result, 'body.items[0]'); - } + // NOTE: This assumes the first pod returned. + const pod = get(result, 'body.items[0]'); - if (pod) { + if (pod && isTSPod(pod)) { if (statusType === 'readiness-probe') { - if (pod.status?.conditions) { + if (pod.status.conditions) { for (const condition of pod.status.conditions) { if ( condition.type === 'ContainersReady' @@ -170,13 +170,13 @@ export class K8s { * | k8s.V1ReplicaSetList * | k8s.V1JobList} list of k8s objects. */ - async list(selector: string, objType: 'deployments', ns?: string): Promise; - async list(selector: string, objType: 'jobs', ns?: string): Promise; - async list(selector: string, objType: 'pods', ns?: string): Promise; - async list(selector: string, objType: 'replicasets', ns?: string): Promise; - async list(selector: string, objType: 'services', ns?: string): Promise; - async list(selector: string, objType: ResourceType, ns?: string): Promise; - async list(selector: string, objType: ResourceType, ns?: string): Promise { + async list(selector: string, objType: 'deployments', ns?: string): Promise; + async list(selector: string, objType: 'jobs', ns?: string): Promise; + async list(selector: string, objType: 'pods', ns?: string): Promise; + async list(selector: string, objType: 'replicasets', ns?: string): Promise; + async list(selector: string, objType: 'services', ns?: string): Promise; + async list(selector: string, objType: ResourceType, ns?: string): Promise; + async list(selector: string, objType: ResourceType, ns?: string): Promise { const namespace = ns || this.defaultNamespace; let responseObj: ResourceListApiResponse; @@ -220,8 +220,7 @@ export class K8s { this.logger.error(error); return Promise.reject(error); } - - return responseObj.body; + return convertToTSResourceList(responseObj.body); } catch (e) { const err = new Error(`Request k8s.list of ${objType} with selector ${selector} failed: ${e}`); this.logger.error(err); @@ -229,7 +228,7 @@ export class K8s { } } - async nonEmptyJobList(selector: string) { + async nonEmptyJobList(selector: string): Promise { const jobs = await this.list(selector, 'jobs'); if (jobs.items.length === 1) { return jobs; @@ -246,15 +245,15 @@ export class K8s { /** * posts manifest to k8s - * @param {Resource} manifest resource manifest - * @return {Resource} body of k8s API response object + * @param {K8sResource} manifest resource manifest + * @return {K8sResource} body of k8s API response object */ - async post(manifest: k8s.V1Deployment): Promise; - async post(manifest: k8s.V1Job): Promise; - async post(manifest: k8s.V1Pod): Promise; - async post(manifest: k8s.V1ReplicaSet): Promise; - async post(manifest: k8s.V1Service): Promise; - async post(manifest: Resource): Promise { + async post(manifest: k8s.V1Deployment): Promise; + async post(manifest: k8s.V1Job): Promise; + async post(manifest: k8s.V1Pod): Promise; + async post(manifest: k8s.V1ReplicaSet): Promise; + async post(manifest: k8s.V1Service): Promise; + async post(manifest: K8sResource): Promise { let responseObj: ResourceApiResponse; try { @@ -278,7 +277,7 @@ export class K8s { return Promise.reject(error); } - return responseObj.body; + return convertToTSResource(responseObj.body); } catch (e) { const err = new Error(`Request k8s.post of ${manifest.kind} with body ${JSON.stringify(manifest)} failed: ${e}`); return Promise.reject(err); @@ -372,7 +371,9 @@ export class K8s { deleteOptions ]; - const deleteWithErrorHandling = async (deleteFn: () => Promise) => { + const deleteWithErrorHandling = async ( + deleteFn: () => Promise + ): Promise => { try { const res = await deleteFn(); return res; @@ -449,7 +450,7 @@ export class K8s { * @param {ResourceType} objType valid object type: `services`, `deployments`, * `jobs`, `pods`, `replicasets` * @param {Boolean} force Forcefully stop all resources - * @return {Promise} + * @return {Promise<(k8s.V1Pod | k8s.V1Status | k8s.V1Service)[]>} */ async _deleteObjByExId( exId: string, nodeType: NodeType, objType: 'pods', force?: boolean @@ -466,7 +467,7 @@ export class K8s { async _deleteObjByExId( exId: string, nodeType: NodeType, objType: ResourceType, force?: boolean ): Promise { - let objList: K8sObjectList; + let objList: TSResourceList; const deleteResponses: Array = []; try { @@ -483,14 +484,8 @@ export class K8s { } for (const obj of objList.items) { - const name = obj.metadata?.name; - const deletionTimestamp = obj.metadata?.deletionTimestamp; - - if (!name) { - const err = new Error(`Cannot delete ${objType} for ExId: ${exId} by name because it has no name`); - this.logger.error(err); - return Promise.reject(err); - } + const name = obj.metadata.name; + const deletionTimestamp = obj.metadata.deletionTimestamp; // If deletionTimestamp is present then the resource is already terminating. // K8s will not change the grace period in this case, so force deletion is not possible @@ -515,10 +510,10 @@ export class K8s { /** * Scales the k8s deployment for the specified exId to the desired number * of workers. - * @param {String} exId exId of execution to scale - * @param {number} numWorkers number of workers to scale by - * @param {ScaleOp} op Scale operation: `set`, `add`, `remove` - * @return {Object} Body of patch response. + * @param {String} exId exId of execution to scale + * @param {number} numWorkers number of workers to scale by + * @param {ScaleOp} op Scale operation: `set`, `add`, `remove` + * @return {Promise} Body of patch response. */ async scaleExecution(exId: string, numWorkers: number, op: ScaleOp): Promise { let newScale: number; @@ -537,11 +532,9 @@ export class K8s { } else if (listResponse.items.length > 1) { throw new TSError(`Unexpected number of Teraslice deployments matching the following selector: ${selector}`); } + const workerDeployment = listResponse.items[0]; - if (workerDeployment.spec?.replicas === undefined) { - throw new Error('replicas is undefined in worker deployment spec'); - } - this.logger.info(`Current Scale for exId=${exId}: ${workerDeployment.spec?.replicas}`); + this.logger.info(`Current Scale for exId=${exId}: ${workerDeployment.spec.replicas}`); if (op === 'set') { newScale = numWorkers; @@ -563,9 +556,6 @@ export class K8s { } ]; - if (!workerDeployment.metadata?.name) { - throw new Error('name is undefined in worker deployment metadata'); - } const patchResponseBody = await this .patch(scalePatch, workerDeployment.metadata.name); this.logger.debug(`k8s.scaleExecution patchResponseBody: ${JSON.stringify(patchResponseBody)}`); diff --git a/packages/teraslice/src/lib/cluster/services/cluster/backends/kubernetesV2/k8sDeploymentResource.ts b/packages/teraslice/src/lib/cluster/services/cluster/backends/kubernetesV2/k8sDeploymentResource.ts index 460eb83236c..b011e2ec4d1 100644 --- a/packages/teraslice/src/lib/cluster/services/cluster/backends/kubernetesV2/k8sDeploymentResource.ts +++ b/packages/teraslice/src/lib/cluster/services/cluster/backends/kubernetesV2/k8sDeploymentResource.ts @@ -1,14 +1,13 @@ -import * as k8s from '@kubernetes/client-node'; import { Logger } from '@terascope/utils'; import type { Config, ExecutionConfig } from '@terascope/types'; import { makeTemplate } from './utils.js'; -import { K8sConfig, NodeType } from './interfaces.js'; +import { K8sConfig, NodeType, TSDeployment } from './interfaces.js'; import { K8sResource } from './k8sResource.js'; -export class K8sDeploymentResource extends K8sResource { +export class K8sDeploymentResource extends K8sResource { nodeType: NodeType = 'worker'; nameInfix = 'wkr'; - templateGenerator: (config: K8sConfig) => k8s.V1Deployment; + templateGenerator: (config: K8sConfig) => TSDeployment; templateConfig; resource; exName: string; diff --git a/packages/teraslice/src/lib/cluster/services/cluster/backends/kubernetesV2/k8sJobResource.ts b/packages/teraslice/src/lib/cluster/services/cluster/backends/kubernetesV2/k8sJobResource.ts index cf1514f440a..35c04ee5e4c 100644 --- a/packages/teraslice/src/lib/cluster/services/cluster/backends/kubernetesV2/k8sJobResource.ts +++ b/packages/teraslice/src/lib/cluster/services/cluster/backends/kubernetesV2/k8sJobResource.ts @@ -1,14 +1,13 @@ -import * as k8s from '@kubernetes/client-node'; import { Logger } from '@terascope/utils'; import type { Config, ExecutionConfig } from '@terascope/types'; import { makeTemplate } from './utils.js'; -import { K8sConfig, NodeType } from './interfaces.js'; +import { K8sConfig, NodeType, TSJob } from './interfaces.js'; import { K8sResource } from './k8sResource.js'; -export class K8sJobResource extends K8sResource { +export class K8sJobResource extends K8sResource { nodeType: NodeType = 'execution_controller'; nameInfix = 'exc'; - templateGenerator: (config: K8sConfig) => k8s.V1Job; + templateGenerator: (config: K8sConfig) => TSJob; templateConfig; resource; diff --git a/packages/teraslice/src/lib/cluster/services/cluster/backends/kubernetesV2/k8sResource.ts b/packages/teraslice/src/lib/cluster/services/cluster/backends/kubernetesV2/k8sResource.ts index 322fd1353f3..e3195c37381 100644 --- a/packages/teraslice/src/lib/cluster/services/cluster/backends/kubernetesV2/k8sResource.ts +++ b/packages/teraslice/src/lib/cluster/services/cluster/backends/kubernetesV2/k8sResource.ts @@ -1,12 +1,14 @@ import _ from 'lodash'; -import * as k8s from '@kubernetes/client-node'; import { isNumber, Logger } from '@terascope/utils'; import type { Config, ExecutionConfig } from '@terascope/types'; import { safeEncode } from '../../../../../utils/encoding_utils.js'; import { setMaxOldSpaceViaEnv } from './utils.js'; -import { K8sConfig, NodeType } from './interfaces.js'; +import { + K8sConfig, NodeType, TSDeployment, + TSJob, TSService +} from './interfaces.js'; -export abstract class K8sResource { +export abstract class K8sResource { execution: ExecutionConfig; jobLabelPrefix: string; jobPropertyLabelPrefix: string; @@ -42,12 +44,12 @@ export abstract class K8sResource { this._setTargetRequired(target, resource); @@ -156,28 +158,28 @@ export abstract class K8sResource { if (isNumber(portValue)) { - resource.spec?.template.spec?.containers[0].ports - ?.push({ containerPort: portValue }); + resource.spec.template.spec.containers[0].ports + .push({ containerPort: portValue }); } else { - resource.spec?.template.spec?.containers[0].ports - ?.push( + resource.spec.template.spec.containers[0].ports + .push( { name: portValue.name, containerPort: portValue.port @@ -188,8 +190,8 @@ export abstract class K8sResource { const key = `${this.jobLabelPrefix}/${_.replace(k, /[^a-zA-Z0-9\-._]/g, '-').substring(0, 63)}`; const value = _.replace(v, /[^a-zA-Z0-9\-._]/g, '-').substring(0, 63); - if (resource.metadata?.labels && resource.spec?.template.metadata?.labels) { + if (resource.metadata.labels && resource.spec.template.metadata.labels) { resource.metadata.labels[key] = value; resource.spec.template.metadata.labels[key] = value; } @@ -255,14 +257,14 @@ export abstract class K8sResource { - resource.spec?.template.spec?.volumes?.push({ + resource.spec.template.spec.volumes.push({ name: volume.name, persistentVolumeClaim: { claimName: volume.name } }); - resource.spec?.template.spec?.containers[0].volumeMounts?.push({ + resource.spec.template.spec.containers[0].volumeMounts.push({ name: volume.name, mountPath: volume.path }); @@ -270,12 +272,12 @@ export abstract class K8sResource { // `required` is the default if no `constraint` is provided for @@ -354,7 +356,7 @@ export abstract class K8sResource { +export class K8sServiceResource extends K8sResource { nodeType: NodeType = 'execution_controller'; nameInfix = 'exc'; - templateGenerator: (config: K8sConfig) => k8s.V1Service; + templateGenerator: (config: K8sConfig) => TSService; templateConfig; resource; exName: string; diff --git a/packages/teraslice/src/lib/cluster/services/cluster/backends/kubernetesV2/k8sState.ts b/packages/teraslice/src/lib/cluster/services/cluster/backends/kubernetesV2/k8sState.ts index 37cfbf3f052..d5d5bea72f2 100644 --- a/packages/teraslice/src/lib/cluster/services/cluster/backends/kubernetesV2/k8sState.ts +++ b/packages/teraslice/src/lib/cluster/services/cluster/backends/kubernetesV2/k8sState.ts @@ -1,18 +1,16 @@ import _ from 'lodash'; -import * as k8s from '@kubernetes/client-node'; -import { Logger } from '@terascope/types'; -import { logError } from '@terascope/utils'; +import { TSPodList } from './interfaces'; /** * Given the k8s Pods API output generates the appropriate Teraslice cluster * state. NOTE: This assumes the pods have already been filtered to ensure they * are teraslice pods and match the cluster in question. - * @param {Object} k8sPods k8s pods API object (k8s v1.10+) + * @param {TSPodList} k8sPods k8s pods API object (k8s v1.10+) * @param {Object} clusterState Teraslice Cluster State * @param {String} clusterNameLabel k8s label containing clusterName * @param {Logger} logger Teraslice logger */ -export function gen(k8sPods: k8s.V1PodList, clusterState: Record, logger: Logger) { +export function gen(k8sPods: TSPodList, clusterState: Record) { // Make sure we clean up the old const hostIPs = _.uniq(_.map(k8sPods.items, 'status.hostIP')); const oldHostIps = _.difference(_.keys(clusterState), hostIPs); @@ -29,50 +27,40 @@ export function gen(k8sPods: k8s.V1PodList, clusterState: Record, l // add a worker for each pod k8sPods.items.forEach((pod) => { - if (pod.status?.hostIP) { - if (!_.has(clusterState, pod.status.hostIP)) { - // If the node isn't in clusterState, add it - clusterState[pod.status.hostIP] = { - node_id: pod.status.hostIP, - hostname: pod.status.hostIP, - pid: 'N/A', - node_version: 'N/A', - teraslice_version: 'N/A', - total: 'N/A', - state: 'connected', - available: 'N/A', - active: [] - }; - } + if (!_.has(clusterState, pod.status.hostIP)) { + // If the node isn't in clusterState, add it + clusterState[pod.status.hostIP] = { + node_id: pod.status.hostIP, + hostname: pod.status.hostIP, + pid: 'N/A', + node_version: 'N/A', + teraslice_version: 'N/A', + total: 'N/A', + state: 'connected', + available: 'N/A', + active: [] + }; + } - if (pod.metadata?.labels && pod.spec) { - const worker = { - assets: [], - assignment: pod.metadata.labels['app.kubernetes.io/component'], - ex_id: pod.metadata.labels['teraslice.terascope.io/exId'], - // WARNING: This makes the assumption that the first container - // in the pod is the teraslice container. Currently it is the - // only container, so this assumption is safe for now. - image: pod.spec.containers[0].image, - job_id: pod.metadata.labels['teraslice.terascope.io/jobId'], - pod_name: pod.metadata.name, - pod_ip: pod.status.podIP, - worker_id: pod.metadata.name, - }; + const worker = { + assets: [], + assignment: pod.metadata.labels['app.kubernetes.io/component'], + ex_id: pod.metadata.labels['teraslice.terascope.io/exId'], + // WARNING: This makes the assumption that the first container + // in the pod is the teraslice container. Currently it is the + // only container, so this assumption is safe for now. + image: pod.spec.containers[0].image, + job_id: pod.metadata.labels['teraslice.terascope.io/jobId'], + pod_name: pod.metadata.name, + pod_ip: pod.status.podIP, + worker_id: pod.metadata.name, + }; - // k8s pods can have status.phase = `Pending`, `Running`, `Succeeded`, - // `Failed`, `Unknown`. We will only add `Running` pods to the - // Teraslice cluster state. - if (pod.status.phase === 'Running') { - clusterState[pod.status.hostIP].active.push(worker); - } - } else { - // TODO: We might need to do more here. I think it's OK to just - // log though. This only gets used to show slicer info through - // the API. We wouldn't want to disrupt the cluster master - // for rare failures to reach the k8s API. - logError(logger, 'K8s pod missing a required field necessary to update cluster state'); - } + // k8s pods can have status.phase = `Pending`, `Running`, `Succeeded`, + // `Failed`, `Unknown`. We will only add `Running` pods to the + // Teraslice cluster state. + if (pod.status.phase === 'Running') { + clusterState[pod.status.hostIP].active.push(worker); } }); } diff --git a/packages/teraslice/src/lib/cluster/services/cluster/backends/kubernetesV2/utils.ts b/packages/teraslice/src/lib/cluster/services/cluster/backends/kubernetesV2/utils.ts index 36342cdfbc0..e283bd2c70d 100644 --- a/packages/teraslice/src/lib/cluster/services/cluster/backends/kubernetesV2/utils.ts +++ b/packages/teraslice/src/lib/cluster/services/cluster/backends/kubernetesV2/utils.ts @@ -4,13 +4,19 @@ import path from 'node:path'; import barbe from 'barbe'; import { isTest } from '@terascope/utils'; import * as k8s from '@kubernetes/client-node'; -import { K8sConfig, NodeType, Resource } from './interfaces.js'; +import { + K8sConfig, NodeType, K8sResource, + TSDeployment, TSJob, TSPod, TSService, + TSReplicaSet, TSResource, K8sResourceList, + TSDeploymentList, TSJobList, TSPodList, + TSReplicaSetList, TSResourceList, TSServiceList +} from './interfaces.js'; const MAX_RETRIES = isTest ? 2 : 3; const RETRY_DELAY = isTest ? 50 : 1000; // time in ms const resourcePath = path.join(process.cwd(), './packages/teraslice/src/lib/cluster/services/cluster/backends/kubernetesV2/'); -export function makeTemplate( +export function makeTemplate( folder: 'deployments' | 'jobs' | 'services', fileName: NodeType ): (config: K8sConfig) => T { @@ -66,21 +72,161 @@ export function getRetryConfig() { }; } -export function isDeployment(manifest: Resource): manifest is k8s.V1Deployment { - return manifest.kind === 'Deployment'; +export function isDeployment(resource: K8sResource): resource is k8s.V1Deployment { + return resource instanceof k8s.V1Deployment || resource.kind === 'Deployment'; } -export function isJob(manifest: Resource): manifest is k8s.V1Job { - return manifest.kind === 'Job'; +export function isJob(resource: K8sResource): resource is k8s.V1Job { + return resource instanceof k8s.V1Job || resource.kind === 'Job'; } -export function isPod(manifest: Resource): manifest is k8s.V1Pod { - return manifest.kind === 'Pod'; +export function isPod(resource: K8sResource): resource is k8s.V1Pod { + return resource instanceof k8s.V1Pod || resource.kind === 'Pod'; } -export function isReplicaSet(manifest: Resource): manifest is k8s.V1ReplicaSet { - return manifest.kind === 'ReplicaSet'; + +export function isReplicaSet(resource: K8sResource): resource is k8s.V1ReplicaSet { + return resource instanceof k8s.V1ReplicaSet || resource.kind === 'ReplicaSet'; +} + +export function isService(resource: K8sResource): resource is k8s.V1Service { + return resource instanceof k8s.V1Service || resource.kind === 'Service'; +} + +export function isTSDeployment(manifest: k8s.V1Deployment): manifest is TSDeployment { + return manifest instanceof k8s.V1Deployment + && manifest.metadata?.labels !== undefined + && manifest.metadata.name !== undefined + && manifest.spec?.replicas !== undefined + && manifest.spec.template.metadata?.labels !== undefined + && manifest.spec.template.spec?.containers[0].volumeMounts !== undefined + && manifest.spec.template.spec.volumes !== undefined; +} + +export function isTSJob(manifest: k8s.V1Job): manifest is TSJob { + return manifest instanceof k8s.V1Job + && manifest.metadata?.labels !== undefined + && manifest.metadata.name !== undefined + && manifest.metadata.uid !== undefined + && manifest.spec?.template.metadata?.labels !== undefined + && manifest.spec.template.spec?.containers[0].volumeMounts !== undefined + && manifest.spec.template.spec.volumes !== undefined + && manifest.spec.selector?.matchLabels !== undefined; +} + +export function isTSPod(manifest: K8sResource): manifest is TSPod { + return manifest instanceof k8s.V1Pod + && manifest.metadata?.name !== undefined + && manifest.status !== undefined; +} + +export function isTSReplicaSet(manifest: k8s.V1ReplicaSet): manifest is TSReplicaSet { + return manifest instanceof k8s.V1ReplicaSet + && manifest.metadata?.name !== undefined + && manifest.status !== undefined; +} + +export function isTSService(manifest: k8s.V1Service): manifest is TSService { + return manifest instanceof k8s.V1Service + && manifest.metadata?.name !== undefined + && manifest.spec?.selector !== undefined + && manifest.spec.ports !== undefined; +} + +export function convertToTSResource(resource: k8s.V1Deployment): TSDeployment; +export function convertToTSResource(resource: k8s.V1Job): TSJob; +export function convertToTSResource(resource: k8s.V1Pod): TSPod; +export function convertToTSResource(resource: k8s.V1ReplicaSet): TSReplicaSet; +export function convertToTSResource(resource: k8s.V1Service): TSService; +export function convertToTSResource(resource: K8sResource): TSResource; +export function convertToTSResource(resource: K8sResource): TSResource { + if (isDeployment(resource) && isTSDeployment(resource)) { + return resource; + } + if (isJob(resource) && isTSJob(resource)) { + return resource; + } + if (isPod(resource) && isTSPod(resource)) { + return resource; + } + if (isReplicaSet(resource) && isTSReplicaSet(resource)) { + return resource; + } + if (isService(resource) && isTSService(resource)) { + return resource; + } + + throw new Error('K8sResource missing required field(s) to be converted to TSResource.'); +} + +export function isDeploymentList(manifest: K8sResourceList): manifest is k8s.V1DeploymentList { + return manifest.kind === 'DeploymentList'; +} + +export function isJobList(manifest: K8sResourceList): manifest is k8s.V1JobList { + return manifest.kind === 'JobList'; +} + +export function isPodList(manifest: K8sResourceList): manifest is k8s.V1PodList { + return manifest.kind === 'PodList'; +} + +export function isReplicaSetList(manifest: K8sResourceList): manifest is k8s.V1ReplicaSetList { + return manifest.kind === 'ReplicaSetList'; +} + +export function isServiceList(manifest: K8sResourceList): manifest is k8s.V1ServiceList { + return manifest.kind === 'ServiceList'; +} + +export function isTSDeploymentList(manifest: k8s.V1DeploymentList): manifest is TSDeploymentList { + return manifest.kind === 'DeploymentList' + && (manifest.items[0] ? isTSDeployment(manifest.items[0]) : true); +} + +export function isTSJobList(manifest: k8s.V1JobList): manifest is TSJobList { + return manifest.kind === 'JobList' + && (manifest.items[0] ? isTSJob(manifest.items[0]) : true); +} + +export function isTSPodList(manifest: k8s.V1PodList): manifest is TSPodList { + return manifest.kind === 'PodList' + && (manifest.items[0] ? isTSPod(manifest.items[0]) : true); +} + +export function isTSReplicaSetList(manifest: k8s.V1ReplicaSetList): manifest is TSReplicaSetList { + return manifest.kind === 'ReplicaSetList' + && (manifest.items[0] ? isTSReplicaSet(manifest.items[0]) : true); } -export function isService(manifest: Resource): manifest is k8s.V1Service { - return manifest.kind === 'Service'; +export function isTSServiceList(manifest: k8s.V1ServiceList): manifest is TSServiceList { + return manifest.kind === 'ServiceList' + && (manifest.items[0] ? isTSService(manifest.items[0]) : true); +} + +export function convertToTSResourceList(resourceList: k8s.V1DeploymentList): TSDeploymentList; +export function convertToTSResourceList(resourceList: k8s.V1JobList): TSJobList; +export function convertToTSResourceList(resourceList: k8s.V1PodList): TSPodList; +export function convertToTSResourceList(resourceList: k8s.V1ReplicaSetList): TSReplicaSetList; +export function convertToTSResourceList(resourceList: k8s.V1ServiceList): TSServiceList; +export function convertToTSResourceList(resourceList: K8sResourceList): TSResourceList; +export function convertToTSResourceList(resourceList: K8sResourceList): TSResourceList { + resourceList.items.map((resource) => convertToTSResource(resource)); + + if (isDeploymentList(resourceList) && isTSDeploymentList(resourceList)) { + return resourceList; + } + if (isJobList(resourceList) && isTSJobList(resourceList)) { + return resourceList; + } + if (isPodList(resourceList) && isTSPodList(resourceList)) { + return resourceList; + } + if (isReplicaSetList(resourceList) && isTSReplicaSetList(resourceList)) { + return resourceList; + } + if (isServiceList(resourceList) && isTSServiceList(resourceList)) { + return resourceList; + } + + throw new Error('K8sResource missing required field(s) to be converted to TSResourceList.'); } diff --git a/packages/teraslice/test/lib/cluster/services/cluster/backends/kubernetes/v2/k8s-v2-spec.ts b/packages/teraslice/test/lib/cluster/services/cluster/backends/kubernetes/v2/k8s-v2-spec.ts index 0453230c801..a6556aa35a6 100644 --- a/packages/teraslice/test/lib/cluster/services/cluster/backends/kubernetes/v2/k8s-v2-spec.ts +++ b/packages/teraslice/test/lib/cluster/services/cluster/backends/kubernetes/v2/k8s-v2-spec.ts @@ -14,6 +14,117 @@ const _url = 'http://mock.kube.api'; describe('k8s', () => { let k8s: K8s; + const job = { + apiVersion: '1.0.0', + kind: 'Job', + metadata: { + labels: { + 'app.kubernetes.io/name': 'teraslice' + }, + name: 'testJob1', + uid: 'uid1' + }, + spec: { + template: { + metadata: { + labels: { + 'app.kubernetes.io/name': 'teraslice' + }, + }, + spec: { + containers: [{ + volumeMounts: [] + }], + volumes: [] + } + }, + selector: { + matchLabels: { + 'app.kubernetes.io/component': 'execution_controller' + } + } + } + }; + + const testPod1 = { + apiVersion: 'v1', + kind: 'Pod', + metadata: { name: 'testPod1' }, + status: {} + }; + + const testPod2 = { + apiVersion: 'v1', + kind: 'Pod', + metadata: { name: 'testPod2' }, + status: {} + }; + + const service = { + kind: 'Service', + metadata: { + name: 'service1' + }, + spec: { + selector: { + 'app.kubernetes.io/component': 'execution_controller' + }, + ports: [ + { port: 45680 } + ] + } + }; + + const deployment = { + apiVersion: 'v1', + kind: 'Deployment', + metadata: { + labels: { + 'app.kubernetes.io/name': 'teraslice' + }, + name: 'dname' + }, + spec: { + replicas: 5, + template: { + metadata: { + labels: { + 'app.kubernetes.io/name': 'teraslice' + }, + }, + spec: { + volumes: [{ name: 'volume1' }], + containers: [{ + volumeMounts: [], + }] + } + }, + selector: {} + }, + status: {} + }; + + const replicaSet = { + kind: 'ReplicaSet', + metadata: { + name: 'replicaset1' + }, + status: {} + + }; + + const status = { + apiVersion: 'v1', + details: { + group: 'batch', + kind: 'jobs', + name: 'testJob1', + uid: 'f29935a1-9f36-4104-a840-f6534d7f2ef8' + }, + kind: 'Status', + status: 'Success' + }; + beforeEach(async () => { nock(_url) .get('/api/v1/namespaces') @@ -68,7 +179,10 @@ describe('k8s', () => { nock(_url) .get('/api/v1/namespaces/default/pods') .query({ labelSelector: 'app=teraslice' }) - .reply(200, { kind: 'PodList' }); + .reply(200, { + kind: 'PodList', + items: [testPod1] + }); const pods = await k8s.list('app=teraslice', 'pods'); expect(pods.kind).toEqual('PodList'); @@ -78,7 +192,10 @@ describe('k8s', () => { nock(_url) .get('/api/v1/namespaces/default/services') .query({ labelSelector: 'app=teraslice' }) - .reply(200, { kind: 'ServiceList' }); + .reply(200, { + kind: 'ServiceList', + items: [service] + }); const pods = await k8s.list('app=teraslice', 'services'); expect(pods.kind).toEqual('ServiceList'); @@ -88,7 +205,10 @@ describe('k8s', () => { nock(_url) .get('/apis/apps/v1/namespaces/default/deployments') .query({ labelSelector: 'app=teraslice' }) - .reply(200, { kind: 'DeploymentList' }); + .reply(200, { + kind: 'DeploymentList', + items: [deployment] + }); const deployments = await k8s.list('app=teraslice', 'deployments'); expect(deployments.kind).toEqual('DeploymentList'); @@ -98,7 +218,10 @@ describe('k8s', () => { nock(_url) .get('/apis/batch/v1/namespaces/default/jobs') .query({ labelSelector: 'app=teraslice' }) - .reply(200, { kind: 'JobList' }); + .reply(200, { + kind: 'JobList', + items: [job] + }); const jobs = await k8s.list('app=teraslice', 'jobs'); expect(jobs.kind).toEqual('JobList'); @@ -108,7 +231,10 @@ describe('k8s', () => { nock(_url) .get('/apis/apps/v1/namespaces/default/replicasets') .query({ labelSelector: 'app=teraslice' }) - .reply(200, { kind: 'ReplicaSetList' }); + .reply(200, { + kind: 'ReplicaSetList', + items: [replicaSet] + }); const jobs = await k8s.list('app=teraslice', 'replicasets'); expect(jobs.kind).toEqual('ReplicaSetList'); @@ -121,27 +247,22 @@ describe('k8s', () => { .get('/apis/batch/v1/namespaces/default/jobs') .query({ labelSelector: 'app=teraslice' }) .reply(200, { - items: [{ - apiVersion: '1.0.0', - kind: 'job' - }] + kind: 'JobList', + items: [job] }); const jobs = await k8s.nonEmptyJobList('app=teraslice'); - expect(jobs.items[0]).toEqual({ - apiVersion: '1.0.0', - kind: 'job', - metadata: undefined, - spec: undefined, - status: undefined - }); + expect(jobs.items[0]).toEqual(job); }); it('throws with an empty list', async () => { nock(_url) .get('/apis/batch/v1/namespaces/default/jobs') .query({ labelSelector: 'app=teraslice' }) - .reply(200, { items: [] }); + .reply(200, { + kind: 'JobList', + items: [] + }); await expect(k8s.nonEmptyJobList('app=teraslice')) .rejects.toThrow('Teraslice job matching the following selector was not found: app=teraslice (retriable)'); @@ -152,7 +273,7 @@ describe('k8s', () => { it('can post a deployment', async () => { nock(_url, { encodedQueryParams: true }) .post('/apis/apps/v1/namespaces/default/deployments') - .reply(201, { kind: 'Deployment' }); + .reply(201, deployment); const response = await k8s.post({ kind: 'Deployment' }); expect(response.kind).toEqual('Deployment'); @@ -161,7 +282,7 @@ describe('k8s', () => { it('can post a job', async () => { nock(_url, { encodedQueryParams: true }) .post('/apis/batch/v1/namespaces/default/jobs') - .reply(201, { kind: 'Job' }); + .reply(201, job); const response = await k8s.post({ kind: 'Job' }); expect(response.kind).toEqual('Job'); @@ -170,7 +291,7 @@ describe('k8s', () => { it('can post a pod', async () => { nock(_url, { encodedQueryParams: true }) .post('/api/v1/namespaces/default/pods') - .reply(201, { kind: 'Pod' }); + .reply(201, testPod1); const response = await k8s.post({ kind: 'Pod' }); expect(response.kind).toEqual('Pod'); @@ -179,7 +300,7 @@ describe('k8s', () => { it('can post a replicaSet', async () => { nock(_url, { encodedQueryParams: true }) .post('/apis/apps/v1/namespaces/default/replicasets') - .reply(201, { kind: 'ReplicaSet' }); + .reply(201, replicaSet); const response = await k8s.post({ kind: 'ReplicaSet' }); expect(response.kind).toEqual('ReplicaSet'); @@ -188,7 +309,7 @@ describe('k8s', () => { it('can post a service', async () => { nock(_url, { encodedQueryParams: true }) .post('/api/v1/namespaces/default/services') - .reply(201, { kind: 'Service' }); + .reply(201, service); const response = await k8s.post({ kind: 'Service' }); expect(response.kind).toEqual('Service'); @@ -312,55 +433,6 @@ describe('k8s', () => { }); describe('->_deletObjByExId', () => { - const job = { - apiVersion: '1.0.0', - kind: 'Job', - metadata: { name: 'testJob1' } - }; - - const jobNoName = { - apiVersion: '1.0.0', - kind: 'Job', - metadata: { name: undefined } - }; - - const testPod1 = { - apiVersion: 'v1', - kind: 'Pod', - metadata: { name: 'testPod1' } - }; - - const testPod2 = { - apiVersion: 'v1', - kind: 'Pod', - metadata: { name: 'testPod2' } - }; - - const status = { - apiVersion: 'v1', - details: { - group: 'batch', - kind: 'jobs', - name: 'testJob1', - uid: 'f29935a1-9f36-4104-a840-f6534d7f2ef8' - }, - kind: 'Status', - status: 'Success' - }; - - it('will throw if name is undefined', async () => { - nock(_url) - .get('/apis/batch/v1/namespaces/default/jobs') - .query({ labelSelector: /app\.kubernetes\.io\/component=execution_controller,teraslice\.terascope\.io\/exId=.*/ }) - .reply(200, { - kind: 'JobList', - items: [jobNoName] - }); - - await expect(k8s._deleteObjByExId('no-name', 'execution_controller', 'jobs')) - .rejects.toThrow('Cannot delete jobs for ExId: no-name by name because it has no name'); - }); - it('can delete a single object', async () => { nock(_url) .get('/apis/batch/v1/namespaces/default/jobs') @@ -384,10 +456,7 @@ describe('k8s', () => { .query({ labelSelector: /app\.kubernetes\.io\/component=worker,teraslice\.terascope\.io\/exId=.*/ }) .reply(200, { kind: 'PodList', - items: [ - { metadata: { name: 'testPod1' } }, - { metadata: { name: 'testPod2' } } - ] + items: [testPod1, testPod2] }) .delete('/api/v1/namespaces/default/pods/testPod1') .reply(200, testPod1) @@ -404,22 +473,8 @@ describe('k8s', () => { describe('->scaleExecution', () => { let scope: nock.Scope; - let deployment: { - apiVersion: string; - kind: string; - metadata: { name: string }; - spec: { replicas: number }; - status: string; - }; beforeEach(() => { - deployment = { - apiVersion: 'v1', - kind: 'Deployment', - metadata: { name: 'dname' }, - spec: { replicas: 5 }, - status: 'Success' - }; - + deployment.spec.replicas = 5; scope = nock(_url) .get('/apis/apps/v1/namespaces/default/deployments') .query({ labelSelector: /app\.kubernetes\.io\/component=worker,teraslice\.terascope\.io\/exId=.*/ }) diff --git a/packages/teraslice/test/lib/cluster/services/cluster/backends/kubernetes/v2/k8sResource-v2-spec.ts b/packages/teraslice/test/lib/cluster/services/cluster/backends/kubernetes/v2/k8sResource-v2-spec.ts index aa11be4af01..d8fde334749 100644 --- a/packages/teraslice/test/lib/cluster/services/cluster/backends/kubernetes/v2/k8sResource-v2-spec.ts +++ b/packages/teraslice/test/lib/cluster/services/cluster/backends/kubernetes/v2/k8sResource-v2-spec.ts @@ -43,22 +43,22 @@ describe('k8sResource', () => { const kr = new K8sDeploymentResource(terasliceConfig, execution, logger, 'example-job-abcd', 'UID1'); expect(kr.resource.kind).toBe('Deployment'); - expect(kr.resource.spec?.replicas).toBe(2); - expect(kr.resource.metadata?.name).toBe('ts-wkr-example-data-generator-job-7ba9afb0-417a'); + expect(kr.resource.spec.replicas).toBe(2); + expect(kr.resource.metadata.name).toBe('ts-wkr-example-data-generator-job-7ba9afb0-417a'); // The following properties should be absent in the default case // Note: This tests that both affinity and podAntiAffinity are absent - expect(kr.resource.spec?.template.spec).not.toHaveProperty('affinity'); - expect(kr.resource.spec?.template.spec).not.toHaveProperty('imagePullSecrets'); - expect(kr.resource.spec?.template.spec).not.toHaveProperty('priorityClassName'); + expect(kr.resource.spec.template.spec).not.toHaveProperty('affinity'); + expect(kr.resource.spec.template.spec).not.toHaveProperty('imagePullSecrets'); + expect(kr.resource.spec.template.spec).not.toHaveProperty('priorityClassName'); let configVolume: k8s.V1Volume | undefined = undefined; - if (kr.resource.spec?.template.spec?.volumes) { + if (kr.resource.spec.template.spec.volumes) { configVolume = kr.resource.spec.template.spec.volumes[0]; } let configVolumeMount: k8s.V1VolumeMount | undefined = undefined; - if (kr.resource.spec?.template.spec?.containers[0].volumeMounts) { + if (kr.resource.spec.template.spec.containers[0].volumeMounts) { configVolumeMount = kr.resource.spec.template.spec.containers[0].volumeMounts[0]; } // Configmaps should be mounted on all workers @@ -80,7 +80,7 @@ describe('k8sResource', () => { const kr = new K8sDeploymentResource(terasliceConfig, execution, logger, 'example-job-abcd', 'UID1'); let firstSecret: k8s.V1LocalObjectReference | undefined = undefined; - if (kr.resource.spec?.template.spec?.imagePullSecrets) { + if (kr.resource.spec.template.spec.imagePullSecrets) { firstSecret = kr.resource.spec.template.spec.imagePullSecrets[0]; } @@ -95,7 +95,7 @@ describe('k8sResource', () => { const kr = new K8sDeploymentResource(terasliceConfig, execution, logger, 'example-job-abcd', 'UID1'); // console.log(yaml.dump(kr.resource.spec.template.spec.affinity)); - expect(kr.resource.spec?.template.spec?.affinity).toEqual( + expect(kr.resource.spec.template.spec.affinity).toEqual( yaml.load(` podAntiAffinity: preferredDuringSchedulingIgnoredDuringExecution: @@ -121,33 +121,23 @@ describe('k8sResource', () => { const kr = new K8sDeploymentResource(terasliceConfig, execution, logger, 'example-job-abcd', 'UID1'); - let configVolume: k8s.V1Volume | undefined = undefined; - if (kr.resource.spec?.template.spec?.volumes) { - configVolume = kr.resource.spec.template.spec.volumes[0]; - } - - let assetVolume: k8s.V1Volume | undefined = undefined; - if (kr.resource.spec?.template.spec?.volumes) { - assetVolume = kr.resource.spec.template.spec.volumes[1]; - } - let configVolumeMount: k8s.V1VolumeMount | undefined = undefined; - if (kr.resource.spec?.template.spec?.containers[0].volumeMounts) { + if (kr.resource.spec.template.spec.containers[0].volumeMounts) { configVolumeMount = kr.resource.spec.template.spec.containers[0].volumeMounts[0]; } let assetVolumeMount: k8s.V1VolumeMount | undefined = undefined; - if (kr.resource.spec?.template.spec?.containers[0].volumeMounts) { + if (kr.resource.spec.template.spec.containers[0].volumeMounts) { assetVolumeMount = kr.resource.spec.template.spec.containers[0].volumeMounts[1]; } - expect(kr.resource.spec?.replicas).toBe(2); - expect(kr.resource.metadata?.name).toBe('ts-wkr-example-data-generator-job-7ba9afb0-417a'); + expect(kr.resource.spec.replicas).toBe(2); + expect(kr.resource.metadata.name).toBe('ts-wkr-example-data-generator-job-7ba9afb0-417a'); // The following properties should be absent in the default case - expect(kr.resource.spec?.template.spec).not.toHaveProperty('affinity'); - expect(kr.resource.spec?.template.spec).not.toHaveProperty('imagePullSecrets'); - expect(configVolume).toEqual(yaml.load(` + expect(kr.resource.spec.template.spec).not.toHaveProperty('affinity'); + expect(kr.resource.spec.template.spec).not.toHaveProperty('imagePullSecrets'); + expect(kr.resource.spec.template.spec.volumes[0]).toEqual(yaml.load(` name: config configMap: name: ts-dev1-worker @@ -160,7 +150,7 @@ describe('k8sResource', () => { name: config`)); // Now check for the assets volume - expect(assetVolume).toEqual(yaml.load(` + expect(kr.resource.spec.template.spec.volumes[1]).toEqual(yaml.load(` name: asset-volume persistentVolumeClaim: claimName: asset-volume`)); @@ -177,42 +167,34 @@ describe('k8sResource', () => { const kr = new K8sDeploymentResource(terasliceConfig, execution, logger, 'example-job-abcd', 'UID1'); - const configVolume1 = kr.resource.spec?.template.spec?.volumes - ? kr.resource.spec.template.spec.volumes[0] - : undefined; - - const configVolume2 = kr.resource.spec?.template.spec?.volumes - ? kr.resource.spec.template.spec.volumes[1] - : undefined; - - const configVolumeMount1 = kr.resource.spec?.template.spec?.containers[0].volumeMounts + const configVolumeMount = kr.resource.spec.template.spec.containers[0].volumeMounts ? kr.resource.spec.template.spec.containers[0].volumeMounts[0] : undefined; - const configVolumeMount2 = kr.resource.spec?.template.spec?.containers[0].volumeMounts + const dataVolumeMount = kr.resource.spec.template.spec.containers[0].volumeMounts ? kr.resource.spec.template.spec.containers[0].volumeMounts[1] : undefined; // First check the configMap volumes, which should be present on all // deployments - expect(configVolume1).toEqual(yaml.load(` + expect(kr.resource.spec.template.spec.volumes[0]).toEqual(yaml.load(` name: config configMap: name: ts-dev1-worker items: - key: teraslice.yaml path: teraslice.yaml`)); - expect(configVolumeMount1) + expect(configVolumeMount) .toEqual(yaml.load(` mountPath: /app/config name: config`)); // Now check for the volume added via config - expect(configVolume2).toEqual(yaml.load(` + expect(kr.resource.spec.template.spec.volumes[1]).toEqual(yaml.load(` name: teraslice-data1 persistentVolumeClaim: claimName: teraslice-data1`)); - expect(configVolumeMount2) + expect(dataVolumeMount) .toEqual(yaml.load(` name: teraslice-data1 mountPath: /data`)); @@ -226,37 +208,29 @@ describe('k8sResource', () => { const kr = new K8sDeploymentResource(terasliceConfig, execution, logger, 'example-job-abcd', 'UID1'); - const configVolume2 = kr.resource.spec?.template.spec?.volumes - ? kr.resource.spec.template.spec.volumes[1] - : undefined; - - const configVolume3 = kr.resource.spec?.template.spec?.volumes - ? kr.resource.spec.template.spec.volumes[2] - : undefined; - - const configVolumeMount2 = kr.resource.spec?.template.spec?.containers[0].volumeMounts + const dataVolumeMount = kr.resource.spec.template.spec.containers[0].volumeMounts ? kr.resource.spec.template.spec.containers[0].volumeMounts[1] : undefined; - const configVolumeMount3 = kr.resource.spec?.template.spec?.containers[0].volumeMounts + const tmpVolumeMount = kr.resource.spec.template.spec.containers[0].volumeMounts ? kr.resource.spec.template.spec.containers[0].volumeMounts[2] : undefined; // Now check for the volumes added via job - expect(configVolume2).toEqual(yaml.load(` + expect(kr.resource.spec.template.spec.volumes[1]).toEqual(yaml.load(` name: teraslice-data1 persistentVolumeClaim: claimName: teraslice-data1`)); - expect(configVolumeMount2) + expect(dataVolumeMount) .toEqual(yaml.load(` name: teraslice-data1 mountPath: /data`)); - expect(configVolume3).toEqual(yaml.load(` + expect(kr.resource.spec.template.spec.volumes[2]).toEqual(yaml.load(` name: tmp persistentVolumeClaim: claimName: tmp`)); - expect(configVolumeMount3) + expect(tmpVolumeMount) .toEqual(yaml.load(` name: tmp mountPath: /tmp`)); @@ -265,15 +239,10 @@ describe('k8sResource', () => { it('does not have memory/cpu limits/requests when not set in config or execution', () => { const kr = new K8sDeploymentResource(terasliceConfig, execution, logger, 'example-job-abcd', 'UID1'); - const exId = kr.resource.metadata?.labels ? kr.resource.metadata.labels['teraslice.terascope.io/exId'] : undefined; - expect(exId) + expect(kr.resource.metadata.labels['teraslice.terascope.io/exId']) .toEqual('e76a0278-d9bc-4d78-bf14-431bcd97528c'); - - const resources = kr.resource.spec?.template.spec?.containers[0].resources; - expect(resources).not.toBeDefined(); - - const envArray = kr.resource.spec?.template.spec?.containers[0].env; - expect(envArray).not.toContain('NODE_OPTIONS'); + expect(kr.resource.spec.template.spec.containers[0].resources).not.toBeDefined(); + expect(kr.resource.spec.template.spec.containers[0].env).not.toContain('NODE_OPTIONS'); }); it('has memory and cpu limits and requests when set on terasliceConfig', () => { @@ -282,12 +251,9 @@ describe('k8sResource', () => { const kr = new K8sDeploymentResource(terasliceConfig, execution, logger, 'example-job-abcd', 'UID1'); - const exId = kr.resource.metadata?.labels ? kr.resource.metadata.labels['teraslice.terascope.io/exId'] : undefined; - expect(exId) + expect(kr.resource.metadata.labels['teraslice.terascope.io/exId']) .toEqual('e76a0278-d9bc-4d78-bf14-431bcd97528c'); - - const resources = kr.resource.spec?.template.spec?.containers[0].resources; - expect(resources).toEqual(yaml.load(` + expect(kr.resource.spec.template.spec.containers[0].resources).toEqual(yaml.load(` requests: memory: 2147483648 cpu: 1 @@ -295,9 +261,8 @@ describe('k8sResource', () => { memory: 2147483648 cpu: 1`)); - const envArray = kr.resource.spec?.template.spec?.containers[0].env; - const nodeOptionsEnv = _.find(envArray, { name: 'NODE_OPTIONS' }); - expect(nodeOptionsEnv?.value) + const envArray = kr.resource.spec.template.spec.containers[0].env; + expect(_.find(envArray, { name: 'NODE_OPTIONS' })?.value) .toEqual('--max-old-space-size=1843'); }); @@ -306,15 +271,9 @@ describe('k8sResource', () => { // NOTE: the env var merge happens in _setResources(), which is // somewhat out of place. - const envArray = kr.resource.spec?.template.spec?.containers[0].env; - - const fooEnv = _.find(envArray, { name: 'FOO' }); - expect(fooEnv?.value) - .toEqual('baz'); - - const exampleEnv = _.find(envArray, { name: 'EXAMPLE' }); - expect(exampleEnv?.value) - .toEqual('test'); + const envArray = kr.resource.spec.template.spec.containers[0].env; + expect(_.find(envArray, { name: 'FOO' })?.value).toEqual('baz'); + expect(_.find(envArray, { name: 'EXAMPLE' })?.value).toEqual('test'); }); it('execution resources override terasliceConfig resources', () => { @@ -325,10 +284,9 @@ describe('k8sResource', () => { const kr = new K8sDeploymentResource(terasliceConfig, execution, logger, 'example-job-abcd', 'UID1'); - const exId = kr.resource.metadata?.labels ? kr.resource.metadata.labels['teraslice.terascope.io/exId'] : undefined; - expect(exId) + expect(kr.resource.metadata.labels['teraslice.terascope.io/exId']) .toEqual('e76a0278-d9bc-4d78-bf14-431bcd97528c'); - expect(kr.resource.spec?.template.spec?.containers[0].resources).toEqual(yaml.load(` + expect(kr.resource.spec.template.spec.containers[0].resources).toEqual(yaml.load(` requests: memory: 1073741824 cpu: 2 @@ -336,9 +294,8 @@ describe('k8sResource', () => { memory: 1073741824 cpu: 2`)); - const envArray = kr.resource.spec?.template.spec?.containers[0].env; - const nodeOptionsEnv = _.find(envArray, { name: 'NODE_OPTIONS' }); - expect(nodeOptionsEnv?.value) + const envArray = kr.resource.spec.template.spec.containers[0].env; + expect(_.find(envArray, { name: 'NODE_OPTIONS' })?.value) .toEqual('--max-old-space-size=922'); }); @@ -349,10 +306,9 @@ describe('k8sResource', () => { const kr = new K8sDeploymentResource(terasliceConfig, execution, logger, 'example-job-abcd', 'UID1'); - const exId = kr.resource.metadata?.labels ? kr.resource.metadata.labels['teraslice.terascope.io/exId'] : undefined; - expect(exId) + expect(kr.resource.metadata.labels['teraslice.terascope.io/exId']) .toEqual('e76a0278-d9bc-4d78-bf14-431bcd97528c'); - expect(kr.resource.spec?.template.spec?.containers[0].resources).toEqual(yaml.load(` + expect(kr.resource.spec.template.spec.containers[0].resources).toEqual(yaml.load(` requests: memory: 2147483648 cpu: 2 @@ -360,9 +316,8 @@ describe('k8sResource', () => { memory: 2147483648 cpu: 2`)); - const envArray = kr.resource.spec?.template.spec?.containers[0].env; - const nodeOptionsEnv = _.find(envArray, { name: 'NODE_OPTIONS' }); - expect(nodeOptionsEnv?.value) + const envArray = kr.resource.spec.template.spec.containers[0].env; + expect(_.find(envArray, { name: 'NODE_OPTIONS' })?.value) .toEqual('--max-old-space-size=1843'); }); @@ -372,10 +327,9 @@ describe('k8sResource', () => { const kr = new K8sDeploymentResource(terasliceConfig, execution, logger, 'example-job-abcd', 'UID1'); - const exId = kr.resource.metadata?.labels ? kr.resource.metadata.labels['teraslice.terascope.io/exId'] : undefined; - expect(exId) + expect(kr.resource.metadata.labels['teraslice.terascope.io/exId']) .toEqual('e76a0278-d9bc-4d78-bf14-431bcd97528c'); - expect(kr.resource.spec?.template.spec?.containers[0].resources).toEqual(yaml.load(` + expect(kr.resource.spec.template.spec.containers[0].resources).toEqual(yaml.load(` requests: memory: 2147483648 cpu: 1 @@ -383,9 +337,8 @@ describe('k8sResource', () => { memory: 2147483648 cpu: 1`)); - const envArray = kr.resource.spec?.template.spec?.containers[0].env; - const nodeOptionsEnv = _.find(envArray, { name: 'NODE_OPTIONS' }); - expect(nodeOptionsEnv?.value) + const envArray = kr.resource.spec.template.spec.containers[0].env; + expect(_.find(envArray, { name: 'NODE_OPTIONS' })?.value) .toEqual('--max-old-space-size=1843'); }); @@ -397,10 +350,9 @@ describe('k8sResource', () => { const kr = new K8sDeploymentResource(terasliceConfig, execution, logger, 'example-job-abcd', 'UID1'); - const exId = kr.resource.metadata?.labels ? kr.resource.metadata.labels['teraslice.terascope.io/exId'] : undefined; - expect(exId) + expect(kr.resource.metadata.labels['teraslice.terascope.io/exId']) .toEqual('e76a0278-d9bc-4d78-bf14-431bcd97528c'); - expect(kr.resource.spec?.template.spec?.containers[0].resources).toEqual(yaml.load(` + expect(kr.resource.spec.template.spec.containers[0].resources).toEqual(yaml.load(` requests: memory: 2147483648 cpu: 1 @@ -408,9 +360,8 @@ describe('k8sResource', () => { memory: 3147483648 cpu: 2`)); - const envArray = kr.resource.spec?.template.spec?.containers[0].env; - const nodeOptionsEnv = _.find(envArray, { name: 'NODE_OPTIONS' }); - expect(nodeOptionsEnv?.value) + const envArray = kr.resource.spec.template.spec.containers[0].env; + expect(_.find(envArray, { name: 'NODE_OPTIONS' })?.value) .toEqual('--max-old-space-size=2702'); }); @@ -419,15 +370,14 @@ describe('k8sResource', () => { const kr = new K8sDeploymentResource(terasliceConfig, execution, logger, 'example-job-abcd', 'UID1'); - expect(kr.resource.spec?.template.spec?.containers[0].resources).toEqual(yaml.load(` + expect(kr.resource.spec.template.spec.containers[0].resources).toEqual(yaml.load(` requests: memory: 2147483648 limits: memory: 2147483648`)); - const envArray = kr.resource.spec?.template.spec?.containers[0].env; - const nodeOptionsEnv = _.find(envArray, { name: 'NODE_OPTIONS' }); - expect(nodeOptionsEnv?.value) + const envArray = kr.resource.spec.template.spec.containers[0].env; + expect(_.find(envArray, { name: 'NODE_OPTIONS' })?.value) .toEqual('--max-old-space-size=1843'); }); @@ -436,7 +386,7 @@ describe('k8sResource', () => { const kr = new K8sDeploymentResource(terasliceConfig, execution, logger, 'example-job-abcd', 'UID1'); - expect(kr.resource.spec?.template.spec?.containers[0].resources).toEqual(yaml.load(` + expect(kr.resource.spec.template.spec.containers[0].resources).toEqual(yaml.load(` requests: cpu: 1 limits: @@ -448,7 +398,7 @@ describe('k8sResource', () => { const kr = new K8sDeploymentResource(terasliceConfig, execution, logger, 'example-job-abcd', 'UID1'); - expect(kr.resource.spec?.template.spec?.containers[0].volumeMounts) + expect(kr.resource.spec.template.spec.containers[0].volumeMounts) .toEqual( [ { mountPath: '/app/config', name: 'config' }, @@ -462,7 +412,7 @@ describe('k8sResource', () => { const kr = new K8sDeploymentResource(terasliceConfig, execution, logger, 'example-job-abcd', 'UID1'); - expect(kr.resource.spec?.template.spec?.containers[0].volumeMounts) + expect(kr.resource.spec.template.spec.containers[0].volumeMounts) .toEqual([{ mountPath: '/app/config', name: 'config' }]); }); }); @@ -472,8 +422,8 @@ describe('k8sResource', () => { execution.targets = []; const kr = new K8sDeploymentResource(terasliceConfig, execution, logger, 'example-job-abcd', 'UID1'); - expect(kr.resource.spec?.template.spec).not.toHaveProperty('affinity'); - expect(kr.resource.spec?.template.spec).not.toHaveProperty('toleration'); + expect(kr.resource.spec.template.spec).not.toHaveProperty('affinity'); + expect(kr.resource.spec.template.spec).not.toHaveProperty('toleration'); }); it('has valid resource object with affinity when execution has one target without constraint', () => { @@ -482,7 +432,7 @@ describe('k8sResource', () => { ]; const kr = new K8sDeploymentResource(terasliceConfig, execution, logger, 'example-job-abcd', 'UID1'); - expect(kr.resource.spec?.template.spec?.affinity).toEqual(yaml.load(` + expect(kr.resource.spec.template.spec.affinity).toEqual(yaml.load(` nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: @@ -500,7 +450,7 @@ describe('k8sResource', () => { terasliceConfig.kubernetes_worker_antiaffinity = true; const kr = new K8sDeploymentResource(terasliceConfig, execution, logger, 'example-job-abcd', 'UID1'); - expect(kr.resource.spec?.template.spec?.affinity).toEqual(yaml.load(` + expect(kr.resource.spec.template.spec.affinity).toEqual(yaml.load(` nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: @@ -532,7 +482,7 @@ describe('k8sResource', () => { ]; const kr = new K8sDeploymentResource(terasliceConfig, execution, logger, 'example-job-abcd', 'UID1'); - expect(kr.resource.spec?.template.spec?.affinity).toEqual(yaml.load(` + expect(kr.resource.spec.template.spec.affinity).toEqual(yaml.load(` nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: @@ -550,7 +500,7 @@ describe('k8sResource', () => { ]; const kr = new K8sDeploymentResource(terasliceConfig, execution, logger, 'example-job-abcd', 'UID1'); - expect(kr.resource.spec?.template.spec?.affinity).toEqual(yaml.load(` + expect(kr.resource.spec.template.spec.affinity).toEqual(yaml.load(` nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: @@ -571,7 +521,7 @@ describe('k8sResource', () => { ]; const kr = new K8sDeploymentResource(terasliceConfig, execution, logger, 'example-job-abcd', 'UID1'); - expect(kr.resource.spec?.template.spec?.affinity).toEqual(yaml.load(` + expect(kr.resource.spec.template.spec.affinity).toEqual(yaml.load(` nodeAffinity: preferredDuringSchedulingIgnoredDuringExecution: - weight: 1 @@ -590,7 +540,7 @@ describe('k8sResource', () => { ]; const kr = new K8sDeploymentResource(terasliceConfig, execution, logger, 'example-job-abcd', 'UID1'); - expect(kr.resource.spec?.template.spec?.affinity).toEqual(yaml.load(` + expect(kr.resource.spec.template.spec.affinity).toEqual(yaml.load(` nodeAffinity: preferredDuringSchedulingIgnoredDuringExecution: - weight: 1 @@ -616,7 +566,7 @@ describe('k8sResource', () => { const kr = new K8sDeploymentResource(terasliceConfig, execution, logger, 'example-job-abcd', 'UID1'); // console.log(yaml.dump(kr.resource.spec.template.spec.tolerations)); - expect(kr.resource.spec?.template.spec?.tolerations).toEqual(yaml.load(` + expect(kr.resource.spec.template.spec.tolerations).toEqual(yaml.load(` - key: zone operator: Equal value: west @@ -631,7 +581,7 @@ describe('k8sResource', () => { const kr = new K8sDeploymentResource(terasliceConfig, execution, logger, 'example-job-abcd', 'UID1'); // console.log(yaml.dump(kr.resource.spec.template.spec.tolerations)); - expect(kr.resource.spec?.template.spec?.tolerations).toEqual(yaml.load(` + expect(kr.resource.spec.template.spec.tolerations).toEqual(yaml.load(` - key: zone operator: Equal value: west @@ -650,7 +600,7 @@ describe('k8sResource', () => { const kr = new K8sDeploymentResource(terasliceConfig, execution, logger, 'example-job-abcd', 'UID1'); // console.log(yaml.dump(kr.resource.spec.template.spec.tolerations)); - expect(kr.resource.spec?.template.spec?.affinity).toEqual(yaml.load(` + expect(kr.resource.spec.template.spec.affinity).toEqual(yaml.load(` nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: @@ -680,7 +630,7 @@ describe('k8sResource', () => { ]; const kr = new K8sDeploymentResource(terasliceConfig, execution, logger, 'example-job-abcd', 'UID1'); - expect(kr.resource.spec?.template.spec?.affinity).toEqual(yaml.load(` + expect(kr.resource.spec.template.spec.affinity).toEqual(yaml.load(` nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: @@ -708,7 +658,7 @@ describe('k8sResource', () => { operator: In values: - texas`)); - expect(kr.resource.spec?.template.spec?.tolerations).toEqual(yaml.load(` + expect(kr.resource.spec.template.spec.tolerations).toEqual(yaml.load(` - key: zone operator: Equal value: west @@ -730,10 +680,8 @@ describe('k8sResource', () => { const kr = new K8sDeploymentResource(terasliceConfig, execution, logger, 'example-job-abcd', 'UID1'); // console.log(yaml.dump(kr.resource)); - const key1Value = kr.resource.metadata?.labels ? kr.resource.metadata.labels['job.teraslice.terascope.io/key1'] : undefined; - const key2Value = kr.resource.metadata?.labels ? kr.resource.metadata.labels['job.teraslice.terascope.io/key2'] : undefined; - expect(key1Value).toEqual('value1'); - expect(key2Value).toEqual('value2'); + expect(kr.resource.metadata.labels['job.teraslice.terascope.io/key1']).toEqual('value1'); + expect(kr.resource.metadata.labels['job.teraslice.terascope.io/key2']).toEqual('value2'); }); it('generates valid k8s resources with keys containing forbidden characters', () => { @@ -746,18 +694,10 @@ describe('k8sResource', () => { const kr = new K8sDeploymentResource(terasliceConfig, execution, logger, 'example-job-abcd', 'UID1'); // console.log(yaml.dump(kr.resource)); - const forbidden1Value = kr.resource.metadata?.labels - ? kr.resource.metadata.labels['job.teraslice.terascope.io/key-1'] - : undefined; - const forbidden2Value = kr.resource.metadata?.labels - ? kr.resource.metadata.labels['job.teraslice.terascope.io/key2-'] - : undefined; - const forbidden3Value = kr.resource.metadata?.labels - ? kr.resource.metadata.labels['job.teraslice.terascope.io/abcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghij123'] - : undefined; - expect(forbidden1Value).toEqual('value1'); - expect(forbidden2Value).toEqual('value2'); - expect(forbidden3Value).toEqual('value3'); + expect(kr.resource.metadata.labels['job.teraslice.terascope.io/key-1']).toEqual('value1'); + expect(kr.resource.metadata.labels['job.teraslice.terascope.io/key2-']).toEqual('value2'); + expect(kr.resource.metadata.labels['job.teraslice.terascope.io/abcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghij123']) + .toEqual('value3'); }); }); @@ -767,7 +707,7 @@ describe('k8sResource', () => { const kr = new K8sDeploymentResource(terasliceConfig, execution, logger, 'example-job-abcd', 'UID1'); // console.log(yaml.dump(kr.resource.spec.template.spec.containers[0].ports)); - expect(kr.resource.spec?.template.spec?.containers[0].ports) + expect(kr.resource.spec.template.spec.containers[0].ports) .toEqual([ { containerPort: 45680 }, { containerPort: 9090 } @@ -779,7 +719,7 @@ describe('k8sResource', () => { const kr = new K8sJobResource(terasliceConfig, execution, logger); // console.log(yaml.dump(kr.resource.spec.template.spec.containers[0].ports)); - expect(kr.resource.spec?.template.spec?.containers[0].ports) + expect(kr.resource.spec.template.spec.containers[0].ports) .toEqual([ { containerPort: 45680 }, { containerPort: 9090 } @@ -796,7 +736,7 @@ describe('k8sResource', () => { const kr = new K8sDeploymentResource(terasliceConfig, execution, logger, 'example-job-abcd', 'UID1'); // console.log(yaml.dump(kr.resource.spec.template.spec.containers[0].ports)); - expect(kr.resource.spec?.template.spec?.containers[0].ports) + expect(kr.resource.spec.template.spec.containers[0].ports) .toEqual([ { containerPort: 45680 }, { containerPort: 9090 }, @@ -809,7 +749,7 @@ describe('k8sResource', () => { const kr = new K8sJobResource(terasliceConfig, execution, logger); // console.log(yaml.dump(kr.resource.spec.template.spec.containers[0].ports)); - expect(kr.resource.spec?.template.spec?.containers[0].ports) + expect(kr.resource.spec.template.spec.containers[0].ports) .toEqual([ { containerPort: 45680 }, { containerPort: 9090 }, @@ -823,33 +763,22 @@ describe('k8sResource', () => { const kr = new K8sJobResource(terasliceConfig, execution, logger); expect(kr.resource.kind).toBe('Job'); - expect(kr.resource.metadata?.name).toBe('ts-exc-example-data-generator-job-7ba9afb0-417a'); + expect(kr.resource.metadata.name).toBe('ts-exc-example-data-generator-job-7ba9afb0-417a'); // The following properties should be absent in the default case - expect(kr.resource.spec?.template.spec).not.toHaveProperty('affinity'); - expect(kr.resource.spec?.template.spec).not.toHaveProperty('imagePullSecrets'); - expect(kr.resource.spec?.template.spec).not.toHaveProperty('priorityClassName'); + expect(kr.resource.spec.template.spec).not.toHaveProperty('affinity'); + expect(kr.resource.spec.template.spec).not.toHaveProperty('imagePullSecrets'); + expect(kr.resource.spec.template.spec).not.toHaveProperty('priorityClassName'); // Configmaps should be mounted on all workers - - let configVolume: k8s.V1Volume | undefined = undefined; - if (kr.resource.spec?.template.spec?.volumes) { - configVolume = kr.resource.spec.template.spec.volumes[0]; - } - - let configVolumeMount: k8s.V1VolumeMount | undefined = undefined; - if (kr.resource.spec?.template.spec?.containers[0].volumeMounts) { - configVolumeMount = kr.resource.spec.template.spec.containers[0].volumeMounts[0]; - } - - expect(configVolume).toEqual(yaml.load(` + expect(kr.resource.spec.template.spec.volumes[0]).toEqual(yaml.load(` name: config configMap: name: ts-dev1-worker items: - key: teraslice.yaml path: teraslice.yaml`)); - expect(configVolumeMount) + expect(kr.resource.spec.template.spec.containers[0].volumeMounts[0]) .toEqual(yaml.load(` mountPath: /app/config name: config`)); @@ -861,7 +790,7 @@ describe('k8sResource', () => { const kr = new K8sJobResource(terasliceConfig, execution, logger); - expect(kr.resource.spec?.template.spec?.containers[0].resources).toEqual(yaml.load(` + expect(kr.resource.spec.template.spec.containers[0].resources).toEqual(yaml.load(` requests: memory: 2147483648 cpu: 1 @@ -869,9 +798,8 @@ describe('k8sResource', () => { memory: 2147483648 cpu: 1`)); - const envArray = kr.resource.spec?.template.spec?.containers[0].env; - const nodeOptionsEnv = _.find(envArray, { name: 'NODE_OPTIONS' }); - expect(nodeOptionsEnv?.value) + const envArray = kr.resource.spec.template.spec.containers[0].env; + expect(_.find(envArray, { name: 'NODE_OPTIONS' })?.value) .toEqual('--max-old-space-size=1843'); }); @@ -883,7 +811,7 @@ describe('k8sResource', () => { const kr = new K8sJobResource(terasliceConfig, execution, logger); - expect(kr.resource.spec?.template.spec?.containers[0].resources).toEqual(yaml.load(` + expect(kr.resource.spec.template.spec.containers[0].resources).toEqual(yaml.load(` requests: memory: 1073741824 cpu: 2 @@ -891,9 +819,8 @@ describe('k8sResource', () => { memory: 1073741824 cpu: 2`)); - const envArray = kr.resource.spec?.template.spec?.containers[0].env; - const nodeOptionsEnv = _.find(envArray, { name: 'NODE_OPTIONS' }); - expect(nodeOptionsEnv?.value) + const envArray = kr.resource.spec.template.spec.containers[0].env; + expect(_.find(envArray, { name: 'NODE_OPTIONS' })?.value) .toEqual('--max-old-space-size=922'); }); }); @@ -911,7 +838,7 @@ describe('k8sResource', () => { execution.name = jobName; const kr = new K8sDeploymentResource(terasliceConfig, execution, logger, 'example-job-abcd', 'UID1'); - expect(kr.resource.metadata?.name).toBe(k8sName); + expect(kr.resource.metadata.name).toBe(k8sName); }); }); @@ -925,11 +852,10 @@ describe('k8sResource', () => { const kr = new K8sJobResource(terasliceConfig, execution, logger); expect(kr.resource.kind).toBe('Job'); - expect(kr.resource.metadata?.name).toBe('ts-exc-example-data-generator-job-7ba9afb0-417a'); - - expect(kr.resource.spec?.template.spec).toHaveProperty('affinity'); - expect(kr.resource.spec?.template.spec).toHaveProperty('tolerations'); - expect(kr.resource.spec?.template.spec?.affinity).toEqual(yaml.load(` + expect(kr.resource.metadata.name).toBe('ts-exc-example-data-generator-job-7ba9afb0-417a'); + expect(kr.resource.spec.template.spec).toHaveProperty('affinity'); + expect(kr.resource.spec.template.spec).toHaveProperty('tolerations'); + expect(kr.resource.spec.template.spec.affinity).toEqual(yaml.load(` nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: @@ -942,7 +868,7 @@ describe('k8sResource', () => { operator: In values: - value2`)); - expect(kr.resource.spec?.template.spec?.tolerations).toEqual(yaml.load(` + expect(kr.resource.spec.template.spec.tolerations).toEqual(yaml.load(` - key: key1 operator: Equal value: value1 @@ -968,13 +894,12 @@ describe('k8sResource', () => { const kr = new K8sJobResource(terasliceConfig, execution, logger); expect(kr.resource.kind).toBe('Job'); - expect(kr.resource.metadata?.name).toBe('ts-exc-example-data-generator-job-7ba9afb0-417a'); - - expect(kr.resource.spec?.template.spec).toHaveProperty('affinity'); - expect(kr.resource.spec?.template.spec).toHaveProperty('tolerations'); + expect(kr.resource.metadata.name).toBe('ts-exc-example-data-generator-job-7ba9afb0-417a'); + expect(kr.resource.spec.template.spec).toHaveProperty('affinity'); + expect(kr.resource.spec.template.spec).toHaveProperty('tolerations'); // console.log(yaml.dump(kr.resource.spec.template.spec.affinity)); - expect(kr.resource.spec?.template.spec?.affinity).toEqual(yaml.load(` + expect(kr.resource.spec.template.spec.affinity).toEqual(yaml.load(` nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: @@ -989,7 +914,7 @@ describe('k8sResource', () => { - value1`)); // console.log(yaml.dump(kr.resource.spec.template.spec.tolerations)); - expect(kr.resource.spec?.template.spec?.tolerations).toEqual(yaml.load(` + expect(kr.resource.spec.template.spec.tolerations).toEqual(yaml.load(` - key: region operator: Equal value: texas @@ -1009,26 +934,19 @@ describe('k8sResource', () => { const krWorker = new K8sDeploymentResource(terasliceConfig, execution, logger, 'example-job-abcd', 'UID1'); expect(krWorker.resource.kind).toBe('Deployment'); - expect(krWorker.resource.spec?.template.spec).toHaveProperty('priorityClassName'); - expect(krWorker.resource.spec?.template.spec?.priorityClassName).toEqual('testPriorityClass'); - - const wkrStatefulLabel = krWorker.resource.spec?.template.metadata?.labels - ? krWorker.resource.spec.template.metadata.labels['job-property.teraslice.terascope.io/stateful'] - : undefined; - expect(wkrStatefulLabel).toEqual('true'); + expect(krWorker.resource.spec.template.spec).toHaveProperty('priorityClassName'); + expect(krWorker.resource.spec.template.spec.priorityClassName).toEqual('testPriorityClass'); + expect(krWorker.resource.spec.template.metadata.labels['job-property.teraslice.terascope.io/stateful']) + .toEqual('true'); const krExporter = new K8sJobResource(terasliceConfig, execution, logger); expect(krExporter.resource.kind).toBe('Job'); - expect(krExporter.resource.metadata?.name).toBe('ts-exc-example-data-generator-job-7ba9afb0-417a'); - - expect(krExporter.resource.spec?.template.spec).toHaveProperty('priorityClassName'); - expect(krExporter.resource.spec?.template.spec?.priorityClassName).toEqual('testPriorityClass'); - - const exporterStatefulLabel = krExporter.resource.spec?.template.metadata?.labels - ? krExporter.resource.spec.template.metadata.labels['job-property.teraslice.terascope.io/stateful'] - : undefined; - expect(exporterStatefulLabel).toEqual('true'); + expect(krExporter.resource.metadata.name).toBe('ts-exc-example-data-generator-job-7ba9afb0-417a'); + expect(krExporter.resource.spec.template.spec).toHaveProperty('priorityClassName'); + expect(krExporter.resource.spec.template.spec.priorityClassName).toEqual('testPriorityClass'); + expect(krExporter.resource.spec.template.metadata.labels['job-property.teraslice.terascope.io/stateful']) + .toEqual('true'); }); }); @@ -1039,7 +957,7 @@ describe('k8sResource', () => { const krWorker = new K8sDeploymentResource(terasliceConfig, execution, logger, 'example-job-abcd', 'UID1'); - expect(krWorker.resource.spec?.template.spec).not.toHaveProperty('initContainers'); + expect(krWorker.resource.spec.template.spec).not.toHaveProperty('initContainers'); }); }); @@ -1050,7 +968,7 @@ describe('k8sResource', () => { const krWorker = new K8sDeploymentResource(terasliceConfig, execution, logger, 'example-job-abcd', 'UID1'); - expect(krWorker.resource.spec?.template.spec).toHaveProperty('initContainers'); + expect(krWorker.resource.spec.template.spec).toHaveProperty('initContainers'); }); }); @@ -1059,14 +977,8 @@ describe('k8sResource', () => { const kr = new K8sServiceResource(terasliceConfig, execution, logger, 'example-job-abcd', 'UID1'); expect(kr.resource.kind).toBe('Service'); - expect(kr.resource.metadata?.name).toBe('svc-ts-exc-example-data-generator-job-7ba9afb0-417a'); - - let serviceSpec: k8s.V1ServiceSpec | undefined = undefined; - if (kr.resource.spec) { - serviceSpec = kr.resource.spec; - } - - expect(serviceSpec).toEqual(yaml.load(` + expect(kr.resource.metadata.name).toBe('svc-ts-exc-example-data-generator-job-7ba9afb0-417a'); + expect(kr.resource.spec).toEqual(yaml.load(` selector: app.kubernetes.io/component: "execution_controller" teraslice.terascope.io/exId: "e76a0278-d9bc-4d78-bf14-431bcd97528c" diff --git a/packages/teraslice/test/lib/cluster/services/cluster/backends/kubernetes/v2/k8sState-multicluster-v2-spec.ts b/packages/teraslice/test/lib/cluster/services/cluster/backends/kubernetes/v2/k8sState-multicluster-v2-spec.ts index b7fd9787a7f..9fd690a9645 100644 --- a/packages/teraslice/test/lib/cluster/services/cluster/backends/kubernetes/v2/k8sState-multicluster-v2-spec.ts +++ b/packages/teraslice/test/lib/cluster/services/cluster/backends/kubernetes/v2/k8sState-multicluster-v2-spec.ts @@ -1,17 +1,14 @@ import _ from 'lodash'; -import { debugLogger } from '@terascope/utils'; -import * as k8s from '@kubernetes/client-node'; import _podsJobRunning from '../files/job-running-v1-k8s-pods-multicluster.json'; import { gen } from '../../../../../../../../src/lib/cluster/services/cluster/backends/kubernetesV2/k8sState.js'; +import { TSPodList } from '../../../../../../../../src/lib/cluster/services/cluster/backends/kubernetesV2/interfaces'; describe('k8sState with pods from multiple clusters', () => { - const logger = debugLogger('k8sResource'); - it('should generate cluster state correctly on first call', () => { - const podsJobRunning: k8s.V1PodList = _.cloneDeep(_podsJobRunning as any); + const podsJobRunning = _.cloneDeep(_podsJobRunning as any); const clusterState = {}; - gen(podsJobRunning, clusterState, logger); + gen(podsJobRunning, clusterState); // console.log(`clusterState\n\n${JSON.stringify(clusterState, null, 2)}`); // console.log(JSON.stringify(podsJobRunning, null, 2)); @@ -42,11 +39,11 @@ describe('k8sState with pods from multiple clusters', () => { }); it('should generate cluster state correctly on second call', () => { - const podsJobRunning = _.cloneDeep(_podsJobRunning as any); + const podsJobRunning = _.cloneDeep(_podsJobRunning as any); const clusterState = {}; - gen(podsJobRunning, clusterState, logger); - gen(podsJobRunning, clusterState, logger); + gen(podsJobRunning, clusterState); + gen(podsJobRunning, clusterState); expect(clusterState['192.168.99.100'].state).toEqual('connected'); expect(clusterState['192.168.99.100'].active.length).toEqual(3); diff --git a/packages/teraslice/test/lib/cluster/services/cluster/backends/kubernetes/v2/k8sState-v2-spec.ts b/packages/teraslice/test/lib/cluster/services/cluster/backends/kubernetes/v2/k8sState-v2-spec.ts index 89db984329e..1e49d74932b 100644 --- a/packages/teraslice/test/lib/cluster/services/cluster/backends/kubernetes/v2/k8sState-v2-spec.ts +++ b/packages/teraslice/test/lib/cluster/services/cluster/backends/kubernetes/v2/k8sState-v2-spec.ts @@ -1,17 +1,14 @@ import _ from 'lodash'; -import { debugLogger } from '@terascope/utils'; -import * as k8s from '@kubernetes/client-node'; import _podsJobRunning from '../files/job-running-v1-k8s-pods.json'; import { gen } from '../../../../../../../../src/lib/cluster/services/cluster/backends/kubernetesV2/k8sState.js'; +import { TSPodList } from '../../../../../../../../src/lib/cluster/services/cluster/backends/kubernetesV2/interfaces.js'; describe('k8sState', () => { - const logger = debugLogger('k8sResource'); - it('should generate cluster state correctly on first call', () => { - const podsJobRunning: k8s.V1PodList = _.cloneDeep(_podsJobRunning as any); + const podsJobRunning = _.cloneDeep(_podsJobRunning as any); const clusterState = {}; - gen(podsJobRunning, clusterState, logger); + gen(podsJobRunning, clusterState); // console.log(`clusterState\n\n${JSON.stringify(clusterState, null, 2)}`); expect(clusterState['192.168.99.100'].state).toEqual('connected'); @@ -41,11 +38,11 @@ describe('k8sState', () => { }); it('should generate cluster state correctly on second call', () => { - const podsJobRunning = _.cloneDeep(_podsJobRunning as any); + const podsJobRunning = _.cloneDeep(_podsJobRunning as any); const clusterState = {}; - gen(podsJobRunning, clusterState, logger); - gen(podsJobRunning, clusterState, logger); + gen(podsJobRunning, clusterState); + gen(podsJobRunning, clusterState); expect(clusterState['192.168.99.100'].state).toEqual('connected'); expect(clusterState['192.168.99.100'].active.length).toEqual(3); @@ -74,14 +71,14 @@ describe('k8sState', () => { }); it('should remove old host ips', () => { - const podsJobRunning = _.cloneDeep(_podsJobRunning as any); + const podsJobRunning = _.cloneDeep(_podsJobRunning as any); const clusterState = {}; clusterState['2.2.2.2'] = { state: 'idk', active: [] }; - gen(podsJobRunning, clusterState, logger); + gen(podsJobRunning, clusterState); expect(clusterState['192.168.99.100'].active.length).toEqual(3); expect(clusterState['2.2.2.2']).toBeUndefined(); }); diff --git a/packages/teraslice/test/lib/cluster/services/cluster/backends/kubernetes/v2/utils-v2-spec.ts b/packages/teraslice/test/lib/cluster/services/cluster/backends/kubernetes/v2/utils-v2-spec.ts index 5324d97833c..53938a0fcc2 100644 --- a/packages/teraslice/test/lib/cluster/services/cluster/backends/kubernetes/v2/utils-v2-spec.ts +++ b/packages/teraslice/test/lib/cluster/services/cluster/backends/kubernetes/v2/utils-v2-spec.ts @@ -1,5 +1,6 @@ -import * as k8s from '@kubernetes/client-node'; -import { K8sConfig } from '../../../../../../../../src/lib/cluster/services/cluster/backends/kubernetesV2/interfaces.js'; +import { + K8sConfig, TSDeployment, TSJob, TSService +} from '../../../../../../../../src/lib/cluster/services/cluster/backends/kubernetesV2/interfaces.js'; import { makeTemplate, getMaxOldSpace } from '../../../../../../../../src/lib/cluster/services/cluster/backends/kubernetesV2/utils.js'; @@ -9,7 +10,7 @@ describe('K8s Utils', () => { describe('->makeTemplate', () => { describe('execution_controller job', () => { it('should be able to support the execution_controller job', () => { - const exJobTemplate = makeTemplate('jobs', 'execution_controller'); + const exJobTemplate = makeTemplate('jobs', 'execution_controller'); const config: K8sConfig = { clusterName: 'teracluster', configMapName: 'teracluster-worker', @@ -94,7 +95,7 @@ describe('K8s Utils', () => { }); describe('worker deployment', () => { - const workerDeploymentTemplate = makeTemplate('deployments', 'worker'); + const workerDeploymentTemplate = makeTemplate('deployments', 'worker'); it('should be able to support the worker deployment', () => { const config: K8sConfig = { clusterName: 'teracluster', @@ -235,7 +236,7 @@ describe('K8s Utils', () => { }); describe('execution_controller service', () => { - const exServiceTemplate = makeTemplate('services', 'execution_controller'); + const exServiceTemplate = makeTemplate('services', 'execution_controller'); it('should be able to support the execution_controller service', () => { const config: K8sConfig = { clusterName: 'teracluster', From 73a00e347d189a1ab225283c3f3e11c0cfced5a0 Mon Sep 17 00:00:00 2001 From: busma13 Date: Wed, 6 Nov 2024 09:21:45 -0700 Subject: [PATCH 21/30] release: (minor) teraslice@2.7.0 --- package.json | 2 +- packages/teraslice/package.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/package.json b/package.json index 811f6188e3e..7faed5abf1f 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { "name": "teraslice-workspace", "displayName": "Teraslice", - "version": "2.6.4", + "version": "2.7.0", "private": true, "homepage": "https://github.com/terascope/teraslice", "bugs": { diff --git a/packages/teraslice/package.json b/packages/teraslice/package.json index 9f9e9b38ac9..4140df8081e 100644 --- a/packages/teraslice/package.json +++ b/packages/teraslice/package.json @@ -1,7 +1,7 @@ { "name": "teraslice", "displayName": "Teraslice", - "version": "2.6.4", + "version": "2.7.0", "description": "Distributed computing platform for processing JSON data", "homepage": "https://github.com/terascope/teraslice#readme", "bugs": { From 2e135009387a7e21c41fd4d64fb07078567bc081 Mon Sep 17 00:00:00 2001 From: busma13 Date: Wed, 6 Nov 2024 10:32:20 -0700 Subject: [PATCH 22/30] make kubernetes_api_poll_delay mandatory in teraslice Config type --- packages/job-components/src/test-helpers.ts | 1 + .../lib/cluster/services/cluster/backends/kubernetes/index.ts | 1 - .../cluster/services/cluster/backends/kubernetesV2/index.ts | 4 ---- packages/types/src/teraslice.ts | 2 +- 4 files changed, 2 insertions(+), 6 deletions(-) diff --git a/packages/job-components/src/test-helpers.ts b/packages/job-components/src/test-helpers.ts index 4a154083da0..dd913b4c31a 100644 --- a/packages/job-components/src/test-helpers.ts +++ b/packages/job-components/src/test-helpers.ts @@ -174,6 +174,7 @@ export class TestContext implements i.Context { analytics: 'yearly', state: 'montly', }, + kubernetes_api_poll_delay: 1000, master_hostname: 'localhost', master: false, name: testName, diff --git a/packages/teraslice/src/lib/cluster/services/cluster/backends/kubernetes/index.ts b/packages/teraslice/src/lib/cluster/services/cluster/backends/kubernetes/index.ts index d5f239c3cd7..9e9b9e26134 100644 --- a/packages/teraslice/src/lib/cluster/services/cluster/backends/kubernetes/index.ts +++ b/packages/teraslice/src/lib/cluster/services/cluster/backends/kubernetes/index.ts @@ -41,7 +41,6 @@ export class KubernetesClusterBackend { this.logger, null, kubernetesNamespace, - // @ts-expect-error context.sysconfig.teraslice.kubernetes_api_poll_delay, context.sysconfig.teraslice.shutdown_timeout ); diff --git a/packages/teraslice/src/lib/cluster/services/cluster/backends/kubernetesV2/index.ts b/packages/teraslice/src/lib/cluster/services/cluster/backends/kubernetesV2/index.ts index 2ca7b06a0ea..86eed65992b 100644 --- a/packages/teraslice/src/lib/cluster/services/cluster/backends/kubernetesV2/index.ts +++ b/packages/teraslice/src/lib/cluster/services/cluster/backends/kubernetesV2/index.ts @@ -31,10 +31,6 @@ export class KubernetesClusterBackendV2 { readonly clusterNameLabel: string; constructor(context: Context, clusterMasterServer: any) { - if (!context.sysconfig.teraslice.kubernetes_api_poll_delay) { - throw new Error('Kubernetes clustering requires kubernetes_api_poll_delay to be defined.'); - } - const kubernetesNamespace = get(context, 'sysconfig.teraslice.kubernetes_namespace', 'default'); const clusterName = get(context, 'sysconfig.teraslice.name'); diff --git a/packages/types/src/teraslice.ts b/packages/types/src/teraslice.ts index 9769e3546ed..8ea4627da5b 100644 --- a/packages/types/src/teraslice.ts +++ b/packages/types/src/teraslice.ts @@ -480,7 +480,7 @@ export interface Config { execution_controller_targets?: ExecutionControllerTargets[]; hostname: string; index_rollover_frequency: IndexRolloverFrequency; - kubernetes_api_poll_delay?: number | 1000; + kubernetes_api_poll_delay: number | 1000; kubernetes_config_map_name?: string | 'teraslice-worker'; kubernetes_image_pull_secret?: string | ''; kubernetes_image?: string | 'terascope/teraslice'; From d27ce691d3e414c6aa962a34c71c2483ef4486a3 Mon Sep 17 00:00:00 2001 From: busma13 Date: Wed, 6 Nov 2024 11:05:57 -0700 Subject: [PATCH 23/30] remove unneeded undefined checks, typos --- .../cluster/backends/kubernetesV2/k8s.ts | 2 +- .../backends/kubernetesV2/k8sResource.ts | 22 ++++--------- .../v2/k8sState-multicluster-v2-spec.ts | 2 +- .../backends/kubernetes/v2/utils-v2-spec.ts | 32 +++++++++---------- 4 files changed, 25 insertions(+), 33 deletions(-) diff --git a/packages/teraslice/src/lib/cluster/services/cluster/backends/kubernetesV2/k8s.ts b/packages/teraslice/src/lib/cluster/services/cluster/backends/kubernetesV2/k8s.ts index 6378e8796f7..878ab62335a 100644 --- a/packages/teraslice/src/lib/cluster/services/cluster/backends/kubernetesV2/k8s.ts +++ b/packages/teraslice/src/lib/cluster/services/cluster/backends/kubernetesV2/k8s.ts @@ -321,7 +321,7 @@ export class K8s { * Deletes k8s object of specified objType * @param {String} name Name of the resource to delete * @param {ResourceType} objType Type of k8s object to get, valid options: - * 'deployments', 'services', 'jobs', 'pods', 'replicasetss' + * 'deployments', 'services', 'jobs', 'pods', 'replicasets' * @param {Boolean} force Forcefully delete resource by setting gracePeriodSeconds to 1 * to be forcefully stopped. * @return {Object} k8s delete response body. diff --git a/packages/teraslice/src/lib/cluster/services/cluster/backends/kubernetesV2/k8sResource.ts b/packages/teraslice/src/lib/cluster/services/cluster/backends/kubernetesV2/k8sResource.ts index e3195c37381..cc76216e7d0 100644 --- a/packages/teraslice/src/lib/cluster/services/cluster/backends/kubernetesV2/k8sResource.ts +++ b/packages/teraslice/src/lib/cluster/services/cluster/backends/kubernetesV2/k8sResource.ts @@ -191,7 +191,7 @@ export abstract class K8sResource { } _setImagePullSecret(resource: TSJob | TSDeployment) { - if (this.terasliceConfig.kubernetes_image_pull_secret && resource.spec.template.spec) { + if (this.terasliceConfig.kubernetes_image_pull_secret) { if (resource.spec.template.spec.imagePullSecrets) { resource.spec.template.spec.imagePullSecrets.push( { name: this.terasliceConfig.kubernetes_image_pull_secret } @@ -209,20 +209,14 @@ export abstract class K8sResource { const className = this.terasliceConfig.kubernetes_priority_class_name; if (this.nodeType === 'execution_controller') { - if (resource.spec.template.spec) { - resource.spec.template.spec.priorityClassName = className; - } - if (this.execution.stateful && resource.spec.template.metadata.labels) { + resource.spec.template.spec.priorityClassName = className; + if (this.execution.stateful) { resource.spec.template.metadata.labels[`${this.jobPropertyLabelPrefix}/stateful`] = 'true'; } } if (this.nodeType === 'worker' && this.execution.stateful) { - if (resource.spec.template.spec) { - resource.spec.template.spec.priorityClassName = className; - } - if (resource.spec.template.metadata.labels) { - resource.spec.template.metadata.labels[`${this.jobPropertyLabelPrefix}/stateful`] = 'true'; - } + resource.spec.template.spec.priorityClassName = className; + resource.spec.template.metadata.labels[`${this.jobPropertyLabelPrefix}/stateful`] = 'true'; } } } @@ -249,10 +243,8 @@ export abstract class K8sResource { const key = `${this.jobLabelPrefix}/${_.replace(k, /[^a-zA-Z0-9\-._]/g, '-').substring(0, 63)}`; const value = _.replace(v, /[^a-zA-Z0-9\-._]/g, '-').substring(0, 63); - if (resource.metadata.labels && resource.spec.template.metadata.labels) { - resource.metadata.labels[key] = value; - resource.spec.template.metadata.labels[key] = value; - } + resource.metadata.labels[key] = value; + resource.spec.template.metadata.labels[key] = value; }); } } diff --git a/packages/teraslice/test/lib/cluster/services/cluster/backends/kubernetes/v2/k8sState-multicluster-v2-spec.ts b/packages/teraslice/test/lib/cluster/services/cluster/backends/kubernetes/v2/k8sState-multicluster-v2-spec.ts index 9fd690a9645..0a383c6619d 100644 --- a/packages/teraslice/test/lib/cluster/services/cluster/backends/kubernetes/v2/k8sState-multicluster-v2-spec.ts +++ b/packages/teraslice/test/lib/cluster/services/cluster/backends/kubernetes/v2/k8sState-multicluster-v2-spec.ts @@ -1,7 +1,7 @@ import _ from 'lodash'; import _podsJobRunning from '../files/job-running-v1-k8s-pods-multicluster.json'; import { gen } from '../../../../../../../../src/lib/cluster/services/cluster/backends/kubernetesV2/k8sState.js'; -import { TSPodList } from '../../../../../../../../src/lib/cluster/services/cluster/backends/kubernetesV2/interfaces'; +import { TSPodList } from '../../../../../../../../src/lib/cluster/services/cluster/backends/kubernetesV2/interfaces.js'; describe('k8sState with pods from multiple clusters', () => { it('should generate cluster state correctly on first call', () => { diff --git a/packages/teraslice/test/lib/cluster/services/cluster/backends/kubernetes/v2/utils-v2-spec.ts b/packages/teraslice/test/lib/cluster/services/cluster/backends/kubernetes/v2/utils-v2-spec.ts index 53938a0fcc2..6a9c9f3d028 100644 --- a/packages/teraslice/test/lib/cluster/services/cluster/backends/kubernetes/v2/utils-v2-spec.ts +++ b/packages/teraslice/test/lib/cluster/services/cluster/backends/kubernetes/v2/utils-v2-spec.ts @@ -43,13 +43,13 @@ describe('K8s Utils', () => { namespace: config.namespace }); - expect(exJob.spec?.template.metadata?.labels).toEqual(exJob.metadata?.labels); + expect(exJob.spec.template.metadata.labels).toEqual(exJob.metadata.labels); - const templateSpec = exJob.spec?.template.spec; + const templateSpec = exJob.spec.template.spec; - expect(templateSpec?.containers[0].image).toEqual(config.dockerImage); - expect(templateSpec?.containers[0].name).toEqual(config.name); - expect(templateSpec?.containers[0].env).toEqual([ + expect(templateSpec.containers[0].image).toEqual(config.dockerImage); + expect(templateSpec.containers[0].name).toEqual(config.name); + expect(templateSpec.containers[0].env).toEqual([ { name: 'NODE_TYPE', value: config.nodeType @@ -67,7 +67,7 @@ describe('K8s Utils', () => { } } ]); - expect(templateSpec?.terminationGracePeriodSeconds).toEqual(config.shutdownTimeout); + expect(templateSpec.terminationGracePeriodSeconds).toEqual(config.shutdownTimeout); }); it('should throw error if docker image undefined on config for job', () => { @@ -140,16 +140,16 @@ describe('K8s Utils', () => { ], }); - expect(workerDeployment.spec?.replicas).toEqual(config.replicas); + expect(workerDeployment.spec.replicas).toEqual(config.replicas); - const labels = workerDeployment.spec?.template.metadata?.labels; - expect(labels).toEqual(workerDeployment.metadata?.labels); + const labels = workerDeployment.spec.template.metadata.labels; + expect(labels).toEqual(workerDeployment.metadata.labels); - const templateSpec = workerDeployment.spec?.template.spec; + const templateSpec = workerDeployment.spec.template.spec; - expect(templateSpec?.containers[0].image).toEqual(config.dockerImage); - expect(templateSpec?.containers[0].name).toEqual(config.name); - expect(templateSpec?.containers[0].env).toEqual([ + expect(templateSpec.containers[0].image).toEqual(config.dockerImage); + expect(templateSpec.containers[0].name).toEqual(config.name); + expect(templateSpec.containers[0].env).toEqual([ { name: 'NODE_TYPE', value: config.nodeType @@ -167,7 +167,7 @@ describe('K8s Utils', () => { } } ]); - expect(templateSpec?.terminationGracePeriodSeconds).toEqual(config.shutdownTimeout); + expect(templateSpec.terminationGracePeriodSeconds).toEqual(config.shutdownTimeout); }); it('should throw error if docker image undefined on config for deployment', () => { @@ -279,12 +279,12 @@ describe('K8s Utils', () => { ], }); - expect(exService.spec?.selector).toEqual({ + expect(exService.spec.selector).toEqual({ 'app.kubernetes.io/component': 'execution_controller', 'teraslice.terascope.io/exId': config.exId }); - expect(exService.spec?.ports).toEqual([ + expect(exService.spec.ports).toEqual([ { port: 45680, targetPort: 45680 From 39425b502e70d37e3afa7a75d0accb422befa3b2 Mon Sep 17 00:00:00 2001 From: busma13 Date: Wed, 6 Nov 2024 11:09:18 -0700 Subject: [PATCH 24/30] missed an unneeded check --- .../cluster/backends/kubernetesV2/k8sResource.ts | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/packages/teraslice/src/lib/cluster/services/cluster/backends/kubernetesV2/k8sResource.ts b/packages/teraslice/src/lib/cluster/services/cluster/backends/kubernetesV2/k8sResource.ts index cc76216e7d0..c12b6bfe88d 100644 --- a/packages/teraslice/src/lib/cluster/services/cluster/backends/kubernetesV2/k8sResource.ts +++ b/packages/teraslice/src/lib/cluster/services/cluster/backends/kubernetesV2/k8sResource.ts @@ -414,11 +414,9 @@ export abstract class K8sResource { * Job setting: `pod_spec_override` */ _mergePodSpecOverlay(resource: TSJob | TSDeployment) { - if (resource.spec.template.spec) { - resource.spec.template.spec = _.merge( - resource.spec.template.spec, - this.execution.pod_spec_override - ); - } + resource.spec.template.spec = _.merge( + resource.spec.template.spec, + this.execution.pod_spec_override + ); } } From cd572d3673eba44da11a4d2be51a36bd9016db1f Mon Sep 17 00:00:00 2001 From: busma13 Date: Wed, 6 Nov 2024 14:52:19 -0700 Subject: [PATCH 25/30] merge resource template into instance of resource class. Update imports --- .../cluster/backends/kubernetesV2/index.ts | 12 +++ .../backends/kubernetesV2/interfaces.ts | 6 -- .../cluster/backends/kubernetesV2/k8s.ts | 83 ++++++++-------- .../kubernetesV2/k8sDeploymentResource.ts | 9 +- .../backends/kubernetesV2/k8sJobResource.ts | 9 +- .../backends/kubernetesV2/k8sResource.ts | 3 +- .../kubernetesV2/k8sServiceResource.ts | 8 +- .../cluster/backends/kubernetesV2/utils.ts | 98 +++++++++---------- 8 files changed, 116 insertions(+), 112 deletions(-) diff --git a/packages/teraslice/src/lib/cluster/services/cluster/backends/kubernetesV2/index.ts b/packages/teraslice/src/lib/cluster/services/cluster/backends/kubernetesV2/index.ts index 86eed65992b..c2086c43855 100644 --- a/packages/teraslice/src/lib/cluster/services/cluster/backends/kubernetesV2/index.ts +++ b/packages/teraslice/src/lib/cluster/services/cluster/backends/kubernetesV2/index.ts @@ -119,6 +119,10 @@ export class KubernetesClusterBackendV2 { const jobResult = await this.k8s.post(exJob); + if (!jobResult.metadata.uid) { + throw new Error('Required field uid missing from jobResult.metadata'); + } + const exServiceResource = new K8sServiceResource( this.context.sysconfig.teraslice, execution, @@ -134,6 +138,10 @@ export class KubernetesClusterBackendV2 { this.logger.debug(jobResult, 'k8s slicer job submitted'); + if (!jobResult.spec.selector?.matchLabels) { + throw new Error('Required field matchLabels missing from jobResult.spec.selector'); + } + let controllerLabel: string; if (jobResult.spec.selector.matchLabels['controller-uid'] !== undefined) { /// If running on kubernetes < v1.27.0 @@ -189,6 +197,10 @@ export class KubernetesClusterBackendV2 { this.context.sysconfig.teraslice.slicer_timeout ); + if (!jobs.items[0].metadata.uid) { + throw new Error('Required field uid missing from kubernetes job metadata'); + } + const kr = new K8sDeploymentResource( this.context.sysconfig.teraslice, execution, diff --git a/packages/teraslice/src/lib/cluster/services/cluster/backends/kubernetesV2/interfaces.ts b/packages/teraslice/src/lib/cluster/services/cluster/backends/kubernetesV2/interfaces.ts index b8b28cd5e01..a220775d82f 100644 --- a/packages/teraslice/src/lib/cluster/services/cluster/backends/kubernetesV2/interfaces.ts +++ b/packages/teraslice/src/lib/cluster/services/cluster/backends/kubernetesV2/interfaces.ts @@ -67,7 +67,6 @@ export interface TSJob extends k8s.V1Job { [key: string]: string; }; name: string; - uid: string; }; spec: NonNullable & { template: k8s.V1PodTemplateSpec & { @@ -84,11 +83,6 @@ export interface TSJob extends k8s.V1Job { volumes: NonNullable; }; }; - selector: NonNullable & { - matchLabels: { - [key: string]: string; - }; - }; }; } diff --git a/packages/teraslice/src/lib/cluster/services/cluster/backends/kubernetesV2/k8s.ts b/packages/teraslice/src/lib/cluster/services/cluster/backends/kubernetesV2/k8s.ts index 878ab62335a..d0757ef4797 100644 --- a/packages/teraslice/src/lib/cluster/services/cluster/backends/kubernetesV2/k8s.ts +++ b/packages/teraslice/src/lib/cluster/services/cluster/backends/kubernetesV2/k8s.ts @@ -7,16 +7,7 @@ import { convertToTSResource, convertToTSResourceList, getRetryConfig, isDeployment, isJob, isPod, isReplicaSet, isService, isTSPod } from './utils.js'; -import { - DeleteApiResponse, DeleteParams, DeleteResponseBody, - KubeConfigOptions, K8sResource, ListParams, - NodeType, PatchApiResponse, ResourceApiResponse, - ResourceListApiResponse, ResourceType, ScaleOp, - TSDeployment, TSDeploymentList, TSJob, - TSJobList, TSPod, TSPodList, - TSReplicaSet, TSReplicaSetList, TSResource, - TSResourceList, TSService, TSServiceList -} from './interfaces.js'; +import * as i from './interfaces.js'; export class K8s { logger: Logger; @@ -30,7 +21,7 @@ export class K8s { constructor( logger: Logger, - clientConfig: KubeConfigOptions | null, + clientConfig: i.KubeConfigOptions | null, defaultNamespace: string | null, apiPollDelay: number, shutdownTimeout: number @@ -170,17 +161,17 @@ export class K8s { * | k8s.V1ReplicaSetList * | k8s.V1JobList} list of k8s objects. */ - async list(selector: string, objType: 'deployments', ns?: string): Promise; - async list(selector: string, objType: 'jobs', ns?: string): Promise; - async list(selector: string, objType: 'pods', ns?: string): Promise; - async list(selector: string, objType: 'replicasets', ns?: string): Promise; - async list(selector: string, objType: 'services', ns?: string): Promise; - async list(selector: string, objType: ResourceType, ns?: string): Promise; - async list(selector: string, objType: ResourceType, ns?: string): Promise { + async list(selector: string, objType: 'deployments', ns?: string): Promise; + async list(selector: string, objType: 'jobs', ns?: string): Promise; + async list(selector: string, objType: 'pods', ns?: string): Promise; + async list(selector: string, objType: 'replicasets', ns?: string): Promise; + async list(selector: string, objType: 'services', ns?: string): Promise; + async list(selector: string, objType: i.ResourceType, ns?: string): Promise; + async list(selector: string, objType: i.ResourceType, ns?: string): Promise { const namespace = ns || this.defaultNamespace; - let responseObj: ResourceListApiResponse; + let responseObj: i.ResourceListApiResponse; - const params: ListParams = [ + const params: i.ListParams = [ namespace, undefined, undefined, @@ -228,7 +219,7 @@ export class K8s { } } - async nonEmptyJobList(selector: string): Promise { + async nonEmptyJobList(selector: string): Promise { const jobs = await this.list(selector, 'jobs'); if (jobs.items.length === 1) { return jobs; @@ -248,13 +239,13 @@ export class K8s { * @param {K8sResource} manifest resource manifest * @return {K8sResource} body of k8s API response object */ - async post(manifest: k8s.V1Deployment): Promise; - async post(manifest: k8s.V1Job): Promise; - async post(manifest: k8s.V1Pod): Promise; - async post(manifest: k8s.V1ReplicaSet): Promise; - async post(manifest: k8s.V1Service): Promise; - async post(manifest: K8sResource): Promise { - let responseObj: ResourceApiResponse; + async post(manifest: k8s.V1Deployment): Promise; + async post(manifest: k8s.V1Job): Promise; + async post(manifest: k8s.V1Pod): Promise; + async post(manifest: k8s.V1ReplicaSet): Promise; + async post(manifest: k8s.V1Service): Promise; + async post(manifest: i.K8sResource): Promise { + let responseObj: i.ResourceApiResponse; try { if (isDeployment(manifest)) { @@ -294,7 +285,7 @@ export class K8s { // the low level k8s api method, I expect to eventually change the interface // on this to require `objType` to support patching other things async patch(record: Record, name: string) { - let responseObj: PatchApiResponse; + let responseObj: i.PatchApiResponse; try { const options = { headers: { 'Content-type': k8s.PatchUtils.PATCH_FORMAT_JSON_PATCH } }; responseObj = await pRetry(() => this.k8sAppsV1Api @@ -336,16 +327,16 @@ export class K8s { name: string, objType: 'services', force?: boolean ): Promise; async delete( - name: string, objType: ResourceType, force?: boolean - ): Promise; + name: string, objType: i.ResourceType, force?: boolean + ): Promise; async delete( - name: string, objType: ResourceType, force?: boolean - ): Promise { + name: string, objType: i.ResourceType, force?: boolean + ): Promise { if (name === undefined || name.trim() === '') { throw new Error(`Name of resource to delete must be specified. Received: "${name}".`); } - let responseObj: DeleteApiResponse; + let responseObj: i.DeleteApiResponse; // To get a Job to remove the associated pods you have to // include a body like the one below with the delete request. @@ -360,7 +351,7 @@ export class K8s { deleteOptions.gracePeriodSeconds = 1; } - const params: DeleteParams = [ + const params: i.DeleteParams = [ name, this.defaultNamespace, undefined, @@ -372,8 +363,8 @@ export class K8s { ]; const deleteWithErrorHandling = async ( - deleteFn: () => Promise - ): Promise => { + deleteFn: () => Promise + ): Promise => { try { const res = await deleteFn(); return res; @@ -453,21 +444,21 @@ export class K8s { * @return {Promise<(k8s.V1Pod | k8s.V1Status | k8s.V1Service)[]>} */ async _deleteObjByExId( - exId: string, nodeType: NodeType, objType: 'pods', force?: boolean + exId: string, nodeType: i.NodeType, objType: 'pods', force?: boolean ): Promise; async _deleteObjByExId( - exId: string, nodeType: NodeType, objType: 'jobs' | 'replicasets' | 'deployments', force?: boolean + exId: string, nodeType: i.NodeType, objType: 'jobs' | 'replicasets' | 'deployments', force?: boolean ): Promise; async _deleteObjByExId( - exId: string, nodeType: NodeType, objType: 'services', force?: boolean + exId: string, nodeType: i.NodeType, objType: 'services', force?: boolean ): Promise; async _deleteObjByExId( - exId: string, nodeType: NodeType, objType: ResourceType, force?: boolean - ): Promise { - let objList: TSResourceList; + exId: string, nodeType: i.NodeType, objType: i.ResourceType, force?: boolean + ): Promise { + let objList: i.TSResourceList; const deleteResponses: Array = []; try { @@ -515,7 +506,11 @@ export class K8s { * @param {ScaleOp} op Scale operation: `set`, `add`, `remove` * @return {Promise} Body of patch response. */ - async scaleExecution(exId: string, numWorkers: number, op: ScaleOp): Promise { + async scaleExecution( + exId: string, + numWorkers: number, + op: i.ScaleOp + ): Promise { let newScale: number; const selector = `app.kubernetes.io/component=worker,teraslice.terascope.io/exId=${exId}`; diff --git a/packages/teraslice/src/lib/cluster/services/cluster/backends/kubernetesV2/k8sDeploymentResource.ts b/packages/teraslice/src/lib/cluster/services/cluster/backends/kubernetesV2/k8sDeploymentResource.ts index b011e2ec4d1..d235dd61f91 100644 --- a/packages/teraslice/src/lib/cluster/services/cluster/backends/kubernetesV2/k8sDeploymentResource.ts +++ b/packages/teraslice/src/lib/cluster/services/cluster/backends/kubernetesV2/k8sDeploymentResource.ts @@ -1,13 +1,14 @@ import { Logger } from '@terascope/utils'; import type { Config, ExecutionConfig } from '@terascope/types'; -import { makeTemplate } from './utils.js'; +import { convertToTSResource, makeTemplate } from './utils.js'; import { K8sConfig, NodeType, TSDeployment } from './interfaces.js'; import { K8sResource } from './k8sResource.js'; +import { V1Deployment } from '@kubernetes/client-node'; export class K8sDeploymentResource extends K8sResource { nodeType: NodeType = 'worker'; nameInfix = 'wkr'; - templateGenerator: (config: K8sConfig) => TSDeployment; + templateGenerator: (config: K8sConfig) => V1Deployment; templateConfig; resource; exName: string; @@ -39,7 +40,9 @@ export class K8sDeploymentResource extends K8sResource { this.exUid = exUid; this.templateGenerator = makeTemplate('deployments', this.nodeType); this.templateConfig = this._makeConfig(this.nameInfix, exName, exUid); - this.resource = this.templateGenerator(this.templateConfig); + const k8sDeployment = new V1Deployment(); + Object.assign(k8sDeployment, this.templateGenerator(this.templateConfig)); + this.resource = convertToTSResource(k8sDeployment); this._setJobLabels(this.resource); diff --git a/packages/teraslice/src/lib/cluster/services/cluster/backends/kubernetesV2/k8sJobResource.ts b/packages/teraslice/src/lib/cluster/services/cluster/backends/kubernetesV2/k8sJobResource.ts index 35c04ee5e4c..b827554fb85 100644 --- a/packages/teraslice/src/lib/cluster/services/cluster/backends/kubernetesV2/k8sJobResource.ts +++ b/packages/teraslice/src/lib/cluster/services/cluster/backends/kubernetesV2/k8sJobResource.ts @@ -1,13 +1,14 @@ +import { V1Job } from '@kubernetes/client-node'; import { Logger } from '@terascope/utils'; import type { Config, ExecutionConfig } from '@terascope/types'; -import { makeTemplate } from './utils.js'; +import { convertToTSResource, makeTemplate } from './utils.js'; import { K8sConfig, NodeType, TSJob } from './interfaces.js'; import { K8sResource } from './k8sResource.js'; export class K8sJobResource extends K8sResource { nodeType: NodeType = 'execution_controller'; nameInfix = 'exc'; - templateGenerator: (config: K8sConfig) => TSJob; + templateGenerator: (config: K8sConfig) => V1Job; templateConfig; resource; @@ -28,7 +29,9 @@ export class K8sJobResource extends K8sResource { super(terasliceConfig, execution, logger); this.templateGenerator = makeTemplate('jobs', this.nodeType); this.templateConfig = this._makeConfig(this.nameInfix); - this.resource = this.templateGenerator(this.templateConfig); + const k8sJob = new V1Job(); + Object.assign(k8sJob, this.templateGenerator(this.templateConfig)); + this.resource = convertToTSResource(k8sJob); this._setJobLabels(this.resource); diff --git a/packages/teraslice/src/lib/cluster/services/cluster/backends/kubernetesV2/k8sResource.ts b/packages/teraslice/src/lib/cluster/services/cluster/backends/kubernetesV2/k8sResource.ts index c12b6bfe88d..93b6972280c 100644 --- a/packages/teraslice/src/lib/cluster/services/cluster/backends/kubernetesV2/k8sResource.ts +++ b/packages/teraslice/src/lib/cluster/services/cluster/backends/kubernetesV2/k8sResource.ts @@ -1,4 +1,5 @@ import _ from 'lodash'; +import { V1Deployment, V1Job, V1Service } from '@kubernetes/client-node'; import { isNumber, Logger } from '@terascope/utils'; import type { Config, ExecutionConfig } from '@terascope/types'; import { safeEncode } from '../../../../../utils/encoding_utils.js'; @@ -16,7 +17,7 @@ export abstract class K8sResource { terasliceConfig: Config; abstract nodeType: NodeType; abstract nameInfix: string; - abstract templateGenerator: (config: K8sConfig) => T; + abstract templateGenerator: (config: K8sConfig) => V1Deployment | V1Job | V1Service; abstract templateConfig: K8sConfig; abstract resource: T; diff --git a/packages/teraslice/src/lib/cluster/services/cluster/backends/kubernetesV2/k8sServiceResource.ts b/packages/teraslice/src/lib/cluster/services/cluster/backends/kubernetesV2/k8sServiceResource.ts index 2f1e871dd7d..c049b9d55c9 100644 --- a/packages/teraslice/src/lib/cluster/services/cluster/backends/kubernetesV2/k8sServiceResource.ts +++ b/packages/teraslice/src/lib/cluster/services/cluster/backends/kubernetesV2/k8sServiceResource.ts @@ -1,8 +1,9 @@ import { Logger } from '@terascope/utils'; import type { Config, ExecutionConfig } from '@terascope/types'; -import { makeTemplate } from './utils.js'; +import { convertToTSResource, makeTemplate } from './utils.js'; import { K8sConfig, NodeType, TSService } from './interfaces.js'; import { K8sResource } from './k8sResource.js'; +import { V1Service } from '@kubernetes/client-node'; export class K8sServiceResource extends K8sResource { nodeType: NodeType = 'execution_controller'; @@ -40,6 +41,9 @@ export class K8sServiceResource extends K8sResource { this.exUid = exUid; this.templateGenerator = makeTemplate('services', this.nodeType); this.templateConfig = this._makeConfig(this.nameInfix, exName, exUid); - this.resource = this.templateGenerator(this.templateConfig); + + const k8sService = new V1Service(); + Object.assign(k8sService, this.templateGenerator(this.templateConfig)); + this.resource = convertToTSResource(k8sService); } } diff --git a/packages/teraslice/src/lib/cluster/services/cluster/backends/kubernetesV2/utils.ts b/packages/teraslice/src/lib/cluster/services/cluster/backends/kubernetesV2/utils.ts index e283bd2c70d..180d9d65ac3 100644 --- a/packages/teraslice/src/lib/cluster/services/cluster/backends/kubernetesV2/utils.ts +++ b/packages/teraslice/src/lib/cluster/services/cluster/backends/kubernetesV2/utils.ts @@ -4,27 +4,21 @@ import path from 'node:path'; import barbe from 'barbe'; import { isTest } from '@terascope/utils'; import * as k8s from '@kubernetes/client-node'; -import { - K8sConfig, NodeType, K8sResource, - TSDeployment, TSJob, TSPod, TSService, - TSReplicaSet, TSResource, K8sResourceList, - TSDeploymentList, TSJobList, TSPodList, - TSReplicaSetList, TSResourceList, TSServiceList -} from './interfaces.js'; +import * as i from './interfaces.js'; const MAX_RETRIES = isTest ? 2 : 3; const RETRY_DELAY = isTest ? 50 : 1000; // time in ms const resourcePath = path.join(process.cwd(), './packages/teraslice/src/lib/cluster/services/cluster/backends/kubernetesV2/'); -export function makeTemplate( +export function makeTemplate( folder: 'deployments' | 'jobs' | 'services', - fileName: NodeType -): (config: K8sConfig) => T { + fileName: i.NodeType +): (config: i.K8sConfig) => T { const filePath = path.join(resourcePath, folder, `${fileName}.hbs`); const templateData = fs.readFileSync(filePath, 'utf-8'); const templateKeys = ['{{', '}}']; - return (config: K8sConfig) => { + return (config: i.K8sConfig) => { if (folder !== 'jobs' && (config.exName === undefined || config.exUid === undefined)) { throw new Error(`K8s config requires ${config.exName === undefined ? 'exName' : 'exUid'} to create a ${folder} template`); } @@ -72,27 +66,27 @@ export function getRetryConfig() { }; } -export function isDeployment(resource: K8sResource): resource is k8s.V1Deployment { - return resource instanceof k8s.V1Deployment || resource.kind === 'Deployment'; +export function isDeployment(resource: i.K8sResource): resource is k8s.V1Deployment { + return resource instanceof k8s.V1Deployment; } -export function isJob(resource: K8sResource): resource is k8s.V1Job { - return resource instanceof k8s.V1Job || resource.kind === 'Job'; +export function isJob(resource: i.K8sResource): resource is k8s.V1Job { + return resource instanceof k8s.V1Job; } -export function isPod(resource: K8sResource): resource is k8s.V1Pod { - return resource instanceof k8s.V1Pod || resource.kind === 'Pod'; +export function isPod(resource: i.K8sResource): resource is k8s.V1Pod { + return resource instanceof k8s.V1Pod; } -export function isReplicaSet(resource: K8sResource): resource is k8s.V1ReplicaSet { - return resource instanceof k8s.V1ReplicaSet || resource.kind === 'ReplicaSet'; +export function isReplicaSet(resource: i.K8sResource): resource is k8s.V1ReplicaSet { + return resource instanceof k8s.V1ReplicaSet; } -export function isService(resource: K8sResource): resource is k8s.V1Service { - return resource instanceof k8s.V1Service || resource.kind === 'Service'; +export function isService(resource: i.K8sResource): resource is k8s.V1Service { + return resource instanceof k8s.V1Service; } -export function isTSDeployment(manifest: k8s.V1Deployment): manifest is TSDeployment { +export function isTSDeployment(manifest: k8s.V1Deployment): manifest is i.TSDeployment { return manifest instanceof k8s.V1Deployment && manifest.metadata?.labels !== undefined && manifest.metadata.name !== undefined @@ -102,43 +96,41 @@ export function isTSDeployment(manifest: k8s.V1Deployment): manifest is TSDeploy && manifest.spec.template.spec.volumes !== undefined; } -export function isTSJob(manifest: k8s.V1Job): manifest is TSJob { +export function isTSJob(manifest: k8s.V1Job): manifest is i.TSJob { return manifest instanceof k8s.V1Job && manifest.metadata?.labels !== undefined && manifest.metadata.name !== undefined - && manifest.metadata.uid !== undefined && manifest.spec?.template.metadata?.labels !== undefined && manifest.spec.template.spec?.containers[0].volumeMounts !== undefined - && manifest.spec.template.spec.volumes !== undefined - && manifest.spec.selector?.matchLabels !== undefined; + && manifest.spec.template.spec.volumes !== undefined; } -export function isTSPod(manifest: K8sResource): manifest is TSPod { +export function isTSPod(manifest: i.K8sResource): manifest is i.TSPod { return manifest instanceof k8s.V1Pod && manifest.metadata?.name !== undefined && manifest.status !== undefined; } -export function isTSReplicaSet(manifest: k8s.V1ReplicaSet): manifest is TSReplicaSet { +export function isTSReplicaSet(manifest: k8s.V1ReplicaSet): manifest is i.TSReplicaSet { return manifest instanceof k8s.V1ReplicaSet && manifest.metadata?.name !== undefined && manifest.status !== undefined; } -export function isTSService(manifest: k8s.V1Service): manifest is TSService { +export function isTSService(manifest: k8s.V1Service): manifest is i.TSService { return manifest instanceof k8s.V1Service && manifest.metadata?.name !== undefined && manifest.spec?.selector !== undefined && manifest.spec.ports !== undefined; } -export function convertToTSResource(resource: k8s.V1Deployment): TSDeployment; -export function convertToTSResource(resource: k8s.V1Job): TSJob; -export function convertToTSResource(resource: k8s.V1Pod): TSPod; -export function convertToTSResource(resource: k8s.V1ReplicaSet): TSReplicaSet; -export function convertToTSResource(resource: k8s.V1Service): TSService; -export function convertToTSResource(resource: K8sResource): TSResource; -export function convertToTSResource(resource: K8sResource): TSResource { +export function convertToTSResource(resource: k8s.V1Deployment): i.TSDeployment; +export function convertToTSResource(resource: k8s.V1Job): i.TSJob; +export function convertToTSResource(resource: k8s.V1Pod): i.TSPod; +export function convertToTSResource(resource: k8s.V1ReplicaSet): i.TSReplicaSet; +export function convertToTSResource(resource: k8s.V1Service): i.TSService; +export function convertToTSResource(resource: i.K8sResource): i.TSResource; +export function convertToTSResource(resource: i.K8sResource): i.TSResource { if (isDeployment(resource) && isTSDeployment(resource)) { return resource; } @@ -158,58 +150,58 @@ export function convertToTSResource(resource: K8sResource): TSResource { throw new Error('K8sResource missing required field(s) to be converted to TSResource.'); } -export function isDeploymentList(manifest: K8sResourceList): manifest is k8s.V1DeploymentList { +export function isDeploymentList(manifest: i.K8sResourceList): manifest is k8s.V1DeploymentList { return manifest.kind === 'DeploymentList'; } -export function isJobList(manifest: K8sResourceList): manifest is k8s.V1JobList { +export function isJobList(manifest: i.K8sResourceList): manifest is k8s.V1JobList { return manifest.kind === 'JobList'; } -export function isPodList(manifest: K8sResourceList): manifest is k8s.V1PodList { +export function isPodList(manifest: i.K8sResourceList): manifest is k8s.V1PodList { return manifest.kind === 'PodList'; } -export function isReplicaSetList(manifest: K8sResourceList): manifest is k8s.V1ReplicaSetList { +export function isReplicaSetList(manifest: i.K8sResourceList): manifest is k8s.V1ReplicaSetList { return manifest.kind === 'ReplicaSetList'; } -export function isServiceList(manifest: K8sResourceList): manifest is k8s.V1ServiceList { +export function isServiceList(manifest: i.K8sResourceList): manifest is k8s.V1ServiceList { return manifest.kind === 'ServiceList'; } -export function isTSDeploymentList(manifest: k8s.V1DeploymentList): manifest is TSDeploymentList { +export function isTSDeploymentList(manifest: k8s.V1DeploymentList): manifest is i.TSDeploymentList { return manifest.kind === 'DeploymentList' && (manifest.items[0] ? isTSDeployment(manifest.items[0]) : true); } -export function isTSJobList(manifest: k8s.V1JobList): manifest is TSJobList { +export function isTSJobList(manifest: k8s.V1JobList): manifest is i.TSJobList { return manifest.kind === 'JobList' && (manifest.items[0] ? isTSJob(manifest.items[0]) : true); } -export function isTSPodList(manifest: k8s.V1PodList): manifest is TSPodList { +export function isTSPodList(manifest: k8s.V1PodList): manifest is i.TSPodList { return manifest.kind === 'PodList' && (manifest.items[0] ? isTSPod(manifest.items[0]) : true); } -export function isTSReplicaSetList(manifest: k8s.V1ReplicaSetList): manifest is TSReplicaSetList { +export function isTSReplicaSetList(manifest: k8s.V1ReplicaSetList): manifest is i.TSReplicaSetList { return manifest.kind === 'ReplicaSetList' && (manifest.items[0] ? isTSReplicaSet(manifest.items[0]) : true); } -export function isTSServiceList(manifest: k8s.V1ServiceList): manifest is TSServiceList { +export function isTSServiceList(manifest: k8s.V1ServiceList): manifest is i.TSServiceList { return manifest.kind === 'ServiceList' && (manifest.items[0] ? isTSService(manifest.items[0]) : true); } -export function convertToTSResourceList(resourceList: k8s.V1DeploymentList): TSDeploymentList; -export function convertToTSResourceList(resourceList: k8s.V1JobList): TSJobList; -export function convertToTSResourceList(resourceList: k8s.V1PodList): TSPodList; -export function convertToTSResourceList(resourceList: k8s.V1ReplicaSetList): TSReplicaSetList; -export function convertToTSResourceList(resourceList: k8s.V1ServiceList): TSServiceList; -export function convertToTSResourceList(resourceList: K8sResourceList): TSResourceList; -export function convertToTSResourceList(resourceList: K8sResourceList): TSResourceList { +export function convertToTSResourceList(resourceList: k8s.V1DeploymentList): i.TSDeploymentList; +export function convertToTSResourceList(resourceList: k8s.V1JobList): i.TSJobList; +export function convertToTSResourceList(resourceList: k8s.V1PodList): i.TSPodList; +export function convertToTSResourceList(resourceList: k8s.V1ReplicaSetList): i.TSReplicaSetList; +export function convertToTSResourceList(resourceList: k8s.V1ServiceList): i.TSServiceList; +export function convertToTSResourceList(resourceList: i.K8sResourceList): i.TSResourceList; +export function convertToTSResourceList(resourceList: i.K8sResourceList): i.TSResourceList { resourceList.items.map((resource) => convertToTSResource(resource)); if (isDeploymentList(resourceList) && isTSDeploymentList(resourceList)) { From 370df23343a0fd11f4600c62c498cb580b93ded8 Mon Sep 17 00:00:00 2001 From: busma13 Date: Wed, 6 Nov 2024 15:19:51 -0700 Subject: [PATCH 26/30] reformat imports --- .../services/cluster/backends/kubernetesV2/interfaces.ts | 2 ++ .../cluster/services/cluster/backends/kubernetesV2/k8s.ts | 5 +++-- .../cluster/backends/kubernetesV2/k8sDeploymentResource.ts | 2 +- .../services/cluster/backends/kubernetesV2/k8sResource.ts | 6 ------ .../cluster/backends/kubernetesV2/k8sServiceResource.ts | 2 +- .../cluster/services/cluster/backends/kubernetesV2/utils.ts | 4 +++- 6 files changed, 10 insertions(+), 11 deletions(-) diff --git a/packages/teraslice/src/lib/cluster/services/cluster/backends/kubernetesV2/interfaces.ts b/packages/teraslice/src/lib/cluster/services/cluster/backends/kubernetesV2/interfaces.ts index a220775d82f..b594d64bd63 100644 --- a/packages/teraslice/src/lib/cluster/services/cluster/backends/kubernetesV2/interfaces.ts +++ b/packages/teraslice/src/lib/cluster/services/cluster/backends/kubernetesV2/interfaces.ts @@ -51,6 +51,7 @@ export interface TSDeployment extends k8s.V1Deployment { }; spec: NonNullable & { containers: k8s.V1Container[] & { + env: NonNullable[]; ports: NonNullable; volumeMounts: [k8s.V1VolumeMount, ...k8s.V1VolumeMount[]]; }[]; @@ -77,6 +78,7 @@ export interface TSJob extends k8s.V1Job { }; spec: NonNullable & { containers: k8s.V1Container[] & { + env: NonNullable[]; ports: NonNullable; volumeMounts: [k8s.V1VolumeMount, ...k8s.V1VolumeMount[]]; }[]; diff --git a/packages/teraslice/src/lib/cluster/services/cluster/backends/kubernetesV2/k8s.ts b/packages/teraslice/src/lib/cluster/services/cluster/backends/kubernetesV2/k8s.ts index d0757ef4797..36da79ec9cb 100644 --- a/packages/teraslice/src/lib/cluster/services/cluster/backends/kubernetesV2/k8s.ts +++ b/packages/teraslice/src/lib/cluster/services/cluster/backends/kubernetesV2/k8s.ts @@ -4,8 +4,9 @@ import { } from '@terascope/utils'; import * as k8s from '@kubernetes/client-node'; import { - convertToTSResource, convertToTSResourceList, getRetryConfig, isDeployment, - isJob, isPod, isReplicaSet, isService, isTSPod + convertToTSResource, convertToTSResourceList, getRetryConfig, + isDeployment, isJob, isPod, + isReplicaSet, isService, isTSPod } from './utils.js'; import * as i from './interfaces.js'; diff --git a/packages/teraslice/src/lib/cluster/services/cluster/backends/kubernetesV2/k8sDeploymentResource.ts b/packages/teraslice/src/lib/cluster/services/cluster/backends/kubernetesV2/k8sDeploymentResource.ts index d235dd61f91..72122e279df 100644 --- a/packages/teraslice/src/lib/cluster/services/cluster/backends/kubernetesV2/k8sDeploymentResource.ts +++ b/packages/teraslice/src/lib/cluster/services/cluster/backends/kubernetesV2/k8sDeploymentResource.ts @@ -1,9 +1,9 @@ +import { V1Deployment } from '@kubernetes/client-node'; import { Logger } from '@terascope/utils'; import type { Config, ExecutionConfig } from '@terascope/types'; import { convertToTSResource, makeTemplate } from './utils.js'; import { K8sConfig, NodeType, TSDeployment } from './interfaces.js'; import { K8sResource } from './k8sResource.js'; -import { V1Deployment } from '@kubernetes/client-node'; export class K8sDeploymentResource extends K8sResource { nodeType: NodeType = 'worker'; diff --git a/packages/teraslice/src/lib/cluster/services/cluster/backends/kubernetesV2/k8sResource.ts b/packages/teraslice/src/lib/cluster/services/cluster/backends/kubernetesV2/k8sResource.ts index 93b6972280c..f0f6899da38 100644 --- a/packages/teraslice/src/lib/cluster/services/cluster/backends/kubernetesV2/k8sResource.ts +++ b/packages/teraslice/src/lib/cluster/services/cluster/backends/kubernetesV2/k8sResource.ts @@ -271,9 +271,6 @@ export abstract class K8sResource { let maxMemory; const container = resource.spec.template.spec.containers[0]; - if (container === undefined) { - throw new Error('Resource container undefined while setting resources.'); - } // use teraslice config as defaults and execution config will override it const envVars = Object.assign({}, this.terasliceConfig.env_vars, this.execution.env_vars); @@ -323,9 +320,6 @@ export abstract class K8sResource { // NOTE: This sucks, this manages the memory env var but it ALSO is // responsible for doing the config and execution env var merge, which // should NOT be in this function - if (container.env === undefined) { - throw new Error('Resource container undefined while setting resources.'); - } setMaxOldSpaceViaEnv(container.env, envVars, maxMemory as number); } diff --git a/packages/teraslice/src/lib/cluster/services/cluster/backends/kubernetesV2/k8sServiceResource.ts b/packages/teraslice/src/lib/cluster/services/cluster/backends/kubernetesV2/k8sServiceResource.ts index c049b9d55c9..9658d2c1eac 100644 --- a/packages/teraslice/src/lib/cluster/services/cluster/backends/kubernetesV2/k8sServiceResource.ts +++ b/packages/teraslice/src/lib/cluster/services/cluster/backends/kubernetesV2/k8sServiceResource.ts @@ -1,9 +1,9 @@ +import { V1Service } from '@kubernetes/client-node'; import { Logger } from '@terascope/utils'; import type { Config, ExecutionConfig } from '@terascope/types'; import { convertToTSResource, makeTemplate } from './utils.js'; import { K8sConfig, NodeType, TSService } from './interfaces.js'; import { K8sResource } from './k8sResource.js'; -import { V1Service } from '@kubernetes/client-node'; export class K8sServiceResource extends K8sResource { nodeType: NodeType = 'execution_controller'; diff --git a/packages/teraslice/src/lib/cluster/services/cluster/backends/kubernetesV2/utils.ts b/packages/teraslice/src/lib/cluster/services/cluster/backends/kubernetesV2/utils.ts index 180d9d65ac3..49dbeb20cba 100644 --- a/packages/teraslice/src/lib/cluster/services/cluster/backends/kubernetesV2/utils.ts +++ b/packages/teraslice/src/lib/cluster/services/cluster/backends/kubernetesV2/utils.ts @@ -2,8 +2,8 @@ import fs from 'node:fs'; import path from 'node:path'; // @ts-expect-error import barbe from 'barbe'; -import { isTest } from '@terascope/utils'; import * as k8s from '@kubernetes/client-node'; +import { isTest } from '@terascope/utils'; import * as i from './interfaces.js'; const MAX_RETRIES = isTest ? 2 : 3; @@ -93,6 +93,7 @@ export function isTSDeployment(manifest: k8s.V1Deployment): manifest is i.TSDepl && manifest.spec?.replicas !== undefined && manifest.spec.template.metadata?.labels !== undefined && manifest.spec.template.spec?.containers[0].volumeMounts !== undefined + && manifest.spec.template.spec?.containers[0].env !== undefined && manifest.spec.template.spec.volumes !== undefined; } @@ -102,6 +103,7 @@ export function isTSJob(manifest: k8s.V1Job): manifest is i.TSJob { && manifest.metadata.name !== undefined && manifest.spec?.template.metadata?.labels !== undefined && manifest.spec.template.spec?.containers[0].volumeMounts !== undefined + && manifest.spec.template.spec?.containers[0].env !== undefined && manifest.spec.template.spec.volumes !== undefined; } From 3787911e7b4893ec7a188b585c3b7aec65692072 Mon Sep 17 00:00:00 2001 From: busma13 Date: Wed, 6 Nov 2024 15:49:48 -0700 Subject: [PATCH 27/30] k8s-v2-spec update resource creation. env can't be required - not returned from list() --- .../backends/kubernetesV2/interfaces.ts | 2 - .../backends/kubernetesV2/k8sResource.ts | 3 + .../cluster/backends/kubernetesV2/utils.ts | 2 - .../backends/kubernetes/v2/k8s-v2-spec.ts | 38 +++++++------ .../kubernetes/v2/k8sResource-v2-spec.ts | 57 ++++--------------- 5 files changed, 36 insertions(+), 66 deletions(-) diff --git a/packages/teraslice/src/lib/cluster/services/cluster/backends/kubernetesV2/interfaces.ts b/packages/teraslice/src/lib/cluster/services/cluster/backends/kubernetesV2/interfaces.ts index b594d64bd63..a220775d82f 100644 --- a/packages/teraslice/src/lib/cluster/services/cluster/backends/kubernetesV2/interfaces.ts +++ b/packages/teraslice/src/lib/cluster/services/cluster/backends/kubernetesV2/interfaces.ts @@ -51,7 +51,6 @@ export interface TSDeployment extends k8s.V1Deployment { }; spec: NonNullable & { containers: k8s.V1Container[] & { - env: NonNullable[]; ports: NonNullable; volumeMounts: [k8s.V1VolumeMount, ...k8s.V1VolumeMount[]]; }[]; @@ -78,7 +77,6 @@ export interface TSJob extends k8s.V1Job { }; spec: NonNullable & { containers: k8s.V1Container[] & { - env: NonNullable[]; ports: NonNullable; volumeMounts: [k8s.V1VolumeMount, ...k8s.V1VolumeMount[]]; }[]; diff --git a/packages/teraslice/src/lib/cluster/services/cluster/backends/kubernetesV2/k8sResource.ts b/packages/teraslice/src/lib/cluster/services/cluster/backends/kubernetesV2/k8sResource.ts index f0f6899da38..b297732aed8 100644 --- a/packages/teraslice/src/lib/cluster/services/cluster/backends/kubernetesV2/k8sResource.ts +++ b/packages/teraslice/src/lib/cluster/services/cluster/backends/kubernetesV2/k8sResource.ts @@ -320,6 +320,9 @@ export abstract class K8sResource { // NOTE: This sucks, this manages the memory env var but it ALSO is // responsible for doing the config and execution env var merge, which // should NOT be in this function + if (container.env === undefined) { + throw new Error('Resource container V1EnvVar[] undefined while setting resources.'); + } setMaxOldSpaceViaEnv(container.env, envVars, maxMemory as number); } diff --git a/packages/teraslice/src/lib/cluster/services/cluster/backends/kubernetesV2/utils.ts b/packages/teraslice/src/lib/cluster/services/cluster/backends/kubernetesV2/utils.ts index 49dbeb20cba..30c99c3ad17 100644 --- a/packages/teraslice/src/lib/cluster/services/cluster/backends/kubernetesV2/utils.ts +++ b/packages/teraslice/src/lib/cluster/services/cluster/backends/kubernetesV2/utils.ts @@ -93,7 +93,6 @@ export function isTSDeployment(manifest: k8s.V1Deployment): manifest is i.TSDepl && manifest.spec?.replicas !== undefined && manifest.spec.template.metadata?.labels !== undefined && manifest.spec.template.spec?.containers[0].volumeMounts !== undefined - && manifest.spec.template.spec?.containers[0].env !== undefined && manifest.spec.template.spec.volumes !== undefined; } @@ -103,7 +102,6 @@ export function isTSJob(manifest: k8s.V1Job): manifest is i.TSJob { && manifest.metadata.name !== undefined && manifest.spec?.template.metadata?.labels !== undefined && manifest.spec.template.spec?.containers[0].volumeMounts !== undefined - && manifest.spec.template.spec?.containers[0].env !== undefined && manifest.spec.template.spec.volumes !== undefined; } diff --git a/packages/teraslice/test/lib/cluster/services/cluster/backends/kubernetes/v2/k8s-v2-spec.ts b/packages/teraslice/test/lib/cluster/services/cluster/backends/kubernetes/v2/k8s-v2-spec.ts index a6556aa35a6..590698fa3f4 100644 --- a/packages/teraslice/test/lib/cluster/services/cluster/backends/kubernetes/v2/k8s-v2-spec.ts +++ b/packages/teraslice/test/lib/cluster/services/cluster/backends/kubernetes/v2/k8s-v2-spec.ts @@ -2,6 +2,10 @@ // output for nock // env DEBUG='nock*' make test +import { + V1Deployment, V1Job, V1Pod, + V1ReplicaSet, V1Service +} from '@kubernetes/client-node'; import nock from 'nock'; import { debugLogger } from '@terascope/job-components'; import { K8s } from '../../../../../../../../src/lib/cluster/services/cluster/backends/kubernetesV2/k8s.js'; @@ -14,7 +18,7 @@ const _url = 'http://mock.kube.api'; describe('k8s', () => { let k8s: K8s; - const job = { + const job = Object.assign(new V1Job(), { apiVersion: '1.0.0', kind: 'Job', metadata: { @@ -44,23 +48,23 @@ describe('k8s', () => { } } } - }; + }); - const testPod1 = { + const testPod1 = Object.assign(new V1Pod(), { apiVersion: 'v1', kind: 'Pod', metadata: { name: 'testPod1' }, status: {} - }; + }); - const testPod2 = { + const testPod2 = Object.assign(new V1Pod(), { apiVersion: 'v1', kind: 'Pod', metadata: { name: 'testPod2' }, status: {} - }; + }); - const service = { + const service = Object.assign(new V1Service(), { kind: 'Service', metadata: { name: 'service1' @@ -73,9 +77,9 @@ describe('k8s', () => { { port: 45680 } ] } - }; + }); - const deployment = { + const deployment = Object.assign(new V1Deployment(), { apiVersion: 'v1', kind: 'Deployment', metadata: { @@ -102,16 +106,16 @@ describe('k8s', () => { selector: {} }, status: {} - }; + }); - const replicaSet = { + const replicaSet = Object.assign(new V1ReplicaSet(), { kind: 'ReplicaSet', metadata: { name: 'replicaset1' }, status: {} - }; + }); const status = { apiVersion: 'v1', @@ -275,7 +279,7 @@ describe('k8s', () => { .post('/apis/apps/v1/namespaces/default/deployments') .reply(201, deployment); - const response = await k8s.post({ kind: 'Deployment' }); + const response = await k8s.post(deployment); expect(response.kind).toEqual('Deployment'); }); @@ -284,7 +288,7 @@ describe('k8s', () => { .post('/apis/batch/v1/namespaces/default/jobs') .reply(201, job); - const response = await k8s.post({ kind: 'Job' }); + const response = await k8s.post(job); expect(response.kind).toEqual('Job'); }); @@ -293,7 +297,7 @@ describe('k8s', () => { .post('/api/v1/namespaces/default/pods') .reply(201, testPod1); - const response = await k8s.post({ kind: 'Pod' }); + const response = await k8s.post(testPod1); expect(response.kind).toEqual('Pod'); }); @@ -302,7 +306,7 @@ describe('k8s', () => { .post('/apis/apps/v1/namespaces/default/replicasets') .reply(201, replicaSet); - const response = await k8s.post({ kind: 'ReplicaSet' }); + const response = await k8s.post(replicaSet); expect(response.kind).toEqual('ReplicaSet'); }); @@ -311,7 +315,7 @@ describe('k8s', () => { .post('/api/v1/namespaces/default/services') .reply(201, service); - const response = await k8s.post({ kind: 'Service' }); + const response = await k8s.post(service); expect(response.kind).toEqual('Service'); }); }); diff --git a/packages/teraslice/test/lib/cluster/services/cluster/backends/kubernetes/v2/k8sResource-v2-spec.ts b/packages/teraslice/test/lib/cluster/services/cluster/backends/kubernetes/v2/k8sResource-v2-spec.ts index d8fde334749..febba386773 100644 --- a/packages/teraslice/test/lib/cluster/services/cluster/backends/kubernetes/v2/k8sResource-v2-spec.ts +++ b/packages/teraslice/test/lib/cluster/services/cluster/backends/kubernetes/v2/k8sResource-v2-spec.ts @@ -52,24 +52,15 @@ describe('k8sResource', () => { expect(kr.resource.spec.template.spec).not.toHaveProperty('imagePullSecrets'); expect(kr.resource.spec.template.spec).not.toHaveProperty('priorityClassName'); - let configVolume: k8s.V1Volume | undefined = undefined; - if (kr.resource.spec.template.spec.volumes) { - configVolume = kr.resource.spec.template.spec.volumes[0]; - } - - let configVolumeMount: k8s.V1VolumeMount | undefined = undefined; - if (kr.resource.spec.template.spec.containers[0].volumeMounts) { - configVolumeMount = kr.resource.spec.template.spec.containers[0].volumeMounts[0]; - } // Configmaps should be mounted on all workers - expect(configVolume).toEqual(yaml.load(` + expect(kr.resource.spec.template.spec.volumes[0]).toEqual(yaml.load(` name: config configMap: name: ts-dev1-worker items: - key: teraslice.yaml path: teraslice.yaml`)); - expect(configVolumeMount) + expect(kr.resource.spec.template.spec.containers[0].volumeMounts[0]) .toEqual(yaml.load(` mountPath: /app/config name: config`)); @@ -121,16 +112,6 @@ describe('k8sResource', () => { const kr = new K8sDeploymentResource(terasliceConfig, execution, logger, 'example-job-abcd', 'UID1'); - let configVolumeMount: k8s.V1VolumeMount | undefined = undefined; - if (kr.resource.spec.template.spec.containers[0].volumeMounts) { - configVolumeMount = kr.resource.spec.template.spec.containers[0].volumeMounts[0]; - } - - let assetVolumeMount: k8s.V1VolumeMount | undefined = undefined; - if (kr.resource.spec.template.spec.containers[0].volumeMounts) { - assetVolumeMount = kr.resource.spec.template.spec.containers[0].volumeMounts[1]; - } - expect(kr.resource.spec.replicas).toBe(2); expect(kr.resource.metadata.name).toBe('ts-wkr-example-data-generator-job-7ba9afb0-417a'); @@ -144,7 +125,7 @@ describe('k8sResource', () => { items: - key: teraslice.yaml path: teraslice.yaml`)); - expect(configVolumeMount) + expect(kr.resource.spec.template.spec.containers[0].volumeMounts[0]) .toEqual(yaml.load(` mountPath: /app/config name: config`)); @@ -154,7 +135,7 @@ describe('k8sResource', () => { name: asset-volume persistentVolumeClaim: claimName: asset-volume`)); - expect(assetVolumeMount) + expect(kr.resource.spec.template.spec.containers[0].volumeMounts[1]) .toEqual(yaml.load(` name: asset-volume mountPath: /assets`)); @@ -167,14 +148,6 @@ describe('k8sResource', () => { const kr = new K8sDeploymentResource(terasliceConfig, execution, logger, 'example-job-abcd', 'UID1'); - const configVolumeMount = kr.resource.spec.template.spec.containers[0].volumeMounts - ? kr.resource.spec.template.spec.containers[0].volumeMounts[0] - : undefined; - - const dataVolumeMount = kr.resource.spec.template.spec.containers[0].volumeMounts - ? kr.resource.spec.template.spec.containers[0].volumeMounts[1] - : undefined; - // First check the configMap volumes, which should be present on all // deployments expect(kr.resource.spec.template.spec.volumes[0]).toEqual(yaml.load(` @@ -184,7 +157,7 @@ describe('k8sResource', () => { items: - key: teraslice.yaml path: teraslice.yaml`)); - expect(configVolumeMount) + expect(kr.resource.spec.template.spec.containers[0].volumeMounts[0]) .toEqual(yaml.load(` mountPath: /app/config name: config`)); @@ -194,7 +167,7 @@ describe('k8sResource', () => { name: teraslice-data1 persistentVolumeClaim: claimName: teraslice-data1`)); - expect(dataVolumeMount) + expect(kr.resource.spec.template.spec.containers[0].volumeMounts[1]) .toEqual(yaml.load(` name: teraslice-data1 mountPath: /data`)); @@ -208,20 +181,12 @@ describe('k8sResource', () => { const kr = new K8sDeploymentResource(terasliceConfig, execution, logger, 'example-job-abcd', 'UID1'); - const dataVolumeMount = kr.resource.spec.template.spec.containers[0].volumeMounts - ? kr.resource.spec.template.spec.containers[0].volumeMounts[1] - : undefined; - - const tmpVolumeMount = kr.resource.spec.template.spec.containers[0].volumeMounts - ? kr.resource.spec.template.spec.containers[0].volumeMounts[2] - : undefined; - // Now check for the volumes added via job expect(kr.resource.spec.template.spec.volumes[1]).toEqual(yaml.load(` name: teraslice-data1 persistentVolumeClaim: claimName: teraslice-data1`)); - expect(dataVolumeMount) + expect(kr.resource.spec.template.spec.containers[0].volumeMounts[1]) .toEqual(yaml.load(` name: teraslice-data1 mountPath: /data`)); @@ -230,7 +195,7 @@ describe('k8sResource', () => { name: tmp persistentVolumeClaim: claimName: tmp`)); - expect(tmpVolumeMount) + expect(kr.resource.spec.template.spec.containers[0].volumeMounts[2]) .toEqual(yaml.load(` name: tmp mountPath: /tmp`)); @@ -272,8 +237,10 @@ describe('k8sResource', () => { // NOTE: the env var merge happens in _setResources(), which is // somewhat out of place. const envArray = kr.resource.spec.template.spec.containers[0].env; - expect(_.find(envArray, { name: 'FOO' })?.value).toEqual('baz'); - expect(_.find(envArray, { name: 'EXAMPLE' })?.value).toEqual('test'); + expect(_.find(envArray, { name: 'FOO' })?.value) + .toEqual('baz'); + expect(_.find(envArray, { name: 'EXAMPLE' })?.value) + .toEqual('test'); }); it('execution resources override terasliceConfig resources', () => { From 60ca244528f807960d5d7651a4a58d1c1bc4dd7d Mon Sep 17 00:00:00 2001 From: busma13 Date: Thu, 7 Nov 2024 09:15:30 -0700 Subject: [PATCH 28/30] remove comment --- .../services/cluster/backends/kubernetes/v2/k8s-v2-spec.ts | 4 ---- 1 file changed, 4 deletions(-) diff --git a/packages/teraslice/test/lib/cluster/services/cluster/backends/kubernetes/v2/k8s-v2-spec.ts b/packages/teraslice/test/lib/cluster/services/cluster/backends/kubernetes/v2/k8s-v2-spec.ts index 590698fa3f4..71a0bfabfc7 100644 --- a/packages/teraslice/test/lib/cluster/services/cluster/backends/kubernetes/v2/k8s-v2-spec.ts +++ b/packages/teraslice/test/lib/cluster/services/cluster/backends/kubernetes/v2/k8s-v2-spec.ts @@ -1,7 +1,3 @@ -// You can set the following environment variable to generate more verbose debug -// output for nock -// env DEBUG='nock*' make test - import { V1Deployment, V1Job, V1Pod, V1ReplicaSet, V1Service From f70948948d4e81d3416a6a35ad6aa07c7abba326 Mon Sep 17 00:00:00 2001 From: busma13 Date: Thu, 7 Nov 2024 11:51:43 -0700 Subject: [PATCH 29/30] remove defaults from teraslice Config type --- docs/configuration/clustering-k8s.md | 1 + packages/job-components/src/test-helpers.ts | 14 ++++- packages/types/src/teraslice.ts | 60 ++++++++++----------- 3 files changed, 44 insertions(+), 31 deletions(-) diff --git a/docs/configuration/clustering-k8s.md b/docs/configuration/clustering-k8s.md index 7282442bf27..eb166a9689e 100644 --- a/docs/configuration/clustering-k8s.md +++ b/docs/configuration/clustering-k8s.md @@ -99,6 +99,7 @@ support k8s based Teraslice deployments. | assets_volume | Name of kubernetes volume to be shared across all pods, where Teraslice assets will be stored | String | optional | | cpu_execution_controller | CPU resources to use for Execution Controller request and limit values | Number | optional | | execution_controller_targets | array of `{"key": "rack", "value": "alpha"}` targets for execution controllers | String | optional | +| kubernetes_api_poll_delay | Specify the delay between attempts to poll the kubernetes API, `1000` by default | Number | optional | | kubernetes_image | Name of docker image, default: `teraslice:k8sdev` | String | optional | | kubernetes_image_pull_secret | Secret used to pull docker images from private repo | String | optional | | kubernetes_config_map_name | Name of the configmap used by worker and execution_controller containers for config. If this is not provided, the default will be `-worker` | String | optional | diff --git a/packages/job-components/src/test-helpers.ts b/packages/job-components/src/test-helpers.ts index dd913b4c31a..01cf7a5a204 100644 --- a/packages/job-components/src/test-helpers.ts +++ b/packages/job-components/src/test-helpers.ts @@ -164,19 +164,31 @@ export class TestContext implements i.Context { teraslice: { action_timeout: 10000, analytics_rate: 10000, + api_response_timeout: 300000, assets_directory: path.join(process.cwd(), 'assets'), + assets_volume: '', asset_storage_connection_type: 'elasticsearch-next', asset_storage_connection: 'default', asset_storage_bucket: '', cluster_manager_type: options.cluster_manager_type || 'native', + cpu_execution_controller: 0.5, + ephemeral_storage: false, hostname: 'localhost', index_rollover_frequency: { analytics: 'yearly', - state: 'montly', + state: 'monthly', }, kubernetes_api_poll_delay: 1000, + kubernetes_config_map_name: 'teraslice-worker', + kubernetes_image_pull_secret: '', + kubernetes_image: 'terascope/teraslice', + kubernetes_namespace: 'default', + kubernetes_overrides_enabled: false, + kubernetes_priority_class_name: undefined, + kubernetes_worker_antiaffinity: false, master_hostname: 'localhost', master: false, + memory_execution_controller: 512000000, name: testName, network_latency_buffer: 100, node_disconnect_timeout: 5000, diff --git a/packages/types/src/teraslice.ts b/packages/types/src/teraslice.ts index 8ea4627da5b..fecd83b7a5d 100644 --- a/packages/types/src/teraslice.ts +++ b/packages/types/src/teraslice.ts @@ -453,7 +453,7 @@ export interface ExecutionControllerTargets { value: string; } -export type RolloverFrequency = 'daily' | 'montly' | 'yearly'; +export type RolloverFrequency = 'daily' | 'monthly' | 'yearly'; // TODO: is this really double? export interface IndexRolloverFrequency { @@ -462,51 +462,51 @@ export interface IndexRolloverFrequency { } export interface Config { - action_timeout: number | 300000; - analytics_rate: number | 60000; - api_response_timeout?: number | 300000; - assets_directory?: string[] | string; + action_timeout: number; + analytics_rate: number; + api_response_timeout: number; + assets_directory: string[] | string; asset_storage_connection_type: string; asset_storage_connection: string; asset_storage_bucket: string; - assets_volume?: string; + assets_volume: string; cluster_manager_type: ClusterManagerType; /** This will only be available in the context of k8s */ cpu?: number; /** This will only be available in the context of k8s */ - cpu_execution_controller?: number | 0.5; + cpu_execution_controller: number; /** This will only be available in the context of k8s */ - ephemeral_storage?: boolean | false; + ephemeral_storage: boolean; execution_controller_targets?: ExecutionControllerTargets[]; hostname: string; index_rollover_frequency: IndexRolloverFrequency; - kubernetes_api_poll_delay: number | 1000; - kubernetes_config_map_name?: string | 'teraslice-worker'; - kubernetes_image_pull_secret?: string | ''; - kubernetes_image?: string | 'terascope/teraslice'; - kubernetes_namespace?: string | 'default'; - kubernetes_overrides_enabled?: boolean | false; - kubernetes_priority_class_name?: string | ''; - kubernetes_worker_antiaffinity?: boolean | false; - master_hostname: string | 'localhost'; - master: boolean | false; + kubernetes_api_poll_delay: number; + kubernetes_config_map_name: string; + kubernetes_image_pull_secret: string; + kubernetes_image: string; + kubernetes_namespace: string; + kubernetes_overrides_enabled: boolean; + kubernetes_priority_class_name?: string; + kubernetes_worker_antiaffinity: boolean; + master_hostname: string; + master: boolean; /** This will only be available in the context of k8s */ memory?: number; /** This will only be available in the context of k8s */ - memory_execution_controller?: number | 512000000; // 512 MB - name: string | 'teracluster'; - network_latency_buffer: number | 15000; - node_disconnect_timeout: number | 300000; - node_state_interval: number | 5000; - port: number | 5678; - shutdown_timeout: number | number; - slicer_allocation_attempts: number | 3; - slicer_port_range: string | '45679:46678'; - slicer_timeout: number | 180000; + memory_execution_controller: number; + name: string; + network_latency_buffer: number; + node_disconnect_timeout: number; + node_state_interval: number; + port: number; + shutdown_timeout: number; + slicer_allocation_attempts: number; + slicer_port_range: string; + slicer_timeout: number; state: { connection: string }; env_vars: { [key: string]: string }; - worker_disconnect_timeout: number | 300000; - workers: number | 4; + worker_disconnect_timeout: number; + workers: number; } export interface TerasliceConfig { From b4d02b50ad6716ca98d3da9f3943803afad3767a Mon Sep 17 00:00:00 2001 From: busma13 Date: Thu, 7 Nov 2024 11:53:22 -0700 Subject: [PATCH 30/30] bump: (minor) @terascope/types@1.3.0, @terascope/utils@1.4.0 bump: (minor) @terascope/data-types@1.4.0, @terascope/data-mate@1.4.0 bump: (minor) elasticsearch-store@1.4.0, terafoundation@1.6.0 bump: (minor) ts-transforms@1.4.0, xlucene-parser@1.4.0 bump: (minor) xlucene-translator@1.4.0, @terascope/elasticsearch-api@4.4.0 bump: (minor) @terascope/teraslice-state-storage@1.4.0, @terascope/job-components@1.6.0 --- e2e/package.json | 4 ++-- packages/data-mate/package.json | 10 +++++----- packages/data-types/package.json | 6 +++--- packages/elasticsearch-api/package.json | 8 ++++---- packages/elasticsearch-store/package.json | 12 ++++++------ packages/job-components/package.json | 6 +++--- packages/scripts/package.json | 4 ++-- packages/terafoundation/package.json | 8 ++++---- packages/teraslice-cli/package.json | 8 ++++---- packages/teraslice-client-js/package.json | 6 +++--- packages/teraslice-messaging/package.json | 6 +++--- packages/teraslice-state-storage/package.json | 6 +++--- packages/teraslice-test-harness/package.json | 4 ++-- packages/teraslice/package.json | 12 ++++++------ packages/ts-transforms/package.json | 8 ++++---- packages/types/package.json | 2 +- packages/utils/package.json | 4 ++-- packages/xlucene-parser/package.json | 6 +++--- packages/xlucene-translator/package.json | 8 ++++---- packages/xpressions/package.json | 6 +++--- 20 files changed, 67 insertions(+), 67 deletions(-) diff --git a/e2e/package.json b/e2e/package.json index d5d09bf2ed5..0ee1b8d0914 100644 --- a/e2e/package.json +++ b/e2e/package.json @@ -43,9 +43,9 @@ "ms": "^2.1.3" }, "devDependencies": { - "@terascope/types": "^1.2.0", + "@terascope/types": "^1.3.0", "bunyan": "^1.8.15", - "elasticsearch-store": "^1.3.3", + "elasticsearch-store": "^1.4.0", "fs-extra": "^11.2.0", "ms": "^2.1.3", "nanoid": "^5.0.7", diff --git a/packages/data-mate/package.json b/packages/data-mate/package.json index 738681ec643..ca65ef07eaf 100644 --- a/packages/data-mate/package.json +++ b/packages/data-mate/package.json @@ -1,7 +1,7 @@ { "name": "@terascope/data-mate", "displayName": "Data-Mate", - "version": "1.3.2", + "version": "1.4.0", "description": "Library of data validations/transformations", "homepage": "https://github.com/terascope/teraslice/tree/master/packages/data-mate#readme", "repository": { @@ -30,9 +30,9 @@ "test:watch": "ts-scripts test --watch . --" }, "dependencies": { - "@terascope/data-types": "^1.3.2", - "@terascope/types": "^1.2.0", - "@terascope/utils": "^1.3.2", + "@terascope/data-types": "^1.4.0", + "@terascope/types": "^1.3.0", + "@terascope/utils": "^1.4.0", "@types/validator": "^13.12.2", "awesome-phonenumber": "^7.2.0", "date-fns": "^4.1.0", @@ -46,7 +46,7 @@ "uuid": "^10.0.0", "valid-url": "^1.0.9", "validator": "^13.12.0", - "xlucene-parser": "^1.3.2" + "xlucene-parser": "^1.4.0" }, "devDependencies": { "@types/ip6addr": "^0.2.6", diff --git a/packages/data-types/package.json b/packages/data-types/package.json index 767a1aebbf8..97b280355ad 100644 --- a/packages/data-types/package.json +++ b/packages/data-types/package.json @@ -1,7 +1,7 @@ { "name": "@terascope/data-types", "displayName": "Data Types", - "version": "1.3.2", + "version": "1.4.0", "description": "A library for defining the data structures and mapping", "homepage": "https://github.com/terascope/teraslice/tree/master/packages/data-types#readme", "bugs": { @@ -27,8 +27,8 @@ "test:watch": "ts-scripts test --watch . --" }, "dependencies": { - "@terascope/types": "^1.2.0", - "@terascope/utils": "^1.3.2", + "@terascope/types": "^1.3.0", + "@terascope/utils": "^1.4.0", "graphql": "^16.9.0", "lodash": "^4.17.21", "yargs": "^17.7.2" diff --git a/packages/elasticsearch-api/package.json b/packages/elasticsearch-api/package.json index 4756f200132..fddd98eeccf 100644 --- a/packages/elasticsearch-api/package.json +++ b/packages/elasticsearch-api/package.json @@ -1,7 +1,7 @@ { "name": "@terascope/elasticsearch-api", "displayName": "Elasticsearch API", - "version": "4.3.2", + "version": "4.4.0", "description": "Elasticsearch client api used across multiple services, handles retries and exponential backoff", "homepage": "https://github.com/terascope/teraslice/tree/master/packages/elasticsearch-api#readme", "bugs": { @@ -24,8 +24,8 @@ "test:watch": "TEST_RESTRAINED_ELASTICSEARCH='true' ts-scripts test --watch . --" }, "dependencies": { - "@terascope/types": "^1.2.0", - "@terascope/utils": "^1.3.2", + "@terascope/types": "^1.3.0", + "@terascope/utils": "^1.4.0", "bluebird": "^3.7.2", "setimmediate": "^1.0.5" }, @@ -33,7 +33,7 @@ "@opensearch-project/opensearch": "^1.2.0", "@types/elasticsearch": "^5.0.43", "elasticsearch": "^15.4.1", - "elasticsearch-store": "^1.3.3", + "elasticsearch-store": "^1.4.0", "elasticsearch6": "npm:@elastic/elasticsearch@^6.7.0", "elasticsearch7": "npm:@elastic/elasticsearch@^7.0.0", "elasticsearch8": "npm:@elastic/elasticsearch@^8.0.0" diff --git a/packages/elasticsearch-store/package.json b/packages/elasticsearch-store/package.json index e7cd1c3515a..42a1b1c63e2 100644 --- a/packages/elasticsearch-store/package.json +++ b/packages/elasticsearch-store/package.json @@ -1,7 +1,7 @@ { "name": "elasticsearch-store", "displayName": "Elasticsearch Store", - "version": "1.3.3", + "version": "1.4.0", "description": "An API for managing an elasticsearch index, with versioning and migration support.", "homepage": "https://github.com/terascope/teraslice/tree/master/packages/elasticsearch-store#readme", "bugs": { @@ -30,10 +30,10 @@ "test:watch": "ts-scripts test --watch . --" }, "dependencies": { - "@terascope/data-mate": "^1.3.2", - "@terascope/data-types": "^1.3.2", - "@terascope/types": "^1.2.0", - "@terascope/utils": "^1.3.2", + "@terascope/data-mate": "^1.4.0", + "@terascope/data-types": "^1.4.0", + "@terascope/types": "^1.3.0", + "@terascope/utils": "^1.4.0", "ajv": "^8.17.1", "ajv-formats": "^3.0.1", "elasticsearch6": "npm:@elastic/elasticsearch@^6.7.0", @@ -43,7 +43,7 @@ "opensearch2": "npm:@opensearch-project/opensearch@^2.2.1", "setimmediate": "^1.0.5", "uuid": "^10.0.0", - "xlucene-translator": "^1.3.2" + "xlucene-translator": "^1.4.0" }, "devDependencies": { "@types/uuid": "^10.0.0" diff --git a/packages/job-components/package.json b/packages/job-components/package.json index 37b66812aca..52a15996a21 100644 --- a/packages/job-components/package.json +++ b/packages/job-components/package.json @@ -1,7 +1,7 @@ { "name": "@terascope/job-components", "displayName": "Job Components", - "version": "1.5.3", + "version": "1.6.0", "description": "A teraslice library for validating jobs schemas, registering apis, and defining and running new Job APIs", "homepage": "https://github.com/terascope/teraslice/tree/master/packages/job-components#readme", "bugs": { @@ -32,8 +32,8 @@ "test:watch": "ts-scripts test --watch . --" }, "dependencies": { - "@terascope/types": "^1.2.0", - "@terascope/utils": "^1.3.2", + "@terascope/types": "^1.3.0", + "@terascope/utils": "^1.4.0", "convict": "^6.2.4", "convict-format-with-moment": "^6.2.0", "convict-format-with-validator": "^6.2.0", diff --git a/packages/scripts/package.json b/packages/scripts/package.json index 8f3ee0624b8..0b8b5422da1 100644 --- a/packages/scripts/package.json +++ b/packages/scripts/package.json @@ -1,7 +1,7 @@ { "name": "@terascope/scripts", "displayName": "Scripts", - "version": "1.4.2", + "version": "1.5.0", "description": "A collection of terascope monorepo scripts", "homepage": "https://github.com/terascope/teraslice/tree/master/packages/scripts#readme", "bugs": { @@ -33,7 +33,7 @@ }, "dependencies": { "@kubernetes/client-node": "^0.22.0", - "@terascope/utils": "^1.3.2", + "@terascope/utils": "^1.4.0", "codecov": "^3.8.3", "execa": "9.4.0", "fs-extra": "^11.2.0", diff --git a/packages/terafoundation/package.json b/packages/terafoundation/package.json index 9a5d7cdd5f1..6b91070b333 100644 --- a/packages/terafoundation/package.json +++ b/packages/terafoundation/package.json @@ -1,7 +1,7 @@ { "name": "terafoundation", "displayName": "Terafoundation", - "version": "1.5.4", + "version": "1.6.0", "description": "A Clustering and Foundation tool for Terascope Tools", "homepage": "https://github.com/terascope/teraslice/tree/master/packages/terafoundation#readme", "bugs": { @@ -29,15 +29,15 @@ }, "dependencies": { "@terascope/file-asset-apis": "^1.0.2", - "@terascope/types": "^1.2.0", - "@terascope/utils": "^1.3.2", + "@terascope/types": "^1.3.0", + "@terascope/utils": "^1.4.0", "bluebird": "^3.7.2", "bunyan": "^1.8.15", "convict": "^6.2.4", "convict-format-with-moment": "^6.2.0", "convict-format-with-validator": "^6.2.0", "elasticsearch": "^15.4.1", - "elasticsearch-store": "^1.3.3", + "elasticsearch-store": "^1.4.0", "express": "^4.21.1", "js-yaml": "^4.1.0", "nanoid": "^5.0.7", diff --git a/packages/teraslice-cli/package.json b/packages/teraslice-cli/package.json index a2a655f5bd9..f6ccbfcf0dc 100644 --- a/packages/teraslice-cli/package.json +++ b/packages/teraslice-cli/package.json @@ -1,7 +1,7 @@ { "name": "teraslice-cli", "displayName": "Teraslice CLI", - "version": "2.7.2", + "version": "2.8.0", "description": "Command line manager for teraslice jobs, assets, and cluster references.", "keywords": [ "teraslice" @@ -38,8 +38,8 @@ }, "dependencies": { "@terascope/fetch-github-release": "^1.0.0", - "@terascope/types": "^1.2.0", - "@terascope/utils": "^1.3.2", + "@terascope/types": "^1.3.0", + "@terascope/utils": "^1.4.0", "chalk": "^5.3.0", "cli-table3": "^0.6.4", "diff": "^7.0.0", @@ -54,7 +54,7 @@ "pretty-bytes": "^6.1.1", "prompts": "^2.4.2", "signale": "^1.4.0", - "teraslice-client-js": "^1.3.2", + "teraslice-client-js": "^1.4.0", "tmp": "^0.2.0", "tty-table": "^4.2.3", "yargs": "^17.7.2", diff --git a/packages/teraslice-client-js/package.json b/packages/teraslice-client-js/package.json index cedb7df72a0..6dc8f89c005 100644 --- a/packages/teraslice-client-js/package.json +++ b/packages/teraslice-client-js/package.json @@ -1,7 +1,7 @@ { "name": "teraslice-client-js", "displayName": "Teraslice Client (JavaScript)", - "version": "1.3.2", + "version": "1.4.0", "description": "A Node.js client for teraslice jobs, assets, and cluster references.", "keywords": [ "elasticsearch", @@ -32,8 +32,8 @@ "test:watch": "ts-scripts test --watch . --" }, "dependencies": { - "@terascope/types": "^1.2.0", - "@terascope/utils": "^1.3.2", + "@terascope/types": "^1.3.0", + "@terascope/utils": "^1.4.0", "auto-bind": "^5.0.1", "got": "^13.0.0" }, diff --git a/packages/teraslice-messaging/package.json b/packages/teraslice-messaging/package.json index 7aa2eb3639c..0be975dc42a 100644 --- a/packages/teraslice-messaging/package.json +++ b/packages/teraslice-messaging/package.json @@ -1,7 +1,7 @@ { "name": "@terascope/teraslice-messaging", "displayName": "Teraslice Messaging", - "version": "1.6.2", + "version": "1.7.0", "description": "An internal teraslice messaging library using socket.io", "homepage": "https://github.com/terascope/teraslice/tree/master/packages/teraslice-messaging#readme", "bugs": { @@ -35,8 +35,8 @@ "ms": "^2.1.3" }, "dependencies": { - "@terascope/types": "^1.2.0", - "@terascope/utils": "^1.3.2", + "@terascope/types": "^1.3.0", + "@terascope/utils": "^1.4.0", "ms": "^2.1.3", "nanoid": "^5.0.7", "p-event": "^6.0.1", diff --git a/packages/teraslice-state-storage/package.json b/packages/teraslice-state-storage/package.json index acb853c54fa..ae715a53ad7 100644 --- a/packages/teraslice-state-storage/package.json +++ b/packages/teraslice-state-storage/package.json @@ -1,7 +1,7 @@ { "name": "@terascope/teraslice-state-storage", "displayName": "Teraslice State Storage", - "version": "1.3.2", + "version": "1.4.0", "description": "State storage operation api for teraslice", "homepage": "https://github.com/terascope/teraslice/tree/master/packages/teraslice-state-storage#readme", "bugs": { @@ -24,8 +24,8 @@ "test:watch": "ts-scripts test --watch . --" }, "dependencies": { - "@terascope/elasticsearch-api": "^4.3.2", - "@terascope/utils": "^1.3.2" + "@terascope/elasticsearch-api": "^4.4.0", + "@terascope/utils": "^1.4.0" }, "engines": { "node": ">=18.18.0", diff --git a/packages/teraslice-test-harness/package.json b/packages/teraslice-test-harness/package.json index 0e0b1fc3d15..daea2cacf7c 100644 --- a/packages/teraslice-test-harness/package.json +++ b/packages/teraslice-test-harness/package.json @@ -36,10 +36,10 @@ "fs-extra": "^11.2.0" }, "devDependencies": { - "@terascope/job-components": "^1.5.3" + "@terascope/job-components": "^1.6.0" }, "peerDependencies": { - "@terascope/job-components": ">=1.5.3" + "@terascope/job-components": ">=1.6.0" }, "engines": { "node": ">=18.18.0", diff --git a/packages/teraslice/package.json b/packages/teraslice/package.json index 4140df8081e..302a26b99b9 100644 --- a/packages/teraslice/package.json +++ b/packages/teraslice/package.json @@ -39,11 +39,11 @@ }, "dependencies": { "@kubernetes/client-node": "^0.22.0", - "@terascope/elasticsearch-api": "^4.3.2", - "@terascope/job-components": "^1.5.3", - "@terascope/teraslice-messaging": "^1.6.2", - "@terascope/types": "^1.2.0", - "@terascope/utils": "^1.3.2", + "@terascope/elasticsearch-api": "^4.4.0", + "@terascope/job-components": "^1.6.0", + "@terascope/teraslice-messaging": "^1.7.0", + "@terascope/types": "^1.3.0", + "@terascope/utils": "^1.4.0", "async-mutex": "^0.5.0", "barbe": "^3.0.16", "body-parser": "^1.20.2", @@ -63,7 +63,7 @@ "semver": "^7.6.3", "socket.io": "^1.7.4", "socket.io-client": "^1.7.4", - "terafoundation": "^1.5.4", + "terafoundation": "^1.6.0", "uuid": "^10.0.0" }, "devDependencies": { diff --git a/packages/ts-transforms/package.json b/packages/ts-transforms/package.json index 0a0b16750bb..ae3fd7215af 100644 --- a/packages/ts-transforms/package.json +++ b/packages/ts-transforms/package.json @@ -1,7 +1,7 @@ { "name": "ts-transforms", "displayName": "TS Transforms", - "version": "1.3.2", + "version": "1.4.0", "description": "An ETL framework built upon xlucene-evaluator", "homepage": "https://github.com/terascope/teraslice/tree/master/packages/ts-transforms#readme", "bugs": { @@ -36,9 +36,9 @@ "test:watch": "ts-scripts test --watch . --" }, "dependencies": { - "@terascope/data-mate": "^1.3.2", - "@terascope/types": "^1.2.0", - "@terascope/utils": "^1.3.2", + "@terascope/data-mate": "^1.4.0", + "@terascope/types": "^1.3.0", + "@terascope/utils": "^1.4.0", "awesome-phonenumber": "^7.2.0", "graphlib": "^2.1.8", "jexl": "^2.2.2", diff --git a/packages/types/package.json b/packages/types/package.json index 14204cc981b..a5bb3e6eb51 100644 --- a/packages/types/package.json +++ b/packages/types/package.json @@ -1,7 +1,7 @@ { "name": "@terascope/types", "displayName": "Types", - "version": "1.2.0", + "version": "1.3.0", "description": "A collection of typescript interfaces", "homepage": "https://github.com/terascope/teraslice/tree/master/packages/types#readme", "bugs": { diff --git a/packages/utils/package.json b/packages/utils/package.json index 36786b85bf5..e4dd8090560 100644 --- a/packages/utils/package.json +++ b/packages/utils/package.json @@ -1,7 +1,7 @@ { "name": "@terascope/utils", "displayName": "Utils", - "version": "1.3.2", + "version": "1.4.0", "description": "A collection of Teraslice Utilities", "homepage": "https://github.com/terascope/teraslice/tree/master/packages/utils#readme", "bugs": { @@ -30,7 +30,7 @@ }, "dependencies": { "@chainsafe/is-ip": "^2.0.2", - "@terascope/types": "^1.2.0", + "@terascope/types": "^1.3.0", "@turf/bbox": "^7.1.0", "@turf/bbox-polygon": "^7.1.0", "@turf/boolean-contains": "^7.1.0", diff --git a/packages/xlucene-parser/package.json b/packages/xlucene-parser/package.json index d5b6a82fea3..803da8bdff2 100644 --- a/packages/xlucene-parser/package.json +++ b/packages/xlucene-parser/package.json @@ -1,7 +1,7 @@ { "name": "xlucene-parser", "displayName": "xLucene Parser", - "version": "1.3.2", + "version": "1.4.0", "description": "Flexible Lucene-like evaluator and language parser", "homepage": "https://github.com/terascope/teraslice/tree/master/packages/xlucene-parser#readme", "repository": { @@ -33,8 +33,8 @@ "test:watch": "ts-scripts test --watch . --" }, "dependencies": { - "@terascope/types": "^1.2.0", - "@terascope/utils": "^1.3.2", + "@terascope/types": "^1.3.0", + "@terascope/utils": "^1.4.0", "peggy": "~4.1.1", "ts-pegjs": "^4.2.1" }, diff --git a/packages/xlucene-translator/package.json b/packages/xlucene-translator/package.json index b0c531e14b3..9a6404b2102 100644 --- a/packages/xlucene-translator/package.json +++ b/packages/xlucene-translator/package.json @@ -1,7 +1,7 @@ { "name": "xlucene-translator", "displayName": "xLucene Translator", - "version": "1.3.2", + "version": "1.4.0", "description": "Translate xlucene query to database queries", "homepage": "https://github.com/terascope/teraslice/tree/master/packages/xlucene-translator#readme", "repository": { @@ -29,10 +29,10 @@ "test:watch": "ts-scripts test --watch . --" }, "dependencies": { - "@terascope/types": "^1.2.0", - "@terascope/utils": "^1.3.2", + "@terascope/types": "^1.3.0", + "@terascope/utils": "^1.4.0", "@types/elasticsearch": "^5.0.43", - "xlucene-parser": "^1.3.2" + "xlucene-parser": "^1.4.0" }, "devDependencies": { "elasticsearch": "^15.4.1" diff --git a/packages/xpressions/package.json b/packages/xpressions/package.json index 554122cf095..169056695b5 100644 --- a/packages/xpressions/package.json +++ b/packages/xpressions/package.json @@ -1,7 +1,7 @@ { "name": "xpressions", "displayName": "Xpressions", - "version": "1.3.2", + "version": "1.4.0", "description": "Variable expressions with date-math support", "homepage": "https://github.com/terascope/teraslice/tree/master/packages/xpressions#readme", "bugs": { @@ -24,10 +24,10 @@ "test:watch": "ts-scripts test --watch . --" }, "dependencies": { - "@terascope/utils": "^1.3.2" + "@terascope/utils": "^1.4.0" }, "devDependencies": { - "@terascope/types": "^1.2.0" + "@terascope/types": "^1.3.0" }, "engines": { "node": ">=18.18.0",