Add k8s-env --dev option (#3667)

This PR makes the following changes: - Adds `--dev` option to k8s-env command - This flag when enabled will add the entire `teraslice` repo as a volume to allow for fast build changes when running in kubernetes. - Adds dev compatibility to k8s master backend - Fixes asset-storage setting in k8s-env - This resolves a bug where launching k8s-env with `asset-storage` set to `s3` would not work correctly
terascope · Jul 8, 2024 · 19f6877 · 19f6877
1 parent 6774a61
commit 19f6877
Show file tree

Hide file tree

Showing 15 changed files with 346 additions and 24 deletions.
diff --git a/Dockerfile.dev b/Dockerfile.dev
@@ -0,0 +1,19 @@
+# NODE_VERSION is set by default in the config.ts, the following value will only
+# be used if you build images by default with docker build
+ARG NODE_VERSION=18.19.1
+FROM terascope/node-base:${NODE_VERSION}
+
+ENV NODE_ENV production
+
+ENV YARN_SETUP_ARGS "--prod=false --silent --frozen-lockfile"
+
+# Check to see if distutils is installed because python 3.12 removed it
+RUN python3 -c "import distutils" || (apk update && apk add py3-setuptools)
+
+EXPOSE 5678
+
+# set up the volumes
+VOLUME /app/config /app/logs /app/assets
+ENV TERAFOUNDATION_CONFIG /app/config/teraslice.yaml
+
+CMD ["yarn", "start:nodemon"]
diff --git a/e2e/k8s/kindConfigDefaultPortsDev.yaml b/e2e/k8s/kindConfigDefaultPortsDev.yaml
@@ -0,0 +1,45 @@
+#
+#   This config file is used in the case that "--dev" is enabled in k8s-env
+#   which will mount the host machines teraslice repo to the k8s resources
+#
+kind: Cluster
+name: k8s-env
+apiVersion: kind.x-k8s.io/v1alpha4
+nodes:
+- role: control-plane
+  extraPortMappings:
+  - containerPort: 30200 # Map internal elasticsearch service to host port
+    hostPort: 9200
+  - containerPort: 30678 # Map internal teraslice service to host port
+    hostPort: 5678
+  - containerPort: 30092 # Map internal kafka service to host port
+    hostPort: 9092
+  - containerPort: 30900 # Map internal minio service to host port
+    hostPort: 9000
+  extraMounts:
+  - hostPath: ./e2e/autoload
+    containerPath: /autoload
+  ### All extra mounts that the Dockerfile.dev image needs to run teraslice
+  - hostPath: ./packages
+    containerPath: /packages
+  - hostPath: ./scripts
+    containerPath: /scripts
+  - hostPath: ./types
+    containerPath: /types
+  - hostPath: ./.yarn
+    containerPath: /.yarn
+  - hostPath: ./.yarnclean.ci
+    containerPath: /.yarnclean.ci
+  - hostPath: ./package.json
+    containerPath: /package.json
+  - hostPath: ./yarn.lock
+    containerPath: /yarn.lock
+  - hostPath: ./tsconfig.json
+    containerPath: /tsconfig.json
+  - hostPath: ./.yarnrc
+    containerPath: /.yarnrc
+  - hostPath: ./service.js
+    containerPath: /service.js
+  - hostPath: ./node_modules
+    containerPath: /node_modules
+  ###
diff --git a/package.json b/package.json
@@ -32,6 +32,7 @@
         "setup": "yarn $YARN_SETUP_ARGS && yarn run build --force",
         "start": "node service.js",
         "start:dev": "yarn build && env NODE_ENV=development node service.js",
+        "start:nodemon": "nodemon --exitcrash service.js",
         "sync": "ts-scripts sync",
         "test": "ts-scripts test"
     },
@@ -57,6 +58,7 @@
         "jest-extended": "^3.2.4",
         "jest-watch-typeahead": "^2.2.2",
         "node-notifier": "^10.0.1",
+        "nodemon": "^3.1.4",
         "ts-jest": "^29.1.4",
         "typescript": "~5.2.2"
     },

diff --git a/packages/scripts/package.json b/packages/scripts/package.json
@@ -1,7 +1,7 @@
 {
     "name": "@terascope/scripts",
     "displayName": "Scripts",
-    "version": "0.78.0",
+    "version": "0.79.0",
     "description": "A collection of terascope monorepo scripts",
     "homepage": "https://github.com/terascope/teraslice/tree/master/packages/scripts#readme",
     "bugs": {

diff --git a/packages/scripts/src/cmds/k8s-env.ts b/packages/scripts/src/cmds/k8s-env.ts
@@ -10,6 +10,7 @@ const cmd: CommandModule = {
     builder(yargs) {
         return yargs
             .example('TEST_ELASTICSEARCH=\'true\' ELASTICSEARCH_PORT=\'9200\' $0 k8s-env', 'Start a kind kubernetes cluster running teraslice from your local repository and elasticsearch.')
+            .example('TEST_ELASTICSEARCH=\'true\' ELASTICSEARCH_PORT=\'9200\' $0 k8s-env --dev', 'Start a kind kubernetes cluster running teraslice in dev mode. Faster build times.')
             .example('TEST_ELASTICSEARCH=\'true\' ELASTICSEARCH_PORT=\'9200\' $0 k8s-env --teraslice-image=terascope/teraslice:v0.91.0-nodev18.18.2', 'Start a kind kubernetes cluster running teraslice from a specific docker image and elasticsearch.')
             .example('TEST_ELASTICSEARCH=\'true\' ELASTICSEARCH_PORT=\'9200\' TEST_KAFKA=\'true\' KAFKA_PORT=\'9092\' $0 k8s-env', 'Start a kind kubernetes cluster running teraslice, elasticsearch, kafka, and zookeeper.')
             .example('TEST_ELASTICSEARCH=\'true\' ELASTICSEARCH_PORT=\'9200\' $0 k8s-env --skip-build', 'Start a kind kubernetes cluster, but skip building a new teraslice docker image.')
@@ -93,6 +94,11 @@ const cmd: CommandModule = {
                 default: 'elasticsearch-next',
                 choices: ['elasticsearch-next', 's3'],
             })
+            .option('dev', {
+                description: 'Mounts local teraslice to k8s resources for faster development.',
+                type: 'boolean',
+                default: false
+            })
             .check((args) => {
                 if (args['asset-storage'] === 's3' && process.env.TEST_MINIO !== 'true') {
                     throw new Error('You chose "s3" as an asset storage but don\'t have the minio service enabled.\n'
@@ -118,7 +124,8 @@ const cmd: CommandModule = {
             kindClusterName: argv['cluster-name'] as string,
             k8sVersion: argv['k8s-version'] as string,
             terasliceImage: argv['teraslice-image'] as string,
-            assetStorage: argv['asset-storage'] as string
+            assetStorage: argv['asset-storage'] as string,
+            dev: Boolean(argv.dev)
         };
 
         if (Boolean(argv.rebuild) === true) {

diff --git a/packages/scripts/src/helpers/interfaces.ts b/packages/scripts/src/helpers/interfaces.ts
@@ -1,4 +1,5 @@
 import { EmptyObject } from '@terascope/utils';
+import type { V1Volume, V1VolumeMount } from '@kubernetes/client-node';
 
 export type PackageInfo = {
     dir: string;
@@ -157,3 +158,9 @@ export type KindCluster = {
         }
     ]
 }
+
+export interface TsVolumeSet {
+    extraMounts: any[],
+    volumes: V1Volume[],
+    volumeMounts: V1VolumeMount[]
+}
diff --git a/packages/scripts/src/helpers/k8s-env/index.ts b/packages/scripts/src/helpers/k8s-env/index.ts
@@ -1,3 +1,4 @@
+import execa from 'execa';
 import {
     dockerTag,
     isKindInstalled,
@@ -20,7 +21,6 @@ const e2eImage = `${rootInfo.name}:e2e-nodev${config.NODE_VERSION}`;
 
 export async function launchK8sEnv(options: K8sEnvOptions) {
     signale.pending('Starting k8s environment with the following options: ', options);
-
     const kind = new Kind(options.k8sVersion, options.kindClusterName);
     // TODO: create a kind class
     const kindInstalled = await isKindInstalled();
@@ -36,9 +36,26 @@ export async function launchK8sEnv(options: K8sEnvOptions) {
         process.exit(1);
     }
 
+    // If --dev is true, we must run yarn setup before creating resources
+    // We need a local node_modules folder built to add it as a volume
+    if (options.dev) {
+        if (process.version.substring(1) !== options.nodeVersion) {
+            throw new Error(`The node version this process is running on (${process.version}) does not match
+            the --node-version set in k8s-env (v${options.nodeVersion}). Check your version by running "node -v"`);
+        }
+        signale.info(`Running yarn setup with node ${process.version}...`);
+        try {
+            execa.commandSync('yarn setup');
+        } catch (err) {
+            signale.fatal(err);
+            await kind.destroyCluster();
+            process.exit(1);
+        }
+    }
+
     signale.pending('Creating kind cluster');
     try {
-        await kind.createCluster(options.tsPort);
+        await kind.createCluster(options.tsPort, options.dev);
     } catch (err) {
         signale.error(err);
         // Do not destroy existing cluster if that was the cause of failure
@@ -144,7 +161,8 @@ async function buildAndTagTerasliceImage(options:K8sEnvOptions) {
                 dryRun: true,
                 nodeSuffix: true,
                 nodeVersion: options.nodeVersion,
-                type: PublishType.Dev
+                type: PublishType.Dev,
+                useDevFile: options.dev
             };
             runImage = await buildDevDockerImage(publishOptions);
         } catch (err) {

diff --git a/packages/scripts/src/helpers/k8s-env/interfaces.ts b/packages/scripts/src/helpers/k8s-env/interfaces.ts
@@ -15,6 +15,7 @@ export interface K8sEnvOptions {
     assetStorage: string;
     terasliceImage?: string;
     resetStore?: boolean;
+    dev: boolean;
 }
 
 // TODO: create a common parent for each resource type,

diff --git a/packages/scripts/src/helpers/k8s-env/k8s.ts b/packages/scripts/src/helpers/k8s-env/k8s.ts
@@ -8,6 +8,7 @@ import { getE2eK8sDir } from '../../helpers/packages';
 import { K8sEnvOptions } from './interfaces';
 import signale from '../signale';
 import * as config from '../config';
+import { getVolumesFromDockerfile } from '../kind';
 
 const logger = debugLogger('ts-scripts:k8s-env');
 export class K8s {
@@ -58,6 +59,35 @@ export class K8s {
         }
     }
 
+    mountLocalTeraslice(masterDeployment: k8sClient.V1Deployment) {
+        const dockerfileMounts = getVolumesFromDockerfile(true, logger);
+        if (masterDeployment.spec?.template.spec?.containers[0].volumeMounts) {
+            masterDeployment.spec.template.spec.containers[0].volumeMounts
+                .push(...dockerfileMounts.volumeMounts);
+        }
+        if (masterDeployment.spec?.template.spec?.volumes) {
+            masterDeployment.spec.template.spec.volumes
+                .push(...dockerfileMounts.volumes);
+        }
+
+        /// Pass in env so master passes volumes to ex's and workers
+        if (masterDeployment.spec?.template.spec?.containers[0].env) {
+            masterDeployment.spec.template.spec.containers[0].env.push(
+                {
+                    name: 'MOUNT_LOCAL_TERASLICE',
+                    value: JSON.stringify(dockerfileMounts)
+                }
+            );
+        } else if (masterDeployment.spec?.template.spec?.containers[0]) {
+            masterDeployment.spec.template.spec.containers[0].env = [
+                {
+                    name: 'MOUNT_LOCAL_TERASLICE',
+                    value: JSON.stringify(dockerfileMounts)
+                }
+            ];
+        }
+    }
+
     async deployK8sTeraslice(wait = false, options: K8sEnvOptions | undefined = undefined) {
         signale.pending('Begin teraslice deployment...');
         const e2eK8sDir = getE2eK8sDir();
@@ -76,7 +106,7 @@ export class K8s {
             masterTerafoundation.teraslice.kubernetes_image = `teraslice-workspace:e2e-nodev${config.NODE_VERSION}`;
             if (options) {
                 const storageType = options.assetStorage;
-                masterTerafoundation.terafoundation.asset_storage_connection_type = storageType;
+                masterTerafoundation.teraslice.asset_storage_connection_type = storageType;
             }
             masterConfigMap.data = { 'teraslice.yaml': k8sClient.dumpYaml(masterTerafoundation) };
             masterConfigMap.metadata = { name: 'teraslice-master' };
@@ -94,7 +124,7 @@ export class K8s {
             workerTerafoundation.teraslice.kubernetes_image = `teraslice-workspace:e2e-nodev${config.NODE_VERSION}`;
             if (options) {
                 const storageType = options.assetStorage;
-                workerTerafoundation.terafoundation.asset_storage_connection_type = storageType;
+                workerTerafoundation.teraslice.asset_storage_connection_type = storageType;
             }
             workerConfigMap.data = { 'teraslice.yaml': k8sClient.dumpYaml(workerTerafoundation) };
             workerConfigMap.metadata = { name: 'teraslice-worker' };
@@ -114,6 +144,9 @@ export class K8s {
             if (yamlTSMasterDeployment.spec?.template.spec?.containers[0]) {
                 yamlTSMasterDeployment.spec.template.spec.containers[0].image = `teraslice-workspace:e2e-nodev${config.NODE_VERSION}`;
             }
+            if (options?.dev) {
+                this.mountLocalTeraslice(yamlTSMasterDeployment);
+            }
             const response = await this.k8sAppsV1Api.createNamespacedDeployment('ts-dev1', yamlTSMasterDeployment);
             logger.debug('deployK8sTeraslice yamlTSMasterDeployment: ', response.body);
         } catch (err) {