Merge pull request #3812 from terascope/k8sV2-types

[teraslice] add typescript types to KubernetesV2 backend

docs/configuration/clustering-k8s.md

@@ -99,6 +99,7 @@ support k8s based Teraslice deployments.
 |         assets_volume          |                               Name of kubernetes volume to be shared across all pods, where Teraslice assets will be stored                                | String  | optional |
 |    cpu_execution_controller    |                                           CPU resources to use for Execution Controller request and limit values                                           | Number  | optional |
 |  execution_controller_targets  |                                       array of `{"key": "rack", "value": "alpha"}` targets for execution controllers                                       | String  | optional |
+|   kubernetes_api_poll_delay    |                               Specify the delay between attempts to poll the kubernetes API, `1000` by default                                            | Number  |  optional  |
 |        kubernetes_image        |                                                     Name of docker image, default: `teraslice:k8sdev`                                                      | String  | optional |
 |  kubernetes_image_pull_secret  |                                                    Secret used to pull docker images from private repo                                                     | String  | optional |
 |   kubernetes_config_map_name   | Name of the configmap used by worker and execution_controller containers for config.  If this is not provided, the default will be `<CLUSTER_NAME>-worker` | String  | optional |

e2e/package.json

@@ -43,9 +43,9 @@
         "ms": "^2.1.3"
     },
     "devDependencies": {
-        "@terascope/types": "^1.2.0",
+        "@terascope/types": "^1.3.0",
         "bunyan": "^1.8.15",
-        "elasticsearch-store": "^1.3.3",
+        "elasticsearch-store": "^1.4.0",
         "fs-extra": "^11.2.0",
         "ms": "^2.1.3",
         "nanoid": "^5.0.7",

package.json

@@ -1,7 +1,7 @@
 {
     "name": "teraslice-workspace",
     "displayName": "Teraslice",
-    "version": "2.6.4",
+    "version": "2.7.0",
     "private": true,
     "homepage": "https://github.com/terascope/teraslice",
     "bugs": {

packages/data-mate/package.json

@@ -1,7 +1,7 @@
 {
     "name": "@terascope/data-mate",
     "displayName": "Data-Mate",
-    "version": "1.3.2",
+    "version": "1.4.0",
     "description": "Library of data validations/transformations",
     "homepage": "https://github.com/terascope/teraslice/tree/master/packages/data-mate#readme",
     "repository": {
@@ -30,9 +30,9 @@
         "test:watch": "ts-scripts test --watch . --"
     },
     "dependencies": {
-        "@terascope/data-types": "^1.3.2",
-        "@terascope/types": "^1.2.0",
-        "@terascope/utils": "^1.3.2",
+        "@terascope/data-types": "^1.4.0",
+        "@terascope/types": "^1.3.0",
+        "@terascope/utils": "^1.4.0",
         "@types/validator": "^13.12.2",
         "awesome-phonenumber": "^7.2.0",
         "date-fns": "^4.1.0",
@@ -46,7 +46,7 @@
         "uuid": "^10.0.0",
         "valid-url": "^1.0.9",
         "validator": "^13.12.0",
-        "xlucene-parser": "^1.3.2"
+        "xlucene-parser": "^1.4.0"
     },
     "devDependencies": {
         "@types/ip6addr": "^0.2.6",

packages/data-types/package.json

@@ -1,7 +1,7 @@
 {
     "name": "@terascope/data-types",
     "displayName": "Data Types",
-    "version": "1.3.2",
+    "version": "1.4.0",
     "description": "A library for defining the data structures and mapping",
     "homepage": "https://github.com/terascope/teraslice/tree/master/packages/data-types#readme",
     "bugs": {
@@ -27,8 +27,8 @@
         "test:watch": "ts-scripts test --watch . --"
     },
     "dependencies": {
-        "@terascope/types": "^1.2.0",
-        "@terascope/utils": "^1.3.2",
+        "@terascope/types": "^1.3.0",
+        "@terascope/utils": "^1.4.0",
         "graphql": "^16.9.0",
         "lodash": "^4.17.21",
         "yargs": "^17.7.2"

packages/elasticsearch-api/package.json

@@ -1,7 +1,7 @@
 {
     "name": "@terascope/elasticsearch-api",
     "displayName": "Elasticsearch API",
-    "version": "4.3.2",
+    "version": "4.4.0",
     "description": "Elasticsearch client api used across multiple services, handles retries and exponential backoff",
     "homepage": "https://github.com/terascope/teraslice/tree/master/packages/elasticsearch-api#readme",
     "bugs": {
@@ -24,16 +24,16 @@
         "test:watch": "TEST_RESTRAINED_ELASTICSEARCH='true' ts-scripts test --watch . --"
     },
     "dependencies": {
-        "@terascope/types": "^1.2.0",
-        "@terascope/utils": "^1.3.2",
+        "@terascope/types": "^1.3.0",
+        "@terascope/utils": "^1.4.0",
         "bluebird": "^3.7.2",
         "setimmediate": "^1.0.5"
     },
     "devDependencies": {
         "@opensearch-project/opensearch": "^1.2.0",
         "@types/elasticsearch": "^5.0.43",
         "elasticsearch": "^15.4.1",
-        "elasticsearch-store": "^1.3.3",
+        "elasticsearch-store": "^1.4.0",
         "elasticsearch6": "npm:@elastic/elasticsearch@^6.7.0",
         "elasticsearch7": "npm:@elastic/elasticsearch@^7.0.0",
         "elasticsearch8": "npm:@elastic/elasticsearch@^8.0.0"

packages/elasticsearch-store/package.json

@@ -1,7 +1,7 @@
 {
     "name": "elasticsearch-store",
     "displayName": "Elasticsearch Store",
-    "version": "1.3.3",
+    "version": "1.4.0",
     "description": "An API for managing an elasticsearch index, with versioning and migration support.",
     "homepage": "https://github.com/terascope/teraslice/tree/master/packages/elasticsearch-store#readme",
     "bugs": {
@@ -30,10 +30,10 @@
         "test:watch": "ts-scripts test --watch . --"
     },
     "dependencies": {
-        "@terascope/data-mate": "^1.3.2",
-        "@terascope/data-types": "^1.3.2",
-        "@terascope/types": "^1.2.0",
-        "@terascope/utils": "^1.3.2",
+        "@terascope/data-mate": "^1.4.0",
+        "@terascope/data-types": "^1.4.0",
+        "@terascope/types": "^1.3.0",
+        "@terascope/utils": "^1.4.0",
         "ajv": "^8.17.1",
         "ajv-formats": "^3.0.1",
         "elasticsearch6": "npm:@elastic/elasticsearch@^6.7.0",
@@ -43,7 +43,7 @@
         "opensearch2": "npm:@opensearch-project/opensearch@^2.2.1",
         "setimmediate": "^1.0.5",
         "uuid": "^10.0.0",
-        "xlucene-translator": "^1.3.2"
+        "xlucene-translator": "^1.4.0"
     },
     "devDependencies": {
         "@types/uuid": "^10.0.0"

packages/job-components/package.json

@@ -1,7 +1,7 @@
 {
     "name": "@terascope/job-components",
     "displayName": "Job Components",
-    "version": "1.5.3",
+    "version": "1.6.0",
     "description": "A teraslice library for validating jobs schemas, registering apis, and defining and running new Job APIs",
     "homepage": "https://github.com/terascope/teraslice/tree/master/packages/job-components#readme",
     "bugs": {
@@ -32,8 +32,8 @@
         "test:watch": "ts-scripts test --watch . --"
     },
     "dependencies": {
-        "@terascope/types": "^1.2.0",
-        "@terascope/utils": "^1.3.2",
+        "@terascope/types": "^1.3.0",
+        "@terascope/utils": "^1.4.0",
         "convict": "^6.2.4",
         "convict-format-with-moment": "^6.2.0",
         "convict-format-with-validator": "^6.2.0",

packages/job-components/src/test-helpers.ts

@@ -164,18 +164,31 @@ export class TestContext implements i.Context {
             teraslice: {
                 action_timeout: 10000,
                 analytics_rate: 10000,
+                api_response_timeout: 300000,
                 assets_directory: path.join(process.cwd(), 'assets'),
+                assets_volume: '',
                 asset_storage_connection_type: 'elasticsearch-next',
                 asset_storage_connection: 'default',
                 asset_storage_bucket: '',
                 cluster_manager_type: options.cluster_manager_type || 'native',
+                cpu_execution_controller: 0.5,
+                ephemeral_storage: false,
                 hostname: 'localhost',
                 index_rollover_frequency: {
                     analytics: 'yearly',
-                    state: 'montly',
+                    state: 'monthly',
                 },
+                kubernetes_api_poll_delay: 1000,
+                kubernetes_config_map_name: 'teraslice-worker',
+                kubernetes_image_pull_secret: '',
+                kubernetes_image: 'terascope/teraslice',
+                kubernetes_namespace: 'default',
+                kubernetes_overrides_enabled: false,
+                kubernetes_priority_class_name: undefined,
+                kubernetes_worker_antiaffinity: false,
                 master_hostname: 'localhost',
                 master: false,
+                memory_execution_controller: 512000000,
                 name: testName,
                 network_latency_buffer: 100,
                 node_disconnect_timeout: 5000,

packages/scripts/package.json

@@ -33,7 +33,7 @@
     },
     "dependencies": {
         "@kubernetes/client-node": "^0.22.0",
-        "@terascope/utils": "^1.3.2",
+        "@terascope/utils": "^1.4.0",
         "codecov": "^3.8.3",
         "execa": "9.4.0",
         "fs-extra": "^11.2.0",

packages/terafoundation/package.json

@@ -1,7 +1,7 @@
 {
     "name": "terafoundation",
     "displayName": "Terafoundation",
-    "version": "1.5.4",
+    "version": "1.6.0",
     "description": "A Clustering and Foundation tool for Terascope Tools",
     "homepage": "https://github.com/terascope/teraslice/tree/master/packages/terafoundation#readme",
     "bugs": {
@@ -29,15 +29,15 @@
     },
     "dependencies": {
         "@terascope/file-asset-apis": "^1.0.2",
-        "@terascope/types": "^1.2.0",
-        "@terascope/utils": "^1.3.2",
+        "@terascope/types": "^1.3.0",
+        "@terascope/utils": "^1.4.0",
         "bluebird": "^3.7.2",
         "bunyan": "^1.8.15",
         "convict": "^6.2.4",
         "convict-format-with-moment": "^6.2.0",
         "convict-format-with-validator": "^6.2.0",
         "elasticsearch": "^15.4.1",
-        "elasticsearch-store": "^1.3.3",
+        "elasticsearch-store": "^1.4.0",
         "express": "^4.21.1",
         "js-yaml": "^4.1.0",
         "nanoid": "^5.0.7",

packages/teraslice-cli/package.json

@@ -1,7 +1,7 @@
 {
     "name": "teraslice-cli",
     "displayName": "Teraslice CLI",
-    "version": "2.7.2",
+    "version": "2.8.0",
     "description": "Command line manager for teraslice jobs, assets, and cluster references.",
     "keywords": [
         "teraslice"
@@ -38,8 +38,8 @@
     },
     "dependencies": {
         "@terascope/fetch-github-release": "^1.0.0",
-        "@terascope/types": "^1.2.0",
-        "@terascope/utils": "^1.3.2",
+        "@terascope/types": "^1.3.0",
+        "@terascope/utils": "^1.4.0",
         "chalk": "^5.3.0",
         "cli-table3": "^0.6.4",
         "diff": "^7.0.0",
@@ -54,7 +54,7 @@
         "pretty-bytes": "^6.1.1",
         "prompts": "^2.4.2",
         "signale": "^1.4.0",
-        "teraslice-client-js": "^1.3.2",
+        "teraslice-client-js": "^1.4.0",
         "tmp": "^0.2.0",
         "tty-table": "^4.2.3",
         "yargs": "^17.7.2",

packages/teraslice-client-js/package.json

@@ -1,7 +1,7 @@
 {
     "name": "teraslice-client-js",
     "displayName": "Teraslice Client (JavaScript)",
-    "version": "1.3.2",
+    "version": "1.4.0",
     "description": "A Node.js client for teraslice jobs, assets, and cluster references.",
     "keywords": [
         "elasticsearch",
@@ -32,8 +32,8 @@
         "test:watch": "ts-scripts test --watch . --"
     },
     "dependencies": {
-        "@terascope/types": "^1.2.0",
-        "@terascope/utils": "^1.3.2",
+        "@terascope/types": "^1.3.0",
+        "@terascope/utils": "^1.4.0",
         "auto-bind": "^5.0.1",
         "got": "^13.0.0"
     },

packages/teraslice-messaging/package.json

@@ -1,7 +1,7 @@
 {
     "name": "@terascope/teraslice-messaging",
     "displayName": "Teraslice Messaging",
-    "version": "1.6.2",
+    "version": "1.7.0",
     "description": "An internal teraslice messaging library using socket.io",
     "homepage": "https://github.com/terascope/teraslice/tree/master/packages/teraslice-messaging#readme",
     "bugs": {
@@ -35,8 +35,8 @@
         "ms": "^2.1.3"
     },
     "dependencies": {
-        "@terascope/types": "^1.2.0",
-        "@terascope/utils": "^1.3.2",
+        "@terascope/types": "^1.3.0",
+        "@terascope/utils": "^1.4.0",
         "ms": "^2.1.3",
         "nanoid": "^5.0.7",
         "p-event": "^6.0.1",

packages/teraslice-state-storage/package.json

@@ -1,7 +1,7 @@
 {
     "name": "@terascope/teraslice-state-storage",
     "displayName": "Teraslice State Storage",
-    "version": "1.3.2",
+    "version": "1.4.0",
     "description": "State storage operation api for teraslice",
     "homepage": "https://github.com/terascope/teraslice/tree/master/packages/teraslice-state-storage#readme",
     "bugs": {
@@ -24,8 +24,8 @@
         "test:watch": "ts-scripts test --watch . --"
     },
     "dependencies": {
-        "@terascope/elasticsearch-api": "^4.3.2",
-        "@terascope/utils": "^1.3.2"
+        "@terascope/elasticsearch-api": "^4.4.0",
+        "@terascope/utils": "^1.4.0"
     },
     "engines": {
         "node": ">=18.18.0",

packages/teraslice-test-harness/package.json

@@ -36,10 +36,10 @@
         "fs-extra": "^11.2.0"
     },
     "devDependencies": {
-        "@terascope/job-components": "^1.5.3"
+        "@terascope/job-components": "^1.6.0"
     },
     "peerDependencies": {
-        "@terascope/job-components": ">=1.5.3"
+        "@terascope/job-components": ">=1.6.0"
     },
     "engines": {
         "node": ">=18.18.0",

packages/teraslice/package.json

@@ -1,7 +1,7 @@
 {
     "name": "teraslice",
     "displayName": "Teraslice",
-    "version": "2.6.4",
+    "version": "2.7.0",
     "description": "Distributed computing platform for processing JSON data",
     "homepage": "https://github.com/terascope/teraslice#readme",
     "bugs": {
@@ -39,11 +39,11 @@
     },
     "dependencies": {
         "@kubernetes/client-node": "^0.22.0",
-        "@terascope/elasticsearch-api": "^4.3.2",
-        "@terascope/job-components": "^1.5.3",
-        "@terascope/teraslice-messaging": "^1.6.2",
-        "@terascope/types": "^1.2.0",
-        "@terascope/utils": "^1.3.2",
+        "@terascope/elasticsearch-api": "^4.4.0",
+        "@terascope/job-components": "^1.6.0",
+        "@terascope/teraslice-messaging": "^1.7.0",
+        "@terascope/types": "^1.3.0",
+        "@terascope/utils": "^1.4.0",
         "async-mutex": "^0.5.0",
         "barbe": "^3.0.16",
         "body-parser": "^1.20.2",
@@ -63,7 +63,7 @@
         "semver": "^7.6.3",
         "socket.io": "^1.7.4",
         "socket.io-client": "^1.7.4",
-        "terafoundation": "^1.5.4",
+        "terafoundation": "^1.6.0",
         "uuid": "^10.0.0"
     },
     "devDependencies": {

packages/teraslice/src/lib/cluster/services/cluster/backends/kubernetes/index.ts

@@ -41,7 +41,6 @@ export class KubernetesClusterBackend {
             this.logger,
             null,
             kubernetesNamespace,
-            // @ts-expect-error
             context.sysconfig.teraslice.kubernetes_api_poll_delay,
             context.sysconfig.teraslice.shutdown_timeout
         );