Skip to content

Commit

Permalink
merge master
Browse files Browse the repository at this point in the history
  • Loading branch information
@jsnoble
jsnoble committed Feb 7, 2024
2 parents f6923c2 + af82efb commit fb4bbdf
Show file tree
Hide file tree
Showing 7 changed files with 98 additions and 62 deletions.
20 changes: 20 additions & 0 deletions docs/development/k8s.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -315,6 +315,16 @@ yarn k8s:rebuild
yarn run ts-scripts k8s-env --rebuild
```

If you would like to reset the elasticsearch store at the same time:

```bash
# from the teraslice root directory:
yarn k8s:rebuild --reset-store

# from any other directory:
yarn run ts-scripts k8s-env --rebuild --reset-store
```

If you need to restart Teraslice without rebuilding you can use the following command:

NOTE: this does not reset state in the elasticsearch store
Expand All @@ -327,6 +337,16 @@ yarn k8s:restart
yarn run ts-scripts k8s-env --rebuild --skip-build
```

If you would like to reset the elasticsearch store at the same time:

```bash
# from the teraslice root directory:
yarn k8s:restart --reset-store

# from any other directory:
yarn run ts-scripts k8s-env --rebuild --skip-build --reset-store
```

## Extras

### Teraslice Kubernetes Job Structure
Expand Down
1 change: 1 addition & 0 deletions packages/scripts/package.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -40,6 +40,7 @@
"got": "^11.8.3",
"ip": "^1.1.8",
"js-yaml": "^4.1.0",
"kafkajs": "^2.2.4",
"lodash": "^4.17.21",
"micromatch": "^4.0.5",
"mnemonist": "^0.39.8",
Expand Down
18 changes: 15 additions & 3 deletions packages/scripts/src/cmds/k8s-env.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@ import { CommandModule } from 'yargs';
import * as config from '../helpers/config';
import { launchK8sEnv, rebuildTeraslice } from '../helpers/k8s-env';
import { kafkaVersionMapper } from '../helpers/mapper';
import { k8sEnvOptions } from '../helpers/k8s-env/interfaces';

const cmd: CommandModule = {
command: 'k8s-env',
Expand All @@ -12,6 +13,9 @@ const cmd: CommandModule = {
.example('TEST_ELASTICSEARCH=\'true\' ELASTICSEARCH_PORT=\'9200\' $0 k8s-env --teraslice-image=terascope/teraslice:v0.91.0-nodev18.18.2', 'Start a kind kubernetes cluster running teraslice from a specific docker image and elasticsearch.')
.example('TEST_ELASTICSEARCH=\'true\' ELASTICSEARCH_PORT=\'9200\' TEST_KAFKA=\'true\' KAFKA_PORT=\'9092\' $0 k8s-env', 'Start a kind kubernetes cluster running teraslice, elasticsearch, kafka, and zookeeper.')
.example('TEST_ELASTICSEARCH=\'true\' ELASTICSEARCH_PORT=\'9200\' $0 k8s-env --skip-build', 'Start a kind kubernetes cluster, but skip building a new teraslice docker image.')
.example('TEST_ELASTICSEARCH=\'true\' ELASTICSEARCH_PORT=\'9200\' $0 k8s-env --rebuild', 'Stop teraslice, rebuild docker image, and restart teraslice.')
.example('TEST_ELASTICSEARCH=\'true\' ELASTICSEARCH_PORT=\'9200\' $0 k8s-env --rebuild -reset-store', 'Rebuild and also clear the elasticsearch store.')
.example('TEST_ELASTICSEARCH=\'true\' ELASTICSEARCH_PORT=\'9200\' $0 k8s-env --rebuild -skip-build', 'Restart teraslice without rebuilding docker image.')
.example('$0 k8s-env --rebuild', 'Rebuild teraslice and redeploy to k8s cluster. ES store data is retained.')
.option('elasticsearch-version', {
description: 'The elasticsearch version to use',
Expand Down Expand Up @@ -53,6 +57,11 @@ const cmd: CommandModule = {
type: 'boolean',
default: false
})
.option('reset-store', {
description: 'Restart the elasticsearch service when rebuilding teraslice. This flag is ignored if not accompanied by --rebuild',
type: 'boolean',
default: false
})
.option('ts-port', {
description: 'Port where teraslice api will be exposed.',
type: 'number',
Expand All @@ -76,7 +85,7 @@ const cmd: CommandModule = {
},
handler(argv) {
const kafkaCPVersion = kafkaVersionMapper(argv.kafkaVersion as string);
const k8sEnvOptions = {
const k8sOptions: k8sEnvOptions = {
elasticsearchVersion: argv.elasticsearchVersion as string,
kafkaVersion: argv.kafkaVersion as string,
kafkaImageVersion: kafkaCPVersion,
Expand All @@ -93,10 +102,13 @@ const cmd: CommandModule = {
};

if (Boolean(argv.rebuild) === true) {
return rebuildTeraslice(k8sEnvOptions);
if (argv['reset-store']) {
k8sOptions.resetStore = true;
}
return rebuildTeraslice(k8sOptions);
}

return launchK8sEnv(k8sEnvOptions);
return launchK8sEnv(k8sOptions);
},
};

Expand Down
16 changes: 14 additions & 2 deletions packages/scripts/src/helpers/k8s-env/index.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,8 @@ import {
dockerTag,
isKindInstalled,
isKubectlInstalled,
k8sStartService,
k8sStopService,
} from '../scripts';
import { Kind } from '../kind';
import { k8sEnvOptions } from './interfaces';
Expand Down Expand Up @@ -38,8 +40,11 @@ export async function launchK8sEnv(options: k8sEnvOptions) {
try {
await kind.createCluster(options.tsPort);
} catch (err) {
signale.fatal(err);
await kind.destroyCluster();
signale.error(err);
// Do not destroy existing cluster if that was the cause of failure
if (!err.stderr.includes('node(s) already exist for a cluster with the name')) {
await kind.destroyCluster();
}
process.exit(1);
}
signale.success('Kind cluster created');
Expand Down Expand Up @@ -111,6 +116,13 @@ export async function rebuildTeraslice(options: k8sEnvOptions) {
await kind.loadTerasliceImage(e2eImage);
signale.success('Teraslice Docker image loaded');

if (options.resetStore) {
signale.pending('Resetting the elasticsearch service');
await k8sStopService('elasticsearch');
await k8sStartService('elasticsearch', config.ELASTICSEARCH_DOCKER_IMAGE, config.ELASTICSEARCH_VERSION, kind);
signale.success('elasticsearch service reset');
}

try {
await k8s.deployK8sTeraslice(true);
} catch (err) {
Expand Down
1 change: 1 addition & 0 deletions packages/scripts/src/helpers/k8s-env/interfaces.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@ export interface k8sEnvOptions {
clusterName: string;
k8sVersion: string;
terasliceImage?: string;
resetStore?: boolean;
}

// TODO: create a common parent for each resource type,
Expand Down
40 changes: 39 additions & 1 deletion packages/scripts/src/helpers/test-runner/services.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@ import semver from 'semver';
import fs from 'fs-extra';
import path from 'path';
import * as ts from '@terascope/utils';
import { Kafka } from 'kafkajs';
import { getServicesForSuite, getRootDir } from '../misc';
import {
dockerRun,
Expand Down Expand Up @@ -710,8 +711,45 @@ async function checkRabbitMQ(options: TestOptions, startTime: number): Promise<v
}

async function checkKafka(options: TestOptions, startTime: number) {
const host = config.KAFKA_HOSTNAME;
const kafkaBroker = config.KAFKA_BROKER;
const retryCount = 5;
const retryTime = 10000;
const totalTime = retryCount * retryTime;

const dockerGateways = ['host.docker.internal', 'gateway.docker.internal'];
if (dockerGateways.includes(config.KAFKA_HOSTNAME)) return;

if (options.trace) {
signale.debug(`checking kafka at ${host}`);
} else {
logger.debug(`checking kafka at ${host}`);
}

const kafka = new Kafka({
clientId: 'tera-test',
brokers: [kafkaBroker],
logLevel: 0,
retry: {
initialRetryTime: retryTime,
maxRetryTime: retryTime,
factor: 0,
retries: retryCount
}
});
const producer = kafka.producer();
const took = ts.toHumanTime(Date.now() - startTime);
signale.success(`kafka@${options.kafkaVersion} *might* be running at ${config.KAFKA_BROKER}, took ${took}`);
try {
await producer.connect();
} catch (err) {
if (err.message.includes('ENOTFOUND') && err.message.includes(config.KAFKA_BROKER)) {
throw new Error(`Unable to connect to kafka broker after ${totalTime}ms at ${kafkaBroker}`);
} else if (err.message.includes('ECONNREFUSED') && err.message.includes(config.KAFKA_BROKER)) {
throw new Error(`Unable to connect to kafka broker after ${totalTime}ms at ${kafkaBroker}`);
}
throw new Error(err.message);
}
signale.success(`kafka@${options.kafkaVersion} is running at ${config.KAFKA_BROKER}, took ${took}`);
}

async function checkZookeeper(options: TestOptions, startTime: number) {
Expand Down
64 changes: 8 additions & 56 deletions yarn.lock
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3044,16 +3044,11 @@
dependencies:
"@types/node" "*"

"@types/lodash@^4.14.195":
"@types/lodash@^4.14.195", "@types/lodash@^4.14.202":
version "4.14.199"
resolved "https://registry.yarnpkg.com/@types/lodash/-/lodash-4.14.199.tgz#c3edb5650149d847a277a8961a7ad360c474e9bf"
integrity sha512-Vrjz5N5Ia4SEzWWgIVwnHNEnb1UE1XMkvY5DGXrAeOGE9imk0hgTHh5GyDjLDJi9OTCn9oo9dXH1uToK1VRfrg==

"@types/lodash@^4.14.202":
version "4.14.202"
resolved "https://registry.yarnpkg.com/@types/lodash/-/lodash-4.14.202.tgz#f09dbd2fb082d507178b2f2a5c7e74bd72ff98f8"
integrity sha512-OvlIYQK9tNneDlS0VN54LLd5uiPCBOp7gS5Z0f1mjoJYBrtStzgmJBxONW3U6OZqdtNzZPmn9BS/7WI7BFFcFQ==

"@types/mem-fs-editor@*":
version "7.0.4"
resolved "https://registry.yarnpkg.com/@types/mem-fs-editor/-/mem-fs-editor-7.0.4.tgz#72a54147274eb86cc63ade8dc51f3d1eaa83ba7f"
Expand Down Expand Up @@ -5346,41 +5341,13 @@ datemath-parser@^1.0.6:
dependencies:
moment "^2.22.2"

debug@2.2.0:
version "2.2.0"
resolved "https://registry.yarnpkg.com/debug/-/debug-2.2.0.tgz#f87057e995b1a1f6ae6a4960664137bc56f039da"
integrity sha512-X0rGvJcskG1c3TgSCPqHJ0XJgwlcvOC7elJ5Y0hYuKBZoVqWpAMfLOeIh2UI/DCQ5ruodIjvsugZtjUYUw2pUw==
dependencies:
ms "0.7.1"

debug@2.3.3:
version "2.3.3"
resolved "https://registry.yarnpkg.com/debug/-/debug-2.3.3.tgz#40c453e67e6e13c901ddec317af8986cda9eff8c"
integrity sha512-dCHp4G+F11zb+RtEu7BE2U8R32AYmM/4bljQfut8LipH3PdwsVBVGh083MXvtKkB7HSQUzSwiXz53c4mzJvYfw==
dependencies:
ms "0.7.2"

debug@2.6.9, debug@^2.2.0:
version "2.6.9"
resolved "https://registry.yarnpkg.com/debug/-/debug-2.6.9.tgz#5d128515df134ff327e90a4c93f4e077a536341f"
integrity sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA==
dependencies:
ms "2.0.0"

debug@4, debug@4.x, debug@^4.1.0, debug@^4.1.1, debug@^4.3.1, debug@^4.3.2, debug@^4.3.3, debug@^4.3.4, debug@~4.3.1:
debug@2.2.0, debug@2.3.3, debug@2.6.9, debug@4, debug@4.x, debug@^2.2.0, debug@^3.2.6, debug@^3.2.7, debug@^4.1.0, debug@^4.1.1, debug@^4.3.1, debug@^4.3.2, debug@^4.3.3, debug@^4.3.4, debug@~4.3.1:
version "4.3.4"
resolved "https://registry.yarnpkg.com/debug/-/debug-4.3.4.tgz#1319f6579357f2338d3337d2cdd4914bb5dcc865"
integrity sha512-PRWFHuSU3eDtQJPvnNY7Jcket1j0t5OuOsFzPPzsekD52Zl8qUfFIPEiswXqIvHWGVHOgX+7G/vCNNhehwxfkQ==
dependencies:
ms "2.1.2"

debug@^3.2.6, debug@^3.2.7:
version "3.2.7"
resolved "https://registry.yarnpkg.com/debug/-/debug-3.2.7.tgz#72580b7e9145fb39b6676f9c5e5fb100b934179a"
integrity sha512-CFjzYYAi4ThfiQvizrFQevTTXHtnCqWfe7x1AhgEscTz6ZbLbfoLRLPugTQyBth6f8ZERVUSyWHFD/7Wu4t1XQ==
dependencies:
ms "^2.1.1"

debuglog@^1.0.1:
version "1.0.1"
resolved "https://registry.yarnpkg.com/debuglog/-/debuglog-1.0.1.tgz#aa24ffb9ac3df9a2351837cfb2d279360cd78492"
Expand Down Expand Up @@ -9195,6 +9162,11 @@ just-extend@^4.0.2:
resolved "https://registry.yarnpkg.com/just-extend/-/just-extend-4.2.1.tgz#ef5e589afb61e5d66b24eca749409a8939a8c744"
integrity sha512-g3UB796vUFIY90VIv/WX3L2c8CS2MdWUww3CNrYmqza1Fg0DURc2K/O4YrnklBdQarSJ/y8JnJYDGc+1iumQjg==

kafkajs@^2.2.4:
version "2.2.4"
resolved "https://registry.yarnpkg.com/kafkajs/-/kafkajs-2.2.4.tgz#59e6e16459d87fdf8b64be73970ed5aa42370a5b"
integrity sha512-j/YeapB1vfPT2iOIUn/vxdyKEuhuY2PxMBvf5JWux6iSaukAccrMtXEY/Lb7OvavDhOWME589bpLrEdnVHjfjA==

kareem@2.4.1:
version "2.4.1"
resolved "https://registry.yarnpkg.com/kareem/-/kareem-2.4.1.tgz#7d81ec518204a48c1cb16554af126806c3cd82b0"
Expand Down Expand Up @@ -10071,27 +10043,7 @@ mquery@5.0.0:
dependencies:
debug "4.x"

ms@0.7.1:
version "0.7.1"
resolved "https://registry.yarnpkg.com/ms/-/ms-0.7.1.tgz#9cd13c03adbff25b65effde7ce864ee952017098"
integrity sha512-lRLiIR9fSNpnP6TC4v8+4OU7oStC01esuNowdQ34L+Gk8e5Puoc88IqJ+XAY/B3Mn2ZKis8l8HX90oU8ivzUHg==

ms@0.7.2:
version "0.7.2"
resolved "https://registry.yarnpkg.com/ms/-/ms-0.7.2.tgz#ae25cf2512b3885a1d95d7f037868d8431124765"
integrity sha512-5NnE67nQSQDJHVahPJna1PQ/zCXMnQop3yUCxjKPNzCxuyPSKWTQ/5Gu5CZmjetwGLWRA+PzeF5thlbOdbQldA==

ms@2.0.0:
version "2.0.0"
resolved "https://registry.yarnpkg.com/ms/-/ms-2.0.0.tgz#5608aeadfc00be6c2901df5f9861788de0d597c8"
integrity sha512-Tpp60P6IUJDTuOq/5Z8cdskzJujfwqfOTkrwIwj7IRISpnkJnT6SyJ4PCPnGMoFjC9ddhal5KVIYtAt97ix05A==

ms@2.1.2:
version "2.1.2"
resolved "https://registry.yarnpkg.com/ms/-/ms-2.1.2.tgz#d09d1f357b443f493382a8eb3ccd183872ae6009"
integrity sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==

ms@2.1.3, ms@^2.0.0, ms@^2.1.1, ms@^2.1.3:
ms@2.1.2, ms@2.1.3, ms@^2.0.0, ms@^2.1.1, ms@^2.1.3:
version "2.1.3"
resolved "https://registry.yarnpkg.com/ms/-/ms-2.1.3.tgz#574c8138ce1d2b5861f0b44579dbadd60c6615b2"
integrity sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==
Expand Down

0 comments on commit fb4bbdf

Please sign in to comment.