Add ssl encryption to Elasticsearch

e2e/helm/helmfile.yaml

@@ -25,7 +25,7 @@ releases:
   - name: opensearch2
     namespace: services-dev1
     chart: opensearch/opensearch
-    version: 2.17.1
+    version: 2.31.0
     installed: {{ .Values | get "opensearch2.enabled" true }}
     values:
       - ./templates/os2.yaml.gotmpl

e2e/helm/templates/os2.yaml.gotmpl

@@ -19,10 +19,28 @@ config:
         {{- if eq (.Values | get "opensearch2.instances" 1) 1 }}
         discovery.type: single-node
         {{- end }}
+        {{- if eq (.Values | get "opensearch2.ssl.enabled" false) true}}
+        plugins.security.ssl.transport.pemcert_filepath: certs/opensearch-cert.pem
+        plugins.security.ssl.transport.pemkey_filepath: certs/opensearch-key.pem
+        plugins.security.ssl.transport.pemtrustedcas_filepath: certs/CAs/rootCA.pem  # Root CA
+
+        plugins.security.ssl.http.enabled: true
+        plugins.security.ssl.http.pemcert_filepath: certs/opensearch-cert.pem
+        plugins.security.ssl.http.pemkey_filepath: certs/opensearch-key.pem
+        plugins.security.ssl.http.pemtrustedcas_filepath: certs/CAs/rootCA.pem  # Root CA
+
+        plugins.security.authcz.admin_dn:
+            - "{{ .Values | get "opensearch2.ssl.admin_dn" "" }}"
+        {{- end }}
 
 extraEnvs:
+    {{- if eq (.Values | get "opensearch2.ssl.enabled" false) true}}
+    - name: OPENSEARCH_INITIAL_ADMIN_PASSWORD
+      value: "passwordsufhbivbU123%$"
+    {{- else }}
     - name: DISABLE_SECURITY_PLUGIN
       value: "true"
+    {{- end }}
 
 clusterName: opensearch2-cluster
 
@@ -37,3 +55,48 @@ resources:
 
 persistence:
     size: {{ .Values | get "opensearch2.persistentVolumeSize" "8Gi" }}
+
+{{- if eq (.Values | get "opensearch2.ssl.enabled" false) true }}
+extraVolumes:
+    - name: certs
+      hostPath:
+        path: /certs
+        type: Directory
+extraVolumeMounts:
+    - name: certs
+      mountPath: /usr/share/opensearch/config/certs
+      readOnly: false
+
+extraContainers:
+  - name: security-admin-init
+    image: opensearchproject/opensearch:{{ .Values | get "opensearch2.version" "2.15.0" }}
+    command: ["/bin/sh", "-c"]
+    args:
+      - |
+        echo "Waiting for OpenSearch to be ready..."
+        until curl -k --silent --output /dev/null https://opensearch2.services-dev1:9200; do
+          echo "Still waiting..."
+          sleep 5
+        done
+        echo "Building securityConfig Directory..."
+        mkdir -p /usr/share/opensearch/plugins/opensearch-security/securityconfig
+        cp -R /usr/share/opensearch/config/opensearch-security/* \
+        /usr/share/opensearch/plugins/opensearch-security/securityconfig/
+        cp /usr/share/opensearch/config/certs/internal_users.yml \
+        /usr/share/opensearch/plugins/opensearch-security/securityconfig/internal_users.yml
+        echo "Successfully build security config!"
+        echo "Running securityadmin.sh..."
+        /usr/share/opensearch/plugins/opensearch-security/tools/securityadmin.sh \
+          -cd /usr/share/opensearch/plugins/opensearch-security/securityconfig/ \
+          -icl \
+          -nhnv \
+          -cacert /usr/share/opensearch/config/certs/CAs/rootCA.pem \
+          -cert /usr/share/opensearch/config/certs/opensearch-cert.pem \
+          -key /usr/share/opensearch/config/certs/opensearch-key.pem \
+          -h opensearch2.services-dev1
+        echo "Security initialized successfully!"
+        sleep 1000000
+    volumeMounts:
+      - name: certs
+        mountPath: /usr/share/opensearch/config/certs
+{{- end }}

e2e/helm/templates/teraslice.yaml.gotmpl

@@ -19,14 +19,29 @@ terafoundation:
             default:
                 node:
                     {{- if eq (.Values | get "teraslice.stateCluster" "none") "opensearch2" }}
+                    {{- if eq (.Values | get "opensearch2.ssl.enabled" false) true}}
+                    - "https://opensearch2.services-dev1:9200"
+                    {{- else}}
                     - "http://opensearch2.services-dev1:9200"
+                    {{- end }}
                     {{- else if eq (.Values | get "teraslice.stateCluster" "none") "opensearch1" }}
                     - "http://opensearch1.services-dev1:9200"
                     {{- else if eq (.Values | get "teraslice.stateCluster" "none") "elasticsearch6" }}
                     - "http://elasticsearch6-master.services-dev1:9200"
                     {{- else if eq (.Values | get "teraslice.stateCluster" "none") "elasticsearch7" }}
                     - "http://elasticsearch7-master.services-dev1:9200"
                     {{- end }}
+                {{- if eq (.Values | get "opensearch2.ssl.enabled" false) true}}
+                username: admin
+                password: "passwordsufhbivbU123%$"
+
+                caCertificate: "{{- if .Values.opensearch2.ssl.caCert -}}
+                {{ .Values.opensearch2.ssl.caCert }}
+                {{- else -}}
+                {{ fail "opensearch2.ssl.caCert MUST be provided so teraslice can get the rootCA" }}
+                {{- end }}"
+
+                {{- end }}
             {{- if eq (.Values | get "opensearch1.enabled" false) true }}
             os1:
                 node:
@@ -35,7 +50,23 @@ terafoundation:
             {{- if eq (.Values | get "opensearch2.enabled" false) true }}
             os2:
                 node:
+                    {{- if eq (.Values | get "opensearch2.ssl.enabled" false) true}}
+                    - "https://opensearch2.services-dev1:9200"
+                    {{- else }}
                     - "http://opensearch2.services-dev1:9200"
+                    {{- end }}
+
+                {{- if eq (.Values | get "opensearch2.ssl.enabled" false) true}}
+                username: admin
+                password: "passwordsufhbivbU123%$"
+
+                caCertificate: "{{- if .Values.opensearch2.ssl.caCert -}}
+                {{ .Values.opensearch2.ssl.caCert }}
+                {{- else -}}
+                {{ fail "opensearch2.ssl.caCert MUST be provided so teraslice can get the rootCA" }}
+                {{- end }}"
+
+                {{- end }}
             {{- end }}
             {{- if eq (.Values | get "elasticsearch6.enabled" false) true }}
             es6:

e2e/helm/values.yaml

@@ -11,6 +11,24 @@ opensearch1:
 opensearch2:
   # If false, opensearch2 will be excluded on helmfile sync
   enabled: true
+
+  ssl:
+    # If true, "caCert" and "admin_dn" need to be set
+    enabled: false
+
+    # CA certificate in a single-line string format with "\\n" as line breaks
+    # Use the following command to convert a rootCA PEM file to the correct format:
+    # awk '{printf "%s\\\\n", $0}' <path-to-pem-file> | pbcopy
+    # This is so teraslice can get the rootCA for it's client validation
+    caCert: null
+
+    # The distinguished name (DN) of the OpenSearch admin user
+    # Required when SSL is enabled
+    # Retrieve this using the following command:
+    # openssl x509 -in <path-to-cert> -noout -subject | awk '{print substr($0,9)}'
+    # Example output: 'O=mkcert development certificate, OU=local'
+    # FIXME: This needs to be auto generated in e2e
+    admin_dn: ""
   version: 2.15.0
   esJavaOpts: -Xmx512M -Xms512M
   memoryLimit: 100Mi
@@ -59,7 +77,7 @@ minio:
 
 kafka:
   # If false, kafka will be excluded on helmfile sync
-  enabled: true
+  enabled: false
   version: 7.7.1
   brokers: 1
   offsets:
@@ -69,7 +87,7 @@ kafka:
 
 zookeeper:
   # If false, zookeeper will be excluded on helmfile sync
-  enabled: true
+  enabled: false
   version: 7.7.1
   instances: 1
 

e2e/package.json

@@ -36,6 +36,7 @@
         "test:k8sNoBuild": "SKIP_DOCKER_BUILD_IN_E2E='true' TEST_ELASTICSEARCH='true' ELASTICSEARCH_VERSION='7.9.3' TEST_KAFKA='true' TEST_PLATFORM='kubernetes' ts-scripts test --suite e2e --",
         "test:k8sV2": "TEST_ELASTICSEARCH='true' ELASTICSEARCH_VERSION='7.9.3' TEST_KAFKA='true' TEST_PLATFORM='kubernetesV2' ts-scripts test --suite e2e --",
         "test:k8sV2Helmfile": "TEST_OPENSEARCH='true' OPENSEARCH_VERSION='2.15.0' TEST_KAFKA='true' TEST_PLATFORM='kubernetesV2' USE_HELMFILE='true' ts-scripts test --suite e2e --",
+        "test:k8sV2HelmfileEncrypted": "TEST_OPENSEARCH='true' OPENSEARCH_VERSION='2.15.0' ENCRYPT_OPENSEARCH='true' OPENSEARCH_PASSWORD='passwordsufhbivbU123%$' TEST_KAFKA='true' TEST_PLATFORM='kubernetesV2' USE_HELMFILE='true' ts-scripts test --suite e2e --debug=true --",
         "test:opensearch1": "TEST_OPENSEARCH='true' TEST_KAFKA='true' ts-scripts test --suite e2e --",
         "test:opensearch2": "TEST_OPENSEARCH='true' OPENSEARCH_VERSION='2.15.0' TEST_KAFKA='true' ts-scripts test --suite e2e --",
         "test:s3AssetStorage": "TEST_OPENSEARCH='true' TEST_KAFKA='true' TEST_MINIO='true' ASSET_STORAGE_CONNECTION_TYPE='s3' ASSET_STORAGE_CONNECTION='default' ts-scripts test --suite e2e --",

e2e/test/config.ts

@@ -9,7 +9,10 @@ const {
     ELASTICSEARCH_HOST,
     ELASTICSEARCH_VERSION,
     OPENSEARCH_HOST,
+    OPENSEARCH_SSL_HOST,
     OPENSEARCH_VERSION,
+    OPENSEARCH_USER,
+    OPENSEARCH_PASSWORD
 } = ElasticsearchTestHelpers;
 
 const filePath = fileURLToPath(new URL(import.meta.url));
@@ -55,12 +58,16 @@ const {
     ASSET_STORAGE_CONNECTION_TYPE = 'elasticsearch-next',
     ASSET_STORAGE_CONNECTION = 'default',
     ENCRYPT_MINIO = false,
+    ENCRYPT_OPENSEARCH = false,
     MINIO_HOST = 'http://127.0.0.1:49000',
     MINIO_ACCESS_KEY = 'minioadmin',
     MINIO_SECRET_KEY = 'minioadmin'
 } = process.env;
 
-const TEST_HOST = TEST_OPENSEARCH ? OPENSEARCH_HOST : ELASTICSEARCH_HOST;
+const TEST_HOST = TEST_OPENSEARCH
+    ? ENCRYPT_OPENSEARCH ? OPENSEARCH_SSL_HOST : OPENSEARCH_HOST
+    : ELASTICSEARCH_HOST;
+
 const USE_HELMFILE = toBoolean(process.env.USE_HELMFILE) || false;
 
 function newId(prefix?: string, lowerCase = false, length = 15) {
@@ -113,7 +120,10 @@ export {
     MINIO_ACCESS_KEY,
     MINIO_SECRET_KEY,
     ENCRYPT_MINIO,
+    ENCRYPT_OPENSEARCH,
     ROOT_CERT_PATH,
     TEST_OPENSEARCH,
-    USE_HELMFILE
+    USE_HELMFILE,
+    OPENSEARCH_USER,
+    OPENSEARCH_PASSWORD
 };

e2e/test/teardown.ts

@@ -3,7 +3,8 @@ import { helmfileDelete, K8s } from '@terascope/scripts';
 import fse from 'fs-extra';
 import {
     KEEP_OPEN, CONFIG_PATH, ASSETS_PATH, TEST_INDEX_PREFIX,
-    TEST_PLATFORM, TERASLICE_PORT, KIND_CLUSTER, USE_HELMFILE
+    TEST_PLATFORM, TERASLICE_PORT, KIND_CLUSTER, USE_HELMFILE,
+    ROOT_CERT_PATH
 } from './config.js';
 import { tearDown } from './docker-helpers.js';
 import signale from './signale.js';
@@ -12,7 +13,7 @@ const { cleanupIndex, makeClient } = ElasticsearchTestHelpers;
 
 async function getClient(client?: Client) {
     if (client) return client;
-    return makeClient();
+    return makeClient(ROOT_CERT_PATH);
 }
 
 export async function teardown(testClient?: Client) {

e2e/test/teraslice-harness.ts

@@ -5,14 +5,14 @@ import {
 } from '@terascope/utils';
 import { showState } from '@terascope/scripts';
 import { JobConfig, Teraslice } from '@terascope/types';
-import { createClient, ElasticsearchTestHelpers, Client } from 'elasticsearch-store';
+import { createClient, ElasticsearchTestHelpers, Client, ClientConfig } from 'elasticsearch-store';
 import { TerasliceClient } from 'teraslice-client-js';
 import fse from 'fs-extra';
 import {
     TEST_HOST, HOST_IP, SPEC_INDEX_PREFIX,
     DEFAULT_NODES, newId, DEFAULT_WORKERS, GENERATE_ONLY,
     EXAMPLE_INDEX_SIZES, EXAMPLE_INDEX_PREFIX, TEST_PLATFORM, TERASLICE_PORT,
-    LOG_PATH
+    LOG_PATH, ENCRYPT_OPENSEARCH, OPENSEARCH_USER, OPENSEARCH_PASSWORD, ROOT_CERT_PATH
 } from './config.js';
 import { scaleWorkers, getElapsed } from './docker-helpers.js';
 import signale from './signale.js';
@@ -65,7 +65,16 @@ export class TerasliceHarness {
     teraslice!: TerasliceClient;
 
     async init() {
-        const { client } = await createClient({ node: TEST_HOST });
+        const esConfig: ClientConfig = { node: TEST_HOST };
+        if (ENCRYPT_OPENSEARCH) {
+            esConfig.username = OPENSEARCH_USER;
+            esConfig.password = OPENSEARCH_PASSWORD;
+            esConfig.caCertificate = fse.readFileSync(ROOT_CERT_PATH, 'utf8');
+        }
+        // I should be able to not do this if I add a probe to os2
+        const { client } = await pRetry(async () => {
+            return await createClient(esConfig);
+        });
         this.client = client;
         this.teraslice = new TerasliceClient({
             host: `http://${HOST_IP}:${TERASLICE_PORT}`,

packages/elasticsearch-store/src/elasticsearch-client/create-client.ts

@@ -17,6 +17,17 @@ export async function createClient(
     config: ClientConfig,
     logger = debugLogger('elasticsearch-client')
 ): Promise<{ log: () => Logger; client: Client }> {
+    if (config.caCertificate) {
+        config.ssl = {
+            ca: config.caCertificate
+        };
+        if (config.username && config.password) {
+            config.auth = {
+                username: config.username,
+                password: config.password
+            };
+        }
+    }
     const distributionMetadata = await getDBMetadata(config, logger);
 
     const baseClient = await getBaseClient(

packages/elasticsearch-store/src/elasticsearch-client/interfaces.ts

@@ -61,6 +61,7 @@ interface ClientOptions {
 export interface ClientConfig extends ClientOptions {
     password?: string;
     username?: string;
+    caCertificate?: string;
 }
 
 export interface ClientOnlyParams {

packages/elasticsearch-store/src/test-helpers/config.ts

@@ -8,6 +8,7 @@ const {
     OPENSEARCH_PASSWORD = 'admin',
     OPENSEARCH_VERSION = '1.3.0',
     OPENSEARCH_HOST = `http://${OPENSEARCH_USER}:${OPENSEARCH_PASSWORD}@${OPENSEARCH_HOSTNAME}`,
+    OPENSEARCH_SSL_HOST = `https://${OPENSEARCH_HOSTNAME}:${OPENSEARCH_PORT}`,
     RESTRAINED_OPENSEARCH_PORT = process.env.RESTRAINED_OPENSEARCH_PORT || '49206',
     RESTRAINED_OPENSEARCH_HOST = `http://${OPENSEARCH_USER}:${OPENSEARCH_PASSWORD}@localhost:${RESTRAINED_OPENSEARCH_PORT}`,
 } = process.env;
@@ -17,6 +18,9 @@ export {
     ELASTICSEARCH_HOST,
     ELASTICSEARCH_VERSION,
     OPENSEARCH_HOST,
+    OPENSEARCH_SSL_HOST,
     OPENSEARCH_VERSION,
-    RESTRAINED_OPENSEARCH_HOST
+    RESTRAINED_OPENSEARCH_HOST,
+    OPENSEARCH_USER,
+    OPENSEARCH_PASSWORD
 };