From d9b5be49f6d816f2820d3afb26db50290064adbb Mon Sep 17 00:00:00 2001 From: jzonthemtn Date: Wed, 5 Jun 2024 09:16:02 -0400 Subject: [PATCH 1/5] #188 Adding support for dataprepper auth. --- dataprepper/pipelines.yaml | 4 +++ docker-compose.yaml | 2 ++ .../org/opensearch/ubi/UbiActionFilter.java | 17 +++++++++--- .../java/org/opensearch/ubi/UbiSettings.java | 27 ++++++++++++++++--- 4 files changed, 43 insertions(+), 7 deletions(-) diff --git a/dataprepper/pipelines.yaml b/dataprepper/pipelines.yaml index ac78a7d..d90ccc3 100644 --- a/dataprepper/pipelines.yaml +++ b/dataprepper/pipelines.yaml @@ -3,6 +3,10 @@ chorus-ubi-pipeline: http: port: 2021 ssl: false +# authentication: +# http_basic: +# username: ubi +# password: ubi sink: - opensearch: hosts: [ "http://ubi-dev-os:9200" ] diff --git a/docker-compose.yaml b/docker-compose.yaml index 2aa1d54..6717d94 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -24,6 +24,8 @@ services: logger.level: info OPENSEARCH_INITIAL_ADMIN_PASSWORD: SuperSecretPassword_123 #ubi.dataprepper.url: "http://dataprepper-dev-os:2021/log/ingest" + #ubi.dataprepper.auth.username: "ubi" + #ubi.dataprepper.auth.password: "ubi" ulimits: memlock: soft: -1 diff --git a/src/main/java/org/opensearch/ubi/UbiActionFilter.java b/src/main/java/org/opensearch/ubi/UbiActionFilter.java index 816024b..fbdb536 100644 --- a/src/main/java/org/opensearch/ubi/UbiActionFilter.java +++ b/src/main/java/org/opensearch/ubi/UbiActionFilter.java @@ -8,10 +8,7 @@ package org.opensearch.ubi; -import com.fasterxml.jackson.annotation.JsonAutoDetect; -import com.fasterxml.jackson.annotation.PropertyAccessor; -import com.fasterxml.jackson.core.JsonProcessingException; -import com.fasterxml.jackson.databind.ObjectMapper; +import org.apache.http.HttpHeaders; import org.apache.http.client.methods.CloseableHttpResponse; import org.apache.http.client.methods.HttpPost; import org.apache.http.entity.StringEntity; @@ -44,9 +41,11 @@ import java.io.ByteArrayOutputStream; import java.io.IOException; import java.io.InputStream; +import java.nio.charset.Charset; import java.nio.charset.StandardCharsets; import java.security.AccessController; import java.security.PrivilegedAction; +import java.util.Base64; import java.util.HashMap; import java.util.LinkedList; import java.util.List; @@ -186,11 +185,21 @@ private void sendToDataPrepper(final String dataPrepperUrl, final QueryRequest q try (CloseableHttpClient httpClient = HttpClients.createDefault()) { + final String dataPrepperUserName = environment.settings().get(UbiSettings.DATA_PREPPER_AUTH_USERNAME); + final HttpPost httpPost = new HttpPost(dataPrepperUrl); httpPost.setEntity(new StringEntity(queryRequest.toString())); httpPost.setHeader("Content-type", "application/json"); + if(dataPrepperUserName != null) { + final String dataPrepperPassword = environment.settings().get(UbiSettings.DATA_PREPPER_AUTH_PASSWORD); + final String auth = dataPrepperUserName + ":" + dataPrepperPassword; + final byte[] encodedAuth = Base64.getEncoder().encode(auth.getBytes(StandardCharsets.ISO_8859_1)); + final String authHeader = "Basic " + new String(encodedAuth, StandardCharsets.ISO_8859_1); + httpPost.setHeader(HttpHeaders.AUTHORIZATION, authHeader); + } + AccessController.doPrivileged((PrivilegedAction) () -> { try (CloseableHttpResponse response = httpClient.execute(httpPost)) { final int status = response.getStatusLine().getStatusCode(); diff --git a/src/main/java/org/opensearch/ubi/UbiSettings.java b/src/main/java/org/opensearch/ubi/UbiSettings.java index 656962a..02257ac 100644 --- a/src/main/java/org/opensearch/ubi/UbiSettings.java +++ b/src/main/java/org/opensearch/ubi/UbiSettings.java @@ -10,7 +10,6 @@ import org.opensearch.common.settings.Setting; -import java.util.Collections; import java.util.List; /** @@ -23,18 +22,40 @@ public class UbiSettings { */ public static final String DATA_PREPPER_URL = "ubi.dataprepper.url"; + /** + * The optional username for Data Prepper's http_source. + */ + public static final String DATA_PREPPER_AUTH_USERNAME = "ubi.dataprepper.auth.username"; + + /** + * The optional password for Data Prepper's http_source. + */ + public static final String DATA_PREPPER_AUTH_PASSWORD = "ubi.dataprepper.auth.password"; + private static final Setting DATA_PREPPER_URL_SETTING = Setting.simpleString( DATA_PREPPER_URL, Setting.Property.Dynamic, Setting.Property.NodeScope); + private static final Setting DATA_PREPPER_AUTH_USERNAME_SETTING = Setting.simpleString( + DATA_PREPPER_AUTH_USERNAME, + Setting.Property.Dynamic, + Setting.Property.NodeScope); + + private static final Setting DATA_PREPPER_AUTH_PASSWORD_PASSWORD = Setting.simpleString( + DATA_PREPPER_AUTH_PASSWORD, + Setting.Property.Dynamic, + Setting.Property.NodeScope); + /** * Gets a list of the UBI plugin settings. * @return A list of the UBI plugin settings. */ public static List> getSettings() { - return Collections.singletonList( - DATA_PREPPER_URL_SETTING + return List.of( + DATA_PREPPER_URL_SETTING, + DATA_PREPPER_AUTH_USERNAME_SETTING, + DATA_PREPPER_AUTH_PASSWORD_PASSWORD ); } From 96b9b852ea406273a7408cdb7949aa5d6e0e76ae Mon Sep 17 00:00:00 2001 From: jzonthemtn Date: Wed, 5 Jun 2024 09:17:48 -0400 Subject: [PATCH 2/5] Moving line. --- src/main/java/org/opensearch/ubi/UbiActionFilter.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/java/org/opensearch/ubi/UbiActionFilter.java b/src/main/java/org/opensearch/ubi/UbiActionFilter.java index fbdb536..46bc1b5 100644 --- a/src/main/java/org/opensearch/ubi/UbiActionFilter.java +++ b/src/main/java/org/opensearch/ubi/UbiActionFilter.java @@ -185,13 +185,13 @@ private void sendToDataPrepper(final String dataPrepperUrl, final QueryRequest q try (CloseableHttpClient httpClient = HttpClients.createDefault()) { - final String dataPrepperUserName = environment.settings().get(UbiSettings.DATA_PREPPER_AUTH_USERNAME); - final HttpPost httpPost = new HttpPost(dataPrepperUrl); httpPost.setEntity(new StringEntity(queryRequest.toString())); httpPost.setHeader("Content-type", "application/json"); + final String dataPrepperUserName = environment.settings().get(UbiSettings.DATA_PREPPER_AUTH_USERNAME); + if(dataPrepperUserName != null) { final String dataPrepperPassword = environment.settings().get(UbiSettings.DATA_PREPPER_AUTH_PASSWORD); final String auth = dataPrepperUserName + ":" + dataPrepperPassword; From 5546f5027477cf5eb25ec6caedd5d69bb3408a76 Mon Sep 17 00:00:00 2001 From: jzonthemtn Date: Wed, 5 Jun 2024 09:43:50 -0400 Subject: [PATCH 3/5] Updating readme. --- README.md | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 16458c5..d52b16d 100644 --- a/README.md +++ b/README.md @@ -213,11 +213,15 @@ Each indexed query will have the following fields: To send queries to Data Prepper, configure the following properties in OpenSearch: -| Property | Description | Example Value | -|---------------------|---------------------------------------|-------------------------------| -| ubi.dataprepper.url | Data Prepper's `http_source` endpoint | `http://localhost:2021/log/ingest` | +| Property | Description | Example Value | +|-------------------------------|--------------------------------------------------------------------|------------------------------------| +| ubi.dataprepper.url | Data Prepper's `http_source` endpoint | `http://localhost:2021/log/ingest` | +| ubi.dataprepper.auth.username | Data Prepper's `http_source` authentication username, if required. | Any string value | +| ubi.dataprepper.auth.password | Data Prepper's `http_source` authentication password, if required. | Any string value | -With these properties set, queries will no longer be indexed into the local OpenSearch. The `ubi_queries` index can be deleted. Queries will be sent to Data Prepper as they are received by OpenSearch. +With these properties set, queries will no longer be indexed into the local OpenSearch. The `ubi_queries` index can be deleted if they have been created. Queries will be sent to Data Prepper as they are received by OpenSearch. + +When Data Prepper is enabled, it is important to verify queries are successfully making it into the Data Prepper sink. If the sink does not contain the expected queries, check the Data Prepper log for errors. ### Capturing Events @@ -338,7 +342,7 @@ The plugin can be built using Gradle: ./gradlew build ``` -To test and debug, build the OpenSearch docker image that contains the built plugin and then start the containers: +The `docker-compose-*.yml` files in this repository are only for development and test. Its configuration can be used for reference, but it is not intended for production use. To test and debug, build the OpenSearch docker image that contains the built plugin and then start the containers: ``` docker compose build && docker compose up From 62adb88f11850e9165aac17e04eafe28c718570f Mon Sep 17 00:00:00 2001 From: jzonthemtn Date: Wed, 5 Jun 2024 09:44:58 -0400 Subject: [PATCH 4/5] Updating readme. --- README.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index d52b16d..1d39369 100644 --- a/README.md +++ b/README.md @@ -213,11 +213,11 @@ Each indexed query will have the following fields: To send queries to Data Prepper, configure the following properties in OpenSearch: -| Property | Description | Example Value | -|-------------------------------|--------------------------------------------------------------------|------------------------------------| -| ubi.dataprepper.url | Data Prepper's `http_source` endpoint | `http://localhost:2021/log/ingest` | -| ubi.dataprepper.auth.username | Data Prepper's `http_source` authentication username, if required. | Any string value | -| ubi.dataprepper.auth.password | Data Prepper's `http_source` authentication password, if required. | Any string value | +| Property | Description | Example Value | +|-------------------------------|-----------------------------------------------------------------------------------------------------------------------------------------------|------------------------------------| +| ubi.dataprepper.url | Data Prepper's `http_source` endpoint | `http://localhost:2021/log/ingest` | +| ubi.dataprepper.auth.username | Data Prepper's `http_source` basic HTTP authentication username, if required. Do not include this property if authentication is not required. | Any string value | +| ubi.dataprepper.auth.password | Data Prepper's `http_source` basic HTTP authentication password, if required. | Any string value | With these properties set, queries will no longer be indexed into the local OpenSearch. The `ubi_queries` index can be deleted if they have been created. Queries will be sent to Data Prepper as they are received by OpenSearch. From 723a350f62f4b5c3eb6b519cd3bbddd1672ab4b5 Mon Sep 17 00:00:00 2001 From: jzonthemtn Date: Wed, 5 Jun 2024 09:46:21 -0400 Subject: [PATCH 5/5] Updating readme. --- README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.md b/README.md index 1d39369..82d4bc3 100644 --- a/README.md +++ b/README.md @@ -221,6 +221,8 @@ To send queries to Data Prepper, configure the following properties in OpenSearc With these properties set, queries will no longer be indexed into the local OpenSearch. The `ubi_queries` index can be deleted if they have been created. Queries will be sent to Data Prepper as they are received by OpenSearch. +For information on configuring Data Prepper's `http_source` see Data Prepper's [http_source](https://opensearch.org/docs/latest/data-prepper/pipelines/configuration/sources/http-source/) documentation. + When Data Prepper is enabled, it is important to verify queries are successfully making it into the Data Prepper sink. If the sink does not contain the expected queries, check the Data Prepper log for errors. ### Capturing Events