diff --git a/README.md b/README.md index 16458c5..82d4bc3 100644 --- a/README.md +++ b/README.md @@ -213,11 +213,17 @@ Each indexed query will have the following fields: To send queries to Data Prepper, configure the following properties in OpenSearch: -| Property | Description | Example Value | -|---------------------|---------------------------------------|-------------------------------| -| ubi.dataprepper.url | Data Prepper's `http_source` endpoint | `http://localhost:2021/log/ingest` | +| Property | Description | Example Value | +|-------------------------------|-----------------------------------------------------------------------------------------------------------------------------------------------|------------------------------------| +| ubi.dataprepper.url | Data Prepper's `http_source` endpoint | `http://localhost:2021/log/ingest` | +| ubi.dataprepper.auth.username | Data Prepper's `http_source` basic HTTP authentication username, if required. Do not include this property if authentication is not required. | Any string value | +| ubi.dataprepper.auth.password | Data Prepper's `http_source` basic HTTP authentication password, if required. | Any string value | -With these properties set, queries will no longer be indexed into the local OpenSearch. The `ubi_queries` index can be deleted. Queries will be sent to Data Prepper as they are received by OpenSearch. +With these properties set, queries will no longer be indexed into the local OpenSearch. The `ubi_queries` index can be deleted if they have been created. Queries will be sent to Data Prepper as they are received by OpenSearch. + +For information on configuring Data Prepper's `http_source` see Data Prepper's [http_source](https://opensearch.org/docs/latest/data-prepper/pipelines/configuration/sources/http-source/) documentation. + +When Data Prepper is enabled, it is important to verify queries are successfully making it into the Data Prepper sink. If the sink does not contain the expected queries, check the Data Prepper log for errors. ### Capturing Events @@ -338,7 +344,7 @@ The plugin can be built using Gradle: ./gradlew build ``` -To test and debug, build the OpenSearch docker image that contains the built plugin and then start the containers: +The `docker-compose-*.yml` files in this repository are only for development and test. Its configuration can be used for reference, but it is not intended for production use. To test and debug, build the OpenSearch docker image that contains the built plugin and then start the containers: ``` docker compose build && docker compose up diff --git a/dataprepper/pipelines.yaml b/dataprepper/pipelines.yaml index ac78a7d..d90ccc3 100644 --- a/dataprepper/pipelines.yaml +++ b/dataprepper/pipelines.yaml @@ -3,6 +3,10 @@ chorus-ubi-pipeline: http: port: 2021 ssl: false +# authentication: +# http_basic: +# username: ubi +# password: ubi sink: - opensearch: hosts: [ "http://ubi-dev-os:9200" ] diff --git a/docker-compose.yaml b/docker-compose.yaml index 2aa1d54..6717d94 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -24,6 +24,8 @@ services: logger.level: info OPENSEARCH_INITIAL_ADMIN_PASSWORD: SuperSecretPassword_123 #ubi.dataprepper.url: "http://dataprepper-dev-os:2021/log/ingest" + #ubi.dataprepper.auth.username: "ubi" + #ubi.dataprepper.auth.password: "ubi" ulimits: memlock: soft: -1 diff --git a/src/main/java/org/opensearch/ubi/UbiActionFilter.java b/src/main/java/org/opensearch/ubi/UbiActionFilter.java index 816024b..46bc1b5 100644 --- a/src/main/java/org/opensearch/ubi/UbiActionFilter.java +++ b/src/main/java/org/opensearch/ubi/UbiActionFilter.java @@ -8,10 +8,7 @@ package org.opensearch.ubi; -import com.fasterxml.jackson.annotation.JsonAutoDetect; -import com.fasterxml.jackson.annotation.PropertyAccessor; -import com.fasterxml.jackson.core.JsonProcessingException; -import com.fasterxml.jackson.databind.ObjectMapper; +import org.apache.http.HttpHeaders; import org.apache.http.client.methods.CloseableHttpResponse; import org.apache.http.client.methods.HttpPost; import org.apache.http.entity.StringEntity; @@ -44,9 +41,11 @@ import java.io.ByteArrayOutputStream; import java.io.IOException; import java.io.InputStream; +import java.nio.charset.Charset; import java.nio.charset.StandardCharsets; import java.security.AccessController; import java.security.PrivilegedAction; +import java.util.Base64; import java.util.HashMap; import java.util.LinkedList; import java.util.List; @@ -191,6 +190,16 @@ private void sendToDataPrepper(final String dataPrepperUrl, final QueryRequest q httpPost.setEntity(new StringEntity(queryRequest.toString())); httpPost.setHeader("Content-type", "application/json"); + final String dataPrepperUserName = environment.settings().get(UbiSettings.DATA_PREPPER_AUTH_USERNAME); + + if(dataPrepperUserName != null) { + final String dataPrepperPassword = environment.settings().get(UbiSettings.DATA_PREPPER_AUTH_PASSWORD); + final String auth = dataPrepperUserName + ":" + dataPrepperPassword; + final byte[] encodedAuth = Base64.getEncoder().encode(auth.getBytes(StandardCharsets.ISO_8859_1)); + final String authHeader = "Basic " + new String(encodedAuth, StandardCharsets.ISO_8859_1); + httpPost.setHeader(HttpHeaders.AUTHORIZATION, authHeader); + } + AccessController.doPrivileged((PrivilegedAction) () -> { try (CloseableHttpResponse response = httpClient.execute(httpPost)) { final int status = response.getStatusLine().getStatusCode(); diff --git a/src/main/java/org/opensearch/ubi/UbiSettings.java b/src/main/java/org/opensearch/ubi/UbiSettings.java index 656962a..02257ac 100644 --- a/src/main/java/org/opensearch/ubi/UbiSettings.java +++ b/src/main/java/org/opensearch/ubi/UbiSettings.java @@ -10,7 +10,6 @@ import org.opensearch.common.settings.Setting; -import java.util.Collections; import java.util.List; /** @@ -23,18 +22,40 @@ public class UbiSettings { */ public static final String DATA_PREPPER_URL = "ubi.dataprepper.url"; + /** + * The optional username for Data Prepper's http_source. + */ + public static final String DATA_PREPPER_AUTH_USERNAME = "ubi.dataprepper.auth.username"; + + /** + * The optional password for Data Prepper's http_source. + */ + public static final String DATA_PREPPER_AUTH_PASSWORD = "ubi.dataprepper.auth.password"; + private static final Setting DATA_PREPPER_URL_SETTING = Setting.simpleString( DATA_PREPPER_URL, Setting.Property.Dynamic, Setting.Property.NodeScope); + private static final Setting DATA_PREPPER_AUTH_USERNAME_SETTING = Setting.simpleString( + DATA_PREPPER_AUTH_USERNAME, + Setting.Property.Dynamic, + Setting.Property.NodeScope); + + private static final Setting DATA_PREPPER_AUTH_PASSWORD_PASSWORD = Setting.simpleString( + DATA_PREPPER_AUTH_PASSWORD, + Setting.Property.Dynamic, + Setting.Property.NodeScope); + /** * Gets a list of the UBI plugin settings. * @return A list of the UBI plugin settings. */ public static List> getSettings() { - return Collections.singletonList( - DATA_PREPPER_URL_SETTING + return List.of( + DATA_PREPPER_URL_SETTING, + DATA_PREPPER_AUTH_USERNAME_SETTING, + DATA_PREPPER_AUTH_PASSWORD_PASSWORD ); }