Adding support for dataprepper auth

README.md

@@ -213,11 +213,17 @@ Each indexed query will have the following fields:
 
 To send queries to Data Prepper, configure the following properties in OpenSearch:
 
-| Property            | Description                           | Example Value                 |
-|---------------------|---------------------------------------|-------------------------------|
-| ubi.dataprepper.url | Data Prepper's `http_source` endpoint | `http://localhost:2021/log/ingest` |
+| Property                      | Description                                                                                                                                   | Example Value                      |
+|-------------------------------|-----------------------------------------------------------------------------------------------------------------------------------------------|------------------------------------|
+| ubi.dataprepper.url           | Data Prepper's `http_source` endpoint                                                                                                         | `http://localhost:2021/log/ingest` |
+| ubi.dataprepper.auth.username | Data Prepper's `http_source` basic HTTP authentication username, if required. Do not include this property if authentication is not required. | Any string value                   |
+| ubi.dataprepper.auth.password | Data Prepper's `http_source` basic HTTP authentication password, if required.                                                                 | Any string value                                   |
 
-With these properties set, queries will no longer be indexed into the local OpenSearch. The `ubi_queries` index can be deleted. Queries will be sent to Data Prepper as they are received by OpenSearch.
+With these properties set, queries will no longer be indexed into the local OpenSearch. The `ubi_queries` index can be deleted if they have been created. Queries will be sent to Data Prepper as they are received by OpenSearch.
+
+For information on configuring Data Prepper's `http_source` see Data Prepper's [http_source](https://opensearch.org/docs/latest/data-prepper/pipelines/configuration/sources/http-source/) documentation.
+
+When Data Prepper is enabled, it is important to verify queries are successfully making it into the Data Prepper sink. If the sink does not contain the expected queries, check the Data Prepper log for errors.
 
 ### Capturing Events
 
@@ -338,7 +344,7 @@ The plugin can be built using Gradle:
 ./gradlew build
 ```
 
-To test and debug, build the OpenSearch docker image that contains the built plugin and then start the containers:
+The `docker-compose-*.yml` files in this repository are only for development and test. Its configuration can be used for reference, but it is not intended for production use. To test and debug, build the OpenSearch docker image that contains the built plugin and then start the containers:
 
 ```
 docker compose build && docker compose up

dataprepper/pipelines.yaml

@@ -3,6 +3,10 @@ chorus-ubi-pipeline:
     http:
       port: 2021
       ssl: false
+#      authentication:
+#        http_basic:
+#          username: ubi
+#          password: ubi
   sink:
     - opensearch:
         hosts: [ "http://ubi-dev-os:9200" ]

docker-compose.yaml

@@ -24,6 +24,8 @@ services:
       logger.level: info
       OPENSEARCH_INITIAL_ADMIN_PASSWORD: SuperSecretPassword_123
       #ubi.dataprepper.url: "http://dataprepper-dev-os:2021/log/ingest"
+      #ubi.dataprepper.auth.username: "ubi"
+      #ubi.dataprepper.auth.password: "ubi"
     ulimits:
       memlock:
         soft: -1

src/main/java/org/opensearch/ubi/UbiActionFilter.java

@@ -8,10 +8,7 @@
 
 package org.opensearch.ubi;
 
-import com.fasterxml.jackson.annotation.JsonAutoDetect;
-import com.fasterxml.jackson.annotation.PropertyAccessor;
-import com.fasterxml.jackson.core.JsonProcessingException;
-import com.fasterxml.jackson.databind.ObjectMapper;
+import org.apache.http.HttpHeaders;
 import org.apache.http.client.methods.CloseableHttpResponse;
 import org.apache.http.client.methods.HttpPost;
 import org.apache.http.entity.StringEntity;
@@ -44,9 +41,11 @@
 import java.io.ByteArrayOutputStream;
 import java.io.IOException;
 import java.io.InputStream;
+import java.nio.charset.Charset;
 import java.nio.charset.StandardCharsets;
 import java.security.AccessController;
 import java.security.PrivilegedAction;
+import java.util.Base64;
 import java.util.HashMap;
 import java.util.LinkedList;
 import java.util.List;
@@ -191,6 +190,16 @@ private void sendToDataPrepper(final String dataPrepperUrl, final QueryRequest q
                 httpPost.setEntity(new StringEntity(queryRequest.toString()));
                 httpPost.setHeader("Content-type", "application/json");
 
+                final String dataPrepperUserName = environment.settings().get(UbiSettings.DATA_PREPPER_AUTH_USERNAME);
+
+                if(dataPrepperUserName != null) {
+                    final String dataPrepperPassword = environment.settings().get(UbiSettings.DATA_PREPPER_AUTH_PASSWORD);
+                    final String auth = dataPrepperUserName + ":" + dataPrepperPassword;
+                    final byte[] encodedAuth = Base64.getEncoder().encode(auth.getBytes(StandardCharsets.ISO_8859_1));
+                    final String authHeader = "Basic " + new String(encodedAuth, StandardCharsets.ISO_8859_1);
+                    httpPost.setHeader(HttpHeaders.AUTHORIZATION, authHeader);
+                }
+
                 AccessController.doPrivileged((PrivilegedAction<Boolean>) () -> {
                     try (CloseableHttpResponse response = httpClient.execute(httpPost)) {
                         final int status = response.getStatusLine().getStatusCode();

src/main/java/org/opensearch/ubi/UbiSettings.java

@@ -10,7 +10,6 @@
 
 import org.opensearch.common.settings.Setting;
 
-import java.util.Collections;
 import java.util.List;
 
 /**
@@ -23,18 +22,40 @@ public class UbiSettings {
      */
     public static final String DATA_PREPPER_URL = "ubi.dataprepper.url";
 
+    /**
+     * The optional username for Data Prepper's http_source.
+     */
+    public static final String DATA_PREPPER_AUTH_USERNAME = "ubi.dataprepper.auth.username";
+
+    /**
+     * The optional password for Data Prepper's http_source.
+     */
+    public static final String DATA_PREPPER_AUTH_PASSWORD = "ubi.dataprepper.auth.password";
+
     private static final Setting<String> DATA_PREPPER_URL_SETTING = Setting.simpleString(
             DATA_PREPPER_URL,
             Setting.Property.Dynamic,
             Setting.Property.NodeScope);
 
+    private static final Setting<String> DATA_PREPPER_AUTH_USERNAME_SETTING = Setting.simpleString(
+            DATA_PREPPER_AUTH_USERNAME,
+            Setting.Property.Dynamic,
+            Setting.Property.NodeScope);
+
+    private static final Setting<String> DATA_PREPPER_AUTH_PASSWORD_PASSWORD = Setting.simpleString(
+            DATA_PREPPER_AUTH_PASSWORD,
+            Setting.Property.Dynamic,
+            Setting.Property.NodeScope);
+
     /**
      * Gets a list of the UBI plugin settings.
      * @return A list of the UBI plugin settings.
      */
     public static List<Setting<?>> getSettings() {
-        return Collections.singletonList(
-                DATA_PREPPER_URL_SETTING
+        return List.of(
+                DATA_PREPPER_URL_SETTING,
+                DATA_PREPPER_AUTH_USERNAME_SETTING,
+                DATA_PREPPER_AUTH_PASSWORD_PASSWORD
         );
     }