Merge branch 'main' into analytics-domain-service-in-cdk

Signed-off-by: Mikayla Thompson <thomika@amazon.com>

TrafficCapture/captureKafkaOffloader/build.gradle

@@ -24,4 +24,13 @@ dependencies {
     testImplementation group: 'org.apache.logging.log4j', name: 'log4j-core', version: '2.20.0'
     testImplementation group: 'org.apache.logging.log4j', name: 'log4j-slf4j2-impl', version: '2.20.0'
     testImplementation group: 'org.slf4j', name: 'slf4j-api', version: '2.0.7'
+}
+
+
+configurations.all {
+    resolutionStrategy.eachDependency { DependencyResolveDetails details ->
+        if (details.requested.group == 'io.netty') {
+            details.useVersion '4.1.100.Final'
+        }
+    }
 }

TrafficCapture/dockerSolution/src/main/docker/migrationConsole/catIndices.sh

@@ -8,7 +8,7 @@ target_no_auth=false
 
 # Check for the presence of MIGRATION_DOMAIN_ENDPOINT environment variable
 if [ -n "$MIGRATION_DOMAIN_ENDPOINT" ]; then
-    target_endpoint="https://${MIGRATION_DOMAIN_ENDPOINT}:443"
+    target_endpoint="${MIGRATION_DOMAIN_ENDPOINT}"
     target_auth_user_and_pass="admin:Admin123!"
 else
     target_endpoint="https://opensearchtarget:9200"

TrafficCapture/dockerSolution/src/main/docker/otelcol/Dockerfile

@@ -1,5 +1,4 @@
 FROM public.ecr.aws/a0w2c5q7/otelcol-with-opensearch:amd-latest
 
 COPY ./otel-config-cdk.yml /etc/otel-config.yml
-# RUN apt-get update && apt-get install file -y
 ENTRYPOINT ["./otelcontribcol", "--config", "/etc/otel-config.yml"]

TrafficCapture/dockerSolution/src/main/docker/otelcol/otel-config-cdk.yml

@@ -31,7 +31,7 @@ exporters:
   opensearch:
     namespace: migrations
     http:
-      endpoint: "https://${ANALYTICS_DOMAIN_ENDPOINT}"
+      endpoint: "${ANALYTICS_DOMAIN_ENDPOINT}"
   logging:
     verbosity: detailed
   debug:

TrafficCapture/replayerPlugins/jsonMessageTransformers/jsonJMESPathMessageTransformer/build.gradle

@@ -0,0 +1,24 @@
+buildscript {
+    dependencies {
+        classpath 'org.junit.platform:junit-platform-gradle-plugin:1.0.1'
+    }
+}
+
+plugins {
+    id("io.freefair.lombok") version "8.0.1"
+}
+
+dependencies {
+    implementation project(':replayerPlugins:jsonMessageTransformers:jsonMessageTransformerInterface')
+
+    implementation group: 'io.burt', name: 'jmespath-core', version: '0.6.0'
+    implementation group: 'org.slf4j', name:"slf4j-api", version:"2.0.7"
+
+    testImplementation project(':trafficReplayer')
+    testImplementation group: 'org.junit.jupiter', name:'junit-jupiter-api', version:'5.9.3'
+    testRuntimeOnly group:'org.junit.jupiter', name:'junit-jupiter-engine', version:'5.x.x'
+}
+
+tasks.named('test') {
+    useJUnitPlatform()
+}

TrafficCapture/replayerPlugins/jsonMessageTransformers/jsonJMESPathMessageTransformer/src/main/java/org/opensearch/migrations/transform/JsonJMESPathTransformer.java

@@ -0,0 +1,26 @@
+package org.opensearch.migrations.transform;
+
+
+import io.burt.jmespath.BaseRuntime;
+import io.burt.jmespath.Expression;
+import lombok.extern.slf4j.Slf4j;
+
+import java.util.Map;
+
+
+@Slf4j
+public class JsonJMESPathTransformer implements IJsonTransformer {
+
+    Expression<Object> expression;
+
+    public JsonJMESPathTransformer(BaseRuntime<Object> runtime, String script) {
+        this.expression = runtime.compile(script);
+    }
+
+    @Override
+    public Map<String,Object> transformJson(Map<String,Object> incomingJson) {
+        var output = expression.search(incomingJson);
+        log.info("output="+output);
+        return (Map<String,Object>) output;
+    }
+}

TrafficCapture/replayerPlugins/jsonMessageTransformers/jsonJMESPathMessageTransformerProvider/build.gradle

@@ -0,0 +1,31 @@
+buildscript {
+    dependencies {
+        classpath 'org.junit.platform:junit-platform-gradle-plugin:1.0.1'
+    }
+}
+
+plugins {
+    id "io.freefair.lombok" version "8.0.1"
+}
+
+dependencies {
+    implementation project(':replayerPlugins:jsonMessageTransformers:jsonMessageTransformerInterface')
+    implementation project(':replayerPlugins:jsonMessageTransformers:jsonJMESPathMessageTransformer')
+
+    implementation group: 'io.burt', name: 'jmespath-core', version: '0.6.0'
+
+    testImplementation project(':trafficReplayer')
+    testImplementation testFixtures(project(path: ':testUtilities'))
+    testImplementation testFixtures(project(path: ':trafficReplayer'))
+
+    testImplementation group: 'com.fasterxml.jackson.core', name: 'jackson-databind', version: '2.15.0'
+    testImplementation group: 'io.netty', name: 'netty-all', version: '4.1.100.Final'
+    testImplementation group: 'org.junit.jupiter', name:'junit-jupiter-api', version:'5.9.3'
+    testImplementation group: 'org.junit.jupiter', name:'junit-jupiter-params', version:'5.9.3'
+    testImplementation group: 'org.slf4j', name: 'slf4j-api', version: '2.0.7'
+    testRuntimeOnly group:'org.junit.jupiter', name:'junit-jupiter-engine', version:'5.x.x'
+}
+
+tasks.named('test') {
+    useJUnitPlatform()
+}

TrafficCapture/replayerPlugins/jsonMessageTransformers/jsonJMESPathMessageTransformerProvider/src/main/java/org/opensearch/migrations/transform/JsonJMESPathTransformerProvider.java

@@ -0,0 +1,58 @@
+package org.opensearch.migrations.transform;
+
+import io.burt.jmespath.BaseRuntime;
+import io.burt.jmespath.jcf.JcfRuntime;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+import java.util.Map;
+import java.util.stream.Collectors;
+
+public class JsonJMESPathTransformerProvider implements IJsonTransformerProvider {
+
+    public static final String SCRIPT_KEY = "script";
+    private BaseRuntime<Object> adapterRuntime;
+
+    public JsonJMESPathTransformerProvider() {
+        this.adapterRuntime = new JcfRuntime();
+    }
+
+    @Override
+    public IJsonTransformer createTransformer(Object jsonConfig) {
+        var transformers = new ArrayList<JsonJMESPathTransformer>();
+        var configs = new ArrayList<Map<String,Object>>();
+        try {
+            if (jsonConfig instanceof Map) {
+                configs.add((Map<String, Object>) jsonConfig);
+            } else if (jsonConfig instanceof List) {
+                for (var c : (List) jsonConfig) {
+                    configs.add((Map<String, Object>) c);
+                }
+            } else {
+                throw new IllegalArgumentException(getConfigUsageStr());
+            }
+            for (var c : configs) {
+                if (c.size() != 1) {
+                    throw new IllegalArgumentException(getConfigUsageStr());
+                }
+                var scriptValue = c.get(SCRIPT_KEY);
+                if (!(scriptValue instanceof String)) {
+                    throw new IllegalArgumentException(getConfigUsageStr());
+                }
+                transformers.add(new JsonJMESPathTransformer(adapterRuntime, (String)scriptValue));
+            }
+        } catch (ClassCastException e) {
+            throw new IllegalArgumentException(getConfigUsageStr(), e);
+        }
+        return new JsonCompositeTransformer(transformers.toArray(IJsonTransformer[]::new));
+    }
+
+    private String getConfigUsageStr() {
+        return this.getClass().getName() + " expects the incoming configuration " +
+                "to be a Map<String,Object> or a List<Map<String,Object>>.  " +
+                "Each of the Maps should have one key-value of \"script\": \"...\".  " +
+                "Script values should be a fully-formed inlined JsonPath queries encoded as a json value.  " +
+                "All of the values within a configuration will be concatenated into one chained transformation.";
+    }
+}

TrafficCapture/replayerPlugins/jsonMessageTransformers/jsonJMESPathMessageTransformerProvider/src/main/resources/META-INF/services/org.opensearch.migrations.transform.IJsonTransformerProvider

@@ -0,0 +1 @@
+org.opensearch.migrations.transform.JsonJMESPathTransformerProvider

TrafficCapture/replayerPlugins/jsonMessageTransformers/jsonJMESPathMessageTransformerProvider/src/test/java/org/opensearch/migrations/replay/JsonTransformerTest.java

@@ -0,0 +1,105 @@
+package org.opensearch.migrations.replay;
+
+import com.fasterxml.jackson.core.JsonProcessingException;
+import com.fasterxml.jackson.core.type.TypeReference;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import io.burt.jmespath.jcf.JcfRuntime;
+import lombok.extern.slf4j.Slf4j;
+import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.Test;
+import org.opensearch.migrations.testutils.WrapWithNettyLeakDetection;
+import org.opensearch.migrations.transform.JsonJMESPathTransformer;
+
+import java.util.LinkedHashMap;
+import java.util.Map;
+
+@Slf4j
+@WrapWithNettyLeakDetection(disableLeakChecks = true)
+class JsonTransformerTest {
+    static final String TEST_INPUT_REQUEST = "{\n" +
+            "  \"method\": \"PUT\",\n" +
+            "  \"URI\": \"/oldStyleIndex\",\n" +
+            "  \"headers\": {\n" +
+            "    \"host\": \"127.0.0.1\"\n" +
+            "  },\n"+
+            "  \"payload\": {\n" +
+            "    \"inlinedJsonBody\": {\n" +
+            "      \"mappings\": {\n" +
+            "        \"oldType\": {\n" +
+            "          \"properties\": {\n" +
+            "            \"field1\": {\n" +
+            "              \"type\": \"text\"\n" +
+            "            },\n" +
+            "            \"field2\": {\n" +
+            "              \"type\": \"keyword\"\n" +
+            "            }\n" +
+            "          }\n" +
+            "        }\n" +
+            "      }\n" +
+            "    }\n" +
+            "  }\n" +
+            "}\n";
+
+    static final String EXCISE_TYPE_EXPRESSION_STRING = "{\n" +
+            "  \"method\": method,\n" +
+            "  \"URI\": URI,\n" +
+            "  \"headers\": headers,\n" +
+            "  \"payload\": {\n" +
+            "    \"inlinedJsonBody\": {\n" +
+            "      \"mappings\": payload.inlinedJsonBody.mappings.oldType\n" +
+            "    }\n" +
+            "  }\n" +
+            "}";
+
+    ObjectMapper mapper = new ObjectMapper();
+    static final TypeReference<LinkedHashMap<String, Object>> TYPE_REFERENCE_FOR_MAP_TYPE = new TypeReference<>(){};
+
+    public JsonTransformerTest() {
+        mapper.configure(com.fasterxml.jackson.core.JsonParser.Feature.ALLOW_COMMENTS, true);
+    }
+
+    static Map<String, Object> parseStringAsJson(ObjectMapper mapper, String jsonStr) throws JsonProcessingException {
+        return mapper.readValue(jsonStr, TYPE_REFERENCE_FOR_MAP_TYPE);
+    }
+
+    static String normalize(ObjectMapper mapper, String input) throws JsonProcessingException {
+        return mapper.writeValueAsString(mapper.readTree(input));
+    }
+
+    static String emitJson(ObjectMapper mapper, Object transformedDocument) throws JsonProcessingException {
+        mapper.configure(com.fasterxml.jackson.core.JsonParser.Feature.ALLOW_COMMENTS, true); //optional
+        return mapper.writeValueAsString(transformedDocument);
+    }
+
+    @Test
+    public void testSimpleTransform() throws JsonProcessingException {
+        var documentJson = parseStringAsJson(mapper, TEST_INPUT_REQUEST);
+        var transformer = new JsonJMESPathTransformer(new JcfRuntime(), EXCISE_TYPE_EXPRESSION_STRING);
+        var transformedDocument = transformer.transformJson(documentJson);
+        var outputStr = emitJson(mapper, transformedDocument);
+
+        final String TEST_OUTPUT_REQUEST = "{\n" +
+                "    \"method\": \"PUT\",\n" +
+                "    \"URI\": \"/oldStyleIndex\",\n" +
+                "  \"headers\": {\n" +
+                "    \"host\": \"127.0.0.1\"\n" +
+                "  },\n"+
+                "    \"payload\": {\n" +
+                "        \"inlinedJsonBody\": {\n" +
+                "            \"mappings\": {\n" +
+                "                \"properties\": {\n" +
+                "                    \"field1\": {\n" +
+                "                        \"type\": \"text\"\n" +
+                "                    },\n" +
+                "                    \"field2\": {\n" +
+                "                        \"type\": \"keyword\"\n" +
+                "                    }\n" +
+                "                }\n" +
+                "            }\n" +
+                "        }\n" +
+                "    }\n" +
+                "}";
+
+        Assertions.assertEquals(normalize(mapper, TEST_OUTPUT_REQUEST), normalize(mapper, outputStr));
+    }
+}

TrafficCapture/replayerPlugins/jsonMessageTransformers/jsonJMESPathMessageTransformerProvider/src/test/java/org/opensearch/migrations/replay/MultipleJMESPathScriptsTest.java

@@ -0,0 +1,70 @@
+package org.opensearch.migrations.replay;
+
+import com.fasterxml.jackson.core.type.TypeReference;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.Test;
+import org.opensearch.migrations.transform.JsonKeysForHttpMessage;
+
+import java.util.Map;
+import java.util.StringJoiner;
+
+public class MultipleJMESPathScriptsTest {
+    static final String HOSTNAME_SCRIPT_TO_INLINE = "{\n" +
+            "  \"method\": method,\n" +
+            "  \"URI\": URI,\n" +
+            "  \"headers\": {\"host\": \"localhost\"},\n" +
+            "  \"payload\": payload\n" +
+            "}";
+    private static final String EXCISE_SCRIPT =
+            "{\\\"method\\\": method,\\\"URI\\\": URI,\\\"headers\\\":headers,\\\"payload\\\":" +
+                    "{\\\"inlinedJsonBody\\\":{\\\"mappings\\\": payload.inlinedJsonBody.mappings.oldType}}}";
+    private static final String HOSTNAME_SCRIPT = "{\\n  \\\"method\\\": method,\\n  \\\"URI\\\": URI,\\n  " +
+            "\\\"headers\\\": {\\\"host\\\": \\\"localhost\\\"},\\n  \\\"payload\\\": payload\\n}";
+    private static final ObjectMapper mapper = new ObjectMapper();
+
+    private static Map<String,Object> parseAsMap(String contents) throws Exception {
+        return mapper.readValue(contents.getBytes(), new TypeReference<>() {});
+    }
+
+    @Test
+    public void testTwoScripts() throws Exception {
+        var aggregateScriptJoiner = new StringJoiner(",\n", "[", "]");
+        for (var script : new String[]{EXCISE_SCRIPT, HOSTNAME_SCRIPT}) {
+            aggregateScriptJoiner.add(
+                    "{\"JsonJMESPathTransformerProvider\": {" +
+                            " \"script\": \"" + script + "\"}}"
+            );
+        }
+
+        var aggregateScriptString = aggregateScriptJoiner.toString();
+        var toNewHostTransformer = new TransformationLoader().getTransformerFactoryLoader("localhost",
+                aggregateScriptString);
+        var origDoc = JsonTransformerTest.parseStringAsJson(mapper, JsonTransformerTest.TEST_INPUT_REQUEST);
+        var newDoc = toNewHostTransformer.transformJson(origDoc);
+
+        final String TEST_OUTPUT_REQUEST = "{\n" +
+                "    \"method\": \"PUT\",\n" +
+                "    \"URI\": \"/oldStyleIndex\",\n" +
+                "  \"headers\": {\n" +
+                "    \"host\": \"localhost\"\n" +
+                "  },\n"+
+                "    \"payload\": {\n" +
+                "        \"inlinedJsonBody\": {\n" +
+                "            \"mappings\": {\n" +
+                "                \"properties\": {\n" +
+                "                    \"field1\": {\n" +
+                "                        \"type\": \"text\"\n" +
+                "                    },\n" +
+                "                    \"field2\": {\n" +
+                "                        \"type\": \"keyword\"\n" +
+                "                    }\n" +
+                "                }\n" +
+                "            }\n" +
+                "        }\n" +
+                "    }\n" +
+                "}";
+        Assertions.assertEquals(JsonTransformerTest.normalize(mapper, TEST_OUTPUT_REQUEST),
+                JsonTransformerTest.emitJson(mapper, newDoc));
+    }
+}

TrafficCapture/replayerPlugins/jsonMessageTransformers/jsonJoltMessageTransformerProvider/src/test/java/org/opensearch/migrations/replay/JsonTransformerTest.java

@@ -33,7 +33,7 @@ private Map<String, Object> parseStringAsJson(String jsonStr) throws JsonProcess
 
     @SneakyThrows
     private Map<String, Object> parseSampleRequestFromResource(String path) {
-        try (InputStream inputStream = JsonJoltTransformBuilder.class.getResourceAsStream("/requests/"+path)) {
+        try (InputStream inputStream = JsonTransformerTest.class.getResourceAsStream("/requests/"+path)) {
             return mapper.readValue(inputStream, TYPE_REFERENCE_FOR_MAP_TYPE);
         }
     }