[Backport 2.x] enabled mockTelemetryPlugin for IT and fixed OOM (#13054)

CHANGELOG.md

@@ -50,6 +50,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 ### Fixed
 - Fix bulk API ignores ingest pipeline for upsert ([#12883](https://github.com/opensearch-project/OpenSearch/pull/12883))
 - Fix issue with feature flags where default value may not be honored ([#12849](https://github.com/opensearch-project/OpenSearch/pull/12849))
+- Enabled mockTelemetryPlugin for IT and fixed OOM issues ([#13054](https://github.com/opensearch-project/OpenSearch/pull/13054))
 - Fix implement mark() and markSupported() in class FilterStreamInput ([#13098](https://github.com/opensearch-project/OpenSearch/pull/13098))
 - Fix snapshot _status API to return correct status for partial snapshots ([#13260](https://github.com/opensearch-project/OpenSearch/pull/13260))
 

TESTING.md

@@ -83,6 +83,7 @@ This will instruct all JVMs (including any that run cli tools such as creating t
 -   In order to remotely attach a debugger to the process: `--debug-jvm`
 -   In order to set a different keystore password: `--keystore-password yourpassword`
 -   In order to set an OpenSearch setting, provide a setting with the following prefix: `-Dtests.opensearch.`
+-   In order to enable stack trace of the MockSpanData during testing, add: `-Dtests.telemetry.span.stack_traces=true` (Storing stack traces alongside span data can be useful for comprehensive debugging and performance optimization during testing, as it provides insights into the exact code paths and execution sequences, facilitating efficient issue identification and resolution. Note: Enabling this might lead to OOM issues while running ITs)
 
 ## Test case filtering
 

test/framework/src/main/java/org/opensearch/test/OpenSearchIntegTestCase.java

@@ -2112,8 +2112,7 @@ protected boolean addMockGeoShapeFieldMapper() {
      * @return boolean.
      */
     protected boolean addMockTelemetryPlugin() {
-        // setting to false until https://github.com/opensearch-project/OpenSearch/issues/12615 is resolved
-        return false;
+        return true;
     }
 
     /**

test/telemetry/src/main/java/org/opensearch/test/telemetry/tracing/StrictCheckSpanProcessor.java

@@ -8,6 +8,7 @@
 
 package org.opensearch.test.telemetry.tracing;
 
+import org.opensearch.common.Booleans;
 import org.opensearch.telemetry.tracing.Span;
 import org.opensearch.test.telemetry.tracing.validators.AllSpansAreEndedProperly;
 import org.opensearch.test.telemetry.tracing.validators.AllSpansHaveUniqueId;
@@ -29,6 +30,14 @@ public StrictCheckSpanProcessor() {}
 
     private static Map<String, MockSpanData> spanMap = new ConcurrentHashMap<>();
 
+    // If you want to see the stack trace for each spanData, then
+    // update the flag to true or set the corresponding system property to true
+    // This is helpful in debugging the tests. Default value is false.
+    // Note: Enabling this might lead to OOM issues while running ITs.
+    private static final boolean isStackTraceForSpanEnabled = Booleans.parseBoolean(
+        System.getProperty("tests.telemetry.span.stack_traces", "false")
+    );
+
     @Override
     public void onStart(Span span) {
         spanMap.put(span.getSpanId(), toMockSpanData(span));
@@ -53,14 +62,15 @@ public List<MockSpanData> getFinishedSpanItems() {
 
     private MockSpanData toMockSpanData(Span span) {
         String parentSpanId = (span.getParentSpan() != null) ? span.getParentSpan().getSpanId() : "";
+        StackTraceElement[] stackTrace = isStackTraceForSpanEnabled ? Thread.currentThread().getStackTrace() : null;
         MockSpanData spanData = new MockSpanData(
             span.getSpanId(),
             parentSpanId,
             span.getTraceId(),
             System.nanoTime(),
             false,
             span.getSpanName(),
-            Thread.currentThread().getStackTrace(),
+            stackTrace,
             (span instanceof MockSpan) ? ((MockSpan) span).getAttributes() : Map.of()
         );
         return spanData;