Extract Otel dependencies to a separate module

Signed-off-by: suranjay <surajkumar.tu@gmail.com>

CHANGELOG.md

@@ -41,6 +41,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - Bump `com.netflix.nebula:nebula-publishing-plugin` from 19.2.0 to 20.3.0
 - Bump `com.diffplug.spotless` from 6.17.0 to 6.18.0
 - Bump `io.opencensus:opencensus-api` from 0.18.0 to 0.31.1 ([#7291](https://github.com/opensearch-project/OpenSearch/pull/7291))
+- Add Opentelemetry dependencies ([#7543](https://github.com/opensearch-project/OpenSearch/issues/7543))
 
 ### Changed
 - [CCR] Add getHistoryOperationsFromTranslog method to fetch the history snapshot from translogs ([#3948](https://github.com/opensearch-project/OpenSearch/pull/3948))
@@ -100,6 +101,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - Added @dbwiddis as on OpenSearch maintainer ([#7665](https://github.com/opensearch-project/OpenSearch/pull/7665))
 - [Extensions] Add ExtensionAwarePlugin extension point to add custom settings for extensions ([#7526](https://github.com/opensearch-project/OpenSearch/pull/7526))
 - Add new cluster setting to set default index replication type ([#7420](https://github.com/opensearch-project/OpenSearch/pull/7420))
+- Add distributed tracing framework ([#7543](https://github.com/opensearch-project/OpenSearch/issues/7543))
 
 ### Dependencies
 - Bump `com.netflix.nebula:gradle-info-plugin` from 12.0.0 to 12.1.3 (#7564)

buildSrc/version.properties

@@ -57,3 +57,6 @@ bytebuddy         = 1.14.3
 
 # benchmark dependencies
 jmh               = 1.35
+
+# opentelemetry dependencies
+opentelemetry    = 1.23.1

distribution/src/config/log4j2.properties

@@ -232,3 +232,22 @@ logger.task_detailslog_rolling.level = trace
 logger.task_detailslog_rolling.appenderRef.task_detailslog_rolling.ref = task_detailslog_rolling
 logger.task_detailslog_rolling.appenderRef.task_detailslog_rolling_old.ref = task_detailslog_rolling_old
 logger.task_detailslog_rolling.additivity = false
+######## Distributed Tracing logs JSON ####################
+appender.tracing_log_rolling.type = RollingFile
+appender.tracing_log_rolling.name = tracing_log_rolling
+appender.tracing_log_rolling.fileName = ${sys:opensearch.logs.base_path}${sys:file.separator}${sys:opensearch.logs.cluster_name}_tracing.trace
+appender.tracing_log_rolling.filePermissions = rw-r-----
+appender.tracing_log_rolling.layout.type = PatternLayout
+appender.tracing_log_rolling.layout.pattern = %m%n
+appender.tracing_log_rolling.filePattern = ${sys:opensearch.logs.base_path}${sys:file.separator}${sys:opensearch.logs.cluster_name}_tracing-%i.trace.gz
+appender.tracing_log_rolling.policies.type = Policies
+appender.tracing_log_rolling.policies.size.type = SizeBasedTriggeringPolicy
+appender.tracing_log_rolling.policies.size.size = 1GB
+appender.tracing_log_rolling.strategy.type = DefaultRolloverStrategy
+appender.tracing_log_rolling.strategy.max = 4
+#################################################
+logger.tracing_log.name = tracing_log
+logger.tracing_log.level = trace
+logger.tracing_log.appenderRef.tracing_log_rolling.ref = tracing_log_rolling
+logger.tracing_log.appenderRef.tracing_log_rolling_old.ref = tracing_log_rolling
+logger.tracing_log.additivity = false

modules/tracer-otel/build.gradle

@@ -0,0 +1,69 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ *
+ * Modifications Copyright OpenSearch Contributors. See
+ * GitHub history for details.
+ */
+
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import org.apache.tools.ant.taskdefs.condition.Os
+import org.opensearch.gradle.Architecture
+import org.opensearch.gradle.OS
+import org.opensearch.gradle.info.BuildParams
+
+opensearchplugin {
+  description 'Opentelemetry based Tracer implementation.'
+  classname 'org.opensearch.tracing.OTelTracerModulePlugin'
+  hasClientJar = true
+}
+
+dependencies {
+  api "io.opentelemetry:opentelemetry-api:${versions.opentelemetry}"
+  api "io.opentelemetry:opentelemetry-context:${versions.opentelemetry}"
+  api "io.opentelemetry:opentelemetry-sdk:${versions.opentelemetry}"
+  api "io.opentelemetry:opentelemetry-sdk-common:${versions.opentelemetry}"
+  api "io.opentelemetry:opentelemetry-sdk-trace:${versions.opentelemetry}"
+  api "io.opentelemetry:opentelemetry-sdk-metrics:${versions.opentelemetry}"
+  api "io.opentelemetry:opentelemetry-semconv:${versions.opentelemetry}-alpha"
+  api "io.opentelemetry:opentelemetry-sdk-logs:${versions.opentelemetry}-alpha"
+  api "io.opentelemetry:opentelemetry-api-logs:${versions.opentelemetry}-alpha"
+}
+
+
+thirdPartyAudit {
+  ignoreViolations(
+   'io.opentelemetry.internal.shaded.jctools.queues.MpscArrayQueueConsumerIndexField',
+   'io.opentelemetry.internal.shaded.jctools.queues.MpscArrayQueueProducerIndexField',
+   'io.opentelemetry.internal.shaded.jctools.queues.MpscArrayQueueProducerLimitField',
+   'io.opentelemetry.internal.shaded.jctools.util.UnsafeAccess',
+   'io.opentelemetry.internal.shaded.jctools.util.UnsafeRefArrayAccess'
+  )
+
+  ignoreMissingClasses(
+    'io.opentelemetry.api.events.EventEmitter',
+    'io.opentelemetry.api.events.EventEmitterBuilder',
+    'io.opentelemetry.api.events.EventEmitterProvider'
+  )
+}

server/src/main/java/org/opensearch/tracing/OSSpan.java → modules/tracer-otel/src/main/java/org/opensearch/tracing/DefaultSpan.java

@@ -11,17 +11,17 @@
 import io.opentelemetry.api.trace.SpanContext;
 
 /**
- * Default implementation of {@link Span}. It keeps a reference of OpenTelemetry Span and handles span
+ * Default implementation of {@link Span} using Otel span. It keeps a reference of OpenTelemetry Span and handles span
  * lifecycle management by delegating calls to it.
  */
-class OSSpan implements Span {
+class DefaultSpan implements Span {
 
     private final String spanName;
     private final io.opentelemetry.api.trace.Span otelSpan;
     private final Span parentSpan;
     private final Level level;
 
-    public OSSpan(String spanName, io.opentelemetry.api.trace.Span span, Span parentSpan, Level level) {
+    public DefaultSpan(String spanName, io.opentelemetry.api.trace.Span span, Span parentSpan, Level level) {
         this.spanName = spanName;
         this.otelSpan = span;
         this.parentSpan = parentSpan;

server/src/main/java/org/opensearch/tracing/DefaultTracer.java → modules/tracer-otel/src/main/java/org/opensearch/tracing/DefaultTracer.java

@@ -15,6 +15,7 @@
 import org.apache.logging.log4j.Logger;
 import org.opensearch.common.util.concurrent.ThreadContext;
 import org.opensearch.threadpool.ThreadPool;
+import org.opensearch.tracing.noop.NoopSpan;
 
 import java.io.Closeable;
 import java.io.IOException;
@@ -30,8 +31,6 @@
  */
 public class DefaultTracer implements Tracer {
 
-    public static final String CURRENT_SPAN = "current_span";
-
     private static final Logger logger = LogManager.getLogger(DefaultTracer.class);
     private static final String TRACE_ID = "trace_id";
     private static final String SPAN_ID = "span_id";
@@ -46,6 +45,13 @@ public class DefaultTracer implements Tracer {
     private final io.opentelemetry.api.trace.Tracer otelTracer;
     private final OpenTelemetry openTelemetry;
 
+    /**
+     * Creates DefaultTracer instance
+     *
+     * @param openTelemetry Otel global Opentelemetry instance
+     * @param threadPool Thread pool
+     * @param tracerSettings tracer related settings
+     */
     public DefaultTracer(OpenTelemetry openTelemetry, ThreadPool threadPool, TracerSettings tracerSettings) {
         this.openTelemetry = openTelemetry;
         this.otelTracer = openTelemetry.getTracer("os-tracer");
@@ -93,8 +99,8 @@ public void addAttribute(String key, boolean value) {
     @Override
     public void addEvent(String event) {
         Span currentSpan = getCurrentSpan();
-        if (currentSpan instanceof OSSpan && ((OSSpan) currentSpan).getOtelSpan() != null) {
-            ((OSSpan) currentSpan).getOtelSpan().addEvent(event);
+        if (currentSpan instanceof DefaultSpan && ((DefaultSpan) currentSpan).getOtelSpan() != null) {
+            ((DefaultSpan) currentSpan).getOtelSpan().addEvent(event);
         }
     }
 
@@ -119,7 +125,7 @@ private Span spanFromHeader() {
         Context context = TracerUtils.extractTracerContextFromHeader(threadPool.getThreadContext().getHeaders());
         if (context != null) {
             io.opentelemetry.api.trace.Span span = io.opentelemetry.api.trace.Span.fromContext(context);
-            return new OSSpan(ROOT_SPAN, span, null, Level.ROOT);
+            return new DefaultSpan(ROOT_SPAN, span, null, Level.ROOT);
         }
         return null;
     }
@@ -132,14 +138,14 @@ private Optional<Span> spanFromThreadContext() {
     }
 
     private Span createSpan(String spanName, Span parentSpan, Level level) {
-        return isLevelEnabled(level) ? createOSSpan(spanName, parentSpan, level) : createNoopSpan(spanName, parentSpan, level);
+        return isLevelEnabled(level) ? createDefaultSpan(spanName, parentSpan, level) : createNoopSpan(spanName, parentSpan, level);
     }
 
-    private Span createOSSpan(String spanName, Span parentSpan, Level level) {
-        OSSpan parentOSSpan = getLastValidSpanInChain(parentSpan);
-        io.opentelemetry.api.trace.Span otelSpan = createOtelSpan(spanName, parentOSSpan);
-        Span span = new OSSpan(spanName, otelSpan, parentSpan, level);
-        logger.debug(
+    private Span createDefaultSpan(String spanName, Span parentSpan, Level level) {
+        DefaultSpan parentDefaultSpan = getLastValidSpanInChain(parentSpan);
+        io.opentelemetry.api.trace.Span otelSpan = createOtelSpan(spanName, parentDefaultSpan);
+        Span span = new DefaultSpan(spanName, otelSpan, parentSpan, level);
+        logger.trace(
             "Starting OtelSpan spanId:{} name:{}: traceId:{}",
             otelSpan.getSpanContext().getSpanId(),
             span.getSpanName(),
@@ -149,22 +155,22 @@ private Span createOSSpan(String spanName, Span parentSpan, Level level) {
     }
 
     private NoopSpan createNoopSpan(String spanName, Span parentSpan, Level level) {
-        logger.debug("Starting Noop span name:{}", spanName);
+        logger.trace("Starting Noop span name:{}", spanName);
         return new NoopSpan(spanName, parentSpan, level);
     }
 
-    private OSSpan getLastValidSpanInChain(Span parentSpan) {
+    private DefaultSpan getLastValidSpanInChain(Span parentSpan) {
         while (parentSpan instanceof NoopSpan) {
             parentSpan = parentSpan.getParentSpan();
         }
-        return (OSSpan) parentSpan;
+        return (DefaultSpan) parentSpan;
     }
 
     // visible for testing
-    io.opentelemetry.api.trace.Span createOtelSpan(String spanName, OSSpan parentOSSpan) {
-        return parentOSSpan == null
+    io.opentelemetry.api.trace.Span createOtelSpan(String spanName, DefaultSpan parentDefaultSpan) {
+        return parentDefaultSpan == null
             ? otelTracer.spanBuilder(spanName).startSpan()
-            : otelTracer.spanBuilder(spanName).setParent(Context.current().with(parentOSSpan.getOtelSpan())).startSpan();
+            : otelTracer.spanBuilder(spanName).setParent(Context.current().with(parentDefaultSpan.getOtelSpan())).startSpan();
     }
 
     private boolean isLevelEnabled(Level level) {
@@ -186,42 +192,45 @@ private void setCurrentSpanInContext(Span span) {
     }
 
     private void endSpan(Span span) {
-        if (span instanceof OSSpan && ((OSSpan) span).getOtelSpan() != null) {
-            OSSpan osSpan = (OSSpan) span;
-            logger.debug(
+        if (span instanceof DefaultSpan && ((DefaultSpan) span).getOtelSpan() != null) {
+            DefaultSpan defaultSpan = (DefaultSpan) span;
+            logger.trace(
                 "Ending span spanId:{} name:{}: traceId:{}",
-                osSpan.getSpanContext().getSpanId(),
+                defaultSpan.getSpanContext().getSpanId(),
                 span.getSpanName(),
-                osSpan.getSpanContext().getTraceId()
+                defaultSpan.getSpanContext().getTraceId()
             );
-            osSpan.getOtelSpan().end();
+            defaultSpan.getOtelSpan().end();
         } else {
-            logger.debug("Ending noop span name:{}", span.getSpanName());
+            logger.trace("Ending noop span name:{}", span.getSpanName());
         }
     }
 
     private void setSpanAttributes(Span span) {
-        if (span instanceof OSSpan) {
-            addDefaultAttributes((OSSpan) span);
+        if (span instanceof DefaultSpan) {
+            addDefaultAttributes((DefaultSpan) span);
         }
     }
 
     private <T> void addSingleAttribute(AttributeKey<T> key, T value) {
         Span currentSpan = getCurrentSpan();
-        if (currentSpan instanceof OSSpan && ((OSSpan) currentSpan).getOtelSpan() != null) {
-            ((OSSpan) currentSpan).getOtelSpan().setAttribute(key, value);
+        if (currentSpan instanceof DefaultSpan && ((DefaultSpan) currentSpan).getOtelSpan() != null) {
+            ((DefaultSpan) currentSpan).getOtelSpan().setAttribute(key, value);
         }
     }
 
-    private void addDefaultAttributes(OSSpan osSpan) {
-        if (osSpan != null) {
-            addSingleAttribute(AttributeKey.stringKey(SPAN_ID), osSpan.getSpanContext().getSpanId());
-            addSingleAttribute(AttributeKey.stringKey(TRACE_ID), osSpan.getSpanContext().getTraceId());
-            addSingleAttribute(AttributeKey.stringKey(SPAN_NAME), osSpan.getSpanName());
+    private void addDefaultAttributes(DefaultSpan defaultSpan) {
+        if (defaultSpan != null) {
+            addSingleAttribute(AttributeKey.stringKey(SPAN_ID), defaultSpan.getSpanContext().getSpanId());
+            addSingleAttribute(AttributeKey.stringKey(TRACE_ID), defaultSpan.getSpanContext().getTraceId());
+            addSingleAttribute(AttributeKey.stringKey(SPAN_NAME), defaultSpan.getSpanName());
             addSingleAttribute(AttributeKey.stringKey(THREAD_NAME), Thread.currentThread().getName());
-            if (osSpan.getParentSpan() != null && osSpan.getParentSpan() instanceof OSSpan) {
-                addSingleAttribute(AttributeKey.stringKey(PARENT_SPAN_ID), ((OSSpan) osSpan.getParentSpan()).getSpanContext().getSpanId());
-                addSingleAttribute(AttributeKey.stringKey(PARENT_SPAN_NAME), osSpan.getParentSpan().getSpanName());
+            if (defaultSpan.getParentSpan() != null && defaultSpan.getParentSpan() instanceof DefaultSpan) {
+                addSingleAttribute(
+                    AttributeKey.stringKey(PARENT_SPAN_ID),
+                    ((DefaultSpan) defaultSpan.getParentSpan()).getSpanContext().getSpanId()
+                );
+                addSingleAttribute(AttributeKey.stringKey(PARENT_SPAN_NAME), defaultSpan.getParentSpan().getSpanName());
             }
         }
     }

server/src/main/java/org/opensearch/tracing/OTelResourceProvider.java → modules/tracer-otel/src/main/java/org/opensearch/tracing/OTelResourceProvider.java

@@ -12,12 +12,12 @@
 import io.opentelemetry.api.common.Attributes;
 import io.opentelemetry.api.trace.propagation.W3CTraceContextPropagator;
 import io.opentelemetry.context.propagation.ContextPropagators;
-import io.opentelemetry.exporter.otlp.trace.OtlpGrpcSpanExporter;
 import io.opentelemetry.sdk.OpenTelemetrySdk;
 import io.opentelemetry.sdk.resources.Resource;
 import io.opentelemetry.sdk.trace.SdkTracerProvider;
 import io.opentelemetry.sdk.trace.export.BatchSpanProcessor;
 import io.opentelemetry.semconv.resource.attributes.ResourceAttributes;
+import org.opensearch.tracing.exporter.FileSpanExporter;
 
 import java.util.concurrent.TimeUnit;
 
@@ -26,20 +26,19 @@
  */
 class OTelResourceProvider {
 
-    private static final Resource resource;
     private static final ContextPropagators contextPropagators;
     private static volatile OpenTelemetry OPEN_TELEMETRY;
 
     static {
-        resource = Resource.getDefault().merge(Resource.create(Attributes.of(ResourceAttributes.SERVICE_NAME, "OpenSearch")));
         contextPropagators = ContextPropagators.create(W3CTraceContextPropagator.getInstance());
     }
 
     static OpenTelemetry getOrCreateOpenTelemetryInstance(TracerSettings tracerSettings) {
         if (OPEN_TELEMETRY == null) {
+            Resource resource = Resource.create(Attributes.of(ResourceAttributes.SERVICE_NAME, "OpenSearch"));
             SdkTracerProvider sdkTracerProvider = SdkTracerProvider.builder()
                 .addSpanProcessor(
-                    BatchSpanProcessor.builder(OtlpGrpcSpanExporter.builder().setTimeout(10, TimeUnit.SECONDS).build())
+                    BatchSpanProcessor.builder(new FileSpanExporter())
                         .setScheduleDelay(tracerSettings.getExporterDelay().getSeconds(), TimeUnit.SECONDS)
                         .setMaxExportBatchSize(tracerSettings.getExporterBatchSize())
                         .setMaxQueueSize(tracerSettings.getExporterMaxQueueSize())

modules/tracer-otel/src/main/java/org/opensearch/tracing/OTelTracerModulePlugin.java

@@ -0,0 +1,55 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.tracing;
+
+import io.opentelemetry.api.OpenTelemetry;
+import org.opensearch.common.settings.Settings;
+import org.opensearch.plugins.Plugin;
+import org.opensearch.plugins.TracerPlugin;
+import org.opensearch.threadpool.ThreadPool;
+
+import java.util.Collections;
+import java.util.Map;
+import java.util.function.Supplier;
+
+/**
+ * Tracer plugin based on Otel
+ */
+public class OTelTracerModulePlugin extends Plugin implements TracerPlugin {
+
+    private static final String OTEL_TRACER_NAME = "otel";
+
+    /**
+     * No-args constructor
+     */
+    public OTelTracerModulePlugin() {}
+
+    @Override
+    public Settings additionalSettings() {
+        return Settings.builder()
+            // set Otel tracer as default tracer
+            .put(TracerModule.TRACER_DEFAULT_TYPE_SETTING.getKey(), OTEL_TRACER_NAME)
+            .build();
+    }
+
+    @Override
+    public Map<String, Supplier<Tracer>> getTracers(ThreadPool threadPool, TracerSettings tracerSettings) {
+        return Collections.singletonMap(OTEL_TRACER_NAME, () -> createDefaultTracer(threadPool, tracerSettings));
+    }
+
+    @Override
+    public Map<String, TracerHeaderInjector> getHeaderInjectors() {
+        return Collections.singletonMap(OTEL_TRACER_NAME, new OtelTracerHeaderInjector());
+    }
+
+    private Tracer createDefaultTracer(ThreadPool threadPool, TracerSettings tracerSettings) {
+        OpenTelemetry openTelemetry = OTelResourceProvider.getOrCreateOpenTelemetryInstance(tracerSettings);
+        return new DefaultTracer(openTelemetry, threadPool, tracerSettings);
+    }
+}