Phase 1 OpenTelemetry Proof of Concept

* configure feature for OpenTelemetry support
* feature gated request fields for traceparent and tracestate headers
* feature gated triggers to OpenTelemetry state machine
* feature gated logic to intialize OpenTelemetry state structure
* temporary call to initialize OpenTelemetry tracer
* temporary logging callback
* OpenTelemetry state machine to manage calls into rust crate
* incoming header/field parsing for traceparent and tracestate
* Rust crate holding all the tracer logic.

Co-authored-by: Gabor Javorszky <g.javorszky@f5.com>
Signed-off-by: Ava Hahn <a.hahn@f5.com>

auto/help

@@ -52,6 +52,8 @@ cat << END
 
   --njs                enable njs library usage
 
+  --otel               enable otel library usage
+
   --debug              enable debug logging
 
   --fuzz=ENGINE        enable fuzz testing

auto/options

@@ -27,6 +27,7 @@ NXT_CYASSL=NO
 NXT_POLARSSL=NO
 
 NXT_NJS=NO
+NXT_OTEL=NO
 
 NXT_TEST_BUILD_EPOLL=NO
 NXT_TEST_BUILD_EVENTPORT=NO
@@ -112,6 +113,7 @@ do
         --polarssl)                      NXT_POLARSSL=YES                    ;;
 
         --njs)                           NXT_NJS=YES                         ;;
+        --otel)                          NXT_OTEL=YES                        ;;
 
         --test-build-epoll)              NXT_TEST_BUILD_EPOLL=YES            ;;
         --test-build-eventport)          NXT_TEST_BUILD_EVENTPORT=YES        ;;

auto/otel

@@ -0,0 +1,32 @@
+
+# Copyright (C) NGINX, Inc.
+
+
+nxt_found=no
+NXT_HAVE_OTEL=NO
+
+# Temporary build step.
+# later we will refactor the otel rust crate into
+# its own (open source) module
+
+cd src/otel
+cargo build
+cd ../../
+
+
+nxt_feature="otel"
+nxt_feature_name=NXT_HAVE_OTEL
+nxt_feature_run=no
+nxt_feature_libs=""
+nxt_feature_test="int main(void){return 0;}"
+
+. auto/feature
+
+if [ $nxt_found = no ]; then
+    $echo
+    $echo $0: error: no OpenTelemetry library found.
+    $echo
+    exit 1;
+fi
+
+NXT_LIB_AUX_LIBS="$NXT_LIB_AUX_LIBS -lssl -lcrypto src/otel/target/debug/libotel.a"

auto/sources

@@ -127,6 +127,10 @@ if [ "$NXT_NJS" != "NO" ]; then
     NXT_LIB_SRCS="$NXT_LIB_SRCS src/nxt_js.c src/nxt_http_js.c src/nxt_script.c"
 fi
 
+if [ "$NXT_OTEL" != "NO" ]; then
+    NXT_LIB_SRCS="$NXT_LIB_SRCS src/nxt_otel.c"
+fi
+
 NXT_LIB_EPOLL_SRCS="src/nxt_epoll_engine.c"
 NXT_LIB_KQUEUE_SRCS="src/nxt_kqueue_engine.c"
 NXT_LIB_EVENTPORT_SRCS="src/nxt_eventport_engine.c"

auto/summary

@@ -30,6 +30,7 @@ Unit configuration summary:
   TLS support: ............... $NXT_OPENSSL
   Regex support: ............. $NXT_REGEX
   njs support: ............... $NXT_NJS
+  otel support: .............. $NXT_OTEL
 
   process isolation: ......... $NXT_ISOLATION
   cgroupv2: .................. $NXT_HAVE_CGROUP

configure

@@ -178,6 +178,10 @@ if [ $NXT_NJS != NO ]; then
     . auto/njs
 fi
 
+if [ $NXT_OTEL != NO ]; then
+    . auto/otel
+fi
+
 . auto/make
 . auto/fuzzing
 . auto/summary

src/nxt_h1proto.c

@@ -178,6 +178,10 @@ static nxt_http_field_proc_t           nxt_h1p_fields[] = {
     { nxt_string("Content-Length"),    &nxt_http_request_content_length, 0 },
     { nxt_string("Authorization"),     &nxt_http_request_field,
         offsetof(nxt_http_request_t, authorization) },
+#if (NXT_HAVE_OTEL)
+    { nxt_string("Traceparent"),       &nxt_otel_parse_traceparent, 0 },
+    { nxt_string("Tracestate"),        &nxt_otel_parse_tracestate,  0 },
+#endif
 };
 
 
@@ -518,6 +522,10 @@ nxt_h1p_conn_request_init(nxt_task_t *task, void *obj, void *data)
             h1p->parser.discard_unsafe_fields = skcf->discard_unsafe_fields;
 
             nxt_h1p_conn_request_header_parse(task, c, h1p);
+#if (NXT_HAVE_OTEL)
+            nxt_otel_test_and_call_state(task, r);
+#endif
+
             return;
         }
 
@@ -1685,6 +1693,11 @@ nxt_h1p_request_discard(nxt_task_t *task, nxt_http_request_t *r,
 
     nxt_sendbuf_drain(task, wq, b);
     nxt_sendbuf_drain(task, wq, last);
+
+#if (NXT_HAVE_OTEL)
+    // TODO Phase 2: Test
+    nxt_otel_test_and_call_state(task, r);
+#endif
 }
 
 
@@ -1824,6 +1837,11 @@ nxt_h1p_conn_sent(nxt_task_t *task, void *obj, void *data)
 {
     nxt_conn_t          *c;
     nxt_event_engine_t  *engine;
+#if (NXT_HAVE_OTEL)
+    nxt_http_request_t  *r;
+    r = ((nxt_h1proto_t *) data)->request;
+    nxt_otel_test_and_call_state(task, r);
+#endif
 
     c = obj;
 

src/nxt_http.h

@@ -8,6 +8,7 @@
 #define _NXT_HTTP_H_INCLUDED_
 
 #include <nxt_regex.h>
+#include <nxt_otel.h>
 
 
 typedef enum {
@@ -190,6 +191,10 @@ struct nxt_http_request_s {
 
     nxt_http_response_t             resp;
 
+#if (NXT_HAVE_OTEL)
+    nxt_otel_state_t                *otel;
+#endif
+
     nxt_http_status_t               status:16;
 
     uint8_t                         log_route;    /* 1 bit */

src/nxt_http_request.c

@@ -285,7 +285,14 @@ nxt_http_request_create(nxt_task_t *task)
     task->thread->engine->requests_cnt++;
 
     r->tstr_cache.var.pool = mp;
-
+#if (NXT_HAVE_OTEL)
+    r->otel = nxt_mp_zget(r->mem_pool, sizeof(nxt_otel_state_t));
+    if (!r->otel) {
+        goto fail;
+    }
+    // TODO: detect and only set if otel is configured
+    r->otel->status = NXT_OTEL_INIT_STATE;
+#endif
     return r;
 
 fail:
@@ -313,6 +320,10 @@ nxt_http_request_start(nxt_task_t *task, void *obj, void *data)
 
     r = obj;
 
+#if (NXT_HAVE_OTEL)
+    nxt_otel_test_and_call_state(task, r);
+#endif
+
     r->state = &nxt_http_request_body_state;
 
     skcf = r->conf->socket_conf;
@@ -584,6 +595,10 @@ nxt_http_request_ready(nxt_task_t *task, void *obj, void *data)
     r = obj;
     action = r->conf->socket_conf->action;
 
+#if (NXT_HAVE_OTEL)
+    nxt_otel_test_and_call_state(task, r);
+#endif
+
     if (r->chunked) {
         ret = nxt_http_request_chunked_transform(r);
         if (nxt_slow_path(ret != NXT_OK)) {
@@ -852,6 +867,11 @@ nxt_http_request_error_handler(nxt_task_t *task, void *obj, void *data)
 
     r->error = 1;
 
+#if (NXT_HAVE_OTEL)
+    // TODO
+    // PHASE 2: add error event to span
+#endif
+
     if (nxt_fast_path(proto.any != NULL)) {
         nxt_http_proto[r->protocol].discard(task, r, nxt_http_buf_last(r));
     }

src/nxt_main.c

@@ -5,12 +5,17 @@
  */
 
 #include <nxt_main.h>
+#include <nxt_otel.h>
 #include <nxt_runtime.h>
 
 
 extern char  **environ;
 
 
+void otel_phase1_log_callback(u_char *arg) {
+    printf("otel: %s", (char *) arg);
+}
+
 int nxt_cdecl
 main(int argc, char **argv)
 {
@@ -24,6 +29,9 @@ main(int argc, char **argv)
 
     nxt_main_log.handler = nxt_log_time_handler;
 
+#if (NXT_HAVE_OTEL)
+    nxt_otel_init(&otel_phase1_log_callback);
+#endif
     ret = nxt_runtime_create(&nxt_main_task);
 
     if (ret != NXT_OK) {

src/nxt_main.h

@@ -154,6 +154,8 @@ typedef void (*nxt_event_conn_handler_t)(nxt_thread_t *thr, nxt_conn_t *c);
 #include <nxt_runtime.h>
 #include <nxt_port_hash.h>
 
+// remove after phase 1
+NXT_EXPORT void otel_phase1_log_callback(u_char *arg);
 
 /*
  * The envp argument must be &environ if application may