Add support for grpc propagation (#67)

* Fix path to flask app

* Add support for gRPC propagation

* Add test for gRPC propagation

.circleci/config.yml

@@ -13,6 +13,8 @@ jobs:
             pip3.5 install setuptools
             pip3.5 install docker
             pip3.5 install docker-compose
+            pip3.5 install protobuf
+            pip3.5 install grpcio
             sudo mkdir /test-log
             sudo chmod a+rwx /test-log
             export LOG_DIR=/test-log

Dockerfile-test

@@ -3,8 +3,6 @@ FROM ubuntu:18.04
 ARG OPENTRACING_CPP_VERSION=v1.5.0
 ARG NGINX_VERSION=1.13.12
 
-COPY . /src
-
 RUN set -x \
   && apt-get update \
   && apt-get install --no-install-recommends --no-install-suggests -y \
@@ -25,9 +23,10 @@ RUN set -x \
               g++-8 \
 ### Use gcc-8 (the default gcc has this problem when using with address sanitizer:
 ### https://gcc.gnu.org/bugzilla/show_bug.cgi?id=84428)
-  && update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-8 800 --slave /usr/bin/g++ g++ /usr/bin/g++-8 \
+  && update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-8 800 --slave /usr/bin/g++ g++ /usr/bin/g++-8
+
 ### Build opentracing-cpp
-  && cd / \
+RUN cd / \
   && git clone -b $OPENTRACING_CPP_VERSION https://github.com/opentracing/opentracing-cpp.git \
   && cd opentracing-cpp \
   && mkdir .build && cd .build \
@@ -36,12 +35,16 @@ RUN set -x \
            -DCMAKE_SHARED_LINKER_FLAGS="-fno-omit-frame-pointer -fsanitize=address" \
            -DCMAKE_EXE_LINKER_FLAGS="-fno-omit-frame-pointer -fsanitize=address" \
            -DBUILD_TESTING=OFF .. \
-  && make && make install \
+  && make && make install
+
+COPY ./opentracing /opentracing
+
 ### Build nginx
-  && cd /src \
+RUN apt-get install -y libssl-dev
+RUN cd / \
   && wget -O nginx-release-${NGINX_VERSION}.tar.gz https://github.com/nginx/nginx/archive/release-${NGINX_VERSION}.tar.gz \
   && tar zxf nginx-release-${NGINX_VERSION}.tar.gz \
-  && cd /src/nginx-release-${NGINX_VERSION} \
+  && cd /nginx-release-${NGINX_VERSION} \
   # Temporarily disable leak sanitizer to get around false positives in build
   && export ASAN_OPTIONS=detect_leaks=0 \
   && export CFLAGS="-Wno-error" \
@@ -50,6 +53,9 @@ RUN set -x \
       --with-debug \
       --with-cc-opt="-O1 -g -fno-omit-frame-pointer -fsanitize=address" \
       --with-ld-opt="-g -fno-omit-frame-pointer -fsanitize=address" \
-      --add-dynamic-module=/src/opentracing \
+      # enables grpc
+      --with-http_v2_module \
+      --add-dynamic-module=/opentracing \
   && make && make install
+
 CMD ["/usr/local/nginx/sbin/nginx", "-g", "daemon off;"]

ci/do_ci.sh

@@ -11,7 +11,10 @@ if [[ "$1" == "system.testing" ]]; then
   docker build -t nginx-opentracing-test/nginx -f Dockerfile-test .
   cd test
   docker build -t nginx-opentracing-test/backend -f Dockerfile-backend .
-  python3 nginx_opentracing_test.py
+  cd environment/grpc
+  docker build -t nginx-opentracing-test/grpc-backend .
+  cd -
+  PYTHONPATH=environment/grpc python3 nginx_opentracing_test.py
   exit 0
 elif [[ "$1" == "module.binaries" ]]; then
   mkdir -p "${BUILD_DIR}"

doc/Reference.md

@@ -35,6 +35,14 @@ Propagates the active span context for upstream requests. (See
 Propagates the active span context for FastCGI requests. (See
 [cross-process-tracing](http://opentracing.io/documentation/pages/api/cross-process-tracing.html))
 
+### `opentracing_grpc_propagate_context`
+
+- **syntax** `opentracing_grpc_propagate_context`
+- **context**: `http`, `server`, `location`
+
+Propagates the active span context for gRPC requests. (See
+[cross-process-tracing](http://opentracing.io/documentation/pages/api/cross-process-tracing.html))
+
 ### `opentracing_trace_locations`
 
 - **syntax** `opentracing_trace_locations on|off`

opentracing/src/ngx_http_opentracing_module.cpp

@@ -223,6 +223,10 @@ static ngx_command_t opentracing_commands[] = {
          NGX_CONF_NOARGS,
      propagate_fastcgi_opentracing_context, NGX_HTTP_LOC_CONF_OFFSET, 0,
      nullptr},
+    {ngx_string("opentracing_grpc_propagate_context"),
+     NGX_HTTP_MAIN_CONF | NGX_HTTP_SRV_CONF | NGX_HTTP_LOC_CONF |
+         NGX_CONF_NOARGS,
+     propagate_grpc_opentracing_context, NGX_HTTP_LOC_CONF_OFFSET, 0, nullptr},
     {ngx_string("opentracing_operation_name"),
      NGX_HTTP_MAIN_CONF | NGX_HTTP_SRV_CONF | NGX_HTTP_LOC_CONF |
          NGX_CONF_TAKE1,

opentracing/src/opentracing_directive.cpp

@@ -208,6 +208,52 @@ char *propagate_fastcgi_opentracing_context(ngx_conf_t *cf,
   return static_cast<char *>(NGX_CONF_ERROR);
 }
 
+//------------------------------------------------------------------------------
+// propagate_grpc_opentracing_context
+//------------------------------------------------------------------------------
+char *propagate_grpc_opentracing_context(ngx_conf_t *cf, ngx_command_t *command,
+                                         void *conf) noexcept try {
+  auto main_conf = static_cast<opentracing_main_conf_t *>(
+      ngx_http_conf_get_module_main_conf(cf, ngx_http_opentracing_module));
+  if (!main_conf->tracer_library.data) {
+    ngx_log_error(NGX_LOG_ERR, cf->log, 0,
+                  "opentracing_grpc_propagate_context before tracer loaded");
+    return static_cast<char *>(NGX_CONF_ERROR);
+  }
+  if (main_conf->span_context_keys == nullptr) {
+    return static_cast<char *>(NGX_CONF_OK);
+  }
+  auto keys = static_cast<opentracing::string_view *>(
+      main_conf->span_context_keys->elts);
+  auto num_keys = static_cast<int>(main_conf->span_context_keys->nelts);
+
+  auto old_args = cf->args;
+
+  ngx_str_t args[] = {ngx_string("grpc_set_header"), {}, {}};
+  ngx_array_t args_array;
+  args_array.elts = static_cast<void *>(&args);
+  args_array.nelts = 3;
+
+  cf->args = &args_array;
+  for (int key_index = 0; key_index < num_keys; ++key_index) {
+    args[1] = ngx_str_t{keys[key_index].size(),
+                        reinterpret_cast<unsigned char *>(
+                            const_cast<char *>(keys[key_index].data()))};
+    args[2] = make_span_context_value_variable(cf->pool, keys[key_index]);
+    auto rcode = opentracing_conf_handler(cf, 0);
+    if (rcode != NGX_OK) {
+      cf->args = old_args;
+      return static_cast<char *>(NGX_CONF_ERROR);
+    }
+  }
+  cf->args = old_args;
+  return static_cast<char *>(NGX_CONF_OK);
+} catch (const std::exception &e) {
+  ngx_log_error(NGX_LOG_ERR, cf->log, 0,
+                "opentracing_grpc_propagate_context failed: %s", e.what());
+  return static_cast<char *>(NGX_CONF_ERROR);
+}
+
 //------------------------------------------------------------------------------
 // set_opentracing_tag
 //------------------------------------------------------------------------------

opentracing/src/opentracing_directive.h

@@ -21,6 +21,12 @@ char *propagate_fastcgi_opentracing_context(ngx_conf_t *cf,
                                             ngx_command_t *command,
                                             void *conf) noexcept;
 
+//------------------------------------------------------------------------------
+// propagate_grpc_opentracing_context
+//------------------------------------------------------------------------------
+char *propagate_grpc_opentracing_context(ngx_conf_t *cf, ngx_command_t *command,
+                                         void *conf) noexcept;
+
 //------------------------------------------------------------------------------
 // add_opentracing_tag
 //------------------------------------------------------------------------------

test/Dockerfile-backend

@@ -5,4 +5,4 @@ COPY . /app
 WORKDIR /app
 RUN pip install -r requirements.txt
 ENTRYPOINT ["python"]
-CMD ["app.py"]
+CMD ["environment/app.py"]

test/environment/docker-compose.yaml

@@ -21,8 +21,10 @@ services:
       - LSAN_OPTIONS=suppressions=/lsan-suppress.txt
     expose:
       - "8080"
+      - "8081"
     ports:
       - "8080:8080"
+      - "8081:8081"
     command:
       - /usr/local/nginx/sbin/nginx
       - -g
@@ -55,6 +57,17 @@ services:
     ports:
       - "9000:9000"
 
+  grpc_backend:
+    image: nginx-opentracing-test/grpc-backend
+    networks:
+      testnet:
+        aliases:
+          - grpc-backend
+    expose:
+      - "50051"
+    ports:
+      - "50051:50051"
+
 networks:
   testnet: {}
 

test/environment/grpc/Dockerfile

@@ -0,0 +1,3 @@
+FROM grpc/python:1.13-onbuild
+ENTRYPOINT ["python", "-u"]
+CMD ["app.py"]

test/environment/grpc/app.proto

@@ -0,0 +1,12 @@
+// install grpcio-tools
+//
+// python3 -m grpc_tools.protoc --proto_path=. --python_out=. --grpc_python_out=. app.proto
+
+syntax = "proto3"; 
+
+service App { 
+  rpc CheckTraceHeader(Empty) returns (Empty); 
+}
+
+message Empty {
+}

test/environment/grpc/app.py

@@ -0,0 +1,32 @@
+from concurrent import futures
+import sys
+import signal
+import time
+import grpc
+import app_pb2 as app_messages
+import app_pb2_grpc as app_service
+
+_ONE_DAY_IN_SECONDS = 60 * 60 * 24
+
+class AppService(app_service.AppServicer):
+    def CheckTraceHeader(self, request, context):
+        metadata = dict(context.invocation_metadata())
+        if 'x-ot-span-context' not in metadata:
+            context.set_code(grpc.StatusCode.INTERNAL)
+            context.set_details("Metadatum x-ot-span-context not found")
+        return app_messages.Empty()
+
+def serve():
+    server = grpc.server(futures.ThreadPoolExecutor(max_workers=10))
+    app_service.add_AppServicer_to_server(AppService(), server)
+    server.add_insecure_port('0.0.0.0:50051')
+    server.start()
+    signal.signal(signal.SIGTERM, signal.getsignal(signal.SIGINT))
+    try:
+        while True:
+            time.sleep(_ONE_DAY_IN_SECONDS)
+    except KeyboardInterrupt:
+        server.stop(0)
+
+if __name__ == '__main__':
+    serve()

test/environment/grpc/app_pb2_grpc.py

@@ -0,0 +1,46 @@
+# Generated by the gRPC Python protocol compiler plugin. DO NOT EDIT!
+import grpc
+
+import app_pb2 as app__pb2
+
+
+class AppStub(object):
+  # missing associated documentation comment in .proto file
+  pass
+
+  def __init__(self, channel):
+    """Constructor.
+
+    Args:
+      channel: A grpc.Channel.
+    """
+    self.CheckTraceHeader = channel.unary_unary(
+        '/App/CheckTraceHeader',
+        request_serializer=app__pb2.Empty.SerializeToString,
+        response_deserializer=app__pb2.Empty.FromString,
+        )
+
+
+class AppServicer(object):
+  # missing associated documentation comment in .proto file
+  pass
+
+  def CheckTraceHeader(self, request, context):
+    # missing associated documentation comment in .proto file
+    pass
+    context.set_code(grpc.StatusCode.UNIMPLEMENTED)
+    context.set_details('Method not implemented!')
+    raise NotImplementedError('Method not implemented!')
+
+
+def add_AppServicer_to_server(servicer, server):
+  rpc_method_handlers = {
+      'CheckTraceHeader': grpc.unary_unary_rpc_method_handler(
+          servicer.CheckTraceHeader,
+          request_deserializer=app__pb2.Empty.FromString,
+          response_serializer=app__pb2.Empty.SerializeToString,
+      ),
+  }
+  generic_handler = grpc.method_handlers_generic_handler(
+      'App', rpc_method_handlers)
+  server.add_generic_rpc_handlers((generic_handler,))

test/environment/grpc/requirements.txt

@@ -0,0 +1 @@
+#grpcio-tools