Fix context propagation

.circleci/config.yml

@@ -1,13 +1,34 @@
 version: 2
 jobs:
-  build:
+  system_testing:
     machine: true
     steps:
       - checkout
       - run:
-          name: Build nginx-opentracing-ci Docker image
-          command: docker build -t nginx-opentracing-ci ci
+          name: system.testing
+          command: |
+            pyenv global 3.5.2
+            python3 --version
+            pip3.5 --version
+            pip3.5 install setuptools
+            pip3.5 install docker
+            pip3.5 install docker-compose
+            sudo mkdir /test-log
+            sudo chmod a+rwx /test-log
+            export LOG_DIR=/test-log
+            ./ci/do_ci.sh system.testing
+      - store_artifacts:
+          path: /test-log
+  docker_image:
+    machine: true
+    steps:
+      - checkout
       - run:
-          name: Build and Test
-          command: ./ci/run_ci_docker.sh './ci/do_ci.sh'
+          command: docker build -t opentracing/nginx-opentracing .
 
+workflows:
+  version: 2
+  build_test_and_deploy:
+    jobs:
+      - system_testing
+      - docker_image

docker/Dockerfile → Dockerfile

@@ -2,12 +2,13 @@ ARG NGINX_LABEL=latest
 
 FROM nginx:${NGINX_LABEL}
 
-ARG OPENTRACING_CPP_VERSION=v1.2.0
-ARG ZIPKIN_CPP_VERSION=v0.2.0
-ARG LIGHTSTEP_VERSION=v0.6.1
-ARG JAEGER_CPP_VERSION=v0.2.0
+ARG OPENTRACING_CPP_VERSION=v1.4.0
+ARG ZIPKIN_CPP_VERSION=v0.3.1
+ARG LIGHTSTEP_VERSION=v0.7.0
+ARG JAEGER_CPP_VERSION=v0.4.1
 ARG GRPC_VERSION=v1.4.x
-ARG NGINX_OPENTRACING_VERSION=v0.2.1
+
+COPY . /src
 
 RUN set -x \
 # install nginx-opentracing package dependencies
@@ -59,16 +60,18 @@ RUN set -x \
   && cmake -DBUILD_SHARED_LIBS=1 -DCMAKE_BUILD_TYPE=Release -DBUILD_TESTING=OFF .. \
   && make && make install \
   && cd "$tempDir" \
+  && ln -s /usr/local/lib/libzipkin_opentracing.so /usr/local/lib/libzipkin_opentracing_plugin.so \
 ### Build Jaeger cpp-client
   && git clone -b $JAEGER_CPP_VERSION https://github.com/jaegertracing/cpp-client.git jaeger-cpp-client \
   && cd jaeger-cpp-client \
   && mkdir .build && cd .build \
   && cmake -DCMAKE_BUILD_TYPE=Release \
            -DBUILD_TESTING=OFF \
-           -DJAEGERTRACING_WITH_YAML_CPP=OFF .. \
+           -DJAEGERTRACING_WITH_YAML_CPP=ON .. \
   && make && make install \
   && export HUNTER_INSTALL_DIR=$(cat _3rdParty/Hunter/install-root-dir) \
   && cd "$tempDir" \
+  && ln -s /usr/local/lib/libjaegertracing.so /usr/local/lib/libjaegertracing_plugin.so \
 ### Build gRPC
   && git clone -b $GRPC_VERSION https://github.com/grpc/grpc \
   && cd grpc \
@@ -83,8 +86,8 @@ RUN set -x \
   && cmake -DBUILD_SHARED_LIBS=1 -DCMAKE_BUILD_TYPE=Release -DBUILD_TESTING=OFF .. \
   && make && make install \
   && cd "$tempDir" \
+  && ln -s /usr/local/lib/liblightstep_tracer.so /usr/local/lib/liblightstep_tracer_plugin.so \
 ### Build nginx-opentracing modules
-  && git clone -b $NGINX_OPENTRACING_VERSION https://github.com/opentracing-contrib/nginx-opentracing.git \
   && NGINX_VERSION=`nginx -v 2>&1` && NGINX_VERSION=${NGINX_VERSION#*nginx/} \
   && echo "deb-src http://nginx.org/packages/mainline/debian/ stretch nginx" >> /etc/apt/sources.list \
   && apt-get update \
@@ -95,18 +98,14 @@ RUN set -x \
   && NGINX_MODULES_PATH=$(nginx -V 2>&1 | grep -oP "modules-path=\K[^\s]*") \
   && auto/configure \
         --with-compat \
-        --add-dynamic-module=${tempDir}/nginx-opentracing/opentracing \
-        --add-dynamic-module=${tempDir}/nginx-opentracing/zipkin \
-        --add-dynamic-module=${tempDir}/nginx-opentracing/lightstep \
-        --add-dynamic-module=${tempDir}/nginx-opentracing/jaeger \
+        --add-dynamic-module=/src/opentracing \
         --with-cc-opt="-I$HUNTER_INSTALL_DIR/include" \
         --with-ld-opt="-L$HUNTER_INSTALL_DIR/lib" \
+        --with-debug \
   && make modules \
   && cp objs/ngx_http_opentracing_module.so $NGINX_MODULES_PATH/ \
-  && cp objs/ngx_http_zipkin_module.so $NGINX_MODULES_PATH/ \
-  && cp objs/ngx_http_lightstep_module.so $NGINX_MODULES_PATH/ \
-  && cp objs/ngx_http_jaeger_module.so $NGINX_MODULES_PATH/ \
 	# if we have leftovers from building, let's purge them (including extra, unnecessary build deps)
+  && rm -rf /src \
   && rm -rf $HOME/.hunter \
   && if [ -n "$tempDir" ]; then \
   	apt-get purge -y --auto-remove \

Dockerfile-test

@@ -0,0 +1,55 @@
+FROM ubuntu:18.04
+
+ARG OPENTRACING_CPP_VERSION=v1.4.0
+ARG NGINX_VERSION=1.13.12
+
+COPY . /src
+
+RUN set -x \
+  && apt-get update \
+  && apt-get install --no-install-recommends --no-install-suggests -y \
+              build-essential \
+              gettext \
+              cmake \
+              git \
+              gnupg2 \
+              software-properties-common \
+              curl \
+              python3 \
+              jq \
+              ca-certificates \
+              wget \
+              libpcre3 libpcre3-dev \
+              zlib1g-dev \
+              gcc-8 \
+              g++-8 \
+### Use gcc-8 (the default gcc has this problem when using with address sanitizer:
+### https://gcc.gnu.org/bugzilla/show_bug.cgi?id=84428)
+  && update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-8 800 --slave /usr/bin/g++ g++ /usr/bin/g++-8 \
+### Build opentracing-cpp
+  && cd / \
+  && git clone -b $OPENTRACING_CPP_VERSION https://github.com/opentracing/opentracing-cpp.git \
+  && cd opentracing-cpp \
+  && mkdir .build && cd .build \
+  && cmake -DCMAKE_BUILD_TYPE=Debug \
+           -DCMAKE_CXX_FLAGS="-fno-omit-frame-pointer -fsanitize=address" \
+           -DCMAKE_SHARED_LINKER_FLAGS="-fno-omit-frame-pointer -fsanitize=address" \
+           -DCMAKE_EXE_LINKER_FLAGS="-fno-omit-frame-pointer -fsanitize=address" \
+           -DBUILD_TESTING=OFF .. \
+  && make && make install \
+### Build nginx
+  && cd /src \
+  && wget -O nginx-release-${NGINX_VERSION}.tar.gz https://github.com/nginx/nginx/archive/release-${NGINX_VERSION}.tar.gz \
+  && tar zxf nginx-release-${NGINX_VERSION}.tar.gz \
+  && cd /src/nginx-release-${NGINX_VERSION} \
+  # Temporarily disable leak sanitizer to get around false positives in build
+  && export ASAN_OPTIONS=detect_leaks=0 \
+  && export CFLAGS="-Wno-error" \
+  && auto/configure \
+      --with-compat \
+      --with-debug \
+      --with-cc-opt="-O1 -g -fno-omit-frame-pointer -fsanitize=address" \
+      --with-ld-opt="-g -fno-omit-frame-pointer -fsanitize=address" \
+      --add-dynamic-module=/src/opentracing \
+  && make && make install
+CMD ["/usr/local/nginx/sbin/nginx", "-g", "daemon off;"]

README.md

@@ -23,63 +23,94 @@ Building
 ```
 $ tar zxvf nginx-1.9.x.tar.gz
 $ cd nginx-1.9.x
-$ ./configure --add-dynamic-module=/absolute/path/to/nginx-opentracing/opentracing \
-              # To enable tracing with Jaeger
-              --add-dynamic-module=/absolute/path/to/nginx-opentracing/jaeger \
-              # To enable tracing with Zipkin
-              --add-dynamic-module=/absolute/path/to/nginx-opentracing/zipkin \  
-              # To enable tracing with LightStep
-              --add-dynamic-module=/absolute/path/to/nginx-opentracing/lightstep
+$ ./configure --add-dynamic-module=/absolute/path/to/nginx-opentracing/opentracing
 $ make && sudo make install
 ```
 
+You will also need to install a C++ tracer for either [Jaeger](https://github.com/jaegertracing/jaeger-client-cpp), [LightStep](
+https://github.com/lightstep/lightstep-tracer-cpp), or [Zipkin](https://github.com/rnburn/zipkin-cpp-opentracing). For linux x86-64, portable binary plugins are available:
+```
+# Jaeger
+wget https://github.com/jaegertracing/jaeger-client-cpp/releases/download/v0.4.0/libjaegertracing_plugin.linux_amd64.so -O /usr/local/lib/libjaegertracing_plugin.so
+
+# LightStep
+wget -O - https://github.com/lightstep/lightstep-tracer-cpp/releases/download/v0.7.0/linux-amd64-liblightstep_tracer_plugin.so.gz | gunzip -c > /usr/local/lib/liblightstep_tracer_plugin.so
+
+# Zipkin
+wget -O - https://github.com/rnburn/zipkin-cpp-opentracing/releases/download/v0.3.1/linux-amd64-libzipkin_opentracing_plugin.so.gz  gunzip -c > /usr/local/lib/libzipkin_opentracing_plugin.so
+
+```
 
 Getting Started
 ---------------
+First, write a configuration for the tracer used. Below's an example of what
+a Jaeger configuration might look like:
+
+/etc/jaeger-nginx-config.json
+```
+{
+  "service_name": "nginx",
+  "sampler": {
+    "type": "const",
+    "param": 1
+  },
+  "reporter": {
+    "localAgentHostPort": "jaeger:6831"
+  },
+  "headers": {
+    "jaegerDebugHeader": "jaeger-debug-id",
+    "jaegerBaggageHeader": "jaeger-baggage",
+    "traceBaggageHeaderPrefix": "uberctx-",
+  },
+  "baggage_restrictions": {
+    "denyBaggageOnInitializationFailure": false,
+    "hostPort": ""
+  }
+}
+```
+
+See the vendor documentation for details on what options are available.
+
+You can then set up NGINX for distributed tracing by adding the following to
+nginx.conf:
 ```
 # Load the OpenTracing dynamic module.
 load_module modules/ngx_http_opentracing_module.so;
 
-# Load a vendor OpenTracing dynamic module.
-# For example,
-#   load_module modules/ngx_http_jaeger_module.so;
-# or
-#   load_module modules/ngx_http_zipkin_module.so;
-# or
-#   load_module modules/ngx_http_lightstep_module.so;
-
 http {
-  # Configure your vendor's tracer.
-  # For example,
-  #     jaeger_service_name my-nginx-server;
-  #     ...
-  # or
-  #     zipkin_collector_host localhost;
-  #     ...
-  # or
-  #     lightstep_access_token ACCESSTOKEN;
-  #     ....
+  # Load a vendor tracer
+  opentracing_load_tracer /usr/local/lib/libjaegertracing_plugin.so /etc/jaeger-nginx-config.json;
+
+  # or 
+  #   opentracing_load_tracer /usr/local/lib/liblightstep_tracer_plugin.so /path/to/config;
+  # or 
+  #   opentracing_load_tracer /usr/local/lib/libzipkin_opentracing_plugin.so /path/to/config;
 
   # Enable tracing for all requests.
   opentracing on;
 
   # Optionally, set additional tags.
   opentracing_tag http_user_agent $http_user_agent;
 
-  location ~ \.php$ {
+  upstream backend {
+    server app-service:9001;
+  }
+
+  location ~ {
     # The operation name used for spans defaults to the name of the location
     # block, but you can use this directive to customize it.
     opentracing_operation_name $uri;
 
-    fastcgi_pass 127.0.0.1:1025;
+    # Propagate the active span context upstream, so that the trace can be
+    # continued by the backend.
+    # See http://opentracing.io/documentation/pages/api/cross-process-tracing.html
+    opentracing_propagate_context;
+
+    proxy_pass http://backend;
   }
 }
 ```
 
 See [Tutorial](doc/Tutorial.md) for a more complete example,
-[Reference](doc/Directives.md) for a list of available OpenTracing-related
-directives, and [Jaeger](jaeger/doc/Directives.md),
-[Zipkin](zipkin/doc/Directives.md), and
-[LightStep](lightstep/doc/Directives.md) for a list of vendor tracing
+[Reference](doc/Reference.md) for a list of available OpenTracing-related
 directives.
-

ci/do_ci.sh

@@ -2,22 +2,12 @@
 
 set -e
 
-NGINX_VERSION=`nginx -v 2>&1` && NGINX_VERSION=${NGINX_VERSION#*nginx/}
-NGINX_MODULES_PATH=$(nginx -V 2>&1 | grep -oP "modules-path=\K[^\s]*")
-cd /src/nginx-release-${NGINX_VERSION}
-auto/configure \
-  --with-compat \
-  --add-dynamic-module=/src/nginx-opentracing/opentracing 
-make modules
-cp objs/ngx_http_opentracing_module.so $NGINX_MODULES_PATH/
-cd /src/nginx-opentracing
-export MOCKTRACER_LIBRARY=/usr/local/lib/libopentracing_mocktracer.so
-export NGINX_OPENTRACING_MODULE="$NGINX_MODULES_PATH/ngx_http_opentracing_module.so"
-export NGINX_OPENTRACING_TEST_DIR=/nginx-opentracing-test
-mkdir $NGINX_OPENTRACING_TEST_DIR
-chmod a+rwx $NGINX_OPENTRACING_TEST_DIR
-cd test
-for i in *;
-do
-  "$i/run.sh"
-done
+[ -z "${SRC_DIR}" ] && export SRC_DIR="`pwd`"
+
+if [[ "$1" == "system.testing" ]]; then
+  docker build -t nginx-opentracing-test/nginx -f Dockerfile-test .
+  cd test
+  docker build -t nginx-opentracing-test/backend -f Dockerfile-backend .
+  python3 nginx_opentracing_test.py
+  exit 0
+fi