From 5d462e844c40aada32684b3ab21a4b2b57d7bd90 Mon Sep 17 00:00:00 2001 From: Luca Comellini Date: Thu, 23 Jun 2022 07:26:24 -0500 Subject: [PATCH] Update Python deps, Docker base images, NGINX used in tests (#289) * Update Python deps * Update Docker image for gRPC * Fix deprecation warning from Docker * Update other Docker images * Print Docker version, use different port * Use old docker-compose --- .circleci/config.yml | 2 +- .github/dependabot.yml | 4 ++ .gitignore | 3 + .python-version | 2 +- Dockerfile-test | 49 +++++++------- ci/Dockerfile | 2 +- ci/install_dependencies.sh | 9 ++- test/Dockerfile-backend | 20 +++--- test/environment/app.py | 20 +++--- test/environment/docker-compose.yaml | 8 +-- test/environment/grpc/.python-version | 1 + test/environment/grpc/Dockerfile | 11 +++- test/environment/grpc/app.proto | 6 +- test/environment/grpc/app_pb2.py | 81 +++++------------------- test/environment/grpc/app_pb2_grpc.py | 80 ++++++++++++++--------- test/environment/grpc/requirements.txt | 6 +- test/environment/nginx.conf | 2 +- test/environment/traces/nginx.json | 1 + test/nginx_opentracing_test.py | 88 +++++++++++++++++--------- test/requirements.ci.txt | 4 +- 20 files changed, 212 insertions(+), 187 deletions(-) create mode 100644 test/environment/grpc/.python-version diff --git a/.circleci/config.yml b/.circleci/config.yml index 86ce20c8..6da43911 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -2,7 +2,7 @@ version: 2 jobs: system_testing: machine: - image: ubuntu-2004:202010-01 + image: ubuntu-2204:2022.04.2 steps: - checkout - run: diff --git a/.github/dependabot.yml b/.github/dependabot.yml index d424db26..9e42de81 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -12,3 +12,7 @@ updates: directory: "/test" schedule: interval: "daily" + - package-ecosystem: "pip" + directory: "/test/environment/grpc" + schedule: + interval: "daily" diff --git a/.gitignore b/.gitignore index 202212d8..92b5f87e 100644 --- a/.gitignore +++ b/.gitignore @@ -33,3 +33,6 @@ __pycache__ # Test test-log/ + +# IDEs +.vscode diff --git a/.python-version b/.python-version index 0cbfaed0..7d4ef04f 100644 --- a/.python-version +++ b/.python-version @@ -1 +1 @@ -3.8.5 +3.10.3 diff --git a/Dockerfile-test b/Dockerfile-test index 5c6d4304..617c1bc5 100644 --- a/Dockerfile-test +++ b/Dockerfile-test @@ -1,29 +1,24 @@ -FROM ubuntu:20.04 +FROM ubuntu:22.04 ARG OPENTRACING_CPP_VERSION=v1.6.0 -ARG NGINX_VERSION=1.19.7 +ARG NGINX_VERSION=1.23.0 RUN set -x \ && apt-get update \ && DEBIAN_FRONTEND="noninteractive" apt-get install --no-install-recommends --no-install-suggests -y \ - build-essential \ - gettext \ - cmake \ - git \ - gnupg2 \ - software-properties-common \ - curl \ - python3 \ - jq \ - ca-certificates \ - wget \ - libpcre3 libpcre3-dev \ - zlib1g-dev \ - gcc-8 \ - g++-8 \ -### Use gcc-8 (the default gcc has this problem when using with address sanitizer: -### https://gcc.gnu.org/bugzilla/show_bug.cgi?id=84428) - && update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-8 800 --slave /usr/bin/g++ g++ /usr/bin/g++-8 + build-essential \ + gettext \ + cmake \ + git \ + gnupg2 \ + software-properties-common \ + curl \ + python3 \ + jq \ + ca-certificates \ + wget \ + libpcre3 libpcre3-dev \ + zlib1g-dev ### Build opentracing-cpp RUN cd / \ @@ -31,7 +26,7 @@ RUN cd / \ && cd opentracing-cpp \ && mkdir .build && cd .build \ && cmake -DCMAKE_BUILD_TYPE=Debug \ - -DBUILD_TESTING=OFF .. \ + -DBUILD_TESTING=OFF .. \ && make && make install COPY ./opentracing /opentracing @@ -46,12 +41,12 @@ RUN cd / \ && export ASAN_OPTIONS=detect_leaks=0 \ && export CFLAGS="-Wno-error" \ && auto/configure \ - --with-http_auth_request_module \ - --with-compat \ - --with-debug \ - # enables grpc - --with-http_v2_module \ - --add-dynamic-module=/opentracing \ + --with-http_auth_request_module \ + --with-compat \ + --with-debug \ + # enables grpc + --with-http_v2_module \ + --add-dynamic-module=/opentracing \ && make && make install CMD ["/usr/local/nginx/sbin/nginx", "-g", "daemon off;"] diff --git a/ci/Dockerfile b/ci/Dockerfile index 4f03cdb3..9f65de3b 100644 --- a/ci/Dockerfile +++ b/ci/Dockerfile @@ -1,4 +1,4 @@ -FROM ubuntu:20.04 +FROM ubuntu:22.04 WORKDIR /3rd_party diff --git a/ci/install_dependencies.sh b/ci/install_dependencies.sh index 7fd3b90a..8cd7b0d0 100755 --- a/ci/install_dependencies.sh +++ b/ci/install_dependencies.sh @@ -3,8 +3,15 @@ set -x set -e +# workaround to install docker-compose v1 +sudo curl -L "https://github.com/docker/compose/releases/download/1.29.2/docker-compose-$(uname -s)-$(uname -m)" -o /usr/local/bin/docker-compose +sudo chmod +x /usr/local/bin/docker-compose + +docker version +docker-compose version + pyenv versions -pyenv local 3.8.5 +pyenv local 3.10.3 python --version pip --version pip install --upgrade pip diff --git a/test/Dockerfile-backend b/test/Dockerfile-backend index 20efa968..9f85d168 100644 --- a/test/Dockerfile-backend +++ b/test/Dockerfile-backend @@ -1,15 +1,15 @@ -FROM ubuntu:20.04 +FROM ubuntu:22.04 RUN apt-get update \ - && apt-get install -y \ - curl \ - python3 \ - python3-dev \ - build-essential \ - && rm -rf /var/lib/apt/lists/* \ - && curl https://bootstrap.pypa.io/get-pip.py -o get-pip.py \ - && python3 get-pip.py \ - && rm get-pip.py + && apt-get install -y \ + curl \ + python3 \ + python3-dev \ + build-essential \ + && rm -rf /var/lib/apt/lists/* \ + && curl https://bootstrap.pypa.io/get-pip.py -o get-pip.py \ + && python3 get-pip.py \ + && rm get-pip.py COPY . /app WORKDIR /app diff --git a/test/environment/app.py b/test/environment/app.py index 9f35a3ea..4e836979 100644 --- a/test/environment/app.py +++ b/test/environment/app.py @@ -1,15 +1,19 @@ from flask import Flask, request + app = Flask(__name__) -@app.route('/has-span-context') + +@app.route("/has-span-context") def has_span_context(): - assert 'x-ot-span-context' in request.headers - return 'Flask Dockerized' + assert "x-ot-span-context" in request.headers + return "Flask Dockerized" -@app.route('/has-span-context-redirect') + +@app.route("/has-span-context-redirect") def has_span_context_redirect(): - assert 'x-ot-span-context' in request.headers - return '', 301 + assert "x-ot-span-context" in request.headers + return "", 301 + -if __name__ == '__main__': - app.run(debug=True,host='0.0.0.0') +if __name__ == "__main__": + app.run(debug=True, host="0.0.0.0", port="5001") diff --git a/test/environment/docker-compose.yaml b/test/environment/docker-compose.yaml index 557345c3..2b7af29d 100644 --- a/test/environment/docker-compose.yaml +++ b/test/environment/docker-compose.yaml @@ -1,4 +1,4 @@ -version: '2' +version: '3' services: nginx: @@ -30,7 +30,6 @@ services: - -g - daemon off; - backend: image: nginx-opentracing-test/backend networks: @@ -40,9 +39,9 @@ services: volumes: - ./app.py:/app/app.py expose: - - "5000" + - "5001" ports: - - "5000:5000" + - "5001:5001" php_fpm: image: php:7-fpm @@ -70,4 +69,3 @@ services: networks: testnet: {} - diff --git a/test/environment/grpc/.python-version b/test/environment/grpc/.python-version new file mode 100644 index 00000000..7d4ef04f --- /dev/null +++ b/test/environment/grpc/.python-version @@ -0,0 +1 @@ +3.10.3 diff --git a/test/environment/grpc/Dockerfile b/test/environment/grpc/Dockerfile index 54654370..a0693c7e 100644 --- a/test/environment/grpc/Dockerfile +++ b/test/environment/grpc/Dockerfile @@ -1,3 +1,12 @@ -FROM grpc/python:1.13-onbuild +FROM python:3.10 + +RUN mkdir -p /usr/src/app +WORKDIR /usr/src/app + +COPY requirements.txt /usr/src/app/ +RUN pip install --no-cache-dir -r requirements.txt + +COPY . /usr/src/app + ENTRYPOINT ["python", "-u"] CMD ["app.py"] diff --git a/test/environment/grpc/app.proto b/test/environment/grpc/app.proto index f559e7ef..12305c34 100644 --- a/test/environment/grpc/app.proto +++ b/test/environment/grpc/app.proto @@ -2,10 +2,10 @@ // // python3 -m grpc_tools.protoc --proto_path=. --python_out=. --grpc_python_out=. app.proto -syntax = "proto3"; +syntax = "proto3"; -service App { - rpc CheckTraceHeader(Empty) returns (Empty); +service App { + rpc CheckTraceHeader(Empty) returns (Empty); } message Empty { diff --git a/test/environment/grpc/app_pb2.py b/test/environment/grpc/app_pb2.py index 909827bf..d223cc09 100644 --- a/test/environment/grpc/app_pb2.py +++ b/test/environment/grpc/app_pb2.py @@ -1,9 +1,9 @@ +# -*- coding: utf-8 -*- # Generated by the protocol buffer compiler. DO NOT EDIT! # source: app.proto - -import sys -_b=sys.version_info[0]<3 and (lambda x:x) or (lambda x:x.encode('latin1')) +"""Generated protocol buffer code.""" from google.protobuf import descriptor as _descriptor +from google.protobuf import descriptor_pool as _descriptor_pool from google.protobuf import message as _message from google.protobuf import reflection as _reflection from google.protobuf import symbol_database as _symbol_database @@ -14,73 +14,24 @@ -DESCRIPTOR = _descriptor.FileDescriptor( - name='app.proto', - package='', - syntax='proto3', - serialized_options=None, - serialized_pb=_b('\n\tapp.proto\"\x07\n\x05\x45mpty2)\n\x03\x41pp\x12\"\n\x10\x43heckTraceHeader\x12\x06.Empty\x1a\x06.Emptyb\x06proto3') -) - - - +DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\tapp.proto\"\x07\n\x05\x45mpty2)\n\x03\x41pp\x12\"\n\x10\x43heckTraceHeader\x12\x06.Empty\x1a\x06.Emptyb\x06proto3') -_EMPTY = _descriptor.Descriptor( - name='Empty', - full_name='Empty', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - serialized_options=None, - is_extendable=False, - syntax='proto3', - extension_ranges=[], - oneofs=[ - ], - serialized_start=13, - serialized_end=20, -) -DESCRIPTOR.message_types_by_name['Empty'] = _EMPTY -_sym_db.RegisterFileDescriptor(DESCRIPTOR) -Empty = _reflection.GeneratedProtocolMessageType('Empty', (_message.Message,), dict( - DESCRIPTOR = _EMPTY, - __module__ = 'app_pb2' +_EMPTY = DESCRIPTOR.message_types_by_name['Empty'] +Empty = _reflection.GeneratedProtocolMessageType('Empty', (_message.Message,), { + 'DESCRIPTOR' : _EMPTY, + '__module__' : 'app_pb2' # @@protoc_insertion_point(class_scope:Empty) - )) + }) _sym_db.RegisterMessage(Empty) +_APP = DESCRIPTOR.services_by_name['App'] +if _descriptor._USE_C_DESCRIPTORS == False: - -_APP = _descriptor.ServiceDescriptor( - name='App', - full_name='App', - file=DESCRIPTOR, - index=0, - serialized_options=None, - serialized_start=22, - serialized_end=63, - methods=[ - _descriptor.MethodDescriptor( - name='CheckTraceHeader', - full_name='App.CheckTraceHeader', - index=0, - containing_service=None, - input_type=_EMPTY, - output_type=_EMPTY, - serialized_options=None, - ), -]) -_sym_db.RegisterServiceDescriptor(_APP) - -DESCRIPTOR.services_by_name['App'] = _APP - + DESCRIPTOR._options = None + _EMPTY._serialized_start=13 + _EMPTY._serialized_end=20 + _APP._serialized_start=22 + _APP._serialized_end=63 # @@protoc_insertion_point(module_scope) diff --git a/test/environment/grpc/app_pb2_grpc.py b/test/environment/grpc/app_pb2_grpc.py index d8fd98ac..9557f020 100644 --- a/test/environment/grpc/app_pb2_grpc.py +++ b/test/environment/grpc/app_pb2_grpc.py @@ -1,46 +1,66 @@ # Generated by the gRPC Python protocol compiler plugin. DO NOT EDIT! +"""Client and server classes corresponding to protobuf-defined services.""" import grpc import app_pb2 as app__pb2 class AppStub(object): - # missing associated documentation comment in .proto file - pass + """Missing associated documentation comment in .proto file.""" - def __init__(self, channel): - """Constructor. + def __init__(self, channel): + """Constructor. - Args: - channel: A grpc.Channel. - """ - self.CheckTraceHeader = channel.unary_unary( - '/App/CheckTraceHeader', - request_serializer=app__pb2.Empty.SerializeToString, - response_deserializer=app__pb2.Empty.FromString, - ) + Args: + channel: A grpc.Channel. + """ + self.CheckTraceHeader = channel.unary_unary( + '/App/CheckTraceHeader', + request_serializer=app__pb2.Empty.SerializeToString, + response_deserializer=app__pb2.Empty.FromString, + ) class AppServicer(object): - # missing associated documentation comment in .proto file - pass + """Missing associated documentation comment in .proto file.""" - def CheckTraceHeader(self, request, context): - # missing associated documentation comment in .proto file - pass - context.set_code(grpc.StatusCode.UNIMPLEMENTED) - context.set_details('Method not implemented!') - raise NotImplementedError('Method not implemented!') + def CheckTraceHeader(self, request, context): + """Missing associated documentation comment in .proto file.""" + context.set_code(grpc.StatusCode.UNIMPLEMENTED) + context.set_details('Method not implemented!') + raise NotImplementedError('Method not implemented!') def add_AppServicer_to_server(servicer, server): - rpc_method_handlers = { - 'CheckTraceHeader': grpc.unary_unary_rpc_method_handler( - servicer.CheckTraceHeader, - request_deserializer=app__pb2.Empty.FromString, - response_serializer=app__pb2.Empty.SerializeToString, - ), - } - generic_handler = grpc.method_handlers_generic_handler( - 'App', rpc_method_handlers) - server.add_generic_rpc_handlers((generic_handler,)) + rpc_method_handlers = { + 'CheckTraceHeader': grpc.unary_unary_rpc_method_handler( + servicer.CheckTraceHeader, + request_deserializer=app__pb2.Empty.FromString, + response_serializer=app__pb2.Empty.SerializeToString, + ), + } + generic_handler = grpc.method_handlers_generic_handler( + 'App', rpc_method_handlers) + server.add_generic_rpc_handlers((generic_handler,)) + + + # This class is part of an EXPERIMENTAL API. +class App(object): + """Missing associated documentation comment in .proto file.""" + + @staticmethod + def CheckTraceHeader(request, + target, + options=(), + channel_credentials=None, + call_credentials=None, + insecure=False, + compression=None, + wait_for_ready=None, + timeout=None, + metadata=None): + return grpc.experimental.unary_unary(request, target, '/App/CheckTraceHeader', + app__pb2.Empty.SerializeToString, + app__pb2.Empty.FromString, + options, channel_credentials, + insecure, call_credentials, compression, wait_for_ready, timeout, metadata) diff --git a/test/environment/grpc/requirements.txt b/test/environment/grpc/requirements.txt index edca13af..0131e1c5 100644 --- a/test/environment/grpc/requirements.txt +++ b/test/environment/grpc/requirements.txt @@ -1 +1,5 @@ -#grpcio-tools +grpcio-tools==1.47.0 +grpcio==1.47.0 +grpcio-reflection==1.47.0 +grpcio-health-checking==1.47.0 +grpcio-testing==1.47.0 diff --git a/test/environment/nginx.conf b/test/environment/nginx.conf index 99fc7559..e87680f9 100644 --- a/test/environment/nginx.conf +++ b/test/environment/nginx.conf @@ -8,7 +8,7 @@ http { opentracing_load_tracer /usr/local/lib/libopentracing_mocktracer.so /tracer-config.json; upstream backend { - server backend:5000; + server backend:5001; } server { diff --git a/test/environment/traces/nginx.json b/test/environment/traces/nginx.json index e69de29b..0637a088 100644 --- a/test/environment/traces/nginx.json +++ b/test/environment/traces/nginx.json @@ -0,0 +1 @@ +[] \ No newline at end of file diff --git a/test/nginx_opentracing_test.py b/test/nginx_opentracing_test.py index 47a2a0fd..df5dd48c 100644 --- a/test/nginx_opentracing_test.py +++ b/test/nginx_opentracing_test.py @@ -1,33 +1,52 @@ -import unittest -import shutil +import http.client +import json import os +import shutil import stat -import tempfile import subprocess +import tempfile import time -import docker -import json -import http.client -import grpc +import unittest +import warnings + import app_pb2 as app_messages import app_pb2_grpc as app_service +import docker +import grpc + + +def get_docker_client(): + with warnings.catch_warnings(): + # Silence warnings due to use of deprecated methods within dockerpy + # See https://github.com/docker/docker-py/pull/2931 + warnings.filterwarnings( + "ignore", + message="distutils Version classes are deprecated.*", + category=DeprecationWarning, + ) + + return docker.from_env() + class NginxOpenTracingTest(unittest.TestCase): def setUp(self): self.testdir = os.getcwd() self.workdir = os.path.join(tempfile.mkdtemp(), "environment") - shutil.copytree(os.path.join(os.getcwd(), "environment"), - self.workdir) + shutil.copytree(os.path.join(os.getcwd(), "environment"), self.workdir) os.chdir(self.workdir) # Make sure trace output is writable - os.chmod(os.path.join(self.workdir, "traces", "nginx.json"), - stat.S_IRWXU | stat.S_IRWXG | stat.S_IRWXO) - - self.environment_handle = subprocess.Popen(["docker-compose", "up"], - stdout=subprocess.PIPE, - stderr=subprocess.PIPE) - self.client = docker.from_env() + os.chmod( + os.path.join(self.workdir, "traces", "nginx.json"), + stat.S_IRWXU | stat.S_IRWXG | stat.S_IRWXO, + ) + + self.environment_handle = subprocess.Popen( + ["docker-compose", "up"], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + ) + self.client = get_docker_client() timeout = time.time() + 60 while len(self.client.containers.list()) != 4: if time.time() > timeout: @@ -36,14 +55,14 @@ def setUp(self): # Wait so that backend can come up. # TODO: replace with something better - time.sleep(2) + time.sleep(5) - self.conn = http.client.HTTPConnection('localhost', 8080, timeout=5) - self.grpcConn = grpc.insecure_channel('localhost:8081') + self.conn = http.client.HTTPConnection("localhost", 8080, timeout=5) + self.grpcConn = grpc.insecure_channel("localhost:8081") try: grpc.channel_ready_future(self.grpcConn).result(timeout=10) except grpc.FutureTimeoutError: - sys.exit('Error connecting to server') + sys.exit("Error connecting to server") self.running = True def _logEnvironment(self): @@ -58,13 +77,19 @@ def _logEnvironment(self): with open(os.path.join(test_log, "docker_compose_stderr"), "w") as out: out.write(str(self.environment_stderr)) - shutil.copyfile(os.path.join(self.workdir, "traces", "nginx.json"), - os.path.join(test_log, "nginx-trace.json")) + shutil.copyfile( + os.path.join(self.workdir, "traces", "nginx.json"), + os.path.join(test_log, "nginx-trace.json"), + ) - shutil.copyfile(os.path.join(self.workdir, "logs", "debug.log"), - os.path.join(test_log, "nginx-debug.log")) - shutil.copyfile(os.path.join(self.workdir, "logs", "error.log"), - os.path.join(test_log, "nginx-error.log")) + shutil.copyfile( + os.path.join(self.workdir, "logs", "debug.log"), + os.path.join(test_log, "nginx-debug.log"), + ) + shutil.copyfile( + os.path.join(self.workdir, "logs", "error.log"), + os.path.join(test_log, "nginx-error.log"), + ) def tearDown(self): self._stopDocker() @@ -164,8 +189,12 @@ def testMultipleTags(self): location_span = self.nginx_traces[0] # assert tags that should NOT be in this trace self.assertEqual(list(location_span["tags"].values()).count("custom_tag_1"), 0) - self.assertEqual(list(location_span["tags"].values()).count("second_server_tag"), 0) - self.assertEqual(list(location_span["tags"].values()).count("second_server_location_tag"), 0) + self.assertEqual( + list(location_span["tags"].values()).count("second_server_tag"), 0 + ) + self.assertEqual( + list(location_span["tags"].values()).count("second_server_location_tag"), 0 + ) # assert tags that should be in this trace self.assertEqual(location_span["tags"]["first_server_tag"], "dummy1") self.assertEqual(location_span["tags"]["custom_tag_2"], "quoted_string") @@ -188,5 +217,6 @@ def testGrpcPropagation(self): self.assertEqual(len(self.nginx_traces), 2) -if __name__ == '__main__': + +if __name__ == "__main__": unittest.main(verbosity=2) diff --git a/test/requirements.ci.txt b/test/requirements.ci.txt index 74d56daf..2f2b93e3 100644 --- a/test/requirements.ci.txt +++ b/test/requirements.ci.txt @@ -1,8 +1,6 @@ -docker==4.3.1 +docker==5.0.3 requests==2.28.0 websocket-client==1.3.3 urllib3==1.26.9 -docker-compose==1.27.4 grpcio==1.46.3 protobuf==4.21.0 -