From 06dbf01573a37da22f5abe3ec3783d1a375fddcc Mon Sep 17 00:00:00 2001 From: Mikko Viitanen Date: Fri, 17 Mar 2023 15:22:27 +0200 Subject: [PATCH 1/6] =?UTF-8?q?What=E2=80=99s=20included=3F?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Enable logs for Ad service and Recommendation service. - Add OTLP exporters for logs - Add the filter processor to prevent an error from the Prometheus exporter for duplicate queueSize metric, see https://github.com/open-telemetry/opentelemetry-collector-contrib/issues/18194. The filter processor can be removed when the fault gets fixed. This PR doesn’t introduce any logs backend. Instead, logs are output only to Logging exporter and can be seen in the console (otelcol). otel-col | 2023-03-17T11:40:22.662Z info LogsExporter {"kind": "exporter", "data_type": "logs", "name": "logging", "#logs": 2} After this PR, different logging backends can be easily tested by configuring an additional exporter. --- .env | 1 + docker-compose.yml | 2 + src/adservice/Dockerfile | 2 +- .../src/main/java/oteldemo/AdService.java | 7 +-- src/otelcollector/otelcol-config.yml | 14 +++++- .../recommendation_server.py | 50 +++++++++++++------ 6 files changed, 55 insertions(+), 21 deletions(-) diff --git a/.env b/.env index a3ba836924..09ee9f7a4c 100644 --- a/.env +++ b/.env @@ -25,6 +25,7 @@ OTEL_COLLECTOR_PORT=4317 OTEL_EXPORTER_OTLP_ENDPOINT=http://${OTEL_COLLECTOR_HOST}:${OTEL_COLLECTOR_PORT} OTEL_EXPORTER_OTLP_TRACES_ENDPOINT=${OTEL_EXPORTER_OTLP_ENDPOINT} OTEL_EXPORTER_OTLP_METRICS_ENDPOINT=${OTEL_EXPORTER_OTLP_ENDPOINT} +OTEL_EXPORTER_OTLP_LOGS_ENDPOINT=${OTEL_EXPORTER_OTLP_ENDPOINT} PUBLIC_OTEL_EXPORTER_OTLP_TRACES_ENDPOINT=http://localhost:4318/v1/traces # OpenTelemetry Resource Definitions diff --git a/docker-compose.yml b/docker-compose.yml index 5d335ea656..5910db9568 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -78,8 +78,10 @@ services: - OTEL_EXPORTER_OTLP_TRACES_ENDPOINT - OTEL_EXPORTER_OTLP_METRICS_ENDPOINT - OTEL_EXPORTER_OTLP_METRICS_TEMPORALITY_PREFERENCE + - OTEL_EXPORTER_OTLP_LOGS_ENDPOINT - OTEL_RESOURCE_ATTRIBUTES - OTEL_SERVICE_NAME=adservice + - OTEL_LOGS_EXPORTER=otlp depends_on: - otelcol logging: *logging diff --git a/src/adservice/Dockerfile b/src/adservice/Dockerfile index 8b6aa1da92..834ddfc6e1 100644 --- a/src/adservice/Dockerfile +++ b/src/adservice/Dockerfile @@ -30,7 +30,7 @@ RUN ./gradlew installDist -PprotoSourceDir=./proto FROM eclipse-temurin:17-jre -ARG version=1.23.0 +ARG version=1.24.0 WORKDIR /usr/src/app/ COPY --from=builder /usr/src/app/ ./ diff --git a/src/adservice/src/main/java/oteldemo/AdService.java b/src/adservice/src/main/java/oteldemo/AdService.java index e8905f7009..d0ec11d3ec 100644 --- a/src/adservice/src/main/java/oteldemo/AdService.java +++ b/src/adservice/src/main/java/oteldemo/AdService.java @@ -99,7 +99,7 @@ private void start() throws IOException { .addService(healthMgr.getHealthService()) .build() .start(); - logger.info("Ad Service started, listening on " + port); + logger.info("Ad service started, listening on " + port); Runtime.getRuntime() .addShutdownHook( new Thread( @@ -160,8 +160,8 @@ public void getAds(AdRequest req, StreamObserver responseObserver) { span.setAttribute("app.ads.contextKeys", req.getContextKeysList().toString()); span.setAttribute("app.ads.contextKeys.count", req.getContextKeysCount()); - logger.info("received ad request (context_words=" + req.getContextKeysList() + ")"); if (req.getContextKeysCount() > 0) { + logger.info("Targeted ad request received for " + req.getContextKeysList()); for (int i = 0; i < req.getContextKeysCount(); i++) { Collection ads = service.getAdsByCategory(req.getContextKeys(i)); allAds.addAll(ads); @@ -169,6 +169,7 @@ public void getAds(AdRequest req, StreamObserver responseObserver) { adRequestType = AdRequestType.TARGETED; adResponseType = AdResponseType.TARGETED; } else { + logger.info("Non-targeted ad request received, preparing random response."); allAds = service.getRandomAds(); adRequestType = AdRequestType.NOT_TARGETED; adResponseType = AdResponseType.RANDOM; @@ -314,7 +315,7 @@ private static ImmutableListMultimap createAdsMap() { /** Main launches the server from the command line. */ public static void main(String[] args) throws IOException, InterruptedException { // Start the RPC server. You shouldn't see any output from gRPC before this. - logger.info("AdService starting."); + logger.info("Ad service starting."); final AdService service = AdService.getInstance(); service.start(); service.blockUntilShutdown(); diff --git a/src/otelcollector/otelcol-config.yml b/src/otelcollector/otelcol-config.yml index ea4ab62aa4..91e0061ad1 100644 --- a/src/otelcollector/otelcol-config.yml +++ b/src/otelcollector/otelcol-config.yml @@ -43,14 +43,24 @@ processors: - context: metric statements: - set(description, "Measures the duration of inbound HTTP requests") where name == "http.server.duration" + filter: + metrics: + exclude: + match_type: strict + metric_names: + - queueSize service: pipelines: traces: receivers: [otlp] processors: [spanmetrics, batch] - exporters: [logging, otlp] + exporters: [otlp, logging] metrics: receivers: [otlp] - processors: [transform, batch] + processors: [filter, transform, batch] exporters: [prometheus, logging] + logs: + receivers: [otlp] + processors: [batch] + exporters: [logging] \ No newline at end of file diff --git a/src/recommendationservice/recommendation_server.py b/src/recommendationservice/recommendation_server.py index 86fd2b274c..d1e80f180d 100644 --- a/src/recommendationservice/recommendation_server.py +++ b/src/recommendationservice/recommendation_server.py @@ -22,14 +22,20 @@ # Pip import grpc from opentelemetry import trace, metrics - +from opentelemetry._logs import set_logger_provider +from opentelemetry.exporter.otlp.proto.grpc._log_exporter import ( + OTLPLogExporter, +) +from opentelemetry.sdk._logs import LoggerProvider, LoggingHandler +from opentelemetry.sdk._logs.export import BatchLogRecordProcessor +from opentelemetry.sdk.resources import Resource # Local +import logging import demo_pb2 import demo_pb2_grpc from grpc_health.v1 import health_pb2 from grpc_health.v1 import health_pb2_grpc -from logger import getJSONLogger from metrics import ( init_metrics @@ -43,7 +49,8 @@ def ListRecommendations(self, request, context): prod_list = get_product_list(request.product_ids) span = trace.get_current_span() span.set_attribute("app.products_recommended.count", len(prod_list)) - logger.info(f"[Recv ListRecommendations] product_ids={prod_list}") + logger.info(f"Receive ListRecommendations for product ids:{prod_list}") + # build and return response response = demo_pb2.ListRecommendationsResponse() response.product_ids.extend(prod_list) @@ -78,7 +85,7 @@ def get_product_list(request_product_ids): if random.random() < 0.5 or first_run: first_run = False span.set_attribute("app.cache_hit", False) - logger.info("cache miss") + logger.info("get_product_list: cache miss") cat_response = product_catalog_stub.ListProducts(demo_pb2.Empty()) response_ids = [x.id for x in cat_response.products] cached_ids = cached_ids + response_ids @@ -86,7 +93,7 @@ def get_product_list(request_product_ids): product_ids = cached_ids else: span.set_attribute("app.cache_hit", True) - logger.info("cache hit") + logger.info("get_product_list: cache hit") product_ids = cached_ids else: span.set_attribute("app.recommendation.cache_enabled", False) @@ -110,7 +117,6 @@ def get_product_list(request_product_ids): return prod_list - def must_map_env(key: str): value = os.environ.get(key) if value is None: @@ -119,19 +125,35 @@ def must_map_env(key: str): def check_feature_flag(flag_name: str): flag = feature_flag_stub.GetFlag(demo_pb2.GetFlagRequest(name=flag_name)).flag - logger.info(flag) return flag.enabled if __name__ == "__main__": + service_name = must_map_env('OTEL_SERVICE_NAME') + # Initialize Traces and Metrics - tracer = trace.get_tracer_provider().get_tracer("recommendationservice") - meter = metrics.get_meter_provider().get_meter("recommendationservice") + tracer = trace.get_tracer_provider().get_tracer(service_name) + meter = metrics.get_meter_provider().get_meter(service_name) rec_svc_metrics = init_metrics(meter) - port = must_map_env('RECOMMENDATION_SERVICE_PORT') + # Initialize Logs + logger_provider = LoggerProvider( + resource=Resource.create( + { + 'service.name': service_name, + } + ), + ) + set_logger_provider(logger_provider) + log_exporter = OTLPLogExporter(insecure=True) + logger_provider.add_log_record_processor(BatchLogRecordProcessor(log_exporter)) + handler = LoggingHandler(level=logging.NOTSET, logger_provider=logger_provider) + + # Attach OTLP handler to logger + logger = logging.getLogger('main') + logger.addHandler(handler) + catalog_addr = must_map_env('PRODUCT_CATALOG_SERVICE_ADDR') ff_addr = must_map_env('FEATURE_FLAG_GRPC_SERVICE_ADDR') - pc_channel = grpc.insecure_channel(catalog_addr) ff_channel = grpc.insecure_channel(ff_addr) product_catalog_stub = demo_pb2_grpc.ProductCatalogServiceStub(pc_channel) @@ -145,11 +167,9 @@ def check_feature_flag(flag_name: str): demo_pb2_grpc.add_RecommendationServiceServicer_to_server(service, server) health_pb2_grpc.add_HealthServicer_to_server(service, server) - # Start logger - logger = getJSONLogger('recommendationservice-server') - logger.info(f"RecommendationService listening on port: {port}") - # Start server + port = must_map_env('RECOMMENDATION_SERVICE_PORT') server.add_insecure_port(f'[::]:{port}') server.start() + logger.info(f'Recommendation service started, listening on port {port}') server.wait_for_termination() From 658fd65758ab7f37c0755a9292326423c25793d3 Mon Sep 17 00:00:00 2001 From: Mikko Viitanen Date: Fri, 17 Mar 2023 15:47:27 +0200 Subject: [PATCH 2/6] Add changelog and fix lint errors. --- CHANGELOG.md | 2 ++ src/otelcollector/otelcol-config.yml | 2 +- src/recommendationservice/recommendation_server.py | 2 +- 3 files changed, 4 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 27d39bb0a2..4052ccdfde 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -275,3 +275,5 @@ significant modifications will be credited to OpenTelemetry Authors. ([#781](https://github.com/open-telemetry/opentelemetry-demo/pull/781)) * [bug] fixing quoteservice metrics exporting (PHP) ([#793](https://github.com/open-telemetry/opentelemetry-demo/pull/793)) +* Add logs for Ad service and Recommendation service + ([#796](https://github.com/open-telemetry/opentelemetry-demo/pull/796)) \ No newline at end of file diff --git a/src/otelcollector/otelcol-config.yml b/src/otelcollector/otelcol-config.yml index 91e0061ad1..43409336e8 100644 --- a/src/otelcollector/otelcol-config.yml +++ b/src/otelcollector/otelcol-config.yml @@ -63,4 +63,4 @@ service: logs: receivers: [otlp] processors: [batch] - exporters: [logging] \ No newline at end of file + exporters: [logging] diff --git a/src/recommendationservice/recommendation_server.py b/src/recommendationservice/recommendation_server.py index d1e80f180d..a3911b211f 100644 --- a/src/recommendationservice/recommendation_server.py +++ b/src/recommendationservice/recommendation_server.py @@ -172,4 +172,4 @@ def check_feature_flag(flag_name: str): server.add_insecure_port(f'[::]:{port}') server.start() logger.info(f'Recommendation service started, listening on port {port}') - server.wait_for_termination() + server.wait_for_termination() \ No newline at end of file From e9cbbae7152e923d6d64ca1d150f999fb608a7ef Mon Sep 17 00:00:00 2001 From: Mikko Viitanen Date: Fri, 17 Mar 2023 16:02:12 +0200 Subject: [PATCH 3/6] Fix changelog and lint --- CHANGELOG.md | 9 +++++---- src/recommendationservice/recommendation_server.py | 2 +- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4052ccdfde..c85355c694 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,11 @@ release. ([#787](https://github.com/open-telemetry/opentelemetry-demo/pull/787)) * [cart] use 60m TTL for cart entries in redis ([#779](https://github.com/open-telemetry/opentelemetry-demo/pull/779)) +* [bug] fixing quoteservice metrics exporting (PHP) +([#793](https://github.com/open-telemetry/opentelemetry-demo/pull/793)) +* Add logs for Ad service and Recommendation service +([#796](https://github.com/open-telemetry/opentelemetry-demo/pull/796)) + ## v0.1.0 @@ -273,7 +278,3 @@ significant modifications will be credited to OpenTelemetry Authors. ([#764](https://github.com/open-telemetry/opentelemetry-demo/pull/764)) * [chore] align memory limits with Helm chart ([#781](https://github.com/open-telemetry/opentelemetry-demo/pull/781)) -* [bug] fixing quoteservice metrics exporting (PHP) - ([#793](https://github.com/open-telemetry/opentelemetry-demo/pull/793)) -* Add logs for Ad service and Recommendation service - ([#796](https://github.com/open-telemetry/opentelemetry-demo/pull/796)) \ No newline at end of file diff --git a/src/recommendationservice/recommendation_server.py b/src/recommendationservice/recommendation_server.py index a3911b211f..d1e80f180d 100644 --- a/src/recommendationservice/recommendation_server.py +++ b/src/recommendationservice/recommendation_server.py @@ -172,4 +172,4 @@ def check_feature_flag(flag_name: str): server.add_insecure_port(f'[::]:{port}') server.start() logger.info(f'Recommendation service started, listening on port {port}') - server.wait_for_termination() \ No newline at end of file + server.wait_for_termination() From 2a1953a7ad8b848919ba529a516c6f1b36e3f0f6 Mon Sep 17 00:00:00 2001 From: Mikko Viitanen Date: Fri, 17 Mar 2023 16:04:43 +0200 Subject: [PATCH 4/6] Fix lint --- CHANGELOG.md | 1 - 1 file changed, 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c85355c694..745842ad67 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,7 +16,6 @@ release. * Add logs for Ad service and Recommendation service ([#796](https://github.com/open-telemetry/opentelemetry-demo/pull/796)) - ## v0.1.0 * The initial code base is donated from a From c343591966933f0b109f86a71ba39941d3b78049 Mon Sep 17 00:00:00 2001 From: Mikko Viitanen Date: Fri, 17 Mar 2023 18:05:04 +0200 Subject: [PATCH 5/6] Move protocol env variables to .env --- .env | 5 +++++ docker-compose.yml | 6 +++--- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/.env b/.env index 09ee9f7a4c..59fae6c33f 100644 --- a/.env +++ b/.env @@ -28,6 +28,11 @@ OTEL_EXPORTER_OTLP_METRICS_ENDPOINT=${OTEL_EXPORTER_OTLP_ENDPOINT} OTEL_EXPORTER_OTLP_LOGS_ENDPOINT=${OTEL_EXPORTER_OTLP_ENDPOINT} PUBLIC_OTEL_EXPORTER_OTLP_TRACES_ENDPOINT=http://localhost:4318/v1/traces +# Exporter protocol configuration +OTEL_TRACES_EXPORTER=otlp +OTEL_METRICS_EXPORTER=otlp +OTEL_LOGS_EXPORTER=otlp + # OpenTelemetry Resource Definitions OTEL_RESOURCE_ATTRIBUTES="service.namespace=opentelemetry-demo" diff --git a/docker-compose.yml b/docker-compose.yml index 5910db9568..2b15f8fed1 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -81,7 +81,7 @@ services: - OTEL_EXPORTER_OTLP_LOGS_ENDPOINT - OTEL_RESOURCE_ATTRIBUTES - OTEL_SERVICE_NAME=adservice - - OTEL_LOGS_EXPORTER=otlp + - OTEL_LOGS_EXPORTER depends_on: - otelcol logging: *logging @@ -487,8 +487,8 @@ services: - PRODUCT_CATALOG_SERVICE_ADDR - FEATURE_FLAG_GRPC_SERVICE_ADDR - OTEL_PYTHON_LOG_CORRELATION=true - - OTEL_TRACES_EXPORTER=otlp - - OTEL_METRICS_EXPORTER=otlp + - OTEL_TRACES_EXPORTER + - OTEL_METRICS_EXPORTER - OTEL_EXPORTER_OTLP_ENDPOINT - OTEL_EXPORTER_OTLP_METRICS_TEMPORALITY_PREFERENCE - OTEL_RESOURCE_ATTRIBUTES From 12f1c9bb09d2b9bb3d7c987979ef292f87c666ca Mon Sep 17 00:00:00 2001 From: Juliano Costa Date: Mon, 20 Mar 2023 08:39:22 +0100 Subject: [PATCH 6/6] Update CHANGELOG.md --- CHANGELOG.md | 1 - 1 file changed, 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 11097789b4..c98d4decec 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -18,7 +18,6 @@ release. * Add logs for Ad service and Recommendation service ([#796](https://github.com/open-telemetry/opentelemetry-demo/pull/796)) - ## v0.1.0 * The initial code base is donated from a