From fce6f5e4dc64217874717129f95c08ad570bb38a Mon Sep 17 00:00:00 2001 From: Daniel Sanche Date: Tue, 27 Sep 2022 15:03:30 -0700 Subject: [PATCH 1/5] structured log handler drops reserved fields in json_fields --- google/cloud/logging_v2/handlers/structured_log.py | 11 +++++++++++ tests/environment | 2 +- 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/google/cloud/logging_v2/handlers/structured_log.py b/google/cloud/logging_v2/handlers/structured_log.py index 4a9a139e5..6e0791fb1 100644 --- a/google/cloud/logging_v2/handlers/structured_log.py +++ b/google/cloud/logging_v2/handlers/structured_log.py @@ -36,6 +36,13 @@ "}" ) +GCP_STRUCTURED_LOGGING_FIELDS = frozenset({ + "severity", "httpRequest", "time", "timestamp", + "timestampSeconds", "timestampNanos", "logging.googleapis.com/insertId", + "logging.googleapis.com/labels", "logging.googleapis.com/operation", + "logging.googleapis.com/sourceLocation", "logging.googleapis.com/spanId", + "logging.googleapis.com/trace", "logging.googleapis.com/trace_sampled" + }) class StructuredLogHandler(logging.StreamHandler): """Handler to format logs into the Cloud Logging structured log format, @@ -70,6 +77,10 @@ def format(self, record): message = _format_and_parse_message(record, super(StructuredLogHandler, self)) if isinstance(message, collections.abc.Mapping): + # remove any special fields + for key in list(message.keys()): + if key in GCP_STRUCTURED_LOGGING_FIELDS: + del message[key] # if input is a dictionary, encode it as a json string encoded_msg = json.dumps(message, ensure_ascii=False) # strip out open and close parentheses diff --git a/tests/environment b/tests/environment index 3845d3c48..d67a68bb8 160000 --- a/tests/environment +++ b/tests/environment @@ -1 +1 @@ -Subproject commit 3845d3c489f45e9366b5fb121846acc7893060b1 +Subproject commit d67a68bb81b33143ddf371d20fa4c2ce4d0c2199 From 81d125e5dac6a6853e2f7308444c5d46ae9634b5 Mon Sep 17 00:00:00 2001 From: Daniel Sanche Date: Tue, 27 Sep 2022 15:17:14 -0700 Subject: [PATCH 2/5] added unit test --- tests/unit/handlers/test_structured_log.py | 46 ++++++++++++++++++++++ 1 file changed, 46 insertions(+) diff --git a/tests/unit/handlers/test_structured_log.py b/tests/unit/handlers/test_structured_log.py index d2d570e21..da53e7a97 100644 --- a/tests/unit/handlers/test_structured_log.py +++ b/tests/unit/handlers/test_structured_log.py @@ -207,6 +207,52 @@ def test_format_with_custom_formatter(self): self.assertIn(expected_result, result) self.assertIn("message", result) + def test_format_with_reserved_json_field(self): + # drop json_field data with reserved names + # related issue: https://github.com/googleapis/python-logging/issues/543 + import logging + import json + + handler = self._make_one() + message = "hello world" + extra = "still here" + json_fields = { + "message": "override?", + "severity": "error", + "logging.googleapis.com/trace_sampled": True, + "time": "none", + "extra": extra, + } + record = logging.LogRecord( + None, + logging.INFO, + None, + None, + message, + None, + None, + ) + record.created = None + setattr(record, "json_fields", json_fields) + expected_payload = { + "message": message, + "severity": "INFO", + "logging.googleapis.com/trace": "", + "logging.googleapis.com/spanId": "", + "logging.googleapis.com/trace_sampled": False, + "logging.googleapis.com/sourceLocation": {}, + "httpRequest": {}, + "logging.googleapis.com/labels": {}, + "extra": extra, + } + handler.filter(record) + result = json.loads(handler.format(record)) + self.assertEqual(set(expected_payload.keys()), set(result.keys())) + for (key, value) in expected_payload.items(): + self.assertEqual( + value, result[key], f"expected_payload[{key}] != result[{key}]" + ) + def test_dict(self): """ Handler should parse json encoded as a string From 9f2ae651ae89fa979ae65e2c06669bd026730431 Mon Sep 17 00:00:00 2001 From: Daniel Sanche Date: Tue, 27 Sep 2022 15:22:17 -0700 Subject: [PATCH 3/5] rand blacken --- .../logging_v2/handlers/structured_log.py | 25 +++++++++++++------ 1 file changed, 18 insertions(+), 7 deletions(-) diff --git a/google/cloud/logging_v2/handlers/structured_log.py b/google/cloud/logging_v2/handlers/structured_log.py index 6e0791fb1..03bf9e818 100644 --- a/google/cloud/logging_v2/handlers/structured_log.py +++ b/google/cloud/logging_v2/handlers/structured_log.py @@ -36,13 +36,24 @@ "}" ) -GCP_STRUCTURED_LOGGING_FIELDS = frozenset({ - "severity", "httpRequest", "time", "timestamp", - "timestampSeconds", "timestampNanos", "logging.googleapis.com/insertId", - "logging.googleapis.com/labels", "logging.googleapis.com/operation", - "logging.googleapis.com/sourceLocation", "logging.googleapis.com/spanId", - "logging.googleapis.com/trace", "logging.googleapis.com/trace_sampled" - }) +GCP_STRUCTURED_LOGGING_FIELDS = frozenset( + { + "severity", + "httpRequest", + "time", + "timestamp", + "timestampSeconds", + "timestampNanos", + "logging.googleapis.com/insertId", + "logging.googleapis.com/labels", + "logging.googleapis.com/operation", + "logging.googleapis.com/sourceLocation", + "logging.googleapis.com/spanId", + "logging.googleapis.com/trace", + "logging.googleapis.com/trace_sampled", + } +) + class StructuredLogHandler(logging.StreamHandler): """Handler to format logs into the Cloud Logging structured log format, From 630df7a696a02bf8ebec3dfc08c901d04358ea0c Mon Sep 17 00:00:00 2001 From: Daniel Sanche Date: Thu, 29 Sep 2022 13:35:31 -0700 Subject: [PATCH 4/5] added comment --- google/cloud/logging_v2/handlers/structured_log.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/google/cloud/logging_v2/handlers/structured_log.py b/google/cloud/logging_v2/handlers/structured_log.py index 03bf9e818..bfaebdab5 100644 --- a/google/cloud/logging_v2/handlers/structured_log.py +++ b/google/cloud/logging_v2/handlers/structured_log.py @@ -36,6 +36,8 @@ "}" ) +# reserved fields taken from Structured Logging documentation: +# https://cloud.google.com/logging/docs/structured-logging GCP_STRUCTURED_LOGGING_FIELDS = frozenset( { "severity", From 9e8c20dfd08a878af159ef75579f2318ed403f34 Mon Sep 17 00:00:00 2001 From: Daniel Sanche Date: Thu, 29 Sep 2022 13:39:11 -0700 Subject: [PATCH 5/5] tested with caps --- tests/unit/handlers/test_structured_log.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/unit/handlers/test_structured_log.py b/tests/unit/handlers/test_structured_log.py index da53e7a97..77bb97ab6 100644 --- a/tests/unit/handlers/test_structured_log.py +++ b/tests/unit/handlers/test_structured_log.py @@ -222,6 +222,7 @@ def test_format_with_reserved_json_field(self): "logging.googleapis.com/trace_sampled": True, "time": "none", "extra": extra, + "SEVERITY": "error", } record = logging.LogRecord( None, @@ -237,6 +238,7 @@ def test_format_with_reserved_json_field(self): expected_payload = { "message": message, "severity": "INFO", + "SEVERITY": "error", "logging.googleapis.com/trace": "", "logging.googleapis.com/spanId": "", "logging.googleapis.com/trace_sampled": False,