diff --git a/CHANGELOG.md b/CHANGELOG.md index 4fb2671c8a..c872762280 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -39,7 +39,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Add metric instrumentation for WSGI ([#1128](https://github.com/open-telemetry/opentelemetry-python-contrib/pull/1128)) - `opentelemetry-instrumentation-requests` Restoring metrics in requests - ([#1110](https://github.com/open-telemetry/opentelemetry-python-contrib/pull/1110) + ([#1110](https://github.com/open-telemetry/opentelemetry-python-contrib/pull/1110)) +- Integrated sqlcommenter plugin into opentelemetry-instrumentation-django + ([#896](https://github.com/open-telemetry/opentelemetry-python-contrib/pull/896)) ## [1.12.0rc1-0.31b0](https://github.com/open-telemetry/opentelemetry-python/releases/tag/v1.12.0rc1-0.31b0) - 2022-05-17 diff --git a/instrumentation/opentelemetry-instrumentation-django/src/opentelemetry/instrumentation/django/__init__.py b/instrumentation/opentelemetry-instrumentation-django/src/opentelemetry/instrumentation/django/__init__.py index ad6fa7bf36..4b8dec4e64 100644 --- a/instrumentation/opentelemetry-instrumentation-django/src/opentelemetry/instrumentation/django/__init__.py +++ b/instrumentation/opentelemetry-instrumentation-django/src/opentelemetry/instrumentation/django/__init__.py @@ -17,6 +17,68 @@ .. _django: https://pypi.org/project/django/ +SQLCOMMENTER +***************************************** +You can optionally configure Django instrumentation to enable sqlcommenter which enriches +the query with contextual information. + +Usage +----- + +.. code:: python + + from opentelemetry.instrumentation.django import DjangoInstrumentor + + DjangoInstrumentor().instrument(is_sql_commentor_enabled=True) + + +For example, +:: + + Invoking Users().objects.all() will lead to sql query "select * from auth_users" but when SQLCommenter is enabled + the query will get appended with some configurable tags like "select * from auth_users /*metrics=value*/;" + + +SQLCommenter Configurations +*************************** +We can configure the tags to be appended to the sqlquery log by adding below variables to the settings.py + +SQLCOMMENTER_WITH_FRAMEWORK = True(Default) or False + +For example, +:: +Enabling this flag will add django framework and it's version which is /*framework='django%3A2.2.3*/ + +SQLCOMMENTER_WITH_CONTROLLER = True(Default) or False + +For example, +:: +Enabling this flag will add controller name that handles the request /*controller='index'*/ + +SQLCOMMENTER_WITH_ROUTE = True(Default) or False + +For example, +:: +Enabling this flag will add url path that handles the request /*route='polls/'*/ + +SQLCOMMENTER_WITH_APP_NAME = True(Default) or False + +For example, +:: +Enabling this flag will add app name that handles the request /*app_name='polls'*/ + +SQLCOMMENTER_WITH_OPENTELEMETRY = True(Default) or False + +For example, +:: +Enabling this flag will add opentelemetry traceparent /*traceparent='00-fd720cffceba94bbf75940ff3caaf3cc-4fd1a2bdacf56388-01'*/ + +SQLCOMMENTER_WITH_DB_DRIVER = True(Default) or False + +For example, +:: +Enabling this flag will add name of the db driver /*db_driver='django.db.backends.postgresql'*/ + Usage ----- @@ -124,6 +186,7 @@ def response_hook(span, request, response): API --- + """ from logging import getLogger @@ -136,7 +199,9 @@ def response_hook(span, request, response): from opentelemetry.instrumentation.django.environment_variables import ( OTEL_PYTHON_DJANGO_INSTRUMENT, ) -from opentelemetry.instrumentation.django.middleware import _DjangoMiddleware +from opentelemetry.instrumentation.django.middleware.otel_middleware import ( + _DjangoMiddleware, +) from opentelemetry.instrumentation.django.package import _instruments from opentelemetry.instrumentation.django.version import __version__ from opentelemetry.instrumentation.instrumentor import BaseInstrumentor @@ -166,6 +231,8 @@ class DjangoInstrumentor(BaseInstrumentor): [_DjangoMiddleware.__module__, _DjangoMiddleware.__qualname__] ) + _sql_commenter_middleware = "opentelemetry.instrumentation.django.middleware.sqlcommenter_middleware.SqlCommenter" + def instrumentation_dependencies(self) -> Collection[str]: return _instruments @@ -204,7 +271,13 @@ def _instrument(self, **kwargs): if isinstance(settings_middleware, tuple): settings_middleware = list(settings_middleware) + is_sql_commentor_enabled = kwargs.pop("is_sql_commentor_enabled", None) + + if is_sql_commentor_enabled: + settings_middleware.insert(0, self._sql_commenter_middleware) + settings_middleware.insert(0, self._opentelemetry_middleware) + setattr(settings, _middleware_setting, settings_middleware) def _uninstrument(self, **kwargs): diff --git a/instrumentation/opentelemetry-instrumentation-django/src/opentelemetry/instrumentation/django/middleware/__init__.py b/instrumentation/opentelemetry-instrumentation-django/src/opentelemetry/instrumentation/django/middleware/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/instrumentation/opentelemetry-instrumentation-django/src/opentelemetry/instrumentation/django/middleware.py b/instrumentation/opentelemetry-instrumentation-django/src/opentelemetry/instrumentation/django/middleware/otel_middleware.py similarity index 100% rename from instrumentation/opentelemetry-instrumentation-django/src/opentelemetry/instrumentation/django/middleware.py rename to instrumentation/opentelemetry-instrumentation-django/src/opentelemetry/instrumentation/django/middleware/otel_middleware.py diff --git a/instrumentation/opentelemetry-instrumentation-django/src/opentelemetry/instrumentation/django/middleware/sqlcommenter_middleware.py b/instrumentation/opentelemetry-instrumentation-django/src/opentelemetry/instrumentation/django/middleware/sqlcommenter_middleware.py new file mode 100644 index 0000000000..556bd92938 --- /dev/null +++ b/instrumentation/opentelemetry-instrumentation-django/src/opentelemetry/instrumentation/django/middleware/sqlcommenter_middleware.py @@ -0,0 +1,153 @@ +#!/usr/bin/python +# +# Copyright The OpenTelemetry Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from logging import getLogger +from typing import Any, Type, TypeVar +from urllib.parse import quote as urllib_quote + +# pylint: disable=no-name-in-module +from django import conf, get_version +from django.db import connection +from django.db.backends.utils import CursorDebugWrapper + +from opentelemetry.trace.propagation.tracecontext import ( + TraceContextTextMapPropagator, +) + +_propagator = TraceContextTextMapPropagator() + +_django_version = get_version() +_logger = getLogger(__name__) + +T = TypeVar("T") # pylint: disable-msg=invalid-name + + +class SqlCommenter: + """ + Middleware to append a comment to each database query with details about + the framework and the execution context. + """ + + def __init__(self, get_response) -> None: + self.get_response = get_response + + def __call__(self, request) -> Any: + with connection.execute_wrapper(_QueryWrapper(request)): + return self.get_response(request) + + +class _QueryWrapper: + def __init__(self, request) -> None: + self.request = request + + def __call__(self, execute: Type[T], sql, params, many, context) -> T: + # pylint: disable-msg=too-many-locals + with_framework = getattr( + conf.settings, "SQLCOMMENTER_WITH_FRAMEWORK", True + ) + with_controller = getattr( + conf.settings, "SQLCOMMENTER_WITH_CONTROLLER", True + ) + with_route = getattr(conf.settings, "SQLCOMMENTER_WITH_ROUTE", True) + with_app_name = getattr( + conf.settings, "SQLCOMMENTER_WITH_APP_NAME", True + ) + with_opentelemetry = getattr( + conf.settings, "SQLCOMMENTER_WITH_OPENTELEMETRY", True + ) + with_db_driver = getattr( + conf.settings, "SQLCOMMENTER_WITH_DB_DRIVER", True + ) + + db_driver = context["connection"].settings_dict.get("ENGINE", "") + resolver_match = self.request.resolver_match + + sql_comment = _generate_sql_comment( + # Information about the controller. + controller=resolver_match.view_name + if resolver_match and with_controller + else None, + # route is the pattern that matched a request with a controller i.e. the regex + # See https://docs.djangoproject.com/en/stable/ref/urlresolvers/#django.urls.ResolverMatch.route + # getattr() because the attribute doesn't exist in Django < 2.2. + route=getattr(resolver_match, "route", None) + if resolver_match and with_route + else None, + # app_name is the application namespace for the URL pattern that matches the URL. + # See https://docs.djangoproject.com/en/stable/ref/urlresolvers/#django.urls.ResolverMatch.app_name + app_name=(resolver_match.app_name or None) + if resolver_match and with_app_name + else None, + # Framework centric information. + framework=f"django:{_django_version}" if with_framework else None, + # Information about the database and driver. + db_driver=db_driver if with_db_driver else None, + **_get_opentelemetry_values() if with_opentelemetry else {}, + ) + + # TODO: MySQL truncates logs > 1024B so prepend comments + # instead of statements, if the engine is MySQL. + # See: + # * https://github.com/basecamp/marginalia/issues/61 + # * https://github.com/basecamp/marginalia/pull/80 + sql += sql_comment + + # Add the query to the query log if debugging. + if context["cursor"].__class__ is CursorDebugWrapper: + context["connection"].queries_log.append(sql) + + return execute(sql, params, many, context) + + +def _generate_sql_comment(**meta) -> str: + """ + Return a SQL comment with comma delimited key=value pairs created from + **meta kwargs. + """ + key_value_delimiter = "," + + if not meta: # No entries added. + return "" + + # Sort the keywords to ensure that caching works and that testing is + # deterministic. It eases visual inspection as well. + return ( + " /*" + + key_value_delimiter.join( + f"{_url_quote(key)}={_url_quote(value)!r}" + for key, value in sorted(meta.items()) + if value is not None + ) + + "*/" + ) + + +def _url_quote(value) -> str: + if not isinstance(value, (str, bytes)): + return value + _quoted = urllib_quote(value) + # Since SQL uses '%' as a keyword, '%' is a by-product of url quoting + # e.g. foo,bar --> foo%2Cbar + # thus in our quoting, we need to escape it too to finally give + # foo,bar --> foo%%2Cbar + return _quoted.replace("%", "%%") + + +def _get_opentelemetry_values() -> dict or None: + """ + Return the OpenTelemetry Trace and Span IDs if Span ID is set in the + OpenTelemetry execution context. + """ + return _propagator.inject({}) diff --git a/instrumentation/opentelemetry-instrumentation-django/tests/test_middleware.py b/instrumentation/opentelemetry-instrumentation-django/tests/test_middleware.py index a0d4e793c6..a40a7b82ee 100644 --- a/instrumentation/opentelemetry-instrumentation-django/tests/test_middleware.py +++ b/instrumentation/opentelemetry-instrumentation-django/tests/test_middleware.py @@ -102,11 +102,11 @@ def setUp(self): ) self.env_patch.start() self.exclude_patch = patch( - "opentelemetry.instrumentation.django.middleware._DjangoMiddleware._excluded_urls", + "opentelemetry.instrumentation.django.middleware.otel_middleware._DjangoMiddleware._excluded_urls", get_excluded_urls("DJANGO"), ) self.traced_patch = patch( - "opentelemetry.instrumentation.django.middleware._DjangoMiddleware._traced_request_attrs", + "opentelemetry.instrumentation.django.middleware.otel_middleware._DjangoMiddleware._traced_request_attrs", get_traced_request_attrs("DJANGO"), ) self.exclude_patch.start() diff --git a/instrumentation/opentelemetry-instrumentation-django/tests/test_middleware_asgi.py b/instrumentation/opentelemetry-instrumentation-django/tests/test_middleware_asgi.py index 14a1ce82a9..941fda49bb 100644 --- a/instrumentation/opentelemetry-instrumentation-django/tests/test_middleware_asgi.py +++ b/instrumentation/opentelemetry-instrumentation-django/tests/test_middleware_asgi.py @@ -104,11 +104,11 @@ def setUp(self): ) self.env_patch.start() self.exclude_patch = patch( - "opentelemetry.instrumentation.django.middleware._DjangoMiddleware._excluded_urls", + "opentelemetry.instrumentation.django.middleware.otel_middleware._DjangoMiddleware._excluded_urls", get_excluded_urls("DJANGO"), ) self.traced_patch = patch( - "opentelemetry.instrumentation.django.middleware._DjangoMiddleware._traced_request_attrs", + "opentelemetry.instrumentation.django.middleware.otel_middleware._DjangoMiddleware._traced_request_attrs", get_traced_request_attrs("DJANGO"), ) self.exclude_patch.start() diff --git a/instrumentation/opentelemetry-instrumentation-django/tests/test_sqlcommenter.py b/instrumentation/opentelemetry-instrumentation-django/tests/test_sqlcommenter.py new file mode 100644 index 0000000000..682dd5f4e9 --- /dev/null +++ b/instrumentation/opentelemetry-instrumentation-django/tests/test_sqlcommenter.py @@ -0,0 +1,100 @@ +# Copyright The OpenTelemetry Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# pylint: disable=no-name-in-module + +from unittest.mock import MagicMock, patch + +from django import VERSION, conf +from django.http import HttpResponse +from django.test.utils import setup_test_environment, teardown_test_environment + +from opentelemetry.instrumentation.django import DjangoInstrumentor +from opentelemetry.instrumentation.django.middleware.sqlcommenter_middleware import ( + _QueryWrapper, +) +from opentelemetry.test.wsgitestutil import WsgiTestBase + +DJANGO_2_0 = VERSION >= (2, 0) + +_django_instrumentor = DjangoInstrumentor() + + +class TestMiddleware(WsgiTestBase): + @classmethod + def setUpClass(cls): + conf.settings.configure( + SQLCOMMENTER_WITH_FRAMEWORK=False, + SQLCOMMENTER_WITH_DB_DRIVER=False, + ) + super().setUpClass() + + def setUp(self): + super().setUp() + setup_test_environment() + _django_instrumentor.instrument(is_sql_commentor_enabled=True) + + def tearDown(self): + super().tearDown() + teardown_test_environment() + _django_instrumentor.uninstrument() + + @classmethod + def tearDownClass(cls): + super().tearDownClass() + conf.settings = conf.LazySettings() + + @patch( + "opentelemetry.instrumentation.django.middleware.sqlcommenter_middleware.SqlCommenter" + ) + def test_middleware_added(self, sqlcommenter_middleware): + instance = sqlcommenter_middleware.return_value + instance.get_response = HttpResponse() + if DJANGO_2_0: + middleware = conf.settings.MIDDLEWARE + else: + middleware = conf.settings.MIDDLEWARE_CLASSES + + self.assertTrue( + "opentelemetry.instrumentation.django.middleware.sqlcommenter_middleware.SqlCommenter" + in middleware + ) + + @patch( + "opentelemetry.instrumentation.django.middleware.sqlcommenter_middleware._get_opentelemetry_values" + ) + def test_query_wrapper(self, trace_capture): + requests_mock = MagicMock() + requests_mock.resolver_match.view_name = "view" + requests_mock.resolver_match.route = "route" + requests_mock.resolver_match.app_name = "app" + + trace_capture.return_value = { + "traceparent": "*traceparent='00-000000000000000000000000deadbeef-000000000000beef-00" + } + qw_instance = _QueryWrapper(requests_mock) + execute_mock_obj = MagicMock() + qw_instance( + execute_mock_obj, + "Select 1", + MagicMock("test"), + MagicMock("test1"), + MagicMock(), + ) + output_sql = execute_mock_obj.call_args[0][0] + self.assertEqual( + output_sql, + "Select 1 /*app_name='app',controller='view',route='route',traceparent='%%2Atraceparent%%3D%%2700-0000000" + "00000000000000000deadbeef-000000000000beef-00'*/", + )