diff --git a/.github/workflows/test-integrations-databases.yml b/.github/workflows/test-integrations-databases.yml index 50d02b72f7..5683bfbd95 100644 --- a/.github/workflows/test-integrations-databases.yml +++ b/.github/workflows/test-integrations-databases.yml @@ -77,10 +77,10 @@ jobs: run: | set -x # print commands that are executed ./scripts/runtox.sh "py${{ matrix.python-version }}-redis-latest" --cov=tests --cov=sentry_sdk --cov-report= --cov-branch - - name: Test rediscluster latest + - name: Test redis_py_cluster_legacy latest run: | set -x # print commands that are executed - ./scripts/runtox.sh "py${{ matrix.python-version }}-rediscluster-latest" --cov=tests --cov=sentry_sdk --cov-report= --cov-branch + ./scripts/runtox.sh "py${{ matrix.python-version }}-redis_py_cluster_legacy-latest" --cov=tests --cov=sentry_sdk --cov-report= --cov-branch - name: Test sqlalchemy latest run: | set -x # print commands that are executed @@ -152,10 +152,10 @@ jobs: run: | set -x # print commands that are executed ./scripts/runtox.sh --exclude-latest "py${{ matrix.python-version }}-redis" --cov=tests --cov=sentry_sdk --cov-report= --cov-branch - - name: Test rediscluster pinned + - name: Test redis_py_cluster_legacy pinned run: | set -x # print commands that are executed - ./scripts/runtox.sh --exclude-latest "py${{ matrix.python-version }}-rediscluster" --cov=tests --cov=sentry_sdk --cov-report= --cov-branch + ./scripts/runtox.sh --exclude-latest "py${{ matrix.python-version }}-redis_py_cluster_legacy" --cov=tests --cov=sentry_sdk --cov-report= --cov-branch - name: Test sqlalchemy pinned run: | set -x # print commands that are executed diff --git a/scripts/split-tox-gh-actions/split-tox-gh-actions.py b/scripts/split-tox-gh-actions/split-tox-gh-actions.py index 9842ff6d39..a4e4038156 100755 --- a/scripts/split-tox-gh-actions/split-tox-gh-actions.py +++ b/scripts/split-tox-gh-actions/split-tox-gh-actions.py @@ -82,7 +82,7 @@ "clickhouse_driver", "pymongo", "redis", - "rediscluster", + "redis_py_cluster_legacy", "sqlalchemy", ], "GraphQL": [ diff --git a/sentry_sdk/consts.py b/sentry_sdk/consts.py index 8cdccc8a53..3829d1278a 100644 --- a/sentry_sdk/consts.py +++ b/sentry_sdk/consts.py @@ -368,7 +368,7 @@ class SPANDATA: class OP: ANTHROPIC_MESSAGES_CREATE = "ai.messages.create.anthropic" CACHE_GET = "cache.get" - CACHE_SET = "cache.set" + CACHE_PUT = "cache.put" COHERE_CHAT_COMPLETIONS_CREATE = "ai.chat_completions.create.cohere" COHERE_EMBEDDINGS_CREATE = "ai.embeddings.create.cohere" DB = "db" diff --git a/sentry_sdk/integrations/django/caching.py b/sentry_sdk/integrations/django/caching.py index 1529aa8a7a..8f5b1b9229 100644 --- a/sentry_sdk/integrations/django/caching.py +++ b/sentry_sdk/integrations/django/caching.py @@ -1,5 +1,6 @@ import functools from typing import TYPE_CHECKING +from sentry_sdk.integrations.redis.utils import _get_safe_key from urllib3.util import parse_url as urlparse from django import VERSION as DJANGO_VERSION @@ -8,7 +9,6 @@ import sentry_sdk from sentry_sdk.consts import OP, SPANDATA from sentry_sdk.utils import ( - SENSITIVE_DATA_SUBSTITUTE, capture_internal_exceptions, ensure_integration_enabled, ) @@ -28,27 +28,9 @@ ] -def _get_key(args, kwargs): - # type: (list[Any], dict[str, Any]) -> str - key = "" - - if args is not None and len(args) >= 1: - key = args[0] - elif kwargs is not None and "key" in kwargs: - key = kwargs["key"] - - if isinstance(key, dict): - # Do not leak sensitive data - # `set_many()` has a dict {"key1": "value1", "key2": "value2"} as first argument. - # Those values could include sensitive data so we replace them with a placeholder - key = {x: SENSITIVE_DATA_SUBSTITUTE for x in key} - - return str(key) - - def _get_span_description(method_name, args, kwargs): - # type: (str, list[Any], dict[str, Any]) -> str - return _get_key(args, kwargs) + # type: (str, tuple[Any], dict[str, Any]) -> str + return _get_safe_key(method_name, args, kwargs) def _patch_cache_method(cache, method_name, address, port): @@ -61,11 +43,11 @@ def _patch_cache_method(cache, method_name, address, port): def _instrument_call( cache, method_name, original_method, args, kwargs, address, port ): - # type: (CacheHandler, str, Callable[..., Any], list[Any], dict[str, Any], Optional[str], Optional[int]) -> Any + # type: (CacheHandler, str, Callable[..., Any], tuple[Any, ...], dict[str, Any], Optional[str], Optional[int]) -> Any is_set_operation = method_name.startswith("set") is_get_operation = not is_set_operation - op = OP.CACHE_SET if is_set_operation else OP.CACHE_GET + op = OP.CACHE_PUT if is_set_operation else OP.CACHE_GET description = _get_span_description(method_name, args, kwargs) with sentry_sdk.start_span(op=op, description=description) as span: @@ -78,7 +60,7 @@ def _instrument_call( if port is not None: span.set_data(SPANDATA.NETWORK_PEER_PORT, port) - key = _get_key(args, kwargs) + key = _get_safe_key(method_name, args, kwargs) if key != "": span.set_data(SPANDATA.CACHE_KEY, key) diff --git a/sentry_sdk/integrations/redis/__init__.py b/sentry_sdk/integrations/redis/__init__.py index 725290407b..dded1bdcc0 100644 --- a/sentry_sdk/integrations/redis/__init__.py +++ b/sentry_sdk/integrations/redis/__init__.py @@ -1,365 +1,23 @@ -import sentry_sdk -from sentry_sdk.consts import OP, SPANDATA -from sentry_sdk.hub import _should_send_default_pii -from sentry_sdk.integrations import Integration, DidNotEnable from sentry_sdk._types import TYPE_CHECKING -from sentry_sdk.utils import ( - SENSITIVE_DATA_SUBSTITUTE, - capture_internal_exceptions, - ensure_integration_enabled, - logger, -) +from sentry_sdk.integrations import Integration, DidNotEnable +from sentry_sdk.integrations.redis.consts import _DEFAULT_MAX_DATA_SIZE +from sentry_sdk.integrations.redis.rb import _patch_rb +from sentry_sdk.integrations.redis.redis import _patch_redis +from sentry_sdk.integrations.redis.redis_cluster import _patch_redis_cluster +from sentry_sdk.integrations.redis.redis_py_cluster_legacy import _patch_rediscluster +from sentry_sdk.utils import logger if TYPE_CHECKING: - from collections.abc import Callable - from typing import Any, Dict, Sequence - from redis import Redis, RedisCluster - from redis.asyncio.cluster import ( - RedisCluster as AsyncRedisCluster, - ClusterPipeline as AsyncClusterPipeline, - ) - from sentry_sdk.tracing import Span - -_SINGLE_KEY_COMMANDS = frozenset( - ["decr", "decrby", "get", "incr", "incrby", "pttl", "set", "setex", "setnx", "ttl"], -) -_MULTI_KEY_COMMANDS = frozenset( - ["del", "touch", "unlink"], -) -_COMMANDS_INCLUDING_SENSITIVE_DATA = [ - "auth", -] -_MAX_NUM_ARGS = 10 # Trim argument lists to this many values -_MAX_NUM_COMMANDS = 10 # Trim command lists to this many values -_DEFAULT_MAX_DATA_SIZE = 1024 - - -def _get_safe_command(name, args): - # type: (str, Sequence[Any]) -> str - command_parts = [name] - - for i, arg in enumerate(args): - if i > _MAX_NUM_ARGS: - break - - name_low = name.lower() - - if name_low in _COMMANDS_INCLUDING_SENSITIVE_DATA: - command_parts.append(SENSITIVE_DATA_SUBSTITUTE) - continue - - arg_is_the_key = i == 0 - if arg_is_the_key: - command_parts.append(repr(arg)) - - else: - if _should_send_default_pii(): - command_parts.append(repr(arg)) - else: - command_parts.append(SENSITIVE_DATA_SUBSTITUTE) - - command = " ".join(command_parts) - return command - - -def _get_span_description(name, *args): - # type: (str, *Any) -> str - description = name - - with capture_internal_exceptions(): - description = _get_safe_command(name, args) - - return description - - -def _get_redis_command_args(command): - # type: (Any) -> Sequence[Any] - return command[0] - - -def _parse_rediscluster_command(command): - # type: (Any) -> Sequence[Any] - return command.args - - -def _set_pipeline_data( - span, is_cluster, get_command_args_fn, is_transaction, command_stack -): - # type: (Span, bool, Any, bool, Sequence[Any]) -> None - span.set_tag("redis.is_cluster", is_cluster) - span.set_tag("redis.transaction", is_transaction) - - commands = [] - for i, arg in enumerate(command_stack): - if i >= _MAX_NUM_COMMANDS: - break - - command = get_command_args_fn(arg) - commands.append(_get_safe_command(command[0], command[1:])) - - span.set_data( - "redis.commands", - { - "count": len(command_stack), - "first_ten": commands, - }, - ) - - -def _set_client_data(span, is_cluster, name, *args): - # type: (Span, bool, str, *Any) -> None - span.set_tag("redis.is_cluster", is_cluster) - if name: - span.set_tag("redis.command", name) - span.set_tag(SPANDATA.DB_OPERATION, name) - - if name and args: - name_low = name.lower() - if (name_low in _SINGLE_KEY_COMMANDS) or ( - name_low in _MULTI_KEY_COMMANDS and len(args) == 1 - ): - span.set_tag("redis.key", args[0]) - - -def _set_db_data_on_span(span, connection_params): - # type: (Span, Dict[str, Any]) -> None - span.set_data(SPANDATA.DB_SYSTEM, "redis") - - db = connection_params.get("db") - if db is not None: - span.set_data(SPANDATA.DB_NAME, str(db)) - - host = connection_params.get("host") - if host is not None: - span.set_data(SPANDATA.SERVER_ADDRESS, host) - - port = connection_params.get("port") - if port is not None: - span.set_data(SPANDATA.SERVER_PORT, port) - - -def _set_db_data(span, redis_instance): - # type: (Span, Redis[Any]) -> None - try: - _set_db_data_on_span(span, redis_instance.connection_pool.connection_kwargs) - except AttributeError: - pass # connections_kwargs may be missing in some cases - - -def _set_cluster_db_data(span, redis_cluster_instance): - # type: (Span, RedisCluster[Any]) -> None - default_node = redis_cluster_instance.get_default_node() - if default_node is not None: - _set_db_data_on_span( - span, {"host": default_node.host, "port": default_node.port} - ) - - -def _set_async_cluster_db_data(span, async_redis_cluster_instance): - # type: (Span, AsyncRedisCluster[Any]) -> None - default_node = async_redis_cluster_instance.get_default_node() - if default_node is not None and default_node.connection_kwargs is not None: - _set_db_data_on_span(span, default_node.connection_kwargs) - - -def _set_async_cluster_pipeline_db_data(span, async_redis_cluster_pipeline_instance): - # type: (Span, AsyncClusterPipeline[Any]) -> None - with capture_internal_exceptions(): - _set_async_cluster_db_data( - span, - # the AsyncClusterPipeline has always had a `_client` attr but it is private so potentially problematic and mypy - # does not recognize it - see https://github.com/redis/redis-py/blame/v5.0.0/redis/asyncio/cluster.py#L1386 - async_redis_cluster_pipeline_instance._client, # type: ignore[attr-defined] - ) - - -def patch_redis_pipeline(pipeline_cls, is_cluster, get_command_args_fn, set_db_data_fn): - # type: (Any, bool, Any, Callable[[Span, Any], None]) -> None - old_execute = pipeline_cls.execute - - @ensure_integration_enabled(RedisIntegration, old_execute) - def sentry_patched_execute(self, *args, **kwargs): - # type: (Any, *Any, **Any) -> Any - with sentry_sdk.start_span( - op=OP.DB_REDIS, description="redis.pipeline.execute" - ) as span: - with capture_internal_exceptions(): - set_db_data_fn(span, self) - _set_pipeline_data( - span, - is_cluster, - get_command_args_fn, - False if is_cluster else self.transaction, - self.command_stack, - ) - - return old_execute(self, *args, **kwargs) - - pipeline_cls.execute = sentry_patched_execute - - -def patch_redis_client(cls, is_cluster, set_db_data_fn): - # type: (Any, bool, Callable[[Span, Any], None]) -> None - """ - This function can be used to instrument custom redis client classes or - subclasses. - """ - old_execute_command = cls.execute_command - - @ensure_integration_enabled(RedisIntegration, old_execute_command) - def sentry_patched_execute_command(self, name, *args, **kwargs): - # type: (Any, str, *Any, **Any) -> Any - integration = sentry_sdk.get_client().get_integration(RedisIntegration) - description = _get_span_description(name, *args) - - data_should_be_truncated = ( - integration.max_data_size and len(description) > integration.max_data_size - ) - if data_should_be_truncated: - description = description[: integration.max_data_size - len("...")] + "..." - - with sentry_sdk.start_span(op=OP.DB_REDIS, description=description) as span: - set_db_data_fn(span, self) - _set_client_data(span, is_cluster, name, *args) - - return old_execute_command(self, name, *args, **kwargs) - - cls.execute_command = sentry_patched_execute_command - - -def _patch_redis(StrictRedis, client): # noqa: N803 - # type: (Any, Any) -> None - patch_redis_client(StrictRedis, is_cluster=False, set_db_data_fn=_set_db_data) - patch_redis_pipeline(client.Pipeline, False, _get_redis_command_args, _set_db_data) - try: - strict_pipeline = client.StrictPipeline - except AttributeError: - pass - else: - patch_redis_pipeline( - strict_pipeline, False, _get_redis_command_args, _set_db_data - ) - - try: - import redis.asyncio - except ImportError: - pass - else: - from sentry_sdk.integrations.redis.asyncio import ( - patch_redis_async_client, - patch_redis_async_pipeline, - ) - - patch_redis_async_client( - redis.asyncio.client.StrictRedis, - is_cluster=False, - set_db_data_fn=_set_db_data, - ) - patch_redis_async_pipeline( - redis.asyncio.client.Pipeline, - False, - _get_redis_command_args, - set_db_data_fn=_set_db_data, - ) - - -def _patch_redis_cluster(): - # type: () -> None - """Patches the cluster module on redis SDK (as opposed to rediscluster library)""" - try: - from redis import RedisCluster, cluster - except ImportError: - pass - else: - patch_redis_client(RedisCluster, True, _set_cluster_db_data) - patch_redis_pipeline( - cluster.ClusterPipeline, - True, - _parse_rediscluster_command, - _set_cluster_db_data, - ) - - try: - from redis.asyncio import cluster as async_cluster - except ImportError: - pass - else: - from sentry_sdk.integrations.redis.asyncio import ( - patch_redis_async_client, - patch_redis_async_pipeline, - ) - - patch_redis_async_client( - async_cluster.RedisCluster, - is_cluster=True, - set_db_data_fn=_set_async_cluster_db_data, - ) - patch_redis_async_pipeline( - async_cluster.ClusterPipeline, - True, - _parse_rediscluster_command, - set_db_data_fn=_set_async_cluster_pipeline_db_data, - ) - - -def _patch_rb(): - # type: () -> None - try: - import rb.clients # type: ignore - except ImportError: - pass - else: - patch_redis_client( - rb.clients.FanoutClient, is_cluster=False, set_db_data_fn=_set_db_data - ) - patch_redis_client( - rb.clients.MappingClient, is_cluster=False, set_db_data_fn=_set_db_data - ) - patch_redis_client( - rb.clients.RoutingClient, is_cluster=False, set_db_data_fn=_set_db_data - ) - - -def _patch_rediscluster(): - # type: () -> None - try: - import rediscluster # type: ignore - except ImportError: - return - - patch_redis_client( - rediscluster.RedisCluster, is_cluster=True, set_db_data_fn=_set_db_data - ) - - # up to v1.3.6, __version__ attribute is a tuple - # from v2.0.0, __version__ is a string and VERSION a tuple - version = getattr(rediscluster, "VERSION", rediscluster.__version__) - - # StrictRedisCluster was introduced in v0.2.0 and removed in v2.0.0 - # https://github.com/Grokzen/redis-py-cluster/blob/master/docs/release-notes.rst - if (0, 2, 0) < version < (2, 0, 0): - pipeline_cls = rediscluster.pipeline.StrictClusterPipeline - patch_redis_client( - rediscluster.StrictRedisCluster, - is_cluster=True, - set_db_data_fn=_set_db_data, - ) - else: - pipeline_cls = rediscluster.pipeline.ClusterPipeline - - patch_redis_pipeline( - pipeline_cls, True, _parse_rediscluster_command, set_db_data_fn=_set_db_data - ) + from typing import Optional class RedisIntegration(Integration): identifier = "redis" - def __init__(self, max_data_size=_DEFAULT_MAX_DATA_SIZE): - # type: (int) -> None + def __init__(self, max_data_size=_DEFAULT_MAX_DATA_SIZE, cache_prefixes=None): + # type: (int, Optional[list[str]]) -> None self.max_data_size = max_data_size - # TODO: add some prefix that users can set to specify a cache key - # GitHub issue: https://github.com/getsentry/sentry-python/issues/2965 + self.cache_prefixes = cache_prefixes if cache_prefixes is not None else [] @staticmethod def setup_once(): diff --git a/sentry_sdk/integrations/redis/asyncio.py b/sentry_sdk/integrations/redis/_async_common.py similarity index 55% rename from sentry_sdk/integrations/redis/asyncio.py rename to sentry_sdk/integrations/redis/_async_common.py index 6cb12b0d51..04c74cc69d 100644 --- a/sentry_sdk/integrations/redis/asyncio.py +++ b/sentry_sdk/integrations/redis/_async_common.py @@ -1,16 +1,18 @@ -import sentry_sdk +from sentry_sdk._types import TYPE_CHECKING from sentry_sdk.consts import OP -from sentry_sdk.integrations.redis import ( - RedisIntegration, - _get_span_description, +from sentry_sdk.integrations.redis.modules.caches import ( + _compile_cache_span_properties, + _set_cache_data, +) +from sentry_sdk.integrations.redis.modules.queries import _compile_db_span_properties +from sentry_sdk.integrations.redis.utils import ( _set_client_data, _set_pipeline_data, ) -from sentry_sdk._types import TYPE_CHECKING from sentry_sdk.tracing import Span -from sentry_sdk.utils import ( - capture_internal_exceptions, -) +from sentry_sdk.utils import capture_internal_exceptions +import sentry_sdk + if TYPE_CHECKING: from collections.abc import Callable @@ -25,6 +27,8 @@ def patch_redis_async_pipeline( # type: (Union[type[Pipeline[Any]], type[ClusterPipeline[Any]]], bool, Any, Callable[[Span, Any], None]) -> None old_execute = pipeline_cls.execute + from sentry_sdk.integrations.redis import RedisIntegration + async def _sentry_execute(self, *args, **kwargs): # type: (Any, *Any, **Any) -> Any if sentry_sdk.get_client().get_integration(RedisIntegration) is None: @@ -52,17 +56,48 @@ def patch_redis_async_client(cls, is_cluster, set_db_data_fn): # type: (Union[type[StrictRedis[Any]], type[RedisCluster[Any]]], bool, Callable[[Span, Any], None]) -> None old_execute_command = cls.execute_command + from sentry_sdk.integrations.redis import RedisIntegration + async def _sentry_execute_command(self, name, *args, **kwargs): # type: (Any, str, *Any, **Any) -> Any - if sentry_sdk.get_client().get_integration(RedisIntegration) is None: + integration = sentry_sdk.get_client().get_integration(RedisIntegration) + if integration is None: return await old_execute_command(self, name, *args, **kwargs) - description = _get_span_description(name, *args) + cache_properties = _compile_cache_span_properties( + name, + args, + kwargs, + integration, + ) - with sentry_sdk.start_span(op=OP.DB_REDIS, description=description) as span: - set_db_data_fn(span, self) - _set_client_data(span, is_cluster, name, *args) + cache_span = None + if cache_properties["is_cache_key"] and cache_properties["op"] is not None: + cache_span = sentry_sdk.start_span( + op=cache_properties["op"], + description=cache_properties["description"], + ) + cache_span.__enter__() - return await old_execute_command(self, name, *args, **kwargs) + db_properties = _compile_db_span_properties(integration, name, args) + + db_span = sentry_sdk.start_span( + op=db_properties["op"], + description=db_properties["description"], + ) + db_span.__enter__() + + set_db_data_fn(db_span, self) + _set_client_data(db_span, is_cluster, name, *args) + + value = await old_execute_command(self, name, *args, **kwargs) + + db_span.__exit__(None, None, None) + + if cache_span: + _set_cache_data(cache_span, self, cache_properties, value) + cache_span.__exit__(None, None, None) + + return value cls.execute_command = _sentry_execute_command # type: ignore diff --git a/sentry_sdk/integrations/redis/_sync_common.py b/sentry_sdk/integrations/redis/_sync_common.py new file mode 100644 index 0000000000..e1578b3194 --- /dev/null +++ b/sentry_sdk/integrations/redis/_sync_common.py @@ -0,0 +1,108 @@ +from sentry_sdk._types import TYPE_CHECKING +from sentry_sdk.consts import OP +from sentry_sdk.integrations.redis.modules.caches import ( + _compile_cache_span_properties, + _set_cache_data, +) +from sentry_sdk.integrations.redis.modules.queries import _compile_db_span_properties +from sentry_sdk.integrations.redis.utils import ( + _set_client_data, + _set_pipeline_data, +) +from sentry_sdk.tracing import Span +from sentry_sdk.utils import capture_internal_exceptions +import sentry_sdk + + +if TYPE_CHECKING: + from collections.abc import Callable + from typing import Any + + +def patch_redis_pipeline( + pipeline_cls, + is_cluster, + get_command_args_fn, + set_db_data_fn, +): + # type: (Any, bool, Any, Callable[[Span, Any], None]) -> None + old_execute = pipeline_cls.execute + + from sentry_sdk.integrations.redis import RedisIntegration + + def sentry_patched_execute(self, *args, **kwargs): + # type: (Any, *Any, **Any) -> Any + if sentry_sdk.get_client().get_integration(RedisIntegration) is None: + return old_execute(self, *args, **kwargs) + + with sentry_sdk.start_span( + op=OP.DB_REDIS, description="redis.pipeline.execute" + ) as span: + with capture_internal_exceptions(): + set_db_data_fn(span, self) + _set_pipeline_data( + span, + is_cluster, + get_command_args_fn, + False if is_cluster else self.transaction, + self.command_stack, + ) + + return old_execute(self, *args, **kwargs) + + pipeline_cls.execute = sentry_patched_execute + + +def patch_redis_client(cls, is_cluster, set_db_data_fn): + # type: (Any, bool, Callable[[Span, Any], None]) -> None + """ + This function can be used to instrument custom redis client classes or + subclasses. + """ + old_execute_command = cls.execute_command + + from sentry_sdk.integrations.redis import RedisIntegration + + def sentry_patched_execute_command(self, name, *args, **kwargs): + # type: (Any, str, *Any, **Any) -> Any + integration = sentry_sdk.get_client().get_integration(RedisIntegration) + if integration is None: + return old_execute_command(self, name, *args, **kwargs) + + cache_properties = _compile_cache_span_properties( + name, + args, + kwargs, + integration, + ) + + cache_span = None + if cache_properties["is_cache_key"] and cache_properties["op"] is not None: + cache_span = sentry_sdk.start_span( + op=cache_properties["op"], + description=cache_properties["description"], + ) + cache_span.__enter__() + + db_properties = _compile_db_span_properties(integration, name, args) + + db_span = sentry_sdk.start_span( + op=db_properties["op"], + description=db_properties["description"], + ) + db_span.__enter__() + + set_db_data_fn(db_span, self) + _set_client_data(db_span, is_cluster, name, *args) + + value = old_execute_command(self, name, *args, **kwargs) + + db_span.__exit__(None, None, None) + + if cache_span: + _set_cache_data(cache_span, self, cache_properties, value) + cache_span.__exit__(None, None, None) + + return value + + cls.execute_command = sentry_patched_execute_command diff --git a/sentry_sdk/integrations/redis/consts.py b/sentry_sdk/integrations/redis/consts.py new file mode 100644 index 0000000000..a8d5509714 --- /dev/null +++ b/sentry_sdk/integrations/redis/consts.py @@ -0,0 +1,17 @@ +_SINGLE_KEY_COMMANDS = frozenset( + ["decr", "decrby", "get", "incr", "incrby", "pttl", "set", "setex", "setnx", "ttl"], +) +_MULTI_KEY_COMMANDS = frozenset( + [ + "del", + "touch", + "unlink", + "mget", + ], +) +_COMMANDS_INCLUDING_SENSITIVE_DATA = [ + "auth", +] +_MAX_NUM_ARGS = 10 # Trim argument lists to this many values +_MAX_NUM_COMMANDS = 10 # Trim command lists to this many values +_DEFAULT_MAX_DATA_SIZE = 1024 diff --git a/sentry_sdk/integrations/redis/modules/__init__.py b/sentry_sdk/integrations/redis/modules/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/sentry_sdk/integrations/redis/modules/caches.py b/sentry_sdk/integrations/redis/modules/caches.py new file mode 100644 index 0000000000..31824aafa3 --- /dev/null +++ b/sentry_sdk/integrations/redis/modules/caches.py @@ -0,0 +1,114 @@ +""" +Code used for the Caches module in Sentry +""" + +from sentry_sdk._types import TYPE_CHECKING +from sentry_sdk.consts import OP, SPANDATA +from sentry_sdk.integrations.redis.utils import _get_safe_key +from sentry_sdk.utils import capture_internal_exceptions + +GET_COMMANDS = ("get", "mget") +SET_COMMANDS = ("set", "setex") + +if TYPE_CHECKING: + from sentry_sdk.integrations.redis import RedisIntegration + from sentry_sdk.tracing import Span + from typing import Any, Optional + + +def _get_op(name): + # type: (str) -> Optional[str] + op = None + if name.lower() in GET_COMMANDS: + op = OP.CACHE_GET + elif name.lower() in SET_COMMANDS: + op = OP.CACHE_PUT + + return op + + +def _compile_cache_span_properties(redis_command, args, kwargs, integration): + # type: (str, tuple[Any, ...], dict[str, Any], RedisIntegration) -> dict[str, Any] + key = _get_safe_key(redis_command, args, kwargs) + + is_cache_key = False + for prefix in integration.cache_prefixes: + if key.startswith(prefix): + is_cache_key = True + break + + value = None + if redis_command.lower() in SET_COMMANDS: + value = args[-1] + + properties = { + "op": _get_op(redis_command), + "description": _get_cache_span_description( + redis_command, args, kwargs, integration + ), + "key": key, + "redis_command": redis_command.lower(), + "is_cache_key": is_cache_key, + "value": value, + } + + return properties + + +def _get_cache_span_description(redis_command, args, kwargs, integration): + # type: (str, tuple[Any, ...], dict[str, Any], RedisIntegration) -> str + description = _get_safe_key(redis_command, args, kwargs) + + data_should_be_truncated = ( + integration.max_data_size and len(description) > integration.max_data_size + ) + if data_should_be_truncated: + description = description[: integration.max_data_size - len("...")] + "..." + + return description + + +def _set_cache_data(span, redis_client, properties, return_value): + # type: (Span, Any, dict[str, Any], Optional[Any]) -> None + with capture_internal_exceptions(): + span.set_data(SPANDATA.CACHE_KEY, properties["key"]) + + if properties["redis_command"] in GET_COMMANDS: + if return_value is not None: + span.set_data(SPANDATA.CACHE_HIT, True) + size = ( + len(str(return_value).encode("utf-8")) + if not isinstance(return_value, bytes) + else len(return_value) + ) + span.set_data(SPANDATA.CACHE_ITEM_SIZE, size) + else: + span.set_data(SPANDATA.CACHE_HIT, False) + + elif properties["redis_command"] in SET_COMMANDS: + if properties["value"] is not None: + size = ( + len(properties["value"].encode("utf-8")) + if not isinstance(properties["value"], bytes) + else len(properties["value"]) + ) + span.set_data(SPANDATA.CACHE_ITEM_SIZE, size) + + try: + connection_params = redis_client.connection_pool.connection_kwargs + except AttributeError: + # If it is a cluster, there is no connection_pool attribute so we + # need to get the default node from the cluster instance + default_node = redis_client.get_default_node() + connection_params = { + "host": default_node.host, + "port": default_node.port, + } + + host = connection_params.get("host") + if host is not None: + span.set_data(SPANDATA.NETWORK_PEER_ADDRESS, host) + + port = connection_params.get("port") + if port is not None: + span.set_data(SPANDATA.NETWORK_PEER_PORT, port) diff --git a/sentry_sdk/integrations/redis/modules/queries.py b/sentry_sdk/integrations/redis/modules/queries.py new file mode 100644 index 0000000000..79f82189ae --- /dev/null +++ b/sentry_sdk/integrations/redis/modules/queries.py @@ -0,0 +1,68 @@ +""" +Code used for the Queries module in Sentry +""" + +from sentry_sdk._types import TYPE_CHECKING +from sentry_sdk.consts import OP, SPANDATA +from sentry_sdk.integrations.redis.utils import _get_safe_command +from sentry_sdk.utils import capture_internal_exceptions + + +if TYPE_CHECKING: + from redis import Redis + from sentry_sdk.integrations.redis import RedisIntegration + from sentry_sdk.tracing import Span + from typing import Any + + +def _compile_db_span_properties(integration, redis_command, args): + # type: (RedisIntegration, str, tuple[Any, ...]) -> dict[str, Any] + description = _get_db_span_description(integration, redis_command, args) + + properties = { + "op": OP.DB_REDIS, + "description": description, + } + + return properties + + +def _get_db_span_description(integration, command_name, args): + # type: (RedisIntegration, str, tuple[Any, ...]) -> str + description = command_name + + with capture_internal_exceptions(): + description = _get_safe_command(command_name, args) + + data_should_be_truncated = ( + integration.max_data_size and len(description) > integration.max_data_size + ) + if data_should_be_truncated: + description = description[: integration.max_data_size - len("...")] + "..." + + return description + + +def _set_db_data_on_span(span, connection_params): + # type: (Span, dict[str, Any]) -> None + span.set_data(SPANDATA.DB_SYSTEM, "redis") + + db = connection_params.get("db") + if db is not None: + span.set_data(SPANDATA.DB_NAME, str(db)) + + host = connection_params.get("host") + if host is not None: + span.set_data(SPANDATA.SERVER_ADDRESS, host) + + port = connection_params.get("port") + if port is not None: + span.set_data(SPANDATA.SERVER_PORT, port) + + +def _set_db_data(span, redis_instance): + # type: (Span, Redis[Any]) -> None + try: + _set_db_data_on_span(span, redis_instance.connection_pool.connection_kwargs) + except AttributeError: + pass # connections_kwargs may be missing in some cases diff --git a/sentry_sdk/integrations/redis/rb.py b/sentry_sdk/integrations/redis/rb.py new file mode 100644 index 0000000000..1b3e2e530c --- /dev/null +++ b/sentry_sdk/integrations/redis/rb.py @@ -0,0 +1,32 @@ +""" +Instrumentation for Redis Blaster (rb) + +https://github.com/getsentry/rb +""" + +from sentry_sdk.integrations.redis._sync_common import patch_redis_client +from sentry_sdk.integrations.redis.modules.queries import _set_db_data + + +def _patch_rb(): + # type: () -> None + try: + import rb.clients # type: ignore + except ImportError: + pass + else: + patch_redis_client( + rb.clients.FanoutClient, + is_cluster=False, + set_db_data_fn=_set_db_data, + ) + patch_redis_client( + rb.clients.MappingClient, + is_cluster=False, + set_db_data_fn=_set_db_data, + ) + patch_redis_client( + rb.clients.RoutingClient, + is_cluster=False, + set_db_data_fn=_set_db_data, + ) diff --git a/sentry_sdk/integrations/redis/redis.py b/sentry_sdk/integrations/redis/redis.py new file mode 100644 index 0000000000..8359d0fcbe --- /dev/null +++ b/sentry_sdk/integrations/redis/redis.py @@ -0,0 +1,69 @@ +""" +Instrumentation for Redis + +https://github.com/redis/redis-py +""" + +from sentry_sdk._types import TYPE_CHECKING +from sentry_sdk.integrations.redis._sync_common import ( + patch_redis_client, + patch_redis_pipeline, +) +from sentry_sdk.integrations.redis.modules.queries import _set_db_data + + +if TYPE_CHECKING: + from typing import Any, Sequence + + +def _get_redis_command_args(command): + # type: (Any) -> Sequence[Any] + return command[0] + + +def _patch_redis(StrictRedis, client): # noqa: N803 + # type: (Any, Any) -> None + patch_redis_client( + StrictRedis, + is_cluster=False, + set_db_data_fn=_set_db_data, + ) + patch_redis_pipeline( + client.Pipeline, + is_cluster=False, + get_command_args_fn=_get_redis_command_args, + set_db_data_fn=_set_db_data, + ) + try: + strict_pipeline = client.StrictPipeline + except AttributeError: + pass + else: + patch_redis_pipeline( + strict_pipeline, + is_cluster=False, + get_command_args_fn=_get_redis_command_args, + set_db_data_fn=_set_db_data, + ) + + try: + import redis.asyncio + except ImportError: + pass + else: + from sentry_sdk.integrations.redis._async_common import ( + patch_redis_async_client, + patch_redis_async_pipeline, + ) + + patch_redis_async_client( + redis.asyncio.client.StrictRedis, + is_cluster=False, + set_db_data_fn=_set_db_data, + ) + patch_redis_async_pipeline( + redis.asyncio.client.Pipeline, + False, + _get_redis_command_args, + set_db_data_fn=_set_db_data, + ) diff --git a/sentry_sdk/integrations/redis/redis_cluster.py b/sentry_sdk/integrations/redis/redis_cluster.py new file mode 100644 index 0000000000..0f42032e0b --- /dev/null +++ b/sentry_sdk/integrations/redis/redis_cluster.py @@ -0,0 +1,98 @@ +""" +Instrumentation for RedisCluster +This is part of the main redis-py client. + +https://github.com/redis/redis-py/blob/master/redis/cluster.py +""" + +from sentry_sdk._types import TYPE_CHECKING +from sentry_sdk.integrations.redis._sync_common import ( + patch_redis_client, + patch_redis_pipeline, +) +from sentry_sdk.integrations.redis.modules.queries import _set_db_data_on_span +from sentry_sdk.integrations.redis.utils import _parse_rediscluster_command + +from sentry_sdk.utils import capture_internal_exceptions + +if TYPE_CHECKING: + from typing import Any + from redis import RedisCluster + from redis.asyncio.cluster import ( + RedisCluster as AsyncRedisCluster, + ClusterPipeline as AsyncClusterPipeline, + ) + from sentry_sdk.tracing import Span + + +def _set_async_cluster_db_data(span, async_redis_cluster_instance): + # type: (Span, AsyncRedisCluster[Any]) -> None + default_node = async_redis_cluster_instance.get_default_node() + if default_node is not None and default_node.connection_kwargs is not None: + _set_db_data_on_span(span, default_node.connection_kwargs) + + +def _set_async_cluster_pipeline_db_data(span, async_redis_cluster_pipeline_instance): + # type: (Span, AsyncClusterPipeline[Any]) -> None + with capture_internal_exceptions(): + _set_async_cluster_db_data( + span, + # the AsyncClusterPipeline has always had a `_client` attr but it is private so potentially problematic and mypy + # does not recognize it - see https://github.com/redis/redis-py/blame/v5.0.0/redis/asyncio/cluster.py#L1386 + async_redis_cluster_pipeline_instance._client, # type: ignore[attr-defined] + ) + + +def _set_cluster_db_data(span, redis_cluster_instance): + # type: (Span, RedisCluster[Any]) -> None + default_node = redis_cluster_instance.get_default_node() + + if default_node is not None: + connection_params = { + "host": default_node.host, + "port": default_node.port, + } + _set_db_data_on_span(span, connection_params) + + +def _patch_redis_cluster(): + # type: () -> None + """Patches the cluster module on redis SDK (as opposed to rediscluster library)""" + try: + from redis import RedisCluster, cluster + except ImportError: + pass + else: + patch_redis_client( + RedisCluster, + is_cluster=True, + set_db_data_fn=_set_cluster_db_data, + ) + patch_redis_pipeline( + cluster.ClusterPipeline, + is_cluster=True, + get_command_args_fn=_parse_rediscluster_command, + set_db_data_fn=_set_cluster_db_data, + ) + + try: + from redis.asyncio import cluster as async_cluster + except ImportError: + pass + else: + from sentry_sdk.integrations.redis._async_common import ( + patch_redis_async_client, + patch_redis_async_pipeline, + ) + + patch_redis_async_client( + async_cluster.RedisCluster, + is_cluster=True, + set_db_data_fn=_set_async_cluster_db_data, + ) + patch_redis_async_pipeline( + async_cluster.ClusterPipeline, + is_cluster=True, + get_command_args_fn=_parse_rediscluster_command, + set_db_data_fn=_set_async_cluster_pipeline_db_data, + ) diff --git a/sentry_sdk/integrations/redis/redis_py_cluster_legacy.py b/sentry_sdk/integrations/redis/redis_py_cluster_legacy.py new file mode 100644 index 0000000000..ad1c23633f --- /dev/null +++ b/sentry_sdk/integrations/redis/redis_py_cluster_legacy.py @@ -0,0 +1,50 @@ +""" +Instrumentation for redis-py-cluster +The project redis-py-cluster is EOL and was integrated into redis-py starting from version 4.1.0 (Dec 26, 2021). + +https://github.com/grokzen/redis-py-cluster +""" + +from sentry_sdk.integrations.redis._sync_common import ( + patch_redis_client, + patch_redis_pipeline, +) +from sentry_sdk.integrations.redis.modules.queries import _set_db_data +from sentry_sdk.integrations.redis.utils import _parse_rediscluster_command + + +def _patch_rediscluster(): + # type: () -> None + try: + import rediscluster # type: ignore + except ImportError: + return + + patch_redis_client( + rediscluster.RedisCluster, + is_cluster=True, + set_db_data_fn=_set_db_data, + ) + + # up to v1.3.6, __version__ attribute is a tuple + # from v2.0.0, __version__ is a string and VERSION a tuple + version = getattr(rediscluster, "VERSION", rediscluster.__version__) + + # StrictRedisCluster was introduced in v0.2.0 and removed in v2.0.0 + # https://github.com/Grokzen/redis-py-cluster/blob/master/docs/release-notes.rst + if (0, 2, 0) < version < (2, 0, 0): + pipeline_cls = rediscluster.pipeline.StrictClusterPipeline + patch_redis_client( + rediscluster.StrictRedisCluster, + is_cluster=True, + set_db_data_fn=_set_db_data, + ) + else: + pipeline_cls = rediscluster.pipeline.ClusterPipeline + + patch_redis_pipeline( + pipeline_cls, + is_cluster=True, + get_command_args_fn=_parse_rediscluster_command, + set_db_data_fn=_set_db_data, + ) diff --git a/sentry_sdk/integrations/redis/utils.py b/sentry_sdk/integrations/redis/utils.py new file mode 100644 index 0000000000..9bfa656158 --- /dev/null +++ b/sentry_sdk/integrations/redis/utils.py @@ -0,0 +1,116 @@ +from sentry_sdk._types import TYPE_CHECKING +from sentry_sdk.consts import SPANDATA +from sentry_sdk.integrations.redis.consts import ( + _COMMANDS_INCLUDING_SENSITIVE_DATA, + _MAX_NUM_ARGS, + _MAX_NUM_COMMANDS, + _MULTI_KEY_COMMANDS, + _SINGLE_KEY_COMMANDS, +) +from sentry_sdk.scope import should_send_default_pii +from sentry_sdk.utils import SENSITIVE_DATA_SUBSTITUTE + + +if TYPE_CHECKING: + from typing import Any, Optional, Sequence + from sentry_sdk.tracing import Span + + +def _get_safe_command(name, args): + # type: (str, Sequence[Any]) -> str + command_parts = [name] + + for i, arg in enumerate(args): + if i > _MAX_NUM_ARGS: + break + + name_low = name.lower() + + if name_low in _COMMANDS_INCLUDING_SENSITIVE_DATA: + command_parts.append(SENSITIVE_DATA_SUBSTITUTE) + continue + + arg_is_the_key = i == 0 + if arg_is_the_key: + command_parts.append(repr(arg)) + + else: + if should_send_default_pii(): + command_parts.append(repr(arg)) + else: + command_parts.append(SENSITIVE_DATA_SUBSTITUTE) + + command = " ".join(command_parts) + return command + + +def _get_safe_key(method_name, args, kwargs): + # type: (str, Optional[tuple[Any, ...]], Optional[dict[str, Any]]) -> str + """ + Gets the keys (or keys) from the given method_name. + The method_name could be a redis command or a django caching command + """ + key = "" + if args is not None and method_name.lower() in _MULTI_KEY_COMMANDS: + # for example redis "mget" + key = ", ".join(args) + elif args is not None and len(args) >= 1: + # for example django "set_many/get_many" or redis "get" + key = args[0] + elif kwargs is not None and "key" in kwargs: + # this is a legacy case for older versions of django (I guess) + key = kwargs["key"] + + if isinstance(key, dict): + # Django caching set_many() has a dictionary {"key": "data", "key2": "data2"} + # as argument. In this case only return the keys of the dictionary (to not leak data) + key = ", ".join(key.keys()) + + if isinstance(key, list): + key = ", ".join(key) + + return str(key) + + +def _parse_rediscluster_command(command): + # type: (Any) -> Sequence[Any] + return command.args + + +def _set_pipeline_data( + span, is_cluster, get_command_args_fn, is_transaction, command_stack +): + # type: (Span, bool, Any, bool, Sequence[Any]) -> None + span.set_tag("redis.is_cluster", is_cluster) + span.set_tag("redis.transaction", is_transaction) + + commands = [] + for i, arg in enumerate(command_stack): + if i >= _MAX_NUM_COMMANDS: + break + + command = get_command_args_fn(arg) + commands.append(_get_safe_command(command[0], command[1:])) + + span.set_data( + "redis.commands", + { + "count": len(command_stack), + "first_ten": commands, + }, + ) + + +def _set_client_data(span, is_cluster, name, *args): + # type: (Span, bool, str, *Any) -> None + span.set_tag("redis.is_cluster", is_cluster) + if name: + span.set_tag("redis.command", name) + span.set_tag(SPANDATA.DB_OPERATION, name) + + if name and args: + name_low = name.lower() + if (name_low in _SINGLE_KEY_COMMANDS) or ( + name_low in _MULTI_KEY_COMMANDS and len(args) == 1 + ): + span.set_tag("redis.key", args[0]) diff --git a/tests/integrations/django/test_cache_module.py b/tests/integrations/django/test_cache_module.py index 3815d4249a..c47b512b02 100644 --- a/tests/integrations/django/test_cache_module.py +++ b/tests/integrations/django/test_cache_module.py @@ -203,8 +203,8 @@ def test_cache_spans_middleware( ) assert not first_event["spans"][0]["data"]["cache.hit"] assert "cache.item_size" not in first_event["spans"][0]["data"] - # first_event - cache.set - assert first_event["spans"][1]["op"] == "cache.set" + # first_event - cache.put + assert first_event["spans"][1]["op"] == "cache.put" assert first_event["spans"][1]["description"].startswith( "views.decorators.cache.cache_header." ) @@ -269,8 +269,8 @@ def test_cache_spans_decorator(sentry_init, client, capture_events, use_django_c ) assert not first_event["spans"][0]["data"]["cache.hit"] assert "cache.item_size" not in first_event["spans"][0]["data"] - # first_event - cache.set - assert first_event["spans"][1]["op"] == "cache.set" + # first_event - cache.put + assert first_event["spans"][1]["op"] == "cache.put" assert first_event["spans"][1]["description"].startswith( "views.decorators.cache.cache_header." ) @@ -327,8 +327,8 @@ def test_cache_spans_templatetag( ) assert not first_event["spans"][0]["data"]["cache.hit"] assert "cache.item_size" not in first_event["spans"][0]["data"] - # first_event - cache.set - assert first_event["spans"][1]["op"] == "cache.set" + # first_event - cache.put + assert first_event["spans"][1]["op"] == "cache.put" assert first_event["spans"][1]["description"].startswith( "template.cache.some_identifier." ) @@ -354,20 +354,21 @@ def test_cache_spans_templatetag( @pytest.mark.parametrize( "method_name, args, kwargs, expected_description", [ + (None, None, None, ""), ("get", None, None, ""), ("get", [], {}, ""), ("get", ["bla", "blub", "foo"], {}, "bla"), ( "get_many", - [["bla 1", "bla 2", "bla 3"], "blub", "foo"], + [["bla1", "bla2", "bla3"], "blub", "foo"], {}, - "['bla 1', 'bla 2', 'bla 3']", + "bla1, bla2, bla3", ), ( "get_many", - [["bla 1", "bla 2", "bla 3"], "blub", "foo"], + [["bla:1", "bla:2", "bla:3"], "blub", "foo"], {"key": "bar"}, - "['bla 1', 'bla 2', 'bla 3']", + "bla:1, bla:2, bla:3", ), ("get", [], {"key": "bar"}, "bar"), ( @@ -375,7 +376,7 @@ def test_cache_spans_templatetag( "something", {}, "s", - ), # this should never happen, just making sure that we are not raising an exception in that case. + ), # this case should never happen, just making sure that we are not raising an exception in that case. ], ) def test_cache_spans_get_span_description( @@ -489,11 +490,11 @@ def test_cache_spans_item_size(sentry_init, client, capture_events, use_django_c assert not first_event["spans"][0]["data"]["cache.hit"] assert "cache.item_size" not in first_event["spans"][0]["data"] - assert first_event["spans"][1]["op"] == "cache.set" + assert first_event["spans"][1]["op"] == "cache.put" assert "cache.hit" not in first_event["spans"][1]["data"] assert first_event["spans"][1]["data"]["cache.item_size"] == 2 - assert first_event["spans"][2]["op"] == "cache.set" + assert first_event["spans"][2]["op"] == "cache.put" assert "cache.hit" not in first_event["spans"][2]["data"] assert first_event["spans"][2]["data"]["cache.item_size"] == 58 @@ -535,7 +536,7 @@ def test_cache_spans_get_many(sentry_init, capture_events, use_django_caching): assert len(transaction["spans"]) == 7 assert transaction["spans"][0]["op"] == "cache.get" - assert transaction["spans"][0]["description"] == f"['S{id}', 'S{id+1}']" + assert transaction["spans"][0]["description"] == f"S{id}, S{id+1}" assert transaction["spans"][1]["op"] == "cache.get" assert transaction["spans"][1]["description"] == f"S{id}" @@ -543,11 +544,11 @@ def test_cache_spans_get_many(sentry_init, capture_events, use_django_caching): assert transaction["spans"][2]["op"] == "cache.get" assert transaction["spans"][2]["description"] == f"S{id+1}" - assert transaction["spans"][3]["op"] == "cache.set" + assert transaction["spans"][3]["op"] == "cache.put" assert transaction["spans"][3]["description"] == f"S{id}" assert transaction["spans"][4]["op"] == "cache.get" - assert transaction["spans"][4]["description"] == f"['S{id}', 'S{id+1}']" + assert transaction["spans"][4]["description"] == f"S{id}, S{id+1}" assert transaction["spans"][5]["op"] == "cache.get" assert transaction["spans"][5]["description"] == f"S{id}" @@ -582,16 +583,13 @@ def test_cache_spans_set_many(sentry_init, capture_events, use_django_caching): (transaction,) = events assert len(transaction["spans"]) == 4 - assert transaction["spans"][0]["op"] == "cache.set" - assert ( - transaction["spans"][0]["description"] - == f"{{'S{id}': '[Filtered]', 'S{id+1}': '[Filtered]'}}" - ) + assert transaction["spans"][0]["op"] == "cache.put" + assert transaction["spans"][0]["description"] == f"S{id}, S{id+1}" - assert transaction["spans"][1]["op"] == "cache.set" + assert transaction["spans"][1]["op"] == "cache.put" assert transaction["spans"][1]["description"] == f"S{id}" - assert transaction["spans"][2]["op"] == "cache.set" + assert transaction["spans"][2]["op"] == "cache.put" assert transaction["spans"][2]["description"] == f"S{id+1}" assert transaction["spans"][3]["op"] == "cache.get" diff --git a/tests/integrations/redis/test_redis.py b/tests/integrations/redis/test_redis.py index 57ac1c9ab1..8203f75130 100644 --- a/tests/integrations/redis/test_redis.py +++ b/tests/integrations/redis/test_redis.py @@ -85,7 +85,8 @@ def test_redis_pipeline( def test_sensitive_data(sentry_init, capture_events): # fakeredis does not support the AUTH command, so we need to mock it with mock.patch( - "sentry_sdk.integrations.redis._COMMANDS_INCLUDING_SENSITIVE_DATA", ["get"] + "sentry_sdk.integrations.redis.utils._COMMANDS_INCLUDING_SENSITIVE_DATA", + ["get"], ): sentry_init( integrations=[RedisIntegration()], diff --git a/tests/integrations/redis/test_redis_cache_module.py b/tests/integrations/redis/test_redis_cache_module.py new file mode 100644 index 0000000000..2459958f13 --- /dev/null +++ b/tests/integrations/redis/test_redis_cache_module.py @@ -0,0 +1,187 @@ +import fakeredis +from fakeredis import FakeStrictRedis + +from sentry_sdk.integrations.redis import RedisIntegration +from sentry_sdk.utils import parse_version +import sentry_sdk + + +FAKEREDIS_VERSION = parse_version(fakeredis.__version__) + + +def test_no_cache_basic(sentry_init, capture_events): + sentry_init( + integrations=[ + RedisIntegration(), + ], + traces_sample_rate=1.0, + ) + events = capture_events() + + connection = FakeStrictRedis() + with sentry_sdk.start_transaction(): + connection.get("mycachekey") + + (event,) = events + spans = event["spans"] + assert len(spans) == 1 + assert spans[0]["op"] == "db.redis" + + +def test_cache_basic(sentry_init, capture_events): + sentry_init( + integrations=[ + RedisIntegration( + cache_prefixes=["mycache"], + ), + ], + traces_sample_rate=1.0, + ) + events = capture_events() + + connection = FakeStrictRedis() + with sentry_sdk.start_transaction(): + connection.hget("mycachekey", "myfield") + connection.get("mycachekey") + connection.set("mycachekey1", "bla") + connection.setex("mycachekey2", 10, "blub") + connection.mget("mycachekey1", "mycachekey2") + + (event,) = events + spans = event["spans"] + assert len(spans) == 9 + + # no cache support for hget command + assert spans[0]["op"] == "db.redis" + assert spans[0]["tags"]["redis.command"] == "HGET" + + assert spans[1]["op"] == "cache.get" + assert spans[2]["op"] == "db.redis" + assert spans[2]["tags"]["redis.command"] == "GET" + + assert spans[3]["op"] == "cache.put" + assert spans[4]["op"] == "db.redis" + assert spans[4]["tags"]["redis.command"] == "SET" + + assert spans[5]["op"] == "cache.put" + assert spans[6]["op"] == "db.redis" + assert spans[6]["tags"]["redis.command"] == "SETEX" + + assert spans[7]["op"] == "cache.get" + assert spans[8]["op"] == "db.redis" + assert spans[8]["tags"]["redis.command"] == "MGET" + + +def test_cache_keys(sentry_init, capture_events): + sentry_init( + integrations=[ + RedisIntegration( + cache_prefixes=["bla", "blub"], + ), + ], + traces_sample_rate=1.0, + ) + events = capture_events() + + connection = FakeStrictRedis() + with sentry_sdk.start_transaction(): + connection.get("somethingelse") + connection.get("blub") + connection.get("blubkeything") + connection.get("bl") + + (event,) = events + spans = event["spans"] + assert len(spans) == 6 + assert spans[0]["op"] == "db.redis" + assert spans[0]["description"] == "GET 'somethingelse'" + + assert spans[1]["op"] == "cache.get" + assert spans[1]["description"] == "blub" + assert spans[2]["op"] == "db.redis" + assert spans[2]["description"] == "GET 'blub'" + + assert spans[3]["op"] == "cache.get" + assert spans[3]["description"] == "blubkeything" + assert spans[4]["op"] == "db.redis" + assert spans[4]["description"] == "GET 'blubkeything'" + + assert spans[5]["op"] == "db.redis" + assert spans[5]["description"] == "GET 'bl'" + + +def test_cache_data(sentry_init, capture_events): + sentry_init( + integrations=[ + RedisIntegration( + cache_prefixes=["mycache"], + ), + ], + traces_sample_rate=1.0, + ) + events = capture_events() + + connection = FakeStrictRedis(host="mycacheserver.io", port=6378) + with sentry_sdk.start_transaction(): + connection.get("mycachekey") + connection.set("mycachekey", "事实胜于雄辩") + connection.get("mycachekey") + + (event,) = events + spans = event["spans"] + + assert len(spans) == 6 + + assert spans[0]["op"] == "cache.get" + assert spans[0]["description"] == "mycachekey" + assert spans[0]["data"]["cache.key"] == "mycachekey" + assert spans[0]["data"]["cache.hit"] == False # noqa: E712 + assert "cache.item_size" not in spans[0]["data"] + # very old fakeredis can not handle port and/or host. + # only applicable for Redis v3 + if FAKEREDIS_VERSION <= (2, 7, 1): + assert "network.peer.port" not in spans[0]["data"] + else: + assert spans[0]["data"]["network.peer.port"] == 6378 + if FAKEREDIS_VERSION <= (1, 7, 1): + assert "network.peer.address" not in spans[0]["data"] + else: + assert spans[0]["data"]["network.peer.address"] == "mycacheserver.io" + + assert spans[1]["op"] == "db.redis" # we ignore db spans in this test. + + assert spans[2]["op"] == "cache.put" + assert spans[2]["description"] == "mycachekey" + assert spans[2]["data"]["cache.key"] == "mycachekey" + assert "cache.hit" not in spans[1]["data"] + assert spans[2]["data"]["cache.item_size"] == 18 + # very old fakeredis can not handle port. + # only used with redis v3 + if FAKEREDIS_VERSION <= (2, 7, 1): + assert "network.peer.port" not in spans[2]["data"] + else: + assert spans[2]["data"]["network.peer.port"] == 6378 + if FAKEREDIS_VERSION <= (1, 7, 1): + assert "network.peer.address" not in spans[2]["data"] + else: + assert spans[2]["data"]["network.peer.address"] == "mycacheserver.io" + + assert spans[3]["op"] == "db.redis" # we ignore db spans in this test. + + assert spans[4]["op"] == "cache.get" + assert spans[4]["description"] == "mycachekey" + assert spans[4]["data"]["cache.key"] == "mycachekey" + assert spans[4]["data"]["cache.hit"] == True # noqa: E712 + assert spans[4]["data"]["cache.item_size"] == 18 + # very old fakeredis can not handle port. + # only used with redis v3 + if FAKEREDIS_VERSION <= (2, 7, 1): + assert "network.peer.port" not in spans[4]["data"] + else: + assert spans[4]["data"]["network.peer.port"] == 6378 + if FAKEREDIS_VERSION <= (1, 7, 1): + assert "network.peer.address" not in spans[4]["data"] + else: + assert spans[4]["data"]["network.peer.address"] == "mycacheserver.io" + + assert spans[5]["op"] == "db.redis" # we ignore db spans in this test. diff --git a/tests/integrations/redis/test_redis_cache_module_async.py b/tests/integrations/redis/test_redis_cache_module_async.py new file mode 100644 index 0000000000..32e4beabea --- /dev/null +++ b/tests/integrations/redis/test_redis_cache_module_async.py @@ -0,0 +1,181 @@ +import pytest + +try: + import fakeredis + from fakeredis.aioredis import FakeRedis as FakeRedisAsync +except ModuleNotFoundError: + FakeRedisAsync = None + +if FakeRedisAsync is None: + pytest.skip( + "Skipping tests because fakeredis.aioredis not available", + allow_module_level=True, + ) + +from sentry_sdk.integrations.redis import RedisIntegration +from sentry_sdk.utils import parse_version +import sentry_sdk + + +FAKEREDIS_VERSION = parse_version(fakeredis.__version__) + + +@pytest.mark.asyncio +async def test_no_cache_basic(sentry_init, capture_events): + sentry_init( + integrations=[ + RedisIntegration(), + ], + traces_sample_rate=1.0, + ) + events = capture_events() + + connection = FakeRedisAsync() + with sentry_sdk.start_transaction(): + await connection.get("myasynccachekey") + + (event,) = events + spans = event["spans"] + assert len(spans) == 1 + assert spans[0]["op"] == "db.redis" + + +@pytest.mark.asyncio +async def test_cache_basic(sentry_init, capture_events): + sentry_init( + integrations=[ + RedisIntegration( + cache_prefixes=["myasynccache"], + ), + ], + traces_sample_rate=1.0, + ) + events = capture_events() + + connection = FakeRedisAsync() + with sentry_sdk.start_transaction(): + await connection.get("myasynccachekey") + + (event,) = events + spans = event["spans"] + assert len(spans) == 2 + + assert spans[0]["op"] == "cache.get" + assert spans[1]["op"] == "db.redis" + + +@pytest.mark.asyncio +async def test_cache_keys(sentry_init, capture_events): + sentry_init( + integrations=[ + RedisIntegration( + cache_prefixes=["abla", "ablub"], + ), + ], + traces_sample_rate=1.0, + ) + events = capture_events() + + connection = FakeRedisAsync() + with sentry_sdk.start_transaction(): + await connection.get("asomethingelse") + await connection.get("ablub") + await connection.get("ablubkeything") + await connection.get("abl") + + (event,) = events + spans = event["spans"] + assert len(spans) == 6 + assert spans[0]["op"] == "db.redis" + assert spans[0]["description"] == "GET 'asomethingelse'" + + assert spans[1]["op"] == "cache.get" + assert spans[1]["description"] == "ablub" + assert spans[2]["op"] == "db.redis" + assert spans[2]["description"] == "GET 'ablub'" + + assert spans[3]["op"] == "cache.get" + assert spans[3]["description"] == "ablubkeything" + assert spans[4]["op"] == "db.redis" + assert spans[4]["description"] == "GET 'ablubkeything'" + + assert spans[5]["op"] == "db.redis" + assert spans[5]["description"] == "GET 'abl'" + + +@pytest.mark.asyncio +async def test_cache_data(sentry_init, capture_events): + sentry_init( + integrations=[ + RedisIntegration( + cache_prefixes=["myasynccache"], + ), + ], + traces_sample_rate=1.0, + ) + events = capture_events() + + connection = FakeRedisAsync(host="mycacheserver.io", port=6378) + with sentry_sdk.start_transaction(): + await connection.get("myasynccachekey") + await connection.set("myasynccachekey", "事实胜于雄辩") + await connection.get("myasynccachekey") + + (event,) = events + spans = event["spans"] + + assert len(spans) == 6 + + assert spans[0]["op"] == "cache.get" + assert spans[0]["description"] == "myasynccachekey" + assert spans[0]["data"]["cache.key"] == "myasynccachekey" + assert spans[0]["data"]["cache.hit"] == False # noqa: E712 + assert "cache.item_size" not in spans[0]["data"] + # very old fakeredis can not handle port and/or host. + # only applicable for Redis v3 + if FAKEREDIS_VERSION <= (2, 7, 1): + assert "network.peer.port" not in spans[0]["data"] + else: + assert spans[0]["data"]["network.peer.port"] == 6378 + if FAKEREDIS_VERSION <= (1, 7, 1): + assert "network.peer.address" not in spans[0]["data"] + else: + assert spans[0]["data"]["network.peer.address"] == "mycacheserver.io" + + assert spans[1]["op"] == "db.redis" # we ignore db spans in this test. + + assert spans[2]["op"] == "cache.put" + assert spans[2]["description"] == "myasynccachekey" + assert spans[2]["data"]["cache.key"] == "myasynccachekey" + assert "cache.hit" not in spans[1]["data"] + assert spans[2]["data"]["cache.item_size"] == 18 + # very old fakeredis can not handle port. + # only used with redis v3 + if FAKEREDIS_VERSION <= (2, 7, 1): + assert "network.peer.port" not in spans[2]["data"] + else: + assert spans[2]["data"]["network.peer.port"] == 6378 + if FAKEREDIS_VERSION <= (1, 7, 1): + assert "network.peer.address" not in spans[2]["data"] + else: + assert spans[2]["data"]["network.peer.address"] == "mycacheserver.io" + + assert spans[3]["op"] == "db.redis" # we ignore db spans in this test. + + assert spans[4]["op"] == "cache.get" + assert spans[4]["description"] == "myasynccachekey" + assert spans[4]["data"]["cache.key"] == "myasynccachekey" + assert spans[4]["data"]["cache.hit"] == True # noqa: E712 + assert spans[4]["data"]["cache.item_size"] == 18 + # very old fakeredis can not handle port. + # only used with redis v3 + if FAKEREDIS_VERSION <= (2, 7, 1): + assert "network.peer.port" not in spans[4]["data"] + else: + assert spans[4]["data"]["network.peer.port"] == 6378 + if FAKEREDIS_VERSION <= (1, 7, 1): + assert "network.peer.address" not in spans[4]["data"] + else: + assert spans[4]["data"]["network.peer.address"] == "mycacheserver.io" + + assert spans[5]["op"] == "db.redis" # we ignore db spans in this test. diff --git a/tests/integrations/rediscluster/__init__.py b/tests/integrations/redis_py_cluster_legacy/__init__.py similarity index 100% rename from tests/integrations/rediscluster/__init__.py rename to tests/integrations/redis_py_cluster_legacy/__init__.py diff --git a/tests/integrations/rediscluster/test_rediscluster.py b/tests/integrations/redis_py_cluster_legacy/test_redis_py_cluster_legacy.py similarity index 100% rename from tests/integrations/rediscluster/test_rediscluster.py rename to tests/integrations/redis_py_cluster_legacy/test_redis_py_cluster_legacy.py diff --git a/tox.ini b/tox.ini index 62d951eb89..6aabb51682 100644 --- a/tox.ini +++ b/tox.ini @@ -196,7 +196,7 @@ envlist = {py3.7,py3.11,py3.12}-redis-latest # Redis Cluster - {py3.6,py3.8}-rediscluster-v{1,2} + {py3.6,py3.8}-redis_py_cluster_legacy-v{1,2} # no -latest, not developed anymore # Requests @@ -528,8 +528,8 @@ deps = redis-latest: redis # Redis Cluster - rediscluster-v1: redis-py-cluster~=1.0 - rediscluster-v2: redis-py-cluster~=2.0 + redis_py_cluster_legacy-v1: redis-py-cluster~=1.0 + redis_py_cluster_legacy-v2: redis-py-cluster~=2.0 # Requests requests: requests>=2.0 @@ -652,7 +652,7 @@ setenv = pyramid: TESTPATH=tests/integrations/pyramid quart: TESTPATH=tests/integrations/quart redis: TESTPATH=tests/integrations/redis - rediscluster: TESTPATH=tests/integrations/rediscluster + redis_py_cluster_legacy: TESTPATH=tests/integrations/redis_py_cluster_legacy requests: TESTPATH=tests/integrations/requests rq: TESTPATH=tests/integrations/rq sanic: TESTPATH=tests/integrations/sanic