diff --git a/changelog.d/638.bugfix b/changelog.d/638.bugfix new file mode 100644 index 00000000..ade737ce --- /dev/null +++ b/changelog.d/638.bugfix @@ -0,0 +1 @@ +Access `django_redis.cache.DJANGO_REDIS_SCAN_ITERSIZE` and `django_redis.client.herd.CACHE_HERD_TIMEOUT` in runtime to not read Django settings in import time. \ No newline at end of file diff --git a/django_redis/cache.py b/django_redis/cache.py index ebf4d14b..0b4d5890 100644 --- a/django_redis/cache.py +++ b/django_redis/cache.py @@ -9,8 +9,6 @@ from .exceptions import ConnectionInterrupted -DJANGO_REDIS_SCAN_ITERSIZE = getattr(settings, "DJANGO_REDIS_SCAN_ITERSIZE", 10) - CONNECTION_INTERRUPTED = object() @@ -45,6 +43,9 @@ def __init__(self, server: str, params: Dict[str, Any]) -> None: super().__init__(params) self._server = server self._params = params + self._default_scan_itersize = getattr( + settings, "DJANGO_REDIS_SCAN_ITERSIZE", 10 + ) options = params.get("OPTIONS", {}) self._client_cls = options.get( @@ -105,7 +106,7 @@ def delete(self, *args, **kwargs): @omit_exception def delete_pattern(self, *args, **kwargs): - kwargs["itersize"] = kwargs.get("itersize", DJANGO_REDIS_SCAN_ITERSIZE) + kwargs.setdefault("itersize", self._default_scan_itersize) return self.client.delete_pattern(*args, **kwargs) @omit_exception diff --git a/django_redis/client/herd.py b/django_redis/client/herd.py index d5ab941d..0c52480f 100644 --- a/django_redis/client/herd.py +++ b/django_redis/client/herd.py @@ -21,15 +21,12 @@ class Marker: pass -CACHE_HERD_TIMEOUT = getattr(settings, "CACHE_HERD_TIMEOUT", 60) - - -def _is_expired(x): - if x >= CACHE_HERD_TIMEOUT: +def _is_expired(x, herd_timeout: int) -> bool: + if x >= herd_timeout: return True - val = x + random.randint(1, CACHE_HERD_TIMEOUT) + val = x + random.randint(1, herd_timeout) - if val >= CACHE_HERD_TIMEOUT: + if val >= herd_timeout: return True return False @@ -37,6 +34,7 @@ def _is_expired(x): class HerdClient(DefaultClient): def __init__(self, *args, **kwargs): self._marker = Marker() + self._herd_timeout = getattr(settings, "CACHE_HERD_TIMEOUT", 60) super().__init__(*args, **kwargs) def _pack(self, value, timeout): @@ -55,7 +53,7 @@ def _unpack(self, value): now = int(time.time()) if herd_timeout < now: x = now - herd_timeout - return unpacked, _is_expired(x) + return unpacked, _is_expired(x, self._herd_timeout) return unpacked, False @@ -84,7 +82,7 @@ def set( ) packed = self._pack(value, timeout) - real_timeout = timeout + CACHE_HERD_TIMEOUT + real_timeout = timeout + self._herd_timeout return super().set( key, packed, timeout=real_timeout, version=version, client=client, nx=nx diff --git a/setup.cfg b/setup.cfg index e0106810..3c76f354 100644 --- a/setup.cfg +++ b/setup.cfg @@ -100,6 +100,7 @@ REDIS = passenv = CI GITHUB* commands = {envpython} -m pytest --cov-report= --ds=settings.sqlite {posargs} + {envpython} -m pytest --cov-append --cov-report= --ds=settings.sqlite_herd {posargs} {envpython} -m pytest --cov-append --cov-report= --ds=settings.sqlite_json {posargs} {envpython} -m pytest --cov-append --cov-report= --ds=settings.sqlite_lz4 {posargs} {envpython} -m pytest --cov-append --cov-report= --ds=settings.sqlite_msgpack {posargs} diff --git a/tests/settings/sqlite_herd.py b/tests/settings/sqlite_herd.py index 7cfb7af4..9bac3a4c 100644 --- a/tests/settings/sqlite_herd.py +++ b/tests/settings/sqlite_herd.py @@ -27,3 +27,5 @@ INSTALLED_APPS = ["django.contrib.sessions"] USE_TZ = False + +CACHE_HERD_TIMEOUT = 2 diff --git a/tests/test_backend.py b/tests/test_backend.py index 9fdba5a4..7f4beb70 100644 --- a/tests/test_backend.py +++ b/tests/test_backend.py @@ -2,21 +2,29 @@ import threading import time from datetime import timedelta -from typing import List, Union, cast +from typing import Iterable, List, Union, cast from unittest.mock import patch import pytest from django.core.cache import caches +from django.test import override_settings from pytest_django.fixtures import SettingsWrapper from pytest_mock import MockerFixture -import django_redis.cache from django_redis.cache import RedisCache from django_redis.client import ShardClient, herd from django_redis.serializers.json import JSONSerializer from django_redis.serializers.msgpack import MSGPackSerializer -herd.CACHE_HERD_TIMEOUT = 2 + +@pytest.fixture +def patch_itersize_setting() -> Iterable[None]: + # destroy cache to force recreation with overriden settings + del caches["default"] + with override_settings(DJANGO_REDIS_SCAN_ITERSIZE=30): + yield + # destroy cache to force recreation with original settings + del caches["default"] class TestDjangoRedisCache: @@ -199,7 +207,12 @@ def test_set_many(self, cache: RedisCache): res = cache.get_many(["a", "b", "c"]) assert res == {"a": 1, "b": 2, "c": 3} - def test_set_call_empty_pipeline(self, cache: RedisCache, mocker: MockerFixture): + def test_set_call_empty_pipeline( + self, + cache: RedisCache, + mocker: MockerFixture, + settings: SettingsWrapper, + ): if isinstance(cache.client, ShardClient): pytest.skip("ShardClient doesn't support get_client") @@ -212,7 +225,7 @@ def test_set_call_empty_pipeline(self, cache: RedisCache, mocker: MockerFixture) if isinstance(cache.client, herd.HerdClient): default_timeout = cache.client._backend.default_timeout - herd_timeout = (default_timeout + herd.CACHE_HERD_TIMEOUT) * 1000 # type: ignore # noqa + herd_timeout = (default_timeout + settings.CACHE_HERD_TIMEOUT) * 1000 herd_pack_value = cache.client._pack(value, default_timeout) mocked_set.assert_called_once_with( cache.client.make_key(key, version=None), @@ -495,11 +508,15 @@ def test_delete_pattern_with_custom_count(self, client_mock, cache: RedisCache): @patch("django_redis.cache.RedisCache.client") def test_delete_pattern_with_settings_default_scan_count( - self, client_mock, cache: RedisCache + self, + client_mock, + patch_itersize_setting, + cache: RedisCache, + settings: SettingsWrapper, ): for key in ["foo-aa", "foo-ab", "foo-bb", "foo-bc"]: cache.set(key, "foo") - expected_count = django_redis.cache.DJANGO_REDIS_SCAN_ITERSIZE + expected_count = settings.DJANGO_REDIS_SCAN_ITERSIZE cache.delete_pattern("*foo-a*")