diff --git a/CHANGELOG.md b/CHANGELOG.md index 6fd66aefec5a..1b8b48a2dae3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -35,6 +35,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Fixed - Helm: Empty password for Redis () +- Preview & chunk cache settings are ignored () ### Security - Fixed vulnerability with social authentication () diff --git a/cvat/apps/engine/cache.py b/cvat/apps/engine/cache.py index 01ed4d514c42..d1b19ef3578b 100644 --- a/cvat/apps/engine/cache.py +++ b/cvat/apps/engine/cache.py @@ -8,7 +8,7 @@ from tempfile import NamedTemporaryFile import pytz -from diskcache import Cache +from django.core.cache import cache from django.conf import settings from rest_framework.exceptions import ValidationError, NotFound @@ -25,42 +25,42 @@ from utils.dataset_manifest import ImageManifestManager -class CacheInteraction: +class MediaCache: def __init__(self, dimension=DimensionType.DIM_2D): - self._cache = Cache(settings.CACHE_ROOT) self._dimension = dimension - def __del__(self): - self._cache.close() + @staticmethod + def _get_or_set_cache_item(key, create_function): + item = cache.get(key) + if not item: + item = create_function() + cache.set(key, item) - def get_buf_chunk_with_mime(self, chunk_number, quality, db_data): - cache_key = f'{db_data.id}_{chunk_number}_{quality}' - chunk, tag = self._cache.get(cache_key, tag=True) + return item - if not chunk: - chunk, tag = self._prepare_chunk_buff(db_data, quality, chunk_number) - self._cache.set(cache_key, chunk, tag=tag) + def get_buf_chunk_with_mime(self, chunk_number, quality, db_data): + item = self._get_or_set_cache_item( + key=f'{db_data.id}_{chunk_number}_{quality}', + create_function=lambda: self._prepare_chunk_buff(db_data, quality, chunk_number), + ) - return chunk, tag + return item def get_local_preview_with_mime(self, frame_number, db_data): - key = f'data_{db_data.id}_{frame_number}_preview' - buf, mime = self._cache.get(key, tag=True) - if not buf: - buf, mime = self._prepare_local_preview(frame_number, db_data) - self._cache.set(key, buf, tag=mime) + item = self._get_or_set_cache_item( + key=f'data_{db_data.id}_{frame_number}_preview', + create_function=lambda: self._prepare_local_preview(frame_number, db_data), + ) - return buf, mime + return item def get_cloud_preview_with_mime(self, db_storage): - key = f'cloudstorage_{db_storage.id}_preview' - preview, mime = self._cache.get(key, tag=True) + item = self._get_or_set_cache_item( + key=f'cloudstorage_{db_storage.id}_preview', + create_function=lambda: self._prepare_cloud_preview(db_storage) + ) - if not preview: - preview, mime = self._prepare_cloud_preview(db_storage) - self._cache.set(key, preview, tag=mime) - - return preview, mime + return item @staticmethod def _get_frame_provider(): @@ -147,9 +147,9 @@ def _prepare_chunk_buff(self, db_data, quality, chunk_number): def _prepare_local_preview(self, frame_number, db_data): FrameProvider = self._get_frame_provider() frame_provider = FrameProvider(db_data, self._dimension) - buf, mime = frame_provider.get_preview(frame_number) + buff, mime_type = frame_provider.get_preview(frame_number) - return buf, mime + return buff, mime_type def _prepare_cloud_preview(self, db_storage): storage = db_storage_to_storage_instance(db_storage) @@ -179,7 +179,7 @@ def _prepare_cloud_preview(self, db_storage): slogger.cloud_storage[db_storage.pk].info(msg) raise NotFound(msg) - preview = storage.download_fileobj(preview_path) - mime = mimetypes.guess_type(preview_path)[0] + buff = storage.download_fileobj(preview_path) + mime_type = mimetypes.guess_type(preview_path)[0] - return preview, mime + return buff, mime_type diff --git a/cvat/apps/engine/frame_provider.py b/cvat/apps/engine/frame_provider.py index 8ab47d8e80ed..ea7b56a57c01 100644 --- a/cvat/apps/engine/frame_provider.py +++ b/cvat/apps/engine/frame_provider.py @@ -12,7 +12,7 @@ import numpy as np from PIL import Image -from cvat.apps.engine.cache import CacheInteraction +from cvat.apps.engine.cache import MediaCache from cvat.apps.engine.media_extractors import VideoReader, ZipReader from cvat.apps.engine.mime_types import mimetypes from cvat.apps.engine.models import DataChoice, StorageMethodChoice, DimensionType @@ -97,7 +97,7 @@ def __init__(self, db_data, dimension=DimensionType.DIM_2D): } if db_data.storage_method == StorageMethodChoice.CACHE: - cache = CacheInteraction(dimension=dimension) + cache = MediaCache(dimension=dimension) self._loaders[self.Quality.COMPRESSED] = self.BuffChunkLoader( reader_class[db_data.compressed_chunk_type], diff --git a/cvat/apps/engine/views.py b/cvat/apps/engine/views.py index 658a8b950e2e..2e06ee914646 100644 --- a/cvat/apps/engine/views.py +++ b/cvat/apps/engine/views.py @@ -75,7 +75,7 @@ from cvat.apps.iam.permissions import (CloudStoragePermission, CommentPermission, IssuePermission, JobPermission, ProjectPermission, TaskPermission, UserPermission) -from cvat.apps.engine.cache import CacheInteraction +from cvat.apps.engine.cache import MediaCache @extend_schema(tags=['server']) @@ -721,7 +721,7 @@ def __call__(self, request, start, stop, db_data): f'[{start}, {stop}] range') if self.type == 'preview': - cache = CacheInteraction(self.dimension) + cache = MediaCache(self.dimension) buf, mime = cache.get_local_preview_with_mime(self.number, db_data) else: buf, mime = frame_provider.get_frame(self.number, self.quality) @@ -2185,7 +2185,7 @@ def content(self, request, pk): def preview(self, request, pk): try: db_storage = self.get_object() - cache = CacheInteraction() + cache = MediaCache() preview, mime = cache.get_cloud_preview_with_mime(db_storage) return HttpResponse(preview, mime) except CloudStorageModel.DoesNotExist: diff --git a/cvat/settings/base.py b/cvat/settings/base.py index 76fa60b491ee..4b0439d8b9e8 100644 --- a/cvat/settings/base.py +++ b/cvat/settings/base.py @@ -514,6 +514,7 @@ class CVAT_QUEUES(Enum): 'BACKEND' : 'diskcache.DjangoCache', 'LOCATION' : CACHE_ROOT, 'TIMEOUT' : None, + 'SHARDS': 32, 'OPTIONS' : { 'size_limit' : 2 ** 40, # 1 Tb }