From ef93d52fb5010c0e034031ead9d2e2b3218f422d Mon Sep 17 00:00:00 2001 From: Paul Schilling Date: Mon, 2 Oct 2023 12:42:12 +0200 Subject: [PATCH] add task and management command for cleaning up logs --- docs/quickstart.rst | 4 ++ log_outgoing_requests/conf.py | 6 +++ log_outgoing_requests/management/__init__.py | 0 .../management/commands/__init__.py | 0 .../commands/prune_outgoing_request_logs.py | 9 ++++ log_outgoing_requests/models.py | 15 ++++++ log_outgoing_requests/tasks.py | 13 +++++ tests/test_tasks.py | 49 +++++++++++++++++++ 8 files changed, 96 insertions(+) create mode 100644 log_outgoing_requests/management/__init__.py create mode 100644 log_outgoing_requests/management/commands/__init__.py create mode 100644 log_outgoing_requests/management/commands/prune_outgoing_request_logs.py diff --git a/docs/quickstart.rst b/docs/quickstart.rst index e5c6770..fd65f13 100644 --- a/docs/quickstart.rst +++ b/docs/quickstart.rst @@ -85,6 +85,7 @@ you likely want to apply the following non-default settings: LOG_OUTGOING_REQUESTS_DB_SAVE = True # save logs enabled/disabled based on the boolean value LOG_OUTGOING_REQUESTS_DB_SAVE_BODY = True # save request/response body LOG_OUTGOING_REQUESTS_EMIT_BODY = True # log request/response body + LOG_OUTGOING_REQUESTS_MAX_AGE = 1 # delete requests older than 1 day .. note:: @@ -100,6 +101,9 @@ you likely want to apply the following non-default settings: (which defines if/when database logging is reset after changes to the library config) should be set to ``None``. + The library provides a Django management command as well as a Celery task to delete + logs which are older than a specified time (by default, 1 day). + See :ref:`reference_settings` for all available settings and their meaning. Usage diff --git a/log_outgoing_requests/conf.py b/log_outgoing_requests/conf.py index 47a65ff..b44abb8 100644 --- a/log_outgoing_requests/conf.py +++ b/log_outgoing_requests/conf.py @@ -52,6 +52,12 @@ class LogOutgoingRequestsConf(AppConf): database, but the body will be missing. """ + MAX_AGE = 1 + """ + The maximum age (in days) of request logs, after which they are deleted (via a Celery + task, Django management command, or the like). + """ + RESET_DB_SAVE_AFTER = 60 """ If the config has been updated, reset the database logging after the specified diff --git a/log_outgoing_requests/management/__init__.py b/log_outgoing_requests/management/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/log_outgoing_requests/management/commands/__init__.py b/log_outgoing_requests/management/commands/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/log_outgoing_requests/management/commands/prune_outgoing_request_logs.py b/log_outgoing_requests/management/commands/prune_outgoing_request_logs.py new file mode 100644 index 0000000..9b5f188 --- /dev/null +++ b/log_outgoing_requests/management/commands/prune_outgoing_request_logs.py @@ -0,0 +1,9 @@ +from django.core.management.base import BaseCommand + +from ...models import OutgoingRequestsLog + + +class Command(BaseCommand): + def handle(self, *args, **kwargs): + num_deleted = OutgoingRequestsLog.objects.prune() + self.stdout.write(f"Deleted {num_deleted} outgoing request log(s)") diff --git a/log_outgoing_requests/models.py b/log_outgoing_requests/models.py index f10cdf1..7de2bd7 100644 --- a/log_outgoing_requests/models.py +++ b/log_outgoing_requests/models.py @@ -1,9 +1,11 @@ import logging +from datetime import timedelta from typing import Union from urllib.parse import urlparse from django.core.validators import MinValueValidator from django.db import models +from django.utils import timezone from django.utils.functional import cached_property from django.utils.translation import gettext_lazy as _ @@ -16,6 +18,17 @@ logger = logging.getLogger(__name__) +class OutgoingRequestsLogQueryset(models.QuerySet): + def prune(self) -> int: + max_age = settings.LOG_OUTGOING_REQUESTS_MAX_AGE + if max_age is None: + return 0 + + now = timezone.now() + num_deleted, _ = self.filter(timestamp__lt=now - timedelta(max_age)).delete() + return num_deleted + + class OutgoingRequestsLog(models.Model): url = models.URLField( verbose_name=_("URL"), @@ -108,6 +121,8 @@ class OutgoingRequestsLog(models.Model): help_text=_("Text providing information in case of request failure."), ) + objects = OutgoingRequestsLogQueryset.as_manager() + class Meta: verbose_name = _("Outgoing request log") verbose_name_plural = _("Outgoing request logs") diff --git a/log_outgoing_requests/tasks.py b/log_outgoing_requests/tasks.py index 4f035e2..3e41557 100644 --- a/log_outgoing_requests/tasks.py +++ b/log_outgoing_requests/tasks.py @@ -1,6 +1,19 @@ +import logging + from .compat import shared_task from .constants import SaveLogsChoice +logger = logging.getLogger(__name__) + + +@shared_task +def prune_logs(): + from .models import OutgoingRequestsLog + + num_deleted = OutgoingRequestsLog.objects.prune() + logger.info("Deleted %d outgoing request log(s)", num_deleted) + return num_deleted + @shared_task def reset_config(): diff --git a/tests/test_tasks.py b/tests/test_tasks.py index c520e8c..4a2dc93 100644 --- a/tests/test_tasks.py +++ b/tests/test_tasks.py @@ -1,5 +1,12 @@ +import logging +from io import StringIO + +from django.core.management import call_command +from django.utils import timezone + import pytest import requests +from freezegun import freeze_time from log_outgoing_requests.config_reset import schedule_config_reset from log_outgoing_requests.models import OutgoingRequestsLog, OutgoingRequestsLogConfig @@ -12,6 +19,48 @@ has_celery = celery is not None +@pytest.mark.django_db +def test_cleanup_request_logs_command(settings, caplog): + settings.LOG_OUTGOING_REQUESTS_MAX_AGE = 1 # delete if > 1 day old + + with freeze_time("2023-10-02T12:00:00Z") as frozen_time: + OutgoingRequestsLog.objects.create(timestamp=timezone.now()) + frozen_time.move_to("2023-10-04T12:00:00Z") + recent_log = OutgoingRequestsLog.objects.create(timestamp=timezone.now()) + + stdout = StringIO() + with caplog.at_level(logging.INFO): + call_command( + "prune_outgoing_request_logs", stdout=stdout, stderr=StringIO() + ) + + output = stdout.getvalue() + assert output == "Deleted 1 outgoing request log(s)\n" + + assert OutgoingRequestsLog.objects.get() == recent_log + + +@pytest.mark.skipif(not has_celery, reason="Celery is optional dependency") +@pytest.mark.django_db +def test_cleanup_request_logs_celery_task(requests_mock, settings, caplog): + from log_outgoing_requests.tasks import prune_logs + + settings.LOG_OUTGOING_REQUESTS_MAX_AGE = 1 # delete if > 1 old old + + with freeze_time("2023-10-02T12:00:00Z") as frozen_time: + OutgoingRequestsLog.objects.create(timestamp=timezone.now()) + frozen_time.move_to("2023-10-04T12:00:00Z") + recent_log = OutgoingRequestsLog.objects.create(timestamp=timezone.now()) + + with caplog.at_level(logging.INFO): + prune_logs() + + assert len(caplog.records) == 1 + assert "Deleted 1 outgoing request log(s)" in caplog.text + + assert OutgoingRequestsLog.objects.get() == recent_log + + @pytest.mark.skipif(not has_celery, reason="Celery is optional dependency") @pytest.mark.django_db def test_reset_config(requests_mock, settings):