send loguru to sentry

neodb-social · Dec 8, 2023 · 4aa121e · 4aa121e
1 parent cd4193b
commit 4aa121e
Show file tree

Hide file tree

Showing 3 changed files with 30 additions and 18 deletions.
diff --git a/boofilsic/settings.py b/boofilsic/settings.py
@@ -536,10 +536,11 @@
 if SENTRY_DSN:
     import sentry_sdk
     from sentry_sdk.integrations.django import DjangoIntegration
+    from sentry_sdk.integrations.loguru import LoguruIntegration
 
     sentry_sdk.init(
         dsn=SENTRY_DSN,
-        integrations=[DjangoIntegration()],
+        integrations=[LoguruIntegration(), DjangoIntegration()],
         release=NEODB_VERSION,
-        traces_sample_rate=1 if DEBUG else 0.01,
+        traces_sample_rate=1 if DEBUG else 0.001,
     )
diff --git a/requirements-dev.txt b/requirements-dev.txt
@@ -6,4 +6,4 @@ djlint~=1.34.0
 isort~=5.12.0
 lxml-stubs
 pre-commit
-pyright==1.1.338
+pyright==1.1.337
diff --git a/takahe/management/commands/backfill_takahe.py b/takahe/management/commands/backfill_takahe.py
@@ -60,6 +60,23 @@ def add_arguments(self, parser):
         parser.add_argument("--count", default=0, action="store")
 
     def process_post(self):
+        def batch(pieces, tracker):
+            with transaction.atomic(using="default"):
+                with transaction.atomic(using="takahe"):
+                    for p in pieces:
+                        if p.__class__ == ShelfMember:
+                            mark = Mark(p.owner, p.item)
+                            Takahe.post_mark(mark, self.post_new)
+                        elif p.__class__ == Comment:
+                            if p.item.__class__ in [PodcastEpisode, TVEpisode]:
+                                Takahe.post_comment(p, self.post_new)
+                        elif p.__class__ == Review:
+                            Takahe.post_review(p, self.post_new)
+                        elif p.__class__ == Collection:
+                            Takahe.post_collection(p)
+                    tracker.set_postfix_str(str(pieces[-1].pk))
+                    tracker.update(len(pieces))
+
         logger.info(f"Generating posts...")
         set_migration_mode(True)
         qs = Piece.objects.filter(
@@ -72,21 +89,15 @@ def process_post(self):
         ).order_by("id")
         if self.starting_id:
             qs = qs.filter(id__gte=self.starting_id)
-        with transaction.atomic(using="default"):
-            with transaction.atomic(using="takahe"):
-                tracker = tqdm(qs.iterator(), total=self.count_est or qs.count())
-                for p in tracker:
-                    tracker.set_postfix_str(f"{p.id}")
-                    if p.__class__ == ShelfMember:
-                        mark = Mark(p.owner, p.item)
-                        Takahe.post_mark(mark, self.post_new)
-                    elif p.__class__ == Comment:
-                        if p.item.__class__ in [PodcastEpisode, TVEpisode]:
-                            Takahe.post_comment(p, self.post_new)
-                    elif p.__class__ == Review:
-                        Takahe.post_review(p, self.post_new)
-                    elif p.__class__ == Collection:
-                        Takahe.post_collection(p)
+        tracker = tqdm(total=self.count_est or qs.count())
+        pieces = []
+        for piece in qs.iterator():
+            pieces.append(piece)
+            if len(pieces) >= BATCH_SIZE:
+                batch(pieces, tracker)
+                pieces = []
+        if pieces:
+            batch(pieces, tracker)
         set_migration_mode(False)
 
     def process_timeline(self):