fix: filter out empty tweets in db

prinsss · Oct 21, 2024 · 2e35ba0 · 2e35ba0
1 parent 9b3ccb8
commit 2e35ba0
Show file tree

Hide file tree

Showing 2 changed files with 31 additions and 4 deletions.
diff --git a/src/core/database/manager.ts b/src/core/database/manager.ts
@@ -56,7 +56,12 @@ export class DatabaseManager {
       return [];
     }
     const tweetIds = captures.map((capture) => capture.data_key);
-    return this.tweets().where('rest_id').anyOf(tweetIds).toArray().catch(this.logError);
+    return this.tweets()
+      .where('rest_id')
+      .anyOf(tweetIds)
+      .filter((t) => this.filterEmptyData(t))
+      .toArray()
+      .catch(this.logError);
   }
 
   async extGetCapturedUsers(extName: string) {
@@ -65,7 +70,12 @@ export class DatabaseManager {
       return [];
     }
     const userIds = captures.map((capture) => capture.data_key);
-    return this.users().where('rest_id').anyOf(userIds).toArray().catch(this.logError);
+    return this.users()
+      .where('rest_id')
+      .anyOf(userIds)
+      .filter((t) => this.filterEmptyData(t))
+      .toArray()
+      .catch(this.logError);
   }
 
   /*
@@ -173,6 +183,14 @@ export class DatabaseManager {
     return this.captures().clear().catch(this.logError);
   }
 
+  private filterEmptyData(data: Tweet | User) {
+    if (!data?.legacy) {
+      logger.warn('Empty data found in DB', data);
+      return false;
+    }
+    return true;
+  }
+
   /*
   |--------------------------------------------------------------------------
   | Migrations

diff --git a/src/utils/api.ts b/src/utils/api.ts
@@ -192,11 +192,11 @@ export function isTimelineEntryListSearch(
 export function extractTweetUnion(tweet: TweetUnion): Tweet | null {
   try {
     if (tweet.__typename === 'Tweet') {
-      return tweet;
+      return filterEmptyTweet(tweet);
     }
 
     if (tweet.__typename === 'TweetWithVisibilityResults') {
-      return tweet.tweet;
+      return filterEmptyTweet(tweet.tweet);
     }
 
     if (tweet.__typename === 'TweetTombstone') {
@@ -256,6 +256,15 @@ export function extractTweetFullText(tweet: Tweet): string {
   return tweet.note_tweet?.note_tweet_results.result.text ?? tweet.legacy.full_text;
 }
 
+export function filterEmptyTweet(tweet: Tweet): Tweet | null {
+  if (!tweet.legacy) {
+    logger.warn('Empty tweet received', tweet);
+    return null;
+  }
+
+  return tweet;
+}
+
 /*
 |--------------------------------------------------------------------------
 | Media operations.