improve index cache storage

ker0olos · Mar 21, 2024 · 73fd44f · 73fd44f
1 parent 58f5466
commit 73fd44f
Show file tree

Hide file tree

Showing 4 changed files with 40 additions and 34 deletions.
diff --git a/.github/workflows/search-index.yml b/.github/workflows/search-index.yml
@@ -1,7 +1,7 @@
 name: search-index
 
 on:
-  # push:
+  push:
   workflow_dispatch:
   # schedule:
   #   # Once a week "At 00:00 on Sunday" (see https://crontab.guru/once-a-week)

diff --git a/deno.json b/deno.json
@@ -48,6 +48,7 @@
     "mongodb": "npm:mongodb",
     "mongodb-memory-server": "npm:mongodb-memory-server-core",
     "orama": "npm:@orama/orama@2.0.11",
+    "orama-persist": "npm:@orama/plugin-data-persistence/server",
     "levenshtein": "https://deno.land/x/fastest_levenshtein@1.0.10/mod.ts"
   }
 }
diff --git a/search-index/cache.ts b/search-index/cache.ts
@@ -1,30 +1,30 @@
 import { create, insert, Orama } from 'orama';
 
-import { join } from '$std/path/mod.ts';
+import { persistToFile } from 'orama-persist';
 
 import utils from '~/src/utils.ts';
 
 import { gql, request } from '~/packs/anilist/graphql.ts';
 
 import { AniListCharacter, AniListMedia } from '~/packs/anilist/types.ts';
 
-import { characterSchema, mediaSchema } from '~/search-index/mod.ts';
+import {
+  characterSchema,
+  charactersIndexCachePath,
+  mediaIndexCachePath,
+  mediaSchema,
+} from '~/search-index/mod.ts';
 
 const anilistAPI = 'https://graphql.anilist.co';
 
-const dirname = new URL('.', import.meta.url).pathname;
-
-const charactersIndexCachePath = join(dirname, './characters.json');
-const mediaIndexCachePath = join(dirname, './media.json');
+const mediaIndex: Orama<typeof mediaSchema> = await create({
+  schema: mediaSchema,
+});
 
 const charactersIndex: Orama<typeof characterSchema> = await create({
   schema: characterSchema,
 });
 
-const mediaIndex: Orama<typeof mediaSchema> = await create({
-  schema: mediaSchema,
-});
-
 type PageInfo = {
   hasNextPage: boolean;
 };
@@ -160,6 +160,10 @@ let mediaPage = 1;
 
 while (true) {
   try {
+    if (mediaPage === 5) {
+      break;
+    }
+
     const { pageInfo, media } = await queryMedia(mediaPage);
 
     for (const { id, characters: firstPage } of media) {
@@ -208,8 +212,6 @@ while (true) {
                     title: mediaTitle,
                     popularity: media?.node.popularity,
                   });
-
-                  console.log(`indexed 1 media with id:${mediaId}`);
                 }
 
                 if (
@@ -225,8 +227,6 @@ while (true) {
                     popularity: media?.node.popularity,
                     role: media.characterRole,
                   });
-
-                  console.log(`indexed 1 character with id:${id}`);
                 }
               }
             }
@@ -242,7 +242,7 @@ while (true) {
           // handle the rate limit
           // (see https://anilist.gitbook.io/anilist-apiv2-docs/overview/rate-limiting)
           if (e.message?.includes('Too Many Requests')) {
-            console.log('sleeping for a minute...');
+            // console.log('sleeping for a minute...');
             await utils.sleep(60);
             continue;
           }
@@ -272,20 +272,8 @@ while (true) {
 }
 
 await Promise.all([
-  Deno.writeTextFile(
-    join(dirname, charactersIndexCachePath),
-    JSON.stringify({
-      data: charactersIndex.data,
-      internalDocumentIDStore: charactersIndex.internalDocumentIDStore,
-    }),
-  ),
-  Deno.writeTextFile(
-    join(dirname, mediaIndexCachePath),
-    JSON.stringify({
-      data: mediaIndex.data,
-      internalDocumentIDStore: mediaIndex.internalDocumentIDStore,
-    }),
-  ),
+  persistToFile(charactersIndex, 'binary', charactersIndexCachePath),
+  persistToFile(mediaIndex, 'binary', mediaIndexCachePath),
 ]);
 
 console.log('\n\nwritten caches to disk');
diff --git a/search-index/mod.ts b/search-index/mod.ts
@@ -1,7 +1,16 @@
+import { join } from '$std/path/mod.ts';
+
+import { restoreFromFile } from 'orama-persist';
+
 import type { Orama } from 'orama';
 
 import type { CharacterRole } from '~/src/types.ts';
 
+const dirname = new URL('.', import.meta.url).pathname;
+
+export const mediaIndexCachePath = join(dirname, './media.msp');
+export const charactersIndexCachePath = join(dirname, './characters.msp');
+
 export type IndexedCharacter = Orama<{
   id: string;
   name: string[];
@@ -34,8 +43,16 @@ export const mediaSchema = {
 //   term: 'luka',
 // };
 
-// const characterDB: Orama<typeof characterSchema> = await create({
-//   schema: characterSchema,
-// });
-
 // const result: Results<TypedDocument<IndexedCharacter>> = await search(characterDB, searchParams);
+
+export const loadMediaIndex = () =>
+  restoreFromFile<Orama<typeof mediaSchema>>(
+    'binary',
+    mediaIndexCachePath,
+  );
+
+export const loadCharactersIndex = () =>
+  restoreFromFile<Orama<typeof characterSchema>>(
+    'binary',
+    charactersIndexCachePath,
+  );