add maxTrigramCount setting

sourcebot-dev · Jan 28, 2025 · f83a4db · f83a4db
1 parent d1ef80d
commit f83a4db
Show file tree

Hide file tree

Showing 6 changed files with 18 additions and 2 deletions.
diff --git a/packages/backend/src/constants.ts b/packages/backend/src/constants.ts
@@ -5,6 +5,7 @@ import { Settings } from "./types.js";
  */
 export const DEFAULT_SETTINGS: Settings = {
     maxFileSize: 2 * 1024 * 1024, // 2MB in bytes
+    maxTrigramCount: 20000,
     autoDeleteStaleRepos: true,
     reindexInterval: 1000 * 60 * 60, // 1 hour in milliseconds
     resyncInterval: 1000 * 60 * 60 * 24, // 1 day in milliseconds

diff --git a/packages/backend/src/main.ts b/packages/backend/src/main.ts
@@ -209,6 +209,7 @@ const syncConfig = async (configPath: string, db: Database, signal: AbortSignal,
     // Update the settings
     const updatedSettings: Settings = {
         maxFileSize: config.settings?.maxFileSize ?? DEFAULT_SETTINGS.maxFileSize,
+        maxTrigramCount: config.settings?.maxTrigramCount ?? DEFAULT_SETTINGS.maxTrigramCount,
         autoDeleteStaleRepos: config.settings?.autoDeleteStaleRepos ?? DEFAULT_SETTINGS.autoDeleteStaleRepos,
         reindexInterval: config.settings?.reindexInterval ?? DEFAULT_SETTINGS.reindexInterval,
         resyncInterval: config.settings?.resyncInterval ?? DEFAULT_SETTINGS.resyncInterval,

diff --git a/packages/backend/src/schemas/v2.ts b/packages/backend/src/schemas/v2.ts
@@ -21,6 +21,10 @@ export interface Settings {
    * The maximum size of a file (in bytes) to be indexed. Files that exceed this maximum will not be inexed. Defaults to 2MB (2097152 bytes).
    */
   maxFileSize?: number;
+  /**
+   * The maximum amount of trigrams per document. Documents that exceed this maximum will not be indexed. Defaults to 20000
+   */
+  maxTrigramCount?: number;
   /**
    * Automatically delete stale repositories from the index. Defaults to true.
    */

diff --git a/packages/backend/src/types.ts b/packages/backend/src/types.ts
@@ -46,9 +46,13 @@ export type AppContext = {
 
 export type Settings = {
     /**
-     * The maximum size of a file (in bytes) to be indexed. Files that exceed this maximum will not be inexed.
+     * The maximum size of a file (in bytes) to be indexed. Files that exceed this maximum will not be indexed.
      */
     maxFileSize: number;
+    /**
+     * The maximum number of trigrams per document. Files that exceed this maximum will not be indexed.
+     */
+    maxTrigramCount: number;
     /**
      * Automatically delete stale repositories from the index. Defaults to true.
      */

diff --git a/packages/backend/src/zoekt.ts b/packages/backend/src/zoekt.ts
@@ -10,7 +10,7 @@ export const indexGitRepository = async (repo: GitRepository, settings: Settings
         ...repo.tags ?? [],
     ];
 
-    const command = `zoekt-git-index -allow_missing_branches -index ${ctx.indexPath} -file_limit ${settings.maxFileSize} -branches ${revisions.join(',')} ${repo.path}`;
+    const command = `zoekt-git-index -allow_missing_branches -index ${ctx.indexPath} -max_trigram_count ${settings.maxTrigramCount} -file_limit ${settings.maxFileSize} -branches ${revisions.join(',')} ${repo.path}`;
 
     return new Promise<{ stdout: string, stderr: string }>((resolve, reject) => {
         exec(command, (error, stdout, stderr) => {

diff --git a/schemas/v2/index.json b/schemas/v2/index.json
@@ -570,6 +570,12 @@
                     "default": 2097152,
                     "minimum": 1
                 },
+                "maxTrigramCount": {
+                    "type": "integer",
+                    "description": "The maximum amount of trigrams per document. Documents that exceed this maximum will not be indexed. Defaults to 20000",
+                    "default": 20000,
+                    "minimum": 1
+                },
                 "autoDeleteStaleRepos": {
                     "type": "boolean",
                     "description": "Automatically delete stale repositories from the index. Defaults to true.",