diff --git a/packages/backend/src/constants.ts b/packages/backend/src/constants.ts index 94b8c76..a9412a7 100644 --- a/packages/backend/src/constants.ts +++ b/packages/backend/src/constants.ts @@ -5,6 +5,7 @@ import { Settings } from "./types.js"; */ export const DEFAULT_SETTINGS: Settings = { maxFileSize: 2 * 1024 * 1024, // 2MB in bytes + maxTrigramCount: 20000, autoDeleteStaleRepos: true, reindexInterval: 1000 * 60 * 60, // 1 hour in milliseconds resyncInterval: 1000 * 60 * 60 * 24, // 1 day in milliseconds diff --git a/packages/backend/src/main.ts b/packages/backend/src/main.ts index 5de504c..2107696 100644 --- a/packages/backend/src/main.ts +++ b/packages/backend/src/main.ts @@ -209,6 +209,7 @@ const syncConfig = async (configPath: string, db: Database, signal: AbortSignal, // Update the settings const updatedSettings: Settings = { maxFileSize: config.settings?.maxFileSize ?? DEFAULT_SETTINGS.maxFileSize, + maxTrigramCount: config.settings?.maxTrigramCount ?? DEFAULT_SETTINGS.maxTrigramCount, autoDeleteStaleRepos: config.settings?.autoDeleteStaleRepos ?? DEFAULT_SETTINGS.autoDeleteStaleRepos, reindexInterval: config.settings?.reindexInterval ?? DEFAULT_SETTINGS.reindexInterval, resyncInterval: config.settings?.resyncInterval ?? DEFAULT_SETTINGS.resyncInterval, diff --git a/packages/backend/src/schemas/v2.ts b/packages/backend/src/schemas/v2.ts index 0c45d6f..5114aa6 100644 --- a/packages/backend/src/schemas/v2.ts +++ b/packages/backend/src/schemas/v2.ts @@ -21,6 +21,10 @@ export interface Settings { * The maximum size of a file (in bytes) to be indexed. Files that exceed this maximum will not be inexed. Defaults to 2MB (2097152 bytes). */ maxFileSize?: number; + /** + * The maximum amount of trigrams per document. Documents that exceed this maximum will not be indexed. Defaults to 20000 + */ + maxTrigramCount?: number; /** * Automatically delete stale repositories from the index. Defaults to true. */ diff --git a/packages/backend/src/types.ts b/packages/backend/src/types.ts index 2b0eca3..2888fc2 100644 --- a/packages/backend/src/types.ts +++ b/packages/backend/src/types.ts @@ -46,9 +46,13 @@ export type AppContext = { export type Settings = { /** - * The maximum size of a file (in bytes) to be indexed. Files that exceed this maximum will not be inexed. + * The maximum size of a file (in bytes) to be indexed. Files that exceed this maximum will not be indexed. */ maxFileSize: number; + /** + * The maximum number of trigrams per document. Files that exceed this maximum will not be indexed. + */ + maxTrigramCount: number; /** * Automatically delete stale repositories from the index. Defaults to true. */ diff --git a/packages/backend/src/zoekt.ts b/packages/backend/src/zoekt.ts index ffeadf7..eab5fea 100644 --- a/packages/backend/src/zoekt.ts +++ b/packages/backend/src/zoekt.ts @@ -10,7 +10,7 @@ export const indexGitRepository = async (repo: GitRepository, settings: Settings ...repo.tags ?? [], ]; - const command = `zoekt-git-index -allow_missing_branches -index ${ctx.indexPath} -file_limit ${settings.maxFileSize} -branches ${revisions.join(',')} ${repo.path}`; + const command = `zoekt-git-index -allow_missing_branches -index ${ctx.indexPath} -max_trigram_count ${settings.maxTrigramCount} -file_limit ${settings.maxFileSize} -branches ${revisions.join(',')} ${repo.path}`; return new Promise<{ stdout: string, stderr: string }>((resolve, reject) => { exec(command, (error, stdout, stderr) => { diff --git a/schemas/v2/index.json b/schemas/v2/index.json index ee70c0c..fc04e9d 100644 --- a/schemas/v2/index.json +++ b/schemas/v2/index.json @@ -570,6 +570,12 @@ "default": 2097152, "minimum": 1 }, + "maxTrigramCount": { + "type": "integer", + "description": "The maximum amount of trigrams per document. Documents that exceed this maximum will not be indexed. Defaults to 20000", + "default": 20000, + "minimum": 1 + }, "autoDeleteStaleRepos": { "type": "boolean", "description": "Automatically delete stale repositories from the index. Defaults to true.",