From d269a8cbe6eca09feee5c6c4103ea6519e6f1a6b Mon Sep 17 00:00:00 2001 From: Michael Sukkarieh Date: Thu, 9 Jan 2025 09:10:30 -0800 Subject: [PATCH] implement raw remote git repo support (#152) * implement raw remote git repo support * add changelog entry --- CHANGELOG.md | 1 + packages/backend/src/git.ts | 81 +++++++++++++++++++++++++++++- packages/backend/src/main.ts | 7 ++- packages/backend/src/schemas/v2.ts | 13 ++++- schemas/v2/index.json | 25 +++++++++ 5 files changed, 124 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e8bc2076..8a92ad79 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added - Added support for creating share links to snippets of code. ([#149](https://github.com/sourcebot-dev/sourcebot/pull/149)) +- Added support for indexing raw remote git repository. ([#152](https://github.com/sourcebot-dev/sourcebot/pull/152)) ## [2.6.3] - 2024-12-18 diff --git a/packages/backend/src/git.ts b/packages/backend/src/git.ts index 3e7d57df..574b2807 100644 --- a/packages/backend/src/git.ts +++ b/packages/backend/src/git.ts @@ -1,7 +1,9 @@ -import { GitRepository } from './types.js'; +import { GitRepository, AppContext } from './types.js'; import { simpleGit, SimpleGitProgressEvent } from 'simple-git'; import { existsSync } from 'fs'; import { createLogger } from './logger.js'; +import { GitConfig } from './schemas/v2.js'; +import path from 'path'; const logger = createLogger('git'); @@ -48,4 +50,81 @@ export const fetchRepository = async (repo: GitRepository, onProgress?: (event: "--progress" ] ); +} + +const isValidGitRepo = async (url: string): Promise => { + const git = simpleGit(); + try { + await git.listRemote([url]); + return true; + } catch (error) { + logger.debug(`Error checking if ${url} is a valid git repo: ${error}`); + return false; + } +} + +const stripProtocolAndGitSuffix = (url: string): string => { + return url.replace(/^[a-zA-Z]+:\/\//, '').replace(/\.git$/, ''); +} + +const getRepoNameFromUrl = (url: string): string => { + const strippedUrl = stripProtocolAndGitSuffix(url); + return strippedUrl.split('/').slice(-2).join('/'); +} + +export const getGitRepoFromConfig = async (config: GitConfig, ctx: AppContext) => { + const repoValid = await isValidGitRepo(config.url); + if (!repoValid) { + logger.error(`Git repo provided in config with url ${config.url} is not valid`); + return null; + } + + const cloneUrl = config.url; + const repoId = stripProtocolAndGitSuffix(cloneUrl); + const repoName = getRepoNameFromUrl(config.url); + const repoPath = path.resolve(path.join(ctx.reposPath, `${repoId}.git`)); + const repo: GitRepository = { + vcs: 'git', + id: repoId, + name: repoName, + path: repoPath, + isStale: false, + cloneUrl: cloneUrl, + branches: [], + tags: [], + } + + if (config.revisions) { + if (config.revisions.branches) { + const branchGlobs = config.revisions.branches; + const git = simpleGit(); + const branchList = await git.listRemote(['--heads', cloneUrl]); + const branches = branchList + .split('\n') + .map(line => line.split('\t')[1]) + .filter(Boolean) + .map(branch => branch.replace('refs/heads/', '')); + + repo.branches = branches.filter(branch => + branchGlobs.some(glob => new RegExp(glob).test(branch)) + ); + } + + if (config.revisions.tags) { + const tagGlobs = config.revisions.tags; + const git = simpleGit(); + const tagList = await git.listRemote(['--tags', cloneUrl]); + const tags = tagList + .split('\n') + .map(line => line.split('\t')[1]) + .filter(Boolean) + .map(tag => tag.replace('refs/tags/', '')); + + repo.tags = tags.filter(tag => + tagGlobs.some(glob => new RegExp(glob).test(tag)) + ); + } + } + + return repo; } \ No newline at end of file diff --git a/packages/backend/src/main.ts b/packages/backend/src/main.ts index c5aa2b66..5de504cf 100644 --- a/packages/backend/src/main.ts +++ b/packages/backend/src/main.ts @@ -6,7 +6,7 @@ import { getGitLabReposFromConfig } from "./gitlab.js"; import { getGiteaReposFromConfig } from "./gitea.js"; import { getGerritReposFromConfig } from "./gerrit.js"; import { AppContext, LocalRepository, GitRepository, Repository, Settings } from "./types.js"; -import { cloneRepository, fetchRepository } from "./git.js"; +import { cloneRepository, fetchRepository, getGitRepoFromConfig } from "./git.js"; import { createLogger } from "./logger.js"; import { createRepository, Database, loadDB, updateRepository, updateSettings } from './db.js'; import { arraysEqualShallow, isRemotePath, measure } from "./utils.js"; @@ -245,6 +245,11 @@ const syncConfig = async (configPath: string, db: Database, signal: AbortSignal, configRepos.push(repo); break; } + case 'git': { + const gitRepo = await getGitRepoFromConfig(repoConfig, ctx); + gitRepo && configRepos.push(gitRepo); + break; + } } } diff --git a/packages/backend/src/schemas/v2.ts b/packages/backend/src/schemas/v2.ts index 67897cd8..0c45d6f3 100644 --- a/packages/backend/src/schemas/v2.ts +++ b/packages/backend/src/schemas/v2.ts @@ -1,6 +1,6 @@ // THIS IS A AUTO-GENERATED FILE. DO NOT MODIFY MANUALLY! -export type Repos = GitHubConfig | GitLabConfig | GiteaConfig | GerritConfig | LocalConfig; +export type Repos = GitHubConfig | GitLabConfig | GiteaConfig | GerritConfig | LocalConfig | GitConfig; /** * A Sourcebot configuration file outlines which repositories Sourcebot should sync and index. @@ -268,3 +268,14 @@ export interface LocalConfig { paths?: string[]; }; } +export interface GitConfig { + /** + * Git Configuration + */ + type: "git"; + /** + * The URL to the git repository. + */ + url: string; + revisions?: GitRevisions; +} diff --git a/schemas/v2/index.json b/schemas/v2/index.json index ffdcff56..ee70c0c2 100644 --- a/schemas/v2/index.json +++ b/schemas/v2/index.json @@ -516,6 +516,28 @@ ], "additionalProperties": false }, + "GitConfig": { + "type": "object", + "properties": { + "type": { + "const": "git", + "description": "Git Configuration" + }, + "url": { + "type": "string", + "format": "url", + "description": "The URL to the git repository." + }, + "revisions": { + "$ref": "#/definitions/GitRevisions" + } + }, + "required": [ + "type", + "url" + ], + "additionalProperties": false + }, "Repos": { "anyOf": [ { @@ -532,6 +554,9 @@ }, { "$ref": "#/definitions/LocalConfig" + }, + { + "$ref": "#/definitions/GitConfig" } ] },