diff --git a/.github/workflows/e2e-tests.yaml b/.github/workflows/e2e-tests.yaml index 74517ed90..ced54e1ca 100644 --- a/.github/workflows/e2e-tests.yaml +++ b/.github/workflows/e2e-tests.yaml @@ -103,5 +103,5 @@ jobs: if: ${{ always() }} run: | Remove-Item -Recurse C:\Users\loft-user\.devpod\ - sh -c "docker ps -a | cut -d' ' -f1 | tail -n+2 | xargs docker rm -f || :" - docker system prune -a -f + sh -c "docker ps -q -a | xargs docker rm -f || :" + sh -c "docker images --format '{{.Repository}}:{{.Tag}},{{.ID}}' | grep devpod | cut -d',' -f2 | xargs docker rmi -f || :" diff --git a/.github/workflows/e2e-win-full-tests.yaml b/.github/workflows/e2e-win-full-tests.yaml index dadc98fc2..f884ec01e 100644 --- a/.github/workflows/e2e-win-full-tests.yaml +++ b/.github/workflows/e2e-win-full-tests.yaml @@ -50,5 +50,5 @@ jobs: if: ${{ always() }} run: | Remove-Item -Recurse C:\Users\loft-user\.devpod\ - sh -c "docker ps -a | cut -d' ' -f1 | tail -n+2 | xargs docker rm -f || :" - docker system prune -a -f + sh -c "docker ps -q -a | xargs docker rm -f || :" + sh -c "docker images --format '{{.Repository}}:{{.Tag}},{{.ID}}' | grep devpod | cut -d',' -f2 | xargs docker rmi -f || :" diff --git a/cmd/agent/workspace/up.go b/cmd/agent/workspace/up.go index a6ac2d170..f3f846863 100644 --- a/cmd/agent/workspace/up.go +++ b/cmd/agent/workspace/up.go @@ -459,7 +459,7 @@ func CloneRepository(ctx context.Context, local bool, workspaceDir string, sourc } // run git command - gitInfo := git.NewGitInfo(source.GitRepository, source.GitBranch, source.GitCommit, source.GitPRReference) + gitInfo := git.NewGitInfo(source.GitRepository, source.GitBranch, source.GitCommit, source.GitPRReference, source.GitSubPath) err := git.CloneRepository(ctx, gitInfo, workspaceDir, helper, false, writer, log) if err != nil { return errors.Wrap(err, "clone repository") diff --git a/cmd/helper/get_workspace_config.go b/cmd/helper/get_workspace_config.go index 3cb1ae579..8ff7a3f3d 100644 --- a/cmd/helper/get_workspace_config.go +++ b/cmd/helper/get_workspace_config.go @@ -146,12 +146,12 @@ func findDevcontainerFiles(ctx context.Context, rawSource, tmpDirPath string, ma } // git repo - gitRepository, gitPRReference, gitBranch, gitCommit := git.NormalizeRepository(rawSource) + gitRepository, gitPRReference, gitBranch, gitCommit, gitSubDir := git.NormalizeRepository(rawSource) if strings.HasSuffix(rawSource, ".git") || git.PingRepository(gitRepository) { log.Debug("Git repository detected") result.IsGitRepository = true - gitInfo := git.NewGitInfo(gitRepository, gitBranch, gitCommit, gitPRReference) + gitInfo := git.NewGitInfo(gitRepository, gitBranch, gitCommit, gitPRReference, gitSubDir) log.Debugf("Cloning git repository into %s", tmpDirPath) err := git.CloneRepository(ctx, gitInfo, tmpDirPath, "", true, log.Writer(logrus.DebugLevel, false), log) if err != nil { diff --git a/cmd/up.go b/cmd/up.go index f4bd57199..59fe56736 100644 --- a/cmd/up.go +++ b/cmd/up.go @@ -8,6 +8,7 @@ import ( "net" "os" "os/exec" + "path/filepath" "strconv" "strings" @@ -175,12 +176,16 @@ func (cmd *UpCmd) Run( workdir = result.MergedConfig.WorkspaceFolder } + if client.WorkspaceConfig().Source.GitSubPath != "" { + result.SubstitutionContext.ContainerWorkspaceFolder = filepath.Join(result.SubstitutionContext.ContainerWorkspaceFolder, client.WorkspaceConfig().Source.GitSubPath) + workdir = result.SubstitutionContext.ContainerWorkspaceFolder + } + // configure container ssh if cmd.ConfigureSSH { err = configureSSH(devPodConfig, client, cmd.SSHConfigPath, user, workdir, cmd.GPGAgentForwarding || devPodConfig.ContextOption(config.ContextOptionGPGAgentForwarding) == "true") - if err != nil { return err } @@ -868,7 +873,6 @@ func performGpgForwarding( "--log-output=raw", "--command", "sleep infinity", ).Run() - if err != nil { log.Error("failure in forwarding gpg-agent") } diff --git a/e2e/tests/ssh/ssh.go b/e2e/tests/ssh/ssh.go index c4f09fafc..5cbdfc724 100644 --- a/e2e/tests/ssh/ssh.go +++ b/e2e/tests/ssh/ssh.go @@ -50,7 +50,7 @@ var _ = DevPodDescribe("devpod ssh test suite", func() { framework.ExpectNoError(err) }) - ginkgo.It("should start a new workspace with a docker provider (default) and forward gpg agent into it", func() { + ginkgo.FIt("should start a new workspace with a docker provider (default) and forward gpg agent into it", func() { // skip windows for now if runtime.GOOS == "windows" { return @@ -83,7 +83,18 @@ var _ = DevPodDescribe("devpod ssh test suite", func() { devpodSSHDeadline := time.Now().Add(20 * time.Second) devpodSSHCtx, cancelSSH := context.WithDeadline(context.Background(), devpodSSHDeadline) defer cancelSSH() - err = f.DevPodSSHGpgTestKey(devpodSSHCtx, tempDir) + + // GPG agent might be not ready, let's try 10 times, 1 second each + retries := 10 + for retries > 0 { + err = f.DevPodSSHGpgTestKey(devpodSSHCtx, tempDir) + if err != nil { + retries-- + time.Sleep(time.Second) + } else { + break + } + } framework.ExpectNoError(err) }) diff --git a/e2e/tests/up/up.go b/e2e/tests/up/up.go index dc9534e20..9d8574460 100644 --- a/e2e/tests/up/up.go +++ b/e2e/tests/up/up.go @@ -277,6 +277,33 @@ var _ = DevPodDescribe("devpod up test suite", func() { framework.ExpectNoError(err) }) + ginkgo.It("create workspace in a subpath", func() { + const providerName = "test-docker" + ctx := context.Background() + + f := framework.NewDefaultFramework(initialDir + "/bin") + + // provider add, use and delete afterwards + err := f.DevPodProviderAdd(ctx, "docker", "--name", providerName) + framework.ExpectNoError(err) + err = f.DevPodProviderUse(ctx, providerName) + framework.ExpectNoError(err) + ginkgo.DeferCleanup(func() { + err = f.DevPodProviderDelete(ctx, providerName) + framework.ExpectNoError(err) + }) + + err = f.DevPodUp(ctx, "https://github.com/loft-sh/examples/@subpath:/devpod/jupyter-notebook-hello-world") + framework.ExpectNoError(err) + + out, err := f.DevPodSSH(ctx, "jupyter-notebook-hello-world", "pwd") + framework.ExpectNoError(err) + framework.ExpectEqual(out, "/workspaces/jupyter-notebook-hello-world\n", "should be subpath") + + err = f.DevPodWorkspaceDelete(ctx, "jupyter-notebook-hello-world") + framework.ExpectNoError(err) + }) + ginkgo.Context("print error message correctly", func() { ginkgo.It("make sure devpod output is correct and log-output works correctly", func(ctx context.Context) { f := framework.NewDefaultFramework(initialDir + "/bin") diff --git a/pkg/devcontainer/run.go b/pkg/devcontainer/run.go index eaf9b2e21..1f519dd6b 100644 --- a/pkg/devcontainer/run.go +++ b/pkg/devcontainer/run.go @@ -180,9 +180,16 @@ func (r *runner) prepare( } else { var err error + localWorkspaceFolder := r.LocalWorkspaceFolder + // if a subpath is specified, let's move to it + + if r.WorkspaceConfig.Workspace.Source.GitSubPath != "" { + localWorkspaceFolder = filepath.Join(r.LocalWorkspaceFolder, r.WorkspaceConfig.Workspace.Source.GitSubPath) + } + // parse the devcontainer json rawParsedConfig, err = config.ParseDevContainerJSON( - r.LocalWorkspaceFolder, + localWorkspaceFolder, r.WorkspaceConfig.Workspace.DevContainerPath, ) diff --git a/pkg/git/git.go b/pkg/git/git.go index 330d14e2e..12b01ed62 100644 --- a/pkg/git/git.go +++ b/pkg/git/git.go @@ -17,12 +17,14 @@ import ( const ( CommitDelimiter string = "@sha256:" PullRequestReference string = "pull/([0-9]+)/head" + SubPathDelimiter string = "@subpath:" ) var ( branchRegEx = regexp.MustCompile(`^([^@]*(?:git@)?[^@/]+/[^@/]+/?[^@/]+)@([a-zA-Z0-9\./\-\_]+)$`) commitRegEx = regexp.MustCompile(`^([^@]*(?:git@)?[^@/]+/[^@]+)` + regexp.QuoteMeta(CommitDelimiter) + `([a-zA-Z0-9]+)$`) prReferenceRegEx = regexp.MustCompile(`^([^@]*(?:git@)?[^@/]+/[^@]+)@(` + PullRequestReference + `)$`) + subPathRegEx = regexp.MustCompile(`^([^@]*(?:git@)?[^@/]+/[^@]+)` + regexp.QuoteMeta(SubPathDelimiter) + `([a-zA-Z0-9\./\-\_]+)$`) ) func CommandContext(ctx context.Context, args ...string) *exec.Cmd { @@ -33,7 +35,7 @@ func CommandContext(ctx context.Context, args ...string) *exec.Cmd { return cmd } -func NormalizeRepository(str string) (string, string, string, string) { +func NormalizeRepository(str string) (string, string, string, string, string) { if !strings.HasPrefix(str, "ssh://") && !strings.HasPrefix(str, "git@") && !strings.HasPrefix(str, "http://") && !strings.HasPrefix(str, "https://") { str = "https://" + str } @@ -44,7 +46,14 @@ func NormalizeRepository(str string) (string, string, string, string) { str = match[1] prReference = match[2] - return str, prReference, "", "" + return str, prReference, "", "", "" + } + + // resolve subpath + subpath := "" + if match := subPathRegEx.FindStringSubmatch(str); match != nil { + str = match[1] + subpath = match[2] } // resolve branch @@ -61,7 +70,7 @@ func NormalizeRepository(str string) (string, string, string, string) { commit = match[2] } - return str, prReference, branch, commit + return str, prReference, branch, commit, subpath } func PingRepository(str string) bool { @@ -86,20 +95,22 @@ type GitInfo struct { Branch string Commit string PR string + SubPath string } -func NewGitInfo(repository, branch, commit, pr string) *GitInfo { +func NewGitInfo(repository, branch, commit, pr, subpath string) *GitInfo { return &GitInfo{ Repository: repository, Branch: branch, Commit: commit, PR: pr, + SubPath: subpath, } } func NormalizeRepositoryGitInfo(str string) *GitInfo { - repository, pr, branch, commit := NormalizeRepository(str) - return NewGitInfo(repository, branch, commit, pr) + repository, pr, branch, commit, subpath := NormalizeRepository(str) + return NewGitInfo(repository, branch, commit, pr, subpath) } func CloneRepository(ctx context.Context, gitInfo *GitInfo, targetDir string, helper string, bare bool, writer io.Writer, log log.Logger) error { diff --git a/pkg/git/git_test.go b/pkg/git/git_test.go index f0a63e904..5c997037e 100644 --- a/pkg/git/git_test.go +++ b/pkg/git/git_test.go @@ -13,6 +13,7 @@ type testCaseNormalizeRepository struct { expectedRepo string expectedBranch string expectedCommit string + expectedSubpath string } type testCaseGetBranchNameForPR struct { @@ -28,6 +29,7 @@ func TestNormalizeRepository(t *testing.T) { expectedPRReference: "", expectedBranch: "", expectedCommit: "", + expectedSubpath: "", }, { in: "ssh://git@github.com/loft-sh/devpod.git", @@ -35,6 +37,7 @@ func TestNormalizeRepository(t *testing.T) { expectedPRReference: "", expectedBranch: "", expectedCommit: "", + expectedSubpath: "", }, { in: "git@github.com/loft-sh/devpod-without-branch.git", @@ -42,6 +45,7 @@ func TestNormalizeRepository(t *testing.T) { expectedPRReference: "", expectedBranch: "", expectedCommit: "", + expectedSubpath: "", }, { in: "https://github.com/loft-sh/devpod.git", @@ -49,6 +53,7 @@ func TestNormalizeRepository(t *testing.T) { expectedPRReference: "", expectedBranch: "", expectedCommit: "", + expectedSubpath: "", }, { in: "github.com/loft-sh/devpod.git", @@ -56,6 +61,7 @@ func TestNormalizeRepository(t *testing.T) { expectedPRReference: "", expectedBranch: "", expectedCommit: "", + expectedSubpath: "", }, { in: "github.com/loft-sh/devpod.git@test-branch", @@ -63,6 +69,7 @@ func TestNormalizeRepository(t *testing.T) { expectedPRReference: "", expectedBranch: "test-branch", expectedCommit: "", + expectedSubpath: "", }, { in: "git@github.com/loft-sh/devpod-with-branch.git@test-branch", @@ -70,6 +77,7 @@ func TestNormalizeRepository(t *testing.T) { expectedPRReference: "", expectedBranch: "test-branch", expectedCommit: "", + expectedSubpath: "", }, { in: "git@github.com/loft-sh/devpod-with-branch.git@test_branch", @@ -77,6 +85,7 @@ func TestNormalizeRepository(t *testing.T) { expectedPRReference: "", expectedBranch: "test_branch", expectedCommit: "", + expectedSubpath: "", }, { in: "ssh://git@github.com/loft-sh/devpod.git@test_branch", @@ -84,6 +93,7 @@ func TestNormalizeRepository(t *testing.T) { expectedPRReference: "", expectedBranch: "test_branch", expectedCommit: "", + expectedSubpath: "", }, { in: "github.com/loft-sh/devpod-without-protocol-with-slash.git@user/branch", @@ -91,6 +101,7 @@ func TestNormalizeRepository(t *testing.T) { expectedPRReference: "", expectedBranch: "user/branch", expectedCommit: "", + expectedSubpath: "", }, { in: "git@github.com/loft-sh/devpod-with-slash.git@user/branch", @@ -98,6 +109,7 @@ func TestNormalizeRepository(t *testing.T) { expectedPRReference: "", expectedBranch: "user/branch", expectedCommit: "", + expectedSubpath: "", }, { in: "github.com/loft-sh/devpod.git@sha256:905ffb0", @@ -105,6 +117,7 @@ func TestNormalizeRepository(t *testing.T) { expectedPRReference: "", expectedBranch: "", expectedCommit: "905ffb0", + expectedSubpath: "", }, { in: "git@github.com:loft-sh/devpod.git@sha256:905ffb0", @@ -112,6 +125,7 @@ func TestNormalizeRepository(t *testing.T) { expectedPRReference: "", expectedBranch: "", expectedCommit: "905ffb0", + expectedSubpath: "", }, { in: "github.com/loft-sh/devpod.git@pull/996/head", @@ -119,6 +133,7 @@ func TestNormalizeRepository(t *testing.T) { expectedPRReference: "pull/996/head", expectedBranch: "", expectedCommit: "", + expectedSubpath: "", }, { in: "git@github.com:loft-sh/devpod.git@pull/996/head", @@ -126,15 +141,25 @@ func TestNormalizeRepository(t *testing.T) { expectedPRReference: "pull/996/head", expectedBranch: "", expectedCommit: "", + expectedSubpath: "", + }, + { + in: "github.com/loft-sh/devpod-without-protocol-with-slash.git@subpath:/test/path", + expectedRepo: "https://github.com/loft-sh/devpod-without-protocol-with-slash.git", + expectedPRReference: "", + expectedBranch: "", + expectedCommit: "", + expectedSubpath: "/test/path", }, } for _, testCase := range testCases { - outRepo, outPRReference, outBranch, outCommit := NormalizeRepository(testCase.in) + outRepo, outPRReference, outBranch, outCommit, outSubpath := NormalizeRepository(testCase.in) assert.Check(t, cmp.Equal(testCase.expectedRepo, outRepo)) assert.Check(t, cmp.Equal(testCase.expectedPRReference, outPRReference)) assert.Check(t, cmp.Equal(testCase.expectedBranch, outBranch)) assert.Check(t, cmp.Equal(testCase.expectedCommit, outCommit)) + assert.Check(t, cmp.Equal(testCase.expectedSubpath, outSubpath)) } } diff --git a/pkg/provider/workspace.go b/pkg/provider/workspace.go index 91e8de773..5a78ca41a 100644 --- a/pkg/provider/workspace.go +++ b/pkg/provider/workspace.go @@ -104,6 +104,9 @@ type WorkspaceSource struct { // GitPRReference is the pull request reference to checkout GitPRReference string `json:"gitPRReference,omitempty"` + // GitSubPath is the subpath in the repo to use + GitSubPath string `json:"gitSubDir,omitempty"` + // LocalFolder is the local folder to use LocalFolder string `json:"localFolder,omitempty"` @@ -216,12 +219,13 @@ func (w WorkspaceSource) String() string { func ParseWorkspaceSource(source string) *WorkspaceSource { if strings.HasPrefix(source, WorkspaceSourceGit) { - gitRepo, gitPRReference, gitBranch, gitCommit := git.NormalizeRepository(strings.TrimPrefix(source, WorkspaceSourceGit)) + gitRepo, gitPRReference, gitBranch, gitCommit, gitSubdir := git.NormalizeRepository(strings.TrimPrefix(source, WorkspaceSourceGit)) return &WorkspaceSource{ GitRepository: gitRepo, GitPRReference: gitPRReference, GitBranch: gitBranch, GitCommit: gitCommit, + GitSubPath: gitSubdir, } } else if strings.HasPrefix(source, WorkspaceSourceLocal) { return &WorkspaceSource{ diff --git a/pkg/workspace/workspace.go b/pkg/workspace/workspace.go index 16e2301a8..985b76b9d 100644 --- a/pkg/workspace/workspace.go +++ b/pkg/workspace/workspace.go @@ -450,7 +450,7 @@ func resolve( } // is git? - gitRepository, gitPRReference, gitBranch, gitCommit := git.NormalizeRepository(name) + gitRepository, gitPRReference, gitBranch, gitCommit, gitSubdir := git.NormalizeRepository(name) if strings.HasSuffix(name, ".git") || git.PingRepository(gitRepository) { workspace.Picture = getProjectImage(name) workspace.Source = provider2.WorkspaceSource{ @@ -458,6 +458,7 @@ func resolve( GitPRReference: gitPRReference, GitBranch: gitBranch, GitCommit: gitCommit, + GitSubPath: gitSubdir, } return workspace, nil } @@ -524,8 +525,10 @@ func getProjectImage(link string) string { return "" } -var workspaceIDRegEx1 = regexp.MustCompile(`[^\w\-]`) -var workspaceIDRegEx2 = regexp.MustCompile(`[^0-9a-z\-]+`) +var ( + workspaceIDRegEx1 = regexp.MustCompile(`[^\w\-]`) + workspaceIDRegEx2 = regexp.MustCompile(`[^0-9a-z\-]+`) +) func ToID(str string) string { str = strings.ToLower(filepath.ToSlash(str))