Merge pull request #898 from 89luca89/feat/git_subfolder

feat: start workspace in a subpath

.github/workflows/e2e-tests.yaml

@@ -103,5 +103,5 @@ jobs:
       if: ${{ always() }}
       run: |
         Remove-Item -Recurse C:\Users\loft-user\.devpod\
-        sh -c "docker ps -a | cut -d' ' -f1 | tail -n+2 | xargs docker rm -f || :"
-        docker system prune -a -f
+        sh -c "docker ps -q -a | xargs docker rm -f || :"
+        sh -c "docker images --format '{{.Repository}}:{{.Tag}},{{.ID}}' | grep devpod | cut -d',' -f2 | xargs docker rmi -f || :"

.github/workflows/e2e-win-full-tests.yaml

@@ -50,5 +50,5 @@ jobs:
       if: ${{ always() }}
       run: |
         Remove-Item -Recurse C:\Users\loft-user\.devpod\
-        sh -c "docker ps -a | cut -d' ' -f1 | tail -n+2 | xargs docker rm -f || :"
-        docker system prune -a -f
+        sh -c "docker ps -q -a | xargs docker rm -f || :"
+        sh -c "docker images --format '{{.Repository}}:{{.Tag}},{{.ID}}' | grep devpod | cut -d',' -f2 | xargs docker rmi -f || :"

cmd/agent/workspace/up.go

@@ -459,7 +459,7 @@ func CloneRepository(ctx context.Context, local bool, workspaceDir string, sourc
 	}
 
 	// run git command
-	gitInfo := git.NewGitInfo(source.GitRepository, source.GitBranch, source.GitCommit, source.GitPRReference)
+	gitInfo := git.NewGitInfo(source.GitRepository, source.GitBranch, source.GitCommit, source.GitPRReference, source.GitSubPath)
 	err := git.CloneRepository(ctx, gitInfo, workspaceDir, helper, false, writer, log)
 	if err != nil {
 		return errors.Wrap(err, "clone repository")

cmd/helper/get_workspace_config.go

@@ -146,12 +146,12 @@ func findDevcontainerFiles(ctx context.Context, rawSource, tmpDirPath string, ma
 	}
 
 	// git repo
-	gitRepository, gitPRReference, gitBranch, gitCommit := git.NormalizeRepository(rawSource)
+	gitRepository, gitPRReference, gitBranch, gitCommit, gitSubDir := git.NormalizeRepository(rawSource)
 	if strings.HasSuffix(rawSource, ".git") || git.PingRepository(gitRepository) {
 		log.Debug("Git repository detected")
 		result.IsGitRepository = true
 
-		gitInfo := git.NewGitInfo(gitRepository, gitBranch, gitCommit, gitPRReference)
+		gitInfo := git.NewGitInfo(gitRepository, gitBranch, gitCommit, gitPRReference, gitSubDir)
 		log.Debugf("Cloning git repository into %s", tmpDirPath)
 		err := git.CloneRepository(ctx, gitInfo, tmpDirPath, "", true, log.Writer(logrus.DebugLevel, false), log)
 		if err != nil {

cmd/up.go

@@ -8,6 +8,7 @@ import (
 	"net"
 	"os"
 	"os/exec"
+	"path/filepath"
 	"strconv"
 	"strings"
 
@@ -175,12 +176,16 @@ func (cmd *UpCmd) Run(
 		workdir = result.MergedConfig.WorkspaceFolder
 	}
 
+	if client.WorkspaceConfig().Source.GitSubPath != "" {
+		result.SubstitutionContext.ContainerWorkspaceFolder = filepath.Join(result.SubstitutionContext.ContainerWorkspaceFolder, client.WorkspaceConfig().Source.GitSubPath)
+		workdir = result.SubstitutionContext.ContainerWorkspaceFolder
+	}
+
 	// configure container ssh
 	if cmd.ConfigureSSH {
 		err = configureSSH(devPodConfig, client, cmd.SSHConfigPath, user, workdir,
 			cmd.GPGAgentForwarding ||
 				devPodConfig.ContextOption(config.ContextOptionGPGAgentForwarding) == "true")
-
 		if err != nil {
 			return err
 		}
@@ -868,7 +873,6 @@ func performGpgForwarding(
 			"--log-output=raw",
 			"--command", "sleep infinity",
 		).Run()
-
 		if err != nil {
 			log.Error("failure in forwarding gpg-agent")
 		}

e2e/tests/ssh/ssh.go

@@ -50,7 +50,7 @@ var _ = DevPodDescribe("devpod ssh test suite", func() {
 			framework.ExpectNoError(err)
 		})
 
-		ginkgo.It("should start a new workspace with a docker provider (default) and forward gpg agent into it", func() {
+		ginkgo.FIt("should start a new workspace with a docker provider (default) and forward gpg agent into it", func() {
 			// skip windows for now
 			if runtime.GOOS == "windows" {
 				return
@@ -83,7 +83,18 @@ var _ = DevPodDescribe("devpod ssh test suite", func() {
 			devpodSSHDeadline := time.Now().Add(20 * time.Second)
 			devpodSSHCtx, cancelSSH := context.WithDeadline(context.Background(), devpodSSHDeadline)
 			defer cancelSSH()
-			err = f.DevPodSSHGpgTestKey(devpodSSHCtx, tempDir)
+
+			// GPG agent might be not ready, let's try 10 times, 1 second each
+			retries := 10
+			for retries > 0 {
+				err = f.DevPodSSHGpgTestKey(devpodSSHCtx, tempDir)
+				if err != nil {
+					retries--
+					time.Sleep(time.Second)
+				} else {
+					break
+				}
+			}
 			framework.ExpectNoError(err)
 		})
 

e2e/tests/up/up.go

@@ -277,6 +277,33 @@ var _ = DevPodDescribe("devpod up test suite", func() {
 			framework.ExpectNoError(err)
 		})
 
+		ginkgo.It("create workspace in a subpath", func() {
+			const providerName = "test-docker"
+			ctx := context.Background()
+
+			f := framework.NewDefaultFramework(initialDir + "/bin")
+
+			// provider add, use and delete afterwards
+			err := f.DevPodProviderAdd(ctx, "docker", "--name", providerName)
+			framework.ExpectNoError(err)
+			err = f.DevPodProviderUse(ctx, providerName)
+			framework.ExpectNoError(err)
+			ginkgo.DeferCleanup(func() {
+				err = f.DevPodProviderDelete(ctx, providerName)
+				framework.ExpectNoError(err)
+			})
+
+			err = f.DevPodUp(ctx, "https://github.com/loft-sh/examples/@subpath:/devpod/jupyter-notebook-hello-world")
+			framework.ExpectNoError(err)
+
+			out, err := f.DevPodSSH(ctx, "jupyter-notebook-hello-world", "pwd")
+			framework.ExpectNoError(err)
+			framework.ExpectEqual(out, "/workspaces/jupyter-notebook-hello-world\n", "should be subpath")
+
+			err = f.DevPodWorkspaceDelete(ctx, "jupyter-notebook-hello-world")
+			framework.ExpectNoError(err)
+		})
+
 		ginkgo.Context("print error message correctly", func() {
 			ginkgo.It("make sure devpod output is correct and log-output works correctly", func(ctx context.Context) {
 				f := framework.NewDefaultFramework(initialDir + "/bin")

pkg/devcontainer/run.go

@@ -180,9 +180,16 @@ func (r *runner) prepare(
 	} else {
 		var err error
 
+		localWorkspaceFolder := r.LocalWorkspaceFolder
+		// if a subpath is specified, let's move to it
+
+		if r.WorkspaceConfig.Workspace.Source.GitSubPath != "" {
+			localWorkspaceFolder = filepath.Join(r.LocalWorkspaceFolder, r.WorkspaceConfig.Workspace.Source.GitSubPath)
+		}
+
 		// parse the devcontainer json
 		rawParsedConfig, err = config.ParseDevContainerJSON(
-			r.LocalWorkspaceFolder,
+			localWorkspaceFolder,
 			r.WorkspaceConfig.Workspace.DevContainerPath,
 		)
 

pkg/git/git.go

@@ -17,12 +17,14 @@ import (
 const (
 	CommitDelimiter      string = "@sha256:"
 	PullRequestReference string = "pull/([0-9]+)/head"
+	SubPathDelimiter     string = "@subpath:"
 )
 
 var (
 	branchRegEx      = regexp.MustCompile(`^([^@]*(?:git@)?[^@/]+/[^@/]+/?[^@/]+)@([a-zA-Z0-9\./\-\_]+)$`)
 	commitRegEx      = regexp.MustCompile(`^([^@]*(?:git@)?[^@/]+/[^@]+)` + regexp.QuoteMeta(CommitDelimiter) + `([a-zA-Z0-9]+)$`)
 	prReferenceRegEx = regexp.MustCompile(`^([^@]*(?:git@)?[^@/]+/[^@]+)@(` + PullRequestReference + `)$`)
+	subPathRegEx     = regexp.MustCompile(`^([^@]*(?:git@)?[^@/]+/[^@]+)` + regexp.QuoteMeta(SubPathDelimiter) + `([a-zA-Z0-9\./\-\_]+)$`)
 )
 
 func CommandContext(ctx context.Context, args ...string) *exec.Cmd {
@@ -33,7 +35,7 @@ func CommandContext(ctx context.Context, args ...string) *exec.Cmd {
 	return cmd
 }
 
-func NormalizeRepository(str string) (string, string, string, string) {
+func NormalizeRepository(str string) (string, string, string, string, string) {
 	if !strings.HasPrefix(str, "ssh://") && !strings.HasPrefix(str, "git@") && !strings.HasPrefix(str, "http://") && !strings.HasPrefix(str, "https://") {
 		str = "https://" + str
 	}
@@ -44,7 +46,14 @@ func NormalizeRepository(str string) (string, string, string, string) {
 		str = match[1]
 		prReference = match[2]
 
-		return str, prReference, "", ""
+		return str, prReference, "", "", ""
+	}
+
+	// resolve subpath
+	subpath := ""
+	if match := subPathRegEx.FindStringSubmatch(str); match != nil {
+		str = match[1]
+		subpath = match[2]
 	}
 
 	// resolve branch
@@ -61,7 +70,7 @@ func NormalizeRepository(str string) (string, string, string, string) {
 		commit = match[2]
 	}
 
-	return str, prReference, branch, commit
+	return str, prReference, branch, commit, subpath
 }
 
 func PingRepository(str string) bool {
@@ -86,20 +95,22 @@ type GitInfo struct {
 	Branch     string
 	Commit     string
 	PR         string
+	SubPath    string
 }
 
-func NewGitInfo(repository, branch, commit, pr string) *GitInfo {
+func NewGitInfo(repository, branch, commit, pr, subpath string) *GitInfo {
 	return &GitInfo{
 		Repository: repository,
 		Branch:     branch,
 		Commit:     commit,
 		PR:         pr,
+		SubPath:    subpath,
 	}
 }
 
 func NormalizeRepositoryGitInfo(str string) *GitInfo {
-	repository, pr, branch, commit := NormalizeRepository(str)
-	return NewGitInfo(repository, branch, commit, pr)
+	repository, pr, branch, commit, subpath := NormalizeRepository(str)
+	return NewGitInfo(repository, branch, commit, pr, subpath)
 }
 
 func CloneRepository(ctx context.Context, gitInfo *GitInfo, targetDir string, helper string, bare bool, writer io.Writer, log log.Logger) error {

pkg/git/git_test.go

@@ -13,6 +13,7 @@ type testCaseNormalizeRepository struct {
 	expectedRepo        string
 	expectedBranch      string
 	expectedCommit      string
+	expectedSubpath     string
 }
 
 type testCaseGetBranchNameForPR struct {
@@ -28,113 +29,137 @@ func TestNormalizeRepository(t *testing.T) {
 			expectedPRReference: "",
 			expectedBranch:      "",
 			expectedCommit:      "",
+			expectedSubpath:     "",
 		},
 		{
 			in:                  "ssh://git@github.com/loft-sh/devpod.git",
 			expectedRepo:        "ssh://git@github.com/loft-sh/devpod.git",
 			expectedPRReference: "",
 			expectedBranch:      "",
 			expectedCommit:      "",
+			expectedSubpath:     "",
 		},
 		{
 			in:                  "git@github.com/loft-sh/devpod-without-branch.git",
 			expectedRepo:        "git@github.com/loft-sh/devpod-without-branch.git",
 			expectedPRReference: "",
 			expectedBranch:      "",
 			expectedCommit:      "",
+			expectedSubpath:     "",
 		},
 		{
 			in:                  "https://github.com/loft-sh/devpod.git",
 			expectedRepo:        "https://github.com/loft-sh/devpod.git",
 			expectedPRReference: "",
 			expectedBranch:      "",
 			expectedCommit:      "",
+			expectedSubpath:     "",
 		},
 		{
 			in:                  "github.com/loft-sh/devpod.git",
 			expectedRepo:        "https://github.com/loft-sh/devpod.git",
 			expectedPRReference: "",
 			expectedBranch:      "",
 			expectedCommit:      "",
+			expectedSubpath:     "",
 		},
 		{
 			in:                  "github.com/loft-sh/devpod.git@test-branch",
 			expectedRepo:        "https://github.com/loft-sh/devpod.git",
 			expectedPRReference: "",
 			expectedBranch:      "test-branch",
 			expectedCommit:      "",
+			expectedSubpath:     "",
 		},
 		{
 			in:                  "git@github.com/loft-sh/devpod-with-branch.git@test-branch",
 			expectedRepo:        "git@github.com/loft-sh/devpod-with-branch.git",
 			expectedPRReference: "",
 			expectedBranch:      "test-branch",
 			expectedCommit:      "",
+			expectedSubpath:     "",
 		},
 		{
 			in:                  "git@github.com/loft-sh/devpod-with-branch.git@test_branch",
 			expectedRepo:        "git@github.com/loft-sh/devpod-with-branch.git",
 			expectedPRReference: "",
 			expectedBranch:      "test_branch",
 			expectedCommit:      "",
+			expectedSubpath:     "",
 		},
 		{
 			in:                  "ssh://git@github.com/loft-sh/devpod.git@test_branch",
 			expectedRepo:        "ssh://git@github.com/loft-sh/devpod.git",
 			expectedPRReference: "",
 			expectedBranch:      "test_branch",
 			expectedCommit:      "",
+			expectedSubpath:     "",
 		},
 		{
 			in:                  "github.com/loft-sh/devpod-without-protocol-with-slash.git@user/branch",
 			expectedRepo:        "https://github.com/loft-sh/devpod-without-protocol-with-slash.git",
 			expectedPRReference: "",
 			expectedBranch:      "user/branch",
 			expectedCommit:      "",
+			expectedSubpath:     "",
 		},
 		{
 			in:                  "git@github.com/loft-sh/devpod-with-slash.git@user/branch",
 			expectedRepo:        "git@github.com/loft-sh/devpod-with-slash.git",
 			expectedPRReference: "",
 			expectedBranch:      "user/branch",
 			expectedCommit:      "",
+			expectedSubpath:     "",
 		},
 		{
 			in:                  "github.com/loft-sh/devpod.git@sha256:905ffb0",
 			expectedRepo:        "https://github.com/loft-sh/devpod.git",
 			expectedPRReference: "",
 			expectedBranch:      "",
 			expectedCommit:      "905ffb0",
+			expectedSubpath:     "",
 		},
 		{
 			in:                  "git@github.com:loft-sh/devpod.git@sha256:905ffb0",
 			expectedRepo:        "git@github.com:loft-sh/devpod.git",
 			expectedPRReference: "",
 			expectedBranch:      "",
 			expectedCommit:      "905ffb0",
+			expectedSubpath:     "",
 		},
 		{
 			in:                  "github.com/loft-sh/devpod.git@pull/996/head",
 			expectedRepo:        "https://github.com/loft-sh/devpod.git",
 			expectedPRReference: "pull/996/head",
 			expectedBranch:      "",
 			expectedCommit:      "",
+			expectedSubpath:     "",
 		},
 		{
 			in:                  "git@github.com:loft-sh/devpod.git@pull/996/head",
 			expectedRepo:        "git@github.com:loft-sh/devpod.git",
 			expectedPRReference: "pull/996/head",
 			expectedBranch:      "",
 			expectedCommit:      "",
+			expectedSubpath:     "",
+		},
+		{
+			in:                  "github.com/loft-sh/devpod-without-protocol-with-slash.git@subpath:/test/path",
+			expectedRepo:        "https://github.com/loft-sh/devpod-without-protocol-with-slash.git",
+			expectedPRReference: "",
+			expectedBranch:      "",
+			expectedCommit:      "",
+			expectedSubpath:     "/test/path",
 		},
 	}
 
 	for _, testCase := range testCases {
-		outRepo, outPRReference, outBranch, outCommit := NormalizeRepository(testCase.in)
+		outRepo, outPRReference, outBranch, outCommit, outSubpath := NormalizeRepository(testCase.in)
 		assert.Check(t, cmp.Equal(testCase.expectedRepo, outRepo))
 		assert.Check(t, cmp.Equal(testCase.expectedPRReference, outPRReference))
 		assert.Check(t, cmp.Equal(testCase.expectedBranch, outBranch))
 		assert.Check(t, cmp.Equal(testCase.expectedCommit, outCommit))
+		assert.Check(t, cmp.Equal(testCase.expectedSubpath, outSubpath))
 	}
 }