Upload only files git doesn't ignore on devpod up of a local folder

CHANGELOG.md

@@ -12,6 +12,17 @@ Add new entries to the top of the "Unreleased" section.
 - `devpod up --git-user-name` and `--git-user-email` set the git identity that the workspace's dev container
   gets, and save it with the workspace. An empty value removes the saved one.
 
+### Changed
+
+- `devpod up` of a local folder in a git repository only uploads the files git doesn't ignore, instead of the
+  whole folder, when the workspace runs on another machine or in Kubernetes. Ignored files (build output,
+  `node_modules`, ...) and nested repositories and git worktrees stay behind. Instead of the repository's
+  `.git`, the workspace gets a shallow one with the checked out commit, the branch, its upstream branch and the
+  remotes, which is enough to commit and push; so does each checked out submodule. `.devpodignore` takes
+  precedence: its patterns leave out more files, and its negation (`!`) patterns upload files git ignores
+  (`!.git` uploads the whole `.git`). See
+  [Which files are uploaded](https://devpod.curityio.net/developing-in-workspaces/create-a-workspace#which-files-are-uploaded).
+
 ### Fixed
 
 - The git identity that DevPod gives a dev container is the one git uses in the repository the workspace comes

docs/pages/developing-in-workspaces/create-a-workspace.mdx

@@ -87,6 +87,43 @@ devpod up ./path/to/my-folder
 
 DevPod will sync the folder into the remote machine and create a development environment from the `devcontainer.json`.
 
+##### Which files are uploaded
+
+When the workspace runs on another machine or in Kubernetes, DevPod uploads the folder. If the folder is in a git
+repository, DevPod only uploads the files git doesn't ignore: tracked files, and untracked files that no
+`.gitignore` file, `.git/info/exclude` or `core.excludesFile` ignores. So build output, `node_modules` and the like
+stay behind, as do other repositories and git worktrees inside the folder. The files of submodules are uploaded.
+
+If the folder is the top of the repository, the workspace gets a small `.git` in place of yours, with only the
+checked out commit, so that you can commit and push from the workspace. It has the same branch, remotes and
+upstream branch as yours, and the commits since your branch and its upstream branch diverged, but no other
+history. Changes you have staged show as not staged in the workspace. Each checked out submodule gets a small
+`.git` of its own in the same way.
+
+A `.devpodignore` file at the top of the folder takes precedence over git. It has the syntax of a
+[`.dockerignore` file](https://docs.docker.com/build/concepts/context/#dockerignore-files), so patterns are
+relative to the folder: `build` only matches the `build` folder at the top, `**/build` matches one anywhere. A
+pattern leaves out files git doesn't ignore, and a negation (`!`) pattern uploads files git ignores:
+
+```
+# a file git ignores, but the workspace needs
+!.env
+# files git doesn't ignore, but the workspace doesn't need
+docs/**/*.mp4
+```
+
+Tracked files that `.devpodignore` leaves out don't show as deleted in the workspace, as git marks them
+[skip-worktree](https://git-scm.com/docs/git-update-index#_skip_worktree_bit), but their content is still uploaded
+in the small `.git`. `!.git` uploads your whole `.git` instead of the small one, and `.git` uploads no `.git` at all.
+
+If the folder is not in a git repository, or git is not installed, DevPod uploads every file `.devpodignore`
+doesn't exclude.
+
+:::info
+When the workspace runs on your machine, e.g. with the `docker` provider and a local Docker, the folder is mounted
+and not uploaded, so the workspace has all of it.
+:::
+
 #### Docker Image
 
 Run the following command in a terminal to create a new workspace from a docker image:

go.mod

@@ -195,7 +195,7 @@ require (
 	github.com/xeipuuv/gojsonschema v1.2.0 // indirect
 	golang.org/x/net v0.37.0 // indirect
 	golang.org/x/sync v0.12.0 // indirect
-	golang.org/x/text v0.23.0 // indirect
+	golang.org/x/text v0.23.0
 	golang.org/x/time v0.9.0 // indirect
 	golang.org/x/tools v0.29.0 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect

pkg/agent/tunnelserver/tunnelserver.go

@@ -5,6 +5,7 @@ import (
 	"context"
 	"encoding/base64"
 	"encoding/json"
+	"errors"
 	"fmt"
 	"io"
 	"os"
@@ -15,6 +16,7 @@ import (
 	"github.com/loft-sh/devpod/pkg/devcontainer/config"
 	"github.com/loft-sh/devpod/pkg/dockercredentials"
 	"github.com/loft-sh/devpod/pkg/extract"
+	"github.com/loft-sh/devpod/pkg/git"
 	"github.com/loft-sh/devpod/pkg/gitcredentials"
 	"github.com/loft-sh/devpod/pkg/gitsshsigning"
 	"github.com/loft-sh/devpod/pkg/gpg"
@@ -325,10 +327,11 @@ func (t *tunnelServer) StreamWorkspace(message *tunnel.Empty, stream tunnel.Tunn
 		return fmt.Errorf("workspace is nil")
 	}
 
-	excludes := t.readIgnoreFile(t.workspace.Source.LocalFolder)
+	opts, cleanup := t.workspaceTarOptions(stream.Context(), t.workspace.Source.LocalFolder)
+	defer cleanup()
 
 	buf := bufio.NewWriterSize(NewStreamWriter(stream, t.log), 10*1024)
-	err := extract.WriteTarWithOptions(buf, t.workspace.Source.LocalFolder, streamTarOptions(excludes))
+	err := extract.WriteTarWithOptions(buf, t.workspace.Source.LocalFolder, opts)
 	if err != nil {
 		return err
 	}
@@ -349,17 +352,19 @@ func (t *tunnelServer) StreamMount(message *tunnel.StreamMountRequest, stream tu
 		return fmt.Errorf("mount %s is not allowed to download", message.Mount)
 	}
 
-	// The ignore file only applies to the workspace folder, other bind mounts
+	// The ignore files only apply to the workspace folder, other bind mounts
 	// are streamed whole
-	var excludes []string
+	opts := extract.TarOptions{}
 	if t.workspaceMount != nil && mount.String() == t.workspaceMount.String() {
-		excludes = t.readIgnoreFile(mount.Source)
+		var cleanup func()
+		opts, cleanup = t.workspaceTarOptions(stream.Context(), mount.Source)
+		defer cleanup()
 	} else {
 		t.log.Debugf("Stream mount %s without excludes, it is not the workspace mount", mount.String())
 	}
 
 	buf := bufio.NewWriterSize(NewStreamWriter(stream, t.log), 10*1024)
-	err := extract.WriteTarWithOptions(buf, mount.Source, streamTarOptions(excludes))
+	err := extract.WriteTarWithOptions(buf, mount.Source, opts)
 	if err != nil {
 		return err
 	}
@@ -399,12 +404,94 @@ func (t *tunnelServer) readIgnoreFile(folder string) []string {
 	return excludes
 }
 
-// streamTarOptions returns the options to stream a folder into the container
-// with excludes. The dockerless build context is always streamed whole, as the
-// build fails without it.
-func streamTarOptions(excludes []string) extract.TarOptions {
-	return extract.TarOptions{
-		Excludes:  excludes,
+// gitFolder is the folder of a git repository in its work tree.
+const gitFolder = ".git"
+
+// workspaceTarOptions returns the options to stream the workspace folder into
+// the container, and a function that removes what they need once streamed.
+//
+// If the folder is in a git work tree, only the files git doesn't ignore are
+// streamed, and the ignore file takes precedence: it can leave out files git
+// doesn't ignore, and re-include files git ignores with negation (!) patterns.
+// If the folder is the top level of the work tree, its .git is replaced by a
+// shallow copy unless the ignore file has a pattern matching .git. The
+// dockerless build context is always streamed whole, as the build fails
+// without it.
+func (t *tunnelServer) workspaceTarOptions(ctx context.Context, folder string) (extract.TarOptions, func()) {
+	opts := extract.TarOptions{
+		Excludes:  t.readIgnoreFile(folder),
 		KeepNames: []string{config.DevPodContextFeatureFolder},
 	}
+	noCleanup := func() {}
+
+	files, err := git.ListFiles(ctx, folder)
+	if err != nil {
+		if errors.Is(err, git.ErrNotInstalled) {
+			t.log.Warnf("Git is not installed, so all files of %s not excluded by %s are uploaded, including those .gitignore files ignore", folder, ignoreFileName)
+		} else {
+			t.log.Debugf("Upload all files of %s not excluded by %s, git can't list its files: %v", folder, ignoreFileName, err)
+		}
+		return opts, noCleanup
+	}
+	// e.g. the folder is in a repository of dotfiles that ignores everything
+	if len(files) == 0 {
+		t.log.Debugf("Upload all files of %s not excluded by %s, git doesn't list any", folder, ignoreFileName)
+		return opts, noCleanup
+	}
+	opts.Files = extract.NewFileList(files, git.IgnoresCase(ctx, folder))
+	t.log.Debugf("Upload the %d files of %s git doesn't ignore, and those %s re-includes", opts.Files.Len(), folder, ignoreFileName)
+
+	if !git.IsTopLevel(ctx, folder) {
+		return opts, noCleanup
+	}
+	// readIgnoreFile only returns valid patterns
+	rules, _ := extract.NewExcludeRules(opts.Excludes)
+	if verdict := gitFolderVerdict(rules); verdict != extract.Unmatched {
+		t.log.Debugf("Upload %s as %s says, instead of a shallow copy", gitFolder, ignoreFileName)
+		return opts, noCleanup
+	}
+
+	snapshot, err := os.MkdirTemp("", "devpod-git-")
+	if err != nil {
+		t.log.Warnf("Upload no %s, can't create a folder for a shallow copy: %v", gitFolder, err)
+		return opts, noCleanup
+	}
+	cleanup := func() { _ = os.RemoveAll(snapshot) }
+	result, err := git.Snapshot(ctx, folder, snapshot, excludedFiles(rules, files))
+	if err != nil {
+		cleanup()
+		t.log.Warnf("Upload no %s, can't create a shallow copy: %v. Add !%s to %s to upload the whole %s", gitFolder, err, gitFolder, ignoreFileName, gitFolder)
+		return opts, noCleanup
+	}
+	if result.Branch != "" {
+		t.log.Infof("Upload a shallow %s with branch %s", gitFolder, result.Branch)
+	} else {
+		t.log.Infof("Upload a shallow %s with the checked out commit", gitFolder)
+	}
+	opts.Overlays = map[string]string{}
+	for _, gitDir := range result.GitDirs {
+		opts.Overlays[gitDir] = filepath.Join(snapshot, filepath.FromSlash(gitDir))
+	}
+	return opts, cleanup
+}
+
+// excludedFiles returns the files the exclude rules leave out.
+func excludedFiles(rules *extract.ExcludeRules, files []string) []string {
+	var excluded []string
+	for _, file := range files {
+		if verdict, err := rules.Match(file); err == nil && verdict == extract.Excluded {
+			excluded = append(excluded, file)
+		}
+	}
+	return excluded
+}
+
+// gitFolderVerdict returns what the exclude rules say about the .git folder
+// at the root.
+func gitFolderVerdict(rules *extract.ExcludeRules) extract.Verdict {
+	verdict, err := rules.Match(gitFolder)
+	if err != nil {
+		return extract.Unmatched
+	}
+	return verdict
 }

pkg/agent/tunnelserver/tunnelserver_git_test.go

@@ -0,0 +1,168 @@
+package tunnelserver
+
+import (
+	"os"
+	"os/exec"
+	"path/filepath"
+	"slices"
+	"strings"
+	"testing"
+
+	"github.com/loft-sh/devpod/pkg/agent/tunnel"
+	"github.com/loft-sh/devpod/pkg/devcontainer/config"
+	"github.com/loft-sh/devpod/pkg/extract"
+	"github.com/loft-sh/devpod/pkg/log"
+	"gotest.tools/assert"
+	is "gotest.tools/assert/cmp"
+)
+
+func runGit(t *testing.T, dir string, args ...string) {
+	t.Helper()
+	cmd := exec.Command("git", args...)
+	cmd.Dir = dir
+	out, err := cmd.CombinedOutput()
+	assert.NilError(t, err, string(out))
+}
+
+// newGitSetupInfo returns a setup result whose workspace folder is a git
+// repository with ignored files, and an ignore file with the given content.
+func newGitSetupInfo(t *testing.T, ignore string) *config.Result {
+	t.Helper()
+	if _, err := exec.LookPath("git"); err != nil {
+		t.Skip("git is not installed")
+	}
+	emptyConfig := filepath.Join(t.TempDir(), "gitconfig")
+	assert.NilError(t, os.WriteFile(emptyConfig, nil, 0o600))
+	t.Setenv("GIT_CONFIG_GLOBAL", emptyConfig)
+	t.Setenv("GIT_CONFIG_NOSYSTEM", "1")
+
+	workspaceFolder := t.TempDir()
+	runGit(t, workspaceFolder, "init", "-q", "-b", "main")
+	writeFiles(t, workspaceFolder, srcMainGo, ".gitignore")
+	assert.NilError(t, os.WriteFile(
+		filepath.Join(workspaceFolder, ".gitignore"), []byte("node_modules/\n.env\nbuild/\n"), 0o600))
+	runGit(t, workspaceFolder, "add", "-A")
+	runGit(t, workspaceFolder, "-c", "user.name=test", "-c", "user.email=test@example.com", "commit", "-q", "-m", "init")
+	writeFiles(t, workspaceFolder,
+		"new.txt",
+		".env",
+		"build/out.bin",
+		"web/node_modules/lib/index.js",
+	)
+	if ignore != "" {
+		assert.NilError(t, os.WriteFile(
+			filepath.Join(workspaceFolder, ignoreFileName), []byte(ignore), 0o600))
+	}
+
+	return &config.Result{
+		SubstitutionContext: &config.SubstitutionContext{
+			WorkspaceMount: "type=bind,src=" + workspaceFolder + ",dst=/workspaces/project",
+		},
+		MergedConfig: &config.MergedDevContainerConfig{},
+	}
+}
+
+// split returns the entries in .git and the others.
+func split(entries []string) ([]string, []string) {
+	var gitEntries, others []string
+	for _, entry := range entries {
+		if strings.HasPrefix(entry, gitFolder+"/") {
+			gitEntries = append(gitEntries, entry)
+		} else {
+			others = append(others, entry)
+		}
+	}
+	return gitEntries, others
+}
+
+func TestStreamMount_GitWorkspaceUploadsWhatGitDoesNotIgnore(t *testing.T) {
+	setupInfo := newGitSetupInfo(t, "")
+	server := newSetupServer(setupInfo, log.Discard)
+
+	gitEntries, others := split(streamMountEntries(t, server, config.GetWorkspaceMount(setupInfo)))
+
+	assert.DeepEqual(t, []string{".gitignore", "new.txt", srcMainGo}, others)
+	// a shallow copy of .git
+	assert.Assert(t, is.Contains(gitEntries, ".git/HEAD"))
+	assert.Assert(t, is.Contains(gitEntries, ".git/index"))
+	assert.Assert(t, is.Contains(gitEntries, ".git/shallow"))
+}
+
+func TestStreamMount_IgnoreFileTakesPrecedenceOverGit(t *testing.T) {
+	setupInfo := newGitSetupInfo(t, "new.txt\n!.env\n!build/\nbuild/*.tmp\n")
+	writeFiles(t, config.GetWorkspaceMount(setupInfo).Source, "build/out.tmp")
+	server := newSetupServer(setupInfo, log.Discard)
+
+	_, others := split(streamMountEntries(t, server, config.GetWorkspaceMount(setupInfo)))
+
+	assert.DeepEqual(t, []string{ignoreFileName, ".env", ".gitignore", "build/out.bin", srcMainGo}, others)
+}
+
+func TestStreamMount_IgnoreFileExcludesGitFolder(t *testing.T) {
+	setupInfo := newGitSetupInfo(t, ".git\n")
+	server := newSetupServer(setupInfo, log.Discard)
+
+	gitEntries, _ := split(streamMountEntries(t, server, config.GetWorkspaceMount(setupInfo)))
+
+	assert.Equal(t, len(gitEntries), 0)
+}
+
+func TestStreamMount_IgnoreFileIncludesWholeGitFolder(t *testing.T) {
+	setupInfo := newGitSetupInfo(t, "!.git\n")
+	server := newSetupServer(setupInfo, log.Discard)
+
+	gitEntries, _ := split(streamMountEntries(t, server, config.GetWorkspaceMount(setupInfo)))
+
+	assert.Assert(t, is.Contains(gitEntries, ".git/HEAD"))
+	assert.Assert(t, !slices.Contains(gitEntries, ".git/shallow"))
+}
+
+func TestStreamMount_GitSubfolderHasNoGitFolder(t *testing.T) {
+	setupInfo := newGitSetupInfo(t, "")
+	workspaceMount := config.GetWorkspaceMount(setupInfo)
+	setupInfo.SubstitutionContext.WorkspaceMount = "type=bind,src=" + filepath.Join(workspaceMount.Source, "src") + ",dst=/workspaces/project"
+	server := newSetupServer(setupInfo, log.Discard)
+
+	entries := streamMountEntries(t, server, config.GetWorkspaceMount(setupInfo))
+
+	assert.DeepEqual(t, []string{"main.go"}, entries)
+}
+
+func gitOutput(t *testing.T, dir string, args ...string) string {
+	t.Helper()
+	cmd := exec.Command("git", args...)
+	cmd.Dir = dir
+	out, err := cmd.CombinedOutput()
+	assert.NilError(t, err, string(out))
+	return strings.TrimSpace(string(out))
+}
+
+func TestStreamMount_ExtractedGitWorkspaceIsClean(t *testing.T) {
+	setupInfo := newGitSetupInfo(t, "docs/**/*.mp4\n")
+	workspaceFolder := config.GetWorkspaceMount(setupInfo).Source
+	commit := []string{"-c", "user.name=test", "-c", "user.email=test@example.com", "commit", "-q", "-m"}
+
+	lib := t.TempDir()
+	runGit(t, lib, "init", "-q", "-b", "main")
+	writeFiles(t, lib, "lib.go")
+	runGit(t, lib, "add", "-A")
+	runGit(t, lib, append(commit, "lib")...)
+	runGit(t, workspaceFolder, "-c", "protocol.file.allow=always", "submodule", "add", "-q", lib, "lib")
+	writeFiles(t, workspaceFolder, "docs/videos/intro.mp4")
+	runGit(t, workspaceFolder, "add", "-A")
+	runGit(t, workspaceFolder, append(commit, "docs and lib")...)
+	server := newSetupServer(setupInfo, log.Discard)
+
+	stream := &mockStreamMountServer{}
+	assert.NilError(t, server.StreamMount(
+		&tunnel.StreamMountRequest{Mount: config.GetWorkspaceMount(setupInfo).String()}, stream))
+	target := t.TempDir()
+	assert.NilError(t, extract.Extract(&stream.content, target))
+
+	_, err := os.Stat(filepath.Join(target, "docs", "videos", "intro.mp4"))
+	assert.Assert(t, os.IsNotExist(err))
+	// the excluded video is not deleted and the submodule is unchanged
+	assert.Equal(t, "", gitOutput(t, target, "status", "--porcelain"))
+	assert.Equal(t, "", gitOutput(t, filepath.Join(target, "lib"), "status", "--porcelain"))
+	assert.Equal(t, gitOutput(t, workspaceFolder, "rev-parse", "HEAD"), gitOutput(t, target, "rev-parse", "HEAD"))
+}