From 031d728cc86198575f7ccd1a9639ae80c241f4bd Mon Sep 17 00:00:00 2001 From: renatoathaydes Date: Thu, 22 Oct 2026 14:21:03 +0200 Subject: [PATCH 1/3] Upload only files git doesn't ignore on devpod up of a local folder Streaming a local folder into a remote machine or a Kubernetes pod uploaded everything not excluded by .devpodignore, including .git, node_modules, build output and nested git worktrees, which made uploads huge (8.8 GB instead of 120 MB on a checkout of this repository). When the folder is in a git work tree, the tunnel server now only uploads the files `git ls-files --cached --others --exclude-standard` lists, including the files of checked out submodules but not nested repositories or worktrees. .devpodignore takes precedence: its patterns leave out listed files and its negation patterns upload files git ignores. When the folder is the top of the work tree, the repository's .git is replaced by a shallow one with the checked out commit, the branch, its upstream branch (deep enough for ahead/behind counts), the remotes and the repository's user config, so that the workspace can commit and push. `!.git` in .devpodignore uploads the whole .git and `.git` uploads none. Without git, or outside a git work tree, nothing changes. Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 10 + .../create-a-workspace.mdx | 34 +++ pkg/agent/tunnelserver/tunnelserver.go | 99 ++++++- .../tunnelserver/tunnelserver_git_test.go | 127 +++++++++ pkg/extract/compress.go | 141 ++++------ pkg/extract/compress_test.go | 199 +++++++++++++ pkg/extract/rules.go | 197 +++++++++++++ pkg/git/worktree.go | 264 ++++++++++++++++++ pkg/git/worktree_test.go | 253 +++++++++++++++++ 9 files changed, 1221 insertions(+), 103 deletions(-) create mode 100644 pkg/agent/tunnelserver/tunnelserver_git_test.go create mode 100644 pkg/extract/rules.go create mode 100644 pkg/git/worktree.go create mode 100644 pkg/git/worktree_test.go diff --git a/CHANGELOG.md b/CHANGELOG.md index 41b6b707d..258881314 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,16 @@ Add new entries to the top of the "Unreleased" section. ## Unreleased +### Changed + +- `devpod up` of a local folder in a git repository only uploads the files git doesn't ignore, instead of the + whole folder, when the workspace runs on another machine or in Kubernetes. Ignored files (build output, + `node_modules`, ...) and nested repositories and git worktrees stay behind. Instead of the repository's + `.git`, the workspace gets a shallow one with the checked out commit, the branch, its upstream branch and the + remotes, which is enough to commit and push. `.devpodignore` takes precedence: its patterns leave out more + files, and its negation (`!`) patterns upload files git ignores (`!.git` uploads the whole `.git`). See + [Which files are uploaded](https://devpod.curityio.net/developing-in-workspaces/create-a-workspace#which-files-are-uploaded). + ### Fixed - `devpod up` of a Kubernetes workspace recreates the workspace's pod when it is dead (its phase is `Failed`, diff --git a/docs/pages/developing-in-workspaces/create-a-workspace.mdx b/docs/pages/developing-in-workspaces/create-a-workspace.mdx index 001852150..cc36f0cc7 100644 --- a/docs/pages/developing-in-workspaces/create-a-workspace.mdx +++ b/docs/pages/developing-in-workspaces/create-a-workspace.mdx @@ -87,6 +87,40 @@ devpod up ./path/to/my-folder DevPod will sync the folder into the remote machine and create a development environment from the `devcontainer.json`. +##### Which files are uploaded + +When the workspace runs on another machine or in Kubernetes, DevPod uploads the folder. If the folder is in a git +repository, DevPod only uploads the files git doesn't ignore: tracked files, and untracked files that no +`.gitignore` file, `.git/info/exclude` or `core.excludesFile` ignores. So build output, `node_modules` and the like +stay behind, as do other repositories and git worktrees inside the folder. The files of submodules are uploaded. + +If the folder is the top of the repository, the workspace gets a small `.git` in place of yours, with only the +checked out commit, so that you can commit and push from the workspace. It has the same branch, remotes and +upstream branch as yours, and the commits since your branch and its upstream branch diverged, but no other +history. Changes you have staged show as not staged in the workspace. + +A `.devpodignore` file at the top of the folder takes precedence over git. It has the syntax of a +[`.dockerignore` file](https://docs.docker.com/build/concepts/context/#dockerignore-files), so patterns are +relative to the folder: `build` only matches the `build` folder at the top, `**/build` matches one anywhere. A +pattern leaves out files git doesn't ignore, and a negation (`!`) pattern uploads files git ignores: + +``` +# a file git ignores, but the workspace needs +!.env +# files git doesn't ignore, but the workspace doesn't need +docs/**/*.mp4 +``` + +`!.git` uploads your whole `.git` instead of the small one, and `.git` uploads no `.git` at all. + +If the folder is not in a git repository, or git is not installed, DevPod uploads every file `.devpodignore` +doesn't exclude. + +:::info +When the workspace runs on your machine, e.g. with the `docker` provider and a local Docker, the folder is mounted +and not uploaded, so the workspace has all of it. +::: + #### Docker Image Run the following command in a terminal to create a new workspace from a docker image: diff --git a/pkg/agent/tunnelserver/tunnelserver.go b/pkg/agent/tunnelserver/tunnelserver.go index 3e7939397..8d2f21854 100644 --- a/pkg/agent/tunnelserver/tunnelserver.go +++ b/pkg/agent/tunnelserver/tunnelserver.go @@ -5,6 +5,7 @@ import ( "context" "encoding/base64" "encoding/json" + "errors" "fmt" "io" "os" @@ -15,6 +16,7 @@ import ( "github.com/loft-sh/devpod/pkg/devcontainer/config" "github.com/loft-sh/devpod/pkg/dockercredentials" "github.com/loft-sh/devpod/pkg/extract" + "github.com/loft-sh/devpod/pkg/git" "github.com/loft-sh/devpod/pkg/gitcredentials" "github.com/loft-sh/devpod/pkg/gitsshsigning" "github.com/loft-sh/devpod/pkg/gpg" @@ -309,10 +311,11 @@ func (t *tunnelServer) StreamWorkspace(message *tunnel.Empty, stream tunnel.Tunn return fmt.Errorf("workspace is nil") } - excludes := t.readIgnoreFile(t.workspace.Source.LocalFolder) + opts, cleanup := t.workspaceTarOptions(stream.Context(), t.workspace.Source.LocalFolder) + defer cleanup() buf := bufio.NewWriterSize(NewStreamWriter(stream, t.log), 10*1024) - err := extract.WriteTarWithOptions(buf, t.workspace.Source.LocalFolder, streamTarOptions(excludes)) + err := extract.WriteTarWithOptions(buf, t.workspace.Source.LocalFolder, opts) if err != nil { return err } @@ -333,17 +336,19 @@ func (t *tunnelServer) StreamMount(message *tunnel.StreamMountRequest, stream tu return fmt.Errorf("mount %s is not allowed to download", message.Mount) } - // The ignore file only applies to the workspace folder, other bind mounts + // The ignore files only apply to the workspace folder, other bind mounts // are streamed whole - var excludes []string + opts := extract.TarOptions{} if t.workspaceMount != nil && mount.String() == t.workspaceMount.String() { - excludes = t.readIgnoreFile(mount.Source) + var cleanup func() + opts, cleanup = t.workspaceTarOptions(stream.Context(), mount.Source) + defer cleanup() } else { t.log.Debugf("Stream mount %s without excludes, it is not the workspace mount", mount.String()) } buf := bufio.NewWriterSize(NewStreamWriter(stream, t.log), 10*1024) - err := extract.WriteTarWithOptions(buf, mount.Source, streamTarOptions(excludes)) + err := extract.WriteTarWithOptions(buf, mount.Source, opts) if err != nil { return err } @@ -383,12 +388,82 @@ func (t *tunnelServer) readIgnoreFile(folder string) []string { return excludes } -// streamTarOptions returns the options to stream a folder into the container -// with excludes. The dockerless build context is always streamed whole, as the -// build fails without it. -func streamTarOptions(excludes []string) extract.TarOptions { - return extract.TarOptions{ - Excludes: excludes, +// gitFolder is the folder of a git repository in its work tree. +const gitFolder = ".git" + +// workspaceTarOptions returns the options to stream the workspace folder into +// the container, and a function that removes what they need once streamed. +// +// If the folder is in a git work tree, only the files git doesn't ignore are +// streamed, and the ignore file takes precedence: it can leave out files git +// doesn't ignore, and re-include files git ignores with negation (!) patterns. +// If the folder is the top level of the work tree, its .git is replaced by a +// shallow copy unless the ignore file has a pattern matching .git. The +// dockerless build context is always streamed whole, as the build fails +// without it. +func (t *tunnelServer) workspaceTarOptions(ctx context.Context, folder string) (extract.TarOptions, func()) { + opts := extract.TarOptions{ + Excludes: t.readIgnoreFile(folder), KeepNames: []string{config.DevPodContextFeatureFolder}, } + noCleanup := func() {} + + files, err := git.ListFiles(ctx, folder) + if err != nil { + if errors.Is(err, git.ErrNotInstalled) { + t.log.Warnf("Git is not installed, so all files of %s not excluded by %s are uploaded, including those .gitignore files ignore", folder, ignoreFileName) + } else { + t.log.Debugf("Upload all files of %s not excluded by %s, git can't list its files: %v", folder, ignoreFileName, err) + } + return opts, noCleanup + } + // e.g. the folder is in a repository of dotfiles that ignores everything + if len(files) == 0 { + t.log.Debugf("Upload all files of %s not excluded by %s, git doesn't list any", folder, ignoreFileName) + return opts, noCleanup + } + opts.Files = extract.NewFileList(files) + t.log.Debugf("Upload the %d files of %s git doesn't ignore, and those %s re-includes", opts.Files.Len(), folder, ignoreFileName) + + if !git.IsTopLevel(ctx, folder) { + return opts, noCleanup + } + if verdict := gitFolderVerdict(opts.Excludes); verdict != extract.Unmatched { + t.log.Debugf("Upload %s as %s says, instead of a shallow copy", gitFolder, ignoreFileName) + return opts, noCleanup + } + + snapshot, err := os.MkdirTemp("", "devpod-git-") + if err != nil { + t.log.Warnf("Upload no %s, can't create a folder for a shallow copy: %v", gitFolder, err) + return opts, noCleanup + } + cleanup := func() { _ = os.RemoveAll(snapshot) } + branch, err := git.Snapshot(ctx, folder, snapshot) + if err != nil { + cleanup() + t.log.Warnf("Upload no %s, can't create a shallow copy: %v. Add !%s to %s to upload the whole %s", gitFolder, err, gitFolder, ignoreFileName, gitFolder) + return opts, noCleanup + } + if branch != "" { + t.log.Infof("Upload a shallow %s with branch %s", gitFolder, branch) + } else { + t.log.Infof("Upload a shallow %s with the checked out commit", gitFolder) + } + opts.Overlays = []string{filepath.Join(snapshot, gitFolder)} + return opts, cleanup +} + +// gitFolderVerdict returns what the exclude patterns say about the .git +// folder at the root. +func gitFolderVerdict(excludes []string) extract.Verdict { + rules, err := extract.NewExcludeRules(excludes) + if err != nil { + return extract.Unmatched + } + verdict, err := rules.Match(gitFolder) + if err != nil { + return extract.Unmatched + } + return verdict } diff --git a/pkg/agent/tunnelserver/tunnelserver_git_test.go b/pkg/agent/tunnelserver/tunnelserver_git_test.go new file mode 100644 index 000000000..1ed0d65cf --- /dev/null +++ b/pkg/agent/tunnelserver/tunnelserver_git_test.go @@ -0,0 +1,127 @@ +package tunnelserver + +import ( + "os" + "os/exec" + "path/filepath" + "slices" + "strings" + "testing" + + "github.com/loft-sh/devpod/pkg/devcontainer/config" + "github.com/loft-sh/devpod/pkg/log" + "gotest.tools/assert" + is "gotest.tools/assert/cmp" +) + +func runGit(t *testing.T, dir string, args ...string) { + t.Helper() + cmd := exec.Command("git", args...) + cmd.Dir = dir + out, err := cmd.CombinedOutput() + assert.NilError(t, err, string(out)) +} + +// newGitSetupInfo returns a setup result whose workspace folder is a git +// repository with ignored files, and an ignore file with the given content. +func newGitSetupInfo(t *testing.T, ignore string) *config.Result { + t.Helper() + if _, err := exec.LookPath("git"); err != nil { + t.Skip("git is not installed") + } + emptyConfig := filepath.Join(t.TempDir(), "gitconfig") + assert.NilError(t, os.WriteFile(emptyConfig, nil, 0o600)) + t.Setenv("GIT_CONFIG_GLOBAL", emptyConfig) + t.Setenv("GIT_CONFIG_NOSYSTEM", "1") + + workspaceFolder := t.TempDir() + runGit(t, workspaceFolder, "init", "-q", "-b", "main") + writeFiles(t, workspaceFolder, srcMainGo, ".gitignore") + assert.NilError(t, os.WriteFile( + filepath.Join(workspaceFolder, ".gitignore"), []byte("node_modules/\n.env\nbuild/\n"), 0o600)) + runGit(t, workspaceFolder, "add", "-A") + runGit(t, workspaceFolder, "-c", "user.name=test", "-c", "user.email=test@example.com", "commit", "-q", "-m", "init") + writeFiles(t, workspaceFolder, + "new.txt", + ".env", + "build/out.bin", + "web/node_modules/lib/index.js", + ) + if ignore != "" { + assert.NilError(t, os.WriteFile( + filepath.Join(workspaceFolder, ignoreFileName), []byte(ignore), 0o600)) + } + + return &config.Result{ + SubstitutionContext: &config.SubstitutionContext{ + WorkspaceMount: "type=bind,src=" + workspaceFolder + ",dst=/workspaces/project", + }, + MergedConfig: &config.MergedDevContainerConfig{}, + } +} + +// split returns the entries in .git and the others. +func split(entries []string) ([]string, []string) { + var gitEntries, others []string + for _, entry := range entries { + if strings.HasPrefix(entry, gitFolder+"/") { + gitEntries = append(gitEntries, entry) + } else { + others = append(others, entry) + } + } + return gitEntries, others +} + +func TestStreamMount_GitWorkspaceUploadsWhatGitDoesNotIgnore(t *testing.T) { + setupInfo := newGitSetupInfo(t, "") + server := newSetupServer(setupInfo, log.Discard) + + gitEntries, others := split(streamMountEntries(t, server, config.GetWorkspaceMount(setupInfo))) + + assert.DeepEqual(t, []string{".gitignore", "new.txt", srcMainGo}, others) + // a shallow copy of .git + assert.Assert(t, is.Contains(gitEntries, ".git/HEAD")) + assert.Assert(t, is.Contains(gitEntries, ".git/index")) + assert.Assert(t, is.Contains(gitEntries, ".git/shallow")) +} + +func TestStreamMount_IgnoreFileTakesPrecedenceOverGit(t *testing.T) { + setupInfo := newGitSetupInfo(t, "new.txt\n!.env\n!build/\nbuild/*.tmp\n") + writeFiles(t, config.GetWorkspaceMount(setupInfo).Source, "build/out.tmp") + server := newSetupServer(setupInfo, log.Discard) + + _, others := split(streamMountEntries(t, server, config.GetWorkspaceMount(setupInfo))) + + assert.DeepEqual(t, []string{ignoreFileName, ".env", ".gitignore", "build/out.bin", srcMainGo}, others) +} + +func TestStreamMount_IgnoreFileExcludesGitFolder(t *testing.T) { + setupInfo := newGitSetupInfo(t, ".git\n") + server := newSetupServer(setupInfo, log.Discard) + + gitEntries, _ := split(streamMountEntries(t, server, config.GetWorkspaceMount(setupInfo))) + + assert.Equal(t, len(gitEntries), 0) +} + +func TestStreamMount_IgnoreFileIncludesWholeGitFolder(t *testing.T) { + setupInfo := newGitSetupInfo(t, "!.git\n") + server := newSetupServer(setupInfo, log.Discard) + + gitEntries, _ := split(streamMountEntries(t, server, config.GetWorkspaceMount(setupInfo))) + + assert.Assert(t, is.Contains(gitEntries, ".git/HEAD")) + assert.Assert(t, !slices.Contains(gitEntries, ".git/shallow")) +} + +func TestStreamMount_GitSubfolderHasNoGitFolder(t *testing.T) { + setupInfo := newGitSetupInfo(t, "") + workspaceMount := config.GetWorkspaceMount(setupInfo) + setupInfo.SubstitutionContext.WorkspaceMount = "type=bind,src=" + filepath.Join(workspaceMount.Source, "src") + ",dst=/workspaces/project" + server := newSetupServer(setupInfo, log.Discard) + + entries := streamMountEntries(t, server, config.GetWorkspaceMount(setupInfo)) + + assert.DeepEqual(t, []string{"main.go"}, entries) +} diff --git a/pkg/extract/compress.go b/pkg/extract/compress.go index 62a20b786..65188cece 100644 --- a/pkg/extract/compress.go +++ b/pkg/extract/compress.go @@ -9,7 +9,6 @@ import ( "path/filepath" "runtime" "slices" - "strings" "time" "github.com/moby/patternmatcher" @@ -21,11 +20,19 @@ type TarOptions struct { // Compress gzips the archive. Compress bool // Excludes are patterns relative to the archived folder, with the same - // syntax as a .dockerignore file. + // syntax as a .dockerignore file. They take precedence over Files: an + // exclude pattern leaves out a listed file, and a negation (!) pattern + // adds a file that isn't listed. Excludes []string + // Files, when not nil, lists the files to archive. A file no exclude + // pattern matches is only archived if it is listed. + Files *FileList // KeepNames are names of folders that are archived whole wherever the walk - // reaches them, whatever the excludes say. + // reaches them, whatever the excludes and files say. KeepNames []string + // Overlays are folders archived whole at the root of the archive under + // their own name, in place of the archived folder's entry of that name. + Overlays []string } func WriteTarExclude(writer io.Writer, localPath string, compress bool, excludedPaths []string) error { @@ -71,8 +78,22 @@ func WriteTarWithOptions(writer io.Writer, localPath string, opts TarOptions) er if err != nil { return err } + archiver.files = opts.Files archiver.keepNames = opts.KeepNames - return archiver.AddToArchive(relativePath) + for _, overlay := range opts.Overlays { + archiver.overlaid = append(archiver.overlaid, filepath.Base(overlay)) + } + if err := archiver.AddToArchive(relativePath); err != nil { + return err + } + + for _, overlay := range opts.Overlays { + whole := &Archiver{basePath: filepath.Dir(overlay), writer: tarWriter, writtenFiles: map[string]bool{}} + if err := whole.AddToArchive(filepath.Base(overlay)); err != nil { + return errors.Wrapf(err, "archive %s", overlay) + } + } + return nil } func WriteTar(writer io.Writer, localPath string, compress bool) error { @@ -87,23 +108,22 @@ type Archiver struct { // excludes holds the exclude patterns, with .dockerignore semantics. It is // nil when nothing is excluded. - excludes *patternmatcher.PatternMatcher - // reincludes holds the components of each negation (!) pattern. - reincludes [][]string + excludes *ExcludeRules + // files lists the files archived when no exclude pattern matches them. It + // is nil when they are all archived. + files *FileList // keepNames are names of folders archived whole, whatever the excludes say. keepNames []string + // overlaid are the names at the root that overlays replace. + overlaid []string } // NewArchiver creates a new archiver. excludedPaths are patterns relative to // basePath, with the same syntax as a .dockerignore file. func NewArchiver(basePath string, writer *tar.Writer, excludedPaths []string) (*Archiver, error) { - var excludes *patternmatcher.PatternMatcher - if len(excludedPaths) > 0 { - var err error - excludes, err = patternmatcher.New(excludedPaths) - if err != nil { - return nil, errors.Wrap(err, "parse exclude patterns") - } + excludes, err := NewExcludeRules(excludedPaths) + if err != nil { + return nil, err } return &Archiver{ @@ -111,25 +131,9 @@ func NewArchiver(basePath string, writer *tar.Writer, excludedPaths []string) (* writer: writer, writtenFiles: map[string]bool{}, excludes: excludes, - reincludes: reincludePatterns(excludes), }, nil } -// reincludePatterns returns the components of each negation pattern. -func reincludePatterns(excludes *patternmatcher.PatternMatcher) [][]string { - if excludes == nil || !excludes.Exclusions() { - return nil - } - - var reincludes [][]string - for _, p := range excludes.Patterns() { - if p.Exclusion() { - reincludes = append(reincludes, strings.Split(filepath.ToSlash(p.String()), "/")) - } - } - return reincludes -} - // AddToArchive adds a new path to the archive func (a *Archiver) AddToArchive(relativePath string) error { return a.addToArchive(relativePath, patternmatcher.MatchInfo{}) @@ -138,7 +142,7 @@ func (a *Archiver) AddToArchive(relativePath string) error { // addToArchive adds a path to the archive. parentInfo holds the exclude // results of the parent directory, or the zero value when they are unknown. func (a *Archiver) addToArchive(relativePath string, parentInfo patternmatcher.MatchInfo) error { - if a.writtenFiles[relativePath] { + if a.writtenFiles[relativePath] || slices.Contains(a.overlaid, relativePath) { return nil } @@ -153,11 +157,11 @@ func (a *Archiver) addToArchive(relativePath string, parentInfo patternmatcher.M return a.addFolder(relativePath, stat, parentInfo) } - excluded, _, err := a.isExcluded(relativePath, parentInfo) + verdict, _, err := a.excludes.match(relativePath, parentInfo, a.files != nil) if err != nil { return err } - if excluded { + if verdict == Excluded || (verdict == Unmatched && !a.files.hasFile(relativePath)) { return nil } return a.tarFile(relativePath, stat) @@ -169,87 +173,42 @@ func (a *Archiver) addFolder(relativePath string, stat os.FileInfo, parentInfo p return a.tarFolderWhole(relativePath, stat) } - excluded, info, err := a.isExcluded(relativePath, parentInfo) + verdict, info, err := a.excludes.match(relativePath, parentInfo, a.files != nil) if err != nil { return err } + archived := verdict == Included || (verdict == Unmatched && a.files.hasFolder(relativePath)) - // Skip an excluded folder without reading it, unless a negation + // Skip a folder that isn't archived without reading it, unless a negation // pattern could re-include something below it. - if excluded && !a.mayReincludeBelow(relativePath) { + if !archived && !a.mayReincludeBelow(relativePath) { return nil } // Recursively tar folder - return a.tarFolder(relativePath, stat, excluded, info) + return a.tarFolder(relativePath, stat, archived, info) } // isKept reports whether the folder relativePath is archived whole. func (a *Archiver) isKept(relativePath string) bool { - return a.excludes != nil && slices.Contains(a.keepNames, path.Base(relativePath)) + return (a.excludes != nil || a.files != nil) && slices.Contains(a.keepNames, path.Base(relativePath)) } // tarFolderWhole archives a folder and everything in it, ignoring the excludes. func (a *Archiver) tarFolderWhole(relativePath string, stat os.FileInfo) error { whole := &Archiver{basePath: a.basePath, writer: a.writer, writtenFiles: a.writtenFiles} - return whole.tarFolder(relativePath, stat, false, patternmatcher.MatchInfo{}) + return whole.tarFolder(relativePath, stat, true, patternmatcher.MatchInfo{}) } // mayReincludeBelow reports whether a negation pattern could match a path -// inside the excluded folder dir. It may return true for a pattern that turns -// out not to match, but never false for one that does. +// inside the folder dir. func (a *Archiver) mayReincludeBelow(dir string) bool { - dirComponents := strings.Split(path.Clean(filepath.ToSlash(dir)), "/") - for _, pattern := range a.reincludes { - if patternMayMatchBelow(pattern, dirComponents) { - return true - } - } - return false -} - -// patternMayMatchBelow reports whether pattern could match a path below the -// folder dir, both given as path components. -func patternMayMatchBelow(pattern, dir []string) bool { - // "**" matches any number of folders, so do not try to rule it out - if slices.Contains(pattern, "**") { - return true - } - // A pattern matching dir itself, or one of its parents, was already - // applied to dir, which is still excluded - if len(pattern) <= len(dir) { - return false - } - for i, component := range dir { - matched, err := path.Match(pattern[i], component) - if err != nil || !matched { - return err != nil - } - } - return true -} - -// isExcluded matches relativePath against the exclude patterns. It also -// returns the match results to pass down to the path's children. -func (a *Archiver) isExcluded(relativePath string, parentInfo patternmatcher.MatchInfo) (bool, patternmatcher.MatchInfo, error) { - if a.excludes == nil { - return false, patternmatcher.MatchInfo{}, nil - } - - // Patterns are relative to the archive root, so the root itself never matches - relativePath = path.Clean(filepath.ToSlash(relativePath)) - if relativePath == "." || relativePath == "/" { - return false, patternmatcher.MatchInfo{}, nil - } - - excluded, info, err := a.excludes.MatchesUsingParentResults(relativePath, parentInfo) - if err != nil { - return false, info, errors.Wrapf(err, "match %s against exclude patterns", relativePath) - } - return excluded, info, nil + return a.excludes.mayReincludeBelow(dir) } -func (a *Archiver) tarFolder(target string, targetStat os.FileInfo, excluded bool, matchInfo patternmatcher.MatchInfo) error { +// tarFolder archives what is in a folder. archived tells whether the folder +// itself is archived, which matters when it is empty. +func (a *Archiver) tarFolder(target string, targetStat os.FileInfo, archived bool, matchInfo patternmatcher.MatchInfo) error { filePath := path.Join(a.basePath, target) files, err := os.ReadDir(filePath) if err != nil { @@ -257,7 +216,7 @@ func (a *Archiver) tarFolder(target string, targetStat os.FileInfo, excluded boo return nil } - if len(files) == 0 && target != "" && !excluded { + if len(files) == 0 && target != "" && archived { // Case empty directory hdr, _ := tar.FileInfoHeader(targetStat, filePath) hdr.Uid = 0 diff --git a/pkg/extract/compress_test.go b/pkg/extract/compress_test.go index 5cc1f2bc8..f69e7b909 100644 --- a/pkg/extract/compress_test.go +++ b/pkg/extract/compress_test.go @@ -268,3 +268,202 @@ func TestWriteTarExcludeInvalidPattern(t *testing.T) { err := WriteTarExclude(&bytes.Buffer{}, root, false, []string{"[a-"}) assert.ErrorContains(t, err, "exclude patterns") } + +// tarEntriesWithOptions archives root with opts and returns the names of the +// regular files and empty directories in the archive, sorted. +func tarEntriesWithOptions(t *testing.T, root string, opts TarOptions) []string { + t.Helper() + + buf := &bytes.Buffer{} + assert.NilError(t, WriteTarWithOptions(buf, root, opts)) + + entries := []string{} + reader := tar.NewReader(buf) + for { + hdr, err := reader.Next() + if errors.Is(err, io.EOF) { + break + } + assert.NilError(t, err) + entries = append(entries, hdr.Name) + } + sort.Strings(entries) + + return entries +} + +// gitFiles are the files of testTree git would list with a .gitignore of +// node_modules, build and .claude/worktrees. +var gitFiles = []string{ + readmeFile, + ".claude/settings.json", + "web/app/main.js", + "buildSrc/plugin.kt", + "docs/guide.tgz", + "docs/guide.md", + "docs/old/archive.tgz", +} + +func sorted(entries ...string) []string { + entries = slices.Clone(entries) + sort.Strings(entries) + return entries +} + +func TestWriteTarWithFiles(t *testing.T) { + tests := []struct { + name string + excludes []string + // files defaults to gitFiles + files []string + expected []string + }{ + { + name: "only listed files are archived", + expected: sorted(gitFiles...), + }, + { + name: "exclude patterns leave out listed files", + excludes: []string{"docs/*.tgz", "web"}, + expected: without(sorted(gitFiles...), "docs/guide.tgz", "web/app/main.js"), + }, + { + name: "negation patterns add files that are not listed", + excludes: []string{"!build/out.bin", "!**/node_modules/lib"}, + expected: sorted(append(slices.Clone(gitFiles), + "build/out.bin", + "node_modules/lib/index.js", + "web/node_modules/lib/index.js", + "web/app/node_modules/lib/index.js", + )...), + }, + { + name: "negation pattern adds a whole folder that is not listed", + excludes: []string{"!.claude/worktrees"}, + expected: sorted(append(slices.Clone(gitFiles), + ".claude/worktrees/feature/src/main.go", + ".claude/worktrees/feature/keep.txt", + )...), + }, + { + name: "later exclude pattern wins over negation pattern", + excludes: []string{"!.claude/worktrees", ".claude/worktrees/feature/src"}, + expected: sorted(append(slices.Clone(gitFiles), ".claude/worktrees/feature/keep.txt")...), + }, + { + // git lists a submodule that isn't checked out as a file + name: "listed empty folder is archived", + files: append(slices.Clone(gitFiles), emptyFolder), + expected: sorted(append(slices.Clone(gitFiles), emptyFolder)...), + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + root := newTestTree(t) + files := tt.files + if files == nil { + files = gitFiles + } + assert.DeepEqual(t, tt.expected, tarEntriesWithOptions(t, root, TarOptions{ + Excludes: tt.excludes, + Files: NewFileList(files), + })) + }) + } +} + +func TestWriteTarWithFilesKeepsFoldersWhole(t *testing.T) { + root := newTestTree(t) + + entries := tarEntriesWithOptions(t, root, TarOptions{ + Files: NewFileList(gitFiles), + KeepNames: []string{"worktrees"}, + }) + + assert.Assert(t, is.Contains(entries, ".claude/worktrees/feature/src/main.go")) + assert.Assert(t, is.Contains(entries, ".claude/worktrees/feature/keep.txt")) +} + +func TestWriteTarWithOverlays(t *testing.T) { + root := newTestTree(t) + overlayParent := t.TempDir() + writeTestFiles(t, overlayParent, "docs/new.md", "docs/sub/other.md") + writeTestFiles(t, root, "web/.git/HEAD") + + entries := tarEntriesWithOptions(t, root, TarOptions{ + Excludes: []string{"!web/.git"}, + Files: NewFileList(gitFiles), + Overlays: []string{filepath.Join(overlayParent, "docs")}, + }) + + // The overlay replaces the folder at the root only + assert.DeepEqual(t, sorted( + readmeFile, + ".claude/settings.json", + "web/app/main.js", + "web/.git/HEAD", + "buildSrc/plugin.kt", + "docs/new.md", + "docs/sub/other.md", + ), entries) +} + +func writeTestFiles(t *testing.T, root string, files ...string) { + t.Helper() + + for _, file := range files { + p := filepath.Join(root, filepath.FromSlash(file)) + assert.NilError(t, os.MkdirAll(filepath.Dir(p), 0o750)) + assert.NilError(t, os.WriteFile(p, []byte(file), 0o600)) + } +} + +func TestExcludeRulesMatch(t *testing.T) { + rules, err := NewExcludeRules([]string{"build", "!build/keep", "!.env", "docs/*.tgz"}) + assert.NilError(t, err) + + tests := map[string]Verdict{ + ".": Unmatched, + "src/main.go": Unmatched, + "build": Excluded, + "build/out.bin": Excluded, + "build/keep": Included, + "build/keep/a": Included, + ".env": Included, + "docs/guide.tgz": Excluded, + "docs/guide.md": Unmatched, + } + for p, expected := range tests { + verdict, err := rules.Match(p) + assert.NilError(t, err) + assert.Equal(t, expected, verdict, p) + } + + var none *ExcludeRules + verdict, err := none.Match("build") + assert.NilError(t, err) + assert.Equal(t, Unmatched, verdict) + + rules, err = NewExcludeRules(nil) + assert.NilError(t, err) + assert.Assert(t, rules == nil) +} + +func TestNewFileList(t *testing.T) { + list := NewFileList([]string{"a/b/c.txt", "a/d.txt", "e.txt", "./f/../g.txt"}) + + assert.Equal(t, 4, list.Len()) + assert.Assert(t, list.hasFolder("")) + assert.Assert(t, list.hasFile("a/b/c.txt")) + assert.Assert(t, list.hasFile("g.txt")) + assert.Assert(t, !list.hasFile("a/b")) + assert.Assert(t, list.hasFolder("a")) + assert.Assert(t, list.hasFolder("a/b")) + assert.Assert(t, !list.hasFolder("f")) + assert.Assert(t, !list.hasFolder("a/b/c")) + + var all *FileList + assert.Assert(t, all.hasFile("anything")) + assert.Assert(t, all.hasFolder("anything")) +} diff --git a/pkg/extract/rules.go b/pkg/extract/rules.go new file mode 100644 index 000000000..7facd8947 --- /dev/null +++ b/pkg/extract/rules.go @@ -0,0 +1,197 @@ +package extract + +import ( + "path" + "path/filepath" + "slices" + "strings" + + "github.com/moby/patternmatcher" + "github.com/pkg/errors" +) + +// Verdict is what exclude patterns say about a path. +type Verdict int + +const ( + // Unmatched means no pattern matches the path or one of its parent folders. + Unmatched Verdict = iota + // Excluded means the last pattern that matches the path or one of its + // parent folders is an exclude pattern. + Excluded + // Included means the last pattern that matches the path or one of its + // parent folders is a negation (!) pattern. + Included +) + +// ExcludeRules matches paths against exclude patterns with the syntax of a +// .dockerignore file. +type ExcludeRules struct { + excludes *patternmatcher.PatternMatcher + // all holds the patterns after one excluding everything, which tells a + // path no pattern matches from one a negation pattern includes. + all *patternmatcher.PatternMatcher + // reincludes holds the components of each negation (!) pattern. + reincludes [][]string +} + +// NewExcludeRules parses patterns relative to an archived folder. It returns +// nil if there are no patterns. +func NewExcludeRules(patterns []string) (*ExcludeRules, error) { + if len(patterns) == 0 { + return nil, nil + } + + excludes, err := patternmatcher.New(patterns) + if err != nil { + return nil, errors.Wrap(err, "parse exclude patterns") + } + all, err := patternmatcher.New(append([]string{"**"}, patterns...)) + if err != nil { + return nil, errors.Wrap(err, "parse exclude patterns") + } + + return &ExcludeRules{ + excludes: excludes, + all: all, + reincludes: reincludePatterns(excludes), + }, nil +} + +// reincludePatterns returns the components of each negation pattern. +func reincludePatterns(excludes *patternmatcher.PatternMatcher) [][]string { + if !excludes.Exclusions() { + return nil + } + + var reincludes [][]string + for _, p := range excludes.Patterns() { + if p.Exclusion() { + reincludes = append(reincludes, strings.Split(filepath.ToSlash(p.String()), "/")) + } + } + return reincludes +} + +// Match returns the verdict of the patterns for relativePath, a path relative +// to the archived folder. +func (r *ExcludeRules) Match(relativePath string) (Verdict, error) { + verdict, _, err := r.match(relativePath, patternmatcher.MatchInfo{}, true) + return verdict, err +} + +// match returns the verdict of the patterns for relativePath, and the match +// results to pass down to the path's children. parentInfo holds the results of +// the parent folder, or the zero value when they are unknown. If unmatched is +// false, it returns Unmatched instead of Included, which is cheaper. +func (r *ExcludeRules) match(relativePath string, parentInfo patternmatcher.MatchInfo, unmatched bool) (Verdict, patternmatcher.MatchInfo, error) { + if r == nil { + return Unmatched, patternmatcher.MatchInfo{}, nil + } + + // Patterns are relative to the archive root, so the root itself never matches + relativePath = path.Clean(filepath.ToSlash(relativePath)) + if relativePath == "." || relativePath == "/" { + return Unmatched, patternmatcher.MatchInfo{}, nil + } + + excluded, info, err := r.excludes.MatchesUsingParentResults(relativePath, parentInfo) + if err != nil { + return Unmatched, info, errors.Wrapf(err, "match %s against exclude patterns", relativePath) + } + if excluded { + return Excluded, info, nil + } + if !unmatched || !r.excludes.Exclusions() { + return Unmatched, info, nil + } + + // MatchesOrParentMatches is exact but slower than using the parent's + // results. The archiver only asks for it with a file list, which keeps it + // out of folders the list leaves out. + all, err := r.all.MatchesOrParentMatches(relativePath) + if err != nil { + return Unmatched, info, errors.Wrapf(err, "match %s against exclude patterns", relativePath) + } + if !all { + return Included, info, nil + } + return Unmatched, info, nil +} + +// mayReincludeBelow reports whether a negation pattern could match a path +// inside the folder dir. It may return true for a pattern that turns out not +// to match, but never false for one that does. +func (r *ExcludeRules) mayReincludeBelow(dir string) bool { + if r == nil { + return false + } + + dirComponents := strings.Split(path.Clean(filepath.ToSlash(dir)), "/") + for _, pattern := range r.reincludes { + if patternMayMatchBelow(pattern, dirComponents) { + return true + } + } + return false +} + +// patternMayMatchBelow reports whether pattern could match a path below the +// folder dir, both given as path components. +func patternMayMatchBelow(pattern, dir []string) bool { + // "**" matches any number of folders, so do not try to rule it out + if slices.Contains(pattern, "**") { + return true + } + // A pattern matching dir itself, or one of its parents, was already + // applied to dir, which is still excluded + if len(pattern) <= len(dir) { + return false + } + for i, component := range dir { + matched, err := path.Match(pattern[i], component) + if err != nil || !matched { + return err != nil + } + } + return true +} + +// FileList is a set of files to archive, relative to the archived folder. +type FileList struct { + files map[string]bool + // folders holds the folders that have a listed file below them. + folders map[string]bool +} + +// NewFileList returns the list of files, given as slash-separated paths +// relative to the archived folder. +func NewFileList(files []string) *FileList { + l := &FileList{files: map[string]bool{}, folders: map[string]bool{}} + for _, file := range files { + file = path.Clean(file) + l.files[file] = true + for dir := path.Dir(file); dir != "." && dir != "/" && !l.folders[dir]; dir = path.Dir(dir) { + l.folders[dir] = true + } + } + return l +} + +// Len returns the number of files in the list. +func (l *FileList) Len() int { + return len(l.files) +} + +// hasFile reports whether the file relativePath is listed. A nil list lists +// every file. +func (l *FileList) hasFile(relativePath string) bool { + return l == nil || l.files[relativePath] +} + +// hasFolder reports whether a file in the folder relativePath is listed, or +// the folder itself is, as git lists a submodule that isn't checked out. A nil +// list lists every folder, and the root is always listed. +func (l *FileList) hasFolder(relativePath string) bool { + return l == nil || relativePath == "" || relativePath == "." || l.folders[relativePath] || l.files[relativePath] +} diff --git a/pkg/git/worktree.go b/pkg/git/worktree.go new file mode 100644 index 000000000..b573933d7 --- /dev/null +++ b/pkg/git/worktree.go @@ -0,0 +1,264 @@ +package git + +import ( + "bytes" + "context" + "errors" + "fmt" + "os" + "os/exec" + "path" + "path/filepath" + "strconv" + "strings" +) + +// ErrNotInstalled is returned when the git binary is not in the PATH. +var ErrNotInstalled = errors.New("git is not installed") + +// ListFiles returns the files of the work tree at dir that git doesn't ignore: tracked files, and untracked files +// that no .gitignore file, .git/info/exclude or core.excludesFile ignores. Paths are slash-separated and relative +// to dir. The files of checked out submodules are listed too, but not those of other repositories or worktrees +// nested in dir, nor anything in .git. +func ListFiles(ctx context.Context, dir string) ([]string, error) { + // The index tells submodules from files, untracked files have no mode + tracked, err := output(ctx, dir, "ls-files", "-z", "--cached", "--stage") + if err != nil { + return nil, err + } + untracked, err := output(ctx, dir, "ls-files", "-z", "--others", "--exclude-standard") + if err != nil { + return nil, err + } + + files := []string{} + for _, entry := range strings.Split(tracked, "\x00") { + // \t + info, file, found := strings.Cut(entry, "\t") + // a file with a merge conflict has an entry for each stage, one after the other + if !found || (len(files) > 0 && files[len(files)-1] == file) { + continue + } + if !strings.HasPrefix(info, "160000 ") { + files = append(files, file) + continue + } + + // A submodule that isn't checked out has no .git, and git would list the parent repository's files in it + submodule := filepath.Join(dir, filepath.FromSlash(file)) + if _, err := os.Lstat(filepath.Join(submodule, ".git")); err != nil { + files = append(files, file) + continue + } + submoduleFiles, err := ListFiles(ctx, submodule) + if err != nil { + return nil, fmt.Errorf("list files of submodule %s: %w", file, err) + } + for _, submoduleFile := range submoduleFiles { + files = append(files, path.Join(file, submoduleFile)) + } + } + for _, file := range strings.Split(untracked, "\x00") { + // Nested repositories and worktrees are listed as a folder with a trailing slash + if file != "" && !strings.HasSuffix(file, "/") { + files = append(files, file) + } + } + + return files, nil +} + +// IsTopLevel reports whether dir is the top level folder of a git work tree. +func IsTopLevel(ctx context.Context, dir string) bool { + prefix, err := output(ctx, dir, "rev-parse", "--show-prefix") + return err == nil && strings.TrimSpace(prefix) == "" +} + +// Snapshot creates a shallow repository in target/.git with the commit checked out in the work tree at dir, so that +// a copy of the work tree can be committed and pushed elsewhere. It has the commit and, if the branch has an upstream +// branch, the commits of both since they diverged, but no other history. The branch, its upstream branch and the +// remotes and user of the repository's config are the same as in dir. The index matches the commit, so changes staged +// in dir show as not staged. It returns the checked out branch, empty if HEAD is detached. +func Snapshot(ctx context.Context, dir, target string) (string, error) { + dir, err := filepath.Abs(dir) + if err != nil { + return "", err + } + + // an empty template leaves out the sample hooks + initArgs := []string{"init", "--quiet", "--template="} + if objectFormat, err := output(ctx, dir, "rev-parse", "--show-object-format"); err == nil { + initArgs = append(initArgs, "--object-format="+strings.TrimSpace(objectFormat)) + } + if _, err := output(ctx, target, initArgs...); err != nil { + return "", err + } + // git init sets these for a case-insensitive file system, as on macOS, but the workspace's file system is not. + // It also sets core.filemode and core.symlinks, which match how the files of the work tree are uploaded. + for _, key := range []string{"core.ignorecase", "core.precomposeunicode"} { + _, _ = output(ctx, target, "config", "--local", "--unset-all", key) + } + + branchRef, err := output(ctx, dir, "symbolic-ref", "--quiet", "HEAD") + branchRef = strings.TrimSpace(branchRef) + if err != nil { + branchRef = "" + } + + // a new repository has no commit to fetch + _, err = output(ctx, dir, "rev-parse", "--quiet", "--verify", "HEAD") + if err == nil { + if err := fetchCommits(ctx, dir, target, branchRef); err != nil { + return "", err + } + } else if branchRef != "" { + if _, err := output(ctx, target, "symbolic-ref", "HEAD", branchRef); err != nil { + return "", err + } + } + + if err := copyConfig(ctx, dir, target, branchRef); err != nil { + return "", err + } + + // info/exclude ignores files in git status, like .gitignore files + exclude, err := output(ctx, dir, "rev-parse", "--path-format=absolute", "--git-path", "info/exclude") + if err == nil { + if content, err := os.ReadFile(strings.TrimSpace(exclude)); err == nil { + infoDir := filepath.Join(target, ".git", "info") + if err := os.MkdirAll(infoDir, 0o755); err != nil { + return "", err + } + if err := os.WriteFile(filepath.Join(infoDir, "exclude"), content, 0o644); err != nil { + return "", err + } + } + } + + _ = os.Remove(filepath.Join(target, ".git", "FETCH_HEAD")) + return strings.TrimPrefix(branchRef, "refs/heads/"), nil +} + +// fetchCommits fetches HEAD of dir into target with as little history as possible, and its upstream branch if +// there is one. +func fetchCommits(ctx context.Context, dir, target, branchRef string) error { + // Fetch the commits since HEAD and the upstream branch diverged, so that git status counts the commits each is + // ahead of the other + depth, upstreamDepth, upstreamRef := 1, 0, "" + if branchRef != "" { + upstreamRef, depth, upstreamDepth = upstream(ctx, dir) + } + + head, err := fetch(ctx, dir, target, "HEAD", depth) + if err != nil { + return err + } + if branchRef == "" { + _, err = output(ctx, target, "update-ref", "--no-deref", "HEAD", head) + if err != nil { + return err + } + } else { + if _, err := output(ctx, target, "update-ref", branchRef, head); err != nil { + return err + } + if _, err := output(ctx, target, "symbolic-ref", "HEAD", branchRef); err != nil { + return err + } + } + + if upstreamRef != "" { + commit, err := fetch(ctx, dir, target, upstreamRef, upstreamDepth) + if err != nil { + return err + } + if _, err := output(ctx, target, "update-ref", upstreamRef, commit); err != nil { + return err + } + } + + // without an index git status shows every file as deleted and untracked + _, err = output(ctx, target, "read-tree", "HEAD") + return err +} + +// upstream returns the upstream branch of HEAD in dir, and how deep to fetch HEAD and the upstream branch to reach +// the commit they diverged from. It returns an empty ref and a depth of 1 for HEAD if there is no upstream branch or +// no common commit. +func upstream(ctx context.Context, dir string) (string, int, int) { + ref, err := output(ctx, dir, "rev-parse", "--quiet", "--verify", "--symbolic-full-name", "@{upstream}") + if err != nil || strings.TrimSpace(ref) == "" { + return "", 1, 0 + } + if _, err := output(ctx, dir, "merge-base", "HEAD", "@{upstream}"); err != nil { + return "", 1, 0 + } + counts, err := output(ctx, dir, "rev-list", "--left-right", "--count", "HEAD...@{upstream}") + if err != nil { + return "", 1, 0 + } + ahead, behind, _ := strings.Cut(strings.TrimSpace(counts), "\t") + aheadCount, err := strconv.Atoi(ahead) + if err != nil { + return "", 1, 0 + } + behindCount, err := strconv.Atoi(behind) + if err != nil { + return "", 1, 0 + } + return strings.TrimSpace(ref), aheadCount + 1, behindCount + 1 +} + +// fetch fetches ref of dir into target with the given depth and returns the fetched commit. +func fetch(ctx context.Context, dir, target, ref string, depth int) (string, error) { + _, err := output(ctx, target, "fetch", "--quiet", "--no-tags", "--depth="+strconv.Itoa(depth), dir, ref) + if err != nil { + return "", err + } + commit, err := output(ctx, target, "rev-parse", "--verify", "FETCH_HEAD^{commit}") + return strings.TrimSpace(commit), err +} + +// copyConfig copies the remotes, the user and the settings of the branch from the config of the repository in dir +// to the repository in target. +func copyConfig(ctx context.Context, dir, target, branchRef string) error { + entries, err := output(ctx, dir, "config", "--local", "--null", "--get-regexp", `^(remote|user|branch)\.`) + if err != nil { + // exit code 1 means there are no such entries + var exitErr *exec.ExitError + if errors.As(err, &exitErr) && exitErr.ExitCode() == 1 { + return nil + } + return err + } + + branchPrefix := "branch." + strings.TrimPrefix(branchRef, "refs/heads/") + "." + for _, entry := range strings.Split(entries, "\x00") { + key, value, found := strings.Cut(entry, "\n") + if !found { + continue + } + if strings.HasPrefix(key, "branch.") && (branchRef == "" || !strings.HasPrefix(key, branchPrefix)) { + continue + } + if _, err := output(ctx, target, "config", "--local", "--add", key, value); err != nil { + return err + } + } + return nil +} + +// output runs git in dir and returns its standard output. +func output(ctx context.Context, dir string, args ...string) (string, error) { + cmd := CommandContext(ctx, nil, append([]string{"-C", dir}, args...)...) + stderr := &bytes.Buffer{} + cmd.Stderr = stderr + out, err := cmd.Output() + if err != nil { + if errors.Is(err, exec.ErrNotFound) { + return "", ErrNotInstalled + } + return string(out), fmt.Errorf("git %s: %s: %w", strings.Join(args, " "), strings.TrimSpace(stderr.String()), err) + } + return string(out), nil +} diff --git a/pkg/git/worktree_test.go b/pkg/git/worktree_test.go new file mode 100644 index 000000000..9567b276b --- /dev/null +++ b/pkg/git/worktree_test.go @@ -0,0 +1,253 @@ +package git + +import ( + "context" + "os" + "os/exec" + "path/filepath" + "sort" + "strings" + "testing" + + "gotest.tools/assert" + "gotest.tools/assert/cmp" +) + +// isolateGit makes the git commands of the code under test ignore the user's +// and the system's git config. +func isolateGit(t *testing.T) { + t.Helper() + requireGit(t) + emptyConfig := filepath.Join(t.TempDir(), "gitconfig") + assert.NilError(t, os.WriteFile(emptyConfig, nil, 0644)) + t.Setenv("GIT_CONFIG_GLOBAL", emptyConfig) + t.Setenv("GIT_CONFIG_NOSYSTEM", "1") +} + +func gitOutput(t *testing.T, dir string, args ...string) string { + t.Helper() + cmd := exec.Command("git", args...) + cmd.Dir = dir + out, err := cmd.CombinedOutput() + assert.NilError(t, err, string(out)) + return strings.TrimSpace(string(out)) +} + +func writeRepoFiles(t *testing.T, root string, files ...string) { + t.Helper() + for _, file := range files { + p := filepath.Join(root, filepath.FromSlash(file)) + assert.NilError(t, os.MkdirAll(filepath.Dir(p), 0o755)) + assert.NilError(t, os.WriteFile(p, []byte(file+"\n"), 0o644)) + } +} + +func commitAll(t *testing.T, repo, message string) { + t.Helper() + runGit(t, repo, "add", "-A") + runGit(t, repo, "-c", "user.name=test", "-c", "user.email=test@example.com", "commit", "-q", "-m", message) +} + +// newRepository creates a repository with a commit of README.md on branch main. +func newRepository(t *testing.T, repo string) { + t.Helper() + runGit(t, filepath.Dir(repo), "init", "-q", "-b", "main", repo) + writeRepoFiles(t, repo, "README.md") + commitAll(t, repo, "init") +} + +func TestListFiles(t *testing.T) { + isolateGit(t) + ctx := context.Background() + dir := t.TempDir() + + lib := filepath.Join(dir, "lib") + newRepository(t, lib) + + repo := filepath.Join(dir, "repo") + newRepository(t, repo) + assert.NilError(t, os.WriteFile(filepath.Join(repo, ".gitignore"), []byte("node_modules/\n*.log\n"), 0o644)) + writeRepoFiles(t, repo, "src/main.go", "forced.log") + runGit(t, repo, "add", "-f", "forced.log") + runGit(t, repo, "-c", "protocol.file.allow=always", "submodule", "add", "-q", lib, "modules/lib") + commitAll(t, repo, "files") + + // ignored and untracked files, a nested repository and a worktree + writeRepoFiles(t, repo, "app.log", "node_modules/x/index.js", "new.txt", "secret.txt") + assert.NilError(t, os.WriteFile(filepath.Join(repo, ".git", "info", "exclude"), []byte("secret.txt\n"), 0o644)) + newRepository(t, filepath.Join(repo, "nested")) + runGit(t, repo, "worktree", "add", "-q", filepath.Join(repo, "worktree")) + + files, err := ListFiles(ctx, repo) + assert.NilError(t, err) + sort.Strings(files) + assert.DeepEqual(t, []string{ + ".gitignore", + ".gitmodules", + "README.md", + "forced.log", + "modules/lib/README.md", + "new.txt", + "src/main.go", + }, files) + + files, err = ListFiles(ctx, filepath.Join(repo, "src")) + assert.NilError(t, err) + assert.DeepEqual(t, []string{"main.go"}, files) +} + +func TestListFilesOfSubmoduleThatIsNotCheckedOut(t *testing.T) { + isolateGit(t) + dir := t.TempDir() + lib := filepath.Join(dir, "lib") + newRepository(t, lib) + repo := filepath.Join(dir, "repo") + newRepository(t, repo) + runGit(t, repo, "-c", "protocol.file.allow=always", "submodule", "add", "-q", lib, "lib") + commitAll(t, repo, "submodule") + runGit(t, repo, "submodule", "deinit", "-q", "lib") + + files, err := ListFiles(context.Background(), repo) + assert.NilError(t, err) + sort.Strings(files) + assert.DeepEqual(t, []string{".gitmodules", "README.md", "lib"}, files) +} + +func TestListFilesWithoutRepository(t *testing.T) { + isolateGit(t) + _, err := ListFiles(context.Background(), t.TempDir()) + assert.ErrorContains(t, err, "ls-files") +} + +func TestIsTopLevel(t *testing.T) { + isolateGit(t) + ctx := context.Background() + repo := filepath.Join(t.TempDir(), "repo") + newRepository(t, repo) + writeRepoFiles(t, repo, "src/main.go") + + assert.Assert(t, IsTopLevel(ctx, repo)) + assert.Assert(t, !IsTopLevel(ctx, filepath.Join(repo, "src"))) + assert.Assert(t, !IsTopLevel(ctx, t.TempDir())) +} + +// copyWorkTree copies the files git lists in repo to target. +func copyWorkTree(t *testing.T, repo, target string) { + t.Helper() + files, err := ListFiles(context.Background(), repo) + assert.NilError(t, err) + for _, file := range files { + content, err := os.ReadFile(filepath.Join(repo, file)) + assert.NilError(t, err) + p := filepath.Join(target, file) + assert.NilError(t, os.MkdirAll(filepath.Dir(p), 0o755)) + assert.NilError(t, os.WriteFile(p, content, 0o644)) + } +} + +func TestSnapshot(t *testing.T) { + isolateGit(t) + ctx := context.Background() + dir := t.TempDir() + + // repo is a commit ahead of and a commit behind origin/main + origin := filepath.Join(dir, "origin.git") + runGit(t, dir, "init", "-q", "--bare", "-b", "main", origin) + other := filepath.Join(dir, "other") + newRepository(t, other) + writeRepoFiles(t, other, "history.txt") + commitAll(t, other, "history") + runGit(t, other, "push", "-q", origin, "main") + repo := filepath.Join(dir, "repo") + runGit(t, dir, "clone", "-q", origin, repo) + runGit(t, repo, "config", "user.email", "repo@example.com") + writeRepoFiles(t, repo, "local.txt") + commitAll(t, repo, "local") + writeRepoFiles(t, other, "remote.txt") + commitAll(t, other, "remote") + runGit(t, other, "push", "-q", origin, "main") + runGit(t, repo, "fetch", "-q") + // uncommitted changes + writeRepoFiles(t, repo, "new.txt") + assert.NilError(t, os.WriteFile(filepath.Join(repo, "README.md"), []byte("changed\n"), 0o644)) + assert.NilError(t, os.WriteFile(filepath.Join(repo, ".git", "info", "exclude"), []byte("*.tmp\n"), 0o644)) + + target := t.TempDir() + branch, err := Snapshot(ctx, repo, target) + assert.NilError(t, err) + assert.Equal(t, "main", branch) + copyWorkTree(t, repo, target) + writeRepoFiles(t, target, "excluded.tmp") + + assert.Equal(t, "refs/heads/main", gitOutput(t, target, "symbolic-ref", "HEAD")) + assert.Equal(t, gitOutput(t, repo, "rev-parse", "HEAD"), gitOutput(t, target, "rev-parse", "HEAD")) + assert.Equal(t, "true", gitOutput(t, target, "rev-parse", "--is-shallow-repository")) + // the local commit and the one it diverged from origin/main + assert.Equal(t, "2", gitOutput(t, target, "rev-list", "--count", "HEAD")) + assert.Equal(t, origin, gitOutput(t, target, "remote", "get-url", "origin")) + assert.Equal(t, "repo@example.com", gitOutput(t, target, "config", "user.email")) + assert.Equal(t, "origin/main", gitOutput(t, target, "rev-parse", "--abbrev-ref", "@{upstream}")) + status := gitOutput(t, target, "status", "--porcelain=v2", "--branch") + assert.Assert(t, cmp.Contains(status, "# branch.ab +1 -1")) + assert.Assert(t, cmp.Contains(status, "1 .M N... 100644 100644 100644")) + assert.Assert(t, cmp.Contains(status, "README.md")) + assert.Assert(t, cmp.Contains(status, "? new.txt")) + assert.Assert(t, !strings.Contains(status, "excluded.tmp"), status) + assert.Assert(t, !strings.Contains(status, "local.txt"), status) + _, err = os.Stat(filepath.Join(target, ".git", "hooks")) + assert.Assert(t, os.IsNotExist(err)) + config, err := os.ReadFile(filepath.Join(target, ".git", "config")) + assert.NilError(t, err) + assert.Assert(t, !strings.Contains(string(config), "ignorecase"), string(config)) + + // the snapshot can be committed and pushed + commitAll(t, target, "from snapshot") + runGit(t, target, "push", "-q", "origin", "main:snapshot") + assert.Equal(t, gitOutput(t, target, "rev-parse", "HEAD"), gitOutput(t, origin, "rev-parse", "snapshot")) +} + +func TestSnapshotOfDetachedHead(t *testing.T) { + isolateGit(t) + repo := filepath.Join(t.TempDir(), "repo") + newRepository(t, repo) + runGit(t, repo, "checkout", "-q", "--detach") + + target := t.TempDir() + branch, err := Snapshot(context.Background(), repo, target) + assert.NilError(t, err) + assert.Equal(t, "", branch) + assert.Equal(t, gitOutput(t, repo, "rev-parse", "HEAD"), gitOutput(t, target, "rev-parse", "HEAD")) + cmd := exec.Command("git", "symbolic-ref", "-q", "HEAD") + cmd.Dir = target + assert.Assert(t, cmd.Run() != nil, "HEAD is not detached") +} + +func TestSnapshotOfWorktree(t *testing.T) { + isolateGit(t) + dir := t.TempDir() + repo := filepath.Join(dir, "repo") + newRepository(t, repo) + worktree := filepath.Join(dir, "worktree") + runGit(t, repo, "worktree", "add", "-q", "-b", "feature", worktree) + writeRepoFiles(t, worktree, "feature.txt") + commitAll(t, worktree, "feature") + + target := t.TempDir() + branch, err := Snapshot(context.Background(), worktree, target) + assert.NilError(t, err) + assert.Equal(t, "feature", branch) + assert.Equal(t, gitOutput(t, worktree, "rev-parse", "HEAD"), gitOutput(t, target, "rev-parse", "HEAD")) + assert.Equal(t, "1", gitOutput(t, target, "rev-list", "--count", "HEAD")) +} + +func TestSnapshotWithoutCommits(t *testing.T) { + isolateGit(t) + repo := filepath.Join(t.TempDir(), "repo") + runGit(t, filepath.Dir(repo), "init", "-q", "-b", "trunk", repo) + + target := t.TempDir() + branch, err := Snapshot(context.Background(), repo, target) + assert.NilError(t, err) + assert.Equal(t, "trunk", branch) + assert.Equal(t, "refs/heads/trunk", gitOutput(t, target, "symbolic-ref", "HEAD")) +} From af26e71c655ba603e66f12ee087b1d3bfefade1d Mon Sep 17 00:00:00 2001 From: renatoathaydes Date: Thu, 22 Oct 2026 14:22:50 +0200 Subject: [PATCH 2/3] Leave the user out of the shallow .git DevPod now gives a workspace the git identity of its repository, or the one given with --git-user-name and --git-user-email, in the container's global git config. A user copied into the snapshot's local config would override it. Co-Authored-By: Claude Opus 5.5 --- pkg/git/worktree.go | 9 +++++---- pkg/git/worktree_test.go | 5 ++++- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/pkg/git/worktree.go b/pkg/git/worktree.go index b573933d7..96fe22474 100644 --- a/pkg/git/worktree.go +++ b/pkg/git/worktree.go @@ -77,8 +77,9 @@ func IsTopLevel(ctx context.Context, dir string) bool { // Snapshot creates a shallow repository in target/.git with the commit checked out in the work tree at dir, so that // a copy of the work tree can be committed and pushed elsewhere. It has the commit and, if the branch has an upstream // branch, the commits of both since they diverged, but no other history. The branch, its upstream branch and the -// remotes and user of the repository's config are the same as in dir. The index matches the commit, so changes staged -// in dir show as not staged. It returns the checked out branch, empty if HEAD is detached. +// remotes are the same as in dir. The user is left out, as DevPod gives the workspace the repository's git identity +// in the global config, which a local one would override. The index matches the commit, so changes staged in dir +// show as not staged. It returns the checked out branch, empty if HEAD is detached. func Snapshot(ctx context.Context, dir, target string) (string, error) { dir, err := filepath.Abs(dir) if err != nil { @@ -219,10 +220,10 @@ func fetch(ctx context.Context, dir, target, ref string, depth int) (string, err return strings.TrimSpace(commit), err } -// copyConfig copies the remotes, the user and the settings of the branch from the config of the repository in dir +// copyConfig copies the remotes and the settings of the branch from the config of the repository in dir // to the repository in target. func copyConfig(ctx context.Context, dir, target, branchRef string) error { - entries, err := output(ctx, dir, "config", "--local", "--null", "--get-regexp", `^(remote|user|branch)\.`) + entries, err := output(ctx, dir, "config", "--local", "--null", "--get-regexp", `^(remote|branch)\.`) if err != nil { // exit code 1 means there are no such entries var exitErr *exec.ExitError diff --git a/pkg/git/worktree_test.go b/pkg/git/worktree_test.go index 9567b276b..83ad048ac 100644 --- a/pkg/git/worktree_test.go +++ b/pkg/git/worktree_test.go @@ -185,7 +185,10 @@ func TestSnapshot(t *testing.T) { // the local commit and the one it diverged from origin/main assert.Equal(t, "2", gitOutput(t, target, "rev-list", "--count", "HEAD")) assert.Equal(t, origin, gitOutput(t, target, "remote", "get-url", "origin")) - assert.Equal(t, "repo@example.com", gitOutput(t, target, "config", "user.email")) + // DevPod gives the workspace the repository's identity in the global git config, which a local one would override + cmd := exec.Command("git", "config", "--local", "user.email") + cmd.Dir = target + assert.Assert(t, cmd.Run() != nil, "the snapshot has a user") assert.Equal(t, "origin/main", gitOutput(t, target, "rev-parse", "--abbrev-ref", "@{upstream}")) status := gitOutput(t, target, "status", "--porcelain=v2", "--branch") assert.Assert(t, cmp.Contains(status, "# branch.ab +1 -1")) From 273a4c333ff0f41bc968126a1feb52d578373801 Mon Sep 17 00:00:00 2001 From: renatoathaydes Date: Thu, 22 Oct 2026 15:13:32 +0200 Subject: [PATCH 3/3] Fix review findings in git-filtered uploads - Match git's file names with those on disk in Unicode normalization form C, and ignoring case when git does (core.ignorecase), and archive listed files with git's spelling. On macOS, files with decomposed names or renamed only in case were left out, and showed as deleted. - Mark tracked files that .devpodignore leaves out skip-worktree in the shallow .git, so they don't show as deleted and get committed as such. - Give each checked out submodule a shallow .git of its own, and copy the submodule config, so submodules stay repositories in the workspace. Overlays now replace folders at any path. - Fetch HEAD and its upstream branch in one shallow fetch, as a second one cut the history of the first and miscounted commits ahead. - Create the shallow .git with the ref format of the repository, not the user's init.defaultRefFormat, which older git can't read. - Copy only the settings of the checked out branch, not of branches whose name starts with it, and keep boolean settings without a value. Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 5 +- .../create-a-workspace.mdx | 7 +- go.mod | 2 +- pkg/agent/tunnelserver/tunnelserver.go | 36 ++- .../tunnelserver/tunnelserver_git_test.go | 41 +++ pkg/extract/compress.go | 48 +++- pkg/extract/compress_test.go | 42 ++- pkg/extract/rules.go | 55 +++- pkg/git/worktree.go | 249 ++++++++++++++---- pkg/git/worktree_test.go | 143 +++++++++- 10 files changed, 515 insertions(+), 113 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 86ba2b2dc..edacf385f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -18,8 +18,9 @@ Add new entries to the top of the "Unreleased" section. whole folder, when the workspace runs on another machine or in Kubernetes. Ignored files (build output, `node_modules`, ...) and nested repositories and git worktrees stay behind. Instead of the repository's `.git`, the workspace gets a shallow one with the checked out commit, the branch, its upstream branch and the - remotes, which is enough to commit and push. `.devpodignore` takes precedence: its patterns leave out more - files, and its negation (`!`) patterns upload files git ignores (`!.git` uploads the whole `.git`). See + remotes, which is enough to commit and push; so does each checked out submodule. `.devpodignore` takes + precedence: its patterns leave out more files, and its negation (`!`) patterns upload files git ignores + (`!.git` uploads the whole `.git`). See [Which files are uploaded](https://devpod.curityio.net/developing-in-workspaces/create-a-workspace#which-files-are-uploaded). ### Fixed diff --git a/docs/pages/developing-in-workspaces/create-a-workspace.mdx b/docs/pages/developing-in-workspaces/create-a-workspace.mdx index cc36f0cc7..964ceac78 100644 --- a/docs/pages/developing-in-workspaces/create-a-workspace.mdx +++ b/docs/pages/developing-in-workspaces/create-a-workspace.mdx @@ -97,7 +97,8 @@ stay behind, as do other repositories and git worktrees inside the folder. The f If the folder is the top of the repository, the workspace gets a small `.git` in place of yours, with only the checked out commit, so that you can commit and push from the workspace. It has the same branch, remotes and upstream branch as yours, and the commits since your branch and its upstream branch diverged, but no other -history. Changes you have staged show as not staged in the workspace. +history. Changes you have staged show as not staged in the workspace. Each checked out submodule gets a small +`.git` of its own in the same way. A `.devpodignore` file at the top of the folder takes precedence over git. It has the syntax of a [`.dockerignore` file](https://docs.docker.com/build/concepts/context/#dockerignore-files), so patterns are @@ -111,7 +112,9 @@ pattern leaves out files git doesn't ignore, and a negation (`!`) pattern upload docs/**/*.mp4 ``` -`!.git` uploads your whole `.git` instead of the small one, and `.git` uploads no `.git` at all. +Tracked files that `.devpodignore` leaves out don't show as deleted in the workspace, as git marks them +[skip-worktree](https://git-scm.com/docs/git-update-index#_skip_worktree_bit), but their content is still uploaded +in the small `.git`. `!.git` uploads your whole `.git` instead of the small one, and `.git` uploads no `.git` at all. If the folder is not in a git repository, or git is not installed, DevPod uploads every file `.devpodignore` doesn't exclude. diff --git a/go.mod b/go.mod index eea206e5e..07ddbc6d8 100644 --- a/go.mod +++ b/go.mod @@ -195,7 +195,7 @@ require ( github.com/xeipuuv/gojsonschema v1.2.0 // indirect golang.org/x/net v0.37.0 // indirect golang.org/x/sync v0.12.0 // indirect - golang.org/x/text v0.23.0 // indirect + golang.org/x/text v0.23.0 golang.org/x/time v0.9.0 // indirect golang.org/x/tools v0.29.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect diff --git a/pkg/agent/tunnelserver/tunnelserver.go b/pkg/agent/tunnelserver/tunnelserver.go index da2ecee60..a3efa9e6d 100644 --- a/pkg/agent/tunnelserver/tunnelserver.go +++ b/pkg/agent/tunnelserver/tunnelserver.go @@ -438,13 +438,15 @@ func (t *tunnelServer) workspaceTarOptions(ctx context.Context, folder string) ( t.log.Debugf("Upload all files of %s not excluded by %s, git doesn't list any", folder, ignoreFileName) return opts, noCleanup } - opts.Files = extract.NewFileList(files) + opts.Files = extract.NewFileList(files, git.IgnoresCase(ctx, folder)) t.log.Debugf("Upload the %d files of %s git doesn't ignore, and those %s re-includes", opts.Files.Len(), folder, ignoreFileName) if !git.IsTopLevel(ctx, folder) { return opts, noCleanup } - if verdict := gitFolderVerdict(opts.Excludes); verdict != extract.Unmatched { + // readIgnoreFile only returns valid patterns + rules, _ := extract.NewExcludeRules(opts.Excludes) + if verdict := gitFolderVerdict(rules); verdict != extract.Unmatched { t.log.Debugf("Upload %s as %s says, instead of a shallow copy", gitFolder, ignoreFileName) return opts, noCleanup } @@ -455,28 +457,38 @@ func (t *tunnelServer) workspaceTarOptions(ctx context.Context, folder string) ( return opts, noCleanup } cleanup := func() { _ = os.RemoveAll(snapshot) } - branch, err := git.Snapshot(ctx, folder, snapshot) + result, err := git.Snapshot(ctx, folder, snapshot, excludedFiles(rules, files)) if err != nil { cleanup() t.log.Warnf("Upload no %s, can't create a shallow copy: %v. Add !%s to %s to upload the whole %s", gitFolder, err, gitFolder, ignoreFileName, gitFolder) return opts, noCleanup } - if branch != "" { - t.log.Infof("Upload a shallow %s with branch %s", gitFolder, branch) + if result.Branch != "" { + t.log.Infof("Upload a shallow %s with branch %s", gitFolder, result.Branch) } else { t.log.Infof("Upload a shallow %s with the checked out commit", gitFolder) } - opts.Overlays = []string{filepath.Join(snapshot, gitFolder)} + opts.Overlays = map[string]string{} + for _, gitDir := range result.GitDirs { + opts.Overlays[gitDir] = filepath.Join(snapshot, filepath.FromSlash(gitDir)) + } return opts, cleanup } -// gitFolderVerdict returns what the exclude patterns say about the .git -// folder at the root. -func gitFolderVerdict(excludes []string) extract.Verdict { - rules, err := extract.NewExcludeRules(excludes) - if err != nil { - return extract.Unmatched +// excludedFiles returns the files the exclude rules leave out. +func excludedFiles(rules *extract.ExcludeRules, files []string) []string { + var excluded []string + for _, file := range files { + if verdict, err := rules.Match(file); err == nil && verdict == extract.Excluded { + excluded = append(excluded, file) + } } + return excluded +} + +// gitFolderVerdict returns what the exclude rules say about the .git folder +// at the root. +func gitFolderVerdict(rules *extract.ExcludeRules) extract.Verdict { verdict, err := rules.Match(gitFolder) if err != nil { return extract.Unmatched diff --git a/pkg/agent/tunnelserver/tunnelserver_git_test.go b/pkg/agent/tunnelserver/tunnelserver_git_test.go index 1ed0d65cf..fcfe523fe 100644 --- a/pkg/agent/tunnelserver/tunnelserver_git_test.go +++ b/pkg/agent/tunnelserver/tunnelserver_git_test.go @@ -8,7 +8,9 @@ import ( "strings" "testing" + "github.com/loft-sh/devpod/pkg/agent/tunnel" "github.com/loft-sh/devpod/pkg/devcontainer/config" + "github.com/loft-sh/devpod/pkg/extract" "github.com/loft-sh/devpod/pkg/log" "gotest.tools/assert" is "gotest.tools/assert/cmp" @@ -125,3 +127,42 @@ func TestStreamMount_GitSubfolderHasNoGitFolder(t *testing.T) { assert.DeepEqual(t, []string{"main.go"}, entries) } + +func gitOutput(t *testing.T, dir string, args ...string) string { + t.Helper() + cmd := exec.Command("git", args...) + cmd.Dir = dir + out, err := cmd.CombinedOutput() + assert.NilError(t, err, string(out)) + return strings.TrimSpace(string(out)) +} + +func TestStreamMount_ExtractedGitWorkspaceIsClean(t *testing.T) { + setupInfo := newGitSetupInfo(t, "docs/**/*.mp4\n") + workspaceFolder := config.GetWorkspaceMount(setupInfo).Source + commit := []string{"-c", "user.name=test", "-c", "user.email=test@example.com", "commit", "-q", "-m"} + + lib := t.TempDir() + runGit(t, lib, "init", "-q", "-b", "main") + writeFiles(t, lib, "lib.go") + runGit(t, lib, "add", "-A") + runGit(t, lib, append(commit, "lib")...) + runGit(t, workspaceFolder, "-c", "protocol.file.allow=always", "submodule", "add", "-q", lib, "lib") + writeFiles(t, workspaceFolder, "docs/videos/intro.mp4") + runGit(t, workspaceFolder, "add", "-A") + runGit(t, workspaceFolder, append(commit, "docs and lib")...) + server := newSetupServer(setupInfo, log.Discard) + + stream := &mockStreamMountServer{} + assert.NilError(t, server.StreamMount( + &tunnel.StreamMountRequest{Mount: config.GetWorkspaceMount(setupInfo).String()}, stream)) + target := t.TempDir() + assert.NilError(t, extract.Extract(&stream.content, target)) + + _, err := os.Stat(filepath.Join(target, "docs", "videos", "intro.mp4")) + assert.Assert(t, os.IsNotExist(err)) + // the excluded video is not deleted and the submodule is unchanged + assert.Equal(t, "", gitOutput(t, target, "status", "--porcelain")) + assert.Equal(t, "", gitOutput(t, filepath.Join(target, "lib"), "status", "--porcelain")) + assert.Equal(t, gitOutput(t, workspaceFolder, "rev-parse", "HEAD"), gitOutput(t, target, "rev-parse", "HEAD")) +} diff --git a/pkg/extract/compress.go b/pkg/extract/compress.go index 65188cece..c5f040a3f 100644 --- a/pkg/extract/compress.go +++ b/pkg/extract/compress.go @@ -4,6 +4,7 @@ import ( "archive/tar" "compress/gzip" "io" + "maps" "os" "path" "path/filepath" @@ -30,9 +31,10 @@ type TarOptions struct { // KeepNames are names of folders that are archived whole wherever the walk // reaches them, whatever the excludes and files say. KeepNames []string - // Overlays are folders archived whole at the root of the archive under - // their own name, in place of the archived folder's entry of that name. - Overlays []string + // Overlays maps slash-separated paths relative to the archived folder to + // local folders that are archived whole at those paths, in place of what + // the archived folder has there. + Overlays map[string]string } func WriteTarExclude(writer io.Writer, localPath string, compress bool, excludedPaths []string) error { @@ -80,17 +82,24 @@ func WriteTarWithOptions(writer io.Writer, localPath string, opts TarOptions) er } archiver.files = opts.Files archiver.keepNames = opts.KeepNames - for _, overlay := range opts.Overlays { - archiver.overlaid = append(archiver.overlaid, filepath.Base(overlay)) + overlays := slices.Sorted(maps.Keys(opts.Overlays)) + archiver.overlaid = map[string]bool{} + for _, overlay := range overlays { + archiver.overlaid[path.Clean(overlay)] = true } if err := archiver.AddToArchive(relativePath); err != nil { return err } - for _, overlay := range opts.Overlays { - whole := &Archiver{basePath: filepath.Dir(overlay), writer: tarWriter, writtenFiles: map[string]bool{}} - if err := whole.AddToArchive(filepath.Base(overlay)); err != nil { - return errors.Wrapf(err, "archive %s", overlay) + for _, overlay := range overlays { + whole := &Archiver{ + basePath: opts.Overlays[overlay], + writer: tarWriter, + writtenFiles: map[string]bool{}, + prefix: path.Clean(overlay), + } + if err := whole.AddToArchive(""); err != nil { + return errors.Wrapf(err, "archive %s", opts.Overlays[overlay]) } } return nil @@ -114,8 +123,10 @@ type Archiver struct { files *FileList // keepNames are names of folders archived whole, whatever the excludes say. keepNames []string - // overlaid are the names at the root that overlays replace. - overlaid []string + // overlaid are the paths that overlays replace. + overlaid map[string]bool + // prefix is the path in the archive of basePath. + prefix string } // NewArchiver creates a new archiver. excludedPaths are patterns relative to @@ -142,7 +153,7 @@ func (a *Archiver) AddToArchive(relativePath string) error { // addToArchive adds a path to the archive. parentInfo holds the exclude // results of the parent directory, or the zero value when they are unknown. func (a *Archiver) addToArchive(relativePath string, parentInfo patternmatcher.MatchInfo) error { - if a.writtenFiles[relativePath] || slices.Contains(a.overlaid, relativePath) { + if a.writtenFiles[relativePath] || a.overlaid[relativePath] { return nil } @@ -206,6 +217,15 @@ func (a *Archiver) mayReincludeBelow(dir string) bool { return a.excludes.mayReincludeBelow(dir) } +// archiveName returns the name in the archive of the path relativePath: the +// spelling of the file list for a listed path. +func (a *Archiver) archiveName(relativePath string) string { + if spelling, listed := a.files.file(relativePath); listed { + relativePath = spelling + } + return path.Join(a.prefix, relativePath) +} + // tarFolder archives what is in a folder. archived tells whether the folder // itself is archived, which matters when it is empty. func (a *Archiver) tarFolder(target string, targetStat os.FileInfo, archived bool, matchInfo patternmatcher.MatchInfo) error { @@ -222,7 +242,7 @@ func (a *Archiver) tarFolder(target string, targetStat os.FileInfo, archived boo hdr.Uid = 0 hdr.Gid = 0 hdr.Mode = fillGo18FileTypeBits(int64(chmodTarEntry(os.FileMode(hdr.Mode))), targetStat) - hdr.Name = target + hdr.Name = a.archiveName(target) if err := a.writer.WriteHeader(hdr); err != nil { return errors.Wrap(err, "tar write header") } @@ -260,7 +280,7 @@ func (a *Archiver) tarFile(target string, targetStat os.FileInfo) error { if err != nil { return errors.Wrap(err, "create tar file info header") } - hdr.Name = target + hdr.Name = a.archiveName(target) hdr.Uid = 0 hdr.Gid = 0 hdr.Mode = fillGo18FileTypeBits(int64(chmodTarEntry(os.FileMode(hdr.Mode))), targetStat) diff --git a/pkg/extract/compress_test.go b/pkg/extract/compress_test.go index f69e7b909..7726faad4 100644 --- a/pkg/extract/compress_test.go +++ b/pkg/extract/compress_test.go @@ -367,7 +367,7 @@ func TestWriteTarWithFiles(t *testing.T) { } assert.DeepEqual(t, tt.expected, tarEntriesWithOptions(t, root, TarOptions{ Excludes: tt.excludes, - Files: NewFileList(files), + Files: NewFileList(files, false), })) }) } @@ -377,7 +377,7 @@ func TestWriteTarWithFilesKeepsFoldersWhole(t *testing.T) { root := newTestTree(t) entries := tarEntriesWithOptions(t, root, TarOptions{ - Files: NewFileList(gitFiles), + Files: NewFileList(gitFiles, false), KeepNames: []string{"worktrees"}, }) @@ -393,15 +393,18 @@ func TestWriteTarWithOverlays(t *testing.T) { entries := tarEntriesWithOptions(t, root, TarOptions{ Excludes: []string{"!web/.git"}, - Files: NewFileList(gitFiles), - Overlays: []string{filepath.Join(overlayParent, "docs")}, + Files: NewFileList(gitFiles, false), + Overlays: map[string]string{ + "docs": filepath.Join(overlayParent, "docs"), + "web/app/": filepath.Join(overlayParent, "docs", "sub"), + }, }) - // The overlay replaces the folder at the root only + // Overlays replace the folders at their paths only assert.DeepEqual(t, sorted( readmeFile, ".claude/settings.json", - "web/app/main.js", + "web/app/other.md", "web/.git/HEAD", "buildSrc/plugin.kt", "docs/new.md", @@ -409,6 +412,24 @@ func TestWriteTarWithOverlays(t *testing.T) { ), entries) } +func TestWriteTarWithFilesSpelledDifferently(t *testing.T) { + root := t.TempDir() + // macOS keeps the decomposed name a file was created with, git lists it composed + decomposed, composed := "cafe\u0301.txt", "caf\u00e9.txt" + writeTestFiles(t, root, "README.md", "Docs/"+decomposed, "other.txt") + + // git lists files with the case they were added with on a case-insensitive file system + files := []string{"Readme.md", "docs/" + composed} + assert.DeepEqual(t, sorted("Readme.md", "docs/"+composed), tarEntriesWithOptions(t, root, TarOptions{ + Files: NewFileList(files, true), + })) + + // without ignoring case, only the normalization differs + assert.DeepEqual(t, []string{"Docs/" + composed}, tarEntriesWithOptions(t, root, TarOptions{ + Files: NewFileList([]string{"Docs/" + composed}, false), + })) +} + func writeTestFiles(t *testing.T, root string, files ...string) { t.Helper() @@ -451,7 +472,7 @@ func TestExcludeRulesMatch(t *testing.T) { } func TestNewFileList(t *testing.T) { - list := NewFileList([]string{"a/b/c.txt", "a/d.txt", "e.txt", "./f/../g.txt"}) + list := NewFileList([]string{"a/b/c.txt", "a/d.txt", "e.txt", "./f/../g.txt"}, false) assert.Equal(t, 4, list.Len()) assert.Assert(t, list.hasFolder("")) @@ -463,6 +484,13 @@ func TestNewFileList(t *testing.T) { assert.Assert(t, !list.hasFolder("f")) assert.Assert(t, !list.hasFolder("a/b/c")) + ignoreCase := NewFileList([]string{"Docs/Read\u00e9.md"}, true) + spelling, listed := ignoreCase.file("docs/READE\u0301.md") + assert.Assert(t, listed) + assert.Equal(t, "Docs/Read\u00e9.md", spelling) + assert.Assert(t, ignoreCase.hasFolder("DOCS")) + assert.Assert(t, !list.hasFile("A/D.txt")) + var all *FileList assert.Assert(t, all.hasFile("anything")) assert.Assert(t, all.hasFolder("anything")) diff --git a/pkg/extract/rules.go b/pkg/extract/rules.go index 7facd8947..d725a87ed 100644 --- a/pkg/extract/rules.go +++ b/pkg/extract/rules.go @@ -8,6 +8,7 @@ import ( "github.com/moby/patternmatcher" "github.com/pkg/errors" + "golang.org/x/text/unicode/norm" ) // Verdict is what exclude patterns say about a path. @@ -158,40 +159,76 @@ func patternMayMatchBelow(pattern, dir []string) bool { } // FileList is a set of files to archive, relative to the archived folder. +// +// Its paths may be spelled differently from those on disk: git lists names +// in Unicode normalization form C, while macOS keeps them as they were +// created, and on a case-insensitive file system git lists a file with the +// case it was added with. So the list matches names in form C, ignoring case +// if it was created to, and listed files are archived with the list's +// spelling, which is what the index of the repository has. type FileList struct { - files map[string]bool - // folders holds the folders that have a listed file below them. + ignoreCase bool + // files maps the key of each file to its spelling. + files map[string]string + // folders holds the keys of the folders that have a listed file below them. folders map[string]bool } // NewFileList returns the list of files, given as slash-separated paths -// relative to the archived folder. -func NewFileList(files []string) *FileList { - l := &FileList{files: map[string]bool{}, folders: map[string]bool{}} +// relative to the archived folder. ignoreCase makes it match paths that +// only differ in case. +func NewFileList(files []string, ignoreCase bool) *FileList { + l := &FileList{ignoreCase: ignoreCase, files: map[string]string{}, folders: map[string]bool{}} for _, file := range files { file = path.Clean(file) - l.files[file] = true - for dir := path.Dir(file); dir != "." && dir != "/" && !l.folders[dir]; dir = path.Dir(dir) { + key := l.key(file) + l.files[key] = file + for dir := path.Dir(key); dir != "." && dir != "/" && !l.folders[dir]; dir = path.Dir(dir) { l.folders[dir] = true } } return l } +// key returns the form of relativePath that the list matches. +func (l *FileList) key(relativePath string) string { + key := norm.NFC.String(relativePath) + if l.ignoreCase { + key = strings.ToLower(key) + } + return key +} + // Len returns the number of files in the list. func (l *FileList) Len() int { return len(l.files) } +// file returns the listed spelling of the file relativePath, and whether it +// is listed. A nil list lists every file as it is spelled. +func (l *FileList) file(relativePath string) (string, bool) { + if l == nil { + return relativePath, true + } + spelling, listed := l.files[l.key(relativePath)] + return spelling, listed +} + // hasFile reports whether the file relativePath is listed. A nil list lists // every file. func (l *FileList) hasFile(relativePath string) bool { - return l == nil || l.files[relativePath] + _, listed := l.file(relativePath) + return listed } // hasFolder reports whether a file in the folder relativePath is listed, or // the folder itself is, as git lists a submodule that isn't checked out. A nil // list lists every folder, and the root is always listed. func (l *FileList) hasFolder(relativePath string) bool { - return l == nil || relativePath == "" || relativePath == "." || l.folders[relativePath] || l.files[relativePath] + if l == nil || relativePath == "" || relativePath == "." { + return true + } + key := l.key(relativePath) + _, listed := l.files[key] + return listed || l.folders[key] } diff --git a/pkg/git/worktree.go b/pkg/git/worktree.go index 96fe22474..8eac5d2b1 100644 --- a/pkg/git/worktree.go +++ b/pkg/git/worktree.go @@ -18,11 +18,10 @@ var ErrNotInstalled = errors.New("git is not installed") // ListFiles returns the files of the work tree at dir that git doesn't ignore: tracked files, and untracked files // that no .gitignore file, .git/info/exclude or core.excludesFile ignores. Paths are slash-separated and relative -// to dir. The files of checked out submodules are listed too, but not those of other repositories or worktrees -// nested in dir, nor anything in .git. +// to dir, spelled as git spells them. The files of checked out submodules are listed too, but not those of other +// repositories or worktrees nested in dir, nor anything in .git. func ListFiles(ctx context.Context, dir string) ([]string, error) { - // The index tells submodules from files, untracked files have no mode - tracked, err := output(ctx, dir, "ls-files", "-z", "--cached", "--stage") + entries, err := indexEntries(ctx, dir) if err != nil { return nil, err } @@ -32,30 +31,17 @@ func ListFiles(ctx context.Context, dir string) ([]string, error) { } files := []string{} - for _, entry := range strings.Split(tracked, "\x00") { - // \t - info, file, found := strings.Cut(entry, "\t") - // a file with a merge conflict has an entry for each stage, one after the other - if !found || (len(files) > 0 && files[len(files)-1] == file) { - continue - } - if !strings.HasPrefix(info, "160000 ") { - files = append(files, file) + for _, entry := range entries { + if !entry.checkedOutSubmodule(dir) { + files = append(files, entry.path) continue } - - // A submodule that isn't checked out has no .git, and git would list the parent repository's files in it - submodule := filepath.Join(dir, filepath.FromSlash(file)) - if _, err := os.Lstat(filepath.Join(submodule, ".git")); err != nil { - files = append(files, file) - continue - } - submoduleFiles, err := ListFiles(ctx, submodule) + submoduleFiles, err := ListFiles(ctx, filepath.Join(dir, filepath.FromSlash(entry.path))) if err != nil { - return nil, fmt.Errorf("list files of submodule %s: %w", file, err) + return nil, fmt.Errorf("list files of submodule %s: %w", entry.path, err) } for _, submoduleFile := range submoduleFiles { - files = append(files, path.Join(file, submoduleFile)) + files = append(files, path.Join(entry.path, submoduleFile)) } } for _, file := range strings.Split(untracked, "\x00") { @@ -68,29 +54,128 @@ func ListFiles(ctx context.Context, dir string) ([]string, error) { return files, nil } +// indexEntry is a path in the index of a repository. +type indexEntry struct { + path string + submodule bool +} + +// checkedOutSubmodule reports whether the entry is a submodule that is checked out in the work tree at dir. One that +// isn't has no .git, and git would take its folder for one of the parent repository. +func (e indexEntry) checkedOutSubmodule(dir string) bool { + if !e.submodule { + return false + } + _, err := os.Lstat(filepath.Join(dir, filepath.FromSlash(e.path), ".git")) + return err == nil +} + +// indexEntries returns the paths in the index of the work tree at dir. +func indexEntries(ctx context.Context, dir string) ([]indexEntry, error) { + out, err := output(ctx, dir, "ls-files", "-z", "--cached", "--stage") + if err != nil { + return nil, err + } + + entries := []indexEntry{} + for _, line := range strings.Split(out, "\x00") { + // \t + info, file, found := strings.Cut(line, "\t") + // a file with a merge conflict has an entry for each stage, one after the other + if !found || (len(entries) > 0 && entries[len(entries)-1].path == file) { + continue + } + entries = append(entries, indexEntry{path: file, submodule: strings.HasPrefix(info, "160000 ")}) + } + return entries, nil +} + // IsTopLevel reports whether dir is the top level folder of a git work tree. func IsTopLevel(ctx context.Context, dir string) bool { prefix, err := output(ctx, dir, "rev-parse", "--show-prefix") return err == nil && strings.TrimSpace(prefix) == "" } +// IgnoresCase reports whether git ignores the case of file names in the work tree at dir, as it does on a +// case-insensitive file system. +func IgnoresCase(ctx context.Context, dir string) bool { + value, err := output(ctx, dir, "config", "--bool", "core.ignorecase") + return err == nil && strings.TrimSpace(value) == "true" +} + +// SnapshotResult describes the repositories Snapshot creates. +type SnapshotResult struct { + // Branch is the checked out branch, empty if HEAD is detached. + Branch string + // GitDirs are the .git folders created, slash-separated and relative to the target: .git, and one for each + // checked out submodule. + GitDirs []string +} + // Snapshot creates a shallow repository in target/.git with the commit checked out in the work tree at dir, so that // a copy of the work tree can be committed and pushed elsewhere. It has the commit and, if the branch has an upstream -// branch, the commits of both since they diverged, but no other history. The branch, its upstream branch and the -// remotes are the same as in dir. The user is left out, as DevPod gives the workspace the repository's git identity -// in the global config, which a local one would override. The index matches the commit, so changes staged in dir -// show as not staged. It returns the checked out branch, empty if HEAD is detached. -func Snapshot(ctx context.Context, dir, target string) (string, error) { +// branch, the commits of both since they diverged, but no other history. The branch, its upstream branch, the remotes +// and the submodules are the same as in dir. The user is left out, as DevPod gives the workspace the repository's git +// identity in the global config, which a local one would override. The index matches the commit, so changes staged +// in dir show as not staged. +// +// Each checked out submodule gets a snapshot of its own in its folder in target. missing are the files of the work +// tree, as ListFiles returns them, that the copy won't have: they are marked skip-worktree in the index, so that git +// doesn't take them for deleted. +func Snapshot(ctx context.Context, dir, target string, missing []string) (*SnapshotResult, error) { dir, err := filepath.Abs(dir) if err != nil { - return "", err + return nil, err + } + + branch, err := snapshotRepository(ctx, dir, target, missing) + if err != nil { + return nil, err } + result := &SnapshotResult{Branch: branch, GitDirs: []string{".git"}} - // an empty template leaves out the sample hooks + entries, err := indexEntries(ctx, dir) + if err != nil { + return nil, err + } + for _, entry := range entries { + if !entry.checkedOutSubmodule(dir) { + continue + } + var submoduleMissing []string + for _, file := range missing { + if rest, found := strings.CutPrefix(file, entry.path+"/"); found { + submoduleMissing = append(submoduleMissing, rest) + } + } + submoduleTarget := filepath.Join(target, filepath.FromSlash(entry.path)) + if err := os.MkdirAll(submoduleTarget, 0o755); err != nil { + return nil, err + } + submodule, err := Snapshot(ctx, filepath.Join(dir, filepath.FromSlash(entry.path)), submoduleTarget, submoduleMissing) + if err != nil { + return nil, fmt.Errorf("snapshot submodule %s: %w", entry.path, err) + } + for _, gitDir := range submodule.GitDirs { + result.GitDirs = append(result.GitDirs, path.Join(entry.path, gitDir)) + } + } + + return result, nil +} + +// snapshotRepository creates the snapshot of the repository at dir, without its submodules, and returns its branch. +func snapshotRepository(ctx context.Context, dir, target string, missing []string) (string, error) { + // An empty template leaves out the sample hooks. The formats are those of dir rather than the defaults of the + // user's config, which may be too new for git in the workspace. initArgs := []string{"init", "--quiet", "--template="} if objectFormat, err := output(ctx, dir, "rev-parse", "--show-object-format"); err == nil { initArgs = append(initArgs, "--object-format="+strings.TrimSpace(objectFormat)) } + // git before 2.45 has no ref formats other than files + if refFormat, err := output(ctx, dir, "rev-parse", "--show-ref-format"); err == nil { + initArgs = append(initArgs, "--ref-format="+strings.TrimSpace(refFormat)) + } if _, err := output(ctx, target, initArgs...); err != nil { return "", err } @@ -112,6 +197,9 @@ func Snapshot(ctx context.Context, dir, target string) (string, error) { if err := fetchCommits(ctx, dir, target, branchRef); err != nil { return "", err } + if err := skipWorktree(ctx, target, missing); err != nil { + return "", err + } } else if branchRef != "" { if _, err := output(ctx, target, "symbolic-ref", "HEAD", branchRef); err != nil { return "", err @@ -144,16 +232,25 @@ func Snapshot(ctx context.Context, dir, target string) (string, error) { // there is one. func fetchCommits(ctx context.Context, dir, target, branchRef string) error { // Fetch the commits since HEAD and the upstream branch diverged, so that git status counts the commits each is - // ahead of the other - depth, upstreamDepth, upstreamRef := 1, 0, "" + // ahead of the other. Both are fetched at once, as a second shallow fetch would cut the history of the first. + const headRef, fetchedUpstreamRef = "refs/devpod-snapshot/head", "refs/devpod-snapshot/upstream" + upstreamRef, depth := "", 1 if branchRef != "" { - upstreamRef, depth, upstreamDepth = upstream(ctx, dir) + upstreamRef, depth = upstream(ctx, dir) + } + args := []string{"fetch", "--quiet", "--no-tags", "--depth=" + strconv.Itoa(depth), dir, "+HEAD:" + headRef} + if upstreamRef != "" { + args = append(args, "+"+upstreamRef+":"+fetchedUpstreamRef) + } + if _, err := output(ctx, target, args...); err != nil { + return err } - head, err := fetch(ctx, dir, target, "HEAD", depth) + head, err := output(ctx, target, "rev-parse", "--verify", headRef+"^{commit}") if err != nil { return err } + head = strings.TrimSpace(head) if branchRef == "" { _, err = output(ctx, target, "update-ref", "--no-deref", "HEAD", head) if err != nil { @@ -167,13 +264,19 @@ func fetchCommits(ctx context.Context, dir, target, branchRef string) error { return err } } + if _, err := output(ctx, target, "update-ref", "-d", headRef); err != nil { + return err + } if upstreamRef != "" { - commit, err := fetch(ctx, dir, target, upstreamRef, upstreamDepth) + commit, err := output(ctx, target, "rev-parse", "--verify", fetchedUpstreamRef+"^{commit}") if err != nil { return err } - if _, err := output(ctx, target, "update-ref", upstreamRef, commit); err != nil { + if _, err := output(ctx, target, "update-ref", upstreamRef, strings.TrimSpace(commit)); err != nil { + return err + } + if _, err := output(ctx, target, "update-ref", "-d", fetchedUpstreamRef); err != nil { return err } } @@ -183,47 +286,69 @@ func fetchCommits(ctx context.Context, dir, target, branchRef string) error { return err } -// upstream returns the upstream branch of HEAD in dir, and how deep to fetch HEAD and the upstream branch to reach -// the commit they diverged from. It returns an empty ref and a depth of 1 for HEAD if there is no upstream branch or -// no common commit. -func upstream(ctx context.Context, dir string) (string, int, int) { +// upstream returns the upstream branch of HEAD in dir, and how deep to fetch HEAD and the upstream branch for both to +// reach the commit they diverged from. It returns an empty ref and a depth of 1 if there is no upstream branch or no +// common commit. +func upstream(ctx context.Context, dir string) (string, int) { ref, err := output(ctx, dir, "rev-parse", "--quiet", "--verify", "--symbolic-full-name", "@{upstream}") if err != nil || strings.TrimSpace(ref) == "" { - return "", 1, 0 + return "", 1 } if _, err := output(ctx, dir, "merge-base", "HEAD", "@{upstream}"); err != nil { - return "", 1, 0 + return "", 1 } counts, err := output(ctx, dir, "rev-list", "--left-right", "--count", "HEAD...@{upstream}") if err != nil { - return "", 1, 0 + return "", 1 } ahead, behind, _ := strings.Cut(strings.TrimSpace(counts), "\t") aheadCount, err := strconv.Atoi(ahead) if err != nil { - return "", 1, 0 + return "", 1 } behindCount, err := strconv.Atoi(behind) if err != nil { - return "", 1, 0 + return "", 1 } - return strings.TrimSpace(ref), aheadCount + 1, behindCount + 1 + return strings.TrimSpace(ref), max(aheadCount, behindCount) + 1 } -// fetch fetches ref of dir into target with the given depth and returns the fetched commit. -func fetch(ctx context.Context, dir, target, ref string, depth int) (string, error) { - _, err := output(ctx, target, "fetch", "--quiet", "--no-tags", "--depth="+strconv.Itoa(depth), dir, ref) +// skipWorktree marks the files of the index of the repository in target that are missing as skip-worktree. +func skipWorktree(ctx context.Context, target string, missing []string) error { + if len(missing) == 0 { + return nil + } + + entries, err := indexEntries(ctx, target) if err != nil { - return "", err + return err + } + inIndex := map[string]bool{} + for _, entry := range entries { + inIndex[entry.path] = true + } + stdin := &bytes.Buffer{} + for _, file := range missing { + if inIndex[file] { + stdin.WriteString(file + "\x00") + } } - commit, err := output(ctx, target, "rev-parse", "--verify", "FETCH_HEAD^{commit}") - return strings.TrimSpace(commit), err + if stdin.Len() == 0 { + return nil + } + + cmd := CommandContext(ctx, nil, "-C", target, "update-index", "-z", "--skip-worktree", "--stdin") + cmd.Stdin = stdin + if out, err := cmd.CombinedOutput(); err != nil { + return fmt.Errorf("git update-index --skip-worktree: %s: %w", strings.TrimSpace(string(out)), err) + } + return nil } -// copyConfig copies the remotes and the settings of the branch from the config of the repository in dir -// to the repository in target. +// copyConfig copies the remotes, the submodules and the settings of the branch from the config of the repository in +// dir to the repository in target. func copyConfig(ctx context.Context, dir, target, branchRef string) error { - entries, err := output(ctx, dir, "config", "--local", "--null", "--get-regexp", `^(remote|branch)\.`) + entries, err := output(ctx, dir, "config", "--local", "--null", "--get-regexp", `^(remote|submodule|branch)\.`) if err != nil { // exit code 1 means there are no such entries var exitErr *exec.ExitError @@ -235,12 +360,20 @@ func copyConfig(ctx context.Context, dir, target, branchRef string) error { branchPrefix := "branch." + strings.TrimPrefix(branchRef, "refs/heads/") + "." for _, entry := range strings.Split(entries, "\x00") { + if entry == "" { + continue + } key, value, found := strings.Cut(entry, "\n") + // a key without a value is a boolean that is true if !found { - continue + value = "true" } - if strings.HasPrefix(key, "branch.") && (branchRef == "" || !strings.HasPrefix(key, branchPrefix)) { - continue + // the settings of branch a.b start with branch.a. too + if strings.HasPrefix(key, "branch.") { + variable, ok := strings.CutPrefix(key, branchPrefix) + if branchRef == "" || !ok || strings.Contains(variable, ".") { + continue + } } if _, err := output(ctx, target, "config", "--local", "--add", key, value); err != nil { return err diff --git a/pkg/git/worktree_test.go b/pkg/git/worktree_test.go index 83ad048ac..aef30a88d 100644 --- a/pkg/git/worktree_test.go +++ b/pkg/git/worktree_test.go @@ -173,9 +173,10 @@ func TestSnapshot(t *testing.T) { assert.NilError(t, os.WriteFile(filepath.Join(repo, ".git", "info", "exclude"), []byte("*.tmp\n"), 0o644)) target := t.TempDir() - branch, err := Snapshot(ctx, repo, target) + result, err := Snapshot(ctx, repo, target, nil) assert.NilError(t, err) - assert.Equal(t, "main", branch) + assert.Equal(t, "main", result.Branch) + assert.DeepEqual(t, []string{".git"}, result.GitDirs) copyWorkTree(t, repo, target) writeRepoFiles(t, target, "excluded.tmp") @@ -216,9 +217,9 @@ func TestSnapshotOfDetachedHead(t *testing.T) { runGit(t, repo, "checkout", "-q", "--detach") target := t.TempDir() - branch, err := Snapshot(context.Background(), repo, target) + result, err := Snapshot(context.Background(), repo, target, nil) assert.NilError(t, err) - assert.Equal(t, "", branch) + assert.Equal(t, "", result.Branch) assert.Equal(t, gitOutput(t, repo, "rev-parse", "HEAD"), gitOutput(t, target, "rev-parse", "HEAD")) cmd := exec.Command("git", "symbolic-ref", "-q", "HEAD") cmd.Dir = target @@ -236,9 +237,9 @@ func TestSnapshotOfWorktree(t *testing.T) { commitAll(t, worktree, "feature") target := t.TempDir() - branch, err := Snapshot(context.Background(), worktree, target) + result, err := Snapshot(context.Background(), worktree, target, nil) assert.NilError(t, err) - assert.Equal(t, "feature", branch) + assert.Equal(t, "feature", result.Branch) assert.Equal(t, gitOutput(t, worktree, "rev-parse", "HEAD"), gitOutput(t, target, "rev-parse", "HEAD")) assert.Equal(t, "1", gitOutput(t, target, "rev-list", "--count", "HEAD")) } @@ -249,8 +250,134 @@ func TestSnapshotWithoutCommits(t *testing.T) { runGit(t, filepath.Dir(repo), "init", "-q", "-b", "trunk", repo) target := t.TempDir() - branch, err := Snapshot(context.Background(), repo, target) + result, err := Snapshot(context.Background(), repo, target, nil) assert.NilError(t, err) - assert.Equal(t, "trunk", branch) + assert.Equal(t, "trunk", result.Branch) assert.Equal(t, "refs/heads/trunk", gitOutput(t, target, "symbolic-ref", "HEAD")) } + +func TestSnapshotCountsCommitsAheadOfMergedUpstream(t *testing.T) { + isolateGit(t) + dir := t.TempDir() + origin := filepath.Join(dir, "origin.git") + runGit(t, dir, "init", "-q", "--bare", "-b", "main", origin) + repo := filepath.Join(dir, "repo") + newRepository(t, repo) + runGit(t, repo, "remote", "add", "origin", origin) + runGit(t, repo, "push", "-q", "-u", "origin", "main") + runGit(t, repo, "checkout", "-q", "-b", "feature", "--track", "origin/main") + for _, file := range []string{"f1", "f2", "f3"} { + writeRepoFiles(t, repo, file) + commitAll(t, repo, file) + } + runGit(t, repo, "checkout", "-q", "main") + writeRepoFiles(t, repo, "m1") + commitAll(t, repo, "m1") + runGit(t, repo, "push", "-q", "origin", "main") + runGit(t, repo, "checkout", "-q", "feature") + runGit(t, repo, "-c", "user.name=test", "-c", "user.email=test@example.com", "merge", "-q", "--no-edit", "origin/main") + expected := gitOutput(t, repo, "rev-list", "--left-right", "--count", "HEAD...@{upstream}") + assert.Equal(t, "4\t0", expected) + + target := t.TempDir() + _, err := Snapshot(context.Background(), repo, target, nil) + assert.NilError(t, err) + assert.Equal(t, expected, gitOutput(t, target, "rev-list", "--left-right", "--count", "HEAD...@{upstream}")) + assert.Equal(t, "", gitOutput(t, target, "for-each-ref", "refs/devpod-snapshot")) +} + +func TestSnapshotMarksMissingFilesSkipWorktree(t *testing.T) { + isolateGit(t) + repo := filepath.Join(t.TempDir(), "repo") + newRepository(t, repo) + writeRepoFiles(t, repo, "docs/big.mp4", "docs/guide.md") + commitAll(t, repo, "docs") + + target := t.TempDir() + _, err := Snapshot(context.Background(), repo, target, []string{"docs/big.mp4", "untracked.txt"}) + assert.NilError(t, err) + writeRepoFiles(t, target, "README.md", "docs/guide.md") + + assert.Equal(t, "", gitOutput(t, target, "status", "--porcelain")) + assert.Equal(t, "S docs/big.mp4", gitOutput(t, target, "ls-files", "-t", "docs/big.mp4")) +} + +func TestSnapshotOfSubmodules(t *testing.T) { + isolateGit(t) + ctx := context.Background() + dir := t.TempDir() + inner := filepath.Join(dir, "inner") + newRepository(t, inner) + lib := filepath.Join(dir, "lib") + newRepository(t, lib) + runGit(t, lib, "-c", "protocol.file.allow=always", "submodule", "add", "-q", inner, "inner") + commitAll(t, lib, "inner") + repo := filepath.Join(dir, "repo") + newRepository(t, repo) + runGit(t, repo, "-c", "protocol.file.allow=always", "submodule", "add", "-q", lib, "modules/lib") + runGit(t, repo, "-c", "protocol.file.allow=always", "submodule", "update", "-q", "--init", "--recursive") + commitAll(t, repo, "lib") + writeRepoFiles(t, filepath.Join(repo, "modules", "lib"), "change.txt") + + target := t.TempDir() + result, err := Snapshot(ctx, repo, target, []string{"modules/lib/README.md"}) + assert.NilError(t, err) + assert.DeepEqual(t, []string{".git", "modules/lib/.git", "modules/lib/inner/.git"}, result.GitDirs) + copyWorkTree(t, repo, target) + assert.NilError(t, os.Remove(filepath.Join(target, "modules", "lib", "README.md"))) + + // the submodules are repositories of their own, with their changes + assert.Equal(t, gitOutput(t, filepath.Join(repo, "modules", "lib"), "rev-parse", "HEAD"), + gitOutput(t, filepath.Join(target, "modules", "lib"), "rev-parse", "HEAD")) + assert.Equal(t, "?? change.txt", gitOutput(t, filepath.Join(target, "modules", "lib"), "status", "--porcelain")) + assert.Equal(t, "", gitOutput(t, filepath.Join(target, "modules", "lib", "inner"), "status", "--porcelain")) + assert.Equal(t, lib, gitOutput(t, target, "config", "submodule.modules/lib.url")) + status := gitOutput(t, target, "submodule", "status", "--recursive") + assert.Assert(t, !strings.Contains(status, "-"), status) + assert.Assert(t, cmp.Contains(gitOutput(t, target, "status", "--porcelain"), "modules/lib")) +} + +func TestSnapshotUsesRefFormatOfRepository(t *testing.T) { + isolateGit(t) + repo := filepath.Join(t.TempDir(), "repo") + newRepository(t, repo) + if _, err := output(context.Background(), repo, "rev-parse", "--show-ref-format"); err != nil { + t.Skip("git has no ref formats") + } + assert.NilError(t, os.WriteFile(os.Getenv("GIT_CONFIG_GLOBAL"), []byte("[init]\n\tdefaultRefFormat = reftable\n"), 0o644)) + + target := t.TempDir() + _, err := Snapshot(context.Background(), repo, target, nil) + assert.NilError(t, err) + assert.Equal(t, "files", gitOutput(t, target, "rev-parse", "--show-ref-format")) +} + +func TestCopyConfig(t *testing.T) { + isolateGit(t) + ctx := context.Background() + repo := filepath.Join(t.TempDir(), "repo") + newRepository(t, repo) + assert.NilError(t, os.WriteFile(filepath.Join(repo, ".git", "config"), []byte(`[branch "a"] + remote = origin + rebase +[branch "a.b"] + remote = other +[remote "origin"] + url = https://example.com/repo.git + fetch = +refs/heads/*:refs/remotes/origin/* + fetch = +refs/notes/*:refs/notes/* +[user] + email = repo@example.com +`), 0o644)) + target := t.TempDir() + runGit(t, target, "init", "-q") + + assert.NilError(t, copyConfig(ctx, repo, target, "refs/heads/a")) + + config := gitOutput(t, target, "config", "--local", "--list") + assert.Assert(t, cmp.Contains(config, "branch.a.remote=origin")) + assert.Assert(t, cmp.Contains(config, "branch.a.rebase=true")) + assert.Assert(t, cmp.Contains(config, "remote.origin.fetch=+refs/notes/*:refs/notes/*")) + assert.Assert(t, !strings.Contains(config, "branch.a.b"), config) + assert.Assert(t, !strings.Contains(config, "user.email"), config) +}