fix(watch): sync into a symlinked directory instead of failing

Signed-off-by: Endika Iglesias <endika2@gmail.com>
This commit is contained in:
Endika Iglesias 2026-08-17 21:56:32 +02:00 • committed by Guillaume Lours
parent 115898dc13
commit f519180b95
4 changed files with 256 additions and 16 deletions

View file

@ -96,7 +96,7 @@ func (t *Tar) Sync(ctx context.Context, service string, paths []*PathMapping) er
eg.SetLimit(16) // arbitrary limit, adjust to taste :D
for i := range containers {
containerID := containers[i].ID
tarReader := tarArchive(pathsToCopy)
tarReader := tarArchive(pathsToCopy, keepImpliedDirectories)
eg.Go(func() error {
if len(deleteCmd) != 0 {
@ -108,9 +108,24 @@ func (t *Tar) Sync(ctx context.Context, service string, paths []*PathMapping) er
}
if err := t.client.Untar(ctx, containerID, tarReader); err != nil {
errMu.Lock()
errs = append(errs, fmt.Errorf("copying files to %s: %w", containerID, err))
errMu.Unlock()
// The engine answers with a plain 500, so the message is the only discriminator.
if !strings.Contains(err.Error(), "cannot overwrite non-directory") {
errMu.Lock()
errs = append(errs, fmt.Errorf("copying files to %s: %w", containerID, err))
errMu.Unlock()
return nil
}
// A directory header for a path the container resolves through a symlink makes
// the engine reject the whole copy, so retry without the headers the archive's
// own entries already imply. That archive is a strict subset of the one that
// just failed, so sending it again is safe.
retry := tarArchive(pathsToCopy, dropImpliedDirectories)
if retryErr := t.client.Untar(ctx, containerID, retry); retryErr != nil {
errMu.Lock()
errs = append(errs, fmt.Errorf("copying files to %s: %w", containerID, errors.Join(err, retryErr)))
errMu.Unlock()
}
}
return nil // don't fail-fast; collect all errors
})
@ -139,7 +154,7 @@ func (a *ArchiveBuilder) Close() error {
}
// ArchivePathsIfExist creates a tar archive of all local files in `paths`. It quietly skips any paths that don't exist.
func (a *ArchiveBuilder) ArchivePathsIfExist(paths []PathMapping) error {
func (a *ArchiveBuilder) ArchivePathsIfExist(paths []PathMapping, dropImpliedDirs bool) error {
// In order to handle overlapping syncs, we
// 1) collect all the entries,
// 2) de-dupe them, with last-one-wins semantics
@ -161,6 +176,9 @@ func (a *ArchiveBuilder) ArchivePathsIfExist(paths []PathMapping) error {
}
entries = dedupeEntries(entries)
if dropImpliedDirs {
entries = withoutImpliedDirectories(entries)
}
for _, entry := range entries {
err := a.writeEntry(entry)
if err != nil {
@ -318,11 +336,17 @@ func (a *ArchiveBuilder) entriesForPath(localPath, containerPath string) ([]arch
return result, nil
}
func tarArchive(ops []PathMapping) io.ReadCloser {
// Whether tarArchive keeps the header of a directory the archive itself fills with entries.
const (
keepImpliedDirectories = false
dropImpliedDirectories = true
)
func tarArchive(ops []PathMapping, dropImpliedDirs bool) io.ReadCloser {
pr, pw := io.Pipe()
go func() {
ab := NewArchiveBuilder(pw)
err := ab.ArchivePathsIfExist(ops)
err := ab.ArchivePathsIfExist(ops, dropImpliedDirs)
if err != nil {
_ = pw.CloseWithError(fmt.Errorf("adding files to tar: %w", err))
} else {
@ -338,6 +362,28 @@ func tarArchive(ops []PathMapping) io.ReadCloser {
return pr
}
// withoutImpliedDirectories removes the header of every directory the archive already fills with
// entries: the engine creates those while extracting, while an explicit header makes it reject the
// whole copy when the container resolves that path through a symlink. A directory nothing else
// would create - an empty one - keeps its header.
//
// The engine creates them with ImpliedDirectoryMode and its own root uid/gid rather than the host's,
// so this is a fallback for a copy that already failed, never the first attempt.
func withoutImpliedDirectories(entries []archiveEntry) []archiveEntry {
holdsEntries := make(map[string]bool, len(entries))
for _, entry := range entries {
holdsEntries[path.Dir(path.Clean(entry.header.Name))] = true
}
result := make([]archiveEntry, 0, len(entries))
for _, entry := range entries {
if entry.header.Typeflag == tar.TypeDir && holdsEntries[path.Clean(entry.header.Name)] {
continue
}
result = append(result, entry)
}
return result
}
// Dedupe the entries with last-entry-wins semantics.
func dedupeEntries(entries []archiveEntry) []archiveEntry {
seenIndex := make(map[string]int, len(entries))

View file

@ -15,7 +15,9 @@
package sync
import (
"archive/tar"
"context"
"errors"
"io"
"os"
"path/filepath"
@ -32,6 +34,10 @@ type fakeLowLevelClient struct {
containers []container.Summary
execCmds [][]string
untarCount int
// untarErrs[i] is what the i-th Untar call returns
untarErrs []error
// untarHeaders[i] holds the headers the i-th Untar call received, by entry name
untarHeaders []map[string]tar.Header
}
func (f *fakeLowLevelClient) ContainersForService(_ context.Context, _ string, _ string) ([]container.Summary, error) {
@ -43,8 +49,25 @@ func (f *fakeLowLevelClient) Exec(_ context.Context, _ string, cmd []string, _ i
return nil
}
func (f *fakeLowLevelClient) Untar(_ context.Context, _ string, _ io.ReadCloser) error {
func (f *fakeLowLevelClient) Untar(_ context.Context, _ string, reader io.ReadCloser) error {
headers := map[string]tar.Header{}
tr := tar.NewReader(reader)
for {
header, err := tr.Next()
if errors.Is(err, io.EOF) {
break
}
if err != nil {
return err
}
headers[header.Name] = *header
}
f.untarHeaders = append(f.untarHeaders, headers)
f.untarCount++
if f.untarCount <= len(f.untarErrs) {
return f.untarErrs[f.untarCount-1]
}
return nil
}
@ -56,9 +79,9 @@ func TestSync_ExistingPath(t *testing.T) {
client := &fakeLowLevelClient{
containers: []container.Summary{{ID: "ctr1"}},
}
tar := NewTar("proj", client)
syncer := NewTar("proj", client)
err := tar.Sync(t.Context(), "svc", []*PathMapping{
err := syncer.Sync(t.Context(), "svc", []*PathMapping{
{HostPath: existingFile, ContainerPath: "/app/exists.txt"},
})
@ -71,9 +94,9 @@ func TestSync_NonExistentPath(t *testing.T) {
client := &fakeLowLevelClient{
containers: []container.Summary{{ID: "ctr1"}},
}
tar := NewTar("proj", client)
syncer := NewTar("proj", client)
err := tar.Sync(t.Context(), "svc", []*PathMapping{
err := syncer.Sync(t.Context(), "svc", []*PathMapping{
{HostPath: "/no/such/file", ContainerPath: "/app/gone.txt"},
})
@ -105,9 +128,9 @@ func TestSync_StatPermissionError(t *testing.T) {
client := &fakeLowLevelClient{
containers: []container.Summary{{ID: "ctr1"}},
}
tar := NewTar("proj", client)
syncer := NewTar("proj", client)
err := tar.Sync(t.Context(), "svc", []*PathMapping{
err := syncer.Sync(t.Context(), "svc", []*PathMapping{
{HostPath: targetFile, ContainerPath: "/app/secret.txt"},
})
@ -125,9 +148,9 @@ func TestSync_MixedPaths(t *testing.T) {
client := &fakeLowLevelClient{
containers: []container.Summary{{ID: "ctr1"}},
}
tar := NewTar("proj", client)
syncer := NewTar("proj", client)
err := tar.Sync(t.Context(), "svc", []*PathMapping{
err := syncer.Sync(t.Context(), "svc", []*PathMapping{
{HostPath: existingFile, ContainerPath: "/app/keep.txt"},
{HostPath: "/no/such/path", ContainerPath: "/app/removed.txt"},
})
@ -137,3 +160,105 @@ func TestSync_MixedPaths(t *testing.T) {
assert.Equal(t, len(client.execCmds), 1)
assert.Check(t, cmp.Contains(client.execCmds[0][len(client.execCmds[0])-1], "removed.txt"))
}
// A directory that arrives already populated keeps its own header: left implied, the engine
// creates it with ImpliedDirectoryMode and its own root uid/gid, and a non-root service can no
// longer write into what it synced.
func TestSync_PopulatedDirectoryKeepsItsHeader(t *testing.T) {
tmpDir := populatedSyncDir(t)
client := &fakeLowLevelClient{
containers: []container.Summary{{ID: "ctr1"}},
}
syncer := NewTar("proj", client)
err := syncer.Sync(t.Context(), "svc", []*PathMapping{
{HostPath: filepath.Join(tmpDir, "sub"), ContainerPath: "/app/sub"},
})
assert.NilError(t, err)
assert.Equal(t, client.untarCount, 1)
sent := client.untarHeaders[0]
dir, sentDirHeader := sent["app/sub/"]
assert.Assert(t, sentDirHeader, "the synced directory needs its own header: %v", sent)
hostDir, err := os.Stat(filepath.Join(tmpDir, "sub"))
assert.NilError(t, err)
assert.Equal(t, dir.FileInfo().Mode().Perm(), hostDir.Mode().Perm(), "the header carries the host mode, not the engine's")
assert.Equal(t, sent["app/sub/data.txt"].Typeflag, byte(tar.TypeReg))
}
// A directory header for a path the container resolves through a symlink makes the engine reject
// the whole copy (docker/compose#13795). The retry leaves those directories implied, so the files
// land through the symlink.
func TestSync_RetriesWithoutImpliedDirectoriesWhenCopyFails(t *testing.T) {
tmpDir := populatedSyncDir(t)
client := &fakeLowLevelClient{
containers: []container.Summary{{ID: "ctr1"}},
untarErrs: []error{errors.New(`cannot overwrite non-directory "/app/sub" with directory "/"`)},
}
syncer := NewTar("proj", client)
err := syncer.Sync(t.Context(), "svc", []*PathMapping{
{HostPath: filepath.Join(tmpDir, "sub"), ContainerPath: "/app/sub"},
})
assert.NilError(t, err)
assert.Equal(t, client.untarCount, 2)
retried := client.untarHeaders[1]
_, sentDirHeader := retried["app/sub/"]
assert.Check(t, !sentDirHeader, "no header expected for the directory holding the synced files: %v", retried)
assert.Equal(t, retried["app/sub/data.txt"].Typeflag, byte(tar.TypeReg))
assert.Equal(t, retried["app/sub/nested/"].Typeflag, byte(tar.TypeDir), "an empty directory still needs its own header")
}
func TestSync_ReportsTheCopyErrorWhenTheRetryFailsToo(t *testing.T) {
tmpDir := populatedSyncDir(t)
client := &fakeLowLevelClient{
containers: []container.Summary{{ID: "ctr1"}},
untarErrs: []error{
errors.New(`cannot overwrite non-directory "/app/sub" with directory "/"`),
errors.New("no such container"),
},
}
syncer := NewTar("proj", client)
err := syncer.Sync(t.Context(), "svc", []*PathMapping{
{HostPath: filepath.Join(tmpDir, "sub"), ContainerPath: "/app/sub"},
})
assert.ErrorContains(t, err, "cannot overwrite non-directory")
assert.ErrorContains(t, err, "no such container")
assert.Equal(t, client.untarCount, 2)
}
// Retrying drops the directory headers, so only the failure they cause may be retried.
func TestSync_DoesNotRetryACopyThatFailedForAnotherReason(t *testing.T) {
tmpDir := populatedSyncDir(t)
client := &fakeLowLevelClient{
containers: []container.Summary{{ID: "ctr1"}},
untarErrs: []error{errors.New("no such container")},
}
syncer := NewTar("proj", client)
err := syncer.Sync(t.Context(), "svc", []*PathMapping{
{HostPath: filepath.Join(tmpDir, "sub"), ContainerPath: "/app/sub"},
})
assert.ErrorContains(t, err, "no such container")
assert.Equal(t, client.untarCount, 1)
assert.Equal(t, client.untarHeaders[0]["app/sub/"].Typeflag, byte(tar.TypeDir), "the archive sent must be the unreduced one")
}
// sub/ holds a file and an empty directory, the shapes the archive treats differently.
func populatedSyncDir(t *testing.T) string {
t.Helper()
tmpDir := t.TempDir()
assert.NilError(t, os.MkdirAll(filepath.Join(tmpDir, "sub", "nested"), 0o700))
assert.NilError(t, os.WriteFile(filepath.Join(tmpDir, "sub", "data.txt"), []byte("data"), 0o644))
return tmpDir
}

View file

@ -0,0 +1,11 @@
services:
app:
image: alpine
init: true
# /app/data/sub is a symlink, so synced files have to land in /var/sub through it
command: sh -c 'mkdir -p /app/data /var/sub && ln -s /var/sub /app/data/sub && sleep infinity'
develop:
watch:
- action: sync
path: ./data
target: /app/data

View file

@ -447,6 +447,64 @@ func TestWatchRebuildIgnoresDependencies(t *testing.T) {
c.RunDockerComposeCmdNoCheck(t, "-p", projectName, "kill", "-s", "9")
}
// Reproduces docker/compose#13795: syncing a directory onto a path the image exposes as a
// symlink failed with `cannot overwrite non-directory "/app/data/sub" with directory "/"`.
func TestWatchSyncIntoSymlinkedDirectory(t *testing.T) {
c := NewCLI(t)
const projectName = "test_watch_symlink"
defer c.cleanupWithDown(t, projectName)
tmpdir := t.TempDir()
composeFilePath := filepath.Join(tmpdir, "compose.yaml")
CopyFile(t, filepath.Join("fixtures", "watch", "symlink.yaml"), composeFilePath)
dataDir := filepath.Join(tmpdir, "data")
assert.NilError(t, os.Mkdir(dataDir, 0o700))
// Fill the directory outside the watched tree and move it in, so a single change carries
// the directory together with its content, the way a branch switch does.
staged := filepath.Join(tmpdir, "staged")
assert.NilError(t, os.Mkdir(staged, 0o700))
assert.NilError(t, os.WriteFile(filepath.Join(staged, "hello.txt"), []byte("hello symlink\n"), 0o600))
cmd := c.NewDockerComposeCmd(t, "-p", projectName, "-f", composeFilePath, "up", "--watch")
buffer := bytes.NewBuffer(nil)
cmd.Stdout = buffer
cmd.Stderr = buffer
watch := icmd.StartCmd(cmd)
assert.NilError(t, watch.Error)
t.Cleanup(func() {
if watch.Cmd.Process != nil {
_ = watch.Cmd.Process.Kill()
}
})
poll.WaitOn(t, func(l poll.LogT) poll.Result {
if strings.Contains(buffer.String(), "Watch enabled") {
return poll.Success()
}
return poll.Continue("waiting for watch to start: %v", buffer.String())
}, poll.WithTimeout(120*time.Second))
assert.NilError(t, os.Rename(staged, filepath.Join(dataDir, "sub")))
poll.WaitOn(t, func(l poll.LogT) poll.Result {
// The image resolves /app/data/sub to /var/sub, where the synced file has to show up.
cat := c.RunDockerComposeCmdNoCheck(t, "-p", projectName, "exec", "app", "cat", "/var/sub/hello.txt")
if strings.Contains(cat.Stdout(), "hello symlink") {
return poll.Success()
}
return poll.Continue("%v\n%v", cat.Combined(), buffer.String())
}, poll.WithTimeout(60*time.Second), poll.WithDelay(time.Second))
// the sync must go through the symlink, not replace it
c.RunDockerComposeCmd(t, "-p", projectName, "exec", "app", "test", "-L", "/app/data/sub")
assert.Assert(t, !strings.Contains(buffer.String(), "cannot overwrite non-directory"),
"sync failed on the symlinked path:\n%s", buffer.String())
c.RunDockerComposeCmdNoCheck(t, "-p", projectName, "kill", "-s", "9")
}
func TestWatchIncludes(t *testing.T) {
c := NewCLI(t)
const projectName = "test_watch_includes"