mirror of
https://github.com/docker/compose.git
synced 2026-11-19 22:50:08 +00:00
fix(watch): sync into a symlinked directory instead of failing
Signed-off-by: Endika Iglesias <endika2@gmail.com>
This commit is contained in:
parent
115898dc13
commit
f519180b95
4 changed files with 256 additions and 16 deletions
|
|
@ -96,7 +96,7 @@ func (t *Tar) Sync(ctx context.Context, service string, paths []*PathMapping) er
|
|||
eg.SetLimit(16) // arbitrary limit, adjust to taste :D
|
||||
for i := range containers {
|
||||
containerID := containers[i].ID
|
||||
tarReader := tarArchive(pathsToCopy)
|
||||
tarReader := tarArchive(pathsToCopy, keepImpliedDirectories)
|
||||
|
||||
eg.Go(func() error {
|
||||
if len(deleteCmd) != 0 {
|
||||
|
|
@ -108,9 +108,24 @@ func (t *Tar) Sync(ctx context.Context, service string, paths []*PathMapping) er
|
|||
}
|
||||
|
||||
if err := t.client.Untar(ctx, containerID, tarReader); err != nil {
|
||||
errMu.Lock()
|
||||
errs = append(errs, fmt.Errorf("copying files to %s: %w", containerID, err))
|
||||
errMu.Unlock()
|
||||
// The engine answers with a plain 500, so the message is the only discriminator.
|
||||
if !strings.Contains(err.Error(), "cannot overwrite non-directory") {
|
||||
errMu.Lock()
|
||||
errs = append(errs, fmt.Errorf("copying files to %s: %w", containerID, err))
|
||||
errMu.Unlock()
|
||||
return nil
|
||||
}
|
||||
|
||||
// A directory header for a path the container resolves through a symlink makes
|
||||
// the engine reject the whole copy, so retry without the headers the archive's
|
||||
// own entries already imply. That archive is a strict subset of the one that
|
||||
// just failed, so sending it again is safe.
|
||||
retry := tarArchive(pathsToCopy, dropImpliedDirectories)
|
||||
if retryErr := t.client.Untar(ctx, containerID, retry); retryErr != nil {
|
||||
errMu.Lock()
|
||||
errs = append(errs, fmt.Errorf("copying files to %s: %w", containerID, errors.Join(err, retryErr)))
|
||||
errMu.Unlock()
|
||||
}
|
||||
}
|
||||
return nil // don't fail-fast; collect all errors
|
||||
})
|
||||
|
|
@ -139,7 +154,7 @@ func (a *ArchiveBuilder) Close() error {
|
|||
}
|
||||
|
||||
// ArchivePathsIfExist creates a tar archive of all local files in `paths`. It quietly skips any paths that don't exist.
|
||||
func (a *ArchiveBuilder) ArchivePathsIfExist(paths []PathMapping) error {
|
||||
func (a *ArchiveBuilder) ArchivePathsIfExist(paths []PathMapping, dropImpliedDirs bool) error {
|
||||
// In order to handle overlapping syncs, we
|
||||
// 1) collect all the entries,
|
||||
// 2) de-dupe them, with last-one-wins semantics
|
||||
|
|
@ -161,6 +176,9 @@ func (a *ArchiveBuilder) ArchivePathsIfExist(paths []PathMapping) error {
|
|||
}
|
||||
|
||||
entries = dedupeEntries(entries)
|
||||
if dropImpliedDirs {
|
||||
entries = withoutImpliedDirectories(entries)
|
||||
}
|
||||
for _, entry := range entries {
|
||||
err := a.writeEntry(entry)
|
||||
if err != nil {
|
||||
|
|
@ -318,11 +336,17 @@ func (a *ArchiveBuilder) entriesForPath(localPath, containerPath string) ([]arch
|
|||
return result, nil
|
||||
}
|
||||
|
||||
func tarArchive(ops []PathMapping) io.ReadCloser {
|
||||
// Whether tarArchive keeps the header of a directory the archive itself fills with entries.
|
||||
const (
|
||||
keepImpliedDirectories = false
|
||||
dropImpliedDirectories = true
|
||||
)
|
||||
|
||||
func tarArchive(ops []PathMapping, dropImpliedDirs bool) io.ReadCloser {
|
||||
pr, pw := io.Pipe()
|
||||
go func() {
|
||||
ab := NewArchiveBuilder(pw)
|
||||
err := ab.ArchivePathsIfExist(ops)
|
||||
err := ab.ArchivePathsIfExist(ops, dropImpliedDirs)
|
||||
if err != nil {
|
||||
_ = pw.CloseWithError(fmt.Errorf("adding files to tar: %w", err))
|
||||
} else {
|
||||
|
|
@ -338,6 +362,28 @@ func tarArchive(ops []PathMapping) io.ReadCloser {
|
|||
return pr
|
||||
}
|
||||
|
||||
// withoutImpliedDirectories removes the header of every directory the archive already fills with
|
||||
// entries: the engine creates those while extracting, while an explicit header makes it reject the
|
||||
// whole copy when the container resolves that path through a symlink. A directory nothing else
|
||||
// would create - an empty one - keeps its header.
|
||||
//
|
||||
// The engine creates them with ImpliedDirectoryMode and its own root uid/gid rather than the host's,
|
||||
// so this is a fallback for a copy that already failed, never the first attempt.
|
||||
func withoutImpliedDirectories(entries []archiveEntry) []archiveEntry {
|
||||
holdsEntries := make(map[string]bool, len(entries))
|
||||
for _, entry := range entries {
|
||||
holdsEntries[path.Dir(path.Clean(entry.header.Name))] = true
|
||||
}
|
||||
result := make([]archiveEntry, 0, len(entries))
|
||||
for _, entry := range entries {
|
||||
if entry.header.Typeflag == tar.TypeDir && holdsEntries[path.Clean(entry.header.Name)] {
|
||||
continue
|
||||
}
|
||||
result = append(result, entry)
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
// Dedupe the entries with last-entry-wins semantics.
|
||||
func dedupeEntries(entries []archiveEntry) []archiveEntry {
|
||||
seenIndex := make(map[string]int, len(entries))
|
||||
|
|
|
|||
|
|
@ -15,7 +15,9 @@
|
|||
package sync
|
||||
|
||||
import (
|
||||
"archive/tar"
|
||||
"context"
|
||||
"errors"
|
||||
"io"
|
||||
"os"
|
||||
"path/filepath"
|
||||
|
|
@ -32,6 +34,10 @@ type fakeLowLevelClient struct {
|
|||
containers []container.Summary
|
||||
execCmds [][]string
|
||||
untarCount int
|
||||
// untarErrs[i] is what the i-th Untar call returns
|
||||
untarErrs []error
|
||||
// untarHeaders[i] holds the headers the i-th Untar call received, by entry name
|
||||
untarHeaders []map[string]tar.Header
|
||||
}
|
||||
|
||||
func (f *fakeLowLevelClient) ContainersForService(_ context.Context, _ string, _ string) ([]container.Summary, error) {
|
||||
|
|
@ -43,8 +49,25 @@ func (f *fakeLowLevelClient) Exec(_ context.Context, _ string, cmd []string, _ i
|
|||
return nil
|
||||
}
|
||||
|
||||
func (f *fakeLowLevelClient) Untar(_ context.Context, _ string, _ io.ReadCloser) error {
|
||||
func (f *fakeLowLevelClient) Untar(_ context.Context, _ string, reader io.ReadCloser) error {
|
||||
headers := map[string]tar.Header{}
|
||||
tr := tar.NewReader(reader)
|
||||
for {
|
||||
header, err := tr.Next()
|
||||
if errors.Is(err, io.EOF) {
|
||||
break
|
||||
}
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
headers[header.Name] = *header
|
||||
}
|
||||
f.untarHeaders = append(f.untarHeaders, headers)
|
||||
|
||||
f.untarCount++
|
||||
if f.untarCount <= len(f.untarErrs) {
|
||||
return f.untarErrs[f.untarCount-1]
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
|
|
@ -56,9 +79,9 @@ func TestSync_ExistingPath(t *testing.T) {
|
|||
client := &fakeLowLevelClient{
|
||||
containers: []container.Summary{{ID: "ctr1"}},
|
||||
}
|
||||
tar := NewTar("proj", client)
|
||||
syncer := NewTar("proj", client)
|
||||
|
||||
err := tar.Sync(t.Context(), "svc", []*PathMapping{
|
||||
err := syncer.Sync(t.Context(), "svc", []*PathMapping{
|
||||
{HostPath: existingFile, ContainerPath: "/app/exists.txt"},
|
||||
})
|
||||
|
||||
|
|
@ -71,9 +94,9 @@ func TestSync_NonExistentPath(t *testing.T) {
|
|||
client := &fakeLowLevelClient{
|
||||
containers: []container.Summary{{ID: "ctr1"}},
|
||||
}
|
||||
tar := NewTar("proj", client)
|
||||
syncer := NewTar("proj", client)
|
||||
|
||||
err := tar.Sync(t.Context(), "svc", []*PathMapping{
|
||||
err := syncer.Sync(t.Context(), "svc", []*PathMapping{
|
||||
{HostPath: "/no/such/file", ContainerPath: "/app/gone.txt"},
|
||||
})
|
||||
|
||||
|
|
@ -105,9 +128,9 @@ func TestSync_StatPermissionError(t *testing.T) {
|
|||
client := &fakeLowLevelClient{
|
||||
containers: []container.Summary{{ID: "ctr1"}},
|
||||
}
|
||||
tar := NewTar("proj", client)
|
||||
syncer := NewTar("proj", client)
|
||||
|
||||
err := tar.Sync(t.Context(), "svc", []*PathMapping{
|
||||
err := syncer.Sync(t.Context(), "svc", []*PathMapping{
|
||||
{HostPath: targetFile, ContainerPath: "/app/secret.txt"},
|
||||
})
|
||||
|
||||
|
|
@ -125,9 +148,9 @@ func TestSync_MixedPaths(t *testing.T) {
|
|||
client := &fakeLowLevelClient{
|
||||
containers: []container.Summary{{ID: "ctr1"}},
|
||||
}
|
||||
tar := NewTar("proj", client)
|
||||
syncer := NewTar("proj", client)
|
||||
|
||||
err := tar.Sync(t.Context(), "svc", []*PathMapping{
|
||||
err := syncer.Sync(t.Context(), "svc", []*PathMapping{
|
||||
{HostPath: existingFile, ContainerPath: "/app/keep.txt"},
|
||||
{HostPath: "/no/such/path", ContainerPath: "/app/removed.txt"},
|
||||
})
|
||||
|
|
@ -137,3 +160,105 @@ func TestSync_MixedPaths(t *testing.T) {
|
|||
assert.Equal(t, len(client.execCmds), 1)
|
||||
assert.Check(t, cmp.Contains(client.execCmds[0][len(client.execCmds[0])-1], "removed.txt"))
|
||||
}
|
||||
|
||||
// A directory that arrives already populated keeps its own header: left implied, the engine
|
||||
// creates it with ImpliedDirectoryMode and its own root uid/gid, and a non-root service can no
|
||||
// longer write into what it synced.
|
||||
func TestSync_PopulatedDirectoryKeepsItsHeader(t *testing.T) {
|
||||
tmpDir := populatedSyncDir(t)
|
||||
|
||||
client := &fakeLowLevelClient{
|
||||
containers: []container.Summary{{ID: "ctr1"}},
|
||||
}
|
||||
syncer := NewTar("proj", client)
|
||||
|
||||
err := syncer.Sync(t.Context(), "svc", []*PathMapping{
|
||||
{HostPath: filepath.Join(tmpDir, "sub"), ContainerPath: "/app/sub"},
|
||||
})
|
||||
|
||||
assert.NilError(t, err)
|
||||
assert.Equal(t, client.untarCount, 1)
|
||||
|
||||
sent := client.untarHeaders[0]
|
||||
dir, sentDirHeader := sent["app/sub/"]
|
||||
assert.Assert(t, sentDirHeader, "the synced directory needs its own header: %v", sent)
|
||||
hostDir, err := os.Stat(filepath.Join(tmpDir, "sub"))
|
||||
assert.NilError(t, err)
|
||||
assert.Equal(t, dir.FileInfo().Mode().Perm(), hostDir.Mode().Perm(), "the header carries the host mode, not the engine's")
|
||||
assert.Equal(t, sent["app/sub/data.txt"].Typeflag, byte(tar.TypeReg))
|
||||
}
|
||||
|
||||
// A directory header for a path the container resolves through a symlink makes the engine reject
|
||||
// the whole copy (docker/compose#13795). The retry leaves those directories implied, so the files
|
||||
// land through the symlink.
|
||||
func TestSync_RetriesWithoutImpliedDirectoriesWhenCopyFails(t *testing.T) {
|
||||
tmpDir := populatedSyncDir(t)
|
||||
|
||||
client := &fakeLowLevelClient{
|
||||
containers: []container.Summary{{ID: "ctr1"}},
|
||||
untarErrs: []error{errors.New(`cannot overwrite non-directory "/app/sub" with directory "/"`)},
|
||||
}
|
||||
syncer := NewTar("proj", client)
|
||||
|
||||
err := syncer.Sync(t.Context(), "svc", []*PathMapping{
|
||||
{HostPath: filepath.Join(tmpDir, "sub"), ContainerPath: "/app/sub"},
|
||||
})
|
||||
|
||||
assert.NilError(t, err)
|
||||
assert.Equal(t, client.untarCount, 2)
|
||||
|
||||
retried := client.untarHeaders[1]
|
||||
_, sentDirHeader := retried["app/sub/"]
|
||||
assert.Check(t, !sentDirHeader, "no header expected for the directory holding the synced files: %v", retried)
|
||||
assert.Equal(t, retried["app/sub/data.txt"].Typeflag, byte(tar.TypeReg))
|
||||
assert.Equal(t, retried["app/sub/nested/"].Typeflag, byte(tar.TypeDir), "an empty directory still needs its own header")
|
||||
}
|
||||
|
||||
func TestSync_ReportsTheCopyErrorWhenTheRetryFailsToo(t *testing.T) {
|
||||
tmpDir := populatedSyncDir(t)
|
||||
|
||||
client := &fakeLowLevelClient{
|
||||
containers: []container.Summary{{ID: "ctr1"}},
|
||||
untarErrs: []error{
|
||||
errors.New(`cannot overwrite non-directory "/app/sub" with directory "/"`),
|
||||
errors.New("no such container"),
|
||||
},
|
||||
}
|
||||
syncer := NewTar("proj", client)
|
||||
|
||||
err := syncer.Sync(t.Context(), "svc", []*PathMapping{
|
||||
{HostPath: filepath.Join(tmpDir, "sub"), ContainerPath: "/app/sub"},
|
||||
})
|
||||
|
||||
assert.ErrorContains(t, err, "cannot overwrite non-directory")
|
||||
assert.ErrorContains(t, err, "no such container")
|
||||
assert.Equal(t, client.untarCount, 2)
|
||||
}
|
||||
|
||||
// Retrying drops the directory headers, so only the failure they cause may be retried.
|
||||
func TestSync_DoesNotRetryACopyThatFailedForAnotherReason(t *testing.T) {
|
||||
tmpDir := populatedSyncDir(t)
|
||||
|
||||
client := &fakeLowLevelClient{
|
||||
containers: []container.Summary{{ID: "ctr1"}},
|
||||
untarErrs: []error{errors.New("no such container")},
|
||||
}
|
||||
syncer := NewTar("proj", client)
|
||||
|
||||
err := syncer.Sync(t.Context(), "svc", []*PathMapping{
|
||||
{HostPath: filepath.Join(tmpDir, "sub"), ContainerPath: "/app/sub"},
|
||||
})
|
||||
|
||||
assert.ErrorContains(t, err, "no such container")
|
||||
assert.Equal(t, client.untarCount, 1)
|
||||
assert.Equal(t, client.untarHeaders[0]["app/sub/"].Typeflag, byte(tar.TypeDir), "the archive sent must be the unreduced one")
|
||||
}
|
||||
|
||||
// sub/ holds a file and an empty directory, the shapes the archive treats differently.
|
||||
func populatedSyncDir(t *testing.T) string {
|
||||
t.Helper()
|
||||
tmpDir := t.TempDir()
|
||||
assert.NilError(t, os.MkdirAll(filepath.Join(tmpDir, "sub", "nested"), 0o700))
|
||||
assert.NilError(t, os.WriteFile(filepath.Join(tmpDir, "sub", "data.txt"), []byte("data"), 0o644))
|
||||
return tmpDir
|
||||
}
|
||||
|
|
|
|||
11
pkg/e2e/fixtures/watch/symlink.yaml
Normal file
11
pkg/e2e/fixtures/watch/symlink.yaml
Normal file
|
|
@ -0,0 +1,11 @@
|
|||
services:
|
||||
app:
|
||||
image: alpine
|
||||
init: true
|
||||
# /app/data/sub is a symlink, so synced files have to land in /var/sub through it
|
||||
command: sh -c 'mkdir -p /app/data /var/sub && ln -s /var/sub /app/data/sub && sleep infinity'
|
||||
develop:
|
||||
watch:
|
||||
- action: sync
|
||||
path: ./data
|
||||
target: /app/data
|
||||
|
|
@ -447,6 +447,64 @@ func TestWatchRebuildIgnoresDependencies(t *testing.T) {
|
|||
c.RunDockerComposeCmdNoCheck(t, "-p", projectName, "kill", "-s", "9")
|
||||
}
|
||||
|
||||
// Reproduces docker/compose#13795: syncing a directory onto a path the image exposes as a
|
||||
// symlink failed with `cannot overwrite non-directory "/app/data/sub" with directory "/"`.
|
||||
func TestWatchSyncIntoSymlinkedDirectory(t *testing.T) {
|
||||
c := NewCLI(t)
|
||||
const projectName = "test_watch_symlink"
|
||||
|
||||
defer c.cleanupWithDown(t, projectName)
|
||||
|
||||
tmpdir := t.TempDir()
|
||||
composeFilePath := filepath.Join(tmpdir, "compose.yaml")
|
||||
CopyFile(t, filepath.Join("fixtures", "watch", "symlink.yaml"), composeFilePath)
|
||||
dataDir := filepath.Join(tmpdir, "data")
|
||||
assert.NilError(t, os.Mkdir(dataDir, 0o700))
|
||||
|
||||
// Fill the directory outside the watched tree and move it in, so a single change carries
|
||||
// the directory together with its content, the way a branch switch does.
|
||||
staged := filepath.Join(tmpdir, "staged")
|
||||
assert.NilError(t, os.Mkdir(staged, 0o700))
|
||||
assert.NilError(t, os.WriteFile(filepath.Join(staged, "hello.txt"), []byte("hello symlink\n"), 0o600))
|
||||
|
||||
cmd := c.NewDockerComposeCmd(t, "-p", projectName, "-f", composeFilePath, "up", "--watch")
|
||||
buffer := bytes.NewBuffer(nil)
|
||||
cmd.Stdout = buffer
|
||||
cmd.Stderr = buffer
|
||||
watch := icmd.StartCmd(cmd)
|
||||
assert.NilError(t, watch.Error)
|
||||
t.Cleanup(func() {
|
||||
if watch.Cmd.Process != nil {
|
||||
_ = watch.Cmd.Process.Kill()
|
||||
}
|
||||
})
|
||||
|
||||
poll.WaitOn(t, func(l poll.LogT) poll.Result {
|
||||
if strings.Contains(buffer.String(), "Watch enabled") {
|
||||
return poll.Success()
|
||||
}
|
||||
return poll.Continue("waiting for watch to start: %v", buffer.String())
|
||||
}, poll.WithTimeout(120*time.Second))
|
||||
|
||||
assert.NilError(t, os.Rename(staged, filepath.Join(dataDir, "sub")))
|
||||
|
||||
poll.WaitOn(t, func(l poll.LogT) poll.Result {
|
||||
// The image resolves /app/data/sub to /var/sub, where the synced file has to show up.
|
||||
cat := c.RunDockerComposeCmdNoCheck(t, "-p", projectName, "exec", "app", "cat", "/var/sub/hello.txt")
|
||||
if strings.Contains(cat.Stdout(), "hello symlink") {
|
||||
return poll.Success()
|
||||
}
|
||||
return poll.Continue("%v\n%v", cat.Combined(), buffer.String())
|
||||
}, poll.WithTimeout(60*time.Second), poll.WithDelay(time.Second))
|
||||
|
||||
// the sync must go through the symlink, not replace it
|
||||
c.RunDockerComposeCmd(t, "-p", projectName, "exec", "app", "test", "-L", "/app/data/sub")
|
||||
assert.Assert(t, !strings.Contains(buffer.String(), "cannot overwrite non-directory"),
|
||||
"sync failed on the symlinked path:\n%s", buffer.String())
|
||||
|
||||
c.RunDockerComposeCmdNoCheck(t, "-p", projectName, "kill", "-s", "9")
|
||||
}
|
||||
|
||||
func TestWatchIncludes(t *testing.T) {
|
||||
c := NewCLI(t)
|
||||
const projectName = "test_watch_includes"
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue