compose/internal/sync/tar_test.go
Guillaume Lours 467fa8ccc9 fix: share --parallel budget across down's ops and up's log-attach
down's outer ops dispatch bounded only how many ops ran concurrently,
separate from the semaphore already shared by image removal -- a
single network plus --rmi could spend up to 2x the configured
budget. Network and volume removal now share that same semaphore.

up's post-attach log streaming called ContainerInspect directly and
passed no limiter to doLogContainer, unlike `compose logs --follow`'s
equivalent path, so it never shared --parallel's budget either.

The plan executor also held a node's slot for its whole dependency
wait instead of just its own work, and internal/sync/tar.go's file
sync used a hardcoded cap of 16 instead of --parallel.

Signed-off-by: Guillaume Lours <glours@users.noreply.github.com>
Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
2026-10-01 16:02:14 +02:00

358 lines
11 KiB
Go

/*
Copyright 2023 Docker Compose CLI authors
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package sync
import (
"archive/tar"
"context"
"errors"
"io"
"os"
"path/filepath"
"runtime"
"sync"
"testing"
"time"
"github.com/moby/moby/api/types/container"
"gotest.tools/v3/assert"
"gotest.tools/v3/assert/cmp"
)
// fakeLowLevelClient records calls made to it for test assertions.
type fakeLowLevelClient struct {
mu sync.Mutex
containers []container.Summary
execCmds [][]string
untarCount int
// untarErrs[i] is what the i-th Untar call returns
untarErrs []error
// untarHeaders[i] holds the headers the i-th Untar call received, by entry name
untarHeaders []map[string]tar.Header
// onUntar, when set, is invoked at the start of every Untar call --
// used to observe how many run concurrently.
onUntar func()
}
func (f *fakeLowLevelClient) ContainersForService(_ context.Context, _ string, _ string) ([]container.Summary, error) {
return f.containers, nil
}
func (f *fakeLowLevelClient) Exec(_ context.Context, _ string, cmd []string, _ io.Reader) error {
f.execCmds = append(f.execCmds, cmd)
return nil
}
func (f *fakeLowLevelClient) Untar(_ context.Context, _ string, reader io.ReadCloser) error {
if f.onUntar != nil {
f.onUntar()
}
headers := map[string]tar.Header{}
tr := tar.NewReader(reader)
for {
header, err := tr.Next()
if errors.Is(err, io.EOF) {
break
}
if err != nil {
return err
}
headers[header.Name] = *header
}
f.mu.Lock()
defer f.mu.Unlock()
f.untarHeaders = append(f.untarHeaders, headers)
f.untarCount++
if f.untarCount <= len(f.untarErrs) {
return f.untarErrs[f.untarCount-1]
}
return nil
}
func TestSync_ExistingPath(t *testing.T) {
tmpDir := t.TempDir()
existingFile := filepath.Join(tmpDir, "exists.txt")
assert.NilError(t, os.WriteFile(existingFile, []byte("data"), 0o644))
client := &fakeLowLevelClient{
containers: []container.Summary{{ID: "ctr1"}},
}
syncer := NewTar("proj", client, 0)
err := syncer.Sync(t.Context(), "svc", []*PathMapping{
{HostPath: existingFile, ContainerPath: "/app/exists.txt"},
})
assert.NilError(t, err)
assert.Equal(t, client.untarCount, 1, "existing path should be copied via Untar")
assert.Equal(t, len(client.execCmds), 0, "no delete command expected for existing path")
}
func TestSync_NonExistentPath(t *testing.T) {
client := &fakeLowLevelClient{
containers: []container.Summary{{ID: "ctr1"}},
}
syncer := NewTar("proj", client, 0)
err := syncer.Sync(t.Context(), "svc", []*PathMapping{
{HostPath: "/no/such/file", ContainerPath: "/app/gone.txt"},
})
assert.NilError(t, err)
assert.Equal(t, len(client.execCmds), 1, "should issue a delete command")
assert.DeepEqual(t, client.execCmds[0], []string{"rm", "-rf", "/app/gone.txt"})
}
func TestSync_StatPermissionError(t *testing.T) {
if runtime.GOOS == "windows" {
t.Skip("permission-based test not reliable on Windows")
}
if os.Getuid() == 0 {
t.Skip("test requires non-root to trigger EACCES")
}
tmpDir := t.TempDir()
restrictedDir := filepath.Join(tmpDir, "noaccess")
assert.NilError(t, os.Mkdir(restrictedDir, 0o700))
targetFile := filepath.Join(restrictedDir, "secret.txt")
assert.NilError(t, os.WriteFile(targetFile, []byte("data"), 0o644))
// Remove all permissions on the parent directory so stat on the child fails with EACCES.
assert.NilError(t, os.Chmod(restrictedDir, 0o000))
t.Cleanup(func() {
// Restore permissions so t.TempDir() cleanup can remove it.
_ = os.Chmod(restrictedDir, 0o700)
})
client := &fakeLowLevelClient{
containers: []container.Summary{{ID: "ctr1"}},
}
syncer := NewTar("proj", client, 0)
err := syncer.Sync(t.Context(), "svc", []*PathMapping{
{HostPath: targetFile, ContainerPath: "/app/secret.txt"},
})
assert.ErrorContains(t, err, "permission denied")
assert.ErrorContains(t, err, "secret.txt")
assert.Equal(t, client.untarCount, 0, "should not attempt copy on stat error")
assert.Equal(t, len(client.execCmds), 0, "should not attempt delete on stat error")
}
// syncConcurrencyTracker records the highest number of overlapping
// enter()/leave() pairs seen, used to assert Sync's concurrency bound.
type syncConcurrencyTracker struct {
mu sync.Mutex
current int
peak int
}
func (t *syncConcurrencyTracker) enter() {
t.mu.Lock()
t.current++
if t.current > t.peak {
t.peak = t.current
}
t.mu.Unlock()
}
func (t *syncConcurrencyTracker) leave() {
t.mu.Lock()
t.current--
t.mu.Unlock()
}
func (t *syncConcurrencyTracker) Peak() int {
t.mu.Lock()
defer t.mu.Unlock()
return t.peak
}
// TestSync_ConcurrencyIsBounded guards internal/sync/tar.go's own fan-out:
// it used to hardcode eg.SetLimit(16) regardless of --parallel, the same
// class of gap fixed everywhere else in this repo for bulk engine calls.
func TestSync_ConcurrencyIsBounded(t *testing.T) {
tracker := &syncConcurrencyTracker{}
client := &fakeLowLevelClient{
containers: []container.Summary{{ID: "ctr1"}, {ID: "ctr2"}, {ID: "ctr3"}},
onUntar: func() {
tracker.enter()
time.Sleep(20 * time.Millisecond) // widen the window for a concurrency violation to show up
tracker.leave()
},
}
syncer := NewTar("proj", client, 1)
tmpDir := t.TempDir()
existingFile := filepath.Join(tmpDir, "exists.txt")
assert.NilError(t, os.WriteFile(existingFile, []byte("data"), 0o644))
err := syncer.Sync(t.Context(), "svc", []*PathMapping{
{HostPath: existingFile, ContainerPath: "/app/exists.txt"},
})
assert.NilError(t, err)
assert.Equal(t, tracker.Peak(), 1, "Sync must never run more than maxConcurrency Untar calls at once")
}
// TestSync_ZeroMaxConcurrencyIsUnlimited guards the errgroup.SetLimit(0)
// footgun: 0 is maxConcurrency's Go zero-value, and SetLimit(0) means "allow
// zero goroutines", not "unlimited" -- which would deadlock Sync forever.
func TestSync_ZeroMaxConcurrencyIsUnlimited(t *testing.T) {
client := &fakeLowLevelClient{
containers: []container.Summary{{ID: "ctr1"}, {ID: "ctr2"}},
}
syncer := NewTar("proj", client, 0)
tmpDir := t.TempDir()
existingFile := filepath.Join(tmpDir, "exists.txt")
assert.NilError(t, os.WriteFile(existingFile, []byte("data"), 0o644))
done := make(chan error, 1)
go func() {
done <- syncer.Sync(t.Context(), "svc", []*PathMapping{
{HostPath: existingFile, ContainerPath: "/app/exists.txt"},
})
}()
select {
case err := <-done:
assert.NilError(t, err)
case <-time.After(2 * time.Second):
t.Fatal("Sync deadlocked with maxConcurrency=0")
}
}
func TestSync_MixedPaths(t *testing.T) {
tmpDir := t.TempDir()
existingFile := filepath.Join(tmpDir, "keep.txt")
assert.NilError(t, os.WriteFile(existingFile, []byte("data"), 0o644))
client := &fakeLowLevelClient{
containers: []container.Summary{{ID: "ctr1"}},
}
syncer := NewTar("proj", client, 0)
err := syncer.Sync(t.Context(), "svc", []*PathMapping{
{HostPath: existingFile, ContainerPath: "/app/keep.txt"},
{HostPath: "/no/such/path", ContainerPath: "/app/removed.txt"},
})
assert.NilError(t, err)
assert.Equal(t, client.untarCount, 1, "existing path should be copied")
assert.Equal(t, len(client.execCmds), 1)
assert.Check(t, cmp.Contains(client.execCmds[0][len(client.execCmds[0])-1], "removed.txt"))
}
// A directory that arrives already populated keeps its own header: left implied, the engine
// creates it with ImpliedDirectoryMode and its own root uid/gid, and a non-root service can no
// longer write into what it synced.
func TestSync_PopulatedDirectoryKeepsItsHeader(t *testing.T) {
tmpDir := populatedSyncDir(t)
client := &fakeLowLevelClient{
containers: []container.Summary{{ID: "ctr1"}},
}
syncer := NewTar("proj", client, 0)
err := syncer.Sync(t.Context(), "svc", []*PathMapping{
{HostPath: filepath.Join(tmpDir, "sub"), ContainerPath: "/app/sub"},
})
assert.NilError(t, err)
assert.Equal(t, client.untarCount, 1)
sent := client.untarHeaders[0]
dir, sentDirHeader := sent["app/sub/"]
assert.Assert(t, sentDirHeader, "the synced directory needs its own header: %v", sent)
hostDir, err := os.Stat(filepath.Join(tmpDir, "sub"))
assert.NilError(t, err)
assert.Equal(t, dir.FileInfo().Mode().Perm(), hostDir.Mode().Perm(), "the header carries the host mode, not the engine's")
assert.Equal(t, sent["app/sub/data.txt"].Typeflag, byte(tar.TypeReg))
}
// A directory header for a path the container resolves through a symlink makes the engine reject
// the whole copy (docker/compose#13795). The retry leaves those directories implied, so the files
// land through the symlink.
func TestSync_RetriesWithoutImpliedDirectoriesWhenCopyFails(t *testing.T) {
tmpDir := populatedSyncDir(t)
client := &fakeLowLevelClient{
containers: []container.Summary{{ID: "ctr1"}},
untarErrs: []error{errors.New(`cannot overwrite non-directory "/app/sub" with directory "/"`)},
}
syncer := NewTar("proj", client, 0)
err := syncer.Sync(t.Context(), "svc", []*PathMapping{
{HostPath: filepath.Join(tmpDir, "sub"), ContainerPath: "/app/sub"},
})
assert.NilError(t, err)
assert.Equal(t, client.untarCount, 2)
retried := client.untarHeaders[1]
_, sentDirHeader := retried["app/sub/"]
assert.Check(t, !sentDirHeader, "no header expected for the directory holding the synced files: %v", retried)
assert.Equal(t, retried["app/sub/data.txt"].Typeflag, byte(tar.TypeReg))
assert.Equal(t, retried["app/sub/nested/"].Typeflag, byte(tar.TypeDir), "an empty directory still needs its own header")
}
func TestSync_ReportsTheCopyErrorWhenTheRetryFailsToo(t *testing.T) {
tmpDir := populatedSyncDir(t)
client := &fakeLowLevelClient{
containers: []container.Summary{{ID: "ctr1"}},
untarErrs: []error{
errors.New(`cannot overwrite non-directory "/app/sub" with directory "/"`),
errors.New("no such container"),
},
}
syncer := NewTar("proj", client, 0)
err := syncer.Sync(t.Context(), "svc", []*PathMapping{
{HostPath: filepath.Join(tmpDir, "sub"), ContainerPath: "/app/sub"},
})
assert.ErrorContains(t, err, "cannot overwrite non-directory")
assert.ErrorContains(t, err, "no such container")
assert.Equal(t, client.untarCount, 2)
}
// Retrying drops the directory headers, so only the failure they cause may be retried.
func TestSync_DoesNotRetryACopyThatFailedForAnotherReason(t *testing.T) {
tmpDir := populatedSyncDir(t)
client := &fakeLowLevelClient{
containers: []container.Summary{{ID: "ctr1"}},
untarErrs: []error{errors.New("no such container")},
}
syncer := NewTar("proj", client, 0)
err := syncer.Sync(t.Context(), "svc", []*PathMapping{
{HostPath: filepath.Join(tmpDir, "sub"), ContainerPath: "/app/sub"},
})
assert.ErrorContains(t, err, "no such container")
assert.Equal(t, client.untarCount, 1)
assert.Equal(t, client.untarHeaders[0]["app/sub/"].Typeflag, byte(tar.TypeDir), "the archive sent must be the unreduced one")
}
// sub/ holds a file and an empty directory, the shapes the archive treats differently.
func populatedSyncDir(t *testing.T) string {
t.Helper()
tmpDir := t.TempDir()
assert.NilError(t, os.MkdirAll(filepath.Join(tmpDir, "sub", "nested"), 0o700))
assert.NilError(t, os.WriteFile(filepath.Join(tmpDir, "sub", "data.txt"), []byte("data"), 0o644))
return tmpDir
}