mirror of
https://github.com/docker/compose.git
synced 2026-11-19 22:50:08 +00:00
fix: bound compose cp's fan-out and panic-safe log slot release
`docker compose cp` fanned out to every matched container through a bare unbounded errgroup, ignoring --parallel entirely -- the same class of gap already closed everywhere else in this package. Routed it through the existing forEachContainerConcurrent helper instead of a bespoke fan-out. inspectWithSlot and doLogContainer both release a semaphore slot early via an explicit call rather than a defer, so a panic between acquire and that release leaked the slot forever. Guarded both with a panic-safe release, scoped tightly in doLogContainer so a later panic in the same function can't trigger a double release. Signed-off-by: Guillaume Lours <glours@users.noreply.github.com> Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
This commit is contained in:
parent
595a600e37
commit
fc92bd93c4
8 changed files with 238 additions and 53 deletions
|
|
@ -200,6 +200,18 @@ func releaseSlot(limiter *semaphore.Weighted) {
|
|||
}
|
||||
}
|
||||
|
||||
// panicSafeReleaseSlot is deferred by callers that release a slot earlier
|
||||
// than function exit on the success path (see inspectWithSlot and
|
||||
// doLogContainer) to avoid leaking it if the guarded call panics before
|
||||
// reaching that point. It is a no-op unless the deferring goroutine is
|
||||
// unwinding from a panic, in which case it releases the slot and re-panics.
|
||||
func panicSafeReleaseSlot(limiter *semaphore.Weighted) {
|
||||
if p := recover(); p != nil {
|
||||
releaseSlot(limiter)
|
||||
panic(p)
|
||||
}
|
||||
}
|
||||
|
||||
// forEachContainerWithLimiter runs fn concurrently for each container,
|
||||
// bounded by limiter. Unlike newLimitedErrgroup, limiter is built by the
|
||||
// caller and can be shared across several concurrently-dispatched calls
|
||||
|
|
|
|||
|
|
@ -29,7 +29,6 @@ import (
|
|||
"github.com/moby/go-archive"
|
||||
"github.com/moby/moby/api/types/container"
|
||||
"github.com/moby/moby/client"
|
||||
"golang.org/x/sync/errgroup"
|
||||
|
||||
"github.com/docker/compose/v5/pkg/api"
|
||||
)
|
||||
|
|
@ -79,37 +78,31 @@ func (s *composeService) copy(ctx context.Context, projectName string, options a
|
|||
return err
|
||||
}
|
||||
|
||||
g := errgroup.Group{}
|
||||
for _, cont := range containers {
|
||||
ctr := cont
|
||||
g.Go(func() error {
|
||||
name := getCanonicalContainerName(ctr)
|
||||
var msg string
|
||||
if direction == fromService {
|
||||
msg = fmt.Sprintf("%s:%s to %s", name, srcPath, dstPath)
|
||||
} else {
|
||||
msg = fmt.Sprintf("%s to %s:%s", srcPath, name, dstPath)
|
||||
}
|
||||
s.events.On(api.Resource{
|
||||
ID: name,
|
||||
Text: api.StatusCopying,
|
||||
Details: msg,
|
||||
Status: api.Working,
|
||||
})
|
||||
if err := copyFunc(ctx, ctr.ID, srcPath, dstPath, options); err != nil {
|
||||
return err
|
||||
}
|
||||
s.events.On(api.Resource{
|
||||
ID: name,
|
||||
Text: api.StatusCopied,
|
||||
Details: msg,
|
||||
Status: api.Done,
|
||||
})
|
||||
return nil
|
||||
return forEachContainerConcurrent(ctx, s.maxConcurrency, containers, func(ctx context.Context, ctr container.Summary) error {
|
||||
name := getCanonicalContainerName(ctr)
|
||||
var msg string
|
||||
if direction == fromService {
|
||||
msg = fmt.Sprintf("%s:%s to %s", name, srcPath, dstPath)
|
||||
} else {
|
||||
msg = fmt.Sprintf("%s to %s:%s", srcPath, name, dstPath)
|
||||
}
|
||||
s.events.On(api.Resource{
|
||||
ID: name,
|
||||
Text: api.StatusCopying,
|
||||
Details: msg,
|
||||
Status: api.Working,
|
||||
})
|
||||
}
|
||||
|
||||
return g.Wait()
|
||||
if err := copyFunc(ctx, ctr.ID, srcPath, dstPath, options); err != nil {
|
||||
return err
|
||||
}
|
||||
s.events.On(api.Resource{
|
||||
ID: name,
|
||||
Text: api.StatusCopied,
|
||||
Details: msg,
|
||||
Status: api.Done,
|
||||
})
|
||||
return nil
|
||||
})
|
||||
}
|
||||
|
||||
func (s *composeService) listContainersTargetedForCopy(ctx context.Context, projectName string, options api.CopyOptions, direction copyDirection, serviceName string) (Containers, error) {
|
||||
|
|
|
|||
75
pkg/compose/cp_test.go
Normal file
75
pkg/compose/cp_test.go
Normal file
|
|
@ -0,0 +1,75 @@
|
|||
/*
|
||||
Copyright 2020 Docker Compose CLI authors
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
package compose
|
||||
|
||||
import (
|
||||
"context"
|
||||
"io"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/moby/moby/api/types/container"
|
||||
"github.com/moby/moby/client"
|
||||
"go.uber.org/mock/gomock"
|
||||
"gotest.tools/v3/assert"
|
||||
|
||||
compose "github.com/docker/compose/v5/pkg/api"
|
||||
)
|
||||
|
||||
// TestCopyToService_ConcurrencyIsBoundedAcrossContainers guards against the
|
||||
// same unbounded-fan-out bug fixed everywhere else in this package: `compose
|
||||
// cp` copying to a service fans out one goroutine per matched container via
|
||||
// a bare errgroup.Group, ignoring --parallel entirely.
|
||||
func TestCopyToService_ConcurrencyIsBoundedAcrossContainers(t *testing.T) {
|
||||
svc, apiClient := newTestService(t, WithMaxConcurrency(1))
|
||||
|
||||
srcFile := filepath.Join(t.TempDir(), "src.txt")
|
||||
assert.NilError(t, os.WriteFile(srcFile, []byte("hello"), 0o644))
|
||||
|
||||
const numContainers = 4
|
||||
var containers []container.Summary
|
||||
for i := range numContainers {
|
||||
containers = append(containers, testContainer("myservice", string(rune('a'+i)), false))
|
||||
}
|
||||
|
||||
apiClient.EXPECT().ContainerList(gomock.Any(), gomock.Any()).
|
||||
Return(client.ContainerListResult{Items: containers}, nil)
|
||||
|
||||
apiClient.EXPECT().ContainerStatPath(gomock.Any(), gomock.Any(), gomock.Any()).
|
||||
Return(client.ContainerStatPathResult{Stat: container.PathStat{Mode: os.ModeDir}}, nil).
|
||||
Times(numContainers)
|
||||
|
||||
tracker := &peakConcurrencyTracker{}
|
||||
apiClient.EXPECT().CopyToContainer(gomock.Any(), gomock.Any(), gomock.Any()).
|
||||
DoAndReturn(func(_ context.Context, _ string, opts client.CopyToContainerOptions) (client.CopyToContainerResult, error) {
|
||||
tracker.enter()
|
||||
time.Sleep(20 * time.Millisecond) // widen the window for a concurrency violation to show up
|
||||
tracker.leave()
|
||||
_, err := io.Copy(io.Discard, opts.Content)
|
||||
return client.CopyToContainerResult{}, err
|
||||
}).
|
||||
Times(numContainers)
|
||||
|
||||
err := svc.copy(t.Context(), "prj", compose.CopyOptions{
|
||||
Source: srcFile,
|
||||
Destination: "myservice:/dest",
|
||||
})
|
||||
assert.NilError(t, err)
|
||||
assert.Equal(t, tracker.Peak(), 1, "cp must never run more than maxConcurrency CopyToContainer calls at once")
|
||||
}
|
||||
|
|
@ -141,10 +141,11 @@ func (t *graphTraversal) visit(ctx context.Context, g *Graph) error {
|
|||
return nil
|
||||
}
|
||||
|
||||
eg, ctx := errgroup.WithContext(ctx)
|
||||
limit := 0
|
||||
if t.maxConcurrency > 0 {
|
||||
eg.SetLimit(t.maxConcurrency + 1)
|
||||
limit = t.maxConcurrency + 1
|
||||
}
|
||||
eg, ctx := newLimitedErrgroup(ctx, limit)
|
||||
nodeCh := make(chan *Vertex, expect)
|
||||
defer close(nodeCh)
|
||||
// nodeCh need to allow n=expect writers while reader goroutine could have returner after ctx.Done
|
||||
|
|
|
|||
|
|
@ -97,11 +97,8 @@ func (s *composeService) down(ctx context.Context, projectName string, options a
|
|||
resourceToRemove = true
|
||||
}
|
||||
|
||||
// shared by every service so the dependency-order fan-out and the
|
||||
// per-service container fan-out combined never exceed maxConcurrency
|
||||
// concurrent container removals — a per-service bound alone allows as
|
||||
// many independent services to run at once as the graph permits, each
|
||||
// with its own maxConcurrency budget (same fix as restart.go)
|
||||
// shared across services — see restart.go's newOptionalLimiter comment
|
||||
// for why a per-service limiter alone isn't enough.
|
||||
limiter := newOptionalLimiter(s.maxConcurrency)
|
||||
|
||||
err = InReverseDependencyOrder(ctx, project, func(c context.Context, service string) error {
|
||||
|
|
|
|||
|
|
@ -114,6 +114,7 @@ func (s *composeService) inspectWithSlot(ctx context.Context, limiter *semaphore
|
|||
if err := acquireSlot(ctx, limiter); err != nil {
|
||||
return container.InspectResponse{}, err
|
||||
}
|
||||
defer panicSafeReleaseSlot(limiter)
|
||||
res, err := s.apiClient().ContainerInspect(ctx, id, client.ContainerInspectOptions{})
|
||||
if err != nil {
|
||||
releaseSlot(limiter)
|
||||
|
|
@ -257,16 +258,24 @@ func logsSinceLastRun(ctr container.InspectResponse) string {
|
|||
// long-lived --follow stream never keeps blocking new connections or the
|
||||
// monitor.
|
||||
func (s *composeService) doLogContainer(ctx context.Context, limiter *semaphore.Weighted, consumer api.LogConsumer, name string, ctr container.InspectResponse, options api.LogOptions) error {
|
||||
r, err := s.apiClient().ContainerLogs(ctx, ctr.ID, client.ContainerLogsOptions{
|
||||
ShowStdout: true,
|
||||
ShowStderr: true,
|
||||
Follow: options.Follow,
|
||||
Since: options.Since,
|
||||
Until: options.Until,
|
||||
Tail: options.Tail,
|
||||
Timestamps: options.Timestamps,
|
||||
})
|
||||
releaseSlot(limiter)
|
||||
// Scoped to a closure so panicSafeReleaseSlot's defer only guards the
|
||||
// acquire-to-release window: releaseSlot below is unconditional once
|
||||
// ContainerLogs returns, so a panic during the copy loop that follows
|
||||
// must not re-trigger it and release the same slot twice.
|
||||
r, err := func() (io.ReadCloser, error) {
|
||||
defer panicSafeReleaseSlot(limiter)
|
||||
r, err := s.apiClient().ContainerLogs(ctx, ctr.ID, client.ContainerLogsOptions{
|
||||
ShowStdout: true,
|
||||
ShowStderr: true,
|
||||
Follow: options.Follow,
|
||||
Since: options.Since,
|
||||
Until: options.Until,
|
||||
Tail: options.Tail,
|
||||
Timestamps: options.Timestamps,
|
||||
})
|
||||
releaseSlot(limiter)
|
||||
return r, err
|
||||
}()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
|
|
|||
|
|
@ -33,6 +33,7 @@ import (
|
|||
"github.com/moby/moby/api/types/events"
|
||||
"github.com/moby/moby/client"
|
||||
"go.uber.org/mock/gomock"
|
||||
"golang.org/x/sync/semaphore"
|
||||
"gotest.tools/v3/assert"
|
||||
is "gotest.tools/v3/assert/cmp"
|
||||
|
||||
|
|
@ -393,6 +394,106 @@ func TestComposeService_Logs_FollowLimitsConcurrentStreamOpens(t *testing.T) {
|
|||
assert.Equal(t, tracker.Peak(), 1, "opening follow-mode log streams must be bounded by maxConcurrency")
|
||||
}
|
||||
|
||||
// TestDoLogContainer_ReleasesSlotOnPanic guards against a semaphore slot leak:
|
||||
// doLogContainer releases limiter's slot right after ContainerLogs returns
|
||||
// rather than via defer (so a --follow stream doesn't pin it for its whole
|
||||
// lifetime, see TestComposeService_Logs_FollowDoesNotStarveMonitor). A panic
|
||||
// inside ContainerLogs must still release it, or the effective concurrency
|
||||
// limit drops by one for the rest of the process.
|
||||
func TestDoLogContainer_ReleasesSlotOnPanic(t *testing.T) {
|
||||
mockCtrl := gomock.NewController(t)
|
||||
defer mockCtrl.Finish()
|
||||
|
||||
api, cli := prepareMocks(mockCtrl)
|
||||
tested, err := NewComposeService(cli)
|
||||
assert.NilError(t, err)
|
||||
svc := tested.(*composeService)
|
||||
|
||||
limiter := semaphore.NewWeighted(1)
|
||||
assert.NilError(t, limiter.Acquire(t.Context(), 1)) // simulates the slot logContainer would have acquired via acquireSlot
|
||||
|
||||
api.EXPECT().ContainerLogs(anyCancellableContext(), "c1", gomock.Any()).
|
||||
DoAndReturn(func(context.Context, string, client.ContainerLogsOptions) (io.ReadCloser, error) {
|
||||
panic("boom")
|
||||
})
|
||||
|
||||
func() {
|
||||
defer func() { recover() }() //nolint:errcheck
|
||||
_ = svc.doLogContainer(t.Context(), limiter, &testLogConsumer{}, "c1", containerType.InspectResponse{ID: "c1"}, compose.LogOptions{})
|
||||
}()
|
||||
|
||||
assert.Assert(t, limiter.TryAcquire(1), "slot must be released even when ContainerLogs panics")
|
||||
}
|
||||
|
||||
// panicReader panics on every Read, simulating a corrupted-stream panic
|
||||
// happening after doLogContainer's slot has already been released.
|
||||
type panicReader struct{}
|
||||
|
||||
func (panicReader) Read([]byte) (int, error) {
|
||||
panic("boom-copy")
|
||||
}
|
||||
|
||||
// TestDoLogContainer_CopyPanicDoesNotDoubleReleaseSlot guards against a
|
||||
// double-release: doLogContainer's slot-release guard must only cover the
|
||||
// acquire-to-release window around ContainerLogs, not the copy loop that
|
||||
// follows. A panic after that point must propagate unmodified, not trigger a
|
||||
// second releaseSlot on an already-released semaphore (which would either
|
||||
// panic on its own — masking the real panic — or silently corrupt the
|
||||
// semaphore's count).
|
||||
func TestDoLogContainer_CopyPanicDoesNotDoubleReleaseSlot(t *testing.T) {
|
||||
mockCtrl := gomock.NewController(t)
|
||||
defer mockCtrl.Finish()
|
||||
|
||||
api, cli := prepareMocks(mockCtrl)
|
||||
tested, err := NewComposeService(cli)
|
||||
assert.NilError(t, err)
|
||||
svc := tested.(*composeService)
|
||||
|
||||
limiter := semaphore.NewWeighted(1)
|
||||
assert.NilError(t, limiter.Acquire(t.Context(), 1)) // simulates the slot logContainer would have acquired via acquireSlot
|
||||
|
||||
api.EXPECT().ContainerLogs(anyCancellableContext(), "c1", gomock.Any()).
|
||||
Return(io.NopCloser(panicReader{}), nil)
|
||||
|
||||
var recovered any
|
||||
func() {
|
||||
defer func() { recovered = recover() }()
|
||||
_ = svc.doLogContainer(t.Context(), limiter, &testLogConsumer{}, "c1", containerType.InspectResponse{ID: "c1", Config: &containerType.Config{Tty: true}}, compose.LogOptions{})
|
||||
}()
|
||||
|
||||
assert.Equal(t, recovered, "boom-copy", "the copy-loop panic must propagate unmodified, not be replaced by a semaphore double-release panic")
|
||||
assert.Assert(t, limiter.TryAcquire(1), "slot must have been released exactly once, right after ContainerLogs returned")
|
||||
}
|
||||
|
||||
// TestInspectWithSlot_ReleasesSlotOnPanic guards against the same semaphore
|
||||
// slot leak as TestDoLogContainer_ReleasesSlotOnPanic, one function earlier
|
||||
// in the acquire/release chain: inspectWithSlot only releases its slot via an
|
||||
// explicit, non-deferred call on the error path (success transfers ownership
|
||||
// to doLogContainer). A panic inside ContainerInspect must still release it.
|
||||
func TestInspectWithSlot_ReleasesSlotOnPanic(t *testing.T) {
|
||||
mockCtrl := gomock.NewController(t)
|
||||
defer mockCtrl.Finish()
|
||||
|
||||
api, cli := prepareMocks(mockCtrl)
|
||||
tested, err := NewComposeService(cli)
|
||||
assert.NilError(t, err)
|
||||
svc := tested.(*composeService)
|
||||
|
||||
limiter := semaphore.NewWeighted(1)
|
||||
|
||||
api.EXPECT().ContainerInspect(anyCancellableContext(), "c1", gomock.Any()).
|
||||
DoAndReturn(func(context.Context, string, client.ContainerInspectOptions) (client.ContainerInspectResult, error) {
|
||||
panic("boom")
|
||||
})
|
||||
|
||||
func() {
|
||||
defer func() { recover() }() //nolint:errcheck
|
||||
_, _ = svc.inspectWithSlot(t.Context(), limiter, "c1")
|
||||
}()
|
||||
|
||||
assert.Assert(t, limiter.TryAcquire(1), "slot must be released even when ContainerInspect panics")
|
||||
}
|
||||
|
||||
type testLogConsumer struct {
|
||||
mu sync.Mutex
|
||||
// logs is keyed by container ID; values are log lines
|
||||
|
|
|
|||
|
|
@ -48,11 +48,8 @@ func (s *composeService) stop(ctx context.Context, projectName string, options a
|
|||
options.Services = project.ServiceNames()
|
||||
}
|
||||
|
||||
// shared by every service so the dependency-order fan-out and the
|
||||
// per-service container fan-out combined never exceed maxConcurrency
|
||||
// concurrent container stops — a per-service bound alone allows as many
|
||||
// independent services to run at once as the graph permits, each with
|
||||
// its own maxConcurrency budget (same fix as restart.go)
|
||||
// shared across services — see restart.go's newOptionalLimiter comment
|
||||
// for why a per-service limiter alone isn't enough.
|
||||
limiter := newOptionalLimiter(s.maxConcurrency)
|
||||
|
||||
return InReverseDependencyOrder(ctx, project, func(c context.Context, service string) error {
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue