diff --git a/pkg/compose/compose.go b/pkg/compose/compose.go index 6f4885f03..d97702de4 100644 --- a/pkg/compose/compose.go +++ b/pkg/compose/compose.go @@ -200,6 +200,18 @@ func releaseSlot(limiter *semaphore.Weighted) { } } +// panicSafeReleaseSlot is deferred by callers that release a slot earlier +// than function exit on the success path (see inspectWithSlot and +// doLogContainer) to avoid leaking it if the guarded call panics before +// reaching that point. It is a no-op unless the deferring goroutine is +// unwinding from a panic, in which case it releases the slot and re-panics. +func panicSafeReleaseSlot(limiter *semaphore.Weighted) { + if p := recover(); p != nil { + releaseSlot(limiter) + panic(p) + } +} + // forEachContainerWithLimiter runs fn concurrently for each container, // bounded by limiter. Unlike newLimitedErrgroup, limiter is built by the // caller and can be shared across several concurrently-dispatched calls diff --git a/pkg/compose/cp.go b/pkg/compose/cp.go index 523b6a022..347672492 100644 --- a/pkg/compose/cp.go +++ b/pkg/compose/cp.go @@ -29,7 +29,6 @@ import ( "github.com/moby/go-archive" "github.com/moby/moby/api/types/container" "github.com/moby/moby/client" - "golang.org/x/sync/errgroup" "github.com/docker/compose/v5/pkg/api" ) @@ -79,37 +78,31 @@ func (s *composeService) copy(ctx context.Context, projectName string, options a return err } - g := errgroup.Group{} - for _, cont := range containers { - ctr := cont - g.Go(func() error { - name := getCanonicalContainerName(ctr) - var msg string - if direction == fromService { - msg = fmt.Sprintf("%s:%s to %s", name, srcPath, dstPath) - } else { - msg = fmt.Sprintf("%s to %s:%s", srcPath, name, dstPath) - } - s.events.On(api.Resource{ - ID: name, - Text: api.StatusCopying, - Details: msg, - Status: api.Working, - }) - if err := copyFunc(ctx, ctr.ID, srcPath, dstPath, options); err != nil { - return err - } - s.events.On(api.Resource{ - ID: name, - Text: api.StatusCopied, - Details: msg, - Status: api.Done, - }) - return nil + return forEachContainerConcurrent(ctx, s.maxConcurrency, containers, func(ctx context.Context, ctr container.Summary) error { + name := getCanonicalContainerName(ctr) + var msg string + if direction == fromService { + msg = fmt.Sprintf("%s:%s to %s", name, srcPath, dstPath) + } else { + msg = fmt.Sprintf("%s to %s:%s", srcPath, name, dstPath) + } + s.events.On(api.Resource{ + ID: name, + Text: api.StatusCopying, + Details: msg, + Status: api.Working, }) - } - - return g.Wait() + if err := copyFunc(ctx, ctr.ID, srcPath, dstPath, options); err != nil { + return err + } + s.events.On(api.Resource{ + ID: name, + Text: api.StatusCopied, + Details: msg, + Status: api.Done, + }) + return nil + }) } func (s *composeService) listContainersTargetedForCopy(ctx context.Context, projectName string, options api.CopyOptions, direction copyDirection, serviceName string) (Containers, error) { diff --git a/pkg/compose/cp_test.go b/pkg/compose/cp_test.go new file mode 100644 index 000000000..99edd47ba --- /dev/null +++ b/pkg/compose/cp_test.go @@ -0,0 +1,75 @@ +/* + Copyright 2020 Docker Compose CLI authors + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package compose + +import ( + "context" + "io" + "os" + "path/filepath" + "testing" + "time" + + "github.com/moby/moby/api/types/container" + "github.com/moby/moby/client" + "go.uber.org/mock/gomock" + "gotest.tools/v3/assert" + + compose "github.com/docker/compose/v5/pkg/api" +) + +// TestCopyToService_ConcurrencyIsBoundedAcrossContainers guards against the +// same unbounded-fan-out bug fixed everywhere else in this package: `compose +// cp` copying to a service fans out one goroutine per matched container via +// a bare errgroup.Group, ignoring --parallel entirely. +func TestCopyToService_ConcurrencyIsBoundedAcrossContainers(t *testing.T) { + svc, apiClient := newTestService(t, WithMaxConcurrency(1)) + + srcFile := filepath.Join(t.TempDir(), "src.txt") + assert.NilError(t, os.WriteFile(srcFile, []byte("hello"), 0o644)) + + const numContainers = 4 + var containers []container.Summary + for i := range numContainers { + containers = append(containers, testContainer("myservice", string(rune('a'+i)), false)) + } + + apiClient.EXPECT().ContainerList(gomock.Any(), gomock.Any()). + Return(client.ContainerListResult{Items: containers}, nil) + + apiClient.EXPECT().ContainerStatPath(gomock.Any(), gomock.Any(), gomock.Any()). + Return(client.ContainerStatPathResult{Stat: container.PathStat{Mode: os.ModeDir}}, nil). + Times(numContainers) + + tracker := &peakConcurrencyTracker{} + apiClient.EXPECT().CopyToContainer(gomock.Any(), gomock.Any(), gomock.Any()). + DoAndReturn(func(_ context.Context, _ string, opts client.CopyToContainerOptions) (client.CopyToContainerResult, error) { + tracker.enter() + time.Sleep(20 * time.Millisecond) // widen the window for a concurrency violation to show up + tracker.leave() + _, err := io.Copy(io.Discard, opts.Content) + return client.CopyToContainerResult{}, err + }). + Times(numContainers) + + err := svc.copy(t.Context(), "prj", compose.CopyOptions{ + Source: srcFile, + Destination: "myservice:/dest", + }) + assert.NilError(t, err) + assert.Equal(t, tracker.Peak(), 1, "cp must never run more than maxConcurrency CopyToContainer calls at once") +} diff --git a/pkg/compose/dependencies.go b/pkg/compose/dependencies.go index b1c0a231f..4917b20e8 100644 --- a/pkg/compose/dependencies.go +++ b/pkg/compose/dependencies.go @@ -141,10 +141,11 @@ func (t *graphTraversal) visit(ctx context.Context, g *Graph) error { return nil } - eg, ctx := errgroup.WithContext(ctx) + limit := 0 if t.maxConcurrency > 0 { - eg.SetLimit(t.maxConcurrency + 1) + limit = t.maxConcurrency + 1 } + eg, ctx := newLimitedErrgroup(ctx, limit) nodeCh := make(chan *Vertex, expect) defer close(nodeCh) // nodeCh need to allow n=expect writers while reader goroutine could have returner after ctx.Done diff --git a/pkg/compose/down.go b/pkg/compose/down.go index 26e127b82..a87617601 100644 --- a/pkg/compose/down.go +++ b/pkg/compose/down.go @@ -97,11 +97,8 @@ func (s *composeService) down(ctx context.Context, projectName string, options a resourceToRemove = true } - // shared by every service so the dependency-order fan-out and the - // per-service container fan-out combined never exceed maxConcurrency - // concurrent container removals — a per-service bound alone allows as - // many independent services to run at once as the graph permits, each - // with its own maxConcurrency budget (same fix as restart.go) + // shared across services — see restart.go's newOptionalLimiter comment + // for why a per-service limiter alone isn't enough. limiter := newOptionalLimiter(s.maxConcurrency) err = InReverseDependencyOrder(ctx, project, func(c context.Context, service string) error { diff --git a/pkg/compose/logs.go b/pkg/compose/logs.go index e97143a27..59640d3e1 100644 --- a/pkg/compose/logs.go +++ b/pkg/compose/logs.go @@ -114,6 +114,7 @@ func (s *composeService) inspectWithSlot(ctx context.Context, limiter *semaphore if err := acquireSlot(ctx, limiter); err != nil { return container.InspectResponse{}, err } + defer panicSafeReleaseSlot(limiter) res, err := s.apiClient().ContainerInspect(ctx, id, client.ContainerInspectOptions{}) if err != nil { releaseSlot(limiter) @@ -257,16 +258,24 @@ func logsSinceLastRun(ctr container.InspectResponse) string { // long-lived --follow stream never keeps blocking new connections or the // monitor. func (s *composeService) doLogContainer(ctx context.Context, limiter *semaphore.Weighted, consumer api.LogConsumer, name string, ctr container.InspectResponse, options api.LogOptions) error { - r, err := s.apiClient().ContainerLogs(ctx, ctr.ID, client.ContainerLogsOptions{ - ShowStdout: true, - ShowStderr: true, - Follow: options.Follow, - Since: options.Since, - Until: options.Until, - Tail: options.Tail, - Timestamps: options.Timestamps, - }) - releaseSlot(limiter) + // Scoped to a closure so panicSafeReleaseSlot's defer only guards the + // acquire-to-release window: releaseSlot below is unconditional once + // ContainerLogs returns, so a panic during the copy loop that follows + // must not re-trigger it and release the same slot twice. + r, err := func() (io.ReadCloser, error) { + defer panicSafeReleaseSlot(limiter) + r, err := s.apiClient().ContainerLogs(ctx, ctr.ID, client.ContainerLogsOptions{ + ShowStdout: true, + ShowStderr: true, + Follow: options.Follow, + Since: options.Since, + Until: options.Until, + Tail: options.Tail, + Timestamps: options.Timestamps, + }) + releaseSlot(limiter) + return r, err + }() if err != nil { return err } diff --git a/pkg/compose/logs_test.go b/pkg/compose/logs_test.go index becc15688..b10635f67 100644 --- a/pkg/compose/logs_test.go +++ b/pkg/compose/logs_test.go @@ -33,6 +33,7 @@ import ( "github.com/moby/moby/api/types/events" "github.com/moby/moby/client" "go.uber.org/mock/gomock" + "golang.org/x/sync/semaphore" "gotest.tools/v3/assert" is "gotest.tools/v3/assert/cmp" @@ -393,6 +394,106 @@ func TestComposeService_Logs_FollowLimitsConcurrentStreamOpens(t *testing.T) { assert.Equal(t, tracker.Peak(), 1, "opening follow-mode log streams must be bounded by maxConcurrency") } +// TestDoLogContainer_ReleasesSlotOnPanic guards against a semaphore slot leak: +// doLogContainer releases limiter's slot right after ContainerLogs returns +// rather than via defer (so a --follow stream doesn't pin it for its whole +// lifetime, see TestComposeService_Logs_FollowDoesNotStarveMonitor). A panic +// inside ContainerLogs must still release it, or the effective concurrency +// limit drops by one for the rest of the process. +func TestDoLogContainer_ReleasesSlotOnPanic(t *testing.T) { + mockCtrl := gomock.NewController(t) + defer mockCtrl.Finish() + + api, cli := prepareMocks(mockCtrl) + tested, err := NewComposeService(cli) + assert.NilError(t, err) + svc := tested.(*composeService) + + limiter := semaphore.NewWeighted(1) + assert.NilError(t, limiter.Acquire(t.Context(), 1)) // simulates the slot logContainer would have acquired via acquireSlot + + api.EXPECT().ContainerLogs(anyCancellableContext(), "c1", gomock.Any()). + DoAndReturn(func(context.Context, string, client.ContainerLogsOptions) (io.ReadCloser, error) { + panic("boom") + }) + + func() { + defer func() { recover() }() //nolint:errcheck + _ = svc.doLogContainer(t.Context(), limiter, &testLogConsumer{}, "c1", containerType.InspectResponse{ID: "c1"}, compose.LogOptions{}) + }() + + assert.Assert(t, limiter.TryAcquire(1), "slot must be released even when ContainerLogs panics") +} + +// panicReader panics on every Read, simulating a corrupted-stream panic +// happening after doLogContainer's slot has already been released. +type panicReader struct{} + +func (panicReader) Read([]byte) (int, error) { + panic("boom-copy") +} + +// TestDoLogContainer_CopyPanicDoesNotDoubleReleaseSlot guards against a +// double-release: doLogContainer's slot-release guard must only cover the +// acquire-to-release window around ContainerLogs, not the copy loop that +// follows. A panic after that point must propagate unmodified, not trigger a +// second releaseSlot on an already-released semaphore (which would either +// panic on its own — masking the real panic — or silently corrupt the +// semaphore's count). +func TestDoLogContainer_CopyPanicDoesNotDoubleReleaseSlot(t *testing.T) { + mockCtrl := gomock.NewController(t) + defer mockCtrl.Finish() + + api, cli := prepareMocks(mockCtrl) + tested, err := NewComposeService(cli) + assert.NilError(t, err) + svc := tested.(*composeService) + + limiter := semaphore.NewWeighted(1) + assert.NilError(t, limiter.Acquire(t.Context(), 1)) // simulates the slot logContainer would have acquired via acquireSlot + + api.EXPECT().ContainerLogs(anyCancellableContext(), "c1", gomock.Any()). + Return(io.NopCloser(panicReader{}), nil) + + var recovered any + func() { + defer func() { recovered = recover() }() + _ = svc.doLogContainer(t.Context(), limiter, &testLogConsumer{}, "c1", containerType.InspectResponse{ID: "c1", Config: &containerType.Config{Tty: true}}, compose.LogOptions{}) + }() + + assert.Equal(t, recovered, "boom-copy", "the copy-loop panic must propagate unmodified, not be replaced by a semaphore double-release panic") + assert.Assert(t, limiter.TryAcquire(1), "slot must have been released exactly once, right after ContainerLogs returned") +} + +// TestInspectWithSlot_ReleasesSlotOnPanic guards against the same semaphore +// slot leak as TestDoLogContainer_ReleasesSlotOnPanic, one function earlier +// in the acquire/release chain: inspectWithSlot only releases its slot via an +// explicit, non-deferred call on the error path (success transfers ownership +// to doLogContainer). A panic inside ContainerInspect must still release it. +func TestInspectWithSlot_ReleasesSlotOnPanic(t *testing.T) { + mockCtrl := gomock.NewController(t) + defer mockCtrl.Finish() + + api, cli := prepareMocks(mockCtrl) + tested, err := NewComposeService(cli) + assert.NilError(t, err) + svc := tested.(*composeService) + + limiter := semaphore.NewWeighted(1) + + api.EXPECT().ContainerInspect(anyCancellableContext(), "c1", gomock.Any()). + DoAndReturn(func(context.Context, string, client.ContainerInspectOptions) (client.ContainerInspectResult, error) { + panic("boom") + }) + + func() { + defer func() { recover() }() //nolint:errcheck + _, _ = svc.inspectWithSlot(t.Context(), limiter, "c1") + }() + + assert.Assert(t, limiter.TryAcquire(1), "slot must be released even when ContainerInspect panics") +} + type testLogConsumer struct { mu sync.Mutex // logs is keyed by container ID; values are log lines diff --git a/pkg/compose/stop.go b/pkg/compose/stop.go index 66e52ab39..c494dd4cf 100644 --- a/pkg/compose/stop.go +++ b/pkg/compose/stop.go @@ -48,11 +48,8 @@ func (s *composeService) stop(ctx context.Context, projectName string, options a options.Services = project.ServiceNames() } - // shared by every service so the dependency-order fan-out and the - // per-service container fan-out combined never exceed maxConcurrency - // concurrent container stops — a per-service bound alone allows as many - // independent services to run at once as the graph permits, each with - // its own maxConcurrency budget (same fix as restart.go) + // shared across services — see restart.go's newOptionalLimiter comment + // for why a per-service limiter alone isn't enough. limiter := newOptionalLimiter(s.maxConcurrency) return InReverseDependencyOrder(ctx, project, func(c context.Context, service string) error {