compose/pkg/compose/executor.go
Nicolas De Loof 80e4a0ae5c refactor: add internal/concurrency for the bounded errgroup helpers
The "errgroup bounded to maxConcurrency, where <= 0 means unlimited" helper was copied in several places (pkg/compose, internal/containerops, internal/inspect, internal/sync), together with the semaphore-based slot helpers used to share one budget across concurrent calls. Gather them in a leaf package so the extractions in progress can import it instead of duplicating it again. Behavior is unchanged.

Signed-off-by: Nicolas De Loof <nicolas.deloof@gmail.com>
2026-10-29 10:18:58 +01:00

626 lines
25 KiB
Go

/*
Copyright 2020 Docker Compose CLI authors
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package compose
import (
"context"
"errors"
"fmt"
"sync"
"sync/atomic"
"time"
"github.com/compose-spec/compose-go/v2/types"
"golang.org/x/sync/errgroup"
"golang.org/x/sync/semaphore"
"github.com/docker/compose/v5/internal/concurrency"
"github.com/docker/compose/v5/pkg/api"
)
// planExecutor executes a reconciliation Plan by walking the DAG and performing
// each atomic operation via the Docker API. It carries no decision logic — all
// decisions were made by the reconciler when building the plan.
type planExecutor struct {
compose *composeService
project *types.Project
pctx *reconciliationContext
// listener streams pre_start/post_start hook logs. It is set once, when
// the Start phase is entered (planRun.runStart): an attached caller only
// has one by then, its printer not existing before the Create phase is
// done. nil leaves hook execution silent, matching detached behavior.
listener api.ContainerEventListener
// containersByService is a live view used to resolve service references
// (network_mode: service:x, volumes_from, ipc, pid) without a daemon
// round-trip per create.
containersMu sync.Mutex
containersByService map[string]Containers
// waitTimeout, when non-zero, bounds OpWaitCondition nodes only (see
// executeNode) -- not the rest of the plan, and with its OWN fresh
// window per wait node rather than a single shared deadline counted
// down from before the plan started. up -d --wait-timeout must cap how
// long a dependency's health/completion condition is waited on, the
// same thing start()'s own per-dependency WaitTimeout already does
// today -- without also capping unrelated create/start/hook work, and
// without letting that unrelated work's duration eat into (or exhaust)
// a wait's own budget (docker/compose#14290 review).
waitTimeout time.Duration
}
// reconciliationContext holds results produced by completed nodes so that downstream
// nodes can reference them (e.g. a RenameContainer node needs the container ID
// created by a prior CreateContainer node).
type reconciliationContext struct {
mu sync.Mutex
results map[int]operationResult
}
type operationResult struct {
ContainerID string
ContainerName string
}
func (pc *reconciliationContext) set(nodeID int, r operationResult) {
pc.mu.Lock()
defer pc.mu.Unlock()
pc.results[nodeID] = r
}
func (pc *reconciliationContext) get(nodeID int) operationResult {
pc.mu.Lock()
defer pc.mu.Unlock()
return pc.results[nodeID]
}
// executePlan walks the plan DAG, executing nodes in parallel where possible
// while respecting dependency edges. It emits progress events and handles
// group-based event aggregation for composite operations like recreate.
//
// It streams no pre_start/post_start hook logs: it hands runStart no listener.
// A caller that does, into an attached session (interactive up), builds the
// executor itself, drives a planRun, and hands its listener to runStart once
// its session exists.
func (s *composeService) executePlan(ctx context.Context, project *types.Project, observed *ObservedState, plan *Plan) error {
return s.newPlanExecutor(project, observed).run(ctx, plan)
}
// newPlanExecutor constructs a planExecutor seeded from the observed state.
// Split out from executePlan so tests can inspect the executor's live state
// (e.g. the containersByService cache) after running a plan.
func (s *composeService) newPlanExecutor(project *types.Project, observed *ObservedState) *planExecutor {
return &planExecutor{
compose: s,
project: project,
pctx: &reconciliationContext{results: map[int]operationResult{}},
containersByService: observed.containersByService(),
}
}
// run walks the plan DAG, executing nodes in parallel where possible while
// respecting dependency edges. Emits progress events and handles group-based
// event aggregation for composite operations like recreate. It is begin
// followed by both phases: a caller that must act between the Create and the
// Start phase drives a planRun itself.
func (exec *planExecutor) run(ctx context.Context, plan *Plan) error {
if plan.IsEmpty() {
return nil
}
pr, err := exec.begin(plan)
if err != nil {
return err
}
if err := pr.runCreate(ctx); err != nil {
return err
}
return pr.runStart(ctx, nil)
}
// planRun is one execution of a Plan, split at the boundary between its two
// phases. begin builds the state both phases share (the done channels and
// failure bookkeeping a Start-phase node needs to see a Create-phase one
// through, the group tracker, the limiter); runCreate and runStart then each
// dispatch their own phase, so a caller can act in between -- interactive up
// sets up its attach/printer/monitor session there -- without a second
// snapshot or a second plan.
//
// The Create phase is a hard barrier: no Start-phase node is even
// dispatched until every Create-phase node has succeeded. This isn't
// just about not canceling unrelated in-flight Create work on a Start
// failure (phaseCancel already keeps that scoped per phase, see its own
// doc comment) -- it's that a Start-phase node whose own DependsOn
// edges are all satisfied has no reason to wait for a COMPLETELY
// UNRELATED Create-phase node elsewhere in the plan, so without this
// barrier a service could already be starting while a sibling it has no
// relationship with is still being created, or fails. The old
// create()-then-start() sequence never allowed that: the whole create
// phase, project-wide, always finished (or failed, with start() never
// invoked at all) before any dependency wait, or anything else in the
// start phase, even began. Interactive up's attach/printer session
// depends on this too: it takes exclusive hold of the terminal for the
// create phase's own progress display, handing it over to continuous
// container log streaming only once the create phase is entirely done
// -- a Start-phase node's log output interleaving with Create's
// progress bars has nowhere consistent to go. runStart's guards enforce it
// for a caller driving the phases itself; run() gets it by calling
// runCreate first.
//
// A planRun is single-use and driven from one goroutine: each phase runs at
// most once, Create first. There is one planRun per executor: two of them
// would share its reconciliation context and service view.
type planRun struct {
exec *planExecutor
rs *runState
createNodes []*PlanNode
startNodes []*PlanNode
createRan bool
createErr error
startRan bool
}
// begin prepares plan for execution without dispatching anything: it builds
// the state shared by both phases, splits the nodes by phase and rejects a
// plan that could only deadlock. An empty plan, or one with a single phase,
// gives a planRun whose other phase dispatches nothing.
func (exec *planExecutor) begin(plan *Plan) (*planRun, error) {
// Build a done-channel per node so dependents can wait
done := make(map[int]chan struct{}, len(plan.Nodes))
for _, node := range plan.Nodes {
done[node.ID] = make(chan struct{})
}
// failures records the terminal error of every node that didn't succeed,
// keyed by node ID. A dependent unblocked from <-done[dep.ID] consults it
// to tell "dependency failed for a genuine reason" apart from "dependency
// succeeded" -- ctx.Err() alone can't: close(done[dep.ID]) happens inside
// the failing node's own goroutine, strictly before errgroup's cancel()
// fires on that goroutine's returned error (cancel() only runs after it
// returns), so a dependent can observe ctx.Err() == nil for a brief
// window even though its dependency just failed. Without this, that
// dependent runs anyway and spuriously emits its own group Working/
// Starting event right after the dependency's Error already went out
// (see TestExecutePlanFailedPreStartGatesStart).
var failuresMu sync.Mutex
failures := make(map[int]error, len(plan.Nodes))
successes := make(map[int]bool, len(plan.Nodes))
recordFailure := func(id int, err error) {
failuresMu.Lock()
failures[id] = err
failuresMu.Unlock()
}
recordSuccess := func(id int) {
failuresMu.Lock()
successes[id] = true
failuresMu.Unlock()
}
succeeded := func(id int) bool {
failuresMu.Lock()
defer failuresMu.Unlock()
return successes[id]
}
failedDependency := func(deps []*PlanNode) error {
failuresMu.Lock()
defer failuresMu.Unlock()
for _, dep := range deps {
if err := failures[dep.ID]; err != nil {
return err
}
}
return nil
}
// Track group event state: first node emits Working, last emits Done
groups := exec.buildGroupTracker(plan)
events := exec.compose.events
rs := &runState{
done: done,
recordFailure: recordFailure,
recordSuccess: recordSuccess,
succeeded: succeeded,
failedDependency: failedDependency,
groups: groups,
events: events,
limiter: concurrency.NewLimiter(exec.compose.maxConcurrency),
}
var createNodes, startNodes []*PlanNode
for _, node := range plan.Nodes {
if node.Phase == PhaseStart {
startNodes = append(startNodes, node)
} else {
createNodes = append(createNodes, node)
}
}
// A Create-phase node depending on a Start-phase node would deadlock
// forever: runPhase(createNodes, ...) only closes rs.done for nodes in
// createNodes, so runNode's `<-rs.done[dep.ID]` would block on a channel
// nothing ever closes, since the Start phase isn't even dispatched until
// the Create phase this node is blocking returns. Nothing in the
// reconciler should ever produce this edge, but failing fast with a
// clear error beats a silent hang if it ever does.
for _, node := range createNodes {
for _, dep := range node.DependsOn {
if dep.Phase == PhaseStart {
return nil, fmt.Errorf("invalid plan: create-phase node %d depends on start-phase node %d", node.ID, dep.ID)
}
}
}
return &planRun{exec: exec, rs: rs, createNodes: createNodes, startNodes: startNodes}, nil
}
// runCreate dispatches the Create phase and waits for it. The Start phase
// cannot be dispatched until it has returned without error.
func (r *planRun) runCreate(ctx context.Context) error {
if r.createRan {
return errors.New("plan run: the create phase already ran")
}
r.createRan = true
r.createErr = r.exec.runPhase(r.createNodes, r.rs, newPhaseCancel(ctx))
return r.createErr
}
// runStart dispatches the Start phase and waits for it. It refuses to run
// before a successful Create phase, and to run twice. This is the only way to
// give the executor a listener: it receives the pre_start/post_start hook
// logs, and nil leaves them unstreamed. It is given here because an attached
// caller only has one once it has set up its session, which happens after the
// Create phase. It is set before any Start-phase node is dispatched, and the
// planner only emits the operations that read it (pre_start, post_start) in
// the Start phase, so no goroutine can observe the previous value.
func (r *planRun) runStart(ctx context.Context, listener api.ContainerEventListener) error {
if !r.createRan || r.createErr != nil {
return errors.New("plan run: the start phase cannot run before the create phase succeeded")
}
if r.startRan {
return errors.New("plan run: the start phase already ran")
}
r.startRan = true
r.exec.listener = listener
return r.exec.runPhase(r.startNodes, r.rs, newPhaseCancel(ctx))
}
// runPhase dispatches every node in nodes concurrently (respecting their own
// DependsOn edges via rs.done, same as a single-phase plan always did) and
// waits for them all to finish. phase is this call's own cancellation scope:
// a node failing here cancels phase.ctx for its siblings in THIS call only,
// never a concurrently-running call for a different phase (not that there
// is one today -- a planRun runs its phases sequentially -- but runNode has
// no way to tell, so this still matters if that ever changes). See phaseCancel's
// doc comment for why a node's failure goes through phase.fail instead of
// errgroup.Group's own first-error-wins race.
func (exec *planExecutor) runPhase(nodes []*PlanNode, rs *runState, phase *phaseCancel) error {
defer phase.cancel()
eg := errgroup.Group{}
for _, node := range nodes {
eg.Go(func() error {
return exec.runNode(node, rs, phase)
})
}
err := eg.Wait()
if phase.err != nil {
return phase.err
}
return err
}
// phaseCancel bundles one phase's cancellation context with the bookkeeping
// needed to make exactly one node's failure authoritative for it: fail is
// called by every node in this phase that errors out, but its body -- the
// event emission and the actual ctx cancellation -- runs for only the first
// caller, via once. That single gate point fixes two related races a plain
// "if nodeCtx.Err() == nil" check can't: two genuinely concurrent failures
// in the same phase racing to emit onNodeError (the old errgroup.WithContext
// made this vanishingly unlikely by canceling ctx atomically with recording
// the error; splitting cancellation into our own explicit call reopened the
// window), and errgroup.Group's own first-error-wins race picking a
// cancellation casualty's bare context.Canceled over the actual originating
// error once cancellation starts unblocking siblings (see run()'s use of
// err, set here, instead of eg.Wait()'s own return value).
type phaseCancel struct {
ctx context.Context
cancel context.CancelFunc
once sync.Once
err error
// failed is the node whose failure canceled the phase, nil while the phase
// is not canceled by one of its own nodes (the caller's context may be done
// as well, e.g. on Ctrl-C). Set strictly before cancel(), so a node that
// sees ctx done because of that failure always finds it here.
failed atomic.Pointer[PlanNode]
}
func newPhaseCancel(parent context.Context) *phaseCancel {
p := &phaseCancel{}
p.ctx, p.cancel = context.WithCancel(parent)
return p
}
// fail registers node's failure as this phase's cause, for the first caller
// only: it emits the group error event and cancels the phase's context. Any
// later caller's err is not registered, and fail tells so (false): such a
// caller either is a cancellation casualty of the first failure, with nothing
// of its own to say, or failed for a genuine reason of its own at the same
// moment, whose row is then for it to close.
func (p *phaseCancel) fail(node *PlanNode, events api.EventProcessor, groups *groupTracker, err error) bool {
first := false
p.once.Do(func() {
first = true
p.err = err
groups.onNodeError(node, events, err)
p.failed.Store(node)
p.cancel()
})
return first
}
// acquireNodeSlot takes node's --parallel slot and returns the function that
// releases it. An OpWaitCondition node holds none: it only polls the daemon on
// a ticker, so a slot held for the whole polling would be capacity lost for
// every unrelated node, and could starve the nodes it is waiting for. The
// latter needs a dependency that is already running, hence with no start
// node for the wait to depend on, whose health relies on a sibling the plan
// has yet to start: with a bounded --parallel the wait could take the slot
// that sibling's start needs and only end at its timeout. The waits
// waitDependencies runs outside any plan stay outside the cap for the same
// reason.
func acquireNodeSlot(ctx context.Context, limiter *semaphore.Weighted, node *PlanNode) (func(), error) {
if node.Operation.Type == OpWaitCondition {
return func() {}, nil
}
if err := concurrency.Acquire(ctx, limiter); err != nil {
return nil, err
}
return func() { concurrency.Release(limiter) }, nil
}
// runState carries the per-planRun state every node's goroutine shares: the
// done-channel per node, dependency-failure bookkeeping, group event
// tracking, and the concurrency limiter. Shared across both phases (built
// once in begin, before either runs) since done/failures must stay visible
// to a Start-phase node depending on a Create-phase one across the barrier
// between them. Split out of begin purely to keep runNode a plain method
// instead of a closure captured in a loop.
type runState struct {
done map[int]chan struct{}
recordFailure func(id int, err error)
recordSuccess func(id int)
succeeded func(id int) bool
failedDependency func(deps []*PlanNode) error
groups *groupTracker
events api.EventProcessor
limiter *semaphore.Weighted
}
// runNode executes one plan node: waits for its dependencies, skips it if
// one of them failed, then dispatches it and records the outcome. phase is
// the cancellation scope of the runPhase call driving it -- see runPhase and
// phaseCancel's own doc comments.
func (exec *planExecutor) runNode(node *PlanNode, rs *runState, phase *phaseCancel) error {
nodeCtx := phase.ctx
// Every exit path below must close rs.done[node.ID] exactly once, or a
// dependent blocks on it forever: deferred once here instead of before
// each individual return, so a future added exit path can't reintroduce
// the gap the original dependency-wait early-return had (no reachable
// hang today -- every dependent in the same phase shares this same
// nodeCtx, already canceled by the time it would matter -- but nothing
// enforces that staying true as the plan DAG grows new shapes).
defer close(rs.done[node.ID])
// Wait for all dependencies
for _, dep := range node.DependsOn {
select {
case <-rs.done[dep.ID]:
case <-nodeCtx.Done():
// Record the cancellation like any other failure: the deferred
// close below makes done[node.ID] ready, and a dependent that
// selects it before seeing nodeCtx.Done() would otherwise find
// no failure recorded for this node, then emit a "Starting"
// event it never finalizes and enter executeNode on a canceled
// context.
err := nodeCtx.Err()
rs.recordFailure(node.ID, err)
exec.reportAborted(node, rs, phase)
return err
}
}
// A dependency may have just failed without nodeCtx being canceled yet
// (see begin's failures comment) -- skip this node exactly like a
// canceled one: no slot acquired, no event emitted, its own dependents
// unblocked with the same failure recorded.
if err := rs.failedDependency(node.DependsOn); err != nil {
// Store the original error, not a wrapped one: this node's own
// dependents look it up the same way, and wrapping it again at
// every hop would compound "dependency failed: " prefixes down a
// multi-node chain (create -> pre_start -> start -> post_start).
rs.recordFailure(node.ID, err)
exec.reportAborted(node, rs, phase)
return fmt.Errorf("dependency failed: %w", err)
}
release, slotErr := acquireNodeSlot(nodeCtx, rs.limiter, node)
if slotErr != nil {
rs.recordFailure(node.ID, slotErr)
exec.reportAborted(node, rs, phase)
return slotErr
}
defer release()
// Emit group start event if this is the first node of a group
if !rs.groups.onNodeStart(node, rs.events) {
// its group was given up meanwhile: the node is skipped like one
// found canceled above, its row already ended
err := nodeCtx.Err()
if err == nil {
err = context.Canceled
}
rs.recordFailure(node.ID, err)
return err
}
err := exec.executeNode(nodeCtx, node)
var wi *waitInterrupted
interrupted := errors.As(err, &wi)
if interrupted {
// waitDependency ends a wait silently when its context is canceled,
// which suits a caller that is going away itself (Ctrl-C): the node
// ends as it always did. Canceled by the failure of another node of
// its phase, the wait is a failure instead: the condition was not
// reached, and the nodes depending on it must not run.
err = exec.interruptedWait(node, rs, phase, wi.cause)
}
if err == nil {
// a wait that ended unreached is not a success its dependents' reports
// could rely on (see abortReason)
if !interrupted {
rs.recordSuccess(node.ID)
}
// Emit group done event if this is the last node of a group
rs.groups.onNodeDone(node, rs.events)
return nil
}
rs.recordFailure(node.ID, err)
switch {
case nodeCtx.Err() == nil:
// This node's own failure, not an inherited cancellation: a node
// whose nodeCtx is already canceled failed because of that
// cancellation, not a new failure of its own to propagate. phase.fail
// itself gates on being the first such failure in this phase (see its
// doc comment) -- this outer check only spares every later,
// cancellation-casualty caller the cost of building the error-event
// payload it would be dropped anyway. One that lost the race to a
// failure of its own, at the same moment, still ends its own row.
if !phase.fail(node, rs.events, rs.groups, err) {
rs.groups.onNodeError(node, rs.events, err)
}
case phase.failed.Load() != nil && !isCancellation(err):
// canceled by the failure of another node while failing, for a reason
// of its own, before it noticed: "interrupted" would hide that reason
rs.groups.onNodeError(node, rs.events, err)
default:
exec.reportAborted(node, rs, phase)
}
return err
}
// isCancellation tells whether err is the end of a context, as opposed to a
// failure of the operation itself.
func isCancellation(err error) bool {
return errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded)
}
// errWaitTimeout marks the failure of an OpWaitCondition node whose own
// --wait-timeout window (planExecutor.waitTimeout) expired, as opposed to any
// other DeadlineExceeded that can surface from a plan: an engine call timing
// out, a hook, or an external deadline on the caller's context. Callers that
// translate a wait timeout into a user-facing message key on this rather than
// on DeadlineExceeded, which is not specific to waits. The wrapped error still
// satisfies errors.Is(err, context.DeadlineExceeded).
var errWaitTimeout = errors.New("timeout waiting for dependencies")
// waitInterrupted marks an OpWaitCondition node whose wait ended because its
// context was canceled, before the condition was reached. cause is what the
// wait would have ended with had nothing but a caller going away (Ctrl-C) been
// at stake: nil for a wait that ends silently, the error of a check cut short
// otherwise.
type waitInterrupted struct{ cause error }
func (e *waitInterrupted) Error() string { return "wait interrupted" }
// translateWaitTimeout turns a plan failure caused by a wait node's own
// --wait-timeout window (see errWaitTimeout) into the "timeout waiting for
// dependencies" message a depends_on wait timing out has always reported,
// with or without --wait, and leaves any other error as is. It is
// deliberately not the "application not healthy" message:
// that one belongs to the final --wait readiness check that runs after the
// plan, which has its own translation.
func translateWaitTimeout(err error) error {
if !errors.Is(err, errWaitTimeout) {
return err
}
return errors.New("timeout waiting for dependencies")
}
// execBoundedWaitCondition runs an OpWaitCondition node under its own fresh
// planExecutor.waitTimeout window, if one is set, and marks a failure caused
// by that window expiring with errWaitTimeout. origCtx.Err() == nil rules out
// an earlier deadline inherited from the caller having fired instead: both
// make the derived context report DeadlineExceeded, but only the former is
// this wait's own timeout.
func (exec *planExecutor) execBoundedWaitCondition(origCtx context.Context, op Operation) error {
ctx := origCtx
if exec.waitTimeout > 0 {
var cancel context.CancelFunc
ctx, cancel = context.WithTimeout(origCtx, exec.waitTimeout)
defer cancel()
}
err := exec.execWaitCondition(ctx, origCtx, op)
if err != nil && exec.waitTimeout > 0 && origCtx.Err() == nil &&
errors.Is(ctx.Err(), context.DeadlineExceeded) && errors.Is(err, context.DeadlineExceeded) {
return fmt.Errorf("%w: %w", errWaitTimeout, err)
}
return err
}
// executeNode dispatches a single plan node to the appropriate API call.
func (exec *planExecutor) executeNode(ctx context.Context, node *PlanNode) error {
op := node.Operation
switch op.Type {
case OpCreateNetwork:
return exec.execCreateNetwork(ctx, op)
case OpRemoveNetwork:
return exec.execRemoveNetwork(ctx, op)
case OpDisconnectNetwork:
return exec.execDisconnectNetwork(ctx, op)
case OpConnectNetwork:
return exec.execConnectNetwork(ctx, op)
case OpCreateVolume:
return exec.execCreateVolume(ctx, op)
case OpRemoveVolume:
return exec.execRemoveVolume(ctx, op)
case OpCreateContainer:
return exec.execCreateContainer(ctx, node)
case OpStartContainer:
return exec.execStartContainer(ctx, op)
case OpStopContainer:
return exec.execStopContainer(ctx, op)
case OpRemoveContainer:
return exec.execRemoveContainer(ctx, op)
case OpRenameContainer:
return exec.execRenameContainer(ctx, node)
case OpCreateHookContainer:
return exec.execCreateHookContainer(ctx, node)
case OpWaitCondition:
return exec.execBoundedWaitCondition(ctx, op)
case OpRunPreStart:
return exec.execRunPreStart(ctx, op)
case OpRunPostStart:
return exec.execRunPostStart(ctx, op)
case OpRunProvider:
return exec.compose.runPlugin(ctx, exec.project, *op.Service, "up")
default:
return fmt.Errorf("unknown operation type: %s", op.Type)
}
}