diff --git a/pkg/compose/executor.go b/pkg/compose/executor.go index aeb528648..9972b7b41 100644 --- a/pkg/compose/executor.go +++ b/pkg/compose/executor.go @@ -160,25 +160,6 @@ func (exec *planExecutor) run(ctx context.Context, plan *Plan) error { groups := exec.buildGroupTracker(plan) events := exec.compose.events - // Cancellation is phase-scoped, not plan-wide: the Create phase is - // canceled by a Create-phase node failing; the Start phase, derived from - // it, is ALSO canceled by a Start-phase node failing. A Start-phase - // failure (an OpWaitCondition hitting --wait-timeout, a container failing - // to start, anything) must never cancel the Create phase -- doing so - // would abort unrelated, still in-flight Create-phase work (a sibling - // service's container create, a network create) that has nothing to do - // with it. The old create()-then-start() sequence never had this failure - // mode: the whole create phase had always finished before any dependency - // wait, or anything else in the start phase, even began. A Create-phase - // failure still cancels the Start phase too, via the Create phase's - // cancellation propagating to the Start phase derived from it -- - // matching the old sequence, where start() was never even called once - // create() failed. - createPhase := newPhaseCancel(ctx) - defer createPhase.cancel() - startPhase := newPhaseCancel(createPhase.ctx) - defer startPhase.cancel() - rs := &runState{ done: done, recordFailure: recordFailure, @@ -186,32 +167,76 @@ func (exec *planExecutor) run(ctx context.Context, plan *Plan) error { groups: groups, events: events, limiter: newOptionalLimiter(exec.compose.maxConcurrency), - createPhase: createPhase, - startPhase: startPhase, - } - eg := errgroup.Group{} - for _, node := range plan.Nodes { - eg.Go(func() error { - return exec.runNode(node, rs) - }) } - err := eg.Wait() - // A phase's own recorded error -- set by whichever node first triggered - // that phase's cancellation (see phaseCancel.fail) -- is authoritative - // over whatever eg.Wait() itself picked: once cancellation starts - // unblocking siblings, more than one goroutine can be mid-return at the - // same time, and errgroup.Group's own first-error-wins race has no idea - // which of them was the actual cause versus just a cancellation casualty - // (a sibling's <-nodeCtx.Done() bail returning a bare context.Canceled - // that reveals nothing). Create takes precedence over Start: a Create - // failure is the more fundamental one of the two (it's what canceled - // Start in the first place, transitively, when both end up set). - if createPhase.err != nil { - return createPhase.err + var createNodes, startNodes []*PlanNode + for _, node := range plan.Nodes { + if node.Phase == PhaseStart { + startNodes = append(startNodes, node) + } else { + createNodes = append(createNodes, node) + } } - if startPhase.err != nil { - return startPhase.err + + // A Create-phase node depending on a Start-phase node would deadlock + // forever: runPhase(createNodes, ...) only closes rs.done for nodes in + // createNodes, so runNode's `<-rs.done[dep.ID]` would block on a channel + // nothing ever closes, since the Start phase isn't even dispatched until + // the Create phase this node is blocking returns. Nothing in the + // reconciler should ever produce this edge, but failing fast with a + // clear error beats a silent hang if it ever does. + for _, node := range createNodes { + for _, dep := range node.DependsOn { + if dep.Phase == PhaseStart { + return fmt.Errorf("invalid plan: create-phase node %d depends on start-phase node %d", node.ID, dep.ID) + } + } + } + + // The Create phase is a hard barrier: no Start-phase node is even + // dispatched until every Create-phase node has succeeded. This isn't + // just about not canceling unrelated in-flight Create work on a Start + // failure (phaseCancel already keeps that scoped per phase, see its own + // doc comment) -- it's that a Start-phase node whose own DependsOn + // edges are all satisfied has no reason to wait for a COMPLETELY + // UNRELATED Create-phase node elsewhere in the plan, so without this + // barrier a service could already be starting while a sibling it has no + // relationship with is still being created, or fails. The old + // create()-then-start() sequence never allowed that: the whole create + // phase, project-wide, always finished (or failed, with start() never + // invoked at all) before any dependency wait, or anything else in the + // start phase, even began. Interactive up's attach/printer session + // depends on this too: it takes exclusive hold of the terminal for the + // create phase's own progress display, handing it over to continuous + // container log streaming only once the create phase is entirely done + // -- a Start-phase node's log output interleaving with Create's + // progress bars has nowhere consistent to go. + if err := exec.runPhase(createNodes, rs, newPhaseCancel(ctx)); err != nil { + return err + } + return exec.runPhase(startNodes, rs, newPhaseCancel(ctx)) +} + +// runPhase dispatches every node in nodes concurrently (respecting their own +// DependsOn edges via rs.done, same as a single-phase plan always did) and +// waits for them all to finish. phase is this call's own cancellation scope: +// a node failing here cancels phase.ctx for its siblings in THIS call only, +// never a concurrently-running call for a different phase (not that there +// is one today -- run() calls this sequentially -- but runNode has no way +// to tell, so this still matters if that ever changes). See phaseCancel's +// doc comment for why a node's failure goes through phase.fail instead of +// errgroup.Group's own first-error-wins race. +func (exec *planExecutor) runPhase(nodes []*PlanNode, rs *runState, phase *phaseCancel) error { + defer phase.cancel() + eg := errgroup.Group{} + for _, node := range nodes { + eg.Go(func() error { + return exec.runNode(node, rs, phase) + }) + } + err := eg.Wait() + if phase.err != nil { + return phase.err } return err } @@ -276,9 +301,11 @@ func acquireNodeSlot(ctx context.Context, limiter *semaphore.Weighted, node *Pla // runState carries the per-run() state every node's goroutine shares: the // done-channel per node, dependency-failure bookkeeping, group event -// tracking, the concurrency limiter, and the phase-scoped cancellation a -// node joins depending on its own Phase. Split out of run() purely to keep -// runNode a plain method instead of a closure captured in a loop. +// tracking, and the concurrency limiter. Shared across both phases (built +// once in run(), before either runs) since done/failures must stay visible +// to a Start-phase node depending on a Create-phase one across the barrier +// between them. Split out of run() purely to keep runNode a plain method +// instead of a closure captured in a loop. type runState struct { done map[int]chan struct{} recordFailure func(id int, err error) @@ -286,19 +313,13 @@ type runState struct { groups *groupTracker events api.EventProcessor limiter *semaphore.Weighted - createPhase *phaseCancel - startPhase *phaseCancel } // runNode executes one plan node: waits for its dependencies, skips it if -// one of them failed, then dispatches it and records the outcome. See run() -// for the phase-scoped cancellation (createPhase/startPhase) this draws -// from. -func (exec *planExecutor) runNode(node *PlanNode, rs *runState) error { - phase := rs.createPhase - if node.Phase == PhaseStart { - phase = rs.startPhase - } +// one of them failed, then dispatches it and records the outcome. phase is +// the cancellation scope of the runPhase call driving it -- see runPhase and +// phaseCancel's own doc comments. +func (exec *planExecutor) runNode(node *PlanNode, rs *runState, phase *phaseCancel) error { nodeCtx := phase.ctx // Every exit path below must close rs.done[node.ID] exactly once, or a diff --git a/pkg/compose/executor_start_test.go b/pkg/compose/executor_start_test.go index 294b6cc11..61490a3ed 100644 --- a/pkg/compose/executor_start_test.go +++ b/pkg/compose/executor_start_test.go @@ -24,6 +24,7 @@ import ( "fmt" "net" "sync" + "sync/atomic" "testing" "time" @@ -264,56 +265,44 @@ func TestExecutePlanWaitDeadlineOnlyBoundsWaitNodes(t *testing.T) { assert.Assert(t, elapsed < 5*time.Second, "waitTimeout did not bound the stuck OpWaitCondition node (took %s)", elapsed) } -// TestExecutePlanStartPhaseFailureDoesNotCancelCreatePhase is a regression -// test for a Copilot review finding on #14290: run() used to dispatch every -// node under one shared errgroup.WithContext, so a Start-phase node failing -// (a required OpWaitCondition hitting its own waitTimeout, here) canceled -// ctx for every other goroutine still in flight -- including a completely -// unrelated Create-phase node (a sibling service's network create) that has -// nothing to do with the dependency that timed out. The old -// create()-then-start() sequence never had this failure mode: the whole -// create phase had always finished before any dependency wait began. This -// pairs a never-satisfied required wait (Phase: PhaseStart, matching what -// planStartPhase actually produces) with a network create (Phase: -// PhaseCreate, the default) deliberately slower than the wait's timeout, and -// asserts the network create's own context is never canceled -- it must -// complete normally, on its own schedule, even though the sibling wait -// already failed and the plan as a whole still fails too. -func TestExecutePlanStartPhaseFailureDoesNotCancelCreatePhase(t *testing.T) { - svc, apiClient, _ := newStartPhaseTestService(t) - - dbSummary := container.Summary{ - ID: "db-id", - Names: []string{"/test-db-1"}, - Labels: map[string]string{api.ServiceLabel: "db", api.OneoffLabel: "False"}, - } - apiClient.EXPECT().ContainerInspect(gomock.Any(), "db-id", gomock.Any()).Return(client.ContainerInspectResult{ - Container: container.InspectResponse{ - ID: "db-id", - Name: "/test-db-1", - State: &container.State{ - Status: container.StateRunning, - Health: &container.Health{Status: container.Starting}, - }, - Config: &container.Config{Healthcheck: &container.HealthConfig{Test: []string{"CMD", "true"}}}, - }, - }, nil).AnyTimes() +// TestExecutePlanCreatePhaseIsABarrierBeforeStartPhase is a regression test +// for a Copilot review finding on #14291: a Start-phase node's own DependsOn +// edges being satisfied is not enough to let it run -- nothing stopped it +// from starting while a COMPLETELY UNRELATED Create-phase node elsewhere in +// the plan was still in flight (or about to fail), something the old +// create()-then-start() sequence never allowed (the whole create phase, +// project-wide, always finished -- or failed, with start() never invoked at +// all -- before any dependency wait, or anything else in the start phase, +// even began). Interactive up's attach/printer session needs that same +// guarantee: it takes exclusive hold of the terminal for the create phase's +// progress display, handing it to continuous container log streaming only +// once the create phase is entirely done. This pairs a slow, unrelated +// network create (Phase: PhaseCreate, the default, no DependsOn edge to +// anything in the Start phase) with an OpStartContainer node (Phase: +// PhaseStart) and asserts ContainerStart is never called before the network +// create has fully returned. Uses OpStartContainer rather than +// OpWaitCondition deliberately: execWaitCondition's underlying poll loop +// only checks on a 500ms ticker (never on entry), which would make a +// same-order-of-magnitude Create delay pass this assertion even with no +// barrier at all, and so couldn't actually catch a regression here. +func TestExecutePlanCreatePhaseIsABarrierBeforeStartPhase(t *testing.T) { + svc, apiClient := newTestService(t) + var createDone atomic.Bool nw := types.NetworkConfig{Name: "test_default"} - var networkCreateCtxErr error - networkCreateDone := make(chan struct{}) apiClient.EXPECT().NetworkCreate(gomock.Any(), "test_default", gomock.Any()). - DoAndReturn(func(ctx context.Context, _ string, _ client.NetworkCreateOptions) (client.NetworkCreateResult, error) { - defer close(networkCreateDone) - // Deliberately outlives the wait's timeout below: if the - // Create-phase ctx were canceled by the sibling Start-phase wait - // failing, this ctx would already be done well before this - // returns. - time.Sleep(300 * time.Millisecond) - networkCreateCtxErr = ctx.Err() + DoAndReturn(func(context.Context, string, client.NetworkCreateOptions) (client.NetworkCreateResult, error) { + time.Sleep(100 * time.Millisecond) + createDone.Store(true) return client.NetworkCreateResult{ID: "net-id"}, nil }) + apiClient.EXPECT().ContainerStart(gomock.Any(), "db-id", gomock.Any()). + DoAndReturn(func(context.Context, string, client.ContainerStartOptions) (client.ContainerStartResult, error) { + assert.Assert(t, createDone.Load(), "the Start-phase container start began before the unrelated Create-phase network create returned") + return client.ContainerStartResult{}, nil + }) + plan := &Plan{} plan.addNode(Operation{ Type: OpCreateNetwork, @@ -321,36 +310,25 @@ func TestExecutePlanStartPhaseFailureDoesNotCancelCreatePhase(t *testing.T) { Name: nw.Name, Network: &nw, }, "") - wait := plan.addNode(Operation{ - Type: OpWaitCondition, - ResourceID: "wait:db:service_healthy", - Name: "db", - Condition: types.ServiceConditionHealthy, + start := plan.addNode(Operation{ + Type: OpStartContainer, + ResourceID: "service:db:1", + Container: &container.Summary{ID: "db-id", Names: []string{"/test-db-1"}}, }, "") - wait.Phase = PhaseStart + start.Phase = PhaseStart exec := svc.newPlanExecutor(&types.Project{Name: "test"}, emptyObservedState("test"), nil) - exec.containersByService["db"] = Containers{dbSummary} - exec.waitTimeout = 100 * time.Millisecond - err := exec.run(t.Context(), plan) - assert.ErrorIs(t, err, context.DeadlineExceeded) - - select { - case <-networkCreateDone: - case <-time.After(5 * time.Second): - t.Fatal("NetworkCreate never completed -- its context was likely canceled by the sibling Start-phase wait timing out") - } - assert.NilError(t, networkCreateCtxErr, "NetworkCreate's own ctx must not be canceled by an unrelated Start-phase failure") + assert.NilError(t, exec.run(t.Context(), plan)) + assert.Assert(t, createDone.Load()) } -// TestExecutePlanCreatePhaseFailureCancelsCreatePhase is the mirror image of -// TestExecutePlanStartPhaseFailureDoesNotCancelCreatePhase: phase-scoped -// cancellation must not weaken the Create phase's own existing fail-fast +// TestExecutePlanCreatePhaseFailureCancelsCreatePhase verifies phase-scoped +// cancellation didn't weaken the Create phase's own existing fail-fast // behavior -- two independent (no DependsOn edge) Create-phase nodes, one // failing immediately, must still cancel the other's in-flight work. Both // nodes default to Phase: PhaseCreate (addNode's zero value), so they share -// createCtx. +// the same runPhase call's phaseCancel. func TestExecutePlanCreatePhaseFailureCancelsCreatePhase(t *testing.T) { svc, apiClient := newTestService(t) @@ -397,6 +375,42 @@ func TestExecutePlanCreatePhaseFailureCancelsCreatePhase(t *testing.T) { assert.ErrorIs(t, volumeCreateCtxErr, context.Canceled, "VolumeCreate's ctx must be canceled by a sibling Create-phase node failing") } +// TestExecutePlanCreatePhaseFailureNeverDispatchesStartPhase complements +// TestExecutePlanCreatePhaseIsABarrierBeforeStartPhase: that test proves the +// Start phase waits for a successful Create phase, this one proves a failed +// Create phase skips the Start phase entirely rather than dispatching it and +// canceling it. run() must return the Create-phase error before runPhase is +// ever called on startNodes -- ContainerStart's .Times(0) fails the test the +// instant it's called at all, not just if it fails to be called. +func TestExecutePlanCreatePhaseFailureNeverDispatchesStartPhase(t *testing.T) { + svc, apiClient := newTestService(t) + + apiClient.EXPECT().NetworkCreate(gomock.Any(), "test_default", gomock.Any()). + Return(client.NetworkCreateResult{}, errors.New("boom")) + apiClient.EXPECT().ContainerStart(gomock.Any(), gomock.Any(), gomock.Any()).Times(0) + + nw := types.NetworkConfig{Name: "test_default"} + + plan := &Plan{} + plan.addNode(Operation{ + Type: OpCreateNetwork, + ResourceID: "network:default", + Name: nw.Name, + Network: &nw, + }, "") + start := plan.addNode(Operation{ + Type: OpStartContainer, + ResourceID: "service:db:1", + Container: &container.Summary{ID: "db-id", Names: []string{"/test-db-1"}}, + }, "") + start.Phase = PhaseStart + + exec := svc.newPlanExecutor(&types.Project{Name: "test"}, emptyObservedState("test"), nil) + + err := exec.run(t.Context(), plan) + assert.ErrorContains(t, err, "boom") +} + // TestExecutePlanConcurrentPhaseFailureKeepsOriginatingError exercises a // Copilot/docker-agent finding on the phase-scoped cancellation fix: // errgroup.Group's plain Wait() returns whichever goroutine's error reaches