From 19aa3d8604f5abb123adf82d73e83d0fc2e1ba06 Mon Sep 17 00:00:00 2001 From: Nicolas De Loof Date: Tue, 6 Oct 2026 22:33:23 +0200 Subject: [PATCH] fix(executor): make Create->Start a hard sequential barrier run() used to dispatch every node concurrently, with Create-phase and Start-phase cancellation scoped separately via phaseCancel so a Start-phase failure couldn't cancel unrelated in-flight Create work. That stopped short of what the interactive up convergence (lot 2's next step) actually needs: its attach/printer session takes exclusive hold of the terminal for the Create phase's own progress display, and hands it over to continuous container log streaming only once the Create phase is entirely done project-wide -- a Start-phase node's log output interleaving with Create's progress bars has nowhere consistent to go. Phase-scoped cancellation alone still let an unrelated Start-phase node begin (and start logging) while a sibling elsewhere in the plan was still being created, or had just failed. run() now splits plan.Nodes into createNodes/startNodes upfront and runs them as two fully sequential runPhase() calls: Start-phase nodes are never even dispatched until every Create-phase node has succeeded, and a Create-phase failure returns immediately without calling runPhase on startNodes at all. This matches the old create()-then- start() sequence's own fail-fast behavior, which the single-plan migration had implicitly relaxed. Added a fast-fail validation: a Create-phase node depending on a Start-phase node would otherwise deadlock forever (the Start phase isn't dispatched until the Create phase blocked on it returns, so nothing would ever close that dependency's done channel). Nothing in the reconciler should produce that edge, but failing fast with a clear error beats a silent hang if it ever does. TestExecutePlanCreatePhaseIsABarrierBeforeStartPhase and TestExecutePlanCreatePhaseFailureNeverDispatchesStartPhase cover both directions of the barrier; both verified by ablation (reverting to a single concurrent runPhase call reliably fails each, 5/5 runs). Verified: go build, go test ./pkg/compose/... (incl. -race), and golangci-lint all clean; full e2e suite re-run clean against a locally built binary (same 3 pre-existing, environment-specific failures as the stack's baseline: TestWatch, TestIPC, TestComposeModel). Local Copilot CLI review found no correctness/deadlock issues in the barrier logic itself; its two actionable findings (missing phase-ordering guard, missing never-dispatched regression test) are both addressed here. Signed-off-by: Nicolas De Loof --- pkg/compose/executor.go | 131 +++++++++++++++----------- pkg/compose/executor_start_test.go | 144 ++++++++++++++++------------- 2 files changed, 155 insertions(+), 120 deletions(-) diff --git a/pkg/compose/executor.go b/pkg/compose/executor.go index aeb528648..9972b7b41 100644 --- a/pkg/compose/executor.go +++ b/pkg/compose/executor.go @@ -160,25 +160,6 @@ func (exec *planExecutor) run(ctx context.Context, plan *Plan) error { groups := exec.buildGroupTracker(plan) events := exec.compose.events - // Cancellation is phase-scoped, not plan-wide: the Create phase is - // canceled by a Create-phase node failing; the Start phase, derived from - // it, is ALSO canceled by a Start-phase node failing. A Start-phase - // failure (an OpWaitCondition hitting --wait-timeout, a container failing - // to start, anything) must never cancel the Create phase -- doing so - // would abort unrelated, still in-flight Create-phase work (a sibling - // service's container create, a network create) that has nothing to do - // with it. The old create()-then-start() sequence never had this failure - // mode: the whole create phase had always finished before any dependency - // wait, or anything else in the start phase, even began. A Create-phase - // failure still cancels the Start phase too, via the Create phase's - // cancellation propagating to the Start phase derived from it -- - // matching the old sequence, where start() was never even called once - // create() failed. - createPhase := newPhaseCancel(ctx) - defer createPhase.cancel() - startPhase := newPhaseCancel(createPhase.ctx) - defer startPhase.cancel() - rs := &runState{ done: done, recordFailure: recordFailure, @@ -186,32 +167,76 @@ func (exec *planExecutor) run(ctx context.Context, plan *Plan) error { groups: groups, events: events, limiter: newOptionalLimiter(exec.compose.maxConcurrency), - createPhase: createPhase, - startPhase: startPhase, - } - eg := errgroup.Group{} - for _, node := range plan.Nodes { - eg.Go(func() error { - return exec.runNode(node, rs) - }) } - err := eg.Wait() - // A phase's own recorded error -- set by whichever node first triggered - // that phase's cancellation (see phaseCancel.fail) -- is authoritative - // over whatever eg.Wait() itself picked: once cancellation starts - // unblocking siblings, more than one goroutine can be mid-return at the - // same time, and errgroup.Group's own first-error-wins race has no idea - // which of them was the actual cause versus just a cancellation casualty - // (a sibling's <-nodeCtx.Done() bail returning a bare context.Canceled - // that reveals nothing). Create takes precedence over Start: a Create - // failure is the more fundamental one of the two (it's what canceled - // Start in the first place, transitively, when both end up set). - if createPhase.err != nil { - return createPhase.err + var createNodes, startNodes []*PlanNode + for _, node := range plan.Nodes { + if node.Phase == PhaseStart { + startNodes = append(startNodes, node) + } else { + createNodes = append(createNodes, node) + } } - if startPhase.err != nil { - return startPhase.err + + // A Create-phase node depending on a Start-phase node would deadlock + // forever: runPhase(createNodes, ...) only closes rs.done for nodes in + // createNodes, so runNode's `<-rs.done[dep.ID]` would block on a channel + // nothing ever closes, since the Start phase isn't even dispatched until + // the Create phase this node is blocking returns. Nothing in the + // reconciler should ever produce this edge, but failing fast with a + // clear error beats a silent hang if it ever does. + for _, node := range createNodes { + for _, dep := range node.DependsOn { + if dep.Phase == PhaseStart { + return fmt.Errorf("invalid plan: create-phase node %d depends on start-phase node %d", node.ID, dep.ID) + } + } + } + + // The Create phase is a hard barrier: no Start-phase node is even + // dispatched until every Create-phase node has succeeded. This isn't + // just about not canceling unrelated in-flight Create work on a Start + // failure (phaseCancel already keeps that scoped per phase, see its own + // doc comment) -- it's that a Start-phase node whose own DependsOn + // edges are all satisfied has no reason to wait for a COMPLETELY + // UNRELATED Create-phase node elsewhere in the plan, so without this + // barrier a service could already be starting while a sibling it has no + // relationship with is still being created, or fails. The old + // create()-then-start() sequence never allowed that: the whole create + // phase, project-wide, always finished (or failed, with start() never + // invoked at all) before any dependency wait, or anything else in the + // start phase, even began. Interactive up's attach/printer session + // depends on this too: it takes exclusive hold of the terminal for the + // create phase's own progress display, handing it over to continuous + // container log streaming only once the create phase is entirely done + // -- a Start-phase node's log output interleaving with Create's + // progress bars has nowhere consistent to go. + if err := exec.runPhase(createNodes, rs, newPhaseCancel(ctx)); err != nil { + return err + } + return exec.runPhase(startNodes, rs, newPhaseCancel(ctx)) +} + +// runPhase dispatches every node in nodes concurrently (respecting their own +// DependsOn edges via rs.done, same as a single-phase plan always did) and +// waits for them all to finish. phase is this call's own cancellation scope: +// a node failing here cancels phase.ctx for its siblings in THIS call only, +// never a concurrently-running call for a different phase (not that there +// is one today -- run() calls this sequentially -- but runNode has no way +// to tell, so this still matters if that ever changes). See phaseCancel's +// doc comment for why a node's failure goes through phase.fail instead of +// errgroup.Group's own first-error-wins race. +func (exec *planExecutor) runPhase(nodes []*PlanNode, rs *runState, phase *phaseCancel) error { + defer phase.cancel() + eg := errgroup.Group{} + for _, node := range nodes { + eg.Go(func() error { + return exec.runNode(node, rs, phase) + }) + } + err := eg.Wait() + if phase.err != nil { + return phase.err } return err } @@ -276,9 +301,11 @@ func acquireNodeSlot(ctx context.Context, limiter *semaphore.Weighted, node *Pla // runState carries the per-run() state every node's goroutine shares: the // done-channel per node, dependency-failure bookkeeping, group event -// tracking, the concurrency limiter, and the phase-scoped cancellation a -// node joins depending on its own Phase. Split out of run() purely to keep -// runNode a plain method instead of a closure captured in a loop. +// tracking, and the concurrency limiter. Shared across both phases (built +// once in run(), before either runs) since done/failures must stay visible +// to a Start-phase node depending on a Create-phase one across the barrier +// between them. Split out of run() purely to keep runNode a plain method +// instead of a closure captured in a loop. type runState struct { done map[int]chan struct{} recordFailure func(id int, err error) @@ -286,19 +313,13 @@ type runState struct { groups *groupTracker events api.EventProcessor limiter *semaphore.Weighted - createPhase *phaseCancel - startPhase *phaseCancel } // runNode executes one plan node: waits for its dependencies, skips it if -// one of them failed, then dispatches it and records the outcome. See run() -// for the phase-scoped cancellation (createPhase/startPhase) this draws -// from. -func (exec *planExecutor) runNode(node *PlanNode, rs *runState) error { - phase := rs.createPhase - if node.Phase == PhaseStart { - phase = rs.startPhase - } +// one of them failed, then dispatches it and records the outcome. phase is +// the cancellation scope of the runPhase call driving it -- see runPhase and +// phaseCancel's own doc comments. +func (exec *planExecutor) runNode(node *PlanNode, rs *runState, phase *phaseCancel) error { nodeCtx := phase.ctx // Every exit path below must close rs.done[node.ID] exactly once, or a diff --git a/pkg/compose/executor_start_test.go b/pkg/compose/executor_start_test.go index 294b6cc11..61490a3ed 100644 --- a/pkg/compose/executor_start_test.go +++ b/pkg/compose/executor_start_test.go @@ -24,6 +24,7 @@ import ( "fmt" "net" "sync" + "sync/atomic" "testing" "time" @@ -264,56 +265,44 @@ func TestExecutePlanWaitDeadlineOnlyBoundsWaitNodes(t *testing.T) { assert.Assert(t, elapsed < 5*time.Second, "waitTimeout did not bound the stuck OpWaitCondition node (took %s)", elapsed) } -// TestExecutePlanStartPhaseFailureDoesNotCancelCreatePhase is a regression -// test for a Copilot review finding on #14290: run() used to dispatch every -// node under one shared errgroup.WithContext, so a Start-phase node failing -// (a required OpWaitCondition hitting its own waitTimeout, here) canceled -// ctx for every other goroutine still in flight -- including a completely -// unrelated Create-phase node (a sibling service's network create) that has -// nothing to do with the dependency that timed out. The old -// create()-then-start() sequence never had this failure mode: the whole -// create phase had always finished before any dependency wait began. This -// pairs a never-satisfied required wait (Phase: PhaseStart, matching what -// planStartPhase actually produces) with a network create (Phase: -// PhaseCreate, the default) deliberately slower than the wait's timeout, and -// asserts the network create's own context is never canceled -- it must -// complete normally, on its own schedule, even though the sibling wait -// already failed and the plan as a whole still fails too. -func TestExecutePlanStartPhaseFailureDoesNotCancelCreatePhase(t *testing.T) { - svc, apiClient, _ := newStartPhaseTestService(t) - - dbSummary := container.Summary{ - ID: "db-id", - Names: []string{"/test-db-1"}, - Labels: map[string]string{api.ServiceLabel: "db", api.OneoffLabel: "False"}, - } - apiClient.EXPECT().ContainerInspect(gomock.Any(), "db-id", gomock.Any()).Return(client.ContainerInspectResult{ - Container: container.InspectResponse{ - ID: "db-id", - Name: "/test-db-1", - State: &container.State{ - Status: container.StateRunning, - Health: &container.Health{Status: container.Starting}, - }, - Config: &container.Config{Healthcheck: &container.HealthConfig{Test: []string{"CMD", "true"}}}, - }, - }, nil).AnyTimes() +// TestExecutePlanCreatePhaseIsABarrierBeforeStartPhase is a regression test +// for a Copilot review finding on #14291: a Start-phase node's own DependsOn +// edges being satisfied is not enough to let it run -- nothing stopped it +// from starting while a COMPLETELY UNRELATED Create-phase node elsewhere in +// the plan was still in flight (or about to fail), something the old +// create()-then-start() sequence never allowed (the whole create phase, +// project-wide, always finished -- or failed, with start() never invoked at +// all -- before any dependency wait, or anything else in the start phase, +// even began). Interactive up's attach/printer session needs that same +// guarantee: it takes exclusive hold of the terminal for the create phase's +// progress display, handing it to continuous container log streaming only +// once the create phase is entirely done. This pairs a slow, unrelated +// network create (Phase: PhaseCreate, the default, no DependsOn edge to +// anything in the Start phase) with an OpStartContainer node (Phase: +// PhaseStart) and asserts ContainerStart is never called before the network +// create has fully returned. Uses OpStartContainer rather than +// OpWaitCondition deliberately: execWaitCondition's underlying poll loop +// only checks on a 500ms ticker (never on entry), which would make a +// same-order-of-magnitude Create delay pass this assertion even with no +// barrier at all, and so couldn't actually catch a regression here. +func TestExecutePlanCreatePhaseIsABarrierBeforeStartPhase(t *testing.T) { + svc, apiClient := newTestService(t) + var createDone atomic.Bool nw := types.NetworkConfig{Name: "test_default"} - var networkCreateCtxErr error - networkCreateDone := make(chan struct{}) apiClient.EXPECT().NetworkCreate(gomock.Any(), "test_default", gomock.Any()). - DoAndReturn(func(ctx context.Context, _ string, _ client.NetworkCreateOptions) (client.NetworkCreateResult, error) { - defer close(networkCreateDone) - // Deliberately outlives the wait's timeout below: if the - // Create-phase ctx were canceled by the sibling Start-phase wait - // failing, this ctx would already be done well before this - // returns. - time.Sleep(300 * time.Millisecond) - networkCreateCtxErr = ctx.Err() + DoAndReturn(func(context.Context, string, client.NetworkCreateOptions) (client.NetworkCreateResult, error) { + time.Sleep(100 * time.Millisecond) + createDone.Store(true) return client.NetworkCreateResult{ID: "net-id"}, nil }) + apiClient.EXPECT().ContainerStart(gomock.Any(), "db-id", gomock.Any()). + DoAndReturn(func(context.Context, string, client.ContainerStartOptions) (client.ContainerStartResult, error) { + assert.Assert(t, createDone.Load(), "the Start-phase container start began before the unrelated Create-phase network create returned") + return client.ContainerStartResult{}, nil + }) + plan := &Plan{} plan.addNode(Operation{ Type: OpCreateNetwork, @@ -321,36 +310,25 @@ func TestExecutePlanStartPhaseFailureDoesNotCancelCreatePhase(t *testing.T) { Name: nw.Name, Network: &nw, }, "") - wait := plan.addNode(Operation{ - Type: OpWaitCondition, - ResourceID: "wait:db:service_healthy", - Name: "db", - Condition: types.ServiceConditionHealthy, + start := plan.addNode(Operation{ + Type: OpStartContainer, + ResourceID: "service:db:1", + Container: &container.Summary{ID: "db-id", Names: []string{"/test-db-1"}}, }, "") - wait.Phase = PhaseStart + start.Phase = PhaseStart exec := svc.newPlanExecutor(&types.Project{Name: "test"}, emptyObservedState("test"), nil) - exec.containersByService["db"] = Containers{dbSummary} - exec.waitTimeout = 100 * time.Millisecond - err := exec.run(t.Context(), plan) - assert.ErrorIs(t, err, context.DeadlineExceeded) - - select { - case <-networkCreateDone: - case <-time.After(5 * time.Second): - t.Fatal("NetworkCreate never completed -- its context was likely canceled by the sibling Start-phase wait timing out") - } - assert.NilError(t, networkCreateCtxErr, "NetworkCreate's own ctx must not be canceled by an unrelated Start-phase failure") + assert.NilError(t, exec.run(t.Context(), plan)) + assert.Assert(t, createDone.Load()) } -// TestExecutePlanCreatePhaseFailureCancelsCreatePhase is the mirror image of -// TestExecutePlanStartPhaseFailureDoesNotCancelCreatePhase: phase-scoped -// cancellation must not weaken the Create phase's own existing fail-fast +// TestExecutePlanCreatePhaseFailureCancelsCreatePhase verifies phase-scoped +// cancellation didn't weaken the Create phase's own existing fail-fast // behavior -- two independent (no DependsOn edge) Create-phase nodes, one // failing immediately, must still cancel the other's in-flight work. Both // nodes default to Phase: PhaseCreate (addNode's zero value), so they share -// createCtx. +// the same runPhase call's phaseCancel. func TestExecutePlanCreatePhaseFailureCancelsCreatePhase(t *testing.T) { svc, apiClient := newTestService(t) @@ -397,6 +375,42 @@ func TestExecutePlanCreatePhaseFailureCancelsCreatePhase(t *testing.T) { assert.ErrorIs(t, volumeCreateCtxErr, context.Canceled, "VolumeCreate's ctx must be canceled by a sibling Create-phase node failing") } +// TestExecutePlanCreatePhaseFailureNeverDispatchesStartPhase complements +// TestExecutePlanCreatePhaseIsABarrierBeforeStartPhase: that test proves the +// Start phase waits for a successful Create phase, this one proves a failed +// Create phase skips the Start phase entirely rather than dispatching it and +// canceling it. run() must return the Create-phase error before runPhase is +// ever called on startNodes -- ContainerStart's .Times(0) fails the test the +// instant it's called at all, not just if it fails to be called. +func TestExecutePlanCreatePhaseFailureNeverDispatchesStartPhase(t *testing.T) { + svc, apiClient := newTestService(t) + + apiClient.EXPECT().NetworkCreate(gomock.Any(), "test_default", gomock.Any()). + Return(client.NetworkCreateResult{}, errors.New("boom")) + apiClient.EXPECT().ContainerStart(gomock.Any(), gomock.Any(), gomock.Any()).Times(0) + + nw := types.NetworkConfig{Name: "test_default"} + + plan := &Plan{} + plan.addNode(Operation{ + Type: OpCreateNetwork, + ResourceID: "network:default", + Name: nw.Name, + Network: &nw, + }, "") + start := plan.addNode(Operation{ + Type: OpStartContainer, + ResourceID: "service:db:1", + Container: &container.Summary{ID: "db-id", Names: []string{"/test-db-1"}}, + }, "") + start.Phase = PhaseStart + + exec := svc.newPlanExecutor(&types.Project{Name: "test"}, emptyObservedState("test"), nil) + + err := exec.run(t.Context(), plan) + assert.ErrorContains(t, err, "boom") +} + // TestExecutePlanConcurrentPhaseFailureKeepsOriginatingError exercises a // Copilot/docker-agent finding on the phase-scoped cancellation fix: // errgroup.Group's plain Wait() returns whichever goroutine's error reaches