From 97943ec9fb2cbe53bcdc170e07cde275d20ef3a0 Mon Sep 17 00:00:00 2001 From: Nicolas De Loof Date: Thu, 15 Oct 2026 13:01:35 +0200 Subject: [PATCH] fix(executor): record a dependency wait canceled by its phase as a failure A node that leaves its dependency wait because its phase was canceled closes its done channel (deferred) without recording any failure. In X -> A -> B with X failing, B could select done[A] before it saw the cancellation, find no failure recorded for A, emit its "Starting" event and enter executeNode on a canceled context, leaving the row never finalized. Record the cancellation as the node's failure at that exit, so a dependent that wins the race is skipped like any other dependent of a failed node. The regression test pins GOMAXPROCS to 1, which makes the bad interleaving the usual one. Signed-off-by: Nicolas De Loof --- pkg/compose/executor.go | 10 ++++++++- pkg/compose/executor_start_test.go | 36 ++++++++++++++++++++++++++++++ 2 files changed, 45 insertions(+), 1 deletion(-) diff --git a/pkg/compose/executor.go b/pkg/compose/executor.go index 45a9ab326..e3e37f1c8 100644 --- a/pkg/compose/executor.go +++ b/pkg/compose/executor.go @@ -337,7 +337,15 @@ func (exec *planExecutor) runNode(node *PlanNode, rs *runState, phase *phaseCanc select { case <-rs.done[dep.ID]: case <-nodeCtx.Done(): - return nodeCtx.Err() + // Record the cancellation like any other failure: the deferred + // close below makes done[node.ID] ready, and a dependent that + // selects it before seeing nodeCtx.Done() would otherwise find + // no failure recorded for this node, then emit a "Starting" + // event it never finalizes and enter executeNode on a canceled + // context. + err := nodeCtx.Err() + rs.recordFailure(node.ID, err) + return err } } diff --git a/pkg/compose/executor_start_test.go b/pkg/compose/executor_start_test.go index 2981f4c5b..cd72ed1e3 100644 --- a/pkg/compose/executor_start_test.go +++ b/pkg/compose/executor_start_test.go @@ -23,6 +23,7 @@ import ( "errors" "fmt" "net" + "runtime" "sync" "sync/atomic" "testing" @@ -498,6 +499,41 @@ func TestExecutePlanCreatePhaseFailureCancelsCreatePhase(t *testing.T) { assert.ErrorIs(t, volumeCreateCtxErr, context.Canceled, "VolumeCreate's ctx must be canceled by a sibling Create-phase node failing, not by the test's own deadline") } +// TestExecutePlanCanceledDependencyIsRecordedAsFailure pins the failure a +// node records when it leaves its dependency wait because its phase was +// canceled. In X -> A -> B with X failing without any I/O, A bails out on the +// cancellation and its done channel is closed; B may select that channel +// before it sees the cancellation itself. With no failure recorded for A, B +// then went on to start and enter executeNode on a canceled context. B must +// be skipped. The bad interleaving needs B to reach its select late, when +// both channels are already ready and the select picks one at random: a +// single P makes that the usual case, and the repetition covers the rest of +// the randomness. +func TestExecutePlanCanceledDependencyIsRecordedAsFailure(t *testing.T) { + prev := runtime.GOMAXPROCS(1) + t.Cleanup(func() { runtime.GOMAXPROCS(prev) }) + + svc, apiClient := newTestService(t) + + apiClient.EXPECT().NetworkCreate(gomock.Any(), "test_default", gomock.Any()). + Return(client.NetworkCreateResult{}, errors.New("boom")).AnyTimes() + apiClient.EXPECT().VolumeCreate(gomock.Any(), gomock.Any()).Times(0) + + nw := types.NetworkConfig{Name: "test_default"} + volA := types.VolumeConfig{Name: "a", Driver: "local"} + volB := types.VolumeConfig{Name: "b", Driver: "local"} + + for range 200 { + plan := &Plan{} + x := plan.addNode(Operation{Type: OpCreateNetwork, ResourceID: "network:default", Name: nw.Name, Network: &nw}, "") + a := plan.addNode(Operation{Type: OpCreateVolume, ResourceID: "volume:a", Name: volA.Name, Volume: &volA}, "", x) + plan.addNode(Operation{Type: OpCreateVolume, ResourceID: "volume:b", Name: volB.Name, Volume: &volB}, "", a) + + exec := svc.newPlanExecutor(&types.Project{Name: "test"}, emptyObservedState("test"), nil) + assert.ErrorContains(t, exec.run(t.Context(), plan), "boom") + } +} + // TestExecutePlanCreatePhaseFailureNeverDispatchesStartPhase complements // TestExecutePlanCreatePhaseIsABarrierBeforeStartPhase: that test proves the // Start phase waits for a successful Create phase, this one proves a failed