fix(executor): record a dependency wait canceled by its phase as a failure

A node that leaves its dependency wait because its phase was canceled
closes its done channel (deferred) without recording any failure. In
X -> A -> B with X failing, B could select done[A] before it saw the
cancellation, find no failure recorded for A, emit its "Starting" event
and enter executeNode on a canceled context, leaving the row never
finalized.

Record the cancellation as the node's failure at that exit, so a
dependent that wins the race is skipped like any other dependent of a
failed node. The regression test pins GOMAXPROCS to 1, which makes the
bad interleaving the usual one.

Signed-off-by: Nicolas De Loof <nicolas.deloof@gmail.com>
This commit is contained in:
Nicolas De Loof 2026-10-15 13:01:35 +02:00 • committed by Guillaume Lours
parent cdbfe54617
commit 97943ec9fb
2 changed files with 45 additions and 1 deletions

View file

@ -337,7 +337,15 @@ func (exec *planExecutor) runNode(node *PlanNode, rs *runState, phase *phaseCanc
select {
case <-rs.done[dep.ID]:
case <-nodeCtx.Done():
return nodeCtx.Err()
// Record the cancellation like any other failure: the deferred
// close below makes done[node.ID] ready, and a dependent that
// selects it before seeing nodeCtx.Done() would otherwise find
// no failure recorded for this node, then emit a "Starting"
// event it never finalizes and enter executeNode on a canceled
// context.
err := nodeCtx.Err()
rs.recordFailure(node.ID, err)
return err
}
}

View file

@ -23,6 +23,7 @@ import (
"errors"
"fmt"
"net"
"runtime"
"sync"
"sync/atomic"
"testing"
@ -498,6 +499,41 @@ func TestExecutePlanCreatePhaseFailureCancelsCreatePhase(t *testing.T) {
assert.ErrorIs(t, volumeCreateCtxErr, context.Canceled, "VolumeCreate's ctx must be canceled by a sibling Create-phase node failing, not by the test's own deadline")
}
// TestExecutePlanCanceledDependencyIsRecordedAsFailure pins the failure a
// node records when it leaves its dependency wait because its phase was
// canceled. In X -> A -> B with X failing without any I/O, A bails out on the
// cancellation and its done channel is closed; B may select that channel
// before it sees the cancellation itself. With no failure recorded for A, B
// then went on to start and enter executeNode on a canceled context. B must
// be skipped. The bad interleaving needs B to reach its select late, when
// both channels are already ready and the select picks one at random: a
// single P makes that the usual case, and the repetition covers the rest of
// the randomness.
func TestExecutePlanCanceledDependencyIsRecordedAsFailure(t *testing.T) {
prev := runtime.GOMAXPROCS(1)
t.Cleanup(func() { runtime.GOMAXPROCS(prev) })
svc, apiClient := newTestService(t)
apiClient.EXPECT().NetworkCreate(gomock.Any(), "test_default", gomock.Any()).
Return(client.NetworkCreateResult{}, errors.New("boom")).AnyTimes()
apiClient.EXPECT().VolumeCreate(gomock.Any(), gomock.Any()).Times(0)
nw := types.NetworkConfig{Name: "test_default"}
volA := types.VolumeConfig{Name: "a", Driver: "local"}
volB := types.VolumeConfig{Name: "b", Driver: "local"}
for range 200 {
plan := &Plan{}
x := plan.addNode(Operation{Type: OpCreateNetwork, ResourceID: "network:default", Name: nw.Name, Network: &nw}, "")
a := plan.addNode(Operation{Type: OpCreateVolume, ResourceID: "volume:a", Name: volA.Name, Volume: &volA}, "", x)
plan.addNode(Operation{Type: OpCreateVolume, ResourceID: "volume:b", Name: volB.Name, Volume: &volB}, "", a)
exec := svc.newPlanExecutor(&types.Project{Name: "test"}, emptyObservedState("test"), nil)
assert.ErrorContains(t, exec.run(t.Context(), plan), "boom")
}
}
// TestExecutePlanCreatePhaseFailureNeverDispatchesStartPhase complements
// TestExecutePlanCreatePhaseIsABarrierBeforeStartPhase: that test proves the
// Start phase waits for a successful Create phase, this one proves a failed