fix(reconcile): keep each dependency's wait timeout independent

Reverts 365f0026f, a behavior-parity fix for a Copilot finding that the
plan-based start engine's per-(dependency,condition) wait node could
start its exec.waitTimeout clock earlier than the old imperative
engine's single waitDependencies(X) call -- which only begins once
EVERY direct dependency of X has itself finished starting, so a slow
sibling with no wait node of its own could silently extend another
dependency's effective grace period.

On reflection that old-engine coupling is itself the bug: a service's
declared --wait-timeout for a given dependency should bound how long
THAT dependency takes to satisfy its condition, period -- not be
inflated by how long an unrelated sibling dependency happens to take
to start. Letting db's health-check timeout silently grow because
migrate's own dependency chain is slow means --wait-timeout no longer
means what it says. Restoring the narrower, per-dependency gating is a
deliberate, accepted behavior change from the old engine, not a parity
regression to chase.

This also removes the dependency-cycle risk the sibling-widening
introduced (an ordinary diamond dependency, or a service whose own
chain end collapses onto a shared wait node, could merge a node back
into its own transitive dependents -- see 365f0026f's description) and
the wouldCreateCycle machinery and five regression tests that existed
solely to guard against it, now dead weight with no reachable scenario
to protect against.

Verified: go build, go test ./pkg/compose/..., and golangci-lint all
clean.

Signed-off-by: Nicolas De Loof <nicolas.deloof@gmail.com>
This commit is contained in:
Nicolas De Loof 2026-10-07 07:56:27 +02:00 • committed by Guillaume Lours
parent 4c82be2569
commit c87307b89f
2 changed files with 12 additions and 307 deletions

View file

@ -1269,52 +1269,24 @@ func (r *reconciler) startPhaseReplicas(service types.ServiceConfig) (replicas [
// gate — errors included: a dependency that is neither declared nor
// disabled is the same hard failure it is today.
func (r *reconciler) startPhaseDependencies(service types.ServiceConfig) ([]*PlanNode, error) {
deps := sortedKeys(service.DependsOn)
depTargets := make(map[string][]*PlanNode, len(deps))
var allTargets []*PlanNode
for _, dep := range deps {
var depNodes []*PlanNode
for _, dep := range sortedKeys(service.DependsOn) {
cfg := service.DependsOn[dep]
targets := r.startChainEnds[dep]
if len(targets) == 0 {
if node, ok := r.serviceNodes[dep]; ok {
targets = []*PlanNode{node}
}
}
depTargets[dep] = targets
for _, target := range targets {
if !slices.Contains(allTargets, target) {
allTargets = append(allTargets, target)
}
}
}
var depNodes []*PlanNode
for _, dep := range deps {
cfg := service.DependsOn[dep]
shouldWait, err := shouldWaitForDependency(dep, cfg, r.project)
if err != nil {
return nil, err
}
if !shouldWait {
for _, target := range depTargets[dep] {
if !slices.Contains(depNodes, target) {
depNodes = append(depNodes, target)
}
}
depNodes = append(depNodes, targets...)
continue
}
// The wait node's own exec.waitTimeout window must not open until
// EVERY dependency this service declares has itself finished
// starting, not just the one this particular condition checks --
// matching the old imperative engine's startService, which only
// calls waitDependencies(X) once every direct dependency's own
// startService call has already returned (see InDependencyOrder's
// graphTraversal.run: a node is dispatched only once none of its
// children are still ServiceStopped, so X's siblings all finish
// starting before X's own wait clock ever starts). Passing just this
// dep's own target here would let an unrelated, slower sibling
// dependency silently shrink the effective grace period this
// condition's health check gets, compared to the old engine.
depNodes = append(depNodes, r.waitConditionNode(dep, cfg, allTargets))
depNodes = append(depNodes, r.waitConditionNode(dep, cfg, targets))
}
return depNodes, nil
}
@ -1322,13 +1294,7 @@ func (r *reconciler) startPhaseDependencies(service types.ServiceConfig) ([]*Pla
// waitConditionNode returns the shared wait node for (dep, condition),
// creating it on first use. required:false marks it best-effort — a missing
// dependency is skipped, not fatal; one required dependent upgrades the
// shared node for everyone. targets is now each caller's FULL sibling
// dependency set (see startPhaseDependencies), not just dep's own target, so
// two dependents of the same (dep, condition) with different sibling sets
// routinely diverge here: the merge below unions them onto the one shared
// node rather than creating a second one, at the cost of a dependent with
// fewer siblings waiting a little longer than its own depends_on strictly
// requires -- preferable to losing the dedup this node exists for.
// shared node for everyone.
func (r *reconciler) waitConditionNode(dep string, cfg types.ServiceDependency, targets []*PlanNode) *PlanNode {
key := dep + ":" + cfg.Condition
wait, ok := r.waitNodes[key]
@ -1348,55 +1314,18 @@ func (r *reconciler) waitConditionNode(dep string, cfg types.ServiceDependency,
if cfg.Required && wait.Operation.BestEffort {
wait.Operation.BestEffort = false
}
// merge this caller's prerequisites into the shared node, skipping
// anything that would create a cycle: an ordinary diamond dependency
// (X depends on dep:cond, Z depends on X AND independently on the same
// dep:cond) puts X's own chain end -- which already depends on this very
// wait node -- into Z's sibling set. Adding it back as wait's own
// prerequisite would deadlock the executor forever (runNode never applies
// exec.waitTimeout until its own DependsOn are satisfied, and nothing
// detects a node depending on itself at a distance). Skipping it loses
// nothing: X cannot even start until wait has already succeeded, so by
// the time X's chain completes wait is necessarily already done.
// merge this caller's prerequisites into the shared node. Today every
// caller passes the same targets (startChainEnds[dep] is fixed before
// any dependent is visited), so this is defensive — but relying on that
// silently would break the day the targets diverge per caller.
for _, target := range targets {
if slices.Contains(wait.DependsOn, target) {
continue
if !slices.Contains(wait.DependsOn, target) {
wait.DependsOn = append(wait.DependsOn, target)
}
if wouldCreateCycle(target, wait) {
continue
}
wait.DependsOn = append(wait.DependsOn, target)
}
return wait
}
// wouldCreateCycle reports whether candidate already (transitively) depends
// on node, via DependsOn edges -- i.e. whether appending candidate to
// node.DependsOn would close a cycle. Only reachable from waitConditionNode's
// merge, where candidate is a sibling target pulled in for an already-shared
// wait node.
func wouldCreateCycle(candidate, node *PlanNode) bool {
if candidate == node {
return true
}
seen := map[*PlanNode]bool{candidate: true}
queue := []*PlanNode{candidate}
for len(queue) > 0 {
n := queue[0]
queue = queue[1:]
for _, dep := range n.DependsOn {
if dep == node {
return true
}
if !seen[dep] {
seen[dep] = true
queue = append(queue, dep)
}
}
}
return false
}
// planProviderStart is the start-phase visit of a provider service: no
// container to start, but its own depends_on still applies — the imperative
// startService waits on it for every service, providers included. Its chain

View file

@ -132,230 +132,6 @@ func TestPlanStart_HealthyConditionDeduplicated(t *testing.T) {
`)+"\n")
}
// TestPlanStart_WaitNodeGatedOnAllSiblingDependencies is a regression test
// for a Copilot review finding on #14290's Create->Start barrier work: a
// wait node's own exec.waitTimeout window used to open as soon as ITS OWN
// dependency's chain finished, ignoring the requesting service's other
// depends_on entries entirely. The old imperative engine never allowed that
// -- InDependencyOrder only calls startService(app), and so only starts
// app's own waitDependencies clock, once EVERY one of app's direct
// dependencies has itself finished starting (see graphTraversal.run: a node
// is dispatched only once none of its children are still ServiceStopped). A
// slow sibling with no wait node of its own (service_started is a plain
// edge, no condition to poll) could silently eat into a health check's
// effective grace period. Here app depends on db:service_healthy (needs a
// wait node) and migrate:service_started (a plain edge, no wait node) -- the
// db wait node must still depend on migrate's own start, or its
// exec.waitTimeout clock starts too early relative to the old engine.
func TestPlanStart_WaitNodeGatedOnAllSiblingDependencies(t *testing.T) {
db := types.ServiceConfig{Name: "db"}
db.HealthCheck = &types.HealthCheckConfig{Test: []string{"CMD", "true"}}
migrate := types.ServiceConfig{Name: "migrate"}
app := serviceWithDeps("app", types.DependsOnConfig{
"db": {Condition: types.ServiceConditionHealthy, Required: true},
"migrate": {Condition: types.ServiceConditionStarted, Required: true},
})
project := &types.Project{
Name: "myproject",
Services: types.Services{"db": db, "migrate": migrate, "app": app},
}
plan, err := reconcile(t.Context(), project, emptyObserved(), startScopeOptions(ScopeCreateStart), noPrompt)
assert.NilError(t, err)
var wait, dbStart, migrateStart *PlanNode
for _, n := range plan.Nodes {
switch {
case n.Operation.Type == OpWaitCondition:
wait = n
case n.Operation.Type == OpStartContainer && n.Operation.ResourceID == "service:db:1":
dbStart = n
case n.Operation.Type == OpStartContainer && n.Operation.ResourceID == "service:migrate:1":
migrateStart = n
}
}
if wait == nil || dbStart == nil || migrateStart == nil {
t.Fatalf("expected a db wait node and both db/migrate start nodes:\n%s", plan)
}
assert.Assert(t, slices.Contains(wait.DependsOn, dbStart),
"db's wait node must depend on db's own start:\n%s", plan)
assert.Assert(t, slices.Contains(wait.DependsOn, migrateStart),
"db's wait node must also depend on app's other sibling dependency (migrate) finishing its own start, not just db's:\n%s", plan)
}
// nodeInOwnDependencyClosure reports whether n is reachable from itself by
// following DependsOn edges -- i.e. whether the plan has a dependency cycle
// through n.
func nodeInOwnDependencyClosure(n *PlanNode) bool {
seen := map[*PlanNode]bool{}
queue := slices.Clone(n.DependsOn)
for len(queue) > 0 {
cur := queue[0]
queue = queue[1:]
if cur == n {
return true
}
if seen[cur] {
continue
}
seen[cur] = true
queue = append(queue, cur.DependsOn...)
}
return false
}
func assertNoCycle(t *testing.T, plan *Plan) {
t.Helper()
for _, n := range plan.Nodes {
if nodeInOwnDependencyClosure(n) {
t.Fatalf("plan has a dependency cycle through node #%d (%s):\n%s", n.ID, n.Operation.ResourceID, plan)
}
}
}
// TestPlanStart_DiamondDependencyNeverCreatesWaitCycle is a regression test
// for a Copilot review finding on the allTargets widening above: an entirely
// ordinary diamond dependency -- app depends on db:service_healthy, worker
// depends on BOTH app:service_started and, independently, db:service_healthy
// -- shares db's wait node between app and worker (same (dep,condition)
// key). app's own start node already depends on that wait node. Widening the
// wait node's own prerequisites with worker's sibling set (app's start node,
// pulled in because worker also depends on app) would append app's start
// node back into the wait node's own DependsOn -- a direct cycle the
// executor can never resolve (runNode blocks on rs.done[dep.ID] for every
// dependency before closing its own, with nothing to detect or time out a
// cycle). The wait node must keep depending on db alone; worker's own start
// node separately depends on both the wait node and app's start node, which
// already encodes everything worker's depends_on requires -- nothing is
// lost by not also routing it through the shared wait node.
func TestPlanStart_DiamondDependencyNeverCreatesWaitCycle(t *testing.T) {
db := types.ServiceConfig{Name: "db"}
db.HealthCheck = &types.HealthCheckConfig{Test: []string{"CMD", "true"}}
app := serviceWithDeps("app", types.DependsOnConfig{
"db": {Condition: types.ServiceConditionHealthy, Required: true},
})
worker := serviceWithDeps("worker", types.DependsOnConfig{
"app": {Condition: types.ServiceConditionStarted, Required: true},
"db": {Condition: types.ServiceConditionHealthy, Required: true},
})
project := &types.Project{
Name: "myproject",
Services: types.Services{"db": db, "app": app, "worker": worker},
}
plan, err := reconcile(t.Context(), project, emptyObserved(), startScopeOptions(ScopeCreateStart), noPrompt)
assert.NilError(t, err)
assertNoCycle(t, plan)
var wait, appStart *PlanNode
for _, n := range plan.Nodes {
switch {
case n.Operation.Type == OpWaitCondition:
wait = n
case n.Operation.Type == OpStartContainer && n.Operation.ResourceID == "service:app:1":
appStart = n
}
}
if wait == nil || appStart == nil {
t.Fatalf("expected db's wait node and app's start node:\n%s", plan)
}
assert.Assert(t, !slices.Contains(wait.DependsOn, appStart),
"db's wait node must not depend on app's start node -- app's own start already depends on the wait node, so this would be a cycle:\n%s", plan)
assert.Assert(t, slices.Contains(appStart.DependsOn, wait),
"app's own start must still depend on db's wait node:\n%s", plan)
}
// TestPlanStart_ExceptionalChainCollapseNeverCreatesWaitCycle is a second
// regression test for the same cycle risk, reproducing the exact shape found
// during review: a steady-state up where every service is already running.
// "a" depends only on db:service_healthy and has nothing left to start, so
// planServiceStart's "nothing to start" branch sets startChainEnds["a"] to
// literally BE the shared db:service_healthy wait node (ends := depNodes).
// "b" depends on both db:service_healthy and a:service_started -- its
// allTargets then contains that very wait node as one of "a"'s targets, and
// merging it into itself would be a direct self-loop with no other node
// in between.
func TestPlanStart_ExceptionalChainCollapseNeverCreatesWaitCycle(t *testing.T) {
db := types.ServiceConfig{Name: "db"}
db.HealthCheck = &types.HealthCheckConfig{Test: []string{"CMD", "true"}}
a := serviceWithDeps("a", types.DependsOnConfig{
"db": {Condition: types.ServiceConditionHealthy, Required: true},
})
b := serviceWithDeps("b", types.DependsOnConfig{
"db": {Condition: types.ServiceConditionHealthy, Required: true},
"a": {Condition: types.ServiceConditionStarted, Required: true},
})
project := &types.Project{
Name: "myproject",
Services: types.Services{"db": db, "a": a, "b": b},
}
observed := emptyObserved()
for _, svc := range []types.ServiceConfig{db, a, b} {
hash, err := serviceHashWithResolvedRefs(svc, nil)
assert.NilError(t, err)
observed.Containers[svc.Name] = []ObservedContainer{
observedServiceContainer(svc.Name, 1, container.StateRunning, hash),
}
}
plan, err := reconcile(t.Context(), project, observed, startScopeOptions(ScopeCreateStart), noPrompt)
assert.NilError(t, err)
assertNoCycle(t, plan)
for _, n := range plan.Nodes {
if n.Operation.Type == OpWaitCondition {
assert.Assert(t, !slices.Contains(n.DependsOn, n),
"wait node #%d must not depend on itself:\n%s", n.ID, plan)
}
}
}
// TestPlanStart_TransitiveChainNeverCreatesWaitCycle exercises
// wouldCreateCycle's BFS beyond a single hop: consumer's sibling "app" isn't
// itself the shared wait node, nor does it depend on it directly -- app's
// own start node depends on mid's start node, which depends on the shared
// db:service_healthy wait node two hops away (app -> mid -> wait). Merging
// app's start node into the wait node's own DependsOn would still close a
// (longer) cycle, so wouldCreateCycle must walk the full closure, not just
// candidate's immediate DependsOn.
func TestPlanStart_TransitiveChainNeverCreatesWaitCycle(t *testing.T) {
db := types.ServiceConfig{Name: "db"}
db.HealthCheck = &types.HealthCheckConfig{Test: []string{"CMD", "true"}}
mid := serviceWithDeps("mid", types.DependsOnConfig{
"db": {Condition: types.ServiceConditionHealthy, Required: true},
})
app := serviceWithDeps("app", types.DependsOnConfig{
"mid": {Condition: types.ServiceConditionStarted, Required: true},
})
consumer := serviceWithDeps("consumer", types.DependsOnConfig{
"app": {Condition: types.ServiceConditionStarted, Required: true},
"db": {Condition: types.ServiceConditionHealthy, Required: true},
})
project := &types.Project{
Name: "myproject",
Services: types.Services{"db": db, "mid": mid, "app": app, "consumer": consumer},
}
plan, err := reconcile(t.Context(), project, emptyObserved(), startScopeOptions(ScopeCreateStart), noPrompt)
assert.NilError(t, err)
assertNoCycle(t, plan)
var wait, appStart *PlanNode
for _, n := range plan.Nodes {
switch {
case n.Operation.Type == OpWaitCondition:
wait = n
case n.Operation.Type == OpStartContainer && n.Operation.ResourceID == "service:app:1":
appStart = n
}
}
if wait == nil || appStart == nil {
t.Fatalf("expected db's wait node and app's start node:\n%s", plan)
}
assert.Assert(t, !slices.Contains(wait.DependsOn, appStart),
"db's wait node must not depend on app's start node two hops away (app -> mid -> wait) -- that would still be a cycle:\n%s", plan)
}
// pre_start runs once per service before the first replica start, only when
// no replica was running at observation; replicas start sequentially, each
// chain link (start, then post_start when declared) gating the next.