From c87307b89fc8b05f136ee2549ada0c5b0c8e730f Mon Sep 17 00:00:00 2001 From: Nicolas De Loof Date: Wed, 7 Oct 2026 07:56:27 +0200 Subject: [PATCH] fix(reconcile): keep each dependency's wait timeout independent Reverts 365f0026f, a behavior-parity fix for a Copilot finding that the plan-based start engine's per-(dependency,condition) wait node could start its exec.waitTimeout clock earlier than the old imperative engine's single waitDependencies(X) call -- which only begins once EVERY direct dependency of X has itself finished starting, so a slow sibling with no wait node of its own could silently extend another dependency's effective grace period. On reflection that old-engine coupling is itself the bug: a service's declared --wait-timeout for a given dependency should bound how long THAT dependency takes to satisfy its condition, period -- not be inflated by how long an unrelated sibling dependency happens to take to start. Letting db's health-check timeout silently grow because migrate's own dependency chain is slow means --wait-timeout no longer means what it says. Restoring the narrower, per-dependency gating is a deliberate, accepted behavior change from the old engine, not a parity regression to chase. This also removes the dependency-cycle risk the sibling-widening introduced (an ordinary diamond dependency, or a service whose own chain end collapses onto a shared wait node, could merge a node back into its own transitive dependents -- see 365f0026f's description) and the wouldCreateCycle machinery and five regression tests that existed solely to guard against it, now dead weight with no reachable scenario to protect against. Verified: go build, go test ./pkg/compose/..., and golangci-lint all clean. Signed-off-by: Nicolas De Loof --- pkg/compose/reconcile.go | 95 ++---------- pkg/compose/reconcile_start_test.go | 224 ---------------------------- 2 files changed, 12 insertions(+), 307 deletions(-) diff --git a/pkg/compose/reconcile.go b/pkg/compose/reconcile.go index 6395b317f..af70e484d 100644 --- a/pkg/compose/reconcile.go +++ b/pkg/compose/reconcile.go @@ -1269,52 +1269,24 @@ func (r *reconciler) startPhaseReplicas(service types.ServiceConfig) (replicas [ // gate — errors included: a dependency that is neither declared nor // disabled is the same hard failure it is today. func (r *reconciler) startPhaseDependencies(service types.ServiceConfig) ([]*PlanNode, error) { - deps := sortedKeys(service.DependsOn) - depTargets := make(map[string][]*PlanNode, len(deps)) - var allTargets []*PlanNode - for _, dep := range deps { + var depNodes []*PlanNode + for _, dep := range sortedKeys(service.DependsOn) { + cfg := service.DependsOn[dep] targets := r.startChainEnds[dep] if len(targets) == 0 { if node, ok := r.serviceNodes[dep]; ok { targets = []*PlanNode{node} } } - depTargets[dep] = targets - for _, target := range targets { - if !slices.Contains(allTargets, target) { - allTargets = append(allTargets, target) - } - } - } - - var depNodes []*PlanNode - for _, dep := range deps { - cfg := service.DependsOn[dep] shouldWait, err := shouldWaitForDependency(dep, cfg, r.project) if err != nil { return nil, err } if !shouldWait { - for _, target := range depTargets[dep] { - if !slices.Contains(depNodes, target) { - depNodes = append(depNodes, target) - } - } + depNodes = append(depNodes, targets...) continue } - // The wait node's own exec.waitTimeout window must not open until - // EVERY dependency this service declares has itself finished - // starting, not just the one this particular condition checks -- - // matching the old imperative engine's startService, which only - // calls waitDependencies(X) once every direct dependency's own - // startService call has already returned (see InDependencyOrder's - // graphTraversal.run: a node is dispatched only once none of its - // children are still ServiceStopped, so X's siblings all finish - // starting before X's own wait clock ever starts). Passing just this - // dep's own target here would let an unrelated, slower sibling - // dependency silently shrink the effective grace period this - // condition's health check gets, compared to the old engine. - depNodes = append(depNodes, r.waitConditionNode(dep, cfg, allTargets)) + depNodes = append(depNodes, r.waitConditionNode(dep, cfg, targets)) } return depNodes, nil } @@ -1322,13 +1294,7 @@ func (r *reconciler) startPhaseDependencies(service types.ServiceConfig) ([]*Pla // waitConditionNode returns the shared wait node for (dep, condition), // creating it on first use. required:false marks it best-effort — a missing // dependency is skipped, not fatal; one required dependent upgrades the -// shared node for everyone. targets is now each caller's FULL sibling -// dependency set (see startPhaseDependencies), not just dep's own target, so -// two dependents of the same (dep, condition) with different sibling sets -// routinely diverge here: the merge below unions them onto the one shared -// node rather than creating a second one, at the cost of a dependent with -// fewer siblings waiting a little longer than its own depends_on strictly -// requires -- preferable to losing the dedup this node exists for. +// shared node for everyone. func (r *reconciler) waitConditionNode(dep string, cfg types.ServiceDependency, targets []*PlanNode) *PlanNode { key := dep + ":" + cfg.Condition wait, ok := r.waitNodes[key] @@ -1348,55 +1314,18 @@ func (r *reconciler) waitConditionNode(dep string, cfg types.ServiceDependency, if cfg.Required && wait.Operation.BestEffort { wait.Operation.BestEffort = false } - // merge this caller's prerequisites into the shared node, skipping - // anything that would create a cycle: an ordinary diamond dependency - // (X depends on dep:cond, Z depends on X AND independently on the same - // dep:cond) puts X's own chain end -- which already depends on this very - // wait node -- into Z's sibling set. Adding it back as wait's own - // prerequisite would deadlock the executor forever (runNode never applies - // exec.waitTimeout until its own DependsOn are satisfied, and nothing - // detects a node depending on itself at a distance). Skipping it loses - // nothing: X cannot even start until wait has already succeeded, so by - // the time X's chain completes wait is necessarily already done. + // merge this caller's prerequisites into the shared node. Today every + // caller passes the same targets (startChainEnds[dep] is fixed before + // any dependent is visited), so this is defensive — but relying on that + // silently would break the day the targets diverge per caller. for _, target := range targets { - if slices.Contains(wait.DependsOn, target) { - continue + if !slices.Contains(wait.DependsOn, target) { + wait.DependsOn = append(wait.DependsOn, target) } - if wouldCreateCycle(target, wait) { - continue - } - wait.DependsOn = append(wait.DependsOn, target) } return wait } -// wouldCreateCycle reports whether candidate already (transitively) depends -// on node, via DependsOn edges -- i.e. whether appending candidate to -// node.DependsOn would close a cycle. Only reachable from waitConditionNode's -// merge, where candidate is a sibling target pulled in for an already-shared -// wait node. -func wouldCreateCycle(candidate, node *PlanNode) bool { - if candidate == node { - return true - } - seen := map[*PlanNode]bool{candidate: true} - queue := []*PlanNode{candidate} - for len(queue) > 0 { - n := queue[0] - queue = queue[1:] - for _, dep := range n.DependsOn { - if dep == node { - return true - } - if !seen[dep] { - seen[dep] = true - queue = append(queue, dep) - } - } - } - return false -} - // planProviderStart is the start-phase visit of a provider service: no // container to start, but its own depends_on still applies — the imperative // startService waits on it for every service, providers included. Its chain diff --git a/pkg/compose/reconcile_start_test.go b/pkg/compose/reconcile_start_test.go index 2769d6980..82b243a60 100644 --- a/pkg/compose/reconcile_start_test.go +++ b/pkg/compose/reconcile_start_test.go @@ -132,230 +132,6 @@ func TestPlanStart_HealthyConditionDeduplicated(t *testing.T) { `)+"\n") } -// TestPlanStart_WaitNodeGatedOnAllSiblingDependencies is a regression test -// for a Copilot review finding on #14290's Create->Start barrier work: a -// wait node's own exec.waitTimeout window used to open as soon as ITS OWN -// dependency's chain finished, ignoring the requesting service's other -// depends_on entries entirely. The old imperative engine never allowed that -// -- InDependencyOrder only calls startService(app), and so only starts -// app's own waitDependencies clock, once EVERY one of app's direct -// dependencies has itself finished starting (see graphTraversal.run: a node -// is dispatched only once none of its children are still ServiceStopped). A -// slow sibling with no wait node of its own (service_started is a plain -// edge, no condition to poll) could silently eat into a health check's -// effective grace period. Here app depends on db:service_healthy (needs a -// wait node) and migrate:service_started (a plain edge, no wait node) -- the -// db wait node must still depend on migrate's own start, or its -// exec.waitTimeout clock starts too early relative to the old engine. -func TestPlanStart_WaitNodeGatedOnAllSiblingDependencies(t *testing.T) { - db := types.ServiceConfig{Name: "db"} - db.HealthCheck = &types.HealthCheckConfig{Test: []string{"CMD", "true"}} - migrate := types.ServiceConfig{Name: "migrate"} - app := serviceWithDeps("app", types.DependsOnConfig{ - "db": {Condition: types.ServiceConditionHealthy, Required: true}, - "migrate": {Condition: types.ServiceConditionStarted, Required: true}, - }) - project := &types.Project{ - Name: "myproject", - Services: types.Services{"db": db, "migrate": migrate, "app": app}, - } - - plan, err := reconcile(t.Context(), project, emptyObserved(), startScopeOptions(ScopeCreateStart), noPrompt) - assert.NilError(t, err) - - var wait, dbStart, migrateStart *PlanNode - for _, n := range plan.Nodes { - switch { - case n.Operation.Type == OpWaitCondition: - wait = n - case n.Operation.Type == OpStartContainer && n.Operation.ResourceID == "service:db:1": - dbStart = n - case n.Operation.Type == OpStartContainer && n.Operation.ResourceID == "service:migrate:1": - migrateStart = n - } - } - if wait == nil || dbStart == nil || migrateStart == nil { - t.Fatalf("expected a db wait node and both db/migrate start nodes:\n%s", plan) - } - assert.Assert(t, slices.Contains(wait.DependsOn, dbStart), - "db's wait node must depend on db's own start:\n%s", plan) - assert.Assert(t, slices.Contains(wait.DependsOn, migrateStart), - "db's wait node must also depend on app's other sibling dependency (migrate) finishing its own start, not just db's:\n%s", plan) -} - -// nodeInOwnDependencyClosure reports whether n is reachable from itself by -// following DependsOn edges -- i.e. whether the plan has a dependency cycle -// through n. -func nodeInOwnDependencyClosure(n *PlanNode) bool { - seen := map[*PlanNode]bool{} - queue := slices.Clone(n.DependsOn) - for len(queue) > 0 { - cur := queue[0] - queue = queue[1:] - if cur == n { - return true - } - if seen[cur] { - continue - } - seen[cur] = true - queue = append(queue, cur.DependsOn...) - } - return false -} - -func assertNoCycle(t *testing.T, plan *Plan) { - t.Helper() - for _, n := range plan.Nodes { - if nodeInOwnDependencyClosure(n) { - t.Fatalf("plan has a dependency cycle through node #%d (%s):\n%s", n.ID, n.Operation.ResourceID, plan) - } - } -} - -// TestPlanStart_DiamondDependencyNeverCreatesWaitCycle is a regression test -// for a Copilot review finding on the allTargets widening above: an entirely -// ordinary diamond dependency -- app depends on db:service_healthy, worker -// depends on BOTH app:service_started and, independently, db:service_healthy -// -- shares db's wait node between app and worker (same (dep,condition) -// key). app's own start node already depends on that wait node. Widening the -// wait node's own prerequisites with worker's sibling set (app's start node, -// pulled in because worker also depends on app) would append app's start -// node back into the wait node's own DependsOn -- a direct cycle the -// executor can never resolve (runNode blocks on rs.done[dep.ID] for every -// dependency before closing its own, with nothing to detect or time out a -// cycle). The wait node must keep depending on db alone; worker's own start -// node separately depends on both the wait node and app's start node, which -// already encodes everything worker's depends_on requires -- nothing is -// lost by not also routing it through the shared wait node. -func TestPlanStart_DiamondDependencyNeverCreatesWaitCycle(t *testing.T) { - db := types.ServiceConfig{Name: "db"} - db.HealthCheck = &types.HealthCheckConfig{Test: []string{"CMD", "true"}} - app := serviceWithDeps("app", types.DependsOnConfig{ - "db": {Condition: types.ServiceConditionHealthy, Required: true}, - }) - worker := serviceWithDeps("worker", types.DependsOnConfig{ - "app": {Condition: types.ServiceConditionStarted, Required: true}, - "db": {Condition: types.ServiceConditionHealthy, Required: true}, - }) - project := &types.Project{ - Name: "myproject", - Services: types.Services{"db": db, "app": app, "worker": worker}, - } - - plan, err := reconcile(t.Context(), project, emptyObserved(), startScopeOptions(ScopeCreateStart), noPrompt) - assert.NilError(t, err) - assertNoCycle(t, plan) - - var wait, appStart *PlanNode - for _, n := range plan.Nodes { - switch { - case n.Operation.Type == OpWaitCondition: - wait = n - case n.Operation.Type == OpStartContainer && n.Operation.ResourceID == "service:app:1": - appStart = n - } - } - if wait == nil || appStart == nil { - t.Fatalf("expected db's wait node and app's start node:\n%s", plan) - } - assert.Assert(t, !slices.Contains(wait.DependsOn, appStart), - "db's wait node must not depend on app's start node -- app's own start already depends on the wait node, so this would be a cycle:\n%s", plan) - assert.Assert(t, slices.Contains(appStart.DependsOn, wait), - "app's own start must still depend on db's wait node:\n%s", plan) -} - -// TestPlanStart_ExceptionalChainCollapseNeverCreatesWaitCycle is a second -// regression test for the same cycle risk, reproducing the exact shape found -// during review: a steady-state up where every service is already running. -// "a" depends only on db:service_healthy and has nothing left to start, so -// planServiceStart's "nothing to start" branch sets startChainEnds["a"] to -// literally BE the shared db:service_healthy wait node (ends := depNodes). -// "b" depends on both db:service_healthy and a:service_started -- its -// allTargets then contains that very wait node as one of "a"'s targets, and -// merging it into itself would be a direct self-loop with no other node -// in between. -func TestPlanStart_ExceptionalChainCollapseNeverCreatesWaitCycle(t *testing.T) { - db := types.ServiceConfig{Name: "db"} - db.HealthCheck = &types.HealthCheckConfig{Test: []string{"CMD", "true"}} - a := serviceWithDeps("a", types.DependsOnConfig{ - "db": {Condition: types.ServiceConditionHealthy, Required: true}, - }) - b := serviceWithDeps("b", types.DependsOnConfig{ - "db": {Condition: types.ServiceConditionHealthy, Required: true}, - "a": {Condition: types.ServiceConditionStarted, Required: true}, - }) - project := &types.Project{ - Name: "myproject", - Services: types.Services{"db": db, "a": a, "b": b}, - } - observed := emptyObserved() - for _, svc := range []types.ServiceConfig{db, a, b} { - hash, err := serviceHashWithResolvedRefs(svc, nil) - assert.NilError(t, err) - observed.Containers[svc.Name] = []ObservedContainer{ - observedServiceContainer(svc.Name, 1, container.StateRunning, hash), - } - } - - plan, err := reconcile(t.Context(), project, observed, startScopeOptions(ScopeCreateStart), noPrompt) - assert.NilError(t, err) - assertNoCycle(t, plan) - - for _, n := range plan.Nodes { - if n.Operation.Type == OpWaitCondition { - assert.Assert(t, !slices.Contains(n.DependsOn, n), - "wait node #%d must not depend on itself:\n%s", n.ID, plan) - } - } -} - -// TestPlanStart_TransitiveChainNeverCreatesWaitCycle exercises -// wouldCreateCycle's BFS beyond a single hop: consumer's sibling "app" isn't -// itself the shared wait node, nor does it depend on it directly -- app's -// own start node depends on mid's start node, which depends on the shared -// db:service_healthy wait node two hops away (app -> mid -> wait). Merging -// app's start node into the wait node's own DependsOn would still close a -// (longer) cycle, so wouldCreateCycle must walk the full closure, not just -// candidate's immediate DependsOn. -func TestPlanStart_TransitiveChainNeverCreatesWaitCycle(t *testing.T) { - db := types.ServiceConfig{Name: "db"} - db.HealthCheck = &types.HealthCheckConfig{Test: []string{"CMD", "true"}} - mid := serviceWithDeps("mid", types.DependsOnConfig{ - "db": {Condition: types.ServiceConditionHealthy, Required: true}, - }) - app := serviceWithDeps("app", types.DependsOnConfig{ - "mid": {Condition: types.ServiceConditionStarted, Required: true}, - }) - consumer := serviceWithDeps("consumer", types.DependsOnConfig{ - "app": {Condition: types.ServiceConditionStarted, Required: true}, - "db": {Condition: types.ServiceConditionHealthy, Required: true}, - }) - project := &types.Project{ - Name: "myproject", - Services: types.Services{"db": db, "mid": mid, "app": app, "consumer": consumer}, - } - - plan, err := reconcile(t.Context(), project, emptyObserved(), startScopeOptions(ScopeCreateStart), noPrompt) - assert.NilError(t, err) - assertNoCycle(t, plan) - - var wait, appStart *PlanNode - for _, n := range plan.Nodes { - switch { - case n.Operation.Type == OpWaitCondition: - wait = n - case n.Operation.Type == OpStartContainer && n.Operation.ResourceID == "service:app:1": - appStart = n - } - } - if wait == nil || appStart == nil { - t.Fatalf("expected db's wait node and app's start node:\n%s", plan) - } - assert.Assert(t, !slices.Contains(wait.DependsOn, appStart), - "db's wait node must not depend on app's start node two hops away (app -> mid -> wait) -- that would still be a cycle:\n%s", plan) -} - // pre_start runs once per service before the first replica start, only when // no replica was running at observation; replicas start sequentially, each // chain link (start, then post_start when declared) gating the next.