diff --git a/pkg/compose/reconcile.go b/pkg/compose/reconcile.go index af70e484d..6395b317f 100644 --- a/pkg/compose/reconcile.go +++ b/pkg/compose/reconcile.go @@ -1269,24 +1269,52 @@ func (r *reconciler) startPhaseReplicas(service types.ServiceConfig) (replicas [ // gate — errors included: a dependency that is neither declared nor // disabled is the same hard failure it is today. func (r *reconciler) startPhaseDependencies(service types.ServiceConfig) ([]*PlanNode, error) { - var depNodes []*PlanNode - for _, dep := range sortedKeys(service.DependsOn) { - cfg := service.DependsOn[dep] + deps := sortedKeys(service.DependsOn) + depTargets := make(map[string][]*PlanNode, len(deps)) + var allTargets []*PlanNode + for _, dep := range deps { targets := r.startChainEnds[dep] if len(targets) == 0 { if node, ok := r.serviceNodes[dep]; ok { targets = []*PlanNode{node} } } + depTargets[dep] = targets + for _, target := range targets { + if !slices.Contains(allTargets, target) { + allTargets = append(allTargets, target) + } + } + } + + var depNodes []*PlanNode + for _, dep := range deps { + cfg := service.DependsOn[dep] shouldWait, err := shouldWaitForDependency(dep, cfg, r.project) if err != nil { return nil, err } if !shouldWait { - depNodes = append(depNodes, targets...) + for _, target := range depTargets[dep] { + if !slices.Contains(depNodes, target) { + depNodes = append(depNodes, target) + } + } continue } - depNodes = append(depNodes, r.waitConditionNode(dep, cfg, targets)) + // The wait node's own exec.waitTimeout window must not open until + // EVERY dependency this service declares has itself finished + // starting, not just the one this particular condition checks -- + // matching the old imperative engine's startService, which only + // calls waitDependencies(X) once every direct dependency's own + // startService call has already returned (see InDependencyOrder's + // graphTraversal.run: a node is dispatched only once none of its + // children are still ServiceStopped, so X's siblings all finish + // starting before X's own wait clock ever starts). Passing just this + // dep's own target here would let an unrelated, slower sibling + // dependency silently shrink the effective grace period this + // condition's health check gets, compared to the old engine. + depNodes = append(depNodes, r.waitConditionNode(dep, cfg, allTargets)) } return depNodes, nil } @@ -1294,7 +1322,13 @@ func (r *reconciler) startPhaseDependencies(service types.ServiceConfig) ([]*Pla // waitConditionNode returns the shared wait node for (dep, condition), // creating it on first use. required:false marks it best-effort — a missing // dependency is skipped, not fatal; one required dependent upgrades the -// shared node for everyone. +// shared node for everyone. targets is now each caller's FULL sibling +// dependency set (see startPhaseDependencies), not just dep's own target, so +// two dependents of the same (dep, condition) with different sibling sets +// routinely diverge here: the merge below unions them onto the one shared +// node rather than creating a second one, at the cost of a dependent with +// fewer siblings waiting a little longer than its own depends_on strictly +// requires -- preferable to losing the dedup this node exists for. func (r *reconciler) waitConditionNode(dep string, cfg types.ServiceDependency, targets []*PlanNode) *PlanNode { key := dep + ":" + cfg.Condition wait, ok := r.waitNodes[key] @@ -1314,18 +1348,55 @@ func (r *reconciler) waitConditionNode(dep string, cfg types.ServiceDependency, if cfg.Required && wait.Operation.BestEffort { wait.Operation.BestEffort = false } - // merge this caller's prerequisites into the shared node. Today every - // caller passes the same targets (startChainEnds[dep] is fixed before - // any dependent is visited), so this is defensive — but relying on that - // silently would break the day the targets diverge per caller. + // merge this caller's prerequisites into the shared node, skipping + // anything that would create a cycle: an ordinary diamond dependency + // (X depends on dep:cond, Z depends on X AND independently on the same + // dep:cond) puts X's own chain end -- which already depends on this very + // wait node -- into Z's sibling set. Adding it back as wait's own + // prerequisite would deadlock the executor forever (runNode never applies + // exec.waitTimeout until its own DependsOn are satisfied, and nothing + // detects a node depending on itself at a distance). Skipping it loses + // nothing: X cannot even start until wait has already succeeded, so by + // the time X's chain completes wait is necessarily already done. for _, target := range targets { - if !slices.Contains(wait.DependsOn, target) { - wait.DependsOn = append(wait.DependsOn, target) + if slices.Contains(wait.DependsOn, target) { + continue } + if wouldCreateCycle(target, wait) { + continue + } + wait.DependsOn = append(wait.DependsOn, target) } return wait } +// wouldCreateCycle reports whether candidate already (transitively) depends +// on node, via DependsOn edges -- i.e. whether appending candidate to +// node.DependsOn would close a cycle. Only reachable from waitConditionNode's +// merge, where candidate is a sibling target pulled in for an already-shared +// wait node. +func wouldCreateCycle(candidate, node *PlanNode) bool { + if candidate == node { + return true + } + seen := map[*PlanNode]bool{candidate: true} + queue := []*PlanNode{candidate} + for len(queue) > 0 { + n := queue[0] + queue = queue[1:] + for _, dep := range n.DependsOn { + if dep == node { + return true + } + if !seen[dep] { + seen[dep] = true + queue = append(queue, dep) + } + } + } + return false +} + // planProviderStart is the start-phase visit of a provider service: no // container to start, but its own depends_on still applies — the imperative // startService waits on it for every service, providers included. Its chain diff --git a/pkg/compose/reconcile_start_test.go b/pkg/compose/reconcile_start_test.go index 82b243a60..2769d6980 100644 --- a/pkg/compose/reconcile_start_test.go +++ b/pkg/compose/reconcile_start_test.go @@ -132,6 +132,230 @@ func TestPlanStart_HealthyConditionDeduplicated(t *testing.T) { `)+"\n") } +// TestPlanStart_WaitNodeGatedOnAllSiblingDependencies is a regression test +// for a Copilot review finding on #14290's Create->Start barrier work: a +// wait node's own exec.waitTimeout window used to open as soon as ITS OWN +// dependency's chain finished, ignoring the requesting service's other +// depends_on entries entirely. The old imperative engine never allowed that +// -- InDependencyOrder only calls startService(app), and so only starts +// app's own waitDependencies clock, once EVERY one of app's direct +// dependencies has itself finished starting (see graphTraversal.run: a node +// is dispatched only once none of its children are still ServiceStopped). A +// slow sibling with no wait node of its own (service_started is a plain +// edge, no condition to poll) could silently eat into a health check's +// effective grace period. Here app depends on db:service_healthy (needs a +// wait node) and migrate:service_started (a plain edge, no wait node) -- the +// db wait node must still depend on migrate's own start, or its +// exec.waitTimeout clock starts too early relative to the old engine. +func TestPlanStart_WaitNodeGatedOnAllSiblingDependencies(t *testing.T) { + db := types.ServiceConfig{Name: "db"} + db.HealthCheck = &types.HealthCheckConfig{Test: []string{"CMD", "true"}} + migrate := types.ServiceConfig{Name: "migrate"} + app := serviceWithDeps("app", types.DependsOnConfig{ + "db": {Condition: types.ServiceConditionHealthy, Required: true}, + "migrate": {Condition: types.ServiceConditionStarted, Required: true}, + }) + project := &types.Project{ + Name: "myproject", + Services: types.Services{"db": db, "migrate": migrate, "app": app}, + } + + plan, err := reconcile(t.Context(), project, emptyObserved(), startScopeOptions(ScopeCreateStart), noPrompt) + assert.NilError(t, err) + + var wait, dbStart, migrateStart *PlanNode + for _, n := range plan.Nodes { + switch { + case n.Operation.Type == OpWaitCondition: + wait = n + case n.Operation.Type == OpStartContainer && n.Operation.ResourceID == "service:db:1": + dbStart = n + case n.Operation.Type == OpStartContainer && n.Operation.ResourceID == "service:migrate:1": + migrateStart = n + } + } + if wait == nil || dbStart == nil || migrateStart == nil { + t.Fatalf("expected a db wait node and both db/migrate start nodes:\n%s", plan) + } + assert.Assert(t, slices.Contains(wait.DependsOn, dbStart), + "db's wait node must depend on db's own start:\n%s", plan) + assert.Assert(t, slices.Contains(wait.DependsOn, migrateStart), + "db's wait node must also depend on app's other sibling dependency (migrate) finishing its own start, not just db's:\n%s", plan) +} + +// nodeInOwnDependencyClosure reports whether n is reachable from itself by +// following DependsOn edges -- i.e. whether the plan has a dependency cycle +// through n. +func nodeInOwnDependencyClosure(n *PlanNode) bool { + seen := map[*PlanNode]bool{} + queue := slices.Clone(n.DependsOn) + for len(queue) > 0 { + cur := queue[0] + queue = queue[1:] + if cur == n { + return true + } + if seen[cur] { + continue + } + seen[cur] = true + queue = append(queue, cur.DependsOn...) + } + return false +} + +func assertNoCycle(t *testing.T, plan *Plan) { + t.Helper() + for _, n := range plan.Nodes { + if nodeInOwnDependencyClosure(n) { + t.Fatalf("plan has a dependency cycle through node #%d (%s):\n%s", n.ID, n.Operation.ResourceID, plan) + } + } +} + +// TestPlanStart_DiamondDependencyNeverCreatesWaitCycle is a regression test +// for a Copilot review finding on the allTargets widening above: an entirely +// ordinary diamond dependency -- app depends on db:service_healthy, worker +// depends on BOTH app:service_started and, independently, db:service_healthy +// -- shares db's wait node between app and worker (same (dep,condition) +// key). app's own start node already depends on that wait node. Widening the +// wait node's own prerequisites with worker's sibling set (app's start node, +// pulled in because worker also depends on app) would append app's start +// node back into the wait node's own DependsOn -- a direct cycle the +// executor can never resolve (runNode blocks on rs.done[dep.ID] for every +// dependency before closing its own, with nothing to detect or time out a +// cycle). The wait node must keep depending on db alone; worker's own start +// node separately depends on both the wait node and app's start node, which +// already encodes everything worker's depends_on requires -- nothing is +// lost by not also routing it through the shared wait node. +func TestPlanStart_DiamondDependencyNeverCreatesWaitCycle(t *testing.T) { + db := types.ServiceConfig{Name: "db"} + db.HealthCheck = &types.HealthCheckConfig{Test: []string{"CMD", "true"}} + app := serviceWithDeps("app", types.DependsOnConfig{ + "db": {Condition: types.ServiceConditionHealthy, Required: true}, + }) + worker := serviceWithDeps("worker", types.DependsOnConfig{ + "app": {Condition: types.ServiceConditionStarted, Required: true}, + "db": {Condition: types.ServiceConditionHealthy, Required: true}, + }) + project := &types.Project{ + Name: "myproject", + Services: types.Services{"db": db, "app": app, "worker": worker}, + } + + plan, err := reconcile(t.Context(), project, emptyObserved(), startScopeOptions(ScopeCreateStart), noPrompt) + assert.NilError(t, err) + assertNoCycle(t, plan) + + var wait, appStart *PlanNode + for _, n := range plan.Nodes { + switch { + case n.Operation.Type == OpWaitCondition: + wait = n + case n.Operation.Type == OpStartContainer && n.Operation.ResourceID == "service:app:1": + appStart = n + } + } + if wait == nil || appStart == nil { + t.Fatalf("expected db's wait node and app's start node:\n%s", plan) + } + assert.Assert(t, !slices.Contains(wait.DependsOn, appStart), + "db's wait node must not depend on app's start node -- app's own start already depends on the wait node, so this would be a cycle:\n%s", plan) + assert.Assert(t, slices.Contains(appStart.DependsOn, wait), + "app's own start must still depend on db's wait node:\n%s", plan) +} + +// TestPlanStart_ExceptionalChainCollapseNeverCreatesWaitCycle is a second +// regression test for the same cycle risk, reproducing the exact shape found +// during review: a steady-state up where every service is already running. +// "a" depends only on db:service_healthy and has nothing left to start, so +// planServiceStart's "nothing to start" branch sets startChainEnds["a"] to +// literally BE the shared db:service_healthy wait node (ends := depNodes). +// "b" depends on both db:service_healthy and a:service_started -- its +// allTargets then contains that very wait node as one of "a"'s targets, and +// merging it into itself would be a direct self-loop with no other node +// in between. +func TestPlanStart_ExceptionalChainCollapseNeverCreatesWaitCycle(t *testing.T) { + db := types.ServiceConfig{Name: "db"} + db.HealthCheck = &types.HealthCheckConfig{Test: []string{"CMD", "true"}} + a := serviceWithDeps("a", types.DependsOnConfig{ + "db": {Condition: types.ServiceConditionHealthy, Required: true}, + }) + b := serviceWithDeps("b", types.DependsOnConfig{ + "db": {Condition: types.ServiceConditionHealthy, Required: true}, + "a": {Condition: types.ServiceConditionStarted, Required: true}, + }) + project := &types.Project{ + Name: "myproject", + Services: types.Services{"db": db, "a": a, "b": b}, + } + observed := emptyObserved() + for _, svc := range []types.ServiceConfig{db, a, b} { + hash, err := serviceHashWithResolvedRefs(svc, nil) + assert.NilError(t, err) + observed.Containers[svc.Name] = []ObservedContainer{ + observedServiceContainer(svc.Name, 1, container.StateRunning, hash), + } + } + + plan, err := reconcile(t.Context(), project, observed, startScopeOptions(ScopeCreateStart), noPrompt) + assert.NilError(t, err) + assertNoCycle(t, plan) + + for _, n := range plan.Nodes { + if n.Operation.Type == OpWaitCondition { + assert.Assert(t, !slices.Contains(n.DependsOn, n), + "wait node #%d must not depend on itself:\n%s", n.ID, plan) + } + } +} + +// TestPlanStart_TransitiveChainNeverCreatesWaitCycle exercises +// wouldCreateCycle's BFS beyond a single hop: consumer's sibling "app" isn't +// itself the shared wait node, nor does it depend on it directly -- app's +// own start node depends on mid's start node, which depends on the shared +// db:service_healthy wait node two hops away (app -> mid -> wait). Merging +// app's start node into the wait node's own DependsOn would still close a +// (longer) cycle, so wouldCreateCycle must walk the full closure, not just +// candidate's immediate DependsOn. +func TestPlanStart_TransitiveChainNeverCreatesWaitCycle(t *testing.T) { + db := types.ServiceConfig{Name: "db"} + db.HealthCheck = &types.HealthCheckConfig{Test: []string{"CMD", "true"}} + mid := serviceWithDeps("mid", types.DependsOnConfig{ + "db": {Condition: types.ServiceConditionHealthy, Required: true}, + }) + app := serviceWithDeps("app", types.DependsOnConfig{ + "mid": {Condition: types.ServiceConditionStarted, Required: true}, + }) + consumer := serviceWithDeps("consumer", types.DependsOnConfig{ + "app": {Condition: types.ServiceConditionStarted, Required: true}, + "db": {Condition: types.ServiceConditionHealthy, Required: true}, + }) + project := &types.Project{ + Name: "myproject", + Services: types.Services{"db": db, "mid": mid, "app": app, "consumer": consumer}, + } + + plan, err := reconcile(t.Context(), project, emptyObserved(), startScopeOptions(ScopeCreateStart), noPrompt) + assert.NilError(t, err) + assertNoCycle(t, plan) + + var wait, appStart *PlanNode + for _, n := range plan.Nodes { + switch { + case n.Operation.Type == OpWaitCondition: + wait = n + case n.Operation.Type == OpStartContainer && n.Operation.ResourceID == "service:app:1": + appStart = n + } + } + if wait == nil || appStart == nil { + t.Fatalf("expected db's wait node and app's start node:\n%s", plan) + } + assert.Assert(t, !slices.Contains(wait.DependsOn, appStart), + "db's wait node must not depend on app's start node two hops away (app -> mid -> wait) -- that would still be a cycle:\n%s", plan) +} + // pre_start runs once per service before the first replica start, only when // no replica was running at observation; replicas start sequentially, each // chain link (start, then post_start when declared) gating the next.