mirror of
https://github.com/docker/compose.git
synced 2026-11-18 14:13:49 +00:00
fix(reconcile): gate wait nodes on every sibling dependency, cycle-safe
Copilot flagged a behavior-parity regression on #14290's Create->Start barrier: a service's OpWaitCondition node only depended on its own dependency's chain end, so its exec.waitTimeout clock could start ticking earlier than the old imperative engine's single waitDependencies(X) call. That call only begins once EVERY direct dependency of X has itself finished starting (confirmed by reading InDependencyOrder's graphTraversal.run: a service's visitorFn only runs once none of its dependency children are still ServiceStopped). A slow sibling with no wait node of its own (service_started is a plain edge, nothing to poll) could silently eat into a health check's effective grace period compared to the old engine. startPhaseDependencies now computes allTargets, the union of every declared dependency's own chain-end for the requesting service, and passes that into waitConditionNode instead of just the one dependency being checked. Because wait nodes are shared/deduplicated across consuming services by (dep, condition) alone, this widening reopened a second, more serious finding from the same review pass: two consumers of the same shared node can have different sibling sets, and merging a sibling that already (transitively) depends on the shared node -- an entirely ordinary diamond dependency, or a service whose own chain end collapses onto the wait node itself when it has nothing left to start -- creates a dependency cycle. The executor has no cycle detection and no timeout on the dependency-wait phase itself (exec.waitTimeout only applies once a node starts executing), so this was a guaranteed permanent hang, not a correctness nicety. Fixed with wouldCreateCycle: a wait node's merge now skips any candidate that already (transitively) depends on it. This is provably safe, not a workaround -- if a candidate already depends on the wait node, the wait node is necessarily already done by the time the candidate's own chain completes, so there is nothing to gain from also adding the reverse edge; the candidate's own direct dependency on the shared node (or on an intermediate sibling) already encodes everything the consumer's depends_on requires. Five new regression tests in reconcile_start_test.go, all verified by ablation (reverting the relevant guard reliably reproduces the original bug, 3-5/5 runs depending on the test): - TestPlanStart_WaitNodeGatedOnAllSiblingDependencies: the original parity fix (db:service_healthy + migrate:service_started siblings). - TestPlanStart_DiamondDependencyNeverCreatesWaitCycle and TestPlanStart_ExceptionalChainCollapseNeverCreatesWaitCycle: the two concrete cycle shapes found in review. - TestPlanStart_TransitiveChainNeverCreatesWaitCycle: exercises wouldCreateCycle's BFS two hops deep, not just a direct edge. - assertNoCycle, a plan-wide helper used by the cycle tests above. Verified: go build, go test ./pkg/compose/..., and golangci-lint all clean; full e2e suite re-run clean against a locally built binary (same 3 pre-existing, environment-specific failures as the stack's baseline: TestWatch, TestIPC, TestComposeModel). Two rounds of local Copilot CLI review: the first caught the cycle risk in the initial allTargets widening, the second confirmed wouldCreateCycle's BFS is correct and complete, the "skipping is safe" reasoning holds in every case, and no other call site in reconcile.go has the same unguarded sharing risk. Signed-off-by: Nicolas De Loof <nicolas.deloof@gmail.com>
This commit is contained in:
parent
cb11968490
commit
365f0026f4
2 changed files with 307 additions and 12 deletions
|
|
@ -1269,24 +1269,52 @@ func (r *reconciler) startPhaseReplicas(service types.ServiceConfig) (replicas [
|
|||
// gate — errors included: a dependency that is neither declared nor
|
||||
// disabled is the same hard failure it is today.
|
||||
func (r *reconciler) startPhaseDependencies(service types.ServiceConfig) ([]*PlanNode, error) {
|
||||
var depNodes []*PlanNode
|
||||
for _, dep := range sortedKeys(service.DependsOn) {
|
||||
cfg := service.DependsOn[dep]
|
||||
deps := sortedKeys(service.DependsOn)
|
||||
depTargets := make(map[string][]*PlanNode, len(deps))
|
||||
var allTargets []*PlanNode
|
||||
for _, dep := range deps {
|
||||
targets := r.startChainEnds[dep]
|
||||
if len(targets) == 0 {
|
||||
if node, ok := r.serviceNodes[dep]; ok {
|
||||
targets = []*PlanNode{node}
|
||||
}
|
||||
}
|
||||
depTargets[dep] = targets
|
||||
for _, target := range targets {
|
||||
if !slices.Contains(allTargets, target) {
|
||||
allTargets = append(allTargets, target)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
var depNodes []*PlanNode
|
||||
for _, dep := range deps {
|
||||
cfg := service.DependsOn[dep]
|
||||
shouldWait, err := shouldWaitForDependency(dep, cfg, r.project)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if !shouldWait {
|
||||
depNodes = append(depNodes, targets...)
|
||||
for _, target := range depTargets[dep] {
|
||||
if !slices.Contains(depNodes, target) {
|
||||
depNodes = append(depNodes, target)
|
||||
}
|
||||
}
|
||||
continue
|
||||
}
|
||||
depNodes = append(depNodes, r.waitConditionNode(dep, cfg, targets))
|
||||
// The wait node's own exec.waitTimeout window must not open until
|
||||
// EVERY dependency this service declares has itself finished
|
||||
// starting, not just the one this particular condition checks --
|
||||
// matching the old imperative engine's startService, which only
|
||||
// calls waitDependencies(X) once every direct dependency's own
|
||||
// startService call has already returned (see InDependencyOrder's
|
||||
// graphTraversal.run: a node is dispatched only once none of its
|
||||
// children are still ServiceStopped, so X's siblings all finish
|
||||
// starting before X's own wait clock ever starts). Passing just this
|
||||
// dep's own target here would let an unrelated, slower sibling
|
||||
// dependency silently shrink the effective grace period this
|
||||
// condition's health check gets, compared to the old engine.
|
||||
depNodes = append(depNodes, r.waitConditionNode(dep, cfg, allTargets))
|
||||
}
|
||||
return depNodes, nil
|
||||
}
|
||||
|
|
@ -1294,7 +1322,13 @@ func (r *reconciler) startPhaseDependencies(service types.ServiceConfig) ([]*Pla
|
|||
// waitConditionNode returns the shared wait node for (dep, condition),
|
||||
// creating it on first use. required:false marks it best-effort — a missing
|
||||
// dependency is skipped, not fatal; one required dependent upgrades the
|
||||
// shared node for everyone.
|
||||
// shared node for everyone. targets is now each caller's FULL sibling
|
||||
// dependency set (see startPhaseDependencies), not just dep's own target, so
|
||||
// two dependents of the same (dep, condition) with different sibling sets
|
||||
// routinely diverge here: the merge below unions them onto the one shared
|
||||
// node rather than creating a second one, at the cost of a dependent with
|
||||
// fewer siblings waiting a little longer than its own depends_on strictly
|
||||
// requires -- preferable to losing the dedup this node exists for.
|
||||
func (r *reconciler) waitConditionNode(dep string, cfg types.ServiceDependency, targets []*PlanNode) *PlanNode {
|
||||
key := dep + ":" + cfg.Condition
|
||||
wait, ok := r.waitNodes[key]
|
||||
|
|
@ -1314,18 +1348,55 @@ func (r *reconciler) waitConditionNode(dep string, cfg types.ServiceDependency,
|
|||
if cfg.Required && wait.Operation.BestEffort {
|
||||
wait.Operation.BestEffort = false
|
||||
}
|
||||
// merge this caller's prerequisites into the shared node. Today every
|
||||
// caller passes the same targets (startChainEnds[dep] is fixed before
|
||||
// any dependent is visited), so this is defensive — but relying on that
|
||||
// silently would break the day the targets diverge per caller.
|
||||
// merge this caller's prerequisites into the shared node, skipping
|
||||
// anything that would create a cycle: an ordinary diamond dependency
|
||||
// (X depends on dep:cond, Z depends on X AND independently on the same
|
||||
// dep:cond) puts X's own chain end -- which already depends on this very
|
||||
// wait node -- into Z's sibling set. Adding it back as wait's own
|
||||
// prerequisite would deadlock the executor forever (runNode never applies
|
||||
// exec.waitTimeout until its own DependsOn are satisfied, and nothing
|
||||
// detects a node depending on itself at a distance). Skipping it loses
|
||||
// nothing: X cannot even start until wait has already succeeded, so by
|
||||
// the time X's chain completes wait is necessarily already done.
|
||||
for _, target := range targets {
|
||||
if !slices.Contains(wait.DependsOn, target) {
|
||||
wait.DependsOn = append(wait.DependsOn, target)
|
||||
if slices.Contains(wait.DependsOn, target) {
|
||||
continue
|
||||
}
|
||||
if wouldCreateCycle(target, wait) {
|
||||
continue
|
||||
}
|
||||
wait.DependsOn = append(wait.DependsOn, target)
|
||||
}
|
||||
return wait
|
||||
}
|
||||
|
||||
// wouldCreateCycle reports whether candidate already (transitively) depends
|
||||
// on node, via DependsOn edges -- i.e. whether appending candidate to
|
||||
// node.DependsOn would close a cycle. Only reachable from waitConditionNode's
|
||||
// merge, where candidate is a sibling target pulled in for an already-shared
|
||||
// wait node.
|
||||
func wouldCreateCycle(candidate, node *PlanNode) bool {
|
||||
if candidate == node {
|
||||
return true
|
||||
}
|
||||
seen := map[*PlanNode]bool{candidate: true}
|
||||
queue := []*PlanNode{candidate}
|
||||
for len(queue) > 0 {
|
||||
n := queue[0]
|
||||
queue = queue[1:]
|
||||
for _, dep := range n.DependsOn {
|
||||
if dep == node {
|
||||
return true
|
||||
}
|
||||
if !seen[dep] {
|
||||
seen[dep] = true
|
||||
queue = append(queue, dep)
|
||||
}
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// planProviderStart is the start-phase visit of a provider service: no
|
||||
// container to start, but its own depends_on still applies — the imperative
|
||||
// startService waits on it for every service, providers included. Its chain
|
||||
|
|
|
|||
|
|
@ -132,6 +132,230 @@ func TestPlanStart_HealthyConditionDeduplicated(t *testing.T) {
|
|||
`)+"\n")
|
||||
}
|
||||
|
||||
// TestPlanStart_WaitNodeGatedOnAllSiblingDependencies is a regression test
|
||||
// for a Copilot review finding on #14290's Create->Start barrier work: a
|
||||
// wait node's own exec.waitTimeout window used to open as soon as ITS OWN
|
||||
// dependency's chain finished, ignoring the requesting service's other
|
||||
// depends_on entries entirely. The old imperative engine never allowed that
|
||||
// -- InDependencyOrder only calls startService(app), and so only starts
|
||||
// app's own waitDependencies clock, once EVERY one of app's direct
|
||||
// dependencies has itself finished starting (see graphTraversal.run: a node
|
||||
// is dispatched only once none of its children are still ServiceStopped). A
|
||||
// slow sibling with no wait node of its own (service_started is a plain
|
||||
// edge, no condition to poll) could silently eat into a health check's
|
||||
// effective grace period. Here app depends on db:service_healthy (needs a
|
||||
// wait node) and migrate:service_started (a plain edge, no wait node) -- the
|
||||
// db wait node must still depend on migrate's own start, or its
|
||||
// exec.waitTimeout clock starts too early relative to the old engine.
|
||||
func TestPlanStart_WaitNodeGatedOnAllSiblingDependencies(t *testing.T) {
|
||||
db := types.ServiceConfig{Name: "db"}
|
||||
db.HealthCheck = &types.HealthCheckConfig{Test: []string{"CMD", "true"}}
|
||||
migrate := types.ServiceConfig{Name: "migrate"}
|
||||
app := serviceWithDeps("app", types.DependsOnConfig{
|
||||
"db": {Condition: types.ServiceConditionHealthy, Required: true},
|
||||
"migrate": {Condition: types.ServiceConditionStarted, Required: true},
|
||||
})
|
||||
project := &types.Project{
|
||||
Name: "myproject",
|
||||
Services: types.Services{"db": db, "migrate": migrate, "app": app},
|
||||
}
|
||||
|
||||
plan, err := reconcile(t.Context(), project, emptyObserved(), startScopeOptions(ScopeCreateStart), noPrompt)
|
||||
assert.NilError(t, err)
|
||||
|
||||
var wait, dbStart, migrateStart *PlanNode
|
||||
for _, n := range plan.Nodes {
|
||||
switch {
|
||||
case n.Operation.Type == OpWaitCondition:
|
||||
wait = n
|
||||
case n.Operation.Type == OpStartContainer && n.Operation.ResourceID == "service:db:1":
|
||||
dbStart = n
|
||||
case n.Operation.Type == OpStartContainer && n.Operation.ResourceID == "service:migrate:1":
|
||||
migrateStart = n
|
||||
}
|
||||
}
|
||||
if wait == nil || dbStart == nil || migrateStart == nil {
|
||||
t.Fatalf("expected a db wait node and both db/migrate start nodes:\n%s", plan)
|
||||
}
|
||||
assert.Assert(t, slices.Contains(wait.DependsOn, dbStart),
|
||||
"db's wait node must depend on db's own start:\n%s", plan)
|
||||
assert.Assert(t, slices.Contains(wait.DependsOn, migrateStart),
|
||||
"db's wait node must also depend on app's other sibling dependency (migrate) finishing its own start, not just db's:\n%s", plan)
|
||||
}
|
||||
|
||||
// nodeInOwnDependencyClosure reports whether n is reachable from itself by
|
||||
// following DependsOn edges -- i.e. whether the plan has a dependency cycle
|
||||
// through n.
|
||||
func nodeInOwnDependencyClosure(n *PlanNode) bool {
|
||||
seen := map[*PlanNode]bool{}
|
||||
queue := slices.Clone(n.DependsOn)
|
||||
for len(queue) > 0 {
|
||||
cur := queue[0]
|
||||
queue = queue[1:]
|
||||
if cur == n {
|
||||
return true
|
||||
}
|
||||
if seen[cur] {
|
||||
continue
|
||||
}
|
||||
seen[cur] = true
|
||||
queue = append(queue, cur.DependsOn...)
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func assertNoCycle(t *testing.T, plan *Plan) {
|
||||
t.Helper()
|
||||
for _, n := range plan.Nodes {
|
||||
if nodeInOwnDependencyClosure(n) {
|
||||
t.Fatalf("plan has a dependency cycle through node #%d (%s):\n%s", n.ID, n.Operation.ResourceID, plan)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// TestPlanStart_DiamondDependencyNeverCreatesWaitCycle is a regression test
|
||||
// for a Copilot review finding on the allTargets widening above: an entirely
|
||||
// ordinary diamond dependency -- app depends on db:service_healthy, worker
|
||||
// depends on BOTH app:service_started and, independently, db:service_healthy
|
||||
// -- shares db's wait node between app and worker (same (dep,condition)
|
||||
// key). app's own start node already depends on that wait node. Widening the
|
||||
// wait node's own prerequisites with worker's sibling set (app's start node,
|
||||
// pulled in because worker also depends on app) would append app's start
|
||||
// node back into the wait node's own DependsOn -- a direct cycle the
|
||||
// executor can never resolve (runNode blocks on rs.done[dep.ID] for every
|
||||
// dependency before closing its own, with nothing to detect or time out a
|
||||
// cycle). The wait node must keep depending on db alone; worker's own start
|
||||
// node separately depends on both the wait node and app's start node, which
|
||||
// already encodes everything worker's depends_on requires -- nothing is
|
||||
// lost by not also routing it through the shared wait node.
|
||||
func TestPlanStart_DiamondDependencyNeverCreatesWaitCycle(t *testing.T) {
|
||||
db := types.ServiceConfig{Name: "db"}
|
||||
db.HealthCheck = &types.HealthCheckConfig{Test: []string{"CMD", "true"}}
|
||||
app := serviceWithDeps("app", types.DependsOnConfig{
|
||||
"db": {Condition: types.ServiceConditionHealthy, Required: true},
|
||||
})
|
||||
worker := serviceWithDeps("worker", types.DependsOnConfig{
|
||||
"app": {Condition: types.ServiceConditionStarted, Required: true},
|
||||
"db": {Condition: types.ServiceConditionHealthy, Required: true},
|
||||
})
|
||||
project := &types.Project{
|
||||
Name: "myproject",
|
||||
Services: types.Services{"db": db, "app": app, "worker": worker},
|
||||
}
|
||||
|
||||
plan, err := reconcile(t.Context(), project, emptyObserved(), startScopeOptions(ScopeCreateStart), noPrompt)
|
||||
assert.NilError(t, err)
|
||||
assertNoCycle(t, plan)
|
||||
|
||||
var wait, appStart *PlanNode
|
||||
for _, n := range plan.Nodes {
|
||||
switch {
|
||||
case n.Operation.Type == OpWaitCondition:
|
||||
wait = n
|
||||
case n.Operation.Type == OpStartContainer && n.Operation.ResourceID == "service:app:1":
|
||||
appStart = n
|
||||
}
|
||||
}
|
||||
if wait == nil || appStart == nil {
|
||||
t.Fatalf("expected db's wait node and app's start node:\n%s", plan)
|
||||
}
|
||||
assert.Assert(t, !slices.Contains(wait.DependsOn, appStart),
|
||||
"db's wait node must not depend on app's start node -- app's own start already depends on the wait node, so this would be a cycle:\n%s", plan)
|
||||
assert.Assert(t, slices.Contains(appStart.DependsOn, wait),
|
||||
"app's own start must still depend on db's wait node:\n%s", plan)
|
||||
}
|
||||
|
||||
// TestPlanStart_ExceptionalChainCollapseNeverCreatesWaitCycle is a second
|
||||
// regression test for the same cycle risk, reproducing the exact shape found
|
||||
// during review: a steady-state up where every service is already running.
|
||||
// "a" depends only on db:service_healthy and has nothing left to start, so
|
||||
// planServiceStart's "nothing to start" branch sets startChainEnds["a"] to
|
||||
// literally BE the shared db:service_healthy wait node (ends := depNodes).
|
||||
// "b" depends on both db:service_healthy and a:service_started -- its
|
||||
// allTargets then contains that very wait node as one of "a"'s targets, and
|
||||
// merging it into itself would be a direct self-loop with no other node
|
||||
// in between.
|
||||
func TestPlanStart_ExceptionalChainCollapseNeverCreatesWaitCycle(t *testing.T) {
|
||||
db := types.ServiceConfig{Name: "db"}
|
||||
db.HealthCheck = &types.HealthCheckConfig{Test: []string{"CMD", "true"}}
|
||||
a := serviceWithDeps("a", types.DependsOnConfig{
|
||||
"db": {Condition: types.ServiceConditionHealthy, Required: true},
|
||||
})
|
||||
b := serviceWithDeps("b", types.DependsOnConfig{
|
||||
"db": {Condition: types.ServiceConditionHealthy, Required: true},
|
||||
"a": {Condition: types.ServiceConditionStarted, Required: true},
|
||||
})
|
||||
project := &types.Project{
|
||||
Name: "myproject",
|
||||
Services: types.Services{"db": db, "a": a, "b": b},
|
||||
}
|
||||
observed := emptyObserved()
|
||||
for _, svc := range []types.ServiceConfig{db, a, b} {
|
||||
hash, err := serviceHashWithResolvedRefs(svc, nil)
|
||||
assert.NilError(t, err)
|
||||
observed.Containers[svc.Name] = []ObservedContainer{
|
||||
observedServiceContainer(svc.Name, 1, container.StateRunning, hash),
|
||||
}
|
||||
}
|
||||
|
||||
plan, err := reconcile(t.Context(), project, observed, startScopeOptions(ScopeCreateStart), noPrompt)
|
||||
assert.NilError(t, err)
|
||||
assertNoCycle(t, plan)
|
||||
|
||||
for _, n := range plan.Nodes {
|
||||
if n.Operation.Type == OpWaitCondition {
|
||||
assert.Assert(t, !slices.Contains(n.DependsOn, n),
|
||||
"wait node #%d must not depend on itself:\n%s", n.ID, plan)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// TestPlanStart_TransitiveChainNeverCreatesWaitCycle exercises
|
||||
// wouldCreateCycle's BFS beyond a single hop: consumer's sibling "app" isn't
|
||||
// itself the shared wait node, nor does it depend on it directly -- app's
|
||||
// own start node depends on mid's start node, which depends on the shared
|
||||
// db:service_healthy wait node two hops away (app -> mid -> wait). Merging
|
||||
// app's start node into the wait node's own DependsOn would still close a
|
||||
// (longer) cycle, so wouldCreateCycle must walk the full closure, not just
|
||||
// candidate's immediate DependsOn.
|
||||
func TestPlanStart_TransitiveChainNeverCreatesWaitCycle(t *testing.T) {
|
||||
db := types.ServiceConfig{Name: "db"}
|
||||
db.HealthCheck = &types.HealthCheckConfig{Test: []string{"CMD", "true"}}
|
||||
mid := serviceWithDeps("mid", types.DependsOnConfig{
|
||||
"db": {Condition: types.ServiceConditionHealthy, Required: true},
|
||||
})
|
||||
app := serviceWithDeps("app", types.DependsOnConfig{
|
||||
"mid": {Condition: types.ServiceConditionStarted, Required: true},
|
||||
})
|
||||
consumer := serviceWithDeps("consumer", types.DependsOnConfig{
|
||||
"app": {Condition: types.ServiceConditionStarted, Required: true},
|
||||
"db": {Condition: types.ServiceConditionHealthy, Required: true},
|
||||
})
|
||||
project := &types.Project{
|
||||
Name: "myproject",
|
||||
Services: types.Services{"db": db, "mid": mid, "app": app, "consumer": consumer},
|
||||
}
|
||||
|
||||
plan, err := reconcile(t.Context(), project, emptyObserved(), startScopeOptions(ScopeCreateStart), noPrompt)
|
||||
assert.NilError(t, err)
|
||||
assertNoCycle(t, plan)
|
||||
|
||||
var wait, appStart *PlanNode
|
||||
for _, n := range plan.Nodes {
|
||||
switch {
|
||||
case n.Operation.Type == OpWaitCondition:
|
||||
wait = n
|
||||
case n.Operation.Type == OpStartContainer && n.Operation.ResourceID == "service:app:1":
|
||||
appStart = n
|
||||
}
|
||||
}
|
||||
if wait == nil || appStart == nil {
|
||||
t.Fatalf("expected db's wait node and app's start node:\n%s", plan)
|
||||
}
|
||||
assert.Assert(t, !slices.Contains(wait.DependsOn, appStart),
|
||||
"db's wait node must not depend on app's start node two hops away (app -> mid -> wait) -- that would still be a cycle:\n%s", plan)
|
||||
}
|
||||
|
||||
// pre_start runs once per service before the first replica start, only when
|
||||
// no replica was running at observation; replicas start sequentially, each
|
||||
// chain link (start, then post_start when declared) gating the next.
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue