mirror of
https://github.com/docker/compose.git
synced 2026-11-19 22:50:08 +00:00
fix(reconcile): keep each dependency's wait timeout independent
Reverts 365f0026f, a behavior-parity fix for a Copilot finding that the
plan-based start engine's per-(dependency,condition) wait node could
start its exec.waitTimeout clock earlier than the old imperative
engine's single waitDependencies(X) call -- which only begins once
EVERY direct dependency of X has itself finished starting, so a slow
sibling with no wait node of its own could silently extend another
dependency's effective grace period.
On reflection that old-engine coupling is itself the bug: a service's
declared --wait-timeout for a given dependency should bound how long
THAT dependency takes to satisfy its condition, period -- not be
inflated by how long an unrelated sibling dependency happens to take
to start. Letting db's health-check timeout silently grow because
migrate's own dependency chain is slow means --wait-timeout no longer
means what it says. Restoring the narrower, per-dependency gating is a
deliberate, accepted behavior change from the old engine, not a parity
regression to chase.
This also removes the dependency-cycle risk the sibling-widening
introduced (an ordinary diamond dependency, or a service whose own
chain end collapses onto a shared wait node, could merge a node back
into its own transitive dependents -- see 365f0026f's description) and
the wouldCreateCycle machinery and five regression tests that existed
solely to guard against it, now dead weight with no reachable scenario
to protect against.
Verified: go build, go test ./pkg/compose/..., and golangci-lint all
clean.
Signed-off-by: Nicolas De Loof <nicolas.deloof@gmail.com>
This commit is contained in:
parent
4c82be2569
commit
c87307b89f
2 changed files with 12 additions and 307 deletions
|
|
@ -1269,52 +1269,24 @@ func (r *reconciler) startPhaseReplicas(service types.ServiceConfig) (replicas [
|
|||
// gate — errors included: a dependency that is neither declared nor
|
||||
// disabled is the same hard failure it is today.
|
||||
func (r *reconciler) startPhaseDependencies(service types.ServiceConfig) ([]*PlanNode, error) {
|
||||
deps := sortedKeys(service.DependsOn)
|
||||
depTargets := make(map[string][]*PlanNode, len(deps))
|
||||
var allTargets []*PlanNode
|
||||
for _, dep := range deps {
|
||||
var depNodes []*PlanNode
|
||||
for _, dep := range sortedKeys(service.DependsOn) {
|
||||
cfg := service.DependsOn[dep]
|
||||
targets := r.startChainEnds[dep]
|
||||
if len(targets) == 0 {
|
||||
if node, ok := r.serviceNodes[dep]; ok {
|
||||
targets = []*PlanNode{node}
|
||||
}
|
||||
}
|
||||
depTargets[dep] = targets
|
||||
for _, target := range targets {
|
||||
if !slices.Contains(allTargets, target) {
|
||||
allTargets = append(allTargets, target)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
var depNodes []*PlanNode
|
||||
for _, dep := range deps {
|
||||
cfg := service.DependsOn[dep]
|
||||
shouldWait, err := shouldWaitForDependency(dep, cfg, r.project)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if !shouldWait {
|
||||
for _, target := range depTargets[dep] {
|
||||
if !slices.Contains(depNodes, target) {
|
||||
depNodes = append(depNodes, target)
|
||||
}
|
||||
}
|
||||
depNodes = append(depNodes, targets...)
|
||||
continue
|
||||
}
|
||||
// The wait node's own exec.waitTimeout window must not open until
|
||||
// EVERY dependency this service declares has itself finished
|
||||
// starting, not just the one this particular condition checks --
|
||||
// matching the old imperative engine's startService, which only
|
||||
// calls waitDependencies(X) once every direct dependency's own
|
||||
// startService call has already returned (see InDependencyOrder's
|
||||
// graphTraversal.run: a node is dispatched only once none of its
|
||||
// children are still ServiceStopped, so X's siblings all finish
|
||||
// starting before X's own wait clock ever starts). Passing just this
|
||||
// dep's own target here would let an unrelated, slower sibling
|
||||
// dependency silently shrink the effective grace period this
|
||||
// condition's health check gets, compared to the old engine.
|
||||
depNodes = append(depNodes, r.waitConditionNode(dep, cfg, allTargets))
|
||||
depNodes = append(depNodes, r.waitConditionNode(dep, cfg, targets))
|
||||
}
|
||||
return depNodes, nil
|
||||
}
|
||||
|
|
@ -1322,13 +1294,7 @@ func (r *reconciler) startPhaseDependencies(service types.ServiceConfig) ([]*Pla
|
|||
// waitConditionNode returns the shared wait node for (dep, condition),
|
||||
// creating it on first use. required:false marks it best-effort — a missing
|
||||
// dependency is skipped, not fatal; one required dependent upgrades the
|
||||
// shared node for everyone. targets is now each caller's FULL sibling
|
||||
// dependency set (see startPhaseDependencies), not just dep's own target, so
|
||||
// two dependents of the same (dep, condition) with different sibling sets
|
||||
// routinely diverge here: the merge below unions them onto the one shared
|
||||
// node rather than creating a second one, at the cost of a dependent with
|
||||
// fewer siblings waiting a little longer than its own depends_on strictly
|
||||
// requires -- preferable to losing the dedup this node exists for.
|
||||
// shared node for everyone.
|
||||
func (r *reconciler) waitConditionNode(dep string, cfg types.ServiceDependency, targets []*PlanNode) *PlanNode {
|
||||
key := dep + ":" + cfg.Condition
|
||||
wait, ok := r.waitNodes[key]
|
||||
|
|
@ -1348,55 +1314,18 @@ func (r *reconciler) waitConditionNode(dep string, cfg types.ServiceDependency,
|
|||
if cfg.Required && wait.Operation.BestEffort {
|
||||
wait.Operation.BestEffort = false
|
||||
}
|
||||
// merge this caller's prerequisites into the shared node, skipping
|
||||
// anything that would create a cycle: an ordinary diamond dependency
|
||||
// (X depends on dep:cond, Z depends on X AND independently on the same
|
||||
// dep:cond) puts X's own chain end -- which already depends on this very
|
||||
// wait node -- into Z's sibling set. Adding it back as wait's own
|
||||
// prerequisite would deadlock the executor forever (runNode never applies
|
||||
// exec.waitTimeout until its own DependsOn are satisfied, and nothing
|
||||
// detects a node depending on itself at a distance). Skipping it loses
|
||||
// nothing: X cannot even start until wait has already succeeded, so by
|
||||
// the time X's chain completes wait is necessarily already done.
|
||||
// merge this caller's prerequisites into the shared node. Today every
|
||||
// caller passes the same targets (startChainEnds[dep] is fixed before
|
||||
// any dependent is visited), so this is defensive — but relying on that
|
||||
// silently would break the day the targets diverge per caller.
|
||||
for _, target := range targets {
|
||||
if slices.Contains(wait.DependsOn, target) {
|
||||
continue
|
||||
if !slices.Contains(wait.DependsOn, target) {
|
||||
wait.DependsOn = append(wait.DependsOn, target)
|
||||
}
|
||||
if wouldCreateCycle(target, wait) {
|
||||
continue
|
||||
}
|
||||
wait.DependsOn = append(wait.DependsOn, target)
|
||||
}
|
||||
return wait
|
||||
}
|
||||
|
||||
// wouldCreateCycle reports whether candidate already (transitively) depends
|
||||
// on node, via DependsOn edges -- i.e. whether appending candidate to
|
||||
// node.DependsOn would close a cycle. Only reachable from waitConditionNode's
|
||||
// merge, where candidate is a sibling target pulled in for an already-shared
|
||||
// wait node.
|
||||
func wouldCreateCycle(candidate, node *PlanNode) bool {
|
||||
if candidate == node {
|
||||
return true
|
||||
}
|
||||
seen := map[*PlanNode]bool{candidate: true}
|
||||
queue := []*PlanNode{candidate}
|
||||
for len(queue) > 0 {
|
||||
n := queue[0]
|
||||
queue = queue[1:]
|
||||
for _, dep := range n.DependsOn {
|
||||
if dep == node {
|
||||
return true
|
||||
}
|
||||
if !seen[dep] {
|
||||
seen[dep] = true
|
||||
queue = append(queue, dep)
|
||||
}
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// planProviderStart is the start-phase visit of a provider service: no
|
||||
// container to start, but its own depends_on still applies — the imperative
|
||||
// startService waits on it for every service, providers included. Its chain
|
||||
|
|
|
|||
|
|
@ -132,230 +132,6 @@ func TestPlanStart_HealthyConditionDeduplicated(t *testing.T) {
|
|||
`)+"\n")
|
||||
}
|
||||
|
||||
// TestPlanStart_WaitNodeGatedOnAllSiblingDependencies is a regression test
|
||||
// for a Copilot review finding on #14290's Create->Start barrier work: a
|
||||
// wait node's own exec.waitTimeout window used to open as soon as ITS OWN
|
||||
// dependency's chain finished, ignoring the requesting service's other
|
||||
// depends_on entries entirely. The old imperative engine never allowed that
|
||||
// -- InDependencyOrder only calls startService(app), and so only starts
|
||||
// app's own waitDependencies clock, once EVERY one of app's direct
|
||||
// dependencies has itself finished starting (see graphTraversal.run: a node
|
||||
// is dispatched only once none of its children are still ServiceStopped). A
|
||||
// slow sibling with no wait node of its own (service_started is a plain
|
||||
// edge, no condition to poll) could silently eat into a health check's
|
||||
// effective grace period. Here app depends on db:service_healthy (needs a
|
||||
// wait node) and migrate:service_started (a plain edge, no wait node) -- the
|
||||
// db wait node must still depend on migrate's own start, or its
|
||||
// exec.waitTimeout clock starts too early relative to the old engine.
|
||||
func TestPlanStart_WaitNodeGatedOnAllSiblingDependencies(t *testing.T) {
|
||||
db := types.ServiceConfig{Name: "db"}
|
||||
db.HealthCheck = &types.HealthCheckConfig{Test: []string{"CMD", "true"}}
|
||||
migrate := types.ServiceConfig{Name: "migrate"}
|
||||
app := serviceWithDeps("app", types.DependsOnConfig{
|
||||
"db": {Condition: types.ServiceConditionHealthy, Required: true},
|
||||
"migrate": {Condition: types.ServiceConditionStarted, Required: true},
|
||||
})
|
||||
project := &types.Project{
|
||||
Name: "myproject",
|
||||
Services: types.Services{"db": db, "migrate": migrate, "app": app},
|
||||
}
|
||||
|
||||
plan, err := reconcile(t.Context(), project, emptyObserved(), startScopeOptions(ScopeCreateStart), noPrompt)
|
||||
assert.NilError(t, err)
|
||||
|
||||
var wait, dbStart, migrateStart *PlanNode
|
||||
for _, n := range plan.Nodes {
|
||||
switch {
|
||||
case n.Operation.Type == OpWaitCondition:
|
||||
wait = n
|
||||
case n.Operation.Type == OpStartContainer && n.Operation.ResourceID == "service:db:1":
|
||||
dbStart = n
|
||||
case n.Operation.Type == OpStartContainer && n.Operation.ResourceID == "service:migrate:1":
|
||||
migrateStart = n
|
||||
}
|
||||
}
|
||||
if wait == nil || dbStart == nil || migrateStart == nil {
|
||||
t.Fatalf("expected a db wait node and both db/migrate start nodes:\n%s", plan)
|
||||
}
|
||||
assert.Assert(t, slices.Contains(wait.DependsOn, dbStart),
|
||||
"db's wait node must depend on db's own start:\n%s", plan)
|
||||
assert.Assert(t, slices.Contains(wait.DependsOn, migrateStart),
|
||||
"db's wait node must also depend on app's other sibling dependency (migrate) finishing its own start, not just db's:\n%s", plan)
|
||||
}
|
||||
|
||||
// nodeInOwnDependencyClosure reports whether n is reachable from itself by
|
||||
// following DependsOn edges -- i.e. whether the plan has a dependency cycle
|
||||
// through n.
|
||||
func nodeInOwnDependencyClosure(n *PlanNode) bool {
|
||||
seen := map[*PlanNode]bool{}
|
||||
queue := slices.Clone(n.DependsOn)
|
||||
for len(queue) > 0 {
|
||||
cur := queue[0]
|
||||
queue = queue[1:]
|
||||
if cur == n {
|
||||
return true
|
||||
}
|
||||
if seen[cur] {
|
||||
continue
|
||||
}
|
||||
seen[cur] = true
|
||||
queue = append(queue, cur.DependsOn...)
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func assertNoCycle(t *testing.T, plan *Plan) {
|
||||
t.Helper()
|
||||
for _, n := range plan.Nodes {
|
||||
if nodeInOwnDependencyClosure(n) {
|
||||
t.Fatalf("plan has a dependency cycle through node #%d (%s):\n%s", n.ID, n.Operation.ResourceID, plan)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// TestPlanStart_DiamondDependencyNeverCreatesWaitCycle is a regression test
|
||||
// for a Copilot review finding on the allTargets widening above: an entirely
|
||||
// ordinary diamond dependency -- app depends on db:service_healthy, worker
|
||||
// depends on BOTH app:service_started and, independently, db:service_healthy
|
||||
// -- shares db's wait node between app and worker (same (dep,condition)
|
||||
// key). app's own start node already depends on that wait node. Widening the
|
||||
// wait node's own prerequisites with worker's sibling set (app's start node,
|
||||
// pulled in because worker also depends on app) would append app's start
|
||||
// node back into the wait node's own DependsOn -- a direct cycle the
|
||||
// executor can never resolve (runNode blocks on rs.done[dep.ID] for every
|
||||
// dependency before closing its own, with nothing to detect or time out a
|
||||
// cycle). The wait node must keep depending on db alone; worker's own start
|
||||
// node separately depends on both the wait node and app's start node, which
|
||||
// already encodes everything worker's depends_on requires -- nothing is
|
||||
// lost by not also routing it through the shared wait node.
|
||||
func TestPlanStart_DiamondDependencyNeverCreatesWaitCycle(t *testing.T) {
|
||||
db := types.ServiceConfig{Name: "db"}
|
||||
db.HealthCheck = &types.HealthCheckConfig{Test: []string{"CMD", "true"}}
|
||||
app := serviceWithDeps("app", types.DependsOnConfig{
|
||||
"db": {Condition: types.ServiceConditionHealthy, Required: true},
|
||||
})
|
||||
worker := serviceWithDeps("worker", types.DependsOnConfig{
|
||||
"app": {Condition: types.ServiceConditionStarted, Required: true},
|
||||
"db": {Condition: types.ServiceConditionHealthy, Required: true},
|
||||
})
|
||||
project := &types.Project{
|
||||
Name: "myproject",
|
||||
Services: types.Services{"db": db, "app": app, "worker": worker},
|
||||
}
|
||||
|
||||
plan, err := reconcile(t.Context(), project, emptyObserved(), startScopeOptions(ScopeCreateStart), noPrompt)
|
||||
assert.NilError(t, err)
|
||||
assertNoCycle(t, plan)
|
||||
|
||||
var wait, appStart *PlanNode
|
||||
for _, n := range plan.Nodes {
|
||||
switch {
|
||||
case n.Operation.Type == OpWaitCondition:
|
||||
wait = n
|
||||
case n.Operation.Type == OpStartContainer && n.Operation.ResourceID == "service:app:1":
|
||||
appStart = n
|
||||
}
|
||||
}
|
||||
if wait == nil || appStart == nil {
|
||||
t.Fatalf("expected db's wait node and app's start node:\n%s", plan)
|
||||
}
|
||||
assert.Assert(t, !slices.Contains(wait.DependsOn, appStart),
|
||||
"db's wait node must not depend on app's start node -- app's own start already depends on the wait node, so this would be a cycle:\n%s", plan)
|
||||
assert.Assert(t, slices.Contains(appStart.DependsOn, wait),
|
||||
"app's own start must still depend on db's wait node:\n%s", plan)
|
||||
}
|
||||
|
||||
// TestPlanStart_ExceptionalChainCollapseNeverCreatesWaitCycle is a second
|
||||
// regression test for the same cycle risk, reproducing the exact shape found
|
||||
// during review: a steady-state up where every service is already running.
|
||||
// "a" depends only on db:service_healthy and has nothing left to start, so
|
||||
// planServiceStart's "nothing to start" branch sets startChainEnds["a"] to
|
||||
// literally BE the shared db:service_healthy wait node (ends := depNodes).
|
||||
// "b" depends on both db:service_healthy and a:service_started -- its
|
||||
// allTargets then contains that very wait node as one of "a"'s targets, and
|
||||
// merging it into itself would be a direct self-loop with no other node
|
||||
// in between.
|
||||
func TestPlanStart_ExceptionalChainCollapseNeverCreatesWaitCycle(t *testing.T) {
|
||||
db := types.ServiceConfig{Name: "db"}
|
||||
db.HealthCheck = &types.HealthCheckConfig{Test: []string{"CMD", "true"}}
|
||||
a := serviceWithDeps("a", types.DependsOnConfig{
|
||||
"db": {Condition: types.ServiceConditionHealthy, Required: true},
|
||||
})
|
||||
b := serviceWithDeps("b", types.DependsOnConfig{
|
||||
"db": {Condition: types.ServiceConditionHealthy, Required: true},
|
||||
"a": {Condition: types.ServiceConditionStarted, Required: true},
|
||||
})
|
||||
project := &types.Project{
|
||||
Name: "myproject",
|
||||
Services: types.Services{"db": db, "a": a, "b": b},
|
||||
}
|
||||
observed := emptyObserved()
|
||||
for _, svc := range []types.ServiceConfig{db, a, b} {
|
||||
hash, err := serviceHashWithResolvedRefs(svc, nil)
|
||||
assert.NilError(t, err)
|
||||
observed.Containers[svc.Name] = []ObservedContainer{
|
||||
observedServiceContainer(svc.Name, 1, container.StateRunning, hash),
|
||||
}
|
||||
}
|
||||
|
||||
plan, err := reconcile(t.Context(), project, observed, startScopeOptions(ScopeCreateStart), noPrompt)
|
||||
assert.NilError(t, err)
|
||||
assertNoCycle(t, plan)
|
||||
|
||||
for _, n := range plan.Nodes {
|
||||
if n.Operation.Type == OpWaitCondition {
|
||||
assert.Assert(t, !slices.Contains(n.DependsOn, n),
|
||||
"wait node #%d must not depend on itself:\n%s", n.ID, plan)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// TestPlanStart_TransitiveChainNeverCreatesWaitCycle exercises
|
||||
// wouldCreateCycle's BFS beyond a single hop: consumer's sibling "app" isn't
|
||||
// itself the shared wait node, nor does it depend on it directly -- app's
|
||||
// own start node depends on mid's start node, which depends on the shared
|
||||
// db:service_healthy wait node two hops away (app -> mid -> wait). Merging
|
||||
// app's start node into the wait node's own DependsOn would still close a
|
||||
// (longer) cycle, so wouldCreateCycle must walk the full closure, not just
|
||||
// candidate's immediate DependsOn.
|
||||
func TestPlanStart_TransitiveChainNeverCreatesWaitCycle(t *testing.T) {
|
||||
db := types.ServiceConfig{Name: "db"}
|
||||
db.HealthCheck = &types.HealthCheckConfig{Test: []string{"CMD", "true"}}
|
||||
mid := serviceWithDeps("mid", types.DependsOnConfig{
|
||||
"db": {Condition: types.ServiceConditionHealthy, Required: true},
|
||||
})
|
||||
app := serviceWithDeps("app", types.DependsOnConfig{
|
||||
"mid": {Condition: types.ServiceConditionStarted, Required: true},
|
||||
})
|
||||
consumer := serviceWithDeps("consumer", types.DependsOnConfig{
|
||||
"app": {Condition: types.ServiceConditionStarted, Required: true},
|
||||
"db": {Condition: types.ServiceConditionHealthy, Required: true},
|
||||
})
|
||||
project := &types.Project{
|
||||
Name: "myproject",
|
||||
Services: types.Services{"db": db, "mid": mid, "app": app, "consumer": consumer},
|
||||
}
|
||||
|
||||
plan, err := reconcile(t.Context(), project, emptyObserved(), startScopeOptions(ScopeCreateStart), noPrompt)
|
||||
assert.NilError(t, err)
|
||||
assertNoCycle(t, plan)
|
||||
|
||||
var wait, appStart *PlanNode
|
||||
for _, n := range plan.Nodes {
|
||||
switch {
|
||||
case n.Operation.Type == OpWaitCondition:
|
||||
wait = n
|
||||
case n.Operation.Type == OpStartContainer && n.Operation.ResourceID == "service:app:1":
|
||||
appStart = n
|
||||
}
|
||||
}
|
||||
if wait == nil || appStart == nil {
|
||||
t.Fatalf("expected db's wait node and app's start node:\n%s", plan)
|
||||
}
|
||||
assert.Assert(t, !slices.Contains(wait.DependsOn, appStart),
|
||||
"db's wait node must not depend on app's start node two hops away (app -> mid -> wait) -- that would still be a cycle:\n%s", plan)
|
||||
}
|
||||
|
||||
// pre_start runs once per service before the first replica start, only when
|
||||
// no replica was running at observation; replicas start sequentially, each
|
||||
// chain link (start, then post_start when declared) gating the next.
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue