diff --git a/pkg/compose/executor.go b/pkg/compose/executor.go index e3e37f1c8..05073a6c4 100644 --- a/pkg/compose/executor.go +++ b/pkg/compose/executor.go @@ -21,6 +21,7 @@ import ( "errors" "fmt" "sync" + "sync/atomic" "time" "github.com/compose-spec/compose-go/v2/types" @@ -141,11 +142,22 @@ func (exec *planExecutor) run(ctx context.Context, plan *Plan) error { // (see TestExecutePlanFailedPreStartGatesStart). var failuresMu sync.Mutex failures := make(map[int]error, len(plan.Nodes)) + successes := make(map[int]bool, len(plan.Nodes)) recordFailure := func(id int, err error) { failuresMu.Lock() failures[id] = err failuresMu.Unlock() } + recordSuccess := func(id int) { + failuresMu.Lock() + successes[id] = true + failuresMu.Unlock() + } + succeeded := func(id int) bool { + failuresMu.Lock() + defer failuresMu.Unlock() + return successes[id] + } failedDependency := func(deps []*PlanNode) error { failuresMu.Lock() defer failuresMu.Unlock() @@ -164,6 +176,8 @@ func (exec *planExecutor) run(ctx context.Context, plan *Plan) error { rs := &runState{ done: done, recordFailure: recordFailure, + recordSuccess: recordSuccess, + succeeded: succeeded, failedDependency: failedDependency, groups: groups, events: events, @@ -260,6 +274,11 @@ type phaseCancel struct { cancel context.CancelFunc once sync.Once err error + // failed is the node whose failure canceled the phase, nil while the phase + // is not canceled by one of its own nodes (the caller's context may be done + // as well, e.g. on Ctrl-C). Set strictly before cancel(), so a node that + // sees ctx done because of that failure always finds it here. + failed atomic.Pointer[PlanNode] } func newPhaseCancel(parent context.Context) *phaseCancel { @@ -277,6 +296,7 @@ func (p *phaseCancel) fail(node *PlanNode, events api.EventProcessor, groups *gr p.once.Do(func() { p.err = err groups.onNodeError(node, events, err) + p.failed.Store(node) p.cancel() }) } @@ -310,6 +330,8 @@ func acquireNodeSlot(ctx context.Context, limiter *semaphore.Weighted, node *Pla type runState struct { done map[int]chan struct{} recordFailure func(id int, err error) + recordSuccess func(id int) + succeeded func(id int) bool failedDependency func(deps []*PlanNode) error groups *groupTracker events api.EventProcessor @@ -345,6 +367,7 @@ func (exec *planExecutor) runNode(node *PlanNode, rs *runState, phase *phaseCanc // context. err := nodeCtx.Err() rs.recordFailure(node.ID, err) + exec.reportAborted(node, rs, phase) return err } } @@ -359,12 +382,14 @@ func (exec *planExecutor) runNode(node *PlanNode, rs *runState, phase *phaseCanc // every hop would compound "dependency failed: " prefixes down a // multi-node chain (create -> pre_start -> start -> post_start). rs.recordFailure(node.ID, err) + exec.reportAborted(node, rs, phase) return fmt.Errorf("dependency failed: %w", err) } release, slotErr := acquireNodeSlot(nodeCtx, rs.limiter, node) if slotErr != nil { rs.recordFailure(node.ID, slotErr) + exec.reportAborted(node, rs, phase) return slotErr } defer release() @@ -373,8 +398,17 @@ func (exec *planExecutor) runNode(node *PlanNode, rs *runState, phase *phaseCanc rs.groups.onNodeStart(node, rs.events) err := exec.executeNode(nodeCtx, node) + if errors.Is(err, errWaitInterrupted) { + // waitDependency ends a wait silently when its context is canceled, + // which suits a caller that is going away itself (Ctrl-C): the node + // ends as it always did. Canceled by the failure of another node of + // its phase, the wait is a failure instead: the condition was not + // reached, and the nodes depending on it must not run. + err = exec.interruptedWait(node, rs, phase) + } if err == nil { + rs.recordSuccess(node.ID) // Emit group done event if this is the last node of a group rs.groups.onNodeDone(node, rs.events) return nil @@ -390,6 +424,8 @@ func (exec *planExecutor) runNode(node *PlanNode, rs *runState, phase *phaseCanc // cancellation-casualty caller the cost of building the error-event // payload it would be dropped anyway. phase.fail(node, rs.events, rs.groups, err) + } else { + exec.reportAborted(node, rs, phase) } return err } @@ -403,6 +439,10 @@ func (exec *planExecutor) runNode(node *PlanNode, rs *runState, phase *phaseCanc // satisfies errors.Is(err, context.DeadlineExceeded). var errWaitTimeout = errors.New("timeout waiting for dependencies") +// errWaitInterrupted marks an OpWaitCondition node whose wait ended because its +// context was canceled, before the condition was reached. +var errWaitInterrupted = errors.New("wait interrupted") + // translateWaitTimeout turns a plan failure caused by a wait node's own // --wait-timeout window (see errWaitTimeout) into the "timeout waiting for // dependencies" message the imperative start path always reported for a diff --git a/pkg/compose/executor_events.go b/pkg/compose/executor_events.go index 81a9b3105..2047a79d2 100644 --- a/pkg/compose/executor_events.go +++ b/pkg/compose/executor_events.go @@ -17,6 +17,8 @@ package compose import ( + "errors" + "fmt" "sync" "github.com/docker/compose/v5/pkg/api" @@ -73,6 +75,7 @@ type groupState struct { started int // nodes that have started done int // nodes that have completed working bool // the group's Working event has been emitted + finished bool // the group's final event (Done or Error) has been emitted } func (exec *planExecutor) buildGroupTracker(plan *Plan) *groupTracker { @@ -154,7 +157,8 @@ func (gt *groupTracker) onNodeDone(node *PlanNode, events api.EventProcessor) { defer gt.mu.Unlock() gs := gt.groups[node.Group] gs.done++ - if gs.done == gs.total { + if gs.done == gs.total && !gs.finished { + gs.finished = true events.On(newEvent(gs.eventName, api.Done, gs.kind.doneText())) } } @@ -176,6 +180,7 @@ func (gt *groupTracker) onNodeError(node *PlanNode, events api.EventProcessor, e gs.working = true events.On(newEvent(gs.eventName, api.Working, gs.kind.workingText())) } + gs.finished = true events.On(api.Resource{ ID: gs.eventName, Status: api.Error, @@ -183,6 +188,97 @@ func (gt *groupTracker) onNodeError(node *PlanNode, events api.EventProcessor, e }) } +// onNodeAborted gives a group a final event when one of its nodes was given up +// because another node of the phase failed. Without it the row stays at its +// last status for good: a replica recreated, then never started, still reads +// "Recreated", as if nothing was left to do. It is a no-op once the group has +// its final event, so the failing node's own Error (which also ends the +// nodes depending on it) is never followed by a second one. +func (gt *groupTracker) onNodeAborted(node *PlanNode, events api.EventProcessor, reason string) { + if node.Group == "" { + return + } + gt.mu.Lock() + defer gt.mu.Unlock() + gs := gt.groups[node.Group] + if gs.finished { + return + } + gs.finished = true + if gs.eventName == "" { + gs.eventName = gt.exec.groupEventName(node.Operation) + } + if !gs.working { + reason = "not started: " + reason + } + events.On(api.Resource{ + ID: gs.eventName, + Status: api.Error, + Text: reason, + }) +} + +// resourceName is the name a node's resource goes by in the progress output. +func (gt *groupTracker) resourceName(node *PlanNode) string { + if node.Group != "" { + gt.mu.Lock() + name := gt.groups[node.Group].eventName + gt.mu.Unlock() + if name != "" { + return name + } + } + if node.Operation.Container != nil { + return getContainerProgressName(*node.Operation.Container) + } + return node.Operation.ResourceID +} + +// abortReason tells why node was given up after the failure of another node +// of its phase: the depends_on condition it was still waiting for, if that is +// what it was held by, and the resource whose failure interrupted the phase. +func (exec *planExecutor) abortReason(node *PlanNode, rs *runState, failed *PlanNode) string { + reason := "interrupted by the failure of " + rs.groups.resourceName(failed) + for _, dep := range node.DependsOn { + if dep.Operation.Type == OpWaitCondition && !rs.succeeded(dep.ID) { + return fmt.Sprintf("condition %q of service %q not reached: %s", dep.Operation.Condition, dep.Operation.Name, reason) + } + } + return reason +} + +// reportAborted reports a node that did not run (or did not complete) because +// the phase was canceled by the failure of another node, so that the row of +// the resource it was about to change ends on something other than its last +// status. A phase canceled by its caller (Ctrl-C) reports nothing, as before. +func (exec *planExecutor) reportAborted(node *PlanNode, rs *runState, phase *phaseCancel) { + failed := phase.failed.Load() + if failed == nil || failed == node || node.Operation.Type == OpWaitCondition { + return + } + rs.groups.onNodeAborted(node, rs.events, exec.abortReason(node, rs, failed)) +} + +// interruptedWait is the failure of a wait node left by the failure of +// another node of its phase: the depends_on condition was not reached. The +// rows of the containers it was waiting on, left on "Waiting", are closed with +// the same reason. A phase not canceled by one of its own nodes is not an +// interruption to report: the wait ends silently, as before. +func (exec *planExecutor) interruptedWait(node *PlanNode, rs *runState, phase *phaseCancel) error { + failed := phase.failed.Load() + if failed == nil { + return nil + } + op := node.Operation + exec.containersMu.Lock() + waitingFor := exec.containersByService[op.Name].filter(isNotOneOff, isNotHookContainer) + exec.containersMu.Unlock() + reason := fmt.Sprintf("condition %q of service %q not reached: interrupted by the failure of %s", + op.Condition, op.Name, rs.groups.resourceName(failed)) + rs.events.On(containerReasonEvents(waitingFor, skippedEvent, reason)...) + return errors.New(reason) +} + // emitStartEvent emits the appropriate Working event for an ungrouped node. func emitStartEvent(node *PlanNode, events api.EventProcessor) { op := node.Operation diff --git a/pkg/compose/executor_ops.go b/pkg/compose/executor_ops.go index 7c36e909e..55f12dedf 100644 --- a/pkg/compose/executor_ops.go +++ b/pkg/compose/executor_ops.go @@ -253,7 +253,11 @@ func (exec *planExecutor) execWaitCondition(ctx, origCtx context.Context, op Ope // practically unreachable log line (an unsupported depends_on condition, // filtered out before a plan is ever built). // - return s.waitDependency(ctx, origCtx, op.ResourceID, op.Name, config, waitingFor) + reached, err := s.waitDependencyReached(ctx, origCtx, op.ResourceID, op.Name, config, waitingFor) + if err == nil && !reached { + return errWaitInterrupted + } + return err } // execRunPreStart runs the service's pre_start hooks against the runner diff --git a/pkg/compose/executor_start_test.go b/pkg/compose/executor_start_test.go index 411aa6bac..06893af06 100644 --- a/pkg/compose/executor_start_test.go +++ b/pkg/compose/executor_start_test.go @@ -1385,3 +1385,213 @@ func TestTranslateWaitTimeout(t *testing.T) { assert.Equal(t, translateWaitTimeout(context.DeadlineExceeded), context.DeadlineExceeded) assert.NilError(t, translateWaitTimeout(nil)) } + +// startFailureFixture builds the start phase of a project where web fails to +// start while api, which depends on db being healthy, is still held by that +// wait. db's health never changes, so only web's failure can end the phase. +func startFailureFixture(t *testing.T) (*recordingEvents, *planExecutor, *Plan) { + t.Helper() + svc, apiClient, recorder := newStartPhaseTestService(t) + + project := &types.Project{ + Name: "test", + Services: types.Services{ + "db": {Name: "db"}, + "api": {Name: "api"}, + "web": {Name: "web"}, + }, + } + summary := func(service string) container.Summary { + return container.Summary{ + ID: service + "-id", + Names: []string{"/test-" + service + "-1"}, + Labels: map[string]string{api.ServiceLabel: service, api.OneoffLabel: "False"}, + } + } + dbSummary, apiSummary, webSummary := summary("db"), summary("api"), summary("web") + + // web fails once the wait on db is polling, so that the failure finds the + // wait in flight rather than not dispatched yet + polling := make(chan struct{}) + var pollingOnce sync.Once + apiClient.EXPECT().ContainerInspect(gomock.Any(), "db-id", gomock.Any()). + DoAndReturn(func(context.Context, string, client.ContainerInspectOptions) (client.ContainerInspectResult, error) { + pollingOnce.Do(func() { close(polling) }) + return client.ContainerInspectResult{Container: container.InspectResponse{ + ID: "db-id", + Name: "/test-db-1", + State: &container.State{ + Status: container.StateRunning, + Health: &container.Health{Status: container.Starting}, + }, + Config: &container.Config{Healthcheck: &container.HealthConfig{Test: []string{"CMD", "true"}}}, + }}, nil + }).AnyTimes() + apiClient.EXPECT().ContainerStart(gomock.Any(), "web-id", gomock.Any()). + DoAndReturn(func(context.Context, string, client.ContainerStartOptions) (client.ContainerStartResult, error) { + select { + case <-polling: + case <-time.After(5 * time.Second): + t.Error("the wait on db never started polling") + } + return client.ContainerStartResult{}, errors.New("port is already allocated") + }) + // no ContainerStart for api: it must never be reached + + apiSvc, webSvc := project.Services["api"], project.Services["web"] + plan := &Plan{} + plan.addNode(Operation{ + Type: OpStartContainer, + ResourceID: "service:web:1", + Cause: "start", + Service: &webSvc, + Container: &webSummary, + }, "start:web:1") + wait := plan.addNode(Operation{ + Type: OpWaitCondition, + ResourceID: "wait:db:service_healthy", + Cause: "api depends on db (service_healthy)", + Name: "db", + Condition: types.ServiceConditionHealthy, + }, "") + plan.addNode(Operation{ + Type: OpStartContainer, + ResourceID: "service:api:1", + Cause: "start", + Service: &apiSvc, + Container: &apiSummary, + }, "start:api:1", wait) + for _, n := range plan.Nodes { + n.Phase = PhaseStart + } + + exec := svc.newPlanExecutor(project, emptyObservedState("test"), nil) + exec.containersByService["db"] = Containers{dbSummary} + return recorder, exec, plan +} + +// TestExecutePlanStartFailureReportsTheServicesItInterrupts pins what the rows +// of the other services say when a start fails: a service held by a depends_on +// wait is reported as not started, with the condition that was not reached, +// and the row of the dependency it was waiting on no longer reads "Waiting". +// A service recreated by the same command would otherwise be left on +// "Recreated" while it is never started. +func TestExecutePlanStartFailureReportsTheServicesItInterrupts(t *testing.T) { + recorder, exec, plan := startFailureFixture(t) + + ctx, cancel := context.WithTimeout(t.Context(), 10*time.Second) + defer cancel() + err := exec.run(ctx, plan) + assert.ErrorContains(t, err, "port is already allocated") + + const reason = `condition "service_healthy" of service "db" not reached: interrupted by the failure of Container test-web-1` + recorder.mu.Lock() + defer recorder.mu.Unlock() + web := recorder.byID["Container test-web-1"] + assert.Equal(t, len(web), 2, "web: %v", web) + assert.Equal(t, web[0], api.StatusStarting) + assert.ErrorContains(t, errors.New(web[1]), "port is already allocated") + assert.DeepEqual(t, recorder.byID["Container test-db-1"], []string{api.StatusWaiting, "Skipped: " + reason}) + assert.DeepEqual(t, recorder.byID["Container test-api-1"], []string{"not started: " + reason}) +} + +// TestGroupTrackerOnNodeAborted pins the final event of a group given up +// because another node failed: "not started" while the group never reported +// Working, the bare interruption once it did, and never a second final event, +// before or after. +func TestGroupTrackerOnNodeAborted(t *testing.T) { + summary := container.Summary{ID: "api-id", Names: []string{"/test-api-1"}} + newTracker := func() (*groupTracker, *PlanNode, *capturingEvents) { + plan := &Plan{} + node := plan.addNode(Operation{Type: OpStartContainer, ResourceID: "service:api:1", Container: &summary}, "start:api:1") + return (&planExecutor{}).buildGroupTracker(plan), node, &capturingEvents{} + } + const row = "Container test-api-1" + + t.Run("never started", func(t *testing.T) { + gt, node, rec := newTracker() + gt.onNodeAborted(node, rec, "interrupted") + gt.onNodeAborted(node, rec, "interrupted") + assert.DeepEqual(t, rec.resources, []api.Resource{{ID: row, Status: api.Error, Text: "not started: interrupted"}}) + }) + t.Run("started", func(t *testing.T) { + gt, node, rec := newTracker() + gt.onNodeStart(node, rec) + gt.onNodeAborted(node, rec, "interrupted") + gt.onNodeDone(node, rec) + assert.DeepEqual(t, rec.resources, []api.Resource{ + {ID: row, Status: api.Working, Text: api.StatusStarting}, + {ID: row, Status: api.Error, Text: "interrupted"}, + }) + }) + t.Run("already failed", func(t *testing.T) { + gt, node, rec := newTracker() + gt.onNodeError(node, rec, errors.New("boom")) + gt.onNodeAborted(node, rec, "interrupted") + assert.DeepEqual(t, rec.resources, []api.Resource{ + {ID: row, Status: api.Working, Text: api.StatusStarting}, + {ID: row, Status: api.Error, Text: "boom"}, + }) + }) +} + +// TestExecutePlanCanceledCallerReportsNothingMore keeps what a canceled caller +// (Ctrl-C) gets as it was: the plan ends, and no row is rewritten as the +// victim of a failure there was none of. +func TestExecutePlanCanceledCallerReportsNothingMore(t *testing.T) { + svc, apiClient, recorder := newStartPhaseTestService(t) + + project := &types.Project{Name: "test", Services: types.Services{"db": {Name: "db"}, "api": {Name: "api"}}} + dbSummary := container.Summary{ + ID: "db-id", + Names: []string{"/test-db-1"}, + Labels: map[string]string{api.ServiceLabel: "db", api.OneoffLabel: "False"}, + } + apiSummary := container.Summary{ + ID: "api-id", + Names: []string{"/test-api-1"}, + Labels: map[string]string{api.ServiceLabel: "api", api.OneoffLabel: "False"}, + } + ctx, cancel := context.WithCancel(t.Context()) + defer cancel() + apiClient.EXPECT().ContainerInspect(gomock.Any(), "db-id", gomock.Any()). + DoAndReturn(func(context.Context, string, client.ContainerInspectOptions) (client.ContainerInspectResult, error) { + cancel() + return client.ContainerInspectResult{Container: container.InspectResponse{ + ID: "db-id", + Name: "/test-db-1", + State: &container.State{ + Status: container.StateRunning, + Health: &container.Health{Status: container.Starting}, + }, + Config: &container.Config{Healthcheck: &container.HealthConfig{Test: []string{"CMD", "true"}}}, + }}, nil + }).AnyTimes() + + apiSvc := project.Services["api"] + plan := &Plan{} + wait := plan.addNode(Operation{ + Type: OpWaitCondition, + ResourceID: "wait:db:service_healthy", + Name: "db", + Condition: types.ServiceConditionHealthy, + }, "") + plan.addNode(Operation{ + Type: OpStartContainer, + ResourceID: "service:api:1", + Service: &apiSvc, + Container: &apiSummary, + }, "start:api:1", wait) + for _, n := range plan.Nodes { + n.Phase = PhaseStart + } + + exec := svc.newPlanExecutor(project, emptyObservedState("test"), nil) + exec.containersByService["db"] = Containers{dbSummary} + _ = exec.run(ctx, plan) + + recorder.mu.Lock() + defer recorder.mu.Unlock() + assert.DeepEqual(t, recorder.byID["Container test-db-1"], []string{api.StatusWaiting}) + assert.Equal(t, len(recorder.byID["Container test-api-1"]), 0, "api: %v", recorder.byID["Container test-api-1"]) +} diff --git a/pkg/compose/service_containers.go b/pkg/compose/service_containers.go index 1ceabc092..6a6b20398 100644 --- a/pkg/compose/service_containers.go +++ b/pkg/compose/service_containers.go @@ -213,6 +213,14 @@ func (s *composeService) waitDependencies(ctx context.Context, project *types.Pr // external deadline, a test harness timeout) fired, and must keep // propagating regardless of Required. func (s *composeService) waitDependency(ctx, origCtx context.Context, dependant, dep string, config types.ServiceDependency, waitingFor Containers) error { + _, err := s.waitDependencyReached(ctx, origCtx, dependant, dep, config, waitingFor) + return err +} + +// waitDependencyReached is waitDependency, which also tells whether the wait is +// over because the dependency is ready (or an optional one was given up on), +// as opposed to a plain cancellation of ctx, which ends it silently. +func (s *composeService) waitDependencyReached(ctx, origCtx context.Context, dependant, dep string, config types.ServiceDependency, waitingFor Containers) (bool, error) { ticker := time.NewTicker(500 * time.Millisecond) defer ticker.Stop() for { @@ -232,11 +240,11 @@ func (s *composeService) waitDependency(ctx, origCtx context.Context, dependant, s.events.On(containerReasonEvents(waitingFor, skippedEvent, fmt.Sprintf("optional dependency %q did not satisfy %q before the timeout", dep, config.Condition))...) logrus.Warnf("optional dependency %q did not satisfy %q before the timeout", dep, config.Condition) - return nil + return true, nil } - return ctx.Err() + return false, ctx.Err() } - return nil + return false, nil } var ( done bool @@ -264,10 +272,10 @@ func (s *composeService) waitDependency(ctx, origCtx context.Context, dependant, // one written by an older/newer Compose version using a // condition value this build doesn't recognize. logrus.Warnf("service %q: unsupported depends_on condition %q, skipping wait for %q", dependant, config.Condition, dep) - return nil + return true, nil } if done || err != nil { - return err + return done, err } } } diff --git a/pkg/e2e/testdata/TestUpStartFailureReportsInterruptedServices/compose.yaml b/pkg/e2e/testdata/TestUpStartFailureReportsInterruptedServices/compose.yaml new file mode 100644 index 000000000..f448e464e --- /dev/null +++ b/pkg/e2e/testdata/TestUpStartFailureReportsInterruptedServices/compose.yaml @@ -0,0 +1,20 @@ +services: + # never becomes healthy: api, which waits for it, cannot start before web fails + db: + image: alpine + command: sleep infinity + healthcheck: + test: ["CMD", "false"] + interval: 1s + + api: + image: alpine + command: sleep infinity + depends_on: + db: + condition: service_healthy + + # fails to start: the executable does not exist + web: + image: alpine + command: ["/does-not-exist"] diff --git a/pkg/e2e/up_start_failure_test.go b/pkg/e2e/up_start_failure_test.go new file mode 100644 index 000000000..e8003ea43 --- /dev/null +++ b/pkg/e2e/up_start_failure_test.go @@ -0,0 +1,34 @@ +//go:build e2e + +/* + Copyright 2020 Docker Compose CLI authors + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package e2e + +import "testing" + +// TestUpStartFailureReportsInterruptedServices locks docker/compose#14372: when +// a service fails to start, the services the command could not start because of +// it are reported as such. A service held by a depends_on wait, created but +// never started, used to be left on its last status with nothing saying why. +func TestUpStartFailureReportsInterruptedServices(t *testing.T) { + NewScenario(t, "up must say why a service is not started when another service failed to start"). + Step("web fails to start, and api, held by the wait for db, is reported as not started", + ComposeCmd("up", "-d").MayFail(), + ExitCode(1), + ServiceState("api", "created"), + OutputContains(`not started: condition "service_healthy" of service "db" not reached: interrupted by the failure of`)) +}