From 0a64ff09a8a04bbd261f8b8c8a3977a65b17aad2 Mon Sep 17 00:00:00 2001 From: Nicolas De Loof Date: Thu, 24 Sep 2026 09:30:38 +0200 Subject: [PATCH] fix: address copilot-pull-request-reviewer's 3 findings on the hook-runner plan A GitHub Copilot PR review (docker/compose#14221#pullrequestreview-5300897216) found three real issues in the reconciliation-plan-based hook runners, all confirmed by direct reproduction against a live daemon before being fixed: - **Hook runners excluded from discovery, stale runners persist.** collectObservedState's single getContainers call requires ConfigHashLabel (getDefaultFilters), which hook runners deliberately never carry -- so they never reached the classification loop that populates HookContainers, and planPreStartHookRunners never saw a stale runner to purge. Reproduced: a failed hook run followed by a second `up` failed with "container name ... already in use" instead of recovering. Fixed by listing hook containers separately, by project+hook label alone (new getHookContainers), and merging them in before classification. Locked in by a new e2e scenario, TestPreStartHookRetryAfterFailureSucceeds, that reproduces the exact failure-then-retry sequence. - **Prepared hook runners misclassified as service replicas.** start's containers.filter(isService(name), isNotOneOff) has no reason to exclude a hook runner: it carries the same project/service labels as a real replica and no one-off label either. Added isNotHookContainer to the filter chain in startService, so a hook runner is never counted as (or started as) a surviving replica. - **Per-replica hooks create runners before unsupported configuration is rejected.** per_replica is not yet supported (#14259): runPreStart validates every hook up front and rejects the whole service before executing any of them, but planPreStartHookRunners had no equivalent check, so create/up would plan (and leave behind) runners for a service whose hooks could never run. Mirrored the same up-front, all-or-nothing validation in planPreStartHookRunners. Signed-off-by: Nicolas De Loof --- pkg/compose/containers.go | 25 +++++++++++++ pkg/compose/observed_state.go | 34 ++++++++++++++++++ pkg/compose/observed_state_test.go | 14 +++++++- pkg/compose/pre_start.go | 1 + pkg/compose/reconcile.go | 11 ++++++ pkg/compose/reconcile_test.go | 35 +++++++++++++++++++ pkg/compose/service_containers.go | 2 +- pkg/e2e/hooks_test.go | 16 +++++++++ .../compose.yaml | 11 ++++++ 9 files changed, 147 insertions(+), 2 deletions(-) create mode 100644 pkg/e2e/testdata/TestPreStartHookRetryAfterFailureSucceeds/compose.yaml diff --git a/pkg/compose/containers.go b/pkg/compose/containers.go index c6ce6474f..0357326a9 100644 --- a/pkg/compose/containers.go +++ b/pkg/compose/containers.go @@ -57,6 +57,23 @@ func (s *composeService) getContainers(ctx context.Context, projectName string, return containers, nil } +// getHookContainers returns every pre_start hook runner container in the +// project, in any state. Hook runners deliberately carry no ConfigHashLabel +// (see createPreStartContainer), so they are invisible to getContainers' +// default filters and must be listed by project+hook label alone. +func (s *composeService) getHookContainers(ctx context.Context, projectName string) (Containers, error) { + f := projectFilter(projectName) + f.Add("label", hookFilter(preStartHookType)) + res, err := s.apiClient().ContainerList(ctx, client.ContainerListOptions{ + All: true, + Filters: f, + }) + if err != nil { + return nil, err + } + return Containers(res.Items), nil +} + // getContainersByService returns all non-oneoff containers for the project, grouped by service name. func (s *composeService) getContainersByService(ctx context.Context, projectName string) (map[string]Containers, error) { all, err := s.getContainers(ctx, projectName, oneOffExclude, true) @@ -170,6 +187,14 @@ func isNotOneOff(c container.Summary) bool { return !ok || v == "False" } +// isNotHookContainer excludes pre_start hook runners: they carry the same +// project/service labels as a real replica (and no one-off label either) +// while being none -- starting one, or counting it as a surviving replica, +// is always wrong. +func isNotHookContainer(c container.Summary) bool { + return c.Labels[api.HookLabel] == "" +} + func isNotRunning(c container.Summary) bool { return c.State != container.StateRunning } diff --git a/pkg/compose/observed_state.go b/pkg/compose/observed_state.go index 32a272ff3..932416ef0 100644 --- a/pkg/compose/observed_state.go +++ b/pkg/compose/observed_state.go @@ -147,6 +147,36 @@ type ObservedVolume struct { // collectObservedState queries the Docker daemon for all resources belonging to // the given project and returns a structured snapshot. +// mergeHookContainers appends the pre_start hook runner containers to raw. +// Runners deliberately carry no ConfigHashLabel (kept invisible to ps/start's +// default listing, which requires it -- see createPreStartContainer) -- so +// getContainers' filters never return them, and they must be listed +// separately, by project+hook label alone, or the classification loop never +// sees them: stale runners would never be purged, and the next deterministic +// create would fail with a name conflict against the one left behind. The +// extra round-trip is only paid when the model declares pre_start hooks at +// all. Accepted corner: a runner whose service left the model is only +// classified (and swept by --remove-orphans) while some remaining service +// still declares pre_start hooks -- `down` removes it by hook label +// regardless (see removePreStartHookContainers). +func (s *composeService) mergeHookContainers(ctx context.Context, project *types.Project, raw Containers) (Containers, error) { + hasPreStartHooks := false + for _, svc := range project.Services { + if len(svc.PreStart) > 0 { + hasPreStartHooks = true + break + } + } + if !hasPreStartHooks { + return raw, nil + } + hookRaw, err := s.getHookContainers(ctx, project.Name) + if err != nil { + return nil, err + } + return append(raw, hookRaw...), nil +} + // The project model is used to classify containers by service and to identify // orphans, and to scope network/volume queries to declared resources. func (s *composeService) collectObservedState(ctx context.Context, project *types.Project) (*ObservedState, error) { @@ -167,6 +197,10 @@ func (s *composeService) collectObservedState(ctx context.Context, project *type if err != nil { return nil, err } + raw, err = s.mergeHookContainers(ctx, project, raw) + if err != nil { + return nil, err + } knownServices := map[string]bool{} for _, service := range project.Services { diff --git a/pkg/compose/observed_state_test.go b/pkg/compose/observed_state_test.go index a09a09201..bff197599 100644 --- a/pkg/compose/observed_state_test.go +++ b/pkg/compose/observed_state_test.go @@ -96,7 +96,9 @@ func TestCollectObservedState(t *testing.T) { project := &types.Project{ Name: "myproject", Services: types.Services{ - "web": {Name: "web"}, + // web declares a hook: the separate runner listing only fires + // when some service declares pre_start at all + "web": {Name: "web", PreStart: []types.PreStartHook{{}}}, "db": {Name: "db"}, }, Networks: types.Networks{ @@ -160,6 +162,14 @@ func TestCollectObservedState(t *testing.T) { api.OneoffLabel: "True", }, }, + }, + }, nil) + + // Mock ContainerList for getHookContainers: hook runners carry no + // ConfigHashLabel, so they are listed separately (see + // collectObservedState) via a project+hook-label filter alone. + apiClient.EXPECT().ContainerList(gomock.Any(), gomock.Any()).Return(client.ContainerListResult{ + Items: []container.Summary{ { // Stale lifecycle-hook runner (a previous run failed before // removing it): neither a replica (it has no container-number @@ -335,6 +345,8 @@ func TestSelectNetwork(t *testing.T) { func collectByNameDiscovery(t *testing.T, project *types.Project, inspect func(apiClient *mocks.MockAPIClient)) (*ObservedState, error) { t.Helper() svc, apiClient := newTestService(t) + // hook-less projects: getHookContainers is gated off, only getContainers + // lists (see mergeHookContainers) apiClient.EXPECT().ContainerList(gomock.Any(), gomock.Any()).Return(client.ContainerListResult{}, nil) apiClient.EXPECT().NetworkList(gomock.Any(), gomock.Any()).Return(client.NetworkListResult{}, nil) apiClient.EXPECT().VolumeList(gomock.Any(), gomock.Any()).Return(client.VolumeListResult{}, nil) diff --git a/pkg/compose/pre_start.go b/pkg/compose/pre_start.go index 5e1d8d388..6542f22a1 100644 --- a/pkg/compose/pre_start.go +++ b/pkg/compose/pre_start.go @@ -100,6 +100,7 @@ func (s *composeService) runPreStart(ctx context.Context, project *types.Project // Validate every hook up front so an unsupported entry never triggers any I/O. for i, hook := range service.PreStart { if hook.PerReplica { + // per_replica is not yet supported: docker/compose#14259. return fmt.Errorf("service %q pre_start[%d]: per_replica is not yet supported; remove per_replica or set it to false", service.Name, i) } } diff --git a/pkg/compose/reconcile.go b/pkg/compose/reconcile.go index 728b9946a..1b1279281 100644 --- a/pkg/compose/reconcile.go +++ b/pkg/compose/reconcile.go @@ -919,6 +919,17 @@ func (r *reconciler) planPreStartHookRunners(service types.ServiceConfig, expect if keptRunning { return } + for _, hook := range service.PreStart { + if hook.PerReplica { + // per_replica is not yet supported (docker/compose#14259): + // runPreStart validates every hook up front and rejects the + // whole service before executing any of them (see runPreStart). + // Planning creates here would only orphan runners that can + // never run — nothing to prepare until the hook declares + // per_replica: false. + return + } + } serviceCopy := service deps := slices.Concat(containerNodes, infraDeps) stale := slices.Clone(r.observed.HookContainers[service.Name]) diff --git a/pkg/compose/reconcile_test.go b/pkg/compose/reconcile_test.go index d7b4c8486..10c23b076 100644 --- a/pkg/compose/reconcile_test.go +++ b/pkg/compose/reconcile_test.go @@ -1499,6 +1499,41 @@ func TestReconcileContainers_HookRunnersPlannedOnColdStart(t *testing.T) { } } +// per_replica is not yet supported (docker/compose#14259): runPreStart +// validates every declared hook up front and rejects the whole service +// before executing any of them, so planning creates for it here would only +// orphan runners that can never run. No hook runner is planned for such a +// service, even for its other, otherwise-valid hooks. +func TestReconcileContainers_HookRunnersNotPlannedWhenPerReplica(t *testing.T) { + project := &types.Project{ + Name: "myproject", + Services: types.Services{ + "app": { + Name: "app", + Scale: intPtr(1), + PreStart: []types.PreStartHook{ + {}, + {PerReplica: true}, + }, + }, + }, + } + observed := &ObservedState{ + ProjectName: "myproject", + Containers: map[string][]ObservedContainer{}, + HookContainers: map[string][]ObservedContainer{}, + Networks: map[string][]ObservedNetwork{}, + Volumes: map[string][]ObservedVolume{}, + } + + plan, err := reconcile(t.Context(), project, observed, defaultReconcileOptions(), noPrompt) + assert.NilError(t, err) + + assert.Equal(t, plan.String(), strings.TrimSpace(` +[] -> #1 service:app:1, CreateContainer, no existing container +`)+"\n") +} + // A running replica whose config diverged is recreated, so it will not be // running when the start phase evaluates pre_start: the runner must be // planned. The observed-running gate applies to replicas that SURVIVE the diff --git a/pkg/compose/service_containers.go b/pkg/compose/service_containers.go index 2a98b8b89..dbed6cd50 100644 --- a/pkg/compose/service_containers.go +++ b/pkg/compose/service_containers.go @@ -598,7 +598,7 @@ func (s *composeService) startService(ctx context.Context, return errNoContainerToStart(service.Name) } - serviceContainers := containers.filter(isService(service.Name), isNotOneOff) + serviceContainers := containers.filter(isService(service.Name), isNotOneOff, isNotHookContainer) toStart := serviceContainers.filter(isNotRunning) if len(toStart) == 0 { return nil diff --git a/pkg/e2e/hooks_test.go b/pkg/e2e/hooks_test.go index 461627bfa..d5494f1a9 100644 --- a/pkg/e2e/hooks_test.go +++ b/pkg/e2e/hooks_test.go @@ -106,6 +106,22 @@ func TestPreStartHookInError(t *testing.T) { ServiceState("sample", "created")) } +// A retained runner from a failed run must not block the next one: it is a +// stale hook container the reconciler has to observe and purge before +// creating a fresh one under the same deterministic name, or the second up +// fails with a container-name conflict instead of recovering. +func TestPreStartHookRetryAfterFailureSucceeds(t *testing.T) { + s := NewScenario(t, "up must recover from a previously failed hook by purging its stale runner") + s.Step("first up fails on the hook", + ComposeCmd("up", "-d").MayFail(), + ExitCode(1), + OutputContains("pre_start"), + ServiceState("sample", "created")). + Step("second up purges the stale runner and succeeds", + ComposeCmd("up", "-d", "--wait").Within(60*time.Second), + ServiceState("sample", "running")) +} + func TestPreStartHookBuildInheritance(t *testing.T) { s := NewScenario(t, "a pre_start hook without an image must run on the service's built image") s.Defer(DockerCmd("image", "rm", "-f", s.Project()+"-sample")). diff --git a/pkg/e2e/testdata/TestPreStartHookRetryAfterFailureSucceeds/compose.yaml b/pkg/e2e/testdata/TestPreStartHookRetryAfterFailureSucceeds/compose.yaml new file mode 100644 index 000000000..7c17fe314 --- /dev/null +++ b/pkg/e2e/testdata/TestPreStartHookRetryAfterFailureSucceeds/compose.yaml @@ -0,0 +1,11 @@ +services: + sample: + image: alpine + command: sh -c 'sleep 30' + volumes: + - data:/mnt + pre_start: + - image: alpine + command: sh -c 'test -f /mnt/marker && exit 0 || { touch /mnt/marker; exit 17; }' +volumes: + data: