From ef277d266b4ffb73100f063b610ad05da945ebda Mon Sep 17 00:00:00 2001 From: Nicolas De Loof Date: Wed, 19 Aug 2026 16:56:05 +0200 Subject: [PATCH] pre_start: inherit the whole container specification generically MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit pre_start hooks carry the full container specification, but the init-container runner only consumed a handful of attributes (image, command, user, env, workdir) — everything else was silently dropped. Instead of wiring attributes one by one, the hook's specification is merged over the service's ContainerSpec through compose-go's own file merge machinery (override.Merge on the canonical yaml tree): command and entrypoint replace, environment merges per key with the hook winning, extra_hosts and dns accumulate entries, ulimits merge — the exact per-attribute rules of multi-file compose, maintained in one place. Every ContainerSpec attribute inherits this way, current and future, with zero attribute-specific code. The merged spec then runs through the standard create path (getCreateConfigs) as a service-shaped one-off, so resources, capabilities, dns, sysctls, logging... materialize exactly as they would for a service container. Hook containers keep their minimal labels and carry no container-number, so tooling telling replicas apart does not count them. The only deliberate exception is volumes: mounts inherit at runtime through volumes_from — the only mechanism that shares the service's anonymous and image volumes — and the hook's own volume declarations, materialized by the standard path, take precedence per target. This is what lets an init container get read-write access to a volume the service mounts read-only (fixes: see PR). e2e scenarios lock extra_hosts inheritance and accumulation, volume override and completion, and the unit tests pin the merge rules. Signed-off-by: Nicolas De Loof --- pkg/compose/hook_spec.go | 89 +++++++++++++++++++ pkg/compose/hook_spec_test.go | 86 ++++++++++++++++++ pkg/compose/pre_start.go | 70 ++++++++------- pkg/compose/pre_start_test.go | 6 ++ pkg/e2e/hooks_test.go | 35 ++++++++ .../TestPreStartHookExtraHosts/compose.yaml | 21 +++++ .../TestPreStartHookVolumes/compose.yaml | 15 ++++ .../TestPreStartHookVolumesMerge/compose.yaml | 17 ++++ 8 files changed, 309 insertions(+), 30 deletions(-) create mode 100644 pkg/compose/hook_spec.go create mode 100644 pkg/compose/hook_spec_test.go create mode 100644 pkg/e2e/testdata/TestPreStartHookExtraHosts/compose.yaml create mode 100644 pkg/e2e/testdata/TestPreStartHookVolumes/compose.yaml create mode 100644 pkg/e2e/testdata/TestPreStartHookVolumesMerge/compose.yaml diff --git a/pkg/compose/hook_spec.go b/pkg/compose/hook_spec.go new file mode 100644 index 000000000..41017947f --- /dev/null +++ b/pkg/compose/hook_spec.go @@ -0,0 +1,89 @@ +/* + Copyright 2020 Docker Compose CLI authors + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package compose + +import ( + "errors" + "fmt" + + "github.com/compose-spec/compose-go/v2/loader" + "github.com/compose-spec/compose-go/v2/override" + "github.com/compose-spec/compose-go/v2/types" + yaml "go.yaml.in/yaml/v4" +) + +// mergedPreStartSpec merges the hook's container specification over the +// service's, following the compose file merge rules: command and entrypoint +// replace, environment merges per key with the hook winning, extra_hosts and +// dns accumulate entries, ulimits merge, ... Every ContainerSpec attribute +// inherits this way, current and future, without attribute-specific code. +// +// Volumes are deliberately dropped from the inherited side: mounts inherit +// at runtime through volumes_from, the only mechanism that shares the +// service's anonymous and image volumes; the merged spec carries the hook's +// own volume declarations only, which take precedence per target. +func mergedPreStartSpec(service types.ServiceConfig, hook types.PreStartHook) (types.ContainerSpec, error) { + base, err := containerSpecDict(service.ContainerSpec) + if err != nil { + return types.ContainerSpec{}, err + } + delete(base, "volumes") + over, err := containerSpecDict(hook.ContainerSpec) + if err != nil { + return types.ContainerSpec{}, err + } + + merged, err := override.Merge( + map[string]any{"services": map[string]any{"hook": base}}, + map[string]any{"services": map[string]any{"hook": over}}, + ) + if err != nil { + return types.ContainerSpec{}, fmt.Errorf("merging pre_start hook specification: %w", err) + } + dict, ok := merged["services"].(map[string]any)["hook"].(map[string]any) + if !ok { + return types.ContainerSpec{}, errors.New("internal: unexpected merged hook specification shape") + } + + var spec types.ContainerSpec + if err := loader.Transform(dict, &spec); err != nil { + return types.ContainerSpec{}, fmt.Errorf("decoding merged pre_start hook specification: %w", err) + } + return spec, nil +} + +// containerSpecDict serializes a ContainerSpec to the canonical yaml tree the +// compose merge rules operate on — the same serialization compose config +// uses. +func containerSpecDict(spec types.ContainerSpec) (map[string]any, error) { + p := &types.Project{Services: types.Services{"hook": {ContainerSpec: spec}}} + raw, err := p.MarshalYAML() + if err != nil { + return nil, err + } + var dict map[string]any + if err := yaml.Unmarshal(raw, &dict); err != nil { + return nil, err + } + svc, ok := dict["services"].(map[string]any)["hook"].(map[string]any) + if !ok { + // a zero spec marshals to a null service entry + return map[string]any{}, nil + } + delete(svc, "name") + return svc, nil +} diff --git a/pkg/compose/hook_spec_test.go b/pkg/compose/hook_spec_test.go new file mode 100644 index 000000000..2a116a962 --- /dev/null +++ b/pkg/compose/hook_spec_test.go @@ -0,0 +1,86 @@ +/* + Copyright 2020 Docker Compose CLI authors + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package compose + +import ( + "testing" + + "github.com/compose-spec/compose-go/v2/types" + "gotest.tools/v3/assert" + is "gotest.tools/v3/assert/cmp" +) + +func strPtr(s string) *string { return &s } + +func TestMergedPreStartSpec(t *testing.T) { + service := types.ServiceConfig{ + Name: "db", + ContainerSpec: types.ContainerSpec{ + Image: "postgres:16", + User: "999", + WorkingDir: "/srv", + Command: types.ShellCommand{"postgres"}, + Environment: types.MappingWithEquals{"PGDATA": strPtr("/data"), "SHARED": strPtr("service")}, + ExtraHosts: types.HostsList{"inherited": {"192.0.2.10"}, "shared": {"192.0.2.11"}}, + DNS: types.StringList{"1.1.1.1"}, + CapAdd: []string{"NET_ADMIN"}, + Volumes: []types.ServiceVolumeConfig{{Type: types.VolumeTypeVolume, Source: "data", Target: "/data", ReadOnly: true}}, + }, + } + hook := types.PreStartHook{ContainerSpec: types.ContainerSpec{ + Command: types.ShellCommand{"init.sh"}, + Environment: types.MappingWithEquals{"SHARED": strPtr("hook"), "ONLY": strPtr("hook")}, + ExtraHosts: types.HostsList{"declared": {"192.0.2.20"}, "shared": {"192.0.2.99"}}, + DNS: types.StringList{"8.8.8.8"}, + Volumes: []types.ServiceVolumeConfig{{Type: types.VolumeTypeVolume, Source: "data", Target: "/data"}}, + }} + + spec, err := mergedPreStartSpec(service, hook) + assert.NilError(t, err) + + // scalars inherit when undeclared + assert.Check(t, is.Equal(spec.Image, "postgres:16")) + assert.Check(t, is.Equal(spec.User, "999")) + assert.Check(t, is.Equal(spec.WorkingDir, "/srv")) + // command replaces + assert.Check(t, is.DeepEqual([]string(spec.Command), []string{"init.sh"})) + // collections merge per the compose file rules: environment per key + // (hook wins), extra_hosts accumulate entries per hostname + assert.Check(t, is.Equal(*spec.Environment["PGDATA"], "/data")) + assert.Check(t, is.Equal(*spec.Environment["SHARED"], "hook")) + assert.Check(t, is.Equal(*spec.Environment["ONLY"], "hook")) + assert.Check(t, is.DeepEqual(spec.ExtraHosts["inherited"], []string{"192.0.2.10"})) + assert.Check(t, is.DeepEqual(spec.ExtraHosts["declared"], []string{"192.0.2.20"})) + assert.Check(t, is.DeepEqual(spec.ExtraHosts["shared"], []string{"192.0.2.11", "192.0.2.99"})) + assert.Check(t, is.DeepEqual([]string(spec.DNS), []string{"1.1.1.1", "8.8.8.8"})) + // previously-unwired attributes inherit too, with zero dedicated code + assert.Check(t, is.DeepEqual(spec.CapAdd, []string{"NET_ADMIN"})) + // volumes are NOT inherited at the model level: runtime volumes_from does + assert.Assert(t, is.Len(spec.Volumes, 1)) + assert.Check(t, !spec.Volumes[0].ReadOnly, "hook keeps its own rw declaration") +} + +func TestMergedPreStartSpecEmptyHook(t *testing.T) { + service := types.ServiceConfig{ContainerSpec: types.ContainerSpec{ + Image: "alpine", Sysctls: types.Mapping{"net.core.somaxconn": "1024"}, + }} + spec, err := mergedPreStartSpec(service, types.PreStartHook{}) + assert.NilError(t, err) + assert.Check(t, is.Equal(spec.Image, "alpine")) + assert.Check(t, is.Equal(spec.Sysctls["net.core.somaxconn"], "1024")) + assert.Check(t, is.Len(spec.Volumes, 0)) +} diff --git a/pkg/compose/pre_start.go b/pkg/compose/pre_start.go index 1d350f605..77cdec2ca 100644 --- a/pkg/compose/pre_start.go +++ b/pkg/compose/pre_start.go @@ -166,48 +166,58 @@ func (s *composeService) createPreStartContainer( ctx context.Context, project *types.Project, service types.ServiceConfig, ctr container.Summary, hook types.PreStartHook, ) (client.ContainerCreateResult, error) { - image := hook.Image - if image == "" { - image = api.GetImageNameOrDefault(service, project.Name) + // A pre_start hook is a full container specification (compose-spec#656) + // inheriting from the service per the compose file merge rules; the + // merged spec runs through the standard create path, so every attribute + // — resources, capabilities, dns, sysctls, ... — materializes exactly as + // it would for a service container. + spec, err := mergedPreStartSpec(service, hook) + if err != nil { + return client.ContainerCreateResult{}, err } - - cfg := &container.Config{ - Image: image, - Cmd: hook.Command, - User: hook.User, - WorkingDir: hook.WorkingDir, - Env: append(ToMobyEnv(service.Environment), ToMobyEnv(hook.Environment)...), - // Tag the ephemeral hook container with the project/service it belongs - // to so it can be found by `compose down` and label-scoped tooling. - // HookLabel also distinguishes hook containers from the real service - // container (which shares ProjectLabel and ServiceLabel). - Labels: map[string]string{ + if spec.Image == "" { + spec.Image = api.GetImageNameOrDefault(service, project.Name) + } + hookService := types.ServiceConfig{ + Name: service.Name, + ContainerSpec: spec, + } + cfgs, err := s.getCreateConfigs(ctx, project, hookService, 0, nil, createOptions{ + // AutoRemove is intentionally false: a failed hook container is + // retained so the operator can inspect its logs. runPreStartHook + // removes it explicitly on success, and the orphan sweep catches + // leftovers on the next run. + AutoRemove: false, + UseNetworkAliases: true, + // Tag the ephemeral hook container with the project/service it + // belongs to so `compose down` and label-scoped tooling can find it. + // HookLabel distinguishes hook containers from the real service + // container; no container-number: tooling telling replicas apart must + // not count hook containers. + Labels: types.Labels{ api.ProjectLabel: project.Name, api.ServiceLabel: service.Name, api.VersionLabel: api.ComposeVersion, api.HookLabel: preStartHookType, }, + }) + if err != nil { + return client.ContainerCreateResult{}, err } - hostCfg := &container.HostConfig{ - // AutoRemove is intentionally false: a failed hook container is retained - // so the operator can inspect its logs. On success runPreStartHook - // removes the container explicitly (see the success path below). - AutoRemove: false, - Privileged: hook.Privileged, - VolumesFrom: []string{ctr.ID}, - } + // Mounts inherit from the live service container: volumes_from carries + // its anonymous and image volumes too. The hook's own mounts, already in + // the host config from the merged spec, take precedence on shared + // targets. + cfgs.Host.VolumesFrom = append(cfgs.Host.VolumesFrom, ctr.ID) + cfg := cfgs.Container + hostCfg := cfgs.Host + networkingConfig := cfgs.Network apiVersion, err := s.RuntimeAPIVersion(ctx) if err != nil { return client.ContainerCreateResult{}, err } - networkMode, networkingConfig, err := defaultNetworkSettings(project, service, 0, nil, true, apiVersion) - if err != nil { - return client.ContainerCreateResult{}, err - } - hostCfg.NetworkMode = networkMode - created, err := s.apiClient().ContainerCreate(ctx, client.ContainerCreateOptions{ Config: cfg, HostConfig: hostCfg, @@ -218,7 +228,7 @@ func (s *composeService) createPreStartContainer( } if versions.LessThan(apiVersion, apiVersion144) { - if err := s.connectPreStartExtraNetworks(ctx, project, service, created.ID, networkMode); err != nil { + if err := s.connectPreStartExtraNetworks(ctx, project, service, created.ID, hostCfg.NetworkMode); err != nil { // AutoRemove is false; remove the container explicitly since it was // never started. Log failures so the orphan is at least visible. if _, removeErr := s.apiClient().ContainerRemove(ctx, created.ID, client.ContainerRemoveOptions{Force: true, RemoveVolumes: true}); removeErr != nil { diff --git a/pkg/compose/pre_start_test.go b/pkg/compose/pre_start_test.go index 1f803f036..b744db9c9 100644 --- a/pkg/compose/pre_start_test.go +++ b/pkg/compose/pre_start_test.go @@ -25,6 +25,7 @@ import ( "testing" "github.com/compose-spec/compose-go/v2/types" + "github.com/docker/cli/cli/config/configfile" "github.com/moby/moby/api/types/container" "github.com/moby/moby/client" "go.uber.org/goleak" @@ -50,9 +51,11 @@ func newPreStartTestServiceWithVersion(t *testing.T, apiVersion string) (*compos apiClient := mocks.NewMockAPIClient(mockCtrl) cli := mocks.NewMockCli(mockCtrl) cli.EXPECT().Client().Return(apiClient).AnyTimes() + cli.EXPECT().ConfigFile().Return(&configfile.ConfigFile{}).AnyTimes() apiClient.EXPECT().Ping(gomock.Any(), client.PingOptions{NegotiateAPIVersion: true}). Return(client.PingResult{APIVersion: apiVersion}, nil).AnyTimes() apiClient.EXPECT().ClientVersion().Return(apiVersion).AnyTimes() + apiClient.EXPECT().DaemonHost().Return("unix:///var/run/docker.sock").AnyTimes() tested, err := NewComposeService(cli) assert.NilError(t, err) return tested.(*composeService), apiClient @@ -1129,6 +1132,9 @@ func TestPreStart_RuntimeAPIVersionError(t *testing.T) { apiClient := mocks.NewMockAPIClient(mockCtrl) cli := mocks.NewMockCli(mockCtrl) cli.EXPECT().Client().Return(apiClient).AnyTimes() + // the merged-spec create path reads the CLI config and daemon host + cli.EXPECT().ConfigFile().Return(&configfile.ConfigFile{}).AnyTimes() + apiClient.EXPECT().DaemonHost().Return("unix:///var/run/docker.sock").AnyTimes() tested, err := NewComposeService(cli) assert.NilError(t, err) s := tested.(*composeService) diff --git a/pkg/e2e/hooks_test.go b/pkg/e2e/hooks_test.go index e7db829f3..77063f505 100644 --- a/pkg/e2e/hooks_test.go +++ b/pkg/e2e/hooks_test.go @@ -209,3 +209,38 @@ func TestPreStartHookRunsOnceForScaledService(t *testing.T) { probeVolume(s, "wc", "-l", "/mnt/log"), OutputContains("1 /mnt/log")) } + +// A pre_start hook is a full container specification inheriting in the +// spirit of the yaml merge rules: collections complete the inherited value, +// the hook's declarations winning on conflicts (docker/compose#13939). +func TestPreStartHookExtraHosts(t *testing.T) { + NewScenario(t, "pre_start hooks must inherit the service's extra_hosts, completed by their own declarations"). + Step("up succeeds: both hooks asserted their /etc/hosts content", + ComposeCmd("up", "-d", "--wait").Within(60*time.Second), + ServiceState("app", "running")) +} + +// The init-container use case of docker/compose#13934: the hook redeclares a +// service volume read-write to prepare content the service mounts read-only. +func TestPreStartHookVolumes(t *testing.T) { + NewScenario(t, "a pre_start hook must write through its own read-write volume declaration while the service stays read-only"). + Step("up succeeds: the hook wrote through its rw mount", + ComposeCmd("up", "-d", "--wait").Within(60*time.Second)). + Step("the service reads what the hook prepared", + ComposeCmd("exec", "app", "cat", "/data/init.txt"), + OutputContains("initialized")). + Step("the service's own mount is still read-only", + ComposeCmd("exec", "app", "touch", "/data/blocked").MayFail(), + OutputContains("Read-only file system")) +} + +// Volumes merge by target: the hook keeps every service mount it did not +// redeclare (docker/compose#13934). +func TestPreStartHookVolumesMerge(t *testing.T) { + NewScenario(t, "a pre_start hook declaring a volume must still see the service's other mounts"). + Step("up succeeds: the hook read the inherited mount and wrote through its own", + ComposeCmd("up", "-d", "--wait").Within(60*time.Second)). + Step("the hook combined the inherited mount's content with its own write", + ComposeCmd("exec", "app", "cat", "/data/init.txt"), + OutputContains("saw-config")) +} diff --git a/pkg/e2e/testdata/TestPreStartHookExtraHosts/compose.yaml b/pkg/e2e/testdata/TestPreStartHookExtraHosts/compose.yaml new file mode 100644 index 000000000..ef78d6cdb --- /dev/null +++ b/pkg/e2e/testdata/TestPreStartHookExtraHosts/compose.yaml @@ -0,0 +1,21 @@ +services: + app: + image: alpine + init: true + command: sleep infinity + extra_hosts: + - "inherited-host:192.0.2.10" + - "shared-host:192.0.2.11" + pre_start: + # no extra_hosts declared: the service's entries are inherited + - command: sh -c 'grep inherited-host /etc/hosts' + # declared extra_hosts complete the inherited ones; entries for the + # same hostname accumulate, exactly like the compose file merge rules + - command: > + sh -c 'grep declared-host /etc/hosts && + grep inherited-host /etc/hosts && + grep "192.0.2.99.*shared-host" /etc/hosts && + grep "192.0.2.11.*shared-host" /etc/hosts' + extra_hosts: + - "declared-host:192.0.2.20" + - "shared-host:192.0.2.99" diff --git a/pkg/e2e/testdata/TestPreStartHookVolumes/compose.yaml b/pkg/e2e/testdata/TestPreStartHookVolumes/compose.yaml new file mode 100644 index 000000000..26b0fd037 --- /dev/null +++ b/pkg/e2e/testdata/TestPreStartHookVolumes/compose.yaml @@ -0,0 +1,15 @@ +services: + app: + image: alpine + init: true + command: sleep infinity + volumes: + - data:/data:ro + pre_start: + # the hook redeclares the volume read-write to prepare content the + # service will only ever read + - command: sh -c 'echo initialized > /data/init.txt' + volumes: + - data:/data:rw +volumes: + data: {} diff --git a/pkg/e2e/testdata/TestPreStartHookVolumesMerge/compose.yaml b/pkg/e2e/testdata/TestPreStartHookVolumesMerge/compose.yaml new file mode 100644 index 000000000..2a48265b9 --- /dev/null +++ b/pkg/e2e/testdata/TestPreStartHookVolumesMerge/compose.yaml @@ -0,0 +1,17 @@ +services: + app: + image: alpine + init: true + command: sleep infinity + volumes: + - data:/data:ro + - config:/config:ro + pre_start: + # declares only /data read-write: /config must still be visible, + # inherited through volumes_from + - command: sh -c 'test -d /config && echo "saw-config" > /data/init.txt' + volumes: + - data:/data:rw +volumes: + data: {} + config: {}