From 1af7ced7cd8d4363cd44da463ad2d5f3dada878a Mon Sep 17 00:00:00 2001 From: Nicolas De Loof Date: Sun, 19 Apr 2026 17:30:25 +0200 Subject: [PATCH] Reconciliation : networks and volumes Signed-off-by: Nicolas De Loof --- pkg/compose/reconcile.go | 292 ++++++++++++++++++++++++++ pkg/compose/reconcile_test.go | 372 ++++++++++++++++++++++++++++++++++ 2 files changed, 664 insertions(+) create mode 100644 pkg/compose/reconcile.go create mode 100644 pkg/compose/reconcile_test.go diff --git a/pkg/compose/reconcile.go b/pkg/compose/reconcile.go new file mode 100644 index 000000000..372cbab9c --- /dev/null +++ b/pkg/compose/reconcile.go @@ -0,0 +1,292 @@ +/* + Copyright 2020 Docker Compose CLI authors + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package compose + +import ( + "context" + "fmt" + + "github.com/compose-spec/compose-go/v2/types" +) + +// ReconcileOptions controls how the reconciler compares desired and observed state. +type ReconcileOptions struct { + Services []string // targeted services (empty = all) + Recreate string // "diverged", "force", "never" for targeted services + RecreateDependencies string // same for non-targeted services + Inherit bool // inherit anonymous volumes on recreate + RemoveOrphans bool + SkipProviders bool +} + +// reconciler compares a types.Project (desired state) with an ObservedState +// (actual state) and produces a Plan — a DAG of atomic operations. +type reconciler struct { + project *types.Project + observed *ObservedState + options ReconcileOptions + prompt Prompt + plan *Plan +} + +// reconcile is the main entry point: it builds a Plan from desired vs observed state. +// The prompt function is called for interactive decisions (e.g. volume divergence). +func reconcile(_ context.Context, project *types.Project, observed *ObservedState, options ReconcileOptions, prompt Prompt) (*Plan, error) { + r := &reconciler{ + project: project, + observed: observed, + options: options, + prompt: prompt, + plan: &Plan{}, + } + + if err := r.reconcileNetworks(); err != nil { + return nil, err + } + + if err := r.reconcileVolumes(); err != nil { + return nil, err + } + + return r.plan, nil +} + +// reconcileNetworks adds plan nodes for network creation or recreation. +func (r *reconciler) reconcileNetworks() error { + for key, desired := range r.project.Networks { + if desired.External { + continue + } + observed, exists := r.observed.Networks[key] + if !exists { + r.planCreateNetwork(key, &desired) + continue + } + + expectedHash, err := NetworkHash(&desired) + if err != nil { + return err + } + if observed.ConfigHash != "" && observed.ConfigHash != expectedHash { + if err := r.planRecreateNetwork(key, &desired); err != nil { + return err + } + } + // else: network exists and config matches, nothing to do + } + return nil +} + +// planCreateNetwork adds a single CreateNetwork node. +func (r *reconciler) planCreateNetwork(key string, nw *types.NetworkConfig) *PlanNode { + return r.plan.addNode(Operation{ + Type: OpCreateNetwork, + ResourceID: fmt.Sprintf("network:%s", key), + Cause: "not found", + Name: nw.Name, + Network: nw, + }, "") +} + +// planRecreateNetwork adds the full sequence for a diverged network: +// stop affected containers → disconnect → remove network → create network. +func (r *reconciler) planRecreateNetwork(key string, nw *types.NetworkConfig) error { + affectedServices := r.servicesUsingNetwork(key) + affectedContainers := r.containersForServices(affectedServices) + + // Stop all affected containers + var stopNodes []*PlanNode + for i := range affectedContainers { + oc := &affectedContainers[i] + node := r.plan.addNode(Operation{ + Type: OpStopContainer, + ResourceID: fmt.Sprintf("service:%s:%d", oc.Summary.Labels[serviceLabel], oc.Number), + Cause: fmt.Sprintf("network %s config changed", key), + Container: &oc.Summary, + }, "") + stopNodes = append(stopNodes, node) + } + + // Disconnect all affected containers (each depends on its own stop) + var disconnectNodes []*PlanNode + for i, oc := range affectedContainers { + node := r.plan.addNode(Operation{ + Type: OpDisconnectNetwork, + ResourceID: fmt.Sprintf("service:%s:%d", oc.Summary.Labels[serviceLabel], oc.Number), + Cause: fmt.Sprintf("network %s recreate", key), + Container: &affectedContainers[i].Summary, + Name: nw.Name, + }, "", stopNodes[i]) + disconnectNodes = append(disconnectNodes, node) + } + + // Remove network (depends on all disconnects) + removeNode := r.plan.addNode(Operation{ + Type: OpRemoveNetwork, + ResourceID: fmt.Sprintf("network:%s", key), + Cause: "config hash diverged", + Name: nw.Name, + Network: nw, + }, "", disconnectNodes...) + + // Create network (depends on remove) + r.plan.addNode(Operation{ + Type: OpCreateNetwork, + ResourceID: fmt.Sprintf("network:%s", key), + Cause: "recreate after config change", + Name: nw.Name, + Network: nw, + }, "", removeNode) + + return nil +} + +// reconcileVolumes adds plan nodes for volume creation or recreation. +func (r *reconciler) reconcileVolumes() error { + for key, desired := range r.project.Volumes { + if desired.External { + continue + } + observed, exists := r.observed.Volumes[key] + if !exists { + r.planCreateVolume(key, &desired) + continue + } + + expectedHash, err := VolumeHash(desired) + if err != nil { + return err + } + if observed.ConfigHash != "" && observed.ConfigHash != expectedHash { + confirmed, err := r.prompt( + fmt.Sprintf("Volume %q exists but doesn't match configuration in compose file. Recreate (data will be lost)?", desired.Name), + false, + ) + if err != nil { + return err + } + if confirmed { + r.planRecreateVolume(key, &desired) + } + } + // else: volume exists and config matches, nothing to do + } + return nil +} + +// planCreateVolume adds a single CreateVolume node. +func (r *reconciler) planCreateVolume(key string, vol *types.VolumeConfig) *PlanNode { + return r.plan.addNode(Operation{ + Type: OpCreateVolume, + ResourceID: fmt.Sprintf("volume:%s", key), + Cause: "not found", + Name: vol.Name, + Volume: vol, + }, "") +} + +// planRecreateVolume adds the full sequence for a diverged volume: +// stop affected containers → remove containers → remove volume → create volume. +// Containers must be removed (not just stopped) because Docker does not allow +// removing a volume that is referenced by any container, even a stopped one. +func (r *reconciler) planRecreateVolume(key string, vol *types.VolumeConfig) { + affectedServices := r.servicesUsingVolume(key) + affectedContainers := r.containersForServices(affectedServices) + + // Stop all affected containers + var stopNodes []*PlanNode + for i := range affectedContainers { + oc := &affectedContainers[i] + node := r.plan.addNode(Operation{ + Type: OpStopContainer, + ResourceID: fmt.Sprintf("service:%s:%d", oc.Summary.Labels[serviceLabel], oc.Number), + Cause: fmt.Sprintf("volume %s config changed", key), + Container: &oc.Summary, + }, "") + stopNodes = append(stopNodes, node) + } + + // Remove all affected containers (each depends on its own stop) + var removeNodes []*PlanNode + for i, oc := range affectedContainers { + node := r.plan.addNode(Operation{ + Type: OpRemoveContainer, + ResourceID: fmt.Sprintf("service:%s:%d", oc.Summary.Labels[serviceLabel], oc.Number), + Cause: fmt.Sprintf("volume %s config changed", key), + Container: &affectedContainers[i].Summary, + }, "", stopNodes[i]) + removeNodes = append(removeNodes, node) + } + + // Remove volume (depends on all container removals) + removeVolNode := r.plan.addNode(Operation{ + Type: OpRemoveVolume, + ResourceID: fmt.Sprintf("volume:%s", key), + Cause: "config hash diverged", + Name: vol.Name, + Volume: vol, + }, "", removeNodes...) + + // Create volume (depends on remove) + r.plan.addNode(Operation{ + Type: OpCreateVolume, + ResourceID: fmt.Sprintf("volume:%s", key), + Cause: "recreate after config change", + Name: vol.Name, + Volume: vol, + }, "", removeVolNode) +} + +// servicesUsingNetwork returns the names of services that reference the given +// compose network key. +func (r *reconciler) servicesUsingNetwork(networkKey string) []string { + var names []string + for _, svc := range r.project.Services { + if _, ok := svc.Networks[networkKey]; ok { + names = append(names, svc.Name) + } + } + return names +} + +// servicesUsingVolume returns the names of services that mount the given +// compose volume key. +func (r *reconciler) servicesUsingVolume(volumeKey string) []string { + var names []string + for _, svc := range r.project.Services { + for _, v := range svc.Volumes { + if v.Source == volumeKey { + names = append(names, svc.Name) + break + } + } + } + return names +} + +// containersForServices returns all observed containers belonging to the given +// service names. +func (r *reconciler) containersForServices(services []string) []ObservedContainer { + var result []ObservedContainer + for _, svc := range services { + result = append(result, r.observed.Containers[svc]...) + } + return result +} + +// serviceLabel is a package-level shorthand for the service label key. +const serviceLabel = "com.docker.compose.service" diff --git a/pkg/compose/reconcile_test.go b/pkg/compose/reconcile_test.go new file mode 100644 index 000000000..5a8a9e0c1 --- /dev/null +++ b/pkg/compose/reconcile_test.go @@ -0,0 +1,372 @@ +/* + Copyright 2020 Docker Compose CLI authors + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package compose + +import ( + "testing" + + "github.com/compose-spec/compose-go/v2/types" + "github.com/moby/moby/api/types/container" + "gotest.tools/v3/assert" + + "github.com/docker/compose/v5/pkg/api" +) + +// noPrompt is a Prompt that should never be called in tests that don't expect it. +func noPrompt(msg string, def bool) (bool, error) { + panic("unexpected prompt call: " + msg) +} + +func alwaysYesPrompt(string, bool) (bool, error) { return true, nil } +func alwaysNoPrompt(string, bool) (bool, error) { return false, nil } + +func defaultReconcileOptions() ReconcileOptions { + return ReconcileOptions{ + Recreate: api.RecreateDiverged, + RecreateDependencies: api.RecreateDiverged, + Inherit: true, + } +} + +// --- Network tests --- + +func TestReconcileNetworks_CreateMissing(t *testing.T) { + project := &types.Project{ + Name: "myproject", + Networks: types.Networks{ + "frontend": {Name: "myproject_frontend"}, + "backend": {Name: "myproject_backend"}, + }, + } + observed := &ObservedState{ + ProjectName: "myproject", + Containers: map[string][]ObservedContainer{}, + Networks: map[string]ObservedNetwork{}, + Volumes: map[string]ObservedVolume{}, + } + + plan, err := reconcile(t.Context(), project, observed, defaultReconcileOptions(), noPrompt) + assert.NilError(t, err) + + s := plan.String() + assert.Assert(t, containsLine(s, "CreateNetwork, not found")) + assert.Equal(t, len(plan.Nodes), 2) +} + +func TestReconcileNetworks_ExistingMatch(t *testing.T) { + nw := types.NetworkConfig{Name: "myproject_frontend"} + hash, err := NetworkHash(&nw) + assert.NilError(t, err) + + project := &types.Project{ + Name: "myproject", + Networks: types.Networks{"frontend": nw}, + } + observed := &ObservedState{ + ProjectName: "myproject", + Containers: map[string][]ObservedContainer{}, + Networks: map[string]ObservedNetwork{ + "frontend": {ID: "net1", Name: "myproject_frontend", ConfigHash: hash}, + }, + Volumes: map[string]ObservedVolume{}, + } + + plan, err := reconcile(t.Context(), project, observed, defaultReconcileOptions(), noPrompt) + assert.NilError(t, err) + assert.Assert(t, plan.IsEmpty()) +} + +func TestReconcileNetworks_ExternalSkipped(t *testing.T) { + project := &types.Project{ + Name: "myproject", + Networks: types.Networks{ + "ext": {Name: "external_net", External: true}, + }, + } + observed := &ObservedState{ + ProjectName: "myproject", + Containers: map[string][]ObservedContainer{}, + Networks: map[string]ObservedNetwork{}, + Volumes: map[string]ObservedVolume{}, + } + + plan, err := reconcile(t.Context(), project, observed, defaultReconcileOptions(), noPrompt) + assert.NilError(t, err) + assert.Assert(t, plan.IsEmpty()) +} + +func TestReconcileNetworks_Diverged(t *testing.T) { + project := &types.Project{ + Name: "myproject", + Networks: types.Networks{ + "frontend": {Name: "myproject_frontend", Driver: "overlay"}, + }, + Services: types.Services{ + "web": { + Name: "web", + Networks: map[string]*types.ServiceNetworkConfig{"frontend": {}}, + }, + }, + } + observed := &ObservedState{ + ProjectName: "myproject", + Containers: map[string][]ObservedContainer{ + "web": { + { + ID: "c1", Number: 1, State: container.StateRunning, + Summary: container.Summary{ + ID: "c1", + Labels: map[string]string{ + api.ServiceLabel: "web", + api.ContainerNumberLabel: "1", + }, + }, + }, + }, + }, + Networks: map[string]ObservedNetwork{ + "frontend": {ID: "net1", Name: "myproject_frontend", ConfigHash: "oldhash"}, + }, + Volumes: map[string]ObservedVolume{}, + } + + plan, err := reconcile(t.Context(), project, observed, defaultReconcileOptions(), noPrompt) + assert.NilError(t, err) + + expected := "[] -> #1 service:web:1, StopContainer, network frontend config changed\n" + + "[1] -> #2 service:web:1, DisconnectNetwork, network frontend recreate\n" + + "[2] -> #3 network:frontend, RemoveNetwork, config hash diverged\n" + + "[3] -> #4 network:frontend, CreateNetwork, recreate after config change\n" + assert.Equal(t, plan.String(), expected) +} + +func TestReconcileNetworks_DivergedMultipleServices(t *testing.T) { + project := &types.Project{ + Name: "myproject", + Networks: types.Networks{ + "frontend": {Name: "myproject_frontend", Driver: "overlay"}, + }, + Services: types.Services{ + "web": { + Name: "web", + Networks: map[string]*types.ServiceNetworkConfig{"frontend": {}}, + }, + "api": { + Name: "api", + Networks: map[string]*types.ServiceNetworkConfig{"frontend": {}}, + }, + }, + } + observed := &ObservedState{ + ProjectName: "myproject", + Containers: map[string][]ObservedContainer{ + "web": {{ + ID: "c1", Number: 1, State: container.StateRunning, + Summary: container.Summary{ID: "c1", Labels: map[string]string{api.ServiceLabel: "web", api.ContainerNumberLabel: "1"}}, + }}, + "api": {{ + ID: "c2", Number: 1, State: container.StateRunning, + Summary: container.Summary{ID: "c2", Labels: map[string]string{api.ServiceLabel: "api", api.ContainerNumberLabel: "1"}}, + }}, + }, + Networks: map[string]ObservedNetwork{ + "frontend": {ID: "net1", Name: "myproject_frontend", ConfigHash: "oldhash"}, + }, + Volumes: map[string]ObservedVolume{}, + } + + plan, err := reconcile(t.Context(), project, observed, defaultReconcileOptions(), noPrompt) + assert.NilError(t, err) + + s := plan.String() + // Both containers stopped and disconnected before network removal + assert.Assert(t, containsLine(s, "StopContainer, network frontend config changed")) + assert.Assert(t, containsLine(s, "DisconnectNetwork, network frontend recreate")) + assert.Assert(t, containsLine(s, "RemoveNetwork, config hash diverged")) + assert.Assert(t, containsLine(s, "CreateNetwork, recreate after config change")) + // 2 stops + 2 disconnects + 1 remove + 1 create = 6 nodes + assert.Equal(t, len(plan.Nodes), 6) +} + +// --- Volume tests --- + +func TestReconcileVolumes_CreateMissing(t *testing.T) { + project := &types.Project{ + Name: "myproject", + Volumes: types.Volumes{"data": {Name: "myproject_data"}}, + } + observed := &ObservedState{ + ProjectName: "myproject", + Containers: map[string][]ObservedContainer{}, + Networks: map[string]ObservedNetwork{}, + Volumes: map[string]ObservedVolume{}, + } + + plan, err := reconcile(t.Context(), project, observed, defaultReconcileOptions(), noPrompt) + assert.NilError(t, err) + + expected := "[] -> #1 volume:data, CreateVolume, not found\n" + assert.Equal(t, plan.String(), expected) +} + +func TestReconcileVolumes_ExistingMatch(t *testing.T) { + vol := types.VolumeConfig{Name: "myproject_data", Driver: "local"} + hash, err := VolumeHash(vol) + assert.NilError(t, err) + + project := &types.Project{ + Name: "myproject", + Volumes: types.Volumes{"data": vol}, + } + observed := &ObservedState{ + ProjectName: "myproject", + Containers: map[string][]ObservedContainer{}, + Networks: map[string]ObservedNetwork{}, + Volumes: map[string]ObservedVolume{ + "data": {Name: "myproject_data", ConfigHash: hash}, + }, + } + + plan, err := reconcile(t.Context(), project, observed, defaultReconcileOptions(), noPrompt) + assert.NilError(t, err) + assert.Assert(t, plan.IsEmpty()) +} + +func TestReconcileVolumes_ExternalSkipped(t *testing.T) { + project := &types.Project{ + Name: "myproject", + Volumes: types.Volumes{"ext": {Name: "external_vol", External: true}}, + } + observed := &ObservedState{ + ProjectName: "myproject", + Containers: map[string][]ObservedContainer{}, + Networks: map[string]ObservedNetwork{}, + Volumes: map[string]ObservedVolume{}, + } + + plan, err := reconcile(t.Context(), project, observed, defaultReconcileOptions(), noPrompt) + assert.NilError(t, err) + assert.Assert(t, plan.IsEmpty()) +} + +func TestReconcileVolumes_DivergedConfirmed(t *testing.T) { + project := &types.Project{ + Name: "myproject", + Volumes: types.Volumes{"data": {Name: "myproject_data", Driver: "local"}}, + Services: types.Services{ + "db": { + Name: "db", + Volumes: []types.ServiceVolumeConfig{ + {Source: "data", Type: "volume"}, + }, + }, + }, + } + observed := &ObservedState{ + ProjectName: "myproject", + Containers: map[string][]ObservedContainer{ + "db": {{ + ID: "c1", Number: 1, State: container.StateRunning, + Summary: container.Summary{ID: "c1", Labels: map[string]string{api.ServiceLabel: "db", api.ContainerNumberLabel: "1"}}, + }}, + }, + Networks: map[string]ObservedNetwork{}, + Volumes: map[string]ObservedVolume{ + "data": {Name: "myproject_data", ConfigHash: "oldhash"}, + }, + } + + plan, err := reconcile(t.Context(), project, observed, defaultReconcileOptions(), alwaysYesPrompt) + assert.NilError(t, err) + + expected := "[] -> #1 service:db:1, StopContainer, volume data config changed\n" + + "[1] -> #2 service:db:1, RemoveContainer, volume data config changed\n" + + "[2] -> #3 volume:data, RemoveVolume, config hash diverged\n" + + "[3] -> #4 volume:data, CreateVolume, recreate after config change\n" + assert.Equal(t, plan.String(), expected) +} + +func TestReconcileVolumes_DivergedDeclined(t *testing.T) { + project := &types.Project{ + Name: "myproject", + Volumes: types.Volumes{"data": {Name: "myproject_data", Driver: "local"}}, + Services: types.Services{ + "db": { + Name: "db", + Volumes: []types.ServiceVolumeConfig{ + {Source: "data", Type: "volume"}, + }, + }, + }, + } + observed := &ObservedState{ + ProjectName: "myproject", + Containers: map[string][]ObservedContainer{ + "db": {{ + ID: "c1", Number: 1, State: container.StateRunning, + Summary: container.Summary{ID: "c1", Labels: map[string]string{api.ServiceLabel: "db", api.ContainerNumberLabel: "1"}}, + }}, + }, + Networks: map[string]ObservedNetwork{}, + Volumes: map[string]ObservedVolume{ + "data": {Name: "myproject_data", ConfigHash: "oldhash"}, + }, + } + + plan, err := reconcile(t.Context(), project, observed, defaultReconcileOptions(), alwaysNoPrompt) + assert.NilError(t, err) + assert.Assert(t, plan.IsEmpty()) +} + +// --- Helpers --- + +func containsLine(s, substr string) bool { + for _, line := range splitLines(s) { + if containsStr(line, substr) { + return true + } + } + return false +} + +func splitLines(s string) []string { + var lines []string + start := 0 + for i := range len(s) { + if s[i] == '\n' { + lines = append(lines, s[start:i]) + start = i + 1 + } + } + if start < len(s) { + lines = append(lines, s[start:]) + } + return lines +} + +func containsStr(s, substr string) bool { + return len(s) >= len(substr) && searchStr(s, substr) +} + +func searchStr(s, substr string) bool { + for i := 0; i <= len(s)-len(substr); i++ { + if s[i:i+len(substr)] == substr { + return true + } + } + return false +}