Reconciliation : containers

Signed-off-by: Nicolas De Loof <nicolas.deloof@gmail.com>

pkg/compose/reconcile.go

@@ -19,16 +19,24 @@ package compose
 import (
 	"context"
 	"fmt"
+	"slices"
+	"sort"
+	"time"
 
 	"github.com/compose-spec/compose-go/v2/types"
+	"github.com/moby/moby/api/types/container"
+	mmount "github.com/moby/moby/api/types/mount"
+
+	"github.com/docker/compose/v5/pkg/api"
 )
 
 // ReconcileOptions controls how the reconciler compares desired and observed state.
 type ReconcileOptions struct {
-	Services             []string // targeted services (empty = all)
-	Recreate             string   // "diverged", "force", "never" for targeted services
-	RecreateDependencies string   // same for non-targeted services
-	Inherit              bool     // inherit anonymous volumes on recreate
+	Services             []string       // targeted services (empty = all)
+	Recreate             string         // "diverged", "force", "never" for targeted services
+	RecreateDependencies string         // same for non-targeted services
+	Inherit              bool           // inherit anonymous volumes on recreate
+	Timeout              *time.Duration // for stop operations
 	RemoveOrphans        bool
 	SkipProviders        bool
 }
@@ -41,17 +49,28 @@ type reconciler struct {
 	options  ReconcileOptions
 	prompt   Prompt
 	plan     *Plan
+
+	// networkNodes and volumeNodes track the last plan node for each
+	// network/volume, so container creation nodes can depend on them.
+	networkNodes map[string]*PlanNode // compose network key → create node
+	volumeNodes  map[string]*PlanNode // compose volume key → create node
+	// serviceNodes tracks the last plan node per service, so dependent
+	// services can order their operations after dependencies.
+	serviceNodes map[string]*PlanNode
 }
 
 // reconcile is the main entry point: it builds a Plan from desired vs observed state.
 // The prompt function is called for interactive decisions (e.g. volume divergence).
 func reconcile(_ context.Context, project *types.Project, observed *ObservedState, options ReconcileOptions, prompt Prompt) (*Plan, error) {
 	r := &reconciler{
-		project:  project,
-		observed: observed,
-		options:  options,
-		prompt:   prompt,
-		plan:     &Plan{},
+		project:      project,
+		observed:     observed,
+		options:      options,
+		prompt:       prompt,
+		plan:         &Plan{},
+		networkNodes: map[string]*PlanNode{},
+		volumeNodes:  map[string]*PlanNode{},
+		serviceNodes: map[string]*PlanNode{},
 	}
 
 	if err := r.reconcileNetworks(); err != nil {
@@ -62,6 +81,14 @@ func reconcile(_ context.Context, project *types.Project, observed *ObservedStat
 		return nil, err
 	}
 
+	if err := r.reconcileContainers(); err != nil {
+		return nil, err
+	}
+
+	if r.options.RemoveOrphans {
+		r.reconcileOrphans()
+	}
+
 	return r.plan, nil
 }
 
@@ -91,15 +118,17 @@ func (r *reconciler) reconcileNetworks() error {
 	return nil
 }
 
-// planCreateNetwork adds a single CreateNetwork node.
+// planCreateNetwork adds a single CreateNetwork node and records it for dependency tracking.
 func (r *reconciler) planCreateNetwork(key string, nw *types.NetworkConfig) *PlanNode {
-	return r.plan.addNode(Operation{
+	node := r.plan.addNode(Operation{
 		Type:       OpCreateNetwork,
 		ResourceID: fmt.Sprintf("network:%s", key),
 		Cause:      "not found",
 		Name:       nw.Name,
 		Network:    nw,
 	}, "")
+	r.networkNodes[key] = node
+	return node
 }
 
 // planRecreateNetwork adds the full sequence for a diverged network:
@@ -144,13 +173,14 @@ func (r *reconciler) planRecreateNetwork(key string, nw *types.NetworkConfig) er
 	}, "", disconnectNodes...)
 
 	// Create network (depends on remove)
-	r.plan.addNode(Operation{
+	createNode := r.plan.addNode(Operation{
 		Type:       OpCreateNetwork,
 		ResourceID: fmt.Sprintf("network:%s", key),
 		Cause:      "recreate after config change",
 		Name:       nw.Name,
 		Network:    nw,
 	}, "", removeNode)
+	r.networkNodes[key] = createNode
 
 	return nil
 }
@@ -188,15 +218,17 @@ func (r *reconciler) reconcileVolumes() error {
 	return nil
 }
 
-// planCreateVolume adds a single CreateVolume node.
+// planCreateVolume adds a single CreateVolume node and records it for dependency tracking.
 func (r *reconciler) planCreateVolume(key string, vol *types.VolumeConfig) *PlanNode {
-	return r.plan.addNode(Operation{
+	node := r.plan.addNode(Operation{
 		Type:       OpCreateVolume,
 		ResourceID: fmt.Sprintf("volume:%s", key),
 		Cause:      "not found",
 		Name:       vol.Name,
 		Volume:     vol,
 	}, "")
+	r.volumeNodes[key] = node
+	return node
 }
 
 // planRecreateVolume adds the full sequence for a diverged volume:
@@ -242,13 +274,14 @@ func (r *reconciler) planRecreateVolume(key string, vol *types.VolumeConfig) {
 	}, "", removeNodes...)
 
 	// Create volume (depends on remove)
-	r.plan.addNode(Operation{
+	createNode := r.plan.addNode(Operation{
 		Type:       OpCreateVolume,
 		ResourceID: fmt.Sprintf("volume:%s", key),
 		Cause:      "recreate after config change",
 		Name:       vol.Name,
 		Volume:     vol,
 	}, "", removeVolNode)
+	r.volumeNodes[key] = createNode
 }
 
 // servicesUsingNetwork returns the names of services that reference the given
@@ -288,5 +321,388 @@ func (r *reconciler) containersForServices(services []string) []ObservedContaine
 	return result
 }
 
+// reconcileContainers processes each service in dependency order, comparing
+// the desired scale and configuration with observed containers.
+func (r *reconciler) reconcileContainers() error {
+	// Build dependency graph and process in order
+	graph, err := NewGraph(r.project, ServiceStopped)
+	if err != nil {
+		return err
+	}
+
+	// Visit in dependency order (leaves first = services with no deps)
+	return r.visitInDependencyOrder(graph)
+}
+
+// visitInDependencyOrder processes services from leaves to roots so that
+// dependencies are reconciled before the services that depend on them.
+func (r *reconciler) visitInDependencyOrder(g *Graph) error {
+	visited := map[string]bool{}
+	for {
+		// Find a vertex whose all children are visited
+		var next *Vertex
+		for _, v := range g.Vertices {
+			if visited[v.Key] {
+				continue
+			}
+			allChildrenVisited := true
+			for _, child := range v.Children {
+				if !visited[child.Key] {
+					allChildrenVisited = false
+					break
+				}
+			}
+			if allChildrenVisited {
+				next = v
+				break
+			}
+		}
+		if next == nil {
+			break // all visited
+		}
+		visited[next.Key] = true
+
+		service, err := r.project.GetService(next.Service)
+		if err != nil {
+			return err
+		}
+		if err := r.reconcileService(service); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+// reconcileService handles a single service: scale down, recreate diverged,
+// start stopped, scale up.
+func (r *reconciler) reconcileService(service types.ServiceConfig) error {
+	if service.Provider != nil && r.options.SkipProviders {
+		return nil
+	}
+	if service.Provider != nil {
+		// Provider services are handled by plugins, not by the reconciler
+		return nil
+	}
+
+	expected, err := getScale(service)
+	if err != nil {
+		return err
+	}
+
+	containers := r.observed.Containers[service.Name]
+	actual := len(containers)
+
+	strategy := r.options.RecreateDependencies
+	if slices.Contains(r.options.Services, service.Name) || len(r.options.Services) == 0 {
+		strategy = r.options.Recreate
+	}
+
+	// Sort containers: obsolete first, then by number descending, then reverse
+	// to get the same ordering as the existing convergence code.
+	r.sortContainers(containers, service, strategy)
+
+	// Collect dependency nodes that container creation should depend on
+	infraDeps := r.infrastructureDeps(service)
+
+	var lastNode *PlanNode
+
+	// Process existing containers
+	for i, oc := range containers {
+		if i >= expected {
+			// Scale down: stop + remove excess containers
+			stopNode := r.plan.addNode(Operation{
+				Type:       OpStopContainer,
+				ResourceID: fmt.Sprintf("service:%s:%d", service.Name, oc.Number),
+				Cause:      "scale down",
+				Container:  &containers[i].Summary,
+				Timeout:    r.options.Timeout,
+			}, "")
+			r.plan.addNode(Operation{
+				Type:       OpRemoveContainer,
+				ResourceID: fmt.Sprintf("service:%s:%d", service.Name, oc.Number),
+				Cause:      "scale down",
+				Container:  &containers[i].Summary,
+			}, "", stopNode)
+			continue
+		}
+
+		recreate, err := r.mustRecreate(service, oc, strategy)
+		if err != nil {
+			return err
+		}
+		if recreate {
+			lastNode = r.planRecreateContainer(service, &containers[i], infraDeps)
+			continue
+		}
+
+		// Container is up-to-date
+		switch oc.State {
+		case container.StateRunning, container.StateCreated, container.StateRestarting, container.StateExited:
+			// Nothing to do (exited containers are left as-is, matching convergence.go behavior)
+		default:
+			// Stopped/exited container that needs starting
+			lastNode = r.plan.addNode(Operation{
+				Type:       OpStartContainer,
+				ResourceID: fmt.Sprintf("service:%s:%d", service.Name, oc.Number),
+				Cause:      "not running",
+				Container:  &containers[i].Summary,
+			}, "", infraDeps...)
+		}
+	}
+
+	// Scale up: create new containers
+	nextNum := nextContainerNumber(r.observedSummaries(service.Name))
+	for i := 0; i < expected-actual; i++ {
+		number := nextNum + i
+		name := getContainerName(r.project.Name, service, number)
+		svc := service // copy for pointer stability
+		lastNode = r.plan.addNode(Operation{
+			Type:       OpCreateContainer,
+			ResourceID: fmt.Sprintf("service:%s:%d", service.Name, number),
+			Cause:      "no existing container",
+			Service:    &svc,
+			Number:     number,
+			Name:       name,
+		}, "", infraDeps...)
+	}
+
+	r.serviceNodes[service.Name] = lastNode
+	return nil
+}
+
+// mustRecreate mirrors the existing convergence.mustRecreate logic.
+func (r *reconciler) mustRecreate(expected types.ServiceConfig, oc ObservedContainer, policy string) (bool, error) {
+	if policy == api.RecreateNever {
+		return false, nil
+	}
+	if policy == api.RecreateForce {
+		return true, nil
+	}
+	configHash, err := ServiceHash(expected)
+	if err != nil {
+		return false, err
+	}
+	if oc.ConfigHash != configHash {
+		return true, nil
+	}
+	if oc.ImageDigest != expected.CustomLabels[api.ImageDigestLabel] {
+		return true, nil
+	}
+
+	if oc.State == container.StateRunning && r.hasNetworkMismatch(expected, oc) {
+		return true, nil
+	}
+	if r.hasVolumeMismatch(expected, oc) {
+		return true, nil
+	}
+
+	return false, nil
+}
+
+// hasNetworkMismatch checks if the container is not connected to all expected networks.
+func (r *reconciler) hasNetworkMismatch(expected types.ServiceConfig, oc ObservedContainer) bool {
+	for net := range expected.Networks {
+		expectedID := ""
+		if obs, ok := r.observed.Networks[net]; ok {
+			expectedID = obs.ID
+		}
+		if expectedID == "" || expectedID == "swarm" {
+			continue
+		}
+		found := false
+		for _, netID := range oc.ConnectedNetworks {
+			if netID == expectedID {
+				found = true
+				break
+			}
+		}
+		if !found {
+			return true
+		}
+	}
+	return false
+}
+
+// hasVolumeMismatch checks if the container is missing any expected volume mounts.
+func (r *reconciler) hasVolumeMismatch(expected types.ServiceConfig, oc ObservedContainer) bool {
+	for _, vol := range expected.Volumes {
+		if vol.Type != string(mmount.TypeVolume) || vol.Source == "" {
+			continue
+		}
+		expectedName := ""
+		if obs, ok := r.observed.Volumes[vol.Source]; ok {
+			expectedName = obs.Name
+		}
+		if expectedName == "" {
+			continue
+		}
+		found := false
+		for _, m := range oc.Summary.Mounts {
+			if m.Type == mmount.TypeVolume && m.Name == expectedName {
+				found = true
+				break
+			}
+		}
+		if !found {
+			return true
+		}
+	}
+	return false
+}
+
+// planRecreateContainer decomposes container recreation into 4 atomic operations:
+// CreateContainer(tmpName) → StopContainer → RemoveContainer → RenameContainer
+func (r *reconciler) planRecreateContainer(service types.ServiceConfig, oc *ObservedContainer, infraDeps []*PlanNode) *PlanNode {
+	resID := fmt.Sprintf("service:%s:%d", service.Name, oc.Number)
+	group := fmt.Sprintf("recreate:%s:%d", service.Name, oc.Number)
+	tmpName := fmt.Sprintf("%s_%s", oc.ID[:min(12, len(oc.ID))], getContainerName(r.project.Name, service, oc.Number))
+	svc := service // copy for pointer stability
+
+	// Stop dependents first
+	depStopNodes := r.planStopDependents(service)
+
+	// All deps: infrastructure + dependent stops
+	allDeps := append(slices.Clone(infraDeps), depStopNodes...)
+
+	var inherited *container.Summary
+	if r.options.Inherit {
+		inherited = &oc.Summary
+	}
+
+	// 1. Create new container with temporary name
+	createNode := r.plan.addNode(Operation{
+		Type:       OpCreateContainer,
+		ResourceID: resID,
+		Cause:      "config changed (tmpName)",
+		Service:    &svc,
+		Inherited:  inherited,
+		Number:     oc.Number,
+		Name:       tmpName,
+	}, group, allDeps...)
+
+	// 2. Stop old container
+	stopNode := r.plan.addNode(Operation{
+		Type:       OpStopContainer,
+		ResourceID: resID,
+		Cause:      fmt.Sprintf("replaced by #%d", createNode.ID),
+		Container:  &oc.Summary,
+		Timeout:    r.options.Timeout,
+	}, group, createNode)
+
+	// 3. Remove old container
+	removeNode := r.plan.addNode(Operation{
+		Type:       OpRemoveContainer,
+		ResourceID: resID,
+		Cause:      fmt.Sprintf("replaced by #%d", createNode.ID),
+		Container:  &oc.Summary,
+	}, group, stopNode)
+
+	// 4. Rename to final name
+	finalName := getContainerName(r.project.Name, service, oc.Number)
+	renameNode := r.plan.addNode(Operation{
+		Type:       OpRenameContainer,
+		ResourceID: resID,
+		Cause:      "finalize recreate",
+		Name:       finalName,
+	}, group, removeNode)
+
+	return renameNode
+}
+
+// planStopDependents plans stop operations for containers of services that
+// depend on the given service with restart: true.
+func (r *reconciler) planStopDependents(service types.ServiceConfig) []*PlanNode {
+	dependents := r.project.GetDependentsForService(service, func(dep types.ServiceDependency) bool {
+		return dep.Restart
+	})
+	var nodes []*PlanNode
+	for _, depName := range dependents {
+		for i, oc := range r.observed.Containers[depName] {
+			node := r.plan.addNode(Operation{
+				Type:       OpStopContainer,
+				ResourceID: fmt.Sprintf("service:%s:%d", depName, oc.Number),
+				Cause:      fmt.Sprintf("dependency %s being recreated", service.Name),
+				Container:  &r.observed.Containers[depName][i].Summary,
+				Timeout:    r.options.Timeout,
+			}, "")
+			nodes = append(nodes, node)
+		}
+	}
+	return nodes
+}
+
+// infrastructureDeps returns the plan nodes that a container creation for this
+// service should depend on: network creates and volume creates that the service
+// references, plus the last node of dependency services.
+func (r *reconciler) infrastructureDeps(service types.ServiceConfig) []*PlanNode {
+	var deps []*PlanNode
+	for net := range service.Networks {
+		if node, ok := r.networkNodes[net]; ok {
+			deps = append(deps, node)
+		}
+	}
+	for _, vol := range service.Volumes {
+		if vol.Type == string(mmount.TypeVolume) && vol.Source != "" {
+			if node, ok := r.volumeNodes[vol.Source]; ok {
+				deps = append(deps, node)
+			}
+		}
+	}
+	for depName := range service.DependsOn {
+		if node, ok := r.serviceNodes[depName]; ok {
+			deps = append(deps, node)
+		}
+	}
+	return deps
+}
+
+// sortContainers sorts containers the same way as convergence.go:138-160:
+// obsolete first, then by container number descending, then reversed.
+func (r *reconciler) sortContainers(containers []ObservedContainer, service types.ServiceConfig, policy string) {
+	sort.Slice(containers, func(i, j int) bool {
+		obsi, _ := r.mustRecreate(service, containers[i], policy)
+		obsj, _ := r.mustRecreate(service, containers[j], policy)
+		if obsi != obsj {
+			return obsi // obsolete first
+		}
+		// preserve low container numbers
+		if containers[i].Number != containers[j].Number {
+			return containers[i].Number > containers[j].Number
+		}
+		return containers[i].Summary.Created < containers[j].Summary.Created
+	})
+	slices.Reverse(containers)
+}
+
+// reconcileOrphans plans stop + remove for orphaned containers.
+func (r *reconciler) reconcileOrphans() {
+	for i, oc := range r.observed.Orphans {
+		stopNode := r.plan.addNode(Operation{
+			Type:       OpStopContainer,
+			ResourceID: fmt.Sprintf("orphan:%s", oc.Name),
+			Cause:      "orphaned container",
+			Container:  &r.observed.Orphans[i].Summary,
+			Timeout:    r.options.Timeout,
+		}, "")
+		r.plan.addNode(Operation{
+			Type:       OpRemoveContainer,
+			ResourceID: fmt.Sprintf("orphan:%s", oc.Name),
+			Cause:      "orphaned container",
+			Container:  &r.observed.Orphans[i].Summary,
+		}, "", stopNode)
+	}
+}
+
+// observedSummaries returns the raw container.Summary list for a service,
+// needed by nextContainerNumber which expects []container.Summary.
+func (r *reconciler) observedSummaries(serviceName string) []container.Summary {
+	ocs := r.observed.Containers[serviceName]
+	result := make([]container.Summary, len(ocs))
+	for i, oc := range ocs {
+		result[i] = oc.Summary
+	}
+	return result
+}
+
 // serviceLabel is a package-level shorthand for the service label key.
 const serviceLabel = "com.docker.compose.service"

pkg/compose/reconcile_test.go

@@ -118,6 +118,7 @@ func TestReconcileNetworks_Diverged(t *testing.T) {
 		Services: types.Services{
 			"web": {
 				Name:     "web",
+				Scale:    intPtr(1),
 				Networks: map[string]*types.ServiceNetworkConfig{"frontend": {}},
 			},
 		},
@@ -147,11 +148,14 @@ func TestReconcileNetworks_Diverged(t *testing.T) {
 	plan, err := reconcile(t.Context(), project, observed, defaultReconcileOptions(), noPrompt)
 	assert.NilError(t, err)
 
-	expected := "[] -> #1 service:web:1, StopContainer, network frontend config changed\n" +
-		"[1] -> #2 service:web:1, DisconnectNetwork, network frontend recreate\n" +
-		"[2] -> #3 network:frontend, RemoveNetwork, config hash diverged\n" +
-		"[3] -> #4 network:frontend, CreateNetwork, recreate after config change\n"
-	assert.Equal(t, plan.String(), expected)
+	s := plan.String()
+	// Network recreation sequence
+	assert.Assert(t, containsLine(s, "StopContainer, network frontend config changed"))
+	assert.Assert(t, containsLine(s, "DisconnectNetwork, network frontend recreate"))
+	assert.Assert(t, containsLine(s, "RemoveNetwork, config hash diverged"))
+	assert.Assert(t, containsLine(s, "CreateNetwork, recreate after config change"))
+	// Container recreation follows (the container was connected to the old network)
+	assert.Assert(t, containsLine(s, "recreate:web:1"))
 }
 
 func TestReconcileNetworks_DivergedMultipleServices(t *testing.T) {
@@ -163,10 +167,12 @@ func TestReconcileNetworks_DivergedMultipleServices(t *testing.T) {
 		Services: types.Services{
 			"web": {
 				Name:     "web",
+				Scale:    intPtr(1),
 				Networks: map[string]*types.ServiceNetworkConfig{"frontend": {}},
 			},
 			"api": {
 				Name:     "api",
+				Scale:    intPtr(1),
 				Networks: map[string]*types.ServiceNetworkConfig{"frontend": {}},
 			},
 		},
@@ -198,8 +204,9 @@ func TestReconcileNetworks_DivergedMultipleServices(t *testing.T) {
 	assert.Assert(t, containsLine(s, "DisconnectNetwork, network frontend recreate"))
 	assert.Assert(t, containsLine(s, "RemoveNetwork, config hash diverged"))
 	assert.Assert(t, containsLine(s, "CreateNetwork, recreate after config change"))
-	// 2 stops + 2 disconnects + 1 remove + 1 create = 6 nodes
-	assert.Equal(t, len(plan.Nodes), 6)
+	// Both containers get recreated after network
+	assert.Assert(t, containsLine(s, "recreate:web:1"))
+	assert.Assert(t, containsLine(s, "recreate:api:1"))
 }
 
 // --- Volume tests ---
@@ -269,7 +276,8 @@ func TestReconcileVolumes_DivergedConfirmed(t *testing.T) {
 		Volumes: types.Volumes{"data": {Name: "myproject_data", Driver: "local"}},
 		Services: types.Services{
 			"db": {
-				Name: "db",
+				Name:  "db",
+				Scale: intPtr(1),
 				Volumes: []types.ServiceVolumeConfig{
 					{Source: "data", Type: "volume"},
 				},
@@ -293,20 +301,28 @@ func TestReconcileVolumes_DivergedConfirmed(t *testing.T) {
 	plan, err := reconcile(t.Context(), project, observed, defaultReconcileOptions(), alwaysYesPrompt)
 	assert.NilError(t, err)
 
-	expected := "[] -> #1 service:db:1, StopContainer, volume data config changed\n" +
-		"[1] -> #2 service:db:1, RemoveContainer, volume data config changed\n" +
-		"[2] -> #3 volume:data, RemoveVolume, config hash diverged\n" +
-		"[3] -> #4 volume:data, CreateVolume, recreate after config change\n"
-	assert.Equal(t, plan.String(), expected)
+	s := plan.String()
+	// Volume recreation sequence
+	assert.Assert(t, containsLine(s, "StopContainer, volume data config changed"))
+	assert.Assert(t, containsLine(s, "RemoveContainer, volume data config changed"))
+	assert.Assert(t, containsLine(s, "RemoveVolume, config hash diverged"))
+	assert.Assert(t, containsLine(s, "CreateVolume, recreate after config change"))
+	// Container is recreated since its config hash diverges after volume removal
+	assert.Assert(t, containsLine(s, "CreateContainer,"))
 }
 
 func TestReconcileVolumes_DivergedDeclined(t *testing.T) {
+	vol := types.VolumeConfig{Name: "myproject_data", Driver: "local"}
+	hash, err := VolumeHash(vol)
+	assert.NilError(t, err)
+
 	project := &types.Project{
 		Name:    "myproject",
 		Volumes: types.Volumes{"data": {Name: "myproject_data", Driver: "local"}},
 		Services: types.Services{
 			"db": {
-				Name: "db",
+				Name:  "db",
+				Scale: intPtr(1),
 				Volumes: []types.ServiceVolumeConfig{
 					{Source: "data", Type: "volume"},
 				},
@@ -318,22 +334,328 @@ func TestReconcileVolumes_DivergedDeclined(t *testing.T) {
 		Containers: map[string][]ObservedContainer{
 			"db": {{
 				ID: "c1", Number: 1, State: container.StateRunning,
-				Summary: container.Summary{ID: "c1", Labels: map[string]string{api.ServiceLabel: "db", api.ContainerNumberLabel: "1"}},
+				ConfigHash: mustServiceHash(t, project.Services["db"]),
+				Summary: container.Summary{
+					ID:    "c1",
+					State: container.StateRunning,
+					Labels: map[string]string{
+						api.ServiceLabel:         "db",
+						api.ContainerNumberLabel: "1",
+						api.ConfigHashLabel:      mustServiceHash(t, project.Services["db"]),
+					},
+					Mounts: []container.MountPoint{{Type: "volume", Name: vol.Name}},
+				},
 			}},
 		},
 		Networks: map[string]ObservedNetwork{},
 		Volumes: map[string]ObservedVolume{
-			"data": {Name: "myproject_data", ConfigHash: "oldhash"},
+			// Volume hash doesn't match, but user declines recreation
+			"data": {Name: vol.Name, ConfigHash: hash + "old"},
 		},
 	}
 
 	plan, err := reconcile(t.Context(), project, observed, defaultReconcileOptions(), alwaysNoPrompt)
 	assert.NilError(t, err)
+	// Volume not recreated, container is up-to-date -> empty plan
 	assert.Assert(t, plan.IsEmpty())
 }
 
+// --- Container tests ---
+
+func TestReconcileContainers_NewProject(t *testing.T) {
+	project := &types.Project{
+		Name: "myproject",
+		Services: types.Services{
+			"web": {Name: "web", Scale: intPtr(1)},
+		},
+	}
+	observed := &ObservedState{
+		ProjectName: "myproject",
+		Containers:  map[string][]ObservedContainer{"web": {}},
+		Networks:    map[string]ObservedNetwork{},
+		Volumes:     map[string]ObservedVolume{},
+	}
+
+	plan, err := reconcile(t.Context(), project, observed, defaultReconcileOptions(), noPrompt)
+	assert.NilError(t, err)
+
+	expected := "[] -> #1 service:web:1, CreateContainer, no existing container\n"
+	assert.Equal(t, plan.String(), expected)
+}
+
+func TestReconcileContainers_AlreadyRunning(t *testing.T) {
+	svc := types.ServiceConfig{Name: "web", Scale: intPtr(1)}
+	hash := mustServiceHash(t, svc)
+
+	project := &types.Project{
+		Name:     "myproject",
+		Services: types.Services{"web": svc},
+	}
+	observed := &ObservedState{
+		ProjectName: "myproject",
+		Containers: map[string][]ObservedContainer{
+			"web": {{
+				ID: "c1", Number: 1, State: container.StateRunning, ConfigHash: hash,
+				Summary: container.Summary{
+					ID: "c1", State: container.StateRunning,
+					Labels: map[string]string{api.ServiceLabel: "web", api.ContainerNumberLabel: "1", api.ConfigHashLabel: hash},
+				},
+			}},
+		},
+		Networks: map[string]ObservedNetwork{},
+		Volumes:  map[string]ObservedVolume{},
+	}
+
+	plan, err := reconcile(t.Context(), project, observed, defaultReconcileOptions(), noPrompt)
+	assert.NilError(t, err)
+	assert.Assert(t, plan.IsEmpty())
+}
+
+func TestReconcileContainers_ConfigChanged(t *testing.T) {
+	project := &types.Project{
+		Name: "myproject",
+		Services: types.Services{
+			"web": {Name: "web", Scale: intPtr(1)},
+		},
+	}
+	observed := &ObservedState{
+		ProjectName: "myproject",
+		Containers: map[string][]ObservedContainer{
+			"web": {{
+				ID: "c1aabbccddee", Number: 1, State: container.StateRunning, ConfigHash: "oldhash",
+				Summary: container.Summary{
+					ID: "c1aabbccddee", State: container.StateRunning,
+					Labels: map[string]string{api.ServiceLabel: "web", api.ContainerNumberLabel: "1", api.ConfigHashLabel: "oldhash"},
+				},
+			}},
+		},
+		Networks: map[string]ObservedNetwork{},
+		Volumes:  map[string]ObservedVolume{},
+	}
+
+	plan, err := reconcile(t.Context(), project, observed, defaultReconcileOptions(), noPrompt)
+	assert.NilError(t, err)
+
+	s := plan.String()
+	assert.Assert(t, containsLine(s, "CreateContainer, config changed (tmpName)"))
+	assert.Assert(t, containsLine(s, "StopContainer, replaced by"))
+	assert.Assert(t, containsLine(s, "RemoveContainer, replaced by"))
+	assert.Assert(t, containsLine(s, "RenameContainer, finalize recreate"))
+	// All 4 nodes share the same group
+	assert.Assert(t, containsLine(s, "[recreate:web:1]"))
+}
+
+func TestReconcileContainers_ScaleUp(t *testing.T) {
+	svc := types.ServiceConfig{Name: "web", Scale: intPtr(3)}
+	hash := mustServiceHash(t, svc)
+
+	project := &types.Project{
+		Name:     "myproject",
+		Services: types.Services{"web": svc},
+	}
+	observed := &ObservedState{
+		ProjectName: "myproject",
+		Containers: map[string][]ObservedContainer{
+			"web": {{
+				ID: "c1", Number: 1, State: container.StateRunning, ConfigHash: hash,
+				Summary: container.Summary{
+					ID: "c1", State: container.StateRunning,
+					Labels: map[string]string{api.ServiceLabel: "web", api.ContainerNumberLabel: "1", api.ConfigHashLabel: hash},
+				},
+			}},
+		},
+		Networks: map[string]ObservedNetwork{},
+		Volumes:  map[string]ObservedVolume{},
+	}
+
+	plan, err := reconcile(t.Context(), project, observed, defaultReconcileOptions(), noPrompt)
+	assert.NilError(t, err)
+
+	s := plan.String()
+	assert.Assert(t, containsLine(s, "service:web:2, CreateContainer, no existing container"))
+	assert.Assert(t, containsLine(s, "service:web:3, CreateContainer, no existing container"))
+	assert.Equal(t, len(plan.Nodes), 2)
+}
+
+func TestReconcileContainers_ScaleDown(t *testing.T) {
+	svc := types.ServiceConfig{Name: "web", Scale: intPtr(1)}
+	hash := mustServiceHash(t, svc)
+
+	project := &types.Project{
+		Name:     "myproject",
+		Services: types.Services{"web": svc},
+	}
+	observed := &ObservedState{
+		ProjectName: "myproject",
+		Containers: map[string][]ObservedContainer{
+			"web": {
+				{
+					ID: "c1", Number: 1, State: container.StateRunning, ConfigHash: hash,
+					Summary: container.Summary{
+						ID: "c1", State: container.StateRunning,
+						Labels: map[string]string{api.ServiceLabel: "web", api.ContainerNumberLabel: "1", api.ConfigHashLabel: hash},
+					},
+				},
+				{
+					ID: "c2", Number: 2, State: container.StateRunning, ConfigHash: hash,
+					Summary: container.Summary{
+						ID: "c2", State: container.StateRunning,
+						Labels: map[string]string{api.ServiceLabel: "web", api.ContainerNumberLabel: "2", api.ConfigHashLabel: hash},
+					},
+				},
+			},
+		},
+		Networks: map[string]ObservedNetwork{},
+		Volumes:  map[string]ObservedVolume{},
+	}
+
+	plan, err := reconcile(t.Context(), project, observed, defaultReconcileOptions(), noPrompt)
+	assert.NilError(t, err)
+
+	s := plan.String()
+	assert.Assert(t, containsLine(s, "StopContainer, scale down"))
+	assert.Assert(t, containsLine(s, "RemoveContainer, scale down"))
+	assert.Equal(t, len(plan.Nodes), 2)
+}
+
+func TestReconcileContainers_ForceRecreate(t *testing.T) {
+	svc := types.ServiceConfig{Name: "web", Scale: intPtr(1)}
+	hash := mustServiceHash(t, svc)
+
+	project := &types.Project{
+		Name:     "myproject",
+		Services: types.Services{"web": svc},
+	}
+	observed := &ObservedState{
+		ProjectName: "myproject",
+		Containers: map[string][]ObservedContainer{
+			"web": {{
+				ID: "c1aabbccddee", Number: 1, State: container.StateRunning, ConfigHash: hash,
+				Summary: container.Summary{
+					ID: "c1aabbccddee", State: container.StateRunning,
+					Labels: map[string]string{api.ServiceLabel: "web", api.ContainerNumberLabel: "1", api.ConfigHashLabel: hash},
+				},
+			}},
+		},
+		Networks: map[string]ObservedNetwork{},
+		Volumes:  map[string]ObservedVolume{},
+	}
+
+	opts := defaultReconcileOptions()
+	opts.Recreate = api.RecreateForce
+
+	plan, err := reconcile(t.Context(), project, observed, opts, noPrompt)
+	assert.NilError(t, err)
+
+	s := plan.String()
+	assert.Assert(t, containsLine(s, "recreate:web:1"))
+	assert.Equal(t, len(plan.Nodes), 4) // create + stop + remove + rename
+}
+
+func TestReconcileContainers_NeverRecreate(t *testing.T) {
+	project := &types.Project{
+		Name: "myproject",
+		Services: types.Services{
+			"web": {Name: "web", Scale: intPtr(1)},
+		},
+	}
+	observed := &ObservedState{
+		ProjectName: "myproject",
+		Containers: map[string][]ObservedContainer{
+			"web": {{
+				ID: "c1", Number: 1, State: container.StateRunning, ConfigHash: "oldhash",
+				Summary: container.Summary{
+					ID: "c1", State: container.StateRunning,
+					Labels: map[string]string{api.ServiceLabel: "web", api.ContainerNumberLabel: "1", api.ConfigHashLabel: "oldhash"},
+				},
+			}},
+		},
+		Networks: map[string]ObservedNetwork{},
+		Volumes:  map[string]ObservedVolume{},
+	}
+
+	opts := defaultReconcileOptions()
+	opts.Recreate = api.RecreateNever
+
+	plan, err := reconcile(t.Context(), project, observed, opts, noPrompt)
+	assert.NilError(t, err)
+	assert.Assert(t, plan.IsEmpty())
+}
+
+func TestReconcileContainers_StoppedNeedsStart(t *testing.T) {
+	svc := types.ServiceConfig{Name: "web", Scale: intPtr(1)}
+	hash := mustServiceHash(t, svc)
+
+	project := &types.Project{
+		Name:     "myproject",
+		Services: types.Services{"web": svc},
+	}
+	observed := &ObservedState{
+		ProjectName: "myproject",
+		Containers: map[string][]ObservedContainer{
+			"web": {{
+				ID: "c1", Number: 1, State: container.StateExited, ConfigHash: hash,
+				Summary: container.Summary{
+					ID: "c1", State: container.StateExited,
+					Labels: map[string]string{api.ServiceLabel: "web", api.ContainerNumberLabel: "1", api.ConfigHashLabel: hash},
+				},
+			}},
+		},
+		Networks: map[string]ObservedNetwork{},
+		Volumes:  map[string]ObservedVolume{},
+	}
+
+	plan, err := reconcile(t.Context(), project, observed, defaultReconcileOptions(), noPrompt)
+	assert.NilError(t, err)
+	// exited state is not "running", "created", or "restarting" → gets a StartContainer
+	// Actually per the code, StateExited is handled explicitly as no-op in current convergence
+	// Let me check: the switch has case StateExited with no action. So it should be empty.
+	// Wait, looking at the reconciler code: the default case starts it. But StateExited is not
+	// explicitly handled — it falls through to default. Let me re-read...
+	// The switch is: Running, Created, Restarting → noop. Exited → is not listed, falls to default → start.
+	// BUT in the original convergence.go:199, StateExited is a noop. Let me fix.
+	// Actually wait: convergence.go:199 shows "case container.StateExited:" with NO body, so it's a noop.
+	// My reconciler should match. Let me verify this is tested correctly.
+	assert.Assert(t, plan.IsEmpty())
+}
+
+func TestReconcileOrphans(t *testing.T) {
+	project := &types.Project{
+		Name:     "myproject",
+		Services: types.Services{},
+	}
+	observed := &ObservedState{
+		ProjectName: "myproject",
+		Containers:  map[string][]ObservedContainer{},
+		Orphans: []ObservedContainer{{
+			ID: "orphan1", Number: 1, Name: "myproject-old-1",
+			Summary: container.Summary{ID: "orphan1"},
+		}},
+		Networks: map[string]ObservedNetwork{},
+		Volumes:  map[string]ObservedVolume{},
+	}
+
+	opts := defaultReconcileOptions()
+	opts.RemoveOrphans = true
+
+	plan, err := reconcile(t.Context(), project, observed, opts, noPrompt)
+	assert.NilError(t, err)
+
+	s := plan.String()
+	assert.Assert(t, containsLine(s, "StopContainer, orphaned container"))
+	assert.Assert(t, containsLine(s, "RemoveContainer, orphaned container"))
+	assert.Equal(t, len(plan.Nodes), 2)
+}
+
 // --- Helpers ---
 
+func mustServiceHash(t *testing.T, svc types.ServiceConfig) string {
+	t.Helper()
+	h, err := ServiceHash(svc)
+	assert.NilError(t, err)
+	return h
+}
+
 func containsLine(s, substr string) bool {
 	for _, line := range splitLines(s) {
 		if containsStr(line, substr) {