diff --git a/internal/images/images.go b/internal/images/images.go new file mode 100644 index 000000000..38bb8ae13 --- /dev/null +++ b/internal/images/images.go @@ -0,0 +1,389 @@ +/* + Copyright 2020 Docker Compose CLI authors + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +// Package images implements the operations on the images of a project: listing +// the images its containers run, working out the identity of a local image, and +// removing the images `down` and `watch` leave behind. +package images + +import ( + "context" + "fmt" + "strings" + "sync" + "time" + + "github.com/containerd/errdefs" + "github.com/containerd/platforms" + "github.com/distribution/reference" + "github.com/moby/moby/api/types/container" + "github.com/moby/moby/api/types/image" + "github.com/moby/moby/client" + "github.com/moby/moby/client/pkg/versions" + godigest "github.com/opencontainers/go-digest" + specs "github.com/opencontainers/image-spec/specs-go/v1" + "github.com/sirupsen/logrus" + + "github.com/docker/compose/v5/internal/concurrency" + "github.com/docker/compose/v5/internal/naming" + "github.com/docker/compose/v5/pkg/api" +) + +// Docker Engine API versions the operations of this package gate on. +const ( + // apiVersion148 is Docker Engine API 1.48 (Engine v28.0): image inspect can + // return per-manifest data. + apiVersion148 = "1.48" + // apiVersion149 is Docker Engine API 1.49 (Engine v28.1): image inspect + // accepts a platform. + apiVersion149 = "1.49" +) + +// Backend is what the operations need from the engine they act on. +type Backend interface { + // APIClient talks to the Docker daemon. + APIClient() client.APIClient + // MaxConcurrency bounds the daemon calls made in parallel, 0 meaning no bound. + MaxConcurrency() int + // RuntimeAPIVersion is the API version negotiated with the daemon. + RuntimeAPIVersion(ctx context.Context) (string, error) + // ClassifyRemovalError reports whether the error of a resource removal is a + // benign "still in use" or "already gone", shared with the removal of the + // other resources of a project so the two agree on what is tolerable. + ClassifyRemovalError(err error) (stillInUse, alreadyGone bool) +} + +// Service runs the image operations on a Backend. +type Service struct { + backend Backend +} + +// New creates a Service acting on the given Backend. +func New(backend Backend) *Service { + return &Service{backend: backend} +} + +// Images lists the images the containers of a project run. +func (s *Service) Images(ctx context.Context, projectName string, options api.ImagesOptions) (map[string]api.ImageSummary, error) { + projectName = strings.ToLower(projectName) + allContainers, err := s.backend.APIClient().ContainerList(ctx, client.ContainerListOptions{ + All: true, + Filters: naming.ProjectFilter(projectName), + }) + if err != nil { + return nil, err + } + var containers []container.Summary + if len(options.Services) > 0 { + containers = naming.InServices(allContainers.Items, options.Services) + } else { + containers = allContainers.Items + } + + // The daemon validates the platform field in ImageInspect against the + // negotiated API version from the request path, not the server's own max version. + version, err := s.backend.RuntimeAPIVersion(ctx) + if err != nil { + return nil, err + } + withPlatform := versions.GreaterThanOrEqualTo(version, apiVersion149) + + summary := map[string]api.ImageSummary{} + var mux sync.Mutex + eg, ctx := concurrency.NewErrgroup(ctx, s.backend.MaxConcurrency()) + for _, ctr := range containers { + eg.Go(func() error { + img, err := s.containerImageSummary(ctx, ctr, withPlatform) + if err != nil { + return err + } + mux.Lock() + defer mux.Unlock() + summary[naming.CanonicalContainerName(ctr)] = img + return nil + }) + } + + err = eg.Wait() + return summary, err +} + +// containerImageSummary describes the image a container was created from. The +// image record may not exist anymore: under the containerd image store the +// daemon drops the dangling old index when a rebuild with identical content +// moves the tag — without compose recreating the container (see #13636) — and +// `docker rmi -f` may remove the image of a running container. In that case +// report what the container itself knows rather than failing the whole +// listing (#14014). +func (s *Service) containerImageSummary(ctx context.Context, c container.Summary, withPlatform bool) (api.ImageSummary, error) { + var repository, tag string + if _, err := godigest.Parse(c.Image); err != nil { + // c.Image is a reference, not the raw image ID the engine reports + // when the image record is gone or its tag was moved + if ref, err := reference.ParseDockerRef(c.Image); err == nil { + repository = reference.FamiliarName(ref) + if tagged, ok := ref.(reference.Tagged); ok { + tag = tagged.Tag() + } + } + } + + img, err := s.backend.APIClient().ImageInspect(ctx, c.Image) + if errdefs.IsNotFound(err) { + return api.ImageSummary{ + ID: c.ImageID, + Repository: repository, + Tag: tag, + }, nil + } + if err != nil { + return api.ImageSummary{}, err + } + id := img.ID // platform-specific image ID can't be combined with image tag, see https://github.com/moby/moby/issues/49995 + + if withPlatform && c.ImageManifestDescriptor != nil && c.ImageManifestDescriptor.Platform != nil { + img, err = s.backend.APIClient().ImageInspect(ctx, c.Image, client.ImageInspectWithPlatform(c.ImageManifestDescriptor.Platform)) + if err != nil { + return api.ImageSummary{}, err + } + } + + var created *time.Time + if img.Created != "" { + t, err := time.Parse(time.RFC3339Nano, img.Created) + if err != nil { + return api.ImageSummary{}, err + } + created = &t + } + + return api.ImageSummary{ + ID: id, + Repository: repository, + Tag: tag, + Platform: platforms.Platform{ + Architecture: img.Architecture, + OS: img.Os, + OSVersion: img.OsVersion, + Variant: img.Variant, + }, + Size: img.Size, + Created: created, + LastTagTime: img.Metadata.LastTagTime, + }, nil +} + +// InspectLocalImages inspects the given references in parallel, requesting +// per-manifest data on engines that support it. References not found locally +// are simply absent from the result. +func (s *Service) InspectLocalImages(ctx context.Context, repoTags []string) (map[string]client.ImageInspectResult, error) { + opts, err := s.ImageInspectOptions(ctx) + if err != nil { + return nil, err + } + inspections := map[string]client.ImageInspectResult{} + l := sync.Mutex{} + eg, ctx := concurrency.NewErrgroup(ctx, s.backend.MaxConcurrency()) + for _, repoTag := range repoTags { + eg.Go(func() error { + inspect, err := s.backend.APIClient().ImageInspect(ctx, repoTag, opts...) + if err != nil { + if errdefs.IsNotFound(err) { + return nil + } + return fmt.Errorf("unable to get image '%s': %w", repoTag, err) + } + l.Lock() + inspections[repoTag] = inspect + l.Unlock() + return nil + }) + } + return inspections, eg.Wait() +} + +// ImageInspectOptions requests per-manifest data when the engine supports it +// (see manifestsSupported). +func (s *Service) ImageInspectOptions(ctx context.Context) ([]client.ImageInspectOption, error) { + withManifests, err := s.manifestsSupported(ctx) + if err != nil { + return nil, err + } + if !withManifests { + return nil, nil + } + return []client.ImageInspectOption{client.ImageInspectWithManifests(true)}, nil +} + +func Summarize(repoTag string, inspect client.ImageInspectResult) api.ImageSummary { + tag := "" + repository := "" + ref, err := reference.ParseDockerRef(repoTag) + if err == nil { + // ParseDockerRef will reject a local image ID + repository = reference.FamiliarName(ref) + if tagged, ok := ref.(reference.Tagged); ok { + tag = tagged.Tag() + } + } + id, _, _ := LocalContentDigest(inspect, "") + return api.ImageSummary{ + ID: id, + Repository: repository, + Tag: tag, + Size: inspect.Size, + LastTagTime: inspect.Metadata.LastTagTime, + } +} + +// manifestsSupported reports whether the engine can return per-manifest data on +// image inspect (Engine >= 28.0 / API >= 1.48). Older engines fall back to the +// plain image ID. +func (s *Service) manifestsSupported(ctx context.Context) (bool, error) { + version, err := s.backend.RuntimeAPIVersion(ctx) + if err != nil { + return false, err + } + return versions.GreaterThanOrEqualTo(version, apiVersion148), nil +} + +// CanonicalBuiltDigest resolves the canonical content digest of a just-built +// image so the recorded identity matches what later runs compute for the same +// local image. Registry-only builds (push-only, multi-platform without load) +// are not locally inspectable and keep the builder-reported digest: a volatile +// but honest value — an actual rebuild is still detected — preferred over a +// stable marker that would hide real image changes. Best effort: never fails +// an already-successful build. +func (s *Service) CanonicalBuiltDigest(ctx context.Context, imageRef, platform, builderDigest string) string { + id, _, err := s.InspectLocalContent(ctx, imageRef, platform) + if err != nil { + logrus.Debugf("unable to resolve content digest for built image %s, keeping builder-reported digest: %v", imageRef, err) + return builderDigest + } + return id +} + +// LocalContentDigest returns the digest identifying an inspected image's +// runnable content (config + layers) for the requested platform (empty means +// the host default), plus whether the local image satisfies that platform. +// Note the satisfied bool is only meaningful for an explicitly requested +// platform: with platform empty, the manifests path reports host-default +// satisfaction while the manifest-less path always reports true. +// +// With BuildKit provenance attestations enabled (the default since recent +// Buildx/BuildKit), the image is stored as an index whose top-level digest +// (inspect.ID) also covers the attestation manifest, so it changes on every +// build even when the runnable image is unchanged — making compose recreate +// containers needlessly (see +// https://github.com/docker/compose/issues/13636). The digest of the "image" +// kind manifest reflects only the image content, which is what compose needs +// to detect staleness. Images inspected without manifest data (engines that +// can't report them) keep the plain image ID — already the config digest — +// with platform satisfaction from the inspect's flat platform fields, the +// pre-manifest behavior. Every image identity compose records for staleness +// comparison must be computed through here, whatever the image's provenance +// (pulled, built, already local), so any two runs produce comparable values. +// +// The converse also holds: this platform-specific digest is ONLY an identity +// for staleness comparison. Pinning image references in a reproducible +// compose model (publish / config --resolve-image-digests) must keep using +// the registry descriptor digest — the multi-platform index — through +// ImageDigestResolver, or the published file would be bound to the platform +// of whoever resolved it. +func LocalContentDigest(inspect client.ImageInspectResult, platform string) (string, bool, error) { + var matcher platforms.Matcher = platforms.Default() + pinned := platform != "" + if pinned { + p, err := platforms.Parse(platform) + if err != nil { + return "", false, err + } + matcher = platforms.NewMatcher(p) + } + if len(inspect.Manifests) > 0 { + id, ok := matchLocalManifest(inspect.InspectResponse, matcher) + return id, ok, nil + } + ok := true + if pinned { + ok = matcher.Match(specs.Platform{ + Architecture: inspect.Architecture, + OS: inspect.Os, + Variant: inspect.Variant, + }) + } + return inspect.ID, ok, nil +} + +// matchLocalManifest selects, among the locally available image manifests of +// inspect, the digest identifying the runnable content for the requested +// platform, and reports whether the local content actually satisfies that +// platform. Selection is platform-aware and deterministic so the same image +// always maps to the same digest across rebuilds: the available manifest +// matching the requested platform wins (satisfied); a lone available image +// manifest keeps its digest — single-platform images stay identifiable +// whatever their platform — without satisfying a different requested +// platform; otherwise fall back to inspect.ID, which never satisfies the +// request. +// +// Both the digest producers and the platform checks must go through this +// single selection so the digest recorded and the platform validated always +// refer to the same manifest. +func matchLocalManifest(inspect image.InspectResponse, platform platforms.Matcher) (string, bool) { + var available []image.ManifestSummary + for _, m := range inspect.Manifests { + if m.Kind == image.ManifestKindImage && m.Available { + available = append(available, m) + } + } + for _, m := range available { + if m.ImageData != nil && platform.Match(m.ImageData.Platform) { + return m.ID, true + } + } + if len(available) == 1 { + m := available[0] + if m.ImageData == nil { + // engines may omit per-manifest image data (seen with locally + // built, never-pushed images): fall back to the inspect's flat + // platform fields, like the manifest-less path does, instead of + // reporting the platform unsatisfied and triggering a pull of a + // possibly local-only image + return m.ID, platform.Match(specs.Platform{ + Architecture: inspect.Architecture, + OS: inspect.Os, + Variant: inspect.Variant, + }) + } + return m.ID, false + } + return inspect.ID, false +} + +// InspectLocalContent inspects ref and returns its canonical content digest +// for the requested platform (empty means the host default), plus whether the +// local image satisfies that platform — see LocalContentDigest. +func (s *Service) InspectLocalContent(ctx context.Context, ref string, platform string) (string, bool, error) { + opts, err := s.ImageInspectOptions(ctx) + if err != nil { + return "", false, err + } + inspected, err := s.backend.APIClient().ImageInspect(ctx, ref, opts...) + if err != nil { + return "", false, err + } + return LocalContentDigest(inspected, platform) +} diff --git a/internal/images/images_test.go b/internal/images/images_test.go new file mode 100644 index 000000000..f01be5a79 --- /dev/null +++ b/internal/images/images_test.go @@ -0,0 +1,452 @@ +/* + Copyright 2024 Docker Compose CLI authors + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package images + +import ( + "context" + "net/netip" + "strings" + "testing" + "time" + + "github.com/containerd/errdefs" + "github.com/containerd/platforms" + "github.com/moby/moby/api/types/container" + "github.com/moby/moby/api/types/image" + "github.com/moby/moby/client" + specs "github.com/opencontainers/image-spec/specs-go/v1" + "go.uber.org/mock/gomock" + "gotest.tools/v3/assert" + + "github.com/docker/compose/v5/internal/naming" + compose "github.com/docker/compose/v5/pkg/api" + "github.com/docker/compose/v5/pkg/mocks" +) + +const testProject = "testProject" + +// fakeBackend serves the API client it is given, and the API version a daemon +// of that version would have negotiated. +type fakeBackend struct { + apiClient client.APIClient + version string +} + +func (f *fakeBackend) APIClient() client.APIClient { return f.apiClient } +func (f *fakeBackend) MaxConcurrency() int { return 0 } + +func (f *fakeBackend) RuntimeAPIVersion(context.Context) (string, error) { return f.version, nil } + +func (f *fakeBackend) ClassifyRemovalError(err error) (stillInUse, alreadyGone bool) { + return errdefs.IsConflict(err), errdefs.IsNotFound(err) +} + +func newTestService(t *testing.T, mockCtrl *gomock.Controller, apiVersion string) (*mocks.MockAPIClient, *Service) { + t.Helper() + api := mocks.NewMockAPIClient(mockCtrl) + return api, New(&fakeBackend{apiClient: api, version: apiVersion}) +} + +func anyCancellableContext() gomock.Matcher { + //nolint:forbidigo // This creates a context type for gomock matching, not for actual test usage + ctxWithCancel, cancel := context.WithCancel(context.Background()) + cancel() + return gomock.AssignableToTypeOf(ctxWithCancel) +} + +func TestImages(t *testing.T) { + mockCtrl := gomock.NewController(t) + defer mockCtrl.Finish() + + api := mocks.NewMockAPIClient(mockCtrl) + tested := New(&fakeBackend{apiClient: api, version: "1.96"}) + + args := naming.ProjectFilter(strings.ToLower(testProject)) + listOpts := client.ContainerListOptions{All: true, Filters: args} + timeStr1 := "2025-06-06T06:06:06.000000000Z" + created1, _ := time.Parse(time.RFC3339Nano, timeStr1) + timeStr2 := "2025-03-03T03:03:03.000000000Z" + created2, _ := time.Parse(time.RFC3339Nano, timeStr2) + image1 := imageInspect("image1", "foo:1", 12345, timeStr1) + image2 := imageInspect("image2", "bar:2", 67890, timeStr2) + api.EXPECT().ImageInspect(anyCancellableContext(), "foo:1").Return(client.ImageInspectResult{InspectResponse: image1}, nil).MaxTimes(2) + api.EXPECT().ImageInspect(anyCancellableContext(), "bar:2").Return(client.ImageInspectResult{InspectResponse: image2}, nil) + c1 := containerDetail("service1", "123", container.StateRunning, "foo:1") + c2 := containerDetail("service1", "456", container.StateRunning, "bar:2") + c2.Ports = []container.PortSummary{{PublicPort: 80, PrivatePort: 90, IP: netip.MustParseAddr("127.0.0.1")}} + c3 := containerDetail("service2", "789", container.StateExited, "foo:1") + api.EXPECT().ContainerList(t.Context(), listOpts).Return(client.ContainerListResult{ + Items: []container.Summary{c1, c2, c3}, + }, nil) + + images, err := tested.Images(t.Context(), strings.ToLower(testProject), compose.ImagesOptions{}) + + expected := map[string]compose.ImageSummary{ + "123": { + ID: "image1", + Repository: "foo", + Tag: "1", + Size: 12345, + Created: &created1, + }, + "456": { + ID: "image2", + Repository: "bar", + Tag: "2", + Size: 67890, + Created: &created2, + }, + "789": { + ID: "image1", + Repository: "foo", + Tag: "1", + Size: 12345, + Created: &created1, + }, + } + assert.NilError(t, err) + assert.DeepEqual(t, images, expected) +} + +func TestImagesMissingImageRecord(t *testing.T) { + // A container may reference an image record that no longer exists: under + // the containerd image store, a rebuild with identical content moves the + // tag and the daemon drops the old index the running container was created + // from (https://github.com/docker/compose/issues/14014), and `docker rmi + // -f` may remove the image of a running container. The listing must + // degrade to what the container itself knows instead of failing. + mockCtrl := gomock.NewController(t) + defer mockCtrl.Finish() + + api := mocks.NewMockAPIClient(mockCtrl) + tested := New(&fakeBackend{apiClient: api, version: "1.96"}) + + args := naming.ProjectFilter(strings.ToLower(testProject)) + listOpts := client.ContainerListOptions{All: true, Filters: args} + + // the raw image ID the engine reports once the image record is gone + const goneID = "sha256:52c159e21b703f2c750d0bfd8d0af93324b46c0df0859331fb9489b7ec033d0c" + gone := containerDetail("service1", "123", container.StateRunning, goneID) + gone.ImageID = goneID + api.EXPECT().ImageInspect(anyCancellableContext(), goneID). + Return(client.ImageInspectResult{}, errdefs.ErrNotFound) + api.EXPECT().ContainerList(t.Context(), listOpts).Return(client.ContainerListResult{ + Items: []container.Summary{gone}, + }, nil) + + images, err := tested.Images(t.Context(), strings.ToLower(testProject), compose.ImagesOptions{}) + assert.NilError(t, err) + assert.DeepEqual(t, images, map[string]compose.ImageSummary{ + "123": {ID: goneID}, + }) +} + +func imageInspect(id string, imageReference string, size int64, created string) image.InspectResponse { + return image.InspectResponse{ + ID: id, + RepoTags: []string{ + "someRepo:someTag", + imageReference, + }, + Size: size, + Created: created, + } +} + +func imageManifest(id, arch string, available bool) image.ManifestSummary { + return image.ManifestSummary{ + ID: id, + Kind: image.ManifestKindImage, + Available: available, + ImageData: &image.ImageProperties{ + Platform: specs.Platform{OS: "linux", Architecture: arch}, + }, + } +} + +func attestationManifest() image.ManifestSummary { + return image.ManifestSummary{ID: "sha256:att", Kind: image.ManifestKindAttestation, Available: true} +} + +func TestMatchLocalManifest(t *testing.T) { + amd64 := platforms.Only(specs.Platform{OS: "linux", Architecture: "amd64"}) + arm64 := platforms.Only(specs.Platform{OS: "linux", Architecture: "arm64"}) + + match := func(t *testing.T, inspect image.InspectResponse, platform platforms.Matcher, wantID string, wantOK bool) { + t.Helper() + id, ok := matchLocalManifest(inspect, platform) + assert.Equal(t, id, wantID) + assert.Equal(t, ok, wantOK) + } + + t.Run("no manifests falls back to the plain image ID, unsatisfied", func(t *testing.T) { + match(t, image.InspectResponse{ID: "sha256:top"}, amd64, "sha256:top", false) + }) + + t.Run("attested image ignores the attestation manifest", func(t *testing.T) { + // sha256:index changes on every build due to attestation metadata churn; + // the image-kind manifest digest is what actually reflects the content. + inspect := image.InspectResponse{ + ID: "sha256:index", + Manifests: []image.ManifestSummary{ + imageManifest("sha256:amd64", "amd64", true), + attestationManifest(), + }, + } + match(t, inspect, amd64, "sha256:amd64", true) + }) + + t.Run("single image manifest keeps its digest but does not satisfy another platform", func(t *testing.T) { + // single-platform image built for a non-host platform stays resolvable + inspect := image.InspectResponse{ + ID: "sha256:index", + Manifests: []image.ManifestSummary{imageManifest("sha256:arm64", "arm64", true)}, + } + match(t, inspect, amd64, "sha256:arm64", false) + match(t, inspect, arm64, "sha256:arm64", true) + }) + + t.Run("multi-platform picks the matching platform manifest", func(t *testing.T) { + inspect := image.InspectResponse{ + ID: "sha256:index", + Manifests: []image.ManifestSummary{ + imageManifest("sha256:amd64", "amd64", true), + imageManifest("sha256:arm64", "arm64", true), + attestationManifest(), + }, + } + match(t, inspect, amd64, "sha256:amd64", true) + match(t, inspect, arm64, "sha256:arm64", true) + }) + + t.Run("unavailable manifests are skipped", func(t *testing.T) { + // amd64 is referenced by the index but not pulled locally; only arm64 is usable + inspect := image.InspectResponse{ + ID: "sha256:index", + Manifests: []image.ManifestSummary{ + imageManifest("sha256:amd64", "amd64", false), + imageManifest("sha256:arm64", "arm64", true), + }, + } + match(t, inspect, amd64, "sha256:arm64", false) + }) + + t.Run("only attestation manifests falls back to the plain image ID", func(t *testing.T) { + inspect := image.InspectResponse{ + ID: "sha256:top", + Manifests: []image.ManifestSummary{attestationManifest()}, + } + match(t, inspect, amd64, "sha256:top", false) + }) + + t.Run("ambiguous multi-platform with no match falls back to the plain image ID", func(t *testing.T) { + inspect := image.InspectResponse{ + ID: "sha256:index", + Manifests: []image.ManifestSummary{ + imageManifest("sha256:amd64", "amd64", true), + imageManifest("sha256:arm64", "arm64", true), + }, + } + windows := platforms.Only(specs.Platform{OS: "windows", Architecture: "amd64"}) + match(t, inspect, windows, "sha256:index", false) + }) +} + +func TestInspectLocalImagesUsesContentDigest(t *testing.T) { + mockCtrl := gomock.NewController(t) + defer mockCtrl.Finish() + api, tested := newTestService(t, mockCtrl, "1.48") + + inspect := image.InspectResponse{ + ID: "sha256:index", // attested index digest, churns every build + Manifests: []image.ManifestSummary{ + imageManifest("sha256:image", "amd64", true), + attestationManifest(), + }, + } + api.EXPECT(). + ImageInspect(anyCancellableContext(), "foo:1", gomock.Any()). + Return(client.ImageInspectResult{InspectResponse: inspect}, nil) + + inspections, err := tested.InspectLocalImages(t.Context(), []string{"foo:1"}) + assert.NilError(t, err) + assert.Equal(t, Summarize("foo:1", inspections["foo:1"]).ID, "sha256:image") +} + +func TestInspectLocalImagesLegacyEngineUsesPlainID(t *testing.T) { + // Engine < 28.0 (API < 1.48) can't report manifests, so we keep the plain ID. + mockCtrl := gomock.NewController(t) + defer mockCtrl.Finish() + api, tested := newTestService(t, mockCtrl, "1.47") + + inspect := image.InspectResponse{ID: "sha256:plain"} + api.EXPECT(). + ImageInspect(anyCancellableContext(), "foo:1"). + Return(client.ImageInspectResult{InspectResponse: inspect}, nil) + + inspections, err := tested.InspectLocalImages(t.Context(), []string{"foo:1"}) + assert.NilError(t, err) + assert.Equal(t, Summarize("foo:1", inspections["foo:1"]).ID, "sha256:plain") +} + +func TestInspectLocalImagesSkipsMissingImages(t *testing.T) { + // Registry-only images (push/multi-platform) aren't inspectable locally; + // they must be omitted so the caller keeps the Bake-reported digest. + mockCtrl := gomock.NewController(t) + defer mockCtrl.Finish() + api, tested := newTestService(t, mockCtrl, "1.48") + + api.EXPECT(). + ImageInspect(anyCancellableContext(), "missing:1", gomock.Any()). + Return(client.ImageInspectResult{}, errdefs.ErrNotFound) + + inspections, err := tested.InspectLocalImages(t.Context(), []string{"missing:1"}) + assert.NilError(t, err) + _, ok := inspections["missing:1"] + assert.Assert(t, !ok) +} + +func TestInspectLocalContent(t *testing.T) { + t.Run("manifests path selects the pinned platform", func(t *testing.T) { + mockCtrl := gomock.NewController(t) + defer mockCtrl.Finish() + api, tested := newTestService(t, mockCtrl, "1.48") + api.EXPECT(). + ImageInspect(anyCancellableContext(), "foo:1", gomock.Any()). + Return(client.ImageInspectResult{InspectResponse: image.InspectResponse{ + ID: "sha256:index", + Manifests: []image.ManifestSummary{ + imageManifest("sha256:amd64", "amd64", true), + imageManifest("sha256:arm64", "arm64", true), + }, + }}, nil) + + id, ok, err := tested.InspectLocalContent(t.Context(), "foo:1", "linux/arm64") + assert.NilError(t, err) + assert.Equal(t, id, "sha256:arm64") + assert.Assert(t, ok) + }) + + t.Run("manifests path reports an unavailable pinned platform", func(t *testing.T) { + mockCtrl := gomock.NewController(t) + defer mockCtrl.Finish() + api, tested := newTestService(t, mockCtrl, "1.48") + api.EXPECT(). + ImageInspect(anyCancellableContext(), "foo:1", gomock.Any()). + Return(client.ImageInspectResult{InspectResponse: image.InspectResponse{ + ID: "sha256:index", + Manifests: []image.ManifestSummary{imageManifest("sha256:amd64", "amd64", true)}, + }}, nil) + + _, ok, err := tested.InspectLocalContent(t.Context(), "foo:1", "linux/arm64") + assert.NilError(t, err) + assert.Assert(t, !ok) + }) + + t.Run("legacy engine falls back to flat platform fields", func(t *testing.T) { + mockCtrl := gomock.NewController(t) + defer mockCtrl.Finish() + api, tested := newTestService(t, mockCtrl, "1.47") + api.EXPECT(). + ImageInspect(anyCancellableContext(), "foo:1"). + Return(client.ImageInspectResult{InspectResponse: image.InspectResponse{ + ID: "sha256:plain", Os: "linux", Architecture: "amd64", + }}, nil).Times(2) + + id, ok, err := tested.InspectLocalContent(t.Context(), "foo:1", "linux/amd64") + assert.NilError(t, err) + assert.Equal(t, id, "sha256:plain") + assert.Assert(t, ok) + + _, ok, err = tested.InspectLocalContent(t.Context(), "foo:1", "linux/arm64") + assert.NilError(t, err) + assert.Assert(t, !ok) + }) +} + +func TestCanonicalBuiltDigest(t *testing.T) { + t.Run("locally inspectable build resolves to the content digest", func(t *testing.T) { + mockCtrl := gomock.NewController(t) + defer mockCtrl.Finish() + api, tested := newTestService(t, mockCtrl, "1.48") + api.EXPECT(). + ImageInspect(anyCancellableContext(), "built:1", gomock.Any()). + Return(client.ImageInspectResult{InspectResponse: image.InspectResponse{ + ID: "sha256:index", + Manifests: []image.ManifestSummary{ + imageManifest("sha256:image", "amd64", true), + attestationManifest(), + }, + }}, nil) + + got := tested.CanonicalBuiltDigest(t.Context(), "built:1", "", "sha256:bakeindex") + assert.Equal(t, got, "sha256:image") + }) + + t.Run("registry-only build keeps the builder-reported digest", func(t *testing.T) { + // push-only / multi-platform-only builds never land in the local + // store: keep the builder digest — volatile but honest (a real + // rebuild is detected) rather than a stable marker hiding changes. + mockCtrl := gomock.NewController(t) + defer mockCtrl.Finish() + api, tested := newTestService(t, mockCtrl, "1.48") + api.EXPECT(). + ImageInspect(anyCancellableContext(), "pushed:1", gomock.Any()). + Return(client.ImageInspectResult{}, errdefs.ErrNotFound) + + got := tested.CanonicalBuiltDigest(t.Context(), "pushed:1", "", "sha256:bakeindex") + assert.Equal(t, got, "sha256:bakeindex") + }) +} + +func containerDetail(service string, id string, status container.ContainerState, imageName string) container.Summary { + return container.Summary{ + ID: id, + Names: []string{"/" + id}, + Image: imageName, + Labels: map[string]string{ + compose.ServiceLabel: service, + compose.ProjectLabel: strings.ToLower(testProject), + }, + State: status, + } +} + +func TestMatchLocalManifestWithoutImageData(t *testing.T) { + // engines may omit per-manifest image data (seen with locally built, + // never-pushed images): the lone available manifest falls back to the + // inspect's flat platform fields instead of reporting the platform + // unsatisfied — which made up try to pull a local-only image. + amd64 := platforms.OnlyStrict(specs.Platform{OS: "linux", Architecture: "amd64"}) + arm64 := platforms.OnlyStrict(specs.Platform{OS: "linux", Architecture: "arm64"}) + inspect := image.InspectResponse{ + ID: "sha256:index", + Os: "linux", + Architecture: "amd64", + Manifests: []image.ManifestSummary{ + {ID: "sha256:lone", Kind: image.ManifestKindImage, Available: true}, + }, + } + + id, satisfied := matchLocalManifest(inspect, amd64) + assert.Equal(t, id, "sha256:lone") + assert.Assert(t, satisfied, "flat platform fields match the requested platform") + + id, satisfied = matchLocalManifest(inspect, arm64) + assert.Equal(t, id, "sha256:lone") + assert.Assert(t, !satisfied, "flat platform fields don't match the requested platform") +} diff --git a/internal/images/prune.go b/internal/images/prune.go new file mode 100644 index 000000000..40880b3a8 --- /dev/null +++ b/internal/images/prune.go @@ -0,0 +1,304 @@ +/* + Copyright 2022 Docker Compose CLI authors + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package images + +import ( + "context" + "errors" + "fmt" + "maps" + "slices" + "sync" + + "github.com/compose-spec/compose-go/v2/types" + "github.com/containerd/errdefs" + "github.com/distribution/reference" + "github.com/moby/moby/api/types/image" + "github.com/moby/moby/client" + "github.com/sirupsen/logrus" + "golang.org/x/sync/semaphore" + + "github.com/docker/compose/v5/internal/concurrency" + "github.com/docker/compose/v5/internal/naming" + "github.com/docker/compose/v5/pkg/api" +) + +// PruneOptions controls the behavior of image pruning. +type PruneOptions struct { + Mode api.ImagePruneMode + + // RemoveOrphans will result in the removal of images that were built for + // the project regardless of whether they are for a known service if true. + RemoveOrphans bool +} + +// Pruner handles image removal during Compose `down` operations. +type Pruner struct { + client client.ImageAPIClient + project *types.Project + // limiter bounds concurrent engine calls, shared with the rest of the + // down op set so tagged- and dangling-image removal (which run as two + // independent, concurrently-dispatched ops) don't each spend their own + // full budget on top of one another. + limiter *semaphore.Weighted +} + +// NewPruner creates a Pruner object for a project. limiter may be +// nil, meaning unbounded. +func NewPruner(imageClient client.ImageAPIClient, project *types.Project, limiter *semaphore.Weighted) *Pruner { + return &Pruner{ + client: imageClient, + project: project, + limiter: limiter, + } +} + +// ImageBelongsToKnownService reports whether img should be pruned: either +// removeOrphans is set (indiscriminately prune all project images even if +// they're not referenced by the current Compose state, e.g. the service was +// removed from YAML), or the image belongs to a service still declared in +// project. Shared by ImagesToPrune (tagged images) and down's own +// dangling-image keep check, so the two can't drift on what counts as +// orphaned — the image-side analog of containers.go's isOrphaned. +func ImageBelongsToKnownService(project *types.Project, removeOrphans bool, img image.Summary) bool { + if removeOrphans { + return true + } + _, err := project.GetService(img.Labels[api.ServiceLabel]) + return err == nil +} + +// ImagesToPrune returns the set of images that should be removed. +func (p *Pruner) ImagesToPrune(ctx context.Context, opts PruneOptions) ([]string, error) { + if opts.Mode == api.ImagePruneNone { + return nil, nil + } else if !opts.Mode.Valid() { + return nil, fmt.Errorf("unsupported image prune mode: %s", opts.Mode) + } + var images []string + + if opts.Mode == api.ImagePruneAll { + namedImages, err := p.namedImages(ctx) + if err != nil { + return nil, err + } + images = append(images, namedImages...) + } + + projectImages, err := p.labeledLocalImages(ctx) + if err != nil { + return nil, err + } + for _, img := range projectImages { + if len(img.RepoTags) == 0 { + // currently, we're only pruning the tagged references, but + // if we start removing the dangling images and grouping by + // service, we can remove this (and should rely on `Image::ID`) + continue + } + + if ImageBelongsToKnownService(p.project, opts.RemoveOrphans, img) { + images = append(images, img.RepoTags[0]) + } + } + + fallbackImages, err := p.unlabeledLocalImages(ctx) + if err != nil { + return nil, err + } + images = append(images, fallbackImages...) + + images = normalizeAndDedupeImages(images) + return images, nil +} + +// namedImages are those that are explicitly named in the service config. +// +// These could be registry-only images (no local build), hybrid (support build +// as a fallback if cannot pull), or local-only (image does not exist in a +// registry). +func (p *Pruner) namedImages(ctx context.Context) ([]string, error) { + var images []string + for _, service := range p.project.Services { + if service.Image == "" { + continue + } + images = append(images, service.Image) + } + return p.filterImagesByExistence(ctx, images) +} + +// labeledLocalImages are images that were locally-built by a current version of +// Compose (it did not always label built images). +// +// The image name could either have been defined by the user or implicitly +// created from the project + service name. +func (p *Pruner) labeledLocalImages(ctx context.Context) ([]image.Summary, error) { + res, err := p.client.ImageList(ctx, client.ImageListOptions{ + Filters: naming.ProjectFilter(p.project.Name).Add("dangling", "false"), + }) + if err != nil { + return nil, err + } + return res.Items, nil +} + +// RemoveImages removes the given images in parallel, tolerating individual +// failures so one bad image doesn't abort the rest. An image already gone +// (a benign race with something else removing it concurrently) or still in +// use is never joined into err — same tolerance removeResource already +// gives a tagged image in either situation (its own distinct, correctly +// worded event, not an error). "Still in use" is returned in its own bucket, +// not folded into removed, since a caller aggregating several images (see +// removeDanglingImagesOp) needs that distinction to report the batch +// honestly; an already-gone image reached the goal the same as an actual +// removal, so it's simply not counted against either bucket. Any other +// removal error is joined into err so callers reporting failures to the +// user keep the actual daemon error instead of just an image ID. +func (s *Service) RemoveImages(ctx context.Context, images []image.Summary, limiter *semaphore.Weighted) (removed, stillInUse []string, err error) { + var mu sync.Mutex + var errs []error + // as in removeTaggedImagesOp, a single image failure must not cancel the + // shared errgroup ctx, so fn always returns nil and collects into the + // buckets below instead. + _ = concurrency.ForEach(ctx, limiter, images, func(ctx context.Context, img image.Summary) error { + if _, err := s.backend.APIClient().ImageRemove(ctx, img.ID, client.ImageRemoveOptions{}); err != nil { + mu.Lock() + defer mu.Unlock() + imgStillInUse, alreadyGone := s.backend.ClassifyRemovalError(err) + switch { + case alreadyGone: + // e.g. removed concurrently by something else + logrus.Debugf("dangling image %s already removed: %v", img.ID, err) + case imgStillInUse: + // the same benign skip the tagged-image path reports via + // removeResource's conflict branch + logrus.Debugf("dangling image %s still in use: %v", img.ID, err) + stillInUse = append(stillInUse, img.ID) + default: + errs = append(errs, fmt.Errorf("image %s: %w", img.ID, err)) + } + return nil + } + mu.Lock() + defer mu.Unlock() + removed = append(removed, img.ID) + return nil + }) + return removed, stillInUse, errors.Join(errs...) +} + +// EligibleDanglingImages lists a project's dangling images and filters out +// those spared by keep. Shared by down --rmi (which also needs the list +// itself, to decide whether there's anything to report) and watch --prune, +// so the two don't drift apart. +func (s *Service) EligibleDanglingImages(ctx context.Context, projectName string, keep func(image.Summary) bool) ([]image.Summary, error) { + res, err := s.backend.APIClient().ImageList(ctx, client.ImageListOptions{ + Filters: naming.ProjectFilter(projectName).Add("dangling", "true"), + }) + if err != nil { + return nil, err + } + + var eligible []image.Summary + for _, img := range res.Items { + if !keep(img) { + eligible = append(eligible, img) + } + } + return eligible, nil +} + +// RemoveDanglingImages lists a project's dangling images and removes those +// not spared by keep. Shared by down --rmi and watch --prune, which differ +// only in what keep spares. +func (s *Service) RemoveDanglingImages(ctx context.Context, projectName string, keep func(image.Summary) bool) ([]string, error) { + eligible, err := s.EligibleDanglingImages(ctx, projectName, keep) + if err != nil { + return nil, err + } + removed, _, err := s.RemoveImages(ctx, eligible, concurrency.NewLimiter(s.backend.MaxConcurrency())) + return removed, err +} + +// unlabeledLocalImages are images that match the implicit naming convention +// for locally-built images but did not get labeled, presumably because they +// were produced by an older version of Compose. +// +// This is transitional to ensure `down` continues to work as expected on +// projects built/launched by previous versions of Compose. It can safely +// be removed after some time. +func (p *Pruner) unlabeledLocalImages(ctx context.Context) ([]string, error) { + var images []string + for _, service := range p.project.Services { + if service.Image != "" { + continue + } + img := api.GetImageNameOrDefault(service, p.project.Name) + images = append(images, img) + } + return p.filterImagesByExistence(ctx, images) +} + +// filterImagesByExistence returns the subset of images that exist in the +// engine store. +// +// NOTE: Any transient errors communicating with the API will result in an +// image being returned as "existing", as this method is exclusively used to +// find images to remove, so the worst case of being conservative here is an +// attempt to remove an image that doesn't exist, which will cause a warning +// but is otherwise harmless. +func (p *Pruner) filterImagesByExistence(ctx context.Context, imageNames []string) ([]string, error) { + var mu sync.Mutex + var ret []string + + err := concurrency.ForEach(ctx, p.limiter, imageNames, func(ctx context.Context, img string) error { + _, err := p.client.ImageInspect(ctx, img) + if errdefs.IsNotFound(err) { + // err on the side of caution: only skip if we successfully + // queried the API and got back a definitive "not exists" + return nil + } + mu.Lock() + defer mu.Unlock() + ret = append(ret, img) + return nil + }) + if err != nil { + return nil, err + } + + return ret, nil +} + +// normalizeAndDedupeImages returns the unique set of images after normalization. +func normalizeAndDedupeImages(images []string) []string { + seen := make(map[string]struct{}, len(images)) + for _, img := range images { + // since some references come from user input (service.image) and some + // come from the engine API, we standardize them, opting for the + // familiar name format since they'll also be displayed in the CLI + ref, err := reference.ParseNormalizedNamed(img) + if err == nil { + ref = reference.TagNameOnly(ref) + img = reference.FamiliarString(ref) + } + seen[img] = struct{}{} + } + // ensure a deterministic return result - the actual ordering is not useful + return slices.Sorted(maps.Keys(seen)) +} diff --git a/pkg/compose/image_pruner_test.go b/internal/images/prune_test.go similarity index 87% rename from pkg/compose/image_pruner_test.go rename to internal/images/prune_test.go index 59d609c86..494115d62 100644 --- a/pkg/compose/image_pruner_test.go +++ b/internal/images/prune_test.go @@ -14,7 +14,7 @@ limitations under the License. */ -package compose +package images import ( "testing" @@ -27,6 +27,8 @@ import ( "gotest.tools/v3/assert" "github.com/docker/compose/v5/internal/naming" + compose "github.com/docker/compose/v5/pkg/api" + "github.com/docker/compose/v5/pkg/mocks" ) // TestRemoveDanglingImages_FiltersKeepsAndToleratesFailures guards the @@ -42,10 +44,8 @@ func TestRemoveDanglingImages_FiltersKeepsAndToleratesFailures(t *testing.T) { mockCtrl := gomock.NewController(t) defer mockCtrl.Finish() - apiClient, cli := prepareMocks(mockCtrl) - svcIface, err := NewComposeService(cli) - assert.NilError(t, err) - svc := svcIface.(*composeService) + apiClient := mocks.NewMockAPIClient(mockCtrl) + svc := New(&fakeBackend{apiClient: apiClient}) apiClient.EXPECT().ImageList(gomock.Any(), client.ImageListOptions{ Filters: naming.ProjectFilter("prj").Add("dangling", "true"), @@ -67,7 +67,7 @@ func TestRemoveDanglingImages_FiltersKeepsAndToleratesFailures(t *testing.T) { // no expectation for "sha256:keep" — a call to ImageRemove for it fails the test keep := func(img image.Summary) bool { return img.ID == "sha256:keep" } - removed, err := svc.removeDanglingImages(t.Context(), "prj", keep) + removed, err := svc.RemoveDanglingImages(t.Context(), "prj", keep) assert.DeepEqual(t, removed, []string{"sha256:removed"}) assert.ErrorContains(t, err, "sha256:fails") assert.ErrorContains(t, err, "permission denied") @@ -84,10 +84,8 @@ func TestRemoveImages_BucketsStillInUseSeparatelyFromRemoved(t *testing.T) { mockCtrl := gomock.NewController(t) defer mockCtrl.Finish() - apiClient, cli := prepareMocks(mockCtrl) - svcIface, err := NewComposeService(cli) - assert.NilError(t, err) - svc := svcIface.(*composeService) + apiClient := mocks.NewMockAPIClient(mockCtrl) + svc := New(&fakeBackend{apiClient: apiClient}) images := []image.Summary{ {ID: "sha256:removed"}, @@ -104,7 +102,7 @@ func TestRemoveImages_BucketsStillInUseSeparatelyFromRemoved(t *testing.T) { apiClient.EXPECT().ImageRemove(gomock.Any(), "sha256:fails", client.ImageRemoveOptions{}). Return(client.ImageRemoveResult{}, errdefs.ErrPermissionDenied.WithMessage("permission denied")) - removed, stillInUse, err := svc.removeImages(t.Context(), images, nil) + removed, stillInUse, err := svc.RemoveImages(t.Context(), images, nil) assert.DeepEqual(t, removed, []string{"sha256:removed"}) assert.DeepEqual(t, stillInUse, []string{"sha256:in-use"}) assert.ErrorContains(t, err, "sha256:fails") @@ -117,16 +115,14 @@ func TestRemoveDanglingImages_NoneFound(t *testing.T) { mockCtrl := gomock.NewController(t) defer mockCtrl.Finish() - apiClient, cli := prepareMocks(mockCtrl) - svcIface, err := NewComposeService(cli) - assert.NilError(t, err) - svc := svcIface.(*composeService) + apiClient := mocks.NewMockAPIClient(mockCtrl) + svc := New(&fakeBackend{apiClient: apiClient}) apiClient.EXPECT().ImageList(gomock.Any(), client.ImageListOptions{ Filters: naming.ProjectFilter("prj").Add("dangling", "true"), }).Return(client.ImageListResult{}, nil) - removed, err := svc.removeDanglingImages(t.Context(), "prj", func(image.Summary) bool { return false }) + removed, err := svc.RemoveDanglingImages(t.Context(), "prj", func(image.Summary) bool { return false }) assert.NilError(t, err) assert.Equal(t, len(removed), 0) } @@ -142,9 +138,9 @@ func TestImagesToPrune_RejectsInvalidMode(t *testing.T) { mockCtrl := gomock.NewController(t) defer mockCtrl.Finish() - apiClient, _ := prepareMocks(mockCtrl) + apiClient := mocks.NewMockAPIClient(mockCtrl) - pruner := NewImagePruner(apiClient, &types.Project{Name: "prj"}, nil) - _, err := pruner.ImagesToPrune(t.Context(), ImagePruneOptions{Mode: ImagePruneMode("bogus")}) + pruner := NewPruner(apiClient, &types.Project{Name: "prj"}, nil) + _, err := pruner.ImagesToPrune(t.Context(), PruneOptions{Mode: compose.ImagePruneMode("bogus")}) assert.ErrorContains(t, err, "unsupported image prune mode: bogus") } diff --git a/pkg/compose/image_pruner.go b/pkg/compose/image_pruner.go index 985604d89..2c274a51a 100644 --- a/pkg/compose/image_pruner.go +++ b/pkg/compose/image_pruner.go @@ -18,22 +18,13 @@ package compose import ( "context" - "errors" - "fmt" - "maps" - "slices" - "sync" "github.com/compose-spec/compose-go/v2/types" - "github.com/containerd/errdefs" - "github.com/distribution/reference" "github.com/moby/moby/api/types/image" "github.com/moby/moby/client" - "github.com/sirupsen/logrus" "golang.org/x/sync/semaphore" - "github.com/docker/compose/v5/internal/concurrency" - "github.com/docker/compose/v5/internal/naming" + "github.com/docker/compose/v5/internal/images" "github.com/docker/compose/v5/pkg/api" ) @@ -53,268 +44,34 @@ const ( ImagePruneAll = api.ImagePruneAll ) -// ImagePruneOptions controls the behavior of image pruning. -type ImagePruneOptions struct { - Mode ImagePruneMode +// The pruning of images lives in internal/images. ImagePruneOptions and +// ImagePruner stay exported from this package as aliases, so the SDK surface +// does not change. - // RemoveOrphans will result in the removal of images that were built for - // the project regardless of whether they are for a known service if true. - RemoveOrphans bool -} +// ImagePruneOptions controls the behavior of image pruning. +type ImagePruneOptions = images.PruneOptions // ImagePruner handles image removal during Compose `down` operations. -type ImagePruner struct { - client client.ImageAPIClient - project *types.Project - // limiter bounds concurrent engine calls, shared with the rest of the - // down op set so tagged- and dangling-image removal (which run as two - // independent, concurrently-dispatched ops) don't each spend their own - // full budget on top of one another. - limiter *semaphore.Weighted -} +type ImagePruner = images.Pruner // NewImagePruner creates an ImagePruner object for a project. limiter may be // nil, meaning unbounded. func NewImagePruner(imageClient client.ImageAPIClient, project *types.Project, limiter *semaphore.Weighted) *ImagePruner { - return &ImagePruner{ - client: imageClient, - project: project, - limiter: limiter, - } + return images.NewPruner(imageClient, project, limiter) } -// imageBelongsToKnownService reports whether img should be pruned: either -// removeOrphans is set (indiscriminately prune all project images even if -// they're not referenced by the current Compose state, e.g. the service was -// removed from YAML), or the image belongs to a service still declared in -// project. Shared by ImagesToPrune (tagged images) and down's own -// dangling-image keep check, so the two can't drift on what counts as -// orphaned — the image-side analog of containers.go's isOrphaned. func imageBelongsToKnownService(project *types.Project, removeOrphans bool, img image.Summary) bool { - if removeOrphans { - return true - } - _, err := project.GetService(img.Labels[api.ServiceLabel]) - return err == nil + return images.ImageBelongsToKnownService(project, removeOrphans, img) } -// ImagesToPrune returns the set of images that should be removed. -func (p *ImagePruner) ImagesToPrune(ctx context.Context, opts ImagePruneOptions) ([]string, error) { - if opts.Mode == ImagePruneNone { - return nil, nil - } else if !opts.Mode.Valid() { - return nil, fmt.Errorf("unsupported image prune mode: %s", opts.Mode) - } - var images []string - - if opts.Mode == ImagePruneAll { - namedImages, err := p.namedImages(ctx) - if err != nil { - return nil, err - } - images = append(images, namedImages...) - } - - projectImages, err := p.labeledLocalImages(ctx) - if err != nil { - return nil, err - } - for _, img := range projectImages { - if len(img.RepoTags) == 0 { - // currently, we're only pruning the tagged references, but - // if we start removing the dangling images and grouping by - // service, we can remove this (and should rely on `Image::ID`) - continue - } - - if imageBelongsToKnownService(p.project, opts.RemoveOrphans, img) { - images = append(images, img.RepoTags[0]) - } - } - - fallbackImages, err := p.unlabeledLocalImages(ctx) - if err != nil { - return nil, err - } - images = append(images, fallbackImages...) - - images = normalizeAndDedupeImages(images) - return images, nil +func (s *composeService) removeImages(ctx context.Context, imgs []image.Summary, limiter *semaphore.Weighted) (removed, stillInUse []string, err error) { + return s.images().RemoveImages(ctx, imgs, limiter) } -// namedImages are those that are explicitly named in the service config. -// -// These could be registry-only images (no local build), hybrid (support build -// as a fallback if cannot pull), or local-only (image does not exist in a -// registry). -func (p *ImagePruner) namedImages(ctx context.Context) ([]string, error) { - var images []string - for _, service := range p.project.Services { - if service.Image == "" { - continue - } - images = append(images, service.Image) - } - return p.filterImagesByExistence(ctx, images) -} - -// labeledLocalImages are images that were locally-built by a current version of -// Compose (it did not always label built images). -// -// The image name could either have been defined by the user or implicitly -// created from the project + service name. -func (p *ImagePruner) labeledLocalImages(ctx context.Context) ([]image.Summary, error) { - res, err := p.client.ImageList(ctx, client.ImageListOptions{ - Filters: naming.ProjectFilter(p.project.Name).Add("dangling", "false"), - }) - if err != nil { - return nil, err - } - return res.Items, nil -} - -// removeImages removes the given images in parallel, tolerating individual -// failures so one bad image doesn't abort the rest. An image already gone -// (a benign race with something else removing it concurrently) or still in -// use is never joined into err — same tolerance removeResource already -// gives a tagged image in either situation (its own distinct, correctly -// worded event, not an error). "Still in use" is returned in its own bucket, -// not folded into removed, since a caller aggregating several images (see -// removeDanglingImagesOp) needs that distinction to report the batch -// honestly; an already-gone image reached the goal the same as an actual -// removal, so it's simply not counted against either bucket. Any other -// removal error is joined into err so callers reporting failures to the -// user keep the actual daemon error instead of just an image ID. -func (s *composeService) removeImages(ctx context.Context, images []image.Summary, limiter *semaphore.Weighted) (removed, stillInUse []string, err error) { - var mu sync.Mutex - var errs []error - // as in removeTaggedImagesOp, a single image failure must not cancel the - // shared errgroup ctx, so fn always returns nil and collects into the - // buckets below instead. - _ = concurrency.ForEach(ctx, limiter, images, func(ctx context.Context, img image.Summary) error { - if _, err := s.apiClient().ImageRemove(ctx, img.ID, client.ImageRemoveOptions{}); err != nil { - mu.Lock() - defer mu.Unlock() - imgStillInUse, alreadyGone := classifyRemovalError(err) - switch { - case alreadyGone: - // e.g. removed concurrently by something else - logrus.Debugf("dangling image %s already removed: %v", img.ID, err) - case imgStillInUse: - // the same benign skip the tagged-image path reports via - // removeResource's conflict branch - logrus.Debugf("dangling image %s still in use: %v", img.ID, err) - stillInUse = append(stillInUse, img.ID) - default: - errs = append(errs, fmt.Errorf("image %s: %w", img.ID, err)) - } - return nil - } - mu.Lock() - defer mu.Unlock() - removed = append(removed, img.ID) - return nil - }) - return removed, stillInUse, errors.Join(errs...) -} - -// eligibleDanglingImages lists a project's dangling images and filters out -// those spared by keep. Shared by down --rmi (which also needs the list -// itself, to decide whether there's anything to report) and watch --prune, -// so the two don't drift apart. -func (s *composeService) eligibleDanglingImages(ctx context.Context, projectName string, keep func(image.Summary) bool) ([]image.Summary, error) { - res, err := s.apiClient().ImageList(ctx, client.ImageListOptions{ - Filters: naming.ProjectFilter(projectName).Add("dangling", "true"), - }) - if err != nil { - return nil, err - } - - var eligible []image.Summary - for _, img := range res.Items { - if !keep(img) { - eligible = append(eligible, img) - } - } - return eligible, nil -} - -// removeDanglingImages lists a project's dangling images and removes those -// not spared by keep. Shared by down --rmi and watch --prune, which differ -// only in what keep spares. func (s *composeService) removeDanglingImages(ctx context.Context, projectName string, keep func(image.Summary) bool) ([]string, error) { - eligible, err := s.eligibleDanglingImages(ctx, projectName, keep) - if err != nil { - return nil, err - } - removed, _, err := s.removeImages(ctx, eligible, concurrency.NewLimiter(s.maxConcurrency)) - return removed, err + return s.images().RemoveDanglingImages(ctx, projectName, keep) } -// unlabeledLocalImages are images that match the implicit naming convention -// for locally-built images but did not get labeled, presumably because they -// were produced by an older version of Compose. -// -// This is transitional to ensure `down` continues to work as expected on -// projects built/launched by previous versions of Compose. It can safely -// be removed after some time. -func (p *ImagePruner) unlabeledLocalImages(ctx context.Context) ([]string, error) { - var images []string - for _, service := range p.project.Services { - if service.Image != "" { - continue - } - img := api.GetImageNameOrDefault(service, p.project.Name) - images = append(images, img) - } - return p.filterImagesByExistence(ctx, images) -} - -// filterImagesByExistence returns the subset of images that exist in the -// engine store. -// -// NOTE: Any transient errors communicating with the API will result in an -// image being returned as "existing", as this method is exclusively used to -// find images to remove, so the worst case of being conservative here is an -// attempt to remove an image that doesn't exist, which will cause a warning -// but is otherwise harmless. -func (p *ImagePruner) filterImagesByExistence(ctx context.Context, imageNames []string) ([]string, error) { - var mu sync.Mutex - var ret []string - - err := concurrency.ForEach(ctx, p.limiter, imageNames, func(ctx context.Context, img string) error { - _, err := p.client.ImageInspect(ctx, img) - if errdefs.IsNotFound(err) { - // err on the side of caution: only skip if we successfully - // queried the API and got back a definitive "not exists" - return nil - } - mu.Lock() - defer mu.Unlock() - ret = append(ret, img) - return nil - }) - if err != nil { - return nil, err - } - - return ret, nil -} - -// normalizeAndDedupeImages returns the unique set of images after normalization. -func normalizeAndDedupeImages(images []string) []string { - seen := make(map[string]struct{}, len(images)) - for _, img := range images { - // since some references come from user input (service.image) and some - // come from the engine API, we standardize them, opting for the - // familiar name format since they'll also be displayed in the CLI - ref, err := reference.ParseNormalizedNamed(img) - if err == nil { - ref = reference.TagNameOnly(ref) - img = reference.FamiliarString(ref) - } - seen[img] = struct{}{} - } - // ensure a deterministic return result - the actual ordering is not useful - return slices.Sorted(maps.Keys(seen)) +func (s *composeService) eligibleDanglingImages(ctx context.Context, projectName string, keep func(image.Summary) bool) ([]image.Summary, error) { + return s.images().EligibleDanglingImages(ctx, projectName, keep) } diff --git a/pkg/compose/images.go b/pkg/compose/images.go index 4dec52c9a..241fa5f4c 100644 --- a/pkg/compose/images.go +++ b/pkg/compose/images.go @@ -18,209 +18,51 @@ package compose import ( "context" - "fmt" - "strings" - "sync" - "time" "github.com/compose-spec/compose-go/v2/types" - "github.com/containerd/errdefs" - "github.com/containerd/platforms" - "github.com/distribution/reference" - "github.com/moby/moby/api/types/container" - "github.com/moby/moby/api/types/image" "github.com/moby/moby/client" - "github.com/moby/moby/client/pkg/versions" - godigest "github.com/opencontainers/go-digest" - specs "github.com/opencontainers/image-spec/specs-go/v1" "github.com/sirupsen/logrus" - "github.com/docker/compose/v5/internal/concurrency" - "github.com/docker/compose/v5/internal/naming" + "github.com/docker/compose/v5/internal/images" "github.com/docker/compose/v5/pkg/api" ) +// The images command, the identity of a local image and the removal of images +// live in internal/images: composeService serves them by handing that package +// the engine they act on. What follows are the forwarders the rest of this +// package still calls, and serviceImageDigest, which depends on the +// pinnedImageDigest type of build.go. + +func (s *composeService) images() *images.Service { + return images.New(imagesBackend{s}) +} + func (s *composeService) Images(ctx context.Context, projectName string, options api.ImagesOptions) (map[string]api.ImageSummary, error) { - projectName = strings.ToLower(projectName) - allContainers, err := s.apiClient().ContainerList(ctx, client.ContainerListOptions{ - All: true, - Filters: naming.ProjectFilter(projectName), - }) - if err != nil { - return nil, err - } - var containers []container.Summary - if len(options.Services) > 0 { - containers = naming.InServices(allContainers.Items, options.Services) - } else { - containers = allContainers.Items - } - - // The daemon validates the platform field in ImageInspect against the - // negotiated API version from the request path, not the server's own max version. - version, err := s.RuntimeAPIVersion(ctx) - if err != nil { - return nil, err - } - withPlatform := versions.GreaterThanOrEqualTo(version, apiVersion149) - - summary := map[string]api.ImageSummary{} - var mux sync.Mutex - eg, ctx := concurrency.NewErrgroup(ctx, s.maxConcurrency) - for _, ctr := range containers { - eg.Go(func() error { - img, err := s.containerImageSummary(ctx, ctr, withPlatform) - if err != nil { - return err - } - mux.Lock() - defer mux.Unlock() - summary[naming.CanonicalContainerName(ctr)] = img - return nil - }) - } - - err = eg.Wait() - return summary, err + return s.images().Images(ctx, projectName, options) } -// containerImageSummary describes the image a container was created from. The -// image record may not exist anymore: under the containerd image store the -// daemon drops the dangling old index when a rebuild with identical content -// moves the tag — without compose recreating the container (see #13636) — and -// `docker rmi -f` may remove the image of a running container. In that case -// report what the container itself knows rather than failing the whole -// listing (#14014). -func (s *composeService) containerImageSummary(ctx context.Context, c container.Summary, withPlatform bool) (api.ImageSummary, error) { - var repository, tag string - if _, err := godigest.Parse(c.Image); err != nil { - // c.Image is a reference, not the raw image ID the engine reports - // when the image record is gone or its tag was moved - if ref, err := reference.ParseDockerRef(c.Image); err == nil { - repository = reference.FamiliarName(ref) - if tagged, ok := ref.(reference.Tagged); ok { - tag = tagged.Tag() - } - } - } - - img, err := s.apiClient().ImageInspect(ctx, c.Image) - if errdefs.IsNotFound(err) { - return api.ImageSummary{ - ID: c.ImageID, - Repository: repository, - Tag: tag, - }, nil - } - if err != nil { - return api.ImageSummary{}, err - } - id := img.ID // platform-specific image ID can't be combined with image tag, see https://github.com/moby/moby/issues/49995 - - if withPlatform && c.ImageManifestDescriptor != nil && c.ImageManifestDescriptor.Platform != nil { - img, err = s.apiClient().ImageInspect(ctx, c.Image, client.ImageInspectWithPlatform(c.ImageManifestDescriptor.Platform)) - if err != nil { - return api.ImageSummary{}, err - } - } - - var created *time.Time - if img.Created != "" { - t, err := time.Parse(time.RFC3339Nano, img.Created) - if err != nil { - return api.ImageSummary{}, err - } - created = &t - } - - return api.ImageSummary{ - ID: id, - Repository: repository, - Tag: tag, - Platform: platforms.Platform{ - Architecture: img.Architecture, - OS: img.Os, - OSVersion: img.OsVersion, - Variant: img.Variant, - }, - Size: img.Size, - Created: created, - LastTagTime: img.Metadata.LastTagTime, - }, nil -} - -// inspectLocalImages inspects the given references in parallel, requesting -// per-manifest data on engines that support it. References not found locally -// are simply absent from the result. func (s *composeService) inspectLocalImages(ctx context.Context, repoTags []string) (map[string]client.ImageInspectResult, error) { - opts, err := s.imageInspectOptions(ctx) - if err != nil { - return nil, err - } - inspections := map[string]client.ImageInspectResult{} - l := sync.Mutex{} - eg, ctx := concurrency.NewErrgroup(ctx, s.maxConcurrency) - for _, repoTag := range repoTags { - eg.Go(func() error { - inspect, err := s.apiClient().ImageInspect(ctx, repoTag, opts...) - if err != nil { - if errdefs.IsNotFound(err) { - return nil - } - return fmt.Errorf("unable to get image '%s': %w", repoTag, err) - } - l.Lock() - inspections[repoTag] = inspect - l.Unlock() - return nil - }) - } - return inspections, eg.Wait() + return s.images().InspectLocalImages(ctx, repoTags) } -// imageInspectOptions requests per-manifest data when the engine supports it -// (see manifestsSupported). func (s *composeService) imageInspectOptions(ctx context.Context) ([]client.ImageInspectOption, error) { - withManifests, err := s.manifestsSupported(ctx) - if err != nil { - return nil, err - } - if !withManifests { - return nil, nil - } - return []client.ImageInspectOption{client.ImageInspectWithManifests(true)}, nil + return s.images().ImageInspectOptions(ctx) } func imageSummary(repoTag string, inspect client.ImageInspectResult) api.ImageSummary { - tag := "" - repository := "" - ref, err := reference.ParseDockerRef(repoTag) - if err == nil { - // ParseDockerRef will reject a local image ID - repository = reference.FamiliarName(ref) - if tagged, ok := ref.(reference.Tagged); ok { - tag = tagged.Tag() - } - } - id, _, _ := localContentDigest(inspect, "") - return api.ImageSummary{ - ID: id, - Repository: repository, - Tag: tag, - Size: inspect.Size, - LastTagTime: inspect.Metadata.LastTagTime, - } + return images.Summarize(repoTag, inspect) } -// manifestsSupported reports whether the engine can return per-manifest data on -// image inspect (Engine >= 28.0 / API >= 1.48). Older engines fall back to the -// plain image ID. -func (s *composeService) manifestsSupported(ctx context.Context) (bool, error) { - version, err := s.RuntimeAPIVersion(ctx) - if err != nil { - return false, err - } - return versions.GreaterThanOrEqualTo(version, apiVersion148), nil +func (s *composeService) canonicalBuiltDigest(ctx context.Context, imageRef, platform, builderDigest string) string { + return s.images().CanonicalBuiltDigest(ctx, imageRef, platform, builderDigest) +} + +func localContentDigest(inspect client.ImageInspectResult, platform string) (string, bool, error) { + return images.LocalContentDigest(inspect, platform) +} + +func (s *composeService) inspectLocalContent(ctx context.Context, ref string, platform string) (string, bool, error) { + return s.images().InspectLocalContent(ctx, ref, platform) } // serviceImageDigest returns the digest to record in a service's @@ -248,130 +90,23 @@ func (s *composeService) serviceImageDigest(ctx context.Context, service types.S return digest } -// canonicalBuiltDigest resolves the canonical content digest of a just-built -// image so the recorded identity matches what later runs compute for the same -// local image. Registry-only builds (push-only, multi-platform without load) -// are not locally inspectable and keep the builder-reported digest: a volatile -// but honest value — an actual rebuild is still detected — preferred over a -// stable marker that would hide real image changes. Best effort: never fails -// an already-successful build. -func (s *composeService) canonicalBuiltDigest(ctx context.Context, imageRef, platform, builderDigest string) string { - id, _, err := s.inspectLocalContent(ctx, imageRef, platform) - if err != nil { - logrus.Debugf("unable to resolve content digest for built image %s, keeping builder-reported digest: %v", imageRef, err) - return builderDigest - } - return id +// imagesBackend is the engine of a composeService, as internal/images sees it. +type imagesBackend struct { + s *composeService } -// localContentDigest returns the digest identifying an inspected image's -// runnable content (config + layers) for the requested platform (empty means -// the host default), plus whether the local image satisfies that platform. -// Note the satisfied bool is only meaningful for an explicitly requested -// platform: with platform empty, the manifests path reports host-default -// satisfaction while the manifest-less path always reports true. -// -// With BuildKit provenance attestations enabled (the default since recent -// Buildx/BuildKit), the image is stored as an index whose top-level digest -// (inspect.ID) also covers the attestation manifest, so it changes on every -// build even when the runnable image is unchanged — making compose recreate -// containers needlessly (see -// https://github.com/docker/compose/issues/13636). The digest of the "image" -// kind manifest reflects only the image content, which is what compose needs -// to detect staleness. Images inspected without manifest data (engines that -// can't report them) keep the plain image ID — already the config digest — -// with platform satisfaction from the inspect's flat platform fields, the -// pre-manifest behavior. Every image identity compose records for staleness -// comparison must be computed through here, whatever the image's provenance -// (pulled, built, already local), so any two runs produce comparable values. -// -// The converse also holds: this platform-specific digest is ONLY an identity -// for staleness comparison. Pinning image references in a reproducible -// compose model (publish / config --resolve-image-digests) must keep using -// the registry descriptor digest — the multi-platform index — through -// ImageDigestResolver, or the published file would be bound to the platform -// of whoever resolved it. -func localContentDigest(inspect client.ImageInspectResult, platform string) (string, bool, error) { - var matcher platforms.Matcher = platforms.Default() - pinned := platform != "" - if pinned { - p, err := platforms.Parse(platform) - if err != nil { - return "", false, err - } - matcher = platforms.NewMatcher(p) - } - if len(inspect.Manifests) > 0 { - id, ok := matchLocalManifest(inspect.InspectResponse, matcher) - return id, ok, nil - } - ok := true - if pinned { - ok = matcher.Match(specs.Platform{ - Architecture: inspect.Architecture, - OS: inspect.Os, - Variant: inspect.Variant, - }) - } - return inspect.ID, ok, nil +func (b imagesBackend) APIClient() client.APIClient { + return b.s.apiClient() } -// matchLocalManifest selects, among the locally available image manifests of -// inspect, the digest identifying the runnable content for the requested -// platform, and reports whether the local content actually satisfies that -// platform. Selection is platform-aware and deterministic so the same image -// always maps to the same digest across rebuilds: the available manifest -// matching the requested platform wins (satisfied); a lone available image -// manifest keeps its digest — single-platform images stay identifiable -// whatever their platform — without satisfying a different requested -// platform; otherwise fall back to inspect.ID, which never satisfies the -// request. -// -// Both the digest producers and the platform checks must go through this -// single selection so the digest recorded and the platform validated always -// refer to the same manifest. -func matchLocalManifest(inspect image.InspectResponse, platform platforms.Matcher) (string, bool) { - var available []image.ManifestSummary - for _, m := range inspect.Manifests { - if m.Kind == image.ManifestKindImage && m.Available { - available = append(available, m) - } - } - for _, m := range available { - if m.ImageData != nil && platform.Match(m.ImageData.Platform) { - return m.ID, true - } - } - if len(available) == 1 { - m := available[0] - if m.ImageData == nil { - // engines may omit per-manifest image data (seen with locally - // built, never-pushed images): fall back to the inspect's flat - // platform fields, like the manifest-less path does, instead of - // reporting the platform unsatisfied and triggering a pull of a - // possibly local-only image - return m.ID, platform.Match(specs.Platform{ - Architecture: inspect.Architecture, - OS: inspect.Os, - Variant: inspect.Variant, - }) - } - return m.ID, false - } - return inspect.ID, false +func (b imagesBackend) MaxConcurrency() int { + return b.s.maxConcurrency } -// inspectLocalContent inspects ref and returns its canonical content digest -// for the requested platform (empty means the host default), plus whether the -// local image satisfies that platform — see localContentDigest. -func (s *composeService) inspectLocalContent(ctx context.Context, ref string, platform string) (string, bool, error) { - opts, err := s.imageInspectOptions(ctx) - if err != nil { - return "", false, err - } - inspected, err := s.apiClient().ImageInspect(ctx, ref, opts...) - if err != nil { - return "", false, err - } - return localContentDigest(inspected, platform) +func (b imagesBackend) RuntimeAPIVersion(ctx context.Context) (string, error) { + return b.s.RuntimeAPIVersion(ctx) +} + +func (b imagesBackend) ClassifyRemovalError(err error) (stillInUse, alreadyGone bool) { + return classifyRemovalError(err) } diff --git a/pkg/compose/images_test.go b/pkg/compose/images_test.go index f7b7a2fb9..bef6cf740 100644 --- a/pkg/compose/images_test.go +++ b/pkg/compose/images_test.go @@ -17,132 +17,21 @@ package compose import ( - "net/netip" - "strings" "testing" - "time" "github.com/compose-spec/compose-go/v2/types" - "github.com/containerd/errdefs" - "github.com/containerd/platforms" "github.com/docker/cli/cli/config/configfile" - "github.com/moby/moby/api/types/container" "github.com/moby/moby/api/types/image" "github.com/moby/moby/client" specs "github.com/opencontainers/image-spec/specs-go/v1" "go.uber.org/mock/gomock" "gotest.tools/v3/assert" - "github.com/docker/compose/v5/internal/naming" compose "github.com/docker/compose/v5/pkg/api" "github.com/docker/compose/v5/pkg/mocks" ) -func TestImages(t *testing.T) { - mockCtrl := gomock.NewController(t) - defer mockCtrl.Finish() - - api, cli := prepareMocks(mockCtrl) - tested, err := NewComposeService(cli) - assert.NilError(t, err) - - args := naming.ProjectFilter(strings.ToLower(testProject)) - listOpts := client.ContainerListOptions{All: true, Filters: args} - api.EXPECT().Ping(gomock.Any(), client.PingOptions{NegotiateAPIVersion: true}).Return(client.PingResult{APIVersion: "1.96"}, nil).AnyTimes() - api.EXPECT().ClientVersion().Return("1.96").AnyTimes() - timeStr1 := "2025-06-06T06:06:06.000000000Z" - created1, _ := time.Parse(time.RFC3339Nano, timeStr1) - timeStr2 := "2025-03-03T03:03:03.000000000Z" - created2, _ := time.Parse(time.RFC3339Nano, timeStr2) - image1 := imageInspect("image1", "foo:1", 12345, timeStr1) - image2 := imageInspect("image2", "bar:2", 67890, timeStr2) - api.EXPECT().ImageInspect(anyCancellableContext(), "foo:1").Return(client.ImageInspectResult{InspectResponse: image1}, nil).MaxTimes(2) - api.EXPECT().ImageInspect(anyCancellableContext(), "bar:2").Return(client.ImageInspectResult{InspectResponse: image2}, nil) - c1 := containerDetail("service1", "123", container.StateRunning, "foo:1") - c2 := containerDetail("service1", "456", container.StateRunning, "bar:2") - c2.Ports = []container.PortSummary{{PublicPort: 80, PrivatePort: 90, IP: netip.MustParseAddr("127.0.0.1")}} - c3 := containerDetail("service2", "789", container.StateExited, "foo:1") - api.EXPECT().ContainerList(t.Context(), listOpts).Return(client.ContainerListResult{ - Items: []container.Summary{c1, c2, c3}, - }, nil) - - images, err := tested.Images(t.Context(), strings.ToLower(testProject), compose.ImagesOptions{}) - - expected := map[string]compose.ImageSummary{ - "123": { - ID: "image1", - Repository: "foo", - Tag: "1", - Size: 12345, - Created: &created1, - }, - "456": { - ID: "image2", - Repository: "bar", - Tag: "2", - Size: 67890, - Created: &created2, - }, - "789": { - ID: "image1", - Repository: "foo", - Tag: "1", - Size: 12345, - Created: &created1, - }, - } - assert.NilError(t, err) - assert.DeepEqual(t, images, expected) -} - -func TestImagesMissingImageRecord(t *testing.T) { - // A container may reference an image record that no longer exists: under - // the containerd image store, a rebuild with identical content moves the - // tag and the daemon drops the old index the running container was created - // from (https://github.com/docker/compose/issues/14014), and `docker rmi - // -f` may remove the image of a running container. The listing must - // degrade to what the container itself knows instead of failing. - mockCtrl := gomock.NewController(t) - defer mockCtrl.Finish() - - api, cli := prepareMocks(mockCtrl) - tested, err := NewComposeService(cli) - assert.NilError(t, err) - - args := naming.ProjectFilter(strings.ToLower(testProject)) - listOpts := client.ContainerListOptions{All: true, Filters: args} - api.EXPECT().Ping(gomock.Any(), client.PingOptions{NegotiateAPIVersion: true}).Return(client.PingResult{APIVersion: "1.96"}, nil).AnyTimes() - api.EXPECT().ClientVersion().Return("1.96").AnyTimes() - - // the raw image ID the engine reports once the image record is gone - const goneID = "sha256:52c159e21b703f2c750d0bfd8d0af93324b46c0df0859331fb9489b7ec033d0c" - gone := containerDetail("service1", "123", container.StateRunning, goneID) - gone.ImageID = goneID - api.EXPECT().ImageInspect(anyCancellableContext(), goneID). - Return(client.ImageInspectResult{}, errdefs.ErrNotFound) - api.EXPECT().ContainerList(t.Context(), listOpts).Return(client.ContainerListResult{ - Items: []container.Summary{gone}, - }, nil) - - images, err := tested.Images(t.Context(), strings.ToLower(testProject), compose.ImagesOptions{}) - assert.NilError(t, err) - assert.DeepEqual(t, images, map[string]compose.ImageSummary{ - "123": {ID: goneID}, - }) -} - -func imageInspect(id string, imageReference string, size int64, created string) image.InspectResponse { - return image.InspectResponse{ - ID: id, - RepoTags: []string{ - "someRepo:someTag", - imageReference, - }, - Size: size, - Created: created, - } -} - +//nolint:unparam // the tests of the engines without manifests moved to internal/images func imageManifest(id, arch string, available bool) image.ManifestSummary { return image.ManifestSummary{ ID: id, @@ -158,90 +47,7 @@ func attestationManifest() image.ManifestSummary { return image.ManifestSummary{ID: "sha256:att", Kind: image.ManifestKindAttestation, Available: true} } -func TestMatchLocalManifest(t *testing.T) { - amd64 := platforms.Only(specs.Platform{OS: "linux", Architecture: "amd64"}) - arm64 := platforms.Only(specs.Platform{OS: "linux", Architecture: "arm64"}) - - match := func(t *testing.T, inspect image.InspectResponse, platform platforms.Matcher, wantID string, wantOK bool) { - t.Helper() - id, ok := matchLocalManifest(inspect, platform) - assert.Equal(t, id, wantID) - assert.Equal(t, ok, wantOK) - } - - t.Run("no manifests falls back to the plain image ID, unsatisfied", func(t *testing.T) { - match(t, image.InspectResponse{ID: "sha256:top"}, amd64, "sha256:top", false) - }) - - t.Run("attested image ignores the attestation manifest", func(t *testing.T) { - // sha256:index changes on every build due to attestation metadata churn; - // the image-kind manifest digest is what actually reflects the content. - inspect := image.InspectResponse{ - ID: "sha256:index", - Manifests: []image.ManifestSummary{ - imageManifest("sha256:amd64", "amd64", true), - attestationManifest(), - }, - } - match(t, inspect, amd64, "sha256:amd64", true) - }) - - t.Run("single image manifest keeps its digest but does not satisfy another platform", func(t *testing.T) { - // single-platform image built for a non-host platform stays resolvable - inspect := image.InspectResponse{ - ID: "sha256:index", - Manifests: []image.ManifestSummary{imageManifest("sha256:arm64", "arm64", true)}, - } - match(t, inspect, amd64, "sha256:arm64", false) - match(t, inspect, arm64, "sha256:arm64", true) - }) - - t.Run("multi-platform picks the matching platform manifest", func(t *testing.T) { - inspect := image.InspectResponse{ - ID: "sha256:index", - Manifests: []image.ManifestSummary{ - imageManifest("sha256:amd64", "amd64", true), - imageManifest("sha256:arm64", "arm64", true), - attestationManifest(), - }, - } - match(t, inspect, amd64, "sha256:amd64", true) - match(t, inspect, arm64, "sha256:arm64", true) - }) - - t.Run("unavailable manifests are skipped", func(t *testing.T) { - // amd64 is referenced by the index but not pulled locally; only arm64 is usable - inspect := image.InspectResponse{ - ID: "sha256:index", - Manifests: []image.ManifestSummary{ - imageManifest("sha256:amd64", "amd64", false), - imageManifest("sha256:arm64", "arm64", true), - }, - } - match(t, inspect, amd64, "sha256:arm64", false) - }) - - t.Run("only attestation manifests falls back to the plain image ID", func(t *testing.T) { - inspect := image.InspectResponse{ - ID: "sha256:top", - Manifests: []image.ManifestSummary{attestationManifest()}, - } - match(t, inspect, amd64, "sha256:top", false) - }) - - t.Run("ambiguous multi-platform with no match falls back to the plain image ID", func(t *testing.T) { - inspect := image.InspectResponse{ - ID: "sha256:index", - Manifests: []image.ManifestSummary{ - imageManifest("sha256:amd64", "amd64", true), - imageManifest("sha256:arm64", "arm64", true), - }, - } - windows := platforms.Only(specs.Platform{OS: "windows", Architecture: "amd64"}) - match(t, inspect, windows, "sha256:index", false) - }) -} - +//nolint:unparam // the tests of older engines moved to internal/images func newTestComposeService(t *testing.T, mockCtrl *gomock.Controller, apiVersion string) (*mocks.MockAPIClient, *composeService) { t.Helper() api, cli := prepareMocks(mockCtrl) @@ -254,118 +60,6 @@ func newTestComposeService(t *testing.T, mockCtrl *gomock.Controller, apiVersion return api, tested.(*composeService) } -func TestInspectLocalImagesUsesContentDigest(t *testing.T) { - mockCtrl := gomock.NewController(t) - defer mockCtrl.Finish() - api, tested := newTestComposeService(t, mockCtrl, "1.48") - - inspect := image.InspectResponse{ - ID: "sha256:index", // attested index digest, churns every build - Manifests: []image.ManifestSummary{ - imageManifest("sha256:image", "amd64", true), - attestationManifest(), - }, - } - api.EXPECT(). - ImageInspect(anyCancellableContext(), "foo:1", gomock.Any()). - Return(client.ImageInspectResult{InspectResponse: inspect}, nil) - - inspections, err := tested.inspectLocalImages(t.Context(), []string{"foo:1"}) - assert.NilError(t, err) - assert.Equal(t, imageSummary("foo:1", inspections["foo:1"]).ID, "sha256:image") -} - -func TestInspectLocalImagesLegacyEngineUsesPlainID(t *testing.T) { - // Engine < 28.0 (API < 1.48) can't report manifests, so we keep the plain ID. - mockCtrl := gomock.NewController(t) - defer mockCtrl.Finish() - api, tested := newTestComposeService(t, mockCtrl, "1.47") - - inspect := image.InspectResponse{ID: "sha256:plain"} - api.EXPECT(). - ImageInspect(anyCancellableContext(), "foo:1"). - Return(client.ImageInspectResult{InspectResponse: inspect}, nil) - - inspections, err := tested.inspectLocalImages(t.Context(), []string{"foo:1"}) - assert.NilError(t, err) - assert.Equal(t, imageSummary("foo:1", inspections["foo:1"]).ID, "sha256:plain") -} - -func TestInspectLocalImagesSkipsMissingImages(t *testing.T) { - // Registry-only images (push/multi-platform) aren't inspectable locally; - // they must be omitted so the caller keeps the Bake-reported digest. - mockCtrl := gomock.NewController(t) - defer mockCtrl.Finish() - api, tested := newTestComposeService(t, mockCtrl, "1.48") - - api.EXPECT(). - ImageInspect(anyCancellableContext(), "missing:1", gomock.Any()). - Return(client.ImageInspectResult{}, errdefs.ErrNotFound) - - inspections, err := tested.inspectLocalImages(t.Context(), []string{"missing:1"}) - assert.NilError(t, err) - _, ok := inspections["missing:1"] - assert.Assert(t, !ok) -} - -func TestInspectLocalContent(t *testing.T) { - t.Run("manifests path selects the pinned platform", func(t *testing.T) { - mockCtrl := gomock.NewController(t) - defer mockCtrl.Finish() - api, tested := newTestComposeService(t, mockCtrl, "1.48") - api.EXPECT(). - ImageInspect(anyCancellableContext(), "foo:1", gomock.Any()). - Return(client.ImageInspectResult{InspectResponse: image.InspectResponse{ - ID: "sha256:index", - Manifests: []image.ManifestSummary{ - imageManifest("sha256:amd64", "amd64", true), - imageManifest("sha256:arm64", "arm64", true), - }, - }}, nil) - - id, ok, err := tested.inspectLocalContent(t.Context(), "foo:1", "linux/arm64") - assert.NilError(t, err) - assert.Equal(t, id, "sha256:arm64") - assert.Assert(t, ok) - }) - - t.Run("manifests path reports an unavailable pinned platform", func(t *testing.T) { - mockCtrl := gomock.NewController(t) - defer mockCtrl.Finish() - api, tested := newTestComposeService(t, mockCtrl, "1.48") - api.EXPECT(). - ImageInspect(anyCancellableContext(), "foo:1", gomock.Any()). - Return(client.ImageInspectResult{InspectResponse: image.InspectResponse{ - ID: "sha256:index", - Manifests: []image.ManifestSummary{imageManifest("sha256:amd64", "amd64", true)}, - }}, nil) - - _, ok, err := tested.inspectLocalContent(t.Context(), "foo:1", "linux/arm64") - assert.NilError(t, err) - assert.Assert(t, !ok) - }) - - t.Run("legacy engine falls back to flat platform fields", func(t *testing.T) { - mockCtrl := gomock.NewController(t) - defer mockCtrl.Finish() - api, tested := newTestComposeService(t, mockCtrl, "1.47") - api.EXPECT(). - ImageInspect(anyCancellableContext(), "foo:1"). - Return(client.ImageInspectResult{InspectResponse: image.InspectResponse{ - ID: "sha256:plain", Os: "linux", Architecture: "amd64", - }}, nil).Times(2) - - id, ok, err := tested.inspectLocalContent(t.Context(), "foo:1", "linux/amd64") - assert.NilError(t, err) - assert.Equal(t, id, "sha256:plain") - assert.Assert(t, ok) - - _, ok, err = tested.inspectLocalContent(t.Context(), "foo:1", "linux/arm64") - assert.NilError(t, err) - assert.Assert(t, !ok) - }) -} - // TestPlatformPinnedDigest covers two historic defects around // `platform:`-pinned services: // - the com.docker.compose.image label must hold the digest of the PINNED @@ -519,73 +213,3 @@ func TestServiceImageDigest(t *testing.T) { assert.Equal(t, got, "sha256:refreshed") }) } - -func TestCanonicalBuiltDigest(t *testing.T) { - t.Run("locally inspectable build resolves to the content digest", func(t *testing.T) { - mockCtrl := gomock.NewController(t) - defer mockCtrl.Finish() - api, tested := newTestComposeService(t, mockCtrl, "1.48") - api.EXPECT(). - ImageInspect(anyCancellableContext(), "built:1", gomock.Any()). - Return(client.ImageInspectResult{InspectResponse: image.InspectResponse{ - ID: "sha256:index", - Manifests: []image.ManifestSummary{ - imageManifest("sha256:image", "amd64", true), - attestationManifest(), - }, - }}, nil) - - got := tested.canonicalBuiltDigest(t.Context(), "built:1", "", "sha256:bakeindex") - assert.Equal(t, got, "sha256:image") - }) - - t.Run("registry-only build keeps the builder-reported digest", func(t *testing.T) { - // push-only / multi-platform-only builds never land in the local - // store: keep the builder digest — volatile but honest (a real - // rebuild is detected) rather than a stable marker hiding changes. - mockCtrl := gomock.NewController(t) - defer mockCtrl.Finish() - api, tested := newTestComposeService(t, mockCtrl, "1.48") - api.EXPECT(). - ImageInspect(anyCancellableContext(), "pushed:1", gomock.Any()). - Return(client.ImageInspectResult{}, errdefs.ErrNotFound) - - got := tested.canonicalBuiltDigest(t.Context(), "pushed:1", "", "sha256:bakeindex") - assert.Equal(t, got, "sha256:bakeindex") - }) -} - -func containerDetail(service string, id string, status container.ContainerState, imageName string) container.Summary { - return container.Summary{ - ID: id, - Names: []string{"/" + id}, - Image: imageName, - Labels: containerLabels(service, false), - State: status, - } -} - -func TestMatchLocalManifestWithoutImageData(t *testing.T) { - // engines may omit per-manifest image data (seen with locally built, - // never-pushed images): the lone available manifest falls back to the - // inspect's flat platform fields instead of reporting the platform - // unsatisfied — which made up try to pull a local-only image. - amd64 := platforms.OnlyStrict(specs.Platform{OS: "linux", Architecture: "amd64"}) - arm64 := platforms.OnlyStrict(specs.Platform{OS: "linux", Architecture: "arm64"}) - inspect := image.InspectResponse{ - ID: "sha256:index", - Os: "linux", - Architecture: "amd64", - Manifests: []image.ManifestSummary{ - {ID: "sha256:lone", Kind: image.ManifestKindImage, Available: true}, - }, - } - - id, satisfied := matchLocalManifest(inspect, amd64) - assert.Equal(t, id, "sha256:lone") - assert.Assert(t, satisfied, "flat platform fields match the requested platform") - - id, satisfied = matchLocalManifest(inspect, arm64) - assert.Equal(t, id, "sha256:lone") - assert.Assert(t, !satisfied, "flat platform fields don't match the requested platform") -}