compose/pkg/remote/oci_cache_entry_test.go
Nicolas De Loof 83c50295cb
Some checks are pending
ci / validate (lint) (push) Waiting to run
ci / validate (relay-lint) (push) Waiting to run
ci / validate (relay-test) (push) Waiting to run
ci / validate (validate-docs) (push) Waiting to run
ci / validate (validate-go-mod) (push) Waiting to run
ci / validate (validate-headers) (push) Waiting to run
ci / validate (validate-mocks) (push) Waiting to run
ci / binary (push) Waiting to run
ci / binary-finalize (push) Blocked by required conditions
ci / bin-image-test (push) Waiting to run
ci / relay-image-test (push) Waiting to run
ci / test (push) Waiting to run
ci / e2e (plugin, oldstable, graphdriver) (push) Waiting to run
ci / e2e (standalone, oldstable, graphdriver) (push) Waiting to run
ci / e2e (plugin, stable, containerd) (push) Waiting to run
ci / e2e (plugin, stable, graphdriver) (push) Waiting to run
ci / e2e (standalone, stable, containerd) (push) Waiting to run
ci / e2e (standalone, stable, graphdriver) (push) Waiting to run
ci / coverage (push) Blocked by required conditions
ci / ci-ok (push) Blocked by required conditions
ci / release (push) Blocked by required conditions
merge / bin-image-prepare (push) Waiting to run
merge / bin-image (push) Blocked by required conditions
merge / relay-image (push) Waiting to run
merge / module-image (push) Waiting to run
Scorecards supply-chain security / Scorecards analysis (push) Waiting to run
zizmor / zizmor (push) Waiting to run
refactor(oci): keep the extracted artifacts in an "oci" directory of the cache
The cache of remote resources now has one directory for each kind of resource:
the checkouts of Git repositories are in "git", and the extracted OCI artifacts
are in "oci" rather than at the root of the cache.

What earlier releases left at the root of the cache is no longer looked for: an
artifact is pulled again the first time it is used.

Signed-off-by: Nicolas De Loof <nicolas.deloof@gmail.com>
2026-11-09 14:38:36 +01:00

276 lines
11 KiB
Go

/*
Copyright 2020 Docker Compose CLI authors
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package remote
import (
"encoding/json"
"net/http"
"net/http/httptest"
"os"
"path/filepath"
"strconv"
"strings"
"sync"
"testing"
"github.com/docker/cli/cli/config/configfile"
"github.com/opencontainers/go-digest"
specs "github.com/opencontainers/image-spec/specs-go"
spec "github.com/opencontainers/image-spec/specs-go/v1"
"gotest.tools/v3/assert"
"github.com/docker/compose/v5/internal/oci"
)
// cacheTestRegistry serves a compose project artifact of a single compose file
// under any reference of test/artifact.
type cacheTestRegistry struct {
manifest []byte
digest digest.Digest
compose []byte
// what the blob endpoint serves, which is the compose file unless a test says otherwise
blob []byte
}
func newCacheTestRegistry(t *testing.T) *cacheTestRegistry {
t.Helper()
compose := []byte("services:\n app:\n image: alpine\n")
manifest, err := json.Marshal(spec.Manifest{
Versioned: specs.Versioned{SchemaVersion: 2},
MediaType: spec.MediaTypeImageManifest,
ArtifactType: oci.ComposeProjectArtifactType,
Config: spec.Descriptor{MediaType: oci.ComposeEmptyConfigMediaType, Digest: digest.FromString("{}"), Size: 2},
Layers: []spec.Descriptor{{MediaType: oci.ComposeYAMLMediaType, Digest: digest.FromBytes(compose), Size: int64(len(compose))}},
})
assert.NilError(t, err)
return &cacheTestRegistry{manifest: manifest, digest: digest.FromBytes(manifest), compose: compose, blob: compose}
}
// serve starts the registry and returns its host, which the loader of the test must be told is insecure.
func (r *cacheTestRegistry) serve(t *testing.T) string {
t.Helper()
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
switch {
case req.URL.Path == "/v2/":
case strings.HasPrefix(req.URL.Path, "/v2/test/artifact/manifests/"):
w.Header().Set("Content-Type", spec.MediaTypeImageManifest)
w.Header().Set("Docker-Content-Digest", r.digest.String())
w.Header().Set("Content-Length", strconv.Itoa(len(r.manifest))) // also for HEAD, which resolves
if req.Method != http.MethodHead {
_, _ = w.Write(r.manifest)
}
case req.URL.Path == "/v2/test/artifact/blobs/"+digest.FromBytes(r.compose).String():
_, _ = w.Write(r.blob)
default:
w.WriteHeader(http.StatusNotFound)
}
}))
t.Cleanup(server.Close)
return server.Listener.Addr().String()
}
func (r *cacheTestRegistry) ref(host string) string {
return OciPrefix + host + "/test/artifact@" + r.digest.String()
}
func newCacheTestLoader(host string) *ociRemoteLoader {
l := &ociRemoteLoader{known: map[string]string{}, insecureRegistries: []string{host}}
l.transportOnce.Do(func() { l.transport = &http.Transport{} })
l.credentialsOnce.Do(func() { l.credentials = &configfile.ConfigFile{} })
return l
}
// setCacheRoot points the cache of extracted projects to a new directory, which it returns.
func setCacheRoot(t *testing.T) string {
t.Helper()
root := t.TempDir()
t.Setenv("XDG_CACHE_HOME", root)
return filepath.Join(root, "docker-compose", cacheOCIDir)
}
// An entry of the cache of extracted projects is complete or is not there: it
// is extracted aside and moved in place in one step, so that a run never uses
// what an interrupted or a concurrent run is still writing, and it is marked
// as complete, so that a directory which is not one is not taken for an entry.
func TestOCILoaderCacheEntryIsComplete(t *testing.T) {
registry := newCacheTestRegistry(t)
host := registry.serve(t)
cache := setCacheRoot(t)
file, err := newCacheTestLoader(host).Load(t.Context(), registry.ref(host))
assert.NilError(t, err)
assert.Equal(t, file, filepath.Join(cache, registry.digest.Hex(), "compose.yaml"))
content, err := os.ReadFile(file)
assert.NilError(t, err)
assert.Equal(t, string(content), string(registry.compose))
entries, err := os.ReadDir(cache)
assert.NilError(t, err)
assert.Equal(t, len(entries), 1, "no directory is left aside")
_, err = os.Stat(filepath.Join(cache, registry.digest.Hex(), cacheEntryComplete))
assert.NilError(t, err)
t.Run("a complete entry is used as it is", func(t *testing.T) {
kept := filepath.Join(cache, registry.digest.Hex(), "kept")
assert.NilError(t, os.WriteFile(kept, nil, 0o600))
_, err := newCacheTestLoader(host).Load(t.Context(), registry.ref(host))
assert.NilError(t, err)
_, err = os.Stat(kept)
assert.NilError(t, err)
})
}
// A directory of the cache which is not marked as complete is extracted again,
// whatever it holds: an interrupted extraction leaves some of the files, and an
// entry written by a release which did not mark them looks like a complete one.
func TestOCILoaderExtractsAgainWhatIsNotACompleteEntry(t *testing.T) {
registry := newCacheTestRegistry(t)
host := registry.serve(t)
for name, compose := range map[string]string{
"an interrupted extraction": "services:\n app:\n ima",
"an entry of a release that did not mark them": string(registry.compose),
} {
t.Run(name, func(t *testing.T) {
cache := setCacheRoot(t)
entry := filepath.Join(cache, registry.digest.Hex())
assert.NilError(t, os.MkdirAll(entry, 0o700))
assert.NilError(t, os.WriteFile(filepath.Join(entry, "compose.yaml"), []byte(compose), 0o600))
assert.NilError(t, os.WriteFile(filepath.Join(entry, "stale"), nil, 0o600))
file, err := newCacheTestLoader(host).Load(t.Context(), registry.ref(host))
assert.NilError(t, err)
assert.Equal(t, file, filepath.Join(entry, "compose.yaml"))
content, err := os.ReadFile(file)
assert.NilError(t, err)
assert.Equal(t, string(content), string(registry.compose))
_, err = os.Stat(filepath.Join(entry, "stale"))
assert.Assert(t, os.IsNotExist(err), "the directory is replaced, not completed")
_, err = os.Stat(filepath.Join(entry, cacheEntryComplete))
assert.NilError(t, err)
})
}
}
// A pull that fails leaves nothing in the cache, neither an entry nor what
// it was extracting.
func TestOCILoaderFailedPullLeavesNothingBehind(t *testing.T) {
registry := newCacheTestRegistry(t)
registry.blob = []byte("services:\n app:\n image: alp1ne\n") // as long as the layer, not the same
host := registry.serve(t)
cache := setCacheRoot(t)
_, err := newCacheTestLoader(host).Load(t.Context(), registry.ref(host))
assert.ErrorContains(t, err, "digest mismatch")
entries, err := os.ReadDir(cache)
assert.Assert(t, err == nil || os.IsNotExist(err))
assert.Equal(t, len(entries), 0)
}
// Runs that load the same artifact at the same time, whether the cache has the
// entry already, has none or has a directory that is not one, leave a single
// complete entry, and nothing aside. A run that loses to another while it
// replaces a directory which is not an entry may return an error, so what is
// pinned for every run is the path of the entry when it succeeds, and the state
// of the cache once the runs are over.
func TestOCILoaderConcurrentRunsLeaveOneCompleteEntry(t *testing.T) {
registry := newCacheTestRegistry(t)
host := registry.serve(t)
for name, seed := range map[string]func(t *testing.T, entry string){
"no entry": func(*testing.T, string) {},
"a directory that is not an entry": func(t *testing.T, entry string) {
assert.NilError(t, os.MkdirAll(entry, 0o700))
assert.NilError(t, os.WriteFile(filepath.Join(entry, "compose.yaml"), []byte("services:\n app:\n ima"), 0o600))
assert.NilError(t, os.WriteFile(filepath.Join(entry, "stale"), nil, 0o600))
},
} {
t.Run(name, func(t *testing.T) {
cache := setCacheRoot(t)
entry := filepath.Join(cache, registry.digest.Hex())
seed(t, entry)
const runs = 8
files := make([]string, runs)
errs := make([]error, runs)
var wg sync.WaitGroup
for i := range runs {
wg.Add(1)
go func() {
defer wg.Done()
files[i], errs[i] = newCacheTestLoader(host).Load(t.Context(), registry.ref(host))
}()
}
wg.Wait()
for i := range runs {
if errs[i] == nil {
assert.Equal(t, files[i], filepath.Join(entry, "compose.yaml"))
}
}
content, err := os.ReadFile(filepath.Join(entry, "compose.yaml"))
assert.NilError(t, err)
assert.Equal(t, string(content), string(registry.compose))
assert.Assert(t, cacheEntryIsComplete(entry))
_, err = os.Stat(filepath.Join(entry, "stale"))
assert.Assert(t, os.IsNotExist(err))
entries, err := os.ReadDir(cache)
assert.NilError(t, err)
assert.Equal(t, len(entries), 1, "no directory is left aside")
})
}
}
// A directory which is not an entry that is replaced, and whose replacement
// then fails, is not put back: only a complete entry is worth restoring. Put
// back, it would be in the way of the run that set aside the entry which took
// its place (see moveCacheEntry), and would leave the cache with it instead.
func TestMoveCacheEntryDoesNotRestoreADirectoryWhichIsNotAnEntry(t *testing.T) {
cache := t.TempDir()
local := filepath.Join(cache, "entry")
assert.NilError(t, os.MkdirAll(local, 0o700))
assert.NilError(t, os.WriteFile(filepath.Join(local, "compose.yaml"), []byte("services:\n app:\n ima"), 0o600))
err := moveCacheEntry(filepath.Join(cache, "missing"), local)
assert.Assert(t, err != nil)
entries, err := os.ReadDir(cache)
assert.NilError(t, err)
assert.Equal(t, len(entries), 0, "neither the directory nor anything aside is left")
}
// The extracted artifacts are kept in a directory of the cache for OCI artifacts: an
// entry, even a complete one, which an earlier release left at the root of the
// cache is not looked for, and the artifact is extracted again in the directory.
func TestOCILoaderKeepsEntriesInTheDirectoryForOCI(t *testing.T) {
registry := newCacheTestRegistry(t)
host := registry.serve(t)
cache := setCacheRoot(t)
legacy := filepath.Join(filepath.Dir(cache), registry.digest.Hex())
assert.NilError(t, os.MkdirAll(legacy, 0o700))
assert.NilError(t, os.WriteFile(filepath.Join(legacy, "compose.yaml"), []byte("services: {}\n"), 0o600))
assert.NilError(t, os.WriteFile(filepath.Join(legacy, cacheEntryComplete), nil, 0o600))
file, err := newCacheTestLoader(host).Load(t.Context(), registry.ref(host))
assert.NilError(t, err)
assert.Equal(t, file, filepath.Join(cache, registry.digest.Hex(), "compose.yaml"))
content, err := os.ReadFile(file)
assert.NilError(t, err)
assert.Equal(t, string(content), string(registry.compose))
}