mirror of
https://github.com/docker/compose.git
synced 2026-11-17 21:57:19 +00:00
Some checks are pending
ci / validate (lint) (push) Waiting to run
ci / validate (relay-lint) (push) Waiting to run
ci / validate (relay-test) (push) Waiting to run
ci / validate (validate-docs) (push) Waiting to run
ci / validate (validate-go-mod) (push) Waiting to run
ci / validate (validate-headers) (push) Waiting to run
ci / validate (validate-mocks) (push) Waiting to run
ci / binary (push) Waiting to run
ci / binary-finalize (push) Blocked by required conditions
ci / bin-image-test (push) Waiting to run
ci / relay-image-test (push) Waiting to run
ci / test (push) Waiting to run
ci / e2e (plugin, oldstable, graphdriver) (push) Waiting to run
ci / e2e (standalone, oldstable, graphdriver) (push) Waiting to run
ci / e2e (plugin, stable, containerd) (push) Waiting to run
ci / e2e (plugin, stable, graphdriver) (push) Waiting to run
ci / e2e (standalone, stable, containerd) (push) Waiting to run
ci / e2e (standalone, stable, graphdriver) (push) Waiting to run
ci / coverage (push) Blocked by required conditions
ci / ci-ok (push) Blocked by required conditions
ci / release (push) Blocked by required conditions
merge / bin-image-prepare (push) Waiting to run
merge / bin-image (push) Blocked by required conditions
merge / relay-image (push) Waiting to run
merge / module-image (push) Waiting to run
Scorecards supply-chain security / Scorecards analysis (push) Waiting to run
zizmor / zizmor (push) Waiting to run
The cache of remote resources now has one directory for each kind of resource: the checkouts of Git repositories are in "git", and the extracted OCI artifacts are in "oci" rather than at the root of the cache. What earlier releases left at the root of the cache is no longer looked for: an artifact is pulled again the first time it is used. Signed-off-by: Nicolas De Loof <nicolas.deloof@gmail.com>
276 lines
11 KiB
Go
276 lines
11 KiB
Go
/*
|
|
Copyright 2020 Docker Compose CLI authors
|
|
|
|
Licensed under the Apache License, Version 2.0 (the "License");
|
|
you may not use this file except in compliance with the License.
|
|
You may obtain a copy of the License at
|
|
|
|
http://www.apache.org/licenses/LICENSE-2.0
|
|
|
|
Unless required by applicable law or agreed to in writing, software
|
|
distributed under the License is distributed on an "AS IS" BASIS,
|
|
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
See the License for the specific language governing permissions and
|
|
limitations under the License.
|
|
*/
|
|
|
|
package remote
|
|
|
|
import (
|
|
"encoding/json"
|
|
"net/http"
|
|
"net/http/httptest"
|
|
"os"
|
|
"path/filepath"
|
|
"strconv"
|
|
"strings"
|
|
"sync"
|
|
"testing"
|
|
|
|
"github.com/docker/cli/cli/config/configfile"
|
|
"github.com/opencontainers/go-digest"
|
|
specs "github.com/opencontainers/image-spec/specs-go"
|
|
spec "github.com/opencontainers/image-spec/specs-go/v1"
|
|
"gotest.tools/v3/assert"
|
|
|
|
"github.com/docker/compose/v5/internal/oci"
|
|
)
|
|
|
|
// cacheTestRegistry serves a compose project artifact of a single compose file
|
|
// under any reference of test/artifact.
|
|
type cacheTestRegistry struct {
|
|
manifest []byte
|
|
digest digest.Digest
|
|
compose []byte
|
|
// what the blob endpoint serves, which is the compose file unless a test says otherwise
|
|
blob []byte
|
|
}
|
|
|
|
func newCacheTestRegistry(t *testing.T) *cacheTestRegistry {
|
|
t.Helper()
|
|
compose := []byte("services:\n app:\n image: alpine\n")
|
|
manifest, err := json.Marshal(spec.Manifest{
|
|
Versioned: specs.Versioned{SchemaVersion: 2},
|
|
MediaType: spec.MediaTypeImageManifest,
|
|
ArtifactType: oci.ComposeProjectArtifactType,
|
|
Config: spec.Descriptor{MediaType: oci.ComposeEmptyConfigMediaType, Digest: digest.FromString("{}"), Size: 2},
|
|
Layers: []spec.Descriptor{{MediaType: oci.ComposeYAMLMediaType, Digest: digest.FromBytes(compose), Size: int64(len(compose))}},
|
|
})
|
|
assert.NilError(t, err)
|
|
return &cacheTestRegistry{manifest: manifest, digest: digest.FromBytes(manifest), compose: compose, blob: compose}
|
|
}
|
|
|
|
// serve starts the registry and returns its host, which the loader of the test must be told is insecure.
|
|
func (r *cacheTestRegistry) serve(t *testing.T) string {
|
|
t.Helper()
|
|
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
|
|
switch {
|
|
case req.URL.Path == "/v2/":
|
|
case strings.HasPrefix(req.URL.Path, "/v2/test/artifact/manifests/"):
|
|
w.Header().Set("Content-Type", spec.MediaTypeImageManifest)
|
|
w.Header().Set("Docker-Content-Digest", r.digest.String())
|
|
w.Header().Set("Content-Length", strconv.Itoa(len(r.manifest))) // also for HEAD, which resolves
|
|
if req.Method != http.MethodHead {
|
|
_, _ = w.Write(r.manifest)
|
|
}
|
|
case req.URL.Path == "/v2/test/artifact/blobs/"+digest.FromBytes(r.compose).String():
|
|
_, _ = w.Write(r.blob)
|
|
default:
|
|
w.WriteHeader(http.StatusNotFound)
|
|
}
|
|
}))
|
|
t.Cleanup(server.Close)
|
|
return server.Listener.Addr().String()
|
|
}
|
|
|
|
func (r *cacheTestRegistry) ref(host string) string {
|
|
return OciPrefix + host + "/test/artifact@" + r.digest.String()
|
|
}
|
|
|
|
func newCacheTestLoader(host string) *ociRemoteLoader {
|
|
l := &ociRemoteLoader{known: map[string]string{}, insecureRegistries: []string{host}}
|
|
l.transportOnce.Do(func() { l.transport = &http.Transport{} })
|
|
l.credentialsOnce.Do(func() { l.credentials = &configfile.ConfigFile{} })
|
|
return l
|
|
}
|
|
|
|
// setCacheRoot points the cache of extracted projects to a new directory, which it returns.
|
|
func setCacheRoot(t *testing.T) string {
|
|
t.Helper()
|
|
root := t.TempDir()
|
|
t.Setenv("XDG_CACHE_HOME", root)
|
|
return filepath.Join(root, "docker-compose", cacheOCIDir)
|
|
}
|
|
|
|
// An entry of the cache of extracted projects is complete or is not there: it
|
|
// is extracted aside and moved in place in one step, so that a run never uses
|
|
// what an interrupted or a concurrent run is still writing, and it is marked
|
|
// as complete, so that a directory which is not one is not taken for an entry.
|
|
func TestOCILoaderCacheEntryIsComplete(t *testing.T) {
|
|
registry := newCacheTestRegistry(t)
|
|
host := registry.serve(t)
|
|
cache := setCacheRoot(t)
|
|
|
|
file, err := newCacheTestLoader(host).Load(t.Context(), registry.ref(host))
|
|
assert.NilError(t, err)
|
|
assert.Equal(t, file, filepath.Join(cache, registry.digest.Hex(), "compose.yaml"))
|
|
content, err := os.ReadFile(file)
|
|
assert.NilError(t, err)
|
|
assert.Equal(t, string(content), string(registry.compose))
|
|
|
|
entries, err := os.ReadDir(cache)
|
|
assert.NilError(t, err)
|
|
assert.Equal(t, len(entries), 1, "no directory is left aside")
|
|
_, err = os.Stat(filepath.Join(cache, registry.digest.Hex(), cacheEntryComplete))
|
|
assert.NilError(t, err)
|
|
|
|
t.Run("a complete entry is used as it is", func(t *testing.T) {
|
|
kept := filepath.Join(cache, registry.digest.Hex(), "kept")
|
|
assert.NilError(t, os.WriteFile(kept, nil, 0o600))
|
|
_, err := newCacheTestLoader(host).Load(t.Context(), registry.ref(host))
|
|
assert.NilError(t, err)
|
|
_, err = os.Stat(kept)
|
|
assert.NilError(t, err)
|
|
})
|
|
}
|
|
|
|
// A directory of the cache which is not marked as complete is extracted again,
|
|
// whatever it holds: an interrupted extraction leaves some of the files, and an
|
|
// entry written by a release which did not mark them looks like a complete one.
|
|
func TestOCILoaderExtractsAgainWhatIsNotACompleteEntry(t *testing.T) {
|
|
registry := newCacheTestRegistry(t)
|
|
host := registry.serve(t)
|
|
|
|
for name, compose := range map[string]string{
|
|
"an interrupted extraction": "services:\n app:\n ima",
|
|
"an entry of a release that did not mark them": string(registry.compose),
|
|
} {
|
|
t.Run(name, func(t *testing.T) {
|
|
cache := setCacheRoot(t)
|
|
entry := filepath.Join(cache, registry.digest.Hex())
|
|
assert.NilError(t, os.MkdirAll(entry, 0o700))
|
|
assert.NilError(t, os.WriteFile(filepath.Join(entry, "compose.yaml"), []byte(compose), 0o600))
|
|
assert.NilError(t, os.WriteFile(filepath.Join(entry, "stale"), nil, 0o600))
|
|
|
|
file, err := newCacheTestLoader(host).Load(t.Context(), registry.ref(host))
|
|
assert.NilError(t, err)
|
|
assert.Equal(t, file, filepath.Join(entry, "compose.yaml"))
|
|
content, err := os.ReadFile(file)
|
|
assert.NilError(t, err)
|
|
assert.Equal(t, string(content), string(registry.compose))
|
|
_, err = os.Stat(filepath.Join(entry, "stale"))
|
|
assert.Assert(t, os.IsNotExist(err), "the directory is replaced, not completed")
|
|
_, err = os.Stat(filepath.Join(entry, cacheEntryComplete))
|
|
assert.NilError(t, err)
|
|
})
|
|
}
|
|
}
|
|
|
|
// A pull that fails leaves nothing in the cache, neither an entry nor what
|
|
// it was extracting.
|
|
func TestOCILoaderFailedPullLeavesNothingBehind(t *testing.T) {
|
|
registry := newCacheTestRegistry(t)
|
|
registry.blob = []byte("services:\n app:\n image: alp1ne\n") // as long as the layer, not the same
|
|
host := registry.serve(t)
|
|
cache := setCacheRoot(t)
|
|
|
|
_, err := newCacheTestLoader(host).Load(t.Context(), registry.ref(host))
|
|
assert.ErrorContains(t, err, "digest mismatch")
|
|
|
|
entries, err := os.ReadDir(cache)
|
|
assert.Assert(t, err == nil || os.IsNotExist(err))
|
|
assert.Equal(t, len(entries), 0)
|
|
}
|
|
|
|
// Runs that load the same artifact at the same time, whether the cache has the
|
|
// entry already, has none or has a directory that is not one, leave a single
|
|
// complete entry, and nothing aside. A run that loses to another while it
|
|
// replaces a directory which is not an entry may return an error, so what is
|
|
// pinned for every run is the path of the entry when it succeeds, and the state
|
|
// of the cache once the runs are over.
|
|
func TestOCILoaderConcurrentRunsLeaveOneCompleteEntry(t *testing.T) {
|
|
registry := newCacheTestRegistry(t)
|
|
host := registry.serve(t)
|
|
|
|
for name, seed := range map[string]func(t *testing.T, entry string){
|
|
"no entry": func(*testing.T, string) {},
|
|
"a directory that is not an entry": func(t *testing.T, entry string) {
|
|
assert.NilError(t, os.MkdirAll(entry, 0o700))
|
|
assert.NilError(t, os.WriteFile(filepath.Join(entry, "compose.yaml"), []byte("services:\n app:\n ima"), 0o600))
|
|
assert.NilError(t, os.WriteFile(filepath.Join(entry, "stale"), nil, 0o600))
|
|
},
|
|
} {
|
|
t.Run(name, func(t *testing.T) {
|
|
cache := setCacheRoot(t)
|
|
entry := filepath.Join(cache, registry.digest.Hex())
|
|
seed(t, entry)
|
|
|
|
const runs = 8
|
|
files := make([]string, runs)
|
|
errs := make([]error, runs)
|
|
var wg sync.WaitGroup
|
|
for i := range runs {
|
|
wg.Add(1)
|
|
go func() {
|
|
defer wg.Done()
|
|
files[i], errs[i] = newCacheTestLoader(host).Load(t.Context(), registry.ref(host))
|
|
}()
|
|
}
|
|
wg.Wait()
|
|
|
|
for i := range runs {
|
|
if errs[i] == nil {
|
|
assert.Equal(t, files[i], filepath.Join(entry, "compose.yaml"))
|
|
}
|
|
}
|
|
content, err := os.ReadFile(filepath.Join(entry, "compose.yaml"))
|
|
assert.NilError(t, err)
|
|
assert.Equal(t, string(content), string(registry.compose))
|
|
assert.Assert(t, cacheEntryIsComplete(entry))
|
|
_, err = os.Stat(filepath.Join(entry, "stale"))
|
|
assert.Assert(t, os.IsNotExist(err))
|
|
entries, err := os.ReadDir(cache)
|
|
assert.NilError(t, err)
|
|
assert.Equal(t, len(entries), 1, "no directory is left aside")
|
|
})
|
|
}
|
|
}
|
|
|
|
// A directory which is not an entry that is replaced, and whose replacement
|
|
// then fails, is not put back: only a complete entry is worth restoring. Put
|
|
// back, it would be in the way of the run that set aside the entry which took
|
|
// its place (see moveCacheEntry), and would leave the cache with it instead.
|
|
func TestMoveCacheEntryDoesNotRestoreADirectoryWhichIsNotAnEntry(t *testing.T) {
|
|
cache := t.TempDir()
|
|
local := filepath.Join(cache, "entry")
|
|
assert.NilError(t, os.MkdirAll(local, 0o700))
|
|
assert.NilError(t, os.WriteFile(filepath.Join(local, "compose.yaml"), []byte("services:\n app:\n ima"), 0o600))
|
|
|
|
err := moveCacheEntry(filepath.Join(cache, "missing"), local)
|
|
assert.Assert(t, err != nil)
|
|
|
|
entries, err := os.ReadDir(cache)
|
|
assert.NilError(t, err)
|
|
assert.Equal(t, len(entries), 0, "neither the directory nor anything aside is left")
|
|
}
|
|
|
|
// The extracted artifacts are kept in a directory of the cache for OCI artifacts: an
|
|
// entry, even a complete one, which an earlier release left at the root of the
|
|
// cache is not looked for, and the artifact is extracted again in the directory.
|
|
func TestOCILoaderKeepsEntriesInTheDirectoryForOCI(t *testing.T) {
|
|
registry := newCacheTestRegistry(t)
|
|
host := registry.serve(t)
|
|
cache := setCacheRoot(t)
|
|
legacy := filepath.Join(filepath.Dir(cache), registry.digest.Hex())
|
|
assert.NilError(t, os.MkdirAll(legacy, 0o700))
|
|
assert.NilError(t, os.WriteFile(filepath.Join(legacy, "compose.yaml"), []byte("services: {}\n"), 0o600))
|
|
assert.NilError(t, os.WriteFile(filepath.Join(legacy, cacheEntryComplete), nil, 0o600))
|
|
|
|
file, err := newCacheTestLoader(host).Load(t.Context(), registry.ref(host))
|
|
assert.NilError(t, err)
|
|
|
|
assert.Equal(t, file, filepath.Join(cache, registry.digest.Hex(), "compose.yaml"))
|
|
content, err := os.ReadFile(file)
|
|
assert.NilError(t, err)
|
|
assert.Equal(t, string(content), string(registry.compose))
|
|
}
|