mirror of
https://github.com/docker/compose.git
synced 2026-11-17 13:45:39 +00:00
fix(git): check a commit out aside, and keep the checkouts in a directory of the cache for Git
A commit was checked out directly into its directory of the cache, which was then taken for complete as soon as it existed: a run interrupted during the fetch or the checkout left a partial directory that every later run used. The checkout is now made in a directory of its own and renamed to the one of the commit once it is complete, so the directory of a commit is never a partial checkout, and no file beside it is needed to tell. Checkouts live in a "git" directory of the cache, and what earlier releases left at the root of the cache is no longer looked for. Signed-off-by: Nicolas De Loof <nicolas.deloof@gmail.com>
This commit is contained in:
parent
95bbca7582
commit
b5709cd309
2 changed files with 183 additions and 4 deletions
|
|
@ -102,12 +102,12 @@ func (g gitRemoteLoader) Load(ctx context.Context, path string) (string, error)
|
|||
return "", fmt.Errorf("initializing remote resource cache: %w", err)
|
||||
}
|
||||
|
||||
local = filepath.Join(cache, ref.Ref)
|
||||
local = gitEntryDir(cache, ref.Ref)
|
||||
if _, err := os.Stat(local); os.IsNotExist(err) {
|
||||
if g.offline {
|
||||
return "", nil
|
||||
}
|
||||
err = g.checkout(ctx, local, ref)
|
||||
err = g.checkout(ctx, cache, ref)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
|
@ -192,11 +192,32 @@ func (g gitRemoteLoader) resolveGitRef(ctx context.Context, path string, ref *gi
|
|||
return nil
|
||||
}
|
||||
|
||||
func (g gitRemoteLoader) checkout(ctx context.Context, path string, ref *gitutil.GitRef) error {
|
||||
err := os.MkdirAll(path, 0o700)
|
||||
// cacheGitDir is the directory of the cache which holds the checkouts of the Git
|
||||
// repositories, one directory for each commit. Earlier releases checked them out at
|
||||
// the root of the cache, where they are no longer looked for: one which was left
|
||||
// incomplete by an interrupted run could be taken for a complete one.
|
||||
const cacheGitDir = "git"
|
||||
|
||||
// gitEntryDir is the directory of the checkout of the commit.
|
||||
func gitEntryDir(cache, sha string) string {
|
||||
return filepath.Join(cache, cacheGitDir, sha)
|
||||
}
|
||||
|
||||
// checkout fetches the commit into the cache. It does so in a directory of its own,
|
||||
// which is renamed to the one of the commit only once the checkout is complete: the
|
||||
// directory of a commit is therefore never a partial checkout, and needs no file
|
||||
// beside it to tell.
|
||||
func (g gitRemoteLoader) checkout(ctx context.Context, cache string, ref *gitutil.GitRef) error {
|
||||
gitCache := filepath.Join(cache, cacheGitDir)
|
||||
if err := os.MkdirAll(gitCache, 0o700); err != nil {
|
||||
return err
|
||||
}
|
||||
path, err := os.MkdirTemp(gitCache, ".checkout-")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer func() { _ = os.RemoveAll(path) }()
|
||||
|
||||
err = exec.CommandContext(ctx, "git", "init", path).Run()
|
||||
if err != nil {
|
||||
return err
|
||||
|
|
@ -224,6 +245,15 @@ func (g gitRemoteLoader) checkout(ctx context.Context, path string, ref *gitutil
|
|||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
local := gitEntryDir(cache, ref.Ref)
|
||||
if err := os.Rename(path, local); err != nil {
|
||||
// another run placed the same commit first
|
||||
if _, statErr := os.Stat(local); statErr == nil {
|
||||
return nil
|
||||
}
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
|
|
|
|||
149
pkg/remote/git_cache_test.go
Normal file
149
pkg/remote/git_cache_test.go
Normal file
|
|
@ -0,0 +1,149 @@
|
|||
/*
|
||||
Copyright 2020 Docker Compose CLI authors
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
package remote
|
||||
|
||||
import (
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
|
||||
gitutil "github.com/moby/buildkit/frontend/dockerfile/dfgitutil"
|
||||
"gotest.tools/v3/assert"
|
||||
)
|
||||
|
||||
// newGitRepository makes a repository of one commit holding a compose file, and
|
||||
// returns its path and the commit.
|
||||
func newGitRepository(t *testing.T) (string, string) {
|
||||
t.Helper()
|
||||
if _, err := exec.LookPath("git"); err != nil {
|
||||
t.Skip("git is not installed")
|
||||
}
|
||||
// keep the configuration of the user, which may sign commits, out of the test
|
||||
t.Setenv("GIT_CONFIG_GLOBAL", os.DevNull)
|
||||
t.Setenv("GIT_CONFIG_SYSTEM", os.DevNull)
|
||||
|
||||
repo := t.TempDir()
|
||||
assert.NilError(t, os.WriteFile(filepath.Join(repo, "compose.yaml"), []byte("services:\n app:\n image: alpine\n"), 0o600))
|
||||
for _, args := range [][]string{
|
||||
{"init", "-q"},
|
||||
{"add", "compose.yaml"},
|
||||
{"-c", "user.name=test", "-c", "user.email=test@example.com", "commit", "-q", "-m", "compose file"},
|
||||
} {
|
||||
cmd := exec.Command("git", args...)
|
||||
cmd.Dir = repo
|
||||
out, err := cmd.CombinedOutput()
|
||||
assert.NilError(t, err, string(out))
|
||||
}
|
||||
cmd := exec.Command("git", "rev-parse", "HEAD")
|
||||
cmd.Dir = repo
|
||||
out, err := cmd.Output()
|
||||
assert.NilError(t, err)
|
||||
return repo, strings.TrimSpace(string(out))
|
||||
}
|
||||
|
||||
func gitCacheNames(t *testing.T, cache string) []string {
|
||||
t.Helper()
|
||||
entries, err := os.ReadDir(filepath.Join(cache, cacheGitDir))
|
||||
if os.IsNotExist(err) {
|
||||
return nil
|
||||
}
|
||||
assert.NilError(t, err)
|
||||
names := make([]string, 0, len(entries))
|
||||
for _, e := range entries {
|
||||
names = append(names, e.Name())
|
||||
}
|
||||
return names
|
||||
}
|
||||
|
||||
// A checkout is put in the cache directory of Git checkouts, named after the commit,
|
||||
// only once it is complete: what is left beside it is the entry and nothing else.
|
||||
func TestGitCheckoutIsPlacedWhenComplete(t *testing.T) {
|
||||
repo, sha := newGitRepository(t)
|
||||
cache := t.TempDir()
|
||||
|
||||
err := gitRemoteLoader{}.checkout(t.Context(), cache, &gitutil.GitRef{Remote: repo, Ref: sha})
|
||||
assert.NilError(t, err)
|
||||
|
||||
assert.DeepEqual(t, gitCacheNames(t, cache), []string{sha})
|
||||
_, err = os.Stat(filepath.Join(gitEntryDir(cache, sha), "compose.yaml"))
|
||||
assert.NilError(t, err)
|
||||
}
|
||||
|
||||
// A checkout which fails leaves nothing in the cache: no entry for the commit,
|
||||
// and no directory of the attempt.
|
||||
func TestGitCheckoutWhichFailsLeavesNothing(t *testing.T) {
|
||||
_, sha := newGitRepository(t)
|
||||
cache := t.TempDir()
|
||||
|
||||
err := gitRemoteLoader{}.checkout(t.Context(), cache, &gitutil.GitRef{Remote: filepath.Join(cache, "missing"), Ref: sha})
|
||||
assert.Assert(t, err != nil)
|
||||
|
||||
assert.Equal(t, len(gitCacheNames(t, cache)), 0)
|
||||
}
|
||||
|
||||
// Runs which check out the same commit together end with one complete entry, and
|
||||
// none of them fails because another one was first.
|
||||
func TestGitConcurrentCheckoutsLeaveOneEntry(t *testing.T) {
|
||||
repo, sha := newGitRepository(t)
|
||||
cache := t.TempDir()
|
||||
|
||||
var wg sync.WaitGroup
|
||||
errs := make([]error, 5)
|
||||
for i := range errs {
|
||||
wg.Add(1)
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
errs[i] = gitRemoteLoader{}.checkout(t.Context(), cache, &gitutil.GitRef{Remote: repo, Ref: sha})
|
||||
}()
|
||||
}
|
||||
wg.Wait()
|
||||
|
||||
for _, err := range errs {
|
||||
assert.NilError(t, err)
|
||||
}
|
||||
assert.DeepEqual(t, gitCacheNames(t, cache), []string{sha})
|
||||
_, err := os.Stat(filepath.Join(gitEntryDir(cache, sha), "compose.yaml"))
|
||||
assert.NilError(t, err)
|
||||
}
|
||||
|
||||
// A directory which an earlier release left at the root of the cache, possibly an
|
||||
// interrupted checkout, is not taken for the commit: the loader finds nothing to
|
||||
// load offline, and finds the checkout once it is in the directory for Git.
|
||||
func TestGitLoaderIgnoresWhatAnEarlierReleaseCached(t *testing.T) {
|
||||
repo, sha := newGitRepository(t)
|
||||
t.Setenv("XDG_CACHE_HOME", t.TempDir())
|
||||
cache, err := cacheDir()
|
||||
assert.NilError(t, err)
|
||||
assert.NilError(t, os.MkdirAll(filepath.Join(cache, sha), 0o700))
|
||||
|
||||
remote := "https://example.invalid/repo.git#" + sha
|
||||
offline := NewGitRemoteLoader(nil, true)
|
||||
|
||||
local, err := offline.Load(t.Context(), remote)
|
||||
assert.NilError(t, err)
|
||||
assert.Equal(t, local, "")
|
||||
|
||||
err = gitRemoteLoader{}.checkout(t.Context(), cache, &gitutil.GitRef{Remote: repo, Ref: sha})
|
||||
assert.NilError(t, err)
|
||||
|
||||
local, err = NewGitRemoteLoader(nil, true).Load(t.Context(), remote)
|
||||
assert.NilError(t, err)
|
||||
assert.Equal(t, local, filepath.Join(gitEntryDir(cache, sha), "compose.yaml"))
|
||||
}
|
||||
Loading…
Add table
Add a link
Reference in a new issue