fix(git): check a commit out aside, and keep the checkouts in a directory of the cache for Git

A commit was checked out directly into its directory of the cache, which was
then taken for complete as soon as it existed: a run interrupted during the
fetch or the checkout left a partial directory that every later run used.

The checkout is now made in a directory of its own and renamed to the one of the
commit once it is complete, so the directory of a commit is never a partial
checkout, and no file beside it is needed to tell. Checkouts live in a "git"
directory of the cache, and what earlier releases left at the root of the cache
is no longer looked for.

Signed-off-by: Nicolas De Loof <nicolas.deloof@gmail.com>
This commit is contained in:
Nicolas De Loof 2026-11-06 18:29:27 +01:00 • committed by Guillaume Lours
parent 95bbca7582
commit b5709cd309
2 changed files with 183 additions and 4 deletions

View file

@ -102,12 +102,12 @@ func (g gitRemoteLoader) Load(ctx context.Context, path string) (string, error)
return "", fmt.Errorf("initializing remote resource cache: %w", err)
}
local = filepath.Join(cache, ref.Ref)
local = gitEntryDir(cache, ref.Ref)
if _, err := os.Stat(local); os.IsNotExist(err) {
if g.offline {
return "", nil
}
err = g.checkout(ctx, local, ref)
err = g.checkout(ctx, cache, ref)
if err != nil {
return "", err
}
@ -192,11 +192,32 @@ func (g gitRemoteLoader) resolveGitRef(ctx context.Context, path string, ref *gi
return nil
}
func (g gitRemoteLoader) checkout(ctx context.Context, path string, ref *gitutil.GitRef) error {
err := os.MkdirAll(path, 0o700)
// cacheGitDir is the directory of the cache which holds the checkouts of the Git
// repositories, one directory for each commit. Earlier releases checked them out at
// the root of the cache, where they are no longer looked for: one which was left
// incomplete by an interrupted run could be taken for a complete one.
const cacheGitDir = "git"
// gitEntryDir is the directory of the checkout of the commit.
func gitEntryDir(cache, sha string) string {
return filepath.Join(cache, cacheGitDir, sha)
}
// checkout fetches the commit into the cache. It does so in a directory of its own,
// which is renamed to the one of the commit only once the checkout is complete: the
// directory of a commit is therefore never a partial checkout, and needs no file
// beside it to tell.
func (g gitRemoteLoader) checkout(ctx context.Context, cache string, ref *gitutil.GitRef) error {
gitCache := filepath.Join(cache, cacheGitDir)
if err := os.MkdirAll(gitCache, 0o700); err != nil {
return err
}
path, err := os.MkdirTemp(gitCache, ".checkout-")
if err != nil {
return err
}
defer func() { _ = os.RemoveAll(path) }()
err = exec.CommandContext(ctx, "git", "init", path).Run()
if err != nil {
return err
@ -224,6 +245,15 @@ func (g gitRemoteLoader) checkout(ctx context.Context, path string, ref *gitutil
if err != nil {
return err
}
local := gitEntryDir(cache, ref.Ref)
if err := os.Rename(path, local); err != nil {
// another run placed the same commit first
if _, statErr := os.Stat(local); statErr == nil {
return nil
}
return err
}
return nil
}

View file

@ -0,0 +1,149 @@
/*
Copyright 2020 Docker Compose CLI authors
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package remote
import (
"os"
"os/exec"
"path/filepath"
"strings"
"sync"
"testing"
gitutil "github.com/moby/buildkit/frontend/dockerfile/dfgitutil"
"gotest.tools/v3/assert"
)
// newGitRepository makes a repository of one commit holding a compose file, and
// returns its path and the commit.
func newGitRepository(t *testing.T) (string, string) {
t.Helper()
if _, err := exec.LookPath("git"); err != nil {
t.Skip("git is not installed")
}
// keep the configuration of the user, which may sign commits, out of the test
t.Setenv("GIT_CONFIG_GLOBAL", os.DevNull)
t.Setenv("GIT_CONFIG_SYSTEM", os.DevNull)
repo := t.TempDir()
assert.NilError(t, os.WriteFile(filepath.Join(repo, "compose.yaml"), []byte("services:\n app:\n image: alpine\n"), 0o600))
for _, args := range [][]string{
{"init", "-q"},
{"add", "compose.yaml"},
{"-c", "user.name=test", "-c", "user.email=test@example.com", "commit", "-q", "-m", "compose file"},
} {
cmd := exec.Command("git", args...)
cmd.Dir = repo
out, err := cmd.CombinedOutput()
assert.NilError(t, err, string(out))
}
cmd := exec.Command("git", "rev-parse", "HEAD")
cmd.Dir = repo
out, err := cmd.Output()
assert.NilError(t, err)
return repo, strings.TrimSpace(string(out))
}
func gitCacheNames(t *testing.T, cache string) []string {
t.Helper()
entries, err := os.ReadDir(filepath.Join(cache, cacheGitDir))
if os.IsNotExist(err) {
return nil
}
assert.NilError(t, err)
names := make([]string, 0, len(entries))
for _, e := range entries {
names = append(names, e.Name())
}
return names
}
// A checkout is put in the cache directory of Git checkouts, named after the commit,
// only once it is complete: what is left beside it is the entry and nothing else.
func TestGitCheckoutIsPlacedWhenComplete(t *testing.T) {
repo, sha := newGitRepository(t)
cache := t.TempDir()
err := gitRemoteLoader{}.checkout(t.Context(), cache, &gitutil.GitRef{Remote: repo, Ref: sha})
assert.NilError(t, err)
assert.DeepEqual(t, gitCacheNames(t, cache), []string{sha})
_, err = os.Stat(filepath.Join(gitEntryDir(cache, sha), "compose.yaml"))
assert.NilError(t, err)
}
// A checkout which fails leaves nothing in the cache: no entry for the commit,
// and no directory of the attempt.
func TestGitCheckoutWhichFailsLeavesNothing(t *testing.T) {
_, sha := newGitRepository(t)
cache := t.TempDir()
err := gitRemoteLoader{}.checkout(t.Context(), cache, &gitutil.GitRef{Remote: filepath.Join(cache, "missing"), Ref: sha})
assert.Assert(t, err != nil)
assert.Equal(t, len(gitCacheNames(t, cache)), 0)
}
// Runs which check out the same commit together end with one complete entry, and
// none of them fails because another one was first.
func TestGitConcurrentCheckoutsLeaveOneEntry(t *testing.T) {
repo, sha := newGitRepository(t)
cache := t.TempDir()
var wg sync.WaitGroup
errs := make([]error, 5)
for i := range errs {
wg.Add(1)
go func() {
defer wg.Done()
errs[i] = gitRemoteLoader{}.checkout(t.Context(), cache, &gitutil.GitRef{Remote: repo, Ref: sha})
}()
}
wg.Wait()
for _, err := range errs {
assert.NilError(t, err)
}
assert.DeepEqual(t, gitCacheNames(t, cache), []string{sha})
_, err := os.Stat(filepath.Join(gitEntryDir(cache, sha), "compose.yaml"))
assert.NilError(t, err)
}
// A directory which an earlier release left at the root of the cache, possibly an
// interrupted checkout, is not taken for the commit: the loader finds nothing to
// load offline, and finds the checkout once it is in the directory for Git.
func TestGitLoaderIgnoresWhatAnEarlierReleaseCached(t *testing.T) {
repo, sha := newGitRepository(t)
t.Setenv("XDG_CACHE_HOME", t.TempDir())
cache, err := cacheDir()
assert.NilError(t, err)
assert.NilError(t, os.MkdirAll(filepath.Join(cache, sha), 0o700))
remote := "https://example.invalid/repo.git#" + sha
offline := NewGitRemoteLoader(nil, true)
local, err := offline.Load(t.Context(), remote)
assert.NilError(t, err)
assert.Equal(t, local, "")
err = gitRemoteLoader{}.checkout(t.Context(), cache, &gitutil.GitRef{Remote: repo, Ref: sha})
assert.NilError(t, err)
local, err = NewGitRemoteLoader(nil, true).Load(t.Context(), remote)
assert.NilError(t, err)
assert.Equal(t, local, filepath.Join(gitEntryDir(cache, sha), "compose.yaml"))
}