mirror of
https://github.com/docker/compose.git
synced 2026-11-19 22:50:08 +00:00
refactor(compose): move publish and transform to internal/publish
Some checks are pending
ci / validate (lint) (push) Waiting to run
ci / validate (relay-lint) (push) Waiting to run
ci / validate (relay-test) (push) Waiting to run
ci / validate (validate-docs) (push) Waiting to run
ci / validate (validate-go-mod) (push) Waiting to run
ci / validate (validate-headers) (push) Waiting to run
ci / validate (validate-mocks) (push) Waiting to run
ci / binary (push) Waiting to run
ci / binary-finalize (push) Blocked by required conditions
ci / bin-image-test (push) Waiting to run
ci / relay-image-test (push) Waiting to run
ci / test (push) Waiting to run
ci / e2e (plugin, oldstable, graphdriver) (push) Waiting to run
ci / e2e (standalone, oldstable, graphdriver) (push) Waiting to run
ci / e2e (plugin, stable, containerd) (push) Waiting to run
ci / e2e (plugin, stable, graphdriver) (push) Waiting to run
ci / e2e (standalone, stable, containerd) (push) Waiting to run
ci / e2e (standalone, stable, graphdriver) (push) Waiting to run
ci / coverage (push) Blocked by required conditions
ci / ci-ok (push) Blocked by required conditions
ci / release (push) Blocked by required conditions
merge / bin-image-prepare (push) Waiting to run
merge / bin-image (push) Blocked by required conditions
merge / relay-image (push) Waiting to run
merge / module-image (push) Waiting to run
Scorecards supply-chain security / Scorecards analysis (push) Waiting to run
zizmor / zizmor (push) Waiting to run
Some checks are pending
ci / validate (lint) (push) Waiting to run
ci / validate (relay-lint) (push) Waiting to run
ci / validate (relay-test) (push) Waiting to run
ci / validate (validate-docs) (push) Waiting to run
ci / validate (validate-go-mod) (push) Waiting to run
ci / validate (validate-headers) (push) Waiting to run
ci / validate (validate-mocks) (push) Waiting to run
ci / binary (push) Waiting to run
ci / binary-finalize (push) Blocked by required conditions
ci / bin-image-test (push) Waiting to run
ci / relay-image-test (push) Waiting to run
ci / test (push) Waiting to run
ci / e2e (plugin, oldstable, graphdriver) (push) Waiting to run
ci / e2e (standalone, oldstable, graphdriver) (push) Waiting to run
ci / e2e (plugin, stable, containerd) (push) Waiting to run
ci / e2e (plugin, stable, graphdriver) (push) Waiting to run
ci / e2e (standalone, stable, containerd) (push) Waiting to run
ci / e2e (standalone, stable, graphdriver) (push) Waiting to run
ci / coverage (push) Blocked by required conditions
ci / ci-ok (push) Blocked by required conditions
ci / release (push) Blocked by required conditions
merge / bin-image-prepare (push) Waiting to run
merge / bin-image (push) Blocked by required conditions
merge / relay-image (push) Waiting to run
merge / module-image (push) Waiting to run
Scorecards supply-chain security / Scorecards analysis (push) Waiting to run
zizmor / zizmor (push) Waiting to run
Signed-off-by: Nicolas De Loof <nicolas.deloof@gmail.com>
This commit is contained in:
parent
5c666993e1
commit
ce67a3ec49
8 changed files with 984 additions and 847 deletions
892
internal/publish/publish.go
Normal file
892
internal/publish/publish.go
Normal file
|
|
@ -0,0 +1,892 @@
|
|||
/*
|
||||
Copyright 2020 Docker Compose CLI authors
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
// Package publish implements publishing a Compose application as an OCI
|
||||
// artifact: the pre-checks on what it would leak, the layers, and the push.
|
||||
package publish
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"maps"
|
||||
"os"
|
||||
"slices"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
"github.com/DefangLabs/secret-detector/pkg/detectors/keyword"
|
||||
"github.com/DefangLabs/secret-detector/pkg/scanner"
|
||||
"github.com/DefangLabs/secret-detector/pkg/secrets"
|
||||
"github.com/compose-spec/compose-go/v2/loader"
|
||||
"github.com/compose-spec/compose-go/v2/types"
|
||||
"github.com/containerd/containerd/v2/core/remotes"
|
||||
"github.com/distribution/reference"
|
||||
"github.com/moby/moby/client"
|
||||
"github.com/opencontainers/go-digest"
|
||||
"github.com/opencontainers/image-spec/specs-go"
|
||||
v1 "github.com/opencontainers/image-spec/specs-go/v1"
|
||||
"github.com/sirupsen/logrus"
|
||||
|
||||
"github.com/docker/compose/v5/internal/desktop"
|
||||
"github.com/docker/compose/v5/internal/oci"
|
||||
"github.com/docker/compose/v5/internal/publish/transform"
|
||||
"github.com/docker/compose/v5/internal/registry"
|
||||
"github.com/docker/compose/v5/pkg/api"
|
||||
)
|
||||
|
||||
// Backend is what publishing needs from the engine it acts on.
|
||||
type Backend interface {
|
||||
// APIClient talks to the Docker daemon.
|
||||
APIClient() client.APIClient
|
||||
// AuthProvider supplies the registry credentials.
|
||||
AuthProvider() registry.AuthProvider
|
||||
// Events receives the progress of the operation.
|
||||
Events() api.EventProcessor
|
||||
// DryRun tells whether the operation only reports what it would do.
|
||||
DryRun() bool
|
||||
// Prompt asks the user a yes/no question.
|
||||
Prompt(message string, defaultValue bool) (bool, error)
|
||||
// ImageDigestResolver resolves an image reference to the digest the
|
||||
// registry holds for it, the same way pull does.
|
||||
ImageDigestResolver(ctx context.Context) func(named reference.Named) (digest.Digest, error)
|
||||
// Push pushes the images built for the project, as part of the operation
|
||||
// already running: it opens no progress bracket of its own.
|
||||
Push(ctx context.Context, project *types.Project, options api.PushOptions) error
|
||||
}
|
||||
|
||||
// Service publishes projects on a Backend.
|
||||
type Service struct {
|
||||
backend Backend
|
||||
}
|
||||
|
||||
// New creates a Service acting on the given Backend.
|
||||
func New(backend Backend) *Service {
|
||||
return &Service{backend: backend}
|
||||
}
|
||||
|
||||
// Publish publishes the project as an OCI artifact to the repository. It does
|
||||
// not open a progress bracket: the caller wraps it in the "publish" one.
|
||||
func (s *Service) Publish(ctx context.Context, project *types.Project, repository string, options api.PublishOptions) error {
|
||||
project, err := project.WithProfiles([]string{"*"})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
accept, err := s.preChecks(ctx, project, options)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if !accept {
|
||||
return api.ErrCanceled
|
||||
}
|
||||
// the Backend's Push, not the public one: publish already runs inside its
|
||||
// own "publish" Start/Done bracket, and Push would open a second one on
|
||||
// the same shared bus. Quiet:true would also suppress push progress
|
||||
// reporting, not just the bracket, so it's not an option here.
|
||||
err = s.backend.Push(ctx, project, api.PushOptions{IgnoreFailures: true, ImageMandatory: true})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
layers, err := s.createLayers(ctx, project, options)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
s.backend.Events().On(api.Resource{
|
||||
ID: repository,
|
||||
Text: "publishing",
|
||||
Status: api.Working,
|
||||
})
|
||||
if logrus.IsLevelEnabled(logrus.DebugLevel) {
|
||||
logrus.Debug("publishing layers")
|
||||
for _, layer := range layers {
|
||||
indent, _ := json.MarshalIndent(layer, "", " ")
|
||||
fmt.Println(string(indent))
|
||||
}
|
||||
}
|
||||
didFallback := false
|
||||
if !s.backend.DryRun() {
|
||||
didFallback, err = s.pushComposeArtifact(ctx, project, repository, layers, options)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
text, status := "published", api.Done
|
||||
if didFallback {
|
||||
text, status = "published (registry rejected OCI 1.1; fell back to OCI 1.0)", api.Warning
|
||||
}
|
||||
s.backend.Events().On(api.Resource{
|
||||
ID: repository,
|
||||
Text: text,
|
||||
Status: status,
|
||||
})
|
||||
return nil
|
||||
}
|
||||
|
||||
// pushComposeArtifact pushes the compose artifact manifest to the repository,
|
||||
// and the application image index when publishing a full application. The
|
||||
// returned bool reports whether the push fell back from OCI 1.1 to OCI 1.0.
|
||||
func (s *Service) pushComposeArtifact(ctx context.Context, project *types.Project, repository string, layers []v1.Descriptor, options api.PublishOptions) (bool, error) {
|
||||
named, err := reference.ParseDockerRef(repository)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
|
||||
var insecureRegistries []string
|
||||
if options.InsecureRegistry {
|
||||
insecureRegistries = append(insecureRegistries, reference.Domain(named))
|
||||
}
|
||||
|
||||
resolver := oci.NewResolver(s.backend.AuthProvider(), desktop.ProxyTransportFor(ctx, s.backend.APIClient()), insecureRegistries...)
|
||||
|
||||
descriptor, didFallback, err := oci.PushManifest(ctx, resolver, named, layers, options.OCIVersion)
|
||||
if err != nil {
|
||||
s.backend.Events().On(api.Resource{
|
||||
ID: repository,
|
||||
Text: "publishing",
|
||||
Status: api.Error,
|
||||
})
|
||||
return false, err
|
||||
}
|
||||
|
||||
if options.Application {
|
||||
return didFallback, pushApplicationIndex(ctx, resolver, named, descriptor, project)
|
||||
}
|
||||
return didFallback, nil
|
||||
}
|
||||
|
||||
// pushApplicationIndex pushes an image index referencing every service image,
|
||||
// so the application can be pulled as a single artifact
|
||||
func pushApplicationIndex(ctx context.Context, resolver remotes.Resolver, named reference.Named, descriptor v1.Descriptor, project *types.Project) error {
|
||||
manifests := []v1.Descriptor{}
|
||||
images := make([]string, 0, len(project.Services)+len(project.Jobs))
|
||||
for _, service := range project.Services {
|
||||
images = append(images, service.Image)
|
||||
}
|
||||
for _, job := range project.Jobs {
|
||||
images = append(images, job.Image)
|
||||
}
|
||||
for _, image := range images {
|
||||
ref, err := reference.ParseDockerRef(image)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
manifest, err := oci.Copy(ctx, resolver, ref, named)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
manifests = append(manifests, manifest)
|
||||
}
|
||||
|
||||
descriptor.Data = nil
|
||||
index, err := json.Marshal(v1.Index{
|
||||
Versioned: specs.Versioned{SchemaVersion: 2},
|
||||
MediaType: v1.MediaTypeImageIndex,
|
||||
Manifests: manifests,
|
||||
Subject: &descriptor,
|
||||
Annotations: map[string]string{
|
||||
"com.docker.compose.version": api.ComposeVersion,
|
||||
},
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
imagesDescriptor := v1.Descriptor{
|
||||
MediaType: v1.MediaTypeImageIndex,
|
||||
ArtifactType: oci.ComposeProjectArtifactType,
|
||||
Digest: digest.FromString(string(index)),
|
||||
Size: int64(len(index)),
|
||||
Annotations: map[string]string{
|
||||
"com.docker.compose.version": api.ComposeVersion,
|
||||
},
|
||||
Data: index,
|
||||
}
|
||||
return oci.Push(ctx, resolver, reference.TrimNamed(named), imagesDescriptor)
|
||||
}
|
||||
|
||||
func (s *Service) createLayers(ctx context.Context, project *types.Project, options api.PublishOptions) ([]v1.Descriptor, error) {
|
||||
var layers []v1.Descriptor
|
||||
extFiles := map[string]string{}
|
||||
envFiles := map[string]string{}
|
||||
for _, file := range project.ComposeFiles {
|
||||
data, err := processFile(ctx, file, project, extFiles, envFiles)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
layerDescriptor := oci.DescriptorForComposeFile(file, data)
|
||||
layers = append(layers, layerDescriptor)
|
||||
}
|
||||
|
||||
extLayers, err := processExtends(ctx, project, extFiles)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
layers = append(layers, extLayers...)
|
||||
|
||||
if options.WithEnvironment {
|
||||
layers = append(layers, envFileLayers(envFiles)...)
|
||||
}
|
||||
|
||||
if options.ResolveImageDigests {
|
||||
yaml, err := s.generateImageDigestsOverride(ctx, project)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
layerDescriptor := oci.DescriptorForComposeFile("image-digests.yaml", yaml)
|
||||
layers = append(layers, layerDescriptor)
|
||||
}
|
||||
return layers, nil
|
||||
}
|
||||
|
||||
func processExtends(ctx context.Context, project *types.Project, extFiles map[string]string) ([]v1.Descriptor, error) {
|
||||
var layers []v1.Descriptor
|
||||
moreExtFiles := map[string]string{}
|
||||
envFiles := map[string]string{}
|
||||
for xf, hash := range extFiles {
|
||||
data, err := processFile(ctx, xf, project, moreExtFiles, envFiles)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
layerDescriptor := oci.DescriptorForComposeFile(hash, data)
|
||||
layerDescriptor.Annotations["com.docker.compose.extends"] = "true"
|
||||
layers = append(layers, layerDescriptor)
|
||||
}
|
||||
for f, hash := range moreExtFiles {
|
||||
if _, ok := extFiles[f]; ok {
|
||||
delete(moreExtFiles, f)
|
||||
}
|
||||
extFiles[f] = hash
|
||||
}
|
||||
if len(moreExtFiles) > 0 {
|
||||
extLayers, err := processExtends(ctx, project, moreExtFiles)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
layers = append(layers, extLayers...)
|
||||
}
|
||||
return layers, nil
|
||||
}
|
||||
|
||||
func processFile(ctx context.Context, file string, project *types.Project, extFiles map[string]string, envFiles map[string]string) ([]byte, error) {
|
||||
f, err := os.ReadFile(file)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
base, err := loader.LoadWithContext(ctx, types.ConfigDetails{
|
||||
WorkingDir: project.WorkingDir,
|
||||
Environment: project.Environment,
|
||||
ConfigFiles: []types.ConfigFile{
|
||||
{
|
||||
Filename: file,
|
||||
Content: f,
|
||||
},
|
||||
},
|
||||
}, func(options *loader.Options) {
|
||||
options.SkipValidation = true
|
||||
options.SkipExtends = true
|
||||
options.SkipConsistencyCheck = true
|
||||
options.ResolvePaths = true
|
||||
options.SkipInclude = true
|
||||
options.Profiles = project.Profiles
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
for name, service := range base.Services {
|
||||
for i, envFile := range service.EnvFiles {
|
||||
// A real stat failure (e.g. permissions) is fatal, but a missing file is not:
|
||||
// the project loader already rejects missing required env files before we get
|
||||
// here, so an absent file at this point is an optional one.
|
||||
_, statErr := os.Stat(envFile.Path)
|
||||
if statErr != nil && !os.IsNotExist(statErr) {
|
||||
return nil, fmt.Errorf("failed to access env file %s: %w", envFile.Path, statErr)
|
||||
}
|
||||
// The hash is derived from the path string alone, so the env_file is always
|
||||
// rewritten to its opaque <hash>.env placeholder, even for a missing optional
|
||||
// file, so the published artifact never leaks the local path. Only files that
|
||||
// exist are registered for upload, mirroring the extends handling below.
|
||||
hash := fmt.Sprintf("%x.env", sha256.Sum256([]byte(envFile.Path)))
|
||||
if statErr == nil {
|
||||
envFiles[envFile.Path] = hash
|
||||
}
|
||||
f, err = transform.ReplaceEnvFile(f, name, i, hash)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
|
||||
if service.Extends == nil {
|
||||
continue
|
||||
}
|
||||
xf := service.Extends.File
|
||||
if xf == "" {
|
||||
continue
|
||||
}
|
||||
if _, err = os.Stat(service.Extends.File); os.IsNotExist(err) {
|
||||
// No local file, while we loaded the project successfully: This is actually a remote resource
|
||||
continue
|
||||
}
|
||||
|
||||
hash := fmt.Sprintf("%x.yaml", sha256.Sum256([]byte(xf)))
|
||||
extFiles[xf] = hash
|
||||
|
||||
f, err = transform.ReplaceExtendsFile(f, name, hash)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
return f, nil
|
||||
}
|
||||
|
||||
func (s *Service) generateImageDigestsOverride(ctx context.Context, project *types.Project) ([]byte, error) {
|
||||
project, err := project.WithImagesResolved(s.backend.ImageDigestResolver(ctx))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
// WithImagesResolved only walks services: dress the jobs as services to
|
||||
// run them through the exact same resolution, then fold the digests back
|
||||
// into a jobs override.
|
||||
if len(project.Jobs) > 0 {
|
||||
jobsAsServices := &types.Project{Services: types.Services{}}
|
||||
for name, job := range project.Jobs {
|
||||
jobsAsServices.Services[name] = types.ServiceConfig{
|
||||
Name: name,
|
||||
ContainerSpec: types.ContainerSpec{Image: job.Image},
|
||||
}
|
||||
}
|
||||
jobsAsServices, err = jobsAsServices.WithImagesResolved(s.backend.ImageDigestResolver(ctx))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
for name, resolved := range jobsAsServices.Services {
|
||||
job := project.Jobs[name]
|
||||
job.Image = resolved.Image
|
||||
project.Jobs[name] = job
|
||||
}
|
||||
}
|
||||
override := types.Project{
|
||||
Services: types.Services{},
|
||||
Jobs: types.Jobs{},
|
||||
}
|
||||
for name, service := range project.Services {
|
||||
override.Services[name] = types.ServiceConfig{
|
||||
ContainerSpec: types.ContainerSpec{Image: service.Image},
|
||||
}
|
||||
}
|
||||
for name, job := range project.Jobs {
|
||||
override.Jobs[name] = types.JobConfig{
|
||||
ContainerSpec: types.ContainerSpec{Image: job.Image},
|
||||
}
|
||||
}
|
||||
return override.MarshalYAML()
|
||||
}
|
||||
|
||||
func (s *Service) preChecks(ctx context.Context, project *types.Project, options api.PublishOptions) (bool, error) {
|
||||
if ok, err := s.checkOnlyBuildSection(project); !ok || err != nil {
|
||||
return false, err
|
||||
}
|
||||
bindMounts := s.checkForBindMount(project)
|
||||
if len(bindMounts) > 0 {
|
||||
b := strings.Builder{}
|
||||
b.WriteString("you are about to publish bind mounts declaration within your OCI artifact.\n" +
|
||||
"only the bind mount declarations will be added to the OCI artifact (not content)\n" +
|
||||
"please double check that you are not mounting potential user's sensitive directories or data\n")
|
||||
for key, val := range bindMounts {
|
||||
b.WriteString(key)
|
||||
for _, v := range val {
|
||||
b.WriteString(v.String())
|
||||
b.WriteRune('\n')
|
||||
}
|
||||
}
|
||||
b.WriteString("Are you ok to publish these bind mount declarations?")
|
||||
confirm, err := s.backend.Prompt(b.String(), false)
|
||||
if err != nil || !confirm {
|
||||
return false, err
|
||||
}
|
||||
}
|
||||
detectedSecrets, err := s.checkForSensitiveData(ctx, project)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
if len(detectedSecrets) > 0 {
|
||||
b := strings.Builder{}
|
||||
b.WriteString("you are about to publish sensitive data within your OCI artifact.\n" +
|
||||
"please double check that you are not leaking sensitive data\n")
|
||||
for _, val := range detectedSecrets {
|
||||
b.WriteString(val.Type)
|
||||
b.WriteRune('\n')
|
||||
fmt.Fprintf(&b, "%q: %s\n", val.Key, val.Value)
|
||||
}
|
||||
b.WriteString("Are you ok to publish these sensitive data?")
|
||||
confirm, err := s.backend.Prompt(b.String(), false)
|
||||
if err != nil || !confirm {
|
||||
return false, err
|
||||
}
|
||||
}
|
||||
err = s.checkEnvironmentVariables(ctx, project, options)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
return true, nil
|
||||
}
|
||||
|
||||
// envCheckFindings groups everything checkEnvironmentVariables surfaces to
|
||||
// the user during publish pre-checks for env-related leak risks.
|
||||
type envCheckFindings struct {
|
||||
// services maps service name -> findings for that service. Only services
|
||||
// with at least one finding are present.
|
||||
services map[string]*serviceEnvFindings
|
||||
// configsLiteralContent lists configs whose inline `content:` is a literal
|
||||
// (not interpolation). Sorted alphabetically. config.content is decoupled
|
||||
// from --with-env because the flag is documented as controlling environment
|
||||
// variable publishing only.
|
||||
configsLiteralContent []string
|
||||
}
|
||||
|
||||
type serviceEnvFindings struct {
|
||||
hasEnvFile bool
|
||||
// suspiciousKeys is the set of environment variable names whose literal
|
||||
// values look sensitive, as classified by the upstream DefangLabs keyword
|
||||
// detector (password, secret, token, api_key, …). A set is used because
|
||||
// the same service may be visited across multiple compose files during
|
||||
// the extends walk; callers convert to a sorted slice via sortedKeys
|
||||
// when surfacing to the user.
|
||||
suspiciousKeys map[string]struct{}
|
||||
}
|
||||
|
||||
// sortedSuspiciousKeys returns the suspicious env var names alphabetically
|
||||
// sorted for stable output.
|
||||
func (f *serviceEnvFindings) sortedSuspiciousKeys() []string {
|
||||
return slices.Sorted(maps.Keys(f.suspiciousKeys))
|
||||
}
|
||||
|
||||
func (f *envCheckFindings) hasEnvFinding() bool {
|
||||
for _, serviceFindings := range f.services {
|
||||
if serviceFindings.hasEnvFile || len(serviceFindings.suspiciousKeys) > 0 {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// checkEnvironmentVariables walks every compose file that will be serialized
|
||||
// into the OCI artifact (the top-level files plus any local extends parents)
|
||||
// and prompts the user to confirm before publishing:
|
||||
//
|
||||
// 1. service env_file declarations and literal environment values whose key
|
||||
// name looks sensitive (password, secret, token, api_key, …) — silenced
|
||||
// by --with-env;
|
||||
// 2. literal inline config.content — always prompts (decoupled from
|
||||
// --with-env, which is documented to cover env vars only).
|
||||
//
|
||||
// Interpolated values like "${SECRET}" or "$VAR" are preserved as placeholders
|
||||
// in the published YAML and don't leak the resolved value; the keyword
|
||||
// detector's value regex skips them automatically.
|
||||
func (s *Service) checkEnvironmentVariables(ctx context.Context, project *types.Project, options api.PublishOptions) error {
|
||||
if len(project.ComposeFiles) == 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
findings, err := collectEnvCheckFindings(ctx, project)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if !options.WithEnvironment && findings.hasEnvFinding() {
|
||||
if err := s.confirmOrCancel(buildEnvPromptMessage(findings.services)); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
if len(findings.configsLiteralContent) > 0 {
|
||||
if err := s.confirmOrCancel(buildConfigContentPromptMessage(findings.configsLiteralContent)); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// confirmOrCancel runs an interactive yes/no prompt and returns:
|
||||
// - the prompt's error verbatim, if it failed;
|
||||
// - api.ErrCanceled if the user declined;
|
||||
// - nil if the user accepted.
|
||||
func (s *Service) confirmOrCancel(message string) error {
|
||||
confirm, err := s.backend.Prompt(message, false)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if !confirm {
|
||||
return api.ErrCanceled
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// collectEnvCheckFindings walks every compose file scheduled for publication
|
||||
// (top-level files plus any local extends parents discovered along the way)
|
||||
// and aggregates per-service and per-config findings. The walk mirrors
|
||||
// processExtends so coverage matches what is actually serialized into the OCI
|
||||
// artifact.
|
||||
func collectEnvCheckFindings(ctx context.Context, project *types.Project) (*envCheckFindings, error) {
|
||||
findings := &envCheckFindings{services: map[string]*serviceEnvFindings{}}
|
||||
literalCfgs := map[string]struct{}{}
|
||||
keywordDetector := keyword.NewDetector("0")
|
||||
|
||||
seen := map[string]struct{}{}
|
||||
queue := slices.Clone(project.ComposeFiles)
|
||||
for len(queue) > 0 {
|
||||
file := queue[0]
|
||||
queue = queue[1:]
|
||||
if _, ok := seen[file]; ok {
|
||||
continue
|
||||
}
|
||||
seen[file] = struct{}{}
|
||||
|
||||
unresolved, err := loadUnresolvedFile(ctx, project, file)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to load compose file %s: %w", file, err)
|
||||
}
|
||||
|
||||
for _, service := range unresolved.Services {
|
||||
recordEnvFindings(findings.services, keywordDetector, service.Name, service.ContainerSpec)
|
||||
if parent := localExtendsParent(service); parent != "" {
|
||||
queue = append(queue, parent)
|
||||
}
|
||||
}
|
||||
for name, job := range unresolved.Jobs {
|
||||
// jobs carry environment/env_file like services: same leak surface
|
||||
recordEnvFindings(findings.services, keywordDetector, name, job.ContainerSpec)
|
||||
}
|
||||
for name, config := range unresolved.Configs {
|
||||
// config.Environment is a variable *name* (only the name is
|
||||
// published, not its resolved value) so it is not a leak. Inline
|
||||
// config.Content is what ends up in the artifact. compose-go
|
||||
// enforces that file, environment, and content are mutually
|
||||
// exclusive. The map key is the name as written in the compose
|
||||
// file; config.Name is the project-namespaced version, which is
|
||||
// less helpful when surfaced to the user.
|
||||
if config.Content != "" && configContentLooksLiteral(config.Content, keywordDetector) {
|
||||
literalCfgs[name] = struct{}{}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if len(literalCfgs) > 0 {
|
||||
findings.configsLiteralContent = slices.Sorted(maps.Keys(literalCfgs))
|
||||
}
|
||||
return findings, nil
|
||||
}
|
||||
|
||||
func recordEnvFindings(services map[string]*serviceEnvFindings, detector secrets.Detector, name string, spec types.ContainerSpec) {
|
||||
envValues := map[string]string{}
|
||||
for key, value := range spec.Environment {
|
||||
if value == nil {
|
||||
continue
|
||||
}
|
||||
envValues[key] = replaceDollarEscape(*value)
|
||||
}
|
||||
|
||||
hits, _ := detector.ScanMap(envValues)
|
||||
if len(hits) == 0 && len(spec.EnvFiles) == 0 {
|
||||
return
|
||||
}
|
||||
|
||||
f := services[name]
|
||||
if f == nil {
|
||||
f = &serviceEnvFindings{suspiciousKeys: map[string]struct{}{}}
|
||||
services[name] = f
|
||||
}
|
||||
if len(spec.EnvFiles) > 0 {
|
||||
f.hasEnvFile = true
|
||||
}
|
||||
for _, hit := range hits {
|
||||
f.suspiciousKeys[hit.Key] = struct{}{}
|
||||
}
|
||||
}
|
||||
|
||||
// configContentLooksLiteral returns true when the inline config.content has
|
||||
// a literal portion that would be published as-is, leaking the value to
|
||||
// consumers of the OCI artifact.
|
||||
//
|
||||
// We piggyback on the keyword detector's value regex (`[^${\s].+[^}\s]`) by
|
||||
// passing a fake "password" key to ScanMap — the regex isn't exported
|
||||
// directly, only via the key+value match path. The regex excludes values
|
||||
// starting with `$` (`${VAR}`/`$VAR` interpolation), ending with `}`
|
||||
// (templates like `key=${SECRET}`), or shorter than 3 chars, which neatly
|
||||
// matches our notion of "looks like a template, not a literal".
|
||||
func configContentLooksLiteral(content string, detector secrets.Detector) bool {
|
||||
hits, _ := detector.ScanMap(map[string]string{"password": replaceDollarEscape(content)})
|
||||
return len(hits) > 0
|
||||
}
|
||||
|
||||
// replaceDollarEscape substitutes the compose-spec `$$` escape (which
|
||||
// represents a literal `$` in the resolved value) with a placeholder. The
|
||||
// placeholder is `X` rather than `$` because the keyword detector's value
|
||||
// regex excludes any value beginning with `$`; using `$` would mask the
|
||||
// literal we're trying to flag. Any non-special char would do — we picked
|
||||
// `X` for readability.
|
||||
func replaceDollarEscape(value string) string {
|
||||
return strings.ReplaceAll(value, "$$", "X")
|
||||
}
|
||||
|
||||
// localExtendsParent returns the path of an extends parent file that exists on
|
||||
// disk, or "" when the service does not extend or extends a remote resource.
|
||||
func localExtendsParent(service types.ServiceConfig) string {
|
||||
if service.Extends == nil || service.Extends.File == "" {
|
||||
return ""
|
||||
}
|
||||
if _, err := os.Stat(service.Extends.File); err != nil {
|
||||
return ""
|
||||
}
|
||||
return service.Extends.File
|
||||
}
|
||||
|
||||
func buildEnvPromptMessage(services map[string]*serviceEnvFindings) string {
|
||||
var b strings.Builder
|
||||
b.WriteString("you are about to publish env-related declarations within your OCI artifact.\n")
|
||||
b.WriteString("env_file paths and literal values for sensitive-looking keys are embedded as-is in the published YAML;\n")
|
||||
b.WriteString("interpolated values like \"${VAR}\" are kept symbolic and have already been excluded.\n")
|
||||
for _, name := range slices.Sorted(maps.Keys(services)) {
|
||||
f := services[name]
|
||||
// name may be a service or a job: findings.services carries both.
|
||||
if f.hasEnvFile {
|
||||
fmt.Fprintf(&b, " %q: env_file declared\n", name)
|
||||
}
|
||||
if keys := f.sortedSuspiciousKeys(); len(keys) > 0 {
|
||||
quoted := make([]string, len(keys))
|
||||
for i, k := range keys {
|
||||
quoted[i] = strconv.Quote(k)
|
||||
}
|
||||
fmt.Fprintf(&b, " %q: literal value for %s\n", name, strings.Join(quoted, ", "))
|
||||
}
|
||||
}
|
||||
b.WriteString("Use --with-env to silence this prompt and always publish env declarations.\n")
|
||||
b.WriteString("Are you ok to publish these env declarations?")
|
||||
return b.String()
|
||||
}
|
||||
|
||||
func buildConfigContentPromptMessage(configs []string) string {
|
||||
var b strings.Builder
|
||||
b.WriteString("you are about to publish literal inline config content within your OCI artifact.\n")
|
||||
for _, name := range configs {
|
||||
fmt.Fprintf(&b, " config %q\n", name)
|
||||
}
|
||||
b.WriteString("Are you ok to publish these config contents?")
|
||||
return b.String()
|
||||
}
|
||||
|
||||
// loadUnresolvedFile loads a single compose file with interpolation and
|
||||
// environment resolution skipped, so callers can inspect raw user-provided
|
||||
// values. Used by both checkEnvironmentVariables and composeFileAsByteReader.
|
||||
func loadUnresolvedFile(ctx context.Context, project *types.Project, filePath string) (*types.Project, error) {
|
||||
return loader.LoadWithContext(ctx, types.ConfigDetails{
|
||||
WorkingDir: project.WorkingDir,
|
||||
Environment: project.Environment,
|
||||
ConfigFiles: []types.ConfigFile{{Filename: filePath}},
|
||||
}, func(options *loader.Options) {
|
||||
options.SkipValidation = true
|
||||
options.SkipExtends = true
|
||||
options.SkipConsistencyCheck = true
|
||||
options.ResolvePaths = true
|
||||
// SkipInclude mirrors processFile: include directives stay symbolic in
|
||||
// the published artifact, so included content must not be inspected
|
||||
// here either (otherwise we'd flag literals that never ship).
|
||||
options.SkipInclude = true
|
||||
options.SkipInterpolation = true
|
||||
options.SkipResolveEnvironment = true
|
||||
options.Profiles = project.Profiles
|
||||
})
|
||||
}
|
||||
|
||||
func envFileLayers(files map[string]string) []v1.Descriptor {
|
||||
var layers []v1.Descriptor
|
||||
for file, hash := range files {
|
||||
f, err := os.ReadFile(file)
|
||||
if err != nil {
|
||||
// if we can't read the file, skip to the next one
|
||||
continue
|
||||
}
|
||||
layerDescriptor := oci.DescriptorForEnvFile(hash, f)
|
||||
layers = append(layers, layerDescriptor)
|
||||
}
|
||||
return layers
|
||||
}
|
||||
|
||||
func (s *Service) checkOnlyBuildSection(project *types.Project) (bool, error) {
|
||||
errorList := []string{}
|
||||
for _, service := range project.Services {
|
||||
if service.Image == "" && service.Build != nil {
|
||||
errorList = append(errorList, service.Name)
|
||||
}
|
||||
}
|
||||
for name, job := range project.Jobs {
|
||||
if job.Image == "" && job.Build != nil {
|
||||
errorList = append(errorList, name)
|
||||
}
|
||||
}
|
||||
if len(errorList) > 0 {
|
||||
var errMsg strings.Builder
|
||||
errMsg.WriteString("your Compose stack cannot be published as it only contains a build section for service(s):\n")
|
||||
for _, serviceInError := range errorList {
|
||||
fmt.Fprintf(&errMsg, "- %q\n", serviceInError)
|
||||
}
|
||||
return false, errors.New(errMsg.String())
|
||||
}
|
||||
return true, nil
|
||||
}
|
||||
|
||||
func (s *Service) checkForBindMount(project *types.Project) map[string][]types.ServiceVolumeConfig {
|
||||
allFindings := map[string][]types.ServiceVolumeConfig{}
|
||||
record := func(name string, volumes []types.ServiceVolumeConfig) {
|
||||
bindMounts := []types.ServiceVolumeConfig{}
|
||||
for _, volume := range volumes {
|
||||
if volume.Type == types.VolumeTypeBind {
|
||||
bindMounts = append(bindMounts, volume)
|
||||
}
|
||||
}
|
||||
if len(bindMounts) > 0 {
|
||||
allFindings[name] = bindMounts
|
||||
}
|
||||
}
|
||||
for serviceName, config := range project.Services {
|
||||
record(serviceName, config.Volumes)
|
||||
}
|
||||
for name, job := range project.Jobs {
|
||||
record(name, job.Volumes)
|
||||
}
|
||||
return allFindings
|
||||
}
|
||||
|
||||
func (s *Service) checkForSensitiveData(ctx context.Context, project *types.Project) ([]secrets.DetectedSecret, error) {
|
||||
var allFindings []secrets.DetectedSecret
|
||||
scan := scanner.NewDefaultScanner()
|
||||
|
||||
// Check all compose files
|
||||
for _, file := range project.ComposeFiles {
|
||||
in, err := composeFileAsByteReader(ctx, file, project)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
findings, err := scan.ScanReader(in)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to scan compose file %s: %w", file, err)
|
||||
}
|
||||
allFindings = append(allFindings, findings...)
|
||||
}
|
||||
|
||||
// Check env files — jobs declare env_file like services do
|
||||
for _, service := range project.Services {
|
||||
findings, err := scanEnvFiles(scan, service.EnvFiles)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
allFindings = append(allFindings, findings...)
|
||||
}
|
||||
for _, job := range project.Jobs {
|
||||
findings, err := scanEnvFiles(scan, job.EnvFiles)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
allFindings = append(allFindings, findings...)
|
||||
}
|
||||
|
||||
// Check configs defined by files
|
||||
configFiles := make([]string, 0, len(project.Configs))
|
||||
for _, config := range project.Configs {
|
||||
configFiles = append(configFiles, config.File)
|
||||
}
|
||||
findings, err := scanFiles(scan, "config", configFiles)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
allFindings = append(allFindings, findings...)
|
||||
|
||||
// Check secrets defined by files
|
||||
secretFiles := make([]string, 0, len(project.Secrets))
|
||||
for _, secret := range project.Secrets {
|
||||
secretFiles = append(secretFiles, secret.File)
|
||||
}
|
||||
findings, err = scanFiles(scan, "secret", secretFiles)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
allFindings = append(allFindings, findings...)
|
||||
|
||||
return allFindings, nil
|
||||
}
|
||||
|
||||
// scanEnvFiles scans declared env files for sensitive data; a missing env
|
||||
// file is only an error when the declaration requires it
|
||||
func scanEnvFiles(scan secrets.Scanner, envFiles []types.EnvFile) ([]secrets.DetectedSecret, error) {
|
||||
var allFindings []secrets.DetectedSecret
|
||||
for _, envFile := range envFiles {
|
||||
if _, statErr := os.Stat(envFile.Path); statErr != nil {
|
||||
if !os.IsNotExist(statErr) {
|
||||
return nil, fmt.Errorf("failed to access env file %s: %w", envFile.Path, statErr)
|
||||
}
|
||||
if envFile.Required {
|
||||
return nil, fmt.Errorf("env file %s not found", envFile.Path)
|
||||
}
|
||||
continue
|
||||
}
|
||||
findings, err := scan.ScanFile(envFile.Path)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to scan env file %s: %w", envFile.Path, err)
|
||||
}
|
||||
allFindings = append(allFindings, findings...)
|
||||
}
|
||||
return allFindings, nil
|
||||
}
|
||||
|
||||
// scanFiles scans file-based resources (configs, secrets) for sensitive data,
|
||||
// ignoring resources not defined by a file
|
||||
func scanFiles(scan secrets.Scanner, kind string, paths []string) ([]secrets.DetectedSecret, error) {
|
||||
var allFindings []secrets.DetectedSecret
|
||||
for _, path := range paths {
|
||||
if path == "" {
|
||||
continue
|
||||
}
|
||||
findings, err := scan.ScanFile(path)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to scan %s file %s: %w", kind, path, err)
|
||||
}
|
||||
allFindings = append(allFindings, findings...)
|
||||
}
|
||||
return allFindings, nil
|
||||
}
|
||||
|
||||
func composeFileAsByteReader(ctx context.Context, filePath string, project *types.Project) (io.Reader, error) {
|
||||
base, err := loadUnresolvedFile(ctx, project, filePath)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to load compose file %s: %w", filePath, err)
|
||||
}
|
||||
in, err := base.MarshalYAML()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return bytes.NewBuffer(in), nil
|
||||
}
|
||||
|
|
@ -14,9 +14,10 @@
|
|||
limitations under the License.
|
||||
*/
|
||||
|
||||
package compose
|
||||
package publish
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"errors"
|
||||
"fmt"
|
||||
|
|
@ -31,17 +32,58 @@ import (
|
|||
"github.com/distribution/reference"
|
||||
"github.com/docker/cli/cli/config/configfile"
|
||||
"github.com/google/go-cmp/cmp"
|
||||
"github.com/moby/moby/api/types/registry"
|
||||
registrytypes "github.com/moby/moby/api/types/registry"
|
||||
"github.com/moby/moby/client"
|
||||
"github.com/opencontainers/go-digest"
|
||||
v1 "github.com/opencontainers/image-spec/specs-go/v1"
|
||||
"go.uber.org/mock/gomock"
|
||||
"gotest.tools/v3/assert"
|
||||
|
||||
"github.com/docker/compose/v5/internal"
|
||||
"github.com/docker/compose/v5/internal/progress"
|
||||
"github.com/docker/compose/v5/internal/registry"
|
||||
"github.com/docker/compose/v5/pkg/api"
|
||||
"github.com/docker/compose/v5/pkg/mocks"
|
||||
)
|
||||
|
||||
// fakeBackend is the engine of the publish tests: it answers the prompts with
|
||||
// the given function, resolves digests through its API client the way the
|
||||
// engine does, and ignores the events.
|
||||
type fakeBackend struct {
|
||||
apiClient client.APIClient
|
||||
prompt func(message string, defaultValue bool) (bool, error)
|
||||
auth registry.AuthProvider
|
||||
}
|
||||
|
||||
func (f *fakeBackend) APIClient() client.APIClient { return f.apiClient }
|
||||
func (f *fakeBackend) AuthProvider() registry.AuthProvider { return f.auth }
|
||||
func (f *fakeBackend) Events() api.EventProcessor { return progress.Noop{} }
|
||||
func (f *fakeBackend) DryRun() bool { return false }
|
||||
|
||||
func (f *fakeBackend) Prompt(message string, defaultValue bool) (bool, error) {
|
||||
return f.prompt(message, defaultValue)
|
||||
}
|
||||
|
||||
func (f *fakeBackend) ImageDigestResolver(ctx context.Context) func(named reference.Named) (digest.Digest, error) {
|
||||
return func(named reference.Named) (digest.Digest, error) {
|
||||
auth, err := registry.EncodedAuth(named, f.auth)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
inspect, err := f.apiClient.DistributionInspect(ctx, named.String(), client.DistributionInspectOptions{
|
||||
EncodedRegistryAuth: auth,
|
||||
})
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("failed to resolve digest for %s: %w", named.String(), err)
|
||||
}
|
||||
return inspect.Descriptor.Digest, nil
|
||||
}
|
||||
}
|
||||
|
||||
func (f *fakeBackend) Push(context.Context, *types.Project, api.PushOptions) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func Test_createLayers(t *testing.T) {
|
||||
project, err := loader.LoadWithContext(t.Context(), types.ConfigDetails{
|
||||
WorkingDir: "testdata/publish/",
|
||||
|
|
@ -55,7 +97,7 @@ func Test_createLayers(t *testing.T) {
|
|||
assert.NilError(t, err)
|
||||
project.ComposeFiles = []string{"testdata/publish/compose.yaml"}
|
||||
|
||||
service := &composeService{}
|
||||
service := &Service{}
|
||||
layers, err := service.createLayers(t.Context(), project, api.PublishOptions{
|
||||
WithEnvironment: true,
|
||||
})
|
||||
|
|
@ -136,9 +178,7 @@ func Test_preChecks_sensitive_data_detected_decline(t *testing.T) {
|
|||
declined := func(message string, defaultValue bool) (bool, error) {
|
||||
return false, nil
|
||||
}
|
||||
svc := &composeService{
|
||||
prompt: declined,
|
||||
}
|
||||
svc := &Service{backend: &fakeBackend{prompt: declined}}
|
||||
|
||||
accept, err := svc.preChecks(t.Context(), project, api.PublishOptions{})
|
||||
assert.NilError(t, err)
|
||||
|
|
@ -227,7 +267,7 @@ func Test_checkForSensitiveData_optional_env_file_missing(t *testing.T) {
|
|||
},
|
||||
}
|
||||
|
||||
svc := &composeService{}
|
||||
svc := &Service{}
|
||||
findings, err := svc.checkForSensitiveData(t.Context(), project)
|
||||
assert.NilError(t, err, "optional missing env file should not cause error during scan")
|
||||
assert.Equal(t, len(findings), 0)
|
||||
|
|
@ -251,7 +291,7 @@ func Test_checkForSensitiveData_optional_env_file_present(t *testing.T) {
|
|||
},
|
||||
}
|
||||
|
||||
svc := &composeService{}
|
||||
svc := &Service{}
|
||||
findings, err := svc.checkForSensitiveData(t.Context(), project)
|
||||
assert.NilError(t, err)
|
||||
assert.Assert(t, len(findings) > 0, "present optional env file should still be scanned for secrets")
|
||||
|
|
@ -272,7 +312,7 @@ func Test_checkForSensitiveData_required_env_file_missing(t *testing.T) {
|
|||
},
|
||||
}
|
||||
|
||||
svc := &composeService{}
|
||||
svc := &Service{}
|
||||
_, err := svc.checkForSensitiveData(t.Context(), project)
|
||||
assert.ErrorContains(t, err, "not found", "required missing env file should fail")
|
||||
}
|
||||
|
|
@ -630,7 +670,7 @@ services:
|
|||
})
|
||||
|
||||
prompt := &fakePrompt{}
|
||||
svc := &composeService{prompt: prompt.handler}
|
||||
svc := &Service{backend: &fakeBackend{prompt: prompt.handler}}
|
||||
|
||||
err := svc.checkEnvironmentVariables(t.Context(), project, api.PublishOptions{})
|
||||
assert.NilError(t, err)
|
||||
|
|
@ -649,7 +689,7 @@ services:
|
|||
})
|
||||
|
||||
prompt := &fakePrompt{answers: []bool{true}}
|
||||
svc := &composeService{prompt: prompt.handler}
|
||||
svc := &Service{backend: &fakeBackend{prompt: prompt.handler}}
|
||||
|
||||
err := svc.checkEnvironmentVariables(t.Context(), project, api.PublishOptions{})
|
||||
assert.NilError(t, err)
|
||||
|
|
@ -670,7 +710,7 @@ services:
|
|||
})
|
||||
|
||||
prompt := &fakePrompt{answers: []bool{false}}
|
||||
svc := &composeService{prompt: prompt.handler}
|
||||
svc := &Service{backend: &fakeBackend{prompt: prompt.handler}}
|
||||
|
||||
err := svc.checkEnvironmentVariables(t.Context(), project, api.PublishOptions{})
|
||||
assert.Assert(t, errors.Is(err, api.ErrCanceled),
|
||||
|
|
@ -700,7 +740,7 @@ configs:
|
|||
})
|
||||
|
||||
prompt := &fakePrompt{answers: []bool{true}}
|
||||
svc := &composeService{prompt: prompt.handler}
|
||||
svc := &Service{backend: &fakeBackend{prompt: prompt.handler}}
|
||||
|
||||
err := svc.checkEnvironmentVariables(t.Context(), project, api.PublishOptions{WithEnvironment: true})
|
||||
assert.NilError(t, err)
|
||||
|
|
@ -724,7 +764,7 @@ configs:
|
|||
})
|
||||
|
||||
prompt := &fakePrompt{answers: []bool{true, true}}
|
||||
svc := &composeService{prompt: prompt.handler}
|
||||
svc := &Service{backend: &fakeBackend{prompt: prompt.handler}}
|
||||
|
||||
err := svc.checkEnvironmentVariables(t.Context(), project, api.PublishOptions{})
|
||||
assert.NilError(t, err)
|
||||
|
|
@ -752,12 +792,9 @@ func Test_publish_decline_returns_ErrCanceled(t *testing.T) {
|
|||
declined := func(message string, defaultValue bool) (bool, error) {
|
||||
return false, nil
|
||||
}
|
||||
svc := &composeService{
|
||||
prompt: declined,
|
||||
events: progress.Noop{},
|
||||
}
|
||||
svc := &Service{backend: &fakeBackend{prompt: declined}}
|
||||
|
||||
err := svc.publish(t.Context(), project, "docker.io/myorg/myapp:latest", api.PublishOptions{})
|
||||
err := svc.Publish(t.Context(), project, "docker.io/myorg/myapp:latest", api.PublishOptions{})
|
||||
assert.Assert(t, errors.Is(err, api.ErrCanceled),
|
||||
"expected api.ErrCanceled when user declines, got: %v", err)
|
||||
}
|
||||
|
|
@ -766,9 +803,8 @@ func Test_generateImageDigestsOverride_resolvesDependentImages(t *testing.T) {
|
|||
mockCtrl := gomock.NewController(t)
|
||||
defer mockCtrl.Finish()
|
||||
|
||||
apiClient, cli := prepareMocks(mockCtrl)
|
||||
cli.EXPECT().ConfigFile().Return(configfile.New("")).AnyTimes()
|
||||
tested := &composeService{dockerCli: cli}
|
||||
apiClient := mocks.NewMockAPIClient(mockCtrl)
|
||||
tested := &Service{backend: &fakeBackend{apiClient: apiClient, auth: configfile.New("")}}
|
||||
|
||||
// distinct digests per resolved reference, so attaching a digest to the wrong image would fail
|
||||
const (
|
||||
|
|
@ -798,15 +834,15 @@ func Test_generateImageDigestsOverride_resolvesDependentImages(t *testing.T) {
|
|||
// per dependent image
|
||||
apiClient.EXPECT().DistributionInspect(gomock.Any(), "docker.io/library/nginx:latest", gomock.Any()).
|
||||
Return(client.DistributionInspectResult{
|
||||
DistributionInspect: registry.DistributionInspect{Descriptor: v1.Descriptor{Digest: serviceDigest}},
|
||||
DistributionInspect: registrytypes.DistributionInspect{Descriptor: v1.Descriptor{Digest: serviceDigest}},
|
||||
}, nil)
|
||||
apiClient.EXPECT().DistributionInspect(gomock.Any(), "docker.io/library/hookimage:latest", gomock.Any()).
|
||||
Return(client.DistributionInspectResult{
|
||||
DistributionInspect: registry.DistributionInspect{Descriptor: v1.Descriptor{Digest: hookDigest}},
|
||||
DistributionInspect: registrytypes.DistributionInspect{Descriptor: v1.Descriptor{Digest: hookDigest}},
|
||||
}, nil)
|
||||
apiClient.EXPECT().DistributionInspect(gomock.Any(), "docker.io/library/someimage:latest", gomock.Any()).
|
||||
Return(client.DistributionInspectResult{
|
||||
DistributionInspect: registry.DistributionInspect{Descriptor: v1.Descriptor{Digest: volumeDigest}},
|
||||
DistributionInspect: registrytypes.DistributionInspect{Descriptor: v1.Descriptor{Digest: volumeDigest}},
|
||||
}, nil)
|
||||
|
||||
override, err := tested.generateImageDigestsOverride(t.Context(), project)
|
||||
|
|
@ -822,9 +858,8 @@ func Test_generateImageDigestsOverride_resolvesJobImages(t *testing.T) {
|
|||
mockCtrl := gomock.NewController(t)
|
||||
defer mockCtrl.Finish()
|
||||
|
||||
apiClient, cli := prepareMocks(mockCtrl)
|
||||
cli.EXPECT().ConfigFile().Return(configfile.New("")).AnyTimes()
|
||||
tested := &composeService{dockerCli: cli}
|
||||
apiClient := mocks.NewMockAPIClient(mockCtrl)
|
||||
tested := &Service{backend: &fakeBackend{apiClient: apiClient, auth: configfile.New("")}}
|
||||
|
||||
const jobDigest = "sha256:dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd"
|
||||
project := &types.Project{
|
||||
|
|
@ -839,7 +874,7 @@ func Test_generateImageDigestsOverride_resolvesJobImages(t *testing.T) {
|
|||
|
||||
apiClient.EXPECT().DistributionInspect(gomock.Any(), "docker.io/library/migrate:latest", gomock.Any()).
|
||||
Return(client.DistributionInspectResult{
|
||||
DistributionInspect: registry.DistributionInspect{Descriptor: v1.Descriptor{Digest: jobDigest}},
|
||||
DistributionInspect: registrytypes.DistributionInspect{Descriptor: v1.Descriptor{Digest: jobDigest}},
|
||||
}, nil)
|
||||
|
||||
override, err := tested.generateImageDigestsOverride(t.Context(), project)
|
||||
|
|
@ -859,7 +894,7 @@ func Test_checkOnlyBuildSection_rejectsJobWithoutImage(t *testing.T) {
|
|||
},
|
||||
}
|
||||
|
||||
svc := &composeService{}
|
||||
svc := &Service{}
|
||||
ok, err := svc.checkOnlyBuildSection(project)
|
||||
assert.Assert(t, !ok)
|
||||
assert.ErrorContains(t, err, `"migrate"`)
|
||||
|
|
@ -876,7 +911,7 @@ func Test_checkOnlyBuildSection_acceptsJobWithImage(t *testing.T) {
|
|||
},
|
||||
}
|
||||
|
||||
svc := &composeService{}
|
||||
svc := &Service{}
|
||||
ok, err := svc.checkOnlyBuildSection(project)
|
||||
assert.NilError(t, err)
|
||||
assert.Assert(t, ok)
|
||||
|
|
@ -900,7 +935,7 @@ func Test_checkForBindMount_flagsJobBindMount(t *testing.T) {
|
|||
},
|
||||
}
|
||||
|
||||
svc := &composeService{}
|
||||
svc := &Service{}
|
||||
findings := svc.checkForBindMount(project)
|
||||
assert.Equal(t, len(findings["migrate"]), 1)
|
||||
assert.Equal(t, findings["migrate"][0].Source, "/host/data")
|
||||
|
|
@ -924,7 +959,7 @@ func Test_checkForSensitiveData_jobEnvFile(t *testing.T) {
|
|||
},
|
||||
}
|
||||
|
||||
svc := &composeService{}
|
||||
svc := &Service{}
|
||||
findings, err := svc.checkForSensitiveData(t.Context(), project)
|
||||
assert.NilError(t, err)
|
||||
assert.Assert(t, len(findings) > 0, "job env file should be scanned for secrets like a service's")
|
||||
|
|
@ -17,846 +17,56 @@
|
|||
package compose
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"maps"
|
||||
"os"
|
||||
"slices"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
"github.com/DefangLabs/secret-detector/pkg/detectors/keyword"
|
||||
"github.com/DefangLabs/secret-detector/pkg/scanner"
|
||||
"github.com/DefangLabs/secret-detector/pkg/secrets"
|
||||
"github.com/compose-spec/compose-go/v2/loader"
|
||||
"github.com/compose-spec/compose-go/v2/types"
|
||||
"github.com/containerd/containerd/v2/core/remotes"
|
||||
"github.com/distribution/reference"
|
||||
"github.com/moby/moby/client"
|
||||
"github.com/opencontainers/go-digest"
|
||||
"github.com/opencontainers/image-spec/specs-go"
|
||||
v1 "github.com/opencontainers/image-spec/specs-go/v1"
|
||||
"github.com/sirupsen/logrus"
|
||||
|
||||
"github.com/docker/compose/v5/internal/desktop"
|
||||
"github.com/docker/compose/v5/internal/oci"
|
||||
"github.com/docker/compose/v5/internal/publish"
|
||||
"github.com/docker/compose/v5/internal/registry"
|
||||
"github.com/docker/compose/v5/pkg/api"
|
||||
"github.com/docker/compose/v5/pkg/compose/transform"
|
||||
)
|
||||
|
||||
// publish lives in internal/publish: composeService serves it by handing that
|
||||
// package the engine it acts on.
|
||||
|
||||
func (s *composeService) Publish(ctx context.Context, project *types.Project, repository string, options api.PublishOptions) error {
|
||||
return Run(ctx, func(ctx context.Context) error {
|
||||
return s.publish(ctx, project, repository, options)
|
||||
return publish.New(publishBackend{s}).Publish(ctx, project, repository, options)
|
||||
}, "publish", s.events)
|
||||
}
|
||||
|
||||
func (s *composeService) publish(ctx context.Context, project *types.Project, repository string, options api.PublishOptions) error {
|
||||
project, err := project.WithProfiles([]string{"*"})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
accept, err := s.preChecks(ctx, project, options)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if !accept {
|
||||
return api.ErrCanceled
|
||||
}
|
||||
// unexported push, not the public Push: publish already runs inside its
|
||||
// own "publish" Start/Done bracket, and Push would open a second one on
|
||||
// the same shared bus. Quiet:true would also suppress push progress
|
||||
// reporting, not just the bracket, so it's not an option here.
|
||||
err = s.push(ctx, project, api.PushOptions{IgnoreFailures: true, ImageMandatory: true})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
layers, err := s.createLayers(ctx, project, options)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
s.events.On(api.Resource{
|
||||
ID: repository,
|
||||
Text: "publishing",
|
||||
Status: api.Working,
|
||||
})
|
||||
if logrus.IsLevelEnabled(logrus.DebugLevel) {
|
||||
logrus.Debug("publishing layers")
|
||||
for _, layer := range layers {
|
||||
indent, _ := json.MarshalIndent(layer, "", " ")
|
||||
fmt.Println(string(indent))
|
||||
}
|
||||
}
|
||||
didFallback := false
|
||||
if !s.dryRun {
|
||||
didFallback, err = s.pushComposeArtifact(ctx, project, repository, layers, options)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
text, status := "published", api.Done
|
||||
if didFallback {
|
||||
text, status = "published (registry rejected OCI 1.1; fell back to OCI 1.0)", api.Warning
|
||||
}
|
||||
s.events.On(api.Resource{
|
||||
ID: repository,
|
||||
Text: text,
|
||||
Status: status,
|
||||
})
|
||||
return nil
|
||||
// publishBackend is the engine of a composeService, as internal/publish sees it.
|
||||
type publishBackend struct {
|
||||
s *composeService
|
||||
}
|
||||
|
||||
// pushComposeArtifact pushes the compose artifact manifest to the repository,
|
||||
// and the application image index when publishing a full application. The
|
||||
// returned bool reports whether the push fell back from OCI 1.1 to OCI 1.0.
|
||||
func (s *composeService) pushComposeArtifact(ctx context.Context, project *types.Project, repository string, layers []v1.Descriptor, options api.PublishOptions) (bool, error) {
|
||||
named, err := reference.ParseDockerRef(repository)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
|
||||
var insecureRegistries []string
|
||||
if options.InsecureRegistry {
|
||||
insecureRegistries = append(insecureRegistries, reference.Domain(named))
|
||||
}
|
||||
|
||||
resolver := oci.NewResolver(s.authProvider(), desktop.ProxyTransportFor(ctx, s.apiClient()), insecureRegistries...)
|
||||
|
||||
descriptor, didFallback, err := oci.PushManifest(ctx, resolver, named, layers, options.OCIVersion)
|
||||
if err != nil {
|
||||
s.events.On(api.Resource{
|
||||
ID: repository,
|
||||
Text: "publishing",
|
||||
Status: api.Error,
|
||||
})
|
||||
return false, err
|
||||
}
|
||||
|
||||
if options.Application {
|
||||
return didFallback, pushApplicationIndex(ctx, resolver, named, descriptor, project)
|
||||
}
|
||||
return didFallback, nil
|
||||
func (b publishBackend) APIClient() client.APIClient {
|
||||
return b.s.apiClient()
|
||||
}
|
||||
|
||||
// pushApplicationIndex pushes an image index referencing every service image,
|
||||
// so the application can be pulled as a single artifact
|
||||
func pushApplicationIndex(ctx context.Context, resolver remotes.Resolver, named reference.Named, descriptor v1.Descriptor, project *types.Project) error {
|
||||
manifests := []v1.Descriptor{}
|
||||
images := make([]string, 0, len(project.Services)+len(project.Jobs))
|
||||
for _, service := range project.Services {
|
||||
images = append(images, service.Image)
|
||||
}
|
||||
for _, job := range project.Jobs {
|
||||
images = append(images, job.Image)
|
||||
}
|
||||
for _, image := range images {
|
||||
ref, err := reference.ParseDockerRef(image)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
manifest, err := oci.Copy(ctx, resolver, ref, named)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
manifests = append(manifests, manifest)
|
||||
}
|
||||
|
||||
descriptor.Data = nil
|
||||
index, err := json.Marshal(v1.Index{
|
||||
Versioned: specs.Versioned{SchemaVersion: 2},
|
||||
MediaType: v1.MediaTypeImageIndex,
|
||||
Manifests: manifests,
|
||||
Subject: &descriptor,
|
||||
Annotations: map[string]string{
|
||||
"com.docker.compose.version": api.ComposeVersion,
|
||||
},
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
imagesDescriptor := v1.Descriptor{
|
||||
MediaType: v1.MediaTypeImageIndex,
|
||||
ArtifactType: oci.ComposeProjectArtifactType,
|
||||
Digest: digest.FromString(string(index)),
|
||||
Size: int64(len(index)),
|
||||
Annotations: map[string]string{
|
||||
"com.docker.compose.version": api.ComposeVersion,
|
||||
},
|
||||
Data: index,
|
||||
}
|
||||
return oci.Push(ctx, resolver, reference.TrimNamed(named), imagesDescriptor)
|
||||
func (b publishBackend) AuthProvider() registry.AuthProvider {
|
||||
return b.s.authProvider()
|
||||
}
|
||||
|
||||
func (s *composeService) createLayers(ctx context.Context, project *types.Project, options api.PublishOptions) ([]v1.Descriptor, error) {
|
||||
var layers []v1.Descriptor
|
||||
extFiles := map[string]string{}
|
||||
envFiles := map[string]string{}
|
||||
for _, file := range project.ComposeFiles {
|
||||
data, err := processFile(ctx, file, project, extFiles, envFiles)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
layerDescriptor := oci.DescriptorForComposeFile(file, data)
|
||||
layers = append(layers, layerDescriptor)
|
||||
}
|
||||
|
||||
extLayers, err := processExtends(ctx, project, extFiles)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
layers = append(layers, extLayers...)
|
||||
|
||||
if options.WithEnvironment {
|
||||
layers = append(layers, envFileLayers(envFiles)...)
|
||||
}
|
||||
|
||||
if options.ResolveImageDigests {
|
||||
yaml, err := s.generateImageDigestsOverride(ctx, project)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
layerDescriptor := oci.DescriptorForComposeFile("image-digests.yaml", yaml)
|
||||
layers = append(layers, layerDescriptor)
|
||||
}
|
||||
return layers, nil
|
||||
func (b publishBackend) Events() api.EventProcessor {
|
||||
return b.s.events
|
||||
}
|
||||
|
||||
func processExtends(ctx context.Context, project *types.Project, extFiles map[string]string) ([]v1.Descriptor, error) {
|
||||
var layers []v1.Descriptor
|
||||
moreExtFiles := map[string]string{}
|
||||
envFiles := map[string]string{}
|
||||
for xf, hash := range extFiles {
|
||||
data, err := processFile(ctx, xf, project, moreExtFiles, envFiles)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
layerDescriptor := oci.DescriptorForComposeFile(hash, data)
|
||||
layerDescriptor.Annotations["com.docker.compose.extends"] = "true"
|
||||
layers = append(layers, layerDescriptor)
|
||||
}
|
||||
for f, hash := range moreExtFiles {
|
||||
if _, ok := extFiles[f]; ok {
|
||||
delete(moreExtFiles, f)
|
||||
}
|
||||
extFiles[f] = hash
|
||||
}
|
||||
if len(moreExtFiles) > 0 {
|
||||
extLayers, err := processExtends(ctx, project, moreExtFiles)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
layers = append(layers, extLayers...)
|
||||
}
|
||||
return layers, nil
|
||||
func (b publishBackend) DryRun() bool {
|
||||
return b.s.dryRun
|
||||
}
|
||||
|
||||
func processFile(ctx context.Context, file string, project *types.Project, extFiles map[string]string, envFiles map[string]string) ([]byte, error) {
|
||||
f, err := os.ReadFile(file)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
base, err := loader.LoadWithContext(ctx, types.ConfigDetails{
|
||||
WorkingDir: project.WorkingDir,
|
||||
Environment: project.Environment,
|
||||
ConfigFiles: []types.ConfigFile{
|
||||
{
|
||||
Filename: file,
|
||||
Content: f,
|
||||
},
|
||||
},
|
||||
}, func(options *loader.Options) {
|
||||
options.SkipValidation = true
|
||||
options.SkipExtends = true
|
||||
options.SkipConsistencyCheck = true
|
||||
options.ResolvePaths = true
|
||||
options.SkipInclude = true
|
||||
options.Profiles = project.Profiles
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
for name, service := range base.Services {
|
||||
for i, envFile := range service.EnvFiles {
|
||||
// A real stat failure (e.g. permissions) is fatal, but a missing file is not:
|
||||
// the project loader already rejects missing required env files before we get
|
||||
// here, so an absent file at this point is an optional one.
|
||||
_, statErr := os.Stat(envFile.Path)
|
||||
if statErr != nil && !os.IsNotExist(statErr) {
|
||||
return nil, fmt.Errorf("failed to access env file %s: %w", envFile.Path, statErr)
|
||||
}
|
||||
// The hash is derived from the path string alone, so the env_file is always
|
||||
// rewritten to its opaque <hash>.env placeholder, even for a missing optional
|
||||
// file, so the published artifact never leaks the local path. Only files that
|
||||
// exist are registered for upload, mirroring the extends handling below.
|
||||
hash := fmt.Sprintf("%x.env", sha256.Sum256([]byte(envFile.Path)))
|
||||
if statErr == nil {
|
||||
envFiles[envFile.Path] = hash
|
||||
}
|
||||
f, err = transform.ReplaceEnvFile(f, name, i, hash)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
|
||||
if service.Extends == nil {
|
||||
continue
|
||||
}
|
||||
xf := service.Extends.File
|
||||
if xf == "" {
|
||||
continue
|
||||
}
|
||||
if _, err = os.Stat(service.Extends.File); os.IsNotExist(err) {
|
||||
// No local file, while we loaded the project successfully: This is actually a remote resource
|
||||
continue
|
||||
}
|
||||
|
||||
hash := fmt.Sprintf("%x.yaml", sha256.Sum256([]byte(xf)))
|
||||
extFiles[xf] = hash
|
||||
|
||||
f, err = transform.ReplaceExtendsFile(f, name, hash)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
return f, nil
|
||||
func (b publishBackend) Prompt(message string, defaultValue bool) (bool, error) {
|
||||
return b.s.prompt(message, defaultValue)
|
||||
}
|
||||
|
||||
func (s *composeService) generateImageDigestsOverride(ctx context.Context, project *types.Project) ([]byte, error) {
|
||||
project, err := project.WithImagesResolved(imageDigestResolver(ctx, s.authProvider(), s.apiClient()))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
// WithImagesResolved only walks services: dress the jobs as services to
|
||||
// run them through the exact same resolution, then fold the digests back
|
||||
// into a jobs override.
|
||||
if len(project.Jobs) > 0 {
|
||||
jobsAsServices := &types.Project{Services: types.Services{}}
|
||||
for name, job := range project.Jobs {
|
||||
jobsAsServices.Services[name] = types.ServiceConfig{
|
||||
Name: name,
|
||||
ContainerSpec: types.ContainerSpec{Image: job.Image},
|
||||
}
|
||||
}
|
||||
jobsAsServices, err = jobsAsServices.WithImagesResolved(imageDigestResolver(ctx, s.authProvider(), s.apiClient()))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
for name, resolved := range jobsAsServices.Services {
|
||||
job := project.Jobs[name]
|
||||
job.Image = resolved.Image
|
||||
project.Jobs[name] = job
|
||||
}
|
||||
}
|
||||
override := types.Project{
|
||||
Services: types.Services{},
|
||||
Jobs: types.Jobs{},
|
||||
}
|
||||
for name, service := range project.Services {
|
||||
override.Services[name] = types.ServiceConfig{
|
||||
ContainerSpec: types.ContainerSpec{Image: service.Image},
|
||||
}
|
||||
}
|
||||
for name, job := range project.Jobs {
|
||||
override.Jobs[name] = types.JobConfig{
|
||||
ContainerSpec: types.ContainerSpec{Image: job.Image},
|
||||
}
|
||||
}
|
||||
return override.MarshalYAML()
|
||||
func (b publishBackend) ImageDigestResolver(ctx context.Context) func(named reference.Named) (digest.Digest, error) {
|
||||
return imageDigestResolver(ctx, b.s.authProvider(), b.s.apiClient())
|
||||
}
|
||||
|
||||
func (s *composeService) preChecks(ctx context.Context, project *types.Project, options api.PublishOptions) (bool, error) {
|
||||
if ok, err := s.checkOnlyBuildSection(project); !ok || err != nil {
|
||||
return false, err
|
||||
}
|
||||
bindMounts := s.checkForBindMount(project)
|
||||
if len(bindMounts) > 0 {
|
||||
b := strings.Builder{}
|
||||
b.WriteString("you are about to publish bind mounts declaration within your OCI artifact.\n" +
|
||||
"only the bind mount declarations will be added to the OCI artifact (not content)\n" +
|
||||
"please double check that you are not mounting potential user's sensitive directories or data\n")
|
||||
for key, val := range bindMounts {
|
||||
b.WriteString(key)
|
||||
for _, v := range val {
|
||||
b.WriteString(v.String())
|
||||
b.WriteRune('\n')
|
||||
}
|
||||
}
|
||||
b.WriteString("Are you ok to publish these bind mount declarations?")
|
||||
confirm, err := s.prompt(b.String(), false)
|
||||
if err != nil || !confirm {
|
||||
return false, err
|
||||
}
|
||||
}
|
||||
detectedSecrets, err := s.checkForSensitiveData(ctx, project)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
if len(detectedSecrets) > 0 {
|
||||
b := strings.Builder{}
|
||||
b.WriteString("you are about to publish sensitive data within your OCI artifact.\n" +
|
||||
"please double check that you are not leaking sensitive data\n")
|
||||
for _, val := range detectedSecrets {
|
||||
b.WriteString(val.Type)
|
||||
b.WriteRune('\n')
|
||||
fmt.Fprintf(&b, "%q: %s\n", val.Key, val.Value)
|
||||
}
|
||||
b.WriteString("Are you ok to publish these sensitive data?")
|
||||
confirm, err := s.prompt(b.String(), false)
|
||||
if err != nil || !confirm {
|
||||
return false, err
|
||||
}
|
||||
}
|
||||
err = s.checkEnvironmentVariables(ctx, project, options)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
return true, nil
|
||||
}
|
||||
|
||||
// envCheckFindings groups everything checkEnvironmentVariables surfaces to
|
||||
// the user during publish pre-checks for env-related leak risks.
|
||||
type envCheckFindings struct {
|
||||
// services maps service name -> findings for that service. Only services
|
||||
// with at least one finding are present.
|
||||
services map[string]*serviceEnvFindings
|
||||
// configsLiteralContent lists configs whose inline `content:` is a literal
|
||||
// (not interpolation). Sorted alphabetically. config.content is decoupled
|
||||
// from --with-env because the flag is documented as controlling environment
|
||||
// variable publishing only.
|
||||
configsLiteralContent []string
|
||||
}
|
||||
|
||||
type serviceEnvFindings struct {
|
||||
hasEnvFile bool
|
||||
// suspiciousKeys is the set of environment variable names whose literal
|
||||
// values look sensitive, as classified by the upstream DefangLabs keyword
|
||||
// detector (password, secret, token, api_key, …). A set is used because
|
||||
// the same service may be visited across multiple compose files during
|
||||
// the extends walk; callers convert to a sorted slice via sortedKeys
|
||||
// when surfacing to the user.
|
||||
suspiciousKeys map[string]struct{}
|
||||
}
|
||||
|
||||
// sortedSuspiciousKeys returns the suspicious env var names alphabetically
|
||||
// sorted for stable output.
|
||||
func (f *serviceEnvFindings) sortedSuspiciousKeys() []string {
|
||||
return slices.Sorted(maps.Keys(f.suspiciousKeys))
|
||||
}
|
||||
|
||||
func (f *envCheckFindings) hasEnvFinding() bool {
|
||||
for _, serviceFindings := range f.services {
|
||||
if serviceFindings.hasEnvFile || len(serviceFindings.suspiciousKeys) > 0 {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// checkEnvironmentVariables walks every compose file that will be serialized
|
||||
// into the OCI artifact (the top-level files plus any local extends parents)
|
||||
// and prompts the user to confirm before publishing:
|
||||
//
|
||||
// 1. service env_file declarations and literal environment values whose key
|
||||
// name looks sensitive (password, secret, token, api_key, …) — silenced
|
||||
// by --with-env;
|
||||
// 2. literal inline config.content — always prompts (decoupled from
|
||||
// --with-env, which is documented to cover env vars only).
|
||||
//
|
||||
// Interpolated values like "${SECRET}" or "$VAR" are preserved as placeholders
|
||||
// in the published YAML and don't leak the resolved value; the keyword
|
||||
// detector's value regex skips them automatically.
|
||||
func (s *composeService) checkEnvironmentVariables(ctx context.Context, project *types.Project, options api.PublishOptions) error {
|
||||
if len(project.ComposeFiles) == 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
findings, err := collectEnvCheckFindings(ctx, project)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if !options.WithEnvironment && findings.hasEnvFinding() {
|
||||
if err := s.confirmOrCancel(buildEnvPromptMessage(findings.services)); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
if len(findings.configsLiteralContent) > 0 {
|
||||
if err := s.confirmOrCancel(buildConfigContentPromptMessage(findings.configsLiteralContent)); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// confirmOrCancel runs an interactive yes/no prompt and returns:
|
||||
// - the prompt's error verbatim, if it failed;
|
||||
// - api.ErrCanceled if the user declined;
|
||||
// - nil if the user accepted.
|
||||
func (s *composeService) confirmOrCancel(message string) error {
|
||||
confirm, err := s.prompt(message, false)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if !confirm {
|
||||
return api.ErrCanceled
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// collectEnvCheckFindings walks every compose file scheduled for publication
|
||||
// (top-level files plus any local extends parents discovered along the way)
|
||||
// and aggregates per-service and per-config findings. The walk mirrors
|
||||
// processExtends so coverage matches what is actually serialized into the OCI
|
||||
// artifact.
|
||||
func collectEnvCheckFindings(ctx context.Context, project *types.Project) (*envCheckFindings, error) {
|
||||
findings := &envCheckFindings{services: map[string]*serviceEnvFindings{}}
|
||||
literalCfgs := map[string]struct{}{}
|
||||
keywordDetector := keyword.NewDetector("0")
|
||||
|
||||
seen := map[string]struct{}{}
|
||||
queue := slices.Clone(project.ComposeFiles)
|
||||
for len(queue) > 0 {
|
||||
file := queue[0]
|
||||
queue = queue[1:]
|
||||
if _, ok := seen[file]; ok {
|
||||
continue
|
||||
}
|
||||
seen[file] = struct{}{}
|
||||
|
||||
unresolved, err := loadUnresolvedFile(ctx, project, file)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to load compose file %s: %w", file, err)
|
||||
}
|
||||
|
||||
for _, service := range unresolved.Services {
|
||||
recordEnvFindings(findings.services, keywordDetector, service.Name, service.ContainerSpec)
|
||||
if parent := localExtendsParent(service); parent != "" {
|
||||
queue = append(queue, parent)
|
||||
}
|
||||
}
|
||||
for name, job := range unresolved.Jobs {
|
||||
// jobs carry environment/env_file like services: same leak surface
|
||||
recordEnvFindings(findings.services, keywordDetector, name, job.ContainerSpec)
|
||||
}
|
||||
for name, config := range unresolved.Configs {
|
||||
// config.Environment is a variable *name* (only the name is
|
||||
// published, not its resolved value) so it is not a leak. Inline
|
||||
// config.Content is what ends up in the artifact. compose-go
|
||||
// enforces that file, environment, and content are mutually
|
||||
// exclusive. The map key is the name as written in the compose
|
||||
// file; config.Name is the project-namespaced version, which is
|
||||
// less helpful when surfaced to the user.
|
||||
if config.Content != "" && configContentLooksLiteral(config.Content, keywordDetector) {
|
||||
literalCfgs[name] = struct{}{}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if len(literalCfgs) > 0 {
|
||||
findings.configsLiteralContent = slices.Sorted(maps.Keys(literalCfgs))
|
||||
}
|
||||
return findings, nil
|
||||
}
|
||||
|
||||
func recordEnvFindings(services map[string]*serviceEnvFindings, detector secrets.Detector, name string, spec types.ContainerSpec) {
|
||||
envValues := map[string]string{}
|
||||
for key, value := range spec.Environment {
|
||||
if value == nil {
|
||||
continue
|
||||
}
|
||||
envValues[key] = replaceDollarEscape(*value)
|
||||
}
|
||||
|
||||
hits, _ := detector.ScanMap(envValues)
|
||||
if len(hits) == 0 && len(spec.EnvFiles) == 0 {
|
||||
return
|
||||
}
|
||||
|
||||
f := services[name]
|
||||
if f == nil {
|
||||
f = &serviceEnvFindings{suspiciousKeys: map[string]struct{}{}}
|
||||
services[name] = f
|
||||
}
|
||||
if len(spec.EnvFiles) > 0 {
|
||||
f.hasEnvFile = true
|
||||
}
|
||||
for _, hit := range hits {
|
||||
f.suspiciousKeys[hit.Key] = struct{}{}
|
||||
}
|
||||
}
|
||||
|
||||
// configContentLooksLiteral returns true when the inline config.content has
|
||||
// a literal portion that would be published as-is, leaking the value to
|
||||
// consumers of the OCI artifact.
|
||||
//
|
||||
// We piggyback on the keyword detector's value regex (`[^${\s].+[^}\s]`) by
|
||||
// passing a fake "password" key to ScanMap — the regex isn't exported
|
||||
// directly, only via the key+value match path. The regex excludes values
|
||||
// starting with `$` (`${VAR}`/`$VAR` interpolation), ending with `}`
|
||||
// (templates like `key=${SECRET}`), or shorter than 3 chars, which neatly
|
||||
// matches our notion of "looks like a template, not a literal".
|
||||
func configContentLooksLiteral(content string, detector secrets.Detector) bool {
|
||||
hits, _ := detector.ScanMap(map[string]string{"password": replaceDollarEscape(content)})
|
||||
return len(hits) > 0
|
||||
}
|
||||
|
||||
// replaceDollarEscape substitutes the compose-spec `$$` escape (which
|
||||
// represents a literal `$` in the resolved value) with a placeholder. The
|
||||
// placeholder is `X` rather than `$` because the keyword detector's value
|
||||
// regex excludes any value beginning with `$`; using `$` would mask the
|
||||
// literal we're trying to flag. Any non-special char would do — we picked
|
||||
// `X` for readability.
|
||||
func replaceDollarEscape(value string) string {
|
||||
return strings.ReplaceAll(value, "$$", "X")
|
||||
}
|
||||
|
||||
// localExtendsParent returns the path of an extends parent file that exists on
|
||||
// disk, or "" when the service does not extend or extends a remote resource.
|
||||
func localExtendsParent(service types.ServiceConfig) string {
|
||||
if service.Extends == nil || service.Extends.File == "" {
|
||||
return ""
|
||||
}
|
||||
if _, err := os.Stat(service.Extends.File); err != nil {
|
||||
return ""
|
||||
}
|
||||
return service.Extends.File
|
||||
}
|
||||
|
||||
func buildEnvPromptMessage(services map[string]*serviceEnvFindings) string {
|
||||
var b strings.Builder
|
||||
b.WriteString("you are about to publish env-related declarations within your OCI artifact.\n")
|
||||
b.WriteString("env_file paths and literal values for sensitive-looking keys are embedded as-is in the published YAML;\n")
|
||||
b.WriteString("interpolated values like \"${VAR}\" are kept symbolic and have already been excluded.\n")
|
||||
for _, name := range slices.Sorted(maps.Keys(services)) {
|
||||
f := services[name]
|
||||
// name may be a service or a job: findings.services carries both.
|
||||
if f.hasEnvFile {
|
||||
fmt.Fprintf(&b, " %q: env_file declared\n", name)
|
||||
}
|
||||
if keys := f.sortedSuspiciousKeys(); len(keys) > 0 {
|
||||
quoted := make([]string, len(keys))
|
||||
for i, k := range keys {
|
||||
quoted[i] = strconv.Quote(k)
|
||||
}
|
||||
fmt.Fprintf(&b, " %q: literal value for %s\n", name, strings.Join(quoted, ", "))
|
||||
}
|
||||
}
|
||||
b.WriteString("Use --with-env to silence this prompt and always publish env declarations.\n")
|
||||
b.WriteString("Are you ok to publish these env declarations?")
|
||||
return b.String()
|
||||
}
|
||||
|
||||
func buildConfigContentPromptMessage(configs []string) string {
|
||||
var b strings.Builder
|
||||
b.WriteString("you are about to publish literal inline config content within your OCI artifact.\n")
|
||||
for _, name := range configs {
|
||||
fmt.Fprintf(&b, " config %q\n", name)
|
||||
}
|
||||
b.WriteString("Are you ok to publish these config contents?")
|
||||
return b.String()
|
||||
}
|
||||
|
||||
// loadUnresolvedFile loads a single compose file with interpolation and
|
||||
// environment resolution skipped, so callers can inspect raw user-provided
|
||||
// values. Used by both checkEnvironmentVariables and composeFileAsByteReader.
|
||||
func loadUnresolvedFile(ctx context.Context, project *types.Project, filePath string) (*types.Project, error) {
|
||||
return loader.LoadWithContext(ctx, types.ConfigDetails{
|
||||
WorkingDir: project.WorkingDir,
|
||||
Environment: project.Environment,
|
||||
ConfigFiles: []types.ConfigFile{{Filename: filePath}},
|
||||
}, func(options *loader.Options) {
|
||||
options.SkipValidation = true
|
||||
options.SkipExtends = true
|
||||
options.SkipConsistencyCheck = true
|
||||
options.ResolvePaths = true
|
||||
// SkipInclude mirrors processFile: include directives stay symbolic in
|
||||
// the published artifact, so included content must not be inspected
|
||||
// here either (otherwise we'd flag literals that never ship).
|
||||
options.SkipInclude = true
|
||||
options.SkipInterpolation = true
|
||||
options.SkipResolveEnvironment = true
|
||||
options.Profiles = project.Profiles
|
||||
})
|
||||
}
|
||||
|
||||
func envFileLayers(files map[string]string) []v1.Descriptor {
|
||||
var layers []v1.Descriptor
|
||||
for file, hash := range files {
|
||||
f, err := os.ReadFile(file)
|
||||
if err != nil {
|
||||
// if we can't read the file, skip to the next one
|
||||
continue
|
||||
}
|
||||
layerDescriptor := oci.DescriptorForEnvFile(hash, f)
|
||||
layers = append(layers, layerDescriptor)
|
||||
}
|
||||
return layers
|
||||
}
|
||||
|
||||
func (s *composeService) checkOnlyBuildSection(project *types.Project) (bool, error) {
|
||||
errorList := []string{}
|
||||
for _, service := range project.Services {
|
||||
if service.Image == "" && service.Build != nil {
|
||||
errorList = append(errorList, service.Name)
|
||||
}
|
||||
}
|
||||
for name, job := range project.Jobs {
|
||||
if job.Image == "" && job.Build != nil {
|
||||
errorList = append(errorList, name)
|
||||
}
|
||||
}
|
||||
if len(errorList) > 0 {
|
||||
var errMsg strings.Builder
|
||||
errMsg.WriteString("your Compose stack cannot be published as it only contains a build section for service(s):\n")
|
||||
for _, serviceInError := range errorList {
|
||||
fmt.Fprintf(&errMsg, "- %q\n", serviceInError)
|
||||
}
|
||||
return false, errors.New(errMsg.String())
|
||||
}
|
||||
return true, nil
|
||||
}
|
||||
|
||||
func (s *composeService) checkForBindMount(project *types.Project) map[string][]types.ServiceVolumeConfig {
|
||||
allFindings := map[string][]types.ServiceVolumeConfig{}
|
||||
record := func(name string, volumes []types.ServiceVolumeConfig) {
|
||||
bindMounts := []types.ServiceVolumeConfig{}
|
||||
for _, volume := range volumes {
|
||||
if volume.Type == types.VolumeTypeBind {
|
||||
bindMounts = append(bindMounts, volume)
|
||||
}
|
||||
}
|
||||
if len(bindMounts) > 0 {
|
||||
allFindings[name] = bindMounts
|
||||
}
|
||||
}
|
||||
for serviceName, config := range project.Services {
|
||||
record(serviceName, config.Volumes)
|
||||
}
|
||||
for name, job := range project.Jobs {
|
||||
record(name, job.Volumes)
|
||||
}
|
||||
return allFindings
|
||||
}
|
||||
|
||||
func (s *composeService) checkForSensitiveData(ctx context.Context, project *types.Project) ([]secrets.DetectedSecret, error) {
|
||||
var allFindings []secrets.DetectedSecret
|
||||
scan := scanner.NewDefaultScanner()
|
||||
|
||||
// Check all compose files
|
||||
for _, file := range project.ComposeFiles {
|
||||
in, err := composeFileAsByteReader(ctx, file, project)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
findings, err := scan.ScanReader(in)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to scan compose file %s: %w", file, err)
|
||||
}
|
||||
allFindings = append(allFindings, findings...)
|
||||
}
|
||||
|
||||
// Check env files — jobs declare env_file like services do
|
||||
for _, service := range project.Services {
|
||||
findings, err := scanEnvFiles(scan, service.EnvFiles)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
allFindings = append(allFindings, findings...)
|
||||
}
|
||||
for _, job := range project.Jobs {
|
||||
findings, err := scanEnvFiles(scan, job.EnvFiles)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
allFindings = append(allFindings, findings...)
|
||||
}
|
||||
|
||||
// Check configs defined by files
|
||||
configFiles := make([]string, 0, len(project.Configs))
|
||||
for _, config := range project.Configs {
|
||||
configFiles = append(configFiles, config.File)
|
||||
}
|
||||
findings, err := scanFiles(scan, "config", configFiles)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
allFindings = append(allFindings, findings...)
|
||||
|
||||
// Check secrets defined by files
|
||||
secretFiles := make([]string, 0, len(project.Secrets))
|
||||
for _, secret := range project.Secrets {
|
||||
secretFiles = append(secretFiles, secret.File)
|
||||
}
|
||||
findings, err = scanFiles(scan, "secret", secretFiles)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
allFindings = append(allFindings, findings...)
|
||||
|
||||
return allFindings, nil
|
||||
}
|
||||
|
||||
// scanEnvFiles scans declared env files for sensitive data; a missing env
|
||||
// file is only an error when the declaration requires it
|
||||
func scanEnvFiles(scan secrets.Scanner, envFiles []types.EnvFile) ([]secrets.DetectedSecret, error) {
|
||||
var allFindings []secrets.DetectedSecret
|
||||
for _, envFile := range envFiles {
|
||||
if _, statErr := os.Stat(envFile.Path); statErr != nil {
|
||||
if !os.IsNotExist(statErr) {
|
||||
return nil, fmt.Errorf("failed to access env file %s: %w", envFile.Path, statErr)
|
||||
}
|
||||
if envFile.Required {
|
||||
return nil, fmt.Errorf("env file %s not found", envFile.Path)
|
||||
}
|
||||
continue
|
||||
}
|
||||
findings, err := scan.ScanFile(envFile.Path)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to scan env file %s: %w", envFile.Path, err)
|
||||
}
|
||||
allFindings = append(allFindings, findings...)
|
||||
}
|
||||
return allFindings, nil
|
||||
}
|
||||
|
||||
// scanFiles scans file-based resources (configs, secrets) for sensitive data,
|
||||
// ignoring resources not defined by a file
|
||||
func scanFiles(scan secrets.Scanner, kind string, paths []string) ([]secrets.DetectedSecret, error) {
|
||||
var allFindings []secrets.DetectedSecret
|
||||
for _, path := range paths {
|
||||
if path == "" {
|
||||
continue
|
||||
}
|
||||
findings, err := scan.ScanFile(path)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to scan %s file %s: %w", kind, path, err)
|
||||
}
|
||||
allFindings = append(allFindings, findings...)
|
||||
}
|
||||
return allFindings, nil
|
||||
}
|
||||
|
||||
func composeFileAsByteReader(ctx context.Context, filePath string, project *types.Project) (io.Reader, error) {
|
||||
base, err := loadUnresolvedFile(ctx, project, filePath)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to load compose file %s: %w", filePath, err)
|
||||
}
|
||||
in, err := base.MarshalYAML()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return bytes.NewBuffer(in), nil
|
||||
func (b publishBackend) Push(ctx context.Context, project *types.Project, options api.PushOptions) error {
|
||||
return b.s.push(ctx, project, options)
|
||||
}
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue