refactor(compose): move publish and transform to internal/publish
Some checks are pending
ci / validate (lint) (push) Waiting to run
ci / validate (relay-lint) (push) Waiting to run
ci / validate (relay-test) (push) Waiting to run
ci / validate (validate-docs) (push) Waiting to run
ci / validate (validate-go-mod) (push) Waiting to run
ci / validate (validate-headers) (push) Waiting to run
ci / validate (validate-mocks) (push) Waiting to run
ci / binary (push) Waiting to run
ci / binary-finalize (push) Blocked by required conditions
ci / bin-image-test (push) Waiting to run
ci / relay-image-test (push) Waiting to run
ci / test (push) Waiting to run
ci / e2e (plugin, oldstable, graphdriver) (push) Waiting to run
ci / e2e (standalone, oldstable, graphdriver) (push) Waiting to run
ci / e2e (plugin, stable, containerd) (push) Waiting to run
ci / e2e (plugin, stable, graphdriver) (push) Waiting to run
ci / e2e (standalone, stable, containerd) (push) Waiting to run
ci / e2e (standalone, stable, graphdriver) (push) Waiting to run
ci / coverage (push) Blocked by required conditions
ci / ci-ok (push) Blocked by required conditions
ci / release (push) Blocked by required conditions
merge / bin-image-prepare (push) Waiting to run
merge / bin-image (push) Blocked by required conditions
merge / relay-image (push) Waiting to run
merge / module-image (push) Waiting to run
Scorecards supply-chain security / Scorecards analysis (push) Waiting to run
zizmor / zizmor (push) Waiting to run

Signed-off-by: Nicolas De Loof <nicolas.deloof@gmail.com>
This commit is contained in:
Nicolas De Loof 2026-10-28 08:06:28 +01:00 • committed by Guillaume Lours
parent 5c666993e1
commit ce67a3ec49
8 changed files with 984 additions and 847 deletions

892
internal/publish/publish.go Normal file
View file

@ -0,0 +1,892 @@
/*
Copyright 2020 Docker Compose CLI authors
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
// Package publish implements publishing a Compose application as an OCI
// artifact: the pre-checks on what it would leak, the layers, and the push.
package publish
import (
"bytes"
"context"
"crypto/sha256"
"encoding/json"
"errors"
"fmt"
"io"
"maps"
"os"
"slices"
"strconv"
"strings"
"github.com/DefangLabs/secret-detector/pkg/detectors/keyword"
"github.com/DefangLabs/secret-detector/pkg/scanner"
"github.com/DefangLabs/secret-detector/pkg/secrets"
"github.com/compose-spec/compose-go/v2/loader"
"github.com/compose-spec/compose-go/v2/types"
"github.com/containerd/containerd/v2/core/remotes"
"github.com/distribution/reference"
"github.com/moby/moby/client"
"github.com/opencontainers/go-digest"
"github.com/opencontainers/image-spec/specs-go"
v1 "github.com/opencontainers/image-spec/specs-go/v1"
"github.com/sirupsen/logrus"
"github.com/docker/compose/v5/internal/desktop"
"github.com/docker/compose/v5/internal/oci"
"github.com/docker/compose/v5/internal/publish/transform"
"github.com/docker/compose/v5/internal/registry"
"github.com/docker/compose/v5/pkg/api"
)
// Backend is what publishing needs from the engine it acts on.
type Backend interface {
// APIClient talks to the Docker daemon.
APIClient() client.APIClient
// AuthProvider supplies the registry credentials.
AuthProvider() registry.AuthProvider
// Events receives the progress of the operation.
Events() api.EventProcessor
// DryRun tells whether the operation only reports what it would do.
DryRun() bool
// Prompt asks the user a yes/no question.
Prompt(message string, defaultValue bool) (bool, error)
// ImageDigestResolver resolves an image reference to the digest the
// registry holds for it, the same way pull does.
ImageDigestResolver(ctx context.Context) func(named reference.Named) (digest.Digest, error)
// Push pushes the images built for the project, as part of the operation
// already running: it opens no progress bracket of its own.
Push(ctx context.Context, project *types.Project, options api.PushOptions) error
}
// Service publishes projects on a Backend.
type Service struct {
backend Backend
}
// New creates a Service acting on the given Backend.
func New(backend Backend) *Service {
return &Service{backend: backend}
}
// Publish publishes the project as an OCI artifact to the repository. It does
// not open a progress bracket: the caller wraps it in the "publish" one.
func (s *Service) Publish(ctx context.Context, project *types.Project, repository string, options api.PublishOptions) error {
project, err := project.WithProfiles([]string{"*"})
if err != nil {
return err
}
accept, err := s.preChecks(ctx, project, options)
if err != nil {
return err
}
if !accept {
return api.ErrCanceled
}
// the Backend's Push, not the public one: publish already runs inside its
// own "publish" Start/Done bracket, and Push would open a second one on
// the same shared bus. Quiet:true would also suppress push progress
// reporting, not just the bracket, so it's not an option here.
err = s.backend.Push(ctx, project, api.PushOptions{IgnoreFailures: true, ImageMandatory: true})
if err != nil {
return err
}
layers, err := s.createLayers(ctx, project, options)
if err != nil {
return err
}
s.backend.Events().On(api.Resource{
ID: repository,
Text: "publishing",
Status: api.Working,
})
if logrus.IsLevelEnabled(logrus.DebugLevel) {
logrus.Debug("publishing layers")
for _, layer := range layers {
indent, _ := json.MarshalIndent(layer, "", " ")
fmt.Println(string(indent))
}
}
didFallback := false
if !s.backend.DryRun() {
didFallback, err = s.pushComposeArtifact(ctx, project, repository, layers, options)
if err != nil {
return err
}
}
text, status := "published", api.Done
if didFallback {
text, status = "published (registry rejected OCI 1.1; fell back to OCI 1.0)", api.Warning
}
s.backend.Events().On(api.Resource{
ID: repository,
Text: text,
Status: status,
})
return nil
}
// pushComposeArtifact pushes the compose artifact manifest to the repository,
// and the application image index when publishing a full application. The
// returned bool reports whether the push fell back from OCI 1.1 to OCI 1.0.
func (s *Service) pushComposeArtifact(ctx context.Context, project *types.Project, repository string, layers []v1.Descriptor, options api.PublishOptions) (bool, error) {
named, err := reference.ParseDockerRef(repository)
if err != nil {
return false, err
}
var insecureRegistries []string
if options.InsecureRegistry {
insecureRegistries = append(insecureRegistries, reference.Domain(named))
}
resolver := oci.NewResolver(s.backend.AuthProvider(), desktop.ProxyTransportFor(ctx, s.backend.APIClient()), insecureRegistries...)
descriptor, didFallback, err := oci.PushManifest(ctx, resolver, named, layers, options.OCIVersion)
if err != nil {
s.backend.Events().On(api.Resource{
ID: repository,
Text: "publishing",
Status: api.Error,
})
return false, err
}
if options.Application {
return didFallback, pushApplicationIndex(ctx, resolver, named, descriptor, project)
}
return didFallback, nil
}
// pushApplicationIndex pushes an image index referencing every service image,
// so the application can be pulled as a single artifact
func pushApplicationIndex(ctx context.Context, resolver remotes.Resolver, named reference.Named, descriptor v1.Descriptor, project *types.Project) error {
manifests := []v1.Descriptor{}
images := make([]string, 0, len(project.Services)+len(project.Jobs))
for _, service := range project.Services {
images = append(images, service.Image)
}
for _, job := range project.Jobs {
images = append(images, job.Image)
}
for _, image := range images {
ref, err := reference.ParseDockerRef(image)
if err != nil {
return err
}
manifest, err := oci.Copy(ctx, resolver, ref, named)
if err != nil {
return err
}
manifests = append(manifests, manifest)
}
descriptor.Data = nil
index, err := json.Marshal(v1.Index{
Versioned: specs.Versioned{SchemaVersion: 2},
MediaType: v1.MediaTypeImageIndex,
Manifests: manifests,
Subject: &descriptor,
Annotations: map[string]string{
"com.docker.compose.version": api.ComposeVersion,
},
})
if err != nil {
return err
}
imagesDescriptor := v1.Descriptor{
MediaType: v1.MediaTypeImageIndex,
ArtifactType: oci.ComposeProjectArtifactType,
Digest: digest.FromString(string(index)),
Size: int64(len(index)),
Annotations: map[string]string{
"com.docker.compose.version": api.ComposeVersion,
},
Data: index,
}
return oci.Push(ctx, resolver, reference.TrimNamed(named), imagesDescriptor)
}
func (s *Service) createLayers(ctx context.Context, project *types.Project, options api.PublishOptions) ([]v1.Descriptor, error) {
var layers []v1.Descriptor
extFiles := map[string]string{}
envFiles := map[string]string{}
for _, file := range project.ComposeFiles {
data, err := processFile(ctx, file, project, extFiles, envFiles)
if err != nil {
return nil, err
}
layerDescriptor := oci.DescriptorForComposeFile(file, data)
layers = append(layers, layerDescriptor)
}
extLayers, err := processExtends(ctx, project, extFiles)
if err != nil {
return nil, err
}
layers = append(layers, extLayers...)
if options.WithEnvironment {
layers = append(layers, envFileLayers(envFiles)...)
}
if options.ResolveImageDigests {
yaml, err := s.generateImageDigestsOverride(ctx, project)
if err != nil {
return nil, err
}
layerDescriptor := oci.DescriptorForComposeFile("image-digests.yaml", yaml)
layers = append(layers, layerDescriptor)
}
return layers, nil
}
func processExtends(ctx context.Context, project *types.Project, extFiles map[string]string) ([]v1.Descriptor, error) {
var layers []v1.Descriptor
moreExtFiles := map[string]string{}
envFiles := map[string]string{}
for xf, hash := range extFiles {
data, err := processFile(ctx, xf, project, moreExtFiles, envFiles)
if err != nil {
return nil, err
}
layerDescriptor := oci.DescriptorForComposeFile(hash, data)
layerDescriptor.Annotations["com.docker.compose.extends"] = "true"
layers = append(layers, layerDescriptor)
}
for f, hash := range moreExtFiles {
if _, ok := extFiles[f]; ok {
delete(moreExtFiles, f)
}
extFiles[f] = hash
}
if len(moreExtFiles) > 0 {
extLayers, err := processExtends(ctx, project, moreExtFiles)
if err != nil {
return nil, err
}
layers = append(layers, extLayers...)
}
return layers, nil
}
func processFile(ctx context.Context, file string, project *types.Project, extFiles map[string]string, envFiles map[string]string) ([]byte, error) {
f, err := os.ReadFile(file)
if err != nil {
return nil, err
}
base, err := loader.LoadWithContext(ctx, types.ConfigDetails{
WorkingDir: project.WorkingDir,
Environment: project.Environment,
ConfigFiles: []types.ConfigFile{
{
Filename: file,
Content: f,
},
},
}, func(options *loader.Options) {
options.SkipValidation = true
options.SkipExtends = true
options.SkipConsistencyCheck = true
options.ResolvePaths = true
options.SkipInclude = true
options.Profiles = project.Profiles
})
if err != nil {
return nil, err
}
for name, service := range base.Services {
for i, envFile := range service.EnvFiles {
// A real stat failure (e.g. permissions) is fatal, but a missing file is not:
// the project loader already rejects missing required env files before we get
// here, so an absent file at this point is an optional one.
_, statErr := os.Stat(envFile.Path)
if statErr != nil && !os.IsNotExist(statErr) {
return nil, fmt.Errorf("failed to access env file %s: %w", envFile.Path, statErr)
}
// The hash is derived from the path string alone, so the env_file is always
// rewritten to its opaque <hash>.env placeholder, even for a missing optional
// file, so the published artifact never leaks the local path. Only files that
// exist are registered for upload, mirroring the extends handling below.
hash := fmt.Sprintf("%x.env", sha256.Sum256([]byte(envFile.Path)))
if statErr == nil {
envFiles[envFile.Path] = hash
}
f, err = transform.ReplaceEnvFile(f, name, i, hash)
if err != nil {
return nil, err
}
}
if service.Extends == nil {
continue
}
xf := service.Extends.File
if xf == "" {
continue
}
if _, err = os.Stat(service.Extends.File); os.IsNotExist(err) {
// No local file, while we loaded the project successfully: This is actually a remote resource
continue
}
hash := fmt.Sprintf("%x.yaml", sha256.Sum256([]byte(xf)))
extFiles[xf] = hash
f, err = transform.ReplaceExtendsFile(f, name, hash)
if err != nil {
return nil, err
}
}
return f, nil
}
func (s *Service) generateImageDigestsOverride(ctx context.Context, project *types.Project) ([]byte, error) {
project, err := project.WithImagesResolved(s.backend.ImageDigestResolver(ctx))
if err != nil {
return nil, err
}
// WithImagesResolved only walks services: dress the jobs as services to
// run them through the exact same resolution, then fold the digests back
// into a jobs override.
if len(project.Jobs) > 0 {
jobsAsServices := &types.Project{Services: types.Services{}}
for name, job := range project.Jobs {
jobsAsServices.Services[name] = types.ServiceConfig{
Name: name,
ContainerSpec: types.ContainerSpec{Image: job.Image},
}
}
jobsAsServices, err = jobsAsServices.WithImagesResolved(s.backend.ImageDigestResolver(ctx))
if err != nil {
return nil, err
}
for name, resolved := range jobsAsServices.Services {
job := project.Jobs[name]
job.Image = resolved.Image
project.Jobs[name] = job
}
}
override := types.Project{
Services: types.Services{},
Jobs: types.Jobs{},
}
for name, service := range project.Services {
override.Services[name] = types.ServiceConfig{
ContainerSpec: types.ContainerSpec{Image: service.Image},
}
}
for name, job := range project.Jobs {
override.Jobs[name] = types.JobConfig{
ContainerSpec: types.ContainerSpec{Image: job.Image},
}
}
return override.MarshalYAML()
}
func (s *Service) preChecks(ctx context.Context, project *types.Project, options api.PublishOptions) (bool, error) {
if ok, err := s.checkOnlyBuildSection(project); !ok || err != nil {
return false, err
}
bindMounts := s.checkForBindMount(project)
if len(bindMounts) > 0 {
b := strings.Builder{}
b.WriteString("you are about to publish bind mounts declaration within your OCI artifact.\n" +
"only the bind mount declarations will be added to the OCI artifact (not content)\n" +
"please double check that you are not mounting potential user's sensitive directories or data\n")
for key, val := range bindMounts {
b.WriteString(key)
for _, v := range val {
b.WriteString(v.String())
b.WriteRune('\n')
}
}
b.WriteString("Are you ok to publish these bind mount declarations?")
confirm, err := s.backend.Prompt(b.String(), false)
if err != nil || !confirm {
return false, err
}
}
detectedSecrets, err := s.checkForSensitiveData(ctx, project)
if err != nil {
return false, err
}
if len(detectedSecrets) > 0 {
b := strings.Builder{}
b.WriteString("you are about to publish sensitive data within your OCI artifact.\n" +
"please double check that you are not leaking sensitive data\n")
for _, val := range detectedSecrets {
b.WriteString(val.Type)
b.WriteRune('\n')
fmt.Fprintf(&b, "%q: %s\n", val.Key, val.Value)
}
b.WriteString("Are you ok to publish these sensitive data?")
confirm, err := s.backend.Prompt(b.String(), false)
if err != nil || !confirm {
return false, err
}
}
err = s.checkEnvironmentVariables(ctx, project, options)
if err != nil {
return false, err
}
return true, nil
}
// envCheckFindings groups everything checkEnvironmentVariables surfaces to
// the user during publish pre-checks for env-related leak risks.
type envCheckFindings struct {
// services maps service name -> findings for that service. Only services
// with at least one finding are present.
services map[string]*serviceEnvFindings
// configsLiteralContent lists configs whose inline `content:` is a literal
// (not interpolation). Sorted alphabetically. config.content is decoupled
// from --with-env because the flag is documented as controlling environment
// variable publishing only.
configsLiteralContent []string
}
type serviceEnvFindings struct {
hasEnvFile bool
// suspiciousKeys is the set of environment variable names whose literal
// values look sensitive, as classified by the upstream DefangLabs keyword
// detector (password, secret, token, api_key, …). A set is used because
// the same service may be visited across multiple compose files during
// the extends walk; callers convert to a sorted slice via sortedKeys
// when surfacing to the user.
suspiciousKeys map[string]struct{}
}
// sortedSuspiciousKeys returns the suspicious env var names alphabetically
// sorted for stable output.
func (f *serviceEnvFindings) sortedSuspiciousKeys() []string {
return slices.Sorted(maps.Keys(f.suspiciousKeys))
}
func (f *envCheckFindings) hasEnvFinding() bool {
for _, serviceFindings := range f.services {
if serviceFindings.hasEnvFile || len(serviceFindings.suspiciousKeys) > 0 {
return true
}
}
return false
}
// checkEnvironmentVariables walks every compose file that will be serialized
// into the OCI artifact (the top-level files plus any local extends parents)
// and prompts the user to confirm before publishing:
//
// 1. service env_file declarations and literal environment values whose key
// name looks sensitive (password, secret, token, api_key, …) — silenced
// by --with-env;
// 2. literal inline config.content — always prompts (decoupled from
// --with-env, which is documented to cover env vars only).
//
// Interpolated values like "${SECRET}" or "$VAR" are preserved as placeholders
// in the published YAML and don't leak the resolved value; the keyword
// detector's value regex skips them automatically.
func (s *Service) checkEnvironmentVariables(ctx context.Context, project *types.Project, options api.PublishOptions) error {
if len(project.ComposeFiles) == 0 {
return nil
}
findings, err := collectEnvCheckFindings(ctx, project)
if err != nil {
return err
}
if !options.WithEnvironment && findings.hasEnvFinding() {
if err := s.confirmOrCancel(buildEnvPromptMessage(findings.services)); err != nil {
return err
}
}
if len(findings.configsLiteralContent) > 0 {
if err := s.confirmOrCancel(buildConfigContentPromptMessage(findings.configsLiteralContent)); err != nil {
return err
}
}
return nil
}
// confirmOrCancel runs an interactive yes/no prompt and returns:
// - the prompt's error verbatim, if it failed;
// - api.ErrCanceled if the user declined;
// - nil if the user accepted.
func (s *Service) confirmOrCancel(message string) error {
confirm, err := s.backend.Prompt(message, false)
if err != nil {
return err
}
if !confirm {
return api.ErrCanceled
}
return nil
}
// collectEnvCheckFindings walks every compose file scheduled for publication
// (top-level files plus any local extends parents discovered along the way)
// and aggregates per-service and per-config findings. The walk mirrors
// processExtends so coverage matches what is actually serialized into the OCI
// artifact.
func collectEnvCheckFindings(ctx context.Context, project *types.Project) (*envCheckFindings, error) {
findings := &envCheckFindings{services: map[string]*serviceEnvFindings{}}
literalCfgs := map[string]struct{}{}
keywordDetector := keyword.NewDetector("0")
seen := map[string]struct{}{}
queue := slices.Clone(project.ComposeFiles)
for len(queue) > 0 {
file := queue[0]
queue = queue[1:]
if _, ok := seen[file]; ok {
continue
}
seen[file] = struct{}{}
unresolved, err := loadUnresolvedFile(ctx, project, file)
if err != nil {
return nil, fmt.Errorf("failed to load compose file %s: %w", file, err)
}
for _, service := range unresolved.Services {
recordEnvFindings(findings.services, keywordDetector, service.Name, service.ContainerSpec)
if parent := localExtendsParent(service); parent != "" {
queue = append(queue, parent)
}
}
for name, job := range unresolved.Jobs {
// jobs carry environment/env_file like services: same leak surface
recordEnvFindings(findings.services, keywordDetector, name, job.ContainerSpec)
}
for name, config := range unresolved.Configs {
// config.Environment is a variable *name* (only the name is
// published, not its resolved value) so it is not a leak. Inline
// config.Content is what ends up in the artifact. compose-go
// enforces that file, environment, and content are mutually
// exclusive. The map key is the name as written in the compose
// file; config.Name is the project-namespaced version, which is
// less helpful when surfaced to the user.
if config.Content != "" && configContentLooksLiteral(config.Content, keywordDetector) {
literalCfgs[name] = struct{}{}
}
}
}
if len(literalCfgs) > 0 {
findings.configsLiteralContent = slices.Sorted(maps.Keys(literalCfgs))
}
return findings, nil
}
func recordEnvFindings(services map[string]*serviceEnvFindings, detector secrets.Detector, name string, spec types.ContainerSpec) {
envValues := map[string]string{}
for key, value := range spec.Environment {
if value == nil {
continue
}
envValues[key] = replaceDollarEscape(*value)
}
hits, _ := detector.ScanMap(envValues)
if len(hits) == 0 && len(spec.EnvFiles) == 0 {
return
}
f := services[name]
if f == nil {
f = &serviceEnvFindings{suspiciousKeys: map[string]struct{}{}}
services[name] = f
}
if len(spec.EnvFiles) > 0 {
f.hasEnvFile = true
}
for _, hit := range hits {
f.suspiciousKeys[hit.Key] = struct{}{}
}
}
// configContentLooksLiteral returns true when the inline config.content has
// a literal portion that would be published as-is, leaking the value to
// consumers of the OCI artifact.
//
// We piggyback on the keyword detector's value regex (`[^${\s].+[^}\s]`) by
// passing a fake "password" key to ScanMap — the regex isn't exported
// directly, only via the key+value match path. The regex excludes values
// starting with `$` (`${VAR}`/`$VAR` interpolation), ending with `}`
// (templates like `key=${SECRET}`), or shorter than 3 chars, which neatly
// matches our notion of "looks like a template, not a literal".
func configContentLooksLiteral(content string, detector secrets.Detector) bool {
hits, _ := detector.ScanMap(map[string]string{"password": replaceDollarEscape(content)})
return len(hits) > 0
}
// replaceDollarEscape substitutes the compose-spec `$$` escape (which
// represents a literal `$` in the resolved value) with a placeholder. The
// placeholder is `X` rather than `$` because the keyword detector's value
// regex excludes any value beginning with `$`; using `$` would mask the
// literal we're trying to flag. Any non-special char would do — we picked
// `X` for readability.
func replaceDollarEscape(value string) string {
return strings.ReplaceAll(value, "$$", "X")
}
// localExtendsParent returns the path of an extends parent file that exists on
// disk, or "" when the service does not extend or extends a remote resource.
func localExtendsParent(service types.ServiceConfig) string {
if service.Extends == nil || service.Extends.File == "" {
return ""
}
if _, err := os.Stat(service.Extends.File); err != nil {
return ""
}
return service.Extends.File
}
func buildEnvPromptMessage(services map[string]*serviceEnvFindings) string {
var b strings.Builder
b.WriteString("you are about to publish env-related declarations within your OCI artifact.\n")
b.WriteString("env_file paths and literal values for sensitive-looking keys are embedded as-is in the published YAML;\n")
b.WriteString("interpolated values like \"${VAR}\" are kept symbolic and have already been excluded.\n")
for _, name := range slices.Sorted(maps.Keys(services)) {
f := services[name]
// name may be a service or a job: findings.services carries both.
if f.hasEnvFile {
fmt.Fprintf(&b, " %q: env_file declared\n", name)
}
if keys := f.sortedSuspiciousKeys(); len(keys) > 0 {
quoted := make([]string, len(keys))
for i, k := range keys {
quoted[i] = strconv.Quote(k)
}
fmt.Fprintf(&b, " %q: literal value for %s\n", name, strings.Join(quoted, ", "))
}
}
b.WriteString("Use --with-env to silence this prompt and always publish env declarations.\n")
b.WriteString("Are you ok to publish these env declarations?")
return b.String()
}
func buildConfigContentPromptMessage(configs []string) string {
var b strings.Builder
b.WriteString("you are about to publish literal inline config content within your OCI artifact.\n")
for _, name := range configs {
fmt.Fprintf(&b, " config %q\n", name)
}
b.WriteString("Are you ok to publish these config contents?")
return b.String()
}
// loadUnresolvedFile loads a single compose file with interpolation and
// environment resolution skipped, so callers can inspect raw user-provided
// values. Used by both checkEnvironmentVariables and composeFileAsByteReader.
func loadUnresolvedFile(ctx context.Context, project *types.Project, filePath string) (*types.Project, error) {
return loader.LoadWithContext(ctx, types.ConfigDetails{
WorkingDir: project.WorkingDir,
Environment: project.Environment,
ConfigFiles: []types.ConfigFile{{Filename: filePath}},
}, func(options *loader.Options) {
options.SkipValidation = true
options.SkipExtends = true
options.SkipConsistencyCheck = true
options.ResolvePaths = true
// SkipInclude mirrors processFile: include directives stay symbolic in
// the published artifact, so included content must not be inspected
// here either (otherwise we'd flag literals that never ship).
options.SkipInclude = true
options.SkipInterpolation = true
options.SkipResolveEnvironment = true
options.Profiles = project.Profiles
})
}
func envFileLayers(files map[string]string) []v1.Descriptor {
var layers []v1.Descriptor
for file, hash := range files {
f, err := os.ReadFile(file)
if err != nil {
// if we can't read the file, skip to the next one
continue
}
layerDescriptor := oci.DescriptorForEnvFile(hash, f)
layers = append(layers, layerDescriptor)
}
return layers
}
func (s *Service) checkOnlyBuildSection(project *types.Project) (bool, error) {
errorList := []string{}
for _, service := range project.Services {
if service.Image == "" && service.Build != nil {
errorList = append(errorList, service.Name)
}
}
for name, job := range project.Jobs {
if job.Image == "" && job.Build != nil {
errorList = append(errorList, name)
}
}
if len(errorList) > 0 {
var errMsg strings.Builder
errMsg.WriteString("your Compose stack cannot be published as it only contains a build section for service(s):\n")
for _, serviceInError := range errorList {
fmt.Fprintf(&errMsg, "- %q\n", serviceInError)
}
return false, errors.New(errMsg.String())
}
return true, nil
}
func (s *Service) checkForBindMount(project *types.Project) map[string][]types.ServiceVolumeConfig {
allFindings := map[string][]types.ServiceVolumeConfig{}
record := func(name string, volumes []types.ServiceVolumeConfig) {
bindMounts := []types.ServiceVolumeConfig{}
for _, volume := range volumes {
if volume.Type == types.VolumeTypeBind {
bindMounts = append(bindMounts, volume)
}
}
if len(bindMounts) > 0 {
allFindings[name] = bindMounts
}
}
for serviceName, config := range project.Services {
record(serviceName, config.Volumes)
}
for name, job := range project.Jobs {
record(name, job.Volumes)
}
return allFindings
}
func (s *Service) checkForSensitiveData(ctx context.Context, project *types.Project) ([]secrets.DetectedSecret, error) {
var allFindings []secrets.DetectedSecret
scan := scanner.NewDefaultScanner()
// Check all compose files
for _, file := range project.ComposeFiles {
in, err := composeFileAsByteReader(ctx, file, project)
if err != nil {
return nil, err
}
findings, err := scan.ScanReader(in)
if err != nil {
return nil, fmt.Errorf("failed to scan compose file %s: %w", file, err)
}
allFindings = append(allFindings, findings...)
}
// Check env files — jobs declare env_file like services do
for _, service := range project.Services {
findings, err := scanEnvFiles(scan, service.EnvFiles)
if err != nil {
return nil, err
}
allFindings = append(allFindings, findings...)
}
for _, job := range project.Jobs {
findings, err := scanEnvFiles(scan, job.EnvFiles)
if err != nil {
return nil, err
}
allFindings = append(allFindings, findings...)
}
// Check configs defined by files
configFiles := make([]string, 0, len(project.Configs))
for _, config := range project.Configs {
configFiles = append(configFiles, config.File)
}
findings, err := scanFiles(scan, "config", configFiles)
if err != nil {
return nil, err
}
allFindings = append(allFindings, findings...)
// Check secrets defined by files
secretFiles := make([]string, 0, len(project.Secrets))
for _, secret := range project.Secrets {
secretFiles = append(secretFiles, secret.File)
}
findings, err = scanFiles(scan, "secret", secretFiles)
if err != nil {
return nil, err
}
allFindings = append(allFindings, findings...)
return allFindings, nil
}
// scanEnvFiles scans declared env files for sensitive data; a missing env
// file is only an error when the declaration requires it
func scanEnvFiles(scan secrets.Scanner, envFiles []types.EnvFile) ([]secrets.DetectedSecret, error) {
var allFindings []secrets.DetectedSecret
for _, envFile := range envFiles {
if _, statErr := os.Stat(envFile.Path); statErr != nil {
if !os.IsNotExist(statErr) {
return nil, fmt.Errorf("failed to access env file %s: %w", envFile.Path, statErr)
}
if envFile.Required {
return nil, fmt.Errorf("env file %s not found", envFile.Path)
}
continue
}
findings, err := scan.ScanFile(envFile.Path)
if err != nil {
return nil, fmt.Errorf("failed to scan env file %s: %w", envFile.Path, err)
}
allFindings = append(allFindings, findings...)
}
return allFindings, nil
}
// scanFiles scans file-based resources (configs, secrets) for sensitive data,
// ignoring resources not defined by a file
func scanFiles(scan secrets.Scanner, kind string, paths []string) ([]secrets.DetectedSecret, error) {
var allFindings []secrets.DetectedSecret
for _, path := range paths {
if path == "" {
continue
}
findings, err := scan.ScanFile(path)
if err != nil {
return nil, fmt.Errorf("failed to scan %s file %s: %w", kind, path, err)
}
allFindings = append(allFindings, findings...)
}
return allFindings, nil
}
func composeFileAsByteReader(ctx context.Context, filePath string, project *types.Project) (io.Reader, error) {
base, err := loadUnresolvedFile(ctx, project, filePath)
if err != nil {
return nil, fmt.Errorf("failed to load compose file %s: %w", filePath, err)
}
in, err := base.MarshalYAML()
if err != nil {
return nil, err
}
return bytes.NewBuffer(in), nil
}

View file

@ -14,9 +14,10 @@
limitations under the License.
*/
package compose
package publish
import (
"context"
"crypto/sha256"
"errors"
"fmt"
@ -31,17 +32,58 @@ import (
"github.com/distribution/reference"
"github.com/docker/cli/cli/config/configfile"
"github.com/google/go-cmp/cmp"
"github.com/moby/moby/api/types/registry"
registrytypes "github.com/moby/moby/api/types/registry"
"github.com/moby/moby/client"
"github.com/opencontainers/go-digest"
v1 "github.com/opencontainers/image-spec/specs-go/v1"
"go.uber.org/mock/gomock"
"gotest.tools/v3/assert"
"github.com/docker/compose/v5/internal"
"github.com/docker/compose/v5/internal/progress"
"github.com/docker/compose/v5/internal/registry"
"github.com/docker/compose/v5/pkg/api"
"github.com/docker/compose/v5/pkg/mocks"
)
// fakeBackend is the engine of the publish tests: it answers the prompts with
// the given function, resolves digests through its API client the way the
// engine does, and ignores the events.
type fakeBackend struct {
apiClient client.APIClient
prompt func(message string, defaultValue bool) (bool, error)
auth registry.AuthProvider
}
func (f *fakeBackend) APIClient() client.APIClient { return f.apiClient }
func (f *fakeBackend) AuthProvider() registry.AuthProvider { return f.auth }
func (f *fakeBackend) Events() api.EventProcessor { return progress.Noop{} }
func (f *fakeBackend) DryRun() bool { return false }
func (f *fakeBackend) Prompt(message string, defaultValue bool) (bool, error) {
return f.prompt(message, defaultValue)
}
func (f *fakeBackend) ImageDigestResolver(ctx context.Context) func(named reference.Named) (digest.Digest, error) {
return func(named reference.Named) (digest.Digest, error) {
auth, err := registry.EncodedAuth(named, f.auth)
if err != nil {
return "", err
}
inspect, err := f.apiClient.DistributionInspect(ctx, named.String(), client.DistributionInspectOptions{
EncodedRegistryAuth: auth,
})
if err != nil {
return "", fmt.Errorf("failed to resolve digest for %s: %w", named.String(), err)
}
return inspect.Descriptor.Digest, nil
}
}
func (f *fakeBackend) Push(context.Context, *types.Project, api.PushOptions) error {
return nil
}
func Test_createLayers(t *testing.T) {
project, err := loader.LoadWithContext(t.Context(), types.ConfigDetails{
WorkingDir: "testdata/publish/",
@ -55,7 +97,7 @@ func Test_createLayers(t *testing.T) {
assert.NilError(t, err)
project.ComposeFiles = []string{"testdata/publish/compose.yaml"}
service := &composeService{}
service := &Service{}
layers, err := service.createLayers(t.Context(), project, api.PublishOptions{
WithEnvironment: true,
})
@ -136,9 +178,7 @@ func Test_preChecks_sensitive_data_detected_decline(t *testing.T) {
declined := func(message string, defaultValue bool) (bool, error) {
return false, nil
}
svc := &composeService{
prompt: declined,
}
svc := &Service{backend: &fakeBackend{prompt: declined}}
accept, err := svc.preChecks(t.Context(), project, api.PublishOptions{})
assert.NilError(t, err)
@ -227,7 +267,7 @@ func Test_checkForSensitiveData_optional_env_file_missing(t *testing.T) {
},
}
svc := &composeService{}
svc := &Service{}
findings, err := svc.checkForSensitiveData(t.Context(), project)
assert.NilError(t, err, "optional missing env file should not cause error during scan")
assert.Equal(t, len(findings), 0)
@ -251,7 +291,7 @@ func Test_checkForSensitiveData_optional_env_file_present(t *testing.T) {
},
}
svc := &composeService{}
svc := &Service{}
findings, err := svc.checkForSensitiveData(t.Context(), project)
assert.NilError(t, err)
assert.Assert(t, len(findings) > 0, "present optional env file should still be scanned for secrets")
@ -272,7 +312,7 @@ func Test_checkForSensitiveData_required_env_file_missing(t *testing.T) {
},
}
svc := &composeService{}
svc := &Service{}
_, err := svc.checkForSensitiveData(t.Context(), project)
assert.ErrorContains(t, err, "not found", "required missing env file should fail")
}
@ -630,7 +670,7 @@ services:
})
prompt := &fakePrompt{}
svc := &composeService{prompt: prompt.handler}
svc := &Service{backend: &fakeBackend{prompt: prompt.handler}}
err := svc.checkEnvironmentVariables(t.Context(), project, api.PublishOptions{})
assert.NilError(t, err)
@ -649,7 +689,7 @@ services:
})
prompt := &fakePrompt{answers: []bool{true}}
svc := &composeService{prompt: prompt.handler}
svc := &Service{backend: &fakeBackend{prompt: prompt.handler}}
err := svc.checkEnvironmentVariables(t.Context(), project, api.PublishOptions{})
assert.NilError(t, err)
@ -670,7 +710,7 @@ services:
})
prompt := &fakePrompt{answers: []bool{false}}
svc := &composeService{prompt: prompt.handler}
svc := &Service{backend: &fakeBackend{prompt: prompt.handler}}
err := svc.checkEnvironmentVariables(t.Context(), project, api.PublishOptions{})
assert.Assert(t, errors.Is(err, api.ErrCanceled),
@ -700,7 +740,7 @@ configs:
})
prompt := &fakePrompt{answers: []bool{true}}
svc := &composeService{prompt: prompt.handler}
svc := &Service{backend: &fakeBackend{prompt: prompt.handler}}
err := svc.checkEnvironmentVariables(t.Context(), project, api.PublishOptions{WithEnvironment: true})
assert.NilError(t, err)
@ -724,7 +764,7 @@ configs:
})
prompt := &fakePrompt{answers: []bool{true, true}}
svc := &composeService{prompt: prompt.handler}
svc := &Service{backend: &fakeBackend{prompt: prompt.handler}}
err := svc.checkEnvironmentVariables(t.Context(), project, api.PublishOptions{})
assert.NilError(t, err)
@ -752,12 +792,9 @@ func Test_publish_decline_returns_ErrCanceled(t *testing.T) {
declined := func(message string, defaultValue bool) (bool, error) {
return false, nil
}
svc := &composeService{
prompt: declined,
events: progress.Noop{},
}
svc := &Service{backend: &fakeBackend{prompt: declined}}
err := svc.publish(t.Context(), project, "docker.io/myorg/myapp:latest", api.PublishOptions{})
err := svc.Publish(t.Context(), project, "docker.io/myorg/myapp:latest", api.PublishOptions{})
assert.Assert(t, errors.Is(err, api.ErrCanceled),
"expected api.ErrCanceled when user declines, got: %v", err)
}
@ -766,9 +803,8 @@ func Test_generateImageDigestsOverride_resolvesDependentImages(t *testing.T) {
mockCtrl := gomock.NewController(t)
defer mockCtrl.Finish()
apiClient, cli := prepareMocks(mockCtrl)
cli.EXPECT().ConfigFile().Return(configfile.New("")).AnyTimes()
tested := &composeService{dockerCli: cli}
apiClient := mocks.NewMockAPIClient(mockCtrl)
tested := &Service{backend: &fakeBackend{apiClient: apiClient, auth: configfile.New("")}}
// distinct digests per resolved reference, so attaching a digest to the wrong image would fail
const (
@ -798,15 +834,15 @@ func Test_generateImageDigestsOverride_resolvesDependentImages(t *testing.T) {
// per dependent image
apiClient.EXPECT().DistributionInspect(gomock.Any(), "docker.io/library/nginx:latest", gomock.Any()).
Return(client.DistributionInspectResult{
DistributionInspect: registry.DistributionInspect{Descriptor: v1.Descriptor{Digest: serviceDigest}},
DistributionInspect: registrytypes.DistributionInspect{Descriptor: v1.Descriptor{Digest: serviceDigest}},
}, nil)
apiClient.EXPECT().DistributionInspect(gomock.Any(), "docker.io/library/hookimage:latest", gomock.Any()).
Return(client.DistributionInspectResult{
DistributionInspect: registry.DistributionInspect{Descriptor: v1.Descriptor{Digest: hookDigest}},
DistributionInspect: registrytypes.DistributionInspect{Descriptor: v1.Descriptor{Digest: hookDigest}},
}, nil)
apiClient.EXPECT().DistributionInspect(gomock.Any(), "docker.io/library/someimage:latest", gomock.Any()).
Return(client.DistributionInspectResult{
DistributionInspect: registry.DistributionInspect{Descriptor: v1.Descriptor{Digest: volumeDigest}},
DistributionInspect: registrytypes.DistributionInspect{Descriptor: v1.Descriptor{Digest: volumeDigest}},
}, nil)
override, err := tested.generateImageDigestsOverride(t.Context(), project)
@ -822,9 +858,8 @@ func Test_generateImageDigestsOverride_resolvesJobImages(t *testing.T) {
mockCtrl := gomock.NewController(t)
defer mockCtrl.Finish()
apiClient, cli := prepareMocks(mockCtrl)
cli.EXPECT().ConfigFile().Return(configfile.New("")).AnyTimes()
tested := &composeService{dockerCli: cli}
apiClient := mocks.NewMockAPIClient(mockCtrl)
tested := &Service{backend: &fakeBackend{apiClient: apiClient, auth: configfile.New("")}}
const jobDigest = "sha256:dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd"
project := &types.Project{
@ -839,7 +874,7 @@ func Test_generateImageDigestsOverride_resolvesJobImages(t *testing.T) {
apiClient.EXPECT().DistributionInspect(gomock.Any(), "docker.io/library/migrate:latest", gomock.Any()).
Return(client.DistributionInspectResult{
DistributionInspect: registry.DistributionInspect{Descriptor: v1.Descriptor{Digest: jobDigest}},
DistributionInspect: registrytypes.DistributionInspect{Descriptor: v1.Descriptor{Digest: jobDigest}},
}, nil)
override, err := tested.generateImageDigestsOverride(t.Context(), project)
@ -859,7 +894,7 @@ func Test_checkOnlyBuildSection_rejectsJobWithoutImage(t *testing.T) {
},
}
svc := &composeService{}
svc := &Service{}
ok, err := svc.checkOnlyBuildSection(project)
assert.Assert(t, !ok)
assert.ErrorContains(t, err, `"migrate"`)
@ -876,7 +911,7 @@ func Test_checkOnlyBuildSection_acceptsJobWithImage(t *testing.T) {
},
}
svc := &composeService{}
svc := &Service{}
ok, err := svc.checkOnlyBuildSection(project)
assert.NilError(t, err)
assert.Assert(t, ok)
@ -900,7 +935,7 @@ func Test_checkForBindMount_flagsJobBindMount(t *testing.T) {
},
}
svc := &composeService{}
svc := &Service{}
findings := svc.checkForBindMount(project)
assert.Equal(t, len(findings["migrate"]), 1)
assert.Equal(t, findings["migrate"][0].Source, "/host/data")
@ -924,7 +959,7 @@ func Test_checkForSensitiveData_jobEnvFile(t *testing.T) {
},
}
svc := &composeService{}
svc := &Service{}
findings, err := svc.checkForSensitiveData(t.Context(), project)
assert.NilError(t, err)
assert.Assert(t, len(findings) > 0, "job env file should be scanned for secrets like a service's")

View file

@ -17,846 +17,56 @@
package compose
import (
"bytes"
"context"
"crypto/sha256"
"encoding/json"
"errors"
"fmt"
"io"
"maps"
"os"
"slices"
"strconv"
"strings"
"github.com/DefangLabs/secret-detector/pkg/detectors/keyword"
"github.com/DefangLabs/secret-detector/pkg/scanner"
"github.com/DefangLabs/secret-detector/pkg/secrets"
"github.com/compose-spec/compose-go/v2/loader"
"github.com/compose-spec/compose-go/v2/types"
"github.com/containerd/containerd/v2/core/remotes"
"github.com/distribution/reference"
"github.com/moby/moby/client"
"github.com/opencontainers/go-digest"
"github.com/opencontainers/image-spec/specs-go"
v1 "github.com/opencontainers/image-spec/specs-go/v1"
"github.com/sirupsen/logrus"
"github.com/docker/compose/v5/internal/desktop"
"github.com/docker/compose/v5/internal/oci"
"github.com/docker/compose/v5/internal/publish"
"github.com/docker/compose/v5/internal/registry"
"github.com/docker/compose/v5/pkg/api"
"github.com/docker/compose/v5/pkg/compose/transform"
)
// publish lives in internal/publish: composeService serves it by handing that
// package the engine it acts on.
func (s *composeService) Publish(ctx context.Context, project *types.Project, repository string, options api.PublishOptions) error {
return Run(ctx, func(ctx context.Context) error {
return s.publish(ctx, project, repository, options)
return publish.New(publishBackend{s}).Publish(ctx, project, repository, options)
}, "publish", s.events)
}
func (s *composeService) publish(ctx context.Context, project *types.Project, repository string, options api.PublishOptions) error {
project, err := project.WithProfiles([]string{"*"})
if err != nil {
return err
}
accept, err := s.preChecks(ctx, project, options)
if err != nil {
return err
}
if !accept {
return api.ErrCanceled
}
// unexported push, not the public Push: publish already runs inside its
// own "publish" Start/Done bracket, and Push would open a second one on
// the same shared bus. Quiet:true would also suppress push progress
// reporting, not just the bracket, so it's not an option here.
err = s.push(ctx, project, api.PushOptions{IgnoreFailures: true, ImageMandatory: true})
if err != nil {
return err
}
layers, err := s.createLayers(ctx, project, options)
if err != nil {
return err
}
s.events.On(api.Resource{
ID: repository,
Text: "publishing",
Status: api.Working,
})
if logrus.IsLevelEnabled(logrus.DebugLevel) {
logrus.Debug("publishing layers")
for _, layer := range layers {
indent, _ := json.MarshalIndent(layer, "", " ")
fmt.Println(string(indent))
}
}
didFallback := false
if !s.dryRun {
didFallback, err = s.pushComposeArtifact(ctx, project, repository, layers, options)
if err != nil {
return err
}
}
text, status := "published", api.Done
if didFallback {
text, status = "published (registry rejected OCI 1.1; fell back to OCI 1.0)", api.Warning
}
s.events.On(api.Resource{
ID: repository,
Text: text,
Status: status,
})
return nil
// publishBackend is the engine of a composeService, as internal/publish sees it.
type publishBackend struct {
s *composeService
}
// pushComposeArtifact pushes the compose artifact manifest to the repository,
// and the application image index when publishing a full application. The
// returned bool reports whether the push fell back from OCI 1.1 to OCI 1.0.
func (s *composeService) pushComposeArtifact(ctx context.Context, project *types.Project, repository string, layers []v1.Descriptor, options api.PublishOptions) (bool, error) {
named, err := reference.ParseDockerRef(repository)
if err != nil {
return false, err
}
var insecureRegistries []string
if options.InsecureRegistry {
insecureRegistries = append(insecureRegistries, reference.Domain(named))
}
resolver := oci.NewResolver(s.authProvider(), desktop.ProxyTransportFor(ctx, s.apiClient()), insecureRegistries...)
descriptor, didFallback, err := oci.PushManifest(ctx, resolver, named, layers, options.OCIVersion)
if err != nil {
s.events.On(api.Resource{
ID: repository,
Text: "publishing",
Status: api.Error,
})
return false, err
}
if options.Application {
return didFallback, pushApplicationIndex(ctx, resolver, named, descriptor, project)
}
return didFallback, nil
func (b publishBackend) APIClient() client.APIClient {
return b.s.apiClient()
}
// pushApplicationIndex pushes an image index referencing every service image,
// so the application can be pulled as a single artifact
func pushApplicationIndex(ctx context.Context, resolver remotes.Resolver, named reference.Named, descriptor v1.Descriptor, project *types.Project) error {
manifests := []v1.Descriptor{}
images := make([]string, 0, len(project.Services)+len(project.Jobs))
for _, service := range project.Services {
images = append(images, service.Image)
}
for _, job := range project.Jobs {
images = append(images, job.Image)
}
for _, image := range images {
ref, err := reference.ParseDockerRef(image)
if err != nil {
return err
}
manifest, err := oci.Copy(ctx, resolver, ref, named)
if err != nil {
return err
}
manifests = append(manifests, manifest)
}
descriptor.Data = nil
index, err := json.Marshal(v1.Index{
Versioned: specs.Versioned{SchemaVersion: 2},
MediaType: v1.MediaTypeImageIndex,
Manifests: manifests,
Subject: &descriptor,
Annotations: map[string]string{
"com.docker.compose.version": api.ComposeVersion,
},
})
if err != nil {
return err
}
imagesDescriptor := v1.Descriptor{
MediaType: v1.MediaTypeImageIndex,
ArtifactType: oci.ComposeProjectArtifactType,
Digest: digest.FromString(string(index)),
Size: int64(len(index)),
Annotations: map[string]string{
"com.docker.compose.version": api.ComposeVersion,
},
Data: index,
}
return oci.Push(ctx, resolver, reference.TrimNamed(named), imagesDescriptor)
func (b publishBackend) AuthProvider() registry.AuthProvider {
return b.s.authProvider()
}
func (s *composeService) createLayers(ctx context.Context, project *types.Project, options api.PublishOptions) ([]v1.Descriptor, error) {
var layers []v1.Descriptor
extFiles := map[string]string{}
envFiles := map[string]string{}
for _, file := range project.ComposeFiles {
data, err := processFile(ctx, file, project, extFiles, envFiles)
if err != nil {
return nil, err
}
layerDescriptor := oci.DescriptorForComposeFile(file, data)
layers = append(layers, layerDescriptor)
}
extLayers, err := processExtends(ctx, project, extFiles)
if err != nil {
return nil, err
}
layers = append(layers, extLayers...)
if options.WithEnvironment {
layers = append(layers, envFileLayers(envFiles)...)
}
if options.ResolveImageDigests {
yaml, err := s.generateImageDigestsOverride(ctx, project)
if err != nil {
return nil, err
}
layerDescriptor := oci.DescriptorForComposeFile("image-digests.yaml", yaml)
layers = append(layers, layerDescriptor)
}
return layers, nil
func (b publishBackend) Events() api.EventProcessor {
return b.s.events
}
func processExtends(ctx context.Context, project *types.Project, extFiles map[string]string) ([]v1.Descriptor, error) {
var layers []v1.Descriptor
moreExtFiles := map[string]string{}
envFiles := map[string]string{}
for xf, hash := range extFiles {
data, err := processFile(ctx, xf, project, moreExtFiles, envFiles)
if err != nil {
return nil, err
}
layerDescriptor := oci.DescriptorForComposeFile(hash, data)
layerDescriptor.Annotations["com.docker.compose.extends"] = "true"
layers = append(layers, layerDescriptor)
}
for f, hash := range moreExtFiles {
if _, ok := extFiles[f]; ok {
delete(moreExtFiles, f)
}
extFiles[f] = hash
}
if len(moreExtFiles) > 0 {
extLayers, err := processExtends(ctx, project, moreExtFiles)
if err != nil {
return nil, err
}
layers = append(layers, extLayers...)
}
return layers, nil
func (b publishBackend) DryRun() bool {
return b.s.dryRun
}
func processFile(ctx context.Context, file string, project *types.Project, extFiles map[string]string, envFiles map[string]string) ([]byte, error) {
f, err := os.ReadFile(file)
if err != nil {
return nil, err
}
base, err := loader.LoadWithContext(ctx, types.ConfigDetails{
WorkingDir: project.WorkingDir,
Environment: project.Environment,
ConfigFiles: []types.ConfigFile{
{
Filename: file,
Content: f,
},
},
}, func(options *loader.Options) {
options.SkipValidation = true
options.SkipExtends = true
options.SkipConsistencyCheck = true
options.ResolvePaths = true
options.SkipInclude = true
options.Profiles = project.Profiles
})
if err != nil {
return nil, err
}
for name, service := range base.Services {
for i, envFile := range service.EnvFiles {
// A real stat failure (e.g. permissions) is fatal, but a missing file is not:
// the project loader already rejects missing required env files before we get
// here, so an absent file at this point is an optional one.
_, statErr := os.Stat(envFile.Path)
if statErr != nil && !os.IsNotExist(statErr) {
return nil, fmt.Errorf("failed to access env file %s: %w", envFile.Path, statErr)
}
// The hash is derived from the path string alone, so the env_file is always
// rewritten to its opaque <hash>.env placeholder, even for a missing optional
// file, so the published artifact never leaks the local path. Only files that
// exist are registered for upload, mirroring the extends handling below.
hash := fmt.Sprintf("%x.env", sha256.Sum256([]byte(envFile.Path)))
if statErr == nil {
envFiles[envFile.Path] = hash
}
f, err = transform.ReplaceEnvFile(f, name, i, hash)
if err != nil {
return nil, err
}
}
if service.Extends == nil {
continue
}
xf := service.Extends.File
if xf == "" {
continue
}
if _, err = os.Stat(service.Extends.File); os.IsNotExist(err) {
// No local file, while we loaded the project successfully: This is actually a remote resource
continue
}
hash := fmt.Sprintf("%x.yaml", sha256.Sum256([]byte(xf)))
extFiles[xf] = hash
f, err = transform.ReplaceExtendsFile(f, name, hash)
if err != nil {
return nil, err
}
}
return f, nil
func (b publishBackend) Prompt(message string, defaultValue bool) (bool, error) {
return b.s.prompt(message, defaultValue)
}
func (s *composeService) generateImageDigestsOverride(ctx context.Context, project *types.Project) ([]byte, error) {
project, err := project.WithImagesResolved(imageDigestResolver(ctx, s.authProvider(), s.apiClient()))
if err != nil {
return nil, err
}
// WithImagesResolved only walks services: dress the jobs as services to
// run them through the exact same resolution, then fold the digests back
// into a jobs override.
if len(project.Jobs) > 0 {
jobsAsServices := &types.Project{Services: types.Services{}}
for name, job := range project.Jobs {
jobsAsServices.Services[name] = types.ServiceConfig{
Name: name,
ContainerSpec: types.ContainerSpec{Image: job.Image},
}
}
jobsAsServices, err = jobsAsServices.WithImagesResolved(imageDigestResolver(ctx, s.authProvider(), s.apiClient()))
if err != nil {
return nil, err
}
for name, resolved := range jobsAsServices.Services {
job := project.Jobs[name]
job.Image = resolved.Image
project.Jobs[name] = job
}
}
override := types.Project{
Services: types.Services{},
Jobs: types.Jobs{},
}
for name, service := range project.Services {
override.Services[name] = types.ServiceConfig{
ContainerSpec: types.ContainerSpec{Image: service.Image},
}
}
for name, job := range project.Jobs {
override.Jobs[name] = types.JobConfig{
ContainerSpec: types.ContainerSpec{Image: job.Image},
}
}
return override.MarshalYAML()
func (b publishBackend) ImageDigestResolver(ctx context.Context) func(named reference.Named) (digest.Digest, error) {
return imageDigestResolver(ctx, b.s.authProvider(), b.s.apiClient())
}
func (s *composeService) preChecks(ctx context.Context, project *types.Project, options api.PublishOptions) (bool, error) {
if ok, err := s.checkOnlyBuildSection(project); !ok || err != nil {
return false, err
}
bindMounts := s.checkForBindMount(project)
if len(bindMounts) > 0 {
b := strings.Builder{}
b.WriteString("you are about to publish bind mounts declaration within your OCI artifact.\n" +
"only the bind mount declarations will be added to the OCI artifact (not content)\n" +
"please double check that you are not mounting potential user's sensitive directories or data\n")
for key, val := range bindMounts {
b.WriteString(key)
for _, v := range val {
b.WriteString(v.String())
b.WriteRune('\n')
}
}
b.WriteString("Are you ok to publish these bind mount declarations?")
confirm, err := s.prompt(b.String(), false)
if err != nil || !confirm {
return false, err
}
}
detectedSecrets, err := s.checkForSensitiveData(ctx, project)
if err != nil {
return false, err
}
if len(detectedSecrets) > 0 {
b := strings.Builder{}
b.WriteString("you are about to publish sensitive data within your OCI artifact.\n" +
"please double check that you are not leaking sensitive data\n")
for _, val := range detectedSecrets {
b.WriteString(val.Type)
b.WriteRune('\n')
fmt.Fprintf(&b, "%q: %s\n", val.Key, val.Value)
}
b.WriteString("Are you ok to publish these sensitive data?")
confirm, err := s.prompt(b.String(), false)
if err != nil || !confirm {
return false, err
}
}
err = s.checkEnvironmentVariables(ctx, project, options)
if err != nil {
return false, err
}
return true, nil
}
// envCheckFindings groups everything checkEnvironmentVariables surfaces to
// the user during publish pre-checks for env-related leak risks.
type envCheckFindings struct {
// services maps service name -> findings for that service. Only services
// with at least one finding are present.
services map[string]*serviceEnvFindings
// configsLiteralContent lists configs whose inline `content:` is a literal
// (not interpolation). Sorted alphabetically. config.content is decoupled
// from --with-env because the flag is documented as controlling environment
// variable publishing only.
configsLiteralContent []string
}
type serviceEnvFindings struct {
hasEnvFile bool
// suspiciousKeys is the set of environment variable names whose literal
// values look sensitive, as classified by the upstream DefangLabs keyword
// detector (password, secret, token, api_key, …). A set is used because
// the same service may be visited across multiple compose files during
// the extends walk; callers convert to a sorted slice via sortedKeys
// when surfacing to the user.
suspiciousKeys map[string]struct{}
}
// sortedSuspiciousKeys returns the suspicious env var names alphabetically
// sorted for stable output.
func (f *serviceEnvFindings) sortedSuspiciousKeys() []string {
return slices.Sorted(maps.Keys(f.suspiciousKeys))
}
func (f *envCheckFindings) hasEnvFinding() bool {
for _, serviceFindings := range f.services {
if serviceFindings.hasEnvFile || len(serviceFindings.suspiciousKeys) > 0 {
return true
}
}
return false
}
// checkEnvironmentVariables walks every compose file that will be serialized
// into the OCI artifact (the top-level files plus any local extends parents)
// and prompts the user to confirm before publishing:
//
// 1. service env_file declarations and literal environment values whose key
// name looks sensitive (password, secret, token, api_key, …) — silenced
// by --with-env;
// 2. literal inline config.content — always prompts (decoupled from
// --with-env, which is documented to cover env vars only).
//
// Interpolated values like "${SECRET}" or "$VAR" are preserved as placeholders
// in the published YAML and don't leak the resolved value; the keyword
// detector's value regex skips them automatically.
func (s *composeService) checkEnvironmentVariables(ctx context.Context, project *types.Project, options api.PublishOptions) error {
if len(project.ComposeFiles) == 0 {
return nil
}
findings, err := collectEnvCheckFindings(ctx, project)
if err != nil {
return err
}
if !options.WithEnvironment && findings.hasEnvFinding() {
if err := s.confirmOrCancel(buildEnvPromptMessage(findings.services)); err != nil {
return err
}
}
if len(findings.configsLiteralContent) > 0 {
if err := s.confirmOrCancel(buildConfigContentPromptMessage(findings.configsLiteralContent)); err != nil {
return err
}
}
return nil
}
// confirmOrCancel runs an interactive yes/no prompt and returns:
// - the prompt's error verbatim, if it failed;
// - api.ErrCanceled if the user declined;
// - nil if the user accepted.
func (s *composeService) confirmOrCancel(message string) error {
confirm, err := s.prompt(message, false)
if err != nil {
return err
}
if !confirm {
return api.ErrCanceled
}
return nil
}
// collectEnvCheckFindings walks every compose file scheduled for publication
// (top-level files plus any local extends parents discovered along the way)
// and aggregates per-service and per-config findings. The walk mirrors
// processExtends so coverage matches what is actually serialized into the OCI
// artifact.
func collectEnvCheckFindings(ctx context.Context, project *types.Project) (*envCheckFindings, error) {
findings := &envCheckFindings{services: map[string]*serviceEnvFindings{}}
literalCfgs := map[string]struct{}{}
keywordDetector := keyword.NewDetector("0")
seen := map[string]struct{}{}
queue := slices.Clone(project.ComposeFiles)
for len(queue) > 0 {
file := queue[0]
queue = queue[1:]
if _, ok := seen[file]; ok {
continue
}
seen[file] = struct{}{}
unresolved, err := loadUnresolvedFile(ctx, project, file)
if err != nil {
return nil, fmt.Errorf("failed to load compose file %s: %w", file, err)
}
for _, service := range unresolved.Services {
recordEnvFindings(findings.services, keywordDetector, service.Name, service.ContainerSpec)
if parent := localExtendsParent(service); parent != "" {
queue = append(queue, parent)
}
}
for name, job := range unresolved.Jobs {
// jobs carry environment/env_file like services: same leak surface
recordEnvFindings(findings.services, keywordDetector, name, job.ContainerSpec)
}
for name, config := range unresolved.Configs {
// config.Environment is a variable *name* (only the name is
// published, not its resolved value) so it is not a leak. Inline
// config.Content is what ends up in the artifact. compose-go
// enforces that file, environment, and content are mutually
// exclusive. The map key is the name as written in the compose
// file; config.Name is the project-namespaced version, which is
// less helpful when surfaced to the user.
if config.Content != "" && configContentLooksLiteral(config.Content, keywordDetector) {
literalCfgs[name] = struct{}{}
}
}
}
if len(literalCfgs) > 0 {
findings.configsLiteralContent = slices.Sorted(maps.Keys(literalCfgs))
}
return findings, nil
}
func recordEnvFindings(services map[string]*serviceEnvFindings, detector secrets.Detector, name string, spec types.ContainerSpec) {
envValues := map[string]string{}
for key, value := range spec.Environment {
if value == nil {
continue
}
envValues[key] = replaceDollarEscape(*value)
}
hits, _ := detector.ScanMap(envValues)
if len(hits) == 0 && len(spec.EnvFiles) == 0 {
return
}
f := services[name]
if f == nil {
f = &serviceEnvFindings{suspiciousKeys: map[string]struct{}{}}
services[name] = f
}
if len(spec.EnvFiles) > 0 {
f.hasEnvFile = true
}
for _, hit := range hits {
f.suspiciousKeys[hit.Key] = struct{}{}
}
}
// configContentLooksLiteral returns true when the inline config.content has
// a literal portion that would be published as-is, leaking the value to
// consumers of the OCI artifact.
//
// We piggyback on the keyword detector's value regex (`[^${\s].+[^}\s]`) by
// passing a fake "password" key to ScanMap — the regex isn't exported
// directly, only via the key+value match path. The regex excludes values
// starting with `$` (`${VAR}`/`$VAR` interpolation), ending with `}`
// (templates like `key=${SECRET}`), or shorter than 3 chars, which neatly
// matches our notion of "looks like a template, not a literal".
func configContentLooksLiteral(content string, detector secrets.Detector) bool {
hits, _ := detector.ScanMap(map[string]string{"password": replaceDollarEscape(content)})
return len(hits) > 0
}
// replaceDollarEscape substitutes the compose-spec `$$` escape (which
// represents a literal `$` in the resolved value) with a placeholder. The
// placeholder is `X` rather than `$` because the keyword detector's value
// regex excludes any value beginning with `$`; using `$` would mask the
// literal we're trying to flag. Any non-special char would do — we picked
// `X` for readability.
func replaceDollarEscape(value string) string {
return strings.ReplaceAll(value, "$$", "X")
}
// localExtendsParent returns the path of an extends parent file that exists on
// disk, or "" when the service does not extend or extends a remote resource.
func localExtendsParent(service types.ServiceConfig) string {
if service.Extends == nil || service.Extends.File == "" {
return ""
}
if _, err := os.Stat(service.Extends.File); err != nil {
return ""
}
return service.Extends.File
}
func buildEnvPromptMessage(services map[string]*serviceEnvFindings) string {
var b strings.Builder
b.WriteString("you are about to publish env-related declarations within your OCI artifact.\n")
b.WriteString("env_file paths and literal values for sensitive-looking keys are embedded as-is in the published YAML;\n")
b.WriteString("interpolated values like \"${VAR}\" are kept symbolic and have already been excluded.\n")
for _, name := range slices.Sorted(maps.Keys(services)) {
f := services[name]
// name may be a service or a job: findings.services carries both.
if f.hasEnvFile {
fmt.Fprintf(&b, " %q: env_file declared\n", name)
}
if keys := f.sortedSuspiciousKeys(); len(keys) > 0 {
quoted := make([]string, len(keys))
for i, k := range keys {
quoted[i] = strconv.Quote(k)
}
fmt.Fprintf(&b, " %q: literal value for %s\n", name, strings.Join(quoted, ", "))
}
}
b.WriteString("Use --with-env to silence this prompt and always publish env declarations.\n")
b.WriteString("Are you ok to publish these env declarations?")
return b.String()
}
func buildConfigContentPromptMessage(configs []string) string {
var b strings.Builder
b.WriteString("you are about to publish literal inline config content within your OCI artifact.\n")
for _, name := range configs {
fmt.Fprintf(&b, " config %q\n", name)
}
b.WriteString("Are you ok to publish these config contents?")
return b.String()
}
// loadUnresolvedFile loads a single compose file with interpolation and
// environment resolution skipped, so callers can inspect raw user-provided
// values. Used by both checkEnvironmentVariables and composeFileAsByteReader.
func loadUnresolvedFile(ctx context.Context, project *types.Project, filePath string) (*types.Project, error) {
return loader.LoadWithContext(ctx, types.ConfigDetails{
WorkingDir: project.WorkingDir,
Environment: project.Environment,
ConfigFiles: []types.ConfigFile{{Filename: filePath}},
}, func(options *loader.Options) {
options.SkipValidation = true
options.SkipExtends = true
options.SkipConsistencyCheck = true
options.ResolvePaths = true
// SkipInclude mirrors processFile: include directives stay symbolic in
// the published artifact, so included content must not be inspected
// here either (otherwise we'd flag literals that never ship).
options.SkipInclude = true
options.SkipInterpolation = true
options.SkipResolveEnvironment = true
options.Profiles = project.Profiles
})
}
func envFileLayers(files map[string]string) []v1.Descriptor {
var layers []v1.Descriptor
for file, hash := range files {
f, err := os.ReadFile(file)
if err != nil {
// if we can't read the file, skip to the next one
continue
}
layerDescriptor := oci.DescriptorForEnvFile(hash, f)
layers = append(layers, layerDescriptor)
}
return layers
}
func (s *composeService) checkOnlyBuildSection(project *types.Project) (bool, error) {
errorList := []string{}
for _, service := range project.Services {
if service.Image == "" && service.Build != nil {
errorList = append(errorList, service.Name)
}
}
for name, job := range project.Jobs {
if job.Image == "" && job.Build != nil {
errorList = append(errorList, name)
}
}
if len(errorList) > 0 {
var errMsg strings.Builder
errMsg.WriteString("your Compose stack cannot be published as it only contains a build section for service(s):\n")
for _, serviceInError := range errorList {
fmt.Fprintf(&errMsg, "- %q\n", serviceInError)
}
return false, errors.New(errMsg.String())
}
return true, nil
}
func (s *composeService) checkForBindMount(project *types.Project) map[string][]types.ServiceVolumeConfig {
allFindings := map[string][]types.ServiceVolumeConfig{}
record := func(name string, volumes []types.ServiceVolumeConfig) {
bindMounts := []types.ServiceVolumeConfig{}
for _, volume := range volumes {
if volume.Type == types.VolumeTypeBind {
bindMounts = append(bindMounts, volume)
}
}
if len(bindMounts) > 0 {
allFindings[name] = bindMounts
}
}
for serviceName, config := range project.Services {
record(serviceName, config.Volumes)
}
for name, job := range project.Jobs {
record(name, job.Volumes)
}
return allFindings
}
func (s *composeService) checkForSensitiveData(ctx context.Context, project *types.Project) ([]secrets.DetectedSecret, error) {
var allFindings []secrets.DetectedSecret
scan := scanner.NewDefaultScanner()
// Check all compose files
for _, file := range project.ComposeFiles {
in, err := composeFileAsByteReader(ctx, file, project)
if err != nil {
return nil, err
}
findings, err := scan.ScanReader(in)
if err != nil {
return nil, fmt.Errorf("failed to scan compose file %s: %w", file, err)
}
allFindings = append(allFindings, findings...)
}
// Check env files — jobs declare env_file like services do
for _, service := range project.Services {
findings, err := scanEnvFiles(scan, service.EnvFiles)
if err != nil {
return nil, err
}
allFindings = append(allFindings, findings...)
}
for _, job := range project.Jobs {
findings, err := scanEnvFiles(scan, job.EnvFiles)
if err != nil {
return nil, err
}
allFindings = append(allFindings, findings...)
}
// Check configs defined by files
configFiles := make([]string, 0, len(project.Configs))
for _, config := range project.Configs {
configFiles = append(configFiles, config.File)
}
findings, err := scanFiles(scan, "config", configFiles)
if err != nil {
return nil, err
}
allFindings = append(allFindings, findings...)
// Check secrets defined by files
secretFiles := make([]string, 0, len(project.Secrets))
for _, secret := range project.Secrets {
secretFiles = append(secretFiles, secret.File)
}
findings, err = scanFiles(scan, "secret", secretFiles)
if err != nil {
return nil, err
}
allFindings = append(allFindings, findings...)
return allFindings, nil
}
// scanEnvFiles scans declared env files for sensitive data; a missing env
// file is only an error when the declaration requires it
func scanEnvFiles(scan secrets.Scanner, envFiles []types.EnvFile) ([]secrets.DetectedSecret, error) {
var allFindings []secrets.DetectedSecret
for _, envFile := range envFiles {
if _, statErr := os.Stat(envFile.Path); statErr != nil {
if !os.IsNotExist(statErr) {
return nil, fmt.Errorf("failed to access env file %s: %w", envFile.Path, statErr)
}
if envFile.Required {
return nil, fmt.Errorf("env file %s not found", envFile.Path)
}
continue
}
findings, err := scan.ScanFile(envFile.Path)
if err != nil {
return nil, fmt.Errorf("failed to scan env file %s: %w", envFile.Path, err)
}
allFindings = append(allFindings, findings...)
}
return allFindings, nil
}
// scanFiles scans file-based resources (configs, secrets) for sensitive data,
// ignoring resources not defined by a file
func scanFiles(scan secrets.Scanner, kind string, paths []string) ([]secrets.DetectedSecret, error) {
var allFindings []secrets.DetectedSecret
for _, path := range paths {
if path == "" {
continue
}
findings, err := scan.ScanFile(path)
if err != nil {
return nil, fmt.Errorf("failed to scan %s file %s: %w", kind, path, err)
}
allFindings = append(allFindings, findings...)
}
return allFindings, nil
}
func composeFileAsByteReader(ctx context.Context, filePath string, project *types.Project) (io.Reader, error) {
base, err := loadUnresolvedFile(ctx, project, filePath)
if err != nil {
return nil, fmt.Errorf("failed to load compose file %s: %w", filePath, err)
}
in, err := base.MarshalYAML()
if err != nil {
return nil, err
}
return bytes.NewBuffer(in), nil
func (b publishBackend) Push(ctx context.Context, project *types.Project, options api.PushOptions) error {
return b.s.push(ctx, project, options)
}