diff --git a/internal/publish/publish.go b/internal/publish/publish.go new file mode 100644 index 000000000..c3aeec5ae --- /dev/null +++ b/internal/publish/publish.go @@ -0,0 +1,892 @@ +/* + Copyright 2020 Docker Compose CLI authors + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +// Package publish implements publishing a Compose application as an OCI +// artifact: the pre-checks on what it would leak, the layers, and the push. +package publish + +import ( + "bytes" + "context" + "crypto/sha256" + "encoding/json" + "errors" + "fmt" + "io" + "maps" + "os" + "slices" + "strconv" + "strings" + + "github.com/DefangLabs/secret-detector/pkg/detectors/keyword" + "github.com/DefangLabs/secret-detector/pkg/scanner" + "github.com/DefangLabs/secret-detector/pkg/secrets" + "github.com/compose-spec/compose-go/v2/loader" + "github.com/compose-spec/compose-go/v2/types" + "github.com/containerd/containerd/v2/core/remotes" + "github.com/distribution/reference" + "github.com/moby/moby/client" + "github.com/opencontainers/go-digest" + "github.com/opencontainers/image-spec/specs-go" + v1 "github.com/opencontainers/image-spec/specs-go/v1" + "github.com/sirupsen/logrus" + + "github.com/docker/compose/v5/internal/desktop" + "github.com/docker/compose/v5/internal/oci" + "github.com/docker/compose/v5/internal/publish/transform" + "github.com/docker/compose/v5/internal/registry" + "github.com/docker/compose/v5/pkg/api" +) + +// Backend is what publishing needs from the engine it acts on. +type Backend interface { + // APIClient talks to the Docker daemon. + APIClient() client.APIClient + // AuthProvider supplies the registry credentials. + AuthProvider() registry.AuthProvider + // Events receives the progress of the operation. + Events() api.EventProcessor + // DryRun tells whether the operation only reports what it would do. + DryRun() bool + // Prompt asks the user a yes/no question. + Prompt(message string, defaultValue bool) (bool, error) + // ImageDigestResolver resolves an image reference to the digest the + // registry holds for it, the same way pull does. + ImageDigestResolver(ctx context.Context) func(named reference.Named) (digest.Digest, error) + // Push pushes the images built for the project, as part of the operation + // already running: it opens no progress bracket of its own. + Push(ctx context.Context, project *types.Project, options api.PushOptions) error +} + +// Service publishes projects on a Backend. +type Service struct { + backend Backend +} + +// New creates a Service acting on the given Backend. +func New(backend Backend) *Service { + return &Service{backend: backend} +} + +// Publish publishes the project as an OCI artifact to the repository. It does +// not open a progress bracket: the caller wraps it in the "publish" one. +func (s *Service) Publish(ctx context.Context, project *types.Project, repository string, options api.PublishOptions) error { + project, err := project.WithProfiles([]string{"*"}) + if err != nil { + return err + } + accept, err := s.preChecks(ctx, project, options) + if err != nil { + return err + } + if !accept { + return api.ErrCanceled + } + // the Backend's Push, not the public one: publish already runs inside its + // own "publish" Start/Done bracket, and Push would open a second one on + // the same shared bus. Quiet:true would also suppress push progress + // reporting, not just the bracket, so it's not an option here. + err = s.backend.Push(ctx, project, api.PushOptions{IgnoreFailures: true, ImageMandatory: true}) + if err != nil { + return err + } + + layers, err := s.createLayers(ctx, project, options) + if err != nil { + return err + } + + s.backend.Events().On(api.Resource{ + ID: repository, + Text: "publishing", + Status: api.Working, + }) + if logrus.IsLevelEnabled(logrus.DebugLevel) { + logrus.Debug("publishing layers") + for _, layer := range layers { + indent, _ := json.MarshalIndent(layer, "", " ") + fmt.Println(string(indent)) + } + } + didFallback := false + if !s.backend.DryRun() { + didFallback, err = s.pushComposeArtifact(ctx, project, repository, layers, options) + if err != nil { + return err + } + } + text, status := "published", api.Done + if didFallback { + text, status = "published (registry rejected OCI 1.1; fell back to OCI 1.0)", api.Warning + } + s.backend.Events().On(api.Resource{ + ID: repository, + Text: text, + Status: status, + }) + return nil +} + +// pushComposeArtifact pushes the compose artifact manifest to the repository, +// and the application image index when publishing a full application. The +// returned bool reports whether the push fell back from OCI 1.1 to OCI 1.0. +func (s *Service) pushComposeArtifact(ctx context.Context, project *types.Project, repository string, layers []v1.Descriptor, options api.PublishOptions) (bool, error) { + named, err := reference.ParseDockerRef(repository) + if err != nil { + return false, err + } + + var insecureRegistries []string + if options.InsecureRegistry { + insecureRegistries = append(insecureRegistries, reference.Domain(named)) + } + + resolver := oci.NewResolver(s.backend.AuthProvider(), desktop.ProxyTransportFor(ctx, s.backend.APIClient()), insecureRegistries...) + + descriptor, didFallback, err := oci.PushManifest(ctx, resolver, named, layers, options.OCIVersion) + if err != nil { + s.backend.Events().On(api.Resource{ + ID: repository, + Text: "publishing", + Status: api.Error, + }) + return false, err + } + + if options.Application { + return didFallback, pushApplicationIndex(ctx, resolver, named, descriptor, project) + } + return didFallback, nil +} + +// pushApplicationIndex pushes an image index referencing every service image, +// so the application can be pulled as a single artifact +func pushApplicationIndex(ctx context.Context, resolver remotes.Resolver, named reference.Named, descriptor v1.Descriptor, project *types.Project) error { + manifests := []v1.Descriptor{} + images := make([]string, 0, len(project.Services)+len(project.Jobs)) + for _, service := range project.Services { + images = append(images, service.Image) + } + for _, job := range project.Jobs { + images = append(images, job.Image) + } + for _, image := range images { + ref, err := reference.ParseDockerRef(image) + if err != nil { + return err + } + + manifest, err := oci.Copy(ctx, resolver, ref, named) + if err != nil { + return err + } + manifests = append(manifests, manifest) + } + + descriptor.Data = nil + index, err := json.Marshal(v1.Index{ + Versioned: specs.Versioned{SchemaVersion: 2}, + MediaType: v1.MediaTypeImageIndex, + Manifests: manifests, + Subject: &descriptor, + Annotations: map[string]string{ + "com.docker.compose.version": api.ComposeVersion, + }, + }) + if err != nil { + return err + } + imagesDescriptor := v1.Descriptor{ + MediaType: v1.MediaTypeImageIndex, + ArtifactType: oci.ComposeProjectArtifactType, + Digest: digest.FromString(string(index)), + Size: int64(len(index)), + Annotations: map[string]string{ + "com.docker.compose.version": api.ComposeVersion, + }, + Data: index, + } + return oci.Push(ctx, resolver, reference.TrimNamed(named), imagesDescriptor) +} + +func (s *Service) createLayers(ctx context.Context, project *types.Project, options api.PublishOptions) ([]v1.Descriptor, error) { + var layers []v1.Descriptor + extFiles := map[string]string{} + envFiles := map[string]string{} + for _, file := range project.ComposeFiles { + data, err := processFile(ctx, file, project, extFiles, envFiles) + if err != nil { + return nil, err + } + + layerDescriptor := oci.DescriptorForComposeFile(file, data) + layers = append(layers, layerDescriptor) + } + + extLayers, err := processExtends(ctx, project, extFiles) + if err != nil { + return nil, err + } + layers = append(layers, extLayers...) + + if options.WithEnvironment { + layers = append(layers, envFileLayers(envFiles)...) + } + + if options.ResolveImageDigests { + yaml, err := s.generateImageDigestsOverride(ctx, project) + if err != nil { + return nil, err + } + + layerDescriptor := oci.DescriptorForComposeFile("image-digests.yaml", yaml) + layers = append(layers, layerDescriptor) + } + return layers, nil +} + +func processExtends(ctx context.Context, project *types.Project, extFiles map[string]string) ([]v1.Descriptor, error) { + var layers []v1.Descriptor + moreExtFiles := map[string]string{} + envFiles := map[string]string{} + for xf, hash := range extFiles { + data, err := processFile(ctx, xf, project, moreExtFiles, envFiles) + if err != nil { + return nil, err + } + + layerDescriptor := oci.DescriptorForComposeFile(hash, data) + layerDescriptor.Annotations["com.docker.compose.extends"] = "true" + layers = append(layers, layerDescriptor) + } + for f, hash := range moreExtFiles { + if _, ok := extFiles[f]; ok { + delete(moreExtFiles, f) + } + extFiles[f] = hash + } + if len(moreExtFiles) > 0 { + extLayers, err := processExtends(ctx, project, moreExtFiles) + if err != nil { + return nil, err + } + layers = append(layers, extLayers...) + } + return layers, nil +} + +func processFile(ctx context.Context, file string, project *types.Project, extFiles map[string]string, envFiles map[string]string) ([]byte, error) { + f, err := os.ReadFile(file) + if err != nil { + return nil, err + } + + base, err := loader.LoadWithContext(ctx, types.ConfigDetails{ + WorkingDir: project.WorkingDir, + Environment: project.Environment, + ConfigFiles: []types.ConfigFile{ + { + Filename: file, + Content: f, + }, + }, + }, func(options *loader.Options) { + options.SkipValidation = true + options.SkipExtends = true + options.SkipConsistencyCheck = true + options.ResolvePaths = true + options.SkipInclude = true + options.Profiles = project.Profiles + }) + if err != nil { + return nil, err + } + for name, service := range base.Services { + for i, envFile := range service.EnvFiles { + // A real stat failure (e.g. permissions) is fatal, but a missing file is not: + // the project loader already rejects missing required env files before we get + // here, so an absent file at this point is an optional one. + _, statErr := os.Stat(envFile.Path) + if statErr != nil && !os.IsNotExist(statErr) { + return nil, fmt.Errorf("failed to access env file %s: %w", envFile.Path, statErr) + } + // The hash is derived from the path string alone, so the env_file is always + // rewritten to its opaque .env placeholder, even for a missing optional + // file, so the published artifact never leaks the local path. Only files that + // exist are registered for upload, mirroring the extends handling below. + hash := fmt.Sprintf("%x.env", sha256.Sum256([]byte(envFile.Path))) + if statErr == nil { + envFiles[envFile.Path] = hash + } + f, err = transform.ReplaceEnvFile(f, name, i, hash) + if err != nil { + return nil, err + } + } + + if service.Extends == nil { + continue + } + xf := service.Extends.File + if xf == "" { + continue + } + if _, err = os.Stat(service.Extends.File); os.IsNotExist(err) { + // No local file, while we loaded the project successfully: This is actually a remote resource + continue + } + + hash := fmt.Sprintf("%x.yaml", sha256.Sum256([]byte(xf))) + extFiles[xf] = hash + + f, err = transform.ReplaceExtendsFile(f, name, hash) + if err != nil { + return nil, err + } + } + return f, nil +} + +func (s *Service) generateImageDigestsOverride(ctx context.Context, project *types.Project) ([]byte, error) { + project, err := project.WithImagesResolved(s.backend.ImageDigestResolver(ctx)) + if err != nil { + return nil, err + } + // WithImagesResolved only walks services: dress the jobs as services to + // run them through the exact same resolution, then fold the digests back + // into a jobs override. + if len(project.Jobs) > 0 { + jobsAsServices := &types.Project{Services: types.Services{}} + for name, job := range project.Jobs { + jobsAsServices.Services[name] = types.ServiceConfig{ + Name: name, + ContainerSpec: types.ContainerSpec{Image: job.Image}, + } + } + jobsAsServices, err = jobsAsServices.WithImagesResolved(s.backend.ImageDigestResolver(ctx)) + if err != nil { + return nil, err + } + for name, resolved := range jobsAsServices.Services { + job := project.Jobs[name] + job.Image = resolved.Image + project.Jobs[name] = job + } + } + override := types.Project{ + Services: types.Services{}, + Jobs: types.Jobs{}, + } + for name, service := range project.Services { + override.Services[name] = types.ServiceConfig{ + ContainerSpec: types.ContainerSpec{Image: service.Image}, + } + } + for name, job := range project.Jobs { + override.Jobs[name] = types.JobConfig{ + ContainerSpec: types.ContainerSpec{Image: job.Image}, + } + } + return override.MarshalYAML() +} + +func (s *Service) preChecks(ctx context.Context, project *types.Project, options api.PublishOptions) (bool, error) { + if ok, err := s.checkOnlyBuildSection(project); !ok || err != nil { + return false, err + } + bindMounts := s.checkForBindMount(project) + if len(bindMounts) > 0 { + b := strings.Builder{} + b.WriteString("you are about to publish bind mounts declaration within your OCI artifact.\n" + + "only the bind mount declarations will be added to the OCI artifact (not content)\n" + + "please double check that you are not mounting potential user's sensitive directories or data\n") + for key, val := range bindMounts { + b.WriteString(key) + for _, v := range val { + b.WriteString(v.String()) + b.WriteRune('\n') + } + } + b.WriteString("Are you ok to publish these bind mount declarations?") + confirm, err := s.backend.Prompt(b.String(), false) + if err != nil || !confirm { + return false, err + } + } + detectedSecrets, err := s.checkForSensitiveData(ctx, project) + if err != nil { + return false, err + } + if len(detectedSecrets) > 0 { + b := strings.Builder{} + b.WriteString("you are about to publish sensitive data within your OCI artifact.\n" + + "please double check that you are not leaking sensitive data\n") + for _, val := range detectedSecrets { + b.WriteString(val.Type) + b.WriteRune('\n') + fmt.Fprintf(&b, "%q: %s\n", val.Key, val.Value) + } + b.WriteString("Are you ok to publish these sensitive data?") + confirm, err := s.backend.Prompt(b.String(), false) + if err != nil || !confirm { + return false, err + } + } + err = s.checkEnvironmentVariables(ctx, project, options) + if err != nil { + return false, err + } + return true, nil +} + +// envCheckFindings groups everything checkEnvironmentVariables surfaces to +// the user during publish pre-checks for env-related leak risks. +type envCheckFindings struct { + // services maps service name -> findings for that service. Only services + // with at least one finding are present. + services map[string]*serviceEnvFindings + // configsLiteralContent lists configs whose inline `content:` is a literal + // (not interpolation). Sorted alphabetically. config.content is decoupled + // from --with-env because the flag is documented as controlling environment + // variable publishing only. + configsLiteralContent []string +} + +type serviceEnvFindings struct { + hasEnvFile bool + // suspiciousKeys is the set of environment variable names whose literal + // values look sensitive, as classified by the upstream DefangLabs keyword + // detector (password, secret, token, api_key, …). A set is used because + // the same service may be visited across multiple compose files during + // the extends walk; callers convert to a sorted slice via sortedKeys + // when surfacing to the user. + suspiciousKeys map[string]struct{} +} + +// sortedSuspiciousKeys returns the suspicious env var names alphabetically +// sorted for stable output. +func (f *serviceEnvFindings) sortedSuspiciousKeys() []string { + return slices.Sorted(maps.Keys(f.suspiciousKeys)) +} + +func (f *envCheckFindings) hasEnvFinding() bool { + for _, serviceFindings := range f.services { + if serviceFindings.hasEnvFile || len(serviceFindings.suspiciousKeys) > 0 { + return true + } + } + return false +} + +// checkEnvironmentVariables walks every compose file that will be serialized +// into the OCI artifact (the top-level files plus any local extends parents) +// and prompts the user to confirm before publishing: +// +// 1. service env_file declarations and literal environment values whose key +// name looks sensitive (password, secret, token, api_key, …) — silenced +// by --with-env; +// 2. literal inline config.content — always prompts (decoupled from +// --with-env, which is documented to cover env vars only). +// +// Interpolated values like "${SECRET}" or "$VAR" are preserved as placeholders +// in the published YAML and don't leak the resolved value; the keyword +// detector's value regex skips them automatically. +func (s *Service) checkEnvironmentVariables(ctx context.Context, project *types.Project, options api.PublishOptions) error { + if len(project.ComposeFiles) == 0 { + return nil + } + + findings, err := collectEnvCheckFindings(ctx, project) + if err != nil { + return err + } + + if !options.WithEnvironment && findings.hasEnvFinding() { + if err := s.confirmOrCancel(buildEnvPromptMessage(findings.services)); err != nil { + return err + } + } + + if len(findings.configsLiteralContent) > 0 { + if err := s.confirmOrCancel(buildConfigContentPromptMessage(findings.configsLiteralContent)); err != nil { + return err + } + } + + return nil +} + +// confirmOrCancel runs an interactive yes/no prompt and returns: +// - the prompt's error verbatim, if it failed; +// - api.ErrCanceled if the user declined; +// - nil if the user accepted. +func (s *Service) confirmOrCancel(message string) error { + confirm, err := s.backend.Prompt(message, false) + if err != nil { + return err + } + if !confirm { + return api.ErrCanceled + } + return nil +} + +// collectEnvCheckFindings walks every compose file scheduled for publication +// (top-level files plus any local extends parents discovered along the way) +// and aggregates per-service and per-config findings. The walk mirrors +// processExtends so coverage matches what is actually serialized into the OCI +// artifact. +func collectEnvCheckFindings(ctx context.Context, project *types.Project) (*envCheckFindings, error) { + findings := &envCheckFindings{services: map[string]*serviceEnvFindings{}} + literalCfgs := map[string]struct{}{} + keywordDetector := keyword.NewDetector("0") + + seen := map[string]struct{}{} + queue := slices.Clone(project.ComposeFiles) + for len(queue) > 0 { + file := queue[0] + queue = queue[1:] + if _, ok := seen[file]; ok { + continue + } + seen[file] = struct{}{} + + unresolved, err := loadUnresolvedFile(ctx, project, file) + if err != nil { + return nil, fmt.Errorf("failed to load compose file %s: %w", file, err) + } + + for _, service := range unresolved.Services { + recordEnvFindings(findings.services, keywordDetector, service.Name, service.ContainerSpec) + if parent := localExtendsParent(service); parent != "" { + queue = append(queue, parent) + } + } + for name, job := range unresolved.Jobs { + // jobs carry environment/env_file like services: same leak surface + recordEnvFindings(findings.services, keywordDetector, name, job.ContainerSpec) + } + for name, config := range unresolved.Configs { + // config.Environment is a variable *name* (only the name is + // published, not its resolved value) so it is not a leak. Inline + // config.Content is what ends up in the artifact. compose-go + // enforces that file, environment, and content are mutually + // exclusive. The map key is the name as written in the compose + // file; config.Name is the project-namespaced version, which is + // less helpful when surfaced to the user. + if config.Content != "" && configContentLooksLiteral(config.Content, keywordDetector) { + literalCfgs[name] = struct{}{} + } + } + } + + if len(literalCfgs) > 0 { + findings.configsLiteralContent = slices.Sorted(maps.Keys(literalCfgs)) + } + return findings, nil +} + +func recordEnvFindings(services map[string]*serviceEnvFindings, detector secrets.Detector, name string, spec types.ContainerSpec) { + envValues := map[string]string{} + for key, value := range spec.Environment { + if value == nil { + continue + } + envValues[key] = replaceDollarEscape(*value) + } + + hits, _ := detector.ScanMap(envValues) + if len(hits) == 0 && len(spec.EnvFiles) == 0 { + return + } + + f := services[name] + if f == nil { + f = &serviceEnvFindings{suspiciousKeys: map[string]struct{}{}} + services[name] = f + } + if len(spec.EnvFiles) > 0 { + f.hasEnvFile = true + } + for _, hit := range hits { + f.suspiciousKeys[hit.Key] = struct{}{} + } +} + +// configContentLooksLiteral returns true when the inline config.content has +// a literal portion that would be published as-is, leaking the value to +// consumers of the OCI artifact. +// +// We piggyback on the keyword detector's value regex (`[^${\s].+[^}\s]`) by +// passing a fake "password" key to ScanMap — the regex isn't exported +// directly, only via the key+value match path. The regex excludes values +// starting with `$` (`${VAR}`/`$VAR` interpolation), ending with `}` +// (templates like `key=${SECRET}`), or shorter than 3 chars, which neatly +// matches our notion of "looks like a template, not a literal". +func configContentLooksLiteral(content string, detector secrets.Detector) bool { + hits, _ := detector.ScanMap(map[string]string{"password": replaceDollarEscape(content)}) + return len(hits) > 0 +} + +// replaceDollarEscape substitutes the compose-spec `$$` escape (which +// represents a literal `$` in the resolved value) with a placeholder. The +// placeholder is `X` rather than `$` because the keyword detector's value +// regex excludes any value beginning with `$`; using `$` would mask the +// literal we're trying to flag. Any non-special char would do — we picked +// `X` for readability. +func replaceDollarEscape(value string) string { + return strings.ReplaceAll(value, "$$", "X") +} + +// localExtendsParent returns the path of an extends parent file that exists on +// disk, or "" when the service does not extend or extends a remote resource. +func localExtendsParent(service types.ServiceConfig) string { + if service.Extends == nil || service.Extends.File == "" { + return "" + } + if _, err := os.Stat(service.Extends.File); err != nil { + return "" + } + return service.Extends.File +} + +func buildEnvPromptMessage(services map[string]*serviceEnvFindings) string { + var b strings.Builder + b.WriteString("you are about to publish env-related declarations within your OCI artifact.\n") + b.WriteString("env_file paths and literal values for sensitive-looking keys are embedded as-is in the published YAML;\n") + b.WriteString("interpolated values like \"${VAR}\" are kept symbolic and have already been excluded.\n") + for _, name := range slices.Sorted(maps.Keys(services)) { + f := services[name] + // name may be a service or a job: findings.services carries both. + if f.hasEnvFile { + fmt.Fprintf(&b, " %q: env_file declared\n", name) + } + if keys := f.sortedSuspiciousKeys(); len(keys) > 0 { + quoted := make([]string, len(keys)) + for i, k := range keys { + quoted[i] = strconv.Quote(k) + } + fmt.Fprintf(&b, " %q: literal value for %s\n", name, strings.Join(quoted, ", ")) + } + } + b.WriteString("Use --with-env to silence this prompt and always publish env declarations.\n") + b.WriteString("Are you ok to publish these env declarations?") + return b.String() +} + +func buildConfigContentPromptMessage(configs []string) string { + var b strings.Builder + b.WriteString("you are about to publish literal inline config content within your OCI artifact.\n") + for _, name := range configs { + fmt.Fprintf(&b, " config %q\n", name) + } + b.WriteString("Are you ok to publish these config contents?") + return b.String() +} + +// loadUnresolvedFile loads a single compose file with interpolation and +// environment resolution skipped, so callers can inspect raw user-provided +// values. Used by both checkEnvironmentVariables and composeFileAsByteReader. +func loadUnresolvedFile(ctx context.Context, project *types.Project, filePath string) (*types.Project, error) { + return loader.LoadWithContext(ctx, types.ConfigDetails{ + WorkingDir: project.WorkingDir, + Environment: project.Environment, + ConfigFiles: []types.ConfigFile{{Filename: filePath}}, + }, func(options *loader.Options) { + options.SkipValidation = true + options.SkipExtends = true + options.SkipConsistencyCheck = true + options.ResolvePaths = true + // SkipInclude mirrors processFile: include directives stay symbolic in + // the published artifact, so included content must not be inspected + // here either (otherwise we'd flag literals that never ship). + options.SkipInclude = true + options.SkipInterpolation = true + options.SkipResolveEnvironment = true + options.Profiles = project.Profiles + }) +} + +func envFileLayers(files map[string]string) []v1.Descriptor { + var layers []v1.Descriptor + for file, hash := range files { + f, err := os.ReadFile(file) + if err != nil { + // if we can't read the file, skip to the next one + continue + } + layerDescriptor := oci.DescriptorForEnvFile(hash, f) + layers = append(layers, layerDescriptor) + } + return layers +} + +func (s *Service) checkOnlyBuildSection(project *types.Project) (bool, error) { + errorList := []string{} + for _, service := range project.Services { + if service.Image == "" && service.Build != nil { + errorList = append(errorList, service.Name) + } + } + for name, job := range project.Jobs { + if job.Image == "" && job.Build != nil { + errorList = append(errorList, name) + } + } + if len(errorList) > 0 { + var errMsg strings.Builder + errMsg.WriteString("your Compose stack cannot be published as it only contains a build section for service(s):\n") + for _, serviceInError := range errorList { + fmt.Fprintf(&errMsg, "- %q\n", serviceInError) + } + return false, errors.New(errMsg.String()) + } + return true, nil +} + +func (s *Service) checkForBindMount(project *types.Project) map[string][]types.ServiceVolumeConfig { + allFindings := map[string][]types.ServiceVolumeConfig{} + record := func(name string, volumes []types.ServiceVolumeConfig) { + bindMounts := []types.ServiceVolumeConfig{} + for _, volume := range volumes { + if volume.Type == types.VolumeTypeBind { + bindMounts = append(bindMounts, volume) + } + } + if len(bindMounts) > 0 { + allFindings[name] = bindMounts + } + } + for serviceName, config := range project.Services { + record(serviceName, config.Volumes) + } + for name, job := range project.Jobs { + record(name, job.Volumes) + } + return allFindings +} + +func (s *Service) checkForSensitiveData(ctx context.Context, project *types.Project) ([]secrets.DetectedSecret, error) { + var allFindings []secrets.DetectedSecret + scan := scanner.NewDefaultScanner() + + // Check all compose files + for _, file := range project.ComposeFiles { + in, err := composeFileAsByteReader(ctx, file, project) + if err != nil { + return nil, err + } + + findings, err := scan.ScanReader(in) + if err != nil { + return nil, fmt.Errorf("failed to scan compose file %s: %w", file, err) + } + allFindings = append(allFindings, findings...) + } + + // Check env files — jobs declare env_file like services do + for _, service := range project.Services { + findings, err := scanEnvFiles(scan, service.EnvFiles) + if err != nil { + return nil, err + } + allFindings = append(allFindings, findings...) + } + for _, job := range project.Jobs { + findings, err := scanEnvFiles(scan, job.EnvFiles) + if err != nil { + return nil, err + } + allFindings = append(allFindings, findings...) + } + + // Check configs defined by files + configFiles := make([]string, 0, len(project.Configs)) + for _, config := range project.Configs { + configFiles = append(configFiles, config.File) + } + findings, err := scanFiles(scan, "config", configFiles) + if err != nil { + return nil, err + } + allFindings = append(allFindings, findings...) + + // Check secrets defined by files + secretFiles := make([]string, 0, len(project.Secrets)) + for _, secret := range project.Secrets { + secretFiles = append(secretFiles, secret.File) + } + findings, err = scanFiles(scan, "secret", secretFiles) + if err != nil { + return nil, err + } + allFindings = append(allFindings, findings...) + + return allFindings, nil +} + +// scanEnvFiles scans declared env files for sensitive data; a missing env +// file is only an error when the declaration requires it +func scanEnvFiles(scan secrets.Scanner, envFiles []types.EnvFile) ([]secrets.DetectedSecret, error) { + var allFindings []secrets.DetectedSecret + for _, envFile := range envFiles { + if _, statErr := os.Stat(envFile.Path); statErr != nil { + if !os.IsNotExist(statErr) { + return nil, fmt.Errorf("failed to access env file %s: %w", envFile.Path, statErr) + } + if envFile.Required { + return nil, fmt.Errorf("env file %s not found", envFile.Path) + } + continue + } + findings, err := scan.ScanFile(envFile.Path) + if err != nil { + return nil, fmt.Errorf("failed to scan env file %s: %w", envFile.Path, err) + } + allFindings = append(allFindings, findings...) + } + return allFindings, nil +} + +// scanFiles scans file-based resources (configs, secrets) for sensitive data, +// ignoring resources not defined by a file +func scanFiles(scan secrets.Scanner, kind string, paths []string) ([]secrets.DetectedSecret, error) { + var allFindings []secrets.DetectedSecret + for _, path := range paths { + if path == "" { + continue + } + findings, err := scan.ScanFile(path) + if err != nil { + return nil, fmt.Errorf("failed to scan %s file %s: %w", kind, path, err) + } + allFindings = append(allFindings, findings...) + } + return allFindings, nil +} + +func composeFileAsByteReader(ctx context.Context, filePath string, project *types.Project) (io.Reader, error) { + base, err := loadUnresolvedFile(ctx, project, filePath) + if err != nil { + return nil, fmt.Errorf("failed to load compose file %s: %w", filePath, err) + } + in, err := base.MarshalYAML() + if err != nil { + return nil, err + } + return bytes.NewBuffer(in), nil +} diff --git a/pkg/compose/publish_test.go b/internal/publish/publish_test.go similarity index 89% rename from pkg/compose/publish_test.go rename to internal/publish/publish_test.go index e60bb9558..0ccbe6d8f 100644 --- a/pkg/compose/publish_test.go +++ b/internal/publish/publish_test.go @@ -14,9 +14,10 @@ limitations under the License. */ -package compose +package publish import ( + "context" "crypto/sha256" "errors" "fmt" @@ -31,17 +32,58 @@ import ( "github.com/distribution/reference" "github.com/docker/cli/cli/config/configfile" "github.com/google/go-cmp/cmp" - "github.com/moby/moby/api/types/registry" + registrytypes "github.com/moby/moby/api/types/registry" "github.com/moby/moby/client" + "github.com/opencontainers/go-digest" v1 "github.com/opencontainers/image-spec/specs-go/v1" "go.uber.org/mock/gomock" "gotest.tools/v3/assert" "github.com/docker/compose/v5/internal" "github.com/docker/compose/v5/internal/progress" + "github.com/docker/compose/v5/internal/registry" "github.com/docker/compose/v5/pkg/api" + "github.com/docker/compose/v5/pkg/mocks" ) +// fakeBackend is the engine of the publish tests: it answers the prompts with +// the given function, resolves digests through its API client the way the +// engine does, and ignores the events. +type fakeBackend struct { + apiClient client.APIClient + prompt func(message string, defaultValue bool) (bool, error) + auth registry.AuthProvider +} + +func (f *fakeBackend) APIClient() client.APIClient { return f.apiClient } +func (f *fakeBackend) AuthProvider() registry.AuthProvider { return f.auth } +func (f *fakeBackend) Events() api.EventProcessor { return progress.Noop{} } +func (f *fakeBackend) DryRun() bool { return false } + +func (f *fakeBackend) Prompt(message string, defaultValue bool) (bool, error) { + return f.prompt(message, defaultValue) +} + +func (f *fakeBackend) ImageDigestResolver(ctx context.Context) func(named reference.Named) (digest.Digest, error) { + return func(named reference.Named) (digest.Digest, error) { + auth, err := registry.EncodedAuth(named, f.auth) + if err != nil { + return "", err + } + inspect, err := f.apiClient.DistributionInspect(ctx, named.String(), client.DistributionInspectOptions{ + EncodedRegistryAuth: auth, + }) + if err != nil { + return "", fmt.Errorf("failed to resolve digest for %s: %w", named.String(), err) + } + return inspect.Descriptor.Digest, nil + } +} + +func (f *fakeBackend) Push(context.Context, *types.Project, api.PushOptions) error { + return nil +} + func Test_createLayers(t *testing.T) { project, err := loader.LoadWithContext(t.Context(), types.ConfigDetails{ WorkingDir: "testdata/publish/", @@ -55,7 +97,7 @@ func Test_createLayers(t *testing.T) { assert.NilError(t, err) project.ComposeFiles = []string{"testdata/publish/compose.yaml"} - service := &composeService{} + service := &Service{} layers, err := service.createLayers(t.Context(), project, api.PublishOptions{ WithEnvironment: true, }) @@ -136,9 +178,7 @@ func Test_preChecks_sensitive_data_detected_decline(t *testing.T) { declined := func(message string, defaultValue bool) (bool, error) { return false, nil } - svc := &composeService{ - prompt: declined, - } + svc := &Service{backend: &fakeBackend{prompt: declined}} accept, err := svc.preChecks(t.Context(), project, api.PublishOptions{}) assert.NilError(t, err) @@ -227,7 +267,7 @@ func Test_checkForSensitiveData_optional_env_file_missing(t *testing.T) { }, } - svc := &composeService{} + svc := &Service{} findings, err := svc.checkForSensitiveData(t.Context(), project) assert.NilError(t, err, "optional missing env file should not cause error during scan") assert.Equal(t, len(findings), 0) @@ -251,7 +291,7 @@ func Test_checkForSensitiveData_optional_env_file_present(t *testing.T) { }, } - svc := &composeService{} + svc := &Service{} findings, err := svc.checkForSensitiveData(t.Context(), project) assert.NilError(t, err) assert.Assert(t, len(findings) > 0, "present optional env file should still be scanned for secrets") @@ -272,7 +312,7 @@ func Test_checkForSensitiveData_required_env_file_missing(t *testing.T) { }, } - svc := &composeService{} + svc := &Service{} _, err := svc.checkForSensitiveData(t.Context(), project) assert.ErrorContains(t, err, "not found", "required missing env file should fail") } @@ -630,7 +670,7 @@ services: }) prompt := &fakePrompt{} - svc := &composeService{prompt: prompt.handler} + svc := &Service{backend: &fakeBackend{prompt: prompt.handler}} err := svc.checkEnvironmentVariables(t.Context(), project, api.PublishOptions{}) assert.NilError(t, err) @@ -649,7 +689,7 @@ services: }) prompt := &fakePrompt{answers: []bool{true}} - svc := &composeService{prompt: prompt.handler} + svc := &Service{backend: &fakeBackend{prompt: prompt.handler}} err := svc.checkEnvironmentVariables(t.Context(), project, api.PublishOptions{}) assert.NilError(t, err) @@ -670,7 +710,7 @@ services: }) prompt := &fakePrompt{answers: []bool{false}} - svc := &composeService{prompt: prompt.handler} + svc := &Service{backend: &fakeBackend{prompt: prompt.handler}} err := svc.checkEnvironmentVariables(t.Context(), project, api.PublishOptions{}) assert.Assert(t, errors.Is(err, api.ErrCanceled), @@ -700,7 +740,7 @@ configs: }) prompt := &fakePrompt{answers: []bool{true}} - svc := &composeService{prompt: prompt.handler} + svc := &Service{backend: &fakeBackend{prompt: prompt.handler}} err := svc.checkEnvironmentVariables(t.Context(), project, api.PublishOptions{WithEnvironment: true}) assert.NilError(t, err) @@ -724,7 +764,7 @@ configs: }) prompt := &fakePrompt{answers: []bool{true, true}} - svc := &composeService{prompt: prompt.handler} + svc := &Service{backend: &fakeBackend{prompt: prompt.handler}} err := svc.checkEnvironmentVariables(t.Context(), project, api.PublishOptions{}) assert.NilError(t, err) @@ -752,12 +792,9 @@ func Test_publish_decline_returns_ErrCanceled(t *testing.T) { declined := func(message string, defaultValue bool) (bool, error) { return false, nil } - svc := &composeService{ - prompt: declined, - events: progress.Noop{}, - } + svc := &Service{backend: &fakeBackend{prompt: declined}} - err := svc.publish(t.Context(), project, "docker.io/myorg/myapp:latest", api.PublishOptions{}) + err := svc.Publish(t.Context(), project, "docker.io/myorg/myapp:latest", api.PublishOptions{}) assert.Assert(t, errors.Is(err, api.ErrCanceled), "expected api.ErrCanceled when user declines, got: %v", err) } @@ -766,9 +803,8 @@ func Test_generateImageDigestsOverride_resolvesDependentImages(t *testing.T) { mockCtrl := gomock.NewController(t) defer mockCtrl.Finish() - apiClient, cli := prepareMocks(mockCtrl) - cli.EXPECT().ConfigFile().Return(configfile.New("")).AnyTimes() - tested := &composeService{dockerCli: cli} + apiClient := mocks.NewMockAPIClient(mockCtrl) + tested := &Service{backend: &fakeBackend{apiClient: apiClient, auth: configfile.New("")}} // distinct digests per resolved reference, so attaching a digest to the wrong image would fail const ( @@ -798,15 +834,15 @@ func Test_generateImageDigestsOverride_resolvesDependentImages(t *testing.T) { // per dependent image apiClient.EXPECT().DistributionInspect(gomock.Any(), "docker.io/library/nginx:latest", gomock.Any()). Return(client.DistributionInspectResult{ - DistributionInspect: registry.DistributionInspect{Descriptor: v1.Descriptor{Digest: serviceDigest}}, + DistributionInspect: registrytypes.DistributionInspect{Descriptor: v1.Descriptor{Digest: serviceDigest}}, }, nil) apiClient.EXPECT().DistributionInspect(gomock.Any(), "docker.io/library/hookimage:latest", gomock.Any()). Return(client.DistributionInspectResult{ - DistributionInspect: registry.DistributionInspect{Descriptor: v1.Descriptor{Digest: hookDigest}}, + DistributionInspect: registrytypes.DistributionInspect{Descriptor: v1.Descriptor{Digest: hookDigest}}, }, nil) apiClient.EXPECT().DistributionInspect(gomock.Any(), "docker.io/library/someimage:latest", gomock.Any()). Return(client.DistributionInspectResult{ - DistributionInspect: registry.DistributionInspect{Descriptor: v1.Descriptor{Digest: volumeDigest}}, + DistributionInspect: registrytypes.DistributionInspect{Descriptor: v1.Descriptor{Digest: volumeDigest}}, }, nil) override, err := tested.generateImageDigestsOverride(t.Context(), project) @@ -822,9 +858,8 @@ func Test_generateImageDigestsOverride_resolvesJobImages(t *testing.T) { mockCtrl := gomock.NewController(t) defer mockCtrl.Finish() - apiClient, cli := prepareMocks(mockCtrl) - cli.EXPECT().ConfigFile().Return(configfile.New("")).AnyTimes() - tested := &composeService{dockerCli: cli} + apiClient := mocks.NewMockAPIClient(mockCtrl) + tested := &Service{backend: &fakeBackend{apiClient: apiClient, auth: configfile.New("")}} const jobDigest = "sha256:dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd" project := &types.Project{ @@ -839,7 +874,7 @@ func Test_generateImageDigestsOverride_resolvesJobImages(t *testing.T) { apiClient.EXPECT().DistributionInspect(gomock.Any(), "docker.io/library/migrate:latest", gomock.Any()). Return(client.DistributionInspectResult{ - DistributionInspect: registry.DistributionInspect{Descriptor: v1.Descriptor{Digest: jobDigest}}, + DistributionInspect: registrytypes.DistributionInspect{Descriptor: v1.Descriptor{Digest: jobDigest}}, }, nil) override, err := tested.generateImageDigestsOverride(t.Context(), project) @@ -859,7 +894,7 @@ func Test_checkOnlyBuildSection_rejectsJobWithoutImage(t *testing.T) { }, } - svc := &composeService{} + svc := &Service{} ok, err := svc.checkOnlyBuildSection(project) assert.Assert(t, !ok) assert.ErrorContains(t, err, `"migrate"`) @@ -876,7 +911,7 @@ func Test_checkOnlyBuildSection_acceptsJobWithImage(t *testing.T) { }, } - svc := &composeService{} + svc := &Service{} ok, err := svc.checkOnlyBuildSection(project) assert.NilError(t, err) assert.Assert(t, ok) @@ -900,7 +935,7 @@ func Test_checkForBindMount_flagsJobBindMount(t *testing.T) { }, } - svc := &composeService{} + svc := &Service{} findings := svc.checkForBindMount(project) assert.Equal(t, len(findings["migrate"]), 1) assert.Equal(t, findings["migrate"][0].Source, "/host/data") @@ -924,7 +959,7 @@ func Test_checkForSensitiveData_jobEnvFile(t *testing.T) { }, } - svc := &composeService{} + svc := &Service{} findings, err := svc.checkForSensitiveData(t.Context(), project) assert.NilError(t, err) assert.Assert(t, len(findings) > 0, "job env file should be scanned for secrets like a service's") diff --git a/pkg/compose/testdata/publish/common.yaml b/internal/publish/testdata/publish/common.yaml similarity index 100% rename from pkg/compose/testdata/publish/common.yaml rename to internal/publish/testdata/publish/common.yaml diff --git a/pkg/compose/testdata/publish/compose.yaml b/internal/publish/testdata/publish/compose.yaml similarity index 100% rename from pkg/compose/testdata/publish/compose.yaml rename to internal/publish/testdata/publish/compose.yaml diff --git a/pkg/compose/testdata/publish/test.env b/internal/publish/testdata/publish/test.env similarity index 100% rename from pkg/compose/testdata/publish/test.env rename to internal/publish/testdata/publish/test.env diff --git a/pkg/compose/transform/replace.go b/internal/publish/transform/replace.go similarity index 100% rename from pkg/compose/transform/replace.go rename to internal/publish/transform/replace.go diff --git a/pkg/compose/transform/replace_test.go b/internal/publish/transform/replace_test.go similarity index 100% rename from pkg/compose/transform/replace_test.go rename to internal/publish/transform/replace_test.go diff --git a/pkg/compose/publish.go b/pkg/compose/publish.go index 29c9f619d..2aa75bc64 100644 --- a/pkg/compose/publish.go +++ b/pkg/compose/publish.go @@ -17,846 +17,56 @@ package compose import ( - "bytes" "context" - "crypto/sha256" - "encoding/json" - "errors" - "fmt" - "io" - "maps" - "os" - "slices" - "strconv" - "strings" - "github.com/DefangLabs/secret-detector/pkg/detectors/keyword" - "github.com/DefangLabs/secret-detector/pkg/scanner" - "github.com/DefangLabs/secret-detector/pkg/secrets" - "github.com/compose-spec/compose-go/v2/loader" "github.com/compose-spec/compose-go/v2/types" - "github.com/containerd/containerd/v2/core/remotes" "github.com/distribution/reference" + "github.com/moby/moby/client" "github.com/opencontainers/go-digest" - "github.com/opencontainers/image-spec/specs-go" - v1 "github.com/opencontainers/image-spec/specs-go/v1" - "github.com/sirupsen/logrus" - "github.com/docker/compose/v5/internal/desktop" - "github.com/docker/compose/v5/internal/oci" + "github.com/docker/compose/v5/internal/publish" + "github.com/docker/compose/v5/internal/registry" "github.com/docker/compose/v5/pkg/api" - "github.com/docker/compose/v5/pkg/compose/transform" ) +// publish lives in internal/publish: composeService serves it by handing that +// package the engine it acts on. + func (s *composeService) Publish(ctx context.Context, project *types.Project, repository string, options api.PublishOptions) error { return Run(ctx, func(ctx context.Context) error { - return s.publish(ctx, project, repository, options) + return publish.New(publishBackend{s}).Publish(ctx, project, repository, options) }, "publish", s.events) } -func (s *composeService) publish(ctx context.Context, project *types.Project, repository string, options api.PublishOptions) error { - project, err := project.WithProfiles([]string{"*"}) - if err != nil { - return err - } - accept, err := s.preChecks(ctx, project, options) - if err != nil { - return err - } - if !accept { - return api.ErrCanceled - } - // unexported push, not the public Push: publish already runs inside its - // own "publish" Start/Done bracket, and Push would open a second one on - // the same shared bus. Quiet:true would also suppress push progress - // reporting, not just the bracket, so it's not an option here. - err = s.push(ctx, project, api.PushOptions{IgnoreFailures: true, ImageMandatory: true}) - if err != nil { - return err - } - - layers, err := s.createLayers(ctx, project, options) - if err != nil { - return err - } - - s.events.On(api.Resource{ - ID: repository, - Text: "publishing", - Status: api.Working, - }) - if logrus.IsLevelEnabled(logrus.DebugLevel) { - logrus.Debug("publishing layers") - for _, layer := range layers { - indent, _ := json.MarshalIndent(layer, "", " ") - fmt.Println(string(indent)) - } - } - didFallback := false - if !s.dryRun { - didFallback, err = s.pushComposeArtifact(ctx, project, repository, layers, options) - if err != nil { - return err - } - } - text, status := "published", api.Done - if didFallback { - text, status = "published (registry rejected OCI 1.1; fell back to OCI 1.0)", api.Warning - } - s.events.On(api.Resource{ - ID: repository, - Text: text, - Status: status, - }) - return nil +// publishBackend is the engine of a composeService, as internal/publish sees it. +type publishBackend struct { + s *composeService } -// pushComposeArtifact pushes the compose artifact manifest to the repository, -// and the application image index when publishing a full application. The -// returned bool reports whether the push fell back from OCI 1.1 to OCI 1.0. -func (s *composeService) pushComposeArtifact(ctx context.Context, project *types.Project, repository string, layers []v1.Descriptor, options api.PublishOptions) (bool, error) { - named, err := reference.ParseDockerRef(repository) - if err != nil { - return false, err - } - - var insecureRegistries []string - if options.InsecureRegistry { - insecureRegistries = append(insecureRegistries, reference.Domain(named)) - } - - resolver := oci.NewResolver(s.authProvider(), desktop.ProxyTransportFor(ctx, s.apiClient()), insecureRegistries...) - - descriptor, didFallback, err := oci.PushManifest(ctx, resolver, named, layers, options.OCIVersion) - if err != nil { - s.events.On(api.Resource{ - ID: repository, - Text: "publishing", - Status: api.Error, - }) - return false, err - } - - if options.Application { - return didFallback, pushApplicationIndex(ctx, resolver, named, descriptor, project) - } - return didFallback, nil +func (b publishBackend) APIClient() client.APIClient { + return b.s.apiClient() } -// pushApplicationIndex pushes an image index referencing every service image, -// so the application can be pulled as a single artifact -func pushApplicationIndex(ctx context.Context, resolver remotes.Resolver, named reference.Named, descriptor v1.Descriptor, project *types.Project) error { - manifests := []v1.Descriptor{} - images := make([]string, 0, len(project.Services)+len(project.Jobs)) - for _, service := range project.Services { - images = append(images, service.Image) - } - for _, job := range project.Jobs { - images = append(images, job.Image) - } - for _, image := range images { - ref, err := reference.ParseDockerRef(image) - if err != nil { - return err - } - - manifest, err := oci.Copy(ctx, resolver, ref, named) - if err != nil { - return err - } - manifests = append(manifests, manifest) - } - - descriptor.Data = nil - index, err := json.Marshal(v1.Index{ - Versioned: specs.Versioned{SchemaVersion: 2}, - MediaType: v1.MediaTypeImageIndex, - Manifests: manifests, - Subject: &descriptor, - Annotations: map[string]string{ - "com.docker.compose.version": api.ComposeVersion, - }, - }) - if err != nil { - return err - } - imagesDescriptor := v1.Descriptor{ - MediaType: v1.MediaTypeImageIndex, - ArtifactType: oci.ComposeProjectArtifactType, - Digest: digest.FromString(string(index)), - Size: int64(len(index)), - Annotations: map[string]string{ - "com.docker.compose.version": api.ComposeVersion, - }, - Data: index, - } - return oci.Push(ctx, resolver, reference.TrimNamed(named), imagesDescriptor) +func (b publishBackend) AuthProvider() registry.AuthProvider { + return b.s.authProvider() } -func (s *composeService) createLayers(ctx context.Context, project *types.Project, options api.PublishOptions) ([]v1.Descriptor, error) { - var layers []v1.Descriptor - extFiles := map[string]string{} - envFiles := map[string]string{} - for _, file := range project.ComposeFiles { - data, err := processFile(ctx, file, project, extFiles, envFiles) - if err != nil { - return nil, err - } - - layerDescriptor := oci.DescriptorForComposeFile(file, data) - layers = append(layers, layerDescriptor) - } - - extLayers, err := processExtends(ctx, project, extFiles) - if err != nil { - return nil, err - } - layers = append(layers, extLayers...) - - if options.WithEnvironment { - layers = append(layers, envFileLayers(envFiles)...) - } - - if options.ResolveImageDigests { - yaml, err := s.generateImageDigestsOverride(ctx, project) - if err != nil { - return nil, err - } - - layerDescriptor := oci.DescriptorForComposeFile("image-digests.yaml", yaml) - layers = append(layers, layerDescriptor) - } - return layers, nil +func (b publishBackend) Events() api.EventProcessor { + return b.s.events } -func processExtends(ctx context.Context, project *types.Project, extFiles map[string]string) ([]v1.Descriptor, error) { - var layers []v1.Descriptor - moreExtFiles := map[string]string{} - envFiles := map[string]string{} - for xf, hash := range extFiles { - data, err := processFile(ctx, xf, project, moreExtFiles, envFiles) - if err != nil { - return nil, err - } - - layerDescriptor := oci.DescriptorForComposeFile(hash, data) - layerDescriptor.Annotations["com.docker.compose.extends"] = "true" - layers = append(layers, layerDescriptor) - } - for f, hash := range moreExtFiles { - if _, ok := extFiles[f]; ok { - delete(moreExtFiles, f) - } - extFiles[f] = hash - } - if len(moreExtFiles) > 0 { - extLayers, err := processExtends(ctx, project, moreExtFiles) - if err != nil { - return nil, err - } - layers = append(layers, extLayers...) - } - return layers, nil +func (b publishBackend) DryRun() bool { + return b.s.dryRun } -func processFile(ctx context.Context, file string, project *types.Project, extFiles map[string]string, envFiles map[string]string) ([]byte, error) { - f, err := os.ReadFile(file) - if err != nil { - return nil, err - } - - base, err := loader.LoadWithContext(ctx, types.ConfigDetails{ - WorkingDir: project.WorkingDir, - Environment: project.Environment, - ConfigFiles: []types.ConfigFile{ - { - Filename: file, - Content: f, - }, - }, - }, func(options *loader.Options) { - options.SkipValidation = true - options.SkipExtends = true - options.SkipConsistencyCheck = true - options.ResolvePaths = true - options.SkipInclude = true - options.Profiles = project.Profiles - }) - if err != nil { - return nil, err - } - for name, service := range base.Services { - for i, envFile := range service.EnvFiles { - // A real stat failure (e.g. permissions) is fatal, but a missing file is not: - // the project loader already rejects missing required env files before we get - // here, so an absent file at this point is an optional one. - _, statErr := os.Stat(envFile.Path) - if statErr != nil && !os.IsNotExist(statErr) { - return nil, fmt.Errorf("failed to access env file %s: %w", envFile.Path, statErr) - } - // The hash is derived from the path string alone, so the env_file is always - // rewritten to its opaque .env placeholder, even for a missing optional - // file, so the published artifact never leaks the local path. Only files that - // exist are registered for upload, mirroring the extends handling below. - hash := fmt.Sprintf("%x.env", sha256.Sum256([]byte(envFile.Path))) - if statErr == nil { - envFiles[envFile.Path] = hash - } - f, err = transform.ReplaceEnvFile(f, name, i, hash) - if err != nil { - return nil, err - } - } - - if service.Extends == nil { - continue - } - xf := service.Extends.File - if xf == "" { - continue - } - if _, err = os.Stat(service.Extends.File); os.IsNotExist(err) { - // No local file, while we loaded the project successfully: This is actually a remote resource - continue - } - - hash := fmt.Sprintf("%x.yaml", sha256.Sum256([]byte(xf))) - extFiles[xf] = hash - - f, err = transform.ReplaceExtendsFile(f, name, hash) - if err != nil { - return nil, err - } - } - return f, nil +func (b publishBackend) Prompt(message string, defaultValue bool) (bool, error) { + return b.s.prompt(message, defaultValue) } -func (s *composeService) generateImageDigestsOverride(ctx context.Context, project *types.Project) ([]byte, error) { - project, err := project.WithImagesResolved(imageDigestResolver(ctx, s.authProvider(), s.apiClient())) - if err != nil { - return nil, err - } - // WithImagesResolved only walks services: dress the jobs as services to - // run them through the exact same resolution, then fold the digests back - // into a jobs override. - if len(project.Jobs) > 0 { - jobsAsServices := &types.Project{Services: types.Services{}} - for name, job := range project.Jobs { - jobsAsServices.Services[name] = types.ServiceConfig{ - Name: name, - ContainerSpec: types.ContainerSpec{Image: job.Image}, - } - } - jobsAsServices, err = jobsAsServices.WithImagesResolved(imageDigestResolver(ctx, s.authProvider(), s.apiClient())) - if err != nil { - return nil, err - } - for name, resolved := range jobsAsServices.Services { - job := project.Jobs[name] - job.Image = resolved.Image - project.Jobs[name] = job - } - } - override := types.Project{ - Services: types.Services{}, - Jobs: types.Jobs{}, - } - for name, service := range project.Services { - override.Services[name] = types.ServiceConfig{ - ContainerSpec: types.ContainerSpec{Image: service.Image}, - } - } - for name, job := range project.Jobs { - override.Jobs[name] = types.JobConfig{ - ContainerSpec: types.ContainerSpec{Image: job.Image}, - } - } - return override.MarshalYAML() +func (b publishBackend) ImageDigestResolver(ctx context.Context) func(named reference.Named) (digest.Digest, error) { + return imageDigestResolver(ctx, b.s.authProvider(), b.s.apiClient()) } -func (s *composeService) preChecks(ctx context.Context, project *types.Project, options api.PublishOptions) (bool, error) { - if ok, err := s.checkOnlyBuildSection(project); !ok || err != nil { - return false, err - } - bindMounts := s.checkForBindMount(project) - if len(bindMounts) > 0 { - b := strings.Builder{} - b.WriteString("you are about to publish bind mounts declaration within your OCI artifact.\n" + - "only the bind mount declarations will be added to the OCI artifact (not content)\n" + - "please double check that you are not mounting potential user's sensitive directories or data\n") - for key, val := range bindMounts { - b.WriteString(key) - for _, v := range val { - b.WriteString(v.String()) - b.WriteRune('\n') - } - } - b.WriteString("Are you ok to publish these bind mount declarations?") - confirm, err := s.prompt(b.String(), false) - if err != nil || !confirm { - return false, err - } - } - detectedSecrets, err := s.checkForSensitiveData(ctx, project) - if err != nil { - return false, err - } - if len(detectedSecrets) > 0 { - b := strings.Builder{} - b.WriteString("you are about to publish sensitive data within your OCI artifact.\n" + - "please double check that you are not leaking sensitive data\n") - for _, val := range detectedSecrets { - b.WriteString(val.Type) - b.WriteRune('\n') - fmt.Fprintf(&b, "%q: %s\n", val.Key, val.Value) - } - b.WriteString("Are you ok to publish these sensitive data?") - confirm, err := s.prompt(b.String(), false) - if err != nil || !confirm { - return false, err - } - } - err = s.checkEnvironmentVariables(ctx, project, options) - if err != nil { - return false, err - } - return true, nil -} - -// envCheckFindings groups everything checkEnvironmentVariables surfaces to -// the user during publish pre-checks for env-related leak risks. -type envCheckFindings struct { - // services maps service name -> findings for that service. Only services - // with at least one finding are present. - services map[string]*serviceEnvFindings - // configsLiteralContent lists configs whose inline `content:` is a literal - // (not interpolation). Sorted alphabetically. config.content is decoupled - // from --with-env because the flag is documented as controlling environment - // variable publishing only. - configsLiteralContent []string -} - -type serviceEnvFindings struct { - hasEnvFile bool - // suspiciousKeys is the set of environment variable names whose literal - // values look sensitive, as classified by the upstream DefangLabs keyword - // detector (password, secret, token, api_key, …). A set is used because - // the same service may be visited across multiple compose files during - // the extends walk; callers convert to a sorted slice via sortedKeys - // when surfacing to the user. - suspiciousKeys map[string]struct{} -} - -// sortedSuspiciousKeys returns the suspicious env var names alphabetically -// sorted for stable output. -func (f *serviceEnvFindings) sortedSuspiciousKeys() []string { - return slices.Sorted(maps.Keys(f.suspiciousKeys)) -} - -func (f *envCheckFindings) hasEnvFinding() bool { - for _, serviceFindings := range f.services { - if serviceFindings.hasEnvFile || len(serviceFindings.suspiciousKeys) > 0 { - return true - } - } - return false -} - -// checkEnvironmentVariables walks every compose file that will be serialized -// into the OCI artifact (the top-level files plus any local extends parents) -// and prompts the user to confirm before publishing: -// -// 1. service env_file declarations and literal environment values whose key -// name looks sensitive (password, secret, token, api_key, …) — silenced -// by --with-env; -// 2. literal inline config.content — always prompts (decoupled from -// --with-env, which is documented to cover env vars only). -// -// Interpolated values like "${SECRET}" or "$VAR" are preserved as placeholders -// in the published YAML and don't leak the resolved value; the keyword -// detector's value regex skips them automatically. -func (s *composeService) checkEnvironmentVariables(ctx context.Context, project *types.Project, options api.PublishOptions) error { - if len(project.ComposeFiles) == 0 { - return nil - } - - findings, err := collectEnvCheckFindings(ctx, project) - if err != nil { - return err - } - - if !options.WithEnvironment && findings.hasEnvFinding() { - if err := s.confirmOrCancel(buildEnvPromptMessage(findings.services)); err != nil { - return err - } - } - - if len(findings.configsLiteralContent) > 0 { - if err := s.confirmOrCancel(buildConfigContentPromptMessage(findings.configsLiteralContent)); err != nil { - return err - } - } - - return nil -} - -// confirmOrCancel runs an interactive yes/no prompt and returns: -// - the prompt's error verbatim, if it failed; -// - api.ErrCanceled if the user declined; -// - nil if the user accepted. -func (s *composeService) confirmOrCancel(message string) error { - confirm, err := s.prompt(message, false) - if err != nil { - return err - } - if !confirm { - return api.ErrCanceled - } - return nil -} - -// collectEnvCheckFindings walks every compose file scheduled for publication -// (top-level files plus any local extends parents discovered along the way) -// and aggregates per-service and per-config findings. The walk mirrors -// processExtends so coverage matches what is actually serialized into the OCI -// artifact. -func collectEnvCheckFindings(ctx context.Context, project *types.Project) (*envCheckFindings, error) { - findings := &envCheckFindings{services: map[string]*serviceEnvFindings{}} - literalCfgs := map[string]struct{}{} - keywordDetector := keyword.NewDetector("0") - - seen := map[string]struct{}{} - queue := slices.Clone(project.ComposeFiles) - for len(queue) > 0 { - file := queue[0] - queue = queue[1:] - if _, ok := seen[file]; ok { - continue - } - seen[file] = struct{}{} - - unresolved, err := loadUnresolvedFile(ctx, project, file) - if err != nil { - return nil, fmt.Errorf("failed to load compose file %s: %w", file, err) - } - - for _, service := range unresolved.Services { - recordEnvFindings(findings.services, keywordDetector, service.Name, service.ContainerSpec) - if parent := localExtendsParent(service); parent != "" { - queue = append(queue, parent) - } - } - for name, job := range unresolved.Jobs { - // jobs carry environment/env_file like services: same leak surface - recordEnvFindings(findings.services, keywordDetector, name, job.ContainerSpec) - } - for name, config := range unresolved.Configs { - // config.Environment is a variable *name* (only the name is - // published, not its resolved value) so it is not a leak. Inline - // config.Content is what ends up in the artifact. compose-go - // enforces that file, environment, and content are mutually - // exclusive. The map key is the name as written in the compose - // file; config.Name is the project-namespaced version, which is - // less helpful when surfaced to the user. - if config.Content != "" && configContentLooksLiteral(config.Content, keywordDetector) { - literalCfgs[name] = struct{}{} - } - } - } - - if len(literalCfgs) > 0 { - findings.configsLiteralContent = slices.Sorted(maps.Keys(literalCfgs)) - } - return findings, nil -} - -func recordEnvFindings(services map[string]*serviceEnvFindings, detector secrets.Detector, name string, spec types.ContainerSpec) { - envValues := map[string]string{} - for key, value := range spec.Environment { - if value == nil { - continue - } - envValues[key] = replaceDollarEscape(*value) - } - - hits, _ := detector.ScanMap(envValues) - if len(hits) == 0 && len(spec.EnvFiles) == 0 { - return - } - - f := services[name] - if f == nil { - f = &serviceEnvFindings{suspiciousKeys: map[string]struct{}{}} - services[name] = f - } - if len(spec.EnvFiles) > 0 { - f.hasEnvFile = true - } - for _, hit := range hits { - f.suspiciousKeys[hit.Key] = struct{}{} - } -} - -// configContentLooksLiteral returns true when the inline config.content has -// a literal portion that would be published as-is, leaking the value to -// consumers of the OCI artifact. -// -// We piggyback on the keyword detector's value regex (`[^${\s].+[^}\s]`) by -// passing a fake "password" key to ScanMap — the regex isn't exported -// directly, only via the key+value match path. The regex excludes values -// starting with `$` (`${VAR}`/`$VAR` interpolation), ending with `}` -// (templates like `key=${SECRET}`), or shorter than 3 chars, which neatly -// matches our notion of "looks like a template, not a literal". -func configContentLooksLiteral(content string, detector secrets.Detector) bool { - hits, _ := detector.ScanMap(map[string]string{"password": replaceDollarEscape(content)}) - return len(hits) > 0 -} - -// replaceDollarEscape substitutes the compose-spec `$$` escape (which -// represents a literal `$` in the resolved value) with a placeholder. The -// placeholder is `X` rather than `$` because the keyword detector's value -// regex excludes any value beginning with `$`; using `$` would mask the -// literal we're trying to flag. Any non-special char would do — we picked -// `X` for readability. -func replaceDollarEscape(value string) string { - return strings.ReplaceAll(value, "$$", "X") -} - -// localExtendsParent returns the path of an extends parent file that exists on -// disk, or "" when the service does not extend or extends a remote resource. -func localExtendsParent(service types.ServiceConfig) string { - if service.Extends == nil || service.Extends.File == "" { - return "" - } - if _, err := os.Stat(service.Extends.File); err != nil { - return "" - } - return service.Extends.File -} - -func buildEnvPromptMessage(services map[string]*serviceEnvFindings) string { - var b strings.Builder - b.WriteString("you are about to publish env-related declarations within your OCI artifact.\n") - b.WriteString("env_file paths and literal values for sensitive-looking keys are embedded as-is in the published YAML;\n") - b.WriteString("interpolated values like \"${VAR}\" are kept symbolic and have already been excluded.\n") - for _, name := range slices.Sorted(maps.Keys(services)) { - f := services[name] - // name may be a service or a job: findings.services carries both. - if f.hasEnvFile { - fmt.Fprintf(&b, " %q: env_file declared\n", name) - } - if keys := f.sortedSuspiciousKeys(); len(keys) > 0 { - quoted := make([]string, len(keys)) - for i, k := range keys { - quoted[i] = strconv.Quote(k) - } - fmt.Fprintf(&b, " %q: literal value for %s\n", name, strings.Join(quoted, ", ")) - } - } - b.WriteString("Use --with-env to silence this prompt and always publish env declarations.\n") - b.WriteString("Are you ok to publish these env declarations?") - return b.String() -} - -func buildConfigContentPromptMessage(configs []string) string { - var b strings.Builder - b.WriteString("you are about to publish literal inline config content within your OCI artifact.\n") - for _, name := range configs { - fmt.Fprintf(&b, " config %q\n", name) - } - b.WriteString("Are you ok to publish these config contents?") - return b.String() -} - -// loadUnresolvedFile loads a single compose file with interpolation and -// environment resolution skipped, so callers can inspect raw user-provided -// values. Used by both checkEnvironmentVariables and composeFileAsByteReader. -func loadUnresolvedFile(ctx context.Context, project *types.Project, filePath string) (*types.Project, error) { - return loader.LoadWithContext(ctx, types.ConfigDetails{ - WorkingDir: project.WorkingDir, - Environment: project.Environment, - ConfigFiles: []types.ConfigFile{{Filename: filePath}}, - }, func(options *loader.Options) { - options.SkipValidation = true - options.SkipExtends = true - options.SkipConsistencyCheck = true - options.ResolvePaths = true - // SkipInclude mirrors processFile: include directives stay symbolic in - // the published artifact, so included content must not be inspected - // here either (otherwise we'd flag literals that never ship). - options.SkipInclude = true - options.SkipInterpolation = true - options.SkipResolveEnvironment = true - options.Profiles = project.Profiles - }) -} - -func envFileLayers(files map[string]string) []v1.Descriptor { - var layers []v1.Descriptor - for file, hash := range files { - f, err := os.ReadFile(file) - if err != nil { - // if we can't read the file, skip to the next one - continue - } - layerDescriptor := oci.DescriptorForEnvFile(hash, f) - layers = append(layers, layerDescriptor) - } - return layers -} - -func (s *composeService) checkOnlyBuildSection(project *types.Project) (bool, error) { - errorList := []string{} - for _, service := range project.Services { - if service.Image == "" && service.Build != nil { - errorList = append(errorList, service.Name) - } - } - for name, job := range project.Jobs { - if job.Image == "" && job.Build != nil { - errorList = append(errorList, name) - } - } - if len(errorList) > 0 { - var errMsg strings.Builder - errMsg.WriteString("your Compose stack cannot be published as it only contains a build section for service(s):\n") - for _, serviceInError := range errorList { - fmt.Fprintf(&errMsg, "- %q\n", serviceInError) - } - return false, errors.New(errMsg.String()) - } - return true, nil -} - -func (s *composeService) checkForBindMount(project *types.Project) map[string][]types.ServiceVolumeConfig { - allFindings := map[string][]types.ServiceVolumeConfig{} - record := func(name string, volumes []types.ServiceVolumeConfig) { - bindMounts := []types.ServiceVolumeConfig{} - for _, volume := range volumes { - if volume.Type == types.VolumeTypeBind { - bindMounts = append(bindMounts, volume) - } - } - if len(bindMounts) > 0 { - allFindings[name] = bindMounts - } - } - for serviceName, config := range project.Services { - record(serviceName, config.Volumes) - } - for name, job := range project.Jobs { - record(name, job.Volumes) - } - return allFindings -} - -func (s *composeService) checkForSensitiveData(ctx context.Context, project *types.Project) ([]secrets.DetectedSecret, error) { - var allFindings []secrets.DetectedSecret - scan := scanner.NewDefaultScanner() - - // Check all compose files - for _, file := range project.ComposeFiles { - in, err := composeFileAsByteReader(ctx, file, project) - if err != nil { - return nil, err - } - - findings, err := scan.ScanReader(in) - if err != nil { - return nil, fmt.Errorf("failed to scan compose file %s: %w", file, err) - } - allFindings = append(allFindings, findings...) - } - - // Check env files — jobs declare env_file like services do - for _, service := range project.Services { - findings, err := scanEnvFiles(scan, service.EnvFiles) - if err != nil { - return nil, err - } - allFindings = append(allFindings, findings...) - } - for _, job := range project.Jobs { - findings, err := scanEnvFiles(scan, job.EnvFiles) - if err != nil { - return nil, err - } - allFindings = append(allFindings, findings...) - } - - // Check configs defined by files - configFiles := make([]string, 0, len(project.Configs)) - for _, config := range project.Configs { - configFiles = append(configFiles, config.File) - } - findings, err := scanFiles(scan, "config", configFiles) - if err != nil { - return nil, err - } - allFindings = append(allFindings, findings...) - - // Check secrets defined by files - secretFiles := make([]string, 0, len(project.Secrets)) - for _, secret := range project.Secrets { - secretFiles = append(secretFiles, secret.File) - } - findings, err = scanFiles(scan, "secret", secretFiles) - if err != nil { - return nil, err - } - allFindings = append(allFindings, findings...) - - return allFindings, nil -} - -// scanEnvFiles scans declared env files for sensitive data; a missing env -// file is only an error when the declaration requires it -func scanEnvFiles(scan secrets.Scanner, envFiles []types.EnvFile) ([]secrets.DetectedSecret, error) { - var allFindings []secrets.DetectedSecret - for _, envFile := range envFiles { - if _, statErr := os.Stat(envFile.Path); statErr != nil { - if !os.IsNotExist(statErr) { - return nil, fmt.Errorf("failed to access env file %s: %w", envFile.Path, statErr) - } - if envFile.Required { - return nil, fmt.Errorf("env file %s not found", envFile.Path) - } - continue - } - findings, err := scan.ScanFile(envFile.Path) - if err != nil { - return nil, fmt.Errorf("failed to scan env file %s: %w", envFile.Path, err) - } - allFindings = append(allFindings, findings...) - } - return allFindings, nil -} - -// scanFiles scans file-based resources (configs, secrets) for sensitive data, -// ignoring resources not defined by a file -func scanFiles(scan secrets.Scanner, kind string, paths []string) ([]secrets.DetectedSecret, error) { - var allFindings []secrets.DetectedSecret - for _, path := range paths { - if path == "" { - continue - } - findings, err := scan.ScanFile(path) - if err != nil { - return nil, fmt.Errorf("failed to scan %s file %s: %w", kind, path, err) - } - allFindings = append(allFindings, findings...) - } - return allFindings, nil -} - -func composeFileAsByteReader(ctx context.Context, filePath string, project *types.Project) (io.Reader, error) { - base, err := loadUnresolvedFile(ctx, project, filePath) - if err != nil { - return nil, fmt.Errorf("failed to load compose file %s: %w", filePath, err) - } - in, err := base.MarshalYAML() - if err != nil { - return nil, err - } - return bytes.NewBuffer(in), nil +func (b publishBackend) Push(ctx context.Context, project *types.Project, options api.PushOptions) error { + return b.s.push(ctx, project, options) }