diff --git a/cmd/compose/create.go b/cmd/compose/create.go index f5ccfca41..3356be634 100644 --- a/cmd/compose/create.go +++ b/cmd/compose/create.go @@ -111,11 +111,6 @@ func normalizeYesFlag(_ *pflag.FlagSet, name string) pflag.NormalizedName { } func runCreate(ctx context.Context, dockerCli command.Cli, backendOptions *BackendOptions, createOpts createOptions, buildOpts buildOptions, project *types.Project, services []string) error { - // same contract as up: an active scheduled job is refused before any - // resource is created — silently not scheduling would break expectations - if err := rejectScheduledJobs(project); err != nil { - return err - } ignoreOrphans, err := resolveIgnoreOrphans(project, createOpts.removeOrphans) if err != nil { return err diff --git a/cmd/compose/start.go b/cmd/compose/start.go index f3f69cec3..062efb680 100644 --- a/cmd/compose/start.go +++ b/cmd/compose/start.go @@ -56,12 +56,6 @@ func runStart(ctx context.Context, dockerCli command.Cli, backendOptions *Backen if err != nil { return err } - if project != nil { - // with a file, refuse active scheduled jobs like up and create do - if err := rejectScheduledJobs(project); err != nil { - return err - } - } var timeout time.Duration if opts.waitTimeout > 0 { diff --git a/cmd/compose/up.go b/cmd/compose/up.go index 740bb21ed..b46bcd0f2 100644 --- a/cmd/compose/up.go +++ b/cmd/compose/up.go @@ -20,9 +20,7 @@ import ( "context" "errors" "fmt" - "maps" "os" - "slices" "sort" "strings" "time" @@ -138,7 +136,7 @@ func upCommand(p *ProjectOptions, dockerCli command.Cli, backendOptions *Backend up.resolveNavigationMenu(dockerCli) - if !p.All && len(project.Services) == 0 { + if !p.All && len(project.Services) == 0 && !hasScheduledJobs(project) { return errors.New("no service selected") } @@ -229,9 +227,6 @@ func runUp( return err } - if err := rejectScheduledJobs(project); err != nil { - return err - } warnIgnoredJobs(project) err := createOptions.Apply(project) @@ -289,7 +284,9 @@ func runUp( var consumer api.LogConsumer var attach []string - if !upOptions.Detach { + // a project with no service, only scheduled jobs to register, has nothing + // to follow: attaching would pick up the containers of the jobs' runs + if !upOptions.Detach && len(project.Services) > 0 { consumer = formatter.NewLogConsumer(ctx, dockerCli.Out(), dockerCli.Err(), !upOptions.noColor, !upOptions.noPrefix, upOptions.timestamp) attach, err = upOptions.attachedServices(project, services) @@ -354,30 +351,43 @@ func (opts upOptions) attachedServices(project *types.Project, services []string return attachSet.Elements(), nil } -// warnIgnoredJobs names the declared jobs up will not act on: manual jobs -// wait for an explicit `compose run ` trigger. +// warnIgnoredJobs names the declared manual jobs up will not act on: they +// wait for an explicit `compose run ` trigger. Scheduled jobs are not +// ignored — they are registered with the engine (see pkg/compose Up). func warnIgnoredJobs(project *types.Project) { - jobs := project.Jobs - if len(jobs) == 0 { + names := manualJobNames(project) + if len(names) == 0 { return } - names := slices.Sorted(maps.Keys(jobs)) logrus.Warnf("jobs are not started by up; trigger them with `docker compose run`: %s", strings.Join(names, ", ")) } -// rejectScheduledJobs refuses to bring a project up when it declares active -// scheduled jobs: silently not scheduling them would break the user's -// expectations, unlike manual jobs which simply wait for an explicit trigger. -func rejectScheduledJobs(project *types.Project) error { +// manualJobNames returns the sorted names of the project's jobs that up +// leaves untouched: a job with a schedule is registered with the engine +// (see pkg/compose Up), so only jobs without one wait for an explicit +// `compose run ` trigger — unless they opt out with `manual: false`. +func manualJobNames(project *types.Project) []string { names := make([]string, 0, len(project.Jobs)) for name, job := range project.Jobs { - if job.Triggers != nil && len(job.Triggers.Schedule) > 0 { - names = append(names, name) + if compose.HasSchedule(job) { + continue } - } - if len(names) == 0 { - return nil + if compose.ManualTriggerDisabled(job) { + continue + } + names = append(names, name) } sort.Strings(names) - return fmt.Errorf("scheduled jobs are not supported in this version: %s", strings.Join(names, ", ")) + return names +} + +// hasScheduledJobs reports whether up has jobs to register, which is enough to +// run on a project that has no service. +func hasScheduledJobs(project *types.Project) bool { + for _, job := range project.Jobs { + if compose.HasSchedule(job) { + return true + } + } + return false } diff --git a/cmd/compose/up_test.go b/cmd/compose/up_test.go index f11aad75d..fbce5a6f0 100644 --- a/cmd/compose/up_test.go +++ b/cmd/compose/up_test.go @@ -183,20 +183,33 @@ services: assert.Assert(t, !strings.Contains(fmt.Sprint(err), "invalid ip address"), fmt.Sprint(err)) } -func TestRejectScheduledJobs(t *testing.T) { +func TestManualJobNames(t *testing.T) { yes := true manual := types.JobConfig{Triggers: &types.TriggerConfig{Manual: &yes}} scheduled := types.JobConfig{Triggers: &types.TriggerConfig{ Schedule: []types.ScheduleConfig{{Cron: "0 3 * * *"}}, }} - assert.NilError(t, rejectScheduledJobs(&types.Project{})) - assert.NilError(t, rejectScheduledJobs(&types.Project{Jobs: types.Jobs{"migrate": manual}})) - // profile-disabled scheduled jobs don't block up - assert.NilError(t, rejectScheduledJobs(&types.Project{DisabledJobs: types.Jobs{"backup": scheduled}})) + assert.DeepEqual(t, manualJobNames(&types.Project{}), []string{}) + assert.DeepEqual(t, manualJobNames(&types.Project{Jobs: types.Jobs{"backup": scheduled}}), []string{}) + assert.DeepEqual(t, + manualJobNames(&types.Project{Jobs: types.Jobs{"backup": scheduled, "sync": manual, "migrate": manual}}), + []string{"migrate", "sync"}, + ) +} - err := rejectScheduledJobs(&types.Project{Jobs: types.Jobs{"backup": scheduled, "sync": scheduled, "migrate": manual}}) - assert.Error(t, err, "scheduled jobs are not supported in this version: backup, sync") +// A project with no service still has something for up to do when a job is +// scheduled: only then is the "no service selected" error not the right answer. +func TestHasScheduledJobs(t *testing.T) { + yes := true + manual := types.JobConfig{Triggers: &types.TriggerConfig{Manual: &yes}} + scheduled := types.JobConfig{Triggers: &types.TriggerConfig{ + Schedule: []types.ScheduleConfig{{Cron: "0 3 * * *"}}, + }} + + assert.Assert(t, !hasScheduledJobs(&types.Project{})) + assert.Assert(t, !hasScheduledJobs(&types.Project{Jobs: types.Jobs{"migrate": manual}})) + assert.Assert(t, hasScheduledJobs(&types.Project{Jobs: types.Jobs{"migrate": manual, "backup": scheduled}})) } // warnIgnoredJobs must only name profile-enabled jobs: a job disabled by @@ -206,9 +219,11 @@ func TestRejectScheduledJobs(t *testing.T) { func TestWarnIgnoredJobs(t *testing.T) { hook := logrustest.NewGlobal() + yes := true + manual := types.JobConfig{Triggers: &types.TriggerConfig{Manual: &yes}} warnIgnoredJobs(&types.Project{ - Jobs: types.Jobs{"migrate": {}}, - DisabledJobs: types.Jobs{"backup": {}}, + Jobs: types.Jobs{"migrate": manual}, + DisabledJobs: types.Jobs{"backup": manual}, }) assert.Equal(t, len(hook.AllEntries()), 1) diff --git a/internal/e2e/jobs_test.go b/internal/e2e/jobs_test.go index b64f1700b..e8764c68e 100644 --- a/internal/e2e/jobs_test.go +++ b/internal/e2e/jobs_test.go @@ -19,19 +19,204 @@ package e2e import ( + "strings" "testing" + "time" + + extensionclient "github.com/moby/extensions/client" + "github.com/moby/moby/client" + "github.com/moby/moby/client/pkg/versions" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" + + jobsv0 "github.com/docker/compose/v5/internal/jobsapi" + jobspb "github.com/docker/compose/v5/internal/jobsapi/protogen" ) -// Scheduled jobs cannot run in this version: silently not scheduling them -// would break the user's expectations, so up must refuse the whole project. +// jobsAPIVersion is the first engine API version shipping the jobs +// extension, see apiVersionJobs in pkg/compose/api_versions.go, which it must +// follow. The extension is disabled by default, so this version does not mean +// jobs are usable. Below it, or with the extension disabled, Compose degrades: +// manual jobs run as one-off services and scheduled jobs are refused. +const jobsAPIVersion = "1.57" + +// JobsExtensionEnabled requires the daemon's jobs extension to be enabled and +// ready: dockerd started with --extension-enable org.mobyproject.jobs.v0. +func JobsExtensionEnabled(t testing.TB, c *CLI) string { + t.Helper() + if !jobsExtensionActive(t, c) { + return "the engine's jobs extension is not enabled" + } + return "" +} + +// JobsExtensionDisabled requires the daemon to have no usable jobs extension: +// either its API predates jobs or the extension is not enabled. +func JobsExtensionDisabled(t testing.TB, c *CLI) string { + t.Helper() + if jobsExtensionActive(t, c) { + return "the engine's jobs extension is enabled" + } + return "" +} + +// jobsExtensionActive probes the daemon the CLI under test talks to, the way +// Compose does: the engine lists no extension, only the jobs service answering +// tells (Unimplemented when the extension is off). An unreachable or not ready +// extension fails the test rather than skipping it, so a misconfigured daemon +// never silently skips every jobs scenario. +func jobsExtensionActive(t testing.TB, c *CLI) bool { + t.Helper() + if versions.LessThan(engineAPIVersion(t, c), jobsAPIVersion) { + return false + } + // The CLI resolves its daemon from DOCKER_HOST/DOCKER_CONTEXT; so does + // `docker context inspect`, without a context name. + host := strings.TrimSpace(c.RunDockerCmd(t, "context", "inspect", "-f", "{{.Endpoints.docker.Host}}").Stdout()) + apiClient, err := client.New(client.WithHost(host)) + if err != nil { + t.Fatalf("connecting to the daemon at %s: %v", host, err) + } + defer apiClient.Close() //nolint:errcheck + exts, err := extensionclient.New(apiClient, extensionclient.WithGRPCPoint(jobspb.ClientPoint)) + if err != nil { + t.Fatalf("extension client: %v", err) + } + defer exts.Close() //nolint:errcheck + jobs, err := extensionclient.Resolve(exts, jobsv0.Point) + if err != nil { + t.Fatalf("resolving the jobs extension: %v", err) + } + _, err = jobs.List(t.Context(), &jobsv0.ListRequest{}) + switch status.Code(err) { + case codes.OK: + return true + case codes.Unimplemented: + return false + default: + t.Fatalf("probing the jobs extension at %s: %v", host, err) + return false + } +} + +// scheduledJobsRefused is the error of an engine without the jobs API refusing +// the project of TestUpRejectsScheduledJobs. +const scheduledJobsRefused = "scheduled jobs are not supported by this version of the engine: backup" + +// scheduledJobsExtensionOffRefused is the error of an engine having the jobs +// API but not the extension enabled, refusing the same project. +const scheduledJobsExtensionOffRefused = "scheduled jobs need the engine's jobs extension, " + + "which is not enabled (start dockerd with --extension-enable org.mobyproject.jobs.v0): backup" + +// Without the jobs API, scheduled jobs cannot run: silently not scheduling +// them would break the user's expectations, so every command bringing the +// project up must refuse it as a whole, before creating anything. func TestUpRejectsScheduledJobs(t *testing.T) { - NewScenario(t, "up must reject a project declaring active scheduled jobs, before creating anything"). + NewScenario(t, "up, create and start must reject a project declaring active scheduled jobs when the engine has no jobs API, before creating anything"). + Requires(EngineAPIVersionBelow(jobsAPIVersion)). Step("up fails naming the scheduled job", ComposeCmd("up", "-d").MayFail(), - StderrContains("scheduled jobs are not supported in this version: backup"), + StderrContains(scheduledJobsRefused), + ServiceNotCreated("web")). + Step("create fails the same way", + ComposeCmd("create").MayFail(), + StderrContains(scheduledJobsRefused), + ServiceNotCreated("web")). + Step("start fails the same way", + ComposeCmd("start").MayFail(), + StderrContains(scheduledJobsRefused), ServiceNotCreated("web")) } +// The jobs API alone does not make jobs usable: the engine ships the extension +// disabled by default, and an engine can only say so when asked. The refusal +// must be the same as on an engine without the API, and just as early, naming +// the dockerd flag that enables the extension. +func TestUpRejectsScheduledJobsWithExtensionOff(t *testing.T) { + NewScenario(t, "up, create and start must reject a project declaring active scheduled jobs when the engine's jobs extension is not enabled, before creating anything"). + Requires(EngineAPIVersionAtLeast(jobsAPIVersion), JobsExtensionDisabled). + Step("up fails naming the scheduled job and the flag enabling the extension", + ComposeCmd("up", "-d").MayFail(), + StderrContains(scheduledJobsExtensionOffRefused), + ServiceNotCreated("web")). + Step("create fails the same way", + ComposeCmd("create").MayFail(), + StderrContains(scheduledJobsExtensionOffRefused), + ServiceNotCreated("web")). + Step("start fails the same way", + ComposeCmd("start").MayFail(), + StderrContains(scheduledJobsExtensionOffRefused), + ServiceNotCreated("web")) +} + +// Without usable jobs support the engine has no job of the project to +// remove: down must not turn that into an error nor into noise. +func TestDownWithoutJobsExtension(t *testing.T) { + NewScenario(t, "down must clean up the project without a word about jobs when the engine cannot run jobs"). + Requires(JobsExtensionDisabled). + Step("up starts the service, the manual job waits for run", + ComposeCmd("up", "-d"), + ServiceState("web", "running")). + Step("down removes the service and ignores the jobs silently", + ComposeCmd("down"), + ServiceNotCreated("web"), + OutputNotContains("Job "), + OutputNotContains("level=warning")) +} + +// A scheduled job registers with the engine instead of being rejected: up is +// safely re-runnable on an unchanged spec, and the schedule fires on the +// engine's own clock, independent of the client. +func TestUpRegistersScheduledJobs(t *testing.T) { + NewScenario(t, "up must register a project's scheduled jobs with the engine and let them fire on their own"). + Requires(JobsExtensionEnabled). + Step("up starts services and registers the scheduled job", + ComposeCmd("up", "-d"), + ServiceState("web", "running")). + Step("re-up is a no-op on the unchanged job spec, and the schedule fires on the engine's own clock", + ComposeCmd("up", "-d"), + ServiceState("web", "running"), + Eventually(ServiceState("backup", "exited"), 90*time.Second)) +} + +// up --no-start stops after creating the containers, and must still register +// the scheduled jobs: the engine's cron does not wait for a Start. +func TestUpNoStartRegistersScheduledJobs(t *testing.T) { + NewScenario(t, "up --no-start must still register a project's scheduled jobs with the engine"). + Requires(JobsExtensionEnabled). + Step("up --no-start creates but never starts web, yet the schedule still fires on its own", + ComposeCmd("up", "--no-start"), + ServiceState("web", "created"), + Eventually(ServiceState("backup", "exited"), 90*time.Second)) +} + +// A project with no service has nothing to follow: an attached up registers +// the jobs and returns, even if an earlier run left its container behind. +func TestUpWithOnlyJobsReturns(t *testing.T) { + NewScenario(t, "up must register the scheduled jobs of a project without service and return"). + Requires(JobsExtensionEnabled). + Step("up registers the scheduled job", + ComposeCmd("up", "-d")). + Step("a failed run leaves its container behind", + ComposeCmd("run", "cron").MayFail()). + // an up that attaches to that container never returns: Within kills it and + // fails the step + Step("an attached up does not wait for, or attach to, that container", + ComposeCmd("up").Within(20*time.Second), + OutputNotContains("Attaching to")) +} + +// create arms the schedule like up does, whether or not the services are +// started: a run that fails for want of them leaves its logs to read. +func TestCreateRegistersScheduledJobs(t *testing.T) { + NewScenario(t, "create must register a project's scheduled jobs with the engine"). + Requires(JobsExtensionEnabled). + Step("create makes web without starting it, yet the schedule fires on its own", + ComposeCmd("create"), + ServiceState("web", "created"), + Eventually(ServiceState("backup", "exited"), 90*time.Second)) +} + // A job runs through `compose run` exactly like a service would: its // declared dependencies start first, its output and exit flow back. Per the // spec, manual execution is always available — scheduled jobs included — diff --git a/internal/e2e/scenario.go b/internal/e2e/scenario.go index 07bb0c2e7..af57adc0c 100644 --- a/internal/e2e/scenario.go +++ b/internal/e2e/scenario.go @@ -28,6 +28,7 @@ import ( "time" "unicode" + "github.com/moby/moby/client" "github.com/moby/moby/client/pkg/versions" "gotest.tools/v3/icmd" ) @@ -674,19 +675,43 @@ func ContainerdImageStore(t testing.TB, c *CLI) string { return "" } -// EngineAPIVersionAtLeast requires a minimum Engine API version (for example -// "1.56"), the unit in which the daemon's features are versioned. -func EngineAPIVersionAtLeast(minimum string) Requirement { +// EngineAPIVersionAtLeast requires the API version Compose negotiates with the +// daemon to be at least the given one, e.g. "1.57". +func EngineAPIVersionAtLeast(version string) Requirement { return func(t testing.TB, c *CLI) string { t.Helper() - version := strings.TrimSpace(c.RunDockerCmd(t, "version", "-f", "{{.Server.APIVersion}}").Stdout()) - if versions.LessThan(version, minimum) { - return fmt.Sprintf("engine API version %s < %s", version, minimum) + if v := engineAPIVersion(t, c); versions.LessThan(v, version) { + return fmt.Sprintf("engine API version %s < %s", v, version) } return "" } } +// EngineAPIVersionBelow requires the API version Compose negotiates with the +// daemon to be older than the given one, for scenarios pinning the behavior of +// Compose on an engine that lacks a feature. +func EngineAPIVersionBelow(version string) Requirement { + return func(t testing.TB, c *CLI) string { + t.Helper() + if v := engineAPIVersion(t, c); !versions.LessThan(v, version) { + return fmt.Sprintf("engine API version %s >= %s", v, version) + } + return "" + } +} + +// engineAPIVersion returns the API version Compose negotiates with the daemon: +// the daemon's, capped at what the client library Compose is built with +// supports. +func engineAPIVersion(t testing.TB, c *CLI) string { + t.Helper() + v := strings.TrimSpace(c.RunDockerCmd(t, "version", "-f", "{{.Server.APIVersion}}").Stdout()) + if versions.LessThan(client.MaxAPIVersion, v) { + return client.MaxAPIVersion + } + return v +} + // EngineVersionAtLeast requires a minimum daemon major version. func EngineVersionAtLeast(major int) Requirement { return func(t testing.TB, c *CLI) string { diff --git a/internal/e2e/testdata/TestCreateRegistersScheduledJobs/compose.yaml b/internal/e2e/testdata/TestCreateRegistersScheduledJobs/compose.yaml new file mode 100644 index 000000000..48e5d07b6 --- /dev/null +++ b/internal/e2e/testdata/TestCreateRegistersScheduledJobs/compose.yaml @@ -0,0 +1,15 @@ +services: + web: + image: alpine + init: true + command: sleep infinity +jobs: + # Fails on purpose: a successful run is removed on exit (RemoveOnSuccess), + # leaving nothing to observe — failing keeps the container around + # (RemoveOnFailure: false) so the fire is verifiable via container state. + backup: + image: alpine + command: sh -c 'exit 1' + triggers: + schedule: + - cron: "* * * * *" diff --git a/internal/e2e/testdata/TestDownWithoutJobsExtension/compose.yaml b/internal/e2e/testdata/TestDownWithoutJobsExtension/compose.yaml new file mode 100644 index 000000000..522e8ccfa --- /dev/null +++ b/internal/e2e/testdata/TestDownWithoutJobsExtension/compose.yaml @@ -0,0 +1,11 @@ +services: + web: + image: alpine + init: true + command: sleep infinity +jobs: + migrate: + image: alpine + command: echo migration done + triggers: + manual: true diff --git a/internal/e2e/testdata/TestUpNoStartRegistersScheduledJobs/compose.yaml b/internal/e2e/testdata/TestUpNoStartRegistersScheduledJobs/compose.yaml new file mode 100644 index 000000000..48e5d07b6 --- /dev/null +++ b/internal/e2e/testdata/TestUpNoStartRegistersScheduledJobs/compose.yaml @@ -0,0 +1,15 @@ +services: + web: + image: alpine + init: true + command: sleep infinity +jobs: + # Fails on purpose: a successful run is removed on exit (RemoveOnSuccess), + # leaving nothing to observe — failing keeps the container around + # (RemoveOnFailure: false) so the fire is verifiable via container state. + backup: + image: alpine + command: sh -c 'exit 1' + triggers: + schedule: + - cron: "* * * * *" diff --git a/internal/e2e/testdata/TestUpRegistersScheduledJobs/compose.yaml b/internal/e2e/testdata/TestUpRegistersScheduledJobs/compose.yaml new file mode 100644 index 000000000..67ac3169f --- /dev/null +++ b/internal/e2e/testdata/TestUpRegistersScheduledJobs/compose.yaml @@ -0,0 +1,21 @@ +services: + web: + image: alpine + init: true + command: sleep infinity +jobs: + # Fails on purpose: a successful run is removed on exit (RemoveOnSuccess), + # leaving nothing to observe — failing keeps the container around + # (RemoveOnFailure: false) so the fire is verifiable via container state. + backup: + image: alpine + command: sh -c 'exit 1' + # repeated `up -d` below must see the identical spec hash each time + environment: + A: "1" + B: "2" + C: "3" + D: "4" + triggers: + schedule: + - cron: "* * * * *" diff --git a/internal/e2e/testdata/TestUpRejectsScheduledJobsWithExtensionOff/compose.yaml b/internal/e2e/testdata/TestUpRejectsScheduledJobsWithExtensionOff/compose.yaml new file mode 100644 index 000000000..5efd658bb --- /dev/null +++ b/internal/e2e/testdata/TestUpRejectsScheduledJobsWithExtensionOff/compose.yaml @@ -0,0 +1,12 @@ +services: + web: + image: alpine + init: true + command: sleep infinity +jobs: + backup: + image: alpine + command: echo backup + triggers: + schedule: + - cron: "0 3 * * *" diff --git a/internal/e2e/testdata/TestUpWithOnlyJobsReturns/compose.yaml b/internal/e2e/testdata/TestUpWithOnlyJobsReturns/compose.yaml new file mode 100644 index 000000000..51e141a73 --- /dev/null +++ b/internal/e2e/testdata/TestUpWithOnlyJobsReturns/compose.yaml @@ -0,0 +1,8 @@ +jobs: + # Fails on purpose: a failed run keeps its container + cron: + image: alpine + command: sh -c 'exit 1' + triggers: + schedule: + - cron: "0 3 * * *" diff --git a/pkg/api/labels.go b/pkg/api/labels.go index a45267348..5a4e40f13 100644 --- a/pkg/api/labels.go +++ b/pkg/api/labels.go @@ -88,6 +88,8 @@ const ( // declared hook with the runner container the reconciliation plan // prepared for it. HookIndexLabel = "com.docker.compose.hook-index" + // JobLabel allow to track resource related to a compose job + JobLabel = "com.docker.compose.job" ) // ComposeVersion is the compose tool version as declared by label VersionLabel diff --git a/pkg/compose/api_versions.go b/pkg/compose/api_versions.go index 53fc5dded..a5043bca6 100644 --- a/pkg/compose/api_versions.go +++ b/pkg/compose/api_versions.go @@ -60,6 +60,16 @@ const ( // - The daemon silently ignores HostConfig.Umask, so the container would // run with the runtime's default umask instead of the requested one apiVersion156 = "1.56" + + // apiVersionJobs is the first Docker Engine API version of an engine + // shipping the jobs extension (one-shot and scheduled jobs). The extension + // is disabled by default (dockerd --extension-enable org.mobyproject.jobs.v0), + // so this version only says it is compiled in: jobsAvailability probes it. + // + // Before this version, or with the extension disabled: + // - Scheduled jobs cannot be registered, they are refused + // - Manual jobs run as one-off service containers + apiVersionJobs = "1.57" ) // Docker Engine version strings for user-facing error messages. diff --git a/pkg/compose/compose.go b/pkg/compose/compose.go index 9e527c9ad..a5b94a9c7 100644 --- a/pkg/compose/compose.go +++ b/pkg/compose/compose.go @@ -30,10 +30,12 @@ import ( clitypes "github.com/docker/cli/cli/config/types" "github.com/docker/cli/cli/flags" "github.com/docker/cli/cli/streams" + extensionclient "github.com/moby/extensions/client" "github.com/moby/moby/api/types/swarm" "github.com/moby/moby/client" "github.com/sirupsen/logrus" + jobsv0 "github.com/docker/compose/v5/internal/jobsapi" "github.com/docker/compose/v5/internal/naming" "github.com/docker/compose/v5/internal/progress" "github.com/docker/compose/v5/internal/projectloader" @@ -241,6 +243,19 @@ type composeService struct { // logStreamDrainTimeout bounds the shutdown drain of re-attached log // streams, see drainTimeout. logStreamDrainTimeout time.Duration + + jobsGRPCOnce sync.Once + jobsExtClient *extensionclient.Client + jobsAPI jobsv0.Jobs + jobsAPIErr error + + // jobsProbe memoizes the first definitive answer of the jobs probe: the + // engine's set of extensions is fixed for the lifetime of its daemon. + jobsProbe struct { + mu sync.Mutex + known bool + status jobsStatus + } } // Close releases any connections/resources held by the underlying clients. @@ -252,6 +267,9 @@ func (s *composeService) Close() error { if s.dockerCli != nil { errs = append(errs, s.apiClient().Close()) } + if s.jobsExtClient != nil { + errs = append(errs, s.jobsExtClient.Close()) + } return errors.Join(errs...) } diff --git a/pkg/compose/containers.go b/pkg/compose/containers.go index fb41fc075..71a3dc6c6 100644 --- a/pkg/compose/containers.go +++ b/pkg/compose/containers.go @@ -178,6 +178,20 @@ func isOrphaned(project *types.Project) containerPredicate { } } +// isLiveJobRun selects a run of a job that has not finished. A job is not a +// service of the model, so its run reads as an orphan whatever its state, which +// `down --remove-orphans` relies on to stop it; `up --remove-orphans` must leave +// it alone, as it does a running one-off. +func isLiveJobRun(c container.Summary) bool { + return isJobRun(c) && c.State != container.StateExited && c.State != container.StateDead +} + +// isJobRun selects the container of a run of a job: the engine labels it. +func isJobRun(c container.Summary) bool { + _, ok := c.Labels[jobContainerIDLabel] + return ok +} + func isNotOneOff(c container.Summary) bool { v, ok := c.Labels[api.OneoffLabel] return !ok || v == "False" diff --git a/pkg/compose/convert.go b/pkg/compose/convert.go index a854564f6..f851cc8f1 100644 --- a/pkg/compose/convert.go +++ b/pkg/compose/convert.go @@ -20,6 +20,7 @@ import ( "context" "errors" "fmt" + "slices" "time" compose "github.com/compose-spec/compose-go/v2/types" @@ -36,6 +37,10 @@ func ToMobyEnv(environment compose.MappingWithEquals) []string { env = append(env, fmt.Sprintf("%s=%s", k, *v)) } } + // environment is a map: iteration order is random, and Env ends up + // marshaled verbatim into the job spec sent to the Engine, whose spec + // hash is order-sensitive. Keys are unique, so sorting is safe. + slices.Sort(env) return env } diff --git a/pkg/compose/create.go b/pkg/compose/create.go index 45dce35e8..cd4813acc 100644 --- a/pkg/compose/create.go +++ b/pkg/compose/create.go @@ -72,7 +72,16 @@ func (s *composeService) Create(ctx context.Context, project *types.Project, opt return err } return Run(ctx, func(ctx context.Context) error { - return s.create(ctx, project, options) + if err := s.rejectUnsupportedScheduledJobs(ctx, project); err != nil { + return err + } + if err := s.create(ctx, project, options); err != nil { + return err + } + if err := s.registerScheduledJobs(ctx, project); err != nil { + return err + } + return s.handleOrphanJobs(ctx, project, options.RemoveOrphans, options.IgnoreOrphans) }, "create", s.events) } @@ -168,11 +177,22 @@ func (s *composeService) preparePlan(ctx context.Context, project *types.Project // requested services get created too warnIgnoredFileReferences(project, project.ServiceNames()) - if len(observed.Orphans) > 0 && !options.IgnoreOrphans && !options.RemoveOrphans { + // A kept job run container (RemoveOnFailure: false) has no matching + // service by design — it's not something the user forgot to clean up, so + // it shouldn't trigger this warning. It's still a real orphan for + // reconcileOrphans (down/up --remove-orphans can sweep it), so the + // exclusion is scoped to this warning's own name list, not observed.Orphans. + var nonJobOrphans []string + for _, o := range observed.Orphans { + if !isJobRun(o.Summary) { + nonJobOrphans = append(nonJobOrphans, o.Name) + } + } + if len(nonJobOrphans) > 0 && !options.IgnoreOrphans && !options.RemoveOrphans { logrus.Warnf("Found orphan containers (%s) for this project. If "+ "you removed or renamed this service in your compose "+ "file, you can run this command with the "+ - "--remove-orphans flag to clean it up.", observed.orphanNames()) + "--remove-orphans flag to clean it up.", strings.Join(nonJobOrphans, ", ")) } reconcileOptions := toReconcileOptions(options) @@ -438,6 +458,11 @@ func (s *composeService) getCreateConfigs(ctx context.Context, dnsIPs = append(dnsIPs, dnsIP) } + // Same ordering requirement as ToMobyEnv: AsList's map iteration order + // is random, and this ends up in the job spec hash. + extraHosts := service.ExtraHosts.AsList(":") + slices.Sort(extraHosts) + hostConfig := container.HostConfig{ AutoRemove: options.AutoRemove, Annotations: service.Annotations, @@ -460,7 +485,7 @@ func (s *composeService) getCreateConfigs(ctx context.Context, DNS: dnsIPs, DNSSearch: service.DNSSearch, DNSOptions: service.DNSOpts, - ExtraHosts: service.ExtraHosts.AsList(":"), + ExtraHosts: extraHosts, SecurityOpt: securityOpts, StorageOpt: service.StorageOpt, UsernsMode: container.UsernsMode(service.UserNSMode), @@ -887,6 +912,11 @@ func toUlimits(m map[string]*types.UlimitsConfig) []*container.Ulimit { Soft: int64(soft), }) } + // Same ordering requirement as ToMobyEnv: m is a map, so iteration + // order is random, and this ends up in the job spec hash. + slices.SortFunc(ulimits, func(a, b *container.Ulimit) int { + return strings.Compare(a.Name, b.Name) + }) return ulimits } @@ -1224,6 +1254,12 @@ func (s *composeService) buildContainerMountOptions(ctx context.Context, p types for _, v := range mounts { values = append(values, v) } + // Same ordering requirement as ToMobyEnv: mounts is keyed by target + // path, so iteration order is random, and this ends up in the job + // spec hash. + slices.SortFunc(values, func(a, b mount.Mount) int { + return strings.Compare(a.Target, b.Target) + }) return values, nil } diff --git a/pkg/compose/create_test.go b/pkg/compose/create_test.go index 350de6ecf..95a9c7797 100644 --- a/pkg/compose/create_test.go +++ b/pkg/compose/create_test.go @@ -21,7 +21,6 @@ import ( "net/netip" "os" "path/filepath" - "sort" "testing" composeloader "github.com/compose-spec/compose-go/v2/loader" @@ -166,9 +165,6 @@ func TestBuildContainerMountOptions(t *testing.T) { mock.EXPECT().ImageInspect(gomock.Any(), "myProject-myService").AnyTimes().Return(client.ImageInspectResult{}, nil) mounts, err := s.buildContainerMountOptions(t.Context(), project, project.Services["myService"], inherit) - sort.Slice(mounts, func(i, j int) bool { - return mounts[i].Target < mounts[j].Target - }) assert.NilError(t, err) assert.Assert(t, len(mounts) == 4) assert.Equal(t, mounts[0].Target, "/var/myvolume1") @@ -178,9 +174,6 @@ func TestBuildContainerMountOptions(t *testing.T) { assert.Equal(t, mounts[3].Target, "\\\\.\\pipe\\docker_engine") mounts, err = s.buildContainerMountOptions(t.Context(), project, project.Services["myService"], inherit) - sort.Slice(mounts, func(i, j int) bool { - return mounts[i].Target < mounts[j].Target - }) assert.NilError(t, err) assert.Assert(t, len(mounts) == 4) assert.Equal(t, mounts[0].Target, "/var/myvolume1") diff --git a/pkg/compose/depends_on_label.go b/pkg/compose/depends_on_label.go index b924f946a..128955204 100644 --- a/pkg/compose/depends_on_label.go +++ b/pkg/compose/depends_on_label.go @@ -18,6 +18,7 @@ package compose import ( "fmt" + "slices" "strconv" "strings" @@ -32,6 +33,9 @@ func encodeDependsOnLabel(dependsOn types.DependsOnConfig) string { for dep, d := range dependsOn { dependencies = append(dependencies, fmt.Sprintf("%s:%s:%t", dep, d.Condition, d.Restart)) } + // map iteration is random: the label is part of a job's spec, hashed by the + // engine, so its entries must come in a stable order + slices.Sort(dependencies) return strings.Join(dependencies, ",") } diff --git a/pkg/compose/down.go b/pkg/compose/down.go index 70066c3a5..fb1e6ec86 100644 --- a/pkg/compose/down.go +++ b/pkg/compose/down.go @@ -92,6 +92,7 @@ func (s *composeService) down(ctx context.Context, projectName string, options a if err != nil { return err } + s.loadJobsToRemove(ctx, project, projectName, options) // keep only the requested services that exist in the model var services []string @@ -158,6 +159,11 @@ func (s *composeService) down(ctx context.Context, projectName string, options a ops := s.ensureNetworksDown(ctx, project, limiter) ops = append(ops, s.ensureRelayLinkNetworksDown(ctx, project, limiter)...) + // unlike a network, which stays while a container still uses it, nothing + // keeps a job that a partial down must leave alone + if len(options.Services) == 0 { + ops = append(ops, s.ensureJobsDown(ctx, project)...) + } if imageProject != nil { ops = append(ops, s.ensureImagesDown(ctx, imageProject, options, limiter)...) diff --git a/pkg/compose/down_test.go b/pkg/compose/down_test.go index c45948b99..307240661 100644 --- a/pkg/compose/down_test.go +++ b/pkg/compose/down_test.go @@ -21,6 +21,7 @@ import ( "errors" "fmt" "os" + "slices" "sort" "strings" "testing" @@ -38,6 +39,7 @@ import ( "gotest.tools/v3/assert" "github.com/docker/compose/v5/internal/concurrency" + jobsv0 "github.com/docker/compose/v5/internal/jobsapi" "github.com/docker/compose/v5/internal/naming" compose "github.com/docker/compose/v5/pkg/api" "github.com/docker/compose/v5/pkg/mocks" @@ -61,11 +63,25 @@ func TestDownRejectsInvalidImagePruneModeUpfront(t *testing.T) { assert.ErrorContains(t, err, `invalid image prune mode "bogus"`) } +// expectNoJobsEngine makes down's lookup of the jobs the engine has for the +// project (no compose file to read them from, or --remove-orphans) see an +// engine whose API predates jobs: no jobs, and no gRPC call to mock. +func expectNoJobsEngine(api *mocks.MockAPIClient) { + expectEngineAPIVersion(api, noJobsEngine) +} + +func expectEngineAPIVersion(api *mocks.MockAPIClient, version string) { + api.EXPECT().Ping(gomock.Any(), client.PingOptions{NegotiateAPIVersion: true}). + Return(client.PingResult{APIVersion: version}, nil).AnyTimes() + api.EXPECT().ClientVersion().Return(version).AnyTimes() +} + func TestDown(t *testing.T) { mockCtrl := gomock.NewController(t) defer mockCtrl.Finish() api, cli := prepareMocks(mockCtrl) + expectNoJobsEngine(api) tested, err := NewComposeService(cli) assert.NilError(t, err) @@ -175,6 +191,7 @@ func TestDown_ConcurrencyIsBoundedAcrossServices(t *testing.T) { // TestDown_ConcurrencyIsBoundedAcrossServices guards for container removal. func TestDown_ImagePruningSharesConcurrencyBudgetAcrossOps(t *testing.T) { svc, apiClient := newTestService(t, WithMaxConcurrency(2)) + expectNoJobsEngine(apiClient) project := &types.Project{Name: "prj"} @@ -225,6 +242,7 @@ func TestDown_ImagePruningSharesConcurrencyBudgetAcrossOps(t *testing.T) { // between the two image ops. func TestDown_NetworkAndImageRemovalShareConcurrencyBudget(t *testing.T) { svc, apiClient := newTestService(t, WithMaxConcurrency(2)) + expectNoJobsEngine(apiClient) project := &types.Project{ Name: "prj", @@ -288,6 +306,7 @@ func TestDownWithGivenServices(t *testing.T) { defer mockCtrl.Finish() api, cli := prepareMocks(mockCtrl) + expectNoJobsEngine(api) tested, err := NewComposeService(cli) assert.NilError(t, err) @@ -339,11 +358,111 @@ func TestDownWithGivenServices(t *testing.T) { assert.NilError(t, err) } +// down --remove-orphans also removes the jobs a previous up registered from a +// file that declared more; without the flag it removes the declared ones only. +func TestDownRemoveOrphansRemovesRegisteredJobsTheFileDroppedKeeps(t *testing.T) { + for name, tc := range map[string]struct { + removeOrphans bool + want []string + }{ + "without the flag": {removeOrphans: false, want: []string{"prj.backup"}}, + "with --remove-orphans": {removeOrphans: true, want: []string{"prj.backup", "prj.old"}}, + } { + t.Run(name, func(t *testing.T) { + svc, apiClient := newTestService(t) + expectEngineAPIVersion(apiClient, apiVersionJobs) + jobs := &fakeJobsClient{listReply: &jobsv0.ListReply{Jobs: []jobsv0.Job{{Name: "prj.backup"}, {Name: "prj.old"}}}} + svc.jobsGRPCOnce.Do(func() {}) + svc.jobsAPI = jobs + + apiClient.EXPECT().ContainerList(gomock.Any(), gomock.Any()).Return(client.ContainerListResult{}, nil).Times(2) + apiClient.EXPECT().NetworkList(gomock.Any(), client.NetworkListOptions{ + Filters: naming.ProjectFilter("prj").Add("label", compose.RelayNetworkLabel), + }).Return(client.NetworkListResult{}, nil) + + project := &types.Project{Name: "prj", Jobs: types.Jobs{"backup": {Name: "backup"}}} + assert.NilError(t, svc.down(t.Context(), "prj", compose.DownOptions{Project: project, RemoveOrphans: tc.removeOrphans})) + + var removed []string + for _, call := range jobs.removeCalls { + removed = append(removed, call.JobRef) + } + slices.Sort(removed) + assert.DeepEqual(t, removed, tc.want) + }) + } +} + +// down with a selection of services leaves the project's jobs alone: unlike a +// network, which stays while a container still uses it, nothing keeps a job. +func TestDownWithGivenServicesKeepsJobs(t *testing.T) { + mockCtrl := gomock.NewController(t) + defer mockCtrl.Finish() + + api, cli := prepareMocks(mockCtrl) + expectEngineAPIVersion(api, apiVersionJobs) + tested, err := NewComposeService(cli) + assert.NilError(t, err) + jobs := &fakeJobsClient{listReply: &jobsv0.ListReply{Jobs: []jobsv0.Job{{Name: strings.ToLower(testProject) + ".backup"}}}} + tested.(*composeService).jobsGRPCOnce.Do(func() {}) + tested.(*composeService).jobsAPI = jobs + + api.EXPECT().ContainerList(gomock.Any(), projectFilterListOpt(false)).Return(client.ContainerListResult{ + Items: []container.Summary{ + testContainer("service1", "123", false), + testContainer("service2", "456", false), + testContainer("service2", "789", false), + testContainer("service_orphan", "321", true), + }, + }, nil) + api.EXPECT().VolumeList( + gomock.Any(), + client.VolumeListOptions{ + Filters: naming.ProjectFilter(strings.ToLower(testProject)), + }). + Return(client.VolumeListResult{}, nil) + + // network names are not guaranteed to be unique, ensure Compose handles + // cleanup properly if duplicates are inadvertently created + api.EXPECT().NetworkList(gomock.Any(), client.NetworkListOptions{Filters: naming.ProjectFilter(strings.ToLower(testProject))}). + Return(client.NetworkListResult{Items: []network.Summary{ + {Network: network.Network{ID: "abc123", Name: "myProject_default", Labels: map[string]string{compose.NetworkLabel: "default"}}}, + {Network: network.Network{ID: "def456", Name: "myProject_default", Labels: map[string]string{compose.NetworkLabel: "default"}}}, + }}, nil) + + api.EXPECT().ContainerStop(gomock.Any(), "123", client.ContainerStopOptions{}).Return(client.ContainerStopResult{}, nil) + + api.EXPECT().ContainerRemove(gomock.Any(), "123", client.ContainerRemoveOptions{Force: true}).Return(client.ContainerRemoveResult{}, nil) + + api.EXPECT().NetworkList(gomock.Any(), client.NetworkListOptions{ + Filters: naming.ProjectFilter(strings.ToLower(testProject)).Add("label", networkFilter("default")), + }).Return(client.NetworkListResult{Items: []network.Summary{ + {Network: network.Network{ID: "abc123", Name: "myProject_default"}}, + }}, nil) + api.EXPECT().NetworkInspect(gomock.Any(), "abc123", gomock.Any()).Return(client.NetworkInspectResult{Network: network.Inspect{Network: network.Network{ID: "abc123"}}}, nil) + api.EXPECT().NetworkRemove(gomock.Any(), "abc123", gomock.Any()).Return(client.NetworkRemoveResult{}, nil) + + // no relay-link network for this project (no provider service involved) + api.EXPECT().NetworkList(gomock.Any(), client.NetworkListOptions{ + Filters: naming.ProjectFilter(strings.ToLower(testProject)).Add("label", compose.RelayNetworkLabel), + }).Return(client.NetworkListResult{}, nil) + + api.EXPECT().ContainerList(gomock.Any(), hookFilterListOpt("service1")).Return(client.ContainerListResult{}, nil) + + err = tested.Down(t.Context(), strings.ToLower(testProject), compose.DownOptions{ + Services: []string{"service1", "not-running-service"}, + }) + assert.NilError(t, err) + assert.Equal(t, len(jobs.listCalls), 0, "a partial down does not look the jobs up") + assert.Equal(t, len(jobs.removeCalls), 0, "a partial down must not remove the project's jobs") +} + func TestDownWithSpecifiedServiceButTheServicesAreNotRunning(t *testing.T) { mockCtrl := gomock.NewController(t) defer mockCtrl.Finish() api, cli := prepareMocks(mockCtrl) + expectNoJobsEngine(api) tested, err := NewComposeService(cli) assert.NilError(t, err) @@ -381,6 +500,7 @@ func TestDownRemoveOrphans(t *testing.T) { defer mockCtrl.Finish() api, cli := prepareMocks(mockCtrl) + expectNoJobsEngine(api) tested, err := NewComposeService(cli) assert.NilError(t, err) @@ -450,6 +570,7 @@ func TestDownRemoveVolumes(t *testing.T) { defer mockCtrl.Finish() api, cli := prepareMocks(mockCtrl) + expectNoJobsEngine(api) tested, err := NewComposeService(cli) assert.NilError(t, err) @@ -721,6 +842,7 @@ func TestDownRemoveImagesWithSelectedServices(t *testing.T) { t.Run(fmt.Sprintf("%s/orphans=%t", mode, removeOrphans), func(t *testing.T) { mockCtrl := gomock.NewController(t) apiClient, cli := prepareMocks(mockCtrl) + expectNoJobsEngine(apiClient) tested, err := NewComposeService(cli) assert.NilError(t, err) @@ -788,6 +910,7 @@ func TestDownRemoveImages_NoLabel(t *testing.T) { defer mockCtrl.Finish() api, cli := prepareMocks(mockCtrl) + expectNoJobsEngine(api) tested, err := NewComposeService(cli) assert.NilError(t, err) @@ -1438,6 +1561,7 @@ func TestDownRemovesRetainedPreStartHookContainers(t *testing.T) { defer mockCtrl.Finish() api, cli := prepareMocks(mockCtrl) + expectNoJobsEngine(api) tested, err := NewComposeService(cli) assert.NilError(t, err) @@ -1484,6 +1608,7 @@ func TestDownHookContainerRemovalFailureIsNonFatal(t *testing.T) { defer mockCtrl.Finish() api, cli := prepareMocks(mockCtrl) + expectNoJobsEngine(api) tested, err := NewComposeService(cli) assert.NilError(t, err) diff --git a/pkg/compose/images_test.go b/pkg/compose/images_test.go index 921edebf8..a541beb74 100644 --- a/pkg/compose/images_test.go +++ b/pkg/compose/images_test.go @@ -47,7 +47,6 @@ func attestationManifest() image.ManifestSummary { return image.ManifestSummary{ID: "sha256:att", Kind: image.ManifestKindAttestation, Available: true} } -//nolint:unparam // the tests of older engines moved to internal/images func newTestComposeService(t *testing.T, mockCtrl *gomock.Controller, apiVersion string) (*mocks.MockAPIClient, *composeService) { t.Helper() api, cli := prepareMocks(mockCtrl) diff --git a/pkg/compose/jobs.go b/pkg/compose/jobs.go new file mode 100644 index 000000000..5dff7d537 --- /dev/null +++ b/pkg/compose/jobs.go @@ -0,0 +1,577 @@ +/* + Copyright 2020 Docker Compose CLI authors + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package compose + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "maps" + "slices" + "strings" + + "github.com/compose-spec/compose-go/v2/types" + "github.com/containerd/errdefs" + extensionclient "github.com/moby/extensions/client" + containerType "github.com/moby/moby/api/types/container" + "github.com/moby/moby/client/pkg/versions" + "github.com/sirupsen/logrus" + "golang.org/x/sync/errgroup" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" + + jobsv0 "github.com/docker/compose/v5/internal/jobsapi" + jobspb "github.com/docker/compose/v5/internal/jobsapi/protogen" + "github.com/docker/compose/v5/internal/naming" + "github.com/docker/compose/v5/pkg/api" +) + +// jobRunHistoryLimit caps retained terminal run records (see +// jobsv0.JobSpec.RunHistoryLimit): low enough that repeated failures of a +// frequently-firing schedule don't accumulate kept containers indefinitely. +const jobRunHistoryLimit = 5 + +// jobsClient lazily resolves the engine's jobs extension point over the same +// dialer the Docker API client already uses. Resolve never dials by itself — +// an engine without the jobs feature only surfaces codes.Unimplemented on +// the first real call — so projects with no jobs never touch the network. +func (s *composeService) jobsClient() (jobsv0.Jobs, error) { + s.jobsGRPCOnce.Do(func() { + exts, err := extensionclient.New(s.apiClient(), extensionclient.WithGRPCPoint(jobspb.ClientPoint)) + if err != nil { + s.jobsAPIErr = err + return + } + s.jobsExtClient = exts + s.jobsAPI, s.jobsAPIErr = extensionclient.Resolve(exts, jobsv0.Point) + }) + return s.jobsAPI, s.jobsAPIErr +} + +// jobsStatus is what the engine offers for jobs. +type jobsStatus int + +const ( + jobsAvailable jobsStatus = iota + jobsEngineTooOld + jobsExtensionOff +) + +// errJobsExtensionNotReady reports a jobs extension that is enabled but does +// not answer yet (or failed to activate): not missing support, a failure. +var errJobsExtensionNotReady = errors.New("the engine's jobs extension is enabled but not ready") + +// jobsAvailability reports what the engine offers for jobs. The API version +// only says the extension is compiled in: it is disabled by default and the +// engine cannot list its extensions, so a call scoped to the project tells +// (Unimplemented: off). Unavailable (enabled, not ready) is an error, never a +// status, so no caller degrades on it. +func (s *composeService) jobsAvailability(ctx context.Context, projectName string) (jobsStatus, error) { + version, err := s.RuntimeAPIVersion(ctx) + if err != nil { + return jobsAvailable, err + } + if versions.LessThan(version, apiVersionJobs) { + return jobsEngineTooOld, nil + } + + s.jobsProbe.mu.Lock() + defer s.jobsProbe.mu.Unlock() + if s.jobsProbe.known { + return s.jobsProbe.status, nil + } + probed, err := s.probeJobs(ctx, projectName) + if err != nil { + return jobsAvailable, err + } + s.jobsProbe.status, s.jobsProbe.known = probed, true + return probed, nil +} + +func (s *composeService) probeJobs(ctx context.Context, projectName string) (jobsStatus, error) { + jc, err := s.jobsClient() + if err != nil { + return jobsAvailable, err + } + _, err = jc.List(ctx, &jobsv0.ListRequest{Labels: []string{naming.LabelFilter(api.ProjectLabel, projectName)}}) + switch status.Code(err) { + case codes.OK: + return jobsAvailable, nil + case codes.Unimplemented: + return jobsExtensionOff, nil + case codes.Unavailable: + return jobsAvailable, fmt.Errorf("%w: %s", errJobsExtensionNotReady, status.Convert(err).Message()) + default: + return jobsAvailable, err + } +} + +// rejectUnsupportedScheduledJobs refuses a project declaring active scheduled +// jobs when the engine cannot run them: silently not scheduling them would +// break the user's expectations, unlike manual jobs which simply wait for an +// explicit trigger. +func (s *composeService) rejectUnsupportedScheduledJobs(ctx context.Context, project *types.Project) error { + names := sortedJobNames(project.Jobs, HasSchedule) + if len(names) == 0 { + return nil + } + st, err := s.jobsAvailability(ctx, project.Name) + if err != nil { + return err + } + switch st { + case jobsEngineTooOld: + return fmt.Errorf("scheduled jobs are not supported by this version of the engine: %s", strings.Join(names, ", ")) + case jobsExtensionOff: + return fmt.Errorf("scheduled jobs need the engine's jobs extension, which is not enabled (start dockerd with %s): %s", + jobsv0.EnableFlag, strings.Join(names, ", ")) + } + return nil +} + +// engineJobNameSeparator joins a project and job name into the engine's +// daemon-wide job name. The engine only accepts [a-zA-Z0-9][a-zA-Z0-9_.-]* for +// job names, which rules out "/" as a separator (confirmed against a live +// engine: it rejects it with InvalidArgument). "." still works: project names +// are restricted to [a-z0-9_-] (see compose-go's NormalizeProjectName, no +// dot), while job names may contain one, so a "." can only originate from the +// job-name side — the join stays unambiguous to reverse. Unlike api.Separator +// ("-"), which both project and job names can contain, this avoids project +// "app-sub" + job "service" colliding with project "app" + job "sub-service". +const engineJobNameSeparator = "." + +// engineJobName is the daemon-wide unique name Compose registers a job +// under: the engine has no notion of Compose projects, so the project name +// is folded into the job name. +func engineJobName(project *types.Project, name string) string { + return project.Name + engineJobNameSeparator + name +} + +// jobTrigger translates compose-go's TriggerConfig into the engine's +// Trigger type: 1:1 field mapping, already verified against the engine +// contract. +func jobTrigger(job types.JobConfig) (*jobsv0.Trigger, error) { + switch { + case job.Triggers == nil: + return nil, fmt.Errorf("job %q has no trigger", job.Name) + case job.Triggers.Manual != nil && *job.Triggers.Manual && len(job.Triggers.Schedule) > 0: + return nil, fmt.Errorf("job %q declares both manual:true and a schedule, exactly one is supported", job.Name) + case job.Triggers.Manual != nil && *job.Triggers.Manual: + return &jobsv0.Trigger{Manual: true}, nil + case len(job.Triggers.Schedule) == 1: + sc := job.Triggers.Schedule[0] + return &jobsv0.Trigger{Schedule: &jobsv0.ScheduleTrigger{ + Cron: sc.Cron, + Timezone: sc.Timezone, + Concurrency: sc.Concurrency, + MissedFires: sc.MissedFires, + }}, nil + case len(job.Triggers.Schedule) > 1: + return nil, fmt.Errorf("job %q declares %d schedules, exactly one is supported", job.Name, len(job.Triggers.Schedule)) + default: + return nil, fmt.Errorf("job %q has no trigger", job.Name) + } +} + +// buildJobSpec builds the engine JobSpec for a job: svc is resolved exactly +// like a service (materializeManualJob or the synthetic ServiceConfig built +// from the JobConfig for scheduled jobs), and getCreateConfigs is the same +// service->container-create-body conversion used to create real containers. +func (s *composeService) buildJobSpec(ctx context.Context, project *types.Project, svc types.ServiceConfig, job types.JobConfig, useNetworkAliases bool) (*jobsv0.JobSpec, error) { + trigger, err := jobTrigger(job) + if err != nil { + return nil, err + } + + // The engine only carries JobSpec.Labels on the job object, not on the run + // container (see jobsv0.JobSpec.Labels godoc): project/service labels must + // be set here on the container spec itself so run containers stay visible + // to the rest of Compose's tooling (ps, label-scoped listings) exactly + // like any other service container. svc.CustomLabels is trustworthy here + // because JobAsService is the single place both materialization paths + // (RunJob's cmd-layer materializeManualJob and registerScheduledJobs' + // scopedProjectForJob) build it — the same job's spec no longer differs + // depending on how it was triggered. + // + // The engine hashes the container spec, labels included, and refuses a job + // whose spec changed: ensureImagesExists' image digest label would make up + // and run fail each time the image behind the same reference is pulled + // again, though the engine creates every run from that reference and uses + // the new image on its own. + labels := mergeLabels(svc.Labels, svc.CustomLabels) + delete(labels, api.ImageDigestLabel) + cfgs, err := s.getCreateConfigs(ctx, project, svc, 1, nil, createOptions{ + UseNetworkAliases: useNetworkAliases, + Labels: labels, + }) + if err != nil { + return nil, err + } + spec, err := json.Marshal(containerType.CreateRequest{ + Config: cfgs.Container, + HostConfig: cfgs.Host, + NetworkingConfig: cfgs.Network, + }) + if err != nil { + return nil, err + } + + return &jobsv0.JobSpec{ + ContainerSpec: spec, + Trigger: trigger, + Labels: map[string]string{ + api.ProjectLabel: project.Name, + api.JobLabel: job.Name, + }, + // Successful runs are disposable: drop the container once its + // terminal record is written. Failed ones are kept for postmortem, + // bounded by RunHistoryLimit so repeated failures don't accumulate. + RemoveOnSuccess: true, + RemoveOnFailure: false, + RunHistoryLimit: jobRunHistoryLimit, + }, nil +} + +// sortedJobNames returns the sorted names of a project's jobs, optionally +// restricted to those matching keep. +func sortedJobNames(jobs types.Jobs, keep func(types.JobConfig) bool) []string { + if keep == nil { + return slices.Sorted(maps.Keys(jobs)) + } + filtered := make(types.Jobs, len(jobs)) + for name, job := range jobs { + if keep(job) { + filtered[name] = job + } + } + return slices.Sorted(maps.Keys(filtered)) +} + +// jobChangedErr reports that the engine already has a job of this name with +// a different spec: re-running the same verb won't reconcile it, only +// `down` removing it first will. +func jobChangedErr(name, verb string) error { + return fmt.Errorf("job %q has changed: run `docker compose down` to remove it, then `%s` again", name, verb) +} + +// mapAlreadyExists maps err through jobsv0.MapError, translating an +// AlreadyExists (the engine already has name registered with a different +// spec) into jobChangedErr instead of the raw engine error. +func mapAlreadyExists(err error, name, verb string) error { + err = jobsv0.MapError(err) + if errdefs.IsAlreadyExists(err) { + return jobChangedErr(name, verb) + } + return err +} + +// HasSchedule reports whether a job declares a schedule trigger — the +// predicate `up` uses to register it with the engine instead of warning +// that it waits for `docker compose run`. +func HasSchedule(job types.JobConfig) bool { + return job.Triggers != nil && len(job.Triggers.Schedule) > 0 +} + +// ManualTriggerDisabled reports whether a job explicitly opts out of manual +// triggering with `triggers.manual: false`. Per the spec, every job accepts +// `docker compose run` regardless of its automated triggers unless it opts +// out this way — this is the single source of truth for that rule, shared +// by every layer that materializes or runs a job manually. +func ManualTriggerDisabled(job types.JobConfig) bool { + return job.Triggers != nil && job.Triggers.Manual != nil && !*job.Triggers.Manual +} + +// JobAsService materializes a job as a service for the one-off machinery: a +// job is a ContainerSpec+WorkloadSpec, the same layers a service is made of. +// It carries the standard custom labels the loader stamps on every service — +// materialization happens after loading, so without them the containers +// created for the job would be invisible to every label-driven path: start +// would silently skip them, ps/down would not see them, and the dependency +// wait would report the job as a missing dependency. Exported as the single +// source of truth both materialization paths (cmd/compose/run.go's +// materializeManualJob for a manual run, scopedProjectForJob below for a +// scheduled registration) share, so the same job's spec doesn't differ +// depending on how it was triggered. +func JobAsService(project *types.Project, name string, job types.JobConfig) types.ServiceConfig { + svc := types.ServiceConfig{ + Name: name, + Profiles: job.Profiles, + Extensions: job.Extensions, + ContainerSpec: job.ContainerSpec, + WorkloadSpec: job.WorkloadSpec, + } + svc.CustomLabels = types.Labels{ + api.ProjectLabel: project.Name, + api.ServiceLabel: name, + api.WorkingDirLabel: project.WorkingDir, + api.ConfigFilesLabel: strings.Join(project.ComposeFiles, ","), + api.OneoffLabel: "False", + } + return svc +} + +// scopedProjectForJob returns a shallow copy of project with name's job +// materialized as its only service, so a scheduled job can run through the same +// project-wide normalization passes (useAPISocket, ensureImagesExists, +// ensureModels) that a manually-run job gets for free once materializeManualJob +// puts it into the real project.Services. The services stay in the copy, as +// disabled ones, for the job to name them (see resolveJobServiceReferences): the +// passes then leave their images alone, `up` already went through them. The +// copy is throwaway: callers discard it once they've read back the resolved +// ServiceConfig, so the job never joins the project the reconciliation loop +// iterates over. +// +// Those passes write into the job's Environment, Volumes and Configs, and into +// the project's Configs, in place: the job and the project's Configs get their +// own copy, so nothing is written through to project.Jobs or to the Configs the +// other jobs registering concurrently copy from. +func scopedProjectForJob(project *types.Project, name string, job types.JobConfig) *types.Project { + scoped := *project + scoped.Services = types.Services{name: withOwnMutableFields(JobAsService(project, name, job))} + scoped.DisabledServices = make(types.Services, len(project.Services)+len(project.DisabledServices)) + maps.Copy(scoped.DisabledServices, project.DisabledServices) + maps.Copy(scoped.DisabledServices, project.Services) + scoped.Configs = maps.Clone(project.Configs) + return &scoped +} + +// withOwnMutableFields returns svc with its own copy of the fields the +// normalization passes write into in place. CustomLabels is not one of them: +// JobAsService builds it afresh. +func withOwnMutableFields(svc types.ServiceConfig) types.ServiceConfig { + // not nil: SetModelVariables and useAPISocket write into it + env := make(types.MappingWithEquals, len(svc.Environment)) + maps.Copy(env, svc.Environment) + svc.Environment = env + svc.Volumes = slices.Clone(svc.Volumes) + svc.Configs = slices.Clone(svc.Configs) + return svc +} + +// resolveJobServiceReferences replaces the services a job shares volumes or +// namespaces with by the name of their first container. The engine knows no +// Compose service, and an ID would not survive the service being recreated: it +// would change the spec, which the engine hashes and holds immutable, and the +// container a cron fires on later. Both `up` and `run` go through here, so the +// spec of a job is the same whichever of them sends it. +func resolveJobServiceReferences(project *types.Project, service *types.ServiceConfig) error { + return resolveReferences(service, func(name string) (string, bool) { + target, ok := project.Services[name] + if !ok { + // the selection (--no-deps, up ) narrows the project + target, ok = project.DisabledServices[name] + } + if !ok { + return "", false + } + return naming.ContainerName(project.Name, target, 1), true + }) +} + +// registerScheduledJobs registers the project's scheduled jobs with the +// engine. Create is idempotent on SpecHash: re-applying the same spec on a +// later `up` is a no-op, which is what makes `up` safely re-runnable. +func (s *composeService) registerScheduledJobs(ctx context.Context, project *types.Project) error { + names := sortedJobNames(project.Jobs, HasSchedule) + if len(names) == 0 { + return nil + } + + jc, err := s.jobsClient() + if err != nil { + return err + } + eg, ctx := errgroup.WithContext(ctx) + eg.SetLimit(s.maxConcurrency) + for _, name := range names { + eg.Go(func() error { + job := project.Jobs[name] + scoped, err := s.useAPISocket(scopedProjectForJob(project, name, job)) + if err != nil { + return err + } + // Without this, a scheduled job declaring only `build:` (no + // `image:`) registers successfully here and then fails, every + // time its schedule fires, because the image was never built — + // silently, since up's own auto-build only ever scoped to + // project.ServiceNames() (see pkg/compose/build.go). + if err := s.ensureImagesExists(ctx, scoped, &api.BuildOptions{Services: []string{name}}, false, false); err != nil { + return err + } + if err := s.ensureModels(ctx, scoped, false); err != nil { + return err + } + svc, err := scoped.GetService(name) + if err != nil { + return err + } + if err := resolveJobServiceReferences(scoped, &svc); err != nil { + return err + } + spec, err := s.buildJobSpec(ctx, scoped, svc, job, false) + if err != nil { + return err + } + if s.dryRun { + return nil // the jobs gRPC dials the real engine, bypassing the dry-run client + } + _, err = jc.Create(ctx, &jobsv0.CreateRequest{ + Name: engineJobName(project, name), + Spec: spec, + }) + return mapAlreadyExists(err, name, "up") + }) + } + return eg.Wait() +} + +// ensureJobsDown removes the project's jobs from the engine. Run history is +// kept by default (RunsRemoval left empty); each removal tolerates the job +// already being gone, matching removeResource's NotFound handling. +func (s *composeService) ensureJobsDown(ctx context.Context, project *types.Project) []downOp { + names := sortedJobNames(project.Jobs, nil) + if len(names) == 0 { + return nil + } + st, err := s.jobsAvailability(ctx, project.Name) + if errors.Is(err, errJobsExtensionNotReady) { + logrus.Warnf("jobs of project %q not removed: %v", project.Name, err) + } + if err != nil || st != jobsAvailable { + return nil + } + + var ops []downOp + for _, name := range names { + ops = append(ops, func() error { + return s.removeResource("Job "+name, func() error { + if s.dryRun { + return nil // see registerScheduledJobs + } + jc, err := s.jobsClient() + if err != nil { + return err + } + err = jc.Remove(ctx, &jobsv0.RemoveRequest{JobRef: engineJobName(project, name)}) + return jobsv0.MapError(err) + }) + }) + } + return ops +} + +// actualJobs reconstructs a project's jobs from the engine's own registry, +// keyed by their local (un-prefixed) name — used when down has no compose +// file to read Jobs from directly (e.g. `compose --project-name X down`). +// Best-effort: any error, including an engine with no jobs feature, is +// treated as "no jobs" so it never blocks an otherwise-successful down; an +// extension enabled but not ready is the one case worth a warning. +func (s *composeService) actualJobs(ctx context.Context, projectName string) types.Jobs { + st, err := s.jobsAvailability(ctx, projectName) + if errors.Is(err, errJobsExtensionNotReady) { + logrus.Warnf("jobs of project %q not removed: %v", projectName, err) + } + if err != nil || st != jobsAvailable { + return nil + } + jc, err := s.jobsClient() + if err != nil { + return nil + } + reply, err := jc.List(ctx, &jobsv0.ListRequest{ + Labels: []string{naming.LabelFilter(api.ProjectLabel, projectName)}, + }) + if err != nil { + // Best-effort: down must not fail or get noisy just because this + // project has no jobs (or the engine has no jobs feature at all — + // which surfaces as anything from a clean Unimplemented to a raw + // transport error, depending on the daemon). Still traceable with + // -v for the case where jobs really were left behind. + logrus.Debugf("failed to list jobs for project %q: %v", projectName, err) + return nil + } + + prefix := projectName + engineJobNameSeparator + jobs := types.Jobs{} + for _, j := range reply.Jobs { + name := strings.TrimPrefix(j.Name, prefix) + jobs[name] = types.JobConfig{Name: name} + } + return jobs +} + +// orphanJobs names the jobs the engine has registered for the project that the +// project no longer declares — a job renamed or removed from the file since +// `up` registered it. A job out of the profiles of this run is declared, not +// an orphan, which is what AllJobs is for. +func (s *composeService) orphanJobs(ctx context.Context, project *types.Project) types.Jobs { + declared := project.AllJobs() + orphans := types.Jobs{} + for name, job := range s.actualJobs(ctx, project.Name) { + if _, ok := declared[name]; !ok { + orphans[name] = job + } + } + return orphans +} + +// loadJobsToRemove completes the jobs down removes with what the engine has +// registered for the project. Without a compose file to read them from they are +// reconstructed from the registry, otherwise a project's scheduled jobs would +// outlive `down` forever. With one, --remove-orphans adds the orphan jobs, the +// ones an earlier up registered from a file that declared more; a partial down +// leaves the jobs alone. +func (s *composeService) loadJobsToRemove(ctx context.Context, project *types.Project, projectName string, options api.DownOptions) { + switch { + case options.Project == nil && len(options.Services) == 0: + project.Jobs = s.actualJobs(ctx, projectName) + case options.RemoveOrphans && len(options.Services) == 0: + if project.Jobs == nil { + project.Jobs = types.Jobs{} + } + maps.Copy(project.Jobs, s.orphanJobs(ctx, project)) + } +} + +// handleOrphanJobs tells the user about the orphan jobs of the project, which +// keep firing on their schedule, or removes them with --remove-orphans. The +// registry is read when it can matter only: by a project that declares +// scheduled jobs, or one that was asked to clean up. +func (s *composeService) handleOrphanJobs(ctx context.Context, project *types.Project, removeOrphans, ignoreOrphans bool) error { + if !removeOrphans && len(sortedJobNames(project.Jobs, HasSchedule)) == 0 { + return nil + } + orphans := s.orphanJobs(ctx, project) + if len(orphans) == 0 { + return nil + } + if !removeOrphans { + if !ignoreOrphans { + logrus.Warnf("Found orphan jobs (%s) still registered with the engine for this project, "+ + "and firing on their schedule if they have one. If you removed or renamed them in your compose file, "+ + "you can run this command with the --remove-orphans flag to clean them up.", strings.Join(slices.Sorted(maps.Keys(orphans)), ", ")) + } + return nil + } + // independent removals: one failing does not stop the others + var eg errgroup.Group + for _, op := range s.ensureJobsDown(ctx, &types.Project{Name: project.Name, Jobs: orphans}) { + eg.Go(op) + } + return eg.Wait() +} diff --git a/pkg/compose/jobs_test.go b/pkg/compose/jobs_test.go new file mode 100644 index 000000000..c128dabbe --- /dev/null +++ b/pkg/compose/jobs_test.go @@ -0,0 +1,843 @@ +/* + Copyright 2020 Docker Compose CLI authors + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package compose + +import ( + "bytes" + "context" + "errors" + "os" + "slices" + "strings" + "sync" + "testing" + + "github.com/compose-spec/compose-go/v2/types" + "github.com/moby/moby/client" + "github.com/sirupsen/logrus" + "go.uber.org/mock/gomock" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" + "gotest.tools/v3/assert" + + jobsv0 "github.com/docker/compose/v5/internal/jobsapi" + "github.com/docker/compose/v5/pkg/api" +) + +func TestJobTrigger(t *testing.T) { + yes, no := true, false + + t.Run("no triggers declared is refused", func(t *testing.T) { + _, err := jobTrigger(types.JobConfig{Name: "migrate"}) + assert.Error(t, err, `job "migrate" has no trigger`) + }) + + t.Run("manual:true translates to a Manual trigger", func(t *testing.T) { + trigger, err := jobTrigger(types.JobConfig{Name: "migrate", Triggers: &types.TriggerConfig{Manual: &yes}}) + assert.NilError(t, err) + assert.DeepEqual(t, trigger, &jobsv0.Trigger{Manual: true}) + }) + + t.Run("manual:false alone (no schedule) is refused: it has no trigger left", func(t *testing.T) { + _, err := jobTrigger(types.JobConfig{Name: "migrate", Triggers: &types.TriggerConfig{Manual: &no}}) + assert.Error(t, err, `job "migrate" has no trigger`) + }) + + t.Run("a single schedule translates to a Schedule trigger", func(t *testing.T) { + trigger, err := jobTrigger(types.JobConfig{Name: "backup", Triggers: &types.TriggerConfig{ + Schedule: []types.ScheduleConfig{{Cron: "0 3 * * *", Timezone: "UTC"}}, + }}) + assert.NilError(t, err) + assert.DeepEqual(t, trigger, &jobsv0.Trigger{Schedule: &jobsv0.ScheduleTrigger{Cron: "0 3 * * *", Timezone: "UTC"}}) + }) + + t.Run("more than one schedule is refused", func(t *testing.T) { + _, err := jobTrigger(types.JobConfig{Name: "backup", Triggers: &types.TriggerConfig{ + Schedule: []types.ScheduleConfig{{Cron: "0 3 * * *"}, {Cron: "0 4 * * *"}}, + }}) + assert.Error(t, err, `job "backup" declares 2 schedules, exactly one is supported`) + }) + + // registerScheduledJobs selects a job on HasSchedule alone, so a job also + // declaring manual:true must be an error here, not a Manual-only Trigger + // that drops the cron silently. + t.Run("manual:true together with a schedule is refused, not silently resolved to Manual", func(t *testing.T) { + _, err := jobTrigger(types.JobConfig{Name: "backup", Triggers: &types.TriggerConfig{ + Manual: &yes, + Schedule: []types.ScheduleConfig{{Cron: "0 3 * * *"}}, + }}) + assert.Error(t, err, `job "backup" declares both manual:true and a schedule, exactly one is supported`) + }) + + t.Run("manual:false together with a schedule keeps the schedule", func(t *testing.T) { + trigger, err := jobTrigger(types.JobConfig{Name: "backup", Triggers: &types.TriggerConfig{ + Manual: &no, + Schedule: []types.ScheduleConfig{{Cron: "0 3 * * *"}}, + }}) + assert.NilError(t, err) + assert.DeepEqual(t, trigger, &jobsv0.Trigger{Schedule: &jobsv0.ScheduleTrigger{Cron: "0 3 * * *"}}) + }) +} + +func TestHasSchedule(t *testing.T) { + assert.Assert(t, !HasSchedule(types.JobConfig{})) + assert.Assert(t, !HasSchedule(types.JobConfig{Triggers: &types.TriggerConfig{}})) + assert.Assert(t, HasSchedule(types.JobConfig{Triggers: &types.TriggerConfig{ + Schedule: []types.ScheduleConfig{{Cron: "0 3 * * *"}}, + }})) +} + +func TestManualTriggerDisabled(t *testing.T) { + yes, no := true, false + + assert.Assert(t, !ManualTriggerDisabled(types.JobConfig{}), "no triggers at all is not an opt-out") + assert.Assert(t, !ManualTriggerDisabled(types.JobConfig{Triggers: &types.TriggerConfig{}}), "an unset Manual is not an opt-out") + assert.Assert(t, !ManualTriggerDisabled(types.JobConfig{Triggers: &types.TriggerConfig{Manual: &yes}})) + assert.Assert(t, ManualTriggerDisabled(types.JobConfig{Triggers: &types.TriggerConfig{Manual: &no}})) +} + +func TestSortedJobNames(t *testing.T) { + jobs := types.Jobs{ + "backup": {Triggers: &types.TriggerConfig{Schedule: []types.ScheduleConfig{{Cron: "0 3 * * *"}}}}, + "migrate": {}, + "report": {}, + } + + assert.DeepEqual(t, sortedJobNames(jobs, nil), []string{"backup", "migrate", "report"}) + assert.DeepEqual(t, sortedJobNames(jobs, HasSchedule), []string{"backup"}) + assert.DeepEqual(t, sortedJobNames(types.Jobs{}, HasSchedule), []string(nil)) +} + +func TestJobChangedErr(t *testing.T) { + assert.Error(t, jobChangedErr("backup", "up"), + `job "backup" has changed: run `+"`docker compose down`"+` to remove it, then `+"`up`"+` again`) + assert.Error(t, jobChangedErr("migrate", "run"), + `job "migrate" has changed: run `+"`docker compose down`"+` to remove it, then `+"`run`"+` again`) +} + +func TestScopedProjectForJob(t *testing.T) { + project := &types.Project{ + Name: "myproject", + DisabledServices: types.Services{"off": {Name: "off"}}, + Services: types.Services{ + "db": {Name: "db"}, + }, + Jobs: types.Jobs{ + "backup": { + Name: "backup", + Extensions: types.Extensions{"x-team": "platform"}, + ContainerSpec: types.ContainerSpec{ + Image: "backup-tool", + Environment: types.MappingWithEquals{"BAR": strPtr("original")}, + Volumes: []types.ServiceVolumeConfig{{Type: types.VolumeTypeImage, Source: "original", Target: "/data"}}, + Configs: []types.ServiceConfigObjConfig{{Source: "original"}}, + }, + }, + }, + Configs: types.Configs{ + "cfg": {Content: "original"}, + }, + } + job := project.Jobs["backup"] + + scoped := scopedProjectForJob(project, "backup", job) + + t.Run("the job is materialized into the scoped copy's Services", func(t *testing.T) { + svc, err := scoped.GetService("backup") + assert.NilError(t, err) + assert.Equal(t, svc.Image, "backup-tool") + assert.Equal(t, svc.Extensions["x-team"], "platform") + }) + + // the passes run on the scoped copy (pull, models, credentials) must see the + // job alone: the services are `up`'s, which already went through them + t.Run("the job is the only service, the others are disabled", func(t *testing.T) { + assert.DeepEqual(t, scoped.ServiceNames(), []string{"backup"}) + assert.DeepEqual(t, scoped.DisabledServiceNames(), []string{"db", "off"}) + }) + + t.Run("the real project is never mutated", func(t *testing.T) { + _, ok := project.Services["backup"] + assert.Assert(t, !ok, "the job must not leak into the shared project's Services") + assert.Equal(t, len(project.Services), 1) + assert.Equal(t, len(project.DisabledServices), 1) + }) + + t.Run("Configs is its own map, not shared with the real project", func(t *testing.T) { + scoped.Configs["new"] = types.ConfigObjConfig{Content: "added"} + _, ok := project.Configs["new"] + assert.Assert(t, !ok, "writing to the scoped copy's Configs must not be visible on the shared project — registerScheduledJobs runs this concurrently per job") + assert.Equal(t, project.Configs["cfg"].Content, "original") + }) + + t.Run("the job's own Environment, Volumes and Configs are not shared with project.Jobs", func(t *testing.T) { + svc, err := scoped.GetService("backup") + assert.NilError(t, err) + // useAPISocket, ensureModels and ensureImagesExists mutate these in + // place on the scoped job service: project.Jobs must stay untouched. + svc.Environment["DOCKER_CONFIG"] = strPtr("added") + svc.Volumes[0].Source = "mutated" + svc.Configs[0].Source = "mutated" + original := project.Jobs["backup"] + _, ok := original.Environment["DOCKER_CONFIG"] + assert.Assert(t, !ok, "writing to the scoped job's Environment must not be visible on project.Jobs") + assert.Equal(t, original.Volumes[0].Source, "original") + assert.Equal(t, original.Configs[0].Source, "original") + }) +} + +// fakeJobsClient is a minimal jobsv0.Jobs double: only the methods a test +// exercises are implemented, every other one is left to the embedded nil +// interface — calling one would panic, which is exactly the loud failure a +// test relying on it deserves. +type fakeJobsClient struct { + jobsv0.Jobs + + mu sync.Mutex // registerScheduledJobs calls Create concurrently + createCalls []*jobsv0.CreateRequest + createErr error + listCalls []*jobsv0.ListRequest + listReply *jobsv0.ListReply + listErr error + removeCalls []*jobsv0.RemoveRequest +} + +func (f *fakeJobsClient) Create(_ context.Context, req *jobsv0.CreateRequest) (*jobsv0.CreateReply, error) { + f.mu.Lock() + defer f.mu.Unlock() + f.createCalls = append(f.createCalls, req) + if f.createErr != nil { + return nil, f.createErr + } + return &jobsv0.CreateReply{Job: &jobsv0.Job{Name: req.Name}}, nil +} + +func (f *fakeJobsClient) Remove(_ context.Context, req *jobsv0.RemoveRequest) error { + f.mu.Lock() + defer f.mu.Unlock() + f.removeCalls = append(f.removeCalls, req) + return nil +} + +func (f *fakeJobsClient) List(_ context.Context, req *jobsv0.ListRequest) (*jobsv0.ListReply, error) { + f.listCalls = append(f.listCalls, req) + if f.listErr != nil { + return nil, f.listErr + } + if f.listReply == nil { + return &jobsv0.ListReply{}, nil + } + return f.listReply, nil +} + +// TestBuildJobSpecIsDeterministic guards against the spec hash churning +// across invocations when a job declares several environment variables: +// buildJobSpec marshals the container spec as JSON and sends it to the +// engine, which identifies the job by a hash of that payload, so any +// map-to-slice conversion that doesn't sort its output (Env, Ulimits, +// ExtraHosts, mounts) makes every other `up`/`run` fail with +// `job "..." has changed`. +func TestBuildJobSpecIsDeterministic(t *testing.T) { + s, _ := newPreStartTestService(t) + project := &types.Project{Name: "myproject"} + svc := types.ServiceConfig{ + Name: "backup", + WorkloadSpec: types.WorkloadSpec{ + DependsOn: types.DependsOnConfig{ + "a": {Condition: types.ServiceConditionStarted, Required: true}, + "b": {Condition: types.ServiceConditionHealthy, Required: true}, + "c": {Condition: types.ServiceConditionStarted, Required: true}, + }, + }, + ContainerSpec: types.ContainerSpec{ + Image: "alpine", + Environment: types.MappingWithEquals{ + "A": strPtr("1"), "B": strPtr("2"), "C": strPtr("3"), "D": strPtr("4"), + "E": strPtr("5"), "F": strPtr("6"), "G": strPtr("7"), "H": strPtr("8"), + }, + Ulimits: map[string]*types.UlimitsConfig{ + "nofile": {Soft: 1024, Hard: 2048}, + "nproc": {Single: 100}, + }, + ExtraHosts: types.HostsList{ + "a.example.com": []string{"10.0.0.1"}, + "b.example.com": []string{"10.0.0.2"}, + }, + Volumes: []types.ServiceVolumeConfig{ + {Type: types.VolumeTypeVolume, Target: "/var/a"}, + {Type: types.VolumeTypeVolume, Target: "/var/b"}, + }, + }, + } + manual := true + job := types.JobConfig{Name: "backup", Triggers: &types.TriggerConfig{Manual: &manual}} + + first, err := s.buildJobSpec(t.Context(), project, svc, job, false) + assert.NilError(t, err) + + msg := "job spec must be byte-identical across invocations, or the engine's spec hash churns and up/run intermittently fail with \"has changed\"" + for i := 0; i < 100; i++ { + spec, err := s.buildJobSpec(t.Context(), project, svc, job, false) + assert.NilError(t, err) + assert.Equal(t, string(spec.ContainerSpec), string(first.ContainerSpec), msg) + } +} + +// The Compose version and the digest of the job's image stay out of its spec. +func TestBuildJobSpecIgnoresVolatileLabels(t *testing.T) { + s, _ := newPreStartTestService(t) + project := &types.Project{Name: "myproject", WorkingDir: "/work"} + manual := true + job := types.JobConfig{Name: "backup", Triggers: &types.TriggerConfig{Manual: &manual}} + svc := JobAsService(project, "backup", job) + svc.Image = "alpine" + svc.CustomLabels[api.ImageDigestLabel] = "sha256:aaa" + + spec, err := s.buildJobSpec(t.Context(), project, svc, job, false) + assert.NilError(t, err) + + labels := string(spec.ContainerSpec) + assert.Assert(t, strings.Contains(labels, api.ProjectLabel), "the labels that identify the run container must stay") + assert.Assert(t, !strings.Contains(labels, api.VersionLabel)) + assert.Assert(t, !strings.Contains(labels, api.ImageDigestLabel)) +} + +// up registers every scheduled job, and only those, under the project-scoped +// engine name; a dry run must not reach the real engine. +func TestRegisterScheduledJobs(t *testing.T) { + newService := func(t *testing.T, fake *fakeJobsClient) *composeService { + t.Helper() + s, mockAPI := newPreStartTestServiceWithVersion(t, apiVersionJobs) + mockAPI.EXPECT().ImageInspect(gomock.Any(), gomock.Any(), gomock.Any()).Return(client.ImageInspectResult{}, nil).AnyTimes() + s.jobsGRPCOnce.Do(func() {}) + s.jobsAPI = fake + return s + } + project := func() *types.Project { + p := scheduledJobsProject() + web := p.Services["web"] + web.Image = "alpine" + p.Services["web"] = web + for name, job := range p.Jobs { + job.Image = "alpine" + p.Jobs[name] = job + } + return p + } + + t.Run("registers the scheduled jobs, not the manual ones", func(t *testing.T) { + fake := &fakeJobsClient{} + assert.NilError(t, newService(t, fake).registerScheduledJobs(t.Context(), project())) + var names []string + for _, call := range fake.createCalls { + names = append(names, call.Name) + assert.Assert(t, call.Spec.Trigger.Schedule != nil, "%s must carry its schedule", call.Name) + assert.Assert(t, call.Spec.RemoveOnSuccess, "%s: a scheduled job's runs are disposable", call.Name) + } + slices.Sort(names) + assert.DeepEqual(t, names, []string{"prj.backup", "prj.sync"}) + }) + + t.Run("a service the job shares with is registered by name, not by service", func(t *testing.T) { + fake := &fakeJobsClient{} + p := project() + backup := p.Jobs["backup"] + backup.VolumesFrom = []string{"web"} + p.Jobs["backup"] = backup + + assert.NilError(t, newService(t, fake).registerScheduledJobs(t.Context(), p)) + + var registered string + for _, call := range fake.createCalls { + if call.Name == "prj.backup" { + registered = string(call.Spec.ContainerSpec) + } + } + assert.Assert(t, strings.Contains(registered, `"VolumesFrom":["prj-web-1"]`), registered) + }) + + // up already pulled the services: registering the jobs must not do it again + t.Run("the images of the services are left alone", func(t *testing.T) { + p := project() + web := p.Services["web"] + web.PullPolicy = types.PullPolicyAlways + p.Services["web"] = web + + // no ImagePull expected on the mock: one would fail the test + assert.NilError(t, newService(t, &fakeJobsClient{}).registerScheduledJobs(t.Context(), p)) + }) + + t.Run("a changed job is reported with the way out", func(t *testing.T) { + fake := &fakeJobsClient{createErr: status.Error(codes.AlreadyExists, "spec differs")} + err := newService(t, fake).registerScheduledJobs(t.Context(), project()) + assert.ErrorContains(t, err, "has changed: run `docker compose down`") + }) + + t.Run("a dry run registers nothing on the real engine", func(t *testing.T) { + fake := &fakeJobsClient{} + s := newService(t, fake) + s.dryRun = true + assert.NilError(t, s.registerScheduledJobs(t.Context(), project())) + assert.Equal(t, len(fake.createCalls), 0) + }) +} + +// create arms the scheduled jobs like up does: the engine's cron is the only +// scheduler, and a job that failed because the services it needs were not +// started leaves its logs for the user. +func TestCreateRegistersScheduledJobs(t *testing.T) { + s, mockAPI := newPreStartTestServiceWithVersion(t, apiVersionJobs) + mockAPI.EXPECT().ImageInspect(gomock.Any(), gomock.Any(), gomock.Any()).Return(client.ImageInspectResult{}, nil).AnyTimes() + mockAPI.EXPECT().ContainerList(gomock.Any(), gomock.Any()).Return(client.ContainerListResult{}, nil).AnyTimes() + mockAPI.EXPECT().NetworkList(gomock.Any(), gomock.Any()).Return(client.NetworkListResult{}, nil).AnyTimes() + mockAPI.EXPECT().VolumeList(gomock.Any(), gomock.Any()).Return(client.VolumeListResult{}, nil).AnyTimes() + fake := &fakeJobsClient{} + s.jobsGRPCOnce.Do(func() {}) + s.jobsAPI = fake + + project := scheduledJobsProject() + project.Services = types.Services{} + for name, job := range project.Jobs { + job.Image = "alpine" + project.Jobs[name] = job + } + + assert.NilError(t, s.Create(t.Context(), project, api.CreateOptions{})) + + var names []string + for _, call := range fake.createCalls { + names = append(names, call.Name) + } + slices.Sort(names) + assert.DeepEqual(t, names, []string{"prj.backup", "prj.sync"}) +} + +// create reports the jobs the file no longer declares, and removes them with +// --remove-orphans. +func TestCreateHandlesOrphanJobs(t *testing.T) { + run := func(t *testing.T, options api.CreateOptions) (*fakeJobsClient, string) { + t.Helper() + out := captureWarnings(t) + s, mockAPI := newPreStartTestServiceWithVersion(t, apiVersionJobs) + mockAPI.EXPECT().ImageInspect(gomock.Any(), gomock.Any(), gomock.Any()).Return(client.ImageInspectResult{}, nil).AnyTimes() + mockAPI.EXPECT().ContainerList(gomock.Any(), gomock.Any()).Return(client.ContainerListResult{}, nil).AnyTimes() + mockAPI.EXPECT().NetworkList(gomock.Any(), gomock.Any()).Return(client.NetworkListResult{}, nil).AnyTimes() + mockAPI.EXPECT().VolumeList(gomock.Any(), gomock.Any()).Return(client.VolumeListResult{}, nil).AnyTimes() + fake := &fakeJobsClient{listReply: &jobsv0.ListReply{Jobs: []jobsv0.Job{{Name: "prj.backup"}, {Name: "prj.sync"}, {Name: "prj.old"}}}} + s.jobsGRPCOnce.Do(func() {}) + s.jobsAPI = fake + + project := scheduledJobsProject() + project.Services = types.Services{} + for name, job := range project.Jobs { + job.Image = "alpine" + project.Jobs[name] = job + } + assert.NilError(t, s.Create(t.Context(), project, options)) + return fake, out.String() + } + + t.Run("without the flag it says so", func(t *testing.T) { + fake, out := run(t, api.CreateOptions{}) + assert.Assert(t, strings.Contains(out, "Found orphan jobs (old)"), out) + assert.Equal(t, len(fake.removeCalls), 0) + }) + + t.Run("with --remove-orphans it removes them", func(t *testing.T) { + fake, out := run(t, api.CreateOptions{RemoveOrphans: true}) + assert.Assert(t, !strings.Contains(out, "orphan jobs"), out) + assert.Equal(t, len(fake.removeCalls), 1) + assert.Equal(t, fake.removeCalls[0].JobRef, "prj.old") + }) +} + +// scheduledJobsProject declares one scheduled and one manual job next to a +// service, the shape the degraded-mode tests below run against. +func scheduledJobsProject() *types.Project { + yes := true + return &types.Project{ + Name: "prj", + Services: types.Services{"web": {Name: "web"}}, + Jobs: types.Jobs{ + "backup": {Name: "backup", Triggers: &types.TriggerConfig{Schedule: []types.ScheduleConfig{{Cron: "0 3 * * *"}}}}, + "sync": {Name: "sync", Triggers: &types.TriggerConfig{Schedule: []types.ScheduleConfig{{Cron: "0 4 * * *"}}}}, + "migrate": {Name: "migrate", Triggers: &types.TriggerConfig{Manual: &yes}}, + }, + } +} + +// newJobsTestService returns a compose service whose engine negotiates +// apiVersion and whose jobs extension is the given fake. The mocks carry no +// other expectation: any daemon call beyond the version negotiation fails the +// test. +func newJobsTestService(t *testing.T, apiVersion string, fake *fakeJobsClient) *composeService { + t.Helper() + _, tested := newTestComposeService(t, gomock.NewController(t), apiVersion) + tested.jobsGRPCOnce.Do(func() {}) // keep jobsClient from dialing: the fake stands in for the extension + tested.jobsAPI = fake + return tested +} + +var ( + // jobsExtensionOffErr is what an engine whose jobs extension is not + // enabled answers to any call of the jobs service. + jobsExtensionOffErr = status.Error(codes.Unimplemented, "unknown service org.mobyproject.extension.jobs.api.v0.Jobs") + // jobsExtensionNotReadyErr is what an engine whose jobs extension is + // enabled but still reconciling (or failed to activate) answers. + jobsExtensionNotReadyErr = status.Error(codes.Unavailable, "the jobs extension is not active") +) + +const ( + // noJobsEngine is the newest API version below apiVersionJobs. + noJobsEngine = "1.56" + // scheduledJobsRefused is what a project declaring the scheduled jobs of + // scheduledJobsProject gets on an engine whose API predates jobs. + scheduledJobsRefused = "scheduled jobs are not supported by this version of the engine: backup, sync" + // scheduledJobsExtensionOffRefused is what it gets on an engine that has + // the jobs API but not the extension enabled: it names the flag to start. + scheduledJobsExtensionOffRefused = "scheduled jobs need the engine's jobs extension, which is not enabled " + + "(start dockerd with --extension-enable org.mobyproject.jobs.v0): backup, sync" + // jobsExtensionNotReady prefixes the distinct error of an enabled but not + // ready extension. + jobsExtensionNotReady = "the engine's jobs extension is enabled but not ready: the jobs extension is not active" +) + +func TestJobsAvailability(t *testing.T) { + t.Run("an engine whose API predates jobs is too old, without any gRPC call", func(t *testing.T) { + for _, version := range []string{"1.44", noJobsEngine} { + fake := &fakeJobsClient{} + got, err := newJobsTestService(t, version, fake).jobsAvailability(t.Context(), "prj") + assert.NilError(t, err) + assert.Equal(t, got, jobsEngineTooOld, version) + assert.Equal(t, len(fake.listCalls), 0, "the API version alone must decide, no probe") + } + }) + + t.Run("an engine answering Unimplemented has the extension off", func(t *testing.T) { + fake := &fakeJobsClient{listErr: jobsExtensionOffErr} + got, err := newJobsTestService(t, apiVersionJobs, fake).jobsAvailability(t.Context(), "prj") + assert.NilError(t, err) + assert.Equal(t, got, jobsExtensionOff) + }) + + t.Run("an engine answering the probe, even with no job, has the extension, probed with a List scoped to the project", func(t *testing.T) { + fake := &fakeJobsClient{} + got, err := newJobsTestService(t, apiVersionJobs, fake).jobsAvailability(t.Context(), "prj") + assert.NilError(t, err) + assert.Equal(t, got, jobsAvailable) + assert.Equal(t, len(fake.listCalls), 1) + assert.DeepEqual(t, fake.listCalls[0].Labels, []string{api.ProjectLabel + "=prj"}) + }) + + t.Run("Unavailable is a distinct error, never an availability", func(t *testing.T) { + fake := &fakeJobsClient{listErr: jobsExtensionNotReadyErr} + _, err := newJobsTestService(t, apiVersionJobs, fake).jobsAvailability(t.Context(), "prj") + assert.ErrorIs(t, err, errJobsExtensionNotReady) + assert.ErrorContains(t, err, jobsExtensionNotReady) + }) + + t.Run("any other error is returned as is", func(t *testing.T) { + boom := errors.New("connection refused") + fake := &fakeJobsClient{listErr: boom} + _, err := newJobsTestService(t, apiVersionJobs, fake).jobsAvailability(t.Context(), "prj") + assert.ErrorIs(t, err, boom) + assert.Assert(t, !errors.Is(err, errJobsExtensionNotReady)) + }) + + t.Run("a definitive answer is probed once per process", func(t *testing.T) { + for _, listErr := range []error{nil, jobsExtensionOffErr} { + fake := &fakeJobsClient{listErr: listErr} + tested := newJobsTestService(t, apiVersionJobs, fake) + first, err := tested.jobsAvailability(t.Context(), "prj") + assert.NilError(t, err) + second, err := tested.jobsAvailability(t.Context(), "prj") + assert.NilError(t, err) + assert.Equal(t, first, second) + assert.Equal(t, len(fake.listCalls), 1) + } + }) + + t.Run("an error is never memoized: the extension may be ready on the next call", func(t *testing.T) { + fake := &fakeJobsClient{listErr: jobsExtensionNotReadyErr} + tested := newJobsTestService(t, apiVersionJobs, fake) + _, err := tested.jobsAvailability(t.Context(), "prj") + assert.ErrorIs(t, err, errJobsExtensionNotReady) + + fake.listErr = nil + got, err := tested.jobsAvailability(t.Context(), "prj") + assert.NilError(t, err) + assert.Equal(t, got, jobsAvailable) + assert.Equal(t, len(fake.listCalls), 2) + }) +} + +func TestRejectUnsupportedScheduledJobs(t *testing.T) { + t.Run("an engine with the extension accepts them", func(t *testing.T) { + err := newJobsTestService(t, apiVersionJobs, &fakeJobsClient{}).rejectUnsupportedScheduledJobs(t.Context(), scheduledJobsProject()) + assert.NilError(t, err) + }) + + t.Run("manual jobs never block nor probe, whatever the engine", func(t *testing.T) { + project := scheduledJobsProject() + delete(project.Jobs, "backup") + delete(project.Jobs, "sync") + fake := &fakeJobsClient{listErr: jobsExtensionOffErr} + assert.NilError(t, newJobsTestService(t, apiVersionJobs, fake).rejectUnsupportedScheduledJobs(t.Context(), project)) + assert.Equal(t, len(fake.listCalls), 0) + }) + + t.Run("profile-disabled scheduled jobs don't block", func(t *testing.T) { + project := &types.Project{DisabledJobs: scheduledJobsProject().Jobs} + assert.NilError(t, newJobsTestService(t, noJobsEngine, &fakeJobsClient{}).rejectUnsupportedScheduledJobs(t.Context(), project)) + }) + + t.Run("a project without jobs never negotiates the engine version", func(t *testing.T) { + mockCtrl := gomock.NewController(t) + _, cli := prepareMocks(mockCtrl) // no Ping expectation: negotiating would fail + tested, err := NewComposeService(cli) + assert.NilError(t, err) + assert.NilError(t, tested.(*composeService).rejectUnsupportedScheduledJobs(t.Context(), &types.Project{Name: "prj"})) + }) +} + +// Without usable jobs support, create, up and start refuse a project +// declaring scheduled jobs before touching the daemon: the mocks carry no +// expectation besides the version negotiation. +func TestScheduledJobsRefusedWithoutJobsEngine(t *testing.T) { + for name, tc := range map[string]struct { + version string + listErr error + refused string + }{ + "API predates jobs": {version: noJobsEngine, refused: scheduledJobsRefused}, + "extension off": {version: apiVersionJobs, listErr: jobsExtensionOffErr, refused: scheduledJobsExtensionOffRefused}, + "extension not ready": {version: apiVersionJobs, listErr: jobsExtensionNotReadyErr, refused: jobsExtensionNotReady}, + } { + t.Run(name, func(t *testing.T) { + tested := newJobsTestService(t, tc.version, &fakeJobsClient{listErr: tc.listErr}) + project := scheduledJobsProject() + + assert.ErrorContains(t, tested.Create(t.Context(), project, api.CreateOptions{}), tc.refused) + assert.ErrorContains(t, tested.Up(t.Context(), project, api.UpOptions{}), tc.refused) + assert.ErrorContains(t, tested.Start(t.Context(), project.Name, api.StartOptions{Project: project}), tc.refused) + }) + } +} + +// captureWarnings captures what the code under test tells the user. +func captureWarnings(t *testing.T) *bytes.Buffer { + t.Helper() + var buf bytes.Buffer + logrus.SetOutput(&buf) + t.Cleanup(func() { logrus.SetOutput(os.Stderr) }) + return &buf +} + +// Down only talks to the engine's jobs registry when the engine has one. +func TestDownJobsWithoutJobsEngine(t *testing.T) { + project := scheduledJobsProject() + + t.Run("an API that predates jobs leaves nothing to remove, silently", func(t *testing.T) { + out := captureWarnings(t) + tested := newJobsTestService(t, noJobsEngine, &fakeJobsClient{}) + assert.Equal(t, len(tested.ensureJobsDown(t.Context(), project)), 0) + assert.Equal(t, out.String(), "") + }) + + t.Run("an extension that is off leaves nothing to remove, silently", func(t *testing.T) { + out := captureWarnings(t) + tested := newJobsTestService(t, apiVersionJobs, &fakeJobsClient{listErr: jobsExtensionOffErr}) + assert.Equal(t, len(tested.ensureJobsDown(t.Context(), project)), 0) + assert.Equal(t, out.String(), "") + }) + + t.Run("an extension enabled but not ready leaves nothing to remove, with a warning", func(t *testing.T) { + out := captureWarnings(t) + tested := newJobsTestService(t, apiVersionJobs, &fakeJobsClient{listErr: jobsExtensionNotReadyErr}) + assert.Equal(t, len(tested.ensureJobsDown(t.Context(), project)), 0, "down must carry on cleaning the rest of the project") + assert.Assert(t, strings.Contains(out.String(), "level=warning"), out.String()) + assert.Assert(t, strings.Contains(out.String(), jobsExtensionNotReady), out.String()) + }) + + t.Run("with the extension every declared job is removed from the engine", func(t *testing.T) { + fake := &fakeJobsClient{} + tested := newJobsTestService(t, apiVersionJobs, fake) + ops := tested.ensureJobsDown(t.Context(), project) + assert.Equal(t, len(ops), len(project.Jobs)) + for _, op := range ops { + assert.NilError(t, op()) + } + var refs []string + for _, call := range fake.removeCalls { + refs = append(refs, call.JobRef) + } + slices.Sort(refs) + assert.DeepEqual(t, refs, []string{"prj.backup", "prj.migrate", "prj.sync"}) + }) + + t.Run("a dry run removes nothing from the real engine", func(t *testing.T) { + fake := &fakeJobsClient{} + tested := newJobsTestService(t, apiVersionJobs, fake) + tested.dryRun = true + for _, op := range tested.ensureJobsDown(t.Context(), project) { + assert.NilError(t, op()) + } + assert.Equal(t, len(fake.removeCalls), 0) + }) +} + +// down without a compose file rebuilds the project's jobs from the engine's +// registry; it tells about jobs it cannot reach only when the extension is +// enabled but not ready, exactly like down with a compose file. +func TestActualJobs(t *testing.T) { + t.Run("an API that predates jobs lists nothing, silently, without probing", func(t *testing.T) { + out := captureWarnings(t) + fake := &fakeJobsClient{} + assert.Equal(t, len(newJobsTestService(t, noJobsEngine, fake).actualJobs(t.Context(), "prj")), 0) + assert.Equal(t, len(fake.listCalls), 0) + assert.Equal(t, out.String(), "") + }) + + t.Run("an extension that is off lists nothing, silently", func(t *testing.T) { + out := captureWarnings(t) + tested := newJobsTestService(t, apiVersionJobs, &fakeJobsClient{listErr: jobsExtensionOffErr}) + assert.Equal(t, len(tested.actualJobs(t.Context(), "prj")), 0) + assert.Equal(t, out.String(), "") + }) + + t.Run("an extension enabled but not ready lists nothing, with a warning", func(t *testing.T) { + out := captureWarnings(t) + tested := newJobsTestService(t, apiVersionJobs, &fakeJobsClient{listErr: jobsExtensionNotReadyErr}) + assert.Equal(t, len(tested.actualJobs(t.Context(), "prj")), 0) + assert.Assert(t, strings.Contains(out.String(), "level=warning"), out.String()) + assert.Assert(t, strings.Contains(out.String(), jobsExtensionNotReady), out.String()) + }) + + t.Run("with the extension the project's jobs come back under their local names", func(t *testing.T) { + fake := &fakeJobsClient{listReply: &jobsv0.ListReply{Jobs: []jobsv0.Job{{Name: "prj.backup"}, {Name: "prj.sync"}}}} + got := newJobsTestService(t, apiVersionJobs, fake).actualJobs(t.Context(), "prj") + assert.DeepEqual(t, sortedJobNames(got, nil), []string{"backup", "sync"}) + }) +} + +func strPtr(s string) *string { return &s } + +// A job names the container of the service it shares with, not its ID. +func TestResolveJobServiceReferences(t *testing.T) { + project := &types.Project{Name: "prj", Services: types.Services{ + "web": {Name: "web"}, + "db": {Name: "db", ContainerName: "the-db"}, + }} + + t.Run("services become the name of their first container", func(t *testing.T) { + volumesFrom := []string{"web", "container:other", "db"} + job := types.ServiceConfig{Name: "backup", ContainerSpec: types.ContainerSpec{ + VolumesFrom: volumesFrom, + NetworkMode: "service:web", + Ipc: "service:db", + Pid: "service:web", + }} + + assert.NilError(t, resolveJobServiceReferences(project, &job)) + + assert.DeepEqual(t, job.VolumesFrom, []string{"prj-web-1", "other", "the-db"}) + assert.Equal(t, job.NetworkMode, "container:prj-web-1") + assert.Equal(t, job.Ipc, "container:the-db") + assert.Equal(t, job.Pid, "container:prj-web-1") + assert.DeepEqual(t, volumesFrom, []string{"web", "container:other", "db"}) // the job of the project keeps its own + }) + + // run --no-deps and up narrow the project: the service the job + // shares with is then disabled, not gone + t.Run("a service the selection left out resolves like the others", func(t *testing.T) { + narrowed := &types.Project{Name: "prj", Services: types.Services{}, DisabledServices: types.Services{"db": {Name: "db"}}} + job := types.ServiceConfig{Name: "backup", ContainerSpec: types.ContainerSpec{VolumesFrom: []string{"db"}}} + + assert.NilError(t, resolveJobServiceReferences(narrowed, &job)) + assert.DeepEqual(t, job.VolumesFrom, []string{"prj-db-1"}) + }) + + t.Run("a service the project does not have is an error", func(t *testing.T) { + job := types.ServiceConfig{Name: "backup", ContainerSpec: types.ContainerSpec{VolumesFrom: []string{"nope"}}} + assert.ErrorContains(t, resolveJobServiceReferences(project, &job), "nope") + }) +} + +// A job the engine still has for the project, which the compose file no longer +// declares, keeps firing: up says so, and --remove-orphans removes it, as it does +// the containers of a service that left the file. +func TestHandleOrphanJobs(t *testing.T) { + registered := &jobsv0.ListReply{Jobs: []jobsv0.Job{ + {Name: "prj.backup"}, {Name: "prj.sync"}, {Name: "prj.migrate"}, {Name: "prj.off"}, {Name: "prj.old"}, + }} + project := func() *types.Project { + p := scheduledJobsProject() + p.DisabledJobs = types.Jobs{"off": {Name: "off"}} // declared, out of the profiles of this run + return p + } + + t.Run("up names the jobs the file no longer declares", func(t *testing.T) { + out := captureWarnings(t) + fake := &fakeJobsClient{listReply: registered} + assert.NilError(t, newJobsTestService(t, apiVersionJobs, fake).handleOrphanJobs(t.Context(), project(), false, false)) + assert.Assert(t, strings.Contains(out.String(), "Found orphan jobs (old)"), out.String()) + assert.Assert(t, strings.Contains(out.String(), "--remove-orphans"), out.String()) + assert.Equal(t, len(fake.removeCalls), 0) + }) + + t.Run("IgnoreOrphans keeps it quiet", func(t *testing.T) { + out := captureWarnings(t) + fake := &fakeJobsClient{listReply: registered} + assert.NilError(t, newJobsTestService(t, apiVersionJobs, fake).handleOrphanJobs(t.Context(), project(), false, true)) + assert.Equal(t, out.String(), "") + }) + + t.Run("--remove-orphans removes them, and only them", func(t *testing.T) { + out := captureWarnings(t) + fake := &fakeJobsClient{listReply: registered} + assert.NilError(t, newJobsTestService(t, apiVersionJobs, fake).handleOrphanJobs(t.Context(), project(), true, false)) + assert.Equal(t, len(fake.removeCalls), 1) + assert.Equal(t, fake.removeCalls[0].JobRef, "prj.old") + assert.Equal(t, out.String(), "") + }) + + t.Run("a dry run removes nothing", func(t *testing.T) { + fake := &fakeJobsClient{listReply: registered} + tested := newJobsTestService(t, apiVersionJobs, fake) + tested.dryRun = true + assert.NilError(t, tested.handleOrphanJobs(t.Context(), project(), true, false)) + assert.Equal(t, len(fake.removeCalls), 0) + }) + + // the registry is looked at when it can matter: a project that declares + // scheduled jobs, or one that was asked to clean up + t.Run("a project with no scheduled job does not look at the registry", func(t *testing.T) { + out := captureWarnings(t) + fake := &fakeJobsClient{listReply: registered} + p := &types.Project{Name: "prj", Services: types.Services{"web": {Name: "web"}}} + assert.NilError(t, newJobsTestService(t, apiVersionJobs, fake).handleOrphanJobs(t.Context(), p, false, false)) + assert.Equal(t, len(fake.listCalls), 0) + assert.Equal(t, out.String(), "") + }) + + t.Run("--remove-orphans looks even when the file declares no scheduled job", func(t *testing.T) { + fake := &fakeJobsClient{listReply: registered} + p := &types.Project{Name: "prj", Services: types.Services{"web": {Name: "web"}}} + assert.NilError(t, newJobsTestService(t, apiVersionJobs, fake).handleOrphanJobs(t.Context(), p, true, false)) + assert.Equal(t, len(fake.removeCalls), 5) + }) +} diff --git a/pkg/compose/observed_state.go b/pkg/compose/observed_state.go index 0bbfe9720..206b7ad4b 100644 --- a/pkg/compose/observed_state.go +++ b/pkg/compose/observed_state.go @@ -21,7 +21,6 @@ import ( "slices" "sort" "strconv" - "strings" "github.com/compose-spec/compose-go/v2/types" "github.com/containerd/errdefs" @@ -34,6 +33,10 @@ import ( "github.com/docker/compose/v5/pkg/api" ) +// jobContainerIDLabel is the engine's reserved label identifying a job run +// container (see the jobsv0 contract); Compose never sets it itself. +const jobContainerIDLabel = "com.docker.job.id" + // ObservedState captures the current state of all Docker resources belonging to // a Compose project. It is a snapshot taken before reconciliation so that the // reconciler can compare desired state (types.Project) with reality without @@ -258,13 +261,13 @@ func (s *composeService) collectObservedState(ctx context.Context, project *type } if isNotOneOff(ctr) && knownServices[svcName] { state.Containers[svcName] = append(state.Containers[svcName], toObservedContainer(ctr)) - } else if isOrphaned(project)(ctr) { + } else if isOrphaned(project)(ctr) && !isLiveJobRun(ctr) { state.Orphans = append(state.Orphans, toObservedContainer(ctr)) } - // else: a still-RUNNING one-off. Deliberately absent from the - // observed state: it is somebody's live `compose run` session — up - // neither reconciles it, nor warns about it, nor removes it (only - // `down` stops running one-offs). + // else: a still-RUNNING one-off, or a run of a job still going. + // Deliberately absent from the observed state: it is somebody's live + // `compose run` session, or a job firing — up neither reconciles it, + // nor warns about it, nor removes it (only `down` stops them). } // Networks and volumes are a create-phase concern: ScopeStart's @@ -496,15 +499,6 @@ func emitRunningEvents(project *types.Project, observed *ObservedState, plan *pl } } -// orphanNames returns the names of orphaned containers as a comma-separated string. -func (s *ObservedState) orphanNames() string { - names := make([]string, len(s.Orphans)) - for i, o := range s.Orphans { - names[i] = o.Name - } - return strings.Join(names, ", ") -} - // containersByService flattens the observed containers into the shape // resolveServiceReferences expects: project service name → raw Summaries. func (s *ObservedState) containersByService() map[string]Containers { diff --git a/pkg/compose/observed_state_test.go b/pkg/compose/observed_state_test.go index c9bf7e1d4..08cde01cf 100644 --- a/pkg/compose/observed_state_test.go +++ b/pkg/compose/observed_state_test.go @@ -815,3 +815,43 @@ func TestCollectObservedState_SelectionDoesNotMakeDisabledServicesOrphans(t *tes assert.Equal(t, len(state.Orphans), 1) assert.Equal(t, state.Orphans[0].ID, "c3", "only the service absent from the model is an orphan") } + +// A run of a job is not a service replica: the project does not define its +// "service", so it reads as an orphan. up --remove-orphans sweeps the finished +// ones, the failures kept for postmortem, but must leave a run still going +// alone, like a live `compose run` session: a cron firing while the user +// deploys would otherwise be killed. +func TestCollectObservedState_RunningJobRunIsNotAnOrphan(t *testing.T) { + svc, apiClient := newTestService(t) + project := &types.Project{Name: "myproject", Services: types.Services{"web": {Name: "web"}}} + + jobRun := func(id string, state container.ContainerState) container.Summary { + return container.Summary{ + ID: id, + Names: []string{"/job-myproject.backup-1-" + id}, + State: state, + Labels: map[string]string{ + api.ServiceLabel: "backup", + api.ProjectLabel: "myproject", + api.ConfigHashLabel: "hash-backup", + api.ContainerNumberLabel: "1", + api.OneoffLabel: "False", + jobContainerIDLabel: "job-1", + }, + } + } + apiClient.EXPECT().ContainerList(gomock.Any(), gomock.Any()).Return(client.ContainerListResult{ + Items: []container.Summary{ + jobRun("running", container.StateRunning), + jobRun("failed", container.StateExited), + }, + }, nil) + apiClient.EXPECT().NetworkList(gomock.Any(), gomock.Any()).Return(client.NetworkListResult{}, nil) + apiClient.EXPECT().VolumeList(gomock.Any(), gomock.Any()).Return(client.VolumeListResult{}, nil) + + state, err := svc.collectObservedState(t.Context(), project, ScopeCreateStart) + assert.NilError(t, err) + + assert.Equal(t, len(state.Orphans), 1) + assert.Equal(t, state.Orphans[0].ID, "failed", "the finished run is swept, the running one is left alone") +} diff --git a/pkg/compose/service_containers.go b/pkg/compose/service_containers.go index eab66349e..a4e2b6c77 100644 --- a/pkg/compose/service_containers.go +++ b/pkg/compose/service_containers.go @@ -20,6 +20,7 @@ import ( "context" "fmt" "maps" + "slices" "strconv" "strings" "sync" @@ -70,13 +71,29 @@ func getScale(config types.ServiceConfig) (int, error) { // shared namespaces. The containersByService map provides the observed containers // grouped by service name. func resolveServiceReferences(service *types.ServiceConfig, containersByService map[string]Containers) error { - if err := resolveVolumeFrom(service, containersByService); err != nil { - return err - } - return resolveSharedNamespaces(service, containersByService) + return resolveReferences(service, func(name string) (string, bool) { + dependencies := containersByService[name] + if len(dependencies) == 0 { + return "", false + } + return dependencies.sorted()[0].ID, true + }) } -func resolveVolumeFrom(service *types.ServiceConfig, containersByService map[string]Containers) error { +// containerRef gives what names a container of the service in another +// container's configuration, false when there is none. +type containerRef func(service string) (string, bool) + +func resolveReferences(service *types.ServiceConfig, ref containerRef) error { + // resolveVolumeFrom writes in place: not into a slice shared with the project + service.VolumesFrom = slices.Clone(service.VolumesFrom) + if err := resolveVolumeFrom(service, ref); err != nil { + return err + } + return resolveSharedNamespaces(service, ref) +} + +func resolveVolumeFrom(service *types.ServiceConfig, ref containerRef) error { for i, vol := range service.VolumesFrom { spec := strings.Split(vol, ":") if len(spec) == 0 { @@ -87,38 +104,38 @@ func resolveVolumeFrom(service *types.ServiceConfig, containersByService map[str continue } name := spec[0] - dependencies := containersByService[name] - if len(dependencies) == 0 { + target, ok := ref(name) + if !ok { return fmt.Errorf("cannot share volume with service %s: container missing", name) } - service.VolumesFrom[i] = dependencies.sorted()[0].ID + service.VolumesFrom[i] = target } return nil } -func resolveSharedNamespaces(service *types.ServiceConfig, containersByService map[string]Containers) error { +func resolveSharedNamespaces(service *types.ServiceConfig, ref containerRef) error { if name := getDependentServiceFromMode(service.NetworkMode); name != "" { - dependencies := containersByService[name] - if len(dependencies) == 0 { + target, ok := ref(name) + if !ok { return fmt.Errorf("cannot share network namespace with service %s: container missing", name) } - service.NetworkMode = types.ContainerPrefix + dependencies.sorted()[0].ID + service.NetworkMode = types.ContainerPrefix + target } if name := getDependentServiceFromMode(service.Ipc); name != "" { - dependencies := containersByService[name] - if len(dependencies) == 0 { + target, ok := ref(name) + if !ok { return fmt.Errorf("cannot share IPC namespace with service %s: container missing", name) } - service.Ipc = types.ContainerPrefix + dependencies.sorted()[0].ID + service.Ipc = types.ContainerPrefix + target } if name := getDependentServiceFromMode(service.Pid); name != "" { - dependencies := containersByService[name] - if len(dependencies) == 0 { + target, ok := ref(name) + if !ok { return fmt.Errorf("cannot share PID namespace with service %s: container missing", name) } - service.Pid = types.ContainerPrefix + dependencies.sorted()[0].ID + service.Pid = types.ContainerPrefix + target } return nil diff --git a/pkg/compose/start.go b/pkg/compose/start.go index bae7710eb..6e311ebb0 100644 --- a/pkg/compose/start.go +++ b/pkg/compose/start.go @@ -29,6 +29,11 @@ import ( func (s *composeService) Start(ctx context.Context, projectName string, options api.StartOptions) error { return Run(ctx, func(ctx context.Context) error { + if options.Project != nil { + if err := s.rejectUnsupportedScheduledJobs(ctx, options.Project); err != nil { + return err + } + } return s.start(ctx, strings.ToLower(projectName), options) }, "start", s.events) } diff --git a/pkg/compose/up.go b/pkg/compose/up.go index 0fd52e590..8d1038d25 100644 --- a/pkg/compose/up.go +++ b/pkg/compose/up.go @@ -53,12 +53,22 @@ func (s *composeService) Up(ctx context.Context, project *types.Project, options detached := options.Start.Attach == nil || (options.Start.Wait && s.dryRun) var created *createdUp err := Run(ctx, tracing.SpanWrapFunc("project/up", tracing.ProjectOptions(ctx, project), func(ctx context.Context) error { - if detached { - return s.upDetached(ctx, project, options) + if err := s.rejectUnsupportedScheduledJobs(ctx, project); err != nil { + return err } var err error - created, err = s.upCreatePhase(ctx, project, options) - return err + if detached { + err = s.upDetached(ctx, project, options) + } else { + created, err = s.upCreatePhase(ctx, project, options) + } + if err != nil { + return err + } + if err := s.registerScheduledJobs(ctx, project); err != nil { + return err + } + return s.handleOrphanJobs(ctx, project, options.Create.RemoveOrphans, options.Create.IgnoreOrphans) }), "up", s.events) if err != nil { return err