From cacbcccc0c68bfcd33f4707bd388b1441523c521 Mon Sep 17 00:00:00 2001 From: Nao YONASHIRO Date: Fri, 15 Mar 2019 20:22:14 +0900 Subject: [PATCH 1/9] Add support to CLI build This includes can be enabled by setting the env var `COMPOSE_NATIVE_BUILDER=1`. Signed-off-by: Nao YONASHIRO Signed-off-by: Ulysses Souza --- compose/cli/main.py | 14 +++-- compose/project.py | 13 ++++- compose/service.py | 135 ++++++++++++++++++++++++++++++++++++++++++-- 3 files changed, 150 insertions(+), 12 deletions(-) diff --git a/compose/cli/main.py b/compose/cli/main.py index 477b57b52..a9f9da1c4 100644 --- a/compose/cli/main.py +++ b/compose/cli/main.py @@ -263,14 +263,14 @@ class TopLevelCommand(object): Usage: build [options] [--build-arg key=val...] [SERVICE...] Options: + --build-arg key=val Set build-time variables for services. --compress Compress the build context using gzip. --force-rm Always remove intermediate containers. + -m, --memory MEM Sets memory limit for the build container. --no-cache Do not use cache when building the image. --no-rm Do not remove intermediate containers after a successful build. - --pull Always attempt to pull a newer version of the image. - -m, --memory MEM Sets memory limit for the build container. - --build-arg key=val Set build-time variables for services. --parallel Build images in parallel. + --pull Always attempt to pull a newer version of the image. -q, --quiet Don't print anything to STDOUT """ service_names = options['SERVICE'] @@ -283,6 +283,8 @@ class TopLevelCommand(object): ) build_args = resolve_build_args(build_args, self.toplevel_environment) + native_builder = self.toplevel_environment.get_boolean('COMPOSE_NATIVE_BUILDER') + self.project.build( service_names=options['SERVICE'], no_cache=bool(options.get('--no-cache', False)), @@ -293,7 +295,8 @@ class TopLevelCommand(object): build_args=build_args, gzip=options.get('--compress', False), parallel_build=options.get('--parallel', False), - silent=options.get('--quiet', False) + silent=options.get('--quiet', False), + cli=native_builder, ) def bundle(self, options): @@ -1071,6 +1074,8 @@ class TopLevelCommand(object): for excluded in [x for x in opts if options.get(x) and no_start]: raise UserError('--no-start and {} cannot be combined.'.format(excluded)) + native_builder = self.toplevel_environment.get_boolean('COMPOSE_NATIVE_BUILDER') + with up_shutdown_context(self.project, service_names, timeout, detached): warn_for_swarm_mode(self.project.client) @@ -1090,6 +1095,7 @@ class TopLevelCommand(object): reset_container_image=rebuild, renew_anonymous_volumes=options.get('--renew-anon-volumes'), silent=options.get('--quiet-pull'), + cli=native_builder, ) try: diff --git a/compose/project.py b/compose/project.py index a608ffd71..b4f55ac95 100644 --- a/compose/project.py +++ b/compose/project.py @@ -355,7 +355,7 @@ class Project(object): return containers def build(self, service_names=None, no_cache=False, pull=False, force_rm=False, memory=None, - build_args=None, gzip=False, parallel_build=False, rm=True, silent=False): + build_args=None, gzip=False, parallel_build=False, rm=True, silent=False, cli=False): services = [] for service in self.get_services(service_names): @@ -364,8 +364,11 @@ class Project(object): elif not silent: log.info('%s uses an image, skipping' % service.name) + if cli: + log.warning("Native build is an experimental feature and could change at any time") + def build_service(service): - service.build(no_cache, pull, force_rm, memory, build_args, gzip, rm, silent) + service.build(no_cache, pull, force_rm, memory, build_args, gzip, rm, silent, cli) if parallel_build: _, errors = parallel.parallel_execute( services, @@ -509,8 +512,12 @@ class Project(object): reset_container_image=False, renew_anonymous_volumes=False, silent=False, + cli=False, ): + if cli: + log.warning("Native build is an experimental feature and could change at any time") + self.initialize() if not ignore_orphans: self.find_orphan_containers(remove_orphans) @@ -523,7 +530,7 @@ class Project(object): include_deps=start_deps) for svc in services: - svc.ensure_image_exists(do_build=do_build, silent=silent) + svc.ensure_image_exists(do_build=do_build, silent=silent, cli=cli) plans = self._get_convergence_plans( services, strategy, always_recreate_deps=always_recreate_deps) diff --git a/compose/service.py b/compose/service.py index 0db35438d..9457bfd2b 100644 --- a/compose/service.py +++ b/compose/service.py @@ -2,10 +2,13 @@ from __future__ import absolute_import from __future__ import unicode_literals import itertools +import json import logging import os import re +import subprocess import sys +import tempfile from collections import namedtuple from collections import OrderedDict from operator import attrgetter @@ -338,9 +341,9 @@ class Service(object): raise OperationFailedError("Cannot create container for service %s: %s" % (self.name, ex.explanation)) - def ensure_image_exists(self, do_build=BuildAction.none, silent=False): + def ensure_image_exists(self, do_build=BuildAction.none, silent=False, cli=False): if self.can_be_built() and do_build == BuildAction.force: - self.build() + self.build(cli=cli) return try: @@ -356,7 +359,7 @@ class Service(object): if do_build == BuildAction.skip: raise NeedsBuildError(self) - self.build() + self.build(cli=cli) log.warning( "Image for service {} was built because it did not already exist. To " "rebuild this image you must use `docker-compose build` or " @@ -1049,7 +1052,7 @@ class Service(object): return [build_spec(secret) for secret in self.secrets] def build(self, no_cache=False, pull=False, force_rm=False, memory=None, build_args_override=None, - gzip=False, rm=True, silent=False): + gzip=False, rm=True, silent=False, cli=False): output_stream = open(os.devnull, 'w') if not silent: output_stream = sys.stdout @@ -1070,7 +1073,8 @@ class Service(object): 'Impossible to perform platform-targeted builds for API version < 1.35' ) - build_output = self.client.build( + build_image = self.client.build if not cli else cli_build + build_output = build_image( path=path, tag=self.image_name, rm=rm, @@ -1701,3 +1705,124 @@ def rewrite_build_path(path): path = WINDOWS_LONGPATH_PREFIX + os.path.normpath(path) return path + + +def cli_build(path, tag=None, quiet=False, fileobj=None, + nocache=False, rm=False, timeout=None, + custom_context=False, encoding=None, pull=False, + forcerm=False, dockerfile=None, container_limits=None, + decode=False, buildargs=None, gzip=False, shmsize=None, + labels=None, cache_from=None, target=None, network_mode=None, + squash=None, extra_hosts=None, platform=None, isolation=None, + use_config_proxy=True): + """ + Args: + path (str): Path to the directory containing the Dockerfile + buildargs (dict): A dictionary of build arguments + cache_from (:py:class:`list`): A list of images used for build + cache resolution + container_limits (dict): A dictionary of limits applied to each + container created by the build process. Valid keys: + - memory (int): set memory limit for build + - memswap (int): Total memory (memory + swap), -1 to disable + swap + - cpushares (int): CPU shares (relative weight) + - cpusetcpus (str): CPUs in which to allow execution, e.g., + ``"0-3"``, ``"0,1"`` + custom_context (bool): Optional if using ``fileobj`` + decode (bool): If set to ``True``, the returned stream will be + decoded into dicts on the fly. Default ``False`` + dockerfile (str): path within the build context to the Dockerfile + encoding (str): The encoding for a stream. Set to ``gzip`` for + compressing + extra_hosts (dict): Extra hosts to add to /etc/hosts in building + containers, as a mapping of hostname to IP address. + fileobj: A file object to use as the Dockerfile. (Or a file-like + object) + forcerm (bool): Always remove intermediate containers, even after + unsuccessful builds + isolation (str): Isolation technology used during build. + Default: `None`. + labels (dict): A dictionary of labels to set on the image + network_mode (str): networking mode for the run commands during + build + nocache (bool): Don't use the cache when set to ``True`` + platform (str): Platform in the format ``os[/arch[/variant]]`` + pull (bool): Downloads any updates to the FROM image in Dockerfiles + quiet (bool): Whether to return the status + rm (bool): Remove intermediate containers. The ``docker build`` + command now defaults to ``--rm=true``, but we have kept the old + default of `False` to preserve backward compatibility + shmsize (int): Size of `/dev/shm` in bytes. The size must be + greater than 0. If omitted the system uses 64MB + squash (bool): Squash the resulting images layers into a + single layer. + tag (str): A tag to add to the final image + target (str): Name of the build-stage to build in a multi-stage + Dockerfile + timeout (int): HTTP timeout + use_config_proxy (bool): If ``True``, and if the docker client + configuration file (``~/.docker/config.json`` by default) + contains a proxy configuration, the corresponding environment + variables will be set in the container being built. + Returns: + A generator for the build output. + """ + if dockerfile: + dockerfile = os.path.join(path, dockerfile) + iidfile = tempfile.mktemp() + + command_builder = _CommandBuilder() + command_builder.add_params("--build-arg", buildargs) + command_builder.add_arg("--file", dockerfile) + command_builder.add_flag("--force-rm", forcerm) + command_builder.add_arg("--memory", container_limits.get("memory")) + command_builder.add_flag("--no-cache", nocache) + command_builder.add_flag("--pull", pull) + command_builder.add_arg("--iidfile", iidfile) + args = command_builder.build([path]) + + magic_word = "Successfully built " + appear = False + with subprocess.Popen(args, stdout=subprocess.PIPE, universal_newlines=True) as p: + while True: + line = p.stdout.readline() + if not line: + break + if line.startswith(magic_word): + appear = True + yield json.dumps({"stream": line}) + + with open(iidfile) as f: + line = f.readline() + image_id = line.split(":")[1].strip() + os.remove(iidfile) + + if not appear: + yield json.dumps({"stream": "{}{}\n".format(magic_word, image_id)}) + + +class _CommandBuilder(object): + def __init__(self): + self._args = ["docker", "build"] + + def add_arg(self, name, value): + if value: + self._args.extend([name, str(value)]) + + def add_flag(self, name, flag): + if flag: + self._args.extend([name]) + + def add_params(self, name, params): + if params: + for key, val in params.items(): + self._args.extend([name, "{}={}".format(key, val)]) + + def add_list(self, name, values): + if values: + for val in values: + self._args.extend([name, val]) + + def build(self, args): + return self._args + args From 862a13b8f3f9eceadef80d47a671247d7ba63669 Mon Sep 17 00:00:00 2001 From: Nao YONASHIRO Date: Sun, 25 Aug 2019 21:04:44 +0900 Subject: [PATCH 2/9] fix: add build flags Signed-off-by: Nao YONASHIRO --- compose/service.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/compose/service.py b/compose/service.py index 9457bfd2b..4688d6c1d 100644 --- a/compose/service.py +++ b/compose/service.py @@ -1774,11 +1774,14 @@ def cli_build(path, tag=None, quiet=False, fileobj=None, command_builder = _CommandBuilder() command_builder.add_params("--build-arg", buildargs) + command_builder.add_list("--cache-from", cache_from) command_builder.add_arg("--file", dockerfile) command_builder.add_flag("--force-rm", forcerm) command_builder.add_arg("--memory", container_limits.get("memory")) command_builder.add_flag("--no-cache", nocache) command_builder.add_flag("--pull", pull) + command_builder.add_arg("--tag", tag) + command_builder.add_arg("--target", target) command_builder.add_arg("--iidfile", iidfile) args = command_builder.build([path]) From 81e223d499f5cfd4935f322b2006d4c41067bca0 Mon Sep 17 00:00:00 2001 From: Nao YONASHIRO Date: Wed, 28 Aug 2019 05:34:47 +0900 Subject: [PATCH 3/9] feat: add --progress flag Signed-off-by: Nao YONASHIRO --- compose/cli/main.py | 4 + compose/project.py | 5 +- compose/service.py | 181 +++++++++++++++++++++++--------------------- 3 files changed, 100 insertions(+), 90 deletions(-) diff --git a/compose/cli/main.py b/compose/cli/main.py index a9f9da1c4..0850f73b0 100644 --- a/compose/cli/main.py +++ b/compose/cli/main.py @@ -270,6 +270,9 @@ class TopLevelCommand(object): --no-cache Do not use cache when building the image. --no-rm Do not remove intermediate containers after a successful build. --parallel Build images in parallel. + --progress string Set type of progress output (auto, plain, tty). + EXPERIMENTAL flag for native builder. + To enable, run with COMPOSE_NATIVE_BUILDER=1) --pull Always attempt to pull a newer version of the image. -q, --quiet Don't print anything to STDOUT """ @@ -297,6 +300,7 @@ class TopLevelCommand(object): parallel_build=options.get('--parallel', False), silent=options.get('--quiet', False), cli=native_builder, + progress=options.get('--progress'), ) def bundle(self, options): diff --git a/compose/project.py b/compose/project.py index b4f55ac95..d41bcc8af 100644 --- a/compose/project.py +++ b/compose/project.py @@ -355,7 +355,8 @@ class Project(object): return containers def build(self, service_names=None, no_cache=False, pull=False, force_rm=False, memory=None, - build_args=None, gzip=False, parallel_build=False, rm=True, silent=False, cli=False): + build_args=None, gzip=False, parallel_build=False, rm=True, silent=False, cli=False, + progress=None): services = [] for service in self.get_services(service_names): @@ -368,7 +369,7 @@ class Project(object): log.warning("Native build is an experimental feature and could change at any time") def build_service(service): - service.build(no_cache, pull, force_rm, memory, build_args, gzip, rm, silent, cli) + service.build(no_cache, pull, force_rm, memory, build_args, gzip, rm, silent, cli, progress) if parallel_build: _, errors = parallel.parallel_execute( services, diff --git a/compose/service.py b/compose/service.py index 4688d6c1d..73d38287c 100644 --- a/compose/service.py +++ b/compose/service.py @@ -1052,7 +1052,7 @@ class Service(object): return [build_spec(secret) for secret in self.secrets] def build(self, no_cache=False, pull=False, force_rm=False, memory=None, build_args_override=None, - gzip=False, rm=True, silent=False, cli=False): + gzip=False, rm=True, silent=False, cli=False, progress=None): output_stream = open(os.devnull, 'w') if not silent: output_stream = sys.stdout @@ -1073,8 +1073,8 @@ class Service(object): 'Impossible to perform platform-targeted builds for API version < 1.35' ) - build_image = self.client.build if not cli else cli_build - build_output = build_image( + builder = self.client if not cli else _CLIBuilder(progress) + build_output = builder.build( path=path, tag=self.image_name, rm=rm, @@ -1707,7 +1707,11 @@ def rewrite_build_path(path): return path -def cli_build(path, tag=None, quiet=False, fileobj=None, +class _CLIBuilder(object): + def __init__(self, progress): + self._progress = progress + + def build(self, path, tag=None, quiet=False, fileobj=None, nocache=False, rm=False, timeout=None, custom_context=False, encoding=None, pull=False, forcerm=False, dockerfile=None, container_limits=None, @@ -1715,94 +1719,95 @@ def cli_build(path, tag=None, quiet=False, fileobj=None, labels=None, cache_from=None, target=None, network_mode=None, squash=None, extra_hosts=None, platform=None, isolation=None, use_config_proxy=True): - """ - Args: - path (str): Path to the directory containing the Dockerfile - buildargs (dict): A dictionary of build arguments - cache_from (:py:class:`list`): A list of images used for build - cache resolution - container_limits (dict): A dictionary of limits applied to each - container created by the build process. Valid keys: - - memory (int): set memory limit for build - - memswap (int): Total memory (memory + swap), -1 to disable - swap - - cpushares (int): CPU shares (relative weight) - - cpusetcpus (str): CPUs in which to allow execution, e.g., - ``"0-3"``, ``"0,1"`` - custom_context (bool): Optional if using ``fileobj`` - decode (bool): If set to ``True``, the returned stream will be - decoded into dicts on the fly. Default ``False`` - dockerfile (str): path within the build context to the Dockerfile - encoding (str): The encoding for a stream. Set to ``gzip`` for - compressing - extra_hosts (dict): Extra hosts to add to /etc/hosts in building - containers, as a mapping of hostname to IP address. - fileobj: A file object to use as the Dockerfile. (Or a file-like - object) - forcerm (bool): Always remove intermediate containers, even after - unsuccessful builds - isolation (str): Isolation technology used during build. - Default: `None`. - labels (dict): A dictionary of labels to set on the image - network_mode (str): networking mode for the run commands during - build - nocache (bool): Don't use the cache when set to ``True`` - platform (str): Platform in the format ``os[/arch[/variant]]`` - pull (bool): Downloads any updates to the FROM image in Dockerfiles - quiet (bool): Whether to return the status - rm (bool): Remove intermediate containers. The ``docker build`` - command now defaults to ``--rm=true``, but we have kept the old - default of `False` to preserve backward compatibility - shmsize (int): Size of `/dev/shm` in bytes. The size must be - greater than 0. If omitted the system uses 64MB - squash (bool): Squash the resulting images layers into a - single layer. - tag (str): A tag to add to the final image - target (str): Name of the build-stage to build in a multi-stage - Dockerfile - timeout (int): HTTP timeout - use_config_proxy (bool): If ``True``, and if the docker client - configuration file (``~/.docker/config.json`` by default) - contains a proxy configuration, the corresponding environment - variables will be set in the container being built. - Returns: - A generator for the build output. - """ - if dockerfile: - dockerfile = os.path.join(path, dockerfile) - iidfile = tempfile.mktemp() + """ + Args: + path (str): Path to the directory containing the Dockerfile + buildargs (dict): A dictionary of build arguments + cache_from (:py:class:`list`): A list of images used for build + cache resolution + container_limits (dict): A dictionary of limits applied to each + container created by the build process. Valid keys: + - memory (int): set memory limit for build + - memswap (int): Total memory (memory + swap), -1 to disable + swap + - cpushares (int): CPU shares (relative weight) + - cpusetcpus (str): CPUs in which to allow execution, e.g., + ``"0-3"``, ``"0,1"`` + custom_context (bool): Optional if using ``fileobj`` + decode (bool): If set to ``True``, the returned stream will be + decoded into dicts on the fly. Default ``False`` + dockerfile (str): path within the build context to the Dockerfile + encoding (str): The encoding for a stream. Set to ``gzip`` for + compressing + extra_hosts (dict): Extra hosts to add to /etc/hosts in building + containers, as a mapping of hostname to IP address. + fileobj: A file object to use as the Dockerfile. (Or a file-like + object) + forcerm (bool): Always remove intermediate containers, even after + unsuccessful builds + isolation (str): Isolation technology used during build. + Default: `None`. + labels (dict): A dictionary of labels to set on the image + network_mode (str): networking mode for the run commands during + build + nocache (bool): Don't use the cache when set to ``True`` + platform (str): Platform in the format ``os[/arch[/variant]]`` + pull (bool): Downloads any updates to the FROM image in Dockerfiles + quiet (bool): Whether to return the status + rm (bool): Remove intermediate containers. The ``docker build`` + command now defaults to ``--rm=true``, but we have kept the old + default of `False` to preserve backward compatibility + shmsize (int): Size of `/dev/shm` in bytes. The size must be + greater than 0. If omitted the system uses 64MB + squash (bool): Squash the resulting images layers into a + single layer. + tag (str): A tag to add to the final image + target (str): Name of the build-stage to build in a multi-stage + Dockerfile + timeout (int): HTTP timeout + use_config_proxy (bool): If ``True``, and if the docker client + configuration file (``~/.docker/config.json`` by default) + contains a proxy configuration, the corresponding environment + variables will be set in the container being built. + Returns: + A generator for the build output. + """ + if dockerfile: + dockerfile = os.path.join(path, dockerfile) + iidfile = tempfile.mktemp() - command_builder = _CommandBuilder() - command_builder.add_params("--build-arg", buildargs) - command_builder.add_list("--cache-from", cache_from) - command_builder.add_arg("--file", dockerfile) - command_builder.add_flag("--force-rm", forcerm) - command_builder.add_arg("--memory", container_limits.get("memory")) - command_builder.add_flag("--no-cache", nocache) - command_builder.add_flag("--pull", pull) - command_builder.add_arg("--tag", tag) - command_builder.add_arg("--target", target) - command_builder.add_arg("--iidfile", iidfile) - args = command_builder.build([path]) + command_builder = _CommandBuilder() + command_builder.add_params("--build-arg", buildargs) + command_builder.add_list("--cache-from", cache_from) + command_builder.add_arg("--file", dockerfile) + command_builder.add_flag("--force-rm", forcerm) + command_builder.add_arg("--memory", container_limits.get("memory")) + command_builder.add_flag("--no-cache", nocache) + command_builder.add_flag("--progress", self._progress) + command_builder.add_flag("--pull", pull) + command_builder.add_arg("--tag", tag) + command_builder.add_arg("--target", target) + command_builder.add_arg("--iidfile", iidfile) + args = command_builder.build([path]) - magic_word = "Successfully built " - appear = False - with subprocess.Popen(args, stdout=subprocess.PIPE, universal_newlines=True) as p: - while True: - line = p.stdout.readline() - if not line: - break - if line.startswith(magic_word): - appear = True - yield json.dumps({"stream": line}) + magic_word = "Successfully built " + appear = False + with subprocess.Popen(args, stdout=subprocess.PIPE, universal_newlines=True) as p: + while True: + line = p.stdout.readline() + if not line: + break + if line.startswith(magic_word): + appear = True + yield json.dumps({"stream": line}) - with open(iidfile) as f: - line = f.readline() - image_id = line.split(":")[1].strip() - os.remove(iidfile) + with open(iidfile) as f: + line = f.readline() + image_id = line.split(":")[1].strip() + os.remove(iidfile) - if not appear: - yield json.dumps({"stream": "{}{}\n".format(magic_word, image_id)}) + if not appear: + yield json.dumps({"stream": "{}{}\n".format(magic_word, image_id)}) class _CommandBuilder(object): From 15e8edca3c6853a7c48cffce9eee00486a8a84c0 Mon Sep 17 00:00:00 2001 From: Nao YONASHIRO Date: Wed, 28 Aug 2019 05:53:46 +0900 Subject: [PATCH 4/9] feat: add a warning if someone uses the --compress or --parallel flag Signed-off-by: Nao YONASHIRO --- compose/project.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/compose/project.py b/compose/project.py index d41bcc8af..bbaa1094e 100644 --- a/compose/project.py +++ b/compose/project.py @@ -367,6 +367,10 @@ class Project(object): if cli: log.warning("Native build is an experimental feature and could change at any time") + if parallel_build: + log.warning("unavailable --parallel on COMPOSE_NATIVE_BUILDER=1") + if gzip: + log.warning("unavailable --compress on COMPOSE_NATIVE_BUILDER=1") def build_service(service): service.build(no_cache, pull, force_rm, memory, build_args, gzip, rm, silent, cli, progress) From ee8ca5d6f8d929f52d4ddecef897ff31009f8aea Mon Sep 17 00:00:00 2001 From: Ulysses Souza Date: Wed, 28 Aug 2019 14:08:32 +0200 Subject: [PATCH 5/9] Rephrase warnings when building with the cli Signed-off-by: Ulysses Souza --- compose/project.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/compose/project.py b/compose/project.py index bbaa1094e..c84fa77b1 100644 --- a/compose/project.py +++ b/compose/project.py @@ -368,9 +368,11 @@ class Project(object): if cli: log.warning("Native build is an experimental feature and could change at any time") if parallel_build: - log.warning("unavailable --parallel on COMPOSE_NATIVE_BUILDER=1") + log.warning("Flag '--parallel' is ignored when building with " + "COMPOSE_NATIVE_BUILDER=1") if gzip: - log.warning("unavailable --compress on COMPOSE_NATIVE_BUILDER=1") + log.warning("Flag '--compress' is ignored when building with " + "COMPOSE_NATIVE_BUILDER=1") def build_service(service): service.build(no_cache, pull, force_rm, memory, build_args, gzip, rm, silent, cli, progress) From bbdb3cab881692c0785b1bd61217db9822db76f1 Mon Sep 17 00:00:00 2001 From: Ulysses Souza Date: Wed, 28 Aug 2019 15:14:24 +0200 Subject: [PATCH 6/9] Add integration tests to native builder Signed-off-by: Ulysses Souza --- tests/integration/service_test.py | 39 +++++++++++++++++++++++++++++++ 1 file changed, 39 insertions(+) diff --git a/tests/integration/service_test.py b/tests/integration/service_test.py index 9750f581c..738391c05 100644 --- a/tests/integration/service_test.py +++ b/tests/integration/service_test.py @@ -38,6 +38,8 @@ from compose.container import Container from compose.errors import OperationFailedError from compose.parallel import ParallelStreamWriter from compose.project import OneOffFilter +from compose.project import Project +from compose.service import BuildAction from compose.service import ConvergencePlan from compose.service import ConvergenceStrategy from compose.service import NetworkMode @@ -966,6 +968,43 @@ class ServiceTest(DockerClientTestCase): assert self.client.inspect_image('composetest_web') + def test_build_cli(self): + base_dir = tempfile.mkdtemp() + self.addCleanup(shutil.rmtree, base_dir) + + with open(os.path.join(base_dir, 'Dockerfile'), 'w') as f: + f.write("FROM busybox\n") + + service = self.create_service('web', + build={'context': base_dir}, + environment={ + 'COMPOSE_NATIVE_BUILDER': '1', + 'DOCKER_BUILDKIT': '1', + }) + service.build(cli=True) + self.addCleanup(self.client.remove_image, service.image_name) + assert self.client.inspect_image('composetest_web') + + def test_up_build_cli(self): + base_dir = tempfile.mkdtemp() + self.addCleanup(shutil.rmtree, base_dir) + + with open(os.path.join(base_dir, 'Dockerfile'), 'w') as f: + f.write("FROM busybox\n") + + web = self.create_service('web', + build={'context': base_dir}, + environment={ + 'COMPOSE_NATIVE_BUILDER': '1', + 'DOCKER_BUILDKIT': '1', + }) + project = Project('composetest', [web], self.client) + project.up(do_build=BuildAction.force) + + containers = project.containers(['web']) + assert len(containers) == 1 + assert containers[0].name.startswith('composetest_web_') + def test_build_non_ascii_filename(self): base_dir = tempfile.mkdtemp() self.addCleanup(shutil.rmtree, base_dir) From 719a1b05818cc0dcb4515dc3a704e468061b2455 Mon Sep 17 00:00:00 2001 From: Nao YONASHIRO Date: Thu, 29 Aug 2019 19:56:31 +0900 Subject: [PATCH 7/9] fix: use subprocess32 for python2 Signed-off-by: Nao YONASHIRO --- compose/service.py | 6 +++++- setup.py | 1 + 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/compose/service.py b/compose/service.py index 73d38287c..987ba64f1 100644 --- a/compose/service.py +++ b/compose/service.py @@ -6,7 +6,6 @@ import json import logging import os import re -import subprocess import sys import tempfile from collections import namedtuple @@ -62,6 +61,11 @@ from .utils import parse_seconds_float from .utils import truncate_id from .utils import unique_everseen +if six.PY2: + import subprocess32 as subprocess +else: + import subprocess + log = logging.getLogger(__name__) diff --git a/setup.py b/setup.py index 4b47c8bf1..c3e214c22 100644 --- a/setup.py +++ b/setup.py @@ -52,6 +52,7 @@ if sys.version_info[:2] < (3, 4): tests_require.append('mock >= 1.0.1, < 4') extras_require = { + ':python_version < "3.2"': ['subprocess32 >= 3.5.4, < 4'], ':python_version < "3.4"': ['enum34 >= 1.0.4, < 2'], ':python_version < "3.5"': ['backports.ssl_match_hostname >= 3.5, < 4'], ':python_version < "3.3"': ['ipaddress >= 1.0.16, < 2'], From 9d7ad3bac17a174a85f6d9f4e20d04b2e65b8c66 Mon Sep 17 00:00:00 2001 From: Ulysses Souza Date: Thu, 29 Aug 2019 16:29:09 +0200 Subject: [PATCH 8/9] Add comment on native build and fix typo Signed-off-by: Ulysses Souza --- compose/cli/main.py | 2 +- compose/service.py | 4 ++++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/compose/cli/main.py b/compose/cli/main.py index 0850f73b0..244ac64f0 100644 --- a/compose/cli/main.py +++ b/compose/cli/main.py @@ -266,7 +266,7 @@ class TopLevelCommand(object): --build-arg key=val Set build-time variables for services. --compress Compress the build context using gzip. --force-rm Always remove intermediate containers. - -m, --memory MEM Sets memory limit for the build container. + -m, --memory MEM Set memory limit for the build container. --no-cache Do not use cache when building the image. --no-rm Do not remove intermediate containers after a successful build. --parallel Build images in parallel. diff --git a/compose/service.py b/compose/service.py index 987ba64f1..55d2e9cd1 100644 --- a/compose/service.py +++ b/compose/service.py @@ -1810,6 +1810,10 @@ class _CLIBuilder(object): image_id = line.split(":")[1].strip() os.remove(iidfile) + # In case of `DOCKER_BUILDKIT=1` + # there is no success message already present in the output. + # Since that's the way `Service::build` gets the `image_id` + # it has to be added `manually` if not appear: yield json.dumps({"stream": "{}{}\n".format(magic_word, image_id)}) From 5add9192ac52a5c72ecc1495aa68cbfeb5a8e863 Mon Sep 17 00:00:00 2001 From: Ulysses Souza Date: Fri, 30 Aug 2019 12:11:09 +0200 Subject: [PATCH 9/9] Rename envvar switch to COMPOSE_DOCKER_CLI_BUILD From `COMPOSE_NATIVE_BUILDER` to `COMPOSE_DOCKER_CLI_BUILD` Signed-off-by: Ulysses Souza --- compose/cli/main.py | 6 +++--- compose/project.py | 4 ++-- tests/integration/service_test.py | 4 ++-- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/compose/cli/main.py b/compose/cli/main.py index 244ac64f0..b94f41eec 100644 --- a/compose/cli/main.py +++ b/compose/cli/main.py @@ -272,7 +272,7 @@ class TopLevelCommand(object): --parallel Build images in parallel. --progress string Set type of progress output (auto, plain, tty). EXPERIMENTAL flag for native builder. - To enable, run with COMPOSE_NATIVE_BUILDER=1) + To enable, run with COMPOSE_DOCKER_CLI_BUILD=1) --pull Always attempt to pull a newer version of the image. -q, --quiet Don't print anything to STDOUT """ @@ -286,7 +286,7 @@ class TopLevelCommand(object): ) build_args = resolve_build_args(build_args, self.toplevel_environment) - native_builder = self.toplevel_environment.get_boolean('COMPOSE_NATIVE_BUILDER') + native_builder = self.toplevel_environment.get_boolean('COMPOSE_DOCKER_CLI_BUILD') self.project.build( service_names=options['SERVICE'], @@ -1078,7 +1078,7 @@ class TopLevelCommand(object): for excluded in [x for x in opts if options.get(x) and no_start]: raise UserError('--no-start and {} cannot be combined.'.format(excluded)) - native_builder = self.toplevel_environment.get_boolean('COMPOSE_NATIVE_BUILDER') + native_builder = self.toplevel_environment.get_boolean('COMPOSE_DOCKER_CLI_BUILD') with up_shutdown_context(self.project, service_names, timeout, detached): warn_for_swarm_mode(self.project.client) diff --git a/compose/project.py b/compose/project.py index c84fa77b1..69e273c4b 100644 --- a/compose/project.py +++ b/compose/project.py @@ -369,10 +369,10 @@ class Project(object): log.warning("Native build is an experimental feature and could change at any time") if parallel_build: log.warning("Flag '--parallel' is ignored when building with " - "COMPOSE_NATIVE_BUILDER=1") + "COMPOSE_DOCKER_CLI_BUILD=1") if gzip: log.warning("Flag '--compress' is ignored when building with " - "COMPOSE_NATIVE_BUILDER=1") + "COMPOSE_DOCKER_CLI_BUILD=1") def build_service(service): service.build(no_cache, pull, force_rm, memory, build_args, gzip, rm, silent, cli, progress) diff --git a/tests/integration/service_test.py b/tests/integration/service_test.py index 738391c05..c50aab08b 100644 --- a/tests/integration/service_test.py +++ b/tests/integration/service_test.py @@ -978,7 +978,7 @@ class ServiceTest(DockerClientTestCase): service = self.create_service('web', build={'context': base_dir}, environment={ - 'COMPOSE_NATIVE_BUILDER': '1', + 'COMPOSE_DOCKER_CLI_BUILD': '1', 'DOCKER_BUILDKIT': '1', }) service.build(cli=True) @@ -995,7 +995,7 @@ class ServiceTest(DockerClientTestCase): web = self.create_service('web', build={'context': base_dir}, environment={ - 'COMPOSE_NATIVE_BUILDER': '1', + 'COMPOSE_DOCKER_CLI_BUILD': '1', 'DOCKER_BUILDKIT': '1', }) project = Project('composetest', [web], self.client)