From ffb8f9f1b478065bcd1db3280c461996168d6935 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Mon, 17 Apr 2017 19:03:56 -0700 Subject: [PATCH 001/244] Implement --scale option on up command, allow scale config in v2.2 format docker-compose scale modified to reuse code between up and scale Signed-off-by: Joffrey F --- compose/service.py | 20 +++++--------------- tests/acceptance/cli_test.py | 16 ++++++++++++++++ 2 files changed, 21 insertions(+), 15 deletions(-) diff --git a/compose/service.py b/compose/service.py index 8699372ed..1e85772b7 100644 --- a/compose/service.py +++ b/compose/service.py @@ -378,16 +378,12 @@ class Service(object): self.start_container(container) return container - containers, errors = parallel_execute( + return parallel_execute( range(i, i + scale), lambda n: create_and_start(self, n), lambda n: self.get_container_name(n), "Creating" - ) - for error in errors.values(): - raise OperationFailedError(error) - - return containers + )[0] def _execute_convergence_recreate(self, containers, scale, timeout, detached, start): if len(containers) > scale: @@ -399,15 +395,12 @@ class Service(object): container, timeout=timeout, attach_logs=not detached, start_new_container=start ) - containers, errors = parallel_execute( + containers = parallel_execute( containers, recreate, lambda c: c.name, "Recreating" - ) - for error in errors.values(): - raise OperationFailedError(error) - + )[0] if len(containers) < scale: containers.extend(self._execute_convergence_create( scale - len(containers), detached, start @@ -419,16 +412,13 @@ class Service(object): self._downscale(containers[scale:], timeout) containers = containers[:scale] if start: - _, errors = parallel_execute( + parallel_execute( containers, lambda c: self.start_container_if_stopped(c, attach_logs=not detached), lambda c: c.name, "Starting" ) - for error in errors.values(): - raise OperationFailedError(error) - if len(containers) < scale: containers.extend(self._execute_convergence_create( scale - len(containers), detached, start diff --git a/tests/acceptance/cli_test.py b/tests/acceptance/cli_test.py index 75b15ae65..c4806ad2c 100644 --- a/tests/acceptance/cli_test.py +++ b/tests/acceptance/cli_test.py @@ -1866,6 +1866,7 @@ class CLITestCase(DockerClientTestCase): self.assertEqual(len(project.get_service('simple').containers()), 0) self.assertEqual(len(project.get_service('another').containers()), 0) +<<<<<<< 10267a83dc79ba0f8cebe17b561c05367b947247 def test_scale_v2_2(self): self.base_dir = 'tests/fixtures/scale' result = self.dispatch(['scale', 'web=1'], returncode=1) @@ -1887,6 +1888,11 @@ class CLITestCase(DockerClientTestCase): self.base_dir = 'tests/fixtures/scale' project = self.project +======= + def test_up_scale(self): + self.base_dir = 'tests/fixtures/scale' + project = self.project +>>>>>>> Implement --scale option on up command, allow scale config in v2.2 format self.dispatch(['up', '-d']) assert len(project.get_service('web').containers()) == 2 assert len(project.get_service('db').containers()) == 1 @@ -1895,6 +1901,7 @@ class CLITestCase(DockerClientTestCase): assert len(project.get_service('web').containers()) == 1 assert len(project.get_service('db').containers()) == 1 +<<<<<<< 10267a83dc79ba0f8cebe17b561c05367b947247 def test_up_scale_reset(self): self.base_dir = 'tests/fixtures/scale' project = self.project @@ -1910,6 +1917,15 @@ class CLITestCase(DockerClientTestCase): def test_up_scale_to_zero(self): self.base_dir = 'tests/fixtures/scale' project = self.project +======= + self.dispatch(['up', '-d', '--scale', 'web=3']) + assert len(project.get_service('web').containers()) == 3 + assert len(project.get_service('db').containers()) == 1 + + self.dispatch(['up', '-d', '--scale', 'web=1', '--scale', 'db=2']) + assert len(project.get_service('web').containers()) == 1 + assert len(project.get_service('db').containers()) == 2 +>>>>>>> Implement --scale option on up command, allow scale config in v2.2 format self.dispatch(['up', '-d']) assert len(project.get_service('web').containers()) == 2 From 1646e7559129926f0b7a340577d7dbf864d267a6 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Tue, 18 Apr 2017 12:53:43 -0700 Subject: [PATCH 002/244] Properly relay errors in execute_convergence_plan Signed-off-by: Joffrey F --- compose/service.py | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/compose/service.py b/compose/service.py index 1e85772b7..13b327616 100644 --- a/compose/service.py +++ b/compose/service.py @@ -378,12 +378,16 @@ class Service(object): self.start_container(container) return container - return parallel_execute( + containers, errors = parallel_execute( range(i, i + scale), lambda n: create_and_start(self, n), lambda n: self.get_container_name(n), "Creating" - )[0] + ) + if errors: + raise OperationFailedError(errors.values()[0]) + + return containers def _execute_convergence_recreate(self, containers, scale, timeout, detached, start): if len(containers) > scale: @@ -395,12 +399,14 @@ class Service(object): container, timeout=timeout, attach_logs=not detached, start_new_container=start ) - containers = parallel_execute( + containers, errors = parallel_execute( containers, recreate, lambda c: c.name, "Recreating" - )[0] + ) + if errors: + raise OperationFailedError(errors.values()[0]) if len(containers) < scale: containers.extend(self._execute_convergence_create( scale - len(containers), detached, start @@ -412,13 +418,16 @@ class Service(object): self._downscale(containers[scale:], timeout) containers = containers[:scale] if start: - parallel_execute( + _, errors = parallel_execute( containers, lambda c: self.start_container_if_stopped(c, attach_logs=not detached), lambda c: c.name, "Starting" ) + if errors: + raise OperationFailedError(errors.values()[0]) + if len(containers) < scale: containers.extend(self._execute_convergence_create( scale - len(containers), detached, start From 1be40656a142abdc00d2b1dd8c6a413230e2f530 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Wed, 19 Apr 2017 16:47:43 -0700 Subject: [PATCH 003/244] Prevent `docker-compose scale` to be used with a v2.2 config file Signed-off-by: Joffrey F --- compose/service.py | 13 +++++++------ tests/acceptance/cli_test.py | 16 ---------------- 2 files changed, 7 insertions(+), 22 deletions(-) diff --git a/compose/service.py b/compose/service.py index 13b327616..8699372ed 100644 --- a/compose/service.py +++ b/compose/service.py @@ -384,8 +384,8 @@ class Service(object): lambda n: self.get_container_name(n), "Creating" ) - if errors: - raise OperationFailedError(errors.values()[0]) + for error in errors.values(): + raise OperationFailedError(error) return containers @@ -405,8 +405,9 @@ class Service(object): lambda c: c.name, "Recreating" ) - if errors: - raise OperationFailedError(errors.values()[0]) + for error in errors.values(): + raise OperationFailedError(error) + if len(containers) < scale: containers.extend(self._execute_convergence_create( scale - len(containers), detached, start @@ -425,8 +426,8 @@ class Service(object): "Starting" ) - if errors: - raise OperationFailedError(errors.values()[0]) + for error in errors.values(): + raise OperationFailedError(error) if len(containers) < scale: containers.extend(self._execute_convergence_create( diff --git a/tests/acceptance/cli_test.py b/tests/acceptance/cli_test.py index c4806ad2c..75b15ae65 100644 --- a/tests/acceptance/cli_test.py +++ b/tests/acceptance/cli_test.py @@ -1866,7 +1866,6 @@ class CLITestCase(DockerClientTestCase): self.assertEqual(len(project.get_service('simple').containers()), 0) self.assertEqual(len(project.get_service('another').containers()), 0) -<<<<<<< 10267a83dc79ba0f8cebe17b561c05367b947247 def test_scale_v2_2(self): self.base_dir = 'tests/fixtures/scale' result = self.dispatch(['scale', 'web=1'], returncode=1) @@ -1888,11 +1887,6 @@ class CLITestCase(DockerClientTestCase): self.base_dir = 'tests/fixtures/scale' project = self.project -======= - def test_up_scale(self): - self.base_dir = 'tests/fixtures/scale' - project = self.project ->>>>>>> Implement --scale option on up command, allow scale config in v2.2 format self.dispatch(['up', '-d']) assert len(project.get_service('web').containers()) == 2 assert len(project.get_service('db').containers()) == 1 @@ -1901,7 +1895,6 @@ class CLITestCase(DockerClientTestCase): assert len(project.get_service('web').containers()) == 1 assert len(project.get_service('db').containers()) == 1 -<<<<<<< 10267a83dc79ba0f8cebe17b561c05367b947247 def test_up_scale_reset(self): self.base_dir = 'tests/fixtures/scale' project = self.project @@ -1917,15 +1910,6 @@ class CLITestCase(DockerClientTestCase): def test_up_scale_to_zero(self): self.base_dir = 'tests/fixtures/scale' project = self.project -======= - self.dispatch(['up', '-d', '--scale', 'web=3']) - assert len(project.get_service('web').containers()) == 3 - assert len(project.get_service('db').containers()) == 1 - - self.dispatch(['up', '-d', '--scale', 'web=1', '--scale', 'db=2']) - assert len(project.get_service('web').containers()) == 1 - assert len(project.get_service('db').containers()) == 2 ->>>>>>> Implement --scale option on up command, allow scale config in v2.2 format self.dispatch(['up', '-d']) assert len(project.get_service('web').containers()) == 2 From d3ad2ae7fe97de3fbdc37f31bcd10076e2516edc Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Wed, 26 Apr 2017 15:00:41 -0700 Subject: [PATCH 004/244] Add deprecation warning to scale command Signed-off-by: Joffrey F --- compose/cli/main.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/compose/cli/main.py b/compose/cli/main.py index 9df3c82ad..37e299d94 100644 --- a/compose/cli/main.py +++ b/compose/cli/main.py @@ -764,6 +764,9 @@ class TopLevelCommand(object): $ docker-compose scale web=2 worker=3 + This command is deprecated. Use the up command with the `--scale` flag + instead. + Usage: scale [options] [SERVICE=NUM...] Options: @@ -777,6 +780,11 @@ class TopLevelCommand(object): 'The scale command is incompatible with the v2.2 format. ' 'Use the up command with the --scale flag instead.' ) + else: + log.warn( + 'The scale command is deprecated. ' + 'Use the up command with the --scale flag instead.' + ) for service_name, num in parse_scale_args(options['SERVICE=NUM']).items(): self.project.get_service(service_name).scale(num, timeout=timeout) From b1e3228d19e6bd50e84671cb58f817a048ecc299 Mon Sep 17 00:00:00 2001 From: mrfly Date: Fri, 28 Apr 2017 16:58:08 +0800 Subject: [PATCH 005/244] Not colon but a dot. hum... Signed-off-by: wrfly --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index d43bd8c4c..e3ca8f833 100644 --- a/README.md +++ b/README.md @@ -17,7 +17,7 @@ Using Compose is basically a three-step process. 1. Define your app's environment with a `Dockerfile` so it can be reproduced anywhere. 2. Define the services that make up your app in `docker-compose.yml` so -they can be run together in an isolated environment: +they can be run together in an isolated environment. 3. Lastly, run `docker-compose up` and Compose will start and run your entire app. A `docker-compose.yml` looks like this: From e27dfe8ccdb7e067c346ff05c75c4bbf8b6de05e Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Mon, 1 May 2017 16:48:16 -0700 Subject: [PATCH 006/244] Script downloading release binaries from bintray and appveyor Signed-off-by: Joffrey F --- script/release/download-binaries | 32 ++++++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) create mode 100755 script/release/download-binaries diff --git a/script/release/download-binaries b/script/release/download-binaries new file mode 100755 index 000000000..5d01f5f75 --- /dev/null +++ b/script/release/download-binaries @@ -0,0 +1,32 @@ +#!/bin/bash + +function usage() { + >&2 cat << EOM +Download Linux, Mac OS and Windows binaries from remote endpoints + +Usage: + + $0 + +Options: + + version version string for the release (ex: 1.6.0) + +EOM + exit 1 +} + + +[ -n "$1" ] || usage +VERSION=$1 +BASE_BINTRAY_URL=https://dl.bintray.com/docker-compose/bump-$VERSION/ +DESTINATION=binaries-$VERSION +APPVEYOR_URL=https://ci.appveyor.com/api/projects/docker/compose/\ +artifacts/dist%2Fdocker-compose-Windows-x86_64.exe?branch=bump-$VERSION + +mkdir $DESTINATION + + +wget -O $DESTINATION/docker-compose-Darwin-x86_64 $BASE_BINTRAY_URL/docker-compose-Darwin-x86_64 +wget -O $DESTINATION/docker-compose-Linux-x86_64 $BASE_BINTRAY_URL/docker-compose-Linux-x86_64 +wget -O $DESTINATION/docker-compose-Windows-x86_64.exe $APPVEYOR_URL From 57f647f03f847f74c3e0d00de56932ca8a00f21f Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Thu, 4 May 2017 12:47:44 -0700 Subject: [PATCH 007/244] 1.14.0dev Signed-off-by: Joffrey F --- compose/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/compose/__init__.py b/compose/__init__.py index 1f4c85725..69307d60e 100644 --- a/compose/__init__.py +++ b/compose/__init__.py @@ -1,4 +1,4 @@ from __future__ import absolute_import from __future__ import unicode_literals -__version__ = '1.13.0' +__version__ = '1.14.0dev' From 9c0bbaad36ec5f7196db15ee5d1c568009c760a8 Mon Sep 17 00:00:00 2001 From: Michael Friis Date: Thu, 4 May 2017 18:08:33 -0700 Subject: [PATCH 008/244] add exception for windows networking Signed-off-by: Michael Friis --- compose/network.py | 1 + 1 file changed, 1 insertion(+) diff --git a/compose/network.py b/compose/network.py index ea6f49631..532686d76 100644 --- a/compose/network.py +++ b/compose/network.py @@ -18,6 +18,7 @@ log = logging.getLogger(__name__) OPTS_EXCEPTIONS = [ 'com.docker.network.driver.overlay.vxlanid_list', + 'com.docker.network.windowsshim.hnsid' ] From 570cf951ac325ab80ddad000a855e36bee6b9fac Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Fri, 5 May 2017 11:40:50 -0700 Subject: [PATCH 009/244] New network config whitelist option in unit test Signed-off-by: Joffrey F --- tests/unit/network_test.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/unit/network_test.py b/tests/unit/network_test.py index d1cf2ccf9..4b40ea884 100644 --- a/tests/unit/network_test.py +++ b/tests/unit/network_test.py @@ -66,7 +66,8 @@ class NetworkTest(unittest.TestCase): options = {'com.docker.network.driver.foo': 'bar'} remote_options = { 'com.docker.network.driver.overlay.vxlanid_list': '257', - 'com.docker.network.driver.foo': 'bar' + 'com.docker.network.driver.foo': 'bar', + 'com.docker.network.windowsshim.hnsid': 'aac3fd4887daaec1e3b', } net = Network( None, 'compose_test', 'net1', 'overlay', From a5837ba358a8ba402f2e31f5223b6da754238bb7 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Fri, 5 May 2017 16:40:45 -0700 Subject: [PATCH 010/244] Use different method to compute ServicePort.repr Workaround for https://bugs.python.org/issue24931 Signed-off-by: Joffrey F --- compose/config/types.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/compose/config/types.py b/compose/config/types.py index dd61a8796..5d3bb5cb7 100644 --- a/compose/config/types.py +++ b/compose/config/types.py @@ -306,7 +306,7 @@ class ServicePort(namedtuple('_ServicePort', 'target published protocol mode ext def repr(self): return dict( - [(k, v) for k, v in self._asdict().items() if v is not None] + [(k, v) for k, v in zip(self._fields, self) if v is not None] ) def legacy_repr(self): From f1fd9eb1d0f783a54ad4ebb89da341c20b4fcf5e Mon Sep 17 00:00:00 2001 From: Victoria Bialas Date: Mon, 8 May 2017 16:31:19 -0700 Subject: [PATCH 011/244] Updated CLI help for docker-compose pull command removed reference to docker-stack.yml in pull command help referenced generic Compose file, consistent naming in Help, init caps Signed-off-by: Victoria Bialas --- compose/cli/main.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/compose/cli/main.py b/compose/cli/main.py index 37e299d94..5b65e5dd9 100644 --- a/compose/cli/main.py +++ b/compose/cli/main.py @@ -171,12 +171,12 @@ class TopLevelCommand(object): in the client certificate (for example if your docker host is an IP address) --project-directory PATH Specify an alternate working directory - (default: the path of the compose file) + (default: the path of the Compose file) Commands: build Build or rebuild services bundle Generate a Docker bundle from the Compose file - config Validate and view the compose file + config Validate and view the Compose file create Create services down Stop and remove containers, networks, images, and volumes events Receive real time events from containers @@ -273,7 +273,7 @@ class TopLevelCommand(object): def config(self, config_options, options): """ - Validate and view the compose file. + Validate and view the Compose file. Usage: config [options] @@ -627,7 +627,7 @@ class TopLevelCommand(object): def pull(self, options): """ - Pulls images for services. + Pulls images for services defined in a Compose file, but does not start the containers. Usage: pull [options] [SERVICE...] From 50437bd6eabdef4d58e76ea756e638e8ff9af7db Mon Sep 17 00:00:00 2001 From: Harald Albers Date: Thu, 11 May 2017 18:19:01 +0200 Subject: [PATCH 012/244] Add docker-compose exec -u to docs and completion Signed-off-by: Harald Albers --- compose/cli/main.py | 2 +- contrib/completion/bash/docker-compose | 4 ++-- contrib/completion/zsh/_docker-compose | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/compose/cli/main.py b/compose/cli/main.py index 5b65e5dd9..49800ba53 100644 --- a/compose/cli/main.py +++ b/compose/cli/main.py @@ -391,7 +391,7 @@ class TopLevelCommand(object): Options: -d Detached mode: Run command in the background. --privileged Give extended privileges to the process. - --user USER Run the command as this user. + -u, --user USER Run the command as this user. -T Disable pseudo-tty allocation. By default `docker-compose exec` allocates a TTY. --index=index index of the container if there are multiple diff --git a/contrib/completion/bash/docker-compose b/contrib/completion/bash/docker-compose index 2c2be61c7..57dfd51f5 100644 --- a/contrib/completion/bash/docker-compose +++ b/contrib/completion/bash/docker-compose @@ -224,14 +224,14 @@ _docker_compose_events() { _docker_compose_exec() { case "$prev" in - --index|--user) + --index|--user|-u) return ;; esac case "$cur" in -*) - COMPREPLY=( $( compgen -W "-d --help --index --privileged -T --user" -- "$cur" ) ) + COMPREPLY=( $( compgen -W "-d --help --index --privileged -T --user -u" -- "$cur" ) ) ;; *) __docker_compose_services_running diff --git a/contrib/completion/zsh/_docker-compose b/contrib/completion/zsh/_docker-compose index 8513884bc..f53f96334 100644 --- a/contrib/completion/zsh/_docker-compose +++ b/contrib/completion/zsh/_docker-compose @@ -241,7 +241,7 @@ __docker-compose_subcommand() { $opts_help \ '-d[Detached mode: Run command in the background.]' \ '--privileged[Give extended privileges to the process.]' \ - '--user=[Run the command as this user.]:username:_users' \ + '(-u --user)'{-u,--user=}'[Run the command as this user.]:username:_users' \ '-T[Disable pseudo-tty allocation. By default `docker-compose exec` allocates a TTY.]' \ '--index=[Index of the container if there are multiple instances of a service \[default: 1\]]:index: ' \ '(-):running services:__docker-compose_runningservices' \ From 9daced4c0433441bd36824b0af06ed0d31392158 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Fri, 12 May 2017 17:39:56 -0700 Subject: [PATCH 013/244] Prevent dependencies rescaling when executing `docker-compose run` Signed-off-by: Joffrey F --- compose/cli/main.py | 4 +++- compose/project.py | 6 ++++-- compose/service.py | 15 ++++++++++----- tests/acceptance/cli_test.py | 11 +++++++++++ 4 files changed, 28 insertions(+), 8 deletions(-) diff --git a/compose/cli/main.py b/compose/cli/main.py index 49800ba53..c91b8d898 100644 --- a/compose/cli/main.py +++ b/compose/cli/main.py @@ -1138,7 +1138,9 @@ def run_one_off_container(container_options, project, service, options): project.up( service_names=deps, start_deps=True, - strategy=ConvergenceStrategy.never) + strategy=ConvergenceStrategy.never, + rescale=False + ) project.initialize() diff --git a/compose/project.py b/compose/project.py index e80b10455..b282f718d 100644 --- a/compose/project.py +++ b/compose/project.py @@ -382,7 +382,8 @@ class Project(object): timeout=None, detached=False, remove_orphans=False, - scale_override=None): + scale_override=None, + rescale=True): warn_for_swarm_mode(self.client) @@ -405,7 +406,8 @@ class Project(object): plans[service.name], timeout=timeout, detached=detached, - scale_override=scale_override.get(service.name) + scale_override=scale_override.get(service.name), + rescale=rescale ) def get_deps(service): diff --git a/compose/service.py b/compose/service.py index 8699372ed..edd0a3764 100644 --- a/compose/service.py +++ b/compose/service.py @@ -390,7 +390,7 @@ class Service(object): return containers def _execute_convergence_recreate(self, containers, scale, timeout, detached, start): - if len(containers) > scale: + if scale is not None and len(containers) > scale: self._downscale(containers[scale:], timeout) containers = containers[:scale] @@ -408,14 +408,14 @@ class Service(object): for error in errors.values(): raise OperationFailedError(error) - if len(containers) < scale: + if scale is not None and len(containers) < scale: containers.extend(self._execute_convergence_create( scale - len(containers), detached, start )) return containers def _execute_convergence_start(self, containers, scale, timeout, detached, start): - if len(containers) > scale: + if scale is not None and len(containers) > scale: self._downscale(containers[scale:], timeout) containers = containers[:scale] if start: @@ -429,7 +429,7 @@ class Service(object): for error in errors.values(): raise OperationFailedError(error) - if len(containers) < scale: + if scale is not None and len(containers) < scale: containers.extend(self._execute_convergence_create( scale - len(containers), detached, start )) @@ -448,7 +448,7 @@ class Service(object): ) def execute_convergence_plan(self, plan, timeout=None, detached=False, - start=True, scale_override=None): + start=True, scale_override=None, rescale=True): (action, containers) = plan scale = scale_override if scale_override is not None else self.scale_num containers = sorted(containers, key=attrgetter('number')) @@ -460,6 +460,11 @@ class Service(object): scale, detached, start ) + # The create action needs always needs an initial scale, but otherwise, + # we set scale to none in no-rescale scenarios (`run` dependencies) + if not rescale: + scale = None + if action == 'recreate': return self._execute_convergence_recreate( containers, scale, timeout, detached, start diff --git a/tests/acceptance/cli_test.py b/tests/acceptance/cli_test.py index 75b15ae65..30eff1b6a 100644 --- a/tests/acceptance/cli_test.py +++ b/tests/acceptance/cli_test.py @@ -1211,6 +1211,17 @@ class CLITestCase(DockerClientTestCase): self.assertEqual(len(db.containers()), 1) self.assertEqual(len(console.containers()), 0) + def test_run_service_with_scaled_dependencies(self): + self.base_dir = 'tests/fixtures/v2-dependencies' + self.dispatch(['up', '-d', '--scale', 'db=2', '--scale', 'console=0']) + db = self.project.get_service('db') + console = self.project.get_service('console') + assert len(db.containers()) == 2 + assert len(console.containers()) == 0 + self.dispatch(['run', 'web', '/bin/true'], None) + assert len(db.containers()) == 2 + assert len(console.containers()) == 0 + def test_run_with_no_deps(self): self.base_dir = 'tests/fixtures/links-composefile' self.dispatch(['run', '--no-deps', 'web', '/bin/true']) From 511b981f11719f5deffbf9b25d829c02fb96d219 Mon Sep 17 00:00:00 2001 From: mengskysama Date: Mon, 15 May 2017 15:22:22 +0800 Subject: [PATCH 014/244] fix python3.x _asdict() return None Signed-off-by: mengskysama --- compose/config/types.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/compose/config/types.py b/compose/config/types.py index 5d3bb5cb7..d853d84f4 100644 --- a/compose/config/types.py +++ b/compose/config/types.py @@ -258,7 +258,7 @@ class ServiceSecret(namedtuple('_ServiceSecret', 'source target uid gid mode')): def repr(self): return dict( - [(k, v) for k, v in self._asdict().items() if v is not None] + [(k, v) for k, v in zip(self._fields, self) if v is not None] ) From 93d1ce5a55d6341fc0a63f518405cff73671026e Mon Sep 17 00:00:00 2001 From: Alexey Rokhin Date: Wed, 17 May 2017 14:50:29 +0300 Subject: [PATCH 015/244] Add cpu_count, cpu_percent, cpus parameters. Signed-off-by: Alexey Rokhin --- compose/config/config.py | 3 +++ compose/config/config_schema_v2.2.json | 3 +++ compose/service.py | 10 ++++++++++ setup.py | 2 +- tests/integration/service_test.py | 25 +++++++++++++++++++++++++ tests/integration/testcases.py | 5 ++++- tests/unit/config/config_test.py | 4 ++++ 7 files changed, 50 insertions(+), 2 deletions(-) diff --git a/compose/config/config.py b/compose/config/config.py index f1195c8ec..056847a85 100644 --- a/compose/config/config.py +++ b/compose/config/config.py @@ -52,8 +52,11 @@ DOCKER_CONFIG_KEYS = [ 'cap_drop', 'cgroup_parent', 'command', + 'cpu_count', + 'cpu_percent', 'cpu_quota', 'cpu_shares', + 'cpus', 'cpuset', 'detach', 'devices', diff --git a/compose/config/config_schema_v2.2.json b/compose/config/config_schema_v2.2.json index a178fccc4..bbf312c6a 100644 --- a/compose/config/config_schema_v2.2.json +++ b/compose/config/config_schema_v2.2.json @@ -74,8 +74,11 @@ ] }, "container_name": {"type": "string"}, + "cpu_count": {"type": "integer", "minimum": 0}, + "cpu_percent": {"type": "integer", "minimum": 0, "maximum": 100}, "cpu_shares": {"type": ["number", "string"]}, "cpu_quota": {"type": ["number", "string"]}, + "cpus": {"type": ["number", "string"], "minimum": 0}, "cpuset": {"type": "string"}, "depends_on": { "oneOf": [ diff --git a/compose/service.py b/compose/service.py index edd0a3764..dc653b165 100644 --- a/compose/service.py +++ b/compose/service.py @@ -52,7 +52,10 @@ HOST_CONFIG_KEYS = [ 'cap_add', 'cap_drop', 'cgroup_parent', + 'cpu_count', + 'cpu_percent', 'cpu_quota', + 'cpus', 'devices', 'dns', 'dns_search', @@ -798,6 +801,10 @@ class Service(object): init_path = options.get('init') options['init'] = True + nano_cpus = None + if options.has_key('cpus'): + nano_cpus = int(options.get('cpus') * 1000000000) + return self.client.create_host_config( links=self._get_links(link_to_self=one_off), port_bindings=build_port_bindings( @@ -837,6 +844,9 @@ class Service(object): init=options.get('init', None), init_path=init_path, isolation=options.get('isolation'), + cpu_count=options.get('cpu_count'), + cpu_percent=options.get('cpu_percent'), + nano_cpus=nano_cpus, ) def get_secret_volumes(self): diff --git a/setup.py b/setup.py index 19a0d4aa0..8dbb337cc 100644 --- a/setup.py +++ b/setup.py @@ -37,7 +37,7 @@ install_requires = [ 'requests >= 2.6.1, != 2.11.0, < 2.12', 'texttable >= 0.8.1, < 0.9', 'websocket-client >= 0.32.0, < 1.0', - 'docker >= 2.2.1, < 3.0', + 'docker >= 2.3.0, < 3.0', 'dockerpty >= 0.4.1, < 0.5', 'six >= 1.3.0, < 2', 'jsonschema >= 2.5.1, < 3', diff --git a/tests/integration/service_test.py b/tests/integration/service_test.py index 87549c506..4ee9c3d19 100644 --- a/tests/integration/service_test.py +++ b/tests/integration/service_test.py @@ -33,6 +33,7 @@ from compose.service import ConvergenceStrategy from compose.service import NetworkMode from compose.service import Service from tests.integration.testcases import v2_1_only +from tests.integration.testcases import v2_2_only from tests.integration.testcases import v2_only from tests.integration.testcases import v3_only @@ -110,6 +111,30 @@ class ServiceTest(DockerClientTestCase): container.start() self.assertEqual(container.get('HostConfig.CpuQuota'), 40000) + @v2_2_only() + def test_create_container_with_cpu_count(self): + self.require_api_version('1.25') + service = self.create_service('db', cpu_count=2) + container = service.create_container() + service.start_container(container) + self.assertEqual(container.get('HostConfig.CpuCount'), 2) + + @v2_2_only() + def test_create_container_with_cpu_percent(self): + self.require_api_version('1.25') + service = self.create_service('db', cpu_percent=12) + container = service.create_container() + service.start_container(container) + self.assertEqual(container.get('HostConfig.CpuPercent'), 12) + + @v2_2_only() + def test_create_container_with_cpus(self): + self.require_api_version('1.25') + service = self.create_service('db', cpus=1) + container = service.create_container() + service.start_container(container) + self.assertEqual(container.get('HostConfig.NanoCpus'), 1000000000) + def test_create_container_with_shm_size(self): self.require_api_version('1.22') service = self.create_service('db', shm_size=67108864) diff --git a/tests/integration/testcases.py b/tests/integration/testcases.py index a5fe999d9..1bed6e8ff 100644 --- a/tests/integration/testcases.py +++ b/tests/integration/testcases.py @@ -15,6 +15,7 @@ from compose.const import API_VERSIONS from compose.const import COMPOSEFILE_V1 as V1 from compose.const import COMPOSEFILE_V2_0 as V2_0 from compose.const import COMPOSEFILE_V2_0 as V2_1 +from compose.const import COMPOSEFILE_V2_2 as V2_2 from compose.const import COMPOSEFILE_V3_2 as V3_2 from compose.const import LABEL_PROJECT from compose.progress_stream import stream_output @@ -69,9 +70,11 @@ def v2_only(): def v2_1_only(): return build_version_required_decorator((V1, V2_0)) +def v2_2_only(): + return build_version_required_decorator((V1, V2_0, V2_1)) def v3_only(): - return build_version_required_decorator((V1, V2_0, V2_1)) + return build_version_required_decorator((V1, V2_0, V2_1, V2_2)) class DockerClientTestCase(unittest.TestCase): diff --git a/tests/unit/config/config_test.py b/tests/unit/config/config_test.py index d3087fffe..e66e952f8 100644 --- a/tests/unit/config/config_test.py +++ b/tests/unit/config/config_test.py @@ -27,6 +27,7 @@ from compose.config.types import VolumeSpec from compose.const import COMPOSEFILE_V1 as V1 from compose.const import COMPOSEFILE_V2_0 as V2_0 from compose.const import COMPOSEFILE_V2_1 as V2_1 +from compose.const import COMPOSEFILE_V2_2 as V2_2 from compose.const import COMPOSEFILE_V3_0 as V3_0 from compose.const import COMPOSEFILE_V3_1 as V3_1 from compose.const import COMPOSEFILE_V3_2 as V3_2 @@ -174,6 +175,9 @@ class ConfigTest(unittest.TestCase): cfg = config.load(build_config_details({'version': '2.1'})) assert cfg.version == V2_1 + cfg = config.load(build_config_details({'version': '2.2'})) + assert cfg.version == V2_2 + for version in ['3', '3.0']: cfg = config.load(build_config_details({'version': version})) assert cfg.version == V3_0 From 2d4fc2cd512758b68500bc0327c44be0ee1f7fb3 Mon Sep 17 00:00:00 2001 From: Alexey Rokhin Date: Wed, 17 May 2017 15:10:44 +0300 Subject: [PATCH 016/244] Fix cpu option checking. Signed-off-by: Alexey Rokhin --- compose/service.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/compose/service.py b/compose/service.py index dc653b165..3956a4782 100644 --- a/compose/service.py +++ b/compose/service.py @@ -802,7 +802,7 @@ class Service(object): options['init'] = True nano_cpus = None - if options.has_key('cpus'): + if 'cpus' in options: nano_cpus = int(options.get('cpus') * 1000000000) return self.client.create_host_config( From e621117ab23c9f7bfc24c8a1e68ef056ac9d69b1 Mon Sep 17 00:00:00 2001 From: Alexey Rokhin Date: Wed, 17 May 2017 15:21:35 +0300 Subject: [PATCH 017/244] Fix testcases.py formatting Signed-off-by: Alexey Rokhin --- tests/integration/testcases.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/integration/testcases.py b/tests/integration/testcases.py index 1bed6e8ff..57814872c 100644 --- a/tests/integration/testcases.py +++ b/tests/integration/testcases.py @@ -70,9 +70,11 @@ def v2_only(): def v2_1_only(): return build_version_required_decorator((V1, V2_0)) + def v2_2_only(): return build_version_required_decorator((V1, V2_0, V2_1)) + def v3_only(): return build_version_required_decorator((V1, V2_0, V2_1, V2_2)) From 56f63c858609e4b0a5a1f5604a7d15eaa02b8071 Mon Sep 17 00:00:00 2001 From: Alexey Rokhin Date: Wed, 17 May 2017 16:18:28 +0300 Subject: [PATCH 018/244] skip cpu_percent test for Linux Signed-off-by: Alexey Rokhin --- tests/integration/service_test.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/integration/service_test.py b/tests/integration/service_test.py index 4ee9c3d19..a1a7497a6 100644 --- a/tests/integration/service_test.py +++ b/tests/integration/service_test.py @@ -25,6 +25,7 @@ from compose.const import LABEL_ONE_OFF from compose.const import LABEL_PROJECT from compose.const import LABEL_SERVICE from compose.const import LABEL_VERSION +from compose.const import IS_WINDOWS_PLATFORM from compose.container import Container from compose.errors import OperationFailedError from compose.project import OneOffFilter @@ -120,6 +121,7 @@ class ServiceTest(DockerClientTestCase): self.assertEqual(container.get('HostConfig.CpuCount'), 2) @v2_2_only() + @pytest.mark.skipif(not IS_WINDOWS_PLATFORM, reason='cpu_percent is not supported for Linux') def test_create_container_with_cpu_percent(self): self.require_api_version('1.25') service = self.create_service('db', cpu_percent=12) From aeeed0cf2fee5e9dc2150b968fa5949d7a468180 Mon Sep 17 00:00:00 2001 From: Alexey Rokhin Date: Wed, 17 May 2017 16:42:43 +0300 Subject: [PATCH 019/244] service_test.py reorder imports Signed-off-by: Alexey Rokhin --- tests/integration/service_test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/integration/service_test.py b/tests/integration/service_test.py index a1a7497a6..a5b5bda57 100644 --- a/tests/integration/service_test.py +++ b/tests/integration/service_test.py @@ -19,13 +19,13 @@ from .testcases import pull_busybox from compose import __version__ from compose.config.types import VolumeFromSpec from compose.config.types import VolumeSpec +from compose.const import IS_WINDOWS_PLATFORM from compose.const import LABEL_CONFIG_HASH from compose.const import LABEL_CONTAINER_NUMBER from compose.const import LABEL_ONE_OFF from compose.const import LABEL_PROJECT from compose.const import LABEL_SERVICE from compose.const import LABEL_VERSION -from compose.const import IS_WINDOWS_PLATFORM from compose.container import Container from compose.errors import OperationFailedError from compose.project import OneOffFilter From b815a00e33166f6bd3b014d4a8a1a1a3b3a367b0 Mon Sep 17 00:00:00 2001 From: Alexey Rokhin Date: Wed, 17 May 2017 23:21:47 +0300 Subject: [PATCH 020/244] Implement review suggestions. Signed-off-by: Alexey Rokhin --- compose/config/config_schema_v2.2.json | 2 +- compose/service.py | 5 ++++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/compose/config/config_schema_v2.2.json b/compose/config/config_schema_v2.2.json index bbf312c6a..a585f2a8c 100644 --- a/compose/config/config_schema_v2.2.json +++ b/compose/config/config_schema_v2.2.json @@ -78,7 +78,7 @@ "cpu_percent": {"type": "integer", "minimum": 0, "maximum": 100}, "cpu_shares": {"type": ["number", "string"]}, "cpu_quota": {"type": ["number", "string"]}, - "cpus": {"type": ["number", "string"], "minimum": 0}, + "cpus": {"type": "number", "minimum": 0}, "cpuset": {"type": "string"}, "depends_on": { "oneOf": [ diff --git a/compose/service.py b/compose/service.py index 3956a4782..515992ad4 100644 --- a/compose/service.py +++ b/compose/service.py @@ -803,7 +803,10 @@ class Service(object): nano_cpus = None if 'cpus' in options: - nano_cpus = int(options.get('cpus') * 1000000000) + nano_cpus = options.get('cpus') * 1000000000 + if isinstance(nano_cpus, float) and not nano_cpus.is_integer(): + raise ValueError("cpus is too precise") + nano_cpus = int(nano_cpus) return self.client.create_host_config( links=self._get_links(link_to_self=one_off), From 201919824f5afd0f73c9d787aec23f4e7004bb0e Mon Sep 17 00:00:00 2001 From: Alexey Rokhin Date: Thu, 18 May 2017 01:43:04 +0300 Subject: [PATCH 021/244] move cpus validation to validation.py Signed-off-by: Alexey Rokhin --- compose/config/config.py | 2 ++ compose/config/validation.py | 11 +++++++++++ compose/const.py | 1 + compose/service.py | 6 ++---- 4 files changed, 16 insertions(+), 4 deletions(-) diff --git a/compose/config/config.py b/compose/config/config.py index 056847a85..4fddac822 100644 --- a/compose/config/config.py +++ b/compose/config/config.py @@ -38,6 +38,7 @@ from .types import VolumeSpec from .validation import match_named_volumes from .validation import validate_against_config_schema from .validation import validate_config_section +from .validation import validate_cpu from .validation import validate_depends_on from .validation import validate_extends_file_path from .validation import validate_links @@ -643,6 +644,7 @@ def validate_service(service_config, service_names, config_file): validate_service_constraints(service_dict, service_name, config_file) validate_paths(service_dict) + validate_cpu(service_config) validate_ulimits(service_config) validate_network_mode(service_config, service_names) validate_depends_on(service_config, service_names) diff --git a/compose/config/validation.py b/compose/config/validation.py index 1df6dd6b7..856f811c5 100644 --- a/compose/config/validation.py +++ b/compose/config/validation.py @@ -15,6 +15,7 @@ from jsonschema import RefResolver from jsonschema import ValidationError from ..const import COMPOSEFILE_V1 as V1 +from ..const import NANOCPUS_SCALE from .errors import ConfigurationError from .errors import VERSION_EXPLANATION from .sort_services import get_service_name_from_network_mode @@ -387,6 +388,16 @@ def validate_service_constraints(config, service_name, config_file): handle_errors(validator.iter_errors(config), handler, None) +def validate_cpu(service_config): + cpus = service_config.config.get('cpus') + if not cpus: + return + nano_cpus = cpus * NANOCPUS_SCALE + if isinstance(nano_cpus, float) and not nano_cpus.is_integer(): + raise ConfigurationError( + "cpus must have nine or less digits after decimal point") + + def get_schema_path(): return os.path.dirname(os.path.abspath(__file__)) diff --git a/compose/const.py b/compose/const.py index 573136d5d..36703138a 100644 --- a/compose/const.py +++ b/compose/const.py @@ -15,6 +15,7 @@ LABEL_NETWORK = 'com.docker.compose.network' LABEL_VERSION = 'com.docker.compose.version' LABEL_VOLUME = 'com.docker.compose.volume' LABEL_CONFIG_HASH = 'com.docker.compose.config-hash' +NANOCPUS_SCALE = 1000000000 SECRETS_PATH = '/run/secrets' diff --git a/compose/service.py b/compose/service.py index 515992ad4..19873d5e5 100644 --- a/compose/service.py +++ b/compose/service.py @@ -34,6 +34,7 @@ from .const import LABEL_ONE_OFF from .const import LABEL_PROJECT from .const import LABEL_SERVICE from .const import LABEL_VERSION +from .const import NANOCPUS_SCALE from .container import Container from .errors import HealthCheckFailed from .errors import NoHealthCheckConfigured @@ -803,10 +804,7 @@ class Service(object): nano_cpus = None if 'cpus' in options: - nano_cpus = options.get('cpus') * 1000000000 - if isinstance(nano_cpus, float) and not nano_cpus.is_integer(): - raise ValueError("cpus is too precise") - nano_cpus = int(nano_cpus) + nano_cpus = int(options.get('cpus') * NANOCPUS_SCALE) return self.client.create_host_config( links=self._get_links(link_to_self=one_off), From d10d64ac82ba46e0cb237e77d2ef846383afe09a Mon Sep 17 00:00:00 2001 From: Colin Hebert Date: Thu, 13 Apr 2017 21:51:41 +1000 Subject: [PATCH 022/244] Add support for labels during build Signed-off-by: Colin Hebert --- compose/config/config_schema_v3.2.json | 1 + compose/service.py | 1 + 2 files changed, 2 insertions(+) diff --git a/compose/config/config_schema_v3.2.json b/compose/config/config_schema_v3.2.json index ea702fcd5..70ff6ce05 100644 --- a/compose/config/config_schema_v3.2.json +++ b/compose/config/config_schema_v3.2.json @@ -72,6 +72,7 @@ "context": {"type": "string"}, "dockerfile": {"type": "string"}, "args": {"$ref": "#/definitions/list_or_dict"}, + "labels": {"$ref": "#/definitions/list_or_dict"}, "cache_from": {"$ref": "#/definitions/list_of_strings"} }, "additionalProperties": false diff --git a/compose/service.py b/compose/service.py index 19873d5e5..dcbbe251e 100644 --- a/compose/service.py +++ b/compose/service.py @@ -884,6 +884,7 @@ class Service(object): nocache=no_cache, dockerfile=build_opts.get('dockerfile', None), cache_from=build_opts.get('cache_from', None), + labels=build_opts.get('labels', None), buildargs=build_args ) From 3f920d515da065f1c19ab53e7eedd2d24b9e9bdc Mon Sep 17 00:00:00 2001 From: Colin Hebert Date: Thu, 13 Apr 2017 22:21:33 +1000 Subject: [PATCH 023/244] Update tests to show labels set to None Signed-off-by: Colin Hebert --- tests/unit/service_test.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/unit/service_test.py b/tests/unit/service_test.py index c32c36339..7b7a078f8 100644 --- a/tests/unit/service_test.py +++ b/tests/unit/service_test.py @@ -471,6 +471,7 @@ class ServiceTest(unittest.TestCase): nocache=False, rm=True, buildargs={}, + labels=None, cache_from=None, ) @@ -508,6 +509,7 @@ class ServiceTest(unittest.TestCase): nocache=False, rm=True, buildargs={}, + labels=None, cache_from=None, ) From 67e48ae4cbd649229c4c1bc81685987abfbff97f Mon Sep 17 00:00:00 2001 From: Colin Hebert Date: Thu, 13 Apr 2017 22:35:21 +1000 Subject: [PATCH 024/244] Add tests for the labels Signed-off-by: Colin Hebert --- tests/unit/config/config_test.py | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/tests/unit/config/config_test.py b/tests/unit/config/config_test.py index e66e952f8..3d42b8392 100644 --- a/tests/unit/config/config_test.py +++ b/tests/unit/config/config_test.py @@ -825,6 +825,34 @@ class ConfigTest(unittest.TestCase): assert service['build']['args']['opt1'] == '42' assert service['build']['args']['opt2'] == 'foobar' + def test_load_with_labels(self): + service = config.load( + build_config_details( + { + 'version': '3.2', + 'services': { + 'web': { + 'build': { + 'context': '.', + 'dockerfile': 'Dockerfile-alt', + 'labels': { + 'label1': 42, + 'label2': 'foobar' + } + } + } + } + }, + 'tests/fixtures/extends', + 'filename.yml' + ) + ).services[0] + assert 'labels' in service['build'] + assert 'label1' in service['build']['labels'] + assert isinstance(service['build']['labels']['label1'], str) + assert service['build']['labels']['label1'] == '42' + assert service['build']['labels']['label2'] == 'foobar' + def test_build_args_allow_empty_properties(self): service = config.load( build_config_details( From 2182329dae52e17984c31b91ab4ab92b7087b273 Mon Sep 17 00:00:00 2001 From: Colin Hebert Date: Thu, 13 Apr 2017 22:40:07 +1000 Subject: [PATCH 025/244] Fix test type Signed-off-by: Colin Hebert --- tests/unit/config/config_test.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tests/unit/config/config_test.py b/tests/unit/config/config_test.py index 3d42b8392..bc5160035 100644 --- a/tests/unit/config/config_test.py +++ b/tests/unit/config/config_test.py @@ -849,8 +849,7 @@ class ConfigTest(unittest.TestCase): ).services[0] assert 'labels' in service['build'] assert 'label1' in service['build']['labels'] - assert isinstance(service['build']['labels']['label1'], str) - assert service['build']['labels']['label1'] == '42' + assert service['build']['labels']['label1'] == 42 assert service['build']['labels']['label2'] == 'foobar' def test_build_args_allow_empty_properties(self): From 2ffa67cf92cc0466f6aafbd59314a779bc6f4880 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Tue, 23 May 2017 11:47:59 -0700 Subject: [PATCH 026/244] Add 3.3 format support Remove build.labels field from 3.2 schema Signed-off-by: Joffrey F --- compose/config/config_schema_v3.2.json | 1 - compose/config/config_schema_v3.3.json | 534 +++++++++++++++++++++++++ compose/config/serialize.py | 5 +- compose/const.py | 3 + docker-compose.spec | 5 + tests/unit/config/config_test.py | 5 +- 6 files changed, 548 insertions(+), 5 deletions(-) create mode 100644 compose/config/config_schema_v3.3.json diff --git a/compose/config/config_schema_v3.2.json b/compose/config/config_schema_v3.2.json index 70ff6ce05..ea702fcd5 100644 --- a/compose/config/config_schema_v3.2.json +++ b/compose/config/config_schema_v3.2.json @@ -72,7 +72,6 @@ "context": {"type": "string"}, "dockerfile": {"type": "string"}, "args": {"$ref": "#/definitions/list_or_dict"}, - "labels": {"$ref": "#/definitions/list_or_dict"}, "cache_from": {"$ref": "#/definitions/list_of_strings"} }, "additionalProperties": false diff --git a/compose/config/config_schema_v3.3.json b/compose/config/config_schema_v3.3.json new file mode 100644 index 000000000..e69116c38 --- /dev/null +++ b/compose/config/config_schema_v3.3.json @@ -0,0 +1,534 @@ +{ + "$schema": "http://json-schema.org/draft-04/schema#", + "id": "config_schema_v3.3.json", + "type": "object", + "required": ["version"], + + "properties": { + "version": { + "type": "string" + }, + + "services": { + "id": "#/properties/services", + "type": "object", + "patternProperties": { + "^[a-zA-Z0-9._-]+$": { + "$ref": "#/definitions/service" + } + }, + "additionalProperties": false + }, + + "networks": { + "id": "#/properties/networks", + "type": "object", + "patternProperties": { + "^[a-zA-Z0-9._-]+$": { + "$ref": "#/definitions/network" + } + } + }, + + "volumes": { + "id": "#/properties/volumes", + "type": "object", + "patternProperties": { + "^[a-zA-Z0-9._-]+$": { + "$ref": "#/definitions/volume" + } + }, + "additionalProperties": false + }, + + "secrets": { + "id": "#/properties/secrets", + "type": "object", + "patternProperties": { + "^[a-zA-Z0-9._-]+$": { + "$ref": "#/definitions/secret" + } + }, + "additionalProperties": false + }, + + "configs": { + "id": "#/properties/configs", + "type": "object", + "patternProperties": { + "^[a-zA-Z0-9._-]+$": { + "$ref": "#/definitions/config" + } + }, + "additionalProperties": false + } + }, + + "additionalProperties": false, + + "definitions": { + + "service": { + "id": "#/definitions/service", + "type": "object", + + "properties": { + "deploy": {"$ref": "#/definitions/deployment"}, + "build": { + "oneOf": [ + {"type": "string"}, + { + "type": "object", + "properties": { + "context": {"type": "string"}, + "dockerfile": {"type": "string"}, + "args": {"$ref": "#/definitions/list_or_dict"}, + "labels": {"$ref": "#/definitions/list_or_dict"}, + "cache_from": {"$ref": "#/definitions/list_of_strings"} + }, + "additionalProperties": false + } + ] + }, + "cap_add": {"type": "array", "items": {"type": "string"}, "uniqueItems": true}, + "cap_drop": {"type": "array", "items": {"type": "string"}, "uniqueItems": true}, + "cgroup_parent": {"type": "string"}, + "command": { + "oneOf": [ + {"type": "string"}, + {"type": "array", "items": {"type": "string"}} + ] + }, + "configs": { + "type": "array", + "items": { + "oneOf": [ + {"type": "string"}, + { + "type": "object", + "properties": { + "source": {"type": "string"}, + "target": {"type": "string"}, + "uid": {"type": "string"}, + "gid": {"type": "string"}, + "mode": {"type": "number"} + } + } + ] + } + }, + "container_name": {"type": "string"}, + "credential_spec": {"type": "object", "properties": { + "file": {"type": "string"}, + "registry": {"type": "string"} + }}, + "depends_on": {"$ref": "#/definitions/list_of_strings"}, + "devices": {"type": "array", "items": {"type": "string"}, "uniqueItems": true}, + "dns": {"$ref": "#/definitions/string_or_list"}, + "dns_search": {"$ref": "#/definitions/string_or_list"}, + "domainname": {"type": "string"}, + "entrypoint": { + "oneOf": [ + {"type": "string"}, + {"type": "array", "items": {"type": "string"}} + ] + }, + "env_file": {"$ref": "#/definitions/string_or_list"}, + "environment": {"$ref": "#/definitions/list_or_dict"}, + + "expose": { + "type": "array", + "items": { + "type": ["string", "number"], + "format": "expose" + }, + "uniqueItems": true + }, + + "external_links": {"type": "array", "items": {"type": "string"}, "uniqueItems": true}, + "extra_hosts": {"$ref": "#/definitions/list_or_dict"}, + "healthcheck": {"$ref": "#/definitions/healthcheck"}, + "hostname": {"type": "string"}, + "image": {"type": "string"}, + "ipc": {"type": "string"}, + "labels": {"$ref": "#/definitions/list_or_dict"}, + "links": {"type": "array", "items": {"type": "string"}, "uniqueItems": true}, + + "logging": { + "type": "object", + + "properties": { + "driver": {"type": "string"}, + "options": { + "type": "object", + "patternProperties": { + "^.+$": {"type": ["string", "number", "null"]} + } + } + }, + "additionalProperties": false + }, + + "mac_address": {"type": "string"}, + "network_mode": {"type": "string"}, + + "networks": { + "oneOf": [ + {"$ref": "#/definitions/list_of_strings"}, + { + "type": "object", + "patternProperties": { + "^[a-zA-Z0-9._-]+$": { + "oneOf": [ + { + "type": "object", + "properties": { + "aliases": {"$ref": "#/definitions/list_of_strings"}, + "ipv4_address": {"type": "string"}, + "ipv6_address": {"type": "string"} + }, + "additionalProperties": false + }, + {"type": "null"} + ] + } + }, + "additionalProperties": false + } + ] + }, + "pid": {"type": ["string", "null"]}, + + "ports": { + "type": "array", + "items": { + "oneOf": [ + {"type": "number", "format": "ports"}, + {"type": "string", "format": "ports"}, + { + "type": "object", + "properties": { + "mode": {"type": "string"}, + "target": {"type": "integer"}, + "published": {"type": "integer"}, + "protocol": {"type": "string"} + }, + "additionalProperties": false + } + ] + }, + "uniqueItems": true + }, + + "privileged": {"type": "boolean"}, + "read_only": {"type": "boolean"}, + "restart": {"type": "string"}, + "security_opt": {"type": "array", "items": {"type": "string"}, "uniqueItems": true}, + "shm_size": {"type": ["number", "string"]}, + "secrets": { + "type": "array", + "items": { + "oneOf": [ + {"type": "string"}, + { + "type": "object", + "properties": { + "source": {"type": "string"}, + "target": {"type": "string"}, + "uid": {"type": "string"}, + "gid": {"type": "string"}, + "mode": {"type": "number"} + } + } + ] + } + }, + "sysctls": {"$ref": "#/definitions/list_or_dict"}, + "stdin_open": {"type": "boolean"}, + "stop_grace_period": {"type": "string", "format": "duration"}, + "stop_signal": {"type": "string"}, + "tmpfs": {"$ref": "#/definitions/string_or_list"}, + "tty": {"type": "boolean"}, + "ulimits": { + "type": "object", + "patternProperties": { + "^[a-z]+$": { + "oneOf": [ + {"type": "integer"}, + { + "type":"object", + "properties": { + "hard": {"type": "integer"}, + "soft": {"type": "integer"} + }, + "required": ["soft", "hard"], + "additionalProperties": false + } + ] + } + } + }, + "user": {"type": "string"}, + "userns_mode": {"type": "string"}, + "volumes": { + "type": "array", + "items": { + "oneOf": [ + {"type": "string"}, + { + "type": "object", + "required": ["type"], + "properties": { + "type": {"type": "string"}, + "source": {"type": "string"}, + "target": {"type": "string"}, + "read_only": {"type": "boolean"}, + "consistency": {"type": "string"}, + "bind": { + "type": "object", + "properties": { + "propagation": {"type": "string"} + } + }, + "volume": { + "type": "object", + "properties": { + "nocopy": {"type": "boolean"} + } + } + } + } + ], + "uniqueItems": true + } + }, + "working_dir": {"type": "string"} + }, + "additionalProperties": false + }, + + "healthcheck": { + "id": "#/definitions/healthcheck", + "type": "object", + "additionalProperties": false, + "properties": { + "disable": {"type": "boolean"}, + "interval": {"type": "string"}, + "retries": {"type": "number"}, + "test": { + "oneOf": [ + {"type": "string"}, + {"type": "array", "items": {"type": "string"}} + ] + }, + "timeout": {"type": "string"} + } + }, + "deployment": { + "id": "#/definitions/deployment", + "type": ["object", "null"], + "properties": { + "mode": {"type": "string"}, + "endpoint_mode": {"type": "string"}, + "replicas": {"type": "integer"}, + "labels": {"$ref": "#/definitions/list_or_dict"}, + "update_config": { + "type": "object", + "properties": { + "parallelism": {"type": "integer"}, + "delay": {"type": "string", "format": "duration"}, + "failure_action": {"type": "string"}, + "monitor": {"type": "string", "format": "duration"}, + "max_failure_ratio": {"type": "number"} + }, + "additionalProperties": false + }, + "resources": { + "type": "object", + "properties": { + "limits": {"$ref": "#/definitions/resource"}, + "reservations": {"$ref": "#/definitions/resource"} + } + }, + "restart_policy": { + "type": "object", + "properties": { + "condition": {"type": "string"}, + "delay": {"type": "string", "format": "duration"}, + "max_attempts": {"type": "integer"}, + "window": {"type": "string", "format": "duration"} + }, + "additionalProperties": false + }, + "placement": { + "type": "object", + "properties": { + "constraints": {"type": "array", "items": {"type": "string"}}, + "preferences": { + "type": "array", + "items": { + "type": "object", + "properties": { + "spread": {"type": "string"} + }, + "additionalProperties": false + } + } + }, + "additionalProperties": false + } + }, + "additionalProperties": false + }, + + "resource": { + "id": "#/definitions/resource", + "type": "object", + "properties": { + "cpus": {"type": "string"}, + "memory": {"type": "string"} + }, + "additionalProperties": false + }, + + "network": { + "id": "#/definitions/network", + "type": ["object", "null"], + "properties": { + "driver": {"type": "string"}, + "driver_opts": { + "type": "object", + "patternProperties": { + "^.+$": {"type": ["string", "number"]} + } + }, + "ipam": { + "type": "object", + "properties": { + "driver": {"type": "string"}, + "config": { + "type": "array", + "items": { + "type": "object", + "properties": { + "subnet": {"type": "string"} + }, + "additionalProperties": false + } + } + }, + "additionalProperties": false + }, + "external": { + "type": ["boolean", "object"], + "properties": { + "name": {"type": "string"} + }, + "additionalProperties": false + }, + "internal": {"type": "boolean"}, + "attachable": {"type": "boolean"}, + "labels": {"$ref": "#/definitions/list_or_dict"} + }, + "additionalProperties": false + }, + + "volume": { + "id": "#/definitions/volume", + "type": ["object", "null"], + "properties": { + "driver": {"type": "string"}, + "driver_opts": { + "type": "object", + "patternProperties": { + "^.+$": {"type": ["string", "number"]} + } + }, + "external": { + "type": ["boolean", "object"], + "properties": { + "name": {"type": "string"} + }, + "additionalProperties": false + }, + "labels": {"$ref": "#/definitions/list_or_dict"} + }, + "additionalProperties": false + }, + + "secret": { + "id": "#/definitions/secret", + "type": "object", + "properties": { + "file": {"type": "string"}, + "external": { + "type": ["boolean", "object"], + "properties": { + "name": {"type": "string"} + } + }, + "labels": {"$ref": "#/definitions/list_or_dict"} + }, + "additionalProperties": false + }, + + "config": { + "id": "#/definitions/config", + "type": "object", + "properties": { + "file": {"type": "string"}, + "external": { + "type": ["boolean", "object"], + "properties": { + "name": {"type": "string"} + } + }, + "labels": {"$ref": "#/definitions/list_or_dict"} + }, + "additionalProperties": false + }, + + "string_or_list": { + "oneOf": [ + {"type": "string"}, + {"$ref": "#/definitions/list_of_strings"} + ] + }, + + "list_of_strings": { + "type": "array", + "items": {"type": "string"}, + "uniqueItems": true + }, + + "list_or_dict": { + "oneOf": [ + { + "type": "object", + "patternProperties": { + ".+": { + "type": ["string", "number", "null"] + } + }, + "additionalProperties": false + }, + {"type": "array", "items": {"type": "string"}, "uniqueItems": true} + ] + }, + + "constraints": { + "service": { + "id": "#/definitions/constraints/service", + "anyOf": [ + {"required": ["build"]}, + {"required": ["image"]} + ], + "properties": { + "build": { + "required": ["context"] + } + } + } + } + } +} diff --git a/compose/config/serialize.py b/compose/config/serialize.py index 040973ae0..ac78b77a2 100644 --- a/compose/config/serialize.py +++ b/compose/config/serialize.py @@ -10,6 +10,7 @@ from compose.const import COMPOSEFILE_V2_1 as V2_1 from compose.const import COMPOSEFILE_V2_2 as V2_2 from compose.const import COMPOSEFILE_V3_1 as V3_1 from compose.const import COMPOSEFILE_V3_2 as V3_2 +from compose.const import COMPOSEFILE_V3_3 as V3_3 def serialize_config_type(dumper, data): @@ -50,7 +51,7 @@ def denormalize_config(config, image_digests=None): if 'external_name' in vol_conf: del vol_conf['external_name'] - if config.version in (V3_1, V3_2): + if config.version in (V3_1, V3_2, V3_3): result['secrets'] = config.secrets.copy() for secret_name, secret_conf in result['secrets'].items(): if 'external_name' in secret_conf: @@ -114,7 +115,7 @@ def denormalize_service_dict(service_dict, version, image_digest=None): service_dict['healthcheck']['timeout'] ) - if 'ports' in service_dict and version not in (V3_2,): + if 'ports' in service_dict and version not in (V3_2, V3_3): service_dict['ports'] = [ p.legacy_repr() if isinstance(p, types.ServicePort) else p for p in service_dict['ports'] diff --git a/compose/const.py b/compose/const.py index 36703138a..36f213897 100644 --- a/compose/const.py +++ b/compose/const.py @@ -27,6 +27,7 @@ COMPOSEFILE_V2_2 = '2.2' COMPOSEFILE_V3_0 = '3.0' COMPOSEFILE_V3_1 = '3.1' COMPOSEFILE_V3_2 = '3.2' +COMPOSEFILE_V3_3 = '3.3' API_VERSIONS = { COMPOSEFILE_V1: '1.21', @@ -36,6 +37,7 @@ API_VERSIONS = { COMPOSEFILE_V3_0: '1.25', COMPOSEFILE_V3_1: '1.25', COMPOSEFILE_V3_2: '1.25', + COMPOSEFILE_V3_3: '1.30', } API_VERSION_TO_ENGINE_VERSION = { @@ -46,4 +48,5 @@ API_VERSION_TO_ENGINE_VERSION = { API_VERSIONS[COMPOSEFILE_V3_0]: '1.13.0', API_VERSIONS[COMPOSEFILE_V3_1]: '1.13.0', API_VERSIONS[COMPOSEFILE_V3_2]: '1.13.0', + API_VERSIONS[COMPOSEFILE_V3_3]: '17.06.0', } diff --git a/docker-compose.spec b/docker-compose.spec index 21b3c1742..8e0d51ae5 100644 --- a/docker-compose.spec +++ b/docker-compose.spec @@ -52,6 +52,11 @@ exe = EXE(pyz, 'compose/config/config_schema_v3.2.json', 'DATA' ), + ( + 'compose/config/config_schema_v3.3.json', + 'compose/config/config_schema_v3.3.json', + 'DATA' + ), ( 'compose/GITSHA', 'compose/GITSHA', diff --git a/tests/unit/config/config_test.py b/tests/unit/config/config_test.py index bc5160035..357244c2c 100644 --- a/tests/unit/config/config_test.py +++ b/tests/unit/config/config_test.py @@ -31,6 +31,7 @@ from compose.const import COMPOSEFILE_V2_2 as V2_2 from compose.const import COMPOSEFILE_V3_0 as V3_0 from compose.const import COMPOSEFILE_V3_1 as V3_1 from compose.const import COMPOSEFILE_V3_2 as V3_2 +from compose.const import COMPOSEFILE_V3_3 as V3_3 from compose.const import IS_WINDOWS_PLATFORM from compose.utils import nanoseconds_from_time_seconds from tests import mock @@ -825,11 +826,11 @@ class ConfigTest(unittest.TestCase): assert service['build']['args']['opt1'] == '42' assert service['build']['args']['opt2'] == 'foobar' - def test_load_with_labels(self): + def test_load_with_build_labels(self): service = config.load( build_config_details( { - 'version': '3.2', + 'version': V3_3, 'services': { 'web': { 'build': { From d0b80f537bee70e2cf312c091966ceabf7547436 Mon Sep 17 00:00:00 2001 From: Eli Atzaba Date: Sat, 29 Apr 2017 02:00:52 +0300 Subject: [PATCH 027/244] Fix for yaml extention does not work with override file Signed-off-by: Eli Atzaba --- compose/config/config.py | 9 ++++++--- tests/acceptance/cli_test.py | 16 ++++++++++++++++ .../docker-compose.override.yaml | 3 +++ .../override-yaml-files/docker-compose.yml | 10 ++++++++++ 4 files changed, 35 insertions(+), 3 deletions(-) create mode 100644 tests/fixtures/override-yaml-files/docker-compose.override.yaml create mode 100644 tests/fixtures/override-yaml-files/docker-compose.yml diff --git a/compose/config/config.py b/compose/config/config.py index 4fddac822..861a3e9bf 100644 --- a/compose/config/config.py +++ b/compose/config/config.py @@ -128,7 +128,7 @@ SUPPORTED_FILENAMES = [ 'docker-compose.yaml', ] -DEFAULT_OVERRIDE_FILENAME = 'docker-compose.override.yml' +DEFAULT_OVERRIDE_FILENAMES = ('docker-compose.override.yml', 'docker-compose.override.yaml') log = logging.getLogger(__name__) @@ -292,8 +292,11 @@ def get_default_config_files(base_dir): def get_default_override_file(path): - override_filename = os.path.join(path, DEFAULT_OVERRIDE_FILENAME) - return [override_filename] if os.path.exists(override_filename) else [] + for default_override_filename in DEFAULT_OVERRIDE_FILENAMES: + override_filename = os.path.join(path, default_override_filename) + if os.path.exists(override_filename): + return [override_filename] + return [] def find_candidates_in_parent_dirs(filenames, path): diff --git a/tests/acceptance/cli_test.py b/tests/acceptance/cli_test.py index 30eff1b6a..f6c074364 100644 --- a/tests/acceptance/cli_test.py +++ b/tests/acceptance/cli_test.py @@ -2149,3 +2149,19 @@ class CLITestCase(DockerClientTestCase): assert 'busybox' in result.stdout assert 'multiplecomposefiles_another_1' in result.stdout assert 'multiplecomposefiles_simple_1' in result.stdout + + def test_up_with_override_yaml(self): + self.base_dir = 'tests/fixtures/override-yaml-files' + self._project = get_project(self.base_dir, []) + self.dispatch( + [ + 'up', '-d', + ], + None) + + containers = self.project.containers() + self.assertEqual(len(containers), 2) + + web, db = containers + self.assertEqual(web.human_readable_command, 'sleep 100') + self.assertEqual(db.human_readable_command, 'top') diff --git a/tests/fixtures/override-yaml-files/docker-compose.override.yaml b/tests/fixtures/override-yaml-files/docker-compose.override.yaml new file mode 100644 index 000000000..58c673482 --- /dev/null +++ b/tests/fixtures/override-yaml-files/docker-compose.override.yaml @@ -0,0 +1,3 @@ + +db: + command: "top" diff --git a/tests/fixtures/override-yaml-files/docker-compose.yml b/tests/fixtures/override-yaml-files/docker-compose.yml new file mode 100644 index 000000000..5f2909d69 --- /dev/null +++ b/tests/fixtures/override-yaml-files/docker-compose.yml @@ -0,0 +1,10 @@ + +web: + image: busybox:latest + command: "sleep 100" + links: + - db + +db: + image: busybox:latest + command: "sleep 200" From 88fa8db79aade1af516ec7f99b9a902cc0696ee8 Mon Sep 17 00:00:00 2001 From: Eli Atzaba Date: Sun, 7 May 2017 18:03:14 +0300 Subject: [PATCH 028/244] Raise exception when override.yaml & override.yml coexist Signed-off-by: Eli Atzaba --- compose/config/config.py | 12 +++++++----- compose/config/errors.py | 12 ++++++++++++ tests/acceptance/cli_test.py | 8 +++++++- .../docker-compose.override.yaml | 3 +++ .../docker-compose.override.yml | 3 +++ .../duplicate-override-yaml-files/docker-compose.yml | 10 ++++++++++ 6 files changed, 42 insertions(+), 6 deletions(-) create mode 100644 tests/fixtures/duplicate-override-yaml-files/docker-compose.override.yaml create mode 100644 tests/fixtures/duplicate-override-yaml-files/docker-compose.override.yml create mode 100644 tests/fixtures/duplicate-override-yaml-files/docker-compose.yml diff --git a/compose/config/config.py b/compose/config/config.py index 861a3e9bf..2a81b93da 100644 --- a/compose/config/config.py +++ b/compose/config/config.py @@ -24,6 +24,7 @@ from .environment import split_env from .errors import CircularReference from .errors import ComposeFileNotFound from .errors import ConfigurationError +from .errors import DuplicateOverrideFileFound from .errors import VERSION_EXPLANATION from .interpolation import interpolate_environment_variables from .sort_services import get_container_name_from_network_mode @@ -292,11 +293,12 @@ def get_default_config_files(base_dir): def get_default_override_file(path): - for default_override_filename in DEFAULT_OVERRIDE_FILENAMES: - override_filename = os.path.join(path, default_override_filename) - if os.path.exists(override_filename): - return [override_filename] - return [] + override_files_in_path = [os.path.join(path, override_filename) for override_filename + in DEFAULT_OVERRIDE_FILENAMES + if os.path.exists(os.path.join(path, override_filename))] + if len(override_files_in_path) > 1: + raise DuplicateOverrideFileFound(override_files_in_path) + return override_files_in_path def find_candidates_in_parent_dirs(filenames, path): diff --git a/compose/config/errors.py b/compose/config/errors.py index 9b82df0ab..060564fc4 100644 --- a/compose/config/errors.py +++ b/compose/config/errors.py @@ -44,3 +44,15 @@ class ComposeFileNotFound(ConfigurationError): Supported filenames: %s """ % ", ".join(supported_filenames)) + + +class DuplicateOverrideFileFound(ConfigurationError): + def __init__(self, override_filenames): + self.override_filenames = override_filenames + + @property + def msg(self): + return """ + Unable to determine with duplicate override files, only a single override file can be used. + Found: %s + """ % ", ".join(self.override_filenames) diff --git a/tests/acceptance/cli_test.py b/tests/acceptance/cli_test.py index f6c074364..1ba64201f 100644 --- a/tests/acceptance/cli_test.py +++ b/tests/acceptance/cli_test.py @@ -21,6 +21,7 @@ from docker import errors from .. import mock from ..helpers import create_host_file from compose.cli.command import get_project +from compose.config.errors import DuplicateOverrideFileFound from compose.container import Container from compose.project import OneOffFilter from compose.utils import nanoseconds_from_time_seconds @@ -31,7 +32,6 @@ from tests.integration.testcases import v2_1_only from tests.integration.testcases import v2_only from tests.integration.testcases import v3_only - ProcessResult = namedtuple('ProcessResult', 'stdout stderr') @@ -2165,3 +2165,9 @@ class CLITestCase(DockerClientTestCase): web, db = containers self.assertEqual(web.human_readable_command, 'sleep 100') self.assertEqual(db.human_readable_command, 'top') + + def test_up_with_duplicate_override_yaml_files(self): + self.base_dir = 'tests/fixtures/duplicate-override-yaml-files' + with self.assertRaises(DuplicateOverrideFileFound): + get_project(self.base_dir, []) + self.base_dir = None diff --git a/tests/fixtures/duplicate-override-yaml-files/docker-compose.override.yaml b/tests/fixtures/duplicate-override-yaml-files/docker-compose.override.yaml new file mode 100644 index 000000000..58c673482 --- /dev/null +++ b/tests/fixtures/duplicate-override-yaml-files/docker-compose.override.yaml @@ -0,0 +1,3 @@ + +db: + command: "top" diff --git a/tests/fixtures/duplicate-override-yaml-files/docker-compose.override.yml b/tests/fixtures/duplicate-override-yaml-files/docker-compose.override.yml new file mode 100644 index 000000000..f1b8ef181 --- /dev/null +++ b/tests/fixtures/duplicate-override-yaml-files/docker-compose.override.yml @@ -0,0 +1,3 @@ + +db: + command: "sleep 300" diff --git a/tests/fixtures/duplicate-override-yaml-files/docker-compose.yml b/tests/fixtures/duplicate-override-yaml-files/docker-compose.yml new file mode 100644 index 000000000..5f2909d69 --- /dev/null +++ b/tests/fixtures/duplicate-override-yaml-files/docker-compose.yml @@ -0,0 +1,10 @@ + +web: + image: busybox:latest + command: "sleep 100" + links: + - db + +db: + image: busybox:latest + command: "sleep 200" From d2a8a9edaaf40542645ba341d04e944dcbd5f675 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Tue, 23 May 2017 12:14:32 -0700 Subject: [PATCH 029/244] Rewrite duplicate override error message Signed-off-by: Joffrey F --- compose/config/errors.py | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/compose/config/errors.py b/compose/config/errors.py index 060564fc4..ac1d3ac19 100644 --- a/compose/config/errors.py +++ b/compose/config/errors.py @@ -49,10 +49,7 @@ class ComposeFileNotFound(ConfigurationError): class DuplicateOverrideFileFound(ConfigurationError): def __init__(self, override_filenames): self.override_filenames = override_filenames - - @property - def msg(self): - return """ - Unable to determine with duplicate override files, only a single override file can be used. - Found: %s - """ % ", ".join(self.override_filenames) + super(DuplicateOverrideFileFound, self).__init__( + "Multiple override files found: {}. You may only use a single " + "override file.".format(", ".join(override_filenames)) + ) From c9ff9023b265644ee6053d8946e7eb39f47aa840 Mon Sep 17 00:00:00 2001 From: Pascal Vibet Date: Wed, 26 Apr 2017 13:50:22 +0200 Subject: [PATCH 030/244] If COMPOSE_FILE is define then set this variable to the container Signed-off-by: Pascal Vibet --- script/run/run.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/script/run/run.sh b/script/run/run.sh index e697d1f6d..d1e1fabaa 100755 --- a/script/run/run.sh +++ b/script/run/run.sh @@ -35,6 +35,7 @@ if [ "$(pwd)" != '/' ]; then VOLUMES="-v $(pwd):$(pwd)" fi if [ -n "$COMPOSE_FILE" ]; then + COMPOSE_OPTIONS="$COMPOSE_OPTIONS -e COMPOSE_FILE=$COMPOSE_FILE" compose_dir=$(realpath $(dirname $COMPOSE_FILE)) fi # TODO: also check --file argument From d29ed0d3e49ccaa59d401a3fc29d4d599fb60fc1 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Tue, 23 May 2017 12:38:54 -0700 Subject: [PATCH 031/244] Fix improper use of project.stop Add some better test coverage for rm --stop Signed-off-by: Joffrey F --- compose/cli/main.py | 8 +------- tests/acceptance/cli_test.py | 18 +++++++++++++++++- 2 files changed, 18 insertions(+), 8 deletions(-) diff --git a/compose/cli/main.py b/compose/cli/main.py index c91b8d898..cfca0f949 100644 --- a/compose/cli/main.py +++ b/compose/cli/main.py @@ -680,13 +680,7 @@ class TopLevelCommand(object): one_off = OneOffFilter.include if options.get('--stop'): - running_containers = self.project.containers( - service_names=options['SERVICE'], stopped=False, one_off=one_off - ) - self.project.stop( - service_names=running_containers, - one_off=one_off - ) + self.project.stop(service_names=options['SERVICE'], one_off=one_off) all_containers = self.project.containers( service_names=options['SERVICE'], stopped=True, one_off=one_off diff --git a/tests/acceptance/cli_test.py b/tests/acceptance/cli_test.py index 1ba64201f..89f4f288b 100644 --- a/tests/acceptance/cli_test.py +++ b/tests/acceptance/cli_test.py @@ -1627,8 +1627,24 @@ class CLITestCase(DockerClientTestCase): service = self.project.get_service('simple') service.create_container() self.dispatch(['rm', '-fs'], None) + self.assertEqual(len(service.containers(stopped=True)), 0) + + def test_rm_stop(self): + self.dispatch(['up', '-d'], None) simple = self.project.get_service('simple') - self.assertEqual(len(simple.containers()), 0) + another = self.project.get_service('another') + assert len(simple.containers()) == 1 + assert len(another.containers()) == 1 + self.dispatch(['rm', '-fs'], None) + assert len(simple.containers(stopped=True)) == 0 + assert len(another.containers(stopped=True)) == 0 + + self.dispatch(['up', '-d'], None) + assert len(simple.containers()) == 1 + assert len(another.containers()) == 1 + self.dispatch(['rm', '-fs', 'another'], None) + assert len(simple.containers()) == 1 + assert len(another.containers(stopped=True)) == 0 def test_rm_all(self): service = self.project.get_service('simple') From 150c44dc364dcb24b3b6b8256e7ec0fce95225fb Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Tue, 23 May 2017 15:17:16 -0700 Subject: [PATCH 032/244] Merge all fields inside build dict Signed-off-by: Joffrey F --- compose/config/config.py | 2 + tests/unit/config/config_test.py | 68 ++++++++++++++++++++++++++++++++ 2 files changed, 70 insertions(+) diff --git a/compose/config/config.py b/compose/config/config.py index 2a81b93da..8dac4fb33 100644 --- a/compose/config/config.py +++ b/compose/config/config.py @@ -952,6 +952,8 @@ def merge_build(output, base, override): md.merge_scalar('context') md.merge_scalar('dockerfile') md.merge_mapping('args', parse_build_arguments) + md.merge_field('cache_from', merge_unique_items_lists, default=[]) + md.merge_mapping('labels', parse_labels) return dict(md) diff --git a/tests/unit/config/config_test.py b/tests/unit/config/config_test.py index 357244c2c..d8973484d 100644 --- a/tests/unit/config/config_test.py +++ b/tests/unit/config/config_test.py @@ -2847,6 +2847,74 @@ class MergeLabelsTest(unittest.TestCase): assert service_dict['labels'] == {'foo': '1', 'bar': ''} +class MergeBuildTest(unittest.TestCase): + def test_full(self): + base = { + 'context': '.', + 'dockerfile': 'Dockerfile', + 'args': { + 'x': '1', + 'y': '2', + }, + 'cache_from': ['ubuntu'], + 'labels': ['com.docker.compose.test=true'] + } + + override = { + 'context': './prod', + 'dockerfile': 'Dockerfile.prod', + 'args': ['x=12'], + 'cache_from': ['debian'], + 'labels': { + 'com.docker.compose.test': 'false', + 'com.docker.compose.prod': 'true', + } + } + + result = config.merge_build(None, {'build': base}, {'build': override}) + assert result['context'] == override['context'] + assert result['dockerfile'] == override['dockerfile'] + assert result['args'] == {'x': '12', 'y': '2'} + assert set(result['cache_from']) == set(['ubuntu', 'debian']) + assert result['labels'] == override['labels'] + + def test_empty_override(self): + base = { + 'context': '.', + 'dockerfile': 'Dockerfile', + 'args': { + 'x': '1', + 'y': '2', + }, + 'cache_from': ['ubuntu'], + 'labels': { + 'com.docker.compose.test': 'true' + } + } + + override = {} + + result = config.merge_build(None, {'build': base}, {'build': override}) + assert result == base + + def test_empty_base(self): + base = {} + + override = { + 'context': './prod', + 'dockerfile': 'Dockerfile.prod', + 'args': {'x': '12'}, + 'cache_from': ['debian'], + 'labels': { + 'com.docker.compose.test': 'false', + 'com.docker.compose.prod': 'true', + } + } + + result = config.merge_build(None, {'build': base}, {'build': override}) + assert result == override + + class MemoryOptionsTest(unittest.TestCase): def test_validation_fails_with_just_memswap_limit(self): From f6aa53ea6c8d9444d2537a3905783d67d2432e6e Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Mon, 22 May 2017 14:52:57 -0700 Subject: [PATCH 033/244] Network label mismatch now prints a warning instead of raising an error Signed-off-by: Joffrey F --- compose/network.py | 7 +++++-- tests/unit/network_test.py | 7 +++++-- 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/compose/network.py b/compose/network.py index 532686d76..fec839162 100644 --- a/compose/network.py +++ b/compose/network.py @@ -188,10 +188,13 @@ def check_remote_network_config(remote, local): local_labels = local.labels or {} remote_labels = remote.get('Labels', {}) for k in set.union(set(remote_labels.keys()), set(local_labels.keys())): - if k.startswith('com.docker.compose.'): # We are only interested in user-specified labels + if k.startswith('com.docker.'): # We are only interested in user-specified labels continue if remote_labels.get(k) != local_labels.get(k): - raise NetworkConfigChangedError(local.full_name, 'label "{}"'.format(k)) + log.warn( + 'Network {}: label "{}" has changed. It may need to be' + ' recreated.'.format(local.full_name, k) + ) def build_networks(name, config_data, client): diff --git a/tests/unit/network_test.py b/tests/unit/network_test.py index 4b40ea884..b27339af8 100644 --- a/tests/unit/network_test.py +++ b/tests/unit/network_test.py @@ -3,6 +3,7 @@ from __future__ import unicode_literals import pytest +from .. import mock from .. import unittest from compose.network import check_remote_network_config from compose.network import Network @@ -152,7 +153,9 @@ class NetworkTest(unittest.TestCase): 'com.project.touhou.character': 'marisa.kirisame', } } - with pytest.raises(NetworkConfigChangedError) as e: + with mock.patch('compose.network.log') as mock_log: check_remote_network_config(remote, net) - assert 'label "com.project.touhou.character" has changed' in str(e.value) + mock_log.warn.assert_called_once_with(mock.ANY) + _, args, kwargs = mock_log.warn.mock_calls[0] + assert 'label "com.project.touhou.character" has changed' in args[0] From 5fb767505554852ab396a82db3eb17c983089b91 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Tue, 23 May 2017 15:53:06 -0700 Subject: [PATCH 034/244] Add support for build labels in 2.1 and 2.2 format Add cache_from in 2.2 format Add integration test for build labels Signed-off-by: Joffrey F --- compose/config/config_schema_v2.1.json | 3 ++- compose/config/config_schema_v2.2.json | 4 +++- tests/integration/service_test.py | 15 +++++++++++++++ 3 files changed, 20 insertions(+), 2 deletions(-) diff --git a/compose/config/config_schema_v2.1.json b/compose/config/config_schema_v2.1.json index aa59d181e..9004000ea 100644 --- a/compose/config/config_schema_v2.1.json +++ b/compose/config/config_schema_v2.1.json @@ -58,7 +58,8 @@ "properties": { "context": {"type": "string"}, "dockerfile": {"type": "string"}, - "args": {"$ref": "#/definitions/list_or_dict"} + "args": {"$ref": "#/definitions/list_or_dict"}, + "labels": {"$ref": "#/definitions/list_or_dict"} }, "additionalProperties": false } diff --git a/compose/config/config_schema_v2.2.json b/compose/config/config_schema_v2.2.json index a585f2a8c..e8edb60ed 100644 --- a/compose/config/config_schema_v2.2.json +++ b/compose/config/config_schema_v2.2.json @@ -58,7 +58,9 @@ "properties": { "context": {"type": "string"}, "dockerfile": {"type": "string"}, - "args": {"$ref": "#/definitions/list_or_dict"} + "args": {"$ref": "#/definitions/list_or_dict"}, + "labels": {"$ref": "#/definitions/list_or_dict"}, + "cache_from": {"$ref": "#/definitions/list_of_strings"} }, "additionalProperties": false } diff --git a/tests/integration/service_test.py b/tests/integration/service_test.py index a5b5bda57..178df1323 100644 --- a/tests/integration/service_test.py +++ b/tests/integration/service_test.py @@ -666,6 +666,21 @@ class ServiceTest(DockerClientTestCase): assert service.image() assert "build_version=2" in service.image()['ContainerConfig']['Cmd'] + def test_build_with_build_labels(self): + base_dir = tempfile.mkdtemp() + self.addCleanup(shutil.rmtree, base_dir) + + with open(os.path.join(base_dir, 'Dockerfile'), 'w') as f: + f.write('FROM busybox\n') + + service = self.create_service('buildlabels', build={ + 'context': text_type(base_dir), + 'labels': {'com.docker.compose.test': 'true'} + }) + service.build() + assert service.image() + assert service.image()['Config']['Labels']['com.docker.compose.test'] == 'true' + def test_start_container_stays_unprivileged(self): service = self.create_service('web') container = create_and_start_container(service).inspect() From 909ef7f4352ee7a107bd6880beaa939a19f385a7 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Tue, 23 May 2017 16:30:48 -0700 Subject: [PATCH 035/244] Add partial support (docker-compose config and warnings) for v3.3 credential_spec Signed-off-by: Joffrey F --- compose/config/config.py | 42 +++++++++++++++++++++++--------- tests/unit/config/config_test.py | 17 +++++++++++++ 2 files changed, 47 insertions(+), 12 deletions(-) diff --git a/compose/config/config.py b/compose/config/config.py index 8dac4fb33..44f84ac82 100644 --- a/compose/config/config.py +++ b/compose/config/config.py @@ -108,6 +108,7 @@ DOCKER_CONFIG_KEYS = [ ALLOWED_KEYS = DOCKER_CONFIG_KEYS + [ 'build', 'container_name', + 'credential_spec', 'dockerfile', 'log_driver', 'log_opt', @@ -320,6 +321,27 @@ def find_candidates_in_parent_dirs(filenames, path): return (candidates, path) +def check_swarm_only_config(service_dicts): + warning_template = ( + "Some services ({services}) use the '{key}' key, which will be ignored. " + "Compose does not support '{key}' configuration - use " + "`docker stack deploy` to deploy to a swarm." + ) + + def check_swarm_only_key(service_dicts, key): + services = [s for s in service_dicts if s.get(key)] + if services: + log.warn( + warning_template.format( + services=", ".join(sorted(s['name'] for s in services)), + key=key + ) + ) + + check_swarm_only_key(service_dicts, 'deploy') + check_swarm_only_key(service_dicts, 'credential_spec') + + def load(config_details): """Load the configuration from a working directory and a list of configuration files. Files are loaded in order, and merged on top @@ -349,13 +371,7 @@ def load(config_details): for service_dict in service_dicts: match_named_volumes(service_dict, volumes) - services_using_deploy = [s for s in service_dicts if s.get('deploy')] - if services_using_deploy: - log.warn( - "Some services ({}) use the 'deploy' key, which will be ignored. " - "Compose does not support deploy configuration - use " - "`docker stack deploy` to deploy to a swarm." - .format(", ".join(sorted(s['name'] for s in services_using_deploy)))) + check_swarm_only_config(service_dicts) return Config(main_file.version, service_dicts, volumes, networks, secrets) @@ -884,7 +900,7 @@ def merge_service_dicts(base, override, version): md.merge_mapping('environment', parse_environment) md.merge_mapping('labels', parse_labels) - md.merge_mapping('ulimits', parse_ulimits) + md.merge_mapping('ulimits', parse_flat_dict) md.merge_mapping('networks', parse_networks) md.merge_mapping('sysctls', parse_sysctls) md.merge_mapping('depends_on', parse_depends_on) @@ -1020,12 +1036,14 @@ parse_depends_on = functools.partial( parse_deploy = functools.partial(parse_dict_or_list, split_kv, 'deploy') -def parse_ulimits(ulimits): - if not ulimits: +def parse_flat_dict(d): + if not d: return {} - if isinstance(ulimits, dict): - return dict(ulimits) + if isinstance(d, dict): + return dict(d) + + raise ConfigurationError("Invalid type: expected mapping") def resolve_env_var(key, val, environment): diff --git a/tests/unit/config/config_test.py b/tests/unit/config/config_test.py index d8973484d..d1160c767 100644 --- a/tests/unit/config/config_test.py +++ b/tests/unit/config/config_test.py @@ -2033,6 +2033,23 @@ class ConfigTest(unittest.TestCase): } } + def test_merge_credential_spec(self): + base = { + 'image': 'bb', + 'credential_spec': { + 'file': '/hello-world', + } + } + + override = { + 'credential_spec': { + 'registry': 'revolution.com', + } + } + + actual = config.merge_service_dicts(base, override, V3_3) + assert actual['credential_spec'] == override['credential_spec'] + def test_external_volume_config(self): config_details = build_config_details({ 'version': '2', From e6000051f7f1ac86cc668bcf927dc9e58389f617 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Fri, 26 May 2017 14:39:47 -0700 Subject: [PATCH 036/244] Bump 1.14.0-rc1 Signed-off-by: Joffrey F --- CHANGELOG.md | 49 +++++++++++++++++++++++++++++++++++++++++++++ compose/__init__.py | 2 +- script/run/run.sh | 2 +- 3 files changed, 51 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d1da62e34..748cce08a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,55 @@ Change log ========== +1.14.0 (2017-06-06) +------------------- + +### New features + +#### Compose file version 3.3 + +- Introduced version 3.3 of the `docker-compose.yml` specification. + This version requires to be used with Docker Engine 17.06.0 or above. + Note: the `credential_spec` key only applies to Swarm services and will + be ignored by Compose + +#### Compose file version 2.2 + +- Added the following parameters in service definitions: `cpu_count`, + `cpu_percent`, `cpus` + +#### Compose file version 2.1 + +- Added support for build labels. This feature is also available in the + 2.2 and 3.3 formats. + +#### All formats + +- Added shorthand `-u` for `--user` flag in `docker-compose exec` + +- Differences in labels between the Compose file and remote network + will now print a warning instead of preventing redeployment. + +### Bugfixes + +- Fixed a bug where service's dependencies were being rescaled to their + default scale when running a `docker-compose run` command + +- Fixed a bug where `docker-compose rm` with the `--stop` flag was not + behaving properly when provided with a list of services to remove + +- Fixed a bug where `cache_from` in the build section would be ignored when + using more than one Compose file. + +- Fixed a bug where override files would not be picked up by Compose if they + had the `.yaml` extension + +- Fixed a bug on Windows Engine where networks would be incorrectly flagged + for recreation + +- Fixed a bug where services declaring ports would cause crashes on some + versions of Python 3 + 1.13.0 (2017-05-02) ------------------- diff --git a/compose/__init__.py b/compose/__init__.py index 69307d60e..445216460 100644 --- a/compose/__init__.py +++ b/compose/__init__.py @@ -1,4 +1,4 @@ from __future__ import absolute_import from __future__ import unicode_literals -__version__ = '1.14.0dev' +__version__ = '1.14.0-rc1' diff --git a/script/run/run.sh b/script/run/run.sh index d1e1fabaa..45063abd5 100755 --- a/script/run/run.sh +++ b/script/run/run.sh @@ -15,7 +15,7 @@ set -e -VERSION="1.13.0" +VERSION="1.14.0-rc1" IMAGE="docker/compose:$VERSION" From ff720ba6b29a6bfb5b081233a7b5d5a0e17b9646 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Mon, 5 Jun 2017 11:51:43 -0700 Subject: [PATCH 037/244] Bump docker version in requirements.txt Signed-off-by: Joffrey F --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index f8061af83..c4545de1e 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,7 +2,7 @@ PyYAML==3.11 backports.ssl-match-hostname==3.5.0.1; python_version < '3' cached-property==1.2.0 colorama==0.3.7 -docker==2.2.1 +docker==2.3.0 dockerpty==0.4.1 docopt==0.6.1 enum34==1.0.4; python_version < '3.4' From bfc7ac4995851097503d78d5af80610adcad3aa6 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Mon, 5 Jun 2017 14:33:20 -0700 Subject: [PATCH 038/244] Always convert port values in ServicePort to integer Signed-off-by: Joffrey F --- compose/config/types.py | 16 ++++++++++++++++ tests/unit/config/types_test.py | 21 +++++++++++++-------- 2 files changed, 29 insertions(+), 8 deletions(-) diff --git a/compose/config/types.py b/compose/config/types.py index d853d84f4..85daa70b0 100644 --- a/compose/config/types.py +++ b/compose/config/types.py @@ -263,6 +263,22 @@ class ServiceSecret(namedtuple('_ServiceSecret', 'source target uid gid mode')): class ServicePort(namedtuple('_ServicePort', 'target published protocol mode external_ip')): + def __new__(cls, target, published, *args, **kwargs): + try: + if target: + target = int(target) + except ValueError: + raise ConfigurationError('Invalid target port: {}'.format(target)) + + try: + if published: + published = int(published) + except ValueError: + raise ConfigurationError('Invalid published port: {}'.format(published)) + + return super(ServicePort, cls).__new__( + cls, target, published, *args, **kwargs + ) @classmethod def parse(cls, spec): diff --git a/tests/unit/config/types_test.py b/tests/unit/config/types_test.py index 83d6270d2..10b698fe3 100644 --- a/tests/unit/config/types_test.py +++ b/tests/unit/config/types_test.py @@ -57,15 +57,15 @@ class TestServicePort(object): def test_parse_simple_target_port(self): ports = ServicePort.parse(8000) assert len(ports) == 1 - assert ports[0].target == '8000' + assert ports[0].target == 8000 def test_parse_complete_port_definition(self): port_def = '1.1.1.1:3000:3000/udp' ports = ServicePort.parse(port_def) assert len(ports) == 1 assert ports[0].repr() == { - 'target': '3000', - 'published': '3000', + 'target': 3000, + 'published': 3000, 'external_ip': '1.1.1.1', 'protocol': 'udp', } @@ -77,7 +77,7 @@ class TestServicePort(object): assert len(ports) == 1 assert ports[0].legacy_repr() == port_def + '/tcp' assert ports[0].repr() == { - 'target': '3000', + 'target': 3000, 'external_ip': '1.1.1.1', } @@ -86,14 +86,19 @@ class TestServicePort(object): assert len(ports) == 2 reprs = [p.repr() for p in ports] assert { - 'target': '4000', - 'published': '25000' + 'target': 4000, + 'published': 25000 } in reprs assert { - 'target': '4001', - 'published': '25001' + 'target': 4001, + 'published': 25001 } in reprs + def test_parse_invalid_port(self): + port_def = '4000p' + with pytest.raises(ConfigurationError): + ServicePort.parse(port_def) + class TestVolumeSpec(object): From 70b2e64c1b126d59c0b6e19333a67df597936e11 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Mon, 5 Jun 2017 16:37:20 -0700 Subject: [PATCH 039/244] Partial support for service configs Signed-off-by: Joffrey F --- compose/config/config.py | 51 +++++++++++++++++-------------------- compose/config/serialize.py | 24 +++++++---------- compose/config/types.py | 11 ++++++-- 3 files changed, 42 insertions(+), 44 deletions(-) diff --git a/compose/config/config.py b/compose/config/config.py index 44f84ac82..fd933d939 100644 --- a/compose/config/config.py +++ b/compose/config/config.py @@ -211,8 +211,11 @@ class ConfigFile(namedtuple('_ConfigFile', 'filename config')): def get_secrets(self): return {} if self.version < const.COMPOSEFILE_V3_1 else self.config.get('secrets', {}) + def get_configs(self): + return {} if self.version < const.COMPOSEFILE_V3_3 else self.config.get('configs', {}) -class Config(namedtuple('_Config', 'version services volumes networks secrets')): + +class Config(namedtuple('_Config', 'version services volumes networks secrets configs')): """ :param version: configuration version :type version: int @@ -224,6 +227,8 @@ class Config(namedtuple('_Config', 'version services volumes networks secrets')) :type networks: :class:`dict` :param secrets: Dictionary mapping secret names to description dictionaries :type secrets: :class:`dict` + :param configs: Dictionary mapping config names to description dictionaries + :type configs: :class:`dict` """ @@ -340,6 +345,7 @@ def check_swarm_only_config(service_dicts): check_swarm_only_key(service_dicts, 'deploy') check_swarm_only_key(service_dicts, 'credential_spec') + check_swarm_only_key(service_dicts, 'configs') def load(config_details): @@ -364,7 +370,12 @@ def load(config_details): networks = load_mapping( config_details.config_files, 'get_networks', 'Network' ) - secrets = load_secrets(config_details.config_files, config_details.working_dir) + secrets = load_mapping( + config_details.config_files, 'get_secrets', 'Secret', config_details.working_dir + ) + configs = load_mapping( + config_details.config_files, 'get_configs', 'Config', config_details.working_dir + ) service_dicts = load_services(config_details, main_file) if main_file.version != V1: @@ -373,10 +384,10 @@ def load(config_details): check_swarm_only_config(service_dicts) - return Config(main_file.version, service_dicts, volumes, networks, secrets) + return Config(main_file.version, service_dicts, volumes, networks, secrets, configs) -def load_mapping(config_files, get_func, entity_type): +def load_mapping(config_files, get_func, entity_type, working_dir=None): mapping = {} for config_file in config_files: @@ -401,6 +412,9 @@ def load_mapping(config_files, get_func, entity_type): if 'labels' in config: config['labels'] = parse_labels(config['labels']) + if 'file' in config: + config['file'] = expand_path(working_dir, config['file']) + return mapping @@ -414,29 +428,6 @@ def validate_external(entity_type, name, config): entity_type, name, ', '.join(k for k in config if k != 'external'))) -def load_secrets(config_files, working_dir): - mapping = {} - - for config_file in config_files: - for name, config in config_file.get_secrets().items(): - mapping[name] = config or {} - if not config: - continue - - external = config.get('external') - if external: - validate_external('Secret', name, config) - if isinstance(external, dict): - config['external_name'] = external.get('name') - else: - config['external_name'] = name - - if 'file' in config: - config['file'] = expand_path(working_dir, config['file']) - - return mapping - - def load_services(config_details, config_file): def build_service(service_name, service_dict, service_names): service_config = ServiceConfig.with_abs_paths( @@ -815,6 +806,11 @@ def finalize_service(service_config, service_names, version, environment): types.ServiceSecret.parse(s) for s in service_dict['secrets'] ] + if 'configs' in service_dict: + service_dict['configs'] = [ + types.ServiceConfig.parse(c) for c in service_dict['configs'] + ] + normalize_build(service_dict, service_config.working_dir, environment) service_dict['name'] = service_config.name @@ -906,6 +902,7 @@ def merge_service_dicts(base, override, version): md.merge_mapping('depends_on', parse_depends_on) md.merge_sequence('links', ServiceLink.parse) md.merge_sequence('secrets', types.ServiceSecret.parse) + md.merge_sequence('configs', types.ServiceConfig.parse) md.merge_mapping('deploy', parse_deploy) for field in ['volumes', 'devices']: diff --git a/compose/config/serialize.py b/compose/config/serialize.py index ac78b77a2..beafe02b9 100644 --- a/compose/config/serialize.py +++ b/compose/config/serialize.py @@ -8,7 +8,6 @@ from compose.config import types from compose.const import COMPOSEFILE_V1 as V1 from compose.const import COMPOSEFILE_V2_1 as V2_1 from compose.const import COMPOSEFILE_V2_2 as V2_2 -from compose.const import COMPOSEFILE_V3_1 as V3_1 from compose.const import COMPOSEFILE_V3_2 as V3_2 from compose.const import COMPOSEFILE_V3_3 as V3_3 @@ -25,6 +24,7 @@ def serialize_dict_type(dumper, data): yaml.SafeDumper.add_representer(types.VolumeFromSpec, serialize_config_type) yaml.SafeDumper.add_representer(types.VolumeSpec, serialize_config_type) yaml.SafeDumper.add_representer(types.ServiceSecret, serialize_dict_type) +yaml.SafeDumper.add_representer(types.ServiceConfig, serialize_dict_type) yaml.SafeDumper.add_representer(types.ServicePort, serialize_dict_type) @@ -41,21 +41,15 @@ def denormalize_config(config, image_digests=None): service_dict.pop('name'): service_dict for service_dict in denormalized_services } - result['networks'] = config.networks.copy() - for net_name, net_conf in result['networks'].items(): - if 'external_name' in net_conf: - del net_conf['external_name'] + for key in ('networks', 'volumes', 'secrets', 'configs'): + config_dict = getattr(config, key) + if not config_dict: + continue + result[key] = config_dict.copy() + for name, conf in result[key].items(): + if 'external_name' in conf: + del conf['external_name'] - result['volumes'] = config.volumes.copy() - for vol_name, vol_conf in result['volumes'].items(): - if 'external_name' in vol_conf: - del vol_conf['external_name'] - - if config.version in (V3_1, V3_2, V3_3): - result['secrets'] = config.secrets.copy() - for secret_name, secret_conf in result['secrets'].items(): - if 'external_name' in secret_conf: - del secret_conf['external_name'] return result diff --git a/compose/config/types.py b/compose/config/types.py index 85daa70b0..6d3ca3f3b 100644 --- a/compose/config/types.py +++ b/compose/config/types.py @@ -238,8 +238,7 @@ class ServiceLink(namedtuple('_ServiceLink', 'target alias')): return self.alias -class ServiceSecret(namedtuple('_ServiceSecret', 'source target uid gid mode')): - +class ServiceConfigBase(namedtuple('_ServiceConfigBase', 'source target uid gid mode')): @classmethod def parse(cls, spec): if isinstance(spec, six.string_types): @@ -262,6 +261,14 @@ class ServiceSecret(namedtuple('_ServiceSecret', 'source target uid gid mode')): ) +class ServiceSecret(ServiceConfigBase): + pass + + +class ServiceConfig(ServiceConfigBase): + pass + + class ServicePort(namedtuple('_ServicePort', 'target published protocol mode external_ip')): def __new__(cls, target, published, *args, **kwargs): try: From bf3b62e2ff72e47a7c39d882051c0b435c72852f Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Mon, 5 Jun 2017 16:57:09 -0700 Subject: [PATCH 040/244] Add configs tests Signed-off-by: Joffrey F --- tests/unit/bundle_test.py | 4 +- tests/unit/config/config_test.py | 168 ++++++++++++++++++++++++++++++- tests/unit/project_test.py | 12 +++ 3 files changed, 182 insertions(+), 2 deletions(-) diff --git a/tests/unit/bundle_test.py b/tests/unit/bundle_test.py index 21bdb31b0..3c6e9ec53 100644 --- a/tests/unit/bundle_test.py +++ b/tests/unit/bundle_test.py @@ -78,7 +78,9 @@ def test_to_bundle(): services=services, volumes={'special': {}}, networks={'extra': {}}, - secrets={}) + secrets={}, + configs={} + ) with mock.patch('compose.bundle.log.warn', autospec=True) as mock_log: output = bundle.to_bundle(config, image_digests) diff --git a/tests/unit/config/config_test.py b/tests/unit/config/config_test.py index d1160c767..d92a35c00 100644 --- a/tests/unit/config/config_test.py +++ b/tests/unit/config/config_test.py @@ -1982,6 +1982,38 @@ class ConfigTest(unittest.TestCase): actual = config.merge_service_dicts(base, override, V3_1) assert actual['secrets'] == override['secrets'] + def test_merge_different_configs(self): + base = { + 'image': 'busybox', + 'configs': [ + {'source': 'src.txt'} + ] + } + override = {'configs': ['other-src.txt']} + + actual = config.merge_service_dicts(base, override, V3_3) + assert secret_sort(actual['configs']) == secret_sort([ + {'source': 'src.txt'}, + {'source': 'other-src.txt'} + ]) + + def test_merge_configs_override(self): + base = { + 'image': 'busybox', + 'configs': ['src.txt'], + } + override = { + 'configs': [ + { + 'source': 'src.txt', + 'target': 'data.txt', + 'mode': 0o400 + } + ] + } + actual = config.merge_service_dicts(base, override, V3_3) + assert actual['configs'] == override['configs'] + def test_merge_deploy(self): base = { 'image': 'busybox', @@ -2214,6 +2246,91 @@ class ConfigTest(unittest.TestCase): ] assert service_sort(service_dicts) == service_sort(expected) + def test_load_configs(self): + base_file = config.ConfigFile( + 'base.yaml', + { + 'version': '3.3', + 'services': { + 'web': { + 'image': 'example/web', + 'configs': [ + 'one', + { + 'source': 'source', + 'target': 'target', + 'uid': '100', + 'gid': '200', + 'mode': 0o777, + }, + ], + }, + }, + 'configs': { + 'one': {'file': 'secret.txt'}, + }, + }) + details = config.ConfigDetails('.', [base_file]) + service_dicts = config.load(details).services + expected = [ + { + 'name': 'web', + 'image': 'example/web', + 'configs': [ + types.ServiceConfig('one', None, None, None, None), + types.ServiceConfig('source', 'target', '100', '200', 0o777), + ], + }, + ] + assert service_sort(service_dicts) == service_sort(expected) + + def test_load_configs_multi_file(self): + base_file = config.ConfigFile( + 'base.yaml', + { + 'version': '3.3', + 'services': { + 'web': { + 'image': 'example/web', + 'configs': ['one'], + }, + }, + 'configs': { + 'one': {'file': 'secret.txt'}, + }, + }) + override_file = config.ConfigFile( + 'base.yaml', + { + 'version': '3.3', + 'services': { + 'web': { + 'configs': [ + { + 'source': 'source', + 'target': 'target', + 'uid': '100', + 'gid': '200', + 'mode': 0o777, + }, + ], + }, + }, + }) + details = config.ConfigDetails('.', [base_file, override_file]) + service_dicts = config.load(details).services + expected = [ + { + 'name': 'web', + 'image': 'example/web', + 'configs': [ + types.ServiceConfig('one', None, None, None, None), + types.ServiceConfig('source', 'target', '100', '200', 0o777), + ], + }, + ] + assert service_sort(service_dicts) == service_sort(expected) + class NetworkModeTest(unittest.TestCase): @@ -2533,6 +2650,24 @@ class InterpolationTest(unittest.TestCase): } } + @mock.patch.dict(os.environ) + def test_interpolation_configs_section(self): + os.environ['FOO'] = 'baz.bar' + config_dict = config.load(build_config_details({ + 'version': '3.3', + 'configs': { + 'configdata': { + 'external': {'name': '$FOO'} + } + } + })) + assert config_dict.configs == { + 'configdata': { + 'external': {'name': 'baz.bar'}, + 'external_name': 'baz.bar' + } + } + class VolumeConfigTest(unittest.TestCase): @@ -3964,7 +4099,38 @@ class SerializeTest(unittest.TestCase): 'image': 'alpine', 'name': 'web' } - ], volumes={}, networks={}, secrets={}) + ], volumes={}, networks={}, secrets={}, configs={}) serialized_config = yaml.load(serialize_config(config_dict)) assert '8080:80/tcp' in serialized_config['services']['web']['ports'] + + def test_serialize_configs(self): + service_dict = { + 'image': 'example/web', + 'configs': [ + {'source': 'one'}, + { + 'source': 'source', + 'target': 'target', + 'uid': '100', + 'gid': '200', + 'mode': 0o777, + } + ] + } + configs_dict = { + 'one': {'file': '/one.txt'}, + 'source': {'file': '/source.pem'}, + 'two': {'external': True}, + } + config_dict = config.load(build_config_details({ + 'version': '3.3', + 'services': {'web': service_dict}, + 'configs': configs_dict + })) + + serialized_config = yaml.load(serialize_config(config_dict)) + serialized_service = serialized_config['services']['web'] + assert secret_sort(serialized_service['configs']) == secret_sort(service_dict['configs']) + assert 'configs' in serialized_config + assert serialized_config['configs']['two'] == configs_dict['two'] diff --git a/tests/unit/project_test.py b/tests/unit/project_test.py index 32d0adfaf..c5366c395 100644 --- a/tests/unit/project_test.py +++ b/tests/unit/project_test.py @@ -37,6 +37,7 @@ class ProjectTest(unittest.TestCase): networks=None, volumes=None, secrets=None, + configs=None, ) project = Project.from_config( name='composetest', @@ -66,6 +67,7 @@ class ProjectTest(unittest.TestCase): networks=None, volumes=None, secrets=None, + configs=None, ) project = Project.from_config('composetest', config, None) self.assertEqual(len(project.services), 2) @@ -173,6 +175,7 @@ class ProjectTest(unittest.TestCase): networks=None, volumes=None, secrets=None, + configs=None, ), ) assert project.get_service('test')._get_volumes_from() == [container_id + ":rw"] @@ -206,6 +209,7 @@ class ProjectTest(unittest.TestCase): networks=None, volumes=None, secrets=None, + configs=None, ), ) assert project.get_service('test')._get_volumes_from() == [container_name + ":rw"] @@ -232,6 +236,7 @@ class ProjectTest(unittest.TestCase): networks=None, volumes=None, secrets=None, + configs=None, ), ) with mock.patch.object(Service, 'containers') as mock_return: @@ -366,6 +371,7 @@ class ProjectTest(unittest.TestCase): networks=None, volumes=None, secrets=None, + configs=None, ), ) service = project.get_service('test') @@ -391,6 +397,7 @@ class ProjectTest(unittest.TestCase): networks=None, volumes=None, secrets=None, + configs=None, ), ) service = project.get_service('test') @@ -425,6 +432,7 @@ class ProjectTest(unittest.TestCase): networks=None, volumes=None, secrets=None, + configs=None, ), ) @@ -446,6 +454,7 @@ class ProjectTest(unittest.TestCase): networks=None, volumes=None, secrets=None, + configs=None, ), ) @@ -467,6 +476,7 @@ class ProjectTest(unittest.TestCase): networks={'custom': {}}, volumes=None, secrets=None, + configs=None, ), ) @@ -498,6 +508,7 @@ class ProjectTest(unittest.TestCase): networks=None, volumes=None, secrets=None, + configs=None, ), ) self.assertEqual([c.id for c in project.containers()], ['1']) @@ -515,6 +526,7 @@ class ProjectTest(unittest.TestCase): networks={'default': {}}, volumes={'data': {}}, secrets=None, + configs=None, ), ) self.mock_client.remove_network.side_effect = NotFound(None, None, 'oops') From e7b74804623b505f0221598e7c5ca8e2f858d406 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Mon, 5 Jun 2017 16:57:24 -0700 Subject: [PATCH 041/244] Interpolate configs values Signed-off-by: Joffrey F --- compose/config/config.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/compose/config/config.py b/compose/config/config.py index fd933d939..b8bffc660 100644 --- a/compose/config/config.py +++ b/compose/config/config.py @@ -507,12 +507,20 @@ def process_config_file(config_file, environment, service_name=None): config_file.get_networks(), 'network', environment) - if config_file.version in (const.COMPOSEFILE_V3_1, const.COMPOSEFILE_V3_2): + if config_file.version in (const.COMPOSEFILE_V3_1, const.COMPOSEFILE_V3_2, + const.COMPOSEFILE_V3_3): processed_config['secrets'] = interpolate_config_section( config_file, config_file.get_secrets(), 'secrets', environment) + if config_file.version in (const.COMPOSEFILE_V3_3): + processed_config['configs'] = interpolate_config_section( + config_file, + config_file.get_configs(), + 'configs', + environment + ) else: processed_config = services From a85dddf83d17985cf14c1a9a5f1316cb45805ae3 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Mon, 5 Jun 2017 19:34:53 -0700 Subject: [PATCH 042/244] Remedy test failures Signed-off-by: Joffrey F --- tests/acceptance/cli_test.py | 6 ------ tests/integration/project_test.py | 4 +++- 2 files changed, 3 insertions(+), 7 deletions(-) diff --git a/tests/acceptance/cli_test.py b/tests/acceptance/cli_test.py index 89f4f288b..dd95fb545 100644 --- a/tests/acceptance/cli_test.py +++ b/tests/acceptance/cli_test.py @@ -258,8 +258,6 @@ class CLITestCase(DockerClientTestCase): 'restart': '' }, }, - 'networks': {}, - 'volumes': {}, } def test_config_external_network(self): @@ -311,8 +309,6 @@ class CLITestCase(DockerClientTestCase): 'network_mode': 'service:net', }, }, - 'networks': {}, - 'volumes': {}, } @v3_only() @@ -322,8 +318,6 @@ class CLITestCase(DockerClientTestCase): assert yaml.load(result.stdout) == { 'version': '3.2', - 'networks': {}, - 'secrets': {}, 'volumes': { 'foobar': { 'labels': { diff --git a/tests/integration/project_test.py b/tests/integration/project_test.py index 69f06b75c..6c5f719ed 100644 --- a/tests/integration/project_test.py +++ b/tests/integration/project_test.py @@ -40,7 +40,9 @@ def build_config(**kwargs): services=kwargs.get('services'), volumes=kwargs.get('volumes'), networks=kwargs.get('networks'), - secrets=kwargs.get('secrets')) + secrets=kwargs.get('secrets'), + configs=kwargs.get('configs'), + ) class ProjectTest(DockerClientTestCase): From cfe152f907bb9b2c1ad430a5bf11ed4b8b9bdf4e Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Tue, 6 Jun 2017 12:36:26 -0700 Subject: [PATCH 043/244] Bump 1.14.0-rc2 Signed-off-by: Joffrey F --- CHANGELOG.md | 7 +++++-- compose/__init__.py | 2 +- script/run/run.sh | 2 +- 3 files changed, 7 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 748cce08a..02e439e4f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,8 +10,8 @@ Change log - Introduced version 3.3 of the `docker-compose.yml` specification. This version requires to be used with Docker Engine 17.06.0 or above. - Note: the `credential_spec` key only applies to Swarm services and will - be ignored by Compose + Note: the `credential_spec` and `configs` keys only apply to Swarm services + and will be ignored by Compose #### Compose file version 2.2 @@ -50,6 +50,9 @@ Change log - Fixed a bug where services declaring ports would cause crashes on some versions of Python 3 +- Fixed a bug where the output of `docker-compose config` would sometimes + contain invalid port definitions + 1.13.0 (2017-05-02) ------------------- diff --git a/compose/__init__.py b/compose/__init__.py index 445216460..9bbae98d5 100644 --- a/compose/__init__.py +++ b/compose/__init__.py @@ -1,4 +1,4 @@ from __future__ import absolute_import from __future__ import unicode_literals -__version__ = '1.14.0-rc1' +__version__ = '1.14.0-rc2' diff --git a/script/run/run.sh b/script/run/run.sh index 45063abd5..ef2f63d8f 100755 --- a/script/run/run.sh +++ b/script/run/run.sh @@ -15,7 +15,7 @@ set -e -VERSION="1.14.0-rc1" +VERSION="1.14.0-rc2" IMAGE="docker/compose:$VERSION" From 5c3d0db3f2eb3aa9ad056c9ec9337295b260a352 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Fri, 9 Jun 2017 16:58:24 -0700 Subject: [PATCH 044/244] ServicePort merge_field should account for external IP and protocol Signed-off-by: Joffrey F --- compose/config/types.py | 2 +- tests/unit/config/config_test.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/compose/config/types.py b/compose/config/types.py index 6d3ca3f3b..4509bfe67 100644 --- a/compose/config/types.py +++ b/compose/config/types.py @@ -325,7 +325,7 @@ class ServicePort(namedtuple('_ServicePort', 'target published protocol mode ext @property def merge_field(self): - return (self.target, self.published) + return (self.target, self.published, self.external_ip, self.protocol) def repr(self): return dict( diff --git a/tests/unit/config/config_test.py b/tests/unit/config/config_test.py index d92a35c00..87bdd8bca 100644 --- a/tests/unit/config/config_test.py +++ b/tests/unit/config/config_test.py @@ -1865,7 +1865,7 @@ class ConfigTest(unittest.TestCase): { 'target': '1245', 'published': '1245', - 'protocol': 'tcp', + 'protocol': 'udp', } ] } From abac2eea37d55bfef8ca443f3f79ccbdb0949db3 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Fri, 9 Jun 2017 16:59:09 -0700 Subject: [PATCH 045/244] Fix `ps` output to show all ports Signed-off-by: Joffrey F --- compose/container.py | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/compose/container.py b/compose/container.py index bda4e659f..4bc7f54f9 100644 --- a/compose/container.py +++ b/compose/container.py @@ -96,12 +96,16 @@ class Container(object): def human_readable_ports(self): def format_port(private, public): if not public: - return private - return '{HostIp}:{HostPort}->{private}'.format( - private=private, **public[0]) + return [private] + return [ + '{HostIp}:{HostPort}->{private}'.format(private=private, **pub) + for pub in public + ] - return ', '.join(format_port(*item) - for item in sorted(six.iteritems(self.ports))) + return ', '.join( + ','.join(format_port(*item)) + for item in sorted(six.iteritems(self.ports)) + ) @property def labels(self): From cffce0880befc1426348eeb8a734b8baa8f790c5 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Thu, 15 Jun 2017 17:05:23 -0700 Subject: [PATCH 046/244] Bump 1.14.0 Signed-off-by: Joffrey F --- CHANGELOG.md | 5 ++++- compose/__init__.py | 2 +- script/run/run.sh | 2 +- 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 02e439e4f..cced3804c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,7 +1,7 @@ Change log ========== -1.14.0 (2017-06-06) +1.14.0 (2017-06-19) ------------------- ### New features @@ -41,6 +41,9 @@ Change log - Fixed a bug where `cache_from` in the build section would be ignored when using more than one Compose file. +- Fixed a bug that prevented binding the same port to different IPs when + using more than one Compose file. + - Fixed a bug where override files would not be picked up by Compose if they had the `.yaml` extension diff --git a/compose/__init__.py b/compose/__init__.py index 9bbae98d5..f6ed1f463 100644 --- a/compose/__init__.py +++ b/compose/__init__.py @@ -1,4 +1,4 @@ from __future__ import absolute_import from __future__ import unicode_literals -__version__ = '1.14.0-rc2' +__version__ = '1.14.0' diff --git a/script/run/run.sh b/script/run/run.sh index ef2f63d8f..e4a2f4199 100755 --- a/script/run/run.sh +++ b/script/run/run.sh @@ -15,7 +15,7 @@ set -e -VERSION="1.14.0-rc2" +VERSION="1.14.0" IMAGE="docker/compose:$VERSION" From 50d405fea33b9eac47954d507c967aa530a465f4 Mon Sep 17 00:00:00 2001 From: Alexey Rokhin Date: Wed, 17 May 2017 16:18:28 +0300 Subject: [PATCH 047/244] skip cpu_percent test for Linux Signed-off-by: Alexey Rokhin --- tests/integration/service_test.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/integration/service_test.py b/tests/integration/service_test.py index 178df1323..79dd4f283 100644 --- a/tests/integration/service_test.py +++ b/tests/integration/service_test.py @@ -26,6 +26,7 @@ from compose.const import LABEL_ONE_OFF from compose.const import LABEL_PROJECT from compose.const import LABEL_SERVICE from compose.const import LABEL_VERSION +from compose.const import IS_WINDOWS_PLATFORM from compose.container import Container from compose.errors import OperationFailedError from compose.project import OneOffFilter From 5067f7a77ba7b0e367d49e19d5a252c31db72003 Mon Sep 17 00:00:00 2001 From: Alexey Rokhin Date: Wed, 17 May 2017 16:42:43 +0300 Subject: [PATCH 048/244] service_test.py reorder imports Signed-off-by: Alexey Rokhin --- tests/integration/service_test.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/integration/service_test.py b/tests/integration/service_test.py index 79dd4f283..178df1323 100644 --- a/tests/integration/service_test.py +++ b/tests/integration/service_test.py @@ -26,7 +26,6 @@ from compose.const import LABEL_ONE_OFF from compose.const import LABEL_PROJECT from compose.const import LABEL_SERVICE from compose.const import LABEL_VERSION -from compose.const import IS_WINDOWS_PLATFORM from compose.container import Container from compose.errors import OperationFailedError from compose.project import OneOffFilter From 645d35612d636e3812abc039bf0c38b0a9a92417 Mon Sep 17 00:00:00 2001 From: Colin Hebert Date: Thu, 13 Apr 2017 21:51:41 +1000 Subject: [PATCH 049/244] Add support for labels during build Signed-off-by: Colin Hebert --- compose/config/config_schema_v3.2.json | 1 + 1 file changed, 1 insertion(+) diff --git a/compose/config/config_schema_v3.2.json b/compose/config/config_schema_v3.2.json index ea702fcd5..70ff6ce05 100644 --- a/compose/config/config_schema_v3.2.json +++ b/compose/config/config_schema_v3.2.json @@ -72,6 +72,7 @@ "context": {"type": "string"}, "dockerfile": {"type": "string"}, "args": {"$ref": "#/definitions/list_or_dict"}, + "labels": {"$ref": "#/definitions/list_or_dict"}, "cache_from": {"$ref": "#/definitions/list_of_strings"} }, "additionalProperties": false From 74f5037f785fe400fb403e433f3c160c50967143 Mon Sep 17 00:00:00 2001 From: Sebastiaan van Stijn Date: Mon, 29 May 2017 14:01:50 +0200 Subject: [PATCH 050/244] Add Joffrey to maintainers Signed-off-by: Sebastiaan van Stijn --- MAINTAINERS | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/MAINTAINERS b/MAINTAINERS index 820b2f829..89f5b4124 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -15,6 +15,7 @@ "bfirsh", "dnephin", "mnowster", + "shin-", ] [people] @@ -44,3 +45,8 @@ Name = "Mazz Mosley" Email = "mazz@houseofmnowster.com" GitHub = "mnowster" + + [People.shin-] + Name = "Joffrey F" + Email = "joffrey@docker.com" + GitHub = "shin-" From 33c7c750e81528fb4c3a6a650821a722505726b5 Mon Sep 17 00:00:00 2001 From: Stefan Pietsch Date: Tue, 30 May 2017 23:54:01 +0200 Subject: [PATCH 051/244] check hash sums of downloaded files Signed-off-by: Stefan Pietsch --- Dockerfile | 23 ++++++++++++++++++----- 1 file changed, 18 insertions(+), 5 deletions(-) diff --git a/Dockerfile b/Dockerfile index a03e15106..154d51510 100644 --- a/Dockerfile +++ b/Dockerfile @@ -19,34 +19,47 @@ RUN set -ex; \ RUN curl https://get.docker.com/builds/Linux/x86_64/docker-1.8.3 \ -o /usr/local/bin/docker && \ + SHA256=f024bc65c45a3778cf07213d26016075e8172de8f6e4b5702bedde06c241650f; \ + echo "${SHA256} /usr/local/bin/docker" | sha256sum -c - && \ chmod +x /usr/local/bin/docker # Build Python 2.7.13 from source RUN set -ex; \ - curl -L https://www.python.org/ftp/python/2.7.13/Python-2.7.13.tgz | tar -xz; \ + curl -LO https://www.python.org/ftp/python/2.7.13/Python-2.7.13.tgz && \ + SHA256=a4f05a0720ce0fd92626f0278b6b433eee9a6173ddf2bced7957dfb599a5ece1; \ + echo "${SHA256} Python-2.7.13.tgz" | sha256sum -c - && \ + tar -xzf Python-2.7.13.tgz; \ cd Python-2.7.13; \ ./configure --enable-shared; \ make; \ make install; \ cd ..; \ - rm -rf /Python-2.7.13 + rm -rf /Python-2.7.13; \ + rm Python-2.7.13.tgz # Build python 3.4 from source RUN set -ex; \ - curl -L https://www.python.org/ftp/python/3.4.6/Python-3.4.6.tgz | tar -xz; \ + curl -LO https://www.python.org/ftp/python/3.4.6/Python-3.4.6.tgz && \ + SHA256=fe59daced99549d1d452727c050ae486169e9716a890cffb0d468b376d916b48; \ + echo "${SHA256} Python-3.4.6.tgz" | sha256sum -c - && \ + tar -xzf Python-3.4.6.tgz; \ cd Python-3.4.6; \ ./configure --enable-shared; \ make; \ make install; \ cd ..; \ - rm -rf /Python-3.4.6 + rm -rf /Python-3.4.6; \ + rm Python-3.4.6.tgz # Make libpython findable ENV LD_LIBRARY_PATH /usr/local/lib # Install pip RUN set -ex; \ - curl -L https://bootstrap.pypa.io/get-pip.py | python + curl -LO https://bootstrap.pypa.io/get-pip.py && \ + SHA256=19dae841a150c86e2a09d475b5eb0602861f2a5b7761ec268049a662dbd2bd0c; \ + echo "${SHA256} get-pip.py" | sha256sum -c - && \ + python get-pip.py # Python3 requires a valid locale RUN echo "en_US.UTF-8 UTF-8" > /etc/locale.gen && locale-gen From 85d2c0a31475fa372388cbb176a3c88425e576ec Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Mon, 5 Jun 2017 19:26:37 -0700 Subject: [PATCH 052/244] Take editions into account when selecting test engine versions Get candidates from moby/moby and docker/docker-ce repos Signed-off-by: Joffrey F --- script/test/all | 2 +- script/test/versions.py | 30 +++++++++++++++++++++--------- 2 files changed, 22 insertions(+), 10 deletions(-) diff --git a/script/test/all b/script/test/all index 7151a75e1..0c6ea6065 100755 --- a/script/test/all +++ b/script/test/all @@ -14,7 +14,7 @@ docker run --rm \ get_versions="docker run --rm --entrypoint=/code/.tox/py27/bin/python $TAG - /code/script/test/versions.py docker/docker" + /code/script/test/versions.py docker/docker-ce,moby/moby" if [ "$DOCKER_VERSIONS" == "" ]; then DOCKER_VERSIONS="$($get_versions default)" diff --git a/script/test/versions.py b/script/test/versions.py index 97383ad99..46872ed9a 100755 --- a/script/test/versions.py +++ b/script/test/versions.py @@ -37,14 +37,22 @@ import requests GITHUB_API = 'https://api.github.com/repos' -class Version(namedtuple('_Version', 'major minor patch rc')): +class Version(namedtuple('_Version', 'major minor patch rc edition')): @classmethod def parse(cls, version): + edition = None version = version.lstrip('v') version, _, rc = version.partition('-') + if rc: + if 'rc' not in rc: + edition = rc + rc = None + elif '-' in rc: + edition, rc = rc.split('-') + major, minor, patch = version.split('.', 3) - return cls(major, minor, patch, rc) + return cls(major, minor, patch, rc, edition) @property def major_minor(self): @@ -61,7 +69,8 @@ class Version(namedtuple('_Version', 'major minor patch rc')): def __str__(self): rc = '-{}'.format(self.rc) if self.rc else '' - return '.'.join(map(str, self[:3])) + rc + edition = '-{}'.format(self.edition) if self.edition else '' + return '.'.join(map(str, self[:3])) + edition + rc def group_versions(versions): @@ -94,6 +103,7 @@ def get_latest_versions(versions, num=1): group. """ versions = group_versions(versions) + num = min(len(versions), num) return [versions[index][0] for index in range(num)] @@ -112,16 +122,18 @@ def get_versions(tags): print("Skipping invalid tag: {name}".format(**tag), file=sys.stderr) -def get_github_releases(project): +def get_github_releases(projects): """Query the Github API for a list of version tags and return them in sorted order. See https://developer.github.com/v3/repos/#list-tags """ - url = '{}/{}/tags'.format(GITHUB_API, project) - response = requests.get(url) - response.raise_for_status() - versions = get_versions(response.json()) + versions = [] + for project in projects: + url = '{}/{}/tags'.format(GITHUB_API, project) + response = requests.get(url) + response.raise_for_status() + versions.extend(get_versions(response.json())) return sorted(versions, reverse=True, key=operator.attrgetter('order')) @@ -136,7 +148,7 @@ def parse_args(argv): def main(argv=None): args = parse_args(argv) - versions = get_github_releases(args.project) + versions = get_github_releases(args.project.split(',')) if args.command == 'recent': print(' '.join(map(str, get_latest_versions(versions, args.num)))) From 86a0e36348c618fc6994b8d4164da8294a0f38da Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Tue, 6 Jun 2017 16:13:34 -0700 Subject: [PATCH 053/244] s/docker daemon/dockerd/ Signed-off-by: Joffrey F --- script/test/all | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/script/test/all b/script/test/all index 0c6ea6065..1200c496e 100755 --- a/script/test/all +++ b/script/test/all @@ -48,7 +48,7 @@ for version in $DOCKER_VERSIONS; do --privileged \ --volume="/var/lib/docker" \ "$repo:$version" \ - docker daemon -H tcp://0.0.0.0:2375 $DOCKER_DAEMON_ARGS \ + dockerd -H tcp://0.0.0.0:2375 $DOCKER_DAEMON_ARGS \ 2>&1 | tail -n 10 docker run \ From 59c4c2388e7828a57e18cf8b47089af78f9ac6b6 Mon Sep 17 00:00:00 2001 From: Joel Barciauskas Date: Wed, 12 Apr 2017 17:45:09 -0400 Subject: [PATCH 054/244] Add --quiet parameter to docker-compose pull, using existing silent flag Signed-off-by: Joel Barciauskas --- compose/cli/main.py | 4 +++- compose/project.py | 6 +++--- tests/acceptance/cli_test.py | 4 ++++ 3 files changed, 10 insertions(+), 4 deletions(-) diff --git a/compose/cli/main.py b/compose/cli/main.py index cfca0f949..20f3b55b4 100644 --- a/compose/cli/main.py +++ b/compose/cli/main.py @@ -634,11 +634,13 @@ class TopLevelCommand(object): Options: --ignore-pull-failures Pull what it can and ignores images with pull failures. --parallel Pull multiple images in parallel. + --quiet Pull without printing progress information """ self.project.pull( service_names=options['SERVICE'], ignore_pull_failures=options.get('--ignore-pull-failures'), - parallel_pull=options.get('--parallel') + parallel_pull=options.get('--parallel'), + silent=options.get('--quiet'), ) def push(self, options): diff --git a/compose/project.py b/compose/project.py index b282f718d..3ad971488 100644 --- a/compose/project.py +++ b/compose/project.py @@ -462,12 +462,12 @@ class Project(object): return plans - def pull(self, service_names=None, ignore_pull_failures=False, parallel_pull=False): + def pull(self, service_names=None, ignore_pull_failures=False, parallel_pull=False, silent=False): services = self.get_services(service_names, include_deps=False) if parallel_pull: def pull_service(service): - service.pull(ignore_pull_failures, True) + service.pull(ignore_pull_failures, True, silent=silent) parallel.parallel_execute( services, @@ -477,7 +477,7 @@ class Project(object): limit=5) else: for service in services: - service.pull(ignore_pull_failures) + service.pull(ignore_pull_failures, silent=silent) def push(self, service_names=None, ignore_push_failures=False): for service in self.get_services(service_names, include_deps=False): diff --git a/tests/acceptance/cli_test.py b/tests/acceptance/cli_test.py index dd95fb545..9a1f5364b 100644 --- a/tests/acceptance/cli_test.py +++ b/tests/acceptance/cli_test.py @@ -431,6 +431,10 @@ class CLITestCase(DockerClientTestCase): assert ('repository nonexisting-image not found' in result.stderr or 'image library/nonexisting-image:latest not found' in result.stderr) + def test_pull_with_quiet(self): + assert self.dispatch(['pull', '--quiet']).stderr == '' + assert self.dispatch(['pull', '--quiet']).stdout == '' + def test_build_plain(self): self.base_dir = 'tests/fixtures/simple-dockerfile' self.dispatch(['build', 'simple']) From a0119ae1a5a8344801d534ddfbdc1a5156a3fe32 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Mon, 22 May 2017 14:58:51 -0700 Subject: [PATCH 055/244] Rewriting tests to be UCP/Swarm compatible - Event may contain more information in some cases. Don't assume order or format - Don't assume ports are always exposed on 0.0.0.0 by default - Absence of HostConfig in a create payload sometimes causes an error at the engine level - In Swarm, volume names are prefixed by "/" - When testing against Swarm, the default network driver is overlay - Ensure custom test networks are always attachable - Handle Swarm network names - Some params moved to host config in recent (1.21+) version - Conditional test skips for Swarm environments Signed-off-by: Joffrey F --- compose/service.py | 6 + tests/acceptance/cli_test.py | 147 ++++++++++-------- .../docker-compose.override.yml | 7 +- .../override-files/docker-compose.yml | 10 +- tests/fixtures/override-files/extra.yml | 9 +- tests/helpers.py | 35 +++++ tests/integration/project_test.py | 134 +++++++++------- tests/integration/service_test.py | 90 +++++++---- tests/integration/state_test.py | 2 +- tests/integration/testcases.py | 15 +- tests/integration/volume_test.py | 21 ++- 11 files changed, 316 insertions(+), 160 deletions(-) diff --git a/compose/service.py b/compose/service.py index dcbbe251e..03c41ce67 100644 --- a/compose/service.py +++ b/compose/service.py @@ -56,7 +56,9 @@ HOST_CONFIG_KEYS = [ 'cpu_count', 'cpu_percent', 'cpu_quota', + 'cpu_shares', 'cpus', + 'cpuset', 'devices', 'dns', 'dns_search', @@ -83,6 +85,7 @@ HOST_CONFIG_KEYS = [ 'sysctls', 'userns_mode', 'volumes_from', + 'volume_driver', ] CONDITION_STARTED = 'service_started' @@ -848,6 +851,9 @@ class Service(object): cpu_count=options.get('cpu_count'), cpu_percent=options.get('cpu_percent'), nano_cpus=nano_cpus, + volume_driver=options.get('volume_driver'), + cpuset_cpus=options.get('cpuset'), + cpu_shares=options.get('cpu_shares'), ) def get_secret_volumes(self): diff --git a/tests/acceptance/cli_test.py b/tests/acceptance/cli_test.py index 9a1f5364b..ba0b53888 100644 --- a/tests/acceptance/cli_test.py +++ b/tests/acceptance/cli_test.py @@ -20,6 +20,8 @@ from docker import errors from .. import mock from ..helpers import create_host_file +from ..helpers import is_cluster +from ..helpers import no_cluster from compose.cli.command import get_project from compose.config.errors import DuplicateOverrideFileFound from compose.container import Container @@ -28,6 +30,7 @@ from compose.utils import nanoseconds_from_time_seconds from tests.integration.testcases import DockerClientTestCase from tests.integration.testcases import get_links from tests.integration.testcases import pull_busybox +from tests.integration.testcases import SWARM_SKIP_RM_VOLUMES from tests.integration.testcases import v2_1_only from tests.integration.testcases import v2_only from tests.integration.testcases import v3_only @@ -68,7 +71,8 @@ def wait_on_condition(condition, delay=0.1, timeout=40): def kill_service(service): for container in service.containers(): - container.kill() + if container.is_running: + container.kill() class ContainerCountCondition(object): @@ -78,7 +82,7 @@ class ContainerCountCondition(object): self.expected = expected def __call__(self): - return len(self.project.containers()) == self.expected + return len([c for c in self.project.containers() if c.is_running]) == self.expected def __str__(self): return "waiting for counter count == %s" % self.expected @@ -116,11 +120,14 @@ class CLITestCase(DockerClientTestCase): for container in self.project.containers(stopped=True, one_off=OneOffFilter.only): container.remove(force=True) - networks = self.client.networks() for n in networks: - if n['Name'].startswith('{}_'.format(self.project.name)): + if n['Name'].split('/')[-1].startswith('{}_'.format(self.project.name)): self.client.remove_network(n['Name']) + volumes = self.client.volumes().get('Volumes') or [] + for v in volumes: + if v['Name'].split('/')[-1].startswith('{}_'.format(self.project.name)): + self.client.remove_volume(v['Name']) if hasattr(self, '_project'): del self._project @@ -175,7 +182,10 @@ class CLITestCase(DockerClientTestCase): def test_host_not_reachable_volumes_from_container(self): self.base_dir = 'tests/fixtures/volumes-from-container' - container = self.client.create_container('busybox', 'true', name='composetest_data_container') + container = self.client.create_container( + 'busybox', 'true', name='composetest_data_container', + host_config={} + ) self.addCleanup(self.client.remove_container, container) result = self.dispatch(['-H=tcp://doesnotexist:8000', 'ps'], returncode=1) @@ -545,42 +555,48 @@ class CLITestCase(DockerClientTestCase): self.dispatch(['create']) service = self.project.get_service('simple') another = self.project.get_service('another') - self.assertEqual(len(service.containers()), 0) - self.assertEqual(len(another.containers()), 0) - self.assertEqual(len(service.containers(stopped=True)), 1) - self.assertEqual(len(another.containers(stopped=True)), 1) + service_containers = service.containers(stopped=True) + another_containers = another.containers(stopped=True) + assert len(service_containers) == 1 + assert len(another_containers) == 1 + assert not service_containers[0].is_running + assert not another_containers[0].is_running def test_create_with_force_recreate(self): self.dispatch(['create'], None) service = self.project.get_service('simple') - self.assertEqual(len(service.containers()), 0) - self.assertEqual(len(service.containers(stopped=True)), 1) + service_containers = service.containers(stopped=True) + assert len(service_containers) == 1 + assert not service_containers[0].is_running old_ids = [c.id for c in service.containers(stopped=True)] self.dispatch(['create', '--force-recreate'], None) - self.assertEqual(len(service.containers()), 0) - self.assertEqual(len(service.containers(stopped=True)), 1) + service_containers = service.containers(stopped=True) + assert len(service_containers) == 1 + assert not service_containers[0].is_running - new_ids = [c.id for c in service.containers(stopped=True)] + new_ids = [c.id for c in service_containers] - self.assertNotEqual(old_ids, new_ids) + assert old_ids != new_ids def test_create_with_no_recreate(self): self.dispatch(['create'], None) service = self.project.get_service('simple') - self.assertEqual(len(service.containers()), 0) - self.assertEqual(len(service.containers(stopped=True)), 1) + service_containers = service.containers(stopped=True) + assert len(service_containers) == 1 + assert not service_containers[0].is_running old_ids = [c.id for c in service.containers(stopped=True)] self.dispatch(['create', '--no-recreate'], None) - self.assertEqual(len(service.containers()), 0) - self.assertEqual(len(service.containers(stopped=True)), 1) + service_containers = service.containers(stopped=True) + assert len(service_containers) == 1 + assert not service_containers[0].is_running - new_ids = [c.id for c in service.containers(stopped=True)] + new_ids = [c.id for c in service_containers] - self.assertEqual(old_ids, new_ids) + assert old_ids == new_ids def test_run_one_off_with_volume(self): self.base_dir = 'tests/fixtures/simple-composefile-volume-ready' @@ -687,7 +703,7 @@ class CLITestCase(DockerClientTestCase): network_name = self.project.networks.networks['default'].full_name networks = self.client.networks(names=[network_name]) self.assertEqual(len(networks), 1) - self.assertEqual(networks[0]['Driver'], 'bridge') + assert networks[0]['Driver'] == 'bridge' if not is_cluster(self.client) else 'overlay' assert 'com.docker.network.bridge.enable_icc' not in networks[0]['Options'] network = self.client.inspect_network(networks[0]['Id']) @@ -733,11 +749,11 @@ class CLITestCase(DockerClientTestCase): networks = [ n for n in self.client.networks() - if n['Name'].startswith('{}_'.format(self.project.name)) + if n['Name'].split('/')[-1].startswith('{}_'.format(self.project.name)) ] # Two networks were created: back and front - assert sorted(n['Name'] for n in networks) == [back_name, front_name] + assert sorted(n['Name'].split('/')[-1] for n in networks) == [back_name, front_name] web_container = self.project.get_service('web').containers()[0] back_aliases = web_container.get( @@ -761,11 +777,11 @@ class CLITestCase(DockerClientTestCase): networks = [ n for n in self.client.networks() - if n['Name'].startswith('{}_'.format(self.project.name)) + if n['Name'].split('/')[-1].startswith('{}_'.format(self.project.name)) ] # One network was created: internal - assert sorted(n['Name'] for n in networks) == [internal_net] + assert sorted(n['Name'].split('/')[-1] for n in networks) == [internal_net] assert networks[0]['Internal'] is True @@ -780,11 +796,11 @@ class CLITestCase(DockerClientTestCase): networks = [ n for n in self.client.networks() - if n['Name'].startswith('{}_'.format(self.project.name)) + if n['Name'].split('/')[-1].startswith('{}_'.format(self.project.name)) ] # One networks was created: front - assert sorted(n['Name'] for n in networks) == [static_net] + assert sorted(n['Name'].split('/')[-1] for n in networks) == [static_net] web_container = self.project.get_service('web').containers()[0] ipam_config = web_container.get( @@ -803,11 +819,11 @@ class CLITestCase(DockerClientTestCase): networks = [ n for n in self.client.networks() - if n['Name'].startswith('{}_'.format(self.project.name)) + if n['Name'].split('/')[-1].startswith('{}_'.format(self.project.name)) ] # Two networks were created: back and front - assert sorted(n['Name'] for n in networks) == [back_name, front_name] + assert sorted(n['Name'].split('/')[-1] for n in networks) == [back_name, front_name] back_network = [n for n in networks if n['Name'] == back_name][0] front_network = [n for n in networks if n['Name'] == front_name][0] @@ -847,8 +863,12 @@ class CLITestCase(DockerClientTestCase): assert 'Service "web" uses an undefined network "foo"' in result.stderr @v2_only() + @no_cluster('container networks not supported in Swarm') def test_up_with_network_mode(self): - c = self.client.create_container('busybox', 'top', name='composetest_network_mode_container') + c = self.client.create_container( + 'busybox', 'top', name='composetest_network_mode_container', + host_config={} + ) self.addCleanup(self.client.remove_container, c, force=True) self.client.start(c) container_mode_source = 'container:{}'.format(c['Id']) @@ -862,7 +882,7 @@ class CLITestCase(DockerClientTestCase): networks = [ n for n in self.client.networks() - if n['Name'].startswith('{}_'.format(self.project.name)) + if n['Name'].split('/')[-1].startswith('{}_'.format(self.project.name)) ] assert not networks @@ -899,7 +919,7 @@ class CLITestCase(DockerClientTestCase): network_names = ['{}_{}'.format(self.project.name, n) for n in ['foo', 'bar']] for name in network_names: - self.client.create_network(name) + self.client.create_network(name, attachable=True) self.dispatch(['-f', filename, 'up', '-d']) container = self.project.containers()[0] @@ -917,12 +937,12 @@ class CLITestCase(DockerClientTestCase): networks = [ n['Name'] for n in self.client.networks() - if n['Name'].startswith('{}_'.format(self.project.name)) + if n['Name'].split('/')[-1].startswith('{}_'.format(self.project.name)) ] assert not networks network_name = 'composetest_external_network' - self.client.create_network(network_name) + self.client.create_network(network_name, attachable=True) self.dispatch(['-f', filename, 'up', '-d']) container = self.project.containers()[0] @@ -941,10 +961,10 @@ class CLITestCase(DockerClientTestCase): networks = [ n for n in self.client.networks() - if n['Name'].startswith('{}_'.format(self.project.name)) + if n['Name'].split('/')[-1].startswith('{}_'.format(self.project.name)) ] - assert [n['Name'] for n in networks] == [network_with_label] + assert [n['Name'].split('/')[-1] for n in networks] == [network_with_label] assert 'label_key' in networks[0]['Labels'] assert networks[0]['Labels']['label_key'] == 'label_val' @@ -961,10 +981,10 @@ class CLITestCase(DockerClientTestCase): volumes = [ v for v in self.client.volumes().get('Volumes', []) - if v['Name'].startswith('{}_'.format(self.project.name)) + if v['Name'].split('/')[-1].startswith('{}_'.format(self.project.name)) ] - assert [v['Name'] for v in volumes] == [volume_with_label] + assert set([v['Name'].split('/')[-1] for v in volumes]) == set([volume_with_label]) assert 'label_key' in volumes[0]['Labels'] assert volumes[0]['Labels']['label_key'] == 'label_val' @@ -975,7 +995,7 @@ class CLITestCase(DockerClientTestCase): network_names = [ n['Name'] for n in self.client.networks() - if n['Name'].startswith('{}_'.format(self.project.name)) + if n['Name'].split('/')[-1].startswith('{}_'.format(self.project.name)) ] assert network_names == [] @@ -1010,6 +1030,7 @@ class CLITestCase(DockerClientTestCase): assert "Unsupported config option for services.bar: 'net'" in result.stderr + @no_cluster("Legacy networking not supported on Swarm") def test_up_with_net_v1(self): self.base_dir = 'tests/fixtures/net-container' self.dispatch(['up', '-d'], None) @@ -1261,6 +1282,7 @@ class CLITestCase(DockerClientTestCase): [u'/bin/true'], ) + @py.test.mark.skipif(SWARM_SKIP_RM_VOLUMES, reason='Swarm DELETE /containers/ bug') def test_run_rm(self): self.base_dir = 'tests/fixtures/volume' proc = start_process(self.base_dir, ['run', '--rm', 'test']) @@ -1274,7 +1296,7 @@ class CLITestCase(DockerClientTestCase): mounts = containers[0].get('Mounts') for mount in mounts: if mount['Destination'] == '/container-path': - anonymousName = mount['Name'] + anonymous_name = mount['Name'] break os.kill(proc.pid, signal.SIGINT) wait_on_process(proc, 1) @@ -1287,9 +1309,11 @@ class CLITestCase(DockerClientTestCase): if volume.internal == '/container-named-path': name = volume.external break - volumeNames = [v['Name'] for v in volumes] - assert name in volumeNames - assert anonymousName not in volumeNames + volume_names = [v['Name'].split('/')[-1] for v in volumes] + assert name in volume_names + if not is_cluster(self.client): + # The `-v` flag for `docker rm` in Swarm seems to be broken + assert anonymous_name not in volume_names def test_run_service_with_dockerfile_entrypoint(self): self.base_dir = 'tests/fixtures/entrypoint-dockerfile' @@ -1411,11 +1435,10 @@ class CLITestCase(DockerClientTestCase): container.stop() # check the ports - self.assertNotEqual(port_random, None) - self.assertIn("0.0.0.0", port_random) - self.assertEqual(port_assigned, "0.0.0.0:49152") - self.assertEqual(port_range[0], "0.0.0.0:49153") - self.assertEqual(port_range[1], "0.0.0.0:49154") + assert port_random is not None + assert port_assigned.endswith(':49152') + assert port_range[0].endswith(':49153') + assert port_range[1].endswith(':49154') def test_run_service_with_explicitly_mapped_ports(self): # create one off container @@ -1431,8 +1454,8 @@ class CLITestCase(DockerClientTestCase): container.stop() # check the ports - self.assertEqual(port_short, "0.0.0.0:30000") - self.assertEqual(port_full, "0.0.0.0:30001") + assert port_short.endswith(':30000') + assert port_full.endswith(':30001') def test_run_service_with_explicitly_mapped_ip_ports(self): # create one off container @@ -1953,9 +1976,9 @@ class CLITestCase(DockerClientTestCase): result = self.dispatch(['port', 'simple', str(number)]) return result.stdout.rstrip() - self.assertEqual(get_port(3000), container.get_local_port(3000)) - self.assertEqual(get_port(3001), "0.0.0.0:49152") - self.assertEqual(get_port(3002), "0.0.0.0:49153") + assert get_port(3000) == container.get_local_port(3000) + assert ':49152' in get_port(3001) + assert ':49153' in get_port(3002) def test_expanded_port(self): self.base_dir = 'tests/fixtures/ports-composefile' @@ -1966,9 +1989,9 @@ class CLITestCase(DockerClientTestCase): result = self.dispatch(['port', 'simple', str(number)]) return result.stdout.rstrip() - self.assertEqual(get_port(3000), container.get_local_port(3000)) - self.assertEqual(get_port(3001), "0.0.0.0:49152") - self.assertEqual(get_port(3002), "0.0.0.0:49153") + assert get_port(3000) == container.get_local_port(3000) + assert ':49152' in get_port(3001) + assert ':49153' in get_port(3002) def test_port_with_scale(self): self.base_dir = 'tests/fixtures/ports-composefile-scale' @@ -2021,12 +2044,14 @@ class CLITestCase(DockerClientTestCase): assert len(lines) == 2 container, = self.project.containers() - expected_template = ( - ' container {} {} (image=busybox:latest, ' - 'name=simplecomposefile_simple_1)') + expected_template = ' container {} {}' + expected_meta_info = ['image=busybox:latest', 'name=simplecomposefile_simple_1'] assert expected_template.format('create', container.id) in lines[0] assert expected_template.format('start', container.id) in lines[1] + for line in lines: + for info in expected_meta_info: + assert info in line assert has_timestamp(lines[0]) @@ -2069,7 +2094,6 @@ class CLITestCase(DockerClientTestCase): 'docker-compose.yml', 'docker-compose.override.yml', 'extra.yml', - ] self._project = get_project(self.base_dir, config_paths) self.dispatch( @@ -2086,7 +2110,6 @@ class CLITestCase(DockerClientTestCase): web, other, db = containers self.assertEqual(web.human_readable_command, 'top') - self.assertTrue({'db', 'other'} <= set(get_links(web))) self.assertEqual(db.human_readable_command, 'top') self.assertEqual(other.human_readable_command, 'top') diff --git a/tests/fixtures/override-files/docker-compose.override.yml b/tests/fixtures/override-files/docker-compose.override.yml index a03d3d6f5..b2c540601 100644 --- a/tests/fixtures/override-files/docker-compose.override.yml +++ b/tests/fixtures/override-files/docker-compose.override.yml @@ -1,6 +1,7 @@ - -web: +version: '2.2' +services: + web: command: "top" -db: + db: command: "top" diff --git a/tests/fixtures/override-files/docker-compose.yml b/tests/fixtures/override-files/docker-compose.yml index 8eb43ddb0..6c3d4e172 100644 --- a/tests/fixtures/override-files/docker-compose.yml +++ b/tests/fixtures/override-files/docker-compose.yml @@ -1,10 +1,10 @@ - -web: +version: '2.2' +services: + web: image: busybox:latest command: "sleep 200" - links: + depends_on: - db - -db: + db: image: busybox:latest command: "sleep 200" diff --git a/tests/fixtures/override-files/extra.yml b/tests/fixtures/override-files/extra.yml index 7b3ade9c2..492c37952 100644 --- a/tests/fixtures/override-files/extra.yml +++ b/tests/fixtures/override-files/extra.yml @@ -1,9 +1,10 @@ - -web: - links: +version: '2.2' +services: + web: + depends_on: - db - other -other: + other: image: busybox:latest command: "top" diff --git a/tests/helpers.py b/tests/helpers.py index 59efd2557..662353c93 100644 --- a/tests/helpers.py +++ b/tests/helpers.py @@ -1,8 +1,12 @@ from __future__ import absolute_import from __future__ import unicode_literals +import functools import os +from docker.errors import APIError +from pytest import skip + from compose.config.config import ConfigDetails from compose.config.config import ConfigFile from compose.config.config import load @@ -44,3 +48,34 @@ def create_host_file(client, filename): "Container exited with code {}:\n{}".format(exitcode, output)) finally: client.remove_container(container, force=True) + + +def is_cluster(client): + nodes = None + + def get_nodes_number(): + try: + return len(client.nodes()) + except APIError: + # If the Engine is not part of a Swarm, the SDK will raise + # an APIError + return 0 + + if nodes is None: + # Only make the API call if the value hasn't been cached yet + nodes = get_nodes_number() + + return nodes > 1 + + +def no_cluster(reason): + def decorator(f): + @functools.wraps(f) + def wrapper(self, *args, **kwargs): + if is_cluster(self.client): + skip("Test will not be run in cluster mode: %s" % reason) + return + return f(self, *args, **kwargs) + return wrapper + + return decorator diff --git a/tests/integration/project_test.py b/tests/integration/project_test.py index 6c5f719ed..6731f25dd 100644 --- a/tests/integration/project_test.py +++ b/tests/integration/project_test.py @@ -6,12 +6,16 @@ import random import py import pytest +from docker.errors import APIError from docker.errors import NotFound from .. import mock from ..helpers import build_config as load_config from ..helpers import create_host_file +from ..helpers import is_cluster +from ..helpers import no_cluster from .testcases import DockerClientTestCase +from .testcases import SWARM_SKIP_CONTAINERS_ALL from compose.config import config from compose.config import ConfigurationError from compose.config import types @@ -57,6 +61,20 @@ class ProjectTest(DockerClientTestCase): containers = project.containers() self.assertEqual(len(containers), 2) + @pytest.mark.skipif(SWARM_SKIP_CONTAINERS_ALL, reason='Swarm /containers/json bug') + def test_containers_stopped(self): + web = self.create_service('web') + db = self.create_service('db') + project = Project('composetest', [web, db], self.client) + + project.up() + assert len(project.containers()) == 2 + assert len(project.containers(stopped=True)) == 2 + + project.stop() + assert len(project.containers()) == 0 + assert len(project.containers(stopped=True)) == 2 + def test_containers_with_service_names(self): web = self.create_service('web') db = self.create_service('db') @@ -110,6 +128,7 @@ class ProjectTest(DockerClientTestCase): volumes=['/var/data'], name='composetest_data_container', labels={LABEL_PROJECT: 'composetest'}, + host_config={}, ) project = Project.from_config( name='composetest', @@ -125,6 +144,7 @@ class ProjectTest(DockerClientTestCase): self.assertEqual(db._get_volumes_from(), [data_container.id + ':rw']) @v2_only() + @no_cluster('container networks not supported in Swarm') def test_network_mode_from_service(self): project = Project.from_config( name='composetest', @@ -152,6 +172,7 @@ class ProjectTest(DockerClientTestCase): self.assertEqual(web.network_mode.mode, 'container:' + net.containers()[0].id) @v2_only() + @no_cluster('container networks not supported in Swarm') def test_network_mode_from_container(self): def get_project(): return Project.from_config( @@ -179,6 +200,7 @@ class ProjectTest(DockerClientTestCase): name='composetest_net_container', command='top', labels={LABEL_PROJECT: 'composetest'}, + host_config={}, ) net_container.start() @@ -188,6 +210,7 @@ class ProjectTest(DockerClientTestCase): web = project.get_service('web') self.assertEqual(web.network_mode.mode, 'container:' + net_container.id) + @no_cluster('container networks not supported in Swarm') def test_net_from_service_v1(self): project = Project.from_config( name='composetest', @@ -211,6 +234,7 @@ class ProjectTest(DockerClientTestCase): net = project.get_service('net') self.assertEqual(web.network_mode.mode, 'container:' + net.containers()[0].id) + @no_cluster('container networks not supported in Swarm') def test_net_from_container_v1(self): def get_project(): return Project.from_config( @@ -235,6 +259,7 @@ class ProjectTest(DockerClientTestCase): name='composetest_net_container', command='top', labels={LABEL_PROJECT: 'composetest'}, + host_config={}, ) net_container.start() @@ -260,12 +285,12 @@ class ProjectTest(DockerClientTestCase): project.start(service_names=['web']) self.assertEqual( - set(c.name for c in project.containers()), + set(c.name for c in project.containers() if c.is_running), set([web_container_1.name, web_container_2.name])) project.start() self.assertEqual( - set(c.name for c in project.containers()), + set(c.name for c in project.containers() if c.is_running), set([web_container_1.name, web_container_2.name, db_container.name])) project.pause(service_names=['web']) @@ -285,10 +310,12 @@ class ProjectTest(DockerClientTestCase): self.assertEqual(len([c.name for c in project.containers() if c.is_paused]), 0) project.stop(service_names=['web'], timeout=1) - self.assertEqual(set(c.name for c in project.containers()), set([db_container.name])) + self.assertEqual( + set(c.name for c in project.containers() if c.is_running), set([db_container.name]) + ) project.kill(service_names=['db']) - self.assertEqual(len(project.containers()), 0) + self.assertEqual(len([c for c in project.containers() if c.is_running]), 0) self.assertEqual(len(project.containers(stopped=True)), 3) project.remove_stopped(service_names=['web']) @@ -303,11 +330,13 @@ class ProjectTest(DockerClientTestCase): project = Project('composetest', [web, db], self.client) project.create(['db']) - self.assertEqual(len(project.containers()), 0) - self.assertEqual(len(project.containers(stopped=True)), 1) - self.assertEqual(len(db.containers()), 0) - self.assertEqual(len(db.containers(stopped=True)), 1) - self.assertEqual(len(web.containers(stopped=True)), 0) + containers = project.containers(stopped=True) + assert len(containers) == 1 + assert not containers[0].is_running + db_containers = db.containers(stopped=True) + assert len(db_containers) == 1 + assert not db_containers[0].is_running + assert len(web.containers(stopped=True)) == 0 def test_create_twice(self): web = self.create_service('web') @@ -316,12 +345,14 @@ class ProjectTest(DockerClientTestCase): project.create(['db', 'web']) project.create(['db', 'web']) - self.assertEqual(len(project.containers()), 0) - self.assertEqual(len(project.containers(stopped=True)), 2) - self.assertEqual(len(db.containers()), 0) - self.assertEqual(len(db.containers(stopped=True)), 1) - self.assertEqual(len(web.containers()), 0) - self.assertEqual(len(web.containers(stopped=True)), 1) + containers = project.containers(stopped=True) + assert len(containers) == 2 + db_containers = db.containers(stopped=True) + assert len(db_containers) == 1 + assert not db_containers[0].is_running + web_containers = web.containers(stopped=True) + assert len(web_containers) == 1 + assert not web_containers[0].is_running def test_create_with_links(self): db = self.create_service('db') @@ -329,12 +360,11 @@ class ProjectTest(DockerClientTestCase): project = Project('composetest', [db, web], self.client) project.create(['web']) - self.assertEqual(len(project.containers()), 0) - self.assertEqual(len(project.containers(stopped=True)), 2) - self.assertEqual(len(db.containers()), 0) - self.assertEqual(len(db.containers(stopped=True)), 1) - self.assertEqual(len(web.containers()), 0) - self.assertEqual(len(web.containers(stopped=True)), 1) + # self.assertEqual(len(project.containers()), 0) + assert len(project.containers(stopped=True)) == 2 + assert not [c for c in project.containers(stopped=True) if c.is_running] + assert len(db.containers(stopped=True)) == 1 + assert len(web.containers(stopped=True)) == 1 def test_create_strategy_always(self): db = self.create_service('db') @@ -343,11 +373,11 @@ class ProjectTest(DockerClientTestCase): old_id = project.containers(stopped=True)[0].id project.create(['db'], strategy=ConvergenceStrategy.always) - self.assertEqual(len(project.containers()), 0) - self.assertEqual(len(project.containers(stopped=True)), 1) + assert len(project.containers(stopped=True)) == 1 db_container = project.containers(stopped=True)[0] - self.assertNotEqual(db_container.id, old_id) + assert not db_container.is_running + assert db_container.id != old_id def test_create_strategy_never(self): db = self.create_service('db') @@ -356,11 +386,11 @@ class ProjectTest(DockerClientTestCase): old_id = project.containers(stopped=True)[0].id project.create(['db'], strategy=ConvergenceStrategy.never) - self.assertEqual(len(project.containers()), 0) - self.assertEqual(len(project.containers(stopped=True)), 1) + assert len(project.containers(stopped=True)) == 1 db_container = project.containers(stopped=True)[0] - self.assertEqual(db_container.id, old_id) + assert not db_container.is_running + assert db_container.id == old_id def test_project_up(self): web = self.create_service('web') @@ -550,8 +580,8 @@ class ProjectTest(DockerClientTestCase): self.assertEqual(len(project.containers(stopped=True)), 2) self.assertEqual(len(project.get_service('web').containers()), 0) self.assertEqual(len(project.get_service('db').containers()), 1) - self.assertEqual(len(project.get_service('data').containers()), 0) self.assertEqual(len(project.get_service('data').containers(stopped=True)), 1) + assert not project.get_service('data').containers(stopped=True)[0].is_running self.assertEqual(len(project.get_service('console').containers()), 0) def test_project_up_recreate_with_tmpfs_volume(self): @@ -737,10 +767,10 @@ class ProjectTest(DockerClientTestCase): "com.docker.compose.network.test": "9-29-045" } - @v2_only() + @v2_1_only() def test_up_with_network_static_addresses(self): config_data = build_config( - version=V2_0, + version=V2_1, services=[{ 'name': 'web', 'image': 'busybox:latest', @@ -766,7 +796,8 @@ class ProjectTest(DockerClientTestCase): {"subnet": "fe80::/64", "gateway": "fe80::1001:1"} ] - } + }, + 'enable_ipv6': True, } } ) @@ -777,13 +808,8 @@ class ProjectTest(DockerClientTestCase): ) project.up(detached=True) - network = self.client.networks(names=['static_test'])[0] service_container = project.get_service('web').containers()[0] - assert network['Options'] == { - "com.docker.network.enable_ipv6": "true" - } - IPAMConfig = (service_container.inspect().get('NetworkSettings', {}). get('Networks', {}).get('composetest_static_test', {}). get('IPAMConfig', {})) @@ -825,7 +851,7 @@ class ProjectTest(DockerClientTestCase): config_data=config_data, ) project.up(detached=True) - network = self.client.networks(names=['static_test'])[0] + network = [n for n in self.client.networks() if 'static_test' in n['Name']][0] service_container = project.get_service('web').containers()[0] assert network['EnableIPv6'] is True @@ -1026,8 +1052,8 @@ class ProjectTest(DockerClientTestCase): project.up() self.assertEqual(len(project.containers()), 1) - volume_data = self.client.inspect_volume(full_vol_name) - self.assertEqual(volume_data['Name'], full_vol_name) + volume_data = self.get_volume_data(full_vol_name) + assert volume_data['Name'].split('/')[-1] == full_vol_name self.assertEqual(volume_data['Driver'], 'local') @v2_1_only() @@ -1062,10 +1088,12 @@ class ProjectTest(DockerClientTestCase): volumes = [ v for v in self.client.volumes().get('Volumes', []) - if v['Name'].startswith('composetest_') + if v['Name'].split('/')[-1].startswith('composetest_') ] - assert [v['Name'] for v in volumes] == ['composetest_{}'.format(volume_name)] + assert set([v['Name'].split('/')[-1] for v in volumes]) == set( + ['composetest_{}'.format(volume_name)] + ) assert 'label_key' in volumes[0]['Labels'] assert volumes[0]['Labels']['label_key'] == 'label_val' @@ -1205,8 +1233,8 @@ class ProjectTest(DockerClientTestCase): ) project.volumes.initialize() - volume_data = self.client.inspect_volume(full_vol_name) - assert volume_data['Name'] == full_vol_name + volume_data = self.get_volume_data(full_vol_name) + assert volume_data['Name'].split('/')[-1] == full_vol_name assert volume_data['Driver'] == 'local' @v2_only() @@ -1229,8 +1257,8 @@ class ProjectTest(DockerClientTestCase): ) project.up() - volume_data = self.client.inspect_volume(full_vol_name) - self.assertEqual(volume_data['Name'], full_vol_name) + volume_data = self.get_volume_data(full_vol_name) + assert volume_data['Name'].split('/')[-1] == full_vol_name self.assertEqual(volume_data['Driver'], 'local') @v3_only() @@ -1287,10 +1315,11 @@ class ProjectTest(DockerClientTestCase): name='composetest', config_data=config_data, client=self.client ) - with self.assertRaises(config.ConfigurationError): + with self.assertRaises(APIError if is_cluster(self.client) else config.ConfigurationError): project.volumes.initialize() @v2_only() + @no_cluster('inspect volume by name defect on Swarm Classic') def test_initialize_volumes_updated_driver(self): vol_name = '{0:x}'.format(random.getrandbits(32)) full_vol_name = 'composetest_{0}'.format(vol_name) @@ -1310,8 +1339,8 @@ class ProjectTest(DockerClientTestCase): ) project.volumes.initialize() - volume_data = self.client.inspect_volume(full_vol_name) - self.assertEqual(volume_data['Name'], full_vol_name) + volume_data = self.get_volume_data(full_vol_name) + assert volume_data['Name'].split('/')[-1] == full_vol_name self.assertEqual(volume_data['Driver'], 'local') config_data = config_data._replace( @@ -1348,8 +1377,8 @@ class ProjectTest(DockerClientTestCase): ) project.volumes.initialize() - volume_data = self.client.inspect_volume(full_vol_name) - self.assertEqual(volume_data['Name'], full_vol_name) + volume_data = self.get_volume_data(full_vol_name) + assert volume_data['Name'].split('/')[-1] == full_vol_name self.assertEqual(volume_data['Driver'], 'local') config_data = config_data._replace( @@ -1361,11 +1390,12 @@ class ProjectTest(DockerClientTestCase): client=self.client ) project.volumes.initialize() - volume_data = self.client.inspect_volume(full_vol_name) - self.assertEqual(volume_data['Name'], full_vol_name) + volume_data = self.get_volume_data(full_vol_name) + assert volume_data['Name'].split('/')[-1] == full_vol_name self.assertEqual(volume_data['Driver'], 'local') @v2_only() + @no_cluster('inspect volume by name defect on Swarm Classic') def test_initialize_volumes_external_volumes(self): # Use composetest_ prefix so it gets garbage-collected in tearDown() vol_name = 'composetest_{0:x}'.format(random.getrandbits(32)) diff --git a/tests/integration/service_test.py b/tests/integration/service_test.py index 178df1323..baf21af3c 100644 --- a/tests/integration/service_test.py +++ b/tests/integration/service_test.py @@ -13,9 +13,13 @@ from six import StringIO from six import text_type from .. import mock +from ..helpers import is_cluster +from ..helpers import no_cluster from .testcases import DockerClientTestCase from .testcases import get_links from .testcases import pull_busybox +from .testcases import SWARM_SKIP_CONTAINERS_ALL +from .testcases import SWARM_SKIP_CPU_SHARES from compose import __version__ from compose.config.types import VolumeFromSpec from compose.config.types import VolumeSpec @@ -100,6 +104,7 @@ class ServiceTest(DockerClientTestCase): service.start_container(container) self.assertEqual('foodriver', container.get('HostConfig.VolumeDriver')) + @pytest.mark.skipif(SWARM_SKIP_CPU_SHARES, reason='Swarm --cpu-shares bug') def test_create_container_with_cpu_shares(self): service = self.create_service('db', cpu_shares=73) container = service.create_container() @@ -151,6 +156,7 @@ class ServiceTest(DockerClientTestCase): service.start_container(container) assert container.get('HostConfig.Init') is True + @pytest.mark.xfail(True, reason='Option has been removed in Engine 17.06.0') def test_create_container_with_init_path(self): self.require_api_version('1.25') docker_init_path = find_executable('docker-init') @@ -249,6 +255,7 @@ class ServiceTest(DockerClientTestCase): 'busybox', 'true', volumes={container_path: {}}, labels={'com.docker.compose.test_image': 'true'}, + host_config={} ) image = self.client.commit(tmp_container)['Id'] @@ -278,6 +285,7 @@ class ServiceTest(DockerClientTestCase): image='busybox:latest', command=["top"], labels={LABEL_PROJECT: 'composetest'}, + host_config={}, ) host_service = self.create_service( 'host', @@ -321,9 +329,15 @@ class ServiceTest(DockerClientTestCase): self.assertIn('FOO=2', new_container.get('Config.Env')) self.assertEqual(new_container.name, 'composetest_db_1') self.assertEqual(new_container.get_mount('/etc')['Source'], volume_path) - self.assertIn( - 'affinity:container==%s' % old_container.id, - new_container.get('Config.Env')) + if not is_cluster(self.client): + assert ( + 'affinity:container==%s' % old_container.id in + new_container.get('Config.Env') + ) + else: + # In Swarm, the env marker is consumed and the container should be deployed + # on the same node. + assert old_container.get('Node.Name') == new_container.get('Node.Name') self.assertEqual(len(self.client.containers(all=True)), num_containers_before) self.assertNotEqual(old_container.id, new_container.id) @@ -350,8 +364,13 @@ class ServiceTest(DockerClientTestCase): ConvergencePlan('recreate', [orig_container])) assert new_container.get_mount('/etc')['Source'] == volume_path - assert ('affinity:container==%s' % orig_container.id in - new_container.get('Config.Env')) + if not is_cluster(self.client): + assert ('affinity:container==%s' % orig_container.id in + new_container.get('Config.Env')) + else: + # In Swarm, the env marker is consumed and the container should be deployed + # on the same node. + assert orig_container.get('Node.Name') == new_container.get('Node.Name') orig_container = new_container @@ -464,18 +483,21 @@ class ServiceTest(DockerClientTestCase): ) containers = service.execute_convergence_plan(ConvergencePlan('create', []), start=False) - self.assertEqual(len(service.containers()), 0) - self.assertEqual(len(service.containers(stopped=True)), 1) + service_containers = service.containers(stopped=True) + assert len(service_containers) == 1 + assert not service_containers[0].is_running containers = service.execute_convergence_plan( ConvergencePlan('recreate', containers), start=False) - self.assertEqual(len(service.containers()), 0) - self.assertEqual(len(service.containers(stopped=True)), 1) + service_containers = service.containers(stopped=True) + assert len(service_containers) == 1 + assert not service_containers[0].is_running service.execute_convergence_plan(ConvergencePlan('start', containers), start=False) - self.assertEqual(len(service.containers()), 0) - self.assertEqual(len(service.containers(stopped=True)), 1) + service_containers = service.containers(stopped=True) + assert len(service_containers) == 1 + assert not service_containers[0].is_running def test_start_container_passes_through_options(self): db = self.create_service('db') @@ -487,6 +509,7 @@ class ServiceTest(DockerClientTestCase): create_and_start_container(db) self.assertEqual(db.containers()[0].environment['FOO'], 'BAR') + @no_cluster('No legacy links support in Swarm') def test_start_container_creates_links(self): db = self.create_service('db') web = self.create_service('web', links=[(db, None)]) @@ -503,6 +526,7 @@ class ServiceTest(DockerClientTestCase): 'db']) ) + @no_cluster('No legacy links support in Swarm') def test_start_container_creates_links_with_names(self): db = self.create_service('db') web = self.create_service('web', links=[(db, 'custom_link_name')]) @@ -519,6 +543,7 @@ class ServiceTest(DockerClientTestCase): 'custom_link_name']) ) + @no_cluster('No legacy links support in Swarm') def test_start_container_with_external_links(self): db = self.create_service('db') web = self.create_service('web', external_links=['composetest_db_1', @@ -537,6 +562,7 @@ class ServiceTest(DockerClientTestCase): 'db_3']), ) + @no_cluster('No legacy links support in Swarm') def test_start_normal_container_does_not_create_links_to_its_own_service(self): db = self.create_service('db') @@ -546,6 +572,7 @@ class ServiceTest(DockerClientTestCase): c = create_and_start_container(db) self.assertEqual(set(get_links(c)), set([])) + @no_cluster('No legacy links support in Swarm') def test_start_one_off_container_creates_links_to_its_own_service(self): db = self.create_service('db') @@ -572,7 +599,7 @@ class ServiceTest(DockerClientTestCase): container = create_and_start_container(service) container.wait() self.assertIn(b'success', container.logs()) - self.assertEqual(len(self.client.images(name='composetest_test')), 1) + assert len(self.client.images(name='composetest_test')) >= 1 def test_start_container_uses_tagged_image_if_it_exists(self): self.check_build('tests/fixtures/simple-dockerfile', tag='composetest_test') @@ -719,20 +746,27 @@ class ServiceTest(DockerClientTestCase): '0.0.0.0:9001:9000/udp', ]) container = create_and_start_container(service).inspect() - self.assertEqual(container['NetworkSettings']['Ports'], { - '8000/tcp': [ - { - 'HostIp': '127.0.0.1', - 'HostPort': '8001', - }, - ], - '9000/udp': [ - { - 'HostIp': '0.0.0.0', - 'HostPort': '9001', - }, - ], - }) + assert container['NetworkSettings']['Ports']['8000/tcp'] == [{ + 'HostIp': '127.0.0.1', + 'HostPort': '8001', + }] + assert container['NetworkSettings']['Ports']['9000/udp'][0]['HostPort'] == '9001' + if not is_cluster(self.client): + assert container['NetworkSettings']['Ports']['9000/udp'][0]['HostIp'] == '0.0.0.0' + # self.assertEqual(container['NetworkSettings']['Ports'], { + # '8000/tcp': [ + # { + # 'HostIp': '127.0.0.1', + # 'HostPort': '8001', + # }, + # ], + # '9000/udp': [ + # { + # 'HostIp': '0.0.0.0', + # 'HostPort': '9001', + # }, + # ], + # }) def test_create_with_image_id(self): # Get image id for the current busybox:latest @@ -760,6 +794,10 @@ class ServiceTest(DockerClientTestCase): service.scale(0) self.assertEqual(len(service.containers()), 0) + @pytest.mark.skipif( + SWARM_SKIP_CONTAINERS_ALL, + reason='Swarm /containers/json bug' + ) def test_scale_with_stopped_containers(self): """ Given there are some stopped containers and scale is called with a diff --git a/tests/integration/state_test.py b/tests/integration/state_test.py index 07b28e784..0dd5f44ad 100644 --- a/tests/integration/state_test.py +++ b/tests/integration/state_test.py @@ -251,7 +251,7 @@ class ServiceStateTest(DockerClientTestCase): container = web.create_container() # update the image - c = self.client.create_container(image, ['touch', '/hello.txt']) + c = self.client.create_container(image, ['touch', '/hello.txt'], host_config={}) self.client.commit(c, repository=repo, tag=tag) self.client.remove_container(c) diff --git a/tests/integration/testcases.py b/tests/integration/testcases.py index 57814872c..1e0d63215 100644 --- a/tests/integration/testcases.py +++ b/tests/integration/testcases.py @@ -8,6 +8,7 @@ from docker.utils import version_lt from pytest import skip from .. import unittest +from ..helpers import is_cluster from compose.cli.docker_client import docker_client from compose.config.config import resolve_environment from compose.config.environment import Environment @@ -21,6 +22,10 @@ from compose.const import LABEL_PROJECT from compose.progress_stream import stream_output from compose.service import Service +SWARM_SKIP_CONTAINERS_ALL = os.environ.get('SWARM_SKIP_CONTAINERS_ALL', '0') != '0' +SWARM_SKIP_CPU_SHARES = os.environ.get('SWARM_SKIP_CPU_SHARES', '0') != '0' +SWARM_SKIP_RM_VOLUMES = os.environ.get('SWARM_SKIP_RM_VOLUMES', '0') != '0' + def pull_busybox(client): client.pull('busybox:latest', stream=False) @@ -97,7 +102,7 @@ class DockerClientTestCase(unittest.TestCase): for i in self.client.images( filters={'label': 'com.docker.compose.test_image'}): - self.client.remove_image(i) + self.client.remove_image(i, force=True) volumes = self.client.volumes().get('Volumes') or [] for v in volumes: @@ -133,3 +138,11 @@ class DockerClientTestCase(unittest.TestCase): api_version = self.client.version()['ApiVersion'] if version_lt(api_version, minimum): skip("API version is too low ({} < {})".format(api_version, minimum)) + + def get_volume_data(self, volume_name): + if not is_cluster(self.client): + return self.client.inspect_volume(volume_name) + + volumes = self.client.volumes(filters={'name': volume_name})['Volumes'] + assert len(volumes) > 0 + return self.client.inspect_volume(volumes[0]['Name']) diff --git a/tests/integration/volume_test.py b/tests/integration/volume_test.py index add169623..772631a5b 100644 --- a/tests/integration/volume_test.py +++ b/tests/integration/volume_test.py @@ -3,6 +3,7 @@ from __future__ import unicode_literals from docker.errors import DockerException +from ..helpers import no_cluster from .testcases import DockerClientTestCase from compose.const import LABEL_PROJECT from compose.const import LABEL_VOLUME @@ -35,26 +36,28 @@ class VolumeTest(DockerClientTestCase): def test_create_volume(self): vol = self.create_volume('volume01') vol.create() - info = self.client.inspect_volume(vol.full_name) - assert info['Name'] == vol.full_name + info = self.get_volume_data(vol.full_name) + assert info['Name'].split('/')[-1] == vol.full_name def test_recreate_existing_volume(self): vol = self.create_volume('volume01') vol.create() - info = self.client.inspect_volume(vol.full_name) - assert info['Name'] == vol.full_name + info = self.get_volume_data(vol.full_name) + assert info['Name'].split('/')[-1] == vol.full_name vol.create() - info = self.client.inspect_volume(vol.full_name) - assert info['Name'] == vol.full_name + info = self.get_volume_data(vol.full_name) + assert info['Name'].split('/')[-1] == vol.full_name + @no_cluster('inspect volume by name defect on Swarm Classic') def test_inspect_volume(self): vol = self.create_volume('volume01') vol.create() info = vol.inspect() assert info['Name'] == vol.full_name + @no_cluster('remove volume by name defect on Swarm Classic') def test_remove_volume(self): vol = Volume(self.client, 'composetest', 'volume01') vol.create() @@ -62,6 +65,7 @@ class VolumeTest(DockerClientTestCase): volumes = self.client.volumes()['Volumes'] assert len([v for v in volumes if v['Name'] == vol.full_name]) == 0 + @no_cluster('inspect volume by name defect on Swarm Classic') def test_external_volume(self): vol = self.create_volume('composetest_volume_ext', external=True) assert vol.external is True @@ -70,6 +74,7 @@ class VolumeTest(DockerClientTestCase): info = vol.inspect() assert info['Name'] == vol.name + @no_cluster('inspect volume by name defect on Swarm Classic') def test_external_aliased_volume(self): alias_name = 'composetest_alias01' vol = self.create_volume('volume01', external=alias_name) @@ -79,24 +84,28 @@ class VolumeTest(DockerClientTestCase): info = vol.inspect() assert info['Name'] == alias_name + @no_cluster('inspect volume by name defect on Swarm Classic') def test_exists(self): vol = self.create_volume('volume01') assert vol.exists() is False vol.create() assert vol.exists() is True + @no_cluster('inspect volume by name defect on Swarm Classic') def test_exists_external(self): vol = self.create_volume('volume01', external=True) assert vol.exists() is False vol.create() assert vol.exists() is True + @no_cluster('inspect volume by name defect on Swarm Classic') def test_exists_external_aliased(self): vol = self.create_volume('volume01', external='composetest_alias01') assert vol.exists() is False vol.create() assert vol.exists() is True + @no_cluster('inspect volume by name defect on Swarm Classic') def test_volume_default_labels(self): vol = self.create_volume('volume01') vol.create() From 3bd5a374290831d6c2f090c6e3e80454d0ffa8bb Mon Sep 17 00:00:00 2001 From: NikitaVlaznev Date: Mon, 19 Jun 2017 17:05:19 +0300 Subject: [PATCH 056/244] Fix double silent argument value Fix for "TypeError: pull() got multiple values for keyword argument 'silent'." This change https://github.com/docker/compose/commit/e9b6cc23fcf01d4768c7e082b7bc91b43ff84e7e caused additional value to be passed for the 'silent' argument, that was already passed there: https://github.com/docker/compose/commit/f85da99ef3273794e855afda8678174419d3bf4f Signed-off-by: Nikita Vlaznev --- compose/project.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/compose/project.py b/compose/project.py index 3ad971488..7f7ade9a2 100644 --- a/compose/project.py +++ b/compose/project.py @@ -467,7 +467,7 @@ class Project(object): if parallel_pull: def pull_service(service): - service.pull(ignore_pull_failures, True, silent=silent) + service.pull(ignore_pull_failures, silent=silent) parallel.parallel_execute( services, From bb4adf2b0f385712e6385901532c738d4e3cc477 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Mon, 19 Jun 2017 13:52:56 -0700 Subject: [PATCH 057/244] 1.15.0dev Signed-off-by: Joffrey F --- compose/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/compose/__init__.py b/compose/__init__.py index f6ed1f463..1898479bf 100644 --- a/compose/__init__.py +++ b/compose/__init__.py @@ -1,4 +1,4 @@ from __future__ import absolute_import from __future__ import unicode_literals -__version__ = '1.14.0' +__version__ = '1.15.0dev' From 1dfdbe6f94db1d51ff5bec90a5785f3db031ba28 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Fri, 23 Jun 2017 15:04:18 -0700 Subject: [PATCH 058/244] Fix ports sorting on Python 3 Signed-off-by: Joffrey F --- compose/config/config.py | 2 +- tests/unit/config/config_test.py | 16 ++++++++++++++++ 2 files changed, 17 insertions(+), 1 deletion(-) diff --git a/compose/config/config.py b/compose/config/config.py index b8bffc660..fdb20df19 100644 --- a/compose/config/config.py +++ b/compose/config/config.py @@ -959,7 +959,7 @@ def merge_ports(md, base, override): merged = parse_sequence_func(md.base.get(field, [])) merged.update(parse_sequence_func(md.override.get(field, []))) - md[field] = [item for item in sorted(merged.values())] + md[field] = [item for item in sorted(merged.values(), key=lambda x: x.target)] def merge_build(output, base, override): diff --git a/tests/unit/config/config_test.py b/tests/unit/config/config_test.py index 87bdd8bca..6178447ae 100644 --- a/tests/unit/config/config_test.py +++ b/tests/unit/config/config_test.py @@ -1615,6 +1615,22 @@ class ConfigTest(unittest.TestCase): 'ports': types.ServicePort.parse('5432') } + def test_merge_service_dicts_ports_sorting(self): + base = { + 'ports': [5432] + } + override = { + 'image': 'alpine:edge', + 'ports': ['5432/udp'] + } + actual = config.merge_service_dicts_from_files( + base, + override, + DEFAULT_VERSION) + assert len(actual['ports']) == 2 + assert types.ServicePort.parse('5432')[0] in actual['ports'] + assert types.ServicePort.parse('5432/udp')[0] in actual['ports'] + def test_merge_service_dicts_heterogeneous_volumes(self): base = { 'volumes': ['/a:/b', '/x:/z'], From 6a957294dff00f905ce6bf5b695753f6558393be Mon Sep 17 00:00:00 2001 From: dinesh Date: Tue, 28 Mar 2017 18:25:27 +0530 Subject: [PATCH 059/244] Add storage_opt in v2.1 Signed-off-by: dinesh --- compose/config/config_schema_v2.1.json | 1 + compose/service.py | 2 ++ tests/integration/service_test.py | 7 +++++++ 3 files changed, 10 insertions(+) diff --git a/compose/config/config_schema_v2.1.json b/compose/config/config_schema_v2.1.json index 9004000ea..5aed9f7b1 100644 --- a/compose/config/config_schema_v2.1.json +++ b/compose/config/config_schema_v2.1.json @@ -229,6 +229,7 @@ "stdin_open": {"type": "boolean"}, "stop_grace_period": {"type": "string", "format": "duration"}, "stop_signal": {"type": "string"}, + "storage_opt": {"type": "object"}, "tmpfs": {"$ref": "#/definitions/string_or_list"}, "tty": {"type": "boolean"}, "ulimits": { diff --git a/compose/service.py b/compose/service.py index 03c41ce67..7ee63771a 100644 --- a/compose/service.py +++ b/compose/service.py @@ -82,6 +82,7 @@ HOST_CONFIG_KEYS = [ 'restart', 'security_opt', 'shm_size', + 'storage_opt', 'sysctls', 'userns_mode', 'volumes_from', @@ -854,6 +855,7 @@ class Service(object): volume_driver=options.get('volume_driver'), cpuset_cpus=options.get('cpuset'), cpu_shares=options.get('cpu_shares'), + storage_opt=options.get('storage_opt') ) def get_secret_volumes(self): diff --git a/tests/integration/service_test.py b/tests/integration/service_test.py index baf21af3c..e0aac2147 100644 --- a/tests/integration/service_test.py +++ b/tests/integration/service_test.py @@ -208,6 +208,13 @@ class ServiceTest(DockerClientTestCase): service.start_container(container) self.assertEqual(set(container.get('HostConfig.SecurityOpt')), set(security_opt)) + def test_create_container_with_storage_opt(self): + storage_opt = {'size': '1G'} + service = self.create_service('db', storage_opt=storage_opt) + container = service.create_container() + service.start_container(container) + self.assertEqual(container.get('HostConfig.StorageOpt'), storage_opt) + def test_create_container_with_mac_address(self): service = self.create_service('db', mac_address='02:42:ac:11:65:43') container = service.create_container() From e22524474aff36461b74c69458c79c5d92553e6c Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Fri, 23 Jun 2017 15:28:35 -0700 Subject: [PATCH 060/244] Ignore test failures in storage_opt test Signed-off-by: Joffrey F --- tests/integration/service_test.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/integration/service_test.py b/tests/integration/service_test.py index e0aac2147..c406a8d5e 100644 --- a/tests/integration/service_test.py +++ b/tests/integration/service_test.py @@ -208,6 +208,7 @@ class ServiceTest(DockerClientTestCase): service.start_container(container) self.assertEqual(set(container.get('HostConfig.SecurityOpt')), set(security_opt)) + @pytest.mark.xfail(True, reason='Not supported on most drivers') def test_create_container_with_storage_opt(self): storage_opt = {'size': '1G'} service = self.create_service('db', storage_opt=storage_opt) From b4eaddf9849d42a0d9a4d8db92956bbde8018314 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Fri, 23 Jun 2017 15:33:02 -0700 Subject: [PATCH 061/244] Add storage_opt to 2.2 schema Signed-off-by: Joffrey F --- compose/config/config_schema_v2.2.json | 1 + 1 file changed, 1 insertion(+) diff --git a/compose/config/config_schema_v2.2.json b/compose/config/config_schema_v2.2.json index e8edb60ed..87ba26ae4 100644 --- a/compose/config/config_schema_v2.2.json +++ b/compose/config/config_schema_v2.2.json @@ -235,6 +235,7 @@ "stdin_open": {"type": "boolean"}, "stop_grace_period": {"type": "string", "format": "duration"}, "stop_signal": {"type": "string"}, + "storage_opt": {"type": "object"}, "tmpfs": {"$ref": "#/definitions/string_or_list"}, "tty": {"type": "boolean"}, "ulimits": { From 5ee7aacca0f6db7d44683d34b5f775017540fe0a Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Wed, 28 Jun 2017 14:31:59 -0700 Subject: [PATCH 062/244] Bump docker Python SDK version -> 2.4.2 Signed-off-by: Joffrey F --- compose/config/types.py | 38 +++++++++++++++++++++----------------- requirements.txt | 2 +- setup.py | 2 +- 3 files changed, 23 insertions(+), 19 deletions(-) diff --git a/compose/config/types.py b/compose/config/types.py index 4509bfe67..be26971c4 100644 --- a/compose/config/types.py +++ b/compose/config/types.py @@ -295,24 +295,28 @@ class ServicePort(namedtuple('_ServicePort', 'target published protocol mode ext if not isinstance(spec, dict): result = [] - for k, v in build_port_bindings([spec]).items(): - if '/' in k: - target, proto = k.split('/', 1) - else: - target, proto = (k, None) - for pub in v: - if pub is None: - result.append( - cls(target, None, proto, None, None) - ) - elif isinstance(pub, tuple): - result.append( - cls(target, pub[1], proto, None, pub[0]) - ) + try: + for k, v in build_port_bindings([spec]).items(): + if '/' in k: + target, proto = k.split('/', 1) else: - result.append( - cls(target, pub, proto, None, None) - ) + target, proto = (k, None) + for pub in v: + if pub is None: + result.append( + cls(target, None, proto, None, None) + ) + elif isinstance(pub, tuple): + result.append( + cls(target, pub[1], proto, None, pub[0]) + ) + else: + result.append( + cls(target, pub, proto, None, None) + ) + except ValueError as e: + raise ConfigurationError(str(e)) + return result return [cls( diff --git a/requirements.txt b/requirements.txt index c4545de1e..4d506b9f4 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,7 +2,7 @@ PyYAML==3.11 backports.ssl-match-hostname==3.5.0.1; python_version < '3' cached-property==1.2.0 colorama==0.3.7 -docker==2.3.0 +docker==2.4.2 dockerpty==0.4.1 docopt==0.6.1 enum34==1.0.4; python_version < '3.4' diff --git a/setup.py b/setup.py index 8dbb337cc..0d5bd6adc 100644 --- a/setup.py +++ b/setup.py @@ -37,7 +37,7 @@ install_requires = [ 'requests >= 2.6.1, != 2.11.0, < 2.12', 'texttable >= 0.8.1, < 0.9', 'websocket-client >= 0.32.0, < 1.0', - 'docker >= 2.3.0, < 3.0', + 'docker >= 2.4.2, < 3.0', 'dockerpty >= 0.4.1, < 0.5', 'six >= 1.3.0, < 2', 'jsonschema >= 2.5.1, < 3', From a891fc1d9a4193dea7dabf1f6cf045d0daf6877e Mon Sep 17 00:00:00 2001 From: Evan Shaw Date: Sat, 1 Jul 2017 13:40:02 +1200 Subject: [PATCH 063/244] Always silence pull output with --parallel This is how things were prior to the addition of the --quiet flag. Making it not silent produces output that's weird and difficult to read. Signed-off-by: Evan Shaw --- compose/project.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/compose/project.py b/compose/project.py index 7f7ade9a2..7951d2974 100644 --- a/compose/project.py +++ b/compose/project.py @@ -467,7 +467,7 @@ class Project(object): if parallel_pull: def pull_service(service): - service.pull(ignore_pull_failures, silent=silent) + service.pull(ignore_pull_failures, True) parallel.parallel_execute( services, From 41976b0f7f8191d1cbc1ae1d9c3b6932d075dc12 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Thu, 15 Jun 2017 17:01:41 -0700 Subject: [PATCH 064/244] Add support for service:name pid config Signed-off-by: Joffrey F --- compose/config/config.py | 2 + compose/config/sort_services.py | 1 + compose/config/validation.py | 15 +++++++ compose/project.py | 26 ++++++++++++ compose/service.py | 49 +++++++++++++++++++++- tests/acceptance/cli_test.py | 25 +++++++++++ tests/fixtures/pid-mode/docker-compose.yml | 17 ++++++++ tests/integration/service_test.py | 5 ++- 8 files changed, 137 insertions(+), 3 deletions(-) create mode 100644 tests/fixtures/pid-mode/docker-compose.yml diff --git a/compose/config/config.py b/compose/config/config.py index fdb20df19..fb5442566 100644 --- a/compose/config/config.py +++ b/compose/config/config.py @@ -44,6 +44,7 @@ from .validation import validate_depends_on from .validation import validate_extends_file_path from .validation import validate_links from .validation import validate_network_mode +from .validation import validate_pid_mode from .validation import validate_service_constraints from .validation import validate_top_level_object from .validation import validate_ulimits @@ -667,6 +668,7 @@ def validate_service(service_config, service_names, config_file): validate_cpu(service_config) validate_ulimits(service_config) validate_network_mode(service_config, service_names) + validate_pid_mode(service_config, service_names) validate_depends_on(service_config, service_names) validate_links(service_config, service_names) diff --git a/compose/config/sort_services.py b/compose/config/sort_services.py index 20ac4461b..42f548a6d 100644 --- a/compose/config/sort_services.py +++ b/compose/config/sort_services.py @@ -38,6 +38,7 @@ def get_service_dependents(service_dict, services): if (name in get_service_names(service.get('links', [])) or name in get_service_names_from_volumes_from(service.get('volumes_from', [])) or name == get_service_name_from_network_mode(service.get('network_mode')) or + name == get_service_name_from_network_mode(service.get('pid')) or name in service.get('depends_on', [])) ] diff --git a/compose/config/validation.py b/compose/config/validation.py index 856f811c5..0b7961e5a 100644 --- a/compose/config/validation.py +++ b/compose/config/validation.py @@ -172,6 +172,21 @@ def validate_network_mode(service_config, service_names): "is undefined.".format(s=service_config, dep=dependency)) +def validate_pid_mode(service_config, service_names): + pid_mode = service_config.config.get('pid') + if not pid_mode: + return + + dependency = get_service_name_from_network_mode(pid_mode) + if not dependency: + return + if dependency not in service_names: + raise ConfigurationError( + "Service '{s.name}' uses the PID namespace of service '{dep}' which " + "is undefined.".format(s=service_config, dep=dependency) + ) + + def validate_links(service_config, service_names): for link in service_config.config.get('links', []): if link.split(':')[0] not in service_names: diff --git a/compose/project.py b/compose/project.py index 7951d2974..28af45c71 100644 --- a/compose/project.py +++ b/compose/project.py @@ -24,10 +24,13 @@ from .network import get_networks from .network import ProjectNetworks from .service import BuildAction from .service import ContainerNetworkMode +from .service import ContainerPidMode from .service import ConvergenceStrategy from .service import NetworkMode +from .service import PidMode from .service import Service from .service import ServiceNetworkMode +from .service import ServicePidMode from .utils import microseconds_from_time_nano from .volume import ProjectVolumes @@ -97,6 +100,7 @@ class Project(object): network_mode = project.get_network_mode( service_dict, list(service_networks.keys()) ) + pid_mode = project.get_pid_mode(service_dict) volumes_from = get_volumes_from(project, service_dict) if config_data.version != V1: @@ -121,6 +125,7 @@ class Project(object): network_mode=network_mode, volumes_from=volumes_from, secrets=secrets, + pid_mode=pid_mode, **service_dict) ) @@ -224,6 +229,27 @@ class Project(object): return NetworkMode(network_mode) + def get_pid_mode(self, service_dict): + pid_mode = service_dict.pop('pid', None) + if not pid_mode: + return PidMode(None) + + service_name = get_service_name_from_network_mode(pid_mode) + if service_name: + return ServicePidMode(self.get_service(service_name)) + + container_name = get_container_name_from_network_mode(pid_mode) + if container_name: + try: + return ContainerPidMode(Container.from_id(self.client, container_name)) + except APIError: + raise ConfigurationError( + "Service '{name}' uses the PID namespace of container '{dep}' which " + "does not exist.".format(name=service_dict['name'], dep=container_name) + ) + + return PidMode(pid_mode) + def start(self, service_names=None, **options): containers = [] diff --git a/compose/service.py b/compose/service.py index 7ee63771a..c4fd96c43 100644 --- a/compose/service.py +++ b/compose/service.py @@ -157,6 +157,7 @@ class Service(object): networks=None, secrets=None, scale=None, + pid_mode=None, **options ): self.name = name @@ -166,6 +167,7 @@ class Service(object): self.links = links or [] self.volumes_from = volumes_from or [] self.network_mode = network_mode or NetworkMode(None) + self.pid_mode = pid_mode or PidMode(None) self.networks = networks or {} self.secrets = secrets or [] self.scale_num = scale or 1 @@ -607,15 +609,19 @@ class Service(object): def get_dependency_names(self): net_name = self.network_mode.service_name + pid_namespace = self.pid_mode.service_name return ( self.get_linked_service_names() + self.get_volumes_from_names() + ([net_name] if net_name else []) + + ([pid_namespace] if pid_namespace else []) + list(self.options.get('depends_on', {}).keys()) ) def get_dependency_configs(self): net_name = self.network_mode.service_name + pid_namespace = self.pid_mode.service_name + configs = dict( [(name, None) for name in self.get_linked_service_names()] ) @@ -623,6 +629,7 @@ class Service(object): [(name, None) for name in self.get_volumes_from_names()] )) configs.update({net_name: None} if net_name else {}) + configs.update({pid_namespace: None} if pid_namespace else {}) configs.update(self.options.get('depends_on', {})) for svc, config in self.options.get('depends_on', {}).items(): if config['condition'] == CONDITION_STARTED: @@ -833,7 +840,7 @@ class Service(object): log_config=log_config, extra_hosts=options.get('extra_hosts'), read_only=options.get('read_only'), - pid_mode=options.get('pid'), + pid_mode=self.pid_mode.mode, security_opt=options.get('security_opt'), ipc_mode=options.get('ipc'), cgroup_parent=options.get('cgroup_parent'), @@ -1056,6 +1063,46 @@ def short_id_alias_exists(container, network): return container.short_id in aliases +class PidMode(object): + def __init__(self, mode): + self._mode = mode + + @property + def mode(self): + return self._mode + + @property + def service_name(self): + return None + + +class ServicePidMode(PidMode): + def __init__(self, service): + self.service = service + + @property + def service_name(self): + return self.service.name + + @property + def mode(self): + containers = self.service.containers() + if containers: + return 'container:' + containers[0].id + + log.warn( + "Service %s is trying to use reuse the PID namespace " + "of another service that is not running." % (self.service_name) + ) + return None + + +class ContainerPidMode(PidMode): + def __init__(self, container): + self.container = container + self._mode = 'container:{}'.format(container.id) + + class NetworkMode(object): """A `standard` network mode (ex: host, bridge)""" diff --git a/tests/acceptance/cli_test.py b/tests/acceptance/cli_test.py index ba0b53888..9058fa35b 100644 --- a/tests/acceptance/cli_test.py +++ b/tests/acceptance/cli_test.py @@ -1183,6 +1183,31 @@ class CLITestCase(DockerClientTestCase): proc.wait() self.assertEqual(proc.returncode, 1) + @v2_only() + def test_up_with_pid_mode(self): + c = self.client.create_container( + 'busybox', 'top', name='composetest_pid_mode_container', + host_config={} + ) + self.addCleanup(self.client.remove_container, c, force=True) + self.client.start(c) + container_mode_source = 'container:{}'.format(c['Id']) + + self.base_dir = 'tests/fixtures/pid-mode' + + self.dispatch(['up', '-d'], None) + + service_mode_source = 'container:{}'.format( + self.project.get_service('container').containers()[0].id) + service_mode_container = self.project.get_service('service').containers()[0] + assert service_mode_container.get('HostConfig.PidMode') == service_mode_source + + container_mode_container = self.project.get_service('container').containers()[0] + assert container_mode_container.get('HostConfig.PidMode') == container_mode_source + + host_mode_container = self.project.get_service('host').containers()[0] + assert host_mode_container.get('HostConfig.PidMode') == 'host' + def test_exec_without_tty(self): self.base_dir = 'tests/fixtures/links-composefile' self.dispatch(['up', '-d', 'console']) diff --git a/tests/fixtures/pid-mode/docker-compose.yml b/tests/fixtures/pid-mode/docker-compose.yml new file mode 100644 index 000000000..fece5a9f0 --- /dev/null +++ b/tests/fixtures/pid-mode/docker-compose.yml @@ -0,0 +1,17 @@ +version: "2.2" + +services: + service: + image: busybox + command: top + pid: "service:container" + + container: + image: busybox + command: top + pid: "container:composetest_pid_mode_container" + + host: + image: busybox + command: top + pid: host diff --git a/tests/integration/service_test.py b/tests/integration/service_test.py index c406a8d5e..ccd6c8b00 100644 --- a/tests/integration/service_test.py +++ b/tests/integration/service_test.py @@ -36,6 +36,7 @@ from compose.project import OneOffFilter from compose.service import ConvergencePlan from compose.service import ConvergenceStrategy from compose.service import NetworkMode +from compose.service import PidMode from compose.service import Service from tests.integration.testcases import v2_1_only from tests.integration.testcases import v2_2_only @@ -968,12 +969,12 @@ class ServiceTest(DockerClientTestCase): self.assertEqual(container.get('HostConfig.NetworkMode'), 'host') def test_pid_mode_none_defined(self): - service = self.create_service('web', pid=None) + service = self.create_service('web', pid_mode=None) container = create_and_start_container(service) self.assertEqual(container.get('HostConfig.PidMode'), '') def test_pid_mode_host(self): - service = self.create_service('web', pid='host') + service = self.create_service('web', pid_mode=PidMode('host')) container = create_and_start_container(service) self.assertEqual(container.get('HostConfig.PidMode'), 'host') From 154adc580776cd3a9742ad48a142e7c2918da60f Mon Sep 17 00:00:00 2001 From: Evan Shaw Date: Sat, 25 Feb 2017 13:48:02 +1300 Subject: [PATCH 065/244] Align status output for parallel_execute Previously docker-compose would output lines that looked like: Starting service ... done Starting short ... Starting service-with-a-long-name ... done It's difficult to scan down this output and get an idea of what's happening. Now the statuses are aligned, and output looks like this: Starting service ... done Starting short ... Starting service-with-a-long-name ... done To me, this is quite a bit easier to read. Signed-off-by: Evan Shaw --- compose/parallel.py | 18 +++++++++++++----- tests/unit/parallel_test.py | 15 +++++++++++++++ 2 files changed, 28 insertions(+), 5 deletions(-) diff --git a/compose/parallel.py b/compose/parallel.py index 34fef71db..a611fd6e0 100644 --- a/compose/parallel.py +++ b/compose/parallel.py @@ -38,7 +38,8 @@ def parallel_execute(objects, func, get_name, msg, get_deps=None, limit=None): writer = ParallelStreamWriter(stream, msg) for obj in objects: - writer.initialize(get_name(obj)) + writer.add_object(get_name(obj)) + writer.write_initial() events = parallel_execute_iter(objects, func, get_deps, limit) @@ -224,12 +225,18 @@ class ParallelStreamWriter(object): self.stream = stream self.msg = msg self.lines = [] + self.width = 0 - def initialize(self, obj_index): + def add_object(self, obj_index): + self.lines.append(obj_index) + self.width = max(self.width, len(obj_index)) + + def write_initial(self): if self.msg is None: return - self.lines.append(obj_index) - self.stream.write("{} {} ... \r\n".format(self.msg, obj_index)) + for line in self.lines: + self.stream.write("{} {:<{width}} ... \r\n".format(self.msg, line, + width=self.width)) self.stream.flush() def write(self, obj_index, status): @@ -241,7 +248,8 @@ class ParallelStreamWriter(object): self.stream.write("%c[%dA" % (27, diff)) # erase self.stream.write("%c[2K\r" % 27) - self.stream.write("{} {} ... {}\r".format(self.msg, obj_index, status)) + self.stream.write("{} {:<{width}} ... {}\r".format(self.msg, obj_index, + status, width=self.width)) # move back down self.stream.write("%c[%dB" % (27, diff)) self.stream.flush() diff --git a/tests/unit/parallel_test.py b/tests/unit/parallel_test.py index d10948eb0..73728fdfd 100644 --- a/tests/unit/parallel_test.py +++ b/tests/unit/parallel_test.py @@ -115,3 +115,18 @@ def test_parallel_execute_with_upstream_errors(): assert (data_volume, None, APIError) in events assert (db, None, UpstreamError) in events assert (web, None, UpstreamError) in events + + +def test_parallel_execute_alignment(capsys): + results, errors = parallel_execute( + objects=["short", "a very long name"], + func=lambda x: x, + get_name=six.text_type, + msg="Aligning", + ) + + assert errors == {} + + _, err = capsys.readouterr() + a, b = err.split('\n')[:2] + assert a.index('...') == b.index('...') From 4796e04cae551a37f6305888a1bb871475e1570f Mon Sep 17 00:00:00 2001 From: Andy Neff Date: Tue, 16 May 2017 14:21:18 -0400 Subject: [PATCH 066/244] Change --volume behavior to add instead of replace mounts Signed-off-by: Andy Neff --- compose/service.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/compose/service.py b/compose/service.py index c4fd96c43..326f50520 100644 --- a/compose/service.py +++ b/compose/service.py @@ -736,6 +736,8 @@ class Service(object): container_options = dict( (k, self.options[k]) for k in DOCKER_CONFIG_KEYS if k in self.options) + override_options['volumes'] = (container_options.get('volumes', []) + + override_options.get('volumes', [])) container_options.update(override_options) if not container_options.get('name'): From ec4ba7752f3633b2bcd4bf0b33baef625bb1309a Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Fri, 23 Jun 2017 13:38:38 -0700 Subject: [PATCH 067/244] Fix override volume merging + add acceptance test Signed-off-by: Joffrey F --- compose/service.py | 8 +++- tests/acceptance/cli_test.py | 37 ++++++++++++++++--- .../docker-compose.merge.yml | 9 +++++ 3 files changed, 46 insertions(+), 8 deletions(-) create mode 100644 tests/fixtures/simple-composefile-volume-ready/docker-compose.merge.yml diff --git a/compose/service.py b/compose/service.py index 326f50520..300ec2852 100644 --- a/compose/service.py +++ b/compose/service.py @@ -736,8 +736,7 @@ class Service(object): container_options = dict( (k, self.options[k]) for k in DOCKER_CONFIG_KEYS if k in self.options) - override_options['volumes'] = (container_options.get('volumes', []) + - override_options.get('volumes', [])) + override_volumes = override_options.pop('volumes', []) container_options.update(override_options) if not container_options.get('name'): @@ -761,6 +760,11 @@ class Service(object): formatted_ports(container_options.get('ports', [])), self.options) + if 'volumes' in container_options or override_volumes: + container_options['volumes'] = list(set( + container_options.get('volumes', []) + override_volumes + )) + container_options['environment'] = merge_environment( self.options.get('environment'), override_options.get('environment')) diff --git a/tests/acceptance/cli_test.py b/tests/acceptance/cli_test.py index 9058fa35b..9d2de622d 100644 --- a/tests/acceptance/cli_test.py +++ b/tests/acceptance/cli_test.py @@ -609,8 +609,13 @@ class CLITestCase(DockerClientTestCase): 'simple', 'test', '-f', '/data/example.txt' ], returncode=0) - # FIXME: does not work with Python 3 - # assert cmd_result.stdout.strip() == 'FILE_CONTENT' + + service = self.project.get_service('simple') + container_data = service.containers(one_off=OneOffFilter.only, stopped=True)[0] + mount = container_data.get('Mounts')[0] + assert mount['Source'] == volume_path + assert mount['Destination'] == '/data' + assert mount['Type'] == 'bind' def test_run_one_off_with_multiple_volumes(self): self.base_dir = 'tests/fixtures/simple-composefile-volume-ready' @@ -624,8 +629,6 @@ class CLITestCase(DockerClientTestCase): 'simple', 'test', '-f', '/data/example.txt' ], returncode=0) - # FIXME: does not work with Python 3 - # assert cmd_result.stdout.strip() == 'FILE_CONTENT' self.dispatch([ 'run', @@ -634,8 +637,30 @@ class CLITestCase(DockerClientTestCase): 'simple', 'test', '-f' '/data1/example.txt' ], returncode=0) - # FIXME: does not work with Python 3 - # assert cmd_result.stdout.strip() == 'FILE_CONTENT' + + def test_run_one_off_with_volume_merge(self): + self.base_dir = 'tests/fixtures/simple-composefile-volume-ready' + volume_path = os.path.abspath(os.path.join(os.getcwd(), self.base_dir, 'files')) + create_host_file(self.client, os.path.join(volume_path, 'example.txt')) + + self.dispatch([ + '-f', 'docker-compose.merge.yml', + 'run', + '-v', '{}:/data'.format(volume_path), + 'simple', + 'test', '-f', '/data/example.txt' + ], returncode=0) + + service = self.project.get_service('simple') + container_data = service.containers(one_off=OneOffFilter.only, stopped=True)[0] + mounts = container_data.get('Mounts') + assert len(mounts) == 2 + config_mount = [m for m in mounts if m['Destination'] == '/data1'][0] + override_mount = [m for m in mounts if m['Destination'] == '/data'][0] + + assert config_mount['Type'] == 'volume' + assert override_mount['Source'] == volume_path + assert override_mount['Type'] == 'bind' def test_create_with_force_recreate_and_no_recreate(self): self.dispatch( diff --git a/tests/fixtures/simple-composefile-volume-ready/docker-compose.merge.yml b/tests/fixtures/simple-composefile-volume-ready/docker-compose.merge.yml new file mode 100644 index 000000000..fe7171516 --- /dev/null +++ b/tests/fixtures/simple-composefile-volume-ready/docker-compose.merge.yml @@ -0,0 +1,9 @@ +version: '2.2' +services: + simple: + image: busybox:latest + volumes: + - datastore:/data1 + +volumes: + datastore: From 0916f124d0d35bc0145b11b82b4721db10c779f1 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Mon, 3 Jul 2017 17:12:39 -0700 Subject: [PATCH 068/244] `scale` property should be merged according to standard scalar rules Signed-off-by: Joffrey F --- compose/config/config.py | 1 + tests/unit/config/config_test.py | 13 +++++++++++++ 2 files changed, 14 insertions(+) diff --git a/compose/config/config.py b/compose/config/config.py index fb5442566..86cf1b39d 100644 --- a/compose/config/config.py +++ b/compose/config/config.py @@ -116,6 +116,7 @@ ALLOWED_KEYS = DOCKER_CONFIG_KEYS + [ 'logging', 'network_mode', 'init', + 'scale', ] DOCKER_VALID_URL_PREFIXES = ( diff --git a/tests/unit/config/config_test.py b/tests/unit/config/config_test.py index 6178447ae..721a428e1 100644 --- a/tests/unit/config/config_test.py +++ b/tests/unit/config/config_test.py @@ -2098,6 +2098,19 @@ class ConfigTest(unittest.TestCase): actual = config.merge_service_dicts(base, override, V3_3) assert actual['credential_spec'] == override['credential_spec'] + def test_merge_scale(self): + base = { + 'image': 'bar', + 'scale': 2, + } + + override = { + 'scale': 4, + } + + actual = config.merge_service_dicts(base, override, V2_2) + assert actual == {'image': 'bar', 'scale': 4} + def test_external_volume_config(self): config_details = build_config_details({ 'version': '2', From d475e0c1e3df983406962addd5e778d8d29ba7b2 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Wed, 5 Jul 2017 15:13:45 -0700 Subject: [PATCH 069/244] Add "network" field to build configuration Signed-off-by: Joffrey F --- compose/config/config.py | 1 + compose/config/config_schema_v2.2.json | 3 ++- compose/service.py | 3 ++- tests/integration/service_test.py | 24 ++++++++++++++++++++++++ tests/unit/service_test.py | 2 ++ 5 files changed, 31 insertions(+), 2 deletions(-) diff --git a/compose/config/config.py b/compose/config/config.py index 86cf1b39d..4be251882 100644 --- a/compose/config/config.py +++ b/compose/config/config.py @@ -975,6 +975,7 @@ def merge_build(output, base, override): md = MergeDict(to_dict(base), to_dict(override)) md.merge_scalar('context') md.merge_scalar('dockerfile') + md.merge_scalar('network') md.merge_mapping('args', parse_build_arguments) md.merge_field('cache_from', merge_unique_items_lists, default=[]) md.merge_mapping('labels', parse_labels) diff --git a/compose/config/config_schema_v2.2.json b/compose/config/config_schema_v2.2.json index 87ba26ae4..9181e606b 100644 --- a/compose/config/config_schema_v2.2.json +++ b/compose/config/config_schema_v2.2.json @@ -60,7 +60,8 @@ "dockerfile": {"type": "string"}, "args": {"$ref": "#/definitions/list_or_dict"}, "labels": {"$ref": "#/definitions/list_or_dict"}, - "cache_from": {"$ref": "#/definitions/list_of_strings"} + "cache_from": {"$ref": "#/definitions/list_of_strings"}, + "network": {"type": "string"} }, "additionalProperties": false } diff --git a/compose/service.py b/compose/service.py index 300ec2852..53ad46362 100644 --- a/compose/service.py +++ b/compose/service.py @@ -906,7 +906,8 @@ class Service(object): dockerfile=build_opts.get('dockerfile', None), cache_from=build_opts.get('cache_from', None), labels=build_opts.get('labels', None), - buildargs=build_args + buildargs=build_args, + network_mode=build_opts.get('network', None), ) try: diff --git a/tests/integration/service_test.py b/tests/integration/service_test.py index ccd6c8b00..350f7398b 100644 --- a/tests/integration/service_test.py +++ b/tests/integration/service_test.py @@ -717,6 +717,30 @@ class ServiceTest(DockerClientTestCase): assert service.image() assert service.image()['Config']['Labels']['com.docker.compose.test'] == 'true' + def test_build_with_network(self): + base_dir = tempfile.mkdtemp() + self.addCleanup(shutil.rmtree, base_dir) + with open(os.path.join(base_dir, 'Dockerfile'), 'w') as f: + f.write('FROM busybox\n') + f.write('RUN ping -c1 google.local\n') + + net_container = self.client.create_container( + 'busybox', 'top', host_config=self.client.create_host_config( + extra_hosts={'google.local': '8.8.8.8'} + ), name='composetest_build_network' + ) + + self.addCleanup(self.client.remove_container, net_container, force=True) + self.client.start(net_container) + + service = self.create_service('buildwithnet', build={ + 'context': text_type(base_dir), + 'network': 'container:{}'.format(net_container['Id']) + }) + + service.build() + assert service.image() + def test_start_container_stays_unprivileged(self): service = self.create_service('web') container = create_and_start_container(service).inspect() diff --git a/tests/unit/service_test.py b/tests/unit/service_test.py index 7b7a078f8..2b0a2762d 100644 --- a/tests/unit/service_test.py +++ b/tests/unit/service_test.py @@ -473,6 +473,7 @@ class ServiceTest(unittest.TestCase): buildargs={}, labels=None, cache_from=None, + network_mode=None, ) def test_ensure_image_exists_no_build(self): @@ -511,6 +512,7 @@ class ServiceTest(unittest.TestCase): buildargs={}, labels=None, cache_from=None, + network_mode=None, ) def test_build_does_not_pull(self): From af182bd3cca710680fb941d7fff7029071e3316f Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Mon, 3 Jul 2017 15:32:22 -0700 Subject: [PATCH 070/244] Add 'socks' extra to help with proxy environment. SOCKS support will be included in the bundled (binary) version Update some packages in requirements.txt and add some implicit deps Signed-off-by: Joffrey F --- requirements.txt | 22 ++++++++++++++-------- setup.py | 1 + 2 files changed, 15 insertions(+), 8 deletions(-) diff --git a/requirements.txt b/requirements.txt index 4d506b9f4..844921ffd 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,16 +1,22 @@ -PyYAML==3.11 +PySocks==1.6.7 +PyYAML==3.12 backports.ssl-match-hostname==3.5.0.1; python_version < '3' -cached-property==1.2.0 -colorama==0.3.7 +cached-property==1.3.0 +certifi==2017.4.17 +chardet==3.0.4 +colorama==0.3.9 docker==2.4.2 +docker-pycreds==0.2.1 dockerpty==0.4.1 -docopt==0.6.1 -enum34==1.0.4; python_version < '3.4' +docopt==0.6.2 +enum34==1.1.6; python_version < '3.4' functools32==3.2.3.post2; python_version < '3.2' -ipaddress==1.0.16 -jsonschema==2.5.1 +idna==2.5 +ipaddress==1.0.18 +jsonschema==2.6.0 pypiwin32==219; sys_platform == 'win32' requests==2.11.1 six==1.10.0 -texttable==0.8.4 +texttable==0.8.8 +urllib3==1.21.1 websocket-client==0.32.0 diff --git a/setup.py b/setup.py index 0d5bd6adc..dab7a6eea 100644 --- a/setup.py +++ b/setup.py @@ -56,6 +56,7 @@ extras_require = { ':python_version < "3.4"': ['enum34 >= 1.0.4, < 2'], ':python_version < "3.5"': ['backports.ssl_match_hostname >= 3.5'], ':python_version < "3.3"': ['ipaddress >= 1.0.16'], + 'socks': ['PySocks >= 1.5.6, != 1.5.7, < 2'], } From 2d21bf6a50a7cda0b7c99d7605b0b2c2a89a191d Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Wed, 5 Jul 2017 19:21:07 -0700 Subject: [PATCH 071/244] Make sure y/n values are quoted in serialized output Signed-off-by: Joffrey F --- compose/config/serialize.py | 15 ++++++++++++++- tests/unit/config/config_test.py | 18 ++++++++++++++++++ 2 files changed, 32 insertions(+), 1 deletion(-) diff --git a/compose/config/serialize.py b/compose/config/serialize.py index beafe02b9..306f86969 100644 --- a/compose/config/serialize.py +++ b/compose/config/serialize.py @@ -21,11 +21,23 @@ def serialize_dict_type(dumper, data): return dumper.represent_dict(data.repr()) +def serialize_string(dumper, data): + """ Ensure boolean-like strings are quoted in the output """ + representer = dumper.represent_str if six.PY3 else dumper.represent_unicode + if data.lower() in ('y', 'n', 'yes', 'no', 'on', 'off', 'true', 'false'): + # Empirically only y/n appears to be an issue, but this might change + # depending on which PyYaml version is being used. Err on safe side. + return dumper.represent_scalar('tag:yaml.org,2002:str', data, style='"') + return representer(data) + + yaml.SafeDumper.add_representer(types.VolumeFromSpec, serialize_config_type) yaml.SafeDumper.add_representer(types.VolumeSpec, serialize_config_type) yaml.SafeDumper.add_representer(types.ServiceSecret, serialize_dict_type) yaml.SafeDumper.add_representer(types.ServiceConfig, serialize_dict_type) yaml.SafeDumper.add_representer(types.ServicePort, serialize_dict_type) +yaml.SafeDumper.add_representer(str, serialize_string) +yaml.SafeDumper.add_representer(six.text_type, serialize_string) def denormalize_config(config, image_digests=None): @@ -58,7 +70,8 @@ def serialize_config(config, image_digests=None): denormalize_config(config, image_digests), default_flow_style=False, indent=2, - width=80) + width=80 + ) def serialize_ns_time_value(value): diff --git a/tests/unit/config/config_test.py b/tests/unit/config/config_test.py index 721a428e1..6731a6bbc 100644 --- a/tests/unit/config/config_test.py +++ b/tests/unit/config/config_test.py @@ -4163,3 +4163,21 @@ class SerializeTest(unittest.TestCase): assert secret_sort(serialized_service['configs']) == secret_sort(service_dict['configs']) assert 'configs' in serialized_config assert serialized_config['configs']['two'] == configs_dict['two'] + + def test_serialize_bool_string(self): + cfg = { + 'version': '2.2', + 'services': { + 'web': { + 'image': 'example/web', + 'command': 'true', + 'environment': {'FOO': 'Y', 'BAR': 'on'} + } + } + } + config_dict = config.load(build_config_details(cfg)) + + serialized_config = serialize_config(config_dict) + assert 'command: "true"\n' in serialized_config + assert 'FOO: "Y"\n' in serialized_config + assert 'BAR: "on"\n' in serialized_config From c41057aa523b62504a61d46cc4b05f387bc3c988 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Thu, 6 Jul 2017 17:54:45 -0700 Subject: [PATCH 072/244] Code warning for the well-intentioned folks that keep wanting to change this Signed-off-by: Joffrey F --- compose/cli/__init__.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/compose/cli/__init__.py b/compose/cli/__init__.py index 379059c1a..2574a311f 100644 --- a/compose/cli/__init__.py +++ b/compose/cli/__init__.py @@ -17,6 +17,8 @@ try: env[str('PIP_DISABLE_PIP_VERSION_CHECK')] = str('1') s_cmd = subprocess.Popen( + # DO NOT replace this call with a `sys.executable` call. It breaks the binary + # distribution (with the binary calling itself recursively over and over). ['pip', 'freeze'], stderr=subprocess.PIPE, stdout=subprocess.PIPE, env=env ) From 6ff6528d45ea2fe6cc511dff75250995f9913c9b Mon Sep 17 00:00:00 2001 From: Vadim Semenov Date: Thu, 15 Jun 2017 16:55:18 +0300 Subject: [PATCH 073/244] Optimize "extends" without file specification Loading the same config file add about 100ms per each extension service, which results in painfully slow CLI calls when a config consists of a couple of dozens of services. This patch makes Compose re-use config files. Signed-off-by: Vadim Semenov --- compose/config/config.py | 21 +++++++++++++++------ 1 file changed, 15 insertions(+), 6 deletions(-) diff --git a/compose/config/config.py b/compose/config/config.py index 4be251882..2b1b99104 100644 --- a/compose/config/config.py +++ b/compose/config/config.py @@ -570,12 +570,21 @@ class ServiceExtendsResolver(object): config_path = self.get_extended_config_path(extends) service_name = extends['service'] - extends_file = ConfigFile.from_filename(config_path) - validate_config_version([self.config_file, extends_file]) - extended_file = process_config_file( - extends_file, self.environment, service_name=service_name - ) - service_config = extended_file.get_service(service_name) + if config_path == self.service_config.filename: + try: + service_config = self.config_file.get_service(service_name) + except KeyError: + raise ConfigurationError( + "Cannot extend service '{}' in {}: Service not found".format( + service_name, config_path) + ) + else: + extends_file = ConfigFile.from_filename(config_path) + validate_config_version([self.config_file, extends_file]) + extended_file = process_config_file( + extends_file, self.environment, service_name=service_name + ) + service_config = extended_file.get_service(service_name) return config_path, service_config, service_name From 56a23bfcd2eec0140589d4b3223e28d47d89fcdb Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Thu, 6 Jul 2017 17:25:41 -0700 Subject: [PATCH 074/244] Improved version comparisons throughout the codebase Signed-off-by: Joffrey F --- compose/config/config.py | 17 ++++++----- compose/config/errors.py | 2 +- compose/config/interpolation.py | 3 +- compose/config/serialize.py | 9 +++--- compose/const.py | 18 +++++++----- compose/version.py | 10 +++++++ tests/acceptance/cli_test.py | 9 ++++-- tests/integration/project_test.py | 22 +++++++------- tests/integration/testcases.py | 39 +++++++++++-------------- tests/unit/bundle_test.py | 3 +- tests/unit/config/config_test.py | 8 ++--- tests/unit/config/interpolation_test.py | 8 +++-- tests/unit/project_test.py | 28 +++++++++--------- 13 files changed, 97 insertions(+), 79 deletions(-) create mode 100644 compose/version.py diff --git a/compose/config/config.py b/compose/config/config.py index 2b1b99104..f5053af8a 100644 --- a/compose/config/config.py +++ b/compose/config/config.py @@ -18,6 +18,7 @@ from ..const import COMPOSEFILE_V1 as V1 from ..utils import build_string_dict from ..utils import parse_nanoseconds_int from ..utils import splitdrive +from ..version import ComposeVersion from .environment import env_vars_from_file from .environment import Environment from .environment import split_env @@ -188,15 +189,16 @@ class ConfigFile(namedtuple('_ConfigFile', 'filename config')): if version == '1': raise ConfigurationError( 'Version in "{}" is invalid. {}' - .format(self.filename, VERSION_EXPLANATION)) + .format(self.filename, VERSION_EXPLANATION) + ) if version == '2': - version = const.COMPOSEFILE_V2_0 + return const.COMPOSEFILE_V2_0 if version == '3': - version = const.COMPOSEFILE_V3_0 + return const.COMPOSEFILE_V3_0 - return version + return ComposeVersion(version) def get_service(self, name): return self.get_service_dicts()[name] @@ -496,7 +498,7 @@ def process_config_file(config_file, environment, service_name=None): 'service', environment) - if config_file.version != V1: + if config_file.version > V1: processed_config = dict(config_file.config) processed_config['services'] = services processed_config['volumes'] = interpolate_config_section( @@ -509,14 +511,13 @@ def process_config_file(config_file, environment, service_name=None): config_file.get_networks(), 'network', environment) - if config_file.version in (const.COMPOSEFILE_V3_1, const.COMPOSEFILE_V3_2, - const.COMPOSEFILE_V3_3): + if config_file.version >= const.COMPOSEFILE_V3_1: processed_config['secrets'] = interpolate_config_section( config_file, config_file.get_secrets(), 'secrets', environment) - if config_file.version in (const.COMPOSEFILE_V3_3): + if config_file.version >= const.COMPOSEFILE_V3_3: processed_config['configs'] = interpolate_config_section( config_file, config_file.get_configs(), diff --git a/compose/config/errors.py b/compose/config/errors.py index ac1d3ac19..f5c038088 100644 --- a/compose/config/errors.py +++ b/compose/config/errors.py @@ -4,7 +4,7 @@ from __future__ import unicode_literals VERSION_EXPLANATION = ( 'You might be seeing this error because you\'re using the wrong Compose file version. ' - 'Either specify a supported version ("2.0", "2.1", "3.0", "3.1", "3.2") and place ' + 'Either specify a supported version (e.g "2.2" or "3.3") and place ' 'your service definitions under the `services` key, or omit the `version` key ' 'and place your service definitions at the root of the file to use ' 'version 1.\nFor more on the Compose file format versions, see ' diff --git a/compose/config/interpolation.py b/compose/config/interpolation.py index 1b270b9ea..b13ac591a 100644 --- a/compose/config/interpolation.py +++ b/compose/config/interpolation.py @@ -7,7 +7,6 @@ from string import Template import six from .errors import ConfigurationError -from compose.const import COMPOSEFILE_V1 as V1 from compose.const import COMPOSEFILE_V2_0 as V2_0 @@ -28,7 +27,7 @@ class Interpolator(object): def interpolate_environment_variables(version, config, section, environment): - if version in (V2_0, V1): + if version <= V2_0: interpolator = Interpolator(Template, environment) else: interpolator = Interpolator(TemplateWithDefaults, environment) diff --git a/compose/config/serialize.py b/compose/config/serialize.py index 306f86969..84521848d 100644 --- a/compose/config/serialize.py +++ b/compose/config/serialize.py @@ -7,9 +7,8 @@ import yaml from compose.config import types from compose.const import COMPOSEFILE_V1 as V1 from compose.const import COMPOSEFILE_V2_1 as V2_1 -from compose.const import COMPOSEFILE_V2_2 as V2_2 +from compose.const import COMPOSEFILE_V3_0 as V3_0 from compose.const import COMPOSEFILE_V3_2 as V3_2 -from compose.const import COMPOSEFILE_V3_3 as V3_3 def serialize_config_type(dumper, data): @@ -41,7 +40,7 @@ yaml.SafeDumper.add_representer(six.text_type, serialize_string) def denormalize_config(config, image_digests=None): - result = {'version': V2_1 if config.version == V1 else config.version} + result = {'version': str(V2_1) if config.version == V1 else str(config.version)} denormalized_services = [ denormalize_service_dict( service_dict, @@ -107,7 +106,7 @@ def denormalize_service_dict(service_dict, version, image_digest=None): if version == V1 and 'network_mode' not in service_dict: service_dict['network_mode'] = 'bridge' - if 'depends_on' in service_dict and version not in (V2_1, V2_2): + if 'depends_on' in service_dict and (version < V2_1 or version >= V3_0): service_dict['depends_on'] = sorted([ svc for svc in service_dict['depends_on'].keys() ]) @@ -122,7 +121,7 @@ def denormalize_service_dict(service_dict, version, image_digest=None): service_dict['healthcheck']['timeout'] ) - if 'ports' in service_dict and version not in (V3_2, V3_3): + if 'ports' in service_dict and version < V3_2: service_dict['ports'] = [ p.legacy_repr() if isinstance(p, types.ServicePort) else p for p in service_dict['ports'] diff --git a/compose/const.py b/compose/const.py index 36f213897..e46de8a73 100644 --- a/compose/const.py +++ b/compose/const.py @@ -3,6 +3,8 @@ from __future__ import unicode_literals import sys +from .version import ComposeVersion + DEFAULT_TIMEOUT = 10 HTTP_TIMEOUT = 60 IMAGE_EVENTS = ['delete', 'import', 'load', 'pull', 'push', 'save', 'tag', 'untag'] @@ -19,15 +21,15 @@ NANOCPUS_SCALE = 1000000000 SECRETS_PATH = '/run/secrets' -COMPOSEFILE_V1 = '1' -COMPOSEFILE_V2_0 = '2.0' -COMPOSEFILE_V2_1 = '2.1' -COMPOSEFILE_V2_2 = '2.2' +COMPOSEFILE_V1 = ComposeVersion('1') +COMPOSEFILE_V2_0 = ComposeVersion('2.0') +COMPOSEFILE_V2_1 = ComposeVersion('2.1') +COMPOSEFILE_V2_2 = ComposeVersion('2.2') -COMPOSEFILE_V3_0 = '3.0' -COMPOSEFILE_V3_1 = '3.1' -COMPOSEFILE_V3_2 = '3.2' -COMPOSEFILE_V3_3 = '3.3' +COMPOSEFILE_V3_0 = ComposeVersion('3.0') +COMPOSEFILE_V3_1 = ComposeVersion('3.1') +COMPOSEFILE_V3_2 = ComposeVersion('3.2') +COMPOSEFILE_V3_3 = ComposeVersion('3.3') API_VERSIONS = { COMPOSEFILE_V1: '1.21', diff --git a/compose/version.py b/compose/version.py new file mode 100644 index 000000000..0532e16c7 --- /dev/null +++ b/compose/version.py @@ -0,0 +1,10 @@ +from __future__ import absolute_import +from __future__ import unicode_literals + +from distutils.version import LooseVersion + + +class ComposeVersion(LooseVersion): + """ A hashable version object """ + def __hash__(self): + return hash(self.vstring) diff --git a/tests/acceptance/cli_test.py b/tests/acceptance/cli_test.py index 9d2de622d..343e4974f 100644 --- a/tests/acceptance/cli_test.py +++ b/tests/acceptance/cli_test.py @@ -850,8 +850,13 @@ class CLITestCase(DockerClientTestCase): # Two networks were created: back and front assert sorted(n['Name'].split('/')[-1] for n in networks) == [back_name, front_name] - back_network = [n for n in networks if n['Name'] == back_name][0] - front_network = [n for n in networks if n['Name'] == front_name][0] + # lookup by ID instead of name in case of duplicates + back_network = self.client.inspect_network( + [n for n in networks if n['Name'] == back_name][0]['Id'] + ) + front_network = self.client.inspect_network( + [n for n in networks if n['Name'] == front_name][0]['Id'] + ) web_container = self.project.get_service('web').containers()[0] app_container = self.project.get_service('app').containers()[0] diff --git a/tests/integration/project_test.py b/tests/integration/project_test.py index 6731f25dd..ce95c5f21 100644 --- a/tests/integration/project_test.py +++ b/tests/integration/project_test.py @@ -34,6 +34,7 @@ from compose.project import Project from compose.project import ProjectError from compose.service import ConvergenceStrategy from tests.integration.testcases import v2_1_only +from tests.integration.testcases import v2_2_only from tests.integration.testcases import v2_only from tests.integration.testcases import v3_only @@ -150,7 +151,7 @@ class ProjectTest(DockerClientTestCase): name='composetest', client=self.client, config_data=load_config({ - 'version': V2_0, + 'version': str(V2_0), 'services': { 'net': { 'image': 'busybox:latest', @@ -178,7 +179,7 @@ class ProjectTest(DockerClientTestCase): return Project.from_config( name='composetest', config_data=load_config({ - 'version': V2_0, + 'version': str(V2_0), 'services': { 'web': { 'image': 'busybox:latest', @@ -820,7 +821,7 @@ class ProjectTest(DockerClientTestCase): def test_up_with_enable_ipv6(self): self.require_api_version('1.23') config_data = build_config( - version=V2_0, + version=V2_1, services=[{ 'name': 'web', 'image': 'busybox:latest', @@ -1003,7 +1004,7 @@ class ProjectTest(DockerClientTestCase): network_name = 'network_with_label' config_data = build_config( - version=V2_0, + version=V2_1, services=[{ 'name': 'web', 'image': 'busybox:latest', @@ -1063,7 +1064,7 @@ class ProjectTest(DockerClientTestCase): volume_name = 'volume_with_label' config_data = build_config( - version=V2_0, + version=V2_1, services=[{ 'name': 'web', 'image': 'busybox:latest', @@ -1103,7 +1104,7 @@ class ProjectTest(DockerClientTestCase): base_file = config.ConfigFile( 'base.yml', { - 'version': V2_0, + 'version': str(V2_0), 'services': { 'simple': {'image': 'busybox:latest', 'command': 'top'}, 'another': { @@ -1122,7 +1123,7 @@ class ProjectTest(DockerClientTestCase): override_file = config.ConfigFile( 'override.yml', { - 'version': V2_0, + 'version': str(V2_0), 'services': { 'another': { 'logging': { @@ -1155,7 +1156,7 @@ class ProjectTest(DockerClientTestCase): base_file = config.ConfigFile( 'base.yml', { - 'version': V2_0, + 'version': str(V2_0), 'services': { 'simple': { 'image': 'busybox:latest', @@ -1168,7 +1169,7 @@ class ProjectTest(DockerClientTestCase): override_file = config.ConfigFile( 'override.yml', { - 'version': V2_0, + 'version': str(V2_0), 'services': { 'simple': { 'ports': ['1234:1234'] @@ -1186,6 +1187,7 @@ class ProjectTest(DockerClientTestCase): containers = project.containers() self.assertEqual(len(containers), 1) + @v2_2_only() def test_project_up_config_scale(self): config_data = build_config( version=V2_2, @@ -1454,7 +1456,7 @@ class ProjectTest(DockerClientTestCase): base_file = config.ConfigFile( 'base.yml', { - 'version': V2_0, + 'version': str(V2_0), 'services': { 'simple': { 'image': 'busybox:latest', diff --git a/tests/integration/testcases.py b/tests/integration/testcases.py index 1e0d63215..7d600f323 100644 --- a/tests/integration/testcases.py +++ b/tests/integration/testcases.py @@ -1,11 +1,10 @@ from __future__ import absolute_import from __future__ import unicode_literals -import functools import os +import pytest from docker.utils import version_lt -from pytest import skip from .. import unittest from ..helpers import is_cluster @@ -17,7 +16,8 @@ from compose.const import COMPOSEFILE_V1 as V1 from compose.const import COMPOSEFILE_V2_0 as V2_0 from compose.const import COMPOSEFILE_V2_0 as V2_1 from compose.const import COMPOSEFILE_V2_2 as V2_2 -from compose.const import COMPOSEFILE_V3_2 as V3_2 +from compose.const import COMPOSEFILE_V3_0 as V3_0 +from compose.const import COMPOSEFILE_V3_3 as V3_3 from compose.const import LABEL_PROJECT from compose.progress_stream import stream_output from compose.service import Service @@ -43,7 +43,7 @@ def get_links(container): def engine_max_version(): if 'DOCKER_VERSION' not in os.environ: - return V3_2 + return V3_3 version = os.environ['DOCKER_VERSION'].partition('-')[0] if version_lt(version, '1.10'): return V1 @@ -51,37 +51,32 @@ def engine_max_version(): return V2_0 if version_lt(version, '1.13'): return V2_1 - return V3_2 + if version_lt(version, '17.06'): + return V2_2 + return V3_3 -def build_version_required_decorator(ignored_versions): - def decorator(f): - @functools.wraps(f) - def wrapper(self, *args, **kwargs): - max_version = engine_max_version() - if max_version in ignored_versions: - skip("Engine version %s is too low" % max_version) - return - return f(self, *args, **kwargs) - return wrapper - - return decorator +def min_version_skip(version): + return pytest.mark.skipif( + engine_max_version() < version, + reason="Engine version %s is too low" % version + ) def v2_only(): - return build_version_required_decorator((V1,)) + return min_version_skip(V2_0) def v2_1_only(): - return build_version_required_decorator((V1, V2_0)) + return min_version_skip(V2_1) def v2_2_only(): - return build_version_required_decorator((V1, V2_0, V2_1)) + return min_version_skip(V2_0) def v3_only(): - return build_version_required_decorator((V1, V2_0, V2_1, V2_2)) + return min_version_skip(V3_0) class DockerClientTestCase(unittest.TestCase): @@ -137,7 +132,7 @@ class DockerClientTestCase(unittest.TestCase): def require_api_version(self, minimum): api_version = self.client.version()['ApiVersion'] if version_lt(api_version, minimum): - skip("API version is too low ({} < {})".format(api_version, minimum)) + pytest.skip("API version is too low ({} < {})".format(api_version, minimum)) def get_volume_data(self, volume_name): if not is_cluster(self.client): diff --git a/tests/unit/bundle_test.py b/tests/unit/bundle_test.py index 3c6e9ec53..847795202 100644 --- a/tests/unit/bundle_test.py +++ b/tests/unit/bundle_test.py @@ -9,6 +9,7 @@ from compose import bundle from compose import service from compose.cli.errors import UserError from compose.config.config import Config +from compose.const import COMPOSEFILE_V2_0 as V2_0 @pytest.fixture @@ -74,7 +75,7 @@ def test_to_bundle(): {'name': 'b', 'build': './b'}, ] config = Config( - version=2, + version=V2_0, services=services, volumes={'special': {}}, networks={'extra': {}}, diff --git a/tests/unit/config/config_test.py b/tests/unit/config/config_test.py index 6731a6bbc..ac742a199 100644 --- a/tests/unit/config/config_test.py +++ b/tests/unit/config/config_test.py @@ -378,7 +378,7 @@ class ConfigTest(unittest.TestCase): base_file = config.ConfigFile( 'base.yaml', { - 'version': V2_1, + 'version': str(V2_1), 'services': { 'web': { 'image': 'example/web', @@ -830,7 +830,7 @@ class ConfigTest(unittest.TestCase): service = config.load( build_config_details( { - 'version': V3_3, + 'version': str(V3_3), 'services': { 'web': { 'build': { @@ -1523,7 +1523,7 @@ class ConfigTest(unittest.TestCase): def test_isolation_option(self): actual = config.load(build_config_details({ - 'version': V2_1, + 'version': str(V2_1), 'services': { 'web': { 'image': 'win10', @@ -4122,7 +4122,7 @@ class SerializeTest(unittest.TestCase): assert serialized_config['secrets']['two'] == secrets_dict['two'] def test_serialize_ports(self): - config_dict = config.Config(version='2.0', services=[ + config_dict = config.Config(version=V2_0, services=[ { 'ports': [types.ServicePort('80', '8080', None, None, None)], 'image': 'alpine', diff --git a/tests/unit/config/interpolation_test.py b/tests/unit/config/interpolation_test.py index 256c74d9b..018a5621a 100644 --- a/tests/unit/config/interpolation_test.py +++ b/tests/unit/config/interpolation_test.py @@ -8,6 +8,8 @@ from compose.config.interpolation import interpolate_environment_variables from compose.config.interpolation import Interpolator from compose.config.interpolation import InvalidInterpolation from compose.config.interpolation import TemplateWithDefaults +from compose.const import COMPOSEFILE_V2_0 as V2_0 +from compose.const import COMPOSEFILE_V3_1 as V3_1 @pytest.fixture @@ -50,7 +52,7 @@ def test_interpolate_environment_variables_in_services(mock_env): } } } - value = interpolate_environment_variables("2.0", services, 'service', mock_env) + value = interpolate_environment_variables(V2_0, services, 'service', mock_env) assert value == expected @@ -75,7 +77,7 @@ def test_interpolate_environment_variables_in_volumes(mock_env): }, 'other': {}, } - value = interpolate_environment_variables("2.0", volumes, 'volume', mock_env) + value = interpolate_environment_variables(V2_0, volumes, 'volume', mock_env) assert value == expected @@ -100,7 +102,7 @@ def test_interpolate_environment_variables_in_secrets(mock_env): }, 'other': {}, } - value = interpolate_environment_variables("3.1", secrets, 'volume', mock_env) + value = interpolate_environment_variables(V3_1, secrets, 'volume', mock_env) assert value == expected diff --git a/tests/unit/project_test.py b/tests/unit/project_test.py index c5366c395..e5f1a175f 100644 --- a/tests/unit/project_test.py +++ b/tests/unit/project_test.py @@ -10,6 +10,8 @@ from .. import mock from .. import unittest from compose.config.config import Config from compose.config.types import VolumeFromSpec +from compose.const import COMPOSEFILE_V1 as V1 +from compose.const import COMPOSEFILE_V2_0 as V2_0 from compose.const import LABEL_SERVICE from compose.container import Container from compose.project import Project @@ -21,9 +23,9 @@ class ProjectTest(unittest.TestCase): def setUp(self): self.mock_client = mock.create_autospec(docker.APIClient) - def test_from_config(self): + def test_from_config_v1(self): config = Config( - version=None, + version=V1, services=[ { 'name': 'web', @@ -53,7 +55,7 @@ class ProjectTest(unittest.TestCase): def test_from_config_v2(self): config = Config( - version=2, + version=V2_0, services=[ { 'name': 'web', @@ -166,7 +168,7 @@ class ProjectTest(unittest.TestCase): name='test', client=self.mock_client, config_data=Config( - version=None, + version=V2_0, services=[{ 'name': 'test', 'image': 'busybox:latest', @@ -194,7 +196,7 @@ class ProjectTest(unittest.TestCase): name='test', client=self.mock_client, config_data=Config( - version=None, + version=V2_0, services=[ { 'name': 'vol', @@ -221,7 +223,7 @@ class ProjectTest(unittest.TestCase): name='test', client=None, config_data=Config( - version=None, + version=V2_0, services=[ { 'name': 'vol', @@ -361,7 +363,7 @@ class ProjectTest(unittest.TestCase): name='test', client=self.mock_client, config_data=Config( - version=None, + version=V1, services=[ { 'name': 'test', @@ -386,7 +388,7 @@ class ProjectTest(unittest.TestCase): name='test', client=self.mock_client, config_data=Config( - version=None, + version=V2_0, services=[ { 'name': 'test', @@ -417,7 +419,7 @@ class ProjectTest(unittest.TestCase): name='test', client=self.mock_client, config_data=Config( - version=None, + version=V2_0, services=[ { 'name': 'aaa', @@ -444,7 +446,7 @@ class ProjectTest(unittest.TestCase): name='test', client=self.mock_client, config_data=Config( - version=2, + version=V2_0, services=[ { 'name': 'foo', @@ -465,7 +467,7 @@ class ProjectTest(unittest.TestCase): name='test', client=self.mock_client, config_data=Config( - version=2, + version=V2_0, services=[ { 'name': 'foo', @@ -500,7 +502,7 @@ class ProjectTest(unittest.TestCase): name='test', client=self.mock_client, config_data=Config( - version=None, + version=V2_0, services=[{ 'name': 'web', 'image': 'busybox:latest', @@ -518,7 +520,7 @@ class ProjectTest(unittest.TestCase): name='test', client=self.mock_client, config_data=Config( - version='2', + version=V2_0, services=[{ 'name': 'web', 'image': 'busybox:latest', From 344a69331cb3842c6f2f9f18eef22dea10ca21a5 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Tue, 11 Jul 2017 19:07:12 -0700 Subject: [PATCH 075/244] Bump 1.15.0-rc1 Signed-off-by: Joffrey F --- CHANGELOG.md | 44 ++++++++++++++++++++++++++++++++++ compose/__init__.py | 2 +- script/run/run.sh | 2 +- tests/integration/testcases.py | 5 ++-- 4 files changed, 49 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index cced3804c..c4b97a756 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,50 @@ Change log ========== +1.15.0 (2017-07-18) +------------------- + +### New features + +#### Compose file version 2.2 + +- Added support for the `network` parameter in build configurations. + +#### Compose file version 2.1 and up + +- The `pid` option in a service's definition now supports a `service:` + value. + +- Added support for the `storage_opt` parameter in in service definitions. + This option is not available for the v3 format + +#### All formats + +- Added `--quiet` flag to `docker-compose pull`, suppressing progress output + +- Some improvements to CLI output + +### Bugfixes + +- Volumes specified through the `--volume` flag of `docker-compose run` now + complement volumes declared in the service's defintion instead of replacing + them + +- Fixed a bug where using multiple Compose files would unset the scale value + defined inside the Compose file. + +- Fixed an issue where the `credHelpers` entries in the `config.json` file + were not being honored by Compose + +- Fixed a bug where using multiple Compose files with port declarations + would cause failures in Python 3 environments + +- Fixed a bug where some proxy-related options present in the user's + environment would prevent Compose from running + +- Fixed an issue where the output of `docker-compose config` would be invalid + if the original file used `Y` or `N` values + 1.14.0 (2017-06-19) ------------------- diff --git a/compose/__init__.py b/compose/__init__.py index 1898479bf..c040b295f 100644 --- a/compose/__init__.py +++ b/compose/__init__.py @@ -1,4 +1,4 @@ from __future__ import absolute_import from __future__ import unicode_literals -__version__ = '1.15.0dev' +__version__ = '1.15.0-rc1' diff --git a/script/run/run.sh b/script/run/run.sh index e4a2f4199..4f0f764b3 100755 --- a/script/run/run.sh +++ b/script/run/run.sh @@ -15,7 +15,7 @@ set -e -VERSION="1.14.0" +VERSION="1.15.0-rc1" IMAGE="docker/compose:$VERSION" diff --git a/tests/integration/testcases.py b/tests/integration/testcases.py index 7d600f323..fd30744b1 100644 --- a/tests/integration/testcases.py +++ b/tests/integration/testcases.py @@ -17,6 +17,7 @@ from compose.const import COMPOSEFILE_V2_0 as V2_0 from compose.const import COMPOSEFILE_V2_0 as V2_1 from compose.const import COMPOSEFILE_V2_2 as V2_2 from compose.const import COMPOSEFILE_V3_0 as V3_0 +from compose.const import COMPOSEFILE_V3_2 as V3_2 from compose.const import COMPOSEFILE_V3_3 as V3_3 from compose.const import LABEL_PROJECT from compose.progress_stream import stream_output @@ -52,7 +53,7 @@ def engine_max_version(): if version_lt(version, '1.13'): return V2_1 if version_lt(version, '17.06'): - return V2_2 + return V3_2 return V3_3 @@ -72,7 +73,7 @@ def v2_1_only(): def v2_2_only(): - return min_version_skip(V2_0) + return min_version_skip(V2_2) def v3_only(): From 31b161045df859e3ca4f4aea865ac2f6aea993c3 Mon Sep 17 00:00:00 2001 From: Alexey Rokhin Date: Wed, 17 May 2017 16:18:28 +0300 Subject: [PATCH 076/244] skip cpu_percent test for Linux Signed-off-by: Alexey Rokhin --- tests/integration/service_test.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/integration/service_test.py b/tests/integration/service_test.py index 350f7398b..feec6a870 100644 --- a/tests/integration/service_test.py +++ b/tests/integration/service_test.py @@ -30,6 +30,7 @@ from compose.const import LABEL_ONE_OFF from compose.const import LABEL_PROJECT from compose.const import LABEL_SERVICE from compose.const import LABEL_VERSION +from compose.const import IS_WINDOWS_PLATFORM from compose.container import Container from compose.errors import OperationFailedError from compose.project import OneOffFilter From fd862f9ca77dcbcf15d818da0d8ce6dea76244e9 Mon Sep 17 00:00:00 2001 From: Alexey Rokhin Date: Wed, 17 May 2017 16:42:43 +0300 Subject: [PATCH 077/244] service_test.py reorder imports Signed-off-by: Alexey Rokhin --- tests/integration/service_test.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/integration/service_test.py b/tests/integration/service_test.py index feec6a870..350f7398b 100644 --- a/tests/integration/service_test.py +++ b/tests/integration/service_test.py @@ -30,7 +30,6 @@ from compose.const import LABEL_ONE_OFF from compose.const import LABEL_PROJECT from compose.const import LABEL_SERVICE from compose.const import LABEL_VERSION -from compose.const import IS_WINDOWS_PLATFORM from compose.container import Container from compose.errors import OperationFailedError from compose.project import OneOffFilter From 24065a15d89fe060330d9732295e2a1eecb3b793 Mon Sep 17 00:00:00 2001 From: Joel Barciauskas Date: Wed, 12 Apr 2017 17:45:09 -0400 Subject: [PATCH 078/244] Add --quiet parameter to docker-compose pull, using existing silent flag Signed-off-by: Joel Barciauskas --- compose/project.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/compose/project.py b/compose/project.py index 28af45c71..a2a398b37 100644 --- a/compose/project.py +++ b/compose/project.py @@ -493,7 +493,7 @@ class Project(object): if parallel_pull: def pull_service(service): - service.pull(ignore_pull_failures, True) + service.pull(ignore_pull_failures, True, silent=silent) parallel.parallel_execute( services, From 051726696b83dedf1a1cc6e6ef796666cfa875b1 Mon Sep 17 00:00:00 2001 From: NikitaVlaznev Date: Mon, 19 Jun 2017 17:05:19 +0300 Subject: [PATCH 079/244] Fix double silent argument value Fix for "TypeError: pull() got multiple values for keyword argument 'silent'." This change https://github.com/docker/compose/commit/e9b6cc23fcf01d4768c7e082b7bc91b43ff84e7e caused additional value to be passed for the 'silent' argument, that was already passed there: https://github.com/docker/compose/commit/f85da99ef3273794e855afda8678174419d3bf4f Signed-off-by: Nikita Vlaznev --- compose/project.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/compose/project.py b/compose/project.py index a2a398b37..e07514895 100644 --- a/compose/project.py +++ b/compose/project.py @@ -493,7 +493,7 @@ class Project(object): if parallel_pull: def pull_service(service): - service.pull(ignore_pull_failures, True, silent=silent) + service.pull(ignore_pull_failures, silent=silent) parallel.parallel_execute( services, From 85b908ebefd2069ccafd2f44af5bced1888ee8b0 Mon Sep 17 00:00:00 2001 From: Evan Shaw Date: Sat, 1 Jul 2017 13:40:02 +1200 Subject: [PATCH 080/244] Always silence pull output with --parallel This is how things were prior to the addition of the --quiet flag. Making it not silent produces output that's weird and difficult to read. Signed-off-by: Evan Shaw --- compose/project.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/compose/project.py b/compose/project.py index e07514895..28af45c71 100644 --- a/compose/project.py +++ b/compose/project.py @@ -493,7 +493,7 @@ class Project(object): if parallel_pull: def pull_service(service): - service.pull(ignore_pull_failures, silent=silent) + service.pull(ignore_pull_failures, True) parallel.parallel_execute( services, From c923ea1320d0a31cc8d62edeb5c60867155ef1d0 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Wed, 19 Jul 2017 15:01:39 -0700 Subject: [PATCH 081/244] Some more test adjustments for Swarm support Signed-off-by: Joffrey F --- .dockerignore | 2 ++ tests/acceptance/cli_test.py | 26 +++++++++----- .../ports-composefile/expanded-notation.yml | 6 ++-- tests/helpers.py | 35 ------------------ tests/integration/project_test.py | 4 +-- tests/integration/service_test.py | 23 +++++++++--- tests/integration/testcases.py | 36 ++++++++++++++++++- tests/integration/volume_test.py | 2 +- tox.ini | 2 ++ 9 files changed, 82 insertions(+), 54 deletions(-) diff --git a/.dockerignore b/.dockerignore index 055ae7ed1..eccd86dda 100644 --- a/.dockerignore +++ b/.dockerignore @@ -7,3 +7,5 @@ coverage-html docs/_site venv .tox +**/__pycache__ +*.pyc diff --git a/tests/acceptance/cli_test.py b/tests/acceptance/cli_test.py index 343e4974f..fc05de351 100644 --- a/tests/acceptance/cli_test.py +++ b/tests/acceptance/cli_test.py @@ -20,8 +20,6 @@ from docker import errors from .. import mock from ..helpers import create_host_file -from ..helpers import is_cluster -from ..helpers import no_cluster from compose.cli.command import get_project from compose.config.errors import DuplicateOverrideFileFound from compose.container import Container @@ -29,6 +27,8 @@ from compose.project import OneOffFilter from compose.utils import nanoseconds_from_time_seconds from tests.integration.testcases import DockerClientTestCase from tests.integration.testcases import get_links +from tests.integration.testcases import is_cluster +from tests.integration.testcases import no_cluster from tests.integration.testcases import pull_busybox from tests.integration.testcases import SWARM_SKIP_RM_VOLUMES from tests.integration.testcases import v2_1_only @@ -116,7 +116,7 @@ class CLITestCase(DockerClientTestCase): def tearDown(self): if self.base_dir: self.project.kill() - self.project.remove_stopped() + self.project.down(None, True) for container in self.project.containers(stopped=True, one_off=OneOffFilter.only): container.remove(force=True) @@ -1214,6 +1214,7 @@ class CLITestCase(DockerClientTestCase): self.assertEqual(proc.returncode, 1) @v2_only() + @no_cluster('Container PID mode does not work across clusters') def test_up_with_pid_mode(self): c = self.client.create_container( 'busybox', 'top', name='composetest_pid_mode_container', @@ -1244,8 +1245,8 @@ class CLITestCase(DockerClientTestCase): self.assertEqual(len(self.project.containers()), 1) stdout, stderr = self.dispatch(['exec', '-T', 'console', 'ls', '-1d', '/']) - self.assertEqual(stdout, "/\n") self.assertEqual(stderr, "") + self.assertEqual(stdout, "/\n") def test_exec_custom_user(self): self.base_dir = 'tests/fixtures/links-composefile' @@ -1826,7 +1827,13 @@ class CLITestCase(DockerClientTestCase): result = self.dispatch(['logs', '-f']) - assert result.stdout.count('\n') == 5 + if not is_cluster(self.client): + assert result.stdout.count('\n') == 5 + else: + # Sometimes logs are picked up from old containers that haven't yet + # been removed (removal in Swarm is async) + assert result.stdout.count('\n') >= 5 + assert 'simple' in result.stdout assert 'another' in result.stdout assert 'exited with code 0' in result.stdout @@ -1882,7 +1889,10 @@ class CLITestCase(DockerClientTestCase): self.dispatch(['up']) result = self.dispatch(['logs', '--tail', '2']) - assert result.stdout.count('\n') == 3 + assert 'c\n' in result.stdout + assert 'd\n' in result.stdout + assert 'a\n' not in result.stdout + assert 'b\n' not in result.stdout def test_kill(self): self.dispatch(['up', '-d'], None) @@ -2045,8 +2055,8 @@ class CLITestCase(DockerClientTestCase): return result.stdout.rstrip() assert get_port(3000) == container.get_local_port(3000) - assert ':49152' in get_port(3001) - assert ':49153' in get_port(3002) + assert ':53222' in get_port(3001) + assert ':53223' in get_port(3002) def test_port_with_scale(self): self.base_dir = 'tests/fixtures/ports-composefile-scale' diff --git a/tests/fixtures/ports-composefile/expanded-notation.yml b/tests/fixtures/ports-composefile/expanded-notation.yml index 6fbe59176..09a7a2bf9 100644 --- a/tests/fixtures/ports-composefile/expanded-notation.yml +++ b/tests/fixtures/ports-composefile/expanded-notation.yml @@ -6,10 +6,10 @@ services: ports: - target: 3000 - target: 3001 - published: 49152 + published: 53222 - target: 3002 - published: 49153 + published: 53223 protocol: tcp - target: 3003 - published: 49154 + published: 53224 protocol: udp diff --git a/tests/helpers.py b/tests/helpers.py index 662353c93..59efd2557 100644 --- a/tests/helpers.py +++ b/tests/helpers.py @@ -1,12 +1,8 @@ from __future__ import absolute_import from __future__ import unicode_literals -import functools import os -from docker.errors import APIError -from pytest import skip - from compose.config.config import ConfigDetails from compose.config.config import ConfigFile from compose.config.config import load @@ -48,34 +44,3 @@ def create_host_file(client, filename): "Container exited with code {}:\n{}".format(exitcode, output)) finally: client.remove_container(container, force=True) - - -def is_cluster(client): - nodes = None - - def get_nodes_number(): - try: - return len(client.nodes()) - except APIError: - # If the Engine is not part of a Swarm, the SDK will raise - # an APIError - return 0 - - if nodes is None: - # Only make the API call if the value hasn't been cached yet - nodes = get_nodes_number() - - return nodes > 1 - - -def no_cluster(reason): - def decorator(f): - @functools.wraps(f) - def wrapper(self, *args, **kwargs): - if is_cluster(self.client): - skip("Test will not be run in cluster mode: %s" % reason) - return - return f(self, *args, **kwargs) - return wrapper - - return decorator diff --git a/tests/integration/project_test.py b/tests/integration/project_test.py index ce95c5f21..5ead7b8e7 100644 --- a/tests/integration/project_test.py +++ b/tests/integration/project_test.py @@ -12,8 +12,6 @@ from docker.errors import NotFound from .. import mock from ..helpers import build_config as load_config from ..helpers import create_host_file -from ..helpers import is_cluster -from ..helpers import no_cluster from .testcases import DockerClientTestCase from .testcases import SWARM_SKIP_CONTAINERS_ALL from compose.config import config @@ -33,6 +31,8 @@ from compose.errors import NoHealthCheckConfigured from compose.project import Project from compose.project import ProjectError from compose.service import ConvergenceStrategy +from tests.integration.testcases import is_cluster +from tests.integration.testcases import no_cluster from tests.integration.testcases import v2_1_only from tests.integration.testcases import v2_2_only from tests.integration.testcases import v2_only diff --git a/tests/integration/service_test.py b/tests/integration/service_test.py index 350f7398b..ff75015df 100644 --- a/tests/integration/service_test.py +++ b/tests/integration/service_test.py @@ -13,8 +13,6 @@ from six import StringIO from six import text_type from .. import mock -from ..helpers import is_cluster -from ..helpers import no_cluster from .testcases import DockerClientTestCase from .testcases import get_links from .testcases import pull_busybox @@ -38,6 +36,8 @@ from compose.service import ConvergenceStrategy from compose.service import NetworkMode from compose.service import PidMode from compose.service import Service +from tests.integration.testcases import is_cluster +from tests.integration.testcases import no_cluster from tests.integration.testcases import v2_1_only from tests.integration.testcases import v2_2_only from tests.integration.testcases import v2_only @@ -635,7 +635,10 @@ class ServiceTest(DockerClientTestCase): with open(os.path.join(base_dir, 'Dockerfile'), 'w') as f: f.write("FROM busybox\n") - self.create_service('web', build={'context': base_dir}).build() + service = self.create_service('web', build={'context': base_dir}) + service.build() + self.addCleanup(self.client.remove_image, service.image_name) + assert self.client.inspect_image('composetest_web') def test_build_non_ascii_filename(self): @@ -648,7 +651,9 @@ class ServiceTest(DockerClientTestCase): with open(os.path.join(base_dir.encode('utf8'), b'foo\xE2bar'), 'w') as f: f.write("hello world\n") - self.create_service('web', build={'context': text_type(base_dir)}).build() + service = self.create_service('web', build={'context': text_type(base_dir)}) + service.build() + self.addCleanup(self.client.remove_image, service.image_name) assert self.client.inspect_image('composetest_web') def test_build_with_image_name(self): @@ -683,6 +688,7 @@ class ServiceTest(DockerClientTestCase): build={'context': text_type(base_dir), 'args': {"build_version": "1"}}) service.build() + self.addCleanup(self.client.remove_image, service.image_name) assert service.image() assert "build_version=1" in service.image()['ContainerConfig']['Cmd'] @@ -699,6 +705,8 @@ class ServiceTest(DockerClientTestCase): build={'context': text_type(base_dir), 'args': {"build_version": "1"}}) service.build(build_args_override={'build_version': '2'}) + self.addCleanup(self.client.remove_image, service.image_name) + assert service.image() assert "build_version=2" in service.image()['ContainerConfig']['Cmd'] @@ -714,9 +722,12 @@ class ServiceTest(DockerClientTestCase): 'labels': {'com.docker.compose.test': 'true'} }) service.build() + self.addCleanup(self.client.remove_image, service.image_name) + assert service.image() assert service.image()['Config']['Labels']['com.docker.compose.test'] == 'true' + @no_cluster('Container networks not on Swarm') def test_build_with_network(self): base_dir = tempfile.mkdtemp() self.addCleanup(shutil.rmtree, base_dir) @@ -739,6 +750,8 @@ class ServiceTest(DockerClientTestCase): }) service.build() + self.addCleanup(self.client.remove_image, service.image_name) + assert service.image() def test_start_container_stays_unprivileged(self): @@ -1130,6 +1143,8 @@ class ServiceTest(DockerClientTestCase): build={'context': base_dir, 'cache_from': ['build1']}) service.build() + self.addCleanup(self.client.remove_image, service.image_name) + assert service.image() @mock.patch.dict(os.environ) diff --git a/tests/integration/testcases.py b/tests/integration/testcases.py index fd30744b1..1b451ef3c 100644 --- a/tests/integration/testcases.py +++ b/tests/integration/testcases.py @@ -1,13 +1,14 @@ from __future__ import absolute_import from __future__ import unicode_literals +import functools import os import pytest +from docker.errors import APIError from docker.utils import version_lt from .. import unittest -from ..helpers import is_cluster from compose.cli.docker_client import docker_client from compose.config.config import resolve_environment from compose.config.environment import Environment @@ -26,6 +27,7 @@ from compose.service import Service SWARM_SKIP_CONTAINERS_ALL = os.environ.get('SWARM_SKIP_CONTAINERS_ALL', '0') != '0' SWARM_SKIP_CPU_SHARES = os.environ.get('SWARM_SKIP_CPU_SHARES', '0') != '0' SWARM_SKIP_RM_VOLUMES = os.environ.get('SWARM_SKIP_RM_VOLUMES', '0') != '0' +SWARM_ASSUME_MULTINODE = os.environ.get('SWARM_ASSUME_MULTINODE', '0') != '0' def pull_busybox(client): @@ -142,3 +144,35 @@ class DockerClientTestCase(unittest.TestCase): volumes = self.client.volumes(filters={'name': volume_name})['Volumes'] assert len(volumes) > 0 return self.client.inspect_volume(volumes[0]['Name']) + + +def is_cluster(client): + if SWARM_ASSUME_MULTINODE: + return True + + def get_nodes_number(): + try: + return len(client.nodes()) + except APIError: + # If the Engine is not part of a Swarm, the SDK will raise + # an APIError + return 0 + + if not hasattr(is_cluster, 'nodes') or is_cluster.nodes is None: + # Only make the API call if the value hasn't been cached yet + is_cluster.nodes = get_nodes_number() + + return is_cluster.nodes > 1 + + +def no_cluster(reason): + def decorator(f): + @functools.wraps(f) + def wrapper(self, *args, **kwargs): + if is_cluster(self.client): + pytest.skip("Test will not be run in cluster mode: %s" % reason) + return + return f(self, *args, **kwargs) + return wrapper + + return decorator diff --git a/tests/integration/volume_test.py b/tests/integration/volume_test.py index 772631a5b..ecc71d0b1 100644 --- a/tests/integration/volume_test.py +++ b/tests/integration/volume_test.py @@ -3,8 +3,8 @@ from __future__ import unicode_literals from docker.errors import DockerException -from ..helpers import no_cluster from .testcases import DockerClientTestCase +from .testcases import no_cluster from compose.const import LABEL_PROJECT from compose.const import LABEL_VOLUME from compose.volume import Volume diff --git a/tox.ini b/tox.ini index 61bc05745..749be3faa 100644 --- a/tox.ini +++ b/tox.ini @@ -9,6 +9,8 @@ passenv = DOCKER_CERT_PATH DOCKER_TLS_VERIFY DOCKER_VERSION + SWARM_SKIP_* + SWARM_ASSUME_MULTINODE setenv = HOME=/tmp deps = From 686a533c9f76ed5994222351f45beb3da7b7040b Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Thu, 20 Jul 2017 14:09:12 -0700 Subject: [PATCH 082/244] Scripts build and push compose-tests image Signed-off-by: Joffrey F --- script/build/test-image | 17 +++++++++++++++++ script/release/build-binaries | 3 +++ script/release/push-release | 4 ++++ 3 files changed, 24 insertions(+) create mode 100755 script/build/test-image diff --git a/script/build/test-image b/script/build/test-image new file mode 100755 index 000000000..216d63f9c --- /dev/null +++ b/script/build/test-image @@ -0,0 +1,17 @@ +#!/bin/bash + +set -e + +if [ -z "$1" ]; then + >&2 echo "First argument must be image tag." + exit 1 +fi + +TAG=$1 + +docker build -t docker-compose-tests:tmp . +ctnr_id=$(docker create --entrypoint=tox docker-compose-tests:tmp) +docker commit $ctnr_id docker/compose-tests:latest +docker tag docker/compose-tests:latest docker/compose-tests:$TAG +docker rm -f $ctnr_id +docker rmi -f docker-compose-tests:tmp \ No newline at end of file diff --git a/script/release/build-binaries b/script/release/build-binaries index 9d4a606e2..a39b186d9 100755 --- a/script/release/build-binaries +++ b/script/release/build-binaries @@ -27,6 +27,9 @@ script/build/linux echo "Building the container distribution" script/build/image $VERSION +echo "Building the compose-tests image" +script/build/test-image $VERSION + echo "Create a github release" # TODO: script more of this https://developer.github.com/v3/repos/releases/ browser https://github.com/$REPO/releases/new diff --git a/script/release/push-release b/script/release/push-release index 9db6f6894..0578aaff8 100755 --- a/script/release/push-release +++ b/script/release/push-release @@ -54,6 +54,10 @@ git push $GITHUB_REPO $VERSION echo "Uploading the docker image" docker push docker/compose:$VERSION +echo "Uploading the compose-tests image" +docker push docker/compose-tests:latest +docker push docker/compose-tests:$VERSION + echo "Uploading package to PyPI" pandoc -f markdown -t rst README.md -o README.rst sed -i -e 's/logo.png?raw=true/https:\/\/github.com\/docker\/compose\/raw\/master\/logo.png?raw=true/' README.rst From 0e9308085094ab94db90cad0e1b12507f438de9a Mon Sep 17 00:00:00 2001 From: Kirin Rastogi Date: Tue, 25 Jul 2017 11:08:01 -0400 Subject: [PATCH 083/244] Add exclusion for networkname Signed-off-by: Kirin Rastogi Signed-off-by: Kirin Rastogi --- compose/network.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/compose/network.py b/compose/network.py index fec839162..0f42eb20a 100644 --- a/compose/network.py +++ b/compose/network.py @@ -18,7 +18,8 @@ log = logging.getLogger(__name__) OPTS_EXCEPTIONS = [ 'com.docker.network.driver.overlay.vxlanid_list', - 'com.docker.network.windowsshim.hnsid' + 'com.docker.network.windowsshim.hnsid', + 'com.docker.network.windowsshim.networkname' ] From ade23b585ef3fdbdb2734ffced1008bb951ef7eb Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Tue, 25 Jul 2017 16:02:11 -0700 Subject: [PATCH 084/244] Bump 1.15.0 Signed-off-by: Joffrey F --- CHANGELOG.md | 5 ++++- compose/__init__.py | 2 +- script/run/run.sh | 2 +- 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c4b97a756..928922782 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,7 +1,7 @@ Change log ========== -1.15.0 (2017-07-18) +1.15.0 (2017-07-26) ------------------- ### New features @@ -45,6 +45,9 @@ Change log - Fixed an issue where the output of `docker-compose config` would be invalid if the original file used `Y` or `N` values +- Fixed an issue preventing `up` operations on a previously created stack on + Windows Engine. + 1.14.0 (2017-06-19) ------------------- diff --git a/compose/__init__.py b/compose/__init__.py index c040b295f..f238607c0 100644 --- a/compose/__init__.py +++ b/compose/__init__.py @@ -1,4 +1,4 @@ from __future__ import absolute_import from __future__ import unicode_literals -__version__ = '1.15.0-rc1' +__version__ = '1.15.0' diff --git a/script/run/run.sh b/script/run/run.sh index 4f0f764b3..47a81c7f8 100755 --- a/script/run/run.sh +++ b/script/run/run.sh @@ -15,7 +15,7 @@ set -e -VERSION="1.15.0-rc1" +VERSION="1.15.0" IMAGE="docker/compose:$VERSION" From ec5d8264c956541a4dd1309da6ba93687d246904 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Mon, 17 Apr 2017 19:03:56 -0700 Subject: [PATCH 085/244] Implement --scale option on up command, allow scale config in v2.2 format docker-compose scale modified to reuse code between up and scale Signed-off-by: Joffrey F --- compose/service.py | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/compose/service.py b/compose/service.py index 53ad46362..c8c2bd982 100644 --- a/compose/service.py +++ b/compose/service.py @@ -392,7 +392,7 @@ class Service(object): range(i, i + scale), lambda n: create_and_start(self, n), lambda n: self.get_container_name(n), - "Creating" + "Creating", ) for error in errors.values(): raise OperationFailedError(error) @@ -413,7 +413,7 @@ class Service(object): containers, recreate, lambda c: c.name, - "Recreating" + "Recreating", ) for error in errors.values(): raise OperationFailedError(error) @@ -433,7 +433,7 @@ class Service(object): containers, lambda c: self.start_container_if_stopped(c, attach_logs=not detached), lambda c: c.name, - "Starting" + "Starting", ) for error in errors.values(): @@ -868,7 +868,7 @@ class Service(object): volume_driver=options.get('volume_driver'), cpuset_cpus=options.get('cpuset'), cpu_shares=options.get('cpu_shares'), - storage_opt=options.get('storage_opt') + storage_opt=options.get('storage_opt'), ) def get_secret_volumes(self): @@ -905,9 +905,7 @@ class Service(object): nocache=no_cache, dockerfile=build_opts.get('dockerfile', None), cache_from=build_opts.get('cache_from', None), - labels=build_opts.get('labels', None), - buildargs=build_args, - network_mode=build_opts.get('network', None), + buildargs=build_args ) try: From 92873edd65ac6f56f2fe57842bfde42baea67853 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Mon, 1 May 2017 14:29:37 -0700 Subject: [PATCH 086/244] Fix external secrets serialization Signed-off-by: Joffrey F --- compose/config/serialize.py | 1 + 1 file changed, 1 insertion(+) diff --git a/compose/config/serialize.py b/compose/config/serialize.py index 84521848d..3fdd4d392 100644 --- a/compose/config/serialize.py +++ b/compose/config/serialize.py @@ -52,6 +52,7 @@ def denormalize_config(config, image_digests=None): service_dict.pop('name'): service_dict for service_dict in denormalized_services } + for key in ('networks', 'volumes', 'secrets', 'configs'): config_dict = getattr(config, key) if not config_dict: From c006add122a85bdf5cf2f981aa91692c9ad2f49f Mon Sep 17 00:00:00 2001 From: Yong Wen Chua Date: Mon, 10 Jul 2017 12:35:34 +0800 Subject: [PATCH 087/244] Add Compose v2.3 Signed-off-by: Yong Wen Chua --- compose/config/config_schema_v2.3.json | 401 +++++++++++++++++++++++++ compose/const.py | 3 + docker-compose.spec | 5 + tests/integration/testcases.py | 5 + tests/unit/config/config_test.py | 4 + 5 files changed, 418 insertions(+) create mode 100644 compose/config/config_schema_v2.3.json diff --git a/compose/config/config_schema_v2.3.json b/compose/config/config_schema_v2.3.json new file mode 100644 index 000000000..abcc2ded2 --- /dev/null +++ b/compose/config/config_schema_v2.3.json @@ -0,0 +1,401 @@ +{ + "$schema": "http://json-schema.org/draft-04/schema#", + "id": "config_schema_v2.3.json", + "type": "object", + + "properties": { + "version": { + "type": "string" + }, + + "services": { + "id": "#/properties/services", + "type": "object", + "patternProperties": { + "^[a-zA-Z0-9._-]+$": { + "$ref": "#/definitions/service" + } + }, + "additionalProperties": false + }, + + "networks": { + "id": "#/properties/networks", + "type": "object", + "patternProperties": { + "^[a-zA-Z0-9._-]+$": { + "$ref": "#/definitions/network" + } + } + }, + + "volumes": { + "id": "#/properties/volumes", + "type": "object", + "patternProperties": { + "^[a-zA-Z0-9._-]+$": { + "$ref": "#/definitions/volume" + } + }, + "additionalProperties": false + } + }, + + "additionalProperties": false, + + "definitions": { + + "service": { + "id": "#/definitions/service", + "type": "object", + + "properties": { + "build": { + "oneOf": [ + {"type": "string"}, + { + "type": "object", + "properties": { + "context": {"type": "string"}, + "dockerfile": {"type": "string"}, + "args": {"$ref": "#/definitions/list_or_dict"}, + "labels": {"$ref": "#/definitions/list_or_dict"}, + "cache_from": {"$ref": "#/definitions/list_of_strings"}, + "network": {"type": "string"} + }, + "additionalProperties": false + } + ] + }, + "cap_add": {"type": "array", "items": {"type": "string"}, "uniqueItems": true}, + "cap_drop": {"type": "array", "items": {"type": "string"}, "uniqueItems": true}, + "cgroup_parent": {"type": "string"}, + "command": { + "oneOf": [ + {"type": "string"}, + {"type": "array", "items": {"type": "string"}} + ] + }, + "container_name": {"type": "string"}, + "cpu_count": {"type": "integer", "minimum": 0}, + "cpu_percent": {"type": "integer", "minimum": 0, "maximum": 100}, + "cpu_shares": {"type": ["number", "string"]}, + "cpu_quota": {"type": ["number", "string"]}, + "cpus": {"type": "number", "minimum": 0}, + "cpuset": {"type": "string"}, + "depends_on": { + "oneOf": [ + {"$ref": "#/definitions/list_of_strings"}, + { + "type": "object", + "additionalProperties": false, + "patternProperties": { + "^[a-zA-Z0-9._-]+$": { + "type": "object", + "additionalProperties": false, + "properties": { + "condition": { + "type": "string", + "enum": ["service_started", "service_healthy"] + } + }, + "required": ["condition"] + } + } + } + ] + }, + "devices": {"type": "array", "items": {"type": "string"}, "uniqueItems": true}, + "dns_opt": { + "type": "array", + "items": { + "type": "string" + }, + "uniqueItems": true + }, + "dns": {"$ref": "#/definitions/string_or_list"}, + "dns_search": {"$ref": "#/definitions/string_or_list"}, + "domainname": {"type": "string"}, + "entrypoint": { + "oneOf": [ + {"type": "string"}, + {"type": "array", "items": {"type": "string"}} + ] + }, + "env_file": {"$ref": "#/definitions/string_or_list"}, + "environment": {"$ref": "#/definitions/list_or_dict"}, + + "expose": { + "type": "array", + "items": { + "type": ["string", "number"], + "format": "expose" + }, + "uniqueItems": true + }, + + "extends": { + "oneOf": [ + { + "type": "string" + }, + { + "type": "object", + + "properties": { + "service": {"type": "string"}, + "file": {"type": "string"} + }, + "required": ["service"], + "additionalProperties": false + } + ] + }, + + "external_links": {"type": "array", "items": {"type": "string"}, "uniqueItems": true}, + "extra_hosts": {"$ref": "#/definitions/list_or_dict"}, + "healthcheck": {"$ref": "#/definitions/healthcheck"}, + "hostname": {"type": "string"}, + "image": {"type": "string"}, + "init": {"type": ["boolean", "string"]}, + "ipc": {"type": "string"}, + "isolation": {"type": "string"}, + "labels": {"$ref": "#/definitions/list_or_dict"}, + "links": {"type": "array", "items": {"type": "string"}, "uniqueItems": true}, + + "logging": { + "type": "object", + + "properties": { + "driver": {"type": "string"}, + "options": {"type": "object"} + }, + "additionalProperties": false + }, + + "mac_address": {"type": "string"}, + "mem_limit": {"type": ["number", "string"]}, + "mem_reservation": {"type": ["string", "integer"]}, + "mem_swappiness": {"type": "integer"}, + "memswap_limit": {"type": ["number", "string"]}, + "network_mode": {"type": "string"}, + + "networks": { + "oneOf": [ + {"$ref": "#/definitions/list_of_strings"}, + { + "type": "object", + "patternProperties": { + "^[a-zA-Z0-9._-]+$": { + "oneOf": [ + { + "type": "object", + "properties": { + "aliases": {"$ref": "#/definitions/list_of_strings"}, + "ipv4_address": {"type": "string"}, + "ipv6_address": {"type": "string"}, + "link_local_ips": {"$ref": "#/definitions/list_of_strings"} + }, + "additionalProperties": false + }, + {"type": "null"} + ] + } + }, + "additionalProperties": false + } + ] + }, + "oom_score_adj": {"type": "integer", "minimum": -1000, "maximum": 1000}, + "group_add": { + "type": "array", + "items": { + "type": ["string", "number"] + }, + "uniqueItems": true + }, + "pid": {"type": ["string", "null"]}, + + "ports": { + "type": "array", + "items": { + "type": ["string", "number"], + "format": "ports" + }, + "uniqueItems": true + }, + + "privileged": {"type": "boolean"}, + "read_only": {"type": "boolean"}, + "restart": {"type": "string"}, + "scale": {"type": "integer"}, + "security_opt": {"type": "array", "items": {"type": "string"}, "uniqueItems": true}, + "shm_size": {"type": ["number", "string"]}, + "sysctls": {"$ref": "#/definitions/list_or_dict"}, + "pids_limit": {"type": ["number", "string"]}, + "stdin_open": {"type": "boolean"}, + "stop_grace_period": {"type": "string", "format": "duration"}, + "stop_signal": {"type": "string"}, + "storage_opt": {"type": "object"}, + "tmpfs": {"$ref": "#/definitions/string_or_list"}, + "tty": {"type": "boolean"}, + "ulimits": { + "type": "object", + "patternProperties": { + "^[a-z]+$": { + "oneOf": [ + {"type": "integer"}, + { + "type":"object", + "properties": { + "hard": {"type": "integer"}, + "soft": {"type": "integer"} + }, + "required": ["soft", "hard"], + "additionalProperties": false + } + ] + } + } + }, + "user": {"type": "string"}, + "userns_mode": {"type": "string"}, + "volumes": {"type": "array", "items": {"type": "string"}, "uniqueItems": true}, + "volume_driver": {"type": "string"}, + "volumes_from": {"type": "array", "items": {"type": "string"}, "uniqueItems": true}, + "working_dir": {"type": "string"} + }, + + "dependencies": { + "memswap_limit": ["mem_limit"] + }, + "additionalProperties": false + }, + + "healthcheck": { + "id": "#/definitions/healthcheck", + "type": "object", + "additionalProperties": false, + "properties": { + "disable": {"type": "boolean"}, + "interval": {"type": "string"}, + "retries": {"type": "number"}, + "test": { + "oneOf": [ + {"type": "string"}, + {"type": "array", "items": {"type": "string"}} + ] + }, + "timeout": {"type": "string"} + } + }, + + "network": { + "id": "#/definitions/network", + "type": "object", + "properties": { + "driver": {"type": "string"}, + "driver_opts": { + "type": "object", + "patternProperties": { + "^.+$": {"type": ["string", "number"]} + } + }, + "ipam": { + "type": "object", + "properties": { + "driver": {"type": "string"}, + "config": { + "type": "array" + }, + "options": { + "type": "object", + "patternProperties": { + "^.+$": {"type": "string"} + }, + "additionalProperties": false + } + }, + "additionalProperties": false + }, + "external": { + "type": ["boolean", "object"], + "properties": { + "name": {"type": "string"} + }, + "additionalProperties": false + }, + "internal": {"type": "boolean"}, + "enable_ipv6": {"type": "boolean"}, + "labels": {"$ref": "#/definitions/list_or_dict"} + }, + "additionalProperties": false + }, + + "volume": { + "id": "#/definitions/volume", + "type": ["object", "null"], + "properties": { + "driver": {"type": "string"}, + "driver_opts": { + "type": "object", + "patternProperties": { + "^.+$": {"type": ["string", "number"]} + } + }, + "external": { + "type": ["boolean", "object"], + "properties": { + "name": {"type": "string"} + }, + "additionalProperties": false + }, + "labels": {"$ref": "#/definitions/list_or_dict"} + }, + "additionalProperties": false + }, + + "string_or_list": { + "oneOf": [ + {"type": "string"}, + {"$ref": "#/definitions/list_of_strings"} + ] + }, + + "list_of_strings": { + "type": "array", + "items": {"type": "string"}, + "uniqueItems": true + }, + + "list_or_dict": { + "oneOf": [ + { + "type": "object", + "patternProperties": { + ".+": { + "type": ["string", "number", "null"] + } + }, + "additionalProperties": false + }, + {"type": "array", "items": {"type": "string"}, "uniqueItems": true} + ] + }, + + "constraints": { + "service": { + "id": "#/definitions/constraints/service", + "anyOf": [ + {"required": ["build"]}, + {"required": ["image"]} + ], + "properties": { + "build": { + "required": ["context"] + } + } + } + } + } +} diff --git a/compose/const.py b/compose/const.py index e46de8a73..6ea0ea79c 100644 --- a/compose/const.py +++ b/compose/const.py @@ -25,6 +25,7 @@ COMPOSEFILE_V1 = ComposeVersion('1') COMPOSEFILE_V2_0 = ComposeVersion('2.0') COMPOSEFILE_V2_1 = ComposeVersion('2.1') COMPOSEFILE_V2_2 = ComposeVersion('2.2') +COMPOSEFILE_V2_3 = ComposeVersion('2.3') COMPOSEFILE_V3_0 = ComposeVersion('3.0') COMPOSEFILE_V3_1 = ComposeVersion('3.1') @@ -36,6 +37,7 @@ API_VERSIONS = { COMPOSEFILE_V2_0: '1.22', COMPOSEFILE_V2_1: '1.24', COMPOSEFILE_V2_2: '1.25', + COMPOSEFILE_V2_3: '1.30', COMPOSEFILE_V3_0: '1.25', COMPOSEFILE_V3_1: '1.25', COMPOSEFILE_V3_2: '1.25', @@ -47,6 +49,7 @@ API_VERSION_TO_ENGINE_VERSION = { API_VERSIONS[COMPOSEFILE_V2_0]: '1.10.0', API_VERSIONS[COMPOSEFILE_V2_1]: '1.12.0', API_VERSIONS[COMPOSEFILE_V2_2]: '1.13.0', + API_VERSIONS[COMPOSEFILE_V2_3]: '17.06.0', API_VERSIONS[COMPOSEFILE_V3_0]: '1.13.0', API_VERSIONS[COMPOSEFILE_V3_1]: '1.13.0', API_VERSIONS[COMPOSEFILE_V3_2]: '1.13.0', diff --git a/docker-compose.spec b/docker-compose.spec index 8e0d51ae5..8dc70c226 100644 --- a/docker-compose.spec +++ b/docker-compose.spec @@ -37,6 +37,11 @@ exe = EXE(pyz, 'compose/config/config_schema_v2.2.json', 'DATA' ), + ( + 'compose/config/config_schema_v2.3.json', + 'compose/config/config_schema_v2.3.json', + 'DATA' + ), ( 'compose/config/config_schema_v3.0.json', 'compose/config/config_schema_v3.0.json', diff --git a/tests/integration/testcases.py b/tests/integration/testcases.py index 1b451ef3c..b1763b113 100644 --- a/tests/integration/testcases.py +++ b/tests/integration/testcases.py @@ -17,6 +17,7 @@ from compose.const import COMPOSEFILE_V1 as V1 from compose.const import COMPOSEFILE_V2_0 as V2_0 from compose.const import COMPOSEFILE_V2_0 as V2_1 from compose.const import COMPOSEFILE_V2_2 as V2_2 +from compose.const import COMPOSEFILE_V2_3 as V2_3 from compose.const import COMPOSEFILE_V3_0 as V3_0 from compose.const import COMPOSEFILE_V3_2 as V3_2 from compose.const import COMPOSEFILE_V3_3 as V3_3 @@ -78,6 +79,10 @@ def v2_2_only(): return min_version_skip(V2_2) +def v2_3_only(): + return min_version_skip(V2_3) + + def v3_only(): return min_version_skip(V3_0) diff --git a/tests/unit/config/config_test.py b/tests/unit/config/config_test.py index ac742a199..9d42f2b59 100644 --- a/tests/unit/config/config_test.py +++ b/tests/unit/config/config_test.py @@ -28,6 +28,7 @@ from compose.const import COMPOSEFILE_V1 as V1 from compose.const import COMPOSEFILE_V2_0 as V2_0 from compose.const import COMPOSEFILE_V2_1 as V2_1 from compose.const import COMPOSEFILE_V2_2 as V2_2 +from compose.const import COMPOSEFILE_V2_3 as V2_3 from compose.const import COMPOSEFILE_V3_0 as V3_0 from compose.const import COMPOSEFILE_V3_1 as V3_1 from compose.const import COMPOSEFILE_V3_2 as V3_2 @@ -179,6 +180,9 @@ class ConfigTest(unittest.TestCase): cfg = config.load(build_config_details({'version': '2.2'})) assert cfg.version == V2_2 + cfg = config.load(build_config_details({'version': '2.3'})) + assert cfg.version == V2_3 + for version in ['3', '3.0']: cfg = config.load(build_config_details({'version': version})) assert cfg.version == V3_0 From 16f8953c786c48fc9febe01bd8b03107598e81d9 Mon Sep 17 00:00:00 2001 From: Yong Wen Chua Date: Mon, 10 Jul 2017 13:02:47 +0800 Subject: [PATCH 088/244] Add `target` to service build configuration Signed-off-by: Yong Wen Chua --- compose/config/config.py | 1 + compose/config/config_schema_v2.3.json | 3 ++- compose/service.py | 5 ++++- tests/integration/service_test.py | 22 ++++++++++++++++++++++ tests/unit/service_test.py | 2 ++ 5 files changed, 31 insertions(+), 2 deletions(-) diff --git a/compose/config/config.py b/compose/config/config.py index f5053af8a..659b6cd59 100644 --- a/compose/config/config.py +++ b/compose/config/config.py @@ -986,6 +986,7 @@ def merge_build(output, base, override): md.merge_scalar('context') md.merge_scalar('dockerfile') md.merge_scalar('network') + md.merge_scalar('target') md.merge_mapping('args', parse_build_arguments) md.merge_field('cache_from', merge_unique_items_lists, default=[]) md.merge_mapping('labels', parse_labels) diff --git a/compose/config/config_schema_v2.3.json b/compose/config/config_schema_v2.3.json index abcc2ded2..877340276 100644 --- a/compose/config/config_schema_v2.3.json +++ b/compose/config/config_schema_v2.3.json @@ -61,7 +61,8 @@ "args": {"$ref": "#/definitions/list_or_dict"}, "labels": {"$ref": "#/definitions/list_or_dict"}, "cache_from": {"$ref": "#/definitions/list_of_strings"}, - "network": {"type": "string"} + "network": {"type": "string"}, + "target": {"type": "string"} }, "additionalProperties": false } diff --git a/compose/service.py b/compose/service.py index c8c2bd982..c43f635b2 100644 --- a/compose/service.py +++ b/compose/service.py @@ -905,7 +905,10 @@ class Service(object): nocache=no_cache, dockerfile=build_opts.get('dockerfile', None), cache_from=build_opts.get('cache_from', None), - buildargs=build_args + labels=build_opts.get('labels', None), + buildargs=build_args, + network_mode=build_opts.get('network', None), + target=build_opts.get('target', None), ) try: diff --git a/tests/integration/service_test.py b/tests/integration/service_test.py index ff75015df..4a5ec5654 100644 --- a/tests/integration/service_test.py +++ b/tests/integration/service_test.py @@ -40,6 +40,7 @@ from tests.integration.testcases import is_cluster from tests.integration.testcases import no_cluster from tests.integration.testcases import v2_1_only from tests.integration.testcases import v2_2_only +from tests.integration.testcases import v2_3_only from tests.integration.testcases import v2_only from tests.integration.testcases import v3_only @@ -754,6 +755,27 @@ class ServiceTest(DockerClientTestCase): assert service.image() + @v2_3_only() + def test_build_with_target(self): + self.require_api_version('1.30') + base_dir = tempfile.mkdtemp() + self.addCleanup(shutil.rmtree, base_dir) + + with open(os.path.join(base_dir, 'Dockerfile'), 'w') as f: + f.write('FROM busybox as one\n') + f.write('LABEL com.docker.compose.test.target=one\n') + f.write('FROM busybox as two\n') + f.write('LABEL com.docker.compose.test.target=two\n') + + service = self.create_service('buildlabels', build={ + 'context': text_type(base_dir), + 'target': 'one' + }) + + service.build() + assert service.image() + assert service.image()['Config']['Labels']['com.docker.compose.test.target'] == 'one' + def test_start_container_stays_unprivileged(self): service = self.create_service('web') container = create_and_start_container(service).inspect() diff --git a/tests/unit/service_test.py b/tests/unit/service_test.py index 2b0a2762d..0293695ab 100644 --- a/tests/unit/service_test.py +++ b/tests/unit/service_test.py @@ -474,6 +474,7 @@ class ServiceTest(unittest.TestCase): labels=None, cache_from=None, network_mode=None, + target=None, ) def test_ensure_image_exists_no_build(self): @@ -513,6 +514,7 @@ class ServiceTest(unittest.TestCase): labels=None, cache_from=None, network_mode=None, + target=None, ) def test_build_does_not_pull(self): From cf0afb071da46c7bec1741d126be050a7a3d35fa Mon Sep 17 00:00:00 2001 From: Harald Albers Date: Fri, 14 Jul 2017 11:34:00 +0200 Subject: [PATCH 089/244] Add bash completion for `pull --quiet` Signed-off-by: Harald Albers --- contrib/completion/bash/docker-compose | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contrib/completion/bash/docker-compose b/contrib/completion/bash/docker-compose index 57dfd51f5..d283a041a 100644 --- a/contrib/completion/bash/docker-compose +++ b/contrib/completion/bash/docker-compose @@ -341,7 +341,7 @@ _docker_compose_ps() { _docker_compose_pull() { case "$cur" in -*) - COMPREPLY=( $( compgen -W "--help --ignore-pull-failures --parallel" -- "$cur" ) ) + COMPREPLY=( $( compgen -W "--help --ignore-pull-failures --parallel --quiet" -- "$cur" ) ) ;; *) __docker_compose_services_from_image From 73fd0abd5b33ee93d4e86764decdbe7d1c2584d6 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Wed, 26 Jul 2017 16:44:54 -0700 Subject: [PATCH 090/244] Fix test issues with Engine 17.07 RC1 Signed-off-by: Joffrey F --- tests/acceptance/cli_test.py | 3 ++- tests/integration/service_test.py | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/tests/acceptance/cli_test.py b/tests/acceptance/cli_test.py index fc05de351..f7ecba9f5 100644 --- a/tests/acceptance/cli_test.py +++ b/tests/acceptance/cli_test.py @@ -439,7 +439,8 @@ class CLITestCase(DockerClientTestCase): assert 'Pulling simple (busybox:latest)...' in result.stderr assert 'Pulling another (nonexisting-image:latest)...' in result.stderr assert ('repository nonexisting-image not found' in result.stderr or - 'image library/nonexisting-image:latest not found' in result.stderr) + 'image library/nonexisting-image:latest not found' in result.stderr or + 'pull access denied for nonexisting-image' in result.stderr) def test_pull_with_quiet(self): assert self.dispatch(['pull', '--quiet']).stderr == '' diff --git a/tests/integration/service_test.py b/tests/integration/service_test.py index 4a5ec5654..3a585ec01 100644 --- a/tests/integration/service_test.py +++ b/tests/integration/service_test.py @@ -210,7 +210,8 @@ class ServiceTest(DockerClientTestCase): service.start_container(container) self.assertEqual(set(container.get('HostConfig.SecurityOpt')), set(security_opt)) - @pytest.mark.xfail(True, reason='Not supported on most drivers') + # @pytest.mark.xfail(True, reason='Not supported on most drivers') + @pytest.mark.skipif(True, reason='https://github.com/moby/moby/issues/34270') def test_create_container_with_storage_opt(self): storage_opt = {'size': '1G'} service = self.create_service('db', storage_opt=storage_opt) From 770d94376a6d2a35e0f81f8652ac4b435844ec23 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Tue, 25 Jul 2017 16:42:31 -0700 Subject: [PATCH 091/244] Escape dollar sign in serialized config output Signed-off-by: Joffrey F --- compose/config/serialize.py | 5 ++++- tests/unit/config/config_test.py | 22 ++++++++++++++++++++++ 2 files changed, 26 insertions(+), 1 deletion(-) diff --git a/compose/config/serialize.py b/compose/config/serialize.py index 3fdd4d392..86fdac38f 100644 --- a/compose/config/serialize.py +++ b/compose/config/serialize.py @@ -21,8 +21,11 @@ def serialize_dict_type(dumper, data): def serialize_string(dumper, data): - """ Ensure boolean-like strings are quoted in the output """ + """ Ensure boolean-like strings are quoted in the output and escape $ characters """ representer = dumper.represent_str if six.PY3 else dumper.represent_unicode + + data = data.replace('$', '$$') + if data.lower() in ('y', 'n', 'yes', 'no', 'on', 'off', 'true', 'false'): # Empirically only y/n appears to be an issue, but this might change # depending on which PyYaml version is being used. Err on safe side. diff --git a/tests/unit/config/config_test.py b/tests/unit/config/config_test.py index 9d42f2b59..63cb7eaef 100644 --- a/tests/unit/config/config_test.py +++ b/tests/unit/config/config_test.py @@ -4185,3 +4185,25 @@ class SerializeTest(unittest.TestCase): assert 'command: "true"\n' in serialized_config assert 'FOO: "Y"\n' in serialized_config assert 'BAR: "on"\n' in serialized_config + + def test_serialize_escape_dollar_sign(self): + cfg = { + 'version': '2.2', + 'services': { + 'web': { + 'image': 'busybox', + 'command': 'echo $$FOO', + 'environment': { + 'CURRENCY': '$$' + }, + 'entrypoint': ['$$SHELL', '-c'], + } + } + } + config_dict = config.load(build_config_details(cfg)) + + serialized_config = yaml.load(serialize_config(config_dict)) + serialized_service = serialized_config['services']['web'] + assert serialized_service['environment']['CURRENCY'] == '$$' + assert serialized_service['command'] == 'echo $$FOO' + assert serialized_service['entrypoint'][0] == '$$SHELL' From 1ae83d41398d55dcf2387e27199338e86010dd39 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Tue, 25 Jul 2017 16:27:10 -0700 Subject: [PATCH 092/244] 0 is a valid value for a published port Signed-off-by: Joffrey F --- compose/config/types.py | 2 +- tests/unit/config/types_test.py | 6 ++++++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/compose/config/types.py b/compose/config/types.py index be26971c4..c410343b8 100644 --- a/compose/config/types.py +++ b/compose/config/types.py @@ -343,7 +343,7 @@ class ServicePort(namedtuple('_ServicePort', 'target published protocol mode ext def normalize_port_dict(port): return '{external_ip}{has_ext_ip}{published}{is_pub}{target}/{protocol}'.format( published=port.get('published', ''), - is_pub=(':' if port.get('published') or port.get('external_ip') else ''), + is_pub=(':' if port.get('published') is not None or port.get('external_ip') else ''), target=port.get('target'), protocol=port.get('protocol', 'tcp'), external_ip=port.get('external_ip', ''), diff --git a/tests/unit/config/types_test.py b/tests/unit/config/types_test.py index 10b698fe3..3a43f727b 100644 --- a/tests/unit/config/types_test.py +++ b/tests/unit/config/types_test.py @@ -81,6 +81,12 @@ class TestServicePort(object): 'external_ip': '1.1.1.1', } + def test_repr_published_port_0(self): + port_def = '0:4000' + ports = ServicePort.parse(port_def) + assert len(ports) == 1 + assert ports[0].legacy_repr() == port_def + '/tcp' + def test_parse_port_range(self): ports = ServicePort.parse('25000-25001:4000-4001') assert len(ports) == 2 From e0f7b075b8c02b40da3e58757d6bf6bc7e9588ed Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Wed, 26 Jul 2017 18:21:30 -0700 Subject: [PATCH 093/244] 1.16.0-dev Signed-off-by: Joffrey F --- compose/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/compose/__init__.py b/compose/__init__.py index f238607c0..cedb7cf04 100644 --- a/compose/__init__.py +++ b/compose/__init__.py @@ -1,4 +1,4 @@ from __future__ import absolute_import from __future__ import unicode_literals -__version__ = '1.15.0' +__version__ = '1.16.0-dev' From bbebf518cfc9b975f04883ef4b3a9a89c071eab2 Mon Sep 17 00:00:00 2001 From: Carl George Date: Wed, 26 Jul 2017 16:50:38 -0500 Subject: [PATCH 094/244] only require colorama on windows Colorama is only useful on Windows by design. Since it has no effect on other platforms, it makes sense to not require it universally. Signed-off-by: Carl George --- compose/cli/colors.py | 6 ++++-- requirements.txt | 2 +- setup.py | 2 +- 3 files changed, 6 insertions(+), 4 deletions(-) diff --git a/compose/cli/colors.py b/compose/cli/colors.py index f1251e431..cb30e3615 100644 --- a/compose/cli/colors.py +++ b/compose/cli/colors.py @@ -1,7 +1,7 @@ from __future__ import absolute_import from __future__ import unicode_literals -import colorama +from ..const import IS_WINDOWS_PLATFORM NAMES = [ 'grey', @@ -33,7 +33,9 @@ def make_color_fn(code): return lambda s: ansi_color(code, s) -colorama.init(strip=False) +if IS_WINDOWS_PLATFORM: + import colorama + colorama.init(strip=False) for (name, code) in get_pairs(): globals()[name] = make_color_fn(code) diff --git a/requirements.txt b/requirements.txt index 844921ffd..81dcdf08c 100644 --- a/requirements.txt +++ b/requirements.txt @@ -4,7 +4,7 @@ backports.ssl-match-hostname==3.5.0.1; python_version < '3' cached-property==1.3.0 certifi==2017.4.17 chardet==3.0.4 -colorama==0.3.9 +colorama==0.3.9; sys_platform == 'win32' docker==2.4.2 docker-pycreds==0.2.1 dockerpty==0.4.1 diff --git a/setup.py b/setup.py index dab7a6eea..a3072c76e 100644 --- a/setup.py +++ b/setup.py @@ -31,7 +31,6 @@ def find_version(*file_paths): install_requires = [ 'cached-property >= 1.2.0, < 2', - 'colorama >= 0.3.7, < 0.4', 'docopt >= 0.6.1, < 0.7', 'PyYAML >= 3.10, < 4', 'requests >= 2.6.1, != 2.11.0, < 2.12', @@ -56,6 +55,7 @@ extras_require = { ':python_version < "3.4"': ['enum34 >= 1.0.4, < 2'], ':python_version < "3.5"': ['backports.ssl_match_hostname >= 3.5'], ':python_version < "3.3"': ['ipaddress >= 1.0.16'], + ':sys_platform == "win32"': ['colorama >= 0.3.7, < 0.4'], 'socks': ['PySocks >= 1.5.6, != 1.5.7, < 2'], } From 4652d3c38a9e3899e77f40f035523650ad625d68 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Wed, 2 Aug 2017 19:55:58 -0700 Subject: [PATCH 095/244] Use newer versions of pre-commit hooks Signed-off-by: Joffrey F --- .pre-commit-config.yaml | 4 ++-- bin/docker-compose | 3 +++ requirements.txt | 4 ++-- script/build/test-image | 2 +- script/setup/osx | 1 - tests/fixtures/default-env-file/.env | 2 +- tests/fixtures/env-file/test.env | 2 +- 7 files changed, 10 insertions(+), 8 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 0e7b9d5f3..b7bcc8466 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,5 +1,5 @@ - repo: git://github.com/pre-commit/pre-commit-hooks - sha: 'v0.4.2' + sha: 'v0.9.1' hooks: - id: check-added-large-files - id: check-docstring-first @@ -14,7 +14,7 @@ - id: requirements-txt-fixer - id: trailing-whitespace - repo: git://github.com/asottile/reorder_python_imports - sha: v0.1.0 + sha: v0.3.5 hooks: - id: reorder-python-imports language_version: 'python2.7' diff --git a/bin/docker-compose b/bin/docker-compose index 5976e1d4a..aeb538703 100755 --- a/bin/docker-compose +++ b/bin/docker-compose @@ -1,3 +1,6 @@ #!/usr/bin/env python +from __future__ import absolute_import +from __future__ import unicode_literals + from compose.cli.main import main main() diff --git a/requirements.txt b/requirements.txt index 81dcdf08c..826c31eb1 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,3 @@ -PySocks==1.6.7 -PyYAML==3.12 backports.ssl-match-hostname==3.5.0.1; python_version < '3' cached-property==1.3.0 certifi==2017.4.17 @@ -15,6 +13,8 @@ idna==2.5 ipaddress==1.0.18 jsonschema==2.6.0 pypiwin32==219; sys_platform == 'win32' +PySocks==1.6.7 +PyYAML==3.12 requests==2.11.1 six==1.10.0 texttable==0.8.8 diff --git a/script/build/test-image b/script/build/test-image index 216d63f9c..a2eb62cdf 100755 --- a/script/build/test-image +++ b/script/build/test-image @@ -14,4 +14,4 @@ ctnr_id=$(docker create --entrypoint=tox docker-compose-tests:tmp) docker commit $ctnr_id docker/compose-tests:latest docker tag docker/compose-tests:latest docker/compose-tests:$TAG docker rm -f $ctnr_id -docker rmi -f docker-compose-tests:tmp \ No newline at end of file +docker rmi -f docker-compose-tests:tmp diff --git a/script/setup/osx b/script/setup/osx index e6ab62a84..e0c2bd0a2 100755 --- a/script/setup/osx +++ b/script/setup/osx @@ -50,4 +50,3 @@ echo "*** Using $(openssl_version)" if !(which virtualenv); then pip install virtualenv fi - diff --git a/tests/fixtures/default-env-file/.env b/tests/fixtures/default-env-file/.env index 996c886cb..9056de724 100644 --- a/tests/fixtures/default-env-file/.env +++ b/tests/fixtures/default-env-file/.env @@ -1,4 +1,4 @@ IMAGE=alpine:latest COMMAND=true PORT1=5643 -PORT2=9999 \ No newline at end of file +PORT2=9999 diff --git a/tests/fixtures/env-file/test.env b/tests/fixtures/env-file/test.env index c9604dad5..d99cd41a4 100644 --- a/tests/fixtures/env-file/test.env +++ b/tests/fixtures/env-file/test.env @@ -1 +1 @@ -FOO=1 \ No newline at end of file +FOO=1 From 467e0d0d31fadb1726e376cc67530e7f750c8633 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Wed, 2 Aug 2017 19:23:39 -0700 Subject: [PATCH 096/244] Fix ServiceExtendsResolver same-file detection Signed-off-by: Joffrey F --- compose/config/config.py | 2 +- tests/unit/config/config_test.py | 36 ++++++++++++++++++++++++++++++++ 2 files changed, 37 insertions(+), 1 deletion(-) diff --git a/compose/config/config.py b/compose/config/config.py index 659b6cd59..cb25a25a6 100644 --- a/compose/config/config.py +++ b/compose/config/config.py @@ -571,7 +571,7 @@ class ServiceExtendsResolver(object): config_path = self.get_extended_config_path(extends) service_name = extends['service'] - if config_path == self.service_config.filename: + if config_path == self.config_file.filename: try: service_config = self.config_file.get_service(service_name) except KeyError: diff --git a/tests/unit/config/config_test.py b/tests/unit/config/config_test.py index 63cb7eaef..fd06db7d1 100644 --- a/tests/unit/config/config_test.py +++ b/tests/unit/config/config_test.py @@ -702,6 +702,42 @@ class ConfigTest(unittest.TestCase): ] self.assertEqual(service_sort(service_dicts), service_sort(expected)) + def test_load_mixed_extends_resolution(self): + main_file = config.ConfigFile( + 'main.yml', { + 'version': '2.2', + 'services': { + 'prodweb': { + 'extends': { + 'service': 'web', + 'file': 'base.yml' + }, + 'environment': {'PROD': 'true'}, + }, + }, + } + ) + + tmpdir = pytest.ensuretemp('config_test') + self.addCleanup(tmpdir.remove) + tmpdir.join('base.yml').write(""" + version: '2.2' + services: + base: + image: base + web: + extends: base + """) + + details = config.ConfigDetails('.', [main_file]) + with tmpdir.as_cwd(): + service_dicts = config.load(details).services + assert service_dicts[0] == { + 'name': 'prodweb', + 'image': 'base', + 'environment': {'PROD': 'true'}, + } + def test_load_with_multiple_files_and_invalid_override(self): base_file = config.ConfigFile( 'base.yaml', From 7feb2685d21b6149e4109a55c2762ebf1d69e8d9 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Wed, 2 Aug 2017 17:02:30 -0700 Subject: [PATCH 097/244] Bump texttable dependency Signed-off-by: Joffrey F --- requirements.txt | 2 +- setup.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/requirements.txt b/requirements.txt index 826c31eb1..d1778990f 100644 --- a/requirements.txt +++ b/requirements.txt @@ -17,6 +17,6 @@ PySocks==1.6.7 PyYAML==3.12 requests==2.11.1 six==1.10.0 -texttable==0.8.8 +texttable==0.9.1 urllib3==1.21.1 websocket-client==0.32.0 diff --git a/setup.py b/setup.py index a3072c76e..16493f52b 100644 --- a/setup.py +++ b/setup.py @@ -34,7 +34,7 @@ install_requires = [ 'docopt >= 0.6.1, < 0.7', 'PyYAML >= 3.10, < 4', 'requests >= 2.6.1, != 2.11.0, < 2.12', - 'texttable >= 0.8.1, < 0.9', + 'texttable >= 0.9.0, < 0.10', 'websocket-client >= 0.32.0, < 1.0', 'docker >= 2.4.2, < 3.0', 'dockerpty >= 0.4.1, < 0.5', From 444d88872059ab87b41e5416682b7793e8845c8e Mon Sep 17 00:00:00 2001 From: Cecile Tonglet Date: Fri, 23 Jun 2017 15:17:38 +0200 Subject: [PATCH 098/244] Add a flag --no-ansi to remove control characters on parallel executions Signed-off-by: Cecile Tonglet --- compose/cli/command.py | 5 +++-- compose/cli/main.py | 1 + compose/parallel.py | 45 ++++++++++++++++++++++++------------- compose/project.py | 32 ++++++++++++++++---------- compose/service.py | 6 +++++ tests/unit/parallel_test.py | 16 +++++++++++++ 6 files changed, 75 insertions(+), 30 deletions(-) diff --git a/compose/cli/command.py b/compose/cli/command.py index e1ae690c0..f5330d1c2 100644 --- a/compose/cli/command.py +++ b/compose/cli/command.py @@ -31,6 +31,7 @@ def project_from_options(project_dir, options): get_config_path_from_options(project_dir, options, environment), project_name=options.get('--project-name'), verbose=options.get('--verbose'), + noansi=options.get('--no-ansi'), host=host, tls_config=tls_config_from_options(options), environment=environment, @@ -81,7 +82,7 @@ def get_client(environment, verbose=False, version=None, tls_config=None, host=N def get_project(project_dir, config_path=None, project_name=None, verbose=False, - host=None, tls_config=None, environment=None, override_dir=None): + noansi=False, host=None, tls_config=None, environment=None, override_dir=None): if not environment: environment = Environment.from_env_file(project_dir) config_details = config.find(project_dir, config_path, environment, override_dir) @@ -100,7 +101,7 @@ def get_project(project_dir, config_path=None, project_name=None, verbose=False, ) with errors.handle_connection_errors(client): - return Project.from_config(project_name, config_data, client) + return Project.from_config(project_name, config_data, client, noansi=noansi) def get_project_name(working_dir, project_name=None, environment=None): diff --git a/compose/cli/main.py b/compose/cli/main.py index 20f3b55b4..c0cf8747f 100644 --- a/compose/cli/main.py +++ b/compose/cli/main.py @@ -159,6 +159,7 @@ class TopLevelCommand(object): -f, --file FILE Specify an alternate compose file (default: docker-compose.yml) -p, --project-name NAME Specify an alternate project name (default: directory name) --verbose Show more output + --no-ansi Do not print ANSI control characters -v, --version Print version and exit -H, --host HOST Daemon socket to connect to diff --git a/compose/parallel.py b/compose/parallel.py index a611fd6e0..89d074e35 100644 --- a/compose/parallel.py +++ b/compose/parallel.py @@ -26,7 +26,7 @@ log = logging.getLogger(__name__) STOP = object() -def parallel_execute(objects, func, get_name, msg, get_deps=None, limit=None): +def parallel_execute(objects, func, get_name, msg, get_deps=None, limit=None, noansi=False): """Runs func on objects in parallel while ensuring that func is ran on object only after it is ran on all its dependencies. @@ -36,7 +36,7 @@ def parallel_execute(objects, func, get_name, msg, get_deps=None, limit=None): objects = list(objects) stream = get_output_stream(sys.stderr) - writer = ParallelStreamWriter(stream, msg) + writer = ParallelStreamWriter(stream, msg, noansi) for obj in objects: writer.add_object(get_name(obj)) writer.write_initial() @@ -221,11 +221,12 @@ class ParallelStreamWriter(object): to jump to the correct line, and write over the line. """ - def __init__(self, stream, msg): + def __init__(self, stream, msg, noansi): self.stream = stream self.msg = msg self.lines = [] self.width = 0 + self.noansi = noansi def add_object(self, obj_index): self.lines.append(obj_index) @@ -239,9 +240,7 @@ class ParallelStreamWriter(object): width=self.width)) self.stream.flush() - def write(self, obj_index, status): - if self.msg is None: - return + def _write_ansi(self, obj_index, status): position = self.lines.index(obj_index) diff = len(self.lines) - position # move up @@ -254,27 +253,41 @@ class ParallelStreamWriter(object): self.stream.write("%c[%dB" % (27, diff)) self.stream.flush() + def _write_noansi(self, obj_index, status): + self.stream.write("{} {:<{width}} ... {}\r\n".format(self.msg, obj_index, + status, width=self.width)) + self.stream.flush() -def parallel_operation(containers, operation, options, message): + def write(self, obj_index, status): + if self.msg is None: + return + if self.noansi: + self._write_noansi(obj_index, status) + else: + self._write_ansi(obj_index, status) + + +def parallel_operation(containers, operation, options, message, noansi=False): parallel_execute( containers, operator.methodcaller(operation, **options), operator.attrgetter('name'), - message) + message, + noansi=noansi) -def parallel_remove(containers, options): +def parallel_remove(containers, options, noansi=False): stopped_containers = [c for c in containers if not c.is_running] - parallel_operation(stopped_containers, 'remove', options, 'Removing') + parallel_operation(stopped_containers, 'remove', options, 'Removing', noansi=noansi) -def parallel_pause(containers, options): - parallel_operation(containers, 'pause', options, 'Pausing') +def parallel_pause(containers, options, noansi=False): + parallel_operation(containers, 'pause', options, 'Pausing', noansi=noansi) -def parallel_unpause(containers, options): - parallel_operation(containers, 'unpause', options, 'Unpausing') +def parallel_unpause(containers, options, noansi=False): + parallel_operation(containers, 'unpause', options, 'Unpausing', noansi=noansi) -def parallel_kill(containers, options): - parallel_operation(containers, 'kill', options, 'Killing') +def parallel_kill(containers, options, noansi=False): + parallel_operation(containers, 'kill', options, 'Killing', noansi=noansi) diff --git a/compose/project.py b/compose/project.py index 28af45c71..9ea6ff6bb 100644 --- a/compose/project.py +++ b/compose/project.py @@ -60,13 +60,15 @@ class Project(object): """ A collection of services. """ - def __init__(self, name, services, client, networks=None, volumes=None, config_version=None): + def __init__(self, name, services, client, networks=None, volumes=None, config_version=None, + noansi=False): self.name = name self.services = services self.client = client self.volumes = volumes or ProjectVolumes({}) self.networks = networks or ProjectNetworks({}, False) self.config_version = config_version + self.noansi = noansi def labels(self, one_off=OneOffFilter.exclude): labels = ['{0}={1}'.format(LABEL_PROJECT, self.name)] @@ -75,7 +77,7 @@ class Project(object): return labels @classmethod - def from_config(cls, name, config_data, client): + def from_config(cls, name, config_data, client, noansi=False): """ Construct a Project from a config.Config object. """ @@ -86,7 +88,7 @@ class Project(object): networks, use_networking) volumes = ProjectVolumes.from_config(name, config_data, client) - project = cls(name, [], client, project_networks, volumes, config_data.version) + project = cls(name, [], client, project_networks, volumes, config_data.version, noansi=noansi) for service_dict in config_data.services: service_dict = dict(service_dict) @@ -126,6 +128,7 @@ class Project(object): volumes_from=volumes_from, secrets=secrets, pid_mode=pid_mode, + noansi=noansi, **service_dict) ) @@ -270,7 +273,8 @@ class Project(object): start_service, operator.attrgetter('name'), 'Starting', - get_deps) + get_deps, + noansi=self.noansi) return containers @@ -288,25 +292,26 @@ class Project(object): self.build_container_operation_with_timeout_func('stop', options), operator.attrgetter('name'), 'Stopping', - get_deps) + get_deps, + noansi=self.noansi) def pause(self, service_names=None, **options): containers = self.containers(service_names) - parallel.parallel_pause(reversed(containers), options) + parallel.parallel_pause(reversed(containers), options, noansi=self.noansi) return containers def unpause(self, service_names=None, **options): containers = self.containers(service_names) - parallel.parallel_unpause(containers, options) + parallel.parallel_unpause(containers, options, noansi=self.noansi) return containers def kill(self, service_names=None, **options): - parallel.parallel_kill(self.containers(service_names), options) + parallel.parallel_kill(self.containers(service_names), options, noansi=self.noansi) def remove_stopped(self, service_names=None, one_off=OneOffFilter.exclude, **options): parallel.parallel_remove(self.containers( service_names, stopped=True, one_off=one_off - ), options) + ), options, noansi=self.noansi) def down(self, remove_image_type, include_volumes, remove_orphans=False): self.stop(one_off=OneOffFilter.include) @@ -331,7 +336,8 @@ class Project(object): containers, self.build_container_operation_with_timeout_func('restart', options), operator.attrgetter('name'), - 'Restarting') + 'Restarting', + noansi=self.noansi) return containers def build(self, service_names=None, no_cache=False, pull=False, force_rm=False, build_args=None): @@ -447,7 +453,8 @@ class Project(object): do, operator.attrgetter('name'), None, - get_deps + get_deps, + noansi=self.noansi, ) if errors: raise ProjectError( @@ -500,7 +507,8 @@ class Project(object): pull_service, operator.attrgetter('name'), 'Pulling', - limit=5) + limit=5, + noansi=self.noansi) else: for service in services: service.pull(ignore_pull_failures, silent=silent) diff --git a/compose/service.py b/compose/service.py index c43f635b2..22aae08b7 100644 --- a/compose/service.py +++ b/compose/service.py @@ -158,6 +158,7 @@ class Service(object): secrets=None, scale=None, pid_mode=None, + noansi=False, **options ): self.name = name @@ -171,6 +172,7 @@ class Service(object): self.networks = networks or {} self.secrets = secrets or [] self.scale_num = scale or 1 + self.noansi = noansi self.options = options def __repr__(self): @@ -393,6 +395,7 @@ class Service(object): lambda n: create_and_start(self, n), lambda n: self.get_container_name(n), "Creating", + noansi=self.noansi, ) for error in errors.values(): raise OperationFailedError(error) @@ -414,6 +417,7 @@ class Service(object): recreate, lambda c: c.name, "Recreating", + noansi=self.noansi, ) for error in errors.values(): raise OperationFailedError(error) @@ -434,6 +438,7 @@ class Service(object): lambda c: self.start_container_if_stopped(c, attach_logs=not detached), lambda c: c.name, "Starting", + noansi=self.noansi, ) for error in errors.values(): @@ -455,6 +460,7 @@ class Service(object): stop_and_remove, lambda c: c.name, "Stopping and removing", + noansi=self.noansi, ) def execute_convergence_plan(self, plan, timeout=None, detached=False, diff --git a/tests/unit/parallel_test.py b/tests/unit/parallel_test.py index 73728fdfd..519c66669 100644 --- a/tests/unit/parallel_test.py +++ b/tests/unit/parallel_test.py @@ -130,3 +130,19 @@ def test_parallel_execute_alignment(capsys): _, err = capsys.readouterr() a, b = err.split('\n')[:2] assert a.index('...') == b.index('...') + + +def test_parallel_execute_alignment_noansi(capsys): + results, errors = parallel_execute( + objects=["short", "a very long name"], + func=lambda x: x, + get_name=six.text_type, + msg="Aligning", + noansi=True, + ) + + assert errors == {} + + _, err = capsys.readouterr() + a, b, c, d = err.split('\n')[:4] + assert a.index('...') == b.index('...') == c.index('...') == d.index('...') From 7882f1fb06da723d957f9038c8a4f64e0cc451a0 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Wed, 9 Aug 2017 16:46:47 -0700 Subject: [PATCH 099/244] Keep no-ansi parameter in the CLI scope Signed-off-by: Joffrey F --- compose/cli/command.py | 5 ++--- compose/cli/main.py | 7 +++++++ compose/parallel.py | 33 +++++++++++++++++++-------------- compose/project.py | 26 +++++++++++--------------- compose/service.py | 6 ------ tests/acceptance/cli_test.py | 8 ++++++++ tests/unit/parallel_test.py | 5 +++-- 7 files changed, 50 insertions(+), 40 deletions(-) diff --git a/compose/cli/command.py b/compose/cli/command.py index f5330d1c2..e1ae690c0 100644 --- a/compose/cli/command.py +++ b/compose/cli/command.py @@ -31,7 +31,6 @@ def project_from_options(project_dir, options): get_config_path_from_options(project_dir, options, environment), project_name=options.get('--project-name'), verbose=options.get('--verbose'), - noansi=options.get('--no-ansi'), host=host, tls_config=tls_config_from_options(options), environment=environment, @@ -82,7 +81,7 @@ def get_client(environment, verbose=False, version=None, tls_config=None, host=N def get_project(project_dir, config_path=None, project_name=None, verbose=False, - noansi=False, host=None, tls_config=None, environment=None, override_dir=None): + host=None, tls_config=None, environment=None, override_dir=None): if not environment: environment = Environment.from_env_file(project_dir) config_details = config.find(project_dir, config_path, environment, override_dir) @@ -101,7 +100,7 @@ def get_project(project_dir, config_path=None, project_name=None, verbose=False, ) with errors.handle_connection_errors(client): - return Project.from_config(project_name, config_data, client, noansi=noansi) + return Project.from_config(project_name, config_data, client) def get_project_name(working_dir, project_name=None, environment=None): diff --git a/compose/cli/main.py b/compose/cli/main.py index c0cf8747f..2bb53f95e 100644 --- a/compose/cli/main.py +++ b/compose/cli/main.py @@ -98,6 +98,7 @@ def dispatch(): options, handler, command_options = dispatcher.parse(sys.argv[1:]) setup_console_handler(console_handler, options.get('--verbose')) + setup_parallel_logger(options.get('--no-ansi')) return functools.partial(perform_command, options, handler, command_options) @@ -127,6 +128,12 @@ def setup_logging(): logging.getLogger("requests").propagate = False +def setup_parallel_logger(noansi): + if noansi: + import compose.parallel + compose.parallel.ParallelStreamWriter.set_noansi() + + def setup_console_handler(handler, verbose): if handler.stream.isatty(): format_class = ConsoleWarningFormatter diff --git a/compose/parallel.py b/compose/parallel.py index 89d074e35..1cf1fb094 100644 --- a/compose/parallel.py +++ b/compose/parallel.py @@ -26,7 +26,7 @@ log = logging.getLogger(__name__) STOP = object() -def parallel_execute(objects, func, get_name, msg, get_deps=None, limit=None, noansi=False): +def parallel_execute(objects, func, get_name, msg, get_deps=None, limit=None): """Runs func on objects in parallel while ensuring that func is ran on object only after it is ran on all its dependencies. @@ -36,7 +36,7 @@ def parallel_execute(objects, func, get_name, msg, get_deps=None, limit=None, no objects = list(objects) stream = get_output_stream(sys.stderr) - writer = ParallelStreamWriter(stream, msg, noansi) + writer = ParallelStreamWriter(stream, msg) for obj in objects: writer.add_object(get_name(obj)) writer.write_initial() @@ -221,12 +221,17 @@ class ParallelStreamWriter(object): to jump to the correct line, and write over the line. """ - def __init__(self, stream, msg, noansi): + noansi = False + + @classmethod + def set_noansi(cls, value=True): + cls.noansi = value + + def __init__(self, stream, msg): self.stream = stream self.msg = msg self.lines = [] self.width = 0 - self.noansi = noansi def add_object(self, obj_index): self.lines.append(obj_index) @@ -267,27 +272,27 @@ class ParallelStreamWriter(object): self._write_ansi(obj_index, status) -def parallel_operation(containers, operation, options, message, noansi=False): +def parallel_operation(containers, operation, options, message): parallel_execute( containers, operator.methodcaller(operation, **options), operator.attrgetter('name'), message, - noansi=noansi) + ) -def parallel_remove(containers, options, noansi=False): +def parallel_remove(containers, options): stopped_containers = [c for c in containers if not c.is_running] - parallel_operation(stopped_containers, 'remove', options, 'Removing', noansi=noansi) + parallel_operation(stopped_containers, 'remove', options, 'Removing') -def parallel_pause(containers, options, noansi=False): - parallel_operation(containers, 'pause', options, 'Pausing', noansi=noansi) +def parallel_pause(containers, options): + parallel_operation(containers, 'pause', options, 'Pausing') -def parallel_unpause(containers, options, noansi=False): - parallel_operation(containers, 'unpause', options, 'Unpausing', noansi=noansi) +def parallel_unpause(containers, options): + parallel_operation(containers, 'unpause', options, 'Unpausing') -def parallel_kill(containers, options, noansi=False): - parallel_operation(containers, 'kill', options, 'Killing', noansi=noansi) +def parallel_kill(containers, options): + parallel_operation(containers, 'kill', options, 'Killing') diff --git a/compose/project.py b/compose/project.py index 9ea6ff6bb..86fbda6ee 100644 --- a/compose/project.py +++ b/compose/project.py @@ -60,15 +60,13 @@ class Project(object): """ A collection of services. """ - def __init__(self, name, services, client, networks=None, volumes=None, config_version=None, - noansi=False): + def __init__(self, name, services, client, networks=None, volumes=None, config_version=None): self.name = name self.services = services self.client = client self.volumes = volumes or ProjectVolumes({}) self.networks = networks or ProjectNetworks({}, False) self.config_version = config_version - self.noansi = noansi def labels(self, one_off=OneOffFilter.exclude): labels = ['{0}={1}'.format(LABEL_PROJECT, self.name)] @@ -77,7 +75,7 @@ class Project(object): return labels @classmethod - def from_config(cls, name, config_data, client, noansi=False): + def from_config(cls, name, config_data, client): """ Construct a Project from a config.Config object. """ @@ -88,7 +86,7 @@ class Project(object): networks, use_networking) volumes = ProjectVolumes.from_config(name, config_data, client) - project = cls(name, [], client, project_networks, volumes, config_data.version, noansi=noansi) + project = cls(name, [], client, project_networks, volumes, config_data.version) for service_dict in config_data.services: service_dict = dict(service_dict) @@ -128,7 +126,6 @@ class Project(object): volumes_from=volumes_from, secrets=secrets, pid_mode=pid_mode, - noansi=noansi, **service_dict) ) @@ -274,7 +271,7 @@ class Project(object): operator.attrgetter('name'), 'Starting', get_deps, - noansi=self.noansi) + ) return containers @@ -293,25 +290,25 @@ class Project(object): operator.attrgetter('name'), 'Stopping', get_deps, - noansi=self.noansi) + ) def pause(self, service_names=None, **options): containers = self.containers(service_names) - parallel.parallel_pause(reversed(containers), options, noansi=self.noansi) + parallel.parallel_pause(reversed(containers), options) return containers def unpause(self, service_names=None, **options): containers = self.containers(service_names) - parallel.parallel_unpause(containers, options, noansi=self.noansi) + parallel.parallel_unpause(containers, options) return containers def kill(self, service_names=None, **options): - parallel.parallel_kill(self.containers(service_names), options, noansi=self.noansi) + parallel.parallel_kill(self.containers(service_names), options) def remove_stopped(self, service_names=None, one_off=OneOffFilter.exclude, **options): parallel.parallel_remove(self.containers( service_names, stopped=True, one_off=one_off - ), options, noansi=self.noansi) + ), options) def down(self, remove_image_type, include_volumes, remove_orphans=False): self.stop(one_off=OneOffFilter.include) @@ -337,7 +334,7 @@ class Project(object): self.build_container_operation_with_timeout_func('restart', options), operator.attrgetter('name'), 'Restarting', - noansi=self.noansi) + ) return containers def build(self, service_names=None, no_cache=False, pull=False, force_rm=False, build_args=None): @@ -454,7 +451,6 @@ class Project(object): operator.attrgetter('name'), None, get_deps, - noansi=self.noansi, ) if errors: raise ProjectError( @@ -508,7 +504,7 @@ class Project(object): operator.attrgetter('name'), 'Pulling', limit=5, - noansi=self.noansi) + ) else: for service in services: service.pull(ignore_pull_failures, silent=silent) diff --git a/compose/service.py b/compose/service.py index 22aae08b7..c43f635b2 100644 --- a/compose/service.py +++ b/compose/service.py @@ -158,7 +158,6 @@ class Service(object): secrets=None, scale=None, pid_mode=None, - noansi=False, **options ): self.name = name @@ -172,7 +171,6 @@ class Service(object): self.networks = networks or {} self.secrets = secrets or [] self.scale_num = scale or 1 - self.noansi = noansi self.options = options def __repr__(self): @@ -395,7 +393,6 @@ class Service(object): lambda n: create_and_start(self, n), lambda n: self.get_container_name(n), "Creating", - noansi=self.noansi, ) for error in errors.values(): raise OperationFailedError(error) @@ -417,7 +414,6 @@ class Service(object): recreate, lambda c: c.name, "Recreating", - noansi=self.noansi, ) for error in errors.values(): raise OperationFailedError(error) @@ -438,7 +434,6 @@ class Service(object): lambda c: self.start_container_if_stopped(c, attach_logs=not detached), lambda c: c.name, "Starting", - noansi=self.noansi, ) for error in errors.values(): @@ -460,7 +455,6 @@ class Service(object): stop_and_remove, lambda c: c.name, "Stopping and removing", - noansi=self.noansi, ) def execute_convergence_plan(self, plan, timeout=None, detached=False, diff --git a/tests/acceptance/cli_test.py b/tests/acceptance/cli_test.py index f7ecba9f5..adf645c2f 100644 --- a/tests/acceptance/cli_test.py +++ b/tests/acceptance/cli_test.py @@ -751,6 +751,14 @@ class CLITestCase(DockerClientTestCase): for service in services: assert self.lookup(container, service.name) + @v2_only() + def test_up_no_ansi(self): + self.base_dir = 'tests/fixtures/v2-simple' + result = self.dispatch(['--no-ansi', 'up', '-d'], None) + assert "%c[2K\r" % 27 not in result.stderr + assert "%c[1A" % 27 not in result.stderr + assert "%c[1B" % 27 not in result.stderr + @v2_only() def test_up_with_default_network_config(self): filename = 'default-network-config.yml' diff --git a/tests/unit/parallel_test.py b/tests/unit/parallel_test.py index 519c66669..f82858eab 100644 --- a/tests/unit/parallel_test.py +++ b/tests/unit/parallel_test.py @@ -8,6 +8,7 @@ from docker.errors import APIError from compose.parallel import parallel_execute from compose.parallel import parallel_execute_iter +from compose.parallel import ParallelStreamWriter from compose.parallel import UpstreamError @@ -62,7 +63,7 @@ def test_parallel_execute_with_limit(): limit=limit, ) - assert results == tasks*[None] + assert results == tasks * [None] assert errors == {} @@ -133,12 +134,12 @@ def test_parallel_execute_alignment(capsys): def test_parallel_execute_alignment_noansi(capsys): + ParallelStreamWriter.set_noansi() results, errors = parallel_execute( objects=["short", "a very long name"], func=lambda x: x, get_name=six.text_type, msg="Aligning", - noansi=True, ) assert errors == {} From 6361d907f6510971ab196e73c7b721422ef7b05f Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Wed, 9 Aug 2017 18:59:17 -0700 Subject: [PATCH 100/244] Add support for blkio config keys Signed-off-by: Joffrey F --- compose/config/config.py | 50 ++++++++++++++++++++++++-- compose/config/config_schema_v2.0.json | 44 +++++++++++++++++++++++ compose/config/config_schema_v2.1.json | 45 +++++++++++++++++++++++ compose/config/config_schema_v2.2.json | 45 +++++++++++++++++++++++ compose/config/config_schema_v2.3.json | 45 +++++++++++++++++++++++ compose/service.py | 26 ++++++++++++++ compose/utils.py | 10 ++++++ tests/integration/service_test.py | 28 +++++++++++++++ tests/unit/config/config_test.py | 47 ++++++++++++++++++++++++ 9 files changed, 338 insertions(+), 2 deletions(-) diff --git a/compose/config/config.py b/compose/config/config.py index cb25a25a6..fb376b325 100644 --- a/compose/config/config.py +++ b/compose/config/config.py @@ -16,6 +16,7 @@ from . import types from .. import const from ..const import COMPOSEFILE_V1 as V1 from ..utils import build_string_dict +from ..utils import parse_bytes from ..utils import parse_nanoseconds_int from ..utils import splitdrive from ..version import ComposeVersion @@ -108,6 +109,7 @@ DOCKER_CONFIG_KEYS = [ ] ALLOWED_KEYS = DOCKER_CONFIG_KEYS + [ + 'blkio_config', 'build', 'container_name', 'credential_spec', @@ -726,8 +728,9 @@ def process_service(service_config): if field in service_dict: service_dict[field] = to_list(service_dict[field]) - service_dict = process_healthcheck(service_dict, service_config.name) - service_dict = process_ports(service_dict) + service_dict = process_blkio_config(process_ports( + process_healthcheck(service_dict, service_config.name) + )) return service_dict @@ -754,6 +757,28 @@ def process_depends_on(service_dict): return service_dict +def process_blkio_config(service_dict): + if not service_dict.get('blkio_config'): + return service_dict + + for field in ['device_read_bps', 'device_write_bps']: + if field in service_dict['blkio_config']: + for v in service_dict['blkio_config'].get(field, []): + v['rate'] = parse_bytes(v.get('rate', 0)) + + for field in ['device_read_iops', 'device_write_iops']: + if field in service_dict['blkio_config']: + for v in service_dict['blkio_config'].get(field, []): + try: + v['rate'] = int(v.get('rate', 0)) + except ValueError: + raise ConfigurationError( + 'Invalid IOPS value: "{}". Must be a positive integer.'.format(v.get('rate')) + ) + + return service_dict + + def process_healthcheck(service_dict, service_name): if 'healthcheck' not in service_dict: return service_dict @@ -940,6 +965,7 @@ def merge_service_dicts(base, override, version): md.merge_field('logging', merge_logging, default={}) merge_ports(md, base, override) + md.merge_field('blkio_config', merge_blkio_config, default={}) for field in set(ALLOWED_KEYS) - set(md): md.merge_scalar(field) @@ -993,6 +1019,26 @@ def merge_build(output, base, override): return dict(md) +def merge_blkio_config(base, override): + md = MergeDict(base, override) + md.merge_scalar('weight') + + def merge_blkio_limits(base, override): + index = dict((b['path'], b) for b in base) + for o in override: + index[o['path']] = o + + return sorted(list(index.values()), key=lambda x: x['path']) + + for field in [ + "device_read_bps", "device_read_iops", "device_write_bps", + "device_write_iops", "weight_device", + ]: + md.merge_field(field, merge_blkio_limits, default=[]) + + return dict(md) + + def merge_logging(base, override): md = MergeDict(base, override) md.merge_scalar('driver') diff --git a/compose/config/config_schema_v2.0.json b/compose/config/config_schema_v2.0.json index f3688685b..14bafab40 100644 --- a/compose/config/config_schema_v2.0.json +++ b/compose/config/config_schema_v2.0.json @@ -50,6 +50,33 @@ "type": "object", "properties": { + "blkio_config": { + "type": "object", + "properties": { + "device_read_bps": { + "type": "array", + "items": {"$ref": "#/definitions/blkio_limit"} + }, + "device_read_iops": { + "type": "array", + "items": {"$ref": "#/definitions/blkio_limit"} + }, + "device_write_bps": { + "type": "array", + "items": {"$ref": "#/definitions/blkio_limit"} + }, + "device_write_iops": { + "type": "array", + "items": {"$ref": "#/definitions/blkio_limit"} + }, + "weight": {"type": "integer"}, + "weight_device": { + "type": "array", + "items": {"$ref": "#/definitions/blkio_weight"} + } + }, + "additionalProperties": false + }, "build": { "oneOf": [ {"type": "string"}, @@ -326,6 +353,23 @@ ] }, + "blkio_limit": { + "type": "object", + "properties": { + "path": {"type": "string"}, + "rate": {"type": ["integer", "string"]} + }, + "additionalProperties": false + }, + "blkio_weight": { + "type": "object", + "properties": { + "path": {"type": "string"}, + "weight": {"type": "integer"} + }, + "additionalProperties": false + }, + "constraints": { "service": { "id": "#/definitions/constraints/service", diff --git a/compose/config/config_schema_v2.1.json b/compose/config/config_schema_v2.1.json index 5aed9f7b1..9d45c324c 100644 --- a/compose/config/config_schema_v2.1.json +++ b/compose/config/config_schema_v2.1.json @@ -50,6 +50,34 @@ "type": "object", "properties": { + "blkio_config": { + "type": "object", + "properties": { + "device_read_bps": { + "type": "array", + "items": {"$ref": "#/definitions/blkio_limit"} + }, + "device_read_iops": { + "type": "array", + "items": {"$ref": "#/definitions/blkio_limit"} + }, + "device_write_bps": { + "type": "array", + "items": {"$ref": "#/definitions/blkio_limit"} + }, + "device_write_iops": { + "type": "array", + "items": {"$ref": "#/definitions/blkio_limit"} + }, + "weight": {"type": "integer"}, + "weight_device": { + "type": "array", + "items": {"$ref": "#/definitions/blkio_weight"} + } + }, + "additionalProperties": false + }, + "build": { "oneOf": [ {"type": "string"}, @@ -376,6 +404,23 @@ ] }, + "blkio_limit": { + "type": "object", + "properties": { + "path": {"type": "string"}, + "rate": {"type": ["integer", "string"]} + }, + "additionalProperties": false + }, + "blkio_weight": { + "type": "object", + "properties": { + "path": {"type": "string"}, + "weight": {"type": "integer"} + }, + "additionalProperties": false + }, + "constraints": { "service": { "id": "#/definitions/constraints/service", diff --git a/compose/config/config_schema_v2.2.json b/compose/config/config_schema_v2.2.json index 9181e606b..954417018 100644 --- a/compose/config/config_schema_v2.2.json +++ b/compose/config/config_schema_v2.2.json @@ -50,6 +50,34 @@ "type": "object", "properties": { + "blkio_config": { + "type": "object", + "properties": { + "device_read_bps": { + "type": "array", + "items": {"$ref": "#/definitions/blkio_limit"} + }, + "device_read_iops": { + "type": "array", + "items": {"$ref": "#/definitions/blkio_limit"} + }, + "device_write_bps": { + "type": "array", + "items": {"$ref": "#/definitions/blkio_limit"} + }, + "device_write_iops": { + "type": "array", + "items": {"$ref": "#/definitions/blkio_limit"} + }, + "weight": {"type": "integer"}, + "weight_device": { + "type": "array", + "items": {"$ref": "#/definitions/blkio_weight"} + } + }, + "additionalProperties": false + }, + "build": { "oneOf": [ {"type": "string"}, @@ -383,6 +411,23 @@ ] }, + "blkio_limit": { + "type": "object", + "properties": { + "path": {"type": "string"}, + "rate": {"type": ["integer", "string"]} + }, + "additionalProperties": false + }, + "blkio_weight": { + "type": "object", + "properties": { + "path": {"type": "string"}, + "weight": {"type": "integer"} + }, + "additionalProperties": false + }, + "constraints": { "service": { "id": "#/definitions/constraints/service", diff --git a/compose/config/config_schema_v2.3.json b/compose/config/config_schema_v2.3.json index 877340276..10a61186e 100644 --- a/compose/config/config_schema_v2.3.json +++ b/compose/config/config_schema_v2.3.json @@ -50,6 +50,34 @@ "type": "object", "properties": { + "blkio_config": { + "type": "object", + "properties": { + "device_read_bps": { + "type": "array", + "items": {"$ref": "#/definitions/blkio_limit"} + }, + "device_read_iops": { + "type": "array", + "items": {"$ref": "#/definitions/blkio_limit"} + }, + "device_write_bps": { + "type": "array", + "items": {"$ref": "#/definitions/blkio_limit"} + }, + "device_write_iops": { + "type": "array", + "items": {"$ref": "#/definitions/blkio_limit"} + }, + "weight": {"type": "integer"}, + "weight_device": { + "type": "array", + "items": {"$ref": "#/definitions/blkio_weight"} + } + }, + "additionalProperties": false + }, + "build": { "oneOf": [ {"type": "string"}, @@ -384,6 +412,23 @@ ] }, + "blkio_limit": { + "type": "object", + "properties": { + "path": {"type": "string"}, + "rate": {"type": ["integer", "string"]} + }, + "additionalProperties": false + }, + "blkio_weight": { + "type": "object", + "properties": { + "path": {"type": "string"}, + "weight": {"type": "integer"} + }, + "additionalProperties": false + }, + "constraints": { "service": { "id": "#/definitions/constraints/service", diff --git a/compose/service.py b/compose/service.py index c43f635b2..2829240f2 100644 --- a/compose/service.py +++ b/compose/service.py @@ -813,6 +813,7 @@ class Service(object): options = dict(self.options, **override_options) logging_dict = options.get('logging', None) + blkio_config = convert_blkio_config(options.get('blkio_config', None)) log_config = get_log_config(logging_dict) init_path = None if isinstance(options.get('init'), six.string_types): @@ -869,6 +870,12 @@ class Service(object): cpuset_cpus=options.get('cpuset'), cpu_shares=options.get('cpu_shares'), storage_opt=options.get('storage_opt'), + blkio_weight=blkio_config.get('weight'), + blkio_weight_device=blkio_config.get('weight_device'), + device_read_bps=blkio_config.get('device_read_bps'), + device_read_iops=blkio_config.get('device_read_iops'), + device_write_bps=blkio_config.get('device_write_bps'), + device_write_iops=blkio_config.get('device_write_iops'), ) def get_secret_volumes(self): @@ -1395,3 +1402,22 @@ def build_container_ports(container_ports, options): port = tuple(port.split('/')) ports.append(port) return ports + + +def convert_blkio_config(blkio_config): + result = {} + if blkio_config is None: + return result + + result['weight'] = blkio_config.get('weight') + for field in [ + "device_read_bps", "device_read_iops", "device_write_bps", + "device_write_iops", "weight_device", + ]: + if field not in blkio_config: + continue + arr = [] + for item in blkio_config[field]: + arr.append(dict([(k.capitalize(), v) for k, v in item.items()])) + result[field] = arr + return result diff --git a/compose/utils.py b/compose/utils.py index b8bdf732f..183a4504d 100644 --- a/compose/utils.py +++ b/compose/utils.py @@ -9,7 +9,10 @@ import logging import ntpath import six +from docker.errors import DockerException +from docker.utils import parse_bytes as sdk_parse_bytes +from .config.errors import ConfigurationError from .errors import StreamParseError from .timeparse import timeparse @@ -133,3 +136,10 @@ def splitdrive(path): if path[0] in ['.', '\\', '/', '~']: return ('', path) return ntpath.splitdrive(path) + + +def parse_bytes(n): + try: + return sdk_parse_bytes(n) + except DockerException: + raise ConfigurationError('Invalid format for bytes value: {}'.format(n)) diff --git a/tests/integration/service_test.py b/tests/integration/service_test.py index 3a585ec01..8fb2251bf 100644 --- a/tests/integration/service_test.py +++ b/tests/integration/service_test.py @@ -203,6 +203,34 @@ class ServiceTest(DockerClientTestCase): service.start_container(container) assert container.get('HostConfig.ReadonlyRootfs') == read_only + def test_create_container_with_blkio_config(self): + blkio_config = { + 'weight': 300, + 'weight_device': [{'path': '/dev/sda', 'weight': 200}], + 'device_read_bps': [{'path': '/dev/sda', 'rate': 1024 * 1024 * 100}], + 'device_read_iops': [{'path': '/dev/sda', 'rate': 1000}], + 'device_write_bps': [{'path': '/dev/sda', 'rate': 1024 * 1024}], + 'device_write_iops': [{'path': '/dev/sda', 'rate': 800}] + } + service = self.create_service('web', blkio_config=blkio_config) + container = service.create_container() + assert container.get('HostConfig.BlkioWeight') == 300 + assert container.get('HostConfig.BlkioWeightDevice') == [{ + 'Path': '/dev/sda', 'Weight': 200 + }] + assert container.get('HostConfig.BlkioDeviceReadBps') == [{ + 'Path': '/dev/sda', 'Rate': 1024 * 1024 * 100 + }] + assert container.get('HostConfig.BlkioDeviceWriteBps') == [{ + 'Path': '/dev/sda', 'Rate': 1024 * 1024 + }] + assert container.get('HostConfig.BlkioDeviceReadIOps') == [{ + 'Path': '/dev/sda', 'Rate': 1000 + }] + assert container.get('HostConfig.BlkioDeviceWriteIOps') == [{ + 'Path': '/dev/sda', 'Rate': 800 + }] + def test_create_container_with_security_opt(self): security_opt = ['label:disable'] service = self.create_service('db', security_opt=security_opt) diff --git a/tests/unit/config/config_test.py b/tests/unit/config/config_test.py index fd06db7d1..8861baa98 100644 --- a/tests/unit/config/config_test.py +++ b/tests/unit/config/config_test.py @@ -2151,6 +2151,53 @@ class ConfigTest(unittest.TestCase): actual = config.merge_service_dicts(base, override, V2_2) assert actual == {'image': 'bar', 'scale': 4} + def test_merge_blkio_config(self): + base = { + 'image': 'bar', + 'blkio_config': { + 'weight': 300, + 'weight_device': [ + {'path': '/dev/sda1', 'weight': 200} + ], + 'device_read_iops': [ + {'path': '/dev/sda1', 'rate': 300} + ], + 'device_write_iops': [ + {'path': '/dev/sda1', 'rate': 1000} + ] + } + } + + override = { + 'blkio_config': { + 'weight': 450, + 'weight_device': [ + {'path': '/dev/sda2', 'weight': 400} + ], + 'device_read_iops': [ + {'path': '/dev/sda1', 'rate': 2000} + ], + 'device_read_bps': [ + {'path': '/dev/sda1', 'rate': 1024} + ] + } + } + + actual = config.merge_service_dicts(base, override, V2_2) + assert actual == { + 'image': 'bar', + 'blkio_config': { + 'weight': override['blkio_config']['weight'], + 'weight_device': ( + base['blkio_config']['weight_device'] + + override['blkio_config']['weight_device'] + ), + 'device_read_iops': override['blkio_config']['device_read_iops'], + 'device_read_bps': override['blkio_config']['device_read_bps'], + 'device_write_iops': base['blkio_config']['device_write_iops'] + } + } + def test_external_volume_config(self): config_details = build_config_details({ 'version': '2', From b893797e03f33733f3678fc115bca9cabca505d1 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Wed, 2 Aug 2017 16:59:43 -0700 Subject: [PATCH 101/244] UCP 2.2.0 test fixes Signed-off-by: Joffrey F --- tests/acceptance/cli_test.py | 16 +++++++++------- tests/helpers.py | 4 ++++ tests/integration/project_test.py | 3 ++- tests/integration/service_test.py | 8 ++++++-- tests/integration/state_test.py | 17 ++++++++++++++++- tests/integration/testcases.py | 6 +++++- tox.ini | 1 + 7 files changed, 43 insertions(+), 12 deletions(-) diff --git a/tests/acceptance/cli_test.py b/tests/acceptance/cli_test.py index adf645c2f..81bce5460 100644 --- a/tests/acceptance/cli_test.py +++ b/tests/acceptance/cli_test.py @@ -451,7 +451,6 @@ class CLITestCase(DockerClientTestCase): self.dispatch(['build', 'simple']) result = self.dispatch(['build', 'simple']) - assert BUILD_CACHE_TEXT in result.stdout assert BUILD_PULL_TEXT not in result.stdout def test_build_no_cache(self): @@ -469,7 +468,9 @@ class CLITestCase(DockerClientTestCase): self.dispatch(['build', 'simple'], None) result = self.dispatch(['build', '--pull', 'simple']) - assert BUILD_CACHE_TEXT in result.stdout + if not is_cluster(self.client): + # If previous build happened on another node, cache won't be available + assert BUILD_CACHE_TEXT in result.stdout assert BUILD_PULL_TEXT in result.stdout def test_build_no_cache_pull(self): @@ -602,11 +603,12 @@ class CLITestCase(DockerClientTestCase): def test_run_one_off_with_volume(self): self.base_dir = 'tests/fixtures/simple-composefile-volume-ready' volume_path = os.path.abspath(os.path.join(os.getcwd(), self.base_dir, 'files')) - create_host_file(self.client, os.path.join(volume_path, 'example.txt')) + node = create_host_file(self.client, os.path.join(volume_path, 'example.txt')) self.dispatch([ 'run', '-v', '{}:/data'.format(volume_path), + '-e', 'constraint:node=={}'.format(node if node is not None else '*'), 'simple', 'test', '-f', '/data/example.txt' ], returncode=0) @@ -621,12 +623,13 @@ class CLITestCase(DockerClientTestCase): def test_run_one_off_with_multiple_volumes(self): self.base_dir = 'tests/fixtures/simple-composefile-volume-ready' volume_path = os.path.abspath(os.path.join(os.getcwd(), self.base_dir, 'files')) - create_host_file(self.client, os.path.join(volume_path, 'example.txt')) + node = create_host_file(self.client, os.path.join(volume_path, 'example.txt')) self.dispatch([ 'run', '-v', '{}:/data'.format(volume_path), '-v', '{}:/data1'.format(volume_path), + '-e', 'constraint:node=={}'.format(node if node is not None else '*'), 'simple', 'test', '-f', '/data/example.txt' ], returncode=0) @@ -635,6 +638,7 @@ class CLITestCase(DockerClientTestCase): 'run', '-v', '{}:/data'.format(volume_path), '-v', '{}:/data1'.format(volume_path), + '-e', 'constraint:node=={}'.format(node if node is not None else '*'), 'simple', 'test', '-f' '/data1/example.txt' ], returncode=0) @@ -1376,9 +1380,7 @@ class CLITestCase(DockerClientTestCase): break volume_names = [v['Name'].split('/')[-1] for v in volumes] assert name in volume_names - if not is_cluster(self.client): - # The `-v` flag for `docker rm` in Swarm seems to be broken - assert anonymous_name not in volume_names + assert anonymous_name not in volume_names def test_run_service_with_dockerfile_entrypoint(self): self.base_dir = 'tests/fixtures/entrypoint-dockerfile' diff --git a/tests/helpers.py b/tests/helpers.py index 59efd2557..a93de993f 100644 --- a/tests/helpers.py +++ b/tests/helpers.py @@ -42,5 +42,9 @@ def create_host_file(client, filename): output = client.logs(container) raise Exception( "Container exited with code {}:\n{}".format(exitcode, output)) + + container_info = client.inspect_container(container) + if 'Node' in container_info: + return container_info['Node']['Name'] finally: client.remove_container(container, force=True) diff --git a/tests/integration/project_test.py b/tests/integration/project_test.py index 5ead7b8e7..4e44c7f6b 100644 --- a/tests/integration/project_test.py +++ b/tests/integration/project_test.py @@ -1265,7 +1265,7 @@ class ProjectTest(DockerClientTestCase): @v3_only() def test_project_up_with_secrets(self): - create_host_file(self.client, os.path.abspath('tests/fixtures/secrets/default')) + node = create_host_file(self.client, os.path.abspath('tests/fixtures/secrets/default')) config_data = build_config( version=V3_1, @@ -1276,6 +1276,7 @@ class ProjectTest(DockerClientTestCase): 'secrets': [ types.ServiceSecret.parse({'source': 'super', 'target': 'special'}), ], + 'environment': ['constraint:node=={}'.format(node if node is not None else '*')] }], secrets={ 'super': { diff --git a/tests/integration/service_test.py b/tests/integration/service_test.py index 8fb2251bf..2abb12c34 100644 --- a/tests/integration/service_test.py +++ b/tests/integration/service_test.py @@ -325,13 +325,15 @@ class ServiceTest(DockerClientTestCase): command=["top"], labels={LABEL_PROJECT: 'composetest'}, host_config={}, + environment=['affinity:container=={}'.format(volume_container_1.id)], ) host_service = self.create_service( 'host', volumes_from=[ VolumeFromSpec(volume_service, 'rw', 'service'), VolumeFromSpec(volume_container_2, 'rw', 'container') - ] + ], + environment=['affinity:container=={}'.format(volume_container_1.id)], ) host_container = host_service.create_container() host_service.start_container(host_container) @@ -785,6 +787,7 @@ class ServiceTest(DockerClientTestCase): assert service.image() @v2_3_only() + @no_cluster('Not supported on UCP 2.2.0-beta1') # FIXME: remove once support is added def test_build_with_target(self): self.require_api_version('1.30') base_dir = tempfile.mkdtemp() @@ -792,11 +795,12 @@ class ServiceTest(DockerClientTestCase): with open(os.path.join(base_dir, 'Dockerfile'), 'w') as f: f.write('FROM busybox as one\n') + f.write('LABEL com.docker.compose.test=true\n') f.write('LABEL com.docker.compose.test.target=one\n') f.write('FROM busybox as two\n') f.write('LABEL com.docker.compose.test.target=two\n') - service = self.create_service('buildlabels', build={ + service = self.create_service('buildtarget', build={ 'context': text_type(base_dir), 'target': 'one' }) diff --git a/tests/integration/state_test.py b/tests/integration/state_test.py index 0dd5f44ad..047dc7046 100644 --- a/tests/integration/state_test.py +++ b/tests/integration/state_test.py @@ -6,9 +6,11 @@ from __future__ import absolute_import from __future__ import unicode_literals import py +from docker.errors import ImageNotFound from .testcases import DockerClientTestCase from .testcases import get_links +from .testcases import no_cluster from compose.config import config from compose.project import Project from compose.service import ConvergenceStrategy @@ -243,21 +245,34 @@ class ServiceStateTest(DockerClientTestCase): tag = 'latest' image = '{}:{}'.format(repo, tag) + def safe_remove_image(image): + try: + self.client.remove_image(image) + except ImageNotFound: + pass + image_id = self.client.images(name='busybox')[0]['Id'] self.client.tag(image_id, repository=repo, tag=tag) - self.addCleanup(self.client.remove_image, image) + self.addCleanup(safe_remove_image, image) web = self.create_service('web', image=image) container = web.create_container() # update the image c = self.client.create_container(image, ['touch', '/hello.txt'], host_config={}) + + # In the case of a cluster, there's a chance we pick up the old image when + # calculating the new hash. To circumvent that, untag the old image first + # See also: https://github.com/moby/moby/issues/26852 + self.client.remove_image(image, force=True) + self.client.commit(c, repository=repo, tag=tag) self.client.remove_container(c) web = self.create_service('web', image=image) self.assertEqual(('recreate', [container]), web.convergence_plan()) + @no_cluster('Can not guarantee the build will be run on the same node the service is deployed') def test_trigger_recreate_with_build(self): context = py.test.ensuretemp('test_trigger_recreate_with_build') self.addCleanup(context.remove) diff --git a/tests/integration/testcases.py b/tests/integration/testcases.py index b1763b113..b72fb53a8 100644 --- a/tests/integration/testcases.py +++ b/tests/integration/testcases.py @@ -105,7 +105,11 @@ class DockerClientTestCase(unittest.TestCase): for i in self.client.images( filters={'label': 'com.docker.compose.test_image'}): - self.client.remove_image(i, force=True) + try: + self.client.remove_image(i, force=True) + except APIError as e: + if e.is_server_error(): + pass volumes = self.client.volumes().get('Volumes') or [] for v in volumes: diff --git a/tox.ini b/tox.ini index 749be3faa..e4f31ec85 100644 --- a/tox.ini +++ b/tox.ini @@ -18,6 +18,7 @@ deps = -rrequirements-dev.txt commands = py.test -v \ + --full-trace \ --cov=compose \ --cov-report html \ --cov-report term \ From b2a3566cf5b9f83c23cdc3461a05877a003d8aab Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Thu, 10 Aug 2017 10:59:23 -0700 Subject: [PATCH 102/244] Prevent null logging options in `docker-compose config` output Signed-off-by: Joffrey F --- compose/config/config.py | 4 ++-- tests/unit/config/config_test.py | 1 - 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/compose/config/config.py b/compose/config/config.py index fb376b325..f3b8e42fd 100644 --- a/compose/config/config.py +++ b/compose/config/config.py @@ -1044,8 +1044,8 @@ def merge_logging(base, override): md.merge_scalar('driver') if md.get('driver') == base.get('driver') or base.get('driver') is None: md.merge_mapping('options', lambda m: m or {}) - else: - md['options'] = override.get('options') + elif override.get('options'): + md['options'] = override.get('options', {}) return dict(md) diff --git a/tests/unit/config/config_test.py b/tests/unit/config/config_test.py index 8861baa98..8a1e16f8a 100644 --- a/tests/unit/config/config_test.py +++ b/tests/unit/config/config_test.py @@ -1864,7 +1864,6 @@ class ConfigTest(unittest.TestCase): 'image': 'alpine:edge', 'logging': { 'driver': 'syslog', - 'options': None } } From b25eb084aefd17229305b084263d2e829d7a522c Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Wed, 9 Aug 2017 19:43:08 -0700 Subject: [PATCH 103/244] Add support for v3.4 files and custom volume names Signed-off-by: Joffrey F --- compose/config/config.py | 7 +- compose/config/config_schema_v3.4.json | 538 +++++++++++++++++++++++++ compose/config/serialize.py | 4 + compose/const.py | 3 + compose/volume.py | 20 +- tests/unit/volume_test.py | 2 +- 6 files changed, 559 insertions(+), 15 deletions(-) create mode 100644 compose/config/config_schema_v3.4.json diff --git a/compose/config/config.py b/compose/config/config.py index f3b8e42fd..aa829a40e 100644 --- a/compose/config/config.py +++ b/compose/config/config.py @@ -404,11 +404,12 @@ def load_mapping(config_files, get_func, entity_type, working_dir=None): external = config.get('external') if external: + name_field = 'name' if entity_type == 'Volume' else 'external_name' validate_external(entity_type, name, config) if isinstance(external, dict): - config['external_name'] = external.get('name') - else: - config['external_name'] = name + config[name_field] = external.get('name') + elif not config.get('name'): + config[name_field] = name if 'driver_opts' in config: config['driver_opts'] = build_string_dict( diff --git a/compose/config/config_schema_v3.4.json b/compose/config/config_schema_v3.4.json new file mode 100644 index 000000000..ce9512076 --- /dev/null +++ b/compose/config/config_schema_v3.4.json @@ -0,0 +1,538 @@ +{ + "$schema": "http://json-schema.org/draft-04/schema#", + "id": "config_schema_v3.4.json", + "type": "object", + "required": ["version"], + + "properties": { + "version": { + "type": "string" + }, + + "services": { + "id": "#/properties/services", + "type": "object", + "patternProperties": { + "^[a-zA-Z0-9._-]+$": { + "$ref": "#/definitions/service" + } + }, + "additionalProperties": false + }, + + "networks": { + "id": "#/properties/networks", + "type": "object", + "patternProperties": { + "^[a-zA-Z0-9._-]+$": { + "$ref": "#/definitions/network" + } + } + }, + + "volumes": { + "id": "#/properties/volumes", + "type": "object", + "patternProperties": { + "^[a-zA-Z0-9._-]+$": { + "$ref": "#/definitions/volume" + } + }, + "additionalProperties": false + }, + + "secrets": { + "id": "#/properties/secrets", + "type": "object", + "patternProperties": { + "^[a-zA-Z0-9._-]+$": { + "$ref": "#/definitions/secret" + } + }, + "additionalProperties": false + }, + + "configs": { + "id": "#/properties/configs", + "type": "object", + "patternProperties": { + "^[a-zA-Z0-9._-]+$": { + "$ref": "#/definitions/config" + } + }, + "additionalProperties": false + } + }, + + "additionalProperties": false, + + "definitions": { + + "service": { + "id": "#/definitions/service", + "type": "object", + + "properties": { + "deploy": {"$ref": "#/definitions/deployment"}, + "build": { + "oneOf": [ + {"type": "string"}, + { + "type": "object", + "properties": { + "context": {"type": "string"}, + "dockerfile": {"type": "string"}, + "args": {"$ref": "#/definitions/list_or_dict"}, + "labels": {"$ref": "#/definitions/list_or_dict"}, + "cache_from": {"$ref": "#/definitions/list_of_strings"} + }, + "additionalProperties": false + } + ] + }, + "cap_add": {"type": "array", "items": {"type": "string"}, "uniqueItems": true}, + "cap_drop": {"type": "array", "items": {"type": "string"}, "uniqueItems": true}, + "cgroup_parent": {"type": "string"}, + "command": { + "oneOf": [ + {"type": "string"}, + {"type": "array", "items": {"type": "string"}} + ] + }, + "configs": { + "type": "array", + "items": { + "oneOf": [ + {"type": "string"}, + { + "type": "object", + "properties": { + "source": {"type": "string"}, + "target": {"type": "string"}, + "uid": {"type": "string"}, + "gid": {"type": "string"}, + "mode": {"type": "number"} + } + } + ] + } + }, + "container_name": {"type": "string"}, + "credential_spec": {"type": "object", "properties": { + "file": {"type": "string"}, + "registry": {"type": "string"} + }}, + "depends_on": {"$ref": "#/definitions/list_of_strings"}, + "devices": {"type": "array", "items": {"type": "string"}, "uniqueItems": true}, + "dns": {"$ref": "#/definitions/string_or_list"}, + "dns_search": {"$ref": "#/definitions/string_or_list"}, + "domainname": {"type": "string"}, + "entrypoint": { + "oneOf": [ + {"type": "string"}, + {"type": "array", "items": {"type": "string"}} + ] + }, + "env_file": {"$ref": "#/definitions/string_or_list"}, + "environment": {"$ref": "#/definitions/list_or_dict"}, + + "expose": { + "type": "array", + "items": { + "type": ["string", "number"], + "format": "expose" + }, + "uniqueItems": true + }, + + "external_links": {"type": "array", "items": {"type": "string"}, "uniqueItems": true}, + "extra_hosts": {"$ref": "#/definitions/list_or_dict"}, + "healthcheck": {"$ref": "#/definitions/healthcheck"}, + "hostname": {"type": "string"}, + "image": {"type": "string"}, + "ipc": {"type": "string"}, + "labels": {"$ref": "#/definitions/list_or_dict"}, + "links": {"type": "array", "items": {"type": "string"}, "uniqueItems": true}, + + "logging": { + "type": "object", + + "properties": { + "driver": {"type": "string"}, + "options": { + "type": "object", + "patternProperties": { + "^.+$": {"type": ["string", "number", "null"]} + } + } + }, + "additionalProperties": false + }, + + "mac_address": {"type": "string"}, + "network_mode": {"type": "string"}, + + "networks": { + "oneOf": [ + {"$ref": "#/definitions/list_of_strings"}, + { + "type": "object", + "patternProperties": { + "^[a-zA-Z0-9._-]+$": { + "oneOf": [ + { + "type": "object", + "properties": { + "aliases": {"$ref": "#/definitions/list_of_strings"}, + "ipv4_address": {"type": "string"}, + "ipv6_address": {"type": "string"} + }, + "additionalProperties": false + }, + {"type": "null"} + ] + } + }, + "additionalProperties": false + } + ] + }, + "pid": {"type": ["string", "null"]}, + + "ports": { + "type": "array", + "items": { + "oneOf": [ + {"type": "number", "format": "ports"}, + {"type": "string", "format": "ports"}, + { + "type": "object", + "properties": { + "mode": {"type": "string"}, + "target": {"type": "integer"}, + "published": {"type": "integer"}, + "protocol": {"type": "string"} + }, + "additionalProperties": false + } + ] + }, + "uniqueItems": true + }, + + "privileged": {"type": "boolean"}, + "read_only": {"type": "boolean"}, + "restart": {"type": "string"}, + "security_opt": {"type": "array", "items": {"type": "string"}, "uniqueItems": true}, + "shm_size": {"type": ["number", "string"]}, + "secrets": { + "type": "array", + "items": { + "oneOf": [ + {"type": "string"}, + { + "type": "object", + "properties": { + "source": {"type": "string"}, + "target": {"type": "string"}, + "uid": {"type": "string"}, + "gid": {"type": "string"}, + "mode": {"type": "number"} + } + } + ] + } + }, + "sysctls": {"$ref": "#/definitions/list_or_dict"}, + "stdin_open": {"type": "boolean"}, + "stop_grace_period": {"type": "string", "format": "duration"}, + "stop_signal": {"type": "string"}, + "tmpfs": {"$ref": "#/definitions/string_or_list"}, + "tty": {"type": "boolean"}, + "ulimits": { + "type": "object", + "patternProperties": { + "^[a-z]+$": { + "oneOf": [ + {"type": "integer"}, + { + "type":"object", + "properties": { + "hard": {"type": "integer"}, + "soft": {"type": "integer"} + }, + "required": ["soft", "hard"], + "additionalProperties": false + } + ] + } + } + }, + "user": {"type": "string"}, + "userns_mode": {"type": "string"}, + "volumes": { + "type": "array", + "items": { + "oneOf": [ + {"type": "string"}, + { + "type": "object", + "required": ["type"], + "properties": { + "type": {"type": "string"}, + "source": {"type": "string"}, + "target": {"type": "string"}, + "read_only": {"type": "boolean"}, + "consistency": {"type": "string"}, + "bind": { + "type": "object", + "properties": { + "propagation": {"type": "string"} + } + }, + "volume": { + "type": "object", + "properties": { + "nocopy": {"type": "boolean"} + } + } + } + } + ], + "uniqueItems": true + } + }, + "working_dir": {"type": "string"} + }, + "additionalProperties": false + }, + + "healthcheck": { + "id": "#/definitions/healthcheck", + "type": "object", + "additionalProperties": false, + "properties": { + "disable": {"type": "boolean"}, + "interval": {"type": "string"}, + "retries": {"type": "number"}, + "test": { + "oneOf": [ + {"type": "string"}, + {"type": "array", "items": {"type": "string"}} + ] + }, + "timeout": {"type": "string"} + } + }, + "deployment": { + "id": "#/definitions/deployment", + "type": ["object", "null"], + "properties": { + "mode": {"type": "string"}, + "endpoint_mode": {"type": "string"}, + "replicas": {"type": "integer"}, + "labels": {"$ref": "#/definitions/list_or_dict"}, + "update_config": { + "type": "object", + "properties": { + "parallelism": {"type": "integer"}, + "delay": {"type": "string", "format": "duration"}, + "failure_action": {"type": "string"}, + "monitor": {"type": "string", "format": "duration"}, + "max_failure_ratio": {"type": "number"}, + "order": {"type": "string", "enum": [ + "start-first", "stop-first" + ]} + }, + "additionalProperties": false + }, + "resources": { + "type": "object", + "properties": { + "limits": {"$ref": "#/definitions/resource"}, + "reservations": {"$ref": "#/definitions/resource"} + } + }, + "restart_policy": { + "type": "object", + "properties": { + "condition": {"type": "string"}, + "delay": {"type": "string", "format": "duration"}, + "max_attempts": {"type": "integer"}, + "window": {"type": "string", "format": "duration"} + }, + "additionalProperties": false + }, + "placement": { + "type": "object", + "properties": { + "constraints": {"type": "array", "items": {"type": "string"}}, + "preferences": { + "type": "array", + "items": { + "type": "object", + "properties": { + "spread": {"type": "string"} + }, + "additionalProperties": false + } + } + }, + "additionalProperties": false + } + }, + "additionalProperties": false + }, + + "resource": { + "id": "#/definitions/resource", + "type": "object", + "properties": { + "cpus": {"type": "string"}, + "memory": {"type": "string"} + }, + "additionalProperties": false + }, + + "network": { + "id": "#/definitions/network", + "type": ["object", "null"], + "properties": { + "driver": {"type": "string"}, + "driver_opts": { + "type": "object", + "patternProperties": { + "^.+$": {"type": ["string", "number"]} + } + }, + "ipam": { + "type": "object", + "properties": { + "driver": {"type": "string"}, + "config": { + "type": "array", + "items": { + "type": "object", + "properties": { + "subnet": {"type": "string"} + }, + "additionalProperties": false + } + } + }, + "additionalProperties": false + }, + "external": { + "type": ["boolean", "object"], + "properties": { + "name": {"type": "string"} + }, + "additionalProperties": false + }, + "internal": {"type": "boolean"}, + "attachable": {"type": "boolean"}, + "labels": {"$ref": "#/definitions/list_or_dict"} + }, + "additionalProperties": false + }, + + "volume": { + "id": "#/definitions/volume", + "type": ["object", "null"], + "properties": { + "name": {"type": "string"}, + "driver": {"type": "string"}, + "driver_opts": { + "type": "object", + "patternProperties": { + "^.+$": {"type": ["string", "number"]} + } + }, + "external": { + "type": ["boolean", "object"], + "properties": { + "name": {"type": "string"} + }, + "additionalProperties": false + }, + "labels": {"$ref": "#/definitions/list_or_dict"} + }, + "additionalProperties": false + }, + + "secret": { + "id": "#/definitions/secret", + "type": "object", + "properties": { + "file": {"type": "string"}, + "external": { + "type": ["boolean", "object"], + "properties": { + "name": {"type": "string"} + } + }, + "labels": {"$ref": "#/definitions/list_or_dict"} + }, + "additionalProperties": false + }, + + "config": { + "id": "#/definitions/config", + "type": "object", + "properties": { + "file": {"type": "string"}, + "external": { + "type": ["boolean", "object"], + "properties": { + "name": {"type": "string"} + } + }, + "labels": {"$ref": "#/definitions/list_or_dict"} + }, + "additionalProperties": false + }, + + "string_or_list": { + "oneOf": [ + {"type": "string"}, + {"$ref": "#/definitions/list_of_strings"} + ] + }, + + "list_of_strings": { + "type": "array", + "items": {"type": "string"}, + "uniqueItems": true + }, + + "list_or_dict": { + "oneOf": [ + { + "type": "object", + "patternProperties": { + ".+": { + "type": ["string", "number", "null"] + } + }, + "additionalProperties": false + }, + {"type": "array", "items": {"type": "string"}, "uniqueItems": true} + ] + }, + + "constraints": { + "service": { + "id": "#/definitions/constraints/service", + "anyOf": [ + {"required": ["build"]}, + {"required": ["image"]} + ], + "properties": { + "build": { + "required": ["context"] + } + } + } + } + } +} diff --git a/compose/config/serialize.py b/compose/config/serialize.py index 86fdac38f..6efe5fc9f 100644 --- a/compose/config/serialize.py +++ b/compose/config/serialize.py @@ -9,6 +9,7 @@ from compose.const import COMPOSEFILE_V1 as V1 from compose.const import COMPOSEFILE_V2_1 as V2_1 from compose.const import COMPOSEFILE_V3_0 as V3_0 from compose.const import COMPOSEFILE_V3_2 as V3_2 +from compose.const import COMPOSEFILE_V3_2 as V3_4 def serialize_config_type(dumper, data): @@ -65,6 +66,9 @@ def denormalize_config(config, image_digests=None): if 'external_name' in conf: del conf['external_name'] + if 'name' in conf and config.version < V3_4: + del conf['name'] + return result diff --git a/compose/const.py b/compose/const.py index 6ea0ea79c..b5970f82a 100644 --- a/compose/const.py +++ b/compose/const.py @@ -31,6 +31,7 @@ COMPOSEFILE_V3_0 = ComposeVersion('3.0') COMPOSEFILE_V3_1 = ComposeVersion('3.1') COMPOSEFILE_V3_2 = ComposeVersion('3.2') COMPOSEFILE_V3_3 = ComposeVersion('3.3') +COMPOSEFILE_V3_4 = ComposeVersion('3.4') API_VERSIONS = { COMPOSEFILE_V1: '1.21', @@ -42,6 +43,7 @@ API_VERSIONS = { COMPOSEFILE_V3_1: '1.25', COMPOSEFILE_V3_2: '1.25', COMPOSEFILE_V3_3: '1.30', + COMPOSEFILE_V3_4: '1.30', } API_VERSION_TO_ENGINE_VERSION = { @@ -54,4 +56,5 @@ API_VERSION_TO_ENGINE_VERSION = { API_VERSIONS[COMPOSEFILE_V3_1]: '1.13.0', API_VERSIONS[COMPOSEFILE_V3_2]: '1.13.0', API_VERSIONS[COMPOSEFILE_V3_3]: '17.06.0', + API_VERSIONS[COMPOSEFILE_V3_4]: '17.06.0', } diff --git a/compose/volume.py b/compose/volume.py index ab6a88fac..da8ba25ca 100644 --- a/compose/volume.py +++ b/compose/volume.py @@ -15,14 +15,15 @@ log = logging.getLogger(__name__) class Volume(object): def __init__(self, client, project, name, driver=None, driver_opts=None, - external_name=None, labels=None): + external=False, labels=None, custom_name=False): self.client = client self.project = project self.name = name self.driver = driver self.driver_opts = driver_opts - self.external_name = external_name + self.external = external self.labels = labels + self.custom_name = custom_name def create(self): return self.client.create_volume( @@ -46,14 +47,10 @@ class Volume(object): return False return True - @property - def external(self): - return bool(self.external_name) - @property def full_name(self): - if self.external_name: - return self.external_name + if self.custom_name: + return self.name return '{0}_{1}'.format(self.project, self.name) @property @@ -80,11 +77,12 @@ class ProjectVolumes(object): vol_name: Volume( client=client, project=name, - name=vol_name, + name=data.get('name', vol_name), driver=data.get('driver'), driver_opts=data.get('driver_opts'), - external_name=data.get('external_name'), - labels=data.get('labels') + custom_name=data.get('name') is not None, + labels=data.get('labels'), + external=bool(data.get('external', False)) ) for vol_name, data in config_volumes.items() } diff --git a/tests/unit/volume_test.py b/tests/unit/volume_test.py index 24829192a..457d85581 100644 --- a/tests/unit/volume_test.py +++ b/tests/unit/volume_test.py @@ -21,6 +21,6 @@ class TestVolume(object): mock_client.remove_volume.assert_called_once_with('foo_project') def test_remove_external_volume(self, mock_client): - vol = volume.Volume(mock_client, 'foo', 'project', external_name='data') + vol = volume.Volume(mock_client, 'foo', 'project', external=True) vol.remove() assert not mock_client.remove_volume.called From 41a5a4a3217e367df30c6a8242a815802057fe28 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Thu, 10 Aug 2017 12:09:05 -0700 Subject: [PATCH 104/244] v2 custom volume name support Signed-off-by: Joffrey F --- compose/config/config_schema_v2.1.json | 3 ++- compose/config/config_schema_v2.2.json | 3 ++- compose/config/config_schema_v2.3.json | 3 ++- compose/config/serialize.py | 7 +++++-- tests/acceptance/cli_test.py | 6 ++++-- tests/integration/project_test.py | 4 ++-- tests/integration/volume_test.py | 19 +++++++++++++++---- 7 files changed, 32 insertions(+), 13 deletions(-) diff --git a/compose/config/config_schema_v2.1.json b/compose/config/config_schema_v2.1.json index 9d45c324c..8a5e12834 100644 --- a/compose/config/config_schema_v2.1.json +++ b/compose/config/config_schema_v2.1.json @@ -371,7 +371,8 @@ }, "additionalProperties": false }, - "labels": {"$ref": "#/definitions/list_or_dict"} + "labels": {"$ref": "#/definitions/list_or_dict"}, + "name": {"type": "string"} }, "additionalProperties": false }, diff --git a/compose/config/config_schema_v2.2.json b/compose/config/config_schema_v2.2.json index 954417018..58ba409ff 100644 --- a/compose/config/config_schema_v2.2.json +++ b/compose/config/config_schema_v2.2.json @@ -378,7 +378,8 @@ }, "additionalProperties": false }, - "labels": {"$ref": "#/definitions/list_or_dict"} + "labels": {"$ref": "#/definitions/list_or_dict"}, + "name": {"type": "string"} }, "additionalProperties": false }, diff --git a/compose/config/config_schema_v2.3.json b/compose/config/config_schema_v2.3.json index 10a61186e..789adf4ab 100644 --- a/compose/config/config_schema_v2.3.json +++ b/compose/config/config_schema_v2.3.json @@ -379,7 +379,8 @@ }, "additionalProperties": false }, - "labels": {"$ref": "#/definitions/list_or_dict"} + "labels": {"$ref": "#/definitions/list_or_dict"}, + "name": {"type": "string"} }, "additionalProperties": false }, diff --git a/compose/config/serialize.py b/compose/config/serialize.py index 6efe5fc9f..1c52fc056 100644 --- a/compose/config/serialize.py +++ b/compose/config/serialize.py @@ -66,8 +66,11 @@ def denormalize_config(config, image_digests=None): if 'external_name' in conf: del conf['external_name'] - if 'name' in conf and config.version < V3_4: - del conf['name'] + if 'name' in conf: + if config.version < V2_1 or (config.version > V3_0 and config.version < V3_4): + del conf['name'] + elif 'external' in conf: + conf['external'] = True return result diff --git a/tests/acceptance/cli_test.py b/tests/acceptance/cli_test.py index 81bce5460..bee7b74a2 100644 --- a/tests/acceptance/cli_test.py +++ b/tests/acceptance/cli_test.py @@ -291,10 +291,12 @@ class CLITestCase(DockerClientTestCase): assert 'volumes' in json_result assert json_result['volumes'] == { 'foo': { - 'external': True + 'external': True, + 'name': 'foo', }, 'bar': { - 'external': {'name': 'some_bar'} + 'external': True, + 'name': 'some_bar', } } diff --git a/tests/integration/project_test.py b/tests/integration/project_test.py index 4e44c7f6b..953dd52be 100644 --- a/tests/integration/project_test.py +++ b/tests/integration/project_test.py @@ -1412,7 +1412,7 @@ class ProjectTest(DockerClientTestCase): 'command': 'top' }], volumes={ - vol_name: {'external': True, 'external_name': vol_name} + vol_name: {'external': True, 'name': vol_name} }, ) project = Project.from_config( @@ -1436,7 +1436,7 @@ class ProjectTest(DockerClientTestCase): 'command': 'top' }], volumes={ - vol_name: {'external': True, 'external_name': vol_name} + vol_name: {'external': True, 'name': vol_name} }, ) project = Project.from_config( diff --git a/tests/integration/volume_test.py b/tests/integration/volume_test.py index ecc71d0b1..2a521d4c5 100644 --- a/tests/integration/volume_test.py +++ b/tests/integration/volume_test.py @@ -1,6 +1,7 @@ from __future__ import absolute_import from __future__ import unicode_literals +import six from docker.errors import DockerException from .testcases import DockerClientTestCase @@ -23,12 +24,15 @@ class VolumeTest(DockerClientTestCase): del self.tmp_volumes super(VolumeTest, self).tearDown() - def create_volume(self, name, driver=None, opts=None, external=None): - if external and isinstance(external, bool): - external = name + def create_volume(self, name, driver=None, opts=None, external=None, custom_name=False): + if external: + custom_name = True + if isinstance(external, six.text_type): + name = external + vol = Volume( self.client, 'composetest', name, driver=driver, driver_opts=opts, - external_name=external + external=bool(external), custom_name=custom_name ) self.tmp_volumes.append(vol) return vol @@ -39,6 +43,13 @@ class VolumeTest(DockerClientTestCase): info = self.get_volume_data(vol.full_name) assert info['Name'].split('/')[-1] == vol.full_name + def test_create_volume_custom_name(self): + vol = self.create_volume('volume01', custom_name=True) + assert vol.name == vol.full_name + vol.create() + info = self.get_volume_data(vol.full_name) + assert info['Name'].split('/')[-1] == vol.name + def test_recreate_existing_volume(self): vol = self.create_volume('volume01') From 43cb1f3dff53b30ba74934622b24b489c5e2b8b1 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Fri, 11 Aug 2017 15:52:43 -0700 Subject: [PATCH 105/244] Add support for start_period in healthcheck config Improve merging strategy for healthcheck configs Signed-off-by: Joffrey F --- compose/config/config.py | 25 +++++---- compose/config/config_schema_v2.3.json | 1 + compose/config/serialize.py | 4 ++ compose/utils.py | 5 +- tests/integration/service_test.py | 19 +++++++ tests/unit/config/config_test.py | 77 +++++++++++++++++++++++++- 6 files changed, 117 insertions(+), 14 deletions(-) diff --git a/compose/config/config.py b/compose/config/config.py index aa829a40e..0c2ab1ab7 100644 --- a/compose/config/config.py +++ b/compose/config/config.py @@ -797,16 +797,12 @@ def process_healthcheck(service_dict, service_name): elif 'test' in raw: hc['test'] = raw['test'] - if 'interval' in raw: - if not isinstance(raw['interval'], six.integer_types): - hc['interval'] = parse_nanoseconds_int(raw['interval']) - else: # Conversion has been done previously - hc['interval'] = raw['interval'] - if 'timeout' in raw: - if not isinstance(raw['timeout'], six.integer_types): - hc['timeout'] = parse_nanoseconds_int(raw['timeout']) - else: # Conversion has been done previously - hc['timeout'] = raw['timeout'] + for field in ['interval', 'timeout', 'start_period']: + if field in raw: + if not isinstance(raw[field], six.integer_types): + hc[field] = parse_nanoseconds_int(raw[field]) + else: # Conversion has been done previously + hc[field] = raw[field] if 'retries' in raw: hc['retries'] = raw['retries'] @@ -967,6 +963,7 @@ def merge_service_dicts(base, override, version): md.merge_field('logging', merge_logging, default={}) merge_ports(md, base, override) md.merge_field('blkio_config', merge_blkio_config, default={}) + md.merge_field('healthcheck', merge_healthchecks, default={}) for field in set(ALLOWED_KEYS) - set(md): md.merge_scalar(field) @@ -985,6 +982,14 @@ def merge_unique_items_lists(base, override): return sorted(set().union(base, override)) +def merge_healthchecks(base, override): + if override.get('disabled') is True: + return override + result = base.copy() + result.update(override) + return result + + def merge_ports(md, base, override): def parse_sequence_func(seq): acc = [] diff --git a/compose/config/config_schema_v2.3.json b/compose/config/config_schema_v2.3.json index 789adf4ab..7a9bdfdf1 100644 --- a/compose/config/config_schema_v2.3.json +++ b/compose/config/config_schema_v2.3.json @@ -309,6 +309,7 @@ "disable": {"type": "boolean"}, "interval": {"type": "string"}, "retries": {"type": "number"}, + "start_period": {"type": "string"}, "test": { "oneOf": [ {"type": "string"}, diff --git a/compose/config/serialize.py b/compose/config/serialize.py index 1c52fc056..606dd7614 100644 --- a/compose/config/serialize.py +++ b/compose/config/serialize.py @@ -132,6 +132,10 @@ def denormalize_service_dict(service_dict, version, image_digest=None): service_dict['healthcheck']['timeout'] ) + if 'start_period' in service_dict['healthcheck']: + service_dict['healthcheck']['start_period'] = serialize_ns_time_value( + service_dict['healthcheck']['start_period'] + ) if 'ports' in service_dict and version < V3_2: service_dict['ports'] = [ p.legacy_repr() if isinstance(p, types.ServicePort) else p diff --git a/compose/utils.py b/compose/utils.py index 183a4504d..1ede4d37d 100644 --- a/compose/utils.py +++ b/compose/utils.py @@ -14,6 +14,7 @@ from docker.utils import parse_bytes as sdk_parse_bytes from .config.errors import ConfigurationError from .errors import StreamParseError +from .timeparse import MULTIPLIERS from .timeparse import timeparse @@ -112,7 +113,7 @@ def microseconds_from_time_nano(time_nano): def nanoseconds_from_time_seconds(time_seconds): - return time_seconds * 1000000000 + return int(time_seconds / MULTIPLIERS['nano']) def parse_seconds_float(value): @@ -123,7 +124,7 @@ def parse_nanoseconds_int(value): parsed = timeparse(value or '') if parsed is None: return None - return int(parsed * 1000000000) + return nanoseconds_from_time_seconds(parsed) def build_string_dict(source_dict): diff --git a/tests/integration/service_test.py b/tests/integration/service_test.py index 2abb12c34..84b54fe41 100644 --- a/tests/integration/service_test.py +++ b/tests/integration/service_test.py @@ -36,6 +36,7 @@ from compose.service import ConvergenceStrategy from compose.service import NetworkMode from compose.service import PidMode from compose.service import Service +from compose.utils import parse_nanoseconds_int from tests.integration.testcases import is_cluster from tests.integration.testcases import no_cluster from tests.integration.testcases import v2_1_only @@ -270,6 +271,24 @@ class ServiceTest(DockerClientTestCase): self.assertTrue(path.basename(actual_host_path) == path.basename(host_path), msg=("Last component differs: %s, %s" % (actual_host_path, host_path))) + def test_create_container_with_healthcheck_config(self): + one_second = parse_nanoseconds_int('1s') + healthcheck = { + 'test': ['true'], + 'interval': 2 * one_second, + 'timeout': 5 * one_second, + 'retries': 5, + 'start_period': 2 * one_second + } + service = self.create_service('db', healthcheck=healthcheck) + container = service.create_container() + remote_healthcheck = container.get('Config.Healthcheck') + assert remote_healthcheck['Test'] == healthcheck['test'] + assert remote_healthcheck['Interval'] == healthcheck['interval'] + assert remote_healthcheck['Timeout'] == healthcheck['timeout'] + assert remote_healthcheck['Retries'] == healthcheck['retries'] + assert remote_healthcheck['StartPeriod'] == healthcheck['start_period'] + def test_recreate_preserves_volume_with_trailing_slash(self): """When the Compose file specifies a trailing slash in the container path, make sure we copy the volume over when recreating. diff --git a/tests/unit/config/config_test.py b/tests/unit/config/config_test.py index 8a1e16f8a..4e355d3bf 100644 --- a/tests/unit/config/config_test.py +++ b/tests/unit/config/config_test.py @@ -2197,6 +2197,75 @@ class ConfigTest(unittest.TestCase): } } + def test_merge_healthcheck_config(self): + base = { + 'image': 'bar', + 'healthcheck': { + 'start_period': 1000, + 'interval': 3000, + 'test': ['true'] + } + } + + override = { + 'healthcheck': { + 'interval': 5000, + 'timeout': 10000, + 'test': ['echo', 'OK'], + } + } + + actual = config.merge_service_dicts(base, override, V2_3) + assert actual['healthcheck'] == { + 'start_period': base['healthcheck']['start_period'], + 'test': override['healthcheck']['test'], + 'interval': override['healthcheck']['interval'], + 'timeout': override['healthcheck']['timeout'], + } + + def test_merge_healthcheck_override_disables(self): + base = { + 'image': 'bar', + 'healthcheck': { + 'start_period': 1000, + 'interval': 3000, + 'timeout': 2000, + 'retries': 3, + 'test': ['true'] + } + } + + override = { + 'healthcheck': { + 'disabled': True + } + } + + actual = config.merge_service_dicts(base, override, V2_3) + assert actual['healthcheck'] == {'disabled': True} + + def test_merge_healthcheck_override_enables(self): + base = { + 'image': 'bar', + 'healthcheck': { + 'disabled': True + } + } + + override = { + 'healthcheck': { + 'disabled': False, + 'start_period': 1000, + 'interval': 3000, + 'timeout': 2000, + 'retries': 3, + 'test': ['true'] + } + } + + actual = config.merge_service_dicts(base, override, V2_3) + assert actual['healthcheck'] == override['healthcheck'] + def test_external_volume_config(self): config_details = build_config_details({ 'version': '2', @@ -4008,6 +4077,7 @@ class HealthcheckTest(unittest.TestCase): 'interval': '1s', 'timeout': '1m', 'retries': 3, + 'start_period': '10s' }}, '.', ) @@ -4017,6 +4087,7 @@ class HealthcheckTest(unittest.TestCase): 'interval': nanoseconds_from_time_seconds(1), 'timeout': nanoseconds_from_time_seconds(60), 'retries': 3, + 'start_period': nanoseconds_from_time_seconds(10) } def test_disable(self): @@ -4147,15 +4218,17 @@ class SerializeTest(unittest.TestCase): 'test': 'exit 1', 'interval': '1m40s', 'timeout': '30s', - 'retries': 5 + 'retries': 5, + 'start_period': '2s90ms' } } processed_service = config.process_service(config.ServiceConfig( '.', 'test', 'test', service_dict )) - denormalized_service = denormalize_service_dict(processed_service, V2_1) + denormalized_service = denormalize_service_dict(processed_service, V2_3) assert denormalized_service['healthcheck']['interval'] == '100s' assert denormalized_service['healthcheck']['timeout'] == '30s' + assert denormalized_service['healthcheck']['start_period'] == '2090ms' def test_denormalize_image_has_digest(self): service_dict = { From f1baee3292f02c8507b2addf538742100abd94c7 Mon Sep 17 00:00:00 2001 From: aronahl Date: Wed, 9 Aug 2017 19:44:12 -0400 Subject: [PATCH 106/244] Fix exit code 0 upon parallel pull failure. Signed-off-by: Aaron Nall --- compose/project.py | 4 +++- tests/acceptance/cli_test.py | 15 +++++++++++++++ 2 files changed, 18 insertions(+), 1 deletion(-) diff --git a/compose/project.py b/compose/project.py index 86fbda6ee..2310a2fcc 100644 --- a/compose/project.py +++ b/compose/project.py @@ -498,13 +498,15 @@ class Project(object): def pull_service(service): service.pull(ignore_pull_failures, True) - parallel.parallel_execute( + _, errors = parallel.parallel_execute( services, pull_service, operator.attrgetter('name'), 'Pulling', limit=5, ) + if len(errors): + raise ProjectError(b"\n".join(errors.values())) else: for service in services: service.pull(ignore_pull_failures, silent=silent) diff --git a/tests/acceptance/cli_test.py b/tests/acceptance/cli_test.py index bee7b74a2..78d1c1eb1 100644 --- a/tests/acceptance/cli_test.py +++ b/tests/acceptance/cli_test.py @@ -6,6 +6,7 @@ import datetime import json import os import os.path +import re import signal import subprocess import time @@ -448,6 +449,20 @@ class CLITestCase(DockerClientTestCase): assert self.dispatch(['pull', '--quiet']).stderr == '' assert self.dispatch(['pull', '--quiet']).stdout == '' + def test_pull_with_parallel_failure(self): + result = self.dispatch([ + '-f', 'ignore-pull-failures.yml', 'pull', '--parallel'], + returncode=1 + ) + + self.assertRegexpMatches(result.stderr, re.compile('^Pulling simple', re.MULTILINE)) + self.assertRegexpMatches(result.stderr, re.compile('^Pulling another', re.MULTILINE)) + self.assertRegexpMatches(result.stderr, + re.compile('^ERROR: for another .*does not exist.*', re.MULTILINE)) + self.assertRegexpMatches(result.stderr, + re.compile('''^(ERROR: )?(b')?.* nonexisting-image''', + re.MULTILINE)) + def test_build_plain(self): self.base_dir = 'tests/fixtures/simple-dockerfile' self.dispatch(['build', 'simple']) From f2aebf8004ce70b8af9464973b0b74b82d25f264 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Thu, 17 Aug 2017 14:31:20 -0700 Subject: [PATCH 107/244] Bump python SDK version -> 2.5.0 Signed-off-by: Joffrey F --- requirements.txt | 2 +- setup.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/requirements.txt b/requirements.txt index d1778990f..d0c0e941b 100644 --- a/requirements.txt +++ b/requirements.txt @@ -3,7 +3,7 @@ cached-property==1.3.0 certifi==2017.4.17 chardet==3.0.4 colorama==0.3.9; sys_platform == 'win32' -docker==2.4.2 +docker==2.5.0 docker-pycreds==0.2.1 dockerpty==0.4.1 docopt==0.6.2 diff --git a/setup.py b/setup.py index 16493f52b..9721fd384 100644 --- a/setup.py +++ b/setup.py @@ -36,7 +36,7 @@ install_requires = [ 'requests >= 2.6.1, != 2.11.0, < 2.12', 'texttable >= 0.9.0, < 0.10', 'websocket-client >= 0.32.0, < 1.0', - 'docker >= 2.4.2, < 3.0', + 'docker >= 2.5.0, < 3.0', 'dockerpty >= 0.4.1, < 0.5', 'six >= 1.3.0, < 2', 'jsonschema >= 2.5.1, < 3', From 7611492f9c1193543e39ec1259e63007bbead6d7 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Thu, 17 Aug 2017 15:52:08 -0700 Subject: [PATCH 108/244] Update schemas to prevent invalid properties in deploy.resources Signed-off-by: Joffrey F --- compose/config/config_schema_v3.0.json | 3 ++- compose/config/config_schema_v3.1.json | 3 ++- compose/config/config_schema_v3.2.json | 3 ++- compose/config/config_schema_v3.3.json | 3 ++- compose/config/config_schema_v3.4.json | 7 +++++-- 5 files changed, 13 insertions(+), 6 deletions(-) diff --git a/compose/config/config_schema_v3.0.json b/compose/config/config_schema_v3.0.json index fbcd8bb85..f39344cfb 100644 --- a/compose/config/config_schema_v3.0.json +++ b/compose/config/config_schema_v3.0.json @@ -240,7 +240,8 @@ "properties": { "limits": {"$ref": "#/definitions/resource"}, "reservations": {"$ref": "#/definitions/resource"} - } + }, + "additionalProperties": false }, "restart_policy": { "type": "object", diff --git a/compose/config/config_schema_v3.1.json b/compose/config/config_schema_v3.1.json index b7037485f..719c0fa7a 100644 --- a/compose/config/config_schema_v3.1.json +++ b/compose/config/config_schema_v3.1.json @@ -269,7 +269,8 @@ "properties": { "limits": {"$ref": "#/definitions/resource"}, "reservations": {"$ref": "#/definitions/resource"} - } + }, + "additionalProperties": false }, "restart_policy": { "type": "object", diff --git a/compose/config/config_schema_v3.2.json b/compose/config/config_schema_v3.2.json index 70ff6ce05..b26b2c6c6 100644 --- a/compose/config/config_schema_v3.2.json +++ b/compose/config/config_schema_v3.2.json @@ -313,7 +313,8 @@ "properties": { "limits": {"$ref": "#/definitions/resource"}, "reservations": {"$ref": "#/definitions/resource"} - } + }, + "additionalProperties": false }, "restart_policy": { "type": "object", diff --git a/compose/config/config_schema_v3.3.json b/compose/config/config_schema_v3.3.json index e69116c38..f1eb9a661 100644 --- a/compose/config/config_schema_v3.3.json +++ b/compose/config/config_schema_v3.3.json @@ -348,7 +348,8 @@ "properties": { "limits": {"$ref": "#/definitions/resource"}, "reservations": {"$ref": "#/definitions/resource"} - } + }, + "additionalProperties": false }, "restart_policy": { "type": "object", diff --git a/compose/config/config_schema_v3.4.json b/compose/config/config_schema_v3.4.json index ce9512076..5a110a888 100644 --- a/compose/config/config_schema_v3.4.json +++ b/compose/config/config_schema_v3.4.json @@ -84,7 +84,9 @@ "dockerfile": {"type": "string"}, "args": {"$ref": "#/definitions/list_or_dict"}, "labels": {"$ref": "#/definitions/list_or_dict"}, - "cache_from": {"$ref": "#/definitions/list_of_strings"} + "cache_from": {"$ref": "#/definitions/list_of_strings"}, + "network": {"type": "string"}, + "target": {"type": "string"} }, "additionalProperties": false } @@ -351,7 +353,8 @@ "properties": { "limits": {"$ref": "#/definitions/resource"}, "reservations": {"$ref": "#/definitions/resource"} - } + }, + "additionalProperties": false }, "restart_policy": { "type": "object", From 7805960a73cbcc58eec49cfbf86fdc66045a57fd Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Fri, 18 Aug 2017 15:37:14 -0700 Subject: [PATCH 109/244] Bump 1.16.0-rc1 Signed-off-by: Joffrey F --- CHANGELOG.md | 60 +++++++++++++++++++++++++++++++++++++++++++++ compose/__init__.py | 2 +- script/run/run.sh | 2 +- 3 files changed, 62 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 928922782..0790f6184 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,66 @@ Change log ========== +1.16.0 (2017-08-31) +------------------- + +### New features + +#### Compose file version 3.4 + +- Introduced version 3.4 of the `docker-compose.yml` specification. + This version requires to be used with Docker Engine 17.06.0 or above. + +#### Compose file version 2.3 + +- Introduced version 2.3 of the `docker-compose.yml` specification. + This version requires to be used with Docker Engine 17.06.0 or above. + +- Added support for the `target` parameter in network configurations + (also available in 3.4) + +- Added support for the `start_period` parameter in healthcheck + configurations + +#### Compose file version 2.x + +- Added support for the `blkio_config` parameter in service definitions + +- Added support for setting a custom name in volume definitions using + the `name` parameter (not available for version 2.0) + +#### All formats + +- Added new CLI flag `--no-ansi` to suppress ANSI control characters in + output + +### Bugfixes + +- Fixed a bug where nested `extends` instructions weren't resolved + properly, causing "file not found" errors + +- Fixed several issues with `.dockerignore` parsing + +- Fixed issues where logs of TTY-enabled services were being printed + incorrectly and causing `MemoryError` exceptions + +- The `$` character in the output of `docker-compose config` is now + properly escaped + +- Fixed a bug where running `docker-compose top` would sometimes fail + with an uncaught exception + +- Fixed a bug where `docker-compose pull` with the `--parallel` flag + would return a `0` exit code when failing + +- Fixed an issue where keys in `deploy.resources` were not being validated + +- Fixed an issue where the `logging` options in the output of + `docker-compose config` would be set to `null`, an invalid value + +- Fixed the output of `docker-compose config` when a port definition used + `0` as the value for the published port + 1.15.0 (2017-07-26) ------------------- diff --git a/compose/__init__.py b/compose/__init__.py index cedb7cf04..b090ccfa2 100644 --- a/compose/__init__.py +++ b/compose/__init__.py @@ -1,4 +1,4 @@ from __future__ import absolute_import from __future__ import unicode_literals -__version__ = '1.16.0-dev' +__version__ = '1.16.0-rc1' diff --git a/script/run/run.sh b/script/run/run.sh index 47a81c7f8..bf9a26cb8 100755 --- a/script/run/run.sh +++ b/script/run/run.sh @@ -15,7 +15,7 @@ set -e -VERSION="1.15.0" +VERSION="1.16.0" IMAGE="docker/compose:$VERSION" From d2543c830df406cb8bbb2e329ba2210c516cbc94 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Tue, 22 Aug 2017 17:14:21 -0700 Subject: [PATCH 110/244] Bump docker SDK -> 2.5.1 Signed-off-by: Joffrey F --- requirements.txt | 2 +- setup.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/requirements.txt b/requirements.txt index d0c0e941b..beeaa2851 100644 --- a/requirements.txt +++ b/requirements.txt @@ -3,7 +3,7 @@ cached-property==1.3.0 certifi==2017.4.17 chardet==3.0.4 colorama==0.3.9; sys_platform == 'win32' -docker==2.5.0 +docker==2.5.1 docker-pycreds==0.2.1 dockerpty==0.4.1 docopt==0.6.2 diff --git a/setup.py b/setup.py index 9721fd384..192a0f6af 100644 --- a/setup.py +++ b/setup.py @@ -36,7 +36,7 @@ install_requires = [ 'requests >= 2.6.1, != 2.11.0, < 2.12', 'texttable >= 0.9.0, < 0.10', 'websocket-client >= 0.32.0, < 1.0', - 'docker >= 2.5.0, < 3.0', + 'docker >= 2.5.1, < 3.0', 'dockerpty >= 0.4.1, < 0.5', 'six >= 1.3.0, < 2', 'jsonschema >= 2.5.1, < 3', From 5f84c0c27afe7ac93e50c89de329748a88a7560a Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Wed, 23 Aug 2017 17:45:56 -0700 Subject: [PATCH 111/244] Rename 3.4 schema to 3.4-beta Signed-off-by: Joffrey F --- ...{config_schema_v3.4.json => config_schema_v3.4-beta.json} | 2 +- compose/const.py | 2 +- docker-compose.spec | 5 +++++ 3 files changed, 7 insertions(+), 2 deletions(-) rename compose/config/{config_schema_v3.4.json => config_schema_v3.4-beta.json} (99%) diff --git a/compose/config/config_schema_v3.4.json b/compose/config/config_schema_v3.4-beta.json similarity index 99% rename from compose/config/config_schema_v3.4.json rename to compose/config/config_schema_v3.4-beta.json index 5a110a888..190c05f2c 100644 --- a/compose/config/config_schema_v3.4.json +++ b/compose/config/config_schema_v3.4-beta.json @@ -1,6 +1,6 @@ { "$schema": "http://json-schema.org/draft-04/schema#", - "id": "config_schema_v3.4.json", + "id": "config_schema_v3.4-beta.json", "type": "object", "required": ["version"], diff --git a/compose/const.py b/compose/const.py index b5970f82a..809f7c7d4 100644 --- a/compose/const.py +++ b/compose/const.py @@ -31,7 +31,7 @@ COMPOSEFILE_V3_0 = ComposeVersion('3.0') COMPOSEFILE_V3_1 = ComposeVersion('3.1') COMPOSEFILE_V3_2 = ComposeVersion('3.2') COMPOSEFILE_V3_3 = ComposeVersion('3.3') -COMPOSEFILE_V3_4 = ComposeVersion('3.4') +COMPOSEFILE_V3_4 = ComposeVersion('3.4-beta') API_VERSIONS = { COMPOSEFILE_V1: '1.21', diff --git a/docker-compose.spec b/docker-compose.spec index 8dc70c226..fe5651f6a 100644 --- a/docker-compose.spec +++ b/docker-compose.spec @@ -62,6 +62,11 @@ exe = EXE(pyz, 'compose/config/config_schema_v3.3.json', 'DATA' ), + ( + 'compose/config/config_schema_v3.4-beta.json', + 'compose/config/config_schema_v3.4-beta.json', + 'DATA' + ), ( 'compose/GITSHA', 'compose/GITSHA', From 177499b6de4b7f5d0bf99549dc3a3f0302ec359d Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Thu, 24 Aug 2017 12:21:31 -0700 Subject: [PATCH 112/244] Account for repo tag values that may contain a port Signed-off-by: Joffrey F --- compose/cli/main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/compose/cli/main.py b/compose/cli/main.py index 2bb53f95e..c07de53f2 100644 --- a/compose/cli/main.py +++ b/compose/cli/main.py @@ -506,7 +506,7 @@ class TopLevelCommand(object): rows = [] for container in containers: image_config = container.image_config - repo_tags = image_config['RepoTags'][0].split(':') + repo_tags = image_config['RepoTags'][0].rsplit(':', 1) image_id = image_config['Id'].split(':')[1][:12] size = human_readable_file_size(image_config['Size']) rows.append([ From c8193821ed7f6420d45da01e30c5c5d76402c3bd Mon Sep 17 00:00:00 2001 From: Cecile Tonglet Date: Tue, 22 Aug 2017 15:37:32 +0200 Subject: [PATCH 113/244] Actually test there is no control characters Signed-off-by: Cecile Tonglet --- tests/unit/parallel_test.py | 26 ++++++++++++++++++++------ 1 file changed, 20 insertions(+), 6 deletions(-) diff --git a/tests/unit/parallel_test.py b/tests/unit/parallel_test.py index f82858eab..3a60f01a6 100644 --- a/tests/unit/parallel_test.py +++ b/tests/unit/parallel_test.py @@ -133,17 +133,31 @@ def test_parallel_execute_alignment(capsys): assert a.index('...') == b.index('...') -def test_parallel_execute_alignment_noansi(capsys): - ParallelStreamWriter.set_noansi() +def test_parallel_execute_ansi(capsys): + ParallelStreamWriter.set_noansi(value=False) results, errors = parallel_execute( - objects=["short", "a very long name"], + objects=["something", "something more"], func=lambda x: x, get_name=six.text_type, - msg="Aligning", + msg="Control characters", ) assert errors == {} _, err = capsys.readouterr() - a, b, c, d = err.split('\n')[:4] - assert a.index('...') == b.index('...') == c.index('...') == d.index('...') + assert "\x1b" in err + + +def test_parallel_execute_noansi(capsys): + ParallelStreamWriter.set_noansi() + results, errors = parallel_execute( + objects=["something", "something more"], + func=lambda x: x, + get_name=six.text_type, + msg="Control characters", + ) + + assert errors == {} + + _, err = capsys.readouterr() + assert "\x1b" not in err From fb531ceaa3e50dcb684c5ba4e5332c9d71b519d9 Mon Sep 17 00:00:00 2001 From: Cecile Tonglet Date: Tue, 22 Aug 2017 15:46:55 +0200 Subject: [PATCH 114/244] Fix --no-ansi flag not working properly Signed-off-by: Cecile Tonglet --- compose/parallel.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/compose/parallel.py b/compose/parallel.py index 1cf1fb094..d455711dd 100644 --- a/compose/parallel.py +++ b/compose/parallel.py @@ -49,16 +49,16 @@ def parallel_execute(objects, func, get_name, msg, get_deps=None, limit=None): for obj, result, exception in events: if exception is None: - writer.write(get_name(obj), green('done')) + writer.write(get_name(obj), 'done', green) results.append(result) elif isinstance(exception, APIError): errors[get_name(obj)] = exception.explanation - writer.write(get_name(obj), red('error')) + writer.write(get_name(obj), 'error', red) elif isinstance(exception, (OperationFailedError, HealthCheckFailed, NoHealthCheckConfigured)): errors[get_name(obj)] = exception.msg - writer.write(get_name(obj), red('error')) + writer.write(get_name(obj), 'error', red) elif isinstance(exception, UpstreamError): - writer.write(get_name(obj), red('error')) + writer.write(get_name(obj), 'error', red) else: errors[get_name(obj)] = exception error_to_reraise = exception @@ -263,13 +263,13 @@ class ParallelStreamWriter(object): status, width=self.width)) self.stream.flush() - def write(self, obj_index, status): + def write(self, obj_index, status, color_func): if self.msg is None: return if self.noansi: self._write_noansi(obj_index, status) else: - self._write_ansi(obj_index, status) + self._write_ansi(obj_index, color_func(status)) def parallel_operation(containers, operation, options, message): From c49837fae0fa82fe43c4fa855ec0737455096d44 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Thu, 24 Aug 2017 12:46:22 -0700 Subject: [PATCH 115/244] Remove all colors in output when --no-ansi is set Signed-off-by: Joffrey F --- compose/cli/main.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/compose/cli/main.py b/compose/cli/main.py index c07de53f2..83bc7d58c 100644 --- a/compose/cli/main.py +++ b/compose/cli/main.py @@ -97,8 +97,10 @@ def dispatch(): {'options_first': True, 'version': get_version_info('compose')}) options, handler, command_options = dispatcher.parse(sys.argv[1:]) - setup_console_handler(console_handler, options.get('--verbose')) + setup_console_handler(console_handler, options.get('--verbose'), options.get('--no-ansi')) setup_parallel_logger(options.get('--no-ansi')) + if options.get('--no-ansi'): + command_options['--no-color'] = True return functools.partial(perform_command, options, handler, command_options) @@ -134,8 +136,8 @@ def setup_parallel_logger(noansi): compose.parallel.ParallelStreamWriter.set_noansi() -def setup_console_handler(handler, verbose): - if handler.stream.isatty(): +def setup_console_handler(handler, verbose, noansi=False): + if handler.stream.isatty() and noansi is False: format_class = ConsoleWarningFormatter else: format_class = logging.Formatter From e62c4033260a07a7b4b4a5eb6fadcbc1b14bf252 Mon Sep 17 00:00:00 2001 From: Harald Albers Date: Fri, 25 Aug 2017 09:42:37 +0200 Subject: [PATCH 116/244] Add bash completion for `--no-ansi` Signed-off-by: Harald Albers --- contrib/completion/bash/docker-compose | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/contrib/completion/bash/docker-compose b/contrib/completion/bash/docker-compose index d283a041a..33c2e2e53 100644 --- a/contrib/completion/bash/docker-compose +++ b/contrib/completion/bash/docker-compose @@ -179,7 +179,7 @@ _docker_compose_docker_compose() { case "$cur" in -*) - COMPREPLY=( $( compgen -W "$top_level_boolean_options $top_level_options_with_args --help -h --verbose --version -v" -- "$cur" ) ) + COMPREPLY=( $( compgen -W "$top_level_boolean_options $top_level_options_with_args --help -h --no-ansi --verbose --version -v" -- "$cur" ) ) ;; *) COMPREPLY=( $( compgen -W "${commands[*]}" -- "$cur" ) ) @@ -569,8 +569,10 @@ _docker_compose() { version ) - # options for the docker daemon that have to be passed to secondary calls to - # docker-compose executed by this script + # Options for the docker daemon that have to be passed to secondary calls to + # docker-compose executed by this script. + # Other global otions that are not relevant for secondary calls are defined in + # `_docker_compose_docker_compose`. local top_level_boolean_options=" --skip-hostname-check --tls From b28bcd613a02aae376a0f16e3d7a457a91347734 Mon Sep 17 00:00:00 2001 From: Harald Albers Date: Fri, 25 Aug 2017 09:59:39 +0200 Subject: [PATCH 117/244] Add bash completion for `create --build` Signed-off-by: Harald Albers --- contrib/completion/bash/docker-compose | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contrib/completion/bash/docker-compose b/contrib/completion/bash/docker-compose index 33c2e2e53..9de156403 100644 --- a/contrib/completion/bash/docker-compose +++ b/contrib/completion/bash/docker-compose @@ -149,7 +149,7 @@ _docker_compose_config() { _docker_compose_create() { case "$cur" in -*) - COMPREPLY=( $( compgen -W "--force-recreate --help --no-build --no-recreate" -- "$cur" ) ) + COMPREPLY=( $( compgen -W "--build --force-recreate --help --no-build --no-recreate" -- "$cur" ) ) ;; *) __docker_compose_services_all From abdeed7bb6ab4e384a9a93fa7020a5258c704ce6 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Fri, 25 Aug 2017 15:39:23 -0700 Subject: [PATCH 118/244] Handle unicode errors in LogPrinter Signed-off-by: Joffrey F --- compose/cli/log_printer.py | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/compose/cli/log_printer.py b/compose/cli/log_printer.py index 043d3d068..60bba8da6 100644 --- a/compose/cli/log_printer.py +++ b/compose/cli/log_printer.py @@ -102,8 +102,18 @@ class LogPrinter(object): # active containers to tail, so continue continue + self.write(line) + + def write(self, line): + try: self.output.write(line) - self.output.flush() + except UnicodeEncodeError: + # This may happen if the user's locale settings don't support UTF-8 + # and UTF-8 characters are present in the log line. The following + # will output a "degraded" log with unsupported characters + # replaced by `?` + self.output.write(line.encode('ascii', 'replace').decode()) + self.output.flush() def remove_stopped_threads(thread_map): From 5cc23c540c95e40d137b77e2aa419ce62c1f17c5 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Fri, 25 Aug 2017 16:06:42 -0700 Subject: [PATCH 119/244] Bump 1.16.0-rc2 Signed-off-by: Joffrey F --- CHANGELOG.md | 12 ++++++------ compose/__init__.py | 2 +- script/run/run.sh | 2 +- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0790f6184..3a92bf10a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,18 +6,12 @@ Change log ### New features -#### Compose file version 3.4 - -- Introduced version 3.4 of the `docker-compose.yml` specification. - This version requires to be used with Docker Engine 17.06.0 or above. - #### Compose file version 2.3 - Introduced version 2.3 of the `docker-compose.yml` specification. This version requires to be used with Docker Engine 17.06.0 or above. - Added support for the `target` parameter in network configurations - (also available in 3.4) - Added support for the `start_period` parameter in healthcheck configurations @@ -44,6 +38,9 @@ Change log - Fixed issues where logs of TTY-enabled services were being printed incorrectly and causing `MemoryError` exceptions +- Fixed a bug where printing application logs would sometimes be interrupted + by a `UnicodeEncodeError` exception on Python 3 + - The `$` character in the output of `docker-compose config` is now properly escaped @@ -58,6 +55,9 @@ Change log - Fixed an issue where the `logging` options in the output of `docker-compose config` would be set to `null`, an invalid value +- Fixed the output of the `docker-compose images` command when an image + would come from a private repository using an explicit port number + - Fixed the output of `docker-compose config` when a port definition used `0` as the value for the published port diff --git a/compose/__init__.py b/compose/__init__.py index b090ccfa2..4e2793b1a 100644 --- a/compose/__init__.py +++ b/compose/__init__.py @@ -1,4 +1,4 @@ from __future__ import absolute_import from __future__ import unicode_literals -__version__ = '1.16.0-rc1' +__version__ = '1.16.0-rc2' diff --git a/script/run/run.sh b/script/run/run.sh index bf9a26cb8..0f0d3d9d9 100755 --- a/script/run/run.sh +++ b/script/run/run.sh @@ -15,7 +15,7 @@ set -e -VERSION="1.16.0" +VERSION="1.16.0-rc2" IMAGE="docker/compose:$VERSION" From 07d5042859d9b23613175ac21a1961b07d9ccc65 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Wed, 30 Aug 2017 15:52:29 -0700 Subject: [PATCH 120/244] Bump 1.16.0 Signed-off-by: Joffrey F --- compose/__init__.py | 2 +- script/run/run.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/compose/__init__.py b/compose/__init__.py index 4e2793b1a..02c325eb8 100644 --- a/compose/__init__.py +++ b/compose/__init__.py @@ -1,4 +1,4 @@ from __future__ import absolute_import from __future__ import unicode_literals -__version__ = '1.16.0-rc2' +__version__ = '1.16.0' diff --git a/script/run/run.sh b/script/run/run.sh index 0f0d3d9d9..bf9a26cb8 100755 --- a/script/run/run.sh +++ b/script/run/run.sh @@ -15,7 +15,7 @@ set -e -VERSION="1.16.0-rc2" +VERSION="1.16.0" IMAGE="docker/compose:$VERSION" From 241931f77605ff2c29de248ec0b7ef00e032d6f3 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Thu, 31 Aug 2017 15:37:33 -0700 Subject: [PATCH 121/244] Merge extra_hosts instead of overwrite Signed-off-by: Joffrey F --- compose/config/config.py | 4 +--- tests/unit/config/config_test.py | 18 ++++++++++++++++++ 2 files changed, 19 insertions(+), 3 deletions(-) diff --git a/compose/config/config.py b/compose/config/config.py index 0c2ab1ab7..0fddfd3a4 100644 --- a/compose/config/config.py +++ b/compose/config/config.py @@ -717,9 +717,6 @@ def process_service(service_config): if 'labels' in service_dict: service_dict['labels'] = parse_labels(service_dict['labels']) - if 'extra_hosts' in service_dict: - service_dict['extra_hosts'] = parse_extra_hosts(service_dict['extra_hosts']) - if 'sysctls' in service_dict: service_dict['sysctls'] = build_string_dict(parse_sysctls(service_dict['sysctls'])) @@ -947,6 +944,7 @@ def merge_service_dicts(base, override, version): md.merge_sequence('secrets', types.ServiceSecret.parse) md.merge_sequence('configs', types.ServiceConfig.parse) md.merge_mapping('deploy', parse_deploy) + md.merge_mapping('extra_hosts', parse_extra_hosts) for field in ['volumes', 'devices']: md.merge_field(field, merge_path_mappings) diff --git a/tests/unit/config/config_test.py b/tests/unit/config/config_test.py index 4e355d3bf..644290157 100644 --- a/tests/unit/config/config_test.py +++ b/tests/unit/config/config_test.py @@ -2197,6 +2197,24 @@ class ConfigTest(unittest.TestCase): } } + def test_merge_extra_hosts(self): + base = { + 'image': 'bar', + 'extra_hosts': { + 'foo': '1.2.3.4', + } + } + + override = { + 'extra_hosts': ['bar:5.6.7.8', 'foo:127.0.0.1'] + } + + actual = config.merge_service_dicts(base, override, V2_0) + assert actual['extra_hosts'] == { + 'foo': '127.0.0.1', + 'bar': '5.6.7.8', + } + def test_merge_healthcheck_config(self): base = { 'image': 'bar', From 4900f099916f31047cdc8b35b8432babe42f87ef Mon Sep 17 00:00:00 2001 From: Andrew Hsu Date: Fri, 1 Sep 2017 13:11:10 -0700 Subject: [PATCH 122/244] Bump 1.16.1 Signed-off-by: Andrew Hsu --- CHANGELOG.md | 8 ++++++++ compose/__init__.py | 2 +- script/run/run.sh | 2 +- 3 files changed, 10 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3a92bf10a..558376855 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,14 @@ Change log ========== +1.16.1 (2017-09-01) +------------------- + +### Bugfixes + +- Fixed bug that prevented using `extra_hosts` in several configuration files. + + 1.16.0 (2017-08-31) ------------------- diff --git a/compose/__init__.py b/compose/__init__.py index 02c325eb8..2e41ca896 100644 --- a/compose/__init__.py +++ b/compose/__init__.py @@ -1,4 +1,4 @@ from __future__ import absolute_import from __future__ import unicode_literals -__version__ = '1.16.0' +__version__ = '1.16.1' diff --git a/script/run/run.sh b/script/run/run.sh index bf9a26cb8..f1754d05a 100755 --- a/script/run/run.sh +++ b/script/run/run.sh @@ -15,7 +15,7 @@ set -e -VERSION="1.16.0" +VERSION="1.16.1" IMAGE="docker/compose:$VERSION" From f64b48f0deb7e684d92dc82f752ff7edf47a6e02 Mon Sep 17 00:00:00 2001 From: Alexey Rokhin Date: Wed, 17 May 2017 15:21:35 +0300 Subject: [PATCH 123/244] Fix testcases.py formatting Signed-off-by: Alexey Rokhin --- tests/integration/testcases.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/integration/testcases.py b/tests/integration/testcases.py index b72fb53a8..8435f97dd 100644 --- a/tests/integration/testcases.py +++ b/tests/integration/testcases.py @@ -75,6 +75,7 @@ def v2_1_only(): return min_version_skip(V2_1) + def v2_2_only(): return min_version_skip(V2_2) @@ -83,6 +84,7 @@ def v2_3_only(): return min_version_skip(V2_3) + def v3_only(): return min_version_skip(V3_0) From 390821e31c1425546892ca58667c11880fb3e66d Mon Sep 17 00:00:00 2001 From: Alexey Rokhin Date: Wed, 17 May 2017 16:18:28 +0300 Subject: [PATCH 124/244] skip cpu_percent test for Linux Signed-off-by: Alexey Rokhin --- tests/integration/service_test.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/integration/service_test.py b/tests/integration/service_test.py index 84b54fe41..28cca4aad 100644 --- a/tests/integration/service_test.py +++ b/tests/integration/service_test.py @@ -28,6 +28,7 @@ from compose.const import LABEL_ONE_OFF from compose.const import LABEL_PROJECT from compose.const import LABEL_SERVICE from compose.const import LABEL_VERSION +from compose.const import IS_WINDOWS_PLATFORM from compose.container import Container from compose.errors import OperationFailedError from compose.project import OneOffFilter From 0367995b8fc7d8bf0e5bddaa6ec1f1a51d918dd6 Mon Sep 17 00:00:00 2001 From: Alexey Rokhin Date: Wed, 17 May 2017 16:42:43 +0300 Subject: [PATCH 125/244] service_test.py reorder imports Signed-off-by: Alexey Rokhin --- tests/integration/service_test.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/integration/service_test.py b/tests/integration/service_test.py index 28cca4aad..84b54fe41 100644 --- a/tests/integration/service_test.py +++ b/tests/integration/service_test.py @@ -28,7 +28,6 @@ from compose.const import LABEL_ONE_OFF from compose.const import LABEL_PROJECT from compose.const import LABEL_SERVICE from compose.const import LABEL_VERSION -from compose.const import IS_WINDOWS_PLATFORM from compose.container import Container from compose.errors import OperationFailedError from compose.project import OneOffFilter From 2eed6939f19b6f7675c907c86ed9277013265e19 Mon Sep 17 00:00:00 2001 From: Joel Barciauskas Date: Wed, 12 Apr 2017 17:45:09 -0400 Subject: [PATCH 126/244] Add --quiet parameter to docker-compose pull, using existing silent flag Signed-off-by: Joel Barciauskas --- compose/project.py | 2 +- tests/acceptance/cli_test.py | 4 ++++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/compose/project.py b/compose/project.py index 2310a2fcc..1faf97d40 100644 --- a/compose/project.py +++ b/compose/project.py @@ -496,7 +496,7 @@ class Project(object): if parallel_pull: def pull_service(service): - service.pull(ignore_pull_failures, True) + service.pull(ignore_pull_failures, True, silent=silent) _, errors = parallel.parallel_execute( services, diff --git a/tests/acceptance/cli_test.py b/tests/acceptance/cli_test.py index 78d1c1eb1..e721b940f 100644 --- a/tests/acceptance/cli_test.py +++ b/tests/acceptance/cli_test.py @@ -463,6 +463,10 @@ class CLITestCase(DockerClientTestCase): re.compile('''^(ERROR: )?(b')?.* nonexisting-image''', re.MULTILINE)) + def test_pull_with_quiet(self): + assert self.dispatch(['pull', '--quiet']).stderr == '' + assert self.dispatch(['pull', '--quiet']).stdout == '' + def test_build_plain(self): self.base_dir = 'tests/fixtures/simple-dockerfile' self.dispatch(['build', 'simple']) From d960f5151efb738acb71d4ea196348d12d7707fa Mon Sep 17 00:00:00 2001 From: NikitaVlaznev Date: Mon, 19 Jun 2017 17:05:19 +0300 Subject: [PATCH 127/244] Fix double silent argument value Fix for "TypeError: pull() got multiple values for keyword argument 'silent'." This change https://github.com/docker/compose/commit/e9b6cc23fcf01d4768c7e082b7bc91b43ff84e7e caused additional value to be passed for the 'silent' argument, that was already passed there: https://github.com/docker/compose/commit/f85da99ef3273794e855afda8678174419d3bf4f Signed-off-by: Nikita Vlaznev --- compose/project.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/compose/project.py b/compose/project.py index 1faf97d40..59d58f2f6 100644 --- a/compose/project.py +++ b/compose/project.py @@ -496,7 +496,7 @@ class Project(object): if parallel_pull: def pull_service(service): - service.pull(ignore_pull_failures, True, silent=silent) + service.pull(ignore_pull_failures, silent=silent) _, errors = parallel.parallel_execute( services, From 4bd2aa3d74c12625b06d1e403e1a1900eff7d11c Mon Sep 17 00:00:00 2001 From: Evan Shaw Date: Sat, 1 Jul 2017 13:40:02 +1200 Subject: [PATCH 128/244] Always silence pull output with --parallel This is how things were prior to the addition of the --quiet flag. Making it not silent produces output that's weird and difficult to read. Signed-off-by: Evan Shaw --- compose/project.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/compose/project.py b/compose/project.py index 59d58f2f6..2310a2fcc 100644 --- a/compose/project.py +++ b/compose/project.py @@ -496,7 +496,7 @@ class Project(object): if parallel_pull: def pull_service(service): - service.pull(ignore_pull_failures, silent=silent) + service.pull(ignore_pull_failures, True) _, errors = parallel.parallel_execute( services, From 376389d7a5fdc29da56e79125728a6b4b071df67 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Mon, 28 Aug 2017 17:35:55 -0700 Subject: [PATCH 129/244] Add --no-start flag to up command. Deprecate create command. Signed-off-by: Joffrey F --- compose/cli/main.py | 18 ++++++++++++++++-- compose/project.py | 6 ++++-- tests/acceptance/cli_test.py | 25 +++++++++++++++++++++++++ 3 files changed, 45 insertions(+), 4 deletions(-) diff --git a/compose/cli/main.py b/compose/cli/main.py index 83bc7d58c..21bf1f308 100644 --- a/compose/cli/main.py +++ b/compose/cli/main.py @@ -319,6 +319,7 @@ class TopLevelCommand(object): def create(self, options): """ Creates containers for a service. + This command is deprecated. Use the `up` command with `--no-start` instead. Usage: create [options] [SERVICE...] @@ -332,6 +333,11 @@ class TopLevelCommand(object): """ service_names = options['SERVICE'] + log.warn( + 'The create command is deprecated. ' + 'Use the up command with the --no-start flag instead.' + ) + self.project.create( service_names=service_names, strategy=convergence_strategy_from_opts(options), @@ -902,6 +908,7 @@ class TopLevelCommand(object): --no-recreate If containers already exist, don't recreate them. Incompatible with --force-recreate. --no-build Don't build an image, even if it's missing. + --no-start Don't start the services after creating them. --build Build images before starting containers. --abort-on-container-exit Stops all containers if any container was stopped. Incompatible with -d. @@ -922,10 +929,16 @@ class TopLevelCommand(object): timeout = timeout_from_opts(options) remove_orphans = options['--remove-orphans'] detached = options.get('-d') + no_start = options.get('--no-start') - if detached and cascade_stop: + if detached and (cascade_stop or exit_value_from): raise UserError("--abort-on-container-exit and -d cannot be combined.") + if no_start: + for excluded in ['-d', '--abort-on-container-exit', '--exit-code-from']: + if options.get(excluded): + raise UserError('--no-start and {} cannot be combined.'.format(excluded)) + with up_shutdown_context(self.project, service_names, timeout, detached): to_attach = self.project.up( service_names=service_names, @@ -936,9 +949,10 @@ class TopLevelCommand(object): detached=detached, remove_orphans=remove_orphans, scale_override=parse_scale_args(options['--scale']), + start=not no_start ) - if detached: + if detached or no_start: return attached_containers = filter_containers_to_service_names(to_attach, service_names) diff --git a/compose/project.py b/compose/project.py index 2310a2fcc..c8b57edd2 100644 --- a/compose/project.py +++ b/compose/project.py @@ -412,7 +412,8 @@ class Project(object): detached=False, remove_orphans=False, scale_override=None, - rescale=True): + rescale=True, + start=True): warn_for_swarm_mode(self.client) @@ -436,7 +437,8 @@ class Project(object): timeout=timeout, detached=detached, scale_override=scale_override.get(service.name), - rescale=rescale + rescale=rescale, + start=start ) def get_deps(service): diff --git a/tests/acceptance/cli_test.py b/tests/acceptance/cli_test.py index e721b940f..3a5e17ad8 100644 --- a/tests/acceptance/cli_test.py +++ b/tests/acceptance/cli_test.py @@ -776,6 +776,31 @@ class CLITestCase(DockerClientTestCase): for service in services: assert self.lookup(container, service.name) + @v2_only() + def test_up_no_start(self): + self.base_dir = 'tests/fixtures/v2-full' + self.dispatch(['up', '--no-start'], None) + + services = self.project.get_services() + + default_network = self.project.networks.networks['default'].full_name + front_network = self.project.networks.networks['front'].full_name + networks = self.client.networks(names=[default_network, front_network]) + assert len(networks) == 2 + + for service in services: + containers = service.containers(stopped=True) + assert len(containers) == 1 + + container = containers[0] + assert not container.is_running + assert container.get('State.Status') == 'created' + + volumes = self.project.volumes.volumes + assert 'data' in volumes + volume = volumes['data'] + assert volume.exists() + @v2_only() def test_up_no_ansi(self): self.base_dir = 'tests/fixtures/v2-simple' From 3cb22fa94a9747f9aa42d689cb35c555638ee7c9 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Mon, 28 Aug 2017 18:18:31 -0700 Subject: [PATCH 130/244] Reduce up() cyclomatic complexity Signed-off-by: Joffrey F --- compose/cli/main.py | 62 +++++++++++++++++++++++++-------------------- 1 file changed, 35 insertions(+), 27 deletions(-) diff --git a/compose/cli/main.py b/compose/cli/main.py index 21bf1f308..face38e6d 100644 --- a/compose/cli/main.py +++ b/compose/cli/main.py @@ -969,33 +969,10 @@ class TopLevelCommand(object): if cascade_stop: print("Aborting on container exit...") - - exit_code = 0 - if exit_value_from: - candidates = list(filter( - lambda c: c.service == exit_value_from, - attached_containers)) - if not candidates: - log.error( - 'No containers matching the spec "{0}" ' - 'were run.'.format(exit_value_from) - ) - exit_code = 2 - elif len(candidates) > 1: - exit_values = filter( - lambda e: e != 0, - [c.inspect()['State']['ExitCode'] for c in candidates] - ) - - exit_code = exit_values[0] - else: - exit_code = candidates[0].inspect()['State']['ExitCode'] - else: - for e in self.project.containers(service_names=options['SERVICE'], stopped=True): - if (not e.is_running and cascade_starter == e.name): - if not e.exit_code == 0: - exit_code = e.exit_code - break + all_containers = self.project.containers(service_names=options['SERVICE'], stopped=True) + exit_code = compute_exit_code( + exit_value_from, attached_containers, cascade_starter, all_containers + ) self.project.stop(service_names=service_names, timeout=timeout) sys.exit(exit_code) @@ -1016,6 +993,37 @@ class TopLevelCommand(object): print(get_version_info('full')) +def compute_exit_code(exit_value_from, attached_containers, cascade_starter, all_containers): + exit_code = 0 + if exit_value_from: + candidates = list(filter( + lambda c: c.service == exit_value_from, + attached_containers)) + if not candidates: + log.error( + 'No containers matching the spec "{0}" ' + 'were run.'.format(exit_value_from) + ) + exit_code = 2 + elif len(candidates) > 1: + exit_values = filter( + lambda e: e != 0, + [c.inspect()['State']['ExitCode'] for c in candidates] + ) + + exit_code = exit_values[0] + else: + exit_code = candidates[0].inspect()['State']['ExitCode'] + else: + for e in all_containers: + if (not e.is_running and cascade_starter == e.name): + if not e.exit_code == 0: + exit_code = e.exit_code + break + + return exit_code + + def convergence_strategy_from_opts(options): no_recreate = options['--no-recreate'] force_recreate = options['--force-recreate'] From 2f61a1dac4abcc22c93d985c73595770ded8c3db Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Fri, 25 Aug 2017 18:09:06 -0700 Subject: [PATCH 131/244] Add support for extension fields in v2.x and v3.4 Signed-off-by: Joffrey F --- compose/config/config_schema_v2.0.json | 1 + compose/config/config_schema_v2.1.json | 1 + compose/config/config_schema_v2.2.json | 1 + compose/config/config_schema_v2.3.json | 1 + compose/config/config_schema_v3.4-beta.json | 1 + compose/config/validation.py | 10 ++++++++++ tests/unit/config/config_test.py | 14 +++++++++++++- 7 files changed, 28 insertions(+), 1 deletion(-) diff --git a/compose/config/config_schema_v2.0.json b/compose/config/config_schema_v2.0.json index 14bafab40..2ad62ac52 100644 --- a/compose/config/config_schema_v2.0.json +++ b/compose/config/config_schema_v2.0.json @@ -41,6 +41,7 @@ } }, + "patternProperties": {"^x-": {}}, "additionalProperties": false, "definitions": { diff --git a/compose/config/config_schema_v2.1.json b/compose/config/config_schema_v2.1.json index 8a5e12834..24e6ba02c 100644 --- a/compose/config/config_schema_v2.1.json +++ b/compose/config/config_schema_v2.1.json @@ -41,6 +41,7 @@ } }, + "patternProperties": {"^x-": {}}, "additionalProperties": false, "definitions": { diff --git a/compose/config/config_schema_v2.2.json b/compose/config/config_schema_v2.2.json index 58ba409ff..86fc5df95 100644 --- a/compose/config/config_schema_v2.2.json +++ b/compose/config/config_schema_v2.2.json @@ -41,6 +41,7 @@ } }, + "patternProperties": {"^x-": {}}, "additionalProperties": false, "definitions": { diff --git a/compose/config/config_schema_v2.3.json b/compose/config/config_schema_v2.3.json index 7a9bdfdf1..a790bb405 100644 --- a/compose/config/config_schema_v2.3.json +++ b/compose/config/config_schema_v2.3.json @@ -41,6 +41,7 @@ } }, + "patternProperties": {"^x-": {}}, "additionalProperties": false, "definitions": { diff --git a/compose/config/config_schema_v3.4-beta.json b/compose/config/config_schema_v3.4-beta.json index 190c05f2c..cba063202 100644 --- a/compose/config/config_schema_v3.4-beta.json +++ b/compose/config/config_schema_v3.4-beta.json @@ -64,6 +64,7 @@ } }, + "patternProperties": {"^x-": {}}, "additionalProperties": false, "definitions": { diff --git a/compose/config/validation.py b/compose/config/validation.py index 0b7961e5a..c6722a14d 100644 --- a/compose/config/validation.py +++ b/compose/config/validation.py @@ -239,6 +239,16 @@ def handle_error_for_schema_with_id(error, path): invalid_config_key = parse_key_from_error_msg(error) return get_unsupported_config_msg(path, invalid_config_key) + if schema_id.startswith('config_schema_v'): + invalid_config_key = parse_key_from_error_msg(error) + return ('Invalid top-level property "{key}". Valid top-level ' + 'sections for this Compose file are: {properties}, and ' + 'extensions starting with "x-".\n\n{explanation}').format( + key=invalid_config_key, + properties=', '.join(error.schema['properties'].keys()), + explanation=VERSION_EXPLANATION + ) + if not error.path: return '{}\n\n{}'.format(error.message, VERSION_EXPLANATION) diff --git a/tests/unit/config/config_test.py b/tests/unit/config/config_test.py index 644290157..14dd01179 100644 --- a/tests/unit/config/config_test.py +++ b/tests/unit/config/config_test.py @@ -251,7 +251,7 @@ class ConfigTest(unittest.TestCase): ) ) - assert 'Additional properties are not allowed' in excinfo.exconly() + assert 'Invalid top-level property "web"' in excinfo.exconly() assert VERSION_EXPLANATION in excinfo.exconly() def test_named_volume_config_empty(self): @@ -773,6 +773,18 @@ class ConfigTest(unittest.TestCase): assert services[1]['name'] == 'db' assert services[2]['name'] == 'web' + def test_load_with_extensions(self): + config_details = build_config_details({ + 'version': '2.3', + 'x-data': { + 'lambda': 3, + 'excess': [True, {}] + } + }) + + config_data = config.load(config_details) + assert config_data.services == [] + def test_config_build_configuration(self): service = config.load( build_config_details( From eab333adb1ca12672bd9a5eb93bbe6b24614862b Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Fri, 1 Sep 2017 12:14:56 -0700 Subject: [PATCH 132/244] Update release process with most recent changes Signed-off-by: Joffrey F --- project/RELEASE-PROCESS.md | 33 +++++++++++++++++++++------------ script/release/make-branch | 3 +-- 2 files changed, 22 insertions(+), 14 deletions(-) diff --git a/project/RELEASE-PROCESS.md b/project/RELEASE-PROCESS.md index c1834f2fb..5b30545f4 100644 --- a/project/RELEASE-PROCESS.md +++ b/project/RELEASE-PROCESS.md @@ -24,7 +24,7 @@ As part of this script you'll be asked to: If the next release will be an RC, append `-rcN`, e.g. `1.4.0-rc1`. -2. Write release notes in `CHANGES.md`. +2. Write release notes in `CHANGELOG.md`. Almost every feature enhancement should be mentioned, with the most visible/exciting ones first. Use descriptive sentences and give context @@ -67,16 +67,13 @@ Check out the bump branch and run the `build-binaries` script When prompted build the non-linux binaries and test them. -1. Download the osx binary from Bintray. Make sure that the latest Travis - build has finished, otherwise you'll be downloading an old binary. +1. Download the different platform binaries by running the following script: - https://dl.bintray.com/docker-compose/$BRANCH_NAME/ + `./script/release/download-binaries $VERSION` -2. Download the windows binary from AppVeyor + The binaries for Linux, OSX and Windows will be downloaded in the `binaries-$VERSION` folder. - https://ci.appveyor.com/project/docker/compose - -3. Draft a release from the tag on GitHub (the script will open the window for +3. Draft a release from the tag on GitHub (the `build-binaries` script will open the window for you) The tag will only be present on Github when you run the `push-release` @@ -87,18 +84,30 @@ When prompted build the non-linux binaries and test them. If you're a Mac or Windows user, the best way to install Compose and keep it up-to-date is **[Docker for Mac and Windows](https://www.docker.com/products/docker)**. - Note that Compose 1.9.0 requires Docker Engine 1.10.0 or later for version 2 of the Compose File format, and Docker Engine 1.9.1 or later for version 1. Docker for Mac and Windows will automatically install the latest version of Docker Engine for you. + Docker for Mac and Windows will automatically install the latest version of Docker Engine for you. Alternatively, you can use the usual commands to install or upgrade Compose: ``` - curl -L https://github.com/docker/compose/releases/download/1.9.0/docker-compose-`uname -s`-`uname -m` > /usr/local/bin/docker-compose + curl -L https://github.com/docker/compose/releases/download/1.16.0/docker-compose-`uname -s`-`uname -m` > /usr/local/bin/docker-compose chmod +x /usr/local/bin/docker-compose ``` See the [install docs](https://docs.docker.com/compose/install/) for more install options and instructions. - Here's what's new: + ## Compose file format compatibility matrix + + | Compose file format | Docker Engine | + | --- | --- | + | 3.3 | 17.06.0+ | + | 3.0 – 3.2 | 1.13.0+ | + | 2.3| 17.06.0+ | + | 2.2 | 1.13.0+ | + | 2.1 | 1.12.0+ | + | 2.0 | 1.10.0+ | + | 1.0 | 1.9.1+ | + + ## Changes ...release notes go here... @@ -119,7 +128,7 @@ When prompted build the non-linux binaries and test them. 9. Check that all the binaries download (following the install instructions) and run. -10. Email maintainers@dockerproject.org and engineering@docker.com about the new release. +10. Announce the release on the appropriate Slack channel(s). ## If it’s a stable release (not an RC) diff --git a/script/release/make-branch b/script/release/make-branch index 7ccf3f055..b8a0cd31e 100755 --- a/script/release/make-branch +++ b/script/release/make-branch @@ -65,8 +65,7 @@ git config "branch.${BRANCH}.release" $VERSION editor=${EDITOR:-vim} -echo "Update versions in docs/install.md, compose/__init__.py, script/run/run.sh" -$editor docs/install.md +echo "Update versions in compose/__init__.py, script/run/run.sh" $editor compose/__init__.py $editor script/run/run.sh From 1610af7e9f833a317f0ecee5d52ccf038a250f95 Mon Sep 17 00:00:00 2001 From: Vincent Demeester Date: Tue, 19 Sep 2017 18:27:02 +0200 Subject: [PATCH 133/244] Sync composefile v3.2 schema with `docker/cli` Signed-off-by: Vincent Demeester --- compose/config/config_schema_v3.2.json | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/compose/config/config_schema_v3.2.json b/compose/config/config_schema_v3.2.json index b26b2c6c6..2ca8e92db 100644 --- a/compose/config/config_schema_v3.2.json +++ b/compose/config/config_schema_v3.2.json @@ -170,7 +170,8 @@ "type": "array", "items": { "oneOf": [ - {"type": ["string", "number"], "format": "ports"}, + {"type": "number", "format": "ports"}, + {"type": "string", "format": "ports"}, { "type": "object", "properties": { @@ -249,6 +250,7 @@ "source": {"type": "string"}, "target": {"type": "string"}, "read_only": {"type": "boolean"}, + "consistency": {"type": "string"}, "bind": { "type": "object", "properties": { From 49b1ac57c315ca8fad4d77253cc536333e51ac6d Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Tue, 26 Sep 2017 16:25:05 -0700 Subject: [PATCH 134/244] Fix oneOf validator parser to correctly process uniqueItems errors Signed-off-by: Joffrey F --- compose/config/validation.py | 15 +++++++-------- tests/unit/config/config_test.py | 25 ++++++++++++++++++++----- 2 files changed, 27 insertions(+), 13 deletions(-) diff --git a/compose/config/validation.py b/compose/config/validation.py index c6722a14d..940775a20 100644 --- a/compose/config/validation.py +++ b/compose/config/validation.py @@ -325,7 +325,6 @@ def _parse_oneof_validator(error): """ types = [] for context in error.context: - if context.validator == 'oneOf': _, error_msg = _parse_oneof_validator(context) return path_string(context.path), error_msg @@ -337,6 +336,13 @@ def _parse_oneof_validator(error): invalid_config_key = parse_key_from_error_msg(context) return (None, "contains unsupported option: '{}'".format(invalid_config_key)) + if context.validator == 'uniqueItems': + return ( + path_string(context.path) if context.path else None, + "contains non-unique items, please remove duplicates from {}".format( + context.instance), + ) + if context.path: return ( path_string(context.path), @@ -345,13 +351,6 @@ def _parse_oneof_validator(error): _parse_valid_types_from_validator(context.validator_value)), ) - if context.validator == 'uniqueItems': - return ( - None, - "contains non unique items, please remove duplicates from {}".format( - context.instance), - ) - if context.validator == 'type': types.append(context.validator_value) diff --git a/tests/unit/config/config_test.py b/tests/unit/config/config_test.py index 14dd01179..de9a61302 100644 --- a/tests/unit/config/config_test.py +++ b/tests/unit/config/config_test.py @@ -581,6 +581,20 @@ class ConfigTest(unittest.TestCase): assert 'Invalid service name \'mong\\o\'' in excinfo.exconly() + def test_config_duplicate_cache_from_values_validation_error(self): + with pytest.raises(ConfigurationError) as exc: + config.load( + build_config_details({ + 'version': '2.3', + 'services': { + 'test': {'build': {'context': '.', 'cache_from': ['a', 'b', 'a']}} + } + + }) + ) + + assert 'build.cache_from contains non-unique items' in exc.exconly() + def test_load_with_multiple_files_v1(self): base_file = config.ConfigFile( 'base.yaml', @@ -2751,11 +2765,12 @@ class PortsTest(unittest.TestCase): def check_config(self, cfg): config.load( - build_config_details( - {'web': dict(image='busybox', **cfg)}, - 'working_dir', - 'filename.yml' - ) + build_config_details({ + 'version': '2.3', + 'services': { + 'web': dict(image='busybox', **cfg) + }, + }, 'working_dir', 'filename.yml') ) From 4bc9d9dbafc9e66294977952d9662223dd99f95f Mon Sep 17 00:00:00 2001 From: French Ben Date: Mon, 18 Sep 2017 16:30:32 -0700 Subject: [PATCH 135/244] Simple patch to allow s390x images to be built Needs integration with CI and s390x machine integration Signed-off-by: French Ben --- Dockerfile.s390x | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100644 Dockerfile.s390x diff --git a/Dockerfile.s390x b/Dockerfile.s390x new file mode 100644 index 000000000..aa71e27bc --- /dev/null +++ b/Dockerfile.s390x @@ -0,0 +1,6 @@ +FROM s390x/python:3.6.2-slim +ARG COMPOSE_VERSION=1.16.1 + +RUN pip install --no-cache-dir docker-compose==$COMPOSE_VERSION + +ENTRYPOINT ["docker-compose"] From c5e871c5a55e873ae1931efec8ccedc649b522a7 Mon Sep 17 00:00:00 2001 From: French Ben Date: Tue, 19 Sep 2017 10:14:55 -0700 Subject: [PATCH 136/244] Use slim alpine instead of bulky debian Signed-off-by: French Ben --- Dockerfile.s390x | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/Dockerfile.s390x b/Dockerfile.s390x index aa71e27bc..3b19bb390 100644 --- a/Dockerfile.s390x +++ b/Dockerfile.s390x @@ -1,6 +1,15 @@ -FROM s390x/python:3.6.2-slim +FROM s390x/alpine:3.6 + ARG COMPOSE_VERSION=1.16.1 -RUN pip install --no-cache-dir docker-compose==$COMPOSE_VERSION +RUN apk add --update --no-cache \ + python \ + py-pip \ + && pip install --no-cache-dir docker-compose==$COMPOSE_VERSION \ + && rm -rf /var/cache/apk/* + +WORKDIR /data +VOLUME /data + ENTRYPOINT ["docker-compose"] From fa63e235202f1e8bd61a1f10d9e908f4cb6a29d3 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Tue, 26 Sep 2017 17:19:29 -0700 Subject: [PATCH 137/244] Revert 3.4-beta temp rename Signed-off-by: Joffrey F --- ...nfig_schema_v3.4-beta.json => config_schema_v3.4.json} | 8 +++++--- compose/const.py | 2 +- docker-compose.spec | 4 ++-- 3 files changed, 8 insertions(+), 6 deletions(-) rename compose/config/{config_schema_v3.4-beta.json => config_schema_v3.4.json} (98%) diff --git a/compose/config/config_schema_v3.4-beta.json b/compose/config/config_schema_v3.4.json similarity index 98% rename from compose/config/config_schema_v3.4-beta.json rename to compose/config/config_schema_v3.4.json index cba063202..dae7d7d23 100644 --- a/compose/config/config_schema_v3.4-beta.json +++ b/compose/config/config_schema_v3.4.json @@ -1,6 +1,7 @@ + { "$schema": "http://json-schema.org/draft-04/schema#", - "id": "config_schema_v3.4-beta.json", + "id": "config_schema_v3.4.json", "type": "object", "required": ["version"], @@ -316,7 +317,7 @@ "additionalProperties": false, "properties": { "disable": {"type": "boolean"}, - "interval": {"type": "string"}, + "interval": {"type": "string", "format": "duration"}, "retries": {"type": "number"}, "test": { "oneOf": [ @@ -324,7 +325,8 @@ {"type": "array", "items": {"type": "string"}} ] }, - "timeout": {"type": "string"} + "timeout": {"type": "string", "format": "duration"}, + "start_period": {"type": "string", "format": "duration"} } }, "deployment": { diff --git a/compose/const.py b/compose/const.py index 809f7c7d4..b5970f82a 100644 --- a/compose/const.py +++ b/compose/const.py @@ -31,7 +31,7 @@ COMPOSEFILE_V3_0 = ComposeVersion('3.0') COMPOSEFILE_V3_1 = ComposeVersion('3.1') COMPOSEFILE_V3_2 = ComposeVersion('3.2') COMPOSEFILE_V3_3 = ComposeVersion('3.3') -COMPOSEFILE_V3_4 = ComposeVersion('3.4-beta') +COMPOSEFILE_V3_4 = ComposeVersion('3.4') API_VERSIONS = { COMPOSEFILE_V1: '1.21', diff --git a/docker-compose.spec b/docker-compose.spec index fe5651f6a..9c46421f0 100644 --- a/docker-compose.spec +++ b/docker-compose.spec @@ -63,8 +63,8 @@ exe = EXE(pyz, 'DATA' ), ( - 'compose/config/config_schema_v3.4-beta.json', - 'compose/config/config_schema_v3.4-beta.json', + 'compose/config/config_schema_v3.4.json', + 'compose/config/config_schema_v3.4.json', 'DATA' ), ( From ce19f431583733a8e0d8b2aa066af1be0a163d52 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Wed, 27 Sep 2017 18:24:46 -0700 Subject: [PATCH 138/244] Avoid import ConfigurationError inside compose.utils (circular import) Signed-off-by: Joffrey F --- compose/config/config.py | 5 ++++- compose/utils.py | 3 +-- tests/unit/utils_test.py | 8 ++++++++ 3 files changed, 13 insertions(+), 3 deletions(-) diff --git a/compose/config/config.py b/compose/config/config.py index 0fddfd3a4..b90ab0305 100644 --- a/compose/config/config.py +++ b/compose/config/config.py @@ -762,7 +762,10 @@ def process_blkio_config(service_dict): for field in ['device_read_bps', 'device_write_bps']: if field in service_dict['blkio_config']: for v in service_dict['blkio_config'].get(field, []): - v['rate'] = parse_bytes(v.get('rate', 0)) + rate = v.get('rate', 0) + v['rate'] = parse_bytes(rate) + if v['rate'] is None: + raise ConfigurationError('Invalid format for bytes value: "{}"'.format(rate)) for field in ['device_read_iops', 'device_write_iops']: if field in service_dict['blkio_config']: diff --git a/compose/utils.py b/compose/utils.py index 1ede4d37d..197ae6eb2 100644 --- a/compose/utils.py +++ b/compose/utils.py @@ -12,7 +12,6 @@ import six from docker.errors import DockerException from docker.utils import parse_bytes as sdk_parse_bytes -from .config.errors import ConfigurationError from .errors import StreamParseError from .timeparse import MULTIPLIERS from .timeparse import timeparse @@ -143,4 +142,4 @@ def parse_bytes(n): try: return sdk_parse_bytes(n) except DockerException: - raise ConfigurationError('Invalid format for bytes value: {}'.format(n)) + return None diff --git a/tests/unit/utils_test.py b/tests/unit/utils_test.py index 85231957e..84becb975 100644 --- a/tests/unit/utils_test.py +++ b/tests/unit/utils_test.py @@ -60,3 +60,11 @@ class TestJsonStream(object): {'three': 'four'}, {'x': 2} ] + + +class TestParseBytes(object): + def test_parse_bytes(self): + assert utils.parse_bytes('123kb') == 123 * 1024 + assert utils.parse_bytes(123) == 123 + assert utils.parse_bytes('foobar') is None + assert utils.parse_bytes('123') == 123 From 38073bbd9fa8a461452db28b89f22617910c43ff Mon Sep 17 00:00:00 2001 From: Marc van den Hoogen Date: Fri, 18 Aug 2017 13:40:11 +0200 Subject: [PATCH 139/244] Add shm_size to build-options (issue #3866) * Add shm_size to build configuration * Make it possible to enlarge/customize shm size during build * Value in bytes, or use string like "512M" or "1G" ... * Add to compose format 2.3 and (provisionally) >=3.5 format * Add automated test for shm_size in build-opts Signed-off-by: Marc van den Hoogen Made unit tests compatible with previously added shm_size build-option Signed-off-by: Marc van den Hoogen Also support shm_size build-opt when conf override Signed-off-by: Marc van den Hoogen Automated test for shm_size build-option Signed-off-by: Marc van den Hoogen Schema 3.4, add shm_size to schema 2.3, updated const.py Signed-off-by: Marc van den Hoogen Corrected typo in config_schema_v3.4 Signed-off-by: Marc van den Hoogen Add support for g/m/k units for shm_size in build-opts Signed-off-by: Marc van den Hoogen Reorder imports in service.py Signed-off-by: Marc van den Hoogen --- compose/config/config.py | 1 + compose/config/config_schema_v2.3.json | 3 +- compose/config/config_schema_v3.5.json | 542 ++++++++++++++++++ compose/const.py | 3 + compose/service.py | 2 + tests/acceptance/cli_test.py | 6 + tests/fixtures/build-shm-size/Dockerfile | 4 + .../build-shm-size/docker-compose.yml | 7 + tests/unit/service_test.py | 2 + 9 files changed, 569 insertions(+), 1 deletion(-) create mode 100644 compose/config/config_schema_v3.5.json create mode 100644 tests/fixtures/build-shm-size/Dockerfile create mode 100644 tests/fixtures/build-shm-size/docker-compose.yml diff --git a/compose/config/config.py b/compose/config/config.py index b90ab0305..f16dd01b3 100644 --- a/compose/config/config.py +++ b/compose/config/config.py @@ -1020,6 +1020,7 @@ def merge_build(output, base, override): md.merge_scalar('dockerfile') md.merge_scalar('network') md.merge_scalar('target') + md.merge_scalar('shm_size') md.merge_mapping('args', parse_build_arguments) md.merge_field('cache_from', merge_unique_items_lists, default=[]) md.merge_mapping('labels', parse_labels) diff --git a/compose/config/config_schema_v2.3.json b/compose/config/config_schema_v2.3.json index a790bb405..ceaf44954 100644 --- a/compose/config/config_schema_v2.3.json +++ b/compose/config/config_schema_v2.3.json @@ -91,7 +91,8 @@ "labels": {"$ref": "#/definitions/list_or_dict"}, "cache_from": {"$ref": "#/definitions/list_of_strings"}, "network": {"type": "string"}, - "target": {"type": "string"} + "target": {"type": "string"}, + "shm_size": {"type": ["integer", "string"]} }, "additionalProperties": false } diff --git a/compose/config/config_schema_v3.5.json b/compose/config/config_schema_v3.5.json new file mode 100644 index 000000000..fa95d6a24 --- /dev/null +++ b/compose/config/config_schema_v3.5.json @@ -0,0 +1,542 @@ +{ + "$schema": "http://json-schema.org/draft-04/schema#", + "id": "config_schema_v3.5.json", + "type": "object", + "required": ["version"], + + "properties": { + "version": { + "type": "string" + }, + + "services": { + "id": "#/properties/services", + "type": "object", + "patternProperties": { + "^[a-zA-Z0-9._-]+$": { + "$ref": "#/definitions/service" + } + }, + "additionalProperties": false + }, + + "networks": { + "id": "#/properties/networks", + "type": "object", + "patternProperties": { + "^[a-zA-Z0-9._-]+$": { + "$ref": "#/definitions/network" + } + } + }, + + "volumes": { + "id": "#/properties/volumes", + "type": "object", + "patternProperties": { + "^[a-zA-Z0-9._-]+$": { + "$ref": "#/definitions/volume" + } + }, + "additionalProperties": false + }, + + "secrets": { + "id": "#/properties/secrets", + "type": "object", + "patternProperties": { + "^[a-zA-Z0-9._-]+$": { + "$ref": "#/definitions/secret" + } + }, + "additionalProperties": false + }, + + "configs": { + "id": "#/properties/configs", + "type": "object", + "patternProperties": { + "^[a-zA-Z0-9._-]+$": { + "$ref": "#/definitions/config" + } + }, + "additionalProperties": false + } + }, + + "additionalProperties": false, + + "definitions": { + + "service": { + "id": "#/definitions/service", + "type": "object", + + "properties": { + "deploy": {"$ref": "#/definitions/deployment"}, + "build": { + "oneOf": [ + {"type": "string"}, + { + "type": "object", + "properties": { + "context": {"type": "string"}, + "dockerfile": {"type": "string"}, + "args": {"$ref": "#/definitions/list_or_dict"}, + "labels": {"$ref": "#/definitions/list_or_dict"}, + "cache_from": {"$ref": "#/definitions/list_of_strings"}, + "network": {"type": "string"}, + "target": {"type": "string"}, + "shm_size": {"type": ["integer", "string"]} + }, + "additionalProperties": false + } + ] + }, + "cap_add": {"type": "array", "items": {"type": "string"}, "uniqueItems": true}, + "cap_drop": {"type": "array", "items": {"type": "string"}, "uniqueItems": true}, + "cgroup_parent": {"type": "string"}, + "command": { + "oneOf": [ + {"type": "string"}, + {"type": "array", "items": {"type": "string"}} + ] + }, + "configs": { + "type": "array", + "items": { + "oneOf": [ + {"type": "string"}, + { + "type": "object", + "properties": { + "source": {"type": "string"}, + "target": {"type": "string"}, + "uid": {"type": "string"}, + "gid": {"type": "string"}, + "mode": {"type": "number"} + } + } + ] + } + }, + "container_name": {"type": "string"}, + "credential_spec": {"type": "object", "properties": { + "file": {"type": "string"}, + "registry": {"type": "string"} + }}, + "depends_on": {"$ref": "#/definitions/list_of_strings"}, + "devices": {"type": "array", "items": {"type": "string"}, "uniqueItems": true}, + "dns": {"$ref": "#/definitions/string_or_list"}, + "dns_search": {"$ref": "#/definitions/string_or_list"}, + "domainname": {"type": "string"}, + "entrypoint": { + "oneOf": [ + {"type": "string"}, + {"type": "array", "items": {"type": "string"}} + ] + }, + "env_file": {"$ref": "#/definitions/string_or_list"}, + "environment": {"$ref": "#/definitions/list_or_dict"}, + + "expose": { + "type": "array", + "items": { + "type": ["string", "number"], + "format": "expose" + }, + "uniqueItems": true + }, + + "external_links": {"type": "array", "items": {"type": "string"}, "uniqueItems": true}, + "extra_hosts": {"$ref": "#/definitions/list_or_dict"}, + "healthcheck": {"$ref": "#/definitions/healthcheck"}, + "hostname": {"type": "string"}, + "image": {"type": "string"}, + "ipc": {"type": "string"}, + "labels": {"$ref": "#/definitions/list_or_dict"}, + "links": {"type": "array", "items": {"type": "string"}, "uniqueItems": true}, + + "logging": { + "type": "object", + + "properties": { + "driver": {"type": "string"}, + "options": { + "type": "object", + "patternProperties": { + "^.+$": {"type": ["string", "number", "null"]} + } + } + }, + "additionalProperties": false + }, + + "mac_address": {"type": "string"}, + "network_mode": {"type": "string"}, + + "networks": { + "oneOf": [ + {"$ref": "#/definitions/list_of_strings"}, + { + "type": "object", + "patternProperties": { + "^[a-zA-Z0-9._-]+$": { + "oneOf": [ + { + "type": "object", + "properties": { + "aliases": {"$ref": "#/definitions/list_of_strings"}, + "ipv4_address": {"type": "string"}, + "ipv6_address": {"type": "string"} + }, + "additionalProperties": false + }, + {"type": "null"} + ] + } + }, + "additionalProperties": false + } + ] + }, + "pid": {"type": ["string", "null"]}, + + "ports": { + "type": "array", + "items": { + "oneOf": [ + {"type": "number", "format": "ports"}, + {"type": "string", "format": "ports"}, + { + "type": "object", + "properties": { + "mode": {"type": "string"}, + "target": {"type": "integer"}, + "published": {"type": "integer"}, + "protocol": {"type": "string"} + }, + "additionalProperties": false + } + ] + }, + "uniqueItems": true + }, + + "privileged": {"type": "boolean"}, + "read_only": {"type": "boolean"}, + "restart": {"type": "string"}, + "security_opt": {"type": "array", "items": {"type": "string"}, "uniqueItems": true}, + "shm_size": {"type": ["number", "string"]}, + "secrets": { + "type": "array", + "items": { + "oneOf": [ + {"type": "string"}, + { + "type": "object", + "properties": { + "source": {"type": "string"}, + "target": {"type": "string"}, + "uid": {"type": "string"}, + "gid": {"type": "string"}, + "mode": {"type": "number"} + } + } + ] + } + }, + "sysctls": {"$ref": "#/definitions/list_or_dict"}, + "stdin_open": {"type": "boolean"}, + "stop_grace_period": {"type": "string", "format": "duration"}, + "stop_signal": {"type": "string"}, + "tmpfs": {"$ref": "#/definitions/string_or_list"}, + "tty": {"type": "boolean"}, + "ulimits": { + "type": "object", + "patternProperties": { + "^[a-z]+$": { + "oneOf": [ + {"type": "integer"}, + { + "type":"object", + "properties": { + "hard": {"type": "integer"}, + "soft": {"type": "integer"} + }, + "required": ["soft", "hard"], + "additionalProperties": false + } + ] + } + } + }, + "user": {"type": "string"}, + "userns_mode": {"type": "string"}, + "volumes": { + "type": "array", + "items": { + "oneOf": [ + {"type": "string"}, + { + "type": "object", + "required": ["type"], + "properties": { + "type": {"type": "string"}, + "source": {"type": "string"}, + "target": {"type": "string"}, + "read_only": {"type": "boolean"}, + "consistency": {"type": "string"}, + "bind": { + "type": "object", + "properties": { + "propagation": {"type": "string"} + } + }, + "volume": { + "type": "object", + "properties": { + "nocopy": {"type": "boolean"} + } + } + } + } + ], + "uniqueItems": true + } + }, + "working_dir": {"type": "string"} + }, + "additionalProperties": false + }, + + "healthcheck": { + "id": "#/definitions/healthcheck", + "type": "object", + "additionalProperties": false, + "properties": { + "disable": {"type": "boolean"}, + "interval": {"type": "string"}, + "retries": {"type": "number"}, + "test": { + "oneOf": [ + {"type": "string"}, + {"type": "array", "items": {"type": "string"}} + ] + }, + "timeout": {"type": "string"} + } + }, + "deployment": { + "id": "#/definitions/deployment", + "type": ["object", "null"], + "properties": { + "mode": {"type": "string"}, + "endpoint_mode": {"type": "string"}, + "replicas": {"type": "integer"}, + "labels": {"$ref": "#/definitions/list_or_dict"}, + "update_config": { + "type": "object", + "properties": { + "parallelism": {"type": "integer"}, + "delay": {"type": "string", "format": "duration"}, + "failure_action": {"type": "string"}, + "monitor": {"type": "string", "format": "duration"}, + "max_failure_ratio": {"type": "number"}, + "order": {"type": "string", "enum": [ + "start-first", "stop-first" + ]} + }, + "additionalProperties": false + }, + "resources": { + "type": "object", + "properties": { + "limits": {"$ref": "#/definitions/resource"}, + "reservations": {"$ref": "#/definitions/resource"} + }, + "additionalProperties": false + }, + "restart_policy": { + "type": "object", + "properties": { + "condition": {"type": "string"}, + "delay": {"type": "string", "format": "duration"}, + "max_attempts": {"type": "integer"}, + "window": {"type": "string", "format": "duration"} + }, + "additionalProperties": false + }, + "placement": { + "type": "object", + "properties": { + "constraints": {"type": "array", "items": {"type": "string"}}, + "preferences": { + "type": "array", + "items": { + "type": "object", + "properties": { + "spread": {"type": "string"} + }, + "additionalProperties": false + } + } + }, + "additionalProperties": false + } + }, + "additionalProperties": false + }, + + "resource": { + "id": "#/definitions/resource", + "type": "object", + "properties": { + "cpus": {"type": "string"}, + "memory": {"type": "string"} + }, + "additionalProperties": false + }, + + "network": { + "id": "#/definitions/network", + "type": ["object", "null"], + "properties": { + "driver": {"type": "string"}, + "driver_opts": { + "type": "object", + "patternProperties": { + "^.+$": {"type": ["string", "number"]} + } + }, + "ipam": { + "type": "object", + "properties": { + "driver": {"type": "string"}, + "config": { + "type": "array", + "items": { + "type": "object", + "properties": { + "subnet": {"type": "string"} + }, + "additionalProperties": false + } + } + }, + "additionalProperties": false + }, + "external": { + "type": ["boolean", "object"], + "properties": { + "name": {"type": "string"} + }, + "additionalProperties": false + }, + "internal": {"type": "boolean"}, + "attachable": {"type": "boolean"}, + "labels": {"$ref": "#/definitions/list_or_dict"} + }, + "additionalProperties": false + }, + + "volume": { + "id": "#/definitions/volume", + "type": ["object", "null"], + "properties": { + "name": {"type": "string"}, + "driver": {"type": "string"}, + "driver_opts": { + "type": "object", + "patternProperties": { + "^.+$": {"type": ["string", "number"]} + } + }, + "external": { + "type": ["boolean", "object"], + "properties": { + "name": {"type": "string"} + }, + "additionalProperties": false + }, + "labels": {"$ref": "#/definitions/list_or_dict"} + }, + "additionalProperties": false + }, + + "secret": { + "id": "#/definitions/secret", + "type": "object", + "properties": { + "file": {"type": "string"}, + "external": { + "type": ["boolean", "object"], + "properties": { + "name": {"type": "string"} + } + }, + "labels": {"$ref": "#/definitions/list_or_dict"} + }, + "additionalProperties": false + }, + + "config": { + "id": "#/definitions/config", + "type": "object", + "properties": { + "file": {"type": "string"}, + "external": { + "type": ["boolean", "object"], + "properties": { + "name": {"type": "string"} + } + }, + "labels": {"$ref": "#/definitions/list_or_dict"} + }, + "additionalProperties": false + }, + + "string_or_list": { + "oneOf": [ + {"type": "string"}, + {"$ref": "#/definitions/list_of_strings"} + ] + }, + + "list_of_strings": { + "type": "array", + "items": {"type": "string"}, + "uniqueItems": true + }, + + "list_or_dict": { + "oneOf": [ + { + "type": "object", + "patternProperties": { + ".+": { + "type": ["string", "number", "null"] + } + }, + "additionalProperties": false + }, + {"type": "array", "items": {"type": "string"}, "uniqueItems": true} + ] + }, + + "constraints": { + "service": { + "id": "#/definitions/constraints/service", + "anyOf": [ + {"required": ["build"]}, + {"required": ["image"]} + ], + "properties": { + "build": { + "required": ["context"] + } + } + } + } + } +} diff --git a/compose/const.py b/compose/const.py index b5970f82a..2ac08b89a 100644 --- a/compose/const.py +++ b/compose/const.py @@ -32,6 +32,7 @@ COMPOSEFILE_V3_1 = ComposeVersion('3.1') COMPOSEFILE_V3_2 = ComposeVersion('3.2') COMPOSEFILE_V3_3 = ComposeVersion('3.3') COMPOSEFILE_V3_4 = ComposeVersion('3.4') +COMPOSEFILE_V3_5 = ComposeVersion('3.5') API_VERSIONS = { COMPOSEFILE_V1: '1.21', @@ -44,6 +45,7 @@ API_VERSIONS = { COMPOSEFILE_V3_2: '1.25', COMPOSEFILE_V3_3: '1.30', COMPOSEFILE_V3_4: '1.30', + COMPOSEFILE_V3_5: '1.30', } API_VERSION_TO_ENGINE_VERSION = { @@ -57,4 +59,5 @@ API_VERSION_TO_ENGINE_VERSION = { API_VERSIONS[COMPOSEFILE_V3_2]: '1.13.0', API_VERSIONS[COMPOSEFILE_V3_3]: '17.06.0', API_VERSIONS[COMPOSEFILE_V3_4]: '17.06.0', + API_VERSIONS[COMPOSEFILE_V3_5]: '17.06.0', } diff --git a/compose/service.py b/compose/service.py index 2829240f2..28c032763 100644 --- a/compose/service.py +++ b/compose/service.py @@ -43,6 +43,7 @@ from .parallel import parallel_execute from .progress_stream import stream_output from .progress_stream import StreamOutputError from .utils import json_hash +from .utils import parse_bytes from .utils import parse_seconds_float @@ -916,6 +917,7 @@ class Service(object): buildargs=build_args, network_mode=build_opts.get('network', None), target=build_opts.get('target', None), + shmsize=parse_bytes(build_opts.get('shm_size')) if build_opts.get('shm_size') else None, ) try: diff --git a/tests/acceptance/cli_test.py b/tests/acceptance/cli_test.py index 3a5e17ad8..ca4bd9ee7 100644 --- a/tests/acceptance/cli_test.py +++ b/tests/acceptance/cli_test.py @@ -531,6 +531,12 @@ class CLITestCase(DockerClientTestCase): ] assert not containers + def test_build_shm_size_build_option(self): + pull_busybox(self.client) + self.base_dir = 'tests/fixtures/build-shm-size' + result = self.dispatch(['build', '--no-cache'], None) + assert 'shm_size: 96' in result.stdout + def test_bundle_with_digests(self): self.base_dir = 'tests/fixtures/bundle-with-digests/' tmpdir = py.test.ensuretemp('cli_test_bundle') diff --git a/tests/fixtures/build-shm-size/Dockerfile b/tests/fixtures/build-shm-size/Dockerfile new file mode 100644 index 000000000..f91733d63 --- /dev/null +++ b/tests/fixtures/build-shm-size/Dockerfile @@ -0,0 +1,4 @@ +FROM busybox + +# Report the shm_size (through the size of /dev/shm) +RUN echo "shm_size:" $(df -h /dev/shm | tail -n 1 | awk '{print $2}') diff --git a/tests/fixtures/build-shm-size/docker-compose.yml b/tests/fixtures/build-shm-size/docker-compose.yml new file mode 100644 index 000000000..238a51322 --- /dev/null +++ b/tests/fixtures/build-shm-size/docker-compose.yml @@ -0,0 +1,7 @@ +version: '3.5' + +services: + custom_shm_size: + build: + context: . + shm_size: 100663296 # =96M diff --git a/tests/unit/service_test.py b/tests/unit/service_test.py index 0293695ab..43ccf081c 100644 --- a/tests/unit/service_test.py +++ b/tests/unit/service_test.py @@ -475,6 +475,7 @@ class ServiceTest(unittest.TestCase): cache_from=None, network_mode=None, target=None, + shmsize=None, ) def test_ensure_image_exists_no_build(self): @@ -515,6 +516,7 @@ class ServiceTest(unittest.TestCase): cache_from=None, network_mode=None, target=None, + shmsize=None ) def test_build_does_not_pull(self): From aecd0a948336dc16bddd67ec81075288b2f8dd14 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Fri, 13 Oct 2017 15:24:34 -0700 Subject: [PATCH 140/244] Temporary xfails for engine bug Signed-off-by: Joffrey F --- tests/acceptance/cli_test.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/tests/acceptance/cli_test.py b/tests/acceptance/cli_test.py index ca4bd9ee7..b598d99d5 100644 --- a/tests/acceptance/cli_test.py +++ b/tests/acceptance/cli_test.py @@ -14,7 +14,7 @@ from collections import Counter from collections import namedtuple from operator import attrgetter -import py +import pytest import six import yaml from docker import errors @@ -504,6 +504,7 @@ class CLITestCase(DockerClientTestCase): assert BUILD_CACHE_TEXT not in result.stdout assert BUILD_PULL_TEXT in result.stdout + @pytest.mark.xfail(reason='17.10.0 RC bug remove after GA https://github.com/moby/moby/issues/35116') def test_build_failed(self): self.base_dir = 'tests/fixtures/simple-failing-dockerfile' self.dispatch(['build', 'simple'], returncode=1) @@ -517,6 +518,7 @@ class CLITestCase(DockerClientTestCase): ] assert len(containers) == 1 + @pytest.mark.xfail(reason='17.10.0 RC bug remove after GA https://github.com/moby/moby/issues/35116') def test_build_failed_forcerm(self): self.base_dir = 'tests/fixtures/simple-failing-dockerfile' self.dispatch(['build', '--force-rm', 'simple'], returncode=1) @@ -539,7 +541,7 @@ class CLITestCase(DockerClientTestCase): def test_bundle_with_digests(self): self.base_dir = 'tests/fixtures/bundle-with-digests/' - tmpdir = py.test.ensuretemp('cli_test_bundle') + tmpdir = pytest.ensuretemp('cli_test_bundle') self.addCleanup(tmpdir.remove) filename = str(tmpdir.join('example.dab')) @@ -1403,7 +1405,7 @@ class CLITestCase(DockerClientTestCase): [u'/bin/true'], ) - @py.test.mark.skipif(SWARM_SKIP_RM_VOLUMES, reason='Swarm DELETE /containers/ bug') + @pytest.mark.skipif(SWARM_SKIP_RM_VOLUMES, reason='Swarm DELETE /containers/ bug') def test_run_rm(self): self.base_dir = 'tests/fixtures/volume' proc = start_process(self.base_dir, ['run', '--rm', 'test']) From f74838676dd1cf759bbb0125505c08baea8ca8f3 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Wed, 27 Sep 2017 17:10:13 -0700 Subject: [PATCH 141/244] Mount with same container path and different mode should override Signed-off-by: Joffrey F --- compose/config/config.py | 35 ++++++++++++++++++++--------- tests/unit/config/config_test.py | 38 +++++++++++++++++++++++++++++--- 2 files changed, 60 insertions(+), 13 deletions(-) diff --git a/compose/config/config.py b/compose/config/config.py index f16dd01b3..948e2376e 100644 --- a/compose/config/config.py +++ b/compose/config/config.py @@ -1137,24 +1137,30 @@ def resolve_volume_paths(working_dir, service_dict): def resolve_volume_path(working_dir, volume): + mount_params = None if isinstance(volume, dict): - host_path = volume.get('source') container_path = volume.get('target') + host_path = volume.get('source') + mode = None if host_path: if volume.get('read_only'): - container_path += ':ro' + mode = 'ro' if volume.get('volume', {}).get('nocopy'): - container_path += ':nocopy' + mode = 'nocopy' + mount_params = (host_path, mode) else: - container_path, host_path = split_path_mapping(volume) + container_path, mount_params = split_path_mapping(volume) - if host_path is not None: + if mount_params is not None: + host_path, mode = mount_params + if host_path is None: + return container_path if host_path.startswith('.'): host_path = expand_path(working_dir, host_path) host_path = os.path.expanduser(host_path) - return u"{}:{}".format(host_path, container_path) - else: - return container_path + return u"{}:{}{}".format(host_path, container_path, (':' + mode if mode else '')) + + return container_path def normalize_build(service_dict, working_dir, environment): @@ -1234,7 +1240,12 @@ def split_path_mapping(volume_path): if ':' in volume_config: (host, container) = volume_config.split(':', 1) - return (container, drive + host) + container_drive, container_path = splitdrive(container) + mode = None + if ':' in container_path: + container_path, mode = container_path.rsplit(':', 1) + + return (container_drive + container_path, (drive + host, mode)) else: return (volume_path, None) @@ -1246,7 +1257,11 @@ def join_path_mapping(pair): elif host is None: return container else: - return ":".join((host, container)) + host, mode = host + result = ":".join((host, container)) + if mode: + result += ":" + mode + return result def expand_path(working_dir, path): diff --git a/tests/unit/config/config_test.py b/tests/unit/config/config_test.py index de9a61302..c5e40130d 100644 --- a/tests/unit/config/config_test.py +++ b/tests/unit/config/config_test.py @@ -1101,6 +1101,38 @@ class ConfigTest(unittest.TestCase): ['/anonymous', '/c:/b:rw', 'vol:/x:ro'] ) + @mock.patch.dict(os.environ) + def test_volume_mode_override(self): + os.environ['COMPOSE_CONVERT_WINDOWS_PATHS'] = 'true' + base_file = config.ConfigFile( + 'base.yaml', + { + 'version': '2.3', + 'services': { + 'web': { + 'image': 'example/web', + 'volumes': ['/c:/b:rw'] + } + }, + } + ) + + override_file = config.ConfigFile( + 'override.yaml', + { + 'version': '2.3', + 'services': { + 'web': { + 'volumes': ['/c:/b:ro'] + } + } + } + ) + details = config.ConfigDetails('.', [base_file, override_file]) + service_dicts = config.load(details).services + svc_volumes = list(map(lambda v: v.repr(), service_dicts[0]['volumes'])) + assert svc_volumes == ['/c:/b:ro'] + def test_undeclared_volume_v2(self): base_file = config.ConfigFile( 'base.yaml', @@ -4018,7 +4050,7 @@ class VolumePathTest(unittest.TestCase): def test_split_path_mapping_with_windows_path(self): host_path = "c:\\Users\\msamblanet\\Documents\\anvil\\connect\\config" windows_volume_path = host_path + ":/opt/connect/config:ro" - expected_mapping = ("/opt/connect/config:ro", host_path) + expected_mapping = ("/opt/connect/config", (host_path, 'ro')) mapping = config.split_path_mapping(windows_volume_path) assert mapping == expected_mapping @@ -4026,7 +4058,7 @@ class VolumePathTest(unittest.TestCase): def test_split_path_mapping_with_windows_path_in_container(self): host_path = 'c:\\Users\\remilia\\data' container_path = 'c:\\scarletdevil\\data' - expected_mapping = (container_path, host_path) + expected_mapping = (container_path, (host_path, None)) mapping = config.split_path_mapping('{0}:{1}'.format(host_path, container_path)) assert mapping == expected_mapping @@ -4034,7 +4066,7 @@ class VolumePathTest(unittest.TestCase): def test_split_path_mapping_with_root_mount(self): host_path = '/' container_path = '/var/hostroot' - expected_mapping = (container_path, host_path) + expected_mapping = (container_path, (host_path, None)) mapping = config.split_path_mapping('{0}:{1}'.format(host_path, container_path)) assert mapping == expected_mapping From 07b30e314592154188d38bc5d0e7167e5c3e7228 Mon Sep 17 00:00:00 2001 From: Andrea Giardini Date: Wed, 20 Sep 2017 23:05:29 +0200 Subject: [PATCH 142/244] Fix secret location with absolute paths Signed-off-by: Andrea Giardini --- compose/service.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/compose/service.py b/compose/service.py index 28c032763..aecafc8ca 100644 --- a/compose/service.py +++ b/compose/service.py @@ -881,9 +881,12 @@ class Service(object): def get_secret_volumes(self): def build_spec(secret): - target = '{}/{}'.format( - const.SECRETS_PATH, - secret['secret'].target or secret['secret'].source) + if secret['secret'].target is not None and secret['secret'].target.startswith('/'): + target = secret['secret'].target + else: + target = '{}/{}'.format( + const.SECRETS_PATH, + secret['secret'].target or secret['secret'].source) return VolumeSpec(secret['file'], target, 'ro') return [build_spec(secret) for secret in self.secrets] From 96882268de43d4a32a708c5ddffb9c219fc65664 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Fri, 13 Oct 2017 17:02:01 -0700 Subject: [PATCH 143/244] Add get_secret_volumes unit tests Signed-off-by: Joffrey F --- compose/service.py | 12 ++++----- tests/unit/service_test.py | 55 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 61 insertions(+), 6 deletions(-) diff --git a/compose/service.py b/compose/service.py index aecafc8ca..1a18c6654 100644 --- a/compose/service.py +++ b/compose/service.py @@ -881,12 +881,12 @@ class Service(object): def get_secret_volumes(self): def build_spec(secret): - if secret['secret'].target is not None and secret['secret'].target.startswith('/'): - target = secret['secret'].target - else: - target = '{}/{}'.format( - const.SECRETS_PATH, - secret['secret'].target or secret['secret'].source) + target = secret['secret'].target + if target is None: + target = '{}/{}'.format(const.SECRETS_PATH, secret['secret'].source) + elif not os.path.isabs(target): + target = '{}/{}'.format(const.SECRETS_PATH, target) + return VolumeSpec(secret['file'], target, 'ro') return [build_spec(secret) for secret in self.secrets] diff --git a/tests/unit/service_test.py b/tests/unit/service_test.py index 43ccf081c..7d61807ba 100644 --- a/tests/unit/service_test.py +++ b/tests/unit/service_test.py @@ -9,12 +9,14 @@ from .. import mock from .. import unittest from compose.config.errors import DependencyError from compose.config.types import ServicePort +from compose.config.types import ServiceSecret from compose.config.types import VolumeFromSpec from compose.config.types import VolumeSpec from compose.const import LABEL_CONFIG_HASH from compose.const import LABEL_ONE_OFF from compose.const import LABEL_PROJECT from compose.const import LABEL_SERVICE +from compose.const import SECRETS_PATH from compose.container import Container from compose.project import OneOffFilter from compose.service import build_ulimits @@ -1089,3 +1091,56 @@ class ServiceVolumesTest(unittest.TestCase): self.assertEqual( self.mock_client.create_host_config.call_args[1]['binds'], [volume]) + + +class ServiceSecretTest(unittest.TestCase): + def setUp(self): + self.mock_client = mock.create_autospec(docker.APIClient) + + def test_get_secret_volumes(self): + secret1 = { + 'secret': ServiceSecret.parse({'source': 'secret1', 'target': 'b.txt'}), + 'file': 'a.txt' + } + service = Service( + 'web', + client=self.mock_client, + image='busybox', + secrets=[secret1] + ) + volumes = service.get_secret_volumes() + + assert volumes[0].external == secret1['file'] + assert volumes[0].internal == '{}/{}'.format(SECRETS_PATH, secret1['secret'].target) + + def test_get_secret_volumes_abspath(self): + secret1 = { + 'secret': ServiceSecret.parse({'source': 'secret1', 'target': '/d.txt'}), + 'file': 'c.txt' + } + service = Service( + 'web', + client=self.mock_client, + image='busybox', + secrets=[secret1] + ) + volumes = service.get_secret_volumes() + + assert volumes[0].external == secret1['file'] + assert volumes[0].internal == secret1['secret'].target + + def test_get_secret_volumes_no_target(self): + secret1 = { + 'secret': ServiceSecret.parse({'source': 'secret1'}), + 'file': 'c.txt' + } + service = Service( + 'web', + client=self.mock_client, + image='busybox', + secrets=[secret1] + ) + volumes = service.get_secret_volumes() + + assert volumes[0].external == secret1['file'] + assert volumes[0].internal == '{}/{}'.format(SECRETS_PATH, secret1['secret'].source) From 2f2259f2d236285b0024d9100e9152e1b81fd63f Mon Sep 17 00:00:00 2001 From: Guillermo Arribas Date: Fri, 6 Oct 2017 19:12:59 -0300 Subject: [PATCH 144/244] Build labels option: array form produces unmarshal error (fixes #5183) Signed-off-by: Guillermo Arribas --- compose/service.py | 3 ++- tests/integration/service_test.py | 19 ++++++++++++++++++- tests/unit/service_test.py | 4 ++-- 3 files changed, 22 insertions(+), 4 deletions(-) diff --git a/compose/service.py b/compose/service.py index 1a18c6654..e2f72aa5a 100644 --- a/compose/service.py +++ b/compose/service.py @@ -23,6 +23,7 @@ from . import const from . import progress_stream from .config import DOCKER_CONFIG_KEYS from .config import merge_environment +from .config.config import parse_labels from .config.errors import DependencyError from .config.types import ServicePort from .config.types import VolumeSpec @@ -916,7 +917,7 @@ class Service(object): nocache=no_cache, dockerfile=build_opts.get('dockerfile', None), cache_from=build_opts.get('cache_from', None), - labels=build_opts.get('labels', None), + labels=parse_labels(build_opts.get('labels', None)), buildargs=build_args, network_mode=build_opts.get('network', None), target=build_opts.get('target', None), diff --git a/tests/integration/service_test.py b/tests/integration/service_test.py index 84b54fe41..a71bc407c 100644 --- a/tests/integration/service_test.py +++ b/tests/integration/service_test.py @@ -761,7 +761,7 @@ class ServiceTest(DockerClientTestCase): assert service.image() assert "build_version=2" in service.image()['ContainerConfig']['Cmd'] - def test_build_with_build_labels(self): + def test_build_with_build_labels_dict(self): base_dir = tempfile.mkdtemp() self.addCleanup(shutil.rmtree, base_dir) @@ -778,6 +778,23 @@ class ServiceTest(DockerClientTestCase): assert service.image() assert service.image()['Config']['Labels']['com.docker.compose.test'] == 'true' + def test_build_with_build_labels_list(self): + base_dir = tempfile.mkdtemp() + self.addCleanup(shutil.rmtree, base_dir) + + with open(os.path.join(base_dir, 'Dockerfile'), 'w') as f: + f.write('FROM busybox\n') + + service = self.create_service('buildlabels', build={ + 'context': text_type(base_dir), + 'labels': ['com.docker.compose.test=true'] + }) + service.build() + self.addCleanup(self.client.remove_image, service.image_name) + + assert service.image() + assert service.image()['Config']['Labels']['com.docker.compose.test'] == 'true' + @no_cluster('Container networks not on Swarm') def test_build_with_network(self): base_dir = tempfile.mkdtemp() diff --git a/tests/unit/service_test.py b/tests/unit/service_test.py index 7d61807ba..5c5c2bf67 100644 --- a/tests/unit/service_test.py +++ b/tests/unit/service_test.py @@ -473,7 +473,7 @@ class ServiceTest(unittest.TestCase): nocache=False, rm=True, buildargs={}, - labels=None, + labels={}, cache_from=None, network_mode=None, target=None, @@ -514,7 +514,7 @@ class ServiceTest(unittest.TestCase): nocache=False, rm=True, buildargs={}, - labels=None, + labels={}, cache_from=None, network_mode=None, target=None, From f6d7eeb129087e4f960946778a4f2ec72cad6ef1 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Mon, 16 Oct 2017 11:43:06 -0700 Subject: [PATCH 145/244] Move build labels parsing to config module Signed-off-by: Joffrey F --- compose/config/config.py | 12 ++++++------ compose/service.py | 3 +-- tests/integration/service_test.py | 19 +------------------ tests/unit/config/config_test.py | 24 +++++++++++++++++++++++- tests/unit/service_test.py | 4 ++-- 5 files changed, 33 insertions(+), 29 deletions(-) diff --git a/compose/config/config.py b/compose/config/config.py index 948e2376e..68b2be3a6 100644 --- a/compose/config/config.py +++ b/compose/config/config.py @@ -707,16 +707,16 @@ def process_service(service_config): if 'build' in service_dict: if isinstance(service_dict['build'], six.string_types): service_dict['build'] = resolve_build_path(working_dir, service_dict['build']) - elif isinstance(service_dict['build'], dict) and 'context' in service_dict['build']: - path = service_dict['build']['context'] - service_dict['build']['context'] = resolve_build_path(working_dir, path) + elif isinstance(service_dict['build'], dict): + if 'context' in service_dict['build']: + path = service_dict['build']['context'] + service_dict['build']['context'] = resolve_build_path(working_dir, path) + if 'labels' in service_dict['build']: + service_dict['build']['labels'] = parse_labels(service_dict['build']['labels']) if 'volumes' in service_dict and service_dict.get('volume_driver') is None: service_dict['volumes'] = resolve_volume_paths(working_dir, service_dict) - if 'labels' in service_dict: - service_dict['labels'] = parse_labels(service_dict['labels']) - if 'sysctls' in service_dict: service_dict['sysctls'] = build_string_dict(parse_sysctls(service_dict['sysctls'])) diff --git a/compose/service.py b/compose/service.py index e2f72aa5a..1a18c6654 100644 --- a/compose/service.py +++ b/compose/service.py @@ -23,7 +23,6 @@ from . import const from . import progress_stream from .config import DOCKER_CONFIG_KEYS from .config import merge_environment -from .config.config import parse_labels from .config.errors import DependencyError from .config.types import ServicePort from .config.types import VolumeSpec @@ -917,7 +916,7 @@ class Service(object): nocache=no_cache, dockerfile=build_opts.get('dockerfile', None), cache_from=build_opts.get('cache_from', None), - labels=parse_labels(build_opts.get('labels', None)), + labels=build_opts.get('labels', None), buildargs=build_args, network_mode=build_opts.get('network', None), target=build_opts.get('target', None), diff --git a/tests/integration/service_test.py b/tests/integration/service_test.py index a71bc407c..84b54fe41 100644 --- a/tests/integration/service_test.py +++ b/tests/integration/service_test.py @@ -761,7 +761,7 @@ class ServiceTest(DockerClientTestCase): assert service.image() assert "build_version=2" in service.image()['ContainerConfig']['Cmd'] - def test_build_with_build_labels_dict(self): + def test_build_with_build_labels(self): base_dir = tempfile.mkdtemp() self.addCleanup(shutil.rmtree, base_dir) @@ -778,23 +778,6 @@ class ServiceTest(DockerClientTestCase): assert service.image() assert service.image()['Config']['Labels']['com.docker.compose.test'] == 'true' - def test_build_with_build_labels_list(self): - base_dir = tempfile.mkdtemp() - self.addCleanup(shutil.rmtree, base_dir) - - with open(os.path.join(base_dir, 'Dockerfile'), 'w') as f: - f.write('FROM busybox\n') - - service = self.create_service('buildlabels', build={ - 'context': text_type(base_dir), - 'labels': ['com.docker.compose.test=true'] - }) - service.build() - self.addCleanup(self.client.remove_image, service.image_name) - - assert service.image() - assert service.image()['Config']['Labels']['com.docker.compose.test'] == 'true' - @no_cluster('Container networks not on Swarm') def test_build_with_network(self): base_dir = tempfile.mkdtemp() diff --git a/tests/unit/config/config_test.py b/tests/unit/config/config_test.py index c5e40130d..8f2266ed8 100644 --- a/tests/unit/config/config_test.py +++ b/tests/unit/config/config_test.py @@ -892,7 +892,7 @@ class ConfigTest(unittest.TestCase): assert service['build']['args']['opt1'] == '42' assert service['build']['args']['opt2'] == 'foobar' - def test_load_with_build_labels(self): + def test_load_build_labels_dict(self): service = config.load( build_config_details( { @@ -919,6 +919,28 @@ class ConfigTest(unittest.TestCase): assert service['build']['labels']['label1'] == 42 assert service['build']['labels']['label2'] == 'foobar' + def test_load_build_labels_list(self): + base_file = config.ConfigFile( + 'base.yml', + { + 'version': '2.3', + 'services': { + 'web': { + 'build': { + 'context': '.', + 'labels': ['foo=bar', 'baz=true', 'foobar=1'] + }, + }, + }, + } + ) + + details = config.ConfigDetails('.', [base_file]) + service = config.load(details).services[0] + assert service['build']['labels'] == { + 'foo': 'bar', 'baz': 'true', 'foobar': '1' + } + def test_build_args_allow_empty_properties(self): service = config.load( build_config_details( diff --git a/tests/unit/service_test.py b/tests/unit/service_test.py index 5c5c2bf67..7d61807ba 100644 --- a/tests/unit/service_test.py +++ b/tests/unit/service_test.py @@ -473,7 +473,7 @@ class ServiceTest(unittest.TestCase): nocache=False, rm=True, buildargs={}, - labels={}, + labels=None, cache_from=None, network_mode=None, target=None, @@ -514,7 +514,7 @@ class ServiceTest(unittest.TestCase): nocache=False, rm=True, buildargs={}, - labels={}, + labels=None, cache_from=None, network_mode=None, target=None, From b9bae89f1d25c05ddd3438f6968c96ef3a9fb16b Mon Sep 17 00:00:00 2001 From: Guillermo Arribas Date: Wed, 11 Oct 2017 13:56:15 -0300 Subject: [PATCH 146/244] Config command generates invalid volumes (fixes #5176) Signed-off-by: Guillermo Arribas --- compose/config/config.py | 19 +++---- compose/config/serialize.py | 4 +- tests/acceptance/cli_test.py | 54 +++++++++++++++++-- .../volumes/external-volumes-v2-x.yml | 17 ++++++ ...al-volumes.yml => external-volumes-v2.yml} | 2 +- .../volumes/external-volumes-v3-4.yml | 17 ++++++ .../volumes/external-volumes-v3-x.yml | 16 ++++++ 7 files changed, 114 insertions(+), 15 deletions(-) create mode 100644 tests/fixtures/volumes/external-volumes-v2-x.yml rename tests/fixtures/volumes/{external-volumes.yml => external-volumes-v2.yml} (92%) create mode 100644 tests/fixtures/volumes/external-volumes-v3-4.yml create mode 100644 tests/fixtures/volumes/external-volumes-v3-x.yml diff --git a/compose/config/config.py b/compose/config/config.py index 68b2be3a6..7bb57076e 100644 --- a/compose/config/config.py +++ b/compose/config/config.py @@ -15,6 +15,9 @@ from cached_property import cached_property from . import types from .. import const from ..const import COMPOSEFILE_V1 as V1 +from ..const import COMPOSEFILE_V2_1 as V2_1 +from ..const import COMPOSEFILE_V3_0 as V3_0 +from ..const import COMPOSEFILE_V3_4 as V3_4 from ..utils import build_string_dict from ..utils import parse_bytes from ..utils import parse_nanoseconds_int @@ -405,7 +408,7 @@ def load_mapping(config_files, get_func, entity_type, working_dir=None): external = config.get('external') if external: name_field = 'name' if entity_type == 'Volume' else 'external_name' - validate_external(entity_type, name, config) + validate_external(entity_type, name, config, config_file.version) if isinstance(external, dict): config[name_field] = external.get('name') elif not config.get('name'): @@ -425,14 +428,12 @@ def load_mapping(config_files, get_func, entity_type, working_dir=None): return mapping -def validate_external(entity_type, name, config): - if len(config.keys()) <= 1: - return - - raise ConfigurationError( - "{} {} declared as external but specifies additional attributes " - "({}).".format( - entity_type, name, ', '.join(k for k in config if k != 'external'))) +def validate_external(entity_type, name, config, version): + if (version < V2_1 or (version >= V3_0 and version < V3_4)) and len(config.keys()) > 1: + raise ConfigurationError( + "{} {} declared as external but specifies additional attributes " + "({}).".format( + entity_type, name, ', '.join(k for k in config if k != 'external'))) def load_services(config_details, config_file): diff --git a/compose/config/serialize.py b/compose/config/serialize.py index 606dd7614..2b8c73f14 100644 --- a/compose/config/serialize.py +++ b/compose/config/serialize.py @@ -9,7 +9,7 @@ from compose.const import COMPOSEFILE_V1 as V1 from compose.const import COMPOSEFILE_V2_1 as V2_1 from compose.const import COMPOSEFILE_V3_0 as V3_0 from compose.const import COMPOSEFILE_V3_2 as V3_2 -from compose.const import COMPOSEFILE_V3_2 as V3_4 +from compose.const import COMPOSEFILE_V3_4 as V3_4 def serialize_config_type(dumper, data): @@ -67,7 +67,7 @@ def denormalize_config(config, image_digests=None): del conf['external_name'] if 'name' in conf: - if config.version < V2_1 or (config.version > V3_0 and config.version < V3_4): + if config.version < V2_1 or (config.version >= V3_0 and config.version < V3_4): del conf['name'] elif 'external' in conf: conf['external'] = True diff --git a/tests/acceptance/cli_test.py b/tests/acceptance/cli_test.py index b598d99d5..43cc89e36 100644 --- a/tests/acceptance/cli_test.py +++ b/tests/acceptance/cli_test.py @@ -285,15 +285,63 @@ class CLITestCase(DockerClientTestCase): } } - def test_config_external_volume(self): + def test_config_external_volume_v2(self): self.base_dir = 'tests/fixtures/volumes' - result = self.dispatch(['-f', 'external-volumes.yml', 'config']) + result = self.dispatch(['-f', 'external-volumes-v2.yml', 'config']) json_result = yaml.load(result.stdout) assert 'volumes' in json_result assert json_result['volumes'] == { 'foo': { 'external': True, - 'name': 'foo', + }, + 'bar': { + 'external': { + 'name': 'some_bar', + }, + } + } + + def test_config_external_volume_v2_x(self): + self.base_dir = 'tests/fixtures/volumes' + result = self.dispatch(['-f', 'external-volumes-v2-x.yml', 'config']) + json_result = yaml.load(result.stdout) + assert 'volumes' in json_result + assert json_result['volumes'] == { + 'foo': { + 'external': True, + 'name': 'some_foo', + }, + 'bar': { + 'external': True, + 'name': 'some_bar', + } + } + + def test_config_external_volume_v3_x(self): + self.base_dir = 'tests/fixtures/volumes' + result = self.dispatch(['-f', 'external-volumes-v3-x.yml', 'config']) + json_result = yaml.load(result.stdout) + assert 'volumes' in json_result + assert json_result['volumes'] == { + 'foo': { + 'external': True, + }, + 'bar': { + 'external': { + 'name': 'some_bar', + }, + } + } + + def test_config_external_volume_v3_4(self): + self.base_dir = 'tests/fixtures/volumes' + result = self.dispatch(['-f', 'external-volumes-v3-4.yml', 'config']) + json_result = yaml.load(result.stdout) + assert 'volumes' in json_result + assert json_result['volumes'] == { + 'foo': { + 'external': True, + 'name': 'some_foo', }, 'bar': { 'external': True, diff --git a/tests/fixtures/volumes/external-volumes-v2-x.yml b/tests/fixtures/volumes/external-volumes-v2-x.yml new file mode 100644 index 000000000..3b736c5f4 --- /dev/null +++ b/tests/fixtures/volumes/external-volumes-v2-x.yml @@ -0,0 +1,17 @@ +version: "2.1" + +services: + web: + image: busybox + command: top + volumes: + - foo:/var/lib/ + - bar:/etc/ + +volumes: + foo: + external: true + name: some_foo + bar: + external: + name: some_bar diff --git a/tests/fixtures/volumes/external-volumes.yml b/tests/fixtures/volumes/external-volumes-v2.yml similarity index 92% rename from tests/fixtures/volumes/external-volumes.yml rename to tests/fixtures/volumes/external-volumes-v2.yml index 05c6c4844..4025b53b1 100644 --- a/tests/fixtures/volumes/external-volumes.yml +++ b/tests/fixtures/volumes/external-volumes-v2.yml @@ -1,4 +1,4 @@ -version: "2.1" +version: "2" services: web: diff --git a/tests/fixtures/volumes/external-volumes-v3-4.yml b/tests/fixtures/volumes/external-volumes-v3-4.yml new file mode 100644 index 000000000..76c8421dc --- /dev/null +++ b/tests/fixtures/volumes/external-volumes-v3-4.yml @@ -0,0 +1,17 @@ +version: "3.4" + +services: + web: + image: busybox + command: top + volumes: + - foo:/var/lib/ + - bar:/etc/ + +volumes: + foo: + external: true + name: some_foo + bar: + external: + name: some_bar diff --git a/tests/fixtures/volumes/external-volumes-v3-x.yml b/tests/fixtures/volumes/external-volumes-v3-x.yml new file mode 100644 index 000000000..903fee647 --- /dev/null +++ b/tests/fixtures/volumes/external-volumes-v3-x.yml @@ -0,0 +1,16 @@ +version: "3.0" + +services: + web: + image: busybox + command: top + volumes: + - foo:/var/lib/ + - bar:/etc/ + +volumes: + foo: + external: true + bar: + external: + name: some_bar From 9e1388eba64c26328af700d357bd71eabe7fc6bd Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Mon, 16 Oct 2017 12:41:29 -0700 Subject: [PATCH 147/244] Add specific handling for pywintypes.error Signed-off-by: Joffrey F --- compose/cli/errors.py | 20 ++++++++++++++++++++ tests/unit/cli/errors_test.py | 21 +++++++++++++++++++++ 2 files changed, 41 insertions(+) diff --git a/compose/cli/errors.py b/compose/cli/errors.py index 23e065c99..1506aa660 100644 --- a/compose/cli/errors.py +++ b/compose/cli/errors.py @@ -57,6 +57,26 @@ def handle_connection_errors(client): except (ReadTimeout, socket.timeout) as e: log_timeout_error(client.timeout) raise ConnectionError() + except Exception as e: + if is_windows(): + import pywintypes + if isinstance(e, pywintypes.error): + log_windows_pipe_error(e) + raise ConnectionError() + raise + + +def log_windows_pipe_error(exc): + if exc.winerror == 232: # https://github.com/docker/compose/issues/5005 + log.error( + "The current Compose file version is not compatible with your engine version. " + "Please upgrade your Compose file to a more recent version, or set " + "a COMPOSE_API_VERSION in your environment." + ) + else: + log.error( + "Windows named pipe error: {} (code: {})".format(exc.strerror, exc.winerror) + ) def log_timeout_error(timeout): diff --git a/tests/unit/cli/errors_test.py b/tests/unit/cli/errors_test.py index 7406a8880..68326d1c7 100644 --- a/tests/unit/cli/errors_test.py +++ b/tests/unit/cli/errors_test.py @@ -7,6 +7,7 @@ from requests.exceptions import ConnectionError from compose.cli import errors from compose.cli.errors import handle_connection_errors +from compose.const import IS_WINDOWS_PLATFORM from tests import mock @@ -65,3 +66,23 @@ class TestHandleConnectionErrors(object): raise APIError(None, None, msg) mock_logging.error.assert_called_once_with(msg) + + @pytest.mark.skipif(not IS_WINDOWS_PLATFORM, reason='Needs pywin32') + def test_windows_pipe_error_no_data(self, mock_logging): + import pywintypes + with pytest.raises(errors.ConnectionError): + with handle_connection_errors(mock.Mock(api_version='1.22')): + raise pywintypes.error(232, 'WriteFile', 'The pipe is being closed.') + + _, args, _ = mock_logging.error.mock_calls[0] + assert "The current Compose file version is not compatible with your engine version." in args[0] + + @pytest.mark.skipif(not IS_WINDOWS_PLATFORM, reason='Needs pywin32') + def test_windows_pipe_error_misc(self, mock_logging): + import pywintypes + with pytest.raises(errors.ConnectionError): + with handle_connection_errors(mock.Mock(api_version='1.22')): + raise pywintypes.error(231, 'WriteFile', 'The pipe is busy.') + + _, args, _ = mock_logging.error.mock_calls[0] + assert "Windows named pipe error: The pipe is busy. (code: 231)" == args[0] From 2556668c8c98a33a43a6bd69fe19c07f890fca8f Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Mon, 16 Oct 2017 13:54:30 -0700 Subject: [PATCH 148/244] Add check_duplicate=True when creating network Signed-off-by: Joffrey F --- compose/network.py | 1 + 1 file changed, 1 insertion(+) diff --git a/compose/network.py b/compose/network.py index 0f42eb20a..2e0a7e6ec 100644 --- a/compose/network.py +++ b/compose/network.py @@ -79,6 +79,7 @@ class Network(object): enable_ipv6=self.enable_ipv6, labels=self._labels, attachable=version_gte(self.client._version, '1.24') or None, + check_duplicate=True, ) def remove(self): From 63b5722b16c3dd57b331d5508eb2a2d13e44cbbd Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Tue, 17 Oct 2017 13:36:06 -0700 Subject: [PATCH 149/244] flake8 Signed-off-by: Joffrey F --- tests/acceptance/cli_test.py | 4 ---- tests/integration/testcases.py | 2 -- 2 files changed, 6 deletions(-) diff --git a/tests/acceptance/cli_test.py b/tests/acceptance/cli_test.py index 43cc89e36..8ba43b00f 100644 --- a/tests/acceptance/cli_test.py +++ b/tests/acceptance/cli_test.py @@ -493,10 +493,6 @@ class CLITestCase(DockerClientTestCase): 'image library/nonexisting-image:latest not found' in result.stderr or 'pull access denied for nonexisting-image' in result.stderr) - def test_pull_with_quiet(self): - assert self.dispatch(['pull', '--quiet']).stderr == '' - assert self.dispatch(['pull', '--quiet']).stdout == '' - def test_pull_with_parallel_failure(self): result = self.dispatch([ '-f', 'ignore-pull-failures.yml', 'pull', '--parallel'], diff --git a/tests/integration/testcases.py b/tests/integration/testcases.py index 8435f97dd..b72fb53a8 100644 --- a/tests/integration/testcases.py +++ b/tests/integration/testcases.py @@ -75,7 +75,6 @@ def v2_1_only(): return min_version_skip(V2_1) - def v2_2_only(): return min_version_skip(V2_2) @@ -84,7 +83,6 @@ def v2_3_only(): return min_version_skip(V2_3) - def v3_only(): return min_version_skip(V3_0) From a0f95afcd1cdf781e7eccdc9ed65d27c260e8534 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Mon, 16 Oct 2017 16:50:51 -0700 Subject: [PATCH 150/244] Bump 1.17.0-rc1 Signed-off-by: Joffrey F --- CHANGELOG.md | 61 +++++++++++++++++++++++++++++++++++++++++++-- compose/__init__.py | 2 +- script/run/run.sh | 2 +- 3 files changed, 61 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 558376855..cff19d879 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,64 @@ Change log ========== +1.17.0 (2017-11-03) +------------------- + +### New features + +#### Compose file version 3.4 + +- Introduced version 3.4 of the `docker-compose.yml` specification. + This version requires to be used with Docker Engine 17.06.0 or above. + +- Added support for `cache_from`, `network` and `target` options in build + configurations + +- Added support for the `order` parameter in the `update_config` section + +- Added support for setting a custom name in volume definitions using + the `name` parameter + +#### Compose file version 2.3 + +- Added support for `shm_size` option in build configuration + +#### Compose file version 2.x + +- Added support for extension fields (`x-*`). Also available for v3.4 files + +#### All formats + +- Added new `--no-start` to the `up` command, allowing users to create all + resources (networks, volumes, containers) without starting services. + The `create` command is deprecated in favor of this new option + +### Bugfixes + +- Fixed a bug where `extra_hosts` values would be overridden by extension + files instead of merging together + +- Fixed a bug where the validation for v3.2 files would prevent using the + `consistency` field in service volume definitions + +- Fixed a bug that would cause a crash when configuration fields expecting + unique items would contain duplicates + +- Fixed a bug where mount overrides with a different mode would create a + duplicate entry instead of overriding the original entry + +- Fixed a bug where build labels declared as a list wouldn't be properly + parsed + +- Fixed a bug where the output of `docker-compose config` would be invalid + for some versions if the file contained custom-named external volumes + +- Improved error handling when issuing a build command on Windows using an + unsupported file version + +- Fixed an issue where networks with identical names would sometimes be + created when running `up` commands concurrently. + 1.16.1 (2017-09-01) ------------------- @@ -8,7 +66,6 @@ Change log - Fixed bug that prevented using `extra_hosts` in several configuration files. - 1.16.0 (2017-08-31) ------------------- @@ -19,7 +76,7 @@ Change log - Introduced version 2.3 of the `docker-compose.yml` specification. This version requires to be used with Docker Engine 17.06.0 or above. -- Added support for the `target` parameter in network configurations +- Added support for the `target` parameter in build configurations - Added support for the `start_period` parameter in healthcheck configurations diff --git a/compose/__init__.py b/compose/__init__.py index 2e41ca896..86542ec44 100644 --- a/compose/__init__.py +++ b/compose/__init__.py @@ -1,4 +1,4 @@ from __future__ import absolute_import from __future__ import unicode_literals -__version__ = '1.16.1' +__version__ = '1.17.0-rc1' diff --git a/script/run/run.sh b/script/run/run.sh index f1754d05a..498226288 100755 --- a/script/run/run.sh +++ b/script/run/run.sh @@ -15,7 +15,7 @@ set -e -VERSION="1.16.1" +VERSION="1.17.0-rc1" IMAGE="docker/compose:$VERSION" From 7f1dc09404f60d704f47bd03bea7f2e86de55cae Mon Sep 17 00:00:00 2001 From: Alexey Rokhin Date: Wed, 17 May 2017 15:21:35 +0300 Subject: [PATCH 151/244] Fix testcases.py formatting Signed-off-by: Alexey Rokhin --- tests/integration/testcases.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/integration/testcases.py b/tests/integration/testcases.py index b72fb53a8..8435f97dd 100644 --- a/tests/integration/testcases.py +++ b/tests/integration/testcases.py @@ -75,6 +75,7 @@ def v2_1_only(): return min_version_skip(V2_1) + def v2_2_only(): return min_version_skip(V2_2) @@ -83,6 +84,7 @@ def v2_3_only(): return min_version_skip(V2_3) + def v3_only(): return min_version_skip(V3_0) From 11bd32b597ca0f710a68a0e3d02908a97542e2ca Mon Sep 17 00:00:00 2001 From: Alexey Rokhin Date: Wed, 17 May 2017 16:18:28 +0300 Subject: [PATCH 152/244] skip cpu_percent test for Linux Signed-off-by: Alexey Rokhin --- tests/integration/service_test.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/integration/service_test.py b/tests/integration/service_test.py index 84b54fe41..28cca4aad 100644 --- a/tests/integration/service_test.py +++ b/tests/integration/service_test.py @@ -28,6 +28,7 @@ from compose.const import LABEL_ONE_OFF from compose.const import LABEL_PROJECT from compose.const import LABEL_SERVICE from compose.const import LABEL_VERSION +from compose.const import IS_WINDOWS_PLATFORM from compose.container import Container from compose.errors import OperationFailedError from compose.project import OneOffFilter From 3089eda5ab58b8898d82d5dca98a184fbcb133d9 Mon Sep 17 00:00:00 2001 From: Alexey Rokhin Date: Wed, 17 May 2017 16:42:43 +0300 Subject: [PATCH 153/244] service_test.py reorder imports Signed-off-by: Alexey Rokhin --- tests/integration/service_test.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/integration/service_test.py b/tests/integration/service_test.py index 28cca4aad..84b54fe41 100644 --- a/tests/integration/service_test.py +++ b/tests/integration/service_test.py @@ -28,7 +28,6 @@ from compose.const import LABEL_ONE_OFF from compose.const import LABEL_PROJECT from compose.const import LABEL_SERVICE from compose.const import LABEL_VERSION -from compose.const import IS_WINDOWS_PLATFORM from compose.container import Container from compose.errors import OperationFailedError from compose.project import OneOffFilter From aee944393e7e27d2ebd6744a90c388134b2a542d Mon Sep 17 00:00:00 2001 From: Joel Barciauskas Date: Wed, 12 Apr 2017 17:45:09 -0400 Subject: [PATCH 154/244] Add --quiet parameter to docker-compose pull, using existing silent flag Signed-off-by: Joel Barciauskas --- compose/project.py | 2 +- tests/acceptance/cli_test.py | 4 ++++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/compose/project.py b/compose/project.py index 2310a2fcc..1faf97d40 100644 --- a/compose/project.py +++ b/compose/project.py @@ -496,7 +496,7 @@ class Project(object): if parallel_pull: def pull_service(service): - service.pull(ignore_pull_failures, True) + service.pull(ignore_pull_failures, True, silent=silent) _, errors = parallel.parallel_execute( services, diff --git a/tests/acceptance/cli_test.py b/tests/acceptance/cli_test.py index 78d1c1eb1..e721b940f 100644 --- a/tests/acceptance/cli_test.py +++ b/tests/acceptance/cli_test.py @@ -463,6 +463,10 @@ class CLITestCase(DockerClientTestCase): re.compile('''^(ERROR: )?(b')?.* nonexisting-image''', re.MULTILINE)) + def test_pull_with_quiet(self): + assert self.dispatch(['pull', '--quiet']).stderr == '' + assert self.dispatch(['pull', '--quiet']).stdout == '' + def test_build_plain(self): self.base_dir = 'tests/fixtures/simple-dockerfile' self.dispatch(['build', 'simple']) From f8b2981fb9b641a92902c2b8d05d5a007a040b0a Mon Sep 17 00:00:00 2001 From: NikitaVlaznev Date: Mon, 19 Jun 2017 17:05:19 +0300 Subject: [PATCH 155/244] Fix double silent argument value Fix for "TypeError: pull() got multiple values for keyword argument 'silent'." This change https://github.com/docker/compose/commit/e9b6cc23fcf01d4768c7e082b7bc91b43ff84e7e caused additional value to be passed for the 'silent' argument, that was already passed there: https://github.com/docker/compose/commit/f85da99ef3273794e855afda8678174419d3bf4f Signed-off-by: Nikita Vlaznev --- compose/project.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/compose/project.py b/compose/project.py index 1faf97d40..59d58f2f6 100644 --- a/compose/project.py +++ b/compose/project.py @@ -496,7 +496,7 @@ class Project(object): if parallel_pull: def pull_service(service): - service.pull(ignore_pull_failures, True, silent=silent) + service.pull(ignore_pull_failures, silent=silent) _, errors = parallel.parallel_execute( services, From 5b4573e7e5e379c0777cb8e38a71d612fe354db5 Mon Sep 17 00:00:00 2001 From: Evan Shaw Date: Sat, 1 Jul 2017 13:40:02 +1200 Subject: [PATCH 156/244] Always silence pull output with --parallel This is how things were prior to the addition of the --quiet flag. Making it not silent produces output that's weird and difficult to read. Signed-off-by: Evan Shaw --- compose/project.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/compose/project.py b/compose/project.py index 59d58f2f6..2310a2fcc 100644 --- a/compose/project.py +++ b/compose/project.py @@ -496,7 +496,7 @@ class Project(object): if parallel_pull: def pull_service(service): - service.pull(ignore_pull_failures, silent=silent) + service.pull(ignore_pull_failures, True) _, errors = parallel.parallel_execute( services, From 9587556e8f7d3ef745399f8423627a9a54b3dfdd Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Mon, 28 Aug 2017 17:35:55 -0700 Subject: [PATCH 157/244] Add --no-start flag to up command. Deprecate create command. Signed-off-by: Joffrey F --- compose/cli/main.py | 18 ++++++++++++++++-- compose/project.py | 6 ++++-- tests/acceptance/cli_test.py | 25 +++++++++++++++++++++++++ 3 files changed, 45 insertions(+), 4 deletions(-) diff --git a/compose/cli/main.py b/compose/cli/main.py index 83bc7d58c..21bf1f308 100644 --- a/compose/cli/main.py +++ b/compose/cli/main.py @@ -319,6 +319,7 @@ class TopLevelCommand(object): def create(self, options): """ Creates containers for a service. + This command is deprecated. Use the `up` command with `--no-start` instead. Usage: create [options] [SERVICE...] @@ -332,6 +333,11 @@ class TopLevelCommand(object): """ service_names = options['SERVICE'] + log.warn( + 'The create command is deprecated. ' + 'Use the up command with the --no-start flag instead.' + ) + self.project.create( service_names=service_names, strategy=convergence_strategy_from_opts(options), @@ -902,6 +908,7 @@ class TopLevelCommand(object): --no-recreate If containers already exist, don't recreate them. Incompatible with --force-recreate. --no-build Don't build an image, even if it's missing. + --no-start Don't start the services after creating them. --build Build images before starting containers. --abort-on-container-exit Stops all containers if any container was stopped. Incompatible with -d. @@ -922,10 +929,16 @@ class TopLevelCommand(object): timeout = timeout_from_opts(options) remove_orphans = options['--remove-orphans'] detached = options.get('-d') + no_start = options.get('--no-start') - if detached and cascade_stop: + if detached and (cascade_stop or exit_value_from): raise UserError("--abort-on-container-exit and -d cannot be combined.") + if no_start: + for excluded in ['-d', '--abort-on-container-exit', '--exit-code-from']: + if options.get(excluded): + raise UserError('--no-start and {} cannot be combined.'.format(excluded)) + with up_shutdown_context(self.project, service_names, timeout, detached): to_attach = self.project.up( service_names=service_names, @@ -936,9 +949,10 @@ class TopLevelCommand(object): detached=detached, remove_orphans=remove_orphans, scale_override=parse_scale_args(options['--scale']), + start=not no_start ) - if detached: + if detached or no_start: return attached_containers = filter_containers_to_service_names(to_attach, service_names) diff --git a/compose/project.py b/compose/project.py index 2310a2fcc..c8b57edd2 100644 --- a/compose/project.py +++ b/compose/project.py @@ -412,7 +412,8 @@ class Project(object): detached=False, remove_orphans=False, scale_override=None, - rescale=True): + rescale=True, + start=True): warn_for_swarm_mode(self.client) @@ -436,7 +437,8 @@ class Project(object): timeout=timeout, detached=detached, scale_override=scale_override.get(service.name), - rescale=rescale + rescale=rescale, + start=start ) def get_deps(service): diff --git a/tests/acceptance/cli_test.py b/tests/acceptance/cli_test.py index e721b940f..3a5e17ad8 100644 --- a/tests/acceptance/cli_test.py +++ b/tests/acceptance/cli_test.py @@ -776,6 +776,31 @@ class CLITestCase(DockerClientTestCase): for service in services: assert self.lookup(container, service.name) + @v2_only() + def test_up_no_start(self): + self.base_dir = 'tests/fixtures/v2-full' + self.dispatch(['up', '--no-start'], None) + + services = self.project.get_services() + + default_network = self.project.networks.networks['default'].full_name + front_network = self.project.networks.networks['front'].full_name + networks = self.client.networks(names=[default_network, front_network]) + assert len(networks) == 2 + + for service in services: + containers = service.containers(stopped=True) + assert len(containers) == 1 + + container = containers[0] + assert not container.is_running + assert container.get('State.Status') == 'created' + + volumes = self.project.volumes.volumes + assert 'data' in volumes + volume = volumes['data'] + assert volume.exists() + @v2_only() def test_up_no_ansi(self): self.base_dir = 'tests/fixtures/v2-simple' From 42aa1c34475f121dbc0710312a6645ee1cc11a9a Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Mon, 28 Aug 2017 18:18:31 -0700 Subject: [PATCH 158/244] Reduce up() cyclomatic complexity Signed-off-by: Joffrey F --- compose/cli/main.py | 62 +++++++++++++++++++++++++-------------------- 1 file changed, 35 insertions(+), 27 deletions(-) diff --git a/compose/cli/main.py b/compose/cli/main.py index 21bf1f308..face38e6d 100644 --- a/compose/cli/main.py +++ b/compose/cli/main.py @@ -969,33 +969,10 @@ class TopLevelCommand(object): if cascade_stop: print("Aborting on container exit...") - - exit_code = 0 - if exit_value_from: - candidates = list(filter( - lambda c: c.service == exit_value_from, - attached_containers)) - if not candidates: - log.error( - 'No containers matching the spec "{0}" ' - 'were run.'.format(exit_value_from) - ) - exit_code = 2 - elif len(candidates) > 1: - exit_values = filter( - lambda e: e != 0, - [c.inspect()['State']['ExitCode'] for c in candidates] - ) - - exit_code = exit_values[0] - else: - exit_code = candidates[0].inspect()['State']['ExitCode'] - else: - for e in self.project.containers(service_names=options['SERVICE'], stopped=True): - if (not e.is_running and cascade_starter == e.name): - if not e.exit_code == 0: - exit_code = e.exit_code - break + all_containers = self.project.containers(service_names=options['SERVICE'], stopped=True) + exit_code = compute_exit_code( + exit_value_from, attached_containers, cascade_starter, all_containers + ) self.project.stop(service_names=service_names, timeout=timeout) sys.exit(exit_code) @@ -1016,6 +993,37 @@ class TopLevelCommand(object): print(get_version_info('full')) +def compute_exit_code(exit_value_from, attached_containers, cascade_starter, all_containers): + exit_code = 0 + if exit_value_from: + candidates = list(filter( + lambda c: c.service == exit_value_from, + attached_containers)) + if not candidates: + log.error( + 'No containers matching the spec "{0}" ' + 'were run.'.format(exit_value_from) + ) + exit_code = 2 + elif len(candidates) > 1: + exit_values = filter( + lambda e: e != 0, + [c.inspect()['State']['ExitCode'] for c in candidates] + ) + + exit_code = exit_values[0] + else: + exit_code = candidates[0].inspect()['State']['ExitCode'] + else: + for e in all_containers: + if (not e.is_running and cascade_starter == e.name): + if not e.exit_code == 0: + exit_code = e.exit_code + break + + return exit_code + + def convergence_strategy_from_opts(options): no_recreate = options['--no-recreate'] force_recreate = options['--force-recreate'] From 8c6f2217c45dd004b4fa10cca1cc7023a6f241ea Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Fri, 25 Aug 2017 18:09:06 -0700 Subject: [PATCH 159/244] Add support for extension fields in v2.x and v3.4 Signed-off-by: Joffrey F --- compose/config/config_schema_v2.0.json | 1 + compose/config/config_schema_v2.1.json | 1 + compose/config/config_schema_v2.2.json | 1 + compose/config/config_schema_v2.3.json | 1 + compose/config/config_schema_v3.4-beta.json | 1 + compose/config/validation.py | 10 ++++++++++ tests/unit/config/config_test.py | 14 +++++++++++++- 7 files changed, 28 insertions(+), 1 deletion(-) diff --git a/compose/config/config_schema_v2.0.json b/compose/config/config_schema_v2.0.json index 14bafab40..2ad62ac52 100644 --- a/compose/config/config_schema_v2.0.json +++ b/compose/config/config_schema_v2.0.json @@ -41,6 +41,7 @@ } }, + "patternProperties": {"^x-": {}}, "additionalProperties": false, "definitions": { diff --git a/compose/config/config_schema_v2.1.json b/compose/config/config_schema_v2.1.json index 8a5e12834..24e6ba02c 100644 --- a/compose/config/config_schema_v2.1.json +++ b/compose/config/config_schema_v2.1.json @@ -41,6 +41,7 @@ } }, + "patternProperties": {"^x-": {}}, "additionalProperties": false, "definitions": { diff --git a/compose/config/config_schema_v2.2.json b/compose/config/config_schema_v2.2.json index 58ba409ff..86fc5df95 100644 --- a/compose/config/config_schema_v2.2.json +++ b/compose/config/config_schema_v2.2.json @@ -41,6 +41,7 @@ } }, + "patternProperties": {"^x-": {}}, "additionalProperties": false, "definitions": { diff --git a/compose/config/config_schema_v2.3.json b/compose/config/config_schema_v2.3.json index 7a9bdfdf1..a790bb405 100644 --- a/compose/config/config_schema_v2.3.json +++ b/compose/config/config_schema_v2.3.json @@ -41,6 +41,7 @@ } }, + "patternProperties": {"^x-": {}}, "additionalProperties": false, "definitions": { diff --git a/compose/config/config_schema_v3.4-beta.json b/compose/config/config_schema_v3.4-beta.json index 190c05f2c..cba063202 100644 --- a/compose/config/config_schema_v3.4-beta.json +++ b/compose/config/config_schema_v3.4-beta.json @@ -64,6 +64,7 @@ } }, + "patternProperties": {"^x-": {}}, "additionalProperties": false, "definitions": { diff --git a/compose/config/validation.py b/compose/config/validation.py index 0b7961e5a..c6722a14d 100644 --- a/compose/config/validation.py +++ b/compose/config/validation.py @@ -239,6 +239,16 @@ def handle_error_for_schema_with_id(error, path): invalid_config_key = parse_key_from_error_msg(error) return get_unsupported_config_msg(path, invalid_config_key) + if schema_id.startswith('config_schema_v'): + invalid_config_key = parse_key_from_error_msg(error) + return ('Invalid top-level property "{key}". Valid top-level ' + 'sections for this Compose file are: {properties}, and ' + 'extensions starting with "x-".\n\n{explanation}').format( + key=invalid_config_key, + properties=', '.join(error.schema['properties'].keys()), + explanation=VERSION_EXPLANATION + ) + if not error.path: return '{}\n\n{}'.format(error.message, VERSION_EXPLANATION) diff --git a/tests/unit/config/config_test.py b/tests/unit/config/config_test.py index 644290157..14dd01179 100644 --- a/tests/unit/config/config_test.py +++ b/tests/unit/config/config_test.py @@ -251,7 +251,7 @@ class ConfigTest(unittest.TestCase): ) ) - assert 'Additional properties are not allowed' in excinfo.exconly() + assert 'Invalid top-level property "web"' in excinfo.exconly() assert VERSION_EXPLANATION in excinfo.exconly() def test_named_volume_config_empty(self): @@ -773,6 +773,18 @@ class ConfigTest(unittest.TestCase): assert services[1]['name'] == 'db' assert services[2]['name'] == 'web' + def test_load_with_extensions(self): + config_details = build_config_details({ + 'version': '2.3', + 'x-data': { + 'lambda': 3, + 'excess': [True, {}] + } + }) + + config_data = config.load(config_details) + assert config_data.services == [] + def test_config_build_configuration(self): service = config.load( build_config_details( From d48296213b76d891db2bfdd7a3fa2eab028d80c3 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Fri, 1 Sep 2017 12:14:56 -0700 Subject: [PATCH 160/244] Update release process with most recent changes Signed-off-by: Joffrey F --- project/RELEASE-PROCESS.md | 33 +++++++++++++++++++++------------ script/release/make-branch | 3 +-- 2 files changed, 22 insertions(+), 14 deletions(-) diff --git a/project/RELEASE-PROCESS.md b/project/RELEASE-PROCESS.md index c1834f2fb..5b30545f4 100644 --- a/project/RELEASE-PROCESS.md +++ b/project/RELEASE-PROCESS.md @@ -24,7 +24,7 @@ As part of this script you'll be asked to: If the next release will be an RC, append `-rcN`, e.g. `1.4.0-rc1`. -2. Write release notes in `CHANGES.md`. +2. Write release notes in `CHANGELOG.md`. Almost every feature enhancement should be mentioned, with the most visible/exciting ones first. Use descriptive sentences and give context @@ -67,16 +67,13 @@ Check out the bump branch and run the `build-binaries` script When prompted build the non-linux binaries and test them. -1. Download the osx binary from Bintray. Make sure that the latest Travis - build has finished, otherwise you'll be downloading an old binary. +1. Download the different platform binaries by running the following script: - https://dl.bintray.com/docker-compose/$BRANCH_NAME/ + `./script/release/download-binaries $VERSION` -2. Download the windows binary from AppVeyor + The binaries for Linux, OSX and Windows will be downloaded in the `binaries-$VERSION` folder. - https://ci.appveyor.com/project/docker/compose - -3. Draft a release from the tag on GitHub (the script will open the window for +3. Draft a release from the tag on GitHub (the `build-binaries` script will open the window for you) The tag will only be present on Github when you run the `push-release` @@ -87,18 +84,30 @@ When prompted build the non-linux binaries and test them. If you're a Mac or Windows user, the best way to install Compose and keep it up-to-date is **[Docker for Mac and Windows](https://www.docker.com/products/docker)**. - Note that Compose 1.9.0 requires Docker Engine 1.10.0 or later for version 2 of the Compose File format, and Docker Engine 1.9.1 or later for version 1. Docker for Mac and Windows will automatically install the latest version of Docker Engine for you. + Docker for Mac and Windows will automatically install the latest version of Docker Engine for you. Alternatively, you can use the usual commands to install or upgrade Compose: ``` - curl -L https://github.com/docker/compose/releases/download/1.9.0/docker-compose-`uname -s`-`uname -m` > /usr/local/bin/docker-compose + curl -L https://github.com/docker/compose/releases/download/1.16.0/docker-compose-`uname -s`-`uname -m` > /usr/local/bin/docker-compose chmod +x /usr/local/bin/docker-compose ``` See the [install docs](https://docs.docker.com/compose/install/) for more install options and instructions. - Here's what's new: + ## Compose file format compatibility matrix + + | Compose file format | Docker Engine | + | --- | --- | + | 3.3 | 17.06.0+ | + | 3.0 – 3.2 | 1.13.0+ | + | 2.3| 17.06.0+ | + | 2.2 | 1.13.0+ | + | 2.1 | 1.12.0+ | + | 2.0 | 1.10.0+ | + | 1.0 | 1.9.1+ | + + ## Changes ...release notes go here... @@ -119,7 +128,7 @@ When prompted build the non-linux binaries and test them. 9. Check that all the binaries download (following the install instructions) and run. -10. Email maintainers@dockerproject.org and engineering@docker.com about the new release. +10. Announce the release on the appropriate Slack channel(s). ## If it’s a stable release (not an RC) diff --git a/script/release/make-branch b/script/release/make-branch index 7ccf3f055..b8a0cd31e 100755 --- a/script/release/make-branch +++ b/script/release/make-branch @@ -65,8 +65,7 @@ git config "branch.${BRANCH}.release" $VERSION editor=${EDITOR:-vim} -echo "Update versions in docs/install.md, compose/__init__.py, script/run/run.sh" -$editor docs/install.md +echo "Update versions in compose/__init__.py, script/run/run.sh" $editor compose/__init__.py $editor script/run/run.sh From 158a78657854a3f82c94861752996a1951c8d02d Mon Sep 17 00:00:00 2001 From: Vincent Demeester Date: Tue, 19 Sep 2017 18:27:02 +0200 Subject: [PATCH 161/244] Sync composefile v3.2 schema with `docker/cli` Signed-off-by: Vincent Demeester --- compose/config/config_schema_v3.2.json | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/compose/config/config_schema_v3.2.json b/compose/config/config_schema_v3.2.json index b26b2c6c6..2ca8e92db 100644 --- a/compose/config/config_schema_v3.2.json +++ b/compose/config/config_schema_v3.2.json @@ -170,7 +170,8 @@ "type": "array", "items": { "oneOf": [ - {"type": ["string", "number"], "format": "ports"}, + {"type": "number", "format": "ports"}, + {"type": "string", "format": "ports"}, { "type": "object", "properties": { @@ -249,6 +250,7 @@ "source": {"type": "string"}, "target": {"type": "string"}, "read_only": {"type": "boolean"}, + "consistency": {"type": "string"}, "bind": { "type": "object", "properties": { From 1da5b54d75024bbe5ceae9a296200681caaa77fd Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Tue, 26 Sep 2017 16:25:05 -0700 Subject: [PATCH 162/244] Fix oneOf validator parser to correctly process uniqueItems errors Signed-off-by: Joffrey F --- compose/config/validation.py | 15 +++++++-------- tests/unit/config/config_test.py | 25 ++++++++++++++++++++----- 2 files changed, 27 insertions(+), 13 deletions(-) diff --git a/compose/config/validation.py b/compose/config/validation.py index c6722a14d..940775a20 100644 --- a/compose/config/validation.py +++ b/compose/config/validation.py @@ -325,7 +325,6 @@ def _parse_oneof_validator(error): """ types = [] for context in error.context: - if context.validator == 'oneOf': _, error_msg = _parse_oneof_validator(context) return path_string(context.path), error_msg @@ -337,6 +336,13 @@ def _parse_oneof_validator(error): invalid_config_key = parse_key_from_error_msg(context) return (None, "contains unsupported option: '{}'".format(invalid_config_key)) + if context.validator == 'uniqueItems': + return ( + path_string(context.path) if context.path else None, + "contains non-unique items, please remove duplicates from {}".format( + context.instance), + ) + if context.path: return ( path_string(context.path), @@ -345,13 +351,6 @@ def _parse_oneof_validator(error): _parse_valid_types_from_validator(context.validator_value)), ) - if context.validator == 'uniqueItems': - return ( - None, - "contains non unique items, please remove duplicates from {}".format( - context.instance), - ) - if context.validator == 'type': types.append(context.validator_value) diff --git a/tests/unit/config/config_test.py b/tests/unit/config/config_test.py index 14dd01179..de9a61302 100644 --- a/tests/unit/config/config_test.py +++ b/tests/unit/config/config_test.py @@ -581,6 +581,20 @@ class ConfigTest(unittest.TestCase): assert 'Invalid service name \'mong\\o\'' in excinfo.exconly() + def test_config_duplicate_cache_from_values_validation_error(self): + with pytest.raises(ConfigurationError) as exc: + config.load( + build_config_details({ + 'version': '2.3', + 'services': { + 'test': {'build': {'context': '.', 'cache_from': ['a', 'b', 'a']}} + } + + }) + ) + + assert 'build.cache_from contains non-unique items' in exc.exconly() + def test_load_with_multiple_files_v1(self): base_file = config.ConfigFile( 'base.yaml', @@ -2751,11 +2765,12 @@ class PortsTest(unittest.TestCase): def check_config(self, cfg): config.load( - build_config_details( - {'web': dict(image='busybox', **cfg)}, - 'working_dir', - 'filename.yml' - ) + build_config_details({ + 'version': '2.3', + 'services': { + 'web': dict(image='busybox', **cfg) + }, + }, 'working_dir', 'filename.yml') ) From 53928a17c0d36c3e1a9b835febdb8e88797196b7 Mon Sep 17 00:00:00 2001 From: French Ben Date: Mon, 18 Sep 2017 16:30:32 -0700 Subject: [PATCH 163/244] Simple patch to allow s390x images to be built Needs integration with CI and s390x machine integration Signed-off-by: French Ben --- Dockerfile.s390x | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100644 Dockerfile.s390x diff --git a/Dockerfile.s390x b/Dockerfile.s390x new file mode 100644 index 000000000..aa71e27bc --- /dev/null +++ b/Dockerfile.s390x @@ -0,0 +1,6 @@ +FROM s390x/python:3.6.2-slim +ARG COMPOSE_VERSION=1.16.1 + +RUN pip install --no-cache-dir docker-compose==$COMPOSE_VERSION + +ENTRYPOINT ["docker-compose"] From cb2d65556b316b58debeb6e5f5ccc3010d0171d7 Mon Sep 17 00:00:00 2001 From: French Ben Date: Tue, 19 Sep 2017 10:14:55 -0700 Subject: [PATCH 164/244] Use slim alpine instead of bulky debian Signed-off-by: French Ben --- Dockerfile.s390x | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/Dockerfile.s390x b/Dockerfile.s390x index aa71e27bc..3b19bb390 100644 --- a/Dockerfile.s390x +++ b/Dockerfile.s390x @@ -1,6 +1,15 @@ -FROM s390x/python:3.6.2-slim +FROM s390x/alpine:3.6 + ARG COMPOSE_VERSION=1.16.1 -RUN pip install --no-cache-dir docker-compose==$COMPOSE_VERSION +RUN apk add --update --no-cache \ + python \ + py-pip \ + && pip install --no-cache-dir docker-compose==$COMPOSE_VERSION \ + && rm -rf /var/cache/apk/* + +WORKDIR /data +VOLUME /data + ENTRYPOINT ["docker-compose"] From 78fe655dbce206e3e6e64ce3e62af94267782487 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Tue, 26 Sep 2017 17:19:29 -0700 Subject: [PATCH 165/244] Revert 3.4-beta temp rename Signed-off-by: Joffrey F --- ...nfig_schema_v3.4-beta.json => config_schema_v3.4.json} | 8 +++++--- compose/const.py | 2 +- docker-compose.spec | 4 ++-- 3 files changed, 8 insertions(+), 6 deletions(-) rename compose/config/{config_schema_v3.4-beta.json => config_schema_v3.4.json} (98%) diff --git a/compose/config/config_schema_v3.4-beta.json b/compose/config/config_schema_v3.4.json similarity index 98% rename from compose/config/config_schema_v3.4-beta.json rename to compose/config/config_schema_v3.4.json index cba063202..dae7d7d23 100644 --- a/compose/config/config_schema_v3.4-beta.json +++ b/compose/config/config_schema_v3.4.json @@ -1,6 +1,7 @@ + { "$schema": "http://json-schema.org/draft-04/schema#", - "id": "config_schema_v3.4-beta.json", + "id": "config_schema_v3.4.json", "type": "object", "required": ["version"], @@ -316,7 +317,7 @@ "additionalProperties": false, "properties": { "disable": {"type": "boolean"}, - "interval": {"type": "string"}, + "interval": {"type": "string", "format": "duration"}, "retries": {"type": "number"}, "test": { "oneOf": [ @@ -324,7 +325,8 @@ {"type": "array", "items": {"type": "string"}} ] }, - "timeout": {"type": "string"} + "timeout": {"type": "string", "format": "duration"}, + "start_period": {"type": "string", "format": "duration"} } }, "deployment": { diff --git a/compose/const.py b/compose/const.py index 809f7c7d4..b5970f82a 100644 --- a/compose/const.py +++ b/compose/const.py @@ -31,7 +31,7 @@ COMPOSEFILE_V3_0 = ComposeVersion('3.0') COMPOSEFILE_V3_1 = ComposeVersion('3.1') COMPOSEFILE_V3_2 = ComposeVersion('3.2') COMPOSEFILE_V3_3 = ComposeVersion('3.3') -COMPOSEFILE_V3_4 = ComposeVersion('3.4-beta') +COMPOSEFILE_V3_4 = ComposeVersion('3.4') API_VERSIONS = { COMPOSEFILE_V1: '1.21', diff --git a/docker-compose.spec b/docker-compose.spec index fe5651f6a..9c46421f0 100644 --- a/docker-compose.spec +++ b/docker-compose.spec @@ -63,8 +63,8 @@ exe = EXE(pyz, 'DATA' ), ( - 'compose/config/config_schema_v3.4-beta.json', - 'compose/config/config_schema_v3.4-beta.json', + 'compose/config/config_schema_v3.4.json', + 'compose/config/config_schema_v3.4.json', 'DATA' ), ( From 21d597c2b4b2c2293bf02a22db6664ae0af7bc62 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Wed, 27 Sep 2017 18:24:46 -0700 Subject: [PATCH 166/244] Avoid import ConfigurationError inside compose.utils (circular import) Signed-off-by: Joffrey F --- compose/config/config.py | 5 ++++- compose/utils.py | 3 +-- tests/unit/utils_test.py | 8 ++++++++ 3 files changed, 13 insertions(+), 3 deletions(-) diff --git a/compose/config/config.py b/compose/config/config.py index 0fddfd3a4..b90ab0305 100644 --- a/compose/config/config.py +++ b/compose/config/config.py @@ -762,7 +762,10 @@ def process_blkio_config(service_dict): for field in ['device_read_bps', 'device_write_bps']: if field in service_dict['blkio_config']: for v in service_dict['blkio_config'].get(field, []): - v['rate'] = parse_bytes(v.get('rate', 0)) + rate = v.get('rate', 0) + v['rate'] = parse_bytes(rate) + if v['rate'] is None: + raise ConfigurationError('Invalid format for bytes value: "{}"'.format(rate)) for field in ['device_read_iops', 'device_write_iops']: if field in service_dict['blkio_config']: diff --git a/compose/utils.py b/compose/utils.py index 1ede4d37d..197ae6eb2 100644 --- a/compose/utils.py +++ b/compose/utils.py @@ -12,7 +12,6 @@ import six from docker.errors import DockerException from docker.utils import parse_bytes as sdk_parse_bytes -from .config.errors import ConfigurationError from .errors import StreamParseError from .timeparse import MULTIPLIERS from .timeparse import timeparse @@ -143,4 +142,4 @@ def parse_bytes(n): try: return sdk_parse_bytes(n) except DockerException: - raise ConfigurationError('Invalid format for bytes value: {}'.format(n)) + return None diff --git a/tests/unit/utils_test.py b/tests/unit/utils_test.py index 85231957e..84becb975 100644 --- a/tests/unit/utils_test.py +++ b/tests/unit/utils_test.py @@ -60,3 +60,11 @@ class TestJsonStream(object): {'three': 'four'}, {'x': 2} ] + + +class TestParseBytes(object): + def test_parse_bytes(self): + assert utils.parse_bytes('123kb') == 123 * 1024 + assert utils.parse_bytes(123) == 123 + assert utils.parse_bytes('foobar') is None + assert utils.parse_bytes('123') == 123 From c7cdd63acf77147fcb3d53112c8c56931d671151 Mon Sep 17 00:00:00 2001 From: Marc van den Hoogen Date: Fri, 18 Aug 2017 13:40:11 +0200 Subject: [PATCH 167/244] Add shm_size to build-options (issue #3866) * Add shm_size to build configuration * Make it possible to enlarge/customize shm size during build * Value in bytes, or use string like "512M" or "1G" ... * Add to compose format 2.3 and (provisionally) >=3.5 format * Add automated test for shm_size in build-opts Signed-off-by: Marc van den Hoogen Made unit tests compatible with previously added shm_size build-option Signed-off-by: Marc van den Hoogen Also support shm_size build-opt when conf override Signed-off-by: Marc van den Hoogen Automated test for shm_size build-option Signed-off-by: Marc van den Hoogen Schema 3.4, add shm_size to schema 2.3, updated const.py Signed-off-by: Marc van den Hoogen Corrected typo in config_schema_v3.4 Signed-off-by: Marc van den Hoogen Add support for g/m/k units for shm_size in build-opts Signed-off-by: Marc van den Hoogen Reorder imports in service.py Signed-off-by: Marc van den Hoogen --- compose/config/config.py | 1 + compose/config/config_schema_v2.3.json | 3 +- compose/config/config_schema_v3.5.json | 542 ++++++++++++++++++ compose/const.py | 3 + compose/service.py | 2 + tests/acceptance/cli_test.py | 6 + tests/fixtures/build-shm-size/Dockerfile | 4 + .../build-shm-size/docker-compose.yml | 7 + tests/unit/service_test.py | 2 + 9 files changed, 569 insertions(+), 1 deletion(-) create mode 100644 compose/config/config_schema_v3.5.json create mode 100644 tests/fixtures/build-shm-size/Dockerfile create mode 100644 tests/fixtures/build-shm-size/docker-compose.yml diff --git a/compose/config/config.py b/compose/config/config.py index b90ab0305..f16dd01b3 100644 --- a/compose/config/config.py +++ b/compose/config/config.py @@ -1020,6 +1020,7 @@ def merge_build(output, base, override): md.merge_scalar('dockerfile') md.merge_scalar('network') md.merge_scalar('target') + md.merge_scalar('shm_size') md.merge_mapping('args', parse_build_arguments) md.merge_field('cache_from', merge_unique_items_lists, default=[]) md.merge_mapping('labels', parse_labels) diff --git a/compose/config/config_schema_v2.3.json b/compose/config/config_schema_v2.3.json index a790bb405..ceaf44954 100644 --- a/compose/config/config_schema_v2.3.json +++ b/compose/config/config_schema_v2.3.json @@ -91,7 +91,8 @@ "labels": {"$ref": "#/definitions/list_or_dict"}, "cache_from": {"$ref": "#/definitions/list_of_strings"}, "network": {"type": "string"}, - "target": {"type": "string"} + "target": {"type": "string"}, + "shm_size": {"type": ["integer", "string"]} }, "additionalProperties": false } diff --git a/compose/config/config_schema_v3.5.json b/compose/config/config_schema_v3.5.json new file mode 100644 index 000000000..fa95d6a24 --- /dev/null +++ b/compose/config/config_schema_v3.5.json @@ -0,0 +1,542 @@ +{ + "$schema": "http://json-schema.org/draft-04/schema#", + "id": "config_schema_v3.5.json", + "type": "object", + "required": ["version"], + + "properties": { + "version": { + "type": "string" + }, + + "services": { + "id": "#/properties/services", + "type": "object", + "patternProperties": { + "^[a-zA-Z0-9._-]+$": { + "$ref": "#/definitions/service" + } + }, + "additionalProperties": false + }, + + "networks": { + "id": "#/properties/networks", + "type": "object", + "patternProperties": { + "^[a-zA-Z0-9._-]+$": { + "$ref": "#/definitions/network" + } + } + }, + + "volumes": { + "id": "#/properties/volumes", + "type": "object", + "patternProperties": { + "^[a-zA-Z0-9._-]+$": { + "$ref": "#/definitions/volume" + } + }, + "additionalProperties": false + }, + + "secrets": { + "id": "#/properties/secrets", + "type": "object", + "patternProperties": { + "^[a-zA-Z0-9._-]+$": { + "$ref": "#/definitions/secret" + } + }, + "additionalProperties": false + }, + + "configs": { + "id": "#/properties/configs", + "type": "object", + "patternProperties": { + "^[a-zA-Z0-9._-]+$": { + "$ref": "#/definitions/config" + } + }, + "additionalProperties": false + } + }, + + "additionalProperties": false, + + "definitions": { + + "service": { + "id": "#/definitions/service", + "type": "object", + + "properties": { + "deploy": {"$ref": "#/definitions/deployment"}, + "build": { + "oneOf": [ + {"type": "string"}, + { + "type": "object", + "properties": { + "context": {"type": "string"}, + "dockerfile": {"type": "string"}, + "args": {"$ref": "#/definitions/list_or_dict"}, + "labels": {"$ref": "#/definitions/list_or_dict"}, + "cache_from": {"$ref": "#/definitions/list_of_strings"}, + "network": {"type": "string"}, + "target": {"type": "string"}, + "shm_size": {"type": ["integer", "string"]} + }, + "additionalProperties": false + } + ] + }, + "cap_add": {"type": "array", "items": {"type": "string"}, "uniqueItems": true}, + "cap_drop": {"type": "array", "items": {"type": "string"}, "uniqueItems": true}, + "cgroup_parent": {"type": "string"}, + "command": { + "oneOf": [ + {"type": "string"}, + {"type": "array", "items": {"type": "string"}} + ] + }, + "configs": { + "type": "array", + "items": { + "oneOf": [ + {"type": "string"}, + { + "type": "object", + "properties": { + "source": {"type": "string"}, + "target": {"type": "string"}, + "uid": {"type": "string"}, + "gid": {"type": "string"}, + "mode": {"type": "number"} + } + } + ] + } + }, + "container_name": {"type": "string"}, + "credential_spec": {"type": "object", "properties": { + "file": {"type": "string"}, + "registry": {"type": "string"} + }}, + "depends_on": {"$ref": "#/definitions/list_of_strings"}, + "devices": {"type": "array", "items": {"type": "string"}, "uniqueItems": true}, + "dns": {"$ref": "#/definitions/string_or_list"}, + "dns_search": {"$ref": "#/definitions/string_or_list"}, + "domainname": {"type": "string"}, + "entrypoint": { + "oneOf": [ + {"type": "string"}, + {"type": "array", "items": {"type": "string"}} + ] + }, + "env_file": {"$ref": "#/definitions/string_or_list"}, + "environment": {"$ref": "#/definitions/list_or_dict"}, + + "expose": { + "type": "array", + "items": { + "type": ["string", "number"], + "format": "expose" + }, + "uniqueItems": true + }, + + "external_links": {"type": "array", "items": {"type": "string"}, "uniqueItems": true}, + "extra_hosts": {"$ref": "#/definitions/list_or_dict"}, + "healthcheck": {"$ref": "#/definitions/healthcheck"}, + "hostname": {"type": "string"}, + "image": {"type": "string"}, + "ipc": {"type": "string"}, + "labels": {"$ref": "#/definitions/list_or_dict"}, + "links": {"type": "array", "items": {"type": "string"}, "uniqueItems": true}, + + "logging": { + "type": "object", + + "properties": { + "driver": {"type": "string"}, + "options": { + "type": "object", + "patternProperties": { + "^.+$": {"type": ["string", "number", "null"]} + } + } + }, + "additionalProperties": false + }, + + "mac_address": {"type": "string"}, + "network_mode": {"type": "string"}, + + "networks": { + "oneOf": [ + {"$ref": "#/definitions/list_of_strings"}, + { + "type": "object", + "patternProperties": { + "^[a-zA-Z0-9._-]+$": { + "oneOf": [ + { + "type": "object", + "properties": { + "aliases": {"$ref": "#/definitions/list_of_strings"}, + "ipv4_address": {"type": "string"}, + "ipv6_address": {"type": "string"} + }, + "additionalProperties": false + }, + {"type": "null"} + ] + } + }, + "additionalProperties": false + } + ] + }, + "pid": {"type": ["string", "null"]}, + + "ports": { + "type": "array", + "items": { + "oneOf": [ + {"type": "number", "format": "ports"}, + {"type": "string", "format": "ports"}, + { + "type": "object", + "properties": { + "mode": {"type": "string"}, + "target": {"type": "integer"}, + "published": {"type": "integer"}, + "protocol": {"type": "string"} + }, + "additionalProperties": false + } + ] + }, + "uniqueItems": true + }, + + "privileged": {"type": "boolean"}, + "read_only": {"type": "boolean"}, + "restart": {"type": "string"}, + "security_opt": {"type": "array", "items": {"type": "string"}, "uniqueItems": true}, + "shm_size": {"type": ["number", "string"]}, + "secrets": { + "type": "array", + "items": { + "oneOf": [ + {"type": "string"}, + { + "type": "object", + "properties": { + "source": {"type": "string"}, + "target": {"type": "string"}, + "uid": {"type": "string"}, + "gid": {"type": "string"}, + "mode": {"type": "number"} + } + } + ] + } + }, + "sysctls": {"$ref": "#/definitions/list_or_dict"}, + "stdin_open": {"type": "boolean"}, + "stop_grace_period": {"type": "string", "format": "duration"}, + "stop_signal": {"type": "string"}, + "tmpfs": {"$ref": "#/definitions/string_or_list"}, + "tty": {"type": "boolean"}, + "ulimits": { + "type": "object", + "patternProperties": { + "^[a-z]+$": { + "oneOf": [ + {"type": "integer"}, + { + "type":"object", + "properties": { + "hard": {"type": "integer"}, + "soft": {"type": "integer"} + }, + "required": ["soft", "hard"], + "additionalProperties": false + } + ] + } + } + }, + "user": {"type": "string"}, + "userns_mode": {"type": "string"}, + "volumes": { + "type": "array", + "items": { + "oneOf": [ + {"type": "string"}, + { + "type": "object", + "required": ["type"], + "properties": { + "type": {"type": "string"}, + "source": {"type": "string"}, + "target": {"type": "string"}, + "read_only": {"type": "boolean"}, + "consistency": {"type": "string"}, + "bind": { + "type": "object", + "properties": { + "propagation": {"type": "string"} + } + }, + "volume": { + "type": "object", + "properties": { + "nocopy": {"type": "boolean"} + } + } + } + } + ], + "uniqueItems": true + } + }, + "working_dir": {"type": "string"} + }, + "additionalProperties": false + }, + + "healthcheck": { + "id": "#/definitions/healthcheck", + "type": "object", + "additionalProperties": false, + "properties": { + "disable": {"type": "boolean"}, + "interval": {"type": "string"}, + "retries": {"type": "number"}, + "test": { + "oneOf": [ + {"type": "string"}, + {"type": "array", "items": {"type": "string"}} + ] + }, + "timeout": {"type": "string"} + } + }, + "deployment": { + "id": "#/definitions/deployment", + "type": ["object", "null"], + "properties": { + "mode": {"type": "string"}, + "endpoint_mode": {"type": "string"}, + "replicas": {"type": "integer"}, + "labels": {"$ref": "#/definitions/list_or_dict"}, + "update_config": { + "type": "object", + "properties": { + "parallelism": {"type": "integer"}, + "delay": {"type": "string", "format": "duration"}, + "failure_action": {"type": "string"}, + "monitor": {"type": "string", "format": "duration"}, + "max_failure_ratio": {"type": "number"}, + "order": {"type": "string", "enum": [ + "start-first", "stop-first" + ]} + }, + "additionalProperties": false + }, + "resources": { + "type": "object", + "properties": { + "limits": {"$ref": "#/definitions/resource"}, + "reservations": {"$ref": "#/definitions/resource"} + }, + "additionalProperties": false + }, + "restart_policy": { + "type": "object", + "properties": { + "condition": {"type": "string"}, + "delay": {"type": "string", "format": "duration"}, + "max_attempts": {"type": "integer"}, + "window": {"type": "string", "format": "duration"} + }, + "additionalProperties": false + }, + "placement": { + "type": "object", + "properties": { + "constraints": {"type": "array", "items": {"type": "string"}}, + "preferences": { + "type": "array", + "items": { + "type": "object", + "properties": { + "spread": {"type": "string"} + }, + "additionalProperties": false + } + } + }, + "additionalProperties": false + } + }, + "additionalProperties": false + }, + + "resource": { + "id": "#/definitions/resource", + "type": "object", + "properties": { + "cpus": {"type": "string"}, + "memory": {"type": "string"} + }, + "additionalProperties": false + }, + + "network": { + "id": "#/definitions/network", + "type": ["object", "null"], + "properties": { + "driver": {"type": "string"}, + "driver_opts": { + "type": "object", + "patternProperties": { + "^.+$": {"type": ["string", "number"]} + } + }, + "ipam": { + "type": "object", + "properties": { + "driver": {"type": "string"}, + "config": { + "type": "array", + "items": { + "type": "object", + "properties": { + "subnet": {"type": "string"} + }, + "additionalProperties": false + } + } + }, + "additionalProperties": false + }, + "external": { + "type": ["boolean", "object"], + "properties": { + "name": {"type": "string"} + }, + "additionalProperties": false + }, + "internal": {"type": "boolean"}, + "attachable": {"type": "boolean"}, + "labels": {"$ref": "#/definitions/list_or_dict"} + }, + "additionalProperties": false + }, + + "volume": { + "id": "#/definitions/volume", + "type": ["object", "null"], + "properties": { + "name": {"type": "string"}, + "driver": {"type": "string"}, + "driver_opts": { + "type": "object", + "patternProperties": { + "^.+$": {"type": ["string", "number"]} + } + }, + "external": { + "type": ["boolean", "object"], + "properties": { + "name": {"type": "string"} + }, + "additionalProperties": false + }, + "labels": {"$ref": "#/definitions/list_or_dict"} + }, + "additionalProperties": false + }, + + "secret": { + "id": "#/definitions/secret", + "type": "object", + "properties": { + "file": {"type": "string"}, + "external": { + "type": ["boolean", "object"], + "properties": { + "name": {"type": "string"} + } + }, + "labels": {"$ref": "#/definitions/list_or_dict"} + }, + "additionalProperties": false + }, + + "config": { + "id": "#/definitions/config", + "type": "object", + "properties": { + "file": {"type": "string"}, + "external": { + "type": ["boolean", "object"], + "properties": { + "name": {"type": "string"} + } + }, + "labels": {"$ref": "#/definitions/list_or_dict"} + }, + "additionalProperties": false + }, + + "string_or_list": { + "oneOf": [ + {"type": "string"}, + {"$ref": "#/definitions/list_of_strings"} + ] + }, + + "list_of_strings": { + "type": "array", + "items": {"type": "string"}, + "uniqueItems": true + }, + + "list_or_dict": { + "oneOf": [ + { + "type": "object", + "patternProperties": { + ".+": { + "type": ["string", "number", "null"] + } + }, + "additionalProperties": false + }, + {"type": "array", "items": {"type": "string"}, "uniqueItems": true} + ] + }, + + "constraints": { + "service": { + "id": "#/definitions/constraints/service", + "anyOf": [ + {"required": ["build"]}, + {"required": ["image"]} + ], + "properties": { + "build": { + "required": ["context"] + } + } + } + } + } +} diff --git a/compose/const.py b/compose/const.py index b5970f82a..2ac08b89a 100644 --- a/compose/const.py +++ b/compose/const.py @@ -32,6 +32,7 @@ COMPOSEFILE_V3_1 = ComposeVersion('3.1') COMPOSEFILE_V3_2 = ComposeVersion('3.2') COMPOSEFILE_V3_3 = ComposeVersion('3.3') COMPOSEFILE_V3_4 = ComposeVersion('3.4') +COMPOSEFILE_V3_5 = ComposeVersion('3.5') API_VERSIONS = { COMPOSEFILE_V1: '1.21', @@ -44,6 +45,7 @@ API_VERSIONS = { COMPOSEFILE_V3_2: '1.25', COMPOSEFILE_V3_3: '1.30', COMPOSEFILE_V3_4: '1.30', + COMPOSEFILE_V3_5: '1.30', } API_VERSION_TO_ENGINE_VERSION = { @@ -57,4 +59,5 @@ API_VERSION_TO_ENGINE_VERSION = { API_VERSIONS[COMPOSEFILE_V3_2]: '1.13.0', API_VERSIONS[COMPOSEFILE_V3_3]: '17.06.0', API_VERSIONS[COMPOSEFILE_V3_4]: '17.06.0', + API_VERSIONS[COMPOSEFILE_V3_5]: '17.06.0', } diff --git a/compose/service.py b/compose/service.py index 2829240f2..28c032763 100644 --- a/compose/service.py +++ b/compose/service.py @@ -43,6 +43,7 @@ from .parallel import parallel_execute from .progress_stream import stream_output from .progress_stream import StreamOutputError from .utils import json_hash +from .utils import parse_bytes from .utils import parse_seconds_float @@ -916,6 +917,7 @@ class Service(object): buildargs=build_args, network_mode=build_opts.get('network', None), target=build_opts.get('target', None), + shmsize=parse_bytes(build_opts.get('shm_size')) if build_opts.get('shm_size') else None, ) try: diff --git a/tests/acceptance/cli_test.py b/tests/acceptance/cli_test.py index 3a5e17ad8..ca4bd9ee7 100644 --- a/tests/acceptance/cli_test.py +++ b/tests/acceptance/cli_test.py @@ -531,6 +531,12 @@ class CLITestCase(DockerClientTestCase): ] assert not containers + def test_build_shm_size_build_option(self): + pull_busybox(self.client) + self.base_dir = 'tests/fixtures/build-shm-size' + result = self.dispatch(['build', '--no-cache'], None) + assert 'shm_size: 96' in result.stdout + def test_bundle_with_digests(self): self.base_dir = 'tests/fixtures/bundle-with-digests/' tmpdir = py.test.ensuretemp('cli_test_bundle') diff --git a/tests/fixtures/build-shm-size/Dockerfile b/tests/fixtures/build-shm-size/Dockerfile new file mode 100644 index 000000000..f91733d63 --- /dev/null +++ b/tests/fixtures/build-shm-size/Dockerfile @@ -0,0 +1,4 @@ +FROM busybox + +# Report the shm_size (through the size of /dev/shm) +RUN echo "shm_size:" $(df -h /dev/shm | tail -n 1 | awk '{print $2}') diff --git a/tests/fixtures/build-shm-size/docker-compose.yml b/tests/fixtures/build-shm-size/docker-compose.yml new file mode 100644 index 000000000..238a51322 --- /dev/null +++ b/tests/fixtures/build-shm-size/docker-compose.yml @@ -0,0 +1,7 @@ +version: '3.5' + +services: + custom_shm_size: + build: + context: . + shm_size: 100663296 # =96M diff --git a/tests/unit/service_test.py b/tests/unit/service_test.py index 0293695ab..43ccf081c 100644 --- a/tests/unit/service_test.py +++ b/tests/unit/service_test.py @@ -475,6 +475,7 @@ class ServiceTest(unittest.TestCase): cache_from=None, network_mode=None, target=None, + shmsize=None, ) def test_ensure_image_exists_no_build(self): @@ -515,6 +516,7 @@ class ServiceTest(unittest.TestCase): cache_from=None, network_mode=None, target=None, + shmsize=None ) def test_build_does_not_pull(self): From 3436145764eb22263dbf9f6b5f2ff6086f767472 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Fri, 13 Oct 2017 15:24:34 -0700 Subject: [PATCH 168/244] Temporary xfails for engine bug Signed-off-by: Joffrey F --- tests/acceptance/cli_test.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/tests/acceptance/cli_test.py b/tests/acceptance/cli_test.py index ca4bd9ee7..b598d99d5 100644 --- a/tests/acceptance/cli_test.py +++ b/tests/acceptance/cli_test.py @@ -14,7 +14,7 @@ from collections import Counter from collections import namedtuple from operator import attrgetter -import py +import pytest import six import yaml from docker import errors @@ -504,6 +504,7 @@ class CLITestCase(DockerClientTestCase): assert BUILD_CACHE_TEXT not in result.stdout assert BUILD_PULL_TEXT in result.stdout + @pytest.mark.xfail(reason='17.10.0 RC bug remove after GA https://github.com/moby/moby/issues/35116') def test_build_failed(self): self.base_dir = 'tests/fixtures/simple-failing-dockerfile' self.dispatch(['build', 'simple'], returncode=1) @@ -517,6 +518,7 @@ class CLITestCase(DockerClientTestCase): ] assert len(containers) == 1 + @pytest.mark.xfail(reason='17.10.0 RC bug remove after GA https://github.com/moby/moby/issues/35116') def test_build_failed_forcerm(self): self.base_dir = 'tests/fixtures/simple-failing-dockerfile' self.dispatch(['build', '--force-rm', 'simple'], returncode=1) @@ -539,7 +541,7 @@ class CLITestCase(DockerClientTestCase): def test_bundle_with_digests(self): self.base_dir = 'tests/fixtures/bundle-with-digests/' - tmpdir = py.test.ensuretemp('cli_test_bundle') + tmpdir = pytest.ensuretemp('cli_test_bundle') self.addCleanup(tmpdir.remove) filename = str(tmpdir.join('example.dab')) @@ -1403,7 +1405,7 @@ class CLITestCase(DockerClientTestCase): [u'/bin/true'], ) - @py.test.mark.skipif(SWARM_SKIP_RM_VOLUMES, reason='Swarm DELETE /containers/ bug') + @pytest.mark.skipif(SWARM_SKIP_RM_VOLUMES, reason='Swarm DELETE /containers/ bug') def test_run_rm(self): self.base_dir = 'tests/fixtures/volume' proc = start_process(self.base_dir, ['run', '--rm', 'test']) From 8c38651196c626c64ff22a22a8d606ed9d88e305 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Wed, 27 Sep 2017 17:10:13 -0700 Subject: [PATCH 169/244] Mount with same container path and different mode should override Signed-off-by: Joffrey F --- compose/config/config.py | 35 ++++++++++++++++++++--------- tests/unit/config/config_test.py | 38 +++++++++++++++++++++++++++++--- 2 files changed, 60 insertions(+), 13 deletions(-) diff --git a/compose/config/config.py b/compose/config/config.py index f16dd01b3..948e2376e 100644 --- a/compose/config/config.py +++ b/compose/config/config.py @@ -1137,24 +1137,30 @@ def resolve_volume_paths(working_dir, service_dict): def resolve_volume_path(working_dir, volume): + mount_params = None if isinstance(volume, dict): - host_path = volume.get('source') container_path = volume.get('target') + host_path = volume.get('source') + mode = None if host_path: if volume.get('read_only'): - container_path += ':ro' + mode = 'ro' if volume.get('volume', {}).get('nocopy'): - container_path += ':nocopy' + mode = 'nocopy' + mount_params = (host_path, mode) else: - container_path, host_path = split_path_mapping(volume) + container_path, mount_params = split_path_mapping(volume) - if host_path is not None: + if mount_params is not None: + host_path, mode = mount_params + if host_path is None: + return container_path if host_path.startswith('.'): host_path = expand_path(working_dir, host_path) host_path = os.path.expanduser(host_path) - return u"{}:{}".format(host_path, container_path) - else: - return container_path + return u"{}:{}{}".format(host_path, container_path, (':' + mode if mode else '')) + + return container_path def normalize_build(service_dict, working_dir, environment): @@ -1234,7 +1240,12 @@ def split_path_mapping(volume_path): if ':' in volume_config: (host, container) = volume_config.split(':', 1) - return (container, drive + host) + container_drive, container_path = splitdrive(container) + mode = None + if ':' in container_path: + container_path, mode = container_path.rsplit(':', 1) + + return (container_drive + container_path, (drive + host, mode)) else: return (volume_path, None) @@ -1246,7 +1257,11 @@ def join_path_mapping(pair): elif host is None: return container else: - return ":".join((host, container)) + host, mode = host + result = ":".join((host, container)) + if mode: + result += ":" + mode + return result def expand_path(working_dir, path): diff --git a/tests/unit/config/config_test.py b/tests/unit/config/config_test.py index de9a61302..c5e40130d 100644 --- a/tests/unit/config/config_test.py +++ b/tests/unit/config/config_test.py @@ -1101,6 +1101,38 @@ class ConfigTest(unittest.TestCase): ['/anonymous', '/c:/b:rw', 'vol:/x:ro'] ) + @mock.patch.dict(os.environ) + def test_volume_mode_override(self): + os.environ['COMPOSE_CONVERT_WINDOWS_PATHS'] = 'true' + base_file = config.ConfigFile( + 'base.yaml', + { + 'version': '2.3', + 'services': { + 'web': { + 'image': 'example/web', + 'volumes': ['/c:/b:rw'] + } + }, + } + ) + + override_file = config.ConfigFile( + 'override.yaml', + { + 'version': '2.3', + 'services': { + 'web': { + 'volumes': ['/c:/b:ro'] + } + } + } + ) + details = config.ConfigDetails('.', [base_file, override_file]) + service_dicts = config.load(details).services + svc_volumes = list(map(lambda v: v.repr(), service_dicts[0]['volumes'])) + assert svc_volumes == ['/c:/b:ro'] + def test_undeclared_volume_v2(self): base_file = config.ConfigFile( 'base.yaml', @@ -4018,7 +4050,7 @@ class VolumePathTest(unittest.TestCase): def test_split_path_mapping_with_windows_path(self): host_path = "c:\\Users\\msamblanet\\Documents\\anvil\\connect\\config" windows_volume_path = host_path + ":/opt/connect/config:ro" - expected_mapping = ("/opt/connect/config:ro", host_path) + expected_mapping = ("/opt/connect/config", (host_path, 'ro')) mapping = config.split_path_mapping(windows_volume_path) assert mapping == expected_mapping @@ -4026,7 +4058,7 @@ class VolumePathTest(unittest.TestCase): def test_split_path_mapping_with_windows_path_in_container(self): host_path = 'c:\\Users\\remilia\\data' container_path = 'c:\\scarletdevil\\data' - expected_mapping = (container_path, host_path) + expected_mapping = (container_path, (host_path, None)) mapping = config.split_path_mapping('{0}:{1}'.format(host_path, container_path)) assert mapping == expected_mapping @@ -4034,7 +4066,7 @@ class VolumePathTest(unittest.TestCase): def test_split_path_mapping_with_root_mount(self): host_path = '/' container_path = '/var/hostroot' - expected_mapping = (container_path, host_path) + expected_mapping = (container_path, (host_path, None)) mapping = config.split_path_mapping('{0}:{1}'.format(host_path, container_path)) assert mapping == expected_mapping From 18df4915f21fb43100b24f181e851a3f0a14eb9a Mon Sep 17 00:00:00 2001 From: Andrea Giardini Date: Wed, 20 Sep 2017 23:05:29 +0200 Subject: [PATCH 170/244] Fix secret location with absolute paths Signed-off-by: Andrea Giardini --- compose/service.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/compose/service.py b/compose/service.py index 28c032763..aecafc8ca 100644 --- a/compose/service.py +++ b/compose/service.py @@ -881,9 +881,12 @@ class Service(object): def get_secret_volumes(self): def build_spec(secret): - target = '{}/{}'.format( - const.SECRETS_PATH, - secret['secret'].target or secret['secret'].source) + if secret['secret'].target is not None and secret['secret'].target.startswith('/'): + target = secret['secret'].target + else: + target = '{}/{}'.format( + const.SECRETS_PATH, + secret['secret'].target or secret['secret'].source) return VolumeSpec(secret['file'], target, 'ro') return [build_spec(secret) for secret in self.secrets] From c4a8cb30ffba475169b01b8d0b653dc07fa5ab60 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Fri, 13 Oct 2017 17:02:01 -0700 Subject: [PATCH 171/244] Add get_secret_volumes unit tests Signed-off-by: Joffrey F --- compose/service.py | 12 ++++----- tests/unit/service_test.py | 55 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 61 insertions(+), 6 deletions(-) diff --git a/compose/service.py b/compose/service.py index aecafc8ca..1a18c6654 100644 --- a/compose/service.py +++ b/compose/service.py @@ -881,12 +881,12 @@ class Service(object): def get_secret_volumes(self): def build_spec(secret): - if secret['secret'].target is not None and secret['secret'].target.startswith('/'): - target = secret['secret'].target - else: - target = '{}/{}'.format( - const.SECRETS_PATH, - secret['secret'].target or secret['secret'].source) + target = secret['secret'].target + if target is None: + target = '{}/{}'.format(const.SECRETS_PATH, secret['secret'].source) + elif not os.path.isabs(target): + target = '{}/{}'.format(const.SECRETS_PATH, target) + return VolumeSpec(secret['file'], target, 'ro') return [build_spec(secret) for secret in self.secrets] diff --git a/tests/unit/service_test.py b/tests/unit/service_test.py index 43ccf081c..7d61807ba 100644 --- a/tests/unit/service_test.py +++ b/tests/unit/service_test.py @@ -9,12 +9,14 @@ from .. import mock from .. import unittest from compose.config.errors import DependencyError from compose.config.types import ServicePort +from compose.config.types import ServiceSecret from compose.config.types import VolumeFromSpec from compose.config.types import VolumeSpec from compose.const import LABEL_CONFIG_HASH from compose.const import LABEL_ONE_OFF from compose.const import LABEL_PROJECT from compose.const import LABEL_SERVICE +from compose.const import SECRETS_PATH from compose.container import Container from compose.project import OneOffFilter from compose.service import build_ulimits @@ -1089,3 +1091,56 @@ class ServiceVolumesTest(unittest.TestCase): self.assertEqual( self.mock_client.create_host_config.call_args[1]['binds'], [volume]) + + +class ServiceSecretTest(unittest.TestCase): + def setUp(self): + self.mock_client = mock.create_autospec(docker.APIClient) + + def test_get_secret_volumes(self): + secret1 = { + 'secret': ServiceSecret.parse({'source': 'secret1', 'target': 'b.txt'}), + 'file': 'a.txt' + } + service = Service( + 'web', + client=self.mock_client, + image='busybox', + secrets=[secret1] + ) + volumes = service.get_secret_volumes() + + assert volumes[0].external == secret1['file'] + assert volumes[0].internal == '{}/{}'.format(SECRETS_PATH, secret1['secret'].target) + + def test_get_secret_volumes_abspath(self): + secret1 = { + 'secret': ServiceSecret.parse({'source': 'secret1', 'target': '/d.txt'}), + 'file': 'c.txt' + } + service = Service( + 'web', + client=self.mock_client, + image='busybox', + secrets=[secret1] + ) + volumes = service.get_secret_volumes() + + assert volumes[0].external == secret1['file'] + assert volumes[0].internal == secret1['secret'].target + + def test_get_secret_volumes_no_target(self): + secret1 = { + 'secret': ServiceSecret.parse({'source': 'secret1'}), + 'file': 'c.txt' + } + service = Service( + 'web', + client=self.mock_client, + image='busybox', + secrets=[secret1] + ) + volumes = service.get_secret_volumes() + + assert volumes[0].external == secret1['file'] + assert volumes[0].internal == '{}/{}'.format(SECRETS_PATH, secret1['secret'].source) From 0d0da0760c3422e6164ec39c3e8edd77ac6d28d3 Mon Sep 17 00:00:00 2001 From: Guillermo Arribas Date: Fri, 6 Oct 2017 19:12:59 -0300 Subject: [PATCH 172/244] Build labels option: array form produces unmarshal error (fixes #5183) Signed-off-by: Guillermo Arribas --- compose/service.py | 3 ++- tests/integration/service_test.py | 19 ++++++++++++++++++- tests/unit/service_test.py | 4 ++-- 3 files changed, 22 insertions(+), 4 deletions(-) diff --git a/compose/service.py b/compose/service.py index 1a18c6654..e2f72aa5a 100644 --- a/compose/service.py +++ b/compose/service.py @@ -23,6 +23,7 @@ from . import const from . import progress_stream from .config import DOCKER_CONFIG_KEYS from .config import merge_environment +from .config.config import parse_labels from .config.errors import DependencyError from .config.types import ServicePort from .config.types import VolumeSpec @@ -916,7 +917,7 @@ class Service(object): nocache=no_cache, dockerfile=build_opts.get('dockerfile', None), cache_from=build_opts.get('cache_from', None), - labels=build_opts.get('labels', None), + labels=parse_labels(build_opts.get('labels', None)), buildargs=build_args, network_mode=build_opts.get('network', None), target=build_opts.get('target', None), diff --git a/tests/integration/service_test.py b/tests/integration/service_test.py index 84b54fe41..a71bc407c 100644 --- a/tests/integration/service_test.py +++ b/tests/integration/service_test.py @@ -761,7 +761,7 @@ class ServiceTest(DockerClientTestCase): assert service.image() assert "build_version=2" in service.image()['ContainerConfig']['Cmd'] - def test_build_with_build_labels(self): + def test_build_with_build_labels_dict(self): base_dir = tempfile.mkdtemp() self.addCleanup(shutil.rmtree, base_dir) @@ -778,6 +778,23 @@ class ServiceTest(DockerClientTestCase): assert service.image() assert service.image()['Config']['Labels']['com.docker.compose.test'] == 'true' + def test_build_with_build_labels_list(self): + base_dir = tempfile.mkdtemp() + self.addCleanup(shutil.rmtree, base_dir) + + with open(os.path.join(base_dir, 'Dockerfile'), 'w') as f: + f.write('FROM busybox\n') + + service = self.create_service('buildlabels', build={ + 'context': text_type(base_dir), + 'labels': ['com.docker.compose.test=true'] + }) + service.build() + self.addCleanup(self.client.remove_image, service.image_name) + + assert service.image() + assert service.image()['Config']['Labels']['com.docker.compose.test'] == 'true' + @no_cluster('Container networks not on Swarm') def test_build_with_network(self): base_dir = tempfile.mkdtemp() diff --git a/tests/unit/service_test.py b/tests/unit/service_test.py index 7d61807ba..5c5c2bf67 100644 --- a/tests/unit/service_test.py +++ b/tests/unit/service_test.py @@ -473,7 +473,7 @@ class ServiceTest(unittest.TestCase): nocache=False, rm=True, buildargs={}, - labels=None, + labels={}, cache_from=None, network_mode=None, target=None, @@ -514,7 +514,7 @@ class ServiceTest(unittest.TestCase): nocache=False, rm=True, buildargs={}, - labels=None, + labels={}, cache_from=None, network_mode=None, target=None, From f680d46d9af868163d3a4887678d3531e88001e0 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Mon, 16 Oct 2017 11:43:06 -0700 Subject: [PATCH 173/244] Move build labels parsing to config module Signed-off-by: Joffrey F --- compose/config/config.py | 12 ++++++------ compose/service.py | 3 +-- tests/integration/service_test.py | 19 +------------------ tests/unit/config/config_test.py | 24 +++++++++++++++++++++++- tests/unit/service_test.py | 4 ++-- 5 files changed, 33 insertions(+), 29 deletions(-) diff --git a/compose/config/config.py b/compose/config/config.py index 948e2376e..68b2be3a6 100644 --- a/compose/config/config.py +++ b/compose/config/config.py @@ -707,16 +707,16 @@ def process_service(service_config): if 'build' in service_dict: if isinstance(service_dict['build'], six.string_types): service_dict['build'] = resolve_build_path(working_dir, service_dict['build']) - elif isinstance(service_dict['build'], dict) and 'context' in service_dict['build']: - path = service_dict['build']['context'] - service_dict['build']['context'] = resolve_build_path(working_dir, path) + elif isinstance(service_dict['build'], dict): + if 'context' in service_dict['build']: + path = service_dict['build']['context'] + service_dict['build']['context'] = resolve_build_path(working_dir, path) + if 'labels' in service_dict['build']: + service_dict['build']['labels'] = parse_labels(service_dict['build']['labels']) if 'volumes' in service_dict and service_dict.get('volume_driver') is None: service_dict['volumes'] = resolve_volume_paths(working_dir, service_dict) - if 'labels' in service_dict: - service_dict['labels'] = parse_labels(service_dict['labels']) - if 'sysctls' in service_dict: service_dict['sysctls'] = build_string_dict(parse_sysctls(service_dict['sysctls'])) diff --git a/compose/service.py b/compose/service.py index e2f72aa5a..1a18c6654 100644 --- a/compose/service.py +++ b/compose/service.py @@ -23,7 +23,6 @@ from . import const from . import progress_stream from .config import DOCKER_CONFIG_KEYS from .config import merge_environment -from .config.config import parse_labels from .config.errors import DependencyError from .config.types import ServicePort from .config.types import VolumeSpec @@ -917,7 +916,7 @@ class Service(object): nocache=no_cache, dockerfile=build_opts.get('dockerfile', None), cache_from=build_opts.get('cache_from', None), - labels=parse_labels(build_opts.get('labels', None)), + labels=build_opts.get('labels', None), buildargs=build_args, network_mode=build_opts.get('network', None), target=build_opts.get('target', None), diff --git a/tests/integration/service_test.py b/tests/integration/service_test.py index a71bc407c..84b54fe41 100644 --- a/tests/integration/service_test.py +++ b/tests/integration/service_test.py @@ -761,7 +761,7 @@ class ServiceTest(DockerClientTestCase): assert service.image() assert "build_version=2" in service.image()['ContainerConfig']['Cmd'] - def test_build_with_build_labels_dict(self): + def test_build_with_build_labels(self): base_dir = tempfile.mkdtemp() self.addCleanup(shutil.rmtree, base_dir) @@ -778,23 +778,6 @@ class ServiceTest(DockerClientTestCase): assert service.image() assert service.image()['Config']['Labels']['com.docker.compose.test'] == 'true' - def test_build_with_build_labels_list(self): - base_dir = tempfile.mkdtemp() - self.addCleanup(shutil.rmtree, base_dir) - - with open(os.path.join(base_dir, 'Dockerfile'), 'w') as f: - f.write('FROM busybox\n') - - service = self.create_service('buildlabels', build={ - 'context': text_type(base_dir), - 'labels': ['com.docker.compose.test=true'] - }) - service.build() - self.addCleanup(self.client.remove_image, service.image_name) - - assert service.image() - assert service.image()['Config']['Labels']['com.docker.compose.test'] == 'true' - @no_cluster('Container networks not on Swarm') def test_build_with_network(self): base_dir = tempfile.mkdtemp() diff --git a/tests/unit/config/config_test.py b/tests/unit/config/config_test.py index c5e40130d..8f2266ed8 100644 --- a/tests/unit/config/config_test.py +++ b/tests/unit/config/config_test.py @@ -892,7 +892,7 @@ class ConfigTest(unittest.TestCase): assert service['build']['args']['opt1'] == '42' assert service['build']['args']['opt2'] == 'foobar' - def test_load_with_build_labels(self): + def test_load_build_labels_dict(self): service = config.load( build_config_details( { @@ -919,6 +919,28 @@ class ConfigTest(unittest.TestCase): assert service['build']['labels']['label1'] == 42 assert service['build']['labels']['label2'] == 'foobar' + def test_load_build_labels_list(self): + base_file = config.ConfigFile( + 'base.yml', + { + 'version': '2.3', + 'services': { + 'web': { + 'build': { + 'context': '.', + 'labels': ['foo=bar', 'baz=true', 'foobar=1'] + }, + }, + }, + } + ) + + details = config.ConfigDetails('.', [base_file]) + service = config.load(details).services[0] + assert service['build']['labels'] == { + 'foo': 'bar', 'baz': 'true', 'foobar': '1' + } + def test_build_args_allow_empty_properties(self): service = config.load( build_config_details( diff --git a/tests/unit/service_test.py b/tests/unit/service_test.py index 5c5c2bf67..7d61807ba 100644 --- a/tests/unit/service_test.py +++ b/tests/unit/service_test.py @@ -473,7 +473,7 @@ class ServiceTest(unittest.TestCase): nocache=False, rm=True, buildargs={}, - labels={}, + labels=None, cache_from=None, network_mode=None, target=None, @@ -514,7 +514,7 @@ class ServiceTest(unittest.TestCase): nocache=False, rm=True, buildargs={}, - labels={}, + labels=None, cache_from=None, network_mode=None, target=None, From 0ec77bf7d58f6a273cb8ae0b65a26f15e13bd6e0 Mon Sep 17 00:00:00 2001 From: Guillermo Arribas Date: Wed, 11 Oct 2017 13:56:15 -0300 Subject: [PATCH 174/244] Config command generates invalid volumes (fixes #5176) Signed-off-by: Guillermo Arribas --- compose/config/config.py | 19 +++---- compose/config/serialize.py | 4 +- tests/acceptance/cli_test.py | 54 +++++++++++++++++-- .../volumes/external-volumes-v2-x.yml | 17 ++++++ ...al-volumes.yml => external-volumes-v2.yml} | 2 +- .../volumes/external-volumes-v3-4.yml | 17 ++++++ .../volumes/external-volumes-v3-x.yml | 16 ++++++ 7 files changed, 114 insertions(+), 15 deletions(-) create mode 100644 tests/fixtures/volumes/external-volumes-v2-x.yml rename tests/fixtures/volumes/{external-volumes.yml => external-volumes-v2.yml} (92%) create mode 100644 tests/fixtures/volumes/external-volumes-v3-4.yml create mode 100644 tests/fixtures/volumes/external-volumes-v3-x.yml diff --git a/compose/config/config.py b/compose/config/config.py index 68b2be3a6..7bb57076e 100644 --- a/compose/config/config.py +++ b/compose/config/config.py @@ -15,6 +15,9 @@ from cached_property import cached_property from . import types from .. import const from ..const import COMPOSEFILE_V1 as V1 +from ..const import COMPOSEFILE_V2_1 as V2_1 +from ..const import COMPOSEFILE_V3_0 as V3_0 +from ..const import COMPOSEFILE_V3_4 as V3_4 from ..utils import build_string_dict from ..utils import parse_bytes from ..utils import parse_nanoseconds_int @@ -405,7 +408,7 @@ def load_mapping(config_files, get_func, entity_type, working_dir=None): external = config.get('external') if external: name_field = 'name' if entity_type == 'Volume' else 'external_name' - validate_external(entity_type, name, config) + validate_external(entity_type, name, config, config_file.version) if isinstance(external, dict): config[name_field] = external.get('name') elif not config.get('name'): @@ -425,14 +428,12 @@ def load_mapping(config_files, get_func, entity_type, working_dir=None): return mapping -def validate_external(entity_type, name, config): - if len(config.keys()) <= 1: - return - - raise ConfigurationError( - "{} {} declared as external but specifies additional attributes " - "({}).".format( - entity_type, name, ', '.join(k for k in config if k != 'external'))) +def validate_external(entity_type, name, config, version): + if (version < V2_1 or (version >= V3_0 and version < V3_4)) and len(config.keys()) > 1: + raise ConfigurationError( + "{} {} declared as external but specifies additional attributes " + "({}).".format( + entity_type, name, ', '.join(k for k in config if k != 'external'))) def load_services(config_details, config_file): diff --git a/compose/config/serialize.py b/compose/config/serialize.py index 606dd7614..2b8c73f14 100644 --- a/compose/config/serialize.py +++ b/compose/config/serialize.py @@ -9,7 +9,7 @@ from compose.const import COMPOSEFILE_V1 as V1 from compose.const import COMPOSEFILE_V2_1 as V2_1 from compose.const import COMPOSEFILE_V3_0 as V3_0 from compose.const import COMPOSEFILE_V3_2 as V3_2 -from compose.const import COMPOSEFILE_V3_2 as V3_4 +from compose.const import COMPOSEFILE_V3_4 as V3_4 def serialize_config_type(dumper, data): @@ -67,7 +67,7 @@ def denormalize_config(config, image_digests=None): del conf['external_name'] if 'name' in conf: - if config.version < V2_1 or (config.version > V3_0 and config.version < V3_4): + if config.version < V2_1 or (config.version >= V3_0 and config.version < V3_4): del conf['name'] elif 'external' in conf: conf['external'] = True diff --git a/tests/acceptance/cli_test.py b/tests/acceptance/cli_test.py index b598d99d5..43cc89e36 100644 --- a/tests/acceptance/cli_test.py +++ b/tests/acceptance/cli_test.py @@ -285,15 +285,63 @@ class CLITestCase(DockerClientTestCase): } } - def test_config_external_volume(self): + def test_config_external_volume_v2(self): self.base_dir = 'tests/fixtures/volumes' - result = self.dispatch(['-f', 'external-volumes.yml', 'config']) + result = self.dispatch(['-f', 'external-volumes-v2.yml', 'config']) json_result = yaml.load(result.stdout) assert 'volumes' in json_result assert json_result['volumes'] == { 'foo': { 'external': True, - 'name': 'foo', + }, + 'bar': { + 'external': { + 'name': 'some_bar', + }, + } + } + + def test_config_external_volume_v2_x(self): + self.base_dir = 'tests/fixtures/volumes' + result = self.dispatch(['-f', 'external-volumes-v2-x.yml', 'config']) + json_result = yaml.load(result.stdout) + assert 'volumes' in json_result + assert json_result['volumes'] == { + 'foo': { + 'external': True, + 'name': 'some_foo', + }, + 'bar': { + 'external': True, + 'name': 'some_bar', + } + } + + def test_config_external_volume_v3_x(self): + self.base_dir = 'tests/fixtures/volumes' + result = self.dispatch(['-f', 'external-volumes-v3-x.yml', 'config']) + json_result = yaml.load(result.stdout) + assert 'volumes' in json_result + assert json_result['volumes'] == { + 'foo': { + 'external': True, + }, + 'bar': { + 'external': { + 'name': 'some_bar', + }, + } + } + + def test_config_external_volume_v3_4(self): + self.base_dir = 'tests/fixtures/volumes' + result = self.dispatch(['-f', 'external-volumes-v3-4.yml', 'config']) + json_result = yaml.load(result.stdout) + assert 'volumes' in json_result + assert json_result['volumes'] == { + 'foo': { + 'external': True, + 'name': 'some_foo', }, 'bar': { 'external': True, diff --git a/tests/fixtures/volumes/external-volumes-v2-x.yml b/tests/fixtures/volumes/external-volumes-v2-x.yml new file mode 100644 index 000000000..3b736c5f4 --- /dev/null +++ b/tests/fixtures/volumes/external-volumes-v2-x.yml @@ -0,0 +1,17 @@ +version: "2.1" + +services: + web: + image: busybox + command: top + volumes: + - foo:/var/lib/ + - bar:/etc/ + +volumes: + foo: + external: true + name: some_foo + bar: + external: + name: some_bar diff --git a/tests/fixtures/volumes/external-volumes.yml b/tests/fixtures/volumes/external-volumes-v2.yml similarity index 92% rename from tests/fixtures/volumes/external-volumes.yml rename to tests/fixtures/volumes/external-volumes-v2.yml index 05c6c4844..4025b53b1 100644 --- a/tests/fixtures/volumes/external-volumes.yml +++ b/tests/fixtures/volumes/external-volumes-v2.yml @@ -1,4 +1,4 @@ -version: "2.1" +version: "2" services: web: diff --git a/tests/fixtures/volumes/external-volumes-v3-4.yml b/tests/fixtures/volumes/external-volumes-v3-4.yml new file mode 100644 index 000000000..76c8421dc --- /dev/null +++ b/tests/fixtures/volumes/external-volumes-v3-4.yml @@ -0,0 +1,17 @@ +version: "3.4" + +services: + web: + image: busybox + command: top + volumes: + - foo:/var/lib/ + - bar:/etc/ + +volumes: + foo: + external: true + name: some_foo + bar: + external: + name: some_bar diff --git a/tests/fixtures/volumes/external-volumes-v3-x.yml b/tests/fixtures/volumes/external-volumes-v3-x.yml new file mode 100644 index 000000000..903fee647 --- /dev/null +++ b/tests/fixtures/volumes/external-volumes-v3-x.yml @@ -0,0 +1,16 @@ +version: "3.0" + +services: + web: + image: busybox + command: top + volumes: + - foo:/var/lib/ + - bar:/etc/ + +volumes: + foo: + external: true + bar: + external: + name: some_bar From d8194cf6f0c5fd5a08f1a0daa6b4022a5582d008 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Mon, 16 Oct 2017 12:41:29 -0700 Subject: [PATCH 175/244] Add specific handling for pywintypes.error Signed-off-by: Joffrey F --- compose/cli/errors.py | 20 ++++++++++++++++++++ tests/unit/cli/errors_test.py | 21 +++++++++++++++++++++ 2 files changed, 41 insertions(+) diff --git a/compose/cli/errors.py b/compose/cli/errors.py index 23e065c99..1506aa660 100644 --- a/compose/cli/errors.py +++ b/compose/cli/errors.py @@ -57,6 +57,26 @@ def handle_connection_errors(client): except (ReadTimeout, socket.timeout) as e: log_timeout_error(client.timeout) raise ConnectionError() + except Exception as e: + if is_windows(): + import pywintypes + if isinstance(e, pywintypes.error): + log_windows_pipe_error(e) + raise ConnectionError() + raise + + +def log_windows_pipe_error(exc): + if exc.winerror == 232: # https://github.com/docker/compose/issues/5005 + log.error( + "The current Compose file version is not compatible with your engine version. " + "Please upgrade your Compose file to a more recent version, or set " + "a COMPOSE_API_VERSION in your environment." + ) + else: + log.error( + "Windows named pipe error: {} (code: {})".format(exc.strerror, exc.winerror) + ) def log_timeout_error(timeout): diff --git a/tests/unit/cli/errors_test.py b/tests/unit/cli/errors_test.py index 7406a8880..68326d1c7 100644 --- a/tests/unit/cli/errors_test.py +++ b/tests/unit/cli/errors_test.py @@ -7,6 +7,7 @@ from requests.exceptions import ConnectionError from compose.cli import errors from compose.cli.errors import handle_connection_errors +from compose.const import IS_WINDOWS_PLATFORM from tests import mock @@ -65,3 +66,23 @@ class TestHandleConnectionErrors(object): raise APIError(None, None, msg) mock_logging.error.assert_called_once_with(msg) + + @pytest.mark.skipif(not IS_WINDOWS_PLATFORM, reason='Needs pywin32') + def test_windows_pipe_error_no_data(self, mock_logging): + import pywintypes + with pytest.raises(errors.ConnectionError): + with handle_connection_errors(mock.Mock(api_version='1.22')): + raise pywintypes.error(232, 'WriteFile', 'The pipe is being closed.') + + _, args, _ = mock_logging.error.mock_calls[0] + assert "The current Compose file version is not compatible with your engine version." in args[0] + + @pytest.mark.skipif(not IS_WINDOWS_PLATFORM, reason='Needs pywin32') + def test_windows_pipe_error_misc(self, mock_logging): + import pywintypes + with pytest.raises(errors.ConnectionError): + with handle_connection_errors(mock.Mock(api_version='1.22')): + raise pywintypes.error(231, 'WriteFile', 'The pipe is busy.') + + _, args, _ = mock_logging.error.mock_calls[0] + assert "Windows named pipe error: The pipe is busy. (code: 231)" == args[0] From 395dce9d2c40632242cead5bffe3b7d55e2fb2a2 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Mon, 16 Oct 2017 13:54:30 -0700 Subject: [PATCH 176/244] Add check_duplicate=True when creating network Signed-off-by: Joffrey F --- compose/network.py | 1 + 1 file changed, 1 insertion(+) diff --git a/compose/network.py b/compose/network.py index 0f42eb20a..2e0a7e6ec 100644 --- a/compose/network.py +++ b/compose/network.py @@ -79,6 +79,7 @@ class Network(object): enable_ipv6=self.enable_ipv6, labels=self._labels, attachable=version_gte(self.client._version, '1.24') or None, + check_duplicate=True, ) def remove(self): From 13c5049dbccf63182bf5d963bf25c817b25b8a66 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Tue, 17 Oct 2017 13:36:06 -0700 Subject: [PATCH 177/244] flake8 Signed-off-by: Joffrey F --- tests/acceptance/cli_test.py | 4 ---- tests/integration/testcases.py | 2 -- 2 files changed, 6 deletions(-) diff --git a/tests/acceptance/cli_test.py b/tests/acceptance/cli_test.py index 43cc89e36..8ba43b00f 100644 --- a/tests/acceptance/cli_test.py +++ b/tests/acceptance/cli_test.py @@ -493,10 +493,6 @@ class CLITestCase(DockerClientTestCase): 'image library/nonexisting-image:latest not found' in result.stderr or 'pull access denied for nonexisting-image' in result.stderr) - def test_pull_with_quiet(self): - assert self.dispatch(['pull', '--quiet']).stderr == '' - assert self.dispatch(['pull', '--quiet']).stdout == '' - def test_pull_with_parallel_failure(self): result = self.dispatch([ '-f', 'ignore-pull-failures.yml', 'pull', '--parallel'], diff --git a/tests/integration/testcases.py b/tests/integration/testcases.py index 8435f97dd..b72fb53a8 100644 --- a/tests/integration/testcases.py +++ b/tests/integration/testcases.py @@ -75,7 +75,6 @@ def v2_1_only(): return min_version_skip(V2_1) - def v2_2_only(): return min_version_skip(V2_2) @@ -84,7 +83,6 @@ def v2_3_only(): return min_version_skip(V2_3) - def v3_only(): return min_version_skip(V3_0) From 2a0dd1401fb832c377a70527d760f34754523f2a Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Mon, 16 Oct 2017 16:50:51 -0700 Subject: [PATCH 178/244] Bump 1.17.0-rc1 Signed-off-by: Joffrey F --- CHANGELOG.md | 61 +++++++++++++++++++++++++++++++++++++++++++-- compose/__init__.py | 2 +- script/run/run.sh | 2 +- 3 files changed, 61 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 558376855..cff19d879 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,64 @@ Change log ========== +1.17.0 (2017-11-03) +------------------- + +### New features + +#### Compose file version 3.4 + +- Introduced version 3.4 of the `docker-compose.yml` specification. + This version requires to be used with Docker Engine 17.06.0 or above. + +- Added support for `cache_from`, `network` and `target` options in build + configurations + +- Added support for the `order` parameter in the `update_config` section + +- Added support for setting a custom name in volume definitions using + the `name` parameter + +#### Compose file version 2.3 + +- Added support for `shm_size` option in build configuration + +#### Compose file version 2.x + +- Added support for extension fields (`x-*`). Also available for v3.4 files + +#### All formats + +- Added new `--no-start` to the `up` command, allowing users to create all + resources (networks, volumes, containers) without starting services. + The `create` command is deprecated in favor of this new option + +### Bugfixes + +- Fixed a bug where `extra_hosts` values would be overridden by extension + files instead of merging together + +- Fixed a bug where the validation for v3.2 files would prevent using the + `consistency` field in service volume definitions + +- Fixed a bug that would cause a crash when configuration fields expecting + unique items would contain duplicates + +- Fixed a bug where mount overrides with a different mode would create a + duplicate entry instead of overriding the original entry + +- Fixed a bug where build labels declared as a list wouldn't be properly + parsed + +- Fixed a bug where the output of `docker-compose config` would be invalid + for some versions if the file contained custom-named external volumes + +- Improved error handling when issuing a build command on Windows using an + unsupported file version + +- Fixed an issue where networks with identical names would sometimes be + created when running `up` commands concurrently. + 1.16.1 (2017-09-01) ------------------- @@ -8,7 +66,6 @@ Change log - Fixed bug that prevented using `extra_hosts` in several configuration files. - 1.16.0 (2017-08-31) ------------------- @@ -19,7 +76,7 @@ Change log - Introduced version 2.3 of the `docker-compose.yml` specification. This version requires to be used with Docker Engine 17.06.0 or above. -- Added support for the `target` parameter in network configurations +- Added support for the `target` parameter in build configurations - Added support for the `start_period` parameter in healthcheck configurations diff --git a/compose/__init__.py b/compose/__init__.py index 2e41ca896..86542ec44 100644 --- a/compose/__init__.py +++ b/compose/__init__.py @@ -1,4 +1,4 @@ from __future__ import absolute_import from __future__ import unicode_literals -__version__ = '1.16.1' +__version__ = '1.17.0-rc1' diff --git a/script/run/run.sh b/script/run/run.sh index f1754d05a..498226288 100755 --- a/script/run/run.sh +++ b/script/run/run.sh @@ -15,7 +15,7 @@ set -e -VERSION="1.16.1" +VERSION="1.17.0-rc1" IMAGE="docker/compose:$VERSION" From 779773b6644b8598a1143d5f370a2fe24eeefac4 Mon Sep 17 00:00:00 2001 From: Harald Albers Date: Thu, 19 Oct 2017 09:18:40 +0200 Subject: [PATCH 179/244] Add bash completion for `up --no-start` Signed-off-by: Harald Albers --- contrib/completion/bash/docker-compose | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contrib/completion/bash/docker-compose b/contrib/completion/bash/docker-compose index 9de156403..1fdb27705 100644 --- a/contrib/completion/bash/docker-compose +++ b/contrib/completion/bash/docker-compose @@ -518,7 +518,7 @@ _docker_compose_up() { case "$cur" in -*) - COMPREPLY=( $( compgen -W "--abort-on-container-exit --build -d --exit-code-from --force-recreate --help --no-build --no-color --no-deps --no-recreate --remove-orphans --scale --timeout -t" -- "$cur" ) ) + COMPREPLY=( $( compgen -W "--abort-on-container-exit --build -d --exit-code-from --force-recreate --help --no-build --no-color --no-deps --no-recreate --no-start --remove-orphans --scale --timeout -t" -- "$cur" ) ) ;; *) __docker_compose_services_all From 0847f8e84be690ad478a425565ab4e6cc9f8371a Mon Sep 17 00:00:00 2001 From: Guillermo Arribas Date: Mon, 23 Oct 2017 13:22:36 -0300 Subject: [PATCH 180/244] flake8 error on master branch (fixes #5298) Signed-off-by: Guillermo Arribas --- compose/bundle.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/compose/bundle.py b/compose/bundle.py index 505ce91fe..937a3708a 100644 --- a/compose/bundle.py +++ b/compose/bundle.py @@ -121,7 +121,7 @@ def get_image_digest(service, allow_push=False): def push_image(service): try: digest = service.push() - except: + except Exception: log.error( "Failed to push image for service '{s.name}'. Please use an " "image tag that can be pushed to a Docker " From 6078736604e7ab77e16b20ec5c3928b939120a7b Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Mon, 23 Oct 2017 13:25:48 -0700 Subject: [PATCH 181/244] Add flake8 to dev requirements Signed-off-by: Joffrey F --- requirements-dev.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/requirements-dev.txt b/requirements-dev.txt index 73b807835..e06cad45c 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -1,4 +1,5 @@ coverage==3.7.1 +flake8==3.5.0 mock>=1.0.1 pytest==2.7.2 pytest-cov==2.1.0 From 9f80ec548e79e1fbfa7adab27a0e50b3fdbb6ba9 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Wed, 1 Nov 2017 14:26:29 -0700 Subject: [PATCH 182/244] Miscellaneous test fixes Signed-off-by: Joffrey F --- tests/acceptance/cli_test.py | 8 +++++++- tests/integration/service_test.py | 2 +- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/tests/acceptance/cli_test.py b/tests/acceptance/cli_test.py index 8ba43b00f..bba2238e7 100644 --- a/tests/acceptance/cli_test.py +++ b/tests/acceptance/cli_test.py @@ -851,7 +851,13 @@ class CLITestCase(DockerClientTestCase): volumes = self.project.volumes.volumes assert 'data' in volumes volume = volumes['data'] - assert volume.exists() + + # The code below is a Swarm-compatible equivalent to volume.exists() + remote_volumes = [ + v for v in self.client.volumes().get('Volumes', []) + if v['Name'].split('/')[-1] == volume.full_name + ] + assert len(remote_volumes) > 0 @v2_only() def test_up_no_ansi(self): diff --git a/tests/integration/service_test.py b/tests/integration/service_test.py index 84b54fe41..3ddf991b3 100644 --- a/tests/integration/service_test.py +++ b/tests/integration/service_test.py @@ -788,7 +788,7 @@ class ServiceTest(DockerClientTestCase): net_container = self.client.create_container( 'busybox', 'top', host_config=self.client.create_host_config( - extra_hosts={'google.local': '8.8.8.8'} + extra_hosts={'google.local': '127.0.0.1'} ), name='composetest_build_network' ) From ac53b73e7958b825f7235a661c208f4f6f6e90f7 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Wed, 1 Nov 2017 13:34:50 -0700 Subject: [PATCH 183/244] Bump 1.17.0 Signed-off-by: Joffrey F --- CHANGELOG.md | 2 +- compose/__init__.py | 2 +- script/run/run.sh | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index cff19d879..f531783e8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,7 +1,7 @@ Change log ========== -1.17.0 (2017-11-03) +1.17.0 (2017-11-02) ------------------- ### New features diff --git a/compose/__init__.py b/compose/__init__.py index 86542ec44..7b0c7d1e4 100644 --- a/compose/__init__.py +++ b/compose/__init__.py @@ -1,4 +1,4 @@ from __future__ import absolute_import from __future__ import unicode_literals -__version__ = '1.17.0-rc1' +__version__ = '1.17.0' diff --git a/script/run/run.sh b/script/run/run.sh index 498226288..38ce87873 100755 --- a/script/run/run.sh +++ b/script/run/run.sh @@ -15,7 +15,7 @@ set -e -VERSION="1.17.0-rc1" +VERSION="1.17.0" IMAGE="docker/compose:$VERSION" From 03fefaca393799ac50e2a9f8bbcfdd26d86682cf Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Fri, 3 Nov 2017 13:51:49 -0700 Subject: [PATCH 184/244] Fix service label parsing Signed-off-by: Joffrey F --- compose/config/config.py | 23 ++++++++----- tests/unit/config/config_test.py | 55 +++++++++++++++++++++----------- 2 files changed, 52 insertions(+), 26 deletions(-) diff --git a/compose/config/config.py b/compose/config/config.py index 7bb57076e..d5aaf9538 100644 --- a/compose/config/config.py +++ b/compose/config/config.py @@ -706,14 +706,7 @@ def process_service(service_config): ] if 'build' in service_dict: - if isinstance(service_dict['build'], six.string_types): - service_dict['build'] = resolve_build_path(working_dir, service_dict['build']) - elif isinstance(service_dict['build'], dict): - if 'context' in service_dict['build']: - path = service_dict['build']['context'] - service_dict['build']['context'] = resolve_build_path(working_dir, path) - if 'labels' in service_dict['build']: - service_dict['build']['labels'] = parse_labels(service_dict['build']['labels']) + process_build_section(service_dict, working_dir) if 'volumes' in service_dict and service_dict.get('volume_driver') is None: service_dict['volumes'] = resolve_volume_paths(working_dir, service_dict) @@ -721,6 +714,9 @@ def process_service(service_config): if 'sysctls' in service_dict: service_dict['sysctls'] = build_string_dict(parse_sysctls(service_dict['sysctls'])) + if 'labels' in service_dict: + service_dict['labels'] = parse_labels(service_dict['labels']) + service_dict = process_depends_on(service_dict) for field in ['dns', 'dns_search', 'tmpfs']: @@ -734,6 +730,17 @@ def process_service(service_config): return service_dict +def process_build_section(service_dict, working_dir): + if isinstance(service_dict['build'], six.string_types): + service_dict['build'] = resolve_build_path(working_dir, service_dict['build']) + elif isinstance(service_dict['build'], dict): + if 'context' in service_dict['build']: + path = service_dict['build']['context'] + service_dict['build']['context'] = resolve_build_path(working_dir, path) + if 'labels' in service_dict['build']: + service_dict['build']['labels'] = parse_labels(service_dict['build']['labels']) + + def process_ports(service_dict): if 'ports' not in service_dict: return service_dict diff --git a/tests/unit/config/config_test.py b/tests/unit/config/config_test.py index 8f2266ed8..8e3d4e2ee 100644 --- a/tests/unit/config/config_test.py +++ b/tests/unit/config/config_test.py @@ -407,6 +407,32 @@ class ConfigTest(unittest.TestCase): } } + def test_load_config_service_labels(self): + base_file = config.ConfigFile( + 'base.yaml', + { + 'version': '2.1', + 'services': { + 'web': { + 'image': 'example/web', + 'labels': ['label_key=label_val'] + }, + 'db': { + 'image': 'example/db', + 'labels': { + 'label_key': 'label_val' + } + } + }, + } + ) + details = config.ConfigDetails('.', [base_file]) + service_dicts = config.load(details).services + for service in service_dicts: + assert service['labels'] == { + 'label_key': 'label_val' + } + def test_load_config_volume_and_network_labels(self): base_file = config.ConfigFile( 'base.yaml', @@ -435,30 +461,23 @@ class ConfigTest(unittest.TestCase): ) details = config.ConfigDetails('.', [base_file]) - network_dict = config.load(details).networks - volume_dict = config.load(details).volumes + loaded_config = config.load(details) - self.assertEqual( - network_dict, - { - 'with_label': { - 'labels': { - 'label_key': 'label_val' - } + assert loaded_config.networks == { + 'with_label': { + 'labels': { + 'label_key': 'label_val' } } - ) + } - self.assertEqual( - volume_dict, - { - 'with_label': { - 'labels': { - 'label_key': 'label_val' - } + assert loaded_config.volumes == { + 'with_label': { + 'labels': { + 'label_key': 'label_val' } } - ) + } def test_load_config_invalid_service_names(self): for invalid_name in ['?not?allowed', ' ', '', '!', '/', '\xe2']: From 6d101fb0686a6e380657aaf974bc3efd1471535e Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Tue, 7 Nov 2017 17:10:22 -0800 Subject: [PATCH 185/244] Bump 1.17.1 Signed-off-by: Joffrey F --- CHANGELOG.md | 8 ++++++++ compose/__init__.py | 2 +- script/run/run.sh | 2 +- 3 files changed, 10 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f531783e8..d0be7ea76 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,14 @@ Change log ========== +1.17.1 (2017-11-08) +------------------ + +### Bugfixes + +- Fixed a bug that would prevent creating new containers when using + container labels in the list format as part of the service's definition. + 1.17.0 (2017-11-02) ------------------- diff --git a/compose/__init__.py b/compose/__init__.py index 7b0c7d1e4..20392ec99 100644 --- a/compose/__init__.py +++ b/compose/__init__.py @@ -1,4 +1,4 @@ from __future__ import absolute_import from __future__ import unicode_literals -__version__ = '1.17.0' +__version__ = '1.17.1' diff --git a/script/run/run.sh b/script/run/run.sh index 38ce87873..58483196d 100755 --- a/script/run/run.sh +++ b/script/run/run.sh @@ -15,7 +15,7 @@ set -e -VERSION="1.17.0" +VERSION="1.17.1" IMAGE="docker/compose:$VERSION" From 700d6aca545fb32eda9cbdb6448f2f37dd66f9e8 Mon Sep 17 00:00:00 2001 From: Alexey Rokhin Date: Wed, 17 May 2017 15:21:35 +0300 Subject: [PATCH 186/244] Fix testcases.py formatting Signed-off-by: Alexey Rokhin --- tests/integration/testcases.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/integration/testcases.py b/tests/integration/testcases.py index b72fb53a8..8435f97dd 100644 --- a/tests/integration/testcases.py +++ b/tests/integration/testcases.py @@ -75,6 +75,7 @@ def v2_1_only(): return min_version_skip(V2_1) + def v2_2_only(): return min_version_skip(V2_2) @@ -83,6 +84,7 @@ def v2_3_only(): return min_version_skip(V2_3) + def v3_only(): return min_version_skip(V3_0) From 9b91f3431b6190e03cd512cc49d49f996f075e2a Mon Sep 17 00:00:00 2001 From: Alexey Rokhin Date: Wed, 17 May 2017 16:18:28 +0300 Subject: [PATCH 187/244] skip cpu_percent test for Linux Signed-off-by: Alexey Rokhin --- tests/integration/service_test.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/integration/service_test.py b/tests/integration/service_test.py index 3ddf991b3..2583e39e8 100644 --- a/tests/integration/service_test.py +++ b/tests/integration/service_test.py @@ -28,6 +28,7 @@ from compose.const import LABEL_ONE_OFF from compose.const import LABEL_PROJECT from compose.const import LABEL_SERVICE from compose.const import LABEL_VERSION +from compose.const import IS_WINDOWS_PLATFORM from compose.container import Container from compose.errors import OperationFailedError from compose.project import OneOffFilter From 3d57f702f18f29c80ae90391b43b94c1c19402e7 Mon Sep 17 00:00:00 2001 From: Alexey Rokhin Date: Wed, 17 May 2017 16:42:43 +0300 Subject: [PATCH 188/244] service_test.py reorder imports Signed-off-by: Alexey Rokhin --- tests/integration/service_test.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/integration/service_test.py b/tests/integration/service_test.py index 2583e39e8..3ddf991b3 100644 --- a/tests/integration/service_test.py +++ b/tests/integration/service_test.py @@ -28,7 +28,6 @@ from compose.const import LABEL_ONE_OFF from compose.const import LABEL_PROJECT from compose.const import LABEL_SERVICE from compose.const import LABEL_VERSION -from compose.const import IS_WINDOWS_PLATFORM from compose.container import Container from compose.errors import OperationFailedError from compose.project import OneOffFilter From 5844dbb38e9e7cc0835cabbf000c3937b80c5c04 Mon Sep 17 00:00:00 2001 From: Joel Barciauskas Date: Wed, 12 Apr 2017 17:45:09 -0400 Subject: [PATCH 189/244] Add --quiet parameter to docker-compose pull, using existing silent flag Signed-off-by: Joel Barciauskas --- compose/project.py | 2 +- tests/acceptance/cli_test.py | 4 ++++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/compose/project.py b/compose/project.py index c8b57edd2..fa536f02c 100644 --- a/compose/project.py +++ b/compose/project.py @@ -498,7 +498,7 @@ class Project(object): if parallel_pull: def pull_service(service): - service.pull(ignore_pull_failures, True) + service.pull(ignore_pull_failures, True, silent=silent) _, errors = parallel.parallel_execute( services, diff --git a/tests/acceptance/cli_test.py b/tests/acceptance/cli_test.py index bba2238e7..746973a2a 100644 --- a/tests/acceptance/cli_test.py +++ b/tests/acceptance/cli_test.py @@ -511,6 +511,10 @@ class CLITestCase(DockerClientTestCase): assert self.dispatch(['pull', '--quiet']).stderr == '' assert self.dispatch(['pull', '--quiet']).stdout == '' + def test_pull_with_quiet(self): + assert self.dispatch(['pull', '--quiet']).stderr == '' + assert self.dispatch(['pull', '--quiet']).stdout == '' + def test_build_plain(self): self.base_dir = 'tests/fixtures/simple-dockerfile' self.dispatch(['build', 'simple']) From 4286315bc907178efb0ce9d53bb28414ebbcd477 Mon Sep 17 00:00:00 2001 From: NikitaVlaznev Date: Mon, 19 Jun 2017 17:05:19 +0300 Subject: [PATCH 190/244] Fix double silent argument value Fix for "TypeError: pull() got multiple values for keyword argument 'silent'." This change https://github.com/docker/compose/commit/e9b6cc23fcf01d4768c7e082b7bc91b43ff84e7e caused additional value to be passed for the 'silent' argument, that was already passed there: https://github.com/docker/compose/commit/f85da99ef3273794e855afda8678174419d3bf4f Signed-off-by: Nikita Vlaznev --- compose/project.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/compose/project.py b/compose/project.py index fa536f02c..9e0a7b02f 100644 --- a/compose/project.py +++ b/compose/project.py @@ -498,7 +498,7 @@ class Project(object): if parallel_pull: def pull_service(service): - service.pull(ignore_pull_failures, True, silent=silent) + service.pull(ignore_pull_failures, silent=silent) _, errors = parallel.parallel_execute( services, From d1289554d505793d9ffc327df81990c475d482bf Mon Sep 17 00:00:00 2001 From: Evan Shaw Date: Sat, 1 Jul 2017 13:40:02 +1200 Subject: [PATCH 191/244] Always silence pull output with --parallel This is how things were prior to the addition of the --quiet flag. Making it not silent produces output that's weird and difficult to read. Signed-off-by: Evan Shaw --- compose/project.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/compose/project.py b/compose/project.py index 9e0a7b02f..c8b57edd2 100644 --- a/compose/project.py +++ b/compose/project.py @@ -498,7 +498,7 @@ class Project(object): if parallel_pull: def pull_service(service): - service.pull(ignore_pull_failures, silent=silent) + service.pull(ignore_pull_failures, True) _, errors = parallel.parallel_execute( services, From 2daf3628e9dda4b58e5c38cd5c2590654ba93329 Mon Sep 17 00:00:00 2001 From: aronahl Date: Wed, 9 Aug 2017 19:44:12 -0400 Subject: [PATCH 192/244] Fix exit code 0 upon parallel pull failure. Signed-off-by: Aaron Nall --- tests/acceptance/cli_test.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/tests/acceptance/cli_test.py b/tests/acceptance/cli_test.py index 746973a2a..22756bd3d 100644 --- a/tests/acceptance/cli_test.py +++ b/tests/acceptance/cli_test.py @@ -515,6 +515,20 @@ class CLITestCase(DockerClientTestCase): assert self.dispatch(['pull', '--quiet']).stderr == '' assert self.dispatch(['pull', '--quiet']).stdout == '' + def test_pull_with_parallel_failure(self): + result = self.dispatch([ + '-f', 'ignore-pull-failures.yml', 'pull', '--parallel'], + returncode=1 + ) + + self.assertRegexpMatches(result.stderr, re.compile('^Pulling simple', re.MULTILINE)) + self.assertRegexpMatches(result.stderr, re.compile('^Pulling another', re.MULTILINE)) + self.assertRegexpMatches(result.stderr, + re.compile('^ERROR: for another .*does not exist.*', re.MULTILINE)) + self.assertRegexpMatches(result.stderr, + re.compile('''^(ERROR: )?(b')?.* nonexisting-image''', + re.MULTILINE)) + def test_build_plain(self): self.base_dir = 'tests/fixtures/simple-dockerfile' self.dispatch(['build', 'simple']) From c8b2dd2fb1346e8efcdcdb6434d6ec860c9581f9 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Fri, 25 Aug 2017 18:09:06 -0700 Subject: [PATCH 193/244] Add support for extension fields in v2.x and v3.4 Signed-off-by: Joffrey F --- compose/config/config_schema_v3.5.json | 1 + 1 file changed, 1 insertion(+) diff --git a/compose/config/config_schema_v3.5.json b/compose/config/config_schema_v3.5.json index fa95d6a24..5400cd99f 100644 --- a/compose/config/config_schema_v3.5.json +++ b/compose/config/config_schema_v3.5.json @@ -64,6 +64,7 @@ } }, + "patternProperties": {"^x-": {}}, "additionalProperties": false, "definitions": { From e1db4f6e191df1abe5c6d4f6ecd85e9dd3d6fbe8 Mon Sep 17 00:00:00 2001 From: Guillermo Arribas Date: Fri, 6 Oct 2017 19:12:59 -0300 Subject: [PATCH 194/244] Build labels option: array form produces unmarshal error (fixes #5183) Signed-off-by: Guillermo Arribas --- compose/service.py | 3 ++- tests/integration/service_test.py | 19 ++++++++++++++++++- tests/unit/service_test.py | 4 ++-- 3 files changed, 22 insertions(+), 4 deletions(-) diff --git a/compose/service.py b/compose/service.py index 1a18c6654..e2f72aa5a 100644 --- a/compose/service.py +++ b/compose/service.py @@ -23,6 +23,7 @@ from . import const from . import progress_stream from .config import DOCKER_CONFIG_KEYS from .config import merge_environment +from .config.config import parse_labels from .config.errors import DependencyError from .config.types import ServicePort from .config.types import VolumeSpec @@ -916,7 +917,7 @@ class Service(object): nocache=no_cache, dockerfile=build_opts.get('dockerfile', None), cache_from=build_opts.get('cache_from', None), - labels=build_opts.get('labels', None), + labels=parse_labels(build_opts.get('labels', None)), buildargs=build_args, network_mode=build_opts.get('network', None), target=build_opts.get('target', None), diff --git a/tests/integration/service_test.py b/tests/integration/service_test.py index 3ddf991b3..6cf8ddaa9 100644 --- a/tests/integration/service_test.py +++ b/tests/integration/service_test.py @@ -761,7 +761,7 @@ class ServiceTest(DockerClientTestCase): assert service.image() assert "build_version=2" in service.image()['ContainerConfig']['Cmd'] - def test_build_with_build_labels(self): + def test_build_with_build_labels_dict(self): base_dir = tempfile.mkdtemp() self.addCleanup(shutil.rmtree, base_dir) @@ -778,6 +778,23 @@ class ServiceTest(DockerClientTestCase): assert service.image() assert service.image()['Config']['Labels']['com.docker.compose.test'] == 'true' + def test_build_with_build_labels_list(self): + base_dir = tempfile.mkdtemp() + self.addCleanup(shutil.rmtree, base_dir) + + with open(os.path.join(base_dir, 'Dockerfile'), 'w') as f: + f.write('FROM busybox\n') + + service = self.create_service('buildlabels', build={ + 'context': text_type(base_dir), + 'labels': ['com.docker.compose.test=true'] + }) + service.build() + self.addCleanup(self.client.remove_image, service.image_name) + + assert service.image() + assert service.image()['Config']['Labels']['com.docker.compose.test'] == 'true' + @no_cluster('Container networks not on Swarm') def test_build_with_network(self): base_dir = tempfile.mkdtemp() diff --git a/tests/unit/service_test.py b/tests/unit/service_test.py index 7d61807ba..5c5c2bf67 100644 --- a/tests/unit/service_test.py +++ b/tests/unit/service_test.py @@ -473,7 +473,7 @@ class ServiceTest(unittest.TestCase): nocache=False, rm=True, buildargs={}, - labels=None, + labels={}, cache_from=None, network_mode=None, target=None, @@ -514,7 +514,7 @@ class ServiceTest(unittest.TestCase): nocache=False, rm=True, buildargs={}, - labels=None, + labels={}, cache_from=None, network_mode=None, target=None, From 6dfd4693548520b3ca4d1fb284984d9781433fc5 Mon Sep 17 00:00:00 2001 From: Guillermo Arribas Date: Tue, 10 Oct 2017 11:55:14 -0300 Subject: [PATCH 195/244] Progress markers are not shown correctly for docker-compose up (fixes #4801) Signed-off-by: Guillermo Arribas --- compose/parallel.py | 23 ++++++++++++++++------- compose/project.py | 25 ++++++++++++++++++++++++- compose/service.py | 23 ++++++++++++----------- tests/unit/service_test.py | 2 +- 4 files changed, 53 insertions(+), 20 deletions(-) diff --git a/compose/parallel.py b/compose/parallel.py index d455711dd..f271561ff 100644 --- a/compose/parallel.py +++ b/compose/parallel.py @@ -26,7 +26,7 @@ log = logging.getLogger(__name__) STOP = object() -def parallel_execute(objects, func, get_name, msg, get_deps=None, limit=None): +def parallel_execute(objects, func, get_name, msg, get_deps=None, limit=None, parent_objects=None): """Runs func on objects in parallel while ensuring that func is ran on object only after it is ran on all its dependencies. @@ -37,9 +37,19 @@ def parallel_execute(objects, func, get_name, msg, get_deps=None, limit=None): stream = get_output_stream(sys.stderr) writer = ParallelStreamWriter(stream, msg) - for obj in objects: + + if parent_objects: + display_objects = list(parent_objects) + else: + display_objects = objects + + for obj in display_objects: writer.add_object(get_name(obj)) - writer.write_initial() + + # write data in a second loop to consider all objects for width alignment + # and avoid duplicates when parent_objects exists + for obj in objects: + writer.write_initial(get_name(obj)) events = parallel_execute_iter(objects, func, get_deps, limit) @@ -237,12 +247,11 @@ class ParallelStreamWriter(object): self.lines.append(obj_index) self.width = max(self.width, len(obj_index)) - def write_initial(self): + def write_initial(self, obj_index): if self.msg is None: return - for line in self.lines: - self.stream.write("{} {:<{width}} ... \r\n".format(self.msg, line, - width=self.width)) + self.stream.write("{} {:<{width}} ... \r\n".format( + self.msg, self.lines[self.lines.index(obj_index)], width=self.width)) self.stream.flush() def _write_ansi(self, obj_index, status): diff --git a/compose/project.py b/compose/project.py index c8b57edd2..f6bd30a88 100644 --- a/compose/project.py +++ b/compose/project.py @@ -29,6 +29,7 @@ from .service import ConvergenceStrategy from .service import NetworkMode from .service import PidMode from .service import Service +from .service import ServiceName from .service import ServiceNetworkMode from .service import ServicePidMode from .utils import microseconds_from_time_nano @@ -190,6 +191,25 @@ class Project(object): service.remove_duplicate_containers() return services + def get_scaled_services(self, services, scale_override): + """ + Returns a list of this project's services as scaled ServiceName objects. + + services: a list of Service objects + scale_override: a dict with the scale to apply to each service (k: service_name, v: scale) + """ + service_names = [] + for service in services: + if service.name in scale_override: + scale = scale_override[service.name] + else: + scale = service.scale_num + + for i in range(1, scale + 1): + service_names.append(ServiceName(self.name, service.name, i)) + + return service_names + def get_links(self, service_dict): links = [] if 'links' in service_dict: @@ -430,15 +450,18 @@ class Project(object): for svc in services: svc.ensure_image_exists(do_build=do_build) plans = self._get_convergence_plans(services, strategy) + scaled_services = self.get_scaled_services(services, scale_override) def do(service): + return service.execute_convergence_plan( plans[service.name], timeout=timeout, detached=detached, scale_override=scale_override.get(service.name), rescale=rescale, - start=start + start=start, + project_services=scaled_services ) def get_deps(service): diff --git a/compose/service.py b/compose/service.py index e2f72aa5a..22a7ca53a 100644 --- a/compose/service.py +++ b/compose/service.py @@ -379,11 +379,11 @@ class Service(object): return has_diverged - def _execute_convergence_create(self, scale, detached, start): + def _execute_convergence_create(self, scale, detached, start, project_services=None): i = self._next_container_number() def create_and_start(service, n): - container = service.create_container(number=n) + container = service.create_container(number=n, quiet=True) if not detached: container.attach_log_stream() if start: @@ -391,10 +391,11 @@ class Service(object): return container containers, errors = parallel_execute( - range(i, i + scale), - lambda n: create_and_start(self, n), - lambda n: self.get_container_name(n), + [ServiceName(self.project, self.name, index) for index in range(i, i + scale)], + lambda service_name: create_and_start(self, service_name.number), + lambda service_name: self.get_container_name(service_name.service, service_name.number), "Creating", + parent_objects=project_services ) for error in errors.values(): raise OperationFailedError(error) @@ -433,7 +434,7 @@ class Service(object): if start: _, errors = parallel_execute( containers, - lambda c: self.start_container_if_stopped(c, attach_logs=not detached), + lambda c: self.start_container_if_stopped(c, attach_logs=not detached, quiet=True), lambda c: c.name, "Starting", ) @@ -460,7 +461,7 @@ class Service(object): ) def execute_convergence_plan(self, plan, timeout=None, detached=False, - start=True, scale_override=None, rescale=True): + start=True, scale_override=None, rescale=True, project_services=None): (action, containers) = plan scale = scale_override if scale_override is not None else self.scale_num containers = sorted(containers, key=attrgetter('number')) @@ -469,7 +470,7 @@ class Service(object): if action == 'create': return self._execute_convergence_create( - scale, detached, start + scale, detached, start, project_services ) # The create action needs always needs an initial scale, but otherwise, @@ -742,7 +743,7 @@ class Service(object): container_options.update(override_options) if not container_options.get('name'): - container_options['name'] = self.get_container_name(number, one_off) + container_options['name'] = self.get_container_name(self.name, number, one_off) container_options.setdefault('detach', True) @@ -961,12 +962,12 @@ class Service(object): def custom_container_name(self): return self.options.get('container_name') - def get_container_name(self, number, one_off=False): + def get_container_name(self, service_name, number, one_off=False): if self.custom_container_name and not one_off: return self.custom_container_name container_name = build_container_name( - self.project, self.name, number, one_off, + self.project, service_name, number, one_off, ) ext_links_origins = [l.split(':')[0] for l in self.options.get('external_links', [])] if container_name in ext_links_origins: diff --git a/tests/unit/service_test.py b/tests/unit/service_test.py index 5c5c2bf67..50b09c87f 100644 --- a/tests/unit/service_test.py +++ b/tests/unit/service_test.py @@ -179,7 +179,7 @@ class ServiceTest(unittest.TestCase): external_links=['default_foo_1'] ) with self.assertRaises(DependencyError): - service.get_container_name(1) + service.get_container_name('foo', 1) def test_mem_reservation(self): self.mock_client.create_host_config.return_value = {} From 8a08eb668876e73d5f18983fb591cce626ca4b27 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Mon, 16 Oct 2017 11:43:06 -0700 Subject: [PATCH 196/244] Move build labels parsing to config module Signed-off-by: Joffrey F --- compose/service.py | 3 +-- tests/integration/service_test.py | 19 +------------------ tests/unit/service_test.py | 4 ++-- 3 files changed, 4 insertions(+), 22 deletions(-) diff --git a/compose/service.py b/compose/service.py index 22a7ca53a..48d428cb8 100644 --- a/compose/service.py +++ b/compose/service.py @@ -23,7 +23,6 @@ from . import const from . import progress_stream from .config import DOCKER_CONFIG_KEYS from .config import merge_environment -from .config.config import parse_labels from .config.errors import DependencyError from .config.types import ServicePort from .config.types import VolumeSpec @@ -918,7 +917,7 @@ class Service(object): nocache=no_cache, dockerfile=build_opts.get('dockerfile', None), cache_from=build_opts.get('cache_from', None), - labels=parse_labels(build_opts.get('labels', None)), + labels=build_opts.get('labels', None), buildargs=build_args, network_mode=build_opts.get('network', None), target=build_opts.get('target', None), diff --git a/tests/integration/service_test.py b/tests/integration/service_test.py index 6cf8ddaa9..3ddf991b3 100644 --- a/tests/integration/service_test.py +++ b/tests/integration/service_test.py @@ -761,7 +761,7 @@ class ServiceTest(DockerClientTestCase): assert service.image() assert "build_version=2" in service.image()['ContainerConfig']['Cmd'] - def test_build_with_build_labels_dict(self): + def test_build_with_build_labels(self): base_dir = tempfile.mkdtemp() self.addCleanup(shutil.rmtree, base_dir) @@ -778,23 +778,6 @@ class ServiceTest(DockerClientTestCase): assert service.image() assert service.image()['Config']['Labels']['com.docker.compose.test'] == 'true' - def test_build_with_build_labels_list(self): - base_dir = tempfile.mkdtemp() - self.addCleanup(shutil.rmtree, base_dir) - - with open(os.path.join(base_dir, 'Dockerfile'), 'w') as f: - f.write('FROM busybox\n') - - service = self.create_service('buildlabels', build={ - 'context': text_type(base_dir), - 'labels': ['com.docker.compose.test=true'] - }) - service.build() - self.addCleanup(self.client.remove_image, service.image_name) - - assert service.image() - assert service.image()['Config']['Labels']['com.docker.compose.test'] == 'true' - @no_cluster('Container networks not on Swarm') def test_build_with_network(self): base_dir = tempfile.mkdtemp() diff --git a/tests/unit/service_test.py b/tests/unit/service_test.py index 50b09c87f..0bf0280de 100644 --- a/tests/unit/service_test.py +++ b/tests/unit/service_test.py @@ -473,7 +473,7 @@ class ServiceTest(unittest.TestCase): nocache=False, rm=True, buildargs={}, - labels={}, + labels=None, cache_from=None, network_mode=None, target=None, @@ -514,7 +514,7 @@ class ServiceTest(unittest.TestCase): nocache=False, rm=True, buildargs={}, - labels={}, + labels=None, cache_from=None, network_mode=None, target=None, From dfa7380f3781f182be236f2ba932afc7b19e6acf Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Wed, 18 Oct 2017 16:34:54 -0700 Subject: [PATCH 197/244] Add missing test constraint Signed-off-by: Joffrey F --- tests/acceptance/cli_test.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/acceptance/cli_test.py b/tests/acceptance/cli_test.py index 22756bd3d..5398f0bb2 100644 --- a/tests/acceptance/cli_test.py +++ b/tests/acceptance/cli_test.py @@ -737,12 +737,13 @@ class CLITestCase(DockerClientTestCase): def test_run_one_off_with_volume_merge(self): self.base_dir = 'tests/fixtures/simple-composefile-volume-ready' volume_path = os.path.abspath(os.path.join(os.getcwd(), self.base_dir, 'files')) - create_host_file(self.client, os.path.join(volume_path, 'example.txt')) + node = create_host_file(self.client, os.path.join(volume_path, 'example.txt')) self.dispatch([ '-f', 'docker-compose.merge.yml', 'run', '-v', '{}:/data'.format(volume_path), + '-e', 'constraint:node=={}'.format(node if node is not None else '*'), 'simple', 'test', '-f', '/data/example.txt' ], returncode=0) From ee6a293ae022877f8113e1fb517cb64b587f0a75 Mon Sep 17 00:00:00 2001 From: Guillermo Arribas Date: Tue, 17 Oct 2017 11:54:06 -0300 Subject: [PATCH 198/244] Placing dots in hostname no longer populates domainname if api >= 1.23 (fixes #4128) Signed-off-by: Guillermo Arribas --- compose/service.py | 8 +++++--- tests/unit/cli_test.py | 3 +++ tests/unit/service_test.py | 22 ++++++++++++++++------ 3 files changed, 24 insertions(+), 9 deletions(-) diff --git a/compose/service.py b/compose/service.py index 48d428cb8..923c3d944 100644 --- a/compose/service.py +++ b/compose/service.py @@ -14,6 +14,7 @@ from docker.errors import APIError from docker.errors import ImageNotFound from docker.errors import NotFound from docker.types import LogConfig +from docker.utils import version_lt from docker.utils.ports import build_port_bindings from docker.utils.ports import split_port from docker.utils.utils import convert_tmpfs_mounts @@ -748,9 +749,10 @@ class Service(object): # If a qualified hostname was given, split it into an # unqualified hostname and a domainname unless domainname - # was also given explicitly. This matches the behavior of - # the official Docker CLI in that scenario. - if ('hostname' in container_options and + # was also given explicitly. This matches behavior + # until Docker Engine 1.11.0 - Docker API 1.23. + if (version_lt(self.client.api_version, '1.23') and + 'hostname' in container_options and 'domainname' not in container_options and '.' in container_options['hostname']): parts = container_options['hostname'].partition('.') diff --git a/tests/unit/cli_test.py b/tests/unit/cli_test.py index f9ce240a3..1a324f50a 100644 --- a/tests/unit/cli_test.py +++ b/tests/unit/cli_test.py @@ -10,6 +10,7 @@ from io import StringIO import docker import py import pytest +from docker.constants import DEFAULT_DOCKER_API_VERSION from .. import mock from .. import unittest @@ -98,6 +99,7 @@ class CLITestCase(unittest.TestCase): @mock.patch('compose.cli.main.PseudoTerminal', autospec=True) def test_run_interactive_passes_logs_false(self, mock_pseudo_terminal, mock_run_operation): mock_client = mock.create_autospec(docker.APIClient) + mock_client.api_version = DEFAULT_DOCKER_API_VERSION project = Project.from_config( name='composetest', client=mock_client, @@ -130,6 +132,7 @@ class CLITestCase(unittest.TestCase): def test_run_service_with_restart_always(self): mock_client = mock.create_autospec(docker.APIClient) + mock_client.api_version = DEFAULT_DOCKER_API_VERSION project = Project.from_config( name='composetest', diff --git a/tests/unit/service_test.py b/tests/unit/service_test.py index 0bf0280de..02b4f6223 100644 --- a/tests/unit/service_test.py +++ b/tests/unit/service_test.py @@ -3,6 +3,7 @@ from __future__ import unicode_literals import docker import pytest +from docker.constants import DEFAULT_DOCKER_API_VERSION from docker.errors import APIError from .. import mock @@ -40,6 +41,7 @@ class ServiceTest(unittest.TestCase): def setUp(self): self.mock_client = mock.create_autospec(docker.APIClient) + self.mock_client.api_version = DEFAULT_DOCKER_API_VERSION def test_containers(self): service = Service('db', self.mock_client, 'myproject', image='foo') @@ -145,12 +147,6 @@ class ServiceTest(unittest.TestCase): self.assertEqual(service._get_volumes_from(), [container_id + ':rw']) from_service.create_container.assert_called_once_with() - def test_split_domainname_none(self): - service = Service('foo', image='foo', hostname='name', client=self.mock_client) - opts = service._get_container_create_options({'image': 'foo'}, 1) - self.assertEqual(opts['hostname'], 'name', 'hostname') - self.assertFalse('domainname' in opts, 'domainname') - def test_memory_swap_limit(self): self.mock_client.create_host_config.return_value = {} @@ -232,7 +228,18 @@ class ServiceTest(unittest.TestCase): {'Type': 'syslog', 'Config': {'syslog-address': 'tcp://192.168.0.42:123'}} ) + def test_split_domainname_none(self): + service = Service( + 'foo', + image='foo', + hostname='name.domain.tld', + client=self.mock_client) + opts = service._get_container_create_options({'image': 'foo'}, 1) + self.assertEqual(opts['hostname'], 'name.domain.tld', 'hostname') + self.assertFalse('domainname' in opts, 'domainname') + def test_split_domainname_fqdn(self): + self.mock_client.api_version = '1.22' service = Service( 'foo', hostname='name.domain.tld', @@ -243,6 +250,7 @@ class ServiceTest(unittest.TestCase): self.assertEqual(opts['domainname'], 'domain.tld', 'domainname') def test_split_domainname_both(self): + self.mock_client.api_version = '1.22' service = Service( 'foo', hostname='name', @@ -254,6 +262,7 @@ class ServiceTest(unittest.TestCase): self.assertEqual(opts['domainname'], 'domain.tld', 'domainname') def test_split_domainname_weird(self): + self.mock_client.api_version = '1.22' service = Service( 'foo', hostname='name.sub', @@ -857,6 +866,7 @@ class ServiceVolumesTest(unittest.TestCase): def setUp(self): self.mock_client = mock.create_autospec(docker.APIClient) + self.mock_client.api_version = DEFAULT_DOCKER_API_VERSION def test_build_volume_binding(self): binding = build_volume_binding(VolumeSpec.parse('/outside:/inside', True)) From 8cd46cd54de66300453b81881087b54e213472ea Mon Sep 17 00:00:00 2001 From: Guillermo Arribas Date: Thu, 19 Oct 2017 22:19:05 -0300 Subject: [PATCH 199/244] Allow empty default values in variable interpolation (fixes #5185) Signed-off-by: Guillermo Arribas --- compose/config/interpolation.py | 2 +- .../docker-compose.yml | 13 +++++++++++ tests/unit/config/config_test.py | 22 +++++++++++++++++++ 3 files changed, 36 insertions(+), 1 deletion(-) create mode 100644 tests/fixtures/environment-interpolation-with-defaults/docker-compose.yml diff --git a/compose/config/interpolation.py b/compose/config/interpolation.py index b13ac591a..df9c988e7 100644 --- a/compose/config/interpolation.py +++ b/compose/config/interpolation.py @@ -71,7 +71,7 @@ def recursive_interpolate(obj, interpolator): class TemplateWithDefaults(Template): - idpattern = r'[_a-z][_a-z0-9]*(?::?-[^}]+)?' + idpattern = r'[_a-z][_a-z0-9]*(?::?-[^}]*)?' # Modified from python2.7/string.py def substitute(self, mapping): diff --git a/tests/fixtures/environment-interpolation-with-defaults/docker-compose.yml b/tests/fixtures/environment-interpolation-with-defaults/docker-compose.yml new file mode 100644 index 000000000..42e7cbb6a --- /dev/null +++ b/tests/fixtures/environment-interpolation-with-defaults/docker-compose.yml @@ -0,0 +1,13 @@ +version: "2.1" + +services: + web: + # set value with default, default must be ignored + image: ${IMAGE:-alpine} + + # unset value with default value + ports: + - "${HOST_PORT:-80}:8000" + + # unset value with empty default + hostname: "host-${UNSET_VALUE:-}" diff --git a/tests/unit/config/config_test.py b/tests/unit/config/config_test.py index 8e3d4e2ee..1c01e52df 100644 --- a/tests/unit/config/config_test.py +++ b/tests/unit/config/config_test.py @@ -2894,6 +2894,28 @@ class InterpolationTest(unittest.TestCase): } ]) + @mock.patch.dict(os.environ) + def test_config_file_with_environment_variable_with_defaults(self): + project_dir = 'tests/fixtures/environment-interpolation-with-defaults' + os.environ.update( + IMAGE="busybox", + ) + + service_dicts = config.load( + config.find( + project_dir, None, Environment.from_env_file(project_dir) + ) + ).services + + self.assertEqual(service_dicts, [ + { + 'name': 'web', + 'image': 'busybox', + 'ports': types.ServicePort.parse('80:8000'), + 'hostname': 'host-', + } + ]) + @mock.patch.dict(os.environ) def test_unset_variable_produces_warning(self): os.environ.pop('FOO', None) From eb51f0fae8d28bccd0bb339472a8c1755d602c4f Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Thu, 19 Oct 2017 17:55:32 -0700 Subject: [PATCH 200/244] Add type converter to interpolation module Signed-off-by: Joffrey F --- compose/config/config.py | 4 +- compose/config/interpolation.py | 95 +++++++++++- tests/unit/config/interpolation_test.py | 198 +++++++++++++++++++++++- 3 files changed, 287 insertions(+), 10 deletions(-) diff --git a/compose/config/config.py b/compose/config/config.py index d5aaf9538..a9f82a29d 100644 --- a/compose/config/config.py +++ b/compose/config/config.py @@ -519,13 +519,13 @@ def process_config_file(config_file, environment, service_name=None): processed_config['secrets'] = interpolate_config_section( config_file, config_file.get_secrets(), - 'secrets', + 'secret', environment) if config_file.version >= const.COMPOSEFILE_V3_3: processed_config['configs'] = interpolate_config_section( config_file, config_file.get_configs(), - 'configs', + 'config', environment ) else: diff --git a/compose/config/interpolation.py b/compose/config/interpolation.py index df9c988e7..9d7e428c9 100644 --- a/compose/config/interpolation.py +++ b/compose/config/interpolation.py @@ -2,6 +2,7 @@ from __future__ import absolute_import from __future__ import unicode_literals import logging +import re from string import Template import six @@ -44,9 +45,13 @@ def interpolate_environment_variables(version, config, section, environment): ) +def get_config_path(config_key, section, name): + return '{}.{}.{}'.format(section, name, config_key) + + def interpolate_value(name, config_key, value, section, interpolator): try: - return recursive_interpolate(value, interpolator) + return recursive_interpolate(value, interpolator, get_config_path(config_key, section, name)) except InvalidInterpolation as e: raise ConfigurationError( 'Invalid interpolation format for "{config_key}" option ' @@ -57,16 +62,19 @@ def interpolate_value(name, config_key, value, section, interpolator): string=e.string)) -def recursive_interpolate(obj, interpolator): +def recursive_interpolate(obj, interpolator, config_path): + def append(config_path, key): + return '{}.{}'.format(config_path, key) + if isinstance(obj, six.string_types): - return interpolator.interpolate(obj) + return converter.convert(config_path, interpolator.interpolate(obj)) if isinstance(obj, dict): return dict( - (key, recursive_interpolate(val, interpolator)) + (key, recursive_interpolate(val, interpolator, append(config_path, key))) for (key, val) in obj.items() ) if isinstance(obj, list): - return [recursive_interpolate(val, interpolator) for val in obj] + return [recursive_interpolate(val, interpolator, config_path) for val in obj] return obj @@ -100,3 +108,80 @@ class TemplateWithDefaults(Template): class InvalidInterpolation(Exception): def __init__(self, string): self.string = string + + +PATH_JOKER = '[^.]+' + + +def re_path(*args): + return re.compile('^{}$'.format('.'.join(args))) + + +def re_path_basic(section, name): + return re_path(section, PATH_JOKER, name) + + +def service_path(*args): + return re_path('service', PATH_JOKER, *args) + + +def to_boolean(s): + s = s.lower() + if s in ['y', 'yes', 'true', 'on']: + return True + elif s in ['n', 'no', 'false', 'off']: + return False + raise ValueError('"{}" is not a valid boolean value'.format(s)) + + +def to_int(s): + # We must be able to handle octal representation for `mode` values notably + if six.PY3 and re.match('^0[0-9]+$', s.strip()): + s = '0o' + s[1:] + return int(s, base=0) + + +class ConversionMap(object): + map = { + service_path('blkio_config', 'weight'): to_int, + service_path('blkio_config', 'weight_device', 'weight'): to_int, + service_path('cpus'): float, + service_path('cpu_count'): to_int, + service_path('configs', 'mode'): to_int, + service_path('secrets', 'mode'): to_int, + service_path('healthcheck', 'retries'): to_int, + service_path('healthcheck', 'disable'): to_boolean, + service_path('deploy', 'replicas'): to_int, + service_path('deploy', 'update_config', 'parallelism'): to_int, + service_path('deploy', 'update_config', 'max_failure_ratio'): float, + service_path('deploy', 'restart_policy', 'max_attempts'): to_int, + service_path('mem_swappiness'): to_int, + service_path('oom_score_adj'): to_int, + service_path('ports', 'target'): to_int, + service_path('ports', 'published'): to_int, + service_path('scale'): to_int, + service_path('ulimits', PATH_JOKER): to_int, + service_path('ulimits', PATH_JOKER, 'soft'): to_int, + service_path('ulimits', PATH_JOKER, 'hard'): to_int, + service_path('privileged'): to_boolean, + service_path('read_only'): to_boolean, + service_path('stdin_open'): to_boolean, + service_path('tty'): to_boolean, + service_path('volumes', 'read_only'): to_boolean, + service_path('volumes', 'volume', 'nocopy'): to_boolean, + re_path_basic('network', 'attachable'): to_boolean, + re_path_basic('network', 'external'): to_boolean, + re_path_basic('network', 'internal'): to_boolean, + re_path_basic('volume', 'external'): to_boolean, + re_path_basic('secret', 'external'): to_boolean, + re_path_basic('config', 'external'): to_boolean, + } + + def convert(self, path, value): + for rexp in self.map.keys(): + if rexp.match(path): + return self.map[rexp](value) + return value + + +converter = ConversionMap() diff --git a/tests/unit/config/interpolation_test.py b/tests/unit/config/interpolation_test.py index 018a5621a..516f5c9e9 100644 --- a/tests/unit/config/interpolation_test.py +++ b/tests/unit/config/interpolation_test.py @@ -9,12 +9,22 @@ from compose.config.interpolation import Interpolator from compose.config.interpolation import InvalidInterpolation from compose.config.interpolation import TemplateWithDefaults from compose.const import COMPOSEFILE_V2_0 as V2_0 -from compose.const import COMPOSEFILE_V3_1 as V3_1 +from compose.const import COMPOSEFILE_V2_3 as V2_3 +from compose.const import COMPOSEFILE_V3_4 as V3_4 @pytest.fixture def mock_env(): - return Environment({'USER': 'jenny', 'FOO': 'bar'}) + return Environment({ + 'USER': 'jenny', + 'FOO': 'bar', + 'TRUE': 'True', + 'FALSE': 'OFF', + 'POSINT': '50', + 'NEGINT': '-200', + 'FLOAT': '0.145', + 'MODE': '0600', + }) @pytest.fixture @@ -102,7 +112,189 @@ def test_interpolate_environment_variables_in_secrets(mock_env): }, 'other': {}, } - value = interpolate_environment_variables(V3_1, secrets, 'volume', mock_env) + value = interpolate_environment_variables(V3_4, secrets, 'secret', mock_env) + assert value == expected + + +def test_interpolate_environment_services_convert_types_v2(mock_env): + entry = { + 'service1': { + 'blkio_config': { + 'weight': '${POSINT}', + 'weight_device': [{'file': '/dev/sda1', 'weight': '${POSINT}'}] + }, + 'cpus': '${FLOAT}', + 'cpu_count': '$POSINT', + 'healthcheck': { + 'retries': '${POSINT:-3}', + 'disable': '${FALSE}', + 'command': 'true' + }, + 'mem_swappiness': '${DEFAULT:-127}', + 'oom_score_adj': '${NEGINT}', + 'scale': '${POSINT}', + 'ulimits': { + 'nproc': '${POSINT}', + 'nofile': { + 'soft': '${POSINT}', + 'hard': '${DEFAULT:-40000}' + }, + }, + 'privileged': '${TRUE}', + 'read_only': '${DEFAULT:-no}', + 'tty': '${DEFAULT:-N}', + 'stdin_open': '${DEFAULT-on}', + } + } + + expected = { + 'service1': { + 'blkio_config': { + 'weight': 50, + 'weight_device': [{'file': '/dev/sda1', 'weight': 50}] + }, + 'cpus': 0.145, + 'cpu_count': 50, + 'healthcheck': { + 'retries': 50, + 'disable': False, + 'command': 'true' + }, + 'mem_swappiness': 127, + 'oom_score_adj': -200, + 'scale': 50, + 'ulimits': { + 'nproc': 50, + 'nofile': { + 'soft': 50, + 'hard': 40000 + }, + }, + 'privileged': True, + 'read_only': False, + 'tty': False, + 'stdin_open': True, + } + } + + value = interpolate_environment_variables(V2_3, entry, 'service', mock_env) + assert value == expected + + +def test_interpolate_environment_services_convert_types_v3(mock_env): + entry = { + 'service1': { + 'healthcheck': { + 'retries': '${POSINT:-3}', + 'disable': '${FALSE}', + 'command': 'true' + }, + 'ulimits': { + 'nproc': '${POSINT}', + 'nofile': { + 'soft': '${POSINT}', + 'hard': '${DEFAULT:-40000}' + }, + }, + 'privileged': '${TRUE}', + 'read_only': '${DEFAULT:-no}', + 'tty': '${DEFAULT:-N}', + 'stdin_open': '${DEFAULT-on}', + 'deploy': { + 'update_config': { + 'parallelism': '${DEFAULT:-2}', + 'max_failure_ratio': '${FLOAT}', + }, + 'restart_policy': { + 'max_attempts': '$POSINT', + }, + 'replicas': '${DEFAULT-3}' + }, + 'ports': [{'target': '${POSINT}', 'published': '${DEFAULT:-5000}'}], + 'configs': [{'mode': '${MODE}', 'source': 'config1'}], + 'secrets': [{'mode': '${MODE}', 'source': 'secret1'}], + } + } + + expected = { + 'service1': { + 'healthcheck': { + 'retries': 50, + 'disable': False, + 'command': 'true' + }, + 'ulimits': { + 'nproc': 50, + 'nofile': { + 'soft': 50, + 'hard': 40000 + }, + }, + 'privileged': True, + 'read_only': False, + 'tty': False, + 'stdin_open': True, + 'deploy': { + 'update_config': { + 'parallelism': 2, + 'max_failure_ratio': 0.145, + }, + 'restart_policy': { + 'max_attempts': 50, + }, + 'replicas': 3 + }, + 'ports': [{'target': 50, 'published': 5000}], + 'configs': [{'mode': 0o600, 'source': 'config1'}], + 'secrets': [{'mode': 0o600, 'source': 'secret1'}], + } + } + + value = interpolate_environment_variables(V3_4, entry, 'service', mock_env) + assert value == expected + + +def test_interpolate_environment_network_convert_types(mock_env): + entry = { + 'network1': { + 'external': '${FALSE}', + 'attachable': '${TRUE}', + 'internal': '${DEFAULT:-false}' + } + } + + expected = { + 'network1': { + 'external': False, + 'attachable': True, + 'internal': False, + } + } + + value = interpolate_environment_variables(V3_4, entry, 'network', mock_env) + assert value == expected + + +def test_interpolate_environment_external_resource_convert_types(mock_env): + entry = { + 'resource1': { + 'external': '${TRUE}', + } + } + + expected = { + 'resource1': { + 'external': True, + } + } + + value = interpolate_environment_variables(V3_4, entry, 'network', mock_env) + assert value == expected + value = interpolate_environment_variables(V3_4, entry, 'volume', mock_env) + assert value == expected + value = interpolate_environment_variables(V3_4, entry, 'secret', mock_env) + assert value == expected + value = interpolate_environment_variables(V3_4, entry, 'config', mock_env) assert value == expected From 7dfb856244fb5bb5690c33db12cb2ad2e072f058 Mon Sep 17 00:00:00 2001 From: Reut Sharabani Date: Mon, 23 Oct 2017 23:21:16 +0300 Subject: [PATCH 201/244] Better installation instruction in release notes Changed sample download script to use the built in `-o` optoin in `curl` instead of redicrecting stdout's output. This allows users to prepend `sudo` to the snippet to make it work in common use cases where root permissions are needed to create the output file. From `curl`: -o, --output Write output to instead of stdout. Signed-off-by: Reut Sharabani --- project/RELEASE-PROCESS.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/project/RELEASE-PROCESS.md b/project/RELEASE-PROCESS.md index 5b30545f4..d4afb87b9 100644 --- a/project/RELEASE-PROCESS.md +++ b/project/RELEASE-PROCESS.md @@ -89,7 +89,7 @@ When prompted build the non-linux binaries and test them. Alternatively, you can use the usual commands to install or upgrade Compose: ``` - curl -L https://github.com/docker/compose/releases/download/1.16.0/docker-compose-`uname -s`-`uname -m` > /usr/local/bin/docker-compose + curl -L https://github.com/docker/compose/releases/download/1.16.0/docker-compose-`uname -s`-`uname -m` -o /usr/local/bin/docker-compose chmod +x /usr/local/bin/docker-compose ``` From e022f32ee99ffa67d6f224d3ac77151c8371774d Mon Sep 17 00:00:00 2001 From: Guillermo Arribas Date: Mon, 23 Oct 2017 12:48:44 -0300 Subject: [PATCH 202/244] Wrong format in the healthcheck test does not issue a warning (fixes #4424) Signed-off-by: Guillermo Arribas --- compose/config/config.py | 33 ++++------ compose/config/validation.py | 24 +++++++ tests/unit/config/config_test.py | 110 ++++++++++++++++++++++--------- 3 files changed, 115 insertions(+), 52 deletions(-) diff --git a/compose/config/config.py b/compose/config/config.py index a9f82a29d..8a2b2a776 100644 --- a/compose/config/config.py +++ b/compose/config/config.py @@ -47,6 +47,7 @@ from .validation import validate_config_section from .validation import validate_cpu from .validation import validate_depends_on from .validation import validate_extends_file_path +from .validation import validate_healthcheck from .validation import validate_links from .validation import validate_network_mode from .validation import validate_pid_mode @@ -686,6 +687,7 @@ def validate_service(service_config, service_names, config_file): validate_pid_mode(service_config, service_names) validate_depends_on(service_config, service_names) validate_links(service_config, service_names) + validate_healthcheck(service_config) if not service_dict.get('image') and has_uppercase(service_name): raise ConfigurationError( @@ -724,7 +726,7 @@ def process_service(service_config): service_dict[field] = to_list(service_dict[field]) service_dict = process_blkio_config(process_ports( - process_healthcheck(service_dict, service_config.name) + process_healthcheck(service_dict) )) return service_dict @@ -788,33 +790,20 @@ def process_blkio_config(service_dict): return service_dict -def process_healthcheck(service_dict, service_name): +def process_healthcheck(service_dict): if 'healthcheck' not in service_dict: return service_dict - hc = {} - raw = service_dict['healthcheck'] - - if raw.get('disable'): - if len(raw) > 1: - raise ConfigurationError( - 'Service "{}" defines an invalid healthcheck: ' - '"disable: true" cannot be combined with other options' - .format(service_name)) - hc['test'] = ['NONE'] - elif 'test' in raw: - hc['test'] = raw['test'] + if 'disable' in service_dict['healthcheck']: + del service_dict['healthcheck']['disable'] + service_dict['healthcheck']['test'] = ['NONE'] for field in ['interval', 'timeout', 'start_period']: - if field in raw: - if not isinstance(raw[field], six.integer_types): - hc[field] = parse_nanoseconds_int(raw[field]) - else: # Conversion has been done previously - hc[field] = raw[field] - if 'retries' in raw: - hc['retries'] = raw['retries'] + if field in service_dict['healthcheck']: + if not isinstance(service_dict['healthcheck'][field], six.integer_types): + service_dict['healthcheck'][field] = parse_nanoseconds_int( + service_dict['healthcheck'][field]) - service_dict['healthcheck'] = hc return service_dict diff --git a/compose/config/validation.py b/compose/config/validation.py index 940775a20..8247cf150 100644 --- a/compose/config/validation.py +++ b/compose/config/validation.py @@ -465,3 +465,27 @@ def handle_errors(errors, format_error_func, filename): "The Compose file{file_msg} is invalid because:\n{error_msg}".format( file_msg=" '{}'".format(filename) if filename else "", error_msg=error_msg)) + + +def validate_healthcheck(service_config): + healthcheck = service_config.config.get('healthcheck', {}) + + if 'test' in healthcheck and isinstance(healthcheck['test'], list): + if len(healthcheck['test']) == 0: + raise ConfigurationError( + 'Service "{}" defines an invalid healthcheck: ' + '"test" is an empty list' + .format(service_config.name)) + + # when disable is true config.py::process_healthcheck adds "test: ['NONE']" to service_config + elif healthcheck['test'][0] == 'NONE' and len(healthcheck) > 1: + raise ConfigurationError( + 'Service "{}" defines an invalid healthcheck: ' + '"disable: true" cannot be combined with other options' + .format(service_config.name)) + + elif healthcheck['test'][0] not in ('NONE', 'CMD', 'CMD-SHELL'): + raise ConfigurationError( + 'Service "{}" defines an invalid healthcheck: ' + 'when "test" is a list the first item must be either NONE, CMD or CMD-SHELL' + .format(service_config.name)) diff --git a/tests/unit/config/config_test.py b/tests/unit/config/config_test.py index 1c01e52df..a758154c0 100644 --- a/tests/unit/config/config_test.py +++ b/tests/unit/config/config_test.py @@ -34,7 +34,6 @@ from compose.const import COMPOSEFILE_V3_1 as V3_1 from compose.const import COMPOSEFILE_V3_2 as V3_2 from compose.const import COMPOSEFILE_V3_3 as V3_3 from compose.const import IS_WINDOWS_PLATFORM -from compose.utils import nanoseconds_from_time_seconds from tests import mock from tests import unittest @@ -4210,52 +4209,103 @@ class BuildPathTest(unittest.TestCase): class HealthcheckTest(unittest.TestCase): def test_healthcheck(self): - service_dict = make_service_dict( - 'test', - {'healthcheck': { - 'test': ['CMD', 'true'], - 'interval': '1s', - 'timeout': '1m', - 'retries': 3, - 'start_period': '10s' - }}, - '.', + config_dict = config.load( + build_config_details({ + 'version': '2.3', + 'services': { + 'test': { + 'image': 'busybox', + 'healthcheck': { + 'test': ['CMD', 'true'], + 'interval': '1s', + 'timeout': '1m', + 'retries': 3, + 'start_period': '10s', + } + } + } + + }) ) - assert service_dict['healthcheck'] == { + serialized_config = yaml.load(serialize_config(config_dict)) + serialized_service = serialized_config['services']['test'] + + assert serialized_service['healthcheck'] == { 'test': ['CMD', 'true'], - 'interval': nanoseconds_from_time_seconds(1), - 'timeout': nanoseconds_from_time_seconds(60), + 'interval': '1s', + 'timeout': '1m', 'retries': 3, - 'start_period': nanoseconds_from_time_seconds(10) + 'start_period': '10s' } def test_disable(self): - service_dict = make_service_dict( - 'test', - {'healthcheck': { - 'disable': True, - }}, - '.', + config_dict = config.load( + build_config_details({ + 'version': '2.3', + 'services': { + 'test': { + 'image': 'busybox', + 'healthcheck': { + 'disable': True, + } + } + } + + }) ) - assert service_dict['healthcheck'] == { + serialized_config = yaml.load(serialize_config(config_dict)) + serialized_service = serialized_config['services']['test'] + + assert serialized_service['healthcheck'] == { 'test': ['NONE'], } def test_disable_with_other_config_is_invalid(self): with pytest.raises(ConfigurationError) as excinfo: - make_service_dict( - 'invalid-healthcheck', - {'healthcheck': { - 'disable': True, - 'interval': '1s', - }}, - '.', + config.load( + build_config_details({ + 'version': '2.3', + 'services': { + 'invalid-healthcheck': { + 'image': 'busybox', + 'healthcheck': { + 'disable': True, + 'interval': '1s', + } + } + } + + }) ) assert 'invalid-healthcheck' in excinfo.exconly() - assert 'disable' in excinfo.exconly() + assert '"disable: true" cannot be combined with other options' in excinfo.exconly() + + def test_healthcheck_with_invalid_test(self): + with pytest.raises(ConfigurationError) as excinfo: + config.load( + build_config_details({ + 'version': '2.3', + 'services': { + 'invalid-healthcheck': { + 'image': 'busybox', + 'healthcheck': { + 'test': ['true'], + 'interval': '1s', + 'timeout': '1m', + 'retries': 3, + 'start_period': '10s', + } + } + } + + }) + ) + + assert 'invalid-healthcheck' in excinfo.exconly() + assert 'the first item must be either NONE, CMD or CMD-SHELL' in excinfo.exconly() class GetDefaultConfigFilesTestCase(unittest.TestCase): From 947e98be387a2c534710a46f49679b5499733581 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Mon, 23 Oct 2017 14:49:36 -0700 Subject: [PATCH 203/244] Improve process_healthcheck readability Signed-off-by: Joffrey F --- compose/config/config.py | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/compose/config/config.py b/compose/config/config.py index 8a2b2a776..af4b69ce7 100644 --- a/compose/config/config.py +++ b/compose/config/config.py @@ -794,15 +794,16 @@ def process_healthcheck(service_dict): if 'healthcheck' not in service_dict: return service_dict - if 'disable' in service_dict['healthcheck']: - del service_dict['healthcheck']['disable'] - service_dict['healthcheck']['test'] = ['NONE'] + hc = service_dict['healthcheck'] + + if 'disable' in hc: + del hc['disable'] + hc['test'] = ['NONE'] for field in ['interval', 'timeout', 'start_period']: - if field in service_dict['healthcheck']: - if not isinstance(service_dict['healthcheck'][field], six.integer_types): - service_dict['healthcheck'][field] = parse_nanoseconds_int( - service_dict['healthcheck'][field]) + if field not in hc or isinstance(hc[field], six.integer_types): + continue + hc[field] = parse_nanoseconds_int(hc[field]) return service_dict From 558df8fe2f3903c06b58e354a1a749ab09c5ebea Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Mon, 23 Oct 2017 17:18:45 -0700 Subject: [PATCH 204/244] Add support for BOM-signed env files Signed-off-by: Joffrey F --- compose/config/environment.py | 2 +- tests/unit/config/environment_test.py | 14 ++++++++++++++ 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/compose/config/environment.py b/compose/config/environment.py index 4ba228c8a..0087b6128 100644 --- a/compose/config/environment.py +++ b/compose/config/environment.py @@ -32,7 +32,7 @@ def env_vars_from_file(filename): elif not os.path.isfile(filename): raise ConfigurationError("%s is not a file." % (filename)) env = {} - with contextlib.closing(codecs.open(filename, 'r', 'utf-8')) as fileobj: + with contextlib.closing(codecs.open(filename, 'r', 'utf-8-sig')) as fileobj: for line in fileobj: line = line.strip() if line and not line.startswith('#'): diff --git a/tests/unit/config/environment_test.py b/tests/unit/config/environment_test.py index 20446d2bf..854aee5a3 100644 --- a/tests/unit/config/environment_test.py +++ b/tests/unit/config/environment_test.py @@ -3,6 +3,11 @@ from __future__ import absolute_import from __future__ import print_function from __future__ import unicode_literals +import codecs + +import pytest + +from compose.config.environment import env_vars_from_file from compose.config.environment import Environment from tests import unittest @@ -38,3 +43,12 @@ class EnvironmentTest(unittest.TestCase): assert env.get_boolean('BAZ') is False assert env.get_boolean('FOOBAR') is True assert env.get_boolean('UNDEFINED') is False + + def test_env_vars_from_file_bom(self): + tmpdir = pytest.ensuretemp('env_file') + self.addCleanup(tmpdir.remove) + with codecs.open('{}/bom.env'.format(str(tmpdir)), 'w', encoding='utf-8') as f: + f.write('\ufeffPARK_BOM=ë°•ë´„\n') + assert env_vars_from_file(str(tmpdir.join('bom.env'))) == { + 'PARK_BOM': 'ë°•ë´„' + } From a1a6fb485b40cc2f4fff19fc7f5067fcf0292bfa Mon Sep 17 00:00:00 2001 From: Guillermo Arribas Date: Thu, 19 Oct 2017 22:07:30 -0300 Subject: [PATCH 205/244] docker-compose exec doesn't have -e option (fixes #4551) Signed-off-by: Guillermo Arribas --- compose/cli/main.py | 59 ++++++++++++------- tests/acceptance/cli_test.py | 26 ++++++++ .../environment-exec/docker-compose.yml | 10 ++++ 3 files changed, 74 insertions(+), 21 deletions(-) create mode 100644 tests/fixtures/environment-exec/docker-compose.yml diff --git a/compose/cli/main.py b/compose/cli/main.py index face38e6d..c3e30919d 100644 --- a/compose/cli/main.py +++ b/compose/cli/main.py @@ -14,6 +14,8 @@ from distutils.spawn import find_executable from inspect import getdoc from operator import attrgetter +import docker + from . import errors from . import signals from .. import __version__ @@ -402,7 +404,7 @@ class TopLevelCommand(object): """ Execute a command in a running container - Usage: exec [options] SERVICE COMMAND [ARGS...] + Usage: exec [options] [-e KEY=VAL...] SERVICE COMMAND [ARGS...] Options: -d Detached mode: Run command in the background. @@ -412,11 +414,16 @@ class TopLevelCommand(object): allocates a TTY. --index=index index of the container if there are multiple instances of a service [default: 1] + -e, --env KEY=VAL Set environment variables (can be used multiple times, + not supported in API < 1.25) """ index = int(options.get('--index')) service = self.project.get_service(options['SERVICE']) detach = options['-d'] + if options['--env'] and docker.utils.version_lt(self.project.client.api_version, '1.25'): + raise UserError("Setting environment for exec is not supported in API < 1.25'") + try: container = service.get_container(number=index) except ValueError as e: @@ -425,26 +432,7 @@ class TopLevelCommand(object): tty = not options["-T"] if IS_WINDOWS_PLATFORM and not detach: - args = ["exec"] - - if options["-d"]: - args += ["--detach"] - else: - args += ["--interactive"] - - if not options["-T"]: - args += ["--tty"] - - if options["--privileged"]: - args += ["--privileged"] - - if options["--user"]: - args += ["--user", options["--user"]] - - args += [container.id] - args += command - - sys.exit(call_docker(args)) + sys.exit(call_docker(build_exec_command(options, container.id, command))) create_exec_options = { "privileged": options["--privileged"], @@ -453,6 +441,9 @@ class TopLevelCommand(object): "stdin": tty, } + if docker.utils.version_gte(self.project.client.api_version, '1.25'): + create_exec_options["environment"] = options["--env"] + exec_id = container.create_exec(command, **create_exec_options) if detach: @@ -1295,3 +1286,29 @@ def parse_scale_args(options): ) res[service_name] = num return res + + +def build_exec_command(options, container_id, command): + args = ["exec"] + + if options["-d"]: + args += ["--detach"] + else: + args += ["--interactive"] + + if not options["-T"]: + args += ["--tty"] + + if options["--privileged"]: + args += ["--privileged"] + + if options["--user"]: + args += ["--user", options["--user"]] + + if options["--env"]: + for env_variable in options["--env"]: + args += ["--env", env_variable] + + args += [container_id] + args += command + return args diff --git a/tests/acceptance/cli_test.py b/tests/acceptance/cli_test.py index 5398f0bb2..0fcf866ff 100644 --- a/tests/acceptance/cli_test.py +++ b/tests/acceptance/cli_test.py @@ -33,6 +33,7 @@ from tests.integration.testcases import no_cluster from tests.integration.testcases import pull_busybox from tests.integration.testcases import SWARM_SKIP_RM_VOLUMES from tests.integration.testcases import v2_1_only +from tests.integration.testcases import v2_2_only from tests.integration.testcases import v2_only from tests.integration.testcases import v3_only @@ -1393,6 +1394,31 @@ class CLITestCase(DockerClientTestCase): self.assertEqual(stdout, "operator\n") self.assertEqual(stderr, "") + @v2_2_only() + def test_exec_service_with_environment_overridden(self): + name = 'service' + self.base_dir = 'tests/fixtures/environment-exec' + self.dispatch(['up', '-d']) + self.assertEqual(len(self.project.containers()), 1) + + stdout, stderr = self.dispatch([ + 'exec', + '-T', + '-e', 'foo=notbar', + '--env', 'alpha=beta', + name, + 'env', + ]) + + # env overridden + assert 'foo=notbar' in stdout + # keep environment from yaml + assert 'hello=world' in stdout + # added option from command line + assert 'alpha=beta' in stdout + + self.assertEqual(stderr, '') + def test_run_service_without_links(self): self.base_dir = 'tests/fixtures/links-composefile' self.dispatch(['run', 'console', '/bin/true']) diff --git a/tests/fixtures/environment-exec/docker-compose.yml b/tests/fixtures/environment-exec/docker-compose.yml new file mode 100644 index 000000000..813606eb8 --- /dev/null +++ b/tests/fixtures/environment-exec/docker-compose.yml @@ -0,0 +1,10 @@ +version: "2.2" + +services: + service: + image: busybox:latest + command: top + + environment: + foo: bar + hello: world From f89a55e4881b096e8cfcbf84fc9ae900eef89798 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Wed, 25 Oct 2017 15:07:00 -0700 Subject: [PATCH 206/244] Add support for oom_kill_disable in service config Signed-off-by: Joffrey F --- compose/config/config.py | 1 + compose/config/config_schema_v2.1.json | 1 + compose/config/config_schema_v2.2.json | 1 + compose/config/config_schema_v2.3.json | 1 + compose/config/interpolation.py | 1 + compose/service.py | 2 ++ tests/integration/service_test.py | 9 +++++++-- 7 files changed, 14 insertions(+), 2 deletions(-) diff --git a/compose/config/config.py b/compose/config/config.py index af4b69ce7..adfb53d8f 100644 --- a/compose/config/config.py +++ b/compose/config/config.py @@ -91,6 +91,7 @@ DOCKER_CONFIG_KEYS = [ 'mem_swappiness', 'net', 'oom_score_adj', + 'oom_kill_disable', 'pid', 'ports', 'privileged', diff --git a/compose/config/config_schema_v2.1.json b/compose/config/config_schema_v2.1.json index 24e6ba02c..6b74f0ed6 100644 --- a/compose/config/config_schema_v2.1.json +++ b/compose/config/config_schema_v2.1.json @@ -229,6 +229,7 @@ } ] }, + "oom_kill_disable": {"type": "boolean"}, "oom_score_adj": {"type": "integer", "minimum": -1000, "maximum": 1000}, "group_add": { "type": "array", diff --git a/compose/config/config_schema_v2.2.json b/compose/config/config_schema_v2.2.json index 86fc5df95..21343b893 100644 --- a/compose/config/config_schema_v2.2.json +++ b/compose/config/config_schema_v2.2.json @@ -235,6 +235,7 @@ } ] }, + "oom_kill_disable": {"type": "boolean"}, "oom_score_adj": {"type": "integer", "minimum": -1000, "maximum": 1000}, "group_add": { "type": "array", diff --git a/compose/config/config_schema_v2.3.json b/compose/config/config_schema_v2.3.json index ceaf44954..0e709e9d9 100644 --- a/compose/config/config_schema_v2.3.json +++ b/compose/config/config_schema_v2.3.json @@ -237,6 +237,7 @@ } ] }, + "oom_kill_disable": {"type": "boolean"}, "oom_score_adj": {"type": "integer", "minimum": -1000, "maximum": 1000}, "group_add": { "type": "array", diff --git a/compose/config/interpolation.py b/compose/config/interpolation.py index 9d7e428c9..45a5f9fc2 100644 --- a/compose/config/interpolation.py +++ b/compose/config/interpolation.py @@ -156,6 +156,7 @@ class ConversionMap(object): service_path('deploy', 'update_config', 'max_failure_ratio'): float, service_path('deploy', 'restart_policy', 'max_attempts'): to_int, service_path('mem_swappiness'): to_int, + service_path('oom_kill_disable'): to_boolean, service_path('oom_score_adj'): to_int, service_path('ports', 'target'): to_int, service_path('ports', 'published'): to_int, diff --git a/compose/service.py b/compose/service.py index 923c3d944..8839c6cfd 100644 --- a/compose/service.py +++ b/compose/service.py @@ -77,6 +77,7 @@ HOST_CONFIG_KEYS = [ 'mem_reservation', 'memswap_limit', 'mem_swappiness', + 'oom_kill_disable', 'oom_score_adj', 'pid', 'pids_limit', @@ -860,6 +861,7 @@ class Service(object): sysctls=options.get('sysctls'), pids_limit=options.get('pids_limit'), tmpfs=options.get('tmpfs'), + oom_kill_disable=options.get('oom_kill_disable'), oom_score_adj=options.get('oom_score_adj'), mem_swappiness=options.get('mem_swappiness'), group_add=options.get('group_add'), diff --git a/tests/integration/service_test.py b/tests/integration/service_test.py index 3ddf991b3..deced2742 100644 --- a/tests/integration/service_test.py +++ b/tests/integration/service_test.py @@ -239,8 +239,7 @@ class ServiceTest(DockerClientTestCase): service.start_container(container) self.assertEqual(set(container.get('HostConfig.SecurityOpt')), set(security_opt)) - # @pytest.mark.xfail(True, reason='Not supported on most drivers') - @pytest.mark.skipif(True, reason='https://github.com/moby/moby/issues/34270') + @pytest.mark.xfail(True, reason='Not supported on most drivers') def test_create_container_with_storage_opt(self): storage_opt = {'size': '1G'} service = self.create_service('db', storage_opt=storage_opt) @@ -248,6 +247,12 @@ class ServiceTest(DockerClientTestCase): service.start_container(container) self.assertEqual(container.get('HostConfig.StorageOpt'), storage_opt) + def test_create_container_with_oom_kill_disable(self): + self.require_api_version('1.20') + service = self.create_service('db', oom_kill_disable=True) + container = service.create_container() + assert container.get('HostConfig.OomKillDisable') is True + def test_create_container_with_mac_address(self): service = self.create_service('db', mac_address='02:42:ac:11:65:43') container = service.create_container() From 574ac9f124f4d5048feb21c0131fdb13138e31c0 Mon Sep 17 00:00:00 2001 From: Andy Neff Date: Thu, 26 Oct 2017 11:42:57 -0400 Subject: [PATCH 207/244] Have stop_grace_period also set StopTimeout on create Signed-off-by: Andy Neff --- compose/service.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/compose/service.py b/compose/service.py index 8839c6cfd..14027a1cc 100644 --- a/compose/service.py +++ b/compose/service.py @@ -15,6 +15,7 @@ from docker.errors import ImageNotFound from docker.errors import NotFound from docker.types import LogConfig from docker.utils import version_lt +from docker.utils import version_gte from docker.utils.ports import build_port_bindings from docker.utils.ports import split_port from docker.utils.utils import convert_tmpfs_mounts @@ -760,6 +761,11 @@ class Service(object): container_options['hostname'] = parts[0] container_options['domainname'] = parts[2] + if (version_gte(self.client.api_version, '1.25') and + 'stop_grace_period' in self.options): + container_options['stop_timeout'] = parse_seconds_float( + self.options.pop('stop_grace_period')) + if 'ports' in container_options or 'expose' in self.options: container_options['ports'] = build_container_ports( formatted_ports(container_options.get('ports', [])), From 41d7d6e45ba56fb02e250ac70cab26110541dd42 Mon Sep 17 00:00:00 2001 From: Andy Neff Date: Fri, 27 Oct 2017 17:44:17 -0400 Subject: [PATCH 208/244] Added unit test and used stop_timeout Signed-off-by: Andy Neff --- compose/service.py | 5 ++--- tests/unit/service_test.py | 11 +++++++++++ 2 files changed, 13 insertions(+), 3 deletions(-) diff --git a/compose/service.py b/compose/service.py index 14027a1cc..245d5f7c7 100644 --- a/compose/service.py +++ b/compose/service.py @@ -14,8 +14,8 @@ from docker.errors import APIError from docker.errors import ImageNotFound from docker.errors import NotFound from docker.types import LogConfig -from docker.utils import version_lt from docker.utils import version_gte +from docker.utils import version_lt from docker.utils.ports import build_port_bindings from docker.utils.ports import split_port from docker.utils.utils import convert_tmpfs_mounts @@ -763,8 +763,7 @@ class Service(object): if (version_gte(self.client.api_version, '1.25') and 'stop_grace_period' in self.options): - container_options['stop_timeout'] = parse_seconds_float( - self.options.pop('stop_grace_period')) + container_options['stop_timeout'] = self.stop_timeout(None) if 'ports' in container_options or 'expose' in self.options: container_options['ports'] = build_container_ports( diff --git a/tests/unit/service_test.py b/tests/unit/service_test.py index 02b4f6223..4c879cae7 100644 --- a/tests/unit/service_test.py +++ b/tests/unit/service_test.py @@ -228,6 +228,17 @@ class ServiceTest(unittest.TestCase): {'Type': 'syslog', 'Config': {'syslog-address': 'tcp://192.168.0.42:123'}} ) + def test_stop_grace_period(self): + self.mock_client.api_version = '1.25' + self.mock_client.create_host_config.return_value = {} + service = Service( + 'foo', + image='foo', + client=self.mock_client, + stop_grace_period="1m35s") + opts = service._get_container_create_options({'image': 'foo'}, 1) + self.assertEqual(opts['stop_timeout'], 95) + def test_split_domainname_none(self): service = Service( 'foo', From 0f978642380c593f46448c4fcd91c23649bf3451 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Fri, 3 Nov 2017 12:26:31 -0700 Subject: [PATCH 209/244] Add shasum computation to download-binaries script Signed-off-by: Joffrey F --- script/release/download-binaries | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/script/release/download-binaries b/script/release/download-binaries index 5d01f5f75..bef5430f4 100755 --- a/script/release/download-binaries +++ b/script/release/download-binaries @@ -30,3 +30,8 @@ mkdir $DESTINATION wget -O $DESTINATION/docker-compose-Darwin-x86_64 $BASE_BINTRAY_URL/docker-compose-Darwin-x86_64 wget -O $DESTINATION/docker-compose-Linux-x86_64 $BASE_BINTRAY_URL/docker-compose-Linux-x86_64 wget -O $DESTINATION/docker-compose-Windows-x86_64.exe $APPVEYOR_URL + +echo -e "\n\nCopy the following lines into the integrity check table in the release notes:\n\n" +cd $DESTINATION +ls | xargs sha256sum | sed 's/ / | /g' | sed -r 's/([^ |]+)/`\1`/g' +cd - From 985010b88707b6b13ec7694d6eb06ca6f6e9d3dc Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Fri, 3 Nov 2017 12:36:53 -0700 Subject: [PATCH 210/244] 1.18.0dev Signed-off-by: Joffrey F --- compose/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/compose/__init__.py b/compose/__init__.py index 20392ec99..7b954eb4f 100644 --- a/compose/__init__.py +++ b/compose/__init__.py @@ -1,4 +1,4 @@ from __future__ import absolute_import from __future__ import unicode_literals -__version__ = '1.17.1' +__version__ = '1.18.0dev' From 183110e0b07453a1826b70f18da0b174d43ef237 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Tue, 7 Nov 2017 17:29:23 -0800 Subject: [PATCH 211/244] Bump SDK version to latest Signed-off-by: Joffrey F --- requirements.txt | 4 ++-- setup.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/requirements.txt b/requirements.txt index beeaa2851..0207b1938 100644 --- a/requirements.txt +++ b/requirements.txt @@ -3,7 +3,7 @@ cached-property==1.3.0 certifi==2017.4.17 chardet==3.0.4 colorama==0.3.9; sys_platform == 'win32' -docker==2.5.1 +docker==2.6.0 docker-pycreds==0.2.1 dockerpty==0.4.1 docopt==0.6.2 @@ -15,7 +15,7 @@ jsonschema==2.6.0 pypiwin32==219; sys_platform == 'win32' PySocks==1.6.7 PyYAML==3.12 -requests==2.11.1 +requests==2.18.4 six==1.10.0 texttable==0.9.1 urllib3==1.21.1 diff --git a/setup.py b/setup.py index 192a0f6af..08d708e95 100644 --- a/setup.py +++ b/setup.py @@ -33,10 +33,10 @@ install_requires = [ 'cached-property >= 1.2.0, < 2', 'docopt >= 0.6.1, < 0.7', 'PyYAML >= 3.10, < 4', - 'requests >= 2.6.1, != 2.11.0, < 2.12', + 'requests >= 2.6.1, != 2.11.0, != 2.12.2, != 2.18.0, < 2.19', 'texttable >= 0.9.0, < 0.10', 'websocket-client >= 0.32.0, < 1.0', - 'docker >= 2.5.1, < 3.0', + 'docker >= 2.6.0, < 3.0', 'dockerpty >= 0.4.1, < 0.5', 'six >= 1.3.0, < 2', 'jsonschema >= 2.5.1, < 3', From 8ecd15e5680b18491f7eb90403baafa6733c0501 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Wed, 8 Nov 2017 16:48:41 -0800 Subject: [PATCH 212/244] Include SDK attach bugfix Signed-off-by: Joffrey F --- requirements.txt | 2 +- setup.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/requirements.txt b/requirements.txt index 0207b1938..8d86b7d3a 100644 --- a/requirements.txt +++ b/requirements.txt @@ -3,7 +3,7 @@ cached-property==1.3.0 certifi==2017.4.17 chardet==3.0.4 colorama==0.3.9; sys_platform == 'win32' -docker==2.6.0 +docker==2.6.1 docker-pycreds==0.2.1 dockerpty==0.4.1 docopt==0.6.2 diff --git a/setup.py b/setup.py index 08d708e95..bc760c3ef 100644 --- a/setup.py +++ b/setup.py @@ -36,7 +36,7 @@ install_requires = [ 'requests >= 2.6.1, != 2.11.0, != 2.12.2, != 2.18.0, < 2.19', 'texttable >= 0.9.0, < 0.10', 'websocket-client >= 0.32.0, < 1.0', - 'docker >= 2.6.0, < 3.0', + 'docker >= 2.6.1, < 3.0', 'dockerpty >= 0.4.1, < 0.5', 'six >= 1.3.0, < 2', 'jsonschema >= 2.5.1, < 3', From b2c13e15343f9a44106eb5a85ba0c17c1de4c19e Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Thu, 9 Nov 2017 14:25:14 -0800 Subject: [PATCH 213/244] Remove redundant log message Signed-off-by: Joffrey F --- compose/service.py | 1 - 1 file changed, 1 deletion(-) diff --git a/compose/service.py b/compose/service.py index 245d5f7c7..366bb3746 100644 --- a/compose/service.py +++ b/compose/service.py @@ -514,7 +514,6 @@ class Service(object): volumes can be copied to the new container, before the original container is removed. """ - log.info("Recreating %s" % container.name) container.stop(timeout=self.stop_timeout(timeout)) container.rename_to_tmp_name() From 67dfcd6951add2460973fe4180459e4076a0f41f Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Wed, 25 Oct 2017 14:51:11 -0700 Subject: [PATCH 214/244] Add support for extra_hosts in build config Signed-off-by: Joffrey F --- compose/config/config.py | 1 + compose/config/config_schema_v2.3.json | 3 ++- compose/service.py | 1 + tests/integration/service_test.py | 23 +++++++++++++++++++++++ tests/unit/service_test.py | 4 +++- tox.ini | 1 - 6 files changed, 30 insertions(+), 3 deletions(-) diff --git a/compose/config/config.py b/compose/config/config.py index adfb53d8f..4c3f93ddb 100644 --- a/compose/config/config.py +++ b/compose/config/config.py @@ -1023,6 +1023,7 @@ def merge_build(output, base, override): md.merge_mapping('args', parse_build_arguments) md.merge_field('cache_from', merge_unique_items_lists, default=[]) md.merge_mapping('labels', parse_labels) + md.merge_mapping('extra_hosts', parse_extra_hosts) return dict(md) diff --git a/compose/config/config_schema_v2.3.json b/compose/config/config_schema_v2.3.json index 0e709e9d9..6f923871b 100644 --- a/compose/config/config_schema_v2.3.json +++ b/compose/config/config_schema_v2.3.json @@ -92,7 +92,8 @@ "cache_from": {"$ref": "#/definitions/list_of_strings"}, "network": {"type": "string"}, "target": {"type": "string"}, - "shm_size": {"type": ["integer", "string"]} + "shm_size": {"type": ["integer", "string"]}, + "extra_hosts": {"$ref": "#/definitions/list_or_dict"} }, "additionalProperties": false } diff --git a/compose/service.py b/compose/service.py index 366bb3746..0b6561d99 100644 --- a/compose/service.py +++ b/compose/service.py @@ -930,6 +930,7 @@ class Service(object): network_mode=build_opts.get('network', None), target=build_opts.get('target', None), shmsize=parse_bytes(build_opts.get('shm_size')) if build_opts.get('shm_size') else None, + extra_hosts=build_opts.get('extra_hosts', None), ) try: diff --git a/tests/integration/service_test.py b/tests/integration/service_test.py index deced2742..00bacebf5 100644 --- a/tests/integration/service_test.py +++ b/tests/integration/service_test.py @@ -833,6 +833,29 @@ class ServiceTest(DockerClientTestCase): assert service.image() assert service.image()['Config']['Labels']['com.docker.compose.test.target'] == 'one' + @v2_3_only() + def test_build_with_extra_hosts(self): + self.require_api_version('1.27') + base_dir = tempfile.mkdtemp() + self.addCleanup(shutil.rmtree, base_dir) + + with open(os.path.join(base_dir, 'Dockerfile'), 'w') as f: + f.write('\n'.join([ + 'FROM busybox', + 'RUN ping -c1 foobar', + 'RUN ping -c1 baz', + ])) + + service = self.create_service('build_extra_hosts', build={ + 'context': text_type(base_dir), + 'extra_hosts': { + 'foobar': '127.0.0.1', + 'baz': '127.0.0.1' + } + }) + service.build() + assert service.image() + def test_start_container_stays_unprivileged(self): service = self.create_service('web') container = create_and_start_container(service).inspect() diff --git a/tests/unit/service_test.py b/tests/unit/service_test.py index 4c879cae7..8e8f60203 100644 --- a/tests/unit/service_test.py +++ b/tests/unit/service_test.py @@ -498,6 +498,7 @@ class ServiceTest(unittest.TestCase): network_mode=None, target=None, shmsize=None, + extra_hosts=None, ) def test_ensure_image_exists_no_build(self): @@ -538,7 +539,8 @@ class ServiceTest(unittest.TestCase): cache_from=None, network_mode=None, target=None, - shmsize=None + shmsize=None, + extra_hosts=None, ) def test_build_does_not_pull(self): diff --git a/tox.ini b/tox.ini index e4f31ec85..749be3faa 100644 --- a/tox.ini +++ b/tox.ini @@ -18,7 +18,6 @@ deps = -rrequirements-dev.txt commands = py.test -v \ - --full-trace \ --cov=compose \ --cov-report html \ --cov-report term \ From fb43b8b6b7a0411f15124f50752e5343b2080d00 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Mon, 16 Oct 2017 16:56:46 -0700 Subject: [PATCH 215/244] Bump colorama (use unreleased fix) Signed-off-by: Joffrey F --- requirements.txt | 2 +- setup.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/requirements.txt b/requirements.txt index 8d86b7d3a..889f87a5a 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,13 +2,13 @@ backports.ssl-match-hostname==3.5.0.1; python_version < '3' cached-property==1.3.0 certifi==2017.4.17 chardet==3.0.4 -colorama==0.3.9; sys_platform == 'win32' docker==2.6.1 docker-pycreds==0.2.1 dockerpty==0.4.1 docopt==0.6.2 enum34==1.1.6; python_version < '3.4' functools32==3.2.3.post2; python_version < '3.2' +git+git://github.com/tartley/colorama.git@bd378c725b45eba0b8e5cc091c3ca76a954c92ff; sys_platform == 'win32' idna==2.5 ipaddress==1.0.18 jsonschema==2.6.0 diff --git a/setup.py b/setup.py index bc760c3ef..d03534040 100644 --- a/setup.py +++ b/setup.py @@ -55,7 +55,7 @@ extras_require = { ':python_version < "3.4"': ['enum34 >= 1.0.4, < 2'], ':python_version < "3.5"': ['backports.ssl_match_hostname >= 3.5'], ':python_version < "3.3"': ['ipaddress >= 1.0.16'], - ':sys_platform == "win32"': ['colorama >= 0.3.7, < 0.4'], + ':sys_platform == "win32"': ['colorama >= 0.3.9, < 0.4'], 'socks': ['PySocks >= 1.5.6, != 1.5.7, < 2'], } From cf782a3dbbe82ccabce8cddfd89ae6b00d6b50ac Mon Sep 17 00:00:00 2001 From: Drew Romanyk Date: Thu, 9 Nov 2017 17:53:27 -0600 Subject: [PATCH 216/244] Implement subnet config validation (fixes #4552) Signed-off-by: Drew Romanyk --- compose/config/config_schema_v3.5.json | 2 +- compose/config/validation.py | 30 +++++++++- tests/unit/config/config_test.py | 82 ++++++++++++++++++++++++++ 3 files changed, 112 insertions(+), 2 deletions(-) diff --git a/compose/config/config_schema_v3.5.json b/compose/config/config_schema_v3.5.json index 5400cd99f..c3ac559ee 100644 --- a/compose/config/config_schema_v3.5.json +++ b/compose/config/config_schema_v3.5.json @@ -419,7 +419,7 @@ "items": { "type": "object", "properties": { - "subnet": {"type": "string"} + "subnet": {"type": "string", "format": "subnet_ip_address"} }, "additionalProperties": false } diff --git a/compose/config/validation.py b/compose/config/validation.py index 8247cf150..a8061a5a4 100644 --- a/compose/config/validation.py +++ b/compose/config/validation.py @@ -5,6 +5,7 @@ import json import logging import os import re +import socket import sys import six @@ -43,6 +44,9 @@ DOCKER_CONFIG_HINTS = { VALID_NAME_CHARS = '[a-zA-Z0-9\._\-]' VALID_EXPOSE_FORMAT = r'^\d+(\-\d+)?(\/[a-zA-Z]+)?$' +VALID_IPV4_FORMAT = r'^(\d{1,3}.){3}\d{1,3}$' +VALID_IPV4_CIDR_FORMAT = r'^(\d|[1-2]\d|3[0-2])$' +VALID_IPV6_CIDR_FORMAT = r'^(\d|[1-9]\d|1[0-1]\d|12[0-8])$' @FormatChecker.cls_checks(format="ports", raises=ValidationError) @@ -64,6 +68,30 @@ def format_expose(instance): return True +@FormatChecker.cls_checks("subnet_ip_address", raises=ValidationError) +def format_subnet_ip_address(instance): + if isinstance(instance, six.string_types): + if '/' not in instance: + raise ValidationError("should be of the format 'IP_ADDRESS/CIDR'") + + ip_address, cidr = instance.split('/') + + if re.match(VALID_IPV4_FORMAT, ip_address): + if not (re.match(VALID_IPV4_CIDR_FORMAT, cidr) and + all(0 <= int(component) <= 255 for component in ip_address.split("."))): + raise ValidationError("should be of the format 'IP_ADDRESS/CIDR'") + elif re.match(VALID_IPV6_CIDR_FORMAT, cidr) and hasattr(socket, "inet_pton"): + try: + if not (socket.inet_pton(socket.AF_INET6, ip_address)): + raise ValidationError("should be of the format 'IP_ADDRESS/CIDR'") + except socket.error as e: + raise ValidationError(six.text_type(e)) + else: + raise ValidationError("should be of the format 'IP_ADDRESS/CIDR'") + + return True + + def match_named_volumes(service_dict, project_volumes): service_volumes = service_dict.get('volumes', []) for volume_spec in service_volumes: @@ -391,7 +419,7 @@ def process_config_schema_errors(error): def validate_against_config_schema(config_file): schema = load_jsonschema(config_file) - format_checker = FormatChecker(["ports", "expose"]) + format_checker = FormatChecker(["ports", "expose", "subnet_ip_address"]) validator = Draft4Validator( schema, resolver=RefResolver(get_resolver_path(), schema), diff --git a/tests/unit/config/config_test.py b/tests/unit/config/config_test.py index a758154c0..819d8f5be 100644 --- a/tests/unit/config/config_test.py +++ b/tests/unit/config/config_test.py @@ -2846,6 +2846,88 @@ class PortsTest(unittest.TestCase): ) +class SubnetTest(unittest.TestCase): + INVALID_SUBNET_TYPES = [ + None, + False, + 10, + ] + + INVALID_SUBNET_MAPPINGS = [ + "", + "192.168.0.1/sdfsdfs", + "192.168.0.1/", + "192.168.0.1/33", + "192.168.0.1/01", + "192.168.0.1", + "fe80:0000:0000:0000:0204:61ff:fe9d:f156/sdfsdfs", + "fe80:0000:0000:0000:0204:61ff:fe9d:f156/", + "fe80:0000:0000:0000:0204:61ff:fe9d:f156/129", + "fe80:0000:0000:0000:0204:61ff:fe9d:f156/01", + "fe80:0000:0000:0000:0204:61ff:fe9d:f156", + ] + + ILLEGAL_SUBNET_MAPPINGS = [ + "ge80:0000:0000:0000:0204:61ff:fe9d:f156/128" + ] + + VALID_SUBNET_MAPPINGS = [ + "192.168.0.1/0", + "192.168.0.1/32", + "fe80:0000:0000:0000:0204:61ff:fe9d:f156/0", + "fe80:0000:0000:0000:0204:61ff:fe9d:f156/128", + ] + + def test_config_invalid_subnet_type_validation(self): + for invalid_subnet in self.INVALID_SUBNET_TYPES: + with pytest.raises(ConfigurationError) as exc: + self.check_config(invalid_subnet) + + assert "contains an invalid type" in exc.value.msg + + def test_config_invalid_subnet_format_validation(self): + for invalid_subnet in self.INVALID_SUBNET_MAPPINGS: + with pytest.raises(ConfigurationError) as exc: + self.check_config(invalid_subnet) + + assert "should be of the format 'IP_ADDRESS/CIDR'" in exc.value.msg + + def test_config_illegal_subnet_type_validation(self): + for invalid_subnet in self.ILLEGAL_SUBNET_MAPPINGS: + with pytest.raises(ConfigurationError) as exc: + self.check_config(invalid_subnet) + + assert "illegal IP address string" in exc.value.msg + + def test_config_valid_subnet_format_validation(self): + for valid_subnet in self.VALID_SUBNET_MAPPINGS: + self.check_config(valid_subnet) + + def check_config(self, subnet): + config.load( + build_config_details({ + 'version': '3.5', + 'services': { + 'web': { + 'image': 'busybox' + } + }, + 'networks': { + 'default': { + 'ipam': { + 'config': [ + { + 'subnet': subnet + } + ], + 'driver': 'default' + } + } + } + }) + ) + + class InterpolationTest(unittest.TestCase): @mock.patch.dict(os.environ) From fa61a91cb5056767ba4b72faf99c295a4372e25b Mon Sep 17 00:00:00 2001 From: Drew Romanyk Date: Thu, 9 Nov 2017 22:57:47 -0600 Subject: [PATCH 217/244] Fix subnet config test for windows Signed-off-by: Drew Romanyk --- compose/config/validation.py | 10 ++++++---- tests/unit/config/config_test.py | 7 +++++-- 2 files changed, 11 insertions(+), 6 deletions(-) diff --git a/compose/config/validation.py b/compose/config/validation.py index a8061a5a4..c2256804b 100644 --- a/compose/config/validation.py +++ b/compose/config/validation.py @@ -72,22 +72,24 @@ def format_expose(instance): def format_subnet_ip_address(instance): if isinstance(instance, six.string_types): if '/' not in instance: - raise ValidationError("should be of the format 'IP_ADDRESS/CIDR'") + raise ValidationError("'{0}' 75 should be of the format 'IP_ADDRESS/CIDR'".format(instance)) ip_address, cidr = instance.split('/') if re.match(VALID_IPV4_FORMAT, ip_address): if not (re.match(VALID_IPV4_CIDR_FORMAT, cidr) and all(0 <= int(component) <= 255 for component in ip_address.split("."))): - raise ValidationError("should be of the format 'IP_ADDRESS/CIDR'") + raise ValidationError( + "'{0}' 83 should be of the format 'IP_ADDRESS/CIDR'".format(instance)) elif re.match(VALID_IPV6_CIDR_FORMAT, cidr) and hasattr(socket, "inet_pton"): try: if not (socket.inet_pton(socket.AF_INET6, ip_address)): - raise ValidationError("should be of the format 'IP_ADDRESS/CIDR'") + raise ValidationError( + "'{0}' 88 should be of the format 'IP_ADDRESS/CIDR'".format(instance)) except socket.error as e: raise ValidationError(six.text_type(e)) else: - raise ValidationError("should be of the format 'IP_ADDRESS/CIDR'") + raise ValidationError("'{0}' 92 should be of the format 'IP_ADDRESS/CIDR'".format(instance)) return True diff --git a/tests/unit/config/config_test.py b/tests/unit/config/config_test.py index 819d8f5be..51323cd32 100644 --- a/tests/unit/config/config_test.py +++ b/tests/unit/config/config_test.py @@ -2896,8 +2896,11 @@ class SubnetTest(unittest.TestCase): for invalid_subnet in self.ILLEGAL_SUBNET_MAPPINGS: with pytest.raises(ConfigurationError) as exc: self.check_config(invalid_subnet) - - assert "illegal IP address string" in exc.value.msg + if IS_WINDOWS_PLATFORM: + assert "An invalid argument was supplied" in exc.value.msg or \ + "illegal IP address string" in exc.value.msg + else: + assert "illegal IP address string" in exc.value.msg def test_config_valid_subnet_format_validation(self): for valid_subnet in self.VALID_SUBNET_MAPPINGS: From df0f7e17d3dc5e806d89865eed060db78a8a0b97 Mon Sep 17 00:00:00 2001 From: Drew Romanyk Date: Fri, 10 Nov 2017 18:04:11 -0600 Subject: [PATCH 218/244] Add format to other v3 configs & remove unix dependency Signed-off-by: Drew Romanyk --- compose/config/config_schema_v3.0.json | 2 +- compose/config/config_schema_v3.1.json | 2 +- compose/config/config_schema_v3.2.json | 2 +- compose/config/config_schema_v3.3.json | 2 +- compose/config/config_schema_v3.4.json | 2 +- compose/config/validation.py | 52 +++++++++++++++++--------- tests/unit/config/config_test.py | 30 ++++++++------- 7 files changed, 55 insertions(+), 37 deletions(-) diff --git a/compose/config/config_schema_v3.0.json b/compose/config/config_schema_v3.0.json index f39344cfb..fa601bed2 100644 --- a/compose/config/config_schema_v3.0.json +++ b/compose/config/config_schema_v3.0.json @@ -294,7 +294,7 @@ "items": { "type": "object", "properties": { - "subnet": {"type": "string"} + "subnet": {"type": "string", "format": "subnet_ip_address"} }, "additionalProperties": false } diff --git a/compose/config/config_schema_v3.1.json b/compose/config/config_schema_v3.1.json index 719c0fa7a..41da89650 100644 --- a/compose/config/config_schema_v3.1.json +++ b/compose/config/config_schema_v3.1.json @@ -323,7 +323,7 @@ "items": { "type": "object", "properties": { - "subnet": {"type": "string"} + "subnet": {"type": "string", "format": "subnet_ip_address"} }, "additionalProperties": false } diff --git a/compose/config/config_schema_v3.2.json b/compose/config/config_schema_v3.2.json index 2ca8e92db..a74e2c66b 100644 --- a/compose/config/config_schema_v3.2.json +++ b/compose/config/config_schema_v3.2.json @@ -369,7 +369,7 @@ "items": { "type": "object", "properties": { - "subnet": {"type": "string"} + "subnet": {"type": "string", "format": "subnet_ip_address"} }, "additionalProperties": false } diff --git a/compose/config/config_schema_v3.3.json b/compose/config/config_schema_v3.3.json index f1eb9a661..96dc1d7d0 100644 --- a/compose/config/config_schema_v3.3.json +++ b/compose/config/config_schema_v3.3.json @@ -412,7 +412,7 @@ "items": { "type": "object", "properties": { - "subnet": {"type": "string"} + "subnet": {"type": "string", "format": "subnet_ip_address"} }, "additionalProperties": false } diff --git a/compose/config/config_schema_v3.4.json b/compose/config/config_schema_v3.4.json index dae7d7d23..8089c7e6d 100644 --- a/compose/config/config_schema_v3.4.json +++ b/compose/config/config_schema_v3.4.json @@ -420,7 +420,7 @@ "items": { "type": "object", "properties": { - "subnet": {"type": "string"} + "subnet": {"type": "string", "format": "subnet_ip_address"} }, "additionalProperties": false } diff --git a/compose/config/validation.py b/compose/config/validation.py index c2256804b..f97069935 100644 --- a/compose/config/validation.py +++ b/compose/config/validation.py @@ -5,7 +5,6 @@ import json import logging import os import re -import socket import sys import six @@ -44,9 +43,32 @@ DOCKER_CONFIG_HINTS = { VALID_NAME_CHARS = '[a-zA-Z0-9\._\-]' VALID_EXPOSE_FORMAT = r'^\d+(\-\d+)?(\/[a-zA-Z]+)?$' -VALID_IPV4_FORMAT = r'^(\d{1,3}.){3}\d{1,3}$' -VALID_IPV4_CIDR_FORMAT = r'^(\d|[1-2]\d|3[0-2])$' -VALID_IPV6_CIDR_FORMAT = r'^(\d|[1-9]\d|1[0-1]\d|12[0-8])$' + +VALID_IPV4_SEG = r'(\d{1,2}|1\d{2}|2[0-4]\d|25[0-5])' +VALID_REGEX_IPV4_CIDR = r'^(\d|[1-2]\d|3[0-2])$' +VALID_IPV4_ADDR = "({IPV4_SEG}\.){{3}}{IPV4_SEG}".format(IPV4_SEG=VALID_IPV4_SEG) +VALID_REGEX_IPV4_ADDR = "^{IPV4_ADDR}$".format(IPV4_ADDR=VALID_IPV4_ADDR) + +VALID_IPV6_SEG = r'[0-9a-fA-F]{1,4}' +VALID_REGEX_IPV6_CIDR = r'^(\d|[1-9]\d|1[0-1]\d|12[0-8])$' +VALID_REGEX_IPV6_ADDR = "".join(""" +^ +( + (({IPV6_SEG}:){{7}}{IPV6_SEG})| + (({IPV6_SEG}:){{1,7}}:)| + (({IPV6_SEG}:){{1,6}}(:{IPV6_SEG}){{1,1}})| + (({IPV6_SEG}:){{1,5}}(:{IPV6_SEG}){{1,2}})| + (({IPV6_SEG}:){{1,4}}(:{IPV6_SEG}){{1,3}})| + (({IPV6_SEG}:){{1,3}}(:{IPV6_SEG}){{1,4}})| + (({IPV6_SEG}:){{1,2}}(:{IPV6_SEG}){{1,5}})| + (({IPV6_SEG}:){{1,1}}(:{IPV6_SEG}){{1,6}})| + (:((:{IPV6_SEG}){{1,7}}|:))| + (fe80:(:{IPV6_SEG}){{0,4}}%[0-9a-zA-Z]{{1,}})| + (::(ffff(:0{{1,4}}){{0,1}}:){{0,1}}{IPV4_ADDR})| + (({IPV6_SEG}:){{1,4}}:{IPV4_ADDR}) +) +$ +""".format(IPV6_SEG=VALID_IPV6_SEG, IPV4_ADDR=VALID_IPV4_ADDR).split()) @FormatChecker.cls_checks(format="ports", raises=ValidationError) @@ -72,24 +94,18 @@ def format_expose(instance): def format_subnet_ip_address(instance): if isinstance(instance, six.string_types): if '/' not in instance: - raise ValidationError("'{0}' 75 should be of the format 'IP_ADDRESS/CIDR'".format(instance)) + raise ValidationError("should be of the format 'IP_ADDRESS/CIDR'") ip_address, cidr = instance.split('/') - if re.match(VALID_IPV4_FORMAT, ip_address): - if not (re.match(VALID_IPV4_CIDR_FORMAT, cidr) and - all(0 <= int(component) <= 255 for component in ip_address.split("."))): - raise ValidationError( - "'{0}' 83 should be of the format 'IP_ADDRESS/CIDR'".format(instance)) - elif re.match(VALID_IPV6_CIDR_FORMAT, cidr) and hasattr(socket, "inet_pton"): - try: - if not (socket.inet_pton(socket.AF_INET6, ip_address)): - raise ValidationError( - "'{0}' 88 should be of the format 'IP_ADDRESS/CIDR'".format(instance)) - except socket.error as e: - raise ValidationError(six.text_type(e)) + if re.match(VALID_REGEX_IPV4_ADDR, ip_address): + if not re.match(VALID_REGEX_IPV4_CIDR, cidr): + raise ValidationError("should be of the format 'IP_ADDRESS/CIDR'") + elif re.match(VALID_REGEX_IPV6_ADDR, ip_address): + if not re.match(VALID_REGEX_IPV6_CIDR, cidr): + raise ValidationError("should be of the format 'IP_ADDRESS/CIDR'") else: - raise ValidationError("'{0}' 92 should be of the format 'IP_ADDRESS/CIDR'".format(instance)) + raise ValidationError("should be of the format 'IP_ADDRESS/CIDR'") return True diff --git a/tests/unit/config/config_test.py b/tests/unit/config/config_test.py index 51323cd32..1cf783c77 100644 --- a/tests/unit/config/config_test.py +++ b/tests/unit/config/config_test.py @@ -2865,10 +2865,7 @@ class SubnetTest(unittest.TestCase): "fe80:0000:0000:0000:0204:61ff:fe9d:f156/129", "fe80:0000:0000:0000:0204:61ff:fe9d:f156/01", "fe80:0000:0000:0000:0204:61ff:fe9d:f156", - ] - - ILLEGAL_SUBNET_MAPPINGS = [ - "ge80:0000:0000:0000:0204:61ff:fe9d:f156/128" + "ge80:0000:0000:0000:0204:61ff:fe9d:f156/128", ] VALID_SUBNET_MAPPINGS = [ @@ -2876,6 +2873,21 @@ class SubnetTest(unittest.TestCase): "192.168.0.1/32", "fe80:0000:0000:0000:0204:61ff:fe9d:f156/0", "fe80:0000:0000:0000:0204:61ff:fe9d:f156/128", + "1:2:3:4:5:6:7:8/0", + "1::/0", + "1:2:3:4:5:6:7::/0", + "1::8/0", + "1:2:3:4:5:6::8/0", + "::/0", + "::8/0", + "::2:3:4:5:6:7:8/0", + "fe80::7:8%eth0/0", + "fe80::7:8%1/0", + "::255.255.255.255/0", + "::ffff:255.255.255.255/0", + "::ffff:0:255.255.255.255/0", + "2001:db8:3:4::192.0.2.33/0", + "64:ff9b::192.0.2.33/0", ] def test_config_invalid_subnet_type_validation(self): @@ -2892,16 +2904,6 @@ class SubnetTest(unittest.TestCase): assert "should be of the format 'IP_ADDRESS/CIDR'" in exc.value.msg - def test_config_illegal_subnet_type_validation(self): - for invalid_subnet in self.ILLEGAL_SUBNET_MAPPINGS: - with pytest.raises(ConfigurationError) as exc: - self.check_config(invalid_subnet) - if IS_WINDOWS_PLATFORM: - assert "An invalid argument was supplied" in exc.value.msg or \ - "illegal IP address string" in exc.value.msg - else: - assert "illegal IP address string" in exc.value.msg - def test_config_valid_subnet_format_validation(self): for valid_subnet in self.VALID_SUBNET_MAPPINGS: self.check_config(valid_subnet) From 76e9076cb714242212a428fe7cfd84159425b5bc Mon Sep 17 00:00:00 2001 From: Drew Romanyk Date: Mon, 13 Nov 2017 21:53:14 -0600 Subject: [PATCH 219/244] Refactor subnet cidr validator & add new test Signed-off-by: Drew Romanyk --- compose/config/validation.py | 23 ++++++----------------- tests/unit/config/config_test.py | 3 ++- 2 files changed, 8 insertions(+), 18 deletions(-) diff --git a/compose/config/validation.py b/compose/config/validation.py index f97069935..0fdcb37e7 100644 --- a/compose/config/validation.py +++ b/compose/config/validation.py @@ -45,13 +45,11 @@ VALID_NAME_CHARS = '[a-zA-Z0-9\._\-]' VALID_EXPOSE_FORMAT = r'^\d+(\-\d+)?(\/[a-zA-Z]+)?$' VALID_IPV4_SEG = r'(\d{1,2}|1\d{2}|2[0-4]\d|25[0-5])' -VALID_REGEX_IPV4_CIDR = r'^(\d|[1-2]\d|3[0-2])$' VALID_IPV4_ADDR = "({IPV4_SEG}\.){{3}}{IPV4_SEG}".format(IPV4_SEG=VALID_IPV4_SEG) -VALID_REGEX_IPV4_ADDR = "^{IPV4_ADDR}$".format(IPV4_ADDR=VALID_IPV4_ADDR) +VALID_REGEX_IPV4_CIDR = "^{IPV4_ADDR}/(\d|[1-2]\d|3[0-2])$".format(IPV4_ADDR=VALID_IPV4_ADDR) VALID_IPV6_SEG = r'[0-9a-fA-F]{1,4}' -VALID_REGEX_IPV6_CIDR = r'^(\d|[1-9]\d|1[0-1]\d|12[0-8])$' -VALID_REGEX_IPV6_ADDR = "".join(""" +VALID_REGEX_IPV6_CIDR = "".join(""" ^ ( (({IPV6_SEG}:){{7}}{IPV6_SEG})| @@ -67,6 +65,7 @@ VALID_REGEX_IPV6_ADDR = "".join(""" (::(ffff(:0{{1,4}}){{0,1}}:){{0,1}}{IPV4_ADDR})| (({IPV6_SEG}:){{1,4}}:{IPV4_ADDR}) ) +/(\d|[1-9]\d|1[0-1]\d|12[0-8]) $ """.format(IPV6_SEG=VALID_IPV6_SEG, IPV4_ADDR=VALID_IPV4_ADDR).split()) @@ -93,19 +92,9 @@ def format_expose(instance): @FormatChecker.cls_checks("subnet_ip_address", raises=ValidationError) def format_subnet_ip_address(instance): if isinstance(instance, six.string_types): - if '/' not in instance: - raise ValidationError("should be of the format 'IP_ADDRESS/CIDR'") - - ip_address, cidr = instance.split('/') - - if re.match(VALID_REGEX_IPV4_ADDR, ip_address): - if not re.match(VALID_REGEX_IPV4_CIDR, cidr): - raise ValidationError("should be of the format 'IP_ADDRESS/CIDR'") - elif re.match(VALID_REGEX_IPV6_ADDR, ip_address): - if not re.match(VALID_REGEX_IPV6_CIDR, cidr): - raise ValidationError("should be of the format 'IP_ADDRESS/CIDR'") - else: - raise ValidationError("should be of the format 'IP_ADDRESS/CIDR'") + if not re.match(VALID_REGEX_IPV4_CIDR, instance) and \ + not re.match(VALID_REGEX_IPV6_CIDR, instance): + raise ValidationError("should use the CIDR format") return True diff --git a/tests/unit/config/config_test.py b/tests/unit/config/config_test.py index 1cf783c77..32ccf1cec 100644 --- a/tests/unit/config/config_test.py +++ b/tests/unit/config/config_test.py @@ -2866,6 +2866,7 @@ class SubnetTest(unittest.TestCase): "fe80:0000:0000:0000:0204:61ff:fe9d:f156/01", "fe80:0000:0000:0000:0204:61ff:fe9d:f156", "ge80:0000:0000:0000:0204:61ff:fe9d:f156/128", + "192.168.0.1/31/31", ] VALID_SUBNET_MAPPINGS = [ @@ -2902,7 +2903,7 @@ class SubnetTest(unittest.TestCase): with pytest.raises(ConfigurationError) as exc: self.check_config(invalid_subnet) - assert "should be of the format 'IP_ADDRESS/CIDR'" in exc.value.msg + assert "should use the CIDR format" in exc.value.msg def test_config_valid_subnet_format_validation(self): for valid_subnet in self.VALID_SUBNET_MAPPINGS: From 6b0138d70f430b6ace9cc57287066ee9ef7a4942 Mon Sep 17 00:00:00 2001 From: Madeline Stager Date: Wed, 22 Nov 2017 16:21:47 -0600 Subject: [PATCH 220/244] implement --timeout flag for docker-compose down Fix #3370 Signed-off-by: Madeline Stager --- compose/cli/main.py | 5 ++++- compose/project.py | 4 ++-- tests/acceptance/cli_test.py | 21 +++++++++++++++++++++ 3 files changed, 27 insertions(+), 3 deletions(-) diff --git a/compose/cli/main.py b/compose/cli/main.py index c3e30919d..f866d5809 100644 --- a/compose/cli/main.py +++ b/compose/cli/main.py @@ -371,9 +371,12 @@ class TopLevelCommand(object): attached to containers. --remove-orphans Remove containers for services not defined in the Compose file + -t, --timeout TIMEOUT Specify a shutdown timeout in seconds. + (default: 10) """ image_type = image_type_from_opt('--rmi', options['--rmi']) - self.project.down(image_type, options['--volumes'], options['--remove-orphans']) + timeout = timeout_from_opts(options) + self.project.down(image_type, options['--volumes'], options['--remove-orphans'], timeout=timeout) def events(self, options): """ diff --git a/compose/project.py b/compose/project.py index f6bd30a88..9cc726e42 100644 --- a/compose/project.py +++ b/compose/project.py @@ -330,8 +330,8 @@ class Project(object): service_names, stopped=True, one_off=one_off ), options) - def down(self, remove_image_type, include_volumes, remove_orphans=False): - self.stop(one_off=OneOffFilter.include) + def down(self, remove_image_type, include_volumes, remove_orphans=False, timeout=None): + self.stop(one_off=OneOffFilter.include, timeout=timeout) self.find_orphan_containers(remove_orphans) self.remove_stopped(v=include_volumes, one_off=OneOffFilter.include) diff --git a/tests/acceptance/cli_test.py b/tests/acceptance/cli_test.py index 0fcf866ff..9d4ae3255 100644 --- a/tests/acceptance/cli_test.py +++ b/tests/acceptance/cli_test.py @@ -794,6 +794,27 @@ class CLITestCase(DockerClientTestCase): assert 'Removing network v2full_default' in result.stderr assert 'Removing network v2full_front' in result.stderr + def test_down_timeout(self): + self.dispatch(['up', '-d'], None) + service = self.project.get_service('simple') + self.assertEqual(len(service.containers()), 1) + self.assertTrue(service.containers()[0].is_running) + "" + + self.dispatch(['down', '-t', '1'], None) + + self.assertEqual(len(service.containers(stopped=True)), 0) + + def test_down_signal(self): + self.base_dir = 'tests/fixtures/stop-signal-composefile' + self.dispatch(['up', '-d'], None) + service = self.project.get_service('simple') + self.assertEqual(len(service.containers()), 1) + self.assertTrue(service.containers()[0].is_running) + + self.dispatch(['down', '-t', '1'], None) + self.assertEqual(len(service.containers(stopped=True)), 0) + def test_up_detached(self): self.dispatch(['up', '-d']) service = self.project.get_service('simple') From a99dd9f2dc51b1f25145c7638933e66817dcd4b3 Mon Sep 17 00:00:00 2001 From: Madeline Stager Date: Wed, 22 Nov 2017 17:32:51 -0600 Subject: [PATCH 221/244] Fixed example in instructions for running tests. Fix #5394 Signed-off-by: Madeline Stager --- CONTRIBUTING.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 16bccf98b..a031e2d68 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -64,8 +64,8 @@ you can specify a test directory, file, module, class or method: $ script/test/default tests/unit $ script/test/default tests/unit/cli_test.py - $ script/test/default tests/unit/config_test.py::ConfigTest - $ script/test/default tests/unit/config_test.py::ConfigTest::test_load + $ script/test/default tests/unit/config/config_test.py::ConfigTest + $ script/test/default tests/unit/config/config_test.py::ConfigTest::test_load ## Finding things to work on From 7835a0755091fd5886cc33276d2a6457151ac981 Mon Sep 17 00:00:00 2001 From: Samantha Miller Date: Sun, 12 Nov 2017 11:33:34 -0600 Subject: [PATCH 222/244] Added a label option to 'docker-compose run' and test. Signed-off-by: Samantha Miller --- compose/cli/main.py | 9 ++++++++- compose/config/__init__.py | 2 ++ compose/config/config.py | 6 ++++++ compose/service.py | 5 +++++ contrib/completion/bash/docker-compose | 4 ++-- tests/acceptance/cli_test.py | 11 +++++++++++ tests/fixtures/run-labels/docker-compose.yml | 7 +++++++ tests/unit/cli_test.py | 4 ++++ 8 files changed, 45 insertions(+), 3 deletions(-) create mode 100644 tests/fixtures/run-labels/docker-compose.yml diff --git a/compose/cli/main.py b/compose/cli/main.py index f866d5809..79f663096 100644 --- a/compose/cli/main.py +++ b/compose/cli/main.py @@ -24,6 +24,7 @@ from ..bundle import MissingDigests from ..bundle import serialize_bundle from ..config import ConfigurationError from ..config import parse_environment +from ..config import parse_labels from ..config import resolve_build_args from ..config.environment import Environment from ..config.serialize import serialize_config @@ -723,7 +724,9 @@ class TopLevelCommand(object): running. If you do not want to start linked services, use `docker-compose run --no-deps SERVICE COMMAND [ARGS...]`. - Usage: run [options] [-v VOLUME...] [-p PORT...] [-e KEY=VAL...] SERVICE [COMMAND] [ARGS...] + Usage: + run [options] [-v VOLUME...] [-p PORT...] [-e KEY=VAL...] [-l KEY=VALUE...] + SERVICE [COMMAND] [ARGS...] Options: -d Detached mode: Run container in the background, print @@ -731,6 +734,7 @@ class TopLevelCommand(object): --name NAME Assign a name to the container --entrypoint CMD Override the entrypoint of the image. -e KEY=VAL Set an environment variable (can be used multiple times) + -l, --label KEY=VAL Add or override a label (can be used multiple times) -u, --user="" Run as specified username or uid --no-deps Don't start linked services. --rm Remove container after run. Ignored in detached mode. @@ -1125,6 +1129,9 @@ def build_container_options(options, detach, command): parse_environment(options['-e']) ) + if options['--label']: + container_options['labels'] = parse_labels(options['--label']) + if options['--entrypoint']: container_options['entrypoint'] = options.get('--entrypoint') diff --git a/compose/config/__init__.py b/compose/config/__init__.py index b629edf66..e1032f3de 100644 --- a/compose/config/__init__.py +++ b/compose/config/__init__.py @@ -8,5 +8,7 @@ from .config import DOCKER_CONFIG_KEYS from .config import find from .config import load from .config import merge_environment +from .config import merge_labels from .config import parse_environment +from .config import parse_labels from .config import resolve_build_args diff --git a/compose/config/config.py b/compose/config/config.py index 4c3f93ddb..864bc7e90 100644 --- a/compose/config/config.py +++ b/compose/config/config.py @@ -1076,6 +1076,12 @@ def merge_environment(base, override): return env +def merge_labels(base, override): + labels = parse_labels(base) + labels.update(parse_labels(override)) + return labels + + def split_kv(kvpair): if '=' in kvpair: return kvpair.split('=', 1) diff --git a/compose/service.py b/compose/service.py index 0b6561d99..b696fd664 100644 --- a/compose/service.py +++ b/compose/service.py @@ -25,6 +25,7 @@ from . import const from . import progress_stream from .config import DOCKER_CONFIG_KEYS from .config import merge_environment +from .config import merge_labels from .config.errors import DependencyError from .config.types import ServicePort from .config.types import VolumeSpec @@ -778,6 +779,10 @@ class Service(object): self.options.get('environment'), override_options.get('environment')) + container_options['labels'] = merge_labels( + self.options.get('labels'), + override_options.get('labels')) + binds, affinity = merge_volume_bindings( container_options.get('volumes') or [], self.options.get('tmpfs') or [], diff --git a/contrib/completion/bash/docker-compose b/contrib/completion/bash/docker-compose index 1fdb27705..af0368177 100644 --- a/contrib/completion/bash/docker-compose +++ b/contrib/completion/bash/docker-compose @@ -403,14 +403,14 @@ _docker_compose_run() { __docker_compose_nospace return ;; - --entrypoint|--name|--user|-u|--volume|-v|--workdir|-w) + --entrypoint|--label|-l|--name|--user|-u|--volume|-v|--workdir|-w) return ;; esac case "$cur" in -*) - COMPREPLY=( $( compgen -W "-d --entrypoint -e --help --name --no-deps --publish -p --rm --service-ports -T --user -u --volume -v --workdir -w" -- "$cur" ) ) + COMPREPLY=( $( compgen -W "-d --entrypoint -e --help --label -l --name --no-deps --publish -p --rm --service-ports -T --user -u --volume -v --workdir -w" -- "$cur" ) ) ;; *) __docker_compose_services_all diff --git a/tests/acceptance/cli_test.py b/tests/acceptance/cli_test.py index 9d4ae3255..0ea5f5a6f 100644 --- a/tests/acceptance/cli_test.py +++ b/tests/acceptance/cli_test.py @@ -1869,6 +1869,17 @@ class CLITestCase(DockerClientTestCase): assert 'FOO=bar' in environment assert 'BAR=baz' not in environment + def test_run_label_flag(self): + self.base_dir = 'tests/fixtures/run-labels' + name = 'service' + self.dispatch(['run', '-l', 'default', '--label', 'foo=baz', name, '/bin/true']) + service = self.project.get_service(name) + container, = service.containers(stopped=True, one_off=OneOffFilter.only) + labels = container.labels + assert labels['default'] == '' + assert labels['foo'] == 'baz' + assert labels['hello'] == 'world' + def test_rm(self): service = self.project.get_service('simple') service.create_container() diff --git a/tests/fixtures/run-labels/docker-compose.yml b/tests/fixtures/run-labels/docker-compose.yml new file mode 100644 index 000000000..e8cd50065 --- /dev/null +++ b/tests/fixtures/run-labels/docker-compose.yml @@ -0,0 +1,7 @@ +service: + image: busybox:latest + command: top + + labels: + foo: bar + hello: world diff --git a/tests/unit/cli_test.py b/tests/unit/cli_test.py index 1a324f50a..c6aa75b26 100644 --- a/tests/unit/cli_test.py +++ b/tests/unit/cli_test.py @@ -114,6 +114,7 @@ class CLITestCase(unittest.TestCase): 'SERVICE': 'service', 'COMMAND': None, '-e': [], + '--label': [], '--user': None, '--no-deps': None, '-d': False, @@ -150,6 +151,7 @@ class CLITestCase(unittest.TestCase): 'SERVICE': 'service', 'COMMAND': None, '-e': [], + '--label': [], '--user': None, '--no-deps': None, '-d': True, @@ -173,6 +175,7 @@ class CLITestCase(unittest.TestCase): 'SERVICE': 'service', 'COMMAND': None, '-e': [], + '--label': [], '--user': None, '--no-deps': None, '-d': True, @@ -205,6 +208,7 @@ class CLITestCase(unittest.TestCase): 'SERVICE': 'service', 'COMMAND': None, '-e': [], + '--label': [], '--user': None, '--no-deps': None, '-d': True, From 3ce2f03d70d9d5688d6a76e6ad7993f994292ad1 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Wed, 29 Nov 2017 12:21:56 -0800 Subject: [PATCH 223/244] Use mounts for secrets instead of volumes Signed-off-by: Joffrey F --- compose/config/types.py | 42 ++++++++++++++++++++++++++++++++++++++ compose/service.py | 27 ++++++++++++++++++++---- tests/unit/service_test.py | 12 +++++------ 3 files changed, 71 insertions(+), 10 deletions(-) diff --git a/compose/config/types.py b/compose/config/types.py index c410343b8..548f2c1cd 100644 --- a/compose/config/types.py +++ b/compose/config/types.py @@ -133,6 +133,48 @@ def normalize_path_for_engine(path): return path.replace('\\', '/') +class MountSpec(object): + options_map = { + 'volume': { + 'nocopy': 'no_copy' + }, + 'bind': { + 'propagation': 'propagation' + } + } + _fields = ['type', 'source', 'target', 'read_only', 'consistency'] + + def __init__(self, type, source=None, target=None, read_only=None, consistency=None, **kwargs): + self.type = type + self.source = source + self.target = target + self.read_only = read_only + self.consistency = consistency + self.options = None + if self.type in kwargs: + self.options = kwargs[self.type] + + def as_volume_spec(self): + mode = 'ro' if self.read_only else 'rw' + return VolumeSpec(external=self.source, internal=self.target, mode=mode) + + def legacy_repr(self): + return self.as_volume_spec().repr() + + def repr(self): + res = {} + for field in self._fields: + if getattr(self, field, None): + res[field] = getattr(self, field) + if self.options: + res[self.type] = self.options + return res + + @property + def is_named_volume(self): + return self.type == 'volume' and self.source + + class VolumeSpec(namedtuple('_VolumeSpec', 'external internal mode')): @classmethod diff --git a/compose/service.py b/compose/service.py index b696fd664..07db3ac5f 100644 --- a/compose/service.py +++ b/compose/service.py @@ -14,6 +14,7 @@ from docker.errors import APIError from docker.errors import ImageNotFound from docker.errors import NotFound from docker.types import LogConfig +from docker.types import Mount from docker.utils import version_gte from docker.utils import version_lt from docker.utils.ports import build_port_bindings @@ -27,6 +28,7 @@ from .config import DOCKER_CONFIG_KEYS from .config import merge_environment from .config import merge_labels from .config.errors import DependencyError +from .config.types import MountSpec from .config.types import ServicePort from .config.types import VolumeSpec from .const import DEFAULT_TIMEOUT @@ -795,9 +797,13 @@ class Service(object): secret_volumes = self.get_secret_volumes() if secret_volumes: - override_options['binds'].extend(v.repr() for v in secret_volumes) - container_options['volumes'].update( - (v.internal, {}) for v in secret_volumes) + if version_lt(self.client.api_version, '1.30'): + override_options['binds'].extend(v.legacy_repr() for v in secret_volumes) + container_options['volumes'].update( + (v.target, {}) for v in secret_volumes + ) + else: + override_options['mounts'] = [build_mount(v) for v in secret_volumes] container_options['image'] = self.image_name @@ -891,6 +897,7 @@ class Service(object): device_read_iops=blkio_config.get('device_read_iops'), device_write_bps=blkio_config.get('device_write_bps'), device_write_iops=blkio_config.get('device_write_iops'), + mounts=options.get('mounts'), ) def get_secret_volumes(self): @@ -901,7 +908,7 @@ class Service(object): elif not os.path.isabs(target): target = '{}/{}'.format(const.SECRETS_PATH, target) - return VolumeSpec(secret['file'], target, 'ro') + return MountSpec('bind', secret['file'], target, read_only=True) return [build_spec(secret) for secret in self.secrets] @@ -1346,6 +1353,18 @@ def build_volume_from(volume_from_spec): return "{}:{}".format(volume_from_spec.source.id, volume_from_spec.mode) +def build_mount(mount_spec): + kwargs = {} + if mount_spec.options: + for option, sdk_name in mount_spec.options_map[mount_spec.type].items(): + if option in mount_spec.options: + kwargs[sdk_name] = mount_spec.options[option] + + return Mount( + type=mount_spec.type, target=mount_spec.target, source=mount_spec.source, + read_only=mount_spec.read_only, consistency=mount_spec.consistency, **kwargs + ) + # Labels diff --git a/tests/unit/service_test.py b/tests/unit/service_test.py index 8e8f60203..87c86a731 100644 --- a/tests/unit/service_test.py +++ b/tests/unit/service_test.py @@ -1133,8 +1133,8 @@ class ServiceSecretTest(unittest.TestCase): ) volumes = service.get_secret_volumes() - assert volumes[0].external == secret1['file'] - assert volumes[0].internal == '{}/{}'.format(SECRETS_PATH, secret1['secret'].target) + assert volumes[0].source == secret1['file'] + assert volumes[0].target == '{}/{}'.format(SECRETS_PATH, secret1['secret'].target) def test_get_secret_volumes_abspath(self): secret1 = { @@ -1149,8 +1149,8 @@ class ServiceSecretTest(unittest.TestCase): ) volumes = service.get_secret_volumes() - assert volumes[0].external == secret1['file'] - assert volumes[0].internal == secret1['secret'].target + assert volumes[0].source == secret1['file'] + assert volumes[0].target == secret1['secret'].target def test_get_secret_volumes_no_target(self): secret1 = { @@ -1165,5 +1165,5 @@ class ServiceSecretTest(unittest.TestCase): ) volumes = service.get_secret_volumes() - assert volumes[0].external == secret1['file'] - assert volumes[0].internal == '{}/{}'.format(SECRETS_PATH, secret1['secret'].source) + assert volumes[0].source == secret1['file'] + assert volumes[0].target == '{}/{}'.format(SECRETS_PATH, secret1['secret'].source) From dba2abd523dc81d30a6e19c3817a93086ed8e301 Mon Sep 17 00:00:00 2001 From: Drew Romanyk Date: Thu, 30 Nov 2017 10:21:27 -0600 Subject: [PATCH 224/244] Add config validation for service volumes, fixes #5352 Signed-off-by: Drew Romanyk --- compose/config/config_schema_v3.2.json | 1 + compose/config/config_schema_v3.3.json | 1 + compose/config/config_schema_v3.4.json | 1 + compose/config/config_schema_v3.5.json | 1 + tests/unit/config/config_test.py | 27 ++++++++++++++++++++++++++ 5 files changed, 31 insertions(+) diff --git a/compose/config/config_schema_v3.2.json b/compose/config/config_schema_v3.2.json index a74e2c66b..0baf6a1a9 100644 --- a/compose/config/config_schema_v3.2.json +++ b/compose/config/config_schema_v3.2.json @@ -245,6 +245,7 @@ { "type": "object", "required": ["type"], + "additionalProperties": false, "properties": { "type": {"type": "string"}, "source": {"type": "string"}, diff --git a/compose/config/config_schema_v3.3.json b/compose/config/config_schema_v3.3.json index 96dc1d7d0..efc0fdbd7 100644 --- a/compose/config/config_schema_v3.3.json +++ b/compose/config/config_schema_v3.3.json @@ -278,6 +278,7 @@ { "type": "object", "required": ["type"], + "additionalProperties": false, "properties": { "type": {"type": "string"}, "source": {"type": "string"}, diff --git a/compose/config/config_schema_v3.4.json b/compose/config/config_schema_v3.4.json index 8089c7e6d..576ecfd84 100644 --- a/compose/config/config_schema_v3.4.json +++ b/compose/config/config_schema_v3.4.json @@ -282,6 +282,7 @@ { "type": "object", "required": ["type"], + "additionalProperties": false, "properties": { "type": {"type": "string"}, "source": {"type": "string"}, diff --git a/compose/config/config_schema_v3.5.json b/compose/config/config_schema_v3.5.json index c3ac559ee..1e65b2087 100644 --- a/compose/config/config_schema_v3.5.json +++ b/compose/config/config_schema_v3.5.json @@ -282,6 +282,7 @@ { "type": "object", "required": ["type"], + "additionalProperties": false, "properties": { "type": {"type": "string"}, "source": {"type": "string"}, diff --git a/tests/unit/config/config_test.py b/tests/unit/config/config_test.py index 32ccf1cec..00ba6c2c6 100644 --- a/tests/unit/config/config_test.py +++ b/tests/unit/config/config_test.py @@ -2631,6 +2631,33 @@ class ConfigTest(unittest.TestCase): ] assert service_sort(service_dicts) == service_sort(expected) + def test_service_volume_invalid_config(self): + config_details = build_config_details( + { + 'version': '3.2', + 'services': { + 'web': { + 'build': { + 'context': '.', + 'args': None, + }, + 'volumes': [ + { + "type": "volume", + "source": "/data", + "garbage": { + "and": "error" + } + } + ] + }, + }, + } + ) + with pytest.raises(ConfigurationError) as exc: + config.load(config_details) + assert "services.web.volumes contains unsupported option: 'garbage'" in exc.exconly() + class NetworkModeTest(unittest.TestCase): From 4099c97758fa4333bfd3b70a82581d4a15a403d7 Mon Sep 17 00:00:00 2001 From: Drew Romanyk Date: Thu, 30 Nov 2017 10:59:25 -0600 Subject: [PATCH 225/244] Add ipam default driver, fixes #5248 Signed-off-by: Drew Romanyk --- compose/network.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/compose/network.py b/compose/network.py index 2e0a7e6ec..ee5939c15 100644 --- a/compose/network.py +++ b/compose/network.py @@ -116,7 +116,7 @@ def create_ipam_config_from_dict(ipam_dict): return None return IPAMConfig( - driver=ipam_dict.get('driver'), + driver=ipam_dict.get('driver') or 'default', pool_configs=[ IPAMPool( subnet=config.get('subnet'), From 7765eed9db5d87c8676e2f8d2fd1dd69ea27e4cb Mon Sep 17 00:00:00 2001 From: Fumiaki MATSUSHIMA Date: Sun, 3 Dec 2017 01:07:17 +0900 Subject: [PATCH 226/244] Specify osx_image to fix CI Signed-off-by: Fumiaki Matsushima --- .travis.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.travis.yml b/.travis.yml index fbf269646..8fef7ed1b 100644 --- a/.travis.yml +++ b/.travis.yml @@ -8,6 +8,7 @@ matrix: services: - docker - os: osx + osx_image: xcode7.3 language: generic install: ./script/travis/install From 20a393d4f95adc5cca092b41138ff41e32627dc9 Mon Sep 17 00:00:00 2001 From: Samantha Miller Date: Fri, 24 Nov 2017 22:53:48 -0600 Subject: [PATCH 227/244] Adds support for a memory flag to docker-compose build. Signed-off-by: Samantha Miller --- compose/cli/main.py | 2 ++ compose/project.py | 5 +++-- compose/service.py | 5 ++++- contrib/completion/bash/docker-compose | 2 +- contrib/completion/zsh/_docker-compose | 1 + tests/acceptance/cli_test.py | 6 ++++++ tests/fixtures/build-memory/Dockerfile | 4 ++++ tests/fixtures/build-memory/docker-compose.yml | 6 ++++++ tests/unit/service_test.py | 2 ++ 9 files changed, 29 insertions(+), 4 deletions(-) create mode 100644 tests/fixtures/build-memory/Dockerfile create mode 100644 tests/fixtures/build-memory/docker-compose.yml diff --git a/compose/cli/main.py b/compose/cli/main.py index 79f663096..f842f05c8 100644 --- a/compose/cli/main.py +++ b/compose/cli/main.py @@ -233,6 +233,7 @@ class TopLevelCommand(object): --force-rm Always remove intermediate containers. --no-cache Do not use cache when building the image. --pull Always attempt to pull a newer version of the image. + -m, --memory MEM Sets memory limit for the bulid container. --build-arg key=val Set build-time variables for one service. """ service_names = options['SERVICE'] @@ -249,6 +250,7 @@ class TopLevelCommand(object): no_cache=bool(options.get('--no-cache', False)), pull=bool(options.get('--pull', False)), force_rm=bool(options.get('--force-rm', False)), + memory=options.get('--memory'), build_args=build_args) def bundle(self, config_options, options): diff --git a/compose/project.py b/compose/project.py index 9cc726e42..411576386 100644 --- a/compose/project.py +++ b/compose/project.py @@ -357,10 +357,11 @@ class Project(object): ) return containers - def build(self, service_names=None, no_cache=False, pull=False, force_rm=False, build_args=None): + def build(self, service_names=None, no_cache=False, pull=False, force_rm=False, memory=None, + build_args=None): for service in self.get_services(service_names): if service.can_be_built(): - service.build(no_cache, pull, force_rm, build_args) + service.build(no_cache, pull, force_rm, memory, build_args) else: log.info('%s uses an image, skipping' % service.name) diff --git a/compose/service.py b/compose/service.py index 07db3ac5f..bfc2e5940 100644 --- a/compose/service.py +++ b/compose/service.py @@ -912,7 +912,7 @@ class Service(object): return [build_spec(secret) for secret in self.secrets] - def build(self, no_cache=False, pull=False, force_rm=False, build_args_override=None): + def build(self, no_cache=False, pull=False, force_rm=False, memory=None, build_args_override=None): log.info('Building %s' % self.name) build_opts = self.options.get('build', {}) @@ -943,6 +943,9 @@ class Service(object): target=build_opts.get('target', None), shmsize=parse_bytes(build_opts.get('shm_size')) if build_opts.get('shm_size') else None, extra_hosts=build_opts.get('extra_hosts', None), + container_limits={ + 'memory': parse_bytes(memory) if memory else None + }, ) try: diff --git a/contrib/completion/bash/docker-compose b/contrib/completion/bash/docker-compose index af0368177..87161d0ac 100644 --- a/contrib/completion/bash/docker-compose +++ b/contrib/completion/bash/docker-compose @@ -120,7 +120,7 @@ _docker_compose_build() { case "$cur" in -*) - COMPREPLY=( $( compgen -W "--build-arg --force-rm --help --no-cache --pull" -- "$cur" ) ) + COMPREPLY=( $( compgen -W "--build-arg --force-rm --help --memory --no-cache --pull" -- "$cur" ) ) ;; *) __docker_compose_services_from_build diff --git a/contrib/completion/zsh/_docker-compose b/contrib/completion/zsh/_docker-compose index f53f96334..c0a54cced 100644 --- a/contrib/completion/zsh/_docker-compose +++ b/contrib/completion/zsh/_docker-compose @@ -196,6 +196,7 @@ __docker-compose_subcommand() { $opts_help \ "*--build-arg=[Set build-time variables for one service.]:=: " \ '--force-rm[Always remove intermediate containers.]' \ + '--memory[Memory limit for the build container.]' \ '--no-cache[Do not use cache when building the image.]' \ '--pull[Always attempt to pull a newer version of the image.]' \ '*:services:__docker-compose_services_from_build' && ret=0 diff --git a/tests/acceptance/cli_test.py b/tests/acceptance/cli_test.py index 0ea5f5a6f..21e716751 100644 --- a/tests/acceptance/cli_test.py +++ b/tests/acceptance/cli_test.py @@ -602,6 +602,12 @@ class CLITestCase(DockerClientTestCase): result = self.dispatch(['build', '--no-cache'], None) assert 'shm_size: 96' in result.stdout + def test_build_memory_build_option(self): + pull_busybox(self.client) + self.base_dir = 'tests/fixtures/build-memory' + result = self.dispatch(['build', '--no-cache', '--memory', '96m', 'service'], None) + assert 'memory: 100663296' in result.stdout # 96 * 1024 * 1024 + def test_bundle_with_digests(self): self.base_dir = 'tests/fixtures/bundle-with-digests/' tmpdir = pytest.ensuretemp('cli_test_bundle') diff --git a/tests/fixtures/build-memory/Dockerfile b/tests/fixtures/build-memory/Dockerfile new file mode 100644 index 000000000..b27349b96 --- /dev/null +++ b/tests/fixtures/build-memory/Dockerfile @@ -0,0 +1,4 @@ +FROM busybox + +# Report the memory (through the size of the group memory) +RUN echo "memory:" $(cat /sys/fs/cgroup/memory/memory.limit_in_bytes) diff --git a/tests/fixtures/build-memory/docker-compose.yml b/tests/fixtures/build-memory/docker-compose.yml new file mode 100644 index 000000000..f98355851 --- /dev/null +++ b/tests/fixtures/build-memory/docker-compose.yml @@ -0,0 +1,6 @@ +version: '3.5' + +services: + service: + build: + context: . diff --git a/tests/unit/service_test.py b/tests/unit/service_test.py index 87c86a731..16670cff5 100644 --- a/tests/unit/service_test.py +++ b/tests/unit/service_test.py @@ -499,6 +499,7 @@ class ServiceTest(unittest.TestCase): target=None, shmsize=None, extra_hosts=None, + container_limits={'memory': None}, ) def test_ensure_image_exists_no_build(self): @@ -541,6 +542,7 @@ class ServiceTest(unittest.TestCase): target=None, shmsize=None, extra_hosts=None, + container_limits={'memory': None}, ) def test_build_does_not_pull(self): From 34ea11fcb72dcf4d314f62c457d2cfab1f81ee6c Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Fri, 1 Dec 2017 15:23:32 -0800 Subject: [PATCH 228/244] Allow port publish ranges Signed-off-by: Joffrey F --- compose/config/types.py | 18 +++++++++++++----- tests/unit/config/types_test.py | 26 ++++++++++++++++++++++++++ 2 files changed, 39 insertions(+), 5 deletions(-) diff --git a/compose/config/types.py b/compose/config/types.py index 548f2c1cd..d3b3cfc53 100644 --- a/compose/config/types.py +++ b/compose/config/types.py @@ -319,11 +319,19 @@ class ServicePort(namedtuple('_ServicePort', 'target published protocol mode ext except ValueError: raise ConfigurationError('Invalid target port: {}'.format(target)) - try: - if published: - published = int(published) - except ValueError: - raise ConfigurationError('Invalid published port: {}'.format(published)) + if published: + if isinstance(published, six.string_types) and '-' in published: # "x-y:z" format + a, b = published.split('-', 1) + try: + int(a) + int(b) + except ValueError: + raise ConfigurationError('Invalid published port: {}'.format(published)) + else: + try: + published = int(published) + except ValueError: + raise ConfigurationError('Invalid published port: {}'.format(published)) return super(ServicePort, cls).__new__( cls, target, published, *args, **kwargs diff --git a/tests/unit/config/types_test.py b/tests/unit/config/types_test.py index 3a43f727b..e7cc67b04 100644 --- a/tests/unit/config/types_test.py +++ b/tests/unit/config/types_test.py @@ -100,11 +100,37 @@ class TestServicePort(object): 'published': 25001 } in reprs + def test_parse_port_publish_range(self): + ports = ServicePort.parse('4440-4450:4000') + assert len(ports) == 1 + reprs = [p.repr() for p in ports] + assert { + 'target': 4000, + 'published': '4440-4450' + } in reprs + def test_parse_invalid_port(self): port_def = '4000p' with pytest.raises(ConfigurationError): ServicePort.parse(port_def) + def test_parse_invalid_publish_range(self): + port_def = '-4000:4000' + with pytest.raises(ConfigurationError): + ServicePort.parse(port_def) + + port_def = 'asdf:4000' + with pytest.raises(ConfigurationError): + ServicePort.parse(port_def) + + port_def = '1234-12f:4000' + with pytest.raises(ConfigurationError): + ServicePort.parse(port_def) + + port_def = '1234-1235-1239:4000' + with pytest.raises(ConfigurationError): + ServicePort.parse(port_def) + class TestVolumeSpec(object): From 58f2f10d49a8b46888173236277658af39971f36 Mon Sep 17 00:00:00 2001 From: Madeline Stager Date: Mon, 4 Dec 2017 20:01:00 -0600 Subject: [PATCH 229/244] Raise error if up used with both -d and --timeout Fix #5434 Signed-off-by: Madeline Stager --- compose/cli/main.py | 10 +++++++--- tests/acceptance/cli_test.py | 15 +++------------ 2 files changed, 10 insertions(+), 15 deletions(-) diff --git a/compose/cli/main.py b/compose/cli/main.py index f842f05c8..222f7d013 100644 --- a/compose/cli/main.py +++ b/compose/cli/main.py @@ -898,8 +898,8 @@ class TopLevelCommand(object): Options: -d Detached mode: Run containers in the background, - print new container names. - Incompatible with --abort-on-container-exit. + print new container names. Incompatible with + --abort-on-container-exit and --timeout. --no-color Produce monochrome output. --no-deps Don't start linked services. --force-recreate Recreate containers even if their configuration @@ -913,7 +913,8 @@ class TopLevelCommand(object): --abort-on-container-exit Stops all containers if any container was stopped. Incompatible with -d. -t, --timeout TIMEOUT Use this timeout in seconds for container shutdown - when attached or when containers are already + when attached or when containers are already. + Incompatible with -d. running. (default: 10) --remove-orphans Remove containers for services not defined in the Compose file @@ -934,6 +935,9 @@ class TopLevelCommand(object): if detached and (cascade_stop or exit_value_from): raise UserError("--abort-on-container-exit and -d cannot be combined.") + if detached and timeout: + raise UserError("-d and --timeout cannot be combined.") + if no_start: for excluded in ['-d', '--abort-on-container-exit', '--exit-code-from']: if options.get(excluded): diff --git a/tests/acceptance/cli_test.py b/tests/acceptance/cli_test.py index 21e716751..251e39db6 100644 --- a/tests/acceptance/cli_test.py +++ b/tests/acceptance/cli_test.py @@ -1325,18 +1325,9 @@ class CLITestCase(DockerClientTestCase): ['up', '-d', '--force-recreate', '--no-recreate'], returncode=1) - def test_up_with_timeout(self): - self.dispatch(['up', '-d', '-t', '1']) - service = self.project.get_service('simple') - another = self.project.get_service('another') - self.assertEqual(len(service.containers()), 1) - self.assertEqual(len(another.containers()), 1) - - # Ensure containers don't have stdin and stdout connected in -d mode - config = service.containers()[0].inspect()['Config'] - self.assertFalse(config['AttachStderr']) - self.assertFalse(config['AttachStdout']) - self.assertFalse(config['AttachStdin']) + def test_up_with_timeout_detached(self): + result = self.dispatch(['up', '-d', '-t', '1'], returncode=1) + assert "-d and --timeout cannot be combined." in result.stderr def test_up_handles_sigint(self): proc = start_process(self.base_dir, ['up', '-t', '2']) From 084818ce2b31e121268228e9b696ed0bab43bad2 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Mon, 4 Dec 2017 22:47:33 -0800 Subject: [PATCH 230/244] Add support for mount syntax Signed-off-by: Joffrey F --- compose/config/config.py | 40 +++++++------ compose/config/config_schema_v2.3.json | 34 ++++++++++- compose/config/serialize.py | 6 ++ compose/config/types.py | 13 ++++ compose/service.py | 63 ++++++++++++++------ compose/utils.py | 2 +- compose/volume.py | 9 ++- tests/acceptance/cli_test.py | 22 ++++--- tests/helpers.py | 13 ++-- tests/integration/project_test.py | 21 +++++++ tests/integration/service_test.py | 82 ++++++++++++++++++++++++++ tests/integration/testcases.py | 6 +- tests/unit/config/config_test.py | 53 ++++++++++++++++- tests/unit/service_test.py | 4 +- 14 files changed, 309 insertions(+), 59 deletions(-) diff --git a/compose/config/config.py b/compose/config/config.py index 864bc7e90..9b4130536 100644 --- a/compose/config/config.py +++ b/compose/config/config.py @@ -35,6 +35,7 @@ from .interpolation import interpolate_environment_variables from .sort_services import get_container_name_from_network_mode from .sort_services import get_service_name_from_network_mode from .sort_services import sort_service_dicts +from .types import MountSpec from .types import parse_extra_hosts from .types import parse_restart_spec from .types import ServiceLink @@ -809,6 +810,20 @@ def process_healthcheck(service_dict): return service_dict +def finalize_service_volumes(service_dict, environment): + if 'volumes' in service_dict: + finalized_volumes = [] + normalize = environment.get_boolean('COMPOSE_CONVERT_WINDOWS_PATHS') + for v in service_dict['volumes']: + if isinstance(v, dict): + finalized_volumes.append(MountSpec.parse(v, normalize)) + else: + finalized_volumes.append(VolumeSpec.parse(v, normalize)) + service_dict['volumes'] = finalized_volumes + + return service_dict + + def finalize_service(service_config, service_names, version, environment): service_dict = dict(service_config.config) @@ -822,12 +837,7 @@ def finalize_service(service_config, service_names, version, environment): for vf in service_dict['volumes_from'] ] - if 'volumes' in service_dict: - service_dict['volumes'] = [ - VolumeSpec.parse( - v, environment.get_boolean('COMPOSE_CONVERT_WINDOWS_PATHS') - ) for v in service_dict['volumes'] - ] + service_dict = finalize_service_volumes(service_dict, environment) if 'net' in service_dict: network_mode = service_dict.pop('net') @@ -1143,19 +1153,13 @@ def resolve_volume_paths(working_dir, service_dict): def resolve_volume_path(working_dir, volume): - mount_params = None if isinstance(volume, dict): - container_path = volume.get('target') - host_path = volume.get('source') - mode = None - if host_path: - if volume.get('read_only'): - mode = 'ro' - if volume.get('volume', {}).get('nocopy'): - mode = 'nocopy' - mount_params = (host_path, mode) - else: - container_path, mount_params = split_path_mapping(volume) + if volume.get('source', '').startswith('.') and volume['type'] == 'mount': + volume['source'] = expand_path(working_dir, volume['source']) + return volume + + mount_params = None + container_path, mount_params = split_path_mapping(volume) if mount_params is not None: host_path, mode = mount_params diff --git a/compose/config/config_schema_v2.3.json b/compose/config/config_schema_v2.3.json index 6f923871b..d50df3e81 100644 --- a/compose/config/config_schema_v2.3.json +++ b/compose/config/config_schema_v2.3.json @@ -293,7 +293,39 @@ }, "user": {"type": "string"}, "userns_mode": {"type": "string"}, - "volumes": {"type": "array", "items": {"type": "string"}, "uniqueItems": true}, + "volumes": { + "type": "array", + "items": { + "oneOf": [ + {"type": "string"}, + { + "type": "object", + "required": ["type"], + "additionalProperties": false, + "properties": { + "type": {"type": "string"}, + "source": {"type": "string"}, + "target": {"type": "string"}, + "read_only": {"type": "boolean"}, + "consistency": {"type": "string"}, + "bind": { + "type": "object", + "properties": { + "propagation": {"type": "string"} + } + }, + "volume": { + "type": "object", + "properties": { + "nocopy": {"type": "boolean"} + } + } + } + } + ], + "uniqueItems": true + } + }, "volume_driver": {"type": "string"}, "volumes_from": {"type": "array", "items": {"type": "string"}, "uniqueItems": true}, "working_dir": {"type": "string"} diff --git a/compose/config/serialize.py b/compose/config/serialize.py index 2b8c73f14..5e80e70e0 100644 --- a/compose/config/serialize.py +++ b/compose/config/serialize.py @@ -7,6 +7,7 @@ import yaml from compose.config import types from compose.const import COMPOSEFILE_V1 as V1 from compose.const import COMPOSEFILE_V2_1 as V2_1 +from compose.const import COMPOSEFILE_V2_3 as V2_3 from compose.const import COMPOSEFILE_V3_0 as V3_0 from compose.const import COMPOSEFILE_V3_2 as V3_2 from compose.const import COMPOSEFILE_V3_4 as V3_4 @@ -34,6 +35,7 @@ def serialize_string(dumper, data): return representer(data) +yaml.SafeDumper.add_representer(types.MountSpec, serialize_dict_type) yaml.SafeDumper.add_representer(types.VolumeFromSpec, serialize_config_type) yaml.SafeDumper.add_representer(types.VolumeSpec, serialize_config_type) yaml.SafeDumper.add_representer(types.ServiceSecret, serialize_dict_type) @@ -141,5 +143,9 @@ def denormalize_service_dict(service_dict, version, image_digest=None): p.legacy_repr() if isinstance(p, types.ServicePort) else p for p in service_dict['ports'] ] + if 'volumes' in service_dict and (version < V2_3 or (version > V3_0 and version < V3_2)): + service_dict['volumes'] = [ + v.legacy_repr() if isinstance(v, types.MountSpec) else v for v in service_dict['volumes'] + ] return service_dict diff --git a/compose/config/types.py b/compose/config/types.py index d3b3cfc53..c134bd7ca 100644 --- a/compose/config/types.py +++ b/compose/config/types.py @@ -144,6 +144,15 @@ class MountSpec(object): } _fields = ['type', 'source', 'target', 'read_only', 'consistency'] + @classmethod + def parse(cls, mount_dict, normalize=False): + if mount_dict.get('source'): + mount_dict['source'] = os.path.normpath(mount_dict['source']) + if normalize: + mount_dict['source'] = normalize_path_for_engine(mount_dict['source']) + + return cls(**mount_dict) + def __init__(self, type, source=None, target=None, read_only=None, consistency=None, **kwargs): self.type = type self.source = source @@ -174,6 +183,10 @@ class MountSpec(object): def is_named_volume(self): return self.type == 'volume' and self.source + @property + def external(self): + return self.source + class VolumeSpec(namedtuple('_VolumeSpec', 'external internal mode')): diff --git a/compose/service.py b/compose/service.py index bfc2e5940..f51f0e5af 100644 --- a/compose/service.py +++ b/compose/service.py @@ -785,15 +785,23 @@ class Service(object): self.options.get('labels'), override_options.get('labels')) + container_volumes = [] + container_mounts = [] + if 'volumes' in container_options: + container_volumes = [ + v for v in container_options.get('volumes') if isinstance(v, VolumeSpec) + ] + container_mounts = [v for v in container_options.get('volumes') if isinstance(v, MountSpec)] + binds, affinity = merge_volume_bindings( - container_options.get('volumes') or [], - self.options.get('tmpfs') or [], - previous_container) + container_volumes, self.options.get('tmpfs') or [], previous_container, + container_mounts + ) override_options['binds'] = binds container_options['environment'].update(affinity) - container_options['volumes'] = dict( - (v.internal, {}) for v in container_options.get('volumes') or {}) + container_options['volumes'] = dict((v.internal, {}) for v in container_volumes or {}) + override_options['mounts'] = [build_mount(v) for v in container_mounts] or None secret_volumes = self.get_secret_volumes() if secret_volumes: @@ -803,7 +811,8 @@ class Service(object): (v.target, {}) for v in secret_volumes ) else: - override_options['mounts'] = [build_mount(v) for v in secret_volumes] + override_options['mounts'] = override_options.get('mounts') or [] + override_options['mounts'].extend([build_mount(v) for v in secret_volumes]) container_options['image'] = self.image_name @@ -1245,32 +1254,40 @@ def parse_repository_tag(repo_path): # Volumes -def merge_volume_bindings(volumes, tmpfs, previous_container): - """Return a list of volume bindings for a container. Container data volumes - are replaced by those from the previous container. +def merge_volume_bindings(volumes, tmpfs, previous_container, mounts): + """ + Return a list of volume bindings for a container. Container data volumes + are replaced by those from the previous container. + Anonymous mounts are updated in place. """ affinity = {} volume_bindings = dict( build_volume_binding(volume) for volume in volumes - if volume.external) + if volume.external + ) if previous_container: - old_volumes = get_container_data_volumes(previous_container, volumes, tmpfs) + old_volumes, old_mounts = get_container_data_volumes( + previous_container, volumes, tmpfs, mounts + ) warn_on_masked_volume(volumes, old_volumes, previous_container.service) volume_bindings.update( - build_volume_binding(volume) for volume in old_volumes) + build_volume_binding(volume) for volume in old_volumes + ) - if old_volumes: + if old_volumes or old_mounts: affinity = {'affinity:container': '=' + previous_container.id} return list(volume_bindings.values()), affinity -def get_container_data_volumes(container, volumes_option, tmpfs_option): - """Find the container data volumes that are in `volumes_option`, and return - a mapping of volume bindings for those volumes. +def get_container_data_volumes(container, volumes_option, tmpfs_option, mounts_option): + """ + Find the container data volumes that are in `volumes_option`, and return + a mapping of volume bindings for those volumes. + Anonymous volume mounts are updated in place instead. """ volumes = [] volumes_option = volumes_option or [] @@ -1309,7 +1326,19 @@ def get_container_data_volumes(container, volumes_option, tmpfs_option): volume = volume._replace(external=mount['Name']) volumes.append(volume) - return volumes + updated_mounts = False + for mount in mounts_option: + if mount.type != 'volume': + continue + + ctnr_mount = container_mounts.get(mount.target) + if not ctnr_mount.get('Name'): + continue + + mount.source = ctnr_mount['Name'] + updated_mounts = True + + return volumes, updated_mounts def warn_on_masked_volume(volumes_option, container_volumes, service): diff --git a/compose/utils.py b/compose/utils.py index 197ae6eb2..00b01df2e 100644 --- a/compose/utils.py +++ b/compose/utils.py @@ -101,7 +101,7 @@ def json_stream(stream): def json_hash(obj): - dump = json.dumps(obj, sort_keys=True, separators=(',', ':')) + dump = json.dumps(obj, sort_keys=True, separators=(',', ':'), default=lambda x: x.repr()) h = hashlib.sha256() h.update(dump.encode('utf8')) return h.hexdigest() diff --git a/compose/volume.py b/compose/volume.py index da8ba25ca..0b148620f 100644 --- a/compose/volume.py +++ b/compose/volume.py @@ -7,6 +7,7 @@ from docker.errors import NotFound from docker.utils import version_lt from .config import ConfigurationError +from .config.types import VolumeSpec from .const import LABEL_PROJECT from .const import LABEL_VOLUME @@ -145,5 +146,9 @@ class ProjectVolumes(object): if not volume_spec.is_named_volume: return volume_spec - volume = self.volumes[volume_spec.external] - return volume_spec._replace(external=volume.full_name) + if isinstance(volume_spec, VolumeSpec): + volume = self.volumes[volume_spec.external] + return volume_spec._replace(external=volume.full_name) + else: + volume_spec.source = self.volumes[volume_spec.source].full_name + return volume_spec diff --git a/tests/acceptance/cli_test.py b/tests/acceptance/cli_test.py index 251e39db6..91e75abad 100644 --- a/tests/acceptance/cli_test.py +++ b/tests/acceptance/cli_test.py @@ -428,13 +428,21 @@ class CLITestCase(DockerClientTestCase): 'timeout': '1s', 'retries': 5, }, - 'volumes': [ - '/host/path:/container/path:ro', - 'foobar:/container/volumepath:rw', - '/anonymous', - 'foobar:/container/volumepath2:nocopy' - ], - + 'volumes': [{ + 'read_only': True, + 'source': '/host/path', + 'target': '/container/path', + 'type': 'bind' + }, { + 'source': 'foobar', 'target': '/container/volumepath', 'type': 'volume' + }, { + 'target': '/anonymous', 'type': 'volume' + }, { + 'source': 'foobar', + 'target': '/container/volumepath2', + 'type': 'volume', + 'volume': {'nocopy': True} + }], 'stop_grace_period': '20s', }, }, diff --git a/tests/helpers.py b/tests/helpers.py index a93de993f..f151f9cde 100644 --- a/tests/helpers.py +++ b/tests/helpers.py @@ -19,12 +19,8 @@ def build_config_details(contents, working_dir='working_dir', filename='filename ) -def create_host_file(client, filename): +def create_custom_host_file(client, filename, content): dirname = os.path.dirname(filename) - - with open(filename, 'r') as fh: - content = fh.read() - container = client.create_container( 'busybox:latest', ['sh', '-c', 'echo -n "{}" > {}'.format(content, filename)], @@ -48,3 +44,10 @@ def create_host_file(client, filename): return container_info['Node']['Name'] finally: client.remove_container(container, force=True) + + +def create_host_file(client, filename): + with open(filename, 'r') as fh: + content = fh.read() + + return create_custom_host_file(client, filename, content) diff --git a/tests/integration/project_test.py b/tests/integration/project_test.py index 953dd52be..6686d96cc 100644 --- a/tests/integration/project_test.py +++ b/tests/integration/project_test.py @@ -35,6 +35,7 @@ from tests.integration.testcases import is_cluster from tests.integration.testcases import no_cluster from tests.integration.testcases import v2_1_only from tests.integration.testcases import v2_2_only +from tests.integration.testcases import v2_3_only from tests.integration.testcases import v2_only from tests.integration.testcases import v3_only @@ -436,6 +437,26 @@ class ProjectTest(DockerClientTestCase): self.assertNotEqual(db_container.id, old_db_id) self.assertEqual(db_container.get('Volumes./etc'), db_volume_path) + @v2_3_only() + def test_recreate_preserves_mounts(self): + web = self.create_service('web') + db = self.create_service('db', volumes=[types.MountSpec(type='volume', target='/etc')]) + project = Project('composetest', [web, db], self.client) + project.start() + assert len(project.containers()) == 0 + + project.up(['db']) + assert len(project.containers()) == 1 + old_db_id = project.containers()[0].id + db_volume_path = project.containers()[0].get_mount('/etc')['Source'] + + project.up(strategy=ConvergenceStrategy.always) + assert len(project.containers()) == 2 + + db_container = [c for c in project.containers() if 'db' in c.name][0] + assert db_container.id != old_db_id + assert db_container.get_mount('/etc')['Source'] == db_volume_path + def test_project_up_with_no_recreate_running(self): web = self.create_service('web') db = self.create_service('db', volumes=[VolumeSpec.parse('/var/db')]) diff --git a/tests/integration/service_test.py b/tests/integration/service_test.py index 00bacebf5..b9005b8e1 100644 --- a/tests/integration/service_test.py +++ b/tests/integration/service_test.py @@ -19,6 +19,7 @@ from .testcases import pull_busybox from .testcases import SWARM_SKIP_CONTAINERS_ALL from .testcases import SWARM_SKIP_CPU_SHARES from compose import __version__ +from compose.config.types import MountSpec from compose.config.types import VolumeFromSpec from compose.config.types import VolumeSpec from compose.const import IS_WINDOWS_PLATFORM @@ -37,6 +38,7 @@ from compose.service import NetworkMode from compose.service import PidMode from compose.service import Service from compose.utils import parse_nanoseconds_int +from tests.helpers import create_custom_host_file from tests.integration.testcases import is_cluster from tests.integration.testcases import no_cluster from tests.integration.testcases import v2_1_only @@ -276,6 +278,54 @@ class ServiceTest(DockerClientTestCase): self.assertTrue(path.basename(actual_host_path) == path.basename(host_path), msg=("Last component differs: %s, %s" % (actual_host_path, host_path))) + @v2_3_only() + def test_create_container_with_host_mount(self): + host_path = '/tmp/host-path' + container_path = '/container-path' + + create_custom_host_file(self.client, path.join(host_path, 'a.txt'), 'test') + + service = self.create_service( + 'db', + volumes=[ + MountSpec(type='bind', source=host_path, target=container_path, read_only=True) + ] + ) + container = service.create_container() + service.start_container(container) + mount = container.get_mount(container_path) + assert mount + assert path.basename(mount['Source']) == path.basename(host_path) + assert mount['RW'] is False + + @v2_3_only() + def test_create_container_with_tmpfs_mount(self): + container_path = '/container-tmpfs' + service = self.create_service( + 'db', + volumes=[MountSpec(type='tmpfs', target=container_path)] + ) + container = service.create_container() + service.start_container(container) + mount = container.get_mount(container_path) + assert mount + assert mount['Type'] == 'tmpfs' + + @v2_3_only() + def test_create_container_with_volume_mount(self): + container_path = '/container-volume' + volume_name = 'composetest_abcde' + self.client.create_volume(volume_name) + service = self.create_service( + 'db', + volumes=[MountSpec(type='volume', source=volume_name, target=container_path)] + ) + container = service.create_container() + service.start_container(container) + mount = container.get_mount(container_path) + assert mount + assert mount['Name'] == volume_name + def test_create_container_with_healthcheck_config(self): one_second = parse_nanoseconds_int('1s') healthcheck = { @@ -439,6 +489,38 @@ class ServiceTest(DockerClientTestCase): orig_container = new_container + @v2_3_only() + def test_execute_convergence_plan_recreate_twice_with_mount(self): + service = self.create_service( + 'db', + volumes=[MountSpec(target='/etc', type='volume')], + entrypoint=['top'], + command=['-d', '1'] + ) + + orig_container = service.create_container() + service.start_container(orig_container) + + orig_container.inspect() # reload volume data + volume_path = orig_container.get_mount('/etc')['Source'] + + # Do this twice to reproduce the bug + for _ in range(2): + new_container, = service.execute_convergence_plan( + ConvergencePlan('recreate', [orig_container]) + ) + + assert new_container.get_mount('/etc')['Source'] == volume_path + if not is_cluster(self.client): + assert ('affinity:container==%s' % orig_container.id in + new_container.get('Config.Env')) + else: + # In Swarm, the env marker is consumed and the container should be deployed + # on the same node. + assert orig_container.get('Node.Name') == new_container.get('Node.Name') + + orig_container = new_container + def test_execute_convergence_plan_when_containers_are_stopped(self): service = self.create_service( 'db', diff --git a/tests/integration/testcases.py b/tests/integration/testcases.py index 8435f97dd..5505df1b4 100644 --- a/tests/integration/testcases.py +++ b/tests/integration/testcases.py @@ -20,7 +20,7 @@ from compose.const import COMPOSEFILE_V2_2 as V2_2 from compose.const import COMPOSEFILE_V2_3 as V2_3 from compose.const import COMPOSEFILE_V3_0 as V3_0 from compose.const import COMPOSEFILE_V3_2 as V3_2 -from compose.const import COMPOSEFILE_V3_3 as V3_3 +from compose.const import COMPOSEFILE_V3_5 as V3_5 from compose.const import LABEL_PROJECT from compose.progress_stream import stream_output from compose.service import Service @@ -47,7 +47,7 @@ def get_links(container): def engine_max_version(): if 'DOCKER_VERSION' not in os.environ: - return V3_3 + return V3_5 version = os.environ['DOCKER_VERSION'].partition('-')[0] if version_lt(version, '1.10'): return V1 @@ -57,7 +57,7 @@ def engine_max_version(): return V2_1 if version_lt(version, '17.06'): return V3_2 - return V3_3 + return V3_5 def min_version_skip(version): diff --git a/tests/unit/config/config_test.py b/tests/unit/config/config_test.py index 00ba6c2c6..d519deb90 100644 --- a/tests/unit/config/config_test.py +++ b/tests/unit/config/config_test.py @@ -1137,9 +1137,12 @@ class ConfigTest(unittest.TestCase): details = config.ConfigDetails('.', [base_file, override_file]) service_dicts = config.load(details).services svc_volumes = map(lambda v: v.repr(), service_dicts[0]['volumes']) - assert sorted(svc_volumes) == sorted( - ['/anonymous', '/c:/b:rw', 'vol:/x:ro'] - ) + for vol in svc_volumes: + assert vol in [ + '/anonymous', + '/c:/b:rw', + {'source': 'vol', 'target': '/x', 'type': 'volume', 'read_only': True} + ] @mock.patch.dict(os.environ) def test_volume_mode_override(self): @@ -1223,6 +1226,50 @@ class ConfigTest(unittest.TestCase): assert volume.external == 'data0028' assert volume.is_named_volume + def test_volumes_long_syntax(self): + base_file = config.ConfigFile( + 'base.yaml', { + 'version': '2.3', + 'services': { + 'web': { + 'image': 'busybox:latest', + 'volumes': [ + { + 'target': '/anonymous', 'type': 'volume' + }, { + 'source': '/abc', 'target': '/xyz', 'type': 'bind' + }, { + 'source': '\\\\.\\pipe\\abcd', 'target': '/named_pipe', 'type': 'npipe' + }, { + 'type': 'tmpfs', 'target': '/tmpfs' + } + ] + }, + }, + }, + ) + details = config.ConfigDetails('.', [base_file]) + config_data = config.load(details) + volumes = config_data.services[0].get('volumes') + anon_volume = [v for v in volumes if v.target == '/anonymous'][0] + tmpfs_mount = [v for v in volumes if v.type == 'tmpfs'][0] + host_mount = [v for v in volumes if v.type == 'bind'][0] + npipe_mount = [v for v in volumes if v.type == 'npipe'][0] + + assert anon_volume.type == 'volume' + assert not anon_volume.is_named_volume + + assert tmpfs_mount.target == '/tmpfs' + assert not tmpfs_mount.is_named_volume + + assert host_mount.source == os.path.normpath('/abc') + assert host_mount.target == '/xyz' + assert not host_mount.is_named_volume + + assert npipe_mount.source == '\\\\.\\pipe\\abcd' + assert npipe_mount.target == '/named_pipe' + assert not npipe_mount.is_named_volume + def test_config_valid_service_names(self): for valid_name in ['_', '-', '.__.', '_what-up.', 'what_.up----', 'whatup']: services = config.load( diff --git a/tests/unit/service_test.py b/tests/unit/service_test.py index 16670cff5..24ed60e94 100644 --- a/tests/unit/service_test.py +++ b/tests/unit/service_test.py @@ -939,7 +939,7 @@ class ServiceVolumesTest(unittest.TestCase): VolumeSpec.parse('imagedata:/mnt/image/data:rw'), ] - volumes = get_container_data_volumes(container, options, ['/dev/tmpfs']) + volumes, _ = get_container_data_volumes(container, options, ['/dev/tmpfs'], []) assert sorted(volumes) == sorted(expected) def test_merge_volume_bindings(self): @@ -975,7 +975,7 @@ class ServiceVolumesTest(unittest.TestCase): 'existingvolume:/existing/volume:rw', ] - binds, affinity = merge_volume_bindings(options, ['/dev/tmpfs'], previous_container) + binds, affinity = merge_volume_bindings(options, ['/dev/tmpfs'], previous_container, []) assert sorted(binds) == sorted(expected) assert affinity == {'affinity:container': '=cdefab'} From 99e9e32d7ebc2da54c4f1634560fbf344ce5b68d Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Wed, 6 Dec 2017 16:48:14 -0800 Subject: [PATCH 231/244] Add support for custom names for networks, secrets, configs Finalize v3.5 schema Signed-off-by: Joffrey F --- compose/config/config.py | 5 +- compose/config/config_schema_v2.1.json | 3 +- compose/config/config_schema_v2.2.json | 3 +- compose/config/config_schema_v2.3.json | 3 +- compose/config/config_schema_v3.5.json | 57 ++++++++++++++----- compose/config/serialize.py | 10 +++- compose/config/types.py | 5 +- compose/network.py | 23 ++++---- compose/project.py | 2 +- docker-compose.spec | 5 ++ tests/acceptance/cli_test.py | 16 ++++++ .../networks/external-networks-v3-5.yml | 17 ++++++ tests/integration/project_test.py | 37 ++++++++++++ tests/unit/config/config_test.py | 54 ++++++++++++++---- 14 files changed, 196 insertions(+), 44 deletions(-) create mode 100644 tests/fixtures/networks/external-networks-v3-5.yml diff --git a/compose/config/config.py b/compose/config/config.py index 9b4130536..98719d6ba 100644 --- a/compose/config/config.py +++ b/compose/config/config.py @@ -410,12 +410,11 @@ def load_mapping(config_files, get_func, entity_type, working_dir=None): external = config.get('external') if external: - name_field = 'name' if entity_type == 'Volume' else 'external_name' validate_external(entity_type, name, config, config_file.version) if isinstance(external, dict): - config[name_field] = external.get('name') + config['name'] = external.get('name') elif not config.get('name'): - config[name_field] = name + config['name'] = name if 'driver_opts' in config: config['driver_opts'] = build_string_dict( diff --git a/compose/config/config_schema_v2.1.json b/compose/config/config_schema_v2.1.json index 6b74f0ed6..15b78e5db 100644 --- a/compose/config/config_schema_v2.1.json +++ b/compose/config/config_schema_v2.1.json @@ -350,7 +350,8 @@ }, "internal": {"type": "boolean"}, "enable_ipv6": {"type": "boolean"}, - "labels": {"$ref": "#/definitions/list_or_dict"} + "labels": {"$ref": "#/definitions/list_or_dict"}, + "name": {"type": "string"} }, "additionalProperties": false }, diff --git a/compose/config/config_schema_v2.2.json b/compose/config/config_schema_v2.2.json index 21343b893..7a3eed0a9 100644 --- a/compose/config/config_schema_v2.2.json +++ b/compose/config/config_schema_v2.2.json @@ -357,7 +357,8 @@ }, "internal": {"type": "boolean"}, "enable_ipv6": {"type": "boolean"}, - "labels": {"$ref": "#/definitions/list_or_dict"} + "labels": {"$ref": "#/definitions/list_or_dict"}, + "name": {"type": "string"} }, "additionalProperties": false }, diff --git a/compose/config/config_schema_v2.3.json b/compose/config/config_schema_v2.3.json index d50df3e81..7c0e54807 100644 --- a/compose/config/config_schema_v2.3.json +++ b/compose/config/config_schema_v2.3.json @@ -393,7 +393,8 @@ }, "internal": {"type": "boolean"}, "enable_ipv6": {"type": "boolean"}, - "labels": {"$ref": "#/definitions/list_or_dict"} + "labels": {"$ref": "#/definitions/list_or_dict"}, + "name": {"type": "string"} }, "additionalProperties": false }, diff --git a/compose/config/config_schema_v3.5.json b/compose/config/config_schema_v3.5.json index 1e65b2087..565da0193 100644 --- a/compose/config/config_schema_v3.5.json +++ b/compose/config/config_schema_v3.5.json @@ -155,6 +155,7 @@ "hostname": {"type": "string"}, "image": {"type": "string"}, "ipc": {"type": "string"}, + "isolation": {"type": "string"}, "labels": {"$ref": "#/definitions/list_or_dict"}, "links": {"type": "array", "items": {"type": "string"}, "uniqueItems": true}, @@ -282,7 +283,6 @@ { "type": "object", "required": ["type"], - "additionalProperties": false, "properties": { "type": {"type": "string"}, "source": {"type": "string"}, @@ -301,7 +301,8 @@ "nocopy": {"type": "boolean"} } } - } + }, + "additionalProperties": false } ], "uniqueItems": true @@ -318,7 +319,7 @@ "additionalProperties": false, "properties": { "disable": {"type": "boolean"}, - "interval": {"type": "string"}, + "interval": {"type": "string", "format": "duration"}, "retries": {"type": "number"}, "test": { "oneOf": [ @@ -326,7 +327,8 @@ {"type": "array", "items": {"type": "string"}} ] }, - "timeout": {"type": "string"} + "timeout": {"type": "string", "format": "duration"}, + "start_period": {"type": "string", "format": "duration"} } }, "deployment": { @@ -354,8 +356,23 @@ "resources": { "type": "object", "properties": { - "limits": {"$ref": "#/definitions/resource"}, - "reservations": {"$ref": "#/definitions/resource"} + "limits": { + "type": "object", + "properties": { + "cpus": {"type": "string"}, + "memory": {"type": "string"} + }, + "additionalProperties": false + }, + "reservations": { + "type": "object", + "properties": { + "cpus": {"type": "string"}, + "memory": {"type": "string"}, + "generic_resources": {"$ref": "#/definitions/generic_resources"} + }, + "additionalProperties": false + } }, "additionalProperties": false }, @@ -390,20 +407,30 @@ "additionalProperties": false }, - "resource": { - "id": "#/definitions/resource", - "type": "object", - "properties": { - "cpus": {"type": "string"}, - "memory": {"type": "string"} - }, - "additionalProperties": false + "generic_resources": { + "id": "#/definitions/generic_resources", + "type": "array", + "items": { + "type": "object", + "properties": { + "discrete_resource_spec": { + "type": "object", + "properties": { + "kind": {"type": "string"}, + "value": {"type": "number"} + }, + "additionalProperties": false + } + }, + "additionalProperties": false + } }, "network": { "id": "#/definitions/network", "type": ["object", "null"], "properties": { + "name": {"type": "string"}, "driver": {"type": "string"}, "driver_opts": { "type": "object", @@ -470,6 +497,7 @@ "id": "#/definitions/secret", "type": "object", "properties": { + "name": {"type": "string"}, "file": {"type": "string"}, "external": { "type": ["boolean", "object"], @@ -486,6 +514,7 @@ "id": "#/definitions/config", "type": "object", "properties": { + "name": {"type": "string"}, "file": {"type": "string"}, "external": { "type": ["boolean", "object"], diff --git a/compose/config/serialize.py b/compose/config/serialize.py index 5e80e70e0..3ab43fc59 100644 --- a/compose/config/serialize.py +++ b/compose/config/serialize.py @@ -11,6 +11,7 @@ from compose.const import COMPOSEFILE_V2_3 as V2_3 from compose.const import COMPOSEFILE_V3_0 as V3_0 from compose.const import COMPOSEFILE_V3_2 as V3_2 from compose.const import COMPOSEFILE_V3_4 as V3_4 +from compose.const import COMPOSEFILE_V3_5 as V3_5 def serialize_config_type(dumper, data): @@ -69,7 +70,8 @@ def denormalize_config(config, image_digests=None): del conf['external_name'] if 'name' in conf: - if config.version < V2_1 or (config.version >= V3_0 and config.version < V3_4): + if config.version < V2_1 or ( + config.version >= V3_0 and config.version < v3_introduced_name_key(key)): del conf['name'] elif 'external' in conf: conf['external'] = True @@ -77,6 +79,12 @@ def denormalize_config(config, image_digests=None): return result +def v3_introduced_name_key(key): + if key == 'volumes': + return V3_4 + return V3_5 + + def serialize_config(config, image_digests=None): return yaml.safe_dump( denormalize_config(config, image_digests), diff --git a/compose/config/types.py b/compose/config/types.py index c134bd7ca..daf25f700 100644 --- a/compose/config/types.py +++ b/compose/config/types.py @@ -293,17 +293,18 @@ class ServiceLink(namedtuple('_ServiceLink', 'target alias')): return self.alias -class ServiceConfigBase(namedtuple('_ServiceConfigBase', 'source target uid gid mode')): +class ServiceConfigBase(namedtuple('_ServiceConfigBase', 'source target uid gid mode name')): @classmethod def parse(cls, spec): if isinstance(spec, six.string_types): - return cls(spec, None, None, None, None) + return cls(spec, None, None, None, None, None) return cls( spec.get('source'), spec.get('target'), spec.get('uid'), spec.get('gid'), spec.get('mode'), + spec.get('name') ) @property diff --git a/compose/network.py b/compose/network.py index ee5939c15..95e2bf60e 100644 --- a/compose/network.py +++ b/compose/network.py @@ -25,21 +25,22 @@ OPTS_EXCEPTIONS = [ class Network(object): def __init__(self, client, project, name, driver=None, driver_opts=None, - ipam=None, external_name=None, internal=False, enable_ipv6=False, - labels=None): + ipam=None, external=False, internal=False, enable_ipv6=False, + labels=None, custom_name=False): self.client = client self.project = project self.name = name self.driver = driver self.driver_opts = driver_opts self.ipam = create_ipam_config_from_dict(ipam) - self.external_name = external_name + self.external = external self.internal = internal self.enable_ipv6 = enable_ipv6 self.labels = labels + self.custom_name = custom_name def ensure(self): - if self.external_name: + if self.external: try: self.inspect() log.debug( @@ -51,7 +52,7 @@ class Network(object): 'Network {name} declared as external, but could' ' not be found. Please create the network manually' ' using `{command} {name}` and try again.'.format( - name=self.external_name, + name=self.full_name, command='docker network create' ) ) @@ -83,7 +84,7 @@ class Network(object): ) def remove(self): - if self.external_name: + if self.external: log.info("Network %s is external, skipping", self.full_name) return @@ -95,8 +96,8 @@ class Network(object): @property def full_name(self): - if self.external_name: - return self.external_name + if self.custom_name: + return self.name return '{0}_{1}'.format(self.project, self.name) @property @@ -203,14 +204,16 @@ def build_networks(name, config_data, client): network_config = config_data.networks or {} networks = { network_name: Network( - client=client, project=name, name=network_name, + client=client, project=name, + name=data.get('name', network_name), driver=data.get('driver'), driver_opts=data.get('driver_opts'), ipam=data.get('ipam'), - external_name=data.get('external_name'), + external=bool(data.get('external', False)), internal=data.get('internal'), enable_ipv6=data.get('enable_ipv6'), labels=data.get('labels'), + custom_name=data.get('name') is not None, ) for network_name, data in network_config.items() } diff --git a/compose/project.py b/compose/project.py index 411576386..11ee4a0b7 100644 --- a/compose/project.py +++ b/compose/project.py @@ -648,7 +648,7 @@ def get_secrets(service, service_secrets, secret_defs): "Service \"{service}\" uses an undefined secret \"{secret}\" " .format(service=service, secret=secret.source)) - if secret_def.get('external_name'): + if secret_def.get('external'): log.warn("Service \"{service}\" uses secret \"{secret}\" which is external. " "External secrets are not available to containers created by " "docker-compose.".format(service=service, secret=secret.source)) diff --git a/docker-compose.spec b/docker-compose.spec index 9c46421f0..83d7389f3 100644 --- a/docker-compose.spec +++ b/docker-compose.spec @@ -67,6 +67,11 @@ exe = EXE(pyz, 'compose/config/config_schema_v3.4.json', 'DATA' ), + ( + 'compose/config/config_schema_v3.5.json', + 'compose/config/config_schema_v3.5.json', + 'DATA' + ), ( 'compose/GITSHA', 'compose/GITSHA', diff --git a/tests/acceptance/cli_test.py b/tests/acceptance/cli_test.py index 91e75abad..3225eb49b 100644 --- a/tests/acceptance/cli_test.py +++ b/tests/acceptance/cli_test.py @@ -350,6 +350,22 @@ class CLITestCase(DockerClientTestCase): } } + def test_config_external_network_v3_5(self): + self.base_dir = 'tests/fixtures/networks' + result = self.dispatch(['-f', 'external-networks-v3-5.yml', 'config']) + json_result = yaml.load(result.stdout) + assert 'networks' in json_result + assert json_result['networks'] == { + 'foo': { + 'external': True, + 'name': 'some_foo', + }, + 'bar': { + 'external': True, + 'name': 'some_bar', + }, + } + def test_config_v1(self): self.base_dir = 'tests/fixtures/v1-config' result = self.dispatch(['config']) diff --git a/tests/fixtures/networks/external-networks-v3-5.yml b/tests/fixtures/networks/external-networks-v3-5.yml new file mode 100644 index 000000000..9ac7b14b5 --- /dev/null +++ b/tests/fixtures/networks/external-networks-v3-5.yml @@ -0,0 +1,17 @@ +version: "3.5" + +services: + web: + image: busybox + command: top + networks: + - foo + - bar + +networks: + foo: + external: true + name: some_foo + bar: + external: + name: some_bar diff --git a/tests/integration/project_test.py b/tests/integration/project_test.py index 6686d96cc..82e0adab3 100644 --- a/tests/integration/project_test.py +++ b/tests/integration/project_test.py @@ -953,6 +953,43 @@ class ProjectTest(DockerClientTestCase): assert 'LinkLocalIPs' in ipam_config assert ipam_config['LinkLocalIPs'] == ['169.254.8.8'] + @v2_1_only() + def test_up_with_custom_name_resources(self): + config_data = build_config( + version=V2_2, + services=[{ + 'name': 'web', + 'volumes': [VolumeSpec.parse('foo:/container-path')], + 'networks': {'foo': {}}, + 'image': 'busybox:latest' + }], + networks={ + 'foo': { + 'name': 'zztop', + 'labels': {'com.docker.compose.test_value': 'sharpdressedman'} + } + }, + volumes={ + 'foo': { + 'name': 'acdc', + 'labels': {'com.docker.compose.test_value': 'thefuror'} + } + } + ) + + project = Project.from_config( + client=self.client, + name='composetest', + config_data=config_data + ) + + project.up(detached=True) + network = [n for n in self.client.networks() if n['Name'] == 'zztop'][0] + volume = [v for v in self.client.volumes()['Volumes'] if v['Name'] == 'acdc'][0] + + assert network['Labels']['com.docker.compose.test_value'] == 'sharpdressedman' + assert volume['Labels']['com.docker.compose.test_value'] == 'thefuror' + @v2_1_only() def test_up_with_isolation(self): self.require_api_version('1.24') diff --git a/tests/unit/config/config_test.py b/tests/unit/config/config_test.py index d519deb90..7029fcb08 100644 --- a/tests/unit/config/config_test.py +++ b/tests/unit/config/config_test.py @@ -432,6 +432,40 @@ class ConfigTest(unittest.TestCase): 'label_key': 'label_val' } + def test_load_config_custom_resource_names(self): + base_file = config.ConfigFile( + 'base.yaml', { + 'version': '3.5', + 'volumes': { + 'abc': { + 'name': 'xyz' + } + }, + 'networks': { + 'abc': { + 'name': 'xyz' + } + }, + 'secrets': { + 'abc': { + 'name': 'xyz' + } + }, + 'configs': { + 'abc': { + 'name': 'xyz' + } + } + } + ) + details = config.ConfigDetails('.', [base_file]) + loaded_config = config.load(details) + + assert loaded_config.networks['abc'] == {'name': 'xyz'} + assert loaded_config.volumes['abc'] == {'name': 'xyz'} + assert loaded_config.secrets['abc']['name'] == 'xyz' + assert loaded_config.configs['abc']['name'] == 'xyz' + def test_load_config_volume_and_network_labels(self): base_file = config.ConfigFile( 'base.yaml', @@ -2539,8 +2573,8 @@ class ConfigTest(unittest.TestCase): 'name': 'web', 'image': 'example/web', 'secrets': [ - types.ServiceSecret('one', None, None, None, None), - types.ServiceSecret('source', 'target', '100', '200', 0o777), + types.ServiceSecret('one', None, None, None, None, None), + types.ServiceSecret('source', 'target', '100', '200', 0o777, None), ], }, ] @@ -2586,8 +2620,8 @@ class ConfigTest(unittest.TestCase): 'name': 'web', 'image': 'example/web', 'secrets': [ - types.ServiceSecret('one', None, None, None, None), - types.ServiceSecret('source', 'target', '100', '200', 0o777), + types.ServiceSecret('one', None, None, None, None, None), + types.ServiceSecret('source', 'target', '100', '200', 0o777, None), ], }, ] @@ -2624,8 +2658,8 @@ class ConfigTest(unittest.TestCase): 'name': 'web', 'image': 'example/web', 'configs': [ - types.ServiceConfig('one', None, None, None, None), - types.ServiceConfig('source', 'target', '100', '200', 0o777), + types.ServiceConfig('one', None, None, None, None, None), + types.ServiceConfig('source', 'target', '100', '200', 0o777, None), ], }, ] @@ -2671,8 +2705,8 @@ class ConfigTest(unittest.TestCase): 'name': 'web', 'image': 'example/web', 'configs': [ - types.ServiceConfig('one', None, None, None, None), - types.ServiceConfig('source', 'target', '100', '200', 0o777), + types.ServiceConfig('one', None, None, None, None, None), + types.ServiceConfig('source', 'target', '100', '200', 0o777, None), ], }, ] @@ -3131,7 +3165,7 @@ class InterpolationTest(unittest.TestCase): assert config_dict.secrets == { 'secretdata': { 'external': {'name': 'baz.bar'}, - 'external_name': 'baz.bar' + 'name': 'baz.bar' } } @@ -3149,7 +3183,7 @@ class InterpolationTest(unittest.TestCase): assert config_dict.configs == { 'configdata': { 'external': {'name': 'baz.bar'}, - 'external_name': 'baz.bar' + 'name': 'baz.bar' } } From 29c02ef598d1888fb389fa959ed2317afb40cc4f Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Wed, 6 Dec 2017 17:39:38 -0800 Subject: [PATCH 232/244] Fix bad rebase Signed-off-by: Joffrey F --- tests/acceptance/cli_test.py | 18 ------------------ tests/integration/testcases.py | 2 -- 2 files changed, 20 deletions(-) diff --git a/tests/acceptance/cli_test.py b/tests/acceptance/cli_test.py index 3225eb49b..c4905f909 100644 --- a/tests/acceptance/cli_test.py +++ b/tests/acceptance/cli_test.py @@ -536,24 +536,6 @@ class CLITestCase(DockerClientTestCase): assert self.dispatch(['pull', '--quiet']).stderr == '' assert self.dispatch(['pull', '--quiet']).stdout == '' - def test_pull_with_quiet(self): - assert self.dispatch(['pull', '--quiet']).stderr == '' - assert self.dispatch(['pull', '--quiet']).stdout == '' - - def test_pull_with_parallel_failure(self): - result = self.dispatch([ - '-f', 'ignore-pull-failures.yml', 'pull', '--parallel'], - returncode=1 - ) - - self.assertRegexpMatches(result.stderr, re.compile('^Pulling simple', re.MULTILINE)) - self.assertRegexpMatches(result.stderr, re.compile('^Pulling another', re.MULTILINE)) - self.assertRegexpMatches(result.stderr, - re.compile('^ERROR: for another .*does not exist.*', re.MULTILINE)) - self.assertRegexpMatches(result.stderr, - re.compile('''^(ERROR: )?(b')?.* nonexisting-image''', - re.MULTILINE)) - def test_build_plain(self): self.base_dir = 'tests/fixtures/simple-dockerfile' self.dispatch(['build', 'simple']) diff --git a/tests/integration/testcases.py b/tests/integration/testcases.py index 5505df1b4..9427f3d0d 100644 --- a/tests/integration/testcases.py +++ b/tests/integration/testcases.py @@ -75,7 +75,6 @@ def v2_1_only(): return min_version_skip(V2_1) - def v2_2_only(): return min_version_skip(V2_2) @@ -84,7 +83,6 @@ def v2_3_only(): return min_version_skip(V2_3) - def v3_only(): return min_version_skip(V3_0) From e96dfbac2a7982b8703abd3774ab661516096931 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Wed, 6 Dec 2017 17:25:37 -0800 Subject: [PATCH 233/244] Bump 1.18.0-rc1 Signed-off-by: Joffrey F --- CHANGELOG.md | 80 +++++++++++++++++++++++++++++++++++++++++++++ compose/__init__.py | 2 +- script/run/run.sh | 2 +- 3 files changed, 82 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d0be7ea76..ba91a505b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,86 @@ Change log ========== +1.18.0 (2017-12-15) +------------------- + +### New features + +#### Compose file version 3.5 + +- Introduced version 3.5 of the `docker-compose.yml` specification. + This version requires to be used with Docker Engine 17.06.0 or above + +- Added support for the `shm_size` parameter in build configurations + +- Added support for the `isolation` parameter in service definitions + +- Added support for custom names for network, secret and config definitions + +#### Compose file version 2.3 + +- Added support for `extra_hosts` in build configuration + +- Added support for the + [long syntax](https://docs.docker.com/compose/compose-file/#long-syntax-3) + for volume entries, as previously introduced in the 3.2 format. + Note that using this syntax will create + [mounts](https://docs.docker.com/engine/admin/volumes/bind-mounts/) + instead of volumes. + +#### Compose file version 2.1 and up + +- Added support for the `oom_kill_disable` parameter in service definitions + (2.x only) + +- Added support for custom names for network, secret and config definitions + (2.x only) + + +#### All formats + +- Values interpolated from the environment will now be converted to the + proper type when used in non-string fields. + +- Added support for `--labels` in `docker-compose run` + +- Added support for `--timeout` in `docker-compose down` + +- Added support for `--memory` in `docker-compose build` + +- Setting `stop_grace_period` in service definitions now also sets the + container's `stop_timeout` + +### Bugfixes + +- Fixed an issue where Compose was still handling service hostname according + to legacy engine behavior, causing hostnames containing dots to be cut up + +- Fixed a bug where the `X-Y:Z` syntax for ports was considered invalid + by Compose + +- Fixed an issue with CLI logging causing duplicate messages and inelegant + output to occur + +- Fixed a bug where the valid `${VAR:-}` syntax would cause Compose to + error out + +- Fixed a bug where `env_file` entries using an UTF-8 BOM were being read + incorrectly + +- Fixed a bug where missing secret files would generate an empty directory + in their place + +- Added validation for the `test` field in healthchecks + +- Added validation for the `subnet` field in IPAM configurations + +- Added validation for `volumes` properties when using the long syntax in + service definitions + +- The CLI now explicit prevents using `-d` and `--timeout` together + in `docker-compose up` + 1.17.1 (2017-11-08) ------------------ diff --git a/compose/__init__.py b/compose/__init__.py index 7b954eb4f..2b363f3be 100644 --- a/compose/__init__.py +++ b/compose/__init__.py @@ -1,4 +1,4 @@ from __future__ import absolute_import from __future__ import unicode_literals -__version__ = '1.18.0dev' +__version__ = '1.18.0-rc1' diff --git a/script/run/run.sh b/script/run/run.sh index 58483196d..441c0d806 100755 --- a/script/run/run.sh +++ b/script/run/run.sh @@ -15,7 +15,7 @@ set -e -VERSION="1.17.1" +VERSION="1.18.0-rc1" IMAGE="docker/compose:$VERSION" From 2e232ee97cb0274060b59336f294c5abcd489db7 Mon Sep 17 00:00:00 2001 From: Harald Albers Date: Thu, 7 Dec 2017 17:55:38 +0100 Subject: [PATCH 234/244] Fix wrong option name in changelog Signed-off-by: Harald Albers --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ba91a505b..74d4f93f6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -42,7 +42,7 @@ Change log - Values interpolated from the environment will now be converted to the proper type when used in non-string fields. -- Added support for `--labels` in `docker-compose run` +- Added support for `--label` in `docker-compose run` - Added support for `--timeout` in `docker-compose down` From ad40a9e65463f81c5b112d996a891c676a8588d4 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Thu, 7 Dec 2017 12:44:17 -0800 Subject: [PATCH 235/244] Expand mount source when type == bind Signed-off-by: Joffrey F --- compose/config/config.py | 2 +- tests/unit/config/config_test.py | 23 +++++++++++++++++++++++ 2 files changed, 24 insertions(+), 1 deletion(-) diff --git a/compose/config/config.py b/compose/config/config.py index 98719d6ba..51391fc7b 100644 --- a/compose/config/config.py +++ b/compose/config/config.py @@ -1153,7 +1153,7 @@ def resolve_volume_paths(working_dir, service_dict): def resolve_volume_path(working_dir, volume): if isinstance(volume, dict): - if volume.get('source', '').startswith('.') and volume['type'] == 'mount': + if volume.get('source', '').startswith('.') and volume['type'] == 'bind': volume['source'] = expand_path(working_dir, volume['source']) return volume diff --git a/tests/unit/config/config_test.py b/tests/unit/config/config_test.py index 7029fcb08..122ab2ef9 100644 --- a/tests/unit/config/config_test.py +++ b/tests/unit/config/config_test.py @@ -1304,6 +1304,29 @@ class ConfigTest(unittest.TestCase): assert npipe_mount.target == '/named_pipe' assert not npipe_mount.is_named_volume + def test_load_bind_mount_relative_path(self): + expected_source = 'C:\\tmp\\web' if IS_WINDOWS_PLATFORM else '/tmp/web' + base_file = config.ConfigFile( + 'base.yaml', { + 'version': '3.4', + 'services': { + 'web': { + 'image': 'busybox:latest', + 'volumes': [ + {'type': 'bind', 'source': './web', 'target': '/web'}, + ], + }, + }, + }, + ) + + details = config.ConfigDetails('/tmp', [base_file]) + config_data = config.load(details) + mount = config_data.services[0].get('volumes')[0] + assert mount.target == '/web' + assert mount.type == 'bind' + assert mount.source == expected_source + def test_config_valid_service_names(self): for valid_name in ['_', '-', '.__.', '_what-up.', 'what_.up----', 'whatup']: services = config.load( From f79f06ca4a4901ad2bf4bdb057e1682973723623 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Thu, 7 Dec 2017 15:46:58 -0800 Subject: [PATCH 236/244] Recover from possible unicode errors in get_conn_error_message Signed-off-by: Joffrey F --- compose/cli/errors.py | 20 +++++++++++++------- 1 file changed, 13 insertions(+), 7 deletions(-) diff --git a/compose/cli/errors.py b/compose/cli/errors.py index 1506aa660..eefa4ebe4 100644 --- a/compose/cli/errors.py +++ b/compose/cli/errors.py @@ -106,7 +106,8 @@ def log_api_error(e, client_version): log.error( "The Docker Engine version is less than the minimum required by " "Compose. Your current project requires a Docker Engine of " - "version {version} or greater.".format(version=version)) + "version {version} or greater.".format(version=version) + ) def exit_with_error(msg): @@ -115,12 +116,17 @@ def exit_with_error(msg): def get_conn_error_message(url): - if find_executable('docker') is None: - return docker_not_found_msg("Couldn't connect to Docker daemon.") - if is_docker_for_mac_installed(): - return conn_error_docker_for_mac - if find_executable('docker-machine') is not None: - return conn_error_docker_machine + try: + if find_executable('docker') is None: + return docker_not_found_msg("Couldn't connect to Docker daemon.") + if is_docker_for_mac_installed(): + return conn_error_docker_for_mac + if find_executable('docker-machine') is not None: + return conn_error_docker_machine + except UnicodeDecodeError: + # https://github.com/docker/compose/issues/5442 + # Ignore the error and print the generic message instead. + pass return conn_error_generic.format(url=url) From 45d2eb40039da06279683e113393d8e286189e3d Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Fri, 8 Dec 2017 14:16:14 -0800 Subject: [PATCH 237/244] Handle non-ascii characters in npipe error handler Signed-off-by: Joffrey F --- compose/cli/errors.py | 10 +++++----- compose/cli/utils.py | 13 +++++++++++++ tests/unit/cli/errors_test.py | 10 ++++++++++ 3 files changed, 28 insertions(+), 5 deletions(-) diff --git a/compose/cli/errors.py b/compose/cli/errors.py index eefa4ebe4..82768970b 100644 --- a/compose/cli/errors.py +++ b/compose/cli/errors.py @@ -7,7 +7,6 @@ import socket from distutils.spawn import find_executable from textwrap import dedent -import six from docker.errors import APIError from requests.exceptions import ConnectionError as RequestsConnectionError from requests.exceptions import ReadTimeout @@ -15,6 +14,7 @@ from requests.exceptions import SSLError from requests.packages.urllib3.exceptions import ReadTimeoutError from ..const import API_VERSION_TO_ENGINE_VERSION +from .utils import binarystr_to_unicode from .utils import is_docker_for_mac_installed from .utils import is_mac from .utils import is_ubuntu @@ -75,7 +75,9 @@ def log_windows_pipe_error(exc): ) else: log.error( - "Windows named pipe error: {} (code: {})".format(exc.strerror, exc.winerror) + "Windows named pipe error: {} (code: {})".format( + binarystr_to_unicode(exc.strerror), exc.winerror + ) ) @@ -89,9 +91,7 @@ def log_timeout_error(timeout): def log_api_error(e, client_version): - explanation = e.explanation - if isinstance(explanation, six.binary_type): - explanation = explanation.decode('utf-8') + explanation = binarystr_to_unicode(e.explanation) if 'client is newer than server' not in explanation: log.error(explanation) diff --git a/compose/cli/utils.py b/compose/cli/utils.py index 4d4fc4c18..a171d6678 100644 --- a/compose/cli/utils.py +++ b/compose/cli/utils.py @@ -10,6 +10,7 @@ import subprocess import sys import docker +import six import compose from ..const import IS_WINDOWS_PLATFORM @@ -148,3 +149,15 @@ def human_readable_file_size(size): size / float(1 << (order * 10)), suffixes[order] ) + + +def binarystr_to_unicode(s): + if not isinstance(s, six.binary_type): + return s + + if IS_WINDOWS_PLATFORM: + try: + return s.decode('windows-1250') + except UnicodeDecodeError: + pass + return s.decode('utf-8', 'replace') diff --git a/tests/unit/cli/errors_test.py b/tests/unit/cli/errors_test.py index 68326d1c7..7b53ed2b1 100644 --- a/tests/unit/cli/errors_test.py +++ b/tests/unit/cli/errors_test.py @@ -86,3 +86,13 @@ class TestHandleConnectionErrors(object): _, args, _ = mock_logging.error.mock_calls[0] assert "Windows named pipe error: The pipe is busy. (code: 231)" == args[0] + + @pytest.mark.skipif(not IS_WINDOWS_PLATFORM, reason='Needs pywin32') + def test_windows_pipe_error_encoding_issue(self, mock_logging): + import pywintypes + with pytest.raises(errors.ConnectionError): + with handle_connection_errors(mock.Mock(api_version='1.22')): + raise pywintypes.error(9999, 'WriteFile', 'I use weird characters \xe9') + + _, args, _ = mock_logging.error.mock_calls[0] + assert 'Windows named pipe error: I use weird characters \xe9 (code: 9999)' == args[0] From 189468b07f295c8df8997e770701be31159ef54e Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Fri, 8 Dec 2017 14:29:43 -0800 Subject: [PATCH 238/244] Bump 1.18.0-rc2 Signed-off-by: Joffrey F --- CHANGELOG.md | 2 ++ compose/__init__.py | 2 +- script/run/run.sh | 2 +- 3 files changed, 4 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 74d4f93f6..ac2050512 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -71,6 +71,8 @@ Change log - Fixed a bug where missing secret files would generate an empty directory in their place +- Fixed character encoding issues in the CLI's error handlers + - Added validation for the `test` field in healthchecks - Added validation for the `subnet` field in IPAM configurations diff --git a/compose/__init__.py b/compose/__init__.py index 2b363f3be..231670a5c 100644 --- a/compose/__init__.py +++ b/compose/__init__.py @@ -1,4 +1,4 @@ from __future__ import absolute_import from __future__ import unicode_literals -__version__ = '1.18.0-rc1' +__version__ = '1.18.0-rc2' diff --git a/script/run/run.sh b/script/run/run.sh index 441c0d806..4be14b722 100755 --- a/script/run/run.sh +++ b/script/run/run.sh @@ -15,7 +15,7 @@ set -e -VERSION="1.18.0-rc1" +VERSION="1.18.0-rc2" IMAGE="docker/compose:$VERSION" From 7d628ad1ab86b19d248f600ea5b5c958c92e6b58 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Mon, 11 Dec 2017 11:03:19 -0800 Subject: [PATCH 239/244] Add stop_grace_period to ALLOWED_KEYS Signed-off-by: Joffrey F --- compose/config/config.py | 1 + tests/unit/config/config_test.py | 4 +++- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/compose/config/config.py b/compose/config/config.py index 51391fc7b..95c12d1cc 100644 --- a/compose/config/config.py +++ b/compose/config/config.py @@ -126,6 +126,7 @@ ALLOWED_KEYS = DOCKER_CONFIG_KEYS + [ 'network_mode', 'init', 'scale', + 'stop_grace_period', ] DOCKER_VALID_URL_PREFIXES = ( diff --git a/tests/unit/config/config_test.py b/tests/unit/config/config_test.py index 122ab2ef9..e16e4bfa1 100644 --- a/tests/unit/config/config_test.py +++ b/tests/unit/config/config_test.py @@ -1150,7 +1150,8 @@ class ConfigTest(unittest.TestCase): 'volumes': [ {'source': '/a', 'target': '/b', 'type': 'bind'}, {'source': 'vol', 'target': '/x', 'type': 'volume', 'read_only': True} - ] + ], + 'stop_grace_period': '30s', } }, 'volumes': {'vol': {}} @@ -1177,6 +1178,7 @@ class ConfigTest(unittest.TestCase): '/c:/b:rw', {'source': 'vol', 'target': '/x', 'type': 'volume', 'read_only': True} ] + assert service_dicts[0]['stop_grace_period'] == '30s' @mock.patch.dict(os.environ) def test_volume_mode_override(self): From 7614becbfeda0457228bd381eee611c140134303 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Mon, 11 Dec 2017 11:03:46 -0800 Subject: [PATCH 240/244] Re-align docstring Signed-off-by: Joffrey F --- compose/cli/main.py | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/compose/cli/main.py b/compose/cli/main.py index 222f7d013..46c6b9652 100644 --- a/compose/cli/main.py +++ b/compose/cli/main.py @@ -365,17 +365,17 @@ class TopLevelCommand(object): Usage: down [options] Options: - --rmi type Remove images. Type must be one of: - 'all': Remove all images used by any service. - 'local': Remove only images that don't have a custom tag - set by the `image` field. - -v, --volumes Remove named volumes declared in the `volumes` section - of the Compose file and anonymous volumes - attached to containers. - --remove-orphans Remove containers for services not defined in the - Compose file - -t, --timeout TIMEOUT Specify a shutdown timeout in seconds. - (default: 10) + --rmi type Remove images. Type must be one of: + 'all': Remove all images used by any service. + 'local': Remove only images that don't have a + custom tag set by the `image` field. + -v, --volumes Remove named volumes declared in the `volumes` + section of the Compose file and anonymous volumes + attached to containers. + --remove-orphans Remove containers for services not defined in the + Compose file + -t, --timeout TIMEOUT Specify a shutdown timeout in seconds. + (default: 10) """ image_type = image_type_from_opt('--rmi', options['--rmi']) timeout = timeout_from_opts(options) From d5167d53290bfab93c42fb3e1884cbff238df303 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Tue, 12 Dec 2017 14:44:15 -0800 Subject: [PATCH 241/244] Avoid CLI crash if image has no tags Signed-off-by: Joffrey F --- compose/cli/main.py | 5 +++- tests/acceptance/cli_test.py | 26 +++++++++++++++---- tests/fixtures/tagless-image/Dockerfile | 2 ++ .../fixtures/tagless-image/docker-compose.yml | 5 ++++ 4 files changed, 32 insertions(+), 6 deletions(-) create mode 100644 tests/fixtures/tagless-image/Dockerfile create mode 100644 tests/fixtures/tagless-image/docker-compose.yml diff --git a/compose/cli/main.py b/compose/cli/main.py index 46c6b9652..308ac5bb2 100644 --- a/compose/cli/main.py +++ b/compose/cli/main.py @@ -511,7 +511,10 @@ class TopLevelCommand(object): rows = [] for container in containers: image_config = container.image_config - repo_tags = image_config['RepoTags'][0].rsplit(':', 1) + repo_tags = ( + image_config['RepoTags'][0].rsplit(':', 1) if image_config['RepoTags'] + else ('', '') + ) image_id = image_config['Id'].split(':')[1][:12] size = human_readable_file_size(image_config['Size']) rows.append([ diff --git a/tests/acceptance/cli_test.py b/tests/acceptance/cli_test.py index c4905f909..e0541f99f 100644 --- a/tests/acceptance/cli_test.py +++ b/tests/acceptance/cli_test.py @@ -2447,14 +2447,30 @@ class CLITestCase(DockerClientTestCase): assert 'multiplecomposefiles_another_1' in result.stdout assert 'multiplecomposefiles_simple_1' in result.stdout + @mock.patch.dict(os.environ) + def test_images_tagless_image(self): + self.base_dir = 'tests/fixtures/tagless-image' + stream = self.client.build(self.base_dir, decode=True) + img_id = None + for data in stream: + if 'aux' in data: + img_id = data['aux']['ID'] + break + if 'stream' in data and 'Successfully built' in data['stream']: + img_id = self.client.inspect_image(data['stream'].split(' ')[2].strip())['Id'] + + assert img_id + + os.environ['IMAGE_ID'] = img_id + self.project.get_service('foo').create_container() + result = self.dispatch(['images']) + assert '' in result.stdout + assert 'taglessimage_foo_1' in result.stdout + def test_up_with_override_yaml(self): self.base_dir = 'tests/fixtures/override-yaml-files' self._project = get_project(self.base_dir, []) - self.dispatch( - [ - 'up', '-d', - ], - None) + self.dispatch(['up', '-d'], None) containers = self.project.containers() self.assertEqual(len(containers), 2) diff --git a/tests/fixtures/tagless-image/Dockerfile b/tests/fixtures/tagless-image/Dockerfile new file mode 100644 index 000000000..567410552 --- /dev/null +++ b/tests/fixtures/tagless-image/Dockerfile @@ -0,0 +1,2 @@ +FROM busybox:latest +RUN touch /blah diff --git a/tests/fixtures/tagless-image/docker-compose.yml b/tests/fixtures/tagless-image/docker-compose.yml new file mode 100644 index 000000000..c4baf2ba1 --- /dev/null +++ b/tests/fixtures/tagless-image/docker-compose.yml @@ -0,0 +1,5 @@ +version: '2.3' +services: + foo: + image: ${IMAGE_ID} + command: top From 5a7ba590fb4ab3311e9caf2b00d04ccf60d5ae18 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Mon, 18 Dec 2017 12:35:59 -0800 Subject: [PATCH 242/244] Convert mounts to legacy volumes if API version < 1.30 Signed-off-by: Joffrey F --- compose/service.py | 65 ++++++++++++++++++------------- tests/integration/service_test.py | 18 +++++++++ 2 files changed, 56 insertions(+), 27 deletions(-) diff --git a/compose/service.py b/compose/service.py index f51f0e5af..fbab9281f 100644 --- a/compose/service.py +++ b/compose/service.py @@ -785,34 +785,9 @@ class Service(object): self.options.get('labels'), override_options.get('labels')) - container_volumes = [] - container_mounts = [] - if 'volumes' in container_options: - container_volumes = [ - v for v in container_options.get('volumes') if isinstance(v, VolumeSpec) - ] - container_mounts = [v for v in container_options.get('volumes') if isinstance(v, MountSpec)] - - binds, affinity = merge_volume_bindings( - container_volumes, self.options.get('tmpfs') or [], previous_container, - container_mounts + container_options, override_options = self._build_container_volume_options( + previous_container, container_options, override_options ) - override_options['binds'] = binds - container_options['environment'].update(affinity) - - container_options['volumes'] = dict((v.internal, {}) for v in container_volumes or {}) - override_options['mounts'] = [build_mount(v) for v in container_mounts] or None - - secret_volumes = self.get_secret_volumes() - if secret_volumes: - if version_lt(self.client.api_version, '1.30'): - override_options['binds'].extend(v.legacy_repr() for v in secret_volumes) - container_options['volumes'].update( - (v.target, {}) for v in secret_volumes - ) - else: - override_options['mounts'] = override_options.get('mounts') or [] - override_options['mounts'].extend([build_mount(v) for v in secret_volumes]) container_options['image'] = self.image_name @@ -838,6 +813,42 @@ class Service(object): container_options['environment']) return container_options + def _build_container_volume_options(self, previous_container, container_options, override_options): + container_volumes = [] + container_mounts = [] + if 'volumes' in container_options: + container_volumes = [ + v for v in container_options.get('volumes') if isinstance(v, VolumeSpec) + ] + container_mounts = [v for v in container_options.get('volumes') if isinstance(v, MountSpec)] + + binds, affinity = merge_volume_bindings( + container_volumes, self.options.get('tmpfs') or [], previous_container, + container_mounts + ) + override_options['binds'] = binds + container_options['environment'].update(affinity) + + container_options['volumes'] = dict((v.internal, {}) for v in container_volumes or {}) + if version_gte(self.client.api_version, '1.30'): + override_options['mounts'] = [build_mount(v) for v in container_mounts] or None + else: + override_options['binds'].extend(m.legacy_repr() for m in container_mounts) + container_options['volumes'].update((m.target, {}) for m in container_mounts) + + secret_volumes = self.get_secret_volumes() + if secret_volumes: + if version_lt(self.client.api_version, '1.30'): + override_options['binds'].extend(v.legacy_repr() for v in secret_volumes) + container_options['volumes'].update( + (v.target, {}) for v in secret_volumes + ) + else: + override_options['mounts'] = override_options.get('mounts') or [] + override_options['mounts'].extend([build_mount(v) for v in secret_volumes]) + + return container_options, override_options + def _get_container_host_config(self, override_options, one_off=False): options = dict(self.options, **override_options) diff --git a/tests/integration/service_test.py b/tests/integration/service_test.py index b9005b8e1..c1681a8de 100644 --- a/tests/integration/service_test.py +++ b/tests/integration/service_test.py @@ -13,6 +13,7 @@ from six import StringIO from six import text_type from .. import mock +from .testcases import docker_client from .testcases import DockerClientTestCase from .testcases import get_links from .testcases import pull_busybox @@ -326,6 +327,23 @@ class ServiceTest(DockerClientTestCase): assert mount assert mount['Name'] == volume_name + @v3_only() + def test_create_container_with_legacy_mount(self): + # Ensure mounts are converted to volumes if API version < 1.30 + # Needed to support long syntax in the 3.2 format + client = docker_client({}, version='1.25') + container_path = '/container-volume' + volume_name = 'composetest_abcde' + self.client.create_volume(volume_name) + service = Service('db', client=client, volumes=[ + MountSpec(type='volume', source=volume_name, target=container_path) + ], image='busybox:latest', command=['top'], project='composetest') + container = service.create_container() + service.start_container(container) + mount = container.get_mount(container_path) + assert mount + assert mount['Name'] == volume_name + def test_create_container_with_healthcheck_config(self): one_second = parse_nanoseconds_int('1s') healthcheck = { From 8dd22a962a4295ada9c8a45a5c58d02f8f66333f Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Wed, 13 Dec 2017 13:40:42 -0800 Subject: [PATCH 243/244] Bump 1.18.0 Signed-off-by: Joffrey F --- CHANGELOG.md | 19 ++++++++++--------- compose/__init__.py | 2 +- script/run/run.sh | 2 +- 3 files changed, 12 insertions(+), 11 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ac2050512..c0b3b5653 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,7 +9,7 @@ Change log #### Compose file version 3.5 - Introduced version 3.5 of the `docker-compose.yml` specification. - This version requires to be used with Docker Engine 17.06.0 or above + This version requires Docker Engine 17.06.0 or above - Added support for the `shm_size` parameter in build configurations @@ -21,20 +21,15 @@ Change log - Added support for `extra_hosts` in build configuration -- Added support for the - [long syntax](https://docs.docker.com/compose/compose-file/#long-syntax-3) - for volume entries, as previously introduced in the 3.2 format. - Note that using this syntax will create - [mounts](https://docs.docker.com/engine/admin/volumes/bind-mounts/) - instead of volumes. +- Added support for the [long syntax](https://docs.docker.com/compose/compose-file/#long-syntax-3) for volume entries, as previously introduced in the 3.2 format. + Note that using this syntax will create [mounts](https://docs.docker.com/engine/admin/volumes/bind-mounts/) instead of volumes. #### Compose file version 2.1 and up - Added support for the `oom_kill_disable` parameter in service definitions (2.x only) -- Added support for custom names for network, secret and config definitions - (2.x only) +- Added support for custom names for network definitions (2.x only) #### All formats @@ -62,6 +57,12 @@ Change log - Fixed an issue with CLI logging causing duplicate messages and inelegant output to occur +- Fixed an issue that caused `stop_grace_period` to be ignored when using + multiple Compose files + +- Fixed a bug that caused `docker-compose images` to crash when using + untagged images + - Fixed a bug where the valid `${VAR:-}` syntax would cause Compose to error out diff --git a/compose/__init__.py b/compose/__init__.py index 231670a5c..a15ad45f3 100644 --- a/compose/__init__.py +++ b/compose/__init__.py @@ -1,4 +1,4 @@ from __future__ import absolute_import from __future__ import unicode_literals -__version__ = '1.18.0-rc2' +__version__ = '1.18.0' diff --git a/script/run/run.sh b/script/run/run.sh index 4be14b722..abb4ff4fe 100755 --- a/script/run/run.sh +++ b/script/run/run.sh @@ -15,7 +15,7 @@ set -e -VERSION="1.18.0-rc2" +VERSION="1.18.0" IMAGE="docker/compose:$VERSION" From 17195d33e6bc6f010bbe15c60a4713c3b37cc799 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Mon, 18 Dec 2017 15:59:23 -0800 Subject: [PATCH 244/244] 1.19.0-dev Signed-off-by: Joffrey F --- compose/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/compose/__init__.py b/compose/__init__.py index 7b954eb4f..60a987ca6 100644 --- a/compose/__init__.py +++ b/compose/__init__.py @@ -1,4 +1,4 @@ from __future__ import absolute_import from __future__ import unicode_literals -__version__ = '1.18.0dev' +__version__ = '1.19.0dev'