From 7f1dc09404f60d704f47bd03bea7f2e86de55cae Mon Sep 17 00:00:00 2001 From: Alexey Rokhin Date: Wed, 17 May 2017 15:21:35 +0300 Subject: [PATCH 01/28] Fix testcases.py formatting Signed-off-by: Alexey Rokhin --- tests/integration/testcases.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/integration/testcases.py b/tests/integration/testcases.py index b72fb53a8..8435f97dd 100644 --- a/tests/integration/testcases.py +++ b/tests/integration/testcases.py @@ -75,6 +75,7 @@ def v2_1_only(): return min_version_skip(V2_1) + def v2_2_only(): return min_version_skip(V2_2) @@ -83,6 +84,7 @@ def v2_3_only(): return min_version_skip(V2_3) + def v3_only(): return min_version_skip(V3_0) From 11bd32b597ca0f710a68a0e3d02908a97542e2ca Mon Sep 17 00:00:00 2001 From: Alexey Rokhin Date: Wed, 17 May 2017 16:18:28 +0300 Subject: [PATCH 02/28] skip cpu_percent test for Linux Signed-off-by: Alexey Rokhin --- tests/integration/service_test.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/integration/service_test.py b/tests/integration/service_test.py index 84b54fe41..28cca4aad 100644 --- a/tests/integration/service_test.py +++ b/tests/integration/service_test.py @@ -28,6 +28,7 @@ from compose.const import LABEL_ONE_OFF from compose.const import LABEL_PROJECT from compose.const import LABEL_SERVICE from compose.const import LABEL_VERSION +from compose.const import IS_WINDOWS_PLATFORM from compose.container import Container from compose.errors import OperationFailedError from compose.project import OneOffFilter From 3089eda5ab58b8898d82d5dca98a184fbcb133d9 Mon Sep 17 00:00:00 2001 From: Alexey Rokhin Date: Wed, 17 May 2017 16:42:43 +0300 Subject: [PATCH 03/28] service_test.py reorder imports Signed-off-by: Alexey Rokhin --- tests/integration/service_test.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/integration/service_test.py b/tests/integration/service_test.py index 28cca4aad..84b54fe41 100644 --- a/tests/integration/service_test.py +++ b/tests/integration/service_test.py @@ -28,7 +28,6 @@ from compose.const import LABEL_ONE_OFF from compose.const import LABEL_PROJECT from compose.const import LABEL_SERVICE from compose.const import LABEL_VERSION -from compose.const import IS_WINDOWS_PLATFORM from compose.container import Container from compose.errors import OperationFailedError from compose.project import OneOffFilter From aee944393e7e27d2ebd6744a90c388134b2a542d Mon Sep 17 00:00:00 2001 From: Joel Barciauskas Date: Wed, 12 Apr 2017 17:45:09 -0400 Subject: [PATCH 04/28] Add --quiet parameter to docker-compose pull, using existing silent flag Signed-off-by: Joel Barciauskas --- compose/project.py | 2 +- tests/acceptance/cli_test.py | 4 ++++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/compose/project.py b/compose/project.py index 2310a2fcc..1faf97d40 100644 --- a/compose/project.py +++ b/compose/project.py @@ -496,7 +496,7 @@ class Project(object): if parallel_pull: def pull_service(service): - service.pull(ignore_pull_failures, True) + service.pull(ignore_pull_failures, True, silent=silent) _, errors = parallel.parallel_execute( services, diff --git a/tests/acceptance/cli_test.py b/tests/acceptance/cli_test.py index 78d1c1eb1..e721b940f 100644 --- a/tests/acceptance/cli_test.py +++ b/tests/acceptance/cli_test.py @@ -463,6 +463,10 @@ class CLITestCase(DockerClientTestCase): re.compile('''^(ERROR: )?(b')?.* nonexisting-image''', re.MULTILINE)) + def test_pull_with_quiet(self): + assert self.dispatch(['pull', '--quiet']).stderr == '' + assert self.dispatch(['pull', '--quiet']).stdout == '' + def test_build_plain(self): self.base_dir = 'tests/fixtures/simple-dockerfile' self.dispatch(['build', 'simple']) From f8b2981fb9b641a92902c2b8d05d5a007a040b0a Mon Sep 17 00:00:00 2001 From: NikitaVlaznev Date: Mon, 19 Jun 2017 17:05:19 +0300 Subject: [PATCH 05/28] Fix double silent argument value Fix for "TypeError: pull() got multiple values for keyword argument 'silent'." This change https://github.com/docker/compose/commit/e9b6cc23fcf01d4768c7e082b7bc91b43ff84e7e caused additional value to be passed for the 'silent' argument, that was already passed there: https://github.com/docker/compose/commit/f85da99ef3273794e855afda8678174419d3bf4f Signed-off-by: Nikita Vlaznev --- compose/project.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/compose/project.py b/compose/project.py index 1faf97d40..59d58f2f6 100644 --- a/compose/project.py +++ b/compose/project.py @@ -496,7 +496,7 @@ class Project(object): if parallel_pull: def pull_service(service): - service.pull(ignore_pull_failures, True, silent=silent) + service.pull(ignore_pull_failures, silent=silent) _, errors = parallel.parallel_execute( services, From 5b4573e7e5e379c0777cb8e38a71d612fe354db5 Mon Sep 17 00:00:00 2001 From: Evan Shaw Date: Sat, 1 Jul 2017 13:40:02 +1200 Subject: [PATCH 06/28] Always silence pull output with --parallel This is how things were prior to the addition of the --quiet flag. Making it not silent produces output that's weird and difficult to read. Signed-off-by: Evan Shaw --- compose/project.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/compose/project.py b/compose/project.py index 59d58f2f6..2310a2fcc 100644 --- a/compose/project.py +++ b/compose/project.py @@ -496,7 +496,7 @@ class Project(object): if parallel_pull: def pull_service(service): - service.pull(ignore_pull_failures, silent=silent) + service.pull(ignore_pull_failures, True) _, errors = parallel.parallel_execute( services, From 9587556e8f7d3ef745399f8423627a9a54b3dfdd Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Mon, 28 Aug 2017 17:35:55 -0700 Subject: [PATCH 07/28] Add --no-start flag to up command. Deprecate create command. Signed-off-by: Joffrey F --- compose/cli/main.py | 18 ++++++++++++++++-- compose/project.py | 6 ++++-- tests/acceptance/cli_test.py | 25 +++++++++++++++++++++++++ 3 files changed, 45 insertions(+), 4 deletions(-) diff --git a/compose/cli/main.py b/compose/cli/main.py index 83bc7d58c..21bf1f308 100644 --- a/compose/cli/main.py +++ b/compose/cli/main.py @@ -319,6 +319,7 @@ class TopLevelCommand(object): def create(self, options): """ Creates containers for a service. + This command is deprecated. Use the `up` command with `--no-start` instead. Usage: create [options] [SERVICE...] @@ -332,6 +333,11 @@ class TopLevelCommand(object): """ service_names = options['SERVICE'] + log.warn( + 'The create command is deprecated. ' + 'Use the up command with the --no-start flag instead.' + ) + self.project.create( service_names=service_names, strategy=convergence_strategy_from_opts(options), @@ -902,6 +908,7 @@ class TopLevelCommand(object): --no-recreate If containers already exist, don't recreate them. Incompatible with --force-recreate. --no-build Don't build an image, even if it's missing. + --no-start Don't start the services after creating them. --build Build images before starting containers. --abort-on-container-exit Stops all containers if any container was stopped. Incompatible with -d. @@ -922,10 +929,16 @@ class TopLevelCommand(object): timeout = timeout_from_opts(options) remove_orphans = options['--remove-orphans'] detached = options.get('-d') + no_start = options.get('--no-start') - if detached and cascade_stop: + if detached and (cascade_stop or exit_value_from): raise UserError("--abort-on-container-exit and -d cannot be combined.") + if no_start: + for excluded in ['-d', '--abort-on-container-exit', '--exit-code-from']: + if options.get(excluded): + raise UserError('--no-start and {} cannot be combined.'.format(excluded)) + with up_shutdown_context(self.project, service_names, timeout, detached): to_attach = self.project.up( service_names=service_names, @@ -936,9 +949,10 @@ class TopLevelCommand(object): detached=detached, remove_orphans=remove_orphans, scale_override=parse_scale_args(options['--scale']), + start=not no_start ) - if detached: + if detached or no_start: return attached_containers = filter_containers_to_service_names(to_attach, service_names) diff --git a/compose/project.py b/compose/project.py index 2310a2fcc..c8b57edd2 100644 --- a/compose/project.py +++ b/compose/project.py @@ -412,7 +412,8 @@ class Project(object): detached=False, remove_orphans=False, scale_override=None, - rescale=True): + rescale=True, + start=True): warn_for_swarm_mode(self.client) @@ -436,7 +437,8 @@ class Project(object): timeout=timeout, detached=detached, scale_override=scale_override.get(service.name), - rescale=rescale + rescale=rescale, + start=start ) def get_deps(service): diff --git a/tests/acceptance/cli_test.py b/tests/acceptance/cli_test.py index e721b940f..3a5e17ad8 100644 --- a/tests/acceptance/cli_test.py +++ b/tests/acceptance/cli_test.py @@ -776,6 +776,31 @@ class CLITestCase(DockerClientTestCase): for service in services: assert self.lookup(container, service.name) + @v2_only() + def test_up_no_start(self): + self.base_dir = 'tests/fixtures/v2-full' + self.dispatch(['up', '--no-start'], None) + + services = self.project.get_services() + + default_network = self.project.networks.networks['default'].full_name + front_network = self.project.networks.networks['front'].full_name + networks = self.client.networks(names=[default_network, front_network]) + assert len(networks) == 2 + + for service in services: + containers = service.containers(stopped=True) + assert len(containers) == 1 + + container = containers[0] + assert not container.is_running + assert container.get('State.Status') == 'created' + + volumes = self.project.volumes.volumes + assert 'data' in volumes + volume = volumes['data'] + assert volume.exists() + @v2_only() def test_up_no_ansi(self): self.base_dir = 'tests/fixtures/v2-simple' From 42aa1c34475f121dbc0710312a6645ee1cc11a9a Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Mon, 28 Aug 2017 18:18:31 -0700 Subject: [PATCH 08/28] Reduce up() cyclomatic complexity Signed-off-by: Joffrey F --- compose/cli/main.py | 62 +++++++++++++++++++++++++-------------------- 1 file changed, 35 insertions(+), 27 deletions(-) diff --git a/compose/cli/main.py b/compose/cli/main.py index 21bf1f308..face38e6d 100644 --- a/compose/cli/main.py +++ b/compose/cli/main.py @@ -969,33 +969,10 @@ class TopLevelCommand(object): if cascade_stop: print("Aborting on container exit...") - - exit_code = 0 - if exit_value_from: - candidates = list(filter( - lambda c: c.service == exit_value_from, - attached_containers)) - if not candidates: - log.error( - 'No containers matching the spec "{0}" ' - 'were run.'.format(exit_value_from) - ) - exit_code = 2 - elif len(candidates) > 1: - exit_values = filter( - lambda e: e != 0, - [c.inspect()['State']['ExitCode'] for c in candidates] - ) - - exit_code = exit_values[0] - else: - exit_code = candidates[0].inspect()['State']['ExitCode'] - else: - for e in self.project.containers(service_names=options['SERVICE'], stopped=True): - if (not e.is_running and cascade_starter == e.name): - if not e.exit_code == 0: - exit_code = e.exit_code - break + all_containers = self.project.containers(service_names=options['SERVICE'], stopped=True) + exit_code = compute_exit_code( + exit_value_from, attached_containers, cascade_starter, all_containers + ) self.project.stop(service_names=service_names, timeout=timeout) sys.exit(exit_code) @@ -1016,6 +993,37 @@ class TopLevelCommand(object): print(get_version_info('full')) +def compute_exit_code(exit_value_from, attached_containers, cascade_starter, all_containers): + exit_code = 0 + if exit_value_from: + candidates = list(filter( + lambda c: c.service == exit_value_from, + attached_containers)) + if not candidates: + log.error( + 'No containers matching the spec "{0}" ' + 'were run.'.format(exit_value_from) + ) + exit_code = 2 + elif len(candidates) > 1: + exit_values = filter( + lambda e: e != 0, + [c.inspect()['State']['ExitCode'] for c in candidates] + ) + + exit_code = exit_values[0] + else: + exit_code = candidates[0].inspect()['State']['ExitCode'] + else: + for e in all_containers: + if (not e.is_running and cascade_starter == e.name): + if not e.exit_code == 0: + exit_code = e.exit_code + break + + return exit_code + + def convergence_strategy_from_opts(options): no_recreate = options['--no-recreate'] force_recreate = options['--force-recreate'] From 8c6f2217c45dd004b4fa10cca1cc7023a6f241ea Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Fri, 25 Aug 2017 18:09:06 -0700 Subject: [PATCH 09/28] Add support for extension fields in v2.x and v3.4 Signed-off-by: Joffrey F --- compose/config/config_schema_v2.0.json | 1 + compose/config/config_schema_v2.1.json | 1 + compose/config/config_schema_v2.2.json | 1 + compose/config/config_schema_v2.3.json | 1 + compose/config/config_schema_v3.4-beta.json | 1 + compose/config/validation.py | 10 ++++++++++ tests/unit/config/config_test.py | 14 +++++++++++++- 7 files changed, 28 insertions(+), 1 deletion(-) diff --git a/compose/config/config_schema_v2.0.json b/compose/config/config_schema_v2.0.json index 14bafab40..2ad62ac52 100644 --- a/compose/config/config_schema_v2.0.json +++ b/compose/config/config_schema_v2.0.json @@ -41,6 +41,7 @@ } }, + "patternProperties": {"^x-": {}}, "additionalProperties": false, "definitions": { diff --git a/compose/config/config_schema_v2.1.json b/compose/config/config_schema_v2.1.json index 8a5e12834..24e6ba02c 100644 --- a/compose/config/config_schema_v2.1.json +++ b/compose/config/config_schema_v2.1.json @@ -41,6 +41,7 @@ } }, + "patternProperties": {"^x-": {}}, "additionalProperties": false, "definitions": { diff --git a/compose/config/config_schema_v2.2.json b/compose/config/config_schema_v2.2.json index 58ba409ff..86fc5df95 100644 --- a/compose/config/config_schema_v2.2.json +++ b/compose/config/config_schema_v2.2.json @@ -41,6 +41,7 @@ } }, + "patternProperties": {"^x-": {}}, "additionalProperties": false, "definitions": { diff --git a/compose/config/config_schema_v2.3.json b/compose/config/config_schema_v2.3.json index 7a9bdfdf1..a790bb405 100644 --- a/compose/config/config_schema_v2.3.json +++ b/compose/config/config_schema_v2.3.json @@ -41,6 +41,7 @@ } }, + "patternProperties": {"^x-": {}}, "additionalProperties": false, "definitions": { diff --git a/compose/config/config_schema_v3.4-beta.json b/compose/config/config_schema_v3.4-beta.json index 190c05f2c..cba063202 100644 --- a/compose/config/config_schema_v3.4-beta.json +++ b/compose/config/config_schema_v3.4-beta.json @@ -64,6 +64,7 @@ } }, + "patternProperties": {"^x-": {}}, "additionalProperties": false, "definitions": { diff --git a/compose/config/validation.py b/compose/config/validation.py index 0b7961e5a..c6722a14d 100644 --- a/compose/config/validation.py +++ b/compose/config/validation.py @@ -239,6 +239,16 @@ def handle_error_for_schema_with_id(error, path): invalid_config_key = parse_key_from_error_msg(error) return get_unsupported_config_msg(path, invalid_config_key) + if schema_id.startswith('config_schema_v'): + invalid_config_key = parse_key_from_error_msg(error) + return ('Invalid top-level property "{key}". Valid top-level ' + 'sections for this Compose file are: {properties}, and ' + 'extensions starting with "x-".\n\n{explanation}').format( + key=invalid_config_key, + properties=', '.join(error.schema['properties'].keys()), + explanation=VERSION_EXPLANATION + ) + if not error.path: return '{}\n\n{}'.format(error.message, VERSION_EXPLANATION) diff --git a/tests/unit/config/config_test.py b/tests/unit/config/config_test.py index 644290157..14dd01179 100644 --- a/tests/unit/config/config_test.py +++ b/tests/unit/config/config_test.py @@ -251,7 +251,7 @@ class ConfigTest(unittest.TestCase): ) ) - assert 'Additional properties are not allowed' in excinfo.exconly() + assert 'Invalid top-level property "web"' in excinfo.exconly() assert VERSION_EXPLANATION in excinfo.exconly() def test_named_volume_config_empty(self): @@ -773,6 +773,18 @@ class ConfigTest(unittest.TestCase): assert services[1]['name'] == 'db' assert services[2]['name'] == 'web' + def test_load_with_extensions(self): + config_details = build_config_details({ + 'version': '2.3', + 'x-data': { + 'lambda': 3, + 'excess': [True, {}] + } + }) + + config_data = config.load(config_details) + assert config_data.services == [] + def test_config_build_configuration(self): service = config.load( build_config_details( From d48296213b76d891db2bfdd7a3fa2eab028d80c3 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Fri, 1 Sep 2017 12:14:56 -0700 Subject: [PATCH 10/28] Update release process with most recent changes Signed-off-by: Joffrey F --- project/RELEASE-PROCESS.md | 33 +++++++++++++++++++++------------ script/release/make-branch | 3 +-- 2 files changed, 22 insertions(+), 14 deletions(-) diff --git a/project/RELEASE-PROCESS.md b/project/RELEASE-PROCESS.md index c1834f2fb..5b30545f4 100644 --- a/project/RELEASE-PROCESS.md +++ b/project/RELEASE-PROCESS.md @@ -24,7 +24,7 @@ As part of this script you'll be asked to: If the next release will be an RC, append `-rcN`, e.g. `1.4.0-rc1`. -2. Write release notes in `CHANGES.md`. +2. Write release notes in `CHANGELOG.md`. Almost every feature enhancement should be mentioned, with the most visible/exciting ones first. Use descriptive sentences and give context @@ -67,16 +67,13 @@ Check out the bump branch and run the `build-binaries` script When prompted build the non-linux binaries and test them. -1. Download the osx binary from Bintray. Make sure that the latest Travis - build has finished, otherwise you'll be downloading an old binary. +1. Download the different platform binaries by running the following script: - https://dl.bintray.com/docker-compose/$BRANCH_NAME/ + `./script/release/download-binaries $VERSION` -2. Download the windows binary from AppVeyor + The binaries for Linux, OSX and Windows will be downloaded in the `binaries-$VERSION` folder. - https://ci.appveyor.com/project/docker/compose - -3. Draft a release from the tag on GitHub (the script will open the window for +3. Draft a release from the tag on GitHub (the `build-binaries` script will open the window for you) The tag will only be present on Github when you run the `push-release` @@ -87,18 +84,30 @@ When prompted build the non-linux binaries and test them. If you're a Mac or Windows user, the best way to install Compose and keep it up-to-date is **[Docker for Mac and Windows](https://www.docker.com/products/docker)**. - Note that Compose 1.9.0 requires Docker Engine 1.10.0 or later for version 2 of the Compose File format, and Docker Engine 1.9.1 or later for version 1. Docker for Mac and Windows will automatically install the latest version of Docker Engine for you. + Docker for Mac and Windows will automatically install the latest version of Docker Engine for you. Alternatively, you can use the usual commands to install or upgrade Compose: ``` - curl -L https://github.com/docker/compose/releases/download/1.9.0/docker-compose-`uname -s`-`uname -m` > /usr/local/bin/docker-compose + curl -L https://github.com/docker/compose/releases/download/1.16.0/docker-compose-`uname -s`-`uname -m` > /usr/local/bin/docker-compose chmod +x /usr/local/bin/docker-compose ``` See the [install docs](https://docs.docker.com/compose/install/) for more install options and instructions. - Here's what's new: + ## Compose file format compatibility matrix + + | Compose file format | Docker Engine | + | --- | --- | + | 3.3 | 17.06.0+ | + | 3.0 – 3.2 | 1.13.0+ | + | 2.3| 17.06.0+ | + | 2.2 | 1.13.0+ | + | 2.1 | 1.12.0+ | + | 2.0 | 1.10.0+ | + | 1.0 | 1.9.1+ | + + ## Changes ...release notes go here... @@ -119,7 +128,7 @@ When prompted build the non-linux binaries and test them. 9. Check that all the binaries download (following the install instructions) and run. -10. Email maintainers@dockerproject.org and engineering@docker.com about the new release. +10. Announce the release on the appropriate Slack channel(s). ## If it’s a stable release (not an RC) diff --git a/script/release/make-branch b/script/release/make-branch index 7ccf3f055..b8a0cd31e 100755 --- a/script/release/make-branch +++ b/script/release/make-branch @@ -65,8 +65,7 @@ git config "branch.${BRANCH}.release" $VERSION editor=${EDITOR:-vim} -echo "Update versions in docs/install.md, compose/__init__.py, script/run/run.sh" -$editor docs/install.md +echo "Update versions in compose/__init__.py, script/run/run.sh" $editor compose/__init__.py $editor script/run/run.sh From 158a78657854a3f82c94861752996a1951c8d02d Mon Sep 17 00:00:00 2001 From: Vincent Demeester Date: Tue, 19 Sep 2017 18:27:02 +0200 Subject: [PATCH 11/28] Sync composefile v3.2 schema with `docker/cli` Signed-off-by: Vincent Demeester --- compose/config/config_schema_v3.2.json | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/compose/config/config_schema_v3.2.json b/compose/config/config_schema_v3.2.json index b26b2c6c6..2ca8e92db 100644 --- a/compose/config/config_schema_v3.2.json +++ b/compose/config/config_schema_v3.2.json @@ -170,7 +170,8 @@ "type": "array", "items": { "oneOf": [ - {"type": ["string", "number"], "format": "ports"}, + {"type": "number", "format": "ports"}, + {"type": "string", "format": "ports"}, { "type": "object", "properties": { @@ -249,6 +250,7 @@ "source": {"type": "string"}, "target": {"type": "string"}, "read_only": {"type": "boolean"}, + "consistency": {"type": "string"}, "bind": { "type": "object", "properties": { From 1da5b54d75024bbe5ceae9a296200681caaa77fd Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Tue, 26 Sep 2017 16:25:05 -0700 Subject: [PATCH 12/28] Fix oneOf validator parser to correctly process uniqueItems errors Signed-off-by: Joffrey F --- compose/config/validation.py | 15 +++++++-------- tests/unit/config/config_test.py | 25 ++++++++++++++++++++----- 2 files changed, 27 insertions(+), 13 deletions(-) diff --git a/compose/config/validation.py b/compose/config/validation.py index c6722a14d..940775a20 100644 --- a/compose/config/validation.py +++ b/compose/config/validation.py @@ -325,7 +325,6 @@ def _parse_oneof_validator(error): """ types = [] for context in error.context: - if context.validator == 'oneOf': _, error_msg = _parse_oneof_validator(context) return path_string(context.path), error_msg @@ -337,6 +336,13 @@ def _parse_oneof_validator(error): invalid_config_key = parse_key_from_error_msg(context) return (None, "contains unsupported option: '{}'".format(invalid_config_key)) + if context.validator == 'uniqueItems': + return ( + path_string(context.path) if context.path else None, + "contains non-unique items, please remove duplicates from {}".format( + context.instance), + ) + if context.path: return ( path_string(context.path), @@ -345,13 +351,6 @@ def _parse_oneof_validator(error): _parse_valid_types_from_validator(context.validator_value)), ) - if context.validator == 'uniqueItems': - return ( - None, - "contains non unique items, please remove duplicates from {}".format( - context.instance), - ) - if context.validator == 'type': types.append(context.validator_value) diff --git a/tests/unit/config/config_test.py b/tests/unit/config/config_test.py index 14dd01179..de9a61302 100644 --- a/tests/unit/config/config_test.py +++ b/tests/unit/config/config_test.py @@ -581,6 +581,20 @@ class ConfigTest(unittest.TestCase): assert 'Invalid service name \'mong\\o\'' in excinfo.exconly() + def test_config_duplicate_cache_from_values_validation_error(self): + with pytest.raises(ConfigurationError) as exc: + config.load( + build_config_details({ + 'version': '2.3', + 'services': { + 'test': {'build': {'context': '.', 'cache_from': ['a', 'b', 'a']}} + } + + }) + ) + + assert 'build.cache_from contains non-unique items' in exc.exconly() + def test_load_with_multiple_files_v1(self): base_file = config.ConfigFile( 'base.yaml', @@ -2751,11 +2765,12 @@ class PortsTest(unittest.TestCase): def check_config(self, cfg): config.load( - build_config_details( - {'web': dict(image='busybox', **cfg)}, - 'working_dir', - 'filename.yml' - ) + build_config_details({ + 'version': '2.3', + 'services': { + 'web': dict(image='busybox', **cfg) + }, + }, 'working_dir', 'filename.yml') ) From 53928a17c0d36c3e1a9b835febdb8e88797196b7 Mon Sep 17 00:00:00 2001 From: French Ben Date: Mon, 18 Sep 2017 16:30:32 -0700 Subject: [PATCH 13/28] Simple patch to allow s390x images to be built Needs integration with CI and s390x machine integration Signed-off-by: French Ben --- Dockerfile.s390x | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100644 Dockerfile.s390x diff --git a/Dockerfile.s390x b/Dockerfile.s390x new file mode 100644 index 000000000..aa71e27bc --- /dev/null +++ b/Dockerfile.s390x @@ -0,0 +1,6 @@ +FROM s390x/python:3.6.2-slim +ARG COMPOSE_VERSION=1.16.1 + +RUN pip install --no-cache-dir docker-compose==$COMPOSE_VERSION + +ENTRYPOINT ["docker-compose"] From cb2d65556b316b58debeb6e5f5ccc3010d0171d7 Mon Sep 17 00:00:00 2001 From: French Ben Date: Tue, 19 Sep 2017 10:14:55 -0700 Subject: [PATCH 14/28] Use slim alpine instead of bulky debian Signed-off-by: French Ben --- Dockerfile.s390x | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/Dockerfile.s390x b/Dockerfile.s390x index aa71e27bc..3b19bb390 100644 --- a/Dockerfile.s390x +++ b/Dockerfile.s390x @@ -1,6 +1,15 @@ -FROM s390x/python:3.6.2-slim +FROM s390x/alpine:3.6 + ARG COMPOSE_VERSION=1.16.1 -RUN pip install --no-cache-dir docker-compose==$COMPOSE_VERSION +RUN apk add --update --no-cache \ + python \ + py-pip \ + && pip install --no-cache-dir docker-compose==$COMPOSE_VERSION \ + && rm -rf /var/cache/apk/* + +WORKDIR /data +VOLUME /data + ENTRYPOINT ["docker-compose"] From 78fe655dbce206e3e6e64ce3e62af94267782487 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Tue, 26 Sep 2017 17:19:29 -0700 Subject: [PATCH 15/28] Revert 3.4-beta temp rename Signed-off-by: Joffrey F --- ...nfig_schema_v3.4-beta.json => config_schema_v3.4.json} | 8 +++++--- compose/const.py | 2 +- docker-compose.spec | 4 ++-- 3 files changed, 8 insertions(+), 6 deletions(-) rename compose/config/{config_schema_v3.4-beta.json => config_schema_v3.4.json} (98%) diff --git a/compose/config/config_schema_v3.4-beta.json b/compose/config/config_schema_v3.4.json similarity index 98% rename from compose/config/config_schema_v3.4-beta.json rename to compose/config/config_schema_v3.4.json index cba063202..dae7d7d23 100644 --- a/compose/config/config_schema_v3.4-beta.json +++ b/compose/config/config_schema_v3.4.json @@ -1,6 +1,7 @@ + { "$schema": "http://json-schema.org/draft-04/schema#", - "id": "config_schema_v3.4-beta.json", + "id": "config_schema_v3.4.json", "type": "object", "required": ["version"], @@ -316,7 +317,7 @@ "additionalProperties": false, "properties": { "disable": {"type": "boolean"}, - "interval": {"type": "string"}, + "interval": {"type": "string", "format": "duration"}, "retries": {"type": "number"}, "test": { "oneOf": [ @@ -324,7 +325,8 @@ {"type": "array", "items": {"type": "string"}} ] }, - "timeout": {"type": "string"} + "timeout": {"type": "string", "format": "duration"}, + "start_period": {"type": "string", "format": "duration"} } }, "deployment": { diff --git a/compose/const.py b/compose/const.py index 809f7c7d4..b5970f82a 100644 --- a/compose/const.py +++ b/compose/const.py @@ -31,7 +31,7 @@ COMPOSEFILE_V3_0 = ComposeVersion('3.0') COMPOSEFILE_V3_1 = ComposeVersion('3.1') COMPOSEFILE_V3_2 = ComposeVersion('3.2') COMPOSEFILE_V3_3 = ComposeVersion('3.3') -COMPOSEFILE_V3_4 = ComposeVersion('3.4-beta') +COMPOSEFILE_V3_4 = ComposeVersion('3.4') API_VERSIONS = { COMPOSEFILE_V1: '1.21', diff --git a/docker-compose.spec b/docker-compose.spec index fe5651f6a..9c46421f0 100644 --- a/docker-compose.spec +++ b/docker-compose.spec @@ -63,8 +63,8 @@ exe = EXE(pyz, 'DATA' ), ( - 'compose/config/config_schema_v3.4-beta.json', - 'compose/config/config_schema_v3.4-beta.json', + 'compose/config/config_schema_v3.4.json', + 'compose/config/config_schema_v3.4.json', 'DATA' ), ( From 21d597c2b4b2c2293bf02a22db6664ae0af7bc62 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Wed, 27 Sep 2017 18:24:46 -0700 Subject: [PATCH 16/28] Avoid import ConfigurationError inside compose.utils (circular import) Signed-off-by: Joffrey F --- compose/config/config.py | 5 ++++- compose/utils.py | 3 +-- tests/unit/utils_test.py | 8 ++++++++ 3 files changed, 13 insertions(+), 3 deletions(-) diff --git a/compose/config/config.py b/compose/config/config.py index 0fddfd3a4..b90ab0305 100644 --- a/compose/config/config.py +++ b/compose/config/config.py @@ -762,7 +762,10 @@ def process_blkio_config(service_dict): for field in ['device_read_bps', 'device_write_bps']: if field in service_dict['blkio_config']: for v in service_dict['blkio_config'].get(field, []): - v['rate'] = parse_bytes(v.get('rate', 0)) + rate = v.get('rate', 0) + v['rate'] = parse_bytes(rate) + if v['rate'] is None: + raise ConfigurationError('Invalid format for bytes value: "{}"'.format(rate)) for field in ['device_read_iops', 'device_write_iops']: if field in service_dict['blkio_config']: diff --git a/compose/utils.py b/compose/utils.py index 1ede4d37d..197ae6eb2 100644 --- a/compose/utils.py +++ b/compose/utils.py @@ -12,7 +12,6 @@ import six from docker.errors import DockerException from docker.utils import parse_bytes as sdk_parse_bytes -from .config.errors import ConfigurationError from .errors import StreamParseError from .timeparse import MULTIPLIERS from .timeparse import timeparse @@ -143,4 +142,4 @@ def parse_bytes(n): try: return sdk_parse_bytes(n) except DockerException: - raise ConfigurationError('Invalid format for bytes value: {}'.format(n)) + return None diff --git a/tests/unit/utils_test.py b/tests/unit/utils_test.py index 85231957e..84becb975 100644 --- a/tests/unit/utils_test.py +++ b/tests/unit/utils_test.py @@ -60,3 +60,11 @@ class TestJsonStream(object): {'three': 'four'}, {'x': 2} ] + + +class TestParseBytes(object): + def test_parse_bytes(self): + assert utils.parse_bytes('123kb') == 123 * 1024 + assert utils.parse_bytes(123) == 123 + assert utils.parse_bytes('foobar') is None + assert utils.parse_bytes('123') == 123 From c7cdd63acf77147fcb3d53112c8c56931d671151 Mon Sep 17 00:00:00 2001 From: Marc van den Hoogen Date: Fri, 18 Aug 2017 13:40:11 +0200 Subject: [PATCH 17/28] Add shm_size to build-options (issue #3866) * Add shm_size to build configuration * Make it possible to enlarge/customize shm size during build * Value in bytes, or use string like "512M" or "1G" ... * Add to compose format 2.3 and (provisionally) >=3.5 format * Add automated test for shm_size in build-opts Signed-off-by: Marc van den Hoogen Made unit tests compatible with previously added shm_size build-option Signed-off-by: Marc van den Hoogen Also support shm_size build-opt when conf override Signed-off-by: Marc van den Hoogen Automated test for shm_size build-option Signed-off-by: Marc van den Hoogen Schema 3.4, add shm_size to schema 2.3, updated const.py Signed-off-by: Marc van den Hoogen Corrected typo in config_schema_v3.4 Signed-off-by: Marc van den Hoogen Add support for g/m/k units for shm_size in build-opts Signed-off-by: Marc van den Hoogen Reorder imports in service.py Signed-off-by: Marc van den Hoogen --- compose/config/config.py | 1 + compose/config/config_schema_v2.3.json | 3 +- compose/config/config_schema_v3.5.json | 542 ++++++++++++++++++ compose/const.py | 3 + compose/service.py | 2 + tests/acceptance/cli_test.py | 6 + tests/fixtures/build-shm-size/Dockerfile | 4 + .../build-shm-size/docker-compose.yml | 7 + tests/unit/service_test.py | 2 + 9 files changed, 569 insertions(+), 1 deletion(-) create mode 100644 compose/config/config_schema_v3.5.json create mode 100644 tests/fixtures/build-shm-size/Dockerfile create mode 100644 tests/fixtures/build-shm-size/docker-compose.yml diff --git a/compose/config/config.py b/compose/config/config.py index b90ab0305..f16dd01b3 100644 --- a/compose/config/config.py +++ b/compose/config/config.py @@ -1020,6 +1020,7 @@ def merge_build(output, base, override): md.merge_scalar('dockerfile') md.merge_scalar('network') md.merge_scalar('target') + md.merge_scalar('shm_size') md.merge_mapping('args', parse_build_arguments) md.merge_field('cache_from', merge_unique_items_lists, default=[]) md.merge_mapping('labels', parse_labels) diff --git a/compose/config/config_schema_v2.3.json b/compose/config/config_schema_v2.3.json index a790bb405..ceaf44954 100644 --- a/compose/config/config_schema_v2.3.json +++ b/compose/config/config_schema_v2.3.json @@ -91,7 +91,8 @@ "labels": {"$ref": "#/definitions/list_or_dict"}, "cache_from": {"$ref": "#/definitions/list_of_strings"}, "network": {"type": "string"}, - "target": {"type": "string"} + "target": {"type": "string"}, + "shm_size": {"type": ["integer", "string"]} }, "additionalProperties": false } diff --git a/compose/config/config_schema_v3.5.json b/compose/config/config_schema_v3.5.json new file mode 100644 index 000000000..fa95d6a24 --- /dev/null +++ b/compose/config/config_schema_v3.5.json @@ -0,0 +1,542 @@ +{ + "$schema": "http://json-schema.org/draft-04/schema#", + "id": "config_schema_v3.5.json", + "type": "object", + "required": ["version"], + + "properties": { + "version": { + "type": "string" + }, + + "services": { + "id": "#/properties/services", + "type": "object", + "patternProperties": { + "^[a-zA-Z0-9._-]+$": { + "$ref": "#/definitions/service" + } + }, + "additionalProperties": false + }, + + "networks": { + "id": "#/properties/networks", + "type": "object", + "patternProperties": { + "^[a-zA-Z0-9._-]+$": { + "$ref": "#/definitions/network" + } + } + }, + + "volumes": { + "id": "#/properties/volumes", + "type": "object", + "patternProperties": { + "^[a-zA-Z0-9._-]+$": { + "$ref": "#/definitions/volume" + } + }, + "additionalProperties": false + }, + + "secrets": { + "id": "#/properties/secrets", + "type": "object", + "patternProperties": { + "^[a-zA-Z0-9._-]+$": { + "$ref": "#/definitions/secret" + } + }, + "additionalProperties": false + }, + + "configs": { + "id": "#/properties/configs", + "type": "object", + "patternProperties": { + "^[a-zA-Z0-9._-]+$": { + "$ref": "#/definitions/config" + } + }, + "additionalProperties": false + } + }, + + "additionalProperties": false, + + "definitions": { + + "service": { + "id": "#/definitions/service", + "type": "object", + + "properties": { + "deploy": {"$ref": "#/definitions/deployment"}, + "build": { + "oneOf": [ + {"type": "string"}, + { + "type": "object", + "properties": { + "context": {"type": "string"}, + "dockerfile": {"type": "string"}, + "args": {"$ref": "#/definitions/list_or_dict"}, + "labels": {"$ref": "#/definitions/list_or_dict"}, + "cache_from": {"$ref": "#/definitions/list_of_strings"}, + "network": {"type": "string"}, + "target": {"type": "string"}, + "shm_size": {"type": ["integer", "string"]} + }, + "additionalProperties": false + } + ] + }, + "cap_add": {"type": "array", "items": {"type": "string"}, "uniqueItems": true}, + "cap_drop": {"type": "array", "items": {"type": "string"}, "uniqueItems": true}, + "cgroup_parent": {"type": "string"}, + "command": { + "oneOf": [ + {"type": "string"}, + {"type": "array", "items": {"type": "string"}} + ] + }, + "configs": { + "type": "array", + "items": { + "oneOf": [ + {"type": "string"}, + { + "type": "object", + "properties": { + "source": {"type": "string"}, + "target": {"type": "string"}, + "uid": {"type": "string"}, + "gid": {"type": "string"}, + "mode": {"type": "number"} + } + } + ] + } + }, + "container_name": {"type": "string"}, + "credential_spec": {"type": "object", "properties": { + "file": {"type": "string"}, + "registry": {"type": "string"} + }}, + "depends_on": {"$ref": "#/definitions/list_of_strings"}, + "devices": {"type": "array", "items": {"type": "string"}, "uniqueItems": true}, + "dns": {"$ref": "#/definitions/string_or_list"}, + "dns_search": {"$ref": "#/definitions/string_or_list"}, + "domainname": {"type": "string"}, + "entrypoint": { + "oneOf": [ + {"type": "string"}, + {"type": "array", "items": {"type": "string"}} + ] + }, + "env_file": {"$ref": "#/definitions/string_or_list"}, + "environment": {"$ref": "#/definitions/list_or_dict"}, + + "expose": { + "type": "array", + "items": { + "type": ["string", "number"], + "format": "expose" + }, + "uniqueItems": true + }, + + "external_links": {"type": "array", "items": {"type": "string"}, "uniqueItems": true}, + "extra_hosts": {"$ref": "#/definitions/list_or_dict"}, + "healthcheck": {"$ref": "#/definitions/healthcheck"}, + "hostname": {"type": "string"}, + "image": {"type": "string"}, + "ipc": {"type": "string"}, + "labels": {"$ref": "#/definitions/list_or_dict"}, + "links": {"type": "array", "items": {"type": "string"}, "uniqueItems": true}, + + "logging": { + "type": "object", + + "properties": { + "driver": {"type": "string"}, + "options": { + "type": "object", + "patternProperties": { + "^.+$": {"type": ["string", "number", "null"]} + } + } + }, + "additionalProperties": false + }, + + "mac_address": {"type": "string"}, + "network_mode": {"type": "string"}, + + "networks": { + "oneOf": [ + {"$ref": "#/definitions/list_of_strings"}, + { + "type": "object", + "patternProperties": { + "^[a-zA-Z0-9._-]+$": { + "oneOf": [ + { + "type": "object", + "properties": { + "aliases": {"$ref": "#/definitions/list_of_strings"}, + "ipv4_address": {"type": "string"}, + "ipv6_address": {"type": "string"} + }, + "additionalProperties": false + }, + {"type": "null"} + ] + } + }, + "additionalProperties": false + } + ] + }, + "pid": {"type": ["string", "null"]}, + + "ports": { + "type": "array", + "items": { + "oneOf": [ + {"type": "number", "format": "ports"}, + {"type": "string", "format": "ports"}, + { + "type": "object", + "properties": { + "mode": {"type": "string"}, + "target": {"type": "integer"}, + "published": {"type": "integer"}, + "protocol": {"type": "string"} + }, + "additionalProperties": false + } + ] + }, + "uniqueItems": true + }, + + "privileged": {"type": "boolean"}, + "read_only": {"type": "boolean"}, + "restart": {"type": "string"}, + "security_opt": {"type": "array", "items": {"type": "string"}, "uniqueItems": true}, + "shm_size": {"type": ["number", "string"]}, + "secrets": { + "type": "array", + "items": { + "oneOf": [ + {"type": "string"}, + { + "type": "object", + "properties": { + "source": {"type": "string"}, + "target": {"type": "string"}, + "uid": {"type": "string"}, + "gid": {"type": "string"}, + "mode": {"type": "number"} + } + } + ] + } + }, + "sysctls": {"$ref": "#/definitions/list_or_dict"}, + "stdin_open": {"type": "boolean"}, + "stop_grace_period": {"type": "string", "format": "duration"}, + "stop_signal": {"type": "string"}, + "tmpfs": {"$ref": "#/definitions/string_or_list"}, + "tty": {"type": "boolean"}, + "ulimits": { + "type": "object", + "patternProperties": { + "^[a-z]+$": { + "oneOf": [ + {"type": "integer"}, + { + "type":"object", + "properties": { + "hard": {"type": "integer"}, + "soft": {"type": "integer"} + }, + "required": ["soft", "hard"], + "additionalProperties": false + } + ] + } + } + }, + "user": {"type": "string"}, + "userns_mode": {"type": "string"}, + "volumes": { + "type": "array", + "items": { + "oneOf": [ + {"type": "string"}, + { + "type": "object", + "required": ["type"], + "properties": { + "type": {"type": "string"}, + "source": {"type": "string"}, + "target": {"type": "string"}, + "read_only": {"type": "boolean"}, + "consistency": {"type": "string"}, + "bind": { + "type": "object", + "properties": { + "propagation": {"type": "string"} + } + }, + "volume": { + "type": "object", + "properties": { + "nocopy": {"type": "boolean"} + } + } + } + } + ], + "uniqueItems": true + } + }, + "working_dir": {"type": "string"} + }, + "additionalProperties": false + }, + + "healthcheck": { + "id": "#/definitions/healthcheck", + "type": "object", + "additionalProperties": false, + "properties": { + "disable": {"type": "boolean"}, + "interval": {"type": "string"}, + "retries": {"type": "number"}, + "test": { + "oneOf": [ + {"type": "string"}, + {"type": "array", "items": {"type": "string"}} + ] + }, + "timeout": {"type": "string"} + } + }, + "deployment": { + "id": "#/definitions/deployment", + "type": ["object", "null"], + "properties": { + "mode": {"type": "string"}, + "endpoint_mode": {"type": "string"}, + "replicas": {"type": "integer"}, + "labels": {"$ref": "#/definitions/list_or_dict"}, + "update_config": { + "type": "object", + "properties": { + "parallelism": {"type": "integer"}, + "delay": {"type": "string", "format": "duration"}, + "failure_action": {"type": "string"}, + "monitor": {"type": "string", "format": "duration"}, + "max_failure_ratio": {"type": "number"}, + "order": {"type": "string", "enum": [ + "start-first", "stop-first" + ]} + }, + "additionalProperties": false + }, + "resources": { + "type": "object", + "properties": { + "limits": {"$ref": "#/definitions/resource"}, + "reservations": {"$ref": "#/definitions/resource"} + }, + "additionalProperties": false + }, + "restart_policy": { + "type": "object", + "properties": { + "condition": {"type": "string"}, + "delay": {"type": "string", "format": "duration"}, + "max_attempts": {"type": "integer"}, + "window": {"type": "string", "format": "duration"} + }, + "additionalProperties": false + }, + "placement": { + "type": "object", + "properties": { + "constraints": {"type": "array", "items": {"type": "string"}}, + "preferences": { + "type": "array", + "items": { + "type": "object", + "properties": { + "spread": {"type": "string"} + }, + "additionalProperties": false + } + } + }, + "additionalProperties": false + } + }, + "additionalProperties": false + }, + + "resource": { + "id": "#/definitions/resource", + "type": "object", + "properties": { + "cpus": {"type": "string"}, + "memory": {"type": "string"} + }, + "additionalProperties": false + }, + + "network": { + "id": "#/definitions/network", + "type": ["object", "null"], + "properties": { + "driver": {"type": "string"}, + "driver_opts": { + "type": "object", + "patternProperties": { + "^.+$": {"type": ["string", "number"]} + } + }, + "ipam": { + "type": "object", + "properties": { + "driver": {"type": "string"}, + "config": { + "type": "array", + "items": { + "type": "object", + "properties": { + "subnet": {"type": "string"} + }, + "additionalProperties": false + } + } + }, + "additionalProperties": false + }, + "external": { + "type": ["boolean", "object"], + "properties": { + "name": {"type": "string"} + }, + "additionalProperties": false + }, + "internal": {"type": "boolean"}, + "attachable": {"type": "boolean"}, + "labels": {"$ref": "#/definitions/list_or_dict"} + }, + "additionalProperties": false + }, + + "volume": { + "id": "#/definitions/volume", + "type": ["object", "null"], + "properties": { + "name": {"type": "string"}, + "driver": {"type": "string"}, + "driver_opts": { + "type": "object", + "patternProperties": { + "^.+$": {"type": ["string", "number"]} + } + }, + "external": { + "type": ["boolean", "object"], + "properties": { + "name": {"type": "string"} + }, + "additionalProperties": false + }, + "labels": {"$ref": "#/definitions/list_or_dict"} + }, + "additionalProperties": false + }, + + "secret": { + "id": "#/definitions/secret", + "type": "object", + "properties": { + "file": {"type": "string"}, + "external": { + "type": ["boolean", "object"], + "properties": { + "name": {"type": "string"} + } + }, + "labels": {"$ref": "#/definitions/list_or_dict"} + }, + "additionalProperties": false + }, + + "config": { + "id": "#/definitions/config", + "type": "object", + "properties": { + "file": {"type": "string"}, + "external": { + "type": ["boolean", "object"], + "properties": { + "name": {"type": "string"} + } + }, + "labels": {"$ref": "#/definitions/list_or_dict"} + }, + "additionalProperties": false + }, + + "string_or_list": { + "oneOf": [ + {"type": "string"}, + {"$ref": "#/definitions/list_of_strings"} + ] + }, + + "list_of_strings": { + "type": "array", + "items": {"type": "string"}, + "uniqueItems": true + }, + + "list_or_dict": { + "oneOf": [ + { + "type": "object", + "patternProperties": { + ".+": { + "type": ["string", "number", "null"] + } + }, + "additionalProperties": false + }, + {"type": "array", "items": {"type": "string"}, "uniqueItems": true} + ] + }, + + "constraints": { + "service": { + "id": "#/definitions/constraints/service", + "anyOf": [ + {"required": ["build"]}, + {"required": ["image"]} + ], + "properties": { + "build": { + "required": ["context"] + } + } + } + } + } +} diff --git a/compose/const.py b/compose/const.py index b5970f82a..2ac08b89a 100644 --- a/compose/const.py +++ b/compose/const.py @@ -32,6 +32,7 @@ COMPOSEFILE_V3_1 = ComposeVersion('3.1') COMPOSEFILE_V3_2 = ComposeVersion('3.2') COMPOSEFILE_V3_3 = ComposeVersion('3.3') COMPOSEFILE_V3_4 = ComposeVersion('3.4') +COMPOSEFILE_V3_5 = ComposeVersion('3.5') API_VERSIONS = { COMPOSEFILE_V1: '1.21', @@ -44,6 +45,7 @@ API_VERSIONS = { COMPOSEFILE_V3_2: '1.25', COMPOSEFILE_V3_3: '1.30', COMPOSEFILE_V3_4: '1.30', + COMPOSEFILE_V3_5: '1.30', } API_VERSION_TO_ENGINE_VERSION = { @@ -57,4 +59,5 @@ API_VERSION_TO_ENGINE_VERSION = { API_VERSIONS[COMPOSEFILE_V3_2]: '1.13.0', API_VERSIONS[COMPOSEFILE_V3_3]: '17.06.0', API_VERSIONS[COMPOSEFILE_V3_4]: '17.06.0', + API_VERSIONS[COMPOSEFILE_V3_5]: '17.06.0', } diff --git a/compose/service.py b/compose/service.py index 2829240f2..28c032763 100644 --- a/compose/service.py +++ b/compose/service.py @@ -43,6 +43,7 @@ from .parallel import parallel_execute from .progress_stream import stream_output from .progress_stream import StreamOutputError from .utils import json_hash +from .utils import parse_bytes from .utils import parse_seconds_float @@ -916,6 +917,7 @@ class Service(object): buildargs=build_args, network_mode=build_opts.get('network', None), target=build_opts.get('target', None), + shmsize=parse_bytes(build_opts.get('shm_size')) if build_opts.get('shm_size') else None, ) try: diff --git a/tests/acceptance/cli_test.py b/tests/acceptance/cli_test.py index 3a5e17ad8..ca4bd9ee7 100644 --- a/tests/acceptance/cli_test.py +++ b/tests/acceptance/cli_test.py @@ -531,6 +531,12 @@ class CLITestCase(DockerClientTestCase): ] assert not containers + def test_build_shm_size_build_option(self): + pull_busybox(self.client) + self.base_dir = 'tests/fixtures/build-shm-size' + result = self.dispatch(['build', '--no-cache'], None) + assert 'shm_size: 96' in result.stdout + def test_bundle_with_digests(self): self.base_dir = 'tests/fixtures/bundle-with-digests/' tmpdir = py.test.ensuretemp('cli_test_bundle') diff --git a/tests/fixtures/build-shm-size/Dockerfile b/tests/fixtures/build-shm-size/Dockerfile new file mode 100644 index 000000000..f91733d63 --- /dev/null +++ b/tests/fixtures/build-shm-size/Dockerfile @@ -0,0 +1,4 @@ +FROM busybox + +# Report the shm_size (through the size of /dev/shm) +RUN echo "shm_size:" $(df -h /dev/shm | tail -n 1 | awk '{print $2}') diff --git a/tests/fixtures/build-shm-size/docker-compose.yml b/tests/fixtures/build-shm-size/docker-compose.yml new file mode 100644 index 000000000..238a51322 --- /dev/null +++ b/tests/fixtures/build-shm-size/docker-compose.yml @@ -0,0 +1,7 @@ +version: '3.5' + +services: + custom_shm_size: + build: + context: . + shm_size: 100663296 # =96M diff --git a/tests/unit/service_test.py b/tests/unit/service_test.py index 0293695ab..43ccf081c 100644 --- a/tests/unit/service_test.py +++ b/tests/unit/service_test.py @@ -475,6 +475,7 @@ class ServiceTest(unittest.TestCase): cache_from=None, network_mode=None, target=None, + shmsize=None, ) def test_ensure_image_exists_no_build(self): @@ -515,6 +516,7 @@ class ServiceTest(unittest.TestCase): cache_from=None, network_mode=None, target=None, + shmsize=None ) def test_build_does_not_pull(self): From 3436145764eb22263dbf9f6b5f2ff6086f767472 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Fri, 13 Oct 2017 15:24:34 -0700 Subject: [PATCH 18/28] Temporary xfails for engine bug Signed-off-by: Joffrey F --- tests/acceptance/cli_test.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/tests/acceptance/cli_test.py b/tests/acceptance/cli_test.py index ca4bd9ee7..b598d99d5 100644 --- a/tests/acceptance/cli_test.py +++ b/tests/acceptance/cli_test.py @@ -14,7 +14,7 @@ from collections import Counter from collections import namedtuple from operator import attrgetter -import py +import pytest import six import yaml from docker import errors @@ -504,6 +504,7 @@ class CLITestCase(DockerClientTestCase): assert BUILD_CACHE_TEXT not in result.stdout assert BUILD_PULL_TEXT in result.stdout + @pytest.mark.xfail(reason='17.10.0 RC bug remove after GA https://github.com/moby/moby/issues/35116') def test_build_failed(self): self.base_dir = 'tests/fixtures/simple-failing-dockerfile' self.dispatch(['build', 'simple'], returncode=1) @@ -517,6 +518,7 @@ class CLITestCase(DockerClientTestCase): ] assert len(containers) == 1 + @pytest.mark.xfail(reason='17.10.0 RC bug remove after GA https://github.com/moby/moby/issues/35116') def test_build_failed_forcerm(self): self.base_dir = 'tests/fixtures/simple-failing-dockerfile' self.dispatch(['build', '--force-rm', 'simple'], returncode=1) @@ -539,7 +541,7 @@ class CLITestCase(DockerClientTestCase): def test_bundle_with_digests(self): self.base_dir = 'tests/fixtures/bundle-with-digests/' - tmpdir = py.test.ensuretemp('cli_test_bundle') + tmpdir = pytest.ensuretemp('cli_test_bundle') self.addCleanup(tmpdir.remove) filename = str(tmpdir.join('example.dab')) @@ -1403,7 +1405,7 @@ class CLITestCase(DockerClientTestCase): [u'/bin/true'], ) - @py.test.mark.skipif(SWARM_SKIP_RM_VOLUMES, reason='Swarm DELETE /containers/ bug') + @pytest.mark.skipif(SWARM_SKIP_RM_VOLUMES, reason='Swarm DELETE /containers/ bug') def test_run_rm(self): self.base_dir = 'tests/fixtures/volume' proc = start_process(self.base_dir, ['run', '--rm', 'test']) From 8c38651196c626c64ff22a22a8d606ed9d88e305 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Wed, 27 Sep 2017 17:10:13 -0700 Subject: [PATCH 19/28] Mount with same container path and different mode should override Signed-off-by: Joffrey F --- compose/config/config.py | 35 ++++++++++++++++++++--------- tests/unit/config/config_test.py | 38 +++++++++++++++++++++++++++++--- 2 files changed, 60 insertions(+), 13 deletions(-) diff --git a/compose/config/config.py b/compose/config/config.py index f16dd01b3..948e2376e 100644 --- a/compose/config/config.py +++ b/compose/config/config.py @@ -1137,24 +1137,30 @@ def resolve_volume_paths(working_dir, service_dict): def resolve_volume_path(working_dir, volume): + mount_params = None if isinstance(volume, dict): - host_path = volume.get('source') container_path = volume.get('target') + host_path = volume.get('source') + mode = None if host_path: if volume.get('read_only'): - container_path += ':ro' + mode = 'ro' if volume.get('volume', {}).get('nocopy'): - container_path += ':nocopy' + mode = 'nocopy' + mount_params = (host_path, mode) else: - container_path, host_path = split_path_mapping(volume) + container_path, mount_params = split_path_mapping(volume) - if host_path is not None: + if mount_params is not None: + host_path, mode = mount_params + if host_path is None: + return container_path if host_path.startswith('.'): host_path = expand_path(working_dir, host_path) host_path = os.path.expanduser(host_path) - return u"{}:{}".format(host_path, container_path) - else: - return container_path + return u"{}:{}{}".format(host_path, container_path, (':' + mode if mode else '')) + + return container_path def normalize_build(service_dict, working_dir, environment): @@ -1234,7 +1240,12 @@ def split_path_mapping(volume_path): if ':' in volume_config: (host, container) = volume_config.split(':', 1) - return (container, drive + host) + container_drive, container_path = splitdrive(container) + mode = None + if ':' in container_path: + container_path, mode = container_path.rsplit(':', 1) + + return (container_drive + container_path, (drive + host, mode)) else: return (volume_path, None) @@ -1246,7 +1257,11 @@ def join_path_mapping(pair): elif host is None: return container else: - return ":".join((host, container)) + host, mode = host + result = ":".join((host, container)) + if mode: + result += ":" + mode + return result def expand_path(working_dir, path): diff --git a/tests/unit/config/config_test.py b/tests/unit/config/config_test.py index de9a61302..c5e40130d 100644 --- a/tests/unit/config/config_test.py +++ b/tests/unit/config/config_test.py @@ -1101,6 +1101,38 @@ class ConfigTest(unittest.TestCase): ['/anonymous', '/c:/b:rw', 'vol:/x:ro'] ) + @mock.patch.dict(os.environ) + def test_volume_mode_override(self): + os.environ['COMPOSE_CONVERT_WINDOWS_PATHS'] = 'true' + base_file = config.ConfigFile( + 'base.yaml', + { + 'version': '2.3', + 'services': { + 'web': { + 'image': 'example/web', + 'volumes': ['/c:/b:rw'] + } + }, + } + ) + + override_file = config.ConfigFile( + 'override.yaml', + { + 'version': '2.3', + 'services': { + 'web': { + 'volumes': ['/c:/b:ro'] + } + } + } + ) + details = config.ConfigDetails('.', [base_file, override_file]) + service_dicts = config.load(details).services + svc_volumes = list(map(lambda v: v.repr(), service_dicts[0]['volumes'])) + assert svc_volumes == ['/c:/b:ro'] + def test_undeclared_volume_v2(self): base_file = config.ConfigFile( 'base.yaml', @@ -4018,7 +4050,7 @@ class VolumePathTest(unittest.TestCase): def test_split_path_mapping_with_windows_path(self): host_path = "c:\\Users\\msamblanet\\Documents\\anvil\\connect\\config" windows_volume_path = host_path + ":/opt/connect/config:ro" - expected_mapping = ("/opt/connect/config:ro", host_path) + expected_mapping = ("/opt/connect/config", (host_path, 'ro')) mapping = config.split_path_mapping(windows_volume_path) assert mapping == expected_mapping @@ -4026,7 +4058,7 @@ class VolumePathTest(unittest.TestCase): def test_split_path_mapping_with_windows_path_in_container(self): host_path = 'c:\\Users\\remilia\\data' container_path = 'c:\\scarletdevil\\data' - expected_mapping = (container_path, host_path) + expected_mapping = (container_path, (host_path, None)) mapping = config.split_path_mapping('{0}:{1}'.format(host_path, container_path)) assert mapping == expected_mapping @@ -4034,7 +4066,7 @@ class VolumePathTest(unittest.TestCase): def test_split_path_mapping_with_root_mount(self): host_path = '/' container_path = '/var/hostroot' - expected_mapping = (container_path, host_path) + expected_mapping = (container_path, (host_path, None)) mapping = config.split_path_mapping('{0}:{1}'.format(host_path, container_path)) assert mapping == expected_mapping From 18df4915f21fb43100b24f181e851a3f0a14eb9a Mon Sep 17 00:00:00 2001 From: Andrea Giardini Date: Wed, 20 Sep 2017 23:05:29 +0200 Subject: [PATCH 20/28] Fix secret location with absolute paths Signed-off-by: Andrea Giardini --- compose/service.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/compose/service.py b/compose/service.py index 28c032763..aecafc8ca 100644 --- a/compose/service.py +++ b/compose/service.py @@ -881,9 +881,12 @@ class Service(object): def get_secret_volumes(self): def build_spec(secret): - target = '{}/{}'.format( - const.SECRETS_PATH, - secret['secret'].target or secret['secret'].source) + if secret['secret'].target is not None and secret['secret'].target.startswith('/'): + target = secret['secret'].target + else: + target = '{}/{}'.format( + const.SECRETS_PATH, + secret['secret'].target or secret['secret'].source) return VolumeSpec(secret['file'], target, 'ro') return [build_spec(secret) for secret in self.secrets] From c4a8cb30ffba475169b01b8d0b653dc07fa5ab60 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Fri, 13 Oct 2017 17:02:01 -0700 Subject: [PATCH 21/28] Add get_secret_volumes unit tests Signed-off-by: Joffrey F --- compose/service.py | 12 ++++----- tests/unit/service_test.py | 55 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 61 insertions(+), 6 deletions(-) diff --git a/compose/service.py b/compose/service.py index aecafc8ca..1a18c6654 100644 --- a/compose/service.py +++ b/compose/service.py @@ -881,12 +881,12 @@ class Service(object): def get_secret_volumes(self): def build_spec(secret): - if secret['secret'].target is not None and secret['secret'].target.startswith('/'): - target = secret['secret'].target - else: - target = '{}/{}'.format( - const.SECRETS_PATH, - secret['secret'].target or secret['secret'].source) + target = secret['secret'].target + if target is None: + target = '{}/{}'.format(const.SECRETS_PATH, secret['secret'].source) + elif not os.path.isabs(target): + target = '{}/{}'.format(const.SECRETS_PATH, target) + return VolumeSpec(secret['file'], target, 'ro') return [build_spec(secret) for secret in self.secrets] diff --git a/tests/unit/service_test.py b/tests/unit/service_test.py index 43ccf081c..7d61807ba 100644 --- a/tests/unit/service_test.py +++ b/tests/unit/service_test.py @@ -9,12 +9,14 @@ from .. import mock from .. import unittest from compose.config.errors import DependencyError from compose.config.types import ServicePort +from compose.config.types import ServiceSecret from compose.config.types import VolumeFromSpec from compose.config.types import VolumeSpec from compose.const import LABEL_CONFIG_HASH from compose.const import LABEL_ONE_OFF from compose.const import LABEL_PROJECT from compose.const import LABEL_SERVICE +from compose.const import SECRETS_PATH from compose.container import Container from compose.project import OneOffFilter from compose.service import build_ulimits @@ -1089,3 +1091,56 @@ class ServiceVolumesTest(unittest.TestCase): self.assertEqual( self.mock_client.create_host_config.call_args[1]['binds'], [volume]) + + +class ServiceSecretTest(unittest.TestCase): + def setUp(self): + self.mock_client = mock.create_autospec(docker.APIClient) + + def test_get_secret_volumes(self): + secret1 = { + 'secret': ServiceSecret.parse({'source': 'secret1', 'target': 'b.txt'}), + 'file': 'a.txt' + } + service = Service( + 'web', + client=self.mock_client, + image='busybox', + secrets=[secret1] + ) + volumes = service.get_secret_volumes() + + assert volumes[0].external == secret1['file'] + assert volumes[0].internal == '{}/{}'.format(SECRETS_PATH, secret1['secret'].target) + + def test_get_secret_volumes_abspath(self): + secret1 = { + 'secret': ServiceSecret.parse({'source': 'secret1', 'target': '/d.txt'}), + 'file': 'c.txt' + } + service = Service( + 'web', + client=self.mock_client, + image='busybox', + secrets=[secret1] + ) + volumes = service.get_secret_volumes() + + assert volumes[0].external == secret1['file'] + assert volumes[0].internal == secret1['secret'].target + + def test_get_secret_volumes_no_target(self): + secret1 = { + 'secret': ServiceSecret.parse({'source': 'secret1'}), + 'file': 'c.txt' + } + service = Service( + 'web', + client=self.mock_client, + image='busybox', + secrets=[secret1] + ) + volumes = service.get_secret_volumes() + + assert volumes[0].external == secret1['file'] + assert volumes[0].internal == '{}/{}'.format(SECRETS_PATH, secret1['secret'].source) From 0d0da0760c3422e6164ec39c3e8edd77ac6d28d3 Mon Sep 17 00:00:00 2001 From: Guillermo Arribas Date: Fri, 6 Oct 2017 19:12:59 -0300 Subject: [PATCH 22/28] Build labels option: array form produces unmarshal error (fixes #5183) Signed-off-by: Guillermo Arribas --- compose/service.py | 3 ++- tests/integration/service_test.py | 19 ++++++++++++++++++- tests/unit/service_test.py | 4 ++-- 3 files changed, 22 insertions(+), 4 deletions(-) diff --git a/compose/service.py b/compose/service.py index 1a18c6654..e2f72aa5a 100644 --- a/compose/service.py +++ b/compose/service.py @@ -23,6 +23,7 @@ from . import const from . import progress_stream from .config import DOCKER_CONFIG_KEYS from .config import merge_environment +from .config.config import parse_labels from .config.errors import DependencyError from .config.types import ServicePort from .config.types import VolumeSpec @@ -916,7 +917,7 @@ class Service(object): nocache=no_cache, dockerfile=build_opts.get('dockerfile', None), cache_from=build_opts.get('cache_from', None), - labels=build_opts.get('labels', None), + labels=parse_labels(build_opts.get('labels', None)), buildargs=build_args, network_mode=build_opts.get('network', None), target=build_opts.get('target', None), diff --git a/tests/integration/service_test.py b/tests/integration/service_test.py index 84b54fe41..a71bc407c 100644 --- a/tests/integration/service_test.py +++ b/tests/integration/service_test.py @@ -761,7 +761,7 @@ class ServiceTest(DockerClientTestCase): assert service.image() assert "build_version=2" in service.image()['ContainerConfig']['Cmd'] - def test_build_with_build_labels(self): + def test_build_with_build_labels_dict(self): base_dir = tempfile.mkdtemp() self.addCleanup(shutil.rmtree, base_dir) @@ -778,6 +778,23 @@ class ServiceTest(DockerClientTestCase): assert service.image() assert service.image()['Config']['Labels']['com.docker.compose.test'] == 'true' + def test_build_with_build_labels_list(self): + base_dir = tempfile.mkdtemp() + self.addCleanup(shutil.rmtree, base_dir) + + with open(os.path.join(base_dir, 'Dockerfile'), 'w') as f: + f.write('FROM busybox\n') + + service = self.create_service('buildlabels', build={ + 'context': text_type(base_dir), + 'labels': ['com.docker.compose.test=true'] + }) + service.build() + self.addCleanup(self.client.remove_image, service.image_name) + + assert service.image() + assert service.image()['Config']['Labels']['com.docker.compose.test'] == 'true' + @no_cluster('Container networks not on Swarm') def test_build_with_network(self): base_dir = tempfile.mkdtemp() diff --git a/tests/unit/service_test.py b/tests/unit/service_test.py index 7d61807ba..5c5c2bf67 100644 --- a/tests/unit/service_test.py +++ b/tests/unit/service_test.py @@ -473,7 +473,7 @@ class ServiceTest(unittest.TestCase): nocache=False, rm=True, buildargs={}, - labels=None, + labels={}, cache_from=None, network_mode=None, target=None, @@ -514,7 +514,7 @@ class ServiceTest(unittest.TestCase): nocache=False, rm=True, buildargs={}, - labels=None, + labels={}, cache_from=None, network_mode=None, target=None, From f680d46d9af868163d3a4887678d3531e88001e0 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Mon, 16 Oct 2017 11:43:06 -0700 Subject: [PATCH 23/28] Move build labels parsing to config module Signed-off-by: Joffrey F --- compose/config/config.py | 12 ++++++------ compose/service.py | 3 +-- tests/integration/service_test.py | 19 +------------------ tests/unit/config/config_test.py | 24 +++++++++++++++++++++++- tests/unit/service_test.py | 4 ++-- 5 files changed, 33 insertions(+), 29 deletions(-) diff --git a/compose/config/config.py b/compose/config/config.py index 948e2376e..68b2be3a6 100644 --- a/compose/config/config.py +++ b/compose/config/config.py @@ -707,16 +707,16 @@ def process_service(service_config): if 'build' in service_dict: if isinstance(service_dict['build'], six.string_types): service_dict['build'] = resolve_build_path(working_dir, service_dict['build']) - elif isinstance(service_dict['build'], dict) and 'context' in service_dict['build']: - path = service_dict['build']['context'] - service_dict['build']['context'] = resolve_build_path(working_dir, path) + elif isinstance(service_dict['build'], dict): + if 'context' in service_dict['build']: + path = service_dict['build']['context'] + service_dict['build']['context'] = resolve_build_path(working_dir, path) + if 'labels' in service_dict['build']: + service_dict['build']['labels'] = parse_labels(service_dict['build']['labels']) if 'volumes' in service_dict and service_dict.get('volume_driver') is None: service_dict['volumes'] = resolve_volume_paths(working_dir, service_dict) - if 'labels' in service_dict: - service_dict['labels'] = parse_labels(service_dict['labels']) - if 'sysctls' in service_dict: service_dict['sysctls'] = build_string_dict(parse_sysctls(service_dict['sysctls'])) diff --git a/compose/service.py b/compose/service.py index e2f72aa5a..1a18c6654 100644 --- a/compose/service.py +++ b/compose/service.py @@ -23,7 +23,6 @@ from . import const from . import progress_stream from .config import DOCKER_CONFIG_KEYS from .config import merge_environment -from .config.config import parse_labels from .config.errors import DependencyError from .config.types import ServicePort from .config.types import VolumeSpec @@ -917,7 +916,7 @@ class Service(object): nocache=no_cache, dockerfile=build_opts.get('dockerfile', None), cache_from=build_opts.get('cache_from', None), - labels=parse_labels(build_opts.get('labels', None)), + labels=build_opts.get('labels', None), buildargs=build_args, network_mode=build_opts.get('network', None), target=build_opts.get('target', None), diff --git a/tests/integration/service_test.py b/tests/integration/service_test.py index a71bc407c..84b54fe41 100644 --- a/tests/integration/service_test.py +++ b/tests/integration/service_test.py @@ -761,7 +761,7 @@ class ServiceTest(DockerClientTestCase): assert service.image() assert "build_version=2" in service.image()['ContainerConfig']['Cmd'] - def test_build_with_build_labels_dict(self): + def test_build_with_build_labels(self): base_dir = tempfile.mkdtemp() self.addCleanup(shutil.rmtree, base_dir) @@ -778,23 +778,6 @@ class ServiceTest(DockerClientTestCase): assert service.image() assert service.image()['Config']['Labels']['com.docker.compose.test'] == 'true' - def test_build_with_build_labels_list(self): - base_dir = tempfile.mkdtemp() - self.addCleanup(shutil.rmtree, base_dir) - - with open(os.path.join(base_dir, 'Dockerfile'), 'w') as f: - f.write('FROM busybox\n') - - service = self.create_service('buildlabels', build={ - 'context': text_type(base_dir), - 'labels': ['com.docker.compose.test=true'] - }) - service.build() - self.addCleanup(self.client.remove_image, service.image_name) - - assert service.image() - assert service.image()['Config']['Labels']['com.docker.compose.test'] == 'true' - @no_cluster('Container networks not on Swarm') def test_build_with_network(self): base_dir = tempfile.mkdtemp() diff --git a/tests/unit/config/config_test.py b/tests/unit/config/config_test.py index c5e40130d..8f2266ed8 100644 --- a/tests/unit/config/config_test.py +++ b/tests/unit/config/config_test.py @@ -892,7 +892,7 @@ class ConfigTest(unittest.TestCase): assert service['build']['args']['opt1'] == '42' assert service['build']['args']['opt2'] == 'foobar' - def test_load_with_build_labels(self): + def test_load_build_labels_dict(self): service = config.load( build_config_details( { @@ -919,6 +919,28 @@ class ConfigTest(unittest.TestCase): assert service['build']['labels']['label1'] == 42 assert service['build']['labels']['label2'] == 'foobar' + def test_load_build_labels_list(self): + base_file = config.ConfigFile( + 'base.yml', + { + 'version': '2.3', + 'services': { + 'web': { + 'build': { + 'context': '.', + 'labels': ['foo=bar', 'baz=true', 'foobar=1'] + }, + }, + }, + } + ) + + details = config.ConfigDetails('.', [base_file]) + service = config.load(details).services[0] + assert service['build']['labels'] == { + 'foo': 'bar', 'baz': 'true', 'foobar': '1' + } + def test_build_args_allow_empty_properties(self): service = config.load( build_config_details( diff --git a/tests/unit/service_test.py b/tests/unit/service_test.py index 5c5c2bf67..7d61807ba 100644 --- a/tests/unit/service_test.py +++ b/tests/unit/service_test.py @@ -473,7 +473,7 @@ class ServiceTest(unittest.TestCase): nocache=False, rm=True, buildargs={}, - labels={}, + labels=None, cache_from=None, network_mode=None, target=None, @@ -514,7 +514,7 @@ class ServiceTest(unittest.TestCase): nocache=False, rm=True, buildargs={}, - labels={}, + labels=None, cache_from=None, network_mode=None, target=None, From 0ec77bf7d58f6a273cb8ae0b65a26f15e13bd6e0 Mon Sep 17 00:00:00 2001 From: Guillermo Arribas Date: Wed, 11 Oct 2017 13:56:15 -0300 Subject: [PATCH 24/28] Config command generates invalid volumes (fixes #5176) Signed-off-by: Guillermo Arribas --- compose/config/config.py | 19 +++---- compose/config/serialize.py | 4 +- tests/acceptance/cli_test.py | 54 +++++++++++++++++-- .../volumes/external-volumes-v2-x.yml | 17 ++++++ ...al-volumes.yml => external-volumes-v2.yml} | 2 +- .../volumes/external-volumes-v3-4.yml | 17 ++++++ .../volumes/external-volumes-v3-x.yml | 16 ++++++ 7 files changed, 114 insertions(+), 15 deletions(-) create mode 100644 tests/fixtures/volumes/external-volumes-v2-x.yml rename tests/fixtures/volumes/{external-volumes.yml => external-volumes-v2.yml} (92%) create mode 100644 tests/fixtures/volumes/external-volumes-v3-4.yml create mode 100644 tests/fixtures/volumes/external-volumes-v3-x.yml diff --git a/compose/config/config.py b/compose/config/config.py index 68b2be3a6..7bb57076e 100644 --- a/compose/config/config.py +++ b/compose/config/config.py @@ -15,6 +15,9 @@ from cached_property import cached_property from . import types from .. import const from ..const import COMPOSEFILE_V1 as V1 +from ..const import COMPOSEFILE_V2_1 as V2_1 +from ..const import COMPOSEFILE_V3_0 as V3_0 +from ..const import COMPOSEFILE_V3_4 as V3_4 from ..utils import build_string_dict from ..utils import parse_bytes from ..utils import parse_nanoseconds_int @@ -405,7 +408,7 @@ def load_mapping(config_files, get_func, entity_type, working_dir=None): external = config.get('external') if external: name_field = 'name' if entity_type == 'Volume' else 'external_name' - validate_external(entity_type, name, config) + validate_external(entity_type, name, config, config_file.version) if isinstance(external, dict): config[name_field] = external.get('name') elif not config.get('name'): @@ -425,14 +428,12 @@ def load_mapping(config_files, get_func, entity_type, working_dir=None): return mapping -def validate_external(entity_type, name, config): - if len(config.keys()) <= 1: - return - - raise ConfigurationError( - "{} {} declared as external but specifies additional attributes " - "({}).".format( - entity_type, name, ', '.join(k for k in config if k != 'external'))) +def validate_external(entity_type, name, config, version): + if (version < V2_1 or (version >= V3_0 and version < V3_4)) and len(config.keys()) > 1: + raise ConfigurationError( + "{} {} declared as external but specifies additional attributes " + "({}).".format( + entity_type, name, ', '.join(k for k in config if k != 'external'))) def load_services(config_details, config_file): diff --git a/compose/config/serialize.py b/compose/config/serialize.py index 606dd7614..2b8c73f14 100644 --- a/compose/config/serialize.py +++ b/compose/config/serialize.py @@ -9,7 +9,7 @@ from compose.const import COMPOSEFILE_V1 as V1 from compose.const import COMPOSEFILE_V2_1 as V2_1 from compose.const import COMPOSEFILE_V3_0 as V3_0 from compose.const import COMPOSEFILE_V3_2 as V3_2 -from compose.const import COMPOSEFILE_V3_2 as V3_4 +from compose.const import COMPOSEFILE_V3_4 as V3_4 def serialize_config_type(dumper, data): @@ -67,7 +67,7 @@ def denormalize_config(config, image_digests=None): del conf['external_name'] if 'name' in conf: - if config.version < V2_1 or (config.version > V3_0 and config.version < V3_4): + if config.version < V2_1 or (config.version >= V3_0 and config.version < V3_4): del conf['name'] elif 'external' in conf: conf['external'] = True diff --git a/tests/acceptance/cli_test.py b/tests/acceptance/cli_test.py index b598d99d5..43cc89e36 100644 --- a/tests/acceptance/cli_test.py +++ b/tests/acceptance/cli_test.py @@ -285,15 +285,63 @@ class CLITestCase(DockerClientTestCase): } } - def test_config_external_volume(self): + def test_config_external_volume_v2(self): self.base_dir = 'tests/fixtures/volumes' - result = self.dispatch(['-f', 'external-volumes.yml', 'config']) + result = self.dispatch(['-f', 'external-volumes-v2.yml', 'config']) json_result = yaml.load(result.stdout) assert 'volumes' in json_result assert json_result['volumes'] == { 'foo': { 'external': True, - 'name': 'foo', + }, + 'bar': { + 'external': { + 'name': 'some_bar', + }, + } + } + + def test_config_external_volume_v2_x(self): + self.base_dir = 'tests/fixtures/volumes' + result = self.dispatch(['-f', 'external-volumes-v2-x.yml', 'config']) + json_result = yaml.load(result.stdout) + assert 'volumes' in json_result + assert json_result['volumes'] == { + 'foo': { + 'external': True, + 'name': 'some_foo', + }, + 'bar': { + 'external': True, + 'name': 'some_bar', + } + } + + def test_config_external_volume_v3_x(self): + self.base_dir = 'tests/fixtures/volumes' + result = self.dispatch(['-f', 'external-volumes-v3-x.yml', 'config']) + json_result = yaml.load(result.stdout) + assert 'volumes' in json_result + assert json_result['volumes'] == { + 'foo': { + 'external': True, + }, + 'bar': { + 'external': { + 'name': 'some_bar', + }, + } + } + + def test_config_external_volume_v3_4(self): + self.base_dir = 'tests/fixtures/volumes' + result = self.dispatch(['-f', 'external-volumes-v3-4.yml', 'config']) + json_result = yaml.load(result.stdout) + assert 'volumes' in json_result + assert json_result['volumes'] == { + 'foo': { + 'external': True, + 'name': 'some_foo', }, 'bar': { 'external': True, diff --git a/tests/fixtures/volumes/external-volumes-v2-x.yml b/tests/fixtures/volumes/external-volumes-v2-x.yml new file mode 100644 index 000000000..3b736c5f4 --- /dev/null +++ b/tests/fixtures/volumes/external-volumes-v2-x.yml @@ -0,0 +1,17 @@ +version: "2.1" + +services: + web: + image: busybox + command: top + volumes: + - foo:/var/lib/ + - bar:/etc/ + +volumes: + foo: + external: true + name: some_foo + bar: + external: + name: some_bar diff --git a/tests/fixtures/volumes/external-volumes.yml b/tests/fixtures/volumes/external-volumes-v2.yml similarity index 92% rename from tests/fixtures/volumes/external-volumes.yml rename to tests/fixtures/volumes/external-volumes-v2.yml index 05c6c4844..4025b53b1 100644 --- a/tests/fixtures/volumes/external-volumes.yml +++ b/tests/fixtures/volumes/external-volumes-v2.yml @@ -1,4 +1,4 @@ -version: "2.1" +version: "2" services: web: diff --git a/tests/fixtures/volumes/external-volumes-v3-4.yml b/tests/fixtures/volumes/external-volumes-v3-4.yml new file mode 100644 index 000000000..76c8421dc --- /dev/null +++ b/tests/fixtures/volumes/external-volumes-v3-4.yml @@ -0,0 +1,17 @@ +version: "3.4" + +services: + web: + image: busybox + command: top + volumes: + - foo:/var/lib/ + - bar:/etc/ + +volumes: + foo: + external: true + name: some_foo + bar: + external: + name: some_bar diff --git a/tests/fixtures/volumes/external-volumes-v3-x.yml b/tests/fixtures/volumes/external-volumes-v3-x.yml new file mode 100644 index 000000000..903fee647 --- /dev/null +++ b/tests/fixtures/volumes/external-volumes-v3-x.yml @@ -0,0 +1,16 @@ +version: "3.0" + +services: + web: + image: busybox + command: top + volumes: + - foo:/var/lib/ + - bar:/etc/ + +volumes: + foo: + external: true + bar: + external: + name: some_bar From d8194cf6f0c5fd5a08f1a0daa6b4022a5582d008 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Mon, 16 Oct 2017 12:41:29 -0700 Subject: [PATCH 25/28] Add specific handling for pywintypes.error Signed-off-by: Joffrey F --- compose/cli/errors.py | 20 ++++++++++++++++++++ tests/unit/cli/errors_test.py | 21 +++++++++++++++++++++ 2 files changed, 41 insertions(+) diff --git a/compose/cli/errors.py b/compose/cli/errors.py index 23e065c99..1506aa660 100644 --- a/compose/cli/errors.py +++ b/compose/cli/errors.py @@ -57,6 +57,26 @@ def handle_connection_errors(client): except (ReadTimeout, socket.timeout) as e: log_timeout_error(client.timeout) raise ConnectionError() + except Exception as e: + if is_windows(): + import pywintypes + if isinstance(e, pywintypes.error): + log_windows_pipe_error(e) + raise ConnectionError() + raise + + +def log_windows_pipe_error(exc): + if exc.winerror == 232: # https://github.com/docker/compose/issues/5005 + log.error( + "The current Compose file version is not compatible with your engine version. " + "Please upgrade your Compose file to a more recent version, or set " + "a COMPOSE_API_VERSION in your environment." + ) + else: + log.error( + "Windows named pipe error: {} (code: {})".format(exc.strerror, exc.winerror) + ) def log_timeout_error(timeout): diff --git a/tests/unit/cli/errors_test.py b/tests/unit/cli/errors_test.py index 7406a8880..68326d1c7 100644 --- a/tests/unit/cli/errors_test.py +++ b/tests/unit/cli/errors_test.py @@ -7,6 +7,7 @@ from requests.exceptions import ConnectionError from compose.cli import errors from compose.cli.errors import handle_connection_errors +from compose.const import IS_WINDOWS_PLATFORM from tests import mock @@ -65,3 +66,23 @@ class TestHandleConnectionErrors(object): raise APIError(None, None, msg) mock_logging.error.assert_called_once_with(msg) + + @pytest.mark.skipif(not IS_WINDOWS_PLATFORM, reason='Needs pywin32') + def test_windows_pipe_error_no_data(self, mock_logging): + import pywintypes + with pytest.raises(errors.ConnectionError): + with handle_connection_errors(mock.Mock(api_version='1.22')): + raise pywintypes.error(232, 'WriteFile', 'The pipe is being closed.') + + _, args, _ = mock_logging.error.mock_calls[0] + assert "The current Compose file version is not compatible with your engine version." in args[0] + + @pytest.mark.skipif(not IS_WINDOWS_PLATFORM, reason='Needs pywin32') + def test_windows_pipe_error_misc(self, mock_logging): + import pywintypes + with pytest.raises(errors.ConnectionError): + with handle_connection_errors(mock.Mock(api_version='1.22')): + raise pywintypes.error(231, 'WriteFile', 'The pipe is busy.') + + _, args, _ = mock_logging.error.mock_calls[0] + assert "Windows named pipe error: The pipe is busy. (code: 231)" == args[0] From 395dce9d2c40632242cead5bffe3b7d55e2fb2a2 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Mon, 16 Oct 2017 13:54:30 -0700 Subject: [PATCH 26/28] Add check_duplicate=True when creating network Signed-off-by: Joffrey F --- compose/network.py | 1 + 1 file changed, 1 insertion(+) diff --git a/compose/network.py b/compose/network.py index 0f42eb20a..2e0a7e6ec 100644 --- a/compose/network.py +++ b/compose/network.py @@ -79,6 +79,7 @@ class Network(object): enable_ipv6=self.enable_ipv6, labels=self._labels, attachable=version_gte(self.client._version, '1.24') or None, + check_duplicate=True, ) def remove(self): From 13c5049dbccf63182bf5d963bf25c817b25b8a66 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Tue, 17 Oct 2017 13:36:06 -0700 Subject: [PATCH 27/28] flake8 Signed-off-by: Joffrey F --- tests/acceptance/cli_test.py | 4 ---- tests/integration/testcases.py | 2 -- 2 files changed, 6 deletions(-) diff --git a/tests/acceptance/cli_test.py b/tests/acceptance/cli_test.py index 43cc89e36..8ba43b00f 100644 --- a/tests/acceptance/cli_test.py +++ b/tests/acceptance/cli_test.py @@ -493,10 +493,6 @@ class CLITestCase(DockerClientTestCase): 'image library/nonexisting-image:latest not found' in result.stderr or 'pull access denied for nonexisting-image' in result.stderr) - def test_pull_with_quiet(self): - assert self.dispatch(['pull', '--quiet']).stderr == '' - assert self.dispatch(['pull', '--quiet']).stdout == '' - def test_pull_with_parallel_failure(self): result = self.dispatch([ '-f', 'ignore-pull-failures.yml', 'pull', '--parallel'], diff --git a/tests/integration/testcases.py b/tests/integration/testcases.py index 8435f97dd..b72fb53a8 100644 --- a/tests/integration/testcases.py +++ b/tests/integration/testcases.py @@ -75,7 +75,6 @@ def v2_1_only(): return min_version_skip(V2_1) - def v2_2_only(): return min_version_skip(V2_2) @@ -84,7 +83,6 @@ def v2_3_only(): return min_version_skip(V2_3) - def v3_only(): return min_version_skip(V3_0) From 2a0dd1401fb832c377a70527d760f34754523f2a Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Mon, 16 Oct 2017 16:50:51 -0700 Subject: [PATCH 28/28] Bump 1.17.0-rc1 Signed-off-by: Joffrey F --- CHANGELOG.md | 61 +++++++++++++++++++++++++++++++++++++++++++-- compose/__init__.py | 2 +- script/run/run.sh | 2 +- 3 files changed, 61 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 558376855..cff19d879 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,64 @@ Change log ========== +1.17.0 (2017-11-03) +------------------- + +### New features + +#### Compose file version 3.4 + +- Introduced version 3.4 of the `docker-compose.yml` specification. + This version requires to be used with Docker Engine 17.06.0 or above. + +- Added support for `cache_from`, `network` and `target` options in build + configurations + +- Added support for the `order` parameter in the `update_config` section + +- Added support for setting a custom name in volume definitions using + the `name` parameter + +#### Compose file version 2.3 + +- Added support for `shm_size` option in build configuration + +#### Compose file version 2.x + +- Added support for extension fields (`x-*`). Also available for v3.4 files + +#### All formats + +- Added new `--no-start` to the `up` command, allowing users to create all + resources (networks, volumes, containers) without starting services. + The `create` command is deprecated in favor of this new option + +### Bugfixes + +- Fixed a bug where `extra_hosts` values would be overridden by extension + files instead of merging together + +- Fixed a bug where the validation for v3.2 files would prevent using the + `consistency` field in service volume definitions + +- Fixed a bug that would cause a crash when configuration fields expecting + unique items would contain duplicates + +- Fixed a bug where mount overrides with a different mode would create a + duplicate entry instead of overriding the original entry + +- Fixed a bug where build labels declared as a list wouldn't be properly + parsed + +- Fixed a bug where the output of `docker-compose config` would be invalid + for some versions if the file contained custom-named external volumes + +- Improved error handling when issuing a build command on Windows using an + unsupported file version + +- Fixed an issue where networks with identical names would sometimes be + created when running `up` commands concurrently. + 1.16.1 (2017-09-01) ------------------- @@ -8,7 +66,6 @@ Change log - Fixed bug that prevented using `extra_hosts` in several configuration files. - 1.16.0 (2017-08-31) ------------------- @@ -19,7 +76,7 @@ Change log - Introduced version 2.3 of the `docker-compose.yml` specification. This version requires to be used with Docker Engine 17.06.0 or above. -- Added support for the `target` parameter in network configurations +- Added support for the `target` parameter in build configurations - Added support for the `start_period` parameter in healthcheck configurations diff --git a/compose/__init__.py b/compose/__init__.py index 2e41ca896..86542ec44 100644 --- a/compose/__init__.py +++ b/compose/__init__.py @@ -1,4 +1,4 @@ from __future__ import absolute_import from __future__ import unicode_literals -__version__ = '1.16.1' +__version__ = '1.17.0-rc1' diff --git a/script/run/run.sh b/script/run/run.sh index f1754d05a..498226288 100755 --- a/script/run/run.sh +++ b/script/run/run.sh @@ -15,7 +15,7 @@ set -e -VERSION="1.16.1" +VERSION="1.17.0-rc1" IMAGE="docker/compose:$VERSION"