From 62aeb767d393bacc047b5a46bcab0eae524f53dc Mon Sep 17 00:00:00 2001 From: Antony MECHIN Date: Mon, 24 Sep 2018 15:55:55 +0200 Subject: [PATCH 01/11] tests.unit.config: Make make_service_dict working dir argument optional. Signed-off-by: Antony MECHIN --- tests/unit/config/config_test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/unit/config/config_test.py b/tests/unit/config/config_test.py index 1d42c10d5..c054c388e 100644 --- a/tests/unit/config/config_test.py +++ b/tests/unit/config/config_test.py @@ -42,7 +42,7 @@ from tests import unittest DEFAULT_VERSION = V2_0 -def make_service_dict(name, service_dict, working_dir, filename=None): +def make_service_dict(name, service_dict, working_dir='.', filename=None): """Test helper function to construct a ServiceExtendsResolver """ resolver = config.ServiceExtendsResolver( From bb87a3d040b515f32ae349986c131697aa09ca29 Mon Sep 17 00:00:00 2001 From: Antony MECHIN Date: Mon, 24 Sep 2018 15:59:02 +0200 Subject: [PATCH 02/11] tests.unit.config: Make sure volume order is preserved. Signed-off-by: Antony MECHIN --- tests/unit/config/config_test.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/tests/unit/config/config_test.py b/tests/unit/config/config_test.py index c054c388e..52c89a9e0 100644 --- a/tests/unit/config/config_test.py +++ b/tests/unit/config/config_test.py @@ -8,6 +8,7 @@ import os import shutil import tempfile from operator import itemgetter +from random import shuffle import py import pytest @@ -3536,6 +3537,13 @@ class VolumeConfigTest(unittest.TestCase): ).services[0] assert d['volumes'] == [VolumeSpec.parse('/host/path:/container/path')] + @pytest.mark.skipif(IS_WINDOWS_PLATFORM, reason='posix paths') + def test_volumes_order_is_preserved(self): + volumes = ['/{0}:/{0}'.format(i) for i in range(0, 6)] + shuffle(volumes) + cfg = make_service_dict('foo', {'build': '.', 'volumes': volumes}) + assert cfg['volumes'] == volumes + @pytest.mark.skipif(IS_WINDOWS_PLATFORM, reason='posix paths') @mock.patch.dict(os.environ) def test_volume_binding_with_home(self): From 18c2d08011fbf7f97d574f10f84ef68717d68ad8 Mon Sep 17 00:00:00 2001 From: Antony MECHIN Date: Mon, 24 Sep 2018 18:08:17 +0200 Subject: [PATCH 03/11] utils: Add unique_everseen (from itertools recipies). Signed-off-by: Antony MECHIN --- compose/utils.py | 9 +++++++++ tests/unit/utils_test.py | 6 ++++++ 2 files changed, 15 insertions(+) diff --git a/compose/utils.py b/compose/utils.py index 8f0b3e549..b9b6ab9bd 100644 --- a/compose/utils.py +++ b/compose/utils.py @@ -170,3 +170,12 @@ def truncate_id(value): if len(value) > 12: return value[:12] return value + + +def unique_everseen(iterable, key=lambda x: x): + "List unique elements, preserving order. Remember all elements ever seen." + seen = set() + for element in iterable: + if key(element) not in seen: + seen.add(element) + yield element diff --git a/tests/unit/utils_test.py b/tests/unit/utils_test.py index 84becb975..186b6b14e 100644 --- a/tests/unit/utils_test.py +++ b/tests/unit/utils_test.py @@ -68,3 +68,9 @@ class TestParseBytes(object): assert utils.parse_bytes(123) == 123 assert utils.parse_bytes('foobar') is None assert utils.parse_bytes('123') == 123 + + +class TestMoreItertools(object): + def test_unique_everseen(self): + assert list(utils.unique_everseen([2, 1, 2, 1])) == [2, 1] + assert list(utils.unique_everseen([2, 1, 2, 1], hash)) == [2, 1] From d5c314b382ae966e3483fd58f5bca4b1fe7dadfb Mon Sep 17 00:00:00 2001 From: Antony MECHIN Date: Mon, 24 Sep 2018 16:57:49 +0200 Subject: [PATCH 04/11] tests.unity.service: Make sure volumes order is preserved. Signed-off-by: Antony MECHIN --- compose/service.py | 6 ++++-- tests/unit/service_test.py | 17 +++++++++++++++++ 2 files changed, 21 insertions(+), 2 deletions(-) diff --git a/compose/service.py b/compose/service.py index aca24ce17..8df061b9e 100644 --- a/compose/service.py +++ b/compose/service.py @@ -56,6 +56,7 @@ from .utils import json_hash from .utils import parse_bytes from .utils import parse_seconds_float from .utils import truncate_id +from .utils import unique_everseen log = logging.getLogger(__name__) @@ -940,8 +941,9 @@ class Service(object): override_options['mounts'] = override_options.get('mounts') or [] override_options['mounts'].extend([build_mount(v) for v in secret_volumes]) - # Remove possible duplicates (see e.g. https://github.com/docker/compose/issues/5885) - override_options['binds'] = list(set(binds)) + # Remove possible duplicates (see e.g. https://github.com/docker/compose/issues/5885). + # unique_everseen preserves order. (see https://github.com/docker/compose/issues/6091). + override_options['binds'] = list(unique_everseen(binds)) return container_options, override_options def _get_container_host_config(self, override_options, one_off=False): diff --git a/tests/unit/service_test.py b/tests/unit/service_test.py index d5dbcbea6..af1cd1bea 100644 --- a/tests/unit/service_test.py +++ b/tests/unit/service_test.py @@ -1037,6 +1037,23 @@ class ServiceTest(unittest.TestCase): assert len(override_opts['binds']) == 1 assert override_opts['binds'][0] == 'vol:/data:rw' + def test_volumes_order_is_preserved(self): + service = Service('foo', client=self.mock_client) + volumes = [ + VolumeSpec.parse(cfg) for cfg in [ + '/v{0}:/v{0}:rw'.format(i) for i in range(6) + ] + ] + ctnr_opts, override_opts = service._build_container_volume_options( + previous_container=None, + container_options={ + 'volumes': volumes, + 'environment': {}, + }, + override_options={}, + ) + assert override_opts['binds'] == [vol.repr() for vol in volumes] + class TestServiceNetwork(unittest.TestCase): def setUp(self): From b64184e388380a6498c3cd9ab15b5da3059b4421 Mon Sep 17 00:00:00 2001 From: Antony MECHIN Date: Wed, 26 Sep 2018 15:15:59 +0200 Subject: [PATCH 05/11] service: Use OrderedDict to preserve volumes order on versions prior 3.6. Signed-off-by: Antony MECHIN --- compose/service.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/compose/service.py b/compose/service.py index 8df061b9e..3327c77f8 100644 --- a/compose/service.py +++ b/compose/service.py @@ -1429,7 +1429,7 @@ def merge_volume_bindings(volumes, tmpfs, previous_container, mounts): """ affinity = {} - volume_bindings = dict( + volume_bindings = OrderedDict( build_volume_binding(volume) for volume in volumes if volume.external From eb86881af17ac4255acd50745a83e10899591da5 Mon Sep 17 00:00:00 2001 From: Antony MECHIN Date: Thu, 27 Sep 2018 13:58:38 +0200 Subject: [PATCH 06/11] utils: Fix typo in unique_everseen. Signed-off-by: Antony MECHIN --- compose/utils.py | 5 +++-- tests/unit/utils_test.py | 6 ++++-- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/compose/utils.py b/compose/utils.py index b9b6ab9bd..72e6ced17 100644 --- a/compose/utils.py +++ b/compose/utils.py @@ -176,6 +176,7 @@ def unique_everseen(iterable, key=lambda x: x): "List unique elements, preserving order. Remember all elements ever seen." seen = set() for element in iterable: - if key(element) not in seen: - seen.add(element) + unique_key = key(element) + if unique_key not in seen: + seen.add(unique_key) yield element diff --git a/tests/unit/utils_test.py b/tests/unit/utils_test.py index 186b6b14e..21b88d962 100644 --- a/tests/unit/utils_test.py +++ b/tests/unit/utils_test.py @@ -72,5 +72,7 @@ class TestParseBytes(object): class TestMoreItertools(object): def test_unique_everseen(self): - assert list(utils.unique_everseen([2, 1, 2, 1])) == [2, 1] - assert list(utils.unique_everseen([2, 1, 2, 1], hash)) == [2, 1] + unique = utils.unique_everseen + assert list(unique([2, 1, 2, 1])) == [2, 1] + assert list(unique([2, 1, 2, 1], hash)) == [2, 1] + assert list(unique([2, 1, 2, 1], lambda x: 'key_%s' % x)) == [2, 1] From 30c91388f31712b47196dd5bfe6352655df12089 Mon Sep 17 00:00:00 2001 From: Harald Albers Date: Thu, 27 Sep 2018 08:46:37 +0200 Subject: [PATCH 07/11] Fix bash completion for `config --hash` Signed-off-by: Harald Albers --- contrib/completion/bash/docker-compose | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/contrib/completion/bash/docker-compose b/contrib/completion/bash/docker-compose index f4c42362c..395888d34 100644 --- a/contrib/completion/bash/docker-compose +++ b/contrib/completion/bash/docker-compose @@ -136,7 +136,18 @@ _docker_compose_bundle() { _docker_compose_config() { - COMPREPLY=( $( compgen -W "--help --quiet -q --resolve-image-digests --services --volumes --hash" -- "$cur" ) ) + case "$prev" in + --hash) + if [[ $cur == \\* ]] ; then + COMPREPLY=( '\*' ) + else + COMPREPLY=( $(compgen -W "$(__docker_compose_services) \\\* " -- "$cur") ) + fi + return + ;; + esac + + COMPREPLY=( $( compgen -W "--hash --help --quiet -q --resolve-image-digests --services --volumes" -- "$cur" ) ) } From 970f8317c51a53431307dd799c63880de5d2151c Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Thu, 4 Oct 2018 00:48:53 -0700 Subject: [PATCH 08/11] Fix twine upload for RC versions Signed-off-by: Joffrey F --- script/release/release.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/script/release/release.py b/script/release/release.py index 749ea49d3..9a5af3aa5 100755 --- a/script/release/release.py +++ b/script/release/release.py @@ -173,9 +173,10 @@ def distclean(): def pypi_upload(args): print('Uploading to PyPi') try: + rel = args.release.replace('-rc', 'rc') twine_upload([ - 'dist/docker_compose-{}*.whl'.format(args.release), - 'dist/docker-compose-{}*.tar.gz'.format(args.release) + 'dist/docker_compose-{}*.whl'.format(rel), + 'dist/docker-compose-{}*.tar.gz'.format(rel) ]) except HTTPError as e: if e.response.status_code == 400 and 'File already exists' in e.message: From 90625cf31b806ee53ecf4dab2cabf498c7541444 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Thu, 4 Oct 2018 01:09:48 -0700 Subject: [PATCH 09/11] Don't attempt iterating on None during parallel pull Signed-off-by: Joffrey F --- compose/project.py | 11 +++++------ compose/utils.py | 6 ++++++ tests/integration/project_test.py | 17 +++++++++++++++++ 3 files changed, 28 insertions(+), 6 deletions(-) diff --git a/compose/project.py b/compose/project.py index 4340577c9..92c352050 100644 --- a/compose/project.py +++ b/compose/project.py @@ -34,6 +34,7 @@ from .service import Service from .service import ServiceNetworkMode from .service import ServicePidMode from .utils import microseconds_from_time_nano +from .utils import truncate_string from .volume import ProjectVolumes @@ -554,12 +555,10 @@ class Project(object): if parallel_pull: def pull_service(service): strm = service.pull(ignore_pull_failures, True, stream=True) - writer = parallel.get_stream_writer() + if strm is None: # Attempting to pull service with no `image` key is a no-op + return - def trunc(s): - if len(s) > 35: - return s[:33] + '...' - return s + writer = parallel.get_stream_writer() for event in strm: if 'status' not in event: @@ -572,7 +571,7 @@ class Project(object): status = '{} ({:.1%})'.format(status, percentage) writer.write( - msg, service.name, trunc(status), lambda s: s + msg, service.name, truncate_string(status), lambda s: s ) _, errors = parallel.parallel_execute( diff --git a/compose/utils.py b/compose/utils.py index 72e6ced17..9f0441d08 100644 --- a/compose/utils.py +++ b/compose/utils.py @@ -180,3 +180,9 @@ def unique_everseen(iterable, key=lambda x: x): if unique_key not in seen: seen.add(unique_key) yield element + + +def truncate_string(s, max_chars=35): + if len(s) > max_chars: + return s[:max_chars - 2] + '...' + return s diff --git a/tests/integration/project_test.py b/tests/integration/project_test.py index 63939676e..57f3b7074 100644 --- a/tests/integration/project_test.py +++ b/tests/integration/project_test.py @@ -105,6 +105,23 @@ class ProjectTest(DockerClientTestCase): project = Project('composetest', [web, db], self.client) assert set(project.containers(stopped=True)) == set([web_1, db_1]) + def test_parallel_pull_with_no_image(self): + config_data = build_config( + version=V2_3, + services=[{ + 'name': 'web', + 'build': {'context': '.'}, + }], + ) + + project = Project.from_config( + name='composetest', + config_data=config_data, + client=self.client + ) + + project.pull(parallel_pull=True) + def test_volumes_from_service(self): project = Project.from_config( name='composetest', From 099c887b597f6883a1dea6086d24edc05c13dfa3 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Thu, 4 Oct 2018 01:40:39 -0700 Subject: [PATCH 10/11] Re-enable testing of TP and beta releases Signed-off-by: Joffrey F --- script/test/versions.py | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/script/test/versions.py b/script/test/versions.py index 6d273a9e6..a06c49f20 100755 --- a/script/test/versions.py +++ b/script/test/versions.py @@ -36,6 +36,8 @@ import requests GITHUB_API = 'https://api.github.com/repos' +STAGES = ['tp', 'beta', 'rc'] + class Version(namedtuple('_Version', 'major minor patch stage edition')): @@ -45,7 +47,7 @@ class Version(namedtuple('_Version', 'major minor patch stage edition')): version = version.lstrip('v') version, _, stage = version.partition('-') if stage: - if not any(marker in stage for marker in ['rc', 'tp', 'beta']): + if not any(marker in stage for marker in STAGES): edition = stage stage = None elif '-' in stage: @@ -62,8 +64,16 @@ class Version(namedtuple('_Version', 'major minor patch stage edition')): """Return a representation that allows this object to be sorted correctly with the default comparator. """ - # rc releases should appear before official releases - stage = (0, self.stage) if self.stage else (1, ) + # non-GA releases should appear before GA releases + # Order: tp -> beta -> rc -> GA + if self.stage: + for st in STAGES: + if st in self.stage: + stage = (STAGES.index(st), self.stage) + break + else: + stage = (len(STAGES),) + return (int(self.major), int(self.minor), int(self.patch)) + stage def __str__(self): @@ -124,9 +134,6 @@ def get_versions(tags): v = Version.parse(tag['name']) if v in BLACKLIST: continue - # FIXME: Temporary. Remove once these versions are built on dockerswarm/dind - if v.stage and 'rc' not in v.stage: - continue yield v except ValueError: print("Skipping invalid tag: {name}".format(**tag), file=sys.stderr) From 350a555e0402c20c06c15a9bf6977ce0b3a2407c Mon Sep 17 00:00:00 2001 From: Silvin Lubecki Date: Mon, 8 Oct 2018 17:10:25 +0200 Subject: [PATCH 11/11] "Bump 1.23.0-rc2" Signed-off-by: Silvin Lubecki --- CHANGELOG.md | 6 ++++++ compose/__init__.py | 2 +- script/run/run.sh | 2 +- 3 files changed, 8 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3f2128090..a37f1664c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -56,6 +56,12 @@ naming scheme accordingly before upgrading. - Fixed a bug causing `external: false` entries in the Compose file to be printed as `external: true` in the output of `docker-compose config` +- Fixed a bug where issuing a `docker-compose pull` command on services + without a defined image key would cause Compose to crash + +- Volumes and binds are now mounted in the order they're declared in the + service definition + ### Miscellaneous - The `zsh` completion script has been updated with new options, and no diff --git a/compose/__init__.py b/compose/__init__.py index f0e3f3274..1f35b9a35 100644 --- a/compose/__init__.py +++ b/compose/__init__.py @@ -1,4 +1,4 @@ from __future__ import absolute_import from __future__ import unicode_literals -__version__ = '1.23.0-rc1' +__version__ = '1.23.0-rc2' diff --git a/script/run/run.sh b/script/run/run.sh index fa2248609..f02135f42 100755 --- a/script/run/run.sh +++ b/script/run/run.sh @@ -15,7 +15,7 @@ set -e -VERSION="1.23.0-rc1" +VERSION="1.23.0-rc2" IMAGE="docker/compose:$VERSION"