From 936e6971f91438394b7fe051ac5ab56797d1feaf Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Mon, 24 Sep 2018 23:46:38 +0000 Subject: [PATCH 01/35] "Bump 1.23.0-rc1" Signed-off-by: Joffrey F --- CHANGELOG.md | 62 +++++++++++++++++++++++++++++++++++++++++++++ compose/__init__.py | 2 +- script/run/run.sh | 2 +- 3 files changed, 64 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d22c16454..e9226db23 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,68 @@ Change log ========== +1.23.0 (2018-10-10) +------------------- + +### Features + +### Important note + +The default naming scheme for containers created by Compose in this version +has changed from `__` to +`___`, where `` is a randomly-generated +hexadecimal string. Please make sure to update scripts relying on the old +naming scheme accordingly before upgrading. + +### All versions + +- Logs for containers restarting after a crash will now appear in the output + of the `up` and `logs` commands. + +- Added `--hash` option to the `docker-compose config` command, allowing users + to print a hash string for each service's configuration to facilitate rolling + updates. + +- Output for the `pull` command now reports status / progress even when pulling + multiple images in parallel. + +- For images with multiple names, Compose will now attempt to match the one + present in the service configuration in the output of the `images` command. + +### Bugfixes + +- Parallel `run` commands for the same service will no longer fail due to name + collisions. + +- Fixed an issue where paths longer than 260 characters on Windows clients would + cause `docker-compose build` to fail. + +- Fixed a bug where attempting to mount `/var/run/docker.sock` with + Docker Desktop for Windows would result in failure. + +- The `--project-directory` option is now used by Compose to determine where to + look for the `.env` file. + +- `docker-compose build` no longer fails when attempting to pull an image with + credentials provided by the gcloud credential helper. + +- Fixed the `--exit-code-from` option in `docker-compose up` to always report + the actual exit code even when the watched container isn't the cause of the + exit. + +- Fixed a bug that caused hash configuration with multiple networks to be + inconsistent, causing some services to be unnecessarily restarted. + +- Fixed a pipe handling issue when using the containerized version of Compose. + +- Fixed a bug causing `external: false` entries in the Compose file to be + printed as `external: true` in the output of `docker-compose config` + +### Miscellaneous + +- The `zsh` completion script has been updated with new options, and no + longer suggests container names where service names are expected. + 1.22.0 (2018-07-17) ------------------- diff --git a/compose/__init__.py b/compose/__init__.py index 3433b63cc..f0e3f3274 100644 --- a/compose/__init__.py +++ b/compose/__init__.py @@ -1,4 +1,4 @@ from __future__ import absolute_import from __future__ import unicode_literals -__version__ = '1.23.0dev' +__version__ = '1.23.0-rc1' diff --git a/script/run/run.sh b/script/run/run.sh index 6b004606c..fa2248609 100755 --- a/script/run/run.sh +++ b/script/run/run.sh @@ -15,7 +15,7 @@ set -e -VERSION="1.22.0" +VERSION="1.23.0-rc1" IMAGE="docker/compose:$VERSION" From ec4ea8d2f14a72a31b6d14d54b274159c39095a6 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Tue, 25 Sep 2018 00:46:52 +0000 Subject: [PATCH 02/35] "Bump 1.23.0-rc1" Signed-off-by: Joffrey F --- CHANGELOG.md | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e9226db23..3f2128090 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,8 +4,6 @@ Change log 1.23.0 (2018-10-10) ------------------- -### Features - ### Important note The default naming scheme for containers created by Compose in this version @@ -14,7 +12,7 @@ has changed from `__` to hexadecimal string. Please make sure to update scripts relying on the old naming scheme accordingly before upgrading. -### All versions +### Features - Logs for containers restarting after a crash will now appear in the output of the `up` and `logs` commands. From 47d740b800addf285bcbf96a4ce35918724a4ba6 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Mon, 24 Sep 2018 20:05:40 -0700 Subject: [PATCH 03/35] Fix some release script issues Signed-off-by: Joffrey F --- script/release/release.py | 21 ++++++++++++++------- 1 file changed, 14 insertions(+), 7 deletions(-) diff --git a/script/release/release.py b/script/release/release.py index c6dc146a7..749ea49d3 100755 --- a/script/release/release.py +++ b/script/release/release.py @@ -77,19 +77,24 @@ def monitor_pr_status(pr_data): 'pending': 0, 'success': 0, 'failure': 0, + 'error': 0, } for detail in status.statuses: if detail.context == 'dco-signed': # dco-signed check breaks on merge remote-tracking ; ignore it continue - summary[detail.state] += 1 - print('{pending} pending, {success} successes, {failure} failures'.format(**summary)) - if summary['pending'] == 0 and summary['failure'] == 0 and summary['success'] > 0: + if detail.state in summary: + summary[detail.state] += 1 + print( + '{pending} pending, {success} successes, {failure} failures, ' + '{error} errors'.format(**summary) + ) + if summary['failure'] > 0 or summary['error'] > 0: + raise ScriptError('CI failures detected!') + elif summary['pending'] == 0 and summary['success'] > 0: # This check assumes at least 1 non-DCO CI check to avoid race conditions. # If testing on a repo without CI, use --skip-ci-check to avoid looping eternally return True - elif summary['failure'] > 0: - raise ScriptError('CI failures detected!') time.sleep(30) elif status.state == 'success': print('{} successes: all clear!'.format(status.total_count)) @@ -97,12 +102,14 @@ def monitor_pr_status(pr_data): def check_pr_mergeable(pr_data): - if not pr_data.mergeable: + if pr_data.mergeable is False: + # mergeable can also be null, in which case the warning would be a false positive. print( 'WARNING!! PR #{} can not currently be merged. You will need to ' 'resolve the conflicts manually before finalizing the release.'.format(pr_data.number) ) - return pr_data.mergeable + + return pr_data.mergeable is True def create_release_draft(repository, version, pr_data, files): From c327a498b03dfcc4137d3e05c904ab620eb12b90 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Tue, 25 Sep 2018 09:13:12 -0700 Subject: [PATCH 04/35] Don't rely on container names containing the db string to identify them Signed-off-by: Joffrey F --- tests/integration/project_test.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/integration/project_test.py b/tests/integration/project_test.py index 858a8dfd7..63939676e 100644 --- a/tests/integration/project_test.py +++ b/tests/integration/project_test.py @@ -432,7 +432,7 @@ class ProjectTest(DockerClientTestCase): project.up(strategy=ConvergenceStrategy.always) assert len(project.containers()) == 2 - db_container = [c for c in project.containers() if 'db' in c.name][0] + db_container = [c for c in project.containers() if c.service == 'db'][0] assert db_container.id != old_db_id assert db_container.get('Volumes./etc') == db_volume_path @@ -452,7 +452,7 @@ class ProjectTest(DockerClientTestCase): project.up(strategy=ConvergenceStrategy.always) assert len(project.containers()) == 2 - db_container = [c for c in project.containers() if 'db' in c.name][0] + db_container = [c for c in project.containers() if c.service == 'db'][0] assert db_container.id != old_db_id assert db_container.get_mount('/etc')['Source'] == db_volume_path @@ -499,7 +499,7 @@ class ProjectTest(DockerClientTestCase): assert len(new_containers) == 2 assert [c.is_running for c in new_containers] == [True, True] - db_container = [c for c in new_containers if 'db' in c.name][0] + db_container = [c for c in new_containers if c.service == 'db'][0] assert db_container.id == old_db_id assert db_container.get_mount('/var/db')['Source'] == db_volume_path From 320e4819d873a857630f41064f6bb6f76ee0ff31 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Wed, 26 Sep 2018 13:44:42 -0700 Subject: [PATCH 05/35] Avoid cred helpers errors in release script Signed-off-by: Joffrey F --- script/release/README.md | 6 ++++++ script/release/release.sh | 14 ++++++++++++-- script/release/release/images.py | 8 ++++++++ 3 files changed, 26 insertions(+), 2 deletions(-) diff --git a/script/release/README.md b/script/release/README.md index c5136c764..65883f5d3 100644 --- a/script/release/README.md +++ b/script/release/README.md @@ -20,6 +20,12 @@ following repositories: - docker/compose - docker/compose-tests +### A local Python environment + +While most of the release script is running inside a Docker container, +fetching local Docker credentials depends on the `docker` Python package +being available locally. + ### A Github account and Github API token Your Github account needs to have write access on the `docker/compose` repo. diff --git a/script/release/release.sh b/script/release/release.sh index 201182657..ee75b13a6 100755 --- a/script/release/release.sh +++ b/script/release/release.sh @@ -15,9 +15,19 @@ if test -z $BINTRAY_TOKEN; then exit 1 fi -docker run -e GITHUB_TOKEN=$GITHUB_TOKEN -e BINTRAY_TOKEN=$BINTRAY_TOKEN -e SSH_AUTH_SOCK=$SSH_AUTH_SOCK -it \ +if test -z $(python -c "import docker; print(docker.version)" 2>/dev/null); then + echo "This script requires the 'docker' Python package to be installed locally" + exit 1 +fi + +hub_credentials=$(python -c "from docker import auth; cfg = auth.load_config(); print(auth.encode_header(auth.resolve_authconfig(cfg, 'docker.io')).decode('ascii'))") + +docker run -it \ + -e GITHUB_TOKEN=$GITHUB_TOKEN \ + -e BINTRAY_TOKEN=$BINTRAY_TOKEN \ + -e SSH_AUTH_SOCK=$SSH_AUTH_SOCK \ + -e HUB_CREDENTIALS=$hub_credentials \ --mount type=bind,source=$(pwd),target=/src \ - --mount type=bind,source=$HOME/.docker,target=/root/.docker \ --mount type=bind,source=$HOME/.gitconfig,target=/root/.gitconfig \ --mount type=bind,source=/var/run/docker.sock,target=/var/run/docker.sock \ --mount type=bind,source=$HOME/.ssh,target=/root/.ssh \ diff --git a/script/release/release/images.py b/script/release/release/images.py index b8f7ed3d6..e247f596d 100644 --- a/script/release/release/images.py +++ b/script/release/release/images.py @@ -2,6 +2,8 @@ from __future__ import absolute_import from __future__ import print_function from __future__ import unicode_literals +import base64 +import json import os import shutil @@ -15,6 +17,12 @@ class ImageManager(object): def __init__(self, version): self.docker_client = docker.APIClient(**docker.utils.kwargs_from_env()) self.version = version + if 'HUB_CREDENTIALS' in os.environ: + print('HUB_CREDENTIALS found in environment, issuing login') + credentials = json.loads(base64.urlsafe_b64decode(os.environ['HUB_CREDENTIALS'])) + self.docker_client.login( + username=credentials['Username'], password=credentials['Password'] + ) def build_images(self, repository, files): print("Building release images...") From 62aeb767d393bacc047b5a46bcab0eae524f53dc Mon Sep 17 00:00:00 2001 From: Antony MECHIN Date: Mon, 24 Sep 2018 15:55:55 +0200 Subject: [PATCH 06/35] tests.unit.config: Make make_service_dict working dir argument optional. Signed-off-by: Antony MECHIN --- tests/unit/config/config_test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/unit/config/config_test.py b/tests/unit/config/config_test.py index 1d42c10d5..c054c388e 100644 --- a/tests/unit/config/config_test.py +++ b/tests/unit/config/config_test.py @@ -42,7 +42,7 @@ from tests import unittest DEFAULT_VERSION = V2_0 -def make_service_dict(name, service_dict, working_dir, filename=None): +def make_service_dict(name, service_dict, working_dir='.', filename=None): """Test helper function to construct a ServiceExtendsResolver """ resolver = config.ServiceExtendsResolver( From bb87a3d040b515f32ae349986c131697aa09ca29 Mon Sep 17 00:00:00 2001 From: Antony MECHIN Date: Mon, 24 Sep 2018 15:59:02 +0200 Subject: [PATCH 07/35] tests.unit.config: Make sure volume order is preserved. Signed-off-by: Antony MECHIN --- tests/unit/config/config_test.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/tests/unit/config/config_test.py b/tests/unit/config/config_test.py index c054c388e..52c89a9e0 100644 --- a/tests/unit/config/config_test.py +++ b/tests/unit/config/config_test.py @@ -8,6 +8,7 @@ import os import shutil import tempfile from operator import itemgetter +from random import shuffle import py import pytest @@ -3536,6 +3537,13 @@ class VolumeConfigTest(unittest.TestCase): ).services[0] assert d['volumes'] == [VolumeSpec.parse('/host/path:/container/path')] + @pytest.mark.skipif(IS_WINDOWS_PLATFORM, reason='posix paths') + def test_volumes_order_is_preserved(self): + volumes = ['/{0}:/{0}'.format(i) for i in range(0, 6)] + shuffle(volumes) + cfg = make_service_dict('foo', {'build': '.', 'volumes': volumes}) + assert cfg['volumes'] == volumes + @pytest.mark.skipif(IS_WINDOWS_PLATFORM, reason='posix paths') @mock.patch.dict(os.environ) def test_volume_binding_with_home(self): From 18c2d08011fbf7f97d574f10f84ef68717d68ad8 Mon Sep 17 00:00:00 2001 From: Antony MECHIN Date: Mon, 24 Sep 2018 18:08:17 +0200 Subject: [PATCH 08/35] utils: Add unique_everseen (from itertools recipies). Signed-off-by: Antony MECHIN --- compose/utils.py | 9 +++++++++ tests/unit/utils_test.py | 6 ++++++ 2 files changed, 15 insertions(+) diff --git a/compose/utils.py b/compose/utils.py index 8f0b3e549..b9b6ab9bd 100644 --- a/compose/utils.py +++ b/compose/utils.py @@ -170,3 +170,12 @@ def truncate_id(value): if len(value) > 12: return value[:12] return value + + +def unique_everseen(iterable, key=lambda x: x): + "List unique elements, preserving order. Remember all elements ever seen." + seen = set() + for element in iterable: + if key(element) not in seen: + seen.add(element) + yield element diff --git a/tests/unit/utils_test.py b/tests/unit/utils_test.py index 84becb975..186b6b14e 100644 --- a/tests/unit/utils_test.py +++ b/tests/unit/utils_test.py @@ -68,3 +68,9 @@ class TestParseBytes(object): assert utils.parse_bytes(123) == 123 assert utils.parse_bytes('foobar') is None assert utils.parse_bytes('123') == 123 + + +class TestMoreItertools(object): + def test_unique_everseen(self): + assert list(utils.unique_everseen([2, 1, 2, 1])) == [2, 1] + assert list(utils.unique_everseen([2, 1, 2, 1], hash)) == [2, 1] From d5c314b382ae966e3483fd58f5bca4b1fe7dadfb Mon Sep 17 00:00:00 2001 From: Antony MECHIN Date: Mon, 24 Sep 2018 16:57:49 +0200 Subject: [PATCH 09/35] tests.unity.service: Make sure volumes order is preserved. Signed-off-by: Antony MECHIN --- compose/service.py | 6 ++++-- tests/unit/service_test.py | 17 +++++++++++++++++ 2 files changed, 21 insertions(+), 2 deletions(-) diff --git a/compose/service.py b/compose/service.py index aca24ce17..8df061b9e 100644 --- a/compose/service.py +++ b/compose/service.py @@ -56,6 +56,7 @@ from .utils import json_hash from .utils import parse_bytes from .utils import parse_seconds_float from .utils import truncate_id +from .utils import unique_everseen log = logging.getLogger(__name__) @@ -940,8 +941,9 @@ class Service(object): override_options['mounts'] = override_options.get('mounts') or [] override_options['mounts'].extend([build_mount(v) for v in secret_volumes]) - # Remove possible duplicates (see e.g. https://github.com/docker/compose/issues/5885) - override_options['binds'] = list(set(binds)) + # Remove possible duplicates (see e.g. https://github.com/docker/compose/issues/5885). + # unique_everseen preserves order. (see https://github.com/docker/compose/issues/6091). + override_options['binds'] = list(unique_everseen(binds)) return container_options, override_options def _get_container_host_config(self, override_options, one_off=False): diff --git a/tests/unit/service_test.py b/tests/unit/service_test.py index d5dbcbea6..af1cd1bea 100644 --- a/tests/unit/service_test.py +++ b/tests/unit/service_test.py @@ -1037,6 +1037,23 @@ class ServiceTest(unittest.TestCase): assert len(override_opts['binds']) == 1 assert override_opts['binds'][0] == 'vol:/data:rw' + def test_volumes_order_is_preserved(self): + service = Service('foo', client=self.mock_client) + volumes = [ + VolumeSpec.parse(cfg) for cfg in [ + '/v{0}:/v{0}:rw'.format(i) for i in range(6) + ] + ] + ctnr_opts, override_opts = service._build_container_volume_options( + previous_container=None, + container_options={ + 'volumes': volumes, + 'environment': {}, + }, + override_options={}, + ) + assert override_opts['binds'] == [vol.repr() for vol in volumes] + class TestServiceNetwork(unittest.TestCase): def setUp(self): From b64184e388380a6498c3cd9ab15b5da3059b4421 Mon Sep 17 00:00:00 2001 From: Antony MECHIN Date: Wed, 26 Sep 2018 15:15:59 +0200 Subject: [PATCH 10/35] service: Use OrderedDict to preserve volumes order on versions prior 3.6. Signed-off-by: Antony MECHIN --- compose/service.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/compose/service.py b/compose/service.py index 8df061b9e..3327c77f8 100644 --- a/compose/service.py +++ b/compose/service.py @@ -1429,7 +1429,7 @@ def merge_volume_bindings(volumes, tmpfs, previous_container, mounts): """ affinity = {} - volume_bindings = dict( + volume_bindings = OrderedDict( build_volume_binding(volume) for volume in volumes if volume.external From eb86881af17ac4255acd50745a83e10899591da5 Mon Sep 17 00:00:00 2001 From: Antony MECHIN Date: Thu, 27 Sep 2018 13:58:38 +0200 Subject: [PATCH 11/35] utils: Fix typo in unique_everseen. Signed-off-by: Antony MECHIN --- compose/utils.py | 5 +++-- tests/unit/utils_test.py | 6 ++++-- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/compose/utils.py b/compose/utils.py index b9b6ab9bd..72e6ced17 100644 --- a/compose/utils.py +++ b/compose/utils.py @@ -176,6 +176,7 @@ def unique_everseen(iterable, key=lambda x: x): "List unique elements, preserving order. Remember all elements ever seen." seen = set() for element in iterable: - if key(element) not in seen: - seen.add(element) + unique_key = key(element) + if unique_key not in seen: + seen.add(unique_key) yield element diff --git a/tests/unit/utils_test.py b/tests/unit/utils_test.py index 186b6b14e..21b88d962 100644 --- a/tests/unit/utils_test.py +++ b/tests/unit/utils_test.py @@ -72,5 +72,7 @@ class TestParseBytes(object): class TestMoreItertools(object): def test_unique_everseen(self): - assert list(utils.unique_everseen([2, 1, 2, 1])) == [2, 1] - assert list(utils.unique_everseen([2, 1, 2, 1], hash)) == [2, 1] + unique = utils.unique_everseen + assert list(unique([2, 1, 2, 1])) == [2, 1] + assert list(unique([2, 1, 2, 1], hash)) == [2, 1] + assert list(unique([2, 1, 2, 1], lambda x: 'key_%s' % x)) == [2, 1] From 30c91388f31712b47196dd5bfe6352655df12089 Mon Sep 17 00:00:00 2001 From: Harald Albers Date: Thu, 27 Sep 2018 08:46:37 +0200 Subject: [PATCH 12/35] Fix bash completion for `config --hash` Signed-off-by: Harald Albers --- contrib/completion/bash/docker-compose | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/contrib/completion/bash/docker-compose b/contrib/completion/bash/docker-compose index f4c42362c..395888d34 100644 --- a/contrib/completion/bash/docker-compose +++ b/contrib/completion/bash/docker-compose @@ -136,7 +136,18 @@ _docker_compose_bundle() { _docker_compose_config() { - COMPREPLY=( $( compgen -W "--help --quiet -q --resolve-image-digests --services --volumes --hash" -- "$cur" ) ) + case "$prev" in + --hash) + if [[ $cur == \\* ]] ; then + COMPREPLY=( '\*' ) + else + COMPREPLY=( $(compgen -W "$(__docker_compose_services) \\\* " -- "$cur") ) + fi + return + ;; + esac + + COMPREPLY=( $( compgen -W "--hash --help --quiet -q --resolve-image-digests --services --volumes" -- "$cur" ) ) } From 970f8317c51a53431307dd799c63880de5d2151c Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Thu, 4 Oct 2018 00:48:53 -0700 Subject: [PATCH 13/35] Fix twine upload for RC versions Signed-off-by: Joffrey F --- script/release/release.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/script/release/release.py b/script/release/release.py index 749ea49d3..9a5af3aa5 100755 --- a/script/release/release.py +++ b/script/release/release.py @@ -173,9 +173,10 @@ def distclean(): def pypi_upload(args): print('Uploading to PyPi') try: + rel = args.release.replace('-rc', 'rc') twine_upload([ - 'dist/docker_compose-{}*.whl'.format(args.release), - 'dist/docker-compose-{}*.tar.gz'.format(args.release) + 'dist/docker_compose-{}*.whl'.format(rel), + 'dist/docker-compose-{}*.tar.gz'.format(rel) ]) except HTTPError as e: if e.response.status_code == 400 and 'File already exists' in e.message: From 90625cf31b806ee53ecf4dab2cabf498c7541444 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Thu, 4 Oct 2018 01:09:48 -0700 Subject: [PATCH 14/35] Don't attempt iterating on None during parallel pull Signed-off-by: Joffrey F --- compose/project.py | 11 +++++------ compose/utils.py | 6 ++++++ tests/integration/project_test.py | 17 +++++++++++++++++ 3 files changed, 28 insertions(+), 6 deletions(-) diff --git a/compose/project.py b/compose/project.py index 4340577c9..92c352050 100644 --- a/compose/project.py +++ b/compose/project.py @@ -34,6 +34,7 @@ from .service import Service from .service import ServiceNetworkMode from .service import ServicePidMode from .utils import microseconds_from_time_nano +from .utils import truncate_string from .volume import ProjectVolumes @@ -554,12 +555,10 @@ class Project(object): if parallel_pull: def pull_service(service): strm = service.pull(ignore_pull_failures, True, stream=True) - writer = parallel.get_stream_writer() + if strm is None: # Attempting to pull service with no `image` key is a no-op + return - def trunc(s): - if len(s) > 35: - return s[:33] + '...' - return s + writer = parallel.get_stream_writer() for event in strm: if 'status' not in event: @@ -572,7 +571,7 @@ class Project(object): status = '{} ({:.1%})'.format(status, percentage) writer.write( - msg, service.name, trunc(status), lambda s: s + msg, service.name, truncate_string(status), lambda s: s ) _, errors = parallel.parallel_execute( diff --git a/compose/utils.py b/compose/utils.py index 72e6ced17..9f0441d08 100644 --- a/compose/utils.py +++ b/compose/utils.py @@ -180,3 +180,9 @@ def unique_everseen(iterable, key=lambda x: x): if unique_key not in seen: seen.add(unique_key) yield element + + +def truncate_string(s, max_chars=35): + if len(s) > max_chars: + return s[:max_chars - 2] + '...' + return s diff --git a/tests/integration/project_test.py b/tests/integration/project_test.py index 63939676e..57f3b7074 100644 --- a/tests/integration/project_test.py +++ b/tests/integration/project_test.py @@ -105,6 +105,23 @@ class ProjectTest(DockerClientTestCase): project = Project('composetest', [web, db], self.client) assert set(project.containers(stopped=True)) == set([web_1, db_1]) + def test_parallel_pull_with_no_image(self): + config_data = build_config( + version=V2_3, + services=[{ + 'name': 'web', + 'build': {'context': '.'}, + }], + ) + + project = Project.from_config( + name='composetest', + config_data=config_data, + client=self.client + ) + + project.pull(parallel_pull=True) + def test_volumes_from_service(self): project = Project.from_config( name='composetest', From 099c887b597f6883a1dea6086d24edc05c13dfa3 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Thu, 4 Oct 2018 01:40:39 -0700 Subject: [PATCH 15/35] Re-enable testing of TP and beta releases Signed-off-by: Joffrey F --- script/test/versions.py | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/script/test/versions.py b/script/test/versions.py index 6d273a9e6..a06c49f20 100755 --- a/script/test/versions.py +++ b/script/test/versions.py @@ -36,6 +36,8 @@ import requests GITHUB_API = 'https://api.github.com/repos' +STAGES = ['tp', 'beta', 'rc'] + class Version(namedtuple('_Version', 'major minor patch stage edition')): @@ -45,7 +47,7 @@ class Version(namedtuple('_Version', 'major minor patch stage edition')): version = version.lstrip('v') version, _, stage = version.partition('-') if stage: - if not any(marker in stage for marker in ['rc', 'tp', 'beta']): + if not any(marker in stage for marker in STAGES): edition = stage stage = None elif '-' in stage: @@ -62,8 +64,16 @@ class Version(namedtuple('_Version', 'major minor patch stage edition')): """Return a representation that allows this object to be sorted correctly with the default comparator. """ - # rc releases should appear before official releases - stage = (0, self.stage) if self.stage else (1, ) + # non-GA releases should appear before GA releases + # Order: tp -> beta -> rc -> GA + if self.stage: + for st in STAGES: + if st in self.stage: + stage = (STAGES.index(st), self.stage) + break + else: + stage = (len(STAGES),) + return (int(self.major), int(self.minor), int(self.patch)) + stage def __str__(self): @@ -124,9 +134,6 @@ def get_versions(tags): v = Version.parse(tag['name']) if v in BLACKLIST: continue - # FIXME: Temporary. Remove once these versions are built on dockerswarm/dind - if v.stage and 'rc' not in v.stage: - continue yield v except ValueError: print("Skipping invalid tag: {name}".format(**tag), file=sys.stderr) From 350a555e0402c20c06c15a9bf6977ce0b3a2407c Mon Sep 17 00:00:00 2001 From: Silvin Lubecki Date: Mon, 8 Oct 2018 17:10:25 +0200 Subject: [PATCH 16/35] "Bump 1.23.0-rc2" Signed-off-by: Silvin Lubecki --- CHANGELOG.md | 6 ++++++ compose/__init__.py | 2 +- script/run/run.sh | 2 +- 3 files changed, 8 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3f2128090..a37f1664c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -56,6 +56,12 @@ naming scheme accordingly before upgrading. - Fixed a bug causing `external: false` entries in the Compose file to be printed as `external: true` in the output of `docker-compose config` +- Fixed a bug where issuing a `docker-compose pull` command on services + without a defined image key would cause Compose to crash + +- Volumes and binds are now mounted in the order they're declared in the + service definition + ### Miscellaneous - The `zsh` completion script has been updated with new options, and no diff --git a/compose/__init__.py b/compose/__init__.py index f0e3f3274..1f35b9a35 100644 --- a/compose/__init__.py +++ b/compose/__init__.py @@ -1,4 +1,4 @@ from __future__ import absolute_import from __future__ import unicode_literals -__version__ = '1.23.0-rc1' +__version__ = '1.23.0-rc2' diff --git a/script/run/run.sh b/script/run/run.sh index fa2248609..f02135f42 100755 --- a/script/run/run.sh +++ b/script/run/run.sh @@ -15,7 +15,7 @@ set -e -VERSION="1.23.0-rc1" +VERSION="1.23.0-rc2" IMAGE="docker/compose:$VERSION" From 5cf25f519e290c46828f9696ce85019a036f829e Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Fri, 5 Oct 2018 08:21:39 -0700 Subject: [PATCH 17/35] Decontainerize release script Credentials management inside containers is a mess. Let's work on the host instead. Signed-off-by: Joffrey F --- script/release/Dockerfile | 15 -------------- script/release/README.md | 21 +++++++++++++------ script/release/release.sh | 39 +++++++++--------------------------- script/release/setup-venv.sh | 30 +++++++++++++++++++++++++++ 4 files changed, 54 insertions(+), 51 deletions(-) delete mode 100644 script/release/Dockerfile create mode 100755 script/release/setup-venv.sh diff --git a/script/release/Dockerfile b/script/release/Dockerfile deleted file mode 100644 index e5af676a5..000000000 --- a/script/release/Dockerfile +++ /dev/null @@ -1,15 +0,0 @@ -FROM python:3.6 -RUN mkdir -p /src && pip install -U Jinja2==2.10 \ - PyGithub==1.39 \ - pypandoc==1.4 \ - GitPython==2.1.9 \ - requests==2.18.4 \ - twine==1.11.0 && \ - apt-get update && apt-get install -y pandoc - -VOLUME /src/script/release -WORKDIR /src -COPY . /src -RUN python setup.py develop -ENTRYPOINT ["python", "script/release/release.py"] -CMD ["--help"] diff --git a/script/release/README.md b/script/release/README.md index 65883f5d3..f7f911e53 100644 --- a/script/release/README.md +++ b/script/release/README.md @@ -9,8 +9,7 @@ The following things are required to bring a release to a successful conclusion ### Local Docker engine (Linux Containers) -The release script runs inside a container and builds images that will be part -of the release. +The release script builds images that will be part of the release. ### Docker Hub account @@ -20,11 +19,9 @@ following repositories: - docker/compose - docker/compose-tests -### A local Python environment +### Python -While most of the release script is running inside a Docker container, -fetching local Docker credentials depends on the `docker` Python package -being available locally. +The release script is written in Python and requires Python 3.3 at minimum. ### A Github account and Github API token @@ -59,6 +56,18 @@ Said account needs to be a member of the maintainers group for the Moreover, the `~/.pypirc` file should exist on your host and contain the relevant pypi credentials. +The following is a sample `.pypirc` provided as a guideline: + +``` +[distutils] +index-servers = + pypi + +[pypi] +username = user +password = pass +``` + ## Start a feature release A feature release is a release that includes all changes present in the diff --git a/script/release/release.sh b/script/release/release.sh index ee75b13a6..7947316e0 100755 --- a/script/release/release.sh +++ b/script/release/release.sh @@ -1,36 +1,15 @@ #!/bin/sh -docker image inspect compose/release-tool > /dev/null -if test $? -ne 0; then - docker build -t compose/release-tool -f $(pwd)/script/release/Dockerfile $(pwd) +if test -d ./.release-venv; then + true +else + ./script/release/setup-venv.sh fi -if test -z $GITHUB_TOKEN; then - echo "GITHUB_TOKEN environment variable must be set" - exit 1 +args=$* + +if test -z $args; then + args="--help" fi -if test -z $BINTRAY_TOKEN; then - echo "BINTRAY_TOKEN environment variable must be set" - exit 1 -fi - -if test -z $(python -c "import docker; print(docker.version)" 2>/dev/null); then - echo "This script requires the 'docker' Python package to be installed locally" - exit 1 -fi - -hub_credentials=$(python -c "from docker import auth; cfg = auth.load_config(); print(auth.encode_header(auth.resolve_authconfig(cfg, 'docker.io')).decode('ascii'))") - -docker run -it \ - -e GITHUB_TOKEN=$GITHUB_TOKEN \ - -e BINTRAY_TOKEN=$BINTRAY_TOKEN \ - -e SSH_AUTH_SOCK=$SSH_AUTH_SOCK \ - -e HUB_CREDENTIALS=$hub_credentials \ - --mount type=bind,source=$(pwd),target=/src \ - --mount type=bind,source=$HOME/.gitconfig,target=/root/.gitconfig \ - --mount type=bind,source=/var/run/docker.sock,target=/var/run/docker.sock \ - --mount type=bind,source=$HOME/.ssh,target=/root/.ssh \ - --mount type=bind,source=/tmp,target=/tmp \ - -v $HOME/.pypirc:/root/.pypirc \ - compose/release-tool $* +./.release-venv/bin/python ./script/release/release.py $args diff --git a/script/release/setup-venv.sh b/script/release/setup-venv.sh new file mode 100755 index 000000000..d3d3f9a42 --- /dev/null +++ b/script/release/setup-venv.sh @@ -0,0 +1,30 @@ +#!/bin/bash + +if test -z $PYTHONBIN; then + PYTHONBIN=$(which python3) + if test -z $PYTHONBIN; then + PYTHONBIN=$(which python) + fi +fi + +VERSION=$($PYTHONBIN -c "import sys; print('{}.{}'.format(*sys.version_info[0:2]))") +if test $(echo $VERSION | cut -d. -f1) -lt 3; then + echo "Python 3.3 or above is required" +fi + +if test $(echo $VERSION | cut -d. -f2) -lt 3; then + echo "Python 3.3 or above is required" +fi + +$PYTHONBIN -m venv ./.release-venv + +VENVBINS=./.release-venv/bin + +$VENVBINS/pip install -U Jinja2==2.10 \ + PyGithub==1.39 \ + pypandoc==1.4 \ + GitPython==2.1.9 \ + requests==2.18.4 \ + twine==1.11.0 + +$VENVBINS/python setup.py develop From 9bccfa8dd0ae29df273824dd9697733555175fc1 Mon Sep 17 00:00:00 2001 From: Andrew Rabert Date: Fri, 5 Oct 2018 14:01:35 -0400 Subject: [PATCH 18/35] Use Docker binary from official Docker image Signed-off-by: Andrew Rabert --- Dockerfile.run | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/Dockerfile.run b/Dockerfile.run index e9ba19fd4..bf87fc335 100644 --- a/Dockerfile.run +++ b/Dockerfile.run @@ -1,7 +1,7 @@ +FROM docker:17.12.1 as docker FROM alpine:3.6 ENV GLIBC 2.27-r0 -ENV DOCKERBINS_SHA 1270dce1bd7e1838d62ae21d2505d87f16efc1d9074645571daaefdfd0c14054 RUN apk update && apk add --no-cache openssl ca-certificates curl libgcc && \ curl -fsSL -o /etc/apk/keys/sgerrand.rsa.pub https://alpine-pkgs.sgerrand.com/sgerrand.rsa.pub && \ @@ -10,14 +10,10 @@ RUN apk update && apk add --no-cache openssl ca-certificates curl libgcc && \ ln -s /lib/libz.so.1 /usr/glibc-compat/lib/ && \ ln -s /lib/libc.musl-x86_64.so.1 /usr/glibc-compat/lib && \ ln -s /usr/lib/libgcc_s.so.1 /usr/glibc-compat/lib && \ - curl -fsSL -o dockerbins.tgz "https://download.docker.com/linux/static/stable/x86_64/docker-17.12.1-ce.tgz" && \ - echo "${DOCKERBINS_SHA} dockerbins.tgz" | sha256sum -c - && \ - tar xvf dockerbins.tgz docker/docker --strip-components 1 && \ - mv docker /usr/local/bin/docker && \ - chmod +x /usr/local/bin/docker && \ - rm dockerbins.tgz /etc/apk/keys/sgerrand.rsa.pub glibc-$GLIBC.apk && \ + rm /etc/apk/keys/sgerrand.rsa.pub glibc-$GLIBC.apk && \ apk del curl +COPY --from=docker /usr/local/bin/docker /usr/local/bin/docker COPY dist/docker-compose-Linux-x86_64 /usr/local/bin/docker-compose ENTRYPOINT ["docker-compose"] From fe347321c952ceb429c5c09c4113f4e4b9676b95 Mon Sep 17 00:00:00 2001 From: Ofek Lev Date: Wed, 10 Oct 2018 22:04:33 -0400 Subject: [PATCH 19/35] Upgrade Windows-specific dependency colorama Signed-off-by: Ofek Lev --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 2819810c2..a0093d8dd 100644 --- a/setup.py +++ b/setup.py @@ -55,7 +55,7 @@ extras_require = { ':python_version < "3.4"': ['enum34 >= 1.0.4, < 2'], ':python_version < "3.5"': ['backports.ssl_match_hostname >= 3.5'], ':python_version < "3.3"': ['ipaddress >= 1.0.16'], - ':sys_platform == "win32"': ['colorama >= 0.3.9, < 0.4'], + ':sys_platform == "win32"': ['colorama >= 0.4, < 0.5'], 'socks': ['PySocks >= 1.5.6, != 1.5.7, < 2'], } From e722190d5011a31eb070435ca165160f4c996f65 Mon Sep 17 00:00:00 2001 From: Ofek Lev Date: Fri, 12 Oct 2018 11:35:27 -0400 Subject: [PATCH 20/35] Update requirements.txt Signed-off-by: Ofek Lev --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 41d21172e..0ea046589 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,13 +2,13 @@ backports.ssl-match-hostname==3.5.0.1; python_version < '3' cached-property==1.3.0 certifi==2017.4.17 chardet==3.0.4 +colorama==0.4.0; sys_platform == 'win32' docker==3.5.0 docker-pycreds==0.3.0 dockerpty==0.4.1 docopt==0.6.2 enum34==1.1.6; python_version < '3.4' functools32==3.2.3.post2; python_version < '3.2' -git+git://github.com/tartley/colorama.git@bd378c725b45eba0b8e5cc091c3ca76a954c92ff; sys_platform == 'win32' idna==2.5 ipaddress==1.0.18 jsonschema==2.6.0 From 51d44c7ebc551ece13c611b348486e967794cf34 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Fri, 12 Oct 2018 06:13:55 -0700 Subject: [PATCH 21/35] Add pypirc check Signed-off-by: Joffrey F --- script/release/release.py | 24 +++---------------- script/release/release/pypi.py | 44 ++++++++++++++++++++++++++++++++++ 2 files changed, 47 insertions(+), 21 deletions(-) create mode 100644 script/release/release/pypi.py diff --git a/script/release/release.py b/script/release/release.py index 9a5af3aa5..15c74c775 100755 --- a/script/release/release.py +++ b/script/release/release.py @@ -17,6 +17,8 @@ from release.const import NAME from release.const import REPO_ROOT from release.downloader import BinaryDownloader from release.images import ImageManager +from release.pypi import check_pypirc +from release.pypi import pypi_upload from release.repository import delete_assets from release.repository import get_contributors from release.repository import Repository @@ -28,8 +30,6 @@ from release.utils import ScriptError from release.utils import update_init_py_version from release.utils import update_run_sh_version from release.utils import yesno -from requests.exceptions import HTTPError -from twine.commands.upload import main as twine_upload def create_initial_branch(repository, args): @@ -170,25 +170,6 @@ def distclean(): shutil.rmtree(folder, ignore_errors=True) -def pypi_upload(args): - print('Uploading to PyPi') - try: - rel = args.release.replace('-rc', 'rc') - twine_upload([ - 'dist/docker_compose-{}*.whl'.format(rel), - 'dist/docker-compose-{}*.tar.gz'.format(rel) - ]) - except HTTPError as e: - if e.response.status_code == 400 and 'File already exists' in e.message: - if not args.finalize_resume: - raise ScriptError( - 'Package already uploaded on PyPi.' - ) - print('Skipping PyPi upload - package already uploaded') - else: - raise ScriptError('Unexpected HTTP error uploading package to PyPi: {}'.format(e)) - - def resume(args): try: distclean() @@ -277,6 +258,7 @@ def start(args): def finalize(args): distclean() try: + check_pypirc() repository = Repository(REPO_ROOT, args.repo) img_manager = ImageManager(args.release) pr_data = repository.find_release_pr(args.release) diff --git a/script/release/release/pypi.py b/script/release/release/pypi.py new file mode 100644 index 000000000..a40e17544 --- /dev/null +++ b/script/release/release/pypi.py @@ -0,0 +1,44 @@ +from __future__ import absolute_import +from __future__ import unicode_literals + +from configparser import Error +from requests.exceptions import HTTPError +from twine.commands.upload import main as twine_upload +from twine.utils import get_config + +from .utils import ScriptError + + +def pypi_upload(args): + print('Uploading to PyPi') + try: + rel = args.release.replace('-rc', 'rc') + twine_upload([ + 'dist/docker_compose-{}*.whl'.format(rel), + 'dist/docker-compose-{}*.tar.gz'.format(rel) + ]) + except HTTPError as e: + if e.response.status_code == 400 and 'File already exists' in e.message: + if not args.finalize_resume: + raise ScriptError( + 'Package already uploaded on PyPi.' + ) + print('Skipping PyPi upload - package already uploaded') + else: + raise ScriptError('Unexpected HTTP error uploading package to PyPi: {}'.format(e)) + + +def check_pypirc(): + try: + config = get_config() + except Error as e: + raise ScriptError('Failed to parse .pypirc file: {}'.format(e)) + + if config is None: + raise ScriptError('Failed to parse .pypirc file') + + if 'pypi' not in config: + raise ScriptError('Missing [pypi] section in .pypirc file') + + if not (config['pypi'].get('username') and config['pypi'].get('password')): + raise ScriptError('Missing login/password pair for pypi repo') From c9107cff39328475ccb3a3efa467a98e4dccba10 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Fri, 12 Oct 2018 06:14:35 -0700 Subject: [PATCH 22/35] Fix arg checks in release.sh Signed-off-by: Joffrey F --- script/release/release.sh | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/script/release/release.sh b/script/release/release.sh index 7947316e0..c10f8aba5 100755 --- a/script/release/release.sh +++ b/script/release/release.sh @@ -6,10 +6,8 @@ else ./script/release/setup-venv.sh fi -args=$* - -if test -z $args; then +if test -z "$*"; then args="--help" fi -./.release-venv/bin/python ./script/release/release.py $args +./.release-venv/bin/python ./script/release/release.py "$@" From da25be8f9944e3962a832396c78f32c0bd4c562a Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Fri, 12 Oct 2018 06:39:56 -0700 Subject: [PATCH 23/35] Fix ImageManager inconsistencies Signed-off-by: Joffrey F --- script/release/release.py | 2 +- script/release/release/images.py | 27 ++++++++++++--------------- 2 files changed, 13 insertions(+), 16 deletions(-) diff --git a/script/release/release.py b/script/release/release.py index 15c74c775..6574bfddd 100755 --- a/script/release/release.py +++ b/script/release/release.py @@ -266,7 +266,7 @@ def finalize(args): raise ScriptError('No PR found for {}'.format(args.release)) if not check_pr_mergeable(pr_data): raise ScriptError('Can not finalize release with an unmergeable PR') - if not img_manager.check_images(args.release): + if not img_manager.check_images(): raise ScriptError('Missing release image') br_name = branch_name(args.release) if not repository.branch_exists(br_name): diff --git a/script/release/release/images.py b/script/release/release/images.py index e247f596d..df6eeda4f 100644 --- a/script/release/release/images.py +++ b/script/release/release/images.py @@ -27,13 +27,12 @@ class ImageManager(object): def build_images(self, repository, files): print("Building release images...") repository.write_git_sha() - docker_client = docker.APIClient(**docker.utils.kwargs_from_env()) distdir = os.path.join(REPO_ROOT, 'dist') os.makedirs(distdir, exist_ok=True) shutil.copy(files['docker-compose-Linux-x86_64'][0], distdir) os.chmod(os.path.join(distdir, 'docker-compose-Linux-x86_64'), 0o755) print('Building docker/compose image') - logstream = docker_client.build( + logstream = self.docker_client.build( REPO_ROOT, tag='docker/compose:{}'.format(self.version), dockerfile='Dockerfile.run', decode=True ) @@ -44,7 +43,7 @@ class ImageManager(object): print(chunk['stream'], end='') print('Building test image (for UCP e2e)') - logstream = docker_client.build( + logstream = self.docker_client.build( REPO_ROOT, tag='docker-compose-tests:tmp', decode=True ) for chunk in logstream: @@ -53,13 +52,15 @@ class ImageManager(object): if 'stream' in chunk: print(chunk['stream'], end='') - container = docker_client.create_container( + container = self.docker_client.create_container( 'docker-compose-tests:tmp', entrypoint='tox' ) - docker_client.commit(container, 'docker/compose-tests', 'latest') - docker_client.tag('docker/compose-tests:latest', 'docker/compose-tests:{}'.format(self.version)) - docker_client.remove_container(container, force=True) - docker_client.remove_image('docker-compose-tests:tmp', force=True) + self.docker_client.commit(container, 'docker/compose-tests', 'latest') + self.docker_client.tag( + 'docker/compose-tests:latest', 'docker/compose-tests:{}'.format(self.version) + ) + self.docker_client.remove_container(container, force=True) + self.docker_client.remove_image('docker-compose-tests:tmp', force=True) @property def image_names(self): @@ -69,23 +70,19 @@ class ImageManager(object): 'docker/compose:{}'.format(self.version) ] - def check_images(self, version): - docker_client = docker.APIClient(**docker.utils.kwargs_from_env()) - + def check_images(self): for name in self.image_names: try: - docker_client.inspect_image(name) + self.docker_client.inspect_image(name) except docker.errors.ImageNotFound: print('Expected image {} was not found'.format(name)) return False return True def push_images(self): - docker_client = docker.APIClient(**docker.utils.kwargs_from_env()) - for name in self.image_names: print('Pushing {} to Docker Hub'.format(name)) - logstream = docker_client.push(name, stream=True, decode=True) + logstream = self.docker_client.push(name, stream=True, decode=True) for chunk in logstream: if 'status' in chunk: print(chunk['status']) From 23beeb353c08e2c53df0d7e7f97d0d70b05d8c25 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Mon, 15 Oct 2018 19:14:58 -0700 Subject: [PATCH 24/35] Update versions in Dockerfiles Signed-off-by: Joffrey F --- Dockerfile | 9 ++------- Dockerfile.run | 6 +++--- 2 files changed, 5 insertions(+), 10 deletions(-) diff --git a/Dockerfile b/Dockerfile index 9df78a826..aa3e1d87b 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,3 +1,4 @@ +FROM docker:18.06.1 as docker FROM python:3.6 RUN set -ex; \ @@ -8,13 +9,7 @@ RUN set -ex; \ python-dev \ git -RUN curl -fsSL -o dockerbins.tgz "https://download.docker.com/linux/static/stable/x86_64/docker-17.12.0-ce.tgz" && \ - SHA256=692e1c72937f6214b1038def84463018d8e320c8eaf8530546c84c2f8f9c767d; \ - echo "${SHA256} dockerbins.tgz" | sha256sum -c - && \ - tar xvf dockerbins.tgz docker/docker --strip-components 1 && \ - mv docker /usr/local/bin/docker && \ - chmod +x /usr/local/bin/docker && \ - rm dockerbins.tgz +COPY --from=docker /usr/local/bin/docker /usr/local/bin/docker # Python3 requires a valid locale RUN echo "en_US.UTF-8 UTF-8" > /etc/locale.gen && locale-gen diff --git a/Dockerfile.run b/Dockerfile.run index bf87fc335..ccc86ea96 100644 --- a/Dockerfile.run +++ b/Dockerfile.run @@ -1,7 +1,7 @@ -FROM docker:17.12.1 as docker -FROM alpine:3.6 +FROM docker:18.06.1 as docker +FROM alpine:3.8 -ENV GLIBC 2.27-r0 +ENV GLIBC 2.28-r0 RUN apk update && apk add --no-cache openssl ca-certificates curl libgcc && \ curl -fsSL -o /etc/apk/keys/sgerrand.rsa.pub https://alpine-pkgs.sgerrand.com/sgerrand.rsa.pub && \ From 12f7e0d2fbcd0876aaf3dc2aec8dfd70ebf8f921 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Mon, 15 Oct 2018 19:22:25 -0700 Subject: [PATCH 25/35] Remove obsolete curl dependency Signed-off-by: Joffrey F --- Dockerfile | 1 - 1 file changed, 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index aa3e1d87b..a14be492e 100644 --- a/Dockerfile +++ b/Dockerfile @@ -5,7 +5,6 @@ RUN set -ex; \ apt-get update -qq; \ apt-get install -y \ locales \ - curl \ python-dev \ git From 5e4098d2280b2b46e2d0ffd08aeaee9fc9b6aec9 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Tue, 16 Oct 2018 13:57:01 -0700 Subject: [PATCH 26/35] Avoid creating duplicate mount points when recreating a service Signed-off-by: Joffrey F --- compose/service.py | 5 +++++ tests/integration/service_test.py | 16 ++++++++++++++++ 2 files changed, 21 insertions(+) diff --git a/compose/service.py b/compose/service.py index 3327c77f8..73744801d 100644 --- a/compose/service.py +++ b/compose/service.py @@ -1489,6 +1489,11 @@ def get_container_data_volumes(container, volumes_option, tmpfs_option, mounts_o if not mount.get('Name'): continue + # Volume (probably an image volume) is overridden by a mount in the service's config + # and would cause a duplicate mountpoint error + if volume.internal in [m.target for m in mounts_option]: + continue + # Copy existing volume from old container volume = volume._replace(external=mount['Name']) volumes.append(volume) diff --git a/tests/integration/service_test.py b/tests/integration/service_test.py index db40409f8..edc195287 100644 --- a/tests/integration/service_test.py +++ b/tests/integration/service_test.py @@ -425,6 +425,22 @@ class ServiceTest(DockerClientTestCase): new_container = service.recreate_container(old_container) assert new_container.get_mount('/data')['Source'] == volume_path + def test_recreate_volume_to_mount(self): + # https://github.com/docker/compose/issues/6280 + service = Service( + project='composetest', + name='db', + client=self.client, + build={'context': 'tests/fixtures/dockerfile-with-volume'}, + volumes=[MountSpec.parse({ + 'type': 'volume', + 'target': '/data', + })] + ) + old_container = create_and_start_container(service) + new_container = service.recreate_container(old_container) + assert new_container.get_mount('/data')['Source'] + def test_duplicate_volume_trailing_slash(self): """ When an image specifies a volume, and the Compose file specifies a host path From 0fa1462b0f9439ff38feec8201871bba4f05c70c Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Tue, 16 Oct 2018 17:21:57 -0700 Subject: [PATCH 27/35] Don't use dot as a path separator as it is a valid character in resource identifiers Signed-off-by: Joffrey F --- compose/config/interpolation.py | 10 ++++---- tests/unit/config/interpolation_test.py | 31 +++++++++++++++++++++++++ 2 files changed, 36 insertions(+), 5 deletions(-) diff --git a/compose/config/interpolation.py b/compose/config/interpolation.py index 4f56dff59..0f878be14 100644 --- a/compose/config/interpolation.py +++ b/compose/config/interpolation.py @@ -48,7 +48,7 @@ def interpolate_environment_variables(version, config, section, environment): def get_config_path(config_key, section, name): - return '{}.{}.{}'.format(section, name, config_key) + return '{}/{}/{}'.format(section, name, config_key) def interpolate_value(name, config_key, value, section, interpolator): @@ -75,7 +75,7 @@ def interpolate_value(name, config_key, value, section, interpolator): def recursive_interpolate(obj, interpolator, config_path): def append(config_path, key): - return '{}.{}'.format(config_path, key) + return '{}/{}'.format(config_path, key) if isinstance(obj, six.string_types): return converter.convert(config_path, interpolator.interpolate(obj)) @@ -160,12 +160,12 @@ class UnsetRequiredSubstitution(Exception): self.err = custom_err_msg -PATH_JOKER = '[^.]+' +PATH_JOKER = '[^/]+' FULL_JOKER = '.+' def re_path(*args): - return re.compile('^{}$'.format('\.'.join(args))) + return re.compile('^{}$'.format('/'.join(args))) def re_path_basic(section, name): @@ -288,7 +288,7 @@ class ConversionMap(object): except ValueError as e: raise ConfigurationError( 'Error while attempting to convert {} to appropriate type: {}'.format( - path, e + path.replace('/', '.'), e ) ) return value diff --git a/tests/unit/config/interpolation_test.py b/tests/unit/config/interpolation_test.py index 0d0e7d28d..91fc3e69d 100644 --- a/tests/unit/config/interpolation_test.py +++ b/tests/unit/config/interpolation_test.py @@ -332,6 +332,37 @@ def test_interpolate_environment_external_resource_convert_types(mock_env): assert value == expected +def test_interpolate_service_name_uses_dot(mock_env): + entry = { + 'service.1': { + 'image': 'busybox', + 'ulimits': { + 'nproc': '${POSINT}', + 'nofile': { + 'soft': '${POSINT}', + 'hard': '${DEFAULT:-40000}' + }, + }, + } + } + + expected = { + 'service.1': { + 'image': 'busybox', + 'ulimits': { + 'nproc': 50, + 'nofile': { + 'soft': 50, + 'hard': 40000 + }, + }, + } + } + + value = interpolate_environment_variables(V3_4, entry, 'service', mock_env) + assert value == expected + + def test_escaped_interpolation(defaults_interpolator): assert defaults_interpolator('$${foo}') == '${foo}' From 5ab3e47b42f004d2cd847051c72f5c10ab16485c Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Wed, 17 Oct 2018 12:10:08 -0700 Subject: [PATCH 28/35] Add workaround for Debian/Ubuntu venv setup failure Signed-off-by: Joffrey F --- script/release/release.sh | 4 ++-- script/release/setup-venv.sh | 25 +++++++++++++++++++++---- 2 files changed, 23 insertions(+), 6 deletions(-) diff --git a/script/release/release.sh b/script/release/release.sh index c10f8aba5..5f853808b 100755 --- a/script/release/release.sh +++ b/script/release/release.sh @@ -1,6 +1,6 @@ #!/bin/sh -if test -d ./.release-venv; then +if test -d ${VENV_DIR:-./.release-venv}; then true else ./script/release/setup-venv.sh @@ -10,4 +10,4 @@ if test -z "$*"; then args="--help" fi -./.release-venv/bin/python ./script/release/release.py "$@" +${VENV_DIR:-./.release-venv}/bin/python ./script/release/release.py "$@" diff --git a/script/release/setup-venv.sh b/script/release/setup-venv.sh index d3d3f9a42..780fc800f 100755 --- a/script/release/setup-venv.sh +++ b/script/release/setup-venv.sh @@ -1,5 +1,11 @@ #!/bin/bash +debian_based() { test -f /etc/debian_version; } + +if test -z $VENV_DIR; then + VENV_DIR=./.release-venv +fi + if test -z $PYTHONBIN; then PYTHONBIN=$(which python3) if test -z $PYTHONBIN; then @@ -16,15 +22,26 @@ if test $(echo $VERSION | cut -d. -f2) -lt 3; then echo "Python 3.3 or above is required" fi -$PYTHONBIN -m venv ./.release-venv +# Debian / Ubuntu workaround: +# https://askubuntu.com/questions/879437/ensurepip-is-disabled-in-debian-ubuntu-for-the-system-python +if debian_based; then + VENV_FLAGS="$VENV_FLAGS --without-pip" +fi -VENVBINS=./.release-venv/bin +$PYTHONBIN -m venv $VENV_DIR $VENV_FLAGS -$VENVBINS/pip install -U Jinja2==2.10 \ +VENV_PYTHONBIN=$VENV_DIR/bin/python + +if debian_based; then + curl https://bootstrap.pypa.io/get-pip.py -o $VENV_DIR/get-pip.py + $VENV_PYTHONBIN $VENV_DIR/get-pip.py +fi + +$VENV_PYTHONBIN -m pip install -U Jinja2==2.10 \ PyGithub==1.39 \ pypandoc==1.4 \ GitPython==2.1.9 \ requests==2.18.4 \ twine==1.11.0 -$VENVBINS/python setup.py develop +$VENV_PYTHONBIN setup.py develop From 45189c134db4669cdd36cc6f041b513a052f831a Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Wed, 17 Oct 2018 12:16:34 -0700 Subject: [PATCH 29/35] "Bump 1.23.0-rc3" Signed-off-by: Joffrey F --- CHANGELOG.md | 7 +++++++ compose/__init__.py | 2 +- script/run/run.sh | 2 +- 3 files changed, 9 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a37f1664c..27f6f3c52 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -48,9 +48,16 @@ naming scheme accordingly before upgrading. the actual exit code even when the watched container isn't the cause of the exit. +- Fixed an issue that would prevent recreating a service in some cases where + a volume would be mapped to the same mountpoint as a volume declared inside + the image's Dockerfile. + - Fixed a bug that caused hash configuration with multiple networks to be inconsistent, causing some services to be unnecessarily restarted. +- Fixed a bug that would cause failures with variable substitution for services + with a name containing one or more dot characters + - Fixed a pipe handling issue when using the containerized version of Compose. - Fixed a bug causing `external: false` entries in the Compose file to be diff --git a/compose/__init__.py b/compose/__init__.py index 1f35b9a35..532f76888 100644 --- a/compose/__init__.py +++ b/compose/__init__.py @@ -1,4 +1,4 @@ from __future__ import absolute_import from __future__ import unicode_literals -__version__ = '1.23.0-rc2' +__version__ = '1.23.0-rc3' diff --git a/script/run/run.sh b/script/run/run.sh index f02135f42..ba945d3ed 100755 --- a/script/run/run.sh +++ b/script/run/run.sh @@ -15,7 +15,7 @@ set -e -VERSION="1.23.0-rc2" +VERSION="1.23.0-rc3" IMAGE="docker/compose:$VERSION" From ea3d406eeda212363f03289770e6c8a7c654dac2 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Wed, 17 Oct 2018 13:39:11 -0700 Subject: [PATCH 30/35] Some additional exclusions in .gitignore / .dockerignore Signed-off-by: Joffrey F --- .dockerignore | 4 +++- .gitignore | 18 ++++++++++-------- 2 files changed, 13 insertions(+), 9 deletions(-) diff --git a/.dockerignore b/.dockerignore index eccd86dda..65ad588d9 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,11 +1,13 @@ *.egg-info .coverage .git +.github .tox build +binaries coverage-html docs/_site -venv +*venv .tox **/__pycache__ *.pyc diff --git a/.gitignore b/.gitignore index 18afd643d..798882748 100644 --- a/.gitignore +++ b/.gitignore @@ -1,16 +1,18 @@ *.egg-info *.pyc +*.swo +*.swp +.cache .coverage* +.DS_Store +.idea + /.tox +/binaries /build +/compose/GITSHA /coverage-html /dist /docs/_site -/venv -README.rst -compose/GITSHA -*.swo -*.swp -.DS_Store -.cache -.idea +/README.rst +/*venv From 8f9ead34d36835cc6eb45dc1a70e86cc02d01876 Mon Sep 17 00:00:00 2001 From: Ofek Lev Date: Wed, 17 Oct 2018 17:11:36 -0400 Subject: [PATCH 31/35] Allow requests 2.20.x Signed-off-by: Ofek Lev --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index a0093d8dd..8260ebc69 100644 --- a/setup.py +++ b/setup.py @@ -33,7 +33,7 @@ install_requires = [ 'cached-property >= 1.2.0, < 2', 'docopt >= 0.6.1, < 0.7', 'PyYAML >= 3.10, < 4', - 'requests >= 2.6.1, != 2.11.0, != 2.12.2, != 2.18.0, < 2.20', + 'requests >= 2.6.1, != 2.11.0, != 2.12.2, != 2.18.0, < 2.21', 'texttable >= 0.9.0, < 0.10', 'websocket-client >= 0.32.0, < 1.0', 'docker >= 3.5.0, < 4.0', From 1c002b584475d20f1dddc347fcf246bf8537e247 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Wed, 24 Oct 2018 15:06:04 -0700 Subject: [PATCH 32/35] Fix new flake8 errors/warnings Signed-off-by: Joffrey F --- compose/cli/errors.py | 2 +- compose/config/types.py | 2 +- compose/config/validation.py | 8 ++++---- tests/acceptance/cli_test.py | 2 +- 4 files changed, 7 insertions(+), 7 deletions(-) diff --git a/compose/cli/errors.py b/compose/cli/errors.py index 82768970b..8c89da6c5 100644 --- a/compose/cli/errors.py +++ b/compose/cli/errors.py @@ -54,7 +54,7 @@ def handle_connection_errors(client): except APIError as e: log_api_error(e, client.api_version) raise ConnectionError() - except (ReadTimeout, socket.timeout) as e: + except (ReadTimeout, socket.timeout): log_timeout_error(client.timeout) raise ConnectionError() except Exception as e: diff --git a/compose/config/types.py b/compose/config/types.py index 838fb9f58..ab8f34e3d 100644 --- a/compose/config/types.py +++ b/compose/config/types.py @@ -125,7 +125,7 @@ def parse_extra_hosts(extra_hosts_config): def normalize_path_for_engine(path): - """Windows paths, c:\my\path\shiny, need to be changed to be compatible with + """Windows paths, c:\\my\\path\\shiny, need to be changed to be compatible with the Engine. Volume paths are expected to be linux style /c/my/path/shiny/ """ drive, tail = splitdrive(path) diff --git a/compose/config/validation.py b/compose/config/validation.py index 0fdcb37e7..87c1f2345 100644 --- a/compose/config/validation.py +++ b/compose/config/validation.py @@ -41,15 +41,15 @@ DOCKER_CONFIG_HINTS = { } -VALID_NAME_CHARS = '[a-zA-Z0-9\._\-]' +VALID_NAME_CHARS = r'[a-zA-Z0-9\._\-]' VALID_EXPOSE_FORMAT = r'^\d+(\-\d+)?(\/[a-zA-Z]+)?$' VALID_IPV4_SEG = r'(\d{1,2}|1\d{2}|2[0-4]\d|25[0-5])' -VALID_IPV4_ADDR = "({IPV4_SEG}\.){{3}}{IPV4_SEG}".format(IPV4_SEG=VALID_IPV4_SEG) -VALID_REGEX_IPV4_CIDR = "^{IPV4_ADDR}/(\d|[1-2]\d|3[0-2])$".format(IPV4_ADDR=VALID_IPV4_ADDR) +VALID_IPV4_ADDR = r"({IPV4_SEG}\.){{3}}{IPV4_SEG}".format(IPV4_SEG=VALID_IPV4_SEG) +VALID_REGEX_IPV4_CIDR = r"^{IPV4_ADDR}/(\d|[1-2]\d|3[0-2])$".format(IPV4_ADDR=VALID_IPV4_ADDR) VALID_IPV6_SEG = r'[0-9a-fA-F]{1,4}' -VALID_REGEX_IPV6_CIDR = "".join(""" +VALID_REGEX_IPV6_CIDR = "".join(r""" ^ ( (({IPV6_SEG}:){{7}}{IPV6_SEG})| diff --git a/tests/acceptance/cli_test.py b/tests/acceptance/cli_test.py index 3d063d853..5b0a0e0fd 100644 --- a/tests/acceptance/cli_test.py +++ b/tests/acceptance/cli_test.py @@ -2361,7 +2361,7 @@ class CLITestCase(DockerClientTestCase): self.dispatch(['up', '-d']) result = self.dispatch(['logs', '-f', '-t']) - assert re.search('(\d{4})-(\d{2})-(\d{2})T(\d{2})\:(\d{2})\:(\d{2})', result.stdout) + assert re.search(r'(\d{4})-(\d{2})-(\d{2})T(\d{2})\:(\d{2})\:(\d{2})', result.stdout) def test_logs_tail(self): self.base_dir = 'tests/fixtures/logs-tail-composefile' From 3104597e7da64fb3efb2c559a77881d58975b2ec Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Mon, 29 Oct 2018 11:43:45 -0700 Subject: [PATCH 33/35] "Bump 1.23.0" Signed-off-by: Joffrey F --- CHANGELOG.md | 3 +++ compose/__init__.py | 2 +- script/run/run.sh | 2 +- 3 files changed, 5 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 27f6f3c52..0f6467098 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -21,6 +21,9 @@ naming scheme accordingly before upgrading. to print a hash string for each service's configuration to facilitate rolling updates. +- Added `--parallel` flag to the `docker-compose build` command, allowing + Compose to build up to 5 images simultaneously. + - Output for the `pull` command now reports status / progress even when pulling multiple images in parallel. diff --git a/compose/__init__.py b/compose/__init__.py index 532f76888..b9088474f 100644 --- a/compose/__init__.py +++ b/compose/__init__.py @@ -1,4 +1,4 @@ from __future__ import absolute_import from __future__ import unicode_literals -__version__ = '1.23.0-rc3' +__version__ = '1.23.0' diff --git a/script/run/run.sh b/script/run/run.sh index ba945d3ed..d82b54f0f 100755 --- a/script/run/run.sh +++ b/script/run/run.sh @@ -15,7 +15,7 @@ set -e -VERSION="1.23.0-rc3" +VERSION="1.23.0" IMAGE="docker/compose:$VERSION" From 140431d3b96f51233d299df131d06d42ed3bbfba Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Mon, 29 Oct 2018 12:22:22 -0700 Subject: [PATCH 34/35] "Bump 1.23.0" Signed-off-by: Joffrey F --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0f6467098..26fd3f883 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,7 +1,7 @@ Change log ========== -1.23.0 (2018-10-10) +1.23.0 (2018-10-30) ------------------- ### Important note From c8524dc1aa54fac0ef27877a07bf4dbc672ba4c8 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Mon, 29 Oct 2018 14:38:50 -0700 Subject: [PATCH 35/35] Bump requests version in requirements.txt Signed-off-by: Joffrey F --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 0ea046589..024b671cc 100644 --- a/requirements.txt +++ b/requirements.txt @@ -16,7 +16,7 @@ pypiwin32==219; sys_platform == 'win32' and python_version < '3.6' pypiwin32==223; sys_platform == 'win32' and python_version >= '3.6' PySocks==1.6.7 PyYAML==3.12 -requests==2.19.1 +requests==2.20.0 six==1.10.0 texttable==0.9.1 urllib3==1.21.1; python_version == '3.3'