From 7aa51a18ff00ebc1425899c7dba643ff033e05e3 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Fri, 30 Mar 2018 18:02:06 -0700 Subject: [PATCH 01/53] Fix port serialization with external IP Signed-off-by: Joffrey F --- compose/config/serialize.py | 5 +++-- tests/unit/config/config_test.py | 12 ++++++++++++ 2 files changed, 15 insertions(+), 2 deletions(-) diff --git a/compose/config/serialize.py b/compose/config/serialize.py index 7de7f41e8..c0cf35c1b 100644 --- a/compose/config/serialize.py +++ b/compose/config/serialize.py @@ -151,9 +151,10 @@ def denormalize_service_dict(service_dict, version, image_digest=None): service_dict['healthcheck']['start_period'] = serialize_ns_time_value( service_dict['healthcheck']['start_period'] ) - if 'ports' in service_dict and version < V3_2: + + if 'ports' in service_dict: service_dict['ports'] = [ - p.legacy_repr() if isinstance(p, types.ServicePort) else p + p.legacy_repr() if p.external_ip or version < V3_2 else p for p in service_dict['ports'] ] if 'volumes' in service_dict and (version < V2_3 or (version > V3_0 and version < V3_2)): diff --git a/tests/unit/config/config_test.py b/tests/unit/config/config_test.py index 0574b215c..8a75648ac 100644 --- a/tests/unit/config/config_test.py +++ b/tests/unit/config/config_test.py @@ -4943,6 +4943,18 @@ class SerializeTest(unittest.TestCase): serialized_config = yaml.load(serialize_config(config_dict)) assert '8080:80/tcp' in serialized_config['services']['web']['ports'] + def test_serialize_ports_with_ext_ip(self): + config_dict = config.Config(version=V3_5, services=[ + { + 'ports': [types.ServicePort('80', '8080', None, None, '127.0.0.1')], + 'image': 'alpine', + 'name': 'web' + } + ], volumes={}, networks={}, secrets={}, configs={}) + + serialized_config = yaml.load(serialize_config(config_dict)) + assert '127.0.0.1:8080:80/tcp' in serialized_config['services']['web']['ports'] + def test_serialize_configs(self): service_dict = { 'image': 'example/web', From 8356576a9a090462a5469c3ca34b47ca0fafb36e Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Fri, 6 Apr 2018 17:46:23 -0700 Subject: [PATCH 02/53] Make sure error messages are unicode strings before combining Signed-off-by: Joffrey F --- compose/parallel.py | 11 ++++++----- compose/project.py | 6 +++++- tests/unit/project_test.py | 29 +++++++++++++++++++++++++++++ 3 files changed, 40 insertions(+), 6 deletions(-) diff --git a/compose/parallel.py b/compose/parallel.py index 5d4791f97..a2eb160e5 100644 --- a/compose/parallel.py +++ b/compose/parallel.py @@ -279,9 +279,7 @@ class ParallelStreamWriter(object): def write_initial(self, msg, obj_index): if msg is None: return - self.stream.write("{:<{width}} ... \r\n".format( - msg + ' ' + obj_index, width=self.width)) - self.stream.flush() + return self._write_noansi(msg, obj_index, '') def _write_ansi(self, msg, obj_index, status): self.lock.acquire() @@ -299,8 +297,11 @@ class ParallelStreamWriter(object): self.lock.release() def _write_noansi(self, msg, obj_index, status): - self.stream.write("{:<{width}} ... {}\r\n".format(msg + ' ' + obj_index, - status, width=self.width)) + self.stream.write( + "{:<{width}} ... {}\r\n".format( + msg + ' ' + obj_index, status, width=self.width + ) + ) self.stream.flush() def write(self, msg, obj_index, status, color_func): diff --git a/compose/project.py b/compose/project.py index afbec183f..924390b4e 100644 --- a/compose/project.py +++ b/compose/project.py @@ -556,7 +556,11 @@ class Project(object): limit=5, ) if len(errors): - raise ProjectError(b"\n".join(errors.values())) + combined_errors = '\n'.join([ + e.decode('utf-8') if isinstance(e, six.binary_type) else e for e in errors.values() + ]) + raise ProjectError(combined_errors) + else: for service in services: service.pull(ignore_pull_failures, silent=silent) diff --git a/tests/unit/project_test.py b/tests/unit/project_test.py index eb6209723..83a014758 100644 --- a/tests/unit/project_test.py +++ b/tests/unit/project_test.py @@ -5,6 +5,7 @@ from __future__ import unicode_literals import datetime import docker +import pytest from docker.errors import NotFound from .. import mock @@ -16,8 +17,10 @@ from compose.const import COMPOSEFILE_V2_0 as V2_0 from compose.const import COMPOSEFILE_V2_4 as V2_4 from compose.const import LABEL_SERVICE from compose.container import Container +from compose.errors import OperationFailedError from compose.project import NoSuchService from compose.project import Project +from compose.project import ProjectError from compose.service import ImageType from compose.service import Service @@ -588,3 +591,29 @@ class ProjectTest(unittest.TestCase): name='test', client=self.mock_client, config_data=config_data, default_platform='windows' ) assert project.get_service('web').options.get('platform') == 'linux/s390x' + + @mock.patch('compose.parallel.ParallelStreamWriter._write_noansi') + def test_error_parallel_pull(self, mock_write): + project = Project.from_config( + name='test', + client=self.mock_client, + config_data=Config( + version=V2_0, + services=[{ + 'name': 'web', + 'image': 'busybox:latest', + }], + networks=None, + volumes=None, + secrets=None, + configs=None, + ), + ) + + self.mock_client.pull.side_effect = OperationFailedError('pull error') + with pytest.raises(ProjectError): + project.pull(parallel_pull=True) + + self.mock_client.pull.side_effect = OperationFailedError(b'pull error') + with pytest.raises(ProjectError): + project.pull(parallel_pull=True) From 20a9ae50b00d6aeb2d68735a56549976e356e6f5 Mon Sep 17 00:00:00 2001 From: Harald Albers Date: Wed, 11 Apr 2018 12:47:10 +0200 Subject: [PATCH 03/53] Refactor bash completion for services Signed-off-by: Harald Albers --- contrib/completion/bash/docker-compose | 91 +++++++++++--------------- 1 file changed, 37 insertions(+), 54 deletions(-) diff --git a/contrib/completion/bash/docker-compose b/contrib/completion/bash/docker-compose index 90c9ce5fc..409796f63 100644 --- a/contrib/completion/bash/docker-compose +++ b/contrib/completion/bash/docker-compose @@ -81,41 +81,24 @@ __docker_compose_nospace() { type compopt &>/dev/null && compopt -o nospace } -# Extracts all service names from the compose file. -___docker_compose_all_services_in_compose_file() { - __docker_compose_q config --services + +# Outputs a list of all defined services, regardless of their running state. +# Arguments for `docker-compose ps` may be passed in order to filter the service list, +# e.g. `status=running`. +__docker_compose_services() { + __docker_compose_q ps --services "$@" } -# All services, even those without an existing container -__docker_compose_services_all() { - COMPREPLY=( $(compgen -W "$(___docker_compose_all_services_in_compose_file)" -- "$cur") ) -} - -# All services that are defined by a Dockerfile reference -__docker_compose_services_from_build() { - COMPREPLY=( $(compgen -W "$(__docker_compose_q ps --services --filter "source=build")" -- "$cur") ) -} - -# All services that are defined by an image -__docker_compose_services_from_image() { - COMPREPLY=( $(compgen -W "$(__docker_compose_q ps --services --filter "source=image")" -- "$cur") ) -} - -# The services for which at least one paused container exists -__docker_compose_services_paused() { - names=$(__docker_compose_q ps --services --filter "status=paused") - COMPREPLY=( $(compgen -W "$names" -- "$cur") ) +# Applies completion of services based on the current value of `$cur`. +# Arguments for `docker-compose ps` may be passed in order to filter the service list, +# see `__docker_compose_services`. +__docker_compose_complete_services() { + COMPREPLY=( $(compgen -W "$(__docker_compose_services "$@")" -- "$cur") ) } # The services for which at least one running container exists -__docker_compose_services_running() { - names=$(__docker_compose_q ps --services --filter "status=running") - COMPREPLY=( $(compgen -W "$names" -- "$cur") ) -} - -# The services for which at least one stopped container exists -__docker_compose_services_stopped() { - names=$(__docker_compose_q ps --services --filter "status=stopped") +__docker_compose_complete_running_services() { + local names=$(__docker_compose_complete_services --filter status=running) COMPREPLY=( $(compgen -W "$names" -- "$cur") ) } @@ -134,7 +117,7 @@ _docker_compose_build() { COMPREPLY=( $( compgen -W "--build-arg --force-rm --help --memory --no-cache --pull" -- "$cur" ) ) ;; *) - __docker_compose_services_from_build + __docker_compose_complete_services --filter source=build ;; esac } @@ -163,7 +146,7 @@ _docker_compose_create() { COMPREPLY=( $( compgen -W "--build --force-recreate --help --no-build --no-recreate" -- "$cur" ) ) ;; *) - __docker_compose_services_all + __docker_compose_complete_services ;; esac } @@ -234,7 +217,7 @@ _docker_compose_events() { COMPREPLY=( $( compgen -W "--help --json" -- "$cur" ) ) ;; *) - __docker_compose_services_all + __docker_compose_complete_services ;; esac } @@ -252,7 +235,7 @@ _docker_compose_exec() { COMPREPLY=( $( compgen -W "-d --detach --help --index --privileged -T --user -u" -- "$cur" ) ) ;; *) - __docker_compose_services_running + __docker_compose_complete_running_services ;; esac } @@ -268,7 +251,7 @@ _docker_compose_images() { COMPREPLY=( $( compgen -W "--help --quiet -q" -- "$cur" ) ) ;; *) - __docker_compose_services_all + __docker_compose_complete_services ;; esac } @@ -286,7 +269,7 @@ _docker_compose_kill() { COMPREPLY=( $( compgen -W "--help -s" -- "$cur" ) ) ;; *) - __docker_compose_services_running + __docker_compose_complete_running_services ;; esac } @@ -304,7 +287,7 @@ _docker_compose_logs() { COMPREPLY=( $( compgen -W "--follow -f --help --no-color --tail --timestamps -t" -- "$cur" ) ) ;; *) - __docker_compose_services_all + __docker_compose_complete_services ;; esac } @@ -316,7 +299,7 @@ _docker_compose_pause() { COMPREPLY=( $( compgen -W "--help" -- "$cur" ) ) ;; *) - __docker_compose_services_running + __docker_compose_complete_running_services ;; esac } @@ -338,7 +321,7 @@ _docker_compose_port() { COMPREPLY=( $( compgen -W "--help --index --protocol" -- "$cur" ) ) ;; *) - __docker_compose_services_all + __docker_compose_complete_services ;; esac } @@ -370,7 +353,7 @@ _docker_compose_ps() { COMPREPLY=( $( compgen -W "--help --quiet -q --services --filter" -- "$cur" ) ) ;; *) - __docker_compose_services_all + __docker_compose_complete_services ;; esac } @@ -382,7 +365,7 @@ _docker_compose_pull() { COMPREPLY=( $( compgen -W "--help --ignore-pull-failures --include-deps --parallel --quiet -q" -- "$cur" ) ) ;; *) - __docker_compose_services_from_image + __docker_compose_complete_services --filter source=image ;; esac } @@ -394,7 +377,7 @@ _docker_compose_push() { COMPREPLY=( $( compgen -W "--help --ignore-push-failures" -- "$cur" ) ) ;; *) - __docker_compose_services_all + __docker_compose_complete_services ;; esac } @@ -412,7 +395,7 @@ _docker_compose_restart() { COMPREPLY=( $( compgen -W "--help --timeout -t" -- "$cur" ) ) ;; *) - __docker_compose_services_running + __docker_compose_complete_running_services ;; esac } @@ -425,9 +408,9 @@ _docker_compose_rm() { ;; *) if __docker_compose_has_option "--stop|-s" ; then - __docker_compose_services_all + __docker_compose_complete_services else - __docker_compose_services_stopped + __docker_compose_complete_services --filter status=stopped fi ;; esac @@ -451,7 +434,7 @@ _docker_compose_run() { COMPREPLY=( $( compgen -W "--detach -d --entrypoint -e --help --label -l --name --no-deps --publish -p --rm --service-ports -T --use-aliases --user -u --volume -v --workdir -w" -- "$cur" ) ) ;; *) - __docker_compose_services_all + __docker_compose_complete_services ;; esac } @@ -473,7 +456,7 @@ _docker_compose_scale() { COMPREPLY=( $( compgen -W "--help --timeout -t" -- "$cur" ) ) ;; *) - COMPREPLY=( $(compgen -S "=" -W "$(___docker_compose_all_services_in_compose_file)" -- "$cur") ) + COMPREPLY=( $(compgen -S "=" -W "$(__docker_compose_services)" -- "$cur") ) __docker_compose_nospace ;; esac @@ -486,7 +469,7 @@ _docker_compose_start() { COMPREPLY=( $( compgen -W "--help" -- "$cur" ) ) ;; *) - __docker_compose_services_stopped + __docker_compose_complete_services --filter status=stopped ;; esac } @@ -504,7 +487,7 @@ _docker_compose_stop() { COMPREPLY=( $( compgen -W "--help --timeout -t" -- "$cur" ) ) ;; *) - __docker_compose_services_running + __docker_compose_complete_running_services ;; esac } @@ -516,7 +499,7 @@ _docker_compose_top() { COMPREPLY=( $( compgen -W "--help" -- "$cur" ) ) ;; *) - __docker_compose_services_running + __docker_compose_complete_running_services ;; esac } @@ -528,7 +511,7 @@ _docker_compose_unpause() { COMPREPLY=( $( compgen -W "--help" -- "$cur" ) ) ;; *) - __docker_compose_services_paused + __docker_compose_complete_services --filter status=paused ;; esac } @@ -541,11 +524,11 @@ _docker_compose_up() { return ;; --exit-code-from) - __docker_compose_services_all + __docker_compose_complete_services return ;; --scale) - COMPREPLY=( $(compgen -S "=" -W "$(___docker_compose_all_services_in_compose_file)" -- "$cur") ) + COMPREPLY=( $(compgen -S "=" -W "$(__docker_compose_services)" -- "$cur") ) __docker_compose_nospace return ;; @@ -559,7 +542,7 @@ _docker_compose_up() { COMPREPLY=( $( compgen -W "--abort-on-container-exit --always-recreate-deps --build -d --detach --exit-code-from --force-recreate --help --no-build --no-color --no-deps --no-recreate --no-start --renew-anon-volumes -V --remove-orphans --scale --timeout -t" -- "$cur" ) ) ;; *) - __docker_compose_services_all + __docker_compose_complete_services ;; esac } From ca396bac6d7220fe61e4255078f85ba4d9d02cfd Mon Sep 17 00:00:00 2001 From: Harald Albers Date: Thu, 12 Apr 2018 08:52:20 +0200 Subject: [PATCH 04/53] Add support for features added in 1.21.0 to bash completion - add support for `docker-compose exec --workdir|-w` - add support for `docker-compose build --compress` - add support for `docker-compose pull --no-parallel`, drop deprecated option `--parallel` Signed-off-by: Harald Albers --- contrib/completion/bash/docker-compose | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/contrib/completion/bash/docker-compose b/contrib/completion/bash/docker-compose index 90c9ce5fc..0713486d3 100644 --- a/contrib/completion/bash/docker-compose +++ b/contrib/completion/bash/docker-compose @@ -131,7 +131,7 @@ _docker_compose_build() { case "$cur" in -*) - COMPREPLY=( $( compgen -W "--build-arg --force-rm --help --memory --no-cache --pull" -- "$cur" ) ) + COMPREPLY=( $( compgen -W "--build-arg --compress --force-rm --help --memory --no-cache --pull" -- "$cur" ) ) ;; *) __docker_compose_services_from_build @@ -242,14 +242,14 @@ _docker_compose_events() { _docker_compose_exec() { case "$prev" in - --index|--user|-u) + --index|--user|-u|--workdir|-w) return ;; esac case "$cur" in -*) - COMPREPLY=( $( compgen -W "-d --detach --help --index --privileged -T --user -u" -- "$cur" ) ) + COMPREPLY=( $( compgen -W "-d --detach --help --index --privileged -T --user -u --workdir -w" -- "$cur" ) ) ;; *) __docker_compose_services_running @@ -379,7 +379,7 @@ _docker_compose_ps() { _docker_compose_pull() { case "$cur" in -*) - COMPREPLY=( $( compgen -W "--help --ignore-pull-failures --include-deps --parallel --quiet -q" -- "$cur" ) ) + COMPREPLY=( $( compgen -W "--help --ignore-pull-failures --include-deps --no-parallel --quiet -q" -- "$cur" ) ) ;; *) __docker_compose_services_from_image From 7078c8740a999c794d7eefa38b7077ee72f2012c Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Thu, 12 Apr 2018 11:31:44 -0700 Subject: [PATCH 05/53] Bump 1.22.0 dev Signed-off-by: Joffrey F --- compose/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/compose/__init__.py b/compose/__init__.py index a5c1364a6..5eb2efd03 100644 --- a/compose/__init__.py +++ b/compose/__init__.py @@ -1,4 +1,4 @@ from __future__ import absolute_import from __future__ import unicode_literals -__version__ = '1.21.0dev' +__version__ = '1.22.0dev' From fc923c3580f88f3743d9b0bd91cae7ae64e6a2e1 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Tue, 17 Apr 2018 12:13:51 -0700 Subject: [PATCH 06/53] Update .gitignore Signed-off-by: Joffrey F --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index ef04ca15f..11266c2e3 100644 --- a/.gitignore +++ b/.gitignore @@ -12,3 +12,4 @@ compose/GITSHA *.swo *.swp .DS_Store +.cache From b1c831c54ae6808a9df2dac7890f5d7dbd9f8b9d Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Wed, 18 Apr 2018 16:01:52 -0700 Subject: [PATCH 07/53] Inital pass on comprehensive automated release script Signed-off-by: Joffrey F --- script/release/release.md.tmpl | 34 +++++ script/release/release.py | 197 +++++++++++++++++++++++++++ script/release/release/__init__.py | 0 script/release/release/bintray.py | 40 ++++++ script/release/release/const.py | 9 ++ script/release/release/downloader.py | 72 ++++++++++ script/release/release/repository.py | 161 ++++++++++++++++++++++ script/release/release/utils.py | 63 +++++++++ 8 files changed, 576 insertions(+) create mode 100644 script/release/release.md.tmpl create mode 100755 script/release/release.py create mode 100644 script/release/release/__init__.py create mode 100644 script/release/release/bintray.py create mode 100644 script/release/release/const.py create mode 100644 script/release/release/downloader.py create mode 100644 script/release/release/repository.py create mode 100644 script/release/release/utils.py diff --git a/script/release/release.md.tmpl b/script/release/release.md.tmpl new file mode 100644 index 000000000..ee97ef104 --- /dev/null +++ b/script/release/release.md.tmpl @@ -0,0 +1,34 @@ +If you're a Mac or Windows user, the best way to install Compose and keep it up-to-date is **[Docker for Mac and Windows](https://www.docker.com/products/docker)**. + +Docker for Mac and Windows will automatically install the latest version of Docker Engine for you. + +Alternatively, you can use the usual commands to install or upgrade Compose: + +``` +curl -L https://github.com/docker/compose/releases/download/{{version}}/docker-compose-`uname -s`-`uname -m` -o /usr/local/bin/docker-compose +chmod +x /usr/local/bin/docker-compose +``` + +See the [install docs](https://docs.docker.com/compose/install/) for more install options and instructions. + +## Compose file format compatibility matrix + +| Compose file format | Docker Engine | +| --- | --- | +{% for engine, formats in compat_matrix.items() -%} +| {% for format in formats %}{{format}}{% if not loop.last %}, {% endif %}{% endfor %} | {{engine}}+ | +{% endfor -%} + +## Changes + +{{changelog}} + +Thanks to {% for name in contributors %}@{{name}}{% if not loop.last %}, {% endif %}{% endfor %} for contributing to this release! + +## Integrity check + +Binary name | SHA-256 sum +| --- | --- | +{% for filename, sha in integrity.items() -%} +| `{{filename}}` | `{{sha[1]}}` | +{% endfor -%} diff --git a/script/release/release.py b/script/release/release.py new file mode 100755 index 000000000..f23146288 --- /dev/null +++ b/script/release/release.py @@ -0,0 +1,197 @@ +from __future__ import absolute_import +from __future__ import print_function +from __future__ import unicode_literals + +import argparse +import os +import sys +import time + +from jinja2 import Template +from release.bintray import BintrayAPI +from release.const import BINTRAY_ORG +from release.const import NAME +from release.const import REPO_ROOT +from release.downloader import BinaryDownloader +from release.repository import get_contributors +from release.repository import Repository +from release.repository import upload_assets +from release.utils import branch_name +from release.utils import compatibility_matrix +from release.utils import read_release_notes_from_changelog +from release.utils import ScriptError +from release.utils import update_init_py_version +from release.utils import update_run_sh_version + + +def create_initial_branch(repository, release, base, bintray_user): + release_branch = repository.create_release_branch(release, base) + print('Updating version info in __init__.py and run.sh') + update_run_sh_version(release) + update_init_py_version(release) + + input('Please add the release notes to the CHANGELOG.md file, then press Enter to continue.') + proceed = '' + while proceed.lower() != 'y': + print(repository.diff()) + proceed = input('Are these changes ok? y/N ') + + repository.create_bump_commit(release_branch, release) + repository.push_branch_to_remote(release_branch) + + bintray_api = BintrayAPI(os.environ['BINTRAY_TOKEN'], bintray_user) + print('Creating data repository {} on bintray'.format(release_branch.name)) + bintray_api.create_repository(BINTRAY_ORG, release_branch.name, 'generic') + + +def monitor_pr_status(pr_data): + print('Waiting for CI to complete...') + last_commit = pr_data.get_commits().reversed[0] + while True: + status = last_commit.get_combined_status() + if status.state == 'pending': + summary = { + 'pending': 0, + 'success': 0, + 'failure': 0, + } + for detail in status.statuses: + summary[detail.state] += 1 + print('{pending} pending, {success} successes, {failure} failures'.format(**summary)) + if status.total_count == 0: + # Mostly for testing purposes against repos with no CI setup + return True + time.sleep(30) + elif status.state == 'success': + print('{} successes: all clear!'.format(status.total_count)) + return True + else: + raise ScriptError('CI failure detected') + + +def create_release_draft(repository, version, pr_data, files): + print('Creating Github release draft') + with open(os.path.join(os.path.dirname(__file__), 'release.md.tmpl'), 'r') as f: + template = Template(f.read()) + print('Rendering release notes based on template') + release_notes = template.render( + version=version, + compat_matrix=compatibility_matrix(), + integrity=files, + contributors=get_contributors(pr_data), + changelog=read_release_notes_from_changelog(), + ) + gh_release = repository.create_release( + version, release_notes, draft=True, prerelease='-rc' in version, + target_commitish='release' + ) + print('Release draft initialized') + return gh_release + + +def resume(args): + raise NotImplementedError() + try: + repository = Repository(REPO_ROOT, args.repo or NAME) + br_name = branch_name(args.release) + if not repository.branch_exists(br_name): + raise ScriptError('No local branch exists for this release.') + # release_branch = repository.checkout_branch(br_name) + except ScriptError as e: + print(e) + return 1 + return 0 + + +def cancel(args): + try: + repository = Repository(REPO_ROOT, args.repo or NAME) + repository.close_release_pr(args.release) + repository.remove_release(args.release) + repository.remove_bump_branch(args.release) + # TODO: uncomment after testing is complete + # bintray_api = BintrayAPI(os.environ['BINTRAY_TOKEN'], args.bintray_user) + # print('Removing Bintray data repository for {}'.format(args.release)) + # bintray_api.delete_repository(BINTRAY_ORG, branch_name(args.release)) + except ScriptError as e: + print(e) + return 1 + print('Release cancellation complete.') + return 0 + + +def start(args): + try: + repository = Repository(REPO_ROOT, args.repo or NAME) + create_initial_branch(repository, args.release, args.base, args.bintray_user) + pr_data = repository.create_release_pull_request(args.release) + monitor_pr_status(pr_data) + downloader = BinaryDownloader(args.destination) + files = downloader.download_all(args.release) + gh_release = create_release_draft(repository, args.release, pr_data, files) + upload_assets(gh_release, files) + except ScriptError as e: + print(e) + return 1 + + return 0 + + +def main(): + if 'GITHUB_TOKEN' not in os.environ: + print('GITHUB_TOKEN environment variable must be set') + return 1 + + if 'BINTRAY_TOKEN' not in os.environ: + print('BINTRAY_TOKEN environment variable must be set') + return 1 + + parser = argparse.ArgumentParser( + description='Orchestrate a new release of docker/compose. This tool assumes that you have' + 'obtained a Github API token and Bintray API key and set the GITHUB_TOKEN and' + 'BINTRAY_TOKEN environment variables accordingly.', + epilog='''Example uses: + * Start a new feature release (includes all changes currently in master) + release.py -b user start 1.23.0 + * Start a new patch release + release.py -b user --patch 1.21.0 start 1.21.1 + * Cancel / rollback an existing release draft + release.py -b user cancel 1.23.0 + * Restart a previously aborted patch release + release.py -b user -p 1.21.0 resume 1.21.1 + ''', formatter_class=argparse.RawTextHelpFormatter) + parser.add_argument( + 'action', choices=['start', 'resume', 'cancel'], + help='The action to be performed for this release' + ) + parser.add_argument('release', help='Release number, e.g. 1.9.0-rc1, 2.1.1') + parser.add_argument( + '--patch', '-p', dest='base', + help='Which version is being patched by this release' + ) + parser.add_argument( + '--repo', '-r', dest='repo', + help='Start a release for the given repo (default: {})'.format(NAME) + ) + parser.add_argument( + '-b', dest='bintray_user', required=True, metavar='USER', + help='Username associated with the Bintray API key' + ) + parser.add_argument( + '--destination', '-o', metavar='DIR', default='binaries', + help='Directory where release binaries will be downloaded relative to the project root' + ) + args = parser.parse_args() + + if args.action == 'start': + return start(args) + elif args.action == 'resume': + return resume(args) + elif args.action == 'cancel': + return cancel(args) + print('Unexpected action "{}"'.format(args.action), file=sys.stderr) + return 1 + + +if __name__ == '__main__': + sys.exit(main()) diff --git a/script/release/release/__init__.py b/script/release/release/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/script/release/release/bintray.py b/script/release/release/bintray.py new file mode 100644 index 000000000..d99d372c6 --- /dev/null +++ b/script/release/release/bintray.py @@ -0,0 +1,40 @@ +from __future__ import absolute_import +from __future__ import unicode_literals + +import json + +import requests + +from .const import NAME + + +class BintrayAPI(requests.Session): + def __init__(self, api_key, user, *args, **kwargs): + super(BintrayAPI, self).__init__(*args, **kwargs) + self.auth = (user, api_key) + self.base_url = 'https://api.bintray.com/' + + def create_repository(self, subject, repo_name, repo_type='generic'): + url = '{base}/repos/{subject}/{repo_name}'.format( + base=self.base_url, subject=subject, repo_name=repo_name, + ) + data = { + 'name': repo_name, + 'type': repo_type, + 'private': False, + 'desc': 'Automated release for {}: {}'.format(NAME, repo_name), + 'labels': ['docker-compose', 'docker', 'release-bot'], + } + return self.post_json(url, data) + + def delete_repository(self, subject, repo_name): + url = '{base}/repos/{subject}/{repo_name}'.format( + base=self.base_url, subject=subject, repo_name=repo_name, + ) + return self.delete(url) + + def post_json(self, url, data, **kwargs): + if 'headers' not in kwargs: + kwargs['headers'] = {} + kwargs['headers']['Content-Type'] = 'application/json' + return self.post(url, data=json.dumps(data), **kwargs) diff --git a/script/release/release/const.py b/script/release/release/const.py new file mode 100644 index 000000000..34f338a89 --- /dev/null +++ b/script/release/release/const.py @@ -0,0 +1,9 @@ +from __future__ import absolute_import +from __future__ import unicode_literals + +import os + + +REPO_ROOT = os.path.join(os.path.dirname(__file__), '..', '..', '..') +NAME = 'shin-/compose' +BINTRAY_ORG = 'shin-compose' diff --git a/script/release/release/downloader.py b/script/release/release/downloader.py new file mode 100644 index 000000000..cd43bc993 --- /dev/null +++ b/script/release/release/downloader.py @@ -0,0 +1,72 @@ +from __future__ import absolute_import +from __future__ import print_function +from __future__ import unicode_literals + +import hashlib +import os + +import requests + +from .const import BINTRAY_ORG +from .const import NAME +from .const import REPO_ROOT +from .utils import branch_name + + +class BinaryDownloader(requests.Session): + base_bintray_url = 'https://dl.bintray.com/{}'.format(BINTRAY_ORG) + base_appveyor_url = 'https://ci.appveyor.com/api/projects/{}/artifacts/'.format(NAME) + + def __init__(self, destination, *args, **kwargs): + super(BinaryDownloader, self).__init__(*args, **kwargs) + self.destination = destination + os.makedirs(self.destination, exist_ok=True) + + def download_from_bintray(self, repo_name, filename): + print('Downloading {} from bintray'.format(filename)) + url = '{base}/{repo_name}/{filename}'.format( + base=self.base_bintray_url, repo_name=repo_name, filename=filename + ) + full_dest = os.path.join(REPO_ROOT, self.destination, filename) + return self._download(url, full_dest) + + def download_from_appveyor(self, branch_name, filename): + print('Downloading {} from appveyor'.format(filename)) + url = '{base}/dist%2F{filename}?branch={branch_name}'.format( + base=self.base_appveyor_url, filename=filename, branch_name=branch_name + ) + full_dest = os.path.join(REPO_ROOT, self.destination, filename) + return self.download(url, full_dest) + + def _download(self, url, full_dest): + m = hashlib.sha256() + with open(full_dest, 'wb') as f: + r = self.get(url, stream=True) + for chunk in r.iter_content(chunk_size=1024 * 600, decode_unicode=False): + print('.', end='', flush=True) + m.update(chunk) + f.write(chunk) + + print(' download complete') + hex_digest = m.hexdigest() + with open(full_dest + '.sha256', 'w') as f: + f.write('{} {}\n'.format(hex_digest, os.path.basename(full_dest))) + return full_dest, hex_digest + + def download_all(self, version): + files = { + 'docker-compose-Darwin-x86_64': None, + 'docker-compose-Linux-x86_64': None, + # 'docker-compose-Windows-x86_64.exe': None, + } + + for filename in files.keys(): + if 'Windows' in filename: + files[filename] = self.download_from_appveyor( + branch_name(version), filename + ) + else: + files[filename] = self.download_from_bintray( + branch_name(version), filename + ) + return files diff --git a/script/release/release/repository.py b/script/release/release/repository.py new file mode 100644 index 000000000..77c697a99 --- /dev/null +++ b/script/release/release/repository.py @@ -0,0 +1,161 @@ +from __future__ import absolute_import +from __future__ import unicode_literals + +import os + +from git import GitCommandError +from git import Repo +from github import Github + +from .const import NAME +from .const import REPO_ROOT +from .utils import branch_name +from .utils import read_release_notes_from_changelog +from .utils import ScriptError + + +class Repository(object): + def __init__(self, root=None, gh_name=None): + if root is None: + root = REPO_ROOT + if gh_name is None: + gh_name = NAME + self.git_repo = Repo(root) + self.gh_client = Github(os.environ['GITHUB_TOKEN']) + self.gh_repo = self.gh_client.get_repo(gh_name) + + def create_release_branch(self, version, base=None): + print('Creating release branch {} based on {}...'.format(version, base or 'master')) + remote = self.find_remote(self.gh_repo.full_name) + remote.fetch() + if self.branch_exists(branch_name(version)): + raise ScriptError( + "Branch {} already exists locally. " + "Please remove it before running the release script.".format(branch_name(version)) + ) + if base is not None: + base = self.git_repo.tag('refs/tags/{}'.format(base)) + else: + base = 'refs/remotes/{}/master'.format(remote.name) + release_branch = self.git_repo.create_head(branch_name(version), commit=base) + release_branch.checkout() + self.git_repo.git.merge('--strategy=ours', '--no-edit', '{}/release'.format(remote.name)) + with release_branch.config_writer() as cfg: + cfg.set_value('release', version) + return release_branch + + def find_remote(self, remote_name=None): + if not remote_name: + remote_name = self.gh_repo.full_name + for remote in self.git_repo.remotes: + for url in remote.urls: + if remote_name in url: + return remote + return None + + def create_bump_commit(self, bump_branch, version): + print('Creating bump commit...') + bump_branch.checkout() + self.git_repo.git.commit('-a', '-s', '-m "Bump {}"'.format(version), '--no-verify') + + def diff(self): + return self.git_repo.git.diff() + + def checkout_branch(self, name): + return self.git_repo.branches[name].checkout() + + def push_branch_to_remote(self, branch, remote_name=None): + print('Pushing branch {} to remote...'.format(branch.name)) + remote = self.find_remote(remote_name) + remote.push(refspec=branch) + + def branch_exists(self, name): + return name in [h.name for h in self.git_repo.heads] + + def create_release_pull_request(self, version): + return self.gh_repo.create_pull( + title='Bump {}'.format(version), + body='Automated release for docker-compose {}\n\n{}'.format( + version, read_release_notes_from_changelog() + ), + base='release', + head=branch_name(version), + ) + + def create_release(self, version, release_notes, **kwargs): + return self.gh_repo.create_git_release( + tag=version, name=version, message=release_notes, **kwargs + ) + + def remove_release(self, version): + print('Removing release draft for {}'.format(version)) + releases = self.gh_repo.get_releases() + for release in releases: + if release.tag_name == version and release.title == version: + if not release.draft: + print( + 'The release at {} is no longer a draft. If you TRULY intend ' + 'to remove it, please do so manually.' + ) + continue + release.delete_release() + + def remove_bump_branch(self, version, remote_name=None): + name = branch_name(version) + if not self.branch_exists(name): + return False + print('Removing local branch "{}"'.format(name)) + if self.git_repo.active_branch.name == name: + print('Active branch is about to be deleted. Checking out to master...') + try: + self.checkout_branch('master') + except GitCommandError: + raise ScriptError( + 'Unable to checkout master. Try stashing local changes before proceeding.' + ) + self.git_repo.branches[name].delete(self.git_repo, name, force=True) + print('Removing remote branch "{}"'.format(name)) + remote = self.find_remote(remote_name) + try: + remote.push(name, delete=True) + except GitCommandError as e: + if 'remote ref does not exist' in str(e): + return False + raise ScriptError( + 'Error trying to remove remote branch: {}'.format(e) + ) + return True + + def close_release_pr(self, version): + print('Retrieving and closing release PR for {}'.format(version)) + name = branch_name(version) + open_prs = self.gh_repo.get_pulls(state='open') + count = 0 + for pr in open_prs: + if pr.head.ref == name: + print('Found matching PR #{}'.format(pr.number)) + pr.edit(state='closed') + count += 1 + if count == 0: + print('No open PR for this release branch.') + return count + + +def get_contributors(pr_data): + commits = pr_data.get_commits() + authors = {} + for commit in commits: + author = commit.author.login + authors[author] = authors.get(author, 0) + 1 + return [x[0] for x in sorted(list(authors.items()), key=lambda x: x[1])] + + +def upload_assets(gh_release, files): + print('Uploading binaries and hash sums') + for filename, filedata in files.items(): + print('Uploading {}...'.format(filename)) + gh_release.upload_asset(filedata[0], content_type='application/octet-stream') + gh_release.upload_asset('{}.sha256'.format(filedata[0]), content_type='text/plain') + gh_release.upload_asset( + os.path.join(REPO_ROOT, 'script', 'run', 'run.sh'), content_type='text/plain' + ) diff --git a/script/release/release/utils.py b/script/release/release/utils.py new file mode 100644 index 000000000..b0e1f6a84 --- /dev/null +++ b/script/release/release/utils.py @@ -0,0 +1,63 @@ +from __future__ import absolute_import +from __future__ import unicode_literals + +import os +import re + +from .const import REPO_ROOT +from compose import const as compose_const + +section_header_re = re.compile(r'^[0-9]+\.[0-9]+\.[0-9]+ \([0-9]{4}-[01][0-9]-[0-3][0-9]\)$') + + +class ScriptError(Exception): + pass + + +def branch_name(version): + return 'bump-{}'.format(version) + + +def read_release_notes_from_changelog(): + with open(os.path.join(REPO_ROOT, 'CHANGELOG.md'), 'r') as f: + lines = f.readlines() + i = 0 + while i < len(lines): + if section_header_re.match(lines[i]): + break + i += 1 + + j = i + 1 + while j < len(lines): + if section_header_re.match(lines[j]): + break + j += 1 + + return ''.join(lines[i + 2:j - 1]) + + +def update_init_py_version(version): + path = os.path.join(REPO_ROOT, 'compose', '__init__.py') + with open(path, 'r') as f: + contents = f.read() + contents = re.sub(r"__version__ = '[0-9a-z.-]+'", "__version__ = '{}'".format(version), contents) + with open(path, 'w') as f: + f.write(contents) + + +def update_run_sh_version(version): + path = os.path.join(REPO_ROOT, 'script', 'run', 'run.sh') + with open(path, 'r') as f: + contents = f.read() + contents = re.sub(r'VERSION="[0-9a-z.-]+"', 'VERSION="{}"'.format(version), contents) + with open(path, 'w') as f: + f.write(contents) + + +def compatibility_matrix(): + result = {} + for engine_version in compose_const.API_VERSION_TO_ENGINE_VERSION.values(): + result[engine_version] = [] + for fmt, api_version in compose_const.API_VERSIONS.items(): + result[compose_const.API_VERSION_TO_ENGINE_VERSION[api_version]].append(fmt.vstring) + return result From ae6dd8a93c22ddda0a193ae1935b4b1df0300f08 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Wed, 18 Apr 2018 16:58:24 -0700 Subject: [PATCH 08/53] Implement resuming a release Signed-off-by: Joffrey F --- script/release/release.py | 52 ++++++++++++++++++++++++---- script/release/release/repository.py | 21 ++++++++++- 2 files changed, 65 insertions(+), 8 deletions(-) diff --git a/script/release/release.py b/script/release/release.py index f23146288..aa6c6198b 100755 --- a/script/release/release.py +++ b/script/release/release.py @@ -26,6 +26,12 @@ from release.utils import update_run_sh_version def create_initial_branch(repository, release, base, bintray_user): release_branch = repository.create_release_branch(release, base) + return create_bump_commit(repository, release_branch, bintray_user) + + +def create_bump_commit(repository, release_branch, bintray_user): + with release_branch.config_reader() as cfg: + release = cfg.get('release') print('Updating version info in __init__.py and run.sh') update_run_sh_version(release) update_init_py_version(release) @@ -36,7 +42,8 @@ def create_initial_branch(repository, release, base, bintray_user): print(repository.diff()) proceed = input('Are these changes ok? y/N ') - repository.create_bump_commit(release_branch, release) + if repository.diff(): + repository.create_bump_commit(release_branch, release) repository.push_branch_to_remote(release_branch) bintray_api = BintrayAPI(os.environ['BINTRAY_TOKEN'], bintray_user) @@ -89,17 +96,48 @@ def create_release_draft(repository, version, pr_data, files): return gh_release +def print_final_instructions(gh_release): + print(""" +You're almost done! The following steps should be executed after you've +verified that everything is in order and are ready to make the release public: +1. +2. +3.""") + + def resume(args): - raise NotImplementedError() try: repository = Repository(REPO_ROOT, args.repo or NAME) br_name = branch_name(args.release) if not repository.branch_exists(br_name): raise ScriptError('No local branch exists for this release.') - # release_branch = repository.checkout_branch(br_name) + release_branch = repository.checkout_branch(br_name) + create_bump_commit(repository, release_branch, args.bintray_user) + pr_data = repository.find_release_pr(args.release) + if not pr_data: + pr_data = repository.create_release_pull_request(args.release) + monitor_pr_status(pr_data) + downloader = BinaryDownloader(args.destination) + files = downloader.download_all(args.release) + gh_release = repository.find_release(args.release) + if not gh_release: + gh_release = create_release_draft(repository, args.release, pr_data, files) + elif not gh_release.draft: + print('WARNING!! Found non-draft (public) release for this version!') + proceed = input( + 'Are you sure you wish to proceed? Modifying an already ' + 'released version is dangerous! y/N' + ) + if proceed.lower() != 'y': + raise ScriptError('Aborting release') + for asset in gh_release.get_assets(): + asset.delete_asset() + upload_assets(gh_release, files) except ScriptError as e: print(e) return 1 + + print_final_instructions(gh_release) return 0 @@ -134,6 +172,7 @@ def start(args): print(e) return 1 + print_final_instructions(gh_release) return 0 @@ -147,8 +186,8 @@ def main(): return 1 parser = argparse.ArgumentParser( - description='Orchestrate a new release of docker/compose. This tool assumes that you have' - 'obtained a Github API token and Bintray API key and set the GITHUB_TOKEN and' + description='Orchestrate a new release of docker/compose. This tool assumes that you have ' + 'obtained a Github API token and Bintray API key and set the GITHUB_TOKEN and ' 'BINTRAY_TOKEN environment variables accordingly.', epilog='''Example uses: * Start a new feature release (includes all changes currently in master) @@ -158,8 +197,7 @@ def main(): * Cancel / rollback an existing release draft release.py -b user cancel 1.23.0 * Restart a previously aborted patch release - release.py -b user -p 1.21.0 resume 1.21.1 - ''', formatter_class=argparse.RawTextHelpFormatter) + release.py -b user -p 1.21.0 resume 1.21.1''', formatter_class=argparse.RawTextHelpFormatter) parser.add_argument( 'action', choices=['start', 'resume', 'cancel'], help='The action to be performed for this release' diff --git a/script/release/release/repository.py b/script/release/release/repository.py index 77c697a99..18c2dbf2c 100644 --- a/script/release/release/repository.py +++ b/script/release/release/repository.py @@ -67,7 +67,7 @@ class Repository(object): def push_branch_to_remote(self, branch, remote_name=None): print('Pushing branch {} to remote...'.format(branch.name)) remote = self.find_remote(remote_name) - remote.push(refspec=branch) + remote.push(refspec=branch, force=True) def branch_exists(self, name): return name in [h.name for h in self.git_repo.heads] @@ -87,6 +87,14 @@ class Repository(object): tag=version, name=version, message=release_notes, **kwargs ) + def find_release(self, version): + print('Retrieving release draft for {}'.format(version)) + releases = self.gh_repo.get_releases() + for release in releases: + if release.tag_name == version and release.title == version: + return release + return None + def remove_release(self, version): print('Removing release draft for {}'.format(version)) releases = self.gh_repo.get_releases() @@ -126,6 +134,17 @@ class Repository(object): ) return True + def find_release_pr(self, version): + print('Retrieving release PR for {}'.format(version)) + name = branch_name(version) + open_prs = self.gh_repo.get_pulls(state='open') + for pr in open_prs: + if pr.head.ref == name: + print('Found matching PR #{}'.format(pr.number)) + return pr + print('No open PR for this release branch.') + return None + def close_release_pr(self, version): print('Retrieving and closing release PR for {}'.format(version)) name = branch_name(version) From 6a710405141a9be99da1d3ca4facca8535669d8e Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Wed, 18 Apr 2018 17:02:15 -0700 Subject: [PATCH 09/53] Temp test Signed-off-by: Joffrey F --- script/release/release/repository.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/script/release/release/repository.py b/script/release/release/repository.py index 18c2dbf2c..c84c9e1b5 100644 --- a/script/release/release/repository.py +++ b/script/release/release/repository.py @@ -36,7 +36,7 @@ class Repository(object): if base is not None: base = self.git_repo.tag('refs/tags/{}'.format(base)) else: - base = 'refs/remotes/{}/master'.format(remote.name) + base = 'refs/remotes/{}/automated-releases'.format(remote.name) release_branch = self.git_repo.create_head(branch_name(version), commit=base) release_branch.checkout() self.git_repo.git.merge('--strategy=ours', '--no-edit', '{}/release'.format(remote.name)) From 599456378bb8af3da4d48e9d4a73c35fb2194adf Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Wed, 18 Apr 2018 17:07:41 -0700 Subject: [PATCH 10/53] Added logging for asset removal Signed-off-by: Joffrey F --- script/release/release.py | 4 ++-- script/release/release/repository.py | 7 +++++++ 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/script/release/release.py b/script/release/release.py index aa6c6198b..b72fb2546 100755 --- a/script/release/release.py +++ b/script/release/release.py @@ -13,6 +13,7 @@ from release.const import BINTRAY_ORG from release.const import NAME from release.const import REPO_ROOT from release.downloader import BinaryDownloader +from release.repository import delete_assets from release.repository import get_contributors from release.repository import Repository from release.repository import upload_assets @@ -130,8 +131,7 @@ def resume(args): ) if proceed.lower() != 'y': raise ScriptError('Aborting release') - for asset in gh_release.get_assets(): - asset.delete_asset() + delete_assets(gh_release) upload_assets(gh_release, files) except ScriptError as e: print(e) diff --git a/script/release/release/repository.py b/script/release/release/repository.py index c84c9e1b5..d7034f8bd 100644 --- a/script/release/release/repository.py +++ b/script/release/release/repository.py @@ -178,3 +178,10 @@ def upload_assets(gh_release, files): gh_release.upload_asset( os.path.join(REPO_ROOT, 'script', 'run', 'run.sh'), content_type='text/plain' ) + + +def delete_assets(gh_release): + print('Removing previously uploaded assets') + for asset in gh_release.get_assets(): + print('Deleting asset {}'.format(asset.name)) + asset.delete_asset() From e9f6abf8f4c6de2295b28df25dd01a46b60a0741 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Wed, 18 Apr 2018 18:33:20 -0700 Subject: [PATCH 11/53] Add images build step and finalize placeholder Signed-off-by: Joffrey F --- script/release/release.py | 116 ++++++++++++++++++++++----- script/release/release/repository.py | 5 ++ 2 files changed, 101 insertions(+), 20 deletions(-) diff --git a/script/release/release.py b/script/release/release.py index b72fb2546..848ab9751 100755 --- a/script/release/release.py +++ b/script/release/release.py @@ -4,9 +4,11 @@ from __future__ import unicode_literals import argparse import os +import shutil import sys import time +import docker from jinja2 import Template from release.bintray import BintrayAPI from release.const import BINTRAY_ORG @@ -77,6 +79,15 @@ def monitor_pr_status(pr_data): raise ScriptError('CI failure detected') +def check_pr_mergeable(pr_data): + if not pr_data.mergeable: + print( + 'WARNING!! PR #{} can not currently be merged. You will need to ' + 'resolve the conflicts manually before finalizing the release.'.format(pr_data.number) + ) + return pr_data.mergeable + + def create_release_draft(repository, version, pr_data, files): print('Creating Github release draft') with open(os.path.join(os.path.dirname(__file__), 'release.md.tmpl'), 'r') as f: @@ -97,13 +108,51 @@ def create_release_draft(repository, version, pr_data, files): return gh_release -def print_final_instructions(gh_release): - print(""" -You're almost done! The following steps should be executed after you've -verified that everything is in order and are ready to make the release public: -1. -2. -3.""") +def build_images(repository, files, version): + print("Building release images...") + repository.write_git_sha() + docker_client = docker.APIClient(**docker.utils.kwargs_from_env()) + distdir = os.path.join(REPO_ROOT, 'dist') + os.makedirs(distdir, exist_ok=True) + shutil.copy(files['docker-compose-Linux-x86_64'][0], distdir) + print('Building docker/compose image') + logstream = docker_client.build( + REPO_ROOT, tag='docker/compose:{}'.format(version), dockerfile='Dockerfile.run', + decode=True + ) + for chunk in logstream: + if 'error' in chunk: + raise ScriptError('Build error: {}'.format(chunk['error'])) + if 'stream' in chunk: + print(chunk['stream'], end='') + + print('Building test image (for UCP e2e)') + logstream = docker_client.build( + REPO_ROOT, tag='docker-compose-tests:tmp', decode=True + ) + for chunk in logstream: + if 'error' in chunk: + raise ScriptError('Build error: {}'.format(chunk['error'])) + if 'stream' in chunk: + print(chunk['stream'], end='') + + container = docker_client.create_container( + 'docker-compose-tests:tmp', entrypoint='tox' + ) + docker_client.commit(container, 'docker/compose-tests:latest') + docker_client.tag('docker/compose-tests:latest', 'docker/compose-tests:{}'.format(version)) + docker_client.remove_container(container, force=True) + docker_client.remove_image('docker-compose-tests:tmp', force=True) + + +def print_final_instructions(args): + print( + "You're almost done! Please verify that everything is in order and " + "you are ready to make the release public, then run the following " + "command:\n{exe} -b {user} finalize {version}".format( + exe=sys.argv[0], user=args.bintray_user, version=args.release + ) + ) def resume(args): @@ -117,6 +166,7 @@ def resume(args): pr_data = repository.find_release_pr(args.release) if not pr_data: pr_data = repository.create_release_pull_request(args.release) + check_pr_mergeable(pr_data) monitor_pr_status(pr_data) downloader = BinaryDownloader(args.destination) files = downloader.download_all(args.release) @@ -133,11 +183,12 @@ def resume(args): raise ScriptError('Aborting release') delete_assets(gh_release) upload_assets(gh_release, files) + build_images(repository, files, args.release) except ScriptError as e: print(e) return 1 - print_final_instructions(gh_release) + print_final_instructions(args) return 0 @@ -163,19 +214,50 @@ def start(args): repository = Repository(REPO_ROOT, args.repo or NAME) create_initial_branch(repository, args.release, args.base, args.bintray_user) pr_data = repository.create_release_pull_request(args.release) + check_pr_mergeable(pr_data) monitor_pr_status(pr_data) downloader = BinaryDownloader(args.destination) files = downloader.download_all(args.release) gh_release = create_release_draft(repository, args.release, pr_data, files) upload_assets(gh_release, files) + build_images(repository, files, args.release) except ScriptError as e: print(e) return 1 - print_final_instructions(gh_release) + print_final_instructions(args) return 0 +def finalize(args): + try: + raise NotImplementedError() + except ScriptError as e: + print(e) + return 1 + + return 0 + + +ACTIONS = [ + 'start', + 'cancel', + 'resume', + 'finalize', +] + +EPILOG = '''Example uses: + * Start a new feature release (includes all changes currently in master) + release.py -b user start 1.23.0 + * Start a new patch release + release.py -b user --patch 1.21.0 start 1.21.1 + * Cancel / rollback an existing release draft + release.py -b user cancel 1.23.0 + * Restart a previously aborted patch release + release.py -b user -p 1.21.0 resume 1.21.1 +''' + + def main(): if 'GITHUB_TOKEN' not in os.environ: print('GITHUB_TOKEN environment variable must be set') @@ -189,18 +271,9 @@ def main(): description='Orchestrate a new release of docker/compose. This tool assumes that you have ' 'obtained a Github API token and Bintray API key and set the GITHUB_TOKEN and ' 'BINTRAY_TOKEN environment variables accordingly.', - epilog='''Example uses: - * Start a new feature release (includes all changes currently in master) - release.py -b user start 1.23.0 - * Start a new patch release - release.py -b user --patch 1.21.0 start 1.21.1 - * Cancel / rollback an existing release draft - release.py -b user cancel 1.23.0 - * Restart a previously aborted patch release - release.py -b user -p 1.21.0 resume 1.21.1''', formatter_class=argparse.RawTextHelpFormatter) + epilog=EPILOG, formatter_class=argparse.RawTextHelpFormatter) parser.add_argument( - 'action', choices=['start', 'resume', 'cancel'], - help='The action to be performed for this release' + 'action', choices=ACTIONS, help='The action to be performed for this release' ) parser.add_argument('release', help='Release number, e.g. 1.9.0-rc1, 2.1.1') parser.add_argument( @@ -227,6 +300,9 @@ def main(): return resume(args) elif args.action == 'cancel': return cancel(args) + elif args.action == 'finalize': + return finalize(args) + print('Unexpected action "{}"'.format(args.action), file=sys.stderr) return 1 diff --git a/script/release/release/repository.py b/script/release/release/repository.py index d7034f8bd..dc4c6c466 100644 --- a/script/release/release/repository.py +++ b/script/release/release/repository.py @@ -159,6 +159,10 @@ class Repository(object): print('No open PR for this release branch.') return count + def write_git_sha(self): + with open(os.path.join(REPO_ROOT, 'compose', 'GITSHA'), 'w') as f: + f.write(self.git_repo.head.commit.hexsha[:7]) + def get_contributors(pr_data): commits = pr_data.get_commits() @@ -175,6 +179,7 @@ def upload_assets(gh_release, files): print('Uploading {}...'.format(filename)) gh_release.upload_asset(filedata[0], content_type='application/octet-stream') gh_release.upload_asset('{}.sha256'.format(filedata[0]), content_type='text/plain') + print('Uploading run.sh...') gh_release.upload_asset( os.path.join(REPO_ROOT, 'script', 'run', 'run.sh'), content_type='text/plain' ) From a120759c9dfea7e77db209004967582965df8a7d Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Thu, 19 Apr 2018 14:47:04 -0700 Subject: [PATCH 12/53] Add finalize step Signed-off-by: Joffrey F --- script/release/release.py | 77 ++++++++++++-------------- script/release/release/images.py | 82 ++++++++++++++++++++++++++++ script/release/release/repository.py | 8 +++ 3 files changed, 125 insertions(+), 42 deletions(-) create mode 100644 script/release/release/images.py diff --git a/script/release/release.py b/script/release/release.py index 848ab9751..92a8c1c0c 100755 --- a/script/release/release.py +++ b/script/release/release.py @@ -4,17 +4,18 @@ from __future__ import unicode_literals import argparse import os -import shutil import sys import time +from distutils.core import run_setup -import docker +import pypandoc from jinja2 import Template from release.bintray import BintrayAPI from release.const import BINTRAY_ORG from release.const import NAME from release.const import REPO_ROOT from release.downloader import BinaryDownloader +from release.images import ImageManager from release.repository import delete_assets from release.repository import get_contributors from release.repository import Repository @@ -108,43 +109,6 @@ def create_release_draft(repository, version, pr_data, files): return gh_release -def build_images(repository, files, version): - print("Building release images...") - repository.write_git_sha() - docker_client = docker.APIClient(**docker.utils.kwargs_from_env()) - distdir = os.path.join(REPO_ROOT, 'dist') - os.makedirs(distdir, exist_ok=True) - shutil.copy(files['docker-compose-Linux-x86_64'][0], distdir) - print('Building docker/compose image') - logstream = docker_client.build( - REPO_ROOT, tag='docker/compose:{}'.format(version), dockerfile='Dockerfile.run', - decode=True - ) - for chunk in logstream: - if 'error' in chunk: - raise ScriptError('Build error: {}'.format(chunk['error'])) - if 'stream' in chunk: - print(chunk['stream'], end='') - - print('Building test image (for UCP e2e)') - logstream = docker_client.build( - REPO_ROOT, tag='docker-compose-tests:tmp', decode=True - ) - for chunk in logstream: - if 'error' in chunk: - raise ScriptError('Build error: {}'.format(chunk['error'])) - if 'stream' in chunk: - print(chunk['stream'], end='') - - container = docker_client.create_container( - 'docker-compose-tests:tmp', entrypoint='tox' - ) - docker_client.commit(container, 'docker/compose-tests:latest') - docker_client.tag('docker/compose-tests:latest', 'docker/compose-tests:{}'.format(version)) - docker_client.remove_container(container, force=True) - docker_client.remove_image('docker-compose-tests:tmp', force=True) - - def print_final_instructions(args): print( "You're almost done! Please verify that everything is in order and " @@ -183,7 +147,8 @@ def resume(args): raise ScriptError('Aborting release') delete_assets(gh_release) upload_assets(gh_release, files) - build_images(repository, files, args.release) + img_manager = ImageManager(args.release) + img_manager.build_images(repository, files, args.release) except ScriptError as e: print(e) return 1 @@ -220,7 +185,8 @@ def start(args): files = downloader.download_all(args.release) gh_release = create_release_draft(repository, args.release, pr_data, files) upload_assets(gh_release, files) - build_images(repository, files, args.release) + img_manager = ImageManager(args.release) + img_manager.build_images(repository, files) except ScriptError as e: print(e) return 1 @@ -231,7 +197,34 @@ def start(args): def finalize(args): try: - raise NotImplementedError() + repository = Repository(REPO_ROOT, args.repo or NAME) + img_manager = ImageManager(args.release) + pr_data = repository.find_release_pr(args.release) + if not pr_data: + raise ScriptError('No PR found for {}'.format(args.release)) + if not check_pr_mergeable(pr_data): + raise ScriptError('Can not finalize release with an unmergeable PR') + if not img_manager.check_images(args.release): + raise ScriptError('Missing release image') + br_name = branch_name(args.release) + if not repository.branch_exists(br_name): + raise ScriptError('No local branch exists for this release.') + gh_release = repository.find_release(args.release) + if not gh_release: + raise ScriptError('No Github release draft for this version') + + pypandoc.convert_file( + os.path.join(REPO_ROOT, 'README.md'), 'rst', outputfile=os.path.join(REPO_ROOT, 'README.rst') + ) + run_setup(os.path.join(REPO_ROOT, 'setup.py'), script_args=['sdist', 'bdist_wheel']) + + merge_status = pr_data.merge() + if not merge_status.merged: + raise ScriptError('Unable to merge PR #{}: {}'.format(pr_data.number, merge_status.message)) + print('Uploading to PyPi') + run_setup(os.path.join(REPO_ROOT, 'setup.py'), script_args=['upload']) + img_manager.push_images(args.release) + repository.publish_release(gh_release) except ScriptError as e: print(e) return 1 diff --git a/script/release/release/images.py b/script/release/release/images.py new file mode 100644 index 000000000..0c7bb2045 --- /dev/null +++ b/script/release/release/images.py @@ -0,0 +1,82 @@ +from __future__ import absolute_import +from __future__ import print_function +from __future__ import unicode_literals + +import os +import shutil + +import docker + +from .const import REPO_ROOT +from .utils import ScriptError + + +class ImageManager(object): + def __init__(self, version): + self.docker_client = docker.APIClient(**docker.utils.kwargs_from_env()) + self.version = version + + def build_images(self, repository, files): + print("Building release images...") + repository.write_git_sha() + docker_client = docker.APIClient(**docker.utils.kwargs_from_env()) + distdir = os.path.join(REPO_ROOT, 'dist') + os.makedirs(distdir, exist_ok=True) + shutil.copy(files['docker-compose-Linux-x86_64'][0], distdir) + print('Building docker/compose image') + logstream = docker_client.build( + REPO_ROOT, tag='docker/compose:{}'.format(self.version), dockerfile='Dockerfile.run', + decode=True + ) + for chunk in logstream: + if 'error' in chunk: + raise ScriptError('Build error: {}'.format(chunk['error'])) + if 'stream' in chunk: + print(chunk['stream'], end='') + + print('Building test image (for UCP e2e)') + logstream = docker_client.build( + REPO_ROOT, tag='docker-compose-tests:tmp', decode=True + ) + for chunk in logstream: + if 'error' in chunk: + raise ScriptError('Build error: {}'.format(chunk['error'])) + if 'stream' in chunk: + print(chunk['stream'], end='') + + container = docker_client.create_container( + 'docker-compose-tests:tmp', entrypoint='tox' + ) + docker_client.commit(container, 'docker/compose-tests:latest') + docker_client.tag('docker/compose-tests:latest', 'docker/compose-tests:{}'.format(self.version)) + docker_client.remove_container(container, force=True) + docker_client.remove_image('docker-compose-tests:tmp', force=True) + + @property + def image_names(self): + return [ + 'docker/compose-tests:latest', + 'docker/compose-tests:{}'.format(self.version), + 'docker/compose:{}'.format(self.version) + ] + + def check_images(self, version): + docker_client = docker.APIClient(**docker.utils.kwargs_from_env()) + + for name in self.image_names: + try: + docker_client.inspect_image(name) + except docker.errors.ImageNotFound: + print('Expected image {} was not found'.format(name)) + return False + return True + + def push_images(self): + docker_client = docker.APIClient(**docker.utils.kwargs_from_env()) + + for name in self.image_names: + print('Pushing {} to Docker Hub'.format(name)) + logstream = docker_client.push(name, stream=True, decode=True) + for chunk in logstream: + if 'status' in chunk: + print(chunk['status']) diff --git a/script/release/release/repository.py b/script/release/release/repository.py index dc4c6c466..122eada8a 100644 --- a/script/release/release/repository.py +++ b/script/release/release/repository.py @@ -95,6 +95,14 @@ class Repository(object): return release return None + def publish_release(self, release): + release.update_release( + name=release.title, + message=release.body, + draft=False, + prerelease=release.prerelease + ) + def remove_release(self, version): print('Removing release draft for {}'.format(version)) releases = self.gh_repo.get_releases() From c49eca41a082c675fef81453ef8cf1ba62c962fd Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Thu, 19 Apr 2018 14:54:17 -0700 Subject: [PATCH 13/53] Avoid accidental prod push Signed-off-by: Joffrey F --- script/release/release.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/script/release/release.py b/script/release/release.py index 92a8c1c0c..3c154012f 100755 --- a/script/release/release.py +++ b/script/release/release.py @@ -213,6 +213,8 @@ def finalize(args): if not gh_release: raise ScriptError('No Github release draft for this version') + repository.checkout_branch(br_name) + pypandoc.convert_file( os.path.join(REPO_ROOT, 'README.md'), 'rst', outputfile=os.path.join(REPO_ROOT, 'README.rst') ) @@ -222,8 +224,9 @@ def finalize(args): if not merge_status.merged: raise ScriptError('Unable to merge PR #{}: {}'.format(pr_data.number, merge_status.message)) print('Uploading to PyPi') - run_setup(os.path.join(REPO_ROOT, 'setup.py'), script_args=['upload']) - img_manager.push_images(args.release) + # TODO: this will do real stuff. Uncomment when done testing + # run_setup(os.path.join(REPO_ROOT, 'setup.py'), script_args=['upload']) + # img_manager.push_images(args.release) repository.publish_release(gh_release) except ScriptError as e: print(e) From e7086091be083e793e920707ee7d87a5ccaacd85 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Thu, 19 Apr 2018 15:22:55 -0700 Subject: [PATCH 14/53] Early check for non-draft release in resume Signed-off-by: Joffrey F --- script/release/release.py | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/script/release/release.py b/script/release/release.py index 3c154012f..06dfdad9a 100755 --- a/script/release/release.py +++ b/script/release/release.py @@ -125,6 +125,15 @@ def resume(args): br_name = branch_name(args.release) if not repository.branch_exists(br_name): raise ScriptError('No local branch exists for this release.') + gh_release = repository.find_release(args.release) + if gh_release and not gh_release.draft: + print('WARNING!! Found non-draft (public) release for this version!') + proceed = input( + 'Are you sure you wish to proceed? Modifying an already ' + 'released version is dangerous! y/N ' + ) + if proceed.lower() != 'y': + raise ScriptError('Aborting release') release_branch = repository.checkout_branch(br_name) create_bump_commit(repository, release_branch, args.bintray_user) pr_data = repository.find_release_pr(args.release) @@ -134,17 +143,8 @@ def resume(args): monitor_pr_status(pr_data) downloader = BinaryDownloader(args.destination) files = downloader.download_all(args.release) - gh_release = repository.find_release(args.release) if not gh_release: gh_release = create_release_draft(repository, args.release, pr_data, files) - elif not gh_release.draft: - print('WARNING!! Found non-draft (public) release for this version!') - proceed = input( - 'Are you sure you wish to proceed? Modifying an already ' - 'released version is dangerous! y/N' - ) - if proceed.lower() != 'y': - raise ScriptError('Aborting release') delete_assets(gh_release) upload_assets(gh_release, files) img_manager = ImageManager(args.release) From 85115707645f7e134023a9635d5926fde7ccef8e Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Thu, 19 Apr 2018 15:24:41 -0700 Subject: [PATCH 15/53] Default base is master Signed-off-by: Joffrey F --- script/release/release/repository.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/script/release/release/repository.py b/script/release/release/repository.py index 122eada8a..e0c581bd5 100644 --- a/script/release/release/repository.py +++ b/script/release/release/repository.py @@ -36,7 +36,7 @@ class Repository(object): if base is not None: base = self.git_repo.tag('refs/tags/{}'.format(base)) else: - base = 'refs/remotes/{}/automated-releases'.format(remote.name) + base = 'refs/remotes/{}/master'.format(remote.name) release_branch = self.git_repo.create_head(branch_name(version), commit=base) release_branch.checkout() self.git_repo.git.merge('--strategy=ours', '--no-edit', '{}/release'.format(remote.name)) From b06bc3cdea4d45a6e3fdbe0d7e44d5bbacadf8be Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Fri, 20 Apr 2018 13:06:41 -0700 Subject: [PATCH 16/53] Add support for PR cherry picks Signed-off-by: Joffrey F --- script/release/release.py | 5 +++++ script/release/release/repository.py | 24 +++++++++++++++++++++++- 2 files changed, 28 insertions(+), 1 deletion(-) diff --git a/script/release/release.py b/script/release/release.py index 06dfdad9a..a38b2aa7d 100755 --- a/script/release/release.py +++ b/script/release/release.py @@ -30,6 +30,11 @@ from release.utils import update_run_sh_version def create_initial_branch(repository, release, base, bintray_user): release_branch = repository.create_release_branch(release, base) + if base: + print('Detected patch version.') + cherries = input('Indicate PR#s to cherry-pick then press Enter:\n') + repository.cherry_pick_prs(release_branch, cherries.split()) + return create_bump_commit(repository, release_branch, bintray_user) diff --git a/script/release/release/repository.py b/script/release/release/repository.py index e0c581bd5..4fcb2712a 100644 --- a/script/release/release/repository.py +++ b/script/release/release/repository.py @@ -2,7 +2,9 @@ from __future__ import absolute_import from __future__ import unicode_literals import os +import tempfile +import requests from git import GitCommandError from git import Repo from github import Github @@ -111,7 +113,7 @@ class Repository(object): if not release.draft: print( 'The release at {} is no longer a draft. If you TRULY intend ' - 'to remove it, please do so manually.' + 'to remove it, please do so manually.'.format(release.url) ) continue release.delete_release() @@ -171,6 +173,26 @@ class Repository(object): with open(os.path.join(REPO_ROOT, 'compose', 'GITSHA'), 'w') as f: f.write(self.git_repo.head.commit.hexsha[:7]) + def cherry_pick_prs(self, release_branch, ids): + if not ids: + return + release_branch.checkout() + for i in ids: + try: + i = int(i) + except ValueError as e: + raise ScriptError('Invalid PR id: {}'.format(e)) + print('Retrieving PR#{}'.format(i)) + pr = self.gh_repo.get_pull(i) + patch_data = requests.get(pr.patch_url).text + self.apply_patch(patch_data) + + def apply_patch(self, patch_data): + with tempfile.NamedTemporaryFile(mode='w', prefix='_compose_cherry', encoding='utf-8') as f: + f.write(patch_data) + f.flush() + self.git_repo.git.am('--3way', f.name) + def get_contributors(pr_data): commits = pr_data.get_commits() From 2b5ad06e003a714c13a5f3f86c5796d191e19bf2 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Fri, 20 Apr 2018 13:34:18 -0700 Subject: [PATCH 17/53] Cleanup Signed-off-by: Joffrey F --- script/release/release.py | 61 +++++++++++++++++----------- script/release/release/const.py | 4 +- script/release/release/repository.py | 11 +++-- script/release/release/utils.py | 22 ++++++++++ 4 files changed, 68 insertions(+), 30 deletions(-) diff --git a/script/release/release.py b/script/release/release.py index a38b2aa7d..c1908f94a 100755 --- a/script/release/release.py +++ b/script/release/release.py @@ -26,19 +26,20 @@ from release.utils import read_release_notes_from_changelog from release.utils import ScriptError from release.utils import update_init_py_version from release.utils import update_run_sh_version +from release.utils import yesno -def create_initial_branch(repository, release, base, bintray_user): - release_branch = repository.create_release_branch(release, base) - if base: +def create_initial_branch(repository, args): + release_branch = repository.create_release_branch(args.release, args.base) + if args.base and args.cherries: print('Detected patch version.') - cherries = input('Indicate PR#s to cherry-pick then press Enter:\n') + cherries = input('Indicate (space-separated) PR numbers to cherry-pick then press Enter:\n') repository.cherry_pick_prs(release_branch, cherries.split()) - return create_bump_commit(repository, release_branch, bintray_user) + return create_bump_commit(repository, release_branch, args.bintray_user, args.bintray_org) -def create_bump_commit(repository, release_branch, bintray_user): +def create_bump_commit(repository, release_branch, bintray_user, bintray_org): with release_branch.config_reader() as cfg: release = cfg.get('release') print('Updating version info in __init__.py and run.sh') @@ -46,10 +47,10 @@ def create_bump_commit(repository, release_branch, bintray_user): update_init_py_version(release) input('Please add the release notes to the CHANGELOG.md file, then press Enter to continue.') - proceed = '' - while proceed.lower() != 'y': + proceed = None + while not proceed: print(repository.diff()) - proceed = input('Are these changes ok? y/N ') + proceed = yesno('Are these changes ok? y/N ', default=False) if repository.diff(): repository.create_bump_commit(release_branch, release) @@ -57,7 +58,7 @@ def create_bump_commit(repository, release_branch, bintray_user): bintray_api = BintrayAPI(os.environ['BINTRAY_TOKEN'], bintray_user) print('Creating data repository {} on bintray'.format(release_branch.name)) - bintray_api.create_repository(BINTRAY_ORG, release_branch.name, 'generic') + bintray_api.create_repository(bintray_org, release_branch.name, 'generic') def monitor_pr_status(pr_data): @@ -126,21 +127,26 @@ def print_final_instructions(args): def resume(args): try: - repository = Repository(REPO_ROOT, args.repo or NAME) + repository = Repository(REPO_ROOT, args.repo) br_name = branch_name(args.release) if not repository.branch_exists(br_name): raise ScriptError('No local branch exists for this release.') gh_release = repository.find_release(args.release) if gh_release and not gh_release.draft: print('WARNING!! Found non-draft (public) release for this version!') - proceed = input( + proceed = yesno( 'Are you sure you wish to proceed? Modifying an already ' - 'released version is dangerous! y/N ' + 'released version is dangerous! y/N ', default=False ) - if proceed.lower() != 'y': + if proceed.lower() is not True: raise ScriptError('Aborting release') + release_branch = repository.checkout_branch(br_name) - create_bump_commit(repository, release_branch, args.bintray_user) + if args.cherries: + cherries = input('Indicate (space-separated) PR numbers to cherry-pick then press Enter:\n') + repository.cherry_pick_prs(release_branch, cherries.split()) + + create_bump_commit(repository, release_branch, args.bintray_user, args.bintray_org) pr_data = repository.find_release_pr(args.release) if not pr_data: pr_data = repository.create_release_pull_request(args.release) @@ -164,14 +170,13 @@ def resume(args): def cancel(args): try: - repository = Repository(REPO_ROOT, args.repo or NAME) + repository = Repository(REPO_ROOT, args.repo) repository.close_release_pr(args.release) repository.remove_release(args.release) repository.remove_bump_branch(args.release) - # TODO: uncomment after testing is complete - # bintray_api = BintrayAPI(os.environ['BINTRAY_TOKEN'], args.bintray_user) - # print('Removing Bintray data repository for {}'.format(args.release)) - # bintray_api.delete_repository(BINTRAY_ORG, branch_name(args.release)) + bintray_api = BintrayAPI(os.environ['BINTRAY_TOKEN'], args.bintray_user) + print('Removing Bintray data repository for {}'.format(args.release)) + bintray_api.delete_repository(args.bintray_org, branch_name(args.release)) except ScriptError as e: print(e) return 1 @@ -181,8 +186,8 @@ def cancel(args): def start(args): try: - repository = Repository(REPO_ROOT, args.repo or NAME) - create_initial_branch(repository, args.release, args.base, args.bintray_user) + repository = Repository(REPO_ROOT, args.repo) + create_initial_branch(repository, args) pr_data = repository.create_release_pull_request(args.release) check_pr_mergeable(pr_data) monitor_pr_status(pr_data) @@ -202,7 +207,7 @@ def start(args): def finalize(args): try: - repository = Repository(REPO_ROOT, args.repo or NAME) + repository = Repository(REPO_ROOT, args.repo) img_manager = ImageManager(args.release) pr_data = repository.find_release_pr(args.release) if not pr_data: @@ -282,17 +287,25 @@ def main(): help='Which version is being patched by this release' ) parser.add_argument( - '--repo', '-r', dest='repo', + '--repo', '-r', dest='repo', default=NAME, help='Start a release for the given repo (default: {})'.format(NAME) ) parser.add_argument( '-b', dest='bintray_user', required=True, metavar='USER', help='Username associated with the Bintray API key' ) + parser.add_argument( + '--bintray-org', dest='bintray_org', metavar='ORG', default=BINTRAY_ORG, + help='Organization name on bintray where the data repository will be created.' + ) parser.add_argument( '--destination', '-o', metavar='DIR', default='binaries', help='Directory where release binaries will be downloaded relative to the project root' ) + parser.add_argument( + '--no-cherries', '-C', dest='cherries', action='store_false', + help='If set, the program will not prompt the user for PR numbers to cherry-pick' + ) args = parser.parse_args() if args.action == 'start': diff --git a/script/release/release/const.py b/script/release/release/const.py index 34f338a89..5a72bde41 100644 --- a/script/release/release/const.py +++ b/script/release/release/const.py @@ -5,5 +5,5 @@ import os REPO_ROOT = os.path.join(os.path.dirname(__file__), '..', '..', '..') -NAME = 'shin-/compose' -BINTRAY_ORG = 'shin-compose' +NAME = 'docker/compose' +BINTRAY_ORG = 'docker-compose' diff --git a/script/release/release/repository.py b/script/release/release/repository.py index 4fcb2712a..d4d1c7201 100644 --- a/script/release/release/repository.py +++ b/script/release/release/repository.py @@ -29,17 +29,20 @@ class Repository(object): def create_release_branch(self, version, base=None): print('Creating release branch {} based on {}...'.format(version, base or 'master')) remote = self.find_remote(self.gh_repo.full_name) + br_name = branch_name(version) remote.fetch() - if self.branch_exists(branch_name(version)): + if self.branch_exists(br_name): raise ScriptError( - "Branch {} already exists locally. " - "Please remove it before running the release script.".format(branch_name(version)) + "Branch {} already exists locally. Please remove it before " + "running the release script, or use `resume` instead.".format( + br_name + ) ) if base is not None: base = self.git_repo.tag('refs/tags/{}'.format(base)) else: base = 'refs/remotes/{}/master'.format(remote.name) - release_branch = self.git_repo.create_head(branch_name(version), commit=base) + release_branch = self.git_repo.create_head(br_name, commit=base) release_branch.checkout() self.git_repo.git.merge('--strategy=ours', '--no-edit', '{}/release'.format(remote.name)) with release_branch.config_writer() as cfg: diff --git a/script/release/release/utils.py b/script/release/release/utils.py index b0e1f6a84..977a0a712 100644 --- a/script/release/release/utils.py +++ b/script/release/release/utils.py @@ -61,3 +61,25 @@ def compatibility_matrix(): for fmt, api_version in compose_const.API_VERSIONS.items(): result[compose_const.API_VERSION_TO_ENGINE_VERSION[api_version]].append(fmt.vstring) return result + + +def yesno(prompt, default=None): + """ + Prompt the user for a yes or no. + + Can optionally specify a default value, which will only be + used if they enter a blank line. + + Unrecognised input (anything other than "y", "n", "yes", + "no" or "") will return None. + """ + answer = input(prompt).strip().lower() + + if answer == "y" or answer == "yes": + return True + elif answer == "n" or answer == "no": + return False + elif answer == "": + return default + else: + return None From 6b83a651f663873ec4d43fd3e8f171f35eedcf42 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Fri, 20 Apr 2018 14:23:58 -0700 Subject: [PATCH 18/53] Improve monitor function Signed-off-by: Joffrey F --- script/release/release.py | 13 +++++++++---- script/release/release/downloader.py | 4 ++-- 2 files changed, 11 insertions(+), 6 deletions(-) diff --git a/script/release/release.py b/script/release/release.py index c1908f94a..338e73af2 100755 --- a/script/release/release.py +++ b/script/release/release.py @@ -66,24 +66,29 @@ def monitor_pr_status(pr_data): last_commit = pr_data.get_commits().reversed[0] while True: status = last_commit.get_combined_status() - if status.state == 'pending': + if status.state == 'pending' or status.state == 'failure': summary = { 'pending': 0, 'success': 0, 'failure': 0, } for detail in status.statuses: + if detail.context == 'dco-signed': + # dco-signed check breaks on merge remote-tracking ; ignore it + continue summary[detail.state] += 1 print('{pending} pending, {success} successes, {failure} failures'.format(**summary)) if status.total_count == 0: # Mostly for testing purposes against repos with no CI setup return True + elif summary['pending'] == 0 and summary['failure'] == 0: + return True + elif summary['failure'] > 0: + raise ScriptError('CI failures detected!') time.sleep(30) elif status.state == 'success': print('{} successes: all clear!'.format(status.total_count)) return True - else: - raise ScriptError('CI failure detected') def check_pr_mergeable(pr_data): @@ -159,7 +164,7 @@ def resume(args): delete_assets(gh_release) upload_assets(gh_release, files) img_manager = ImageManager(args.release) - img_manager.build_images(repository, files, args.release) + img_manager.build_images(repository, files) except ScriptError as e: print(e) return 1 diff --git a/script/release/release/downloader.py b/script/release/release/downloader.py index cd43bc993..d92ae78b5 100644 --- a/script/release/release/downloader.py +++ b/script/release/release/downloader.py @@ -36,7 +36,7 @@ class BinaryDownloader(requests.Session): base=self.base_appveyor_url, filename=filename, branch_name=branch_name ) full_dest = os.path.join(REPO_ROOT, self.destination, filename) - return self.download(url, full_dest) + return self._download(url, full_dest) def _download(self, url, full_dest): m = hashlib.sha256() @@ -57,7 +57,7 @@ class BinaryDownloader(requests.Session): files = { 'docker-compose-Darwin-x86_64': None, 'docker-compose-Linux-x86_64': None, - # 'docker-compose-Windows-x86_64.exe': None, + 'docker-compose-Windows-x86_64.exe': None, } for filename in files.keys(): From a752208621ef77b5ea34ffefef4230027f95ad7a Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Fri, 20 Apr 2018 15:29:37 -0700 Subject: [PATCH 19/53] Fix appveyor build Signed-off-by: Joffrey F --- script/build/windows.ps1 | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/script/build/windows.ps1 b/script/build/windows.ps1 index 98a748158..1de9bbfa4 100644 --- a/script/build/windows.ps1 +++ b/script/build/windows.ps1 @@ -44,16 +44,10 @@ virtualenv .\venv # pip and pyinstaller generate lots of warnings, so we need to ignore them $ErrorActionPreference = "Continue" -# Install dependencies -# Fix for https://github.com/pypa/pip/issues/3964 -# Remove-Item -Recurse -Force .\venv\Lib\site-packages\pip -# .\venv\Scripts\easy_install pip==9.0.1 -# .\venv\Scripts\pip install --upgrade pip setuptools -# End fix .\venv\Scripts\pip install pypiwin32==220 .\venv\Scripts\pip install -r requirements.txt .\venv\Scripts\pip install --no-deps . -.\venv\Scripts\pip install --allow-external pyinstaller -r requirements-build.txt +.\venv\Scripts\pip install -r requirements-build.txt git rev-parse --short HEAD | out-file -encoding ASCII compose\GITSHA From eba67910f3b59f121e0832e4edb01c64b05a10b9 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Fri, 20 Apr 2018 16:21:13 -0700 Subject: [PATCH 20/53] Containerize release tool Signed-off-by: Joffrey F --- script/release/Dockerfile | 14 ++++++++++++++ script/release/release.sh | 25 +++++++++++++++++++++++++ 2 files changed, 39 insertions(+) create mode 100644 script/release/Dockerfile create mode 100755 script/release/release.sh diff --git a/script/release/Dockerfile b/script/release/Dockerfile new file mode 100644 index 000000000..0d4ec27e1 --- /dev/null +++ b/script/release/Dockerfile @@ -0,0 +1,14 @@ +FROM python:3.6 +RUN mkdir -p /src && pip install -U Jinja2==2.10 \ + PyGithub==1.39 \ + pypandoc==1.4 \ + GitPython==2.1.9 \ + requests==2.18.4 && \ + apt-get update && apt-get install -y pandoc + +VOLUME /src/script/release +WORKDIR /src +COPY . /src +RUN python setup.py develop +ENTRYPOINT ["python", "script/release/release.py"] +CMD ["--help"] diff --git a/script/release/release.sh b/script/release/release.sh new file mode 100755 index 000000000..2310429aa --- /dev/null +++ b/script/release/release.sh @@ -0,0 +1,25 @@ +#!/bin/sh + +docker image inspect compose/release-tool > /dev/null +if test $? -ne 0; then + docker build -t compose/release-tool -f $(pwd)/script/release/Dockerfile $(pwd) +fi + +if test -z $GITHUB_TOKEN; then + echo "GITHUB_TOKEN environment variable must be set" + exit 1 +fi + +if test -z $BINTRAY_TOKEN; then + echo "BINTRAY_TOKEN environment variable must be set" + exit 1 +fi + +docker run -e GITHUB_TOKEN=$GITHUB_TOKEN -e BINTRAY_TOKEN=$BINTRAY_TOKEN -it \ + --mount type=bind,source=$(pwd),target=/src \ + --mount type=bind,source=$(pwd)/.git,target=/src/.git \ + --mount type=bind,source=$HOME/.docker,target=/root/.docker \ + --mount type=bind,source=/var/run/docker.sock,target=/var/run/docker.sock \ + --mount type=bind,source=$HOME/.ssh,target=/root/.ssh \ + -v $HOME/.pypirc:/root/.pypirc \ + compose/release-tool $* From 62fc24eb279edaab628609bf14d910cb34d58dc6 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Fri, 20 Apr 2018 17:15:45 -0700 Subject: [PATCH 21/53] Uncomment deploy steps Signed-off-by: Joffrey F --- script/release/release.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/script/release/release.py b/script/release/release.py index 338e73af2..add8fb2d3 100755 --- a/script/release/release.py +++ b/script/release/release.py @@ -239,9 +239,8 @@ def finalize(args): if not merge_status.merged: raise ScriptError('Unable to merge PR #{}: {}'.format(pr_data.number, merge_status.message)) print('Uploading to PyPi') - # TODO: this will do real stuff. Uncomment when done testing - # run_setup(os.path.join(REPO_ROOT, 'setup.py'), script_args=['upload']) - # img_manager.push_images(args.release) + run_setup(os.path.join(REPO_ROOT, 'setup.py'), script_args=['upload']) + img_manager.push_images(args.release) repository.publish_release(gh_release) except ScriptError as e: print(e) From 7536c331e0cc63be834bee03007db544cf5b656d Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Mon, 23 Apr 2018 14:37:34 -0700 Subject: [PATCH 22/53] Document new release process Signed-off-by: Joffrey F --- project/RELEASE-PROCESS.md | 149 +----------------------------- script/release/README.md | 184 +++++++++++++++++++++++++++++++++++++ 2 files changed, 185 insertions(+), 148 deletions(-) mode change 100644 => 120000 project/RELEASE-PROCESS.md create mode 100644 script/release/README.md diff --git a/project/RELEASE-PROCESS.md b/project/RELEASE-PROCESS.md deleted file mode 100644 index d4afb87b9..000000000 --- a/project/RELEASE-PROCESS.md +++ /dev/null @@ -1,148 +0,0 @@ -Building a Compose release -========================== - -## Prerequisites - -The release scripts require the following tools installed on the host: - -* https://hub.github.com/ -* https://stedolan.github.io/jq/ -* http://pandoc.org/ - -## To get started with a new release - -Create a branch, update version, and add release notes by running `make-branch` - - ./script/release/make-branch $VERSION [$BASE_VERSION] - -`$BASE_VERSION` will default to master. Use the last version tag for a bug fix -release. - -As part of this script you'll be asked to: - -1. Update the version in `compose/__init__.py` and `script/run/run.sh`. - - If the next release will be an RC, append `-rcN`, e.g. `1.4.0-rc1`. - -2. Write release notes in `CHANGELOG.md`. - - Almost every feature enhancement should be mentioned, with the most - visible/exciting ones first. Use descriptive sentences and give context - where appropriate. - - Bug fixes are worth mentioning if it's likely that they've affected lots - of people, or if they were regressions in the previous version. - - Improvements to the code are not worth mentioning. - -3. Create a new repository on [bintray](https://bintray.com/docker-compose). - The name has to match the name of the branch (e.g. `bump-1.9.0`) and the - type should be "Generic". Other fields can be left blank. - -4. Check that the `vnext-compose` branch on - [the docs repo](https://github.com/docker/docker.github.io/) has - documentation for all the new additions in the upcoming release, and create - a PR there for what needs to be amended. - - -## When a PR is merged into master that we want in the release - -1. Check out the bump branch and run the cherry pick script - - git checkout bump-$VERSION - ./script/release/cherry-pick-pr $PR_NUMBER - -2. When you are done cherry-picking branches move the bump version commit to HEAD - - ./script/release/rebase-bump-commit - git push --force $USERNAME bump-$VERSION - - -## To release a version (whether RC or stable) - -Check out the bump branch and run the `build-binaries` script - - git checkout bump-$VERSION - ./script/release/build-binaries - -When prompted build the non-linux binaries and test them. - -1. Download the different platform binaries by running the following script: - - `./script/release/download-binaries $VERSION` - - The binaries for Linux, OSX and Windows will be downloaded in the `binaries-$VERSION` folder. - -3. Draft a release from the tag on GitHub (the `build-binaries` script will open the window for - you) - - The tag will only be present on Github when you run the `push-release` - script in step 7, but you can pre-fill it at that point. - -4. Paste in installation instructions and release notes. Here's an example - - change the Compose version and Docker version as appropriate: - - If you're a Mac or Windows user, the best way to install Compose and keep it up-to-date is **[Docker for Mac and Windows](https://www.docker.com/products/docker)**. - - Docker for Mac and Windows will automatically install the latest version of Docker Engine for you. - - Alternatively, you can use the usual commands to install or upgrade Compose: - - ``` - curl -L https://github.com/docker/compose/releases/download/1.16.0/docker-compose-`uname -s`-`uname -m` -o /usr/local/bin/docker-compose - chmod +x /usr/local/bin/docker-compose - ``` - - See the [install docs](https://docs.docker.com/compose/install/) for more install options and instructions. - - ## Compose file format compatibility matrix - - | Compose file format | Docker Engine | - | --- | --- | - | 3.3 | 17.06.0+ | - | 3.0 – 3.2 | 1.13.0+ | - | 2.3| 17.06.0+ | - | 2.2 | 1.13.0+ | - | 2.1 | 1.12.0+ | - | 2.0 | 1.10.0+ | - | 1.0 | 1.9.1+ | - - ## Changes - - ...release notes go here... - -5. Attach the binaries and `script/run/run.sh` - -6. Add "Thanks" with a list of contributors. The contributor list can be generated - by running `./script/release/contributors`. - -7. If everything looks good, it's time to push the release. - - - ./script/release/push-release - - -8. Merge the bump PR. - -8. Publish the release on GitHub. - -9. Check that all the binaries download (following the install instructions) and run. - -10. Announce the release on the appropriate Slack channel(s). - -## If it’s a stable release (not an RC) - -1. Close the release’s milestone. - -## If it’s a minor release (1.x.0), rather than a patch release (1.x.y) - -1. Open a PR against `master` to: - - - update `CHANGELOG.md` to bring it in line with `release` - - bump the version in `compose/__init__.py` to the *next* minor version number with `dev` appended. For example, if you just released `1.4.0`, update it to `1.5.0dev`. - -2. Get the PR merged. - -## Finally - -1. Celebrate, however you’d like. diff --git a/project/RELEASE-PROCESS.md b/project/RELEASE-PROCESS.md new file mode 120000 index 000000000..c8457671a --- /dev/null +++ b/project/RELEASE-PROCESS.md @@ -0,0 +1 @@ +../script/release/README.md \ No newline at end of file diff --git a/script/release/README.md b/script/release/README.md new file mode 100644 index 000000000..c5136c764 --- /dev/null +++ b/script/release/README.md @@ -0,0 +1,184 @@ +# Release HOWTO + +This file describes the process of making a public release of `docker-compose`. +Please read it carefully before proceeding! + +## Prerequisites + +The following things are required to bring a release to a successful conclusion + +### Local Docker engine (Linux Containers) + +The release script runs inside a container and builds images that will be part +of the release. + +### Docker Hub account + +You should be logged into a Docker Hub account that allows pushing to the +following repositories: + +- docker/compose +- docker/compose-tests + +### A Github account and Github API token + +Your Github account needs to have write access on the `docker/compose` repo. +To generate a Github token, head over to the +[Personal access tokens](https://github.com/settings/tokens) page in your +Github settings and select "Generate new token". Your token should include +(at minimum) the following scopes: + +- `repo:status` +- `public_repo` + +This API token should be exposed to the release script through the +`GITHUB_TOKEN` environment variable. + +### A Bintray account and Bintray API key + +Your Bintray account will need to be an admin member of the +[docker-compose organization](https://github.com/settings/tokens). +Additionally, you should generate a personal API key. To do so, click your +username in the top-right hand corner and select "Edit profile" ; on the new +page, select "API key" in the left-side menu. + +This API key should be exposed to the release script through the +`BINTRAY_TOKEN` environment variable. + +### A PyPi account + +Said account needs to be a member of the maintainers group for the +[`docker-compose` project](https://pypi.org/project/docker-compose/). + +Moreover, the `~/.pypirc` file should exist on your host and contain the +relevant pypi credentials. + +## Start a feature release + +A feature release is a release that includes all changes present in the +`master` branch when initiated. It's typically versioned `X.Y.0-rc1`, where +Y is the minor version of the previous release incremented by one. A series +of one or more Release Candidates (RCs) should be made available to the public +to find and squash potential bugs. + +From the root of the Compose repository, run the following command: +``` +./script/release/release.sh -b start X.Y.0-rc1 +``` + +After a short initialization period, the script will invite you to edit the +`CHANGELOG.md` file. Do so by being careful to respect the same format as +previous releases. Once done, the script will display a `diff` of the staged +changes for the bump commit. Once you validate these, a bump commit will be +created on the newly created release branch and pushed remotely. + +The release tool then waits for the CI to conclude before proceeding. +If failures are reported, the release will be aborted until these are fixed. +Please refer to the "Resume a draft release" section below for more details. + +Once all resources have been prepared, the release script will exit with a +message resembling this one: + +``` +You're almost done! Please verify that everything is in order and you are ready +to make the release public, then run the following command: +./script/release/release.sh -b user finalize X.Y.0-rc1 +``` + +Once you are ready to finalize the release (making binaries and other versioned +assets public), proceed to the "Finalize a release" section of this guide. + +## Start a patch release + +A patch release is a release that builds off a previous release with discrete +additions. This can be an RC release after RC1 (`X.Y.0-rcZ`, `Z > 1`), a GA release +based off the final RC (`X.Y.0`), or a bugfix release based off a previous +GA release (`X.Y.Z`, `Z > 0`). + +From the root of the Compose repository, run the following command: +``` +./script/release/release.sh -b start --patch=BASE_VERSION RELEASE_VERSION +``` + +The process of starting a patch release is identical to starting a feature +release except for one difference ; at the beginning, the script will ask for +PR numbers you wish to cherry-pick into the release. These numbers should +correspond to existing PRs on the docker/compose repository. Multiple numbers +should be separated by whitespace. + +Once you are ready to finalize the release (making binaries and other versioned +assets public), proceed to the "Finalize a release" section of this guide. + +## Finalize a release + +Once you're ready to make your release public, you may execute the following +command from the root of the Compose repository: +``` +./script/release/release.sh -b finalize RELEAE_VERSION +``` + +Note that this command will create and publish versioned assets to the public. +As a result, it can not be reverted. The command will perform some basic +sanity checks before doing so, but it is your responsibility to ensure +everything is in order before pushing the button. + +After the command exits, you should make sure: + +- The `docker/compose:VERSION` image is available on Docker Hub and functional +- The `pip install -U docker-compose==VERSION` command correctly installs the + specified version +- The install command on the Github release page installs the new release + +## Resume a draft release + +"Resuming" a release lets you address the following situations occurring before +a release is made final: + +- Cherry-pick additional PRs to include in the release +- Resume a release that was aborted because of CI failures after they've been + addressed +- Rebuild / redownload assets after manual changes have been made to the + release branch +- etc. + +From the root of the Compose repository, run the following command: +``` +./script/release/release.sh -b resume RELEASE_VERSION +``` + +The release tool will attempt to determine what steps it's already been through +for the specified release and pick up where it left off. Some steps are +executed again no matter what as it's assumed they'll produce different +results, like building images or downloading binaries. + +## Cancel a draft release + +If issues snuck into your release branch, it is sometimes easier to start from +scratch. Before a release has been finalized, it is possible to cancel it using +the following command: +``` +./script/release/release.sh -b cancel RELEASE_VERSION +``` + +This will remove the release branch with this release (locally and remotely), +close the associated PR, remove the release page draft on Github and delete +the Bintray repository for it, allowing you to start fresh. + +## Manual operations + +Some common, release-related operations are not covered by this tool and should +be handled manually by the operator: + +- After any release: + - Announce new release on Slack +- After a GA release: + - Close the release milestone + - Merge back `CHANGELOG.md` changes from the `release` branch into `master` + - Bump the version in `compose/__init__.py` to the *next* minor version + number with `dev` appended. For example, if you just released `1.4.0`, + update it to `1.5.0dev` + +## Advanced options + +You can consult the full list of options for the release tool by executing +`./script/release/release.sh --help`. From 0578a58471d6dbca16fc6ed218f5affe76ac813e Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Mon, 23 Apr 2018 15:01:30 -0700 Subject: [PATCH 23/53] Remove obsolete release scripts Signed-off-by: Joffrey F --- script/release/build-binaries | 40 --------------- script/release/contributors | 30 ----------- script/release/download-binaries | 39 --------------- script/release/make-branch | 86 -------------------------------- 4 files changed, 195 deletions(-) delete mode 100755 script/release/build-binaries delete mode 100755 script/release/contributors delete mode 100755 script/release/download-binaries delete mode 100755 script/release/make-branch diff --git a/script/release/build-binaries b/script/release/build-binaries deleted file mode 100755 index a39b186d9..000000000 --- a/script/release/build-binaries +++ /dev/null @@ -1,40 +0,0 @@ -#!/bin/bash -# -# Build the release binaries -# - -. "$(dirname "${BASH_SOURCE[0]}")/utils.sh" - -function usage() { - >&2 cat << EOM -Build binaries for the release. - -This script requires that 'git config branch.${BRANCH}.release' is set to the -release version for the release branch. - -EOM - exit 1 -} - -BRANCH="$(git rev-parse --abbrev-ref HEAD)" -VERSION="$(git config "branch.${BRANCH}.release")" || usage -REPO=docker/compose - -# Build the binaries -script/clean -script/build/linux - -echo "Building the container distribution" -script/build/image $VERSION - -echo "Building the compose-tests image" -script/build/test-image $VERSION - -echo "Create a github release" -# TODO: script more of this https://developer.github.com/v3/repos/releases/ -browser https://github.com/$REPO/releases/new - -echo "Don't forget to download the osx and windows binaries from appveyor/bintray\!" -echo "https://dl.bintray.com/docker-compose/$BRANCH/" -echo "https://ci.appveyor.com/project/docker/compose" -echo diff --git a/script/release/contributors b/script/release/contributors deleted file mode 100755 index 4657dd805..000000000 --- a/script/release/contributors +++ /dev/null @@ -1,30 +0,0 @@ -#!/bin/bash -set -e - - -function usage() { - >&2 cat << EOM -Print the list of github contributors for the release - -Usage: - - $0 -EOM - exit 1 -} - -[[ -n "$1" ]] || usage -PREV_RELEASE=$1 -BRANCH="$(git rev-parse --abbrev-ref HEAD)" -URL="https://api.github.com/repos/docker/compose/compare" - -contribs=$(curl -sf "$URL/$PREV_RELEASE...$BRANCH" | \ - jq -r '.commits[].author.login' | \ - sort | \ - uniq -c | \ - sort -nr) - -echo "Contributions by user: " -echo "$contribs" -echo -echo "$contribs" | awk '{print "@"$2","}' | xargs diff --git a/script/release/download-binaries b/script/release/download-binaries deleted file mode 100755 index 0b187f6c2..000000000 --- a/script/release/download-binaries +++ /dev/null @@ -1,39 +0,0 @@ -#!/bin/bash - -function usage() { - >&2 cat << EOM -Download Linux, Mac OS and Windows binaries from remote endpoints - -Usage: - - $0 - -Options: - - version version string for the release (ex: 1.6.0) - -EOM - exit 1 -} - - -[ -n "$1" ] || usage -VERSION=$1 -BASE_BINTRAY_URL=https://dl.bintray.com/docker-compose/bump-$VERSION/ -DESTINATION=binaries-$VERSION -APPVEYOR_URL=https://ci.appveyor.com/api/projects/docker/compose/\ -artifacts/dist%2Fdocker-compose-Windows-x86_64.exe?branch=bump-$VERSION - -mkdir $DESTINATION - - -wget -O $DESTINATION/docker-compose-Darwin-x86_64 $BASE_BINTRAY_URL/docker-compose-Darwin-x86_64 -wget -O $DESTINATION/docker-compose-Linux-x86_64 $BASE_BINTRAY_URL/docker-compose-Linux-x86_64 -wget -O $DESTINATION/docker-compose-Windows-x86_64.exe $APPVEYOR_URL - -echo -e "\n\nCopy the following lines into the integrity check table in the release notes:\n\n" -cd $DESTINATION -rm -rf *.sha256 -ls | xargs sha256sum | sed 's/ / | /g' | sed -r 's/([^ |]+)/`\1`/g' -ls | xargs -I@ bash -c "sha256sum @ | cut -d' ' -f1 > @.sha256" -cd - diff --git a/script/release/make-branch b/script/release/make-branch deleted file mode 100755 index b8a0cd31e..000000000 --- a/script/release/make-branch +++ /dev/null @@ -1,86 +0,0 @@ -#!/bin/bash -# -# Prepare a new release branch -# - -. "$(dirname "${BASH_SOURCE[0]}")/utils.sh" - -function usage() { - >&2 cat << EOM -Create a new release branch 'release-' - -Usage: - - $0 [] - -Options: - - version version string for the release (ex: 1.6.0) - base_version branch or tag to start from. Defaults to master. For - bug-fix releases use the previous stage release tag. - -EOM - exit 1 -} - - -[ -n "$1" ] || usage -VERSION=$1 -BRANCH=bump-$VERSION -REPO=docker/compose -GITHUB_REPO=git@github.com:$REPO - -if [ -z "$2" ]; then - BASE_VERSION="master" -else - BASE_VERSION=$2 -fi - - -DEFAULT_REMOTE=release -REMOTE="$(find_remote "$GITHUB_REPO")" -# If we don't have a docker remote add one -if [ -z "$REMOTE" ]; then - echo "Creating $DEFAULT_REMOTE remote" - git remote add ${DEFAULT_REMOTE} ${GITHUB_REPO} -fi - -# handle the difference between a branch and a tag -if [ -z "$(git name-rev --tags $BASE_VERSION | grep tags)" ]; then - BASE_VERSION=$REMOTE/$BASE_VERSION -fi - -echo "Creating a release branch $VERSION from $BASE_VERSION" -read -n1 -r -p "Continue? (ctrl+c to cancel)" -git fetch $REMOTE -p -git checkout -b $BRANCH $BASE_VERSION - -echo "Merging remote release branch into new release branch" -git merge --strategy=ours --no-edit $REMOTE/release - -# Store the release version for this branch in git, so that other release -# scripts can use it -git config "branch.${BRANCH}.release" $VERSION - - -editor=${EDITOR:-vim} - -echo "Update versions in compose/__init__.py, script/run/run.sh" -$editor compose/__init__.py -$editor script/run/run.sh - - -echo "Write release notes in CHANGELOG.md" -browser "https://github.com/docker/compose/issues?q=milestone%3A$VERSION+is%3Aclosed" -$editor CHANGELOG.md - - -git diff -echo "Verify changes before commit. Exit the shell to commit changes" -$SHELL || true -git commit -a -m "Bump $VERSION" --signoff --no-verify - - -echo "Push branch to docker remote" -git push $REMOTE -browser https://github.com/$REPO/compare/docker:release...$BRANCH?expand=1 From 4dece7fcb20f4cae3a77291d20d1c83e34cbc4c3 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Mon, 23 Apr 2018 16:41:10 -0700 Subject: [PATCH 24/53] Retrieve objects using legacy (< 1.21) project names Signed-off-by: Joffrey F --- compose/network.py | 66 ++++++++++++++++++++++++++++++++++------------ compose/service.py | 36 +++++++++++++++++++++---- compose/volume.py | 43 +++++++++++++++++++++++++----- 3 files changed, 117 insertions(+), 28 deletions(-) diff --git a/compose/network.py b/compose/network.py index 1a080c40c..e0fdb3e27 100644 --- a/compose/network.py +++ b/compose/network.py @@ -2,6 +2,7 @@ from __future__ import absolute_import from __future__ import unicode_literals import logging +import re from collections import OrderedDict from docker.errors import NotFound @@ -10,9 +11,11 @@ from docker.types import IPAMPool from docker.utils import version_gte from docker.utils import version_lt +from . import __version__ from .config import ConfigurationError from .const import LABEL_NETWORK from .const import LABEL_PROJECT +from .const import LABEL_VERSION log = logging.getLogger(__name__) @@ -39,6 +42,7 @@ class Network(object): self.enable_ipv6 = enable_ipv6 self.labels = labels self.custom_name = custom_name + self.legacy = False def ensure(self): if self.external: @@ -68,6 +72,14 @@ class Network(object): data = self.inspect() check_remote_network_config(data, self) except NotFound: + try: + data = self.inspect(legacy=True) + self.legacy = True + check_remote_network_config(data, self) + return + except NotFound: + pass + driver_name = 'the default driver' if self.driver: driver_name = 'driver "{}"'.format(self.driver) @@ -94,18 +106,37 @@ class Network(object): log.info("Network %s is external, skipping", self.full_name) return - log.info("Removing network {}".format(self.full_name)) - self.client.remove_network(self.full_name) + log.info("Removing network {}".format(self.true_name)) + try: + self.client.remove_network(self.full_name) + except NotFound: + self.client.remove_network(self.legacy_full_name) - def inspect(self): + def inspect(self, legacy=False): + if legacy: + return self.client.inspect_network(self.legacy_full_name) return self.client.inspect_network(self.full_name) + @property + def legacy_full_name(self): + if self.custom_name: + return self.name + return '{0}_{1}'.format( + re.sub(r'[_-]', '', self.project), self.name + ) + @property def full_name(self): if self.custom_name: return self.name return '{0}_{1}'.format(self.project, self.name) + @property + def true_name(self): + if self.legacy: + return self.legacy_full_name + return self.full_name + @property def _labels(self): if version_lt(self.client._version, '1.23'): @@ -114,6 +145,7 @@ class Network(object): labels.update({ LABEL_PROJECT: self.project, LABEL_NETWORK: self.name, + LABEL_VERSION: __version__, }) return labels @@ -150,49 +182,49 @@ def check_remote_ipam_config(remote, local): remote_ipam = remote.get('IPAM') ipam_dict = create_ipam_config_from_dict(local.ipam) if local.ipam.get('driver') and local.ipam.get('driver') != remote_ipam.get('Driver'): - raise NetworkConfigChangedError(local.full_name, 'IPAM driver') + raise NetworkConfigChangedError(local.true_name, 'IPAM driver') if len(ipam_dict['Config']) != 0: if len(ipam_dict['Config']) != len(remote_ipam['Config']): - raise NetworkConfigChangedError(local.full_name, 'IPAM configs') + raise NetworkConfigChangedError(local.true_name, 'IPAM configs') remote_configs = sorted(remote_ipam['Config'], key='Subnet') local_configs = sorted(ipam_dict['Config'], key='Subnet') while local_configs: lc = local_configs.pop() rc = remote_configs.pop() if lc.get('Subnet') != rc.get('Subnet'): - raise NetworkConfigChangedError(local.full_name, 'IPAM config subnet') + raise NetworkConfigChangedError(local.true_name, 'IPAM config subnet') if lc.get('Gateway') is not None and lc.get('Gateway') != rc.get('Gateway'): - raise NetworkConfigChangedError(local.full_name, 'IPAM config gateway') + raise NetworkConfigChangedError(local.true_name, 'IPAM config gateway') if lc.get('IPRange') != rc.get('IPRange'): - raise NetworkConfigChangedError(local.full_name, 'IPAM config ip_range') + raise NetworkConfigChangedError(local.true_name, 'IPAM config ip_range') if sorted(lc.get('AuxiliaryAddresses')) != sorted(rc.get('AuxiliaryAddresses')): - raise NetworkConfigChangedError(local.full_name, 'IPAM config aux_addresses') + raise NetworkConfigChangedError(local.true_name, 'IPAM config aux_addresses') remote_opts = remote_ipam.get('Options') or {} local_opts = local.ipam.get('options') or {} for k in set.union(set(remote_opts.keys()), set(local_opts.keys())): if remote_opts.get(k) != local_opts.get(k): - raise NetworkConfigChangedError(local.full_name, 'IPAM option "{}"'.format(k)) + raise NetworkConfigChangedError(local.true_name, 'IPAM option "{}"'.format(k)) def check_remote_network_config(remote, local): if local.driver and remote.get('Driver') != local.driver: - raise NetworkConfigChangedError(local.full_name, 'driver') + raise NetworkConfigChangedError(local.true_name, 'driver') local_opts = local.driver_opts or {} remote_opts = remote.get('Options') or {} for k in set.union(set(remote_opts.keys()), set(local_opts.keys())): if k in OPTS_EXCEPTIONS: continue if remote_opts.get(k) != local_opts.get(k): - raise NetworkConfigChangedError(local.full_name, 'option "{}"'.format(k)) + raise NetworkConfigChangedError(local.true_name, 'option "{}"'.format(k)) if local.ipam is not None: check_remote_ipam_config(remote, local) if local.internal is not None and local.internal != remote.get('Internal', False): - raise NetworkConfigChangedError(local.full_name, 'internal') + raise NetworkConfigChangedError(local.true_name, 'internal') if local.enable_ipv6 is not None and local.enable_ipv6 != remote.get('EnableIPv6', False): - raise NetworkConfigChangedError(local.full_name, 'enable_ipv6') + raise NetworkConfigChangedError(local.true_name, 'enable_ipv6') local_labels = local.labels or {} remote_labels = remote.get('Labels', {}) @@ -202,7 +234,7 @@ def check_remote_network_config(remote, local): if remote_labels.get(k) != local_labels.get(k): log.warn( 'Network {}: label "{}" has changed. It may need to be' - ' recreated.'.format(local.full_name, k) + ' recreated.'.format(local.true_name, k) ) @@ -257,7 +289,7 @@ class ProjectNetworks(object): try: network.remove() except NotFound: - log.warn("Network %s not found.", network.full_name) + log.warn("Network %s not found.", network.true_name) def initialize(self): if not self.use_networking: @@ -286,7 +318,7 @@ def get_networks(service_dict, network_definitions): for name, netdef in get_network_defs_for_service(service_dict).items(): network = network_definitions.get(name) if network: - networks[network.full_name] = netdef + networks[network.true_name] = netdef else: raise ConfigurationError( 'Service "{}" uses an undefined network "{}"' diff --git a/compose/service.py b/compose/service.py index bb9e26baa..0a866161c 100644 --- a/compose/service.py +++ b/compose/service.py @@ -51,6 +51,7 @@ from .progress_stream import StreamOutputError from .utils import json_hash from .utils import parse_bytes from .utils import parse_seconds_float +from .version import ComposeVersion log = logging.getLogger(__name__) @@ -192,11 +193,25 @@ class Service(object): def containers(self, stopped=False, one_off=False, filters={}): filters.update({'label': self.labels(one_off=one_off)}) - return list(filter(None, [ + result = list(filter(None, [ Container.from_ps(self.client, container) for container in self.client.containers( all=stopped, - filters=filters)])) + filters=filters)]) + ) + if result: + return result + + filters.update({'label': self.labels(one_off=one_off, legacy=True)}) + return list( + filter( + self.has_legacy_proj_name, filter(None, [ + Container.from_ps(self.client, container) + for container in self.client.containers( + all=stopped, + filters=filters)]) + ) + ) def get_container(self, number=1): """Return a :class:`compose.container.Container` for this service. The @@ -380,6 +395,10 @@ class Service(object): has_diverged = False for c in containers: + if self.has_legacy_proj_name(c): + log.debug('%s has diverged: Legacy project name' % c.name) + has_diverged = True + continue container_config_hash = c.labels.get(LABEL_CONFIG_HASH, None) if container_config_hash != config_hash: log.debug( @@ -1053,11 +1072,12 @@ class Service(object): def can_be_built(self): return 'build' in self.options - def labels(self, one_off=False): + def labels(self, one_off=False, legacy=False): + proj_name = self.project if not legacy else re.sub(r'[_-]', '', self.project) return [ - '{0}={1}'.format(LABEL_PROJECT, self.project), + '{0}={1}'.format(LABEL_PROJECT, proj_name), '{0}={1}'.format(LABEL_SERVICE, self.name), - '{0}={1}'.format(LABEL_ONE_OFF, "True" if one_off else "False") + '{0}={1}'.format(LABEL_ONE_OFF, "True" if one_off else "False"), ] @property @@ -1214,6 +1234,12 @@ class Service(object): return result + def has_legacy_proj_name(self, ctnr): + return ( + ComposeVersion(ctnr.labels.get(LABEL_VERSION)) < ComposeVersion('1.21.0') and + ctnr.project != self.project + ) + def short_id_alias_exists(container, network): aliases = container.get( diff --git a/compose/volume.py b/compose/volume.py index 6bf184045..6cad1e0de 100644 --- a/compose/volume.py +++ b/compose/volume.py @@ -2,15 +2,19 @@ from __future__ import absolute_import from __future__ import unicode_literals import logging +import re from docker.errors import NotFound from docker.utils import version_lt +from . import __version__ from .config import ConfigurationError from .config.types import VolumeSpec from .const import LABEL_PROJECT +from .const import LABEL_VERSION from .const import LABEL_VOLUME + log = logging.getLogger(__name__) @@ -25,6 +29,7 @@ class Volume(object): self.external = external self.labels = labels self.custom_name = custom_name + self.legacy = False def create(self): return self.client.create_volume( @@ -36,15 +41,26 @@ class Volume(object): log.info("Volume %s is external, skipping", self.full_name) return log.info("Removing volume %s", self.full_name) - return self.client.remove_volume(self.full_name) + try: + return self.client.remove_volume(self.full_name) + except NotFound: + self.client.remove_volume(self.legacy_full_name) - def inspect(self): + def inspect(self, legacy=False): + if legacy: + return self.client.inspect_volume(self.legacy_full_name) return self.client.inspect_volume(self.full_name) def exists(self): try: self.inspect() except NotFound: + try: + self.inspect(legacy=True) + self.legacy = True + return True + except NotFound: + pass return False return True @@ -54,6 +70,20 @@ class Volume(object): return self.name return '{0}_{1}'.format(self.project, self.name) + @property + def legacy_full_name(self): + if self.custom_name: + return self.name + return '{0}_{1}'.format( + re.sub(r'[_-]', '', self.project), self.name + ) + + @property + def true_name(self): + if self.legacy: + return self.legacy_full_name + return self.full_name + @property def _labels(self): if version_lt(self.client._version, '1.23'): @@ -62,6 +92,7 @@ class Volume(object): labels.update({ LABEL_PROJECT: self.project, LABEL_VOLUME: self.name, + LABEL_VERSION: __version__, }) return labels @@ -94,7 +125,7 @@ class ProjectVolumes(object): try: volume.remove() except NotFound: - log.warn("Volume %s not found.", volume.full_name) + log.warn("Volume %s not found.", volume.true_name) def initialize(self): try: @@ -136,9 +167,9 @@ class ProjectVolumes(object): if isinstance(volume_spec, VolumeSpec): volume = self.volumes[volume_spec.external] - return volume_spec._replace(external=volume.full_name) + return volume_spec._replace(external=volume.true_name) else: - volume_spec.source = self.volumes[volume_spec.source].full_name + volume_spec.source = self.volumes[volume_spec.source].true_name return volume_spec @@ -152,7 +183,7 @@ class VolumeConfigChangedError(ConfigurationError): 'first:\n$ docker volume rm {full_name}'.format( vol_name=local.name, property_name=property_name, local_value=local_value, remote_value=remote_value, - full_name=local.full_name + full_name=local.true_name ) ) From 299ce6ad009fa9f56564c51a4cd86eafa6e7ccee Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Mon, 23 Apr 2018 18:16:58 -0700 Subject: [PATCH 25/53] Incorrect key name for IPAM options check Signed-off-by: Joffrey F --- compose/network.py | 2 +- tests/unit/network_test.py | 8 +++++++- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/compose/network.py b/compose/network.py index 1a080c40c..7803db979 100644 --- a/compose/network.py +++ b/compose/network.py @@ -169,7 +169,7 @@ def check_remote_ipam_config(remote, local): raise NetworkConfigChangedError(local.full_name, 'IPAM config aux_addresses') remote_opts = remote_ipam.get('Options') or {} - local_opts = local.ipam.get('options') or {} + local_opts = local.ipam.get('Options') or {} for k in set.union(set(remote_opts.keys()), set(local_opts.keys())): if remote_opts.get(k) != local_opts.get(k): raise NetworkConfigChangedError(local.full_name, 'IPAM option "{}"'.format(k)) diff --git a/tests/unit/network_test.py b/tests/unit/network_test.py index b27339af8..0e03fc10e 100644 --- a/tests/unit/network_test.py +++ b/tests/unit/network_test.py @@ -23,7 +23,10 @@ class NetworkTest(unittest.TestCase): 'aux_addresses': ['11.0.0.1', '24.25.26.27'], 'ip_range': '156.0.0.1-254' } - ] + ], + 'options': { + 'iface': 'eth0', + } } labels = { 'com.project.tests.istest': 'true', @@ -57,6 +60,9 @@ class NetworkTest(unittest.TestCase): 'Subnet': '172.0.0.1/16', 'Gateway': '172.0.0.1' }], + 'Options': { + 'iface': 'eth0', + }, }, 'Labels': remote_labels }, From fa6d837b49a607d3fd480f50d2e1adec3dabd9e2 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Mon, 23 Apr 2018 19:08:55 -0700 Subject: [PATCH 26/53] Clearly define IPAM config schema for validation Signed-off-by: Joffrey F --- compose/config/config_schema_v2.0.json | 21 ++++++++++++++++++++- compose/config/config_schema_v2.1.json | 21 ++++++++++++++++++++- compose/config/config_schema_v2.2.json | 21 ++++++++++++++++++++- compose/config/config_schema_v2.3.json | 21 ++++++++++++++++++++- compose/config/config_schema_v2.4.json | 21 ++++++++++++++++++++- tests/unit/config/config_test.py | 22 ++++++++++++++++++++++ 6 files changed, 122 insertions(+), 5 deletions(-) diff --git a/compose/config/config_schema_v2.0.json b/compose/config/config_schema_v2.0.json index eddf787ea..793cef1d6 100644 --- a/compose/config/config_schema_v2.0.json +++ b/compose/config/config_schema_v2.0.json @@ -281,7 +281,8 @@ "properties": { "driver": {"type": "string"}, "config": { - "type": "array" + "type": "array", + "items": {"$ref": "#/definitions/ipam_config"} }, "options": { "type": "object", @@ -305,6 +306,24 @@ "additionalProperties": false }, + "ipam_config": { + "id": "#/definitions/ipam_config", + "type": "object", + "properties": { + "subnet": {"type": "string"}, + "iprange": {"type": "string"}, + "gateway": {"type": "string"}, + "aux_addresses": { + "type": "object", + "patternProperties": { + "^.+$": {"type": "string"} + }, + "additionalProperties": false + } + }, + "additionalProperties": false + }, + "volume": { "id": "#/definitions/volume", "type": ["object", "null"], diff --git a/compose/config/config_schema_v2.1.json b/compose/config/config_schema_v2.1.json index 5ad5a20ea..5ea763544 100644 --- a/compose/config/config_schema_v2.1.json +++ b/compose/config/config_schema_v2.1.json @@ -332,7 +332,8 @@ "properties": { "driver": {"type": "string"}, "config": { - "type": "array" + "type": "array", + "items": {"$ref": "#/definitions/ipam_config"} }, "options": { "type": "object", @@ -359,6 +360,24 @@ "additionalProperties": false }, + "ipam_config": { + "id": "#/definitions/ipam_config", + "type": "object", + "properties": { + "subnet": {"type": "string"}, + "iprange": {"type": "string"}, + "gateway": {"type": "string"}, + "aux_addresses": { + "type": "object", + "patternProperties": { + "^.+$": {"type": "string"} + }, + "additionalProperties": false + } + }, + "additionalProperties": false + }, + "volume": { "id": "#/definitions/volume", "type": ["object", "null"], diff --git a/compose/config/config_schema_v2.2.json b/compose/config/config_schema_v2.2.json index 26044b651..a19d4c945 100644 --- a/compose/config/config_schema_v2.2.json +++ b/compose/config/config_schema_v2.2.json @@ -341,7 +341,8 @@ "properties": { "driver": {"type": "string"}, "config": { - "type": "array" + "type": "array", + "items": {"$ref": "#/definitions/ipam_config"} }, "options": { "type": "object", @@ -368,6 +369,24 @@ "additionalProperties": false }, + "ipam_config": { + "id": "#/definitions/ipam_config", + "type": "object", + "properties": { + "subnet": {"type": "string"}, + "iprange": {"type": "string"}, + "gateway": {"type": "string"}, + "aux_addresses": { + "type": "object", + "patternProperties": { + "^.+$": {"type": "string"} + }, + "additionalProperties": false + } + }, + "additionalProperties": false + }, + "volume": { "id": "#/definitions/volume", "type": ["object", "null"], diff --git a/compose/config/config_schema_v2.3.json b/compose/config/config_schema_v2.3.json index ac0778f2a..78b716a7a 100644 --- a/compose/config/config_schema_v2.3.json +++ b/compose/config/config_schema_v2.3.json @@ -385,7 +385,8 @@ "properties": { "driver": {"type": "string"}, "config": { - "type": "array" + "type": "array", + "items": {"$ref": "#/definitions/ipam_config"} }, "options": { "type": "object", @@ -412,6 +413,24 @@ "additionalProperties": false }, + "ipam_config": { + "id": "#/definitions/ipam_config", + "type": "object", + "properties": { + "subnet": {"type": "string"}, + "iprange": {"type": "string"}, + "gateway": {"type": "string"}, + "aux_addresses": { + "type": "object", + "patternProperties": { + "^.+$": {"type": "string"} + }, + "additionalProperties": false + } + }, + "additionalProperties": false + }, + "volume": { "id": "#/definitions/volume", "type": ["object", "null"], diff --git a/compose/config/config_schema_v2.4.json b/compose/config/config_schema_v2.4.json index 731fa2f9b..a5796d5b1 100644 --- a/compose/config/config_schema_v2.4.json +++ b/compose/config/config_schema_v2.4.json @@ -384,7 +384,8 @@ "properties": { "driver": {"type": "string"}, "config": { - "type": "array" + "type": "array", + "items": {"$ref": "#/definitions/ipam_config"} }, "options": { "type": "object", @@ -411,6 +412,24 @@ "additionalProperties": false }, + "ipam_config": { + "id": "#/definitions/ipam_config", + "type": "object", + "properties": { + "subnet": {"type": "string"}, + "iprange": {"type": "string"}, + "gateway": {"type": "string"}, + "aux_addresses": { + "type": "object", + "patternProperties": { + "^.+$": {"type": "string"} + }, + "additionalProperties": false + } + }, + "additionalProperties": false + }, + "volume": { "id": "#/definitions/volume", "type": ["object", "null"], diff --git a/tests/unit/config/config_test.py b/tests/unit/config/config_test.py index 8a75648ac..4562a99ca 100644 --- a/tests/unit/config/config_test.py +++ b/tests/unit/config/config_test.py @@ -1322,6 +1322,28 @@ class ConfigTest(unittest.TestCase): assert mount.type == 'bind' assert mount.source == expected_source + def test_config_invalid_ipam_config(self): + with pytest.raises(ConfigurationError) as excinfo: + config.load( + build_config_details( + { + 'version': str(V2_1), + 'networks': { + 'foo': { + 'driver': 'default', + 'ipam': { + 'driver': 'default', + 'config': ['172.18.0.0/16'], + } + } + } + }, + filename='filename.yml', + ) + ) + assert ('networks.foo.ipam.config contains an invalid type,' + ' it should be an object') in excinfo.exconly() + def test_config_valid_service_names(self): for valid_name in ['_', '-', '.__.', '_what-up.', 'what_.up----', 'whatup']: services = config.load( From c1657dc46aab2edafbd175b27756e367839129d5 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Tue, 24 Apr 2018 15:48:02 -0700 Subject: [PATCH 27/53] Improve legacy network and volume detection Signed-off-by: Joffrey F --- compose/network.py | 26 ++++++++++++++------------ compose/volume.py | 25 +++++++++++++++---------- 2 files changed, 29 insertions(+), 22 deletions(-) diff --git a/compose/network.py b/compose/network.py index e0fdb3e27..b00e2ae99 100644 --- a/compose/network.py +++ b/compose/network.py @@ -42,7 +42,7 @@ class Network(object): self.enable_ipv6 = enable_ipv6 self.labels = labels self.custom_name = custom_name - self.legacy = False + self.legacy = None def ensure(self): if self.external: @@ -68,25 +68,17 @@ class Network(object): ) return + self._set_legacy_flag() try: - data = self.inspect() + data = self.inspect(legacy=self.legacy) check_remote_network_config(data, self) except NotFound: - try: - data = self.inspect(legacy=True) - self.legacy = True - check_remote_network_config(data, self) - return - except NotFound: - pass - driver_name = 'the default driver' if self.driver: driver_name = 'driver "{}"'.format(self.driver) log.info( - 'Creating network "{}" with {}' - .format(self.full_name, driver_name) + 'Creating network "{}" with {}'.format(self.full_name, driver_name) ) self.client.create_network( @@ -133,6 +125,7 @@ class Network(object): @property def true_name(self): + self._set_legacy_flag() if self.legacy: return self.legacy_full_name return self.full_name @@ -149,6 +142,15 @@ class Network(object): }) return labels + def _set_legacy_flag(self): + if self.legacy is not None: + return + try: + data = self.inspect(legacy=True) + self.legacy = data is not None + except NotFound: + self.legacy = False + def create_ipam_config_from_dict(ipam_dict): if not ipam_dict: diff --git a/compose/volume.py b/compose/volume.py index 6cad1e0de..56ff601cd 100644 --- a/compose/volume.py +++ b/compose/volume.py @@ -29,7 +29,7 @@ class Volume(object): self.external = external self.labels = labels self.custom_name = custom_name - self.legacy = False + self.legacy = None def create(self): return self.client.create_volume( @@ -46,21 +46,16 @@ class Volume(object): except NotFound: self.client.remove_volume(self.legacy_full_name) - def inspect(self, legacy=False): + def inspect(self, legacy=None): if legacy: return self.client.inspect_volume(self.legacy_full_name) return self.client.inspect_volume(self.full_name) def exists(self): + self._set_legacy_flag() try: - self.inspect() + self.inspect(legacy=self.legacy) except NotFound: - try: - self.inspect(legacy=True) - self.legacy = True - return True - except NotFound: - pass return False return True @@ -80,6 +75,7 @@ class Volume(object): @property def true_name(self): + self._set_legacy_flag() if self.legacy: return self.legacy_full_name return self.full_name @@ -96,6 +92,15 @@ class Volume(object): }) return labels + def _set_legacy_flag(self): + if self.legacy is not None: + return + try: + data = self.inspect(legacy=True) + self.legacy = data is not None + except NotFound: + self.legacy = False + class ProjectVolumes(object): @@ -155,7 +160,7 @@ class ProjectVolumes(object): ) volume.create() else: - check_remote_volume_config(volume.inspect(), volume) + check_remote_volume_config(volume.inspect(legacy=volume.legacy), volume) except NotFound: raise ConfigurationError( 'Volume %s specifies nonexistent driver %s' % (volume.name, volume.driver) From 3b2ce82fa1c2131a542d3af7768c6b4c56094a18 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Tue, 24 Apr 2018 16:10:59 -0700 Subject: [PATCH 28/53] Use true_name for remove operation Signed-off-by: Joffrey F --- compose/network.py | 7 ++----- compose/volume.py | 9 +++------ tests/unit/network_test.py | 3 +++ tests/unit/project_test.py | 2 ++ 4 files changed, 10 insertions(+), 11 deletions(-) diff --git a/compose/network.py b/compose/network.py index b00e2ae99..4ac3a0ea4 100644 --- a/compose/network.py +++ b/compose/network.py @@ -95,14 +95,11 @@ class Network(object): def remove(self): if self.external: - log.info("Network %s is external, skipping", self.full_name) + log.info("Network %s is external, skipping", self.true_name) return log.info("Removing network {}".format(self.true_name)) - try: - self.client.remove_network(self.full_name) - except NotFound: - self.client.remove_network(self.legacy_full_name) + self.client.remove_network(self.true_name) def inspect(self, legacy=False): if legacy: diff --git a/compose/volume.py b/compose/volume.py index 56ff601cd..7618417ff 100644 --- a/compose/volume.py +++ b/compose/volume.py @@ -38,13 +38,10 @@ class Volume(object): def remove(self): if self.external: - log.info("Volume %s is external, skipping", self.full_name) + log.info("Volume %s is external, skipping", self.true_name) return - log.info("Removing volume %s", self.full_name) - try: - return self.client.remove_volume(self.full_name) - except NotFound: - self.client.remove_volume(self.legacy_full_name) + log.info("Removing volume %s", self.true_name) + return self.client.remove_volume(self.true_name) def inspect(self, legacy=None): if legacy: diff --git a/tests/unit/network_test.py b/tests/unit/network_test.py index b27339af8..426156662 100644 --- a/tests/unit/network_test.py +++ b/tests/unit/network_test.py @@ -78,6 +78,7 @@ class NetworkTest(unittest.TestCase): {'Driver': 'overlay', 'Options': remote_options}, net ) + @mock.patch('compose.network.Network.true_name', lambda n: n.full_name) def test_check_remote_network_config_driver_mismatch(self): net = Network(None, 'compose_test', 'net1', 'overlay') with pytest.raises(NetworkConfigChangedError) as e: @@ -87,6 +88,7 @@ class NetworkTest(unittest.TestCase): assert 'driver has changed' in str(e.value) + @mock.patch('compose.network.Network.true_name', lambda n: n.full_name) def test_check_remote_network_config_options_mismatch(self): net = Network(None, 'compose_test', 'net1', 'overlay') with pytest.raises(NetworkConfigChangedError) as e: @@ -140,6 +142,7 @@ class NetworkTest(unittest.TestCase): net ) + @mock.patch('compose.network.Network.true_name', lambda n: n.full_name) def test_check_remote_network_labels_mismatch(self): net = Network(None, 'compose_test', 'net1', 'overlay', labels={ 'com.project.touhou.character': 'sakuya.izayoi' diff --git a/tests/unit/project_test.py b/tests/unit/project_test.py index 83a014758..1b6b6651f 100644 --- a/tests/unit/project_test.py +++ b/tests/unit/project_test.py @@ -60,6 +60,7 @@ class ProjectTest(unittest.TestCase): assert project.get_service('db').options['image'] == 'busybox:latest' assert not project.networks.use_networking + @mock.patch('compose.network.Network.true_name', lambda n: n.full_name) def test_from_config_v2(self): config = Config( version=V2_0, @@ -217,6 +218,7 @@ class ProjectTest(unittest.TestCase): ) assert project.get_service('test')._get_volumes_from() == [container_name + ":rw"] + @mock.patch('compose.network.Network.true_name', lambda n: n.full_name) def test_use_volumes_from_service_container(self): container_ids = ['aabbccddee', '12345'] From 6e09e37114b59196661a08d94cd4b36ea3f1f25f Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Wed, 25 Apr 2018 18:08:34 -0700 Subject: [PATCH 29/53] Bump SDK version to latest Signed-off-by: Joffrey F --- requirements.txt | 4 ++-- setup.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/requirements.txt b/requirements.txt index 7dce40246..93a0cce35 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,8 +2,8 @@ backports.ssl-match-hostname==3.5.0.1; python_version < '3' cached-property==1.3.0 certifi==2017.4.17 chardet==3.0.4 -docker==3.2.1 -docker-pycreds==0.2.1 +docker==3.3.0 +docker-pycreds==0.2.3 dockerpty==0.4.1 docopt==0.6.2 enum34==1.1.6; python_version < '3.4' diff --git a/setup.py b/setup.py index a7a333634..422ba5466 100644 --- a/setup.py +++ b/setup.py @@ -36,7 +36,7 @@ install_requires = [ 'requests >= 2.6.1, != 2.11.0, != 2.12.2, != 2.18.0, < 2.19', 'texttable >= 0.9.0, < 0.10', 'websocket-client >= 0.32.0, < 1.0', - 'docker >= 3.2.1, < 4.0', + 'docker >= 3.3.0, < 4.0', 'dockerpty >= 0.4.1, < 0.5', 'six >= 1.3.0, < 2', 'jsonschema >= 2.5.1, < 3', From aecc0de28fdcbeb3a4df3ef876ef7f58cc998396 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Thu, 26 Apr 2018 15:20:45 -0700 Subject: [PATCH 30/53] Prevent duplicate binds in generated container config Signed-off-by: Joffrey F --- compose/service.py | 7 ++++--- tests/unit/service_test.py | 19 +++++++++++++++++++ 2 files changed, 23 insertions(+), 3 deletions(-) diff --git a/compose/service.py b/compose/service.py index 0a866161c..ae9e0bb08 100644 --- a/compose/service.py +++ b/compose/service.py @@ -877,7 +877,6 @@ class Service(object): container_volumes, self.options.get('tmpfs') or [], previous_container, container_mounts ) - override_options['binds'] = binds container_options['environment'].update(affinity) container_options['volumes'] = dict((v.internal, {}) for v in container_volumes or {}) @@ -890,13 +889,13 @@ class Service(object): if m.is_tmpfs: override_options['tmpfs'].append(m.target) else: - override_options['binds'].append(m.legacy_repr()) + binds.append(m.legacy_repr()) container_options['volumes'][m.target] = {} secret_volumes = self.get_secret_volumes() if secret_volumes: if version_lt(self.client.api_version, '1.30'): - override_options['binds'].extend(v.legacy_repr() for v in secret_volumes) + binds.extend(v.legacy_repr() for v in secret_volumes) container_options['volumes'].update( (v.target, {}) for v in secret_volumes ) @@ -904,6 +903,8 @@ class Service(object): override_options['mounts'] = override_options.get('mounts') or [] override_options['mounts'].extend([build_mount(v) for v in secret_volumes]) + # Remove possible duplicates (see e.g. https://github.com/docker/compose/issues/5885) + override_options['binds'] = list(set(binds)) return container_options, override_options def _get_container_host_config(self, override_options, one_off=False): diff --git a/tests/unit/service_test.py b/tests/unit/service_test.py index 4ccc48653..d50db9044 100644 --- a/tests/unit/service_test.py +++ b/tests/unit/service_test.py @@ -10,6 +10,7 @@ from docker.errors import NotFound from .. import mock from .. import unittest from compose.config.errors import DependencyError +from compose.config.types import MountSpec from compose.config.types import ServicePort from compose.config.types import ServiceSecret from compose.config.types import VolumeFromSpec @@ -955,6 +956,24 @@ class ServiceTest(unittest.TestCase): assert service.create_container().id == 'new_cont_id' + def test_build_volume_options_duplicate_binds(self): + self.mock_client.api_version = '1.29' # Trigger 3.2 format workaround + service = Service('foo', client=self.mock_client) + ctnr_opts, override_opts = service._build_container_volume_options( + previous_container=None, + container_options={ + 'volumes': [ + MountSpec.parse({'source': 'vol', 'target': '/data', 'type': 'volume'}), + VolumeSpec.parse('vol:/data:rw'), + ], + 'environment': {}, + }, + override_options={}, + ) + assert 'binds' in override_opts + assert len(override_opts['binds']) == 1 + assert override_opts['binds'][0] == 'vol:/data:rw' + class TestServiceNetwork(unittest.TestCase): def setUp(self): From d469113b3742e0761f70e0f5b073f7061d8c854e Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Fri, 27 Apr 2018 12:24:43 -0700 Subject: [PATCH 31/53] Improve release automation Signed-off-by: Joffrey F --- script/release/release.py | 17 +++++++++++------ script/release/release.sh | 1 + 2 files changed, 12 insertions(+), 6 deletions(-) diff --git a/script/release/release.py b/script/release/release.py index add8fb2d3..4357e36b9 100755 --- a/script/release/release.py +++ b/script/release/release.py @@ -78,10 +78,9 @@ def monitor_pr_status(pr_data): continue summary[detail.state] += 1 print('{pending} pending, {success} successes, {failure} failures'.format(**summary)) - if status.total_count == 0: - # Mostly for testing purposes against repos with no CI setup - return True - elif summary['pending'] == 0 and summary['failure'] == 0: + if summary['pending'] == 0 and summary['failure'] == 0 and summary['success'] > 0: + # This check assumes at least 1 non-DCO CI check to avoid race conditions. + # If testing on a repo without CI, use --skip-ci-check to avoid looping eternally return True elif summary['failure'] > 0: raise ScriptError('CI failures detected!') @@ -156,7 +155,8 @@ def resume(args): if not pr_data: pr_data = repository.create_release_pull_request(args.release) check_pr_mergeable(pr_data) - monitor_pr_status(pr_data) + if not args.skip_ci: + monitor_pr_status(pr_data) downloader = BinaryDownloader(args.destination) files = downloader.download_all(args.release) if not gh_release: @@ -195,7 +195,8 @@ def start(args): create_initial_branch(repository, args) pr_data = repository.create_release_pull_request(args.release) check_pr_mergeable(pr_data) - monitor_pr_status(pr_data) + if not args.skip_ci: + monitor_pr_status(pr_data) downloader = BinaryDownloader(args.destination) files = downloader.download_all(args.release) gh_release = create_release_draft(repository, args.release, pr_data, files) @@ -310,6 +311,10 @@ def main(): '--no-cherries', '-C', dest='cherries', action='store_false', help='If set, the program will not prompt the user for PR numbers to cherry-pick' ) + parser.add_argument( + '--skip-ci-checks', dest='skip_ci', action='store_true', + help='If set, the program will not wait for CI jobs to complete' + ) args = parser.parse_args() if args.action == 'start': diff --git a/script/release/release.sh b/script/release/release.sh index 2310429aa..f592365d3 100755 --- a/script/release/release.sh +++ b/script/release/release.sh @@ -19,6 +19,7 @@ docker run -e GITHUB_TOKEN=$GITHUB_TOKEN -e BINTRAY_TOKEN=$BINTRAY_TOKEN -it \ --mount type=bind,source=$(pwd),target=/src \ --mount type=bind,source=$(pwd)/.git,target=/src/.git \ --mount type=bind,source=$HOME/.docker,target=/root/.docker \ + --mount type=bind,source=$HOME/.gitconfig,target=/root/.gitconfig --mount type=bind,source=/var/run/docker.sock,target=/var/run/docker.sock \ --mount type=bind,source=$HOME/.ssh,target=/root/.ssh \ -v $HOME/.pypirc:/root/.pypirc \ From 90c89e34f19364023f9ff06a32c059c3c86efaaa Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Fri, 27 Apr 2018 14:35:13 -0700 Subject: [PATCH 32/53] Finalize fixes Signed-off-by: Joffrey F --- script/release/Dockerfile | 3 ++- script/release/release.py | 5 +++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/script/release/Dockerfile b/script/release/Dockerfile index 0d4ec27e1..e5af676a5 100644 --- a/script/release/Dockerfile +++ b/script/release/Dockerfile @@ -3,7 +3,8 @@ RUN mkdir -p /src && pip install -U Jinja2==2.10 \ PyGithub==1.39 \ pypandoc==1.4 \ GitPython==2.1.9 \ - requests==2.18.4 && \ + requests==2.18.4 \ + twine==1.11.0 && \ apt-get update && apt-get install -y pandoc VOLUME /src/script/release diff --git a/script/release/release.py b/script/release/release.py index 4357e36b9..d0545a7e6 100755 --- a/script/release/release.py +++ b/script/release/release.py @@ -27,6 +27,7 @@ from release.utils import ScriptError from release.utils import update_init_py_version from release.utils import update_run_sh_version from release.utils import yesno +from twine.commands.upload import main as twine_upload def create_initial_branch(repository, args): @@ -240,8 +241,8 @@ def finalize(args): if not merge_status.merged: raise ScriptError('Unable to merge PR #{}: {}'.format(pr_data.number, merge_status.message)) print('Uploading to PyPi') - run_setup(os.path.join(REPO_ROOT, 'setup.py'), script_args=['upload']) - img_manager.push_images(args.release) + twine_upload(['dist/*']) + img_manager.push_images() repository.publish_release(gh_release) except ScriptError as e: print(e) From bc0344155016d713587db564a39800899f6be04c Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Fri, 27 Apr 2018 18:36:48 -0700 Subject: [PATCH 33/53] Automatically detect pickable PRs for patch releases Signed-off-by: Joffrey F --- script/release/release.py | 9 +++++++++ script/release/release/repository.py | 18 ++++++++++++++++++ 2 files changed, 27 insertions(+) diff --git a/script/release/release.py b/script/release/release.py index d0545a7e6..e9a52c4aa 100755 --- a/script/release/release.py +++ b/script/release/release.py @@ -34,6 +34,15 @@ def create_initial_branch(repository, args): release_branch = repository.create_release_branch(args.release, args.base) if args.base and args.cherries: print('Detected patch version.') + auto_prs = repository.get_prs_in_milestone(args.release) + if auto_prs: + print( + 'Found the following PRs in this release\'s milestone: {}'.format(', '.join(auto_prs)) + ) + proceed = yesno('Automatically cherry-pick detected PRs? Y/n', default=True) + if proceed: + repository.cherry_pick_prs(release_branch, auto_prs) + cherries = input('Indicate (space-separated) PR numbers to cherry-pick then press Enter:\n') repository.cherry_pick_prs(release_branch, cherries.split()) diff --git a/script/release/release/repository.py b/script/release/release/repository.py index d4d1c7201..9a5d432c0 100644 --- a/script/release/release/repository.py +++ b/script/release/release/repository.py @@ -196,6 +196,24 @@ class Repository(object): f.flush() self.git_repo.git.am('--3way', f.name) + def get_prs_in_milestone(self, version): + milestones = self.gh_repo.get_milestones(state='open') + milestone = None + for ms in milestones: + if ms.title == version: + milestone = ms + break + if not milestone: + print('Didn\'t find a milestone matching "{}"'.format(version)) + return None + + issues = self.gh_repo.get_issues(milestone=milestone, state='all') + prs = [] + for issue in issues: + if issue.pull_request is not None: + prs.append(issue.number) + return sorted(prs) + def get_contributors(pr_data): commits = pr_data.get_commits() From 5eb3f4b32f04c4471b2d537a8bb52480d6eae1c0 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Fri, 27 Apr 2018 18:43:05 -0700 Subject: [PATCH 34/53] Typo fix Signed-off-by: Joffrey F --- script/release/release.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/script/release/release.sh b/script/release/release.sh index f592365d3..affbce37b 100755 --- a/script/release/release.sh +++ b/script/release/release.sh @@ -19,7 +19,7 @@ docker run -e GITHUB_TOKEN=$GITHUB_TOKEN -e BINTRAY_TOKEN=$BINTRAY_TOKEN -it \ --mount type=bind,source=$(pwd),target=/src \ --mount type=bind,source=$(pwd)/.git,target=/src/.git \ --mount type=bind,source=$HOME/.docker,target=/root/.docker \ - --mount type=bind,source=$HOME/.gitconfig,target=/root/.gitconfig + --mount type=bind,source=$HOME/.gitconfig,target=/root/.gitconfig \ --mount type=bind,source=/var/run/docker.sock,target=/var/run/docker.sock \ --mount type=bind,source=$HOME/.ssh,target=/root/.ssh \ -v $HOME/.pypirc:/root/.pypirc \ From e6aedb1ce0837918b53d20d3501b740b47a4b920 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Fri, 27 Apr 2018 18:48:30 -0700 Subject: [PATCH 35/53] Partial revert bc034415501 Signed-off-by: Joffrey F --- script/release/release.py | 9 --------- 1 file changed, 9 deletions(-) diff --git a/script/release/release.py b/script/release/release.py index e9a52c4aa..d0545a7e6 100755 --- a/script/release/release.py +++ b/script/release/release.py @@ -34,15 +34,6 @@ def create_initial_branch(repository, args): release_branch = repository.create_release_branch(args.release, args.base) if args.base and args.cherries: print('Detected patch version.') - auto_prs = repository.get_prs_in_milestone(args.release) - if auto_prs: - print( - 'Found the following PRs in this release\'s milestone: {}'.format(', '.join(auto_prs)) - ) - proceed = yesno('Automatically cherry-pick detected PRs? Y/n', default=True) - if proceed: - repository.cherry_pick_prs(release_branch, auto_prs) - cherries = input('Indicate (space-separated) PR numbers to cherry-pick then press Enter:\n') repository.cherry_pick_prs(release_branch, cherries.split()) From 05638ab5ead4819b6af5dfb1f40b3ea37c8a0e12 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Sat, 28 Apr 2018 13:42:24 -0700 Subject: [PATCH 36/53] Esnure docker-compose binary is executable (fixes #5917) Signed-off-by: Joffrey F --- script/release/release/images.py | 1 + 1 file changed, 1 insertion(+) diff --git a/script/release/release/images.py b/script/release/release/images.py index 0c7bb2045..d238d4d7f 100644 --- a/script/release/release/images.py +++ b/script/release/release/images.py @@ -23,6 +23,7 @@ class ImageManager(object): distdir = os.path.join(REPO_ROOT, 'dist') os.makedirs(distdir, exist_ok=True) shutil.copy(files['docker-compose-Linux-x86_64'][0], distdir) + os.chmod(os.path.join(distdir, 'docker-compose-Linux-x86_64'), 0o755) print('Building docker/compose image') logstream = docker_client.build( REPO_ROOT, tag='docker/compose:{}'.format(self.version), dockerfile='Dockerfile.run', From 5aafa54667f95660e4863d4a0a40f8f20353191b Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Tue, 1 May 2018 17:11:14 -0700 Subject: [PATCH 37/53] iprange -> ip_range Signed-off-by: Joffrey F --- compose/config/config_schema_v2.0.json | 2 +- compose/config/config_schema_v2.1.json | 2 +- compose/config/config_schema_v2.2.json | 2 +- compose/config/config_schema_v2.3.json | 2 +- compose/config/config_schema_v2.4.json | 2 +- tests/unit/config/config_test.py | 32 ++++++++++++++++++++++++++ 6 files changed, 37 insertions(+), 5 deletions(-) diff --git a/compose/config/config_schema_v2.0.json b/compose/config/config_schema_v2.0.json index 793cef1d6..419f2e28c 100644 --- a/compose/config/config_schema_v2.0.json +++ b/compose/config/config_schema_v2.0.json @@ -311,7 +311,7 @@ "type": "object", "properties": { "subnet": {"type": "string"}, - "iprange": {"type": "string"}, + "ip_range": {"type": "string"}, "gateway": {"type": "string"}, "aux_addresses": { "type": "object", diff --git a/compose/config/config_schema_v2.1.json b/compose/config/config_schema_v2.1.json index 5ea763544..3cb1ee213 100644 --- a/compose/config/config_schema_v2.1.json +++ b/compose/config/config_schema_v2.1.json @@ -365,7 +365,7 @@ "type": "object", "properties": { "subnet": {"type": "string"}, - "iprange": {"type": "string"}, + "ip_range": {"type": "string"}, "gateway": {"type": "string"}, "aux_addresses": { "type": "object", diff --git a/compose/config/config_schema_v2.2.json b/compose/config/config_schema_v2.2.json index a19d4c945..8e1f288ba 100644 --- a/compose/config/config_schema_v2.2.json +++ b/compose/config/config_schema_v2.2.json @@ -374,7 +374,7 @@ "type": "object", "properties": { "subnet": {"type": "string"}, - "iprange": {"type": "string"}, + "ip_range": {"type": "string"}, "gateway": {"type": "string"}, "aux_addresses": { "type": "object", diff --git a/compose/config/config_schema_v2.3.json b/compose/config/config_schema_v2.3.json index 78b716a7a..659dbcd1a 100644 --- a/compose/config/config_schema_v2.3.json +++ b/compose/config/config_schema_v2.3.json @@ -418,7 +418,7 @@ "type": "object", "properties": { "subnet": {"type": "string"}, - "iprange": {"type": "string"}, + "ip_range": {"type": "string"}, "gateway": {"type": "string"}, "aux_addresses": { "type": "object", diff --git a/compose/config/config_schema_v2.4.json b/compose/config/config_schema_v2.4.json index a5796d5b1..47e118755 100644 --- a/compose/config/config_schema_v2.4.json +++ b/compose/config/config_schema_v2.4.json @@ -417,7 +417,7 @@ "type": "object", "properties": { "subnet": {"type": "string"}, - "iprange": {"type": "string"}, + "ip_range": {"type": "string"}, "gateway": {"type": "string"}, "aux_addresses": { "type": "object", diff --git a/tests/unit/config/config_test.py b/tests/unit/config/config_test.py index 4562a99ca..085a2d010 100644 --- a/tests/unit/config/config_test.py +++ b/tests/unit/config/config_test.py @@ -1344,6 +1344,38 @@ class ConfigTest(unittest.TestCase): assert ('networks.foo.ipam.config contains an invalid type,' ' it should be an object') in excinfo.exconly() + def test_config_valid_ipam_config(self): + ipam_config = { + 'subnet': '172.28.0.0/16', + 'ip_range': '172.28.5.0/24', + 'gateway': '172.28.5.254', + 'aux_addresses': { + 'host1': '172.28.1.5', + 'host2': '172.28.1.6', + 'host3': '172.28.1.7', + }, + } + networks = config.load( + build_config_details( + { + 'version': str(V2_1), + 'networks': { + 'foo': { + 'driver': 'default', + 'ipam': { + 'driver': 'default', + 'config': [ipam_config], + } + } + } + }, + filename='filename.yml', + ) + ).networks + + assert 'foo' in networks + assert networks['foo']['ipam']['config'] == [ipam_config] + def test_config_valid_service_names(self): for valid_name in ['_', '-', '.__.', '_what-up.', 'what_.up----', 'whatup']: services = config.load( From c3bb9588651969c3b6161aa7530d8d4becd2753f Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Fri, 4 May 2018 14:06:03 -0700 Subject: [PATCH 38/53] Ignore default platform if API version doesn't support platform param Signed-off-by: Joffrey F --- compose/project.py | 3 +- compose/service.py | 18 ++++++++---- tests/unit/project_test.py | 9 +++--- tests/unit/service_test.py | 57 +++++++++++++++++++++++++++++++++++++- 4 files changed, 76 insertions(+), 11 deletions(-) diff --git a/compose/project.py b/compose/project.py index 924390b4e..c27794fc3 100644 --- a/compose/project.py +++ b/compose/project.py @@ -128,7 +128,8 @@ class Project(object): volumes_from=volumes_from, secrets=secrets, pid_mode=pid_mode, - platform=service_dict.pop('platform', default_platform), + platform=service_dict.pop('platform', None), + default_platform=default_platform, **service_dict) ) diff --git a/compose/service.py b/compose/service.py index ae9e0bb08..4ff56eea7 100644 --- a/compose/service.py +++ b/compose/service.py @@ -172,6 +172,7 @@ class Service(object): secrets=None, scale=None, pid_mode=None, + default_platform=None, **options ): self.name = name @@ -185,6 +186,7 @@ class Service(object): self.networks = networks or {} self.secrets = secrets or [] self.scale_num = scale or 1 + self.default_platform = default_platform self.options = options def __repr__(self): @@ -358,6 +360,13 @@ class Service(object): def image_name(self): return self.options.get('image', '{s.project}_{s.name}'.format(s=self)) + @property + def platform(self): + platform = self.options.get('platform') + if not platform and version_gte(self.client.api_version, '1.35'): + platform = self.default_platform + return platform + def convergence_plan(self, strategy=ConvergenceStrategy.changed): containers = self.containers(stopped=True) @@ -1018,8 +1027,7 @@ class Service(object): if not six.PY3 and not IS_WINDOWS_PLATFORM: path = path.encode('utf8') - platform = self.options.get('platform') - if platform and version_lt(self.client.api_version, '1.35'): + if self.platform and version_lt(self.client.api_version, '1.35'): raise OperationFailedError( 'Impossible to perform platform-targeted builds for API version < 1.35' ) @@ -1044,7 +1052,7 @@ class Service(object): }, gzip=gzip, isolation=build_opts.get('isolation', self.options.get('isolation', None)), - platform=platform, + platform=self.platform, ) try: @@ -1150,14 +1158,14 @@ class Service(object): kwargs = { 'tag': tag or 'latest', 'stream': True, - 'platform': self.options.get('platform'), + 'platform': self.platform, } if not silent: log.info('Pulling %s (%s%s%s)...' % (self.name, repo, separator, tag)) if kwargs['platform'] and version_lt(self.client.api_version, '1.35'): raise OperationFailedError( - 'Impossible to perform platform-targeted builds for API version < 1.35' + 'Impossible to perform platform-targeted pulls for API version < 1.35' ) try: output = self.client.pull(repo, **kwargs) diff --git a/tests/unit/project_test.py b/tests/unit/project_test.py index 1b6b6651f..1cc841814 100644 --- a/tests/unit/project_test.py +++ b/tests/unit/project_test.py @@ -29,6 +29,7 @@ class ProjectTest(unittest.TestCase): def setUp(self): self.mock_client = mock.create_autospec(docker.APIClient) self.mock_client._general_configs = {} + self.mock_client.api_version = docker.constants.DEFAULT_DOCKER_API_VERSION def test_from_config_v1(self): config = Config( @@ -578,21 +579,21 @@ class ProjectTest(unittest.TestCase): ) project = Project.from_config(name='test', client=self.mock_client, config_data=config_data) - assert project.get_service('web').options.get('platform') is None + assert project.get_service('web').platform is None project = Project.from_config( name='test', client=self.mock_client, config_data=config_data, default_platform='windows' ) - assert project.get_service('web').options.get('platform') == 'windows' + assert project.get_service('web').platform == 'windows' service_config['platform'] = 'linux/s390x' project = Project.from_config(name='test', client=self.mock_client, config_data=config_data) - assert project.get_service('web').options.get('platform') == 'linux/s390x' + assert project.get_service('web').platform == 'linux/s390x' project = Project.from_config( name='test', client=self.mock_client, config_data=config_data, default_platform='windows' ) - assert project.get_service('web').options.get('platform') == 'linux/s390x' + assert project.get_service('web').platform == 'linux/s390x' @mock.patch('compose.parallel.ParallelStreamWriter._write_noansi') def test_error_parallel_pull(self, mock_write): diff --git a/tests/unit/service_test.py b/tests/unit/service_test.py index d50db9044..f5a35d814 100644 --- a/tests/unit/service_test.py +++ b/tests/unit/service_test.py @@ -446,6 +446,20 @@ class ServiceTest(unittest.TestCase): with pytest.raises(OperationFailedError): service.pull() + def test_pull_image_with_default_platform(self): + self.mock_client.api_version = '1.35' + + service = Service( + 'foo', client=self.mock_client, image='someimage:sometag', + default_platform='linux' + ) + assert service.platform == 'linux' + service.pull() + + assert self.mock_client.pull.call_count == 1 + call_args = self.mock_client.pull.call_args + assert call_args[1]['platform'] == 'linux' + @mock.patch('compose.service.Container', autospec=True) def test_recreate_container(self, _): mock_container = mock.create_autospec(Container) @@ -538,7 +552,7 @@ class ServiceTest(unittest.TestCase): assert self.mock_client.build.call_count == 1 assert not self.mock_client.build.call_args[1]['pull'] - def test_build_does_with_platform(self): + def test_build_with_platform(self): self.mock_client.api_version = '1.35' self.mock_client.build.return_value = [ b'{"stream": "Successfully built 12345"}', @@ -551,6 +565,47 @@ class ServiceTest(unittest.TestCase): call_args = self.mock_client.build.call_args assert call_args[1]['platform'] == 'linux' + def test_build_with_default_platform(self): + self.mock_client.api_version = '1.35' + self.mock_client.build.return_value = [ + b'{"stream": "Successfully built 12345"}', + ] + + service = Service( + 'foo', client=self.mock_client, build={'context': '.'}, + default_platform='linux' + ) + assert service.platform == 'linux' + service.build() + + assert self.mock_client.build.call_count == 1 + call_args = self.mock_client.build.call_args + assert call_args[1]['platform'] == 'linux' + + def test_service_platform_precedence(self): + self.mock_client.api_version = '1.35' + + service = Service( + 'foo', client=self.mock_client, platform='linux/arm', + default_platform='osx' + ) + assert service.platform == 'linux/arm' + + def test_service_ignore_default_platform_with_unsupported_api(self): + self.mock_client.api_version = '1.32' + self.mock_client.build.return_value = [ + b'{"stream": "Successfully built 12345"}', + ] + + service = Service( + 'foo', client=self.mock_client, default_platform='windows', build={'context': '.'} + ) + assert service.platform is None + service.build() + assert self.mock_client.build.call_count == 1 + call_args = self.mock_client.build.call_args + assert call_args[1]['platform'] is None + def test_build_with_override_build_args(self): self.mock_client.build.return_value = [ b'{"stream": "Successfully built 12345"}', From f368b4846f84afc2fa9fc0701408fd3b6eaed132 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Fri, 4 May 2018 15:05:28 -0700 Subject: [PATCH 39/53] Ignore attachable property on networks in compatibility mode Signed-off-by: Joffrey F --- compose/config/config.py | 9 +++++---- compose/config/serialize.py | 4 ++++ tests/acceptance/cli_test.py | 6 ++++-- tests/fixtures/compatibility-mode/docker-compose.yml | 6 ++++++ 4 files changed, 19 insertions(+), 6 deletions(-) diff --git a/compose/config/config.py b/compose/config/config.py index 9f8a50c62..aab78be5f 100644 --- a/compose/config/config.py +++ b/compose/config/config.py @@ -918,6 +918,11 @@ def convert_restart_policy(name): def translate_deploy_keys_to_container_config(service_dict): + if 'credential_spec' in service_dict: + del service_dict['credential_spec'] + if 'configs' in service_dict: + del service_dict['configs'] + if 'deploy' not in service_dict: return service_dict, [] @@ -946,10 +951,6 @@ def translate_deploy_keys_to_container_config(service_dict): ) del service_dict['deploy'] - if 'credential_spec' in service_dict: - del service_dict['credential_spec'] - if 'configs' in service_dict: - del service_dict['configs'] return service_dict, ignored_keys diff --git a/compose/config/serialize.py b/compose/config/serialize.py index c0cf35c1b..ccddbf532 100644 --- a/compose/config/serialize.py +++ b/compose/config/serialize.py @@ -80,6 +80,10 @@ def denormalize_config(config, image_digests=None): elif 'external' in conf: conf['external'] = True + if 'attachable' in conf and config.version < V3_2: + # For compatibility mode, this option is invalid in v2 + del conf['attachable'] + return result diff --git a/tests/acceptance/cli_test.py b/tests/acceptance/cli_test.py index 075705804..43e8fa822 100644 --- a/tests/acceptance/cli_test.py +++ b/tests/acceptance/cli_test.py @@ -481,6 +481,7 @@ class CLITestCase(DockerClientTestCase): assert yaml.load(result.stdout) == { 'version': '2.3', 'volumes': {'foo': {'driver': 'default'}}, + 'networks': {'bar': {}}, 'services': { 'foo': { 'command': '/bin/true', @@ -490,9 +491,10 @@ class CLITestCase(DockerClientTestCase): 'mem_limit': '300M', 'mem_reservation': '100M', 'cpus': 0.7, - 'volumes': ['foo:/bar:rw'] + 'volumes': ['foo:/bar:rw'], + 'networks': {'bar': None}, } - } + }, } def test_ps(self): diff --git a/tests/fixtures/compatibility-mode/docker-compose.yml b/tests/fixtures/compatibility-mode/docker-compose.yml index aac6fd4cb..8187b110c 100644 --- a/tests/fixtures/compatibility-mode/docker-compose.yml +++ b/tests/fixtures/compatibility-mode/docker-compose.yml @@ -16,7 +16,13 @@ services: memory: 100M volumes: - foo:/bar + networks: + - bar volumes: foo: driver: default + +networks: + bar: + attachable: true From d5ebc734829f86054388fced14665c348e1d1ef5 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Fri, 4 May 2018 16:15:52 -0700 Subject: [PATCH 40/53] Don't attempt to create resources with name starting with illegal characters Signed-off-by: Joffrey F --- compose/service.py | 2 +- compose/volume.py | 2 +- tests/integration/project_test.py | 62 +++++++++++++++++++++++++++++++ 3 files changed, 64 insertions(+), 2 deletions(-) diff --git a/compose/service.py b/compose/service.py index 4ff56eea7..932ed8b34 100644 --- a/compose/service.py +++ b/compose/service.py @@ -1355,7 +1355,7 @@ class ServiceNetworkMode(object): def build_container_name(project, service, number, one_off=False): - bits = [project, service] + bits = [project.lstrip('-_'), service] if one_off: bits.append('run') return '_'.join(bits + [str(number)]) diff --git a/compose/volume.py b/compose/volume.py index 7618417ff..60c1e0fe8 100644 --- a/compose/volume.py +++ b/compose/volume.py @@ -60,7 +60,7 @@ class Volume(object): def full_name(self): if self.custom_name: return self.name - return '{0}_{1}'.format(self.project, self.name) + return '{0}_{1}'.format(self.project.lstrip('-_'), self.name) @property def legacy_full_name(self): diff --git a/tests/integration/project_test.py b/tests/integration/project_test.py index 3960d12e5..8813e84ce 100644 --- a/tests/integration/project_test.py +++ b/tests/integration/project_test.py @@ -1915,3 +1915,65 @@ class ProjectTest(DockerClientTestCase): assert len(remote_secopts) == 1 assert remote_secopts[0].startswith('seccomp=') assert json.loads(remote_secopts[0].lstrip('seccomp=')) == seccomp_data + + @no_cluster('inspect volume by name defect on Swarm Classic') + def test_project_up_name_starts_with_illegal_char(self): + config_dict = { + 'version': '2.3', + 'services': { + 'svc1': { + 'image': 'busybox:latest', + 'command': 'ls', + 'volumes': ['foo:/foo:rw'], + 'networks': ['bar'], + }, + }, + 'volumes': { + 'foo': {}, + }, + 'networks': { + 'bar': {}, + } + } + config_data = load_config(config_dict) + project = Project.from_config( + name='_underscoretest', config_data=config_data, client=self.client + ) + project.up() + self.addCleanup(project.down, None, True) + + containers = project.containers(stopped=True) + assert len(containers) == 1 + assert containers[0].name == 'underscoretest_svc1_1' + assert containers[0].project == '_underscoretest' + + full_vol_name = 'underscoretest_foo' + vol_data = self.get_volume_data(full_vol_name) + assert vol_data + assert vol_data['Labels'][LABEL_PROJECT] == '_underscoretest' + + full_net_name = '_underscoretest_bar' + net_data = self.client.inspect_network(full_net_name) + assert net_data + assert net_data['Labels'][LABEL_PROJECT] == '_underscoretest' + + project2 = Project.from_config( + name='-dashtest', config_data=config_data, client=self.client + ) + project2.up() + self.addCleanup(project2.down, None, True) + + containers = project2.containers(stopped=True) + assert len(containers) == 1 + assert containers[0].name == 'dashtest_svc1_1' + assert containers[0].project == '-dashtest' + + full_vol_name = 'dashtest_foo' + vol_data = self.get_volume_data(full_vol_name) + assert vol_data + assert vol_data['Labels'][LABEL_PROJECT] == '-dashtest' + + full_net_name = '-dashtest_bar' + net_data = self.client.inspect_network(full_net_name) + assert net_data + assert net_data['Labels'][LABEL_PROJECT] == '-dashtest' From 7846f6e2a06ac0e0acfda41b9f0e869b411c7225 Mon Sep 17 00:00:00 2001 From: Harald Albers Date: Thu, 17 May 2018 15:57:07 +0200 Subject: [PATCH 41/53] Fix bash completion for running services Signed-off-by: Harald Albers --- contrib/completion/bash/docker-compose | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contrib/completion/bash/docker-compose b/contrib/completion/bash/docker-compose index 7aa69a463..b90af45d1 100644 --- a/contrib/completion/bash/docker-compose +++ b/contrib/completion/bash/docker-compose @@ -98,7 +98,7 @@ __docker_compose_complete_services() { # The services for which at least one running container exists __docker_compose_complete_running_services() { - local names=$(__docker_compose_complete_services --filter status=running) + local names=$(__docker_compose_services --filter status=running) COMPREPLY=( $(compgen -W "$names" -- "$cur") ) } From e245fb04cfb2703950e458cf3a81ac6e50db1886 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Wed, 23 May 2018 16:28:41 -0700 Subject: [PATCH 42/53] Allow all Compose commands to retrieve and handle legacy-name containers Signed-off-by: Joffrey F --- compose/container.py | 8 ++++++++ compose/project.py | 19 ++++++++++++++++--- compose/service.py | 40 ++++++++++++++++++++++------------------ 3 files changed, 46 insertions(+), 21 deletions(-) diff --git a/compose/container.py b/compose/container.py index 0c2ca9902..8dac8cacd 100644 --- a/compose/container.py +++ b/compose/container.py @@ -9,6 +9,8 @@ from docker.errors import ImageNotFound from .const import LABEL_CONTAINER_NUMBER from .const import LABEL_PROJECT from .const import LABEL_SERVICE +from .const import LABEL_VERSION +from .version import ComposeVersion class Container(object): @@ -283,6 +285,12 @@ class Container(object): def attach(self, *args, **kwargs): return self.client.attach(self.id, *args, **kwargs) + def has_legacy_proj_name(self, project_name): + return ( + ComposeVersion(self.labels.get(LABEL_VERSION)) < ComposeVersion('1.21.0') and + self.project != project_name + ) + def __repr__(self): return '' % (self.name, self.id[:6]) diff --git a/compose/project.py b/compose/project.py index c27794fc3..005b7e240 100644 --- a/compose/project.py +++ b/compose/project.py @@ -4,6 +4,7 @@ from __future__ import unicode_literals import datetime import logging import operator +import re from functools import reduce import enum @@ -70,8 +71,11 @@ class Project(object): self.networks = networks or ProjectNetworks({}, False) self.config_version = config_version - def labels(self, one_off=OneOffFilter.exclude): - labels = ['{0}={1}'.format(LABEL_PROJECT, self.name)] + def labels(self, one_off=OneOffFilter.exclude, legacy=False): + name = self.name + if legacy: + name = re.sub(r'[_-]', '', name) + labels = ['{0}={1}'.format(LABEL_PROJECT, name)] OneOffFilter.update_labels(one_off, labels) return labels @@ -571,12 +575,21 @@ class Project(object): service.push(ignore_push_failures) def _labeled_containers(self, stopped=False, one_off=OneOffFilter.exclude): - return list(filter(None, [ + ctnrs = list(filter(None, [ Container.from_ps(self.client, container) for container in self.client.containers( all=stopped, filters={'label': self.labels(one_off=one_off)})]) ) + if ctnrs: + return ctnrs + + return list(filter(lambda c: c.has_legacy_proj_name(self.name), filter(None, [ + Container.from_ps(self.client, container) + for container in self.client.containers( + all=stopped, + filters={'label': self.labels(one_off=one_off, legacy=True)})]) + )) def containers(self, service_names=None, stopped=False, one_off=OneOffFilter.exclude): if service_names: diff --git a/compose/service.py b/compose/service.py index 932ed8b34..48cbc1702 100644 --- a/compose/service.py +++ b/compose/service.py @@ -1,6 +1,7 @@ from __future__ import absolute_import from __future__ import unicode_literals +import itertools import logging import os import re @@ -51,7 +52,6 @@ from .progress_stream import StreamOutputError from .utils import json_hash from .utils import parse_bytes from .utils import parse_seconds_float -from .version import ComposeVersion log = logging.getLogger(__name__) @@ -192,8 +192,8 @@ class Service(object): def __repr__(self): return ''.format(self.name) - def containers(self, stopped=False, one_off=False, filters={}): - filters.update({'label': self.labels(one_off=one_off)}) + def containers(self, stopped=False, one_off=False, filters={}, labels=None): + filters.update({'label': self.labels(one_off=one_off) + (labels or [])}) result = list(filter(None, [ Container.from_ps(self.client, container) @@ -204,10 +204,10 @@ class Service(object): if result: return result - filters.update({'label': self.labels(one_off=one_off, legacy=True)}) + filters.update({'label': self.labels(one_off=one_off, legacy=True) + (labels or [])}) return list( filter( - self.has_legacy_proj_name, filter(None, [ + lambda c: c.has_legacy_proj_name(self.project), filter(None, [ Container.from_ps(self.client, container) for container in self.client.containers( all=stopped, @@ -219,9 +219,9 @@ class Service(object): """Return a :class:`compose.container.Container` for this service. The container must be active, and match `number`. """ - labels = self.labels() + ['{0}={1}'.format(LABEL_CONTAINER_NUMBER, number)] - for container in self.client.containers(filters={'label': labels}): - return Container.from_ps(self.client, container) + + for container in self.containers(labels=['{0}={1}'.format(LABEL_CONTAINER_NUMBER, number)]): + return container raise ValueError("No container found for %s_%s" % (self.name, number)) @@ -258,6 +258,11 @@ class Service(object): running_containers = self.containers(stopped=False) num_running = len(running_containers) + for c in running_containers: + if not c.has_legacy_proj_name(self.project): + continue + log.info('Recreating container with legacy name %s' % c.name) + self.recreate_container(c, timeout, start_new_container=False) if desired_num == num_running: # do nothing as we already have the desired number @@ -404,7 +409,7 @@ class Service(object): has_diverged = False for c in containers: - if self.has_legacy_proj_name(c): + if c.has_legacy_proj_name(self.project): log.debug('%s has diverged: Legacy project name' % c.name) has_diverged = True continue @@ -713,9 +718,14 @@ class Service(object): # TODO: this would benefit from github.com/docker/docker/pull/14699 # to remove the need to inspect every container def _next_container_number(self, one_off=False): - containers = self._fetch_containers( - all=True, - filters={'label': self.labels(one_off=one_off)} + containers = itertools.chain( + self._fetch_containers( + all=True, + filters={'label': self.labels(one_off=one_off)} + ), self._fetch_containers( + all=True, + filters={'label': self.labels(one_off=one_off, legacy=True)} + ) ) numbers = [c.number for c in containers] return 1 if not numbers else max(numbers) + 1 @@ -1243,12 +1253,6 @@ class Service(object): return result - def has_legacy_proj_name(self, ctnr): - return ( - ComposeVersion(ctnr.labels.get(LABEL_VERSION)) < ComposeVersion('1.21.0') and - ctnr.project != self.project - ) - def short_id_alias_exists(container, network): aliases = container.get( From 025fb7f86075042c3fb9d3201aebb71c0a1a5003 Mon Sep 17 00:00:00 2001 From: Vincent Demeester Date: Tue, 29 May 2018 11:58:54 +0200 Subject: [PATCH 43/53] Add composefile v3.7 Signed-off-by: Vincent Demeester --- compose/config/config_schema_v3.7.json | 582 +++++++++++++++++++++++++ compose/const.py | 3 + docker-compose.spec | 5 + 3 files changed, 590 insertions(+) create mode 100644 compose/config/config_schema_v3.7.json diff --git a/compose/config/config_schema_v3.7.json b/compose/config/config_schema_v3.7.json new file mode 100644 index 000000000..f85efe34f --- /dev/null +++ b/compose/config/config_schema_v3.7.json @@ -0,0 +1,582 @@ +{ + "$schema": "http://json-schema.org/draft-04/schema#", + "id": "config_schema_v3.7.json", + "type": "object", + "required": ["version"], + + "properties": { + "version": { + "type": "string" + }, + + "services": { + "id": "#/properties/services", + "type": "object", + "patternProperties": { + "^[a-zA-Z0-9._-]+$": { + "$ref": "#/definitions/service" + } + }, + "additionalProperties": false + }, + + "networks": { + "id": "#/properties/networks", + "type": "object", + "patternProperties": { + "^[a-zA-Z0-9._-]+$": { + "$ref": "#/definitions/network" + } + } + }, + + "volumes": { + "id": "#/properties/volumes", + "type": "object", + "patternProperties": { + "^[a-zA-Z0-9._-]+$": { + "$ref": "#/definitions/volume" + } + }, + "additionalProperties": false + }, + + "secrets": { + "id": "#/properties/secrets", + "type": "object", + "patternProperties": { + "^[a-zA-Z0-9._-]+$": { + "$ref": "#/definitions/secret" + } + }, + "additionalProperties": false + }, + + "configs": { + "id": "#/properties/configs", + "type": "object", + "patternProperties": { + "^[a-zA-Z0-9._-]+$": { + "$ref": "#/definitions/config" + } + }, + "additionalProperties": false + } + }, + + "patternProperties": {"^x-": {}}, + "additionalProperties": false, + + "definitions": { + + "service": { + "id": "#/definitions/service", + "type": "object", + + "properties": { + "deploy": {"$ref": "#/definitions/deployment"}, + "build": { + "oneOf": [ + {"type": "string"}, + { + "type": "object", + "properties": { + "context": {"type": "string"}, + "dockerfile": {"type": "string"}, + "args": {"$ref": "#/definitions/list_or_dict"}, + "labels": {"$ref": "#/definitions/list_or_dict"}, + "cache_from": {"$ref": "#/definitions/list_of_strings"}, + "network": {"type": "string"}, + "target": {"type": "string"}, + "shm_size": {"type": ["integer", "string"]} + }, + "additionalProperties": false + } + ] + }, + "cap_add": {"type": "array", "items": {"type": "string"}, "uniqueItems": true}, + "cap_drop": {"type": "array", "items": {"type": "string"}, "uniqueItems": true}, + "cgroup_parent": {"type": "string"}, + "command": { + "oneOf": [ + {"type": "string"}, + {"type": "array", "items": {"type": "string"}} + ] + }, + "configs": { + "type": "array", + "items": { + "oneOf": [ + {"type": "string"}, + { + "type": "object", + "properties": { + "source": {"type": "string"}, + "target": {"type": "string"}, + "uid": {"type": "string"}, + "gid": {"type": "string"}, + "mode": {"type": "number"} + } + } + ] + } + }, + "container_name": {"type": "string"}, + "credential_spec": {"type": "object", "properties": { + "file": {"type": "string"}, + "registry": {"type": "string"} + }}, + "depends_on": {"$ref": "#/definitions/list_of_strings"}, + "devices": {"type": "array", "items": {"type": "string"}, "uniqueItems": true}, + "dns": {"$ref": "#/definitions/string_or_list"}, + "dns_search": {"$ref": "#/definitions/string_or_list"}, + "domainname": {"type": "string"}, + "entrypoint": { + "oneOf": [ + {"type": "string"}, + {"type": "array", "items": {"type": "string"}} + ] + }, + "env_file": {"$ref": "#/definitions/string_or_list"}, + "environment": {"$ref": "#/definitions/list_or_dict"}, + + "expose": { + "type": "array", + "items": { + "type": ["string", "number"], + "format": "expose" + }, + "uniqueItems": true + }, + + "external_links": {"type": "array", "items": {"type": "string"}, "uniqueItems": true}, + "extra_hosts": {"$ref": "#/definitions/list_or_dict"}, + "healthcheck": {"$ref": "#/definitions/healthcheck"}, + "hostname": {"type": "string"}, + "image": {"type": "string"}, + "ipc": {"type": "string"}, + "isolation": {"type": "string"}, + "labels": {"$ref": "#/definitions/list_or_dict"}, + "links": {"type": "array", "items": {"type": "string"}, "uniqueItems": true}, + + "logging": { + "type": "object", + + "properties": { + "driver": {"type": "string"}, + "options": { + "type": "object", + "patternProperties": { + "^.+$": {"type": ["string", "number", "null"]} + } + } + }, + "additionalProperties": false + }, + + "mac_address": {"type": "string"}, + "network_mode": {"type": "string"}, + + "networks": { + "oneOf": [ + {"$ref": "#/definitions/list_of_strings"}, + { + "type": "object", + "patternProperties": { + "^[a-zA-Z0-9._-]+$": { + "oneOf": [ + { + "type": "object", + "properties": { + "aliases": {"$ref": "#/definitions/list_of_strings"}, + "ipv4_address": {"type": "string"}, + "ipv6_address": {"type": "string"} + }, + "additionalProperties": false + }, + {"type": "null"} + ] + } + }, + "additionalProperties": false + } + ] + }, + "pid": {"type": ["string", "null"]}, + + "ports": { + "type": "array", + "items": { + "oneOf": [ + {"type": "number", "format": "ports"}, + {"type": "string", "format": "ports"}, + { + "type": "object", + "properties": { + "mode": {"type": "string"}, + "target": {"type": "integer"}, + "published": {"type": "integer"}, + "protocol": {"type": "string"} + }, + "additionalProperties": false + } + ] + }, + "uniqueItems": true + }, + + "privileged": {"type": "boolean"}, + "read_only": {"type": "boolean"}, + "restart": {"type": "string"}, + "security_opt": {"type": "array", "items": {"type": "string"}, "uniqueItems": true}, + "shm_size": {"type": ["number", "string"]}, + "secrets": { + "type": "array", + "items": { + "oneOf": [ + {"type": "string"}, + { + "type": "object", + "properties": { + "source": {"type": "string"}, + "target": {"type": "string"}, + "uid": {"type": "string"}, + "gid": {"type": "string"}, + "mode": {"type": "number"} + } + } + ] + } + }, + "sysctls": {"$ref": "#/definitions/list_or_dict"}, + "stdin_open": {"type": "boolean"}, + "stop_grace_period": {"type": "string", "format": "duration"}, + "stop_signal": {"type": "string"}, + "tmpfs": {"$ref": "#/definitions/string_or_list"}, + "tty": {"type": "boolean"}, + "ulimits": { + "type": "object", + "patternProperties": { + "^[a-z]+$": { + "oneOf": [ + {"type": "integer"}, + { + "type":"object", + "properties": { + "hard": {"type": "integer"}, + "soft": {"type": "integer"} + }, + "required": ["soft", "hard"], + "additionalProperties": false + } + ] + } + } + }, + "user": {"type": "string"}, + "userns_mode": {"type": "string"}, + "volumes": { + "type": "array", + "items": { + "oneOf": [ + {"type": "string"}, + { + "type": "object", + "required": ["type"], + "properties": { + "type": {"type": "string"}, + "source": {"type": "string"}, + "target": {"type": "string"}, + "read_only": {"type": "boolean"}, + "consistency": {"type": "string"}, + "bind": { + "type": "object", + "properties": { + "propagation": {"type": "string"} + } + }, + "volume": { + "type": "object", + "properties": { + "nocopy": {"type": "boolean"} + } + }, + "tmpfs": { + "type": "object", + "properties": { + "size": { + "type": "integer", + "minimum": 0 + } + } + } + }, + "additionalProperties": false + } + ], + "uniqueItems": true + } + }, + "working_dir": {"type": "string"} + }, + "additionalProperties": false + }, + + "healthcheck": { + "id": "#/definitions/healthcheck", + "type": "object", + "additionalProperties": false, + "properties": { + "disable": {"type": "boolean"}, + "interval": {"type": "string", "format": "duration"}, + "retries": {"type": "number"}, + "test": { + "oneOf": [ + {"type": "string"}, + {"type": "array", "items": {"type": "string"}} + ] + }, + "timeout": {"type": "string", "format": "duration"}, + "start_period": {"type": "string", "format": "duration"} + } + }, + "deployment": { + "id": "#/definitions/deployment", + "type": ["object", "null"], + "properties": { + "mode": {"type": "string"}, + "endpoint_mode": {"type": "string"}, + "replicas": {"type": "integer"}, + "labels": {"$ref": "#/definitions/list_or_dict"}, + "update_config": { + "type": "object", + "properties": { + "parallelism": {"type": "integer"}, + "delay": {"type": "string", "format": "duration"}, + "failure_action": {"type": "string"}, + "monitor": {"type": "string", "format": "duration"}, + "max_failure_ratio": {"type": "number"}, + "order": {"type": "string", "enum": [ + "start-first", "stop-first" + ]} + }, + "additionalProperties": false + }, + "resources": { + "type": "object", + "properties": { + "limits": { + "type": "object", + "properties": { + "cpus": {"type": "string"}, + "memory": {"type": "string"} + }, + "additionalProperties": false + }, + "reservations": { + "type": "object", + "properties": { + "cpus": {"type": "string"}, + "memory": {"type": "string"}, + "generic_resources": {"$ref": "#/definitions/generic_resources"} + }, + "additionalProperties": false + } + }, + "additionalProperties": false + }, + "restart_policy": { + "type": "object", + "properties": { + "condition": {"type": "string"}, + "delay": {"type": "string", "format": "duration"}, + "max_attempts": {"type": "integer"}, + "window": {"type": "string", "format": "duration"} + }, + "additionalProperties": false + }, + "placement": { + "type": "object", + "properties": { + "constraints": {"type": "array", "items": {"type": "string"}}, + "preferences": { + "type": "array", + "items": { + "type": "object", + "properties": { + "spread": {"type": "string"} + }, + "additionalProperties": false + } + } + }, + "additionalProperties": false + } + }, + "additionalProperties": false + }, + + "generic_resources": { + "id": "#/definitions/generic_resources", + "type": "array", + "items": { + "type": "object", + "properties": { + "discrete_resource_spec": { + "type": "object", + "properties": { + "kind": {"type": "string"}, + "value": {"type": "number"} + }, + "additionalProperties": false + } + }, + "additionalProperties": false + } + }, + + "network": { + "id": "#/definitions/network", + "type": ["object", "null"], + "properties": { + "name": {"type": "string"}, + "driver": {"type": "string"}, + "driver_opts": { + "type": "object", + "patternProperties": { + "^.+$": {"type": ["string", "number"]} + } + }, + "ipam": { + "type": "object", + "properties": { + "driver": {"type": "string"}, + "config": { + "type": "array", + "items": { + "type": "object", + "properties": { + "subnet": {"type": "string"} + }, + "additionalProperties": false + } + } + }, + "additionalProperties": false + }, + "external": { + "type": ["boolean", "object"], + "properties": { + "name": {"type": "string"} + }, + "additionalProperties": false + }, + "internal": {"type": "boolean"}, + "attachable": {"type": "boolean"}, + "labels": {"$ref": "#/definitions/list_or_dict"} + }, + "additionalProperties": false + }, + + "volume": { + "id": "#/definitions/volume", + "type": ["object", "null"], + "properties": { + "name": {"type": "string"}, + "driver": {"type": "string"}, + "driver_opts": { + "type": "object", + "patternProperties": { + "^.+$": {"type": ["string", "number"]} + } + }, + "external": { + "type": ["boolean", "object"], + "properties": { + "name": {"type": "string"} + }, + "additionalProperties": false + }, + "labels": {"$ref": "#/definitions/list_or_dict"} + }, + "additionalProperties": false + }, + + "secret": { + "id": "#/definitions/secret", + "type": "object", + "properties": { + "name": {"type": "string"}, + "file": {"type": "string"}, + "external": { + "type": ["boolean", "object"], + "properties": { + "name": {"type": "string"} + } + }, + "labels": {"$ref": "#/definitions/list_or_dict"} + }, + "additionalProperties": false + }, + + "config": { + "id": "#/definitions/config", + "type": "object", + "properties": { + "name": {"type": "string"}, + "file": {"type": "string"}, + "external": { + "type": ["boolean", "object"], + "properties": { + "name": {"type": "string"} + } + }, + "labels": {"$ref": "#/definitions/list_or_dict"} + }, + "additionalProperties": false + }, + + "string_or_list": { + "oneOf": [ + {"type": "string"}, + {"$ref": "#/definitions/list_of_strings"} + ] + }, + + "list_of_strings": { + "type": "array", + "items": {"type": "string"}, + "uniqueItems": true + }, + + "list_or_dict": { + "oneOf": [ + { + "type": "object", + "patternProperties": { + ".+": { + "type": ["string", "number", "null"] + } + }, + "additionalProperties": false + }, + {"type": "array", "items": {"type": "string"}, "uniqueItems": true} + ] + }, + + "constraints": { + "service": { + "id": "#/definitions/constraints/service", + "anyOf": [ + {"required": ["build"]}, + {"required": ["image"]} + ], + "properties": { + "build": { + "required": ["context"] + } + } + } + } + } +} diff --git a/compose/const.py b/compose/const.py index 200a458a1..f5632de74 100644 --- a/compose/const.py +++ b/compose/const.py @@ -36,6 +36,7 @@ COMPOSEFILE_V3_3 = ComposeVersion('3.3') COMPOSEFILE_V3_4 = ComposeVersion('3.4') COMPOSEFILE_V3_5 = ComposeVersion('3.5') COMPOSEFILE_V3_6 = ComposeVersion('3.6') +COMPOSEFILE_V3_7 = ComposeVersion('3.7') API_VERSIONS = { COMPOSEFILE_V1: '1.21', @@ -51,6 +52,7 @@ API_VERSIONS = { COMPOSEFILE_V3_4: '1.30', COMPOSEFILE_V3_5: '1.30', COMPOSEFILE_V3_6: '1.36', + COMPOSEFILE_V3_7: '1.36', } API_VERSION_TO_ENGINE_VERSION = { @@ -67,4 +69,5 @@ API_VERSION_TO_ENGINE_VERSION = { API_VERSIONS[COMPOSEFILE_V3_4]: '17.06.0', API_VERSIONS[COMPOSEFILE_V3_5]: '17.06.0', API_VERSIONS[COMPOSEFILE_V3_6]: '18.02.0', + API_VERSIONS[COMPOSEFILE_V3_7]: '18.02.0', } diff --git a/docker-compose.spec b/docker-compose.spec index b8c3a4191..70db5c29e 100644 --- a/docker-compose.spec +++ b/docker-compose.spec @@ -82,6 +82,11 @@ exe = EXE(pyz, 'compose/config/config_schema_v3.6.json', 'DATA' ), + ( + 'compose/config/config_schema_v3.7.json', + 'compose/config/config_schema_v3.7.json', + 'DATA' + ), ( 'compose/GITSHA', 'compose/GITSHA', From 70574efd5bb882efd22027eba5853255c9026892 Mon Sep 17 00:00:00 2001 From: Vincent Demeester Date: Wed, 30 May 2018 13:36:59 +0200 Subject: [PATCH 44/53] Support for rollback config in compose 3.7 Ignoring it on docker-compose Signed-off-by: Vincent Demeester --- compose/config/config.py | 3 ++- compose/config/config_schema_v3.7.json | 14 ++++++++++++++ compose/config/interpolation.py | 2 ++ compose/const.py | 2 +- 4 files changed, 19 insertions(+), 2 deletions(-) diff --git a/compose/config/config.py b/compose/config/config.py index aab78be5f..5c614fadc 100644 --- a/compose/config/config.py +++ b/compose/config/config.py @@ -928,7 +928,7 @@ def translate_deploy_keys_to_container_config(service_dict): deploy_dict = service_dict['deploy'] ignored_keys = [ - k for k in ['endpoint_mode', 'labels', 'update_config', 'placement'] + k for k in ['endpoint_mode', 'labels', 'update_config', 'rollback_config', 'placement'] if k in deploy_dict ] @@ -1136,6 +1136,7 @@ def merge_deploy(base, override): md.merge_scalar('replicas') md.merge_mapping('labels', parse_labels) md.merge_mapping('update_config') + md.merge_mapping('rollback_config') md.merge_mapping('restart_policy') if md.needs_merge('resources'): resources_md = MergeDict(md.base.get('resources') or {}, md.override.get('resources') or {}) diff --git a/compose/config/config_schema_v3.7.json b/compose/config/config_schema_v3.7.json index f85efe34f..4c3d24dcd 100644 --- a/compose/config/config_schema_v3.7.json +++ b/compose/config/config_schema_v3.7.json @@ -348,6 +348,20 @@ "endpoint_mode": {"type": "string"}, "replicas": {"type": "integer"}, "labels": {"$ref": "#/definitions/list_or_dict"}, + "rollback_config": { + "type": "object", + "properties": { + "parallelism": {"type": "integer"}, + "delay": {"type": "string", "format": "duration"}, + "failure_action": {"type": "string"}, + "monitor": {"type": "string", "format": "duration"}, + "max_failure_ratio": {"type": "number"}, + "order": {"type": "string", "enum": [ + "start-first", "stop-first" + ]} + }, + "additionalProperties": false + }, "update_config": { "type": "object", "properties": { diff --git a/compose/config/interpolation.py b/compose/config/interpolation.py index 8845d73b5..4f56dff59 100644 --- a/compose/config/interpolation.py +++ b/compose/config/interpolation.py @@ -248,6 +248,8 @@ class ConversionMap(object): service_path('deploy', 'replicas'): to_int, service_path('deploy', 'update_config', 'parallelism'): to_int, service_path('deploy', 'update_config', 'max_failure_ratio'): to_float, + service_path('deploy', 'rollback_config', 'parallelism'): to_int, + service_path('deploy', 'rollback_config', 'max_failure_ratio'): to_float, service_path('deploy', 'restart_policy', 'max_attempts'): to_int, service_path('mem_swappiness'): to_int, service_path('labels', FULL_JOKER): to_str, diff --git a/compose/const.py b/compose/const.py index f5632de74..374a09711 100644 --- a/compose/const.py +++ b/compose/const.py @@ -69,5 +69,5 @@ API_VERSION_TO_ENGINE_VERSION = { API_VERSIONS[COMPOSEFILE_V3_4]: '17.06.0', API_VERSIONS[COMPOSEFILE_V3_5]: '17.06.0', API_VERSIONS[COMPOSEFILE_V3_6]: '18.02.0', - API_VERSIONS[COMPOSEFILE_V3_7]: '18.02.0', + API_VERSIONS[COMPOSEFILE_V3_7]: '18.06.0', } From 7a19b7548ff44955108a6a3bd4be9527babd1622 Mon Sep 17 00:00:00 2001 From: Vincent Demeester Date: Thu, 31 May 2018 14:12:10 +0200 Subject: [PATCH 45/53] Allow `x-*` extension on 3rd level objects As for top-level key, any 3rd-level key which starts with `x-` will be ignored by compose. This allows for users to: * include additional metadata in their compose files * create YAML anchor objects that can be re-used in other parts of the config This matches a similar feature in the swagger spec definition: https://swagger.io/specification/#specificationExtensions This means a composefile like the following is valid ``` verison: "3.7" services: foo: image: foo/bar x-foo: bar network: bar: x-bar: baz ``` It concerns services, volumes, networks, configs and secrets. Signed-off-by: Vincent Demeester --- compose/config/config_schema_v2.4.json | 3 +++ compose/config/config_schema_v3.7.json | 5 +++++ 2 files changed, 8 insertions(+) diff --git a/compose/config/config_schema_v2.4.json b/compose/config/config_schema_v2.4.json index 47e118755..4e6417887 100644 --- a/compose/config/config_schema_v2.4.json +++ b/compose/config/config_schema_v2.4.json @@ -346,6 +346,7 @@ "dependencies": { "memswap_limit": ["mem_limit"] }, + "patternProperties": {"^x-": {}}, "additionalProperties": false }, @@ -409,6 +410,7 @@ "labels": {"$ref": "#/definitions/labels"}, "name": {"type": "string"} }, + "patternProperties": {"^x-": {}}, "additionalProperties": false }, @@ -451,6 +453,7 @@ "labels": {"$ref": "#/definitions/labels"}, "name": {"type": "string"} }, + "patternProperties": {"^x-": {}}, "additionalProperties": false }, diff --git a/compose/config/config_schema_v3.7.json b/compose/config/config_schema_v3.7.json index 4c3d24dcd..05566cf81 100644 --- a/compose/config/config_schema_v3.7.json +++ b/compose/config/config_schema_v3.7.json @@ -319,6 +319,7 @@ }, "working_dir": {"type": "string"} }, + "patternProperties": {"^x-": {}}, "additionalProperties": false }, @@ -489,6 +490,7 @@ "attachable": {"type": "boolean"}, "labels": {"$ref": "#/definitions/list_or_dict"} }, + "patternProperties": {"^x-": {}}, "additionalProperties": false }, @@ -513,6 +515,7 @@ }, "labels": {"$ref": "#/definitions/list_or_dict"} }, + "patternProperties": {"^x-": {}}, "additionalProperties": false }, @@ -530,6 +533,7 @@ }, "labels": {"$ref": "#/definitions/list_or_dict"} }, + "patternProperties": {"^x-": {}}, "additionalProperties": false }, @@ -547,6 +551,7 @@ }, "labels": {"$ref": "#/definitions/list_or_dict"} }, + "patternProperties": {"^x-": {}}, "additionalProperties": false }, From c584ad67fce14ab9ee221945ccc10b52b5ab0aeb Mon Sep 17 00:00:00 2001 From: Vincent Demeester Date: Mon, 18 Jun 2018 10:52:57 +0200 Subject: [PATCH 46/53] Add `init` support in 3.7 schema > Run an init inside the container that forwards signals and reaps > processes This is already supported in 2.4 schema Signed-off-by: Vincent Demeester --- compose/config/config_schema_v3.7.json | 1 + 1 file changed, 1 insertion(+) diff --git a/compose/config/config_schema_v3.7.json b/compose/config/config_schema_v3.7.json index 05566cf81..cd7882f5b 100644 --- a/compose/config/config_schema_v3.7.json +++ b/compose/config/config_schema_v3.7.json @@ -154,6 +154,7 @@ "healthcheck": {"$ref": "#/definitions/healthcheck"}, "hostname": {"type": "string"}, "image": {"type": "string"}, + "init": {"type": "boolean"}, "ipc": {"type": "string"}, "isolation": {"type": "string"}, "labels": {"$ref": "#/definitions/list_or_dict"}, From a728ff6a599fb4aaefba066fd71fb276598e2cd7 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Mon, 18 Jun 2018 15:31:25 -0700 Subject: [PATCH 47/53] Bump Python SDK -> 3.4.0 Signed-off-by: Joffrey F --- requirements.txt | 4 ++-- setup.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/requirements.txt b/requirements.txt index 93a0cce35..05b38526a 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,8 +2,8 @@ backports.ssl-match-hostname==3.5.0.1; python_version < '3' cached-property==1.3.0 certifi==2017.4.17 chardet==3.0.4 -docker==3.3.0 -docker-pycreds==0.2.3 +docker==3.4.0 +docker-pycreds==0.3.0 dockerpty==0.4.1 docopt==0.6.2 enum34==1.1.6; python_version < '3.4' diff --git a/setup.py b/setup.py index 422ba5466..fc024078e 100644 --- a/setup.py +++ b/setup.py @@ -36,7 +36,7 @@ install_requires = [ 'requests >= 2.6.1, != 2.11.0, != 2.12.2, != 2.18.0, < 2.19', 'texttable >= 0.9.0, < 0.10', 'websocket-client >= 0.32.0, < 1.0', - 'docker >= 3.3.0, < 4.0', + 'docker >= 3.4.0, < 4.0', 'dockerpty >= 0.4.1, < 0.5', 'six >= 1.3.0, < 2', 'jsonschema >= 2.5.1, < 3', From c187d3c39fece75144d4e7c0b04994dca0098190 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Fri, 8 Jun 2018 16:32:55 -0700 Subject: [PATCH 48/53] Use original LD_LIBRARY_PATH when shelling out to credential stores Signed-off-by: Joffrey F --- compose/cli/docker_client.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/compose/cli/docker_client.py b/compose/cli/docker_client.py index 939e95bf0..a01704fd2 100644 --- a/compose/cli/docker_client.py +++ b/compose/cli/docker_client.py @@ -117,6 +117,13 @@ def docker_client(environment, version=None, tls_config=None, host=None, kwargs['user_agent'] = generate_user_agent() + # Workaround for + # https://pyinstaller.readthedocs.io/en/v3.3.1/runtime-information.html#ld-library-path-libpath-considerations + if 'LD_LIBRARY_PATH_ORIG' in environment: + kwargs['credstore_env'] = { + 'LD_LIBRARY_PATH': environment.get('LD_LIBRARY_PATH_ORIG'), + } + client = APIClient(**kwargs) client._original_base_url = kwargs.get('base_url') From 80322cfa5b00162c22f8abe6442daf3d0f57f230 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Tue, 12 Jun 2018 15:01:53 -0700 Subject: [PATCH 49/53] Better support for UTF8+bom Compose files Signed-off-by: Joffrey F --- compose/config/config.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/compose/config/config.py b/compose/config/config.py index 5c614fadc..7abab2546 100644 --- a/compose/config/config.py +++ b/compose/config/config.py @@ -1436,15 +1436,15 @@ def has_uppercase(name): return any(char in string.ascii_uppercase for char in name) -def load_yaml(filename, encoding=None): +def load_yaml(filename, encoding=None, binary=True): try: - with io.open(filename, 'r', encoding=encoding) as fh: + with io.open(filename, 'rb' if binary else 'r', encoding=encoding) as fh: return yaml.safe_load(fh) except (IOError, yaml.YAMLError, UnicodeDecodeError) as e: if encoding is None: # Sometimes the user's locale sets an encoding that doesn't match # the YAML files. Im such cases, retry once with the "default" # UTF-8 encoding - return load_yaml(filename, encoding='utf-8') + return load_yaml(filename, encoding='utf-8-sig', binary=False) error_name = getattr(e, '__module__', '') + '.' + e.__class__.__name__ raise ConfigurationError(u"{}: {}".format(error_name, e)) From 709ba0975da55e1367a1bb29f5316db98ea7e0c6 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Thu, 21 Jun 2018 11:40:32 -0700 Subject: [PATCH 50/53] Release script fixes Signed-off-by: Joffrey F --- script/release/release.sh | 4 ++-- script/release/release/bintray.py | 4 +++- script/release/release/images.py | 2 +- 3 files changed, 6 insertions(+), 4 deletions(-) diff --git a/script/release/release.sh b/script/release/release.sh index affbce37b..201182657 100755 --- a/script/release/release.sh +++ b/script/release/release.sh @@ -15,12 +15,12 @@ if test -z $BINTRAY_TOKEN; then exit 1 fi -docker run -e GITHUB_TOKEN=$GITHUB_TOKEN -e BINTRAY_TOKEN=$BINTRAY_TOKEN -it \ +docker run -e GITHUB_TOKEN=$GITHUB_TOKEN -e BINTRAY_TOKEN=$BINTRAY_TOKEN -e SSH_AUTH_SOCK=$SSH_AUTH_SOCK -it \ --mount type=bind,source=$(pwd),target=/src \ - --mount type=bind,source=$(pwd)/.git,target=/src/.git \ --mount type=bind,source=$HOME/.docker,target=/root/.docker \ --mount type=bind,source=$HOME/.gitconfig,target=/root/.gitconfig \ --mount type=bind,source=/var/run/docker.sock,target=/var/run/docker.sock \ --mount type=bind,source=$HOME/.ssh,target=/root/.ssh \ + --mount type=bind,source=/tmp,target=/tmp \ -v $HOME/.pypirc:/root/.pypirc \ compose/release-tool $* diff --git a/script/release/release/bintray.py b/script/release/release/bintray.py index d99d372c6..554611a40 100644 --- a/script/release/release/bintray.py +++ b/script/release/release/bintray.py @@ -25,7 +25,9 @@ class BintrayAPI(requests.Session): 'desc': 'Automated release for {}: {}'.format(NAME, repo_name), 'labels': ['docker-compose', 'docker', 'release-bot'], } - return self.post_json(url, data) + result = self.post_json(url, data) + result.raise_for_status() + return result def delete_repository(self, subject, repo_name): url = '{base}/repos/{subject}/{repo_name}'.format( diff --git a/script/release/release/images.py b/script/release/release/images.py index d238d4d7f..24672f2ba 100644 --- a/script/release/release/images.py +++ b/script/release/release/images.py @@ -48,7 +48,7 @@ class ImageManager(object): container = docker_client.create_container( 'docker-compose-tests:tmp', entrypoint='tox' ) - docker_client.commit(container, 'docker/compose-tests:latest') + docker_client.commit(container, 'docker/compose-tests', 'latest') docker_client.tag('docker/compose-tests:latest', 'docker/compose-tests:{}'.format(self.version)) docker_client.remove_container(container, force=True) docker_client.remove_image('docker-compose-tests:tmp', force=True) From e8af19daa32b5b5ccbad966df425b57cd3926299 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Thu, 21 Jun 2018 13:27:42 -0700 Subject: [PATCH 51/53] Fix release script Signed-off-by: Joffrey F --- script/release/release.py | 7 +++++-- script/release/release/bintray.py | 10 ++++++++++ 2 files changed, 15 insertions(+), 2 deletions(-) diff --git a/script/release/release.py b/script/release/release.py index d0545a7e6..476adc4c3 100755 --- a/script/release/release.py +++ b/script/release/release.py @@ -58,8 +58,11 @@ def create_bump_commit(repository, release_branch, bintray_user, bintray_org): repository.push_branch_to_remote(release_branch) bintray_api = BintrayAPI(os.environ['BINTRAY_TOKEN'], bintray_user) - print('Creating data repository {} on bintray'.format(release_branch.name)) - bintray_api.create_repository(bintray_org, release_branch.name, 'generic') + if not bintray_api.repository_exists(bintray_org, release_branch.name): + print('Creating data repository {} on bintray'.format(release_branch.name)) + bintray_api.create_repository(bintray_org, release_branch.name, 'generic') + else: + print('Bintray repository {} already exists. Skipping'.format(release_branch.name)) def monitor_pr_status(pr_data): diff --git a/script/release/release/bintray.py b/script/release/release/bintray.py index 554611a40..d9986875d 100644 --- a/script/release/release/bintray.py +++ b/script/release/release/bintray.py @@ -29,6 +29,16 @@ class BintrayAPI(requests.Session): result.raise_for_status() return result + def repository_exists(self, subject, repo_name): + url = '{base}/repos/{subject}/{repo_name}'.format( + base=self.base_url, subject=subject, repo_name=repo_name, + ) + result = self.get(url) + if result.status_code == 404: + return False + result.raise_for_status() + return True + def delete_repository(self, subject, repo_name): url = '{base}/repos/{subject}/{repo_name}'.format( base=self.base_url, subject=subject, repo_name=repo_name, From 1fb50395850295adf405243e7f093107fc55301a Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Thu, 21 Jun 2018 18:48:04 +0000 Subject: [PATCH 52/53] Bump 1.22.0-rc1 Signed-off-by: Joffrey F --- CHANGELOG.md | 57 +++++++++++++++++++++++++++++++++++++++++++++ compose/__init__.py | 2 +- script/run/run.sh | 2 +- 3 files changed, 59 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 18742324f..b5a22aad2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,63 @@ Change log ========== +1.22.0 (2018-06-30) +------------------- + +### Features + +#### Compose format version 3.7 + +- Introduced version 3.7 of the `docker-compose.yml` specification. + This version requires Docker Engine 18.06.0 or above. + +- Added support for `rollback_config` in the deploy configuration + +- Added support for the `init` parameter in service configurations + +- Added support for extension fields in service, network, volume, secret, + and config configurations + +#### Compose format version 2.4 + +- Added support for extension fields in service, network, + and volume configurations + +### Bugfixes + +- Fixed a bug that prevented deployment with some Compose files when + `DOCKER_DEFAULT_PLATFORM` was set + +- Compose will no longer try to create containers or volumes with + invalid starting characters + +- Fixed several bugs that prevented Compose commands from working properly + with containers created with an older version of Compose + +- Fixed an issue with the output of `docker-compose config` with the + `--compatibility-mode` flag enabled when the source file contains + attachable networks + +- Fixed a bug that prevented the `gcloud` credential store from working + properly when used with the Compose binary on UNIX + +- Fixed a bug that caused connection errors when trying to operate + over a non-HTTPS TCP connection on Windows + +- Fixed a bug that caused builds to fail on Windows if the Dockerfile + was located in a subdirectory of the build context + +- Fixed an issue that prevented proper parsing of UTF-8 BOM encoded + Compose files on Windows + +1.21.2 (2018-05-03) +------------------- + +### Bugfixes + +- Fixed a bug where the ip_range attirbute in IPAM configs was prevented + from passing validation + 1.21.1 (2018-04-27) ------------------- diff --git a/compose/__init__.py b/compose/__init__.py index 5eb2efd03..eb7195176 100644 --- a/compose/__init__.py +++ b/compose/__init__.py @@ -1,4 +1,4 @@ from __future__ import absolute_import from __future__ import unicode_literals -__version__ = '1.22.0dev' +__version__ = '1.22.0-rc1' diff --git a/script/run/run.sh b/script/run/run.sh index 45e74febd..86679bbec 100755 --- a/script/run/run.sh +++ b/script/run/run.sh @@ -15,7 +15,7 @@ set -e -VERSION="1.21.1" +VERSION="1.22.0-rc1" IMAGE="docker/compose:$VERSION" From e7de1bc3c9c9f7d65b1bf7f58136154618c90db1 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Thu, 21 Jun 2018 13:47:44 -0700 Subject: [PATCH 53/53] 3.7 --> API v1.38 Signed-off-by: Joffrey F --- compose/const.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/compose/const.py b/compose/const.py index 374a09711..ffb68db01 100644 --- a/compose/const.py +++ b/compose/const.py @@ -52,7 +52,7 @@ API_VERSIONS = { COMPOSEFILE_V3_4: '1.30', COMPOSEFILE_V3_5: '1.30', COMPOSEFILE_V3_6: '1.36', - COMPOSEFILE_V3_7: '1.36', + COMPOSEFILE_V3_7: '1.38', } API_VERSION_TO_ENGINE_VERSION = {