From a0119ae1a5a8344801d534ddfbdc1a5156a3fe32 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Mon, 22 May 2017 14:58:51 -0700 Subject: [PATCH] Rewriting tests to be UCP/Swarm compatible - Event may contain more information in some cases. Don't assume order or format - Don't assume ports are always exposed on 0.0.0.0 by default - Absence of HostConfig in a create payload sometimes causes an error at the engine level - In Swarm, volume names are prefixed by "/" - When testing against Swarm, the default network driver is overlay - Ensure custom test networks are always attachable - Handle Swarm network names - Some params moved to host config in recent (1.21+) version - Conditional test skips for Swarm environments Signed-off-by: Joffrey F --- compose/service.py | 6 + tests/acceptance/cli_test.py | 147 ++++++++++-------- .../docker-compose.override.yml | 7 +- .../override-files/docker-compose.yml | 10 +- tests/fixtures/override-files/extra.yml | 9 +- tests/helpers.py | 35 +++++ tests/integration/project_test.py | 134 +++++++++------- tests/integration/service_test.py | 90 +++++++---- tests/integration/state_test.py | 2 +- tests/integration/testcases.py | 15 +- tests/integration/volume_test.py | 21 ++- 11 files changed, 316 insertions(+), 160 deletions(-) diff --git a/compose/service.py b/compose/service.py index dcbbe251e..03c41ce67 100644 --- a/compose/service.py +++ b/compose/service.py @@ -56,7 +56,9 @@ HOST_CONFIG_KEYS = [ 'cpu_count', 'cpu_percent', 'cpu_quota', + 'cpu_shares', 'cpus', + 'cpuset', 'devices', 'dns', 'dns_search', @@ -83,6 +85,7 @@ HOST_CONFIG_KEYS = [ 'sysctls', 'userns_mode', 'volumes_from', + 'volume_driver', ] CONDITION_STARTED = 'service_started' @@ -848,6 +851,9 @@ class Service(object): cpu_count=options.get('cpu_count'), cpu_percent=options.get('cpu_percent'), nano_cpus=nano_cpus, + volume_driver=options.get('volume_driver'), + cpuset_cpus=options.get('cpuset'), + cpu_shares=options.get('cpu_shares'), ) def get_secret_volumes(self): diff --git a/tests/acceptance/cli_test.py b/tests/acceptance/cli_test.py index 9a1f5364b..ba0b53888 100644 --- a/tests/acceptance/cli_test.py +++ b/tests/acceptance/cli_test.py @@ -20,6 +20,8 @@ from docker import errors from .. import mock from ..helpers import create_host_file +from ..helpers import is_cluster +from ..helpers import no_cluster from compose.cli.command import get_project from compose.config.errors import DuplicateOverrideFileFound from compose.container import Container @@ -28,6 +30,7 @@ from compose.utils import nanoseconds_from_time_seconds from tests.integration.testcases import DockerClientTestCase from tests.integration.testcases import get_links from tests.integration.testcases import pull_busybox +from tests.integration.testcases import SWARM_SKIP_RM_VOLUMES from tests.integration.testcases import v2_1_only from tests.integration.testcases import v2_only from tests.integration.testcases import v3_only @@ -68,7 +71,8 @@ def wait_on_condition(condition, delay=0.1, timeout=40): def kill_service(service): for container in service.containers(): - container.kill() + if container.is_running: + container.kill() class ContainerCountCondition(object): @@ -78,7 +82,7 @@ class ContainerCountCondition(object): self.expected = expected def __call__(self): - return len(self.project.containers()) == self.expected + return len([c for c in self.project.containers() if c.is_running]) == self.expected def __str__(self): return "waiting for counter count == %s" % self.expected @@ -116,11 +120,14 @@ class CLITestCase(DockerClientTestCase): for container in self.project.containers(stopped=True, one_off=OneOffFilter.only): container.remove(force=True) - networks = self.client.networks() for n in networks: - if n['Name'].startswith('{}_'.format(self.project.name)): + if n['Name'].split('/')[-1].startswith('{}_'.format(self.project.name)): self.client.remove_network(n['Name']) + volumes = self.client.volumes().get('Volumes') or [] + for v in volumes: + if v['Name'].split('/')[-1].startswith('{}_'.format(self.project.name)): + self.client.remove_volume(v['Name']) if hasattr(self, '_project'): del self._project @@ -175,7 +182,10 @@ class CLITestCase(DockerClientTestCase): def test_host_not_reachable_volumes_from_container(self): self.base_dir = 'tests/fixtures/volumes-from-container' - container = self.client.create_container('busybox', 'true', name='composetest_data_container') + container = self.client.create_container( + 'busybox', 'true', name='composetest_data_container', + host_config={} + ) self.addCleanup(self.client.remove_container, container) result = self.dispatch(['-H=tcp://doesnotexist:8000', 'ps'], returncode=1) @@ -545,42 +555,48 @@ class CLITestCase(DockerClientTestCase): self.dispatch(['create']) service = self.project.get_service('simple') another = self.project.get_service('another') - self.assertEqual(len(service.containers()), 0) - self.assertEqual(len(another.containers()), 0) - self.assertEqual(len(service.containers(stopped=True)), 1) - self.assertEqual(len(another.containers(stopped=True)), 1) + service_containers = service.containers(stopped=True) + another_containers = another.containers(stopped=True) + assert len(service_containers) == 1 + assert len(another_containers) == 1 + assert not service_containers[0].is_running + assert not another_containers[0].is_running def test_create_with_force_recreate(self): self.dispatch(['create'], None) service = self.project.get_service('simple') - self.assertEqual(len(service.containers()), 0) - self.assertEqual(len(service.containers(stopped=True)), 1) + service_containers = service.containers(stopped=True) + assert len(service_containers) == 1 + assert not service_containers[0].is_running old_ids = [c.id for c in service.containers(stopped=True)] self.dispatch(['create', '--force-recreate'], None) - self.assertEqual(len(service.containers()), 0) - self.assertEqual(len(service.containers(stopped=True)), 1) + service_containers = service.containers(stopped=True) + assert len(service_containers) == 1 + assert not service_containers[0].is_running - new_ids = [c.id for c in service.containers(stopped=True)] + new_ids = [c.id for c in service_containers] - self.assertNotEqual(old_ids, new_ids) + assert old_ids != new_ids def test_create_with_no_recreate(self): self.dispatch(['create'], None) service = self.project.get_service('simple') - self.assertEqual(len(service.containers()), 0) - self.assertEqual(len(service.containers(stopped=True)), 1) + service_containers = service.containers(stopped=True) + assert len(service_containers) == 1 + assert not service_containers[0].is_running old_ids = [c.id for c in service.containers(stopped=True)] self.dispatch(['create', '--no-recreate'], None) - self.assertEqual(len(service.containers()), 0) - self.assertEqual(len(service.containers(stopped=True)), 1) + service_containers = service.containers(stopped=True) + assert len(service_containers) == 1 + assert not service_containers[0].is_running - new_ids = [c.id for c in service.containers(stopped=True)] + new_ids = [c.id for c in service_containers] - self.assertEqual(old_ids, new_ids) + assert old_ids == new_ids def test_run_one_off_with_volume(self): self.base_dir = 'tests/fixtures/simple-composefile-volume-ready' @@ -687,7 +703,7 @@ class CLITestCase(DockerClientTestCase): network_name = self.project.networks.networks['default'].full_name networks = self.client.networks(names=[network_name]) self.assertEqual(len(networks), 1) - self.assertEqual(networks[0]['Driver'], 'bridge') + assert networks[0]['Driver'] == 'bridge' if not is_cluster(self.client) else 'overlay' assert 'com.docker.network.bridge.enable_icc' not in networks[0]['Options'] network = self.client.inspect_network(networks[0]['Id']) @@ -733,11 +749,11 @@ class CLITestCase(DockerClientTestCase): networks = [ n for n in self.client.networks() - if n['Name'].startswith('{}_'.format(self.project.name)) + if n['Name'].split('/')[-1].startswith('{}_'.format(self.project.name)) ] # Two networks were created: back and front - assert sorted(n['Name'] for n in networks) == [back_name, front_name] + assert sorted(n['Name'].split('/')[-1] for n in networks) == [back_name, front_name] web_container = self.project.get_service('web').containers()[0] back_aliases = web_container.get( @@ -761,11 +777,11 @@ class CLITestCase(DockerClientTestCase): networks = [ n for n in self.client.networks() - if n['Name'].startswith('{}_'.format(self.project.name)) + if n['Name'].split('/')[-1].startswith('{}_'.format(self.project.name)) ] # One network was created: internal - assert sorted(n['Name'] for n in networks) == [internal_net] + assert sorted(n['Name'].split('/')[-1] for n in networks) == [internal_net] assert networks[0]['Internal'] is True @@ -780,11 +796,11 @@ class CLITestCase(DockerClientTestCase): networks = [ n for n in self.client.networks() - if n['Name'].startswith('{}_'.format(self.project.name)) + if n['Name'].split('/')[-1].startswith('{}_'.format(self.project.name)) ] # One networks was created: front - assert sorted(n['Name'] for n in networks) == [static_net] + assert sorted(n['Name'].split('/')[-1] for n in networks) == [static_net] web_container = self.project.get_service('web').containers()[0] ipam_config = web_container.get( @@ -803,11 +819,11 @@ class CLITestCase(DockerClientTestCase): networks = [ n for n in self.client.networks() - if n['Name'].startswith('{}_'.format(self.project.name)) + if n['Name'].split('/')[-1].startswith('{}_'.format(self.project.name)) ] # Two networks were created: back and front - assert sorted(n['Name'] for n in networks) == [back_name, front_name] + assert sorted(n['Name'].split('/')[-1] for n in networks) == [back_name, front_name] back_network = [n for n in networks if n['Name'] == back_name][0] front_network = [n for n in networks if n['Name'] == front_name][0] @@ -847,8 +863,12 @@ class CLITestCase(DockerClientTestCase): assert 'Service "web" uses an undefined network "foo"' in result.stderr @v2_only() + @no_cluster('container networks not supported in Swarm') def test_up_with_network_mode(self): - c = self.client.create_container('busybox', 'top', name='composetest_network_mode_container') + c = self.client.create_container( + 'busybox', 'top', name='composetest_network_mode_container', + host_config={} + ) self.addCleanup(self.client.remove_container, c, force=True) self.client.start(c) container_mode_source = 'container:{}'.format(c['Id']) @@ -862,7 +882,7 @@ class CLITestCase(DockerClientTestCase): networks = [ n for n in self.client.networks() - if n['Name'].startswith('{}_'.format(self.project.name)) + if n['Name'].split('/')[-1].startswith('{}_'.format(self.project.name)) ] assert not networks @@ -899,7 +919,7 @@ class CLITestCase(DockerClientTestCase): network_names = ['{}_{}'.format(self.project.name, n) for n in ['foo', 'bar']] for name in network_names: - self.client.create_network(name) + self.client.create_network(name, attachable=True) self.dispatch(['-f', filename, 'up', '-d']) container = self.project.containers()[0] @@ -917,12 +937,12 @@ class CLITestCase(DockerClientTestCase): networks = [ n['Name'] for n in self.client.networks() - if n['Name'].startswith('{}_'.format(self.project.name)) + if n['Name'].split('/')[-1].startswith('{}_'.format(self.project.name)) ] assert not networks network_name = 'composetest_external_network' - self.client.create_network(network_name) + self.client.create_network(network_name, attachable=True) self.dispatch(['-f', filename, 'up', '-d']) container = self.project.containers()[0] @@ -941,10 +961,10 @@ class CLITestCase(DockerClientTestCase): networks = [ n for n in self.client.networks() - if n['Name'].startswith('{}_'.format(self.project.name)) + if n['Name'].split('/')[-1].startswith('{}_'.format(self.project.name)) ] - assert [n['Name'] for n in networks] == [network_with_label] + assert [n['Name'].split('/')[-1] for n in networks] == [network_with_label] assert 'label_key' in networks[0]['Labels'] assert networks[0]['Labels']['label_key'] == 'label_val' @@ -961,10 +981,10 @@ class CLITestCase(DockerClientTestCase): volumes = [ v for v in self.client.volumes().get('Volumes', []) - if v['Name'].startswith('{}_'.format(self.project.name)) + if v['Name'].split('/')[-1].startswith('{}_'.format(self.project.name)) ] - assert [v['Name'] for v in volumes] == [volume_with_label] + assert set([v['Name'].split('/')[-1] for v in volumes]) == set([volume_with_label]) assert 'label_key' in volumes[0]['Labels'] assert volumes[0]['Labels']['label_key'] == 'label_val' @@ -975,7 +995,7 @@ class CLITestCase(DockerClientTestCase): network_names = [ n['Name'] for n in self.client.networks() - if n['Name'].startswith('{}_'.format(self.project.name)) + if n['Name'].split('/')[-1].startswith('{}_'.format(self.project.name)) ] assert network_names == [] @@ -1010,6 +1030,7 @@ class CLITestCase(DockerClientTestCase): assert "Unsupported config option for services.bar: 'net'" in result.stderr + @no_cluster("Legacy networking not supported on Swarm") def test_up_with_net_v1(self): self.base_dir = 'tests/fixtures/net-container' self.dispatch(['up', '-d'], None) @@ -1261,6 +1282,7 @@ class CLITestCase(DockerClientTestCase): [u'/bin/true'], ) + @py.test.mark.skipif(SWARM_SKIP_RM_VOLUMES, reason='Swarm DELETE /containers/ bug') def test_run_rm(self): self.base_dir = 'tests/fixtures/volume' proc = start_process(self.base_dir, ['run', '--rm', 'test']) @@ -1274,7 +1296,7 @@ class CLITestCase(DockerClientTestCase): mounts = containers[0].get('Mounts') for mount in mounts: if mount['Destination'] == '/container-path': - anonymousName = mount['Name'] + anonymous_name = mount['Name'] break os.kill(proc.pid, signal.SIGINT) wait_on_process(proc, 1) @@ -1287,9 +1309,11 @@ class CLITestCase(DockerClientTestCase): if volume.internal == '/container-named-path': name = volume.external break - volumeNames = [v['Name'] for v in volumes] - assert name in volumeNames - assert anonymousName not in volumeNames + volume_names = [v['Name'].split('/')[-1] for v in volumes] + assert name in volume_names + if not is_cluster(self.client): + # The `-v` flag for `docker rm` in Swarm seems to be broken + assert anonymous_name not in volume_names def test_run_service_with_dockerfile_entrypoint(self): self.base_dir = 'tests/fixtures/entrypoint-dockerfile' @@ -1411,11 +1435,10 @@ class CLITestCase(DockerClientTestCase): container.stop() # check the ports - self.assertNotEqual(port_random, None) - self.assertIn("0.0.0.0", port_random) - self.assertEqual(port_assigned, "0.0.0.0:49152") - self.assertEqual(port_range[0], "0.0.0.0:49153") - self.assertEqual(port_range[1], "0.0.0.0:49154") + assert port_random is not None + assert port_assigned.endswith(':49152') + assert port_range[0].endswith(':49153') + assert port_range[1].endswith(':49154') def test_run_service_with_explicitly_mapped_ports(self): # create one off container @@ -1431,8 +1454,8 @@ class CLITestCase(DockerClientTestCase): container.stop() # check the ports - self.assertEqual(port_short, "0.0.0.0:30000") - self.assertEqual(port_full, "0.0.0.0:30001") + assert port_short.endswith(':30000') + assert port_full.endswith(':30001') def test_run_service_with_explicitly_mapped_ip_ports(self): # create one off container @@ -1953,9 +1976,9 @@ class CLITestCase(DockerClientTestCase): result = self.dispatch(['port', 'simple', str(number)]) return result.stdout.rstrip() - self.assertEqual(get_port(3000), container.get_local_port(3000)) - self.assertEqual(get_port(3001), "0.0.0.0:49152") - self.assertEqual(get_port(3002), "0.0.0.0:49153") + assert get_port(3000) == container.get_local_port(3000) + assert ':49152' in get_port(3001) + assert ':49153' in get_port(3002) def test_expanded_port(self): self.base_dir = 'tests/fixtures/ports-composefile' @@ -1966,9 +1989,9 @@ class CLITestCase(DockerClientTestCase): result = self.dispatch(['port', 'simple', str(number)]) return result.stdout.rstrip() - self.assertEqual(get_port(3000), container.get_local_port(3000)) - self.assertEqual(get_port(3001), "0.0.0.0:49152") - self.assertEqual(get_port(3002), "0.0.0.0:49153") + assert get_port(3000) == container.get_local_port(3000) + assert ':49152' in get_port(3001) + assert ':49153' in get_port(3002) def test_port_with_scale(self): self.base_dir = 'tests/fixtures/ports-composefile-scale' @@ -2021,12 +2044,14 @@ class CLITestCase(DockerClientTestCase): assert len(lines) == 2 container, = self.project.containers() - expected_template = ( - ' container {} {} (image=busybox:latest, ' - 'name=simplecomposefile_simple_1)') + expected_template = ' container {} {}' + expected_meta_info = ['image=busybox:latest', 'name=simplecomposefile_simple_1'] assert expected_template.format('create', container.id) in lines[0] assert expected_template.format('start', container.id) in lines[1] + for line in lines: + for info in expected_meta_info: + assert info in line assert has_timestamp(lines[0]) @@ -2069,7 +2094,6 @@ class CLITestCase(DockerClientTestCase): 'docker-compose.yml', 'docker-compose.override.yml', 'extra.yml', - ] self._project = get_project(self.base_dir, config_paths) self.dispatch( @@ -2086,7 +2110,6 @@ class CLITestCase(DockerClientTestCase): web, other, db = containers self.assertEqual(web.human_readable_command, 'top') - self.assertTrue({'db', 'other'} <= set(get_links(web))) self.assertEqual(db.human_readable_command, 'top') self.assertEqual(other.human_readable_command, 'top') diff --git a/tests/fixtures/override-files/docker-compose.override.yml b/tests/fixtures/override-files/docker-compose.override.yml index a03d3d6f5..b2c540601 100644 --- a/tests/fixtures/override-files/docker-compose.override.yml +++ b/tests/fixtures/override-files/docker-compose.override.yml @@ -1,6 +1,7 @@ - -web: +version: '2.2' +services: + web: command: "top" -db: + db: command: "top" diff --git a/tests/fixtures/override-files/docker-compose.yml b/tests/fixtures/override-files/docker-compose.yml index 8eb43ddb0..6c3d4e172 100644 --- a/tests/fixtures/override-files/docker-compose.yml +++ b/tests/fixtures/override-files/docker-compose.yml @@ -1,10 +1,10 @@ - -web: +version: '2.2' +services: + web: image: busybox:latest command: "sleep 200" - links: + depends_on: - db - -db: + db: image: busybox:latest command: "sleep 200" diff --git a/tests/fixtures/override-files/extra.yml b/tests/fixtures/override-files/extra.yml index 7b3ade9c2..492c37952 100644 --- a/tests/fixtures/override-files/extra.yml +++ b/tests/fixtures/override-files/extra.yml @@ -1,9 +1,10 @@ - -web: - links: +version: '2.2' +services: + web: + depends_on: - db - other -other: + other: image: busybox:latest command: "top" diff --git a/tests/helpers.py b/tests/helpers.py index 59efd2557..662353c93 100644 --- a/tests/helpers.py +++ b/tests/helpers.py @@ -1,8 +1,12 @@ from __future__ import absolute_import from __future__ import unicode_literals +import functools import os +from docker.errors import APIError +from pytest import skip + from compose.config.config import ConfigDetails from compose.config.config import ConfigFile from compose.config.config import load @@ -44,3 +48,34 @@ def create_host_file(client, filename): "Container exited with code {}:\n{}".format(exitcode, output)) finally: client.remove_container(container, force=True) + + +def is_cluster(client): + nodes = None + + def get_nodes_number(): + try: + return len(client.nodes()) + except APIError: + # If the Engine is not part of a Swarm, the SDK will raise + # an APIError + return 0 + + if nodes is None: + # Only make the API call if the value hasn't been cached yet + nodes = get_nodes_number() + + return nodes > 1 + + +def no_cluster(reason): + def decorator(f): + @functools.wraps(f) + def wrapper(self, *args, **kwargs): + if is_cluster(self.client): + skip("Test will not be run in cluster mode: %s" % reason) + return + return f(self, *args, **kwargs) + return wrapper + + return decorator diff --git a/tests/integration/project_test.py b/tests/integration/project_test.py index 6c5f719ed..6731f25dd 100644 --- a/tests/integration/project_test.py +++ b/tests/integration/project_test.py @@ -6,12 +6,16 @@ import random import py import pytest +from docker.errors import APIError from docker.errors import NotFound from .. import mock from ..helpers import build_config as load_config from ..helpers import create_host_file +from ..helpers import is_cluster +from ..helpers import no_cluster from .testcases import DockerClientTestCase +from .testcases import SWARM_SKIP_CONTAINERS_ALL from compose.config import config from compose.config import ConfigurationError from compose.config import types @@ -57,6 +61,20 @@ class ProjectTest(DockerClientTestCase): containers = project.containers() self.assertEqual(len(containers), 2) + @pytest.mark.skipif(SWARM_SKIP_CONTAINERS_ALL, reason='Swarm /containers/json bug') + def test_containers_stopped(self): + web = self.create_service('web') + db = self.create_service('db') + project = Project('composetest', [web, db], self.client) + + project.up() + assert len(project.containers()) == 2 + assert len(project.containers(stopped=True)) == 2 + + project.stop() + assert len(project.containers()) == 0 + assert len(project.containers(stopped=True)) == 2 + def test_containers_with_service_names(self): web = self.create_service('web') db = self.create_service('db') @@ -110,6 +128,7 @@ class ProjectTest(DockerClientTestCase): volumes=['/var/data'], name='composetest_data_container', labels={LABEL_PROJECT: 'composetest'}, + host_config={}, ) project = Project.from_config( name='composetest', @@ -125,6 +144,7 @@ class ProjectTest(DockerClientTestCase): self.assertEqual(db._get_volumes_from(), [data_container.id + ':rw']) @v2_only() + @no_cluster('container networks not supported in Swarm') def test_network_mode_from_service(self): project = Project.from_config( name='composetest', @@ -152,6 +172,7 @@ class ProjectTest(DockerClientTestCase): self.assertEqual(web.network_mode.mode, 'container:' + net.containers()[0].id) @v2_only() + @no_cluster('container networks not supported in Swarm') def test_network_mode_from_container(self): def get_project(): return Project.from_config( @@ -179,6 +200,7 @@ class ProjectTest(DockerClientTestCase): name='composetest_net_container', command='top', labels={LABEL_PROJECT: 'composetest'}, + host_config={}, ) net_container.start() @@ -188,6 +210,7 @@ class ProjectTest(DockerClientTestCase): web = project.get_service('web') self.assertEqual(web.network_mode.mode, 'container:' + net_container.id) + @no_cluster('container networks not supported in Swarm') def test_net_from_service_v1(self): project = Project.from_config( name='composetest', @@ -211,6 +234,7 @@ class ProjectTest(DockerClientTestCase): net = project.get_service('net') self.assertEqual(web.network_mode.mode, 'container:' + net.containers()[0].id) + @no_cluster('container networks not supported in Swarm') def test_net_from_container_v1(self): def get_project(): return Project.from_config( @@ -235,6 +259,7 @@ class ProjectTest(DockerClientTestCase): name='composetest_net_container', command='top', labels={LABEL_PROJECT: 'composetest'}, + host_config={}, ) net_container.start() @@ -260,12 +285,12 @@ class ProjectTest(DockerClientTestCase): project.start(service_names=['web']) self.assertEqual( - set(c.name for c in project.containers()), + set(c.name for c in project.containers() if c.is_running), set([web_container_1.name, web_container_2.name])) project.start() self.assertEqual( - set(c.name for c in project.containers()), + set(c.name for c in project.containers() if c.is_running), set([web_container_1.name, web_container_2.name, db_container.name])) project.pause(service_names=['web']) @@ -285,10 +310,12 @@ class ProjectTest(DockerClientTestCase): self.assertEqual(len([c.name for c in project.containers() if c.is_paused]), 0) project.stop(service_names=['web'], timeout=1) - self.assertEqual(set(c.name for c in project.containers()), set([db_container.name])) + self.assertEqual( + set(c.name for c in project.containers() if c.is_running), set([db_container.name]) + ) project.kill(service_names=['db']) - self.assertEqual(len(project.containers()), 0) + self.assertEqual(len([c for c in project.containers() if c.is_running]), 0) self.assertEqual(len(project.containers(stopped=True)), 3) project.remove_stopped(service_names=['web']) @@ -303,11 +330,13 @@ class ProjectTest(DockerClientTestCase): project = Project('composetest', [web, db], self.client) project.create(['db']) - self.assertEqual(len(project.containers()), 0) - self.assertEqual(len(project.containers(stopped=True)), 1) - self.assertEqual(len(db.containers()), 0) - self.assertEqual(len(db.containers(stopped=True)), 1) - self.assertEqual(len(web.containers(stopped=True)), 0) + containers = project.containers(stopped=True) + assert len(containers) == 1 + assert not containers[0].is_running + db_containers = db.containers(stopped=True) + assert len(db_containers) == 1 + assert not db_containers[0].is_running + assert len(web.containers(stopped=True)) == 0 def test_create_twice(self): web = self.create_service('web') @@ -316,12 +345,14 @@ class ProjectTest(DockerClientTestCase): project.create(['db', 'web']) project.create(['db', 'web']) - self.assertEqual(len(project.containers()), 0) - self.assertEqual(len(project.containers(stopped=True)), 2) - self.assertEqual(len(db.containers()), 0) - self.assertEqual(len(db.containers(stopped=True)), 1) - self.assertEqual(len(web.containers()), 0) - self.assertEqual(len(web.containers(stopped=True)), 1) + containers = project.containers(stopped=True) + assert len(containers) == 2 + db_containers = db.containers(stopped=True) + assert len(db_containers) == 1 + assert not db_containers[0].is_running + web_containers = web.containers(stopped=True) + assert len(web_containers) == 1 + assert not web_containers[0].is_running def test_create_with_links(self): db = self.create_service('db') @@ -329,12 +360,11 @@ class ProjectTest(DockerClientTestCase): project = Project('composetest', [db, web], self.client) project.create(['web']) - self.assertEqual(len(project.containers()), 0) - self.assertEqual(len(project.containers(stopped=True)), 2) - self.assertEqual(len(db.containers()), 0) - self.assertEqual(len(db.containers(stopped=True)), 1) - self.assertEqual(len(web.containers()), 0) - self.assertEqual(len(web.containers(stopped=True)), 1) + # self.assertEqual(len(project.containers()), 0) + assert len(project.containers(stopped=True)) == 2 + assert not [c for c in project.containers(stopped=True) if c.is_running] + assert len(db.containers(stopped=True)) == 1 + assert len(web.containers(stopped=True)) == 1 def test_create_strategy_always(self): db = self.create_service('db') @@ -343,11 +373,11 @@ class ProjectTest(DockerClientTestCase): old_id = project.containers(stopped=True)[0].id project.create(['db'], strategy=ConvergenceStrategy.always) - self.assertEqual(len(project.containers()), 0) - self.assertEqual(len(project.containers(stopped=True)), 1) + assert len(project.containers(stopped=True)) == 1 db_container = project.containers(stopped=True)[0] - self.assertNotEqual(db_container.id, old_id) + assert not db_container.is_running + assert db_container.id != old_id def test_create_strategy_never(self): db = self.create_service('db') @@ -356,11 +386,11 @@ class ProjectTest(DockerClientTestCase): old_id = project.containers(stopped=True)[0].id project.create(['db'], strategy=ConvergenceStrategy.never) - self.assertEqual(len(project.containers()), 0) - self.assertEqual(len(project.containers(stopped=True)), 1) + assert len(project.containers(stopped=True)) == 1 db_container = project.containers(stopped=True)[0] - self.assertEqual(db_container.id, old_id) + assert not db_container.is_running + assert db_container.id == old_id def test_project_up(self): web = self.create_service('web') @@ -550,8 +580,8 @@ class ProjectTest(DockerClientTestCase): self.assertEqual(len(project.containers(stopped=True)), 2) self.assertEqual(len(project.get_service('web').containers()), 0) self.assertEqual(len(project.get_service('db').containers()), 1) - self.assertEqual(len(project.get_service('data').containers()), 0) self.assertEqual(len(project.get_service('data').containers(stopped=True)), 1) + assert not project.get_service('data').containers(stopped=True)[0].is_running self.assertEqual(len(project.get_service('console').containers()), 0) def test_project_up_recreate_with_tmpfs_volume(self): @@ -737,10 +767,10 @@ class ProjectTest(DockerClientTestCase): "com.docker.compose.network.test": "9-29-045" } - @v2_only() + @v2_1_only() def test_up_with_network_static_addresses(self): config_data = build_config( - version=V2_0, + version=V2_1, services=[{ 'name': 'web', 'image': 'busybox:latest', @@ -766,7 +796,8 @@ class ProjectTest(DockerClientTestCase): {"subnet": "fe80::/64", "gateway": "fe80::1001:1"} ] - } + }, + 'enable_ipv6': True, } } ) @@ -777,13 +808,8 @@ class ProjectTest(DockerClientTestCase): ) project.up(detached=True) - network = self.client.networks(names=['static_test'])[0] service_container = project.get_service('web').containers()[0] - assert network['Options'] == { - "com.docker.network.enable_ipv6": "true" - } - IPAMConfig = (service_container.inspect().get('NetworkSettings', {}). get('Networks', {}).get('composetest_static_test', {}). get('IPAMConfig', {})) @@ -825,7 +851,7 @@ class ProjectTest(DockerClientTestCase): config_data=config_data, ) project.up(detached=True) - network = self.client.networks(names=['static_test'])[0] + network = [n for n in self.client.networks() if 'static_test' in n['Name']][0] service_container = project.get_service('web').containers()[0] assert network['EnableIPv6'] is True @@ -1026,8 +1052,8 @@ class ProjectTest(DockerClientTestCase): project.up() self.assertEqual(len(project.containers()), 1) - volume_data = self.client.inspect_volume(full_vol_name) - self.assertEqual(volume_data['Name'], full_vol_name) + volume_data = self.get_volume_data(full_vol_name) + assert volume_data['Name'].split('/')[-1] == full_vol_name self.assertEqual(volume_data['Driver'], 'local') @v2_1_only() @@ -1062,10 +1088,12 @@ class ProjectTest(DockerClientTestCase): volumes = [ v for v in self.client.volumes().get('Volumes', []) - if v['Name'].startswith('composetest_') + if v['Name'].split('/')[-1].startswith('composetest_') ] - assert [v['Name'] for v in volumes] == ['composetest_{}'.format(volume_name)] + assert set([v['Name'].split('/')[-1] for v in volumes]) == set( + ['composetest_{}'.format(volume_name)] + ) assert 'label_key' in volumes[0]['Labels'] assert volumes[0]['Labels']['label_key'] == 'label_val' @@ -1205,8 +1233,8 @@ class ProjectTest(DockerClientTestCase): ) project.volumes.initialize() - volume_data = self.client.inspect_volume(full_vol_name) - assert volume_data['Name'] == full_vol_name + volume_data = self.get_volume_data(full_vol_name) + assert volume_data['Name'].split('/')[-1] == full_vol_name assert volume_data['Driver'] == 'local' @v2_only() @@ -1229,8 +1257,8 @@ class ProjectTest(DockerClientTestCase): ) project.up() - volume_data = self.client.inspect_volume(full_vol_name) - self.assertEqual(volume_data['Name'], full_vol_name) + volume_data = self.get_volume_data(full_vol_name) + assert volume_data['Name'].split('/')[-1] == full_vol_name self.assertEqual(volume_data['Driver'], 'local') @v3_only() @@ -1287,10 +1315,11 @@ class ProjectTest(DockerClientTestCase): name='composetest', config_data=config_data, client=self.client ) - with self.assertRaises(config.ConfigurationError): + with self.assertRaises(APIError if is_cluster(self.client) else config.ConfigurationError): project.volumes.initialize() @v2_only() + @no_cluster('inspect volume by name defect on Swarm Classic') def test_initialize_volumes_updated_driver(self): vol_name = '{0:x}'.format(random.getrandbits(32)) full_vol_name = 'composetest_{0}'.format(vol_name) @@ -1310,8 +1339,8 @@ class ProjectTest(DockerClientTestCase): ) project.volumes.initialize() - volume_data = self.client.inspect_volume(full_vol_name) - self.assertEqual(volume_data['Name'], full_vol_name) + volume_data = self.get_volume_data(full_vol_name) + assert volume_data['Name'].split('/')[-1] == full_vol_name self.assertEqual(volume_data['Driver'], 'local') config_data = config_data._replace( @@ -1348,8 +1377,8 @@ class ProjectTest(DockerClientTestCase): ) project.volumes.initialize() - volume_data = self.client.inspect_volume(full_vol_name) - self.assertEqual(volume_data['Name'], full_vol_name) + volume_data = self.get_volume_data(full_vol_name) + assert volume_data['Name'].split('/')[-1] == full_vol_name self.assertEqual(volume_data['Driver'], 'local') config_data = config_data._replace( @@ -1361,11 +1390,12 @@ class ProjectTest(DockerClientTestCase): client=self.client ) project.volumes.initialize() - volume_data = self.client.inspect_volume(full_vol_name) - self.assertEqual(volume_data['Name'], full_vol_name) + volume_data = self.get_volume_data(full_vol_name) + assert volume_data['Name'].split('/')[-1] == full_vol_name self.assertEqual(volume_data['Driver'], 'local') @v2_only() + @no_cluster('inspect volume by name defect on Swarm Classic') def test_initialize_volumes_external_volumes(self): # Use composetest_ prefix so it gets garbage-collected in tearDown() vol_name = 'composetest_{0:x}'.format(random.getrandbits(32)) diff --git a/tests/integration/service_test.py b/tests/integration/service_test.py index 178df1323..baf21af3c 100644 --- a/tests/integration/service_test.py +++ b/tests/integration/service_test.py @@ -13,9 +13,13 @@ from six import StringIO from six import text_type from .. import mock +from ..helpers import is_cluster +from ..helpers import no_cluster from .testcases import DockerClientTestCase from .testcases import get_links from .testcases import pull_busybox +from .testcases import SWARM_SKIP_CONTAINERS_ALL +from .testcases import SWARM_SKIP_CPU_SHARES from compose import __version__ from compose.config.types import VolumeFromSpec from compose.config.types import VolumeSpec @@ -100,6 +104,7 @@ class ServiceTest(DockerClientTestCase): service.start_container(container) self.assertEqual('foodriver', container.get('HostConfig.VolumeDriver')) + @pytest.mark.skipif(SWARM_SKIP_CPU_SHARES, reason='Swarm --cpu-shares bug') def test_create_container_with_cpu_shares(self): service = self.create_service('db', cpu_shares=73) container = service.create_container() @@ -151,6 +156,7 @@ class ServiceTest(DockerClientTestCase): service.start_container(container) assert container.get('HostConfig.Init') is True + @pytest.mark.xfail(True, reason='Option has been removed in Engine 17.06.0') def test_create_container_with_init_path(self): self.require_api_version('1.25') docker_init_path = find_executable('docker-init') @@ -249,6 +255,7 @@ class ServiceTest(DockerClientTestCase): 'busybox', 'true', volumes={container_path: {}}, labels={'com.docker.compose.test_image': 'true'}, + host_config={} ) image = self.client.commit(tmp_container)['Id'] @@ -278,6 +285,7 @@ class ServiceTest(DockerClientTestCase): image='busybox:latest', command=["top"], labels={LABEL_PROJECT: 'composetest'}, + host_config={}, ) host_service = self.create_service( 'host', @@ -321,9 +329,15 @@ class ServiceTest(DockerClientTestCase): self.assertIn('FOO=2', new_container.get('Config.Env')) self.assertEqual(new_container.name, 'composetest_db_1') self.assertEqual(new_container.get_mount('/etc')['Source'], volume_path) - self.assertIn( - 'affinity:container==%s' % old_container.id, - new_container.get('Config.Env')) + if not is_cluster(self.client): + assert ( + 'affinity:container==%s' % old_container.id in + new_container.get('Config.Env') + ) + else: + # In Swarm, the env marker is consumed and the container should be deployed + # on the same node. + assert old_container.get('Node.Name') == new_container.get('Node.Name') self.assertEqual(len(self.client.containers(all=True)), num_containers_before) self.assertNotEqual(old_container.id, new_container.id) @@ -350,8 +364,13 @@ class ServiceTest(DockerClientTestCase): ConvergencePlan('recreate', [orig_container])) assert new_container.get_mount('/etc')['Source'] == volume_path - assert ('affinity:container==%s' % orig_container.id in - new_container.get('Config.Env')) + if not is_cluster(self.client): + assert ('affinity:container==%s' % orig_container.id in + new_container.get('Config.Env')) + else: + # In Swarm, the env marker is consumed and the container should be deployed + # on the same node. + assert orig_container.get('Node.Name') == new_container.get('Node.Name') orig_container = new_container @@ -464,18 +483,21 @@ class ServiceTest(DockerClientTestCase): ) containers = service.execute_convergence_plan(ConvergencePlan('create', []), start=False) - self.assertEqual(len(service.containers()), 0) - self.assertEqual(len(service.containers(stopped=True)), 1) + service_containers = service.containers(stopped=True) + assert len(service_containers) == 1 + assert not service_containers[0].is_running containers = service.execute_convergence_plan( ConvergencePlan('recreate', containers), start=False) - self.assertEqual(len(service.containers()), 0) - self.assertEqual(len(service.containers(stopped=True)), 1) + service_containers = service.containers(stopped=True) + assert len(service_containers) == 1 + assert not service_containers[0].is_running service.execute_convergence_plan(ConvergencePlan('start', containers), start=False) - self.assertEqual(len(service.containers()), 0) - self.assertEqual(len(service.containers(stopped=True)), 1) + service_containers = service.containers(stopped=True) + assert len(service_containers) == 1 + assert not service_containers[0].is_running def test_start_container_passes_through_options(self): db = self.create_service('db') @@ -487,6 +509,7 @@ class ServiceTest(DockerClientTestCase): create_and_start_container(db) self.assertEqual(db.containers()[0].environment['FOO'], 'BAR') + @no_cluster('No legacy links support in Swarm') def test_start_container_creates_links(self): db = self.create_service('db') web = self.create_service('web', links=[(db, None)]) @@ -503,6 +526,7 @@ class ServiceTest(DockerClientTestCase): 'db']) ) + @no_cluster('No legacy links support in Swarm') def test_start_container_creates_links_with_names(self): db = self.create_service('db') web = self.create_service('web', links=[(db, 'custom_link_name')]) @@ -519,6 +543,7 @@ class ServiceTest(DockerClientTestCase): 'custom_link_name']) ) + @no_cluster('No legacy links support in Swarm') def test_start_container_with_external_links(self): db = self.create_service('db') web = self.create_service('web', external_links=['composetest_db_1', @@ -537,6 +562,7 @@ class ServiceTest(DockerClientTestCase): 'db_3']), ) + @no_cluster('No legacy links support in Swarm') def test_start_normal_container_does_not_create_links_to_its_own_service(self): db = self.create_service('db') @@ -546,6 +572,7 @@ class ServiceTest(DockerClientTestCase): c = create_and_start_container(db) self.assertEqual(set(get_links(c)), set([])) + @no_cluster('No legacy links support in Swarm') def test_start_one_off_container_creates_links_to_its_own_service(self): db = self.create_service('db') @@ -572,7 +599,7 @@ class ServiceTest(DockerClientTestCase): container = create_and_start_container(service) container.wait() self.assertIn(b'success', container.logs()) - self.assertEqual(len(self.client.images(name='composetest_test')), 1) + assert len(self.client.images(name='composetest_test')) >= 1 def test_start_container_uses_tagged_image_if_it_exists(self): self.check_build('tests/fixtures/simple-dockerfile', tag='composetest_test') @@ -719,20 +746,27 @@ class ServiceTest(DockerClientTestCase): '0.0.0.0:9001:9000/udp', ]) container = create_and_start_container(service).inspect() - self.assertEqual(container['NetworkSettings']['Ports'], { - '8000/tcp': [ - { - 'HostIp': '127.0.0.1', - 'HostPort': '8001', - }, - ], - '9000/udp': [ - { - 'HostIp': '0.0.0.0', - 'HostPort': '9001', - }, - ], - }) + assert container['NetworkSettings']['Ports']['8000/tcp'] == [{ + 'HostIp': '127.0.0.1', + 'HostPort': '8001', + }] + assert container['NetworkSettings']['Ports']['9000/udp'][0]['HostPort'] == '9001' + if not is_cluster(self.client): + assert container['NetworkSettings']['Ports']['9000/udp'][0]['HostIp'] == '0.0.0.0' + # self.assertEqual(container['NetworkSettings']['Ports'], { + # '8000/tcp': [ + # { + # 'HostIp': '127.0.0.1', + # 'HostPort': '8001', + # }, + # ], + # '9000/udp': [ + # { + # 'HostIp': '0.0.0.0', + # 'HostPort': '9001', + # }, + # ], + # }) def test_create_with_image_id(self): # Get image id for the current busybox:latest @@ -760,6 +794,10 @@ class ServiceTest(DockerClientTestCase): service.scale(0) self.assertEqual(len(service.containers()), 0) + @pytest.mark.skipif( + SWARM_SKIP_CONTAINERS_ALL, + reason='Swarm /containers/json bug' + ) def test_scale_with_stopped_containers(self): """ Given there are some stopped containers and scale is called with a diff --git a/tests/integration/state_test.py b/tests/integration/state_test.py index 07b28e784..0dd5f44ad 100644 --- a/tests/integration/state_test.py +++ b/tests/integration/state_test.py @@ -251,7 +251,7 @@ class ServiceStateTest(DockerClientTestCase): container = web.create_container() # update the image - c = self.client.create_container(image, ['touch', '/hello.txt']) + c = self.client.create_container(image, ['touch', '/hello.txt'], host_config={}) self.client.commit(c, repository=repo, tag=tag) self.client.remove_container(c) diff --git a/tests/integration/testcases.py b/tests/integration/testcases.py index 57814872c..1e0d63215 100644 --- a/tests/integration/testcases.py +++ b/tests/integration/testcases.py @@ -8,6 +8,7 @@ from docker.utils import version_lt from pytest import skip from .. import unittest +from ..helpers import is_cluster from compose.cli.docker_client import docker_client from compose.config.config import resolve_environment from compose.config.environment import Environment @@ -21,6 +22,10 @@ from compose.const import LABEL_PROJECT from compose.progress_stream import stream_output from compose.service import Service +SWARM_SKIP_CONTAINERS_ALL = os.environ.get('SWARM_SKIP_CONTAINERS_ALL', '0') != '0' +SWARM_SKIP_CPU_SHARES = os.environ.get('SWARM_SKIP_CPU_SHARES', '0') != '0' +SWARM_SKIP_RM_VOLUMES = os.environ.get('SWARM_SKIP_RM_VOLUMES', '0') != '0' + def pull_busybox(client): client.pull('busybox:latest', stream=False) @@ -97,7 +102,7 @@ class DockerClientTestCase(unittest.TestCase): for i in self.client.images( filters={'label': 'com.docker.compose.test_image'}): - self.client.remove_image(i) + self.client.remove_image(i, force=True) volumes = self.client.volumes().get('Volumes') or [] for v in volumes: @@ -133,3 +138,11 @@ class DockerClientTestCase(unittest.TestCase): api_version = self.client.version()['ApiVersion'] if version_lt(api_version, minimum): skip("API version is too low ({} < {})".format(api_version, minimum)) + + def get_volume_data(self, volume_name): + if not is_cluster(self.client): + return self.client.inspect_volume(volume_name) + + volumes = self.client.volumes(filters={'name': volume_name})['Volumes'] + assert len(volumes) > 0 + return self.client.inspect_volume(volumes[0]['Name']) diff --git a/tests/integration/volume_test.py b/tests/integration/volume_test.py index add169623..772631a5b 100644 --- a/tests/integration/volume_test.py +++ b/tests/integration/volume_test.py @@ -3,6 +3,7 @@ from __future__ import unicode_literals from docker.errors import DockerException +from ..helpers import no_cluster from .testcases import DockerClientTestCase from compose.const import LABEL_PROJECT from compose.const import LABEL_VOLUME @@ -35,26 +36,28 @@ class VolumeTest(DockerClientTestCase): def test_create_volume(self): vol = self.create_volume('volume01') vol.create() - info = self.client.inspect_volume(vol.full_name) - assert info['Name'] == vol.full_name + info = self.get_volume_data(vol.full_name) + assert info['Name'].split('/')[-1] == vol.full_name def test_recreate_existing_volume(self): vol = self.create_volume('volume01') vol.create() - info = self.client.inspect_volume(vol.full_name) - assert info['Name'] == vol.full_name + info = self.get_volume_data(vol.full_name) + assert info['Name'].split('/')[-1] == vol.full_name vol.create() - info = self.client.inspect_volume(vol.full_name) - assert info['Name'] == vol.full_name + info = self.get_volume_data(vol.full_name) + assert info['Name'].split('/')[-1] == vol.full_name + @no_cluster('inspect volume by name defect on Swarm Classic') def test_inspect_volume(self): vol = self.create_volume('volume01') vol.create() info = vol.inspect() assert info['Name'] == vol.full_name + @no_cluster('remove volume by name defect on Swarm Classic') def test_remove_volume(self): vol = Volume(self.client, 'composetest', 'volume01') vol.create() @@ -62,6 +65,7 @@ class VolumeTest(DockerClientTestCase): volumes = self.client.volumes()['Volumes'] assert len([v for v in volumes if v['Name'] == vol.full_name]) == 0 + @no_cluster('inspect volume by name defect on Swarm Classic') def test_external_volume(self): vol = self.create_volume('composetest_volume_ext', external=True) assert vol.external is True @@ -70,6 +74,7 @@ class VolumeTest(DockerClientTestCase): info = vol.inspect() assert info['Name'] == vol.name + @no_cluster('inspect volume by name defect on Swarm Classic') def test_external_aliased_volume(self): alias_name = 'composetest_alias01' vol = self.create_volume('volume01', external=alias_name) @@ -79,24 +84,28 @@ class VolumeTest(DockerClientTestCase): info = vol.inspect() assert info['Name'] == alias_name + @no_cluster('inspect volume by name defect on Swarm Classic') def test_exists(self): vol = self.create_volume('volume01') assert vol.exists() is False vol.create() assert vol.exists() is True + @no_cluster('inspect volume by name defect on Swarm Classic') def test_exists_external(self): vol = self.create_volume('volume01', external=True) assert vol.exists() is False vol.create() assert vol.exists() is True + @no_cluster('inspect volume by name defect on Swarm Classic') def test_exists_external_aliased(self): vol = self.create_volume('volume01', external='composetest_alias01') assert vol.exists() is False vol.create() assert vol.exists() is True + @no_cluster('inspect volume by name defect on Swarm Classic') def test_volume_default_labels(self): vol = self.create_volume('volume01') vol.create()