diff --git a/compose/config/config.py b/compose/config/config.py index 4c9cf423bb..bb7d18a1c5 100644 --- a/compose/config/config.py +++ b/compose/config/config.py @@ -35,6 +35,7 @@ from .types import parse_extra_hosts from .types import parse_restart_spec from .types import ServiceLink +from .types import ServicePort from .types import VolumeFromSpec from .types import VolumeSpec from .validation import match_named_volumes @@ -683,10 +684,25 @@ def process_service(service_config): service_dict[field] = to_list(service_dict[field]) service_dict = process_healthcheck(service_dict, service_config.name) + service_dict = process_ports(service_dict) return service_dict +def process_ports(service_dict): + if 'ports' not in service_dict: + return service_dict + + ports = [] + for port_definition in service_dict['ports']: + if isinstance(port_definition, ServicePort): + ports.append(port_definition) + else: + ports.extend(ServicePort.parse(port_definition)) + service_dict['ports'] = ports + return service_dict + + def process_depends_on(service_dict): if 'depends_on' in service_dict and not isinstance(service_dict['depends_on'], dict): service_dict['depends_on'] = dict([ @@ -864,7 +880,7 @@ def merge_service_dicts(base, override, version): md.merge_field(field, merge_path_mappings) for field in [ - 'ports', 'cap_add', 'cap_drop', 'expose', 'external_links', + 'cap_add', 'cap_drop', 'expose', 'external_links', 'security_opt', 'volumes_from', ]: md.merge_field(field, merge_unique_items_lists, default=[]) @@ -873,6 +889,7 @@ def merge_service_dicts(base, override, version): md.merge_field(field, merge_list_or_string) md.merge_field('logging', merge_logging, default={}) + merge_ports(md, base, override) for field in set(ALLOWED_KEYS) - set(md): md.merge_scalar(field) @@ -889,6 +906,23 @@ def merge_unique_items_lists(base, override): return sorted(set().union(base, override)) +def merge_ports(md, base, override): + def parse_sequence_func(seq): + acc = [] + for item in seq: + acc.extend(ServicePort.parse(item)) + return to_mapping(acc, 'merge_field') + + field = 'ports' + + if not md.needs_merge(field): + return + + merged = parse_sequence_func(md.base.get(field, [])) + merged.update(parse_sequence_func(md.override.get(field, []))) + md[field] = [item for item in sorted(merged.values())] + + def merge_build(output, base, override): def to_dict(service): build_config = service.get('build', {}) diff --git a/compose/config/config_schema_v3.1.json b/compose/config/config_schema_v3.1.json index 77c2d35cbc..219ccdd488 100644 --- a/compose/config/config_schema_v3.1.json +++ b/compose/config/config_schema_v3.1.json @@ -168,8 +168,21 @@ "ports": { "type": "array", "items": { - "type": ["string", "number"], - "format": "ports" + "oneOf": [ + {"type": "number", "format": "ports"}, + {"type": "string", "format": "ports"}, + { + "type": "object", + "properties": { + "mode": {"type": "string"}, + "target": {"type": "integer"}, + "published": {"type": "integer"}, + "protocol": {"type": "string"} + }, + "required": ["target"], + "additionalProperties": false + } + ] }, "uniqueItems": true }, diff --git a/compose/config/serialize.py b/compose/config/serialize.py index 46d283f080..58581f7cc5 100644 --- a/compose/config/serialize.py +++ b/compose/config/serialize.py @@ -7,6 +7,7 @@ from compose.config import types from compose.config.config import V1 from compose.config.config import V2_1 +from compose.config.config import V3_1 def serialize_config_type(dumper, data): @@ -14,8 +15,14 @@ def serialize_config_type(dumper, data): return representer(data.repr()) +def serialize_dict_type(dumper, data): + return dumper.represent_dict(data.repr()) + + yaml.SafeDumper.add_representer(types.VolumeFromSpec, serialize_config_type) yaml.SafeDumper.add_representer(types.VolumeSpec, serialize_config_type) +yaml.SafeDumper.add_representer(types.ServiceSecret, serialize_dict_type) +yaml.SafeDumper.add_representer(types.ServicePort, serialize_dict_type) def denormalize_config(config): @@ -102,7 +109,10 @@ def denormalize_service_dict(service_dict, version): service_dict['healthcheck']['timeout'] ) - if 'secrets' in service_dict: - service_dict['secrets'] = map(lambda s: s.repr(), service_dict['secrets']) + if 'ports' in service_dict and version != V3_1: + service_dict['ports'] = map( + lambda p: p.legacy_repr() if isinstance(p, types.ServicePort) else p, + service_dict['ports'] + ) return service_dict diff --git a/compose/config/types.py b/compose/config/types.py index 811e6c1fc3..aa1edcf0b1 100644 --- a/compose/config/types.py +++ b/compose/config/types.py @@ -9,6 +9,7 @@ from collections import namedtuple import six +from docker.utils.ports import build_port_bindings from ..const import COMPOSEFILE_V1 as V1 from .errors import ConfigurationError @@ -258,3 +259,61 @@ def repr(self): return dict( [(k, v) for k, v in self._asdict().items() if v is not None] ) + + +class ServicePort(namedtuple('_ServicePort', 'target published protocol mode external_ip')): + + @classmethod + def parse(cls, spec): + if not isinstance(spec, dict): + result = [] + for k, v in build_port_bindings([spec]).items(): + if '/' in k: + target, proto = k.split('/', 1) + else: + target, proto = (k, None) + for pub in v: + if pub is None: + result.append( + cls(target, None, proto, None, None) + ) + elif isinstance(pub, tuple): + result.append( + cls(target, pub[1], proto, None, pub[0]) + ) + else: + result.append( + cls(target, pub, proto, None, None) + ) + return result + + return [cls( + spec.get('target'), + spec.get('published'), + spec.get('protocol'), + spec.get('mode'), + None + )] + + @property + def merge_field(self): + return (self.target, self.published) + + def repr(self): + return dict( + [(k, v) for k, v in self._asdict().items() if v is not None] + ) + + def legacy_repr(self): + return normalize_port_dict(self.repr()) + + +def normalize_port_dict(port): + return '{external_ip}{has_ext_ip}{published}{is_pub}{target}/{protocol}'.format( + published=port.get('published', ''), + is_pub=(':' if port.get('published') else ''), + target=port.get('target'), + protocol=port.get('protocol', 'tcp'), + external_ip=port.get('external_ip', ''), + has_ext_ip=(':' if port.get('external_ip') else ''), + ) diff --git a/compose/service.py b/compose/service.py index 0f0dc57dd0..ef593d84d9 100644 --- a/compose/service.py +++ b/compose/service.py @@ -21,6 +21,7 @@ from . import progress_stream from .config import DOCKER_CONFIG_KEYS from .config import merge_environment +from .config.types import ServicePort from .config.types import VolumeSpec from .const import DEFAULT_TIMEOUT from .const import IS_WINDOWS_PLATFORM @@ -693,7 +694,7 @@ def _get_container_create_options( if 'ports' in container_options or 'expose' in self.options: container_options['ports'] = build_container_ports( - container_options, + formatted_ports(container_options.get('ports', [])), self.options) container_options['environment'] = merge_environment( @@ -747,7 +748,9 @@ def _get_container_host_config(self, override_options, one_off=False): host_config = self.client.create_host_config( links=self._get_links(link_to_self=one_off), - port_bindings=build_port_bindings(options.get('ports') or []), + port_bindings=build_port_bindings( + formatted_ports(options.get('ports', [])) + ), binds=options.get('binds'), volumes_from=self._get_volumes_from(), privileged=options.get('privileged', False), @@ -875,7 +878,10 @@ def remove_image(self, image_type): def specifies_host_port(self): def has_host_port(binding): - _, external_bindings = split_port(binding) + if isinstance(binding, dict): + external_bindings = binding.get('published') + else: + _, external_bindings = split_port(binding) # there are no external bindings if external_bindings is None: @@ -1214,12 +1220,21 @@ def format_env(key, value): return '{key}={value}'.format(key=key, value=value) return [format_env(*item) for item in environment.items()] + # Ports +def formatted_ports(ports): + result = [] + for port in ports: + if isinstance(port, ServicePort): + result.append(port.legacy_repr()) + else: + result.append(port) + return result -def build_container_ports(container_options, options): +def build_container_ports(container_ports, options): ports = [] - all_ports = container_options.get('ports', []) + options.get('expose', []) + all_ports = container_ports + options.get('expose', []) for port_range in all_ports: internal_range, _ = split_port(port_range) for port in internal_range: diff --git a/tests/acceptance/cli_test.py b/tests/acceptance/cli_test.py index 8366ca75e2..c1605aa766 100644 --- a/tests/acceptance/cli_test.py +++ b/tests/acceptance/cli_test.py @@ -1759,6 +1759,19 @@ def get_port(number): self.assertEqual(get_port(3001), "0.0.0.0:49152") self.assertEqual(get_port(3002), "0.0.0.0:49153") + def test_expanded_port(self): + self.base_dir = 'tests/fixtures/ports-composefile' + self.dispatch(['-f', 'expanded-notation.yml', 'up', '-d']) + container = self.project.get_service('simple').get_container() + + def get_port(number): + result = self.dispatch(['port', 'simple', str(number)]) + return result.stdout.rstrip() + + self.assertEqual(get_port(3000), container.get_local_port(3000)) + self.assertEqual(get_port(3001), "0.0.0.0:49152") + self.assertEqual(get_port(3002), "0.0.0.0:49153") + def test_port_with_scale(self): self.base_dir = 'tests/fixtures/ports-composefile-scale' self.dispatch(['scale', 'simple=2'], None) diff --git a/tests/fixtures/ports-composefile/expanded-notation.yml b/tests/fixtures/ports-composefile/expanded-notation.yml new file mode 100644 index 0000000000..46d587363f --- /dev/null +++ b/tests/fixtures/ports-composefile/expanded-notation.yml @@ -0,0 +1,15 @@ +version: '3.1' +services: + simple: + image: busybox:latest + command: top + ports: + - target: 3000 + - target: 3001 + published: 49152 + - target: 3002 + published: 49153 + protocol: tcp + - target: 3003 + published: 49154 + protocol: udp diff --git a/tests/unit/config/config_test.py b/tests/unit/config/config_test.py index e5104f7692..4348129c8f 100644 --- a/tests/unit/config/config_test.py +++ b/tests/unit/config/config_test.py @@ -10,6 +10,7 @@ import py import pytest +import yaml from ...helpers import build_config_details from compose.config import config @@ -25,6 +26,7 @@ from compose.config.errors import ConfigurationError from compose.config.errors import VERSION_EXPLANATION from compose.config.serialize import denormalize_service_dict +from compose.config.serialize import serialize_config from compose.config.serialize import serialize_ns_time_value from compose.config.types import VolumeSpec from compose.const import IS_WINDOWS_PLATFORM @@ -1737,6 +1739,30 @@ def test_merge_logging_v2_no_override(self): } } + def test_merge_mixed_ports(self): + base = { + 'image': 'busybox:latest', + 'command': 'top', + 'ports': [ + { + 'target': '1245', + 'published': '1245', + 'protocol': 'tcp', + } + ] + } + + override = { + 'ports': ['1245:1245/udp'] + } + + actual = config.merge_service_dicts(base, override, V3_1) + assert actual == { + 'image': 'busybox:latest', + 'command': 'top', + 'ports': [types.ServicePort('1245', '1245', 'udp', None, None)] + } + def test_merge_depends_on_no_override(self): base = { 'image': 'busybox', @@ -2210,7 +2236,10 @@ def test_config_file_with_environment_file(self): self.assertEqual(service_dicts[0], { 'name': 'web', 'image': 'alpine:latest', - 'ports': ['5643', '9999'], + 'ports': [ + types.ServicePort.parse('5643')[0], + types.ServicePort.parse('9999')[0] + ], 'command': 'true' }) @@ -2233,7 +2262,7 @@ def test_config_file_with_environment_variable(self): { 'name': 'web', 'image': 'busybox', - 'ports': ['80:8000'], + 'ports': types.ServicePort.parse('80:8000'), 'labels': {'mylabel': 'myvalue'}, 'hostname': 'host-', 'command': '${ESCAPED}', @@ -2515,13 +2544,37 @@ class MergePortsTest(unittest.TestCase, MergeListsTest): base_config = ['10:8000', '9000'] override_config = ['20:8000'] + def merged_config(self): + return self.convert(self.base_config) | self.convert(self.override_config) + + def convert(self, port_config): + return set(config.merge_service_dicts( + {self.config_name: port_config}, + {self.config_name: []}, + DEFAULT_VERSION + )[self.config_name]) + def test_duplicate_port_mappings(self): service_dict = config.merge_service_dicts( {self.config_name: self.base_config}, {self.config_name: self.base_config}, DEFAULT_VERSION ) - assert set(service_dict[self.config_name]) == set(self.base_config) + assert set(service_dict[self.config_name]) == self.convert(self.base_config) + + def test_no_override(self): + service_dict = config.merge_service_dicts( + {self.config_name: self.base_config}, + {}, + DEFAULT_VERSION) + assert set(service_dict[self.config_name]) == self.convert(self.base_config) + + def test_no_base(self): + service_dict = config.merge_service_dicts( + {}, + {self.config_name: self.base_config}, + DEFAULT_VERSION) + assert set(service_dict[self.config_name]) == self.convert(self.base_config) class MergeNetworksTest(unittest.TestCase, MergeListsTest): @@ -3542,23 +3595,25 @@ def test_denormalize_healthcheck(self): assert denormalized_service['healthcheck']['interval'] == '100s' assert denormalized_service['healthcheck']['timeout'] == '30s' - def test_denormalize_secrets(self): + def test_serialize_secrets(self): service_dict = { - 'name': 'web', 'image': 'example/web', 'secrets': [ - types.ServiceSecret('one', None, None, None, None), - types.ServiceSecret('source', 'target', '100', '200', 0o777), - ], + {'source': 'one'}, + { + 'source': 'source', + 'target': 'target', + 'uid': '100', + 'gid': '200', + 'mode': 0o777, + } + ] } - denormalized_service = denormalize_service_dict(service_dict, V3_1) - assert secret_sort(denormalized_service['secrets']) == secret_sort([ - {'source': 'one'}, - { - 'source': 'source', - 'target': 'target', - 'uid': '100', - 'gid': '200', - 'mode': 0o777, - }, - ]) + config_dict = config.load(build_config_details({ + 'version': '3.1', + 'services': {'web': service_dict} + })) + + serialized_config = yaml.load(serialize_config(config_dict)) + serialized_service = serialized_config['services']['web'] + assert secret_sort(serialized_service['secrets']) == secret_sort(service_dict['secrets']) diff --git a/tests/unit/config/types_test.py b/tests/unit/config/types_test.py index 114273520e..22d7aa88a6 100644 --- a/tests/unit/config/types_test.py +++ b/tests/unit/config/types_test.py @@ -7,6 +7,7 @@ from compose.config.config import V2_0 from compose.config.errors import ConfigurationError from compose.config.types import parse_extra_hosts +from compose.config.types import ServicePort from compose.config.types import VolumeFromSpec from compose.config.types import VolumeSpec @@ -41,6 +42,49 @@ def test_parse_extra_hosts_dict(): } +class TestServicePort(object): + def test_parse_dict(self): + data = { + 'target': 8000, + 'published': 8000, + 'protocol': 'udp', + 'mode': 'global', + } + ports = ServicePort.parse(data) + assert len(ports) == 1 + assert ports[0].repr() == data + + def test_parse_simple_target_port(self): + ports = ServicePort.parse(8000) + assert len(ports) == 1 + assert ports[0].target == '8000' + + def test_parse_complete_port_definition(self): + port_def = '1.1.1.1:3000:3000/udp' + ports = ServicePort.parse(port_def) + assert len(ports) == 1 + assert ports[0].repr() == { + 'target': '3000', + 'published': '3000', + 'external_ip': '1.1.1.1', + 'protocol': 'udp', + } + assert ports[0].legacy_repr() == port_def + + def test_parse_port_range(self): + ports = ServicePort.parse('25000-25001:4000-4001') + assert len(ports) == 2 + reprs = [p.repr() for p in ports] + assert { + 'target': '4000', + 'published': '25000' + } in reprs + assert { + 'target': '4001', + 'published': '25001' + } in reprs + + class TestVolumeSpec(object): def test_parse_volume_spec_only_one_path(self): diff --git a/tests/unit/service_test.py b/tests/unit/service_test.py index 0a66e4f3e2..4d81623baf 100644 --- a/tests/unit/service_test.py +++ b/tests/unit/service_test.py @@ -7,6 +7,7 @@ from .. import mock from .. import unittest +from compose.config.types import ServicePort from compose.config.types import VolumeFromSpec from compose.config.types import VolumeSpec from compose.const import LABEL_CONFIG_HASH @@ -19,6 +20,7 @@ from compose.service import build_volume_binding from compose.service import BuildAction from compose.service import ContainerNetworkMode +from compose.service import formatted_ports from compose.service import get_container_data_volumes from compose.service import ImageType from compose.service import merge_volume_bindings @@ -778,6 +780,25 @@ def test_network_mode_service_no_containers(self): self.assertEqual(network_mode.service_name, service_name) +class ServicePortsTest(unittest.TestCase): + def test_formatted_ports(self): + ports = [ + '3000', + '0.0.0.0:4025-4030:23000-23005', + ServicePort(6000, None, None, None, None), + ServicePort(8080, 8080, None, None, None), + ServicePort('20000', '20000', 'udp', 'ingress', None), + ServicePort(30000, '30000', 'tcp', None, '127.0.0.1'), + ] + formatted = formatted_ports(ports) + assert ports[0] in formatted + assert ports[1] in formatted + assert '6000/tcp' in formatted + assert '8080:8080/tcp' in formatted + assert '20000:20000/udp' in formatted + assert '127.0.0.1:30000:30000/tcp' in formatted + + def build_mount(destination, source, mode='rw'): return {'Source': source, 'Destination': destination, 'Mode': mode}