From 96d9725d7db399ccb11a11df67cf52e5feb25c36 Mon Sep 17 00:00:00 2001 From: Adam Wierzbicki Date: Tue, 2 Jul 2019 16:39:12 +0200 Subject: [PATCH 1/3] Port mapping for docker machine hypervisors --- golem/docker/commands/docker_machine.py | 1 + golem/docker/hypervisor/docker_machine.py | 9 ++++++++- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/golem/docker/commands/docker_machine.py b/golem/docker/commands/docker_machine.py index d871b86bdc..d817348ecf 100644 --- a/golem/docker/commands/docker_machine.py +++ b/golem/docker/commands/docker_machine.py @@ -18,6 +18,7 @@ class DockerMachineCommandHandler(DockerCommandHandler): 'docker-machine', '--native-ssh', 'regenerate-certs', '--force'], restart=['docker-machine', '--native-ssh', 'restart'], execute=['docker-machine', '--native-ssh', 'ssh'], + ip=['docker-machine', 'ip'], ) commands.update(DockerCommandHandler.commands) diff --git a/golem/docker/hypervisor/docker_machine.py b/golem/docker/hypervisor/docker_machine.py index 105f6dccc1..03a4d78d52 100644 --- a/golem/docker/hypervisor/docker_machine.py +++ b/golem/docker/hypervisor/docker_machine.py @@ -5,6 +5,7 @@ from contextlib import contextmanager from typing import Optional, ClassVar, Any, List, Tuple +from golem.docker.client import local_client from golem.docker.commands.docker_machine import DockerMachineCommandHandler from golem.docker.config import DOCKER_VM_NAME, GetConfigFunction, \ CONSTRAINT_KEYS @@ -92,7 +93,13 @@ def requires_ports_publishing(self) -> bool: return True def get_port_mapping(self, container_id: str, port: int) -> Tuple[str, int]: - pass + api_client = local_client() + c_config = api_client.inspect_container(container_id) + port = int( + c_config['NetworkSettings']['Ports'][f'{port}/tcp'][0]['HostPort']) + ip = self.command('ip', self._vm_name) + assert isinstance(ip, str) + return ip, port @property def config_dir(self): From fb625cc0707f87359251eb3195b7e9e1f6b94cfb Mon Sep 17 00:00:00 2001 From: Adam Wierzbicki Date: Thu, 4 Jul 2019 13:25:48 +0200 Subject: [PATCH 2/3] Unit tests for get_port_mapping --- tests/golem/docker/test_hypervisor.py | 60 ++++++++++++++++++++++++++- 1 file changed, 58 insertions(+), 2 deletions(-) diff --git a/tests/golem/docker/test_hypervisor.py b/tests/golem/docker/test_hypervisor.py index 7587bae57c..3686503da4 100644 --- a/tests/golem/docker/test_hypervisor.py +++ b/tests/golem/docker/test_hypervisor.py @@ -1,17 +1,17 @@ import functools import json import os -import types import uuid from contextlib import contextmanager from subprocess import CalledProcessError from typing import Optional, Dict -from unittest import mock +from unittest import mock, TestCase from golem.docker.commands.docker_machine import DockerMachineCommandHandler from golem.docker.config import DOCKER_VM_NAME as VM_NAME, DEFAULTS from golem.docker.hypervisor.docker_for_mac import DockerForMac from golem.docker.hypervisor.docker_machine import DockerMachineHypervisor +from golem.docker.hypervisor.dummy import DummyHypervisor from golem.docker.hypervisor.virtualbox import VirtualBoxHypervisor from golem.docker.manager import DockerManager from golem.testutils import TempDirFixture @@ -167,6 +167,25 @@ def test_remove(self): with self.assertLogs(level='WARN'): assert not hypervisor.remove('test') + @mock.patch('golem.docker.hypervisor.docker_machine.local_client') + def test_get_port_mapping(self, local_client): + local_client().inspect_container.return_value = { + 'NetworkSettings': { + 'Ports': { + '12345/tcp': [{ + 'HostIp': '0.0.0.0', + 'HostPort': '54321' + }] + } + } + } + hypervisor = MockHypervisor() + vm_ip = '192.168.64.151' + with mock.patch.object(hypervisor, 'command', return_value=vm_ip): + host, port = hypervisor.get_port_mapping('container_id', 12345) + self.assertEqual(host, vm_ip) + self.assertEqual(port, 54321) + class TestVirtualBoxHypervisor(LogTestCase): @@ -442,3 +461,40 @@ def _assert_dns_configured(config_file) -> Dict: assert all(daemon_config['dns']) return daemon_config + + @mock.patch('golem.docker.hypervisor.docker_for_mac.local_client') + def test_get_port_mapping(self, local_client): + local_client().inspect_container.return_value = { + 'NetworkSettings': { + 'Ports': { + '12345/tcp': [{ + 'HostIp': '0.0.0.0', + 'HostPort': '54321' + }] + } + } + } + hypervisor = DockerForMac.instance(mock.Mock()) + host, port = hypervisor.get_port_mapping('container_id', 12345) + self.assertEqual(host, '127.0.0.1') + self.assertEqual(port, 54321) + + +class TestDummyHypervisor(TestCase): + + @mock.patch('golem.docker.hypervisor.dummy.local_client') + def test_get_port_mapping(self, local_client): + container_ip = '172.17.0.2' + local_client().inspect_container.return_value = { + 'NetworkSettings': { + 'Networks': { + 'bridge': { + 'IPAddress': container_ip, + } + } + } + } + hypervisor = DummyHypervisor(mock.Mock()) + host, port = hypervisor.get_port_mapping('container_id', 12345) + self.assertEqual(host, container_ip) + self.assertEqual(port, 12345) From 45cd5247cb9f56c42b1cac7768aef615b3f252bf Mon Sep 17 00:00:00 2001 From: Adam Wierzbicki Date: Thu, 4 Jul 2019 13:26:06 +0200 Subject: [PATCH 3/3] Don't defer docker whitelist check to another thread This check is not time-consuming so it can be performed in the calling thread instead of being deferred to another thread. Actually, this is a workaround for integration tests failing on Windows. For some reason if database was used in another thread (as it happened before this change) the unit test cleanup would fail because "The golem.db is used by another process and cannot be removed". --- golem/envs/docker/cpu.py | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/golem/envs/docker/cpu.py b/golem/envs/docker/cpu.py index c614c8b4a4..45e392acfe 100644 --- a/golem/envs/docker/cpu.py +++ b/golem/envs/docker/cpu.py @@ -8,7 +8,7 @@ NamedTuple, Tuple, Iterator, Union, Iterable from docker.errors import APIError -from twisted.internet.defer import Deferred, inlineCallbacks +from twisted.internet.defer import Deferred, inlineCallbacks, succeed from twisted.internet.threads import deferToThread from urllib3.contrib.pyopenssl import WrappedSocket @@ -580,13 +580,14 @@ def install_prerequisites(self, prerequisites: Prerequisites) -> Deferred: f"is in invalid state: '{self._status}'") logger.info("Preparing prerequisites...") + if not Whitelist.is_whitelisted(prerequisites.image): + logger.info( + "Docker image '%s' is not whitelisted.", + prerequisites.image, + ) + return succeed(False) + def _prepare(): - if not Whitelist.is_whitelisted(prerequisites.image): - logger.info( - "Docker image '%s' is not whitelisted.", - prerequisites.image, - ) - return False try: client = local_client() client.pull(