Merge remote-tracking branch 'origin2/master' into composemaster

Signed-off-by: Yuval Kohavi <yuval.kohavi@gmail.com>

Conflicts:
	tests/unit/service_test.py

Author: yuval-k

compose/__init__.py

@@ -1,4 +1,3 @@
 from __future__ import unicode_literals
-from .service import Service  # noqa:flake8
 
 __version__ = '1.3.0dev'

compose/cli/main.py

@@ -11,8 +11,9 @@
 import dockerpty
 
 from .. import __version__
+from .. import migration
 from ..project import NoSuchService, ConfigurationError
-from ..service import BuildError, CannotBeScaledError
+from ..service import BuildError, CannotBeScaledError, NeedsBuildError
 from ..config import parse_environment
 from .command import Command
 from .docopt_command import NoSuchCommand
@@ -46,6 +47,9 @@ def main():
     except BuildError as e:
         log.error("Service '%s' failed to build: %s" % (e.service.name, e.reason))
         sys.exit(1)
+    except NeedsBuildError as e:
+        log.error("Service '%s' needs to be built, but --no-build was passed." % e.service.name)
+        sys.exit(1)
 
 
 def setup_logging():
@@ -81,20 +85,21 @@ class TopLevelCommand(Command):
       -v, --version             Print version and exit
 
     Commands:
-      build     Build or rebuild services
-      help      Get help on a command
-      kill      Kill containers
-      logs      View output from containers
-      port      Print the public port for a port binding
-      ps        List containers
-      pull      Pulls service images
-      restart   Restart services
-      rm        Remove stopped containers
-      run       Run a one-off command
-      scale     Set number of containers for a service
-      start     Start services
-      stop      Stop services
-      up        Create and start containers
+      build              Build or rebuild services
+      help               Get help on a command
+      kill               Kill containers
+      logs               View output from containers
+      port               Print the public port for a port binding
+      ps                 List containers
+      pull               Pulls service images
+      restart            Restart services
+      rm                 Remove stopped containers
+      run                Run a one-off command
+      scale              Set number of containers for a service
+      start              Start services
+      stop               Stop services
+      up                 Create and start containers
+      migrate_to_labels  Recreate containers to add labels
 
     """
     def docopt_options(self):
@@ -295,9 +300,8 @@ def run(self, project, options):
                 project.up(
                     service_names=deps,
                     start_deps=True,
-                    recreate=False,
+                    allow_recreate=False,
                     insecure_registry=insecure_registry,
-                    detach=options['-d']
                 )
 
         tty = True
@@ -341,7 +345,6 @@ def run(self, project, options):
             service.start_container(container)
             print(container.name)
         else:
-            service.start_container(container)
             dockerpty.start(project.client, container.id, interactive=not options['-T'])
             exit_code = container.wait()
             if options['--rm']:
@@ -440,6 +443,8 @@ def up(self, project, options):
                                    print new container names.
             --no-color             Produce monochrome output.
             --no-deps              Don't start linked services.
+            --x-smart-recreate     Only recreate containers whose configuration or
+                                   image needs to be updated. (EXPERIMENTAL)
             --no-recreate          If containers already exist, don't recreate them.
             --no-build             Don't build an image, even if it's missing
             -t, --timeout TIMEOUT  When attached, use this timeout in seconds
@@ -452,15 +457,16 @@ def up(self, project, options):
         monochrome = options['--no-color']
 
         start_deps = not options['--no-deps']
-        recreate = not options['--no-recreate']
+        allow_recreate = not options['--no-recreate']
+        smart_recreate = options['--x-smart-recreate']
         service_names = options['SERVICE']
 
         project.up(
             service_names=service_names,
             start_deps=start_deps,
-            recreate=recreate,
+            allow_recreate=allow_recreate,
+            smart_recreate=smart_recreate,
             insecure_registry=insecure_registry,
-            detach=detached,
             do_build=not options['--no-build'],
         )
 
@@ -483,6 +489,9 @@ def handler(signal, frame):
                 params = {} if timeout is None else {'timeout': int(timeout)}
                 project.stop(service_names=service_names, **params)
 
+    def migrate_to_labels(self, project, _options):
+        migration.migrate_project_to_labels(project)
+
 
 def list_containers(containers):
     return ", ".join(c.name for c in containers)

compose/const.py

@@ -0,0 +1,7 @@
+
+LABEL_CONTAINER_NUMBER = 'com.docker.compose.container-number'
+LABEL_ONE_OFF = 'com.docker.compose.oneoff'
+LABEL_PROJECT = 'com.docker.compose.project'
+LABEL_SERVICE = 'com.docker.compose.service'
+LABEL_VERSION = 'com.docker.compose.version'
+LABEL_CONFIG_HASH = 'com.docker.compose.config-hash'

compose/container.py

@@ -4,6 +4,8 @@
 import six
 from functools import reduce
 
+from .const import LABEL_CONTAINER_NUMBER, LABEL_SERVICE
+
 
 class Container(object):
     """
@@ -58,14 +60,15 @@ def name(self):
 
     @property
     def name_without_project(self):
-        return '_'.join(self.dictionary['Name'].split('_')[1:])
+        return '{0}_{1}'.format(self.labels.get(LABEL_SERVICE), self.number)
 
     @property
     def number(self):
-        try:
-            return int(self.name.split('_')[-1])
-        except ValueError:
-            return None
+        number = self.labels.get(LABEL_CONTAINER_NUMBER)
+        if not number:
+            raise ValueError("Container {0} does not have a {1} label".format(
+                self.short_id, LABEL_CONTAINER_NUMBER))
+        return int(number)
 
     @property
     def ports(self):
@@ -159,6 +162,7 @@ def inspect(self):
         self.has_been_inspected = True
         return self.dictionary
 
+    # TODO: only used by tests, move to test module
     def links(self):
         links = []
         for container in self.client.containers():
@@ -175,13 +179,16 @@ def attach_socket(self, **kwargs):
         return self.client.attach_socket(self.id, **kwargs)
 
     def __repr__(self):
-        return '<Container: %s>' % self.name
+        return '<Container: %s (%s)>' % (self.name, self.id[:6])
 
     def __eq__(self, other):
         if type(self) != type(other):
             return False
         return self.id == other.id
 
+    def __hash__(self):
+        return self.id.__hash__()
+
 
 def get_container_name(container):
     if not container.get('Name') and not container.get('Names'):

compose/migration.py

@@ -0,0 +1,35 @@
+import logging
+import re
+
+from .container import get_container_name, Container
+
+
+log = logging.getLogger(__name__)
+
+
+# TODO: remove this section when migrate_project_to_labels is removed
+NAME_RE = re.compile(r'^([^_]+)_([^_]+)_(run_)?(\d+)$')
+
+
+def is_valid_name(name):
+    match = NAME_RE.match(name)
+    return match is not None
+
+
+def add_labels(project, container, name):
+    project_name, service_name, one_off, number = NAME_RE.match(name).groups()
+    if project_name != project.name or service_name not in project.service_names:
+        return
+    service = project.get_service(service_name)
+    service.recreate_container(container)
+
+
+def migrate_project_to_labels(project):
+    log.info("Running migration to labels for project %s", project.name)
+
+    client = project.client
+    for container in client.containers(all=True):
+        name = get_container_name(container)
+        if not is_valid_name(name):
+            continue
+        add_labels(project, Container.from_ps(client, container), name)

compose/project.py

@@ -1,12 +1,14 @@
 from __future__ import unicode_literals
 from __future__ import absolute_import
 import logging
-
 from functools import reduce
+
+from docker.errors import APIError
+
 from .config import get_service_name_from_net, ConfigurationError
-from .service import Service
+from .const import LABEL_PROJECT, LABEL_ONE_OFF
+from .service import Service, check_for_legacy_containers
 from .container import Container
-from docker.errors import APIError
 
 log = logging.getLogger(__name__)
 
@@ -60,6 +62,12 @@ def __init__(self, name, services, client):
         self.services = services
         self.client = client
 
+    def labels(self, one_off=False):
+        return [
+            '{0}={1}'.format(LABEL_PROJECT, self.name),
+            '{0}={1}'.format(LABEL_ONE_OFF, "True" if one_off else "False"),
+        ]
+
     @classmethod
     def from_dicts(cls, name, service_dicts, client):
         """
@@ -75,6 +83,10 @@ def from_dicts(cls, name, service_dicts, client):
                                             volumes_from=volumes_from, **service_dict))
         return project
 
+    @property
+    def service_names(self):
+        return [service.name for service in self.services]
+
     def get_service(self, name):
         """
         Retrieve a service by name. Raises NoSuchService
@@ -102,7 +114,7 @@ def get_services(self, service_names=None, include_deps=False):
         """
         if service_names is None or len(service_names) == 0:
             return self.get_services(
-                service_names=[s.name for s in self.services],
+                service_names=self.service_names,
                 include_deps=include_deps
             )
         else:
@@ -195,24 +207,59 @@ def build(self, service_names=None, no_cache=False):
     def up(self,
            service_names=None,
            start_deps=True,
-           recreate=True,
+           allow_recreate=True,
+           smart_recreate=False,
            insecure_registry=False,
-           detach=False,
            do_build=True):
-        running_containers = []
-        for service in self.get_services(service_names, include_deps=start_deps):
-            if recreate:
-                create_func = service.recreate_containers
+
+        services = self.get_services(service_names, include_deps=start_deps)
+
+        plans = self._get_convergence_plans(
+            services,
+            allow_recreate=allow_recreate,
+            smart_recreate=smart_recreate,
+        )
+
+        return [
+            container
+            for service in services
+            for container in service.execute_convergence_plan(
+                plans[service.name],
+                insecure_registry=insecure_registry,
+                do_build=do_build,
+            )
+        ]
+
+    def _get_convergence_plans(self,
+                               services,
+                               allow_recreate=True,
+                               smart_recreate=False):
+
+        plans = {}
+
+        for service in services:
+            updated_dependencies = [
+                name
+                for name in service.get_dependency_names()
+                if name in plans
+                and plans[name].action == 'recreate'
+            ]
+
+            if updated_dependencies:
+                log.debug(
+                    '%s has not changed but its dependencies (%s) have, so recreating',
+                    service.name, ", ".join(updated_dependencies),
+                )
+                plan = service.recreate_plan()
             else:
-                create_func = service.start_or_create_containers
+                plan = service.convergence_plan(
+                    allow_recreate=allow_recreate,
+                    smart_recreate=smart_recreate,
+                )
 
-            for container in create_func(
-                    insecure_registry=insecure_registry,
-                    detach=detach,
-                    do_build=do_build):
-                running_containers.append(container)
+            plans[service.name] = plan
 
-        return running_containers
+        return plans
 
     def pull(self, service_names=None, insecure_registry=False):
         for service in self.get_services(service_names, include_deps=True):
@@ -223,16 +270,24 @@ def remove_stopped(self, service_names=None, **options):
             service.remove_stopped(**options)
 
     def containers(self, service_names=None, stopped=False, one_off=False):
-        return [Container.from_ps(self.client, container)
-                for container in self.client.containers(all=stopped)
-                for service in self.get_services(service_names)
-                if service.has_container(container, one_off=one_off)]
+        containers = [
+            Container.from_ps(self.client, container)
+            for container in self.client.containers(
+                all=stopped,
+                filters={'label': self.labels(one_off=one_off)})]
+
+        if not containers:
+            check_for_legacy_containers(
+                self.client,
+                self.name,
+                self.service_names,
+                stopped=stopped,
+                one_off=one_off)
+
+        return containers
 
     def _inject_deps(self, acc, service):
-        net_name = service.get_net_name()
-        dep_names = (service.get_linked_names() +
-                     service.get_volumes_from_names() +
-                     ([net_name] if net_name else []))
+        dep_names = service.get_dependency_names()
 
         if len(dep_names) > 0:
             dep_services = self.get_services(