From 02eb9b6989b86c0fc6bd18cf2f5d8796d48e735d Mon Sep 17 00:00:00 2001 From: Greg Gulrajani Date: Fri, 17 Jun 2016 21:53:28 +0100 Subject: [PATCH] docker container fetch limit is now configurable Signed-off-by: Gregory Gulrajani --- doc/manual/external-configuration.md | 5 ++-- doc/manual/global-configuration.md | 2 ++ .../maven/docker/AbstractDockerMojo.java | 11 ++++++-- .../maven/docker/access/UrlBuilder.java | 10 +++++-- .../access/hc/DockerAccessWithHcClient.java | 12 ++++---- .../docker/config/RunImageConfiguration.java | 23 +++++++++++---- .../config/handler/property/ConfigKey.java | 5 ++-- .../property/PropertyConfigHandler.java | 28 +++++++++++-------- .../maven/docker/service/QueryService.java | 17 +++++------ .../maven/docker/service/ServiceHub.java | 4 +-- .../docker/service/ServiceHubFactory.java | 6 ++-- 11 files changed, 78 insertions(+), 45 deletions(-) diff --git a/doc/manual/external-configuration.md b/doc/manual/external-configuration.md index 8f9b4c618..821cc0f5a 100644 --- a/doc/manual/external-configuration.md +++ b/doc/manual/external-configuration.md @@ -36,8 +36,8 @@ Example: ``` -Given this example configuration a single image configuration is build -up from the following properties, which correspond to corresponding +Given this example configuration a single image configuration is built +up from the following properties, which correspond to the corresponding values in the `` and `` sections. * **docker.alias** Alias name @@ -83,6 +83,7 @@ used to create the image. The dockerfile must be name `Dockerfile` used as environment variables. The environment variables takes precedence over any other environment variables specified. * **docker.extraHosts.idx** List of `host:ip` to add to `/etc/hosts` +* **docker.fetchLimit** Number of running and stopped containers to return. A value of 0 returns all running and stopped containers. Default is 100. * **docker.from** Base image for building an image * **docker.hostname** Container hostname * **docker.labels.LABEL** Sets a label which works similarly like setting environment variables. diff --git a/doc/manual/global-configuration.md b/doc/manual/global-configuration.md index 097ab0c95..211b53554 100644 --- a/doc/manual/global-configuration.md +++ b/doc/manual/global-configuration.md @@ -47,6 +47,8 @@ parentheses. * **skip** (`docker.skip`) With this parameter the execution of this plugin can be skipped completely. +* **fetchLimit** (`docker.fetchLimit`) + Number of running and stopped containers to return. A value of 0 returns all running and stopped containers. Default is 100. * **skipTags** (`docker.skipTags`) If set to `true` the plugin won't add any tags to images that have been built. * **registry** (`docker.registry`) diff --git a/src/main/java/io/fabric8/maven/docker/AbstractDockerMojo.java b/src/main/java/io/fabric8/maven/docker/AbstractDockerMojo.java index f0f349570..c193e5ca5 100644 --- a/src/main/java/io/fabric8/maven/docker/AbstractDockerMojo.java +++ b/src/main/java/io/fabric8/maven/docker/AbstractDockerMojo.java @@ -120,6 +120,10 @@ public abstract class AbstractDockerMojo extends AbstractMojo implements Context @Parameter(property = "docker.skip", defaultValue = "false") private boolean skip; + // Max number of containers to fetch when stopping/removing + @Parameter(property = "docker.fetchLimit", defaultValue = "100") + protected int fetchLimit; + /** * Whether to restrict operation to a single image. This can be either * the image or an alias name. It can also be comma separated list. @@ -139,7 +143,7 @@ public abstract class AbstractDockerMojo extends AbstractMojo implements Context // property file to write out with port mappings @Parameter protected String portPropertyFile; - + // Authentication information @Parameter Map authConfig; @@ -178,7 +182,9 @@ public void execute() throws MojoExecutionException, MojoFailureException { DockerAccess access = null; try { access = createDockerAccess(minimalApiVersion); - ServiceHub serviceHub = serviceHubFactory.createServiceHub(project, session, access, log, logSpecFactory); + fetchLimit = resolvedImages.get(0).getRunConfiguration().getFetchLimit() != null + ? resolvedImages.get(0).getRunConfiguration().getFetchLimit() : fetchLimit; + ServiceHub serviceHub = serviceHubFactory.createServiceHub(project, session, access, log, logSpecFactory, fetchLimit); executeInternal(serviceHub); } catch (DockerAccessException exp) { log.error("%s", exp.getMessage()); @@ -225,6 +231,7 @@ public List resolve(ImageConfiguration image) { // Customization hook for subclasses to influence the final configuration. This method is called // before initialization and validation of the configuration. + @Override public List customizeConfig(List imageConfigs) { return imageConfigs; } diff --git a/src/main/java/io/fabric8/maven/docker/access/UrlBuilder.java b/src/main/java/io/fabric8/maven/docker/access/UrlBuilder.java index 619e5c801..806e89612 100644 --- a/src/main/java/io/fabric8/maven/docker/access/UrlBuilder.java +++ b/src/main/java/io/fabric8/maven/docker/access/UrlBuilder.java @@ -53,7 +53,7 @@ public String containerLogs(String containerId, boolean follow) { .p("follow", follow) .build(); } - + public String createContainer(String name) { return u("containers/create") .p("name", name) @@ -70,13 +70,19 @@ public String inspectContainer(String containerId) { return u("containers/%s/json", containerId) .build(); } - + public String listContainers(int limit) { return u("containers/json") .p("limit", limit) .build(); } + public String listAllContainers() { + return u("containers/json") + .p("all", 0) + .build(); + } + public String pullImage(ImageName name, String registry) { return u("images/create") .p("fromImage", name.getNameWithoutTag(registry)) diff --git a/src/main/java/io/fabric8/maven/docker/access/hc/DockerAccessWithHcClient.java b/src/main/java/io/fabric8/maven/docker/access/hc/DockerAccessWithHcClient.java index 99314225c..b5e064767 100644 --- a/src/main/java/io/fabric8/maven/docker/access/hc/DockerAccessWithHcClient.java +++ b/src/main/java/io/fabric8/maven/docker/access/hc/DockerAccessWithHcClient.java @@ -188,7 +188,7 @@ public void stopContainer(String containerId, int killWait) throws DockerAccessE @Override public void buildImage(String image, File dockerArchive, String dockerfileName, boolean forceRemove, boolean noCache, - Map buildArgs) throws DockerAccessException { + Map buildArgs) throws DockerAccessException { try { String url = urlBuilder.buildImage(image, dockerfileName, forceRemove, noCache, buildArgs); delegate.post(url, dockerArchive, createBuildResponseHandler(), HTTP_OK); @@ -198,7 +198,7 @@ public void buildImage(String image, File dockerArchive, String dockerfileName, } @Override - public void copyArchive(String containerId,File archive, String targetPath) + public void copyArchive(String containerId, File archive, String targetPath) throws DockerAccessException { try { String url = urlBuilder.copyArchive(containerId, targetPath); @@ -235,7 +235,7 @@ public Container inspectContainer(String containerId) throws DockerAccessExcepti @Override public List listContainers(int limit) throws DockerAccessException { - String url = urlBuilder.listContainers(limit); + String url = limit == 0 ? urlBuilder.listAllContainers() : urlBuilder.listContainers(limit); try { String response = delegate.get(url, HTTP_OK); @@ -300,7 +300,7 @@ public void pullImage(String image, AuthConfig authConfig, String registry) try { delegate.post(pullUrl, null, createAuthHeader(authConfig), - createPullOrPushResponseHandler(), HTTP_OK); + createPullOrPushResponseHandler(), HTTP_OK); } catch (IOException e) { throw new DockerAccessException(e, "Unable to pull '%s'%s", image, (registry != null) ? " from registry '" + registry + "'" : ""); } @@ -314,7 +314,7 @@ public void pushImage(String image, AuthConfig authConfig, String registry) String temporaryImage = tagTemporaryImage(name, registry); try { delegate.post(pushUrl, null, createAuthHeader(authConfig), - createPullOrPushResponseHandler(), HTTP_OK); + createPullOrPushResponseHandler(), HTTP_OK); } catch (IOException e) { throw new DockerAccessException(e, "Unable to push '%s'%s", image, (registry != null) ? " from registry '" + registry + "'" : ""); } finally { @@ -334,7 +334,7 @@ public void tag(String sourceImage, String targetImage, boolean force) delegate.post(url, HTTP_CREATED); } catch (IOException e) { throw new DockerAccessException(e, "Unable to add tag [%s] to image [%s]", targetImage, - sourceImage, e); + sourceImage, e); } } diff --git a/src/main/java/io/fabric8/maven/docker/config/RunImageConfiguration.java b/src/main/java/io/fabric8/maven/docker/config/RunImageConfiguration.java index 0676fbf6a..d73a539f0 100644 --- a/src/main/java/io/fabric8/maven/docker/config/RunImageConfiguration.java +++ b/src/main/java/io/fabric8/maven/docker/config/RunImageConfiguration.java @@ -109,15 +109,18 @@ public class RunImageConfiguration { @Parameter private WaitConfiguration wait; + @Parameter + private Integer fetchLimit; + @Parameter private LogConfiguration log; - + @Parameter private RestartPolicy restartPolicy; @Parameter private boolean skip = false; - + public RunImageConfiguration() { } public String initAndValidate() { @@ -197,6 +200,10 @@ public WaitConfiguration getWaitConfiguration() { return wait; } + public Integer getFetchLimit() { + return fetchLimit; + } + public LogConfiguration getLogConfiguration() { return log; } @@ -224,7 +231,7 @@ public List getDnsSearch() { public List getExtraHosts() { return extraHosts; } - + public VolumeConfiguration getVolumeConfiguration() { return volumes; } @@ -248,7 +255,7 @@ public enum NamingStrategy { public NamingStrategy getNamingStrategy() { return namingStrategy == null ? NamingStrategy.none : namingStrategy; } - + public Boolean getPrivileged() { return privileged; } @@ -260,7 +267,7 @@ public RestartPolicy getRestartPolicy() { public boolean skip() { return skip; } - + // ====================================================================================== public static class Builder { @@ -387,12 +394,16 @@ public Builder wait(WaitConfiguration wait) { return this; } + public Builder fetchLimit(Integer fetchLimit) { + config.fetchLimit = fetchLimit; + return this; + } + public Builder log(LogConfiguration log) { config.log = log; return this; } - public Builder namingStrategy(String namingStrategy) { config.namingStrategy = namingStrategy == null ? NamingStrategy.none : diff --git a/src/main/java/io/fabric8/maven/docker/config/handler/property/ConfigKey.java b/src/main/java/io/fabric8/maven/docker/config/handler/property/ConfigKey.java index dde901b4b..9ff1640ed 100644 --- a/src/main/java/io/fabric8/maven/docker/config/handler/property/ConfigKey.java +++ b/src/main/java/io/fabric8/maven/docker/config/handler/property/ConfigKey.java @@ -1,5 +1,5 @@ package io.fabric8.maven.docker.config.handler.property;/* - * + * * Copyright 2014 Roland Huss * * Licensed under the Apache License, Version 2.0 (the "License"); @@ -15,8 +15,6 @@ * limitations under the License. */ -import static io.fabric8.maven.docker.assembly.DockerFileKeyword.WORKDIR; - /** * Enum holding possible configuration keys * @@ -51,6 +49,7 @@ public enum ConfigKey { ENV, ENV_PROPERTY_FILE, EXTRA_HOSTS, + FETCH_LIMIT, FROM, HOSTNAME, LABELS, diff --git a/src/main/java/io/fabric8/maven/docker/config/handler/property/PropertyConfigHandler.java b/src/main/java/io/fabric8/maven/docker/config/handler/property/PropertyConfigHandler.java index f958305fc..cef1bdbb4 100644 --- a/src/main/java/io/fabric8/maven/docker/config/handler/property/PropertyConfigHandler.java +++ b/src/main/java/io/fabric8/maven/docker/config/handler/property/PropertyConfigHandler.java @@ -1,5 +1,5 @@ package io.fabric8.maven.docker.config.handler.property;/* - * + * * Copyright 2014 Roland Huss * * Licensed under the Apache License, Version 2.0 (the "License"); @@ -40,14 +40,14 @@ public String getType() { @Override public List resolve(ImageConfiguration config, Properties properties) throws IllegalArgumentException { String prefix = getPrefix(config); - + RunImageConfiguration run = extractRunConfiguration(prefix,properties); BuildImageConfiguration build = extractBuildConfiguration(prefix,properties); WatchImageConfiguration watch = extractWatchConfig(prefix, properties); String name = extractName(prefix, properties); String alias = withPrefix(prefix, ALIAS, properties); - + return Collections.singletonList( new ImageConfiguration.Builder() .name(name) @@ -85,7 +85,7 @@ private BuildImageConfiguration extractBuildConfiguration(String prefix, Propert } private RunImageConfiguration extractRunConfiguration(String prefix, Properties properties) { - + return new RunImageConfiguration.Builder() .capAdd(listWithPrefix(prefix, CAP_ADD, properties)) .capDrop(listWithPrefix(prefix, CAP_DROP, properties)) @@ -113,6 +113,7 @@ private RunImageConfiguration extractRunConfiguration(String prefix, Properties .workingDir(withPrefix(prefix, WORKING_DIR, properties)) .log(extractLogConfig(prefix,properties)) .wait(extractWaitConfig(prefix, properties)) + .fetchLimit(intWithPrefix(prefix, FETCH_LIMIT, properties)) .volumes(extractVolumeConfig(prefix, properties)) .skip(withPrefix(prefix, SKIP_RUN, properties)) .build(); @@ -120,7 +121,7 @@ private RunImageConfiguration extractRunConfiguration(String prefix, Properties private AssemblyConfiguration extractAssembly(String prefix, Properties properties) { return new AssemblyConfiguration.Builder() - .basedir(withPrefix(prefix, ASSEMBLY_BASEDIR, properties)) + .basedir(withPrefix(prefix, ASSEMBLY_BASEDIR, properties)) .descriptor(withPrefix(prefix, ASSEMBLY_DESCRIPTOR, properties)) .descriptorRef(withPrefix(prefix, ASSEMBLY_DESCRIPTOR_REF, properties)) .dockerFileDir(withPrefix(prefix, ASSEMBLY_DOCKER_FILE_DIR, properties)) @@ -130,7 +131,7 @@ private AssemblyConfiguration extractAssembly(String prefix, Properties properti .mode(withPrefix(prefix, ASSEMBLY_MODE, properties)) .build(); } - + private String extractName(String prefix, Properties properties) throws IllegalArgumentException { String name = withPrefix(prefix, NAME, properties); if (name == null) { @@ -138,7 +139,7 @@ private String extractName(String prefix, Properties properties) throws IllegalA } return name; } - + // Extract only the values of the port mapping private List extractPortValues(String prefix, Properties properties) { List ret = new ArrayList<>(); @@ -204,7 +205,7 @@ private WaitConfiguration extractWaitConfig(String prefix, Properties properties .tcpPorts(asIntList(listWithPrefix(prefix, WAIT_TCP_PORT, properties))) .build(); } - + private WatchImageConfiguration extractWatchConfig(String prefix, Properties properties) { return new WatchImageConfiguration.Builder() .interval(asInt(withPrefix(prefix, WATCH_INTERVAL, properties))) @@ -219,7 +220,7 @@ private VolumeConfiguration extractVolumeConfig(String prefix, Properties proper .from(listWithPrefix(prefix, VOLUMES_FROM, properties)) .build(); } - + private int asInt(String s) { return s != null ? Integer.parseInt(s) : 0; } @@ -241,11 +242,11 @@ private List asIntList(List strings) { private List listWithPrefix(String prefix, ConfigKey key, Properties properties) { return extractFromPropertiesAsList(key.asPropertyKey(prefix), properties); } - + private Map mapWithPrefix(String prefix, ConfigKey key, Properties properties) { return extractFromPropertiesAsMap(key.asPropertyKey(prefix), properties); } - + private String withPrefix(String prefix, ConfigKey key, Properties properties) { return properties.getProperty(key.asPropertyKey(prefix)); } @@ -255,6 +256,11 @@ private Long longWithPrefix(String prefix, ConfigKey key, Properties properties) return prop == null ? null : Long.valueOf(prop); } + private Integer intWithPrefix(String prefix, ConfigKey key, Properties properties) { + String prop = withPrefix(prefix, key, properties); + return prop == null ? null : Integer.valueOf(prop); + } + private Boolean booleanWithPrefix(String prefix, ConfigKey key, Properties properties) { String prop = withPrefix(prefix,key,properties); return prop == null ? null : Boolean.valueOf(prop); diff --git a/src/main/java/io/fabric8/maven/docker/service/QueryService.java b/src/main/java/io/fabric8/maven/docker/service/QueryService.java index e382351e5..82c39afe7 100644 --- a/src/main/java/io/fabric8/maven/docker/service/QueryService.java +++ b/src/main/java/io/fabric8/maven/docker/service/QueryService.java @@ -17,7 +17,7 @@ public class QueryService { // Default limit when listing containers - private static final int CONTAINER_LIMIT = 100; + private final int containerLimit; // Access to docker daemon & logger private DockerAccess docker; @@ -29,7 +29,8 @@ public class QueryService { * @param docker remote access to docker daemon * @param log logger */ - public QueryService(DockerAccess docker, Logger log) { + public QueryService(DockerAccess docker, Logger log, int dockerFetchLimit) { + this.containerLimit = dockerFetchLimit; this.docker = docker; this.log = log; } @@ -52,7 +53,7 @@ public Container getContainer(String containerId) throws DockerAccessException { * @throws DockerAccessException in case of an remote error */ public Container getContainerByName(final String containerName) throws DockerAccessException { - for (Container el : docker.listContainers(CONTAINER_LIMIT)) { + for (Container el : docker.listContainers(containerLimit)) { if (containerName.equals(el.getName())) { return el; } @@ -72,14 +73,14 @@ public String getContainerName(String containerId) throws DockerAccessException } /** - * Get all containers which are build from an image. Only the last 100 containers are considered + * Get all containers which are build from an image. By default only the last 100 containers are considered * * @param image for which its container are looked up * @return list of Container objects * @throws DockerAccessException if the request fails */ public List getContainersForImage(final String image) throws DockerAccessException { - List list = docker.listContainers(CONTAINER_LIMIT); + List list = docker.listContainers(containerLimit); List ret = new ArrayList<>(); for (Container el : list) { if (image.equals(el.getImage())) { @@ -91,7 +92,7 @@ public List getContainersForImage(final String image) throws DockerAc /** * Finds the id of an image. - * + * * @param imageName name of the image. * @return the id of the image * @throws DockerAccessException if the request fails @@ -99,7 +100,7 @@ public List getContainersForImage(final String image) throws DockerAc public String getImageId(String imageName) throws DockerAccessException { return docker.getImageId(imageName); } - + /** * Get the id of the latest container started for an image * @@ -135,7 +136,7 @@ public Container getLatestContainerForImage(String image) throws DockerAccessExc public boolean hasContainer(String containerName) throws DockerAccessException { return getContainerByName(containerName) != null; } - + /** * Check whether the given Image is locally available. * diff --git a/src/main/java/io/fabric8/maven/docker/service/ServiceHub.java b/src/main/java/io/fabric8/maven/docker/service/ServiceHub.java index 4932f2fe2..bd6113e2a 100644 --- a/src/main/java/io/fabric8/maven/docker/service/ServiceHub.java +++ b/src/main/java/io/fabric8/maven/docker/service/ServiceHub.java @@ -44,7 +44,7 @@ public class ServiceHub { ServiceHub(DockerAccess dockerAccess, ContainerTracker containerTracker, BuildPluginManager pluginManager, DockerAssemblyManager dockerAssemblyManager, MavenProject project, MavenSession session, - Logger logger, LogOutputSpecFactory logSpecFactory) { + Logger logger, LogOutputSpecFactory logSpecFactory, int dockerFetchLimit) { this.dockerAccess = dockerAccess; this.logSpecFactory = logSpecFactory; @@ -53,7 +53,7 @@ public class ServiceHub { archiveService = new ArchiveService(dockerAssemblyManager, logger); if (dockerAccess != null) { - queryService = new QueryService(dockerAccess, logger); + queryService = new QueryService(dockerAccess, logger, dockerFetchLimit); runService = new RunService(dockerAccess, queryService, containerTracker, logSpecFactory, logger); buildService = new BuildService(dockerAccess, queryService, archiveService, logger); } else { diff --git a/src/main/java/io/fabric8/maven/docker/service/ServiceHubFactory.java b/src/main/java/io/fabric8/maven/docker/service/ServiceHubFactory.java index 4637b83da..83e7e5b2f 100644 --- a/src/main/java/io/fabric8/maven/docker/service/ServiceHubFactory.java +++ b/src/main/java/io/fabric8/maven/docker/service/ServiceHubFactory.java @@ -18,7 +18,7 @@ public class ServiceHubFactory { // Track started containers private final ContainerTracker containerTracker = new ContainerTracker(); - + @Requirement protected BuildPluginManager pluginManager; @@ -27,10 +27,10 @@ public class ServiceHubFactory { private LogOutputSpecFactory logOutputSpecFactory; - public ServiceHub createServiceHub(MavenProject project, MavenSession session, DockerAccess access, Logger log, LogOutputSpecFactory logSpecFactory) { + public ServiceHub createServiceHub(MavenProject project, MavenSession session, DockerAccess access, Logger log, LogOutputSpecFactory logSpecFactory, int dockerFetchLimit) { this.logOutputSpecFactory = logSpecFactory; return new ServiceHub(access, containerTracker, pluginManager, dockerAssemblyManager, project, session, - log, logSpecFactory); + log, logSpecFactory, dockerFetchLimit); } public LogOutputSpecFactory getLogOutputSpecFactory() {