Merge pull request fabric8io#645 from Katsuya-Tomioka/docker-load-sup…

…port

PR merged! Thanks!

doc/changelog.md

@@ -4,7 +4,7 @@
   - Better log message when waiting for URL (#640)
   - Extended authentication for AWS ECR (#663)
   - Add two new goals: "volume-create" and "volume-remove" for volume handling independent of images. 
-
+  - Support for loading from an tar archive (option `<build><dockerArchive>`) (#645)
 
 * **0.18.1** (2016-11-17)
   - Renamed `basedir` and `exportBasedir` in an `<assembly>` configuration to `targetDir` and `exportTargetDir` since this better reflects the purpose, i.e. the target in the Docker image to which the assembly is copied. The old name is still recognized but deprecated. 

src/main/asciidoc/inc/build/_overview.adoc

@@ -8,12 +8,13 @@ With an inline plugin configuration all information required to build the image
 When using this mode, the Dockerfile is created on the fly with all instructions extracted from the configuration given.
 
 [[external-dockerfile]]
-.External Dockerfile
-Alternatively an external Dockerfile template can be used. This mode is switch on by using one of these two configuration options within
+.External Dockerfile or docker archive
+Alternatively an external Dockerfile template or docker archive can be used. This mode is switch on by using one of these three configuration options within
 the `<build>` configuration section.
 
 * *dockerFileDir* specifies a directory containing a `Dockerfile` that will be used to create the image.
 * *dockerFile* specifies a specific Dockerfile. The `dockerFileDir` is set to the directory containing the file.
+* *dockerArchive* specifies a previously saved image archive to load directly.
 
 If `dockerFileDir` is a relative path it is looked up in `${project.basedir}/src/main/docker`. You can make easily an absolute path by prefixing with `${project.basedir}`.
 

src/main/asciidoc/inc/external/_property_configuration.adoc

@@ -48,7 +48,7 @@ values in the `<build>` and `<run>` sections.
 | can be `ignore` to use the permission as found on files regardless on any assembly configuration, `keep` to respect the assembly provided permissions, `exec` for setting the executable bit on all files (required for Windows when using an assembly mode `dir`) or `auto` to let the plugin select `exec` on Windows and `keep` on others. `keep` is the default value.
 
 | *docker.assembly.dockerFileDir*
-| specifies a directory containing an external Dockerfile that will be used to create the image. This is deprecated p ease use `docker.dockerFileDir` or `docker.dockerFile` instead.
+| specifies a directory containing an external Dockerfile that will be used to create the image. This is deprecated please use `docker.dockerFileDir` or `docker.dockerFile` instead.
 
 | *docker.nocache*
 | Don't use Docker's build cache.This can be overwritten by setting a system property `docker.nocache` when running Maven.
@@ -77,6 +77,9 @@ values in the `<build>` and `<run>` sections.
 | *docker.dnsSearch.idx*
 | List of dns search domains
 
+| *docker.dockerArchive*
+| specify an archive which can be loaded with `docker load`. Use this as an alternative to `docker.dockerFile` or `docker.dockerFileDir`
+
 | *docker.dockerFile*
 | specifies a Dockerfile to use. This property must point to the Dockerfile itself.
 

src/main/java/io/fabric8/maven/docker/AbstractBuildSupportMojo.java

@@ -60,6 +60,7 @@ protected MojoParameters createMojoParameters() {
     protected void buildImage(ServiceHub hub, ImageConfiguration imageConfig)
             throws DockerAccessException, MojoExecutionException {
         EnvUtil.storeTimestamp(getBuildTimestampFile(), getBuildTimestamp());
+
         autoPullBaseImage(hub, imageConfig);
 
         MojoParameters params = createMojoParameters();
@@ -98,6 +99,11 @@ private void autoPullBaseImage(ServiceHub hub, ImageConfiguration imageConfig)
             throws DockerAccessException, MojoExecutionException {
         BuildImageConfiguration buildConfig = imageConfig.getBuildConfiguration();
 
+        if (buildConfig.getDockerArchive() != null) {
+            // No autopull need in archive mode
+            return;
+        }
+
         String fromImage;
         if (buildConfig.isDockerFileMode()) {
             fromImage = extractBaseFromDockerfile(buildConfig);

src/main/java/io/fabric8/maven/docker/StartMojo.java

@@ -292,7 +292,8 @@ private Queue<ImageConfiguration> prepareStart(ServiceHub hub, QueryService quer
 
             String imageName = imageConfig.getName();
             checkImageWithAutoPull(hub, imageName,
-                                   getConfiguredRegistry(imageConfig, pullRegistry), imageConfig.getBuildConfiguration() == null);
+                                   getConfiguredRegistry(imageConfig, pullRegistry),
+                                   imageConfig.getBuildConfiguration() == null);
 
             RunImageConfiguration runConfig = imageConfig.getRunConfiguration();
             NetworkConfig config = runConfig.getNetworkingConfig();

src/main/java/io/fabric8/maven/docker/access/DockerAccess.java

@@ -149,6 +149,15 @@ void copyArchive(String containerId, File archive, String targetPath)
      */
     void removeContainer(String containerId, boolean removeVolumes) throws DockerAccessException;
 
+    /**
+     * Load an image from an archive.
+     *
+     * @param image the image to pull.
+     * @param filepath a URL string of the archive
+     * @throws DockerAccessException if the image couldn't be loaded.
+     */
+    void loadImage(String image, String filepath) throws DockerAccessException;
+
     /**
      * Pull an image from a remote registry and store it locally.
      *

src/main/java/io/fabric8/maven/docker/access/UrlBuilder.java

@@ -82,6 +82,11 @@ public String listContainers(String ... filter) {
         return builder.build();
     }
 
+    public String loadImage() {
+        return u("images/load")
+            .build();
+    }
+
     public String pullImage(ImageName name, String registry) {
         return u("images/create")
                 .p("fromImage", name.getNameWithoutTag(registry))

src/main/java/io/fabric8/maven/docker/access/hc/DockerAccessWithHcClient.java

@@ -14,6 +14,8 @@
 import java.io.InputStreamReader;
 import java.io.LineNumberReader;
 import java.net.URI;
+import java.nio.file.Files;
+import java.nio.file.Paths;
 import java.util.ArrayList;
 import java.util.Collections;
 import java.util.List;
@@ -23,6 +25,8 @@
 import org.apache.http.HttpResponse;
 import org.apache.http.client.HttpResponseException;
 import org.apache.http.client.ResponseHandler;
+import org.apache.http.entity.InputStreamEntity;
+import org.apache.http.impl.client.CloseableHttpClient;
 import org.json.JSONArray;
 import org.json.JSONObject;
 
@@ -357,6 +361,17 @@ public void removeContainer(String containerId, boolean removeVolumes)
         }
     }
 
+    @Override
+    public void loadImage(String image, String filepath) throws DockerAccessException {
+        String url = urlBuilder.loadImage();
+
+        try {
+            delegate.post(url, new File(filepath), new BodyAndStatusResponseHandler(), HTTP_OK);
+        } catch (IOException e) {
+            throw new DockerAccessException(e, "Unable to load %s", filepath);
+        }
+    }
+
     @Override
     public void pullImage(String image, AuthConfig authConfig, String registry)
             throws DockerAccessException {

src/main/java/io/fabric8/maven/docker/assembly/DockerAssemblyManager.java

@@ -93,7 +93,7 @@ public File createDockerTarArchive(String imageName, MojoParameters params, Buil
                 final File dockerFile = buildConfig.getAbsoluteDockerFilePath(params);
                 if (!dockerFile.exists()) {
                     throw new MojoExecutionException("Configured Dockerfile \"" +
-                                                     buildConfig.getDockerFile() + "\" (resolved to \"" + dockerFile + "\") doesnt exist");
+                                                     buildConfig.getDockerFile() + "\" (resolved to \"" + dockerFile + "\") doesn't exist");
                 }
                 // User dedicated Dockerfile from extra director
                 customizer = new ArchiverCustomizer() {

src/main/java/io/fabric8/maven/docker/config/BuildImageConfiguration.java

@@ -29,6 +29,14 @@ public class BuildImageConfiguration implements Serializable {
      */
     private String dockerFile;
 
+    /**
+     * Path to a docker archive to load an image instead of building from scratch. Note only either dockerFile or
+     * dockerArchive can be used.
+     *
+     * @parameter
+     */
+    private String dockerArchive;
+
     // Base Image name of the data image to use.
     /**
      * @parameter
@@ -145,20 +153,22 @@ public class BuildImageConfiguration implements Serializable {
     private BuildTarArchiveCompression compression = BuildTarArchiveCompression.none;
 
     // Path to Dockerfile to use, initialized lazily ....
-    File dockerFileFile;
-    private boolean dockerFileMode;
+    private File dockerFileFile;
 
     public BuildImageConfiguration() {}
 
-
     public boolean isDockerFileMode() {
-        return dockerFileMode;
+        return dockerFileFile != null;
     }
 
     public File getDockerFile() {
         return dockerFileFile;
     }
 
+    public String getDockerArchive() {
+        return dockerArchive;
+    }
+
     public String getFrom() {
         if (from == null && getFromExt() != null) {
             return getFromExt().get("name");
@@ -284,6 +294,11 @@ public Builder dockerFile(String file) {
             return this;
         }
 
+        public Builder dockerArchive(String archive) {
+            config.dockerArchive = archive;
+            return this;
+        }
+
         public Builder from(String from) {
             config.from = from;
             return this;
@@ -457,24 +472,37 @@ public String initAndValidate(Logger log) throws IllegalArgumentException {
 
     // Initialize the dockerfile location and the build mode
     private void initDockerFileFile(Logger log) {
+        // can't have dockerFile/dockerFileDir and dockerArchive
+        if ((dockerFile != null || dockerFileDir != null) && dockerArchive != null) {
+            throw new IllegalArgumentException("Both <dockerFile> (<dockerFileDir>) and <dockerArchive> are set. " +
+                                               "Only one of them can be specified.");
+        }
+        dockerFileFile = findDockerFileFile(log);
+    }
+
+    private File findDockerFileFile(Logger log) {
         if (dockerFile != null) {
-            dockerFileFile = new File(dockerFile);
-            dockerFileMode = true;
-        } else if (dockerFileDir != null) {
-            dockerFileFile = new File(dockerFileDir, "Dockerfile");
-            dockerFileMode = true;
-        } else {
-            String deprecatedDockerFileDir = getAssemblyConfiguration() != null ?
-                getAssemblyConfiguration().getDockerFileDir() :
-                null;
+            return new File(dockerFile);
+        }
+
+        if (dockerFileDir != null) {
+            return new File(dockerFileDir, "Dockerfile");
+        }
+
+        // TODO: Remove the following deprecated handling section
+        if (dockerArchive == null) {
+            String deprecatedDockerFileDir =
+                getAssemblyConfiguration() != null ?
+                    getAssemblyConfiguration().getDockerFileDir() :
+                    null;
             if (deprecatedDockerFileDir != null) {
                 log.warn("<dockerFileDir> in the <assembly> section of a <build> configuration is deprecated");
                 log.warn("Please use <dockerFileDir> or <dockerFile> directly within the <build> configuration instead");
-                dockerFileFile = new File(deprecatedDockerFileDir,"Dockerfile");
-                dockerFileMode = true;
-            } else {
-                dockerFileMode = false;
+                return new File(deprecatedDockerFileDir,"Dockerfile");
             }
         }
+
+        // No dockerfile mode
+        return null;
     }
 }

src/main/java/io/fabric8/maven/docker/config/handler/property/ConfigKey.java

@@ -46,6 +46,7 @@ public enum ConfigKey {
     DOMAINNAME,
     DNS,
     DNS_SEARCH,
+    DOCKER_ARCHIVE,
     DOCKER_FILE,
     DOCKER_FILE_DIR,
     ENTRYPOINT,

src/main/java/io/fabric8/maven/docker/config/handler/property/PropertyConfigHandler.java

@@ -85,6 +85,7 @@ private BuildImageConfiguration extractBuildConfiguration(String prefix, Propert
                 .maintainer(withPrefix(prefix, MAINTAINER, properties))
                 .workdir(withPrefix(prefix, WORKDIR, properties))
                 .skip(withPrefix(prefix, SKIP_BUILD, properties))
+                .dockerArchive(withPrefix(prefix, DOCKER_ARCHIVE, properties))
                 .dockerFile(withPrefix(prefix, DOCKER_FILE, properties))
                 .dockerFileDir(withPrefix(prefix, DOCKER_FILE_DIR, properties))
                 .user(withPrefix(prefix, USER, properties))

src/main/java/io/fabric8/maven/docker/service/BuildService.java

@@ -55,6 +55,13 @@ public void buildImage(ImageConfiguration imageConfig, MojoParameters params, bo
         }
 
         long time = System.currentTimeMillis();
+
+        if (buildConfig.getDockerArchive() != null) {
+            docker.loadImage(imageName, buildConfig.getDockerArchive());
+            log.info("%s: Loaded tarball in %s", buildConfig.getDockerArchive(), EnvUtil.formatDurationTill(time));
+            return;
+        }
+
         File dockerArchive = archiveService.createArchive(imageName, buildConfig, params, log);
         log.info("%s: Created %s in %s", dockerArchive.getName(), imageConfig.getDescription(), EnvUtil.formatDurationTill(time));
 

src/test/java/integration/DockerAccessIT.java

@@ -32,6 +32,7 @@ public class DockerAccessIT {
     private static final String IMAGE = "busybox:buildroot-2014.02";
 
     private static final String IMAGE_TAG = "busybox:tagged";
+    private static final String IMAGE_LATEST = "busybox:latest";
     private static final int PORT = 5677;
 
     private String containerId;
@@ -85,6 +86,14 @@ public void testPullStartStopRemove() throws DockerAccessException, InterruptedE
         }
     }
 
+    @Test
+    public void testLoadImage() throws DockerAccessException {
+        testDoesNotHave();
+        dockerClient.loadImage(IMAGE_LATEST, "integration/busybox-image.tar.gz");
+        assertTrue(hasImage(IMAGE_LATEST));
+        testRemoveImage(IMAGE_LATEST);
+    }
+
     private DockerAccessWithHcClient createClient(String baseUrl, Logger logger) {
         try {
             String certPath = createDockerConnectionDetector(logger).detectConnectionParameter(null,null).getCertPath();

src/test/java/io/fabric8/maven/docker/UrlBuilderTest.java

@@ -18,8 +18,10 @@
 
 import java.io.UnsupportedEncodingException;
 import java.net.*;
+import java.util.HashMap;
 
 import io.fabric8.maven.docker.access.UrlBuilder;
+import io.fabric8.maven.docker.util.ImageName;
 import org.junit.Test;
 
 import static org.junit.Assert.*;
@@ -30,6 +32,42 @@
  */
 public class UrlBuilderTest {
 
+    @Test
+    public void buildImage() {
+        UrlBuilder builder = new UrlBuilder("","1.0");
+        assertEquals("/1.0/build?t=image1&dockerfile=df&nocache=0&rm=1",
+            builder.buildImage("image1", "df", false, false, null));
+        assertEquals("/1.0/build?t=image1&dockerfile=df&nocache=1&forcerm=1",
+            builder.buildImage("image1", "df", true, true, null));
+        HashMap<String, String> m = new HashMap<>();
+        m.put("k1", "v1");
+        m.put("k2", "v2");
+        assertEquals("/1.0/build?t=image1&dockerfile=df&nocache=0&rm=1&buildargs=%7B%22k1%22%3A%22v1%22%2C%22k2%22%3A%22v2%22%7D",
+            builder.buildImage("image1", "df", false, false, m));
+    }
+
+    @Test
+    public void copyArchive() {
+        UrlBuilder builder = new UrlBuilder("","1.0");
+        assertEquals("/1.0/containers/cid/archive?path=tp", builder.copyArchive("cid", "tp"));
+
+    }
+
+    @Test
+    public void containerLogs() {
+        UrlBuilder builder = new UrlBuilder("","1.0");
+        assertEquals("/1.0/containers/cid/logs?stdout=1&timestamps=1&stderr=1&follow=0",
+            builder.containerLogs("cid", false));
+
+    }
+
+    @Test
+    public void deleteImage() {
+        UrlBuilder builder = new UrlBuilder("","1.0");
+        assertEquals("/1.0/images/n1?force=0", builder.deleteImage("n1", false));
+
+    }
+
     @Test
     public void listContainers() throws MalformedURLException, UnsupportedEncodingException {
         UrlBuilder builder = new UrlBuilder("","1.0");
@@ -45,4 +83,27 @@ public void listContainers() throws MalformedURLException, UnsupportedEncodingEx
             assertTrue(exp.getMessage().contains("pair"));
         }
     }
+
+    @Test
+    public void loadImage() {
+        UrlBuilder builder = new UrlBuilder("", "1.0");
+        assertEquals("/1.0/images/load",builder.loadImage());
+    }
+
+    @Test
+    public void pullImage() {
+        UrlBuilder builder = new UrlBuilder("", "1.0");
+        assertEquals("/1.0/images/create?fromImage=reg%2Ft1&tag=latest",
+            builder.pullImage(new ImageName("t1:latest"), "reg"));
+        assertEquals("/1.0/images/create?fromImage=reg%2Ft1",
+            builder.pullImage(new ImageName("t1"), "reg"));
+    }
+
+    @Test
+    public void tagContainer() {
+        UrlBuilder builder = new UrlBuilder("", "1.0");
+        assertEquals("/1.0/images/t1%3Alatest/tag?repo=new&force=1&tag=tag1",
+            builder.tagContainer(new ImageName("t1:latest"), new ImageName("new:tag1"), true));
+
+    }
 }