Skip to content

Commit

Permalink
CODENVY-560 : Introduce new type for providing docker recipes and rem…
Browse files Browse the repository at this point in the history 
…ove InstanceKey

#1 new docker recipe type

currently we have type:"dockerfile", location: "http://path-to-recipe"

now we could provide
type:"dockerfile", content: "FROM codenvy/foo\nENV FLORENT=TRUE\"

and
type:"image", location or content: "codenvy/foo"

#2 InstanceKey
Up to now, InstanceKey was used to perform snapshot recovery.
But machine source is a way to provide this information.
So remove InstanceKey and replace it by MachineSource (and DockerMachineSource instead of DockerInstanceKey)

InstanceProvider:
void removeInstanceSnapshot(InstanceKey instanceKey)
--> void removeInstanceSnapshot(MachineSource machineSource)

Instance:
InstanceKey saveToSnapshot(String owner)
--> MachineSource saveToSnapshot(String owner)

#3 InstanceProvider model
To avoid also that MachineManager "knows" the inner type, the recipe handling is moved to the instance provider implementation
And as the snapshot handling is with MachineSource (included in MachineConfig included in Machine), no need to give extra InstanceKey parameter

Replace two previous methods

Instance createInstance(Recipe recipe,
                            Machine machine,
                            LineConsumer creationLogsOutput)

 Instance createInstance(InstanceKey instanceKey,
                            Machine machine,
                            LineConsumer creationLogsOutput) throws NotFoundException, InvalidInstanceSnapshotException, MachineException;

by only one:
   createInstance(Machine machine,
                            LineConsumer creationLogsOutput)

Change-Id: Ia7ea97bc1a44059b4892f5db387f54f2e1709fa3
Signed-off-by: Florent BENOIT <fbenoit@codenvy.com>
  • Loading branch information
@benoitf
benoitf committed May 30, 2016
1 parent 9727335 commit 3d2ef51
Show file tree
Hide file tree
Showing 34 changed files with 919 additions and 373 deletions.
21 changes: 21 additions & 0 deletions ...he-core-api-model/src/main/java/org/eclipse/che/api/core/model/machine/MachineSource.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,4 +24,25 @@ public interface MachineSource {
* Returns URL or ID
*/
String getLocation();

/**
* @return content of the machine source. No need to use an external link.
*/
String getContent();

/**
* Defines the new content to use for this machine source.
* Alternate way is to provide a location
* @param content the content instead of an external link like with location
*/
void setContent(String content);

/**
* Defines the new content to use for this machine source.
* Alternate way is to provide a location
* @param content the content instead of an external link like with location
* @return the current intance of the object
*/
MachineSource withContent(String content);

}
20 changes: 12 additions & 8 deletions ...in-docker-machine/src/main/java/org/eclipse/che/plugin/docker/machine/DockerInstance.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,12 +16,12 @@
import org.eclipse.che.api.core.NotFoundException;
import org.eclipse.che.api.core.model.machine.Command;
import org.eclipse.che.api.core.model.machine.Machine;
import org.eclipse.che.api.core.model.machine.MachineSource;
import org.eclipse.che.api.core.util.LineConsumer;
import org.eclipse.che.api.core.util.ListLineConsumer;
import org.eclipse.che.api.machine.server.exception.MachineException;
import org.eclipse.che.api.machine.server.model.impl.MachineRuntimeInfoImpl;
import org.eclipse.che.api.machine.server.spi.Instance;
import org.eclipse.che.api.machine.server.spi.InstanceKey;
import org.eclipse.che.api.machine.server.spi.InstanceProcess;
import org.eclipse.che.api.machine.server.spi.impl.AbstractInstance;
import org.eclipse.che.commons.lang.NameGenerator;
Expand Down Expand Up @@ -60,6 +60,11 @@
public class DockerInstance extends AbstractInstance {
private static final Logger LOG = LoggerFactory.getLogger(DockerInstance.class);

/**
* Name of the latest tag used in Docker image.
*/
private static final String LATEST_TAG = "latest";

private static final AtomicInteger pidSequence = new AtomicInteger(1);
private static final String PID_FILE_TEMPLATE = "/tmp/docker-exec-%s.pid";
private static final Pattern PID_FILE_PATH_PATTERN = Pattern.compile(String.format(PID_FILE_TEMPLATE, "([0-9]+)"));
Expand Down Expand Up @@ -191,21 +196,20 @@ public InstanceProcess createProcess(Command command, String outputChannel) thro
}

@Override
public InstanceKey saveToSnapshot(String owner) throws MachineException {
public MachineSource saveToSnapshot(String owner) throws MachineException {
try {
final String repository = generateRepository();
final String tag = "latest";
if(!snapshotUseRegistry) {
commitContainer(owner, repository, tag);
return new DockerInstanceKey(repository, tag);
commitContainer(owner, repository, LATEST_TAG);
new DockerMachineSource(repository).setTag(LATEST_TAG);
}
final String repositoryName = registry + '/' + repository;
commitContainer(owner, repositoryName, tag);
commitContainer(owner, repositoryName, LATEST_TAG);
//TODO fix this workaround. Docker image is not visible after commit when using swarm
Thread.sleep(2000);
final ProgressLineFormatterImpl lineFormatter = new ProgressLineFormatterImpl();
final String digest = docker.push(PushParams.create(repository)
.withTag(tag)
.withTag(LATEST_TAG)
.withRegistry(registry),
progressMonitor -> {
try {
Expand All @@ -214,7 +218,7 @@ public InstanceKey saveToSnapshot(String owner) throws MachineException {
}
});
docker.removeImage(RemoveImageParams.create(repositoryName).withForce(false));
return new DockerInstanceKey(repository, tag, registry, digest);
return new DockerMachineSource(repository).setRegistry(registry).setDigest(digest).setTag(LATEST_TAG);
} catch (IOException ioEx) {
throw new MachineException(ioEx);
} catch (InterruptedException e) {
Expand Down
116 changes: 90 additions & 26 deletions ...r-machine/src/main/java/org/eclipse/che/plugin/docker/machine/DockerInstanceProvider.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,8 @@

import org.eclipse.che.api.core.NotFoundException;
import org.eclipse.che.api.core.model.machine.Machine;
import org.eclipse.che.api.core.model.machine.MachineConfig;
import org.eclipse.che.api.core.model.machine.MachineSource;
import org.eclipse.che.api.core.model.machine.Recipe;
import org.eclipse.che.api.core.model.machine.ServerConf;
import org.eclipse.che.api.core.util.FileCleaner;
Expand All @@ -28,8 +30,8 @@
import org.eclipse.che.api.machine.server.exception.SnapshotException;
import org.eclipse.che.api.machine.server.exception.UnsupportedRecipeException;
import org.eclipse.che.api.machine.server.spi.Instance;
import org.eclipse.che.api.machine.server.spi.InstanceKey;
import org.eclipse.che.api.machine.server.spi.InstanceProvider;
import org.eclipse.che.api.machine.server.util.RecipeRetriever;
import org.eclipse.che.commons.annotation.Nullable;
import org.eclipse.che.commons.env.EnvironmentContext;
import org.eclipse.che.commons.lang.IoUtil;
Expand Down Expand Up @@ -78,6 +80,16 @@
public class DockerInstanceProvider implements InstanceProvider {
private static final Logger LOG = LoggerFactory.getLogger(DockerInstanceProvider.class);

/**
* dockerfile type support with recipe being a content of Dockerfile
*/
public static final String DOCKER_FILE_TYPE = "dockerfile";

/**
* image type support with recipe script being the name of the repository + image name
*/
public static final String DOCKER_IMAGE_TYPE = "image";

private final DockerConnector docker;
private final DockerInstanceStopDetector dockerInstanceStopDetector;
private final DockerContainerNameGenerator containerNameGenerator;
Expand All @@ -95,13 +107,15 @@ public class DockerInstanceProvider implements InstanceProvider {
private final String[] allMachinesExtraHosts;
private final String projectFolderPath;
private final boolean snapshotUseRegistry;
private final RecipeRetriever recipeRetriever;

@Inject
public DockerInstanceProvider(DockerConnector docker,
DockerConnectorConfiguration dockerConnectorConfiguration,
DockerMachineFactory dockerMachineFactory,
DockerInstanceStopDetector dockerInstanceStopDetector,
DockerContainerNameGenerator containerNameGenerator,
RecipeRetriever recipeRetriever,
@Named("machine.docker.dev_machine.machine_servers") Set<ServerConf> devMachineServers,
@Named("machine.docker.machine_servers") Set<ServerConf> allMachinesServers,
@Named("machine.docker.dev_machine.machine_volumes") Set<String> devMachineSystemVolumes,
Expand All @@ -118,10 +132,11 @@ public DockerInstanceProvider(DockerConnector docker,
this.dockerMachineFactory = dockerMachineFactory;
this.dockerInstanceStopDetector = dockerInstanceStopDetector;
this.containerNameGenerator = containerNameGenerator;
this.recipeRetriever = recipeRetriever;
this.workspaceFolderPathProvider = workspaceFolderPathProvider;
this.doForcePullOnBuild = doForcePullOnBuild;
this.privilegeMode = privilegeMode;
this.supportedRecipeTypes = Collections.singleton("dockerfile");
this.supportedRecipeTypes = Sets.newHashSet(DOCKER_FILE_TYPE, DOCKER_IMAGE_TYPE);
this.projectFolderPath = projectFolderPath;
this.snapshotUseRegistry = snapshotUseRegistry;

Expand Down Expand Up @@ -214,10 +229,50 @@ public Set<String> getRecipeTypes() {
return supportedRecipeTypes;
}



/**
* Creates instance from scratch or by reusing a previously one by using specified {@link MachineSource}
* data in {@link MachineConfig}.
*
* @param machine
* machine description
* @param creationLogsOutput
* output for instance creation logs
* @return newly created {@link Instance}
* @throws UnsupportedRecipeException
* if specified {@code recipe} is not supported
* @throws InvalidRecipeException
* if {@code recipe} is invalid
* @throws NotFoundException
* if instance described by {@link MachineSource} doesn't exists
* @throws MachineException
* if other error occurs
*/
@Override
public Instance createInstance(Recipe recipe,
Machine machine,
LineConsumer creationLogsOutput) throws MachineException, UnsupportedRecipeException {
public Instance createInstance(Machine machine, LineConsumer creationLogsOutput)
throws UnsupportedRecipeException, InvalidRecipeException, NotFoundException, MachineException {

// based on machine source, do the right steps
MachineConfig machineConfig = machine.getConfig();
MachineSource machineSource = machineConfig.getSource();
String type = machineSource.getType();

// get recipe
// - it's a dockerfile type:
// - location defined : download this location and get script as recipe
// - content defined : use this content as recipe script
// - it's an image:
// - use either location or content as image ([registry:port]/<repository-image>[:tag][@digest])
final Recipe recipe;
if (DOCKER_FILE_TYPE.equals(type)) {
recipe = this.recipeRetriever.getRecipe(machineConfig);
} else if (DOCKER_IMAGE_TYPE.equals(type)) {
return doCreateInstanceImage(machine, creationLogsOutput);
} else {
// not supported
throw new UnsupportedRecipeException("The type '" + type + "' is not supported");
}
final Dockerfile dockerfile = parseRecipe(recipe);

final String userName = EnvironmentContext.getCurrent().getSubject().getUserName();
Expand All @@ -236,21 +291,21 @@ public Instance createInstance(Recipe recipe,
creationLogsOutput);
}

@Override
public Instance createInstance(InstanceKey instanceKey,
Machine machine,
LineConsumer creationLogsOutput) throws NotFoundException, MachineException {
final DockerInstanceKey dockerInstanceKey = new DockerInstanceKey(instanceKey);
if (snapshotUseRegistry) {
pullImage(dockerInstanceKey, creationLogsOutput);
}
protected Instance doCreateInstanceImage(final Machine machine,
final LineConsumer creationLogsOutput) throws NotFoundException, MachineException {
final DockerMachineSource dockerMachineSource = new DockerMachineSource(machine.getConfig().getSource());

if (snapshotUseRegistry) {
pullImage(dockerMachineSource, creationLogsOutput);
}

final String userName = EnvironmentContext.getCurrent().getSubject().getUserName();
final String machineContainerName = containerNameGenerator.generateContainerName(machine.getWorkspaceId(),
machine.getId(),
userName,
machine.getConfig().getName());
final String machineImageName = "eclipse-che/" + machineContainerName;
final String fullNameOfPulledImage = dockerInstanceKey.getFullName();
final String fullNameOfPulledImage = dockerMachineSource.getFullName();
try {
// tag image with generated name to allow sysadmin recognize it
docker.tag(fullNameOfPulledImage, machineImageName, null);
Expand Down Expand Up @@ -337,15 +392,15 @@ protected void buildImage(Dockerfile dockerfile,
}
}

private void pullImage(DockerInstanceKey dockerInstanceKey, final LineConsumer creationLogsOutput) throws MachineException {
if (dockerInstanceKey.getRepository() == null) {
private void pullImage(DockerMachineSource dockerMachineSource, final LineConsumer creationLogsOutput) throws MachineException {
if (dockerMachineSource.getRepository() == null) {
throw new MachineException("Machine creation failed. Snapshot state is invalid. Please, contact support.");
}
try {
final ProgressLineFormatterImpl progressLineFormatter = new ProgressLineFormatterImpl();
docker.pull(dockerInstanceKey.getRepository(),
dockerInstanceKey.getTag(),
dockerInstanceKey.getRegistry(),
docker.pull(dockerMachineSource.getRepository(),
dockerMachineSource.getTag(),
dockerMachineSource.getRegistry(),
currentProgressStatus -> {
try {
creationLogsOutput.writeLine(progressLineFormatter.format(currentProgressStatus));
Expand All @@ -358,30 +413,39 @@ private void pullImage(DockerInstanceKey dockerInstanceKey, final LineConsumer c
}
}


/**
* Removes snapshot of the instance in implementation specific way.
*
* @param machineSource
* contains implementation specific key of the snapshot of the instance that should be removed
* @throws SnapshotException
* if exception occurs on instance snapshot removal
*/
@Override
public void removeInstanceSnapshot(InstanceKey instanceKey) throws SnapshotException {
public void removeInstanceSnapshot(MachineSource machineSource) throws SnapshotException {
// use registry API directly because docker doesn't have such API yet
// https://github.com/docker/docker-registry/issues/45
final DockerInstanceKey dockerInstanceKey = new DockerInstanceKey(instanceKey);
final DockerMachineSource dockerMachineSource = new DockerMachineSource(machineSource);
if (!snapshotUseRegistry) {
try {
docker.removeImage(RemoveImageParams.create(dockerInstanceKey.getFullName()));
docker.removeImage(RemoveImageParams.create(dockerMachineSource.getFullName()));
} catch (IOException ignore) {
}
return;
}

final String registry = dockerInstanceKey.getRegistry();
final String repository = dockerInstanceKey.getRepository();
final String registry = dockerMachineSource.getRegistry();
final String repository = dockerMachineSource.getRepository();
if (registry == null || repository == null) {
LOG.error("Failed to remove instance snapshot: invalid instance key: {}", instanceKey);
LOG.error("Failed to remove instance snapshot: invalid instance key: {}", dockerMachineSource);
throw new SnapshotException("Snapshot removing failed. Snapshot attributes are not valid");
}

try {
URL url = UriBuilder.fromUri("http://" + registry) // TODO make possible to use https here
.path("/v2/{repository}/manifests/{digest}")
.build(repository, dockerInstanceKey.getDigest())
.build(repository, dockerMachineSource.getDigest())
.toURL();
final HttpURLConnection conn = (HttpURLConnection)url.openConnection();
try {
Expand Down
Loading

0 comments on commit 3d2ef51

Please sign in to comment.