Skip to content

Commit

Permalink
CHE-2300: fix usage of dockerfile as a source of machine in case of s…
Browse files Browse the repository at this point in the history 
…ecured API (eclipse-che#2421)

Signed-off-by: Alexander Garagatyi <agaragatyi@codenvy.com>
  • Loading branch information
Alexander Garagatyi authored Sep 13, 2016
1 parent 2176c35 commit be851cb
Show file tree
Hide file tree
Showing 8 changed files with 498 additions and 38 deletions.
7 changes: 5 additions & 2 deletions ...r/che-core-api-agent/src/main/java/org/eclipse/che/api/agent/server/impl/AgentSorter.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@
import org.eclipse.che.api.agent.server.model.impl.AgentKeyImpl;
import org.eclipse.che.api.agent.shared.model.Agent;
import org.eclipse.che.api.agent.shared.model.AgentKey;
import org.eclipse.che.commons.annotation.Nullable;

import java.util.ArrayList;
import java.util.HashSet;
Expand Down Expand Up @@ -54,13 +55,15 @@ public AgentSorter(AgentRegistry agentRegistry) {
* @throws AgentException
* if circular dependency found or agent creation failed or other unexpected error
*/
public List<AgentKey> sort(List<String> agentKeys) throws AgentException {
public List<AgentKey> sort(@Nullable List<String> agentKeys) throws AgentException {
List<AgentKey> sorted = new ArrayList<>();
Set<String> pending = new HashSet<>();

if (agentKeys != null) {
for (String agentKey : agentKeys) {
doSort(agentKeys, AgentKeyImpl.parse(agentKey), sorted, pending);
if (agentKey != null) {
doSort(agentKeys, AgentKeyImpl.parse(agentKey), sorted, pending);
}
}
}

Expand Down
3 changes: 2 additions & 1 deletion ...pi-workspace/src/main/java/org/eclipse/che/api/environment/server/AgentConfigApplier.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@
import org.eclipse.che.api.agent.shared.model.AgentKey;
import org.eclipse.che.api.core.model.workspace.compose.ComposeService;
import org.eclipse.che.api.environment.server.compose.model.ComposeServiceImpl;
import org.eclipse.che.commons.annotation.Nullable;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

Expand Down Expand Up @@ -76,7 +77,7 @@ public AgentConfigApplier(AgentSorter sorter, AgentRegistry agentRegistry) {
*
* @throws AgentException
*/
public void modify(ComposeServiceImpl composeService, List<String> agentKeys) throws AgentException {
public void modify(ComposeServiceImpl composeService, @Nullable List<String> agentKeys) throws AgentException {
for (AgentKey agentKey : sorter.sort(agentKeys)) {
Agent agent = agentRegistry.getAgent(agentKey);
addEnv(composeService, agent.getProperties());
Expand Down
77 changes: 60 additions & 17 deletions ...-workspace/src/main/java/org/eclipse/che/api/environment/server/CheEnvironmentEngine.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@
import org.eclipse.che.api.core.model.machine.MachineStatus;
import org.eclipse.che.api.core.model.machine.ServerConf;
import org.eclipse.che.api.core.model.workspace.Environment;
import org.eclipse.che.api.core.model.workspace.ExtendedMachine;
import org.eclipse.che.api.core.notification.EventService;
import org.eclipse.che.api.core.notification.EventSubscriber;
import org.eclipse.che.api.core.util.AbstractLineConsumer;
Expand All @@ -50,8 +51,11 @@
import org.eclipse.che.api.machine.server.model.impl.SnapshotImpl;
import org.eclipse.che.api.machine.server.spi.Instance;
import org.eclipse.che.api.machine.server.spi.InstanceProvider;
import org.eclipse.che.api.machine.server.util.RecipeDownloader;
import org.eclipse.che.api.machine.shared.dto.event.MachineStatusEvent;
import org.eclipse.che.api.workspace.server.StripedLocks;
import org.eclipse.che.api.workspace.server.model.impl.ExtendedMachineImpl;
import org.eclipse.che.commons.annotation.Nullable;
import org.eclipse.che.commons.env.EnvironmentContext;
import org.eclipse.che.commons.lang.IoUtil;
import org.eclipse.che.commons.lang.NameGenerator;
Expand All @@ -73,6 +77,7 @@
import java.util.Queue;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.CopyOnWriteArrayList;
import java.util.regex.Pattern;

import static java.lang.String.format;
import static org.eclipse.che.api.machine.server.event.InstanceStateEvent.Type.DIE;
Expand Down Expand Up @@ -102,6 +107,8 @@ public class CheEnvironmentEngine {
private final ComposeServicesStartStrategy startStrategy;
private final ComposeMachineInstanceProvider composeProvider;
private final AgentConfigApplier agentConfigApplier;
private final RecipeDownloader recipeDownloader;
private final Pattern recipeApiPattern;

private volatile boolean isPreDestroyInvoked;

Expand All @@ -114,19 +121,24 @@ public CheEnvironmentEngine(SnapshotDao snapshotDao,
EnvironmentParser environmentParser,
ComposeServicesStartStrategy startStrategy,
ComposeMachineInstanceProvider composeProvider,
AgentConfigApplier agentConfigApplier) {
AgentConfigApplier agentConfigApplier,
@Named("api.endpoint") String apiEndpoint,
RecipeDownloader recipeDownloader) {
this.snapshotDao = snapshotDao;
this.eventService = eventService;
this.environmentParser = environmentParser;
this.startStrategy = startStrategy;
this.composeProvider = composeProvider;
this.agentConfigApplier = agentConfigApplier;
this.recipeDownloader = recipeDownloader;
this.environments = new ConcurrentHashMap<>();
this.machineInstanceProviders = machineInstanceProviders;
this.machineLogsDir = new File(machineLogsDir);
this.defaultMachineMemorySizeBytes = Size.parseSize(defaultMachineMemorySizeMB + "MB");
// 16 - experimental value for stripes count, it comes from default hash map size
this.stripedLocks = new StripedLocks(16);
this.recipeApiPattern = Pattern.compile("^" + apiEndpoint + "/recipe/.*$");

eventService.subscribe(new MachineCleaner());
}

Expand Down Expand Up @@ -289,9 +301,11 @@ public void stop(String workspaceId) throws EnvironmentNotRunningException,
* if any other error occurs
*/
public Instance startMachine(String workspaceId,
MachineConfig machineConfig) throws ServerException,
NotFoundException,
ConflictException {
MachineConfig machineConfig,
List<String> agents) throws ServerException,
NotFoundException,
ConflictException {

MachineConfig machineConfigCopy = new MachineConfigImpl(machineConfig);
EnvironmentHolder environmentHolder;
try (StripedLocks.ReadLock lock = stripedLocks.acquireReadLock(workspaceId)) {
Expand Down Expand Up @@ -325,8 +339,16 @@ public Instance startMachine(String workspaceId,
// needed to reuse startInstance method and
// create machine instances by different implementation-specific providers
ComposeServiceImpl composeService = machineConfigToComposeService(machineConfig);
normalize(composeService);

machineStarter = (machineLogger, machineSource) -> {
ComposeServiceImpl serviceWithCorrectSource = getServiceWithCorrectSource(composeService, machineSource);

normalize(serviceWithCorrectSource);
ExtendedMachineImpl extendedMachine = new ExtendedMachineImpl();
extendedMachine.setAgents(agents);
applyAgents(extendedMachine, serviceWithCorrectSource);

return composeProvider.startService(namespace,
workspaceId,
environmentHolder.name,
Expand Down Expand Up @@ -489,9 +511,10 @@ private void initializeEnvironment(String workspaceId,
ConflictException {

ComposeEnvironmentImpl composeEnvironment = environmentParser.parse(env);

applyAgents(env, composeEnvironment);

normalizeEnvironment(composeEnvironment);
normalize(composeEnvironment);

List<String> servicesOrder = startStrategy.order(composeEnvironment);

Expand All @@ -510,26 +533,45 @@ private void initializeEnvironment(String workspaceId,
}

private void applyAgents(Environment env, ComposeEnvironmentImpl composeEnvironment) throws ServerException {
for (Map.Entry<String, ComposeServiceImpl> entry : composeEnvironment.getServices().entrySet()) {
String machineName = entry.getKey();
ComposeServiceImpl composeService = entry.getValue();
for (Map.Entry<String, ? extends ExtendedMachine> machineEntry : env.getMachines().entrySet()) {
String machineName = machineEntry.getKey();
ExtendedMachine extendedMachine = machineEntry.getValue();
ComposeServiceImpl service = composeEnvironment.getServices().get(machineName);

List<String> agents = env.getMachines().get(machineName).getAgents();
applyAgents(extendedMachine, service);
}
}

private void applyAgents(@Nullable ExtendedMachine extendedMachine,
ComposeServiceImpl composeService) throws ServerException {
if (extendedMachine != null) {
try {
agentConfigApplier.modify(composeService, agents);
agentConfigApplier.modify(composeService, extendedMachine.getAgents());
} catch (AgentException e) {
throw new ServerException("Can't apply agent config", e);
}
}
}

private void normalizeEnvironment(ComposeEnvironmentImpl composeEnvironment) {
for (Map.Entry<String, ComposeServiceImpl> serviceEntry : composeEnvironment.getServices()
.entrySet()) {
if (serviceEntry.getValue().getMemLimit() == null || serviceEntry.getValue().getMemLimit() == 0) {
serviceEntry.getValue().setMemLimit(defaultMachineMemorySizeBytes);
}
private void normalize(ComposeEnvironmentImpl composeEnvironment) throws ServerException {
for (ComposeServiceImpl service : composeEnvironment.getServices().values()) {
normalize(service);
}
}

private void normalize(ComposeServiceImpl service) throws ServerException {
// set default mem limit for service if it is not set
if (service.getMemLimit() == null || service.getMemLimit() == 0) {
service.setMemLimit(defaultMachineMemorySizeBytes);
}
// download dockerfile if it is hosted by API to avoid problems with unauthorized requests from docker daemon
if (service.getBuild() != null &&
service.getBuild().getContext() != null &&
recipeApiPattern.matcher(service.getBuild().getContext()).matches()) {

String recipeContent = recipeDownloader.getRecipe(service.getBuild().getContext());
service.getBuild().setDockerfile(recipeContent);
service.getBuild().setContext(null);
}
}

Expand Down Expand Up @@ -792,7 +834,8 @@ Instance startMachine(LineConsumer machineLogger,
NotFoundException;
}

private ComposeServiceImpl getServiceWithCorrectSource(ComposeServiceImpl composeService, MachineSource machineSource)
private ComposeServiceImpl getServiceWithCorrectSource(ComposeServiceImpl composeService,
MachineSource machineSource)
throws ServerException {
ComposeServiceImpl serviceWithCorrectSource = composeService;
if (machineSource != null) {
Expand Down
15 changes: 11 additions & 4 deletions ...rkspace/src/main/java/org/eclipse/che/api/environment/server/CheEnvironmentValidator.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -136,14 +136,13 @@ public void validate(String envName, Environment env) throws IllegalArgumentExce
List<String> devMachines = env.getMachines()
.entrySet()
.stream()
.filter(entry -> entry.getValue()
.getAgents()
.contains("org.eclipse.che.ws-agent"))
.filter(entry -> entry.getValue().getAgents() != null &&
entry.getValue().getAgents().contains("org.eclipse.che.ws-agent"))
.map(Map.Entry::getKey)
.collect(toList());

checkArgument(devMachines.size() == 1,
"Environment '%s' should contain exactly 1 machine with org.eclipse.che.ws-agent, but contains '%s'. " +
"Environment '%s' should contain exactly 1 machine with agent 'org.eclipse.che.ws-agent', but contains '%s'. " +
"All machines with this agent: %s",
envName, devMachines.size(), Joiner.on(", ").join(devMachines));

Expand Down Expand Up @@ -276,6 +275,14 @@ private void validateExtendedMachine(ExtendedMachine extendedMachine, String env
});
}

if (extendedMachine.getAgents() != null) {
for (String agent : extendedMachine.getAgents()) {
checkArgument(!isNullOrEmpty(agent),
"Machine '%s' in environment '%s' contains invalid agent '%s'",
machineName, envName, agent);
}
}

}

public void validateMachine(MachineConfig machineCfg) throws IllegalArgumentException {
Expand Down
5 changes: 3 additions & 2 deletions ...e-api-workspace/src/main/java/org/eclipse/che/api/workspace/server/WorkspaceRuntimes.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -407,8 +407,9 @@ public Instance startMachine(String workspaceId,
}
}

Instance instance = environmentEngine.startMachine(workspaceId, machineConfig);
launchAgents(instance, Collections.singletonList("org.eclipse.che.terminal"));
List<String> agents = Collections.singletonList("org.eclipse.che.terminal");
Instance instance = environmentEngine.startMachine(workspaceId, machineConfig, agents);
launchAgents(instance, agents);

try (StripedLocks.WriteLock lock = stripedLocks.acquireWriteLock(workspaceId)) {
WorkspaceState workspaceState = workspaces.get(workspaceId);
Expand Down
Loading

0 comments on commit be851cb

Please sign in to comment.