Skip to content

Commit

Permalink
Issue #1636 Node pools for Azure and GCP (#1641)
Browse files Browse the repository at this point in the history
* Issue #1636 Implement scaling/reassigning in node pool for AzureInstanceService

* Issue #1636 Implement scaling/reassigning in node pool for GCPInstanceService

* Issue #1636 Call suitable method for building node down command in AzureInstanceService

* Issue #1636 Code cleanup for recently implemented methods

* Issue #1636 Add pre_pull_images argument support

* Issue #1636 Add pipe_log changes to common utils

* Issue #1636 Use empty list as a default value for pre_pull_images
  • Loading branch information
Wedds authored Dec 30, 2020
1 parent 4329809 commit 80e22ab
Show file tree
Hide file tree
Showing 12 changed files with 168 additions and 64 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@
import com.epam.pipeline.entity.pipeline.DiskAttachRequest;
import com.epam.pipeline.entity.pipeline.RunInstance;
import com.epam.pipeline.entity.region.AbstractCloudRegion;
import com.epam.pipeline.manager.cluster.KubernetesConstants;
import com.epam.pipeline.manager.cluster.autoscale.AutoscalerServiceImpl;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
Expand All @@ -33,6 +34,7 @@
import java.time.Duration;
import java.time.LocalDateTime;
import java.time.format.DateTimeParseException;
import java.util.Collections;
import java.util.List;
import java.util.Map;
import java.util.Optional;
Expand All @@ -44,6 +46,10 @@ public interface CloudInstanceService<T extends AbstractCloudRegion>
int TIME_DELIMITER = 60;
int TIME_TO_SHUT_DOWN_NODE = 1;

default Map<String, String> getPoolLabels(final NodePool pool) {
return Collections.singletonMap(KubernetesConstants.NODE_POOL_ID_LABEL, String.valueOf(pool.getId()));
}

/**
* Creates new instance using specified cloud and adds it to cluster
* @param runId
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -37,7 +37,6 @@
import com.epam.pipeline.manager.cloud.commands.ClusterCommandService;
import com.epam.pipeline.manager.cloud.commands.NodeUpCommand;
import com.epam.pipeline.manager.cluster.InstanceOfferManager;
import com.epam.pipeline.manager.cluster.KubernetesConstants;
import com.epam.pipeline.manager.execution.SystemParams;
import com.epam.pipeline.manager.preference.PreferenceManager;
import com.epam.pipeline.manager.preference.SystemPreferences;
Expand Down Expand Up @@ -118,9 +117,7 @@ public RunInstance scaleUpPoolNode(final AwsRegion region,
final String nodeIdLabel,
final NodePool node) {
final RunInstance instance = node.toRunInstance();
final Map<String, String> labels = Collections.singletonMap(
KubernetesConstants.NODE_POOL_ID_LABEL, String.valueOf(node.getId()));
final String command = buildNodeUpCommand(region, nodeIdLabel, instance, labels);
final String command = buildNodeUpCommand(region, nodeIdLabel, instance, getPoolLabels(node));
return instanceService.runNodeUpScript(cmdExecutor, null, instance, command, buildScriptEnvVars());
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -48,6 +48,7 @@
import org.springframework.stereotype.Service;

import java.time.LocalDateTime;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
Expand Down Expand Up @@ -105,29 +106,33 @@ public AzureInstanceService(final CommonCloudInstanceService instanceService,
public RunInstance scaleUpNode(final AzureRegion region,
final Long runId,
final RunInstance instance) {
final String command = buildNodeUpCommand(region, runId, instance);
final Map<String, String> envVars = buildScriptAzureEnvVars(region);
return instanceService.runNodeUpScript(cmdExecutor, runId, instance, command, envVars);
final String command = buildNodeUpCommand(region, String.valueOf(runId), instance, Collections.emptyMap());
return instanceService.runNodeUpScript(cmdExecutor, runId, instance, command, buildScriptAzureEnvVars(region));
}

@Override
public RunInstance scaleUpPoolNode(final AzureRegion region,
final String nodeId,
final NodePool node) {
throw new UnsupportedOperationException();
final NodePool nodePool) {
final RunInstance instance = nodePool.toRunInstance();
final String command = buildNodeUpCommand(region, nodeId, instance, getPoolLabels(nodePool));
return instanceService.runNodeUpScript(cmdExecutor, null, instance, command, buildScriptAzureEnvVars(region));
}

@Override
public void scaleDownNode(final AzureRegion region, final Long runId) {
final String command = buildNodeDownCommand(runId);
final String command = buildNodeDownCommand(String.valueOf(runId));
final Map<String, String> envVars = buildScriptAzureEnvVars(region);
CompletableFuture.runAsync(() -> instanceService.runNodeDownScript(cmdExecutor, command, envVars),
executorService.getExecutorService());
}

@Override
public void scaleDownPoolNode(final AzureRegion region, final String nodeLabel) {
throw new UnsupportedOperationException();
final String command = buildNodeDownCommand(nodeLabel);
final Map<String, String> envVars = buildScriptAzureEnvVars(region);
CompletableFuture.runAsync(() -> instanceService.runNodeDownScript(cmdExecutor, command, envVars),
executorService.getExecutorService());
}

@Override
Expand All @@ -139,10 +144,11 @@ public boolean reassignNode(final AzureRegion region, final Long oldId, final Lo
}

@Override
public boolean reassignPoolNode(final AzureRegion region,
final String nodeLabel,
final Long newId) {
throw new UnsupportedOperationException();
public boolean reassignPoolNode(final AzureRegion region, final String nodeLabel, final Long newId) {
final String command = commandService.
buildNodeReassignCommand(nodeReassignScript, nodeLabel, String.valueOf(newId), getProvider().name());
return instanceService.runNodeReassignScript(cmdExecutor, command, nodeLabel,
String.valueOf(newId), buildScriptAzureEnvVars(region));
}

@Override
Expand Down Expand Up @@ -276,19 +282,22 @@ private Map<String, String> buildScriptAzureEnvVars(final AzureRegion region) {
return envVars;
}

private String buildNodeUpCommand(final AzureRegion region, final Long runId, final RunInstance instance) {
private String buildNodeUpCommand(final AzureRegion region, final String nodeLabel, final RunInstance instance,
final Map<String, String> labels) {

final NodeUpCommand.NodeUpCommandBuilder commandBuilder = NodeUpCommand.builder()
.executable(AbstractClusterCommand.EXECUTABLE)
.script(nodeUpScript)
.runId(String.valueOf(runId))
.runId(nodeLabel)
.sshKey(region.getSshPublicKeyPath())
.instanceImage(instance.getNodeImage())
.instanceType(instance.getNodeType())
.instanceDisk(String.valueOf(instance.getEffectiveNodeDisk()))
.kubeIP(kubeMasterIP)
.kubeToken(kubeToken)
.region(region.getRegionCode());
.region(region.getRegionCode())
.prePulledImages(instance.getPrePulledDockerImages())
.additionalLabels(labels);

final Boolean clusterSpotStrategy = instance.getSpot() == null
? preferenceManager.getPreference(SystemPreferences.CLUSTER_SPOT)
Expand All @@ -300,11 +309,11 @@ private String buildNodeUpCommand(final AzureRegion region, final Long runId, fi
return commandBuilder.build().getCommand();
}

private String buildNodeDownCommand(final Long runId) {
private String buildNodeDownCommand(final String nodeLabel) {
return RunIdArgCommand.builder()
.executable(AbstractClusterCommand.EXECUTABLE)
.script(nodeDownScript)
.runId(String.valueOf(runId))
.runId(nodeLabel)
.build()
.getCommand();
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -33,14 +33,6 @@ public ClusterCommandService(@Value("${kube.master.ip}") final String kubeMaster
this.kubeToken = kubeToken;
}

public NodeUpCommand.NodeUpCommandBuilder buildNodeUpCommand(final String nodeUpScript,
final AbstractCloudRegion region,
final Long runId,
final RunInstance instance,
final String cloud) {
return buildNodeUpCommand(nodeUpScript, region, String.valueOf(runId), instance, cloud);
}

public NodeUpCommand.NodeUpCommandBuilder buildNodeUpCommand(final String nodeUpScript,
final AbstractCloudRegion region,
final String nodeLabel,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -44,6 +44,7 @@
import java.nio.file.InvalidPathException;
import java.nio.file.Paths;
import java.time.LocalDateTime;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
Expand Down Expand Up @@ -89,20 +90,15 @@ public GCPInstanceService(final ClusterCommandService commandService,
@Override
public RunInstance scaleUpNode(final GCPRegion region, final Long runId, final RunInstance instance) {

final String command = commandService.buildNodeUpCommand(nodeUpScript, region, runId, instance,
getProviderName()).sshKey(region.getSshPublicKeyPath())
.isSpot(Optional.ofNullable(instance.getSpot())
.orElse(false))
.bidPrice(StringUtils.EMPTY)
.build()
.getCommand();
final Map<String, String> envVars = buildScriptGCPEnvVars(region);
return instanceService.runNodeUpScript(cmdExecutor, runId, instance, command, envVars);
final String command = buildNodeUpCommand(region, String.valueOf(runId), instance, Collections.emptyMap());
return instanceService.runNodeUpScript(cmdExecutor, runId, instance, command, buildScriptGCPEnvVars(region));
}

@Override
public RunInstance scaleUpPoolNode(final GCPRegion region, final String nodeId, final NodePool node) {
throw new UnsupportedOperationException();
final RunInstance instance = node.toRunInstance();
final String command = buildNodeUpCommand(region, nodeId, instance, getPoolLabels(node));
return instanceService.runNodeUpScript(cmdExecutor, null, instance, command, buildScriptGCPEnvVars(region));
}

@Override
Expand All @@ -114,7 +110,8 @@ public void scaleDownNode(final GCPRegion region, final Long runId) {

@Override
public void scaleDownPoolNode(final GCPRegion region, final String nodeLabel) {
throw new UnsupportedOperationException();
final String command = commandService.buildNodeDownCommand(nodeDownScript, nodeLabel, getProviderName());
instanceService.runNodeDownScript(cmdExecutor, command, buildScriptGCPEnvVars(region));
}

@Override
Expand All @@ -127,10 +124,12 @@ public boolean reassignNode(final GCPRegion region, final Long oldId, final Long

@Override
public boolean reassignPoolNode(final GCPRegion region, final String nodeLabel, final Long newId) {
throw new UnsupportedOperationException();
final String command = commandService
.buildNodeReassignCommand(nodeReassignScript, nodeLabel, String.valueOf(newId), getProviderName());
return instanceService.runNodeReassignScript(cmdExecutor, command, nodeLabel, String.valueOf(newId),
buildScriptGCPEnvVars(region));
}


@Override
public void terminateNode(final GCPRegion region, final String internalIp, final String nodeName) {
final String command = commandService.buildTerminateNodeCommand(nodeTerminateScript, internalIp,
Expand Down Expand Up @@ -259,6 +258,20 @@ public CloudInstanceState getInstanceState(final GCPRegion region, final String
}
}

private String buildNodeUpCommand(final GCPRegion region, final String nodeLabel, final RunInstance instance,
final Map<String, String> labels) {
return commandService
.buildNodeUpCommand(nodeUpScript, region, nodeLabel, instance, getProviderName())
.sshKey(region.getSshPublicKeyPath())
.isSpot(Optional.ofNullable(instance.getSpot())
.orElse(false))
.bidPrice(StringUtils.EMPTY)
.additionalLabels(labels)
.prePulledImages(instance.getPrePulledDockerImages())
.build()
.getCommand();
}

private String getCredentialsFilePath(GCPRegion region) {
return StringUtils.isEmpty(region.getAuthFile())
? System.getenv(GOOGLE_APPLICATION_CREDENTIALS)
Expand Down
Loading

0 comments on commit 80e22ab

Please sign in to comment.