Skip to content

Commit

Permalink
Issue #1636 Node pools for Azure and GCP (#1641)
Browse files Browse the repository at this point in the history 
* Issue #1636 Implement scaling/reassigning in node pool for AzureInstanceService

* Issue #1636 Implement scaling/reassigning in node pool for GCPInstanceService

* Issue #1636 Call suitable method for building node down command in AzureInstanceService

* Issue #1636 Code cleanup for recently implemented methods

* Issue #1636 Add pre_pull_images argument support

* Issue #1636 Add pipe_log changes to common utils

* Issue #1636 Use empty list as a default value for pre_pull_images
  • Loading branch information
@Wedds
Wedds authored Dec 30, 2020
1 parent 4329809 commit 80e22ab
Show file tree
Hide file tree
Showing 12 changed files with 168 additions and 64 deletions.
6 changes: 6 additions & 0 deletions api/src/main/java/com/epam/pipeline/manager/cloud/CloudInstanceService.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@
import com.epam.pipeline.entity.pipeline.DiskAttachRequest;
import com.epam.pipeline.entity.pipeline.RunInstance;
import com.epam.pipeline.entity.region.AbstractCloudRegion;
import com.epam.pipeline.manager.cluster.KubernetesConstants;
import com.epam.pipeline.manager.cluster.autoscale.AutoscalerServiceImpl;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
Expand All @@ -33,6 +34,7 @@
import java.time.Duration;
import java.time.LocalDateTime;
import java.time.format.DateTimeParseException;
import java.util.Collections;
import java.util.List;
import java.util.Map;
import java.util.Optional;
Expand All @@ -44,6 +46,10 @@ public interface CloudInstanceService<T extends AbstractCloudRegion>
int TIME_DELIMITER = 60;
int TIME_TO_SHUT_DOWN_NODE = 1;

default Map<String, String> getPoolLabels(final NodePool pool) {
return Collections.singletonMap(KubernetesConstants.NODE_POOL_ID_LABEL, String.valueOf(pool.getId()));
}

/**
* Creates new instance using specified cloud and adds it to cluster
* @param runId
Expand Down
5 changes: 1 addition & 4 deletions api/src/main/java/com/epam/pipeline/manager/cloud/aws/AWSInstanceService.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -37,7 +37,6 @@
import com.epam.pipeline.manager.cloud.commands.ClusterCommandService;
import com.epam.pipeline.manager.cloud.commands.NodeUpCommand;
import com.epam.pipeline.manager.cluster.InstanceOfferManager;
import com.epam.pipeline.manager.cluster.KubernetesConstants;
import com.epam.pipeline.manager.execution.SystemParams;
import com.epam.pipeline.manager.preference.PreferenceManager;
import com.epam.pipeline.manager.preference.SystemPreferences;
Expand Down Expand Up @@ -118,9 +117,7 @@ public RunInstance scaleUpPoolNode(final AwsRegion region,
final String nodeIdLabel,
final NodePool node) {
final RunInstance instance = node.toRunInstance();
final Map<String, String> labels = Collections.singletonMap(
KubernetesConstants.NODE_POOL_ID_LABEL, String.valueOf(node.getId()));
final String command = buildNodeUpCommand(region, nodeIdLabel, instance, labels);
final String command = buildNodeUpCommand(region, nodeIdLabel, instance, getPoolLabels(node));
return instanceService.runNodeUpScript(cmdExecutor, null, instance, command, buildScriptEnvVars());
}

Expand Down
41 changes: 25 additions & 16 deletions api/src/main/java/com/epam/pipeline/manager/cloud/azure/AzureInstanceService.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -48,6 +48,7 @@
import org.springframework.stereotype.Service;

import java.time.LocalDateTime;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
Expand Down Expand Up @@ -105,29 +106,33 @@ public AzureInstanceService(final CommonCloudInstanceService instanceService,
public RunInstance scaleUpNode(final AzureRegion region,
final Long runId,
final RunInstance instance) {
final String command = buildNodeUpCommand(region, runId, instance);
final Map<String, String> envVars = buildScriptAzureEnvVars(region);
return instanceService.runNodeUpScript(cmdExecutor, runId, instance, command, envVars);
final String command = buildNodeUpCommand(region, String.valueOf(runId), instance, Collections.emptyMap());
return instanceService.runNodeUpScript(cmdExecutor, runId, instance, command, buildScriptAzureEnvVars(region));
}

@Override
public RunInstance scaleUpPoolNode(final AzureRegion region,
final String nodeId,
final NodePool node) {
throw new UnsupportedOperationException();
final NodePool nodePool) {
final RunInstance instance = nodePool.toRunInstance();
final String command = buildNodeUpCommand(region, nodeId, instance, getPoolLabels(nodePool));
return instanceService.runNodeUpScript(cmdExecutor, null, instance, command, buildScriptAzureEnvVars(region));
}

@Override
public void scaleDownNode(final AzureRegion region, final Long runId) {
final String command = buildNodeDownCommand(runId);
final String command = buildNodeDownCommand(String.valueOf(runId));
final Map<String, String> envVars = buildScriptAzureEnvVars(region);
CompletableFuture.runAsync(() -> instanceService.runNodeDownScript(cmdExecutor, command, envVars),
executorService.getExecutorService());
}

@Override
public void scaleDownPoolNode(final AzureRegion region, final String nodeLabel) {
throw new UnsupportedOperationException();
final String command = buildNodeDownCommand(nodeLabel);
final Map<String, String> envVars = buildScriptAzureEnvVars(region);
CompletableFuture.runAsync(() -> instanceService.runNodeDownScript(cmdExecutor, command, envVars),
executorService.getExecutorService());
}

@Override
Expand All @@ -139,10 +144,11 @@ public boolean reassignNode(final AzureRegion region, final Long oldId, final Lo
}

@Override
public boolean reassignPoolNode(final AzureRegion region,
final String nodeLabel,
final Long newId) {
throw new UnsupportedOperationException();
public boolean reassignPoolNode(final AzureRegion region, final String nodeLabel, final Long newId) {
final String command = commandService.
buildNodeReassignCommand(nodeReassignScript, nodeLabel, String.valueOf(newId), getProvider().name());
return instanceService.runNodeReassignScript(cmdExecutor, command, nodeLabel,
String.valueOf(newId), buildScriptAzureEnvVars(region));
}

@Override
Expand Down Expand Up @@ -276,19 +282,22 @@ private Map<String, String> buildScriptAzureEnvVars(final AzureRegion region) {
return envVars;
}

private String buildNodeUpCommand(final AzureRegion region, final Long runId, final RunInstance instance) {
private String buildNodeUpCommand(final AzureRegion region, final String nodeLabel, final RunInstance instance,
final Map<String, String> labels) {

final NodeUpCommand.NodeUpCommandBuilder commandBuilder = NodeUpCommand.builder()
.executable(AbstractClusterCommand.EXECUTABLE)
.script(nodeUpScript)
.runId(String.valueOf(runId))
.runId(nodeLabel)
.sshKey(region.getSshPublicKeyPath())
.instanceImage(instance.getNodeImage())
.instanceType(instance.getNodeType())
.instanceDisk(String.valueOf(instance.getEffectiveNodeDisk()))
.kubeIP(kubeMasterIP)
.kubeToken(kubeToken)
.region(region.getRegionCode());
.region(region.getRegionCode())
.prePulledImages(instance.getPrePulledDockerImages())
.additionalLabels(labels);

final Boolean clusterSpotStrategy = instance.getSpot() == null
? preferenceManager.getPreference(SystemPreferences.CLUSTER_SPOT)
Expand All @@ -300,11 +309,11 @@ private String buildNodeUpCommand(final AzureRegion region, final Long runId, fi
return commandBuilder.build().getCommand();
}

private String buildNodeDownCommand(final Long runId) {
private String buildNodeDownCommand(final String nodeLabel) {
return RunIdArgCommand.builder()
.executable(AbstractClusterCommand.EXECUTABLE)
.script(nodeDownScript)
.runId(String.valueOf(runId))
.runId(nodeLabel)
.build()
.getCommand();
}
Expand Down
8 changes: 0 additions & 8 deletions api/src/main/java/com/epam/pipeline/manager/cloud/commands/ClusterCommandService.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -33,14 +33,6 @@ public ClusterCommandService(@Value("${kube.master.ip}") final String kubeMaster
this.kubeToken = kubeToken;
}

public NodeUpCommand.NodeUpCommandBuilder buildNodeUpCommand(final String nodeUpScript,
final AbstractCloudRegion region,
final Long runId,
final RunInstance instance,
final String cloud) {
return buildNodeUpCommand(nodeUpScript, region, String.valueOf(runId), instance, cloud);
}

public NodeUpCommand.NodeUpCommandBuilder buildNodeUpCommand(final String nodeUpScript,
final AbstractCloudRegion region,
final String nodeLabel,
Expand Down
39 changes: 26 additions & 13 deletions api/src/main/java/com/epam/pipeline/manager/cloud/gcp/GCPInstanceService.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -44,6 +44,7 @@
import java.nio.file.InvalidPathException;
import java.nio.file.Paths;
import java.time.LocalDateTime;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
Expand Down Expand Up @@ -89,20 +90,15 @@ public GCPInstanceService(final ClusterCommandService commandService,
@Override
public RunInstance scaleUpNode(final GCPRegion region, final Long runId, final RunInstance instance) {

final String command = commandService.buildNodeUpCommand(nodeUpScript, region, runId, instance,
getProviderName()).sshKey(region.getSshPublicKeyPath())
.isSpot(Optional.ofNullable(instance.getSpot())
.orElse(false))
.bidPrice(StringUtils.EMPTY)
.build()
.getCommand();
final Map<String, String> envVars = buildScriptGCPEnvVars(region);
return instanceService.runNodeUpScript(cmdExecutor, runId, instance, command, envVars);
final String command = buildNodeUpCommand(region, String.valueOf(runId), instance, Collections.emptyMap());
return instanceService.runNodeUpScript(cmdExecutor, runId, instance, command, buildScriptGCPEnvVars(region));
}

@Override
public RunInstance scaleUpPoolNode(final GCPRegion region, final String nodeId, final NodePool node) {
throw new UnsupportedOperationException();
final RunInstance instance = node.toRunInstance();
final String command = buildNodeUpCommand(region, nodeId, instance, getPoolLabels(node));
return instanceService.runNodeUpScript(cmdExecutor, null, instance, command, buildScriptGCPEnvVars(region));
}

@Override
Expand All @@ -114,7 +110,8 @@ public void scaleDownNode(final GCPRegion region, final Long runId) {

@Override
public void scaleDownPoolNode(final GCPRegion region, final String nodeLabel) {
throw new UnsupportedOperationException();
final String command = commandService.buildNodeDownCommand(nodeDownScript, nodeLabel, getProviderName());
instanceService.runNodeDownScript(cmdExecutor, command, buildScriptGCPEnvVars(region));
}

@Override
Expand All @@ -127,10 +124,12 @@ public boolean reassignNode(final GCPRegion region, final Long oldId, final Long

@Override
public boolean reassignPoolNode(final GCPRegion region, final String nodeLabel, final Long newId) {
throw new UnsupportedOperationException();
final String command = commandService
.buildNodeReassignCommand(nodeReassignScript, nodeLabel, String.valueOf(newId), getProviderName());
return instanceService.runNodeReassignScript(cmdExecutor, command, nodeLabel, String.valueOf(newId),
buildScriptGCPEnvVars(region));
}


@Override
public void terminateNode(final GCPRegion region, final String internalIp, final String nodeName) {
final String command = commandService.buildTerminateNodeCommand(nodeTerminateScript, internalIp,
Expand Down Expand Up @@ -259,6 +258,20 @@ public CloudInstanceState getInstanceState(final GCPRegion region, final String
}
}

private String buildNodeUpCommand(final GCPRegion region, final String nodeLabel, final RunInstance instance,
final Map<String, String> labels) {
return commandService
.buildNodeUpCommand(nodeUpScript, region, nodeLabel, instance, getProviderName())
.sshKey(region.getSshPublicKeyPath())
.isSpot(Optional.ofNullable(instance.getSpot())
.orElse(false))
.bidPrice(StringUtils.EMPTY)
.additionalLabels(labels)
.prePulledImages(instance.getPrePulledDockerImages())
.build()
.getCommand();
}

private String getCredentialsFilePath(GCPRegion region) {
return StringUtils.isEmpty(region.getAuthFile())
? System.getenv(GOOGLE_APPLICATION_CREDENTIALS)
Expand Down
Loading

0 comments on commit 80e22ab

Please sign in to comment.