Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Issue #1636 Node pools for Azure and GCP #1641

Merged
merged 7 commits into from
Dec 30, 2020
Merged

Issue #1636 Node pools for Azure and GCP #1641

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
506bafc
Issue #1636 Implement scaling/reassigning in node pool for AzureInsta…
Wedds Dec 8, 2020
4e2226d
Issue #1636 Implement scaling/reassigning in node pool for GCPInstanc…
Wedds Dec 8, 2020
063bb1f
Issue #1636 Call suitable method for building node down command in Az…
Wedds Dec 8, 2020
10f3085
Issue #1636 Code cleanup for recently implemented methods
Wedds Dec 9, 2020
57c3753
Issue #1636 Add pre_pull_images argument support
Wedds Dec 9, 2020
07de210
Issue #1636 Add pipe_log changes to common utils
Wedds Dec 10, 2020
c099d98
Issue #1636 Use empty list as a default value for pre_pull_images
Wedds Dec 10, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 6 additions & 0 deletions api/src/main/java/com/epam/pipeline/manager/cloud/CloudInstanceService.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@
import com.epam.pipeline.entity.pipeline.DiskAttachRequest;
import com.epam.pipeline.entity.pipeline.RunInstance;
import com.epam.pipeline.entity.region.AbstractCloudRegion;
import com.epam.pipeline.manager.cluster.KubernetesConstants;
import com.epam.pipeline.manager.cluster.autoscale.AutoscalerServiceImpl;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
Expand All @@ -32,6 +33,7 @@
import java.time.Duration;
import java.time.LocalDateTime;
import java.time.format.DateTimeParseException;
import java.util.Collections;
import java.util.List;
import java.util.Map;
import java.util.Optional;
Expand All @@ -43,6 +45,10 @@ public interface CloudInstanceService<T extends AbstractCloudRegion>
int TIME_DELIMITER = 60;
int TIME_TO_SHUT_DOWN_NODE = 1;

default Map<String, String> getPoolLabels(final NodePool pool) {
return Collections.singletonMap(KubernetesConstants.NODE_POOL_ID_LABEL, String.valueOf(pool.getId()));
}

/**
* Creates new instance using specified cloud and adds it to cluster
* @param runId
Expand Down
5 changes: 1 addition & 4 deletions api/src/main/java/com/epam/pipeline/manager/cloud/aws/AWSInstanceService.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -36,7 +36,6 @@
import com.epam.pipeline.manager.cloud.commands.ClusterCommandService;
import com.epam.pipeline.manager.cloud.commands.NodeUpCommand;
import com.epam.pipeline.manager.cluster.InstanceOfferManager;
import com.epam.pipeline.manager.cluster.KubernetesConstants;
import com.epam.pipeline.manager.execution.SystemParams;
import com.epam.pipeline.manager.preference.PreferenceManager;
import com.epam.pipeline.manager.preference.SystemPreferences;
Expand Down Expand Up @@ -109,9 +108,7 @@ public RunInstance scaleUpPoolNode(final AwsRegion region,
final String nodeIdLabel,
final NodePool node) {
final RunInstance instance = node.toRunInstance();
final Map<String, String> labels = Collections.singletonMap(
KubernetesConstants.NODE_POOL_ID_LABEL, String.valueOf(node.getId()));
final String command = buildNodeUpCommand(region, nodeIdLabel, instance, labels);
final String command = buildNodeUpCommand(region, nodeIdLabel, instance, getPoolLabels(node));
return instanceService.runNodeUpScript(cmdExecutor, null, instance, command, buildScriptEnvVars());
}

Expand Down
41 changes: 25 additions & 16 deletions api/src/main/java/com/epam/pipeline/manager/cloud/azure/AzureInstanceService.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -47,6 +47,7 @@
import org.springframework.stereotype.Service;

import java.time.LocalDateTime;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
Expand Down Expand Up @@ -104,29 +105,33 @@ public AzureInstanceService(final CommonCloudInstanceService instanceService,
public RunInstance scaleUpNode(final AzureRegion region,
final Long runId,
final RunInstance instance) {
final String command = buildNodeUpCommand(region, runId, instance);
final Map<String, String> envVars = buildScriptAzureEnvVars(region);
return instanceService.runNodeUpScript(cmdExecutor, runId, instance, command, envVars);
final String command = buildNodeUpCommand(region, String.valueOf(runId), instance, Collections.emptyMap());
return instanceService.runNodeUpScript(cmdExecutor, runId, instance, command, buildScriptAzureEnvVars(region));
}

@Override
public RunInstance scaleUpPoolNode(final AzureRegion region,
final String nodeId,
final NodePool node) {
throw new UnsupportedOperationException();
final NodePool nodePool) {
final RunInstance instance = nodePool.toRunInstance();
final String command = buildNodeUpCommand(region, nodeId, instance, getPoolLabels(nodePool));
return instanceService.runNodeUpScript(cmdExecutor, null, instance, command, buildScriptAzureEnvVars(region));
}

@Override
public void scaleDownNode(final AzureRegion region, final Long runId) {
final String command = buildNodeDownCommand(runId);
final String command = buildNodeDownCommand(String.valueOf(runId));
final Map<String, String> envVars = buildScriptAzureEnvVars(region);
CompletableFuture.runAsync(() -> instanceService.runNodeDownScript(cmdExecutor, command, envVars),
executorService.getExecutorService());
}

@Override
public void scaleDownPoolNode(final AzureRegion region, final String nodeLabel) {
throw new UnsupportedOperationException();
final String command = buildNodeDownCommand(nodeLabel);
final Map<String, String> envVars = buildScriptAzureEnvVars(region);
CompletableFuture.runAsync(() -> instanceService.runNodeDownScript(cmdExecutor, command, envVars),
executorService.getExecutorService());
}

@Override
Expand All @@ -138,10 +143,11 @@ public boolean reassignNode(final AzureRegion region, final Long oldId, final Lo
}

@Override
public boolean reassignPoolNode(final AzureRegion region,
final String nodeLabel,
final Long newId) {
throw new UnsupportedOperationException();
public boolean reassignPoolNode(final AzureRegion region, final String nodeLabel, final Long newId) {
final String command = commandService.
buildNodeReassignCommand(nodeReassignScript, nodeLabel, String.valueOf(newId), getProvider().name());
return instanceService.runNodeReassignScript(cmdExecutor, command, nodeLabel,
String.valueOf(newId), buildScriptAzureEnvVars(region));
}

@Override
Expand Down Expand Up @@ -266,19 +272,22 @@ private Map<String, String> buildScriptAzureEnvVars(final AzureRegion region) {
return envVars;
}

private String buildNodeUpCommand(final AzureRegion region, final Long runId, final RunInstance instance) {
private String buildNodeUpCommand(final AzureRegion region, final String nodeLabel, final RunInstance instance,
final Map<String, String> labels) {

final NodeUpCommand.NodeUpCommandBuilder commandBuilder = NodeUpCommand.builder()
.executable(AbstractClusterCommand.EXECUTABLE)
.script(nodeUpScript)
.runId(String.valueOf(runId))
.runId(nodeLabel)
.sshKey(region.getSshPublicKeyPath())
.instanceImage(instance.getNodeImage())
.instanceType(instance.getNodeType())
.instanceDisk(String.valueOf(instance.getEffectiveNodeDisk()))
.kubeIP(kubeMasterIP)
.kubeToken(kubeToken)
.region(region.getRegionCode());
.region(region.getRegionCode())
.prePulledImages(instance.getPrePulledDockerImages())
.additionalLabels(labels);

final Boolean clusterSpotStrategy = instance.getSpot() == null
? preferenceManager.getPreference(SystemPreferences.CLUSTER_SPOT)
Expand All @@ -290,11 +299,11 @@ private String buildNodeUpCommand(final AzureRegion region, final Long runId, fi
return commandBuilder.build().getCommand();
}

private String buildNodeDownCommand(final Long runId) {
private String buildNodeDownCommand(final String nodeLabel) {
return RunIdArgCommand.builder()
.executable(AbstractClusterCommand.EXECUTABLE)
.script(nodeDownScript)
.runId(String.valueOf(runId))
.runId(nodeLabel)
.build()
.getCommand();
}
Expand Down
8 changes: 0 additions & 8 deletions api/src/main/java/com/epam/pipeline/manager/cloud/commands/ClusterCommandService.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -33,14 +33,6 @@ public ClusterCommandService(@Value("${kube.master.ip}") final String kubeMaster
this.kubeToken = kubeToken;
}

public NodeUpCommand.NodeUpCommandBuilder buildNodeUpCommand(final String nodeUpScript,
final AbstractCloudRegion region,
final Long runId,
final RunInstance instance,
final String cloud) {
return buildNodeUpCommand(nodeUpScript, region, String.valueOf(runId), instance, cloud);
}

public NodeUpCommand.NodeUpCommandBuilder buildNodeUpCommand(final String nodeUpScript,
final AbstractCloudRegion region,
final String nodeLabel,
Expand Down
39 changes: 26 additions & 13 deletions api/src/main/java/com/epam/pipeline/manager/cloud/gcp/GCPInstanceService.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -43,6 +43,7 @@
import java.nio.file.InvalidPathException;
import java.nio.file.Paths;
import java.time.LocalDateTime;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
Expand Down Expand Up @@ -88,20 +89,15 @@ public GCPInstanceService(final ClusterCommandService commandService,
@Override
public RunInstance scaleUpNode(final GCPRegion region, final Long runId, final RunInstance instance) {

final String command = commandService.buildNodeUpCommand(nodeUpScript, region, runId, instance,
getProviderName()).sshKey(region.getSshPublicKeyPath())
.isSpot(Optional.ofNullable(instance.getSpot())
.orElse(false))
.bidPrice(StringUtils.EMPTY)
.build()
.getCommand();
final Map<String, String> envVars = buildScriptGCPEnvVars(region);
return instanceService.runNodeUpScript(cmdExecutor, runId, instance, command, envVars);
final String command = buildNodeUpCommand(region, String.valueOf(runId), instance, Collections.emptyMap());
return instanceService.runNodeUpScript(cmdExecutor, runId, instance, command, buildScriptGCPEnvVars(region));
}

@Override
public RunInstance scaleUpPoolNode(final GCPRegion region, final String nodeId, final NodePool node) {
throw new UnsupportedOperationException();
final RunInstance instance = node.toRunInstance();
final String command = buildNodeUpCommand(region, nodeId, instance, getPoolLabels(node));
return instanceService.runNodeUpScript(cmdExecutor, null, instance, command, buildScriptGCPEnvVars(region));
}

@Override
Expand All @@ -113,7 +109,8 @@ public void scaleDownNode(final GCPRegion region, final Long runId) {

@Override
public void scaleDownPoolNode(final GCPRegion region, final String nodeLabel) {
throw new UnsupportedOperationException();
final String command = commandService.buildNodeDownCommand(nodeDownScript, nodeLabel, getProviderName());
instanceService.runNodeDownScript(cmdExecutor, command, buildScriptGCPEnvVars(region));
}

@Override
Expand All @@ -126,10 +123,12 @@ public boolean reassignNode(final GCPRegion region, final Long oldId, final Long

@Override
public boolean reassignPoolNode(final GCPRegion region, final String nodeLabel, final Long newId) {
throw new UnsupportedOperationException();
final String command = commandService
.buildNodeReassignCommand(nodeReassignScript, nodeLabel, String.valueOf(newId), getProviderName());
return instanceService.runNodeReassignScript(cmdExecutor, command, nodeLabel, String.valueOf(newId),
buildScriptGCPEnvVars(region));
}


@Override
public void terminateNode(final GCPRegion region, final String internalIp, final String nodeName) {
final String command = commandService.buildTerminateNodeCommand(nodeTerminateScript, internalIp,
Expand Down Expand Up @@ -248,6 +247,20 @@ public CloudInstanceState getInstanceState(final GCPRegion region, final String
}
}

private String buildNodeUpCommand(final GCPRegion region, final String nodeLabel, final RunInstance instance,
final Map<String, String> labels) {
return commandService
.buildNodeUpCommand(nodeUpScript, region, nodeLabel, instance, getProviderName())
.sshKey(region.getSshPublicKeyPath())
.isSpot(Optional.ofNullable(instance.getSpot())
.orElse(false))
.bidPrice(StringUtils.EMPTY)
.additionalLabels(labels)
.prePulledImages(instance.getPrePulledDockerImages())
.build()
.getCommand();
}

private String getCredentialsFilePath(GCPRegion region) {
return StringUtils.isEmpty(region.getAuthFile())
? System.getenv(GOOGLE_APPLICATION_CREDENTIALS)
Expand Down
Loading