Skip to content

Commit

Permalink
AWS statefull instances deallocation (#43)
Browse files Browse the repository at this point in the history
  • Loading branch information
@sitay93
sitay93 authored Sep 21, 2023
1 parent 491106b commit 7a4b0c7
Show file tree
Hide file tree
Showing 22 changed files with 754 additions and 48 deletions.
6 changes: 6 additions & 0 deletions JenkinsWiki.adoc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -40,6 +40,12 @@ termination"

[SpotinstPlugin-Versionhistory]
== Version history
[SpotinstPlugin-Version2.2.14(Sep21,2023)]
=== Version 2.2.14 (Sep 21, 2023)

* AWS stateful groups: deallocate stateful instances resources when removed from Jenkins
* Fixed : monitor alert won't show up for Spotinst Clouds Communication Monitor when all groups are properly connected

[SpotinstPlugin-Version2.2.13(Apr19,2023)]
=== Version 2.2.13 (Apr 19, 2023)

Expand Down
68 changes: 68 additions & 0 deletions src/main/java/hudson/plugins/spotinst/api/SpotinstApi.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,10 @@
import hudson.plugins.spotinst.api.infra.*;
import hudson.plugins.spotinst.common.SpotinstContext;
import hudson.plugins.spotinst.model.aws.*;
import hudson.plugins.spotinst.model.aws.stateful.AwsDeallocateStatefulInstanceRequest;
import hudson.plugins.spotinst.model.aws.stateful.AwsStatefulDeallocationConfig;
import hudson.plugins.spotinst.model.aws.stateful.AwsStatefulInstance;
import hudson.plugins.spotinst.model.aws.stateful.AwsStatefulInstancesResponse;
import hudson.plugins.spotinst.model.azure.*;
import hudson.plugins.spotinst.model.gcp.*;
import hudson.plugins.spotinst.model.redis.DeleteGroupControllerResponse;
Expand Down Expand Up @@ -72,6 +76,23 @@ else if (response.getStatusCode() == HttpStatus.SC_UNAUTHORIZED) {
}

//region AWS
public static AwsGroup getAwsGroup(String groupId, String accountId) throws ApiException {
AwsGroup retVal = null;
Map<String, String> headers = buildHeaders();
Map<String, String> queryParams = buildQueryParams(accountId);

RestResponse response =
RestClient.sendGet(SPOTINST_API_HOST + "/aws/ec2/group/" + groupId, headers, queryParams);

AwsGroupResponse instancesResponse = getCastedResponse(response, AwsGroupResponse.class);

if (instancesResponse.getResponse().getItems().size() > 0) {
retVal = instancesResponse.getResponse().getItems().get(0);
}

return retVal;
}

public static List<AwsGroupInstance> getAwsGroupInstances(String groupId, String accountId) throws ApiException {
List<AwsGroupInstance> retVal = new LinkedList<>();
Map<String, String> headers = buildHeaders();
Expand All @@ -89,6 +110,27 @@ public static List<AwsGroupInstance> getAwsGroupInstances(String groupId, String
return retVal;
}


public static List<AwsStatefulInstance> getAwsStatefulInstances(String groupId,
String accountId) throws ApiException {
List<AwsStatefulInstance> retVal = new LinkedList<>();
Map<String, String> headers = buildHeaders();
Map<String, String> queryParams = buildQueryParams(accountId);

RestResponse response =
RestClient.sendGet(SPOTINST_API_HOST + "/aws/ec2/group/" + groupId + "/statefulInstance", headers,
queryParams);

AwsStatefulInstancesResponse instancesResponse =
getCastedResponse(response, AwsStatefulInstancesResponse.class);

if (instancesResponse.getResponse().getItems().size() > 0) {
retVal = instancesResponse.getResponse().getItems();
}

return retVal;
}

public static AwsScaleUpResult awsScaleUp(String groupId, int adjustment, String accountId) throws ApiException {
AwsScaleUpResult retVal = null;
Map<String, String> headers = buildHeaders();
Expand Down Expand Up @@ -128,6 +170,32 @@ public static Boolean awsDetachInstance(String instanceId, String accountId) thr
return retVal;
}


public static Boolean awsDeallocateInstance(String groupId, String statefulInstanceId,
String accountId) throws ApiException {
Map<String, String> headers = buildHeaders();
Map<String, String> queryParams = buildQueryParams(accountId);

AwsDeallocateStatefulInstanceRequest request = new AwsDeallocateStatefulInstanceRequest();
AwsStatefulDeallocationConfig statefulDeallocation = new AwsStatefulDeallocationConfig();
statefulDeallocation.setShouldDeleteImages(true);
statefulDeallocation.setShouldDeleteSnapshots(true);
statefulDeallocation.setShouldDeleteVolumes(true);
statefulDeallocation.setShouldDeleteNetworkInterfaces(true);
request.setStatefulDeallocation(statefulDeallocation);

String body = JsonMapper.toJson(request);

RestResponse response = RestClient.sendPut(
SPOTINST_API_HOST + "/aws/ec2/group/" + groupId + "/statefulInstance/" + statefulInstanceId +
"/deallocate", body, headers, queryParams);

getCastedResponse(response, ApiEmptyResponse.class);
Boolean retVal = true;

return retVal;
}

public static List<AwsInstanceType> getAllAwsInstanceTypes(String accountId) throws ApiException {
List<AwsInstanceType> retVal;
Map<String, String> headers = buildHeaders();
Expand Down
137 changes: 131 additions & 6 deletions src/main/java/hudson/plugins/spotinst/cloud/AwsSpotinstCloud.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,21 +6,27 @@
import hudson.plugins.spotinst.api.infra.JsonMapper;
import hudson.plugins.spotinst.common.ConnectionMethodEnum;
import hudson.plugins.spotinst.common.SpotAwsInstanceTypesHelper;
import hudson.plugins.spotinst.common.stateful.StatefulInstanceStateEnum;
import hudson.plugins.spotinst.model.aws.*;
import hudson.plugins.spotinst.model.aws.stateful.AwsStatefulInstance;
import hudson.plugins.spotinst.model.common.BlResponse;
import hudson.plugins.spotinst.repos.IAwsGroupRepo;
import hudson.plugins.spotinst.repos.RepoManager;
import hudson.plugins.spotinst.slave.*;
import hudson.slaves.ComputerConnector;
import hudson.slaves.EnvironmentVariablesNodeProperty;
import hudson.tools.ToolLocationNodeProperty;
import jenkins.model.Jenkins;
import org.apache.commons.lang.BooleanUtils;
import org.apache.commons.lang.StringUtils;
import org.kohsuke.stapler.DataBoundConstructor;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import javax.annotation.Nonnull;
import java.io.IOException;
import java.util.*;
import java.util.stream.Collectors;

/**
* Created by ohadmuchnik on 20/03/2017.
Expand All @@ -34,6 +40,7 @@ public class AwsSpotinstCloud extends BaseSpotinstCloud {
protected Map<String, Integer> executorsByInstanceType;
private List<? extends SpotinstInstanceWeight> executorsForTypes;
private List<String> invalidInstanceTypes;
private Map<String, AwsStatefulInstance> ssiByInstanceId;
//endregion

//region Constructor
Expand Down Expand Up @@ -100,7 +107,50 @@ List<SpotinstSlave> scaleUp(ProvisionRequest request) {
}

@Override
public Boolean detachInstance(String instanceId) {
protected String getSsiId(String instanceId) {
String retVal = null;
AwsStatefulInstance statefulInstance = getStatefulInstance(instanceId);

if (statefulInstance != null) {
retVal = statefulInstance.getId();
}

return retVal;
}

private AwsStatefulInstance getStatefulInstance(String instanceId) {
AwsStatefulInstance retVal = null;
boolean isInstanceStateful = ssiByInstanceId != null && ssiByInstanceId.containsKey(instanceId);

if (isInstanceStateful) {
retVal = ssiByInstanceId.get(instanceId);
}

return retVal;
}

@Override
public Boolean deallocateInstance(String statefulInstanceId) {
boolean retVal = false;

IAwsGroupRepo awsGroupRepo = RepoManager.getInstance().getAwsGroupRepo();
ApiResponse<Boolean> detachInstanceResponse =
awsGroupRepo.deallocateInstance(groupId, statefulInstanceId, accountId);

if (detachInstanceResponse.isRequestSucceed()) {
LOGGER.info(String.format("Stateful Instance %s deallocated", statefulInstanceId));
retVal = true;
}
else {
LOGGER.error(String.format("Failed to deallocate instance %s. Errors: %s", statefulInstanceId,
detachInstanceResponse.getErrors()));
}

return retVal;
}

@Override
protected Boolean detachInstance(String instanceId) {
Boolean retVal = false;

IAwsGroupRepo awsGroupRepo = RepoManager.getInstance().getAwsGroupRepo();
Expand All @@ -119,7 +169,7 @@ public Boolean detachInstance(String instanceId) {
}

@Override
protected void internalSyncGroupInstances() {
protected void syncGroupInstances() {
IAwsGroupRepo awsGroupRepo = RepoManager.getInstance().getAwsGroupRepo();
ApiResponse<List<AwsGroupInstance>> instancesResponse = awsGroupRepo.getGroupInstances(groupId, this.accountId);

Expand All @@ -136,6 +186,10 @@ protected void internalSyncGroupInstances() {

this.slaveInstancesDetailsByInstanceId = new HashMap<>(slaveInstancesDetailsByInstanceId);

if (isStatefulGroup()) {
syncGroupStatefulInstances();
}

addNewSlaveInstances(instances);
removeOldSlaveInstances(instances);
}
Expand All @@ -145,7 +199,6 @@ protected void internalSyncGroupInstances() {
}
}


@Override
public Map<String, String> getInstanceIpsById() {
Map<String, String> retVal = new HashMap<>();
Expand Down Expand Up @@ -214,6 +267,55 @@ protected int getOverriddenNumberOfExecutors(String instanceType) {
return retVal;
}

@Override
protected BlResponse<Boolean> checkIsStatefulGroup() {
BlResponse<Boolean> retVal;
IAwsGroupRepo awsGroupRepo = RepoManager.getInstance().getAwsGroupRepo();
ApiResponse<AwsGroup> groupResponse = awsGroupRepo.getGroup(groupId, this.accountId);

if (groupResponse.isRequestSucceed()) {
Boolean result = false;
AwsGroup awsGroup = groupResponse.getValue();
AwsGroupStrategy groupStrategy = awsGroup.getStrategy();

if (groupStrategy != null) {
AwsGroupPersistence groupPersistence = groupStrategy.getPersistence();

if (groupPersistence != null) {
result = BooleanUtils.isTrue(groupPersistence.getShouldPersistPrivateIp()) ||
BooleanUtils.isTrue(groupPersistence.getShouldPersistBlockDevices()) ||
BooleanUtils.isTrue(groupPersistence.getShouldPersistRootDevice()) ||
StringUtils.isNotEmpty(groupPersistence.getBlockDevicesMode());
}
}

retVal = new BlResponse<>(result);
}
else {
LOGGER.error(String.format("Failed to get group %s. Errors: %s", groupId, groupResponse.getErrors()));
retVal = new BlResponse<>(false);
}

return retVal;
}

private void syncGroupStatefulInstances() {
IAwsGroupRepo awsGroupRepo = RepoManager.getInstance().getAwsGroupRepo();
ApiResponse<List<AwsStatefulInstance>> statefulInstancesResponse =
awsGroupRepo.getStatefulInstances(groupId, this.accountId);

if (statefulInstancesResponse.isRequestSucceed()) {
List<AwsStatefulInstance> statefulInstances = statefulInstancesResponse.getValue();
this.ssiByInstanceId = statefulInstances.stream().collect(
Collectors.toMap(AwsStatefulInstance::getInstanceId, statefulInstance -> statefulInstance));
}
else {
LOGGER.error(String.format("Failed to get group %s stateful instances. Errors: %s", groupId,
statefulInstancesResponse.getErrors()));
}

}

private List<SpotinstSlave> handleNewAwsSpots(AwsScaleUpResult scaleUpResult, String label) {
List<SpotinstSlave> retVal = new LinkedList<>();

Expand Down Expand Up @@ -253,9 +355,9 @@ private SpotinstSlave handleNewAwsInstance(String instanceId, String instanceTyp
private void addNewSlaveInstances(List<AwsGroupInstance> elastigroupInstances) {
if (elastigroupInstances.size() > 0) {
for (AwsGroupInstance instance : elastigroupInstances) {
Boolean isSlaveExist = isSlaveExistForInstance(instance);
boolean shouldAddSlaveInstance = shouldAddNewSlaveInstance(instance);

if (isSlaveExist == false) {
if (shouldAddSlaveInstance) {
LOGGER.info(String.format("Instance: %s of group: %s doesn't have slave , adding new one",
JsonMapper.toJson(instance), groupId));
addSpotinstSlave(instance);
Expand All @@ -267,6 +369,29 @@ private void addNewSlaveInstances(List<AwsGroupInstance> elastigroupInstances) {
}
}

private Boolean shouldAddNewSlaveInstance(AwsGroupInstance instance) {
boolean retVal;
boolean isSlaveExist = isSlaveExistForInstance(instance);

if (isSlaveExist) {
retVal = false;
}
else {
if (isStatefulGroup()) {
AwsStatefulInstance statefulInstance = getStatefulInstance(instance.getInstanceId());
boolean isStatefulInstanceReadyForUse = statefulInstance != null &&
Objects.equals(statefulInstance.getState(),
StatefulInstanceStateEnum.ACTIVE);
retVal = isStatefulInstanceReadyForUse;
}
else {
retVal = true;
}
}

return retVal;
}

private void removeOldSlaveInstances(List<AwsGroupInstance> elastigroupInstances) {
List<SpotinstSlave> allGroupsSlaves = getAllSpotinstSlaves();

Expand Down Expand Up @@ -326,7 +451,7 @@ private List<String> getGroupInstanceAndSpotIds(List<AwsGroupInstance> elastigro
}

private Boolean isSlaveExistForInstance(AwsGroupInstance instance) {
Boolean retVal = false;
boolean retVal = false;
Node node;

String instanceId = instance.getInstanceId();
Expand Down
20 changes: 18 additions & 2 deletions src/main/java/hudson/plugins/spotinst/cloud/AzureSpotCloud.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@
import hudson.plugins.spotinst.model.azure.AzureGroupVm;
import hudson.plugins.spotinst.model.azure.AzureScaleUpResultNewVm;
import hudson.plugins.spotinst.model.azure.AzureVmSizeEnum;
import hudson.plugins.spotinst.model.common.BlResponse;
import hudson.plugins.spotinst.repos.IAzureVmGroupRepo;
import hudson.plugins.spotinst.repos.RepoManager;
import hudson.plugins.spotinst.slave.SlaveInstanceDetails;
Expand Down Expand Up @@ -86,7 +87,22 @@ List<SpotinstSlave> scaleUp(ProvisionRequest request) {
}

@Override
public Boolean detachInstance(String instanceId) {
protected BlResponse<Boolean> checkIsStatefulGroup() {
return new BlResponse<>(false);
}

@Override
protected String getSsiId(String instanceId) {
return null;//TODO: implement
}

@Override
protected Boolean deallocateInstance(String statefulInstanceId) {
return false;//TODO: implement
}

@Override
protected Boolean detachInstance(String instanceId) {
Boolean retVal = false;
IAzureVmGroupRepo azureVmGroupRepo = RepoManager.getInstance().getAzureVmGroupRepo();
ApiResponse<Boolean> detachVmResponse = azureVmGroupRepo.detachVM(groupId, instanceId, this.accountId);
Expand All @@ -109,7 +125,7 @@ public String getCloudUrl() {
}

@Override
protected void internalSyncGroupInstances() {
protected void syncGroupInstances() {
IAzureVmGroupRepo azureVmGroupRepo = RepoManager.getInstance().getAzureVmGroupRepo();
ApiResponse<List<AzureGroupVm>> instancesResponse = azureVmGroupRepo.getGroupVms(groupId, this.accountId);

Expand Down
Loading

0 comments on commit 7a4b0c7

Please sign in to comment.