Skip to content

Commit

Permalink
GCP: Implement custom extended machine types - tests
Browse files Browse the repository at this point in the history
  • Loading branch information
@ekazachkova
ekazachkova committed Sep 12, 2023
1 parent 010dfe1 commit 95a5c06
Show file tree
Hide file tree
Showing 3 changed files with 124 additions and 59 deletions.
54 changes: 42 additions & 12 deletions api/src/test/java/com/epam/pipeline/manager/cloud/gcp/GCPCustomMachineExtractorTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@

import com.epam.pipeline.entity.cluster.GpuDevice;
import com.epam.pipeline.entity.region.GCPCustomInstanceType;
import com.epam.pipeline.entity.region.GCPCustomVMType;
import com.epam.pipeline.entity.region.GCPRegion;
import com.epam.pipeline.manager.cloud.gcp.extractor.GCPCustomMachineExtractor;
import com.epam.pipeline.manager.cloud.gcp.extractor.GCPObjectExtractor;
Expand All @@ -40,6 +41,7 @@ public class GCPCustomMachineExtractorTest {
private static final double RAM = 2.0;
private static final double EXTENDED_RAM = 2.0;
private static final double EXTENDED_RAM_FACTOR = 6.5;
private static final double N2_EXTENDED_RAM_FACTOR = 8.0;
private static final int GPU = 3;
private static final String K_80 = "K80";
public static final String CUSTOM_FAMILY = "custom";
Expand All @@ -51,41 +53,67 @@ public class GCPCustomMachineExtractorTest {
public void testCustomCpuMachineExtraction() {
final GCPRegion region = new GCPRegion();
region.setCustomInstanceTypes(Collections.singletonList(GCPCustomInstanceType.withCpu(CPU, RAM)));
final GCPMachine expectedMachine = GCPMachine.withCpu("custom-1-2048", CUSTOM_FAMILY, CPU, RAM, 0, null);
final GCPMachine expectedMachine = GCPMachine.withCpu("custom-1-2048",
CUSTOM_FAMILY, CPU, RAM, 0, GCPCustomVMType.n1);

final List<AbstractGCPObject> actualMachines = extractor.extract(region);

assertThat(actualMachines.size(), is(1));
final AbstractGCPObject actualObject = actualMachines.get(0);
assertThat(actualObject, instanceOf(GCPMachine.class));
assertMachineEquals(expectedMachine, (GCPMachine) actualObject);
assertGCPMachine(expectedMachine, actualMachines);
}

@Test
public void testCustomGpuMachineExtraction() {
final GCPRegion region = new GCPRegion();
region.setCustomInstanceTypes(Collections.singletonList(GCPCustomInstanceType.withGpu(CPU, RAM, GPU, K_80)));
final GCPMachine expectedMachine = GCPMachine.withGpu("gpu-custom-1-2048-k80-3", CUSTOM_FAMILY, CPU, RAM, 0,
GPU, GpuDevice.from(K_80, NVIDIA), null);
GPU, GpuDevice.from(K_80, NVIDIA), GCPCustomVMType.n1);

final List<AbstractGCPObject> actualMachines = extractor.extract(region);

assertThat(actualMachines.size(), is(1));
final AbstractGCPObject actualObject = actualMachines.get(0);
assertThat(actualObject, instanceOf(GCPMachine.class));
assertMachineEquals(expectedMachine, (GCPMachine) actualObject);
assertGCPMachine(expectedMachine, actualMachines);
}

@Test
public void testExtendedCustomMachineExtraction() {
final GCPRegion region = new GCPRegion();
region.setCustomInstanceTypes(Collections.singletonList(GCPCustomInstanceType.withCpu(2 * CPU,
EXTENDED_RAM_FACTOR * 2 + EXTENDED_RAM)));
final GCPMachine expectedMachine = GCPMachine.withCpu("gpu-custom-1-2048-k80-3", CUSTOM_FAMILY, 2 * CPU,
EXTENDED_RAM_FACTOR * 2, EXTENDED_RAM, null);
final GCPMachine expectedMachine = GCPMachine.withCpu("custom-2-15360-ext", CUSTOM_FAMILY, 2 * CPU,
EXTENDED_RAM_FACTOR * 2, EXTENDED_RAM, GCPCustomVMType.n1);

final List<AbstractGCPObject> actualMachines = extractor.extract(region);

assertGCPMachine(expectedMachine, actualMachines);
}

@Test
public void testExtendedCustomGpuMachineExtraction() {
final GCPRegion region = new GCPRegion();
region.setCustomInstanceTypes(Collections.singletonList(GCPCustomInstanceType
.withGpu(2 * CPU, EXTENDED_RAM_FACTOR * 2 + EXTENDED_RAM, GPU, K_80)));
final GCPMachine expectedMachine = GCPMachine.withGpu("gpu-custom-2-15360-k80-3-ext",
CUSTOM_FAMILY, 2 * CPU, EXTENDED_RAM_FACTOR * 2, EXTENDED_RAM,
GPU, GpuDevice.from(K_80, NVIDIA), GCPCustomVMType.n1);

final List<AbstractGCPObject> actualMachines = extractor.extract(region);

assertGCPMachine(expectedMachine, actualMachines);
}

@Test
public void testExtendedCustomMachineWithFamilyExtraction() {
final GCPRegion region = new GCPRegion();
region.setCustomInstanceTypes(Collections.singletonList(GCPCustomInstanceType.withCpu(2 * CPU,
N2_EXTENDED_RAM_FACTOR * 2 + EXTENDED_RAM, "n2")));
final GCPMachine expectedMachine = GCPMachine.withCpu("n2-custom-2-18432-ext", CUSTOM_FAMILY, 2 * CPU,
N2_EXTENDED_RAM_FACTOR * 2, EXTENDED_RAM, GCPCustomVMType.n2);

final List<AbstractGCPObject> actualMachines = extractor.extract(region);

assertGCPMachine(expectedMachine, actualMachines);
}

private void assertGCPMachine(final GCPMachine expectedMachine, final List<AbstractGCPObject> actualMachines) {
assertThat(actualMachines.size(), is(1));
final AbstractGCPObject actualObject = actualMachines.get(0);
assertThat(actualObject, instanceOf(GCPMachine.class));
Expand All @@ -97,5 +125,7 @@ private void assertMachineEquals(final GCPMachine expectedMachine, final GCPMach
assertEquals(actualMachine.getRam(), expectedMachine.getRam(), DELTA);
assertEquals(actualMachine.getExtendedRam(), expectedMachine.getExtendedRam(), DELTA);
assertEquals(actualMachine.getGpu(), expectedMachine.getGpu());
assertEquals(actualMachine.getName(), expectedMachine.getName());
assertEquals(actualMachine.getFamily(), expectedMachine.getFamily());
}
}
123 changes: 76 additions & 47 deletions api/src/test/java/com/epam/pipeline/manager/cloud/gcp/GCPInstancePriceServiceTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@
import com.epam.pipeline.dao.cluster.InstanceOfferDao;
import com.epam.pipeline.entity.cluster.GpuDevice;
import com.epam.pipeline.entity.cluster.InstanceOffer;
import com.epam.pipeline.entity.region.GCPCustomVMType;
import com.epam.pipeline.entity.region.GCPRegion;
import com.epam.pipeline.manager.cloud.CloudInstancePriceService;
import com.epam.pipeline.manager.cloud.gcp.extractor.GCPObjectExtractor;
Expand Down Expand Up @@ -88,14 +89,17 @@ public class GCPInstancePriceServiceTest {
private final GCPMachine gpuMachine = GCPMachine.withGpu("a2-highgpu-2g", HIGHGPU_FAMILY, 24, 170, 0, 2,
NVIDIA_A100, null);
private final GCPMachine customCpuMachine = GCPMachine.withCpu("custom-2-15360", CUSTOM_FAMILY, 2, 13, 2, null);
private final GCPMachine customCpuN2Machine = GCPMachine.withCpu("n2-custom-2-15360", CUSTOM_FAMILY,
2, 13, 2, GCPCustomVMType.n2);
private final GCPMachine customGpuMachine = GCPMachine.withGpu("gpu-custom-2-8192-k80-1", CUSTOM_FAMILY, 2, 8, 0, 3,
NVIDIA_K80, null);
private final GCPMachine cpuMachineWithNoPrices = GCPMachine.withCpu("n1-familywithoutprices-1",
"familywithoutprices", 10, 20, 0, null);
private final GCPDisk disk = new GCPDisk("SSD", "SSD");
private final List<AbstractGCPObject> predefinedMachines = Arrays.asList(cpuMachine, gpuMachine,
cpuMachineWithNoPrices);
private final List<AbstractGCPObject> customMachines = Arrays.asList(customGpuMachine, customCpuMachine);
private final List<AbstractGCPObject> customMachines =
Arrays.asList(customGpuMachine, customCpuMachine, customCpuN2Machine);
private final List<AbstractGCPObject> disks = Collections.singletonList(disk);

private final GCPObjectExtractor predefinedMachinesExtractor = mock(GCPObjectExtractor.class);
Expand All @@ -121,6 +125,7 @@ public void setUp() {
"cpu_ondemand_standard", "cpu_preemptible_standard",
"ram_ondemand_standard", "ram_preemptible_standard",
"cpu_ondemand_custom", "ram_ondemand_custom", "extendedram_ondemand_custom",
"cpu_ondemand_n2-custom", "ram_ondemand_n2-custom", "extendedram_ondemand_n2-custom",
"gpu_ondemand_a100", "gpu_preemptible_a100",
"gpu_ondemand_k80", "gpu_preemptible_k80",
"disk_ondemand", "disk_preemptible"));
Expand All @@ -130,35 +135,20 @@ public void setUp() {

@Test
public void refreshShouldGenerateRequestsForExtractedObject() {
final GCPObjectExtractor extractor = mock(GCPObjectExtractor.class);
final GCPMachine machine = GCPMachine.withCpu("name", STANDARD_FAMILY, 2, 8.0, 0, null);
when(extractor.extract(any())).thenReturn(Collections.singletonList(machine));
final GCPInstancePriceService service = getService(extractor);
final List<GCPResourceRequest> expectedRequests = Arrays.asList(
new GCPResourceRequest(GCPResourceType.CPU, GCPBilling.ON_DEMAND, machine, MAPPING),
new GCPResourceRequest(GCPResourceType.RAM, GCPBilling.ON_DEMAND, machine, MAPPING),
new GCPResourceRequest(GCPResourceType.CPU, GCPBilling.PREEMPTIBLE, machine, MAPPING),
new GCPResourceRequest(GCPResourceType.RAM, GCPBilling.PREEMPTIBLE, machine, MAPPING)
);

service.refreshPriceListForRegion(region);

final ArgumentCaptor<List<GCPResourceRequest>> captor = ArgumentCaptor.forClass((Class) List.class);
verify(priceLoader).load(eq(region), captor.capture());
final List<GCPResourceRequest> actualRequests = captor.getValue();
assertThat(actualRequests.size(), is(expectedRequests.size()));
assertTrue(expectedRequests.stream()
.map(request -> actualRequests.stream().filter(request::equals).findFirst())
.allMatch(Optional::isPresent));
refreshShouldGenerateRequestsForExtractedMachines(expectedRequests, Collections.singletonList(machine));
}

@Test
public void refreshShouldGenerateRequestsForAllExtractedObjects() {
final GCPObjectExtractor extractor = mock(GCPObjectExtractor.class);
final GCPMachine machine1 = GCPMachine.withCpu("name", STANDARD_FAMILY, 2, 8.0, 0, null);
final GCPMachine machine2 = GCPMachine.withCpu("another-name", STANDARD_FAMILY, 3, 10.0, 0, null);
when(extractor.extract(any())).thenReturn(Arrays.asList(machine1, machine2));
final GCPInstancePriceService service = getService(extractor);
final List<GCPResourceRequest> expectedRequests = Arrays.asList(
new GCPResourceRequest(GCPResourceType.CPU, GCPBilling.ON_DEMAND, machine1, MAPPING),
new GCPResourceRequest(GCPResourceType.RAM, GCPBilling.ON_DEMAND, machine1, MAPPING),
Expand All @@ -169,16 +159,29 @@ public void refreshShouldGenerateRequestsForAllExtractedObjects() {
new GCPResourceRequest(GCPResourceType.CPU, GCPBilling.PREEMPTIBLE, machine1, MAPPING),
new GCPResourceRequest(GCPResourceType.RAM, GCPBilling.PREEMPTIBLE, machine1, MAPPING)
);
refreshShouldGenerateRequestsForExtractedMachines(expectedRequests, Arrays.asList(machine1, machine2));
}

service.refreshPriceListForRegion(region);
@Test
public void refreshShouldGenerateRequestsForCustomExtractedMachines() {
final List<GCPResourceRequest> expectedRequests = Arrays.asList(
new GCPResourceRequest(GCPResourceType.CPU, GCPBilling.ON_DEMAND, customCpuMachine, MAPPING),
new GCPResourceRequest(GCPResourceType.RAM, GCPBilling.ON_DEMAND, customCpuMachine, MAPPING),
new GCPResourceRequest(GCPResourceType.EXTENDED_RAM, GCPBilling.ON_DEMAND, customCpuMachine, MAPPING)
);
final List<AbstractGCPObject> extractedMachines = Collections.singletonList(customCpuMachine);
refreshShouldGenerateRequestsForExtractedMachines(expectedRequests, extractedMachines);
}

final ArgumentCaptor<List<GCPResourceRequest>> captor = ArgumentCaptor.forClass((Class) List.class);
verify(priceLoader).load(eq(region), captor.capture());
final List<GCPResourceRequest> actualRequests = captor.getValue();
assertThat(actualRequests.size(), is(expectedRequests.size()));
assertTrue(expectedRequests.stream()
.map(request -> actualRequests.stream().filter(request::equals).findFirst())
.allMatch(Optional::isPresent));
@Test
public void refreshShouldGenerateRequestsForCustomExtractedMachinesWithFamily() {
final List<GCPResourceRequest> expectedRequests = Arrays.asList(
new GCPResourceRequest(GCPResourceType.CPU, GCPBilling.ON_DEMAND, customCpuN2Machine, MAPPING),
new GCPResourceRequest(GCPResourceType.RAM, GCPBilling.ON_DEMAND, customCpuN2Machine, MAPPING),
new GCPResourceRequest(GCPResourceType.EXTENDED_RAM, GCPBilling.ON_DEMAND, customCpuN2Machine, MAPPING)
);
final List<AbstractGCPObject> extractedMachines = Collections.singletonList(customCpuN2Machine);
refreshShouldGenerateRequestsForExtractedMachines(expectedRequests, extractedMachines);
}

@Test
Expand Down Expand Up @@ -342,29 +345,12 @@ public void refreshShouldReturnPreemptibleMachineInstanceOffers() {

@Test
public void refreshShouldReturnExtendedCustomMachineInstanceOffers() {
when(priceLoader.load(any(), any())).thenReturn(new HashSet<>(Arrays.asList(
price(GCPResourceType.CPU, GCPBilling.ON_DEMAND, customCpuMachine, CUSTOM_CPU_COST),
price(GCPResourceType.RAM, GCPBilling.ON_DEMAND, customCpuMachine, CUSTOM_RAM_COST),
price(GCPResourceType.EXTENDED_RAM, GCPBilling.ON_DEMAND, customCpuMachine, CUSTOM_EXTENDED_RAM_COST)
)));
final List<InstanceOffer> offers = service.refreshPriceListForRegion(region);

final Optional<InstanceOffer> optionalOffer = offers.stream()
.filter(it -> it.getInstanceType().equals(customCpuMachine.getName()))
.filter(it -> it.getTermType().equals(CloudInstancePriceService.TermType.ON_DEMAND.getName()))
.findFirst();
shouldReturnExtendedCustomMachineInstanceOffers(customCpuMachine);
}

assertTrue(optionalOffer.isPresent());
final InstanceOffer offer = optionalOffer.get();
assertThat(offer.getVCPU(), is(customCpuMachine.getCpu()));
assertThat(offer.getMemory(), is(customCpuMachine.getRam() + customCpuMachine.getExtendedRam()));
assertThat(offer.getGpu(), is(customCpuMachine.getGpu()));
assertTrue(offer.getProductFamily().toLowerCase().contains(INSTANCE_PRODUCT_FAMILY));
final double expectedNanos = CUSTOM_CPU_COST * customCpuMachine.getCpu()
+ CUSTOM_RAM_COST * customCpuMachine.getRam()
+ CUSTOM_EXTENDED_RAM_COST * customCpuMachine.getExtendedRam();
final double expectedPrice = expectedNanos / GIGABYTE;
assertEquals(expectedPrice, offer.getPricePerUnit(), DELTA);
@Test
public void refreshShouldReturnExtendedCustomMachineInstanceOffersWithFamilies() {
shouldReturnExtendedCustomMachineInstanceOffers(customCpuN2Machine);
}

@Test
Expand Down Expand Up @@ -403,4 +389,47 @@ private static GCPRegion defaultRegion() {
region.setRegionCode("us-east1-b");
return region;
}

private void shouldReturnExtendedCustomMachineInstanceOffers(final GCPMachine customMachine) {
when(priceLoader.load(any(), any())).thenReturn(new HashSet<>(Arrays.asList(
price(GCPResourceType.CPU, GCPBilling.ON_DEMAND, customMachine, CUSTOM_CPU_COST),
price(GCPResourceType.RAM, GCPBilling.ON_DEMAND, customMachine, CUSTOM_RAM_COST),
price(GCPResourceType.EXTENDED_RAM, GCPBilling.ON_DEMAND, customMachine, CUSTOM_EXTENDED_RAM_COST)
)));
final List<InstanceOffer> offers = service.refreshPriceListForRegion(region);

final Optional<InstanceOffer> optionalOffer = offers.stream()
.filter(it -> it.getInstanceType().equals(customMachine.getName()))
.filter(it -> it.getTermType().equals(CloudInstancePriceService.TermType.ON_DEMAND.getName()))
.findFirst();

assertTrue(optionalOffer.isPresent());
final InstanceOffer offer = optionalOffer.get();
assertThat(offer.getVCPU(), is(customMachine.getCpu()));
assertThat(offer.getMemory(), is(customMachine.getRam() + customMachine.getExtendedRam()));
assertThat(offer.getGpu(), is(customMachine.getGpu()));
assertTrue(offer.getProductFamily().toLowerCase().contains(INSTANCE_PRODUCT_FAMILY));
final double expectedNanos = CUSTOM_CPU_COST * customMachine.getCpu()
+ CUSTOM_RAM_COST * customMachine.getRam()
+ CUSTOM_EXTENDED_RAM_COST * customMachine.getExtendedRam();
final double expectedPrice = expectedNanos / GIGABYTE;
assertEquals(expectedPrice, offer.getPricePerUnit(), DELTA);
}

private void refreshShouldGenerateRequestsForExtractedMachines(final List<GCPResourceRequest> expectedRequests,
final List<AbstractGCPObject> extractedObjects) {
final GCPObjectExtractor extractor = mock(GCPObjectExtractor.class);
when(extractor.extract(any())).thenReturn(extractedObjects);
final GCPInstancePriceService service = getService(extractor);

service.refreshPriceListForRegion(region);

final ArgumentCaptor<List<GCPResourceRequest>> captor = ArgumentCaptor.forClass((Class) List.class);
verify(priceLoader).load(eq(region), captor.capture());
final List<GCPResourceRequest> actualRequests = captor.getValue();
assertThat(actualRequests.size(), is(expectedRequests.size()));
assertTrue(expectedRequests.stream()
.map(request -> actualRequests.stream().filter(request::equals).findFirst())
.allMatch(Optional::isPresent));
}
}
6 changes: 6 additions & 0 deletions core/src/main/java/com/epam/pipeline/entity/region/GCPCustomInstanceType.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -37,6 +37,12 @@ public static GCPCustomInstanceType withCpu(final int cpu,
return new GCPCustomInstanceType(cpu, ram, 0, null, null);
}

public static GCPCustomInstanceType withCpu(final int cpu,
final double ram,
final String family) {
return new GCPCustomInstanceType(cpu, ram, 0, null, family);
}

public static GCPCustomInstanceType withGpu(final int cpu,
final double ram,
final int gpu,
Expand Down

0 comments on commit 95a5c06

Please sign in to comment.