Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

chore: use the same localstack version in all tests #259

Merged
merged 1 commit into from
Sep 26, 2023
Merged

chore: use the same localstack version in all tests #259

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
31 changes: 31 additions & 0 deletions src/test/java/io/kestra/plugin/aws/AbstractLocalStackTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,31 @@
package io.kestra.plugin.aws;

import io.micronaut.test.extensions.junit5.annotation.MicronautTest;
import org.junit.jupiter.api.AfterAll;
import org.junit.jupiter.api.BeforeAll;
import org.testcontainers.containers.localstack.LocalStackContainer;
import org.testcontainers.junit.jupiter.Testcontainers;
import org.testcontainers.utility.DockerImageName;

@MicronautTest
@Testcontainers
public class AbstractLocalStackTest {
public static String LOCALSTACK_VERSION = "localstack/localstack:1.4.0";

protected static LocalStackContainer localstack;

@BeforeAll
static void startLocalstack() {
localstack = new LocalStackContainer(DockerImageName.parse(LOCALSTACK_VERSION));
// some tests use a real flow with hardcoded configuration, so we have to fix the binding port
localstack.setPortBindings(java.util.List.of("4566:4566"));
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I wonder if we should'nt allow to configure the local stack before starting it (in an abstract protected method) as not all tst need this port binding and some test may need at some point other thing

localstack.start();
}

@AfterAll
static void stopLocalstack() {
if (localstack != null) {
localstack.stop();
}
}
}
60 changes: 30 additions & 30 deletions src/test/java/io/kestra/plugin/aws/cli/AwsCLITest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@
import io.kestra.core.runners.RunContextFactory;
import io.kestra.core.utils.IdUtils;
import io.kestra.core.utils.TestsUtils;
import io.kestra.plugin.aws.s3.AbstractTest;
import io.kestra.plugin.aws.AbstractLocalStackTest;
import io.kestra.plugin.scripts.exec.scripts.models.DockerOptions;
import io.kestra.plugin.scripts.exec.scripts.models.ScriptOutput;
import jakarta.inject.Inject;
Expand All @@ -20,7 +20,7 @@
import static org.hamcrest.Matchers.hasEntry;
import static org.hamcrest.Matchers.is;

public class AwsCLITest extends AbstractTest {
public class AwsCLITest extends AbstractLocalStackTest {
@Inject
private RunContextFactory runContextFactory;

Expand All @@ -31,32 +31,32 @@ void run() throws Exception {
String envValue = "MY_VALUE";

AwsCLI execute = AwsCLI.builder()
.id(IdUtils.create())
.type(AwsCLI.class.getName())
.docker(DockerOptions.builder()
// needed to be able to reach localstack from inside the container
.networkMode("host")
.image("amazon/aws-cli")
.entryPoint(Collections.emptyList())
.build())
.endpointOverride(localstack.getEndpointOverride(LocalStackContainer.Service.S3).toString())
.accessKeyId(localstack.getAccessKey())
.secretKeyId(localstack.getSecretKey())
.region(localstack.getRegion())
.env(Map.of("{{ inputs.envKey }}", "{{ inputs.envValue }}"))
.commands(List.of(
"echo \"::{\\\"outputs\\\":{" +
"\\\"endpoint\\\":\\\"$AWS_ENDPOINT_URL\\\"," +
"\\\"accessKeyId\\\":\\\"$AWS_ACCESS_KEY_ID\\\"," +
"\\\"secretKeyId\\\":\\\"$AWS_SECRET_ACCESS_KEY\\\"," +
"\\\"region\\\":\\\"$AWS_DEFAULT_REGION\\\"," +
"\\\"format\\\":\\\"$AWS_DEFAULT_OUTPUT\\\"," +
"\\\"customEnv\\\":\\\"$" + envKey + "\\\"" +
"}}::\"",
"aws s3 mb s3://test-bucket",
"aws s3api list-buckets | tr -d ' \n' | xargs -0 -I {} echo '::{\"outputs\":{}}::'"
))
.build();
.id(IdUtils.create())
.type(AwsCLI.class.getName())
.docker(DockerOptions.builder()
// needed to be able to reach localstack from inside the container
.networkMode("host")
.image("amazon/aws-cli")
.entryPoint(Collections.emptyList())
.build())
.endpointOverride(localstack.getEndpointOverride(LocalStackContainer.Service.S3).toString())
.accessKeyId(localstack.getAccessKey())
.secretKeyId(localstack.getSecretKey())
.region(localstack.getRegion())
.env(Map.of("{{ inputs.envKey }}", "{{ inputs.envValue }}"))
.commands(List.of(
"echo \"::{\\\"outputs\\\":{" +
"\\\"endpoint\\\":\\\"$AWS_ENDPOINT_URL\\\"," +
"\\\"accessKeyId\\\":\\\"$AWS_ACCESS_KEY_ID\\\"," +
"\\\"secretKeyId\\\":\\\"$AWS_SECRET_ACCESS_KEY\\\"," +
"\\\"region\\\":\\\"$AWS_DEFAULT_REGION\\\"," +
"\\\"format\\\":\\\"$AWS_DEFAULT_OUTPUT\\\"," +
"\\\"customEnv\\\":\\\"$" + envKey + "\\\"" +
"}}::\"",
"aws s3 mb s3://test-bucket",
"aws s3api list-buckets | tr -d ' \n' | xargs -0 -I {} echo '::{\"outputs\":{}}::'"
))
.build();

RunContext runContext = TestsUtils.mockRunContext(runContextFactory, execute, Map.of("envKey", envKey, "envValue", envValue));

Expand All @@ -70,8 +70,8 @@ void run() throws Exception {
assertThat(runOutput.getVars().get("format"), is("json"));
assertThat(runOutput.getVars().get("customEnv"), is(envValue));
assertThat(((Iterable<Map<String, String>>) runOutput.getVars().get("Buckets")), Matchers.allOf(
Matchers.iterableWithSize(1),
Matchers.hasItem(hasEntry("Name", "test-bucket"))
Matchers.iterableWithSize(1),
Matchers.hasItem(hasEntry("Name", "test-bucket"))
));
assertThat(((Map<String, Object>) runOutput.getVars().get("Owner")), hasEntry("DisplayName", "webfile"));
}
Expand Down
25 changes: 3 additions & 22 deletions src/test/java/io/kestra/plugin/aws/dynamodb/AbstractDynamoDbTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,44 +4,25 @@
import io.kestra.core.runners.RunContext;
import io.kestra.core.runners.RunContextFactory;
import io.kestra.core.storages.StorageInterface;
import io.micronaut.context.annotation.Value;
import io.kestra.plugin.aws.AbstractLocalStackTest;
import io.micronaut.test.extensions.junit5.annotation.MicronautTest;
import jakarta.inject.Inject;
import org.junit.jupiter.api.AfterAll;
import org.junit.jupiter.api.BeforeAll;
import org.testcontainers.containers.localstack.LocalStackContainer;
import org.testcontainers.junit.jupiter.Testcontainers;
import org.testcontainers.utility.DockerImageName;
import software.amazon.awssdk.services.dynamodb.model.*;

@MicronautTest
@Testcontainers
public abstract class AbstractDynamoDbTest {
protected static LocalStackContainer localstack;
public abstract class AbstractDynamoDbTest extends AbstractLocalStackTest {

@Inject
protected RunContextFactory runContextFactory;

@Inject
protected StorageInterface storageInterface;

@BeforeAll
static void startLocalstack() {
localstack = new LocalStackContainer(DockerImageName.parse("localstack/localstack:1.3.1"))
.withServices(LocalStackContainer.Service.DYNAMODB);
localstack.start();
}

@AfterAll
static void stopLocalstack() {
if(localstack != null) {
localstack.stop();
}
}

void createTable(RunContext runContext, AbstractDynamoDb dynamoDb) throws IllegalVariableEvaluationException {
try (var dynamoDbClient = dynamoDb.client(runContext)) {
if(!dynamoDbClient.listTables().tableNames().contains("persons")){
if (!dynamoDbClient.listTables().tableNames().contains("persons")) {
var request = CreateTableRequest.builder()
.tableName("persons")
.attributeDefinitions(AttributeDefinition.builder().attributeName("id").attributeType(ScalarAttributeType.S).build())
Expand Down
26 changes: 2 additions & 24 deletions src/test/java/io/kestra/plugin/aws/eventbridge/PutEventsTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,12 +1,10 @@
package io.kestra.plugin.aws.eventbridge;

import com.fasterxml.jackson.annotation.JsonInclude;
import com.fasterxml.jackson.databind.ObjectMapper;
import io.kestra.core.runners.RunContext;
import io.kestra.core.runners.RunContextFactory;
import io.kestra.core.serializers.FileSerde;
import io.kestra.core.serializers.JacksonMapper;
import io.kestra.core.storages.StorageInterface;
import io.kestra.plugin.aws.AbstractLocalStackTest;
import io.kestra.plugin.aws.eventbridge.model.Entry;
import io.micronaut.test.extensions.junit5.annotation.MicronautTest;
import io.reactivex.BackpressureStrategy;
Expand All @@ -16,12 +14,8 @@
import lombok.EqualsAndHashCode;
import lombok.Getter;
import lombok.extern.jackson.Jacksonized;
import org.junit.jupiter.api.AfterAll;
import org.junit.jupiter.api.BeforeAll;
import org.junit.jupiter.api.Test;
import org.testcontainers.containers.localstack.LocalStackContainer;
import org.testcontainers.junit.jupiter.Testcontainers;
import org.testcontainers.utility.DockerImageName;

import java.io.BufferedReader;
import java.io.File;
Expand All @@ -37,29 +31,13 @@

@MicronautTest
@Testcontainers
class PutEventsTest {
private static final ObjectMapper MAPPER = JacksonMapper.ofIon()
.setSerializationInclusion(JsonInclude.Include.ALWAYS);
class PutEventsTest extends AbstractLocalStackTest {

protected static LocalStackContainer localstack;
@Inject
protected RunContextFactory runContextFactory;
@Inject
protected StorageInterface storageInterface;

@BeforeAll
static void startLocalstack() {
localstack = new LocalStackContainer(DockerImageName.parse("localstack/localstack:1.3.1"));
localstack.start();
}

@AfterAll
static void stopLocalstack() {
if (localstack != null) {
localstack.stop();
}
}

private static List<PutEvents.OutputEntry> getOutputEntries(PutEvents put, RunContext runContext) throws Exception {
var output = put.run(runContext);
List<PutEvents.OutputEntry> outputEntries;
Expand Down
7 changes: 4 additions & 3 deletions src/test/java/io/kestra/plugin/aws/kinesis/PutRecordsTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@
import io.kestra.core.serializers.FileSerde;
import io.kestra.core.serializers.JacksonMapper;
import io.kestra.core.storages.StorageInterface;
import io.kestra.plugin.aws.AbstractLocalStackTest;
import io.kestra.plugin.aws.kinesis.model.Record;
import io.micronaut.test.extensions.junit5.annotation.MicronautTest;
import io.reactivex.BackpressureStrategy;
Expand Down Expand Up @@ -46,16 +47,16 @@
class PutRecordsTest {
private static final ObjectMapper MAPPER = JacksonMapper.ofIon()
.setSerializationInclusion(JsonInclude.Include.ALWAYS);
private static LocalStackContainer localstack;

protected static LocalStackContainer localstack;
@Inject
protected RunContextFactory runContextFactory;
@Inject
protected StorageInterface storageInterface;

@BeforeAll
static void startLocalstack() throws IllegalVariableEvaluationException, InterruptedException {
localstack = new LocalStackContainer(DockerImageName.parse("localstack/localstack:1.3.1"));
localstack = new LocalStackContainer(DockerImageName.parse(AbstractLocalStackTest.LOCALSTACK_VERSION));
localstack.start();

KinesisClient client = client(localstack);
Expand Down Expand Up @@ -223,7 +224,7 @@ void runStorageUpperCase() throws Exception {
try (var stream = new FileOutputStream(tempFile)) {
List.of(record, record2, record3).forEach(throwConsumer(r -> FileSerde.write(stream, r)));
}

var put = PutRecords.builder()
.endpointOverride(localstack.getEndpoint().toString())
.region(localstack.getRegion())
Expand Down
53 changes: 16 additions & 37 deletions src/test/java/io/kestra/plugin/aws/lambda/AbstractInvokeTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,39 +1,31 @@
package io.kestra.plugin.aws.lambda;

import java.io.InputStream;
import java.util.stream.Collectors;
import org.junit.jupiter.api.AfterAll;
import org.junit.jupiter.api.BeforeAll;
import org.testcontainers.containers.localstack.LocalStackContainer;
import org.testcontainers.junit.jupiter.Testcontainers;
import org.testcontainers.utility.DockerImageName;
import io.kestra.core.runners.RunContextFactory;
import io.kestra.core.storages.StorageInterface;
import io.kestra.plugin.aws.AbstractLocalStackTest;
import io.micronaut.test.extensions.junit5.annotation.MicronautTest;
import jakarta.inject.Inject;
import org.testcontainers.junit.jupiter.Testcontainers;
import software.amazon.awssdk.core.SdkBytes;
import software.amazon.awssdk.core.waiters.WaiterResponse;
import software.amazon.awssdk.services.lambda.LambdaClient;
import software.amazon.awssdk.services.lambda.model.CreateFunctionRequest;
import software.amazon.awssdk.services.lambda.model.CreateFunctionResponse;
import software.amazon.awssdk.services.lambda.model.FunctionCode;
import software.amazon.awssdk.services.lambda.model.GetFunctionRequest;
import software.amazon.awssdk.services.lambda.model.GetFunctionResponse;
import software.amazon.awssdk.services.lambda.model.Runtime;
import software.amazon.awssdk.services.lambda.model.*;
import software.amazon.awssdk.services.lambda.waiters.LambdaWaiter;

import java.io.InputStream;
import java.util.stream.Collectors;

@MicronautTest
@Testcontainers
public class AbstractInvokeTest {
public class AbstractInvokeTest extends AbstractLocalStackTest {

static final String FUNCTION_NAME = "Test-Lambda";

static final String FUNCTION_ROLE_ARN = "arn:aws:iam::000000000000:role/lambda-role";

static final String FUNCTION_CODE = "lambda/test.py.zip";

protected static LocalStackContainer localstack;

@Inject
protected RunContextFactory runContextFactory;

Expand All @@ -42,24 +34,11 @@ public class AbstractInvokeTest {

protected String functionArn;

@BeforeAll
static void startLocalstack() {
localstack = new LocalStackContainer(DockerImageName.parse("localstack/localstack:1.3.1"))
.withServices(LocalStackContainer.Service.LAMBDA);
localstack.start();
}

@AfterAll
static void stopLocalstack() {
if (localstack != null) {
localstack.stop();
}
}

void createFunction(LambdaClient client) {
if (client.listFunctions().functions().stream()
.filter(config -> config.functionName().equals(FUNCTION_NAME))
.collect(Collectors.toList()).size() == 0) {
.filter(config -> config.functionName().equals(FUNCTION_NAME))
.collect(Collectors.toList()).size() == 0) {
LambdaWaiter waiter = client.waiter();

InputStream is = getClass().getClassLoader().getResourceAsStream(FUNCTION_CODE);
Expand All @@ -69,18 +48,18 @@ void createFunction(LambdaClient client) {
FunctionCode code = FunctionCode.builder().zipFile(codeToUpload).build();

CreateFunctionRequest functionRequest = CreateFunctionRequest.builder()
.functionName(FUNCTION_NAME).description("Created by the Lambda Java API")
.code(code).handler("test.handler")
.role(FUNCTION_ROLE_ARN)
.runtime(Runtime.PYTHON3_9)
.build();
.functionName(FUNCTION_NAME).description("Created by the Lambda Java API")
.code(code).handler("test.handler")
.role(FUNCTION_ROLE_ARN)
.runtime(Runtime.PYTHON3_9)
.build();

// Create a Lambda function using a waiter.
CreateFunctionResponse functionResponse = client.createFunction(functionRequest);
GetFunctionRequest getFunctionRequest =
GetFunctionRequest.builder().functionName(FUNCTION_NAME).build();
GetFunctionRequest.builder().functionName(FUNCTION_NAME).build();
WaiterResponse<GetFunctionResponse> waiterResponse =
waiter.waitUntilFunctionExists(getFunctionRequest);
waiter.waitUntilFunctionExists(getFunctionRequest);
waiterResponse.matched().response().ifPresent(System.out::println);
// FYI ARN can be found as follows
//functionArn = functionResponse.functionArn();
Expand Down
Loading