Skip to content

Commit

Permalink
chore: enable codebuild workflows
Browse files Browse the repository at this point in the history
  • Loading branch information
karenc-bq committed Apr 13, 2024
1 parent 365e40e commit c98f86b
Show file tree
Hide file tree
Showing 13 changed files with 195 additions and 64 deletions.
76 changes: 76 additions & 0 deletions .github/workflows/run-integration-tests-codebuild.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,76 @@
name: Run Aurora Integration Tests CodeBuild

on:
workflow_dispatch:
push:
branches:
- main
- workflow

concurrency:
group: ${{ github.workflow }}-${{ github.ref }}
cancel-in-progress: true

jobs:
aurora-integration-tests:
strategy:
matrix:
engine_version: ["lts"]
environment: ["pg_integ"]
runs-on: codebuild-jdbcWrapper-${{ github.run_id }}-${{ github.run_attempt }}
environment: ${{ matrix.environment }}
steps:
- name: 'Clone repository'
uses: actions/checkout@v4
with:
fetch-depth: 1
- name: 'Set up JDK 8'
uses: actions/setup-java@v4
with:
distribution: 'corretto'
java-version: 8
- name: 'Configure AWS credentials'
uses: aws-actions/configure-aws-credentials@v4
with:
aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
aws-region: ${{ secrets.AWS_DEFAULT_REGION }}
- name: 'Set up temp AWS credentials'
run: |
creds=($(aws sts get-session-token \
--duration-seconds 21600 \
--query 'Credentials.[AccessKeyId, SecretAccessKey, SessionToken]' \
--output text \
| xargs));
echo "::add-mask::${creds[0]}"
echo "::add-mask::${creds[1]}"
echo "::add-mask::${creds[2]}"
echo "TEMP_AWS_ACCESS_KEY_ID=${creds[0]}" >> $GITHUB_ENV
echo "TEMP_AWS_SECRET_ACCESS_KEY=${creds[1]}" >> $GITHUB_ENV
echo "TEMP_AWS_SESSION_TOKEN=${creds[2]}" >> $GITHUB_ENV
- name: Run integration tests
run: |
./gradlew --no-parallel --no-daemon test-all-aurora
env:
AURORA_CLUSTER_DOMAIN: ${{ secrets.DB_CONN_SUFFIX }}
AURORA_DB_REGION: ${{ secrets.AWS_DEFAULT_REGION }}
AWS_ACCESS_KEY_ID: ${{ env.TEMP_AWS_ACCESS_KEY_ID }}
AWS_SECRET_ACCESS_KEY: ${{ env.TEMP_AWS_SECRET_ACCESS_KEY }}
AWS_SESSION_TOKEN: ${{ env.TEMP_AWS_SESSION_TOKEN }}
RDS_ENDPOINT: ${{ secrets.RDS_ENDPOINT }}
AURORA_MYSQL_DB_ENGINE_VERSION: ${{ matrix.engine_version }}
AURORA_PG_ENGINE_VERSION: ${{ matrix.engine_version }}
- name: 'Archive junit results ${{ matrix.engine_version }}'
if: always()
uses: actions/upload-artifact@v4
with:
name: junit-report-${{ matrix.engine_version }}
path: ./wrapper/build/test-results
retention-days: 5
- name: 'Archive html summary report ${{ matrix.engine_version }}'
if: always()
uses: actions/upload-artifact@v4
with:
name: html-summary-report-${{ matrix.engine_version }}
path: ./wrapper/build/report
retention-days: 5
1 change: 1 addition & 0 deletions examples/AWSDriverExample/build.gradle.kts
Original file line number Diff line number Diff line change
Expand Up @@ -28,4 +28,5 @@ dependencies {
implementation("io.opentelemetry:opentelemetry-exporter-otlp:1.36.0")
implementation("com.amazonaws:aws-xray-recorder-sdk-core:2.15.0")
implementation("org.jsoup:jsoup:1.17.2")
implementation("software.amazon.awssdk:ec2:2.25.2")
}
Original file line number Diff line number Diff line change
Expand Up @@ -22,9 +22,13 @@
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicLong;
import java.util.function.Function;
import java.util.logging.Logger;

public class SlidingExpirationCache<K, V> {

private static final Logger LOGGER =
Logger.getLogger(SlidingExpirationCache.class.getName());

protected final Map<K, CacheItem> cache = new ConcurrentHashMap<>();
protected long cleanupIntervalNanos = TimeUnit.MINUTES.toNanos(10);
protected final AtomicLong cleanupTimeNanos = new AtomicLong(System.nanoTime() + cleanupIntervalNanos);
Expand Down Expand Up @@ -225,6 +229,7 @@ public CacheItem(final V item, final long expirationTimeNano) {
* false.
*/
boolean shouldCleanup() {
LOGGER.finest("shouldCleanup: " + expirationTimeNano);
if (shouldDisposeFunc != null) {
return System.nanoTime() > expirationTimeNano && shouldDisposeFunc.shouldDispose(this.item);
}
Expand Down
9 changes: 9 additions & 0 deletions wrapper/src/test/java/integration/TestEnvironmentInfo.java
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@ public class TestEnvironmentInfo {
private String awsSessionToken;

private String auroraRegion;
private String rdsEndpoint;
private String auroraClusterName;
private String iamUsername;

Expand Down Expand Up @@ -79,6 +80,10 @@ public String getAuroraRegion() {
return this.auroraRegion;
}

public String getRdsEndpoint() {
return this.rdsEndpoint;
}

public String getAuroraClusterName() {
return this.auroraClusterName;
}
Expand All @@ -95,6 +100,10 @@ public void setAuroraRegion(String auroraRegion) {
this.auroraRegion = auroraRegion;
}

public void setRdsEndpoint(String rdsEndpoint) {
this.rdsEndpoint = rdsEndpoint;
}

public void setAuroraClusterName(String auroraClusterName) {
this.auroraClusterName = auroraClusterName;
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -146,7 +146,7 @@ public void beforeEach(ExtensionContext context) throws Exception {
}

if (testRequest.getDatabaseEngineDeployment() == DatabaseEngineDeployment.AURORA) {
AuroraTestUtility auroraUtil = new AuroraTestUtility(testInfo.getAuroraRegion());
AuroraTestUtility auroraUtil = new AuroraTestUtility(testInfo.getAuroraRegion(), testInfo.getRdsEndpoint());
auroraUtil.waitUntilClusterHasRightState(testInfo.getAuroraClusterName());

boolean makeSureFirstInstanceWriter =
Expand All @@ -161,7 +161,7 @@ public void beforeEach(ExtensionContext context) throws Exception {
// Wait up to 5min
long startTimeNano = System.nanoTime();
while ((instanceIDs.size() != testRequest.getNumOfInstances()
|| instanceIDs.size() == 0
|| instanceIDs.isEmpty()
|| !auroraUtil.isDBInstanceWriter(instanceIDs.get(0)))
&& TimeUnit.NANOSECONDS.toMinutes(System.nanoTime() - startTimeNano) < 5) {

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,7 @@
import static software.amazon.jdbc.plugin.failover.FailoverConnectionPlugin.FAILOVER_TIMEOUT_MS;

import integration.TestEnvironmentFeatures;
import integration.TestEnvironmentInfo;
import integration.container.ConnectionStringHelper;
import integration.container.TestDriverProvider;
import integration.container.TestEnvironment;
Expand All @@ -37,6 +38,7 @@
import java.io.FileOutputStream;
import java.io.IOException;
import java.net.InetAddress;
import java.net.URISyntaxException;
import java.net.UnknownHostException;
import java.sql.Connection;
import java.sql.DriverManager;
Expand Down Expand Up @@ -97,8 +99,16 @@ public class AdvancedPerformanceTest {

private static final ConcurrentLinkedQueue<PerfStat> perfDataList = new ConcurrentLinkedQueue<>();

protected static final AuroraTestUtility auroraUtil =
new AuroraTestUtility(TestEnvironment.getCurrent().getInfo().getAuroraRegion());
protected static final AuroraTestUtility auroraUtil;

static {
try {
final TestEnvironmentInfo info = TestEnvironment.getCurrent().getInfo();
auroraUtil = new AuroraTestUtility(info.getAuroraRegion(), info.getRdsEndpoint());
} catch (URISyntaxException e) {
throw new RuntimeException(e);
}
}

private static void doWritePerfDataToFile(
String fileName, ConcurrentLinkedQueue<PerfStat> dataList) throws IOException {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -23,21 +23,7 @@
import static org.junit.jupiter.api.Assertions.assertTrue;

import com.mysql.cj.conf.PropertyKey;
import integration.DatabaseEngine;
import integration.DriverHelper;
import integration.TestEnvironmentFeatures;
import integration.TestInstanceInfo;
import integration.container.ConnectionStringHelper;
import integration.container.ProxyHelper;
import integration.container.TestDriver;
import integration.container.TestDriverProvider;
import integration.container.TestEnvironment;
import integration.container.condition.DisableOnTestFeature;
import integration.container.condition.EnableOnNumOfInstances;
import integration.container.condition.EnableOnTestDriver;
import integration.container.condition.EnableOnTestFeature;
import integration.container.condition.MakeSureFirstInstanceWriter;
import integration.util.AuroraTestUtility;
import java.net.URISyntaxException;
import java.sql.Connection;
import java.sql.DriverManager;
import java.sql.ResultSet;
Expand All @@ -53,6 +39,22 @@
import org.junit.jupiter.api.TestMethodOrder;
import org.junit.jupiter.api.TestTemplate;
import org.junit.jupiter.api.extension.ExtendWith;
import integration.DatabaseEngine;
import integration.DriverHelper;
import integration.TestEnvironmentFeatures;
import integration.TestEnvironmentInfo;
import integration.TestInstanceInfo;
import integration.container.ConnectionStringHelper;
import integration.container.ProxyHelper;
import integration.container.TestDriver;
import integration.container.TestDriverProvider;
import integration.container.TestEnvironment;
import integration.container.condition.DisableOnTestFeature;
import integration.container.condition.EnableOnNumOfInstances;
import integration.container.condition.EnableOnTestDriver;
import integration.container.condition.EnableOnTestFeature;
import integration.container.condition.MakeSureFirstInstanceWriter;
import integration.util.AuroraTestUtility;
import software.amazon.jdbc.PropertyDefinition;
import software.amazon.jdbc.ds.AwsWrapperDataSource;
import software.amazon.jdbc.hostlistprovider.AuroraHostListProvider;
Expand All @@ -73,8 +75,17 @@ public class AuroraFailoverTest {

private static final Logger LOGGER = Logger.getLogger(AuroraFailoverTest.class.getName());

protected static final AuroraTestUtility auroraUtil =
new AuroraTestUtility(TestEnvironment.getCurrent().getInfo().getAuroraRegion());
protected static final AuroraTestUtility auroraUtil;

static {
try {
final TestEnvironmentInfo info = TestEnvironment.getCurrent().getInfo();
auroraUtil = new AuroraTestUtility(info.getAuroraRegion(), info.getRdsEndpoint());
} catch (URISyntaxException e) {
throw new RuntimeException(e);
}
}

protected static final int IS_VALID_TIMEOUT = 5;

protected String currentWriter;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,7 @@
import integration.container.condition.EnableOnTestFeature;
import integration.container.condition.MakeSureFirstInstanceWriter;
import integration.util.AuroraTestUtility;
import java.net.URISyntaxException;
import java.sql.Connection;
import java.sql.DriverManager;
import java.sql.SQLException;
Expand All @@ -62,8 +63,16 @@
@EnableOnNumOfInstances(min = 5)
@MakeSureFirstInstanceWriter
public class AutoscalingTests {
protected static final AuroraTestUtility auroraUtil =
new AuroraTestUtility(TestEnvironment.getCurrent().getInfo().getAuroraRegion());
protected static final AuroraTestUtility auroraUtil;

static {
try {
final TestEnvironmentInfo info = TestEnvironment.getCurrent().getInfo();
auroraUtil = new AuroraTestUtility(info.getAuroraRegion(), info.getRdsEndpoint());
} catch (URISyntaxException e) {
throw new RuntimeException(e);
}
}

protected static Properties getDefaultPropsNoPlugins() {
final Properties props = ConnectionStringHelper.getDefaultProperties();
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -280,6 +280,7 @@ void test_TokenGenerators() {
protected Properties initAwsIamProps(String user, String password) {
final Properties props = ConnectionStringHelper.getDefaultProperties();
props.setProperty(PropertyDefinition.PLUGINS.name, "iam");
props.setProperty(IamAuthConnectionPlugin.IAM_REGION.name, TestEnvironment.getCurrent().getInfo().getAuroraRegion());
props.setProperty(PropertyDefinition.USER.name, user);
props.setProperty(PropertyDefinition.PASSWORD.name, password);
DriverHelper.setTcpKeepAlive(TestEnvironment.getCurrent().getCurrentDriver(), props, false);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -47,6 +47,7 @@
import integration.container.condition.EnableOnTestFeature;
import integration.container.condition.MakeSureFirstInstanceWriter;
import integration.util.AuroraTestUtility;
import java.net.URISyntaxException;
import java.sql.Connection;
import java.sql.DriverManager;
import java.sql.ResultSet;
Expand Down Expand Up @@ -86,8 +87,17 @@
@MakeSureFirstInstanceWriter
public class ReadWriteSplittingTests {

protected static final AuroraTestUtility auroraUtil =
new AuroraTestUtility(TestEnvironment.getCurrent().getInfo().getAuroraRegion());
protected static final AuroraTestUtility auroraUtil;

static {
try {
final TestEnvironmentInfo info = TestEnvironment.getCurrent().getInfo();
auroraUtil = new AuroraTestUtility(info.getAuroraRegion(), info.getRdsEndpoint());
} catch (URISyntaxException e) {
throw new RuntimeException(e);
}
}

private static final Logger LOGGER = Logger.getLogger(ReadWriteSplittingTests.class.getName());

protected static Properties getProxiedPropsWithFailover() {
Expand Down
13 changes: 9 additions & 4 deletions wrapper/src/test/java/integration/host/TestEnvironment.java
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,7 @@
import integration.util.AuroraTestUtility;
import integration.util.ContainerHelper;
import java.io.IOException;
import java.net.URISyntaxException;
import java.net.UnknownHostException;
import java.sql.SQLException;
import java.util.ArrayList;
Expand Down Expand Up @@ -82,6 +83,7 @@ public class TestEnvironment implements AutoCloseable {
private boolean reuseAuroraDbCluster;
private String auroraClusterName; // "cluster-mysql"
private String auroraClusterDomain; // "XYZ.us-west-2.rds.amazonaws.com"
private String rdsEndpoint; // "https://rds-int.amazon.com"

private String awsAccessKeyId;
private String awsSecretAccessKey;
Expand All @@ -103,7 +105,7 @@ private TestEnvironment(TestEnvironmentRequest request) {
this.info.setRequest(request);
}

public static TestEnvironment build(TestEnvironmentRequest request) throws IOException {
public static TestEnvironment build(TestEnvironmentRequest request) throws IOException, URISyntaxException {
LOGGER.finest("Building test env: " + request.getEnvPreCreateIndex());
preCreateEnvironment(request.getEnvPreCreateIndex());

Expand Down Expand Up @@ -155,7 +157,7 @@ public static TestEnvironment build(TestEnvironmentRequest request) throws IOExc
return env;
}

private static TestEnvironment createAuroraEnvironment(TestEnvironmentRequest request) {
private static TestEnvironment createAuroraEnvironment(TestEnvironmentRequest request) throws URISyntaxException {

EnvPreCreateInfo preCreateInfo =
TestEnvironmentProvider.preCreateInfos.get(request.getEnvPreCreateIndex());
Expand Down Expand Up @@ -302,7 +304,7 @@ private static void createDatabaseContainers(TestEnvironment env) {
}
}

private static void createAuroraDbCluster(TestEnvironment env) {
private static void createAuroraDbCluster(TestEnvironment env) throws URISyntaxException {

switch (env.info.getRequest().getDatabaseInstances()) {
case SINGLE_INSTANCE:
Expand All @@ -328,7 +330,7 @@ private static void createAuroraDbCluster(TestEnvironment env) {
}
}

private static void createAuroraDbCluster(TestEnvironment env, int numOfInstances) {
private static void createAuroraDbCluster(TestEnvironment env, int numOfInstances) throws URISyntaxException {

env.info.setAuroraRegion(
!StringUtils.isNullOrEmpty(config.auroraDbRegion)
Expand All @@ -340,6 +342,8 @@ private static void createAuroraDbCluster(TestEnvironment env, int numOfInstance
&& Boolean.parseBoolean(config.reuseAuroraCluster);
env.auroraClusterName = config.auroraClusterName; // "cluster-mysql"
env.auroraClusterDomain = config.auroraClusterDomain; // "XYZ.us-west-2.rds.amazonaws.com"
env.rdsEndpoint = config.rdsEndpoint; // "XYZ.us-west-2.rds.amazonaws.com"
env.info.setRdsEndpoint(env.rdsEndpoint);

if (StringUtils.isNullOrEmpty(env.auroraClusterDomain)) {
throw new RuntimeException("Environment variable AURORA_CLUSTER_DOMAIN is required.");
Expand All @@ -348,6 +352,7 @@ private static void createAuroraDbCluster(TestEnvironment env, int numOfInstance
env.auroraUtil =
new AuroraTestUtility(
env.info.getAuroraRegion(),
env.rdsEndpoint,
env.awsAccessKeyId,
env.awsSecretAccessKey,
env.awsSessionToken);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -78,6 +78,8 @@ public class TestEnvironmentConfiguration {
public String auroraClusterName = System.getenv("AURORA_CLUSTER_NAME"); // "cluster-mysql"
public String auroraClusterDomain =
System.getenv("AURORA_CLUSTER_DOMAIN"); // "XYZ.us-west-2.rds.amazonaws.com"
public String rdsEndpoint =
System.getenv("RDS_ENDPOINT"); // "https://rds-int.amazon.com"

// Expected values: "latest", "lts", or engine version, for example, "15.4"
// If left as empty, will use LTS version
Expand Down
Loading

0 comments on commit c98f86b

Please sign in to comment.