diff --git a/CHANGELOG.md b/CHANGELOG.md index 1268b3fa8f0..d783aca09a3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,7 @@ All notable changes to this project will be documented in this file. - New module: Apache Pulsar ([\#713](https://github.com/testcontainers/testcontainers-java/pull/713)) - Add support for defining container labels ([\#725](https://github.com/testcontainers/testcontainers-java/pull/725)) - Use `quay.io/testcontainers/ryuk` instead of `bsideup/ryuk` ([\#721](https://github.com/testcontainers/testcontainers-java/pull/721)) +- Added Couchbase module ([\#688](https://github.com/testcontainers/testcontainers-java/pull/688)) ## [1.7.3] - 2018-05-16 diff --git a/modules/couchbase/AUTHORS b/modules/couchbase/AUTHORS new file mode 100644 index 00000000000..db292de72f1 --- /dev/null +++ b/modules/couchbase/AUTHORS @@ -0,0 +1 @@ +Tayeb Chlyah diff --git a/modules/couchbase/README.md b/modules/couchbase/README.md new file mode 100644 index 00000000000..7a18a235eb4 --- /dev/null +++ b/modules/couchbase/README.md @@ -0,0 +1,59 @@ + + +# TestContainers Couchbase Module +Testcontainers module for Couchbase. [Couchbase](https://www.couchbase.com/) is a Document oriented NoSQL database. + +## Usage example + +Running Couchbase as a stand-in in a test: + +### Create you own bucket + +```java +public class SomeTest { + + @Rule + public CouchbaseContainer couchbase = new CouchbaseContainer() + .withNewBucket(DefaultBucketSettings.builder() + .enableFlush(true) + .name('bucket-name') + .quota(100) + .type(BucketType.COUCHBASE) + .build()); + + @Test + public void someTestMethod() { + Bucket bucket = couchbase.getCouchbaseCluster().openBucket('bucket-name') + + ... interact with client as if using Couchbase normally +``` + +### Use preconfigured default bucket + +Bucket is cleared after each test + +```java +public class SomeTest extends AbstractCouchbaseTest { + + @Test + public void someTestMethod() { + Bucket bucket = getBucket(); + + ... interact with client as if using Couchbase normally +``` + +### Special consideration + +Couchbase container is configured to use random available [ports](https://developer.couchbase.com/documentation/server/current/install/install-ports.html) for some ports only, as [Couchbase Java SDK](https://developer.couchbase.com/documentation/server/current/sdk/java/start-using-sdk.html) permit to configure only some ports : +- **8091** : REST/HTTP traffic ([bootstrapHttpDirectPort](http://docs.couchbase.com/sdk-api/couchbase-java-client-2.4.6/com/couchbase/client/java/env/DefaultCouchbaseEnvironment.Builder.html#bootstrapCarrierDirectPort-int-)) +- **18091** : REST/HTTP traffic with SSL ([bootstrapHttpSslPort](http://docs.couchbase.com/sdk-api/couchbase-java-client-2.4.6/com/couchbase/client/java/env/DefaultCouchbaseEnvironment.Builder.html#bootstrapCarrierSslPort-int-)) +- **11210** : memcached ([bootstrapCarrierDirectPort](http://docs.couchbase.com/sdk-api/couchbase-java-client-2.4.6/com/couchbase/client/java/env/DefaultCouchbaseEnvironment.Builder.html#bootstrapCarrierDirectPort-int-)) +- **11207** : memcached SSL ([bootstrapCarrierSslPort](http://docs.couchbase.com/sdk-api/couchbase-java-client-2.4.6/com/couchbase/client/java/env/DefaultCouchbaseEnvironment.Builder.html#bootstrapCarrierSslPort-int-)) + +All other ports cannot be changed by Java SDK, there are sadly fixed : +- **8092** : Queries, views, XDCR +- **8093** : REST/HTTP Query service +- **8094** : REST/HTTP Search Service +- **8095** : REST/HTTP Analytic service + +So if you disable Query, Search and Analytic service, you can run multiple instance of this container, otherwise, you're stuck with one instance, for now. diff --git a/modules/couchbase/build.gradle b/modules/couchbase/build.gradle new file mode 100644 index 00000000000..218adce2d5a --- /dev/null +++ b/modules/couchbase/build.gradle @@ -0,0 +1,6 @@ +description = "Testcontainers :: Couchbase" + +dependencies { + compile project(':testcontainers') + compile 'com.couchbase.client:java-client:2.5.7' +} diff --git a/modules/couchbase/src/main/java/org/testcontainers/couchbase/AbstractCouchbaseTest.java b/modules/couchbase/src/main/java/org/testcontainers/couchbase/AbstractCouchbaseTest.java new file mode 100644 index 00000000000..b8cdaa14cb2 --- /dev/null +++ b/modules/couchbase/src/main/java/org/testcontainers/couchbase/AbstractCouchbaseTest.java @@ -0,0 +1,59 @@ +package org.testcontainers.couchbase; + +import com.couchbase.client.java.Bucket; +import com.couchbase.client.java.CouchbaseCluster; +import com.couchbase.client.java.bucket.BucketType; +import com.couchbase.client.java.cluster.DefaultBucketSettings; +import com.couchbase.client.java.query.N1qlParams; +import com.couchbase.client.java.query.N1qlQuery; +import com.couchbase.client.java.query.consistency.ScanConsistency; +import lombok.Getter; +import org.junit.After; + +/** + * @author ctayeb + */ +public abstract class AbstractCouchbaseTest { + + public static final String TEST_BUCKET = "test"; + + public static final String DEFAULT_PASSWORD = "password"; + + @Getter(lazy = true) + private final static CouchbaseContainer couchbaseContainer = initCouchbaseContainer(); + + @Getter(lazy = true) + private final static Bucket bucket = openBucket(TEST_BUCKET, DEFAULT_PASSWORD); + + @After + public void clear() { + if (getCouchbaseContainer().isIndex() && getCouchbaseContainer().isQuery() && getCouchbaseContainer().isPrimaryIndex()) { + getBucket().query( + N1qlQuery.simple(String.format("DELETE FROM `%s`", getBucket().name()), + N1qlParams.build().consistency(ScanConsistency.STATEMENT_PLUS))); + } else { + getBucket().bucketManager().flush(); + } + } + + private static CouchbaseContainer initCouchbaseContainer() { + CouchbaseContainer couchbaseContainer = new CouchbaseContainer() + .withNewBucket(DefaultBucketSettings.builder() + .enableFlush(true) + .name(TEST_BUCKET) + .password(DEFAULT_PASSWORD) + .quota(100) + .replicas(0) + .type(BucketType.COUCHBASE) + .build()); + couchbaseContainer.start(); + return couchbaseContainer; + } + + private static Bucket openBucket(String bucketName, String password) { + CouchbaseCluster cluster = getCouchbaseContainer().getCouchbaseCluster(); + Bucket bucket = cluster.openBucket(bucketName, password); + Runtime.getRuntime().addShutdownHook(new Thread(bucket::close)); + return bucket; + } +} diff --git a/modules/couchbase/src/main/java/org/testcontainers/couchbase/CouchbaseContainer.java b/modules/couchbase/src/main/java/org/testcontainers/couchbase/CouchbaseContainer.java new file mode 100644 index 00000000000..356bbc91fea --- /dev/null +++ b/modules/couchbase/src/main/java/org/testcontainers/couchbase/CouchbaseContainer.java @@ -0,0 +1,276 @@ +/* + * Copyright (c) 2016 Couchbase, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.testcontainers.couchbase; + +import com.couchbase.client.core.utils.Base64; +import com.couchbase.client.java.Bucket; +import com.couchbase.client.java.CouchbaseCluster; +import com.couchbase.client.java.cluster.*; +import com.couchbase.client.java.env.CouchbaseEnvironment; +import com.couchbase.client.java.env.DefaultCouchbaseEnvironment; +import com.couchbase.client.java.query.Index; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.github.dockerjava.api.command.InspectContainerResponse; +import lombok.AllArgsConstructor; +import lombok.Cleanup; +import lombok.Getter; +import lombok.SneakyThrows; +import lombok.experimental.Wither; +import org.apache.commons.compress.utils.Sets; +import org.jetbrains.annotations.NotNull; +import org.testcontainers.containers.GenericContainer; +import org.testcontainers.containers.wait.strategy.HttpWaitStrategy; + +import java.io.DataOutputStream; +import java.io.IOException; +import java.net.HttpURLConnection; +import java.net.URL; +import java.net.URLEncoder; +import java.util.*; + +import static java.net.HttpURLConnection.HTTP_OK; + +/** + * Based on Laurent Doguin version, + *

+ * optimized by Tayeb Chlyah + */ +@AllArgsConstructor +public class CouchbaseContainer extends GenericContainer { + + public static final String VERSION = "5.1.0"; + public static final ObjectMapper MAPPER = new ObjectMapper(); + + @Wither + private String memoryQuota = "300"; + + @Wither + private String indexMemoryQuota = "300"; + + @Wither + private String clusterUsername = "Administrator"; + + @Wither + private String clusterPassword = "password"; + + @Wither + private boolean keyValue = true; + + @Getter + @Wither + private boolean query = true; + + @Getter + @Wither + private boolean index = true; + + @Getter + @Wither + private boolean primaryIndex = true; + + @Getter + @Wither + private boolean fts = false; + + @Wither + private boolean beerSample = false; + + @Wither + private boolean travelSample = false; + + @Wither + private boolean gamesIMSample = false; + + @Getter(lazy = true) + private final CouchbaseEnvironment couchbaseEnvironment = createCouchbaseEnvironment(); + + @Getter(lazy = true) + private final CouchbaseCluster couchbaseCluster = createCouchbaseCluster(); + + private List newBuckets = new ArrayList<>(); + + private String urlBase; + + public CouchbaseContainer() { + super("couchbase/server:" + VERSION); + } + + public CouchbaseContainer(String containerName) { + super(containerName); + } + + @Override + protected Integer getLivenessCheckPort() { + return getMappedPort(8091); + } + + @Override + public Set getLivenessCheckPortNumbers() { + return Sets.newHashSet(getLivenessCheckPort()); + } + + @Override + protected void configure() { + // Configurable ports + addExposedPorts(11210, 11207, 8091, 18091); + + // Non configurable ports + addFixedExposedPort(8092, 8092); + addFixedExposedPort(8093, 8093); + addFixedExposedPort(8094, 8094); + addFixedExposedPort(8095, 8095); + addFixedExposedPort(18092, 18092); + addFixedExposedPort(18093, 18093); + setWaitStrategy(new HttpWaitStrategy().forPath("/ui/index.html#/")); + } + + public CouchbaseContainer withNewBucket(BucketSettings bucketSettings) { + newBuckets.add(bucketSettings); + return self(); + } + + @SneakyThrows + public void initCluster() { + urlBase = String.format("http://%s:%s", getContainerIpAddress(), getMappedPort(8091)); + String poolURL = "/pools/default"; + String poolPayload = "memoryQuota=" + URLEncoder.encode(memoryQuota, "UTF-8") + "&indexMemoryQuota=" + URLEncoder.encode(indexMemoryQuota, "UTF-8"); + + String setupServicesURL = "/node/controller/setupServices"; + StringBuilder servicePayloadBuilder = new StringBuilder(); + if (keyValue) { + servicePayloadBuilder.append("kv,"); + } + if (query) { + servicePayloadBuilder.append("n1ql,"); + } + if (index) { + servicePayloadBuilder.append("index,"); + } + if (fts) { + servicePayloadBuilder.append("fts,"); + } + String setupServiceContent = "services=" + URLEncoder.encode(servicePayloadBuilder.toString(), "UTF-8"); + + String webSettingsURL = "/settings/web"; + String webSettingsContent = "username=" + URLEncoder.encode(clusterUsername, "UTF-8") + "&password=" + URLEncoder.encode(clusterPassword, "UTF-8") + "&port=8091"; + + String bucketURL = "/sampleBuckets/install"; + + StringBuilder sampleBucketPayloadBuilder = new StringBuilder(); + sampleBucketPayloadBuilder.append('['); + if (travelSample) { + sampleBucketPayloadBuilder.append("\"travel-sample\","); + } + if (beerSample) { + sampleBucketPayloadBuilder.append("\"beer-sample\","); + } + if (gamesIMSample) { + sampleBucketPayloadBuilder.append("\"gamesim-sample\","); + } + sampleBucketPayloadBuilder.append(']'); + + callCouchbaseRestAPI(poolURL, poolPayload); + callCouchbaseRestAPI(setupServicesURL, setupServiceContent); + callCouchbaseRestAPI(webSettingsURL, webSettingsContent); + callCouchbaseRestAPI(bucketURL, sampleBucketPayloadBuilder.toString()); + + createNodeWaitStrategy().waitUntilReady(this); + callCouchbaseRestAPI("/settings/indexes", "indexerThreads=0&logLevel=info&maxRollbackPoints=5&storageMode=memory_optimized"); + } + + @NotNull + private HttpWaitStrategy createNodeWaitStrategy() { + return new HttpWaitStrategy() + .forPath("/pools/default/") + .withBasicCredentials(clusterUsername, clusterPassword) + .forStatusCode(HTTP_OK) + .forResponsePredicate(response -> { + try { + return Optional.of(MAPPER.readTree(response)) + .map(n -> n.at("/nodes/0/status")) + .map(JsonNode::asText) + .map("healthy"::equals) + .orElse(false); + } catch (IOException e) { + logger().error("Unable to parse response {}", response); + return false; + } + }); + } + + public void createBucket(BucketSettings bucketSetting, boolean primaryIndex) { + ClusterManager clusterManager = getCouchbaseCluster().clusterManager(clusterUsername, clusterPassword); + // Insert Bucket + BucketSettings bucketSettings = clusterManager.insertBucket(bucketSetting); + // Insert Bucket admin user + UserSettings userSettings = UserSettings.build() + .password(bucketSetting.password()) + .roles(Collections.singletonList(new UserRole("bucket_admin", bucketSetting.name()))); + try { + clusterManager.upsertUser(AuthDomain.LOCAL, bucketSetting.name(), userSettings); + } catch (Exception e) { + logger().warn("Unable to insert user '" + bucketSetting.name() + "', maybe you are using older version"); + } + if (index) { + Bucket bucket = getCouchbaseCluster().openBucket(bucketSettings.name(), bucketSettings.password()); + new CouchbaseQueryServiceWaitStrategy(bucket).waitUntilReady(this); + if (primaryIndex) { + bucket.query(Index.createPrimaryIndex().on(bucketSetting.name())); + } + } + } + + public void callCouchbaseRestAPI(String url, String payload) throws IOException { + String fullUrl = urlBase + url; + @Cleanup("disconnect") + HttpURLConnection httpConnection = (HttpURLConnection) ((new URL(fullUrl).openConnection())); + httpConnection.setDoOutput(true); + httpConnection.setRequestMethod("POST"); + httpConnection.setRequestProperty("Content-Type", + "application/x-www-form-urlencoded"); + String encoded = Base64.encode((clusterUsername + ":" + clusterPassword).getBytes("UTF-8")); + httpConnection.setRequestProperty("Authorization", "Basic " + encoded); + @Cleanup + DataOutputStream out = new DataOutputStream(httpConnection.getOutputStream()); + out.writeBytes(payload); + out.flush(); + httpConnection.getResponseCode(); + } + + @Override + protected void containerIsStarted(InspectContainerResponse containerInfo) { + if (!newBuckets.isEmpty()) { + for (BucketSettings bucketSetting : newBuckets) { + createBucket(bucketSetting, primaryIndex); + } + } + } + + private CouchbaseCluster createCouchbaseCluster() { + return CouchbaseCluster.create(getCouchbaseEnvironment(), getContainerIpAddress()); + } + + private DefaultCouchbaseEnvironment createCouchbaseEnvironment() { + initCluster(); + return DefaultCouchbaseEnvironment.builder() + .bootstrapCarrierDirectPort(getMappedPort(11210)) + .bootstrapCarrierSslPort(getMappedPort(11207)) + .bootstrapHttpDirectPort(getMappedPort(8091)) + .bootstrapHttpSslPort(getMappedPort(18091)) + .build(); + } +} diff --git a/modules/couchbase/src/main/java/org/testcontainers/couchbase/CouchbaseQueryServiceWaitStrategy.java b/modules/couchbase/src/main/java/org/testcontainers/couchbase/CouchbaseQueryServiceWaitStrategy.java new file mode 100644 index 00000000000..853dd7d8eb7 --- /dev/null +++ b/modules/couchbase/src/main/java/org/testcontainers/couchbase/CouchbaseQueryServiceWaitStrategy.java @@ -0,0 +1,55 @@ +package org.testcontainers.couchbase; + +import com.couchbase.client.core.message.cluster.GetClusterConfigRequest; +import com.couchbase.client.core.message.cluster.GetClusterConfigResponse; +import com.couchbase.client.core.service.ServiceType; +import com.couchbase.client.java.Bucket; +import lombok.extern.slf4j.Slf4j; +import org.rnorth.ducttape.TimeoutException; +import org.testcontainers.containers.ContainerLaunchException; +import org.testcontainers.containers.wait.strategy.AbstractWaitStrategy; + +import java.time.Duration; +import java.util.concurrent.TimeUnit; + +import static org.rnorth.ducttape.unreliables.Unreliables.retryUntilSuccess; + +/** + * @author ctayeb + * Created on 06/06/2017 + */ +@Slf4j +public class CouchbaseQueryServiceWaitStrategy extends AbstractWaitStrategy { + + private final Bucket bucket; + + public CouchbaseQueryServiceWaitStrategy(Bucket bucket) { + this.bucket = bucket; + startupTimeout = Duration.ofSeconds(120); + } + + @Override + protected void waitUntilReady() { + log.info("Waiting for {} seconds for QUERY service", startupTimeout.getSeconds()); + + // try to connect to the URL + try { + retryUntilSuccess((int) startupTimeout.getSeconds(), TimeUnit.SECONDS, () -> { + getRateLimiter().doWhenReady(() -> { + GetClusterConfigResponse clusterConfig = bucket.core() + .send(new GetClusterConfigRequest()) + .toBlocking().single(); + boolean queryServiceEnabled = clusterConfig.config() + .bucketConfig(bucket.name()) + .serviceEnabled(ServiceType.QUERY); + if (!queryServiceEnabled) { + throw new ContainerLaunchException("Query service not ready yet"); + } + }); + return true; + }); + } catch (TimeoutException e) { + throw new ContainerLaunchException("Timed out waiting for QUERY service"); + } + } +} diff --git a/modules/couchbase/src/test/java/org/testcontainers/couchbase/CouchbaseContainerTest.java b/modules/couchbase/src/test/java/org/testcontainers/couchbase/CouchbaseContainerTest.java new file mode 100644 index 00000000000..52660787796 --- /dev/null +++ b/modules/couchbase/src/test/java/org/testcontainers/couchbase/CouchbaseContainerTest.java @@ -0,0 +1,41 @@ +package org.testcontainers.couchbase; + +import com.couchbase.client.java.document.RawJsonDocument; +import com.couchbase.client.java.query.N1qlQuery; +import com.couchbase.client.java.query.N1qlQueryResult; +import com.couchbase.client.java.query.N1qlQueryRow; +import org.junit.Assert; +import org.junit.Test; + +import java.util.List; + +/** + * @author ctayeb + * created on 18/06/2017 + */ +public class CouchbaseContainerTest extends AbstractCouchbaseTest { + + private static final String ID = "toto"; + + private static final String DOCUMENT = "{\"name\":\"toto\"}"; + + @Test + public void shouldInsertDocument() { + RawJsonDocument expected = RawJsonDocument.create(ID, DOCUMENT); + getBucket().upsert(expected); + RawJsonDocument result = getBucket().get(ID, RawJsonDocument.class); + Assert.assertEquals(expected.content(), result.content()); + } + + @Test + public void shouldExecuteN1ql() { + getBucket().query(N1qlQuery.simple("INSERT INTO " + TEST_BUCKET + " (KEY, VALUE) VALUES ('" + ID + "', " + DOCUMENT + ")")); + + N1qlQueryResult query = getBucket().query(N1qlQuery.simple("SELECT * FROM " + TEST_BUCKET + " USE KEYS '" + ID + "'")); + Assert.assertTrue(query.parseSuccess()); + Assert.assertTrue(query.finalSuccess()); + List n1qlQueryRows = query.allRows(); + Assert.assertEquals(1, n1qlQueryRows.size()); + Assert.assertEquals(DOCUMENT, n1qlQueryRows.get(0).value().get(TEST_BUCKET).toString()); + } +} diff --git a/modules/couchbase/src/test/resources/logback-test.xml b/modules/couchbase/src/test/resources/logback-test.xml new file mode 100644 index 00000000000..fe273feb881 --- /dev/null +++ b/modules/couchbase/src/test/resources/logback-test.xml @@ -0,0 +1,29 @@ + + + + + + %d{HH:mm:ss.SSS} %-5level %logger - %msg%n + + + + + + + + + + + + + + + + + + + PROFILER + DENY + + \ No newline at end of file