forked from airbytehq/airbyte
-
Notifications
You must be signed in to change notification settings - Fork 0
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
16250 Destination Redis: Add SSH support (airbytehq#17951)
* 16250 Destination Redis: Add SSH support * 16250 Resolve port issue * 11679 Bump version * auto-bump connector version Co-authored-by: Octavia Squidington III <octavia-squidington-iii@users.noreply.github.com>
- Loading branch information
1 parent
224ace1
commit 8aa8506
Showing
9 changed files
with
283 additions
and
5 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
16 changes: 16 additions & 0 deletions
16
.../java/io/airbyte/integrations/destination/redis/SshKeyRedisDestinationAcceptanceTest.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,16 @@ | ||
| /* | ||
| * Copyright (c) 2022 Airbyte, Inc., all rights reserved. | ||
| */ | ||
|
|
||
| package io.airbyte.integrations.destination.redis; | ||
|
|
||
| import io.airbyte.integrations.base.ssh.SshTunnel; | ||
|
|
||
| public class SshKeyRedisDestinationAcceptanceTest extends SshRedisDestinationAcceptanceTest { | ||
|
|
||
| @Override | ||
| public SshTunnel.TunnelMethod getTunnelMethod() { | ||
| return SshTunnel.TunnelMethod.SSH_KEY_AUTH; | ||
| } | ||
|
|
||
| } |
16 changes: 16 additions & 0 deletions
16
.../io/airbyte/integrations/destination/redis/SshPasswordRedisDestinationAcceptanceTest.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,16 @@ | ||
| /* | ||
| * Copyright (c) 2022 Airbyte, Inc., all rights reserved. | ||
| */ | ||
|
|
||
| package io.airbyte.integrations.destination.redis; | ||
|
|
||
| import io.airbyte.integrations.base.ssh.SshTunnel; | ||
|
|
||
| public class SshPasswordRedisDestinationAcceptanceTest extends SshRedisDestinationAcceptanceTest { | ||
|
|
||
| @Override | ||
| public SshTunnel.TunnelMethod getTunnelMethod() { | ||
| return SshTunnel.TunnelMethod.SSH_PASSWORD_AUTH; | ||
| } | ||
|
|
||
| } |
126 changes: 126 additions & 0 deletions
126
...ion/java/io/airbyte/integrations/destination/redis/SshRedisDestinationAcceptanceTest.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,126 @@ | ||
| /* | ||
| * Copyright (c) 2022 Airbyte, Inc., all rights reserved. | ||
| */ | ||
|
|
||
| package io.airbyte.integrations.destination.redis; | ||
|
|
||
| import com.fasterxml.jackson.databind.JsonNode; | ||
| import com.google.common.collect.ImmutableMap; | ||
| import io.airbyte.commons.json.Jsons; | ||
| import io.airbyte.integrations.base.ssh.SshBastionContainer; | ||
| import io.airbyte.integrations.base.ssh.SshTunnel; | ||
| import io.airbyte.integrations.destination.redis.RedisContainerInitializr.RedisContainer; | ||
| import io.airbyte.integrations.standardtest.destination.DestinationAcceptanceTest; | ||
| import io.airbyte.integrations.standardtest.destination.comparator.AdvancedTestDataComparator; | ||
| import io.airbyte.integrations.standardtest.destination.comparator.TestDataComparator; | ||
| import io.airbyte.integrations.util.HostPortResolver; | ||
| import java.util.Comparator; | ||
| import java.util.List; | ||
| import java.util.stream.Collectors; | ||
| import org.junit.jupiter.api.AfterAll; | ||
| import org.junit.jupiter.api.BeforeAll; | ||
| import org.testcontainers.containers.Network; | ||
|
|
||
| public abstract class SshRedisDestinationAcceptanceTest extends DestinationAcceptanceTest { | ||
|
|
||
| private static final SshBastionContainer bastion = new SshBastionContainer(); | ||
| private static final Network network = Network.newNetwork(); | ||
| private static RedisContainerInitializr.RedisContainer redisContainer; | ||
| private JsonNode configJson; | ||
| private RedisCache redisCache; | ||
| private RedisNameTransformer redisNameTransformer; | ||
|
|
||
| @BeforeAll | ||
| static void initContainers() { | ||
| redisContainer = new RedisContainer() | ||
| .withExposedPorts(6379) | ||
| .withNetwork(network); | ||
| redisContainer.start(); | ||
| bastion.initAndStartBastion(network); | ||
| } | ||
|
|
||
| @AfterAll | ||
| static void stop() { | ||
| redisContainer.close(); | ||
| bastion.stopAndClose(); | ||
| } | ||
|
|
||
| public abstract SshTunnel.TunnelMethod getTunnelMethod(); | ||
|
|
||
| @Override | ||
| protected void setup(TestDestinationEnv testEnv) { | ||
| configJson = RedisDataFactory.jsonConfig( | ||
| redisContainer.getHost(), | ||
| redisContainer.getFirstMappedPort()); | ||
| var redisConfig = new RedisConfig(configJson); | ||
| redisCache = new RedisHCache(redisConfig); | ||
| redisNameTransformer = new RedisNameTransformer(); | ||
| } | ||
|
|
||
| @Override | ||
| protected void tearDown(TestDestinationEnv testEnv) { | ||
| redisCache.flushAll(); | ||
| } | ||
|
|
||
| @Override | ||
| protected String getImageName() { | ||
| return "airbyte/destination-redis:dev"; | ||
| } | ||
|
|
||
| @Override | ||
| protected JsonNode getConfig() throws Exception { | ||
| return bastion.getTunnelConfig(getTunnelMethod(), ImmutableMap.builder() | ||
| .put("host", HostPortResolver.resolveIpAddress(redisContainer)) | ||
| .put("port", redisContainer.getExposedPorts().get(0)) | ||
| .put("username", configJson.get("username")) | ||
| .put("password", configJson.get("password")) | ||
| .put("cache_type", configJson.get("cache_type"))); | ||
| } | ||
|
|
||
| @Override | ||
| protected JsonNode getFailCheckConfig() { | ||
| return RedisDataFactory.jsonConfig( | ||
| "127.0.0.9", | ||
| 8080); | ||
| } | ||
|
|
||
| @Override | ||
| protected List<JsonNode> retrieveRecords(TestDestinationEnv testEnv, | ||
| String streamName, | ||
| String namespace, | ||
| JsonNode streamSchema) { | ||
| var key = redisNameTransformer.keyName(namespace, streamName); | ||
| return redisCache.getAll(key).stream() | ||
| .sorted(Comparator.comparing(RedisRecord::getTimestamp)) | ||
| .map(RedisRecord::getData) | ||
| .map(Jsons::deserialize) | ||
| .collect(Collectors.toList()); | ||
| } | ||
|
|
||
|
|
||
| @Override | ||
| protected boolean implementsNamespaces() { | ||
| return true; | ||
| } | ||
|
|
||
| @Override | ||
| protected TestDataComparator getTestDataComparator() { | ||
| return new AdvancedTestDataComparator(); | ||
| } | ||
|
|
||
| @Override | ||
| protected boolean supportBasicDataTypeTest() { | ||
| return true; | ||
| } | ||
|
|
||
| @Override | ||
| protected boolean supportArrayDataTypeTest() { | ||
| return true; | ||
| } | ||
|
|
||
| @Override | ||
| protected boolean supportObjectDataTypeTest() { | ||
| return true; | ||
| } | ||
|
|
||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters