Skip to content

Commit

Permalink
remove usages of YamlSeedConfigPersistence (airbytehq#17895)
Browse files Browse the repository at this point in the history
  • Loading branch information
pedroslopez authored and jhammarstedt committed Oct 31, 2022
1 parent f48973d commit d89c87c
Show file tree
Hide file tree
Showing 5 changed files with 13 additions and 15 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,6 @@
import io.airbyte.config.init.ApplyDefinitionsHelper;
import io.airbyte.config.init.DefinitionsProvider;
import io.airbyte.config.init.LocalDefinitionsProvider;
import io.airbyte.config.init.YamlSeedConfigPersistence;
import io.airbyte.config.persistence.ConfigPersistence;
import io.airbyte.config.persistence.ConfigRepository;
import io.airbyte.config.persistence.DatabaseConfigPersistence;
Expand Down Expand Up @@ -206,7 +205,7 @@ private static ConfigPersistence getConfigPersistence(final Database configDatab
}

private static DefinitionsProvider getLocalDefinitionsProvider() throws IOException {
return new LocalDefinitionsProvider(YamlSeedConfigPersistence.DEFAULT_SEED_DEFINITION_RESOURCE_CLASS);
return new LocalDefinitionsProvider(LocalDefinitionsProvider.DEFAULT_SEED_DEFINITION_RESOURCE_CLASS);
}

private static Database getJobDatabase(final DSLContext dslContext) throws IOException {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,9 @@
import io.airbyte.config.Geography;
import io.airbyte.config.SourceConnection;
import io.airbyte.config.StandardWorkspace;
import io.airbyte.config.init.YamlSeedConfigPersistence;
import io.airbyte.config.init.DefinitionProviderToConfigPersistenceAdapter;
import io.airbyte.config.init.DefinitionsProvider;
import io.airbyte.config.init.LocalDefinitionsProvider;
import io.airbyte.config.persistence.ConfigPersistence;
import io.airbyte.config.persistence.ConfigRepository;
import io.airbyte.config.persistence.DatabaseConfigPersistence;
Expand Down Expand Up @@ -199,9 +201,10 @@ void testBootloaderAppRunSecretMigration() throws Exception {
val initBootloader = new BootloaderApp(mockedConfigs, mockedFeatureFlags, null, configsDslContext, jobsDslContext, configsFlyway, jobsFlyway);
initBootloader.load();

final ConfigPersistence localSchema = new YamlSeedConfigPersistence(YamlSeedConfigPersistence.DEFAULT_SEED_DEFINITION_RESOURCE_CLASS);
final DefinitionsProvider localDefinitions = new LocalDefinitionsProvider(LocalDefinitionsProvider.DEFAULT_SEED_DEFINITION_RESOURCE_CLASS);
final ConfigRepository configRepository = new ConfigRepository(configPersistence, configDatabase);
configRepository.loadDataNoSecrets(localSchema);
final ConfigPersistence localConfigPersistence = new DefinitionProviderToConfigPersistenceAdapter(localDefinitions);
configRepository.loadDataNoSecrets(localConfigPersistence);

final String sourceSpecs = """
{
Expand Down
2 changes: 1 addition & 1 deletion airbyte-config/config-persistence/readme.md
Original file line number Diff line number Diff line change
Expand Up @@ -3,5 +3,5 @@
This module contains the logic for accessing the config database. This database is primarily used by the `airbyte-server` but is also accessed from `airbyte-workers`. It contains all configuration information for Airbyte.

## Key files
* `ConfigPersistence.java` is the interface over "low-level" access to the db. The most commonly used implementation of it is `DatabaseConfigPersistence.java` The only other one that is used is the `YamlSeedConfigPersistence.java` which is used for loading configs that ship with the app.
* `ConfigPersistence.java` is the interface over "low-level" access to the db. The most commonly used implementation of it is `DatabaseConfigPersistence.java`.
* `ConfigRepository.java` is what is most used for accessing the databases. The `ConfigPersistence` iface was hard to work with. `ConfigRepository` builds on top of it and houses any databases queries to keep them from proliferating throughout the codebase.
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,8 @@
*/
final public class LocalDefinitionsProvider implements DefinitionsProvider {

public static final Class<?> DEFAULT_SEED_DEFINITION_RESOURCE_CLASS = SeedType.class;

private final static String PROTOCOL_VERSION = "protocol_version";
private final static String SPEC = "spec";

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -6,10 +6,6 @@

import com.fasterxml.jackson.databind.JsonNode;
import io.airbyte.commons.json.JsonSchemas;
import io.airbyte.config.ConfigSchema;
import io.airbyte.config.StandardDestinationDefinition;
import io.airbyte.config.StandardSourceDefinition;
import io.airbyte.config.persistence.ConfigPersistence;
import io.airbyte.config.persistence.split_secrets.JsonSecretsProcessor;
import io.airbyte.validation.json.JsonValidationException;
import java.io.IOException;
Expand All @@ -25,16 +21,14 @@ class SpecFormatTest {

@Test
void testOnAllExistingConfig() throws IOException, JsonValidationException {
final ConfigPersistence configPersistence = new YamlSeedConfigPersistence(YamlSeedConfigPersistence.DEFAULT_SEED_DEFINITION_RESOURCE_CLASS);
final DefinitionsProvider definitionsProvider = new LocalDefinitionsProvider(LocalDefinitionsProvider.DEFAULT_SEED_DEFINITION_RESOURCE_CLASS);

final List<JsonNode> sourceSpecs = configPersistence.listConfigs(
ConfigSchema.STANDARD_SOURCE_DEFINITION, StandardSourceDefinition.class)
final List<JsonNode> sourceSpecs = definitionsProvider.getSourceDefinitions()
.stream()
.map(standardSourceDefinition -> standardSourceDefinition.getSpec().getConnectionSpecification())
.toList();

final List<JsonNode> destinationSpecs = configPersistence.listConfigs(
ConfigSchema.STANDARD_DESTINATION_DEFINITION, StandardDestinationDefinition.class)
final List<JsonNode> destinationSpecs = definitionsProvider.getDestinationDefinitions()
.stream()
.map(standardDestinationDefinition -> standardDestinationDefinition.getSpec().getConnectionSpecification())
.toList();
Expand Down

0 comments on commit d89c87c

Please sign in to comment.