-
Notifications
You must be signed in to change notification settings - Fork 948
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
chore(core): migrate StorageInterface to service-loader mechanism
part-of: kestra-io/storage-minio#80 part-of: kestra-io/storage-s3#80 part-of: kestra-io/storage-gcs#119 part-of: kestra-io/storage-azure#53
- Loading branch information
1 parent
0ab3f7d
commit b381ee5
Showing
13 changed files
with
340 additions
and
112 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
77 changes: 0 additions & 77 deletions
77
core/src/main/java/io/kestra/core/contexts/KestraApplicationContext.java
This file was deleted.
Oops, something went wrong.
56 changes: 56 additions & 0 deletions
56
core/src/main/java/io/kestra/core/contexts/KestraBeansFactory.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,17 +1,73 @@ | ||
| package io.kestra.core.contexts; | ||
|
|
||
| import io.kestra.core.exceptions.KestraRuntimeException; | ||
| import io.kestra.core.plugins.DefaultPluginRegistry; | ||
| import io.kestra.core.plugins.PluginRegistry; | ||
| import io.kestra.core.storages.StorageInterface; | ||
| import io.kestra.core.storages.StorageInterfaceFactory; | ||
| import io.micronaut.context.annotation.Bean; | ||
| import io.micronaut.context.annotation.ConfigurationProperties; | ||
| import io.micronaut.context.annotation.Factory; | ||
| import io.micronaut.context.annotation.Requires; | ||
| import io.micronaut.context.annotation.Value; | ||
| import io.micronaut.core.annotation.Nullable; | ||
| import io.micronaut.core.convert.format.MapFormat; | ||
| import io.micronaut.core.naming.conventions.StringConvention; | ||
| import jakarta.inject.Inject; | ||
| import jakarta.inject.Singleton; | ||
| import jakarta.validation.Validator; | ||
|
|
||
| import java.io.IOException; | ||
| import java.util.Map; | ||
| import java.util.Optional; | ||
|
|
||
| import static io.kestra.core.storages.StorageInterfaceFactory.KESTRA_STORAGE_TYPE_CONFIG; | ||
|
|
||
| @Factory | ||
| public class KestraBeansFactory { | ||
|
|
||
| @Inject | ||
| Validator validator; | ||
|
|
||
| @Inject | ||
| StorageConfig storageConfig; | ||
|
|
||
| @Value("${kestra.storage.type}") | ||
| Optional<String> storageType; | ||
|
|
||
| @Requires(missingBeans = PluginRegistry.class) | ||
| @Singleton | ||
| public PluginRegistry pluginRegistry() { | ||
| return DefaultPluginRegistry.getOrCreate(); | ||
| } | ||
|
|
||
| @Requires(missingBeans = StorageInterface.class) | ||
| @Singleton | ||
| @Bean(preDestroy = "close") | ||
| public StorageInterface storageInterface(final PluginRegistry pluginRegistry) throws IOException { | ||
| String pluginId = storageType.orElseThrow(() -> new KestraRuntimeException(String.format( | ||
| "No storage configured through the application property '%s'. Support types are: %s" | ||
| , KESTRA_STORAGE_TYPE_CONFIG, | ||
| StorageInterfaceFactory.getLoggableStorageIds(pluginRegistry) | ||
| ))); | ||
| return StorageInterfaceFactory.make(pluginRegistry, pluginId, storageConfig.getStorageConfig(pluginId), validator); | ||
| } | ||
|
|
||
| @ConfigurationProperties("kestra") | ||
| public record StorageConfig( | ||
| @Nullable | ||
| @MapFormat(keyFormat = StringConvention.CAMEL_CASE, transformation = MapFormat.MapTransformation.NESTED) | ||
| Map<String, Object> storage | ||
| ) { | ||
|
|
||
| /** | ||
| * Returns the configuration for the configured storage. | ||
| * | ||
| * @return the configuration. | ||
| */ | ||
| @SuppressWarnings("unchecked") | ||
| private Map<String, Object> getStorageConfig(String type) { | ||
| return (Map<String, Object>) storage.get(StringConvention.CAMEL_CASE.format(type)); | ||
| } | ||
| } | ||
| } |
22 changes: 22 additions & 0 deletions
22
core/src/main/java/io/kestra/core/exceptions/KestraRuntimeException.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,22 @@ | ||
| package io.kestra.core.exceptions; | ||
|
|
||
| /** | ||
| * The top-level {@link KestraRuntimeException} for non-recoverable errors. | ||
| */ | ||
| public class KestraRuntimeException extends RuntimeException { | ||
|
|
||
| public KestraRuntimeException() { | ||
| } | ||
|
|
||
| public KestraRuntimeException(String message) { | ||
| super(message); | ||
| } | ||
|
|
||
| public KestraRuntimeException(String message, Throwable cause) { | ||
| super(message, cause); | ||
| } | ||
|
|
||
| public KestraRuntimeException(Throwable cause) { | ||
| super(cause); | ||
| } | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
118 changes: 118 additions & 0 deletions
118
core/src/main/java/io/kestra/core/storages/StorageInterfaceFactory.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,118 @@ | ||
| package io.kestra.core.storages; | ||
|
|
||
| import io.kestra.core.exceptions.KestraRuntimeException; | ||
| import io.kestra.core.models.Plugin; | ||
| import io.kestra.core.plugins.PluginRegistry; | ||
| import io.kestra.core.plugins.RegisteredPlugin; | ||
| import io.kestra.core.serializers.JacksonMapper; | ||
| import jakarta.validation.ConstraintViolation; | ||
| import jakarta.validation.ConstraintViolationException; | ||
| import jakarta.validation.Validator; | ||
| import org.slf4j.Logger; | ||
| import org.slf4j.LoggerFactory; | ||
|
|
||
| import java.io.IOException; | ||
| import java.util.List; | ||
| import java.util.Map; | ||
| import java.util.Optional; | ||
| import java.util.Set; | ||
| import java.util.stream.Collectors; | ||
| import java.util.stream.Stream; | ||
|
|
||
| /** | ||
| * Factor class for constructing {@link StorageInterface} objects. | ||
| */ | ||
| public final class StorageInterfaceFactory { | ||
|
|
||
| private static final Logger log = LoggerFactory.getLogger(StorageInterfaceFactory.class); | ||
| public static final String KESTRA_STORAGE_TYPE_CONFIG = "kestra.storage.type"; | ||
|
|
||
| /** | ||
| * Factory method for constructing and validating new {@link StorageInterface} of the given type with the given configuration. | ||
| * | ||
| * @param pluginRegistry The {@link PluginRegistry}. cannot be {@code null}. | ||
| * @param pluginId The ID of the storage. cannot be {@code null}. | ||
| * @param pluginConfiguration The configuration of the storage. cannot be {@code null}. | ||
| * @param validator The {@link Validator}. | ||
| * @return a new {@link StorageInterface}. | ||
| * @throws KestraRuntimeException if no storage can be found. | ||
| */ | ||
| public static StorageInterface make(final PluginRegistry pluginRegistry, | ||
| final String pluginId, | ||
| final Map<String, Object> pluginConfiguration, | ||
| final Validator validator) { | ||
| Optional<Class<? extends StorageInterface>> optional = allStorageClasses(pluginRegistry) | ||
| .filter(clazz -> Plugin.getId(clazz).map(id -> id.equalsIgnoreCase(pluginId)).orElse(false)) | ||
| .findFirst(); | ||
|
|
||
| if (optional.isEmpty()) { | ||
| String storageIds = getLoggableStorageIds(pluginRegistry); | ||
| throw new KestraRuntimeException(String.format( | ||
| "No storage interface can be found for '%s=%s'. Supported types are: %s", KESTRA_STORAGE_TYPE_CONFIG, pluginId, storageIds | ||
| )); | ||
| } | ||
|
|
||
| Class<? extends StorageInterface> pluginClass = optional.get(); | ||
|
|
||
| // Storage are handle as any serializable/deserialize plugins. | ||
| StorageInterface plugin; | ||
| try { | ||
| // Make sure config is not null, otherwise deserialization result will be null too. | ||
| Map<String, Object> nonEmptyConfig = Optional.ofNullable(pluginConfiguration).orElse(Map.of()); | ||
| plugin = JacksonMapper.toMap(nonEmptyConfig, pluginClass); | ||
| } catch (Exception e) { | ||
| throw new KestraRuntimeException(String.format( | ||
| "Failed to create storage '%s'. Error: %s", pluginId, e.getMessage()) | ||
| ); | ||
| } | ||
|
|
||
| // Validate configuration. | ||
| Set<ConstraintViolation<StorageInterface>> violations; | ||
| try { | ||
| violations = validator.validate(plugin); | ||
| } catch (ConstraintViolationException e) { | ||
| throw new KestraRuntimeException(String.format( | ||
| "Failed to validate configuration for storage '%s'. Error: %s", pluginId, e.getMessage()) | ||
| ); | ||
| } | ||
| if (!violations.isEmpty()) { | ||
| ConstraintViolationException e = new ConstraintViolationException(violations); | ||
| throw new KestraRuntimeException(String.format( | ||
| "Invalid configuration for storage '%s'. Error: '%s'", pluginId, e.getMessage()), e | ||
| ); | ||
| } | ||
|
|
||
| try { | ||
| plugin.init(); | ||
| } catch (IOException e) { | ||
| throw new KestraRuntimeException(String.format( | ||
| "Failed to initialize storage '%s'. Error: %s", pluginId, e.getMessage()), e | ||
| ); | ||
| } | ||
| return plugin; | ||
| } | ||
|
|
||
| public static String getLoggableStorageIds(final PluginRegistry pluginRegistry) { | ||
| return allIdsFor(allStorageClasses(pluginRegistry)); | ||
| } | ||
|
|
||
| /** | ||
| * @return all plugin classes for the {@link StorageInterface}s. | ||
| */ | ||
| private static Stream<Class<? extends StorageInterface>> allStorageClasses(final PluginRegistry pluginRegistry) { | ||
| return pluginRegistry.plugins() | ||
| .stream() | ||
| .map(RegisteredPlugin::getStorages) | ||
| .flatMap(List::stream); | ||
| } | ||
|
|
||
| /** | ||
| * @return all plugin identifier for the {@link StorageInterface}s. | ||
| */ | ||
| private static String allIdsFor(final Stream<Class<? extends StorageInterface>> classes) { | ||
| return classes | ||
| .map(Plugin::getId) | ||
| .flatMap(Optional::stream) | ||
| .collect(Collectors.joining(",", "[", "]")); | ||
| } | ||
| } |
Oops, something went wrong.