-
Notifications
You must be signed in to change notification settings - Fork 948
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
WIP chore(core): migrate StorageInterface to service-loader mechanism
part-of: kestra-io/storage-minio#80 part-of: kestra-io/storage-s3#80 part-of: kestra-io/storage-gcs#119 part-of: kestra-io/storage-azure#53
- Loading branch information
1 parent
f2e7a3a
commit 229e95b
Showing
13 changed files
with
335 additions
and
112 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
77 changes: 0 additions & 77 deletions
77
core/src/main/java/io/kestra/core/contexts/KestraApplicationContext.java
This file was deleted.
Oops, something went wrong.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
22 changes: 22 additions & 0 deletions
22
core/src/main/java/io/kestra/core/exceptions/KestraRuntimeException.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,22 @@ | ||
| package io.kestra.core.exceptions; | ||
|
|
||
| /** | ||
| * The top-level {@link KestraRuntimeException} for non-recoverable errors. | ||
| */ | ||
| public class KestraRuntimeException extends RuntimeException { | ||
|
|
||
| public KestraRuntimeException() { | ||
| } | ||
|
|
||
| public KestraRuntimeException(String message) { | ||
| super(message); | ||
| } | ||
|
|
||
| public KestraRuntimeException(String message, Throwable cause) { | ||
| super(message, cause); | ||
| } | ||
|
|
||
| public KestraRuntimeException(Throwable cause) { | ||
| super(cause); | ||
| } | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
119 changes: 119 additions & 0 deletions
119
core/src/main/java/io/kestra/core/storages/StorageInterfaceFactory.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,119 @@ | ||
| package io.kestra.core.storages; | ||
|
|
||
| import io.kestra.core.exceptions.KestraRuntimeException; | ||
| import io.kestra.core.models.Plugin; | ||
| import io.kestra.core.plugins.PluginRegistry; | ||
| import io.kestra.core.plugins.RegisteredPlugin; | ||
| import io.kestra.core.serializers.JacksonMapper; | ||
| import jakarta.annotation.Nullable; | ||
| import jakarta.validation.ConstraintViolation; | ||
| import jakarta.validation.ConstraintViolationException; | ||
| import jakarta.validation.Validator; | ||
| import org.slf4j.Logger; | ||
| import org.slf4j.LoggerFactory; | ||
|
|
||
| import java.io.IOException; | ||
| import java.util.List; | ||
| import java.util.Map; | ||
| import java.util.Optional; | ||
| import java.util.Set; | ||
| import java.util.stream.Collectors; | ||
| import java.util.stream.Stream; | ||
|
|
||
| /** | ||
| * Factor class for constructing {@link StorageInterface} objects. | ||
| */ | ||
| public final class StorageInterfaceFactory { | ||
|
|
||
| private static final Logger log = LoggerFactory.getLogger(StorageInterfaceFactory.class); | ||
|
|
||
| /** | ||
| * Factory method for constructing and validating new {@link StorageInterface} of the given type with the given configuration. | ||
| * | ||
| * @param pluginRegistry The {@link PluginRegistry}. cannot be {@code null}. | ||
| * @param pluginId The ID of the storage. cannot be {@code null}. | ||
| * @param pluginConfiguration The configuration of the storage. cannot be {@code null}. | ||
| * @param validator The {@link Validator}. | ||
| * @return a new {@link StorageInterface}. | ||
| * @throws KestraRuntimeException if no storage can be found. | ||
| */ | ||
| public static StorageInterface make(final PluginRegistry pluginRegistry, | ||
| final String pluginId, | ||
| final Map<String, Object> pluginConfiguration, | ||
| @Nullable final Validator validator) { | ||
| List<Class<? extends StorageInterface>> classes = allStorageClasses(pluginRegistry) | ||
| .filter(clazz -> Plugin.getId(clazz).map(id -> id.equalsIgnoreCase(pluginId)).orElse(false)) | ||
| .toList(); | ||
|
|
||
| if (classes.isEmpty()) { | ||
| String storageIds = allIdsFor(allStorageClasses(pluginRegistry)); | ||
| throw new KestraRuntimeException(String.format( | ||
| "No storage interface can be found for 'kestra.storage.type=%s'. Supported types are: %s", pluginId, storageIds | ||
| )); | ||
| } | ||
|
|
||
| if (classes.size() > 1) { | ||
| String storageIds = allIdsFor(classes.stream()); | ||
| log.warn("Multiple storage interface candidates was found for 'kestra.storage.type={}' ({}). First one is used.", pluginId, storageIds); | ||
| } | ||
|
|
||
| Class<? extends StorageInterface> storageClass = classes.get(0); | ||
|
|
||
| // Storage are handle as any serializable/deserialize plugins. | ||
| StorageInterface plugin; | ||
| try { | ||
| plugin = JacksonMapper.toMap(pluginConfiguration, storageClass); | ||
| } catch (Exception e) { | ||
| throw new KestraRuntimeException(String.format( | ||
| "Failed to create storage '%s'. Error: %s", pluginId, e.getMessage()) | ||
| ); | ||
| } | ||
|
|
||
| // Validate configuration. | ||
| if (validator != null) { | ||
| Set<ConstraintViolation<StorageInterface>> violations; | ||
| try { | ||
| violations = validator.validate(plugin); | ||
| } catch (ConstraintViolationException e) { | ||
| throw new KestraRuntimeException(String.format( | ||
| "Failed to validate configuration for storage '%s'. Error: %s", pluginId, e.getMessage()) | ||
| ); | ||
| } | ||
| if (!violations.isEmpty()) { | ||
| ConstraintViolationException e = new ConstraintViolationException(violations); | ||
| throw new KestraRuntimeException(String.format( | ||
| "Invalid configuration for storage '%s'. Error: '%s'", pluginId, e.getMessage()), e | ||
| ); | ||
| } | ||
| } | ||
|
|
||
| try { | ||
| plugin.init(); | ||
| } catch (IOException e) { | ||
| throw new KestraRuntimeException(String.format( | ||
| "Failed to initialize storage '%s'. Error: %s", pluginId, e.getMessage()), e | ||
| ); | ||
| } | ||
| return plugin; | ||
| } | ||
|
|
||
| /** | ||
| * @return all plugin classes for the {@link StorageInterface}s. | ||
| */ | ||
| private static Stream<Class<? extends StorageInterface>> allStorageClasses(final PluginRegistry pluginRegistry) { | ||
| return pluginRegistry.plugins() | ||
| .stream() | ||
| .map(RegisteredPlugin::getStorages) | ||
| .flatMap(List::stream); | ||
| } | ||
|
|
||
| /** | ||
| * @return all plugin identifier for the {@link StorageInterface}s. | ||
| */ | ||
| private static String allIdsFor(final Stream<Class<? extends StorageInterface>> classes) { | ||
| return classes | ||
| .map(Plugin::getId) | ||
| .flatMap(Optional::stream) | ||
| .collect(Collectors.joining(",", "[", "]")); | ||
| } | ||
| } |
Oops, something went wrong.