-
Notifications
You must be signed in to change notification settings - Fork 948
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
WIP chore(core): migrate StorageInterface to service-loader mechanism
part-of: kestra-io/storage-minio#80 part-of: kestra-io/storage-s3#80 part-of: kestra-io/storage-gcs#119
- Loading branch information
1 parent
93f9f70
commit c0fd070
Showing
12 changed files
with
304 additions
and
112 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
77 changes: 0 additions & 77 deletions
77
core/src/main/java/io/kestra/core/contexts/KestraApplicationContext.java
This file was deleted.
Oops, something went wrong.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
22 changes: 22 additions & 0 deletions
22
core/src/main/java/io/kestra/core/exceptions/KestraRuntimeException.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,22 @@ | ||
| package io.kestra.core.exceptions; | ||
|
|
||
| /** | ||
| * The top-level {@link KestraRuntimeException} for non-recoverable errors. | ||
| */ | ||
| public class KestraRuntimeException extends RuntimeException { | ||
|
|
||
| public KestraRuntimeException() { | ||
| } | ||
|
|
||
| public KestraRuntimeException(String message) { | ||
| super(message); | ||
| } | ||
|
|
||
| public KestraRuntimeException(String message, Throwable cause) { | ||
| super(message, cause); | ||
| } | ||
|
|
||
| public KestraRuntimeException(Throwable cause) { | ||
| super(cause); | ||
| } | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
121 changes: 121 additions & 0 deletions
121
core/src/main/java/io/kestra/core/storages/StorageInterfaceFactory.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,121 @@ | ||
| package io.kestra.core.storages; | ||
|
|
||
| import io.kestra.core.exceptions.KestraRuntimeException; | ||
| import io.kestra.core.models.Plugin; | ||
| import io.kestra.core.plugins.PluginRegistry; | ||
| import io.kestra.core.plugins.RegisteredPlugin; | ||
| import io.kestra.core.serializers.JacksonMapper; | ||
| import jakarta.annotation.Nullable; | ||
| import jakarta.validation.ConstraintViolation; | ||
| import jakarta.validation.ConstraintViolationException; | ||
| import jakarta.validation.Validator; | ||
| import org.slf4j.Logger; | ||
| import org.slf4j.LoggerFactory; | ||
|
|
||
| import java.io.IOException; | ||
| import java.util.List; | ||
| import java.util.Map; | ||
| import java.util.Optional; | ||
| import java.util.Set; | ||
| import java.util.function.Function; | ||
| import java.util.stream.Collectors; | ||
| import java.util.stream.Stream; | ||
|
|
||
| /** | ||
| * Factor class for constructing {@link StorageInterface} objects. | ||
| */ | ||
| public final class StorageInterfaceFactory { | ||
|
|
||
| private static final Logger log = LoggerFactory.getLogger(StorageInterfaceFactory.class); | ||
|
|
||
| /** | ||
| * Factory method for constructing and validating new {@link StorageInterface} of the given type with the given configuration. | ||
| * | ||
| * @param pluginRegistry The {@link PluginRegistry}. cannot be {@code null}. | ||
| * @param storageType The type of the storage. cannot be {@code null}. | ||
| * @param storageConfiguration The configuration of the storage. cannot be {@code null}. | ||
| * @param validator The {@link Validator}. | ||
| * @return a new {@link StorageInterface}. | ||
| * @throws KestraRuntimeException if no storage can be found. | ||
| */ | ||
| public static StorageInterface make(final PluginRegistry pluginRegistry, | ||
| final String storageType, | ||
| final Map<String, Object> storageConfiguration, | ||
| @Nullable final Validator validator) { | ||
| List<Class<? extends StorageInterface>> classes = pluginRegistry.plugins() | ||
| .stream() | ||
| .map(RegisteredPlugin::getStorages) | ||
| .flatMap(List::stream) | ||
| .filter(clazz -> Plugin.getId(clazz).map(id -> id.equalsIgnoreCase(storageType)).orElse(false)) | ||
| .toList(); | ||
|
|
||
| if (classes.isEmpty()) { | ||
| String storageIds = allStorageIds(pluginRegistry); | ||
| throw new KestraRuntimeException(String.format( | ||
| "No storage interface can be found for 'kestra.storage.type=%s'. Supported types are: %s", storageType, storageIds | ||
| )); | ||
| } | ||
|
|
||
| if (classes.size() > 1) { | ||
| String storageIds = allIdsFor(classes.stream().map(Function.identity())); | ||
| log.warn("Multiple StorageInterface candidates was found for 'kestra.storage.type={}' ({}). First one is used.", storageType, storageIds); | ||
| } | ||
|
|
||
| Class<? extends StorageInterface> storageClass = classes.get(0); | ||
|
|
||
| // Storage are handle as any serializable/deserialize plugins. | ||
| StorageInterface storage; | ||
| try { | ||
| storage = JacksonMapper.toMap(storageConfiguration, storageClass); | ||
| } catch (Exception e) { | ||
| throw new KestraRuntimeException(String.format( | ||
| "Failed to create storage '%s'. Error: %s", storageType, e.getMessage()) | ||
| ); | ||
| } | ||
|
|
||
| // Validate configuration. | ||
| if (validator != null) { | ||
| Set<ConstraintViolation<StorageInterface>> violations; | ||
| try { | ||
| violations = validator.validate(storage); | ||
| } catch (ConstraintViolationException e) { | ||
| throw new KestraRuntimeException(String.format( | ||
| "Failed to validate configuration for storage '%s'. Error: %s", storageType, e.getMessage()) | ||
| ); | ||
| } | ||
| if (!violations.isEmpty()) { | ||
| ConstraintViolationException e = new ConstraintViolationException(violations); | ||
| throw new KestraRuntimeException(String.format( | ||
| "Invalid configuration for storage '%s'. Error: '%s'", storageType, e.getMessage()), e | ||
| ); | ||
| } | ||
| } | ||
|
|
||
| try { | ||
| storage.init(); | ||
| } catch (IOException e) { | ||
| throw new KestraRuntimeException(String.format( | ||
| "Failed to initialize storage '%s'. Error: %s", storageType, e.getMessage()), e | ||
| ); | ||
| } | ||
| return storage; | ||
| } | ||
|
|
||
| private static String allStorageIds(final PluginRegistry pluginRegistry) { | ||
| return allIdsFor(allStorageClasses(pluginRegistry).map(Function.identity())); | ||
| } | ||
|
|
||
| private static Stream<Class<? extends StorageInterface>> allStorageClasses(final PluginRegistry pluginRegistry) { | ||
| return pluginRegistry.plugins() | ||
| .stream() | ||
| .map(RegisteredPlugin::getStorages) | ||
| .flatMap(List::stream); | ||
| } | ||
|
|
||
| private static String allIdsFor(final Stream<Class<?>> classes) { | ||
| return classes | ||
| .map(Plugin::getId) | ||
| .flatMap(Optional::stream) | ||
| .collect(Collectors.joining(",", "[", "]")); | ||
| } | ||
| } |
Oops, something went wrong.