-
Notifications
You must be signed in to change notification settings - Fork 949
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
feat(core): ForEachItem task (#2359)
close #2131
- Loading branch information
1 parent
382a5df
commit a570cc6
Showing
25 changed files
with
851 additions
and
207 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
12 changes: 6 additions & 6 deletions
12
core/src/main/java/io/kestra/core/models/hierarchies/SubflowGraphTask.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -1,19 +1,19 @@ | ||
package io.kestra.core.models.hierarchies; | ||
|
||
import io.kestra.core.models.executions.TaskRun; | ||
import io.kestra.core.tasks.flows.Flow; | ||
import io.kestra.core.models.tasks.ExecutableTask; | ||
import io.kestra.core.models.tasks.Task; | ||
import lombok.Getter; | ||
|
||
import java.util.List; | ||
|
||
@Getter | ||
public class SubflowGraphTask extends AbstractGraphTask { | ||
public SubflowGraphTask(Flow task, TaskRun taskRun, List<String> values, RelationType relationType) { | ||
super(task, taskRun, values, relationType); | ||
public SubflowGraphTask(ExecutableTask task, TaskRun taskRun, List<String> values, RelationType relationType) { | ||
super((Task) task, taskRun, values, relationType); | ||
} | ||
|
||
@Override | ||
public Flow getTask() { | ||
return (Flow) super.getTask(); | ||
public ExecutableTask getExecutableTask() { | ||
return (ExecutableTask) super.getTask(); | ||
} | ||
} |
57 changes: 57 additions & 0 deletions
57
core/src/main/java/io/kestra/core/models/tasks/ExecutableTask.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,57 @@ | ||
package io.kestra.core.models.tasks; | ||
|
||
import io.kestra.core.exceptions.InternalException; | ||
import io.kestra.core.models.executions.Execution; | ||
import io.kestra.core.models.executions.TaskRun; | ||
import io.kestra.core.models.flows.Flow; | ||
import io.kestra.core.runners.FlowExecutorInterface; | ||
import io.kestra.core.runners.RunContext; | ||
import io.kestra.core.runners.WorkerTaskExecution; | ||
import io.kestra.core.runners.WorkerTaskResult; | ||
|
||
import java.util.List; | ||
import java.util.Optional; | ||
|
||
/** | ||
* Interface for tasks that generates subflow execution(s). Those tasks are handled in the Executor. | ||
*/ | ||
public interface ExecutableTask { | ||
/** | ||
* Creates a list of WorkerTaskExecution for this task definition. | ||
* Each WorkerTaskExecution will generate a subflow execution. | ||
*/ | ||
List<WorkerTaskExecution<?>> createWorkerTaskExecutions(RunContext runContext, | ||
FlowExecutorInterface flowExecutorInterface, | ||
Flow currentFlow, Execution currentExecution, | ||
TaskRun currentTaskRun) throws InternalException; | ||
|
||
/** | ||
* Creates a WorkerTaskResult for a given WorkerTaskExecution | ||
*/ | ||
Optional<WorkerTaskResult> createWorkerTaskResult(RunContext runContext, | ||
WorkerTaskExecution<?> workerTaskExecution, | ||
Flow flow, | ||
Execution execution); | ||
|
||
/** | ||
* Whether to wait for the execution(s) of the subflow before terminating this tasks | ||
*/ | ||
boolean waitForExecution(); | ||
|
||
/** | ||
* @return the subflow identifier, used by the flow topology and related dependency code. | ||
*/ | ||
SubflowId subflowId(); | ||
|
||
record SubflowId(String namespace, String flowId, Optional<Integer> revision) { | ||
public String flowUid() { | ||
// as the Flow task can only be used in the same tenant we can hardcode null here | ||
return Flow.uid(null, this.namespace, this.flowId, this.revision); | ||
} | ||
|
||
public String flowUidWithoutRevision() { | ||
// as the Flow task can only be used in the same tenant we can hardcode null here | ||
return Flow.uidWithoutRevision(null, this.namespace, this.flowId); | ||
} | ||
} | ||
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
105 changes: 105 additions & 0 deletions
105
core/src/main/java/io/kestra/core/runners/ExecutableUtils.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,105 @@ | ||
package io.kestra.core.runners; | ||
|
||
import com.google.common.collect.ImmutableMap; | ||
import io.kestra.core.exceptions.IllegalVariableEvaluationException; | ||
import io.kestra.core.models.Label; | ||
import io.kestra.core.models.executions.Execution; | ||
import io.kestra.core.models.executions.ExecutionTrigger; | ||
import io.kestra.core.models.executions.TaskRun; | ||
import io.kestra.core.models.executions.TaskRunAttempt; | ||
import io.kestra.core.models.flows.Flow; | ||
import io.kestra.core.models.flows.FlowWithException; | ||
import io.kestra.core.models.flows.State; | ||
import io.kestra.core.models.tasks.ExecutableTask; | ||
import io.kestra.core.models.tasks.Task; | ||
|
||
import java.util.Collections; | ||
import java.util.HashMap; | ||
import java.util.List; | ||
import java.util.Map; | ||
import java.util.Optional; | ||
|
||
public final class ExecutableUtils { | ||
|
||
private ExecutableUtils() { | ||
// prevent initialization | ||
} | ||
|
||
public static State.Type guessState(Execution execution, boolean transmitFailed) { | ||
if (transmitFailed && | ||
(execution.getState().isFailed() || execution.getState().isPaused() || execution.getState().getCurrent() == State.Type.KILLED || execution.getState().getCurrent() == State.Type.WARNING) | ||
) { | ||
return execution.getState().getCurrent(); | ||
} else { | ||
return State.Type.SUCCESS; | ||
} | ||
} | ||
|
||
public static WorkerTaskResult workerTaskResult(TaskRun taskRun) { | ||
return WorkerTaskResult.builder() | ||
.taskRun(taskRun.withAttempts( | ||
Collections.singletonList(TaskRunAttempt.builder().state(new State().withState(taskRun.getState().getCurrent())).build()) | ||
)) | ||
.build(); | ||
} | ||
|
||
public static <T extends Task & ExecutableTask> WorkerTaskExecution<?> workerTaskExecution( | ||
RunContext runContext, | ||
FlowExecutorInterface flowExecutorInterface, | ||
Execution currentExecution, | ||
Flow currentFlow, | ||
T currentTask, | ||
TaskRun currentTaskRun, | ||
Map<String, Object> inputs, | ||
List<Label> labels | ||
) throws IllegalVariableEvaluationException { | ||
String subflowNamespace = runContext.render(currentTask.subflowId().namespace()); | ||
String subflowId = runContext.render(currentTask.subflowId().flowId()); | ||
Optional<Integer> subflowRevision = currentTask.subflowId().revision(); | ||
|
||
io.kestra.core.models.flows.Flow flow = flowExecutorInterface.findByIdFromFlowTask( | ||
currentExecution.getTenantId(), | ||
subflowNamespace, | ||
subflowId, | ||
subflowRevision, | ||
currentExecution.getTenantId(), | ||
currentFlow.getNamespace(), | ||
currentFlow.getId() | ||
) | ||
.orElseThrow(() -> new IllegalStateException("Unable to find flow '" + subflowNamespace + "'.'" + subflowId + "' with revision + '" + subflowRevision.orElse(0) + "'")); | ||
|
||
if (flow.isDisabled()) { | ||
throw new IllegalStateException("Cannot execute a flow which is disabled"); | ||
} | ||
|
||
if (flow instanceof FlowWithException fwe) { | ||
throw new IllegalStateException("Cannot execute an invalid flow: " + fwe.getException()); | ||
} | ||
|
||
Map<String, Object> variables = ImmutableMap.of( | ||
"executionId", currentExecution.getId(), | ||
"namespace", currentFlow.getNamespace(), | ||
"flowId", currentFlow.getId(), | ||
"flowRevision", currentFlow.getRevision() | ||
); | ||
|
||
RunnerUtils runnerUtils = runContext.getApplicationContext().getBean(RunnerUtils.class); | ||
Execution execution = runnerUtils | ||
.newExecution( | ||
flow, | ||
(f, e) -> runnerUtils.typedInputs(f, e, inputs), | ||
labels) | ||
.withTrigger(ExecutionTrigger.builder() | ||
.id(currentTask.getId()) | ||
.type(currentTask.getType()) | ||
.variables(variables) | ||
.build() | ||
); | ||
|
||
return WorkerTaskExecution.builder() | ||
.task(currentTask) | ||
.taskRun(currentTaskRun) | ||
.execution(execution) | ||
.build(); | ||
} | ||
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.