diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 185418677d..7a374e3dd7 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -32,7 +32,6 @@ updates: timezone: "Europe/Paris" open-pull-requests-limit: 50 labels: ["dependency-upgrade"] - reviewers: ["MilosPaunovic"] ignore: # Ignore major versions greater than 8, as it's still known to be flaky - dependency-name: "eslint" diff --git a/.github/workflows/generate_translations.yml b/.github/workflows/generate_translations.yml index a86f53f105..0d37c9edaa 100644 --- a/.github/workflows/generate_translations.yml +++ b/.github/workflows/generate_translations.yml @@ -108,4 +108,4 @@ jobs: exit 0 fi git commit -m "chore(translations): auto generate values for languages other than english" - git push origin $BRANCH_NAME + git push origin $BRANCH_NAME \ No newline at end of file diff --git a/README.md b/README.md index 6bb58967fb..1d0a86c771 100644 --- a/README.md +++ b/README.md @@ -19,7 +19,7 @@

- twitter   + X(formerly Twitter)   linkedin   youtube  

@@ -167,7 +167,7 @@ Create custom plugins to extend Kestra's capabilities. Check out our [Plugin Dev Stay connected and get support: - **Slack:** Join our [Slack community](https://kestra.io/slack) to ask questions and share ideas. -- **Twitter:** Follow us on [Twitter](https://twitter.com/kestra_io) for the latest updates. +- **X(formerly Twitter):** Follow us on [X(formerly Twitter)](https://x.com/kestra_io) for the latest updates. - **YouTube:** Subscribe to our [YouTube channel](https://www.youtube.com/@kestra-io) for tutorials and webinars. - **LinkedIn:** Connect with us on [LinkedIn](https://www.linkedin.com/company/kestra/). diff --git a/build.gradle b/build.gradle index 9c943ff6df..577f7e1596 100644 --- a/build.gradle +++ b/build.gradle @@ -27,7 +27,7 @@ plugins { id "com.github.ben-manes.versions" version "0.51.0" // front - id 'org.siouan.frontend-jdk17' version '8.1.0' apply false + id 'org.siouan.frontend-jdk21' version '9.0.0' apply false // release id "io.github.gradle-nexus.publish-plugin" version "2.0.0" diff --git a/cli/src/main/java/io/kestra/cli/AbstractValidateCommand.java b/cli/src/main/java/io/kestra/cli/AbstractValidateCommand.java index d550e84c02..8d5fce6ee0 100644 --- a/cli/src/main/java/io/kestra/cli/AbstractValidateCommand.java +++ b/cli/src/main/java/io/kestra/cli/AbstractValidateCommand.java @@ -1,6 +1,5 @@ package io.kestra.cli; -import io.kestra.cli.commands.flows.FlowValidateCommand; import io.kestra.core.models.validations.ModelValidator; import io.kestra.core.models.validations.ValidateConstraintViolation; import io.kestra.core.serializers.YamlFlowParser; @@ -75,7 +74,8 @@ public Integer call( YamlFlowParser yamlFlowParser, ModelValidator modelValidator, Function identity, - Function> warningsFunction + Function> warningsFunction, + Function> infosFunction ) throws Exception { super.call(); @@ -93,6 +93,8 @@ public Integer call( stdOut("@|green \u2713|@ - " + identity.apply(parse)); List warnings = warningsFunction.apply(parse); warnings.forEach(warning -> stdOut("@|bold,yellow \u26A0|@ - " + warning)); + List infos = infosFunction.apply(parse); + infos.forEach(info -> stdOut("@|bold,blue \u2139|@ - " + info)); } catch (ConstraintViolationException e) { stdErr("@|red \u2718|@ - " + path); AbstractValidateCommand.handleException(e, clsName); diff --git a/cli/src/main/java/io/kestra/cli/commands/flows/FlowValidateCommand.java b/cli/src/main/java/io/kestra/cli/commands/flows/FlowValidateCommand.java index 7783b904e4..2b3abb5215 100644 --- a/cli/src/main/java/io/kestra/cli/commands/flows/FlowValidateCommand.java +++ b/cli/src/main/java/io/kestra/cli/commands/flows/FlowValidateCommand.java @@ -39,9 +39,12 @@ public Integer call() throws Exception { Flow flow = (Flow) object; List warnings = new ArrayList<>(); warnings.addAll(flowService.deprecationPaths(flow).stream().map(deprecation -> deprecation + " is deprecated").toList()); - warnings.addAll(flowService.relocations(flow.generateSource()).stream().map(relocation -> relocation.from() + " is replaced by " + relocation.to()).toList()); warnings.addAll(flowService.warnings(flow)); return warnings; + }, + (Object object) -> { + Flow flow = (Flow) object; + return flowService.relocations(flow.generateSource()).stream().map(relocation -> relocation.from() + " is replaced by " + relocation.to()).toList(); } ); } diff --git a/cli/src/main/java/io/kestra/cli/commands/templates/TemplateValidateCommand.java b/cli/src/main/java/io/kestra/cli/commands/templates/TemplateValidateCommand.java index 74e3e29868..fec68750c6 100644 --- a/cli/src/main/java/io/kestra/cli/commands/templates/TemplateValidateCommand.java +++ b/cli/src/main/java/io/kestra/cli/commands/templates/TemplateValidateCommand.java @@ -32,6 +32,7 @@ public Integer call() throws Exception { Template template = (Template) object; return template.getNamespace() + " / " + template.getId(); }, + (Object object) -> Collections.emptyList(), (Object object) -> Collections.emptyList() ); } diff --git a/cli/src/main/java/io/kestra/cli/services/FileChangedEventListener.java b/cli/src/main/java/io/kestra/cli/services/FileChangedEventListener.java index e56a34f651..622f02a6bd 100644 --- a/cli/src/main/java/io/kestra/cli/services/FileChangedEventListener.java +++ b/cli/src/main/java/io/kestra/cli/services/FileChangedEventListener.java @@ -121,7 +121,7 @@ public void startListening(List paths) throws IOException, InterruptedExce WatchEvent.Kind kind = watchEvent.kind(); Path entry = (Path) watchEvent.context(); - if (entry.endsWith(".yml") || entry.endsWith(".yaml")) { + if (entry.toString().endsWith(".yml") || entry.toString().endsWith(".yaml")) { if (kind == StandardWatchEventKinds.ENTRY_CREATE || kind == StandardWatchEventKinds.ENTRY_MODIFY) { diff --git a/cli/src/main/java/io/kestra/cli/services/LocalFlowFileWatcher.java b/cli/src/main/java/io/kestra/cli/services/LocalFlowFileWatcher.java index d151204b01..2cf8ee53ce 100644 --- a/cli/src/main/java/io/kestra/cli/services/LocalFlowFileWatcher.java +++ b/cli/src/main/java/io/kestra/cli/services/LocalFlowFileWatcher.java @@ -23,12 +23,12 @@ public FlowWithSource createOrUpdateFlow(Flow flow, String content) { public void deleteFlow(FlowWithSource toDelete) { flowRepositoryInterface.findByIdWithSource(toDelete.getTenantId(), toDelete.getNamespace(), toDelete.getId()).ifPresent(flowRepositoryInterface::delete); - log.debug("Flow {} has been deleted", toDelete.getId()); + log.error("Flow {} has been deleted", toDelete.getId()); } @Override public void deleteFlow(String tenantId, String namespace, String id) { flowRepositoryInterface.findByIdWithSource(tenantId, namespace, id).ifPresent(flowRepositoryInterface::delete); - log.debug("Flow {} has been deleted", id); + log.error("Flow {} has been deleted", id); } } diff --git a/cli/src/main/resources/application.yml b/cli/src/main/resources/application.yml index ac1e695eeb..1576eb9083 100644 --- a/cli/src/main/resources/application.yml +++ b/cli/src/main/resources/application.yml @@ -188,3 +188,8 @@ kestra: uri: https://api.kestra.io/v1/reports/usages initial-delay: 5m fixed-delay: 1h + + hidden-labels: + prefixes: + - system_ + - internal_ diff --git a/cli/src/test/java/io/kestra/cli/commands/flows/FlowValidateCommandTest.java b/cli/src/test/java/io/kestra/cli/commands/flows/FlowValidateCommandTest.java index a552e20ae1..f760e2c590 100644 --- a/cli/src/test/java/io/kestra/cli/commands/flows/FlowValidateCommandTest.java +++ b/cli/src/test/java/io/kestra/cli/commands/flows/FlowValidateCommandTest.java @@ -42,7 +42,9 @@ void warning() { Integer call = PicocliRunner.call(FlowValidateCommand.class, ctx, args); assertThat(call, is(0)); - assertThat(out.toString(), containsString("tasks[0] is deprecated")); + assertThat(out.toString(), containsString("✓ - system / warning")); + assertThat(out.toString(), containsString("⚠ - tasks[0] is deprecated")); + assertThat(out.toString(), containsString("ℹ - io.kestra.core.tasks.log.Log is replaced by io.kestra.plugin.core.log.Log")); } } } \ No newline at end of file diff --git a/cli/src/test/resources/warning/flow-with-warning.yaml b/cli/src/test/resources/warning/flow-with-warning.yaml index e57b43df4b..a84ef569e3 100644 --- a/cli/src/test/resources/warning/flow-with-warning.yaml +++ b/cli/src/test/resources/warning/flow-with-warning.yaml @@ -4,4 +4,7 @@ namespace: system tasks: - id: deprecated type: io.kestra.plugin.core.debug.Echo - format: Hello World \ No newline at end of file + format: Hello World + - id: alias + type: io.kestra.core.tasks.log.Log + message: I'm an alias \ No newline at end of file diff --git a/core/src/main/java/io/kestra/core/metrics/MetricRegistry.java b/core/src/main/java/io/kestra/core/metrics/MetricRegistry.java index cc563b05be..36d7ae35f5 100644 --- a/core/src/main/java/io/kestra/core/metrics/MetricRegistry.java +++ b/core/src/main/java/io/kestra/core/metrics/MetricRegistry.java @@ -19,58 +19,60 @@ @Singleton @Slf4j public class MetricRegistry { - public final static String METRIC_WORKER_JOB_PENDING_COUNT = "worker.job.pending"; - public final static String METRIC_WORKER_JOB_RUNNING_COUNT = "worker.job.running"; - public final static String METRIC_WORKER_JOB_THREAD_COUNT = "worker.job.thread"; - public final static String METRIC_WORKER_RUNNING_COUNT = "worker.running.count"; - public final static String METRIC_WORKER_QUEUED_DURATION = "worker.queued.duration"; - public final static String METRIC_WORKER_STARTED_COUNT = "worker.started.count"; - public final static String METRIC_WORKER_TIMEOUT_COUNT = "worker.timeout.count"; - public final static String METRIC_WORKER_ENDED_COUNT = "worker.ended.count"; - public final static String METRIC_WORKER_ENDED_DURATION = "worker.ended.duration"; - public final static String METRIC_WORKER_TRIGGER_DURATION = "worker.trigger.duration"; - public final static String METRIC_WORKER_TRIGGER_RUNNING_COUNT = "worker.trigger.running.count"; - public final static String METRIC_WORKER_TRIGGER_STARTED_COUNT = "worker.trigger.started.count"; - public final static String METRIC_WORKER_TRIGGER_ENDED_COUNT = "worker.trigger.ended.count"; - public final static String METRIC_WORKER_TRIGGER_ERROR_COUNT = "worker.trigger.error.count"; - public final static String METRIC_WORKER_TRIGGER_EXECUTION_COUNT = "worker.trigger.execution.count"; - - public final static String EXECUTOR_TASKRUN_NEXT_COUNT = "executor.taskrun.next.count"; - public final static String EXECUTOR_TASKRUN_ENDED_COUNT = "executor.taskrun.ended.count"; - public final static String EXECUTOR_TASKRUN_ENDED_DURATION = "executor.taskrun.ended.duration"; - public final static String EXECUTOR_WORKERTASKRESULT_COUNT = "executor.workertaskresult.count"; - public final static String EXECUTOR_EXECUTION_STARTED_COUNT = "executor.execution.started.count"; - public final static String EXECUTOR_EXECUTION_END_COUNT = "executor.execution.end.count"; - public final static String EXECUTOR_EXECUTION_DURATION = "executor.execution.duration"; - - public final static String METRIC_INDEXER_REQUEST_COUNT = "indexer.request.count"; - public final static String METRIC_INDEXER_REQUEST_DURATION = "indexer.request.duration"; - public final static String METRIC_INDEXER_REQUEST_RETRY_COUNT = "indexer.request.retry.count"; - public final static String METRIC_INDEXER_SERVER_DURATION = "indexer.server.duration"; - public final static String METRIC_INDEXER_MESSAGE_FAILED_COUNT = "indexer.message.failed.count"; - public final static String METRIC_INDEXER_MESSAGE_IN_COUNT = "indexer.message.in.count"; - public final static String METRIC_INDEXER_MESSAGE_OUT_COUNT = "indexer.message.out.count"; - - public final static String SCHEDULER_LOOP_COUNT = "scheduler.loop.count"; - public final static String SCHEDULER_TRIGGER_COUNT = "scheduler.trigger.count"; - public final static String SCHEDULER_TRIGGER_DELAY_DURATION = "scheduler.trigger.delay.duration"; - public final static String SCHEDULER_EVALUATE_COUNT = "scheduler.evaluate.count"; - public final static String SCHEDULER_EXECUTION_RUNNING_DURATION = "scheduler.execution.running.duration"; - public final static String SCHEDULER_EXECUTION_MISSING_DURATION = "scheduler.execution.missing.duration"; - - public final static String STREAMS_STATE_COUNT = "stream.state.count"; - - - public final static String JDBC_QUERY_DURATION = "jdbc.query.duration"; - - public final static String TAG_TASK_TYPE = "task_type"; - public final static String TAG_TRIGGER_TYPE = "trigger_type"; - public final static String TAG_FLOW_ID = "flow_id"; - public final static String TAG_NAMESPACE_ID = "namespace_id"; - public final static String TAG_STATE = "state"; - public final static String TAG_ATTEMPT_COUNT = "attempt_count"; - public final static String TAG_WORKER_GROUP = "worker_group"; - public final static String TAG_TENANT_ID = "tenant_id"; + public static final String METRIC_WORKER_JOB_PENDING_COUNT = "worker.job.pending"; + public static final String METRIC_WORKER_JOB_RUNNING_COUNT = "worker.job.running"; + public static final String METRIC_WORKER_JOB_THREAD_COUNT = "worker.job.thread"; + public static final String METRIC_WORKER_RUNNING_COUNT = "worker.running.count"; + public static final String METRIC_WORKER_QUEUED_DURATION = "worker.queued.duration"; + public static final String METRIC_WORKER_STARTED_COUNT = "worker.started.count"; + public static final String METRIC_WORKER_TIMEOUT_COUNT = "worker.timeout.count"; + public static final String METRIC_WORKER_ENDED_COUNT = "worker.ended.count"; + public static final String METRIC_WORKER_ENDED_DURATION = "worker.ended.duration"; + public static final String METRIC_WORKER_TRIGGER_DURATION = "worker.trigger.duration"; + public static final String METRIC_WORKER_TRIGGER_RUNNING_COUNT = "worker.trigger.running.count"; + public static final String METRIC_WORKER_TRIGGER_STARTED_COUNT = "worker.trigger.started.count"; + public static final String METRIC_WORKER_TRIGGER_ENDED_COUNT = "worker.trigger.ended.count"; + public static final String METRIC_WORKER_TRIGGER_ERROR_COUNT = "worker.trigger.error.count"; + public static final String METRIC_WORKER_TRIGGER_EXECUTION_COUNT = "worker.trigger.execution.count"; + + public static final String EXECUTOR_TASKRUN_NEXT_COUNT = "executor.taskrun.next.count"; + public static final String EXECUTOR_TASKRUN_ENDED_COUNT = "executor.taskrun.ended.count"; + public static final String EXECUTOR_TASKRUN_ENDED_DURATION = "executor.taskrun.ended.duration"; + public static final String EXECUTOR_WORKERTASKRESULT_COUNT = "executor.workertaskresult.count"; + public static final String EXECUTOR_EXECUTION_STARTED_COUNT = "executor.execution.started.count"; + public static final String EXECUTOR_EXECUTION_END_COUNT = "executor.execution.end.count"; + public static final String EXECUTOR_EXECUTION_DURATION = "executor.execution.duration"; + + public static final String METRIC_INDEXER_REQUEST_COUNT = "indexer.request.count"; + public static final String METRIC_INDEXER_REQUEST_DURATION = "indexer.request.duration"; + public static final String METRIC_INDEXER_REQUEST_RETRY_COUNT = "indexer.request.retry.count"; + public static final String METRIC_INDEXER_SERVER_DURATION = "indexer.server.duration"; + public static final String METRIC_INDEXER_MESSAGE_FAILED_COUNT = "indexer.message.failed.count"; + public static final String METRIC_INDEXER_MESSAGE_IN_COUNT = "indexer.message.in.count"; + public static final String METRIC_INDEXER_MESSAGE_OUT_COUNT = "indexer.message.out.count"; + + public static final String SCHEDULER_LOOP_COUNT = "scheduler.loop.count"; + public static final String SCHEDULER_TRIGGER_COUNT = "scheduler.trigger.count"; + public static final String SCHEDULER_TRIGGER_DELAY_DURATION = "scheduler.trigger.delay.duration"; + public static final String SCHEDULER_EVALUATE_COUNT = "scheduler.evaluate.count"; + public static final String SCHEDULER_EXECUTION_RUNNING_DURATION = "scheduler.execution.running.duration"; + public static final String SCHEDULER_EXECUTION_MISSING_DURATION = "scheduler.execution.missing.duration"; + + public static final String STREAMS_STATE_COUNT = "stream.state.count"; + + public static final String JDBC_QUERY_DURATION = "jdbc.query.duration"; + + public static final String QUEUE_BIG_MESSAGE_COUNT = "queue.big_message.count"; + + public static final String TAG_TASK_TYPE = "task_type"; + public static final String TAG_TRIGGER_TYPE = "trigger_type"; + public static final String TAG_FLOW_ID = "flow_id"; + public static final String TAG_NAMESPACE_ID = "namespace_id"; + public static final String TAG_STATE = "state"; + public static final String TAG_ATTEMPT_COUNT = "attempt_count"; + public static final String TAG_WORKER_GROUP = "worker_group"; + public static final String TAG_TENANT_ID = "tenant_id"; + public static final String TAG_CLASS_NAME = "class_name"; @Inject private MeterRegistry meterRegistry; diff --git a/core/src/main/java/io/kestra/core/models/Label.java b/core/src/main/java/io/kestra/core/models/Label.java index 5879703085..bd2fbd8359 100644 --- a/core/src/main/java/io/kestra/core/models/Label.java +++ b/core/src/main/java/io/kestra/core/models/Label.java @@ -2,4 +2,11 @@ import jakarta.validation.constraints.NotNull; -public record Label(@NotNull String key, @NotNull String value) {} +public record Label(@NotNull String key, @NotNull String value) { + public static final String SYSTEM_PREFIX = "system_"; + + // system labels + public static final String CORRELATION_ID = SYSTEM_PREFIX + "correlationId"; + public static final String USERNAME = SYSTEM_PREFIX + "username"; + public static final String APP = SYSTEM_PREFIX + "app"; +} diff --git a/core/src/main/java/io/kestra/core/models/executions/Execution.java b/core/src/main/java/io/kestra/core/models/executions/Execution.java index 890c629765..89407f5933 100644 --- a/core/src/main/java/io/kestra/core/models/executions/Execution.java +++ b/core/src/main/java/io/kestra/core/models/executions/Execution.java @@ -39,6 +39,8 @@ import java.util.stream.Stream; import java.util.zip.CRC32; +import static io.kestra.core.models.Label.SYSTEM_PREFIX; + @Builder(toBuilder = true) @Slf4j @Getter @@ -76,7 +78,6 @@ public class Execution implements DeletedInterface, TenantInterface { @JsonInclude(JsonInclude.Include.NON_EMPTY) Map outputs; - @With @JsonSerialize(using = ListOrMapOfLabelSerializer.class) @JsonDeserialize(using = ListOrMapOfLabelDeserializer.class) List