From e739c34a810c33ec5acb3fefa8386527c10ab6b6 Mon Sep 17 00:00:00 2001 From: Anoop Panicker Date: Thu, 14 Apr 2022 17:14:05 -0700 Subject: [PATCH] remove guava from common module;handle exceptions in sweeper --- annotations-processor/dependencies.lock | 8 +- azureblob-storage/dependencies.lock | 100 --------- build.gradle | 1 + cassandra-persistence/dependencies.lock | 98 +-------- .../cassandra/dao/CassandraExecutionDAO.java | 58 +----- client-spring/dependencies.lock | 84 +------- client/build.gradle | 2 - client/dependencies.lock | 148 +------------- .../client/automator/TaskPollExecutor.java | 82 ++++---- .../automator/TaskPollExecutorTest.java | 59 +++--- common/build.gradle | 2 - common/dependencies.lock | 192 +----------------- .../OwnerEmailMandatoryConstraint.java | 4 +- .../metadata/workflow/SubWorkflowParams.java | 8 +- .../conductor/common/run/Workflow.java | 18 +- .../conductor/common/utils/RetryUtil.java | 147 -------------- contribs/dependencies.lock | 22 +- .../contribs/storage/S3PayloadStorage.java | 17 +- core/build.gradle | 4 +- core/dependencies.lock | 188 ++--------------- .../annotations/VisibleForTesting.java | 24 +++ .../config/ConductorCoreConfiguration.java | 38 +++- .../core/config/SchedulerConfiguration.java | 8 +- .../core/events/DefaultEventProcessor.java | 73 +++---- .../core/execution/DeciderService.java | 3 +- .../core/execution/WorkflowExecutor.java | 60 ++---- .../execution/mapper/DecisionTaskMapper.java | 3 +- .../execution/mapper/DynamicTaskMapper.java | 3 +- .../mapper/ForkJoinDynamicTaskMapper.java | 10 +- .../mapper/SubWorkflowTaskMapper.java | 3 +- .../core/execution/tasks/DoWhile.java | 3 +- .../conductor/core/execution/tasks/Event.java | 2 +- .../core/execution/tasks/ExecutionConfig.java | 6 +- .../tasks/IsolatedTaskQueueProducer.java | 3 +- .../execution/tasks/SystemTaskWorker.java | 3 +- .../tasks/SystemTaskWorkerCoordinator.java | 3 +- .../core/metadata/MetadataMapperService.java | 25 ++- .../reconciliation/WorkflowRepairService.java | 3 +- .../core/reconciliation/WorkflowSweeper.java | 11 +- .../utils/ExternalPayloadStorageUtils.java | 2 +- .../conductor/core/utils/ParametersUtils.java | 3 +- .../netflix/conductor/core/utils/Utils.java | 38 ++-- .../conductor/model/WorkflowModel.java | 6 +- .../conductor/service/TaskServiceImpl.java | 16 +- .../events/TestDefaultEventProcessor.java | 48 +++-- .../core/execution/TestWorkflowExecutor.java | 9 +- .../mapper/ForkJoinDynamicTaskMapperTest.java | 9 +- .../metadata/MetadataMapperServiceTest.java | 18 +- dependencies.gradle | 1 - es6-persistence/build.gradle | 2 + es6-persistence/dependencies.lock | 38 ++-- .../config/ElasticSearchV6Configuration.java | 23 ++- .../es6/dao/index/ElasticSearchDAOV6.java | 45 +--- .../es6/dao/index/ElasticSearchRestDAOV6.java | 124 ++++------- .../dao/index/ElasticSearchDaoBaseTest.java | 7 +- .../index/ElasticSearchRestDaoBaseTest.java | 5 +- es7-persistence/build.gradle | 2 + es7-persistence/dependencies.lock | 46 ++--- .../config/ElasticSearchV7Configuration.java | 16 +- .../es7/dao/index/ElasticSearchRestDAOV7.java | 124 ++++------- .../index/ElasticSearchRestDaoBaseTest.java | 5 +- grpc-client/dependencies.lock | 76 +++---- grpc-server/dependencies.lock | 90 ++++---- grpc/dependencies.lock | 94 +++------ java-sdk/dependencies.lock | 18 -- mysql-persistence/build.gradle | 2 + mysql-persistence/dependencies.lock | 38 ++-- .../mysql/config/MySQLConfiguration.java | 70 ++++++- .../mysql/config/MySQLProperties.java | 10 + .../conductor/mysql/dao/MySQLBaseDAO.java | 68 ++----- .../mysql/dao/MySQLExecutionDAO.java | 10 +- .../conductor/mysql/dao/MySQLMetadataDAO.java | 9 +- .../conductor/mysql/dao/MySQLQueueDAO.java | 7 +- postgres-external-storage/dependencies.lock | 114 ++--------- postgres-persistence/build.gradle | 2 + postgres-persistence/dependencies.lock | 38 ++-- .../config/PostgresConfiguration.java | 69 ++++++- .../postgres/config/PostgresProperties.java | 10 + .../postgres/dao/PostgresBaseDAO.java | 68 +------ .../postgres/dao/PostgresExecutionDAO.java | 7 +- .../postgres/dao/PostgresMetadataDAO.java | 15 +- .../postgres/dao/PostgresQueueDAO.java | 13 +- redis-concurrency-limit/dependencies.lock | 100 --------- redis-lock/dependencies.lock | 99 +-------- redis-persistence/dependencies.lock | 88 +------- rest/dependencies.lock | 124 ++--------- server/build.gradle | 1 + server/dependencies.lock | 162 ++++++++------- test-harness/build.gradle | 2 + test-harness/dependencies.lock | 46 +++-- .../resiliency/QueueResiliencySpec.groovy | 2 +- zookeeper-lock/dependencies.lock | 94 +-------- 92 files changed, 990 insertions(+), 2669 deletions(-) delete mode 100644 common/src/main/java/com/netflix/conductor/common/utils/RetryUtil.java create mode 100644 core/src/main/java/com/netflix/conductor/annotations/VisibleForTesting.java diff --git a/annotations-processor/dependencies.lock b/annotations-processor/dependencies.lock index 4c7067343d..9e0727bb97 100644 --- a/annotations-processor/dependencies.lock +++ b/annotations-processor/dependencies.lock @@ -42,7 +42,7 @@ ] }, "com.google.protobuf:protobuf-java": { - "locked": "3.20.0" + "locked": "3.20.1" }, "com.netflix.conductor:conductor-annotations": { "project": true @@ -181,7 +181,7 @@ ] }, "com.google.protobuf:protobuf-java": { - "locked": "3.20.0" + "locked": "3.20.1" }, "com.netflix.conductor:conductor-annotations": { "project": true @@ -280,7 +280,7 @@ ] }, "com.google.protobuf:protobuf-java": { - "locked": "3.20.0" + "locked": "3.20.1" }, "com.jayway.jsonpath:json-path": { "locked": "2.4.0", @@ -661,7 +661,7 @@ ] }, "com.google.protobuf:protobuf-java": { - "locked": "3.20.0" + "locked": "3.20.1" }, "com.jayway.jsonpath:json-path": { "locked": "2.4.0", diff --git a/azureblob-storage/dependencies.lock b/azureblob-storage/dependencies.lock index 86bbb09764..b123f7ed5c 100644 --- a/azureblob-storage/dependencies.lock +++ b/azureblob-storage/dependencies.lock @@ -439,50 +439,6 @@ "com.fasterxml.jackson.dataformat:jackson-dataformat-xml" ] }, - "com.github.rholder:guava-retrying": { - "locked": "2.0.0", - "transitive": [ - "com.netflix.conductor:conductor-common" - ] - }, - "com.google.code.findbugs:jsr305": { - "locked": "3.0.2", - "transitive": [ - "com.github.rholder:guava-retrying", - "com.google.guava:guava" - ] - }, - "com.google.errorprone:error_prone_annotations": { - "locked": "2.3.4", - "transitive": [ - "com.google.guava:guava" - ] - }, - "com.google.guava:failureaccess": { - "locked": "1.0.1", - "transitive": [ - "com.google.guava:guava" - ] - }, - "com.google.guava:guava": { - "locked": "30.0-jre", - "transitive": [ - "com.github.rholder:guava-retrying", - "com.netflix.conductor:conductor-core" - ] - }, - "com.google.guava:listenablefuture": { - "locked": "9999.0-empty-to-avoid-conflict-with-guava", - "transitive": [ - "com.google.guava:guava" - ] - }, - "com.google.j2objc:j2objc-annotations": { - "locked": "1.3", - "transitive": [ - "com.google.guava:guava" - ] - }, "com.google.protobuf:protobuf-java": { "locked": "3.13.0", "transitive": [ @@ -754,12 +710,6 @@ "com.netflix.conductor:conductor-core" ] }, - "org.checkerframework:checker-qual": { - "locked": "3.5.0", - "transitive": [ - "com.google.guava:guava" - ] - }, "org.codehaus.woodstox:stax2-api": { "locked": "4.2.1", "transitive": [ @@ -1424,50 +1374,6 @@ "com.fasterxml.jackson.dataformat:jackson-dataformat-xml" ] }, - "com.github.rholder:guava-retrying": { - "locked": "2.0.0", - "transitive": [ - "com.netflix.conductor:conductor-common" - ] - }, - "com.google.code.findbugs:jsr305": { - "locked": "3.0.2", - "transitive": [ - "com.github.rholder:guava-retrying", - "com.google.guava:guava" - ] - }, - "com.google.errorprone:error_prone_annotations": { - "locked": "2.3.4", - "transitive": [ - "com.google.guava:guava" - ] - }, - "com.google.guava:failureaccess": { - "locked": "1.0.1", - "transitive": [ - "com.google.guava:guava" - ] - }, - "com.google.guava:guava": { - "locked": "30.0-jre", - "transitive": [ - "com.github.rholder:guava-retrying", - "com.netflix.conductor:conductor-core" - ] - }, - "com.google.guava:listenablefuture": { - "locked": "9999.0-empty-to-avoid-conflict-with-guava", - "transitive": [ - "com.google.guava:guava" - ] - }, - "com.google.j2objc:j2objc-annotations": { - "locked": "1.3", - "transitive": [ - "com.google.guava:guava" - ] - }, "com.google.protobuf:protobuf-java": { "locked": "3.13.0", "transitive": [ @@ -1791,12 +1697,6 @@ "org.springframework.boot:spring-boot-starter-test" ] }, - "org.checkerframework:checker-qual": { - "locked": "3.5.0", - "transitive": [ - "com.google.guava:guava" - ] - }, "org.codehaus.woodstox:stax2-api": { "locked": "4.2.1", "transitive": [ diff --git a/build.gradle b/build.gradle index d45c52756f..ffb66ca11a 100644 --- a/build.gradle +++ b/build.gradle @@ -136,6 +136,7 @@ allprojects { testLogging { events = ["SKIPPED", "FAILED"] exceptionFormat = "full" + displayGranularity = 1 showStandardStreams = false } } diff --git a/cassandra-persistence/dependencies.lock b/cassandra-persistence/dependencies.lock index 4f43e9299c..c4de7b73d7 100644 --- a/cassandra-persistence/dependencies.lock +++ b/cassandra-persistence/dependencies.lock @@ -330,49 +330,10 @@ "com.github.jnr:jnr-ffi" ] }, - "com.github.rholder:guava-retrying": { - "locked": "2.0.0", - "transitive": [ - "com.netflix.conductor:conductor-common" - ] - }, - "com.google.code.findbugs:jsr305": { - "locked": "3.0.2", - "transitive": [ - "com.github.rholder:guava-retrying", - "com.google.guava:guava" - ] - }, - "com.google.errorprone:error_prone_annotations": { - "locked": "2.3.4", - "transitive": [ - "com.google.guava:guava" - ] - }, - "com.google.guava:failureaccess": { - "locked": "1.0.1", - "transitive": [ - "com.google.guava:guava" - ] - }, "com.google.guava:guava": { - "locked": "30.0-jre", - "transitive": [ - "com.datastax.cassandra:cassandra-driver-core", - "com.github.rholder:guava-retrying", - "com.netflix.conductor:conductor-core" - ] - }, - "com.google.guava:listenablefuture": { - "locked": "9999.0-empty-to-avoid-conflict-with-guava", - "transitive": [ - "com.google.guava:guava" - ] - }, - "com.google.j2objc:j2objc-annotations": { - "locked": "1.3", + "locked": "19.0", "transitive": [ - "com.google.guava:guava" + "com.datastax.cassandra:cassandra-driver-core" ] }, "com.google.protobuf:protobuf-java": { @@ -562,12 +523,6 @@ "com.netflix.conductor:conductor-core" ] }, - "org.checkerframework:checker-qual": { - "locked": "3.5.0", - "transitive": [ - "com.google.guava:guava" - ] - }, "org.ow2.asm:asm": { "locked": "5.0.4", "transitive": [ @@ -1496,49 +1451,10 @@ "com.github.jnr:jnr-ffi" ] }, - "com.github.rholder:guava-retrying": { - "locked": "2.0.0", - "transitive": [ - "com.netflix.conductor:conductor-common" - ] - }, - "com.google.code.findbugs:jsr305": { - "locked": "3.0.2", - "transitive": [ - "com.github.rholder:guava-retrying", - "com.google.guava:guava" - ] - }, - "com.google.errorprone:error_prone_annotations": { - "locked": "2.3.4", - "transitive": [ - "com.google.guava:guava" - ] - }, - "com.google.guava:failureaccess": { - "locked": "1.0.1", - "transitive": [ - "com.google.guava:guava" - ] - }, "com.google.guava:guava": { - "locked": "30.0-jre", - "transitive": [ - "com.datastax.cassandra:cassandra-driver-core", - "com.github.rholder:guava-retrying", - "com.netflix.conductor:conductor-core" - ] - }, - "com.google.guava:listenablefuture": { - "locked": "9999.0-empty-to-avoid-conflict-with-guava", - "transitive": [ - "com.google.guava:guava" - ] - }, - "com.google.j2objc:j2objc-annotations": { - "locked": "1.3", + "locked": "19.0", "transitive": [ - "com.google.guava:guava" + "com.datastax.cassandra:cassandra-driver-core" ] }, "com.google.protobuf:protobuf-java": { @@ -1846,12 +1762,6 @@ "org.springframework.boot:spring-boot-starter-test" ] }, - "org.checkerframework:checker-qual": { - "locked": "3.5.0", - "transitive": [ - "com.google.guava:guava" - ] - }, "org.codehaus.groovy:groovy": { "locked": "2.5.14", "transitive": [ diff --git a/cassandra-persistence/src/main/java/com/netflix/conductor/cassandra/dao/CassandraExecutionDAO.java b/cassandra-persistence/src/main/java/com/netflix/conductor/cassandra/dao/CassandraExecutionDAO.java index 4341186d70..f8184f4b56 100644 --- a/cassandra-persistence/src/main/java/com/netflix/conductor/cassandra/dao/CassandraExecutionDAO.java +++ b/cassandra-persistence/src/main/java/com/netflix/conductor/cassandra/dao/CassandraExecutionDAO.java @@ -12,12 +12,7 @@ */ package com.netflix.conductor.cassandra.dao; -import java.util.ArrayList; -import java.util.Comparator; -import java.util.LinkedList; -import java.util.List; -import java.util.Optional; -import java.util.UUID; +import java.util.*; import java.util.stream.Collectors; import org.slf4j.Logger; @@ -28,7 +23,6 @@ import com.netflix.conductor.cassandra.util.Statements; import com.netflix.conductor.common.metadata.events.EventExecution; import com.netflix.conductor.common.metadata.tasks.TaskDef; -import com.netflix.conductor.common.utils.RetryUtil; import com.netflix.conductor.core.exception.ApplicationException; import com.netflix.conductor.core.exception.ApplicationException.Code; import com.netflix.conductor.dao.ConcurrentExecutionLimitDAO; @@ -37,25 +31,12 @@ import com.netflix.conductor.model.TaskModel; import com.netflix.conductor.model.WorkflowModel; -import com.datastax.driver.core.BatchStatement; -import com.datastax.driver.core.PreparedStatement; -import com.datastax.driver.core.ResultSet; -import com.datastax.driver.core.Row; -import com.datastax.driver.core.Session; +import com.datastax.driver.core.*; import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Preconditions; -import static com.netflix.conductor.cassandra.util.Constants.DEFAULT_SHARD_ID; -import static com.netflix.conductor.cassandra.util.Constants.DEFAULT_TOTAL_PARTITIONS; -import static com.netflix.conductor.cassandra.util.Constants.ENTITY_KEY; -import static com.netflix.conductor.cassandra.util.Constants.ENTITY_TYPE_TASK; -import static com.netflix.conductor.cassandra.util.Constants.ENTITY_TYPE_WORKFLOW; -import static com.netflix.conductor.cassandra.util.Constants.PAYLOAD_KEY; -import static com.netflix.conductor.cassandra.util.Constants.TASK_ID_KEY; -import static com.netflix.conductor.cassandra.util.Constants.TOTAL_PARTITIONS_KEY; -import static com.netflix.conductor.cassandra.util.Constants.TOTAL_TASKS_KEY; -import static com.netflix.conductor.cassandra.util.Constants.WORKFLOW_ID_KEY; +import static com.netflix.conductor.cassandra.util.Constants.*; @Trace public class CassandraExecutionDAO extends CassandraBaseDAO @@ -746,19 +727,11 @@ public void addTaskToLimit(TaskModel task) { try { recordCassandraDaoRequests( "addTaskToLimit", task.getTaskType(), task.getWorkflowType()); - new RetryUtil<>() - .retryOnException( - () -> - session.execute( - updateTaskDefLimitStatement.bind( - UUID.fromString(task.getWorkflowInstanceId()), - task.getTaskDefName(), - UUID.fromString(task.getTaskId()))), - null, - null, - 3, - "Adding to task_def_limit", - "addTaskToLimit"); + session.execute( + updateTaskDefLimitStatement.bind( + UUID.fromString(task.getWorkflowInstanceId()), + task.getTaskDefName(), + UUID.fromString(task.getTaskId()))); } catch (Exception e) { Monitors.error(CLASS_NAME, "addTaskToLimit"); String errorMsg = @@ -775,18 +748,9 @@ public void removeTaskFromLimit(TaskModel task) { try { recordCassandraDaoRequests( "removeTaskFromLimit", task.getTaskType(), task.getWorkflowType()); - new RetryUtil<>() - .retryOnException( - () -> - session.execute( - deleteTaskDefLimitStatement.bind( - task.getTaskDefName(), - UUID.fromString(task.getTaskId()))), - null, - null, - 3, - "Deleting from task_def_limit", - "removeTaskFromLimit"); + session.execute( + deleteTaskDefLimitStatement.bind( + task.getTaskDefName(), UUID.fromString(task.getTaskId()))); } catch (Exception e) { Monitors.error(CLASS_NAME, "removeTaskFromLimit"); String errorMsg = diff --git a/client-spring/dependencies.lock b/client-spring/dependencies.lock index 8a725b089a..7a115e4427 100644 --- a/client-spring/dependencies.lock +++ b/client-spring/dependencies.lock @@ -385,12 +385,6 @@ "com.github.vlsi.compactmap:compactmap" ] }, - "com.github.rholder:guava-retrying": { - "locked": "2.0.0", - "transitive": [ - "com.netflix.conductor:conductor-common" - ] - }, "com.github.vlsi.compactmap:compactmap": { "locked": "2.0", "transitive": [ @@ -400,8 +394,6 @@ "com.google.code.findbugs:jsr305": { "locked": "3.0.2", "transitive": [ - "com.github.rholder:guava-retrying", - "com.google.guava:guava", "com.netflix.archaius:archaius-core", "com.netflix.eureka:eureka-client", "com.netflix.netflix-commons:netflix-infix" @@ -413,47 +405,21 @@ "com.netflix.netflix-commons:netflix-infix" ] }, - "com.google.errorprone:error_prone_annotations": { - "locked": "2.3.4", - "transitive": [ - "com.google.guava:guava" - ] - }, - "com.google.guava:failureaccess": { - "locked": "1.0.1", - "transitive": [ - "com.google.guava:guava" - ] - }, "com.google.guava:guava": { - "locked": "30.0-jre", + "locked": "19.0", "transitive": [ - "com.github.rholder:guava-retrying", "com.google.inject:guice", "com.netflix.archaius:archaius-core", - "com.netflix.conductor:conductor-client", "com.netflix.netflix-commons:netflix-infix", "com.netflix.servo:servo-core" ] }, - "com.google.guava:listenablefuture": { - "locked": "9999.0-empty-to-avoid-conflict-with-guava", - "transitive": [ - "com.google.guava:guava" - ] - }, "com.google.inject:guice": { "locked": "4.1.0", "transitive": [ "com.netflix.eureka:eureka-client" ] }, - "com.google.j2objc:j2objc-annotations": { - "locked": "1.3", - "transitive": [ - "com.google.guava:guava" - ] - }, "com.google.protobuf:protobuf-java": { "locked": "3.13.0", "transitive": [ @@ -715,12 +681,6 @@ "com.netflix.conductor:conductor-common" ] }, - "org.checkerframework:checker-qual": { - "locked": "3.5.0", - "transitive": [ - "com.google.guava:guava" - ] - }, "org.codehaus.jettison:jettison": { "locked": "1.4.0", "transitive": [ @@ -1427,12 +1387,6 @@ "com.github.vlsi.compactmap:compactmap" ] }, - "com.github.rholder:guava-retrying": { - "locked": "2.0.0", - "transitive": [ - "com.netflix.conductor:conductor-common" - ] - }, "com.github.vlsi.compactmap:compactmap": { "locked": "2.0", "transitive": [ @@ -1442,8 +1396,6 @@ "com.google.code.findbugs:jsr305": { "locked": "3.0.2", "transitive": [ - "com.github.rholder:guava-retrying", - "com.google.guava:guava", "com.netflix.archaius:archaius-core", "com.netflix.eureka:eureka-client", "com.netflix.netflix-commons:netflix-infix" @@ -1455,47 +1407,21 @@ "com.netflix.netflix-commons:netflix-infix" ] }, - "com.google.errorprone:error_prone_annotations": { - "locked": "2.3.4", - "transitive": [ - "com.google.guava:guava" - ] - }, - "com.google.guava:failureaccess": { - "locked": "1.0.1", - "transitive": [ - "com.google.guava:guava" - ] - }, "com.google.guava:guava": { - "locked": "30.0-jre", + "locked": "19.0", "transitive": [ - "com.github.rholder:guava-retrying", "com.google.inject:guice", "com.netflix.archaius:archaius-core", - "com.netflix.conductor:conductor-client", "com.netflix.netflix-commons:netflix-infix", "com.netflix.servo:servo-core" ] }, - "com.google.guava:listenablefuture": { - "locked": "9999.0-empty-to-avoid-conflict-with-guava", - "transitive": [ - "com.google.guava:guava" - ] - }, "com.google.inject:guice": { "locked": "4.1.0", "transitive": [ "com.netflix.eureka:eureka-client" ] }, - "com.google.j2objc:j2objc-annotations": { - "locked": "1.3", - "transitive": [ - "com.google.guava:guava" - ] - }, "com.google.protobuf:protobuf-java": { "locked": "3.13.0", "transitive": [ @@ -1820,12 +1746,6 @@ "org.springframework.boot:spring-boot-starter-test" ] }, - "org.checkerframework:checker-qual": { - "locked": "3.5.0", - "transitive": [ - "com.google.guava:guava" - ] - }, "org.codehaus.jettison:jettison": { "locked": "1.4.0", "transitive": [ diff --git a/client/build.gradle b/client/build.gradle index 1a5ba7f2ef..285ac6ceb7 100644 --- a/client/build.gradle +++ b/client/build.gradle @@ -18,8 +18,6 @@ configurations.all { dependencies { implementation project(':conductor-common') - // SBMTODO: remove guava dep - implementation "com.google.guava:guava:${revGuava}" implementation "com.sun.jersey:jersey-client:${revJersey}" diff --git a/client/dependencies.lock b/client/dependencies.lock index b885225afc..5186500693 100644 --- a/client/dependencies.lock +++ b/client/dependencies.lock @@ -84,47 +84,22 @@ "com.google.code.findbugs:jsr305": { "locked": "3.0.2", "transitive": [ - "com.google.guava:guava", "com.netflix.eureka:eureka-client" ] }, - "com.google.errorprone:error_prone_annotations": { - "locked": "2.3.4", - "transitive": [ - "com.google.guava:guava" - ] - }, - "com.google.guava:failureaccess": { - "locked": "1.0.1", - "transitive": [ - "com.google.guava:guava" - ] - }, "com.google.guava:guava": { - "locked": "30.0-jre", + "locked": "19.0", "transitive": [ "com.google.inject:guice", "com.netflix.servo:servo-core" ] }, - "com.google.guava:listenablefuture": { - "locked": "9999.0-empty-to-avoid-conflict-with-guava", - "transitive": [ - "com.google.guava:guava" - ] - }, "com.google.inject:guice": { "locked": "4.1.0", "transitive": [ "com.netflix.eureka:eureka-client" ] }, - "com.google.j2objc:j2objc-annotations": { - "locked": "1.3", - "transitive": [ - "com.google.guava:guava" - ] - }, "com.netflix.archaius:archaius-core": { "locked": "0.7.6", "transitive": [ @@ -280,12 +255,6 @@ "org.apache.logging.log4j:log4j-web": { "locked": "2.17.1" }, - "org.checkerframework:checker-qual": { - "locked": "3.5.0", - "transitive": [ - "com.google.guava:guava" - ] - }, "org.slf4j:slf4j-api": { "locked": "1.7.30", "transitive": [ @@ -452,12 +421,6 @@ "com.github.vlsi.compactmap:compactmap" ] }, - "com.github.rholder:guava-retrying": { - "locked": "2.0.0", - "transitive": [ - "com.netflix.conductor:conductor-common" - ] - }, "com.github.vlsi.compactmap:compactmap": { "locked": "2.0", "transitive": [ @@ -467,8 +430,6 @@ "com.google.code.findbugs:jsr305": { "locked": "3.0.2", "transitive": [ - "com.github.rholder:guava-retrying", - "com.google.guava:guava", "com.netflix.archaius:archaius-core", "com.netflix.eureka:eureka-client", "com.netflix.netflix-commons:netflix-infix" @@ -480,46 +441,21 @@ "com.netflix.netflix-commons:netflix-infix" ] }, - "com.google.errorprone:error_prone_annotations": { - "locked": "2.3.4", - "transitive": [ - "com.google.guava:guava" - ] - }, - "com.google.guava:failureaccess": { - "locked": "1.0.1", - "transitive": [ - "com.google.guava:guava" - ] - }, "com.google.guava:guava": { - "locked": "30.0-jre", + "locked": "19.0", "transitive": [ - "com.github.rholder:guava-retrying", "com.google.inject:guice", "com.netflix.archaius:archaius-core", "com.netflix.netflix-commons:netflix-infix", "com.netflix.servo:servo-core" ] }, - "com.google.guava:listenablefuture": { - "locked": "9999.0-empty-to-avoid-conflict-with-guava", - "transitive": [ - "com.google.guava:guava" - ] - }, "com.google.inject:guice": { "locked": "4.1.0", "transitive": [ "com.netflix.eureka:eureka-client" ] }, - "com.google.j2objc:j2objc-annotations": { - "locked": "1.3", - "transitive": [ - "com.google.guava:guava" - ] - }, "com.google.protobuf:protobuf-java": { "locked": "3.13.0", "transitive": [ @@ -753,12 +689,6 @@ "com.netflix.conductor:conductor-common" ] }, - "org.checkerframework:checker-qual": { - "locked": "3.5.0", - "transitive": [ - "com.google.guava:guava" - ] - }, "org.codehaus.jettison:jettison": { "locked": "1.4.0", "transitive": [ @@ -1003,47 +933,22 @@ "com.google.code.findbugs:jsr305": { "locked": "3.0.2", "transitive": [ - "com.google.guava:guava", "com.netflix.eureka:eureka-client" ] }, - "com.google.errorprone:error_prone_annotations": { - "locked": "2.3.4", - "transitive": [ - "com.google.guava:guava" - ] - }, - "com.google.guava:failureaccess": { - "locked": "1.0.1", - "transitive": [ - "com.google.guava:guava" - ] - }, "com.google.guava:guava": { - "locked": "30.0-jre", + "locked": "19.0", "transitive": [ "com.google.inject:guice", "com.netflix.servo:servo-core" ] }, - "com.google.guava:listenablefuture": { - "locked": "9999.0-empty-to-avoid-conflict-with-guava", - "transitive": [ - "com.google.guava:guava" - ] - }, "com.google.inject:guice": { "locked": "4.1.0", "transitive": [ "com.netflix.eureka:eureka-client" ] }, - "com.google.j2objc:j2objc-annotations": { - "locked": "1.3", - "transitive": [ - "com.google.guava:guava" - ] - }, "com.jayway.jsonpath:json-path": { "locked": "2.4.0", "transitive": [ @@ -1277,12 +1182,6 @@ "org.springframework.boot:spring-boot-starter-test" ] }, - "org.checkerframework:checker-qual": { - "locked": "3.5.0", - "transitive": [ - "com.google.guava:guava" - ] - }, "org.hamcrest:hamcrest": { "locked": "2.2", "transitive": [ @@ -1652,12 +1551,6 @@ "com.github.vlsi.compactmap:compactmap" ] }, - "com.github.rholder:guava-retrying": { - "locked": "2.0.0", - "transitive": [ - "com.netflix.conductor:conductor-common" - ] - }, "com.github.vlsi.compactmap:compactmap": { "locked": "2.0", "transitive": [ @@ -1667,8 +1560,6 @@ "com.google.code.findbugs:jsr305": { "locked": "3.0.2", "transitive": [ - "com.github.rholder:guava-retrying", - "com.google.guava:guava", "com.netflix.archaius:archaius-core", "com.netflix.eureka:eureka-client", "com.netflix.netflix-commons:netflix-infix" @@ -1680,46 +1571,21 @@ "com.netflix.netflix-commons:netflix-infix" ] }, - "com.google.errorprone:error_prone_annotations": { - "locked": "2.3.4", - "transitive": [ - "com.google.guava:guava" - ] - }, - "com.google.guava:failureaccess": { - "locked": "1.0.1", - "transitive": [ - "com.google.guava:guava" - ] - }, "com.google.guava:guava": { - "locked": "30.0-jre", + "locked": "19.0", "transitive": [ - "com.github.rholder:guava-retrying", "com.google.inject:guice", "com.netflix.archaius:archaius-core", "com.netflix.netflix-commons:netflix-infix", "com.netflix.servo:servo-core" ] }, - "com.google.guava:listenablefuture": { - "locked": "9999.0-empty-to-avoid-conflict-with-guava", - "transitive": [ - "com.google.guava:guava" - ] - }, "com.google.inject:guice": { "locked": "4.1.0", "transitive": [ "com.netflix.eureka:eureka-client" ] }, - "com.google.j2objc:j2objc-annotations": { - "locked": "1.3", - "transitive": [ - "com.google.guava:guava" - ] - }, "com.google.protobuf:protobuf-java": { "locked": "3.13.0", "transitive": [ @@ -2028,12 +1894,6 @@ "org.springframework.boot:spring-boot-starter-test" ] }, - "org.checkerframework:checker-qual": { - "locked": "3.5.0", - "transitive": [ - "com.google.guava:guava" - ] - }, "org.codehaus.jettison:jettison": { "locked": "1.4.0", "transitive": [ diff --git a/client/src/main/java/com/netflix/conductor/client/automator/TaskPollExecutor.java b/client/src/main/java/com/netflix/conductor/client/automator/TaskPollExecutor.java index e2ae52f70f..815ec338f8 100644 --- a/client/src/main/java/com/netflix/conductor/client/automator/TaskPollExecutor.java +++ b/client/src/main/java/com/netflix/conductor/client/automator/TaskPollExecutor.java @@ -1,5 +1,5 @@ /* - * Copyright 2020 Netflix, Inc. + * Copyright 2022 Netflix, Inc. *

* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at @@ -18,11 +18,8 @@ import java.util.Map; import java.util.Objects; import java.util.Optional; -import java.util.concurrent.CompletableFuture; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; -import java.util.concurrent.ThreadPoolExecutor; -import java.util.concurrent.TimeUnit; +import java.util.concurrent.*; +import java.util.function.Function; import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.concurrent.BasicThreadFactory; @@ -36,7 +33,6 @@ import com.netflix.conductor.client.worker.Worker; import com.netflix.conductor.common.metadata.tasks.Task; import com.netflix.conductor.common.metadata.tasks.TaskResult; -import com.netflix.conductor.common.utils.RetryUtil; import com.netflix.discovery.EurekaClient; import com.netflix.spectator.api.Registry; import com.netflix.spectator.api.Spectator; @@ -259,7 +255,7 @@ private void executeTask(Worker worker, Task task) { worker.getClass().getSimpleName(), worker.getIdentity(), result.getStatus()); - updateWithRetry(updateRetryCount, task, result, worker); + updateTaskResult(updateRetryCount, task, result, worker); } private void finalizeTask(Task task, Throwable throwable) { @@ -279,44 +275,25 @@ private void finalizeTask(Task task, Throwable throwable) { } } - private void updateWithRetry(int count, Task task, TaskResult result, Worker worker) { + private void updateTaskResult(int count, Task task, TaskResult result, Worker worker) { try { - String updateTaskDesc = - String.format( - "Retry updating task result: %s for task: %s in worker: %s", - result.toString(), task.getTaskDefName(), worker.getIdentity()); - String evaluatePayloadDesc = - String.format( - "Evaluate Task payload for task: %s in worker: %s", - task.getTaskDefName(), worker.getIdentity()); - String methodName = "updateWithRetry"; - TaskResult finalResult = - new RetryUtil() - .retryOnException( - () -> { - TaskResult taskResult = result.copy(); - taskClient.evaluateAndUploadLargePayload( - taskResult, task.getTaskType()); - return taskResult; - }, - null, - null, - count, - evaluatePayloadDesc, - methodName); - - new RetryUtil<>() - .retryOnException( - () -> { - taskClient.updateTask(finalResult); + retryOperation( + (TaskResult taskResult) -> { + taskClient.evaluateAndUploadLargePayload( + taskResult, task.getTaskType()); return null; }, - null, - null, count, - updateTaskDesc, - methodName); + result); + + retryOperation( + (TaskResult taskResult) -> { + taskClient.updateTask(taskResult); + return null; + }, + count, + finalResult); } catch (Exception e) { worker.onErrorUpdate(task); MetricsContainer.incrementTaskUpdateErrorCount(worker.getTaskDefName(), e); @@ -328,6 +305,27 @@ private void updateWithRetry(int count, Task task, TaskResult result, Worker wor } } + private TaskResult retryOperation( + Function operation, int count, TaskResult result) { + int index = 0; + while (index < count) { + try { + TaskResult taskResult = result.copy(); + operation.apply(taskResult); + return taskResult; + } catch (Exception e) { + index++; + try { + Thread.sleep(500L); + } catch (InterruptedException ie) { + LOGGER.error("Retry interrupted", ie); + } + } + } + throw new RuntimeException( + String.format("Exhausted retries for updating task: %s", result.getTaskId())); + } + private void handleException(Throwable t, TaskResult result, Worker worker, Task task) { LOGGER.error(String.format("Error while executing task %s", task.toString()), t); MetricsContainer.incrementTaskExecutionErrorCount(worker.getTaskDefName(), t); @@ -338,7 +336,7 @@ private void handleException(Throwable t, TaskResult result, Worker worker, Task t.printStackTrace(new PrintWriter(stringWriter)); result.log(stringWriter.toString()); - updateWithRetry(updateRetryCount, task, result, worker); + updateTaskResult(updateRetryCount, task, result, worker); } private PollingSemaphore getPollingSemaphore(String taskType) { diff --git a/client/src/test/java/com/netflix/conductor/client/automator/TaskPollExecutorTest.java b/client/src/test/java/com/netflix/conductor/client/automator/TaskPollExecutorTest.java index 54287ca566..d4029d2e2b 100644 --- a/client/src/test/java/com/netflix/conductor/client/automator/TaskPollExecutorTest.java +++ b/client/src/test/java/com/netflix/conductor/client/automator/TaskPollExecutorTest.java @@ -1,5 +1,5 @@ /* - * Copyright 2020 Netflix, Inc. + * Copyright 2022 Netflix, Inc. *

* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at @@ -15,11 +15,7 @@ import java.util.HashMap; import java.util.Map; import java.util.UUID; -import java.util.concurrent.CountDownLatch; -import java.util.concurrent.ExecutionException; -import java.util.concurrent.Executors; -import java.util.concurrent.ScheduledFuture; -import java.util.concurrent.TimeUnit; +import java.util.concurrent.*; import org.junit.Test; import org.mockito.Mockito; @@ -34,25 +30,19 @@ import com.netflix.conductor.common.metadata.tasks.TaskResult; import com.netflix.discovery.EurekaClient; -import com.google.common.util.concurrent.Uninterruptibles; - import static com.netflix.conductor.common.metadata.tasks.TaskResult.Status.IN_PROGRESS; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNull; import static org.mockito.ArgumentMatchers.any; -import static org.mockito.Mockito.doAnswer; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.times; -import static org.mockito.Mockito.verify; -import static org.mockito.Mockito.when; +import static org.mockito.Mockito.*; public class TaskPollExecutorTest { private static final String TEST_TASK_DEF_NAME = "test"; @Test - public void testTaskExecutionException() { + public void testTaskExecutionException() throws InterruptedException { Worker worker = Worker.create( TEST_TASK_DEF_NAME, @@ -83,13 +73,13 @@ public void testTaskExecutionException() { .scheduleAtFixedRate( () -> taskPollExecutor.pollAndExecute(worker), 0, 1, TimeUnit.SECONDS); - Uninterruptibles.awaitUninterruptibly(latch); + latch.await(); verify(taskClient).updateTask(any()); } @SuppressWarnings("rawtypes") @Test - public void testMultipleTasksExecution() { + public void testMultipleTasksExecution() throws InterruptedException { String outputKey = "KEY"; Task task = testTask(); Worker worker = mock(Worker.class); @@ -101,9 +91,10 @@ public void testMultipleTasksExecution() { private int count = 0; Map outputMap = new HashMap<>(); - public TaskResult answer(InvocationOnMock invocation) { + public TaskResult answer(InvocationOnMock invocation) + throws InterruptedException { // Sleep for 2 seconds to simulate task execution - Uninterruptibles.sleepUninterruptibly(2, TimeUnit.SECONDS); + Thread.sleep(2000L); TaskResult taskResult = new TaskResult(task); outputMap.put(outputKey, count++); taskResult.setOutputData(outputMap); @@ -138,7 +129,7 @@ public TaskResult answer(InvocationOnMock invocation) { Executors.newSingleThreadScheduledExecutor() .scheduleAtFixedRate( () -> taskPollExecutor.pollAndExecute(worker), 0, 1, TimeUnit.SECONDS); - Uninterruptibles.awaitUninterruptibly(latch); + latch.await(); // execute() is called 3 times on the worker (once for each task) verify(worker, times(3)).execute(any()); @@ -146,7 +137,7 @@ public TaskResult answer(InvocationOnMock invocation) { } @Test - public void testLargePayloadCanFailUpdateWithRetry() { + public void testLargePayloadCanFailUpdateWithRetry() throws InterruptedException { Task task = testTask(); Worker worker = mock(Worker.class); @@ -163,7 +154,7 @@ public void testLargePayloadCanFailUpdateWithRetry() { Object[] args = invocation.getArguments(); TaskResult result = (TaskResult) args[0]; assertNull(result.getReasonForIncompletion()); - result.setReasonForIncompletion("some_reason"); + result.setReasonForIncompletion("some_reason_1"); throw new ConductorClientException(); }) .when(taskClient) @@ -184,14 +175,14 @@ public void testLargePayloadCanFailUpdateWithRetry() { Executors.newSingleThreadScheduledExecutor() .scheduleAtFixedRate( () -> taskPollExecutor.pollAndExecute(worker), 0, 1, TimeUnit.SECONDS); - Uninterruptibles.awaitUninterruptibly(latch); + latch.await(); // When evaluateAndUploadLargePayload fails indefinitely, task update shouldn't be called. verify(taskClient, times(0)).updateTask(any()); } @Test - public void testTaskPollException() { + public void testTaskPollException() throws InterruptedException { Task task = testTask(); Worker worker = mock(Worker.class); @@ -224,12 +215,12 @@ public void testTaskPollException() { .scheduleAtFixedRate( () -> taskPollExecutor.pollAndExecute(worker), 0, 1, TimeUnit.SECONDS); - Uninterruptibles.awaitUninterruptibly(latch); + latch.await(); verify(taskClient).updateTask(any()); } @Test - public void testTaskPoll() { + public void testTaskPoll() throws InterruptedException { Task task = testTask(); Worker worker = mock(Worker.class); @@ -260,12 +251,12 @@ public void testTaskPoll() { .scheduleAtFixedRate( () -> taskPollExecutor.pollAndExecute(worker), 0, 1, TimeUnit.SECONDS); - Uninterruptibles.awaitUninterruptibly(latch); + latch.await(); verify(taskClient).updateTask(any()); } @Test - public void testTaskPollDomain() { + public void testTaskPollDomain() throws InterruptedException { TaskClient taskClient = Mockito.mock(TaskClient.class); String testDomain = "foo"; Map taskToDomain = new HashMap<>(); @@ -292,12 +283,12 @@ public void testTaskPollDomain() { .scheduleAtFixedRate( () -> taskPollExecutor.pollAndExecute(worker), 0, 1, TimeUnit.SECONDS); - Uninterruptibles.awaitUninterruptibly(latch); + latch.await(); verify(taskClient).pollTask(TEST_TASK_DEF_NAME, workerName, testDomain); } @Test - public void testPollOutOfDiscoveryForTask() { + public void testPollOutOfDiscoveryForTask() throws InterruptedException { Task task = testTask(); EurekaClient client = mock(EurekaClient.class); @@ -331,7 +322,7 @@ public void testPollOutOfDiscoveryForTask() { .scheduleAtFixedRate( () -> taskPollExecutor.pollAndExecute(worker), 0, 1, TimeUnit.SECONDS); - Uninterruptibles.awaitUninterruptibly(latch); + latch.await(); verify(taskClient).updateTask(any()); } @@ -418,7 +409,7 @@ public void testPollOutOfDiscoveryAsExplicitFalseForTask() } @Test - public void testPollOutOfDiscoveryIsIgnoredWhenDiscoveryIsUp() { + public void testPollOutOfDiscoveryIsIgnoredWhenDiscoveryIsUp() throws InterruptedException { Task task = testTask(); EurekaClient client = mock(EurekaClient.class); @@ -452,12 +443,12 @@ public void testPollOutOfDiscoveryIsIgnoredWhenDiscoveryIsUp() { .scheduleAtFixedRate( () -> taskPollExecutor.pollAndExecute(worker), 0, 1, TimeUnit.SECONDS); - Uninterruptibles.awaitUninterruptibly(latch); + latch.await(); verify(taskClient).updateTask(any()); } @Test - public void testTaskThreadCount() { + public void testTaskThreadCount() throws InterruptedException { TaskClient taskClient = Mockito.mock(TaskClient.class); Map taskThreadCount = new HashMap<>(); @@ -485,7 +476,7 @@ public void testTaskThreadCount() { .scheduleAtFixedRate( () -> taskPollExecutor.pollAndExecute(worker), 0, 1, TimeUnit.SECONDS); - Uninterruptibles.awaitUninterruptibly(latch); + latch.await(); verify(taskClient).pollTask(TEST_TASK_DEF_NAME, workerName, null); } diff --git a/common/build.gradle b/common/build.gradle index 47901c6208..0d2aab4e0e 100644 --- a/common/build.gradle +++ b/common/build.gradle @@ -13,8 +13,6 @@ dependencies { implementation "org.apache.commons:commons-lang3" - implementation "com.github.rholder:guava-retrying:${revGuavaRetrying}" - implementation "org.apache.bval:bval-jsr:${revBval}" implementation "com.google.protobuf:protobuf-java:${revProtoBuf}" diff --git a/common/dependencies.lock b/common/dependencies.lock index a387d087e7..78957aef7b 100644 --- a/common/dependencies.lock +++ b/common/dependencies.lock @@ -48,7 +48,7 @@ ] }, "com.google.protobuf:protobuf-java": { - "locked": "3.20.0", + "locked": "3.20.1", "transitive": [ "com.netflix.conductor:conductor-annotations-processor" ] @@ -174,46 +174,6 @@ "org.hibernate.validator:hibernate-validator" ] }, - "com.github.rholder:guava-retrying": { - "locked": "2.0.0" - }, - "com.google.code.findbugs:jsr305": { - "locked": "3.0.2", - "transitive": [ - "com.github.rholder:guava-retrying", - "com.google.guava:guava" - ] - }, - "com.google.errorprone:error_prone_annotations": { - "locked": "2.11.0", - "transitive": [ - "com.google.guava:guava" - ] - }, - "com.google.guava:failureaccess": { - "locked": "1.0.1", - "transitive": [ - "com.google.guava:guava" - ] - }, - "com.google.guava:guava": { - "locked": "31.1-jre", - "transitive": [ - "com.github.rholder:guava-retrying" - ] - }, - "com.google.guava:listenablefuture": { - "locked": "9999.0-empty-to-avoid-conflict-with-guava", - "transitive": [ - "com.google.guava:guava" - ] - }, - "com.google.j2objc:j2objc-annotations": { - "locked": "1.3", - "transitive": [ - "com.google.guava:guava" - ] - }, "com.google.protobuf:protobuf-java": { "locked": "3.13.0" }, @@ -302,12 +262,6 @@ "org.apache.logging.log4j:log4j-web": { "locked": "2.17.1" }, - "org.checkerframework:checker-qual": { - "locked": "3.12.0", - "transitive": [ - "com.google.guava:guava" - ] - }, "org.glassfish:jakarta.el": { "locked": "3.0.3", "transitive": [ @@ -342,16 +296,16 @@ ] }, "org.springdoc:springdoc-openapi-common": { - "locked": "1.6.7", + "locked": "1.6.8", "transitive": [ "org.springdoc:springdoc-openapi-webmvc-core" ] }, "org.springdoc:springdoc-openapi-ui": { - "locked": "1.6.7" + "locked": "1.6.8" }, "org.springdoc:springdoc-openapi-webmvc-core": { - "locked": "1.6.7", + "locked": "1.6.8", "transitive": [ "org.springdoc:springdoc-openapi-ui" ] @@ -483,46 +437,6 @@ "com.fasterxml.jackson.core:jackson-databind": { "locked": "2.11.4" }, - "com.github.rholder:guava-retrying": { - "locked": "2.0.0" - }, - "com.google.code.findbugs:jsr305": { - "locked": "3.0.2", - "transitive": [ - "com.github.rholder:guava-retrying", - "com.google.guava:guava" - ] - }, - "com.google.errorprone:error_prone_annotations": { - "locked": "2.11.0", - "transitive": [ - "com.google.guava:guava" - ] - }, - "com.google.guava:failureaccess": { - "locked": "1.0.1", - "transitive": [ - "com.google.guava:guava" - ] - }, - "com.google.guava:guava": { - "locked": "31.1-jre", - "transitive": [ - "com.github.rholder:guava-retrying" - ] - }, - "com.google.guava:listenablefuture": { - "locked": "9999.0-empty-to-avoid-conflict-with-guava", - "transitive": [ - "com.google.guava:guava" - ] - }, - "com.google.j2objc:j2objc-annotations": { - "locked": "1.3", - "transitive": [ - "com.google.guava:guava" - ] - }, "com.google.protobuf:protobuf-java": { "locked": "3.13.0" }, @@ -571,12 +485,6 @@ "com.netflix.conductor:conductor-annotations" ] }, - "org.checkerframework:checker-qual": { - "locked": "3.12.0", - "transitive": [ - "com.google.guava:guava" - ] - }, "org.slf4j:slf4j-api": { "locked": "1.7.30", "transitive": [ @@ -606,46 +514,6 @@ "org.hibernate.validator:hibernate-validator" ] }, - "com.github.rholder:guava-retrying": { - "locked": "2.0.0" - }, - "com.google.code.findbugs:jsr305": { - "locked": "3.0.2", - "transitive": [ - "com.github.rholder:guava-retrying", - "com.google.guava:guava" - ] - }, - "com.google.errorprone:error_prone_annotations": { - "locked": "2.11.0", - "transitive": [ - "com.google.guava:guava" - ] - }, - "com.google.guava:failureaccess": { - "locked": "1.0.1", - "transitive": [ - "com.google.guava:guava" - ] - }, - "com.google.guava:guava": { - "locked": "31.1-jre", - "transitive": [ - "com.github.rholder:guava-retrying" - ] - }, - "com.google.guava:listenablefuture": { - "locked": "9999.0-empty-to-avoid-conflict-with-guava", - "transitive": [ - "com.google.guava:guava" - ] - }, - "com.google.j2objc:j2objc-annotations": { - "locked": "1.3", - "transitive": [ - "com.google.guava:guava" - ] - }, "com.google.protobuf:protobuf-java": { "locked": "3.13.0" }, @@ -771,12 +639,6 @@ "org.springframework.boot:spring-boot-starter-test" ] }, - "org.checkerframework:checker-qual": { - "locked": "3.12.0", - "transitive": [ - "com.google.guava:guava" - ] - }, "org.glassfish:jakarta.el": { "locked": "3.0.3", "transitive": [ @@ -1039,46 +901,6 @@ "org.hibernate.validator:hibernate-validator" ] }, - "com.github.rholder:guava-retrying": { - "locked": "2.0.0" - }, - "com.google.code.findbugs:jsr305": { - "locked": "3.0.2", - "transitive": [ - "com.github.rholder:guava-retrying", - "com.google.guava:guava" - ] - }, - "com.google.errorprone:error_prone_annotations": { - "locked": "2.11.0", - "transitive": [ - "com.google.guava:guava" - ] - }, - "com.google.guava:failureaccess": { - "locked": "1.0.1", - "transitive": [ - "com.google.guava:guava" - ] - }, - "com.google.guava:guava": { - "locked": "31.1-jre", - "transitive": [ - "com.github.rholder:guava-retrying" - ] - }, - "com.google.guava:listenablefuture": { - "locked": "9999.0-empty-to-avoid-conflict-with-guava", - "transitive": [ - "com.google.guava:guava" - ] - }, - "com.google.j2objc:j2objc-annotations": { - "locked": "1.3", - "transitive": [ - "com.google.guava:guava" - ] - }, "com.google.protobuf:protobuf-java": { "locked": "3.13.0" }, @@ -1213,12 +1035,6 @@ "org.springframework.boot:spring-boot-starter-test" ] }, - "org.checkerframework:checker-qual": { - "locked": "3.12.0", - "transitive": [ - "com.google.guava:guava" - ] - }, "org.glassfish:jakarta.el": { "locked": "3.0.3", "transitive": [ diff --git a/common/src/main/java/com/netflix/conductor/common/constraints/OwnerEmailMandatoryConstraint.java b/common/src/main/java/com/netflix/conductor/common/constraints/OwnerEmailMandatoryConstraint.java index 6ec57d9acb..55347529dd 100644 --- a/common/src/main/java/com/netflix/conductor/common/constraints/OwnerEmailMandatoryConstraint.java +++ b/common/src/main/java/com/netflix/conductor/common/constraints/OwnerEmailMandatoryConstraint.java @@ -22,7 +22,7 @@ import javax.validation.ConstraintValidatorContext; import javax.validation.Payload; -import com.google.common.base.Strings; +import org.apache.commons.lang3.StringUtils; import static java.lang.annotation.ElementType.FIELD; import static java.lang.annotation.ElementType.TYPE; @@ -51,7 +51,7 @@ public void initialize(OwnerEmailMandatoryConstraint constraintAnnotation) {} @Override public boolean isValid(String ownerEmail, ConstraintValidatorContext context) { - return !ownerEmailMandatory || !Strings.isNullOrEmpty(ownerEmail); + return !ownerEmailMandatory || !StringUtils.isEmpty(ownerEmail); } private static boolean ownerEmailMandatory = true; diff --git a/common/src/main/java/com/netflix/conductor/common/metadata/workflow/SubWorkflowParams.java b/common/src/main/java/com/netflix/conductor/common/metadata/workflow/SubWorkflowParams.java index 82fae46370..8fe77533ab 100644 --- a/common/src/main/java/com/netflix/conductor/common/metadata/workflow/SubWorkflowParams.java +++ b/common/src/main/java/com/netflix/conductor/common/metadata/workflow/SubWorkflowParams.java @@ -23,7 +23,6 @@ import com.fasterxml.jackson.annotation.JsonGetter; import com.fasterxml.jackson.annotation.JsonSetter; -import com.google.common.base.Preconditions; @ProtoMessage public class SubWorkflowParams { @@ -95,9 +94,10 @@ public WorkflowDef getWorkflowDef() { /** @param workflowDef the workflowDefinition to set */ public void setWorkflowDefinition(Object workflowDef) { - Preconditions.checkArgument( - workflowDef == null || workflowDef instanceof WorkflowDef, - "workflowDefinition must be either null or WorkflowDef"); + if (!(workflowDef == null || workflowDef instanceof WorkflowDef)) { + throw new IllegalArgumentException( + "workflowDefinition must be either null or WorkflowDef"); + } this.workflowDefinition = workflowDef; } diff --git a/common/src/main/java/com/netflix/conductor/common/run/Workflow.java b/common/src/main/java/com/netflix/conductor/common/run/Workflow.java index 946ab90ed4..0bd8c4eadb 100644 --- a/common/src/main/java/com/netflix/conductor/common/run/Workflow.java +++ b/common/src/main/java/com/netflix/conductor/common/run/Workflow.java @@ -12,13 +12,7 @@ */ package com.netflix.conductor.common.run; -import java.util.HashMap; -import java.util.HashSet; -import java.util.LinkedList; -import java.util.List; -import java.util.Map; -import java.util.Objects; -import java.util.Set; +import java.util.*; import java.util.stream.Collectors; import javax.validation.constraints.Max; @@ -33,8 +27,6 @@ import com.netflix.conductor.common.metadata.tasks.Task; import com.netflix.conductor.common.metadata.workflow.WorkflowDef; -import com.google.common.base.Preconditions; - @ProtoMessage public class Workflow extends Auditable { @@ -328,7 +320,9 @@ public void setPriority(int priority) { * @return the workflow definition name. */ public String getWorkflowName() { - Preconditions.checkNotNull(workflowDefinition, "Workflow definition is null"); + if (workflowDefinition == null) { + throw new NullPointerException("Workflow definition is null"); + } return workflowDefinition.getName(); } @@ -338,7 +332,9 @@ public String getWorkflowName() { * @return the workflow definition version. */ public int getWorkflowVersion() { - Preconditions.checkNotNull(workflowDefinition, "Workflow definition is null"); + if (workflowDefinition == null) { + throw new NullPointerException("Workflow definition is null"); + } return workflowDefinition.getVersion(); } diff --git a/common/src/main/java/com/netflix/conductor/common/utils/RetryUtil.java b/common/src/main/java/com/netflix/conductor/common/utils/RetryUtil.java deleted file mode 100644 index 49672c8382..0000000000 --- a/common/src/main/java/com/netflix/conductor/common/utils/RetryUtil.java +++ /dev/null @@ -1,147 +0,0 @@ -/* - * Copyright 2020 Netflix, Inc. - *

- * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - *

- * http://www.apache.org/licenses/LICENSE-2.0 - *

- * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on - * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the - * specific language governing permissions and limitations under the License. - */ -package com.netflix.conductor.common.utils; - -import java.util.Optional; -import java.util.concurrent.ExecutionException; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicInteger; -import java.util.function.Supplier; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import com.github.rholder.retry.Attempt; -import com.github.rholder.retry.BlockStrategies; -import com.github.rholder.retry.RetryException; -import com.github.rholder.retry.RetryListener; -import com.github.rholder.retry.Retryer; -import com.github.rholder.retry.RetryerBuilder; -import com.github.rholder.retry.StopStrategies; -import com.github.rholder.retry.WaitStrategies; -import com.google.common.base.Predicate; - -import static java.lang.String.format; - -/** - * Utility class that deals with retries in case of transient failures. - * - *

Note: Create a new {@link RetryUtil} for every operation that needs to retried for the - * stated retries. - * - *

Limitations: - * - *

- * - * @param The type of the object that will be returned by the flaky supplier function - */ -@SuppressWarnings("UnstableApiUsage") -public class RetryUtil { - - private static final Logger LOGGER = LoggerFactory.getLogger(RetryUtil.class); - - private final AtomicInteger internalNumberOfRetries = new AtomicInteger(); - - /** - * A helper method which has the ability to execute a flaky supplier function and retry in case - * of failures. - * - * @param supplierCommand: Any function that is flaky and needs multiple retries. - * @param throwablePredicate: A Guava {@link Predicate} housing the exceptional criteria to - * perform informed filtering before retrying. - * @param resultRetryPredicate: a predicate to be evaluated for a valid condition of the - * expected result - * @param retryCount: Number of times the function is to be retried before failure - * @param shortDescription: A short description of the function that will be used in logging and - * error propagation. The intention of this description is to provide context for - * Operability. - * @param operationName: The name of the function for traceability in logs - * @return an instance of return type of the supplierCommand - * @throws RuntimeException in case of failed attempts to get T, which needs to be returned by - * the supplierCommand. The instance of the returned exception has: - *
    - *
  • A message with shortDescription and operationName with the number of retries made - *
  • And a reference to the original exception generated during the last {@link Attempt} - * of the retry - *
- */ - @SuppressWarnings("Guava") - public T retryOnException( - Supplier supplierCommand, - Predicate throwablePredicate, - Predicate resultRetryPredicate, - int retryCount, - String shortDescription, - String operationName) - throws RuntimeException { - - Retryer retryer = - RetryerBuilder.newBuilder() - .retryIfException( - Optional.ofNullable(throwablePredicate).orElse(exception -> true)) - .retryIfResult( - Optional.ofNullable(resultRetryPredicate).orElse(result -> false)) - .withWaitStrategy( - WaitStrategies.join( - WaitStrategies.exponentialWait(1000, 90, TimeUnit.SECONDS), - WaitStrategies.randomWait( - 100, - TimeUnit.MILLISECONDS, - 500, - TimeUnit.MILLISECONDS))) - .withStopStrategy(StopStrategies.stopAfterAttempt(retryCount)) - .withBlockStrategy(BlockStrategies.threadSleepStrategy()) - .withRetryListener( - new RetryListener() { - @Override - public void onRetry(Attempt attempt) { - LOGGER.debug( - "Attempt # {}, {} millis since first attempt. Operation: {}, description:{}", - attempt.getAttemptNumber(), - attempt.getDelaySinceFirstAttempt(), - operationName, - shortDescription); - internalNumberOfRetries.incrementAndGet(); - } - }) - .build(); - - try { - return retryer.call(supplierCommand::get); - } catch (ExecutionException executionException) { - String errorMessage = - format( - "Operation '%s:%s' failed for the %d time in RetryUtil", - operationName, shortDescription, internalNumberOfRetries.get()); - LOGGER.debug(errorMessage); - throw new RuntimeException(errorMessage, executionException.getCause()); - } catch (RetryException retryException) { - String errorMessage = - format( - "Operation '%s:%s' failed after retrying %d times, retry limit %d", - operationName, shortDescription, internalNumberOfRetries.get(), 3); - LOGGER.error(errorMessage, retryException.getLastFailedAttempt().getExceptionCause()); - throw new RuntimeException( - errorMessage, retryException.getLastFailedAttempt().getExceptionCause()); - } - } -} diff --git a/contribs/dependencies.lock b/contribs/dependencies.lock index 13aaa9339e..e6f17c079b 100644 --- a/contribs/dependencies.lock +++ b/contribs/dependencies.lock @@ -511,16 +511,9 @@ "org.apache.kafka:kafka-clients" ] }, - "com.github.rholder:guava-retrying": { - "locked": "2.0.0", - "transitive": [ - "com.netflix.conductor:conductor-common" - ] - }, "com.google.code.findbugs:jsr305": { "locked": "3.0.2", "transitive": [ - "com.github.rholder:guava-retrying", "com.google.guava:guava" ] }, @@ -537,11 +530,7 @@ ] }, "com.google.guava:guava": { - "locked": "30.0-jre", - "transitive": [ - "com.github.rholder:guava-retrying", - "com.netflix.conductor:conductor-core" - ] + "locked": "30.0-jre" }, "com.google.guava:listenablefuture": { "locked": "9999.0-empty-to-avoid-conflict-with-guava", @@ -2130,12 +2119,6 @@ "org.apache.kafka:kafka-clients" ] }, - "com.github.rholder:guava-retrying": { - "locked": "2.0.0", - "transitive": [ - "com.netflix.conductor:conductor-common" - ] - }, "com.google.code.findbugs:jsr305": { "locked": "3.0.2", "transitive": [ @@ -2145,7 +2128,6 @@ "com.github.java-json-tools:json-schema-validator", "com.github.java-json-tools:msg-simple", "com.github.java-json-tools:uri-template", - "com.github.rholder:guava-retrying", "com.google.guava:guava" ] }, @@ -2165,8 +2147,6 @@ "locked": "30.0-jre", "transitive": [ "com.github.java-json-tools:json-schema-validator", - "com.github.rholder:guava-retrying", - "com.netflix.conductor:conductor-core", "io.swagger:swagger-core", "org.mock-server:mockserver-client-java", "org.mock-server:mockserver-core" diff --git a/contribs/src/main/java/com/netflix/conductor/contribs/storage/S3PayloadStorage.java b/contribs/src/main/java/com/netflix/conductor/contribs/storage/S3PayloadStorage.java index 5be8becb9f..fa53f0b0bf 100644 --- a/contribs/src/main/java/com/netflix/conductor/contribs/storage/S3PayloadStorage.java +++ b/contribs/src/main/java/com/netflix/conductor/contribs/storage/S3PayloadStorage.java @@ -30,11 +30,7 @@ import com.amazonaws.SdkClientException; import com.amazonaws.services.s3.AmazonS3; import com.amazonaws.services.s3.AmazonS3ClientBuilder; -import com.amazonaws.services.s3.model.GeneratePresignedUrlRequest; -import com.amazonaws.services.s3.model.GetObjectRequest; -import com.amazonaws.services.s3.model.ObjectMetadata; -import com.amazonaws.services.s3.model.PutObjectRequest; -import com.amazonaws.services.s3.model.S3Object; +import com.amazonaws.services.s3.model.*; /** * An implementation of {@link ExternalPayloadStorage} using AWS S3 for storing large JSON payload @@ -103,7 +99,10 @@ public ExternalStorageLocation getLocation( .toASCIIString()); return externalStorageLocation; } catch (SdkClientException e) { - String msg = "Error communicating with S3"; + String msg = + String.format( + "Error communicating with S3 - operation:%s, payloadType: %s, path: %s", + operation, payloadType, path); LOGGER.error(msg, e); throw new ApplicationException(ApplicationException.Code.BACKEND_ERROR, msg, e); } catch (URISyntaxException e) { @@ -132,7 +131,9 @@ public void upload(String path, InputStream payload, long payloadSize) { new PutObjectRequest(bucketName, path, payload, objectMetadata); s3Client.putObject(request); } catch (SdkClientException e) { - String msg = "Error communicating with S3"; + String msg = + String.format( + "Error uploading to S3 - path:%s, payloadSize: %d", path, payloadSize); LOGGER.error(msg, e); throw new ApplicationException(ApplicationException.Code.BACKEND_ERROR, msg, e); } @@ -151,7 +152,7 @@ public InputStream download(String path) { S3Object s3Object = s3Client.getObject(new GetObjectRequest(bucketName, path)); return s3Object.getObjectContent(); } catch (SdkClientException e) { - String msg = "Error communicating with S3"; + String msg = String.format("Error downloading from S3 - path:%s", path); LOGGER.error(msg, e); throw new ApplicationException(ApplicationException.Code.BACKEND_ERROR, msg, e); } diff --git a/core/build.gradle b/core/build.gradle index 1347aa5e3b..600b8f219e 100644 --- a/core/build.gradle +++ b/core/build.gradle @@ -16,6 +16,7 @@ dependencies { implementation project(':conductor-common') compileOnly 'org.springframework.boot:spring-boot-starter' compileOnly 'org.springframework.boot:spring-boot-starter-validation' + compileOnly 'org.springframework.retry:spring-retry' implementation "com.fasterxml.jackson.core:jackson-annotations" implementation "com.fasterxml.jackson.core:jackson-databind" @@ -34,8 +35,6 @@ dependencies { implementation "io.reactivex:rxjava:${revRxJava}" - implementation "com.google.guava:guava:${revGuava}" - implementation "com.netflix.spectator:spectator-api:${revSpectator}" implementation "org.apache.bval:bval-jsr:${revBval}" @@ -49,6 +48,7 @@ dependencies { testImplementation "org.glassfish.jaxb:jaxb-runtime:${revJAXB}" testImplementation 'org.springframework.boot:spring-boot-starter-validation' + testImplementation 'org.springframework.retry:spring-retry' testImplementation project(':conductor-common').sourceSets.test.output testImplementation "org.codehaus.groovy:groovy-all:${revGroovy}" diff --git a/core/dependencies.lock b/core/dependencies.lock index 5da10457de..4dca2dbed9 100644 --- a/core/dependencies.lock +++ b/core/dependencies.lock @@ -26,39 +26,6 @@ "org.hibernate.validator:hibernate-validator" ] }, - "com.google.code.findbugs:jsr305": { - "locked": "3.0.2", - "transitive": [ - "com.google.guava:guava" - ] - }, - "com.google.errorprone:error_prone_annotations": { - "locked": "2.3.4", - "transitive": [ - "com.google.guava:guava" - ] - }, - "com.google.guava:failureaccess": { - "locked": "1.0.1", - "transitive": [ - "com.google.guava:guava" - ] - }, - "com.google.guava:guava": { - "locked": "30.0-jre" - }, - "com.google.guava:listenablefuture": { - "locked": "9999.0-empty-to-avoid-conflict-with-guava", - "transitive": [ - "com.google.guava:guava" - ] - }, - "com.google.j2objc:j2objc-annotations": { - "locked": "1.3", - "transitive": [ - "com.google.guava:guava" - ] - }, "com.google.protobuf:protobuf-java": { "locked": "3.13.0" }, @@ -143,12 +110,6 @@ "org.apache.logging.log4j:log4j-web": { "locked": "2.17.1" }, - "org.checkerframework:checker-qual": { - "locked": "3.5.0", - "transitive": [ - "com.google.guava:guava" - ] - }, "org.glassfish:jakarta.el": { "locked": "3.0.3", "transitive": [ @@ -215,6 +176,9 @@ "org.springframework.boot:spring-boot-starter-validation": { "locked": "2.3.12.RELEASE" }, + "org.springframework.retry:spring-retry": { + "locked": "1.2.5.RELEASE" + }, "org.springframework:spring-aop": { "locked": "5.2.15.RELEASE", "transitive": [ @@ -239,6 +203,7 @@ "transitive": [ "org.springframework.boot:spring-boot", "org.springframework.boot:spring-boot-starter", + "org.springframework.retry:spring-retry", "org.springframework:spring-aop", "org.springframework:spring-beans", "org.springframework:spring-context", @@ -284,49 +249,6 @@ "com.netflix.conductor:conductor-common" ] }, - "com.github.rholder:guava-retrying": { - "locked": "2.0.0", - "transitive": [ - "com.netflix.conductor:conductor-common" - ] - }, - "com.google.code.findbugs:jsr305": { - "locked": "3.0.2", - "transitive": [ - "com.github.rholder:guava-retrying", - "com.google.guava:guava" - ] - }, - "com.google.errorprone:error_prone_annotations": { - "locked": "2.3.4", - "transitive": [ - "com.google.guava:guava" - ] - }, - "com.google.guava:failureaccess": { - "locked": "1.0.1", - "transitive": [ - "com.google.guava:guava" - ] - }, - "com.google.guava:guava": { - "locked": "30.0-jre", - "transitive": [ - "com.github.rholder:guava-retrying" - ] - }, - "com.google.guava:listenablefuture": { - "locked": "9999.0-empty-to-avoid-conflict-with-guava", - "transitive": [ - "com.google.guava:guava" - ] - }, - "com.google.j2objc:j2objc-annotations": { - "locked": "1.3", - "transitive": [ - "com.google.guava:guava" - ] - }, "com.google.protobuf:protobuf-java": { "locked": "3.13.0", "transitive": [ @@ -431,12 +353,6 @@ "com.netflix.conductor:conductor-common" ] }, - "org.checkerframework:checker-qual": { - "locked": "3.5.0", - "transitive": [ - "com.google.guava:guava" - ] - }, "org.ow2.asm:asm": { "locked": "5.0.4", "transitive": [ @@ -474,39 +390,6 @@ "org.hibernate.validator:hibernate-validator" ] }, - "com.google.code.findbugs:jsr305": { - "locked": "3.0.2", - "transitive": [ - "com.google.guava:guava" - ] - }, - "com.google.errorprone:error_prone_annotations": { - "locked": "2.3.4", - "transitive": [ - "com.google.guava:guava" - ] - }, - "com.google.guava:failureaccess": { - "locked": "1.0.1", - "transitive": [ - "com.google.guava:guava" - ] - }, - "com.google.guava:guava": { - "locked": "30.0-jre" - }, - "com.google.guava:listenablefuture": { - "locked": "9999.0-empty-to-avoid-conflict-with-guava", - "transitive": [ - "com.google.guava:guava" - ] - }, - "com.google.j2objc:j2objc-annotations": { - "locked": "1.3", - "transitive": [ - "com.google.guava:guava" - ] - }, "com.google.protobuf:protobuf-java": { "locked": "3.13.0" }, @@ -689,12 +572,6 @@ "org.springframework.boot:spring-boot-starter-test" ] }, - "org.checkerframework:checker-qual": { - "locked": "3.5.0", - "transitive": [ - "com.google.guava:guava" - ] - }, "org.codehaus.groovy:groovy": { "locked": "2.5.14", "transitive": [ @@ -1091,6 +968,9 @@ "org.springframework.boot:spring-boot-starter-test" ] }, + "org.springframework.retry:spring-retry": { + "locked": "1.2.5.RELEASE" + }, "org.springframework:spring-aop": { "locked": "5.2.15.RELEASE", "transitive": [ @@ -1116,6 +996,7 @@ "org.springframework.boot:spring-boot", "org.springframework.boot:spring-boot-starter", "org.springframework.boot:spring-boot-starter-test", + "org.springframework.retry:spring-retry", "org.springframework:spring-aop", "org.springframework:spring-beans", "org.springframework:spring-context", @@ -1186,49 +1067,6 @@ "org.hibernate.validator:hibernate-validator" ] }, - "com.github.rholder:guava-retrying": { - "locked": "2.0.0", - "transitive": [ - "com.netflix.conductor:conductor-common" - ] - }, - "com.google.code.findbugs:jsr305": { - "locked": "3.0.2", - "transitive": [ - "com.github.rholder:guava-retrying", - "com.google.guava:guava" - ] - }, - "com.google.errorprone:error_prone_annotations": { - "locked": "2.3.4", - "transitive": [ - "com.google.guava:guava" - ] - }, - "com.google.guava:failureaccess": { - "locked": "1.0.1", - "transitive": [ - "com.google.guava:guava" - ] - }, - "com.google.guava:guava": { - "locked": "30.0-jre", - "transitive": [ - "com.github.rholder:guava-retrying" - ] - }, - "com.google.guava:listenablefuture": { - "locked": "9999.0-empty-to-avoid-conflict-with-guava", - "transitive": [ - "com.google.guava:guava" - ] - }, - "com.google.j2objc:j2objc-annotations": { - "locked": "1.3", - "transitive": [ - "com.google.guava:guava" - ] - }, "com.google.protobuf:protobuf-java": { "locked": "3.13.0", "transitive": [ @@ -1460,12 +1298,6 @@ "org.springframework.boot:spring-boot-starter-test" ] }, - "org.checkerframework:checker-qual": { - "locked": "3.5.0", - "transitive": [ - "com.google.guava:guava" - ] - }, "org.codehaus.groovy:groovy": { "locked": "2.5.14", "transitive": [ @@ -1875,6 +1707,9 @@ "org.springframework.boot:spring-boot-starter-test" ] }, + "org.springframework.retry:spring-retry": { + "locked": "1.2.5.RELEASE" + }, "org.springframework:spring-aop": { "locked": "5.2.15.RELEASE", "transitive": [ @@ -1900,6 +1735,7 @@ "org.springframework.boot:spring-boot", "org.springframework.boot:spring-boot-starter", "org.springframework.boot:spring-boot-starter-test", + "org.springframework.retry:spring-retry", "org.springframework:spring-aop", "org.springframework:spring-beans", "org.springframework:spring-context", diff --git a/core/src/main/java/com/netflix/conductor/annotations/VisibleForTesting.java b/core/src/main/java/com/netflix/conductor/annotations/VisibleForTesting.java new file mode 100644 index 0000000000..4929311283 --- /dev/null +++ b/core/src/main/java/com/netflix/conductor/annotations/VisibleForTesting.java @@ -0,0 +1,24 @@ +/* + * Copyright 2022 Netflix, Inc. + *

+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ +package com.netflix.conductor.annotations; + +import java.lang.annotation.*; + +/** + * Annotates a program element that exists, or is more widely visible than otherwise necessary, only + * for use in test code. + */ +@Retention(RetentionPolicy.CLASS) +@Target({ElementType.FIELD, ElementType.TYPE, ElementType.METHOD}) +@Documented +public @interface VisibleForTesting {} diff --git a/core/src/main/java/com/netflix/conductor/core/config/ConductorCoreConfiguration.java b/core/src/main/java/com/netflix/conductor/core/config/ConductorCoreConfiguration.java index 100cab754d..42f3fcd4c7 100644 --- a/core/src/main/java/com/netflix/conductor/core/config/ConductorCoreConfiguration.java +++ b/core/src/main/java/com/netflix/conductor/core/config/ConductorCoreConfiguration.java @@ -14,12 +14,14 @@ import java.util.List; import java.util.Map; +import java.util.Optional; import java.util.Set; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.ThreadFactory; import java.util.stream.Collectors; +import org.apache.commons.lang3.concurrent.BasicThreadFactory; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Qualifier; @@ -27,6 +29,10 @@ import org.springframework.boot.context.properties.EnableConfigurationProperties; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; +import org.springframework.retry.RetryContext; +import org.springframework.retry.backoff.NoBackOffPolicy; +import org.springframework.retry.policy.SimpleRetryPolicy; +import org.springframework.retry.support.RetryTemplate; import com.netflix.conductor.common.metadata.tasks.TaskType; import com.netflix.conductor.common.utils.ExternalPayloadStorage; @@ -39,10 +45,9 @@ import com.netflix.conductor.core.sync.Lock; import com.netflix.conductor.core.sync.NoopLock; -import com.google.common.util.concurrent.ThreadFactoryBuilder; - import static com.netflix.conductor.core.events.EventQueues.EVENT_QUEUE_PROVIDERS_QUALIFIER; import static com.netflix.conductor.core.execution.tasks.SystemTaskRegistry.ASYNC_SYSTEM_TASKS_QUALIFIER; +import static com.netflix.conductor.core.utils.Utils.isTransientException; import static java.util.function.Function.identity; @@ -83,9 +88,9 @@ public WorkflowStatusListener workflowStatusListener() { @Bean public ExecutorService executorService(ConductorProperties conductorProperties) { ThreadFactory threadFactory = - new ThreadFactoryBuilder() - .setNameFormat("conductor-worker-%d") - .setDaemon(true) + new BasicThreadFactory.Builder() + .namingPattern("conductor-worker-%d") + .daemon(true) .build(); return Executors.newFixedThreadPool( conductorProperties.getExecutorServiceMaxThreadCount(), threadFactory); @@ -112,4 +117,27 @@ public Map getEventQueueProviders( return eventQueueProviders.stream() .collect(Collectors.toMap(EventQueueProvider::getQueueType, identity())); } + + @Bean + public RetryTemplate onTransientErrorRetryTemplate() { + SimpleRetryPolicy retryPolicy = new CustomRetryPolicy(); + retryPolicy.setMaxAttempts(3); + + RetryTemplate retryTemplate = new RetryTemplate(); + retryTemplate.setRetryPolicy(retryPolicy); + retryTemplate.setBackOffPolicy(new NoBackOffPolicy()); + return retryTemplate; + } + + public static class CustomRetryPolicy extends SimpleRetryPolicy { + + @Override + public boolean canRetry(final RetryContext context) { + final Optional lastThrowable = + Optional.ofNullable(context.getLastThrowable()); + return lastThrowable + .map(throwable -> super.canRetry(context) && isTransientException(throwable)) + .orElseGet(() -> super.canRetry(context)); + } + } } diff --git a/core/src/main/java/com/netflix/conductor/core/config/SchedulerConfiguration.java b/core/src/main/java/com/netflix/conductor/core/config/SchedulerConfiguration.java index f5a453b034..364406e102 100644 --- a/core/src/main/java/com/netflix/conductor/core/config/SchedulerConfiguration.java +++ b/core/src/main/java/com/netflix/conductor/core/config/SchedulerConfiguration.java @@ -16,6 +16,7 @@ import java.util.concurrent.Executors; import java.util.concurrent.ThreadFactory; +import org.apache.commons.lang3.concurrent.BasicThreadFactory; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; import org.springframework.scheduling.annotation.EnableAsync; @@ -24,7 +25,6 @@ import org.springframework.scheduling.concurrent.ThreadPoolTaskScheduler; import org.springframework.scheduling.config.ScheduledTaskRegistrar; -import com.google.common.util.concurrent.ThreadFactoryBuilder; import rx.Scheduler; import rx.schedulers.Schedulers; @@ -43,8 +43,8 @@ public class SchedulerConfiguration implements SchedulingConfigurer { @Bean public Scheduler scheduler(ConductorProperties properties) { ThreadFactory threadFactory = - new ThreadFactoryBuilder() - .setNameFormat("event-queue-poll-scheduler-thread-%d") + new BasicThreadFactory.Builder() + .namingPattern("event-queue-poll-scheduler-thread-%d") .build(); Executor executorService = Executors.newFixedThreadPool( @@ -60,7 +60,7 @@ public Executor sweeperExecutor(ConductorProperties properties) { "conductor.app.sweeper-thread-count must be greater than 0."); } ThreadFactory threadFactory = - new ThreadFactoryBuilder().setNameFormat("sweeper-thread-%d").build(); + new BasicThreadFactory.Builder().namingPattern("sweeper-thread-%d").build(); return Executors.newFixedThreadPool(properties.getSweeperThreadCount(), threadFactory); } diff --git a/core/src/main/java/com/netflix/conductor/core/events/DefaultEventProcessor.java b/core/src/main/java/com/netflix/conductor/core/events/DefaultEventProcessor.java index c9422cd8c0..19767c4780 100644 --- a/core/src/main/java/com/netflix/conductor/core/events/DefaultEventProcessor.java +++ b/core/src/main/java/com/netflix/conductor/core/events/DefaultEventProcessor.java @@ -22,9 +22,12 @@ import java.util.concurrent.ThreadFactory; import org.apache.commons.lang3.StringUtils; +import org.apache.commons.lang3.concurrent.BasicThreadFactory; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import org.springframework.retry.support.RetryTemplate; import org.springframework.stereotype.Component; import org.springframework.util.CollectionUtils; @@ -32,11 +35,9 @@ import com.netflix.conductor.common.metadata.events.EventExecution.Status; import com.netflix.conductor.common.metadata.events.EventHandler; import com.netflix.conductor.common.metadata.events.EventHandler.Action; -import com.netflix.conductor.common.utils.RetryUtil; import com.netflix.conductor.core.config.ConductorProperties; import com.netflix.conductor.core.events.queue.Message; import com.netflix.conductor.core.events.queue.ObservableQueue; -import com.netflix.conductor.core.exception.ApplicationException; import com.netflix.conductor.core.execution.evaluators.Evaluator; import com.netflix.conductor.core.utils.JsonUtils; import com.netflix.conductor.metrics.Monitors; @@ -44,9 +45,10 @@ import com.netflix.conductor.service.MetadataService; import com.fasterxml.jackson.databind.ObjectMapper; -import com.google.common.util.concurrent.ThreadFactoryBuilder; import com.spotify.futures.CompletableFutures; +import static com.netflix.conductor.core.utils.Utils.isTransientException; + /** * Event Processor is used to dispatch actions configured in the event handlers, based on incoming * events to the event queues. @@ -61,7 +63,6 @@ public class DefaultEventProcessor { private static final Logger LOGGER = LoggerFactory.getLogger(DefaultEventProcessor.class); - private static final int RETRY_COUNT = 3; private final MetadataService metadataService; private final ExecutionService executionService; @@ -72,6 +73,7 @@ public class DefaultEventProcessor { private final JsonUtils jsonUtils; private final boolean isEventMessageIndexingEnabled; private final Map evaluators; + private final RetryTemplate retryTemplate; public DefaultEventProcessor( ExecutionService executionService, @@ -80,7 +82,8 @@ public DefaultEventProcessor( JsonUtils jsonUtils, ConductorProperties properties, ObjectMapper objectMapper, - Map evaluators) { + Map evaluators, + @Qualifier("onTransientErrorRetryTemplate") RetryTemplate retryTemplate) { this.executionService = executionService; this.metadataService = metadataService; this.actionProcessor = actionProcessor; @@ -94,12 +97,15 @@ public DefaultEventProcessor( + "processing, set conductor.default-event-processor.enabled=false."); } ThreadFactory threadFactory = - new ThreadFactoryBuilder().setNameFormat("event-action-executor-thread-%d").build(); + new BasicThreadFactory.Builder() + .namingPattern("event-action-executor-thread-%d") + .build(); eventActionExecutorService = Executors.newFixedThreadPool( properties.getEventProcessorThreadCount(), threadFactory); this.isEventMessageIndexingEnabled = properties.isEventMessageIndexingEnabled(); + this.retryTemplate = retryTemplate; LOGGER.info("Event Processing is ENABLED"); } @@ -254,30 +260,21 @@ protected CompletableFuture> executeActionsForEventHandler( */ protected EventExecution execute(EventExecution eventExecution, Action action, Object payload) { try { - String methodName = "executeEventAction"; - String description = - String.format( - "Executing action: %s for event: %s with messageId: %s with payload: %s", - action.getAction(), - eventExecution.getId(), - eventExecution.getMessageId(), - payload); - LOGGER.debug(description); + LOGGER.debug( + "Executing action: {} for event: {} with messageId: {} with payload: {}", + action.getAction(), + eventExecution.getId(), + eventExecution.getMessageId(), + payload); Map output = - new RetryUtil>() - .retryOnException( - () -> - actionProcessor.execute( - action, - payload, - eventExecution.getEvent(), - eventExecution.getMessageId()), - this::isTransientException, - null, - RETRY_COUNT, - description, - methodName); + retryTemplate.execute( + context -> + actionProcessor.execute( + action, + payload, + eventExecution.getEvent(), + eventExecution.getMessageId())); if (output != null) { eventExecution.getOutput().putAll(output); } @@ -293,7 +290,7 @@ protected EventExecution execute(EventExecution eventExecution, Action action, O eventExecution.getEvent(), eventExecution.getMessageId(), e); - if (!isTransientException(e.getCause())) { + if (!isTransientException(e)) { // not a transient error, fail the event execution eventExecution.setStatus(Status.FAILED); eventExecution.getOutput().put("exception", e.getMessage()); @@ -307,24 +304,6 @@ protected EventExecution execute(EventExecution eventExecution, Action action, O return eventExecution; } - /** - * Used to determine if the exception is thrown due to a transient failure and the operation is - * expected to succeed upon retrying. - * - * @param throwableException the exception that is thrown - * @return true - if the exception is a transient failure false - if the exception is - * non-transient - */ - protected boolean isTransientException(Throwable throwableException) { - if (throwableException != null) { - return !((throwableException instanceof UnsupportedOperationException) - || (throwableException instanceof ApplicationException - && ((ApplicationException) throwableException).getCode() - != ApplicationException.Code.BACKEND_ERROR)); - } - return true; - } - private Object getPayloadObject(String payload) { Object payloadObject = null; if (payload != null) { diff --git a/core/src/main/java/com/netflix/conductor/core/execution/DeciderService.java b/core/src/main/java/com/netflix/conductor/core/execution/DeciderService.java index e2bd2f8b69..55192b2447 100644 --- a/core/src/main/java/com/netflix/conductor/core/execution/DeciderService.java +++ b/core/src/main/java/com/netflix/conductor/core/execution/DeciderService.java @@ -24,6 +24,7 @@ import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Service; +import com.netflix.conductor.annotations.VisibleForTesting; import com.netflix.conductor.common.metadata.tasks.TaskDef; import com.netflix.conductor.common.metadata.tasks.TaskType; import com.netflix.conductor.common.metadata.workflow.WorkflowDef; @@ -43,8 +44,6 @@ import com.netflix.conductor.model.TaskModel; import com.netflix.conductor.model.WorkflowModel; -import com.google.common.annotations.VisibleForTesting; - import static com.netflix.conductor.common.metadata.tasks.TaskType.TERMINATE; import static com.netflix.conductor.model.TaskModel.Status.*; diff --git a/core/src/main/java/com/netflix/conductor/core/execution/WorkflowExecutor.java b/core/src/main/java/com/netflix/conductor/core/execution/WorkflowExecutor.java index 8bc32df2a5..0519ba7afd 100644 --- a/core/src/main/java/com/netflix/conductor/core/execution/WorkflowExecutor.java +++ b/core/src/main/java/com/netflix/conductor/core/execution/WorkflowExecutor.java @@ -22,13 +22,13 @@ import org.springframework.stereotype.Component; import com.netflix.conductor.annotations.Trace; +import com.netflix.conductor.annotations.VisibleForTesting; import com.netflix.conductor.common.metadata.tasks.*; import com.netflix.conductor.common.metadata.workflow.RerunWorkflowRequest; import com.netflix.conductor.common.metadata.workflow.SkipTaskRequest; import com.netflix.conductor.common.metadata.workflow.WorkflowDef; import com.netflix.conductor.common.metadata.workflow.WorkflowTask; import com.netflix.conductor.common.run.Workflow; -import com.netflix.conductor.common.utils.RetryUtil; import com.netflix.conductor.common.utils.TaskUtils; import com.netflix.conductor.core.WorkflowContext; import com.netflix.conductor.core.config.ConductorProperties; @@ -44,6 +44,7 @@ import com.netflix.conductor.core.utils.IDGenerator; import com.netflix.conductor.core.utils.ParametersUtils; import com.netflix.conductor.core.utils.QueueUtils; +import com.netflix.conductor.core.utils.Utils; import com.netflix.conductor.dao.MetadataDAO; import com.netflix.conductor.dao.QueueDAO; import com.netflix.conductor.metrics.Monitors; @@ -51,9 +52,6 @@ import com.netflix.conductor.model.WorkflowModel; import com.netflix.conductor.service.ExecutionLockService; -import com.google.common.annotations.VisibleForTesting; -import com.google.common.base.Preconditions; - import static com.netflix.conductor.core.exception.ApplicationException.Code.*; import static com.netflix.conductor.core.utils.Utils.DECIDER_QUEUE; import static com.netflix.conductor.model.TaskModel.Status.*; @@ -490,8 +488,7 @@ public void resetCallbacksForWorkflow(String workflowId) { } public String rerun(RerunWorkflowRequest request) { - Preconditions.checkNotNull( - request.getReRunFromWorkflowId(), "reRunFromWorkflowId is missing"); + Utils.checkNotNull(request.getReRunFromWorkflowId(), "reRunFromWorkflowId is missing"); if (!rerunWF( request.getReRunFromWorkflowId(), request.getReRunFromTaskId(), @@ -1163,33 +1160,15 @@ public void updateTask(TaskResult taskResult) { case IN_PROGRESS: case SCHEDULED: try { - String postponeTaskMessageDesc = - "Postponing Task message in queue for taskId: " + task.getTaskId(); - String postponeTaskMessageOperation = "postponeTaskMessage"; - - new RetryUtil<>() - .retryOnException( - () -> { - // postpone based on callbackAfterSeconds - long callBack = taskResult.getCallbackAfterSeconds(); - queueDAO.postpone( - taskQueueName, - task.getTaskId(), - task.getWorkflowPriority(), - callBack); - LOGGER.debug( - "Task: {} postponed in taskQueue: {} since the task status is {} with callbackAfterSeconds: {}", - task, - taskQueueName, - task.getStatus().name(), - callBack); - return null; - }, - null, - null, - 2, - postponeTaskMessageDesc, - postponeTaskMessageOperation); + long callBack = taskResult.getCallbackAfterSeconds(); + queueDAO.postpone( + taskQueueName, task.getTaskId(), task.getWorkflowPriority(), callBack); + LOGGER.debug( + "Task: {} postponed in taskQueue: {} since the task status is {} with callbackAfterSeconds: {}", + task, + taskQueueName, + task.getStatus().name(), + callBack); } catch (Exception e) { // Throw exceptions on queue postpone, this would impact task execution String errorMsg = @@ -1208,20 +1187,7 @@ public void updateTask(TaskResult taskResult) { // Throw an ApplicationException if below operations fail to avoid workflow inconsistencies. try { - String updateTaskDesc = "Updating Task with taskId: " + task.getTaskId(); - String updateTaskOperation = "updateTask"; - - new RetryUtil<>() - .retryOnException( - () -> { - executionDAOFacade.updateTask(task); - return null; - }, - null, - null, - 2, - updateTaskDesc, - updateTaskOperation); + executionDAOFacade.updateTask(task); } catch (Exception e) { String errorMsg = String.format( diff --git a/core/src/main/java/com/netflix/conductor/core/execution/mapper/DecisionTaskMapper.java b/core/src/main/java/com/netflix/conductor/core/execution/mapper/DecisionTaskMapper.java index b2fd2fed1a..45b2add899 100644 --- a/core/src/main/java/com/netflix/conductor/core/execution/mapper/DecisionTaskMapper.java +++ b/core/src/main/java/com/netflix/conductor/core/execution/mapper/DecisionTaskMapper.java @@ -24,6 +24,7 @@ import org.slf4j.LoggerFactory; import org.springframework.stereotype.Component; +import com.netflix.conductor.annotations.VisibleForTesting; import com.netflix.conductor.common.metadata.tasks.TaskType; import com.netflix.conductor.common.metadata.workflow.WorkflowDef; import com.netflix.conductor.common.metadata.workflow.WorkflowTask; @@ -32,8 +33,6 @@ import com.netflix.conductor.model.TaskModel; import com.netflix.conductor.model.WorkflowModel; -import com.google.common.annotations.VisibleForTesting; - /** * An implementation of {@link TaskMapper} to map a {@link WorkflowTask} of type {@link * TaskType#DECISION} to a List {@link TaskModel} starting with Task of type {@link diff --git a/core/src/main/java/com/netflix/conductor/core/execution/mapper/DynamicTaskMapper.java b/core/src/main/java/com/netflix/conductor/core/execution/mapper/DynamicTaskMapper.java index 5d92add458..506fbbdc65 100644 --- a/core/src/main/java/com/netflix/conductor/core/execution/mapper/DynamicTaskMapper.java +++ b/core/src/main/java/com/netflix/conductor/core/execution/mapper/DynamicTaskMapper.java @@ -22,6 +22,7 @@ import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; +import com.netflix.conductor.annotations.VisibleForTesting; import com.netflix.conductor.common.metadata.tasks.TaskDef; import com.netflix.conductor.common.metadata.tasks.TaskType; import com.netflix.conductor.common.metadata.workflow.WorkflowDef; @@ -32,8 +33,6 @@ import com.netflix.conductor.model.TaskModel; import com.netflix.conductor.model.WorkflowModel; -import com.google.common.annotations.VisibleForTesting; - /** * An implementation of {@link TaskMapper} to map a {@link WorkflowTask} of type {@link * TaskType#DYNAMIC} to a {@link TaskModel} based on definition derived from the dynamic task name diff --git a/core/src/main/java/com/netflix/conductor/core/execution/mapper/ForkJoinDynamicTaskMapper.java b/core/src/main/java/com/netflix/conductor/core/execution/mapper/ForkJoinDynamicTaskMapper.java index 4ce63851c8..0a013b3b91 100644 --- a/core/src/main/java/com/netflix/conductor/core/execution/mapper/ForkJoinDynamicTaskMapper.java +++ b/core/src/main/java/com/netflix/conductor/core/execution/mapper/ForkJoinDynamicTaskMapper.java @@ -28,6 +28,7 @@ import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; +import com.netflix.conductor.annotations.VisibleForTesting; import com.netflix.conductor.common.metadata.tasks.TaskType; import com.netflix.conductor.common.metadata.workflow.DynamicForkJoinTaskList; import com.netflix.conductor.common.metadata.workflow.WorkflowDef; @@ -41,7 +42,6 @@ import com.fasterxml.jackson.core.type.TypeReference; import com.fasterxml.jackson.databind.ObjectMapper; -import com.google.common.annotations.VisibleForTesting; /** * An implementation of {@link TaskMapper} to map a {@link WorkflowTask} of type {@link @@ -231,9 +231,11 @@ public List getMappedTasks(TaskMapperContext taskMapperContext) * This method creates a FORK task and adds the list of dynamic fork tasks keyed by * "forkedTaskDefs" and their names keyed by "forkedTasks" into {@link TaskModel#getInputData()} * - * @param taskMapperContext - * @param dynForkTasks - * @return + * @param taskMapperContext: The {@link TaskMapperContext} which wraps workflowTask, workflowDef + * and workflowModel + * @param dynForkTasks: The list of dynamic forked tasks, the reference names of these tasks + * will be added to the forkDynamicTask + * @return A new instance of {@link TaskModel} representing a {@link TaskType#TASK_TYPE_FORK} */ @VisibleForTesting TaskModel createDynamicForkTask( diff --git a/core/src/main/java/com/netflix/conductor/core/execution/mapper/SubWorkflowTaskMapper.java b/core/src/main/java/com/netflix/conductor/core/execution/mapper/SubWorkflowTaskMapper.java index 64a77df1db..167cd3aef5 100644 --- a/core/src/main/java/com/netflix/conductor/core/execution/mapper/SubWorkflowTaskMapper.java +++ b/core/src/main/java/com/netflix/conductor/core/execution/mapper/SubWorkflowTaskMapper.java @@ -18,6 +18,7 @@ import org.slf4j.LoggerFactory; import org.springframework.stereotype.Component; +import com.netflix.conductor.annotations.VisibleForTesting; import com.netflix.conductor.common.metadata.tasks.TaskType; import com.netflix.conductor.common.metadata.workflow.SubWorkflowParams; import com.netflix.conductor.common.metadata.workflow.WorkflowDef; @@ -28,8 +29,6 @@ import com.netflix.conductor.model.TaskModel; import com.netflix.conductor.model.WorkflowModel; -import com.google.common.annotations.VisibleForTesting; - import static com.netflix.conductor.common.metadata.tasks.TaskType.TASK_TYPE_SUB_WORKFLOW; @Component diff --git a/core/src/main/java/com/netflix/conductor/core/execution/tasks/DoWhile.java b/core/src/main/java/com/netflix/conductor/core/execution/tasks/DoWhile.java index 58366fdcf7..2d3b5dee85 100644 --- a/core/src/main/java/com/netflix/conductor/core/execution/tasks/DoWhile.java +++ b/core/src/main/java/com/netflix/conductor/core/execution/tasks/DoWhile.java @@ -21,6 +21,7 @@ import org.slf4j.LoggerFactory; import org.springframework.stereotype.Component; +import com.netflix.conductor.annotations.VisibleForTesting; import com.netflix.conductor.common.metadata.tasks.TaskDef; import com.netflix.conductor.common.utils.TaskUtils; import com.netflix.conductor.core.events.ScriptEvaluator; @@ -30,8 +31,6 @@ import com.netflix.conductor.model.TaskModel; import com.netflix.conductor.model.WorkflowModel; -import com.google.common.annotations.VisibleForTesting; - import static com.netflix.conductor.common.metadata.tasks.TaskType.TASK_TYPE_DO_WHILE; @Component(TASK_TYPE_DO_WHILE) diff --git a/core/src/main/java/com/netflix/conductor/core/execution/tasks/Event.java b/core/src/main/java/com/netflix/conductor/core/execution/tasks/Event.java index 0dec908c8f..5e9a9aef4c 100644 --- a/core/src/main/java/com/netflix/conductor/core/execution/tasks/Event.java +++ b/core/src/main/java/com/netflix/conductor/core/execution/tasks/Event.java @@ -20,6 +20,7 @@ import org.slf4j.LoggerFactory; import org.springframework.stereotype.Component; +import com.netflix.conductor.annotations.VisibleForTesting; import com.netflix.conductor.core.events.EventQueues; import com.netflix.conductor.core.events.queue.Message; import com.netflix.conductor.core.events.queue.ObservableQueue; @@ -31,7 +32,6 @@ import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.ObjectMapper; -import com.google.common.annotations.VisibleForTesting; import static com.netflix.conductor.common.metadata.tasks.TaskType.TASK_TYPE_EVENT; diff --git a/core/src/main/java/com/netflix/conductor/core/execution/tasks/ExecutionConfig.java b/core/src/main/java/com/netflix/conductor/core/execution/tasks/ExecutionConfig.java index 890a33ebf5..7115dfd1dc 100644 --- a/core/src/main/java/com/netflix/conductor/core/execution/tasks/ExecutionConfig.java +++ b/core/src/main/java/com/netflix/conductor/core/execution/tasks/ExecutionConfig.java @@ -15,9 +15,9 @@ import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; -import com.netflix.conductor.core.utils.SemaphoreUtil; +import org.apache.commons.lang3.concurrent.BasicThreadFactory; -import com.google.common.util.concurrent.ThreadFactoryBuilder; +import com.netflix.conductor.core.utils.SemaphoreUtil; class ExecutionConfig { @@ -29,7 +29,7 @@ class ExecutionConfig { this.executorService = Executors.newFixedThreadPool( threadCount, - new ThreadFactoryBuilder().setNameFormat(threadNameFormat).build()); + new BasicThreadFactory.Builder().namingPattern(threadNameFormat).build()); this.semaphoreUtil = new SemaphoreUtil(threadCount); } diff --git a/core/src/main/java/com/netflix/conductor/core/execution/tasks/IsolatedTaskQueueProducer.java b/core/src/main/java/com/netflix/conductor/core/execution/tasks/IsolatedTaskQueueProducer.java index e357a80e2a..5a5ac33a87 100644 --- a/core/src/main/java/com/netflix/conductor/core/execution/tasks/IsolatedTaskQueueProducer.java +++ b/core/src/main/java/com/netflix/conductor/core/execution/tasks/IsolatedTaskQueueProducer.java @@ -29,12 +29,11 @@ import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; import org.springframework.stereotype.Component; +import com.netflix.conductor.annotations.VisibleForTesting; import com.netflix.conductor.common.metadata.tasks.TaskDef; import com.netflix.conductor.core.utils.QueueUtils; import com.netflix.conductor.service.MetadataService; -import com.google.common.annotations.VisibleForTesting; - import static com.netflix.conductor.core.execution.tasks.SystemTaskRegistry.ASYNC_SYSTEM_TASKS_QUALIFIER; @Component diff --git a/core/src/main/java/com/netflix/conductor/core/execution/tasks/SystemTaskWorker.java b/core/src/main/java/com/netflix/conductor/core/execution/tasks/SystemTaskWorker.java index 04ac69d555..29bf6bb2a6 100644 --- a/core/src/main/java/com/netflix/conductor/core/execution/tasks/SystemTaskWorker.java +++ b/core/src/main/java/com/netflix/conductor/core/execution/tasks/SystemTaskWorker.java @@ -25,6 +25,7 @@ import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; import org.springframework.stereotype.Component; +import com.netflix.conductor.annotations.VisibleForTesting; import com.netflix.conductor.core.LifecycleAwareComponent; import com.netflix.conductor.core.config.ConductorProperties; import com.netflix.conductor.core.execution.AsyncSystemTaskExecutor; @@ -34,8 +35,6 @@ import com.netflix.conductor.metrics.Monitors; import com.netflix.conductor.service.ExecutionService; -import com.google.common.annotations.VisibleForTesting; - /** The worker that polls and executes an async system task. */ @Component @ConditionalOnProperty( diff --git a/core/src/main/java/com/netflix/conductor/core/execution/tasks/SystemTaskWorkerCoordinator.java b/core/src/main/java/com/netflix/conductor/core/execution/tasks/SystemTaskWorkerCoordinator.java index 6992727a14..b1a9ed9887 100644 --- a/core/src/main/java/com/netflix/conductor/core/execution/tasks/SystemTaskWorkerCoordinator.java +++ b/core/src/main/java/com/netflix/conductor/core/execution/tasks/SystemTaskWorkerCoordinator.java @@ -23,11 +23,10 @@ import org.springframework.context.event.EventListener; import org.springframework.stereotype.Component; +import com.netflix.conductor.annotations.VisibleForTesting; import com.netflix.conductor.core.config.ConductorProperties; import com.netflix.conductor.core.utils.QueueUtils; -import com.google.common.annotations.VisibleForTesting; - import static com.netflix.conductor.core.execution.tasks.SystemTaskRegistry.ASYNC_SYSTEM_TASKS_QUALIFIER; @Component diff --git a/core/src/main/java/com/netflix/conductor/core/metadata/MetadataMapperService.java b/core/src/main/java/com/netflix/conductor/core/metadata/MetadataMapperService.java index bd761c3b25..b5d3fd4006 100644 --- a/core/src/main/java/com/netflix/conductor/core/metadata/MetadataMapperService.java +++ b/core/src/main/java/com/netflix/conductor/core/metadata/MetadataMapperService.java @@ -21,6 +21,7 @@ import org.slf4j.LoggerFactory; import org.springframework.stereotype.Component; +import com.netflix.conductor.annotations.VisibleForTesting; import com.netflix.conductor.common.metadata.tasks.TaskType; import com.netflix.conductor.common.metadata.workflow.SubWorkflowParams; import com.netflix.conductor.common.metadata.workflow.WorkflowDef; @@ -28,14 +29,12 @@ import com.netflix.conductor.core.WorkflowContext; import com.netflix.conductor.core.exception.ApplicationException; import com.netflix.conductor.core.exception.TerminateWorkflowException; +import com.netflix.conductor.core.utils.Utils; import com.netflix.conductor.dao.MetadataDAO; import com.netflix.conductor.metrics.Monitors; import com.netflix.conductor.model.TaskModel; import com.netflix.conductor.model.WorkflowModel; -import com.google.common.annotations.VisibleForTesting; -import com.google.common.base.Preconditions; - /** * Populates metadata definitions within workflow objects. Benefits of loading and populating * metadata definitions upfront could be: @@ -79,7 +78,7 @@ public WorkflowDef lookupForWorkflowDefinition(String name, Integer version) { @VisibleForTesting Optional lookupWorkflowDefinition(String workflowName, int workflowVersion) { - Preconditions.checkArgument( + Utils.checkArgument( StringUtils.isNotBlank(workflowName), "Workflow name must be specified when searching for a definition"); return metadataDAO.getWorkflowDef(workflowName, workflowVersion); @@ -87,14 +86,14 @@ Optional lookupWorkflowDefinition(String workflowName, int workflow @VisibleForTesting Optional lookupLatestWorkflowDefinition(String workflowName) { - Preconditions.checkArgument( + Utils.checkArgument( StringUtils.isNotBlank(workflowName), "Workflow name must be specified when searching for a definition"); return metadataDAO.getLatestWorkflowDef(workflowName); } public WorkflowModel populateWorkflowWithDefinitions(WorkflowModel workflow) { - Preconditions.checkNotNull(workflow, "workflow cannot be null"); + Utils.checkNotNull(workflow, "workflow cannot be null"); WorkflowDef workflowDefinition = Optional.ofNullable(workflow.getWorkflowDefinition()) .orElseGet( @@ -114,14 +113,14 @@ public WorkflowModel populateWorkflowWithDefinitions(WorkflowModel workflow) { } public WorkflowDef populateTaskDefinitions(WorkflowDef workflowDefinition) { - Preconditions.checkNotNull(workflowDefinition, "workflowDefinition cannot be null"); + Utils.checkNotNull(workflowDefinition, "workflowDefinition cannot be null"); workflowDefinition.collectTasks().forEach(this::populateWorkflowTaskWithDefinition); checkNotEmptyDefinitions(workflowDefinition); return workflowDefinition; } private void populateWorkflowTaskWithDefinition(WorkflowTask workflowTask) { - Preconditions.checkNotNull(workflowTask, "WorkflowTask cannot be null"); + Utils.checkNotNull(workflowTask, "WorkflowTask cannot be null"); if (shouldPopulateTaskDefinition(workflowTask)) { workflowTask.setTaskDefinition(metadataDAO.getTaskDef(workflowTask.getName())); } @@ -131,7 +130,7 @@ private void populateWorkflowTaskWithDefinition(WorkflowTask workflowTask) { } private void populateVersionForSubWorkflow(WorkflowTask workflowTask) { - Preconditions.checkNotNull(workflowTask, "WorkflowTask cannot be null"); + Utils.checkNotNull(workflowTask, "WorkflowTask cannot be null"); SubWorkflowParams subworkflowParams = workflowTask.getSubWorkflowParam(); if (subworkflowParams.getVersion() == null) { String subWorkflowName = subworkflowParams.getName(); @@ -153,7 +152,7 @@ private void populateVersionForSubWorkflow(WorkflowTask workflowTask) { } private void checkNotEmptyDefinitions(WorkflowDef workflowDefinition) { - Preconditions.checkNotNull(workflowDefinition, "WorkflowDefinition cannot be null"); + Utils.checkNotNull(workflowDefinition, "WorkflowDefinition cannot be null"); // Obtain the names of the tasks with missing definitions Set missingTaskDefinitionNames = @@ -179,15 +178,15 @@ private void checkNotEmptyDefinitions(WorkflowDef workflowDefinition) { } public TaskModel populateTaskWithDefinition(TaskModel task) { - Preconditions.checkNotNull(task, "Task cannot be null"); + Utils.checkNotNull(task, "Task cannot be null"); populateWorkflowTaskWithDefinition(task.getWorkflowTask()); return task; } @VisibleForTesting boolean shouldPopulateTaskDefinition(WorkflowTask workflowTask) { - Preconditions.checkNotNull(workflowTask, "WorkflowTask cannot be null"); - Preconditions.checkNotNull(workflowTask.getType(), "WorkflowTask type cannot be null"); + Utils.checkNotNull(workflowTask, "WorkflowTask cannot be null"); + Utils.checkNotNull(workflowTask.getType(), "WorkflowTask type cannot be null"); return workflowTask.getTaskDefinition() == null && StringUtils.isNotBlank(workflowTask.getName()); } diff --git a/core/src/main/java/com/netflix/conductor/core/reconciliation/WorkflowRepairService.java b/core/src/main/java/com/netflix/conductor/core/reconciliation/WorkflowRepairService.java index 5ac05e24f0..769fe8ddb6 100644 --- a/core/src/main/java/com/netflix/conductor/core/reconciliation/WorkflowRepairService.java +++ b/core/src/main/java/com/netflix/conductor/core/reconciliation/WorkflowRepairService.java @@ -21,6 +21,7 @@ import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; import org.springframework.stereotype.Service; +import com.netflix.conductor.annotations.VisibleForTesting; import com.netflix.conductor.core.config.ConductorProperties; import com.netflix.conductor.core.execution.tasks.SystemTaskRegistry; import com.netflix.conductor.core.execution.tasks.WorkflowSystemTask; @@ -32,8 +33,6 @@ import com.netflix.conductor.model.TaskModel; import com.netflix.conductor.model.WorkflowModel; -import com.google.common.annotations.VisibleForTesting; - /** * A helper service that tries to keep ExecutionDAO and QueueDAO in sync, based on the task or * workflow state. diff --git a/core/src/main/java/com/netflix/conductor/core/reconciliation/WorkflowSweeper.java b/core/src/main/java/com/netflix/conductor/core/reconciliation/WorkflowSweeper.java index e21894da1b..2d3c1eaa59 100644 --- a/core/src/main/java/com/netflix/conductor/core/reconciliation/WorkflowSweeper.java +++ b/core/src/main/java/com/netflix/conductor/core/reconciliation/WorkflowSweeper.java @@ -76,11 +76,7 @@ public void sweep(String workflowId) { boolean done = workflowExecutor.decide(workflowId); if (done) { queueDAO.remove(DECIDER_QUEUE, workflowId); - } else { - queueDAO.setUnackTimeout( - DECIDER_QUEUE, - workflowId, - properties.getWorkflowOffsetTimeout().toMillis()); + return; } } catch (ApplicationException e) { if (e.getCode() == ApplicationException.Code.NOT_FOUND) { @@ -89,12 +85,13 @@ public void sweep(String workflowId) { "Workflow NOT found for id:{}. Removed it from decider queue", workflowId, e); + return; } } catch (Exception e) { - queueDAO.setUnackTimeout( - DECIDER_QUEUE, workflowId, properties.getWorkflowOffsetTimeout().toMillis()); Monitors.error(CLASS_NAME, "sweep"); LOGGER.error("Error running sweep for " + workflowId, e); } + queueDAO.setUnackTimeout( + DECIDER_QUEUE, workflowId, properties.getWorkflowOffsetTimeout().toMillis()); } } diff --git a/core/src/main/java/com/netflix/conductor/core/utils/ExternalPayloadStorageUtils.java b/core/src/main/java/com/netflix/conductor/core/utils/ExternalPayloadStorageUtils.java index 1c1227912d..eb6e862659 100644 --- a/core/src/main/java/com/netflix/conductor/core/utils/ExternalPayloadStorageUtils.java +++ b/core/src/main/java/com/netflix/conductor/core/utils/ExternalPayloadStorageUtils.java @@ -25,6 +25,7 @@ import org.slf4j.LoggerFactory; import org.springframework.stereotype.Component; +import com.netflix.conductor.annotations.VisibleForTesting; import com.netflix.conductor.common.run.ExternalStorageLocation; import com.netflix.conductor.common.utils.ExternalPayloadStorage; import com.netflix.conductor.common.utils.ExternalPayloadStorage.PayloadType; @@ -36,7 +37,6 @@ import com.netflix.conductor.model.WorkflowModel; import com.fasterxml.jackson.databind.ObjectMapper; -import com.google.common.annotations.VisibleForTesting; /** Provides utility functions to upload and download payloads to {@link ExternalPayloadStorage} */ @Component diff --git a/core/src/main/java/com/netflix/conductor/core/utils/ParametersUtils.java b/core/src/main/java/com/netflix/conductor/core/utils/ParametersUtils.java index b7397807df..40749a1796 100644 --- a/core/src/main/java/com/netflix/conductor/core/utils/ParametersUtils.java +++ b/core/src/main/java/com/netflix/conductor/core/utils/ParametersUtils.java @@ -34,7 +34,6 @@ import com.fasterxml.jackson.core.type.TypeReference; import com.fasterxml.jackson.databind.ObjectMapper; -import com.google.common.base.Preconditions; import com.jayway.jsonpath.Configuration; import com.jayway.jsonpath.DocumentContext; import com.jayway.jsonpath.JsonPath; @@ -286,7 +285,7 @@ private Map getTaskInputV1( (paramName, value) -> { String paramPath = "" + value; String[] paramPathComponents = paramPath.split("\\."); - Preconditions.checkArgument( + Utils.checkArgument( paramPathComponents.length == 3, "Invalid input expression for " + paramName diff --git a/core/src/main/java/com/netflix/conductor/core/utils/Utils.java b/core/src/main/java/com/netflix/conductor/core/utils/Utils.java index 013f7dc950..537d1137a4 100644 --- a/core/src/main/java/com/netflix/conductor/core/utils/Utils.java +++ b/core/src/main/java/com/netflix/conductor/core/utils/Utils.java @@ -14,18 +14,12 @@ import java.net.InetAddress; import java.net.UnknownHostException; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collection; -import java.util.List; -import java.util.Map; +import java.util.*; import org.apache.commons.lang3.StringUtils; import com.netflix.conductor.core.exception.ApplicationException; -import com.google.common.base.Preconditions; - public class Utils { public static final String DECIDER_QUEUE = "_deciderQueue"; @@ -66,7 +60,7 @@ public static List convertStringToList(String inputStr) { */ public static void checkArgument(boolean condition, String errorMessage) { if (!condition) { - throw new ApplicationException(ApplicationException.Code.INVALID_INPUT, errorMessage); + throw new IllegalArgumentException(errorMessage); } } @@ -104,9 +98,7 @@ public static void checkNotNullOrEmpty(Map map, String errorMessage) { * @throws ApplicationException if input string is not valid */ public static void checkNotNullOrEmpty(String input, String errorMessage) { - try { - Preconditions.checkArgument(StringUtils.isNotBlank(input), errorMessage); - } catch (IllegalArgumentException exception) { + if (StringUtils.isEmpty(input)) { throw new ApplicationException(ApplicationException.Code.INVALID_INPUT, errorMessage); } } @@ -119,10 +111,26 @@ public static void checkNotNullOrEmpty(String input, String errorMessage) { * @throws ApplicationException if input object is not valid */ public static void checkNotNull(Object object, String errorMessage) { - try { - Preconditions.checkNotNull(object, errorMessage); - } catch (NullPointerException exception) { - throw new ApplicationException(ApplicationException.Code.INVALID_INPUT, errorMessage); + if (object == null) { + throw new NullPointerException(errorMessage); + } + } + + /** + * Used to determine if the exception is thrown due to a transient failure and the operation is + * expected to succeed upon retrying. + * + * @param throwable the exception that is thrown + * @return true - if the exception is a transient failure + *

false - if the exception is non-transient + */ + public static boolean isTransientException(Throwable throwable) { + if (throwable != null) { + return !((throwable instanceof UnsupportedOperationException) + || (throwable instanceof ApplicationException + && ((ApplicationException) throwable).getCode() + != ApplicationException.Code.BACKEND_ERROR)); } + return true; } } diff --git a/core/src/main/java/com/netflix/conductor/model/WorkflowModel.java b/core/src/main/java/com/netflix/conductor/model/WorkflowModel.java index 5968725970..523d8be7f0 100644 --- a/core/src/main/java/com/netflix/conductor/model/WorkflowModel.java +++ b/core/src/main/java/com/netflix/conductor/model/WorkflowModel.java @@ -20,10 +20,10 @@ import com.netflix.conductor.common.metadata.workflow.WorkflowDef; import com.netflix.conductor.common.run.Workflow; +import com.netflix.conductor.core.utils.Utils; import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.annotation.JsonProperty; -import com.google.common.base.Preconditions; public class WorkflowModel { @@ -378,7 +378,7 @@ public void setFailedTaskId(String failedTaskId) { * @return the workflow definition name. */ public String getWorkflowName() { - Preconditions.checkNotNull(workflowDefinition, "Workflow definition is null"); + Utils.checkNotNull(workflowDefinition, "Workflow definition is null"); return workflowDefinition.getName(); } @@ -388,7 +388,7 @@ public String getWorkflowName() { * @return the workflow definition version. */ public int getWorkflowVersion() { - Preconditions.checkNotNull(workflowDefinition, "Workflow definition is null"); + Utils.checkNotNull(workflowDefinition, "Workflow definition is null"); return workflowDefinition.getVersion(); } diff --git a/core/src/main/java/com/netflix/conductor/service/TaskServiceImpl.java b/core/src/main/java/com/netflix/conductor/service/TaskServiceImpl.java index 9865dccd3e..fdcb503410 100644 --- a/core/src/main/java/com/netflix/conductor/service/TaskServiceImpl.java +++ b/core/src/main/java/com/netflix/conductor/service/TaskServiceImpl.java @@ -34,7 +34,6 @@ import com.netflix.conductor.common.run.SearchResult; import com.netflix.conductor.common.run.TaskSummary; import com.netflix.conductor.common.utils.ExternalPayloadStorage; -import com.netflix.conductor.common.utils.RetryUtil; import com.netflix.conductor.dao.QueueDAO; import com.netflix.conductor.metrics.Monitors; @@ -160,22 +159,9 @@ public String ackTaskReceived(String taskId, String workerId) { */ public boolean ackTaskReceived(String taskId) { LOGGER.debug("Ack received for task: {}", taskId); - String ackTaskDesc = "Ack Task with taskId: " + taskId; - String ackTaskOperation = "ackTaskReceived"; AtomicBoolean ackResult = new AtomicBoolean(false); try { - new RetryUtil<>() - .retryOnException( - () -> { - ackResult.set(executionService.ackTaskReceived(taskId)); - return null; - }, - null, - null, - 3, - ackTaskDesc, - ackTaskOperation); - + ackResult.set(executionService.ackTaskReceived(taskId)); } catch (Exception e) { // Fail the task and let decide reevaluate the workflow, thereby preventing workflow // being stuck from transient ack errors. diff --git a/core/src/test/java/com/netflix/conductor/core/events/TestDefaultEventProcessor.java b/core/src/test/java/com/netflix/conductor/core/events/TestDefaultEventProcessor.java index 2c99279bc6..8558904023 100644 --- a/core/src/test/java/com/netflix/conductor/core/events/TestDefaultEventProcessor.java +++ b/core/src/test/java/com/netflix/conductor/core/events/TestDefaultEventProcessor.java @@ -12,11 +12,7 @@ */ package com.netflix.conductor.core.events; -import java.util.Collections; -import java.util.HashMap; -import java.util.LinkedHashMap; -import java.util.Map; -import java.util.UUID; +import java.util.*; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicInteger; @@ -25,8 +21,10 @@ import org.junit.runner.RunWith; import org.mockito.stubbing.Answer; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.context.annotation.ComponentScan; import org.springframework.context.annotation.Configuration; +import org.springframework.retry.support.RetryTemplate; import org.springframework.test.context.ContextConfiguration; import org.springframework.test.context.junit4.SpringRunner; @@ -37,6 +35,7 @@ import com.netflix.conductor.common.metadata.events.EventHandler.Action.Type; import com.netflix.conductor.common.metadata.events.EventHandler.StartWorkflow; import com.netflix.conductor.common.metadata.events.EventHandler.TaskDetails; +import com.netflix.conductor.core.config.ConductorCoreConfiguration; import com.netflix.conductor.core.config.ConductorProperties; import com.netflix.conductor.core.events.queue.Message; import com.netflix.conductor.core.events.queue.ObservableQueue; @@ -54,19 +53,15 @@ import com.fasterxml.jackson.databind.ObjectMapper; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.assertNull; -import static org.junit.Assert.assertTrue; -import static org.mockito.ArgumentMatchers.any; -import static org.mockito.ArgumentMatchers.anyMap; -import static org.mockito.ArgumentMatchers.eq; +import static org.junit.Assert.*; +import static org.mockito.ArgumentMatchers.*; import static org.mockito.Mockito.*; @ContextConfiguration( classes = { TestObjectMapperConfiguration.class, - TestDefaultEventProcessor.TestConfiguration.class + TestDefaultEventProcessor.TestConfiguration.class, + ConductorCoreConfiguration.class }) @RunWith(SpringRunner.class) public class TestDefaultEventProcessor { @@ -87,6 +82,9 @@ public class TestDefaultEventProcessor { @Autowired private ObjectMapper objectMapper; + @Autowired + private @Qualifier("onTransientErrorRetryTemplate") RetryTemplate retryTemplate; + @Configuration @ComponentScan(basePackageClasses = {Evaluator.class}) // load all Evaluator beans public static class TestConfiguration {} @@ -201,7 +199,8 @@ public void testEventProcessor() { jsonUtils, properties, objectMapper, - evaluators); + evaluators, + retryTemplate); eventProcessor.handle(queue, message); assertTrue(started.get()); assertTrue(completed.get()); @@ -267,7 +266,8 @@ public void testEventHandlerWithCondition() { jsonUtils, properties, objectMapper, - evaluators); + evaluators, + retryTemplate); eventProcessor.handle(queue, message); assertTrue(started.get()); } @@ -331,7 +331,8 @@ public void testEventHandlerWithConditionEvaluator() { jsonUtils, properties, objectMapper, - evaluators); + evaluators, + retryTemplate); eventProcessor.handle(queue, message); assertTrue(started.get()); } @@ -368,7 +369,8 @@ public void testEventProcessorWithRetriableError() { jsonUtils, properties, objectMapper, - evaluators); + evaluators, + retryTemplate); eventProcessor.handle(queue, message); verify(queue, never()).ack(any()); verify(queue, never()).publish(any()); @@ -407,7 +409,8 @@ public void testEventProcessorWithNonRetriableError() { jsonUtils, properties, objectMapper, - evaluators); + evaluators, + retryTemplate); eventProcessor.handle(queue, message); verify(queue, atMost(1)).ack(any()); verify(queue, never()).publish(any()); @@ -433,7 +436,8 @@ public void testExecuteInvalidAction() { jsonUtils, properties, objectMapper, - evaluators); + evaluators, + retryTemplate); EventExecution eventExecution = new EventExecution("id", "messageId"); eventExecution.setName("handler"); eventExecution.setStatus(EventExecution.Status.IN_PROGRESS); @@ -469,7 +473,8 @@ public void testExecuteNonRetriableApplicationException() { jsonUtils, properties, objectMapper, - evaluators); + evaluators, + retryTemplate); EventExecution eventExecution = new EventExecution("id", "messageId"); eventExecution.setStatus(EventExecution.Status.IN_PROGRESS); eventExecution.setEvent("event"); @@ -506,7 +511,8 @@ public void testExecuteRetriableApplicationException() { jsonUtils, properties, objectMapper, - evaluators); + evaluators, + retryTemplate); EventExecution eventExecution = new EventExecution("id", "messageId"); eventExecution.setStatus(EventExecution.Status.IN_PROGRESS); eventExecution.setEvent("event"); diff --git a/core/src/test/java/com/netflix/conductor/core/execution/TestWorkflowExecutor.java b/core/src/test/java/com/netflix/conductor/core/execution/TestWorkflowExecutor.java index 178b54c503..0707811a99 100644 --- a/core/src/test/java/com/netflix/conductor/core/execution/TestWorkflowExecutor.java +++ b/core/src/test/java/com/netflix/conductor/core/execution/TestWorkflowExecutor.java @@ -14,7 +14,6 @@ import java.time.Duration; import java.util.*; -import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicInteger; import java.util.stream.Collectors; @@ -58,7 +57,6 @@ import com.netflix.conductor.service.ExecutionLockService; import com.fasterxml.jackson.databind.ObjectMapper; -import com.google.common.util.concurrent.Uninterruptibles; import static com.netflix.conductor.common.metadata.tasks.TaskType.*; import static com.netflix.conductor.core.exception.ApplicationException.Code.CONFLICT; @@ -1604,7 +1602,7 @@ public void testRerunSubWorkflowWithTaskId() { } @Test - public void testGetActiveDomain() { + public void testGetActiveDomain() throws Exception { String taskType = "test-task"; String[] domains = new String[] {"domain1", "domain2"}; @@ -1615,8 +1613,7 @@ public void testGetActiveDomain() { .thenReturn(pollData1); String activeDomain = workflowExecutor.getActiveDomain(taskType, domains); assertEquals(domains[0], activeDomain); - - Uninterruptibles.sleepUninterruptibly(2, TimeUnit.SECONDS); + Thread.sleep(2000L); PollData pollData2 = new PollData( @@ -1626,7 +1623,7 @@ public void testGetActiveDomain() { activeDomain = workflowExecutor.getActiveDomain(taskType, domains); assertEquals(domains[1], activeDomain); - Uninterruptibles.sleepUninterruptibly(2, TimeUnit.SECONDS); + Thread.sleep(2000L); activeDomain = workflowExecutor.getActiveDomain(taskType, domains); assertEquals(domains[1], activeDomain); diff --git a/core/src/test/java/com/netflix/conductor/core/execution/mapper/ForkJoinDynamicTaskMapperTest.java b/core/src/test/java/com/netflix/conductor/core/execution/mapper/ForkJoinDynamicTaskMapperTest.java index 2475443aa2..7c7f747964 100644 --- a/core/src/test/java/com/netflix/conductor/core/execution/mapper/ForkJoinDynamicTaskMapperTest.java +++ b/core/src/test/java/com/netflix/conductor/core/execution/mapper/ForkJoinDynamicTaskMapperTest.java @@ -12,11 +12,7 @@ */ package com.netflix.conductor.core.execution.mapper; -import java.util.Arrays; -import java.util.Collections; -import java.util.HashMap; -import java.util.List; -import java.util.Map; +import java.util.*; import org.apache.commons.lang3.tuple.Pair; import org.junit.Before; @@ -40,7 +36,6 @@ import com.fasterxml.jackson.core.type.TypeReference; import com.fasterxml.jackson.databind.ObjectMapper; -import com.google.common.collect.Lists; import static com.netflix.conductor.common.metadata.tasks.TaskType.TASK_TYPE_FORK; import static com.netflix.conductor.common.metadata.tasks.TaskType.TASK_TYPE_JOIN; @@ -487,7 +482,7 @@ public void testDynamicTaskDuplicateTaskRefName() { // Empty list, this is a bad state, workflow should terminate when(deciderService.getTasksToBeScheduled(workflowModel, wt2, 0)) - .thenReturn(Lists.newArrayList()); + .thenReturn(new ArrayList<>()); String taskId = idGenerator.generate(); TaskMapperContext taskMapperContext = diff --git a/core/src/test/java/com/netflix/conductor/core/metadata/MetadataMapperServiceTest.java b/core/src/test/java/com/netflix/conductor/core/metadata/MetadataMapperServiceTest.java index dbc74e8c21..3d5e0627e4 100644 --- a/core/src/test/java/com/netflix/conductor/core/metadata/MetadataMapperServiceTest.java +++ b/core/src/test/java/com/netflix/conductor/core/metadata/MetadataMapperServiceTest.java @@ -1,5 +1,5 @@ /* - * Copyright 2020 Netflix, Inc. + * Copyright 2022 Netflix, Inc. *

* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at @@ -37,8 +37,6 @@ import com.netflix.conductor.core.exception.TerminateWorkflowException; import com.netflix.conductor.dao.MetadataDAO; -import com.google.common.collect.ImmutableList; - import static com.netflix.conductor.TestUtils.getConstraintViolationMessages; import static org.junit.Assert.assertEquals; @@ -89,7 +87,7 @@ public void testMetadataPopulationOnSimpleTask() { when(metadataDAO.getTaskDef(nameTaskDefinition)).thenReturn(taskDefinition); WorkflowDef workflowDefinition = createWorkflowDefinition("testMetadataPopulation"); - workflowDefinition.setTasks(ImmutableList.of(workflowTask)); + workflowDefinition.setTasks(List.of(workflowTask)); metadataMapperService.populateTaskDefinitions(workflowDefinition); @@ -107,7 +105,7 @@ public void testNoMetadataPopulationOnEmbeddedTaskDefinition() { workflowTask.setTaskDefinition(taskDefinition); WorkflowDef workflowDefinition = createWorkflowDefinition("testMetadataPopulation"); - workflowDefinition.setTasks(ImmutableList.of(workflowTask)); + workflowDefinition.setTasks(List.of(workflowTask)); metadataMapperService.populateTaskDefinitions(workflowDefinition); @@ -128,7 +126,7 @@ public void testMetadataPopulationOnlyOnNecessaryWorkflowTasks() { WorkflowTask workflowTask2 = createWorkflowTask(nameTaskDefinition2); WorkflowDef workflowDefinition = createWorkflowDefinition("testMetadataPopulation"); - workflowDefinition.setTasks(ImmutableList.of(workflowTask1, workflowTask2)); + workflowDefinition.setTasks(List.of(workflowTask1, workflowTask2)); when(metadataDAO.getTaskDef(nameTaskDefinition2)).thenReturn(taskDefinition); @@ -154,7 +152,7 @@ public void testMetadataPopulationMissingDefinitions() { TaskDef taskDefinition = createTaskDefinition(nameTaskDefinition1); WorkflowDef workflowDefinition = createWorkflowDefinition("testMetadataPopulation"); - workflowDefinition.setTasks(ImmutableList.of(workflowTask1, workflowTask2)); + workflowDefinition.setTasks(List.of(workflowTask1, workflowTask2)); when(metadataDAO.getTaskDef(nameTaskDefinition1)).thenReturn(taskDefinition); when(metadataDAO.getTaskDef(nameTaskDefinition2)).thenReturn(null); @@ -178,7 +176,7 @@ public void testVersionPopulationForSubworkflowTaskIfVersionIsNotAvailable() { workflowTask.setSubWorkflowParam(subWorkflowParams); WorkflowDef workflowDefinition = createWorkflowDefinition("testMetadataPopulation"); - workflowDefinition.setTasks(ImmutableList.of(workflowTask)); + workflowDefinition.setTasks(List.of(workflowTask)); when(metadataDAO.getLatestWorkflowDef(workflowDefinitionName)) .thenReturn(Optional.of(subWorkflowDefinition)); @@ -211,7 +209,7 @@ public void testNoVersionPopulationForSubworkflowTaskIfAvailable() { workflowTask.setSubWorkflowParam(subWorkflowParams); WorkflowDef workflowDefinition = createWorkflowDefinition("testMetadataPopulation"); - workflowDefinition.setTasks(ImmutableList.of(workflowTask)); + workflowDefinition.setTasks(List.of(workflowTask)); metadataMapperService.populateTaskDefinitions(workflowDefinition); @@ -238,7 +236,7 @@ public void testExceptionWhenWorkflowDefinitionNotAvailable() { workflowTask.setSubWorkflowParam(subWorkflowParams); WorkflowDef workflowDefinition = createWorkflowDefinition("testMetadataPopulation"); - workflowDefinition.setTasks(ImmutableList.of(workflowTask)); + workflowDefinition.setTasks(List.of(workflowTask)); when(metadataDAO.getLatestWorkflowDef(workflowDefinitionName)).thenReturn(Optional.empty()); diff --git a/dependencies.gradle b/dependencies.gradle index d117d05fc3..213a6c0406 100644 --- a/dependencies.gradle +++ b/dependencies.gradle @@ -34,7 +34,6 @@ ext { revGroovy = '2.5.13' revGrpc = '1.+' revGuava = '30.0-jre' - revGuavaRetrying = '2.0.0' revHamcrestAllMatchers = '1.8' revHealth = '1.1.+' revJAXB = '2.3.3' diff --git a/es6-persistence/build.gradle b/es6-persistence/build.gradle index f4d2d430bf..2bf0d49c89 100644 --- a/es6-persistence/build.gradle +++ b/es6-persistence/build.gradle @@ -16,6 +16,7 @@ dependencies { implementation project(':conductor-core') compileOnly 'org.springframework.boot:spring-boot-starter' + compileOnly 'org.springframework.retry:spring-retry' implementation "commons-io:commons-io:${revCommonsIo}" implementation "org.apache.commons:commons-lang3" @@ -26,6 +27,7 @@ dependencies { implementation "org.elasticsearch.client:elasticsearch-rest-client" implementation "org.elasticsearch.client:elasticsearch-rest-high-level-client" + testImplementation 'org.springframework.retry:spring-retry' testImplementation "org.awaitility:awaitility:${revAwaitility}" testImplementation "org.testcontainers:elasticsearch:${revTestContainer}" testImplementation project(':conductor-common').sourceSets.test.output diff --git a/es6-persistence/dependencies.lock b/es6-persistence/dependencies.lock index 5229213a92..9945fa5cee 100644 --- a/es6-persistence/dependencies.lock +++ b/es6-persistence/dependencies.lock @@ -463,6 +463,9 @@ "org.springframework.boot:spring-boot-starter" ] }, + "org.springframework.retry:spring-retry": { + "locked": "1.2.5.RELEASE" + }, "org.springframework:spring-aop": { "locked": "5.2.15.RELEASE", "transitive": [ @@ -487,6 +490,7 @@ "transitive": [ "org.springframework.boot:spring-boot", "org.springframework.boot:spring-boot-starter", + "org.springframework.retry:spring-retry", "org.springframework:spring-aop", "org.springframework:spring-beans", "org.springframework:spring-context", @@ -564,12 +568,6 @@ "org.elasticsearch:elasticsearch-x-content" ] }, - "com.github.rholder:guava-retrying": { - "locked": "2.0.0", - "transitive": [ - "com.netflix.conductor:conductor-common" - ] - }, "com.github.spullara.mustache.java:compiler": { "locked": "0.9.3", "transitive": [ @@ -579,7 +577,6 @@ "com.google.code.findbugs:jsr305": { "locked": "3.0.2", "transitive": [ - "com.github.rholder:guava-retrying", "com.google.guava:guava" ] }, @@ -596,11 +593,7 @@ ] }, "com.google.guava:guava": { - "locked": "30.0-jre", - "transitive": [ - "com.github.rholder:guava-retrying", - "com.netflix.conductor:conductor-core" - ] + "locked": "30.0-jre" }, "com.google.guava:listenablefuture": { "locked": "9999.0-empty-to-avoid-conflict-with-guava", @@ -1793,6 +1786,9 @@ "org.springframework.boot:spring-boot-starter-test" ] }, + "org.springframework.retry:spring-retry": { + "locked": "1.2.5.RELEASE" + }, "org.springframework:spring-aop": { "locked": "5.2.15.RELEASE", "transitive": [ @@ -1818,6 +1814,7 @@ "org.springframework.boot:spring-boot", "org.springframework.boot:spring-boot-starter", "org.springframework.boot:spring-boot-starter-test", + "org.springframework.retry:spring-retry", "org.springframework:spring-aop", "org.springframework:spring-beans", "org.springframework:spring-context", @@ -1936,12 +1933,6 @@ "org.testcontainers:testcontainers" ] }, - "com.github.rholder:guava-retrying": { - "locked": "2.0.0", - "transitive": [ - "com.netflix.conductor:conductor-common" - ] - }, "com.github.spullara.mustache.java:compiler": { "locked": "0.9.3", "transitive": [ @@ -1951,7 +1942,6 @@ "com.google.code.findbugs:jsr305": { "locked": "3.0.2", "transitive": [ - "com.github.rholder:guava-retrying", "com.google.guava:guava" ] }, @@ -1968,11 +1958,7 @@ ] }, "com.google.guava:guava": { - "locked": "30.0-jre", - "transitive": [ - "com.github.rholder:guava-retrying", - "com.netflix.conductor:conductor-core" - ] + "locked": "30.0-jre" }, "com.google.guava:listenablefuture": { "locked": "9999.0-empty-to-avoid-conflict-with-guava", @@ -2687,6 +2673,9 @@ "org.springframework.boot:spring-boot-starter-test" ] }, + "org.springframework.retry:spring-retry": { + "locked": "1.2.5.RELEASE" + }, "org.springframework:spring-aop": { "locked": "5.2.15.RELEASE", "transitive": [ @@ -2712,6 +2701,7 @@ "org.springframework.boot:spring-boot", "org.springframework.boot:spring-boot-starter", "org.springframework.boot:spring-boot-starter-test", + "org.springframework.retry:spring-retry", "org.springframework:spring-aop", "org.springframework:spring-beans", "org.springframework:spring-context", diff --git a/es6-persistence/src/main/java/com/netflix/conductor/es6/config/ElasticSearchV6Configuration.java b/es6-persistence/src/main/java/com/netflix/conductor/es6/config/ElasticSearchV6Configuration.java index 1f5f8469dc..a16bea44db 100644 --- a/es6-persistence/src/main/java/com/netflix/conductor/es6/config/ElasticSearchV6Configuration.java +++ b/es6-persistence/src/main/java/com/netflix/conductor/es6/config/ElasticSearchV6Configuration.java @@ -34,10 +34,13 @@ import org.elasticsearch.transport.client.PreBuiltTransportClient; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.boot.context.properties.EnableConfigurationProperties; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Conditional; import org.springframework.context.annotation.Configuration; +import org.springframework.retry.backoff.FixedBackOffPolicy; +import org.springframework.retry.support.RetryTemplate; import com.netflix.conductor.dao.IndexDAO; import com.netflix.conductor.es6.dao.index.ElasticSearchDAOV6; @@ -122,15 +125,29 @@ public RestClientBuilder restClientBuilder(ElasticSearchProperties properties) { public IndexDAO es6IndexRestDAO( RestClientBuilder restClientBuilder, ElasticSearchProperties properties, + @Qualifier("es6RetryTemplate") RetryTemplate retryTemplate, ObjectMapper objectMapper) { - return new ElasticSearchRestDAOV6(restClientBuilder, properties, objectMapper); + return new ElasticSearchRestDAOV6( + restClientBuilder, retryTemplate, properties, objectMapper); } @Bean @Conditional(IsTcpProtocol.class) public IndexDAO es6IndexDAO( - Client client, ElasticSearchProperties properties, ObjectMapper objectMapper) { - return new ElasticSearchDAOV6(client, properties, objectMapper); + Client client, + @Qualifier("es6RetryTemplate") RetryTemplate retryTemplate, + ElasticSearchProperties properties, + ObjectMapper objectMapper) { + return new ElasticSearchDAOV6(client, retryTemplate, properties, objectMapper); + } + + @Bean + public RetryTemplate es6RetryTemplate() { + RetryTemplate retryTemplate = new RetryTemplate(); + FixedBackOffPolicy fixedBackOffPolicy = new FixedBackOffPolicy(); + fixedBackOffPolicy.setBackOffPeriod(1000L); + retryTemplate.setBackOffPolicy(fixedBackOffPolicy); + return retryTemplate; } private HttpHost[] convertToHttpHosts(List hosts) { diff --git a/es6-persistence/src/main/java/com/netflix/conductor/es6/dao/index/ElasticSearchDAOV6.java b/es6-persistence/src/main/java/com/netflix/conductor/es6/dao/index/ElasticSearchDAOV6.java index 0b36f01f9e..0d40fd61ad 100644 --- a/es6-persistence/src/main/java/com/netflix/conductor/es6/dao/index/ElasticSearchDAOV6.java +++ b/es6-persistence/src/main/java/com/netflix/conductor/es6/dao/index/ElasticSearchDAOV6.java @@ -31,7 +31,6 @@ import org.elasticsearch.action.admin.indices.mapping.get.GetMappingsResponse; import org.elasticsearch.action.admin.indices.template.get.GetIndexTemplatesResponse; import org.elasticsearch.action.bulk.BulkRequestBuilder; -import org.elasticsearch.action.bulk.BulkResponse; import org.elasticsearch.action.delete.DeleteRequest; import org.elasticsearch.action.delete.DeleteResponse; import org.elasticsearch.action.get.GetRequest; @@ -40,7 +39,6 @@ import org.elasticsearch.action.search.SearchRequestBuilder; import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.action.update.UpdateRequest; -import org.elasticsearch.action.update.UpdateResponse; import org.elasticsearch.client.Client; import org.elasticsearch.common.Strings; import org.elasticsearch.common.settings.Settings; @@ -56,6 +54,7 @@ import org.elasticsearch.search.sort.SortOrder; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.springframework.retry.support.RetryTemplate; import com.netflix.conductor.annotations.Trace; import com.netflix.conductor.common.metadata.events.EventExecution; @@ -63,7 +62,6 @@ import com.netflix.conductor.common.run.SearchResult; import com.netflix.conductor.common.run.TaskSummary; import com.netflix.conductor.common.run.WorkflowSummary; -import com.netflix.conductor.common.utils.RetryUtil; import com.netflix.conductor.core.events.queue.Message; import com.netflix.conductor.core.exception.ApplicationException; import com.netflix.conductor.dao.IndexDAO; @@ -86,7 +84,6 @@ public class ElasticSearchDAOV6 extends ElasticSearchBaseDAO implements IndexDAO private static final String EVENT_DOC_TYPE = "event"; private static final String MSG_DOC_TYPE = "message"; - private static final int RETRY_COUNT = 3; private static final int CORE_POOL_SIZE = 6; private static final long KEEP_ALIVE_TIME = 1L; private static final int UPDATE_REQUEST_RETRY_COUNT = 5; @@ -117,12 +114,15 @@ public class ElasticSearchDAOV6 extends ElasticSearchBaseDAO implements IndexDAO private final long asyncBufferFlushTimeout; private final ElasticSearchProperties properties; + private final RetryTemplate retryTemplate; + static { SIMPLE_DATE_FORMAT.setTimeZone(GMT); } public ElasticSearchDAOV6( Client elasticSearchClient, + RetryTemplate retryTemplate, ElasticSearchProperties properties, ObjectMapper objectMapper) { this.objectMapper = objectMapper; @@ -182,6 +182,7 @@ public ElasticSearchDAOV6( Executors.newSingleThreadScheduledExecutor() .scheduleAtFixedRate(this::flushBulkRequests, 60, 30, TimeUnit.SECONDS); + this.retryTemplate = retryTemplate; } @PreDestroy @@ -360,14 +361,7 @@ public void indexWorkflow(WorkflowSummary workflow) { StringUtils.isBlank(docTypeOverride) ? WORKFLOW_DOC_TYPE : docTypeOverride; UpdateRequest req = buildUpdateRequest(id, doc, workflowIndexName, docType); - new RetryUtil() - .retryOnException( - () -> elasticSearchClient.update(req).actionGet(), - null, - null, - RETRY_COUNT, - "Indexing workflow document: " + workflow.getWorkflowId(), - "indexWorkflow"); + elasticSearchClient.update(req).actionGet(); long endTime = Instant.now().toEpochMilli(); LOGGER.debug( @@ -461,14 +455,7 @@ public void addTaskExecutionLogs(List taskExecLogs) { request.source(objectMapper.writeValueAsBytes(log), XContentType.JSON); bulkRequestBuilder.add(request); } - new RetryUtil() - .retryOnException( - () -> bulkRequestBuilder.execute().actionGet(5, TimeUnit.SECONDS), - null, - BulkResponse::hasFailures, - RETRY_COUNT, - "Indexing task execution logs", - "addTaskExecutionLogs"); + bulkRequestBuilder.execute().actionGet(5, TimeUnit.SECONDS); long endTime = Instant.now().toEpochMilli(); LOGGER.debug("Time taken {} for indexing taskExecutionLogs", endTime - startTime); Monitors.recordESIndexTime( @@ -656,14 +643,7 @@ private List mapEventExecutionsResponse(SearchResponse response) private void updateWithRetry(BulkRequestBuilderWrapper request, String docType) { try { long startTime = Instant.now().toEpochMilli(); - new RetryUtil() - .retryOnException( - () -> request.execute().actionGet(5, TimeUnit.SECONDS), - null, - BulkResponse::hasFailures, - RETRY_COUNT, - "Bulk Indexing " + docType, - "updateWithRetry"); + retryTemplate.execute(context -> request.execute().actionGet(5, TimeUnit.SECONDS)); long endTime = Instant.now().toEpochMilli(); LOGGER.debug( "Time taken {} for indexing object of type: {}", endTime - startTime, docType); @@ -738,14 +718,7 @@ public void updateWorkflow(String workflowInstanceId, String[] keys, Object[] va "Updating workflow {} in elasticsearch index: {}", workflowInstanceId, workflowIndexName); - new RetryUtil<>() - .retryOnException( - () -> elasticSearchClient.update(request).actionGet(), - null, - null, - RETRY_COUNT, - "Updating index for doc_type workflow", - "updateWorkflow"); + elasticSearchClient.update(request).actionGet(); long endTime = Instant.now().toEpochMilli(); LOGGER.debug( "Time taken {} for updating workflow: {}", endTime - startTime, workflowInstanceId); diff --git a/es6-persistence/src/main/java/com/netflix/conductor/es6/dao/index/ElasticSearchRestDAOV6.java b/es6-persistence/src/main/java/com/netflix/conductor/es6/dao/index/ElasticSearchRestDAOV6.java index 2e2083ea43..a55d2f7d3f 100644 --- a/es6-persistence/src/main/java/com/netflix/conductor/es6/dao/index/ElasticSearchRestDAOV6.java +++ b/es6-persistence/src/main/java/com/netflix/conductor/es6/dao/index/ElasticSearchRestDAOV6.java @@ -35,17 +35,14 @@ import org.apache.http.util.EntityUtils; import org.elasticsearch.action.DocWriteResponse; import org.elasticsearch.action.bulk.BulkRequest; -import org.elasticsearch.action.bulk.BulkResponse; import org.elasticsearch.action.delete.DeleteRequest; import org.elasticsearch.action.delete.DeleteResponse; import org.elasticsearch.action.get.GetRequest; import org.elasticsearch.action.get.GetResponse; import org.elasticsearch.action.index.IndexRequest; -import org.elasticsearch.action.index.IndexResponse; import org.elasticsearch.action.search.SearchRequest; import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.action.update.UpdateRequest; -import org.elasticsearch.action.update.UpdateResponse; import org.elasticsearch.client.*; import org.elasticsearch.client.core.CountRequest; import org.elasticsearch.client.core.CountResponse; @@ -59,6 +56,7 @@ import org.elasticsearch.search.sort.SortOrder; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.springframework.retry.support.RetryTemplate; import com.netflix.conductor.annotations.Trace; import com.netflix.conductor.common.metadata.events.EventExecution; @@ -66,7 +64,6 @@ import com.netflix.conductor.common.run.SearchResult; import com.netflix.conductor.common.run.TaskSummary; import com.netflix.conductor.common.run.WorkflowSummary; -import com.netflix.conductor.common.utils.RetryUtil; import com.netflix.conductor.core.events.queue.Message; import com.netflix.conductor.core.exception.ApplicationException; import com.netflix.conductor.dao.IndexDAO; @@ -86,7 +83,6 @@ public class ElasticSearchRestDAOV6 extends ElasticSearchBaseDAO implements Inde private static final Logger LOGGER = LoggerFactory.getLogger(ElasticSearchRestDAOV6.class); - private static final int RETRY_COUNT = 3; private static final int CORE_POOL_SIZE = 6; private static final long KEEP_ALIVE_TIME = 1L; @@ -130,12 +126,15 @@ public class ElasticSearchRestDAOV6 extends ElasticSearchBaseDAO implements Inde private final long asyncBufferFlushTimeout; private final ElasticSearchProperties properties; + private final RetryTemplate retryTemplate; + static { SIMPLE_DATE_FORMAT.setTimeZone(GMT); } public ElasticSearchRestDAOV6( RestClientBuilder restClientBuilder, + RetryTemplate retryTemplate, ElasticSearchProperties properties, ObjectMapper objectMapper) { @@ -202,6 +201,7 @@ public ElasticSearchRestDAOV6( Executors.newSingleThreadScheduledExecutor() .scheduleAtFixedRate(this::flushBulkRequests, 60, 30, TimeUnit.SECONDS); + this.retryTemplate = retryTemplate; } @PreDestroy @@ -453,21 +453,7 @@ public void indexWorkflow(WorkflowSummary workflow) { IndexRequest request = new IndexRequest(workflowIndexName, docType, workflowId); request.source(docBytes, XContentType.JSON); - new RetryUtil() - .retryOnException( - () -> { - try { - return elasticSearchClient.index(request); - } catch (IOException e) { - throw new RuntimeException(e); - } - }, - null, - null, - RETRY_COUNT, - "Indexing workflow document: " + workflow.getWorkflowId(), - "indexWorkflow"); - + elasticSearchClient.index(request, RequestOptions.DEFAULT); long endTime = Instant.now().toEpochMilli(); LOGGER.debug( "Time taken {} for indexing workflow: {}", endTime - startTime, workflowId); @@ -537,20 +523,7 @@ public void addTaskExecutionLogs(List taskExecLogs) { } try { - new RetryUtil() - .retryOnException( - () -> { - try { - return elasticSearchClient.bulk(bulkRequest); - } catch (IOException e) { - throw new RuntimeException(e); - } - }, - null, - BulkResponse::hasFailures, - RETRY_COUNT, - "Indexing task execution logs", - "addTaskExecutionLogs"); + elasticSearchClient.bulk(bulkRequest, RequestOptions.DEFAULT); long endTime = Instant.now().toEpochMilli(); LOGGER.debug("Time taken {} for indexing taskExecutionLogs", endTime - startTime); Monitors.recordESIndexTime( @@ -799,43 +772,38 @@ public CompletableFuture asyncRemoveWorkflow(String workflowId) { @Override public void updateWorkflow(String workflowInstanceId, String[] keys, Object[] values) { - if (keys.length != values.length) { - throw new ApplicationException( - ApplicationException.Code.INVALID_INPUT, - "Number of keys and values do not match"); - } + try { + if (keys.length != values.length) { + throw new ApplicationException( + ApplicationException.Code.INVALID_INPUT, + "Number of keys and values do not match"); + } - long startTime = Instant.now().toEpochMilli(); - String docType = StringUtils.isBlank(docTypeOverride) ? WORKFLOW_DOC_TYPE : docTypeOverride; - UpdateRequest request = new UpdateRequest(workflowIndexName, docType, workflowInstanceId); - Map source = - IntStream.range(0, keys.length) - .boxed() - .collect(Collectors.toMap(i -> keys[i], i -> values[i])); - request.doc(source); - - LOGGER.debug("Updating workflow {} with {}", workflowInstanceId, source); - - new RetryUtil() - .retryOnException( - () -> { - try { - return elasticSearchClient.update(request); - } catch (IOException e) { - throw new RuntimeException(e); - } - }, - null, - null, - RETRY_COUNT, - "Updating workflow document: " + workflowInstanceId, - "updateWorkflow"); - long endTime = Instant.now().toEpochMilli(); - LOGGER.debug( - "Time taken {} for updating workflow: {}", endTime - startTime, workflowInstanceId); - Monitors.recordESIndexTime("update_workflow", WORKFLOW_DOC_TYPE, endTime - startTime); - Monitors.recordWorkerQueueSize( - "indexQueue", ((ThreadPoolExecutor) executorService).getQueue().size()); + long startTime = Instant.now().toEpochMilli(); + String docType = + StringUtils.isBlank(docTypeOverride) ? WORKFLOW_DOC_TYPE : docTypeOverride; + UpdateRequest request = + new UpdateRequest(workflowIndexName, docType, workflowInstanceId); + Map source = + IntStream.range(0, keys.length) + .boxed() + .collect(Collectors.toMap(i -> keys[i], i -> values[i])); + request.doc(source); + + LOGGER.debug("Updating workflow {} with {}", workflowInstanceId, source); + elasticSearchClient.update(request, RequestOptions.DEFAULT); + long endTime = Instant.now().toEpochMilli(); + LOGGER.debug( + "Time taken {} for updating workflow: {}", + endTime - startTime, + workflowInstanceId); + Monitors.recordESIndexTime("update_workflow", WORKFLOW_DOC_TYPE, endTime - startTime); + Monitors.recordWorkerQueueSize( + "indexQueue", ((ThreadPoolExecutor) executorService).getQueue().size()); + } catch (Exception e) { + LOGGER.error("Failed to update workflow {}", workflowInstanceId, e); + Monitors.error(className, "update"); + } } @Override @@ -1057,20 +1025,8 @@ private void indexWithRetry( final BulkRequest request, final String operationDescription, String docType) { try { long startTime = Instant.now().toEpochMilli(); - new RetryUtil() - .retryOnException( - () -> { - try { - return elasticSearchClient.bulk(request); - } catch (IOException e) { - throw new RuntimeException(e); - } - }, - null, - null, - RETRY_COUNT, - operationDescription, - "indexWithRetry"); + retryTemplate.execute( + context -> elasticSearchClient.bulk(request, RequestOptions.DEFAULT)); long endTime = Instant.now().toEpochMilli(); LOGGER.debug( "Time taken {} for indexing object of type: {}", endTime - startTime, docType); diff --git a/es6-persistence/src/test/java/com/netflix/conductor/es6/dao/index/ElasticSearchDaoBaseTest.java b/es6-persistence/src/test/java/com/netflix/conductor/es6/dao/index/ElasticSearchDaoBaseTest.java index 6dc8cd8c4d..529f22e12f 100644 --- a/es6-persistence/src/test/java/com/netflix/conductor/es6/dao/index/ElasticSearchDaoBaseTest.java +++ b/es6-persistence/src/test/java/com/netflix/conductor/es6/dao/index/ElasticSearchDaoBaseTest.java @@ -1,5 +1,5 @@ /* - * Copyright 2020 Netflix, Inc. + * Copyright 2022 Netflix, Inc. *

* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at @@ -25,6 +25,7 @@ import org.junit.After; import org.junit.AfterClass; import org.junit.Before; +import org.springframework.retry.support.RetryTemplate; abstract class ElasticSearchDaoBaseTest extends ElasticSearchTest { @@ -45,7 +46,9 @@ public void setup() throws Exception { new TransportAddress( InetAddress.getByName("localhost"), mappedPort)); - indexDAO = new ElasticSearchDAOV6(elasticSearchClient, properties, objectMapper); + indexDAO = + new ElasticSearchDAOV6( + elasticSearchClient, new RetryTemplate(), properties, objectMapper); indexDAO.setup(); } diff --git a/es6-persistence/src/test/java/com/netflix/conductor/es6/dao/index/ElasticSearchRestDaoBaseTest.java b/es6-persistence/src/test/java/com/netflix/conductor/es6/dao/index/ElasticSearchRestDaoBaseTest.java index 1ac35c8546..59584e1dc7 100644 --- a/es6-persistence/src/test/java/com/netflix/conductor/es6/dao/index/ElasticSearchRestDaoBaseTest.java +++ b/es6-persistence/src/test/java/com/netflix/conductor/es6/dao/index/ElasticSearchRestDaoBaseTest.java @@ -23,6 +23,7 @@ import org.elasticsearch.client.RestClientBuilder; import org.junit.After; import org.junit.Before; +import org.springframework.retry.support.RetryTemplate; abstract class ElasticSearchRestDaoBaseTest extends ElasticSearchTest { @@ -40,7 +41,9 @@ public void setup() throws Exception { RestClientBuilder restClientBuilder = RestClient.builder(new HttpHost(host, port, "http")); restClient = restClientBuilder.build(); - indexDAO = new ElasticSearchRestDAOV6(restClientBuilder, properties, objectMapper); + indexDAO = + new ElasticSearchRestDAOV6( + restClientBuilder, new RetryTemplate(), properties, objectMapper); indexDAO.setup(); } diff --git a/es7-persistence/build.gradle b/es7-persistence/build.gradle index 646a35ea31..793053deda 100644 --- a/es7-persistence/build.gradle +++ b/es7-persistence/build.gradle @@ -29,6 +29,7 @@ dependencies { implementation project(':conductor-core') compileOnly 'org.springframework.boot:spring-boot-starter' + compileOnly 'org.springframework.retry:spring-retry' implementation "commons-io:commons-io:${revCommonsIo}" implementation "org.apache.commons:commons-lang3" @@ -41,6 +42,7 @@ dependencies { implementation "org.elasticsearch.client:elasticsearch-rest-client" implementation "org.elasticsearch.client:elasticsearch-rest-high-level-client" + testImplementation 'org.springframework.retry:spring-retry' testImplementation "org.awaitility:awaitility:${revAwaitility}" testImplementation "org.testcontainers:elasticsearch:${revTestContainer}" testImplementation project(':conductor-common').sourceSets.test.output diff --git a/es7-persistence/dependencies.lock b/es7-persistence/dependencies.lock index 03c35ae676..a929e007b8 100644 --- a/es7-persistence/dependencies.lock +++ b/es7-persistence/dependencies.lock @@ -395,6 +395,9 @@ "org.springframework.boot:spring-boot-starter" ] }, + "org.springframework.retry:spring-retry": { + "locked": "1.2.5.RELEASE" + }, "org.springframework:spring-aop": { "locked": "5.2.15.RELEASE", "transitive": [ @@ -419,6 +422,7 @@ "transitive": [ "org.springframework.boot:spring-boot", "org.springframework.boot:spring-boot-starter", + "org.springframework.retry:spring-retry", "org.springframework:spring-aop", "org.springframework:spring-beans", "org.springframework:spring-context", @@ -493,12 +497,6 @@ "org.elasticsearch:elasticsearch-x-content" ] }, - "com.github.rholder:guava-retrying": { - "locked": "2.0.0", - "transitive": [ - "com.netflix.conductor:conductor-common" - ] - }, "com.github.spullara.mustache.java:compiler": { "locked": "0.9.6", "transitive": [ @@ -508,7 +506,6 @@ "com.google.code.findbugs:jsr305": { "locked": "3.0.2", "transitive": [ - "com.github.rholder:guava-retrying", "com.google.guava:guava" ] }, @@ -525,11 +522,7 @@ ] }, "com.google.guava:guava": { - "locked": "30.0-jre", - "transitive": [ - "com.github.rholder:guava-retrying", - "com.netflix.conductor:conductor-core" - ] + "locked": "30.0-jre" }, "com.google.guava:listenablefuture": { "locked": "9999.0-empty-to-avoid-conflict-with-guava", @@ -976,6 +969,9 @@ "org.springframework.boot:spring-boot-starter" ] }, + "org.springframework.retry:spring-retry": { + "locked": "1.2.5.RELEASE" + }, "org.springframework:spring-aop": { "locked": "5.2.15.RELEASE", "transitive": [ @@ -1000,6 +996,7 @@ "transitive": [ "org.springframework.boot:spring-boot", "org.springframework.boot:spring-boot-starter", + "org.springframework.retry:spring-retry", "org.springframework:spring-aop", "org.springframework:spring-beans", "org.springframework:spring-context", @@ -1680,6 +1677,9 @@ "org.springframework.boot:spring-boot-starter-test" ] }, + "org.springframework.retry:spring-retry": { + "locked": "1.2.5.RELEASE" + }, "org.springframework:spring-aop": { "locked": "5.2.15.RELEASE", "transitive": [ @@ -1705,6 +1705,7 @@ "org.springframework.boot:spring-boot", "org.springframework.boot:spring-boot-starter", "org.springframework.boot:spring-boot-starter-test", + "org.springframework.retry:spring-retry", "org.springframework:spring-aop", "org.springframework:spring-beans", "org.springframework:spring-context", @@ -1794,6 +1795,9 @@ "org.springframework.boot:spring-boot-starter" ] }, + "org.springframework.retry:spring-retry": { + "locked": "1.2.5.RELEASE" + }, "org.springframework:spring-aop": { "locked": "5.2.15.RELEASE", "transitive": [ @@ -1818,6 +1822,7 @@ "transitive": [ "org.springframework.boot:spring-boot", "org.springframework.boot:spring-boot-starter", + "org.springframework.retry:spring-retry", "org.springframework:spring-aop", "org.springframework:spring-beans", "org.springframework:spring-context", @@ -1910,12 +1915,6 @@ "org.testcontainers:testcontainers" ] }, - "com.github.rholder:guava-retrying": { - "locked": "2.0.0", - "transitive": [ - "com.netflix.conductor:conductor-common" - ] - }, "com.github.spullara.mustache.java:compiler": { "locked": "0.9.6", "transitive": [ @@ -1925,7 +1924,6 @@ "com.google.code.findbugs:jsr305": { "locked": "3.0.2", "transitive": [ - "com.github.rholder:guava-retrying", "com.google.guava:guava" ] }, @@ -1942,11 +1940,7 @@ ] }, "com.google.guava:guava": { - "locked": "30.0-jre", - "transitive": [ - "com.github.rholder:guava-retrying", - "com.netflix.conductor:conductor-core" - ] + "locked": "30.0-jre" }, "com.google.guava:listenablefuture": { "locked": "9999.0-empty-to-avoid-conflict-with-guava", @@ -2597,6 +2591,9 @@ "org.springframework.boot:spring-boot-starter-test" ] }, + "org.springframework.retry:spring-retry": { + "locked": "1.2.5.RELEASE" + }, "org.springframework:spring-aop": { "locked": "5.2.15.RELEASE", "transitive": [ @@ -2622,6 +2619,7 @@ "org.springframework.boot:spring-boot", "org.springframework.boot:spring-boot-starter", "org.springframework.boot:spring-boot-starter-test", + "org.springframework.retry:spring-retry", "org.springframework:spring-aop", "org.springframework:spring-beans", "org.springframework:spring-context", diff --git a/es7-persistence/src/main/java/com/netflix/conductor/es7/config/ElasticSearchV7Configuration.java b/es7-persistence/src/main/java/com/netflix/conductor/es7/config/ElasticSearchV7Configuration.java index bc241b446d..6aab66f0d8 100644 --- a/es7-persistence/src/main/java/com/netflix/conductor/es7/config/ElasticSearchV7Configuration.java +++ b/es7-persistence/src/main/java/com/netflix/conductor/es7/config/ElasticSearchV7Configuration.java @@ -24,10 +24,13 @@ import org.elasticsearch.client.RestClientBuilder; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.boot.context.properties.EnableConfigurationProperties; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Conditional; import org.springframework.context.annotation.Configuration; +import org.springframework.retry.backoff.FixedBackOffPolicy; +import org.springframework.retry.support.RetryTemplate; import com.netflix.conductor.dao.IndexDAO; import com.netflix.conductor.es7.dao.index.ElasticSearchRestDAOV7; @@ -79,10 +82,21 @@ public RestClientBuilder restClientBuilder(ElasticSearchProperties properties) { @Bean public IndexDAO es7IndexDAO( RestClientBuilder restClientBuilder, + @Qualifier("es7RetryTemplate") RetryTemplate retryTemplate, ElasticSearchProperties properties, ObjectMapper objectMapper) { String url = properties.getUrl(); - return new ElasticSearchRestDAOV7(restClientBuilder, properties, objectMapper); + return new ElasticSearchRestDAOV7( + restClientBuilder, retryTemplate, properties, objectMapper); + } + + @Bean + public RetryTemplate es7RetryTemplate() { + RetryTemplate retryTemplate = new RetryTemplate(); + FixedBackOffPolicy fixedBackOffPolicy = new FixedBackOffPolicy(); + fixedBackOffPolicy.setBackOffPeriod(1000L); + retryTemplate.setBackOffPolicy(fixedBackOffPolicy); + return retryTemplate; } private HttpHost[] convertToHttpHosts(List hosts) { diff --git a/es7-persistence/src/main/java/com/netflix/conductor/es7/dao/index/ElasticSearchRestDAOV7.java b/es7-persistence/src/main/java/com/netflix/conductor/es7/dao/index/ElasticSearchRestDAOV7.java index dd32f39827..d56b26b622 100644 --- a/es7-persistence/src/main/java/com/netflix/conductor/es7/dao/index/ElasticSearchRestDAOV7.java +++ b/es7-persistence/src/main/java/com/netflix/conductor/es7/dao/index/ElasticSearchRestDAOV7.java @@ -34,17 +34,14 @@ import org.apache.http.util.EntityUtils; import org.elasticsearch.action.DocWriteResponse; import org.elasticsearch.action.bulk.BulkRequest; -import org.elasticsearch.action.bulk.BulkResponse; import org.elasticsearch.action.delete.DeleteRequest; import org.elasticsearch.action.delete.DeleteResponse; import org.elasticsearch.action.get.GetRequest; import org.elasticsearch.action.get.GetResponse; import org.elasticsearch.action.index.IndexRequest; -import org.elasticsearch.action.index.IndexResponse; import org.elasticsearch.action.search.SearchRequest; import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.action.update.UpdateRequest; -import org.elasticsearch.action.update.UpdateResponse; import org.elasticsearch.client.*; import org.elasticsearch.client.core.CountRequest; import org.elasticsearch.client.core.CountResponse; @@ -59,6 +56,7 @@ import org.joda.time.DateTime; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.springframework.retry.support.RetryTemplate; import com.netflix.conductor.annotations.Trace; import com.netflix.conductor.common.metadata.events.EventExecution; @@ -66,7 +64,6 @@ import com.netflix.conductor.common.run.SearchResult; import com.netflix.conductor.common.run.TaskSummary; import com.netflix.conductor.common.run.WorkflowSummary; -import com.netflix.conductor.common.utils.RetryUtil; import com.netflix.conductor.core.events.queue.Message; import com.netflix.conductor.core.exception.ApplicationException; import com.netflix.conductor.dao.IndexDAO; @@ -86,7 +83,6 @@ public class ElasticSearchRestDAOV7 extends ElasticSearchBaseDAO implements Inde private static final Logger logger = LoggerFactory.getLogger(ElasticSearchRestDAOV7.class); - private static final int RETRY_COUNT = 3; private static final int CORE_POOL_SIZE = 6; private static final long KEEP_ALIVE_TIME = 1L; @@ -127,6 +123,7 @@ public class ElasticSearchRestDAOV7 extends ElasticSearchBaseDAO implements Inde private final int indexBatchSize; private final int asyncBufferFlushTimeout; private final ElasticSearchProperties properties; + private final RetryTemplate retryTemplate; static { SIMPLE_DATE_FORMAT.setTimeZone(GMT); @@ -134,6 +131,7 @@ public class ElasticSearchRestDAOV7 extends ElasticSearchBaseDAO implements Inde public ElasticSearchRestDAOV7( RestClientBuilder restClientBuilder, + RetryTemplate retryTemplate, ElasticSearchProperties properties, ObjectMapper objectMapper) { @@ -194,6 +192,7 @@ public ElasticSearchRestDAOV7( Executors.newSingleThreadScheduledExecutor() .scheduleAtFixedRate(this::flushBulkRequests, 60, 30, TimeUnit.SECONDS); + this.retryTemplate = retryTemplate; } @PreDestroy @@ -493,22 +492,7 @@ public void indexWorkflow(WorkflowSummary workflow) { new IndexRequest(workflowIndexName) .id(workflowId) .source(docBytes, XContentType.JSON); - new RetryUtil() - .retryOnException( - () -> { - try { - return elasticSearchClient.index( - request, RequestOptions.DEFAULT); - } catch (IOException e) { - throw new RuntimeException(e); - } - }, - null, - null, - RETRY_COUNT, - "Indexing workflow document: " + workflow.getWorkflowId(), - "indexWorkflow"); - + elasticSearchClient.index(request, RequestOptions.DEFAULT); long endTime = Instant.now().toEpochMilli(); logger.debug( "Time taken {} for indexing workflow: {}", endTime - startTime, workflowId); @@ -576,21 +560,7 @@ public void addTaskExecutionLogs(List taskExecLogs) { } try { - new RetryUtil() - .retryOnException( - () -> { - try { - return elasticSearchClient.bulk( - bulkRequest, RequestOptions.DEFAULT); - } catch (IOException e) { - throw new RuntimeException(e); - } - }, - null, - BulkResponse::hasFailures, - RETRY_COUNT, - "Indexing task execution logs", - "addTaskExecutionLogs"); + elasticSearchClient.bulk(bulkRequest, RequestOptions.DEFAULT); long endTime = Instant.now().toEpochMilli(); logger.debug("Time taken {} for indexing taskExecutionLogs", endTime - startTime); Monitors.recordESIndexTime( @@ -831,42 +801,35 @@ public CompletableFuture asyncRemoveWorkflow(String workflowId) { @Override public void updateWorkflow(String workflowInstanceId, String[] keys, Object[] values) { - if (keys.length != values.length) { - throw new ApplicationException( - ApplicationException.Code.INVALID_INPUT, - "Number of keys and values do not match"); - } + try { + if (keys.length != values.length) { + throw new ApplicationException( + ApplicationException.Code.INVALID_INPUT, + "Number of keys and values do not match"); + } - long startTime = Instant.now().toEpochMilli(); - UpdateRequest request = new UpdateRequest(workflowIndexName, workflowInstanceId); - Map source = - IntStream.range(0, keys.length) - .boxed() - .collect(Collectors.toMap(i -> keys[i], i -> values[i])); - request.doc(source); - - logger.debug("Updating workflow {} with {}", workflowInstanceId, source); - - new RetryUtil() - .retryOnException( - () -> { - try { - return elasticSearchClient.update(request, RequestOptions.DEFAULT); - } catch (IOException e) { - throw new RuntimeException(e); - } - }, - null, - null, - RETRY_COUNT, - "Updating workflow document: " + workflowInstanceId, - "updateWorkflow"); - long endTime = Instant.now().toEpochMilli(); - logger.debug( - "Time taken {} for updating workflow: {}", endTime - startTime, workflowInstanceId); - Monitors.recordESIndexTime("update_workflow", WORKFLOW_DOC_TYPE, endTime - startTime); - Monitors.recordWorkerQueueSize( - "indexQueue", ((ThreadPoolExecutor) executorService).getQueue().size()); + long startTime = Instant.now().toEpochMilli(); + UpdateRequest request = new UpdateRequest(workflowIndexName, workflowInstanceId); + Map source = + IntStream.range(0, keys.length) + .boxed() + .collect(Collectors.toMap(i -> keys[i], i -> values[i])); + request.doc(source); + + logger.debug("Updating workflow {} with {}", workflowInstanceId, source); + elasticSearchClient.update(request, RequestOptions.DEFAULT); + long endTime = Instant.now().toEpochMilli(); + logger.debug( + "Time taken {} for updating workflow: {}", + endTime - startTime, + workflowInstanceId); + Monitors.recordESIndexTime("update_workflow", WORKFLOW_DOC_TYPE, endTime - startTime); + Monitors.recordWorkerQueueSize( + "indexQueue", ((ThreadPoolExecutor) executorService).getQueue().size()); + } catch (Exception e) { + logger.error("Failed to update workflow {}", workflowInstanceId, e); + Monitors.error(className, "update"); + } } @Override @@ -1110,21 +1073,8 @@ private void indexWithRetry( final BulkRequest request, final String operationDescription, String docType) { try { long startTime = Instant.now().toEpochMilli(); - new RetryUtil() - .retryOnException( - () -> { - try { - return elasticSearchClient.bulk( - request, RequestOptions.DEFAULT); - } catch (IOException e) { - throw new RuntimeException(e); - } - }, - null, - null, - RETRY_COUNT, - operationDescription, - "indexWithRetry"); + retryTemplate.execute( + context -> elasticSearchClient.bulk(request, RequestOptions.DEFAULT)); long endTime = Instant.now().toEpochMilli(); logger.debug( "Time taken {} for indexing object of type: {}", endTime - startTime, docType); @@ -1149,7 +1099,7 @@ private void flushBulkRequests() { .filter( entry -> (System.currentTimeMillis() - entry.getValue().getLastFlushTime()) - >= asyncBufferFlushTimeout * 1000) + >= asyncBufferFlushTimeout * 1000L) .filter( entry -> entry.getValue().getBulkRequest() != null diff --git a/es7-persistence/src/test/java/com/netflix/conductor/es7/dao/index/ElasticSearchRestDaoBaseTest.java b/es7-persistence/src/test/java/com/netflix/conductor/es7/dao/index/ElasticSearchRestDaoBaseTest.java index 3541a89e14..97829ddf7c 100644 --- a/es7-persistence/src/test/java/com/netflix/conductor/es7/dao/index/ElasticSearchRestDaoBaseTest.java +++ b/es7-persistence/src/test/java/com/netflix/conductor/es7/dao/index/ElasticSearchRestDaoBaseTest.java @@ -24,6 +24,7 @@ import org.elasticsearch.client.RestClientBuilder; import org.junit.After; import org.junit.Before; +import org.springframework.retry.support.RetryTemplate; public abstract class ElasticSearchRestDaoBaseTest extends ElasticSearchTest { @@ -41,7 +42,9 @@ public void setup() throws Exception { RestClientBuilder restClientBuilder = RestClient.builder(new HttpHost(host, port, "http")); restClient = restClientBuilder.build(); - indexDAO = new ElasticSearchRestDAOV7(restClientBuilder, properties, objectMapper); + indexDAO = + new ElasticSearchRestDAOV7( + restClientBuilder, new RetryTemplate(), properties, objectMapper); indexDAO.setup(); } diff --git a/grpc-client/dependencies.lock b/grpc-client/dependencies.lock index edf9cc629b..4a8f3ac014 100644 --- a/grpc-client/dependencies.lock +++ b/grpc-client/dependencies.lock @@ -64,7 +64,7 @@ "project": true }, "io.grpc:grpc-api": { - "locked": "1.45.1", + "locked": "1.46.0", "transitive": [ "io.grpc:grpc-core", "io.grpc:grpc-protobuf", @@ -73,31 +73,31 @@ ] }, "io.grpc:grpc-context": { - "locked": "1.45.1", + "locked": "1.46.0", "transitive": [ "io.grpc:grpc-api" ] }, "io.grpc:grpc-core": { - "locked": "1.45.1", + "locked": "1.46.0", "transitive": [ "io.grpc:grpc-netty" ] }, "io.grpc:grpc-netty": { - "locked": "1.45.1" + "locked": "1.46.0" }, "io.grpc:grpc-protobuf": { - "locked": "1.45.1" + "locked": "1.46.0" }, "io.grpc:grpc-protobuf-lite": { - "locked": "1.45.1", + "locked": "1.46.0", "transitive": [ "io.grpc:grpc-protobuf" ] }, "io.grpc:grpc-stub": { - "locked": "1.45.1" + "locked": "1.46.0" }, "io.netty:netty-buffer": { "locked": "4.1.65.Final", @@ -224,12 +224,6 @@ "com.netflix.conductor:conductor-common" ] }, - "com.github.rholder:guava-retrying": { - "locked": "2.0.0", - "transitive": [ - "com.netflix.conductor:conductor-common" - ] - }, "com.google.android:annotations": { "locked": "4.1.1.4", "transitive": [ @@ -245,7 +239,6 @@ "com.google.code.findbugs:jsr305": { "locked": "3.0.2", "transitive": [ - "com.github.rholder:guava-retrying", "com.google.guava:guava", "io.grpc:grpc-api", "io.grpc:grpc-protobuf", @@ -277,7 +270,6 @@ "com.google.guava:guava": { "locked": "30.0-jre", "transitive": [ - "com.github.rholder:guava-retrying", "io.grpc:grpc-api", "io.grpc:grpc-core", "io.grpc:grpc-netty", @@ -323,7 +315,7 @@ "project": true }, "io.grpc:grpc-api": { - "locked": "1.45.1", + "locked": "1.46.0", "transitive": [ "io.grpc:grpc-core", "io.grpc:grpc-protobuf", @@ -332,34 +324,34 @@ ] }, "io.grpc:grpc-context": { - "locked": "1.45.1", + "locked": "1.46.0", "transitive": [ "io.grpc:grpc-api" ] }, "io.grpc:grpc-core": { - "locked": "1.45.1", + "locked": "1.46.0", "transitive": [ "io.grpc:grpc-netty" ] }, "io.grpc:grpc-netty": { - "locked": "1.45.1" + "locked": "1.46.0" }, "io.grpc:grpc-protobuf": { - "locked": "1.45.1", + "locked": "1.46.0", "transitive": [ "com.netflix.conductor:conductor-grpc" ] }, "io.grpc:grpc-protobuf-lite": { - "locked": "1.45.1", + "locked": "1.46.0", "transitive": [ "io.grpc:grpc-protobuf" ] }, "io.grpc:grpc-stub": { - "locked": "1.45.1", + "locked": "1.46.0", "transitive": [ "com.netflix.conductor:conductor-grpc" ] @@ -451,7 +443,7 @@ ] }, "io.perfmark:perfmark-api": { - "locked": "0.23.0", + "locked": "0.25.0", "transitive": [ "io.grpc:grpc-core", "io.grpc:grpc-netty" @@ -612,7 +604,7 @@ ] }, "io.grpc:grpc-api": { - "locked": "1.45.1", + "locked": "1.46.0", "transitive": [ "io.grpc:grpc-core", "io.grpc:grpc-protobuf", @@ -621,31 +613,31 @@ ] }, "io.grpc:grpc-context": { - "locked": "1.45.1", + "locked": "1.46.0", "transitive": [ "io.grpc:grpc-api" ] }, "io.grpc:grpc-core": { - "locked": "1.45.1", + "locked": "1.46.0", "transitive": [ "io.grpc:grpc-netty" ] }, "io.grpc:grpc-netty": { - "locked": "1.45.1" + "locked": "1.46.0" }, "io.grpc:grpc-protobuf": { - "locked": "1.45.1" + "locked": "1.46.0" }, "io.grpc:grpc-protobuf-lite": { - "locked": "1.45.1", + "locked": "1.46.0", "transitive": [ "io.grpc:grpc-protobuf" ] }, "io.grpc:grpc-stub": { - "locked": "1.45.1" + "locked": "1.46.0" }, "io.netty:netty-buffer": { "locked": "4.1.65.Final", @@ -1054,12 +1046,6 @@ "com.netflix.conductor:conductor-common" ] }, - "com.github.rholder:guava-retrying": { - "locked": "2.0.0", - "transitive": [ - "com.netflix.conductor:conductor-common" - ] - }, "com.google.android:annotations": { "locked": "4.1.1.4", "transitive": [ @@ -1075,7 +1061,6 @@ "com.google.code.findbugs:jsr305": { "locked": "3.0.2", "transitive": [ - "com.github.rholder:guava-retrying", "com.google.guava:guava", "io.grpc:grpc-api", "io.grpc:grpc-protobuf", @@ -1107,7 +1092,6 @@ "com.google.guava:guava": { "locked": "30.0-jre", "transitive": [ - "com.github.rholder:guava-retrying", "io.grpc:grpc-api", "io.grpc:grpc-core", "io.grpc:grpc-netty", @@ -1165,7 +1149,7 @@ ] }, "io.grpc:grpc-api": { - "locked": "1.45.1", + "locked": "1.46.0", "transitive": [ "io.grpc:grpc-core", "io.grpc:grpc-protobuf", @@ -1174,34 +1158,34 @@ ] }, "io.grpc:grpc-context": { - "locked": "1.45.1", + "locked": "1.46.0", "transitive": [ "io.grpc:grpc-api" ] }, "io.grpc:grpc-core": { - "locked": "1.45.1", + "locked": "1.46.0", "transitive": [ "io.grpc:grpc-netty" ] }, "io.grpc:grpc-netty": { - "locked": "1.45.1" + "locked": "1.46.0" }, "io.grpc:grpc-protobuf": { - "locked": "1.45.1", + "locked": "1.46.0", "transitive": [ "com.netflix.conductor:conductor-grpc" ] }, "io.grpc:grpc-protobuf-lite": { - "locked": "1.45.1", + "locked": "1.46.0", "transitive": [ "io.grpc:grpc-protobuf" ] }, "io.grpc:grpc-stub": { - "locked": "1.45.1", + "locked": "1.46.0", "transitive": [ "com.netflix.conductor:conductor-grpc" ] @@ -1293,7 +1277,7 @@ ] }, "io.perfmark:perfmark-api": { - "locked": "0.23.0", + "locked": "0.25.0", "transitive": [ "io.grpc:grpc-core", "io.grpc:grpc-netty" diff --git a/grpc-server/dependencies.lock b/grpc-server/dependencies.lock index 27198da0a1..31a913f66c 100644 --- a/grpc-server/dependencies.lock +++ b/grpc-server/dependencies.lock @@ -67,7 +67,7 @@ "project": true }, "io.grpc:grpc-api": { - "locked": "1.45.1", + "locked": "1.46.0", "transitive": [ "io.grpc:grpc-core", "io.grpc:grpc-protobuf", @@ -76,38 +76,38 @@ ] }, "io.grpc:grpc-context": { - "locked": "1.45.1", + "locked": "1.46.0", "transitive": [ "io.grpc:grpc-api" ] }, "io.grpc:grpc-core": { - "locked": "1.45.1", + "locked": "1.46.0", "transitive": [ "io.grpc:grpc-netty", "io.grpc:grpc-services" ] }, "io.grpc:grpc-netty": { - "locked": "1.45.1" + "locked": "1.46.0" }, "io.grpc:grpc-protobuf": { - "locked": "1.45.1", + "locked": "1.46.0", "transitive": [ "io.grpc:grpc-services" ] }, "io.grpc:grpc-protobuf-lite": { - "locked": "1.45.1", + "locked": "1.46.0", "transitive": [ "io.grpc:grpc-protobuf" ] }, "io.grpc:grpc-services": { - "locked": "1.45.1" + "locked": "1.46.0" }, "io.grpc:grpc-stub": { - "locked": "1.45.1", + "locked": "1.46.0", "transitive": [ "io.grpc:grpc-services" ] @@ -329,12 +329,6 @@ "com.netflix.conductor:conductor-core" ] }, - "com.github.rholder:guava-retrying": { - "locked": "2.0.0", - "transitive": [ - "com.netflix.conductor:conductor-common" - ] - }, "com.google.android:annotations": { "locked": "4.1.1.4", "transitive": [ @@ -350,7 +344,6 @@ "com.google.code.findbugs:jsr305": { "locked": "3.0.2", "transitive": [ - "com.github.rholder:guava-retrying", "com.google.guava:guava", "com.google.protobuf:protobuf-java-util", "io.grpc:grpc-api", @@ -386,9 +379,7 @@ "com.google.guava:guava": { "locked": "31.0.1-android", "transitive": [ - "com.github.rholder:guava-retrying", "com.google.protobuf:protobuf-java-util", - "com.netflix.conductor:conductor-core", "io.grpc:grpc-api", "io.grpc:grpc-core", "io.grpc:grpc-netty", @@ -472,7 +463,7 @@ ] }, "io.grpc:grpc-api": { - "locked": "1.45.1", + "locked": "1.46.0", "transitive": [ "io.grpc:grpc-core", "io.grpc:grpc-protobuf", @@ -481,39 +472,39 @@ ] }, "io.grpc:grpc-context": { - "locked": "1.45.1", + "locked": "1.46.0", "transitive": [ "io.grpc:grpc-api" ] }, "io.grpc:grpc-core": { - "locked": "1.45.1", + "locked": "1.46.0", "transitive": [ "io.grpc:grpc-netty", "io.grpc:grpc-services" ] }, "io.grpc:grpc-netty": { - "locked": "1.45.1" + "locked": "1.46.0" }, "io.grpc:grpc-protobuf": { - "locked": "1.45.1", + "locked": "1.46.0", "transitive": [ "com.netflix.conductor:conductor-grpc", "io.grpc:grpc-services" ] }, "io.grpc:grpc-protobuf-lite": { - "locked": "1.45.1", + "locked": "1.46.0", "transitive": [ "io.grpc:grpc-protobuf" ] }, "io.grpc:grpc-services": { - "locked": "1.45.1" + "locked": "1.46.0" }, "io.grpc:grpc-stub": { - "locked": "1.45.1", + "locked": "1.46.0", "transitive": [ "com.netflix.conductor:conductor-grpc", "io.grpc:grpc-services" @@ -606,7 +597,7 @@ ] }, "io.perfmark:perfmark-api": { - "locked": "0.23.0", + "locked": "0.25.0", "transitive": [ "io.grpc:grpc-core", "io.grpc:grpc-netty" @@ -828,7 +819,7 @@ ] }, "io.grpc:grpc-api": { - "locked": "1.45.1", + "locked": "1.46.0", "transitive": [ "io.grpc:grpc-core", "io.grpc:grpc-protobuf", @@ -837,13 +828,13 @@ ] }, "io.grpc:grpc-context": { - "locked": "1.45.1", + "locked": "1.46.0", "transitive": [ "io.grpc:grpc-api" ] }, "io.grpc:grpc-core": { - "locked": "1.45.1", + "locked": "1.46.0", "transitive": [ "io.grpc:grpc-netty", "io.grpc:grpc-services", @@ -851,32 +842,32 @@ ] }, "io.grpc:grpc-netty": { - "locked": "1.45.1" + "locked": "1.46.0" }, "io.grpc:grpc-protobuf": { - "locked": "1.45.1", + "locked": "1.46.0", "transitive": [ "io.grpc:grpc-services" ] }, "io.grpc:grpc-protobuf-lite": { - "locked": "1.45.1", + "locked": "1.46.0", "transitive": [ "io.grpc:grpc-protobuf" ] }, "io.grpc:grpc-services": { - "locked": "1.45.1" + "locked": "1.46.0" }, "io.grpc:grpc-stub": { - "locked": "1.45.1", + "locked": "1.46.0", "transitive": [ "io.grpc:grpc-services", "io.grpc:grpc-testing" ] }, "io.grpc:grpc-testing": { - "locked": "1.45.1" + "locked": "1.46.0" }, "io.netty:netty-buffer": { "locked": "4.1.65.Final", @@ -1371,12 +1362,6 @@ "com.netflix.conductor:conductor-core" ] }, - "com.github.rholder:guava-retrying": { - "locked": "2.0.0", - "transitive": [ - "com.netflix.conductor:conductor-common" - ] - }, "com.google.android:annotations": { "locked": "4.1.1.4", "transitive": [ @@ -1392,7 +1377,6 @@ "com.google.code.findbugs:jsr305": { "locked": "3.0.2", "transitive": [ - "com.github.rholder:guava-retrying", "com.google.guava:guava", "com.google.protobuf:protobuf-java-util", "io.grpc:grpc-api", @@ -1434,9 +1418,7 @@ "com.google.guava:guava": { "locked": "31.0.1-android", "transitive": [ - "com.github.rholder:guava-retrying", "com.google.protobuf:protobuf-java-util", - "com.netflix.conductor:conductor-core", "io.grpc:grpc-api", "io.grpc:grpc-core", "io.grpc:grpc-netty", @@ -1527,7 +1509,7 @@ ] }, "io.grpc:grpc-api": { - "locked": "1.45.1", + "locked": "1.46.0", "transitive": [ "io.grpc:grpc-core", "io.grpc:grpc-protobuf", @@ -1536,14 +1518,14 @@ ] }, "io.grpc:grpc-context": { - "locked": "1.45.1", + "locked": "1.46.0", "transitive": [ "io.grpc:grpc-api", "io.grpc:grpc-testing" ] }, "io.grpc:grpc-core": { - "locked": "1.45.1", + "locked": "1.46.0", "transitive": [ "io.grpc:grpc-netty", "io.grpc:grpc-services", @@ -1551,26 +1533,26 @@ ] }, "io.grpc:grpc-netty": { - "locked": "1.45.1" + "locked": "1.46.0" }, "io.grpc:grpc-protobuf": { - "locked": "1.45.1", + "locked": "1.46.0", "transitive": [ "com.netflix.conductor:conductor-grpc", "io.grpc:grpc-services" ] }, "io.grpc:grpc-protobuf-lite": { - "locked": "1.45.1", + "locked": "1.46.0", "transitive": [ "io.grpc:grpc-protobuf" ] }, "io.grpc:grpc-services": { - "locked": "1.45.1" + "locked": "1.46.0" }, "io.grpc:grpc-stub": { - "locked": "1.45.1", + "locked": "1.46.0", "transitive": [ "com.netflix.conductor:conductor-grpc", "io.grpc:grpc-services", @@ -1578,7 +1560,7 @@ ] }, "io.grpc:grpc-testing": { - "locked": "1.45.1" + "locked": "1.46.0" }, "io.netty:netty-buffer": { "locked": "4.1.65.Final", @@ -1667,7 +1649,7 @@ ] }, "io.perfmark:perfmark-api": { - "locked": "0.23.0", + "locked": "0.25.0", "transitive": [ "io.grpc:grpc-core", "io.grpc:grpc-netty" diff --git a/grpc/dependencies.lock b/grpc/dependencies.lock index a08a0ca560..67a1140636 100644 --- a/grpc/dependencies.lock +++ b/grpc/dependencies.lock @@ -61,7 +61,7 @@ "project": true }, "io.grpc:grpc-api": { - "locked": "1.45.1", + "locked": "1.46.0", "transitive": [ "io.grpc:grpc-protobuf", "io.grpc:grpc-protobuf-lite", @@ -69,22 +69,22 @@ ] }, "io.grpc:grpc-context": { - "locked": "1.45.1", + "locked": "1.46.0", "transitive": [ "io.grpc:grpc-api" ] }, "io.grpc:grpc-protobuf": { - "locked": "1.45.1" + "locked": "1.46.0" }, "io.grpc:grpc-protobuf-lite": { - "locked": "1.45.1", + "locked": "1.46.0", "transitive": [ "io.grpc:grpc-protobuf" ] }, "io.grpc:grpc-stub": { - "locked": "1.45.1" + "locked": "1.46.0" }, "javax.annotation:javax.annotation-api": { "locked": "1.3.2" @@ -152,12 +152,6 @@ "com.netflix.conductor:conductor-common" ] }, - "com.github.rholder:guava-retrying": { - "locked": "2.0.0", - "transitive": [ - "com.netflix.conductor:conductor-common" - ] - }, "com.google.api.grpc:proto-google-common-protos": { "locked": "2.0.1", "transitive": [ @@ -167,7 +161,6 @@ "com.google.code.findbugs:jsr305": { "locked": "3.0.2", "transitive": [ - "com.github.rholder:guava-retrying", "com.google.guava:guava", "io.grpc:grpc-api", "io.grpc:grpc-protobuf", @@ -191,7 +184,6 @@ "com.google.guava:guava": { "locked": "31.0.1-android", "transitive": [ - "com.github.rholder:guava-retrying", "io.grpc:grpc-api", "io.grpc:grpc-protobuf", "io.grpc:grpc-protobuf-lite", @@ -228,7 +220,7 @@ "project": true }, "io.grpc:grpc-api": { - "locked": "1.45.1", + "locked": "1.46.0", "transitive": [ "io.grpc:grpc-protobuf", "io.grpc:grpc-protobuf-lite", @@ -236,22 +228,22 @@ ] }, "io.grpc:grpc-context": { - "locked": "1.45.1", + "locked": "1.46.0", "transitive": [ "io.grpc:grpc-api" ] }, "io.grpc:grpc-protobuf": { - "locked": "1.45.1" + "locked": "1.46.0" }, "io.grpc:grpc-protobuf-lite": { - "locked": "1.45.1", + "locked": "1.46.0", "transitive": [ "io.grpc:grpc-protobuf" ] }, "io.grpc:grpc-stub": { - "locked": "1.45.1" + "locked": "1.46.0" }, "javax.annotation:javax.annotation-api": { "locked": "1.3.2" @@ -330,7 +322,7 @@ }, "protobufToolsLocator_grpc": { "io.grpc:protoc-gen-grpc-java": { - "locked": "1.45.1" + "locked": "1.46.0" } }, "protobufToolsLocator_protoc": { @@ -358,12 +350,6 @@ "com.netflix.conductor:conductor-common" ] }, - "com.github.rholder:guava-retrying": { - "locked": "2.0.0", - "transitive": [ - "com.netflix.conductor:conductor-common" - ] - }, "com.google.api.grpc:proto-google-common-protos": { "locked": "2.0.1", "transitive": [ @@ -373,7 +359,6 @@ "com.google.code.findbugs:jsr305": { "locked": "3.0.2", "transitive": [ - "com.github.rholder:guava-retrying", "com.google.guava:guava", "io.grpc:grpc-api", "io.grpc:grpc-protobuf", @@ -397,7 +382,6 @@ "com.google.guava:guava": { "locked": "31.0.1-android", "transitive": [ - "com.github.rholder:guava-retrying", "io.grpc:grpc-api", "io.grpc:grpc-protobuf", "io.grpc:grpc-protobuf-lite", @@ -434,7 +418,7 @@ "project": true }, "io.grpc:grpc-api": { - "locked": "1.45.1", + "locked": "1.46.0", "transitive": [ "io.grpc:grpc-protobuf", "io.grpc:grpc-protobuf-lite", @@ -442,22 +426,22 @@ ] }, "io.grpc:grpc-context": { - "locked": "1.45.1", + "locked": "1.46.0", "transitive": [ "io.grpc:grpc-api" ] }, "io.grpc:grpc-protobuf": { - "locked": "1.45.1" + "locked": "1.46.0" }, "io.grpc:grpc-protobuf-lite": { - "locked": "1.45.1", + "locked": "1.46.0", "transitive": [ "io.grpc:grpc-protobuf" ] }, "io.grpc:grpc-stub": { - "locked": "1.45.1" + "locked": "1.46.0" }, "javax.annotation:javax.annotation-api": { "locked": "1.3.2" @@ -603,7 +587,7 @@ ] }, "io.grpc:grpc-api": { - "locked": "1.45.1", + "locked": "1.46.0", "transitive": [ "io.grpc:grpc-protobuf", "io.grpc:grpc-protobuf-lite", @@ -611,22 +595,22 @@ ] }, "io.grpc:grpc-context": { - "locked": "1.45.1", + "locked": "1.46.0", "transitive": [ "io.grpc:grpc-api" ] }, "io.grpc:grpc-protobuf": { - "locked": "1.45.1" + "locked": "1.46.0" }, "io.grpc:grpc-protobuf-lite": { - "locked": "1.45.1", + "locked": "1.46.0", "transitive": [ "io.grpc:grpc-protobuf" ] }, "io.grpc:grpc-stub": { - "locked": "1.45.1" + "locked": "1.46.0" }, "jakarta.activation:jakarta.activation-api": { "locked": "1.2.2", @@ -976,12 +960,6 @@ "com.netflix.conductor:conductor-common" ] }, - "com.github.rholder:guava-retrying": { - "locked": "2.0.0", - "transitive": [ - "com.netflix.conductor:conductor-common" - ] - }, "com.google.api.grpc:proto-google-common-protos": { "locked": "2.0.1", "transitive": [ @@ -991,7 +969,6 @@ "com.google.code.findbugs:jsr305": { "locked": "3.0.2", "transitive": [ - "com.github.rholder:guava-retrying", "com.google.guava:guava", "io.grpc:grpc-api", "io.grpc:grpc-protobuf", @@ -1015,7 +992,6 @@ "com.google.guava:guava": { "locked": "31.0.1-android", "transitive": [ - "com.github.rholder:guava-retrying", "io.grpc:grpc-api", "io.grpc:grpc-protobuf", "io.grpc:grpc-protobuf-lite", @@ -1064,7 +1040,7 @@ ] }, "io.grpc:grpc-api": { - "locked": "1.45.1", + "locked": "1.46.0", "transitive": [ "io.grpc:grpc-protobuf", "io.grpc:grpc-protobuf-lite", @@ -1072,22 +1048,22 @@ ] }, "io.grpc:grpc-context": { - "locked": "1.45.1", + "locked": "1.46.0", "transitive": [ "io.grpc:grpc-api" ] }, "io.grpc:grpc-protobuf": { - "locked": "1.45.1" + "locked": "1.46.0" }, "io.grpc:grpc-protobuf-lite": { - "locked": "1.45.1", + "locked": "1.46.0", "transitive": [ "io.grpc:grpc-protobuf" ] }, "io.grpc:grpc-stub": { - "locked": "1.45.1" + "locked": "1.46.0" }, "jakarta.activation:jakarta.activation-api": { "locked": "1.2.2", @@ -1473,12 +1449,6 @@ "com.netflix.conductor:conductor-common" ] }, - "com.github.rholder:guava-retrying": { - "locked": "2.0.0", - "transitive": [ - "com.netflix.conductor:conductor-common" - ] - }, "com.google.api.grpc:proto-google-common-protos": { "locked": "2.0.1", "transitive": [ @@ -1488,7 +1458,6 @@ "com.google.code.findbugs:jsr305": { "locked": "3.0.2", "transitive": [ - "com.github.rholder:guava-retrying", "com.google.guava:guava", "io.grpc:grpc-api", "io.grpc:grpc-protobuf", @@ -1512,7 +1481,6 @@ "com.google.guava:guava": { "locked": "31.0.1-android", "transitive": [ - "com.github.rholder:guava-retrying", "io.grpc:grpc-api", "io.grpc:grpc-protobuf", "io.grpc:grpc-protobuf-lite", @@ -1561,7 +1529,7 @@ ] }, "io.grpc:grpc-api": { - "locked": "1.45.1", + "locked": "1.46.0", "transitive": [ "io.grpc:grpc-protobuf", "io.grpc:grpc-protobuf-lite", @@ -1569,22 +1537,22 @@ ] }, "io.grpc:grpc-context": { - "locked": "1.45.1", + "locked": "1.46.0", "transitive": [ "io.grpc:grpc-api" ] }, "io.grpc:grpc-protobuf": { - "locked": "1.45.1" + "locked": "1.46.0" }, "io.grpc:grpc-protobuf-lite": { - "locked": "1.45.1", + "locked": "1.46.0", "transitive": [ "io.grpc:grpc-protobuf" ] }, "io.grpc:grpc-stub": { - "locked": "1.45.1" + "locked": "1.46.0" }, "jakarta.activation:jakarta.activation-api": { "locked": "1.2.2", diff --git a/java-sdk/dependencies.lock b/java-sdk/dependencies.lock index da107df236..c2119c67ea 100644 --- a/java-sdk/dependencies.lock +++ b/java-sdk/dependencies.lock @@ -215,12 +215,6 @@ "com.github.vlsi.compactmap:compactmap" ] }, - "com.github.rholder:guava-retrying": { - "locked": "2.0.0", - "transitive": [ - "com.netflix.conductor:conductor-common" - ] - }, "com.github.vlsi.compactmap:compactmap": { "locked": "2.0", "transitive": [ @@ -230,7 +224,6 @@ "com.google.code.findbugs:jsr305": { "locked": "3.0.2", "transitive": [ - "com.github.rholder:guava-retrying", "com.google.guava:guava", "com.netflix.archaius:archaius-core", "com.netflix.eureka:eureka-client", @@ -258,10 +251,8 @@ "com.google.guava:guava": { "locked": "30.0-jre", "transitive": [ - "com.github.rholder:guava-retrying", "com.google.inject:guice", "com.netflix.archaius:archaius-core", - "com.netflix.conductor:conductor-client", "com.netflix.netflix-commons:netflix-infix", "com.netflix.servo:servo-core" ] @@ -1350,12 +1341,6 @@ "com.github.vlsi.compactmap:compactmap" ] }, - "com.github.rholder:guava-retrying": { - "locked": "2.0.0", - "transitive": [ - "com.netflix.conductor:conductor-common" - ] - }, "com.github.vlsi.compactmap:compactmap": { "locked": "2.0", "transitive": [ @@ -1365,7 +1350,6 @@ "com.google.code.findbugs:jsr305": { "locked": "3.0.2", "transitive": [ - "com.github.rholder:guava-retrying", "com.google.guava:guava", "com.netflix.archaius:archaius-core", "com.netflix.eureka:eureka-client", @@ -1393,10 +1377,8 @@ "com.google.guava:guava": { "locked": "30.0-jre", "transitive": [ - "com.github.rholder:guava-retrying", "com.google.inject:guice", "com.netflix.archaius:archaius-core", - "com.netflix.conductor:conductor-client", "com.netflix.netflix-commons:netflix-infix", "com.netflix.servo:servo-core" ] diff --git a/mysql-persistence/build.gradle b/mysql-persistence/build.gradle index 11cd9eb073..081da89612 100644 --- a/mysql-persistence/build.gradle +++ b/mysql-persistence/build.gradle @@ -15,6 +15,7 @@ dependencies { implementation project(':conductor-common') implementation project(':conductor-core') compileOnly 'org.springframework.boot:spring-boot-starter' + compileOnly 'org.springframework.retry:spring-retry' // SBMTODO: remove guava dep implementation "com.google.guava:guava:${revGuava}" @@ -28,6 +29,7 @@ dependencies { implementation "org.springframework.boot:spring-boot-starter-jdbc" implementation "org.flywaydb:flyway-core" + testImplementation 'org.springframework.retry:spring-retry' testImplementation "org.testcontainers:mysql:${revTestContainer}" testImplementation project(':conductor-core').sourceSets.test.output diff --git a/mysql-persistence/dependencies.lock b/mysql-persistence/dependencies.lock index d734c99a5f..22ac9d9e0f 100644 --- a/mysql-persistence/dependencies.lock +++ b/mysql-persistence/dependencies.lock @@ -158,6 +158,9 @@ "org.springframework.boot:spring-boot-starter" ] }, + "org.springframework.retry:spring-retry": { + "locked": "1.2.5.RELEASE" + }, "org.springframework:spring-aop": { "locked": "5.2.15.RELEASE", "transitive": [ @@ -184,6 +187,7 @@ "transitive": [ "org.springframework.boot:spring-boot", "org.springframework.boot:spring-boot-starter", + "org.springframework.retry:spring-retry", "org.springframework:spring-aop", "org.springframework:spring-beans", "org.springframework:spring-context", @@ -246,16 +250,9 @@ "com.netflix.conductor:conductor-core" ] }, - "com.github.rholder:guava-retrying": { - "locked": "2.0.0", - "transitive": [ - "com.netflix.conductor:conductor-common" - ] - }, "com.google.code.findbugs:jsr305": { "locked": "3.0.2", "transitive": [ - "com.github.rholder:guava-retrying", "com.google.guava:guava" ] }, @@ -272,11 +269,7 @@ ] }, "com.google.guava:guava": { - "locked": "30.0-jre", - "transitive": [ - "com.github.rholder:guava-retrying", - "com.netflix.conductor:conductor-core" - ] + "locked": "30.0-jre" }, "com.google.guava:listenablefuture": { "locked": "9999.0-empty-to-avoid-conflict-with-guava", @@ -964,6 +957,9 @@ "org.springframework.boot:spring-boot-starter-test" ] }, + "org.springframework.retry:spring-retry": { + "locked": "1.2.5.RELEASE" + }, "org.springframework:spring-aop": { "locked": "5.2.15.RELEASE", "transitive": [ @@ -991,6 +987,7 @@ "org.springframework.boot:spring-boot", "org.springframework.boot:spring-boot-starter", "org.springframework.boot:spring-boot-starter-test", + "org.springframework.retry:spring-retry", "org.springframework:spring-aop", "org.springframework:spring-beans", "org.springframework:spring-context", @@ -1106,16 +1103,9 @@ "org.testcontainers:testcontainers" ] }, - "com.github.rholder:guava-retrying": { - "locked": "2.0.0", - "transitive": [ - "com.netflix.conductor:conductor-common" - ] - }, "com.google.code.findbugs:jsr305": { "locked": "3.0.2", "transitive": [ - "com.github.rholder:guava-retrying", "com.google.guava:guava" ] }, @@ -1132,11 +1122,7 @@ ] }, "com.google.guava:guava": { - "locked": "30.0-jre", - "transitive": [ - "com.github.rholder:guava-retrying", - "com.netflix.conductor:conductor-core" - ] + "locked": "30.0-jre" }, "com.google.guava:listenablefuture": { "locked": "9999.0-empty-to-avoid-conflict-with-guava", @@ -1558,6 +1544,9 @@ "org.springframework.boot:spring-boot-starter-test" ] }, + "org.springframework.retry:spring-retry": { + "locked": "1.2.5.RELEASE" + }, "org.springframework:spring-aop": { "locked": "5.2.15.RELEASE", "transitive": [ @@ -1585,6 +1574,7 @@ "org.springframework.boot:spring-boot", "org.springframework.boot:spring-boot-starter", "org.springframework.boot:spring-boot-starter-test", + "org.springframework.retry:spring-retry", "org.springframework:spring-aop", "org.springframework:spring-beans", "org.springframework:spring-context", diff --git a/mysql-persistence/src/main/java/com/netflix/conductor/mysql/config/MySQLConfiguration.java b/mysql-persistence/src/main/java/com/netflix/conductor/mysql/config/MySQLConfiguration.java index d0ec108207..e45fc3b2bb 100644 --- a/mysql-persistence/src/main/java/com/netflix/conductor/mysql/config/MySQLConfiguration.java +++ b/mysql-persistence/src/main/java/com/netflix/conductor/mysql/config/MySQLConfiguration.java @@ -12,8 +12,12 @@ */ package com.netflix.conductor.mysql.config; +import java.sql.SQLException; +import java.util.Optional; + import javax.sql.DataSource; +import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; import org.springframework.boot.autoconfigure.jdbc.DataSourceAutoConfiguration; import org.springframework.boot.context.properties.EnableConfigurationProperties; @@ -21,6 +25,10 @@ import org.springframework.context.annotation.Configuration; import org.springframework.context.annotation.DependsOn; import org.springframework.context.annotation.Import; +import org.springframework.retry.RetryContext; +import org.springframework.retry.backoff.NoBackOffPolicy; +import org.springframework.retry.policy.SimpleRetryPolicy; +import org.springframework.retry.support.RetryTemplate; import com.netflix.conductor.mysql.dao.MySQLExecutionDAO; import com.netflix.conductor.mysql.dao.MySQLMetadataDAO; @@ -28,6 +36,8 @@ import com.fasterxml.jackson.databind.ObjectMapper; +import static com.mysql.cj.exceptions.MysqlErrorNumbers.ER_LOCK_DEADLOCK; + @Configuration(proxyBeanMethods = false) @EnableConfigurationProperties(MySQLProperties.class) @ConditionalOnProperty(name = "conductor.db.type", havingValue = "mysql") @@ -39,19 +49,67 @@ public class MySQLConfiguration { @Bean @DependsOn({"flyway", "flywayInitializer"}) public MySQLMetadataDAO mySqlMetadataDAO( - ObjectMapper objectMapper, DataSource dataSource, MySQLProperties properties) { - return new MySQLMetadataDAO(objectMapper, dataSource, properties); + @Qualifier("mysqlRetryTemplate") RetryTemplate retryTemplate, + ObjectMapper objectMapper, + DataSource dataSource, + MySQLProperties properties) { + return new MySQLMetadataDAO(retryTemplate, objectMapper, dataSource, properties); } @Bean @DependsOn({"flyway", "flywayInitializer"}) - public MySQLExecutionDAO mySqlExecutionDAO(ObjectMapper objectMapper, DataSource dataSource) { - return new MySQLExecutionDAO(objectMapper, dataSource); + public MySQLExecutionDAO mySqlExecutionDAO( + @Qualifier("mysqlRetryTemplate") RetryTemplate retryTemplate, + ObjectMapper objectMapper, + DataSource dataSource) { + return new MySQLExecutionDAO(retryTemplate, objectMapper, dataSource); } @Bean @DependsOn({"flyway", "flywayInitializer"}) - public MySQLQueueDAO mySqlQueueDAO(ObjectMapper objectMapper, DataSource dataSource) { - return new MySQLQueueDAO(objectMapper, dataSource); + public MySQLQueueDAO mySqlQueueDAO( + @Qualifier("mysqlRetryTemplate") RetryTemplate retryTemplate, + ObjectMapper objectMapper, + DataSource dataSource) { + return new MySQLQueueDAO(retryTemplate, objectMapper, dataSource); + } + + @Bean + public RetryTemplate mysqlRetryTemplate(MySQLProperties properties) { + SimpleRetryPolicy retryPolicy = new CustomRetryPolicy(); + retryPolicy.setMaxAttempts(properties.getDeadlockRetryMax()); + + RetryTemplate retryTemplate = new RetryTemplate(); + retryTemplate.setRetryPolicy(retryPolicy); + retryTemplate.setBackOffPolicy(new NoBackOffPolicy()); + return retryTemplate; + } + + public static class CustomRetryPolicy extends SimpleRetryPolicy { + + @Override + public boolean canRetry(final RetryContext context) { + final Optional lastThrowable = + Optional.ofNullable(context.getLastThrowable()); + return lastThrowable + .map(throwable -> super.canRetry(context) && isDeadLockError(throwable)) + .orElseGet(() -> super.canRetry(context)); + } + + private boolean isDeadLockError(Throwable throwable) { + SQLException sqlException = findCauseSQLException(throwable); + if (sqlException == null) { + return false; + } + return ER_LOCK_DEADLOCK == sqlException.getErrorCode(); + } + + private SQLException findCauseSQLException(Throwable throwable) { + Throwable causeException = throwable; + while (null != causeException && !(causeException instanceof SQLException)) { + causeException = causeException.getCause(); + } + return (SQLException) causeException; + } } } diff --git a/mysql-persistence/src/main/java/com/netflix/conductor/mysql/config/MySQLProperties.java b/mysql-persistence/src/main/java/com/netflix/conductor/mysql/config/MySQLProperties.java index dfef0eb787..42f9f74a3a 100644 --- a/mysql-persistence/src/main/java/com/netflix/conductor/mysql/config/MySQLProperties.java +++ b/mysql-persistence/src/main/java/com/netflix/conductor/mysql/config/MySQLProperties.java @@ -22,6 +22,8 @@ public class MySQLProperties { /** The time (in seconds) after which the in-memory task definitions cache will be refreshed */ private Duration taskDefCacheRefreshInterval = Duration.ofSeconds(60); + private Integer deadlockRetryMax = 3; + public Duration getTaskDefCacheRefreshInterval() { return taskDefCacheRefreshInterval; } @@ -29,4 +31,12 @@ public Duration getTaskDefCacheRefreshInterval() { public void setTaskDefCacheRefreshInterval(Duration taskDefCacheRefreshInterval) { this.taskDefCacheRefreshInterval = taskDefCacheRefreshInterval; } + + public Integer getDeadlockRetryMax() { + return deadlockRetryMax; + } + + public void setDeadlockRetryMax(Integer deadlockRetryMax) { + this.deadlockRetryMax = deadlockRetryMax; + } } diff --git a/mysql-persistence/src/main/java/com/netflix/conductor/mysql/dao/MySQLBaseDAO.java b/mysql-persistence/src/main/java/com/netflix/conductor/mysql/dao/MySQLBaseDAO.java index 7302e61b5e..d48b14ed2d 100644 --- a/mysql-persistence/src/main/java/com/netflix/conductor/mysql/dao/MySQLBaseDAO.java +++ b/mysql-persistence/src/main/java/com/netflix/conductor/mysql/dao/MySQLBaseDAO.java @@ -25,34 +25,20 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.springframework.retry.support.RetryTemplate; -import com.netflix.conductor.common.utils.RetryUtil; import com.netflix.conductor.core.exception.ApplicationException; -import com.netflix.conductor.mysql.util.ExecuteFunction; -import com.netflix.conductor.mysql.util.LazyToString; -import com.netflix.conductor.mysql.util.Query; -import com.netflix.conductor.mysql.util.QueryFunction; -import com.netflix.conductor.mysql.util.TransactionalFunction; +import com.netflix.conductor.mysql.util.*; import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.core.type.TypeReference; import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.collect.ImmutableList; -import static com.netflix.conductor.core.exception.ApplicationException.Code.BACKEND_ERROR; -import static com.netflix.conductor.core.exception.ApplicationException.Code.CONFLICT; -import static com.netflix.conductor.core.exception.ApplicationException.Code.INTERNAL_ERROR; - -import static com.mysql.cj.exceptions.MysqlErrorNumbers.ER_LOCK_DEADLOCK; -import static java.lang.Integer.parseInt; -import static java.lang.System.getProperty; +import static com.netflix.conductor.core.exception.ApplicationException.Code.*; public abstract class MySQLBaseDAO { - private static final String MAX_RETRY_ON_DEADLOCK_PROPERTY_NAME = - "conductor.mysql.deadlock.retry.max"; - private static final String MAX_RETRY_ON_DEADLOCK_PROPERTY_DEFAULT_VALUE = "3"; - private static final int MAX_RETRY_ON_DEADLOCK = getMaxRetriesOnDeadLock(); private static final List EXCLUDED_STACKTRACE_CLASS = ImmutableList.of(MySQLBaseDAO.class.getName(), Thread.class.getName()); @@ -60,8 +46,12 @@ public abstract class MySQLBaseDAO { protected final ObjectMapper objectMapper; protected final DataSource dataSource; - protected MySQLBaseDAO(ObjectMapper om, DataSource dataSource) { - this.objectMapper = om; + private final RetryTemplate retryTemplate; + + protected MySQLBaseDAO( + RetryTemplate retryTemplate, ObjectMapper objectMapper, DataSource dataSource) { + this.retryTemplate = retryTemplate; + this.objectMapper = objectMapper; this.dataSource = dataSource; } @@ -155,16 +145,9 @@ private R getWithTransaction(final TransactionalFunction function) { R getWithRetriedTransactions(final TransactionalFunction function) { try { - return new RetryUtil() - .retryOnException( - () -> getWithTransaction(function), - this::isDeadLockError, - null, - MAX_RETRY_ON_DEADLOCK, - "retry on deadlock", - "transactional"); - } catch (RuntimeException e) { - throw (ApplicationException) e.getCause(); + return retryTemplate.execute(context -> getWithTransaction(function)); + } catch (Exception e) { + throw (ApplicationException) e; } } @@ -272,31 +255,4 @@ protected void execute(Connection tx, String query, ExecuteFunction function) { protected void executeWithTransaction(String query, ExecuteFunction function) { withTransaction(tx -> execute(tx, query, function)); } - - private boolean isDeadLockError(Throwable throwable) { - SQLException sqlException = findCauseSQLException(throwable); - if (sqlException == null) { - return false; - } - return ER_LOCK_DEADLOCK == sqlException.getErrorCode(); - } - - private SQLException findCauseSQLException(Throwable throwable) { - Throwable causeException = throwable; - while (null != causeException && !(causeException instanceof SQLException)) { - causeException = causeException.getCause(); - } - return (SQLException) causeException; - } - - private static int getMaxRetriesOnDeadLock() { - try { - return parseInt( - getProperty( - MAX_RETRY_ON_DEADLOCK_PROPERTY_NAME, - MAX_RETRY_ON_DEADLOCK_PROPERTY_DEFAULT_VALUE)); - } catch (Exception e) { - return parseInt(MAX_RETRY_ON_DEADLOCK_PROPERTY_DEFAULT_VALUE); - } - } } diff --git a/mysql-persistence/src/main/java/com/netflix/conductor/mysql/dao/MySQLExecutionDAO.java b/mysql-persistence/src/main/java/com/netflix/conductor/mysql/dao/MySQLExecutionDAO.java index 9c50ad7d64..d4f0c54c4a 100644 --- a/mysql-persistence/src/main/java/com/netflix/conductor/mysql/dao/MySQLExecutionDAO.java +++ b/mysql-persistence/src/main/java/com/netflix/conductor/mysql/dao/MySQLExecutionDAO.java @@ -20,6 +20,8 @@ import javax.sql.DataSource; +import org.springframework.retry.support.RetryTemplate; + import com.netflix.conductor.common.metadata.events.EventExecution; import com.netflix.conductor.common.metadata.tasks.PollData; import com.netflix.conductor.common.metadata.tasks.TaskDef; @@ -43,11 +45,9 @@ public class MySQLExecutionDAO extends MySQLBaseDAO implements ExecutionDAO, RateLimitingDAO, PollDataDAO, ConcurrentExecutionLimitDAO { - private static final String ARCHIVED_FIELD = "archived"; - private static final String RAW_JSON_FIELD = "rawJSON"; - - public MySQLExecutionDAO(ObjectMapper objectMapper, DataSource dataSource) { - super(objectMapper, dataSource); + public MySQLExecutionDAO( + RetryTemplate retryTemplate, ObjectMapper objectMapper, DataSource dataSource) { + super(retryTemplate, objectMapper, dataSource); } private static String dateStr(Long timeInMs) { diff --git a/mysql-persistence/src/main/java/com/netflix/conductor/mysql/dao/MySQLMetadataDAO.java b/mysql-persistence/src/main/java/com/netflix/conductor/mysql/dao/MySQLMetadataDAO.java index 775c825887..ca26a51661 100644 --- a/mysql-persistence/src/main/java/com/netflix/conductor/mysql/dao/MySQLMetadataDAO.java +++ b/mysql-persistence/src/main/java/com/netflix/conductor/mysql/dao/MySQLMetadataDAO.java @@ -24,6 +24,8 @@ import javax.sql.DataSource; +import org.springframework.retry.support.RetryTemplate; + import com.netflix.conductor.common.metadata.events.EventHandler; import com.netflix.conductor.common.metadata.tasks.TaskDef; import com.netflix.conductor.common.metadata.workflow.WorkflowDef; @@ -42,8 +44,11 @@ public class MySQLMetadataDAO extends MySQLBaseDAO implements MetadataDAO, Event private static final String CLASS_NAME = MySQLMetadataDAO.class.getSimpleName(); public MySQLMetadataDAO( - ObjectMapper objectMapper, DataSource dataSource, MySQLProperties properties) { - super(objectMapper, dataSource); + RetryTemplate retryTemplate, + ObjectMapper objectMapper, + DataSource dataSource, + MySQLProperties properties) { + super(retryTemplate, objectMapper, dataSource); long cacheRefreshTime = properties.getTaskDefCacheRefreshInterval().getSeconds(); Executors.newSingleThreadScheduledExecutor() diff --git a/mysql-persistence/src/main/java/com/netflix/conductor/mysql/dao/MySQLQueueDAO.java b/mysql-persistence/src/main/java/com/netflix/conductor/mysql/dao/MySQLQueueDAO.java index e6164eab7a..cd068a228b 100644 --- a/mysql-persistence/src/main/java/com/netflix/conductor/mysql/dao/MySQLQueueDAO.java +++ b/mysql-persistence/src/main/java/com/netflix/conductor/mysql/dao/MySQLQueueDAO.java @@ -23,6 +23,8 @@ import javax.sql.DataSource; +import org.springframework.retry.support.RetryTemplate; + import com.netflix.conductor.core.events.queue.Message; import com.netflix.conductor.dao.QueueDAO; import com.netflix.conductor.mysql.util.Query; @@ -36,8 +38,9 @@ public class MySQLQueueDAO extends MySQLBaseDAO implements QueueDAO { private static final Long UNACK_SCHEDULE_MS = 60_000L; - public MySQLQueueDAO(ObjectMapper objectMapper, DataSource dataSource) { - super(objectMapper, dataSource); + public MySQLQueueDAO( + RetryTemplate retryTemplate, ObjectMapper objectMapper, DataSource dataSource) { + super(retryTemplate, objectMapper, dataSource); Executors.newSingleThreadScheduledExecutor() .scheduleAtFixedRate( diff --git a/postgres-external-storage/dependencies.lock b/postgres-external-storage/dependencies.lock index f0fa4f6570..97578c6a75 100644 --- a/postgres-external-storage/dependencies.lock +++ b/postgres-external-storage/dependencies.lock @@ -194,16 +194,16 @@ ] }, "org.springdoc:springdoc-openapi-common": { - "locked": "1.6.7", + "locked": "1.6.8", "transitive": [ "org.springdoc:springdoc-openapi-webmvc-core" ] }, "org.springdoc:springdoc-openapi-ui": { - "locked": "1.6.7" + "locked": "1.6.8" }, "org.springdoc:springdoc-openapi-webmvc-core": { - "locked": "1.6.7", + "locked": "1.6.8", "transitive": [ "org.springdoc:springdoc-openapi-ui" ] @@ -399,50 +399,6 @@ "io.swagger.core.v3:swagger-core" ] }, - "com.github.rholder:guava-retrying": { - "locked": "2.0.0", - "transitive": [ - "com.netflix.conductor:conductor-common" - ] - }, - "com.google.code.findbugs:jsr305": { - "locked": "3.0.2", - "transitive": [ - "com.github.rholder:guava-retrying", - "com.google.guava:guava" - ] - }, - "com.google.errorprone:error_prone_annotations": { - "locked": "2.3.4", - "transitive": [ - "com.google.guava:guava" - ] - }, - "com.google.guava:failureaccess": { - "locked": "1.0.1", - "transitive": [ - "com.google.guava:guava" - ] - }, - "com.google.guava:guava": { - "locked": "30.0-jre", - "transitive": [ - "com.github.rholder:guava-retrying", - "com.netflix.conductor:conductor-core" - ] - }, - "com.google.guava:listenablefuture": { - "locked": "9999.0-empty-to-avoid-conflict-with-guava", - "transitive": [ - "com.google.guava:guava" - ] - }, - "com.google.j2objc:j2objc-annotations": { - "locked": "1.3", - "transitive": [ - "com.google.guava:guava" - ] - }, "com.google.protobuf:protobuf-java": { "locked": "3.13.0", "transitive": [ @@ -627,7 +583,6 @@ "org.checkerframework:checker-qual": { "locked": "3.5.0", "transitive": [ - "com.google.guava:guava", "org.postgresql:postgresql" ] }, @@ -662,16 +617,16 @@ ] }, "org.springdoc:springdoc-openapi-common": { - "locked": "1.6.7", + "locked": "1.6.8", "transitive": [ "org.springdoc:springdoc-openapi-webmvc-core" ] }, "org.springdoc:springdoc-openapi-ui": { - "locked": "1.6.7" + "locked": "1.6.8" }, "org.springdoc:springdoc-openapi-webmvc-core": { - "locked": "1.6.7", + "locked": "1.6.8", "transitive": [ "org.springdoc:springdoc-openapi-ui" ] @@ -1202,16 +1157,16 @@ ] }, "org.springdoc:springdoc-openapi-common": { - "locked": "1.6.7", + "locked": "1.6.8", "transitive": [ "org.springdoc:springdoc-openapi-webmvc-core" ] }, "org.springdoc:springdoc-openapi-ui": { - "locked": "1.6.7" + "locked": "1.6.8" }, "org.springdoc:springdoc-openapi-webmvc-core": { - "locked": "1.6.7", + "locked": "1.6.8", "transitive": [ "org.springdoc:springdoc-openapi-ui" ] @@ -1502,50 +1457,6 @@ "org.testcontainers:testcontainers" ] }, - "com.github.rholder:guava-retrying": { - "locked": "2.0.0", - "transitive": [ - "com.netflix.conductor:conductor-common" - ] - }, - "com.google.code.findbugs:jsr305": { - "locked": "3.0.2", - "transitive": [ - "com.github.rholder:guava-retrying", - "com.google.guava:guava" - ] - }, - "com.google.errorprone:error_prone_annotations": { - "locked": "2.3.4", - "transitive": [ - "com.google.guava:guava" - ] - }, - "com.google.guava:failureaccess": { - "locked": "1.0.1", - "transitive": [ - "com.google.guava:guava" - ] - }, - "com.google.guava:guava": { - "locked": "30.0-jre", - "transitive": [ - "com.github.rholder:guava-retrying", - "com.netflix.conductor:conductor-core" - ] - }, - "com.google.guava:listenablefuture": { - "locked": "9999.0-empty-to-avoid-conflict-with-guava", - "transitive": [ - "com.google.guava:guava" - ] - }, - "com.google.j2objc:j2objc-annotations": { - "locked": "1.3", - "transitive": [ - "com.google.guava:guava" - ] - }, "com.google.protobuf:protobuf-java": { "locked": "3.13.0", "transitive": [ @@ -1804,7 +1715,6 @@ "org.checkerframework:checker-qual": { "locked": "3.5.0", "transitive": [ - "com.google.guava:guava", "org.postgresql:postgresql" ] }, @@ -1958,16 +1868,16 @@ ] }, "org.springdoc:springdoc-openapi-common": { - "locked": "1.6.7", + "locked": "1.6.8", "transitive": [ "org.springdoc:springdoc-openapi-webmvc-core" ] }, "org.springdoc:springdoc-openapi-ui": { - "locked": "1.6.7" + "locked": "1.6.8" }, "org.springdoc:springdoc-openapi-webmvc-core": { - "locked": "1.6.7", + "locked": "1.6.8", "transitive": [ "org.springdoc:springdoc-openapi-ui" ] diff --git a/postgres-persistence/build.gradle b/postgres-persistence/build.gradle index d07299bf9d..2b17febbb2 100644 --- a/postgres-persistence/build.gradle +++ b/postgres-persistence/build.gradle @@ -15,6 +15,7 @@ dependencies { implementation project(':conductor-common') implementation project(':conductor-core') compileOnly 'org.springframework.boot:spring-boot-starter' + compileOnly 'org.springframework.retry:spring-retry' // SBMTODO: remove guava dep implementation "com.google.guava:guava:${revGuava}" @@ -27,6 +28,7 @@ dependencies { implementation "org.springframework.boot:spring-boot-starter-jdbc" implementation "org.flywaydb:flyway-core" + testImplementation 'org.springframework.retry:spring-retry' testImplementation "org.testcontainers:postgresql:${revTestContainer}" testImplementation project(':conductor-core').sourceSets.test.output diff --git a/postgres-persistence/dependencies.lock b/postgres-persistence/dependencies.lock index f67423ebf0..11890d7c0d 100644 --- a/postgres-persistence/dependencies.lock +++ b/postgres-persistence/dependencies.lock @@ -152,6 +152,9 @@ "org.springframework.boot:spring-boot-starter" ] }, + "org.springframework.retry:spring-retry": { + "locked": "1.2.5.RELEASE" + }, "org.springframework:spring-aop": { "locked": "5.2.15.RELEASE", "transitive": [ @@ -178,6 +181,7 @@ "transitive": [ "org.springframework.boot:spring-boot", "org.springframework.boot:spring-boot-starter", + "org.springframework.retry:spring-retry", "org.springframework:spring-aop", "org.springframework:spring-beans", "org.springframework:spring-context", @@ -240,16 +244,9 @@ "com.netflix.conductor:conductor-core" ] }, - "com.github.rholder:guava-retrying": { - "locked": "2.0.0", - "transitive": [ - "com.netflix.conductor:conductor-common" - ] - }, "com.google.code.findbugs:jsr305": { "locked": "3.0.2", "transitive": [ - "com.github.rholder:guava-retrying", "com.google.guava:guava" ] }, @@ -266,11 +263,7 @@ ] }, "com.google.guava:guava": { - "locked": "30.0-jre", - "transitive": [ - "com.github.rholder:guava-retrying", - "com.netflix.conductor:conductor-core" - ] + "locked": "30.0-jre" }, "com.google.guava:listenablefuture": { "locked": "9999.0-empty-to-avoid-conflict-with-guava", @@ -952,6 +945,9 @@ "org.springframework.boot:spring-boot-starter-test" ] }, + "org.springframework.retry:spring-retry": { + "locked": "1.2.5.RELEASE" + }, "org.springframework:spring-aop": { "locked": "5.2.15.RELEASE", "transitive": [ @@ -979,6 +975,7 @@ "org.springframework.boot:spring-boot", "org.springframework.boot:spring-boot-starter", "org.springframework.boot:spring-boot-starter-test", + "org.springframework.retry:spring-retry", "org.springframework:spring-aop", "org.springframework:spring-beans", "org.springframework:spring-context", @@ -1094,16 +1091,9 @@ "org.testcontainers:testcontainers" ] }, - "com.github.rholder:guava-retrying": { - "locked": "2.0.0", - "transitive": [ - "com.netflix.conductor:conductor-common" - ] - }, "com.google.code.findbugs:jsr305": { "locked": "3.0.2", "transitive": [ - "com.github.rholder:guava-retrying", "com.google.guava:guava" ] }, @@ -1120,11 +1110,7 @@ ] }, "com.google.guava:guava": { - "locked": "30.0-jre", - "transitive": [ - "com.github.rholder:guava-retrying", - "com.netflix.conductor:conductor-core" - ] + "locked": "30.0-jre" }, "com.google.guava:listenablefuture": { "locked": "9999.0-empty-to-avoid-conflict-with-guava", @@ -1546,6 +1532,9 @@ "org.springframework.boot:spring-boot-starter-test" ] }, + "org.springframework.retry:spring-retry": { + "locked": "1.2.5.RELEASE" + }, "org.springframework:spring-aop": { "locked": "5.2.15.RELEASE", "transitive": [ @@ -1573,6 +1562,7 @@ "org.springframework.boot:spring-boot", "org.springframework.boot:spring-boot-starter", "org.springframework.boot:spring-boot-starter-test", + "org.springframework.retry:spring-retry", "org.springframework:spring-aop", "org.springframework:spring-beans", "org.springframework:spring-context", diff --git a/postgres-persistence/src/main/java/com/netflix/conductor/postgres/config/PostgresConfiguration.java b/postgres-persistence/src/main/java/com/netflix/conductor/postgres/config/PostgresConfiguration.java index c1b9e533d7..bb86096e52 100644 --- a/postgres-persistence/src/main/java/com/netflix/conductor/postgres/config/PostgresConfiguration.java +++ b/postgres-persistence/src/main/java/com/netflix/conductor/postgres/config/PostgresConfiguration.java @@ -12,10 +12,14 @@ */ package com.netflix.conductor.postgres.config; +import java.sql.SQLException; +import java.util.Optional; + import javax.annotation.PostConstruct; import javax.sql.DataSource; import org.flywaydb.core.Flyway; +import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; import org.springframework.boot.autoconfigure.jdbc.DataSourceAutoConfiguration; import org.springframework.boot.context.properties.EnableConfigurationProperties; @@ -23,6 +27,10 @@ import org.springframework.context.annotation.Configuration; import org.springframework.context.annotation.DependsOn; import org.springframework.context.annotation.Import; +import org.springframework.retry.RetryContext; +import org.springframework.retry.backoff.NoBackOffPolicy; +import org.springframework.retry.policy.SimpleRetryPolicy; +import org.springframework.retry.support.RetryTemplate; import com.netflix.conductor.postgres.dao.PostgresExecutionDAO; import com.netflix.conductor.postgres.dao.PostgresMetadataDAO; @@ -58,19 +66,68 @@ public Flyway flywayForPrimaryDb() { @Bean @DependsOn({"flywayForPrimaryDb"}) public PostgresMetadataDAO postgresMetadataDAO( - ObjectMapper objectMapper, PostgresProperties properties) { - return new PostgresMetadataDAO(objectMapper, dataSource, properties); + @Qualifier("postgresRetryTemplate") RetryTemplate retryTemplate, + ObjectMapper objectMapper, + PostgresProperties properties) { + return new PostgresMetadataDAO(retryTemplate, objectMapper, dataSource, properties); } @Bean @DependsOn({"flywayForPrimaryDb"}) - public PostgresExecutionDAO postgresExecutionDAO(ObjectMapper objectMapper) { - return new PostgresExecutionDAO(objectMapper, dataSource); + public PostgresExecutionDAO postgresExecutionDAO( + @Qualifier("postgresRetryTemplate") RetryTemplate retryTemplate, + ObjectMapper objectMapper) { + return new PostgresExecutionDAO(retryTemplate, objectMapper, dataSource); } @Bean @DependsOn({"flywayForPrimaryDb"}) - public PostgresQueueDAO postgresQueueDAO(ObjectMapper objectMapper) { - return new PostgresQueueDAO(objectMapper, dataSource); + public PostgresQueueDAO postgresQueueDAO( + @Qualifier("postgresRetryTemplate") RetryTemplate retryTemplate, + ObjectMapper objectMapper) { + return new PostgresQueueDAO(retryTemplate, objectMapper, dataSource); + } + + @Bean + public RetryTemplate postgresRetryTemplate(PostgresProperties properties) { + SimpleRetryPolicy retryPolicy = new CustomRetryPolicy(); + retryPolicy.setMaxAttempts(3); + + RetryTemplate retryTemplate = new RetryTemplate(); + retryTemplate.setRetryPolicy(retryPolicy); + retryTemplate.setBackOffPolicy(new NoBackOffPolicy()); + return retryTemplate; + } + + public static class CustomRetryPolicy extends SimpleRetryPolicy { + + private static final String ER_LOCK_DEADLOCK = "40P01"; + private static final String ER_SERIALIZATION_FAILURE = "40001"; + + @Override + public boolean canRetry(final RetryContext context) { + final Optional lastThrowable = + Optional.ofNullable(context.getLastThrowable()); + return lastThrowable + .map(throwable -> super.canRetry(context) && isDeadLockError(throwable)) + .orElseGet(() -> super.canRetry(context)); + } + + private boolean isDeadLockError(Throwable throwable) { + SQLException sqlException = findCauseSQLException(throwable); + if (sqlException == null) { + return false; + } + return ER_LOCK_DEADLOCK.equals(sqlException.getSQLState()) + || ER_SERIALIZATION_FAILURE.equals(sqlException.getSQLState()); + } + + private SQLException findCauseSQLException(Throwable throwable) { + Throwable causeException = throwable; + while (null != causeException && !(causeException instanceof SQLException)) { + causeException = causeException.getCause(); + } + return (SQLException) causeException; + } } } diff --git a/postgres-persistence/src/main/java/com/netflix/conductor/postgres/config/PostgresProperties.java b/postgres-persistence/src/main/java/com/netflix/conductor/postgres/config/PostgresProperties.java index 93f0442d0a..3226b5d17f 100644 --- a/postgres-persistence/src/main/java/com/netflix/conductor/postgres/config/PostgresProperties.java +++ b/postgres-persistence/src/main/java/com/netflix/conductor/postgres/config/PostgresProperties.java @@ -25,6 +25,8 @@ public class PostgresProperties { @DurationUnit(ChronoUnit.SECONDS) private Duration taskDefCacheRefreshInterval = Duration.ofSeconds(60); + private Integer deadlockRetryMax = 3; + public Duration getTaskDefCacheRefreshInterval() { return taskDefCacheRefreshInterval; } @@ -32,4 +34,12 @@ public Duration getTaskDefCacheRefreshInterval() { public void setTaskDefCacheRefreshInterval(Duration taskDefCacheRefreshInterval) { this.taskDefCacheRefreshInterval = taskDefCacheRefreshInterval; } + + public Integer getDeadlockRetryMax() { + return deadlockRetryMax; + } + + public void setDeadlockRetryMax(Integer deadlockRetryMax) { + this.deadlockRetryMax = deadlockRetryMax; + } } diff --git a/postgres-persistence/src/main/java/com/netflix/conductor/postgres/dao/PostgresBaseDAO.java b/postgres-persistence/src/main/java/com/netflix/conductor/postgres/dao/PostgresBaseDAO.java index 173cad887b..74fb9abc11 100644 --- a/postgres-persistence/src/main/java/com/netflix/conductor/postgres/dao/PostgresBaseDAO.java +++ b/postgres-persistence/src/main/java/com/netflix/conductor/postgres/dao/PostgresBaseDAO.java @@ -25,35 +25,20 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.springframework.retry.support.RetryTemplate; -import com.netflix.conductor.common.utils.RetryUtil; import com.netflix.conductor.core.exception.ApplicationException; -import com.netflix.conductor.postgres.util.ExecuteFunction; -import com.netflix.conductor.postgres.util.LazyToString; -import com.netflix.conductor.postgres.util.Query; -import com.netflix.conductor.postgres.util.QueryFunction; -import com.netflix.conductor.postgres.util.TransactionalFunction; +import com.netflix.conductor.postgres.util.*; import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.core.type.TypeReference; import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.collect.ImmutableList; -import static com.netflix.conductor.core.exception.ApplicationException.Code.BACKEND_ERROR; -import static com.netflix.conductor.core.exception.ApplicationException.Code.CONFLICT; -import static com.netflix.conductor.core.exception.ApplicationException.Code.INTERNAL_ERROR; - -import static java.lang.Integer.parseInt; -import static java.lang.System.getProperty; +import static com.netflix.conductor.core.exception.ApplicationException.Code.*; public abstract class PostgresBaseDAO { - private static final String ER_LOCK_DEADLOCK = "40P01"; - private static final String ER_SERIALIZATION_FAILURE = "40001"; - private static final String MAX_RETRY_ON_DEADLOCK_PROPERTY_NAME = - "conductor.postgres.deadlock.retry.max"; - private static final String MAX_RETRY_ON_DEADLOCK_PROPERTY_DEFAULT_VALUE = "3"; - private static final int MAX_RETRY_ON_DEADLOCK = getMaxRetriesOnDeadLock(); private static final List EXCLUDED_STACKTRACE_CLASS = ImmutableList.of(PostgresBaseDAO.class.getName(), Thread.class.getName()); @@ -61,7 +46,11 @@ public abstract class PostgresBaseDAO { protected final ObjectMapper objectMapper; protected final DataSource dataSource; - protected PostgresBaseDAO(ObjectMapper objectMapper, DataSource dataSource) { + private final RetryTemplate retryTemplate; + + protected PostgresBaseDAO( + RetryTemplate retryTemplate, ObjectMapper objectMapper, DataSource dataSource) { + this.retryTemplate = retryTemplate; this.objectMapper = objectMapper; this.dataSource = dataSource; } @@ -156,16 +145,9 @@ private R getWithTransaction(final TransactionalFunction function) { R getWithRetriedTransactions(final TransactionalFunction function) { try { - return new RetryUtil() - .retryOnException( - () -> getWithTransaction(function), - this::isDeadLockError, - null, - MAX_RETRY_ON_DEADLOCK, - "retry on deadlock", - "transactional"); - } catch (RuntimeException e) { - throw (ApplicationException) e.getCause(); + return retryTemplate.execute(context -> getWithTransaction(function)); + } catch (Exception e) { + throw (ApplicationException) e; } } @@ -273,32 +255,4 @@ protected void execute(Connection tx, String query, ExecuteFunction function) { protected void executeWithTransaction(String query, ExecuteFunction function) { withTransaction(tx -> execute(tx, query, function)); } - - private boolean isDeadLockError(Throwable throwable) { - SQLException sqlException = findCauseSQLException(throwable); - if (sqlException == null) { - return false; - } - return ER_LOCK_DEADLOCK.equals(sqlException.getSQLState()) - || ER_SERIALIZATION_FAILURE.equals(sqlException.getSQLState()); - } - - private SQLException findCauseSQLException(Throwable throwable) { - Throwable causeException = throwable; - while (null != causeException && !(causeException instanceof SQLException)) { - causeException = causeException.getCause(); - } - return (SQLException) causeException; - } - - private static int getMaxRetriesOnDeadLock() { - try { - return parseInt( - getProperty( - MAX_RETRY_ON_DEADLOCK_PROPERTY_NAME, - MAX_RETRY_ON_DEADLOCK_PROPERTY_DEFAULT_VALUE)); - } catch (Exception e) { - return parseInt(MAX_RETRY_ON_DEADLOCK_PROPERTY_DEFAULT_VALUE); - } - } } diff --git a/postgres-persistence/src/main/java/com/netflix/conductor/postgres/dao/PostgresExecutionDAO.java b/postgres-persistence/src/main/java/com/netflix/conductor/postgres/dao/PostgresExecutionDAO.java index 510563d363..df90aae8bd 100644 --- a/postgres-persistence/src/main/java/com/netflix/conductor/postgres/dao/PostgresExecutionDAO.java +++ b/postgres-persistence/src/main/java/com/netflix/conductor/postgres/dao/PostgresExecutionDAO.java @@ -21,6 +21,8 @@ import javax.sql.DataSource; +import org.springframework.retry.support.RetryTemplate; + import com.netflix.conductor.common.metadata.events.EventExecution; import com.netflix.conductor.common.metadata.tasks.PollData; import com.netflix.conductor.common.metadata.tasks.TaskDef; @@ -47,8 +49,9 @@ public class PostgresExecutionDAO extends PostgresBaseDAO private static final String ARCHIVED_FIELD = "archived"; private static final String RAW_JSON_FIELD = "rawJSON"; - public PostgresExecutionDAO(ObjectMapper objectMapper, DataSource dataSource) { - super(objectMapper, dataSource); + public PostgresExecutionDAO( + RetryTemplate retryTemplate, ObjectMapper objectMapper, DataSource dataSource) { + super(retryTemplate, objectMapper, dataSource); } private static String dateStr(Long timeInMs) { diff --git a/postgres-persistence/src/main/java/com/netflix/conductor/postgres/dao/PostgresMetadataDAO.java b/postgres-persistence/src/main/java/com/netflix/conductor/postgres/dao/PostgresMetadataDAO.java index b461a399e1..ffe752e4b2 100644 --- a/postgres-persistence/src/main/java/com/netflix/conductor/postgres/dao/PostgresMetadataDAO.java +++ b/postgres-persistence/src/main/java/com/netflix/conductor/postgres/dao/PostgresMetadataDAO.java @@ -13,17 +13,15 @@ package com.netflix.conductor.postgres.dao; import java.sql.Connection; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Optional; +import java.util.*; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.Executors; import java.util.concurrent.TimeUnit; import javax.sql.DataSource; +import org.springframework.retry.support.RetryTemplate; + import com.netflix.conductor.common.metadata.events.EventHandler; import com.netflix.conductor.common.metadata.tasks.TaskDef; import com.netflix.conductor.common.metadata.workflow.WorkflowDef; @@ -42,8 +40,11 @@ public class PostgresMetadataDAO extends PostgresBaseDAO implements MetadataDAO, private static final String CLASS_NAME = PostgresMetadataDAO.class.getSimpleName(); public PostgresMetadataDAO( - ObjectMapper objectMapper, DataSource dataSource, PostgresProperties properties) { - super(objectMapper, dataSource); + RetryTemplate retryTemplate, + ObjectMapper objectMapper, + DataSource dataSource, + PostgresProperties properties) { + super(retryTemplate, objectMapper, dataSource); long cacheRefreshTime = properties.getTaskDefCacheRefreshInterval().getSeconds(); Executors.newSingleThreadScheduledExecutor() diff --git a/postgres-persistence/src/main/java/com/netflix/conductor/postgres/dao/PostgresQueueDAO.java b/postgres-persistence/src/main/java/com/netflix/conductor/postgres/dao/PostgresQueueDAO.java index 8b4175f653..ed95b3433d 100644 --- a/postgres-persistence/src/main/java/com/netflix/conductor/postgres/dao/PostgresQueueDAO.java +++ b/postgres-persistence/src/main/java/com/netflix/conductor/postgres/dao/PostgresQueueDAO.java @@ -13,17 +13,15 @@ package com.netflix.conductor.postgres.dao; import java.sql.Connection; -import java.util.ArrayList; -import java.util.Collections; -import java.util.HashMap; -import java.util.List; -import java.util.Map; +import java.util.*; import java.util.concurrent.Executors; import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; import javax.sql.DataSource; +import org.springframework.retry.support.RetryTemplate; + import com.netflix.conductor.core.events.queue.Message; import com.netflix.conductor.dao.QueueDAO; import com.netflix.conductor.postgres.util.Query; @@ -37,8 +35,9 @@ public class PostgresQueueDAO extends PostgresBaseDAO implements QueueDAO { private static final Long UNACK_SCHEDULE_MS = 60_000L; - public PostgresQueueDAO(ObjectMapper om, DataSource ds) { - super(om, ds); + public PostgresQueueDAO( + RetryTemplate retryTemplate, ObjectMapper objectMapper, DataSource dataSource) { + super(retryTemplate, objectMapper, dataSource); Executors.newSingleThreadScheduledExecutor() .scheduleAtFixedRate( diff --git a/redis-concurrency-limit/dependencies.lock b/redis-concurrency-limit/dependencies.lock index 9ff784141e..e5a8166299 100644 --- a/redis-concurrency-limit/dependencies.lock +++ b/redis-concurrency-limit/dependencies.lock @@ -209,50 +209,6 @@ "com.netflix.conductor:conductor-core" ] }, - "com.github.rholder:guava-retrying": { - "locked": "2.0.0", - "transitive": [ - "com.netflix.conductor:conductor-common" - ] - }, - "com.google.code.findbugs:jsr305": { - "locked": "3.0.2", - "transitive": [ - "com.github.rholder:guava-retrying", - "com.google.guava:guava" - ] - }, - "com.google.errorprone:error_prone_annotations": { - "locked": "2.3.4", - "transitive": [ - "com.google.guava:guava" - ] - }, - "com.google.guava:failureaccess": { - "locked": "1.0.1", - "transitive": [ - "com.google.guava:guava" - ] - }, - "com.google.guava:guava": { - "locked": "30.0-jre", - "transitive": [ - "com.github.rholder:guava-retrying", - "com.netflix.conductor:conductor-core" - ] - }, - "com.google.guava:listenablefuture": { - "locked": "9999.0-empty-to-avoid-conflict-with-guava", - "transitive": [ - "com.google.guava:guava" - ] - }, - "com.google.j2objc:j2objc-annotations": { - "locked": "1.3", - "transitive": [ - "com.google.guava:guava" - ] - }, "com.google.protobuf:protobuf-java": { "locked": "3.13.0", "transitive": [ @@ -396,12 +352,6 @@ "com.netflix.conductor:conductor-core" ] }, - "org.checkerframework:checker-qual": { - "locked": "3.5.0", - "transitive": [ - "com.google.guava:guava" - ] - }, "org.ow2.asm:asm": { "locked": "5.0.4", "transitive": [ @@ -1176,50 +1126,6 @@ "org.testcontainers:testcontainers" ] }, - "com.github.rholder:guava-retrying": { - "locked": "2.0.0", - "transitive": [ - "com.netflix.conductor:conductor-common" - ] - }, - "com.google.code.findbugs:jsr305": { - "locked": "3.0.2", - "transitive": [ - "com.github.rholder:guava-retrying", - "com.google.guava:guava" - ] - }, - "com.google.errorprone:error_prone_annotations": { - "locked": "2.3.4", - "transitive": [ - "com.google.guava:guava" - ] - }, - "com.google.guava:failureaccess": { - "locked": "1.0.1", - "transitive": [ - "com.google.guava:guava" - ] - }, - "com.google.guava:guava": { - "locked": "30.0-jre", - "transitive": [ - "com.github.rholder:guava-retrying", - "com.netflix.conductor:conductor-core" - ] - }, - "com.google.guava:listenablefuture": { - "locked": "9999.0-empty-to-avoid-conflict-with-guava", - "transitive": [ - "com.google.guava:guava" - ] - }, - "com.google.j2objc:j2objc-annotations": { - "locked": "1.3", - "transitive": [ - "com.google.guava:guava" - ] - }, "com.google.protobuf:protobuf-java": { "locked": "3.13.0", "transitive": [ @@ -1481,12 +1387,6 @@ "org.springframework.boot:spring-boot-starter-test" ] }, - "org.checkerframework:checker-qual": { - "locked": "3.5.0", - "transitive": [ - "com.google.guava:guava" - ] - }, "org.codehaus.groovy:groovy": { "locked": "2.5.14", "transitive": [ diff --git a/redis-lock/dependencies.lock b/redis-lock/dependencies.lock index 42284995e6..e22ef46e5e 100644 --- a/redis-lock/dependencies.lock +++ b/redis-lock/dependencies.lock @@ -316,50 +316,6 @@ "org.redisson:redisson" ] }, - "com.github.rholder:guava-retrying": { - "locked": "2.0.0", - "transitive": [ - "com.netflix.conductor:conductor-common" - ] - }, - "com.google.code.findbugs:jsr305": { - "locked": "3.0.2", - "transitive": [ - "com.github.rholder:guava-retrying", - "com.google.guava:guava" - ] - }, - "com.google.errorprone:error_prone_annotations": { - "locked": "2.3.4", - "transitive": [ - "com.google.guava:guava" - ] - }, - "com.google.guava:failureaccess": { - "locked": "1.0.1", - "transitive": [ - "com.google.guava:guava" - ] - }, - "com.google.guava:guava": { - "locked": "30.0-jre", - "transitive": [ - "com.github.rholder:guava-retrying", - "com.netflix.conductor:conductor-core" - ] - }, - "com.google.guava:listenablefuture": { - "locked": "9999.0-empty-to-avoid-conflict-with-guava", - "transitive": [ - "com.google.guava:guava" - ] - }, - "com.google.j2objc:j2objc-annotations": { - "locked": "1.3", - "transitive": [ - "com.google.guava:guava" - ] - }, "com.google.protobuf:protobuf-java": { "locked": "3.13.0", "transitive": [ @@ -591,12 +547,6 @@ "com.netflix.conductor:conductor-core" ] }, - "org.checkerframework:checker-qual": { - "locked": "3.5.0", - "transitive": [ - "com.google.guava:guava" - ] - }, "org.jboss.marshalling:jboss-marshalling": { "locked": "2.0.9.Final", "transitive": [ @@ -1190,49 +1140,10 @@ "com.github.kstyrc:embedded-redis": { "locked": "0.6" }, - "com.github.rholder:guava-retrying": { - "locked": "2.0.0", - "transitive": [ - "com.netflix.conductor:conductor-common" - ] - }, - "com.google.code.findbugs:jsr305": { - "locked": "3.0.2", - "transitive": [ - "com.github.rholder:guava-retrying", - "com.google.guava:guava" - ] - }, - "com.google.errorprone:error_prone_annotations": { - "locked": "2.3.4", - "transitive": [ - "com.google.guava:guava" - ] - }, - "com.google.guava:failureaccess": { - "locked": "1.0.1", - "transitive": [ - "com.google.guava:guava" - ] - }, "com.google.guava:guava": { - "locked": "30.0-jre", - "transitive": [ - "com.github.kstyrc:embedded-redis", - "com.github.rholder:guava-retrying", - "com.netflix.conductor:conductor-core" - ] - }, - "com.google.guava:listenablefuture": { - "locked": "9999.0-empty-to-avoid-conflict-with-guava", - "transitive": [ - "com.google.guava:guava" - ] - }, - "com.google.j2objc:j2objc-annotations": { - "locked": "1.3", + "locked": "18.0", "transitive": [ - "com.google.guava:guava" + "com.github.kstyrc:embedded-redis" ] }, "com.google.protobuf:protobuf-java": { @@ -1514,12 +1425,6 @@ "org.springframework.boot:spring-boot-starter-test" ] }, - "org.checkerframework:checker-qual": { - "locked": "3.5.0", - "transitive": [ - "com.google.guava:guava" - ] - }, "org.hamcrest:hamcrest": { "locked": "2.2", "transitive": [ diff --git a/redis-persistence/dependencies.lock b/redis-persistence/dependencies.lock index 29cc6ed6d1..6ea21f96f1 100644 --- a/redis-persistence/dependencies.lock +++ b/redis-persistence/dependencies.lock @@ -455,12 +455,6 @@ "com.github.vlsi.compactmap:compactmap" ] }, - "com.github.rholder:guava-retrying": { - "locked": "2.0.0", - "transitive": [ - "com.netflix.conductor:conductor-common" - ] - }, "com.github.vlsi.compactmap:compactmap": { "locked": "1.2.1", "transitive": [ @@ -468,10 +462,8 @@ ] }, "com.google.code.findbugs:jsr305": { - "locked": "3.0.2", + "locked": "3.0.1", "transitive": [ - "com.github.rholder:guava-retrying", - "com.google.guava:guava", "com.netflix.archaius:archaius-core", "com.netflix.netflix-commons:netflix-infix" ] @@ -484,35 +476,15 @@ "com.netflix.netflix-commons:netflix-infix" ] }, - "com.google.errorprone:error_prone_annotations": { - "locked": "2.3.4", - "transitive": [ - "com.google.guava:guava" - ] - }, - "com.google.guava:failureaccess": { - "locked": "1.0.1", - "transitive": [ - "com.google.guava:guava" - ] - }, "com.google.guava:guava": { - "locked": "30.0-jre", + "locked": "19.0", "transitive": [ - "com.github.rholder:guava-retrying", "com.google.inject:guice", "com.netflix.archaius:archaius-core", - "com.netflix.conductor:conductor-core", "com.netflix.netflix-commons:netflix-infix", "com.netflix.servo:servo-core" ] }, - "com.google.guava:listenablefuture": { - "locked": "9999.0-empty-to-avoid-conflict-with-guava", - "transitive": [ - "com.google.guava:guava" - ] - }, "com.google.inject:guice": { "locked": "4.1.0", "transitive": [ @@ -521,12 +493,6 @@ "com.netflix.eureka:eureka-client" ] }, - "com.google.j2objc:j2objc-annotations": { - "locked": "1.3", - "transitive": [ - "com.google.guava:guava" - ] - }, "com.google.protobuf:protobuf-java": { "locked": "3.13.0", "transitive": [ @@ -920,12 +886,6 @@ "com.netflix.conductor:conductor-core" ] }, - "org.checkerframework:checker-qual": { - "locked": "3.5.0", - "transitive": [ - "com.google.guava:guava" - ] - }, "org.codehaus.jettison:jettison": { "locked": "1.3.7", "transitive": [ @@ -1654,12 +1614,6 @@ "com.github.vlsi.compactmap:compactmap" ] }, - "com.github.rholder:guava-retrying": { - "locked": "2.0.0", - "transitive": [ - "com.netflix.conductor:conductor-common" - ] - }, "com.github.vlsi.compactmap:compactmap": { "locked": "1.2.1", "transitive": [ @@ -1667,10 +1621,8 @@ ] }, "com.google.code.findbugs:jsr305": { - "locked": "3.0.2", + "locked": "3.0.1", "transitive": [ - "com.github.rholder:guava-retrying", - "com.google.guava:guava", "com.netflix.archaius:archaius-core", "com.netflix.netflix-commons:netflix-infix" ] @@ -1683,35 +1635,15 @@ "com.netflix.netflix-commons:netflix-infix" ] }, - "com.google.errorprone:error_prone_annotations": { - "locked": "2.3.4", - "transitive": [ - "com.google.guava:guava" - ] - }, - "com.google.guava:failureaccess": { - "locked": "1.0.1", - "transitive": [ - "com.google.guava:guava" - ] - }, "com.google.guava:guava": { - "locked": "30.0-jre", + "locked": "19.0", "transitive": [ - "com.github.rholder:guava-retrying", "com.google.inject:guice", "com.netflix.archaius:archaius-core", - "com.netflix.conductor:conductor-core", "com.netflix.netflix-commons:netflix-infix", "com.netflix.servo:servo-core" ] }, - "com.google.guava:listenablefuture": { - "locked": "9999.0-empty-to-avoid-conflict-with-guava", - "transitive": [ - "com.google.guava:guava" - ] - }, "com.google.inject:guice": { "locked": "4.1.0", "transitive": [ @@ -1720,12 +1652,6 @@ "com.netflix.eureka:eureka-client" ] }, - "com.google.j2objc:j2objc-annotations": { - "locked": "1.3", - "transitive": [ - "com.google.guava:guava" - ] - }, "com.google.protobuf:protobuf-java": { "locked": "3.13.0", "transitive": [ @@ -2171,12 +2097,6 @@ "org.springframework.boot:spring-boot-starter-test" ] }, - "org.checkerframework:checker-qual": { - "locked": "3.5.0", - "transitive": [ - "com.google.guava:guava" - ] - }, "org.codehaus.jettison:jettison": { "locked": "1.3.7", "transitive": [ diff --git a/rest/dependencies.lock b/rest/dependencies.lock index 26c1344b04..c573511cc5 100644 --- a/rest/dependencies.lock +++ b/rest/dependencies.lock @@ -184,16 +184,16 @@ ] }, "org.springdoc:springdoc-openapi-common": { - "locked": "1.6.7", + "locked": "1.6.8", "transitive": [ "org.springdoc:springdoc-openapi-webmvc-core" ] }, "org.springdoc:springdoc-openapi-ui": { - "locked": "1.6.7" + "locked": "1.6.8" }, "org.springdoc:springdoc-openapi-webmvc-core": { - "locked": "1.6.7", + "locked": "1.6.8", "transitive": [ "org.springdoc:springdoc-openapi-ui" ] @@ -387,50 +387,6 @@ "org.springframework.boot:spring-boot-starter-json" ] }, - "com.github.rholder:guava-retrying": { - "locked": "2.0.0", - "transitive": [ - "com.netflix.conductor:conductor-common" - ] - }, - "com.google.code.findbugs:jsr305": { - "locked": "3.0.2", - "transitive": [ - "com.github.rholder:guava-retrying", - "com.google.guava:guava" - ] - }, - "com.google.errorprone:error_prone_annotations": { - "locked": "2.3.4", - "transitive": [ - "com.google.guava:guava" - ] - }, - "com.google.guava:failureaccess": { - "locked": "1.0.1", - "transitive": [ - "com.google.guava:guava" - ] - }, - "com.google.guava:guava": { - "locked": "30.0-jre", - "transitive": [ - "com.github.rholder:guava-retrying", - "com.netflix.conductor:conductor-core" - ] - }, - "com.google.guava:listenablefuture": { - "locked": "9999.0-empty-to-avoid-conflict-with-guava", - "transitive": [ - "com.google.guava:guava" - ] - }, - "com.google.j2objc:j2objc-annotations": { - "locked": "1.3", - "transitive": [ - "com.google.guava:guava" - ] - }, "com.google.protobuf:protobuf-java": { "locked": "3.13.0", "transitive": [ @@ -623,12 +579,6 @@ "org.springframework.boot:spring-boot-starter-tomcat" ] }, - "org.checkerframework:checker-qual": { - "locked": "3.5.0", - "transitive": [ - "com.google.guava:guava" - ] - }, "org.glassfish:jakarta.el": { "locked": "3.0.3", "transitive": [ @@ -659,16 +609,16 @@ ] }, "org.springdoc:springdoc-openapi-common": { - "locked": "1.6.7", + "locked": "1.6.8", "transitive": [ "org.springdoc:springdoc-openapi-webmvc-core" ] }, "org.springdoc:springdoc-openapi-ui": { - "locked": "1.6.7" + "locked": "1.6.8" }, "org.springdoc:springdoc-openapi-webmvc-core": { - "locked": "1.6.7", + "locked": "1.6.8", "transitive": [ "org.springdoc:springdoc-openapi-ui" ] @@ -1141,16 +1091,16 @@ ] }, "org.springdoc:springdoc-openapi-common": { - "locked": "1.6.7", + "locked": "1.6.8", "transitive": [ "org.springdoc:springdoc-openapi-webmvc-core" ] }, "org.springdoc:springdoc-openapi-ui": { - "locked": "1.6.7" + "locked": "1.6.8" }, "org.springdoc:springdoc-openapi-webmvc-core": { - "locked": "1.6.7", + "locked": "1.6.8", "transitive": [ "org.springdoc:springdoc-openapi-ui" ] @@ -1381,50 +1331,6 @@ "org.springframework.boot:spring-boot-starter-json" ] }, - "com.github.rholder:guava-retrying": { - "locked": "2.0.0", - "transitive": [ - "com.netflix.conductor:conductor-common" - ] - }, - "com.google.code.findbugs:jsr305": { - "locked": "3.0.2", - "transitive": [ - "com.github.rholder:guava-retrying", - "com.google.guava:guava" - ] - }, - "com.google.errorprone:error_prone_annotations": { - "locked": "2.3.4", - "transitive": [ - "com.google.guava:guava" - ] - }, - "com.google.guava:failureaccess": { - "locked": "1.0.1", - "transitive": [ - "com.google.guava:guava" - ] - }, - "com.google.guava:guava": { - "locked": "30.0-jre", - "transitive": [ - "com.github.rholder:guava-retrying", - "com.netflix.conductor:conductor-core" - ] - }, - "com.google.guava:listenablefuture": { - "locked": "9999.0-empty-to-avoid-conflict-with-guava", - "transitive": [ - "com.google.guava:guava" - ] - }, - "com.google.j2objc:j2objc-annotations": { - "locked": "1.3", - "transitive": [ - "com.google.guava:guava" - ] - }, "com.google.protobuf:protobuf-java": { "locked": "3.13.0", "transitive": [ @@ -1663,12 +1569,6 @@ "org.springframework.boot:spring-boot-starter-test" ] }, - "org.checkerframework:checker-qual": { - "locked": "3.5.0", - "transitive": [ - "com.google.guava:guava" - ] - }, "org.glassfish:jakarta.el": { "locked": "3.0.3", "transitive": [ @@ -1797,16 +1697,16 @@ ] }, "org.springdoc:springdoc-openapi-common": { - "locked": "1.6.7", + "locked": "1.6.8", "transitive": [ "org.springdoc:springdoc-openapi-webmvc-core" ] }, "org.springdoc:springdoc-openapi-ui": { - "locked": "1.6.7" + "locked": "1.6.8" }, "org.springdoc:springdoc-openapi-webmvc-core": { - "locked": "1.6.7", + "locked": "1.6.8", "transitive": [ "org.springdoc:springdoc-openapi-ui" ] diff --git a/server/build.gradle b/server/build.gradle index 2b6297f61d..e0010d93bb 100644 --- a/server/build.gradle +++ b/server/build.gradle @@ -33,6 +33,7 @@ dependencies { implementation 'org.springframework.boot:spring-boot-starter' implementation 'org.springframework.boot:spring-boot-starter-validation' implementation 'org.springframework.boot:spring-boot-starter-web' + implementation 'org.springframework.retry:spring-retry' implementation 'org.springframework.boot:spring-boot-starter-log4j2' implementation 'org.apache.logging.log4j:log4j-web' diff --git a/server/dependencies.lock b/server/dependencies.lock index 6be593c3a0..253276a080 100644 --- a/server/dependencies.lock +++ b/server/dependencies.lock @@ -255,16 +255,16 @@ ] }, "org.springdoc:springdoc-openapi-common": { - "locked": "1.6.7", + "locked": "1.6.8", "transitive": [ "org.springdoc:springdoc-openapi-webmvc-core" ] }, "org.springdoc:springdoc-openapi-ui": { - "locked": "1.6.7" + "locked": "1.6.8" }, "org.springdoc:springdoc-openapi-webmvc-core": { - "locked": "1.6.7", + "locked": "1.6.8", "transitive": [ "org.springdoc:springdoc-openapi-ui" ] @@ -338,6 +338,12 @@ "org.springframework.boot:spring-boot-starter-web": { "locked": "2.3.12.RELEASE" }, + "org.springframework.retry:spring-retry": { + "locked": "1.2.5.RELEASE", + "transitive": [ + "com.netflix.conductor:conductor-es7-persistence" + ] + }, "org.springframework:spring-aop": { "locked": "5.2.15.RELEASE", "transitive": [ @@ -366,6 +372,7 @@ "transitive": [ "org.springframework.boot:spring-boot", "org.springframework.boot:spring-boot-starter", + "org.springframework.retry:spring-retry", "org.springframework:spring-aop", "org.springframework:spring-beans", "org.springframework:spring-context", @@ -637,12 +644,6 @@ "org.apache.kafka:kafka-clients" ] }, - "com.github.rholder:guava-retrying": { - "locked": "2.0.0", - "transitive": [ - "com.netflix.conductor:conductor-common" - ] - }, "com.github.spullara.mustache.java:compiler": { "locked": "0.9.3", "transitive": [ @@ -670,7 +671,6 @@ "com.google.code.findbugs:jsr305": { "locked": "3.0.2", "transitive": [ - "com.github.rholder:guava-retrying", "com.google.guava:guava", "com.google.protobuf:protobuf-java-util", "com.netflix.archaius:archaius-core", @@ -712,12 +712,10 @@ "locked": "31.0.1-android", "transitive": [ "com.datastax.cassandra:cassandra-driver-core", - "com.github.rholder:guava-retrying", "com.google.inject:guice", "com.google.protobuf:protobuf-java-util", "com.netflix.archaius:archaius-core", "com.netflix.conductor:conductor-contribs", - "com.netflix.conductor:conductor-core", "com.netflix.conductor:conductor-es6-persistence", "com.netflix.conductor:conductor-mysql-persistence", "com.netflix.conductor:conductor-postgres-persistence", @@ -1121,7 +1119,7 @@ ] }, "io.grpc:grpc-api": { - "locked": "1.45.1", + "locked": "1.46.0", "transitive": [ "io.grpc:grpc-core", "io.grpc:grpc-protobuf", @@ -1130,45 +1128,45 @@ ] }, "io.grpc:grpc-context": { - "locked": "1.45.1", + "locked": "1.46.0", "transitive": [ "io.grpc:grpc-api" ] }, "io.grpc:grpc-core": { - "locked": "1.45.1", + "locked": "1.46.0", "transitive": [ "io.grpc:grpc-netty", "io.grpc:grpc-services" ] }, "io.grpc:grpc-netty": { - "locked": "1.45.1", + "locked": "1.46.0", "transitive": [ "com.netflix.conductor:conductor-grpc-server" ] }, "io.grpc:grpc-protobuf": { - "locked": "1.45.1", + "locked": "1.46.0", "transitive": [ "com.netflix.conductor:conductor-grpc", "io.grpc:grpc-services" ] }, "io.grpc:grpc-protobuf-lite": { - "locked": "1.45.1", + "locked": "1.46.0", "transitive": [ "io.grpc:grpc-protobuf" ] }, "io.grpc:grpc-services": { - "locked": "1.45.1", + "locked": "1.46.0", "transitive": [ "com.netflix.conductor:conductor-grpc-server" ] }, "io.grpc:grpc-stub": { - "locked": "1.45.1", + "locked": "1.46.0", "transitive": [ "com.netflix.conductor:conductor-grpc", "io.grpc:grpc-services" @@ -1320,7 +1318,7 @@ ] }, "io.perfmark:perfmark-api": { - "locked": "0.23.0", + "locked": "0.25.0", "transitive": [ "io.grpc:grpc-core", "io.grpc:grpc-netty" @@ -2123,20 +2121,20 @@ ] }, "org.springdoc:springdoc-openapi-common": { - "locked": "1.6.7", + "locked": "1.6.8", "transitive": [ "org.springdoc:springdoc-openapi-webmvc-core" ] }, "org.springdoc:springdoc-openapi-ui": { - "locked": "1.6.7", + "locked": "1.6.8", "transitive": [ "com.netflix.conductor:conductor-postgres-external-storage", "com.netflix.conductor:conductor-rest" ] }, "org.springdoc:springdoc-openapi-webmvc-core": { - "locked": "1.6.7", + "locked": "1.6.8", "transitive": [ "org.springdoc:springdoc-openapi-ui" ] @@ -2222,6 +2220,12 @@ "com.netflix.conductor:conductor-rest" ] }, + "org.springframework.retry:spring-retry": { + "locked": "1.2.5.RELEASE", + "transitive": [ + "com.netflix.conductor:conductor-es7-persistence" + ] + }, "org.springframework:spring-aop": { "locked": "5.2.15.RELEASE", "transitive": [ @@ -2252,6 +2256,7 @@ "transitive": [ "org.springframework.boot:spring-boot", "org.springframework.boot:spring-boot-starter", + "org.springframework.retry:spring-retry", "org.springframework:spring-aop", "org.springframework:spring-beans", "org.springframework:spring-context", @@ -2571,12 +2576,6 @@ "org.apache.kafka:kafka-clients" ] }, - "com.github.rholder:guava-retrying": { - "locked": "2.0.0", - "transitive": [ - "com.netflix.conductor:conductor-common" - ] - }, "com.github.spullara.mustache.java:compiler": { "locked": "0.9.3", "transitive": [ @@ -2604,7 +2603,6 @@ "com.google.code.findbugs:jsr305": { "locked": "3.0.2", "transitive": [ - "com.github.rholder:guava-retrying", "com.google.guava:guava", "com.google.protobuf:protobuf-java-util", "com.netflix.archaius:archaius-core", @@ -2646,12 +2644,10 @@ "locked": "31.0.1-android", "transitive": [ "com.datastax.cassandra:cassandra-driver-core", - "com.github.rholder:guava-retrying", "com.google.inject:guice", "com.google.protobuf:protobuf-java-util", "com.netflix.archaius:archaius-core", "com.netflix.conductor:conductor-contribs", - "com.netflix.conductor:conductor-core", "com.netflix.conductor:conductor-es6-persistence", "com.netflix.conductor:conductor-mysql-persistence", "com.netflix.conductor:conductor-postgres-persistence", @@ -3055,7 +3051,7 @@ ] }, "io.grpc:grpc-api": { - "locked": "1.45.1", + "locked": "1.46.0", "transitive": [ "io.grpc:grpc-core", "io.grpc:grpc-protobuf", @@ -3064,45 +3060,45 @@ ] }, "io.grpc:grpc-context": { - "locked": "1.45.1", + "locked": "1.46.0", "transitive": [ "io.grpc:grpc-api" ] }, "io.grpc:grpc-core": { - "locked": "1.45.1", + "locked": "1.46.0", "transitive": [ "io.grpc:grpc-netty", "io.grpc:grpc-services" ] }, "io.grpc:grpc-netty": { - "locked": "1.45.1", + "locked": "1.46.0", "transitive": [ "com.netflix.conductor:conductor-grpc-server" ] }, "io.grpc:grpc-protobuf": { - "locked": "1.45.1", + "locked": "1.46.0", "transitive": [ "com.netflix.conductor:conductor-grpc", "io.grpc:grpc-services" ] }, "io.grpc:grpc-protobuf-lite": { - "locked": "1.45.1", + "locked": "1.46.0", "transitive": [ "io.grpc:grpc-protobuf" ] }, "io.grpc:grpc-services": { - "locked": "1.45.1", + "locked": "1.46.0", "transitive": [ "com.netflix.conductor:conductor-grpc-server" ] }, "io.grpc:grpc-stub": { - "locked": "1.45.1", + "locked": "1.46.0", "transitive": [ "com.netflix.conductor:conductor-grpc", "io.grpc:grpc-services" @@ -3254,7 +3250,7 @@ ] }, "io.perfmark:perfmark-api": { - "locked": "0.23.0", + "locked": "0.25.0", "transitive": [ "io.grpc:grpc-core", "io.grpc:grpc-netty" @@ -4057,20 +4053,20 @@ ] }, "org.springdoc:springdoc-openapi-common": { - "locked": "1.6.7", + "locked": "1.6.8", "transitive": [ "org.springdoc:springdoc-openapi-webmvc-core" ] }, "org.springdoc:springdoc-openapi-ui": { - "locked": "1.6.7", + "locked": "1.6.8", "transitive": [ "com.netflix.conductor:conductor-postgres-external-storage", "com.netflix.conductor:conductor-rest" ] }, "org.springdoc:springdoc-openapi-webmvc-core": { - "locked": "1.6.7", + "locked": "1.6.8", "transitive": [ "org.springdoc:springdoc-openapi-ui" ] @@ -4156,6 +4152,12 @@ "com.netflix.conductor:conductor-rest" ] }, + "org.springframework.retry:spring-retry": { + "locked": "1.2.5.RELEASE", + "transitive": [ + "com.netflix.conductor:conductor-es7-persistence" + ] + }, "org.springframework:spring-aop": { "locked": "5.2.15.RELEASE", "transitive": [ @@ -4186,6 +4188,7 @@ "transitive": [ "org.springframework.boot:spring-boot", "org.springframework.boot:spring-boot-starter", + "org.springframework.retry:spring-retry", "org.springframework:spring-aop", "org.springframework:spring-beans", "org.springframework:spring-context", @@ -4467,7 +4470,7 @@ ] }, "io.grpc:grpc-api": { - "locked": "1.45.1", + "locked": "1.46.0", "transitive": [ "io.grpc:grpc-core", "io.grpc:grpc-protobuf", @@ -4476,34 +4479,34 @@ ] }, "io.grpc:grpc-context": { - "locked": "1.45.1", + "locked": "1.46.0", "transitive": [ "io.grpc:grpc-api" ] }, "io.grpc:grpc-core": { - "locked": "1.45.1", + "locked": "1.46.0", "transitive": [ "io.grpc:grpc-testing" ] }, "io.grpc:grpc-protobuf": { - "locked": "1.45.1" + "locked": "1.46.0" }, "io.grpc:grpc-protobuf-lite": { - "locked": "1.45.1", + "locked": "1.46.0", "transitive": [ "io.grpc:grpc-protobuf" ] }, "io.grpc:grpc-stub": { - "locked": "1.45.1", + "locked": "1.46.0", "transitive": [ "io.grpc:grpc-testing" ] }, "io.grpc:grpc-testing": { - "locked": "1.45.1" + "locked": "1.46.0" }, "io.micrometer:micrometer-core": { "locked": "1.5.14", @@ -4809,16 +4812,16 @@ ] }, "org.springdoc:springdoc-openapi-common": { - "locked": "1.6.7", + "locked": "1.6.8", "transitive": [ "org.springdoc:springdoc-openapi-webmvc-core" ] }, "org.springdoc:springdoc-openapi-ui": { - "locked": "1.6.7" + "locked": "1.6.8" }, "org.springdoc:springdoc-openapi-webmvc-core": { - "locked": "1.6.7", + "locked": "1.6.8", "transitive": [ "org.springdoc:springdoc-openapi-ui" ] @@ -4912,6 +4915,12 @@ "org.springframework.boot:spring-boot-starter-test" ] }, + "org.springframework.retry:spring-retry": { + "locked": "1.2.5.RELEASE", + "transitive": [ + "com.netflix.conductor:conductor-es7-persistence" + ] + }, "org.springframework:spring-aop": { "locked": "5.2.15.RELEASE", "transitive": [ @@ -4941,6 +4950,7 @@ "org.springframework.boot:spring-boot", "org.springframework.boot:spring-boot-starter", "org.springframework.boot:spring-boot-starter-test", + "org.springframework.retry:spring-retry", "org.springframework:spring-aop", "org.springframework:spring-beans", "org.springframework:spring-context", @@ -5225,12 +5235,6 @@ "org.apache.kafka:kafka-clients" ] }, - "com.github.rholder:guava-retrying": { - "locked": "2.0.0", - "transitive": [ - "com.netflix.conductor:conductor-common" - ] - }, "com.github.spullara.mustache.java:compiler": { "locked": "0.9.3", "transitive": [ @@ -5258,7 +5262,6 @@ "com.google.code.findbugs:jsr305": { "locked": "3.0.2", "transitive": [ - "com.github.rholder:guava-retrying", "com.google.guava:guava", "com.google.protobuf:protobuf-java-util", "com.netflix.archaius:archaius-core", @@ -5300,12 +5303,10 @@ "locked": "31.0.1-android", "transitive": [ "com.datastax.cassandra:cassandra-driver-core", - "com.github.rholder:guava-retrying", "com.google.inject:guice", "com.google.protobuf:protobuf-java-util", "com.netflix.archaius:archaius-core", "com.netflix.conductor:conductor-contribs", - "com.netflix.conductor:conductor-core", "com.netflix.conductor:conductor-es6-persistence", "com.netflix.conductor:conductor-mysql-persistence", "com.netflix.conductor:conductor-postgres-persistence", @@ -5716,7 +5717,7 @@ ] }, "io.grpc:grpc-api": { - "locked": "1.45.1", + "locked": "1.46.0", "transitive": [ "io.grpc:grpc-core", "io.grpc:grpc-protobuf", @@ -5725,14 +5726,14 @@ ] }, "io.grpc:grpc-context": { - "locked": "1.45.1", + "locked": "1.46.0", "transitive": [ "io.grpc:grpc-api", "io.grpc:grpc-testing" ] }, "io.grpc:grpc-core": { - "locked": "1.45.1", + "locked": "1.46.0", "transitive": [ "io.grpc:grpc-netty", "io.grpc:grpc-services", @@ -5740,32 +5741,32 @@ ] }, "io.grpc:grpc-netty": { - "locked": "1.45.1", + "locked": "1.46.0", "transitive": [ "com.netflix.conductor:conductor-grpc-server" ] }, "io.grpc:grpc-protobuf": { - "locked": "1.45.1", + "locked": "1.46.0", "transitive": [ "com.netflix.conductor:conductor-grpc", "io.grpc:grpc-services" ] }, "io.grpc:grpc-protobuf-lite": { - "locked": "1.45.1", + "locked": "1.46.0", "transitive": [ "io.grpc:grpc-protobuf" ] }, "io.grpc:grpc-services": { - "locked": "1.45.1", + "locked": "1.46.0", "transitive": [ "com.netflix.conductor:conductor-grpc-server" ] }, "io.grpc:grpc-stub": { - "locked": "1.45.1", + "locked": "1.46.0", "transitive": [ "com.netflix.conductor:conductor-grpc", "io.grpc:grpc-services", @@ -5773,7 +5774,7 @@ ] }, "io.grpc:grpc-testing": { - "locked": "1.45.1" + "locked": "1.46.0" }, "io.micrometer:micrometer-core": { "locked": "1.5.14", @@ -5921,7 +5922,7 @@ ] }, "io.perfmark:perfmark-api": { - "locked": "0.23.0", + "locked": "0.25.0", "transitive": [ "io.grpc:grpc-core", "io.grpc:grpc-netty" @@ -6860,20 +6861,20 @@ ] }, "org.springdoc:springdoc-openapi-common": { - "locked": "1.6.7", + "locked": "1.6.8", "transitive": [ "org.springdoc:springdoc-openapi-webmvc-core" ] }, "org.springdoc:springdoc-openapi-ui": { - "locked": "1.6.7", + "locked": "1.6.8", "transitive": [ "com.netflix.conductor:conductor-postgres-external-storage", "com.netflix.conductor:conductor-rest" ] }, "org.springdoc:springdoc-openapi-webmvc-core": { - "locked": "1.6.7", + "locked": "1.6.8", "transitive": [ "org.springdoc:springdoc-openapi-ui" ] @@ -6979,6 +6980,12 @@ "org.springframework.boot:spring-boot-starter-test" ] }, + "org.springframework.retry:spring-retry": { + "locked": "1.2.5.RELEASE", + "transitive": [ + "com.netflix.conductor:conductor-es7-persistence" + ] + }, "org.springframework:spring-aop": { "locked": "5.2.15.RELEASE", "transitive": [ @@ -7010,6 +7017,7 @@ "org.springframework.boot:spring-boot", "org.springframework.boot:spring-boot-starter", "org.springframework.boot:spring-boot-starter-test", + "org.springframework.retry:spring-retry", "org.springframework:spring-aop", "org.springframework:spring-beans", "org.springframework:spring-context", diff --git a/test-harness/build.gradle b/test-harness/build.gradle index 28911ab80c..eaf4016e3b 100644 --- a/test-harness/build.gradle +++ b/test-harness/build.gradle @@ -15,6 +15,8 @@ dependencies { testImplementation project(':conductor-client') testImplementation project(':conductor-grpc-client') + testImplementation "org.springframework.retry:spring-retry" + testImplementation "com.fasterxml.jackson.core:jackson-databind" testImplementation "com.fasterxml.jackson.core:jackson-core" diff --git a/test-harness/dependencies.lock b/test-harness/dependencies.lock index 04f10a6dfb..b1d373509c 100644 --- a/test-harness/dependencies.lock +++ b/test-harness/dependencies.lock @@ -1180,6 +1180,9 @@ "org.springframework.boot:spring-boot-starter-test" ] }, + "org.springframework.retry:spring-retry": { + "locked": "1.2.5.RELEASE" + }, "org.springframework:spring-aop": { "locked": "5.2.15.RELEASE", "transitive": [ @@ -1206,6 +1209,7 @@ "org.springframework.boot:spring-boot", "org.springframework.boot:spring-boot-starter", "org.springframework.boot:spring-boot-starter-test", + "org.springframework.retry:spring-retry", "org.springframework:spring-aop", "org.springframework:spring-beans", "org.springframework:spring-context", @@ -1548,12 +1552,6 @@ "org.apache.kafka:kafka-clients" ] }, - "com.github.rholder:guava-retrying": { - "locked": "2.0.0", - "transitive": [ - "com.netflix.conductor:conductor-common" - ] - }, "com.github.spullara.mustache.java:compiler": { "locked": "0.9.3", "transitive": [ @@ -1581,7 +1579,6 @@ "com.google.code.findbugs:jsr305": { "locked": "3.0.2", "transitive": [ - "com.github.rholder:guava-retrying", "com.google.guava:guava", "com.google.protobuf:protobuf-java-util", "com.netflix.archaius:archaius-core", @@ -1624,13 +1621,10 @@ "locked": "30.0-jre", "transitive": [ "com.datastax.cassandra:cassandra-driver-core", - "com.github.rholder:guava-retrying", "com.google.inject:guice", "com.google.protobuf:protobuf-java-util", "com.netflix.archaius:archaius-core", - "com.netflix.conductor:conductor-client", "com.netflix.conductor:conductor-contribs", - "com.netflix.conductor:conductor-core", "com.netflix.conductor:conductor-es6-persistence", "com.netflix.conductor:conductor-grpc-client", "com.netflix.conductor:conductor-mysql-persistence", @@ -2110,7 +2104,7 @@ ] }, "io.grpc:grpc-api": { - "locked": "1.45.1", + "locked": "1.46.0", "transitive": [ "io.grpc:grpc-core", "io.grpc:grpc-protobuf", @@ -2119,27 +2113,27 @@ ] }, "io.grpc:grpc-context": { - "locked": "1.45.1", + "locked": "1.46.0", "transitive": [ "io.grpc:grpc-api" ] }, "io.grpc:grpc-core": { - "locked": "1.45.1", + "locked": "1.46.0", "transitive": [ "io.grpc:grpc-netty", "io.grpc:grpc-services" ] }, "io.grpc:grpc-netty": { - "locked": "1.45.1", + "locked": "1.46.0", "transitive": [ "com.netflix.conductor:conductor-grpc-client", "com.netflix.conductor:conductor-grpc-server" ] }, "io.grpc:grpc-protobuf": { - "locked": "1.45.1", + "locked": "1.46.0", "transitive": [ "com.netflix.conductor:conductor-grpc", "com.netflix.conductor:conductor-grpc-client", @@ -2147,19 +2141,19 @@ ] }, "io.grpc:grpc-protobuf-lite": { - "locked": "1.45.1", + "locked": "1.46.0", "transitive": [ "io.grpc:grpc-protobuf" ] }, "io.grpc:grpc-services": { - "locked": "1.45.1", + "locked": "1.46.0", "transitive": [ "com.netflix.conductor:conductor-grpc-server" ] }, "io.grpc:grpc-stub": { - "locked": "1.45.1", + "locked": "1.46.0", "transitive": [ "com.netflix.conductor:conductor-grpc", "com.netflix.conductor:conductor-grpc-client", @@ -2315,7 +2309,7 @@ ] }, "io.perfmark:perfmark-api": { - "locked": "0.23.0", + "locked": "0.25.0", "transitive": [ "io.grpc:grpc-core", "io.grpc:grpc-netty" @@ -3539,13 +3533,13 @@ "locked": "1.3-groovy-2.5" }, "org.springdoc:springdoc-openapi-common": { - "locked": "1.6.7", + "locked": "1.6.8", "transitive": [ "org.springdoc:springdoc-openapi-webmvc-core" ] }, "org.springdoc:springdoc-openapi-ui": { - "locked": "1.6.7", + "locked": "1.6.8", "transitive": [ "com.netflix.conductor:conductor-postgres-external-storage", "com.netflix.conductor:conductor-rest", @@ -3553,7 +3547,7 @@ ] }, "org.springdoc:springdoc-openapi-webmvc-core": { - "locked": "1.6.7", + "locked": "1.6.8", "transitive": [ "org.springdoc:springdoc-openapi-ui" ] @@ -3670,6 +3664,13 @@ "org.springframework.boot:spring-boot-starter-test" ] }, + "org.springframework.retry:spring-retry": { + "locked": "1.2.5.RELEASE", + "transitive": [ + "com.netflix.conductor:conductor-es7-persistence", + "com.netflix.conductor:conductor-server" + ] + }, "org.springframework:spring-aop": { "locked": "5.2.15.RELEASE", "transitive": [ @@ -3701,6 +3702,7 @@ "org.springframework.boot:spring-boot", "org.springframework.boot:spring-boot-starter", "org.springframework.boot:spring-boot-starter-test", + "org.springframework.retry:spring-retry", "org.springframework:spring-aop", "org.springframework:spring-beans", "org.springframework:spring-context", diff --git a/test-harness/src/test/groovy/com/netflix/conductor/test/resiliency/QueueResiliencySpec.groovy b/test-harness/src/test/groovy/com/netflix/conductor/test/resiliency/QueueResiliencySpec.groovy index 719db6c071..1a04b59b63 100644 --- a/test-harness/src/test/groovy/com/netflix/conductor/test/resiliency/QueueResiliencySpec.groovy +++ b/test-harness/src/test/groovy/com/netflix/conductor/test/resiliency/QueueResiliencySpec.groovy @@ -479,7 +479,7 @@ class QueueResiliencySpec extends AbstractResiliencySpecification { def result = taskResource.updateTask(taskResult) then: "updateTask fails with an exception" - 2 * queueDAO.postpone(*_) >> { throw new IllegalStateException("Queue postpone failed from Spy") } + 1 * queueDAO.postpone(*_) >> { throw new IllegalStateException("Queue postpone failed from Spy") } thrown(Exception) } diff --git a/zookeeper-lock/dependencies.lock b/zookeeper-lock/dependencies.lock index 29e0ae63dc..e2c2a63e65 100644 --- a/zookeeper-lock/dependencies.lock +++ b/zookeeper-lock/dependencies.lock @@ -211,53 +211,14 @@ "com.netflix.conductor:conductor-core" ] }, - "com.github.rholder:guava-retrying": { - "locked": "2.0.0", - "transitive": [ - "com.netflix.conductor:conductor-common" - ] - }, - "com.google.code.findbugs:jsr305": { - "locked": "3.0.2", - "transitive": [ - "com.github.rholder:guava-retrying", - "com.google.guava:guava" - ] - }, - "com.google.errorprone:error_prone_annotations": { - "locked": "2.3.4", - "transitive": [ - "com.google.guava:guava" - ] - }, - "com.google.guava:failureaccess": { - "locked": "1.0.1", - "transitive": [ - "com.google.guava:guava" - ] - }, "com.google.guava:guava": { - "locked": "30.0-jre", + "locked": "14.0.1", "transitive": [ - "com.github.rholder:guava-retrying", - "com.netflix.conductor:conductor-core", "org.apache.curator:curator-client", "org.apache.curator:curator-framework", "org.apache.curator:curator-recipes" ] }, - "com.google.guava:listenablefuture": { - "locked": "9999.0-empty-to-avoid-conflict-with-guava", - "transitive": [ - "com.google.guava:guava" - ] - }, - "com.google.j2objc:j2objc-annotations": { - "locked": "1.3", - "transitive": [ - "com.google.guava:guava" - ] - }, "com.google.protobuf:protobuf-java": { "locked": "3.13.0", "transitive": [ @@ -442,12 +403,6 @@ "org.apache.curator:curator-recipes" ] }, - "org.checkerframework:checker-qual": { - "locked": "3.5.0", - "transitive": [ - "com.google.guava:guava" - ] - }, "org.jboss.netty:netty": { "locked": "3.2.2.Final", "transitive": [ @@ -906,54 +861,15 @@ "com.netflix.conductor:conductor-core" ] }, - "com.github.rholder:guava-retrying": { - "locked": "2.0.0", - "transitive": [ - "com.netflix.conductor:conductor-common" - ] - }, - "com.google.code.findbugs:jsr305": { - "locked": "3.0.2", - "transitive": [ - "com.github.rholder:guava-retrying", - "com.google.guava:guava" - ] - }, - "com.google.errorprone:error_prone_annotations": { - "locked": "2.3.4", - "transitive": [ - "com.google.guava:guava" - ] - }, - "com.google.guava:failureaccess": { - "locked": "1.0.1", - "transitive": [ - "com.google.guava:guava" - ] - }, "com.google.guava:guava": { - "locked": "30.0-jre", + "locked": "14.0.1", "transitive": [ - "com.github.rholder:guava-retrying", - "com.netflix.conductor:conductor-core", "org.apache.curator:curator-client", "org.apache.curator:curator-framework", "org.apache.curator:curator-recipes", "org.apache.curator:curator-test" ] }, - "com.google.guava:listenablefuture": { - "locked": "9999.0-empty-to-avoid-conflict-with-guava", - "transitive": [ - "com.google.guava:guava" - ] - }, - "com.google.j2objc:j2objc-annotations": { - "locked": "1.3", - "transitive": [ - "com.google.guava:guava" - ] - }, "com.google.protobuf:protobuf-java": { "locked": "3.13.0", "transitive": [ @@ -1200,12 +1116,6 @@ "org.springframework.boot:spring-boot-starter-test" ] }, - "org.checkerframework:checker-qual": { - "locked": "3.5.0", - "transitive": [ - "com.google.guava:guava" - ] - }, "org.hamcrest:hamcrest": { "locked": "2.2", "transitive": [