diff --git a/TESTING.md b/TESTING.md
index e12e54b0c7..dc84efff60 100644
--- a/TESTING.md
+++ b/TESTING.md
@@ -35,10 +35,14 @@ All of JanusGraph's tests are written for JUnit.  JanusGraph's JUnit tests are a
 
 ### Marking tests as flaky
 
-If a test should be marked as flaky add following annotation to the test and open an issue.
+If a test should be marked as flaky, then first [open an issue](https://github.com/JanusGraph/janusgraph/issues/new?assignees=&labels=testing%2Fflaky&projects=&template=flaky-test.md)
+where you can add information about the flaky test that could be helpful later to others to understand why the test is
+marked as flaky and hopefully for fixing it.
+Afterwards, add the annotation to the test and link to the issue you just created:
 
 ```java
-@RepeatedIfExceptionsTest(repeats = 4, minSuccess = 2)
+// flaky test: https://github.com/JanusGraph/janusgraph/issues/[ISSUE_NUMBER]
+@RepeatedIfExceptionsTest(repeats = 3)
 public void testFlakyFailsSometimes(){}
 ```
 
@@ -127,7 +131,7 @@ mvn clean install -pl janusgraph-es -Delasticsearch.docker.version=6.0.0 -Delast
 
 **Note** Running CQL tests require Docker.
 
-CQL tests are executed using [testcontainers-java](https://www.testcontainers.org/). 
+CQL tests are executed using [testcontainers-java](https://www.testcontainers.org/).
 CQL tests can be executed against a Cassandra 3 using the profile `cassandra3`, or a Scylla 3 using the profile `scylladb`.
 
 ```bash
@@ -157,10 +161,10 @@ mvn clean install -pl janusgraph-cql -Dcassandra.docker.image=cassandra -Dcassan
 
 ### TinkerPop tests
 
-The CQL backend is tested with TinkerPop tests using following command. 
+The CQL backend is tested with TinkerPop tests using following command.
 
-**Note: Profiles are not supported during running TinkerPop tests. 
-If you do not want to use the default config, you can set `cassandra.docker.image`, 
+**Note: Profiles are not supported during running TinkerPop tests.
+If you do not want to use the default config, you can set `cassandra.docker.image`,
 `cassandra.docker.version`, or `cassandra.docker.partitioner`.**
 
 ```bash
@@ -170,8 +174,8 @@ mvn clean install -Dtest.skip.tp=false -DskipTests=true -pl janusgraph-cql \
 
 ### Create new configuration files for new Versions of Cassandra
 
-The file `janusgraph-cql/src/test/resources/docker/docker-compose.yml` can be used to generate new configuration files. 
-Therefore, you have to start a Cassandra instance using `docker-compose up`. 
+The file `janusgraph-cql/src/test/resources/docker/docker-compose.yml` can be used to generate new configuration files.
+Therefore, you have to start a Cassandra instance using `docker-compose up`.
 Afterward, you can extract the configuration which is located in the following file `/etc/cassandra/cassandra.yaml`.
 
 ## Running hbase tests
diff --git a/janusgraph-backend-testutils/src/main/java/org/janusgraph/diskstorage/indexing/IndexProviderTest.java b/janusgraph-backend-testutils/src/main/java/org/janusgraph/diskstorage/indexing/IndexProviderTest.java
index 17498d9a8b..b39c5f1e69 100644
--- a/janusgraph-backend-testutils/src/main/java/org/janusgraph/diskstorage/indexing/IndexProviderTest.java
+++ b/janusgraph-backend-testutils/src/main/java/org/janusgraph/diskstorage/indexing/IndexProviderTest.java
@@ -845,7 +845,8 @@ public void testRestore() throws Exception {
         assertTrue(results.contains("restore-doc1"));
     }
 
-    @RepeatedIfExceptionsTest(repeats = 4, minSuccess = 2)
+    // flaky test: https://github.com/JanusGraph/janusgraph/issues/1091
+    @RepeatedIfExceptionsTest(repeats = 3)
     public void testTTL() throws Exception {
         if (!index.getFeatures().supportsDocumentTTL())
             return;
diff --git a/janusgraph-backend-testutils/src/main/java/org/janusgraph/diskstorage/log/LogTest.java b/janusgraph-backend-testutils/src/main/java/org/janusgraph/diskstorage/log/LogTest.java
index dd35f1fcb0..89717e002a 100644
--- a/janusgraph-backend-testutils/src/main/java/org/janusgraph/diskstorage/log/LogTest.java
+++ b/janusgraph-backend-testutils/src/main/java/org/janusgraph/diskstorage/log/LogTest.java
@@ -99,7 +99,8 @@ public void mediumSendReceiveSerial() throws Exception {
         simpleSendReceive(2000,1);
     }
 
-    @RepeatedIfExceptionsTest(repeats = 4, minSuccess = 2)
+    // flaky test: https://github.com/JanusGraph/janusgraph/issues/1445
+    @RepeatedIfExceptionsTest(repeats = 3)
     @Tag(LogTest.requiresOrderPreserving)
     public void testMultipleReadersOnSingleLogSerial() throws Exception {
         sendReceive(4, 2000, 5, true, TIMEOUT_MS);
diff --git a/janusgraph-backend-testutils/src/main/java/org/janusgraph/graphdb/JanusGraphIndexTest.java b/janusgraph-backend-testutils/src/main/java/org/janusgraph/graphdb/JanusGraphIndexTest.java
index 330017f302..3247440c26 100644
--- a/janusgraph-backend-testutils/src/main/java/org/janusgraph/graphdb/JanusGraphIndexTest.java
+++ b/janusgraph-backend-testutils/src/main/java/org/janusgraph/graphdb/JanusGraphIndexTest.java
@@ -2188,7 +2188,8 @@ public void testIndexReplay() throws Exception {
         assertEquals(4, recoveryStats[1]); //all 4 index transaction had provoked errors in the indexing backend
     }
 
-    @Test
+    // flaky test: https://github.com/JanusGraph/janusgraph/issues/2272
+    @RepeatedIfExceptionsTest(repeats = 3)
     public void testIndexUpdatesWithoutReindex() throws InterruptedException, ExecutionException {
         final Object[] settings = new Object[]{option(LOG_SEND_DELAY, MANAGEMENT_LOG), Duration.ofMillis(0),
                 option(KCVSLog.LOG_READ_LAG_TIME, MANAGEMENT_LOG), Duration.ofMillis(50),
@@ -3197,7 +3198,8 @@ public void testOrderByWithRange() {
 
     }
 
-    @RepeatedIfExceptionsTest(repeats = 4, minSuccess = 2)
+    // flaky test: https://github.com/JanusGraph/janusgraph/issues/3976
+    @RepeatedIfExceptionsTest(repeats = 3)
     public void shouldUpdateIndexFieldsAfterIndexModification() throws InterruptedException, ExecutionException {
         clopen(option(FORCE_INDEX_USAGE), true, option(LOG_READ_INTERVAL, MANAGEMENT_LOG), Duration.ofMillis(5000));
         String key1 = "testKey1";
diff --git a/janusgraph-backend-testutils/src/main/java/org/janusgraph/graphdb/JanusGraphOperationCountingTest.java b/janusgraph-backend-testutils/src/main/java/org/janusgraph/graphdb/JanusGraphOperationCountingTest.java
index 711d6d583d..8c5de41dfa 100644
--- a/janusgraph-backend-testutils/src/main/java/org/janusgraph/graphdb/JanusGraphOperationCountingTest.java
+++ b/janusgraph-backend-testutils/src/main/java/org/janusgraph/graphdb/JanusGraphOperationCountingTest.java
@@ -20,6 +20,7 @@
 import com.google.common.collect.ImmutableMap;
 import com.google.common.collect.Iterables;
 import com.google.common.collect.Iterators;
+import io.github.artsok.RepeatedIfExceptionsTest;
 import org.apache.tinkerpop.gremlin.structure.Direction;
 import org.apache.tinkerpop.gremlin.structure.Edge;
 import org.apache.tinkerpop.gremlin.structure.Vertex;
@@ -127,7 +128,8 @@ private void verifyLockingOverwrite(long num) {
         }
     }
 
-    @Test
+    // flaky test: https://github.com/JanusGraph/janusgraph/issues/1459
+    @RepeatedIfExceptionsTest(repeats = 3)
     public void testIdCounts() {
         makeVertexIndexedUniqueKey("uid", Integer.class);
         mgmt.setConsistency(mgmt.getGraphIndex("uid"), ConsistencyModifier.LOCK);
diff --git a/janusgraph-berkeleyje/src/test/java/org/janusgraph/graphdb/berkeleyje/BerkeleyOperationCountingTest.java b/janusgraph-berkeleyje/src/test/java/org/janusgraph/graphdb/berkeleyje/BerkeleyOperationCountingTest.java
index 1df9a51634..2044fb57d3 100644
--- a/janusgraph-berkeleyje/src/test/java/org/janusgraph/graphdb/berkeleyje/BerkeleyOperationCountingTest.java
+++ b/janusgraph-berkeleyje/src/test/java/org/janusgraph/graphdb/berkeleyje/BerkeleyOperationCountingTest.java
@@ -14,7 +14,6 @@
 
 package org.janusgraph.graphdb.berkeleyje;
 
-import io.github.artsok.RepeatedIfExceptionsTest;
 import org.janusgraph.BerkeleyStorageSetup;
 import org.janusgraph.diskstorage.configuration.WriteConfiguration;
 import org.janusgraph.graphdb.JanusGraphOperationCountingTest;
@@ -27,12 +26,6 @@ public WriteConfiguration getBaseConfiguration() {
         return BerkeleyStorageSetup.getBerkeleyJEGraphConfiguration();
     }
 
-    @Override
-    @RepeatedIfExceptionsTest(repeats = 4, minSuccess = 2)
-    public void testIdCounts() {
-        super.testIdCounts();
-    }
-
     @AfterEach
     public void resetCounts() {
         resetMetrics(); // Metrics is a singleton, so subsequents test runs have wrong counts if we don't clean up.
diff --git a/janusgraph-cql/src/test/java/org/janusgraph/core/cql/CQLConfiguredGraphFactoryTest.java b/janusgraph-cql/src/test/java/org/janusgraph/core/cql/CQLConfiguredGraphFactoryTest.java
index 704b39b516..2d91879de6 100644
--- a/janusgraph-cql/src/test/java/org/janusgraph/core/cql/CQLConfiguredGraphFactoryTest.java
+++ b/janusgraph-cql/src/test/java/org/janusgraph/core/cql/CQLConfiguredGraphFactoryTest.java
@@ -109,7 +109,8 @@ public void createConfigurationShouldSupportMultiHosts() throws Exception {
         }
     }
 
-    @Test
+    // flaky test: https://github.com/JanusGraph/janusgraph/issues/3393
+    @RepeatedIfExceptionsTest(repeats = 3)
     public void dropGraphShouldRemoveGraphKeyspace() throws Exception {
         final MapConfiguration graphConfig = getGraphConfig();
         final String graphName = graphConfig.getString(GRAPH_NAME.toStringWithoutRoot());
@@ -132,10 +133,18 @@ public void dropGraphShouldRemoveGraphKeyspace() throws Exception {
         }
     }
 
+    // flaky test: https://github.com/JanusGraph/janusgraph/issues/3096
     @RepeatedIfExceptionsTest(repeats = 3)
     @Override
     public void updateConfigurationShouldRemoveGraphFromCache() throws Exception {
         super.updateConfigurationShouldRemoveGraphFromCache();
     }
+
+    // flaky test: https://github.com/JanusGraph/janusgraph/issues/3959
+    @RepeatedIfExceptionsTest(repeats = 3)
+    @Override
+    public void dropShouldCleanUpTraversalSourceAndBindings() throws Exception {
+        super.dropShouldCleanUpTraversalSourceAndBindings();
+    }
 }
 
diff --git a/janusgraph-cql/src/test/java/org/janusgraph/graphdb/cql/CQLGraphCacheTest.java b/janusgraph-cql/src/test/java/org/janusgraph/graphdb/cql/CQLGraphCacheTest.java
index d074765885..fd2e1c27cd 100644
--- a/janusgraph-cql/src/test/java/org/janusgraph/graphdb/cql/CQLGraphCacheTest.java
+++ b/janusgraph-cql/src/test/java/org/janusgraph/graphdb/cql/CQLGraphCacheTest.java
@@ -14,13 +14,18 @@
 
 package org.janusgraph.graphdb.cql;
 
+import io.github.artsok.ParameterizedRepeatedIfExceptionsTest;
+import io.github.artsok.RepeatedIfExceptionsTest;
 import org.janusgraph.JanusGraphCassandraContainer;
 import org.janusgraph.StorageSetup;
 import org.janusgraph.diskstorage.configuration.WriteConfiguration;
 import org.janusgraph.graphdb.JanusGraphTest;
+import org.junit.jupiter.params.provider.ValueSource;
 import org.testcontainers.junit.jupiter.Container;
 import org.testcontainers.junit.jupiter.Testcontainers;
 
+import java.util.concurrent.ExecutionException;
+
 @Testcontainers
 public class CQLGraphCacheTest extends JanusGraphTest {
 
@@ -31,4 +36,62 @@ public class CQLGraphCacheTest extends JanusGraphTest {
     public WriteConfiguration getConfiguration() {
         return StorageSetup.addPermanentCache(cqlContainer.getConfiguration(getClass().getSimpleName()));
     }
+
+    // flaky test: https://github.com/JanusGraph/janusgraph/issues/1498
+    @RepeatedIfExceptionsTest(repeats = 3)
+    @Override
+    public void testIndexUpdatesWithReindexAndRemove() throws InterruptedException, ExecutionException {
+        super.testIndexUpdatesWithReindexAndRemove();
+    }
+
+    // flaky test: https://github.com/JanusGraph/janusgraph/issues/1457
+    @ParameterizedRepeatedIfExceptionsTest(repeats = 4, minSuccess = 2)
+    @ValueSource(booleans = {true, false})
+    @Override
+    public void simpleLogTest(boolean useStringId) throws InterruptedException {
+        super.simpleLogTest(useStringId);
+    }
+
+    // flaky test: https://github.com/JanusGraph/janusgraph/issues/1457
+    @ParameterizedRepeatedIfExceptionsTest(repeats = 4, minSuccess = 2)
+    @ValueSource(booleans = {true, false})
+    @Override
+    public void simpleLogTestWithFailure(boolean useStringId) throws InterruptedException {
+        super.simpleLogTestWithFailure(useStringId);
+    }
+
+    // flaky test: https://github.com/JanusGraph/janusgraph/issues/1497
+    @RepeatedIfExceptionsTest(repeats = 3)
+    @Override
+    public void testEdgeTTLTiming() throws Exception {
+        super.testEdgeTTLTiming();
+    }
+
+    // flaky test: https://github.com/JanusGraph/janusgraph/issues/1462
+    @RepeatedIfExceptionsTest(repeats = 3)
+    @Override
+    public void testEdgeTTLWithTransactions() throws Exception {
+        super.testEdgeTTLWithTransactions();
+    }
+
+    // flaky test: https://github.com/JanusGraph/janusgraph/issues/1464
+    @RepeatedIfExceptionsTest(repeats = 3)
+    @Override
+    public void testVertexTTLWithCompositeIndex() throws Exception {
+        super.testVertexTTLWithCompositeIndex();
+    }
+
+    // flaky test: https://github.com/JanusGraph/janusgraph/issues/1465
+    @RepeatedIfExceptionsTest(repeats = 3)
+    @Override
+    public void testVertexTTLImplicitKey() throws Exception {
+        super.testVertexTTLImplicitKey();
+    }
+
+    // flaky test: https://github.com/JanusGraph/janusgraph/issues/3142
+    @RepeatedIfExceptionsTest(repeats = 3)
+    @Override
+    public void testReindexingForEdgeIndex() throws ExecutionException, InterruptedException {
+        super.testReindexingForEdgeIndex();
+    }
 }
diff --git a/janusgraph-cql/src/test/java/org/janusgraph/graphdb/cql/CQLGraphTest.java b/janusgraph-cql/src/test/java/org/janusgraph/graphdb/cql/CQLGraphTest.java
index 0c1f80102f..a2aa514c2f 100644
--- a/janusgraph-cql/src/test/java/org/janusgraph/graphdb/cql/CQLGraphTest.java
+++ b/janusgraph-cql/src/test/java/org/janusgraph/graphdb/cql/CQLGraphTest.java
@@ -14,6 +14,7 @@
 
 package org.janusgraph.graphdb.cql;
 
+import io.github.artsok.ParameterizedRepeatedIfExceptionsTest;
 import io.github.artsok.RepeatedIfExceptionsTest;
 import org.apache.commons.lang3.RandomStringUtils;
 import org.apache.commons.lang3.exception.ExceptionUtils;
@@ -227,20 +228,44 @@ public void testHasTTL() {
         assertTrue(features.hasCellTTL());
     }
 
-    @ParameterizedTest
+    // flaky test: https://github.com/JanusGraph/janusgraph/issues/1457
+    @ParameterizedRepeatedIfExceptionsTest(repeats = 4, minSuccess = 2)
     @ValueSource(booleans = {true, false})
-    public void simpleLogTest(boolean useStringId) {
-        for (int i = 0; i < 3; i++) {
-            try {
-                super.simpleLogTest(useStringId);
-                // If the test passes, break out of the loop.
-                break;
-            } catch (Exception ex) {
-                log.info("Attempt #{} fails", i, ex);
-            }
-        }
+    @Override
+    public void simpleLogTest(boolean useStringId) throws InterruptedException {
+        super.simpleLogTest(useStringId);
+    }
+
+    // flaky test: https://github.com/JanusGraph/janusgraph/issues/1457
+    @ParameterizedRepeatedIfExceptionsTest(repeats = 4, minSuccess = 2)
+    @ValueSource(booleans = {true, false})
+    @Override
+    public void simpleLogTestWithFailure(boolean useStringId) throws InterruptedException {
+        super.simpleLogTestWithFailure(useStringId);
+    }
+
+    // flaky test: https://github.com/JanusGraph/janusgraph/issues/1497
+    @RepeatedIfExceptionsTest(repeats = 3)
+    @Override
+    public void testEdgeTTLTiming() throws Exception {
+        super.testEdgeTTLTiming();
+    }
+
+    // flaky test: https://github.com/JanusGraph/janusgraph/issues/1464
+    @RepeatedIfExceptionsTest(repeats = 3)
+    @Override
+    public void testVertexTTLWithCompositeIndex() throws Exception {
+        super.testVertexTTLWithCompositeIndex();
+    }
+
+    // flaky test: https://github.com/JanusGraph/janusgraph/issues/1465
+    @RepeatedIfExceptionsTest(repeats = 3)
+    @Override
+    public void testVertexTTLImplicitKey() throws Exception {
+        super.testVertexTTLImplicitKey();
     }
 
+    // flaky test: https://github.com/JanusGraph/janusgraph/issues/3142
     @RepeatedIfExceptionsTest(repeats = 3)
     @Override
     public void testReindexingForEdgeIndex() throws ExecutionException, InterruptedException {
diff --git a/janusgraph-cql/src/test/java/org/janusgraph/graphdb/cql/CQLOLAPTest.java b/janusgraph-cql/src/test/java/org/janusgraph/graphdb/cql/CQLOLAPTest.java
index 8b33a51969..2499b9d50d 100644
--- a/janusgraph-cql/src/test/java/org/janusgraph/graphdb/cql/CQLOLAPTest.java
+++ b/janusgraph-cql/src/test/java/org/janusgraph/graphdb/cql/CQLOLAPTest.java
@@ -14,6 +14,7 @@
 
 package org.janusgraph.graphdb.cql;
 
+import io.github.artsok.RepeatedIfExceptionsTest;
 import org.janusgraph.JanusGraphCassandraContainer;
 import org.janusgraph.diskstorage.configuration.WriteConfiguration;
 import org.janusgraph.olap.OLAPTest;
@@ -29,4 +30,11 @@ public class CQLOLAPTest extends OLAPTest {
     public WriteConfiguration getConfiguration() {
         return cqlContainer.getConfiguration(getClass().getSimpleName()).getConfiguration();
     }
+
+    // flaky test: https://github.com/JanusGraph/janusgraph/issues/3392
+    @RepeatedIfExceptionsTest(repeats = 3)
+    @Override
+    public void testShortestDistance() throws Exception {
+        super.testShortestDistance();
+    }
 }
diff --git a/janusgraph-cql/src/test/java/org/janusgraph/hadoop/CQLIndexManagementIT.java b/janusgraph-cql/src/test/java/org/janusgraph/hadoop/CQLIndexManagementIT.java
index 9e60dda318..0ad70021eb 100644
--- a/janusgraph-cql/src/test/java/org/janusgraph/hadoop/CQLIndexManagementIT.java
+++ b/janusgraph-cql/src/test/java/org/janusgraph/hadoop/CQLIndexManagementIT.java
@@ -33,6 +33,7 @@ public WriteConfiguration getConfiguration() {
         return cql.getConfiguration(getClass().getSimpleName().toLowerCase()).getConfiguration();
     }
 
+    // flaky test: https://github.com/JanusGraph/janusgraph/issues/3132
     @RepeatedIfExceptionsTest(repeats = 3)
     @Override
     public void testRepairRelationIndex() throws ExecutionException, InterruptedException, BackendException {
diff --git a/janusgraph-es/src/test/java/org/janusgraph/diskstorage/es/BerkeleyElasticsearchTest.java b/janusgraph-es/src/test/java/org/janusgraph/diskstorage/es/BerkeleyElasticsearchTest.java
index 87d287ca32..45dca41fdb 100644
--- a/janusgraph-es/src/test/java/org/janusgraph/diskstorage/es/BerkeleyElasticsearchTest.java
+++ b/janusgraph-es/src/test/java/org/janusgraph/diskstorage/es/BerkeleyElasticsearchTest.java
@@ -41,6 +41,13 @@ public ModifiableConfiguration getStorageConfiguration() {
         return getBerkeleyJEConfiguration();
     }
 
+    // flaky test: https://github.com/JanusGraph/janusgraph/issues/3960
+    @RepeatedIfExceptionsTest(repeats = 3)
+    @Override
+    public void indexShouldNotExistAfterDeletion() throws Exception {
+        super.indexShouldNotExistAfterDeletion();
+    }
+
     /**
      * Test {@link org.janusgraph.example.GraphOfTheGodsFactory#create(String)}.
      */
@@ -55,9 +62,17 @@ public void testGraphOfTheGodsFactoryCreate() {
         gotg.close();
     }
 
+    // flaky test: https://github.com/JanusGraph/janusgraph/issues/3651
+    @RepeatedIfExceptionsTest(repeats = 3)
+    @Override
+    public void testDisableAndDiscardManuallyAndDropEnabledIndex() throws Exception {
+        super.testDisableAndDiscardManuallyAndDropEnabledIndex();
+    }
+
+    // flaky test: https://github.com/JanusGraph/janusgraph/issues/3931
     @RepeatedIfExceptionsTest(repeats = 3)
     @Override
-    public void testIndexUpdatesWithoutReindex() throws InterruptedException, ExecutionException {
-        super.testIndexUpdatesWithoutReindex();
+    public void testDiscardAndDropRegisteredIndex() throws ExecutionException, InterruptedException {
+        super.testDiscardAndDropRegisteredIndex();
     }
 }
diff --git a/janusgraph-es/src/test/java/org/janusgraph/diskstorage/es/CQLElasticsearchTest.java b/janusgraph-es/src/test/java/org/janusgraph/diskstorage/es/CQLElasticsearchTest.java
index 4fa030f195..f38c2ae867 100644
--- a/janusgraph-es/src/test/java/org/janusgraph/diskstorage/es/CQLElasticsearchTest.java
+++ b/janusgraph-es/src/test/java/org/janusgraph/diskstorage/es/CQLElasticsearchTest.java
@@ -14,15 +14,12 @@
 
 package org.janusgraph.diskstorage.es;
 
-import io.github.artsok.RepeatedIfExceptionsTest;
 import org.janusgraph.JanusGraphCassandraContainer;
 import org.janusgraph.diskstorage.configuration.ModifiableConfiguration;
 import org.junit.jupiter.api.Disabled;
 import org.testcontainers.junit.jupiter.Container;
 import org.testcontainers.junit.jupiter.Testcontainers;
 
-import java.util.concurrent.ExecutionException;
-
 @Testcontainers
 public class CQLElasticsearchTest extends ElasticsearchJanusGraphIndexTest {
 
@@ -37,10 +34,4 @@ public ModifiableConfiguration getStorageConfiguration() {
     @Override
     @Disabled("CQL seems to not clear storage correctly")
     public void testClearStorage() {}
-
-    @RepeatedIfExceptionsTest(repeats = 3)
-    @Override
-    public void testIndexUpdatesWithoutReindex() throws InterruptedException, ExecutionException {
-        super.testIndexUpdatesWithoutReindex();
-    }
 }
diff --git a/janusgraph-solr/src/test/java/org/janusgraph/diskstorage/solr/CQLSolrTest.java b/janusgraph-solr/src/test/java/org/janusgraph/diskstorage/solr/CQLSolrTest.java
index f0433400eb..ddec938844 100644
--- a/janusgraph-solr/src/test/java/org/janusgraph/diskstorage/solr/CQLSolrTest.java
+++ b/janusgraph-solr/src/test/java/org/janusgraph/diskstorage/solr/CQLSolrTest.java
@@ -14,6 +14,7 @@
 
 package org.janusgraph.diskstorage.solr;
 
+import io.github.artsok.RepeatedIfExceptionsTest;
 import org.janusgraph.JanusGraphCassandraContainer;
 import org.janusgraph.diskstorage.configuration.ModifiableConfiguration;
 import org.junit.jupiter.api.Test;
@@ -38,6 +39,13 @@ public boolean supportsWildcardQuery() {
         return false;
     }
 
+    // flaky test: https://github.com/JanusGraph/janusgraph/issues/3356
+    @RepeatedIfExceptionsTest(repeats = 3)
+    @Override
+    public void testSetIndexing() {
+        super.testSetIndexing();
+    }
+
     @Test
     @Override
     public void testDiscardAndDropRegisteredIndex() {
diff --git a/janusgraph-solr/src/test/java/org/janusgraph/diskstorage/solr/SolrJanusGraphIndexTest.java b/janusgraph-solr/src/test/java/org/janusgraph/diskstorage/solr/SolrJanusGraphIndexTest.java
index 62bf34bf85..694fbd119b 100644
--- a/janusgraph-solr/src/test/java/org/janusgraph/diskstorage/solr/SolrJanusGraphIndexTest.java
+++ b/janusgraph-solr/src/test/java/org/janusgraph/diskstorage/solr/SolrJanusGraphIndexTest.java
@@ -81,6 +81,7 @@ public void testClearStorage() throws Exception {
         super.testClearStorage();
     }
 
+    // flaky test: https://github.com/JanusGraph/janusgraph/issues/2271
     @Override
     @RepeatedIfExceptionsTest(repeats = 10, suspend = 1000L)
     public void testIndexReplay() throws Exception {
diff --git a/janusgraph-test/src/test/java/org/janusgraph/diskstorage/cache/ExpirationCacheTest.java b/janusgraph-test/src/test/java/org/janusgraph/diskstorage/cache/ExpirationCacheTest.java
index 5ee9a601e4..256e4ffc8d 100644
--- a/janusgraph-test/src/test/java/org/janusgraph/diskstorage/cache/ExpirationCacheTest.java
+++ b/janusgraph-test/src/test/java/org/janusgraph/diskstorage/cache/ExpirationCacheTest.java
@@ -52,7 +52,8 @@ private static KCVSCache getCache(KeyColumnValueStore store, Duration expiration
         return new ExpirationKCVSCache(store,METRICS_STRING,expirationTime.toMillis(),graceWait.toMillis(),CACHE_SIZE);
     }
 
-    @RepeatedIfExceptionsTest(repeats = 4, minSuccess = 2)
+    // flaky test: https://github.com/JanusGraph/janusgraph/issues/2934
+    @RepeatedIfExceptionsTest(repeats = 3)
     public void testExpiration() throws Exception {
         testExpiration(Duration.ofMillis(200));
         testExpiration(Duration.ofSeconds(4));
@@ -96,6 +97,7 @@ private void testExpiration(Duration expirationTime) throws Exception {
         verifyResults(key, keys, query, 4);
     }
 
+    // flaky test: https://github.com/JanusGraph/janusgraph/issues/3352
     @RepeatedIfExceptionsTest(repeats = 3)
     public void testGracePeriod() throws Exception {
         testGracePeriod(Duration.ofMillis(200));
diff --git a/janusgraph-test/src/test/java/org/janusgraph/graphdb/vertices/StandardVertexTest.java b/janusgraph-test/src/test/java/org/janusgraph/graphdb/vertices/StandardVertexTest.java
index 1cf16aac84..b974919166 100644
--- a/janusgraph-test/src/test/java/org/janusgraph/graphdb/vertices/StandardVertexTest.java
+++ b/janusgraph-test/src/test/java/org/janusgraph/graphdb/vertices/StandardVertexTest.java
@@ -59,7 +59,9 @@ public void setup(){
         standardVertex = spy(new StandardVertex(tx, 1, (byte) 1));
     }
 
-    @RepeatedIfExceptionsTest(repeats = 3, minSuccess = 1)
+    // This test is flaky simply because it tests a possible deadlock
+    // https://github.com/JanusGraph/janusgraph/pull/1486
+    @RepeatedIfExceptionsTest(repeats = 3)
     public void shouldNotStuckInDeadlockWhenTheVerticeAndItsRelationIsDeletedInParallel()
         throws InterruptedException, TimeoutException, ExecutionException {