Mark more tests as flaky

TESTING.md

@@ -35,10 +35,14 @@ All of JanusGraph's tests are written for JUnit.  JanusGraph's JUnit tests are a
 
 ### Marking tests as flaky
 
-If a test should be marked as flaky add following annotation to the test and open an issue.
+If a test should be marked as flaky, then first [open an issue](https://github.com/JanusGraph/janusgraph/issues/new?assignees=&labels=testing%2Fflaky&projects=&template=flaky-test.md)
+where you can add information about the flaky test that could be helpful later to others to understand why the test is
+marked as flaky and hopefully for fixing it.
+Afterwards, add the annotation to the test and link to the issue you just created:
 
 ```java
-@RepeatedIfExceptionsTest(repeats = 4, minSuccess = 2)
+// flaky test: https://github.com/JanusGraph/janusgraph/issues/[ISSUE_NUMBER]
+@RepeatedIfExceptionsTest(repeats = 3)
 public void testFlakyFailsSometimes(){}
 ```
 
@@ -127,7 +131,7 @@ mvn clean install -pl janusgraph-es -Delasticsearch.docker.version=6.0.0 -Delast
 
 **Note** Running CQL tests require Docker.
 
-CQL tests are executed using [testcontainers-java](https://www.testcontainers.org/). 
+CQL tests are executed using [testcontainers-java](https://www.testcontainers.org/).
 CQL tests can be executed against a Cassandra 3 using the profile `cassandra3`, or a Scylla 3 using the profile `scylladb`.
 
 ```bash
@@ -157,10 +161,10 @@ mvn clean install -pl janusgraph-cql -Dcassandra.docker.image=cassandra -Dcassan
 
 ### TinkerPop tests
 
-The CQL backend is tested with TinkerPop tests using following command. 
+The CQL backend is tested with TinkerPop tests using following command.
 
-**Note: Profiles are not supported during running TinkerPop tests. 
-If you do not want to use the default config, you can set `cassandra.docker.image`, 
+**Note: Profiles are not supported during running TinkerPop tests.
+If you do not want to use the default config, you can set `cassandra.docker.image`,
 `cassandra.docker.version`, or `cassandra.docker.partitioner`.**
 
 ```bash
@@ -170,8 +174,8 @@ mvn clean install -Dtest.skip.tp=false -DskipTests=true -pl janusgraph-cql \
 
 ### Create new configuration files for new Versions of Cassandra
 
-The file `janusgraph-cql/src/test/resources/docker/docker-compose.yml` can be used to generate new configuration files. 
-Therefore, you have to start a Cassandra instance using `docker-compose up`. 
+The file `janusgraph-cql/src/test/resources/docker/docker-compose.yml` can be used to generate new configuration files.
+Therefore, you have to start a Cassandra instance using `docker-compose up`.
 Afterward, you can extract the configuration which is located in the following file `/etc/cassandra/cassandra.yaml`.
 
 ## Running hbase tests

janusgraph-backend-testutils/src/main/java/org/janusgraph/diskstorage/indexing/IndexProviderTest.java

@@ -845,7 +845,8 @@ public void testRestore() throws Exception {
         assertTrue(results.contains("restore-doc1"));
     }
 
-    @RepeatedIfExceptionsTest(repeats = 4, minSuccess = 2)
+    // flaky test: https://github.com/JanusGraph/janusgraph/issues/1091
+    @RepeatedIfExceptionsTest(repeats = 3)
     public void testTTL() throws Exception {
         if (!index.getFeatures().supportsDocumentTTL())
             return;

janusgraph-backend-testutils/src/main/java/org/janusgraph/diskstorage/log/LogTest.java

@@ -99,7 +99,8 @@ public void mediumSendReceiveSerial() throws Exception {
         simpleSendReceive(2000,1);
     }
 
-    @RepeatedIfExceptionsTest(repeats = 4, minSuccess = 2)
+    // flaky test: https://github.com/JanusGraph/janusgraph/issues/1445
+    @RepeatedIfExceptionsTest(repeats = 3)
     @Tag(LogTest.requiresOrderPreserving)
     public void testMultipleReadersOnSingleLogSerial() throws Exception {
         sendReceive(4, 2000, 5, true, TIMEOUT_MS);

janusgraph-backend-testutils/src/main/java/org/janusgraph/graphdb/JanusGraphIndexTest.java

@@ -2188,7 +2188,8 @@ public void testIndexReplay() throws Exception {
         assertEquals(4, recoveryStats[1]); //all 4 index transaction had provoked errors in the indexing backend
     }
 
-    @Test
+    // flaky test: https://github.com/JanusGraph/janusgraph/issues/2272
+    @RepeatedIfExceptionsTest(repeats = 3)
     public void testIndexUpdatesWithoutReindex() throws InterruptedException, ExecutionException {
         final Object[] settings = new Object[]{option(LOG_SEND_DELAY, MANAGEMENT_LOG), Duration.ofMillis(0),
                 option(KCVSLog.LOG_READ_LAG_TIME, MANAGEMENT_LOG), Duration.ofMillis(50),
@@ -3197,7 +3198,8 @@ public void testOrderByWithRange() {
 
     }
 
-    @RepeatedIfExceptionsTest(repeats = 4, minSuccess = 2)
+    // flaky test: https://github.com/JanusGraph/janusgraph/issues/3976
+    @RepeatedIfExceptionsTest(repeats = 3)
     public void shouldUpdateIndexFieldsAfterIndexModification() throws InterruptedException, ExecutionException {
         clopen(option(FORCE_INDEX_USAGE), true, option(LOG_READ_INTERVAL, MANAGEMENT_LOG), Duration.ofMillis(5000));
         String key1 = "testKey1";

janusgraph-backend-testutils/src/main/java/org/janusgraph/graphdb/JanusGraphOperationCountingTest.java

@@ -20,6 +20,7 @@
 import com.google.common.collect.ImmutableMap;
 import com.google.common.collect.Iterables;
 import com.google.common.collect.Iterators;
+import io.github.artsok.RepeatedIfExceptionsTest;
 import org.apache.tinkerpop.gremlin.structure.Direction;
 import org.apache.tinkerpop.gremlin.structure.Edge;
 import org.apache.tinkerpop.gremlin.structure.Vertex;
@@ -127,7 +128,8 @@ private void verifyLockingOverwrite(long num) {
         }
     }
 
-    @Test
+    // flaky test: https://github.com/JanusGraph/janusgraph/issues/1459
+    @RepeatedIfExceptionsTest(repeats = 3)
     public void testIdCounts() {
         makeVertexIndexedUniqueKey("uid", Integer.class);
         mgmt.setConsistency(mgmt.getGraphIndex("uid"), ConsistencyModifier.LOCK);

janusgraph-berkeleyje/src/test/java/org/janusgraph/graphdb/berkeleyje/BerkeleyOperationCountingTest.java

@@ -14,7 +14,6 @@
 
 package org.janusgraph.graphdb.berkeleyje;
 
-import io.github.artsok.RepeatedIfExceptionsTest;
 import org.janusgraph.BerkeleyStorageSetup;
 import org.janusgraph.diskstorage.configuration.WriteConfiguration;
 import org.janusgraph.graphdb.JanusGraphOperationCountingTest;
@@ -27,12 +26,6 @@ public WriteConfiguration getBaseConfiguration() {
         return BerkeleyStorageSetup.getBerkeleyJEGraphConfiguration();
     }
 
-    @Override
-    @RepeatedIfExceptionsTest(repeats = 4, minSuccess = 2)
-    public void testIdCounts() {
-        super.testIdCounts();
-    }
-
     @AfterEach
     public void resetCounts() {
         resetMetrics(); // Metrics is a singleton, so subsequents test runs have wrong counts if we don't clean up.

janusgraph-cql/src/test/java/org/janusgraph/core/cql/CQLConfiguredGraphFactoryTest.java

@@ -109,7 +109,8 @@ public void createConfigurationShouldSupportMultiHosts() throws Exception {
         }
     }
 
-    @Test
+    // flaky test: https://github.com/JanusGraph/janusgraph/issues/3393
+    @RepeatedIfExceptionsTest(repeats = 3)
     public void dropGraphShouldRemoveGraphKeyspace() throws Exception {
         final MapConfiguration graphConfig = getGraphConfig();
         final String graphName = graphConfig.getString(GRAPH_NAME.toStringWithoutRoot());
@@ -132,10 +133,18 @@ public void dropGraphShouldRemoveGraphKeyspace() throws Exception {
         }
     }
 
+    // flaky test: https://github.com/JanusGraph/janusgraph/issues/3096
     @RepeatedIfExceptionsTest(repeats = 3)
     @Override
     public void updateConfigurationShouldRemoveGraphFromCache() throws Exception {
         super.updateConfigurationShouldRemoveGraphFromCache();
     }
+
+    // flaky test: https://github.com/JanusGraph/janusgraph/issues/3959
+    @RepeatedIfExceptionsTest(repeats = 3)
+    @Override
+    public void dropShouldCleanUpTraversalSourceAndBindings() throws Exception {
+        super.dropShouldCleanUpTraversalSourceAndBindings();
+    }
 }
 

janusgraph-cql/src/test/java/org/janusgraph/graphdb/cql/CQLGraphCacheTest.java

@@ -14,13 +14,18 @@
 
 package org.janusgraph.graphdb.cql;
 
+import io.github.artsok.ParameterizedRepeatedIfExceptionsTest;
+import io.github.artsok.RepeatedIfExceptionsTest;
 import org.janusgraph.JanusGraphCassandraContainer;
 import org.janusgraph.StorageSetup;
 import org.janusgraph.diskstorage.configuration.WriteConfiguration;
 import org.janusgraph.graphdb.JanusGraphTest;
+import org.junit.jupiter.params.provider.ValueSource;
 import org.testcontainers.junit.jupiter.Container;
 import org.testcontainers.junit.jupiter.Testcontainers;
 
+import java.util.concurrent.ExecutionException;
+
 @Testcontainers
 public class CQLGraphCacheTest extends JanusGraphTest {
 
@@ -31,4 +36,62 @@ public class CQLGraphCacheTest extends JanusGraphTest {
     public WriteConfiguration getConfiguration() {
         return StorageSetup.addPermanentCache(cqlContainer.getConfiguration(getClass().getSimpleName()));
     }
+
+    // flaky test: https://github.com/JanusGraph/janusgraph/issues/1498
+    @RepeatedIfExceptionsTest(repeats = 3)
+    @Override
+    public void testIndexUpdatesWithReindexAndRemove() throws InterruptedException, ExecutionException {
+        super.testIndexUpdatesWithReindexAndRemove();
+    }
+
+    // flaky test: https://github.com/JanusGraph/janusgraph/issues/1457
+    @ParameterizedRepeatedIfExceptionsTest(repeats = 4, minSuccess = 2)
+    @ValueSource(booleans = {true, false})
+    @Override
+    public void simpleLogTest(boolean useStringId) throws InterruptedException {
+        super.simpleLogTest(useStringId);
+    }
+
+    // flaky test: https://github.com/JanusGraph/janusgraph/issues/1457
+    @ParameterizedRepeatedIfExceptionsTest(repeats = 4, minSuccess = 2)
+    @ValueSource(booleans = {true, false})
+    @Override
+    public void simpleLogTestWithFailure(boolean useStringId) throws InterruptedException {
+        super.simpleLogTestWithFailure(useStringId);
+    }
+
+    // flaky test: https://github.com/JanusGraph/janusgraph/issues/1497
+    @RepeatedIfExceptionsTest(repeats = 3)
+    @Override
+    public void testEdgeTTLTiming() throws Exception {
+        super.testEdgeTTLTiming();
+    }
+
+    // flaky test: https://github.com/JanusGraph/janusgraph/issues/1462
+    @RepeatedIfExceptionsTest(repeats = 3)
+    @Override
+    public void testEdgeTTLWithTransactions() throws Exception {
+        super.testEdgeTTLWithTransactions();
+    }
+
+    // flaky test: https://github.com/JanusGraph/janusgraph/issues/1464
+    @RepeatedIfExceptionsTest(repeats = 3)
+    @Override
+    public void testVertexTTLWithCompositeIndex() throws Exception {
+        super.testVertexTTLWithCompositeIndex();
+    }
+
+    // flaky test: https://github.com/JanusGraph/janusgraph/issues/1465
+    @RepeatedIfExceptionsTest(repeats = 3)
+    @Override
+    public void testVertexTTLImplicitKey() throws Exception {
+        super.testVertexTTLImplicitKey();
+    }
+
+    // flaky test: https://github.com/JanusGraph/janusgraph/issues/3142
+    @RepeatedIfExceptionsTest(repeats = 3)
+    @Override
+    public void testReindexingForEdgeIndex() throws ExecutionException, InterruptedException {
+        super.testReindexingForEdgeIndex();
+    }
 }

janusgraph-cql/src/test/java/org/janusgraph/graphdb/cql/CQLGraphTest.java

@@ -14,6 +14,7 @@
 
 package org.janusgraph.graphdb.cql;
 
+import io.github.artsok.ParameterizedRepeatedIfExceptionsTest;
 import io.github.artsok.RepeatedIfExceptionsTest;
 import org.apache.commons.lang3.RandomStringUtils;
 import org.apache.commons.lang3.exception.ExceptionUtils;
@@ -227,20 +228,44 @@ public void testHasTTL() {
         assertTrue(features.hasCellTTL());
     }
 
-    @ParameterizedTest
+    // flaky test: https://github.com/JanusGraph/janusgraph/issues/1457
+    @ParameterizedRepeatedIfExceptionsTest(repeats = 4, minSuccess = 2)
     @ValueSource(booleans = {true, false})
-    public void simpleLogTest(boolean useStringId) {
-        for (int i = 0; i < 3; i++) {
-            try {
-                super.simpleLogTest(useStringId);
-                // If the test passes, break out of the loop.
-                break;
-            } catch (Exception ex) {
-                log.info("Attempt #{} fails", i, ex);
-            }
-        }
+    @Override
+    public void simpleLogTest(boolean useStringId) throws InterruptedException {
+        super.simpleLogTest(useStringId);
+    }
+
+    // flaky test: https://github.com/JanusGraph/janusgraph/issues/1457
+    @ParameterizedRepeatedIfExceptionsTest(repeats = 4, minSuccess = 2)
+    @ValueSource(booleans = {true, false})
+    @Override
+    public void simpleLogTestWithFailure(boolean useStringId) throws InterruptedException {
+        super.simpleLogTestWithFailure(useStringId);
+    }
+
+    // flaky test: https://github.com/JanusGraph/janusgraph/issues/1497
+    @RepeatedIfExceptionsTest(repeats = 3)
+    @Override
+    public void testEdgeTTLTiming() throws Exception {
+        super.testEdgeTTLTiming();
+    }
+
+    // flaky test: https://github.com/JanusGraph/janusgraph/issues/1464
+    @RepeatedIfExceptionsTest(repeats = 3)
+    @Override
+    public void testVertexTTLWithCompositeIndex() throws Exception {
+        super.testVertexTTLWithCompositeIndex();
+    }
+
+    // flaky test: https://github.com/JanusGraph/janusgraph/issues/1465
+    @RepeatedIfExceptionsTest(repeats = 3)
+    @Override
+    public void testVertexTTLImplicitKey() throws Exception {
+        super.testVertexTTLImplicitKey();
     }
 
+    // flaky test: https://github.com/JanusGraph/janusgraph/issues/3142
     @RepeatedIfExceptionsTest(repeats = 3)
     @Override
     public void testReindexingForEdgeIndex() throws ExecutionException, InterruptedException {

janusgraph-cql/src/test/java/org/janusgraph/graphdb/cql/CQLOLAPTest.java

@@ -14,6 +14,7 @@
 
 package org.janusgraph.graphdb.cql;
 
+import io.github.artsok.RepeatedIfExceptionsTest;
 import org.janusgraph.JanusGraphCassandraContainer;
 import org.janusgraph.diskstorage.configuration.WriteConfiguration;
 import org.janusgraph.olap.OLAPTest;
@@ -29,4 +30,11 @@ public class CQLOLAPTest extends OLAPTest {
     public WriteConfiguration getConfiguration() {
         return cqlContainer.getConfiguration(getClass().getSimpleName()).getConfiguration();
     }
+
+    // flaky test: https://github.com/JanusGraph/janusgraph/issues/3392
+    @RepeatedIfExceptionsTest(repeats = 3)
+    @Override
+    public void testShortestDistance() throws Exception {
+        super.testShortestDistance();
+    }
 }

janusgraph-cql/src/test/java/org/janusgraph/hadoop/CQLIndexManagementIT.java

@@ -33,6 +33,7 @@ public WriteConfiguration getConfiguration() {
         return cql.getConfiguration(getClass().getSimpleName().toLowerCase()).getConfiguration();
     }
 
+    // flaky test: https://github.com/JanusGraph/janusgraph/issues/3132
     @RepeatedIfExceptionsTest(repeats = 3)
     @Override
     public void testRepairRelationIndex() throws ExecutionException, InterruptedException, BackendException {

janusgraph-es/src/test/java/org/janusgraph/diskstorage/es/BerkeleyElasticsearchTest.java

@@ -41,6 +41,13 @@ public ModifiableConfiguration getStorageConfiguration() {
         return getBerkeleyJEConfiguration();
     }
 
+    // flaky test: https://github.com/JanusGraph/janusgraph/issues/3960
+    @RepeatedIfExceptionsTest(repeats = 3)
+    @Override
+    public void indexShouldNotExistAfterDeletion() throws Exception {
+        super.indexShouldNotExistAfterDeletion();
+    }
+
     /**
      * Test {@link org.janusgraph.example.GraphOfTheGodsFactory#create(String)}.
      */
@@ -55,9 +62,17 @@ public void testGraphOfTheGodsFactoryCreate() {
         gotg.close();
     }
 
+    // flaky test: https://github.com/JanusGraph/janusgraph/issues/3651
+    @RepeatedIfExceptionsTest(repeats = 3)
+    @Override
+    public void testDisableAndDiscardManuallyAndDropEnabledIndex() throws Exception {
+        super.testDisableAndDiscardManuallyAndDropEnabledIndex();
+    }
+
+    // flaky test: https://github.com/JanusGraph/janusgraph/issues/3931
     @RepeatedIfExceptionsTest(repeats = 3)
     @Override
-    public void testIndexUpdatesWithoutReindex() throws InterruptedException, ExecutionException {
-        super.testIndexUpdatesWithoutReindex();
+    public void testDiscardAndDropRegisteredIndex() throws ExecutionException, InterruptedException {
+        super.testDiscardAndDropRegisteredIndex();
     }
 }

janusgraph-es/src/test/java/org/janusgraph/diskstorage/es/CQLElasticsearchTest.java

@@ -14,15 +14,12 @@
 
 package org.janusgraph.diskstorage.es;
 
-import io.github.artsok.RepeatedIfExceptionsTest;
 import org.janusgraph.JanusGraphCassandraContainer;
 import org.janusgraph.diskstorage.configuration.ModifiableConfiguration;
 import org.junit.jupiter.api.Disabled;
 import org.testcontainers.junit.jupiter.Container;
 import org.testcontainers.junit.jupiter.Testcontainers;
 
-import java.util.concurrent.ExecutionException;
-
 @Testcontainers
 public class CQLElasticsearchTest extends ElasticsearchJanusGraphIndexTest {
 
@@ -37,10 +34,4 @@ public ModifiableConfiguration getStorageConfiguration() {
     @Override
     @Disabled("CQL seems to not clear storage correctly")
     public void testClearStorage() {}
-
-    @RepeatedIfExceptionsTest(repeats = 3)
-    @Override
-    public void testIndexUpdatesWithoutReindex() throws InterruptedException, ExecutionException {
-        super.testIndexUpdatesWithoutReindex();
-    }
 }