From 87911fda38f2ac3b38eb250f05a8bf07d20b276b Mon Sep 17 00:00:00 2001 From: Florian Hockmann Date: Wed, 6 Sep 2023 14:29:51 +0200 Subject: [PATCH] Mark more tests as flaky This adds the necessary annotation to repeat flaky tests if necessary for tests described in these issues: * #1457 * #1459 * #1462 * #1464 * #1465 * #1497 * #1498 * #2272 * #3142 * #3356 * #3392 * #3393 * #3651 * #3931 * #3959 * #3960 I went through the recent builds on `master` and checked all failed builds. I think this should handle all flaky tests that occurred in these builds, apart from the TinkerPop test where I just don't know how to deal with it: #3841, but we can of course just address that in another PR. We however also sometimes get failed jobs because of some intermediate issues where a backend cannot be properly set up or teared down for some reason or downloading some artifact just fails. I don't know what we can do to improve this, but I would also consider that out of scope of this PR where I just wanted to deal with failing jobs due to specific tests being flaky. Hopefully, this will already lead to significantly fewer cases where we have to manually restart failed jobs (which I had to do a lot lately for all these Dependabot PRs). I also added a link to the relevant issue to all flaky tests. Signed-off-by: Florian Hockmann --- TESTING.md | 20 +++--- .../indexing/IndexProviderTest.java | 3 +- .../janusgraph/diskstorage/log/LogTest.java | 3 +- .../graphdb/JanusGraphIndexTest.java | 6 +- .../JanusGraphOperationCountingTest.java | 4 +- .../BerkeleyOperationCountingTest.java | 7 --- .../cql/CQLConfiguredGraphFactoryTest.java | 11 +++- .../graphdb/cql/CQLGraphCacheTest.java | 63 +++++++++++++++++++ .../janusgraph/graphdb/cql/CQLGraphTest.java | 47 ++++++++++---- .../janusgraph/graphdb/cql/CQLOLAPTest.java | 8 +++ .../hadoop/CQLIndexManagementIT.java | 1 + .../es/BerkeleyElasticsearchTest.java | 19 +++++- .../diskstorage/es/CQLElasticsearchTest.java | 9 --- .../diskstorage/solr/CQLSolrTest.java | 8 +++ .../solr/SolrJanusGraphIndexTest.java | 1 + .../cache/ExpirationCacheTest.java | 4 +- .../graphdb/vertices/StandardVertexTest.java | 4 +- 17 files changed, 173 insertions(+), 45 deletions(-) diff --git a/TESTING.md b/TESTING.md index e12e54b0c7..dc84efff60 100644 --- a/TESTING.md +++ b/TESTING.md @@ -35,10 +35,14 @@ All of JanusGraph's tests are written for JUnit. JanusGraph's JUnit tests are a ### Marking tests as flaky -If a test should be marked as flaky add following annotation to the test and open an issue. +If a test should be marked as flaky, then first [open an issue](https://github.com/JanusGraph/janusgraph/issues/new?assignees=&labels=testing%2Fflaky&projects=&template=flaky-test.md) +where you can add information about the flaky test that could be helpful later to others to understand why the test is +marked as flaky and hopefully for fixing it. +Afterwards, add the annotation to the test and link to the issue you just created: ```java -@RepeatedIfExceptionsTest(repeats = 4, minSuccess = 2) +// flaky test: https://github.com/JanusGraph/janusgraph/issues/[ISSUE_NUMBER] +@RepeatedIfExceptionsTest(repeats = 3) public void testFlakyFailsSometimes(){} ``` @@ -127,7 +131,7 @@ mvn clean install -pl janusgraph-es -Delasticsearch.docker.version=6.0.0 -Delast **Note** Running CQL tests require Docker. -CQL tests are executed using [testcontainers-java](https://www.testcontainers.org/). +CQL tests are executed using [testcontainers-java](https://www.testcontainers.org/). CQL tests can be executed against a Cassandra 3 using the profile `cassandra3`, or a Scylla 3 using the profile `scylladb`. ```bash @@ -157,10 +161,10 @@ mvn clean install -pl janusgraph-cql -Dcassandra.docker.image=cassandra -Dcassan ### TinkerPop tests -The CQL backend is tested with TinkerPop tests using following command. +The CQL backend is tested with TinkerPop tests using following command. -**Note: Profiles are not supported during running TinkerPop tests. -If you do not want to use the default config, you can set `cassandra.docker.image`, +**Note: Profiles are not supported during running TinkerPop tests. +If you do not want to use the default config, you can set `cassandra.docker.image`, `cassandra.docker.version`, or `cassandra.docker.partitioner`.** ```bash @@ -170,8 +174,8 @@ mvn clean install -Dtest.skip.tp=false -DskipTests=true -pl janusgraph-cql \ ### Create new configuration files for new Versions of Cassandra -The file `janusgraph-cql/src/test/resources/docker/docker-compose.yml` can be used to generate new configuration files. -Therefore, you have to start a Cassandra instance using `docker-compose up`. +The file `janusgraph-cql/src/test/resources/docker/docker-compose.yml` can be used to generate new configuration files. +Therefore, you have to start a Cassandra instance using `docker-compose up`. Afterward, you can extract the configuration which is located in the following file `/etc/cassandra/cassandra.yaml`. ## Running hbase tests diff --git a/janusgraph-backend-testutils/src/main/java/org/janusgraph/diskstorage/indexing/IndexProviderTest.java b/janusgraph-backend-testutils/src/main/java/org/janusgraph/diskstorage/indexing/IndexProviderTest.java index 17498d9a8b..b39c5f1e69 100644 --- a/janusgraph-backend-testutils/src/main/java/org/janusgraph/diskstorage/indexing/IndexProviderTest.java +++ b/janusgraph-backend-testutils/src/main/java/org/janusgraph/diskstorage/indexing/IndexProviderTest.java @@ -845,7 +845,8 @@ public void testRestore() throws Exception { assertTrue(results.contains("restore-doc1")); } - @RepeatedIfExceptionsTest(repeats = 4, minSuccess = 2) + // flaky test: https://github.com/JanusGraph/janusgraph/issues/1091 + @RepeatedIfExceptionsTest(repeats = 3) public void testTTL() throws Exception { if (!index.getFeatures().supportsDocumentTTL()) return; diff --git a/janusgraph-backend-testutils/src/main/java/org/janusgraph/diskstorage/log/LogTest.java b/janusgraph-backend-testutils/src/main/java/org/janusgraph/diskstorage/log/LogTest.java index dd35f1fcb0..89717e002a 100644 --- a/janusgraph-backend-testutils/src/main/java/org/janusgraph/diskstorage/log/LogTest.java +++ b/janusgraph-backend-testutils/src/main/java/org/janusgraph/diskstorage/log/LogTest.java @@ -99,7 +99,8 @@ public void mediumSendReceiveSerial() throws Exception { simpleSendReceive(2000,1); } - @RepeatedIfExceptionsTest(repeats = 4, minSuccess = 2) + // flaky test: https://github.com/JanusGraph/janusgraph/issues/1445 + @RepeatedIfExceptionsTest(repeats = 3) @Tag(LogTest.requiresOrderPreserving) public void testMultipleReadersOnSingleLogSerial() throws Exception { sendReceive(4, 2000, 5, true, TIMEOUT_MS); diff --git a/janusgraph-backend-testutils/src/main/java/org/janusgraph/graphdb/JanusGraphIndexTest.java b/janusgraph-backend-testutils/src/main/java/org/janusgraph/graphdb/JanusGraphIndexTest.java index 330017f302..3247440c26 100644 --- a/janusgraph-backend-testutils/src/main/java/org/janusgraph/graphdb/JanusGraphIndexTest.java +++ b/janusgraph-backend-testutils/src/main/java/org/janusgraph/graphdb/JanusGraphIndexTest.java @@ -2188,7 +2188,8 @@ public void testIndexReplay() throws Exception { assertEquals(4, recoveryStats[1]); //all 4 index transaction had provoked errors in the indexing backend } - @Test + // flaky test: https://github.com/JanusGraph/janusgraph/issues/2272 + @RepeatedIfExceptionsTest(repeats = 3) public void testIndexUpdatesWithoutReindex() throws InterruptedException, ExecutionException { final Object[] settings = new Object[]{option(LOG_SEND_DELAY, MANAGEMENT_LOG), Duration.ofMillis(0), option(KCVSLog.LOG_READ_LAG_TIME, MANAGEMENT_LOG), Duration.ofMillis(50), @@ -3197,7 +3198,8 @@ public void testOrderByWithRange() { } - @RepeatedIfExceptionsTest(repeats = 4, minSuccess = 2) + // flaky test: https://github.com/JanusGraph/janusgraph/issues/3976 + @RepeatedIfExceptionsTest(repeats = 3) public void shouldUpdateIndexFieldsAfterIndexModification() throws InterruptedException, ExecutionException { clopen(option(FORCE_INDEX_USAGE), true, option(LOG_READ_INTERVAL, MANAGEMENT_LOG), Duration.ofMillis(5000)); String key1 = "testKey1"; diff --git a/janusgraph-backend-testutils/src/main/java/org/janusgraph/graphdb/JanusGraphOperationCountingTest.java b/janusgraph-backend-testutils/src/main/java/org/janusgraph/graphdb/JanusGraphOperationCountingTest.java index 711d6d583d..8c5de41dfa 100644 --- a/janusgraph-backend-testutils/src/main/java/org/janusgraph/graphdb/JanusGraphOperationCountingTest.java +++ b/janusgraph-backend-testutils/src/main/java/org/janusgraph/graphdb/JanusGraphOperationCountingTest.java @@ -20,6 +20,7 @@ import com.google.common.collect.ImmutableMap; import com.google.common.collect.Iterables; import com.google.common.collect.Iterators; +import io.github.artsok.RepeatedIfExceptionsTest; import org.apache.tinkerpop.gremlin.structure.Direction; import org.apache.tinkerpop.gremlin.structure.Edge; import org.apache.tinkerpop.gremlin.structure.Vertex; @@ -127,7 +128,8 @@ private void verifyLockingOverwrite(long num) { } } - @Test + // flaky test: https://github.com/JanusGraph/janusgraph/issues/1459 + @RepeatedIfExceptionsTest(repeats = 3) public void testIdCounts() { makeVertexIndexedUniqueKey("uid", Integer.class); mgmt.setConsistency(mgmt.getGraphIndex("uid"), ConsistencyModifier.LOCK); diff --git a/janusgraph-berkeleyje/src/test/java/org/janusgraph/graphdb/berkeleyje/BerkeleyOperationCountingTest.java b/janusgraph-berkeleyje/src/test/java/org/janusgraph/graphdb/berkeleyje/BerkeleyOperationCountingTest.java index 1df9a51634..2044fb57d3 100644 --- a/janusgraph-berkeleyje/src/test/java/org/janusgraph/graphdb/berkeleyje/BerkeleyOperationCountingTest.java +++ b/janusgraph-berkeleyje/src/test/java/org/janusgraph/graphdb/berkeleyje/BerkeleyOperationCountingTest.java @@ -14,7 +14,6 @@ package org.janusgraph.graphdb.berkeleyje; -import io.github.artsok.RepeatedIfExceptionsTest; import org.janusgraph.BerkeleyStorageSetup; import org.janusgraph.diskstorage.configuration.WriteConfiguration; import org.janusgraph.graphdb.JanusGraphOperationCountingTest; @@ -27,12 +26,6 @@ public WriteConfiguration getBaseConfiguration() { return BerkeleyStorageSetup.getBerkeleyJEGraphConfiguration(); } - @Override - @RepeatedIfExceptionsTest(repeats = 4, minSuccess = 2) - public void testIdCounts() { - super.testIdCounts(); - } - @AfterEach public void resetCounts() { resetMetrics(); // Metrics is a singleton, so subsequents test runs have wrong counts if we don't clean up. diff --git a/janusgraph-cql/src/test/java/org/janusgraph/core/cql/CQLConfiguredGraphFactoryTest.java b/janusgraph-cql/src/test/java/org/janusgraph/core/cql/CQLConfiguredGraphFactoryTest.java index 704b39b516..2d91879de6 100644 --- a/janusgraph-cql/src/test/java/org/janusgraph/core/cql/CQLConfiguredGraphFactoryTest.java +++ b/janusgraph-cql/src/test/java/org/janusgraph/core/cql/CQLConfiguredGraphFactoryTest.java @@ -109,7 +109,8 @@ public void createConfigurationShouldSupportMultiHosts() throws Exception { } } - @Test + // flaky test: https://github.com/JanusGraph/janusgraph/issues/3393 + @RepeatedIfExceptionsTest(repeats = 3) public void dropGraphShouldRemoveGraphKeyspace() throws Exception { final MapConfiguration graphConfig = getGraphConfig(); final String graphName = graphConfig.getString(GRAPH_NAME.toStringWithoutRoot()); @@ -132,10 +133,18 @@ public void dropGraphShouldRemoveGraphKeyspace() throws Exception { } } + // flaky test: https://github.com/JanusGraph/janusgraph/issues/3096 @RepeatedIfExceptionsTest(repeats = 3) @Override public void updateConfigurationShouldRemoveGraphFromCache() throws Exception { super.updateConfigurationShouldRemoveGraphFromCache(); } + + // flaky test: https://github.com/JanusGraph/janusgraph/issues/3959 + @RepeatedIfExceptionsTest(repeats = 3) + @Override + public void dropShouldCleanUpTraversalSourceAndBindings() throws Exception { + super.dropShouldCleanUpTraversalSourceAndBindings(); + } } diff --git a/janusgraph-cql/src/test/java/org/janusgraph/graphdb/cql/CQLGraphCacheTest.java b/janusgraph-cql/src/test/java/org/janusgraph/graphdb/cql/CQLGraphCacheTest.java index d074765885..fd2e1c27cd 100644 --- a/janusgraph-cql/src/test/java/org/janusgraph/graphdb/cql/CQLGraphCacheTest.java +++ b/janusgraph-cql/src/test/java/org/janusgraph/graphdb/cql/CQLGraphCacheTest.java @@ -14,13 +14,18 @@ package org.janusgraph.graphdb.cql; +import io.github.artsok.ParameterizedRepeatedIfExceptionsTest; +import io.github.artsok.RepeatedIfExceptionsTest; import org.janusgraph.JanusGraphCassandraContainer; import org.janusgraph.StorageSetup; import org.janusgraph.diskstorage.configuration.WriteConfiguration; import org.janusgraph.graphdb.JanusGraphTest; +import org.junit.jupiter.params.provider.ValueSource; import org.testcontainers.junit.jupiter.Container; import org.testcontainers.junit.jupiter.Testcontainers; +import java.util.concurrent.ExecutionException; + @Testcontainers public class CQLGraphCacheTest extends JanusGraphTest { @@ -31,4 +36,62 @@ public class CQLGraphCacheTest extends JanusGraphTest { public WriteConfiguration getConfiguration() { return StorageSetup.addPermanentCache(cqlContainer.getConfiguration(getClass().getSimpleName())); } + + // flaky test: https://github.com/JanusGraph/janusgraph/issues/1498 + @RepeatedIfExceptionsTest(repeats = 3) + @Override + public void testIndexUpdatesWithReindexAndRemove() throws InterruptedException, ExecutionException { + super.testIndexUpdatesWithReindexAndRemove(); + } + + // flaky test: https://github.com/JanusGraph/janusgraph/issues/1457 + @ParameterizedRepeatedIfExceptionsTest(repeats = 4, minSuccess = 2) + @ValueSource(booleans = {true, false}) + @Override + public void simpleLogTest(boolean useStringId) throws InterruptedException { + super.simpleLogTest(useStringId); + } + + // flaky test: https://github.com/JanusGraph/janusgraph/issues/1457 + @ParameterizedRepeatedIfExceptionsTest(repeats = 4, minSuccess = 2) + @ValueSource(booleans = {true, false}) + @Override + public void simpleLogTestWithFailure(boolean useStringId) throws InterruptedException { + super.simpleLogTestWithFailure(useStringId); + } + + // flaky test: https://github.com/JanusGraph/janusgraph/issues/1497 + @RepeatedIfExceptionsTest(repeats = 3) + @Override + public void testEdgeTTLTiming() throws Exception { + super.testEdgeTTLTiming(); + } + + // flaky test: https://github.com/JanusGraph/janusgraph/issues/1462 + @RepeatedIfExceptionsTest(repeats = 3) + @Override + public void testEdgeTTLWithTransactions() throws Exception { + super.testEdgeTTLWithTransactions(); + } + + // flaky test: https://github.com/JanusGraph/janusgraph/issues/1464 + @RepeatedIfExceptionsTest(repeats = 3) + @Override + public void testVertexTTLWithCompositeIndex() throws Exception { + super.testVertexTTLWithCompositeIndex(); + } + + // flaky test: https://github.com/JanusGraph/janusgraph/issues/1465 + @RepeatedIfExceptionsTest(repeats = 3) + @Override + public void testVertexTTLImplicitKey() throws Exception { + super.testVertexTTLImplicitKey(); + } + + // flaky test: https://github.com/JanusGraph/janusgraph/issues/3142 + @RepeatedIfExceptionsTest(repeats = 3) + @Override + public void testReindexingForEdgeIndex() throws ExecutionException, InterruptedException { + super.testReindexingForEdgeIndex(); + } } diff --git a/janusgraph-cql/src/test/java/org/janusgraph/graphdb/cql/CQLGraphTest.java b/janusgraph-cql/src/test/java/org/janusgraph/graphdb/cql/CQLGraphTest.java index 0c1f80102f..a2aa514c2f 100644 --- a/janusgraph-cql/src/test/java/org/janusgraph/graphdb/cql/CQLGraphTest.java +++ b/janusgraph-cql/src/test/java/org/janusgraph/graphdb/cql/CQLGraphTest.java @@ -14,6 +14,7 @@ package org.janusgraph.graphdb.cql; +import io.github.artsok.ParameterizedRepeatedIfExceptionsTest; import io.github.artsok.RepeatedIfExceptionsTest; import org.apache.commons.lang3.RandomStringUtils; import org.apache.commons.lang3.exception.ExceptionUtils; @@ -227,20 +228,44 @@ public void testHasTTL() { assertTrue(features.hasCellTTL()); } - @ParameterizedTest + // flaky test: https://github.com/JanusGraph/janusgraph/issues/1457 + @ParameterizedRepeatedIfExceptionsTest(repeats = 4, minSuccess = 2) @ValueSource(booleans = {true, false}) - public void simpleLogTest(boolean useStringId) { - for (int i = 0; i < 3; i++) { - try { - super.simpleLogTest(useStringId); - // If the test passes, break out of the loop. - break; - } catch (Exception ex) { - log.info("Attempt #{} fails", i, ex); - } - } + @Override + public void simpleLogTest(boolean useStringId) throws InterruptedException { + super.simpleLogTest(useStringId); + } + + // flaky test: https://github.com/JanusGraph/janusgraph/issues/1457 + @ParameterizedRepeatedIfExceptionsTest(repeats = 4, minSuccess = 2) + @ValueSource(booleans = {true, false}) + @Override + public void simpleLogTestWithFailure(boolean useStringId) throws InterruptedException { + super.simpleLogTestWithFailure(useStringId); + } + + // flaky test: https://github.com/JanusGraph/janusgraph/issues/1497 + @RepeatedIfExceptionsTest(repeats = 3) + @Override + public void testEdgeTTLTiming() throws Exception { + super.testEdgeTTLTiming(); + } + + // flaky test: https://github.com/JanusGraph/janusgraph/issues/1464 + @RepeatedIfExceptionsTest(repeats = 3) + @Override + public void testVertexTTLWithCompositeIndex() throws Exception { + super.testVertexTTLWithCompositeIndex(); + } + + // flaky test: https://github.com/JanusGraph/janusgraph/issues/1465 + @RepeatedIfExceptionsTest(repeats = 3) + @Override + public void testVertexTTLImplicitKey() throws Exception { + super.testVertexTTLImplicitKey(); } + // flaky test: https://github.com/JanusGraph/janusgraph/issues/3142 @RepeatedIfExceptionsTest(repeats = 3) @Override public void testReindexingForEdgeIndex() throws ExecutionException, InterruptedException { diff --git a/janusgraph-cql/src/test/java/org/janusgraph/graphdb/cql/CQLOLAPTest.java b/janusgraph-cql/src/test/java/org/janusgraph/graphdb/cql/CQLOLAPTest.java index 8b33a51969..2499b9d50d 100644 --- a/janusgraph-cql/src/test/java/org/janusgraph/graphdb/cql/CQLOLAPTest.java +++ b/janusgraph-cql/src/test/java/org/janusgraph/graphdb/cql/CQLOLAPTest.java @@ -14,6 +14,7 @@ package org.janusgraph.graphdb.cql; +import io.github.artsok.RepeatedIfExceptionsTest; import org.janusgraph.JanusGraphCassandraContainer; import org.janusgraph.diskstorage.configuration.WriteConfiguration; import org.janusgraph.olap.OLAPTest; @@ -29,4 +30,11 @@ public class CQLOLAPTest extends OLAPTest { public WriteConfiguration getConfiguration() { return cqlContainer.getConfiguration(getClass().getSimpleName()).getConfiguration(); } + + // flaky test: https://github.com/JanusGraph/janusgraph/issues/3392 + @RepeatedIfExceptionsTest(repeats = 3) + @Override + public void testShortestDistance() throws Exception { + super.testShortestDistance(); + } } diff --git a/janusgraph-cql/src/test/java/org/janusgraph/hadoop/CQLIndexManagementIT.java b/janusgraph-cql/src/test/java/org/janusgraph/hadoop/CQLIndexManagementIT.java index 9e60dda318..0ad70021eb 100644 --- a/janusgraph-cql/src/test/java/org/janusgraph/hadoop/CQLIndexManagementIT.java +++ b/janusgraph-cql/src/test/java/org/janusgraph/hadoop/CQLIndexManagementIT.java @@ -33,6 +33,7 @@ public WriteConfiguration getConfiguration() { return cql.getConfiguration(getClass().getSimpleName().toLowerCase()).getConfiguration(); } + // flaky test: https://github.com/JanusGraph/janusgraph/issues/3132 @RepeatedIfExceptionsTest(repeats = 3) @Override public void testRepairRelationIndex() throws ExecutionException, InterruptedException, BackendException { diff --git a/janusgraph-es/src/test/java/org/janusgraph/diskstorage/es/BerkeleyElasticsearchTest.java b/janusgraph-es/src/test/java/org/janusgraph/diskstorage/es/BerkeleyElasticsearchTest.java index 87d287ca32..45dca41fdb 100644 --- a/janusgraph-es/src/test/java/org/janusgraph/diskstorage/es/BerkeleyElasticsearchTest.java +++ b/janusgraph-es/src/test/java/org/janusgraph/diskstorage/es/BerkeleyElasticsearchTest.java @@ -41,6 +41,13 @@ public ModifiableConfiguration getStorageConfiguration() { return getBerkeleyJEConfiguration(); } + // flaky test: https://github.com/JanusGraph/janusgraph/issues/3960 + @RepeatedIfExceptionsTest(repeats = 3) + @Override + public void indexShouldNotExistAfterDeletion() throws Exception { + super.indexShouldNotExistAfterDeletion(); + } + /** * Test {@link org.janusgraph.example.GraphOfTheGodsFactory#create(String)}. */ @@ -55,9 +62,17 @@ public void testGraphOfTheGodsFactoryCreate() { gotg.close(); } + // flaky test: https://github.com/JanusGraph/janusgraph/issues/3651 + @RepeatedIfExceptionsTest(repeats = 3) + @Override + public void testDisableAndDiscardManuallyAndDropEnabledIndex() throws Exception { + super.testDisableAndDiscardManuallyAndDropEnabledIndex(); + } + + // flaky test: https://github.com/JanusGraph/janusgraph/issues/3931 @RepeatedIfExceptionsTest(repeats = 3) @Override - public void testIndexUpdatesWithoutReindex() throws InterruptedException, ExecutionException { - super.testIndexUpdatesWithoutReindex(); + public void testDiscardAndDropRegisteredIndex() throws ExecutionException, InterruptedException { + super.testDiscardAndDropRegisteredIndex(); } } diff --git a/janusgraph-es/src/test/java/org/janusgraph/diskstorage/es/CQLElasticsearchTest.java b/janusgraph-es/src/test/java/org/janusgraph/diskstorage/es/CQLElasticsearchTest.java index 4fa030f195..f38c2ae867 100644 --- a/janusgraph-es/src/test/java/org/janusgraph/diskstorage/es/CQLElasticsearchTest.java +++ b/janusgraph-es/src/test/java/org/janusgraph/diskstorage/es/CQLElasticsearchTest.java @@ -14,15 +14,12 @@ package org.janusgraph.diskstorage.es; -import io.github.artsok.RepeatedIfExceptionsTest; import org.janusgraph.JanusGraphCassandraContainer; import org.janusgraph.diskstorage.configuration.ModifiableConfiguration; import org.junit.jupiter.api.Disabled; import org.testcontainers.junit.jupiter.Container; import org.testcontainers.junit.jupiter.Testcontainers; -import java.util.concurrent.ExecutionException; - @Testcontainers public class CQLElasticsearchTest extends ElasticsearchJanusGraphIndexTest { @@ -37,10 +34,4 @@ public ModifiableConfiguration getStorageConfiguration() { @Override @Disabled("CQL seems to not clear storage correctly") public void testClearStorage() {} - - @RepeatedIfExceptionsTest(repeats = 3) - @Override - public void testIndexUpdatesWithoutReindex() throws InterruptedException, ExecutionException { - super.testIndexUpdatesWithoutReindex(); - } } diff --git a/janusgraph-solr/src/test/java/org/janusgraph/diskstorage/solr/CQLSolrTest.java b/janusgraph-solr/src/test/java/org/janusgraph/diskstorage/solr/CQLSolrTest.java index f0433400eb..ddec938844 100644 --- a/janusgraph-solr/src/test/java/org/janusgraph/diskstorage/solr/CQLSolrTest.java +++ b/janusgraph-solr/src/test/java/org/janusgraph/diskstorage/solr/CQLSolrTest.java @@ -14,6 +14,7 @@ package org.janusgraph.diskstorage.solr; +import io.github.artsok.RepeatedIfExceptionsTest; import org.janusgraph.JanusGraphCassandraContainer; import org.janusgraph.diskstorage.configuration.ModifiableConfiguration; import org.junit.jupiter.api.Test; @@ -38,6 +39,13 @@ public boolean supportsWildcardQuery() { return false; } + // flaky test: https://github.com/JanusGraph/janusgraph/issues/3356 + @RepeatedIfExceptionsTest(repeats = 3) + @Override + public void testSetIndexing() { + super.testSetIndexing(); + } + @Test @Override public void testDiscardAndDropRegisteredIndex() { diff --git a/janusgraph-solr/src/test/java/org/janusgraph/diskstorage/solr/SolrJanusGraphIndexTest.java b/janusgraph-solr/src/test/java/org/janusgraph/diskstorage/solr/SolrJanusGraphIndexTest.java index 62bf34bf85..694fbd119b 100644 --- a/janusgraph-solr/src/test/java/org/janusgraph/diskstorage/solr/SolrJanusGraphIndexTest.java +++ b/janusgraph-solr/src/test/java/org/janusgraph/diskstorage/solr/SolrJanusGraphIndexTest.java @@ -81,6 +81,7 @@ public void testClearStorage() throws Exception { super.testClearStorage(); } + // flaky test: https://github.com/JanusGraph/janusgraph/issues/2271 @Override @RepeatedIfExceptionsTest(repeats = 10, suspend = 1000L) public void testIndexReplay() throws Exception { diff --git a/janusgraph-test/src/test/java/org/janusgraph/diskstorage/cache/ExpirationCacheTest.java b/janusgraph-test/src/test/java/org/janusgraph/diskstorage/cache/ExpirationCacheTest.java index 5ee9a601e4..256e4ffc8d 100644 --- a/janusgraph-test/src/test/java/org/janusgraph/diskstorage/cache/ExpirationCacheTest.java +++ b/janusgraph-test/src/test/java/org/janusgraph/diskstorage/cache/ExpirationCacheTest.java @@ -52,7 +52,8 @@ private static KCVSCache getCache(KeyColumnValueStore store, Duration expiration return new ExpirationKCVSCache(store,METRICS_STRING,expirationTime.toMillis(),graceWait.toMillis(),CACHE_SIZE); } - @RepeatedIfExceptionsTest(repeats = 4, minSuccess = 2) + // flaky test: https://github.com/JanusGraph/janusgraph/issues/2934 + @RepeatedIfExceptionsTest(repeats = 3) public void testExpiration() throws Exception { testExpiration(Duration.ofMillis(200)); testExpiration(Duration.ofSeconds(4)); @@ -96,6 +97,7 @@ private void testExpiration(Duration expirationTime) throws Exception { verifyResults(key, keys, query, 4); } + // flaky test: https://github.com/JanusGraph/janusgraph/issues/3352 @RepeatedIfExceptionsTest(repeats = 3) public void testGracePeriod() throws Exception { testGracePeriod(Duration.ofMillis(200)); diff --git a/janusgraph-test/src/test/java/org/janusgraph/graphdb/vertices/StandardVertexTest.java b/janusgraph-test/src/test/java/org/janusgraph/graphdb/vertices/StandardVertexTest.java index 1cf16aac84..b974919166 100644 --- a/janusgraph-test/src/test/java/org/janusgraph/graphdb/vertices/StandardVertexTest.java +++ b/janusgraph-test/src/test/java/org/janusgraph/graphdb/vertices/StandardVertexTest.java @@ -59,7 +59,9 @@ public void setup(){ standardVertex = spy(new StandardVertex(tx, 1, (byte) 1)); } - @RepeatedIfExceptionsTest(repeats = 3, minSuccess = 1) + // This test is flaky simply because it tests a possible deadlock + // https://github.com/JanusGraph/janusgraph/pull/1486 + @RepeatedIfExceptionsTest(repeats = 3) public void shouldNotStuckInDeadlockWhenTheVerticeAndItsRelationIsDeletedInParallel() throws InterruptedException, TimeoutException, ExecutionException {