From 80ca9208826cae07de54caeb6ac339a76f9bde89 Mon Sep 17 00:00:00 2001 From: Florian Hockmann Date: Wed, 6 Sep 2023 14:29:51 +0200 Subject: [PATCH] Mark more tests as flaky This adds the necessary annotation to repeat flaky tests if necessary for tests described in these issues: * #1457 * #1459 * #1462 * #1464 * #1465 * #1497 * #1498 * #2272 * #3142 * #3356 * #3392 * #3393 * #3651 * #3931 * #3959 * #3960 I went through the recent builds on `master` and checked all failed builds. I think this should handle all flaky tests that occurred in these builds, apart from the TinkerPop test where I just don't know how to deal with it: #3841, but we can of course just address that in another PR. We however also sometimes get failed jobs because of some intermediate issues where a backend cannot be properly set up or teared down for some reason or downloading some artifact just fails. I don't know what we can do to improve this, but I would also consider that out of scope of this PR where I just wanted to deal with failing jobs due to specific tests being flaky. Hopefully, this will already lead to significantly fewer cases where we have to manually restart failed jobs (which I had to do a lot lately for all these Dependabot PRs). Signed-off-by: Florian Hockmann --- .../graphdb/JanusGraphIndexTest.java | 2 +- .../JanusGraphOperationCountingTest.java | 3 +- .../BerkeleyOperationCountingTest.java | 7 --- .../cql/CQLConfiguredGraphFactoryTest.java | 8 ++- .../graphdb/cql/CQLGraphCacheTest.java | 55 +++++++++++++++++++ .../janusgraph/graphdb/cql/CQLGraphTest.java | 41 ++++++++++---- .../janusgraph/graphdb/cql/CQLOLAPTest.java | 7 +++ .../es/BerkeleyElasticsearchTest.java | 18 +++++- .../diskstorage/es/CQLElasticsearchTest.java | 9 --- .../diskstorage/solr/CQLSolrTest.java | 7 +++ 10 files changed, 124 insertions(+), 33 deletions(-) diff --git a/janusgraph-backend-testutils/src/main/java/org/janusgraph/graphdb/JanusGraphIndexTest.java b/janusgraph-backend-testutils/src/main/java/org/janusgraph/graphdb/JanusGraphIndexTest.java index 330017f3020..b17592d28ce 100644 --- a/janusgraph-backend-testutils/src/main/java/org/janusgraph/graphdb/JanusGraphIndexTest.java +++ b/janusgraph-backend-testutils/src/main/java/org/janusgraph/graphdb/JanusGraphIndexTest.java @@ -2188,7 +2188,7 @@ public void testIndexReplay() throws Exception { assertEquals(4, recoveryStats[1]); //all 4 index transaction had provoked errors in the indexing backend } - @Test + @RepeatedIfExceptionsTest(repeats = 4, minSuccess = 2) public void testIndexUpdatesWithoutReindex() throws InterruptedException, ExecutionException { final Object[] settings = new Object[]{option(LOG_SEND_DELAY, MANAGEMENT_LOG), Duration.ofMillis(0), option(KCVSLog.LOG_READ_LAG_TIME, MANAGEMENT_LOG), Duration.ofMillis(50), diff --git a/janusgraph-backend-testutils/src/main/java/org/janusgraph/graphdb/JanusGraphOperationCountingTest.java b/janusgraph-backend-testutils/src/main/java/org/janusgraph/graphdb/JanusGraphOperationCountingTest.java index 711d6d583dc..b625df7cbf0 100644 --- a/janusgraph-backend-testutils/src/main/java/org/janusgraph/graphdb/JanusGraphOperationCountingTest.java +++ b/janusgraph-backend-testutils/src/main/java/org/janusgraph/graphdb/JanusGraphOperationCountingTest.java @@ -20,6 +20,7 @@ import com.google.common.collect.ImmutableMap; import com.google.common.collect.Iterables; import com.google.common.collect.Iterators; +import io.github.artsok.RepeatedIfExceptionsTest; import org.apache.tinkerpop.gremlin.structure.Direction; import org.apache.tinkerpop.gremlin.structure.Edge; import org.apache.tinkerpop.gremlin.structure.Vertex; @@ -127,7 +128,7 @@ private void verifyLockingOverwrite(long num) { } } - @Test + @RepeatedIfExceptionsTest(repeats = 4, minSuccess = 2) public void testIdCounts() { makeVertexIndexedUniqueKey("uid", Integer.class); mgmt.setConsistency(mgmt.getGraphIndex("uid"), ConsistencyModifier.LOCK); diff --git a/janusgraph-berkeleyje/src/test/java/org/janusgraph/graphdb/berkeleyje/BerkeleyOperationCountingTest.java b/janusgraph-berkeleyje/src/test/java/org/janusgraph/graphdb/berkeleyje/BerkeleyOperationCountingTest.java index 1df9a51634a..2044fb57d33 100644 --- a/janusgraph-berkeleyje/src/test/java/org/janusgraph/graphdb/berkeleyje/BerkeleyOperationCountingTest.java +++ b/janusgraph-berkeleyje/src/test/java/org/janusgraph/graphdb/berkeleyje/BerkeleyOperationCountingTest.java @@ -14,7 +14,6 @@ package org.janusgraph.graphdb.berkeleyje; -import io.github.artsok.RepeatedIfExceptionsTest; import org.janusgraph.BerkeleyStorageSetup; import org.janusgraph.diskstorage.configuration.WriteConfiguration; import org.janusgraph.graphdb.JanusGraphOperationCountingTest; @@ -27,12 +26,6 @@ public WriteConfiguration getBaseConfiguration() { return BerkeleyStorageSetup.getBerkeleyJEGraphConfiguration(); } - @Override - @RepeatedIfExceptionsTest(repeats = 4, minSuccess = 2) - public void testIdCounts() { - super.testIdCounts(); - } - @AfterEach public void resetCounts() { resetMetrics(); // Metrics is a singleton, so subsequents test runs have wrong counts if we don't clean up. diff --git a/janusgraph-cql/src/test/java/org/janusgraph/core/cql/CQLConfiguredGraphFactoryTest.java b/janusgraph-cql/src/test/java/org/janusgraph/core/cql/CQLConfiguredGraphFactoryTest.java index 704b39b516e..c781fa85eb8 100644 --- a/janusgraph-cql/src/test/java/org/janusgraph/core/cql/CQLConfiguredGraphFactoryTest.java +++ b/janusgraph-cql/src/test/java/org/janusgraph/core/cql/CQLConfiguredGraphFactoryTest.java @@ -109,7 +109,7 @@ public void createConfigurationShouldSupportMultiHosts() throws Exception { } } - @Test + @RepeatedIfExceptionsTest(repeats = 4, minSuccess = 2) public void dropGraphShouldRemoveGraphKeyspace() throws Exception { final MapConfiguration graphConfig = getGraphConfig(); final String graphName = graphConfig.getString(GRAPH_NAME.toStringWithoutRoot()); @@ -137,5 +137,11 @@ public void dropGraphShouldRemoveGraphKeyspace() throws Exception { public void updateConfigurationShouldRemoveGraphFromCache() throws Exception { super.updateConfigurationShouldRemoveGraphFromCache(); } + + @RepeatedIfExceptionsTest(repeats = 4, minSuccess = 2) + @Override + public void dropShouldCleanUpTraversalSourceAndBindings() throws Exception { + super.dropShouldCleanUpTraversalSourceAndBindings(); + } } diff --git a/janusgraph-cql/src/test/java/org/janusgraph/graphdb/cql/CQLGraphCacheTest.java b/janusgraph-cql/src/test/java/org/janusgraph/graphdb/cql/CQLGraphCacheTest.java index d0747658859..752fe28f374 100644 --- a/janusgraph-cql/src/test/java/org/janusgraph/graphdb/cql/CQLGraphCacheTest.java +++ b/janusgraph-cql/src/test/java/org/janusgraph/graphdb/cql/CQLGraphCacheTest.java @@ -14,13 +14,18 @@ package org.janusgraph.graphdb.cql; +import io.github.artsok.ParameterizedRepeatedIfExceptionsTest; +import io.github.artsok.RepeatedIfExceptionsTest; import org.janusgraph.JanusGraphCassandraContainer; import org.janusgraph.StorageSetup; import org.janusgraph.diskstorage.configuration.WriteConfiguration; import org.janusgraph.graphdb.JanusGraphTest; +import org.junit.jupiter.params.provider.ValueSource; import org.testcontainers.junit.jupiter.Container; import org.testcontainers.junit.jupiter.Testcontainers; +import java.util.concurrent.ExecutionException; + @Testcontainers public class CQLGraphCacheTest extends JanusGraphTest { @@ -31,4 +36,54 @@ public class CQLGraphCacheTest extends JanusGraphTest { public WriteConfiguration getConfiguration() { return StorageSetup.addPermanentCache(cqlContainer.getConfiguration(getClass().getSimpleName())); } + + @RepeatedIfExceptionsTest + @Override + public void testIndexUpdatesWithReindexAndRemove() throws InterruptedException, ExecutionException { + super.testIndexUpdatesWithReindexAndRemove(); + } + + @ParameterizedRepeatedIfExceptionsTest(repeats = 4, minSuccess = 2) + @ValueSource(booleans = {true, false}) + @Override + public void simpleLogTest(boolean useStringId) throws InterruptedException { + super.simpleLogTest(useStringId); + } + + @ParameterizedRepeatedIfExceptionsTest(repeats = 4, minSuccess = 2) + @ValueSource(booleans = {true, false}) + @Override + public void simpleLogTestWithFailure(boolean useStringId) throws InterruptedException { + super.simpleLogTestWithFailure(useStringId); + } + + @RepeatedIfExceptionsTest(repeats = 4, minSuccess = 2) + @Override + public void testEdgeTTLTiming() throws Exception { + super.testEdgeTTLTiming(); + } + + @RepeatedIfExceptionsTest(repeats = 4, minSuccess = 2) + @Override + public void testEdgeTTLWithTransactions() throws Exception { + super.testEdgeTTLWithTransactions(); + } + + @RepeatedIfExceptionsTest(repeats = 4, minSuccess = 2) + @Override + public void testVertexTTLWithCompositeIndex() throws Exception { + super.testVertexTTLWithCompositeIndex(); + } + + @RepeatedIfExceptionsTest(repeats = 4, minSuccess = 2) + @Override + public void testVertexTTLImplicitKey() throws Exception { + super.testVertexTTLImplicitKey(); + } + + @RepeatedIfExceptionsTest(repeats = 3) + @Override + public void testReindexingForEdgeIndex() throws ExecutionException, InterruptedException { + super.testReindexingForEdgeIndex(); + } } diff --git a/janusgraph-cql/src/test/java/org/janusgraph/graphdb/cql/CQLGraphTest.java b/janusgraph-cql/src/test/java/org/janusgraph/graphdb/cql/CQLGraphTest.java index 0c1f80102f6..2d245d508ec 100644 --- a/janusgraph-cql/src/test/java/org/janusgraph/graphdb/cql/CQLGraphTest.java +++ b/janusgraph-cql/src/test/java/org/janusgraph/graphdb/cql/CQLGraphTest.java @@ -14,6 +14,7 @@ package org.janusgraph.graphdb.cql; +import io.github.artsok.ParameterizedRepeatedIfExceptionsTest; import io.github.artsok.RepeatedIfExceptionsTest; import org.apache.commons.lang3.RandomStringUtils; import org.apache.commons.lang3.exception.ExceptionUtils; @@ -227,18 +228,36 @@ public void testHasTTL() { assertTrue(features.hasCellTTL()); } - @ParameterizedTest + @ParameterizedRepeatedIfExceptionsTest(repeats = 4, minSuccess = 2) @ValueSource(booleans = {true, false}) - public void simpleLogTest(boolean useStringId) { - for (int i = 0; i < 3; i++) { - try { - super.simpleLogTest(useStringId); - // If the test passes, break out of the loop. - break; - } catch (Exception ex) { - log.info("Attempt #{} fails", i, ex); - } - } + @Override + public void simpleLogTest(boolean useStringId) throws InterruptedException { + super.simpleLogTest(useStringId); + } + + @ParameterizedRepeatedIfExceptionsTest(repeats = 4, minSuccess = 2) + @ValueSource(booleans = {true, false}) + @Override + public void simpleLogTestWithFailure(boolean useStringId) throws InterruptedException { + super.simpleLogTestWithFailure(useStringId); + } + + @RepeatedIfExceptionsTest(repeats = 4, minSuccess = 2) + @Override + public void testEdgeTTLTiming() throws Exception { + super.testEdgeTTLTiming(); + } + + @RepeatedIfExceptionsTest(repeats = 4, minSuccess = 2) + @Override + public void testVertexTTLWithCompositeIndex() throws Exception { + super.testVertexTTLWithCompositeIndex(); + } + + @RepeatedIfExceptionsTest(repeats = 4, minSuccess = 2) + @Override + public void testVertexTTLImplicitKey() throws Exception { + super.testVertexTTLImplicitKey(); } @RepeatedIfExceptionsTest(repeats = 3) diff --git a/janusgraph-cql/src/test/java/org/janusgraph/graphdb/cql/CQLOLAPTest.java b/janusgraph-cql/src/test/java/org/janusgraph/graphdb/cql/CQLOLAPTest.java index 8b33a51969c..83b4be07fa3 100644 --- a/janusgraph-cql/src/test/java/org/janusgraph/graphdb/cql/CQLOLAPTest.java +++ b/janusgraph-cql/src/test/java/org/janusgraph/graphdb/cql/CQLOLAPTest.java @@ -14,6 +14,7 @@ package org.janusgraph.graphdb.cql; +import io.github.artsok.RepeatedIfExceptionsTest; import org.janusgraph.JanusGraphCassandraContainer; import org.janusgraph.diskstorage.configuration.WriteConfiguration; import org.janusgraph.olap.OLAPTest; @@ -29,4 +30,10 @@ public class CQLOLAPTest extends OLAPTest { public WriteConfiguration getConfiguration() { return cqlContainer.getConfiguration(getClass().getSimpleName()).getConfiguration(); } + + @RepeatedIfExceptionsTest(repeats = 4, minSuccess = 2) + @Override + public void testShortestDistance() throws Exception { + super.testShortestDistance(); + } } diff --git a/janusgraph-es/src/test/java/org/janusgraph/diskstorage/es/BerkeleyElasticsearchTest.java b/janusgraph-es/src/test/java/org/janusgraph/diskstorage/es/BerkeleyElasticsearchTest.java index 87d287ca321..97a78730c03 100644 --- a/janusgraph-es/src/test/java/org/janusgraph/diskstorage/es/BerkeleyElasticsearchTest.java +++ b/janusgraph-es/src/test/java/org/janusgraph/diskstorage/es/BerkeleyElasticsearchTest.java @@ -41,6 +41,12 @@ public ModifiableConfiguration getStorageConfiguration() { return getBerkeleyJEConfiguration(); } + @RepeatedIfExceptionsTest(repeats = 4, minSuccess = 2) + @Override + public void indexShouldNotExistAfterDeletion() throws Exception { + super.indexShouldNotExistAfterDeletion(); + } + /** * Test {@link org.janusgraph.example.GraphOfTheGodsFactory#create(String)}. */ @@ -55,9 +61,15 @@ public void testGraphOfTheGodsFactoryCreate() { gotg.close(); } - @RepeatedIfExceptionsTest(repeats = 3) + @RepeatedIfExceptionsTest(repeats = 4, minSuccess = 2) + @Override + public void testDisableAndDiscardManuallyAndDropEnabledIndex() throws Exception { + super.testDisableAndDiscardManuallyAndDropEnabledIndex(); + } + + @RepeatedIfExceptionsTest(repeats = 4, minSuccess = 2) @Override - public void testIndexUpdatesWithoutReindex() throws InterruptedException, ExecutionException { - super.testIndexUpdatesWithoutReindex(); + public void testDiscardAndDropRegisteredIndex() throws ExecutionException, InterruptedException { + super.testDiscardAndDropRegisteredIndex(); } } diff --git a/janusgraph-es/src/test/java/org/janusgraph/diskstorage/es/CQLElasticsearchTest.java b/janusgraph-es/src/test/java/org/janusgraph/diskstorage/es/CQLElasticsearchTest.java index 4fa030f1951..f38c2ae8677 100644 --- a/janusgraph-es/src/test/java/org/janusgraph/diskstorage/es/CQLElasticsearchTest.java +++ b/janusgraph-es/src/test/java/org/janusgraph/diskstorage/es/CQLElasticsearchTest.java @@ -14,15 +14,12 @@ package org.janusgraph.diskstorage.es; -import io.github.artsok.RepeatedIfExceptionsTest; import org.janusgraph.JanusGraphCassandraContainer; import org.janusgraph.diskstorage.configuration.ModifiableConfiguration; import org.junit.jupiter.api.Disabled; import org.testcontainers.junit.jupiter.Container; import org.testcontainers.junit.jupiter.Testcontainers; -import java.util.concurrent.ExecutionException; - @Testcontainers public class CQLElasticsearchTest extends ElasticsearchJanusGraphIndexTest { @@ -37,10 +34,4 @@ public ModifiableConfiguration getStorageConfiguration() { @Override @Disabled("CQL seems to not clear storage correctly") public void testClearStorage() {} - - @RepeatedIfExceptionsTest(repeats = 3) - @Override - public void testIndexUpdatesWithoutReindex() throws InterruptedException, ExecutionException { - super.testIndexUpdatesWithoutReindex(); - } } diff --git a/janusgraph-solr/src/test/java/org/janusgraph/diskstorage/solr/CQLSolrTest.java b/janusgraph-solr/src/test/java/org/janusgraph/diskstorage/solr/CQLSolrTest.java index f0433400ebe..a117c4af052 100644 --- a/janusgraph-solr/src/test/java/org/janusgraph/diskstorage/solr/CQLSolrTest.java +++ b/janusgraph-solr/src/test/java/org/janusgraph/diskstorage/solr/CQLSolrTest.java @@ -14,6 +14,7 @@ package org.janusgraph.diskstorage.solr; +import io.github.artsok.RepeatedIfExceptionsTest; import org.janusgraph.JanusGraphCassandraContainer; import org.janusgraph.diskstorage.configuration.ModifiableConfiguration; import org.junit.jupiter.api.Test; @@ -38,6 +39,12 @@ public boolean supportsWildcardQuery() { return false; } + @RepeatedIfExceptionsTest(repeats = 4, minSuccess = 2) + @Override + public void testSetIndexing() { + super.testSetIndexing(); + } + @Test @Override public void testDiscardAndDropRegisteredIndex() {