From e3f17aec94d358cfcc8047ed51a215088fe04451 Mon Sep 17 00:00:00 2001 From: emreyigit Date: Fri, 23 Aug 2024 13:16:29 +0300 Subject: [PATCH 01/20] Fix Partition Group selection on ClusterView event. --- .../Clustering/MemberPartitionGroupTests.cs | 71 ++++++++++++++++--- src/Hazelcast.Net/Clustering/ClusterEvents.cs | 15 ++-- .../Clustering/ClusterMembers.cs | 2 +- 3 files changed, 73 insertions(+), 15 deletions(-) diff --git a/src/Hazelcast.Net.Tests/Clustering/MemberPartitionGroupTests.cs b/src/Hazelcast.Net.Tests/Clustering/MemberPartitionGroupTests.cs index c9dff491e..699f51a9b 100644 --- a/src/Hazelcast.Net.Tests/Clustering/MemberPartitionGroupTests.cs +++ b/src/Hazelcast.Net.Tests/Clustering/MemberPartitionGroupTests.cs @@ -25,6 +25,7 @@ using Hazelcast.Messaging; using Hazelcast.Models; using Hazelcast.Networking; +using Hazelcast.Partitioning; using Hazelcast.Protocol.Codecs; using Hazelcast.Protocol.Models; using Hazelcast.Serialization; @@ -34,6 +35,7 @@ using Hazelcast.Testing.TestServer; using Microsoft.Extensions.Logging.Abstractions; using NUnit.Framework; +using static NSubstitute.Substitute; namespace Hazelcast.Tests.Clustering { // internal for class MemberPartitionGroup : ISubsetClusterMembers @@ -50,8 +52,8 @@ public async Task TearDown() } } - [TestCase(@"[{ ""raftId"":{ ""seed"":9, ""id"":3, ""name"":""grp1"" }, ""leaderUUID"":""fa270257-5767-45bf-a3c6-bafe17bed525"" }]", "grp1",9, 3, "fa270257-5767-45bf-a3c6-bafe17bed525",1)] - [TestCase(@"[]", "grp1",0, 0, "",0)] + [TestCase(@"[{ ""raftId"":{ ""seed"":9, ""id"":3, ""name"":""grp1"" }, ""leaderUUID"":""fa270257-5767-45bf-a3c6-bafe17bed525"" }]", "grp1", 9, 3, "fa270257-5767-45bf-a3c6-bafe17bed525", 1)] + [TestCase(@"[]", "grp1", 0, 0, "", 0)] public void TestAuthenticatorCanParseCPGroups(string jsonString, string groupName, int seed, int id, string leaderId, int count) { ClientAuthenticationCodec.ResponseParameters response = new ClientAuthenticationCodec.ResponseParameters(); @@ -60,16 +62,16 @@ public void TestAuthenticatorCanParseCPGroups(string jsonString, string groupNam response.IsKeyValuePairsExists = true; var authenticator = CreateAuthenticator(); - + var cpGroups = authenticator.ParseCPGroupLeaderIds(response); - + Assert.That(cpGroups.Count, Is.EqualTo(count)); foreach (var g in cpGroups) { Assert.That(g.Key.Name, Is.EqualTo(groupName)); Assert.That(g.Key.Seed, Is.EqualTo(seed)); Assert.That(g.Key.Id, Is.EqualTo(id)); - Assert.That(g.Value, Is.EqualTo(Guid.Parse(leaderId))); + Assert.That(g.Value, Is.EqualTo(Guid.Parse(leaderId))); } } @@ -131,6 +133,55 @@ public void TestMemberPartitionGroupPicksCorrectGroup(MemberGroups group1, Assert.That(((MemberPartitionGroup) memberPartitionGroup).CurrentGroups.SelectedGroup.Count, Is.EqualTo(expectedGroupSize)); } + [TestCaseSource(nameof(MemberGroupCases))] + public void TestMemberFilteringWorks(MemberGroups group1, + MemberGroups group2, + Guid expectedMemberId, + int expectedVersion, + int expectedGroupSize) + { + // Create a list of members + var memberIds = group1.Groups.SelectMany(p => p).Distinct().ToList(); + memberIds.AddRange(group2.Groups.SelectMany(p => p).Distinct()); + memberIds = memberIds.Distinct().ToList(); + var members = memberIds.Select(p => new MemberInfo(p, new NetworkAddress("127.0.0.1"), + new MemberVersion(5, 5, 0), false, new Dictionary())).ToList(); + + // Prepare the member partition group + ISubsetClusterMembers memberPartitionGroup = new MemberPartitionGroup(new NetworkingOptions(), NullLogger.Instance); + + var clusterMembers = For(For(new HazelcastOptions(), null, null, For(), NullLoggerFactory.Instance), For(NullLoggerFactory.Instance), memberPartitionGroup); + + memberPartitionGroup.SetSubsetMembers(group1); + + // Assume that MembersView is received from the server, and picking the filtered members for MultiMember mode. + var filteredMembers = clusterMembers.FilterMembers(members).Select(p => p.Id).ToList(); + + Assert.That(filteredMembers.Count, Is.EqualTo(group1.SelectedGroup.Count)); + foreach (var memberId in filteredMembers) + { + Assert.That(group1.SelectedGroup.Contains(memberId), Is.True, + $"Member {memberId} is not in the selected group [{string.Join(", ", group1.SelectedGroup.ToArray())}]"); + } + + if (group2.Version != MemberPartitionGroup.InvalidVersion) + { + //Assume new CP group is received from the server, then MemberView is received from the server. + memberPartitionGroup.SetSubsetMembers(group2); + filteredMembers = clusterMembers.FilterMembers(members).Select(p => p.Id).ToList(); + + // If Group 2 is selected than assert members. + if (((MemberPartitionGroup) memberPartitionGroup).CurrentGroups.MemberReceivedFrom == group2.MemberReceivedFrom) + { + foreach (var memberId in filteredMembers) + { + Assert.That(group2.SelectedGroup.Contains(memberId), Is.True, + $"Member {memberId} is not in the selected group [{string.Join(", ", group2.SelectedGroup.ToArray())}]"); + } + } + } + } + [Test] public async Task TestServerNotSupportMultiMemberRoutingAndClusterVersion() { @@ -322,7 +373,7 @@ await AssertEx.SucceedsEventually(() => }, 20_000, 200); } - + // Mock server over real TPC connection private static async Task CreateClient(RoutingModes routingMode = RoutingModes.MultiMember, params NetworkAddress[] addresses) { @@ -467,7 +518,7 @@ private async ValueTask ServerHandler(ClientRequest request) break; } - + // unexpected message = error default: { @@ -480,6 +531,9 @@ private async ValueTask ServerHandler(ClientRequest request) } + + + public static object[] MemberGroupCases = { // Case 1: New version wins. @@ -490,7 +544,8 @@ private async ValueTask ServerHandler(ClientRequest request) { // Selected Group new List() { Guid.Parse("64082773-bc1b-408c-8ea6-1150c3c6477e"), Guid.Parse("bfc5a01b-884f-421b-b8a0-a7ce643b4085") }, - new List() { Guid.Parse("efdb670f-d0d5-4482-84eb-0354e4278112"), Guid.Parse("6965aaa2-d6eb-483a-bb6c-99388c348bc6") } + new List() { Guid.Parse("efdb670f-d0d5-4482-84eb-0354e4278112"), Guid.Parse("6965aaa2-d6eb-483a-bb6c-99388c348bc6"), Guid.Parse("07988585-f4cf-4473-8299-5575fa6bc93a") }, + new List() { Guid.Parse("a722ab7a-d266-4911-91b3-5f9f9c7dbcab"), Guid.Parse("b6c07420-3971-4982-bdd4-4ad52b355b7a"), Guid.Parse("e29f6919-7d8a-4f44-91fe-0b48376993a2") } }, 1, Guid.Parse("81b1ac67-1238-42d6-84b7-ef869e60f262"), diff --git a/src/Hazelcast.Net/Clustering/ClusterEvents.cs b/src/Hazelcast.Net/Clustering/ClusterEvents.cs index 4da1dfff4..389614e7c 100644 --- a/src/Hazelcast.Net/Clustering/ClusterEvents.cs +++ b/src/Hazelcast.Net/Clustering/ClusterEvents.cs @@ -105,9 +105,9 @@ public ClusterEvents(ClusterState clusterState, ClusterMessaging clusterMessagin // Register cluster views _clusterViewProperties[typeof(ClientAddClusterViewListenerCodec)] = new ClusterViewProperties(SubscribeToClusterViewsAsync, typeof(ClientAddClusterViewListenerCodec)); - + // Subscribe when CP direct to leader is enabled - if(_clusterState.Options.Networking.CPDirectToLeaderEnabled) + if (_clusterState.Options.Networking.CPDirectToLeaderEnabled) _clusterViewProperties[typeof(ClientAddCPGroupViewListenerCodec)] = new ClusterViewProperties(SubscribeToClusterCPViewAsync, typeof(ClientAddCPGroupViewListenerCodec)); _logger = _clusterState.LoggerFactory.CreateLogger(); @@ -697,7 +697,7 @@ private async ValueTask HandleCodecPartitionViewEvent(int version, IList> memberGroups, object state, Guid clusterId, Guid memberId) { _logger.IfDebug()?.LogDebug("Handle MemberGroups event for cluster {ClusterId} and member {MemberId}. Received version:{Version} Member Groups: [{Groups}]", - clusterId, memberId, version, (memberGroups == null ? "null" : string.Join(", ", memberGroups.Select(x => string.Join(", ", x))))); - _clusterMembers.SubsetClusterMembers.SetSubsetMembers(new MemberGroups(memberGroups, version, clusterId, memberId)); + clusterId, memberId, version, (memberGroups == null ? "null" : string.Join(", ", memberGroups.Select(x => $"[{string.Join(", ", x)}]")))); + // Received memberId is Guid.Empty because we don't want to change the current member + // group of the member that received the event during auth. + _clusterMembers.SubsetClusterMembers + .SetSubsetMembers(new MemberGroups(memberGroups, version, clusterId, Guid.Empty)); await _memberPartitionGroupsUpdated.AwaitEach().CfAwait(); } @@ -861,7 +864,7 @@ public Func PartitionsUpdated _partitionsUpdated = value; } } - + /// /// Internal event to emit when member partition groups have been updated. /// diff --git a/src/Hazelcast.Net/Clustering/ClusterMembers.cs b/src/Hazelcast.Net/Clustering/ClusterMembers.cs index 46b7ad387..7069dd9c0 100644 --- a/src/Hazelcast.Net/Clustering/ClusterMembers.cs +++ b/src/Hazelcast.Net/Clustering/ClusterMembers.cs @@ -903,7 +903,7 @@ public IEnumerable GetMembers(bool liteOnly = false) /// Members allowed to connect public IEnumerable GetMembersForConnection() { - return _members.Version == InvalidMemberTableVersion ? Enumerable.Empty() : _filteredMembersToConnect.Members; + return _filteredMembersToConnect.Version == InvalidMemberTableVersion ? Enumerable.Empty() : _filteredMembersToConnect.Members; } /// From 65bc83b6705ff1cc6bc4332d9f45bbc3b5743154 Mon Sep 17 00:00:00 2001 From: emreyigit Date: Fri, 23 Aug 2024 14:40:32 +0300 Subject: [PATCH 02/20] Fix most overlapped group selection --- .../Clustering/MemberPartitionGroupTests.cs | 153 ++++++++++++++++-- .../Clustering/MemberPartitionGroup.cs | 21 ++- 2 files changed, 151 insertions(+), 23 deletions(-) diff --git a/src/Hazelcast.Net.Tests/Clustering/MemberPartitionGroupTests.cs b/src/Hazelcast.Net.Tests/Clustering/MemberPartitionGroupTests.cs index 699f51a9b..7d04a4493 100644 --- a/src/Hazelcast.Net.Tests/Clustering/MemberPartitionGroupTests.cs +++ b/src/Hazelcast.Net.Tests/Clustering/MemberPartitionGroupTests.cs @@ -122,12 +122,15 @@ public void TestMemberPartitionGroupPicksCorrectGroup(MemberGroups group1, ISubsetClusterMembers memberPartitionGroup = new MemberPartitionGroup(new NetworkingOptions(), NullLogger.Instance); - memberPartitionGroup.SetSubsetMembers(group1); - if (group2.Version != MemberPartitionGroup.InvalidVersion) + for (int i = 0; i < 2; i++) { - memberPartitionGroup.SetSubsetMembers(group2); + memberPartitionGroup.SetSubsetMembers(group1); + if (group2.Version != MemberPartitionGroup.InvalidVersion) + { + memberPartitionGroup.SetSubsetMembers(group2); + } } - + Assert.AreEqual(expectedVersion, ((MemberPartitionGroup) memberPartitionGroup).CurrentGroups.Version); Assert.That(memberPartitionGroup.GetSubsetMemberIds(), Contains.Item(expectedMemberId)); Assert.That(((MemberPartitionGroup) memberPartitionGroup).CurrentGroups.SelectedGroup.Count, Is.EqualTo(expectedGroupSize)); @@ -150,7 +153,9 @@ public void TestMemberFilteringWorks(MemberGroups group1, // Prepare the member partition group ISubsetClusterMembers memberPartitionGroup = new MemberPartitionGroup(new NetworkingOptions(), NullLogger.Instance); - var clusterMembers = For(For(new HazelcastOptions(), null, null, For(), NullLoggerFactory.Instance), For(NullLoggerFactory.Instance), memberPartitionGroup); + var clusterMembers = For(For(new HazelcastOptions(), + null, null, For(), NullLoggerFactory.Instance), + For(NullLoggerFactory.Instance), memberPartitionGroup); memberPartitionGroup.SetSubsetMembers(group1); @@ -595,7 +600,7 @@ private async ValueTask ServerHandler(ClientRequest request) }, 1, Guid.Parse("81b1ac67-1238-42d6-84b7-ef869e60f262"), - Guid.Parse("64082773-bc1b-408c-8ea6-1150c3c6477e")), + Guid.Empty), Guid.Parse("64082773-bc1b-408c-8ea6-1150c3c6477e"), // Expected group member 1, // Expected version @@ -608,10 +613,15 @@ private async ValueTask ServerHandler(ClientRequest request) // Group 1 new MemberGroups(new List>() { - new List() { Guid.Parse("efdb670f-d0d5-4482-84eb-0354e4278112"), Guid.Parse("6965aaa2-d6eb-483a-bb6c-99388c348bc6") }, new List() { - Guid.Parse("64082773-bc1b-408c-8ea6-1150c3c6477e"), Guid.Parse("d3c34048-a055-4025-8c00-c70b2dcd47b9") + Guid.Parse("efdb670f-d0d5-4482-84eb-0354e4278112"), + Guid.Parse("6965aaa2-d6eb-483a-bb6c-99388c348bc6") + }, + new List() + { + Guid.Parse("64082773-bc1b-408c-8ea6-1150c3c6477e"), + Guid.Parse("d3c34048-a055-4025-8c00-c70b2dcd47b9") } }, 1, @@ -624,30 +634,40 @@ private async ValueTask ServerHandler(ClientRequest request) // Selected Group new List() { - Guid.Parse("64082773-bc1b-408c-8ea6-1150c3c6477e"), Guid.Parse("d3c34048-a055-4025-8c00-c70b2dcd47b9"), + Guid.Parse("64082773-bc1b-408c-8ea6-1150c3c6477e"), + Guid.Parse("d3c34048-a055-4025-8c00-c70b2dcd47b9"), Guid.Parse("03d9a78b-6380-49c8-9bf3-8044f2cfa75d") }, - new List() { Guid.Parse("dd8803e6-16ae-4d62-921e-ffe4f5c8ce49"), Guid.Parse("6965aaa2-d6eb-483a-bb6c-99388c348bc6") } + new List() + { + Guid.Parse("dd8803e6-16ae-4d62-921e-ffe4f5c8ce49"), + Guid.Parse("6965aaa2-d6eb-483a-bb6c-99388c348bc6") + } }, 1, Guid.Parse("a810df4f-a54c-437d-a945-99218688cf31"), - Guid.Parse("64082773-bc1b-408c-8ea6-1150c3c6477e")), + Guid.Empty), Guid.Parse("64082773-bc1b-408c-8ea6-1150c3c6477e"), // Expected group member 1, // Expected version 3 // Expected group size }, - // Case 2: Biggest wins + // Case 3: Biggest wins new object[] { // Group 1 new MemberGroups(new List>() { - new List() { Guid.Parse("efdb670f-d0d5-4482-84eb-0354e4278112"), Guid.Parse("6965aaa2-d6eb-483a-bb6c-99388c348bc6") }, new List() { - Guid.Parse("64082773-bc1b-408c-8ea6-1150c3c6477e"), Guid.Parse("d3c34048-a055-4025-8c00-c70b2dcd47b9") + Guid.Parse("efdb670f-d0d5-4482-84eb-0354e4278112"), + Guid.Parse("6965aaa2-d6eb-483a-bb6c-99388c348bc6") + }, + new List() + { + Guid.Parse("64082773-bc1b-408c-8ea6-1150c3c6477e"), + Guid.Parse("d3c34048-a055-4025-8c00-c70b2dcd47b9") } }, 1, @@ -660,10 +680,15 @@ private async ValueTask ServerHandler(ClientRequest request) // Selected Group new List() { - Guid.Parse("64082773-bc1b-408c-8ea6-1150c3c6477e"), Guid.Parse("d3c34048-a055-4025-8c00-c70b2dcd47b9"), + Guid.Parse("64082773-bc1b-408c-8ea6-1150c3c6477e"), + Guid.Parse("d3c34048-a055-4025-8c00-c70b2dcd47b9"), Guid.Parse("03d9a78b-6380-49c8-9bf3-8044f2cfa75d") }, - new List() { Guid.Parse("dd8803e6-16ae-4d62-921e-ffe4f5c8ce49"), Guid.Parse("6965aaa2-d6eb-483a-bb6c-99388c348bc6") } + new List() + { + Guid.Parse("dd8803e6-16ae-4d62-921e-ffe4f5c8ce49"), + Guid.Parse("6965aaa2-d6eb-483a-bb6c-99388c348bc6") + } }, 1, Guid.Parse("a810df4f-a54c-437d-a945-99218688cf31"), @@ -674,6 +699,102 @@ private async ValueTask ServerHandler(ClientRequest request) 3 // Expected group size }, + // Case 4: PG Group not changing after auth. + new object[] + { + // Group 1 + new MemberGroups(new List>() + { + new List() + { + Guid.Parse("efdb670f-d0d5-4482-84eb-0354e4278112"), + Guid.Parse("6965aaa2-d6eb-483a-bb6c-99388c348bc6"), + Guid.Parse("d8ee9c15-ac9a-4357-9698-ade761ced554") + }, + // Selected Group + new List() + { + Guid.Parse("64082773-bc1b-408c-8ea6-1150c3c6477e"), + Guid.Parse("d3c34048-a055-4025-8c00-c70b2dcd47b9") + } + }, + 1, + Guid.Parse("81b1ac67-1238-42d6-84b7-ef869e60f262"), + Guid.Parse("64082773-bc1b-408c-8ea6-1150c3c6477e")), + + // Group 2 + new MemberGroups(new List>() + { + new List() + { + Guid.Parse("efdb670f-d0d5-4482-84eb-0354e4278112"), + Guid.Parse("6965aaa2-d6eb-483a-bb6c-99388c348bc6"), + Guid.Parse("d8ee9c15-ac9a-4357-9698-ade761ced554"), + Guid.Parse("79d63bcf-339d-449b-aa55-a2cb4f3bad8b") + }, + new List() + { + Guid.Parse("64082773-bc1b-408c-8ea6-1150c3c6477e"), + Guid.Parse("d3c34048-a055-4025-8c00-c70b2dcd47b9") + } + }, + 2, + Guid.Parse("81b1ac67-1238-42d6-84b7-ef869e60f262"), + Guid.Empty), + Guid.Parse("64082773-bc1b-408c-8ea6-1150c3c6477e"), // Expected group member + 2, // Expected version + 2 // Expected group size + }, + + // Case 5: Bigger overlap with auth group wins. + new object[] + { + // Group 1 + new MemberGroups(new List>() + { + new List() + { + Guid.Parse("efdb670f-d0d5-4482-84eb-0354e4278112"), + Guid.Parse("6965aaa2-d6eb-483a-bb6c-99388c348bc6"), + Guid.Parse("d8ee9c15-ac9a-4357-9698-ade761ced554") + }, + // Selected Group + new List() + { + Guid.Parse("64082773-bc1b-408c-8ea6-1150c3c6477e"), + Guid.Parse("d3c34048-a055-4025-8c00-c70b2dcd47b9") + } + }, + 1, + Guid.Parse("81b1ac67-1238-42d6-84b7-ef869e60f262"), + Guid.Parse("64082773-bc1b-408c-8ea6-1150c3c6477e")), + + // Group 2 + new MemberGroups(new List>() + { + new List() + { + Guid.Parse("efdb670f-d0d5-4482-84eb-0354e4278112"), + Guid.Parse("6965aaa2-d6eb-483a-bb6c-99388c348bc6"), + Guid.Parse("d8ee9c15-ac9a-4357-9698-ade761ced554"), + Guid.Parse("79d63bcf-339d-449b-aa55-a2cb4f3bad8b") + }, + new List() + { + Guid.Parse("64082773-bc1b-408c-8ea6-1150c3c6477e"), + Guid.Parse("d3c34048-a055-4025-8c00-c70b2dcd47b9"), + Guid.Parse("6878f2be-5153-4b7a-8896-edab76011c9c"), + Guid.Parse("7bff264d-426d-44e3-928e-a6200a0a5271"), + Guid.Parse("ce74ca59-3061-47c4-b56b-ddf5727fa312") + } + }, + 2, + Guid.Parse("81b1ac67-1238-42d6-84b7-ef869e60f262"), + Guid.Empty), + Guid.Parse("64082773-bc1b-408c-8ea6-1150c3c6477e"), // Expected group member + 2, // Expected version + 5 // Expected group size + }, }; } diff --git a/src/Hazelcast.Net/Clustering/MemberPartitionGroup.cs b/src/Hazelcast.Net/Clustering/MemberPartitionGroup.cs index 3c23efe9b..df8e2965a 100644 --- a/src/Hazelcast.Net/Clustering/MemberPartitionGroup.cs +++ b/src/Hazelcast.Net/Clustering/MemberPartitionGroup.cs @@ -29,7 +29,7 @@ internal class MemberPartitionGroup : ISubsetClusterMembers public const string PartitionGroupJsonField = "groups"; public const int InvalidVersion = -1; - private NetworkingOptions _networkingOptions; + private NetworkingOptions _networkingOptions; private ILogger _logger; private readonly ReaderWriterLockSlim _mutex = new ReaderWriterLockSlim(); private MemberGroups _currentGroups @@ -58,9 +58,9 @@ internal MemberGroups PickBestGroup(MemberGroups newGroup) return newGroup; } - if (_currentGroups.SelectedGroup.Count == 0) + if (_currentGroups.SelectedGroup.Count > 0) { - var pickedGroup = GetMostOverlappedGroup(newGroup.ClusterId, newGroup.MemberReceivedFrom, newGroup); + var pickedGroup = GetMostOverlappedGroup(newGroup.ClusterId, _currentGroups.MemberReceivedFrom, newGroup); if (pickedGroup.SelectedGroup.Count > 0) return pickedGroup; @@ -81,8 +81,12 @@ internal MemberGroups GetMostOverlappedGroup(Guid clusterId, Guid memberIdOfGrou // Find the group that has the most overlap with the given groups. var maxOverlap = 0; ICollection mostOverlappedGroup = null; + foreach (var examinedGroup in newGroups.Groups) { + if (examinedGroup.Contains(memberIdOfGroup) == false) + continue; + var overlap = _currentGroups.SelectedGroup.Intersect(examinedGroup).Count(); if (overlap > maxOverlap) { @@ -90,8 +94,11 @@ internal MemberGroups GetMostOverlappedGroup(Guid clusterId, Guid memberIdOfGrou mostOverlappedGroup = examinedGroup; } } - - return mostOverlappedGroup is { Count: > 0 } ? newGroups : _currentGroups; + + return mostOverlappedGroup is { Count: > 0 } + // Selected the new group that has the most overlap with the current group. + ? new MemberGroups(newGroups.Groups, newGroups.Version, newGroups.ClusterId, memberIdOfGroup) + : _currentGroups; } // internal for testing @@ -99,7 +106,7 @@ internal MemberGroups GetBiggestGroup(MemberGroups newGroup) { var maxCount = int.MinValue; IList biggestGroup = null; - + for (var i = 0; i < newGroup.Groups.Count; i++) { if (newGroup.Groups[i].Count > maxCount) @@ -153,7 +160,7 @@ public void SetSubsetMembers(MemberGroups newGroup) _mutex.ExitUpgradeableReadLock(); } } - + public void RemoveSubsetMember(Guid memberId) { try From 9a1ebb892752c42b949b5d52c866e67b66135e99 Mon Sep 17 00:00:00 2001 From: emreyigit Date: Fri, 23 Aug 2024 16:34:47 +0300 Subject: [PATCH 03/20] More detailed login on member removal from group. --- .../Clustering/MemberPartitionGroupTests.cs | 80 ++++++++++++++++++- .../Clustering/MemberPartitionGroup.cs | 3 +- 2 files changed, 78 insertions(+), 5 deletions(-) diff --git a/src/Hazelcast.Net.Tests/Clustering/MemberPartitionGroupTests.cs b/src/Hazelcast.Net.Tests/Clustering/MemberPartitionGroupTests.cs index 7d04a4493..f9dea9556 100644 --- a/src/Hazelcast.Net.Tests/Clustering/MemberPartitionGroupTests.cs +++ b/src/Hazelcast.Net.Tests/Clustering/MemberPartitionGroupTests.cs @@ -110,7 +110,29 @@ private static Authenticator CreateAuthenticator() return authenticator; } - // TODO: add integration tests after having config. + + [TestCaseSource(nameof(MemberGroupCases))] + public void TestMemberPartitionGroupRemovesCorrectMember(MemberGroups group1, + MemberGroups group2, + Guid expectedMemberId, + int expectedVersion, + int expectedGroupSize) + { + + ISubsetClusterMembers memberPartitionGroup = new MemberPartitionGroup(new NetworkingOptions(), NullLogger.Instance); + + memberPartitionGroup.SetSubsetMembers(group1); + memberPartitionGroup.SetSubsetMembers(group2); + + var removedId = memberPartitionGroup.GetSubsetMemberIds().Where(p => p != expectedMemberId).FirstOrDefault(); + memberPartitionGroup.RemoveSubsetMember(removedId); + + Assert.AreEqual(expectedVersion, ((MemberPartitionGroup) memberPartitionGroup).CurrentGroups.Version); + Assert.That(memberPartitionGroup.GetSubsetMemberIds(), Is.Not.Contains(removedId)); + + if (removedId != Guid.Empty) + Assert.That(((MemberPartitionGroup) memberPartitionGroup).CurrentGroups.SelectedGroup.Count, Is.EqualTo(expectedGroupSize - 1)); + } [TestCaseSource(nameof(MemberGroupCases))] public void TestMemberPartitionGroupPicksCorrectGroup(MemberGroups group1, @@ -128,9 +150,9 @@ public void TestMemberPartitionGroupPicksCorrectGroup(MemberGroups group1, if (group2.Version != MemberPartitionGroup.InvalidVersion) { memberPartitionGroup.SetSubsetMembers(group2); - } + } } - + Assert.AreEqual(expectedVersion, ((MemberPartitionGroup) memberPartitionGroup).CurrentGroups.Version); Assert.That(memberPartitionGroup.GetSubsetMemberIds(), Contains.Item(expectedMemberId)); Assert.That(((MemberPartitionGroup) memberPartitionGroup).CurrentGroups.SelectedGroup.Count, Is.EqualTo(expectedGroupSize)); @@ -745,7 +767,7 @@ private async ValueTask ServerHandler(ClientRequest request) 2, // Expected version 2 // Expected group size }, - + // Case 5: Bigger overlap with auth group wins. new object[] { @@ -796,6 +818,56 @@ private async ValueTask ServerHandler(ClientRequest request) 5 // Expected group size }, + // Case 5: Scale down + new object[] + { + // Group 1 + new MemberGroups(new List>() + { + new List() + { + Guid.Parse("efdb670f-d0d5-4482-84eb-0354e4278112"), + Guid.Parse("6965aaa2-d6eb-483a-bb6c-99388c348bc6"), + Guid.Parse("d8ee9c15-ac9a-4357-9698-ade761ced554"), + Guid.Parse("79d63bcf-339d-449b-aa55-a2cb4f3bad8b") + }, + // Selected Group + new List() + { + Guid.Parse("64082773-bc1b-408c-8ea6-1150c3c6477e"), + Guid.Parse("d3c34048-a055-4025-8c00-c70b2dcd47b9"), + Guid.Parse("6878f2be-5153-4b7a-8896-edab76011c9c"), + Guid.Parse("7bff264d-426d-44e3-928e-a6200a0a5271"), + Guid.Parse("ce74ca59-3061-47c4-b56b-ddf5727fa312") + } + }, + 1, + Guid.Parse("81b1ac67-1238-42d6-84b7-ef869e60f262"), + Guid.Parse("64082773-bc1b-408c-8ea6-1150c3c6477e")), + + // Group 2 + new MemberGroups(new List>() + { + new List() + { + Guid.Parse("efdb670f-d0d5-4482-84eb-0354e4278112"), + Guid.Parse("6965aaa2-d6eb-483a-bb6c-99388c348bc6"), + Guid.Parse("d8ee9c15-ac9a-4357-9698-ade761ced554"), + Guid.Parse("79d63bcf-339d-449b-aa55-a2cb4f3bad8b") + }, + new List() + { + Guid.Parse("64082773-bc1b-408c-8ea6-1150c3c6477e") + } + }, + 2, + Guid.Parse("81b1ac67-1238-42d6-84b7-ef869e60f262"), + Guid.Empty), + Guid.Parse("64082773-bc1b-408c-8ea6-1150c3c6477e"), // Expected group member + 2, // Expected version + 1 // Expected group size + }, + }; } } diff --git a/src/Hazelcast.Net/Clustering/MemberPartitionGroup.cs b/src/Hazelcast.Net/Clustering/MemberPartitionGroup.cs index df8e2965a..dd061c05d 100644 --- a/src/Hazelcast.Net/Clustering/MemberPartitionGroup.cs +++ b/src/Hazelcast.Net/Clustering/MemberPartitionGroup.cs @@ -180,8 +180,9 @@ public void RemoveSubsetMember(Guid memberId) } var newGroup = new MemberGroups(clearedGroup, _currentGroups.Version, _currentGroups.ClusterId, _currentGroups.MemberReceivedFrom); - + var old = _currentGroups; _currentGroups = newGroup.SelectedGroup.Count > 0 ? newGroup : new MemberGroups(new List>(0), MemberPartitionGroup.InvalidVersion, Guid.Empty, Guid.Empty); + _logger.IfDebug()?.LogDebug("Removed Member[{MemberId}] and updated member partition group. Old group: {OldGroup} New group: {PickedGroup}", memberId, old, _currentGroups); } } finally From 60be92865668ac46abb2d78fe0eab54b562b6308 Mon Sep 17 00:00:00 2001 From: emreyigit Date: Fri, 23 Aug 2024 16:44:02 +0300 Subject: [PATCH 04/20] Fix the removal. --- src/Hazelcast.Net/Clustering/MemberPartitionGroup.cs | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/src/Hazelcast.Net/Clustering/MemberPartitionGroup.cs b/src/Hazelcast.Net/Clustering/MemberPartitionGroup.cs index dd061c05d..56732deae 100644 --- a/src/Hazelcast.Net/Clustering/MemberPartitionGroup.cs +++ b/src/Hazelcast.Net/Clustering/MemberPartitionGroup.cs @@ -181,8 +181,16 @@ public void RemoveSubsetMember(Guid memberId) var newGroup = new MemberGroups(clearedGroup, _currentGroups.Version, _currentGroups.ClusterId, _currentGroups.MemberReceivedFrom); var old = _currentGroups; - _currentGroups = newGroup.SelectedGroup.Count > 0 ? newGroup : new MemberGroups(new List>(0), MemberPartitionGroup.InvalidVersion, Guid.Empty, Guid.Empty); - _logger.IfDebug()?.LogDebug("Removed Member[{MemberId}] and updated member partition group. Old group: {OldGroup} New group: {PickedGroup}", memberId, old, _currentGroups); + _currentGroups = newGroup.SelectedGroup.Count > 0 + && newGroup.SelectedGroup.Contains(_currentGroups.MemberReceivedFrom) + ? newGroup + : new MemberGroups(new List>(0), + MemberPartitionGroup.InvalidVersion, + Guid.Empty, + Guid.Empty); + + _logger.IfDebug()?.LogDebug("Removed Member[{MemberId}] and updated member partition group. " + + "Old group: {OldGroup} New group: {PickedGroup}", memberId, old, _currentGroups); } } finally From 5b9de960f345f5f96adf9c9f7e690c612229985a Mon Sep 17 00:00:00 2001 From: emreyigit Date: Fri, 23 Aug 2024 18:43:46 +0300 Subject: [PATCH 05/20] Partion group change should manage connections --- src/Hazelcast.Net/Clustering/ClusterMembers.cs | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/src/Hazelcast.Net/Clustering/ClusterMembers.cs b/src/Hazelcast.Net/Clustering/ClusterMembers.cs index 7069dd9c0..e3b4b4954 100644 --- a/src/Hazelcast.Net/Clustering/ClusterMembers.cs +++ b/src/Hazelcast.Net/Clustering/ClusterMembers.cs @@ -433,6 +433,7 @@ public async Task SetMembersAsync(int version, ICollect : members; var newMembers = new MemberTable(version, members); + lock (_mutex) { _members = newMembers; @@ -471,8 +472,8 @@ public async Task SetMembersAsync(int version, ICollect // update members foreach (var member in members) member.UsePublicAddress = _usePublicAddresses; - // compute changes - var (added, removed) = ComputeChanges(previousMembers, newMembers, members); + // compute changes and if routing mode is MultiMember try to connect to the members + var (added, removed) = ComputeChanges(previousMembers, newMembers, members, _clusterState.IsRoutingModeMultiMember); var maybeDisconnected = false; lock (_mutex) @@ -591,7 +592,7 @@ public async Task SetMembersAsync(int version, ICollect return new MembersUpdatedEventArgs(added, removed, members.ToList()); } - private (List Added, List Removed) ComputeChanges(MemberTable previousTable, MemberTable currentTable, ICollection members) + private (List Added, List Removed) ComputeChanges(MemberTable previousTable, MemberTable currentTable, IEnumerable members, bool forceToConnect = false) { // compute changes // count 1 for old members, 2 for new members, and then the result is @@ -643,6 +644,10 @@ public async Task SetMembersAsync(int version, ICollect break; case 3: // old and new = no change + if (forceToConnect) + { + _memberConnectionQueue?.Add(member); + } break; default: From af20570c0cf60d06d8bdcbe13a4efcec918c87e7 Mon Sep 17 00:00:00 2001 From: emreyigit Date: Fri, 23 Aug 2024 21:52:35 +0300 Subject: [PATCH 06/20] Fix scale down --- .../NoOpSubsetMembers.cs | 6 ++++ src/Hazelcast.Net/Clustering/Authenticator.cs | 6 ++-- src/Hazelcast.Net/Clustering/ClusterEvents.cs | 5 ++-- .../Clustering/ClusterMembers.cs | 9 ++++-- .../Clustering/ISubsetClusterMembers.cs | 4 ++- .../Clustering/MemberPartitionGroup.cs | 30 +++++++++++-------- src/Hazelcast.Net/Models/MemberGroups.cs | 15 ++++++---- .../Polyfills/CollectionExtensions.cs | 18 +++++++++-- 8 files changed, 62 insertions(+), 31 deletions(-) diff --git a/src/Hazelcast.Net.Testing/NoOpSubsetMembers.cs b/src/Hazelcast.Net.Testing/NoOpSubsetMembers.cs index 581288dec..04e953476 100644 --- a/src/Hazelcast.Net.Testing/NoOpSubsetMembers.cs +++ b/src/Hazelcast.Net.Testing/NoOpSubsetMembers.cs @@ -22,10 +22,16 @@ internal class NoOpSubsetMembers : ISubsetClusterMembers public IReadOnlyList GetSubsetMemberIds() => throw new NotImplementedException(); + HashSet ISubsetClusterMembers.GetSubsetMemberIds() + => throw new NotImplementedException(); public void SetSubsetMembers(MemberGroups newGroup) { throw new NotImplementedException(); } + public MemberGroups CurrentGroups + { + get; + } public void RemoveSubsetMember(Guid memberId) { throw new NotImplementedException(); diff --git a/src/Hazelcast.Net/Clustering/Authenticator.cs b/src/Hazelcast.Net/Clustering/Authenticator.cs index ee0e1e9e6..1d22f5ec4 100644 --- a/src/Hazelcast.Net/Clustering/Authenticator.cs +++ b/src/Hazelcast.Net/Clustering/Authenticator.cs @@ -233,7 +233,7 @@ private ClusterVersion ParseClusterVersion(ClientAuthenticationCodec.ResponsePar /// internal MemberGroups ParsePartitionMemberGroups(ClientAuthenticationCodec.ResponseParameters response) { - var emptyMemberGroups = new MemberGroups(new List>(0), MemberPartitionGroup.InvalidVersion, response.ClusterId, response.MemberUuid); + var emptyMemberGroups = new MemberGroups(new List>(0), MemberPartitionGroup.InvalidVersion, response.ClusterId, response.MemberUuid); if (!response.IsKeyValuePairsExists) return emptyMemberGroups; @@ -245,7 +245,7 @@ internal MemberGroups ParsePartitionMemberGroups(ClientAuthenticationCodec.Respo return emptyMemberGroups; } - var partitionGroups = new List>(); + var partitionGroups = new List>(); var version = MemberPartitionGroup.InvalidVersion; try { @@ -262,7 +262,7 @@ internal MemberGroups ParsePartitionMemberGroups(ClientAuthenticationCodec.Respo { group.Add(Guid.Parse(member.ToString())); } - partitionGroups.Add(group.ToList()); + partitionGroups.Add(group); } } catch (Exception e) diff --git a/src/Hazelcast.Net/Clustering/ClusterEvents.cs b/src/Hazelcast.Net/Clustering/ClusterEvents.cs index 389614e7c..f3def189d 100644 --- a/src/Hazelcast.Net/Clustering/ClusterEvents.cs +++ b/src/Hazelcast.Net/Clustering/ClusterEvents.cs @@ -706,10 +706,9 @@ private async ValueTask HandleCodecMemberGroupsViewEvent(int version, IList $"[{string.Join(", ", x)}]")))); - // Received memberId is Guid.Empty because we don't want to change the current member - // group of the member that received the event during auth. + _clusterMembers.SubsetClusterMembers - .SetSubsetMembers(new MemberGroups(memberGroups, version, clusterId, Guid.Empty)); + .SetSubsetMembers(new MemberGroups(memberGroups, version, clusterId, memberId)); await _memberPartitionGroupsUpdated.AwaitEach().CfAwait(); } diff --git a/src/Hazelcast.Net/Clustering/ClusterMembers.cs b/src/Hazelcast.Net/Clustering/ClusterMembers.cs index e3b4b4954..31cafe5e8 100644 --- a/src/Hazelcast.Net/Clustering/ClusterMembers.cs +++ b/src/Hazelcast.Net/Clustering/ClusterMembers.cs @@ -473,7 +473,7 @@ public async Task SetMembersAsync(int version, ICollect foreach (var member in members) member.UsePublicAddress = _usePublicAddresses; // compute changes and if routing mode is MultiMember try to connect to the members - var (added, removed) = ComputeChanges(previousMembers, newMembers, members, _clusterState.IsRoutingModeMultiMember); + var (added, removed) = ComputeChanges(previousMembers, newMembers, members); var maybeDisconnected = false; lock (_mutex) @@ -592,7 +592,7 @@ public async Task SetMembersAsync(int version, ICollect return new MembersUpdatedEventArgs(added, removed, members.ToList()); } - private (List Added, List Removed) ComputeChanges(MemberTable previousTable, MemberTable currentTable, IEnumerable members, bool forceToConnect = false) + private (List Added, List Removed) ComputeChanges(MemberTable previousTable, MemberTable currentTable, IEnumerable members) { // compute changes // count 1 for old members, 2 for new members, and then the result is @@ -644,7 +644,10 @@ public async Task SetMembersAsync(int version, ICollect break; case 3: // old and new = no change - if (forceToConnect) + // In MultiMember mode, while group is changed, the members table can be same. + // In this case, we need to add the member to the queue to connect. + if (_clusterState.IsRoutingModeMultiMember + && _subsetClusterMembers.GetSubsetMemberIds().Contains(member.Id)) { _memberConnectionQueue?.Add(member); } diff --git a/src/Hazelcast.Net/Clustering/ISubsetClusterMembers.cs b/src/Hazelcast.Net/Clustering/ISubsetClusterMembers.cs index 3ee106cb9..05410addd 100644 --- a/src/Hazelcast.Net/Clustering/ISubsetClusterMembers.cs +++ b/src/Hazelcast.Net/Clustering/ISubsetClusterMembers.cs @@ -18,10 +18,12 @@ namespace Hazelcast.Clustering { internal interface ISubsetClusterMembers { - IReadOnlyList GetSubsetMemberIds(); + HashSet GetSubsetMemberIds(); void SetSubsetMembers(MemberGroups newGroup); + MemberGroups CurrentGroups { get; } + void RemoveSubsetMember(Guid memberId); } } diff --git a/src/Hazelcast.Net/Clustering/MemberPartitionGroup.cs b/src/Hazelcast.Net/Clustering/MemberPartitionGroup.cs index 56732deae..16548326e 100644 --- a/src/Hazelcast.Net/Clustering/MemberPartitionGroup.cs +++ b/src/Hazelcast.Net/Clustering/MemberPartitionGroup.cs @@ -33,7 +33,7 @@ internal class MemberPartitionGroup : ISubsetClusterMembers private ILogger _logger; private readonly ReaderWriterLockSlim _mutex = new ReaderWriterLockSlim(); private MemberGroups _currentGroups - = new MemberGroups(null, InvalidVersion, Guid.Empty, Guid.Empty); + = new MemberGroups(new List>(), InvalidVersion, Guid.Empty, Guid.Empty); public MemberPartitionGroup(NetworkingOptions networkingOptions, ILogger logger) { _networkingOptions = networkingOptions; @@ -48,9 +48,13 @@ public MemberPartitionGroup(NetworkingOptions networkingOptions, ILogger logger) /// internal MemberGroups PickBestGroup(MemberGroups newGroup) { - if (newGroup is null) + if (newGroup is null || newGroup.Version <= 0) return _currentGroups; + // Pick authenticator's group. + if (_currentGroups.MemberReceivedFrom != Guid.Empty && _currentGroups.MemberReceivedFrom != newGroup.MemberReceivedFrom) + return new MemberGroups(newGroup.Groups, newGroup.Version, newGroup.ClusterId, _currentGroups.MemberReceivedFrom); + if ((_currentGroups.ClusterId != newGroup.ClusterId || _currentGroups.SelectedGroup.Count == 0) && newGroup.MemberReceivedFrom != Guid.Empty && newGroup.SelectedGroup.Count > 0) @@ -105,7 +109,7 @@ internal MemberGroups GetMostOverlappedGroup(Guid clusterId, Guid memberIdOfGrou internal MemberGroups GetBiggestGroup(MemberGroups newGroup) { var maxCount = int.MinValue; - IList biggestGroup = null; + HashSet biggestGroup = null; for (var i = 0; i < newGroup.Groups.Count; i++) { @@ -117,14 +121,14 @@ internal MemberGroups GetBiggestGroup(MemberGroups newGroup) } return biggestGroup == null - ? new MemberGroups(null, 0, Guid.Empty, Guid.Empty) + ? new MemberGroups(new List>(), 0, Guid.Empty, Guid.Empty) : new MemberGroups(newGroup.Groups, newGroup.Version, newGroup.ClusterId, biggestGroup.First()); } #endregion // internal for testing - internal MemberGroups CurrentGroups + public MemberGroups CurrentGroups { get { @@ -140,7 +144,7 @@ internal MemberGroups CurrentGroups } } - public IReadOnlyList GetSubsetMemberIds() => CurrentGroups.SelectedGroup; + public HashSet GetSubsetMemberIds() => CurrentGroups.SelectedGroup; public void SetSubsetMembers(MemberGroups newGroup) { @@ -148,7 +152,7 @@ public void SetSubsetMembers(MemberGroups newGroup) try { - var pickedGroup = _currentGroups.Version == InvalidVersion ? newGroup : PickBestGroup(newGroup); + var pickedGroup = PickBestGroup(newGroup); _mutex.EnterWriteLock(); var old = _currentGroups; _currentGroups = pickedGroup; @@ -168,11 +172,11 @@ public void RemoveSubsetMember(Guid memberId) _mutex.EnterWriteLock(); if (_currentGroups.SelectedGroup.Contains(memberId)) { - var clearedGroup = new List>(); + var clearedGroup = new List>(); foreach (var group in _currentGroups.Groups) { - var cleared = group.Where(id => id != memberId).ToList(); + var cleared = group.Where(id => id != memberId).ToHashSet(); if (cleared.Count > 0) { clearedGroup.Add(cleared); @@ -181,14 +185,14 @@ public void RemoveSubsetMember(Guid memberId) var newGroup = new MemberGroups(clearedGroup, _currentGroups.Version, _currentGroups.ClusterId, _currentGroups.MemberReceivedFrom); var old = _currentGroups; - _currentGroups = newGroup.SelectedGroup.Count > 0 + _currentGroups = newGroup.SelectedGroup.Count > 0 && newGroup.SelectedGroup.Contains(_currentGroups.MemberReceivedFrom) - ? newGroup - : new MemberGroups(new List>(0), + ? newGroup + : new MemberGroups(new List>(0), MemberPartitionGroup.InvalidVersion, Guid.Empty, Guid.Empty); - + _logger.IfDebug()?.LogDebug("Removed Member[{MemberId}] and updated member partition group. " + "Old group: {OldGroup} New group: {PickedGroup}", memberId, old, _currentGroups); } diff --git a/src/Hazelcast.Net/Models/MemberGroups.cs b/src/Hazelcast.Net/Models/MemberGroups.cs index 5ac0fce6d..f37acd06c 100644 --- a/src/Hazelcast.Net/Models/MemberGroups.cs +++ b/src/Hazelcast.Net/Models/MemberGroups.cs @@ -19,28 +19,31 @@ namespace Hazelcast.Models { internal class MemberGroups { + public MemberGroups(IList> groups, int version, Guid clusterId, Guid memberReceivedFrom) : + this(groups.Select(group => new HashSet(group)).ToList(), version, clusterId, memberReceivedFrom) + { } - public MemberGroups(IList> groups, int version, Guid clusterId, Guid memberReceivedFrom) + public MemberGroups(IList> groups, int version, Guid clusterId, Guid memberReceivedFrom) { - Groups = groups ?? Enumerable.Empty>().ToList(); + Groups = groups ?? Enumerable.Empty>().ToList(); Version = version; ClusterId = clusterId; MemberReceivedFrom = memberReceivedFrom; SelectedGroup = GetGroupOf(MemberReceivedFrom); } - public IList> Groups { get; } + public IList> Groups { get; } public int Version { get; } public Guid ClusterId { get; } public Guid MemberReceivedFrom { get; } /// Group of the member that group information received from. - public IReadOnlyList SelectedGroup { get; } + public HashSet SelectedGroup { get; } // internal for testing - public IReadOnlyList GetGroupOf(Guid memberId) + public HashSet GetGroupOf(Guid memberId) { // Find given member's group. - return (Groups.FirstOrDefault(group => group.Contains(memberId)) ?? Enumerable.Empty()).ToList(); + return Groups.FirstOrDefault(group => group.Contains(memberId)) ?? new HashSet(); } public override string ToString() diff --git a/src/Hazelcast.Net/Polyfills/CollectionExtensions.cs b/src/Hazelcast.Net/Polyfills/CollectionExtensions.cs index 2cc176493..dc9a853b7 100644 --- a/src/Hazelcast.Net/Polyfills/CollectionExtensions.cs +++ b/src/Hazelcast.Net/Polyfills/CollectionExtensions.cs @@ -13,10 +13,10 @@ // limitations under the License. // ReSharper disable once CheckNamespace +using System.Diagnostics.Tracing; namespace System.Collections.Generic; #if NETSTANDARD2_0 - internal static class CollectionExtensions { /// Tries to add the specified and to the . @@ -41,6 +41,20 @@ public static bool TryAdd( dictionary.Add(key, value); return true; } + + /// + /// Creates a HashSet from an IEnumerable. + /// + /// An IEnumerable to create a HashSet from. + /// The type of the elements of source. + /// A HashSet that contains values of type TSource selected from the input sequence. + public static System.Collections.Generic.HashSet ToHashSet(this System.Collections.Generic.IEnumerable source) + { + var set = new HashSet(); + foreach (var item in source) + set.Add(item); + return set; + } } -#endif \ No newline at end of file +#endif From bec87a9eedf4014cbd0b4321b5a938ccad7f8445 Mon Sep 17 00:00:00 2001 From: emreyigit Date: Fri, 23 Aug 2024 22:28:53 +0300 Subject: [PATCH 07/20] Refactor group picking --- .../Clustering/MemberPartitionGroupTests.cs | 17 +++----- .../Clustering/MemberPartitionGroup.cs | 41 +++++++++---------- 2 files changed, 26 insertions(+), 32 deletions(-) diff --git a/src/Hazelcast.Net.Tests/Clustering/MemberPartitionGroupTests.cs b/src/Hazelcast.Net.Tests/Clustering/MemberPartitionGroupTests.cs index f9dea9556..7380b1165 100644 --- a/src/Hazelcast.Net.Tests/Clustering/MemberPartitionGroupTests.cs +++ b/src/Hazelcast.Net.Tests/Clustering/MemberPartitionGroupTests.cs @@ -14,7 +14,6 @@ using System; using System.Collections.Generic; using System.Linq; -using System.Net.Sockets; using System.Threading; using System.Threading.Tasks; using Hazelcast.Aggregation; @@ -557,10 +556,6 @@ private async ValueTask ServerHandler(ClientRequest request) } } - - - - public static object[] MemberGroupCases = { // Case 1: New version wins. @@ -622,7 +617,7 @@ private async ValueTask ServerHandler(ClientRequest request) }, 1, Guid.Parse("81b1ac67-1238-42d6-84b7-ef869e60f262"), - Guid.Empty), + Guid.Parse("64082773-bc1b-408c-8ea6-1150c3c6477e")), Guid.Parse("64082773-bc1b-408c-8ea6-1150c3c6477e"), // Expected group member 1, // Expected version @@ -668,7 +663,7 @@ private async ValueTask ServerHandler(ClientRequest request) }, 1, Guid.Parse("a810df4f-a54c-437d-a945-99218688cf31"), - Guid.Empty), + Guid.Parse("64082773-bc1b-408c-8ea6-1150c3c6477e")), Guid.Parse("64082773-bc1b-408c-8ea6-1150c3c6477e"), // Expected group member 1, // Expected version @@ -714,7 +709,7 @@ private async ValueTask ServerHandler(ClientRequest request) }, 1, Guid.Parse("a810df4f-a54c-437d-a945-99218688cf31"), - Guid.Empty), + Guid.Parse("64082773-bc1b-408c-8ea6-1150c3c6477e")), Guid.Parse("64082773-bc1b-408c-8ea6-1150c3c6477e"), // Expected group member 1, // Expected version @@ -762,7 +757,7 @@ private async ValueTask ServerHandler(ClientRequest request) }, 2, Guid.Parse("81b1ac67-1238-42d6-84b7-ef869e60f262"), - Guid.Empty), + Guid.Parse("64082773-bc1b-408c-8ea6-1150c3c6477e")), Guid.Parse("64082773-bc1b-408c-8ea6-1150c3c6477e"), // Expected group member 2, // Expected version 2 // Expected group size @@ -812,7 +807,7 @@ private async ValueTask ServerHandler(ClientRequest request) }, 2, Guid.Parse("81b1ac67-1238-42d6-84b7-ef869e60f262"), - Guid.Empty), + Guid.Parse("64082773-bc1b-408c-8ea6-1150c3c6477e")), Guid.Parse("64082773-bc1b-408c-8ea6-1150c3c6477e"), // Expected group member 2, // Expected version 5 // Expected group size @@ -862,7 +857,7 @@ private async ValueTask ServerHandler(ClientRequest request) }, 2, Guid.Parse("81b1ac67-1238-42d6-84b7-ef869e60f262"), - Guid.Empty), + Guid.Parse("64082773-bc1b-408c-8ea6-1150c3c6477e")), Guid.Parse("64082773-bc1b-408c-8ea6-1150c3c6477e"), // Expected group member 2, // Expected version 1 // Expected group size diff --git a/src/Hazelcast.Net/Clustering/MemberPartitionGroup.cs b/src/Hazelcast.Net/Clustering/MemberPartitionGroup.cs index 16548326e..e2ec3b10e 100644 --- a/src/Hazelcast.Net/Clustering/MemberPartitionGroup.cs +++ b/src/Hazelcast.Net/Clustering/MemberPartitionGroup.cs @@ -19,7 +19,6 @@ using Hazelcast.Models; using Hazelcast.Networking; using Microsoft.Extensions.Logging; -using System.Threading; namespace Hazelcast.Clustering { internal class MemberPartitionGroup : ISubsetClusterMembers @@ -51,37 +50,37 @@ internal MemberGroups PickBestGroup(MemberGroups newGroup) if (newGroup is null || newGroup.Version <= 0) return _currentGroups; - // Pick authenticator's group. - if (_currentGroups.MemberReceivedFrom != Guid.Empty && _currentGroups.MemberReceivedFrom != newGroup.MemberReceivedFrom) - return new MemberGroups(newGroup.Groups, newGroup.Version, newGroup.ClusterId, _currentGroups.MemberReceivedFrom); + var isCurrentNull = _currentGroups.MemberReceivedFrom == Guid.Empty + || _currentGroups.SelectedGroup.Count == 0 + || _currentGroups.ClusterId != newGroup.ClusterId; - if ((_currentGroups.ClusterId != newGroup.ClusterId || _currentGroups.SelectedGroup.Count == 0) - && newGroup.MemberReceivedFrom != Guid.Empty - && newGroup.SelectedGroup.Count > 0) - { + // Pick authenticator's group. + if (isCurrentNull && _currentGroups.MemberReceivedFrom != newGroup.MemberReceivedFrom) return newGroup; - } - if (_currentGroups.SelectedGroup.Count > 0) - { - var pickedGroup = GetMostOverlappedGroup(newGroup.ClusterId, _currentGroups.MemberReceivedFrom, newGroup); + // Pick most overlapped group. + if (isCurrentNull == false) + { // Given group is stale. Stick with current one. + if (_currentGroups.Version <= newGroup.Version) + { + var pickedGroup = GetMostOverlappedGroup(newGroup.ClusterId, _currentGroups.MemberReceivedFrom, newGroup); - if (pickedGroup.SelectedGroup.Count > 0) - return pickedGroup; + if (pickedGroup.SelectedGroup.Count > 0) + return pickedGroup; + } + else + { + return _currentGroups; + } } + // Pick biggest group. return GetBiggestGroup(newGroup); } // internal for testing internal MemberGroups GetMostOverlappedGroup(Guid clusterId, Guid memberIdOfGroup, MemberGroups newGroups) { - if (_currentGroups.Version > newGroups.Version) - { - // Given group is stale. Stick with current one. - return _currentGroups; - } - // Find the group that has the most overlap with the given groups. var maxOverlap = 0; ICollection mostOverlappedGroup = null; @@ -189,7 +188,7 @@ public void RemoveSubsetMember(Guid memberId) && newGroup.SelectedGroup.Contains(_currentGroups.MemberReceivedFrom) ? newGroup : new MemberGroups(new List>(0), - MemberPartitionGroup.InvalidVersion, + InvalidVersion, Guid.Empty, Guid.Empty); From d4bf18f175a97c4b2b2373b21f87721b7120c1c9 Mon Sep 17 00:00:00 2001 From: emreyigit Date: Wed, 28 Aug 2024 13:48:06 +0300 Subject: [PATCH 08/20] Add more test. --- .../MemberPartitionGroupServerTests.cs | 111 ++++++++++++++---- 1 file changed, 85 insertions(+), 26 deletions(-) diff --git a/src/Hazelcast.Net.Tests/Clustering/MemberPartitionGroupServerTests.cs b/src/Hazelcast.Net.Tests/Clustering/MemberPartitionGroupServerTests.cs index 608c13584..9bbb32fb2 100644 --- a/src/Hazelcast.Net.Tests/Clustering/MemberPartitionGroupServerTests.cs +++ b/src/Hazelcast.Net.Tests/Clustering/MemberPartitionGroupServerTests.cs @@ -44,10 +44,12 @@ public async Task TestMultiMemberRoutingWorks(RoutingStrategy routingStrategy) var address2 = "127.0.0.1:5702"; var address3 = "127.0.0.1:5703"; + var addreses = new string[] { address1, address2, address3 }; + // create a client with the given routing strategy - var client1 = await CreateClient(routingStrategy, address1); - var client2 = await CreateClient(routingStrategy, address2); - var client3 = await CreateClient(routingStrategy, address3); + var client1 = await CreateClient(routingStrategy, new string[] { address1 }, "client1"); + var client2 = await CreateClient(routingStrategy, new string[] { address2 }, "client2"); + var client3 = await CreateClient(routingStrategy, new string[] { address3 }, "client3"); AssertClientOnlySees(client1, address1); AssertClientOnlySees(client2, address2); @@ -68,7 +70,7 @@ await AssertEx.SucceedsEventually(() => { AssertClientOnlySees(client1, address1); }, 10_000, 500, "Client1 did not see the correct members"); - + AssertClientOnlySees(client2, address2); AssertClientOnlySees(client3, address3); @@ -79,37 +81,89 @@ await AssertEx.SucceedsEventually(() => AssertClientOnlySees(client1, address1); AssertClientOnlySees(client2, address2); + var nAddress3 = await AssertClientReConnected(client3, address3); + + // Check if client1 is connected to the new member + // cannot use the old member id since we can only either kill or create member + Assert.That(client3.Cluster.Connections.Count, Is.EqualTo(1)); + Assert.That(client3.Members.Where(p => p.IsConnected).Select(p => p.Member.ConnectAddress.ToString()), Contains.Item(address3)); + + AssertClientOnlySees(client1, address1); + AssertClientOnlySees(client2, address2); + AssertClientOnlySees(client3, address3); + } + + + [TestCase(RoutingStrategy.PartitionGroups)] + public async Task TestMultiMemberRoutingConnectsNextGroupWhenDisconnected(RoutingStrategy routingStrategy) + { + var address1 = "127.0.0.1:5701"; + var address2 = "127.0.0.1:5702"; + var address3 = "127.0.0.1:5703"; + + var addreses = new string[] { address1, address2, address3 }; + + // create a client with the given routing strategy + var client = await CreateClient(routingStrategy, addreses, "client1"); + + // it should connect to first address + Assert.That(client.Cluster.Connections.Count, Is.EqualTo(1)); + + var connectedAddress = client.Members.First(p => p.IsConnected).Member.ConnectAddress.ToString(); + + AssertClientOnlySees(client, connectedAddress); + + var effectiveMembers = client.Cluster.Members.GetMembersForConnection(); + Assert.That(effectiveMembers.Count(), Is.EqualTo(1)); + Assert.That(effectiveMembers.Select(p => p.ConnectAddress.ToString()), Contains.Item(connectedAddress)); + // Kill the connected member so that client can go to next group + var connectedMember = RcMembers.Values.Where(m => connectedAddress.Equals($"{m.Host}:{m.Port}")).Select(m => m.Uuid).First(); + RemoveMember(connectedMember); + + await AssertEx.SucceedsEventually(() => Assert.That(client.State, Is.EqualTo(ClientState.Disconnected)), 10_000, 500); + + await AssertEx.SucceedsEventually(() => + { + Assert.That(client.Cluster.Connections.Count, Is.EqualTo(1)); + Assert.That(client.State, Is.EqualTo(ClientState.Connected)); + }, 20_000, 500); + + var reConnectedAddress = client.Members.First(p => p.IsConnected).Member.ConnectAddress.ToString(); + effectiveMembers = client.Cluster.Members.GetMembersForConnection(); + + Assert.That(reConnectedAddress, Is.Not.EqualTo(connectedAddress)); + Assert.That(client.Cluster.Connections.Count, Is.EqualTo(1)); + Assert.That(client.Members.Where(p => p.IsConnected).Select(p => p.Member.ConnectAddress.ToString()), Contains.Item(reConnectedAddress)); + Assert.That(effectiveMembers.Count(), Is.EqualTo(1)); + Assert.That(effectiveMembers.Select(p => p.ConnectAddress.ToString()), Contains.Item(reConnectedAddress)); + } + + private async Task AssertClientReConnected(HazelcastClient client, string address, bool createMember = true) + { await AssertEx.SucceedsEventually(() => { - Assert.That(client3.Cluster.Connections.Count, Is.EqualTo(0)); - Assert.That(client3.State, Is.EqualTo(ClientState.Disconnected)); + Assert.That(client.Cluster.Connections.Count, Is.EqualTo(0)); + Assert.That(client.State, Is.EqualTo(ClientState.Disconnected)); }, 15_000, 500); - Member member3; - var nAddress3 = NetworkAddress.Parse(address3); - while (true) + Member member; + var nAddress3 = NetworkAddress.Parse(address); + while (createMember) { - member3 = await AddMember(); - var created = new NetworkAddress(member3.Host, member3.Port); + member = await AddMember(); + var created = new NetworkAddress(member.Host, member.Port); if (created == nAddress3) break; - await RemoveMember(member3.Uuid); + await RemoveMember(member.Uuid); } await AssertEx.SucceedsEventually(() - => Assert.That(client3.State, Is.EqualTo(ClientState.Connected)), + => Assert.That(client.State, Is.EqualTo(ClientState.Connected)), 10_000, 500); - // Check if client1 is connected to the new member - // cannot use the old member id since we can only either kill or create member - Assert.That(client3.Cluster.Connections.Count, Is.EqualTo(1)); - Assert.That(client3.Members.Select(p => p.Member.ConnectAddress), Contains.Item(nAddress3)); - - AssertClientOnlySees(client1, address1); - AssertClientOnlySees(client2, address2); - AssertClientOnlySees(client3, address3); + return nAddress3; } [TestCase(RoutingModes.MultiMember)] @@ -119,10 +173,11 @@ public async Task TestClientCollectsClusterEvents(RoutingModes routingMode) { var address1 = "127.0.0.1:5701"; var address2 = "127.0.0.1:5702"; + var addreses = new string[] { address1, address2 }; var keyCount = 3 * 271; // create a client with the given routing strategy for catching the events. - var client1 = await CreateClient(RoutingStrategy.PartitionGroups, address1, routingMode); + var client1 = await CreateClient(RoutingStrategy.PartitionGroups, addreses, "client1", routingMode); var client1Count = 0; var mapName = $"map_{routingMode}"; var map1 = await client1.GetMapAsync(mapName); @@ -137,7 +192,7 @@ await map1.SubscribeAsync((eventHandler) => // create a dummy client for creating events - var client2 = await CreateClient(RoutingStrategy.PartitionGroups, address2, RoutingModes.SingleMember); + var client2 = await CreateClient(RoutingStrategy.PartitionGroups, addreses, "client2", RoutingModes.SingleMember); var map2 = await client2.GetMapAsync(map1.Name); @@ -168,13 +223,17 @@ private void AssertClientOnlySees(HazelcastClient client, string address, int cl Assert.That(members.SubsetClusterMembers.GetSubsetMemberIds().Count(), Is.EqualTo(1)); Assert.That(members.SubsetClusterMembers.GetSubsetMemberIds(), Contains.Item(memberId)); } - private async Task CreateClient(RoutingStrategy routingStrategy, string address, RoutingModes routingMode = RoutingModes.MultiMember) + private async Task CreateClient(RoutingStrategy routingStrategy, string[] address, string clientName, RoutingModes routingMode = RoutingModes.MultiMember) { var options = new HazelcastOptionsBuilder() .With(args => { - args.ClientName = $"Client:{address}"; - args.Networking.Addresses.Add(address); + for (int i = 0; i < address.Length; i++) + { + + args.Networking.Addresses.Add(address[i]); + } + args.ClientName = clientName; args.ClusterName = RcCluster.Id; args.Networking.RoutingMode.Mode = routingMode; args.Networking.RoutingMode.Strategy = routingStrategy; From 856d0ecd2594ecf3f40ec2121ea5fe4eb81dfdd4 Mon Sep 17 00:00:00 2001 From: emreyigit Date: Wed, 28 Aug 2024 16:38:53 +0300 Subject: [PATCH 09/20] Increase test timeout. --- .../Clustering/MemberPartitionGroupServerTests.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Hazelcast.Net.Tests/Clustering/MemberPartitionGroupServerTests.cs b/src/Hazelcast.Net.Tests/Clustering/MemberPartitionGroupServerTests.cs index 9bbb32fb2..906c5eab2 100644 --- a/src/Hazelcast.Net.Tests/Clustering/MemberPartitionGroupServerTests.cs +++ b/src/Hazelcast.Net.Tests/Clustering/MemberPartitionGroupServerTests.cs @@ -26,7 +26,7 @@ namespace Hazelcast.Tests.Clustering { [Category("enterprise")] [ServerCondition("5.5")] - [Timeout(30_000)] + [Timeout(60_000)] public class MemberPartitionGroupServerTests : MultiMembersRemoteTestBase { protected override string RcClusterConfiguration => Resources.ClusterPGEnabled; From be9e73d01a8f2a6212c9214d5af6c16b12407820 Mon Sep 17 00:00:00 2001 From: emreyigit Date: Thu, 29 Aug 2024 08:45:18 +0300 Subject: [PATCH 10/20] Fix failing test --- .../MemberPartitionGroupServerTests.cs | 51 ++++++++----------- 1 file changed, 22 insertions(+), 29 deletions(-) diff --git a/src/Hazelcast.Net.Tests/Clustering/MemberPartitionGroupServerTests.cs b/src/Hazelcast.Net.Tests/Clustering/MemberPartitionGroupServerTests.cs index 906c5eab2..cd31eaf78 100644 --- a/src/Hazelcast.Net.Tests/Clustering/MemberPartitionGroupServerTests.cs +++ b/src/Hazelcast.Net.Tests/Clustering/MemberPartitionGroupServerTests.cs @@ -81,7 +81,28 @@ await AssertEx.SucceedsEventually(() => AssertClientOnlySees(client1, address1); AssertClientOnlySees(client2, address2); - var nAddress3 = await AssertClientReConnected(client3, address3); + await AssertEx.SucceedsEventually(() + => + { + Assert.That(client3.Cluster.Connections.Count, Is.EqualTo(0)); + Assert.That(client3.State, Is.EqualTo(ClientState.Disconnected)); + }, + 15_000, 500); + + Member member; + var nAddress3 = NetworkAddress.Parse(address3); + while (true) + { + member = await AddMember(); + var created = new NetworkAddress(member.Host, member.Port); + if (created == nAddress3) break; + + await RemoveMember(member.Uuid); + } + + await AssertEx.SucceedsEventually(() + => Assert.That(client3.State, Is.EqualTo(ClientState.Connected)), + 10_000, 500); // Check if client1 is connected to the new member // cannot use the old member id since we can only either kill or create member @@ -138,34 +159,6 @@ await AssertEx.SucceedsEventually(() => Assert.That(effectiveMembers.Select(p => p.ConnectAddress.ToString()), Contains.Item(reConnectedAddress)); } - private async Task AssertClientReConnected(HazelcastClient client, string address, bool createMember = true) - { - await AssertEx.SucceedsEventually(() - => - { - Assert.That(client.Cluster.Connections.Count, Is.EqualTo(0)); - Assert.That(client.State, Is.EqualTo(ClientState.Disconnected)); - }, - 15_000, 500); - - Member member; - var nAddress3 = NetworkAddress.Parse(address); - while (createMember) - { - member = await AddMember(); - var created = new NetworkAddress(member.Host, member.Port); - if (created == nAddress3) break; - - await RemoveMember(member.Uuid); - } - - await AssertEx.SucceedsEventually(() - => Assert.That(client.State, Is.EqualTo(ClientState.Connected)), - 10_000, 500); - - return nAddress3; - } - [TestCase(RoutingModes.MultiMember)] [TestCase(RoutingModes.SingleMember)] [TestCase(RoutingModes.AllMembers)] From e20998951fec1e60aa2da2335ad29b7f13085206 Mon Sep 17 00:00:00 2001 From: emreyigit Date: Thu, 29 Aug 2024 13:57:38 +0300 Subject: [PATCH 11/20] Fix test timeout. --- .../Clustering/MemberPartitionGroupServerTests.cs | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/src/Hazelcast.Net.Tests/Clustering/MemberPartitionGroupServerTests.cs b/src/Hazelcast.Net.Tests/Clustering/MemberPartitionGroupServerTests.cs index cd31eaf78..a23f2a44f 100644 --- a/src/Hazelcast.Net.Tests/Clustering/MemberPartitionGroupServerTests.cs +++ b/src/Hazelcast.Net.Tests/Clustering/MemberPartitionGroupServerTests.cs @@ -32,11 +32,20 @@ public class MemberPartitionGroupServerTests : MultiMembersRemoteTestBase protected override string RcClusterConfiguration => Resources.ClusterPGEnabled; [OneTimeSetUp] - public async Task TearDown() + public async Task OneTimeSetUp() { await CreateCluster(); } + [SetUp] + public async Task TearUp() + { + var diff = 3 - RcMembers.Count; + if (diff is > 0 and < 3) + for (var i = 0; i < diff; i++) + await AddMember(); + } + [TestCase(RoutingStrategy.PartitionGroups)] public async Task TestMultiMemberRoutingWorks(RoutingStrategy routingStrategy) { From 6889c3b4c9b352f5cde9848575bd8e8a83d820c0 Mon Sep 17 00:00:00 2001 From: emreyigit Date: Thu, 29 Aug 2024 16:20:36 +0300 Subject: [PATCH 12/20] Reset cluster for each run --- .../MemberPartitionGroupServerTests.cs | 56 ++++++++++++++----- 1 file changed, 41 insertions(+), 15 deletions(-) diff --git a/src/Hazelcast.Net.Tests/Clustering/MemberPartitionGroupServerTests.cs b/src/Hazelcast.Net.Tests/Clustering/MemberPartitionGroupServerTests.cs index a23f2a44f..d53ff9474 100644 --- a/src/Hazelcast.Net.Tests/Clustering/MemberPartitionGroupServerTests.cs +++ b/src/Hazelcast.Net.Tests/Clustering/MemberPartitionGroupServerTests.cs @@ -15,13 +15,13 @@ using System.Linq; using System.Threading; using System.Threading.Tasks; +using Hazelcast.Core; using Hazelcast.Networking; using Hazelcast.Testing; using Hazelcast.Testing.Conditions; using Hazelcast.Testing.Remote; using Microsoft.Extensions.Logging; using NUnit.Framework; -using Thrift.Protocol; namespace Hazelcast.Tests.Clustering { [Category("enterprise")] @@ -31,24 +31,36 @@ public class MemberPartitionGroupServerTests : MultiMembersRemoteTestBase { protected override string RcClusterConfiguration => Resources.ClusterPGEnabled; - [OneTimeSetUp] + [SetUp] public async Task OneTimeSetUp() { await CreateCluster(); } + [TearDown] + public async Task TearDown() + { + await MembersOneTimeTearDown(); + } - [SetUp] + /*[SetUp] public async Task TearUp() { var diff = 3 - RcMembers.Count; if (diff is > 0 and < 3) for (var i = 0; i < diff; i++) - await AddMember(); - } + { + var member = await AddMember(); + HConsole.WriteLine(this, member.Uuid+ " was missing, added."); + } + + }*/ [TestCase(RoutingStrategy.PartitionGroups)] public async Task TestMultiMemberRoutingWorks(RoutingStrategy routingStrategy) { + Assert.That(RcMembers.Count, Is.EqualTo(3)); + HConsole.Configure(c => c.ConfigureDefaults(this)); + var address1 = "127.0.0.1:5701"; var address2 = "127.0.0.1:5702"; var address3 = "127.0.0.1:5703"; @@ -60,7 +72,8 @@ public async Task TestMultiMemberRoutingWorks(RoutingStrategy routingStrategy) var client2 = await CreateClient(routingStrategy, new string[] { address2 }, "client2"); var client3 = await CreateClient(routingStrategy, new string[] { address3 }, "client3"); - AssertClientOnlySees(client1, address1); + // wait until cluster forms of 3 members + await AssertEx.SucceedsEventually(() => AssertClientOnlySees(client1, address1), 15_000, 500); AssertClientOnlySees(client2, address2); AssertClientOnlySees(client3, address3); @@ -148,7 +161,7 @@ public async Task TestMultiMemberRoutingConnectsNextGroupWhenDisconnected(Routin Assert.That(effectiveMembers.Select(p => p.ConnectAddress.ToString()), Contains.Item(connectedAddress)); // Kill the connected member so that client can go to next group var connectedMember = RcMembers.Values.Where(m => connectedAddress.Equals($"{m.Host}:{m.Port}")).Select(m => m.Uuid).First(); - RemoveMember(connectedMember); + await RemoveMember(connectedMember); await AssertEx.SucceedsEventually(() => Assert.That(client.State, Is.EqualTo(ClientState.Disconnected)), 10_000, 500); @@ -156,16 +169,29 @@ await AssertEx.SucceedsEventually(() => { Assert.That(client.Cluster.Connections.Count, Is.EqualTo(1)); Assert.That(client.State, Is.EqualTo(ClientState.Connected)); + var reConnectedAddress = client.Members.First(p => p.IsConnected).Member.ConnectAddress.ToString(); + effectiveMembers = client.Cluster.Members.GetMembersForConnection(); + + Assert.That(reConnectedAddress, Is.Not.EqualTo(connectedAddress)); + Assert.That(client.Cluster.Connections.Count, Is.EqualTo(1)); + Assert.That(client.Members.Where(p => p.IsConnected).Select(p => p.Member.ConnectAddress.ToString()), Contains.Item(reConnectedAddress)); + Assert.That(effectiveMembers.Count(), Is.EqualTo(1)); + Assert.That(effectiveMembers.Select(p => p.ConnectAddress.ToString()), Contains.Item(reConnectedAddress)); }, 20_000, 500); - var reConnectedAddress = client.Members.First(p => p.IsConnected).Member.ConnectAddress.ToString(); - effectiveMembers = client.Cluster.Members.GetMembersForConnection(); + } + private async Task AddMemberFor(string connectedAddress) + { + Member member; + var nAddress3 = NetworkAddress.Parse(connectedAddress); + while (true) + { + member = await AddMember(); + var created = new NetworkAddress(member.Host, member.Port); + if (created == nAddress3) break; - Assert.That(reConnectedAddress, Is.Not.EqualTo(connectedAddress)); - Assert.That(client.Cluster.Connections.Count, Is.EqualTo(1)); - Assert.That(client.Members.Where(p => p.IsConnected).Select(p => p.Member.ConnectAddress.ToString()), Contains.Item(reConnectedAddress)); - Assert.That(effectiveMembers.Count(), Is.EqualTo(1)); - Assert.That(effectiveMembers.Select(p => p.ConnectAddress.ToString()), Contains.Item(reConnectedAddress)); + await RemoveMember(member.Uuid); + } } [TestCase(RoutingModes.MultiMember)] @@ -219,7 +245,7 @@ private void AssertClientOnlySees(HazelcastClient client, string address, int cl var members = client.Cluster.Members; var memberId = Guid.Parse(member.Uuid); - Assert.That(members.GetMembers().Count(), Is.EqualTo(clusterSize)); + Assert.That(members.GetMembers().Count(), Is.EqualTo(clusterSize), "Current cluster size " + RcMembers.Count); Assert.That(client.Cluster.Connections.Count, Is.EqualTo(1)); Assert.True(client.Cluster.Connections.Contains(memberId), "Member is not connected"); Assert.That(members.SubsetClusterMembers.GetSubsetMemberIds().Count(), Is.EqualTo(1)); From 280d7be4b5fa8c0df5745e048433d0c8290ac407 Mon Sep 17 00:00:00 2001 From: emreyigit Date: Thu, 29 Aug 2024 17:28:59 +0300 Subject: [PATCH 13/20] increase test timeout. --- .../Clustering/MemberPartitionGroupServerTests.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Hazelcast.Net.Tests/Clustering/MemberPartitionGroupServerTests.cs b/src/Hazelcast.Net.Tests/Clustering/MemberPartitionGroupServerTests.cs index d53ff9474..3b0706f96 100644 --- a/src/Hazelcast.Net.Tests/Clustering/MemberPartitionGroupServerTests.cs +++ b/src/Hazelcast.Net.Tests/Clustering/MemberPartitionGroupServerTests.cs @@ -163,7 +163,7 @@ public async Task TestMultiMemberRoutingConnectsNextGroupWhenDisconnected(Routin var connectedMember = RcMembers.Values.Where(m => connectedAddress.Equals($"{m.Host}:{m.Port}")).Select(m => m.Uuid).First(); await RemoveMember(connectedMember); - await AssertEx.SucceedsEventually(() => Assert.That(client.State, Is.EqualTo(ClientState.Disconnected)), 10_000, 500); + await AssertEx.SucceedsEventually(() => Assert.That(client.State, Is.EqualTo(ClientState.Disconnected)), 20_000, 500); await AssertEx.SucceedsEventually(() => { From 715325f2fa7105b34a72606a2651263ad93f771a Mon Sep 17 00:00:00 2001 From: emreyigit Date: Tue, 10 Sep 2024 12:24:46 +0300 Subject: [PATCH 14/20] increase timeout. --- .../Clustering/MemberPartitionGroupServerTests.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Hazelcast.Net.Tests/Clustering/MemberPartitionGroupServerTests.cs b/src/Hazelcast.Net.Tests/Clustering/MemberPartitionGroupServerTests.cs index 3b0706f96..0c65692e7 100644 --- a/src/Hazelcast.Net.Tests/Clustering/MemberPartitionGroupServerTests.cs +++ b/src/Hazelcast.Net.Tests/Clustering/MemberPartitionGroupServerTests.cs @@ -177,7 +177,7 @@ await AssertEx.SucceedsEventually(() => Assert.That(client.Members.Where(p => p.IsConnected).Select(p => p.Member.ConnectAddress.ToString()), Contains.Item(reConnectedAddress)); Assert.That(effectiveMembers.Count(), Is.EqualTo(1)); Assert.That(effectiveMembers.Select(p => p.ConnectAddress.ToString()), Contains.Item(reConnectedAddress)); - }, 20_000, 500); + }, 60_000, 500); } private async Task AddMemberFor(string connectedAddress) From 898ac264d5ef1f83fd65b4e3bb305f475e8ac54c Mon Sep 17 00:00:00 2001 From: emreyigit Date: Tue, 10 Sep 2024 16:10:26 +0300 Subject: [PATCH 15/20] Increase timeout --- .../Clustering/MemberPartitionGroupServerTests.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Hazelcast.Net.Tests/Clustering/MemberPartitionGroupServerTests.cs b/src/Hazelcast.Net.Tests/Clustering/MemberPartitionGroupServerTests.cs index 0c65692e7..cd211f8d0 100644 --- a/src/Hazelcast.Net.Tests/Clustering/MemberPartitionGroupServerTests.cs +++ b/src/Hazelcast.Net.Tests/Clustering/MemberPartitionGroupServerTests.cs @@ -163,7 +163,7 @@ public async Task TestMultiMemberRoutingConnectsNextGroupWhenDisconnected(Routin var connectedMember = RcMembers.Values.Where(m => connectedAddress.Equals($"{m.Host}:{m.Port}")).Select(m => m.Uuid).First(); await RemoveMember(connectedMember); - await AssertEx.SucceedsEventually(() => Assert.That(client.State, Is.EqualTo(ClientState.Disconnected)), 20_000, 500); + await AssertEx.SucceedsEventually(() => Assert.That(client.State, Is.EqualTo(ClientState.Disconnected)), 60_000, 500); await AssertEx.SucceedsEventually(() => { From 3b7a7d29a3783b36fc6a5af1eb4e0eeabdf0ef76 Mon Sep 17 00:00:00 2001 From: emreyigit Date: Tue, 10 Sep 2024 18:17:32 +0300 Subject: [PATCH 16/20] wait for cluster --- .../Clustering/MemberPartitionGroupServerTests.cs | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/Hazelcast.Net.Tests/Clustering/MemberPartitionGroupServerTests.cs b/src/Hazelcast.Net.Tests/Clustering/MemberPartitionGroupServerTests.cs index cd211f8d0..bc701ba75 100644 --- a/src/Hazelcast.Net.Tests/Clustering/MemberPartitionGroupServerTests.cs +++ b/src/Hazelcast.Net.Tests/Clustering/MemberPartitionGroupServerTests.cs @@ -13,6 +13,7 @@ // limitations under the License. using System; using System.Linq; +using System.Reflection; using System.Threading; using System.Threading.Tasks; using Hazelcast.Core; @@ -32,7 +33,7 @@ public class MemberPartitionGroupServerTests : MultiMembersRemoteTestBase protected override string RcClusterConfiguration => Resources.ClusterPGEnabled; [SetUp] - public async Task OneTimeSetUp() + public async Task Setup() { await CreateCluster(); } @@ -58,7 +59,7 @@ public async Task TearUp() [TestCase(RoutingStrategy.PartitionGroups)] public async Task TestMultiMemberRoutingWorks(RoutingStrategy routingStrategy) { - Assert.That(RcMembers.Count, Is.EqualTo(3)); + await AssertEx.SucceedsEventually(() => Assert.That(RcMembers.Count, Is.EqualTo(3)), 30_000, 500); HConsole.Configure(c => c.ConfigureDefaults(this)); var address1 = "127.0.0.1:5701"; From 1c622ccdf21be9640c3e6a371996d96b0d8a9290 Mon Sep 17 00:00:00 2001 From: emreyigit Date: Wed, 11 Sep 2024 12:07:04 +0300 Subject: [PATCH 17/20] Separate tests into different suits due to timeout. --- .../MemberPartitionGroupServerTests.cs | 58 -------- .../MemberPartitionGroupServerTests2.cs | 139 ++++++++++++++++++ 2 files changed, 139 insertions(+), 58 deletions(-) create mode 100644 src/Hazelcast.Net.Tests/Clustering/MemberPartitionGroupServerTests2.cs diff --git a/src/Hazelcast.Net.Tests/Clustering/MemberPartitionGroupServerTests.cs b/src/Hazelcast.Net.Tests/Clustering/MemberPartitionGroupServerTests.cs index bc701ba75..1a22a9661 100644 --- a/src/Hazelcast.Net.Tests/Clustering/MemberPartitionGroupServerTests.cs +++ b/src/Hazelcast.Net.Tests/Clustering/MemberPartitionGroupServerTests.cs @@ -13,7 +13,6 @@ // limitations under the License. using System; using System.Linq; -using System.Reflection; using System.Threading; using System.Threading.Tasks; using Hazelcast.Core; @@ -43,19 +42,6 @@ public async Task TearDown() await MembersOneTimeTearDown(); } - /*[SetUp] - public async Task TearUp() - { - var diff = 3 - RcMembers.Count; - if (diff is > 0 and < 3) - for (var i = 0; i < diff; i++) - { - var member = await AddMember(); - HConsole.WriteLine(this, member.Uuid+ " was missing, added."); - } - - }*/ - [TestCase(RoutingStrategy.PartitionGroups)] public async Task TestMultiMemberRoutingWorks(RoutingStrategy routingStrategy) { @@ -137,50 +123,6 @@ await AssertEx.SucceedsEventually(() AssertClientOnlySees(client3, address3); } - - [TestCase(RoutingStrategy.PartitionGroups)] - public async Task TestMultiMemberRoutingConnectsNextGroupWhenDisconnected(RoutingStrategy routingStrategy) - { - var address1 = "127.0.0.1:5701"; - var address2 = "127.0.0.1:5702"; - var address3 = "127.0.0.1:5703"; - - var addreses = new string[] { address1, address2, address3 }; - - // create a client with the given routing strategy - var client = await CreateClient(routingStrategy, addreses, "client1"); - - // it should connect to first address - Assert.That(client.Cluster.Connections.Count, Is.EqualTo(1)); - - var connectedAddress = client.Members.First(p => p.IsConnected).Member.ConnectAddress.ToString(); - - AssertClientOnlySees(client, connectedAddress); - - var effectiveMembers = client.Cluster.Members.GetMembersForConnection(); - Assert.That(effectiveMembers.Count(), Is.EqualTo(1)); - Assert.That(effectiveMembers.Select(p => p.ConnectAddress.ToString()), Contains.Item(connectedAddress)); - // Kill the connected member so that client can go to next group - var connectedMember = RcMembers.Values.Where(m => connectedAddress.Equals($"{m.Host}:{m.Port}")).Select(m => m.Uuid).First(); - await RemoveMember(connectedMember); - - await AssertEx.SucceedsEventually(() => Assert.That(client.State, Is.EqualTo(ClientState.Disconnected)), 60_000, 500); - - await AssertEx.SucceedsEventually(() => - { - Assert.That(client.Cluster.Connections.Count, Is.EqualTo(1)); - Assert.That(client.State, Is.EqualTo(ClientState.Connected)); - var reConnectedAddress = client.Members.First(p => p.IsConnected).Member.ConnectAddress.ToString(); - effectiveMembers = client.Cluster.Members.GetMembersForConnection(); - - Assert.That(reConnectedAddress, Is.Not.EqualTo(connectedAddress)); - Assert.That(client.Cluster.Connections.Count, Is.EqualTo(1)); - Assert.That(client.Members.Where(p => p.IsConnected).Select(p => p.Member.ConnectAddress.ToString()), Contains.Item(reConnectedAddress)); - Assert.That(effectiveMembers.Count(), Is.EqualTo(1)); - Assert.That(effectiveMembers.Select(p => p.ConnectAddress.ToString()), Contains.Item(reConnectedAddress)); - }, 60_000, 500); - - } private async Task AddMemberFor(string connectedAddress) { Member member; diff --git a/src/Hazelcast.Net.Tests/Clustering/MemberPartitionGroupServerTests2.cs b/src/Hazelcast.Net.Tests/Clustering/MemberPartitionGroupServerTests2.cs new file mode 100644 index 000000000..806a49bca --- /dev/null +++ b/src/Hazelcast.Net.Tests/Clustering/MemberPartitionGroupServerTests2.cs @@ -0,0 +1,139 @@ +// Copyright (c) 2008-2024, Hazelcast, Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +using System; +using System.Linq; +using System.Threading.Tasks; +using Hazelcast.Networking; +using Hazelcast.Testing; +using Hazelcast.Testing.Conditions; +using Microsoft.Extensions.Logging; +using NUnit.Framework; +namespace Hazelcast.Tests.Clustering +{ + [Category("enterprise")] + [ServerCondition("5.5")] + [Timeout(60_000)] + public class MemberPartitionGroupServerTests2 : MultiMembersRemoteTestBase + { + protected override string RcClusterConfiguration => Resources.ClusterPGEnabled; + + [SetUp] + public async Task Setup() + { + await CreateCluster(); + } + [TearDown] + public async Task TearDown() + { + await MembersOneTimeTearDown(); + } + + [TestCase(RoutingStrategy.PartitionGroups)] + public async Task TestMultiMemberRoutingConnectsNextGroupWhenDisconnected(RoutingStrategy routingStrategy) + { + var address1 = "127.0.0.1:5701"; + var address2 = "127.0.0.1:5702"; + var address3 = "127.0.0.1:5703"; + + var addreses = new string[] { address1, address2, address3 }; + + // create a client with the given routing strategy + var client = await CreateClient(routingStrategy, addreses, "client1"); + + // it should connect to first address + Assert.That(client.Cluster.Connections.Count, Is.EqualTo(1)); + + var connectedAddress = client.Members.First(p => p.IsConnected).Member.ConnectAddress.ToString(); + + AssertClientOnlySees(client, connectedAddress); + + var effectiveMembers = client.Cluster.Members.GetMembersForConnection(); + Assert.That(effectiveMembers.Count(), Is.EqualTo(1)); + Assert.That(effectiveMembers.Select(p => p.ConnectAddress.ToString()), Contains.Item(connectedAddress)); + // Kill the connected member so that client can go to next group + var connectedMember = RcMembers.Values.Where(m => connectedAddress.Equals($"{m.Host}:{m.Port}")).Select(m => m.Uuid).First(); + await RemoveMember(connectedMember); + + await AssertEx.SucceedsEventually(() => Assert.That(client.State, Is.EqualTo(ClientState.Disconnected)), 60_000, 500); + + await AssertEx.SucceedsEventually(() => + { + Assert.That(client.Cluster.Connections.Count, Is.EqualTo(1)); + Assert.That(client.State, Is.EqualTo(ClientState.Connected)); + var reConnectedAddress = client.Members.First(p => p.IsConnected).Member.ConnectAddress.ToString(); + effectiveMembers = client.Cluster.Members.GetMembersForConnection(); + + Assert.That(reConnectedAddress, Is.Not.EqualTo(connectedAddress)); + Assert.That(client.Cluster.Connections.Count, Is.EqualTo(1)); + Assert.That(client.Members.Where(p => p.IsConnected).Select(p => p.Member.ConnectAddress.ToString()), Contains.Item(reConnectedAddress)); + Assert.That(effectiveMembers.Count(), Is.EqualTo(1)); + Assert.That(effectiveMembers.Select(p => p.ConnectAddress.ToString()), Contains.Item(reConnectedAddress)); + }, 60_000, 500); + + } + + private void AssertClientOnlySees(HazelcastClient client, string address, int clusterSize = 3) + { + var member = RcMembers.Values.FirstOrDefault(m => address.Equals($"{m.Host}:{m.Port}")); + + Assert.IsNotNull(member); + + var members = client.Cluster.Members; + var memberId = Guid.Parse(member.Uuid); + + Assert.That(members.GetMembers().Count(), Is.EqualTo(clusterSize), "Current cluster size " + RcMembers.Count); + Assert.That(client.Cluster.Connections.Count, Is.EqualTo(1)); + Assert.True(client.Cluster.Connections.Contains(memberId), "Member is not connected"); + Assert.That(members.SubsetClusterMembers.GetSubsetMemberIds().Count(), Is.EqualTo(1)); + Assert.That(members.SubsetClusterMembers.GetSubsetMemberIds(), Contains.Item(memberId)); + } + private async Task CreateClient(RoutingStrategy routingStrategy, string[] address, string clientName, RoutingModes routingMode = RoutingModes.MultiMember) + { + var options = new HazelcastOptionsBuilder() + .With(args => + { + for (int i = 0; i < address.Length; i++) + { + + args.Networking.Addresses.Add(address[i]); + } + args.ClientName = clientName; + args.ClusterName = RcCluster.Id; + args.Networking.RoutingMode.Mode = routingMode; + args.Networking.RoutingMode.Strategy = routingStrategy; + args.Networking.ReconnectMode = ReconnectMode.ReconnectSync; + + args.AddSubscriber(on => on.StateChanged((client, eventArgs) => + { + Console.WriteLine(eventArgs.State); + })); + + args.LoggerFactory.Creator = () => Microsoft.Extensions.Logging.LoggerFactory.Create( + conf => conf.AddConsole().SetMinimumLevel(LogLevel.Debug)); + + }) + .Build(); + + var client = (HazelcastClient) await HazelcastClientFactory.StartNewClientAsync(options); + return client; + } + private async Task CreateCluster(int size = 3) + { + for (int i = 0; i < size; i++) + { + await AddMember(); + } + } + } +} From 39121b2176aae74ab5998bb9c5ee2e80dd79df08 Mon Sep 17 00:00:00 2001 From: emreyigit Date: Wed, 11 Sep 2024 12:45:22 +0300 Subject: [PATCH 18/20] Heavy test shouldn't run every time. --- .../Clustering/MemberPartitionGroupServerTests2.cs | 1 + 1 file changed, 1 insertion(+) diff --git a/src/Hazelcast.Net.Tests/Clustering/MemberPartitionGroupServerTests2.cs b/src/Hazelcast.Net.Tests/Clustering/MemberPartitionGroupServerTests2.cs index 806a49bca..2b4088c5c 100644 --- a/src/Hazelcast.Net.Tests/Clustering/MemberPartitionGroupServerTests2.cs +++ b/src/Hazelcast.Net.Tests/Clustering/MemberPartitionGroupServerTests2.cs @@ -23,6 +23,7 @@ namespace Hazelcast.Tests.Clustering { [Category("enterprise")] [ServerCondition("5.5")] + [Explicit("This test is not working as expected. It is failing on the GitHub Actions due to resource consumption.")] [Timeout(60_000)] public class MemberPartitionGroupServerTests2 : MultiMembersRemoteTestBase { From de7900074f6f4619e9ba3c04e8f4d1d1b0bbfcd4 Mon Sep 17 00:00:00 2001 From: emreyigit Date: Thu, 12 Sep 2024 10:26:29 +0300 Subject: [PATCH 19/20] Review changes. --- .../Clustering/MemberPartitionGroupServerTests.cs | 14 -------------- ...s => MemberPartitionGroupServerTestsNightly.cs} | 5 ++--- .../Clustering/MemberPartitionGroup.cs | 3 +-- 3 files changed, 3 insertions(+), 19 deletions(-) rename src/Hazelcast.Net.Tests/Clustering/{MemberPartitionGroupServerTests2.cs => MemberPartitionGroupServerTestsNightly.cs} (96%) diff --git a/src/Hazelcast.Net.Tests/Clustering/MemberPartitionGroupServerTests.cs b/src/Hazelcast.Net.Tests/Clustering/MemberPartitionGroupServerTests.cs index 1a22a9661..1baba8867 100644 --- a/src/Hazelcast.Net.Tests/Clustering/MemberPartitionGroupServerTests.cs +++ b/src/Hazelcast.Net.Tests/Clustering/MemberPartitionGroupServerTests.cs @@ -123,20 +123,6 @@ await AssertEx.SucceedsEventually(() AssertClientOnlySees(client3, address3); } - private async Task AddMemberFor(string connectedAddress) - { - Member member; - var nAddress3 = NetworkAddress.Parse(connectedAddress); - while (true) - { - member = await AddMember(); - var created = new NetworkAddress(member.Host, member.Port); - if (created == nAddress3) break; - - await RemoveMember(member.Uuid); - } - } - [TestCase(RoutingModes.MultiMember)] [TestCase(RoutingModes.SingleMember)] [TestCase(RoutingModes.AllMembers)] diff --git a/src/Hazelcast.Net.Tests/Clustering/MemberPartitionGroupServerTests2.cs b/src/Hazelcast.Net.Tests/Clustering/MemberPartitionGroupServerTestsNightly.cs similarity index 96% rename from src/Hazelcast.Net.Tests/Clustering/MemberPartitionGroupServerTests2.cs rename to src/Hazelcast.Net.Tests/Clustering/MemberPartitionGroupServerTestsNightly.cs index 2b4088c5c..d403a8dc3 100644 --- a/src/Hazelcast.Net.Tests/Clustering/MemberPartitionGroupServerTests2.cs +++ b/src/Hazelcast.Net.Tests/Clustering/MemberPartitionGroupServerTestsNightly.cs @@ -21,11 +21,10 @@ using NUnit.Framework; namespace Hazelcast.Tests.Clustering { - [Category("enterprise")] + [Category("enterprise,nightly")] [ServerCondition("5.5")] - [Explicit("This test is not working as expected. It is failing on the GitHub Actions due to resource consumption.")] [Timeout(60_000)] - public class MemberPartitionGroupServerTests2 : MultiMembersRemoteTestBase + public class MemberPartitionGroupServerTestsNightly : MultiMembersRemoteTestBase { protected override string RcClusterConfiguration => Resources.ClusterPGEnabled; diff --git a/src/Hazelcast.Net/Clustering/MemberPartitionGroup.cs b/src/Hazelcast.Net/Clustering/MemberPartitionGroup.cs index e2ec3b10e..a02aeb8c7 100644 --- a/src/Hazelcast.Net/Clustering/MemberPartitionGroup.cs +++ b/src/Hazelcast.Net/Clustering/MemberPartitionGroup.cs @@ -125,8 +125,7 @@ internal MemberGroups GetBiggestGroup(MemberGroups newGroup) } #endregion - - // internal for testing + public MemberGroups CurrentGroups { get From 693ab22faf0c5adb9b162f3b8161afce579f7a4b Mon Sep 17 00:00:00 2001 From: emreyigit Date: Thu, 12 Sep 2024 10:27:27 +0300 Subject: [PATCH 20/20] Fix typo --- .../Clustering/MemberPartitionGroupServerTests.cs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/Hazelcast.Net.Tests/Clustering/MemberPartitionGroupServerTests.cs b/src/Hazelcast.Net.Tests/Clustering/MemberPartitionGroupServerTests.cs index 1baba8867..c43480f6e 100644 --- a/src/Hazelcast.Net.Tests/Clustering/MemberPartitionGroupServerTests.cs +++ b/src/Hazelcast.Net.Tests/Clustering/MemberPartitionGroupServerTests.cs @@ -130,11 +130,11 @@ public async Task TestClientCollectsClusterEvents(RoutingModes routingMode) { var address1 = "127.0.0.1:5701"; var address2 = "127.0.0.1:5702"; - var addreses = new string[] { address1, address2 }; + var addresses = new string[] { address1, address2 }; var keyCount = 3 * 271; // create a client with the given routing strategy for catching the events. - var client1 = await CreateClient(RoutingStrategy.PartitionGroups, addreses, "client1", routingMode); + var client1 = await CreateClient(RoutingStrategy.PartitionGroups, addresses, "client1", routingMode); var client1Count = 0; var mapName = $"map_{routingMode}"; var map1 = await client1.GetMapAsync(mapName); @@ -149,7 +149,7 @@ await map1.SubscribeAsync((eventHandler) => // create a dummy client for creating events - var client2 = await CreateClient(RoutingStrategy.PartitionGroups, addreses, "client2", RoutingModes.SingleMember); + var client2 = await CreateClient(RoutingStrategy.PartitionGroups, addresses, "client2", RoutingModes.SingleMember); var map2 = await client2.GetMapAsync(map1.Name);