added assert invariants to build script

cleaned up gossip class to assert more invariants

build.fsx

@@ -348,6 +348,7 @@ Target "MultiNodeTests" (fun _ ->
 
 Target "MultiNodeTestsNetCore" (fun _ ->
     if not skipBuild.Value then
+        setEnvironVar "akka.cluster.assert" "on" // needed to enable assert invariants for Akka.Cluster
         let multiNodeTestPath = findToolInSubPath "Akka.MultiNodeTestRunner.dll" (currentDirectory @@ "src" @@ "core" @@ "Akka.MultiNodeTestRunner" @@ "bin" @@ "Release" @@ testNetCoreVersion @@ "win10-x64" @@ "publish")
 
         let projects =
@@ -388,6 +389,7 @@ Target "MultiNodeTestsNetCore" (fun _ ->
 )
 Target "MultiNodeTestsNet" (fun _ ->
     if not skipBuild.Value then
+        setEnvironVar "akka.cluster.assert" "on" // needed to enable assert invariants for Akka.Cluster
         let multiNodeTestPath = findToolInSubPath "Akka.MultiNodeTestRunner.dll" (currentDirectory @@ "src" @@ "core" @@ "Akka.MultiNodeTestRunner" @@ "bin" @@ "Release" @@ testNetVersion @@ "win10-x64" @@ "publish")
 
         let projects =

src/core/Akka.Cluster.Tests.MultiNode/StressSpec.cs

@@ -1,4 +1,11 @@
-using System;
+//-----------------------------------------------------------------------
+// <copyright file="StressSpec.cs" company="Akka.NET Project">
+//     Copyright (C) 2009-2021 Lightbend Inc. <http://www.lightbend.com>
+//     Copyright (C) 2013-2021 .NET Foundation <https://github.com/akkadotnet/akka.net>
+// </copyright>
+//-----------------------------------------------------------------------
+
+using System;
 using System.Collections.Generic;
 using System.Collections.Immutable;
 using System.Diagnostics;
@@ -68,8 +75,8 @@ public StressSpecConfig()
     }
     akka.actor.provider = cluster
     akka.cluster {
-      failure-detector.acceptable-heartbeat-pause =  3s
-      downing-provider-class = ""Akka.Cluster.SBR.SplitBrainResolverProvider, Akka.Cluster""
+      failure-detector.acceptable-heartbeat-pause = 3s
+      downing-provider-class = ""Akka.Cluster.SplitBrainResolver, Akka.Cluster""
       split-brain-resolver {
           active-strategy = keep-majority #TODO: remove this once it's been made default
           stable-after = 10s
@@ -266,8 +273,7 @@ public string FormatPhi()
             if (_phiValuesObservedByNode.IsEmpty) return string.Empty;
             else
             {
-                //var lines = _phiValuesObservedByNode.Select(
-                //    x => x.Value.SelectMany(y => FormatPhiLine(x.Key, y.Address, y)));
+
                 var lines = (from mon in _phiValuesObservedByNode from phi in mon.Value select FormatPhiLine(mon.Key, phi.Address, phi));
                 return FormatPhiHeader + Environment.NewLine + string.Join(Environment.NewLine, lines);
             }
@@ -917,26 +923,36 @@ string FormatNodeLeave()
                 var currentRoles = Roles.Take(NbrUsedRoles - 1).ToArray();
                 var title = $"{FormatNodeLeave()} one from {NbrUsedRoles} nodes cluster";
                 CreateResultAggregator(title, expectedResults:currentRoles.Length, true);
+
                 var removeRole = Roles[NbrUsedRoles - 1];
                 var removeAddress = GetAddress(removeRole);
-
+                Console.WriteLine($"Preparing to {FormatNodeLeave()}[{removeAddress}] role [{removeRole.Name}] out of [{Roles.Count}]");
                 RunOn(() =>
                 {
                     var watchee = Sys.ActorOf(Props.Create(() => new Watchee()), "watchee");
-                    if(!shutdown)
-                        Cluster.Leave(GetAddress(Myself));
+                    Console.WriteLine("Created watchee [{0}]", watchee);
                 }, removeRole);
 
                 EnterBarrier("watchee-created-" + Step);
 
                 RunOn(() =>
                 {
-                    Sys.ActorSelection(Node(removeRole) / "user" / "watchee").Tell(new Identify("watchee"), IdentifyProbe.Ref);
-                    var watchee = IdentifyProbe.ExpectMsg<ActorIdentity>().Subject;
-                    Watch(watchee);
+                    AwaitAssert(() =>
+                    {
+                        Sys.ActorSelection(new RootActorPath(removeAddress) / "user" / "watchee").Tell(new Identify("watchee"), IdentifyProbe.Ref);
+                        var watchee = IdentifyProbe.ExpectMsg<ActorIdentity>(TimeSpan.FromSeconds(1)).Subject;
+                        Watch(watchee);
+                    }, interval:TimeSpan.FromSeconds(1.25d));
+
                 }, Roles.First());
                 EnterBarrier("watchee-established-" + Step);
 
+                RunOn(() =>
+                {
+                    if (!shutdown)
+                        Cluster.Leave(GetAddress(Myself));
+                }, removeRole);
+
                 RunOn(() =>
                 {
                     ReportResult(() =>
@@ -950,7 +966,7 @@ string FormatNodeLeave()
                                     Log.Info("Shutting down [{0}]", removeAddress);
                                 }
 
-                                TestConductor.Exit(removeRole, 0).Wait(RemainingOrDefault);
+                                TestConductor.Exit(removeRole, 0).Wait();
                             }
                         }, Roles.First());
 
@@ -1041,7 +1057,7 @@ public void PartitionSeveral(int numberOfNodes)
                         {
                             foreach (var y in removeRoles)
                             {
-                                TestConductor.Blackhole(x, y, ThrottleTransportAdapter.Direction.Both);
+                                TestConductor.Blackhole(x, y, ThrottleTransportAdapter.Direction.Both).Wait();
                             }
                         }
                     }, Roles.First());
@@ -1222,6 +1238,17 @@ public void Cluster_under_stress()
             IncrementStep();
             MustLeaveNodesOneByOneFromLargeCluster();
             IncrementStep();
+            MustShutdownNodesOneByOneFromLargeCluster();
+            IncrementStep();
+            MustLeaveSeveralNodes();
+            IncrementStep();
+            MustShutdownSeveralNodes();
+            IncrementStep();
+            MustShutdownNodesOneByOneFromSmallCluster();
+            IncrementStep();
+            MustLeaveNodesOneByOneFromSmallCluster();
+            IncrementStep();
+            MustLogClrInfo();
         }
 
         public void MustLogSettings()
@@ -1307,7 +1334,7 @@ public void MustGossipWhenIdle()
         public void MustDownPartitionedNodes()
         {
             PartitionSeveral(Settings.NumberOfNodesPartition);
-            NbrUsedRoles += Settings.NumberOfNodesPartition;
+            NbrUsedRoles -= Settings.NumberOfNodesPartition;
             EnterBarrier("after-" + Step);
         }
 
@@ -1316,5 +1343,46 @@ public void MustLeaveNodesOneByOneFromLargeCluster()
             RemoveOneByOne(Settings.NumberOfNodesLeavingOneByOneLarge, shutdown:false);
             EnterBarrier("after-" + Step);
         }
+
+        public void MustShutdownNodesOneByOneFromLargeCluster()
+        {
+            RemoveOneByOne(Settings.NumberOfNodesShutdownOneByOneLarge, shutdown: true);
+            EnterBarrier("after-" + Step);
+        }
+
+        public void MustLeaveSeveralNodes()
+        {
+            RemoveSeveral(Settings.NumberOfNodesLeaving, shutdown: false);
+            NbrUsedRoles -= Settings.NumberOfNodesLeaving;
+            EnterBarrier("after-" + Step);
+        }
+
+        public void MustShutdownSeveralNodes()
+        {
+            RemoveSeveral(Settings.NumberOfNodesShutdown, shutdown: true);
+            NbrUsedRoles -= Settings.NumberOfNodesShutdown;
+            EnterBarrier("after-" + Step);
+        }
+
+        public void MustShutdownNodesOneByOneFromSmallCluster()
+        {
+            RemoveOneByOne(Settings.NumberOfNodesShutdownOneByOneSmall, true);
+            EnterBarrier("after-" + Step);
+        }
+
+        public void MustLeaveNodesOneByOneFromSmallCluster()
+        {
+            RemoveOneByOne(Settings.NumberOfNodesLeavingOneByOneSmall, false);
+            EnterBarrier("after-" + Step);
+        }
+
+        public void MustLogClrInfo()
+        {
+            if (Settings.Infolog)
+            {
+                Log.Info("StressSpec CLR: " + Environment.NewLine + "{0}", ClrInfo());
+            }
+            EnterBarrier("after-" + Step);
+        }
     }
 }

src/core/Akka.Cluster/Gossip.cs

@@ -156,25 +156,33 @@ public Gossip Copy(ImmutableSortedSet<Member> members = null, GossipOverview ove
 
         private void AssertInvariants()
         {
-            if (_members.Any(m => m.Status == MemberStatus.Removed))
+            void IfTrueThrow(bool func, string expected, string actual)
             {
-                var members = string.Join(", ", _members.Where(m => m.Status == MemberStatus.Removed).Select(m => m.ToString()));
-                throw new ArgumentException($"Live members must not have status [Removed], got {members}", nameof(_members));
+                if (func) throw new ArgumentException($"{expected}, but found [{actual}]");
             }
 
+
+            IfTrueThrow(_members.Any(m => m.Status == MemberStatus.Removed),
+                expected: "Live members must not have status [Removed]",
+                actual: string.Join(", ",
+                    _members.Where(m => m.Status == MemberStatus.Removed).Select(m => m.ToString())));
+
+
             var inReachabilityButNotMember = _overview.Reachability.AllObservers.Except(_members.Select(m => m.UniqueAddress));
-            if (!inReachabilityButNotMember.IsEmpty)
-            {
-                var inreachability = string.Join(", ", inReachabilityButNotMember.Select(a => a.ToString()));
-                throw new ArgumentException($"Nodes not part of cluster in reachability table, got {inreachability}", nameof(_overview));
-            }
+            IfTrueThrow(!inReachabilityButNotMember.IsEmpty,
+                expected: "Nodes not part of cluster in reachability table",
+                actual: string.Join(", ", inReachabilityButNotMember.Select(a => a.ToString())));
+
+            var inReachabilityVersionsButNotMember =
+                _overview.Reachability.Versions.Keys.Except(Members.Select(x => x.UniqueAddress)).ToImmutableHashSet();
+            IfTrueThrow(!inReachabilityVersionsButNotMember.IsEmpty,
+                expected: "Nodes not part of cluster in reachability versions table",
+                actual: string.Join(", ", inReachabilityVersionsButNotMember.Select(a => a.ToString())));
 
             var seenButNotMember = _overview.Seen.Except(_members.Select(m => m.UniqueAddress));
-            if (!seenButNotMember.IsEmpty)
-            {
-                var seen = string.Join(", ", seenButNotMember.Select(a => a.ToString()));
-                throw new ArgumentException($"Nodes not part of cluster have marked the Gossip as seen, got {seen}", nameof(_overview));
-            }
+            IfTrueThrow(!seenButNotMember.IsEmpty,
+                expected: "Nodes not part of cluster have marked the Gossip as seen",
+                actual: string.Join(", ", seenButNotMember.Select(a => a.ToString())));
         }
 
         //TODO: Serializer should ignore
@@ -274,7 +282,7 @@ public Gossip Merge(Gossip that)
             var mergedMembers = EmptyMembers.Union(Member.PickHighestPriority(this._members, that._members));
 
             // 3. merge reachability table by picking records with highest version
-            var mergedReachability = this._overview.Reachability.Merge(mergedMembers.Select(m => m.UniqueAddress),
+            var mergedReachability = _overview.Reachability.Merge(mergedMembers.Select(m => m.UniqueAddress),
                 that._overview.Reachability);
 
             // 4. Nobody can have seen this new gossip yet
@@ -448,7 +456,7 @@ public override string ToString()
     /// <summary>
     /// Represents the overview of the cluster, holds the cluster convergence table and set with unreachable nodes.
     /// </summary>
-    class GossipOverview
+    internal class GossipOverview
     {
         readonly ImmutableHashSet<UniqueAddress> _seen;
         readonly Reachability _reachability;