Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Akka.Delivery: fix ProducerControllerImpl<T> state bug #7034

Merged
merged 5 commits into from
Jan 4, 2024
Merged

Akka.Delivery: fix ProducerControllerImpl<T> state bug #7034

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
b1d80bf
Troubleshoot ReliableDeliveryRandomSpecs
Aaronontheweb Jan 3, 2024
a5b47f8
fixed logging bug
Aaronontheweb Jan 3, 2024
1f7a683
adding better logging around fuzz factor
Aaronontheweb Jan 3, 2024
7d84b8e
Merge branch 'dev' into fix-racy-tests
Aaronontheweb Jan 4, 2024
d784f5d
found the bug
Aaronontheweb Jan 4, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
67 changes: 35 additions & 32 deletions src/core/Akka.Tests/Delivery/ReliableDeliveryRandomSpecs.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,11 +20,13 @@ namespace Akka.Tests.Delivery;

public class ReliableDeliveryRandomSpecs : TestKit.Xunit2.TestKit
{
internal static readonly Config Config = @"akka.reliable-delivery.consumer-controller{
private static readonly Config Config = @"akka.reliable-delivery.consumer-controller{
flow-control-window = 20
resend-interval-min = 500ms
resend-interval-max = 2s
}";
}
akka.loglevel = DEBUG
";

public ReliableDeliveryRandomSpecs(ITestOutputHelper output) : this(output, Config)
{
Expand Down Expand Up @@ -57,16 +59,6 @@ private async Task Test(int numberOfMessages, double producerDropProbability, do
consumerDropProbability, producerDropProbability,
consumerDelay, producerDelay, durableFailProbability, durableDelay);

// RandomFlakyNetwork to simulate lost messages from producerController to consumerController
double ConsumerDrop(object msg)
{
return msg switch
{
ConsumerController.SequencedMessage<TestConsumer.Job> _ => consumerDropProbability,
_ => 0
};
}

var consumerEndProbe = CreateTestProbe();
var consumerController = Sys.ActorOf(ConsumerController.CreateWithFuzzing<TestConsumer.Job>(Sys, Option<IActorRef>.None, ConsumerDrop, consumerControllerSettings), $"consumer-controller-{_idCount}");

Expand All @@ -75,18 +67,6 @@ double ConsumerDrop(object msg)
TestConsumer.PropsFor(consumerDelay, numberOfMessages, consumerEndProbe.Ref, consumerController),
$"consumer-{_idCount}");

// RandomFlakyNetwork to simulate lost messages from consumerController to producerController
double ProducerDrop(object msg)
{
return msg switch
{
ProducerController.Request _ => producerDropProbability,
ProducerController.Resend _ => producerDropProbability,
ProducerController.RegisterConsumer<TestConsumer.Job> _ => producerDropProbability,
_ => 0
};
}

var stateHolder = new AtomicReference<DurableProducerQueueStateHolder<TestConsumer.Job>>(DurableProducerQueueStateHolder<TestConsumer.Job>.Empty);
var durableQueue = durableFailProbability.Select(p =>
{
Expand All @@ -105,50 +85,73 @@ double ProducerDrop(object msg)
new ConsumerController.RegisterToProducerController<TestConsumer.Job>(producerController));

await consumerEndProbe.ExpectMsgAsync<TestConsumer.Collected>(TimeSpan.FromSeconds(120));
return;

// RandomFlakyNetwork to simulate lost messages from producerController to consumerController
double ConsumerDrop(object msg)
{
return msg switch
{
ConsumerController.SequencedMessage<TestConsumer.Job> _ => consumerDropProbability,
_ => 0
};
}

// RandomFlakyNetwork to simulate lost messages from consumerController to producerController
double ProducerDrop(object msg)
{
return msg switch
{
ProducerController.Request _ => producerDropProbability,
ProducerController.Resend _ => producerDropProbability,
ProducerController.RegisterConsumer<TestConsumer.Job> _ => producerDropProbability,
_ => 0
};
}
}

[Fact]
public async Task ReliableDelivery_with_random_failures_must_work_with_flaky_network()
public Task ReliableDelivery_with_random_failures_must_work_with_flaky_network()
{
NextId();
var consumerDropProbability = 0.1 + ThreadLocalRandom.Current.NextDouble() * 0.2;
var producerDropProbability = 0.1 + ThreadLocalRandom.Current.NextDouble() * 0.2;

await Test(numberOfMessages: 63, producerDropProbability: producerDropProbability,
return Test(numberOfMessages: 63, producerDropProbability: producerDropProbability,
consumerDropProbability: consumerDropProbability, Option<double>.None, true);
}

[Fact]
public async Task ReliableDelivery_with_random_failures_must_work_with_flaky_DurableProducerQueue()
public Task ReliableDelivery_with_random_failures_must_work_with_flaky_DurableProducerQueue()
{
NextId();
var durableFailProbability = 0.1 + ThreadLocalRandom.Current.NextDouble() * 0.1;

await Test(numberOfMessages: 31, producerDropProbability: 0.0,
return Test(numberOfMessages: 31, producerDropProbability: 0.0,
consumerDropProbability:0.0, durableFailProbability, true);
}

[Fact]
public async Task ReliableDelivery_with_random_failures_must_work_with_flaky_network_and_flaky_DurableProducerQueue()
public Task ReliableDelivery_with_random_failures_must_work_with_flaky_network_and_flaky_DurableProducerQueue()
{
NextId();
var consumerDropProbability = 0.1 + ThreadLocalRandom.Current.NextDouble() * 0.1;
var producerDropProbability = 0.1 + ThreadLocalRandom.Current.NextDouble() * 0.1;
var durableFailProbability = 0.1 + ThreadLocalRandom.Current.NextDouble() * 0.1;


await Test(numberOfMessages: 17, producerDropProbability: producerDropProbability,
return Test(numberOfMessages: 17, producerDropProbability: producerDropProbability,
consumerDropProbability: consumerDropProbability, durableFailProbability, true);
}

[Fact]
public async Task ReliableDelivery_with_random_failures_must_work_with_flaky_network_without_resending()
public Task ReliableDelivery_with_random_failures_must_work_with_flaky_network_without_resending()
{
NextId();
var consumerDropProbability = 0.1 + ThreadLocalRandom.Current.NextDouble() * 0.4;
var producerDropProbability = 0.1 + ThreadLocalRandom.Current.NextDouble() * 0.3;

await Test(numberOfMessages: 63, producerDropProbability: producerDropProbability,
return Test(numberOfMessages: 63, producerDropProbability: producerDropProbability,
consumerDropProbability: consumerDropProbability, Option<double>.None, false);
}
}
70 changes: 38 additions & 32 deletions src/core/Akka/Delivery/Internal/ConsumerControllerImpl.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -57,7 +57,11 @@ protected internal override bool AroundReceive(Receive receive, object message)
{
// TESTING PURPOSES ONLY - used to simulate network failures.
if (_fuzzingControl != null && ThreadLocalRandom.Current.NextDouble() < _fuzzingControl(message))
{
_log.Debug("[Testing] dropping message [{0}] due to fuzzing factor", message);
return true;
}

return base.AroundReceive(receive, message);
}

Expand Down Expand Up @@ -281,38 +285,6 @@ private void WaitingForConfirmation(SequencedMessage<T> sequencedMessage)
if (_log.IsDebugEnabled)
_log.Debug("Received Confirmed seqNr [{0}] from consumer, stashed size [{1}].", seqNr, Stash.Count);

long ComputeNextSeqNr()
{
if (sequencedMessage.First)
{
// confirm the first message immediately to cancel resending of first
var newRequestedSeqNr = seqNr - 1 + Settings.FlowControlWindow;
_log.Debug("Sending Request after first with confirmedSeqNr [{0}], requestUpToSeqNr [{1}]", seqNr,
newRequestedSeqNr);
CurrentState.ProducerController.Tell(new Request(seqNr, newRequestedSeqNr, ResendLost, false));
return newRequestedSeqNr;
}

if (CurrentState.RequestedSeqNr - seqNr == Settings.FlowControlWindow / 2)
{
var newRequestedSeqNr = CurrentState.RequestedSeqNr + Settings.FlowControlWindow / 2;
_log.Debug("Sending Request with confirmedSeqNr [{0}], requestUpToSeqNr [{1}]", seqNr,
newRequestedSeqNr);
CurrentState.ProducerController.Tell(new Request(seqNr, newRequestedSeqNr, ResendLost, false));
_retryTimer.Start(); // reset interval since Request was just sent
return newRequestedSeqNr;
}

if (sequencedMessage.Ack)
{
if (_log.IsDebugEnabled)
_log.Debug("Sending Ack seqNr [{0}]", seqNr);
CurrentState.ProducerController.Tell(new Ack(seqNr));
}

return CurrentState.RequestedSeqNr;
}

var requestedSeqNr = ComputeNextSeqNr();
if (CurrentState.Stopping && Stash.IsEmpty)
{
Expand Down Expand Up @@ -351,6 +323,40 @@ async Task ShutDownAndStop()
Stash.Unstash();
Become(Active);
}

return;

long ComputeNextSeqNr()
{
if (sequencedMessage.First)
{
// confirm the first message immediately to cancel resending of first
var newRequestedSeqNr = seqNr - 1 + Settings.FlowControlWindow;
_log.Debug("Sending Request after first with confirmedSeqNr [{0}], requestUpToSeqNr [{1}]", seqNr,
newRequestedSeqNr);
CurrentState.ProducerController.Tell(new Request(seqNr, newRequestedSeqNr, ResendLost, false));
return newRequestedSeqNr;
}

if (CurrentState.RequestedSeqNr - seqNr == Settings.FlowControlWindow / 2)
{
var newRequestedSeqNr = CurrentState.RequestedSeqNr + Settings.FlowControlWindow / 2;
_log.Debug("Sending Request with confirmedSeqNr [{0}], requestUpToSeqNr [{1}]", seqNr,
newRequestedSeqNr);
CurrentState.ProducerController.Tell(new Request(seqNr, newRequestedSeqNr, ResendLost, false));
_retryTimer.Start(); // reset interval since Request was just sent
return newRequestedSeqNr;
}

if (sequencedMessage.Ack)
{
if (_log.IsDebugEnabled)
_log.Debug("Sending Ack seqNr [{0}]", seqNr);
CurrentState.ProducerController.Tell(new Ack(seqNr));
}

return CurrentState.RequestedSeqNr;
}
});

Receive<SequencedMessage<T>>(msg =>
Expand Down
4 changes: 2 additions & 2 deletions src/core/Akka/Delivery/Internal/ProducerControllerImpl.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -490,10 +490,10 @@ private void ReceiveRequest(long newConfirmedSeqNr, long newRequestedSeqNr, bool
: newRequestedSeqNr;

if (newRequestedSeqNr2 != newRequestedSeqNr)
_log.Debug("Expanded requestedSeqNr from [{0}] to [{1}], because current [{3}] and all were probably lost.",
_log.Debug("Expanded requestedSeqNr from [{0}] to [{1}], because current [{2}] and all were probably lost.",
newRequestedSeqNr, newRequestedSeqNr2, stateAfterAck.CurrentSeqNr);

if (newRequestedSeqNr > CurrentState.RequestedSeqNr)
if (newRequestedSeqNr2 > CurrentState.RequestedSeqNr)
Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This was the bug - we need to evaluate newRequestedSeqNr2 here for when the producer believes that the consumer has lost data (and re-delivery isn't enabled). This is what was causing this test to get stuck and probably some of the others in this suite.

{
bool newRequested;
if (CurrentState.StoreMessageSentInProgress != 0)
Expand Down
Loading