Skip to content

Commit

Permalink
feat: Notify application when a critical ISS event is detected by the…
Browse files Browse the repository at this point in the history
… platform (#17274)

Signed-off-by: Tim Farber-Newman <tim.farber-newman@swirldslabs.com>
  • Loading branch information
timfn-hg authored Jan 9, 2025
1 parent 5a5b7c6 commit 45f108d
Show file tree
Hide file tree
Showing 9 changed files with 254 additions and 10 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -146,6 +146,7 @@
import com.swirlds.platform.system.SwirldMain;
import com.swirlds.platform.system.address.AddressBook;
import com.swirlds.platform.system.events.Event;
import com.swirlds.platform.system.state.notifications.AsyncFatalIssListener;
import com.swirlds.platform.system.state.notifications.StateHashedListener;
import com.swirlds.platform.system.status.PlatformStatus;
import com.swirlds.platform.system.transaction.Transaction;
Expand Down Expand Up @@ -1003,6 +1004,7 @@ private void initializeDagger(@NonNull final State state, @NonNull final InitTri
notifications.unregister(PlatformStatusChangeListener.class, this);
notifications.unregister(ReconnectCompleteListener.class, daggerApp.reconnectListener());
notifications.unregister(StateWriteToDiskCompleteListener.class, daggerApp.stateWriteToDiskListener());
notifications.unregister(AsyncFatalIssListener.class, daggerApp.fatalIssListener());
if (blockStreamEnabled) {
notifications.unregister(StateHashedListener.class, daggerApp.blockStreamManager());
}
Expand Down Expand Up @@ -1056,6 +1058,7 @@ private void initializeDagger(@NonNull final State state, @NonNull final InitTri
notifications.register(PlatformStatusChangeListener.class, this);
notifications.register(ReconnectCompleteListener.class, daggerApp.reconnectListener());
notifications.register(StateWriteToDiskCompleteListener.class, daggerApp.stateWriteToDiskListener());
notifications.register(AsyncFatalIssListener.class, daggerApp.fatalIssListener());
if (blockStreamEnabled) {
notifications.register(StateHashedListener.class, daggerApp.blockStreamManager());
daggerApp
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -66,6 +66,7 @@
import com.swirlds.platform.listeners.StateWriteToDiskCompleteListener;
import com.swirlds.platform.system.InitTrigger;
import com.swirlds.platform.system.Platform;
import com.swirlds.platform.system.state.notifications.AsyncFatalIssListener;
import com.swirlds.state.State;
import com.swirlds.state.lifecycle.StartupNetworks;
import com.swirlds.state.lifecycle.info.NetworkInfo;
Expand Down Expand Up @@ -146,6 +147,8 @@ public interface HederaInjectionComponent {

SubmissionManager submissionManager();

AsyncFatalIssListener fatalIssListener();

@Component.Builder
interface Builder {
@BindsInstance
Expand Down
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
/*
* Copyright (C) 2020-2024 Hedera Hashgraph, LLC
* Copyright (C) 2020-2025 Hedera Hashgraph, LLC
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
Expand All @@ -17,12 +17,14 @@
package com.hedera.node.app.platform;

import com.hedera.node.app.annotations.CommonExecutor;
import com.hedera.node.app.state.listeners.FatalIssListenerImpl;
import com.hedera.node.app.state.listeners.ReconnectListener;
import com.hedera.node.app.state.listeners.WriteStateToDiskListener;
import com.swirlds.common.stream.Signer;
import com.swirlds.platform.listeners.ReconnectCompleteListener;
import com.swirlds.platform.listeners.StateWriteToDiskCompleteListener;
import com.swirlds.platform.system.Platform;
import com.swirlds.platform.system.state.notifications.AsyncFatalIssListener;
import dagger.Binds;
import dagger.Module;
import dagger.Provides;
Expand Down Expand Up @@ -68,4 +70,8 @@ static IntSupplier provideFrontendThrottleSplit(@NonNull final Platform platform
@Binds
@Singleton
StateWriteToDiskCompleteListener bindStateWrittenToDiskListener(WriteStateToDiskListener writeStateToDiskListener);

@Binds
@Singleton
AsyncFatalIssListener bindFatalIssListener(FatalIssListenerImpl listener);
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,41 @@
/*
* Copyright (C) 2025 Hedera Hashgraph, LLC
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/

package com.hedera.node.app.state.listeners;

import com.swirlds.platform.system.state.notifications.AsyncFatalIssListener;
import com.swirlds.platform.system.state.notifications.IssNotification;
import edu.umd.cs.findbugs.annotations.NonNull;
import javax.inject.Inject;
import javax.inject.Singleton;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;

@Singleton
public class FatalIssListenerImpl implements AsyncFatalIssListener {

private static final Logger log = LogManager.getLogger(FatalIssListenerImpl.class);

@Inject
public FatalIssListenerImpl() {
// no-op
}

@Override
public void notify(@NonNull final IssNotification data) {
log.warn("ISS detected (type={}, round={})", data.getIssType(), data.getRound());
}
}
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
/*
* Copyright (C) 2024 Hedera Hashgraph, LLC
* Copyright (C) 2024-2025 Hedera Hashgraph, LLC
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
Expand Down
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
/*
* Copyright (C) 2024 Hedera Hashgraph, LLC
* Copyright (C) 2024-2025 Hedera Hashgraph, LLC
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
Expand All @@ -24,8 +24,10 @@
import com.swirlds.platform.listeners.ReconnectCompleteNotification;
import com.swirlds.platform.listeners.StateWriteToDiskCompleteListener;
import com.swirlds.platform.listeners.StateWriteToDiskCompleteNotification;
import com.swirlds.platform.system.state.notifications.AsyncFatalIssListener;
import com.swirlds.platform.system.state.notifications.IssListener;
import com.swirlds.platform.system.state.notifications.IssNotification;
import com.swirlds.platform.system.state.notifications.IssNotification.IssType;
import com.swirlds.platform.system.state.notifications.NewSignedStateListener;
import com.swirlds.platform.system.state.notifications.StateHashedListener;
import com.swirlds.platform.system.state.notifications.StateHashedNotification;
Expand Down Expand Up @@ -84,5 +86,10 @@ public void sendLatestCompleteStateNotification(
@Override
public void sendIssNotification(@NonNull final IssNotification notification) {
notificationEngine.dispatch(IssListener.class, notification);

if (IssType.CATASTROPHIC_ISS == notification.getIssType() || IssType.SELF_ISS == notification.getIssType()) {
// Forward notification to application
notificationEngine.dispatch(AsyncFatalIssListener.class, notification);
}
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,29 @@
/*
* Copyright (C) 2025 Hedera Hashgraph, LLC
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/

package com.swirlds.platform.system.state.notifications;

import com.swirlds.common.notification.DispatchMode;
import com.swirlds.common.notification.DispatchModel;
import com.swirlds.common.notification.DispatchOrder;
import com.swirlds.common.notification.Listener;

/**
* Listener for fatal ISS events (i.e. of type SELF or CATASTROPHIC). This listener is ordered and asynchronous.
* If you require ordered and synchronous dispatch that includes all ISS events, then use {@link IssListener}.
*/
@DispatchModel(mode = DispatchMode.ASYNC, order = DispatchOrder.ORDERED)
public interface AsyncFatalIssListener extends Listener<IssNotification> {}
Original file line number Diff line number Diff line change
@@ -0,0 +1,155 @@
/*
* Copyright (C) 2025 Hedera Hashgraph, LLC
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/

package com.swirlds.platform.components;

import static org.junit.jupiter.api.Assertions.assertDoesNotThrow;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertNotNull;
import static org.mockito.ArgumentMatchers.eq;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.times;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.verifyNoMoreInteractions;

import com.swirlds.common.crypto.DigestType;
import com.swirlds.common.crypto.Hash;
import com.swirlds.common.notification.NotificationEngine;
import com.swirlds.common.notification.NotificationResult;
import com.swirlds.common.threading.futures.StandardFuture.CompletionCallback;
import com.swirlds.platform.components.appcomm.CompleteStateNotificationWithCleanup;
import com.swirlds.platform.listeners.PlatformStatusChangeListener;
import com.swirlds.platform.listeners.PlatformStatusChangeNotification;
import com.swirlds.platform.listeners.ReconnectCompleteListener;
import com.swirlds.platform.listeners.ReconnectCompleteNotification;
import com.swirlds.platform.listeners.StateWriteToDiskCompleteListener;
import com.swirlds.platform.listeners.StateWriteToDiskCompleteNotification;
import com.swirlds.platform.system.SwirldState;
import com.swirlds.platform.system.state.notifications.AsyncFatalIssListener;
import com.swirlds.platform.system.state.notifications.IssListener;
import com.swirlds.platform.system.state.notifications.IssNotification;
import com.swirlds.platform.system.state.notifications.IssNotification.IssType;
import com.swirlds.platform.system.state.notifications.NewSignedStateListener;
import com.swirlds.platform.system.state.notifications.NewSignedStateNotification;
import com.swirlds.platform.system.state.notifications.StateHashedListener;
import com.swirlds.platform.system.state.notifications.StateHashedNotification;
import com.swirlds.platform.system.status.PlatformStatus;
import java.time.Instant;
import java.util.List;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.params.ParameterizedTest;
import org.junit.jupiter.params.provider.Arguments;
import org.junit.jupiter.params.provider.MethodSource;
import org.mockito.ArgumentCaptor;

public class DefaultAppNotifierTest {

NotificationEngine notificationEngine;
AppNotifier notifier;

@BeforeEach
void beforeEach() {
notificationEngine = mock(NotificationEngine.class);
notifier = new DefaultAppNotifier(notificationEngine);
}

@Test
void testStateWrittenToDiskNotificationSent() {
final StateWriteToDiskCompleteNotification notification =
new StateWriteToDiskCompleteNotification(100, Instant.now(), false);

assertDoesNotThrow(() -> notifier.sendStateWrittenToDiskNotification(notification));
verify(notificationEngine, times(1)).dispatch(StateWriteToDiskCompleteListener.class, notification);
verifyNoMoreInteractions(notificationEngine);
}

@Test
void testStateHashNotificationSent() {
final StateHashedNotification notification = new StateHashedNotification(100L, new Hash(DigestType.SHA_384));

assertDoesNotThrow(() -> notifier.sendStateHashedNotification(notification));
verify(notificationEngine, times(1)).dispatch(StateHashedListener.class, notification);
verifyNoMoreInteractions(notificationEngine);
}

@Test
void testReconnectCompleteNotificationSent() {
final SwirldState state = mock(SwirldState.class);
final ReconnectCompleteNotification notification =
new ReconnectCompleteNotification(100L, Instant.now(), state);

assertDoesNotThrow(() -> notifier.sendReconnectCompleteNotification(notification));
verify(notificationEngine, times(1)).dispatch(ReconnectCompleteListener.class, notification);
verifyNoMoreInteractions(notificationEngine);
}

@Test
void testPlatformStatusChangeNotificationSent() {
final PlatformStatus status = PlatformStatus.ACTIVE;
final ArgumentCaptor<PlatformStatusChangeNotification> captor =
ArgumentCaptor.forClass(PlatformStatusChangeNotification.class);

assertDoesNotThrow(() -> notifier.sendPlatformStatusChangeNotification(status));
verify(notificationEngine, times(1)).dispatch(eq(PlatformStatusChangeListener.class), captor.capture());
verifyNoMoreInteractions(notificationEngine);

final PlatformStatusChangeNotification notification = captor.getValue();
assertNotNull(notification);
assertEquals(status, notification.getNewStatus());
}

@Test
void testLatestCompleteStateNotificationSent() {
final SwirldState state = mock(SwirldState.class);
final CompletionCallback<NotificationResult<NewSignedStateNotification>> cleanup =
mock(CompletionCallback.class);
final NewSignedStateNotification signedStateNotification =
new NewSignedStateNotification(state, 100L, Instant.now());
final CompleteStateNotificationWithCleanup notificationWithCleanup =
new CompleteStateNotificationWithCleanup(signedStateNotification, cleanup);

assertDoesNotThrow(() -> notifier.sendLatestCompleteStateNotification(notificationWithCleanup));
verify(notificationEngine, times(1)).dispatch(NewSignedStateListener.class, signedStateNotification, cleanup);
verifyNoMoreInteractions(notificationEngine);
}

public static List<Arguments> issTypes() {
return List.of(
Arguments.of(IssType.CATASTROPHIC_ISS, true),
Arguments.of(IssType.SELF_ISS, true),
Arguments.of(IssType.OTHER_ISS, false));
}

@ParameterizedTest
@MethodSource("issTypes")
void testIssNotificationSent(final IssType type, final boolean isFatal) {
final IssNotification notification = new IssNotification(100L, type);

assertDoesNotThrow(() -> notifier.sendIssNotification(notification));

// verify the ISS notification is always sent to the IssListener
verify(notificationEngine, times(1)).dispatch(IssListener.class, notification);

if (isFatal) {
// if the ISS event is considered fatal to the local node, verify the event is also sent to the
// FatalIssListener
verify(notificationEngine, times(1)).dispatch(AsyncFatalIssListener.class, notification);
}

verifyNoMoreInteractions(notificationEngine);
}
}
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
/*
* Copyright (C) 2022-2024 Hedera Hashgraph, LLC
* Copyright (C) 2022-2025 Hedera Hashgraph, LLC
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
Expand Down Expand Up @@ -135,7 +135,7 @@ void selfIssAutomatedRecovery() {

final SerializableLong issRound = simpleScratchpad.get(IssScratchpad.LAST_ISS_ROUND);
assertNotNull(issRound);
assertEquals(issRound.getValue(), 1234L);
assertEquals(1234L, issRound.getValue());
}

@Test
Expand Down Expand Up @@ -167,7 +167,7 @@ void selfIssNoAction() {

final SerializableLong issRound = simpleScratchpad.get(IssScratchpad.LAST_ISS_ROUND);
assertNotNull(issRound);
assertEquals(issRound.getValue(), 1234L);
assertEquals(1234L, issRound.getValue());
}

@Test
Expand Down Expand Up @@ -205,7 +205,7 @@ void selfIssAlwaysFreeze() {

final SerializableLong issRound = simpleScratchpad.get(IssScratchpad.LAST_ISS_ROUND);
assertNotNull(issRound);
assertEquals(issRound.getValue(), 1234L);
assertEquals(1234L, issRound.getValue());
}

@Test
Expand Down Expand Up @@ -237,7 +237,7 @@ void catastrophicIssNoAction() {

final SerializableLong issRound = simpleScratchpad.get(IssScratchpad.LAST_ISS_ROUND);
assertNotNull(issRound);
assertEquals(issRound.getValue(), 1234L);
assertEquals(1234L, issRound.getValue());
}

@Test
Expand Down Expand Up @@ -275,7 +275,7 @@ void catastrophicIssAlwaysFreeze() {

final SerializableLong issRound = simpleScratchpad.get(IssScratchpad.LAST_ISS_ROUND);
assertNotNull(issRound);
assertEquals(issRound.getValue(), 1234L);
assertEquals(1234L, issRound.getValue());
}

@Test
Expand Down Expand Up @@ -313,6 +313,6 @@ void catastrophicIssFreezeOnCatastrophic() {

final SerializableLong issRound = simpleScratchpad.get(IssScratchpad.LAST_ISS_ROUND);
assertNotNull(issRound);
assertEquals(issRound.getValue(), 1234L);
assertEquals(1234L, issRound.getValue());
}
}

0 comments on commit 45f108d

Please sign in to comment.