feat: overhaul SignedState.getAddressBook()

hedera-node/hedera-app/src/main/java/com/hedera/node/app/roster/RosterService.java

@@ -20,10 +20,10 @@
 import static java.util.Objects.requireNonNull;
 
 import com.hedera.hapi.node.state.roster.Roster;
-import com.hedera.node.app.roster.schemas.V0540RosterSchema;
 import com.hedera.node.app.roster.schemas.V057RosterSchema;
 import com.swirlds.platform.state.service.ReadablePlatformStateStore;
 import com.swirlds.platform.state.service.WritableRosterStore;
+import com.swirlds.platform.state.service.schemas.V0540RosterSchema;
 import com.swirlds.state.lifecycle.SchemaRegistry;
 import com.swirlds.state.lifecycle.Service;
 import edu.umd.cs.findbugs.annotations.NonNull;

hedera-node/hedera-app/src/main/java/com/hedera/node/app/tss/TssBaseServiceImpl.java

@@ -32,7 +32,6 @@
 import com.hedera.hapi.services.auxiliary.tss.TssMessageTransactionBody;
 import com.hedera.hapi.services.auxiliary.tss.TssShareSignatureTransactionBody;
 import com.hedera.node.app.roster.RosterService;
-import com.hedera.node.app.roster.schemas.V0540RosterSchema;
 import com.hedera.node.app.services.ServiceMigrator;
 import com.hedera.node.app.spi.AppContext;
 import com.hedera.node.app.spi.workflows.HandleContext;
@@ -53,6 +52,7 @@
 import com.swirlds.metrics.api.Metrics;
 import com.swirlds.platform.roster.RosterUtils;
 import com.swirlds.platform.state.service.ReadableRosterStore;
+import com.swirlds.platform.state.service.schemas.V0540RosterSchema;
 import com.swirlds.platform.system.InitTrigger;
 import com.swirlds.state.State;
 import com.swirlds.state.lifecycle.SchemaRegistry;

hedera-node/hedera-app/src/test/java/com/hedera/node/app/info/DiskStartupNetworksTest.java

@@ -20,14 +20,14 @@
 import static com.hedera.node.app.info.DiskStartupNetworks.ARCHIVE;
 import static com.hedera.node.app.info.DiskStartupNetworks.GENESIS_NETWORK_JSON;
 import static com.hedera.node.app.info.DiskStartupNetworks.OVERRIDE_NETWORK_JSON;
-import static com.hedera.node.app.roster.schemas.V0540RosterSchema.ROSTER_KEY;
-import static com.hedera.node.app.roster.schemas.V0540RosterSchema.ROSTER_STATES_KEY;
 import static com.hedera.node.app.service.addressbook.impl.schemas.V053AddressBookSchema.NODES_KEY;
 import static com.hedera.node.app.spi.AppContext.Gossip.UNAVAILABLE_GOSSIP;
 import static com.hedera.node.app.tss.schemas.V0560TssBaseSchema.TSS_MESSAGE_MAP_KEY;
 import static com.hedera.node.app.tss.schemas.V0560TssBaseSchema.TSS_VOTE_MAP_KEY;
 import static com.hedera.node.app.workflows.standalone.TransactionExecutorsTest.FAKE_NETWORK_INFO;
 import static com.hedera.node.app.workflows.standalone.TransactionExecutorsTest.NO_OP_METRICS;
+import static com.swirlds.platform.state.service.schemas.V0540RosterSchema.ROSTER_KEY;
+import static com.swirlds.platform.state.service.schemas.V0540RosterSchema.ROSTER_STATES_KEY;
 import static java.util.Collections.emptyList;
 import static java.util.Objects.requireNonNull;
 import static org.assertj.core.api.AssertionsForClassTypes.assertThat;

hedera-node/hedera-app/src/test/java/com/hedera/node/app/roster/RosterServiceTest.java

@@ -22,8 +22,8 @@
 import static org.mockito.Mockito.verify;
 
 import com.hedera.hapi.node.state.roster.Roster;
-import com.hedera.node.app.roster.schemas.V0540RosterSchema;
 import com.hedera.node.app.roster.schemas.V057RosterSchema;
+import com.swirlds.platform.state.service.schemas.V0540RosterSchema;
 import com.swirlds.state.lifecycle.Schema;
 import com.swirlds.state.lifecycle.SchemaRegistry;
 import java.util.function.Predicate;

hedera-node/hedera-app/src/test/java/com/hedera/node/app/roster/schemas/V0540RosterSchemaTest.java

@@ -16,8 +16,8 @@
 
 package com.hedera.node.app.roster.schemas;
 
-import static com.hedera.node.app.roster.schemas.V0540RosterSchema.ROSTER_KEY;
-import static com.hedera.node.app.roster.schemas.V0540RosterSchema.ROSTER_STATES_KEY;
+import static com.swirlds.platform.state.service.schemas.V0540RosterSchema.ROSTER_KEY;
+import static com.swirlds.platform.state.service.schemas.V0540RosterSchema.ROSTER_STATES_KEY;
 import static org.assertj.core.api.Assertions.assertThat;
 import static org.junit.jupiter.api.Assertions.assertEquals;
 import static org.mockito.Mockito.mock;
@@ -28,6 +28,7 @@
 import com.hedera.hapi.node.base.SemanticVersion;
 import com.hedera.hapi.node.state.roster.RosterState;
 import com.hedera.node.app.spi.fixtures.util.LoggingSubject;
+import com.swirlds.platform.state.service.schemas.V0540RosterSchema;
 import com.swirlds.state.lifecycle.MigrationContext;
 import com.swirlds.state.lifecycle.StateDefinition;
 import com.swirlds.state.spi.WritableSingletonState;

hedera-node/hedera-app/src/test/java/com/hedera/node/app/workflows/handle/steps/NodeStakeUpdatesTest.java

@@ -44,7 +44,6 @@
 import com.hedera.hapi.node.transaction.ExchangeRateSet;
 import com.hedera.node.app.fees.ExchangeRateManager;
 import com.hedera.node.app.records.ReadableBlockRecordStore;
-import com.hedera.node.app.roster.schemas.V0540RosterSchema;
 import com.hedera.node.app.service.addressbook.AddressBookService;
 import com.hedera.node.app.service.addressbook.ReadableNodeStore;
 import com.hedera.node.app.service.addressbook.impl.ReadableNodeStoreImpl;
@@ -62,6 +61,7 @@
 import com.hedera.pbj.runtime.io.buffer.Bytes;
 import com.swirlds.config.api.Configuration;
 import com.swirlds.platform.roster.RosterUtils;
+import com.swirlds.platform.state.service.schemas.V0540RosterSchema;
 import com.swirlds.state.spi.WritableKVState;
 import com.swirlds.state.spi.WritableSingletonState;
 import com.swirlds.state.spi.WritableStates;

hedera-node/hedera-app/src/test/java/com/hedera/node/app/workflows/handle/steps/PlatformStateUpdatesTest.java

@@ -18,10 +18,10 @@
 
 import static com.hedera.hapi.node.freeze.FreezeType.FREEZE_UPGRADE;
 import static com.hedera.node.app.fixtures.AppTestBase.DEFAULT_CONFIG;
-import static com.hedera.node.app.roster.schemas.V0540RosterSchema.ROSTER_KEY;
-import static com.hedera.node.app.roster.schemas.V0540RosterSchema.ROSTER_STATES_KEY;
 import static com.hedera.node.app.service.addressbook.impl.schemas.V053AddressBookSchema.NODES_KEY;
 import static com.hedera.node.app.service.networkadmin.impl.schemas.V0490FreezeSchema.FREEZE_TIME_KEY;
+import static com.swirlds.platform.state.service.schemas.V0540RosterSchema.ROSTER_KEY;
+import static com.swirlds.platform.state.service.schemas.V0540RosterSchema.ROSTER_STATES_KEY;
 import static org.assertj.core.api.Assertions.assertThatThrownBy;
 import static org.junit.jupiter.api.Assertions.assertEquals;
 import static org.mockito.Mock.Strictness.LENIENT;

hedera-node/test-clients/src/main/java/com/hedera/services/bdd/spec/HapiSpec.java

@@ -16,7 +16,6 @@
 
 package com.hedera.services.bdd.spec;
 
-import static com.hedera.node.app.roster.schemas.V0540RosterSchema.ROSTER_STATES_KEY;
 import static com.hedera.node.app.service.addressbook.impl.schemas.V053AddressBookSchema.NODES_KEY;
 import static com.hedera.node.app.service.schedule.impl.schemas.V0570ScheduleSchema.SCHEDULE_IDS_BY_EXPIRY_SEC_KEY;
 import static com.hedera.node.app.service.token.impl.schemas.V0490TokenSchema.ACCOUNTS_KEY;
@@ -60,6 +59,7 @@
 import static com.hederahashgraph.api.proto.java.ResponseCodeEnum.NO_NEW_VALID_SIGNATURES;
 import static com.hederahashgraph.api.proto.java.ResponseCodeEnum.OK;
 import static com.hederahashgraph.api.proto.java.ResponseCodeEnum.SUCCESS;
+import static com.swirlds.platform.state.service.schemas.V0540RosterSchema.ROSTER_STATES_KEY;
 import static java.util.Collections.emptyList;
 import static java.util.Objects.requireNonNull;
 import static java.util.concurrent.CompletableFuture.allOf;

hedera-node/test-clients/src/main/java/com/hedera/services/bdd/suites/integration/ConcurrentIntegrationTests.java

@@ -20,8 +20,6 @@
 import static com.hedera.hapi.node.base.ResponseCodeEnum.BUSY;
 import static com.hedera.hapi.node.base.ResponseCodeEnum.FAIL_INVALID;
 import static com.hedera.node.app.blocks.schemas.V0560BlockStreamSchema.BLOCK_STREAM_INFO_KEY;
-import static com.hedera.node.app.roster.schemas.V0540RosterSchema.ROSTER_KEY;
-import static com.hedera.node.app.roster.schemas.V0540RosterSchema.ROSTER_STATES_KEY;
 import static com.hedera.services.bdd.junit.EmbeddedReason.MANIPULATES_EVENT_VERSION;
 import static com.hedera.services.bdd.junit.SharedNetworkLauncherSessionListener.CLASSIC_HAPI_TEST_NETWORK_SIZE;
 import static com.hedera.services.bdd.junit.TestTags.INTEGRATION;
@@ -69,6 +67,8 @@
 import static com.hedera.services.bdd.suites.hip869.NodeCreateTest.generateX509Certificates;
 import static com.hederahashgraph.api.proto.java.ResponseCodeEnum.RECORD_NOT_FOUND;
 import static com.hederahashgraph.api.proto.java.ResponseCodeEnum.TOKEN_NOT_ASSOCIATED_TO_ACCOUNT;
+import static com.swirlds.platform.state.service.schemas.V0540RosterSchema.ROSTER_KEY;
+import static com.swirlds.platform.state.service.schemas.V0540RosterSchema.ROSTER_STATES_KEY;
 import static org.junit.jupiter.api.Assertions.assertEquals;
 
 import com.hedera.hapi.block.stream.BlockItem;

platform-sdk/swirlds-common/src/main/java/com/swirlds/common/RosterStateId.java

@@ -0,0 +1,31 @@
+/*
+ * Copyright (C) 2024 Hedera Hashgraph, LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.swirlds.common;
+
+/**
+ * A class with constants identifying Roster entities in state.
+ */
+public final class RosterStateId {
+    private RosterStateId() {}
+
+    /** The name of a service that owns Roster entities in state. */
+    public static final String NAME = "RosterService";
+    /** The name of the RosterMap. */
+    public static final String ROSTER_KEY = "ROSTERS";
+    /** The name of the RosterState. */
+    public static final String ROSTER_STATES_KEY = "ROSTER_STATE";
+}

platform-sdk/swirlds-platform-core/src/main/java/com/swirlds/platform/reconnect/ReconnectLearner.java

@@ -36,6 +36,7 @@
 import com.swirlds.platform.state.signed.SignedStateInvalidException;
 import com.swirlds.platform.state.signed.SignedStateValidationData;
 import com.swirlds.platform.state.signed.SignedStateValidator;
+import com.swirlds.platform.state.snapshot.SignedStateFileReader;
 import com.swirlds.platform.system.address.AddressBook;
 import edu.umd.cs.findbugs.annotations.NonNull;
 import java.io.IOException;
@@ -211,6 +212,7 @@ private ReservedSignedState reconnect() throws InterruptedException {
                 false,
                 false,
                 false);
+        SignedStateFileReader.registerServiceStates(newSignedState);
         newSignedState.setSigSet(sigSet);
 
         final double mbReceived = connection.getDis().getSyncByteCounter().getMebiBytes();

hedera-node/hedera-app/src/main/java/com/hedera/node/app/roster/schemas/V0540RosterSchema.java → platform-sdk/swirlds-platform-core/src/main/java/com/swirlds/platform/state/service/schemas/V0540RosterSchema.java

@@ -14,7 +14,7 @@
  * limitations under the License.
  */
 
-package com.hedera.node.app.roster.schemas;
+package com.swirlds.platform.state.service.schemas;
 
 import com.hedera.hapi.node.base.SemanticVersion;
 import com.hedera.hapi.node.state.primitives.ProtoBytes;

platform-sdk/swirlds-platform-core/src/main/java/com/swirlds/platform/state/signed/SignedState.java

@@ -34,7 +34,10 @@
 import com.swirlds.config.api.Configuration;
 import com.swirlds.platform.config.StateConfig;
 import com.swirlds.platform.crypto.SignatureVerifier;
+import com.swirlds.platform.roster.RosterRetriever;
+import com.swirlds.platform.roster.RosterUtils;
 import com.swirlds.platform.state.MerkleRoot;
+import com.swirlds.platform.state.MerkleStateRoot;
 import com.swirlds.platform.state.signed.SignedStateHistory.SignedStateAction;
 import com.swirlds.platform.state.snapshot.StateToDiskReason;
 import com.swirlds.platform.system.SwirldState;
@@ -258,7 +261,8 @@ public void setSigSet(@NonNull final SigSet sigSet) {
     @Override
     public @NonNull AddressBook getAddressBook() {
         return Objects.requireNonNull(
-                getState().getReadablePlatformState().getAddressBook(),
+                RosterUtils.buildAddressBook(RosterRetriever.retrieveActiveOrGenesisRoster(
+                        (MerkleStateRoot) getState().getSwirldState())),
                 "address book stored in this signed state is null, this should never happen");
     }
 

platform-sdk/swirlds-platform-core/src/main/java/com/swirlds/platform/state/snapshot/SignedStateFileReader.java

@@ -21,23 +21,33 @@
 import static com.swirlds.platform.state.snapshot.SignedStateFileUtils.SUPPORTED_SIGSET_VERSIONS;
 import static java.nio.file.Files.exists;
 
+import com.swirlds.common.RosterStateId;
 import com.swirlds.common.config.StateCommonConfig;
 import com.swirlds.common.config.singleton.ConfigurationHolder;
 import com.swirlds.common.context.PlatformContext;
 import com.swirlds.common.io.streams.MerkleDataInputStream;
 import com.swirlds.common.platform.NodeId;
 import com.swirlds.config.api.Configuration;
 import com.swirlds.platform.crypto.CryptoStatic;
+import com.swirlds.platform.state.MerkleStateRoot;
 import com.swirlds.platform.state.MerkleTreeSnapshotReader;
+import com.swirlds.platform.state.service.PlatformStateService;
+import com.swirlds.platform.state.service.schemas.V0540PlatformStateSchema;
+import com.swirlds.platform.state.service.schemas.V0540RosterSchema;
 import com.swirlds.platform.state.signed.SigSet;
 import com.swirlds.platform.state.signed.SignedState;
+import com.swirlds.state.State;
+import com.swirlds.state.lifecycle.Schema;
+import com.swirlds.state.lifecycle.StateDefinition;
+import com.swirlds.state.merkle.StateMetadata;
 import edu.umd.cs.findbugs.annotations.NonNull;
 import java.io.BufferedInputStream;
 import java.io.File;
 import java.io.FileInputStream;
 import java.io.IOException;
 import java.nio.file.Files;
 import java.nio.file.Path;
+import java.util.Comparator;
 import java.util.List;
 import java.util.Objects;
 
@@ -109,6 +119,8 @@
                 false,
                 false);
 
+        registerServiceStates(newSignedState);
+
         newSignedState.setSigSet(normalizedData.sigSet());
 
         returnState = new DeserializedSignedState(
@@ -145,4 +157,53 @@
         }
         in.readProtocolVersion();
     }
+
+    /**
+     * Register stub states for PlatformStateService and RosterService so that the State knows about them per the metadata and services registry.
+     * <p>
+     * Note that the state data objects associated with these services MUST ALREADY EXIST in the merkle tree (or on disk.)
+     * These stubs WILL NOT create missing nodes in the state, or run any state migration code. The stubs assume that the
+     * data structures present in the snapshot match the version of the software where this code runs.
+     * <p>
+     * These stubs are necessary to enable a state (a SignedState, in particular) to read the roster (or fall back
+     * to reading the legacy AddressBook) from the state using the States API which would normally require
+     * the complete initialization of services and all the schemas. However, only the PlatformState and RosterState/RosterMap
+     * are really required to support reading the Roster (or AddressBook.) So we only initialize the schemas for these two.
+     * <p>
+     * If/when this SignedState object needs to become a real state to support the node operation, the services/app
+     * code will be responsible for initializing all the supported services. Note that the {@code merkleStateRoot.putServiceStateIfAbsent}
+     * operation that we use to actually perform the registration is assumed to be idempotent. When the app re-initializes
+     * the PlatformStateService and the RosterService, the information about the registration will simply be overwritten
+     * and/or amended as necessary with the complete registration logic, and the "double-registration" per se
+     * shouldn't cause any issues.
+     *
+     * @param signedState a signed state to register schemas in
+     */
+    public static void registerServiceStates(@NonNull final SignedState signedState) {
+        registerServiceState(
+                (MerkleStateRoot) signedState.getState(), new V0540PlatformStateSchema(), PlatformStateService.NAME);
+        registerServiceState((MerkleStateRoot) signedState.getState(), new V0540RosterSchema(), RosterStateId.NAME);
+    }
+
+    private static void registerServiceState(
+            @NonNull final State state, @NonNull final Schema schema, @NonNull final String name) {
+        if (!(state instanceof MerkleStateRoot merkleStateRoot)) {
+            throw new IllegalArgumentException("Can only be used with MerkleStateRoot instances");
+        }
+        schema.statesToCreate().stream()
+                .sorted(Comparator.comparing(StateDefinition::stateKey))
+                .forEach(def -> {
+                    final var md = new StateMetadata<>(name, schema, def);
+                    if (def.singleton() || def.onDisk()) {
+                        merkleStateRoot.putServiceStateIfAbsent(md, () -> {
+                            throw new IllegalStateException(
+                                    "State nodes " + md.stateDefinition().stateKey() + " for service " + name
+                                            + " are supposed to exist in the state snapshot already.");
+                        });
+                    } else {
+                        throw new IllegalStateException(
+                                "Only singletons and onDisk virtual maps are supported as stub states");
+                    }
+                });
+    }
 }

platform-sdk/swirlds-platform-core/src/test/java/com/swirlds/platform/SignedStateFileReadWriteTest.java

@@ -46,6 +46,7 @@
 import com.swirlds.common.test.fixtures.platform.TestPlatformContextBuilder;
 import com.swirlds.config.api.Configuration;
 import com.swirlds.config.extensions.test.fixtures.TestConfigBuilder;
+import com.swirlds.merkledb.MerkleDb;
 import com.swirlds.platform.config.StateConfig;
 import com.swirlds.platform.state.MerkleRoot;
 import com.swirlds.platform.state.signed.SignedState;
@@ -64,22 +65,17 @@
 import java.nio.file.Files;
 import java.nio.file.Path;
 import java.nio.file.StandardOpenOption;
+import org.junit.jupiter.api.AfterEach;
 import org.junit.jupiter.api.BeforeAll;
 import org.junit.jupiter.api.BeforeEach;
 import org.junit.jupiter.api.DisplayName;
 import org.junit.jupiter.api.Test;
-import org.junit.jupiter.api.io.TempDir;
 
 @DisplayName("SignedState Read/Write Test")
 class SignedStateFileReadWriteTest {
-    /**
-     * Temporary directory provided by JUnit
-     */
-    @TempDir
     Path testDirectory;
 
     private static SemanticVersion platformVersion;
-    private SignedState signedState;
 
     @BeforeAll
     static void beforeAll() throws ConstructableRegistryException {
@@ -89,22 +85,33 @@ static void beforeAll() throws ConstructableRegistryException {
         registry.registerConstructables("com.swirlds.common");
         registry.registerConstructables("com.swirlds.platform");
         registry.registerConstructables("com.swirlds.state");
+        registry.registerConstructables("com.swirlds.virtualmap");
+        registry.registerConstructables("com.swirlds.merkledb");
         FakeMerkleStateLifecycles.registerMerkleStateRootClassIds();
     }
 
     @BeforeEach
     void beforeEach() throws IOException {
+        // Don't use JUnit @TempDir as it runs into a thread race with Merkle DB DataSource release...
+        testDirectory = LegacyTemporaryFileBuilder.buildTemporaryFile(
+                "SignedStateFileReadWriteTest", FakeMerkleStateLifecycles.CONFIGURATION);
         LegacyTemporaryFileBuilder.overrideTemporaryFileLocation(testDirectory.resolve("tmp"));
-        signedState = new RandomSignedStateGenerator()
-                .setSoftwareVersion(new BasicSoftwareVersion(platformVersion.minor()))
-                .build();
+        MerkleDb.resetDefaultInstancePath();
+    }
+
+    @AfterEach
+    void tearDown() {
+        RandomSignedStateGenerator.releaseAllBuiltSignedStates();
     }
 
     @Test
     @DisplayName("writeHashInfoFile() Test")
     void writeHashInfoFileTest() throws IOException {
         final PlatformContext platformContext =
                 TestPlatformContextBuilder.create().build();
+        final SignedState signedState = new RandomSignedStateGenerator()
+                .setSoftwareVersion(new BasicSoftwareVersion(platformVersion.minor()))
+                .build();
         MerkleRoot state = signedState.getState();
         writeHashInfoFile(platformContext, testDirectory, state);
         final StateConfig stateConfig =
@@ -209,6 +216,9 @@ void writeThenReadStateFileTest_v1() throws IOException {
     @Test
     @DisplayName("writeSavedStateToDisk() Test")
     void writeSavedStateToDiskTest() throws IOException {
+        final SignedState signedState = new RandomSignedStateGenerator()
+                .setSoftwareVersion(new BasicSoftwareVersion(platformVersion.minor()))
+                .build();
         final Path directory = testDirectory.resolve("state");
 
         final Path stateFile = directory.resolve(SIGNED_STATE_FILE_NAME);