From bac40bdab9f904faebc4993d0437a4a6499a1b68 Mon Sep 17 00:00:00 2001 From: Edward Wertz <123979964+edward-swirldslabs@users.noreply.github.com> Date: Wed, 6 Dec 2023 10:32:08 -0600 Subject: [PATCH 1/7] feat(event serialization): add `birthRound`, `EventDescriptor` and multiple other parents to serialized event (#9344) Signed-off-by: Edward Wertz Signed-off-by: Edward Wertz <123979964+edward-swirldslabs@users.noreply.github.com> Signed-off-by: Austin Littley Co-authored-by: Cody Littley <56973212+cody-littley@users.noreply.github.com> Co-authored-by: Austin Littley <102969658+alittley@users.noreply.github.com> --- .../system/events/BaseEventHashedData.java | 318 ++++++++++++------ .../system/events/BaseEventUnhashedData.java | 114 +++++-- .../system/events/DetailedConsensusEvent.java | 15 +- .../common/system/events}/EventConstants.java | 6 +- .../common/system/events/EventDescriptor.java | 107 ++++-- .../consensus/RoundCalculationUtils.java | 2 +- .../platform/consensus/RoundElections.java | 2 +- .../platform/event/EventStringBuilder.java | 1 + .../swirlds/platform/event/EventUtils.java | 1 + .../swirlds/platform/event/GossipEvent.java | 56 ++- .../creation/tipset/TipsetEventCreator.java | 20 +- .../event/creation/tipset/TipsetUtils.java | 20 +- .../platform/event/orphan/ParentIterator.java | 16 +- .../hashgraph/internal/CachingGuiSource.java | 2 +- .../internal/HashgraphGuiControls.java | 2 +- .../state/signed/SignedStateFileManager.java | 2 +- .../com/swirlds/platform/EventImplTests.java | 16 +- .../java/com/swirlds/platform/TestUtils.java | 15 +- .../consensus/RoundCalculationUtilsTest.java | 2 +- .../platform/event/DetGenerateUtils.java | 25 +- .../event/EventDeduplicatorTests.java | 2 +- .../event/linking/InOrderLinkerTests.java | 2 +- .../event/orphan/OrphanBufferTests.java | 43 ++- .../platform/recovery/RecoveryTestUtils.java | 25 +- .../state/TransactionHandlerTest.java | 15 +- .../test/fixtures/event/RandomEventUtils.java | 48 ++- .../generator/AbstractGraphGenerator.java | 2 +- .../chatter/simulator/SimulatedEvent.java | 3 +- .../framework/ConsensusTestOrchestrator.java | 2 +- .../platform/test/utils/EqualsVerifier.java | 18 +- .../platform/test/EventStringsTest.java | 19 +- .../messages/EventDescriptorTest.java | 14 +- .../test/components/EventUtilsTests.java | 2 +- .../TransactionHandlingTestUtils.java | 15 +- .../consensus/IntakeAndConsensusTests.java | 2 +- .../test/event/GossipEventBuilder.java | 52 ++- .../platform/test/event/GossipEventTest.java | 32 +- .../tipset/ChildlessEventTrackerTests.java | 33 +- .../event/tipset/TipsetEventCreatorTests.java | 34 +- .../test/event/tipset/TipsetTrackerTests.java | 4 +- .../tipset/TipsetWeightCalculatorTests.java | 97 +++--- .../event/validation/EventValidatorTests.java | 27 +- .../platform/test/sync/EventFactory.java | 27 +- .../swirlds/platform/test/sync/SyncTests.java | 2 +- .../sampleGossipEvent.evts | Bin 0 -> 947 bytes 45 files changed, 879 insertions(+), 383 deletions(-) rename platform-sdk/{swirlds-platform-core/src/main/java/com/swirlds/platform/event => swirlds-common/src/main/java/com/swirlds/common/system/events}/EventConstants.java (85%) create mode 100644 platform-sdk/swirlds-unit-tests/core/swirlds-platform-test/src/test/resources/eventFiles/eventSerializationV45/sampleGossipEvent.evts diff --git a/platform-sdk/swirlds-common/src/main/java/com/swirlds/common/system/events/BaseEventHashedData.java b/platform-sdk/swirlds-common/src/main/java/com/swirlds/common/system/events/BaseEventHashedData.java index 0dfbb12810ec..857d4c531d21 100644 --- a/platform-sdk/swirlds-common/src/main/java/com/swirlds/common/system/events/BaseEventHashedData.java +++ b/platform-sdk/swirlds-common/src/main/java/com/swirlds/common/system/events/BaseEventHashedData.java @@ -28,6 +28,7 @@ import com.swirlds.common.io.streams.SerializableDataOutputStream; import com.swirlds.common.system.NodeId; import com.swirlds.common.system.SoftwareVersion; +import com.swirlds.common.system.address.AddressBook; import com.swirlds.common.system.transaction.internal.ConsensusTransactionImpl; import com.swirlds.common.utility.CommonUtils; import edu.umd.cs.findbugs.annotations.NonNull; @@ -35,6 +36,8 @@ import java.io.IOException; import java.time.Instant; import java.util.Arrays; +import java.util.Collections; +import java.util.List; import java.util.Objects; /** @@ -50,7 +53,7 @@ public class BaseEventHashedData extends AbstractSerializableHashable public static final int TO_STRING_BYTE_ARRAY_LENGTH = 5; private static final long CLASS_ID = 0x21c2620e9b6a2243L; - private static class ClassVersion { + public static class ClassVersion { /** * In this version, the transactions contained by this event are encoded using * LegacyTransaction class. No longer supported. @@ -66,7 +69,24 @@ private static class ClassVersion { * In this version, the software version of the node that created this event is included in the event. */ public static final int SOFTWARE_VERSION = 3; + + /** + * Event descriptors replace the hashes and generation of the parents in the event. + * Multiple otherParents are supported. + * birthRound is added for lookup of the effective roster at the time of event creation. + * + * @since 0.46.0 + */ + public static final int BIRTH_ROUND = 4; } + + /** + * The version of the serialization to use. May be overridden by the version encountered when deserializing. + * + * DEPRECATED: remove after 0.46.0 goes to mainnet. + */ + private int serializedVersion = ClassVersion.BIRTH_ROUND; + /////////////////////////////////////// // immutable, sent during normal syncs, affects the hash that is signed: /////////////////////////////////////// @@ -75,14 +95,12 @@ private static class ClassVersion { private SoftwareVersion softwareVersion; /** ID of this event's creator (translate before sending) */ private NodeId creatorId; - /** the generation for the self parent */ - private long selfParentGen; - /** the generation for the other parent */ - private long otherParentGen; - /** self parent hash value */ - private Hash selfParentHash; - /** other parent hash value */ - private Hash otherParentHash; + /** the round number in which this event was created, used to look up the effective roster at that time. */ + private long birthRound; + /** the self parent event descriptor */ + private EventDescriptor selfParent; + /** the other parents' event descriptors */ + private List otherParents; /** creation time, as claimed by its creator */ private Instant timeCreated; /** the payload: an array of transactions */ @@ -97,14 +115,12 @@ public BaseEventHashedData() {} * the software version of the node that created this event. * @param creatorId * ID of this event's creator - * @param selfParentGen - * the generation for the self parent - * @param otherParentGen - * the generation for the other parent - * @param selfParentHash - * self parent hash value - * @param otherParentHash - * other parent hash value + * @param selfParent + * self parent event descriptor + * @param otherParents + * other parent event descriptors + * @param birthRound + * the round in which this event was created. * @param timeCreated * creation time, as claimed by its creator * @param transactions @@ -113,62 +129,22 @@ public BaseEventHashedData() {} public BaseEventHashedData( @NonNull SoftwareVersion softwareVersion, @NonNull final NodeId creatorId, - final long selfParentGen, - final long otherParentGen, - @Nullable final Hash selfParentHash, - @Nullable final Hash otherParentHash, + @Nullable final EventDescriptor selfParent, + @NonNull final List otherParents, + final long birthRound, @NonNull final Instant timeCreated, @Nullable final ConsensusTransactionImpl[] transactions) { this.softwareVersion = Objects.requireNonNull(softwareVersion, "The softwareVersion must not be null"); this.creatorId = Objects.requireNonNull(creatorId, "The creatorId must not be null"); - this.selfParentGen = selfParentGen; - this.otherParentGen = otherParentGen; - this.selfParentHash = selfParentHash; - this.otherParentHash = otherParentHash; + this.selfParent = selfParent; + Objects.requireNonNull(otherParents, "The otherParents must not be null"); + otherParents.forEach(Objects::requireNonNull); + this.otherParents = otherParents; + this.birthRound = birthRound; this.timeCreated = Objects.requireNonNull(timeCreated, "The timeCreated must not be null"); this.transactions = transactions; } - /** - * Create a BaseEventHashedData object - * - * @param softwareVersion - * the software version of the node that created this event. - * @param creatorId - * ID of this event's creator - * @param selfParentGen - * the generation for the self parent - * @param otherParentGen - * the generation for the other parent - * @param selfParentHash - * self parent hash value in byte array format - * @param otherParentHash - * other parent hash value in byte array format - * @param timeCreated - * creation time, as claimed by its creator - * @param transactions - * the payload: an array of transactions included in this event instance - */ - public BaseEventHashedData( - @NonNull final SoftwareVersion softwareVersion, - @NonNull final NodeId creatorId, - final long selfParentGen, - final long otherParentGen, - @Nullable final byte[] selfParentHash, - @Nullable final byte[] otherParentHash, - @NonNull final Instant timeCreated, - @Nullable final ConsensusTransactionImpl[] transactions) { - this( - softwareVersion, - creatorId, - selfParentGen, - otherParentGen, - selfParentHash == null ? null : new Hash(selfParentHash), - otherParentHash == null ? null : new Hash(otherParentHash), - timeCreated, - transactions); - } - @Override public int getMinimumSupportedVersion() { return ClassVersion.TRANSACTION_SUBCLASSES; @@ -179,13 +155,21 @@ public void serialize( @NonNull final SerializableDataOutputStream out, @NonNull final EventSerializationOptions option) throws IOException { out.writeSerializable(softwareVersion, true); - // FUTURE WORK: The creatorId should be a selfSerializable NodeId at some point. - // Changing the event format may require a HIP. The old format is preserved for now. - out.writeLong(creatorId.id()); - out.writeLong(selfParentGen); - out.writeLong(otherParentGen); - out.writeSerializable(selfParentHash, false); - out.writeSerializable(otherParentHash, false); + if (serializedVersion < ClassVersion.BIRTH_ROUND) { + out.writeLong(creatorId.id()); + out.writeLong(selfParent != null ? selfParent.getGeneration() : EventConstants.GENERATION_UNDEFINED); + out.writeLong( + !otherParents.isEmpty() + ? otherParents.get(0).getGeneration() + : EventConstants.GENERATION_UNDEFINED); + out.writeSerializable(selfParent != null ? selfParent.getHash() : null, false); + out.writeSerializable(!otherParents.isEmpty() ? otherParents.get(0).getHash() : null, false); + } else { + out.writeSerializable(creatorId, false); + out.writeSerializable(selfParent, false); + out.writeSerializableList(otherParents, false, true); + out.writeLong(birthRound); + } out.writeInstant(timeCreated); // write serialized length of transaction array first, so during the deserialization proces @@ -216,18 +200,39 @@ public void deserialize( @NonNull final SerializableDataInputStream in, final int version, final int maxTransactionCount) throws IOException { Objects.requireNonNull(in, "The input stream must not be null"); + serializedVersion = version; if (version >= ClassVersion.SOFTWARE_VERSION) { softwareVersion = in.readSerializable(); } else { softwareVersion = SoftwareVersion.NO_VERSION; } - // FUTURE WORK: The creatorId should be a selfSerializable NodeId at some point. - // Changing the event format may require a HIP. The old format is preserved for now. - creatorId = NodeId.deserializeLong(in, false); - selfParentGen = in.readLong(); - otherParentGen = in.readLong(); - selfParentHash = in.readSerializable(false, Hash::new); - otherParentHash = in.readSerializable(false, Hash::new); + if (version < ClassVersion.BIRTH_ROUND) { + // FUTURE WORK: The creatorId should be a selfSerializable NodeId at some point. + // Changing the event format may require a HIP. The old format is preserved for now. + creatorId = NodeId.deserializeLong(in, false); + final long selfParentGen = in.readLong(); + final long otherParentGen = in.readLong(); + final Hash selfParentHash = in.readSerializable(false, Hash::new); + final Hash otherParentHash = in.readSerializable(false, Hash::new); + selfParent = selfParentHash == null + ? null + : new EventDescriptor( + selfParentHash, creatorId, selfParentGen, EventConstants.BIRTH_ROUND_UNDEFINED); + // The creator for the other parent descriptor is not here and should be retrieved from the unhashed data. + otherParents = otherParentHash == null + ? Collections.emptyList() + : Collections.singletonList( + new EventDescriptor(otherParentHash, otherParentGen, EventConstants.BIRTH_ROUND_UNDEFINED)); + birthRound = EventConstants.BIRTH_ROUND_UNDEFINED; + } else { + creatorId = in.readSerializable(false, NodeId::new); + if (creatorId == null) { + throw new IOException("creatorId is null"); + } + selfParent = in.readSerializable(false, EventDescriptor::new); + otherParents = in.readSerializableList(AddressBook.MAX_ADDRESSES, false, EventDescriptor::new); + birthRound = in.readLong(); + } timeCreated = in.readInstant(); in.readInt(); // read serialized length transactions = in.readSerializableArray(ConsensusTransactionImpl[]::new, maxTransactionCount, true); @@ -246,10 +251,9 @@ public boolean equals(final Object o) { final BaseEventHashedData that = (BaseEventHashedData) o; return (Objects.equals(creatorId, that.creatorId)) - && (selfParentGen == that.selfParentGen) - && (otherParentGen == that.otherParentGen) - && Objects.equals(selfParentHash, that.selfParentHash) - && Objects.equals(otherParentHash, that.otherParentHash) + && Objects.equals(selfParent, that.selfParent) + && Objects.equals(otherParents, that.otherParents) + && birthRound == that.birthRound && Objects.equals(timeCreated, that.timeCreated) && Arrays.equals(transactions, that.transactions) && (softwareVersion.compareTo(that.softwareVersion) == 0); @@ -257,14 +261,7 @@ public boolean equals(final Object o) { @Override public int hashCode() { - int result = Objects.hash( - softwareVersion, - creatorId, - selfParentGen, - otherParentGen, - selfParentHash, - otherParentHash, - timeCreated); + int result = Objects.hash(softwareVersion, creatorId, selfParent, otherParents, birthRound, timeCreated); result = 31 * result + Arrays.hashCode(transactions); return result; } @@ -274,16 +271,16 @@ public String toString() { return new ToStringBuilder(this) .append("softwareVersion", softwareVersion) .append("creatorId", creatorId) - .append("selfParentGen", selfParentGen) - .append("otherParentGen", otherParentGen) - .append("selfParentHash", CommonUtils.hex(valueOrNull(selfParentHash), TO_STRING_BYTE_ARRAY_LENGTH)) - .append("otherParentHash", CommonUtils.hex(valueOrNull(otherParentHash), TO_STRING_BYTE_ARRAY_LENGTH)) + .append("selfParent", selfParent) + .append("otherParents", otherParents) + .append("birthRound", birthRound) .append("timeCreated", timeCreated) .append("transactions size", transactions == null ? "null" : transactions.length) .append("hash", CommonUtils.hex(valueOrNull(getHash()), TO_STRING_BYTE_ARRAY_LENGTH)) .toString(); } + @Nullable private byte[] valueOrNull(final Hash hash) { return hash == null ? null : hash.getValue(); } @@ -295,7 +292,7 @@ public long getClassId() { @Override public int getVersion() { - return ClassVersion.SOFTWARE_VERSION; + return serializedVersion; } /** @@ -318,30 +315,134 @@ public NodeId getCreatorId() { return creatorId; } + /** + * Get the birth round of the event. + * + * @return the birth round of the event + */ + public long getBirthRound() { + return birthRound; + } + + /** + * Get the event descriptor for the self parent. + * + * @return the event descriptor for the self parent + */ + @Nullable + public EventDescriptor getSelfParent() { + return selfParent; + } + + /** + * Get the event descriptors for the other parents. + * + * @return the event descriptors for the other parents + */ + @NonNull + public List getOtherParents() { + return otherParents; + } + + /** + * Get the self parent generation + * + * @return the self parent generation + */ public long getSelfParentGen() { - return selfParentGen; + if (selfParent == null) { + return EventConstants.GENERATION_UNDEFINED; + } + return selfParent.getGeneration(); } + /** + * Get the maximum generation of the other parents. + * + * @return the maximum generation of the other parents + * @deprecated this method should be replaced since there can be multiple other parents. + */ public long getOtherParentGen() { - return otherParentGen; + if (otherParents == null || otherParents.isEmpty()) { + return EventConstants.GENERATION_UNDEFINED; + } + if (otherParents.size() == 1) { + return otherParents.get(0).getGeneration(); + } + // 0.46.0 adds support for multiple other parents in the serialization scheme, but not yet in the + // implementation. This exception should never be reached unless we have multiple parents and need to + // update the implementation. + throw new UnsupportedOperationException("Multiple other parents is not supported yet"); } + /** + * Get the hash of the self parent. + * + * @return the hash of the self parent + */ + @Nullable public Hash getSelfParentHash() { - return selfParentHash; + if (selfParent == null) { + return null; + } + return selfParent.getHash(); } + /** + * Get the hash of the other parent with the maximum generation. + * + * @return the hash of the other parent with the maximum generation + * @deprecated + */ + @Nullable public Hash getOtherParentHash() { - return otherParentHash; + if (otherParents == null || otherParents.isEmpty()) { + return null; + } + if (otherParents.size() == 1) { + return otherParents.get(0).getHash(); + } + // 0.46.0 adds support for multiple other parents in the serialization scheme, but not yet in the + // implementation. This exception should never be reached unless we have multiple parents and need to + // update the implementation. + throw new UnsupportedOperationException("Multiple other parents is not supported yet"); } + /** + * Check if the event has a self parent. + * @return true if the event has a self parent + */ public boolean hasSelfParent() { - return selfParentHash != null; + return selfParent != null; } + /** + * Check if the event has other parents. + * @return true if the event has other parents + */ public boolean hasOtherParent() { - return otherParentHash != null; + return otherParents != null && !otherParents.isEmpty(); + } + + /** + * Get the hash value of the parent event. + * @return the hash value of the parent event + */ + @Nullable + public byte[] getSelfParentHashValue() { + return selfParent == null ? null : getSelfParentHash().getValue(); + } + + /** + * Get the hash value of the other parent with the maximum generation. + * @return the hash value of the other parent with the maximum generation + */ + @Nullable + public byte[] getOtherParentHashValue() { + return otherParents.isEmpty() ? null : getOtherParentHash().getValue(); } + @NonNull public Instant getTimeCreated() { return timeCreated; } @@ -349,12 +450,13 @@ public Instant getTimeCreated() { /** * @return array of transactions inside this event instance */ + @Nullable public ConsensusTransactionImpl[] getTransactions() { return transactions; } public long getGeneration() { - return calculateGeneration(selfParentGen, otherParentGen); + return calculateGeneration(getSelfParentGen(), getOtherParentGen()); } /** @@ -369,4 +471,14 @@ public long getGeneration() { public static long calculateGeneration(final long selfParentGeneration, final long otherParentGeneration) { return 1 + Math.max(selfParentGeneration, otherParentGeneration); } + + /** + * Create an event descriptor for this event. + * + * @return an event descriptor for this event + */ + @NonNull + public EventDescriptor createEventDescriptor() { + return new EventDescriptor(getHash(), getCreatorId(), getGeneration(), getBirthRound()); + } } diff --git a/platform-sdk/swirlds-common/src/main/java/com/swirlds/common/system/events/BaseEventUnhashedData.java b/platform-sdk/swirlds-common/src/main/java/com/swirlds/common/system/events/BaseEventUnhashedData.java index 4fcd4a361c52..424108edcf12 100644 --- a/platform-sdk/swirlds-common/src/main/java/com/swirlds/common/system/events/BaseEventUnhashedData.java +++ b/platform-sdk/swirlds-common/src/main/java/com/swirlds/common/system/events/BaseEventUnhashedData.java @@ -16,6 +16,7 @@ package com.swirlds.common.system.events; +import com.swirlds.base.utility.ToStringBuilder; import com.swirlds.common.io.SelfSerializable; import com.swirlds.common.io.streams.SerializableDataInputStream; import com.swirlds.common.io.streams.SerializableDataOutputStream; @@ -31,16 +32,36 @@ * A class used to store base event data that does not affect the hash of that event. *

* A base event is a set of data describing an event at the point when it is created, before it is added to the - * hashgraph and before its consensus can be determined. Some of this data is used to create a hash of an event - * that is signed, and some data is additional and does not affect that hash. This data is split into 2 classes: + * hashgraph and before its consensus can be determined. Some of this data is used to create a hash of an event that is + * signed, and some data is additional and does not affect that hash. This data is split into 2 classes: * {@link BaseEventHashedData} and {@link BaseEventUnhashedData}. */ public class BaseEventUnhashedData implements SelfSerializable { private static final long CLASS_ID = 0x33cb9d4ae38c9e91L; - private static final int CLASS_VERSION = 1; - private static final int MAX_SIG_LENGTH = 384; + public static final int MAX_SIG_LENGTH = 384; private static final long SEQUENCE_UNUSED = -1; + public static class ClassVersion { + /** + * The original version of the BaseEventUnhashedData class. + */ + public static final int ORIGINAL = 1; + + /** + * Removes the serialization of the sequence information and other parent creator id. + * + * @since 0.46.0 + */ + public static final int BIRTH_ROUND = 2; + } + + /** + * The version of the software discovered during deserialization that needs to be preserved in serialization. + *

+ * DEPRECATED: Remove after 0.46.0 is delivered to mainnet. + */ + private int serializedVersion = ClassVersion.BIRTH_ROUND; + /////////////////////////////////////// // immutable, sent during normal syncs, does NOT affect the hash that is signed: /////////////////////////////////////// @@ -52,42 +73,43 @@ public class BaseEventUnhashedData implements SelfSerializable { // otherId is also probably not needed anymore, so at some point this class can be replaced with just a Signature // ---------------------------------------------------------------------------------------------------------------- - /** sequence number for this by its creator (0 is first) */ - private long creatorSeq; /** ID of otherParent (translate before sending) */ private NodeId otherId; - /** sequence number for otherParent event (by its creator) */ - private long otherSeq; /** creator's sig for this */ private byte[] signature; public BaseEventUnhashedData() {} public BaseEventUnhashedData(@Nullable final NodeId otherId, @NonNull final byte[] signature) { - this.creatorSeq = SEQUENCE_UNUSED; this.otherId = otherId; this.signature = Objects.requireNonNull(signature, "signature must not be null"); - this.otherSeq = SEQUENCE_UNUSED; } @Override public void serialize(@NonNull final SerializableDataOutputStream out) throws IOException { - out.writeLong(creatorSeq); - // FUTURE WORK: The otherId should be a selfSerializable NodeId at some point. - // Changing the event format may require a HIP. The old format is preserved for now. - out.writeLong(otherId == null ? -1 : otherId.id()); - out.writeLong(otherSeq); - out.writeByteArray(signature); + if (serializedVersion < ClassVersion.BIRTH_ROUND) { + out.writeLong(SEQUENCE_UNUSED); + out.writeLong(otherId == null ? -1 : otherId.id()); + out.writeLong(SEQUENCE_UNUSED); + out.writeByteArray(signature); + } else { + out.writeByteArray(signature); + } } @Override public void deserialize(@NonNull final SerializableDataInputStream in, final int version) throws IOException { - creatorSeq = in.readLong(); - // FUTURE WORK: The otherId should be a nullable selfSerializable NodeId at some point. - // Changing the event format may require a HIP. The old format is preserved for now. - otherId = NodeId.deserializeLong(in, true); - otherSeq = in.readLong(); - signature = in.readByteArray(MAX_SIG_LENGTH); + serializedVersion = version; + if (version < ClassVersion.BIRTH_ROUND) { + in.readLong(); // unused + otherId = NodeId.deserializeLong(in, true); + in.readLong(); // unused + signature = in.readByteArray(MAX_SIG_LENGTH); + } else { + signature = in.readByteArray(MAX_SIG_LENGTH); + // initialize unused fields (can be removed when the unused fields are removed) + otherId = null; + } } @Override @@ -102,27 +124,21 @@ public boolean equals(final Object o) { final BaseEventUnhashedData that = (BaseEventUnhashedData) o; - return (creatorSeq == that.creatorSeq) - && (Objects.equals(otherId, that.otherId)) - && (otherSeq == that.otherSeq) - && Arrays.equals(signature, that.signature); + return Objects.equals(otherId, that.otherId) && Arrays.equals(signature, that.signature); } @Override public int hashCode() { - int result = Objects.hash(creatorSeq, otherId, otherSeq); - result = 31 * result + Arrays.hashCode(signature); - return result; + return Arrays.hashCode(signature); } @Override public String toString() { final int signatureLength = signature == null ? 0 : signature.length; - return "BaseEventUnhashedData{" + "creatorSeq=" - + creatorSeq + ", otherId=" - + otherId + ", otherSeq=" - + otherSeq + ", signature=" - + CommonUtils.hex(signature, signatureLength) + '}'; + return new ToStringBuilder(this) + .append("otherId", otherId) + .append("signature", CommonUtils.hex(signature, signatureLength)) + .toString(); } @Override @@ -132,7 +148,12 @@ public long getClassId() { @Override public int getVersion() { - return CLASS_VERSION; + return serializedVersion; + } + + @Override + public int getMinimumSupportedVersion() { + return ClassVersion.ORIGINAL; } /** @@ -145,7 +166,30 @@ public NodeId getOtherId() { return otherId; } + /** + * Get the signature + * + * @return the signature + */ public byte[] getSignature() { return signature; } + + /** + * Synchronize the creator data between the hashed event's other parent data and the creatorId. This method is to + * support backwards compatibility with the previous event serialization format. + * + * @param event the event to update + * @deprecated This method should be deleted when we are no longer supporting the previous event serialization + * format. + */ + public void updateOtherParentEventDescriptor(@NonNull final BaseEventHashedData event) { + if (event.hasOtherParent()) { + if (otherId == null) { + otherId = event.getOtherParents().get(0).getCreator(); + } else { + event.getOtherParents().get(0).setCreator(otherId); + } + } + } } diff --git a/platform-sdk/swirlds-common/src/main/java/com/swirlds/common/system/events/DetailedConsensusEvent.java b/platform-sdk/swirlds-common/src/main/java/com/swirlds/common/system/events/DetailedConsensusEvent.java index fc3f11879f56..ccde5d1cb997 100644 --- a/platform-sdk/swirlds-common/src/main/java/com/swirlds/common/system/events/DetailedConsensusEvent.java +++ b/platform-sdk/swirlds-common/src/main/java/com/swirlds/common/system/events/DetailedConsensusEvent.java @@ -23,6 +23,7 @@ import com.swirlds.common.io.OptionalSelfSerializable; import com.swirlds.common.io.streams.SerializableDataInputStream; import com.swirlds.common.io.streams.SerializableDataOutputStream; +import com.swirlds.common.system.events.BaseEventHashedData.ClassVersion; import java.io.IOException; import java.util.Objects; @@ -86,7 +87,11 @@ public static void serialize( final EventSerializationOptions option) throws IOException { out.writeOptionalSerializable(baseEventHashedData, false, option); - out.writeSerializable(baseEventUnhashedData, false); + if (baseEventHashedData.getVersion() < ClassVersion.BIRTH_ROUND) { + out.writeSerializable(baseEventUnhashedData, false); + } else { + out.writeByteArray(baseEventUnhashedData.getSignature()); + } out.writeSerializable(consensusData, false); } @@ -104,7 +109,13 @@ public void serialize(final SerializableDataOutputStream out) throws IOException @Override public void deserialize(final SerializableDataInputStream in, final int version) throws IOException { baseEventHashedData = in.readSerializable(false, BaseEventHashedData::new); - baseEventUnhashedData = in.readSerializable(false, BaseEventUnhashedData::new); + if (baseEventHashedData.getVersion() < ClassVersion.BIRTH_ROUND) { + baseEventUnhashedData = in.readSerializable(false, BaseEventUnhashedData::new); + baseEventUnhashedData.updateOtherParentEventDescriptor(baseEventHashedData); + } else { + final byte[] signature = in.readByteArray(BaseEventUnhashedData.MAX_SIG_LENGTH); + baseEventUnhashedData = new BaseEventUnhashedData(null, signature); + } consensusData = in.readSerializable(false, ConsensusData::new); } diff --git a/platform-sdk/swirlds-platform-core/src/main/java/com/swirlds/platform/event/EventConstants.java b/platform-sdk/swirlds-common/src/main/java/com/swirlds/common/system/events/EventConstants.java similarity index 85% rename from platform-sdk/swirlds-platform-core/src/main/java/com/swirlds/platform/event/EventConstants.java rename to platform-sdk/swirlds-common/src/main/java/com/swirlds/common/system/events/EventConstants.java index 471f2423ee5b..d596ba7d4221 100644 --- a/platform-sdk/swirlds-platform-core/src/main/java/com/swirlds/platform/event/EventConstants.java +++ b/platform-sdk/swirlds-common/src/main/java/com/swirlds/common/system/events/EventConstants.java @@ -1,5 +1,5 @@ /* - * Copyright (C) 2016-2023 Hedera Hashgraph, LLC + * Copyright (C) 2023 Hedera Hashgraph, LLC * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -14,7 +14,7 @@ * limitations under the License. */ -package com.swirlds.platform.event; +package com.swirlds.common.system.events; import com.swirlds.common.system.NodeId; @@ -35,4 +35,6 @@ private EventConstants() {} public static final NodeId CREATOR_ID_UNDEFINED = NodeId.UNDEFINED_NODE_ID; /** the smallest round an event can belong to */ public static final long MINIMUM_ROUND_CREATED = 1; + /** the round number to represent that the birth round is undefined */ + public static final long BIRTH_ROUND_UNDEFINED = -1; } diff --git a/platform-sdk/swirlds-common/src/main/java/com/swirlds/common/system/events/EventDescriptor.java b/platform-sdk/swirlds-common/src/main/java/com/swirlds/common/system/events/EventDescriptor.java index 8da7147d062a..8e3d112c11da 100644 --- a/platform-sdk/swirlds-common/src/main/java/com/swirlds/common/system/events/EventDescriptor.java +++ b/platform-sdk/swirlds-common/src/main/java/com/swirlds/common/system/events/EventDescriptor.java @@ -18,6 +18,7 @@ import static com.swirlds.common.utility.CommonUtils.hex; +import com.swirlds.base.utility.ToStringBuilder; import com.swirlds.common.crypto.Hash; import com.swirlds.common.io.SelfSerializable; import com.swirlds.common.io.streams.SerializableDataInputStream; @@ -38,14 +39,22 @@ private static final class ClassVersion { public static final int ORIGINAL = 1; /** * The creator field is serialized as a self serializable node id. + * * @since 0.40.0 */ public static final int SELF_SERIALIZABLE_NODE_ID = 2; + /** + * The birthRound field is added. + * + * @since 0.46.0 + */ + public static final int BIRTH_ROUND = 3; } private Hash hash; private NodeId creator; private long generation; + private long birthRound; /** * Zero arg constructor, required for deserialization. Do not use manually. @@ -55,14 +64,52 @@ public EventDescriptor() {} /** * Create a new event descriptor. * - * @param hash the hash of the event - * @param creator the creator of the event - * @param generation the age of an event, smaller is older + * @param hash the hash of the event + * @param creator the creator of the event + * @param generation the age of an event, smaller is older + * @param birthRound the round when the event was created */ - public EventDescriptor(@NonNull final Hash hash, @NonNull final NodeId creator, final long generation) { + public EventDescriptor( + @NonNull final Hash hash, @NonNull final NodeId creator, final long generation, final long birthRound) { this.hash = Objects.requireNonNull(hash, "hash must not be null"); this.creator = Objects.requireNonNull(creator, "creator must not be null"); this.generation = generation; + this.birthRound = birthRound; + } + + /** + * Create a new event descriptor. This is package protected to only allow related classes to use it. The creator + * must be set before retrieval. + * + * @param hash the hash of the event + * @param generation the age of an event, smaller is older + * @param birthRound the round when the event was created + * @deprecated (since = "0.46.0", forRemoval = true) + */ + protected EventDescriptor(@NonNull final Hash hash, final long generation, final long birthRound) { + this.hash = Objects.requireNonNull(hash, "hash must not be null"); + this.generation = generation; + this.birthRound = birthRound; + this.creator = null; + } + + /** + * Set the creator node of the event. This is package protected to only allow related classes to use it. + * + * @param creator the creator node id + * @deprecated (since = "0.46.0", forRemoval = true) + */ + protected void setCreator(@NonNull final NodeId creator) { + this.creator = Objects.requireNonNull(creator, "creator must not be null"); + } + + private void checkInitialization() { + if (hash == null) { + throw new IllegalStateException("EventDescriptor improperly initialized: the hash is null"); + } + if (creator == null) { + throw new IllegalStateException("EventDescriptor improperly initialized: the creator node id is null"); + } } /** @@ -72,9 +119,7 @@ public EventDescriptor(@NonNull final Hash hash, @NonNull final NodeId creator, */ @NonNull public Hash getHash() { - if (hash == null) { - throw new IllegalStateException("EventDescriptor improperly initialized: the hash is null"); - } + checkInitialization(); return hash; } @@ -85,9 +130,7 @@ public Hash getHash() { */ @NonNull public NodeId getCreator() { - if (hash == null) { - throw new IllegalStateException("EventDescriptor improperly initialized: the hash is null"); - } + checkInitialization(); return creator; } @@ -100,6 +143,15 @@ public long getGeneration() { return generation; } + /** + * Get the round when the event was created. + * + * @return the round when the event was created + */ + public long getBirthRound() { + return birthRound; + } + /** * {@inheritDoc} */ @@ -113,6 +165,11 @@ public long getClassId() { */ @Override public int getVersion() { + return ClassVersion.BIRTH_ROUND; + } + + @Override + public int getMinimumSupportedVersion() { return ClassVersion.SELF_SERIALIZABLE_NODE_ID; } @@ -124,6 +181,7 @@ public void serialize(@NonNull final SerializableDataOutputStream out) throws IO out.writeSerializable(hash, false); out.writeSerializable(creator, false); out.writeLong(generation); + out.writeLong(birthRound); } /** @@ -135,15 +193,16 @@ public void deserialize(@NonNull final SerializableDataInputStream in, final int if (hash == null) { throw new IOException("hash cannot be null"); } - if (version < ClassVersion.SELF_SERIALIZABLE_NODE_ID) { - creator = new NodeId(in.readLong()); - } else { - creator = in.readSerializable(false, NodeId::new); - if (creator == null) { - throw new IOException("creator cannot be null"); - } + creator = in.readSerializable(false, NodeId::new); + if (creator == null) { + throw new IOException("creator cannot be null"); } generation = in.readLong(); + if (version < ClassVersion.BIRTH_ROUND) { + birthRound = EventConstants.BIRTH_ROUND_UNDEFINED; + } else { + birthRound = in.readLong(); + } } /** @@ -160,7 +219,10 @@ public boolean equals(final Object o) { final EventDescriptor that = (EventDescriptor) o; - return Objects.equals(creator, that.creator) && generation == that.generation && hash.equals(that.hash); + return Objects.equals(creator, that.creator) + && generation == that.generation + && birthRound == that.birthRound + && hash.equals(that.hash); } /** @@ -176,8 +238,11 @@ public int hashCode() { @Override public String toString() { - return "(creator: " + creator + ", generation: " - + generation + ", hash: " - + hex(hash.getValue()).substring(0, 12) + ")"; + return new ToStringBuilder(this) + .append("creator", creator) + .append("generation", generation) + .append("birthRound", birthRound) + .append("hash", hex(hash.getValue()).substring(0, 12)) + .toString(); } } diff --git a/platform-sdk/swirlds-platform-core/src/main/java/com/swirlds/platform/consensus/RoundCalculationUtils.java b/platform-sdk/swirlds-platform-core/src/main/java/com/swirlds/platform/consensus/RoundCalculationUtils.java index b3d35b9bf344..c12182892161 100644 --- a/platform-sdk/swirlds-platform-core/src/main/java/com/swirlds/platform/consensus/RoundCalculationUtils.java +++ b/platform-sdk/swirlds-platform-core/src/main/java/com/swirlds/platform/consensus/RoundCalculationUtils.java @@ -16,7 +16,7 @@ package com.swirlds.platform.consensus; -import com.swirlds.platform.event.EventConstants; +import com.swirlds.common.system.events.EventConstants; import com.swirlds.platform.state.signed.SignedState; import java.util.function.LongUnaryOperator; diff --git a/platform-sdk/swirlds-platform-core/src/main/java/com/swirlds/platform/consensus/RoundElections.java b/platform-sdk/swirlds-platform-core/src/main/java/com/swirlds/platform/consensus/RoundElections.java index 342333d89bda..abf775be0250 100644 --- a/platform-sdk/swirlds-platform-core/src/main/java/com/swirlds/platform/consensus/RoundElections.java +++ b/platform-sdk/swirlds-platform-core/src/main/java/com/swirlds/platform/consensus/RoundElections.java @@ -19,9 +19,9 @@ import static com.swirlds.logging.legacy.LogMarker.CONSENSUS_VOTING; import com.swirlds.common.system.NodeId; +import com.swirlds.common.system.events.EventConstants; import com.swirlds.common.utility.IntReference; import com.swirlds.platform.Utilities; -import com.swirlds.platform.event.EventConstants; import com.swirlds.platform.event.EventMetadata; import com.swirlds.platform.internal.EventImpl; import com.swirlds.platform.state.MinGenInfo; diff --git a/platform-sdk/swirlds-platform-core/src/main/java/com/swirlds/platform/event/EventStringBuilder.java b/platform-sdk/swirlds-platform-core/src/main/java/com/swirlds/platform/event/EventStringBuilder.java index e724ed7b3a25..a1d01f0bf351 100644 --- a/platform-sdk/swirlds-platform-core/src/main/java/com/swirlds/platform/event/EventStringBuilder.java +++ b/platform-sdk/swirlds-platform-core/src/main/java/com/swirlds/platform/event/EventStringBuilder.java @@ -21,6 +21,7 @@ import com.swirlds.common.system.events.BaseEvent; import com.swirlds.common.system.events.BaseEventHashedData; import com.swirlds.common.system.events.BaseEventUnhashedData; +import com.swirlds.common.system.events.EventConstants; import com.swirlds.common.utility.CommonUtils; import com.swirlds.platform.internal.EventImpl; import edu.umd.cs.findbugs.annotations.Nullable; diff --git a/platform-sdk/swirlds-platform-core/src/main/java/com/swirlds/platform/event/EventUtils.java b/platform-sdk/swirlds-platform-core/src/main/java/com/swirlds/platform/event/EventUtils.java index a8a8da73517d..f0c7c9611d1e 100644 --- a/platform-sdk/swirlds-platform-core/src/main/java/com/swirlds/platform/event/EventUtils.java +++ b/platform-sdk/swirlds-platform-core/src/main/java/com/swirlds/platform/event/EventUtils.java @@ -18,6 +18,7 @@ import com.swirlds.common.system.NodeId; import com.swirlds.common.system.events.BaseEvent; +import com.swirlds.common.system.events.EventConstants; import com.swirlds.common.system.events.PlatformEvent; import com.swirlds.logging.legacy.LogMarker; import com.swirlds.platform.EventStrings; diff --git a/platform-sdk/swirlds-platform-core/src/main/java/com/swirlds/platform/event/GossipEvent.java b/platform-sdk/swirlds-platform-core/src/main/java/com/swirlds/platform/event/GossipEvent.java index ab9e87b78baf..63e896b500eb 100644 --- a/platform-sdk/swirlds-platform-core/src/main/java/com/swirlds/platform/event/GossipEvent.java +++ b/platform-sdk/swirlds-platform-core/src/main/java/com/swirlds/platform/event/GossipEvent.java @@ -36,6 +36,20 @@ */ public class GossipEvent implements BaseEvent, ChatterEvent { private static final long CLASS_ID = 0xfe16b46795bfb8dcL; + private static final int MAX_SIG_LENGTH = 384; + + private static final class ClassVersion { + public static final int ORIGINAL = 1; + public static final int REMOVED_ROUND = 2; + /** + * Event serialization changes + * + * @since 0.46.0 + */ + public static final int BIRTH_ROUND = 3; + } + + private int serializedVersion = ClassVersion.BIRTH_ROUND; private BaseEventHashedData hashedData; private BaseEventUnhashedData unhashedData; private EventDescriptor descriptor; @@ -44,8 +58,8 @@ public class GossipEvent implements BaseEvent, ChatterEvent { /** * The id of the node which sent us this event *

- * The sender ID of an event should not be serialized when an event is serialized, and it should not affect the - * hash of the event in any way. + * The sender ID of an event should not be serialized when an event is serialized, and it should not affect the hash + * of the event in any way. */ private NodeId senderId; @@ -59,6 +73,8 @@ public GossipEvent() {} public GossipEvent(final BaseEventHashedData hashedData, final BaseEventUnhashedData unhashedData) { this.hashedData = hashedData; this.unhashedData = unhashedData; + // remove update of other parent event descriptor after 0.46.0 hits mainnet. + unhashedData.updateOtherParentEventDescriptor(hashedData); this.timeReceived = Instant.now(); this.senderId = null; } @@ -68,8 +84,13 @@ public GossipEvent(final BaseEventHashedData hashedData, final BaseEventUnhashed */ @Override public void serialize(final SerializableDataOutputStream out) throws IOException { - out.writeSerializable(hashedData, false); - out.writeSerializable(unhashedData, false); + if (serializedVersion < ClassVersion.BIRTH_ROUND) { + out.writeSerializable(hashedData, false); + out.writeSerializable(unhashedData, false); + } else { + out.writeSerializable(hashedData, false); + out.writeByteArray(unhashedData.getSignature()); + } } /** @@ -77,11 +98,17 @@ public void serialize(final SerializableDataOutputStream out) throws IOException */ @Override public void deserialize(final SerializableDataInputStream in, final int version) throws IOException { - hashedData = in.readSerializable(false, BaseEventHashedData::new); - unhashedData = in.readSerializable(false, BaseEventUnhashedData::new); - if (version == ClassVersion.ORIGINAL) { - in.readLong(); // roundCreated + serializedVersion = version; + if (version < ClassVersion.BIRTH_ROUND) { + hashedData = in.readSerializable(false, BaseEventHashedData::new); + unhashedData = in.readSerializable(false, BaseEventUnhashedData::new); + } else { + hashedData = in.readSerializable(false, BaseEventHashedData::new); + final byte[] signature = in.readByteArray(MAX_SIG_LENGTH); + unhashedData = new BaseEventUnhashedData(null, signature); } + // remove update of other parent event descriptor after 0.46.0 hits mainnet. + unhashedData.updateOtherParentEventDescriptor(hashedData); timeReceived = Instant.now(); } @@ -117,8 +144,7 @@ public EventDescriptor getDescriptor() { * hashed before the descriptor can be built. */ public void buildDescriptor() { - this.descriptor = - new EventDescriptor(hashedData.getHash(), hashedData.getCreatorId(), hashedData.getGeneration()); + this.descriptor = hashedData.createEventDescriptor(); } @Override @@ -166,6 +192,11 @@ public long getClassId() { */ @Override public int getVersion() { + return serializedVersion; + } + + @Override + public int getMinimumSupportedVersion() { return ClassVersion.REMOVED_ROUND; } @@ -201,9 +232,4 @@ public boolean equals(final Object o) { public int hashCode() { return hashedData.getHash().hashCode(); } - - private static final class ClassVersion { - public static final int ORIGINAL = 1; - public static final int REMOVED_ROUND = 2; - } } diff --git a/platform-sdk/swirlds-platform-core/src/main/java/com/swirlds/platform/event/creation/tipset/TipsetEventCreator.java b/platform-sdk/swirlds-platform-core/src/main/java/com/swirlds/platform/event/creation/tipset/TipsetEventCreator.java index c8826dbfb103..4a2c1885a399 100644 --- a/platform-sdk/swirlds-platform-core/src/main/java/com/swirlds/platform/event/creation/tipset/TipsetEventCreator.java +++ b/platform-sdk/swirlds-platform-core/src/main/java/com/swirlds/platform/event/creation/tipset/TipsetEventCreator.java @@ -17,10 +17,10 @@ package com.swirlds.platform.event.creation.tipset; import static com.swirlds.common.system.NodeId.UNDEFINED_NODE_ID; +import static com.swirlds.common.system.events.EventConstants.CREATOR_ID_UNDEFINED; +import static com.swirlds.common.system.events.EventConstants.GENERATION_UNDEFINED; import static com.swirlds.logging.legacy.LogMarker.EXCEPTION; import static com.swirlds.platform.consensus.GraphGenerations.FIRST_GENERATION; -import static com.swirlds.platform.event.EventConstants.CREATOR_ID_UNDEFINED; -import static com.swirlds.platform.event.EventConstants.GENERATION_UNDEFINED; import static com.swirlds.platform.event.creation.tipset.TipsetAdvancementWeight.ZERO_ADVANCEMENT_WEIGHT; import static com.swirlds.platform.event.creation.tipset.TipsetUtils.buildDescriptor; import static com.swirlds.platform.event.creation.tipset.TipsetUtils.getParentDescriptors; @@ -178,7 +178,8 @@ public void registerEvent(@NonNull final EventImpl event) { final EventDescriptor parentDescriptor = new EventDescriptor( event.getBaseEventHashedData().getOtherParentHash(), event.getBaseEventUnhashedData().getOtherId(), - event.getBaseEventHashedData().getOtherParentGen()); + event.getBaseEventHashedData().getOtherParentGen(), + event.getBaseEventHashedData().getBirthRound()); childlessOtherEventTracker.registerSelfEventParents(List.of(parentDescriptor)); } @@ -410,12 +411,6 @@ private GossipEvent buildAndProcessEvent(@Nullable final EventDescriptor otherPa */ @NonNull private GossipEvent assembleEventObject(@Nullable final EventDescriptor otherParent) { - - final long selfParentGeneration = getGeneration(lastSelfEvent); - final Hash selfParentHash = getHash(lastSelfEvent); - - final long otherParentGeneration = getGeneration(otherParent); - final Hash otherParentHash = getHash(otherParent); final NodeId otherParentId = getCreator(otherParent); final Instant now = time.now(); @@ -430,10 +425,9 @@ private GossipEvent assembleEventObject(@Nullable final EventDescriptor otherPar final BaseEventHashedData hashedData = new BaseEventHashedData( softwareVersion, selfId, - selfParentGeneration, - otherParentGeneration, - selfParentHash, - otherParentHash, + lastSelfEvent, + otherParent == null ? Collections.emptyList() : Collections.singletonList(otherParent), + addressBook.getRound(), timeCreated, transactionSupplier.getTransactions()); cryptography.digestSync(hashedData); diff --git a/platform-sdk/swirlds-platform-core/src/main/java/com/swirlds/platform/event/creation/tipset/TipsetUtils.java b/platform-sdk/swirlds-platform-core/src/main/java/com/swirlds/platform/event/creation/tipset/TipsetUtils.java index 64a6606a2247..864e06c0b97d 100644 --- a/platform-sdk/swirlds-platform-core/src/main/java/com/swirlds/platform/event/creation/tipset/TipsetUtils.java +++ b/platform-sdk/swirlds-platform-core/src/main/java/com/swirlds/platform/event/creation/tipset/TipsetUtils.java @@ -16,6 +16,7 @@ package com.swirlds.platform.event.creation.tipset; +import com.swirlds.common.system.events.BaseEventHashedData; import com.swirlds.common.system.events.EventDescriptor; import com.swirlds.platform.event.GossipEvent; import com.swirlds.platform.internal.EventImpl; @@ -40,7 +41,7 @@ public static EventDescriptor buildDescriptor(@NonNull final EventImpl event) { if (event.getBaseHash() == null) { throw new IllegalStateException("event is not hashed"); } - return new EventDescriptor(event.getBaseHash(), event.getCreatorId(), event.getGeneration()); + return event.getHashedData().createEventDescriptor(); } /** @@ -81,19 +82,12 @@ public static List getParentDescriptors(@NonNull final EventImp public static List getParentDescriptors(@NonNull final GossipEvent event) { final List parentDescriptors = new ArrayList<>(2); - if (event.getHashedData().getSelfParentHash() != null) { - final EventDescriptor parent = new EventDescriptor( - event.getHashedData().getSelfParentHash(), - event.getHashedData().getCreatorId(), - event.getHashedData().getSelfParentGen()); - parentDescriptors.add(parent); + final BaseEventHashedData hashedData = event.getHashedData(); + if (hashedData.hasSelfParent()) { + parentDescriptors.add(event.getHashedData().getSelfParent()); } - if (event.getHashedData().getOtherParentHash() != null) { - final EventDescriptor parent = new EventDescriptor( - event.getHashedData().getOtherParentHash(), - event.getUnhashedData().getOtherId(), - event.getHashedData().getOtherParentGen()); - parentDescriptors.add(parent); + if (hashedData.hasOtherParent()) { + hashedData.getOtherParents().forEach(parentDescriptors::add); } return parentDescriptors; diff --git a/platform-sdk/swirlds-platform-core/src/main/java/com/swirlds/platform/event/orphan/ParentIterator.java b/platform-sdk/swirlds-platform-core/src/main/java/com/swirlds/platform/event/orphan/ParentIterator.java index 2fd45d7e6e42..f1cfea2a82b6 100644 --- a/platform-sdk/swirlds-platform-core/src/main/java/com/swirlds/platform/event/orphan/ParentIterator.java +++ b/platform-sdk/swirlds-platform-core/src/main/java/com/swirlds/platform/event/orphan/ParentIterator.java @@ -47,19 +47,13 @@ class ParentIterator implements Iterator { ParentIterator(@NonNull final GossipEvent event) { parents = new ArrayList<>(); - if (event.getHashedData().getSelfParentHash() != null) { - parents.add(new EventDescriptor( - event.getHashedData().getSelfParentHash(), - event.getHashedData().getCreatorId(), - event.getHashedData().getSelfParentGen())); - } + final EventDescriptor selfParent = event.getHashedData().getSelfParent(); + final List otherParents = event.getHashedData().getOtherParents(); - if (event.getHashedData().getOtherParentHash() != null) { - parents.add(new EventDescriptor( - event.getHashedData().getOtherParentHash(), - event.getUnhashedData().getOtherId(), - event.getHashedData().getOtherParentGen())); + if (selfParent != null) { + parents.add(selfParent); } + otherParents.forEach(parents::add); } /** diff --git a/platform-sdk/swirlds-platform-core/src/main/java/com/swirlds/platform/gui/hashgraph/internal/CachingGuiSource.java b/platform-sdk/swirlds-platform-core/src/main/java/com/swirlds/platform/gui/hashgraph/internal/CachingGuiSource.java index 08b66e695e0b..71cf0da34498 100644 --- a/platform-sdk/swirlds-platform-core/src/main/java/com/swirlds/platform/gui/hashgraph/internal/CachingGuiSource.java +++ b/platform-sdk/swirlds-platform-core/src/main/java/com/swirlds/platform/gui/hashgraph/internal/CachingGuiSource.java @@ -17,10 +17,10 @@ package com.swirlds.platform.gui.hashgraph.internal; import com.swirlds.common.system.address.AddressBook; +import com.swirlds.common.system.events.EventConstants; import com.swirlds.common.system.events.PlatformEvent; import com.swirlds.gui.hashgraph.HashgraphGuiConstants; import com.swirlds.gui.hashgraph.HashgraphGuiSource; -import com.swirlds.platform.event.EventConstants; import com.swirlds.platform.gossip.shadowgraph.Generations; /** diff --git a/platform-sdk/swirlds-platform-core/src/main/java/com/swirlds/platform/gui/hashgraph/internal/HashgraphGuiControls.java b/platform-sdk/swirlds-platform-core/src/main/java/com/swirlds/platform/gui/hashgraph/internal/HashgraphGuiControls.java index d24d75aed569..b0509953b0f2 100644 --- a/platform-sdk/swirlds-platform-core/src/main/java/com/swirlds/platform/gui/hashgraph/internal/HashgraphGuiControls.java +++ b/platform-sdk/swirlds-platform-core/src/main/java/com/swirlds/platform/gui/hashgraph/internal/HashgraphGuiControls.java @@ -19,10 +19,10 @@ import static com.swirlds.gui.GuiUtils.wrap; import static com.swirlds.gui.hashgraph.HashgraphGuiConstants.DEFAULT_GENERATIONS_TO_DISPLAY; +import com.swirlds.common.system.events.EventConstants; import com.swirlds.gui.GuiUtils; import com.swirlds.gui.hashgraph.HashgraphPictureOptions; import com.swirlds.platform.consensus.GraphGenerations; -import com.swirlds.platform.event.EventConstants; import java.awt.Checkbox; import java.awt.Color; import java.awt.Component; diff --git a/platform-sdk/swirlds-platform-core/src/main/java/com/swirlds/platform/state/signed/SignedStateFileManager.java b/platform-sdk/swirlds-platform-core/src/main/java/com/swirlds/platform/state/signed/SignedStateFileManager.java index 4ea0357f860b..b22d7f2b1e64 100644 --- a/platform-sdk/swirlds-platform-core/src/main/java/com/swirlds/platform/state/signed/SignedStateFileManager.java +++ b/platform-sdk/swirlds-platform-core/src/main/java/com/swirlds/platform/state/signed/SignedStateFileManager.java @@ -28,9 +28,9 @@ import com.swirlds.common.config.StateConfig; import com.swirlds.common.context.PlatformContext; import com.swirlds.common.system.NodeId; +import com.swirlds.common.system.events.EventConstants; import com.swirlds.config.api.Configuration; import com.swirlds.logging.legacy.payload.InsufficientSignaturesPayload; -import com.swirlds.platform.event.EventConstants; import edu.umd.cs.findbugs.annotations.NonNull; import edu.umd.cs.findbugs.annotations.Nullable; import java.io.IOException; diff --git a/platform-sdk/swirlds-platform-core/src/test/java/com/swirlds/platform/EventImplTests.java b/platform-sdk/swirlds-platform-core/src/test/java/com/swirlds/platform/EventImplTests.java index 2e80b244480a..e7dfc686d080 100644 --- a/platform-sdk/swirlds-platform-core/src/test/java/com/swirlds/platform/EventImplTests.java +++ b/platform-sdk/swirlds-platform-core/src/test/java/com/swirlds/platform/EventImplTests.java @@ -22,11 +22,13 @@ import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertTrue; -import com.swirlds.common.crypto.CryptographyHolder; +import com.swirlds.common.crypto.Hash; import com.swirlds.common.system.BasicSoftwareVersion; import com.swirlds.common.system.NodeId; import com.swirlds.common.system.events.BaseEventHashedData; import com.swirlds.common.system.events.BaseEventUnhashedData; +import com.swirlds.common.system.events.EventConstants; +import com.swirlds.common.system.events.EventDescriptor; import com.swirlds.common.system.transaction.ConsensusTransaction; import com.swirlds.common.system.transaction.Transaction; import com.swirlds.common.system.transaction.internal.ConsensusTransactionImpl; @@ -37,6 +39,7 @@ import com.swirlds.platform.internal.EventImpl; import java.time.Instant; import java.util.ArrayList; +import java.util.Collections; import java.util.HashSet; import java.util.Iterator; import java.util.List; @@ -238,14 +241,17 @@ void testConsensusReached() { } private static EventImpl newEvent(final ConsensusTransactionImpl[] transactions) { + + final EventDescriptor selfDescriptor = new EventDescriptor(new Hash(), new NodeId(0), -1, -1); + final EventDescriptor otherDescriptor = new EventDescriptor(new Hash(), new NodeId(0), -1, -1); + return new EventImpl( new BaseEventHashedData( new BasicSoftwareVersion(1), new NodeId(0L), - 0L, - 0L, - CryptographyHolder.get().getNullHash(), - CryptographyHolder.get().getNullHash(), + selfDescriptor, + Collections.singletonList(otherDescriptor), + EventConstants.BIRTH_ROUND_UNDEFINED, Instant.now(), transactions), new BaseEventUnhashedData(new NodeId(0L), new byte[0])); diff --git a/platform-sdk/swirlds-platform-core/src/test/java/com/swirlds/platform/TestUtils.java b/platform-sdk/swirlds-platform-core/src/test/java/com/swirlds/platform/TestUtils.java index b4d8514d613c..ceabf7e1350f 100644 --- a/platform-sdk/swirlds-platform-core/src/test/java/com/swirlds/platform/TestUtils.java +++ b/platform-sdk/swirlds-platform-core/src/test/java/com/swirlds/platform/TestUtils.java @@ -17,16 +17,20 @@ package com.swirlds.platform; import com.swirlds.common.crypto.CryptographyHolder; +import com.swirlds.common.crypto.Hash; import com.swirlds.common.system.BasicSoftwareVersion; import com.swirlds.common.system.NodeId; import com.swirlds.common.system.events.BaseEventHashedData; import com.swirlds.common.system.events.BaseEventUnhashedData; +import com.swirlds.common.system.events.EventConstants; +import com.swirlds.common.system.events.EventDescriptor; import com.swirlds.common.system.transaction.internal.SwirldTransaction; import com.swirlds.platform.internal.EventImpl; import edu.umd.cs.findbugs.annotations.NonNull; import edu.umd.cs.findbugs.annotations.Nullable; import java.time.Instant; import java.util.Arrays; +import java.util.Collections; import java.util.Objects; public class TestUtils { @@ -53,14 +57,17 @@ public static boolean equals(final EventImpl e1, final EventImpl e2) { static EventImpl createTestEvent(@NonNull final NodeId creatorId, @Nullable final NodeId otherId) { Objects.requireNonNull(creatorId, "creatorId must not be null"); final Instant startTime = Instant.ofEpochSecond(1554466913); + + final EventDescriptor selfDescriptor = new EventDescriptor(new Hash(), creatorId, -1, -1); + final EventDescriptor otherDescriptor = new EventDescriptor(new Hash(), otherId, -1, -1); + final EventImpl e = new EventImpl( new BaseEventHashedData( new BasicSoftwareVersion(1), creatorId, - -1, - -1, - (byte[]) null, - (byte[]) null, + selfDescriptor, + Collections.singletonList(otherDescriptor), + EventConstants.BIRTH_ROUND_UNDEFINED, startTime, new SwirldTransaction[0]), new BaseEventUnhashedData(otherId, new byte[] {0, 0, 0, 0}), diff --git a/platform-sdk/swirlds-platform-core/src/test/java/com/swirlds/platform/consensus/RoundCalculationUtilsTest.java b/platform-sdk/swirlds-platform-core/src/test/java/com/swirlds/platform/consensus/RoundCalculationUtilsTest.java index c52ee3e5a191..ce0db1be1e48 100644 --- a/platform-sdk/swirlds-platform-core/src/test/java/com/swirlds/platform/consensus/RoundCalculationUtilsTest.java +++ b/platform-sdk/swirlds-platform-core/src/test/java/com/swirlds/platform/consensus/RoundCalculationUtilsTest.java @@ -18,7 +18,7 @@ import static org.mockito.Mockito.when; -import com.swirlds.platform.event.EventConstants; +import com.swirlds.common.system.events.EventConstants; import com.swirlds.platform.state.PlatformData; import com.swirlds.platform.state.PlatformState; import com.swirlds.platform.state.State; diff --git a/platform-sdk/swirlds-platform-core/src/test/java/com/swirlds/platform/event/DetGenerateUtils.java b/platform-sdk/swirlds-platform-core/src/test/java/com/swirlds/platform/event/DetGenerateUtils.java index 3a79da013cb9..af2ea5db7bd6 100644 --- a/platform-sdk/swirlds-platform-core/src/test/java/com/swirlds/platform/event/DetGenerateUtils.java +++ b/platform-sdk/swirlds-platform-core/src/test/java/com/swirlds/platform/event/DetGenerateUtils.java @@ -27,12 +27,15 @@ import com.swirlds.common.system.events.BaseEventHashedData; import com.swirlds.common.system.events.BaseEventUnhashedData; import com.swirlds.common.system.events.ConsensusData; +import com.swirlds.common.system.events.EventConstants; +import com.swirlds.common.system.events.EventDescriptor; import com.swirlds.common.system.transaction.Transaction; import com.swirlds.common.system.transaction.internal.ConsensusTransactionImpl; import com.swirlds.common.system.transaction.internal.StateSignatureTransaction; import com.swirlds.common.system.transaction.internal.SwirldTransaction; import java.time.Instant; import java.util.ArrayList; +import java.util.Collections; import java.util.List; import java.util.Random; @@ -47,13 +50,25 @@ public abstract class DetGenerateUtils { private static final int DEFAULT_SIGNATURE_SIZE = 384; public static BaseEventHashedData generateBaseEventHashedData(final Random random) { + + final NodeId selfId = new NodeId(nextLong(random, 0)); + final EventDescriptor selfDescriptor = new EventDescriptor( + generateRandomHash(random, DEFAULT_HASH_TYPE), + selfId, + nextLong(random, 0), + EventConstants.BIRTH_ROUND_UNDEFINED); + final EventDescriptor otherDescriptor = new EventDescriptor( + generateRandomHash(random, DEFAULT_HASH_TYPE), + new NodeId(nextLong(random, 0)), + nextLong(random, 0), + EventConstants.BIRTH_ROUND_UNDEFINED); + return new BaseEventHashedData( new BasicSoftwareVersion(1), new NodeId(nextLong(random, 0)), // creatorId, must be positive - nextLong(random, 0), // selfParentGen, must be positive - nextLong(random, 0), // otherParentGen, must be positive - generateRandomHash(random, DEFAULT_HASH_TYPE), // selfParentHash - generateRandomHash(random, DEFAULT_HASH_TYPE), // otherParentHash + selfDescriptor, // selfParent + Collections.singletonList(otherDescriptor), // otherParents + EventConstants.BIRTH_ROUND_UNDEFINED, // birth round generateRandomInstant(random, DEFAULT_MAX_EPOCH), // timeCreated generateTransactions(DEFAULT_TRANSACTION_NUMBER, DEFAULT_TRANSACTION_MAX_SIZE, random) .toArray(new ConsensusTransactionImpl[0])); // transactions @@ -61,7 +76,7 @@ public static BaseEventHashedData generateBaseEventHashedData(final Random rando public static BaseEventUnhashedData generateBaseEventUnhashedData(final Random random) { return new BaseEventUnhashedData( - new NodeId(nextLong(random, 0)), // otherId, must be positive + null, // the other node id is no longer stored here. generateRandomByteArray(random, DEFAULT_SIGNATURE_SIZE)); // signature } diff --git a/platform-sdk/swirlds-platform-core/src/test/java/com/swirlds/platform/event/EventDeduplicatorTests.java b/platform-sdk/swirlds-platform-core/src/test/java/com/swirlds/platform/event/EventDeduplicatorTests.java index 8b7f6f3a6906..aa37772b0884 100644 --- a/platform-sdk/swirlds-platform-core/src/test/java/com/swirlds/platform/event/EventDeduplicatorTests.java +++ b/platform-sdk/swirlds-platform-core/src/test/java/com/swirlds/platform/event/EventDeduplicatorTests.java @@ -83,7 +83,7 @@ private GossipEvent createGossipEvent( final long generation, @NonNull final byte[] signature) { - final EventDescriptor descriptor = new EventDescriptor(hash, creatorId, generation); + final EventDescriptor descriptor = new EventDescriptor(hash, creatorId, generation, -1); final BaseEventUnhashedData unhashedData = mock(BaseEventUnhashedData.class); when(unhashedData.getSignature()).thenReturn(signature); diff --git a/platform-sdk/swirlds-platform-core/src/test/java/com/swirlds/platform/event/linking/InOrderLinkerTests.java b/platform-sdk/swirlds-platform-core/src/test/java/com/swirlds/platform/event/linking/InOrderLinkerTests.java index caffb69b1a93..f8bc8d3d76e8 100644 --- a/platform-sdk/swirlds-platform-core/src/test/java/com/swirlds/platform/event/linking/InOrderLinkerTests.java +++ b/platform-sdk/swirlds-platform-core/src/test/java/com/swirlds/platform/event/linking/InOrderLinkerTests.java @@ -16,9 +16,9 @@ package com.swirlds.platform.event.linking; +import static com.swirlds.common.system.events.EventConstants.GENERATION_UNDEFINED; import static com.swirlds.common.test.fixtures.RandomUtils.getRandomPrintSeed; import static com.swirlds.common.test.fixtures.RandomUtils.randomHash; -import static com.swirlds.platform.event.EventConstants.GENERATION_UNDEFINED; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertNotEquals; import static org.junit.jupiter.api.Assertions.assertNull; diff --git a/platform-sdk/swirlds-platform-core/src/test/java/com/swirlds/platform/event/orphan/OrphanBufferTests.java b/platform-sdk/swirlds-platform-core/src/test/java/com/swirlds/platform/event/orphan/OrphanBufferTests.java index 81a8f9975c9f..70dcba0037fc 100644 --- a/platform-sdk/swirlds-platform-core/src/test/java/com/swirlds/platform/event/orphan/OrphanBufferTests.java +++ b/platform-sdk/swirlds-platform-core/src/test/java/com/swirlds/platform/event/orphan/OrphanBufferTests.java @@ -30,6 +30,7 @@ import com.swirlds.common.system.NodeId; import com.swirlds.common.system.events.BaseEventHashedData; import com.swirlds.common.system.events.BaseEventUnhashedData; +import com.swirlds.common.system.events.EventConstants; import com.swirlds.common.system.events.EventDescriptor; import com.swirlds.platform.event.GossipEvent; import com.swirlds.platform.gossip.IntakeEventCounter; @@ -98,7 +99,8 @@ class OrphanBufferTests { */ private EventDescriptor createBootstrapEvent( @NonNull final NodeId nodeId, @NonNull final List parentCandidates) { - final EventDescriptor bootstrapEvent = new EventDescriptor(randomHash(random), nodeId, 0); + final EventDescriptor bootstrapEvent = + new EventDescriptor(randomHash(random), nodeId, 0, EventConstants.BIRTH_ROUND_UNDEFINED); parentCandidates.add(bootstrapEvent); @@ -130,6 +132,9 @@ private static GossipEvent createGossipEvent( when(hashedData.getSelfParentHash()).thenReturn(selfParent.getHash()); when(hashedData.getOtherParentHash()).thenReturn(otherParent.getHash()); when(hashedData.getTimeCreated()).thenReturn(Instant.now()); + when(hashedData.getSelfParent()).thenReturn(selfParent == null ? null : selfParent); + when(hashedData.getOtherParents()) + .thenReturn(otherParent == null ? Collections.emptyList() : Collections.singletonList(otherParent)); final BaseEventUnhashedData unhashedData = mock(BaseEventUnhashedData.class); when(unhashedData.getOtherId()).thenReturn(otherParent.getCreator()); @@ -137,7 +142,9 @@ private static GossipEvent createGossipEvent( final GossipEvent event = mock(GossipEvent.class); when(event.getHashedData()).thenReturn(hashedData); when(event.getUnhashedData()).thenReturn(unhashedData); - when(event.getDescriptor()).thenReturn(new EventDescriptor(eventHash, eventCreator, eventGeneration)); + when(event.getDescriptor()) + .thenReturn(new EventDescriptor( + eventHash, eventCreator, eventGeneration, EventConstants.BIRTH_ROUND_UNDEFINED)); when(event.getGeneration()).thenReturn(eventGeneration); when(event.getSenderId()).thenReturn(eventCreator); @@ -334,4 +341,36 @@ void pause() { assertEquals(halfEventCount, eventsExitedIntakePipeline.get() + emittedEvents.size()); assertEquals(0, orphanBuffer.getCurrentOrphanCount()); } + + @Test + @DisplayName("Test Parent Iterator") + void testParentIterator() { + final GossipEvent event = mock(GossipEvent.class); + + final EventDescriptor selfParent = + new EventDescriptor(new Hash(), new NodeId(0), 0, EventConstants.BIRTH_ROUND_UNDEFINED); + final EventDescriptor otherParent1 = + new EventDescriptor(new Hash(), new NodeId(1), 1, EventConstants.BIRTH_ROUND_UNDEFINED); + final EventDescriptor otherParent2 = + new EventDescriptor(new Hash(), new NodeId(2), 2, EventConstants.BIRTH_ROUND_UNDEFINED); + final EventDescriptor otherParent3 = + new EventDescriptor(new Hash(), new NodeId(3), 3, EventConstants.BIRTH_ROUND_UNDEFINED); + final List otherParents = new ArrayList<>(); + otherParents.add(otherParent1); + otherParents.add(otherParent2); + otherParents.add(otherParent3); + + final BaseEventHashedData eventBase = mock(BaseEventHashedData.class); + when(eventBase.getSelfParent()).thenReturn(selfParent); + when(eventBase.getOtherParents()).thenReturn(otherParents); + when(event.getHashedData()).thenReturn(eventBase); + + final ParentIterator iterator = new ParentIterator(event); + + assertEquals(selfParent, iterator.next(), "The first parent should be the self parent"); + int index = 0; + while (iterator.hasNext()) { + assertEquals(otherParents.get(index++), iterator.next(), "The next parent should be the next other parent"); + } + } } diff --git a/platform-sdk/swirlds-platform-core/src/test/java/com/swirlds/platform/recovery/RecoveryTestUtils.java b/platform-sdk/swirlds-platform-core/src/test/java/com/swirlds/platform/recovery/RecoveryTestUtils.java index 63b31039dfff..e47f0e70521d 100644 --- a/platform-sdk/swirlds-platform-core/src/test/java/com/swirlds/platform/recovery/RecoveryTestUtils.java +++ b/platform-sdk/swirlds-platform-core/src/test/java/com/swirlds/platform/recovery/RecoveryTestUtils.java @@ -36,6 +36,8 @@ import com.swirlds.common.system.events.BaseEventHashedData; import com.swirlds.common.system.events.BaseEventUnhashedData; import com.swirlds.common.system.events.ConsensusData; +import com.swirlds.common.system.events.EventConstants; +import com.swirlds.common.system.events.EventDescriptor; import com.swirlds.common.system.transaction.internal.ConsensusTransactionImpl; import com.swirlds.common.system.transaction.internal.SwirldTransaction; import com.swirlds.platform.internal.EventImpl; @@ -54,6 +56,7 @@ import java.time.Duration; import java.time.Instant; import java.util.ArrayList; +import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -86,19 +89,25 @@ public static EventImpl generateRandomEvent( transactions[transactionIndex] = new SwirldTransaction(contents); } + final NodeId selfId = new NodeId(random.nextLong(Long.MAX_VALUE)); + final NodeId otherId = new NodeId(random.nextLong(Long.MAX_VALUE)); + + final EventDescriptor selfDescriptor = new EventDescriptor( + randomHash(random), selfId, random.nextLong(), EventConstants.BIRTH_ROUND_UNDEFINED); + final EventDescriptor otherDescriptor = new EventDescriptor( + randomHash(random), otherId, random.nextLong(), EventConstants.BIRTH_ROUND_UNDEFINED); + final BaseEventHashedData baseEventHashedData = new BaseEventHashedData( new BasicSoftwareVersion(1), - new NodeId(random.nextLong(Long.MAX_VALUE)), - random.nextLong(), - random.nextLong(), - randomHash(random), - randomHash(random), + selfId, + selfDescriptor, + Collections.singletonList(otherDescriptor), + EventConstants.BIRTH_ROUND_UNDEFINED, now, transactions); - final BaseEventUnhashedData baseEventUnhashedData = new BaseEventUnhashedData( - new NodeId(random.nextLong(Long.MAX_VALUE)), - randomSignature(random).getSignatureBytes()); + final BaseEventUnhashedData baseEventUnhashedData = + new BaseEventUnhashedData(otherId, randomSignature(random).getSignatureBytes()); final ConsensusData consensusData = new ConsensusData(); consensusData.setConsensusTimestamp(now); diff --git a/platform-sdk/swirlds-platform-core/src/test/java/com/swirlds/platform/state/TransactionHandlerTest.java b/platform-sdk/swirlds-platform-core/src/test/java/com/swirlds/platform/state/TransactionHandlerTest.java index f8ba5de33006..6eb837ec0553 100644 --- a/platform-sdk/swirlds-platform-core/src/test/java/com/swirlds/platform/state/TransactionHandlerTest.java +++ b/platform-sdk/swirlds-platform-core/src/test/java/com/swirlds/platform/state/TransactionHandlerTest.java @@ -28,12 +28,15 @@ import com.swirlds.common.system.SwirldState; import com.swirlds.common.system.events.BaseEventHashedData; import com.swirlds.common.system.events.BaseEventUnhashedData; +import com.swirlds.common.system.events.EventConstants; +import com.swirlds.common.system.events.EventDescriptor; import com.swirlds.common.system.transaction.internal.ConsensusTransactionImpl; import com.swirlds.common.test.fixtures.RandomUtils; import com.swirlds.common.test.fixtures.TransactionUtils; import com.swirlds.platform.internal.EventImpl; import com.swirlds.platform.metrics.SwirldStateMetrics; import java.time.Instant; +import java.util.Collections; import java.util.List; import java.util.Random; import java.util.function.Supplier; @@ -76,14 +79,18 @@ void testSwirldStatePreHandle() { } private static EventImpl newEvent(final ConsensusTransactionImpl[] transactions) { + + final EventDescriptor selfDescriptor = new EventDescriptor( + CryptographyHolder.get().getNullHash(), new NodeId(0), 0, EventConstants.BIRTH_ROUND_UNDEFINED); + final EventDescriptor otherDescriptor = new EventDescriptor( + CryptographyHolder.get().getNullHash(), new NodeId(0), 0, EventConstants.BIRTH_ROUND_UNDEFINED); return new EventImpl( new BaseEventHashedData( new BasicSoftwareVersion(1), new NodeId(0L), - 0L, - 0L, - CryptographyHolder.get().getNullHash(), - CryptographyHolder.get().getNullHash(), + selfDescriptor, + Collections.singletonList(otherDescriptor), + EventConstants.BIRTH_ROUND_UNDEFINED, Instant.now(), transactions), new BaseEventUnhashedData(new NodeId(0L), new byte[0])); diff --git a/platform-sdk/swirlds-platform-core/src/testFixtures/java/com/swirlds/platform/test/fixtures/event/RandomEventUtils.java b/platform-sdk/swirlds-platform-core/src/testFixtures/java/com/swirlds/platform/test/fixtures/event/RandomEventUtils.java index 97b2bc0f0ba3..cad2cd023798 100644 --- a/platform-sdk/swirlds-platform-core/src/testFixtures/java/com/swirlds/platform/test/fixtures/event/RandomEventUtils.java +++ b/platform-sdk/swirlds-platform-core/src/testFixtures/java/com/swirlds/platform/test/fixtures/event/RandomEventUtils.java @@ -22,6 +22,8 @@ import com.swirlds.common.system.NodeId; import com.swirlds.common.system.events.BaseEventHashedData; import com.swirlds.common.system.events.BaseEventUnhashedData; +import com.swirlds.common.system.events.EventConstants; +import com.swirlds.common.system.events.EventDescriptor; import com.swirlds.common.system.transaction.internal.ConsensusTransactionImpl; import com.swirlds.common.system.transaction.internal.SwirldTransaction; import com.swirlds.common.test.fixtures.RandomUtils; @@ -30,6 +32,7 @@ import edu.umd.cs.findbugs.annotations.NonNull; import edu.umd.cs.findbugs.annotations.Nullable; import java.time.Instant; +import java.util.Collections; import java.util.Objects; import java.util.Random; @@ -203,13 +206,28 @@ public static BaseEventHashedData randomEventHashedData( @Nullable final EventImpl selfParent, @Nullable final EventImpl otherParent, final boolean fakeHash) { + + final EventDescriptor selfDescriptor = selfParent == null + ? null + : new EventDescriptor( + selfParent.getHash(), + selfParent.getCreatorId(), + selfParent.getGeneration(), + EventConstants.BIRTH_ROUND_UNDEFINED); + final EventDescriptor otherDescriptor = otherParent == null + ? null + : new EventDescriptor( + otherParent.getHash(), + otherParent.getCreatorId(), + otherParent.getGeneration(), + EventConstants.BIRTH_ROUND_UNDEFINED); + final BaseEventHashedData hashedData = new BaseEventHashedData( new BasicSoftwareVersion(1), creatorId, - selfParent != null ? selfParent.getGeneration() : -1, - otherParent != null ? otherParent.getGeneration() : -1, - selfParent != null ? selfParent.getBaseHash() : null, - otherParent != null ? otherParent.getBaseHash() : null, + selfDescriptor, + otherDescriptor == null ? Collections.emptyList() : Collections.singletonList(otherDescriptor), + EventConstants.BIRTH_ROUND_UNDEFINED, selfParent != null ? selfParent.getTimeCreated().plusMillis(1 + random.nextInt(DEFAULT_MAX_NEXT_EVENT_MILLIS)) : firstTimeCreated, @@ -236,13 +254,27 @@ public static BaseEventHashedData randomEventHashedDataWithTimestamp( @Nullable final EventImpl otherParent, final boolean fakeHash) { + final EventDescriptor selfDescriptor = (selfParent == null || selfParent.getBaseHash() == null) + ? null + : new EventDescriptor( + selfParent.getBaseHash(), + selfParent.getCreatorId(), + selfParent.getGeneration(), + EventConstants.BIRTH_ROUND_UNDEFINED); + final EventDescriptor otherDescriptor = (otherParent == null || otherParent.getBaseHash() == null) + ? null + : new EventDescriptor( + otherParent.getBaseHash(), + otherParent.getCreatorId(), + otherParent.getGeneration(), + EventConstants.BIRTH_ROUND_UNDEFINED); + final BaseEventHashedData hashedData = new BaseEventHashedData( new BasicSoftwareVersion(1), creatorId, - selfParent != null ? selfParent.getGeneration() : -1, - otherParent != null ? otherParent.getGeneration() : -1, - selfParent != null ? selfParent.getBaseHash() : null, - otherParent != null ? otherParent.getBaseHash() : null, + selfDescriptor, + otherDescriptor == null ? Collections.emptyList() : Collections.singletonList(otherDescriptor), + EventConstants.BIRTH_ROUND_UNDEFINED, timestamp, transactions); diff --git a/platform-sdk/swirlds-platform-core/src/testFixtures/java/com/swirlds/platform/test/fixtures/event/generator/AbstractGraphGenerator.java b/platform-sdk/swirlds-platform-core/src/testFixtures/java/com/swirlds/platform/test/fixtures/event/generator/AbstractGraphGenerator.java index 8ab6bee888d4..5a497b71bfff 100644 --- a/platform-sdk/swirlds-platform-core/src/testFixtures/java/com/swirlds/platform/test/fixtures/event/generator/AbstractGraphGenerator.java +++ b/platform-sdk/swirlds-platform-core/src/testFixtures/java/com/swirlds/platform/test/fixtures/event/generator/AbstractGraphGenerator.java @@ -17,8 +17,8 @@ package com.swirlds.platform.test.fixtures.event.generator; import com.swirlds.common.system.NodeId; +import com.swirlds.common.system.events.EventConstants; import com.swirlds.platform.consensus.GraphGenerations; -import com.swirlds.platform.event.EventConstants; import com.swirlds.platform.test.fixtures.event.IndexedEvent; import edu.umd.cs.findbugs.annotations.Nullable; import java.util.HashMap; diff --git a/platform-sdk/swirlds-unit-tests/core/swirlds-platform-test/src/main/java/com/swirlds/platform/test/chatter/simulator/SimulatedEvent.java b/platform-sdk/swirlds-unit-tests/core/swirlds-platform-test/src/main/java/com/swirlds/platform/test/chatter/simulator/SimulatedEvent.java index 043e93f21dfc..2474740a96a1 100644 --- a/platform-sdk/swirlds-unit-tests/core/swirlds-platform-test/src/main/java/com/swirlds/platform/test/chatter/simulator/SimulatedEvent.java +++ b/platform-sdk/swirlds-unit-tests/core/swirlds-platform-test/src/main/java/com/swirlds/platform/test/chatter/simulator/SimulatedEvent.java @@ -21,6 +21,7 @@ import com.swirlds.common.io.streams.SerializableDataInputStream; import com.swirlds.common.io.streams.SerializableDataOutputStream; import com.swirlds.common.system.NodeId; +import com.swirlds.common.system.events.EventConstants; import com.swirlds.common.system.events.EventDescriptor; import com.swirlds.platform.gossip.chatter.protocol.messages.ChatterEvent; import edu.umd.cs.findbugs.annotations.NonNull; @@ -72,7 +73,7 @@ public SimulatedEvent( random.nextBytes(hashBytes); final Hash hash = new Hash(hashBytes, DigestType.SHA_384); - this.descriptor = new EventDescriptor(hash, creator, round); + this.descriptor = new EventDescriptor(hash, creator, EventConstants.GENERATION_UNDEFINED, round); } /** diff --git a/platform-sdk/swirlds-unit-tests/core/swirlds-platform-test/src/main/java/com/swirlds/platform/test/consensus/framework/ConsensusTestOrchestrator.java b/platform-sdk/swirlds-unit-tests/core/swirlds-platform-test/src/main/java/com/swirlds/platform/test/consensus/framework/ConsensusTestOrchestrator.java index a56a9e3d925b..a5ea8260127e 100644 --- a/platform-sdk/swirlds-unit-tests/core/swirlds-platform-test/src/main/java/com/swirlds/platform/test/consensus/framework/ConsensusTestOrchestrator.java +++ b/platform-sdk/swirlds-unit-tests/core/swirlds-platform-test/src/main/java/com/swirlds/platform/test/consensus/framework/ConsensusTestOrchestrator.java @@ -17,7 +17,7 @@ package com.swirlds.platform.test.consensus.framework; import com.swirlds.common.system.address.AddressBook; -import com.swirlds.platform.event.EventConstants; +import com.swirlds.common.system.events.EventConstants; import com.swirlds.platform.state.signed.SignedState; import com.swirlds.platform.test.consensus.framework.validation.ConsensusOutputValidation; import com.swirlds.platform.test.consensus.framework.validation.Validations; diff --git a/platform-sdk/swirlds-unit-tests/core/swirlds-platform-test/src/main/java/com/swirlds/platform/test/utils/EqualsVerifier.java b/platform-sdk/swirlds-unit-tests/core/swirlds-platform-test/src/main/java/com/swirlds/platform/test/utils/EqualsVerifier.java index bed13c871a6d..83b10fc36b9c 100644 --- a/platform-sdk/swirlds-unit-tests/core/swirlds-platform-test/src/main/java/com/swirlds/platform/test/utils/EqualsVerifier.java +++ b/platform-sdk/swirlds-unit-tests/core/swirlds-platform-test/src/main/java/com/swirlds/platform/test/utils/EqualsVerifier.java @@ -23,11 +23,14 @@ import com.swirlds.common.system.events.BaseEventHashedData; import com.swirlds.common.system.events.BaseEventUnhashedData; import com.swirlds.common.system.events.ConsensusData; +import com.swirlds.common.system.events.EventConstants; +import com.swirlds.common.system.events.EventDescriptor; import com.swirlds.common.system.transaction.internal.SwirldTransaction; import com.swirlds.platform.event.GossipEvent; import com.swirlds.platform.internal.EventImpl; import java.time.Instant; import java.util.ArrayList; +import java.util.Collections; import java.util.List; import java.util.Random; import java.util.function.Function; @@ -71,13 +74,18 @@ public static BaseEventHashedData randomBaseEventHashedData(final RandomGenerato transactions[i] = randomSwirldTransaction(r); } + final NodeId selfId = new NodeId(r.nextLong(Long.MAX_VALUE)); + final EventDescriptor selfParent = new EventDescriptor( + randomHash(r), selfId, r.nextLong(Long.MAX_VALUE), EventConstants.BIRTH_ROUND_UNDEFINED); + final EventDescriptor otherParent = new EventDescriptor( + randomHash(r), selfId, r.nextLong(Long.MAX_VALUE), EventConstants.BIRTH_ROUND_UNDEFINED); + final BaseEventHashedData data = new BaseEventHashedData( new BasicSoftwareVersion(1), - new NodeId(r.nextLong(Long.MAX_VALUE)), - r.nextLong(Long.MAX_VALUE), - r.nextLong(Long.MAX_VALUE), - randomHash(r), - randomHash(r), + selfId, + selfParent, + Collections.singletonList(otherParent), + EventConstants.BIRTH_ROUND_UNDEFINED, randomInstant(r), transactions); data.setHash(randomHash(r)); diff --git a/platform-sdk/swirlds-unit-tests/core/swirlds-platform-test/src/test/java/com/swirlds/platform/test/EventStringsTest.java b/platform-sdk/swirlds-unit-tests/core/swirlds-platform-test/src/test/java/com/swirlds/platform/test/EventStringsTest.java index cc47b9e70f0d..9e19c019c004 100644 --- a/platform-sdk/swirlds-unit-tests/core/swirlds-platform-test/src/test/java/com/swirlds/platform/test/EventStringsTest.java +++ b/platform-sdk/swirlds-unit-tests/core/swirlds-platform-test/src/test/java/com/swirlds/platform/test/EventStringsTest.java @@ -23,6 +23,8 @@ import com.swirlds.common.system.NodeId; import com.swirlds.common.system.events.BaseEventHashedData; import com.swirlds.common.system.events.BaseEventUnhashedData; +import com.swirlds.common.system.events.EventConstants; +import com.swirlds.common.system.events.EventDescriptor; import com.swirlds.common.system.transaction.internal.ConsensusTransactionImpl; import com.swirlds.platform.EventStrings; import com.swirlds.platform.event.GossipEvent; @@ -30,6 +32,7 @@ import com.swirlds.test.framework.TestComponentTags; import com.swirlds.test.framework.TestTypeTags; import java.time.Instant; +import java.util.Collections; import org.junit.jupiter.api.DisplayName; import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; @@ -50,13 +53,19 @@ void testCopy() { final long spGen = 10; final long opGen = 10; + final NodeId selfId = new NodeId(id); + final NodeId otherId = new NodeId(opId); + final EventDescriptor selfParent = + new EventDescriptor(new Hash(), selfId, spGen, EventConstants.BIRTH_ROUND_UNDEFINED); + final EventDescriptor otherParent = + new EventDescriptor(new Hash(), otherId, opGen, EventConstants.BIRTH_ROUND_UNDEFINED); + BaseEventHashedData hashedData = new BaseEventHashedData( new BasicSoftwareVersion(1), - new NodeId(id), - spGen, - opGen, - new Hash(), - new Hash(), + selfId, + selfParent, + Collections.singletonList(otherParent), + EventConstants.BIRTH_ROUND_UNDEFINED, Instant.now(), new ConsensusTransactionImpl[0]); hashedData.setHash(new Hash()); diff --git a/platform-sdk/swirlds-unit-tests/core/swirlds-platform-test/src/test/java/com/swirlds/platform/test/chatter/protocol/messages/EventDescriptorTest.java b/platform-sdk/swirlds-unit-tests/core/swirlds-platform-test/src/test/java/com/swirlds/platform/test/chatter/protocol/messages/EventDescriptorTest.java index 8bca77e2e947..e2dc56c3144d 100644 --- a/platform-sdk/swirlds-unit-tests/core/swirlds-platform-test/src/test/java/com/swirlds/platform/test/chatter/protocol/messages/EventDescriptorTest.java +++ b/platform-sdk/swirlds-unit-tests/core/swirlds-platform-test/src/test/java/com/swirlds/platform/test/chatter/protocol/messages/EventDescriptorTest.java @@ -25,6 +25,7 @@ import com.swirlds.common.constructable.ConstructableRegistry; import com.swirlds.common.constructable.ConstructableRegistryException; import com.swirlds.common.system.NodeId; +import com.swirlds.common.system.events.EventConstants; import com.swirlds.common.system.events.EventDescriptor; import com.swirlds.common.test.fixtures.RandomUtils; import com.swirlds.common.test.fixtures.io.SerializationUtils; @@ -34,20 +35,25 @@ class EventDescriptorTest { @Test void testSerialization() throws IOException, ConstructableRegistryException { - final EventDescriptor descriptor = new EventDescriptor(RandomUtils.randomHash(), new NodeId(1), 123); + final EventDescriptor descriptor = + new EventDescriptor(RandomUtils.randomHash(), new NodeId(1), 123, EventConstants.BIRTH_ROUND_UNDEFINED); ConstructableRegistry.getInstance() .registerConstructable(new ClassConstructorPair(EventDescriptor.class, EventDescriptor::new)); final EventDescriptor copy = SerializationUtils.serializeDeserialize(descriptor); assertEquals(descriptor, copy, "deserialized version should be the same"); assertThrows( - Exception.class, () -> new EventDescriptor(null, new NodeId(0), 0), "we should not permit a null hash"); + Exception.class, + () -> new EventDescriptor(null, new NodeId(0), 0, EventConstants.BIRTH_ROUND_UNDEFINED), + "we should not permit a null hash"); } @Test void testEquals() { - final EventDescriptor d1 = new EventDescriptor(RandomUtils.randomHash(), new NodeId(1), 123); - final EventDescriptor d2 = new EventDescriptor(RandomUtils.randomHash(), new NodeId(2), 234); + final EventDescriptor d1 = + new EventDescriptor(RandomUtils.randomHash(), new NodeId(1), 123, EventConstants.BIRTH_ROUND_UNDEFINED); + final EventDescriptor d2 = + new EventDescriptor(RandomUtils.randomHash(), new NodeId(2), 234, EventConstants.BIRTH_ROUND_UNDEFINED); assertTrue(d1.equals(d1), "should be equal to itself"); assertFalse(d1.equals(null), "should not be equal to null"); assertFalse(d1.equals(new Object()), "should not be equal to a different class"); diff --git a/platform-sdk/swirlds-unit-tests/core/swirlds-platform-test/src/test/java/com/swirlds/platform/test/components/EventUtilsTests.java b/platform-sdk/swirlds-unit-tests/core/swirlds-platform-test/src/test/java/com/swirlds/platform/test/components/EventUtilsTests.java index 4b4fe6ea4bb6..98341c2d07b8 100644 --- a/platform-sdk/swirlds-unit-tests/core/swirlds-platform-test/src/test/java/com/swirlds/platform/test/components/EventUtilsTests.java +++ b/platform-sdk/swirlds-unit-tests/core/swirlds-platform-test/src/test/java/com/swirlds/platform/test/components/EventUtilsTests.java @@ -25,8 +25,8 @@ import com.swirlds.common.system.NodeId; import com.swirlds.common.system.events.BaseEventHashedData; +import com.swirlds.common.system.events.EventConstants; import com.swirlds.common.system.transaction.internal.SwirldTransaction; -import com.swirlds.platform.event.EventConstants; import com.swirlds.platform.event.EventUtils; import com.swirlds.platform.test.event.EventMocks; import com.swirlds.test.framework.TestComponentTags; diff --git a/platform-sdk/swirlds-unit-tests/core/swirlds-platform-test/src/test/java/com/swirlds/platform/test/components/TransactionHandlingTestUtils.java b/platform-sdk/swirlds-unit-tests/core/swirlds-platform-test/src/test/java/com/swirlds/platform/test/components/TransactionHandlingTestUtils.java index a5d178f6a5b7..51570ff2d5ee 100644 --- a/platform-sdk/swirlds-unit-tests/core/swirlds-platform-test/src/test/java/com/swirlds/platform/test/components/TransactionHandlingTestUtils.java +++ b/platform-sdk/swirlds-unit-tests/core/swirlds-platform-test/src/test/java/com/swirlds/platform/test/components/TransactionHandlingTestUtils.java @@ -24,6 +24,8 @@ import com.swirlds.common.system.address.AddressBook; import com.swirlds.common.system.events.BaseEventHashedData; import com.swirlds.common.system.events.BaseEventUnhashedData; +import com.swirlds.common.system.events.EventConstants; +import com.swirlds.common.system.events.EventDescriptor; import com.swirlds.common.system.transaction.internal.SystemTransaction; import com.swirlds.common.test.fixtures.DummySystemTransaction; import com.swirlds.platform.consensus.ConsensusSnapshot; @@ -32,6 +34,7 @@ import com.swirlds.platform.internal.EventImpl; import java.time.Instant; import java.util.ArrayList; +import java.util.Collections; import java.util.List; /** @@ -51,14 +54,18 @@ public static EventImpl newDummyEvent(final int transactionCount) { transactions[index] = new DummySystemTransaction(); } + final EventDescriptor selfParent = new EventDescriptor( + CryptographyHolder.get().getNullHash(), new NodeId(0), 0, EventConstants.BIRTH_ROUND_UNDEFINED); + final EventDescriptor otherParent = new EventDescriptor( + CryptographyHolder.get().getNullHash(), new NodeId(0), 0, EventConstants.BIRTH_ROUND_UNDEFINED); + return new EventImpl( new BaseEventHashedData( new BasicSoftwareVersion(1), new NodeId(0), - 0L, - 0L, - CryptographyHolder.get().getNullHash(), - CryptographyHolder.get().getNullHash(), + selfParent, + Collections.singletonList(otherParent), + EventConstants.BIRTH_ROUND_UNDEFINED, Instant.now(), transactions), new BaseEventUnhashedData(new NodeId(0L), new byte[0])); diff --git a/platform-sdk/swirlds-unit-tests/core/swirlds-platform-test/src/test/java/com/swirlds/platform/test/consensus/IntakeAndConsensusTests.java b/platform-sdk/swirlds-unit-tests/core/swirlds-platform-test/src/test/java/com/swirlds/platform/test/consensus/IntakeAndConsensusTests.java index 67f8b2a6d8ff..3a6f1b64f9c2 100644 --- a/platform-sdk/swirlds-unit-tests/core/swirlds-platform-test/src/test/java/com/swirlds/platform/test/consensus/IntakeAndConsensusTests.java +++ b/platform-sdk/swirlds-unit-tests/core/swirlds-platform-test/src/test/java/com/swirlds/platform/test/consensus/IntakeAndConsensusTests.java @@ -22,8 +22,8 @@ import com.swirlds.common.config.ConsensusConfig_; import com.swirlds.common.system.NodeId; import com.swirlds.common.system.address.AddressBook; +import com.swirlds.common.system.events.EventConstants; import com.swirlds.config.api.Configuration; -import com.swirlds.platform.event.EventConstants; import com.swirlds.platform.test.consensus.framework.validation.ConsensusRoundValidation; import com.swirlds.platform.test.fixtures.event.DynamicValue; import com.swirlds.platform.test.fixtures.event.IndexedEvent; diff --git a/platform-sdk/swirlds-unit-tests/core/swirlds-platform-test/src/test/java/com/swirlds/platform/test/event/GossipEventBuilder.java b/platform-sdk/swirlds-unit-tests/core/swirlds-platform-test/src/test/java/com/swirlds/platform/test/event/GossipEventBuilder.java index 39b3ab5f5be3..07c56b8c294e 100644 --- a/platform-sdk/swirlds-unit-tests/core/swirlds-platform-test/src/test/java/com/swirlds/platform/test/event/GossipEventBuilder.java +++ b/platform-sdk/swirlds-unit-tests/core/swirlds-platform-test/src/test/java/com/swirlds/platform/test/event/GossipEventBuilder.java @@ -22,14 +22,16 @@ import com.swirlds.common.system.NodeId; import com.swirlds.common.system.events.BaseEventHashedData; import com.swirlds.common.system.events.BaseEventUnhashedData; +import com.swirlds.common.system.events.EventConstants; +import com.swirlds.common.system.events.EventDescriptor; import com.swirlds.common.system.transaction.internal.ConsensusTransactionImpl; import com.swirlds.common.system.transaction.internal.SwirldTransaction; import com.swirlds.common.test.fixtures.RandomUtils; import com.swirlds.platform.consensus.GraphGenerations; -import com.swirlds.platform.event.EventConstants; import com.swirlds.platform.event.GossipEvent; import com.swirlds.platform.internal.EventImpl; import java.time.Instant; +import java.util.Collections; import java.util.Random; public class GossipEventBuilder { @@ -178,18 +180,42 @@ public GossipEvent buildGossipEvent() { : EventConstants.GENERATION_UNDEFINED; final long otherParentGen = fakeGeneration >= GraphGenerations.FIRST_GENERATION ? fakeGeneration - 1 - : getOtherParentGossip() != null ? getOtherParentGossip().getGeneration() : -1; + : getOtherParentGossip() != null + ? getOtherParentGossip().getGeneration() + : EventConstants.GENERATION_UNDEFINED; + + final EventDescriptor selfParent = getSelfParentGossip() != null + ? new EventDescriptor( + getSelfParentGossip().getHashedData().getHash(), + creatorId, + selfParentGen, + EventConstants.BIRTH_ROUND_UNDEFINED) + : selfParentGen > EventConstants.GENERATION_UNDEFINED + ? new EventDescriptor( + RandomUtils.randomHash(random), + creatorId, + selfParentGen, + EventConstants.BIRTH_ROUND_UNDEFINED) + : null; + final EventDescriptor otherParent = getOtherParentGossip() != null + ? new EventDescriptor( + getOtherParentGossip().getHashedData().getHash(), + getOtherParentGossip().getHashedData().getCreatorId(), + otherParentGen, + EventConstants.BIRTH_ROUND_UNDEFINED) + : otherParentGen > EventConstants.GENERATION_UNDEFINED + ? new EventDescriptor( + RandomUtils.randomHash(random), + creatorId, + otherParentGen, + EventConstants.BIRTH_ROUND_UNDEFINED) + : null; final BaseEventHashedData hashedData = new BaseEventHashedData( new BasicSoftwareVersion(1), creatorId, - selfParentGen, - otherParentGen, - getSelfParentGossip() != null - ? getSelfParentGossip().getHashedData().getHash() - : null, - getOtherParentGossip() != null - ? getOtherParentGossip().getHashedData().getHash() - : null, + selfParent, + otherParent == null ? Collections.emptyList() : Collections.singletonList(otherParent), + EventConstants.BIRTH_ROUND_UNDEFINED, timestamp == null ? getParentTime().plusMillis(1 + creatorId.id()) : timestamp, tr); @@ -203,9 +229,9 @@ public GossipEvent buildGossipEvent() { random.nextBytes(sig); final BaseEventUnhashedData unhashedData = new BaseEventUnhashedData( - getOtherParentGossip() != null - ? getOtherParentGossip().getHashedData().getCreatorId() - : null, + getOtherParentGossip() == null + ? null + : getOtherParentGossip().getHashedData().getCreatorId(), sig); final GossipEvent gossipEvent = new GossipEvent(hashedData, unhashedData); gossipEvent.buildDescriptor(); diff --git a/platform-sdk/swirlds-unit-tests/core/swirlds-platform-test/src/test/java/com/swirlds/platform/test/event/GossipEventTest.java b/platform-sdk/swirlds-unit-tests/core/swirlds-platform-test/src/test/java/com/swirlds/platform/test/event/GossipEventTest.java index cf77ebd58114..ba8b173ee8f7 100644 --- a/platform-sdk/swirlds-unit-tests/core/swirlds-platform-test/src/test/java/com/swirlds/platform/test/event/GossipEventTest.java +++ b/platform-sdk/swirlds-unit-tests/core/swirlds-platform-test/src/test/java/com/swirlds/platform/test/event/GossipEventTest.java @@ -16,11 +16,14 @@ package com.swirlds.platform.test.event; +import static org.junit.jupiter.api.Assertions.assertArrayEquals; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertTrue; import com.swirlds.common.constructable.ConstructableRegistry; import com.swirlds.common.constructable.ConstructableRegistryException; +import com.swirlds.common.io.streams.SerializableDataInputStream; +import com.swirlds.common.io.streams.SerializableDataOutputStream; import com.swirlds.common.system.NodeId; import com.swirlds.common.test.fixtures.io.SerializationUtils; import com.swirlds.platform.event.GossipEvent; @@ -28,10 +31,14 @@ import com.swirlds.platform.test.fixtures.event.RandomEventUtils; import com.swirlds.platform.test.utils.EqualsVerifier; import com.swirlds.test.framework.config.TestConfigBuilder; +import java.io.ByteArrayOutputStream; +import java.io.File; +import java.io.FileInputStream; import java.io.FileNotFoundException; import java.io.IOException; import java.util.Random; import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.DisplayName; import org.junit.jupiter.api.Test; class GossipEventTest { @@ -42,7 +49,8 @@ public static void setup() throws FileNotFoundException, ConstructableRegistryEx } @Test - void test() throws IOException, ConstructableRegistryException { + @DisplayName("Serialize and deserialize event") + void serializeDeserialize() throws IOException, ConstructableRegistryException { final IndexedEvent indexedEvent = RandomEventUtils.randomEvent(new Random(), new NodeId(0), null, null); final GossipEvent gossipEvent = new GossipEvent(indexedEvent.getBaseEventHashedData(), indexedEvent.getBaseEventUnhashedData()); @@ -51,6 +59,28 @@ void test() throws IOException, ConstructableRegistryException { assertEquals(gossipEvent, copy, "deserialized version should be the same"); } + @Test + @DisplayName("Deserialize prior version of event") + void deserializePriorVersion() throws IOException, ConstructableRegistryException { + ConstructableRegistry.getInstance().registerConstructables("com.swirlds"); + final File file = new File("src/test/resources/eventFiles/eventSerializationV45/sampleGossipEvent.evts"); + final SerializableDataInputStream in = new SerializableDataInputStream(new FileInputStream(file)); + final GossipEvent gossipEvent = in.readSerializable(false, GossipEvent::new); + assertEquals(3, gossipEvent.getHashedData().getVersion()); + assertEquals(1, gossipEvent.getUnhashedData().getVersion()); + final GossipEvent copy = SerializationUtils.serializeDeserialize(gossipEvent); + assertEquals(gossipEvent, copy, "deserialized version should be the same"); + assertEquals( + gossipEvent.getHashedData().getVersion(), copy.getHashedData().getVersion()); + + final byte[] original = new FileInputStream(file).readAllBytes(); + final ByteArrayOutputStream outBytes = new ByteArrayOutputStream(); + final SerializableDataOutputStream out = new SerializableDataOutputStream(outBytes); + out.writeSerializable(gossipEvent, false); + final byte[] serialized = outBytes.toByteArray(); + assertArrayEquals(original, serialized, "serialized bytes should be the same"); + } + @Test void validateEqualsHashCode() { assertTrue(EqualsVerifier.verify(EqualsVerifier::randomGossipEvent)); diff --git a/platform-sdk/swirlds-unit-tests/core/swirlds-platform-test/src/test/java/com/swirlds/platform/test/event/tipset/ChildlessEventTrackerTests.java b/platform-sdk/swirlds-unit-tests/core/swirlds-platform-test/src/test/java/com/swirlds/platform/test/event/tipset/ChildlessEventTrackerTests.java index e273842bb739..69df60961903 100644 --- a/platform-sdk/swirlds-unit-tests/core/swirlds-platform-test/src/test/java/com/swirlds/platform/test/event/tipset/ChildlessEventTrackerTests.java +++ b/platform-sdk/swirlds-unit-tests/core/swirlds-platform-test/src/test/java/com/swirlds/platform/test/event/tipset/ChildlessEventTrackerTests.java @@ -20,6 +20,7 @@ import static com.swirlds.common.test.fixtures.RandomUtils.randomHash; import static org.junit.jupiter.api.Assertions.assertEquals; +import com.swirlds.common.crypto.Hash; import com.swirlds.common.system.NodeId; import com.swirlds.common.system.events.EventDescriptor; import com.swirlds.platform.event.creation.tipset.ChildlessEventTracker; @@ -66,7 +67,7 @@ void basicBehaviorTest() { // Adding some event with no parents final List batch1 = new ArrayList<>(); for (int i = 0; i < 10; i++) { - final EventDescriptor descriptor = new EventDescriptor(randomHash(random), new NodeId(i), 0); + final EventDescriptor descriptor = newEventDescriptor(randomHash(random), new NodeId(i), 0); tracker.addEvent(descriptor, List.of()); batch1.add(descriptor); } @@ -80,11 +81,11 @@ void basicBehaviorTest() { final List batch2 = new ArrayList<>(); for (int i = 0; i < 10; i++) { final NodeId nonExistentParentId = new NodeId(i + 100); - final EventDescriptor nonExistentParent = new EventDescriptor(randomHash(random), nonExistentParentId, 0); + final EventDescriptor nonExistentParent = newEventDescriptor(randomHash(random), nonExistentParentId, 0); final int oddParentId = (i * 2 + 1) % 10; final EventDescriptor oddParent = batch1.get(oddParentId); - final EventDescriptor descriptor = new EventDescriptor(randomHash(), new NodeId(i), 1); + final EventDescriptor descriptor = newEventDescriptor(randomHash(), new NodeId(i), 1); tracker.addEvent(descriptor, List.of(nonExistentParent, oddParent)); batch2.add(descriptor); } @@ -111,9 +112,9 @@ void branchingTest() { final ChildlessEventTracker tracker = new ChildlessEventTracker(); - final EventDescriptor e0 = new EventDescriptor(randomHash(random), new NodeId(0), 0); - final EventDescriptor e1 = new EventDescriptor(randomHash(random), new NodeId(0), 1); - final EventDescriptor e2 = new EventDescriptor(randomHash(random), new NodeId(0), 2); + final EventDescriptor e0 = newEventDescriptor(randomHash(random), new NodeId(0), 0); + final EventDescriptor e1 = newEventDescriptor(randomHash(random), new NodeId(0), 1); + final EventDescriptor e2 = newEventDescriptor(randomHash(random), new NodeId(0), 2); tracker.addEvent(e0, List.of()); tracker.addEvent(e1, List.of(e0)); @@ -123,8 +124,8 @@ void branchingTest() { assertEquals(1, batch1.size()); assertEquals(e2, batch1.get(0)); - final EventDescriptor e3 = new EventDescriptor(randomHash(random), new NodeId(0), 3); - final EventDescriptor e3Branch = new EventDescriptor(randomHash(random), new NodeId(0), 3); + final EventDescriptor e3 = newEventDescriptor(randomHash(random), new NodeId(0), 3); + final EventDescriptor e3Branch = newEventDescriptor(randomHash(random), new NodeId(0), 3); // Branch with the same generation, existing event should not be discarded. tracker.addEvent(e3, List.of(e2)); @@ -135,18 +136,30 @@ void branchingTest() { assertEquals(e3, batch2.get(0)); // Branch with a lower generation, existing event should not be discarded. - final EventDescriptor e2Branch = new EventDescriptor(randomHash(random), new NodeId(0), 2); + final EventDescriptor e2Branch = newEventDescriptor(randomHash(random), new NodeId(0), 2); tracker.addEvent(e2Branch, List.of(e1)); assertEquals(1, batch2.size()); assertEquals(e3, batch2.get(0)); // Branch with a higher generation, existing event should be discarded. - final EventDescriptor e99Branch = new EventDescriptor(randomHash(random), new NodeId(0), 99); + final EventDescriptor e99Branch = newEventDescriptor(randomHash(random), new NodeId(0), 99); tracker.addEvent(e99Branch, List.of()); final List batch3 = tracker.getChildlessEvents(); assertEquals(1, batch3.size()); assertEquals(e99Branch, batch3.get(0)); } + + /** + * Create a new event descriptor with the given parameters and -1 for the address book round. + * @param hash the hash of the event + * @param creator the creator of the event + * @param generation the generation of the event + * @return the event descriptor + */ + private EventDescriptor newEventDescriptor( + @NonNull final Hash hash, @NonNull final NodeId creator, final long generation) { + return new EventDescriptor(hash, creator, generation, -1); + } } diff --git a/platform-sdk/swirlds-unit-tests/core/swirlds-platform-test/src/test/java/com/swirlds/platform/test/event/tipset/TipsetEventCreatorTests.java b/platform-sdk/swirlds-unit-tests/core/swirlds-platform-test/src/test/java/com/swirlds/platform/test/event/tipset/TipsetEventCreatorTests.java index 62f3aa38e37d..37015bd87248 100644 --- a/platform-sdk/swirlds-unit-tests/core/swirlds-platform-test/src/test/java/com/swirlds/platform/test/event/tipset/TipsetEventCreatorTests.java +++ b/platform-sdk/swirlds-unit-tests/core/swirlds-platform-test/src/test/java/com/swirlds/platform/test/event/tipset/TipsetEventCreatorTests.java @@ -42,6 +42,7 @@ import com.swirlds.common.system.address.AddressBook; import com.swirlds.common.system.events.BaseEventHashedData; import com.swirlds.common.system.events.BaseEventUnhashedData; +import com.swirlds.common.system.events.EventConstants; import com.swirlds.common.system.events.EventDescriptor; import com.swirlds.common.system.transaction.internal.ConsensusTransactionImpl; import com.swirlds.common.system.transaction.internal.SwirldTransaction; @@ -79,7 +80,7 @@ class TipsetEventCreatorTests { /** * @param nodeId the node ID of the simulated node * @param tipsetTracker tracks tipsets of events - * @param eventCreator the event creator for the simulated node + * @param eventCreator the event creator for the simulated node * @param tipsetWeightCalculator used to sanity check event creation logic */ private record SimulatedNode( @@ -152,9 +153,11 @@ private void validateNewEvent( final boolean slowNode) { final EventImpl selfParent = events.get(newEvent.getHashedData().getSelfParentHash()); - final long selfParentGeneration = selfParent == null ? -1 : selfParent.getGeneration(); + final long selfParentGeneration = + selfParent == null ? EventConstants.GENERATION_UNDEFINED : selfParent.getGeneration(); final EventImpl otherParent = events.get(newEvent.getHashedData().getOtherParentHash()); - final long otherParentGeneration = otherParent == null ? -1 : otherParent.getGeneration(); + final long otherParentGeneration = + otherParent == null ? EventConstants.GENERATION_UNDEFINED : otherParent.getGeneration(); if (selfParent == null) { // The only legal time to have a null self parent is genesis. @@ -709,7 +712,11 @@ private EventImpl createMockEvent( when(hashedData.getHash()).thenReturn(hash); when(event.getBaseHash()).thenReturn(hash); + when(hashedData.createEventDescriptor()) + .thenReturn(new EventDescriptor(hash, creator, generation, -EventConstants.BIRTH_ROUND_UNDEFINED)); + when(event.getHashedData()).thenReturn(hashedData); + when(event.getBaseEventHashedData()).thenReturn(hashedData); final BaseEventUnhashedData unhashedData = mock(BaseEventUnhashedData.class); when(unhashedData.getOtherId()).thenReturn(otherParentId); @@ -750,9 +757,12 @@ void frozenEventCreationBug() { final GossipEvent eventA1 = eventCreator.maybeCreateEvent(); assertNotNull(eventA1); - final EventImpl eventB1 = createMockEvent(random, nodeB, -1, null, -1); - final EventImpl eventC1 = createMockEvent(random, nodeC, -1, null, -1); - final EventImpl eventD1 = createMockEvent(random, nodeD, -1, null, -1); + final EventImpl eventB1 = createMockEvent( + random, nodeB, EventConstants.GENERATION_UNDEFINED, null, EventConstants.GENERATION_UNDEFINED); + final EventImpl eventC1 = createMockEvent( + random, nodeC, EventConstants.GENERATION_UNDEFINED, null, EventConstants.GENERATION_UNDEFINED); + final EventImpl eventD1 = createMockEvent( + random, nodeD, EventConstants.GENERATION_UNDEFINED, null, EventConstants.GENERATION_UNDEFINED); eventCreator.registerEvent(eventB1); eventCreator.registerEvent(eventC1); @@ -820,10 +830,14 @@ void notRegisteringEventsFromNodesNotInAddressBook() { final GossipEvent eventA1 = eventCreator.maybeCreateEvent(); assertNotNull(eventA1); - final EventImpl eventB1 = createMockEvent(random, nodeB, -1, null, -1); - final EventImpl eventC1 = createMockEvent(random, nodeC, -1, null, -1); - final EventImpl eventD1 = createMockEvent(random, nodeD, -1, null, -1); - final EventImpl eventE1 = createMockEvent(random, nodeE, -1, null, -1); + final EventImpl eventB1 = createMockEvent( + random, nodeB, EventConstants.GENERATION_UNDEFINED, null, EventConstants.GENERATION_UNDEFINED); + final EventImpl eventC1 = createMockEvent( + random, nodeC, EventConstants.GENERATION_UNDEFINED, null, EventConstants.GENERATION_UNDEFINED); + final EventImpl eventD1 = createMockEvent( + random, nodeD, EventConstants.GENERATION_UNDEFINED, null, EventConstants.GENERATION_UNDEFINED); + final EventImpl eventE1 = createMockEvent( + random, nodeE, EventConstants.GENERATION_UNDEFINED, null, EventConstants.GENERATION_UNDEFINED); eventCreator.registerEvent(eventB1); eventCreator.registerEvent(eventC1); diff --git a/platform-sdk/swirlds-unit-tests/core/swirlds-platform-test/src/test/java/com/swirlds/platform/test/event/tipset/TipsetTrackerTests.java b/platform-sdk/swirlds-unit-tests/core/swirlds-platform-test/src/test/java/com/swirlds/platform/test/event/tipset/TipsetTrackerTests.java index 2fc4bc2e7d8d..3ba9dd4b57f7 100644 --- a/platform-sdk/swirlds-unit-tests/core/swirlds-platform-test/src/test/java/com/swirlds/platform/test/event/tipset/TipsetTrackerTests.java +++ b/platform-sdk/swirlds-unit-tests/core/swirlds-platform-test/src/test/java/com/swirlds/platform/test/event/tipset/TipsetTrackerTests.java @@ -27,6 +27,7 @@ import com.swirlds.common.system.NodeId; import com.swirlds.common.system.address.Address; import com.swirlds.common.system.address.AddressBook; +import com.swirlds.common.system.events.EventConstants; import com.swirlds.common.system.events.EventDescriptor; import com.swirlds.common.test.fixtures.RandomAddressBookGenerator; import com.swirlds.platform.event.creation.tipset.Tipset; @@ -81,7 +82,8 @@ void basicBehaviorTest() { } final EventDescriptor selfParent = latestEvents.get(creator); - final EventDescriptor fingerprint = new EventDescriptor(randomHash(random), creator, generation); + final EventDescriptor fingerprint = + new EventDescriptor(randomHash(random), creator, generation, EventConstants.BIRTH_ROUND_UNDEFINED); latestEvents.put(creator, fingerprint); // Select some nodes we'd like to be our parents. diff --git a/platform-sdk/swirlds-unit-tests/core/swirlds-platform-test/src/test/java/com/swirlds/platform/test/event/tipset/TipsetWeightCalculatorTests.java b/platform-sdk/swirlds-unit-tests/core/swirlds-platform-test/src/test/java/com/swirlds/platform/test/event/tipset/TipsetWeightCalculatorTests.java index e27acbf28656..0bfaebb40fe3 100644 --- a/platform-sdk/swirlds-unit-tests/core/swirlds-platform-test/src/test/java/com/swirlds/platform/test/event/tipset/TipsetWeightCalculatorTests.java +++ b/platform-sdk/swirlds-unit-tests/core/swirlds-platform-test/src/test/java/com/swirlds/platform/test/event/tipset/TipsetWeightCalculatorTests.java @@ -30,9 +30,11 @@ import com.swirlds.base.time.Time; import com.swirlds.common.context.PlatformContext; +import com.swirlds.common.crypto.Hash; import com.swirlds.common.system.NodeId; import com.swirlds.common.system.address.Address; import com.swirlds.common.system.address.AddressBook; +import com.swirlds.common.system.events.EventConstants; import com.swirlds.common.system.events.EventDescriptor; import com.swirlds.common.test.fixtures.RandomAddressBookGenerator; import com.swirlds.common.test.fixtures.RandomAddressBookGenerator.WeightDistributionStrategy; @@ -42,6 +44,7 @@ import com.swirlds.platform.event.creation.tipset.TipsetTracker; import com.swirlds.platform.event.creation.tipset.TipsetWeightCalculator; import com.swirlds.test.framework.context.TestPlatformContextBuilder; +import edu.umd.cs.findbugs.annotations.NonNull; import java.util.ArrayList; import java.util.HashMap; import java.util.HashSet; @@ -55,6 +58,18 @@ @DisplayName("TipsetWeightCalculator Tests") class TipsetWeightCalculatorTests { + /** + * Create a new event descriptor with the given parameters and {@link EventConstants#BIRTH_ROUND_UNDEFINED} for the birth round. + * @param hash the hash of the event + * @param creator the creator of the event + * @param generation the generation of the event + * @return the event descriptor + */ + private EventDescriptor newEventDescriptor( + @NonNull final Hash hash, @NonNull final NodeId creator, final long generation) { + return new EventDescriptor(hash, creator, generation, EventConstants.BIRTH_ROUND_UNDEFINED); + } + @Test @DisplayName("Basic Behavior Test") void basicBehaviorTest() { @@ -97,7 +112,7 @@ void basicBehaviorTest() { } final EventDescriptor selfParent = latestEvents.get(creator); - final EventDescriptor fingerprint = new EventDescriptor(randomHash(random), creator, generation); + final EventDescriptor fingerprint = newEventDescriptor(randomHash(random), creator, generation); latestEvents.put(creator, fingerprint); // Select some nodes we'd like to be our parents. @@ -217,16 +232,16 @@ void selfishNodeTest() { final Tipset snapshot1 = calculator.getSnapshot(); // Each node creates an event. - final EventDescriptor eventA1 = new EventDescriptor(randomHash(random), nodeA, 1); + final EventDescriptor eventA1 = newEventDescriptor(randomHash(random), nodeA, 1); tracker.addEvent(eventA1, List.of()); childlessEventTracker.addEvent(eventA1, List.of()); - final EventDescriptor eventB1 = new EventDescriptor(randomHash(random), nodeB, 1); + final EventDescriptor eventB1 = newEventDescriptor(randomHash(random), nodeB, 1); tracker.addEvent(eventB1, List.of()); childlessEventTracker.addEvent(eventB1, List.of()); - final EventDescriptor eventC1 = new EventDescriptor(randomHash(random), nodeC, 1); + final EventDescriptor eventC1 = newEventDescriptor(randomHash(random), nodeC, 1); tracker.addEvent(eventC1, List.of()); childlessEventTracker.addEvent(eventC1, List.of()); - final EventDescriptor eventD1 = new EventDescriptor(randomHash(random), nodeD, 1); + final EventDescriptor eventD1 = newEventDescriptor(randomHash(random), nodeD, 1); tracker.addEvent(eventD1, List.of()); childlessEventTracker.addEvent(eventD1, List.of()); @@ -235,16 +250,16 @@ void selfishNodeTest() { assertSame(snapshot1, calculator.getSnapshot()); // Each node creates another event. All nodes use all available other parents except the event from D. - final EventDescriptor eventA2 = new EventDescriptor(randomHash(random), nodeA, 2); + final EventDescriptor eventA2 = newEventDescriptor(randomHash(random), nodeA, 2); tracker.addEvent(eventA2, List.of(eventA1, eventB1, eventC1)); childlessEventTracker.addEvent(eventA2, List.of(eventA1, eventB1, eventC1)); - final EventDescriptor eventB2 = new EventDescriptor(randomHash(random), nodeB, 2); + final EventDescriptor eventB2 = newEventDescriptor(randomHash(random), nodeB, 2); tracker.addEvent(eventB2, List.of(eventA1, eventB1, eventC1)); childlessEventTracker.addEvent(eventB2, List.of(eventA1, eventB1, eventC1)); - final EventDescriptor eventC2 = new EventDescriptor(randomHash(random), nodeC, 2); + final EventDescriptor eventC2 = newEventDescriptor(randomHash(random), nodeC, 2); tracker.addEvent(eventC2, List.of(eventA1, eventB1, eventC1)); childlessEventTracker.addEvent(eventC2, List.of(eventA1, eventB1, eventC1)); - final EventDescriptor eventD2 = new EventDescriptor(randomHash(random), nodeD, 2); + final EventDescriptor eventD2 = newEventDescriptor(randomHash(random), nodeD, 2); tracker.addEvent(eventD2, List.of(eventA1, eventB1, eventC1, eventD1)); childlessEventTracker.addEvent(eventD2, List.of(eventA1, eventB1, eventC1, eventD1)); @@ -265,16 +280,16 @@ void selfishNodeTest() { assertEquals(1, calculator.getMaxSelfishnessScore()); // Create another batch of events where D is bullied. - final EventDescriptor eventA3 = new EventDescriptor(randomHash(random), nodeA, 3); + final EventDescriptor eventA3 = newEventDescriptor(randomHash(random), nodeA, 3); tracker.addEvent(eventA3, List.of(eventA2, eventB2, eventC2)); childlessEventTracker.addEvent(eventA3, List.of(eventA2, eventB2, eventC2)); - final EventDescriptor eventB3 = new EventDescriptor(randomHash(random), nodeB, 3); + final EventDescriptor eventB3 = newEventDescriptor(randomHash(random), nodeB, 3); tracker.addEvent(eventB3, List.of(eventA2, eventB2, eventC2)); childlessEventTracker.addEvent(eventB3, List.of(eventA2, eventB2, eventC2)); - final EventDescriptor eventC3 = new EventDescriptor(randomHash(random), nodeC, 3); + final EventDescriptor eventC3 = newEventDescriptor(randomHash(random), nodeC, 3); tracker.addEvent(eventC3, List.of(eventA2, eventB2, eventC2)); childlessEventTracker.addEvent(eventC3, List.of(eventA2, eventB2, eventC2)); - final EventDescriptor eventD3 = new EventDescriptor(randomHash(random), nodeD, 3); + final EventDescriptor eventD3 = newEventDescriptor(randomHash(random), nodeD, 3); tracker.addEvent(eventD3, List.of(eventA2, eventB2, eventC2, eventD2)); childlessEventTracker.addEvent(eventD3, List.of(eventA2, eventB2, eventC2, eventD2)); @@ -294,16 +309,16 @@ void selfishNodeTest() { assertEquals(2, calculator.getMaxSelfishnessScore()); // Create a bach of events that don't ignore D. Let's all ignore C, because C is a jerk. - final EventDescriptor eventA4 = new EventDescriptor(randomHash(random), nodeA, 4); + final EventDescriptor eventA4 = newEventDescriptor(randomHash(random), nodeA, 4); tracker.addEvent(eventA4, List.of(eventA3, eventB3, eventD3)); childlessEventTracker.addEvent(eventA4, List.of(eventA3, eventB3, eventD3)); - final EventDescriptor eventB4 = new EventDescriptor(randomHash(random), nodeB, 4); + final EventDescriptor eventB4 = newEventDescriptor(randomHash(random), nodeB, 4); tracker.addEvent(eventB4, List.of(eventA3, eventB3, eventD3)); childlessEventTracker.addEvent(eventB4, List.of(eventA3, eventB3, eventD3)); - final EventDescriptor eventC4 = new EventDescriptor(randomHash(random), nodeC, 4); + final EventDescriptor eventC4 = newEventDescriptor(randomHash(random), nodeC, 4); tracker.addEvent(eventC4, List.of(eventA3, eventB3, eventC3, eventD3)); childlessEventTracker.addEvent(eventC4, List.of(eventA3, eventB3, eventC3, eventD3)); - final EventDescriptor eventD4 = new EventDescriptor(randomHash(random), nodeD, 4); + final EventDescriptor eventD4 = newEventDescriptor(randomHash(random), nodeD, 4); tracker.addEvent(eventD4, List.of(eventA3, eventB3, eventD3)); childlessEventTracker.addEvent(eventD4, List.of(eventA3, eventB3, eventD3)); @@ -323,13 +338,13 @@ void selfishNodeTest() { assertEquals(1, calculator.getMaxSelfishnessScore()); // Stop ignoring C. D stops creating events. - final EventDescriptor eventA5 = new EventDescriptor(randomHash(random), nodeA, 5); + final EventDescriptor eventA5 = newEventDescriptor(randomHash(random), nodeA, 5); tracker.addEvent(eventA5, List.of(eventA4, eventB4, eventC4, eventD4)); childlessEventTracker.addEvent(eventA5, List.of(eventA4, eventB4, eventC4, eventD4)); - final EventDescriptor eventB5 = new EventDescriptor(randomHash(random), nodeB, 5); + final EventDescriptor eventB5 = newEventDescriptor(randomHash(random), nodeB, 5); tracker.addEvent(eventB5, List.of(eventA4, eventB4, eventC4, eventD4)); childlessEventTracker.addEvent(eventB5, List.of(eventA4, eventB4, eventC4, eventD4)); - final EventDescriptor eventC5 = new EventDescriptor(randomHash(random), nodeC, 5); + final EventDescriptor eventC5 = newEventDescriptor(randomHash(random), nodeC, 5); tracker.addEvent(eventC5, List.of(eventA4, eventB4, eventC4, eventD4)); childlessEventTracker.addEvent(eventC5, List.of(eventA4, eventB4, eventC4, eventD4)); @@ -349,13 +364,13 @@ void selfishNodeTest() { // D still is not creating events. Since there is no legal event from D to use as a parent, this doesn't // count as being selfish. - final EventDescriptor eventA6 = new EventDescriptor(randomHash(random), nodeA, 6); + final EventDescriptor eventA6 = newEventDescriptor(randomHash(random), nodeA, 6); tracker.addEvent(eventA6, List.of(eventA5, eventB5, eventC5)); childlessEventTracker.addEvent(eventA6, List.of(eventA5, eventB5, eventC5)); - final EventDescriptor eventB6 = new EventDescriptor(randomHash(random), nodeB, 6); + final EventDescriptor eventB6 = newEventDescriptor(randomHash(random), nodeB, 6); tracker.addEvent(eventB6, List.of(eventA5, eventB5, eventC5)); childlessEventTracker.addEvent(eventB6, List.of(eventA5, eventB5, eventC5)); - final EventDescriptor eventC6 = new EventDescriptor(randomHash(random), nodeC, 6); + final EventDescriptor eventC6 = newEventDescriptor(randomHash(random), nodeC, 6); tracker.addEvent(eventC6, List.of(eventA5, eventB5, eventC5)); childlessEventTracker.addEvent(eventC6, List.of(eventA5, eventB5, eventC5)); @@ -374,13 +389,13 @@ void selfishNodeTest() { assertEquals(0, calculator.getMaxSelfishnessScore()); // Rinse and repeat. - final EventDescriptor eventA7 = new EventDescriptor(randomHash(random), nodeA, 7); + final EventDescriptor eventA7 = newEventDescriptor(randomHash(random), nodeA, 7); tracker.addEvent(eventA7, List.of(eventA6, eventB6, eventC6)); childlessEventTracker.addEvent(eventA7, List.of(eventA6, eventB6, eventC6)); - final EventDescriptor eventB7 = new EventDescriptor(randomHash(random), nodeB, 7); + final EventDescriptor eventB7 = newEventDescriptor(randomHash(random), nodeB, 7); tracker.addEvent(eventB7, List.of(eventA6, eventB6, eventC6)); childlessEventTracker.addEvent(eventB7, List.of(eventA6, eventB6, eventC6)); - final EventDescriptor eventC7 = new EventDescriptor(randomHash(random), nodeC, 7); + final EventDescriptor eventC7 = newEventDescriptor(randomHash(random), nodeC, 7); tracker.addEvent(eventC7, List.of(eventA6, eventB6, eventC6)); childlessEventTracker.addEvent(eventC7, List.of(eventA6, eventB6, eventC6)); @@ -431,13 +446,13 @@ void zeroWeightNodeTest() { final Tipset snapshot1 = calculator.getSnapshot(); // Each node creates an event. - final EventDescriptor eventA1 = new EventDescriptor(randomHash(random), nodeA, 1); + final EventDescriptor eventA1 = newEventDescriptor(randomHash(random), nodeA, 1); builder.addEvent(eventA1, List.of()); - final EventDescriptor eventB1 = new EventDescriptor(randomHash(random), nodeB, 1); + final EventDescriptor eventB1 = newEventDescriptor(randomHash(random), nodeB, 1); builder.addEvent(eventB1, List.of()); - final EventDescriptor eventC1 = new EventDescriptor(randomHash(random), nodeC, 1); + final EventDescriptor eventC1 = newEventDescriptor(randomHash(random), nodeC, 1); builder.addEvent(eventC1, List.of()); - final EventDescriptor eventD1 = new EventDescriptor(randomHash(random), nodeD, 1); + final EventDescriptor eventD1 = newEventDescriptor(randomHash(random), nodeD, 1); builder.addEvent(eventD1, List.of()); assertEquals(ZERO_ADVANCEMENT_WEIGHT, calculator.getTheoreticalAdvancementWeight(List.of())); @@ -445,7 +460,7 @@ void zeroWeightNodeTest() { assertSame(snapshot1, calculator.getSnapshot()); // Create a node "on top of" B1. - final EventDescriptor eventA2 = new EventDescriptor(randomHash(random), nodeA, 2); + final EventDescriptor eventA2 = newEventDescriptor(randomHash(random), nodeA, 2); builder.addEvent(eventA2, List.of(eventA1, eventB1)); final TipsetAdvancementWeight advancement1 = calculator.addEventAndGetAdvancementWeight(eventA2); assertEquals(TipsetAdvancementWeight.of(1, 0), advancement1); @@ -456,7 +471,7 @@ void zeroWeightNodeTest() { // If we get 1 more advancement point then the snapshot will advance. But building // on top of a zero stake node will not contribute to this and the snapshot will not // advance. Build on top of node D. - final EventDescriptor eventA3 = new EventDescriptor(randomHash(random), nodeA, 3); + final EventDescriptor eventA3 = newEventDescriptor(randomHash(random), nodeA, 3); builder.addEvent(eventA3, List.of(eventA2, eventD1)); final TipsetAdvancementWeight advancement2 = calculator.addEventAndGetAdvancementWeight(eventA3); assertEquals(TipsetAdvancementWeight.of(0, 1), advancement2); @@ -465,7 +480,7 @@ void zeroWeightNodeTest() { assertSame(snapshot1, calculator.getSnapshot()); // Now, build on top of C. This should push us into the next snapshot. - final EventDescriptor eventA4 = new EventDescriptor(randomHash(random), nodeA, 4); + final EventDescriptor eventA4 = newEventDescriptor(randomHash(random), nodeA, 4); builder.addEvent(eventA4, List.of(eventA3, eventC1)); final TipsetAdvancementWeight advancement3 = calculator.addEventAndGetAdvancementWeight(eventA4); assertEquals(TipsetAdvancementWeight.of(1, 0), advancement3); @@ -501,21 +516,21 @@ void ancientParentTest() { platformContext, Time.getCurrent(), addressBook, nodeA, builder, childlessEventTracker); // Create generation 1 events. - final EventDescriptor eventA1 = new EventDescriptor(randomHash(random), nodeA, 1); + final EventDescriptor eventA1 = newEventDescriptor(randomHash(random), nodeA, 1); builder.addEvent(eventA1, List.of()); - final EventDescriptor eventB1 = new EventDescriptor(randomHash(random), nodeB, 1); + final EventDescriptor eventB1 = newEventDescriptor(randomHash(random), nodeB, 1); builder.addEvent(eventB1, List.of()); - final EventDescriptor eventC1 = new EventDescriptor(randomHash(random), nodeC, 1); + final EventDescriptor eventC1 = newEventDescriptor(randomHash(random), nodeC, 1); builder.addEvent(eventC1, List.of()); - final EventDescriptor eventD1 = new EventDescriptor(randomHash(random), nodeD, 1); + final EventDescriptor eventD1 = newEventDescriptor(randomHash(random), nodeD, 1); builder.addEvent(eventD1, List.of()); // Create some generation 2 events. A does not create an event yet. - final EventDescriptor eventB2 = new EventDescriptor(randomHash(random), nodeB, 2); + final EventDescriptor eventB2 = newEventDescriptor(randomHash(random), nodeB, 2); builder.addEvent(eventB2, List.of(eventA1, eventB1, eventC1, eventD1)); - final EventDescriptor eventC2 = new EventDescriptor(randomHash(random), nodeC, 2); + final EventDescriptor eventC2 = newEventDescriptor(randomHash(random), nodeC, 2); builder.addEvent(eventC2, List.of(eventA1, eventB1, eventC1, eventD1)); - final EventDescriptor eventD2 = new EventDescriptor(randomHash(random), nodeD, 2); + final EventDescriptor eventD2 = newEventDescriptor(randomHash(random), nodeD, 2); builder.addEvent(eventD2, List.of(eventA1, eventB1, eventC1, eventD1)); // Mark generation 1 as ancient. @@ -531,7 +546,7 @@ void ancientParentTest() { // Including generation 1 events as parents shouldn't cause us to throw. (Angry log messages are ok). assertDoesNotThrow(() -> { calculator.getTheoreticalAdvancementWeight(List.of(eventA1, eventB2, eventC2, eventD1)); - final EventDescriptor eventA2 = new EventDescriptor(randomHash(random), nodeA, 2); + final EventDescriptor eventA2 = newEventDescriptor(randomHash(random), nodeA, 2); builder.addEvent(eventA2, List.of(eventA1, eventB2, eventC2, eventD1)); calculator.addEventAndGetAdvancementWeight(eventA2); }); diff --git a/platform-sdk/swirlds-unit-tests/core/swirlds-platform-test/src/test/java/com/swirlds/platform/test/event/validation/EventValidatorTests.java b/platform-sdk/swirlds-unit-tests/core/swirlds-platform-test/src/test/java/com/swirlds/platform/test/event/validation/EventValidatorTests.java index 46b7ed4a9cd7..be6a523902bd 100644 --- a/platform-sdk/swirlds-unit-tests/core/swirlds-platform-test/src/test/java/com/swirlds/platform/test/event/validation/EventValidatorTests.java +++ b/platform-sdk/swirlds-unit-tests/core/swirlds-platform-test/src/test/java/com/swirlds/platform/test/event/validation/EventValidatorTests.java @@ -29,10 +29,11 @@ import com.swirlds.common.system.BasicSoftwareVersion; import com.swirlds.common.system.NodeId; import com.swirlds.common.system.events.BaseEventHashedData; +import com.swirlds.common.system.events.EventConstants; +import com.swirlds.common.system.events.EventDescriptor; import com.swirlds.common.system.transaction.Transaction; import com.swirlds.common.system.transaction.internal.ConsensusTransactionImpl; import com.swirlds.common.test.fixtures.RandomUtils; -import com.swirlds.platform.event.EventConstants; import com.swirlds.platform.event.GossipEvent; import com.swirlds.platform.event.validation.EventDeduplication; import com.swirlds.platform.event.validation.EventValidator; @@ -45,6 +46,7 @@ import com.swirlds.platform.metrics.EventIntakeMetrics; import com.swirlds.platform.test.event.GossipEventBuilder; import java.time.Instant; +import java.util.Collections; import java.util.HashSet; import java.util.List; import java.util.Set; @@ -67,14 +69,19 @@ private static GossipEvent eventWithParents( final long creatorId = 0; final Instant timeCreated = Instant.now(); final ConsensusTransactionImpl[] transactions = new ConsensusTransactionImpl[0]; + final NodeId selfId = new NodeId(creatorId); + final NodeId otherId = new NodeId(0); + final EventDescriptor selfParent = + selfParentHash == null ? null : new EventDescriptor(selfParentHash, selfId, selfParentGen, -1); + final EventDescriptor otherParent = + otherParentHash == null ? null : new EventDescriptor(otherParentHash, otherId, otherParentGen, -1); Mockito.when(event.getHashedData()) .thenReturn(new BaseEventHashedData( new BasicSoftwareVersion(1), - new NodeId(creatorId), - selfParentGen, - otherParentGen, - selfParentHash, - otherParentHash, + selfId, + selfParent, + otherParent == null ? Collections.emptyList() : Collections.singletonList(otherParent), + EventConstants.BIRTH_ROUND_UNDEFINED, timeCreated, transactions)); return event; @@ -177,10 +184,6 @@ void parentValidity() { final GossipEventValidator validator = StaticValidators.buildParentValidator(10); - // has generation but no hash - assertFalse(validator.isEventValid(eventWithParents(validGeneration, undefinedGeneration, nullHash, nullHash))); - assertFalse(validator.isEventValid(eventWithParents(undefinedGeneration, validGeneration, nullHash, nullHash))); - // has hash but no generation assertFalse( validator.isEventValid(eventWithParents(undefinedGeneration, undefinedGeneration, hash1, nullHash))); @@ -210,10 +213,6 @@ void parentValiditySizeOneNetwork() { final GossipEventValidator validator = StaticValidators.buildParentValidator(1); - // has generation but no hash - assertFalse(validator.isEventValid(eventWithParents(validGeneration, undefinedGeneration, nullHash, nullHash))); - assertFalse(validator.isEventValid(eventWithParents(undefinedGeneration, validGeneration, nullHash, nullHash))); - // has hash but no generation assertFalse( validator.isEventValid(eventWithParents(undefinedGeneration, undefinedGeneration, hash1, nullHash))); diff --git a/platform-sdk/swirlds-unit-tests/core/swirlds-platform-test/src/test/java/com/swirlds/platform/test/sync/EventFactory.java b/platform-sdk/swirlds-unit-tests/core/swirlds-platform-test/src/test/java/com/swirlds/platform/test/sync/EventFactory.java index 28e9b846552c..b2470e8bba09 100644 --- a/platform-sdk/swirlds-unit-tests/core/swirlds-platform-test/src/test/java/com/swirlds/platform/test/sync/EventFactory.java +++ b/platform-sdk/swirlds-unit-tests/core/swirlds-platform-test/src/test/java/com/swirlds/platform/test/sync/EventFactory.java @@ -22,11 +22,12 @@ import com.swirlds.common.system.NodeId; import com.swirlds.common.system.events.BaseEventHashedData; import com.swirlds.common.system.events.BaseEventUnhashedData; -import com.swirlds.platform.event.EventConstants; +import com.swirlds.common.system.events.EventConstants; +import com.swirlds.common.system.events.EventDescriptor; import com.swirlds.platform.gossip.shadowgraph.ShadowEvent; import com.swirlds.platform.internal.EventImpl; import java.time.Instant; -import java.util.Random; +import java.util.Collections; /** * A simple, deterministic factory for Event instances @@ -52,19 +53,25 @@ public static EventImpl makeEventWithRandomHash() { } public static EventImpl makeEvent(final EventImpl selfParent, final EventImpl otherParent) { + final NodeId selfId = selfParent != null ? selfParent.getCreatorId() : new NodeId(0); + final NodeId otherId = otherParent != null ? otherParent.getCreatorId() : new NodeId(0); + final EventDescriptor selfDescriptor = selfParent == null + ? null + : new EventDescriptor(selfParent.getBaseHash(), selfId, selfParent.getGeneration(), -1); + final EventDescriptor otherDescriptor = otherParent == null + ? null + : new EventDescriptor(otherParent.getBaseHash(), otherId, otherParent.getGeneration(), -1); final BaseEventHashedData hashedEventData = new BaseEventHashedData( new BasicSoftwareVersion(1), - new NodeId(selfParent != null ? selfParent.getCreatorId().id() : 0), - (selfParent != null ? selfParent.getGeneration() : EventConstants.GENERATION_UNDEFINED), - (otherParent != null ? otherParent.getGeneration() : EventConstants.GENERATION_UNDEFINED), - (selfParent != null ? selfParent.getBaseHash() : null), - (otherParent != null ? otherParent.getBaseHash() : null), + selfId, + selfDescriptor, + otherDescriptor == null ? Collections.emptyList() : Collections.singletonList(otherDescriptor), + EventConstants.BIRTH_ROUND_UNDEFINED, Instant.EPOCH, null); - final BaseEventUnhashedData unhashedEventData = new BaseEventUnhashedData( - otherParent != null ? otherParent.getCreatorId() : new NodeId(Math.abs(new Random().nextLong())), - HashGenerator.random().getValue()); + final BaseEventUnhashedData unhashedEventData = + new BaseEventUnhashedData(otherId, HashGenerator.random().getValue()); final EventImpl e = new EventImpl(hashedEventData, unhashedEventData, selfParent, otherParent); e.getBaseEventHashedData().setHash(HashGenerator.random()); diff --git a/platform-sdk/swirlds-unit-tests/core/swirlds-platform-test/src/test/java/com/swirlds/platform/test/sync/SyncTests.java b/platform-sdk/swirlds-unit-tests/core/swirlds-platform-test/src/test/java/com/swirlds/platform/test/sync/SyncTests.java index c908779a7b77..0e73499af1a4 100644 --- a/platform-sdk/swirlds-unit-tests/core/swirlds-platform-test/src/test/java/com/swirlds/platform/test/sync/SyncTests.java +++ b/platform-sdk/swirlds-unit-tests/core/swirlds-platform-test/src/test/java/com/swirlds/platform/test/sync/SyncTests.java @@ -29,13 +29,13 @@ import com.swirlds.common.constructable.ConstructableRegistry; import com.swirlds.common.constructable.ConstructableRegistryException; import com.swirlds.common.system.NodeId; +import com.swirlds.common.system.events.EventConstants; import com.swirlds.common.test.fixtures.threading.ReplaceSyncPhaseParallelExecutor; import com.swirlds.common.test.fixtures.threading.SyncPhaseParallelExecutor; import com.swirlds.common.threading.pool.CachedPoolParallelExecutor; import com.swirlds.common.threading.pool.ParallelExecutionException; import com.swirlds.common.threading.pool.ParallelExecutor; import com.swirlds.platform.consensus.GraphGenerations; -import com.swirlds.platform.event.EventConstants; import com.swirlds.platform.gossip.shadowgraph.ShadowEvent; import com.swirlds.platform.internal.EventImpl; import com.swirlds.platform.test.event.emitter.EventEmitterFactory; diff --git a/platform-sdk/swirlds-unit-tests/core/swirlds-platform-test/src/test/resources/eventFiles/eventSerializationV45/sampleGossipEvent.evts b/platform-sdk/swirlds-unit-tests/core/swirlds-platform-test/src/test/resources/eventFiles/eventSerializationV45/sampleGossipEvent.evts new file mode 100644 index 0000000000000000000000000000000000000000..530f5f39aea5246eda4bfb1ce3bc37b0e5a7d277 GIT binary patch literal 947 zcmZQzU|?ckU|=qD~V$dg|t=WzAjM32Sm?wFyOEp}L__e$Q| zrr8^EUFOf2x;%1+XKm%XIy6-`g2|Mm1wiqky#M}2lu?RvU|z9_ionDQ=jS% z%;uVQA|2^X|uju0Yla6hB_i)}RY53mwTOt2g%{k_< z-#XFNeHB9Y6B6ED`o^Yv>}7;)@`;%uMauG59aAdhm3ML9n`o~1kA-=E+{4=ouebH_ zFZjQ>dUKe`#qyWGi?{F7c% zwI7~R_xN)xNk6_`|GTMr*SqlRq9L=_AMsT9`%wRR<(+~XN$GypYaRGjzZOybx=t+A z@N>0Y{~p;!wVmN}UR&0!RpR|Vxv}x%-TG%LFLv*($v<`9$?2+RiLcrXQ;n7qt~p;s n<(1~fly3?QT)6whhk1)nuJkSo`RTp5a2hZIP1dp3-+BxHQFqVh literal 0 HcmV?d00001 From f8f0317550e626e9c163ca7812b155b40337efa1 Mon Sep 17 00:00:00 2001 From: David Bakin <117694041+david-bakin-sl@users.noreply.github.com> Date: Wed, 6 Dec 2023 15:53:07 -0800 Subject: [PATCH 2/7] fix(migration): Migrate contract slots from mono- to modular- in deterministic order (#10310) Nodes added to the merkle tree _must be added in deterministic order_. Especially for the migrators, which add so many nodes they cross _commit_ boundaries. Remove the concurrent sinking of contract slots; replace with accumulating all `fromState` slots into a concurrent queue, then placing them in an array, and then sorting them. Fixes #10293 Signed-off-by: David S Bakin <117694041+david-bakin-sl@users.noreply.github.com> --- .../BrokenTransformationException.java | 32 +++ .../migration/ContractStateMigrator.java | 263 +++++++++--------- 2 files changed, 159 insertions(+), 136 deletions(-) create mode 100644 hedera-node/hedera-mono-service/src/main/java/com/hedera/node/app/service/mono/state/migration/BrokenTransformationException.java diff --git a/hedera-node/hedera-mono-service/src/main/java/com/hedera/node/app/service/mono/state/migration/BrokenTransformationException.java b/hedera-node/hedera-mono-service/src/main/java/com/hedera/node/app/service/mono/state/migration/BrokenTransformationException.java new file mode 100644 index 000000000000..1a71d11748ff --- /dev/null +++ b/hedera-node/hedera-mono-service/src/main/java/com/hedera/node/app/service/mono/state/migration/BrokenTransformationException.java @@ -0,0 +1,32 @@ +/* + * Copyright (C) 2023 Hedera Hashgraph, LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.hedera.node.app.service.mono.state.migration; + +import edu.umd.cs.findbugs.annotations.NonNull; +import java.util.concurrent.CompletionException; + +/** Exception to indicate the migration failed. */ +class BrokenTransformationException extends CompletionException { + + public BrokenTransformationException(@NonNull final String message) { + super(message); + } + + public BrokenTransformationException(@NonNull final String message, @NonNull final Throwable t) { + super(message, t); + } +} diff --git a/hedera-node/hedera-mono-service/src/main/java/com/hedera/node/app/service/mono/state/migration/ContractStateMigrator.java b/hedera-node/hedera-mono-service/src/main/java/com/hedera/node/app/service/mono/state/migration/ContractStateMigrator.java index 4f4f1df7b0dc..89ebdf1fb136 100644 --- a/hedera-node/hedera-mono-service/src/main/java/com/hedera/node/app/service/mono/state/migration/ContractStateMigrator.java +++ b/hedera-node/hedera-mono-service/src/main/java/com/hedera/node/app/service/mono/state/migration/ContractStateMigrator.java @@ -31,23 +31,18 @@ import com.hedera.node.app.spi.state.WritableKVState; import com.hedera.pbj.runtime.io.buffer.Bytes; import com.swirlds.common.threading.interrupt.InterruptableConsumer; -import com.swirlds.common.threading.interrupt.InterruptableSupplier; import edu.umd.cs.findbugs.annotations.NonNull; import edu.umd.cs.findbugs.annotations.Nullable; import java.nio.ByteBuffer; -import java.time.Duration; -import java.time.temporal.ChronoUnit; +import java.nio.IntBuffer; import java.util.ArrayList; -import java.util.EnumSet; +import java.util.Arrays; +import java.util.Comparator; import java.util.List; -import java.util.Optional; -import java.util.concurrent.ArrayBlockingQueue; -import java.util.concurrent.CompletableFuture; -import java.util.concurrent.CompletionException; -import java.util.concurrent.TimeUnit; -import java.util.function.Supplier; +import java.util.concurrent.ConcurrentLinkedDeque; import java.util.function.UnaryOperator; import java.util.stream.Collectors; +import java.util.stream.Stream; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -62,7 +57,8 @@ public class ContractStateMigrator { private static final Logger log = LogManager.getLogger(ContractStateMigrator.class); - /** Return status from the contract storage migrator. The migration methods can also return a list of specific + /** + * Return status from the contract storage migrator. The migration methods can also return a list of specific * migration validation consistency failures (strings). */ public enum Status { @@ -71,11 +67,12 @@ public enum Status { INCOMPLETE_TRAVERSAL } - /** Given a `WritableKVState` for the contract storage, makes a copy of it - thus flushing it to disk! - and then + /** + * Given a `WritableKVState` for the contract storage, makes a copy of it - thus flushing it to disk! - and then * returns the copy for further writing. - * - * Supplied by the caller because only it knows how to get (or hold) the underlying datastore for a `WritableKVState`. - * (There's an example over at `DumpContractStoresSubcommand`.) + *

+ * Supplied by the caller because only it knows how to get (or hold) the underlying datastore for a + * `WritableKVState`. (There's an example over at `DumpContractStoresSubcommand`.) */ public interface StateFlusher extends UnaryOperator> {} @@ -114,6 +111,7 @@ public static Status migrateFromContractStorageVirtualMap( final var migrator = new ContractStateMigrator(fromState, initialToState, stateFlusher) .withValidation(doFullValidation) + .withValidationFailureMessages(validationFailures) .withThreadCount(threadCount); final var status = migrator.doit(validationFailures); @@ -126,21 +124,15 @@ public static Status migrateFromContractStorageVirtualMap( return status; } - /** A reasonable guess as to the number of threads to devote to the source tree traversal. - * - * a) Don't wish to use a configuration option because I want to be able to use this class in test code or a CLI - * tool that doesn't necessarily have the services (or platform) configuration system (easily) available. - * b) And though `availableProcessors()` may in fact give different results during the _same_ JVM invocation: IDC. + /** + * A reasonable guess as to the number of threads to devote to the source tree traversal. + *

+ * a) Don't wish to use a configuration option because I want to be able to use this class in test code or a CLI + * tool that doesn't necessarily have the services (or platform) configuration system (easily) available. b) And + * though `availableProcessors()` may in fact give different results during the _same_ JVM invocation: IDC. */ static final int DEFAULT_THREAD_COUNT = Math.max(1, Runtime.getRuntime().availableProcessors() * 2 / 3); - /** Slots are read from `fromState` concurrently with their being transformed and put into `toState`, with a queue - * between the source and the sink. This is the maximum size of that queue. - * - * (Current value of 1M holds both mainnet and testnet state as of 2023-11.) - */ - static final int MAXIMUM_SLOTS_IN_FLIGHT = 1_000_000; - static final int EVM_WORD_WIDTH_IN_BYTES = 32; static final int EVM_WORD_WIDTH_IN_INTS = 8; static final String LOG_CAPTION = "contract-storage mono-to-modular migration"; @@ -148,14 +140,13 @@ public static Status migrateFromContractStorageVirtualMap( /** Need to commit and flush at intervals; should be VirtualMapConfig.preferredFlushQueueSize() but no access here */ private static final int COMMIT_STATE_EVERY_N_INSERTS = 10_000; - /** Timeout to prevent a thread hanging in case of some design flaw - any guess for timeout is good, it's safety only */ - private static final Duration MIGRATION_QUEUE_POLL_TIMEOUT = Duration.of(100, ChronoUnit.MILLIS); - final VirtualMapLike fromState; WritableKVState toState; final StateFlusher stateFlusher; boolean doFullValidation; + + List validationFailures; int threadCount; final ContractMigrationValidationCounters counters; int nInsertionsDone = 0; @@ -173,10 +164,12 @@ public static Status migrateFromContractStorageVirtualMap( this.stateFlusher = stateFlusher; this.threadCount = DEFAULT_THREAD_COUNT; this.doFullValidation = true; + this.validationFailures = new ArrayList<>(); this.counters = ContractMigrationValidationCounters.create(); } - /** Perform (optional) validation that `toState` contains the same entities as `fromState`. Not a direct + /** + * Perform (optional) validation that `toState` contains the same entities as `fromState`. Not a direct * comparision, but a rough safety check. (Default: do not do this validation.) */ @NonNull @@ -185,6 +178,13 @@ ContractStateMigrator withValidation(final boolean doFullValidation) { return this; } + @NonNull + ContractStateMigrator withValidationFailureMessages(@NonNull final List validationFailures) { + requireNonNull(validationFailures); + this.validationFailures = validationFailures; + return this; + } + @NonNull ContractStateMigrator withThreadCount(final int threadCount) { this.threadCount = threadCount; @@ -198,27 +198,25 @@ ContractStateMigrator withThreadCount(final int threadCount) { @NonNull Status doit(@NonNull final List validationsFailed) { - final var completedProcesses = new NonAtomicReference>(); + final var sourcingCompleted = new NonAtomicReference(); withLoggedDuration( - () -> completedProcesses.set(transformStorage(doFullValidation)), + () -> sourcingCompleted.set(transformStorage(doFullValidation)), log, LOG_CAPTION + ": complete transform"); - requireNonNull(completedProcesses.get(), "must have valid completed processes set at this point"); - - return validateTransform(completedProcesses.get(), validationsFailed, doFullValidation); + return validateTransform(validationsFailed, sourcingCompleted.get(), doFullValidation); } /** * The intermediate representation of a contract slot (K/V pair, plus prev/next linked list references). */ - @SuppressWarnings("java:S6218") // should overload equals/hashcode - but will never test for equality or hash this private record ContractSlotLocal( - long contractId, @NonNull int[] key, @NonNull byte[] value, @Nullable int[] prev, @Nullable int[] next) { + long contractId, @NonNull int[] key, @NonNull byte[] value, @Nullable int[] prev, @Nullable int[] next) + implements Comparable { public static final ContractSlotLocal SENTINEL = new ContractSlotLocal( - 0L, new int[EVM_WORD_WIDTH_IN_INTS], new byte[EVM_WORD_WIDTH_IN_BYTES], null, null); + -1L, new int[EVM_WORD_WIDTH_IN_INTS], new byte[EVM_WORD_WIDTH_IN_BYTES], null, null); private ContractSlotLocal { requireNonNull(key); @@ -232,91 +230,108 @@ private record ContractSlotLocal( public ContractSlotLocal(@NonNull final ContractKey k, @NonNull final IterableContractValue v) { this(k.getContractId(), k.getKey(), v.getValue(), v.getExplicitPrevKey(), v.getExplicitNextKey()); } - } - /** Indicates that the source or sink process processed all slots */ - enum CompletedProcesses { - SOURCING, - SINKING - } + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + ContractSlotLocal that = (ContractSlotLocal) o; - /** - * Operates the source and sink processes to transform the mono-state ("from") into the modular-state ("to"). - * */ - @NonNull - EnumSet transformStorage(final boolean doFullValidation) { + return this.contractId == that.contractId && Arrays.equals(this.key, that.key); + } - final var slotQueue = new ArrayBlockingQueue(MAXIMUM_SLOTS_IN_FLIGHT); + @Override + public int hashCode() { + int result = Long.hashCode(this.contractId); + result = 31 * result + Arrays.hashCode(this.key); + return result; + } - // Sinking and sourcing happen concurrently. (Though sourcing is multithreaded, sinking is all on one thread.) - // Consider: For debugging, don't create (and start) `processSlotQueue` until after sourcing is complete. Just - // accumulate everything in the `fromState` in the queue before sinking anything. + static final Comparator comparator = Comparator.comparingLong(ContractSlotLocal::contractId) + .thenComparing(ContractSlotLocal::key, Comparator.comparing(IntBuffer::wrap)); - CompletableFuture processSlotQueue = CompletableFuture.runAsync(() -> iterateOverAllQueuedSlots( - () -> slotQueue.poll(MIGRATION_QUEUE_POLL_TIMEOUT.toMillis(), TimeUnit.MILLISECONDS), - fromState.size(), - doFullValidation)); + @Override + public int compareTo(@NonNull final ContractSlotLocal that) { + return comparator.compare(this, that); + } - final var completedTasks = EnumSet.noneOf(CompletedProcesses.class); + @Override + public String toString() { + return "ContractKeyLocal{id:%d,key:%s,value:%s,prev:%s,next:%s}" + .formatted( + contractId, + Arrays.toString(key), + Arrays.toString(value), + Arrays.toString(prev), + Arrays.toString(next)); + } + } - boolean didCompleteSourcing = iterateOverAllCurrentData(slotQueue::put, doFullValidation); - if (didCompleteSourcing) completedTasks.add(CompletedProcesses.SOURCING); + enum SourcingCompleted { + NO, + YES + } - boolean didCompleteSinking = true; - try { - processSlotQueue.join(); - } catch (BrokenTransformationException bex) { - log.error("%s: interrupt when sinking slots: %s".formatted(LOG_CAPTION, bex.getMessage()), bex.getCause()); - didCompleteSinking = false; - } - if (didCompleteSinking) completedTasks.add(CompletedProcesses.SINKING); + /** + * Operates the source and sink processes to transform the mono-state ("from") into the modular-state ("to"). + */ + @SuppressWarnings( + "java:S5411") // Avoid using boxed "Boolean" types directly - suggested rewrite is far worse to read + SourcingCompleted transformStorage(final boolean doFullValidation) { + + // Nodes must be added to the merkle tree in _deterministic_ order. Because this migrator writes nodes over + // _multiple_ commit batches it is responsible for ordering _all_ contract slots written to `toState`. Thus + // _all_ nodes are sourced from the `fromState` (in non-deterministic order) and only after _all_ have been + // read are they processed: first sorted, then transformed. + + final var slotQueue = new ConcurrentLinkedDeque(); + + // Multipart test here, during and after the traversal of `fromState`, to ensure that all `fromState` slots got + // handled and put in the queue + final boolean didCompleteSourcing = addMsgIfFailed( + iterateOverAllCurrentData(slotQueue::add, doFullValidation), + "traversal of `fromState` interrupted") + && addMsgIfFailed( + slotQueue.size() == fromState.size() + 1, + "did not source expected number of slots (expected %d+1 slots, found %d)" + .formatted(fromState.size(), slotQueue.size())) + && addMsgIfFailed( + slotQueue.getLast() == ContractSlotLocal.SENTINEL, + "slot source did not end with expected sentinel"); + + if (!didCompleteSourcing) return SourcingCompleted.NO; + + // Achieve the deterministic ordering of all slots + final var slotArray = slotQueue.toArray(new ContractSlotLocal[0]); + Arrays.parallelSort(slotArray, Comparator.naturalOrder()); + + // Prepare to strip out the sentinel + final var slotsToProcess = Arrays.stream(slotArray).filter(e -> e != ContractSlotLocal.SENTINEL); + + // Now process all slots + iterateOverAllQueuedSlots(slotsToProcess, doFullValidation); + + return SourcingCompleted.YES; + } - return completedTasks; + boolean addMsgIfFailed(final boolean condition, @NonNull final String messageOnFailure) { + if (!condition) validationFailures.add(messageOnFailure); + return condition; } /** - * Pull slots off the queue and add each one immediately to the modular-service state. This is "sinking" the slots. + * Pull slots off the stream and add each one immediately to the modular-service state. This is "sinking" the slots. * - * This is single-threaded. (But does operate concurrently with sourcing.) + * (Caller checked to make sure that the source did run to completion and stripped away the sentinel.) */ @SuppressWarnings("java:S2142") // must rethrow InterruptedException - Sonar doesn't understand wrapping it to throw void iterateOverAllQueuedSlots( - @NonNull final InterruptableSupplier slotSource, - final long expectedSlotCount, - final boolean doFullValidation) { + @NonNull final Stream slotSource, final boolean doFullValidation) { requireNonNull(slotSource); - // Lambda to get the next item off the concurrent queue between source and sink. Handles end-of-stream - // (finding the sentinel element) by returning an empty `Optional`. Transforms an `InterruptedException` - // (which is _not_ a `RuntimeException`) into the more easily handled exception defined in this class. - final Supplier> getter = () -> { - try { - final var slot = slotSource.get(); - if (slot == ContractSlotLocal.SENTINEL) return Optional.empty(); - return Optional.of(slot); - } catch (final InterruptedException ex) { - throw new BrokenTransformationException( - LOG_CAPTION + ": timeout reading contract slots from queue", ex); - } - }; - - // We know precisely how many slots we have to process. And they're followed, in the queue, by a sentinel - // element. But this loop checks for too _few_ and too _many_ slots in the queue, for safety. (The check for - // too few is by either finding the sentinel too soon or hitting a timeout waiting for an element from the - // queue.) - for (int i = 0; i < expectedSlotCount; i++) { - final var oslot = getter.get(); - if (oslot.isEmpty()) { - final var msg = - "%s: not enough contract slots read from queue (%d expected, %d read), SENTINEL found too soon" - .formatted(LOG_CAPTION, expectedSlotCount, i); - throw new BrokenTransformationException(msg); - } - + slotSource.forEachOrdered(slot -> { // The transform from the mono-service slot representation to the modular-service slot representation // is here: - - final var slot = oslot.get(); final var key = SlotKey.newBuilder() .contractNumber(slot.contractId()) .key(bytesFromInts(slot.key())) @@ -339,16 +354,9 @@ void iterateOverAllQueuedSlots( if (value.nextKey() == Bytes.EMPTY) counters.addMissingNext(); else counters.xorAnotherLink(value.nextKey().toByteArray()); } - } + }); commitToStateNow(); - - // Read final sentinel value - if (getter.get().isPresent()) { - final var msg = "%s: too many contract slots read (%d expected), SENTINEL not yet found" - .formatted(LOG_CAPTION, expectedSlotCount); - throw new BrokenTransformationException(msg); - } } /** State needs to be flushed to disk every so many inserts (for performance reasons w.r.t. the underlying @@ -397,17 +405,12 @@ boolean iterateOverAllCurrentData( /** If processing did not complete, report that. Otherwise, perform (optionally) some simple consistency checks. */ @NonNull Status validateTransform( - @NonNull final EnumSet completedProcesses, - @NonNull final List validationsFailed, + @NonNull final List validationFailures, + @NonNull final SourcingCompleted sourcingCompleted, final boolean doFullValidation) { - requireNonNull(completedProcesses); // Validate that both the source and sink finished processing slots - if (completedProcesses.size() != EnumSet.allOf(CompletedProcesses.class).size()) { - if (!completedProcesses.contains(CompletedProcesses.SOURCING)) - validationsFailed.add("Sourcing process didn't finish"); - if (!completedProcesses.contains(CompletedProcesses.SINKING)) - validationsFailed.add("Sinking process didn't finish"); + if (sourcingCompleted == SourcingCompleted.NO) { return Status.INCOMPLETE_TRAVERSAL; } @@ -418,7 +421,7 @@ Status validateTransform( if (fromSize != counters.slotsSourced().get() || fromSize != counters.slotsSunk().get() || fromSize != toState.size()) { - validationsFailed.add( + validationFailures.add( "counters of slots processed don't match: %d source size, %d #slots sourced, %d slots sunk, %d final destination size" .formatted( fromSize, @@ -433,7 +436,7 @@ Status validateTransform( final var nContracts = counters.contracts().size(); if (nContracts != counters.nMissingPrevs().get() || nContracts != counters.nMissingNexts().get()) { - validationsFailed.add( + validationFailures.add( "distinct contracts doesn't match #null prev links and/or #null next links: %d contract, %d null prevs, %d null nexts" .formatted( nContracts, @@ -442,10 +445,10 @@ Status validateTransform( } if (!counters.runningXorOfLinksIsZero()) { - validationsFailed.add("prev/next links (over all contracts) aren't properly paired"); + validationFailures.add("prev/next links (over all contracts) aren't properly paired"); } - return validationsFailed.isEmpty() ? Status.VALIDATION_ERRORS : Status.SUCCESS; + return validationFailures.isEmpty() ? Status.VALIDATION_ERRORS : Status.SUCCESS; } /** Convert int[] to byte[] and then to Bytes. If argument is null or 0-length then return `Bytes.EMPTY`. */ @@ -466,16 +469,4 @@ static Bytes bytesFromInts(@Nullable final int[] ints) { public static void validateArgument(final boolean b, @NonNull final String msg) { if (!b) throw new IllegalArgumentException(msg); } - - /** Exception to indicate the migration failed. */ - static class BrokenTransformationException extends CompletionException { - - public BrokenTransformationException(@NonNull final String message) { - super(message); - } - - public BrokenTransformationException(@NonNull final String message, @NonNull final Throwable t) { - super(message, t); - } - } } From 7610d394c5d719bdbc0091148882f627b114330d Mon Sep 17 00:00:00 2001 From: Nathan Klick Date: Wed, 6 Dec 2023 19:24:29 -0600 Subject: [PATCH 3/7] chore(ci): add a new CI workflow to ensure release artifact determinism (#10347) Signed-off-by: Nathan Klick --- .../workflows/flow-artifact-determinism.yaml | 61 +++++ .../generate-gradle-artifact-baseline.sh | 149 +++++++++++ .../zxc-verify-gradle-build-determinism.yaml | 237 ++++++++++++++++++ 3 files changed, 447 insertions(+) create mode 100644 .github/workflows/flow-artifact-determinism.yaml create mode 100755 .github/workflows/support/scripts/generate-gradle-artifact-baseline.sh create mode 100644 .github/workflows/zxc-verify-gradle-build-determinism.yaml diff --git a/.github/workflows/flow-artifact-determinism.yaml b/.github/workflows/flow-artifact-determinism.yaml new file mode 100644 index 000000000000..ab007d6aee8e --- /dev/null +++ b/.github/workflows/flow-artifact-determinism.yaml @@ -0,0 +1,61 @@ +## +# Copyright (C) 2023 Hedera Hashgraph, LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +## + +name: Artifact Determinism +on: + workflow_dispatch: + inputs: + ref: + description: "The branch, tag, or commit to checkout:" + type: string + required: false + default: "" + java-distribution: + description: "Java JDK Distribution:" + type: string + required: false + default: "temurin" + java-version: + description: "Java JDK Version:" + type: string + required: false + default: "17.0.9" + push: + branches: + - develop + - 'release/**' + tags: + - 'v*.*.*' + +defaults: + run: + shell: bash + +permissions: + id-token: write + contents: read + +jobs: + check: + name: Gradle + uses: ./.github/workflows/zxc-verify-gradle-build-determinism.yaml + with: + ref: ${{ github.event.inputs.ref || '' }} + java-distribution: ${{ inputs.java-distribution || 'temurin' }} + java-version: ${{ inputs.java-version || '17.0.9' }} + secrets: + gradle-cache-username: ${{ secrets.GRADLE_CACHE_USERNAME }} + gradle-cache-password: ${{ secrets.GRADLE_CACHE_PASSWORD }} diff --git a/.github/workflows/support/scripts/generate-gradle-artifact-baseline.sh b/.github/workflows/support/scripts/generate-gradle-artifact-baseline.sh new file mode 100755 index 000000000000..53ae01ef2bdc --- /dev/null +++ b/.github/workflows/support/scripts/generate-gradle-artifact-baseline.sh @@ -0,0 +1,149 @@ +#!/usr/bin/env bash +set -o pipefail +set +e + +readonly RELEASE_LIB_PATH="hedera-node/data/lib" +readonly RELEASE_APPS_PATH="hedera-node/data/apps" + +GROUP_ACTIVE="false" + +function fail { + printf '%s\n' "$1" >&2 ## Send message to stderr. Exclude >&2 if you don't want it that way. + if [[ "${GROUP_ACTIVE}" == "true" ]]; then + end_group + fi + exit "${2-1}" ## Return a code specified by $2 or 1 by default. +} + +function start_group { + if [[ "${GROUP_ACTIVE}" == "true" ]]; then + end_group + fi + + GROUP_ACTIVE="true" + printf "::group::%s\n" "${1}" +} + +function end_group { + GROUP_ACTIVE="false" + printf "::endgroup::\n" +} + +function log { + local message="${1}" + shift + # shellcheck disable=SC2059 + printf "${message}" "${@}" +} + +function log_line { + local message="${1}" + shift + # shellcheck disable=SC2059 + printf "${message}\n" "${@}" +} + +function start_task { + local message="${1}" + shift + # shellcheck disable=SC2059 + printf "${message} .....\t" "${@}" +} + +function end_task { + printf "%s\n" "${1:-DONE}" +} + +start_group "Configuring Environment" + # Access workflow environment variables + export GITHUB_WORKSPACE GITHUB_SHA GITHUB_OUTPUT MANIFEST_PATH + + start_task "Initializing Temporary Directory" + TEMP_DIR="$(mktemp -d)" || fail "ERROR (Exit Code: ${?})" "${?}" + trap 'rm -rf "${TEMP_DIR}"' EXIT + end_task "DONE (Path: ${TEMP_DIR})" + + start_task "Resolving the GITHUB_WORKSPACE path" + # Ensure GITHUB_WORKSPACE is provided or default to the repository root + if [[ -z "${GITHUB_WORKSPACE}" || ! -d "${GITHUB_WORKSPACE}" ]]; then + GITHUB_WORKSPACE="$(cd "$(dirname "${BASH_SOURCE[0]}")/../../../../" && pwd)" + fi + end_task "DONE (Path: ${GITHUB_WORKSPACE})" + + start_task "Resolving the GITHUB_OUTPUT path" + # Ensure GITHUB_OUTPUT is provided or default to the repository root + if [[ -z "${GITHUB_OUTPUT}" ]]; then + GITHUB_OUTPUT="${TEMP_DIR}/workflow-output.txt" + fi + end_task "DONE (Path: ${GITHUB_OUTPUT})" + + start_task "Resolving the GITHUB_SHA hash" + if [[ -z "${GITHUB_SHA}" ]]; then + GITHUB_SHA="$(git rev-parse HEAD | tr -d '[:space:]')" || fail "ERROR (Exit Code: ${?})" "${?}" + fi + end_task "DONE (Commit: ${GITHUB_SHA})" + + start_task "Resolving the MANIFEST_PATH variable" + if [[ -z "${MANIFEST_PATH}" ]]; then + MANIFEST_PATH="${GITHUB_WORKSPACE}/.manifests/gradle" + fi + end_task "DONE (Path: ${MANIFEST_PATH})" + + start_task "Ensuring the MANIFEST_PATH location is present" + if [[ ! -d "${MANIFEST_PATH}" ]]; then + mkdir -p "${MANIFEST_PATH}" || fail "ERROR (Exit Code: ${?})" "${?}" + fi + end_task + + start_task "Checking for the sha256sum command" + if command -v sha256sum >/dev/null 2>&1; then + SHA256SUM="$(command -v sha256sum)" || fail "ERROR (Exit Code: ${?})" "${?}" + else + fail "ERROR (Exit Code: ${?})" "${?}" + fi + end_task "DONE (Found: ${SHA256SUM})" + + start_task "Checking for prebuilt libraries" + ls -al "${GITHUB_WORKSPACE}/${RELEASE_LIB_PATH}"/*.jar >/dev/null 2>&1 || fail "ERROR (Exit Code: ${?})" "${?}" + end_task "FOUND (Path: ${GITHUB_WORKSPACE}/${RELEASE_LIB_PATH}/*.jar)" + + start_task "Checking for prebuilt applications" + ls -al "${GITHUB_WORKSPACE}/${RELEASE_APPS_PATH}"/*.jar >/dev/null 2>&1 || fail "ERROR (Exit Code: ${?})" "${?}" + end_task "FOUND (Path: ${GITHUB_WORKSPACE}/${RELEASE_APPS_PATH}/*.jar)" +end_group + +start_group "Generating Library Hashes (${GITHUB_WORKSPACE}/${RELEASE_LIB_PATH}/*.jar)" + pushd "${GITHUB_WORKSPACE}/${RELEASE_LIB_PATH}" >/dev/null 2>&1 || fail "PUSHD ERROR (Exit Code: ${?})" "${?}" + ${SHA256SUM} -- *.jar | sort -k 2 | tee -a "${TEMP_DIR}"/libraries.sha256 + popd >/dev/null 2>&1 || fail "POPD ERROR (Exit Code: ${?})" "${?}" +end_group + +start_group "Generating Application Hashes (${GITHUB_WORKSPACE}/${RELEASE_APPS_PATH}/*.jar)" + pushd "${GITHUB_WORKSPACE}/${RELEASE_APPS_PATH}" >/dev/null 2>&1 || fail "PUSHD ERROR (Exit Code: ${?})" "${?}" + ${SHA256SUM} -- *.jar | sort -k 2 | tee -a "${TEMP_DIR}"/applications.sha256 + popd >/dev/null 2>&1 || fail "POPD ERROR (Exit Code: ${?})" "${?}" +end_group + +start_group "Generating Final Release Manifests" + + start_task "Generating the manifest archive" + tar -czf "${TEMP_DIR}/manifest.tar.gz" -C "${TEMP_DIR}" libraries.sha256 applications.sha256 >/dev/null 2>&1 || fail "TAR ERROR (Exit Code: ${?})" "${?}" + end_task + + start_task "Copying the manifest files" + cp "${TEMP_DIR}/manifest.tar.gz" "${MANIFEST_PATH}/${GITHUB_SHA}.tar.gz" || fail "COPY ERROR (Exit Code: ${?})" "${?}" + cp "${TEMP_DIR}/libraries.sha256" "${MANIFEST_PATH}/libraries.sha256" || fail "COPY ERROR (Exit Code: ${?})" "${?}" + cp "${TEMP_DIR}/applications.sha256" "${MANIFEST_PATH}/applications.sha256" || fail "COPY ERROR (Exit Code: ${?})" "${?}" + end_task "DONE (Path: ${MANIFEST_PATH}/${GITHUB_SHA}.tar.gz)" + + start_task "Setting Step Outputs" + { + printf "path=%s\n" "${MANIFEST_PATH}" + printf "file=%s\n" "${MANIFEST_PATH}/${GITHUB_SHA}.tar.gz" + printf "name=%s\n" "${GITHUB_SHA}.tar.gz" + printf "applications=%s\n" "${MANIFEST_PATH}/applications.sha256" + printf "libraries=%s\n" "${MANIFEST_PATH}/libraries.sha256" + } >> "${GITHUB_OUTPUT}" + end_task +end_group + diff --git a/.github/workflows/zxc-verify-gradle-build-determinism.yaml b/.github/workflows/zxc-verify-gradle-build-determinism.yaml new file mode 100644 index 000000000000..708da293a382 --- /dev/null +++ b/.github/workflows/zxc-verify-gradle-build-determinism.yaml @@ -0,0 +1,237 @@ +## +# Copyright (C) 2023 Hedera Hashgraph, LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +## + +name: "ZXC: Verify Gradle Build Determinism" +on: + workflow_call: + inputs: + ref: + description: "The branch, tag, or commit to checkout:" + type: string + required: false + default: "" + java-distribution: + description: "Java JDK Distribution:" + type: string + required: false + default: "temurin" + java-version: + description: "Java JDK Version:" + type: string + required: false + default: "17.0.9" + + secrets: + gradle-cache-username: + description: "The username used to authenticate with the Gradle Build Cache Node." + required: true + gradle-cache-password: + description: "The password used to authenticate with the Gradle Build Cache Node." + required: true + +defaults: + run: + shell: bash + +permissions: + id-token: write + contents: read + +env: + GRADLE_MANIFEST_PATH: ${{ github.workspace }}/.manifests/gradle + GRADLE_MANIFEST_GENERATOR: .github/workflows/support/scripts/generate-gradle-artifact-baseline.sh + GRADLE_CACHE_USERNAME: ${{ secrets.gradle-cache-username }} + GRADLE_CACHE_PASSWORD: ${{ secrets.gradle-cache-password }} + +jobs: + generate-baseline: + name: Generate Baseline + runs-on: [self-hosted, Linux, medium, ephemeral] + outputs: + sha: ${{ steps.commit.outputs.sha }} + path: ${{ steps.baseline.outputs.path }} + file: ${{ steps.baseline.outputs.file }} + name: ${{ steps.baseline.outputs.name }} + steps: + - name: Checkout Code + uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 + with: + ref: ${{ inputs.ref }} + + - name: Setup Java + uses: actions/setup-java@0ab4596768b603586c0de567f2430c30f5b0d2b0 # v3.13.0 + with: + distribution: ${{ inputs.java-distribution }} + java-version: ${{ inputs.java-version }} + + - name: Setup Gradle + uses: gradle/gradle-build-action@842c587ad8aa4c68eeba24c396e15af4c2e9f30a # v2.9.0 + with: + cache-disabled: true + + - name: Authenticate to Google Cloud + id: google-auth + uses: google-github-actions/auth@f105ef0cdb3b102a020be1767fcc8a974898b7c6 # v1.2.0 + with: + workload_identity_provider: "projects/235822363393/locations/global/workloadIdentityPools/hedera-builds-pool/providers/hedera-builds-gh-actions" + service_account: "swirlds-automation@hedera-registry.iam.gserviceaccount.com" + + - name: Setup Google Cloud SDK + uses: google-github-actions/setup-gcloud@e30db14379863a8c79331b04a9969f4c1e225e0b # v1.1.1 + + - name: Retrieve Commit Hash + id: commit + run: echo "sha=$(git rev-parse HEAD)" >> "${GITHUB_OUTPUT}" + + - name: Baseline Existence Check + id: baseline + run: | + BASELINE_NAME="${{ steps.commit.outputs.sha }}.tar.gz" + BASELINE_PATH="gs://hedera-ci-ephemeral-artifacts/${{ github.repository }}/gradle/baselines" + BASELINE_FILE="${BASELINE_PATH}/${BASELINE_NAME}" + BASELINE_EXISTS="false" + + if gsutil ls "${BASELINE_FILE}" >/dev/null 2>&1; then + BASELINE_EXISTS="true" + fi + + echo "exists=${BASELINE_EXISTS}" >> "${GITHUB_OUTPUT}" + echo "path=${BASELINE_PATH}" >> "${GITHUB_OUTPUT}" + echo "name=${BASELINE_NAME}" >> "${GITHUB_OUTPUT}" + echo "file=${BASELINE_FILE}" >> "${GITHUB_OUTPUT}" + + - name: Build Artifacts + id: gradle-build + if: ${{ steps.baseline.outputs.exists == 'false' && !failure() && !cancelled() }} + run: ./gradlew assemble --scan + + - name: Generate Manifest + id: manifest + env: + MANIFEST_PATH: ${{ env.GRADLE_MANIFEST_PATH }} + if: ${{ steps.baseline.outputs.exists == 'false' && !failure() && !cancelled() }} + run: ${{ env.GRADLE_MANIFEST_GENERATOR }} + + - name: Upload Baseline + if: ${{ steps.baseline.outputs.exists == 'false' && !failure() && !cancelled() }} + run: gsutil cp "${{ steps.manifest.outputs.file }}" "${{ steps.baseline.outputs.file }}" + + verify-artifacts: + name: "Verify Artifacts (${{ join(matrix.os, ', ') }})" + runs-on: ${{ matrix.os }} + needs: + - generate-baseline + strategy: + fail-fast: false + matrix: + os: + - ubuntu-22.04 + - ubuntu-20.04 + - macos-12 + - macos-11 + - windows-2022 + - windows-2019 + - [self-hosted, Linux, medium, ephemeral] + - [self-hosted, Linux, large, ephemeral] + steps: + - name: Checkout Code + uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 + with: + ref: ${{ inputs.ref }} + + - name: Setup Python + uses: actions/setup-python@b64ffcaf5b410884ad320a9cfac8866006a109aa # v4.8.0 + with: + python-version: 3.9 + + - name: Setup Java + uses: actions/setup-java@0ab4596768b603586c0de567f2430c30f5b0d2b0 # v3.13.0 + with: + distribution: ${{ inputs.java-distribution }} + java-version: ${{ inputs.java-version }} + + - name: Setup Gradle + uses: gradle/gradle-build-action@842c587ad8aa4c68eeba24c396e15af4c2e9f30a # v2.9.0 + with: + cache-disabled: true + + - name: Setup CoreUtils (macOS) + if: ${{ runner.os == 'macOS' }} + run: brew install coreutils + + - name: Authenticate to Google Cloud + id: google-auth + uses: google-github-actions/auth@f105ef0cdb3b102a020be1767fcc8a974898b7c6 # v1.2.0 + with: + workload_identity_provider: "projects/235822363393/locations/global/workloadIdentityPools/hedera-builds-pool/providers/hedera-builds-gh-actions" + service_account: "swirlds-automation@hedera-registry.iam.gserviceaccount.com" + + - name: Setup Google Cloud SDK + uses: google-github-actions/setup-gcloud@e30db14379863a8c79331b04a9969f4c1e225e0b # v1.1.1 + env: + CLOUDSDK_PYTHON: ${{ format('{0}{1}', env.pythonLocation, runner.os == 'Windows' && '\python.exe' || '/bin/python3') }} + + - name: Download Baseline + env: + CLOUDSDK_PYTHON: ${{ format('{0}{1}', env.pythonLocation, runner.os == 'Windows' && '\python.exe' || '/bin/python3') }} + run: | + mkdir -p "${GRADLE_MANIFEST_PATH}" + cd "${GRADLE_MANIFEST_PATH}" + gsutil cp "${{ needs.generate-baseline.outputs.file }}" . + tar -xzf "${{ needs.generate-baseline.outputs.name }}" + + - name: Build Artifacts + id: gradle-build + run: ./gradlew assemble --scan --no-build-cache + + - name: Regenerate Manifest + id: regen-manifest + env: + MANIFEST_PATH: ${{ env.GRADLE_MANIFEST_PATH }}/regenerated + run: ${{ env.GRADLE_MANIFEST_GENERATOR }} + + - name: Validate Libraries + working-directory: ${{ github.workspace }}/hedera-node/data/lib + run: sha256sum -c "${GRADLE_MANIFEST_PATH}/libraries.sha256" + + - name: Validate Applications + working-directory: ${{ github.workspace }}/hedera-node/data/apps + run: sha256sum -c "${GRADLE_MANIFEST_PATH}/applications.sha256" + + - name: Compare Library Manifests + run: | + if ! diff -u "${GRADLE_MANIFEST_PATH}/libraries.sha256" "${{ steps.regen-manifest.outputs.libraries }}" >/dev/null 2>&1; then + echo "::group::Library Manifest Differences" + diff -u "${GRADLE_MANIFEST_PATH}/libraries.sha256" "${{ steps.regen-manifest.outputs.libraries }}" + echo "::endgroup::" + exit 1 + fi + + - name: Compare Application Manifests + run: | + if ! diff -u "${GRADLE_MANIFEST_PATH}/applications.sha256" "${{ steps.regen-manifest.outputs.applications }}" >/dev/null 2>&1; then + echo "::group::Application Manifest Differences" + diff -u "${GRADLE_MANIFEST_PATH}/applications.sha256" "${{ steps.regen-manifest.outputs.applications }}" + echo "::endgroup::" + exit 1 + fi + + - name: Publish Manifests + uses: actions/upload-artifact@a8a3f3ad30e3422c9c7b888a15615d19a852ae32 # v3.1.3 + if: ${{ steps.regen-manifest.conclusion == 'success' && failure() && !cancelled() }} + with: + name: Gradle Manifests [${{ join(matrix.os, ', ') }}] + path: ${{ env.GRADLE_MANIFEST_PATH }}/** From f0702214cbacbaf543560ffcf2890f1da3888004 Mon Sep 17 00:00:00 2001 From: lukelee-sl <109538178+lukelee-sl@users.noreply.github.com> Date: Wed, 6 Dec 2023 17:29:19 -0800 Subject: [PATCH 4/7] feat: Implement lazyCreationCostInGas method (#10337) Signed-off-by: lukelee-sl --- .../java/contract/AbstractContractXTest.java | 9 ++-- .../exec/scope/HandleHederaOperations.java | 46 +++++++++++++++++-- .../impl/exec/scope/HederaOperations.java | 4 +- .../exec/scope/QueryHederaOperations.java | 3 +- .../impl/state/ProxyWorldUpdater.java | 2 +- .../scope/HandleHederaOperationsTest.java | 21 +++++++-- .../test/state/ProxyWorldUpdaterTest.java | 6 +-- 7 files changed, 75 insertions(+), 16 deletions(-) diff --git a/hedera-node/hedera-app/src/xtest/java/contract/AbstractContractXTest.java b/hedera-node/hedera-app/src/xtest/java/contract/AbstractContractXTest.java index 5a38cfa31797..059e264efccb 100644 --- a/hedera-node/hedera-app/src/xtest/java/contract/AbstractContractXTest.java +++ b/hedera-node/hedera-app/src/xtest/java/contract/AbstractContractXTest.java @@ -276,12 +276,17 @@ private void runHtsCallAndExpect( context.exchangeRateInfo().activeRate(Instant.now()), context.resourcePricesFor(HederaFunctionality.CONTRACT_CALL, SubType.DEFAULT), context.resourcePricesFor(HederaFunctionality.CONTRACT_CALL, SubType.DEFAULT)); + final var systemContractGasCalculator = new SystemContractGasCalculator( + tinybarValues, + new CanonicalDispatchPrices(new AssetsLoader()), + (body, payerId) -> context.dispatchComputeFees(body, payerId).totalFee()); final var enhancement = new HederaWorldUpdater.Enhancement( new HandleHederaOperations( component.config().getConfigData(LedgerConfig.class), component.config().getConfigData(ContractsConfig.class), context, tinybarValues, + systemContractGasCalculator, component.config().getConfigData(HederaConfig.class)), new HandleHederaNativeOperations(context), new HandleSystemContractOperations(context)); @@ -290,10 +295,6 @@ private void runHtsCallAndExpect( given(frame.getSenderAddress()).willReturn(sender); final Deque stack = new ArrayDeque<>(); given(initialFrame.getContextVariable(CONFIG_CONTEXT_VARIABLE)).willReturn(component.config()); - final var systemContractGasCalculator = new SystemContractGasCalculator( - tinybarValues, - new CanonicalDispatchPrices(new AssetsLoader()), - (body, payerId) -> context.dispatchComputeFees(body, payerId).totalFee()); given(initialFrame.getContextVariable(SYSTEM_CONTRACT_GAS_CALCULATOR_CONTEXT_VARIABLE)) .willReturn(systemContractGasCalculator); stack.push(initialFrame); diff --git a/hedera-node/hedera-smart-contract-service-impl/src/main/java/com/hedera/node/app/service/contract/impl/exec/scope/HandleHederaOperations.java b/hedera-node/hedera-smart-contract-service-impl/src/main/java/com/hedera/node/app/service/contract/impl/exec/scope/HandleHederaOperations.java index 75d528323f80..1dcd74540e39 100644 --- a/hedera-node/hedera-smart-contract-service-impl/src/main/java/com/hedera/node/app/service/contract/impl/exec/scope/HandleHederaOperations.java +++ b/hedera-node/hedera-smart-contract-service-impl/src/main/java/com/hedera/node/app/service/contract/impl/exec/scope/HandleHederaOperations.java @@ -18,7 +18,11 @@ import static com.hedera.hapi.node.base.ResponseCodeEnum.OK; import static com.hedera.hapi.node.base.ResponseCodeEnum.SUCCESS; +import static com.hedera.node.app.service.contract.impl.utils.ConversionUtils.tuweniToPbjBytes; import static com.hedera.node.app.service.contract.impl.utils.SynthTxnUtils.*; +import static com.hedera.node.app.service.mono.txns.crypto.AbstractAutoCreationLogic.LAZY_MEMO; +import static com.hedera.node.app.service.mono.txns.crypto.AbstractAutoCreationLogic.THREE_MONTHS_IN_SECONDS; +import static com.hedera.node.app.spi.key.KeyUtils.IMMUTABILITY_SENTINEL_KEY; import static com.hedera.node.app.spi.workflows.record.ExternalizedRecordCustomizer.SUPPRESSING_EXTERNALIZED_RECORD_CUSTOMIZER; import static java.util.Objects.requireNonNull; @@ -26,9 +30,12 @@ import com.hedera.hapi.node.contract.ContractCreateTransactionBody; import com.hedera.hapi.node.contract.ContractFunctionResult; import com.hedera.hapi.node.token.CryptoCreateTransactionBody; +import com.hedera.hapi.node.token.CryptoUpdateTransactionBody; import com.hedera.hapi.node.transaction.SignedTransaction; import com.hedera.hapi.node.transaction.TransactionBody; import com.hedera.node.app.service.contract.impl.annotations.TransactionScope; +import com.hedera.node.app.service.contract.impl.exec.gas.DispatchType; +import com.hedera.node.app.service.contract.impl.exec.gas.SystemContractGasCalculator; import com.hedera.node.app.service.contract.impl.exec.gas.TinybarValues; import com.hedera.node.app.service.contract.impl.records.ContractCreateRecordBuilder; import com.hedera.node.app.service.contract.impl.state.ContractStateStore; @@ -50,6 +57,7 @@ import java.util.List; import java.util.Optional; import javax.inject.Inject; +import org.hyperledger.besu.datatypes.Address; /** * A fully mutable {@link HederaOperations} implementation based on a {@link HandleContext}. @@ -59,10 +67,24 @@ public class HandleHederaOperations implements HederaOperations { public static final Bytes ZERO_ENTROPY = Bytes.fromHex( "000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000"); + private static final CryptoUpdateTransactionBody.Builder UPDATE_TXN_BODY_BUILDER = + CryptoUpdateTransactionBody.newBuilder() + .key(Key.newBuilder().ecdsaSecp256k1(Bytes.EMPTY).build()); + + private static final CryptoCreateTransactionBody.Builder CREATE_TXN_BODY_BUILDER = + CryptoCreateTransactionBody.newBuilder() + .initialBalance(0) + .maxAutomaticTokenAssociations(0) + .autoRenewPeriod(Duration.newBuilder().seconds(THREE_MONTHS_IN_SECONDS)) + .key(IMMUTABILITY_SENTINEL_KEY) + .memo(LAZY_MEMO); + private final TinybarValues tinybarValues; private final LedgerConfig ledgerConfig; private final ContractsConfig contractsConfig; private final HederaConfig hederaConfig; + private final SystemContractGasCalculator gasCalculator; + private final HandleContext context; @Inject @@ -71,12 +93,14 @@ public HandleHederaOperations( @NonNull final ContractsConfig contractsConfig, @NonNull final HandleContext context, @NonNull final TinybarValues tinybarValues, + @NonNull final SystemContractGasCalculator gasCalculator, @NonNull final HederaConfig hederaConfig) { this.ledgerConfig = requireNonNull(ledgerConfig); this.contractsConfig = requireNonNull(contractsConfig); this.context = requireNonNull(context); this.tinybarValues = requireNonNull(tinybarValues); this.hederaConfig = requireNonNull(hederaConfig); + this.gasCalculator = requireNonNull(gasCalculator); } /** @@ -154,9 +178,25 @@ public long contractCreationLimit() { * {@inheritDoc} */ @Override - public long lazyCreationCostInGas() { - // TODO - implement correctly - return 1L; + public long lazyCreationCostInGas(@NonNull final Address recipient) { + final var payerId = context.payer(); + // Calculate gas for a CryptoCreateTransactionBody with an alias address + final var createFee = gasCalculator.gasRequirement( + TransactionBody.newBuilder() + .cryptoCreateAccount(CREATE_TXN_BODY_BUILDER.alias(tuweniToPbjBytes(recipient))) + .build(), + DispatchType.CRYPTO_CREATE, + payerId); + + // Calculate gas for an update TransactionBody + final var updateFee = gasCalculator.gasRequirement( + TransactionBody.newBuilder() + .cryptoUpdateAccount(UPDATE_TXN_BODY_BUILDER) + .build(), + DispatchType.CRYPTO_UPDATE, + payerId); + + return createFee + updateFee; } /** diff --git a/hedera-node/hedera-smart-contract-service-impl/src/main/java/com/hedera/node/app/service/contract/impl/exec/scope/HederaOperations.java b/hedera-node/hedera-smart-contract-service-impl/src/main/java/com/hedera/node/app/service/contract/impl/exec/scope/HederaOperations.java index b9bfed6cbaca..e75601780be0 100644 --- a/hedera-node/hedera-smart-contract-service-impl/src/main/java/com/hedera/node/app/service/contract/impl/exec/scope/HederaOperations.java +++ b/hedera-node/hedera-smart-contract-service-impl/src/main/java/com/hedera/node/app/service/contract/impl/exec/scope/HederaOperations.java @@ -32,6 +32,7 @@ import edu.umd.cs.findbugs.annotations.NonNull; import edu.umd.cs.findbugs.annotations.Nullable; import java.util.List; +import org.hyperledger.besu.datatypes.Address; /** * Provides the Hedera operations that only a {@link ProxyWorldUpdater} needs (but not a {@link DispatchingEvmFrameState}. @@ -109,10 +110,11 @@ public interface HederaOperations { /** * Returns the lazy creation cost within this scope. + * @param recipient the recipient contract address * * @return the lazy creation cost in gas */ - long lazyCreationCostInGas(); + long lazyCreationCostInGas(@NonNull final Address recipient); /** * Returns the gas price in tinybars within this scope. diff --git a/hedera-node/hedera-smart-contract-service-impl/src/main/java/com/hedera/node/app/service/contract/impl/exec/scope/QueryHederaOperations.java b/hedera-node/hedera-smart-contract-service-impl/src/main/java/com/hedera/node/app/service/contract/impl/exec/scope/QueryHederaOperations.java index 615ddc38ad4b..df5d95b8a42b 100644 --- a/hedera-node/hedera-smart-contract-service-impl/src/main/java/com/hedera/node/app/service/contract/impl/exec/scope/QueryHederaOperations.java +++ b/hedera-node/hedera-smart-contract-service-impl/src/main/java/com/hedera/node/app/service/contract/impl/exec/scope/QueryHederaOperations.java @@ -34,6 +34,7 @@ import java.util.Objects; import java.util.Optional; import javax.inject.Inject; +import org.hyperledger.besu.datatypes.Address; /** * TODO - a read-only {@link HederaOperations} implementation based on a {@link QueryContext}. @@ -128,7 +129,7 @@ public long contractCreationLimit() { * @throws UnsupportedOperationException always */ @Override - public long lazyCreationCostInGas() { + public long lazyCreationCostInGas(@NonNull final Address recipient) { throw new UnsupportedOperationException("Queries cannot get lazy creation cost"); } diff --git a/hedera-node/hedera-smart-contract-service-impl/src/main/java/com/hedera/node/app/service/contract/impl/state/ProxyWorldUpdater.java b/hedera-node/hedera-smart-contract-service-impl/src/main/java/com/hedera/node/app/service/contract/impl/state/ProxyWorldUpdater.java index 147304012dda..ac47e1ef3fa8 100644 --- a/hedera-node/hedera-smart-contract-service-impl/src/main/java/com/hedera/node/app/service/contract/impl/state/ProxyWorldUpdater.java +++ b/hedera-node/hedera-smart-contract-service-impl/src/main/java/com/hedera/node/app/service/contract/impl/state/ProxyWorldUpdater.java @@ -217,7 +217,7 @@ public Optional tryTransfer( @Override public Optional tryLazyCreation( @NonNull final Address recipient, @NonNull final MessageFrame frame) { - final var gasCost = enhancement.operations().lazyCreationCostInGas(); + final var gasCost = enhancement.operations().lazyCreationCostInGas(recipient); if (gasCost > frame.getRemainingGas()) { return Optional.of(INSUFFICIENT_GAS); } diff --git a/hedera-node/hedera-smart-contract-service-impl/src/test/java/com/hedera/node/app/service/contract/impl/test/exec/scope/HandleHederaOperationsTest.java b/hedera-node/hedera-smart-contract-service-impl/src/test/java/com/hedera/node/app/service/contract/impl/test/exec/scope/HandleHederaOperationsTest.java index 423257ac2d30..4a5a7ab83e19 100644 --- a/hedera-node/hedera-smart-contract-service-impl/src/test/java/com/hedera/node/app/service/contract/impl/test/exec/scope/HandleHederaOperationsTest.java +++ b/hedera-node/hedera-smart-contract-service-impl/src/test/java/com/hedera/node/app/service/contract/impl/test/exec/scope/HandleHederaOperationsTest.java @@ -38,6 +38,8 @@ import com.hedera.hapi.node.token.TokenCreateTransactionBody; import com.hedera.hapi.node.transaction.SignedTransaction; import com.hedera.hapi.node.transaction.TransactionBody; +import com.hedera.node.app.service.contract.impl.exec.gas.DispatchType; +import com.hedera.node.app.service.contract.impl.exec.gas.SystemContractGasCalculator; import com.hedera.node.app.service.contract.impl.exec.gas.TinybarValues; import com.hedera.node.app.service.contract.impl.exec.scope.HandleHederaOperations; import com.hedera.node.app.service.contract.impl.exec.scope.HederaOperations; @@ -93,12 +95,20 @@ class HandleHederaOperationsTest { @Mock private FeeCalculator feeCalculator; + @Mock + private SystemContractGasCalculator gasCalculator; + private HandleHederaOperations subject; @BeforeEach void setUp() { subject = new HandleHederaOperations( - DEFAULT_LEDGER_CONFIG, DEFAULT_CONTRACTS_CONFIG, context, tinybarValues, DEFAULT_HEDERA_CONFIG); + DEFAULT_LEDGER_CONFIG, + DEFAULT_CONTRACTS_CONFIG, + context, + tinybarValues, + gasCalculator, + DEFAULT_HEDERA_CONFIG); } @Test @@ -194,8 +204,13 @@ void commitIsNoopUntilSavepointExposesIt() { } @Test - void lazyCreationCostInGasHardcoded() { - assertEquals(1L, subject.lazyCreationCostInGas()); + void lazyCreationCostInGasTest() { + given(context.payer()).willReturn(A_NEW_ACCOUNT_ID); + given(gasCalculator.gasRequirement(any(), eq(DispatchType.CRYPTO_CREATE), eq(A_NEW_ACCOUNT_ID))) + .willReturn(6L); + given(gasCalculator.gasRequirement(any(), eq(DispatchType.CRYPTO_UPDATE), eq(A_NEW_ACCOUNT_ID))) + .willReturn(5L); + assertEquals(11L, subject.lazyCreationCostInGas(NON_SYSTEM_LONG_ZERO_ADDRESS)); } @Test diff --git a/hedera-node/hedera-smart-contract-service-impl/src/test/java/com/hedera/node/app/service/contract/impl/test/state/ProxyWorldUpdaterTest.java b/hedera-node/hedera-smart-contract-service-impl/src/test/java/com/hedera/node/app/service/contract/impl/test/state/ProxyWorldUpdaterTest.java index efeb4e16d723..8bd0c92a5a06 100644 --- a/hedera-node/hedera-smart-contract-service-impl/src/test/java/com/hedera/node/app/service/contract/impl/test/state/ProxyWorldUpdaterTest.java +++ b/hedera-node/hedera-smart-contract-service-impl/src/test/java/com/hedera/node/app/service/contract/impl/test/state/ProxyWorldUpdaterTest.java @@ -437,7 +437,7 @@ void delegatesTransfer() { @Test void abortsLazyCreationIfRemainingGasInsufficient() { final var pretendCost = 1_234L; - given(hederaOperations.lazyCreationCostInGas()).willReturn(pretendCost); + given(hederaOperations.lazyCreationCostInGas(SOME_EVM_ADDRESS)).willReturn(pretendCost); given(frame.getRemainingGas()).willReturn(pretendCost - 1); final var maybeHaltReason = subject.tryLazyCreation(SOME_EVM_ADDRESS, frame); assertTrue(maybeHaltReason.isPresent()); @@ -447,7 +447,7 @@ void abortsLazyCreationIfRemainingGasInsufficient() { @Test void delegatesLazyCreationAndDecrementsGasCostOnSuccess() { final var pretendCost = 1_234L; - given(hederaOperations.lazyCreationCostInGas()).willReturn(pretendCost); + given(hederaOperations.lazyCreationCostInGas(SOME_EVM_ADDRESS)).willReturn(pretendCost); given(frame.getRemainingGas()).willReturn(pretendCost * 2); given(evmFrameState.tryLazyCreation(SOME_EVM_ADDRESS)).willReturn(Optional.empty()); final var maybeHaltReason = subject.tryLazyCreation(SOME_EVM_ADDRESS, frame); @@ -459,7 +459,7 @@ void delegatesLazyCreationAndDecrementsGasCostOnSuccess() { void doesntBothDecrementingGasOnLazyCreationFailureSinceAboutToHalt() { final var pretendCost = 1_234L; final var haltReason = Optional.of(FAILURE_DURING_LAZY_ACCOUNT_CREATION); - given(hederaOperations.lazyCreationCostInGas()).willReturn(pretendCost); + given(hederaOperations.lazyCreationCostInGas(SOME_EVM_ADDRESS)).willReturn(pretendCost); given(frame.getRemainingGas()).willReturn(pretendCost * 2); given(evmFrameState.tryLazyCreation(SOME_EVM_ADDRESS)).willReturn(haltReason); final var maybeHaltReason = subject.tryLazyCreation(SOME_EVM_ADDRESS, frame); From 62604878ad3c0c9facf390d8a3a69ee110867619 Mon Sep 17 00:00:00 2001 From: Nathan Klick Date: Wed, 6 Dec 2023 19:44:43 -0600 Subject: [PATCH 5/7] fix(ci): disable Gradle configuration cache before executing Snyk (#10349) Signed-off-by: Nathan Klick --- .../node-zxc-compile-application-code.yaml | 16 +++++++++++++++- .github/workflows/node-zxf-snyk-monitor.yaml | 7 +++++-- .snyk | 6 +++--- 3 files changed, 23 insertions(+), 6 deletions(-) diff --git a/.github/workflows/node-zxc-compile-application-code.yaml b/.github/workflows/node-zxc-compile-application-code.yaml index b40156642b56..d0425cc4e6a4 100644 --- a/.github/workflows/node-zxc-compile-application-code.yaml +++ b/.github/workflows/node-zxc-compile-application-code.yaml @@ -459,6 +459,20 @@ jobs: }} run: ${GRADLE_EXEC} sonar --info --scan --no-parallel ${{ steps.sonar-cloud.outputs.options }} + - name: Disable Gradle Configuration Cache + if: >- + ${{ + inputs.enable-snyk-scan && + steps.gradle-build.conclusion == 'success' && + ( + github.event.pull_request.head.repo.full_name == github.repository || + github.event_name == 'push' || + github.event_name == 'workflow_dispatch' + ) && + !cancelled() + }} + run: sed -i 's/^org.gradle.configuration-cache=.*$/org.gradle.configuration-cache=false/' gradle.properties + - name: Setup Snyk env: SNYK_TOKEN: ${{ secrets.snyk-token }} @@ -490,7 +504,7 @@ jobs: ) && !cancelled() }} - run: ${CG_EXEC} snyk test --all-sub-projects --severity-threshold=high --json-file-output=snyk-test.json + run: ${CG_EXEC} snyk test --all-sub-projects --severity-threshold=high --policy-path=.snyk --json-file-output=snyk-test.json - name: Snyk Code id: snyk-code diff --git a/.github/workflows/node-zxf-snyk-monitor.yaml b/.github/workflows/node-zxf-snyk-monitor.yaml index cc40c02a3d57..3f6e4a99bb27 100644 --- a/.github/workflows/node-zxf-snyk-monitor.yaml +++ b/.github/workflows/node-zxf-snyk-monitor.yaml @@ -38,7 +38,7 @@ jobs: uses: actions/setup-java@0ab4596768b603586c0de567f2430c30f5b0d2b0 # v3.13.0 with: distribution: temurin - java-version: 17.0.3 + java-version: 17.0.9 - name: Setup Gradle uses: gradle/gradle-build-action@842c587ad8aa4c68eeba24c396e15af4c2e9f30a # v2.9.0 @@ -52,6 +52,9 @@ jobs: gradle-version: wrapper arguments: assemble --scan + - name: Disable Gradle Configuration Cache + run: sed -i 's/^org.gradle.configuration-cache=.*$/org.gradle.configuration-cache=false/' gradle.properties + - name: Setup NodeJS uses: actions/setup-node@8f152de45cc393bb48ce5d89d36b731f54556e65 # v4.0.0 with: @@ -66,4 +69,4 @@ jobs: continue-on-error: true env: SNYK_TOKEN: ${{ secrets.SNYK_TOKEN }} - run: snyk monitor --all-sub-projects + run: snyk monitor --all-sub-projects --policy-path=.snyk --trust-policies diff --git a/.snyk b/.snyk index b1e553196c2b..73acd34c6988 100644 --- a/.snyk +++ b/.snyk @@ -4,9 +4,9 @@ version: v1.25.0 ignore: SNYK-JAVA-IONETTY-5953332: - '*': - reason: No matching gRPC version is available at this time - expires: 2024-01-01T00:00:00.000Z - created: 2023-10-20T01:20:35.772Z + reason: No gRPC version with a fix is available + expires: 2024-03-30T00:00:00.000Z + created: 2023-12-06T23:35:31.268Z patch: {} exclude: global: From 644fd5eb0436cf7b76e1ccb6f4a38b442fd66cc3 Mon Sep 17 00:00:00 2001 From: jamesnguyentech <82211356+jamesnguyentech@users.noreply.github.com> Date: Thu, 7 Dec 2023 10:29:15 +0700 Subject: [PATCH 6/7] fix: 10227 use dev config with modrun (#10249) Signed-off-by: Nguyen Pham Truong Giang Signed-off-by: jamesnguyentech <82211356+jamesnguyentech@users.noreply.github.com> Co-authored-by: Nathan Klick --- hedera-node/hedera-app/build.gradle.kts | 36 ++++++++++++++++++++++--- 1 file changed, 32 insertions(+), 4 deletions(-) diff --git a/hedera-node/hedera-app/build.gradle.kts b/hedera-node/hedera-app/build.gradle.kts index 5d5ec948af60..496832c719f3 100644 --- a/hedera-node/hedera-app/build.gradle.kts +++ b/hedera-node/hedera-app/build.gradle.kts @@ -141,6 +141,32 @@ tasks.jar { } } +// Define build directory +val buildDir = layout.projectDirectory.dir("./build/node") + +// Copy everything from hedera-node/data +val copyNodeData = + tasks.register("copyNodeData") { + dependsOn(copyLib) + dependsOn(copyApp) + from(layout.projectDirectory.dir("../data/keys")) { into("data/keys") } + from(layout.projectDirectory.dir("../data")) + into(buildDir) + exclude("config", "keys") // Exclude config directory + shouldRunAfter(tasks.named("copyApp")) + shouldRunAfter(tasks.named("copyLib")) + } + +//// Copy hedera-node/configuration/dev as hedera-node/hedera-app/build/node/data/config } +val copyConfig = + tasks.register("copyConfig") { + from(layout.projectDirectory.dir("../configuration/dev")) { into("data/config") } + from(layout.projectDirectory.file("../config.txt")) + from(layout.projectDirectory.file("../log4j2.xml")) + into(buildDir) + shouldRunAfter(tasks.named("copyNodeData")) + } + // Copy dependencies into `data/lib` val copyLib = tasks.register("copyLib") { @@ -160,22 +186,24 @@ val copyApp = tasks.assemble { dependsOn(copyLib) dependsOn(copyApp) + dependsOn(copyNodeData) + dependsOn(copyConfig) } // Create the "run" task for running a Hedera consensus node tasks.register("run") { group = "application" dependsOn(tasks.assemble) - workingDir = layout.projectDirectory.dir("..").asFile - jvmArgs = listOf("-cp", "data/lib/*") + workingDir = layout.projectDirectory.dir("./build/node").asFile + jvmArgs = listOf("-cp", "lib/*") mainClass.set("com.swirlds.platform.Browser") } tasks.register("modrun") { group = "application" dependsOn(tasks.assemble) - workingDir = layout.projectDirectory.dir("..").asFile - jvmArgs = listOf("-cp", "data/lib/*:data/apps/*", "-Dhedera.workflows.enabled=true") + workingDir = layout.projectDirectory.dir("./build/node").asFile + jvmArgs = listOf("-cp", "lib/*:apps/*", "-Dhedera.workflows.enabled=true") mainClass.set("com.hedera.node.app.ServicesMain") } From 1d6aedd9417d43c469edb97df83401dc6f59f9d1 Mon Sep 17 00:00:00 2001 From: Nathan Klick Date: Wed, 6 Dec 2023 23:30:58 -0600 Subject: [PATCH 7/7] fix(ci): snyk workflow should not run on dependabot or forked pull requests (#10355) Signed-off-by: Nathan Klick --- .github/workflows/node-flow-pull-request-checks.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/node-flow-pull-request-checks.yaml b/.github/workflows/node-flow-pull-request-checks.yaml index f4dacd0d242c..edb73f99ec73 100644 --- a/.github/workflows/node-flow-pull-request-checks.yaml +++ b/.github/workflows/node-flow-pull-request-checks.yaml @@ -214,6 +214,7 @@ jobs: snyk-scan: name: Snyk Scan uses: ./.github/workflows/node-zxc-compile-application-code.yaml + if: ${{ github.actor != 'dependabot[bot]' && github.event.pull_request.head.repo.full_name == github.repository }} with: custom-job-label: Standard enable-unit-tests: false