Merge branch 'develop' into 16602-add-new-workflow-to-capture-logs-on…

…-xts-failure-and-report-to-slack

gradle/modules.properties

@@ -5,4 +5,9 @@
 # Consider contributing the missing entry back to the plugin by creating a PR for the 'modules.properties' linked above.
 com.hedera.cryptography.pairings.api=com.hedera.cryptography:hedera-cryptography-pairings-api
 com.hedera.cryptography.pairings.signatures=com.hedera.cryptography:hedera-cryptography-pairings-signatures
+com.hedera.cryptography.altbn128=com.hedera.cryptography:hedera-cryptography-altbn128
+com.hedera.cryptography.bls=com.hedera.cryptography:hedera-cryptography-bls
+com.hedera.cryptography.tss=com.hedera.cryptography:hedera-cryptography-tss
+com.hedera.cryptography.tss.impl=com.hedera.cryptography:hedera-cryptography-tss:impl
+com.hedera.cryptography.tss.api=com.hedera.cryptography:hedera-cryptography-tss:api
 jmh.core=org.openjdk.jmh:jmh-core

hapi/hedera-protobufs/block/stream/output/state_changes.proto

@@ -354,9 +354,10 @@ enum StateIdentifier {
     STATE_ID_ROSTERS = 28;
 
     /**
-    * A state identifier for scheduled transaction expiration.
+    * A state identifier for counts of transactions scheduled and
+    * processed in a second.
     */
-    STATE_ID_SCHEDULE_IDS_BY_EXPIRY = 29;
+    STATE_ID_SCHEDULED_COUNTS = 29;
 
     /**
      * A state identifier for scheduled transaction deduplication.
@@ -373,6 +374,16 @@ enum StateIdentifier {
      */
     STATE_ID_TSS_VOTES = 32;
 
+    /**
+    * A state identifier for the ordering of scheduled transactions.
+    */
+    STATE_ID_SCHEDULED_ORDERS = 33;
+
+    /**
+     * A state identifier for scheduled throttle usage snapshots.
+     */
+    STATE_ID_SCHEDULED_USAGES = 34;
+
     /**
      * A state identifier for the round receipts queue.
      */
@@ -711,6 +722,21 @@ message MapChangeKey {
          * A change to a virtual map keyed by pending airdrop id identifier.
          */
         proto.PendingAirdropId pending_airdrop_id_key = 14;
+
+        /**
+         * An exact date and time,  with a resolution of one second
+         */
+        proto.TimestampSeconds timestamp_seconds_key = 15;
+
+        /**
+         * An ordering key mapped to a particular schedule.<br/>
+         * This identifies the order in which long term scheduled transactions
+         * that are requested to execute in the same consensus second will
+         * be executed. The value will be the `ScheduleID` for the schedule
+         * to be executed at a particular consensus second and order within
+         * that second.
+         */
+        proto.ScheduledOrder scheduled_order_key = 16;
     }
 }
 
@@ -817,14 +843,20 @@ message MapChangeValue {
         com.hedera.hapi.node.state.roster.Roster roster_value = 16;
 
         /**
-         * A list of scheduled ids.
+         * The value of a map summarizing the counts of scheduled and processed transactions
+         * within a particular consensus second.
          */
-        proto.ScheduleIdList schedule_id_list_value = 17;
+        proto.ScheduledCounts scheduled_counts_value = 17;
 
         /**
          * A scheduled id value.
          */
         proto.ScheduleID schedule_id_value = 18;
+
+        /**
+         * A change to the scheduled throttle usage snapshots.
+         */
+        proto.ThrottleUsageSnapshots throttle_usage_snapshots_value = 19;
     }
 }
 

hapi/hedera-protobufs/services/response_code.proto

@@ -1593,4 +1593,30 @@ enum ResponseCodeEnum {
      * airdrop and whether the sender can fulfill the offer.
      */
     INVALID_TOKEN_IN_PENDING_AIRDROP = 369;
+
+    /**
+     * A scheduled transaction configured to wait for expiry to execute was given
+     * an expiry time not strictly after the time at which its creation reached
+     * consensus.
+     */
+    SCHEDULE_EXPIRY_MUST_BE_FUTURE = 370;
+
+    /**
+     * A scheduled transaction configured to wait for expiry to execute was given
+     * an expiry time too far in the future after the time at which its creation
+     * reached consensus.
+     */
+    SCHEDULE_EXPIRY_TOO_LONG = 371;
+
+    /**
+     * A scheduled transaction configured to wait for expiry to execute was given
+     * an expiry time at which there is already too many transactions scheduled to
+     * expire; its creation must be retried with a different expiry.
+     */
+    SCHEDULE_EXPIRY_IS_BUSY = 372;
+
+    /**
+     * The provided gRPC certificate hash is invalid.
+     */
+    INVALID_GRPC_CERTIFICATE_HASH = 373;
 }

hapi/hedera-protobufs/services/state/blockstream/block_stream_info.proto

@@ -155,4 +155,12 @@ message BlockStreamInfo {
      * at which an interval of time-dependent events were processed.
      */
     proto.Timestamp last_interval_process_time = 12;
+
+    /**
+     * The time stamp at which the last user transaction was handled.
+     * <p>
+     * This field SHALL hold the consensus time for the last time
+     * at which a user transaction was handled.
+     */
+    proto.Timestamp last_handle_time = 13;
 }

hapi/hedera-protobufs/services/state/schedule/schedule.proto

@@ -160,3 +160,33 @@ message ScheduleIdList {
    */
   repeated ScheduleID schedule_ids = 1;
 }
+
+/**
+ * The value of a map summarizing the counts of scheduled and processed transactions
+ * within a particular consensus second.
+ */
+message ScheduledCounts {
+    /**
+     * The number of transactions scheduled to expire at a consensus second.
+     */
+    uint32 number_scheduled = 1;
+    /**
+     * The number of scheduled transactions that have been processed at a consensus second.
+     */
+    uint32 number_processed = 2;
+}
+
+/**
+ * A key mapping to a particular ScheduleID that will execute at a given order number
+ * within a given consensus second.
+ */
+message ScheduledOrder {
+    /**
+     * The consensus second in which the transaction is expired.
+     */
+    uint64 expiry_second = 1;
+    /*
+     * The ordered position within the consensus second that the transaction will be executed.
+     */
+    uint32 order_number = 2;
+}

hedera-dependency-versions/build.gradle.kts

@@ -235,7 +235,19 @@ dependencies.constraints {
     api("uk.org.webcompere:system-stubs-jupiter:2.1.5") {
         because("uk.org.webcompere.systemstubs.jupiter")
     }
-
     api("com.google.protobuf:protoc:3.25.4")
     api("io.grpc:protoc-gen-grpc-java:1.66.0")
+
+    api("com.hedera.cryptography:hedera-cryptography-pairings-api:0.1.0-SNAPSHOT") {
+        because("com.hedera.cryptography.pairings.api")
+    }
+    api("com.hedera.cryptography:hedera-cryptography-altbn128:0.1.0-SNAPSHOT") {
+        because("com.hedera.cryptography.altbn128")
+    }
+    api("com.hedera.cryptography:hedera-cryptography-bls:0.1.0-SNAPSHOT") {
+        because("com.hedera.cryptography.bls")
+    }
+    api("com.hedera.cryptography:hedera-cryptography-tss:0.1.0-SNAPSHOT") {
+        because("com.hedera.cryptography.tss")
+    }
 }

hedera-node/hapi-utils/src/main/java/com/hedera/node/app/hapi/utils/sysfiles/domain/throttling/ScaleFactor.java

@@ -47,6 +47,14 @@ public int scaling(int nominalOps) {
         return Math.max(1, nominalOps * numerator / denominator);
     }
 
+    /**
+     * Returns the scale factor as an approximate 1:n split of capacity, rounding up.
+     * @return the approximate capacity split
+     */
+    public int asApproxCapacitySplit() {
+        return (denominator + numerator - 1) / numerator;
+    }
+
     @Override
     public int compareTo(final ScaleFactor that) {
         return Integer.compare(this.numerator * that.denominator, that.numerator * this.denominator);

hedera-node/hedera-addressbook-service-impl/src/main/java/com/hedera/node/app/service/addressbook/impl/handlers/NodeUpdateHandler.java

@@ -18,6 +18,7 @@
 
 import static com.hedera.hapi.node.base.ResponseCodeEnum.INVALID_ADMIN_KEY;
 import static com.hedera.hapi.node.base.ResponseCodeEnum.INVALID_GOSSIP_CA_CERTIFICATE;
+import static com.hedera.hapi.node.base.ResponseCodeEnum.INVALID_GRPC_CERTIFICATE_HASH;
 import static com.hedera.hapi.node.base.ResponseCodeEnum.INVALID_NODE_ACCOUNT_ID;
 import static com.hedera.hapi.node.base.ResponseCodeEnum.INVALID_NODE_ID;
 import static com.hedera.hapi.node.base.ResponseCodeEnum.UPDATE_NODE_ACCOUNT_NOT_ALLOWED;
@@ -75,6 +76,9 @@ public void pureChecks(@NonNull final TransactionBody txn) throws PreCheckExcept
             final var adminKey = op.adminKey();
             addressBookValidator.validateAdminKey(adminKey);
         }
+        if (op.hasGrpcCertificateHash()) {
+            validateFalsePreCheck(op.grpcCertificateHash().equals(Bytes.EMPTY), INVALID_GRPC_CERTIFICATE_HASH);
+        }
     }
 
     @Override

hedera-node/hedera-addressbook-service-impl/src/main/java/com/hedera/node/app/service/addressbook/impl/helpers/AddressBookHelper.java

@@ -46,10 +46,12 @@ public AddressBookHelper() {}
     /**
      * Adjusts the node metadata after upgrade. This method will mark nodes as deleted if they are not present in the
      * address book and add new nodes to the node store.
+     * <b>IMPORTANT:</b> Once DAB is enabled, should always be a no-op.
      * @param networkInfo the network info
      * @param config configuration
      * @param nodeStore the node store
      */
+    @Deprecated
     public void adjustPostUpgradeNodeMetadata(
             @NonNull final NetworkInfo networkInfo,
             @NonNull final Configuration config,

hedera-node/hedera-addressbook-service-impl/src/test/java/com/hedera/node/app/service/addressbook/impl/test/handlers/NodeUpdateHandlerTest.java

@@ -18,6 +18,7 @@
 
 import static com.hedera.hapi.node.base.ResponseCodeEnum.INVALID_ADMIN_KEY;
 import static com.hedera.hapi.node.base.ResponseCodeEnum.INVALID_GOSSIP_CA_CERTIFICATE;
+import static com.hedera.hapi.node.base.ResponseCodeEnum.INVALID_GRPC_CERTIFICATE_HASH;
 import static com.hedera.hapi.node.base.ResponseCodeEnum.INVALID_NODE_ACCOUNT_ID;
 import static com.hedera.hapi.node.base.ResponseCodeEnum.INVALID_NODE_ID;
 import static com.hedera.hapi.node.base.ResponseCodeEnum.UPDATE_NODE_ACCOUNT_NOT_ALLOWED;
@@ -122,6 +123,18 @@ void gossipCaCertificateCannotEmpty() {
         assertThat(msg.responseCode()).isEqualTo(INVALID_GOSSIP_CA_CERTIFICATE);
     }
 
+    @Test
+    @DisplayName("pureChecks fail when grpcCertHash is empty")
+    void grpcCertHashCannotEmpty() {
+        txn = new NodeUpdateBuilder()
+                .withNodeId(1)
+                .withAccountId(accountId)
+                .withGrpcCertificateHash(Bytes.EMPTY)
+                .build();
+        final var msg = assertThrows(PreCheckException.class, () -> subject.pureChecks(txn));
+        assertThat(msg.responseCode()).isEqualTo(INVALID_GRPC_CERTIFICATE_HASH);
+    }
+
     @Test
     @DisplayName("invalid adminKey fail")
     void adminKeyInvalid() {

hedera-node/hedera-app-spi/src/main/java/com/hedera/node/app/spi/AppContext.java

@@ -19,6 +19,7 @@
 import com.hedera.hapi.node.base.ResponseCodeEnum;
 import com.hedera.hapi.node.transaction.TransactionBody;
 import com.hedera.node.app.spi.signatures.SignatureVerifier;
+import com.hedera.node.app.spi.throttle.Throttle;
 import com.swirlds.common.crypto.Signature;
 import com.swirlds.config.api.Configuration;
 import com.swirlds.state.lifecycle.Service;
@@ -103,4 +104,10 @@ public Signature sign(final byte[] ledgerId) {
      * @return the supplier
      */
     Supplier<NodeInfo> selfNodeInfoSupplier();
+
+    /**
+     * The application's strategy for creating {@link Throttle} instances.
+     * @return the throttle factory
+     */
+    Throttle.Factory throttleFactory();
 }

hedera-node/hedera-app-spi/src/main/java/com/hedera/node/app/spi/throttle/Throttle.java

@@ -0,0 +1,64 @@
+/*
+ * Copyright (C) 2024 Hedera Hashgraph, LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.hedera.node.app.spi.throttle;
+
+import com.hedera.hapi.node.base.AccountID;
+import com.hedera.hapi.node.base.HederaFunctionality;
+import com.hedera.hapi.node.state.throttles.ThrottleUsageSnapshots;
+import com.hedera.hapi.node.transaction.TransactionBody;
+import com.hedera.node.app.spi.AppContext;
+import edu.umd.cs.findbugs.annotations.NonNull;
+import edu.umd.cs.findbugs.annotations.Nullable;
+import java.time.Instant;
+
+/**
+ * A throttle that can be used to limit the rate of transactions. Provided in the {@link AppContext} so that services
+ * can align to the application's strategy for throttling transactions.
+ */
+public interface Throttle {
+    /**
+     * A factory for creating {@link Throttle} instances.
+     */
+    interface Factory {
+        /**
+         * Creates a new throttle based on the capacity split and usage snapshots.
+         * @param capacitySplit the split of the capacity
+         * @param initialUsageSnapshots if not null, the usage snapshots the throttle should start with
+         * @return the new throttle
+         */
+        Throttle newThrottle(int capacitySplit, @Nullable ThrottleUsageSnapshots initialUsageSnapshots);
+    }
+
+    /**
+     * Tries to consume throttle capacity for the given payer, transaction, function, time, and state.
+     * @param payerId the account ID of the payer
+     * @param body the transaction body
+     * @param function the functionality of the transaction
+     * @param now the current time
+     * @return whether the capacity could be consumed
+     */
+    boolean allow(
+            @NonNull AccountID payerId,
+            @NonNull TransactionBody body,
+            @NonNull HederaFunctionality function,
+            @NonNull Instant now);
+
+    /**
+     * Returns the usage snapshots of the throttle.
+     */
+    ThrottleUsageSnapshots usageSnapshots();
+}

hedera-node/hedera-app/build.gradle.kts

@@ -33,6 +33,7 @@ mainModuleInfo {
     runtimeOnly("io.helidon.grpc.core")
     runtimeOnly("io.helidon.webclient")
     runtimeOnly("io.helidon.webclient.grpc")
+    runtimeOnly("com.hedera.cryptography.altbn128")
 }
 
 testModuleInfo {

hedera-node/hedera-app/src/jmh/java/com/hedera/node/app/blocks/BlockStreamManagerBenchmark.java

@@ -44,9 +44,10 @@
 import com.hedera.node.app.config.ConfigProviderImpl;
 import com.hedera.node.app.fixtures.state.FakeState;
 import com.hedera.node.app.services.AppContextImpl;
+import com.hedera.node.app.spi.AppContext;
 import com.hedera.node.app.spi.signatures.SignatureVerifier;
-import com.hedera.node.app.tss.PlaceholderTssLibrary;
 import com.hedera.node.app.tss.TssBaseServiceImpl;
+import com.hedera.node.app.tss.TssLibraryImpl;
 import com.hedera.node.config.ConfigProvider;
 import com.hedera.pbj.runtime.OneOf;
 import com.hedera.pbj.runtime.ParseException;
@@ -117,16 +118,20 @@ public static void main(String... args) throws Exception {
                     "blockStream.hashCombineBatchSize", "64",
                     "blockStream.serializationBatchSize", "32"));
     private final List<BlockItem> roundItems = new ArrayList<>();
+    final AppContext appContext = new AppContextImpl(
+            Instant::now,
+            fakeSignatureVerifier(),
+            UNAVAILABLE_GOSSIP,
+            configProvider::getConfiguration,
+            () -> DEFAULT_NODE_INFO,
+            (split, snapshots) -> {
+                throw new UnsupportedOperationException();
+            });
     private final TssBaseServiceImpl tssBaseService = new TssBaseServiceImpl(
-            new AppContextImpl(
-                    Instant::now,
-                    fakeSignatureVerifier(),
-                    UNAVAILABLE_GOSSIP,
-                    () -> configProvider.getConfiguration(),
-                    () -> DEFAULT_NODE_INFO),
+            appContext,
             ForkJoinPool.commonPool(),
             ForkJoinPool.commonPool(),
-            new PlaceholderTssLibrary(),
+            new TssLibraryImpl(appContext),
             ForkJoinPool.commonPool(),
             new NoOpMetrics());
     private final BlockStreamManagerImpl subject = new BlockStreamManagerImpl(