diff --git a/hapi/hedera-protobufs/services/response_code.proto b/hapi/hedera-protobufs/services/response_code.proto index 6945f591b420..38d1d5f94c52 100644 --- a/hapi/hedera-protobufs/services/response_code.proto +++ b/hapi/hedera-protobufs/services/response_code.proto @@ -1619,4 +1619,10 @@ enum ResponseCodeEnum { * The provided gRPC certificate hash is invalid. */ INVALID_GRPC_CERTIFICATE_HASH = 373; + + /** + * A scheduled transaction configured to wait for expiry to execute was not + * given an explicit expiration time. + */ + MISSING_EXPIRY_TIME = 374; } diff --git a/hedera-node/hedera-schedule-service-impl/src/main/java/com/hedera/node/app/service/schedule/impl/handlers/ScheduleCreateHandler.java b/hedera-node/hedera-schedule-service-impl/src/main/java/com/hedera/node/app/service/schedule/impl/handlers/ScheduleCreateHandler.java index eaba5519a14e..f89f57fad2ce 100644 --- a/hedera-node/hedera-schedule-service-impl/src/main/java/com/hedera/node/app/service/schedule/impl/handlers/ScheduleCreateHandler.java +++ b/hedera-node/hedera-schedule-service-impl/src/main/java/com/hedera/node/app/service/schedule/impl/handlers/ScheduleCreateHandler.java @@ -23,6 +23,7 @@ import static com.hedera.hapi.node.base.ResponseCodeEnum.INVALID_TRANSACTION_BODY; import static com.hedera.hapi.node.base.ResponseCodeEnum.MAX_ENTITIES_IN_PRICE_REGIME_HAVE_BEEN_CREATED; import static com.hedera.hapi.node.base.ResponseCodeEnum.MEMO_TOO_LONG; +import static com.hedera.hapi.node.base.ResponseCodeEnum.MISSING_EXPIRY_TIME; import static com.hedera.hapi.node.base.ResponseCodeEnum.SCHEDULED_TRANSACTION_NOT_IN_WHITELIST; import static com.hedera.hapi.node.base.ResponseCodeEnum.SCHEDULE_EXPIRY_IS_BUSY; import static com.hedera.hapi.node.base.ResponseCodeEnum.SCHEDULE_EXPIRY_MUST_BE_FUTURE; @@ -45,6 +46,8 @@ import com.hedera.hapi.node.scheduled.SchedulableTransactionBody; import com.hedera.hapi.node.scheduled.ScheduleCreateTransactionBody; import com.hedera.hapi.node.state.schedule.Schedule; +import com.hedera.hapi.node.state.schedule.ScheduledOrder; +import com.hedera.hapi.node.state.throttles.ThrottleUsageSnapshots; import com.hedera.hapi.node.transaction.TransactionBody; import com.hedera.node.app.hapi.fees.usage.SigUsage; import com.hedera.node.app.hapi.fees.usage.schedule.ScheduleOpsUsage; @@ -72,12 +75,16 @@ import java.util.Objects; import javax.inject.Inject; import javax.inject.Singleton; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; /** * This class contains all workflow-related functionality regarding {@link HederaFunctionality#SCHEDULE_CREATE}. */ @Singleton public class ScheduleCreateHandler extends AbstractScheduleHandler implements TransactionHandler { + private static final Logger log = LogManager.getLogger(ScheduleCreateHandler.class); + private final ScheduleOpsUsage scheduleOpsUsage = new ScheduleOpsUsage(); private final InstantSource instantSource; private final Throttle.Factory throttleFactory; @@ -96,7 +103,7 @@ public void pureChecks(@NonNull final TransactionBody body) throws PreCheckExcep final var op = body.scheduleCreateOrThrow(); validateTruePreCheck(op.hasScheduledTransactionBody(), INVALID_TRANSACTION); // (FUTURE) Add a dedicated response code for an op waiting for an unspecified expiration time - validateFalsePreCheck(op.waitForExpiry() && !op.hasExpirationTime(), INVALID_TRANSACTION); + validateFalsePreCheck(op.waitForExpiry() && !op.hasExpirationTime(), MISSING_EXPIRY_TIME); } @Override @@ -195,7 +202,8 @@ public void handle(@NonNull final HandleContext context) throws HandleException MAX_ENTITIES_IN_PRICE_REGIME_HAVE_BEEN_CREATED); final var capacityFraction = schedulingConfig.schedulableCapacityFraction(); final var usageSnapshots = scheduleStore.usageSnapshotsForScheduled(then); - final var throttle = throttleFactory.newThrottle(capacityFraction.asApproxCapacitySplit(), usageSnapshots); + final var throttle = + upToDateThrottle(then, capacityFraction.asApproxCapacitySplit(), usageSnapshots, scheduleStore); validateTrue( throttle.allow( provisionalSchedule.payerAccountIdOrThrow(), @@ -305,4 +313,46 @@ private HederaFunctionality functionOf(@NonNull final Schedule schedule) { return functionalityForType( schedule.scheduledTransactionOrThrow().data().kind()); } + + /** + * Attempts to recover a throttle from the given usage snapshots, or creates a new throttle if the recovery fails. + * (This edge case can occur if the network throttle definitions changed since a transaction was last scheduled + * in the given second and snapshots were taken.) + * @param then the second for which the throttle is being recovered + * @param capacitySplit the capacity split for the throttle + * @param usageSnapshots the usage snapshots to recover from + * @return the throttle + */ + private Throttle upToDateThrottle( + final long then, + final int capacitySplit, + @Nullable final ThrottleUsageSnapshots usageSnapshots, + @NonNull final WritableScheduleStore scheduleStore) { + requireNonNull(scheduleStore); + try { + return throttleFactory.newThrottle(capacitySplit, usageSnapshots); + } catch (Exception e) { + final var instantThen = Instant.ofEpochSecond(then); + log.info( + "Could not recreate throttle at {} from {} ({}), rebuilding with up-to-date throttle", + instantThen, + usageSnapshots, + e.getMessage()); + final var throttle = throttleFactory.newThrottle(capacitySplit, null); + final var counts = requireNonNull(scheduleStore.scheduledCountsAt(then)); + final int n = counts.numberScheduled(); + for (int i = 0; i < n; i++) { + final var scheduleId = requireNonNull(scheduleStore.getByOrder(new ScheduledOrder(then, i))); + final var schedule = requireNonNull(scheduleStore.get(scheduleId)); + // Consume capacity from every already-scheduled transaction in the new throttle + throttle.allow( + schedule.payerAccountIdOrThrow(), + childAsOrdinary(schedule), + functionOf(schedule), + Instant.ofEpochSecond(then)); + } + log.info("Rebuilt throttle at {} from {} scheduled transactions", instantThen, n); + return throttle; + } + } } diff --git a/hedera-node/test-clients/src/main/java/com/hedera/services/bdd/suites/hip423/ScheduleLongTermSignTest.java b/hedera-node/test-clients/src/main/java/com/hedera/services/bdd/suites/hip423/ScheduleLongTermSignTest.java index 264a3129494a..ff58c604ebf6 100644 --- a/hedera-node/test-clients/src/main/java/com/hedera/services/bdd/suites/hip423/ScheduleLongTermSignTest.java +++ b/hedera-node/test-clients/src/main/java/com/hedera/services/bdd/suites/hip423/ScheduleLongTermSignTest.java @@ -702,6 +702,7 @@ public Stream sharedKeyWorksAsExpected() { @Order(15) public Stream overlappingKeysTreatedAsExpected() { var keyGen = OverlappingKeyGenerator.withAtLeastOneOverlappingByte(2); + final long scheduleLifetime = 6; return defaultHapiSpec("OverlappingKeysTreatedAsExpectedAtExpiry") .given( @@ -717,7 +718,7 @@ public Stream overlappingKeysTreatedAsExpected() { tinyBarsFromTo("aSender", ADDRESS_BOOK_CONTROL, 1), tinyBarsFromTo("cSender", ADDRESS_BOOK_CONTROL, 1))) .waitForExpiry() - .withRelativeExpiry(SENDER_TXN, 5) + .withRelativeExpiry(SENDER_TXN, scheduleLifetime) .recordingScheduledTxn()) .then( scheduleSign(DEFERRED_XFER).alsoSigningWith("aKey"), @@ -743,9 +744,9 @@ public Stream overlappingKeysTreatedAsExpected() { .hasWaitForExpiry() .isNotExecuted() .isNotDeleted() - .hasRelativeExpiry(SENDER_TXN, 5) + .hasRelativeExpiry(SENDER_TXN, scheduleLifetime) .hasRecordedScheduledTxn(), - sleepFor(TimeUnit.SECONDS.toMillis(6)), + sleepFor(TimeUnit.SECONDS.toMillis(scheduleLifetime)), cryptoCreate("foo"), getScheduleInfo(DEFERRED_XFER).hasCostAnswerPrecheck(INVALID_SCHEDULE_ID), getAccountBalance(ADDRESS_BOOK_CONTROL).hasTinyBars(changeFromSnapshot(BEFORE, +2))); diff --git a/hedera-node/test-clients/src/main/java/com/hedera/services/bdd/suites/integration/RepeatableHip423Tests.java b/hedera-node/test-clients/src/main/java/com/hedera/services/bdd/suites/integration/RepeatableHip423Tests.java index 278149ea1a33..98eeb0bf52cf 100644 --- a/hedera-node/test-clients/src/main/java/com/hedera/services/bdd/suites/integration/RepeatableHip423Tests.java +++ b/hedera-node/test-clients/src/main/java/com/hedera/services/bdd/suites/integration/RepeatableHip423Tests.java @@ -52,6 +52,7 @@ import static com.hedera.services.bdd.spec.utilops.UtilVerbs.newKeyNamed; import static com.hedera.services.bdd.spec.utilops.UtilVerbs.overriding; import static com.hedera.services.bdd.spec.utilops.UtilVerbs.overridingAllOf; +import static com.hedera.services.bdd.spec.utilops.UtilVerbs.overridingThrottles; import static com.hedera.services.bdd.spec.utilops.UtilVerbs.sleepFor; import static com.hedera.services.bdd.spec.utilops.UtilVerbs.sleepForSeconds; import static com.hedera.services.bdd.spec.utilops.UtilVerbs.sourcing; @@ -66,6 +67,7 @@ import static com.hedera.services.bdd.suites.HapiSuite.ONE_MILLION_HBARS; import static com.hederahashgraph.api.proto.java.HederaFunctionality.ConsensusCreateTopic; import static com.hederahashgraph.api.proto.java.ResponseCodeEnum.BUSY; +import static com.hederahashgraph.api.proto.java.ResponseCodeEnum.MISSING_EXPIRY_TIME; import static com.hederahashgraph.api.proto.java.ResponseCodeEnum.SCHEDULE_EXPIRY_IS_BUSY; import static com.hederahashgraph.api.proto.java.ResponseCodeEnum.SCHEDULE_EXPIRY_MUST_BE_FUTURE; import static com.hederahashgraph.api.proto.java.ResponseCodeEnum.SCHEDULE_EXPIRY_TOO_LONG; @@ -179,7 +181,10 @@ final Stream expiryMustBeValid() { exposeSpecSecondTo(lastSecond::set), sourcing(() -> scheduleCreate("tooLate", cryptoTransfer(tinyBarsFromTo(DEFAULT_PAYER, FUNDING, 34L))) .expiringAt(lastSecond.get() + 1 + ONE_MINUTE + 1) - .hasKnownStatus(SCHEDULE_EXPIRY_TOO_LONG))); + .hasKnownStatus(SCHEDULE_EXPIRY_TOO_LONG)), + scheduleCreate("unspecified", cryptoTransfer(tinyBarsFromTo(DEFAULT_PAYER, FUNDING, 56L))) + .waitForExpiry() + .hasPrecheck(MISSING_EXPIRY_TIME)); } /** @@ -230,6 +235,64 @@ final Stream throttlingAndExecutionAsExpected() { sourcingContextual(spec -> purgeExpiringWithin(maxLifetime.get()))); } + /** + * Tests that the consensus {@link com.hedera.hapi.node.base.HederaFunctionality#SCHEDULE_CREATE} throttle is + * enforced by overriding the dev throttles to the more restrictive mainnet throttles and scheduling one more + * {@link com.hedera.hapi.node.base.HederaFunctionality#CONSENSUS_CREATE_TOPIC} that is allowed. + */ + @LeakyRepeatableHapiTest( + value = { + NEEDS_LAST_ASSIGNED_CONSENSUS_TIME, + NEEDS_VIRTUAL_TIME_FOR_FAST_EXECUTION, + NEEDS_STATE_ACCESS, + THROTTLE_OVERRIDES + }, + overrides = { + "scheduling.whitelist", + }, + throttles = "testSystemFiles/mainnet-throttles-sans-reservations.json") + final Stream throttlingRebuiltForSecondWhenSnapshotsNoLongerMatch() { + final var expirySecond = new AtomicLong(); + final var maxLifetime = new AtomicLong(); + final var maxSchedulableTopicCreates = new AtomicInteger(); + return hapiTest( + overriding("scheduling.whitelist", "ConsensusCreateTopic"), + doWithStartupConfigNow( + "scheduling.maxExpirationFutureSeconds", + (value, specTime) -> doAdhoc(() -> { + maxLifetime.set(Long.parseLong(value)); + expirySecond.set(specTime.getEpochSecond() + maxLifetime.get()); + })), + cryptoCreate(CIVILIAN_PAYER).balance(ONE_MILLION_HBARS), + exposeMaxSchedulable(ConsensusCreateTopic, maxSchedulableTopicCreates::set), + // Schedule one fewer than the maximum number of topic creations allowed using + // the initial throttles without the PriorityReservations bucket + sourcing(() -> blockingOrder(IntStream.range(0, maxSchedulableTopicCreates.get() - 1) + .mapToObj(i -> scheduleCreate( + "topic" + i, createTopic("t" + i).topicMemo("m" + i)) + .expiringAt(expirySecond.get()) + .payingWith(CIVILIAN_PAYER) + .fee(ONE_HUNDRED_HBARS)) + .toArray(SpecOperation[]::new))), + // Now override the throttles to the mainnet throttles with the PriorityReservations bucket + // (so that the throttle snapshots in state for this second don't match the new throttles) + overridingThrottles("testSystemFiles/mainnet-throttles.json"), + // And confirm we can schedule one more + sourcing(() -> scheduleCreate( + "lastTopicCreation", createTopic("oneMore").topicMemo("N-1")) + .expiringAt(expirySecond.get()) + .payingWith(CIVILIAN_PAYER) + .fee(ONE_HUNDRED_HBARS)), + // But then the next is throttled + sourcing(() -> scheduleCreate( + "throttledTopicCreation", createTopic("NTB").topicMemo("NOPE")) + .expiringAt(expirySecond.get()) + .payingWith(CIVILIAN_PAYER) + .fee(ONE_HUNDRED_HBARS) + .hasKnownStatus(SCHEDULE_EXPIRY_IS_BUSY)), + sourcingContextual(spec -> purgeExpiringWithin(maxLifetime.get()))); + } + /** * Tests that execution of scheduled transactions purges the associated state as expected when a single * user transaction fully executes multiple seconds. The test uses three scheduled transactions, two of diff --git a/hedera-node/test-clients/src/main/java/com/hedera/services/bdd/suites/queries/AsNodeOperatorQueriesTest.java b/hedera-node/test-clients/src/main/java/com/hedera/services/bdd/suites/queries/AsNodeOperatorQueriesTest.java index 57d7f56aca7c..201caa17d5f1 100644 --- a/hedera-node/test-clients/src/main/java/com/hedera/services/bdd/suites/queries/AsNodeOperatorQueriesTest.java +++ b/hedera-node/test-clients/src/main/java/com/hedera/services/bdd/suites/queries/AsNodeOperatorQueriesTest.java @@ -192,7 +192,7 @@ final Stream nodeOperatorAccountInfoQueryCharged() { return hapiTest(flattened( nodeOperatorAccount(), getAccountInfo(NODE_OPERATOR).payingWith(NODE_OPERATOR).via("accountInfoQueryTxn"), - sleepFor(1000), + sleepFor(2000), getAccountBalance(NODE_OPERATOR).hasTinyBars(lessThan(ONE_HUNDRED_HBARS)))); } diff --git a/hedera-node/test-clients/src/main/resources/testSystemFiles/mainnet-throttles-sans-reservations.json b/hedera-node/test-clients/src/main/resources/testSystemFiles/mainnet-throttles-sans-reservations.json new file mode 100644 index 000000000000..763fd0d8328d --- /dev/null +++ b/hedera-node/test-clients/src/main/resources/testSystemFiles/mainnet-throttles-sans-reservations.json @@ -0,0 +1,173 @@ +{ + "buckets": [ + { + "burstPeriod": 0, + "burstPeriodMs": 1000, + "name": "ThroughputLimits", + "throttleGroups": [ + { + "opsPerSec": 0, + "milliOpsPerSec": 10500000, + "operations": [ + "NodeCreate", + "NodeDelete", + "NodeUpdate", + "ScheduleCreate", + "CryptoCreate", + "CryptoTransfer", + "CryptoUpdate", + "CryptoDelete", + "CryptoGetInfo", + "CryptoGetAccountRecords", + "ConsensusCreateTopic", + "ConsensusSubmitMessage", + "ConsensusUpdateTopic", + "ConsensusDeleteTopic", + "ConsensusGetTopicInfo", + "TokenGetNftInfo", + "TokenGetInfo", + "ScheduleDelete", + "ScheduleGetInfo", + "FileGetContents", + "FileGetInfo", + "ContractUpdate", + "ContractDelete", + "ContractGetInfo", + "ContractGetBytecode", + "ContractGetRecords", + "ContractCallLocal", + "TransactionGetRecord", + "GetVersionInfo", + "TokenGetAccountNftInfos", + "TokenGetNftInfos", + "CryptoApproveAllowance", + "CryptoDeleteAllowance", + "UtilPrng" + ] + }, + { + "opsPerSec": 0, + "milliOpsPerSec": 13000, + "operations": [ + "FileCreate", + "FileUpdate", + "FileAppend", + "FileDelete" + ] + }, + { + "opsPerSec": 0, + "milliOpsPerSec": 100000, + "operations": [ + "ScheduleSign" + ] + }, + { + "opsPerSec": 0, + "milliOpsPerSec": 125000, + "operations": [ + "TokenMint" + ] + }, + { + "opsPerSec": 0, + "milliOpsPerSec": 350000, + "operations": [ + "ContractCall", + "ContractCreate", + "EthereumTransaction" + ] + }, + { + "opsPerSec": 0, + "milliOpsPerSec": 3000000, + "operations": [ + "TokenCreate", + "TokenDelete", + "TokenBurn", + "TokenUpdate", + "TokenFeeScheduleUpdate", + "TokenAssociateToAccount", + "TokenAccountWipe", + "TokenDissociateFromAccount", + "TokenFreezeAccount", + "TokenUnfreezeAccount", + "TokenGrantKycToAccount", + "TokenRevokeKycFromAccount", + "TokenPause", + "TokenUnpause", + "TokenUpdateNfts", + "TokenReject", + "TokenAirdrop", + "TokenClaimAirdrop", + "TokenCancelAirdrop" + ] + } + ] + }, + { + "burstPeriod": 0, + "burstPeriodMs": 1000, + "name": "OffHeapQueryLimits", + "throttleGroups": [ + { + "opsPerSec": 0, + "milliOpsPerSec": 700000, + "operations": [ + "FileGetContents", + "FileGetInfo", + "ContractGetInfo", + "ContractGetBytecode", + "ContractCallLocal" + ] + } + ] + }, + { + "burstPeriod": 0, + "burstPeriodMs": 1000, + "name": "CreationLimits", + "throttleGroups": [ + { + "opsPerSec": 0, + "milliOpsPerSec": 2000, + "operations": [ + "CryptoCreate" + ] + }, + { + "opsPerSec": 0, + "milliOpsPerSec": 5000, + "operations": [ + "ConsensusCreateTopic" + ] + }, + { + "opsPerSec": 0, + "milliOpsPerSec": 100000, + "operations": [ + "TokenCreate", + "TokenAssociateToAccount", + "ScheduleCreate", + "TokenAirdrop" + ] + } + ] + }, + { + "burstPeriod": 0, + "burstPeriodMs": 1000, + "name": "FreeQueryLimits", + "throttleGroups": [ + { + "opsPerSec": 0, + "milliOpsPerSec": 1000000000, + "operations": [ + "CryptoGetAccountBalance", + "TransactionGetReceipt" + ] + } + ] + } + ] +}