Merge branch 'develop' into 09445-D-ci-performance-testing

.github/workflows/node-flow-fsts-daily-interval-01.yaml

@@ -39,35 +39,14 @@ concurrency:
   group: ${{ github.event.inputs.concurrency-group || format('{0}-{1}-flow-jrs-daily-interval-groups', github.ref_name, github.sha) }}
 
 # Panel Definition & Timings:
-#   "configs/services/suites/daily/GCP-Daily-Services-Crypto-Update-Ubuntu1804-4N-2C.json"    ----         90m
 #   "configs/services/suites/daily/GCP-Daily-Services-Crypto-Update-Rhel7-4N-2C.json"         ----         90m
 #   "configs/services/suites/daily/GCP-Daily-Services-Crypto-Update-Rhel8-4N-2C.json"         ----         90m
 #   "configs/services/suites/daily/GCP-Daily-Services-Crypto-Update-CentOS7-4N-2C.json"       ----         90m
 #   "configs/services/suites/daily/GCP-Daily-Services-Crypto-Update-4N-2C.json"               ----         90m
 #   "configs/services/suites/daily/GCP-Daily-Services-Crypto-Update-Ubuntu2204-4N-2C.json"    ----         90m
-#   TOTAL RUN TIME:             540 minutes, 9 hours
+#   TOTAL RUN TIME:             450 minutes, 7.5 hours
 
 jobs:
-  crypto-update-ubuntu1804-4n-2c:
-    name: Crypto-Update-Ubuntu1804-4N-2C
-    uses: ./.github/workflows/zxc-jrs-regression.yaml
-    with:
-      ref: ${{ github.event.inputs.ref }}
-      branch-name: ${{ github.event.inputs.branch-name }}
-      hedera-tests-enabled: true
-      use-branch-for-slack-channel: true
-      panel-config: "configs/services/suites/daily/GCP-Daily-Services-Crypto-Update-Ubuntu1804-4N-2C.json"
-    secrets:
-      access-token: ${{ secrets.PLATFORM_GH_ACCESS_TOKEN }}
-      jrs-ssh-user-name: ${{ secrets.PLATFORM_JRS_SSH_USER_NAME }}
-      jrs-ssh-key-file: ${{ secrets.PLATFORM_JRS_SSH_KEY_FILE }}
-      gcp-project-number: ${{ secrets.PLATFORM_GCP_PROJECT_NUMBER }}
-      gcp-sa-key-contents: ${{ secrets.PLATFORM_GCP_KEY_FILE }}
-      slack-api-token: ${{ secrets.PLATFORM_SLACK_API_TOKEN }}
-      gradle-cache-username: ${{ secrets.GRADLE_CACHE_USERNAME }}
-      gradle-cache-password: ${{ secrets.GRADLE_CACHE_PASSWORD }}
-    if: ${{ !cancelled() && always() }}
-
 
   crypto-update-rhel7-4n-2c:
     name: Crypto-Update-Rhel7-4N-2C

.github/workflows/node-flow-fsts-daily-regression.yaml

@@ -56,24 +56,24 @@ jobs:
     secrets:
       access-token: ${{ secrets.PLATFORM_GH_ACCESS_TOKEN }}
 
-#  interval-03:
-#    name: Interval 3
-#    uses: ./.github/workflows/platform-zxc-launch-jrs-workflow.yaml
-#    needs:
-#      - interval-02
-#    with:
-#      ref: ${{ github.event.inputs.ref || github.sha }}
-#      branch-name: ${{ github.event.inputs.branch-name || github.ref_name }}
-#      workflow-file: node-flow-fsts-daily-interval-03.yaml
-#      concurrency-group: ${{ github.event.inputs.ref || github.sha }}-node-flow-fsts-daily-group-03
-#    secrets:
-#      access-token: ${{ secrets.PLATFORM_GH_ACCESS_TOKEN }}
+  interval-03:
+    name: Interval 3
+    uses: ./.github/workflows/platform-zxc-launch-jrs-workflow.yaml
+    needs:
+      - interval-02
+    with:
+      ref: ${{ github.event.inputs.ref || github.sha }}
+      branch-name: ${{ github.event.inputs.branch-name || github.ref_name }}
+      workflow-file: node-flow-fsts-daily-interval-03.yaml
+      concurrency-group: ${{ github.event.inputs.ref || github.sha }}-node-flow-fsts-daily-group-03
+    secrets:
+      access-token: ${{ secrets.PLATFORM_GH_ACCESS_TOKEN }}
 
   interval-04:
     name: Interval 4
     uses: ./.github/workflows/platform-zxc-launch-jrs-workflow.yaml
     needs:
-      - interval-02
+      - interval-03
     with:
       ref: ${{ github.event.inputs.ref || github.sha }}
       branch-name: ${{ github.event.inputs.branch-name || github.ref_name }}

.github/workflows/node-flow-pull-request-checks.yaml

@@ -214,6 +214,7 @@ jobs:
   snyk-scan:
     name: Snyk Scan
     uses: ./.github/workflows/node-zxc-compile-application-code.yaml
+    if: ${{ github.actor != 'dependabot[bot]' && github.event.pull_request.head.repo.full_name == github.repository }}
     with:
       custom-job-label: Standard
       enable-unit-tests: false

hedera-node/hedera-app-spi/src/main/java/com/hedera/node/app/spi/workflows/record/GenesisRecordsBuilder.java

@@ -17,9 +17,8 @@
 package com.hedera.node.app.spi.workflows.record;
 
 import com.hedera.hapi.node.state.token.Account;
-import com.hedera.hapi.node.token.CryptoCreateTransactionBody;
 import edu.umd.cs.findbugs.annotations.NonNull;
-import java.util.Map;
+import java.util.SortedSet;
 
 /**
  * A class that stores entities created during node startup, for the purpose of creating synthetic
@@ -29,25 +28,25 @@ public interface GenesisRecordsBuilder {
     /**
      * Tracks the system accounts created during node startup
      */
-    void systemAccounts(@NonNull final Map<Account, CryptoCreateTransactionBody.Builder> accounts);
+    void systemAccounts(@NonNull final SortedSet<Account> accounts);
 
     /**
      * Tracks the staking accounts created during node startup
      */
-    void stakingAccounts(@NonNull final Map<Account, CryptoCreateTransactionBody.Builder> accounts);
+    void stakingAccounts(@NonNull final SortedSet<Account> accounts);
 
     /**
      * Tracks miscellaneous accounts created during node startup. These accounts are typically used for testing
      */
-    void miscAccounts(@NonNull final Map<Account, CryptoCreateTransactionBody.Builder> accounts);
+    void miscAccounts(@NonNull final SortedSet<Account> accounts);
 
     /**
      * Tracks the treasury clones created during node startup
      */
-    void treasuryClones(@NonNull final Map<Account, CryptoCreateTransactionBody.Builder> accounts);
+    void treasuryClones(@NonNull final SortedSet<Account> accounts);
 
     /**
      * Tracks the blocklist accounts created during node startup
      */
-    void blocklistAccounts(@NonNull final Map<Account, CryptoCreateTransactionBody.Builder> accounts);
+    void blocklistAccounts(@NonNull final SortedSet<Account> accounts);
 }

hedera-node/hedera-app-spi/src/testFixtures/java/com/hedera/node/app/spi/fixtures/state/NoOpGenesisRecordsBuilder.java

@@ -33,34 +33,33 @@
  */
 
 import com.hedera.hapi.node.state.token.Account;
-import com.hedera.hapi.node.token.CryptoCreateTransactionBody;
 import com.hedera.node.app.spi.workflows.record.GenesisRecordsBuilder;
 import edu.umd.cs.findbugs.annotations.NonNull;
-import java.util.Map;
+import java.util.SortedSet;
 
 public class NoOpGenesisRecordsBuilder implements GenesisRecordsBuilder {
     @Override
-    public void systemAccounts(@NonNull final Map<Account, CryptoCreateTransactionBody.Builder> accounts) {
+    public void systemAccounts(@NonNull final SortedSet<Account> accounts) {
         // Intentional no-op
     }
 
     @Override
-    public void stakingAccounts(@NonNull final Map<Account, CryptoCreateTransactionBody.Builder> accounts) {
+    public void stakingAccounts(@NonNull final SortedSet<Account> accounts) {
         // Intentional no-op
     }
 
     @Override
-    public void miscAccounts(@NonNull final Map<Account, CryptoCreateTransactionBody.Builder> accounts) {
+    public void miscAccounts(@NonNull final SortedSet<Account> accounts) {
         // Intentional no-op
     }
 
     @Override
-    public void treasuryClones(@NonNull final Map<Account, CryptoCreateTransactionBody.Builder> accounts) {
+    public void treasuryClones(@NonNull final SortedSet<Account> accounts) {
         // Intentional no-op
     }
 
     @Override
-    public void blocklistAccounts(@NonNull Map<Account, CryptoCreateTransactionBody.Builder> accounts) {
+    public void blocklistAccounts(@NonNull SortedSet<Account> accounts) {
         // Intentional no-op
     }
 }

hedera-node/hedera-app/build.gradle.kts

@@ -141,6 +141,32 @@ tasks.jar {
     }
 }
 
+// Define build directory
+val buildDir = layout.projectDirectory.dir("./build/node")
+
+// Copy everything from hedera-node/data
+val copyNodeData =
+    tasks.register<Sync>("copyNodeData") {
+        dependsOn(copyLib)
+        dependsOn(copyApp)
+        from(layout.projectDirectory.dir("../data/keys")) { into("data/keys") }
+        from(layout.projectDirectory.dir("../data"))
+        into(buildDir)
+        exclude("config", "keys") // Exclude config directory
+        shouldRunAfter(tasks.named("copyApp"))
+        shouldRunAfter(tasks.named("copyLib"))
+    }
+
+//// Copy hedera-node/configuration/dev as hedera-node/hedera-app/build/node/data/config  }
+val copyConfig =
+    tasks.register<Copy>("copyConfig") {
+        from(layout.projectDirectory.dir("../configuration/dev")) { into("data/config") }
+        from(layout.projectDirectory.file("../config.txt"))
+        from(layout.projectDirectory.file("../log4j2.xml"))
+        into(buildDir)
+        shouldRunAfter(tasks.named("copyNodeData"))
+    }
+
 // Copy dependencies into `data/lib`
 val copyLib =
     tasks.register<Sync>("copyLib") {
@@ -160,22 +186,24 @@ val copyApp =
 tasks.assemble {
     dependsOn(copyLib)
     dependsOn(copyApp)
+    dependsOn(copyNodeData)
+    dependsOn(copyConfig)
 }
 
 // Create the "run" task for running a Hedera consensus node
 tasks.register<JavaExec>("run") {
     group = "application"
     dependsOn(tasks.assemble)
-    workingDir = layout.projectDirectory.dir("..").asFile
-    jvmArgs = listOf("-cp", "data/lib/*")
+    workingDir = layout.projectDirectory.dir("./build/node").asFile
+    jvmArgs = listOf("-cp", "lib/*")
     mainClass.set("com.swirlds.platform.Browser")
 }
 
 tasks.register<JavaExec>("modrun") {
     group = "application"
     dependsOn(tasks.assemble)
-    workingDir = layout.projectDirectory.dir("..").asFile
-    jvmArgs = listOf("-cp", "data/lib/*:data/apps/*", "-Dhedera.workflows.enabled=true")
+    workingDir = layout.projectDirectory.dir("./build/node").asFile
+    jvmArgs = listOf("-cp", "lib/*:apps/*", "-Dhedera.workflows.enabled=true")
     mainClass.set("com.hedera.node.app.ServicesMain")
 }
 

hedera-node/hedera-app/src/main/java/com/hedera/node/app/Hedera.java

@@ -53,6 +53,7 @@
 import com.hedera.node.app.service.networkadmin.impl.NetworkServiceImpl;
 import com.hedera.node.app.service.schedule.impl.ScheduleServiceImpl;
 import com.hedera.node.app.service.token.impl.TokenServiceImpl;
+import com.hedera.node.app.service.token.impl.schemas.SyntheticRecordsGenerator;
 import com.hedera.node.app.service.util.impl.UtilServiceImpl;
 import com.hedera.node.app.services.ServicesRegistryImpl;
 import com.hedera.node.app.spi.HapiUtils;
@@ -187,6 +188,7 @@ public final class Hedera implements SwirldMain {
     private ThrottleAccumulator backendThrottle;
     private ThrottleAccumulator frontendThrottle;
     private CongestionMultipliers congestionMultipliers;
+    private final SyntheticRecordsGenerator recordsGenerator;
 
     /**
      * The application name from the platform's perspective. This is currently locked in at the old main class name and
@@ -242,7 +244,9 @@ public Hedera(@NonNull final ConstructableRegistry constructableRegistry) {
                 () -> HapiUtils.toString(version.getServicesVersion()),
                 () -> HapiUtils.toString(version.getHapiVersion()));
 
-        // Create a record builder for any genesis records that need to be created
+        // Create a records generator for any synthetic records that need to be CREATED
+        this.recordsGenerator = new SyntheticRecordsGenerator();
+        // Create a records builder for any genesis records that need to be RECORDED
         this.genesisRecordsBuilder = new GenesisRecordsConsensusHook();
 
         // Create all the service implementations
@@ -258,7 +262,12 @@ public Hedera(@NonNull final ConstructableRegistry constructableRegistry) {
                         new FreezeServiceImpl(),
                         new NetworkServiceImpl(),
                         new ScheduleServiceImpl(),
-                        new TokenServiceImpl(),
+                        new TokenServiceImpl(
+                                recordsGenerator::sysAcctRecords,
+                                recordsGenerator::stakingAcctRecords,
+                                recordsGenerator::treasuryAcctRecords,
+                                recordsGenerator::multiUseAcctRecords,
+                                recordsGenerator::blocklistAcctRecords),
                         new UtilServiceImpl(),
                         new RecordCacheService(),
                         new BlockRecordService(),
@@ -355,8 +364,34 @@ private void onStateInitialized(
             @NonNull final SwirldDualState dualState,
             @NonNull final InitTrigger trigger,
             @Nullable final SoftwareVersion previousVersion) {
+        // Initialize the configuration from disk. We must do this BEFORE we run migration, because the various
+        // migration methods may depend on configuration to do their work. For example, the token service migration code
+        // needs to know the token treasury account, which has an account ID specified in config. The initial config
+        // file in state, created by the file service migration, will match what we have here, so we don't have to worry
+        // about re-loading config after migration.
+        logger.info("Initializing configuration with trigger {}", trigger);
+        configProvider = new ConfigProviderImpl(trigger == GENESIS);
+        logConfiguration();
 
-        // We do nothing for EVENT_STREAM_RECOVERY. This is a special case that is handled by the platform.
+        // Determine if we need to create synthetic records for system entities
+        final var blockRecordState = state.createReadableStates(BlockRecordService.NAME);
+        boolean createSynthRecords = false;
+        if (!blockRecordState.isEmpty()) {
+            final var blockInfo = blockRecordState
+                    .<BlockInfo>getSingleton(BlockRecordService.BLOCK_INFO_STATE_KEY)
+                    .get();
+            if (blockInfo == null || blockInfo.consTimeOfLastHandledTxn() == null) {
+                createSynthRecords = true;
+            }
+        } else {
+            createSynthRecords = true;
+        }
+        if (createSynthRecords) {
+            recordsGenerator.createRecords(configProvider.getConfiguration(), genesisRecordsBuilder);
+        }
+
+        // We do nothing else for EVENT_STREAM_RECOVERY, which for now is broken. This is a special case that is handled
+        // by the platform, and we need to figure out how to make it work with the modular app.
         if (trigger == EVENT_STREAM_RECOVERY) {
             logger.debug("Skipping state initialization for trigger {}", trigger);
             return;
@@ -718,15 +753,6 @@ public void shutdownGrpcServer() {
     private void genesis(@NonNull final MerkleHederaState state) {
         logger.debug("Genesis Initialization");
 
-        // Initialize the configuration from disk (genesis case). We must do this BEFORE we run migration, because
-        // the various migration methods may depend on configuration to do their work. For example, the token service
-        // migration code needs to know the token treasury account, which has an account ID specified in config.
-        // The initial config file in state, created by the file service migration, will match what we have here,
-        // so we don't have to worry about re-loading config after migration.
-        logger.info("Initializing genesis configuration");
-        this.configProvider = new ConfigProviderImpl(true);
-        logConfiguration();
-
         logger.info("Initializing ThrottleManager");
         this.throttleManager = new ThrottleManager();
 

hedera-node/hedera-app/src/main/java/com/hedera/node/app/fees/FeeManager.java

@@ -179,9 +179,6 @@ public FeeCalculator createFeeCalculator(
         // If it is not known, that is, if we have no fee data for that transaction, then we MUST NOT execute that
         // transaction! We will not be able to charge appropriately for it.
         final var feeData = getFeeData(functionality, consensusTime, subType);
-        if (feeData == null) {
-            throw new IllegalStateException("No fee data found for transaction type " + functionality);
-        }
 
         // Create the fee calculator
         return new FeeCalculatorImpl(

hedera-node/hedera-app/src/main/java/com/hedera/node/app/workflows/SolvencyPreCheck.java

@@ -220,9 +220,6 @@ public long estimateAdditionalCosts(
     private long estimatedGasPriceInTinybars(
             @NonNull final HederaFunctionality functionality, @NonNull final Instant consensusTime) {
         final var feeData = feeManager.getFeeData(functionality, consensusTime, SubType.DEFAULT);
-        if (feeData == null) {
-            throw new IllegalStateException("No fee data found for transaction type " + functionality);
-        }
         final long priceInTinyCents = feeData.servicedataOrThrow().gas() / FEE_DIVISOR_FACTOR;
         final long priceInTinyBars = exchangeRateManager.getTinybarsFromTinyCents(priceInTinyCents, consensusTime);
         return Math.max(priceInTinyBars, 1L);